agentskeptic 0.1.1

package/schemas/workflow-truth-report.schema.json

@@ -0,0 +1,761 @@
+{
+  "$id": "https://agentskeptic.com/schemas/workflow-truth-report.schema.json",
+  "title": "WorkflowTruthReport",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "schemaVersion",
+    "workflowId",
+    "workflowStatus",
+    "trustSummary",
+    "runLevelIssues",
+    "eventSequence",
+    "steps",
+    "failureAnalysis",
+    "failureExplanation",
+    "correctnessDefinition",
+    "executionPathFindings",
+    "executionPathSummary"
+  ],
+  "properties": {
+    "schemaVersion": { "const": 9 },
+    "workflowId": { "type": "string" },
+    "workflowStatus": {
+      "type": "string",
+      "enum": ["complete", "incomplete", "inconsistent"]
+    },
+    "trustSummary": { "type": "string" },
+    "runLevelIssues": {
+      "type": "array",
+      "items": { "$ref": "#/$defs/truthIssue" }
+    },
+    "eventSequence": {
+      "oneOf": [
+        {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["kind"],
+          "properties": {
+            "kind": { "const": "normal" }
+          }
+        },
+        {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["kind", "issues"],
+          "properties": {
+            "kind": { "const": "irregular" },
+            "issues": {
+              "type": "array",
+              "minItems": 1,
+              "items": { "$ref": "#/$defs/truthIssue" }
+            }
+          }
+        }
+      ]
+    },
+    "steps": {
+      "type": "array",
+      "items": { "$ref": "#/$defs/truthStep" }
+    },
+    "failureAnalysis": {
+      "oneOf": [{ "type": "null" }, { "$ref": "#/$defs/failureAnalysis" }]
+    },
+    "failureExplanation": {
+      "oneOf": [{ "type": "null" }, { "$ref": "#/$defs/failureExplanationV1" }]
+    },
+    "correctnessDefinition": {
+      "oneOf": [{ "type": "null" }, { "$ref": "#/$defs/correctnessDefinitionV1" }]
+    },
+    "executionPathFindings": {
+      "type": "array",
+      "items": { "$ref": "#/$defs/executionPathFinding" }
+    },
+    "executionPathSummary": { "type": "string", "maxLength": 4096 }
+  },
+  "allOf": [
+    {
+      "if": {
+        "properties": { "workflowStatus": { "const": "complete" } },
+        "required": ["workflowStatus"]
+      },
+      "then": {
+        "properties": {
+          "failureExplanation": { "type": "null" },
+          "failureAnalysis": { "type": "null" },
+          "correctnessDefinition": { "type": "null" }
+        }
+      }
+    },
+    {
+      "if": {
+        "properties": {
+          "workflowStatus": { "enum": ["incomplete", "inconsistent"] }
+        },
+        "required": ["workflowStatus"]
+      },
+      "then": {
+        "properties": {
+          "failureExplanation": { "$ref": "#/$defs/failureExplanationV1" },
+          "failureAnalysis": { "$ref": "#/$defs/failureAnalysis" },
+          "correctnessDefinition": { "$ref": "#/$defs/correctnessDefinitionV1" }
+        }
+      }
+    }
+  ],
+  "$defs": {
+    "failureExplanationFactId": {
+      "type": "string",
+      "enum": [
+        "trust_summary",
+        "workflow_status",
+        "verification_policy",
+        "primary_origin",
+        "classification_confidence",
+        "failure_analysis_summary",
+        "primary_scope",
+        "primary_codes",
+        "primary_ingest_index",
+        "primary_tool_id",
+        "primary_source",
+        "primary_run_event_id",
+        "primary_seq",
+        "primary_effect_id",
+        "verify_target",
+        "intended_effect_narrative",
+        "evidence_summary_field",
+        "evidence_summary_expected",
+        "evidence_summary_actual",
+        "evidence_summary_row_count"
+      ]
+    },
+    "failureExplanationUnknownId": {
+      "type": "string",
+      "enum": ["unknown_reason_code", "classification_confidence_band", "competing_hypothesis"]
+    },
+    "failureExplanationFactRow": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["id", "value"],
+      "properties": {
+        "id": { "$ref": "#/$defs/failureExplanationFactId" },
+        "value": { "type": "string", "maxLength": 4096 }
+      }
+    },
+    "failureExplanationUnknownRow": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["id", "value"],
+      "properties": {
+        "id": { "$ref": "#/$defs/failureExplanationUnknownId" },
+        "value": { "type": "string", "maxLength": 4096 }
+      }
+    },
+    "failureExplanationV1": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["schemaVersion", "expected", "observed", "divergence", "knownFacts", "unknowns"],
+      "properties": {
+        "schemaVersion": { "const": 1 },
+        "expected": { "type": "string", "maxLength": 2048 },
+        "observed": { "type": "string", "maxLength": 2048 },
+        "divergence": { "type": "string", "maxLength": 2048 },
+        "knownFacts": {
+          "type": "array",
+          "minItems": 1,
+          "items": { "$ref": "#/$defs/failureExplanationFactRow" }
+        },
+        "unknowns": {
+          "type": "array",
+          "items": { "$ref": "#/$defs/failureExplanationUnknownRow" }
+        }
+      }
+    },
+    "pathConcernCategory": {
+      "type": "string",
+      "enum": [
+        "context_quality",
+        "decision_execution",
+        "tool_selection_execution",
+        "action_inputs_invalid",
+        "workflow_completeness",
+        "capture_integrity"
+      ]
+    },
+    "pathFindingSeverity": {
+      "type": "string",
+      "enum": ["high", "medium", "low"]
+    },
+    "executionPathEvidence": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["scope"],
+      "properties": {
+        "scope": {
+          "type": "string",
+          "enum": ["run_context", "run_level", "event_sequence", "step"]
+        },
+        "codes": {
+          "type": "array",
+          "items": { "type": "string" }
+        },
+        "ingestIndex": { "type": "integer", "minimum": 0 },
+        "seq": { "type": "integer", "minimum": 0 },
+        "toolId": { "type": "string" },
+        "source": { "type": "string" },
+        "runEventId": { "type": ["string", "null"] }
+      }
+    },
+    "executionPathFinding": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["code", "severity", "concernCategory", "message", "evidence"],
+      "properties": {
+        "code": { "type": "string", "minLength": 1 },
+        "severity": { "$ref": "#/$defs/pathFindingSeverity" },
+        "concernCategory": { "$ref": "#/$defs/pathConcernCategory" },
+        "message": { "type": "string", "maxLength": 2048 },
+        "evidence": { "$ref": "#/$defs/executionPathEvidence" }
+      }
+    },
+    "failureOrigin": {
+      "type": "string",
+      "enum": [
+        "decision_making",
+        "inputs",
+        "retrieval",
+        "tool_use",
+        "workflow_flow",
+        "downstream_system_state"
+      ]
+    },
+    "failureConfidence": {
+      "type": "string",
+      "enum": ["high", "medium", "low"]
+    },
+    "failureAnalysisEvidence": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["scope"],
+      "properties": {
+        "scope": {
+          "type": "string",
+          "enum": ["run_context", "run_level", "event_sequence", "step", "effect"]
+        },
+        "codes": {
+          "type": "array",
+          "items": { "type": "string" }
+        },
+        "ingestIndex": { "type": "integer", "minimum": 0 },
+        "seq": { "type": "integer", "minimum": 0 },
+        "toolId": { "type": "string" },
+        "effectId": { "type": "string" },
+        "source": { "type": "string" },
+        "runEventId": { "type": ["string", "null"] }
+      }
+    },
+    "failureAnalysisAlternative": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["primaryOrigin", "rationale"],
+      "properties": {
+        "primaryOrigin": { "$ref": "#/$defs/failureOrigin" },
+        "rationale": { "type": "string", "maxLength": 2048 }
+      }
+    },
+    "actionableFailureCategory": {
+      "type": "string",
+      "enum": [
+        "decision_error",
+        "bad_input",
+        "retrieval_failure",
+        "control_flow_problem",
+        "state_inconsistency",
+        "downstream_execution_failure",
+        "ambiguous",
+        "unclassified"
+      ]
+    },
+    "actionableFailureSeverity": {
+      "type": "string",
+      "enum": ["high", "medium", "low"]
+    },
+    "recommendedAction": {
+      "type": "string",
+      "enum": [
+        "none",
+        "manual_review",
+        "deduplicate",
+        "reconcile_downstream_state",
+        "correct_verification_inputs",
+        "improve_read_connectivity",
+        "resolve_multi_effect_failures",
+        "align_tool_observations",
+        "fix_event_ingest_and_steps",
+        "fix_event_sequence_order",
+        "fix_run_context_controls",
+        "fix_cli_usage",
+        "fix_registry_events_or_compare_files",
+        "fix_verification_database_connection",
+        "fix_saved_workflow_json",
+        "fix_compare_workflow_inputs",
+        "fix_execution_trace_structure",
+        "fix_verification_policy_and_hook",
+        "fix_plan_document_and_patterns",
+        "fix_plan_transition_cli_and_refs",
+        "upgrade_git_or_retry_git"
+      ]
+    },
+    "actionableFailure": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["category", "severity", "recommendedAction", "automationSafe"],
+      "properties": {
+        "category": { "$ref": "#/$defs/actionableFailureCategory" },
+        "severity": { "$ref": "#/$defs/actionableFailureSeverity" },
+        "recommendedAction": { "$ref": "#/$defs/recommendedAction" },
+        "automationSafe": { "type": "boolean" }
+      }
+    },
+    "failureAnalysis": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "summary",
+        "primaryOrigin",
+        "confidence",
+        "unknownReasonCodes",
+        "evidence",
+        "actionableFailure"
+      ],
+      "properties": {
+        "summary": { "type": "string", "maxLength": 2048 },
+        "primaryOrigin": { "$ref": "#/$defs/failureOrigin" },
+        "confidence": { "$ref": "#/$defs/failureConfidence" },
+        "unknownReasonCodes": {
+          "type": "array",
+          "items": { "type": "string" }
+        },
+        "evidence": {
+          "type": "array",
+          "minItems": 1,
+          "items": { "$ref": "#/$defs/failureAnalysisEvidence" }
+        },
+        "alternativeHypotheses": {
+          "type": "array",
+          "items": { "$ref": "#/$defs/failureAnalysisAlternative" }
+        },
+        "actionableFailure": { "$ref": "#/$defs/actionableFailure" }
+      }
+    },
+    "truthIssue": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["code", "message", "category"],
+      "properties": {
+        "code": { "type": "string" },
+        "message": { "type": "string" },
+        "category": {
+          "type": "string",
+          "enum": [
+            "workflow_execution",
+            "verification_setup",
+            "observation_uncertainty"
+          ]
+        }
+      }
+    },
+    "reason": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["code", "message"],
+      "properties": {
+        "code": { "type": "string" },
+        "message": { "type": "string" },
+        "field": { "type": "string" }
+      }
+    },
+    "truthEffect": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["id", "outcomeLabel", "reasons"],
+      "properties": {
+        "id": { "type": "string" },
+        "outcomeLabel": {
+          "type": "string",
+          "enum": [
+            "VERIFIED",
+            "FAILED_ROW_MISSING",
+            "FAILED_VALUE_MISMATCH",
+            "INCOMPLETE_CANNOT_VERIFY"
+          ]
+        },
+        "reasons": {
+          "type": "array",
+          "items": { "$ref": "#/$defs/reason" }
+        }
+      }
+    },
+    "truthStep": {
+      "allOf": [
+        {
+          "type": "object",
+          "additionalProperties": false,
+          "required": [
+            "seq",
+            "toolId",
+            "outcomeLabel",
+            "observations",
+            "reasons",
+            "intendedEffect",
+            "observedExecution",
+            "verifyTarget",
+            "observedStateSummary"
+          ],
+          "properties": {
+            "seq": { "type": "integer", "minimum": 0 },
+            "toolId": { "type": "string" },
+            "outcomeLabel": {
+              "type": "string",
+              "enum": [
+                "VERIFIED",
+                "FAILED_ROW_MISSING",
+                "FAILED_VALUE_MISMATCH",
+                "INCOMPLETE_CANNOT_VERIFY",
+                "PARTIALLY_VERIFIED",
+                "UNCERTAIN_NOT_OBSERVED_WITHIN_WINDOW"
+              ]
+            },
+            "observations": {
+              "type": "object",
+              "additionalProperties": false,
+              "required": ["evaluatedOrdinal", "repeatCount"],
+              "properties": {
+                "evaluatedOrdinal": { "type": "integer", "minimum": 1 },
+                "repeatCount": { "type": "integer", "minimum": 1 }
+              }
+            },
+            "reasons": {
+              "type": "array",
+              "items": { "$ref": "#/$defs/reason" }
+            },
+            "intendedEffect": {
+              "type": "object",
+              "additionalProperties": false,
+              "required": ["narrative"],
+              "properties": {
+                "narrative": { "type": "string" }
+              }
+            },
+            "observedExecution": {
+              "type": "object",
+              "additionalProperties": false,
+              "required": ["paramsCanonical"],
+              "properties": {
+                "paramsCanonical": { "type": "string" }
+              }
+            },
+            "failureCategory": {
+              "type": "string",
+              "enum": [
+                "workflow_execution",
+                "verification_setup",
+                "observation_uncertainty"
+              ]
+            },
+            "verifyTarget": {
+              "type": ["string", "null"]
+            },
+            "observedStateSummary": { "type": "string" },
+            "effects": {
+              "type": "array",
+              "minItems": 1,
+              "items": { "$ref": "#/$defs/truthEffect" }
+            }
+          }
+        },
+        {
+          "if": {
+            "type": "object",
+            "required": ["outcomeLabel"],
+            "properties": { "outcomeLabel": { "const": "VERIFIED" } }
+          },
+          "then": {
+            "type": "object",
+            "properties": { "failureCategory": false }
+          },
+          "else": {
+            "type": "object",
+            "required": ["failureCategory"]
+          }
+        }
+      ]
+    },
+    "correctnessEnforcementKind": {
+      "type": "string",
+      "enum": [
+        "run_ingest_integrity",
+        "event_capture_integrity",
+        "run_context_fairness",
+        "step_sql_expectation",
+        "plan_transition_expectation",
+        "quick_inferred_sql_row",
+        "quick_inferred_relational",
+        "quick_mapping_gap"
+      ]
+    },
+    "correctnessRequiredUpstreamContract": {
+      "type": "string",
+      "enum": [
+        "retrieval_ok_before_observation",
+        "model_turn_completed_before_observation",
+        "no_interrupt_before_observation",
+        "branch_not_skipped_before_observation",
+        "gate_not_skipped_before_observation",
+        "tool_not_skipped_before_observation"
+      ]
+    },
+    "correctnessIngestContractRequirement": {
+      "type": "string",
+      "enum": ["no_run_level_failures", "non_empty_tool_observed_steps"]
+    },
+    "correctnessRemediationAlignment": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["recommendedAction", "automationSafe"],
+      "properties": {
+        "recommendedAction": { "$ref": "#/$defs/recommendedAction" },
+        "automationSafe": { "type": "boolean" }
+      }
+    },
+    "correctnessEnforceableProjection": {
+      "oneOf": [
+        {
+          "type": "object",
+          "additionalProperties": false,
+          "required": [
+            "projectionKind",
+            "workflowId",
+            "verificationPolicyFragment",
+            "primaryFailureCodes",
+            "ingestContractRequirement"
+          ],
+          "properties": {
+            "projectionKind": { "const": "run_ingest_integrity" },
+            "workflowId": { "type": "string" },
+            "verificationPolicyFragment": { "type": "string" },
+            "primaryFailureCodes": {
+              "type": "array",
+              "items": { "type": "string" }
+            },
+            "ingestContractRequirement": { "$ref": "#/$defs/correctnessIngestContractRequirement" }
+          }
+        },
+        {
+          "type": "object",
+          "additionalProperties": false,
+          "required": [
+            "projectionKind",
+            "workflowId",
+            "verificationPolicyFragment",
+            "forbiddenEventSequenceCodes"
+          ],
+          "properties": {
+            "projectionKind": { "const": "event_capture_integrity" },
+            "workflowId": { "type": "string" },
+            "verificationPolicyFragment": { "type": "string" },
+            "forbiddenEventSequenceCodes": {
+              "type": "array",
+              "items": { "type": "string" }
+            }
+          }
+        },
+        {
+          "type": "object",
+          "additionalProperties": false,
+          "required": [
+            "projectionKind",
+            "workflowId",
+            "verificationPolicyFragment",
+            "ingestIndex",
+            "requiredUpstreamContract",
+            "primaryRunContextCodes"
+          ],
+          "properties": {
+            "projectionKind": { "const": "run_context_fairness" },
+            "workflowId": { "type": "string" },
+            "verificationPolicyFragment": { "type": "string" },
+            "ingestIndex": { "type": "integer", "minimum": 0 },
+            "requiredUpstreamContract": { "$ref": "#/$defs/correctnessRequiredUpstreamContract" },
+            "primaryRunContextCodes": {
+              "type": "array",
+              "items": { "type": "string" }
+            }
+          }
+        },
+        {
+          "type": "object",
+          "additionalProperties": false,
+          "required": [
+            "projectionKind",
+            "workflowId",
+            "verificationPolicyFragment",
+            "seq",
+            "toolId",
+            "verificationRequest"
+          ],
+          "properties": {
+            "projectionKind": { "const": "step_sql_expectation" },
+            "workflowId": { "type": "string" },
+            "verificationPolicyFragment": { "type": "string" },
+            "seq": { "type": "integer", "minimum": 0 },
+            "toolId": { "type": "string" },
+            "verificationRequest": {
+              "oneOf": [
+                { "type": "null" },
+                {
+                  "$ref": "https://agentskeptic.com/schemas/workflow-engine-result.schema.json#/$defs/verificationRequestSqlRow"
+                },
+                {
+                  "$ref": "https://agentskeptic.com/schemas/workflow-engine-result.schema.json#/$defs/verificationRequestSqlRowAbsent"
+                },
+                {
+                  "$ref": "https://agentskeptic.com/schemas/workflow-engine-result.schema.json#/$defs/verificationRequestSqlEffects"
+                },
+                {
+                  "$ref": "https://agentskeptic.com/schemas/workflow-engine-result.schema.json#/$defs/verificationRequestSqlRelational"
+                }
+              ]
+            }
+          }
+        },
+        {
+          "type": "object",
+          "additionalProperties": false,
+          "required": [
+            "projectionKind",
+            "workflowId",
+            "verificationPolicyFragment",
+            "seq",
+            "toolId",
+            "verifyTarget",
+            "primaryCodes"
+          ],
+          "properties": {
+            "projectionKind": { "const": "plan_transition_expectation" },
+            "workflowId": { "type": "string" },
+            "verificationPolicyFragment": { "type": "string" },
+            "seq": { "type": "integer", "minimum": 0 },
+            "toolId": { "type": "string" },
+            "verifyTarget": { "type": ["string", "null"] },
+            "primaryCodes": {
+              "type": "array",
+              "items": { "type": "string" }
+            }
+          }
+        },
+        {
+          "type": "object",
+          "additionalProperties": false,
+          "required": [
+            "projectionKind",
+            "quickProvisional",
+            "unitId",
+            "toolName",
+            "actionIndex",
+            "table",
+            "sqlRowRequest"
+          ],
+          "properties": {
+            "projectionKind": { "const": "quick_inferred_sql_row" },
+            "quickProvisional": { "const": true },
+            "unitId": { "type": "string", "minLength": 1 },
+            "toolName": { "type": "string" },
+            "actionIndex": { "type": "integer", "minimum": 0 },
+            "table": { "type": "string" },
+            "sqlRowRequest": {
+              "$ref": "https://agentskeptic.com/schemas/workflow-engine-result.schema.json#/$defs/verificationRequestSqlRow"
+            }
+          }
+        },
+        {
+          "type": "object",
+          "additionalProperties": false,
+          "required": [
+            "projectionKind",
+            "quickProvisional",
+            "unitId",
+            "toolName",
+            "actionIndex",
+            "childTable",
+            "checkId",
+            "matchEq"
+          ],
+          "properties": {
+            "projectionKind": { "const": "quick_inferred_relational" },
+            "quickProvisional": { "const": true },
+            "unitId": { "type": "string", "minLength": 1 },
+            "toolName": { "type": "string" },
+            "actionIndex": { "type": "integer", "minimum": 0 },
+            "childTable": { "type": "string" },
+            "checkId": { "type": "string" },
+            "matchEq": {
+              "type": "array",
+              "minItems": 1,
+              "items": {
+                "$ref": "https://agentskeptic.com/schemas/workflow-engine-result.schema.json#/$defs/identityEqPair"
+              }
+            }
+          }
+        },
+        {
+          "type": "object",
+          "additionalProperties": false,
+          "required": [
+            "projectionKind",
+            "quickProvisional",
+            "toolName",
+            "actionIndex",
+            "reasonCodes",
+            "table"
+          ],
+          "properties": {
+            "projectionKind": { "const": "quick_mapping_gap" },
+            "quickProvisional": { "const": true },
+            "toolName": { "type": "string" },
+            "actionIndex": { "type": "integer", "minimum": 0 },
+            "reasonCodes": {
+              "type": "array",
+              "items": { "type": "string" }
+            },
+            "table": { "type": ["string", "null"] }
+          }
+        }
+      ]
+    },
+    "correctnessDefinitionV1": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "schemaVersion",
+        "enforcementKind",
+        "mustAlwaysHold",
+        "enforceAs",
+        "enforceableProjection",
+        "remediationAlignment"
+      ],
+      "properties": {
+        "schemaVersion": { "const": 1 },
+        "enforcementKind": { "$ref": "#/$defs/correctnessEnforcementKind" },
+        "mustAlwaysHold": { "type": "string", "maxLength": 2048 },
+        "enforceAs": {
+          "type": "array",
+          "minItems": 2,
+          "items": { "type": "string", "maxLength": 2048 }
+        },
+        "enforceableProjection": { "$ref": "#/$defs/correctnessEnforceableProjection" },
+        "remediationAlignment": { "$ref": "#/$defs/correctnessRemediationAlignment" }
+      }
+    }
+  }
+}