agentskeptic 0.2.1 → 1.0.1

package/schemas/outcome-certificate-v1.schema.json

@@ -0,0 +1,89 @@
+{
+  "$id": "https://agentskeptic.com/schemas/outcome-certificate-v1.schema.json",
+  "title": "OutcomeCertificateV1",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "schemaVersion",
+    "workflowId",
+    "runKind",
+    "stateRelation",
+    "highStakesReliance",
+    "relianceRationale",
+    "intentSummary",
+    "explanation",
+    "steps",
+    "humanReport"
+  ],
+  "properties": {
+    "schemaVersion": { "type": "integer", "const": 1 },
+    "workflowId": { "type": "string", "minLength": 1, "maxLength": 512 },
+    "runKind": {
+      "type": "string",
+      "enum": ["contract_sql", "contract_sql_langgraph_checkpoint_trust", "quick_preview"]
+    },
+    "stateRelation": {
+      "type": "string",
+      "enum": ["matches_expectations", "does_not_match", "not_established"]
+    },
+    "highStakesReliance": { "type": "string", "enum": ["permitted", "prohibited"] },
+    "relianceRationale": { "type": "string", "minLength": 1, "maxLength": 8192 },
+    "intentSummary": { "type": "string", "minLength": 1, "maxLength": 8192 },
+    "explanation": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["headline", "details"],
+      "properties": {
+        "headline": { "type": "string", "minLength": 1, "maxLength": 2048 },
+        "details": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["code", "message"],
+            "properties": {
+              "code": { "type": "string", "minLength": 1, "maxLength": 256 },
+              "message": { "type": "string", "minLength": 1, "maxLength": 4096 }
+            }
+          }
+        }
+      }
+    },
+    "steps": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": ["seq", "declaredAction", "expectedOutcome", "observedOutcome"],
+        "properties": {
+          "seq": { "type": "integer", "minimum": 0 },
+          "toolId": { "type": "string", "maxLength": 512 },
+          "declaredAction": { "type": "string", "minLength": 1, "maxLength": 4096 },
+          "expectedOutcome": { "type": "string", "minLength": 1, "maxLength": 4096 },
+          "observedOutcome": { "type": "string", "minLength": 1, "maxLength": 8192 }
+        }
+      }
+    },
+    "humanReport": { "type": "string", "minLength": 1, "maxLength": 1048576 },
+    "checkpointVerdicts": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": ["checkpointKey", "verdict", "seqs", "productionMeaning"],
+        "properties": {
+          "checkpointKey": { "type": "string", "minLength": 1, "maxLength": 2048 },
+          "verdict": {
+            "type": "string",
+            "enum": ["verified", "inconsistent", "incomplete"]
+          },
+          "seqs": {
+            "type": "array",
+            "items": { "type": "integer", "minimum": 0 }
+          },
+          "productionMeaning": { "type": "string", "minLength": 1, "maxLength": 8192 }
+        }
+      }
+    }
+  }
+}

package/schemas/public-verification-report-v2.schema.json

@@ -0,0 +1,15 @@
+{
+  "$id": "https://agentskeptic.com/schemas/public-verification-report-v2.schema.json",
+  "title": "PublicVerificationReportEnvelopeV2",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["schemaVersion", "certificate"],
+  "properties": {
+    "schemaVersion": { "type": "integer", "const": 2 },
+    "certificate": {
+      "$ref": "https://agentskeptic.com/schemas/outcome-certificate-v1.schema.json"
+    },
+    "cliVersion": { "type": "string", "maxLength": 128 },
+    "createdFrom": { "type": "string", "maxLength": 256 }
+  }
+}

package/schemas/quick-verify-report.schema.json

@@ -30,7 +30,7 @@
         "groundTruth"
       ],
       "properties": {
-        "quickVerifyVersion": { "type": "string", "const": "1.2.0" },
+        "quickVerifyVersion": { "type": "string", "const": "1.2.1" },
         "capabilities": {
           "type": "array",
           "const": ["inferred_row", "inferred_related_exists"]

package/schemas/regression-artifact-v1.schema.json

@@ -0,0 +1,212 @@
+{
+  "$id": "https://agentskeptic.com/schemas/regression-artifact-v1.schema.json",
+  "title": "RegressionArtifactV1",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "schemaVersion",
+    "artifactSource",
+    "manifestSha256",
+    "workflowId",
+    "certificateProfile",
+    "verification",
+    "outcomeCertificates",
+    "tracePairwise",
+    "narrative",
+    "humanText",
+    "narrativeHtml"
+  ],
+  "properties": {
+    "schemaVersion": { "const": 1 },
+    "artifactSource": { "type": "string", "enum": ["cli_manifest", "debug_corpus"] },
+    "manifestSha256": { "type": "string", "pattern": "^[a-f0-9]{64}$" },
+    "workflowId": { "type": "string", "minLength": 1 },
+    "certificateProfile": {
+      "oneOf": [
+        {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["mode", "outcomeCertificateRunKind"],
+          "properties": {
+            "mode": { "const": "uniform" },
+            "outcomeCertificateRunKind": {
+              "type": "string",
+              "enum": ["contract_sql", "contract_sql_langgraph_checkpoint_trust"]
+            }
+          }
+        },
+        {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["mode", "entries"],
+          "properties": {
+            "mode": { "const": "perRun" },
+            "entries": {
+              "type": "array",
+              "minItems": 1,
+              "items": {
+                "type": "object",
+                "additionalProperties": false,
+                "required": ["runIndex", "outcomeCertificateRunKind"],
+                "properties": {
+                  "runIndex": { "type": "integer", "minimum": 0 },
+                  "outcomeCertificateRunKind": {
+                    "type": "string",
+                    "enum": ["contract_sql", "contract_sql_langgraph_checkpoint_trust"]
+                  }
+                }
+              }
+            }
+          }
+        }
+      ]
+    },
+    "verification": { "$ref": "https://agentskeptic.com/schemas/run-comparison-report.schema.json" },
+    "outcomeCertificates": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": [
+          "runIndex",
+          "displayLabel",
+          "outcomeCertificateRunKind",
+          "certificateCanonicalDigest",
+          "certificate"
+        ],
+        "properties": {
+          "runIndex": { "type": "integer", "minimum": 0 },
+          "displayLabel": { "type": "string", "minLength": 1 },
+          "outcomeCertificateRunKind": {
+            "type": "string",
+            "enum": ["contract_sql", "contract_sql_langgraph_checkpoint_trust"]
+          },
+          "certificateCanonicalDigest": { "type": "string", "pattern": "^[a-f0-9]{64}$" },
+          "certificate": { "$ref": "https://agentskeptic.com/schemas/outcome-certificate-v1.schema.json" }
+        }
+      }
+    },
+    "tracePairwise": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "priorRunIndex",
+        "currentRunIndex",
+        "viewBuild",
+        "bySeq",
+        "seqTimelineOrderDiverged",
+        "nonToolEventCounts"
+      ],
+      "properties": {
+        "priorRunIndex": { "type": "integer", "minimum": 0 },
+        "currentRunIndex": { "type": "integer", "minimum": 0 },
+        "viewBuild": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["prior", "current"],
+          "properties": {
+            "prior": { "$ref": "https://agentskeptic.com/schemas/execution-trace-view.schema.json" },
+            "current": { "$ref": "https://agentskeptic.com/schemas/execution-trace-view.schema.json" }
+          }
+        },
+        "bySeq": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["seq", "class", "prior", "current"],
+            "properties": {
+              "seq": { "type": "integer", "minimum": 0 },
+              "class": { "type": "string", "enum": ["only_in_prior", "only_in_current", "in_both"] },
+              "prior": { "oneOf": [{ "type": "null" }, { "$ref": "#/$defs/evaluatedNodeSummary" }] },
+              "current": { "oneOf": [{ "type": "null" }, { "$ref": "#/$defs/evaluatedNodeSummary" }] }
+            }
+          }
+        },
+        "seqTimelineOrderDiverged": { "type": "boolean" },
+        "nonToolEventCounts": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["prior", "current"],
+          "properties": {
+            "prior": { "type": "object", "additionalProperties": { "type": "integer", "minimum": 0 } },
+            "current": { "type": "object", "additionalProperties": { "type": "integer", "minimum": 0 } }
+          }
+        }
+      }
+    },
+    "narrative": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "classification",
+        "whyItMatters",
+        "headline",
+        "structural",
+        "traceSummary"
+      ],
+      "properties": {
+        "classification": {
+          "type": "string",
+          "enum": ["improving", "worsening", "unchanged", "mixed"]
+        },
+        "whyItMatters": { "type": "string" },
+        "headline": { "type": "string" },
+        "structural": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": [
+            "introducedLogicalStepKeysCount",
+            "resolvedLogicalStepKeysCount",
+            "recurringSignatureCount"
+          ],
+          "properties": {
+            "introducedLogicalStepKeysCount": { "type": "integer", "minimum": 0 },
+            "resolvedLogicalStepKeysCount": { "type": "integer", "minimum": 0 },
+            "recurringSignatureCount": { "type": "integer", "minimum": 0 }
+          }
+        },
+        "traceSummary": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": [
+            "seqTimelineOrderDiverged",
+            "onlyInPriorSeqCount",
+            "onlyInCurrentSeqCount",
+            "inBothCount",
+            "nonToolCountsDiverged"
+          ],
+          "properties": {
+            "seqTimelineOrderDiverged": { "type": "boolean" },
+            "onlyInPriorSeqCount": { "type": "integer", "minimum": 0 },
+            "onlyInCurrentSeqCount": { "type": "integer", "minimum": 0 },
+            "inBothCount": { "type": "integer", "minimum": 0 },
+            "nonToolCountsDiverged": { "type": "boolean" }
+          }
+        }
+      }
+    },
+    "humanText": { "type": "string" },
+    "narrativeHtml": { "type": "string", "minLength": 1 }
+  },
+  "$defs": {
+    "evaluatedNodeSummary": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "ingestIndex",
+        "traceStepKind",
+        "engineStepStatus",
+        "truthOutcomeLabel",
+        "toolId"
+      ],
+      "properties": {
+        "ingestIndex": { "type": "integer", "minimum": 0 },
+        "traceStepKind": { "type": "string" },
+        "engineStepStatus": { "oneOf": [{ "type": "string" }, { "type": "null" }] },
+        "truthOutcomeLabel": { "oneOf": [{ "type": "string" }, { "type": "null" }] },
+        "toolId": { "oneOf": [{ "type": "string" }, { "type": "null" }] }
+      }
+    }
+  }
+}

package/schemas/tools-registry.schema.json

@@ -72,12 +72,115 @@
                 "items": { "$ref": "#/$defs/sqlRelationalCheck" }
               }
             }
+          },
+          {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["kind", "provider", "documentId", "indexName"],
+            "properties": {
+              "kind": { "const": "vector_document" },
+              "provider": { "type": "string", "enum": ["pinecone", "weaviate", "chroma"] },
+              "documentId": { "$ref": "#/$defs/stringSpec" },
+              "indexName": { "$ref": "#/$defs/stringSpec" },
+              "namespace": { "$ref": "#/$defs/stringSpec" },
+              "host": { "$ref": "#/$defs/stringSpec" },
+              "metadataEq": { "$ref": "#/$defs/requiredFieldsPointer" },
+              "expectPayloadSha256": { "$ref": "#/$defs/stringSpec" }
+            }
+          },
+          {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["kind", "bucket", "key"],
+            "properties": {
+              "kind": { "const": "object_storage_object" },
+              "bucket": { "$ref": "#/$defs/stringSpec" },
+              "key": { "$ref": "#/$defs/stringSpec" },
+              "endpoint": { "$ref": "#/$defs/stringSpec" },
+              "expectSizeBytes": { "$ref": "#/$defs/numberOrPointerSpec" },
+              "expectSha256": { "$ref": "#/$defs/stringSpec" },
+              "expectEtag": { "$ref": "#/$defs/stringSpec" }
+            }
+          },
+          {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["kind", "url"],
+            "properties": {
+              "kind": { "const": "http_witness" },
+              "method": { "type": "string", "enum": ["GET", "POST"] },
+              "url": { "$ref": "#/$defs/stringSpec" },
+              "expectedStatus": { "$ref": "#/$defs/numberOrPointerSpec" },
+              "assertions": {
+                "type": "array",
+                "items": { "$ref": "#/$defs/httpWitnessAssertion" }
+              }
+            }
+          },
+          {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["kind", "collection", "filterPointer", "requiredFields"],
+            "properties": {
+              "kind": { "const": "mongo_document" },
+              "collection": { "$ref": "#/$defs/stringSpec" },
+              "filterPointer": {
+                "type": "object",
+                "additionalProperties": false,
+                "required": ["pointer"],
+                "properties": {
+                  "pointer": { "type": "string", "pattern": "^/" }
+                }
+              },
+              "requiredFields": { "$ref": "#/$defs/requiredFieldsPointer" }
+            }
           }
         ]
       }
     }
   },
   "$defs": {
+    "stringSpec": {
+      "oneOf": [
+        {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["const"],
+          "properties": { "const": { "type": "string", "minLength": 1 } }
+        },
+        {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["pointer"],
+          "properties": { "pointer": { "type": "string", "pattern": "^/" } }
+        }
+      ]
+    },
+    "numberOrPointerSpec": {
+      "oneOf": [
+        {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["const"],
+          "properties": { "const": { "type": "number" } }
+        },
+        {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["pointer"],
+          "properties": { "pointer": { "type": "string", "pattern": "^/" } }
+        }
+      ]
+    },
+    "httpWitnessAssertion": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["jsonPointer", "value"],
+      "properties": {
+        "jsonPointer": { "type": "string", "minLength": 1 },
+        "value": { "$ref": "#/$defs/scalarOrPointer" }
+      }
+    },
     "tableSpec": {
       "oneOf": [
         {

package/schemas/workflow-engine-result.schema.json

@@ -81,7 +81,7 @@
         "lastRunEvent"
       ],
       "properties": {
-        "maxWireSchemaVersion": { "type": "integer", "enum": [1, 2] },
+        "maxWireSchemaVersion": { "type": "integer", "enum": [1, 2, 3] },
         "retrievalEvents": {
           "type": "array",
           "items": {
@@ -414,6 +414,70 @@
         }
       }
     },
+    "verificationRequestVectorDocument": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["kind", "provider", "documentId", "indexName"],
+      "properties": {
+        "kind": { "const": "vector_document" },
+        "provider": { "type": "string", "enum": ["pinecone", "weaviate", "chroma"] },
+        "documentId": { "type": "string", "minLength": 1 },
+        "indexName": { "type": "string", "minLength": 1 },
+        "namespace": { "type": "string" },
+        "host": { "type": "string", "minLength": 1 },
+        "metadataSubset": { "type": "object", "additionalProperties": true },
+        "expectPayloadSha256": { "type": "string", "minLength": 1 }
+      }
+    },
+    "verificationRequestObjectStorageObject": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["kind", "bucket", "key"],
+      "properties": {
+        "kind": { "const": "object_storage_object" },
+        "bucket": { "type": "string", "minLength": 1 },
+        "key": { "type": "string", "minLength": 1 },
+        "endpoint": { "type": "string", "minLength": 1 },
+        "expectSizeBytes": { "type": "number", "minimum": 0 },
+        "expectSha256": { "type": "string", "minLength": 1 },
+        "expectEtag": { "type": "string", "minLength": 1 }
+      }
+    },
+    "httpWitnessResolvedAssertion": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["jsonPointer", "value"],
+      "properties": {
+        "jsonPointer": { "type": "string", "minLength": 1 },
+        "value": { "type": ["string", "number", "boolean", "null"] }
+      }
+    },
+    "verificationRequestHttpWitness": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["kind", "method", "url"],
+      "properties": {
+        "kind": { "const": "http_witness" },
+        "method": { "type": "string", "enum": ["GET", "POST"] },
+        "url": { "type": "string", "minLength": 1 },
+        "expectedStatus": { "type": "integer", "minimum": 100, "maximum": 599 },
+        "assertions": {
+          "type": "array",
+          "items": { "$ref": "#/$defs/httpWitnessResolvedAssertion" }
+        }
+      }
+    },
+    "verificationRequestMongoDocument": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["kind", "collection", "filter", "requiredFields"],
+      "properties": {
+        "kind": { "const": "mongo_document" },
+        "collection": { "type": "string", "minLength": 1 },
+        "filter": { "type": "object", "additionalProperties": true },
+        "requiredFields": { "$ref": "#/$defs/requiredFieldsMap" }
+      }
+    },
     "effectOutcome": {
       "type": "object",
       "additionalProperties": false,
@@ -478,7 +542,11 @@
                 { "$ref": "#/$defs/verificationRequestSqlRow" },
                 { "$ref": "#/$defs/verificationRequestSqlRowAbsent" },
                 { "$ref": "#/$defs/verificationRequestSqlEffects" },
-                { "$ref": "#/$defs/verificationRequestSqlRelational" }
+                { "$ref": "#/$defs/verificationRequestSqlRelational" },
+                { "$ref": "#/$defs/verificationRequestVectorDocument" },
+                { "$ref": "#/$defs/verificationRequestObjectStorageObject" },
+                { "$ref": "#/$defs/verificationRequestHttpWitness" },
+                { "$ref": "#/$defs/verificationRequestMongoDocument" }
               ]
             },
             "status": {
@@ -502,6 +570,7 @@
             },
             "repeatObservationCount": { "type": "integer", "minimum": 1 },
             "evaluatedObservationOrdinal": { "type": "integer", "minimum": 1 },
+            "langgraphCheckpointKey": { "type": "string", "minLength": 1 },
             "failureDiagnostic": {
               "type": "string",
               "enum": [

package/schemas/workflow-truth-report.schema.json

@@ -624,6 +624,18 @@
                 },
                 {
                   "$ref": "https://agentskeptic.com/schemas/workflow-engine-result.schema.json#/$defs/verificationRequestSqlRelational"
+                },
+                {
+                  "$ref": "https://agentskeptic.com/schemas/workflow-engine-result.schema.json#/$defs/verificationRequestVectorDocument"
+                },
+                {
+                  "$ref": "https://agentskeptic.com/schemas/workflow-engine-result.schema.json#/$defs/verificationRequestObjectStorageObject"
+                },
+                {
+                  "$ref": "https://agentskeptic.com/schemas/workflow-engine-result.schema.json#/$defs/verificationRequestHttpWitness"
+                },
+                {
+                  "$ref": "https://agentskeptic.com/schemas/workflow-engine-result.schema.json#/$defs/verificationRequestMongoDocument"
                 }
               ]
             }