agentskeptic 1.0.4 → 1.1.1

package/schemas/regression-artifact-v1.schema.json

@@ -0,0 +1,212 @@
+{
+  "$id": "https://agentskeptic.com/schemas/regression-artifact-v1.schema.json",
+  "title": "RegressionArtifactV1",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "schemaVersion",
+    "artifactSource",
+    "manifestSha256",
+    "workflowId",
+    "certificateProfile",
+    "verification",
+    "outcomeCertificates",
+    "tracePairwise",
+    "narrative",
+    "humanText",
+    "narrativeHtml"
+  ],
+  "properties": {
+    "schemaVersion": { "const": 1 },
+    "artifactSource": { "type": "string", "enum": ["cli_manifest", "debug_corpus"] },
+    "manifestSha256": { "type": "string", "pattern": "^[a-f0-9]{64}$" },
+    "workflowId": { "type": "string", "minLength": 1 },
+    "certificateProfile": {
+      "oneOf": [
+        {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["mode", "outcomeCertificateRunKind"],
+          "properties": {
+            "mode": { "const": "uniform" },
+            "outcomeCertificateRunKind": {
+              "type": "string",
+              "enum": ["contract_sql", "contract_sql_langgraph_checkpoint_trust"]
+            }
+          }
+        },
+        {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["mode", "entries"],
+          "properties": {
+            "mode": { "const": "perRun" },
+            "entries": {
+              "type": "array",
+              "minItems": 1,
+              "items": {
+                "type": "object",
+                "additionalProperties": false,
+                "required": ["runIndex", "outcomeCertificateRunKind"],
+                "properties": {
+                  "runIndex": { "type": "integer", "minimum": 0 },
+                  "outcomeCertificateRunKind": {
+                    "type": "string",
+                    "enum": ["contract_sql", "contract_sql_langgraph_checkpoint_trust"]
+                  }
+                }
+              }
+            }
+          }
+        }
+      ]
+    },
+    "verification": { "$ref": "https://agentskeptic.com/schemas/run-comparison-report.schema.json" },
+    "outcomeCertificates": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": [
+          "runIndex",
+          "displayLabel",
+          "outcomeCertificateRunKind",
+          "certificateCanonicalDigest",
+          "certificate"
+        ],
+        "properties": {
+          "runIndex": { "type": "integer", "minimum": 0 },
+          "displayLabel": { "type": "string", "minLength": 1 },
+          "outcomeCertificateRunKind": {
+            "type": "string",
+            "enum": ["contract_sql", "contract_sql_langgraph_checkpoint_trust"]
+          },
+          "certificateCanonicalDigest": { "type": "string", "pattern": "^[a-f0-9]{64}$" },
+          "certificate": { "$ref": "https://agentskeptic.com/schemas/outcome-certificate-v1.schema.json" }
+        }
+      }
+    },
+    "tracePairwise": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "priorRunIndex",
+        "currentRunIndex",
+        "viewBuild",
+        "bySeq",
+        "seqTimelineOrderDiverged",
+        "nonToolEventCounts"
+      ],
+      "properties": {
+        "priorRunIndex": { "type": "integer", "minimum": 0 },
+        "currentRunIndex": { "type": "integer", "minimum": 0 },
+        "viewBuild": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["prior", "current"],
+          "properties": {
+            "prior": { "$ref": "https://agentskeptic.com/schemas/execution-trace-view.schema.json" },
+            "current": { "$ref": "https://agentskeptic.com/schemas/execution-trace-view.schema.json" }
+          }
+        },
+        "bySeq": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["seq", "class", "prior", "current"],
+            "properties": {
+              "seq": { "type": "integer", "minimum": 0 },
+              "class": { "type": "string", "enum": ["only_in_prior", "only_in_current", "in_both"] },
+              "prior": { "oneOf": [{ "type": "null" }, { "$ref": "#/$defs/evaluatedNodeSummary" }] },
+              "current": { "oneOf": [{ "type": "null" }, { "$ref": "#/$defs/evaluatedNodeSummary" }] }
+            }
+          }
+        },
+        "seqTimelineOrderDiverged": { "type": "boolean" },
+        "nonToolEventCounts": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["prior", "current"],
+          "properties": {
+            "prior": { "type": "object", "additionalProperties": { "type": "integer", "minimum": 0 } },
+            "current": { "type": "object", "additionalProperties": { "type": "integer", "minimum": 0 } }
+          }
+        }
+      }
+    },
+    "narrative": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "classification",
+        "whyItMatters",
+        "headline",
+        "structural",
+        "traceSummary"
+      ],
+      "properties": {
+        "classification": {
+          "type": "string",
+          "enum": ["improving", "worsening", "unchanged", "mixed"]
+        },
+        "whyItMatters": { "type": "string" },
+        "headline": { "type": "string" },
+        "structural": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": [
+            "introducedLogicalStepKeysCount",
+            "resolvedLogicalStepKeysCount",
+            "recurringSignatureCount"
+          ],
+          "properties": {
+            "introducedLogicalStepKeysCount": { "type": "integer", "minimum": 0 },
+            "resolvedLogicalStepKeysCount": { "type": "integer", "minimum": 0 },
+            "recurringSignatureCount": { "type": "integer", "minimum": 0 }
+          }
+        },
+        "traceSummary": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": [
+            "seqTimelineOrderDiverged",
+            "onlyInPriorSeqCount",
+            "onlyInCurrentSeqCount",
+            "inBothCount",
+            "nonToolCountsDiverged"
+          ],
+          "properties": {
+            "seqTimelineOrderDiverged": { "type": "boolean" },
+            "onlyInPriorSeqCount": { "type": "integer", "minimum": 0 },
+            "onlyInCurrentSeqCount": { "type": "integer", "minimum": 0 },
+            "inBothCount": { "type": "integer", "minimum": 0 },
+            "nonToolCountsDiverged": { "type": "boolean" }
+          }
+        }
+      }
+    },
+    "humanText": { "type": "string" },
+    "narrativeHtml": { "type": "string", "minLength": 1 }
+  },
+  "$defs": {
+    "evaluatedNodeSummary": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "ingestIndex",
+        "traceStepKind",
+        "engineStepStatus",
+        "truthOutcomeLabel",
+        "toolId"
+      ],
+      "properties": {
+        "ingestIndex": { "type": "integer", "minimum": 0 },
+        "traceStepKind": { "type": "string" },
+        "engineStepStatus": { "oneOf": [{ "type": "string" }, { "type": "null" }] },
+        "truthOutcomeLabel": { "oneOf": [{ "type": "string" }, { "type": "null" }] },
+        "toolId": { "oneOf": [{ "type": "string" }, { "type": "null" }] }
+      }
+    }
+  }
+}

package/schemas/tools-registry.schema.json

@@ -72,12 +72,115 @@
                 "items": { "$ref": "#/$defs/sqlRelationalCheck" }
               }
             }
+          },
+          {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["kind", "provider", "documentId", "indexName"],
+            "properties": {
+              "kind": { "const": "vector_document" },
+              "provider": { "type": "string", "enum": ["pinecone", "weaviate", "chroma"] },
+              "documentId": { "$ref": "#/$defs/stringSpec" },
+              "indexName": { "$ref": "#/$defs/stringSpec" },
+              "namespace": { "$ref": "#/$defs/stringSpec" },
+              "host": { "$ref": "#/$defs/stringSpec" },
+              "metadataEq": { "$ref": "#/$defs/requiredFieldsPointer" },
+              "expectPayloadSha256": { "$ref": "#/$defs/stringSpec" }
+            }
+          },
+          {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["kind", "bucket", "key"],
+            "properties": {
+              "kind": { "const": "object_storage_object" },
+              "bucket": { "$ref": "#/$defs/stringSpec" },
+              "key": { "$ref": "#/$defs/stringSpec" },
+              "endpoint": { "$ref": "#/$defs/stringSpec" },
+              "expectSizeBytes": { "$ref": "#/$defs/numberOrPointerSpec" },
+              "expectSha256": { "$ref": "#/$defs/stringSpec" },
+              "expectEtag": { "$ref": "#/$defs/stringSpec" }
+            }
+          },
+          {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["kind", "url"],
+            "properties": {
+              "kind": { "const": "http_witness" },
+              "method": { "type": "string", "enum": ["GET", "POST"] },
+              "url": { "$ref": "#/$defs/stringSpec" },
+              "expectedStatus": { "$ref": "#/$defs/numberOrPointerSpec" },
+              "assertions": {
+                "type": "array",
+                "items": { "$ref": "#/$defs/httpWitnessAssertion" }
+              }
+            }
+          },
+          {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["kind", "collection", "filterPointer", "requiredFields"],
+            "properties": {
+              "kind": { "const": "mongo_document" },
+              "collection": { "$ref": "#/$defs/stringSpec" },
+              "filterPointer": {
+                "type": "object",
+                "additionalProperties": false,
+                "required": ["pointer"],
+                "properties": {
+                  "pointer": { "type": "string", "pattern": "^/" }
+                }
+              },
+              "requiredFields": { "$ref": "#/$defs/requiredFieldsPointer" }
+            }
           }
         ]
       }
     }
   },
   "$defs": {
+    "stringSpec": {
+      "oneOf": [
+        {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["const"],
+          "properties": { "const": { "type": "string", "minLength": 1 } }
+        },
+        {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["pointer"],
+          "properties": { "pointer": { "type": "string", "pattern": "^/" } }
+        }
+      ]
+    },
+    "numberOrPointerSpec": {
+      "oneOf": [
+        {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["const"],
+          "properties": { "const": { "type": "number" } }
+        },
+        {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["pointer"],
+          "properties": { "pointer": { "type": "string", "pattern": "^/" } }
+        }
+      ]
+    },
+    "httpWitnessAssertion": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["jsonPointer", "value"],
+      "properties": {
+        "jsonPointer": { "type": "string", "minLength": 1 },
+        "value": { "$ref": "#/$defs/scalarOrPointer" }
+      }
+    },
     "tableSpec": {
       "oneOf": [
         {

package/schemas/workflow-engine-result.schema.json

@@ -414,6 +414,70 @@
         }
       }
     },
+    "verificationRequestVectorDocument": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["kind", "provider", "documentId", "indexName"],
+      "properties": {
+        "kind": { "const": "vector_document" },
+        "provider": { "type": "string", "enum": ["pinecone", "weaviate", "chroma"] },
+        "documentId": { "type": "string", "minLength": 1 },
+        "indexName": { "type": "string", "minLength": 1 },
+        "namespace": { "type": "string" },
+        "host": { "type": "string", "minLength": 1 },
+        "metadataSubset": { "type": "object", "additionalProperties": true },
+        "expectPayloadSha256": { "type": "string", "minLength": 1 }
+      }
+    },
+    "verificationRequestObjectStorageObject": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["kind", "bucket", "key"],
+      "properties": {
+        "kind": { "const": "object_storage_object" },
+        "bucket": { "type": "string", "minLength": 1 },
+        "key": { "type": "string", "minLength": 1 },
+        "endpoint": { "type": "string", "minLength": 1 },
+        "expectSizeBytes": { "type": "number", "minimum": 0 },
+        "expectSha256": { "type": "string", "minLength": 1 },
+        "expectEtag": { "type": "string", "minLength": 1 }
+      }
+    },
+    "httpWitnessResolvedAssertion": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["jsonPointer", "value"],
+      "properties": {
+        "jsonPointer": { "type": "string", "minLength": 1 },
+        "value": { "type": ["string", "number", "boolean", "null"] }
+      }
+    },
+    "verificationRequestHttpWitness": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["kind", "method", "url"],
+      "properties": {
+        "kind": { "const": "http_witness" },
+        "method": { "type": "string", "enum": ["GET", "POST"] },
+        "url": { "type": "string", "minLength": 1 },
+        "expectedStatus": { "type": "integer", "minimum": 100, "maximum": 599 },
+        "assertions": {
+          "type": "array",
+          "items": { "$ref": "#/$defs/httpWitnessResolvedAssertion" }
+        }
+      }
+    },
+    "verificationRequestMongoDocument": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["kind", "collection", "filter", "requiredFields"],
+      "properties": {
+        "kind": { "const": "mongo_document" },
+        "collection": { "type": "string", "minLength": 1 },
+        "filter": { "type": "object", "additionalProperties": true },
+        "requiredFields": { "$ref": "#/$defs/requiredFieldsMap" }
+      }
+    },
     "effectOutcome": {
       "type": "object",
       "additionalProperties": false,
@@ -478,7 +542,11 @@
                 { "$ref": "#/$defs/verificationRequestSqlRow" },
                 { "$ref": "#/$defs/verificationRequestSqlRowAbsent" },
                 { "$ref": "#/$defs/verificationRequestSqlEffects" },
-                { "$ref": "#/$defs/verificationRequestSqlRelational" }
+                { "$ref": "#/$defs/verificationRequestSqlRelational" },
+                { "$ref": "#/$defs/verificationRequestVectorDocument" },
+                { "$ref": "#/$defs/verificationRequestObjectStorageObject" },
+                { "$ref": "#/$defs/verificationRequestHttpWitness" },
+                { "$ref": "#/$defs/verificationRequestMongoDocument" }
               ]
             },
             "status": {

package/schemas/workflow-truth-report.schema.json

@@ -624,6 +624,18 @@
                 },
                 {
                   "$ref": "https://agentskeptic.com/schemas/workflow-engine-result.schema.json#/$defs/verificationRequestSqlRelational"
+                },
+                {
+                  "$ref": "https://agentskeptic.com/schemas/workflow-engine-result.schema.json#/$defs/verificationRequestVectorDocument"
+                },
+                {
+                  "$ref": "https://agentskeptic.com/schemas/workflow-engine-result.schema.json#/$defs/verificationRequestObjectStorageObject"
+                },
+                {
+                  "$ref": "https://agentskeptic.com/schemas/workflow-engine-result.schema.json#/$defs/verificationRequestHttpWitness"
+                },
+                {
+                  "$ref": "https://agentskeptic.com/schemas/workflow-engine-result.schema.json#/$defs/verificationRequestMongoDocument"
                 }
               ]
             }

package/scripts/discovery-acquisition.lib.cjs

@@ -2,8 +2,6 @@
 
 const { existsSync, readFileSync, readdirSync } = require("node:fs");
 const { join } = require("node:path");
-const Ajv = require("ajv");
-const addFormats = require("ajv-formats");
 
 const PLACEHOLDER_KEYS = [
   ["{{ORIGIN}}", "ORIGIN"],
@@ -20,8 +18,7 @@ const PLACEHOLDER_KEYS = [
  */
 function discoveryPaths(root) {
   return {
-    jsonPath: join(root, "config", "discovery-acquisition.json"),
-    schemaPath: join(root, "config", "discovery-acquisition.schema.json"),
+    jsonPath: join(root, "config", "marketing.json"),
   };
 }
 
@@ -177,20 +174,12 @@ function appendDiscoveryLlmsAppendix(baseLlms, discovery, canonicalOrigin) {
  * @param {string} root
  */
 function validateDiscoveryAcquisition(root) {
-  const { jsonPath, schemaPath } = discoveryPaths(root);
+  const { jsonPath } = discoveryPaths(root);
+  const { validateMarketing } = require("./validate-marketing.cjs");
+  validateMarketing(root);
   const discovery = JSON.parse(readFileSync(jsonPath, "utf8"));
-  const schema = JSON.parse(readFileSync(schemaPath, "utf8"));
-  const ajv = new Ajv({ allErrors: true, strict: true });
-  addFormats(ajv);
-  const validate = ajv.compile(schema);
-  if (!validate(discovery)) {
-    const msg = ajv.errorsText(validate.errors, { separator: "\n" });
-    throw new Error(`discovery-acquisition: schema validation failed:\n${msg}`);
-  }
-  const anchorsPath = join(root, "config", "public-product-anchors.json");
-  const anchors = JSON.parse(readFileSync(anchorsPath, "utf8"));
-  const { normalize } = require("./public-product-anchors.cjs");
-  const origin = normalize(anchors.productionCanonicalOrigin);
+  const { normalize } = require("./origin.cjs");
+  const origin = normalize(String(discovery.productionCanonicalOrigin));
   buildDiscoveryFoldBody(discovery, origin);
   const demo = discovery.shareableTerminalDemo;
   if (demo && String(demo.transcript).includes("```")) {

package/scripts/discovery-payload.lib.cjs

@@ -132,8 +132,10 @@ function buildDiscoveryPayload(root) {
     problemAnchor: padAnchor(path),
     embedKey: embedKeyForExamplePath(path),
   }));
-  const anchorsPath = join(root, "config", "public-product-anchors.json");
-  const anchors = JSON.parse(readFileSync(anchorsPath, "utf8"));
+  const pm = /** @type {{ productionCanonicalOrigin: string; identityOneLiner: string; gitRepositoryUrl: string; npmPackageUrl: string }} */ (
+    discovery
+  );
+  const anchors = pm;
   const canonicalOrigin = normalizeOrigin(anchors.productionCanonicalOrigin);
   const integrateUrl = `${canonicalOrigin}/integrate`;
   const learnHubUrl = `${canonicalOrigin}/guides`;
@@ -141,7 +143,7 @@ function buildDiscoveryPayload(root) {
   const { owner, repo } = parseGithubRepoFromUrl(anchors.gitRepositoryUrl);
   const llmsRaw = `https://raw.githubusercontent.com/${owner}/${repo}/refs/heads/${DISCOVERY_LLM_BRANCH}/llms.txt`;
   const llmsBlob = `https://github.com/${owner}/${repo}/blob/${DISCOVERY_LLM_BRANCH}/llms.txt`;
-  const integratorVerificationSsotRaw = `https://raw.githubusercontent.com/${owner}/${repo}/refs/heads/${DISCOVERY_LLM_BRANCH}/docs/integrator-verification-ssot.md`;
+  const integratorVerificationSsotRaw = `https://raw.githubusercontent.com/${owner}/${repo}/refs/heads/${DISCOVERY_LLM_BRANCH}/docs/integrator-verification.md`;
   const openapiRaw = `https://raw.githubusercontent.com/${owner}/${repo}/refs/heads/${DISCOVERY_LLM_BRANCH}/schemas/openapi-commercial-v1.yaml`;
   const llms = /** @type {{ intentPhrases: string[]; notFor: string[]; relatedQueries: string[] }} */ (
     discovery.llms