ai-capability-framework 1.0.3 → 1.1.0

package/examples/12-capability-discovery-selection/selection-example.mjs

@@ -0,0 +1,88 @@
+import {
+  createCapabilityDiscovery,
+  createInMemoryDiscoveryBudget,
+  aggregateDiscoveryUsage
+} from "ai-capability-framework/discovery";
+import { createDiscoveryExample } from "./discovery-example.mjs";
+import {
+  MockSelectionClient,
+  selectorCases,
+  selectionResponse,
+  selectedDecision,
+  abstainedDecision
+} from "./mock-selection.mjs";
+
+export async function runSelectionExample() {
+  const harness = await createDiscoveryExample();
+  const discovery = createCapabilityDiscovery({ registry: harness.registry });
+  const results = await discovery.search({
+    tasks: selectorCases.map(({ id, text }) => ({ id, text })),
+    runtimeContext: harness.runtimeContext
+  });
+  const budget = createInMemoryDiscoveryBudget({
+    runId: harness.runtimeContext.runId,
+    maxProviderCalls: 3,
+    maxInputTokens: 12_000
+  });
+  const client = new MockSelectionClient([
+    selectionResponse(
+      selectedDecision("competition", "support.ticket.get"),
+      abstainedDecision("single")
+    )
+  ]);
+  const initial = await discovery.select({
+    results,
+    client,
+    budget,
+    maxInputTokens: 4000,
+    maxOutputTokens: 1000
+  });
+  const repairClient = new MockSelectionClient([
+    selectionResponse(selectedDecision("competition", "support.foreign")),
+    selectionResponse(selectedDecision("competition", "support.ticket.get"))
+  ]);
+  const repaired = await discovery.select({
+    results: [results[0]],
+    client: repairClient,
+    budget,
+    maxInputTokens: 4000,
+    maxOutputTokens: 1000,
+    limits: { maxAttempts: 2 }
+  });
+  if (
+    initial.outcomes[0]?.status !== "selected" ||
+    initial.outcomes[1]?.status !== "abstained" ||
+    initial.outcomes.length !== 2 ||
+    initial.discovery[2].status !== "no_match" ||
+    repaired.outcomes[0]?.status !== "selected"
+  ) {
+    throw new Error(
+      "The mock selector must choose, abstain, skip and repair as specified."
+    );
+  }
+  for (const batch of [initial, repaired])
+    for (const outcome of batch.outcomes) {
+      const fixture = selectorCases.find((entry) => entry.id === outcome.queryId);
+      if (
+        outcome.status === "selected" &&
+        (!fixture.compatibleIds.includes(outcome.capabilityId) ||
+          fixture.forbiddenIds.includes(outcome.capabilityId))
+      ) {
+        throw new Error(
+          "The selected capability must be compatible and not forbidden."
+        );
+      }
+    }
+  return {
+    mock: true,
+    initial,
+    repaired,
+    mockSelectionAttempts: client.calls.length + repairClient.calls.length,
+    toolCalls: harness.readCalls.length,
+    embeddingAttempts: 0,
+    usage: aggregateDiscoveryUsage(
+      [...initial.usage.stages, ...repaired.usage.stages],
+      initial.usage.totals.elapsedMs + repaired.usage.totals.elapsedMs
+    )
+  };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ai-capability-framework",
-  "version": "1.0.3",
+  "version": "1.1.0",
   "private": false,
   "description": "Provider-agnostic AI capability contracts, runtime controls, evals, and model/provider adapters.",
   "license": "MIT",
@@ -198,7 +198,17 @@
     "./schemas/security-packs/promptfoo-red-team-config.schema.json": "./schemas/security-packs/promptfoo-red-team-config.schema.json",
     "./schemas/security-packs/security-case-suite.schema.json": "./schemas/security-packs/security-case-suite.schema.json",
     "./schemas/security-packs/security-pack.schema.json": "./schemas/security-packs/security-pack.schema.json",
-    "./package.json": "./package.json"
+    "./package.json": "./package.json",
+    "./discovery": {
+      "types": "./dist/discovery/index.d.ts",
+      "import": "./dist/discovery/index.js"
+    },
+    "./schemas/discovery/metadata.schema.json": "./schemas/discovery/metadata.schema.json",
+    "./schemas/discovery/embedding-index.schema.json": "./schemas/discovery/embedding-index.schema.json",
+    "./schemas/discovery/selection-response.schema.json": "./schemas/discovery/selection-response.schema.json",
+    "./schemas/discovery/eval-suite.schema.json": "./schemas/discovery/eval-suite.schema.json",
+    "./schemas/discovery/eval-observations.schema.json": "./schemas/discovery/eval-observations.schema.json",
+    "./schemas/replay/discovery-snapshot.schema.json": "./schemas/replay/discovery-snapshot.schema.json"
   },
   "publishConfig": {
     "access": "public"
@@ -223,7 +233,7 @@
     "archive:source": "node scripts/create-source-archive.mjs",
     "build": "tsc -p tsconfig.json",
     "typecheck": "tsc -p tsconfig.json --noEmit",
-    "check": "npm run check:generated && npm run build && npm run typecheck && npm run lint && npm test && npm run validate && npm run conformance && npm run format:check && npm run check:readability && npm run gate:examples && npm run docs:build && npm run check:package && npm run check:workspace-public",
+    "check": "npm run check:generated && npm run build && npm run typecheck && npm run lint && npm test && npm run validate && npm run conformance && npm run eval:discovery:mock && npm run format:check && npm run check:readability && npm run gate:examples && npm run docs:build && npm run check:package && npm run check:workspace-public",
     "check:docs": "node scripts/check-docs.mjs",
     "check:generated": "node scripts/check-generated.mjs",
     "check:readability": "node scripts/check-public-readability.mjs",
@@ -296,7 +306,10 @@
     "validate": "npm run build --silent && node dist/cli.js validate examples",
     "skills:ci": "npm --prefix agent-skills ci",
     "skills:pack:dry": "npm --prefix agent-skills run pack:dry",
-    "skills:publish:dry": "npm publish ./agent-skills --dry-run --access public"
+    "skills:publish:dry": "npm publish ./agent-skills --dry-run --access public",
+    "eval:discovery:mock": "npm run build --silent && node examples/12-capability-discovery-selection/run-evaluation-mock.mjs",
+    "eval:discovery:live": "npm run build --silent && node examples/12-capability-discovery-selection/run-evaluation-live.mjs",
+    "eval:discovery:campaign:mock": "npm run build --silent && node examples/12-capability-discovery-selection/run-campaign-mock.mjs"
   },
   "engines": {
     "node": ">=20"

package/schemas/discovery/embedding-index.schema.json

@@ -0,0 +1,71 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://raw.githubusercontent.com/xennon101/ai-capability-framework/main/schemas/discovery/embedding-index.schema.json",
+  "title": "CapabilityEmbeddingIndexArtifact",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "schemaVersion",
+    "projectionVersion",
+    "registrySourceFingerprint",
+    "corpusFingerprint",
+    "embedding",
+    "embeddingConfigurationFingerprint",
+    "mock",
+    "vectorDimension",
+    "entries",
+    "indexFingerprint",
+    "builtAt"
+  ],
+  "properties": {
+    "schemaVersion": { "const": "1.0" },
+    "projectionVersion": { "type": "string", "minLength": 1 },
+    "registrySourceFingerprint": { "$ref": "#/$defs/hash" },
+    "corpusFingerprint": { "$ref": "#/$defs/hash" },
+    "embeddingConfigurationFingerprint": { "$ref": "#/$defs/hash" },
+    "indexFingerprint": { "$ref": "#/$defs/hash" },
+    "mock": { "type": "boolean" },
+    "vectorDimension": { "type": "integer", "minimum": 1 },
+    "builtAt": {
+      "type": "string",
+      "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$"
+    },
+    "embedding": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["provider", "model", "configurationId"],
+      "properties": {
+        "provider": { "$ref": "#/$defs/nonblank" },
+        "model": { "$ref": "#/$defs/nonblank" },
+        "configurationId": { "$ref": "#/$defs/nonblank" },
+        "revision": { "$ref": "#/$defs/nonblank" },
+        "dimensions": { "type": "integer", "minimum": 1 }
+      }
+    },
+    "entries": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": [
+          "capabilityId",
+          "capabilityVersion",
+          "documentHash",
+          "vector",
+          "vectorHash"
+        ],
+        "properties": {
+          "capabilityId": { "$ref": "#/$defs/nonblank" },
+          "capabilityVersion": { "$ref": "#/$defs/nonblank" },
+          "documentHash": { "$ref": "#/$defs/hash" },
+          "vectorHash": { "$ref": "#/$defs/hash" },
+          "vector": { "type": "array", "minItems": 1, "items": { "type": "number" } }
+        }
+      }
+    }
+  },
+  "$defs": {
+    "nonblank": { "type": "string", "minLength": 1, "pattern": "\\S" },
+    "hash": { "type": "string", "pattern": "^sha256:[a-f0-9]{64}$" }
+  }
+}

package/schemas/discovery/eval-observations.schema.json

@@ -0,0 +1,409 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://raw.githubusercontent.com/xennon101/ai-capability-framework/main/schemas/discovery/eval-observations.schema.json",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "schema_version",
+    "suite_id",
+    "suite_fingerprint",
+    "registry_source_fingerprint",
+    "corpus_fingerprint",
+    "retrieval_policy_fingerprint",
+    "configuration",
+    "usage_complete",
+    "attempts",
+    "cases"
+  ],
+  "properties": {
+    "schema_version": {
+      "const": "1.0"
+    },
+    "suite_id": {
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 200
+    },
+    "suite_fingerprint": {
+      "type": "string",
+      "pattern": "^sha256:[a-f0-9]{64}$"
+    },
+    "registry_source_fingerprint": {
+      "type": "string",
+      "pattern": "^sha256:[a-f0-9]{64}$"
+    },
+    "corpus_fingerprint": {
+      "type": "string",
+      "pattern": "^sha256:[a-f0-9]{64}$"
+    },
+    "retrieval_policy_fingerprint": {
+      "type": "string",
+      "pattern": "^sha256:[a-f0-9]{64}$"
+    },
+    "configuration": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "mode",
+        "policyVersion",
+        "limits",
+        "aliasWeight",
+        "minRetrievalScore"
+      ],
+      "properties": {
+        "mode": {
+          "enum": ["lexical", "hybrid"]
+        },
+        "policyVersion": {
+          "type": "string",
+          "minLength": 1,
+          "maxLength": 200
+        },
+        "limits": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": [
+            "maxQueries",
+            "maxFragmentsPerQuery",
+            "maxQueryCharacters",
+            "maxCandidates",
+            "aliasDiversitySlots"
+          ],
+          "properties": {
+            "maxQueries": {
+              "type": "integer",
+              "minimum": 1,
+              "maximum": 32
+            },
+            "maxFragmentsPerQuery": {
+              "type": "integer",
+              "minimum": 1,
+              "maximum": 32
+            },
+            "maxQueryCharacters": {
+              "type": "integer",
+              "minimum": 1,
+              "maximum": 16000
+            },
+            "maxCandidates": {
+              "type": "integer",
+              "minimum": 1,
+              "maximum": 20
+            },
+            "aliasDiversitySlots": {
+              "enum": [0, 1]
+            }
+          }
+        },
+        "aliasWeight": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": 1
+        },
+        "minRetrievalScore": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": 1
+        },
+        "weights": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["semantic", "lexical"],
+          "properties": {
+            "semantic": {
+              "type": "number",
+              "minimum": 0,
+              "maximum": 1
+            },
+            "lexical": {
+              "type": "number",
+              "minimum": 0,
+              "maximum": 1
+            }
+          }
+        },
+        "failurePolicy": {
+          "enum": ["unavailable", "lexical_fallback"]
+        }
+      }
+    },
+    "custom_ranker_version": {
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 200
+    },
+    "usage_complete": {
+      "type": "boolean"
+    },
+    "attempts": {
+      "type": "array",
+      "maxItems": 100000,
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": [
+          "run_id",
+          "attempt_id",
+          "stage",
+          "configuration",
+          "mock",
+          "provider_attempts",
+          "completed_responses",
+          "retry_attempts",
+          "repair_attempts",
+          "usage",
+          "elapsed_ms",
+          "pending",
+          "outcome"
+        ],
+        "properties": {
+          "run_id": {
+            "type": "string",
+            "minLength": 1,
+            "maxLength": 200
+          },
+          "attempt_id": {
+            "type": "string",
+            "minLength": 1,
+            "maxLength": 200
+          },
+          "stage": {
+            "enum": [
+              "embedding_build",
+              "query_embedding",
+              "selection",
+              "selection_repair",
+              "model_tool_loop",
+              "planning",
+              "response_assembly"
+            ]
+          },
+          "configuration": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["provider", "model", "configurationId"],
+            "properties": {
+              "provider": {
+                "type": "string",
+                "minLength": 1,
+                "maxLength": 200
+              },
+              "model": {
+                "type": "string",
+                "minLength": 1,
+                "maxLength": 200
+              },
+              "configurationId": {
+                "type": "string",
+                "minLength": 1,
+                "maxLength": 200
+              },
+              "revision": {
+                "type": "string",
+                "minLength": 1,
+                "maxLength": 200
+              }
+            }
+          },
+          "mock": {
+            "type": "boolean"
+          },
+          "provider_attempts": {
+            "type": "integer",
+            "minimum": 0
+          },
+          "completed_responses": {
+            "type": "integer",
+            "minimum": 0
+          },
+          "retry_attempts": {
+            "type": "integer",
+            "minimum": 0
+          },
+          "repair_attempts": {
+            "type": "integer",
+            "minimum": 0
+          },
+          "usage": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": [],
+            "properties": {
+              "inputTokens": {
+                "type": "integer",
+                "minimum": 0
+              },
+              "outputTokens": {
+                "type": "integer",
+                "minimum": 0
+              },
+              "totalTokens": {
+                "type": "integer",
+                "minimum": 0
+              },
+              "cachedInputTokens": {
+                "type": "integer",
+                "minimum": 0
+              },
+              "reasoningTokens": {
+                "type": "integer",
+                "minimum": 0
+              },
+              "estimatedCost": {
+                "type": "number",
+                "minimum": 0
+              }
+            }
+          },
+          "elapsed_ms": {
+            "type": "number",
+            "minimum": 0
+          },
+          "pending": {
+            "type": "boolean"
+          },
+          "outcome": {
+            "enum": ["completed", "failed", "cancelled", "discarded"]
+          }
+        }
+      }
+    },
+    "cases": {
+      "type": "array",
+      "maxItems": 10000,
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": [
+          "case_id",
+          "status",
+          "actual_mode",
+          "eligibility_fingerprint",
+          "candidates",
+          "latency_ms"
+        ],
+        "properties": {
+          "case_id": {
+            "type": "string",
+            "minLength": 1,
+            "maxLength": 200
+          },
+          "status": {
+            "enum": ["matched", "no_match", "unavailable", "cancelled"]
+          },
+          "actual_mode": {
+            "enum": ["lexical", "hybrid", null]
+          },
+          "eligibility_fingerprint": {
+            "type": "string",
+            "pattern": "^sha256:[a-f0-9]{64}$"
+          },
+          "index_fingerprint": {
+            "type": "string",
+            "pattern": "^sha256:[a-f0-9]{64}$"
+          },
+          "candidates": {
+            "type": "array",
+            "maxItems": 20,
+            "items": {
+              "type": "object",
+              "additionalProperties": false,
+              "required": ["capability_id", "capability_version", "score"],
+              "properties": {
+                "capability_id": {
+                  "type": "string",
+                  "minLength": 1,
+                  "maxLength": 200
+                },
+                "capability_version": {
+                  "type": "string",
+                  "minLength": 1,
+                  "maxLength": 200
+                },
+                "score": {
+                  "type": "number",
+                  "exclusiveMinimum": 0,
+                  "maximum": 1
+                }
+              }
+            }
+          },
+          "latency_ms": {
+            "type": "number",
+            "minimum": 0
+          },
+          "selection": {
+            "anyOf": [
+              {
+                "type": "object",
+                "additionalProperties": false,
+                "required": [
+                  "mode",
+                  "status",
+                  "capability_id",
+                  "capability_version",
+                  "operation"
+                ],
+                "properties": {
+                  "mode": {
+                    "enum": ["native", "dedicated"]
+                  },
+                  "status": {
+                    "const": "selected"
+                  },
+                  "capability_id": {
+                    "type": "string",
+                    "minLength": 1,
+                    "maxLength": 200
+                  },
+                  "capability_version": {
+                    "type": "string",
+                    "minLength": 1,
+                    "maxLength": 200
+                  },
+                  "operation": {
+                    "enum": ["read", "prepare", "commit"]
+                  }
+                }
+              },
+              {
+                "type": "object",
+                "additionalProperties": false,
+                "required": [
+                  "mode",
+                  "status",
+                  "capability_id",
+                  "capability_version",
+                  "operation"
+                ],
+                "properties": {
+                  "mode": {
+                    "enum": ["native", "dedicated"]
+                  },
+                  "status": {
+                    "enum": [
+                      "abstained",
+                      "unavailable",
+                      "invalid_output",
+                      "cancelled",
+                      "not_performed"
+                    ]
+                  },
+                  "capability_id": {
+                    "type": "null"
+                  },
+                  "capability_version": {
+                    "type": "null"
+                  },
+                  "operation": {
+                    "type": "null"
+                  }
+                }
+              }
+            ]
+          }
+        }
+      }
+    }
+  }
+}