ai-capability-framework 1.0.3 → 1.1.0

package/dist/discovery/evaluation.d.ts

@@ -0,0 +1,84 @@
+import type { DiscoveryEvaluationSuite } from "../generated/manifest-types.js";
+import type { ProviderUsage } from "../runtime/provider-types.js";
+import type { ManifestRegistry } from "../types.js";
+import type { DiscoveryEvaluationDiagnostic } from "./evaluation-validation.js";
+export type { DiscoveryEvaluationSuite, DiscoveryEvaluationObservations } from "../generated/manifest-types.js";
+export type { DiscoveryEvaluationDiagnostic } from "./evaluation-validation.js";
+export interface DiscoveryEvaluationRate {
+    numerator: number;
+    denominator: number;
+    value: number | null;
+}
+export interface DiscoveryEvaluationDistribution {
+    count: number;
+    minimum: number | null;
+    mean: number | null;
+    p95: number | null;
+    maximum: number | null;
+}
+export interface DiscoveryEvaluationReport {
+    suiteFingerprint: string;
+    scopeFingerprint: string;
+    complete: boolean;
+    semanticQualityQualified: false;
+    counts: {
+        expected: number;
+        observed: number;
+        missing: number;
+        byPartition: Record<string, number>;
+        byDomain: Record<string, number>;
+        byCohort: Record<string, number>;
+    };
+    retrieval: {
+        k: number;
+        compatibleRecallAtK: DiscoveryEvaluationRate;
+        topOnePrecision: DiscoveryEvaluationRate;
+        noMatchFalsePositiveRate: DiscoveryEvaluationRate;
+        shortlistSize: DiscoveryEvaluationDistribution;
+        outcomes: Record<string, number>;
+        forbiddenCandidateCases: number;
+        policyViolationCases: number;
+        readToMutationCases: number;
+    };
+    selection: {
+        outcomes: Record<string, number>;
+        compatibleAccuracy: DiscoveryEvaluationRate;
+        abstentionRate: DiscoveryEvaluationRate;
+        coverage: DiscoveryEvaluationRate;
+        noMatchFalseSelectionRate: DiscoveryEvaluationRate;
+        forbiddenChoices: number;
+        policyViolations: number;
+        readToMutationChoices: number;
+    };
+    latencyMs: DiscoveryEvaluationDistribution;
+    accountingComplete: boolean;
+    usageComplete: boolean;
+    usage: ProviderUsage;
+    provenance: {
+        mockAttempts: number;
+        liveAttempts: number;
+        pendingAttempts: number;
+    };
+}
+export type DiscoveryEvaluationResult = {
+    valid: true;
+    report: Readonly<DiscoveryEvaluationReport>;
+    diagnostics: readonly [];
+} | {
+    valid: false;
+    diagnostics: readonly DiscoveryEvaluationDiagnostic[];
+};
+export interface EvaluateCapabilityDiscoveryRequest {
+    suite: unknown;
+    registry: ManifestRegistry;
+    observations: unknown;
+    /** Explicit expected cases; omitted means the entire frozen suite. Missing observations remain missing. */
+    caseIds?: readonly string[];
+}
+/** Includes authored cases, labels, partitions and capability content; excludes no suite fields. */
+export declare function fingerprintDiscoveryEvaluationSuite(value: DiscoveryEvaluationSuite): string;
+/** Semantic fixture checks used by the CLI without executing retrieval or providers. */
+export declare function validateDiscoveryEvaluationSuite(value: unknown, registry?: ManifestRegistry): readonly DiscoveryEvaluationDiagnostic[];
+/** Scores observations only. It never calls a ranker, model, embedding client, or executor. */
+export declare function evaluateCapabilityDiscovery(request: EvaluateCapabilityDiscoveryRequest): DiscoveryEvaluationResult;
+//# sourceMappingURL=evaluation.d.ts.map

package/dist/discovery/evaluation.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluation.d.ts","sourceRoot":"","sources":["../../src/discovery/evaluation.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAmC,wBAAwB,EAAE,MAAM,gCAAgC,CAAC;AAKhH,OAAO,KAAK,EAAsB,aAAa,EAAE,MAAM,8BAA8B,CAAC;AAEtF,OAAO,KAAK,EAAgD,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAQlG,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,4BAA4B,CAAC;AAIhF,YAAY,EAAE,wBAAwB,EAAE,+BAA+B,EAAE,MAAM,gCAAgC,CAAC;AAChH,YAAY,EAAE,6BAA6B,EAAE,MAAM,4BAA4B,CAAC;AAChF,MAAM,WAAW,uBAAuB;IAAG,SAAS,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CAAE;AAC1G,MAAM,WAAW,+BAA+B;IAAG,KAAK,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;CAAE;AAC5J,MAAM,WAAW,yBAAyB;IACxC,gBAAgB,EAAE,MAAM,CAAC;IACzB,gBAAgB,EAAE,MAAM,CAAC;IACzB,QAAQ,EAAE,OAAO,CAAC;IAClB,wBAAwB,EAAE,KAAK,CAAC;IAChC,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAAE,CAAC;IAC1K,SAAS,EAAE;QACT,CAAC,EAAE,MAAM,CAAC;QAAC,mBAAmB,EAAE,uBAAuB,CAAC;QAAC,eAAe,EAAE,uBAAuB,CAAC;QAClG,wBAAwB,EAAE,uBAAuB,CAAC;QAAC,aAAa,EAAE,+BAA+B,CAAC;QAClG,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAAC,uBAAuB,EAAE,MAAM,CAAC;QAAC,oBAAoB,EAAE,MAAM,CAAC;QAAC,mBAAmB,EAAE,MAAM,CAAC;KAC9H,CAAC;IACF,SAAS,EAAE;QACT,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAAC,kBAAkB,EAAE,uBAAuB,CAAC;QAAC,cAAc,EAAE,uBAAuB,CAAC;QACvH,QAAQ,EAAE,uBAAuB,CAAC;QAAC,yBAAyB,EAAE,uBAAuB,CAAC;QACtF,gBAAgB,EAAE,MAAM,CAAC;QAAC,gBAAgB,EAAE,MAAM,CAAC;QAAC,qBAAqB,EAAE,MAAM,CAAC;KACnF,CAAC;IACF,SAAS,EAAE,+BAA+B,CAAC;IAC3C,kBAAkB,EAAE,OAAO,CAAC;IAC5B,aAAa,EAAE,OAAO,CAAC;IACvB,KAAK,EAAE,aAAa,CAAC;IACrB,UAAU,EAAE;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAC;QAAC,eAAe,EAAE,MAAM,CAAC;KAAE,CAAC;CACtF;AACD,MAAM,MAAM,yBAAyB,GACjC;IAAE,KAAK,EAAE,IAAI,CAAC;IAAC,MAAM,EAAE,QAAQ,CAAC,yBAAyB,CAAC,CAAC;IAAC,WAAW,EAAE,SAAS,EAAE,CAAA;CAAE,GACtF;IAAE,KAAK,EAAE,KAAK,CAAC;IAAC,WAAW,EAAE,SAAS,6BAA6B,EAAE,CAAA;CAAE,CAAC;AAC5E,MAAM,WAAW,kCAAkC;IACjD,KAAK,EAAE,OAAO,CAAC;IACf,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,YAAY,EAAE,OAAO,CAAC;IACtB,2GAA2G;IAC3G,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CAC7B;AAED,oGAAoG;AACpG,wBAAgB,mCAAmC,CAAC,KAAK,EAAE,wBAAwB,GAAG,MAAM,CAI3F;AAiBD,wFAAwF;AACxF,wBAAgB,gCAAgC,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,CAAC,EAAE,gBAAgB,GAAG,SAAS,6BAA6B,EAAE,CAwCtI;AAcD,+FAA+F;AAC/F,wBAAgB,2BAA2B,CAAC,OAAO,EAAE,kCAAkC,GAAG,yBAAyB,CAalH"}

package/dist/discovery/evaluation.js

@@ -0,0 +1,310 @@
+import { buildRegistry } from "../registry.js";
+import { validateRuntimeContext } from "../runtime/context.js";
+import { deepFreeze } from "../runtime/bounded-json.js";
+import { aggregateProviderUsage, usageQuantities, USAGE_QUANTITIES } from "../runtime/provider-usage.js";
+import { validateManifests } from "../validator.js";
+import { captureEligibility } from "./bindings.js";
+import { buildCapabilitySearchCorpus, registrySourceFingerprint } from "./corpus.js";
+import { resolveDiscoveryConfiguration } from "./configuration.js";
+import { DiscoveryError } from "./diagnostics.js";
+import { hashDiscoveryContent } from "./fingerprint.js";
+import { evaluationObservationsSnapshot, evaluationSuiteSnapshot } from "./evaluation-validation.js";
+import { retrievalPolicyFingerprint } from "./retrieval-policy.js";
+/** Includes authored cases, labels, partitions and capability content; excludes no suite fields. */
+export function fingerprintDiscoveryEvaluationSuite(value) {
+    const suite = evaluationSuiteSnapshot(value);
+    if (!suite)
+        throw new DiscoveryError({ code: "invalid_input", message: "A valid discovery evaluation suite is required." });
+    return hashDiscoveryContent({ kind: "discovery_evaluation_suite", version: "1.0", suite });
+}
+function loadedSuite(suite) {
+    return suite.capabilities.map(manifest => ({ kind: "capability", path: `synthetic/${manifest.id}.yaml`,
+        absolutePath: `/synthetic/${manifest.id}.yaml`, manifest: manifest }));
+}
+function routeFor(suite, entry, registry) {
+    const context = suite.contexts[entry.context_id];
+    const runtimeContext = context.runtime_context;
+    validateRuntimeContext(runtimeContext);
+    if (!Array.isArray(runtimeContext.subject.permissions) || !runtimeContext.subject.permissions.every(x => typeof x === "string")
+        || !runtimeContext.autonomy || !["low", "medium", "high", "critical"].includes(runtimeContext.autonomy.maxRiskTier))
+        throw new Error("context");
+    return { ...context.eligibility, registry, userInput: { text: entry.fragments.map(fragment => fragment.text).join("\n") },
+        builtContext: { runtimeContext, items: [], modelContextText: "", redactions: [], warnings: [] } };
+}
+/** Semantic fixture checks used by the CLI without executing retrieval or providers. */
+export function validateDiscoveryEvaluationSuite(value, registry) {
+    const suite = evaluationSuiteSnapshot(value);
+    if (!suite)
+        return [{ code: "invalid_suite_schema" }];
+    const errors = [];
+    const add = (code, caseId) => { if (errors.length < 64)
+        errors.push({ code, ...(caseId ? { caseId } : {}) }); };
+    try {
+        const definitions = loadedSuite(suite);
+        if (!validateManifests(definitions).valid)
+            return [{ code: "invalid_suite_capabilities" }];
+        const authoredRegistry = buildRegistry(definitions);
+        if (registry && registrySourceFingerprint(registry) !== registrySourceFingerprint(authoredRegistry))
+            return [{ code: "suite_registry_mismatch" }];
+        registry ??= authoredRegistry;
+        for (const context of Object.values(suite.contexts)) {
+            try {
+                validateRuntimeContext(context.runtime_context);
+            }
+            catch {
+                add("invalid_context");
+            }
+            for (const id of [...context.eligibility?.includeCapabilityIds ?? [], ...context.eligibility?.excludeCapabilityIds ?? []]) {
+                if (!registry.capabilityById.has(id))
+                    add("unknown_eligibility_capability");
+            }
+        }
+        const ids = new Set(), families = new Map();
+        for (const entry of suite.cases) {
+            if (ids.has(entry.id))
+                add("duplicate_case", entry.id);
+            ids.add(entry.id);
+            if (families.has(entry.family_id) && families.get(entry.family_id) !== entry.partition)
+                add("family_partition_leak", entry.id);
+            families.set(entry.family_id, entry.partition);
+            if (!Object.hasOwn(suite.contexts, entry.context_id)) {
+                add("unknown_context", entry.id);
+                continue;
+            }
+            if (entry.compatible_ids.some(id => entry.forbidden_ids.includes(id))
+                || entry.compatible_ids.length && entry.forbidden_operations.includes(entry.expected_operation)
+                || entry.cohorts.includes("true_no_match") && entry.compatible_ids.length)
+                add("contradictory_labels", entry.id);
+            let eligible;
+            try {
+                eligible = new Set(captureEligibility(routeFor(suite, entry, registry)).eligible.map(x => x.manifest.id));
+            }
+            catch {
+                add("invalid_context", entry.id);
+                continue;
+            }
+            for (const id of [...entry.compatible_ids, ...entry.forbidden_ids]) {
+                const capability = registry.capabilityById.get(id)?.manifest;
+                if (!capability) {
+                    add("unknown_capability", entry.id);
+                    continue;
+                }
+                if (entry.compatible_ids.includes(id) && (!eligible.has(id) || operationFor(capability) !== entry.expected_operation
+                    || capability.lifecycle.commit || capability.capability_type === "write_commit"))
+                    add("invalid_compatible_capability", entry.id);
+            }
+        }
+    }
+    catch {
+        add("invalid_suite_bindings");
+    }
+    return errors;
+}
+const operationFor = (capability) => capability.lifecycle.commit
+    || capability.capability_type === "write_commit" ? "commit" : capability.lifecycle.prepare ? "prepare" : "read";
+const rate = (numerator, denominator) => ({ numerator, denominator, value: denominator ? numerator / denominator : null });
+function distribution(values) {
+    const sorted = [...values].sort((a, b) => a - b);
+    let mean = 0;
+    values.forEach((value, index) => { mean += (value - mean) / (index + 1); });
+    return { count: values.length, minimum: sorted[0] ?? null, maximum: sorted.at(-1) ?? null,
+        mean: values.length ? mean : null,
+        p95: sorted[Math.ceil(sorted.length * 0.95) - 1] ?? null };
+}
+/** Scores observations only. It never calls a ranker, model, embedding client, or executor. */
+export function evaluateCapabilityDiscovery(request) {
+    const errors = [...validateDiscoveryEvaluationSuite(request.suite, request.registry)];
+    const suite = evaluationSuiteSnapshot(request.suite);
+    const observations = evaluationObservationsSnapshot(request.observations);
+    if (!observations)
+        errors.push({ code: "invalid_observations_schema" });
+    const caseIds = request.caseIds;
+    if (caseIds !== undefined && (!Array.isArray(caseIds) || caseIds.length > (suite?.cases.length ?? 0) || !Array.from(caseIds).every(id => typeof id === "string")
+        || new Set(caseIds).size !== caseIds.length || caseIds.some(id => !suite?.cases.some(entry => entry.id === id)))) {
+        errors.push({ code: "invalid_evaluation_scope" });
+    }
+    if (errors.length || !suite || !observations)
+        return { valid: false, diagnostics: deepFreeze(errors) };
+    try {
+        return evaluate(suite, observations, request.registry, caseIds);
+    }
+    catch {
+        return { valid: false, diagnostics: [{ code: "invalid_evaluation_bindings" }] };
+    }
+}
+function evaluate(suite, observations, registry, caseIds) {
+    const errors = [];
+    const add = (code, caseId) => { if (errors.length < 64)
+        errors.push({ code, ...(caseId ? { caseId } : {}) }); };
+    const configuration = resolveDiscoveryConfiguration(observations.configuration);
+    if (hashDiscoveryContent(configuration) !== hashDiscoveryContent(observations.configuration))
+        add("unresolved_configuration");
+    const corpus = buildCapabilitySearchCorpus(registry);
+    const suiteFingerprint = fingerprintDiscoveryEvaluationSuite(suite);
+    if (observations.suite_id !== suite.id || observations.suite_fingerprint !== suiteFingerprint
+        || observations.registry_source_fingerprint !== corpus.registrySourceFingerprint || observations.corpus_fingerprint !== corpus.corpusFingerprint
+        || observations.retrieval_policy_fingerprint !== retrievalPolicyFingerprint(configuration, observations.custom_ranker_version))
+        add("observation_binding_mismatch");
+    const definitions = new Map(suite.cases.map(entry => [entry.id, entry]));
+    const scope = new Set(caseIds ?? suite.cases.map(entry => entry.id));
+    const expectedCases = suite.cases.filter(entry => scope.has(entry.id));
+    const scopeFingerprint = hashDiscoveryContent({ kind: "discovery_evaluation_scope", version: "1.0", suiteFingerprint,
+        caseIds: [...scope].sort() });
+    const seen = new Set();
+    const retrievalOutcomes = { matched: 0, no_match: 0, unavailable: 0, cancelled: 0 };
+    const selectionOutcomes = { selected: 0, abstained: 0, unavailable: 0, invalid_output: 0, cancelled: 0, not_performed: 0 };
+    const byPartition = Object.create(null), byDomain = Object.create(null), byCohort = Object.create(null);
+    let hits = 0, topHits = 0, nonempty = 0, noMatchFalsePositives = 0;
+    let forbiddenCandidates = 0, policyCandidateCases = 0, readMutationCases = 0;
+    let selectedMatched = 0, correctSelected = 0, covered = 0, answerableMatched = 0, submitted = 0;
+    let forbiddenChoices = 0, policyChoices = 0, readMutationChoices = 0;
+    let noMatchSelections = 0;
+    const sizes = [], latencies = [];
+    for (const observed of observations.cases) {
+        const entry = definitions.get(observed.case_id);
+        if (!entry) {
+            add("foreign_observation", observed.case_id);
+            continue;
+        }
+        if (!scope.has(entry.id)) {
+            add("observation_outside_scope", entry.id);
+            continue;
+        }
+        if (seen.has(entry.id)) {
+            add("duplicate_observation", entry.id);
+            continue;
+        }
+        seen.add(entry.id);
+        const eligibility = captureEligibility(routeFor(suite, entry, registry));
+        const eligible = new Set(eligibility.eligible.map(x => x.manifest.id));
+        if (observed.eligibility_fingerprint !== eligibility.fingerprint)
+            add("eligibility_binding_mismatch", entry.id);
+        if ((observed.status === "matched") !== !!observed.candidates.length || observed.candidates.length > configuration.limits.maxCandidates
+            || (observed.status === "unavailable" || observed.status === "cancelled") !== (observed.actual_mode === null)
+            || configuration.mode === "lexical" && observed.actual_mode === "hybrid"
+            || configuration.mode === "hybrid" && observed.actual_mode === "lexical" && configuration.failurePolicy !== "lexical_fallback"
+            || observed.actual_mode === "hybrid" && observed.status === "matched" && !observed.index_fingerprint)
+            add("invalid_retrieval_observation", entry.id);
+        const candidateIds = new Set();
+        let forbidden = false, policy = false, readMutation = false;
+        for (const candidate of observed.candidates) {
+            const manifest = registry.capabilityById.get(candidate.capability_id)?.manifest;
+            if (!manifest || manifest.version !== candidate.capability_version || candidateIds.has(candidate.capability_id)) {
+                add("invalid_candidate_binding", entry.id);
+                continue;
+            }
+            candidateIds.add(candidate.capability_id);
+            const operation = operationFor(manifest);
+            forbidden ||= entry.forbidden_ids.includes(manifest.id) || entry.forbidden_operations.includes(operation);
+            policy ||= !eligible.has(manifest.id) || operation === "commit";
+            readMutation ||= entry.expected_operation === "read" && operation !== "read";
+        }
+        if (forbidden)
+            forbiddenCandidates++;
+        if (policy)
+            policyCandidateCases++;
+        if (readMutation)
+            readMutationCases++;
+        retrievalOutcomes[observed.status]++;
+        if (observed.candidates.length)
+            nonempty++;
+        if (entry.compatible_ids.some(id => candidateIds.has(id)))
+            hits++;
+        if (observed.candidates[0] && entry.compatible_ids.includes(observed.candidates[0].capability_id))
+            topHits++;
+        if (entry.cohorts.includes("true_no_match") && candidateIds.size)
+            noMatchFalsePositives++;
+        sizes.push(observed.candidates.length);
+        latencies.push(observed.latency_ms);
+        byPartition[entry.partition] = (byPartition[entry.partition] ?? 0) + 1;
+        byDomain[entry.domain] = (byDomain[entry.domain] ?? 0) + 1;
+        for (const cohort of entry.cohorts)
+            byCohort[cohort] = (byCohort[cohort] ?? 0) + 1;
+        const selection = observed.selection;
+        const answerable = entry.compatible_ids.length > 0;
+        if (answerable && observed.status === "matched")
+            answerableMatched++;
+        const status = selection?.status ?? "not_performed";
+        selectionOutcomes[status]++;
+        if (status !== "not_performed") {
+            submitted++;
+            if (observed.status !== "matched")
+                add("selection_without_shortlist", entry.id);
+        }
+        if (selection?.status === "selected") {
+            const manifest = registry.capabilityById.get(selection.capability_id)?.manifest;
+            if (!manifest || !candidateIds.has(manifest.id) || manifest.version !== selection.capability_version
+                || operationFor(manifest) !== selection.operation) {
+                add("invalid_selection_binding", entry.id);
+                continue;
+            }
+            if (answerable) {
+                selectedMatched++;
+                covered++;
+                if (entry.compatible_ids.includes(manifest.id))
+                    correctSelected++;
+            }
+            if (entry.cohorts.includes("true_no_match"))
+                noMatchSelections++;
+            if (entry.forbidden_ids.includes(manifest.id) || entry.forbidden_operations.includes(selection.operation))
+                forbiddenChoices++;
+            if (!eligible.has(manifest.id) || selection.operation === "commit")
+                policyChoices++;
+            if (entry.expected_operation === "read" && selection.operation !== "read")
+                readMutationChoices++;
+        }
+    }
+    const attempts = new Map();
+    for (const attempt of observations.attempts) {
+        const key = JSON.stringify([attempt.run_id, attempt.attempt_id]);
+        const previous = attempts.get(key);
+        if (previous && hashDiscoveryContent(previous) !== hashDiscoveryContent(attempt))
+            add("conflicting_attempt_receipt");
+        attempts.set(key, attempt);
+        if (attempt.completed_responses > attempt.provider_attempts || attempt.retry_attempts > attempt.provider_attempts
+            || attempt.repair_attempts > attempt.provider_attempts)
+            add("invalid_attempt_counts");
+    }
+    if (errors.length)
+        return { valid: false, diagnostics: deepFreeze(errors) };
+    const stages = [...attempts.values()].map(attempt => ({
+        attemptId: JSON.stringify([attempt.run_id, attempt.attempt_id]), stage: attempt.stage, configuration: attempt.configuration,
+        mock: attempt.mock, pending: attempt.pending, outcome: attempt.outcome, cacheHits: 0,
+        providerAttempts: attempt.provider_attempts, completedResponses: attempt.completed_responses,
+        retryAttempts: attempt.retry_attempts, repairAttempts: attempt.repair_attempts, elapsedMs: attempt.elapsed_ms,
+        ...usageQuantities(attempt.usage, attempt.provider_attempts === 0)
+    }));
+    const elapsed = stages.reduce((sum, stage) => sum + stage.elapsedMs, 0);
+    if (!Number.isFinite(elapsed)
+        || ["providerAttempts", "completedResponses", "retryAttempts", "repairAttempts"].some(key => !Number.isSafeInteger(stages.reduce((sum, stage) => sum + stage[key], 0)))
+        || USAGE_QUANTITIES.some(key => {
+            const total = stages.reduce((sum, stage) => sum + stage[key].knownSubtotal, 0);
+            return !Number.isFinite(total) || key !== "estimatedCost" && !Number.isSafeInteger(total);
+        }))
+        return { valid: false, diagnostics: [{ code: "invalid_accounting_totals" }] };
+    let usage = aggregateProviderUsage(stages, elapsed);
+    if (!observations.usage_complete) {
+        const totals = { ...usage.totals };
+        for (const key of USAGE_QUANTITIES)
+            totals[key] = { total: null, complete: false, knownSubtotal: totals[key].knownSubtotal };
+        usage = deepFreeze({ stages: usage.stages, totals });
+    }
+    // Missing cases stay in recall/no-match denominators; partial runs cannot inflate them.
+    const answerable = expectedCases.filter(entry => entry.compatible_ids.length).length;
+    const trueNoMatch = expectedCases.filter(entry => entry.cohorts.includes("true_no_match")).length;
+    return { valid: true, diagnostics: [], report: deepFreeze({
+            suiteFingerprint, scopeFingerprint, complete: seen.size === expectedCases.length, semanticQualityQualified: false,
+            counts: { expected: expectedCases.length, observed: seen.size, missing: expectedCases.length - seen.size, byPartition, byDomain, byCohort },
+            retrieval: { k: configuration.limits.maxCandidates, compatibleRecallAtK: rate(hits, answerable), topOnePrecision: rate(topHits, nonempty),
+                noMatchFalsePositiveRate: rate(noMatchFalsePositives, trueNoMatch), shortlistSize: distribution(sizes), outcomes: retrievalOutcomes,
+                forbiddenCandidateCases: forbiddenCandidates, policyViolationCases: policyCandidateCases, readToMutationCases: readMutationCases },
+            selection: { outcomes: selectionOutcomes, compatibleAccuracy: rate(correctSelected, selectedMatched),
+                abstentionRate: rate(selectionOutcomes.abstained, submitted), coverage: rate(covered, answerableMatched),
+                noMatchFalseSelectionRate: rate(noMatchSelections, trueNoMatch),
+                forbiddenChoices, policyViolations: policyChoices, readToMutationChoices: readMutationChoices },
+            latencyMs: distribution(latencies), accountingComplete: observations.usage_complete,
+            usageComplete: observations.usage_complete && stages.every(stage => !stage.pending) && USAGE_QUANTITIES.every(key => usage.totals[key].complete), usage,
+            provenance: { mockAttempts: stages.filter(stage => stage.mock).reduce((sum, x) => sum + x.providerAttempts, 0),
+                liveAttempts: stages.filter(stage => !stage.mock).reduce((sum, x) => sum + x.providerAttempts, 0), pendingAttempts: stages.filter(stage => stage.pending).length }
+        }) };
+}

package/dist/discovery/export.d.ts

@@ -0,0 +1,14 @@
+import type { CapabilityRouteRequest } from "../runtime/types.js";
+import type { CapabilityDiscoveryResult, CapabilitySelectionBatchResult, DiscoveryRouteResult } from "./types.js";
+export declare function validateRouterLimit(limit: number | undefined): void;
+/** A task-local selector decision narrows an already validated shortlist. */
+export declare function exportSelectedResult(request: CapabilityRouteRequest & {
+    signal?: AbortSignal;
+    deadlineAt?: number;
+}, original: CapabilityDiscoveryResult, batch: CapabilitySelectionBatchResult, source: string, current?: boolean): DiscoveryRouteResult;
+/** Shared final gate for fresh retrieval and request-bound shortlist reuse. */
+export declare function exportDiscoveryResult(request: CapabilityRouteRequest & {
+    signal?: AbortSignal;
+    deadlineAt?: number;
+}, original: CapabilityDiscoveryResult, sourceFingerprint: string, bindingCurrent?: boolean): DiscoveryRouteResult;
+//# sourceMappingURL=export.d.ts.map

package/dist/discovery/export.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"export.d.ts","sourceRoot":"","sources":["../../src/discovery/export.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAKlE,OAAO,KAAK,EAAE,yBAAyB,EAAE,8BAA8B,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAElH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,CAInE;AAED,6EAA6E;AAC7E,wBAAgB,oBAAoB,CAClC,OAAO,EAAE,sBAAsB,GAAG;IAAE,MAAM,CAAC,EAAE,WAAW,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAA;CAAE,EAC/E,QAAQ,EAAE,yBAAyB,EAAE,KAAK,EAAE,8BAA8B,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,UAAO,GACzG,oBAAoB,CAetB;AAED,+EAA+E;AAC/E,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,sBAAsB,GAAG;IAAE,MAAM,CAAC,EAAE,WAAW,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAA;CAAE,EAC/E,QAAQ,EAAE,yBAAyB,EACnC,iBAAiB,EAAE,MAAM,EACzB,cAAc,UAAO,GACpB,oBAAoB,CAkDtB"}

package/dist/discovery/export.js

@@ -0,0 +1,81 @@
+import { captureEligibility } from "./bindings.js";
+import { registrySourceFingerprint } from "./corpus.js";
+import { discoveryStopReason } from "./discoverer.js";
+import { DiscoveryError } from "./diagnostics.js";
+export function validateRouterLimit(limit) {
+    if (limit !== undefined && (!Number.isInteger(limit) || limit < 1 || limit > 20)) {
+        throw new DiscoveryError({ code: "invalid_input", message: "maxCapabilities must be an integer from 1 through 20." });
+    }
+}
+/** A task-local selector decision narrows an already validated shortlist. */
+export function exportSelectedResult(request, original, batch, source, current = true) {
+    const exported = exportDiscoveryResult(request, original, source, current);
+    const outcome = batch.outcomes.find(item => item.queryId === original.queryId);
+    const selected = outcome?.status === "selected" && exported.discovery[0]?.status === "matched"
+        && request.registry.capabilityById.get(outcome.capabilityId)?.manifest.version === outcome.capabilityVersion
+        ? outcome.capabilityId : undefined;
+    const items = exported.slice.items.filter(item => item.capabilityId === selected).map(item => ({ ...item,
+        reasons: ["Selected for this task by the explicitly configured selector; execution authorization remains required."] }));
+    return { ...exported, selection: batch.outcomes, selectionBatch: batch, slice: {
+            items, excluded: [...exported.slice.excluded, ...exported.slice.items.filter(item => item.capabilityId !== selected)
+                    .map(item => ({ capabilityId: item.capabilityId, reason: "Capability was not selected for this task." }))],
+            warnings: items.length ? exported.slice.warnings : [...exported.slice.warnings, {
+                    code: "selection_" + (outcome?.status ?? "not_submitted"), message: "Selection did not provide a current actionable capability for this task."
+                }]
+        } };
+}
+/** Shared final gate for fresh retrieval and request-bound shortlist reuse. */
+export function exportDiscoveryResult(request, original, sourceFingerprint, bindingCurrent = true) {
+    let result = original;
+    let eligibility;
+    let current = false;
+    try {
+        eligibility = captureEligibility(request);
+        current = bindingCurrent && eligibility.fingerprint === result.eligibilityFingerprint
+            && registrySourceFingerprint(request.registry) === sourceFingerprint;
+    }
+    catch { /* Invalidated live registries cannot yield a tool slice. */ }
+    const stop = discoveryStopReason(request);
+    if (stop || (!current && result.status === "matched")) {
+        const code = stop ?? "stale_binding";
+        result = {
+            ...result, status: stop ? "cancelled" : "unavailable", actualMode: null, candidates: [],
+            diagnostics: [{ code, queryId: result.queryId, message: stop
+                        ? "Discovery was cancelled or its deadline elapsed." : "Discovery bindings changed before tool export." }]
+        };
+        delete result.indexFingerprint;
+    }
+    const candidates = result.status === "matched" ? result.candidates.slice(0, request.maxCapabilities ?? result.candidates.length) : [];
+    const selected = new Set(candidates.map(candidate => candidate.capabilityId));
+    const excluded = [...(eligibility?.excluded ?? [])];
+    for (const entry of eligibility?.eligible ?? []) {
+        if (!selected.has(entry.manifest.id))
+            excluded.push({
+                capabilityId: entry.manifest.id, reason: "Capability was not included in the retrieval shortlist."
+            });
+    }
+    const warnings = result.status === "matched" ? [] : [{
+            code: "discovery_" + result.status,
+            message: result.status === "no_match" ? "Discovery found no eligible candidates meeting the retrieval policy."
+                : result.status === "cancelled" ? "Discovery was cancelled or its deadline elapsed."
+                    : "Discovery is unavailable; no tools were shortlisted."
+        }];
+    if (result.actualMode === "lexical" && result.degradedReason)
+        warnings.push({
+            code: "discovery_lexical_fallback", message: "Discovery used the configured whole-query lexical fallback policy."
+        });
+    return {
+        discovery: [result],
+        slice: {
+            items: candidates.map(candidate => {
+                const manifest = request.registry.capabilityById.get(candidate.capabilityId).manifest;
+                return {
+                    capabilityId: candidate.capabilityId, score: candidate.scores.final,
+                    exposedOperations: manifest.lifecycle.prepare ? ["select", "prepare"] : ["select"],
+                    reasons: ["Included by the retrieval policy; ordinary tool selection is required."]
+                };
+            }),
+            excluded, warnings
+        }
+    };
+}

package/dist/discovery/facade.d.ts

@@ -0,0 +1,82 @@
+import type { AicfProviderRun } from "../runtime/provider-types.js";
+import type { AicfCapabilityRouter, AicfContextBuilder, AicfRuntimeContext, CapabilityRouteRequest } from "../runtime/types.js";
+import type { ManifestRegistry } from "../types.js";
+import type { DiscoveryBudgetScope } from "./budget.js";
+import type { EmbeddingExecutionOptions, EmbeddingIndexLimits } from "./embedding-types.js";
+import type { CapabilitySelectRequest } from "./selection-types.js";
+import type { CapabilityDiscoveryResult, CapabilityLexicalRanker, CapabilitySelectionBatchResult, DiscoveryLimits, DiscoveryRouteResult, EmbeddingClient, HybridDiscoveryConfiguration } from "./types.js";
+export declare const DISCOVERY_STANDARD_PROFILE: {
+    policyVersion: string;
+    timeoutMs: number;
+    weights: {
+        semantic: number;
+        lexical: number;
+    };
+    failurePolicy: "unavailable";
+    indexLimits: {
+        maxArtifactBytes: number;
+        maxEntries: number;
+        maxVectorDimension: number;
+    };
+    maxCharactersPerText: number;
+    maxCharactersPerCall: number;
+};
+/** @inline */
+interface CapabilityDiscoveryOptionsBase {
+    registry: ManifestRegistry;
+    contextBuilder?: AicfContextBuilder;
+    ranker?: CapabilityLexicalRanker;
+    policyVersion?: string;
+    timeoutMs?: number;
+    limits?: Partial<DiscoveryLimits>;
+    aliasWeight?: number;
+    minRetrievalScore?: number;
+}
+export interface CapabilityDiscoveryEmbeddingOptions {
+    client: EmbeddingClient;
+    index: unknown;
+    maxInputTokens: number;
+    indexLimits?: Partial<EmbeddingIndexLimits>;
+    maxCharactersPerText?: number;
+    maxCharactersPerCall?: number;
+    maxEstimatedCostPerCall?: number;
+    estimateCost?: EmbeddingExecutionOptions["estimateCost"];
+}
+export type CapabilityDiscoveryOptions = CapabilityDiscoveryOptionsBase & ({
+    mode?: "lexical";
+    embedding?: never;
+    weights?: never;
+    failurePolicy?: never;
+} | {
+    mode: "hybrid";
+    embedding: CapabilityDiscoveryEmbeddingOptions;
+    weights?: HybridDiscoveryConfiguration["weights"];
+    failurePolicy?: HybridDiscoveryConfiguration["failurePolicy"];
+});
+export type CapabilityDiscoveryEligibility = Omit<CapabilityRouteRequest, "registry" | "builtContext" | "userInput" | "controls">;
+export interface CapabilitySearchRequest {
+    providerRun?: AicfProviderRun;
+    tasks: readonly {
+        id: string;
+        text: string;
+    }[];
+    runtimeContext: AicfRuntimeContext;
+    controls?: CapabilityRouteRequest["controls"];
+    eligibility?: CapabilityDiscoveryEligibility;
+    signal?: AbortSignal;
+    deadlineAt?: number;
+    budget?: DiscoveryBudgetScope;
+}
+export interface CapabilityResultRouter extends AicfCapabilityRouter {
+    routeWithDiagnostics(request: CapabilityRouteRequest): DiscoveryRouteResult;
+}
+export interface CapabilityDiscovery {
+    search(request: CapabilitySearchRequest): Promise<readonly CapabilityDiscoveryResult[]>;
+    select(request: CapabilitySelectRequest): Promise<CapabilitySelectionBatchResult>;
+    routerFor(result: CapabilityDiscoveryResult): CapabilityResultRouter;
+    routerFor(batch: CapabilitySelectionBatchResult, queryId: string): CapabilityResultRouter;
+}
+/** Simple application-driven search. Model choice and execution stay in the ordinary runtime. */
+export declare function createCapabilityDiscovery(options: CapabilityDiscoveryOptions): CapabilityDiscovery;
+export {};
+//# sourceMappingURL=facade.d.ts.map

package/dist/discovery/facade.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"facade.d.ts","sourceRoot":"","sources":["../../src/discovery/facade.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AAGpE,OAAO,KAAK,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAChI,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AACpD,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAMxD,OAAO,KAAK,EAAE,yBAAyB,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAK5F,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAGpE,OAAO,KAAK,EACV,yBAAyB,EAAE,uBAAuB,EAAE,8BAA8B,EAClF,eAAe,EAAE,oBAAoB,EAAE,eAAe,EAAE,4BAA4B,EACrF,MAAM,YAAY,CAAC;AAEpB,eAAO,MAAM,0BAA0B;;;;;;;;;;;;;;;CAKrC,CAAC;AAEH,cAAc;AACd,UAAU,8BAA8B;IACtC,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,cAAc,CAAC,EAAE,kBAAkB,CAAC;IACpC,MAAM,CAAC,EAAE,uBAAuB,CAAC;IACjC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,CAAC;IAClC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,MAAM,WAAW,mCAAmC;IAClD,MAAM,EAAE,eAAe,CAAC;IACxB,KAAK,EAAE,OAAO,CAAC;IACf,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,CAAC,EAAE,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAC5C,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,YAAY,CAAC,EAAE,yBAAyB,CAAC,cAAc,CAAC,CAAC;CAC1D;AAED,MAAM,MAAM,0BAA0B,GAAG,8BAA8B,GAAG,CACtE;IAAE,IAAI,CAAC,EAAE,SAAS,CAAC;IAAC,SAAS,CAAC,EAAE,KAAK,CAAC;IAAC,OAAO,CAAC,EAAE,KAAK,CAAC;IAAC,aAAa,CAAC,EAAE,KAAK,CAAA;CAAE,GAC/E;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,SAAS,EAAE,mCAAmC,CAAC;IAC/D,OAAO,CAAC,EAAE,4BAA4B,CAAC,SAAS,CAAC,CAAC;IAAC,aAAa,CAAC,EAAE,4BAA4B,CAAC,eAAe,CAAC,CAAA;CAAE,CACvH,CAAC;AAEF,MAAM,MAAM,8BAA8B,GAAG,IAAI,CAAC,sBAAsB,EAAE,UAAU,GAAG,cAAc,GAAG,WAAW,GAAG,UAAU,CAAC,CAAC;AAClI,MAAM,WAAW,uBAAuB;IACtC,WAAW,CAAC,EAAE,eAAe,CAAC;IAC9B,KAAK,EAAE,SAAS;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAC/C,cAAc,EAAE,kBAAkB,CAAC;IACnC,QAAQ,CAAC,EAAE,sBAAsB,CAAC,UAAU,CAAC,CAAC;IAC9C,WAAW,CAAC,EAAE,8BAA8B,CAAC;IAC7C,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,oBAAoB,CAAC;CAC/B;AAED,MAAM,WAAW,sBAAuB,SAAQ,oBAAoB;IAClE,oBAAoB,CAAC,OAAO,EAAE,sBAAsB,GAAG,oBAAoB,CAAC;CAC7E;AAED,MAAM,WAAW,mBAAmB;IAClC,MAAM,CAAC,OAAO,EAAE,uBAAuB,GAAG,OAAO,CAAC,SAAS,yBAAyB,EAAE,CAAC,CAAC;IACxF,MAAM,CAAC,OAAO,EAAE,uBAAuB,GAAG,OAAO,CAAC,8BAA8B,CAAC,CAAC;IAClF,SAAS,CAAC,MAAM,EAAE,yBAAyB,GAAG,sBAAsB,CAAC;IACrE,SAAS,CAAC,KAAK,EAAE,8BAA8B,EAAE,OAAO,EAAE,MAAM,GAAG,sBAAsB,CAAC;CAC3F;AA4BD,iGAAiG;AACjG,wBAAgB,yBAAyB,CAAC,OAAO,EAAE,0BAA0B,GAAG,mBAAmB,CAsMlG"}