@shrkcrft/inspector 0.1.0-alpha.30 → 0.1.0-alpha.31

package/dist/test-runner.js

@@ -1,64 +1,142 @@
 import { existsSync } from 'node:fs';
 import * as nodePath from 'node:path';
 import { buildContext } from '@shrkcrft/context';
-import { listConstructs } from "./construct-registry.js";
-import { listPolicyIds } from "./policy-registry.js";
-import { listPlaybooks } from "./playbook-registry.js";
 import { buildTaskPacket } from "./task-packet.js";
 import { rankKnowledgeEntries } from "./task-ranker.js";
 import { contextTuningBoostFor } from "./context-tuning.js";
 import { buildProjectOverview, renderOverviewText } from "./project-overview.js";
-import { HELPERS } from "./helper-registry.js";
-import { importModuleViaLoader } from '@shrkcrft/core';
+import { importModuleViaLoader, RejectionCause } from '@shrkcrft/core';
+import { CommandResolutionStatus } from "./command-resolution-status.js";
+import { COMMAND_INDEX_NOT_INJECTED, isCacheBackedKind, isCommandResolved, isReferenceCacheWarm, referenceIdExists, referenceIdsFor, resolveShrkCommandReference, warmReferenceRegistries, } from "./reference-registry.js";
+/** The `list` verb that prints each registry — named in every diagnostic. */
+const LIST_VERBS = Object.freeze({
+    template: 'shrk templates list',
+    rule: 'shrk rules list',
+    pipeline: 'shrk pipelines list',
+    helper: 'shrk helper list',
+    playbook: 'shrk playbooks list',
+    policy: 'shrk policy list',
+    construct: 'shrk constructs list',
+    knowledge: 'shrk knowledge list',
+    command: 'shrk surface list',
+});
+function consulted(inspection, kind) {
+    return {
+        kind,
+        listVerb: LIST_VERBS[kind] ?? `shrk ${kind} list`,
+        size: kind === 'command' ? 0 : referenceIdsFor(inspection, kind).length,
+    };
+}
+/**
+ * THE existence answer for a test expectation — the shared reference registry,
+ * never a private set. A cache-backed kind read before the registries were
+ * warmed is `unverifiable`, not "your correct id does not exist".
+ */
+function existence(inspection, kind, id) {
+    if (referenceIdExists(inspection, kind, id))
+        return 'exists';
+    if (isCacheBackedKind(kind) && !isReferenceCacheWarm(inspection))
+        return 'unverifiable';
+    return 'missing';
+}
+/** A test file's default array; `error` (first line) when the import threw. A missing file is `[]`. */
 async function importDefaultArray(absPath) {
     if (!existsSync(absPath))
-        return [];
+        return { items: [] };
     try {
         const mod = (await importModuleViaLoader(absPath));
-        return Array.isArray(mod.default) ? mod.default : [];
+        return { items: Array.isArray(mod.default) ? mod.default : [] };
     }
-    catch {
-        return [];
+    catch (e) {
+        return { items: [], error: (e.message ?? String(e)).split('\n')[0].trim() };
     }
 }
 /**
- * Load context tests from local sharkcraft config + any pack contributions.
+ * THE context / agent-contract test acceptance predicate (round 12, 12.1): a
+ * non-empty string `id` and a string `task` — `[]` means accepted. Tests used
+ * to load with no per-entry validation (a test without a `task` ran against
+ * `undefined`), and a file that failed to import became `[]` in silence.
  */
-export async function loadContextTests(inspection) {
+export function testDefinitionRejectionReasons(raw) {
+    if (!raw || typeof raw !== 'object' || Array.isArray(raw))
+        return ['(entry): must be an object'];
+    const t = raw;
+    const out = [];
+    if (typeof t.id !== 'string' || t.id.length === 0)
+        out.push('id: must be a non-empty string');
+    if (typeof t.task !== 'string')
+        out.push('task: must be a string');
+    return out;
+}
+async function loadTestDefinitions(inspection, localName, slot) {
     const tests = [];
+    const entries = [];
+    const issues = [];
+    const rejected = [];
+    const readInto = async (file, packageName) => {
+        const r = await importDefaultArray(file);
+        const rel = nodePath.relative(inspection.projectRoot, file) || file;
+        if (r.error !== undefined) {
+            issues.push({
+                severity: 'warning',
+                code: 'load-failed',
+                message: `${packageName ? `Pack ${packageName} (${rel})` : `Failed to load ${rel}`}: ${r.error}`,
+                source: file,
+                ...(packageName ? { packageName } : {}),
+            });
+            return;
+        }
+        r.items.forEach((item, index) => {
+            const reasons = testDefinitionRejectionReasons(item);
+            if (reasons.length > 0) {
+                const id = item && typeof item === 'object' ? item.id : undefined;
+                rejected.push({
+                    file,
+                    index,
+                    exportName: 'default',
+                    ...(typeof id === 'string' && id.length > 0 ? { entryId: id } : {}),
+                    reasons,
+                    cause: RejectionCause.Invalid,
+                });
+                return;
+            }
+            tests.push(item);
+            entries.push({ id: item.id, file, ...(packageName ? { packageName } : {}) });
+        });
+    };
     // Local config file.
     if (inspection.sharkcraftDir) {
-        const local = nodePath.join(inspection.sharkcraftDir, 'context-tests.ts');
-        if (existsSync(local)) {
-            tests.push(...(await importDefaultArray(local)));
-        }
+        const local = nodePath.join(inspection.sharkcraftDir, localName);
+        if (existsSync(local))
+            await readInto(local);
     }
     // Pack contributions.
     for (const pack of inspection.packs.validPacks) {
         const c = pack.manifest.contributions;
-        for (const rel of c.contextTestFiles ?? []) {
-            const full = nodePath.resolve(pack.packageRoot, rel);
-            tests.push(...(await importDefaultArray(full)));
-        }
+        for (const rel of c[slot] ?? [])
+            await readInto(nodePath.resolve(pack.packageRoot, rel), pack.packageName);
     }
-    return tests;
+    return { tests, entries, issues, rejected };
+}
+/**
+ * Context tests WITH what did not take effect (round 12, 12.1): files that
+ * failed to import and every declared test the loader refused.
+ */
+export async function loadContextTestsWithIssues(inspection) {
+    return loadTestDefinitions(inspection, 'context-tests.ts', 'contextTestFiles');
+}
+/** Agent-contract tests WITH what did not take effect (round 12, 12.1). */
+export async function loadAgentContractTestsWithIssues(inspection) {
+    return loadTestDefinitions(inspection, 'agent-tests.ts', 'agentTestFiles');
+}
+/**
+ * Load context tests from local sharkcraft config + any pack contributions.
+ */
+export async function loadContextTests(inspection) {
+    return [...(await loadContextTestsWithIssues(inspection)).tests];
 }
 export async function loadAgentContractTests(inspection) {
-    const tests = [];
-    if (inspection.sharkcraftDir) {
-        const local = nodePath.join(inspection.sharkcraftDir, 'agent-tests.ts');
-        if (existsSync(local)) {
-            tests.push(...(await importDefaultArray(local)));
-        }
-    }
-    for (const pack of inspection.packs.validPacks) {
-        const c = pack.manifest.contributions;
-        for (const rel of c.agentTestFiles ?? []) {
-            const full = nodePath.resolve(pack.packageRoot, rel);
-            tests.push(...(await importDefaultArray(full)));
-        }
-    }
-    return tests;
+    return [...(await loadAgentContractTestsWithIssues(inspection)).tests];
 }
 export function runContextTest(inspection, test) {
     const overview = buildProjectOverview(inspection.workspace, inspection.config?.projectName);
@@ -91,29 +169,33 @@ export function runContextTest(inspection, test) {
     // Diagnostics: only populate on failure. Show the top-ranked alternatives
     // the ranker chose for the task, so the test author can see what beat the
     // expected entry — and tighten the rule's appliesWhen/tags accordingly.
+    // `mustInclude` is a RANKER-SURFACED assertion; whether the id exists at all
+    // is answered by the shared registry, the set `shrk knowledge list` prints.
     let diagnostics;
     if (!passed) {
         const ranked = rankKnowledgeEntries(inspection.knowledgeEntries, test.task);
+        const knowledgeRegistry = consulted(inspection, 'knowledge');
         diagnostics = [];
         for (const id of missingInclude) {
-            const exists = inspection.knowledgeEntries.some((e) => e.id === id);
+            const exists = referenceIdExists(inspection, 'knowledge', id);
             diagnostics.push({
                 id,
                 existsInRegistry: exists,
+                consulted: knowledgeRegistry,
                 topAlternatives: ranked.slice(0, 5).map((r) => ({
                     id: r.item.id,
                     score: r.score,
                     reasons: r.reasons,
                 })),
-                suggestions: buildMissingIncludeSuggestions(id, exists, test.task),
+                suggestions: buildMissingIncludeSuggestions(id, exists, test.task, knowledgeRegistry),
             });
         }
         for (const id of unexpectedInclude) {
-            const entry = inspection.knowledgeEntries.find((e) => e.id === id);
             const reasons = ranked.find((r) => r.item.id === id)?.reasons ?? [];
             diagnostics.push({
                 id,
-                existsInRegistry: !!entry,
+                existsInRegistry: referenceIdExists(inspection, 'knowledge', id),
+                consulted: knowledgeRegistry,
                 topAlternatives: reasons.length
                     ? [{ id, score: ranked.find((r) => r.item.id === id)?.score ?? 0, reasons }]
                     : undefined,
@@ -141,10 +223,10 @@ export function runContextTest(inspection, test) {
         result.diagnostics = diagnostics;
     return result;
 }
-function buildMissingIncludeSuggestions(id, existsInRegistry, task) {
+function buildMissingIncludeSuggestions(id, existsInRegistry, task, registry) {
     if (!existsInRegistry) {
         return [
-            `Entry "${id}" does not exist in the inspection — either install the pack that ships it or correct the id.`,
+            `Entry "${id}" is not in the knowledge registry (the ${registry.size} ids \`${registry.listVerb}\` prints) — this expectation can never pass. Install the pack that ships it or correct the id.`,
             `If the test expectation is wrong, update sharkcraft/context-tests.ts.`,
         ];
     }
@@ -156,38 +238,91 @@ function buildMissingIncludeSuggestions(id, existsInRegistry, task) {
         `Or raise its priority / shorten the title to include task-relevant tokens.`,
     ];
 }
-export function runAgentContractTest(inspection, test, registries) {
+export function runAgentContractTest(inspection, test, 
+/** @deprecated Ignored — existence resolves through the shared reference registry. */
+_registries) {
     // Self-tests assert specific rules/templates appear for a task — they
     // test semantics, not packet size. Opt out of the compact cap so the
     // assertions see the full ranking.
     const packet = buildTaskPacket(inspection, test.task, { maxTokens: 3500, compact: false });
     const actualPipelines = packet.recommendedPipelines.map((p) => p.pipelineId);
+    // ── Ranker-SURFACED expectations (order-sensitive) ─────────────────────
     const missingTemplates = (test.expectedTemplates ?? []).filter((id) => !packet.relevantTemplates.some((t) => t.id === id));
     const missingRules = (test.expectedRules ?? []).filter((id) => !packet.relevantRules.some((r) => r.id === id));
     const missingForbidden = (test.expectedForbiddenActions ?? []).filter((a) => !packet.forbiddenActions.includes(a));
     const missingVerification = (test.expectedVerificationCommands ?? []).filter((c) => !packet.verificationCommands.includes(c));
-    // New expectation checks. Each one fails when the asset is missing
-    // from the relevant registry (drift detection) or — for commands — also
-    // not surfaced in the packet's recommended set.
-    const helperRegistry = registries?.helpers ?? loadHelperRegistryQuietly();
-    const playbookList = registries?.playbooks ?? listPlaybookIdsQuietly(inspection);
-    const policyChecks = registries?.policies ?? listPolicyIdsQuietly(inspection);
-    const constructList = registries?.constructs ?? listConstructIdsQuietly(inspection);
-    const commandCatalog = registries?.commands ?? listCommandIdsQuietly(inspection);
-    const knowledgeIds = registries?.knowledge ?? new Set(inspection.knowledgeEntries.map((e) => e.id));
-    const missingHelpers = (test.expectedHelpers ?? []).filter((id) => !helperRegistry.has(id));
-    const missingPlaybooks = (test.expectedPlaybooks ?? []).filter((id) => !playbookList.has(id));
-    const missingPolicies = (test.expectedPolicies ?? []).filter((id) => !policyChecks.has(id));
-    const missingConstructs = (test.expectedConstructs ?? []).filter((id) => !constructList.has(id));
-    const missingCommands = (test.expectedCommands ?? []).filter((id) => {
-        if (packet.recommendedCliCommands.includes(id))
-            return false;
-        // Accept either the catalog id form ("dev start") or the bare
-        // first token ("dev"). The catalog uses dotted ids that map to the
-        // command shape; we accept either form.
-        return !commandCatalog.has(id) && !commandCatalog.has(id.split(' ')[0]);
-    });
-    const missingKnowledge = (test.expectedKnowledge ?? []).filter((id) => !knowledgeIds.has(id));
+    // ── Registry-EXISTENCE expectations (stable) ───────────────────────────
+    // One authority: the shared reference registry. The runner used to keep a
+    // parallel resolver (a `*Quietly` helper per kind) whose helper set missed
+    // pack helpers, and whose command set read a property no inspection has —
+    // so every correct `expectedCommands` entry failed.
+    const unverified = [];
+    const unverifiedIds = new Set();
+    const existsCheck = (kind, ids) => {
+        const missing = [];
+        for (const id of ids ?? []) {
+            const status = existence(inspection, kind, id);
+            if (status === 'exists')
+                continue;
+            missing.push(id);
+            if (status === 'unverifiable') {
+                unverified.push(`${kind}:${id}`);
+                unverifiedIds.add(`${kind}:${id}`);
+            }
+        }
+        return missing;
+    };
+    const missingHelpers = existsCheck('helper', test.expectedHelpers);
+    const missingPlaybooks = existsCheck('playbook', test.expectedPlaybooks);
+    const missingPolicies = existsCheck('policy', test.expectedPolicies);
+    const missingConstructs = existsCheck('construct', test.expectedConstructs);
+    const missingKnowledge = existsCheck('knowledge', test.expectedKnowledge);
+    // ── Commands: surfaced by the packet, OR a command that really resolves ─
+    const missingCommands = [];
+    const commandDiagnostics = [];
+    for (const cmd of test.expectedCommands ?? []) {
+        if (packet.recommendedCliCommands.includes(cmd))
+            continue;
+        // A command REFERENCE: the bare catalog form (`dev start`) is read as
+        // `shrk dev start` by the one resolver — no private normalisation here.
+        const resolution = resolveShrkCommandReference(inspection, cmd);
+        if (resolution.status === CommandResolutionStatus.Unverified) {
+            missingCommands.push(cmd);
+            unverified.push(`command:${cmd}`);
+            unverifiedIds.add(`command:${cmd}`);
+            commandDiagnostics.push({
+                id: cmd,
+                kind: 'command',
+                existsInRegistry: false,
+                assertion: 'exists',
+                code: 'unverifiable',
+                consulted: consulted(inspection, 'command'),
+                suggestions: [
+                    `Command "${cmd}" was not surfaced for "${test.task}", and it could not be resolved: ${COMMAND_INDEX_NOT_INJECTED}.`,
+                    'Run the test through the CLI (`shrk test agent`), which injects the live command index.',
+                ],
+            });
+            continue;
+        }
+        if (isCommandResolved(resolution))
+            continue;
+        missingCommands.push(cmd);
+        commandDiagnostics.push({
+            id: cmd,
+            kind: 'command',
+            existsInRegistry: false,
+            assertion: 'exists',
+            code: 'unknown-command',
+            consulted: consulted(inspection, 'command'),
+            ...(resolution.closest && resolution.closest.length > 0 ? { closest: resolution.closest } : {}),
+            suggestions: [
+                `Command "${cmd}" does not resolve (${resolution.status}${resolution.reason ? ` — ${resolution.reason}` : ''}) — this expectation can never pass.`,
+                resolution.closest && resolution.closest.length > 0
+                    ? `Did you mean \`${resolution.closest[0]}\`?`
+                    : 'Run `shrk surface list` for every real command.',
+            ],
+        });
+    }
     // mustNotInclude — fail if any of these ids ended up surfaced in the packet.
     const surfacedIds = new Set();
     for (const t of packet.relevantTemplates)
@@ -202,77 +337,82 @@ export function runAgentContractTest(inspection, test, registries) {
         surfacedIds.add(p.preset.id);
     const unexpectedlyIncluded = (test.mustNotInclude ?? []).filter((id) => surfacedIds.has(id));
     const pipelineOk = !test.expectedPipeline || actualPipelines.includes(test.expectedPipeline);
-    const passed = pipelineOk &&
-        missingTemplates.length === 0 &&
-        missingRules.length === 0 &&
-        missingForbidden.length === 0 &&
-        missingVerification.length === 0 &&
-        missingHelpers.length === 0 &&
-        missingPlaybooks.length === 0 &&
-        missingPolicies.length === 0 &&
-        missingConstructs.length === 0 &&
-        missingCommands.length === 0 &&
-        missingKnowledge.length === 0 &&
-        unexpectedlyIncluded.length === 0;
-    // Diagnostics on failure: per missing id, explain whether the entry
-    // exists in the registry and how to make it surface.
+    const failing = (pipelineOk ? 0 : 1) +
+        missingTemplates.length +
+        missingRules.length +
+        missingForbidden.length +
+        missingVerification.length +
+        missingHelpers.length +
+        missingPlaybooks.length +
+        missingPolicies.length +
+        missingConstructs.length +
+        missingCommands.length +
+        missingKnowledge.length +
+        unexpectedlyIncluded.length;
+    const passed = failing === 0;
+    const verdict = passed
+        ? 'pass'
+        : failing === unverified.length
+            ? 'not-verified'
+            : 'fail';
+    // Diagnostics on failure: per missing id, whether it exists, WHICH registry
+    // said so, and how to make it surface.
     let diagnostics;
     if (!passed) {
         diagnostics = [];
-        for (const id of missingTemplates) {
-            const exists = inspection.templates.some((t) => t.id === id);
-            diagnostics.push({
+        const surfacedDiagnostic = (kind, id, whenExists) => {
+            const exists = referenceIdExists(inspection, kind, id);
+            const registry = consulted(inspection, kind);
+            const label = kind[0].toUpperCase() + kind.slice(1);
+            return {
                 id,
-                kind: 'template',
+                kind,
                 existsInRegistry: exists,
+                assertion: 'surfaced',
+                code: exists ? 'not-surfaced' : 'unknown-id',
+                consulted: registry,
                 suggestions: exists
-                    ? [
-                        `Template "${id}" exists but the ranker did not place it in the top results.`,
-                        'Add task-aligned tags/appliesWhen to the template (e.g. tags:["service"], appliesWhen:["generate-service"]).',
-                        'Reference the template from a preset that matches the task profile (includes.templateIds).',
-                        'Reference it from a pipeline step (`step.references`).',
-                    ]
+                    ? whenExists
                     : [
-                        `Template "${id}" does not exist — install the pack that ships it or correct the id.`,
+                        `${label} "${id}" is not registered — it is not among the ${registry.size} ids \`${registry.listVerb}\` prints, so this expectation can never pass.`,
+                        'Install the pack that ships it or correct the id.',
                     ],
-            });
+            };
+        };
+        for (const id of missingTemplates) {
+            diagnostics.push(surfacedDiagnostic('template', id, [
+                `Template "${id}" exists but the ranker did not place it in the top results.`,
+                'Add task-aligned tags/appliesWhen to the template (e.g. tags:["service"], appliesWhen:["generate-service"]).',
+                'Reference the template from a preset that matches the task profile (includes.templateIds).',
+                'Reference it from a pipeline step (`step.references`).',
+            ]));
         }
         for (const id of missingRules) {
-            const exists = inspection.knowledgeEntries.some((e) => e.id === id);
-            diagnostics.push({
-                id,
-                kind: 'rule',
-                existsInRegistry: exists,
-                suggestions: exists
-                    ? [
-                        `Rule "${id}" exists but the ranker did not surface it for "${test.task}".`,
-                        'Align its appliesWhen with the domain (e.g. generate-service / generate-utility / create-pipeline).',
-                        'Add domain tags (service / utility / route / pipeline).',
-                        'Reference it from actionHints.relatedTemplates on a rule that *is* ranking, or include it in a preset.',
-                    ]
-                    : [
-                        `Rule "${id}" does not exist — install the pack that ships it or correct the id.`,
-                    ],
-            });
+            // The RULE registry (what `shrk rules list` prints) — it used to be the
+            // knowledge entries, so a knowledge-but-not-rule id was told "exists but
+            // not surfaced", a permanently red test with a false hint.
+            diagnostics.push(surfacedDiagnostic('rule', id, [
+                `Rule "${id}" exists but the ranker did not surface it for "${test.task}".`,
+                'Align its appliesWhen with the domain (e.g. generate-service / generate-utility / create-pipeline).',
+                'Add domain tags (service / utility / route / pipeline).',
+                'Reference it from actionHints.relatedTemplates on a rule that *is* ranking, or include it in a preset.',
+            ]));
         }
         if (!pipelineOk && test.expectedPipeline) {
-            diagnostics.push({
-                id: test.expectedPipeline,
-                kind: 'pipeline',
-                existsInRegistry: inspection.pipelines.some((p) => p.id === test.expectedPipeline),
-                suggestions: [
-                    `Pipeline "${test.expectedPipeline}" did not rank in the top 3 for "${test.task}".`,
-                    'Add domain-aligned tags to the pipeline (tags:["plugin"], …).',
-                    'Add task tokens to its title / description so the ranker catches them.',
-                    'Reference the pipeline from a preset that the task profile recommends.',
-                ],
-            });
+            diagnostics.push(surfacedDiagnostic('pipeline', test.expectedPipeline, [
+                `Pipeline "${test.expectedPipeline}" did not rank in the top 3 for "${test.task}".`,
+                'Add domain-aligned tags to the pipeline (tags:["plugin"], …).',
+                'Add task tokens to its title / description so the ranker catches them.',
+                'Reference the pipeline from a preset that the task profile recommends.',
+            ]));
         }
         for (const action of missingForbidden) {
             diagnostics.push({
                 id: action,
                 kind: 'forbidden-action',
                 existsInRegistry: false,
+                assertion: 'surfaced',
+                code: 'not-aggregated',
                 suggestions: [
                     `No rule's actionHints.forbiddenActions contains "${action}" for this task's relevant set.`,
                     'Add the forbiddenAction to one of the high-priority rules that match this task.',
@@ -284,83 +424,58 @@ export function runAgentContractTest(inspection, test, registries) {
                 id: cmd,
                 kind: 'verification-command',
                 existsInRegistry: false,
+                assertion: 'surfaced',
+                code: 'not-aggregated',
                 suggestions: [
                     `Verification command "${cmd}" is not aggregated for this task.`,
                     'Add it to a high-priority rule\'s actionHints.verificationCommands.',
                 ],
             });
         }
-        // Diagnostics for the new expectation slots.
-        for (const id of missingHelpers) {
-            diagnostics.push({
+        const existsDiagnostic = (kind, id, hint) => {
+            const registry = consulted(inspection, kind);
+            const cold = unverifiedIds.has(`${kind}:${id}`);
+            return {
                 id,
-                kind: 'helper',
-                existsInRegistry: helperRegistry.has(id),
-                suggestions: [
-                    `Helper "${id}" not found in the helper registry — install the pack that ships it or correct the id.`,
-                    'Make sure the helper is registered in HELPERS in packages/inspector/src/helper-registry.ts (or a pack contribution).',
-                ],
-            });
+                kind,
+                existsInRegistry: false,
+                assertion: 'exists',
+                code: cold ? 'unverifiable' : 'unknown-id',
+                consulted: registry,
+                suggestions: cold
+                    ? [
+                        `The ${kind} registry was not warmed, so "${id}" could not be looked up — NOT VERIFIED.`,
+                        'Call warmReferenceRegistries(inspection) before running agent tests.',
+                    ]
+                    : [
+                        `${kind[0].toUpperCase() + kind.slice(1)} "${id}" is not registered — it is not among the ${registry.size} ids \`${registry.listVerb}\` prints.`,
+                        hint,
+                    ],
+            };
+        };
+        for (const id of missingHelpers) {
+            diagnostics.push(existsDiagnostic('helper', id, 'Register it in HELPERS (packages/inspector/src/helper-registry.ts) or ship it from a pack helperFiles contribution.'));
         }
         for (const id of missingPlaybooks) {
-            diagnostics.push({
-                id,
-                kind: 'playbook',
-                existsInRegistry: playbookList.has(id),
-                suggestions: [
-                    `Playbook "${id}" not found in the playbook registry — install the pack that ships it or correct the id.`,
-                    'Ensure the playbook file is referenced in sharkcraft.config.ts / pack contributions and exports a default array.',
-                ],
-            });
+            diagnostics.push(existsDiagnostic('playbook', id, 'Ensure the playbook file is referenced in sharkcraft.config.ts / pack contributions and exports a default array.'));
         }
         for (const id of missingPolicies) {
-            diagnostics.push({
-                id,
-                kind: 'policy',
-                existsInRegistry: policyChecks.has(id),
-                suggestions: [
-                    `Policy "${id}" not found in the policy-engine checks — install the pack that ships it or correct the id.`,
-                    'Add it to sharkcraft/policies.ts or a pack policyCheckFile.',
-                ],
-            });
+            diagnostics.push(existsDiagnostic('policy', id, 'Add it to sharkcraft/policies.ts or a pack policyCheckFile.'));
         }
         for (const id of missingConstructs) {
-            diagnostics.push({
-                id,
-                kind: 'construct',
-                existsInRegistry: constructList.has(id),
-                suggestions: [
-                    `Construct "${id}" not found in the construct registry — install the pack that ships it or correct the id.`,
-                    'Add a defineConstruct() entry in sharkcraft/constructs.ts or a pack constructFile.',
-                ],
-            });
-        }
-        for (const id of missingCommands) {
-            diagnostics.push({
-                id,
-                kind: 'command',
-                existsInRegistry: commandCatalog.has(id) || commandCatalog.has(id.split(' ')[0]),
-                suggestions: [
-                    `Command "${id}" was not surfaced for "${test.task}" and is not in the command catalog.`,
-                    'Add it to a high-priority rule\'s actionHints.commands or check the command-catalog.ts entries.',
-                ],
-            });
+            diagnostics.push(existsDiagnostic('construct', id, 'Add a defineConstruct() entry in sharkcraft/constructs.ts or a pack constructFile.'));
         }
+        diagnostics.push(...commandDiagnostics);
         for (const id of missingKnowledge) {
-            diagnostics.push({
-                id,
-                kind: 'knowledge',
-                existsInRegistry: knowledgeIds.has(id),
-                suggestions: [
-                    `Knowledge entry "${id}" is not present in the inspection — install the pack that ships it or correct the id.`,
-                ],
-            });
+            diagnostics.push(existsDiagnostic('knowledge', id, 'Install the pack that ships it or correct the id.'));
         }
         for (const id of unexpectedlyIncluded) {
             diagnostics.push({
                 id,
                 kind: 'must-not-include',
                 existsInRegistry: true,
+                assertion: 'surfaced',
+                code: 'surfaced-but-forbidden',
                 suggestions: [
                     `"${id}" was surfaced by the ranker but the test forbids it.`,
                     'Narrow its appliesWhen / tags to avoid matching this task wording.',
@@ -372,6 +487,7 @@ export function runAgentContractTest(inspection, test, registries) {
         id: test.id,
         task: test.task,
         passed,
+        verdict,
         actualPipelines,
         missingTemplates,
         missingRules,
@@ -385,6 +501,8 @@ export function runAgentContractTest(inspection, test, registries) {
         missingKnowledge,
         unexpectedlyIncluded,
     };
+    if (unverified.length > 0)
+        result.unverified = unverified;
     if (test.expectedPipeline)
         result.expectedPipeline = test.expectedPipeline;
     if (!passed) {
@@ -400,149 +518,32 @@ export function runAgentContractTest(inspection, test, registries) {
             `missingKnowledge=${missingKnowledge.length}`,
             `unexpectedlyIncluded=${unexpectedlyIncluded.length}`,
         ];
+        if (unverified.length > 0)
+            partsList.push(`unverified=${unverified.length}`);
         result.failureSummary = partsList.join(' ');
     }
     if (diagnostics)
         result.diagnostics = diagnostics;
     return result;
 }
-// ── Registry helpers ────────────────────────────────────────────────
-function loadHelperRegistryQuietly() {
-    try {
-        return new Set(HELPERS.map((h) => h.id));
-    }
-    catch {
-        return new Set();
-    }
-}
-function listPlaybookIdsQuietly(inspection) {
-    try {
-        return new Set(listPlaybooks(inspection).map((p) => p.id));
-    }
-    catch {
-        return new Set();
-    }
-}
-function listPolicyIdsQuietly(inspection) {
-    return new Set(listPolicyIds(inspection));
-}
-function listConstructIdsQuietly(inspection) {
-    try {
-        return new Set(listConstructs(inspection).map((c) => c.id));
-    }
-    catch {
-        return new Set();
-    }
-}
-function listCommandIdsQuietly(inspection) {
-    const cat = inspection.commandCatalog ?? [];
-    const out = new Set();
-    for (const c of cat) {
-        if (c.id)
-            out.add(c.id);
-        if (c.command)
-            out.add(c.command);
-    }
-    return out;
-}
 /**
- * Async pre-loader for agent-contract test registries. Reads local
- * sharkcraft/policies.ts, sharkcraft/constructs.ts, sharkcraft/playbooks/
- * (and pack contributions where present). Result is sync-friendly so the
- * runner can stay sync.
+ * Warm the shared reference registry and return a snapshot of it.
+ *
+ * @deprecated Call `warmReferenceRegistries(inspection, options)` directly (the
+ * CLI: `warmCliReferenceRegistries`) — the runners read the registry itself.
+ * The snapshot is a projection of that one authority, never a second source:
+ * the old loader re-read policies / playbooks / constructs from its own file
+ * list and returned the built-in helpers only (no pack helpers).
  */
-export async function loadAgentContractRegistries(inspection) {
+export async function loadAgentContractRegistries(inspection, options = {}) {
+    await warmReferenceRegistries(inspection, options);
+    const ids = (kind) => new Set(referenceIdsFor(inspection, kind));
     return {
-        helpers: loadHelperRegistryQuietly(),
-        playbooks: await loadPlaybookIdsAsync(inspection),
-        policies: await loadPolicyIdsAsync(inspection),
-        constructs: await loadConstructIdsAsync(inspection),
-        commands: listCommandIdsQuietly(inspection),
-        knowledge: new Set(inspection.knowledgeEntries.map((e) => e.id)),
+        helpers: ids('helper'),
+        playbooks: ids('playbook'),
+        policies: ids('policy'),
+        constructs: ids('construct'),
+        commands: new Set(),
+        knowledge: ids('knowledge'),
     };
 }
-async function importDefaultIdsArray(absPath) {
-    if (!existsSync(absPath))
-        return [];
-    try {
-        const mod = (await importModuleViaLoader(absPath));
-        return Array.isArray(mod.default) ? mod.default : [];
-    }
-    catch {
-        return [];
-    }
-}
-async function loadPolicyIdsAsync(inspection) {
-    const out = new Set();
-    if (inspection.sharkcraftDir) {
-        const local = nodePath.join(inspection.sharkcraftDir, 'policies.ts');
-        for (const entry of await importDefaultIdsArray(local)) {
-            if (entry?.id)
-                out.add(entry.id);
-        }
-    }
-    for (const pack of inspection.packs.validPacks) {
-        const c = pack.manifest.contributions;
-        for (const rel of c.policyCheckFiles ?? []) {
-            const full = nodePath.resolve(pack.packageRoot, rel);
-            for (const entry of await importDefaultIdsArray(full)) {
-                if (entry?.id)
-                    out.add(entry.id);
-            }
-        }
-    }
-    // Also include any policyChecks already on the inspection (sync fallback).
-    for (const id of listPolicyIdsQuietly(inspection))
-        out.add(id);
-    return out;
-}
-async function loadPlaybookIdsAsync(inspection) {
-    const out = new Set();
-    // Sync fallback from registry.
-    for (const id of listPlaybookIdsQuietly(inspection))
-        out.add(id);
-    // Local file.
-    if (inspection.sharkcraftDir) {
-        const local = nodePath.join(inspection.sharkcraftDir, 'playbooks.ts');
-        for (const entry of await importDefaultIdsArray(local)) {
-            if (entry?.id)
-                out.add(entry.id);
-        }
-    }
-    // Pack contributions.
-    for (const pack of inspection.packs.validPacks) {
-        const c = pack.manifest.contributions;
-        for (const rel of c.playbookFiles ?? []) {
-            const full = nodePath.resolve(pack.packageRoot, rel);
-            for (const entry of await importDefaultIdsArray(full)) {
-                if (entry?.id)
-                    out.add(entry.id);
-            }
-        }
-    }
-    return out;
-}
-async function loadConstructIdsAsync(inspection) {
-    const out = new Set();
-    // Sync fallback first.
-    for (const id of listConstructIdsQuietly(inspection))
-        out.add(id);
-    if (inspection.sharkcraftDir) {
-        const local = nodePath.join(inspection.sharkcraftDir, 'constructs.ts');
-        for (const entry of await importDefaultIdsArray(local)) {
-            if (entry?.id)
-                out.add(entry.id);
-        }
-    }
-    for (const pack of inspection.packs.validPacks) {
-        const c = pack.manifest.contributions;
-        for (const rel of c.constructFiles ?? []) {
-            const full = nodePath.resolve(pack.packageRoot, rel);
-            for (const entry of await importDefaultIdsArray(full)) {
-                if (entry?.id)
-                    out.add(entry.id);
-            }
-        }
-    }
-    return out;
-}