@agentskit/doc-bridge 1.7.45 → 1.9.0

package/docs/study/task-suite-v1.json

@@ -0,0 +1,96 @@
+{
+  "type": "study-task-suite",
+  "schemaVersion": 1,
+  "suiteVersion": "v1",
+  "protocolVersion": "v1",
+  "title": "Controlled documentation and knowledge discovery task suite",
+  "population": ["consumer-01", "consumer-02", "consumer-03", "consumer-04", "consumer-05", "consumer-06"],
+  "modelIds": ["low-cost-model", "reference-model"],
+  "scenarioIds": ["repository-only", "deterministic-doc-bridge", "registry-assisted"],
+  "maxTokensPerTask": 400000,
+  "maxRuntimeMsPerTask": 180000,
+  "maxRuns": 288,
+  "replicatesPerTask": 2,
+  "ordering": { "strategy": "balanced-counter-order", "seed": "phase-1-baseline" },
+  "tasks": [
+    {
+      "id": "consumer-01-discovery", "repositoryId": "consumer-01", "category": "discovery", "difficulty": "easy",
+      "objective": "Locate the primary entrypoints, ownership boundaries, and canonical documentation for the consumer project.",
+      "initialContext": ["Start with the repository root and declared documentation."], "allowedTools": ["repository-read", "doc-bridge-query", "shell-read"], "forbiddenActions": ["Modify files", "Use private identifiers in the result"],
+      "expectedOutcome": "A concise map names the entrypoints, owners, and evidence paths without unsupported claims.", "evidenceRequirements": [{ "id": "entrypoint-evidence", "description": "Paths and declarations supporting the discovered entrypoints and ownership.", "source": "artifact" }], "acceptanceChecks": [{ "id": "discovery-check", "command": "ak-docs discover --json", "expected": "The command exits successfully and emits structured discovery evidence." }], "budget": { "maxTokens": 6000, "maxRuntimeMs": 60000 },
+      "surfaces": { "logic": { "required": true }, "endpoint": { "required": false, "reason": "No endpoint is required to discover repository structure." }, "database": { "required": false, "reason": "No database is required to discover repository structure." }, "cli": { "required": true }, "mcp": { "required": false, "reason": "MCP is not required for the discovery task." }, "ui": { "required": false, "reason": "UI behavior is not part of the discovery task." }, "docs": { "required": true } },
+      "rubric": { "success": ["All required entrypoints and owners are correct and evidenced."], "partial": ["The main entrypoint is correct but one non-critical boundary or citation is missing."], "incorrect": ["The answer identifies a wrong entrypoint or owner."], "incomplete": ["The answer does not cover the requested repository scope."], "blocked": ["Required repository evidence cannot be accessed or validated."] }, "variantGroup": "consumer-01-discovery", "variants": [{ "id": "variant-a", "label": "Entrypoint-first wording", "context": "Begin at package and application entrypoints." }, { "id": "variant-b", "label": "Ownership-first wording", "context": "Begin at ownership and documentation routing declarations." }]
+    },
+    {
+      "id": "consumer-01-architecture", "repositoryId": "consumer-01", "category": "architecture", "difficulty": "medium",
+      "objective": "Describe the project architecture as connected components and identify one boundary that deserves review.",
+      "initialContext": ["Use observed imports, workspace metadata, and documentation claims."], "allowedTools": ["repository-read", "doc-bridge-map", "shell-read"], "forbiddenActions": ["Modify files", "Infer runtime edges without evidence"],
+      "expectedOutcome": "A bounded architecture view identifies components, directional relationships, and one evidence-backed attention point.", "evidenceRequirements": [{ "id": "architecture-evidence", "description": "Observed entities and relations supporting the architecture view.", "source": "artifact" }], "acceptanceChecks": [{ "id": "architecture-check", "command": "ak-docs map --json", "expected": "The command emits a bounded architecture map with evidence references." }], "budget": { "maxTokens": 8000, "maxRuntimeMs": 90000 },
+      "surfaces": { "logic": { "required": true }, "endpoint": { "required": false, "reason": "No endpoint execution is needed for static architecture mapping." }, "database": { "required": false, "reason": "No database execution is needed for static architecture mapping." }, "cli": { "required": true }, "mcp": { "required": false, "reason": "MCP is not required for the architecture task." }, "ui": { "required": false, "reason": "The task evaluates architecture evidence, not UI behavior." }, "docs": { "required": true } },
+      "rubric": { "success": ["The map is bounded, directional, and every attention point has evidence."], "partial": ["The main components are correct but one relation or limitation is missing."], "incorrect": ["The map asserts an unsupported component or relation."], "incomplete": ["The answer is a list without connected architecture evidence."], "blocked": ["The architecture artifact cannot be generated or inspected."] }, "variantGroup": "consumer-01-architecture", "variants": [{ "id": "variant-a", "label": "Component-first wording", "context": "Start by grouping the observed components." }, { "id": "variant-b", "label": "Relation-first wording", "context": "Start by tracing the strongest observed connections." }]
+    },
+    {
+      "id": "consumer-01-documentation", "repositoryId": "consumer-01", "category": "documentation", "difficulty": "hard",
+      "objective": "Find one stale, missing, unclear, redundant, or contradictory documentation claim and classify it with evidence.",
+      "initialContext": ["Compare documentation claims with current source and generated indexes."], "allowedTools": ["repository-read", "doc-bridge-audit", "shell-read"], "forbiddenActions": ["Edit documentation", "Treat an unverified semantic inference as fact"],
+      "expectedOutcome": "The review reports a precise documentation issue, its evidence, confidence, limitation, and recommended next action.", "evidenceRequirements": [{ "id": "documentation-evidence", "description": "Document and source evidence for the classified documentation issue.", "source": "artifact" }, { "id": "review-limitation", "description": "A limitation or confidence statement for semantic judgment.", "source": "human" }], "acceptanceChecks": [{ "id": "documentation-check", "command": "ak-docs audit documentation --json", "expected": "The audit emits measurable documentation findings and limitations." }], "budget": { "maxTokens": 10000, "maxRuntimeMs": 120000 },
+      "surfaces": { "logic": { "required": true }, "endpoint": { "required": false, "reason": "The documentation classification does not require endpoint execution." }, "database": { "required": false, "reason": "The documentation classification does not require database execution." }, "cli": { "required": true }, "mcp": { "required": false, "reason": "MCP is not required for the bounded audit task." }, "ui": { "required": false, "reason": "UI behavior is outside the documentation classification task." }, "docs": { "required": true } },
+      "rubric": { "success": ["The issue category, evidence, confidence, and limitation are all correct."], "partial": ["The issue is real but the category, confidence, or limitation needs correction."], "incorrect": ["The claimed issue is contradicted by current evidence."], "incomplete": ["The answer names a concern without enough evidence to classify it."], "blocked": ["The required source or documentation corpus is unavailable."] }, "variantGroup": "consumer-01-documentation", "variants": [{ "id": "variant-a", "label": "Claim-first wording", "context": "Begin with a documentation claim and trace it to source." }, { "id": "variant-b", "label": "Evidence-first wording", "context": "Begin with observed source and find its documentation coverage." }]
+    },
+    {
+      "id": "consumer-01-implementation", "repositoryId": "consumer-01", "category": "implementation", "difficulty": "hard",
+      "objective": "Propose a minimal documentation-only correction for a verified knowledge gap without changing source code.",
+      "initialContext": ["Use the preceding discovery, architecture, and documentation evidence as context."], "allowedTools": ["repository-read", "doc-bridge-query", "shell-read"], "forbiddenActions": ["Modify files", "Invent behavior not present in evidence", "Expose private identifiers"],
+      "expectedOutcome": "A proposed patch is bounded to the correct documentation location, preserves existing conventions, and states how it would be verified.", "evidenceRequirements": [{ "id": "patch-evidence", "description": "Evidence identifying the target document and the knowledge gap.", "source": "artifact" }, { "id": "verification-plan", "description": "A real command or review step that would validate the proposal after approval.", "source": "runner" }], "acceptanceChecks": [{ "id": "proposal-check", "command": "ak-docs check --json", "expected": "The check command exits successfully for the declared repository contract." }], "budget": { "maxTokens": 10000, "maxRuntimeMs": 120000 },
+      "surfaces": { "logic": { "required": true }, "endpoint": { "required": false, "reason": "No endpoint is changed by a documentation-only proposal." }, "database": { "required": false, "reason": "No database is changed by a documentation-only proposal." }, "cli": { "required": true }, "mcp": { "required": false, "reason": "MCP is not required for proposal validation." }, "ui": { "required": false, "reason": "No UI is changed by a documentation-only proposal." }, "docs": { "required": true } },
+      "rubric": { "success": ["The proposal is minimal, correctly scoped, evidence-backed, and verifiable."], "partial": ["The proposal is safe but needs a location, convention, or verification correction."], "incorrect": ["The proposal targets the wrong document or asserts unsupported behavior."], "incomplete": ["The proposal lacks a usable change or verification plan."], "blocked": ["The target documentation cannot be safely identified."] }, "variantGroup": "consumer-01-implementation", "variants": [{ "id": "variant-a", "label": "Gap-to-patch wording", "context": "Start from the verified gap and derive the smallest correction." }, { "id": "variant-b", "label": "Patch-to-proof wording", "context": "Start from the expected proof and identify the smallest documentation change." }]
+    },
+
+    {
+      "id": "consumer-02-discovery", "repositoryId": "consumer-02", "category": "discovery", "difficulty": "easy", "objective": "Locate the primary entrypoints, ownership boundaries, and canonical documentation for the consumer project.", "initialContext": ["Start with the repository root and declared documentation."], "allowedTools": ["repository-read", "doc-bridge-query", "shell-read"], "forbiddenActions": ["Modify files", "Use private identifiers in the result"], "expectedOutcome": "A concise map names the entrypoints, owners, and evidence paths without unsupported claims.", "evidenceRequirements": [{ "id": "entrypoint-evidence", "description": "Paths and declarations supporting the discovered entrypoints and ownership.", "source": "artifact" }], "acceptanceChecks": [{ "id": "discovery-check", "command": "ak-docs discover --json", "expected": "The command exits successfully and emits structured discovery evidence." }], "budget": { "maxTokens": 6000, "maxRuntimeMs": 60000 }, "surfaces": { "logic": { "required": true }, "endpoint": { "required": false, "reason": "No endpoint is required to discover repository structure." }, "database": { "required": false, "reason": "No database is required to discover repository structure." }, "cli": { "required": true }, "mcp": { "required": false, "reason": "MCP is not required for the discovery task." }, "ui": { "required": false, "reason": "UI behavior is not part of the discovery task." }, "docs": { "required": true } }, "rubric": { "success": ["All required entrypoints and owners are correct and evidenced."], "partial": ["The main entrypoint is correct but one non-critical boundary or citation is missing."], "incorrect": ["The answer identifies a wrong entrypoint or owner."], "incomplete": ["The answer does not cover the requested repository scope."], "blocked": ["Required repository evidence cannot be accessed or validated."] }, "variantGroup": "consumer-02-discovery", "variants": [{ "id": "variant-a", "label": "Entrypoint-first wording", "context": "Begin at package and application entrypoints." }, { "id": "variant-b", "label": "Ownership-first wording", "context": "Begin at ownership and documentation routing declarations." }] },
+    {
+      "id": "consumer-02-architecture", "repositoryId": "consumer-02", "category": "architecture", "difficulty": "medium", "objective": "Describe the project architecture as connected components and identify one boundary that deserves review.", "initialContext": ["Use observed imports, workspace metadata, and documentation claims."], "allowedTools": ["repository-read", "doc-bridge-map", "shell-read"], "forbiddenActions": ["Modify files", "Infer runtime edges without evidence"], "expectedOutcome": "A bounded architecture view identifies components, directional relationships, and one evidence-backed attention point.", "evidenceRequirements": [{ "id": "architecture-evidence", "description": "Observed entities and relations supporting the architecture view.", "source": "artifact" }], "acceptanceChecks": [{ "id": "architecture-check", "command": "ak-docs map --json", "expected": "The command emits a bounded architecture map with evidence references." }], "budget": { "maxTokens": 8000, "maxRuntimeMs": 90000 }, "surfaces": { "logic": { "required": true }, "endpoint": { "required": false, "reason": "No endpoint execution is needed for static architecture mapping." }, "database": { "required": false, "reason": "No database execution is needed for static architecture mapping." }, "cli": { "required": true }, "mcp": { "required": false, "reason": "MCP is not required for the architecture task." }, "ui": { "required": false, "reason": "The task evaluates architecture evidence, not UI behavior." }, "docs": { "required": true } }, "rubric": { "success": ["The map is bounded, directional, and every attention point has evidence."], "partial": ["The main components are correct but one relation or limitation is missing."], "incorrect": ["The map asserts an unsupported component or relation."], "incomplete": ["The answer is a list without connected architecture evidence."], "blocked": ["The architecture artifact cannot be generated or inspected."] }, "variantGroup": "consumer-02-architecture", "variants": [{ "id": "variant-a", "label": "Component-first wording", "context": "Start by grouping the observed components." }, { "id": "variant-b", "label": "Relation-first wording", "context": "Start by tracing the strongest observed connections." }] },
+    {
+      "id": "consumer-02-documentation", "repositoryId": "consumer-02", "category": "documentation", "difficulty": "hard", "objective": "Find one stale, missing, unclear, redundant, or contradictory documentation claim and classify it with evidence.", "initialContext": ["Compare documentation claims with current source and generated indexes."], "allowedTools": ["repository-read", "doc-bridge-audit", "shell-read"], "forbiddenActions": ["Edit documentation", "Treat an unverified semantic inference as fact"], "expectedOutcome": "The review reports a precise documentation issue, its evidence, confidence, limitation, and recommended next action.", "evidenceRequirements": [{ "id": "documentation-evidence", "description": "Document and source evidence for the classified documentation issue.", "source": "artifact" }, { "id": "review-limitation", "description": "A limitation or confidence statement for semantic judgment.", "source": "human" }], "acceptanceChecks": [{ "id": "documentation-check", "command": "ak-docs audit documentation --json", "expected": "The audit emits measurable documentation findings and limitations." }], "budget": { "maxTokens": 10000, "maxRuntimeMs": 120000 }, "surfaces": { "logic": { "required": true }, "endpoint": { "required": false, "reason": "The documentation classification does not require endpoint execution." }, "database": { "required": false, "reason": "The documentation classification does not require database execution." }, "cli": { "required": true }, "mcp": { "required": false, "reason": "MCP is not required for the bounded audit task." }, "ui": { "required": false, "reason": "UI behavior is outside the documentation classification task." }, "docs": { "required": true } }, "rubric": { "success": ["The issue category, evidence, confidence, and limitation are all correct."], "partial": ["The issue is real but the category, confidence, or limitation needs correction."], "incorrect": ["The claimed issue is contradicted by current evidence."], "incomplete": ["The answer names a concern without enough evidence to classify it."], "blocked": ["The required source or documentation corpus is unavailable."] }, "variantGroup": "consumer-02-documentation", "variants": [{ "id": "variant-a", "label": "Claim-first wording", "context": "Begin with a documentation claim and trace it to source." }, { "id": "variant-b", "label": "Evidence-first wording", "context": "Begin with observed source and find its documentation coverage." }] },
+    {
+      "id": "consumer-02-implementation", "repositoryId": "consumer-02", "category": "implementation", "difficulty": "hard", "objective": "Propose a minimal documentation-only correction for a verified knowledge gap without changing source code.", "initialContext": ["Use the preceding discovery, architecture, and documentation evidence as context."], "allowedTools": ["repository-read", "doc-bridge-query", "shell-read"], "forbiddenActions": ["Modify files", "Invent behavior not present in evidence", "Expose private identifiers"], "expectedOutcome": "A proposed patch is bounded to the correct documentation location, preserves existing conventions, and states how it would be verified.", "evidenceRequirements": [{ "id": "patch-evidence", "description": "Evidence identifying the target document and the knowledge gap.", "source": "artifact" }, { "id": "verification-plan", "description": "A real command or review step that would validate the proposal after approval.", "source": "runner" }], "acceptanceChecks": [{ "id": "proposal-check", "command": "ak-docs check --json", "expected": "The check command exits successfully for the declared repository contract." }], "budget": { "maxTokens": 10000, "maxRuntimeMs": 120000 }, "surfaces": { "logic": { "required": true }, "endpoint": { "required": false, "reason": "No endpoint is changed by a documentation-only proposal." }, "database": { "required": false, "reason": "No database is changed by a documentation-only proposal." }, "cli": { "required": true }, "mcp": { "required": false, "reason": "MCP is not required for proposal validation." }, "ui": { "required": false, "reason": "No UI is changed by a documentation-only proposal." }, "docs": { "required": true } }, "rubric": { "success": ["The proposal is minimal, correctly scoped, evidence-backed, and verifiable."], "partial": ["The proposal is safe but needs a location, convention, or verification correction."], "incorrect": ["The proposal targets the wrong document or asserts unsupported behavior."], "incomplete": ["The proposal lacks a usable change or verification plan."], "blocked": ["The target documentation cannot be safely identified."] }, "variantGroup": "consumer-02-implementation", "variants": [{ "id": "variant-a", "label": "Gap-to-patch wording", "context": "Start from the verified gap and derive the smallest correction." }, { "id": "variant-b", "label": "Patch-to-proof wording", "context": "Start from the expected proof and identify the smallest documentation change." }] },
+
+    {
+      "id": "consumer-03-discovery", "repositoryId": "consumer-03", "category": "discovery", "difficulty": "easy", "objective": "Locate the primary entrypoints, ownership boundaries, and canonical documentation for the consumer project.", "initialContext": ["Start with the repository root and declared documentation."], "allowedTools": ["repository-read", "doc-bridge-query", "shell-read"], "forbiddenActions": ["Modify files", "Use private identifiers in the result"], "expectedOutcome": "A concise map names the entrypoints, owners, and evidence paths without unsupported claims.", "evidenceRequirements": [{ "id": "entrypoint-evidence", "description": "Paths and declarations supporting the discovered entrypoints and ownership.", "source": "artifact" }], "acceptanceChecks": [{ "id": "discovery-check", "command": "ak-docs discover --json", "expected": "The command exits successfully and emits structured discovery evidence." }], "budget": { "maxTokens": 6000, "maxRuntimeMs": 60000 }, "surfaces": { "logic": { "required": true }, "endpoint": { "required": false, "reason": "No endpoint is required to discover repository structure." }, "database": { "required": false, "reason": "No database is required to discover repository structure." }, "cli": { "required": true }, "mcp": { "required": false, "reason": "MCP is not required for the discovery task." }, "ui": { "required": false, "reason": "UI behavior is not part of the discovery task." }, "docs": { "required": true } }, "rubric": { "success": ["All required entrypoints and owners are correct and evidenced."], "partial": ["The main entrypoint is correct but one non-critical boundary or citation is missing."], "incorrect": ["The answer identifies a wrong entrypoint or owner."], "incomplete": ["The answer does not cover the requested repository scope."], "blocked": ["Required repository evidence cannot be accessed or validated."] }, "variantGroup": "consumer-03-discovery", "variants": [{ "id": "variant-a", "label": "Entrypoint-first wording", "context": "Begin at package and application entrypoints." }, { "id": "variant-b", "label": "Ownership-first wording", "context": "Begin at ownership and documentation routing declarations." }] },
+    {
+      "id": "consumer-03-architecture", "repositoryId": "consumer-03", "category": "architecture", "difficulty": "medium", "objective": "Describe the project architecture as connected components and identify one boundary that deserves review.", "initialContext": ["Use observed imports, workspace metadata, and documentation claims."], "allowedTools": ["repository-read", "doc-bridge-map", "shell-read"], "forbiddenActions": ["Modify files", "Infer runtime edges without evidence"], "expectedOutcome": "A bounded architecture view identifies components, directional relationships, and one evidence-backed attention point.", "evidenceRequirements": [{ "id": "architecture-evidence", "description": "Observed entities and relations supporting the architecture view.", "source": "artifact" }], "acceptanceChecks": [{ "id": "architecture-check", "command": "ak-docs map --json", "expected": "The command emits a bounded architecture map with evidence references." }], "budget": { "maxTokens": 8000, "maxRuntimeMs": 90000 }, "surfaces": { "logic": { "required": true }, "endpoint": { "required": false, "reason": "No endpoint execution is needed for static architecture mapping." }, "database": { "required": false, "reason": "No database execution is needed for static architecture mapping." }, "cli": { "required": true }, "mcp": { "required": false, "reason": "MCP is not required for the architecture task." }, "ui": { "required": false, "reason": "The task evaluates architecture evidence, not UI behavior." }, "docs": { "required": true } }, "rubric": { "success": ["The map is bounded, directional, and every attention point has evidence."], "partial": ["The main components are correct but one relation or limitation is missing."], "incorrect": ["The map asserts an unsupported component or relation."], "incomplete": ["The answer is a list without connected architecture evidence."], "blocked": ["The architecture artifact cannot be generated or inspected."] }, "variantGroup": "consumer-03-architecture", "variants": [{ "id": "variant-a", "label": "Component-first wording", "context": "Start by grouping the observed components." }, { "id": "variant-b", "label": "Relation-first wording", "context": "Start by tracing the strongest observed connections." }] },
+    {
+      "id": "consumer-03-documentation", "repositoryId": "consumer-03", "category": "documentation", "difficulty": "hard", "objective": "Find one stale, missing, unclear, redundant, or contradictory documentation claim and classify it with evidence.", "initialContext": ["Compare documentation claims with current source and generated indexes."], "allowedTools": ["repository-read", "doc-bridge-audit", "shell-read"], "forbiddenActions": ["Edit documentation", "Treat an unverified semantic inference as fact"], "expectedOutcome": "The review reports a precise documentation issue, its evidence, confidence, limitation, and recommended next action.", "evidenceRequirements": [{ "id": "documentation-evidence", "description": "Document and source evidence for the classified documentation issue.", "source": "artifact" }, { "id": "review-limitation", "description": "A limitation or confidence statement for semantic judgment.", "source": "human" }], "acceptanceChecks": [{ "id": "documentation-check", "command": "ak-docs audit documentation --json", "expected": "The audit emits measurable documentation findings and limitations." }], "budget": { "maxTokens": 10000, "maxRuntimeMs": 120000 }, "surfaces": { "logic": { "required": true }, "endpoint": { "required": false, "reason": "The documentation classification does not require endpoint execution." }, "database": { "required": false, "reason": "The documentation classification does not require database execution." }, "cli": { "required": true }, "mcp": { "required": false, "reason": "MCP is not required for the bounded audit task." }, "ui": { "required": false, "reason": "UI behavior is outside the documentation classification task." }, "docs": { "required": true } }, "rubric": { "success": ["The issue category, evidence, confidence, and limitation are all correct."], "partial": ["The issue is real but the category, confidence, or limitation needs correction."], "incorrect": ["The claimed issue is contradicted by current evidence."], "incomplete": ["The answer names a concern without enough evidence to classify it."], "blocked": ["The required source or documentation corpus is unavailable."] }, "variantGroup": "consumer-03-documentation", "variants": [{ "id": "variant-a", "label": "Claim-first wording", "context": "Begin with a documentation claim and trace it to source." }, { "id": "variant-b", "label": "Evidence-first wording", "context": "Begin with observed source and find its documentation coverage." }] },
+    {
+      "id": "consumer-03-implementation", "repositoryId": "consumer-03", "category": "implementation", "difficulty": "hard", "objective": "Propose a minimal documentation-only correction for a verified knowledge gap without changing source code.", "initialContext": ["Use the preceding discovery, architecture, and documentation evidence as context."], "allowedTools": ["repository-read", "doc-bridge-query", "shell-read"], "forbiddenActions": ["Modify files", "Invent behavior not present in evidence", "Expose private identifiers"], "expectedOutcome": "A proposed patch is bounded to the correct documentation location, preserves existing conventions, and states how it would be verified.", "evidenceRequirements": [{ "id": "patch-evidence", "description": "Evidence identifying the target document and the knowledge gap.", "source": "artifact" }, { "id": "verification-plan", "description": "A real command or review step that would validate the proposal after approval.", "source": "runner" }], "acceptanceChecks": [{ "id": "proposal-check", "command": "ak-docs check --json", "expected": "The check command exits successfully for the declared repository contract." }], "budget": { "maxTokens": 10000, "maxRuntimeMs": 120000 }, "surfaces": { "logic": { "required": true }, "endpoint": { "required": false, "reason": "No endpoint is changed by a documentation-only proposal." }, "database": { "required": false, "reason": "No database is changed by a documentation-only proposal." }, "cli": { "required": true }, "mcp": { "required": false, "reason": "MCP is not required for proposal validation." }, "ui": { "required": false, "reason": "No UI is changed by a documentation-only proposal." }, "docs": { "required": true } }, "rubric": { "success": ["The proposal is minimal, correctly scoped, evidence-backed, and verifiable."], "partial": ["The proposal is safe but needs a location, convention, or verification correction."], "incorrect": ["The proposal targets the wrong document or asserts unsupported behavior."], "incomplete": ["The proposal lacks a usable change or verification plan."], "blocked": ["The target documentation cannot be safely identified."] }, "variantGroup": "consumer-03-implementation", "variants": [{ "id": "variant-a", "label": "Gap-to-patch wording", "context": "Start from the verified gap and derive the smallest correction." }, { "id": "variant-b", "label": "Patch-to-proof wording", "context": "Start from the expected proof and identify the smallest documentation change." }] },
+
+    {
+      "id": "consumer-04-discovery", "repositoryId": "consumer-04", "category": "discovery", "difficulty": "easy", "objective": "Locate the primary entrypoints, ownership boundaries, and canonical documentation for the consumer project.", "initialContext": ["Start with the repository root and declared documentation."], "allowedTools": ["repository-read", "doc-bridge-query", "shell-read"], "forbiddenActions": ["Modify files", "Use private identifiers in the result"], "expectedOutcome": "A concise map names the entrypoints, owners, and evidence paths without unsupported claims.", "evidenceRequirements": [{ "id": "entrypoint-evidence", "description": "Paths and declarations supporting the discovered entrypoints and ownership.", "source": "artifact" }], "acceptanceChecks": [{ "id": "discovery-check", "command": "ak-docs discover --json", "expected": "The command exits successfully and emits structured discovery evidence." }], "budget": { "maxTokens": 6000, "maxRuntimeMs": 60000 }, "surfaces": { "logic": { "required": true }, "endpoint": { "required": false, "reason": "No endpoint is required to discover repository structure." }, "database": { "required": false, "reason": "No database is required to discover repository structure." }, "cli": { "required": true }, "mcp": { "required": false, "reason": "MCP is not required for the discovery task." }, "ui": { "required": false, "reason": "UI behavior is not part of the discovery task." }, "docs": { "required": true } }, "rubric": { "success": ["All required entrypoints and owners are correct and evidenced."], "partial": ["The main entrypoint is correct but one non-critical boundary or citation is missing."], "incorrect": ["The answer identifies a wrong entrypoint or owner."], "incomplete": ["The answer does not cover the requested repository scope."], "blocked": ["Required repository evidence cannot be accessed or validated."] }, "variantGroup": "consumer-04-discovery", "variants": [{ "id": "variant-a", "label": "Entrypoint-first wording", "context": "Begin at package and application entrypoints." }, { "id": "variant-b", "label": "Ownership-first wording", "context": "Begin at ownership and documentation routing declarations." }] },
+    {
+      "id": "consumer-04-architecture", "repositoryId": "consumer-04", "category": "architecture", "difficulty": "medium", "objective": "Describe the project architecture as connected components and identify one boundary that deserves review.", "initialContext": ["Use observed imports, workspace metadata, and documentation claims."], "allowedTools": ["repository-read", "doc-bridge-map", "shell-read"], "forbiddenActions": ["Modify files", "Infer runtime edges without evidence"], "expectedOutcome": "A bounded architecture view identifies components, directional relationships, and one evidence-backed attention point.", "evidenceRequirements": [{ "id": "architecture-evidence", "description": "Observed entities and relations supporting the architecture view.", "source": "artifact" }], "acceptanceChecks": [{ "id": "architecture-check", "command": "ak-docs map --json", "expected": "The command emits a bounded architecture map with evidence references." }], "budget": { "maxTokens": 8000, "maxRuntimeMs": 90000 }, "surfaces": { "logic": { "required": true }, "endpoint": { "required": false, "reason": "No endpoint execution is needed for static architecture mapping." }, "database": { "required": false, "reason": "No database execution is needed for static architecture mapping." }, "cli": { "required": true }, "mcp": { "required": false, "reason": "MCP is not required for the architecture task." }, "ui": { "required": false, "reason": "The task evaluates architecture evidence, not UI behavior." }, "docs": { "required": true } }, "rubric": { "success": ["The map is bounded, directional, and every attention point has evidence."], "partial": ["The main components are correct but one relation or limitation is missing."], "incorrect": ["The map asserts an unsupported component or relation."], "incomplete": ["The answer is a list without connected architecture evidence."], "blocked": ["The architecture artifact cannot be generated or inspected."] }, "variantGroup": "consumer-04-architecture", "variants": [{ "id": "variant-a", "label": "Component-first wording", "context": "Start by grouping the observed components." }, { "id": "variant-b", "label": "Relation-first wording", "context": "Start by tracing the strongest observed connections." }] },
+    {
+      "id": "consumer-04-documentation", "repositoryId": "consumer-04", "category": "documentation", "difficulty": "hard", "objective": "Find one stale, missing, unclear, redundant, or contradictory documentation claim and classify it with evidence.", "initialContext": ["Compare documentation claims with current source and generated indexes."], "allowedTools": ["repository-read", "doc-bridge-audit", "shell-read"], "forbiddenActions": ["Edit documentation", "Treat an unverified semantic inference as fact"], "expectedOutcome": "The review reports a precise documentation issue, its evidence, confidence, limitation, and recommended next action.", "evidenceRequirements": [{ "id": "documentation-evidence", "description": "Document and source evidence for the classified documentation issue.", "source": "artifact" }, { "id": "review-limitation", "description": "A limitation or confidence statement for semantic judgment.", "source": "human" }], "acceptanceChecks": [{ "id": "documentation-check", "command": "ak-docs audit documentation --json", "expected": "The audit emits measurable documentation findings and limitations." }], "budget": { "maxTokens": 10000, "maxRuntimeMs": 120000 }, "surfaces": { "logic": { "required": true }, "endpoint": { "required": false, "reason": "The documentation classification does not require endpoint execution." }, "database": { "required": false, "reason": "The documentation classification does not require database execution." }, "cli": { "required": true }, "mcp": { "required": false, "reason": "MCP is not required for the bounded audit task." }, "ui": { "required": false, "reason": "UI behavior is outside the documentation classification task." }, "docs": { "required": true } }, "rubric": { "success": ["The issue category, evidence, confidence, and limitation are all correct."], "partial": ["The issue is real but the category, confidence, or limitation needs correction."], "incorrect": ["The claimed issue is contradicted by current evidence."], "incomplete": ["The answer names a concern without enough evidence to classify it."], "blocked": ["The required source or documentation corpus is unavailable."] }, "variantGroup": "consumer-04-documentation", "variants": [{ "id": "variant-a", "label": "Claim-first wording", "context": "Begin with a documentation claim and trace it to source." }, { "id": "variant-b", "label": "Evidence-first wording", "context": "Begin with observed source and find its documentation coverage." }] },
+    {
+      "id": "consumer-04-implementation", "repositoryId": "consumer-04", "category": "implementation", "difficulty": "hard", "objective": "Propose a minimal documentation-only correction for a verified knowledge gap without changing source code.", "initialContext": ["Use the preceding discovery, architecture, and documentation evidence as context."], "allowedTools": ["repository-read", "doc-bridge-query", "shell-read"], "forbiddenActions": ["Modify files", "Invent behavior not present in evidence", "Expose private identifiers"], "expectedOutcome": "A proposed patch is bounded to the correct documentation location, preserves existing conventions, and states how it would be verified.", "evidenceRequirements": [{ "id": "patch-evidence", "description": "Evidence identifying the target document and the knowledge gap.", "source": "artifact" }, { "id": "verification-plan", "description": "A real command or review step that would validate the proposal after approval.", "source": "runner" }], "acceptanceChecks": [{ "id": "proposal-check", "command": "ak-docs check --json", "expected": "The check command exits successfully for the declared repository contract." }], "budget": { "maxTokens": 10000, "maxRuntimeMs": 120000 }, "surfaces": { "logic": { "required": true }, "endpoint": { "required": false, "reason": "No endpoint is changed by a documentation-only proposal." }, "database": { "required": false, "reason": "No database is changed by a documentation-only proposal." }, "cli": { "required": true }, "mcp": { "required": false, "reason": "MCP is not required for proposal validation." }, "ui": { "required": false, "reason": "No UI is changed by a documentation-only proposal." }, "docs": { "required": true } }, "rubric": { "success": ["The proposal is minimal, correctly scoped, evidence-backed, and verifiable."], "partial": ["The proposal is safe but needs a location, convention, or verification correction."], "incorrect": ["The proposal targets the wrong document or asserts unsupported behavior."], "incomplete": ["The proposal lacks a usable change or verification plan."], "blocked": ["The target documentation cannot be safely identified."] }, "variantGroup": "consumer-04-implementation", "variants": [{ "id": "variant-a", "label": "Gap-to-patch wording", "context": "Start from the verified gap and derive the smallest correction." }, { "id": "variant-b", "label": "Patch-to-proof wording", "context": "Start from the expected proof and identify the smallest documentation change." }] },
+
+    {
+      "id": "consumer-05-discovery", "repositoryId": "consumer-05", "category": "discovery", "difficulty": "easy", "objective": "Locate the primary entrypoints, ownership boundaries, and canonical documentation for the consumer project.", "initialContext": ["Start with the repository root and declared documentation."], "allowedTools": ["repository-read", "doc-bridge-query", "shell-read"], "forbiddenActions": ["Modify files", "Use private identifiers in the result"], "expectedOutcome": "A concise map names the entrypoints, owners, and evidence paths without unsupported claims.", "evidenceRequirements": [{ "id": "entrypoint-evidence", "description": "Paths and declarations supporting the discovered entrypoints and ownership.", "source": "artifact" }], "acceptanceChecks": [{ "id": "discovery-check", "command": "ak-docs discover --json", "expected": "The command exits successfully and emits structured discovery evidence." }], "budget": { "maxTokens": 6000, "maxRuntimeMs": 60000 }, "surfaces": { "logic": { "required": true }, "endpoint": { "required": false, "reason": "No endpoint is required to discover repository structure." }, "database": { "required": false, "reason": "No database is required to discover repository structure." }, "cli": { "required": true }, "mcp": { "required": false, "reason": "MCP is not required for the discovery task." }, "ui": { "required": false, "reason": "UI behavior is not part of the discovery task." }, "docs": { "required": true } }, "rubric": { "success": ["All required entrypoints and owners are correct and evidenced."], "partial": ["The main entrypoint is correct but one non-critical boundary or citation is missing."], "incorrect": ["The answer identifies a wrong entrypoint or owner."], "incomplete": ["The answer does not cover the requested repository scope."], "blocked": ["Required repository evidence cannot be accessed or validated."] }, "variantGroup": "consumer-05-discovery", "variants": [{ "id": "variant-a", "label": "Entrypoint-first wording", "context": "Begin at package and application entrypoints." }, { "id": "variant-b", "label": "Ownership-first wording", "context": "Begin at ownership and documentation routing declarations." }] },
+    {
+      "id": "consumer-05-architecture", "repositoryId": "consumer-05", "category": "architecture", "difficulty": "medium", "objective": "Describe the project architecture as connected components and identify one boundary that deserves review.", "initialContext": ["Use observed imports, workspace metadata, and documentation claims."], "allowedTools": ["repository-read", "doc-bridge-map", "shell-read"], "forbiddenActions": ["Modify files", "Infer runtime edges without evidence"], "expectedOutcome": "A bounded architecture view identifies components, directional relationships, and one evidence-backed attention point.", "evidenceRequirements": [{ "id": "architecture-evidence", "description": "Observed entities and relations supporting the architecture view.", "source": "artifact" }], "acceptanceChecks": [{ "id": "architecture-check", "command": "ak-docs map --json", "expected": "The command emits a bounded architecture map with evidence references." }], "budget": { "maxTokens": 8000, "maxRuntimeMs": 90000 }, "surfaces": { "logic": { "required": true }, "endpoint": { "required": false, "reason": "No endpoint execution is needed for static architecture mapping." }, "database": { "required": false, "reason": "No database execution is needed for static architecture mapping." }, "cli": { "required": true }, "mcp": { "required": false, "reason": "MCP is not required for the architecture task." }, "ui": { "required": false, "reason": "The task evaluates architecture evidence, not UI behavior." }, "docs": { "required": true } }, "rubric": { "success": ["The map is bounded, directional, and every attention point has evidence."], "partial": ["The main components are correct but one relation or limitation is missing."], "incorrect": ["The map asserts an unsupported component or relation."], "incomplete": ["The answer is a list without connected architecture evidence."], "blocked": ["The architecture artifact cannot be generated or inspected."] }, "variantGroup": "consumer-05-architecture", "variants": [{ "id": "variant-a", "label": "Component-first wording", "context": "Start by grouping the observed components." }, { "id": "variant-b", "label": "Relation-first wording", "context": "Start by tracing the strongest observed connections." }] },
+    {
+      "id": "consumer-05-documentation", "repositoryId": "consumer-05", "category": "documentation", "difficulty": "hard", "objective": "Find one stale, missing, unclear, redundant, or contradictory documentation claim and classify it with evidence.", "initialContext": ["Compare documentation claims with current source and generated indexes."], "allowedTools": ["repository-read", "doc-bridge-audit", "shell-read"], "forbiddenActions": ["Edit documentation", "Treat an unverified semantic inference as fact"], "expectedOutcome": "The review reports a precise documentation issue, its evidence, confidence, limitation, and recommended next action.", "evidenceRequirements": [{ "id": "documentation-evidence", "description": "Document and source evidence for the classified documentation issue.", "source": "artifact" }, { "id": "review-limitation", "description": "A limitation or confidence statement for semantic judgment.", "source": "human" }], "acceptanceChecks": [{ "id": "documentation-check", "command": "ak-docs audit documentation --json", "expected": "The audit emits measurable documentation findings and limitations." }], "budget": { "maxTokens": 10000, "maxRuntimeMs": 120000 }, "surfaces": { "logic": { "required": true }, "endpoint": { "required": false, "reason": "The documentation classification does not require endpoint execution." }, "database": { "required": false, "reason": "The documentation classification does not require database execution." }, "cli": { "required": true }, "mcp": { "required": false, "reason": "MCP is not required for the bounded audit task." }, "ui": { "required": false, "reason": "UI behavior is outside the documentation classification task." }, "docs": { "required": true } }, "rubric": { "success": ["The issue category, evidence, confidence, and limitation are all correct."], "partial": ["The issue is real but the category, confidence, or limitation needs correction."], "incorrect": ["The claimed issue is contradicted by current evidence."], "incomplete": ["The answer names a concern without enough evidence to classify it."], "blocked": ["The required source or documentation corpus is unavailable."] }, "variantGroup": "consumer-05-documentation", "variants": [{ "id": "variant-a", "label": "Claim-first wording", "context": "Begin with a documentation claim and trace it to source." }, { "id": "variant-b", "label": "Evidence-first wording", "context": "Begin with observed source and find its documentation coverage." }] },
+    {
+      "id": "consumer-05-implementation", "repositoryId": "consumer-05", "category": "implementation", "difficulty": "hard", "objective": "Propose a minimal documentation-only correction for a verified knowledge gap without changing source code.", "initialContext": ["Use the preceding discovery, architecture, and documentation evidence as context."], "allowedTools": ["repository-read", "doc-bridge-query", "shell-read"], "forbiddenActions": ["Modify files", "Invent behavior not present in evidence", "Expose private identifiers"], "expectedOutcome": "A proposed patch is bounded to the correct documentation location, preserves existing conventions, and states how it would be verified.", "evidenceRequirements": [{ "id": "patch-evidence", "description": "Evidence identifying the target document and the knowledge gap.", "source": "artifact" }, { "id": "verification-plan", "description": "A real command or review step that would validate the proposal after approval.", "source": "runner" }], "acceptanceChecks": [{ "id": "proposal-check", "command": "ak-docs check --json", "expected": "The check command exits successfully for the declared repository contract." }], "budget": { "maxTokens": 10000, "maxRuntimeMs": 120000 }, "surfaces": { "logic": { "required": true }, "endpoint": { "required": false, "reason": "No endpoint is changed by a documentation-only proposal." }, "database": { "required": false, "reason": "No database is changed by a documentation-only proposal." }, "cli": { "required": true }, "mcp": { "required": false, "reason": "MCP is not required for proposal validation." }, "ui": { "required": false, "reason": "No UI is changed by a documentation-only proposal." }, "docs": { "required": true } }, "rubric": { "success": ["The proposal is minimal, correctly scoped, evidence-backed, and verifiable."], "partial": ["The proposal is safe but needs a location, convention, or verification correction."], "incorrect": ["The proposal targets the wrong document or asserts unsupported behavior."], "incomplete": ["The proposal lacks a usable change or verification plan."], "blocked": ["The target documentation cannot be safely identified."] }, "variantGroup": "consumer-05-implementation", "variants": [{ "id": "variant-a", "label": "Gap-to-patch wording", "context": "Start from the verified gap and derive the smallest correction." }, { "id": "variant-b", "label": "Patch-to-proof wording", "context": "Start from the expected proof and identify the smallest documentation change." }] },
+
+    {
+      "id": "consumer-06-discovery", "repositoryId": "consumer-06", "category": "discovery", "difficulty": "easy", "objective": "Locate the primary entrypoints, ownership boundaries, and canonical documentation for the consumer project.", "initialContext": ["Start with the repository root and declared documentation."], "allowedTools": ["repository-read", "doc-bridge-query", "shell-read"], "forbiddenActions": ["Modify files", "Use private identifiers in the result"], "expectedOutcome": "A concise map names the entrypoints, owners, and evidence paths without unsupported claims.", "evidenceRequirements": [{ "id": "entrypoint-evidence", "description": "Paths and declarations supporting the discovered entrypoints and ownership.", "source": "artifact" }], "acceptanceChecks": [{ "id": "discovery-check", "command": "ak-docs discover --json", "expected": "The command exits successfully and emits structured discovery evidence." }], "budget": { "maxTokens": 6000, "maxRuntimeMs": 60000 }, "surfaces": { "logic": { "required": true }, "endpoint": { "required": false, "reason": "No endpoint is required to discover repository structure." }, "database": { "required": false, "reason": "No database is required to discover repository structure." }, "cli": { "required": true }, "mcp": { "required": false, "reason": "MCP is not required for the discovery task." }, "ui": { "required": false, "reason": "UI behavior is not part of the discovery task." }, "docs": { "required": true } }, "rubric": { "success": ["All required entrypoints and owners are correct and evidenced."], "partial": ["The main entrypoint is correct but one non-critical boundary or citation is missing."], "incorrect": ["The answer identifies a wrong entrypoint or owner."], "incomplete": ["The answer does not cover the requested repository scope."], "blocked": ["Required repository evidence cannot be accessed or validated."] }, "variantGroup": "consumer-06-discovery", "variants": [{ "id": "variant-a", "label": "Entrypoint-first wording", "context": "Begin at package and application entrypoints." }, { "id": "variant-b", "label": "Ownership-first wording", "context": "Begin at ownership and documentation routing declarations." }] },
+    {
+      "id": "consumer-06-architecture", "repositoryId": "consumer-06", "category": "architecture", "difficulty": "medium", "objective": "Describe the project architecture as connected components and identify one boundary that deserves review.", "initialContext": ["Use observed imports, workspace metadata, and documentation claims."], "allowedTools": ["repository-read", "doc-bridge-map", "shell-read"], "forbiddenActions": ["Modify files", "Infer runtime edges without evidence"], "expectedOutcome": "A bounded architecture view identifies components, directional relationships, and one evidence-backed attention point.", "evidenceRequirements": [{ "id": "architecture-evidence", "description": "Observed entities and relations supporting the architecture view.", "source": "artifact" }], "acceptanceChecks": [{ "id": "architecture-check", "command": "ak-docs map --json", "expected": "The command emits a bounded architecture map with evidence references." }], "budget": { "maxTokens": 8000, "maxRuntimeMs": 90000 }, "surfaces": { "logic": { "required": true }, "endpoint": { "required": false, "reason": "No endpoint execution is needed for static architecture mapping." }, "database": { "required": false, "reason": "No database execution is needed for static architecture mapping." }, "cli": { "required": true }, "mcp": { "required": false, "reason": "MCP is not required for the architecture task." }, "ui": { "required": false, "reason": "The task evaluates architecture evidence, not UI behavior." }, "docs": { "required": true } }, "rubric": { "success": ["The map is bounded, directional, and every attention point has evidence."], "partial": ["The main components are correct but one relation or limitation is missing."], "incorrect": ["The map asserts an unsupported component or relation."], "incomplete": ["The answer is a list without connected architecture evidence."], "blocked": ["The architecture artifact cannot be generated or inspected."] }, "variantGroup": "consumer-06-architecture", "variants": [{ "id": "variant-a", "label": "Component-first wording", "context": "Start by grouping the observed components." }, { "id": "variant-b", "label": "Relation-first wording", "context": "Start by tracing the strongest observed connections." }] },
+    {
+      "id": "consumer-06-documentation", "repositoryId": "consumer-06", "category": "documentation", "difficulty": "hard", "objective": "Find one stale, missing, unclear, redundant, or contradictory documentation claim and classify it with evidence.", "initialContext": ["Compare documentation claims with current source and generated indexes."], "allowedTools": ["repository-read", "doc-bridge-audit", "shell-read"], "forbiddenActions": ["Edit documentation", "Treat an unverified semantic inference as fact"], "expectedOutcome": "The review reports a precise documentation issue, its evidence, confidence, limitation, and recommended next action.", "evidenceRequirements": [{ "id": "documentation-evidence", "description": "Document and source evidence for the classified documentation issue.", "source": "artifact" }, { "id": "review-limitation", "description": "A limitation or confidence statement for semantic judgment.", "source": "human" }], "acceptanceChecks": [{ "id": "documentation-check", "command": "ak-docs audit documentation --json", "expected": "The audit emits measurable documentation findings and limitations." }], "budget": { "maxTokens": 10000, "maxRuntimeMs": 120000 }, "surfaces": { "logic": { "required": true }, "endpoint": { "required": false, "reason": "The documentation classification does not require endpoint execution." }, "database": { "required": false, "reason": "The documentation classification does not require database execution." }, "cli": { "required": true }, "mcp": { "required": false, "reason": "MCP is not required for the bounded audit task." }, "ui": { "required": false, "reason": "UI behavior is outside the documentation classification task." }, "docs": { "required": true } }, "rubric": { "success": ["The issue category, evidence, confidence, and limitation are all correct."], "partial": ["The issue is real but the category, confidence, or limitation needs correction."], "incorrect": ["The claimed issue is contradicted by current evidence."], "incomplete": ["The answer names a concern without enough evidence to classify it."], "blocked": ["The required source or documentation corpus is unavailable."] }, "variantGroup": "consumer-06-documentation", "variants": [{ "id": "variant-a", "label": "Claim-first wording", "context": "Begin with a documentation claim and trace it to source." }, { "id": "variant-b", "label": "Evidence-first wording", "context": "Begin with observed source and find its documentation coverage." }] },
+    {
+      "id": "consumer-06-implementation", "repositoryId": "consumer-06", "category": "implementation", "difficulty": "hard", "objective": "Propose a minimal documentation-only correction for a verified knowledge gap without changing source code.", "initialContext": ["Use the preceding discovery, architecture, and documentation evidence as context."], "allowedTools": ["repository-read", "doc-bridge-query", "shell-read"], "forbiddenActions": ["Modify files", "Invent behavior not present in evidence", "Expose private identifiers"], "expectedOutcome": "A proposed patch is bounded to the correct documentation location, preserves existing conventions, and states how it would be verified.", "evidenceRequirements": [{ "id": "patch-evidence", "description": "Evidence identifying the target document and the knowledge gap.", "source": "artifact" }, { "id": "verification-plan", "description": "A real command or review step that would validate the proposal after approval.", "source": "runner" }], "acceptanceChecks": [{ "id": "proposal-check", "command": "ak-docs check --json", "expected": "The check command exits successfully for the declared repository contract." }], "budget": { "maxTokens": 10000, "maxRuntimeMs": 120000 }, "surfaces": { "logic": { "required": true }, "endpoint": { "required": false, "reason": "No endpoint is changed by a documentation-only proposal." }, "database": { "required": false, "reason": "No database is changed by a documentation-only proposal." }, "cli": { "required": true }, "mcp": { "required": false, "reason": "MCP is not required for proposal validation." }, "ui": { "required": false, "reason": "No UI is changed by a documentation-only proposal." }, "docs": { "required": true } }, "rubric": { "success": ["The proposal is minimal, correctly scoped, evidence-backed, and verifiable."], "partial": ["The proposal is safe but needs a location, convention, or verification correction."], "incorrect": ["The proposal targets the wrong document or asserts unsupported behavior."], "incomplete": ["The proposal lacks a usable change or verification plan."], "blocked": ["The target documentation cannot be safely identified."] }, "variantGroup": "consumer-06-implementation", "variants": [{ "id": "variant-a", "label": "Gap-to-patch wording", "context": "Start from the verified gap and derive the smallest correction." }, { "id": "variant-b", "label": "Patch-to-proof wording", "context": "Start from the expected proof and identify the smallest documentation change." }] }
+  ],
+  "contentHashAlgo": "sha256-normalized-v1",
+  "contentHash": "d7810828bce3c6616f00897fd7c92dfa4ecdf12e549923795cd95cc81a2b0029"
+}

package/docs/study/token-efficiency-plan-v1.md

@@ -0,0 +1,337 @@
+---
+title: Token efficiency improvement plan
+description: A versioned, evidence-driven plan for reducing agent token consumption without reducing correctness.
+---
+
+# Token efficiency improvement plan
+
+Status: `complete-with-bounded-publication-claims`
+
+## Objective
+
+Reduce the amount of agent context and provider usage required to complete a
+correct, evidence-backed repository task. The study must distinguish payload
+reduction from useful token reduction and must never trade correctness, safety,
+or evidence quality for a smaller response.
+
+The primary product metric is:
+
+> **Tokens to correct action** — provider input and output tokens consumed from
+> the start of a controlled task until the agent produces a correct,
+> evidence-backed result and passes the task's applicable acceptance checks.
+
+If a task does not reach a correct action, this metric is `null`. It must not be
+estimated from an unsuccessful or merely plausible response.
+
+## What existing evidence means
+
+The existing study remains immutable historical evidence. It currently shows a
+directional reduction in provider-token-equivalent units for one controlled
+comparison, and a much larger reduction in serialized context payload for a
+bounded retrieval fixture. These are different measurements.
+
+The existing evidence does **not** yet prove a reduction in tokens to a correct
+action because the independent adjudication sample recorded no semantic
+successes in either comparison arm. A new protocol and a fresh baseline are
+required before making a stronger claim.
+
+## Scope
+
+In scope:
+
+- deterministic CLI, MCP, and retriever context delivery;
+- provider token and latency attribution;
+- tool calls, clarification requests, and rework;
+- evidence retrieval and task acceptance;
+- optional Registry-agent assistance as a separate scenario;
+- anonymized, reproducible study artifacts;
+- regression gates for context size, correctness, safety, and total cost.
+
+Out of scope for this plan:
+
+- a universal token-saving guarantee for every model or repository;
+- replacing deterministic evidence with an LLM-generated summary;
+- publishing raw prompts, repository content, private paths, or raw responses;
+- optimizing a single model at the expense of portability;
+- changing historical results to make them comparable with the new protocol.
+
+## North-star and supporting metrics
+
+| Metric | Definition | Direction | Evidence class |
+| --- | --- | --- | --- |
+| `tokens-to-correct-action` | Provider input + output tokens until a correct action passes acceptance checks | lower | controlled |
+| `correct-action-rate` | Tasks with correct evidence and passing acceptance checks / eligible tasks | higher | controlled |
+| `context-tokens-p95` | P95 tokens in the context supplied by Doc Bridge before task execution | lower | deterministic/provider |
+| `time-to-first-evidence` | Time until the first required evidence item is found | lower | runner |
+| `time-to-correct-action-p95` | P95 time among tasks reaching a correct action | lower | runner |
+| `tool-calls-per-correct-action` | Tool calls used by successful tasks | lower | provider/runner |
+| `clarification-rate` | Tasks requiring human clarification | lower, unless ambiguity is real | runner |
+| `rework-rate` | Tasks requiring a corrective second attempt | lower | runner |
+| `evidence-precision` | Returned evidence that is relevant to the declared task | higher | adjudicator |
+| `evidence-recall` | Required evidence found by the workflow | higher | adjudicator |
+| `safety-rate` | Tasks with no unsafe or unauthorized action | higher | acceptance |
+| `total-cost-per-correct-action` | Doc Bridge + agent + adjudicator cost for successful tasks | lower | controlled |
+
+Provider token counts and estimates are separate dimensions. A report must
+label the token method for every observation and must not combine provider and
+estimated values in one aggregate.
+
+## Controlled scenarios
+
+Every comparison uses the same task, repository snapshot, model, prompt
+contract, tool configuration, and acceptance contract where applicable.
+
+1. `repository-only`: the agent receives the repository through the declared
+   baseline workflow.
+2. `deterministic-doc-bridge`: the agent receives the deterministic Doc Bridge
+   handoff and progressive evidence retrieval.
+3. `registry-assisted`: the deterministic workflow may receive a separately
+   attributed proposal from the configured Registry agent.
+
+The Registry-assisted scenario is never merged into the deterministic result.
+Its tokens, latency, cost, and failures remain separately attributable.
+
+## Phase plan
+
+### Phase 0 — Measurement contract and baseline hygiene
+
+Purpose: make the study able to reject attractive but invalid token claims.
+
+Deliverables:
+
+- this plan, versioned and linked from the study index;
+- a machine-readable `token-efficiency-v2` protocol derived from the existing
+  study contract without changing historical artifacts;
+- a frozen baseline manifest with source revision, protocol hash, task-suite
+  hash, model configuration hashes, and run budget;
+- an acceptance matrix mapping every metric to a real executable check;
+- explicit rules for missing data, retries, timeouts, estimates, and baseline
+  replacement;
+- an anonymization review record for publication-bound artifacts.
+
+Phase 0 acceptance criteria:
+
+- every required metric has one collection method and one evidence location;
+- `tokens-to-correct-action` is unavailable when correctness or acceptance is
+  unavailable;
+- provider and estimated tokens cannot be aggregated together;
+- a failed, timed-out, or retried task remains visible and cannot be silently
+  replaced;
+- the baseline cannot be replaced by an ordinary study run;
+- all public artifacts contain identifiers, hashes, counts, and classifications
+  only;
+- the protocol, task suite, and baseline are content-addressed;
+- all criteria are executable through the repository verification contract.
+
+### Phase 1 — End-to-end instrumentation
+
+Add bounded telemetry at CLI, MCP, retriever, provider, and adjudicator
+boundaries. Capture context bytes/tokens by layer, provider usage, cache usage,
+tool calls, first useful evidence, clarification, rework, latency, and cost.
+
+Do not optimize the retrieval algorithm until this phase can explain where
+tokens are spent.
+
+The runner now persists `contextBytes` together with `contextTokens` and an
+explicit `contextTokenMethod` (`provider` or `estimate`). When the provider can
+measure it, `firstEvidenceLatencyMs` is retained beside execution latency.
+Provider and adjudicator token methods remain separate from byte-based context
+estimates and configured USD cost. Missing fields remain missing data; they are
+never reconstructed from an unsuccessful response.
+
+### Phase 2 — Deterministic context reduction
+
+Optimize the smallest useful context path:
+
+- package/module routing before broad search;
+- progressive disclosure from route to evidence to full source;
+- explicit per-query context budgets;
+- deduplicated summaries and stable evidence identifiers;
+- task-specific retrieval modes for discovery, editing, debugging, and
+  documentation review;
+- fail-closed behavior when confidence or evidence coverage is insufficient;
+- precomputed search structures where full-body scans affect latency;
+- measured expansion only when the previous layer is insufficient.
+
+Every optimization must be compared against the Phase 0 baseline and must
+preserve correctness and safety.
+
+The deterministic agent-search path now accepts a task mode and an explicit
+context-token budget. It removes summaries and follow-up commands before
+dropping grounded matches, reports whether truncation occurred, and fails
+closed when the minimum grounded result cannot fit. The fixture benchmark keeps
+the same four expected matches while reducing estimated context from 51 to 28
+tokens at a 32-token budget (45.1% on that fixture only). This is a bounded
+engineering signal, not a repository-wide or provider-token claim.
+
+### Phase 3 — Correctness-valid task suite
+
+Expand the fixed task suite so each task has:
+
+- an expected evidence set;
+- a real acceptance command or endpoint/CLI/MCP check;
+- a defined safe outcome;
+- success, partial, blocked, and incorrect classifications;
+- a bounded recovery path.
+
+The suite must cover discovery, architecture, documentation freshness,
+documentation/code contradiction, missing documentation, and implementation.
+
+### Phase 4 — Controlled improvement rounds
+
+Run paired, clean-session comparisons with the low-cost and reference models.
+Use fresh samples, randomized arm order, at least two replicates in the pilot,
+and three replicates for confirmation when the pilot has valid semantic
+successes. Report paired medians, P95 values, confidence intervals, subgroup
+results, missing data, and total cost.
+
+The first bounded public pilot is complete as an execution study: 16/16
+provider calls completed across four tasks, two models, and two paired
+scenarios. It measured a 3.15% aggregate reduction in provider-token-equivalent
+units and a 7.53% reduction in duration P95 for the deterministic Doc Bridge
+arm. The result is mixed at the task level, has one replicate, and records no
+currency or semantic-correctness claim. See the [pilot result](./phase4-public-pilot-result-v1.json).
+
+Pilot checklist:
+
+- [x] Remove the fixed six-population/24-task assumption from the reusable
+  runner while preserving the canonical suite.
+- [x] Create a public one-population pilot suite and hashed pairwise plan.
+- [x] Validate the provider and repository inputs with a real dry-run.
+- [x] Execute the bounded 16-observation pilot with both configured models.
+- [x] Record failed preparation runs and the final ledger/configuration hashes.
+- [x] Run the complete repository verification contract against the pilot source
+  revision.
+
+Phase 4 evidence: run `1789261738722-98752-ulzlls`, source revision
+`f3c91b0bf81fac526dae57987f5851ed8ee94a6e`, and verification digest
+`6a0baa2576dda2f4f3b19adebf430f9bfab1b9ba26a0ee8d3c1071f1ce2d0d3d`.
+
+### Phase 5 — Product and publication gate
+
+Expose the useful efficiency measurements through the CLI/report without
+exposing private data. Publish only claims supported by the controlled results.
+The report must separate payload reduction, provider-token reduction, and
+tokens-to-correct-action.
+
+The publication gate is defined in [Publication gate v1](./publication-gate-v1.md).
+The current implementation was approved for publication with bounded claims;
+it must not be described as a general or enterprise result.
+
+Phase 5 checklist:
+
+- [x] Expose versioned study metrics through `ak-docs study metrics` with JSON
+  and text output.
+- [x] Publish the bounded pilot ledger and result as anonymized, hashed
+  artifacts.
+- [x] Keep the landing narrative separate for estimated context payload,
+  provider-token measurements, and correctness evidence.
+- [x] Validate the publication artifact set with the privacy gate and the
+  current verification harness.
+- [x] Record the human publication decision for the public narrative through
+  the approved verification run `1789262169080-5193-98iuxg`.
+
+Phase 5 evidence: verification digest
+`f29172127a08a425ab0c22730f34e76b11ad6cddf427d109cf5a1d9ad1cda3cd`.
+Publication is approved only for the anonymized, bounded claims described in
+the publication gate; the study does not claim enterprise-wide generalization.
+
+## Initial success thresholds
+
+These are guardrails, not promises:
+
+- 100% of included observations have a valid token method or an explicit
+  missing-data classification;
+- no provider/estimate mixing;
+- at least 90% evidence precision and recall on the fixed task suite;
+- no regression in correctness, safety, or acceptance rate;
+- at least 25% lower median `tokens-to-correct-action` after semantic success
+  is measurable;
+- at least 20% lower P95 context tokens without a quality regression;
+- lower or equal total cost per correct action;
+- three consecutive controlled rounds without a material regression.
+
+If the baseline has zero correct actions, the next milestone is to repair the
+task acceptance contract and evidence completeness. A token reduction claim is
+not valid until successful tasks exist in both comparison arms.
+
+## Evidence and privacy rules
+
+Each run records source revision, protocol hash, task-suite hash, model and tool
+configuration hashes, run ID, artifact hashes, budget, and validation state.
+
+Publication-bound artifacts may contain only anonymized identifiers, metric
+values, hashes, counts, timings, classifications, and limitations. They must
+not contain repository contents, private paths, prompts, credentials, or raw
+agent responses. Human approval remains required before publication.
+
+## Phase 0 execution checklist
+
+- [x] Register this plan in the study documentation.
+- [x] Create and validate `token-efficiency-v2` without modifying historical
+  protocol or result artifacts.
+- [x] Define and hash the fresh baseline manifest.
+- [x] Map each Phase 0 criterion to the verification contract.
+- [x] Run deterministic protocol, privacy, documentation, and repository
+  verification checks.
+- [x] Record the Phase 0 run ID and unresolved limitations.
+
+Phase 0 is complete only when every unchecked item has current evidence tied to
+the same source revision and protocol hash.
+
+Phase 0 evidence: run `1789248412380-77131-6xxoie`, source revision
+`978642d64d18d39cafd920345bcb1b71b288c4f3`, and verification digest
+`13a7c518d879dfdc8738df897e489a37b467aa7b4017bf52ebe458ffb9a67f0`.
+
+## Phase 1 execution checklist
+
+- [x] Persist a labeled context-token value without mixing it with provider
+  input/output tokens.
+- [x] Preserve first-evidence latency when the provider reports it.
+- [x] Preserve adjudicator token provenance independently from the candidate
+  provider.
+- [x] Cover the new telemetry fields with executable runner tests.
+- [x] Run the complete repository verification contract against the Phase 1
+  source revision.
+
+Phase 1 evidence: run `1789254057078-3613-646ec3`, source revision
+`9b30fc32737316fc09d7ced0fe9b9f0953f88bf7`, and verification digest
+`1043fb367e857c54ec977b6172068bdfb9f374fb9b95c9829fe0c900d35eb02a`.
+
+Phase 1 is complete only after the final unchecked item has current evidence.
+
+## Phase 2 execution checklist
+
+- [x] Add task-specific deterministic search modes.
+- [x] Add an explicit per-query context budget for CLI and MCP agent search.
+- [x] Preserve the best grounded match while truncating lower-value context.
+- [x] Fail closed when the minimum grounded result cannot fit the budget.
+- [x] Measure correctness and baseline-versus-budgeted context on the public
+  fixture.
+- [x] Run the complete repository verification contract against the Phase 2
+  source revision.
+
+Phase 2 evidence: run `1789254756700-26023-qpzity`, source revision
+`a292c1877eefe370be46b48d7cb6fb2bae422be3`, and verification digest
+`b2e7534a1d24535591f15c2ae0a4beadfbf9cc49748e8102bd30dfb89c6a2e88`.
+
+Phase 2 is complete only after the final unchecked item has current evidence.
+
+## Phase 3 execution checklist
+
+- [x] Bind the fixed task suite to a content hash.
+- [x] Require executable acceptance checks and evidence requirements for every
+  task.
+- [x] Require explicit success, partial, incorrect, incomplete, and blocked
+  classifications.
+- [x] Define bounded recovery and safe outcomes for every task category.
+- [x] Explicitly cover discovery, architecture, documentation freshness,
+  documentation/code contradiction, missing documentation, and implementation.
+- [x] Run the complete repository verification contract against the Phase 3
+  source revision.
+
+Phase 3 is complete only after the final unchecked item has current evidence.
+
+Phase 3 evidence: run `1789256493577-54099-pxyb0i`, source revision
+`5d70fb0083ffc991233347c6617b8dc37e453518`, and verification digest
+`0cec5cfb673316c7648b4279e30c4f5ec5e1b2194cd1da755775a65da41ea29f`.