@multileaf/ai-workflow 0.7.0 → 0.8.0

package/dist/workflow.js

@@ -37,6 +37,12 @@ import { createSummaryCache, isSummaryFresh, parseSummaryCache, serializeSummary
 import { parseTokenUsage, recordTokenUsage, serializeTokenUsage, summarizeUsageByModel, } from "./token-usage.js";
 import { APPROVAL_STAGES, assertWorkflowId, validateTaskTicket, approveStage, beginStageApproval, createApprovalLedger, getUnresolvedPrerequisites, invalidateStageAndDescendants, parseApprovalLedger, parseWorkflowSession, rejectStage, serializeApprovalLedger, serializeWorkflowSession, skipOptionalStage, stageDefinition, stageFingerprint, } from "./human-approval.js";
 import { measureContextQuality, serializeContextQuality } from "./context-quality.js";
+import { collectContextBench, compareContextBench, parseContextBench, serializeContextBench, } from "./context-benchmark.js";
+import { materializeParityTask, parseParityTask, runParitySuite, serializeParitySuite, TASK_ASSEMBLER, } from "./task-parity.js";
+import { appendUsageSidecarEntry, parseUsageSidecar, serializeUsageSidecar, serializeUsageSidecarSummary, summarizeUsageSidecar, USAGE_SIDECAR_PATH, USAGE_TIERS, } from "./usage-sidecar.js";
+import { computeCostAttribution, serializeCostAttribution, serializeTaskUsage, summarizeTaskUsage, } from "./cost-attribution.js";
+import { defaultRoutingTable, parseRoutingTable, resolveRoute, ROUTING_TABLE_PATH, serializeResolvedRoute, serializeRoutingTable, } from "./model-routing.js";
+import { parseTraceabilityGraph } from "./traceability.js";
 import { serializeAdapterCapabilities } from "./adapter-contract.js";
 import { buildTraceabilityGraph, serializeTraceabilityGraph } from "./traceability.js";
 import { evaluateQualityGate } from "./quality-gates.js";
@@ -530,6 +536,105 @@ export async function runCommand(args, root, fs, services = {}) {
             }
             return { exitCode: 1, error: usage };
         }
+        if (command === "context-bench") {
+            if (!fs.exists(join(aiw, "manifest.yml")))
+                return { exitCode: 1, error: "Run `aiw install` first." };
+            const saveCount = args.filter((arg) => arg === "--save").length;
+            if (saveCount > 1)
+                return { exitCode: 1, error: "Usage: aiw context-bench [--baseline=path] [--save]" };
+            const save = saveCount === 1;
+            const parsed = parseNamedOptions(args.slice(1).filter((arg) => arg !== "--save"), ["baseline"]);
+            if (!parsed)
+                return { exitCode: 1, error: "Usage: aiw context-bench [--baseline=path] [--save]" };
+            const report = collectContextBench(root, fs);
+            const serialized = serializeContextBench(report);
+            const lines = [serialized.trimEnd()];
+            if (parsed.baseline) {
+                const baselinePath = safeProjectFile(root, parsed.baseline, fs);
+                if (!fs.exists(baselinePath))
+                    return { exitCode: 1, error: `Baseline not found: ${parsed.baseline}` };
+                const comparison = compareContextBench(parseContextBench(fs.read(baselinePath)), report);
+                lines.push(`Baseline: ${parsed.baseline} (bundle ${comparison.bundleChanged ? "changed" : "identical"}, token delta ${comparison.totalTokenDelta >= 0 ? "+" : ""}${comparison.totalTokenDelta})`);
+                if (comparison.regressions.length)
+                    return {
+                        exitCode: 1,
+                        error: `Standing context regressed against baseline:\n${comparison.regressions.map((regression) => `- ${regression}`).join("\n")}`,
+                        output: lines.join("\n"),
+                    };
+                lines.push("No regressions against baseline.");
+            }
+            if (save) {
+                const checkpoint = join(aiw, `checkpoints/context-bench-${report.bundleHash}.yml`);
+                if (!fs.createExclusive)
+                    return { exitCode: 1, error: "The filesystem does not support atomic checkpoints." };
+                if (!fs.createExclusive(checkpoint, serialized)) {
+                    if (fs.read(checkpoint) !== serialized)
+                        return {
+                            exitCode: 1,
+                            error: `Checkpoint already exists with different content: ${checkpoint}`,
+                        };
+                    lines.push(`Checkpoint already recorded: ${checkpoint}`);
+                }
+                else {
+                    lines.push(`Checkpoint saved: ${checkpoint}`);
+                }
+            }
+            return { exitCode: 0, output: lines.join("\n") };
+        }
+        if (command === "parity") {
+            if (!fs.exists(join(aiw, "manifest.yml")))
+                return { exitCode: 1, error: "Run `aiw install` first." };
+            const parsed = parseNamedOptions(args.slice(1), ["tasks"]);
+            if (!parsed)
+                return { exitCode: 1, error: "Usage: aiw parity [--tasks=directory]" };
+            const tasksPath = parsed.tasks;
+            if (tasksPath && (isAbsolute(tasksPath) || tasksPath.split(/[\\/]/).includes("..")))
+                return {
+                    exitCode: 1,
+                    error: "Parity tasks directory must be a safe project-relative path.",
+                };
+            const tasksDirectory = tasksPath ? join(root, tasksPath) : join(aiw, "parity");
+            if (fs.pathType?.(tasksDirectory) !== "directory" || !fs.list)
+                return {
+                    exitCode: 1,
+                    error: `Parity tasks directory not found: ${relative(root, tasksDirectory).split(sep).join("/")}. Add YAML tasks (see docs/token-efficiency.md).`,
+                };
+            const taskFiles = (fs.list(tasksDirectory) ?? [])
+                .filter((name) => name.endsWith(".yml"))
+                .sort();
+            if (!taskFiles.length)
+                return {
+                    exitCode: 1,
+                    error: `No parity tasks found in ${relative(root, tasksDirectory).split(sep).join("/")}. Each task is one YAML file with schema, id, layers, critical_facts, and completion_markers.`,
+                };
+            const tasks = taskFiles.map((name) => {
+                const path = join(tasksDirectory, name);
+                try {
+                    return materializeParityTask(parseParityTask(fs.read(path)), (source) => {
+                        const absolute = join(root, source);
+                        if (fs.pathType?.(absolute) !== "file")
+                            throw new Error("source is not a file");
+                        return fs.read(absolute);
+                    });
+                }
+                catch (error) {
+                    throw new Error(`${name}: ${error.message}`, { cause: error });
+                }
+            });
+            const suite = runParitySuite(tasks, TASK_ASSEMBLER);
+            const serialized = serializeParitySuite(suite);
+            if (suite.failed)
+                return {
+                    exitCode: 1,
+                    output: serialized.trimEnd(),
+                    error: `Task parity failed for ${suite.failed} of ${suite.total} task(s): critical facts or completion markers were lost by the ${TASK_ASSEMBLER.name} assembler. Fix context assembly or move the facts to a retrieved layer; do not delete the expectations.`,
+                };
+            return {
+                exitCode: 0,
+                output: serialized.trimEnd() +
+                    `\nTask parity passed: every critical fact survived the ${TASK_ASSEMBLER.name} assembler.`,
+            };
+        }
         if (command === "scan") {
             if (!fs.exists(join(aiw, "manifest.yml")))
                 return { exitCode: 1, error: "Run `aiw install` first." };
@@ -647,6 +752,28 @@ export async function runCommand(args, root, fs, services = {}) {
                         model: usage.model,
                     });
                 fs.write(usagePath, serializeTokenUsage(next));
+                const sidecarPath = join(aiw, USAGE_SIDECAR_PATH);
+                const ledger = fs.exists(sidecarPath) ? parseUsageSidecar(fs.read(sidecarPath)) : [];
+                let sidecar = ledger;
+                for (const usage of modelUsage) {
+                    const measured = usage.detail.includes("(measured)");
+                    const isInterpreter = usage.label === "interpreter" && measured && interpreterUsage;
+                    sidecar = appendUsageSidecarEntry(sidecar, {
+                        command: `scan.${usage.label}`,
+                        label: measured ? "measured" : "estimated",
+                        tiers: isInterpreter
+                            ? {
+                                "cache-read": 0,
+                                "fresh-input": interpreterUsage.promptTokens,
+                                "cache-write": 0,
+                                output: interpreterUsage.completionTokens,
+                            }
+                            : { "cache-read": 0, "fresh-input": usage.tokens, "cache-write": 0, output: 0 },
+                        model: usage.model,
+                        harness: "aiw-cli",
+                    });
+                }
+                fs.write(sidecarPath, serializeUsageSidecar(sidecar));
             }
             return {
                 exitCode: 0,
@@ -1922,6 +2049,183 @@ export async function runCommand(args, root, fs, services = {}) {
             fs.write(cachePath, serializeSummaryCache(createSummaryCache(context, summaryArg)));
             return { exitCode: 0, output: summaryArg };
         }
+        if (command === "routing") {
+            if (!fs.exists(join(aiw, "manifest.yml")))
+                return { exitCode: 1, error: "Run `aiw install` first." };
+            const usage = "Usage: aiw routing init | show | resolve --category=name";
+            const action = args[1];
+            const routingPath = join(aiw, ROUTING_TABLE_PATH);
+            if (action === "init") {
+                if (fs.exists(routingPath))
+                    return {
+                        exitCode: 1,
+                        error: `Routing table already exists: .aiw/${ROUTING_TABLE_PATH}. Edit it directly instead of re-initializing.`,
+                    };
+                fs.write(routingPath, serializeRoutingTable(defaultRoutingTable()) +
+                    "# Add one rule per task category (dot-separated kebab-case, e.g. scan.interpreter).\n" +
+                    "# Name the evidence behind every rule — a benchmark, parity run, or ADR — before trusting it.\n" +
+                    "# Routing is advisory: humans can always override the resolved model.\n");
+                return {
+                    exitCode: 0,
+                    output: `Routing table created: .aiw/${ROUTING_TABLE_PATH}. Add rules with named evidence, then resolve with \`aiw routing resolve --category=...\`.`,
+                };
+            }
+            if (action === "show") {
+                if (!fs.exists(routingPath))
+                    return {
+                        exitCode: 1,
+                        error: `No routing table found. Run \`aiw routing init\` to create .aiw/${ROUTING_TABLE_PATH}.`,
+                    };
+                return {
+                    exitCode: 0,
+                    output: serializeRoutingTable(parseRoutingTable(fs.read(routingPath))).trimEnd(),
+                };
+            }
+            if (action === "resolve") {
+                const category = args.find((arg) => arg.startsWith("--category="))?.slice(11);
+                if (!category)
+                    return { exitCode: 1, error: usage };
+                if (!fs.exists(routingPath))
+                    return {
+                        exitCode: 1,
+                        error: `No routing table found. Run \`aiw routing init\` first.`,
+                    };
+                let route;
+                try {
+                    route = resolveRoute(parseRoutingTable(fs.read(routingPath)), category);
+                }
+                catch (error) {
+                    throw new Error(`Routing failed: ${error.message}`, { cause: error });
+                }
+                return {
+                    exitCode: 0,
+                    output: serializeResolvedRoute(route) +
+                        "\noverride: advisory only — a human may choose a different model at any time.",
+                };
+            }
+            return { exitCode: 1, error: usage };
+        }
+        if (command === "task-usage") {
+            if (!fs.exists(join(aiw, "manifest.yml")))
+                return { exitCode: 1, error: "Run `aiw install` first." };
+            const parsed = parseNamedOptions(args.slice(1), ["workflow"]);
+            if (!parsed)
+                return { exitCode: 1, error: "Usage: aiw task-usage [--workflow=id-or-ticket]" };
+            const sidecarPath = join(aiw, USAGE_SIDECAR_PATH);
+            if (!fs.exists(sidecarPath))
+                return {
+                    exitCode: 1,
+                    error: "No usage sidecar recorded yet. Correlate runs with `aiw usage-sidecar record --workflow=<id-or-ticket>` or run `aiw scan`.",
+                };
+            const sessionPath = join(aiw, "workflow.yml");
+            const workflow = parsed.workflow ??
+                (fs.exists(sessionPath)
+                    ? (parseWorkflowSession(fs.read(sessionPath)).ticket ??
+                        parseWorkflowSession(fs.read(sessionPath)).workflowId)
+                    : undefined);
+            if (!workflow)
+                return {
+                    exitCode: 1,
+                    error: "No workflow specified and no active SDD task. Pass --workflow=<id-or-ticket>.",
+                };
+            const summary = summarizeTaskUsage(parseUsageSidecar(fs.read(sidecarPath)), workflow);
+            return {
+                exitCode: 0,
+                output: serializeTaskUsage(summary),
+            };
+        }
+        if (command === "cost-report") {
+            if (!fs.exists(join(aiw, "manifest.yml")))
+                return { exitCode: 1, error: "Run `aiw install` first." };
+            const sidecarPath = join(aiw, USAGE_SIDECAR_PATH);
+            if (!fs.exists(sidecarPath))
+                return {
+                    exitCode: 1,
+                    error: "No usage sidecar recorded yet. Record runs with `aiw usage-sidecar record` or run `aiw scan`.",
+                };
+            const tracePath = join(aiw, "generated/artifacts/traceability.yml");
+            if (!fs.exists(tracePath))
+                return {
+                    exitCode: 1,
+                    error: "Traceability graph is missing. Run `aiw trace` to build it from the approved specification and plan.",
+                };
+            const attribution = computeCostAttribution(parseUsageSidecar(fs.read(sidecarPath)), parseTraceabilityGraph(fs.read(tracePath)));
+            return {
+                exitCode: 0,
+                output: serializeCostAttribution(attribution) +
+                    "\nSources: .aiw/checkpoints/usage-sidecar.yml + .aiw/generated/artifacts/traceability.yml (verified = linked evidence present).",
+            };
+        }
+        if (command === "usage-sidecar") {
+            if (!fs.exists(join(aiw, "manifest.yml")))
+                return { exitCode: 1, error: "Run `aiw install` first." };
+            const usage = "Usage: aiw usage-sidecar record --command=name (--cache-read=N | --fresh-input=N | --cache-write=N | --output=N) [--model=name] [--harness=name] [--harness-version=text] [--estimated] | aiw usage-sidecar report";
+            const action = args[1];
+            const sidecarPath = join(aiw, USAGE_SIDECAR_PATH);
+            if (action === "report") {
+                const entries = fs.exists(sidecarPath) ? parseUsageSidecar(fs.read(sidecarPath)) : [];
+                return {
+                    exitCode: 0,
+                    output: serializeUsageSidecarSummary(summarizeUsageSidecar(entries)) +
+                        (entries.length
+                            ? `\nLedger: .aiw/${USAGE_SIDECAR_PATH} (${entries.length} entries, sequence-ordered, no timestamps)`
+                            : ""),
+                };
+            }
+            if (action === "record") {
+                const recordArgs = args.slice(2).filter((arg) => arg !== "--estimated");
+                if (recordArgs.some((arg) => arg !== "--estimated" && !arg.startsWith("--")))
+                    return { exitCode: 1, error: usage };
+                const parsed = parseNamedOptions(recordArgs, [
+                    "command",
+                    "model",
+                    "harness",
+                    "harness-version",
+                    "requirement",
+                    "workflow",
+                    ...USAGE_TIERS,
+                ]);
+                if (!parsed?.command)
+                    return { exitCode: 1, error: usage };
+                const tierValues = USAGE_TIERS.map((tier) => [tier, parsed[tier]]).filter(([, value]) => value !== undefined);
+                if (!tierValues.length)
+                    return {
+                        exitCode: 1,
+                        error: "At least one token tier is required: --cache-read, --fresh-input, --cache-write, or --output.",
+                    };
+                for (const [, value] of tierValues)
+                    if (!/^\d+$/.test(value))
+                        return { exitCode: 1, error: "Token tiers must be non-negative integers." };
+                const entry = {
+                    command: parsed.command,
+                    label: (args.includes("--estimated") ? "estimated" : "measured"),
+                    tiers: Object.fromEntries(USAGE_TIERS.map((tier) => [tier, Number(parsed[tier] ?? 0)])),
+                    ...(parsed.model ? { model: parsed.model } : {}),
+                    ...(parsed.harness ? { harness: parsed.harness } : {}),
+                    ...(parsed["harness-version"] ? { harnessVersion: parsed["harness-version"] } : {}),
+                    ...(parsed.requirement ? { requirement: parsed.requirement } : {}),
+                    ...(parsed.workflow ? { workflow: parsed.workflow } : {}),
+                };
+                let ledger;
+                try {
+                    ledger = fs.exists(sidecarPath) ? parseUsageSidecar(fs.read(sidecarPath)) : [];
+                }
+                catch (error) {
+                    throw new Error(`Usage sidecar is unreadable: ${error.message}`, {
+                        cause: error,
+                    });
+                }
+                const next = appendUsageSidecarEntry(ledger, entry);
+                fs.write(sidecarPath, serializeUsageSidecar(next));
+                return {
+                    exitCode: 0,
+                    output: `Usage recorded (seq ${next[next.length - 1].seq}, ${entry.label}): ${entry.command} — ${USAGE_TIERS.filter((tier) => entry.tiers[tier] > 0)
+                        .map((tier) => `${tier} ${entry.tiers[tier].toLocaleString("en-US")}`)
+                        .join(", ")}`,
+                };
+            }
+            return { exitCode: 1, error: usage };
+        }
         if (command === "token-usage") {
             const usagePath = join(aiw, "usage.yml");
             const entries = fs.exists(usagePath) ? parseTokenUsage(fs.read(usagePath)) : [];

package/docs/README.md

@@ -12,6 +12,8 @@ AI Workflow is a portable, spec-driven development system for AI coding agents.
 - [Security and trust](security.md)
 - [Quality and testing](quality.md)
 - [Token efficiency](token-efficiency.md)
+- [Token efficiency research](token-efficiency-research.md)
+- [Token efficiency study notes](token-efficiency-study-notes.md)
 - [Project intelligence](project-intelligence.md)
 - [Semantic mapping](semantic-mapping.md)
 - [Decision model evaluation](decision-model-evaluation.md)

package/docs/token-efficiency-research.md

@@ -0,0 +1,135 @@
+# Token Efficiency Research
+
+Research notes and external references for the open problem: **guaranteeing low token consumption without sacrificing output quality**. This document consolidates published methodologies, open-source implementations, and a recommended path for AI Workflow.
+
+> Methodology note: sources were collected via GitHub Search API and web search. Descriptions and savings figures come from the projects' own READMEs and blog posts and have not been independently validated. Last reviewed: October 2026.
+
+## The core problem
+
+Token optimization without a quality constraint is indistinguishable from degradation. Every credible source converges on the same inversion of the problem:
+
+1. **Fix quality first** as a measurable constraint (benchmark of real work).
+2. **Then minimize cost subject to that constraint.**
+
+"Guaranteeing" low consumption is only possible once quality is pinned to a benchmark that an optimization cannot silently regress.
+
+## Reference publications
+
+| Publication                                                                                                                                                                       | Key contribution                                                                                                                                                                                                 | Relevance to AI Workflow                                                                   |
+| --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------ |
+| Uber Engineering — [Running a Software Factory Efficiently at Uber Scale](https://www.uber.com/blog/efficient-software-factory/)                                                  | Cost equation decomposed into six multiplicative, independently measurable terms; sequential driver decomposition with no unexplained residual; outcome-denominated cost (cost per merged PR, per triaged alert) | The measurement model to adopt: decompose, attribute, and denominate cost by unit of value |
+| Anthropic — [Prompt Caching](https://docs.anthropic.com/en/docs/build-with-claude/prompt-caching) and [Token-Saving Updates](https://www.anthropic.com/news/token-saving-updates) | Cache reads at ~0.1x input price; TTL economics (5 min at 1.25x write, 1 h at 2x); thinking blocks cached in tool-use loops                                                                                      | Cache-friendly ordering of system prompt, rules, and project profile                       |
+| Microsoft Azure — [The Economics of Agent Optimization](https://azure.microsoft.com/en-us/blog/the-economics-of-agent-optimization-context-engineering-for-enterprise-ai-agents/) | Context engineering framed as the primary economic lever for enterprise agents                                                                                                                                   | Aligns with product principle 6 (just-in-time context)                                     |
+| LMSYS — [RouteLLM](https://www.lmsys.org/blog/2024-07-01-routellm/)                                                                                                               | Formal cost/quality trade-off for model routing; 2x+ cost reduction with quality held                                                                                                                            | Routing strategy for the provider-neutral `AiProvider` interface                           |
+| GetBeam — [AI Agent Token Cost Optimization](https://getbeam.dev/blog/ai-agent-token-cost-optimization.html)                                                                      | Practical case study with spend decomposition                                                                                                                                                                    | Cross-check on decomposition methodology                                                   |
+
+### Common pattern across publications
+
+Savings never come from a single technique. The pattern is:
+
+1. Measure per task, not per aggregate.
+2. Pin quality with a benchmark built from real work.
+3. Optimize each cost term separately (price/token, tokens/request, requests/turn).
+4. Hold the model constant when attributing savings (behavior shifts with every upgrade).
+
+## Repositories by category
+
+### Context engineering and token reduction
+
+| Repository                                                                                                          | Stars | Notes                                                                                                                                                                 |
+| ------------------------------------------------------------------------------------------------------------------- | ----- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [microsoft/LLMLingua](https://github.com/microsoft/LLMLingua)                                                       | ~6.8k | Prompt compression (EMNLP'23, ACL'24); 5–20x compression. `gladehq/claude-shorthand` applies it to Claude Code (~55% claimed). Last-resort layer due to quality risk. |
+| [ratel-ai/ratel](https://github.com/ratel-ai/ratel)                                                                 | ~484  | Context engineering for agents: ~80% fewer tokens via fixing tool overload; in-process BM25 skills and memory. Architecturally the closest peer.                      |
+| [jia-gao/leanctx](https://github.com/jia-gao/leanctx)                                                               | ~323  | Drop-in prompt compression, 40–60% claimed without app changes.                                                                                                       |
+| [rossoclt/context-guru](https://github.com/rossoclt/context-guru)                                                   | ~70   | Context optimization for coding agents: lower cost and latency "without sacrificing accuracy".                                                                        |
+| [pleasedodisturb/awesome-llm-token-optimization](https://github.com/pleasedodisturb/awesome-llm-token-optimization) | ~99   | **The reference map.** Quick-wins table (cache 90% input, routing 60–95%, batch 50%, compression 5–20x), papers, and tools by category. Read first.                   |
+| [wai-21/llm-token-optimization](https://github.com/wai-21/llm-token-optimization)                                   | ~2    | Decision Distillation, skill routing; 60–99% claimed. Unvalidated.                                                                                                    |
+| [8b-is/smart-tree](https://github.com/8b-is/smart-tree)                                                             | ~271  | Context-aware filesystem tooling replacing 20+ tools; token reduction via tool design.                                                                                |
+
+### Telemetry across harnesses
+
+| Repository                                                                        | Stars | Notes                                                                                                                                                                                            |
+| --------------------------------------------------------------------------------- | ----- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| [pulsemcp/switchboard](https://github.com/pulsemcp/switchboard)                   | new   | Self-hosted capture of agent transcripts (Claude Code, Codex, others). Cross-harness data source without vendor lock-in — the piece Uber gets from its gateway but a portable system must build. |
+| [DeepAgentLabs/agenticlens](https://github.com/DeepAgentLabs/agenticlens)         | ~58   | Open-source profiler: token usage, cost, latency, and optimization opportunities per session.                                                                                                    |
+| [traceloop/openllmetry](https://github.com/traceloop/openllmetry)                 | ~7.5k | GenAI observability on OpenTelemetry. The interoperable schema to adopt if telemetry grows beyond local use.                                                                                     |
+| [gocenalper/agent-optimization](https://github.com/gocenalper/agent-optimization) | ~12   | Live token monitor for Claude Code / Codex: cost tracking, analysis, LLM-powered suggestions.                                                                                                    |
+| [BerriAI/litellm](https://github.com/BerriAI/litellm)                             | high  | Proxy with routing and cost tracking for 100+ providers; candidate neutral measurement layer.                                                                                                    |
+| [timharris707/modeldeck](https://github.com/timharris707/modeldeck)               | ~116  | Usage/quota tracking across Claude Code and Codex accounts.                                                                                                                                      |
+
+### Harnesses, workflows, and SDD
+
+| Repository                                                                          | Stars | Notes                                                                                                           |
+| ----------------------------------------------------------------------------------- | ----- | --------------------------------------------------------------------------------------------------------------- |
+| [github/spec-kit](https://github.com/github/spec-kit)                               | ~142k | De-facto standard for spec-driven development; direct philosophical competitor.                                 |
+| [cisco-open/ai-harness-toolkit](https://github.com/cisco-open/ai-harness-toolkit)   | ~15   | Skills for assessing AI-native engineering workflows in repositories.                                           |
+| [Zoria-Lind/dsh-token-optimizer](https://github.com/Zoria-Lind/dsh-token-optimizer) | ~20   | Layered pipeline: output ladder, MCP lazy loading, compaction driver. Small but architecturally interesting.    |
+| [ComposioHQ/composio](https://github.com/ComposioHQ/composio)                       | ~30k  | 1000+ toolkits with tool search and context management — the "tool search" pattern Uber describes, productized. |
+
+### Benchmarks with quality data (fix quality before optimizing)
+
+| Repository                                                                      | Stars | Notes                                                                                                                                                                                                            |
+| ------------------------------------------------------------------------------- | ----- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [HumphreySun98/repoagentbench](https://github.com/HumphreySun98/repoagentbench) | ~27   | Mines merged PRs into a local, contamination-free coding-agent benchmark. This is Uber's uReview methodology, open-sourced.                                                                                      |
+| [mupt-ai/self-bench](https://github.com/mupt-ai/self-bench)                     | ~33   | Benchmarks agents on your own repository, Harbor-formatted evals.                                                                                                                                                |
+| [jettison-ai/jettison-parity](https://github.com/jettison-ai/jettison-parity)   | new   | Benchmark harness for context-optimization middleware: task parity, session cost, harness-agnostic. Defines exactly the experiment needed: same task, different context configuration, measure cost and quality. |
+
+## Savings reference table
+
+From the awesome-list and Uber's post, ordered by effort-to-savings ratio:
+
+| Lever                                        | Claimed savings        | Quality risk               | Applies at              |
+| -------------------------------------------- | ---------------------- | -------------------------- | ----------------------- |
+| Prompt prefix caching                        | ~90% of cached input   | None (transparent)         | Provider layer          |
+| Tool schema lazy loading / tool search       | 50–70K tokens/session  | Low                        | Harness / MCP layer     |
+| Model routing (cheap model for simple tasks) | 60–95% on routed calls | Medium — needs benchmark   | Provider / orchestrator |
+| Batch APIs for non-urgent work               | ~50%                   | None (same output)         | Provider layer          |
+| Code-mode / batched tool calls               | 55–100% per workflow   | Low                        | Harness layer           |
+| Compaction budget                            | Context-dependent      | Medium                     | Harness layer           |
+| Prompt compression (LLMLingua)               | 5–20x                  | High — degrades edge cases | Last resort, middleware |
+
+Combined pipelines claim 95–99% versus a naive baseline, but every high-risk row requires the quality gate below.
+
+## Recommended path for AI Workflow
+
+A four-step guarantee pipeline, reusing existing pieces:
+
+### Step 1 — Fix quality with a self-benchmark
+
+Mine real SDD tasks (specs, plans, verification results from `.aiw/checkpoints/` and `.aiw/generated/`) into an evaluable set, following the `repoagentbench` / `self-bench` pattern. Define the acceptance rule: **an optimization ships only if outcome does not fall below threshold X% on the benchmark.** AI Workflow already has the ingredients: `evidence.ts`, `traceability.ts`, quality gates, and `src/fixtures`.
+
+### Step 2 — Decompose cost per term
+
+Extend `token-usage.ts` toward Uber-style decomposition, per SDD stage and per skill:
+
+- input vs output vs cached tokens per request;
+- requests per turn and turns per task;
+- cost per **verified requirement** (outcome-denominated), not per session.
+
+Data sources: agent JSONL logs (Claude Code, Codex — the `switchboard` approach) plus AIW checkpoints. Deliver through the existing `dashboard.ts`, evolving it toward anti-pattern detection (Uber reports 16 distinct session anti-patterns; the traces to detect them are already captured).
+
+### Step 3 — Apply levers in ROI order
+
+1. Stable cacheable prefix: system prompt + confirmed profile + rules first; dynamic context last.
+2. Routing: subagents and mechanical skills to cheaper models, manual override preserved (AIW's provider neutrality makes this a configuration, not a rewrite).
+3. Tool overload: lazy loading / on-demand tool schemas for MCP surfaces.
+4. Stage-budgeted compaction (already designed in `token-efficiency.md`).
+5. Prompt compression only behind the quality gate.
+
+### Step 4 — Continuous guardrail
+
+A CI gate that reruns the self-benchmark on changes to models, skills, or configuration and reports Δcost and Δquality. This is the "model held constant" discipline from Uber: savings claims are only attributed when the model is fixed. Fail the gate when quality regresses or cost exceeds threshold.
+
+### Suggested study order
+
+1. `pleasedodisturb/awesome-llm-token-optimization` — full map.
+2. `jettison-ai/jettison-parity` — how to design the cost/quality experiment.
+3. `HumphreySun98/repoagentbench` — how to build the benchmark.
+4. `pulsemcp/switchboard` and `DeepAgentLabs/agenticlens` — cross-harness telemetry layer.
+
+## Gaps and open questions
+
+- **Absolute cost data:** none of the sources publish per-task absolute costs in a form transferable to another codebase; only relative deltas. The self-benchmark must establish its own baseline.
+- **Quality drift across model upgrades:** Uber acknowledges behavior shifts with every upgrade; savings measured against one model do not transfer. The CI guardrail (step 4) is the only mitigation found.
+- **Compression safety:** no source provides a formal guarantee that prompt compression preserves behavior on edge cases; treat it as untrusted until benchmarked.
+- **Telemetry privacy:** most third-party trackers are local-first, but AI Workflow's telemetry consent model (opt-in, minimal payload) should gate any integration with observability backends.

package/docs/token-efficiency-study-notes.md

@@ -0,0 +1,128 @@
+# Token Efficiency Study Notes
+
+Deep-dive notes from studying the five reference repositories recommended in [token-efficiency-research.md](token-efficiency-research.md). Local clones live outside the repository at `../token-research/` (sibling of this repo, not committed).
+
+| Repository                       | Role in the plan                             | Maturity                                                                    |
+| -------------------------------- | -------------------------------------------- | --------------------------------------------------------------------------- |
+| `awesome-llm-token-optimization` | Complete map of levers, tools, and papers    | Curated list; strong curation discipline                                    |
+| `jettison-ai/jettison-parity`    | Cost/quality experiment design               | Small but rigorous: offline, deterministic, 29 tests, honest-labeling rules |
+| `HumphreySun98/repoagentbench`   | How to build a benchmark from real work      | Published on PyPI, real leaderboards (~$11 API spend for 12 runs)           |
+| `pulsemcp/switchboard`           | Cross-harness telemetry architecture         | Early (docs-first repo); architecture doc is the value                      |
+| `DeepAgentLabs/agenticlens`      | Evaluation, diagnosis, and model-fit scoring | Large active codebase (400+ files), versioned releases, research program    |
+
+## 1. awesome-llm-token-optimization — the map
+
+Beyond the levers already captured in the research doc, the full read surfaced:
+
+- **GitHub's own engineering result** ([github.blog](https://github.blog/ai-and-ml/github-copilot/improving-token-efficiency-in-github-agentic-workflows/)): pruning unused MCP tool schemas and swapping MCP calls for the `gh` CLI cut agentic-CI token spend **43–62%** across real workflows. This independently replicates Uber's CLI-resolved MCP lever.
+- **Cockroach Labs**: re-sent context can be **62% of agent inference bills** — quantifies why per-turn context, not generation, is the dominant cost.
+- **ProjectDiscovery**: raising cache hit rate from 7% to 84% across 9.8B cached tokens — the single largest documented saving, achieved purely by prompt-structure discipline (stable prefix).
+- **Anthropic tool search**: 85% token reduction for large tool libraries; token-efficient tool use: 70% output reduction. Both are official provider features, not middleware.
+- **Browser tool efficiency**: accessibility-tree approaches are 10–50x smaller than raw HTML; AI-summarized fetch ~20x better than Playwright MCP (1.5 KB vs 10–33 KB per page). Relevant to any web-driven skill.
+- **Lossless rule-based compression achieves 3–4:1** with no model: strip prose/hedging, preserve numbers/entities/constraints/risks, dense bullets, self-contained 3–5K token sections. This is essentially AI Workflow's existing context strategy, externally validated.
+- **Tokenizer warning**: newer Anthropic tokenizers produce ~30% more tokens for identical text at unchanged per-token prices — model upgrades can silently raise cost without any price change. Any cost attribution must be tokenizer-aware (Uber's "hold the model constant" rule exists for the same reason).
+- **Router research caveat**: a 2026 game-theoretic analysis (arxiv:2602.09902) argues optimal routing is usually _static_ — dynamism mostly adds failure modes. Prefer a small fixed routing table per task category over clever per-request routing.
+
+## 2. jettison-parity — the experiment design
+
+The most directly transferable artifact. It answers cost and quality **together**, deterministically and offline.
+
+### Four benchmark families
+
+| Family           | Measures                                                                                                                                                                     | Transferable idea                                                                                                   |
+| ---------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------- |
+| Standing context | Tokens per turn of persistent context (tool schemas, skill bodies, instruction files), per category, byte-reproducible                                                       | Measure AIW's rendered context (AGENTS.md/CLAUDE.md, skills, rules) the same way — before/after any resource change |
+| Task parity      | Same task run baseline vs optimized; scores completion, correct tool + params, **critical-fact retention**, rounds, tokens. **Exits nonzero on any regression**              | The quality gate: `critical_facts` containment checks that facts which must survive compression actually survive    |
+| Session cost     | Dollars of an N-turn session pricing cache-read / fresh-input / cache-write **separately**, in cache-hit and cache-miss regimes                                              | Naive `tokens_saved × price` overstates savings ~10x on cache-heavy agents — AIW cost math must be tier-aware       |
+| Holdout RCT      | Randomized trial over recorded request logs: sha256 arm assignment, mean tokens/request, 95% CI. Synthetic output is always labeled "synthetic demonstration — not evidence" | The standard for claiming a real-world saving once logs exist                                                       |
+
+### Honest-measurement rules (adopt verbatim)
+
+1. Every number carries a label: `measured` (exact tokenizer / provider price) vs `estimated`; labels propagate pessimistically.
+2. Synthetic is never evidence.
+3. Disclose the config alongside any number (content hash).
+4. Cache-aware dollars (per billing tier).
+5. No network, no clocks — byte-identical reruns.
+
+### Adapter surface
+
+A single method makes any optimizer scoreable: `optimize_request(body, provider) -> body`. AIW's equivalent boundary is the context assembly that produces a session's initial payload.
+
+### Task file schema (quality pin)
+
+```json
+"expected": {
+  "tool": "pg_describe_table",
+  "arguments": {"table": "orders"},
+  "critical_facts": ["Never run a migration", "db-staging.internal.example.com"],
+  "final_answer_contains": "DONE"
+}
+```
+
+Six shipped scenarios cover: tool selection among 20 tools, search→load→call chains, numeric constraints, security rules, output-format requirements, mixed turns.
+
+## 3. repoagentbench — building the benchmark
+
+### Mining pipeline (PR → task)
+
+1. PR title/body/base SHA/diff via `gh`.
+2. Clone at base commit.
+3. Split diff into `solution_tests.patch` + `solution_source.patch`; apply **tests only**, so the starting state is "post-PR tests vs pre-PR source".
+4. Auto-generate `verify.sh` for the detected framework.
+
+### Run protocol (the part to copy)
+
+- Fresh workdir + per-task venv per run.
+- `pre_verify` must **FAIL** (proves the task is broken), agent runs, `post_verify` must **PASS** (proves the fix).
+- Artifacts: `diff.patch`, `agent.log`, `events.jsonl` — replayable and diffable.
+- `mock-fix` oracle agent (applies the known PR diff) validates the harness itself: 3/3 pass.
+
+### Findings that matter
+
+- **The harness is part of the system under test**: same Anthropic model via aider (no-op fix, FAIL) vs native Claude Code (canonical fix, PASS) on the identical PR. Portability claims must be measured per-harness, not assumed.
+- Contamination control: mine PRs post-dating the model's training cutoff (cites arxiv:2512.10218 — frontier models score 3x better on potentially-contaminated benchmarks).
+- Cheap: ~$11 for 12 real-agent runs. A self-benchmark is not expensive.
+
+**AIW translation**: our unit is not a PR but an SDD task — spec → plan → implementation → verification. The analog of "PR tests" is the AIW quality gate / verification evidence; the analog of `pre_verify` is "the requirement has no passing evidence yet".
+
+## 4. switchboard — cross-harness telemetry architecture
+
+Early-stage, docs-first. The architectural pattern is the takeaway:
+
+- **Canonical format ("OpenTranscripts")**: every agent's transcript is converted by a deterministic, versioned, per-source converter into one schema. New agent → new converter; spec change → version bump.
+- **One hard determinism boundary**: raw → canonical is deterministic ETL; only segmentation and outcome scoring are LLM-driven, downstream and optional (MVP makes zero LLM calls).
+- **Hooks, not interception**: a small separately-distributed package per agent pipes sessions to user-owned storage (S3/GCS/local/HTTP behind one interface). The platform never owns the ingestion edge.
+- Sidecar metadata per session: who ran it, trigger, model, token usage, harness + version. Deterministic rollups over the canonical table feed org-level dashboards.
+
+**AIW translation**: AIW checkpoints are already a deterministic, versioned artifact per harness-agnostic run. The missing piece is per-session token/cost sidecar metadata recorded at checkpoint time (model, harness, tokens in/out/cached), which turns existing checkpoints into the `open_transcripts` equivalent without any new ingestion machinery.
+
+## 5. agenticlens — evaluation and model-fit
+
+Large active project (v0.10.0). Most relevant constructs:
+
+- **Evaluator versioning**: every score carries its evaluator's version; evidence audits tie recommendations to the step they concern.
+- **Experiments**: pass@k, pass^k, confidence intervals, significance tests, factor comparison — statistics done properly for agent evals.
+- **Failure taxonomy + attribution**: deterministic diagnosis rules attribute failures to faulty step/agent, validated on controlled failures.
+- **ModelFit**: task taxonomy, **hard constraints on quality/cost/latency/safety**, weighted scoring, per-category **Pareto frontiers with cost-per-success rankings**, recommendation confidence + explanations.
+- **Advisory routing**: turns saved ModelFit evidence into explainable runtime decisions — advisory by default, execution requires explicit opt-in, with a local audit journal. Aligns with the static-routing finding from §1.
+
+**AIW translation**: ModelFit's constraint-then-Pareto structure is exactly step 1–3 of our recommended path; "cost per success" is the outcome-denominated metric from the Uber post, productized.
+
+## Synthesis: implementation blueprint for steps 1–2
+
+Combining the five sources, the sequence with concrete designs:
+
+1. **Standing-context benchmark** (jettison-parity, family 1): a command that renders AIW context for a fixture project and reports tokens per category (root context file, skills, rules, tool schemas) with a content hash. Deterministic, offline, zero API cost. First deliverable.
+2. **Task parity suite** (jettison-parity, family 2 + repoagentbench protocol): SDD tasks from `src/fixtures` with `expected` blocks (required artifacts, `critical_facts` that must survive context filtering, completion markers). Baseline vs optimized context; **exit nonzero on regression**. `pre_verify must fail / post_verify must pass` discipline via existing quality gates.
+3. **Checkpoint sidecar metadata** (switchboard pattern): record model, harness+version, token tiers (cache read / fresh input / cache write / output) into every `.aiw/checkpoints/` entry. Tier-aware, `measured` vs `estimated` labeled.
+4. **Cost-per-verified-requirement** (Uber + agenticlens ModelFit): join step-3 metadata with `evidence.ts`/`traceability.ts` to report cost per unit of value, per SDD stage and per skill.
+5. **Static routing table** (research consensus): a small per-task-category model matrix with hard quality constraints from step 2's benchmark; advisory with manual override, audit-logged.
+6. **Holdout RCT** (jettison-parity, family 4): once real session logs accumulate, upgrade claims from "benchmark delta" to CI-backed RCT evidence.
+
+### Guardrails to adopt wholesale
+
+- Honest labeling (`measured`/`estimated`, pessimistic propagation).
+- Synthetic results are never evidence.
+- Config hash disclosed with every published number.
+- Model (and tokenizer) held constant when attributing savings.
+- Oracle baseline (mock-fix equivalent) to validate the harness itself.