@multileaf/ai-workflow 0.6.1 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/README.md +9 -1
- package/dist/context-benchmark.d.ts +52 -0
- package/dist/context-benchmark.js +170 -0
- package/dist/cost-attribution.d.ts +65 -0
- package/dist/cost-attribution.js +119 -0
- package/dist/human-approval.d.ts +4 -0
- package/dist/human-approval.js +22 -2
- package/dist/interpreter.d.ts +8 -1
- package/dist/interpreter.js +8 -1
- package/dist/model-routing.d.ts +38 -0
- package/dist/model-routing.js +116 -0
- package/dist/recommendations.js +13 -0
- package/dist/task-parity.d.ts +67 -0
- package/dist/task-parity.js +202 -0
- package/dist/token-usage.d.ts +6 -0
- package/dist/token-usage.js +32 -5
- package/dist/traceability.d.ts +2 -0
- package/dist/traceability.js +35 -0
- package/dist/usage-sidecar.d.ts +56 -0
- package/dist/usage-sidecar.js +187 -0
- package/dist/workflow.js +470 -50
- package/docs/README.md +2 -0
- package/docs/telemetry.md +3 -1
- package/docs/token-efficiency-research.md +135 -0
- package/docs/token-efficiency-study-notes.md +128 -0
- package/docs/token-efficiency.md +104 -0
- package/package.json +1 -1
- package/resources/package.yaml +1 -0
- package/resources/skills/ticket.md +18 -0
package/CHANGELOG.md
CHANGED
|
@@ -4,6 +4,25 @@ All notable changes to this project are documented in this file. The format foll
|
|
|
4
4
|
|
|
5
5
|
## [Unreleased]
|
|
6
6
|
|
|
7
|
+
## [0.8.0] - 2026-10-20
|
|
8
|
+
|
|
9
|
+
### Added
|
|
10
|
+
|
|
11
|
+
- Standing-context benchmark: `aiw context-bench` measures the persistent context AI Workflow renders into every agent session (root context file, neutral resources under `.aiw/resources/`, `.aiw/context/` layers) per category, offline and deterministically. Byte and file counts are measured; tokens are estimated (characters/4) and labeled as such; a content hash pins the fixture and the report contains no timestamps. `--save` records a checkpoint under `.aiw/checkpoints/` and `--baseline=<path>` exits nonzero on any token regression, giving context changes a cost gate.
|
|
12
|
+
- Task parity suite: `aiw parity [--tasks=directory]` runs YAML parity tasks (`.aiw/parity/` by default) that pin quality constraints on context assembly. Each task compares the full composed context against the just-in-time task-retrieval assembler and fails, naming the exact fact, when a critical fact or completion marker does not survive retrieval. Exit nonzero on any failure makes it a CI gate for context changes; estimated token cost is reported but never gates here. Inter-stage tasks load real artifacts via `sources` (project-relative paths into any layer) and enforce `required_patterns` — every extracted identifier (e.g. `REQ-\d+` from the approved specification) must survive assembly, guarding the identifiers traceability links depend on.
|
|
13
|
+
- Usage sidecar: `aiw usage-sidecar record/report` maintains a versioned ledger at `.aiw/checkpoints/usage-sidecar.yml` with per-run tokens split into billing tiers (cache-read, fresh-input, cache-write, output), a `measured`/`estimated` label that propagates pessimistically to aggregates, and attribution by model and harness (with version). The report exposes `cache_hit_rate` (cache-read over billable input); dollar conversion is intentionally left out. `aiw scan` auto-records interpreter and embedding usage into the ledger with their real measurement label.
|
|
14
|
+
- Cost attribution: `aiw cost-report` joins the usage sidecar with the traceability graph (`aiw trace`) to report tokens per verified requirement — the outcome-denominated cost metric — with attributed and unattributed tokens shown side by side. Sidecar entries accept `--requirement=REQ-###`, and `parseTraceabilityGraph` round-trips the serialized graph.
|
|
15
|
+
- Model routing table: `aiw routing init|show|resolve --category=name` maintains a static, evidence-linked table at `.aiw/routing.yml`. Rules map task categories to models deterministically; each rule names its evidence, and the resolver reports its advisory level (`evidenced`, `unevidenced-rule`, `default-fallback`, `default-unset`). Routing is advisory only and never invokes a model.
|
|
16
|
+
- Task-lifecycle token usage: `aiw task-usage [--workflow=id-or-ticket]` aggregates sidecar entries correlated by `--workflow` into one task's totals — input tokens (all billing tiers), output tokens, cache hit rate, per-stage breakdown with models, requirements touched, and harness attribution. Tokens are the model-invariant metric, so lifecycle totals stay comparable across model routing changes; `usage-sidecar record` accepts `--workflow` for correlation.
|
|
17
|
+
|
|
18
|
+
## [0.7.0] - 2026-10-13
|
|
19
|
+
|
|
20
|
+
### Added
|
|
21
|
+
|
|
22
|
+
- Ticket-driven task entry point: `aiw task start --ticket=TEAM-123 [--title=text]` starts an SDD session that records the ticket, and `aiw task status` reports the ticket, stage statuses, and the next pending step with ticket-aware guidance. `aiw spec --ticket=TEAM-123` links the specification scaffold to the ticket. A new `ticket` skill (recommended as the `ticket-driven-development` capability) drives the flow in the host agent, keeping the brainstorming skip an explicit human decision.
|
|
23
|
+
- `aiw telemetry show` summarizes buffered events by command and outcome (fill level against the retention cap, per-command success/failure counts); `--verbose` adds each event verbatim. `telemetry status` keeps reporting configuration only.
|
|
24
|
+
- Per-model token usage: `usage.yml` records carry model attribution (`stage|budget|used|model`; legacy three-column rows still parse) and `aiw token-usage` summarizes total consumption by model. The OpenAI-compatible provider reports the response usage block, so interpreter usage is measured (marked `(measured)`) when the endpoint provides it and estimated otherwise; `scan.embedding`, `scan.interpreter`, and `recommend.decision` all record their model.
|
|
25
|
+
|
|
7
26
|
## [0.6.1] - 2026-10-13
|
|
8
27
|
|
|
9
28
|
### Added
|
package/README.md
CHANGED
|
@@ -204,6 +204,13 @@ npx @multileaf/ai-workflow recommend --select=typescript-quality,verification
|
|
|
204
204
|
|
|
205
205
|
# Track a feature from its first artifact through final human approval.
|
|
206
206
|
npx @multileaf/ai-workflow workflow start
|
|
207
|
+
|
|
208
|
+
# Or start directly from a ticket / already-clear requirement (no brainstorm needed).
|
|
209
|
+
npx @multileaf/ai-workflow task start --ticket=PROJ-142 --title="Add export endpoint"
|
|
210
|
+
npx @multileaf/ai-workflow task status
|
|
211
|
+
# Record an accepted brainstorm skip, then link the specification to the ticket:
|
|
212
|
+
npx @multileaf/ai-workflow skip brainstorming --reason="Requirement provided by ticket PROJ-142"
|
|
213
|
+
npx @multileaf/ai-workflow spec --ticket=PROJ-142 --title="Add export endpoint"
|
|
207
214
|
npx @multileaf/ai-workflow brainstorm --title="Feature name"
|
|
208
215
|
npx @multileaf/ai-workflow gate brainstorming
|
|
209
216
|
# Present evidence and wait for a human reply before each approve command.
|
|
@@ -343,7 +350,8 @@ The typed SDK is available from `@multileaf/ai-workflow/plugin-sdk`. It exports
|
|
|
343
350
|
Telemetry is disabled by default and requires explicit consent — offered once during interactive installation with an explicit default of No. Each event contains only the command name and its outcome; never arguments, file paths, or file contents. When telemetry is enabled but no host transport is configured, events accumulate in a rotated local buffer (`.aiw/telemetry-buffer.jsonl`, 200-event cap) that `aiw telemetry status --verbose` can inspect and `aiw telemetry disable` purges.
|
|
344
351
|
|
|
345
352
|
```bash
|
|
346
|
-
npx @multileaf/ai-workflow telemetry
|
|
353
|
+
npx @multileaf/ai-workflow telemetry show # summarized by command and outcome
|
|
354
|
+
npx @multileaf/ai-workflow telemetry show --verbose # events verbatim
|
|
347
355
|
npx @multileaf/ai-workflow telemetry enable
|
|
348
356
|
npx @multileaf/ai-workflow telemetry disable
|
|
349
357
|
```
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import type { FileSystem } from "./types.js";
|
|
2
|
+
/**
|
|
3
|
+
* Standing-context benchmark (deterministic, offline, zero API cost).
|
|
4
|
+
*
|
|
5
|
+
* Measures the persistent context AI Workflow renders into every agent
|
|
6
|
+
* session: the root context file for the active target, neutral resources
|
|
7
|
+
* under `.aiw/resources/`, and composed context layers under `.aiw/context/`.
|
|
8
|
+
*
|
|
9
|
+
* Honest-measurement rules (see docs/token-efficiency-study-notes.md):
|
|
10
|
+
* - byte and file counts are `measured`; token counts are `estimated`
|
|
11
|
+
* (characters/4) and always labeled as such;
|
|
12
|
+
* - the bundle hash pins the exact fixture content behind every number;
|
|
13
|
+
* - output contains no timestamps or randomness, so reruns are
|
|
14
|
+
* byte-identical for unchanged projects.
|
|
15
|
+
*/
|
|
16
|
+
export declare const CONTEXT_BENCH_SCHEMA = 1;
|
|
17
|
+
export declare const ROOT_CONTEXT_FILES: readonly ["AGENTS.md", "CLAUDE.md", "GEMINI.md"];
|
|
18
|
+
export declare const RESOURCE_CATEGORIES: readonly ["skills", "rules", "agents", "hooks", "templates"];
|
|
19
|
+
export type ResourceCategory = (typeof RESOURCE_CATEGORIES)[number];
|
|
20
|
+
export type ContextBenchCategory = ResourceCategory | "root-context" | "context-layers" | "other-resources";
|
|
21
|
+
export declare const CONTEXT_BENCH_CATEGORIES: readonly ContextBenchCategory[];
|
|
22
|
+
export type ContextBenchCategoryMetrics = {
|
|
23
|
+
name: ContextBenchCategory;
|
|
24
|
+
files: number;
|
|
25
|
+
characters: number;
|
|
26
|
+
estimatedTokens: number;
|
|
27
|
+
paths: string[];
|
|
28
|
+
};
|
|
29
|
+
export type ContextBenchReport = {
|
|
30
|
+
schema: number;
|
|
31
|
+
bundleHash: string;
|
|
32
|
+
totalCharacters: number;
|
|
33
|
+
totalEstimatedTokens: number;
|
|
34
|
+
categories: ContextBenchCategoryMetrics[];
|
|
35
|
+
};
|
|
36
|
+
/** Estimate tokens from characters. Always labeled `estimated`, never `measured`. */
|
|
37
|
+
export declare function estimateTokens(text: string): number;
|
|
38
|
+
/** Collect the standing-context report for a project. Deterministic: same files, same bytes, same report. */
|
|
39
|
+
export declare function collectContextBench(root: string, fs: FileSystem): ContextBenchReport;
|
|
40
|
+
export declare function serializeContextBench(report: ContextBenchReport): string;
|
|
41
|
+
/** Parse a previously saved report for baseline comparison. */
|
|
42
|
+
export declare function parseContextBench(content: string): ContextBenchReport;
|
|
43
|
+
export type ContextBenchComparison = {
|
|
44
|
+
bundleChanged: boolean;
|
|
45
|
+
totalTokenDelta: number;
|
|
46
|
+
regressions: string[];
|
|
47
|
+
};
|
|
48
|
+
/**
|
|
49
|
+
* Compare a current report against a saved baseline.
|
|
50
|
+
* Follows the parity-harness rule: any token increase is a regression.
|
|
51
|
+
*/
|
|
52
|
+
export declare function compareContextBench(baseline: ContextBenchReport, current: ContextBenchReport): ContextBenchComparison;
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
/**
|
|
4
|
+
* Standing-context benchmark (deterministic, offline, zero API cost).
|
|
5
|
+
*
|
|
6
|
+
* Measures the persistent context AI Workflow renders into every agent
|
|
7
|
+
* session: the root context file for the active target, neutral resources
|
|
8
|
+
* under `.aiw/resources/`, and composed context layers under `.aiw/context/`.
|
|
9
|
+
*
|
|
10
|
+
* Honest-measurement rules (see docs/token-efficiency-study-notes.md):
|
|
11
|
+
* - byte and file counts are `measured`; token counts are `estimated`
|
|
12
|
+
* (characters/4) and always labeled as such;
|
|
13
|
+
* - the bundle hash pins the exact fixture content behind every number;
|
|
14
|
+
* - output contains no timestamps or randomness, so reruns are
|
|
15
|
+
* byte-identical for unchanged projects.
|
|
16
|
+
*/
|
|
17
|
+
export const CONTEXT_BENCH_SCHEMA = 1;
|
|
18
|
+
export const ROOT_CONTEXT_FILES = ["AGENTS.md", "CLAUDE.md", "GEMINI.md"];
|
|
19
|
+
export const RESOURCE_CATEGORIES = ["skills", "rules", "agents", "hooks", "templates"];
|
|
20
|
+
export const CONTEXT_BENCH_CATEGORIES = [
|
|
21
|
+
"root-context",
|
|
22
|
+
...RESOURCE_CATEGORIES,
|
|
23
|
+
"context-layers",
|
|
24
|
+
"other-resources",
|
|
25
|
+
];
|
|
26
|
+
/** Estimate tokens from characters. Always labeled `estimated`, never `measured`. */
|
|
27
|
+
export function estimateTokens(text) {
|
|
28
|
+
return Math.ceil(text.length / 4);
|
|
29
|
+
}
|
|
30
|
+
function hashBundle(entries) {
|
|
31
|
+
const hash = createHash("sha256");
|
|
32
|
+
for (const entry of entries.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0)))
|
|
33
|
+
hash.update(`${entry.path}\0${Buffer.byteLength(entry.content, "utf8")}\0`);
|
|
34
|
+
return hash.digest("hex").slice(0, 16);
|
|
35
|
+
}
|
|
36
|
+
function emptyCategory(name) {
|
|
37
|
+
return { name, files: 0, characters: 0, estimatedTokens: 0, paths: [] };
|
|
38
|
+
}
|
|
39
|
+
function addFile(categories, name, path, content) {
|
|
40
|
+
const category = categories.get(name) ?? emptyCategory(name);
|
|
41
|
+
category.files += 1;
|
|
42
|
+
category.characters += content.length;
|
|
43
|
+
category.estimatedTokens += estimateTokens(content);
|
|
44
|
+
category.paths.push(path);
|
|
45
|
+
categories.set(name, category);
|
|
46
|
+
}
|
|
47
|
+
/** Collect the standing-context report for a project. Deterministic: same files, same bytes, same report. */
|
|
48
|
+
export function collectContextBench(root, fs) {
|
|
49
|
+
const categories = new Map();
|
|
50
|
+
const bundle = [];
|
|
51
|
+
const record = (name, path, content) => {
|
|
52
|
+
addFile(categories, name, path, content);
|
|
53
|
+
bundle.push({ path, content });
|
|
54
|
+
};
|
|
55
|
+
for (const file of ROOT_CONTEXT_FILES) {
|
|
56
|
+
const path = join(root, file);
|
|
57
|
+
if (fs.pathType?.(path) === "file")
|
|
58
|
+
record("root-context", file, fs.read(path));
|
|
59
|
+
}
|
|
60
|
+
const resourceRoot = join(root, ".aiw/resources");
|
|
61
|
+
if (fs.pathType?.(resourceRoot) === "directory" && fs.listFiles) {
|
|
62
|
+
for (const relative of fs.listFiles(resourceRoot)) {
|
|
63
|
+
const absolute = join(resourceRoot, relative);
|
|
64
|
+
if (fs.pathType?.(absolute) !== "file")
|
|
65
|
+
continue;
|
|
66
|
+
const top = relative.split(/[\\/]/)[0];
|
|
67
|
+
const name = RESOURCE_CATEGORIES.includes(top)
|
|
68
|
+
? top
|
|
69
|
+
: "other-resources";
|
|
70
|
+
record(name, `.aiw/resources/${relative.split("\\").join("/")}`, fs.read(absolute));
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
const contextRoot = join(root, ".aiw/context");
|
|
74
|
+
if (fs.pathType?.(contextRoot) === "directory") {
|
|
75
|
+
for (const file of (fs.list?.(contextRoot) ?? [])
|
|
76
|
+
.filter((name) => name.endsWith(".md"))
|
|
77
|
+
.sort()) {
|
|
78
|
+
const absolute = join(contextRoot, file);
|
|
79
|
+
if (fs.pathType?.(absolute) !== "file")
|
|
80
|
+
continue;
|
|
81
|
+
record("context-layers", `.aiw/context/${file}`, fs.read(absolute));
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
const ordered = CONTEXT_BENCH_CATEGORIES.map((name) => categories.get(name) ?? emptyCategory(name));
|
|
85
|
+
return {
|
|
86
|
+
schema: CONTEXT_BENCH_SCHEMA,
|
|
87
|
+
bundleHash: hashBundle(bundle),
|
|
88
|
+
totalCharacters: ordered.reduce((total, category) => total + category.characters, 0),
|
|
89
|
+
totalEstimatedTokens: ordered.reduce((total, category) => total + category.estimatedTokens, 0),
|
|
90
|
+
categories: ordered,
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
export function serializeContextBench(report) {
|
|
94
|
+
const lines = [
|
|
95
|
+
`schema: ${report.schema}`,
|
|
96
|
+
`bundle_hash: ${report.bundleHash}`,
|
|
97
|
+
`total_characters: ${report.totalCharacters}`,
|
|
98
|
+
`total_estimated_tokens: ${report.totalEstimatedTokens}`,
|
|
99
|
+
"token_estimate: characters/4 (estimated; byte counts are measured)",
|
|
100
|
+
"categories:",
|
|
101
|
+
];
|
|
102
|
+
for (const category of report.categories) {
|
|
103
|
+
lines.push(` - name: ${category.name}`, ` files: ${category.files}`, ` characters: ${category.characters}`, ` estimated_tokens: ${category.estimatedTokens}`);
|
|
104
|
+
if (category.paths.length) {
|
|
105
|
+
lines.push(" paths:");
|
|
106
|
+
lines.push(...category.paths.map((path) => ` - ${path}`));
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
return lines.join("\n") + "\n";
|
|
110
|
+
}
|
|
111
|
+
/** Parse a previously saved report for baseline comparison. */
|
|
112
|
+
export function parseContextBench(content) {
|
|
113
|
+
const lines = content.split("\n");
|
|
114
|
+
if (lines[0]?.trim() !== `schema: ${CONTEXT_BENCH_SCHEMA}`)
|
|
115
|
+
throw new Error("Unrecognized context benchmark report schema.");
|
|
116
|
+
const report = {
|
|
117
|
+
schema: CONTEXT_BENCH_SCHEMA,
|
|
118
|
+
bundleHash: requiredValue(lines, "bundle_hash"),
|
|
119
|
+
totalCharacters: Number(requiredValue(lines, "total_characters")),
|
|
120
|
+
totalEstimatedTokens: Number(requiredValue(lines, "total_estimated_tokens")),
|
|
121
|
+
categories: [],
|
|
122
|
+
};
|
|
123
|
+
let current;
|
|
124
|
+
for (const line of lines.slice(1)) {
|
|
125
|
+
const category = line.match(/^ {2}- name: (.+)$/);
|
|
126
|
+
if (category) {
|
|
127
|
+
current = emptyCategory(category[1]);
|
|
128
|
+
report.categories.push(current);
|
|
129
|
+
continue;
|
|
130
|
+
}
|
|
131
|
+
const path = line.match(/^ {6}- (.+)$/);
|
|
132
|
+
if (path && current)
|
|
133
|
+
current.paths.push(path[1]);
|
|
134
|
+
const match = line.match(/^ {4}(files|characters|estimated_tokens): (\d+)$/);
|
|
135
|
+
if (match && current) {
|
|
136
|
+
const key = match[1] === "estimated_tokens" ? "estimatedTokens" : match[1];
|
|
137
|
+
current[key] = Number(match[2]);
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
if (!report.categories.length)
|
|
141
|
+
throw new Error("Context benchmark report has no categories.");
|
|
142
|
+
return report;
|
|
143
|
+
}
|
|
144
|
+
function requiredValue(lines, key) {
|
|
145
|
+
const value = lines.find((line) => line.startsWith(`${key}: `))?.slice(key.length + 2);
|
|
146
|
+
if (!value)
|
|
147
|
+
throw new Error(`Context benchmark report is missing ${key}.`);
|
|
148
|
+
return value;
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* Compare a current report against a saved baseline.
|
|
152
|
+
* Follows the parity-harness rule: any token increase is a regression.
|
|
153
|
+
*/
|
|
154
|
+
export function compareContextBench(baseline, current) {
|
|
155
|
+
const baselineByCategory = new Map(baseline.categories.map((category) => [category.name, category]));
|
|
156
|
+
const regressions = [];
|
|
157
|
+
for (const category of current.categories) {
|
|
158
|
+
const previous = baselineByCategory.get(category.name);
|
|
159
|
+
if (previous && category.estimatedTokens > previous.estimatedTokens)
|
|
160
|
+
regressions.push(`${category.name}: ${previous.estimatedTokens} -> ${category.estimatedTokens} estimated tokens`);
|
|
161
|
+
}
|
|
162
|
+
const totalTokenDelta = current.totalEstimatedTokens - baseline.totalEstimatedTokens;
|
|
163
|
+
if (totalTokenDelta > 0)
|
|
164
|
+
regressions.push(`total: ${baseline.totalEstimatedTokens} -> ${current.totalEstimatedTokens} estimated tokens`);
|
|
165
|
+
return {
|
|
166
|
+
bundleChanged: baseline.bundleHash !== current.bundleHash,
|
|
167
|
+
totalTokenDelta,
|
|
168
|
+
regressions,
|
|
169
|
+
};
|
|
170
|
+
}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import { type UsageLabel, type UsageSidecarEntry } from "./usage-sidecar.js";
|
|
2
|
+
import type { TraceabilityGraph } from "./traceability.js";
|
|
3
|
+
/**
|
|
4
|
+
* Cost attribution per verified requirement — the outcome-denominated cost
|
|
5
|
+
* metric (Uber's "cost per merged PR", adapted to AI Workflow's unit of
|
|
6
|
+
* value: a requirement with verification evidence).
|
|
7
|
+
*
|
|
8
|
+
* Honesty rules:
|
|
9
|
+
* - tokens attributed to requirements and unattributed tokens are reported
|
|
10
|
+
* side by side, so the ratio cannot be gamed by leaving expensive work
|
|
11
|
+
* unattributed;
|
|
12
|
+
* - the denominator counts only requirements the traceability graph marks
|
|
13
|
+
* as verified (linked evidence present);
|
|
14
|
+
* - labels propagate pessimistically (any estimated entry poisons the total).
|
|
15
|
+
*/
|
|
16
|
+
export type RequirementCost = {
|
|
17
|
+
requirement: string;
|
|
18
|
+
tokens: number;
|
|
19
|
+
label: UsageLabel;
|
|
20
|
+
verified: boolean;
|
|
21
|
+
};
|
|
22
|
+
export type CostAttribution = {
|
|
23
|
+
attributed: RequirementCost[];
|
|
24
|
+
unattributedTokens: number;
|
|
25
|
+
unattributedLabel: UsageLabel;
|
|
26
|
+
verifiedRequirements: number;
|
|
27
|
+
unverifiedRequirements: number;
|
|
28
|
+
totalTokens: number;
|
|
29
|
+
totalLabel: UsageLabel;
|
|
30
|
+
/** Outcome-denominated metric: attributed tokens per verified requirement. */
|
|
31
|
+
tokensPerVerifiedRequirement?: number;
|
|
32
|
+
};
|
|
33
|
+
export declare function computeCostAttribution(entries: UsageSidecarEntry[], graph: TraceabilityGraph): CostAttribution;
|
|
34
|
+
export declare function serializeCostAttribution(attribution: CostAttribution): string;
|
|
35
|
+
/**
|
|
36
|
+
* Task-lifecycle usage: total tokens from task start to merged PR.
|
|
37
|
+
*
|
|
38
|
+
* Tokens — not dollars — are the model-invariant metric: switching between a
|
|
39
|
+
* frontier and an entry-level model changes the price per token but not the
|
|
40
|
+
* input/output consumption, so lifecycle token totals stay comparable across
|
|
41
|
+
* model routing changes and harness migrations. All input tiers count as
|
|
42
|
+
* input tokens processed (cached reads still consume tokens; they just bill
|
|
43
|
+
* less), and the tier split stays visible for cache-behavior analysis.
|
|
44
|
+
*/
|
|
45
|
+
export type TaskUsageStage = {
|
|
46
|
+
stage: string;
|
|
47
|
+
tokens: number;
|
|
48
|
+
label: UsageLabel;
|
|
49
|
+
models: string[];
|
|
50
|
+
};
|
|
51
|
+
export type TaskUsageSummary = {
|
|
52
|
+
workflow: string;
|
|
53
|
+
entries: number;
|
|
54
|
+
inputTokens: number;
|
|
55
|
+
outputTokens: number;
|
|
56
|
+
totalTokens: number;
|
|
57
|
+
label: UsageLabel;
|
|
58
|
+
cacheHitRate?: number;
|
|
59
|
+
stages: TaskUsageStage[];
|
|
60
|
+
requirements: string[];
|
|
61
|
+
models: string[];
|
|
62
|
+
harnesses: string[];
|
|
63
|
+
};
|
|
64
|
+
export declare function summarizeTaskUsage(entries: UsageSidecarEntry[], workflow: string): TaskUsageSummary;
|
|
65
|
+
export declare function serializeTaskUsage(summary: TaskUsageSummary): string;
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
import { USAGE_TIERS } from "./usage-sidecar.js";
|
|
2
|
+
const totalTokens = (entry) => USAGE_TIERS.reduce((total, tier) => total + entry.tiers[tier], 0);
|
|
3
|
+
export function computeCostAttribution(entries, graph) {
|
|
4
|
+
const verifiedByRequirement = new Map(graph.links.map((link) => [link.requirement, link.evidence.length > 0]));
|
|
5
|
+
const byRequirement = new Map();
|
|
6
|
+
const unattributed = [];
|
|
7
|
+
for (const entry of entries) {
|
|
8
|
+
if (entry.requirement) {
|
|
9
|
+
const bucket = byRequirement.get(entry.requirement) ?? [];
|
|
10
|
+
bucket.push(entry);
|
|
11
|
+
byRequirement.set(entry.requirement, bucket);
|
|
12
|
+
}
|
|
13
|
+
else
|
|
14
|
+
unattributed.push(entry);
|
|
15
|
+
}
|
|
16
|
+
const labelOf = (group) => group.every((entry) => entry.label === "measured") ? "measured" : "estimated";
|
|
17
|
+
const attributed = [...byRequirement]
|
|
18
|
+
.map(([requirement, group]) => ({
|
|
19
|
+
requirement,
|
|
20
|
+
tokens: group.reduce((total, entry) => total + totalTokens(entry), 0),
|
|
21
|
+
label: labelOf(group),
|
|
22
|
+
verified: verifiedByRequirement.get(requirement) ?? false,
|
|
23
|
+
}))
|
|
24
|
+
.sort((a, b) => b.tokens - a.tokens || a.requirement.localeCompare(b.requirement));
|
|
25
|
+
const graphRequirements = new Set(graph.links.map((link) => link.requirement));
|
|
26
|
+
const verifiedRequirements = attributed.filter(({ verified }) => verified).length;
|
|
27
|
+
const unverifiedRequirements = attributed.filter(({ requirement }) => graphRequirements.has(requirement)).length -
|
|
28
|
+
verifiedRequirements;
|
|
29
|
+
const attributedToVerified = attributed
|
|
30
|
+
.filter(({ verified }) => verified)
|
|
31
|
+
.reduce((total, entry) => total + entry.tokens, 0);
|
|
32
|
+
return {
|
|
33
|
+
attributed,
|
|
34
|
+
unattributedTokens: unattributed.reduce((total, entry) => total + totalTokens(entry), 0),
|
|
35
|
+
unattributedLabel: labelOf(unattributed),
|
|
36
|
+
verifiedRequirements,
|
|
37
|
+
unverifiedRequirements,
|
|
38
|
+
totalTokens: entries.reduce((total, entry) => total + totalTokens(entry), 0),
|
|
39
|
+
totalLabel: labelOf(entries),
|
|
40
|
+
...(verifiedRequirements > 0
|
|
41
|
+
? { tokensPerVerifiedRequirement: Math.ceil(attributedToVerified / verifiedRequirements) }
|
|
42
|
+
: {}),
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
export function serializeCostAttribution(attribution) {
|
|
46
|
+
if (!attribution.attributed.length && !attribution.unattributedTokens)
|
|
47
|
+
return "No usage recorded against requirements yet. Record with `aiw usage-sidecar record --requirement=REQ-###`.";
|
|
48
|
+
const lines = [
|
|
49
|
+
`total_tokens: ${attribution.totalTokens.toLocaleString("en-US")} (${attribution.totalLabel})`,
|
|
50
|
+
`unattributed_tokens: ${attribution.unattributedTokens.toLocaleString("en-US")} (${attribution.unattributedLabel})`,
|
|
51
|
+
`verified_requirements: ${attribution.verifiedRequirements}`,
|
|
52
|
+
`unverified_requirements: ${attribution.unverifiedRequirements}`,
|
|
53
|
+
...(attribution.tokensPerVerifiedRequirement === undefined
|
|
54
|
+
? ["tokens_per_verified_requirement: n/a (no verified requirements yet)"]
|
|
55
|
+
: [
|
|
56
|
+
`tokens_per_verified_requirement: ${attribution.tokensPerVerifiedRequirement.toLocaleString("en-US")} (estimated; outcome-denominated)`,
|
|
57
|
+
]),
|
|
58
|
+
"by requirement:",
|
|
59
|
+
...attribution.attributed.map(({ requirement, tokens, label, verified }) => ` - ${requirement}: ${tokens.toLocaleString("en-US")} tokens (${label}, ${verified ? "verified" : "unverified"})`),
|
|
60
|
+
];
|
|
61
|
+
return lines.join("\n");
|
|
62
|
+
}
|
|
63
|
+
export function summarizeTaskUsage(entries, workflow) {
|
|
64
|
+
const scoped = entries.filter((entry) => entry.workflow === workflow);
|
|
65
|
+
const labelOf = (group) => group.every((entry) => entry.label === "measured") ? "measured" : "estimated";
|
|
66
|
+
const inputTokens = scoped.reduce((total, entry) => total + entry.tiers["cache-read"] + entry.tiers["fresh-input"] + entry.tiers["cache-write"], 0);
|
|
67
|
+
const billableInput = scoped.reduce((total, entry) => total + entry.tiers["cache-read"] + entry.tiers["fresh-input"], 0);
|
|
68
|
+
const byCommand = new Map();
|
|
69
|
+
for (const entry of scoped) {
|
|
70
|
+
const bucket = byCommand.get(entry.command) ?? [];
|
|
71
|
+
bucket.push(entry);
|
|
72
|
+
byCommand.set(entry.command, bucket);
|
|
73
|
+
}
|
|
74
|
+
const stages = [...byCommand]
|
|
75
|
+
.map(([stage, group]) => ({
|
|
76
|
+
stage,
|
|
77
|
+
tokens: group.reduce((total, entry) => total + totalTokens(entry), 0),
|
|
78
|
+
label: labelOf(group),
|
|
79
|
+
models: [...new Set(group.map((entry) => entry.model ?? "unattributed"))].sort(),
|
|
80
|
+
}))
|
|
81
|
+
.sort((a, b) => b.tokens - a.tokens || a.stage.localeCompare(b.stage));
|
|
82
|
+
const uniqueSorted = (values) => [...new Set(values.filter((value) => value !== undefined))].sort();
|
|
83
|
+
return {
|
|
84
|
+
workflow,
|
|
85
|
+
entries: scoped.length,
|
|
86
|
+
inputTokens,
|
|
87
|
+
outputTokens: scoped.reduce((total, entry) => total + entry.tiers.output, 0),
|
|
88
|
+
totalTokens: scoped.reduce((total, entry) => total + totalTokens(entry), 0),
|
|
89
|
+
label: labelOf(scoped),
|
|
90
|
+
...(billableInput > 0
|
|
91
|
+
? {
|
|
92
|
+
cacheHitRate: scoped.reduce((total, entry) => total + entry.tiers["cache-read"], 0) / billableInput,
|
|
93
|
+
}
|
|
94
|
+
: {}),
|
|
95
|
+
stages,
|
|
96
|
+
requirements: uniqueSorted(scoped.map((entry) => entry.requirement)),
|
|
97
|
+
models: uniqueSorted(scoped.map((entry) => entry.model)),
|
|
98
|
+
harnesses: uniqueSorted(scoped.map((entry) => entry.harness)),
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
export function serializeTaskUsage(summary) {
|
|
102
|
+
if (!summary.entries)
|
|
103
|
+
return `No usage recorded for workflow ${summary.workflow}. Correlate runs with \`aiw usage-sidecar record --workflow=${summary.workflow}\`.`;
|
|
104
|
+
return [
|
|
105
|
+
`workflow: ${summary.workflow} (${summary.entries} entries, ${summary.label})`,
|
|
106
|
+
`input_tokens: ${summary.inputTokens.toLocaleString("en-US")} (cache-read + fresh-input + cache-write)`,
|
|
107
|
+
`output_tokens: ${summary.outputTokens.toLocaleString("en-US")}`,
|
|
108
|
+
`total_tokens: ${summary.totalTokens.toLocaleString("en-US")}`,
|
|
109
|
+
...(summary.cacheHitRate === undefined
|
|
110
|
+
? []
|
|
111
|
+
: [`cache_hit_rate: ${Math.round(summary.cacheHitRate * 100)}%`]),
|
|
112
|
+
...(summary.models.length ? [`models: ${summary.models.join(", ")}`] : []),
|
|
113
|
+
...(summary.harnesses.length ? [`harnesses: ${summary.harnesses.join(", ")}`] : []),
|
|
114
|
+
...(summary.requirements.length ? [`requirements: ${summary.requirements.join(", ")}`] : []),
|
|
115
|
+
"by stage:",
|
|
116
|
+
...summary.stages.map(({ stage, tokens, label, models }) => ` - ${stage}: ${tokens.toLocaleString("en-US")} tokens (${label}${models.length ? `; ${models.join(", ")}` : ""})`),
|
|
117
|
+
"metric: tokens are model-invariant — comparable across model routing changes and harness migrations.",
|
|
118
|
+
].join("\n");
|
|
119
|
+
}
|
package/dist/human-approval.d.ts
CHANGED
|
@@ -20,7 +20,10 @@ export type WorkflowSession = {
|
|
|
20
20
|
status: "active" | "complete";
|
|
21
21
|
startedAt: string;
|
|
22
22
|
updatedAt: string;
|
|
23
|
+
ticket?: string;
|
|
24
|
+
title?: string;
|
|
23
25
|
};
|
|
26
|
+
export declare function validateTaskTicket(ticket: string): void;
|
|
24
27
|
export type ApprovalStageDefinition = {
|
|
25
28
|
optional: boolean;
|
|
26
29
|
requiresQualityGate: boolean;
|
|
@@ -49,3 +52,4 @@ export declare function skipOptionalStage(ledger: ApprovalLedger, stage: HumanAp
|
|
|
49
52
|
export declare function invalidateStageAndDescendants(ledger: ApprovalLedger, stage: HumanApprovalStage, updatedAt: string): ApprovalLedger;
|
|
50
53
|
export declare function serializeApprovalLedger(ledger: ApprovalLedger): string;
|
|
51
54
|
export declare function parseApprovalLedger(content: string): ApprovalLedger;
|
|
55
|
+
export declare function assertWorkflowId(value: string): void;
|
package/dist/human-approval.js
CHANGED
|
@@ -2,6 +2,10 @@ import { createHash } from "node:crypto";
|
|
|
2
2
|
import { parseDocument, stringify } from "yaml";
|
|
3
3
|
import { HUMAN_APPROVAL_STAGES } from "./types.js";
|
|
4
4
|
export const APPROVAL_STAGES = HUMAN_APPROVAL_STAGES;
|
|
5
|
+
export function validateTaskTicket(ticket) {
|
|
6
|
+
if (!/^[A-Z][A-Z0-9_]*-\d+$/.test(ticket))
|
|
7
|
+
throw new Error("Task ticket must match TEAM-123, for example PROJ-142.");
|
|
8
|
+
}
|
|
5
9
|
const STAGE_DEFINITIONS = {
|
|
6
10
|
brainstorming: { optional: true, requiresQualityGate: true },
|
|
7
11
|
specification: { optional: false, requiresQualityGate: true },
|
|
@@ -22,7 +26,15 @@ const APPROVAL_RECORD_KEYS = new Set([
|
|
|
22
26
|
"updatedAt",
|
|
23
27
|
]);
|
|
24
28
|
const LEDGER_KEYS = new Set(["schema", "workflowId", "stages"]);
|
|
25
|
-
const SESSION_KEYS = new Set([
|
|
29
|
+
const SESSION_KEYS = new Set([
|
|
30
|
+
"schema",
|
|
31
|
+
"workflowId",
|
|
32
|
+
"status",
|
|
33
|
+
"startedAt",
|
|
34
|
+
"updatedAt",
|
|
35
|
+
"ticket",
|
|
36
|
+
"title",
|
|
37
|
+
]);
|
|
26
38
|
export function createApprovalLedger(workflowId) {
|
|
27
39
|
assertWorkflowId(workflowId);
|
|
28
40
|
return { schema: 1, workflowId, stages: {} };
|
|
@@ -47,12 +59,20 @@ export function parseWorkflowSession(content) {
|
|
|
47
59
|
throw new Error("Workflow session status must be active or complete.");
|
|
48
60
|
if (typeof value.startedAt !== "string" || typeof value.updatedAt !== "string")
|
|
49
61
|
throw new Error("Workflow session requires start and update timestamps.");
|
|
62
|
+
if (value.ticket !== undefined && typeof value.ticket !== "string")
|
|
63
|
+
throw new Error("Workflow session ticket must be a string.");
|
|
64
|
+
if (value.title !== undefined && typeof value.title !== "string")
|
|
65
|
+
throw new Error("Workflow session title must be a string.");
|
|
66
|
+
if (typeof value.ticket === "string")
|
|
67
|
+
validateTaskTicket(value.ticket);
|
|
50
68
|
const session = {
|
|
51
69
|
schema: 1,
|
|
52
70
|
workflowId: value.workflowId,
|
|
53
71
|
status: value.status,
|
|
54
72
|
startedAt: value.startedAt,
|
|
55
73
|
updatedAt: value.updatedAt,
|
|
74
|
+
...(value.ticket === undefined ? {} : { ticket: value.ticket }),
|
|
75
|
+
...(value.title === undefined ? {} : { title: value.title }),
|
|
56
76
|
};
|
|
57
77
|
validateWorkflowSession(session);
|
|
58
78
|
return session;
|
|
@@ -189,7 +209,7 @@ function assertTimestamp(value) {
|
|
|
189
209
|
!Number.isFinite(Date.parse(value)))
|
|
190
210
|
throw new Error("Approval timestamp must be a valid date-time.");
|
|
191
211
|
}
|
|
192
|
-
function assertWorkflowId(value) {
|
|
212
|
+
export function assertWorkflowId(value) {
|
|
193
213
|
if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/.test(value))
|
|
194
214
|
throw new Error("Workflow ID must be 1–64 characters using letters, numbers, dot, underscore, or hyphen.");
|
|
195
215
|
}
|
package/dist/interpreter.d.ts
CHANGED
|
@@ -10,11 +10,16 @@ export type InterpretationRequest = {
|
|
|
10
10
|
export type Interpreter = {
|
|
11
11
|
interpret(request: InterpretationRequest): Promise<EvidenceFact[]>;
|
|
12
12
|
};
|
|
13
|
+
export type AiProviderUsage = {
|
|
14
|
+
promptTokens: number;
|
|
15
|
+
completionTokens: number;
|
|
16
|
+
};
|
|
13
17
|
export type AiProviderRequest = {
|
|
14
18
|
system: string;
|
|
15
19
|
prompt: string;
|
|
16
20
|
maxTokens: number;
|
|
17
21
|
responseFormat: "json";
|
|
22
|
+
onUsage?: (usage: AiProviderUsage) => void;
|
|
18
23
|
};
|
|
19
24
|
export type AiProvider = {
|
|
20
25
|
generate(request: AiProviderRequest): Promise<string>;
|
|
@@ -35,5 +40,7 @@ export declare function buildInterpreterQuery(profile: ProjectProfile): string;
|
|
|
35
40
|
type AiFetchLike = (url: string, init?: RequestInit) => Promise<Response>;
|
|
36
41
|
export declare function loadAiProviderConfig(fs: FileSystem, aiwPath: string): AiProviderConfig | undefined;
|
|
37
42
|
export declare function createOpenAiCompatibleProvider(config: AiProviderConfig, fetchImpl?: AiFetchLike, env?: Readonly<Record<string, string | undefined>>): AiProvider;
|
|
38
|
-
export declare function createProjectInterpreter(root: string, provider: AiProvider, contextReader?: ContextReader
|
|
43
|
+
export declare function createProjectInterpreter(root: string, provider: AiProvider, contextReader?: ContextReader, options?: {
|
|
44
|
+
onUsage?: (usage: AiProviderUsage) => void;
|
|
45
|
+
}): Interpreter;
|
|
39
46
|
export {};
|
package/dist/interpreter.js
CHANGED
|
@@ -212,6 +212,12 @@ export function createOpenAiCompatibleProvider(config, fetchImpl = (url, init) =
|
|
|
212
212
|
if (!response.ok)
|
|
213
213
|
throw new Error(`AI provider request failed for model ${config.model}: ${await response.text()}`);
|
|
214
214
|
const body = (await response.json());
|
|
215
|
+
if (typeof body.usage?.prompt_tokens === "number" &&
|
|
216
|
+
typeof body.usage?.completion_tokens === "number")
|
|
217
|
+
request.onUsage?.({
|
|
218
|
+
promptTokens: body.usage.prompt_tokens,
|
|
219
|
+
completionTokens: body.usage.completion_tokens,
|
|
220
|
+
});
|
|
215
221
|
const content = body.choices?.[0]?.message?.content;
|
|
216
222
|
if (typeof content !== "string")
|
|
217
223
|
throw new Error(`AI provider ${config.model} returned no message content`);
|
|
@@ -219,7 +225,7 @@ export function createOpenAiCompatibleProvider(config, fetchImpl = (url, init) =
|
|
|
219
225
|
},
|
|
220
226
|
};
|
|
221
227
|
}
|
|
222
|
-
export function createProjectInterpreter(root, provider, contextReader = async (path) => new TextDecoder("utf-8", { fatal: true }).decode(await readFile(path))) {
|
|
228
|
+
export function createProjectInterpreter(root, provider, contextReader = async (path) => new TextDecoder("utf-8", { fatal: true }).decode(await readFile(path)), options = {}) {
|
|
223
229
|
const projectRoot = resolve(root);
|
|
224
230
|
return {
|
|
225
231
|
async interpret(request) {
|
|
@@ -261,6 +267,7 @@ export function createProjectInterpreter(root, provider, contextReader = async (
|
|
|
261
267
|
prompt: `Detected profile:\n${safeProfile}\n\nScoped project context:\n${sections.join("\n\n")}`,
|
|
262
268
|
maxTokens: scoped.maxTokens,
|
|
263
269
|
responseFormat: "json",
|
|
270
|
+
...(options.onUsage ? { onUsage: options.onUsage } : {}),
|
|
264
271
|
});
|
|
265
272
|
return parseFacts(output, new Set(scoped.files));
|
|
266
273
|
},
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Static model routing table — advisory, versioned, evidence-linked.
|
|
3
|
+
*
|
|
4
|
+
* Research consensus (docs/token-efficiency-study-notes.md §1 and §5):
|
|
5
|
+
* optimal routing is usually static — per-request dynamism mostly adds
|
|
6
|
+
* failure modes. Rules follow the ModelFit discipline: every recommendation
|
|
7
|
+
* names the evidence behind it, execution is advisory only, and humans can
|
|
8
|
+
* always override. The table never invokes a model; it answers "which model
|
|
9
|
+
* should this category use, and why".
|
|
10
|
+
*/
|
|
11
|
+
export declare const ROUTING_TABLE_SCHEMA = 1;
|
|
12
|
+
export declare const ROUTING_TABLE_PATH = "routing.yml";
|
|
13
|
+
export declare const ROUTING_DEFAULT_UNSET = "unset";
|
|
14
|
+
export type RoutingRule = {
|
|
15
|
+
category: string;
|
|
16
|
+
model: string;
|
|
17
|
+
/** Named evidence (benchmark, parity run, ADR) backing this rule. Empty means unevidenced. */
|
|
18
|
+
evidence: string;
|
|
19
|
+
};
|
|
20
|
+
export type RoutingTable = {
|
|
21
|
+
schema: number;
|
|
22
|
+
default: string;
|
|
23
|
+
rules: RoutingRule[];
|
|
24
|
+
};
|
|
25
|
+
export type ResolvedRoute = {
|
|
26
|
+
category: string;
|
|
27
|
+
model: string;
|
|
28
|
+
matchedRule: boolean;
|
|
29
|
+
evidence?: string;
|
|
30
|
+
advisory: "evidenced" | "unevidenced-rule" | "default-fallback" | "default-unset";
|
|
31
|
+
};
|
|
32
|
+
export declare function validateRoutingTable(table: RoutingTable): RoutingTable;
|
|
33
|
+
export declare function defaultRoutingTable(): RoutingTable;
|
|
34
|
+
export declare function parseRoutingTable(content: string): RoutingTable;
|
|
35
|
+
export declare function serializeRoutingTable(table: RoutingTable): string;
|
|
36
|
+
/** Resolve a category to a model. Deterministic: same table, same category, same answer. */
|
|
37
|
+
export declare function resolveRoute(table: RoutingTable, category: string): ResolvedRoute;
|
|
38
|
+
export declare function serializeResolvedRoute(route: ResolvedRoute): string;
|