@multileaf/ai-workflow 0.6.1 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/README.md +9 -1
- package/dist/context-benchmark.d.ts +52 -0
- package/dist/context-benchmark.js +170 -0
- package/dist/cost-attribution.d.ts +65 -0
- package/dist/cost-attribution.js +119 -0
- package/dist/human-approval.d.ts +4 -0
- package/dist/human-approval.js +22 -2
- package/dist/interpreter.d.ts +8 -1
- package/dist/interpreter.js +8 -1
- package/dist/model-routing.d.ts +38 -0
- package/dist/model-routing.js +116 -0
- package/dist/recommendations.js +13 -0
- package/dist/task-parity.d.ts +67 -0
- package/dist/task-parity.js +202 -0
- package/dist/token-usage.d.ts +6 -0
- package/dist/token-usage.js +32 -5
- package/dist/traceability.d.ts +2 -0
- package/dist/traceability.js +35 -0
- package/dist/usage-sidecar.d.ts +56 -0
- package/dist/usage-sidecar.js +187 -0
- package/dist/workflow.js +470 -50
- package/docs/README.md +2 -0
- package/docs/telemetry.md +3 -1
- package/docs/token-efficiency-research.md +135 -0
- package/docs/token-efficiency-study-notes.md +128 -0
- package/docs/token-efficiency.md +104 -0
- package/package.json +1 -1
- package/resources/package.yaml +1 -0
- package/resources/skills/ticket.md +18 -0
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
import { parseDocument } from "yaml";
|
|
2
|
+
/**
|
|
3
|
+
* Static model routing table — advisory, versioned, evidence-linked.
|
|
4
|
+
*
|
|
5
|
+
* Research consensus (docs/token-efficiency-study-notes.md §1 and §5):
|
|
6
|
+
* optimal routing is usually static — per-request dynamism mostly adds
|
|
7
|
+
* failure modes. Rules follow the ModelFit discipline: every recommendation
|
|
8
|
+
* names the evidence behind it, execution is advisory only, and humans can
|
|
9
|
+
* always override. The table never invokes a model; it answers "which model
|
|
10
|
+
* should this category use, and why".
|
|
11
|
+
*/
|
|
12
|
+
export const ROUTING_TABLE_SCHEMA = 1;
|
|
13
|
+
export const ROUTING_TABLE_PATH = "routing.yml";
|
|
14
|
+
export const ROUTING_DEFAULT_UNSET = "unset";
|
|
15
|
+
const CATEGORY_PATTERN = /^[a-z][a-z0-9-]*(?:\.[a-z][a-z0-9-]*)*$/;
|
|
16
|
+
function validateRule(rule) {
|
|
17
|
+
if (!CATEGORY_PATTERN.test(rule.category))
|
|
18
|
+
throw new Error(`Routing category must be dot-separated kebab-case: ${rule.category}.`);
|
|
19
|
+
if (!rule.model.trim())
|
|
20
|
+
throw new Error(`Routing rule for ${rule.category} needs a model.`);
|
|
21
|
+
}
|
|
22
|
+
export function validateRoutingTable(table) {
|
|
23
|
+
if (table.schema !== ROUTING_TABLE_SCHEMA)
|
|
24
|
+
throw new Error(`Routing table schema must be version ${ROUTING_TABLE_SCHEMA}.`);
|
|
25
|
+
if (!table.default.trim())
|
|
26
|
+
throw new Error("Routing table default model must not be empty.");
|
|
27
|
+
table.rules.forEach(validateRule);
|
|
28
|
+
const categories = table.rules.map((rule) => rule.category);
|
|
29
|
+
if (new Set(categories).size !== categories.length)
|
|
30
|
+
throw new Error("Routing table categories must be unique.");
|
|
31
|
+
return table;
|
|
32
|
+
}
|
|
33
|
+
export function defaultRoutingTable() {
|
|
34
|
+
return validateRoutingTable({
|
|
35
|
+
schema: ROUTING_TABLE_SCHEMA,
|
|
36
|
+
default: ROUTING_DEFAULT_UNSET,
|
|
37
|
+
rules: [],
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
export function parseRoutingTable(content) {
|
|
41
|
+
const document = parseDocument(content);
|
|
42
|
+
if (document.errors.length)
|
|
43
|
+
throw new Error("Routing table is not valid YAML.");
|
|
44
|
+
const schema = document.get("schema");
|
|
45
|
+
if (schema !== ROUTING_TABLE_SCHEMA)
|
|
46
|
+
throw new Error(`Routing table schema must be version ${ROUTING_TABLE_SCHEMA}.`);
|
|
47
|
+
const pick = (key) => document.get(key);
|
|
48
|
+
const rulesNode = pick("rules");
|
|
49
|
+
const rules = [];
|
|
50
|
+
if (rulesNode !== null && rulesNode !== undefined) {
|
|
51
|
+
const items = Array.isArray(rulesNode)
|
|
52
|
+
? rulesNode
|
|
53
|
+
: typeof rulesNode.items === "object"
|
|
54
|
+
? rulesNode.items
|
|
55
|
+
: undefined;
|
|
56
|
+
if (items === undefined)
|
|
57
|
+
throw new Error("Routing table rules must be a list.");
|
|
58
|
+
for (const raw of items) {
|
|
59
|
+
const read = (key) => typeof raw.get === "function"
|
|
60
|
+
? raw.get(key)
|
|
61
|
+
: raw[key];
|
|
62
|
+
rules.push({
|
|
63
|
+
category: String(read("category")),
|
|
64
|
+
model: String(read("model")),
|
|
65
|
+
evidence: read("evidence") === null || read("evidence") === undefined
|
|
66
|
+
? ""
|
|
67
|
+
: String(read("evidence")),
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
return validateRoutingTable({
|
|
72
|
+
schema: ROUTING_TABLE_SCHEMA,
|
|
73
|
+
default: String(pick("default") ?? ""),
|
|
74
|
+
rules,
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
export function serializeRoutingTable(table) {
|
|
78
|
+
validateRoutingTable(table);
|
|
79
|
+
const lines = [`schema: ${table.schema}`, `default: ${table.default}`, "rules:"];
|
|
80
|
+
if (!table.rules.length)
|
|
81
|
+
lines.push(" []");
|
|
82
|
+
for (const rule of table.rules) {
|
|
83
|
+
lines.push(` - category: ${rule.category}`, ` model: ${rule.model}`, ` evidence: ${rule.evidence ? JSON.stringify(rule.evidence) : '""'}`);
|
|
84
|
+
}
|
|
85
|
+
return lines.join("\n") + "\n";
|
|
86
|
+
}
|
|
87
|
+
/** Resolve a category to a model. Deterministic: same table, same category, same answer. */
|
|
88
|
+
export function resolveRoute(table, category) {
|
|
89
|
+
if (!CATEGORY_PATTERN.test(category))
|
|
90
|
+
throw new Error(`Routing category must be dot-separated kebab-case: ${category}.`);
|
|
91
|
+
const rule = table.rules.find((candidate) => candidate.category === category);
|
|
92
|
+
if (rule)
|
|
93
|
+
return {
|
|
94
|
+
category,
|
|
95
|
+
model: rule.model,
|
|
96
|
+
matchedRule: true,
|
|
97
|
+
...(rule.evidence ? { evidence: rule.evidence } : {}),
|
|
98
|
+
advisory: rule.evidence ? "evidenced" : "unevidenced-rule",
|
|
99
|
+
};
|
|
100
|
+
if (table.default === ROUTING_DEFAULT_UNSET)
|
|
101
|
+
return { category, model: table.default, matchedRule: false, advisory: "default-unset" };
|
|
102
|
+
return { category, model: table.default, matchedRule: false, advisory: "default-fallback" };
|
|
103
|
+
}
|
|
104
|
+
export function serializeResolvedRoute(route) {
|
|
105
|
+
const advisoryText = {
|
|
106
|
+
evidenced: `advisory: rule is evidence-backed (${route.evidence})`,
|
|
107
|
+
"unevidenced-rule": "advisory: rule has no named evidence; benchmark it before trusting the routing",
|
|
108
|
+
"default-fallback": "advisory: no rule for this category; fell back to the table default",
|
|
109
|
+
"default-unset": "advisory: no rule for this category and the default is unset; a human must choose the model",
|
|
110
|
+
};
|
|
111
|
+
return [
|
|
112
|
+
`category: ${route.category}`,
|
|
113
|
+
`model: ${route.model}`,
|
|
114
|
+
advisoryText[route.advisory],
|
|
115
|
+
].join("\n");
|
|
116
|
+
}
|
package/dist/recommendations.js
CHANGED
|
@@ -14,6 +14,19 @@ const CATALOG = [
|
|
|
14
14
|
? { evidence: projectEvidence(profile), confidence: 0.92 }
|
|
15
15
|
: undefined,
|
|
16
16
|
},
|
|
17
|
+
{
|
|
18
|
+
id: "ticket-driven-development",
|
|
19
|
+
provider: "multileaf",
|
|
20
|
+
rationale: "Work often arrives as a ticket or an already-clear requirement; this entry point starts the SDD flow from the ticket without a brainstorming phase.",
|
|
21
|
+
permissions: [],
|
|
22
|
+
resources: local([
|
|
23
|
+
{ type: "skills", id: "ticket" },
|
|
24
|
+
{ type: "templates", id: "task" },
|
|
25
|
+
]),
|
|
26
|
+
matches: (profile) => hasProjectSignals(profile)
|
|
27
|
+
? { evidence: projectEvidence(profile), confidence: 0.9 }
|
|
28
|
+
: undefined,
|
|
29
|
+
},
|
|
17
30
|
{
|
|
18
31
|
id: "requirements-specification",
|
|
19
32
|
provider: "multileaf",
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { type ContextLayer, type ContextStore } from "./context-store.js";
|
|
2
|
+
/**
|
|
3
|
+
* Task parity suite (deterministic, offline, zero API cost).
|
|
4
|
+
*
|
|
5
|
+
* Each parity task pins a quality constraint on context assembly: the
|
|
6
|
+
* critical facts and completion markers a task needs must survive the
|
|
7
|
+
* just-in-time assembly path (`retrieveTaskContext`), not only the full
|
|
8
|
+
* composed context. If a fact only exists in a layer that retrieval drops,
|
|
9
|
+
* the task fails — that is a true-positive detection of context loss, not
|
|
10
|
+
* a benchmark artifact (see docs/token-efficiency-study-notes.md, step 2).
|
|
11
|
+
*
|
|
12
|
+
* Cost is reported (estimated, characters/4) but never gates: this suite is
|
|
13
|
+
* the quality gate; the cost gate is `aiw context-bench`.
|
|
14
|
+
*/
|
|
15
|
+
export declare const PARITY_TASK_SCHEMA = 1;
|
|
16
|
+
export type ParityTask = {
|
|
17
|
+
schema: number;
|
|
18
|
+
id: string;
|
|
19
|
+
description: string;
|
|
20
|
+
layers: Partial<Record<ContextLayer, string>>;
|
|
21
|
+
criticalFacts: string[];
|
|
22
|
+
completionMarkers: string[];
|
|
23
|
+
/** Project-relative artifact paths loaded into layers before assembly (inter-stage parity). */
|
|
24
|
+
sources?: Partial<Record<ContextLayer, string>>;
|
|
25
|
+
/** Every literal match of these patterns in the loaded sources must survive assembly. */
|
|
26
|
+
requiredPatterns?: string[];
|
|
27
|
+
};
|
|
28
|
+
export type ParityAssembler = {
|
|
29
|
+
name: string;
|
|
30
|
+
assemble(store: ContextStore): string;
|
|
31
|
+
};
|
|
32
|
+
/** The default just-in-time assembly path under test. */
|
|
33
|
+
export declare const TASK_ASSEMBLER: ParityAssembler;
|
|
34
|
+
/** The full-context baseline every assembler is compared against. */
|
|
35
|
+
export declare const FULL_ASSEMBLER: ParityAssembler;
|
|
36
|
+
export type ParityTaskResult = {
|
|
37
|
+
id: string;
|
|
38
|
+
passed: boolean;
|
|
39
|
+
factsTotal: number;
|
|
40
|
+
factsRetained: number;
|
|
41
|
+
missingFacts: string[];
|
|
42
|
+
missingMarkers: string[];
|
|
43
|
+
missingPatterns: string[];
|
|
44
|
+
baselineTokens: number;
|
|
45
|
+
assembledTokens: number;
|
|
46
|
+
tokenDelta: number;
|
|
47
|
+
};
|
|
48
|
+
export type ParitySuiteResult = {
|
|
49
|
+
schema: number;
|
|
50
|
+
assembler: string;
|
|
51
|
+
total: number;
|
|
52
|
+
passed: number;
|
|
53
|
+
failed: number;
|
|
54
|
+
totalBaselineTokens: number;
|
|
55
|
+
totalAssembledTokens: number;
|
|
56
|
+
results: ParityTaskResult[];
|
|
57
|
+
};
|
|
58
|
+
export declare function parseParityTask(content: string): ParityTask;
|
|
59
|
+
/**
|
|
60
|
+
* Load declared artifact sources into their layers (inter-stage parity).
|
|
61
|
+
* Source content is appended after any declared layer content.
|
|
62
|
+
*/
|
|
63
|
+
export declare function materializeParityTask(task: ParityTask, readSource: (path: string) => string): ParityTask;
|
|
64
|
+
/** Run one task: baseline (full context) vs the assembler under test. Exits are pass/fail only. */
|
|
65
|
+
export declare function runParityTask(task: ParityTask, assembler?: ParityAssembler): ParityTaskResult;
|
|
66
|
+
export declare function runParitySuite(tasks: ParityTask[], assembler?: ParityAssembler): ParitySuiteResult;
|
|
67
|
+
export declare function serializeParitySuite(result: ParitySuiteResult): string;
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
import { isAbsolute } from "node:path";
|
|
2
|
+
import { isMap, isScalar, isSeq, parseDocument } from "yaml";
|
|
3
|
+
import { composeContext, CONTEXT_LAYERS, } from "./context-store.js";
|
|
4
|
+
import { retrieveTaskContext } from "./context-retrieval.js";
|
|
5
|
+
import { estimateTokens } from "./context-benchmark.js";
|
|
6
|
+
/**
|
|
7
|
+
* Task parity suite (deterministic, offline, zero API cost).
|
|
8
|
+
*
|
|
9
|
+
* Each parity task pins a quality constraint on context assembly: the
|
|
10
|
+
* critical facts and completion markers a task needs must survive the
|
|
11
|
+
* just-in-time assembly path (`retrieveTaskContext`), not only the full
|
|
12
|
+
* composed context. If a fact only exists in a layer that retrieval drops,
|
|
13
|
+
* the task fails — that is a true-positive detection of context loss, not
|
|
14
|
+
* a benchmark artifact (see docs/token-efficiency-study-notes.md, step 2).
|
|
15
|
+
*
|
|
16
|
+
* Cost is reported (estimated, characters/4) but never gates: this suite is
|
|
17
|
+
* the quality gate; the cost gate is `aiw context-bench`.
|
|
18
|
+
*/
|
|
19
|
+
export const PARITY_TASK_SCHEMA = 1;
|
|
20
|
+
/** The default just-in-time assembly path under test. */
|
|
21
|
+
export const TASK_ASSEMBLER = {
|
|
22
|
+
name: "task-retrieval",
|
|
23
|
+
assemble: retrieveTaskContext,
|
|
24
|
+
};
|
|
25
|
+
/** The full-context baseline every assembler is compared against. */
|
|
26
|
+
export const FULL_ASSEMBLER = {
|
|
27
|
+
name: "full-context",
|
|
28
|
+
assemble: composeContext,
|
|
29
|
+
};
|
|
30
|
+
export function parseParityTask(content) {
|
|
31
|
+
const document = parseDocument(content);
|
|
32
|
+
if (document.errors.length)
|
|
33
|
+
throw new Error(`Parity task is not valid YAML.`);
|
|
34
|
+
const schema = document.get("schema");
|
|
35
|
+
if (schema !== PARITY_TASK_SCHEMA)
|
|
36
|
+
throw new Error(`Parity task schema must be version ${PARITY_TASK_SCHEMA}.`);
|
|
37
|
+
const id = String(document.get("id") ?? "").trim();
|
|
38
|
+
if (!id)
|
|
39
|
+
throw new Error("Parity task id is required.");
|
|
40
|
+
if (!/^[a-z0-9][a-z0-9-]*$/.test(id))
|
|
41
|
+
throw new Error(`Parity task id must be lowercase letters, numbers, and hyphens: ${id}.`);
|
|
42
|
+
const description = String(document.get("description") ?? "").trim();
|
|
43
|
+
const layersNode = document.get("layers");
|
|
44
|
+
const layers = {};
|
|
45
|
+
if (layersNode !== null && layersNode !== undefined) {
|
|
46
|
+
if (!isMap(layersNode))
|
|
47
|
+
throw new Error("Parity task layers must be a mapping.");
|
|
48
|
+
for (const layer of CONTEXT_LAYERS) {
|
|
49
|
+
const value = layersNode.get(layer);
|
|
50
|
+
if (value !== null && value !== undefined) {
|
|
51
|
+
if (typeof value !== "string")
|
|
52
|
+
throw new Error(`Parity task layer ${layer} must be a string.`);
|
|
53
|
+
layers[layer] = value;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
if (!CONTEXT_LAYERS.some((layer) => (layers[layer] ?? "").trim()))
|
|
58
|
+
throw new Error(`Parity task ${id} needs at least one non-empty context layer.`);
|
|
59
|
+
const listOf = (key) => {
|
|
60
|
+
const node = document.get(key);
|
|
61
|
+
if (node === null || node === undefined)
|
|
62
|
+
return [];
|
|
63
|
+
if (!isSeq(node))
|
|
64
|
+
throw new Error(`Parity task ${key} must be a list.`);
|
|
65
|
+
return node.items.map((item) => {
|
|
66
|
+
const value = isScalar(item) ? item.value : item;
|
|
67
|
+
if (typeof value !== "string" || !value.trim())
|
|
68
|
+
throw new Error(`Parity task ${key} entries must be non-empty strings.`);
|
|
69
|
+
return value;
|
|
70
|
+
});
|
|
71
|
+
};
|
|
72
|
+
const criticalFacts = listOf("critical_facts");
|
|
73
|
+
const completionMarkers = listOf("completion_markers");
|
|
74
|
+
const requiredPatterns = listOf("required_patterns");
|
|
75
|
+
if (!criticalFacts.length && !completionMarkers.length && !requiredPatterns.length)
|
|
76
|
+
throw new Error(`Parity task ${id} needs at least one critical fact, completion marker, or required pattern.`);
|
|
77
|
+
for (const pattern of requiredPatterns)
|
|
78
|
+
try {
|
|
79
|
+
new RegExp(pattern);
|
|
80
|
+
}
|
|
81
|
+
catch {
|
|
82
|
+
throw new Error(`Parity task ${id} has an invalid required pattern: ${pattern}`);
|
|
83
|
+
}
|
|
84
|
+
const sourcesNode = document.get("sources");
|
|
85
|
+
const sources = {};
|
|
86
|
+
if (sourcesNode !== null && sourcesNode !== undefined) {
|
|
87
|
+
if (!isMap(sourcesNode))
|
|
88
|
+
throw new Error("Parity task sources must be a mapping of layer to path.");
|
|
89
|
+
for (const layer of CONTEXT_LAYERS) {
|
|
90
|
+
const value = sourcesNode.get(layer);
|
|
91
|
+
if (value === null || value === undefined)
|
|
92
|
+
continue;
|
|
93
|
+
if (typeof value !== "string" || !value.trim())
|
|
94
|
+
throw new Error(`Parity task source for ${layer} must be a non-empty project-relative path.`);
|
|
95
|
+
if (isAbsolute(value) || value.split(/[\\/]/).includes(".."))
|
|
96
|
+
throw new Error(`Parity task source for ${layer} must be a safe project-relative path: ${value}`);
|
|
97
|
+
sources[layer] = value;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
return {
|
|
101
|
+
schema: PARITY_TASK_SCHEMA,
|
|
102
|
+
id,
|
|
103
|
+
description,
|
|
104
|
+
layers,
|
|
105
|
+
criticalFacts,
|
|
106
|
+
completionMarkers,
|
|
107
|
+
...(Object.keys(sources).length ? { sources } : {}),
|
|
108
|
+
...(requiredPatterns.length ? { requiredPatterns } : {}),
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
/**
|
|
112
|
+
* Load declared artifact sources into their layers (inter-stage parity).
|
|
113
|
+
* Source content is appended after any declared layer content.
|
|
114
|
+
*/
|
|
115
|
+
export function materializeParityTask(task, readSource) {
|
|
116
|
+
if (!task.sources)
|
|
117
|
+
return task;
|
|
118
|
+
const layers = { ...task.layers };
|
|
119
|
+
for (const [layer, path] of Object.entries(task.sources)) {
|
|
120
|
+
let content;
|
|
121
|
+
try {
|
|
122
|
+
content = readSource(path);
|
|
123
|
+
}
|
|
124
|
+
catch (error) {
|
|
125
|
+
throw new Error(`Parity task ${task.id} cannot read source ${path}: ${error.message}`, {
|
|
126
|
+
cause: error,
|
|
127
|
+
});
|
|
128
|
+
}
|
|
129
|
+
layers[layer] = `${layers[layer] ? `${layers[layer].trimEnd()}\n\n` : ""}${content}`;
|
|
130
|
+
}
|
|
131
|
+
return { ...task, layers };
|
|
132
|
+
}
|
|
133
|
+
/** Run one task: baseline (full context) vs the assembler under test. Exits are pass/fail only. */
|
|
134
|
+
export function runParityTask(task, assembler = TASK_ASSEMBLER) {
|
|
135
|
+
const store = {
|
|
136
|
+
layers: Object.fromEntries(CONTEXT_LAYERS.map((layer) => [layer, (task.layers[layer] ?? "").trim()])),
|
|
137
|
+
};
|
|
138
|
+
const baseline = FULL_ASSEMBLER.assemble(store);
|
|
139
|
+
const assembled = assembler.assemble(store);
|
|
140
|
+
const missingFacts = task.criticalFacts.filter((fact) => !assembled.includes(fact));
|
|
141
|
+
const missingMarkers = task.completionMarkers.filter((marker) => !assembled.includes(marker));
|
|
142
|
+
const missingPatterns = [];
|
|
143
|
+
if (task.requiredPatterns?.length) {
|
|
144
|
+
const sourceContent = CONTEXT_LAYERS.map((layer) => store.layers[layer]).join("\n");
|
|
145
|
+
for (const pattern of task.requiredPatterns) {
|
|
146
|
+
const matches = new Set(sourceContent.match(new RegExp(pattern, "g")) ?? []);
|
|
147
|
+
for (const match of matches)
|
|
148
|
+
if (!assembled.includes(match))
|
|
149
|
+
missingPatterns.push(`${pattern} -> ${match}`);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
const baselineTokens = estimateTokens(baseline);
|
|
153
|
+
const assembledTokens = estimateTokens(assembled);
|
|
154
|
+
return {
|
|
155
|
+
id: task.id,
|
|
156
|
+
passed: missingFacts.length === 0 && missingMarkers.length === 0 && missingPatterns.length === 0,
|
|
157
|
+
factsTotal: task.criticalFacts.length,
|
|
158
|
+
factsRetained: task.criticalFacts.length - missingFacts.length,
|
|
159
|
+
missingFacts,
|
|
160
|
+
missingMarkers,
|
|
161
|
+
missingPatterns,
|
|
162
|
+
baselineTokens,
|
|
163
|
+
assembledTokens,
|
|
164
|
+
tokenDelta: assembledTokens - baselineTokens,
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
export function runParitySuite(tasks, assembler = TASK_ASSEMBLER) {
|
|
168
|
+
const results = tasks.map((task) => runParityTask(task, assembler));
|
|
169
|
+
return {
|
|
170
|
+
schema: PARITY_TASK_SCHEMA,
|
|
171
|
+
assembler: assembler.name,
|
|
172
|
+
total: results.length,
|
|
173
|
+
passed: results.filter((result) => result.passed).length,
|
|
174
|
+
failed: results.filter((result) => !result.passed).length,
|
|
175
|
+
totalBaselineTokens: results.reduce((total, result) => total + result.baselineTokens, 0),
|
|
176
|
+
totalAssembledTokens: results.reduce((total, result) => total + result.assembledTokens, 0),
|
|
177
|
+
results,
|
|
178
|
+
};
|
|
179
|
+
}
|
|
180
|
+
export function serializeParitySuite(result) {
|
|
181
|
+
const lines = [
|
|
182
|
+
`schema: ${result.schema}`,
|
|
183
|
+
`assembler: ${result.assembler}`,
|
|
184
|
+
`tasks: ${result.total}`,
|
|
185
|
+
`passed: ${result.passed}`,
|
|
186
|
+
`failed: ${result.failed}`,
|
|
187
|
+
`total_baseline_tokens: ${result.totalBaselineTokens}`,
|
|
188
|
+
`total_assembled_tokens: ${result.totalAssembledTokens}`,
|
|
189
|
+
"token_estimate: characters/4 (estimated; quality gates are exact containment checks)",
|
|
190
|
+
"results:",
|
|
191
|
+
];
|
|
192
|
+
for (const entry of result.results) {
|
|
193
|
+
lines.push(` - id: ${entry.id}`, ` passed: ${entry.passed}`, ` facts: ${entry.factsRetained}/${entry.factsTotal}`, ` baseline_tokens: ${entry.baselineTokens}`, ` assembled_tokens: ${entry.assembledTokens}`);
|
|
194
|
+
if (entry.missingFacts.length)
|
|
195
|
+
lines.push(...entry.missingFacts.map((fact) => ` missing_fact: ${fact}`));
|
|
196
|
+
if (entry.missingMarkers.length)
|
|
197
|
+
lines.push(...entry.missingMarkers.map((marker) => ` missing_marker: ${marker}`));
|
|
198
|
+
if (entry.missingPatterns.length)
|
|
199
|
+
lines.push(...entry.missingPatterns.map((pattern) => ` missing_pattern: ${pattern}`));
|
|
200
|
+
}
|
|
201
|
+
return lines.join("\n") + "\n";
|
|
202
|
+
}
|
package/dist/token-usage.d.ts
CHANGED
|
@@ -2,8 +2,14 @@ export type TokenUsage = {
|
|
|
2
2
|
stage: string;
|
|
3
3
|
budget: number;
|
|
4
4
|
used: number;
|
|
5
|
+
model?: string;
|
|
5
6
|
};
|
|
6
7
|
export declare function validateTokenUsage(usage: TokenUsage): TokenUsage;
|
|
7
8
|
export declare function parseTokenUsage(content: string): TokenUsage[];
|
|
8
9
|
export declare function serializeTokenUsage(entries: TokenUsage[]): string;
|
|
9
10
|
export declare function recordTokenUsage(entries: TokenUsage[], usage: TokenUsage): TokenUsage[];
|
|
11
|
+
export declare function summarizeUsageByModel(entries: TokenUsage[]): {
|
|
12
|
+
model: string;
|
|
13
|
+
tokens: number;
|
|
14
|
+
stages: string[];
|
|
15
|
+
}[];
|
package/dist/token-usage.js
CHANGED
|
@@ -10,6 +10,8 @@ export function validateTokenUsage(usage) {
|
|
|
10
10
|
}
|
|
11
11
|
if (usage.used > usage.budget)
|
|
12
12
|
throw new Error(`Token budget exceeded for stage: ${usage.stage}`);
|
|
13
|
+
if (usage.model !== undefined && (typeof usage.model !== "string" || !usage.model.trim()))
|
|
14
|
+
throw new Error("Token usage model must be a non-empty string when present.");
|
|
13
15
|
return usage;
|
|
14
16
|
}
|
|
15
17
|
export function parseTokenUsage(content) {
|
|
@@ -20,19 +22,44 @@ export function parseTokenUsage(content) {
|
|
|
20
22
|
.slice(1)
|
|
21
23
|
.filter(Boolean)
|
|
22
24
|
.map((line) => {
|
|
23
|
-
const [stage, budget, used] = line.split("|");
|
|
24
|
-
return validateTokenUsage({
|
|
25
|
+
const [stage, budget, used, model] = line.split("|");
|
|
26
|
+
return validateTokenUsage({
|
|
27
|
+
stage,
|
|
28
|
+
budget: Number(budget),
|
|
29
|
+
used: Number(used),
|
|
30
|
+
...(model ? { model } : {}),
|
|
31
|
+
});
|
|
25
32
|
});
|
|
26
33
|
}
|
|
27
34
|
export function serializeTokenUsage(entries) {
|
|
28
35
|
return ([
|
|
29
36
|
"schema: 1",
|
|
30
37
|
...entries
|
|
31
|
-
.sort((a, b) => a.stage
|
|
32
|
-
|
|
38
|
+
.sort((a, b) => a.stage < b.stage
|
|
39
|
+
? -1
|
|
40
|
+
: a.stage > b.stage
|
|
41
|
+
? 1
|
|
42
|
+
: (a.model ?? "").localeCompare(b.model ?? ""))
|
|
43
|
+
.map((entry) => `${entry.stage}|${entry.budget}|${entry.used}${entry.model ? `|${entry.model}` : ""}`),
|
|
33
44
|
].join("\n") + "\n");
|
|
34
45
|
}
|
|
35
46
|
export function recordTokenUsage(entries, usage) {
|
|
36
47
|
validateTokenUsage(usage);
|
|
37
|
-
return [
|
|
48
|
+
return [
|
|
49
|
+
...entries.filter((entry) => entry.stage !== usage.stage || entry.model !== usage.model),
|
|
50
|
+
usage,
|
|
51
|
+
];
|
|
52
|
+
}
|
|
53
|
+
export function summarizeUsageByModel(entries) {
|
|
54
|
+
const byModel = new Map();
|
|
55
|
+
for (const entry of entries) {
|
|
56
|
+
const model = entry.model ?? "unattributed";
|
|
57
|
+
const bucket = byModel.get(model) ?? { tokens: 0, stages: new Set() };
|
|
58
|
+
bucket.tokens += entry.used;
|
|
59
|
+
bucket.stages.add(entry.stage);
|
|
60
|
+
byModel.set(model, bucket);
|
|
61
|
+
}
|
|
62
|
+
return [...byModel]
|
|
63
|
+
.map(([model, { tokens, stages }]) => ({ model, tokens, stages: [...stages].sort() }))
|
|
64
|
+
.sort((a, b) => b.tokens - a.tokens || a.model.localeCompare(b.model));
|
|
38
65
|
}
|
package/dist/traceability.d.ts
CHANGED
|
@@ -10,6 +10,8 @@ export type TraceabilityGraph = {
|
|
|
10
10
|
schema: 1;
|
|
11
11
|
links: TraceabilityLink[];
|
|
12
12
|
};
|
|
13
|
+
/** Parse a serialized traceability graph (round-trips serializeTraceabilityGraph). */
|
|
14
|
+
export declare function parseTraceabilityGraph(content: string): TraceabilityGraph;
|
|
13
15
|
type TraceabilityInput = {
|
|
14
16
|
specification: string;
|
|
15
17
|
plan: string;
|
package/dist/traceability.js
CHANGED
|
@@ -1,3 +1,38 @@
|
|
|
1
|
+
/** Parse a serialized traceability graph (round-trips serializeTraceabilityGraph). */
|
|
2
|
+
export function parseTraceabilityGraph(content) {
|
|
3
|
+
if (!content.trim())
|
|
4
|
+
throw new Error("Traceability graph is empty.");
|
|
5
|
+
const schema = content.match(/^schema: (\d+)$/m)?.[1];
|
|
6
|
+
if (schema !== "1")
|
|
7
|
+
throw new Error("Traceability graph schema must be version 1.");
|
|
8
|
+
const links = [];
|
|
9
|
+
for (const match of content.matchAll(/^ {2}- requirement: (REQ-\d+)$/gm)) {
|
|
10
|
+
const requirement = match[1];
|
|
11
|
+
const blockStart = (match.index ?? 0) + match[0].length;
|
|
12
|
+
const next = content.slice(blockStart).match(/^ {2}- requirement: /m);
|
|
13
|
+
const block = content.slice(blockStart, next?.index === undefined ? undefined : blockStart + next.index);
|
|
14
|
+
const listOf = (field) => {
|
|
15
|
+
const raw = block?.match(new RegExp(`^ {4}${field}: (.+)$`, "m"))?.[1] ?? "[]";
|
|
16
|
+
if (raw === "[]")
|
|
17
|
+
return [];
|
|
18
|
+
return raw
|
|
19
|
+
.slice(1, -1)
|
|
20
|
+
.split(/,\s*/)
|
|
21
|
+
.map((value) => (value.startsWith('"') && value.endsWith('"') ? JSON.parse(value) : value));
|
|
22
|
+
};
|
|
23
|
+
links.push({
|
|
24
|
+
requirement,
|
|
25
|
+
decisions: listOf("decisions"),
|
|
26
|
+
tasks: listOf("tasks"),
|
|
27
|
+
code: listOf("code"),
|
|
28
|
+
tests: listOf("tests"),
|
|
29
|
+
evidence: listOf("evidence"),
|
|
30
|
+
});
|
|
31
|
+
}
|
|
32
|
+
if (!links.length)
|
|
33
|
+
throw new Error("Traceability graph has no requirement links.");
|
|
34
|
+
return { schema: 1, links };
|
|
35
|
+
}
|
|
1
36
|
const uniqueSorted = (values) => [...new Set(values)].sort();
|
|
2
37
|
function valuesForField(block, field) {
|
|
3
38
|
const expression = new RegExp(`^\\s*-?\\s*${field}:\\s*(.+)$`, "gim");
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Usage sidecar (switchboard pattern): per-run token records with billing
|
|
3
|
+
* tiers, measurement labels, and harness attribution, stored alongside
|
|
4
|
+
* workflow checkpoints.
|
|
5
|
+
*
|
|
6
|
+
* Honest-measurement rules (docs/token-efficiency-study-notes.md):
|
|
7
|
+
* - every entry carries `measured` or `estimated`; aggregates propagate the
|
|
8
|
+
* label pessimistically (any estimated component makes the total estimated);
|
|
9
|
+
* - no timestamps: entries are ordered by a monotonic sequence number, so
|
|
10
|
+
* unchanged input produces byte-identical output;
|
|
11
|
+
* - naive `tokens x price` accounting overstates savings on cache-heavy
|
|
12
|
+
* agents, so tiers are kept separate and cache hit rate is a first-class
|
|
13
|
+
* metric. Cost conversion is deliberately not built in: provider prices
|
|
14
|
+
* rot, and unlabeled dollars would be dishonest by default.
|
|
15
|
+
*/
|
|
16
|
+
export declare const USAGE_SIDECAR_SCHEMA = 1;
|
|
17
|
+
export declare const USAGE_SIDECAR_PATH = "checkpoints/usage-sidecar.yml";
|
|
18
|
+
export declare const USAGE_SIDECAR_CAP = 500;
|
|
19
|
+
export declare const USAGE_TIERS: readonly ["cache-read", "fresh-input", "cache-write", "output"];
|
|
20
|
+
export type UsageTier = (typeof USAGE_TIERS)[number];
|
|
21
|
+
export type UsageLabel = "measured" | "estimated";
|
|
22
|
+
export type UsageSidecarEntry = {
|
|
23
|
+
seq: number;
|
|
24
|
+
command: string;
|
|
25
|
+
label: UsageLabel;
|
|
26
|
+
tiers: Record<UsageTier, number>;
|
|
27
|
+
model?: string;
|
|
28
|
+
harness?: string;
|
|
29
|
+
harnessVersion?: string;
|
|
30
|
+
requirement?: string;
|
|
31
|
+
/** Correlates entries with one SDD task run (`aiw task start --id=` or ticket), start to merged PR. */
|
|
32
|
+
workflow?: string;
|
|
33
|
+
};
|
|
34
|
+
export type UsageSidecarSummary = {
|
|
35
|
+
entries: number;
|
|
36
|
+
label: UsageLabel;
|
|
37
|
+
tiers: Record<UsageTier, number>;
|
|
38
|
+
cacheHitRate?: number;
|
|
39
|
+
byModel: {
|
|
40
|
+
model: string;
|
|
41
|
+
tokens: number;
|
|
42
|
+
label: UsageLabel;
|
|
43
|
+
}[];
|
|
44
|
+
byHarness: {
|
|
45
|
+
harness: string;
|
|
46
|
+
tokens: number;
|
|
47
|
+
label: UsageLabel;
|
|
48
|
+
}[];
|
|
49
|
+
};
|
|
50
|
+
export declare function validateUsageSidecarEntry(entry: UsageSidecarEntry): UsageSidecarEntry;
|
|
51
|
+
export declare function parseUsageSidecar(content: string): UsageSidecarEntry[];
|
|
52
|
+
export declare function serializeUsageSidecar(entries: UsageSidecarEntry[]): string;
|
|
53
|
+
/** Append an entry, assigning the next sequence number. Drops nothing: the ledger is capped by the caller. */
|
|
54
|
+
export declare function appendUsageSidecarEntry(entries: UsageSidecarEntry[], entry: Omit<UsageSidecarEntry, "seq">): UsageSidecarEntry[];
|
|
55
|
+
export declare function summarizeUsageSidecar(entries: UsageSidecarEntry[]): UsageSidecarSummary;
|
|
56
|
+
export declare function serializeUsageSidecarSummary(summary: UsageSidecarSummary): string;
|