@multileaf/ai-workflow 0.7.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/dist/context-benchmark.d.ts +52 -0
- package/dist/context-benchmark.js +170 -0
- package/dist/cost-attribution.d.ts +65 -0
- package/dist/cost-attribution.js +119 -0
- package/dist/heartbeat.d.ts +61 -0
- package/dist/heartbeat.js +158 -0
- package/dist/hook-installer.d.ts +30 -0
- package/dist/hook-installer.js +52 -0
- package/dist/model-routing.d.ts +38 -0
- package/dist/model-routing.js +116 -0
- package/dist/task-parity.d.ts +67 -0
- package/dist/task-parity.js +202 -0
- package/dist/traceability.d.ts +2 -0
- package/dist/traceability.js +35 -0
- package/dist/usage-import.d.ts +51 -0
- package/dist/usage-import.js +190 -0
- package/dist/usage-sidecar.d.ts +62 -0
- package/dist/usage-sidecar.js +200 -0
- package/dist/workflow.js +519 -0
- package/docs/README.md +2 -0
- package/docs/token-efficiency-research.md +135 -0
- package/docs/token-efficiency-study-notes.md +128 -0
- package/docs/token-efficiency.md +157 -0
- package/package.json +1 -1
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import type { FileSystem } from "./types.js";
|
|
2
|
+
/**
|
|
3
|
+
* Claude Code hook installer: wires harness events to `aiw heartbeat`
|
|
4
|
+
* without a scheduler. Project-scoped: writes `.claude/settings.json` in
|
|
5
|
+
* the repository, never the user's global settings.
|
|
6
|
+
*
|
|
7
|
+
* Merge policy: existing hooks the project already declares are preserved
|
|
8
|
+
* untouched; AIW only appends its own entries, identified by the
|
|
9
|
+
* `aiw heartbeat` command, and skips them when already present
|
|
10
|
+
* (idempotent). Any JSON that is not a settings object is a hard conflict.
|
|
11
|
+
*/
|
|
12
|
+
export declare const CLAUDE_HOOK_COMMAND = "npx --yes --package=@multileaf/ai-workflow -- aiw heartbeat";
|
|
13
|
+
export type ClaudeHookEntry = {
|
|
14
|
+
hooks: {
|
|
15
|
+
type: string;
|
|
16
|
+
command: string;
|
|
17
|
+
timeout?: number;
|
|
18
|
+
}[];
|
|
19
|
+
};
|
|
20
|
+
export type ClaudeSettings = {
|
|
21
|
+
hooks?: Record<string, ClaudeHookEntry[]>;
|
|
22
|
+
} & Record<string, unknown>;
|
|
23
|
+
export declare const AIW_HEARTBEAT_EVENTS: readonly ["Stop", "UserPromptSubmit"];
|
|
24
|
+
export declare function parseClaudeSettings(content: string): ClaudeSettings;
|
|
25
|
+
export declare function installHeartbeatHooks(settings: ClaudeSettings): {
|
|
26
|
+
settings: ClaudeSettings;
|
|
27
|
+
added: string[];
|
|
28
|
+
alreadyPresent: string[];
|
|
29
|
+
};
|
|
30
|
+
export declare function writeClaudeSettings(fs: FileSystem, path: string, settings: ClaudeSettings): void;
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Claude Code hook installer: wires harness events to `aiw heartbeat`
|
|
3
|
+
* without a scheduler. Project-scoped: writes `.claude/settings.json` in
|
|
4
|
+
* the repository, never the user's global settings.
|
|
5
|
+
*
|
|
6
|
+
* Merge policy: existing hooks the project already declares are preserved
|
|
7
|
+
* untouched; AIW only appends its own entries, identified by the
|
|
8
|
+
* `aiw heartbeat` command, and skips them when already present
|
|
9
|
+
* (idempotent). Any JSON that is not a settings object is a hard conflict.
|
|
10
|
+
*/
|
|
11
|
+
export const CLAUDE_HOOK_COMMAND = "npx --yes --package=@multileaf/ai-workflow -- aiw heartbeat";
|
|
12
|
+
export const AIW_HEARTBEAT_EVENTS = ["Stop", "UserPromptSubmit"];
|
|
13
|
+
export function parseClaudeSettings(content) {
|
|
14
|
+
if (!content.trim())
|
|
15
|
+
return {};
|
|
16
|
+
const parsed = JSON.parse(content);
|
|
17
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed))
|
|
18
|
+
throw new Error(".claude/settings.json must be a JSON object.");
|
|
19
|
+
const settings = parsed;
|
|
20
|
+
if (settings.hooks !== undefined) {
|
|
21
|
+
if (typeof settings.hooks !== "object" ||
|
|
22
|
+
settings.hooks === null ||
|
|
23
|
+
Array.isArray(settings.hooks))
|
|
24
|
+
throw new Error(".claude/settings.json hooks must be an object keyed by event name.");
|
|
25
|
+
for (const [event, entries] of Object.entries(settings.hooks))
|
|
26
|
+
if (!Array.isArray(entries))
|
|
27
|
+
throw new Error(`.claude/settings.json hooks.${event} must be a list.`);
|
|
28
|
+
}
|
|
29
|
+
return settings;
|
|
30
|
+
}
|
|
31
|
+
export function installHeartbeatHooks(settings) {
|
|
32
|
+
const hooks = { ...(settings.hooks ?? {}) };
|
|
33
|
+
const added = [];
|
|
34
|
+
const alreadyPresent = [];
|
|
35
|
+
for (const event of AIW_HEARTBEAT_EVENTS) {
|
|
36
|
+
const entries = hooks[event] ? [...hooks[event]] : [];
|
|
37
|
+
const exists = entries.some((entry) => entry.hooks?.some((hook) => hook.command === CLAUDE_HOOK_COMMAND));
|
|
38
|
+
if (exists)
|
|
39
|
+
alreadyPresent.push(event);
|
|
40
|
+
else {
|
|
41
|
+
entries.push({
|
|
42
|
+
hooks: [{ type: "command", command: CLAUDE_HOOK_COMMAND, timeout: 15 }],
|
|
43
|
+
});
|
|
44
|
+
added.push(event);
|
|
45
|
+
}
|
|
46
|
+
hooks[event] = entries;
|
|
47
|
+
}
|
|
48
|
+
return { settings: { ...settings, hooks }, added, alreadyPresent };
|
|
49
|
+
}
|
|
50
|
+
export function writeClaudeSettings(fs, path, settings) {
|
|
51
|
+
fs.write(path, `${JSON.stringify(settings, null, 2)}\n`);
|
|
52
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Static model routing table — advisory, versioned, evidence-linked.
|
|
3
|
+
*
|
|
4
|
+
* Research consensus (docs/token-efficiency-study-notes.md §1 and §5):
|
|
5
|
+
* optimal routing is usually static — per-request dynamism mostly adds
|
|
6
|
+
* failure modes. Rules follow the ModelFit discipline: every recommendation
|
|
7
|
+
* names the evidence behind it, execution is advisory only, and humans can
|
|
8
|
+
* always override. The table never invokes a model; it answers "which model
|
|
9
|
+
* should this category use, and why".
|
|
10
|
+
*/
|
|
11
|
+
export declare const ROUTING_TABLE_SCHEMA = 1;
|
|
12
|
+
export declare const ROUTING_TABLE_PATH = "routing.yml";
|
|
13
|
+
export declare const ROUTING_DEFAULT_UNSET = "unset";
|
|
14
|
+
export type RoutingRule = {
|
|
15
|
+
category: string;
|
|
16
|
+
model: string;
|
|
17
|
+
/** Named evidence (benchmark, parity run, ADR) backing this rule. Empty means unevidenced. */
|
|
18
|
+
evidence: string;
|
|
19
|
+
};
|
|
20
|
+
export type RoutingTable = {
|
|
21
|
+
schema: number;
|
|
22
|
+
default: string;
|
|
23
|
+
rules: RoutingRule[];
|
|
24
|
+
};
|
|
25
|
+
export type ResolvedRoute = {
|
|
26
|
+
category: string;
|
|
27
|
+
model: string;
|
|
28
|
+
matchedRule: boolean;
|
|
29
|
+
evidence?: string;
|
|
30
|
+
advisory: "evidenced" | "unevidenced-rule" | "default-fallback" | "default-unset";
|
|
31
|
+
};
|
|
32
|
+
export declare function validateRoutingTable(table: RoutingTable): RoutingTable;
|
|
33
|
+
export declare function defaultRoutingTable(): RoutingTable;
|
|
34
|
+
export declare function parseRoutingTable(content: string): RoutingTable;
|
|
35
|
+
export declare function serializeRoutingTable(table: RoutingTable): string;
|
|
36
|
+
/** Resolve a category to a model. Deterministic: same table, same category, same answer. */
|
|
37
|
+
export declare function resolveRoute(table: RoutingTable, category: string): ResolvedRoute;
|
|
38
|
+
export declare function serializeResolvedRoute(route: ResolvedRoute): string;
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
import { parseDocument } from "yaml";
|
|
2
|
+
/**
|
|
3
|
+
* Static model routing table — advisory, versioned, evidence-linked.
|
|
4
|
+
*
|
|
5
|
+
* Research consensus (docs/token-efficiency-study-notes.md §1 and §5):
|
|
6
|
+
* optimal routing is usually static — per-request dynamism mostly adds
|
|
7
|
+
* failure modes. Rules follow the ModelFit discipline: every recommendation
|
|
8
|
+
* names the evidence behind it, execution is advisory only, and humans can
|
|
9
|
+
* always override. The table never invokes a model; it answers "which model
|
|
10
|
+
* should this category use, and why".
|
|
11
|
+
*/
|
|
12
|
+
export const ROUTING_TABLE_SCHEMA = 1;
|
|
13
|
+
export const ROUTING_TABLE_PATH = "routing.yml";
|
|
14
|
+
export const ROUTING_DEFAULT_UNSET = "unset";
|
|
15
|
+
const CATEGORY_PATTERN = /^[a-z][a-z0-9-]*(?:\.[a-z][a-z0-9-]*)*$/;
|
|
16
|
+
function validateRule(rule) {
|
|
17
|
+
if (!CATEGORY_PATTERN.test(rule.category))
|
|
18
|
+
throw new Error(`Routing category must be dot-separated kebab-case: ${rule.category}.`);
|
|
19
|
+
if (!rule.model.trim())
|
|
20
|
+
throw new Error(`Routing rule for ${rule.category} needs a model.`);
|
|
21
|
+
}
|
|
22
|
+
export function validateRoutingTable(table) {
|
|
23
|
+
if (table.schema !== ROUTING_TABLE_SCHEMA)
|
|
24
|
+
throw new Error(`Routing table schema must be version ${ROUTING_TABLE_SCHEMA}.`);
|
|
25
|
+
if (!table.default.trim())
|
|
26
|
+
throw new Error("Routing table default model must not be empty.");
|
|
27
|
+
table.rules.forEach(validateRule);
|
|
28
|
+
const categories = table.rules.map((rule) => rule.category);
|
|
29
|
+
if (new Set(categories).size !== categories.length)
|
|
30
|
+
throw new Error("Routing table categories must be unique.");
|
|
31
|
+
return table;
|
|
32
|
+
}
|
|
33
|
+
export function defaultRoutingTable() {
|
|
34
|
+
return validateRoutingTable({
|
|
35
|
+
schema: ROUTING_TABLE_SCHEMA,
|
|
36
|
+
default: ROUTING_DEFAULT_UNSET,
|
|
37
|
+
rules: [],
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
export function parseRoutingTable(content) {
|
|
41
|
+
const document = parseDocument(content);
|
|
42
|
+
if (document.errors.length)
|
|
43
|
+
throw new Error("Routing table is not valid YAML.");
|
|
44
|
+
const schema = document.get("schema");
|
|
45
|
+
if (schema !== ROUTING_TABLE_SCHEMA)
|
|
46
|
+
throw new Error(`Routing table schema must be version ${ROUTING_TABLE_SCHEMA}.`);
|
|
47
|
+
const pick = (key) => document.get(key);
|
|
48
|
+
const rulesNode = pick("rules");
|
|
49
|
+
const rules = [];
|
|
50
|
+
if (rulesNode !== null && rulesNode !== undefined) {
|
|
51
|
+
const items = Array.isArray(rulesNode)
|
|
52
|
+
? rulesNode
|
|
53
|
+
: typeof rulesNode.items === "object"
|
|
54
|
+
? rulesNode.items
|
|
55
|
+
: undefined;
|
|
56
|
+
if (items === undefined)
|
|
57
|
+
throw new Error("Routing table rules must be a list.");
|
|
58
|
+
for (const raw of items) {
|
|
59
|
+
const read = (key) => typeof raw.get === "function"
|
|
60
|
+
? raw.get(key)
|
|
61
|
+
: raw[key];
|
|
62
|
+
rules.push({
|
|
63
|
+
category: String(read("category")),
|
|
64
|
+
model: String(read("model")),
|
|
65
|
+
evidence: read("evidence") === null || read("evidence") === undefined
|
|
66
|
+
? ""
|
|
67
|
+
: String(read("evidence")),
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
return validateRoutingTable({
|
|
72
|
+
schema: ROUTING_TABLE_SCHEMA,
|
|
73
|
+
default: String(pick("default") ?? ""),
|
|
74
|
+
rules,
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
export function serializeRoutingTable(table) {
|
|
78
|
+
validateRoutingTable(table);
|
|
79
|
+
const lines = [`schema: ${table.schema}`, `default: ${table.default}`, "rules:"];
|
|
80
|
+
if (!table.rules.length)
|
|
81
|
+
lines.push(" []");
|
|
82
|
+
for (const rule of table.rules) {
|
|
83
|
+
lines.push(` - category: ${rule.category}`, ` model: ${rule.model}`, ` evidence: ${rule.evidence ? JSON.stringify(rule.evidence) : '""'}`);
|
|
84
|
+
}
|
|
85
|
+
return lines.join("\n") + "\n";
|
|
86
|
+
}
|
|
87
|
+
/** Resolve a category to a model. Deterministic: same table, same category, same answer. */
|
|
88
|
+
export function resolveRoute(table, category) {
|
|
89
|
+
if (!CATEGORY_PATTERN.test(category))
|
|
90
|
+
throw new Error(`Routing category must be dot-separated kebab-case: ${category}.`);
|
|
91
|
+
const rule = table.rules.find((candidate) => candidate.category === category);
|
|
92
|
+
if (rule)
|
|
93
|
+
return {
|
|
94
|
+
category,
|
|
95
|
+
model: rule.model,
|
|
96
|
+
matchedRule: true,
|
|
97
|
+
...(rule.evidence ? { evidence: rule.evidence } : {}),
|
|
98
|
+
advisory: rule.evidence ? "evidenced" : "unevidenced-rule",
|
|
99
|
+
};
|
|
100
|
+
if (table.default === ROUTING_DEFAULT_UNSET)
|
|
101
|
+
return { category, model: table.default, matchedRule: false, advisory: "default-unset" };
|
|
102
|
+
return { category, model: table.default, matchedRule: false, advisory: "default-fallback" };
|
|
103
|
+
}
|
|
104
|
+
export function serializeResolvedRoute(route) {
|
|
105
|
+
const advisoryText = {
|
|
106
|
+
evidenced: `advisory: rule is evidence-backed (${route.evidence})`,
|
|
107
|
+
"unevidenced-rule": "advisory: rule has no named evidence; benchmark it before trusting the routing",
|
|
108
|
+
"default-fallback": "advisory: no rule for this category; fell back to the table default",
|
|
109
|
+
"default-unset": "advisory: no rule for this category and the default is unset; a human must choose the model",
|
|
110
|
+
};
|
|
111
|
+
return [
|
|
112
|
+
`category: ${route.category}`,
|
|
113
|
+
`model: ${route.model}`,
|
|
114
|
+
advisoryText[route.advisory],
|
|
115
|
+
].join("\n");
|
|
116
|
+
}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { type ContextLayer, type ContextStore } from "./context-store.js";
|
|
2
|
+
/**
|
|
3
|
+
* Task parity suite (deterministic, offline, zero API cost).
|
|
4
|
+
*
|
|
5
|
+
* Each parity task pins a quality constraint on context assembly: the
|
|
6
|
+
* critical facts and completion markers a task needs must survive the
|
|
7
|
+
* just-in-time assembly path (`retrieveTaskContext`), not only the full
|
|
8
|
+
* composed context. If a fact only exists in a layer that retrieval drops,
|
|
9
|
+
* the task fails — that is a true-positive detection of context loss, not
|
|
10
|
+
* a benchmark artifact (see docs/token-efficiency-study-notes.md, step 2).
|
|
11
|
+
*
|
|
12
|
+
* Cost is reported (estimated, characters/4) but never gates: this suite is
|
|
13
|
+
* the quality gate; the cost gate is `aiw context-bench`.
|
|
14
|
+
*/
|
|
15
|
+
export declare const PARITY_TASK_SCHEMA = 1;
|
|
16
|
+
export type ParityTask = {
|
|
17
|
+
schema: number;
|
|
18
|
+
id: string;
|
|
19
|
+
description: string;
|
|
20
|
+
layers: Partial<Record<ContextLayer, string>>;
|
|
21
|
+
criticalFacts: string[];
|
|
22
|
+
completionMarkers: string[];
|
|
23
|
+
/** Project-relative artifact paths loaded into layers before assembly (inter-stage parity). */
|
|
24
|
+
sources?: Partial<Record<ContextLayer, string>>;
|
|
25
|
+
/** Every literal match of these patterns in the loaded sources must survive assembly. */
|
|
26
|
+
requiredPatterns?: string[];
|
|
27
|
+
};
|
|
28
|
+
export type ParityAssembler = {
|
|
29
|
+
name: string;
|
|
30
|
+
assemble(store: ContextStore): string;
|
|
31
|
+
};
|
|
32
|
+
/** The default just-in-time assembly path under test. */
|
|
33
|
+
export declare const TASK_ASSEMBLER: ParityAssembler;
|
|
34
|
+
/** The full-context baseline every assembler is compared against. */
|
|
35
|
+
export declare const FULL_ASSEMBLER: ParityAssembler;
|
|
36
|
+
export type ParityTaskResult = {
|
|
37
|
+
id: string;
|
|
38
|
+
passed: boolean;
|
|
39
|
+
factsTotal: number;
|
|
40
|
+
factsRetained: number;
|
|
41
|
+
missingFacts: string[];
|
|
42
|
+
missingMarkers: string[];
|
|
43
|
+
missingPatterns: string[];
|
|
44
|
+
baselineTokens: number;
|
|
45
|
+
assembledTokens: number;
|
|
46
|
+
tokenDelta: number;
|
|
47
|
+
};
|
|
48
|
+
export type ParitySuiteResult = {
|
|
49
|
+
schema: number;
|
|
50
|
+
assembler: string;
|
|
51
|
+
total: number;
|
|
52
|
+
passed: number;
|
|
53
|
+
failed: number;
|
|
54
|
+
totalBaselineTokens: number;
|
|
55
|
+
totalAssembledTokens: number;
|
|
56
|
+
results: ParityTaskResult[];
|
|
57
|
+
};
|
|
58
|
+
export declare function parseParityTask(content: string): ParityTask;
|
|
59
|
+
/**
|
|
60
|
+
* Load declared artifact sources into their layers (inter-stage parity).
|
|
61
|
+
* Source content is appended after any declared layer content.
|
|
62
|
+
*/
|
|
63
|
+
export declare function materializeParityTask(task: ParityTask, readSource: (path: string) => string): ParityTask;
|
|
64
|
+
/** Run one task: baseline (full context) vs the assembler under test. Exits are pass/fail only. */
|
|
65
|
+
export declare function runParityTask(task: ParityTask, assembler?: ParityAssembler): ParityTaskResult;
|
|
66
|
+
export declare function runParitySuite(tasks: ParityTask[], assembler?: ParityAssembler): ParitySuiteResult;
|
|
67
|
+
export declare function serializeParitySuite(result: ParitySuiteResult): string;
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
import { isAbsolute } from "node:path";
|
|
2
|
+
import { isMap, isScalar, isSeq, parseDocument } from "yaml";
|
|
3
|
+
import { composeContext, CONTEXT_LAYERS, } from "./context-store.js";
|
|
4
|
+
import { retrieveTaskContext } from "./context-retrieval.js";
|
|
5
|
+
import { estimateTokens } from "./context-benchmark.js";
|
|
6
|
+
/**
|
|
7
|
+
* Task parity suite (deterministic, offline, zero API cost).
|
|
8
|
+
*
|
|
9
|
+
* Each parity task pins a quality constraint on context assembly: the
|
|
10
|
+
* critical facts and completion markers a task needs must survive the
|
|
11
|
+
* just-in-time assembly path (`retrieveTaskContext`), not only the full
|
|
12
|
+
* composed context. If a fact only exists in a layer that retrieval drops,
|
|
13
|
+
* the task fails — that is a true-positive detection of context loss, not
|
|
14
|
+
* a benchmark artifact (see docs/token-efficiency-study-notes.md, step 2).
|
|
15
|
+
*
|
|
16
|
+
* Cost is reported (estimated, characters/4) but never gates: this suite is
|
|
17
|
+
* the quality gate; the cost gate is `aiw context-bench`.
|
|
18
|
+
*/
|
|
19
|
+
export const PARITY_TASK_SCHEMA = 1;
|
|
20
|
+
/** The default just-in-time assembly path under test. */
|
|
21
|
+
export const TASK_ASSEMBLER = {
|
|
22
|
+
name: "task-retrieval",
|
|
23
|
+
assemble: retrieveTaskContext,
|
|
24
|
+
};
|
|
25
|
+
/** The full-context baseline every assembler is compared against. */
|
|
26
|
+
export const FULL_ASSEMBLER = {
|
|
27
|
+
name: "full-context",
|
|
28
|
+
assemble: composeContext,
|
|
29
|
+
};
|
|
30
|
+
export function parseParityTask(content) {
|
|
31
|
+
const document = parseDocument(content);
|
|
32
|
+
if (document.errors.length)
|
|
33
|
+
throw new Error(`Parity task is not valid YAML.`);
|
|
34
|
+
const schema = document.get("schema");
|
|
35
|
+
if (schema !== PARITY_TASK_SCHEMA)
|
|
36
|
+
throw new Error(`Parity task schema must be version ${PARITY_TASK_SCHEMA}.`);
|
|
37
|
+
const id = String(document.get("id") ?? "").trim();
|
|
38
|
+
if (!id)
|
|
39
|
+
throw new Error("Parity task id is required.");
|
|
40
|
+
if (!/^[a-z0-9][a-z0-9-]*$/.test(id))
|
|
41
|
+
throw new Error(`Parity task id must be lowercase letters, numbers, and hyphens: ${id}.`);
|
|
42
|
+
const description = String(document.get("description") ?? "").trim();
|
|
43
|
+
const layersNode = document.get("layers");
|
|
44
|
+
const layers = {};
|
|
45
|
+
if (layersNode !== null && layersNode !== undefined) {
|
|
46
|
+
if (!isMap(layersNode))
|
|
47
|
+
throw new Error("Parity task layers must be a mapping.");
|
|
48
|
+
for (const layer of CONTEXT_LAYERS) {
|
|
49
|
+
const value = layersNode.get(layer);
|
|
50
|
+
if (value !== null && value !== undefined) {
|
|
51
|
+
if (typeof value !== "string")
|
|
52
|
+
throw new Error(`Parity task layer ${layer} must be a string.`);
|
|
53
|
+
layers[layer] = value;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
if (!CONTEXT_LAYERS.some((layer) => (layers[layer] ?? "").trim()))
|
|
58
|
+
throw new Error(`Parity task ${id} needs at least one non-empty context layer.`);
|
|
59
|
+
const listOf = (key) => {
|
|
60
|
+
const node = document.get(key);
|
|
61
|
+
if (node === null || node === undefined)
|
|
62
|
+
return [];
|
|
63
|
+
if (!isSeq(node))
|
|
64
|
+
throw new Error(`Parity task ${key} must be a list.`);
|
|
65
|
+
return node.items.map((item) => {
|
|
66
|
+
const value = isScalar(item) ? item.value : item;
|
|
67
|
+
if (typeof value !== "string" || !value.trim())
|
|
68
|
+
throw new Error(`Parity task ${key} entries must be non-empty strings.`);
|
|
69
|
+
return value;
|
|
70
|
+
});
|
|
71
|
+
};
|
|
72
|
+
const criticalFacts = listOf("critical_facts");
|
|
73
|
+
const completionMarkers = listOf("completion_markers");
|
|
74
|
+
const requiredPatterns = listOf("required_patterns");
|
|
75
|
+
if (!criticalFacts.length && !completionMarkers.length && !requiredPatterns.length)
|
|
76
|
+
throw new Error(`Parity task ${id} needs at least one critical fact, completion marker, or required pattern.`);
|
|
77
|
+
for (const pattern of requiredPatterns)
|
|
78
|
+
try {
|
|
79
|
+
new RegExp(pattern);
|
|
80
|
+
}
|
|
81
|
+
catch {
|
|
82
|
+
throw new Error(`Parity task ${id} has an invalid required pattern: ${pattern}`);
|
|
83
|
+
}
|
|
84
|
+
const sourcesNode = document.get("sources");
|
|
85
|
+
const sources = {};
|
|
86
|
+
if (sourcesNode !== null && sourcesNode !== undefined) {
|
|
87
|
+
if (!isMap(sourcesNode))
|
|
88
|
+
throw new Error("Parity task sources must be a mapping of layer to path.");
|
|
89
|
+
for (const layer of CONTEXT_LAYERS) {
|
|
90
|
+
const value = sourcesNode.get(layer);
|
|
91
|
+
if (value === null || value === undefined)
|
|
92
|
+
continue;
|
|
93
|
+
if (typeof value !== "string" || !value.trim())
|
|
94
|
+
throw new Error(`Parity task source for ${layer} must be a non-empty project-relative path.`);
|
|
95
|
+
if (isAbsolute(value) || value.split(/[\\/]/).includes(".."))
|
|
96
|
+
throw new Error(`Parity task source for ${layer} must be a safe project-relative path: ${value}`);
|
|
97
|
+
sources[layer] = value;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
return {
|
|
101
|
+
schema: PARITY_TASK_SCHEMA,
|
|
102
|
+
id,
|
|
103
|
+
description,
|
|
104
|
+
layers,
|
|
105
|
+
criticalFacts,
|
|
106
|
+
completionMarkers,
|
|
107
|
+
...(Object.keys(sources).length ? { sources } : {}),
|
|
108
|
+
...(requiredPatterns.length ? { requiredPatterns } : {}),
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
/**
|
|
112
|
+
* Load declared artifact sources into their layers (inter-stage parity).
|
|
113
|
+
* Source content is appended after any declared layer content.
|
|
114
|
+
*/
|
|
115
|
+
export function materializeParityTask(task, readSource) {
|
|
116
|
+
if (!task.sources)
|
|
117
|
+
return task;
|
|
118
|
+
const layers = { ...task.layers };
|
|
119
|
+
for (const [layer, path] of Object.entries(task.sources)) {
|
|
120
|
+
let content;
|
|
121
|
+
try {
|
|
122
|
+
content = readSource(path);
|
|
123
|
+
}
|
|
124
|
+
catch (error) {
|
|
125
|
+
throw new Error(`Parity task ${task.id} cannot read source ${path}: ${error.message}`, {
|
|
126
|
+
cause: error,
|
|
127
|
+
});
|
|
128
|
+
}
|
|
129
|
+
layers[layer] = `${layers[layer] ? `${layers[layer].trimEnd()}\n\n` : ""}${content}`;
|
|
130
|
+
}
|
|
131
|
+
return { ...task, layers };
|
|
132
|
+
}
|
|
133
|
+
/** Run one task: baseline (full context) vs the assembler under test. Exits are pass/fail only. */
|
|
134
|
+
export function runParityTask(task, assembler = TASK_ASSEMBLER) {
|
|
135
|
+
const store = {
|
|
136
|
+
layers: Object.fromEntries(CONTEXT_LAYERS.map((layer) => [layer, (task.layers[layer] ?? "").trim()])),
|
|
137
|
+
};
|
|
138
|
+
const baseline = FULL_ASSEMBLER.assemble(store);
|
|
139
|
+
const assembled = assembler.assemble(store);
|
|
140
|
+
const missingFacts = task.criticalFacts.filter((fact) => !assembled.includes(fact));
|
|
141
|
+
const missingMarkers = task.completionMarkers.filter((marker) => !assembled.includes(marker));
|
|
142
|
+
const missingPatterns = [];
|
|
143
|
+
if (task.requiredPatterns?.length) {
|
|
144
|
+
const sourceContent = CONTEXT_LAYERS.map((layer) => store.layers[layer]).join("\n");
|
|
145
|
+
for (const pattern of task.requiredPatterns) {
|
|
146
|
+
const matches = new Set(sourceContent.match(new RegExp(pattern, "g")) ?? []);
|
|
147
|
+
for (const match of matches)
|
|
148
|
+
if (!assembled.includes(match))
|
|
149
|
+
missingPatterns.push(`${pattern} -> ${match}`);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
const baselineTokens = estimateTokens(baseline);
|
|
153
|
+
const assembledTokens = estimateTokens(assembled);
|
|
154
|
+
return {
|
|
155
|
+
id: task.id,
|
|
156
|
+
passed: missingFacts.length === 0 && missingMarkers.length === 0 && missingPatterns.length === 0,
|
|
157
|
+
factsTotal: task.criticalFacts.length,
|
|
158
|
+
factsRetained: task.criticalFacts.length - missingFacts.length,
|
|
159
|
+
missingFacts,
|
|
160
|
+
missingMarkers,
|
|
161
|
+
missingPatterns,
|
|
162
|
+
baselineTokens,
|
|
163
|
+
assembledTokens,
|
|
164
|
+
tokenDelta: assembledTokens - baselineTokens,
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
export function runParitySuite(tasks, assembler = TASK_ASSEMBLER) {
|
|
168
|
+
const results = tasks.map((task) => runParityTask(task, assembler));
|
|
169
|
+
return {
|
|
170
|
+
schema: PARITY_TASK_SCHEMA,
|
|
171
|
+
assembler: assembler.name,
|
|
172
|
+
total: results.length,
|
|
173
|
+
passed: results.filter((result) => result.passed).length,
|
|
174
|
+
failed: results.filter((result) => !result.passed).length,
|
|
175
|
+
totalBaselineTokens: results.reduce((total, result) => total + result.baselineTokens, 0),
|
|
176
|
+
totalAssembledTokens: results.reduce((total, result) => total + result.assembledTokens, 0),
|
|
177
|
+
results,
|
|
178
|
+
};
|
|
179
|
+
}
|
|
180
|
+
export function serializeParitySuite(result) {
|
|
181
|
+
const lines = [
|
|
182
|
+
`schema: ${result.schema}`,
|
|
183
|
+
`assembler: ${result.assembler}`,
|
|
184
|
+
`tasks: ${result.total}`,
|
|
185
|
+
`passed: ${result.passed}`,
|
|
186
|
+
`failed: ${result.failed}`,
|
|
187
|
+
`total_baseline_tokens: ${result.totalBaselineTokens}`,
|
|
188
|
+
`total_assembled_tokens: ${result.totalAssembledTokens}`,
|
|
189
|
+
"token_estimate: characters/4 (estimated; quality gates are exact containment checks)",
|
|
190
|
+
"results:",
|
|
191
|
+
];
|
|
192
|
+
for (const entry of result.results) {
|
|
193
|
+
lines.push(` - id: ${entry.id}`, ` passed: ${entry.passed}`, ` facts: ${entry.factsRetained}/${entry.factsTotal}`, ` baseline_tokens: ${entry.baselineTokens}`, ` assembled_tokens: ${entry.assembledTokens}`);
|
|
194
|
+
if (entry.missingFacts.length)
|
|
195
|
+
lines.push(...entry.missingFacts.map((fact) => ` missing_fact: ${fact}`));
|
|
196
|
+
if (entry.missingMarkers.length)
|
|
197
|
+
lines.push(...entry.missingMarkers.map((marker) => ` missing_marker: ${marker}`));
|
|
198
|
+
if (entry.missingPatterns.length)
|
|
199
|
+
lines.push(...entry.missingPatterns.map((pattern) => ` missing_pattern: ${pattern}`));
|
|
200
|
+
}
|
|
201
|
+
return lines.join("\n") + "\n";
|
|
202
|
+
}
|
package/dist/traceability.d.ts
CHANGED
|
@@ -10,6 +10,8 @@ export type TraceabilityGraph = {
|
|
|
10
10
|
schema: 1;
|
|
11
11
|
links: TraceabilityLink[];
|
|
12
12
|
};
|
|
13
|
+
/** Parse a serialized traceability graph (round-trips serializeTraceabilityGraph). */
|
|
14
|
+
export declare function parseTraceabilityGraph(content: string): TraceabilityGraph;
|
|
13
15
|
type TraceabilityInput = {
|
|
14
16
|
specification: string;
|
|
15
17
|
plan: string;
|
package/dist/traceability.js
CHANGED
|
@@ -1,3 +1,38 @@
|
|
|
1
|
+
/** Parse a serialized traceability graph (round-trips serializeTraceabilityGraph). */
|
|
2
|
+
export function parseTraceabilityGraph(content) {
|
|
3
|
+
if (!content.trim())
|
|
4
|
+
throw new Error("Traceability graph is empty.");
|
|
5
|
+
const schema = content.match(/^schema: (\d+)$/m)?.[1];
|
|
6
|
+
if (schema !== "1")
|
|
7
|
+
throw new Error("Traceability graph schema must be version 1.");
|
|
8
|
+
const links = [];
|
|
9
|
+
for (const match of content.matchAll(/^ {2}- requirement: (REQ-\d+)$/gm)) {
|
|
10
|
+
const requirement = match[1];
|
|
11
|
+
const blockStart = (match.index ?? 0) + match[0].length;
|
|
12
|
+
const next = content.slice(blockStart).match(/^ {2}- requirement: /m);
|
|
13
|
+
const block = content.slice(blockStart, next?.index === undefined ? undefined : blockStart + next.index);
|
|
14
|
+
const listOf = (field) => {
|
|
15
|
+
const raw = block?.match(new RegExp(`^ {4}${field}: (.+)$`, "m"))?.[1] ?? "[]";
|
|
16
|
+
if (raw === "[]")
|
|
17
|
+
return [];
|
|
18
|
+
return raw
|
|
19
|
+
.slice(1, -1)
|
|
20
|
+
.split(/,\s*/)
|
|
21
|
+
.map((value) => (value.startsWith('"') && value.endsWith('"') ? JSON.parse(value) : value));
|
|
22
|
+
};
|
|
23
|
+
links.push({
|
|
24
|
+
requirement,
|
|
25
|
+
decisions: listOf("decisions"),
|
|
26
|
+
tasks: listOf("tasks"),
|
|
27
|
+
code: listOf("code"),
|
|
28
|
+
tests: listOf("tests"),
|
|
29
|
+
evidence: listOf("evidence"),
|
|
30
|
+
});
|
|
31
|
+
}
|
|
32
|
+
if (!links.length)
|
|
33
|
+
throw new Error("Traceability graph has no requirement links.");
|
|
34
|
+
return { schema: 1, links };
|
|
35
|
+
}
|
|
1
36
|
const uniqueSorted = (values) => [...new Set(values)].sort();
|
|
2
37
|
function valuesForField(block, field) {
|
|
3
38
|
const expression = new RegExp(`^\\s*-?\\s*${field}:\\s*(.+)$`, "gim");
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import type { UsageSidecarEntry } from "./usage-sidecar.js";
|
|
2
|
+
/**
|
|
3
|
+
* Deterministic importers that turn local harness session logs into
|
|
4
|
+
* measured sidecar entries (the switchboard pattern: raw logs -> canonical
|
|
5
|
+
* ledger via per-source converters; no LLM, no network, no timestamps).
|
|
6
|
+
*
|
|
7
|
+
* Formats were validated against real logs:
|
|
8
|
+
* - Claude Code: the .jsonl transcripts under ~/.claude/projects/<munged-root>
|
|
9
|
+
* carry message.usage with all four tiers; stream chunks
|
|
10
|
+
* duplicate per message.id, so entries are deduplicated by message id.
|
|
11
|
+
* - Codex: the recursive rollout-*.jsonl tree under ~/.codex/sessions — token_usage_record entries; input_tokens includes the cached subset; session_meta.cwd
|
|
12
|
+
* scopes a session to this project.
|
|
13
|
+
* - Gemini (antigravity-cli): the transcript_full.jsonl logs under the ~/.gemini/antigravity-cli/brain sessions — PLANNER_RESPONSE entries with
|
|
14
|
+
* input/cache_read/output. Sessions do not record their working
|
|
15
|
+
* directory, so Gemini imports cannot be scoped per project.
|
|
16
|
+
*
|
|
17
|
+
* - Pi (pi.dev): the session JSONL tree under ~/.pi/agent/sessions, grouped
|
|
18
|
+
* by project directory. Assistant messages carry per-message usage with
|
|
19
|
+
* all four tiers plus the provider and model, and dedicated usage entries
|
|
20
|
+
* (for example cache warming) record model-attributed consumption that
|
|
21
|
+
* never becomes a message.
|
|
22
|
+
*
|
|
23
|
+
* Import is idempotent: each event derives a stable command identity
|
|
24
|
+
* (`import.<source>.<hash>`), and events already present in the ledger
|
|
25
|
+
* are skipped, so reruns never double-count.
|
|
26
|
+
*/
|
|
27
|
+
export type ImportSource = "claude-code" | "codex" | "gemini" | "pi";
|
|
28
|
+
/** An event ready to append: no seq (assigned by the ledger). */
|
|
29
|
+
export type ImportedUsageEvent = Omit<UsageSidecarEntry, "seq">;
|
|
30
|
+
export type ImportResult = {
|
|
31
|
+
source: ImportSource;
|
|
32
|
+
files: number;
|
|
33
|
+
events: number;
|
|
34
|
+
imported: number;
|
|
35
|
+
skipped: number;
|
|
36
|
+
};
|
|
37
|
+
/** Claude Code: one JSONL transcript per session; dedupe assistant chunks by message id. */
|
|
38
|
+
export declare function parseClaudeCodeTranscript(content: string): ImportedUsageEvent[];
|
|
39
|
+
/** Codex: rollout JSONL; returns undefined when the session belongs to another project. */
|
|
40
|
+
export declare function parseCodexSession(content: string, projectRoot: string): ImportedUsageEvent[] | undefined;
|
|
41
|
+
/** Gemini (antigravity-cli): transcript_full JSONL; PLANNER_RESPONSE steps carry token counts. */
|
|
42
|
+
export declare function parseGeminiTranscript(content: string): ImportedUsageEvent[];
|
|
43
|
+
/** Pi (pi.dev): session JSONL; assistant messages and dedicated usage entries both carry per-message tiers and provider. */
|
|
44
|
+
export declare function parsePiSession(content: string): ImportedUsageEvent[];
|
|
45
|
+
/** Merge imported events into a ledger idempotently (already-present command ids are skipped). */
|
|
46
|
+
export declare function mergeImportedEvents(ledger: UsageSidecarEntry[], events: ImportedUsageEvent[]): {
|
|
47
|
+
ledger: UsageSidecarEntry[];
|
|
48
|
+
imported: number;
|
|
49
|
+
skipped: number;
|
|
50
|
+
};
|
|
51
|
+
export declare function summarizeImport(result: ImportResult): string;
|