@multileaf/ai-workflow 0.7.0 → 0.8.0

package/dist/task-parity.js

@@ -0,0 +1,202 @@
+import { isAbsolute } from "node:path";
+import { isMap, isScalar, isSeq, parseDocument } from "yaml";
+import { composeContext, CONTEXT_LAYERS, } from "./context-store.js";
+import { retrieveTaskContext } from "./context-retrieval.js";
+import { estimateTokens } from "./context-benchmark.js";
+/**
+ * Task parity suite (deterministic, offline, zero API cost).
+ *
+ * Each parity task pins a quality constraint on context assembly: the
+ * critical facts and completion markers a task needs must survive the
+ * just-in-time assembly path (`retrieveTaskContext`), not only the full
+ * composed context. If a fact only exists in a layer that retrieval drops,
+ * the task fails — that is a true-positive detection of context loss, not
+ * a benchmark artifact (see docs/token-efficiency-study-notes.md, step 2).
+ *
+ * Cost is reported (estimated, characters/4) but never gates: this suite is
+ * the quality gate; the cost gate is `aiw context-bench`.
+ */
+export const PARITY_TASK_SCHEMA = 1;
+/** The default just-in-time assembly path under test. */
+export const TASK_ASSEMBLER = {
+    name: "task-retrieval",
+    assemble: retrieveTaskContext,
+};
+/** The full-context baseline every assembler is compared against. */
+export const FULL_ASSEMBLER = {
+    name: "full-context",
+    assemble: composeContext,
+};
+export function parseParityTask(content) {
+    const document = parseDocument(content);
+    if (document.errors.length)
+        throw new Error(`Parity task is not valid YAML.`);
+    const schema = document.get("schema");
+    if (schema !== PARITY_TASK_SCHEMA)
+        throw new Error(`Parity task schema must be version ${PARITY_TASK_SCHEMA}.`);
+    const id = String(document.get("id") ?? "").trim();
+    if (!id)
+        throw new Error("Parity task id is required.");
+    if (!/^[a-z0-9][a-z0-9-]*$/.test(id))
+        throw new Error(`Parity task id must be lowercase letters, numbers, and hyphens: ${id}.`);
+    const description = String(document.get("description") ?? "").trim();
+    const layersNode = document.get("layers");
+    const layers = {};
+    if (layersNode !== null && layersNode !== undefined) {
+        if (!isMap(layersNode))
+            throw new Error("Parity task layers must be a mapping.");
+        for (const layer of CONTEXT_LAYERS) {
+            const value = layersNode.get(layer);
+            if (value !== null && value !== undefined) {
+                if (typeof value !== "string")
+                    throw new Error(`Parity task layer ${layer} must be a string.`);
+                layers[layer] = value;
+            }
+        }
+    }
+    if (!CONTEXT_LAYERS.some((layer) => (layers[layer] ?? "").trim()))
+        throw new Error(`Parity task ${id} needs at least one non-empty context layer.`);
+    const listOf = (key) => {
+        const node = document.get(key);
+        if (node === null || node === undefined)
+            return [];
+        if (!isSeq(node))
+            throw new Error(`Parity task ${key} must be a list.`);
+        return node.items.map((item) => {
+            const value = isScalar(item) ? item.value : item;
+            if (typeof value !== "string" || !value.trim())
+                throw new Error(`Parity task ${key} entries must be non-empty strings.`);
+            return value;
+        });
+    };
+    const criticalFacts = listOf("critical_facts");
+    const completionMarkers = listOf("completion_markers");
+    const requiredPatterns = listOf("required_patterns");
+    if (!criticalFacts.length && !completionMarkers.length && !requiredPatterns.length)
+        throw new Error(`Parity task ${id} needs at least one critical fact, completion marker, or required pattern.`);
+    for (const pattern of requiredPatterns)
+        try {
+            new RegExp(pattern);
+        }
+        catch {
+            throw new Error(`Parity task ${id} has an invalid required pattern: ${pattern}`);
+        }
+    const sourcesNode = document.get("sources");
+    const sources = {};
+    if (sourcesNode !== null && sourcesNode !== undefined) {
+        if (!isMap(sourcesNode))
+            throw new Error("Parity task sources must be a mapping of layer to path.");
+        for (const layer of CONTEXT_LAYERS) {
+            const value = sourcesNode.get(layer);
+            if (value === null || value === undefined)
+                continue;
+            if (typeof value !== "string" || !value.trim())
+                throw new Error(`Parity task source for ${layer} must be a non-empty project-relative path.`);
+            if (isAbsolute(value) || value.split(/[\\/]/).includes(".."))
+                throw new Error(`Parity task source for ${layer} must be a safe project-relative path: ${value}`);
+            sources[layer] = value;
+        }
+    }
+    return {
+        schema: PARITY_TASK_SCHEMA,
+        id,
+        description,
+        layers,
+        criticalFacts,
+        completionMarkers,
+        ...(Object.keys(sources).length ? { sources } : {}),
+        ...(requiredPatterns.length ? { requiredPatterns } : {}),
+    };
+}
+/**
+ * Load declared artifact sources into their layers (inter-stage parity).
+ * Source content is appended after any declared layer content.
+ */
+export function materializeParityTask(task, readSource) {
+    if (!task.sources)
+        return task;
+    const layers = { ...task.layers };
+    for (const [layer, path] of Object.entries(task.sources)) {
+        let content;
+        try {
+            content = readSource(path);
+        }
+        catch (error) {
+            throw new Error(`Parity task ${task.id} cannot read source ${path}: ${error.message}`, {
+                cause: error,
+            });
+        }
+        layers[layer] = `${layers[layer] ? `${layers[layer].trimEnd()}\n\n` : ""}${content}`;
+    }
+    return { ...task, layers };
+}
+/** Run one task: baseline (full context) vs the assembler under test. Exits are pass/fail only. */
+export function runParityTask(task, assembler = TASK_ASSEMBLER) {
+    const store = {
+        layers: Object.fromEntries(CONTEXT_LAYERS.map((layer) => [layer, (task.layers[layer] ?? "").trim()])),
+    };
+    const baseline = FULL_ASSEMBLER.assemble(store);
+    const assembled = assembler.assemble(store);
+    const missingFacts = task.criticalFacts.filter((fact) => !assembled.includes(fact));
+    const missingMarkers = task.completionMarkers.filter((marker) => !assembled.includes(marker));
+    const missingPatterns = [];
+    if (task.requiredPatterns?.length) {
+        const sourceContent = CONTEXT_LAYERS.map((layer) => store.layers[layer]).join("\n");
+        for (const pattern of task.requiredPatterns) {
+            const matches = new Set(sourceContent.match(new RegExp(pattern, "g")) ?? []);
+            for (const match of matches)
+                if (!assembled.includes(match))
+                    missingPatterns.push(`${pattern} -> ${match}`);
+        }
+    }
+    const baselineTokens = estimateTokens(baseline);
+    const assembledTokens = estimateTokens(assembled);
+    return {
+        id: task.id,
+        passed: missingFacts.length === 0 && missingMarkers.length === 0 && missingPatterns.length === 0,
+        factsTotal: task.criticalFacts.length,
+        factsRetained: task.criticalFacts.length - missingFacts.length,
+        missingFacts,
+        missingMarkers,
+        missingPatterns,
+        baselineTokens,
+        assembledTokens,
+        tokenDelta: assembledTokens - baselineTokens,
+    };
+}
+export function runParitySuite(tasks, assembler = TASK_ASSEMBLER) {
+    const results = tasks.map((task) => runParityTask(task, assembler));
+    return {
+        schema: PARITY_TASK_SCHEMA,
+        assembler: assembler.name,
+        total: results.length,
+        passed: results.filter((result) => result.passed).length,
+        failed: results.filter((result) => !result.passed).length,
+        totalBaselineTokens: results.reduce((total, result) => total + result.baselineTokens, 0),
+        totalAssembledTokens: results.reduce((total, result) => total + result.assembledTokens, 0),
+        results,
+    };
+}
+export function serializeParitySuite(result) {
+    const lines = [
+        `schema: ${result.schema}`,
+        `assembler: ${result.assembler}`,
+        `tasks: ${result.total}`,
+        `passed: ${result.passed}`,
+        `failed: ${result.failed}`,
+        `total_baseline_tokens: ${result.totalBaselineTokens}`,
+        `total_assembled_tokens: ${result.totalAssembledTokens}`,
+        "token_estimate: characters/4 (estimated; quality gates are exact containment checks)",
+        "results:",
+    ];
+    for (const entry of result.results) {
+        lines.push(`  - id: ${entry.id}`, `    passed: ${entry.passed}`, `    facts: ${entry.factsRetained}/${entry.factsTotal}`, `    baseline_tokens: ${entry.baselineTokens}`, `    assembled_tokens: ${entry.assembledTokens}`);
+        if (entry.missingFacts.length)
+            lines.push(...entry.missingFacts.map((fact) => `    missing_fact: ${fact}`));
+        if (entry.missingMarkers.length)
+            lines.push(...entry.missingMarkers.map((marker) => `    missing_marker: ${marker}`));
+        if (entry.missingPatterns.length)
+            lines.push(...entry.missingPatterns.map((pattern) => `    missing_pattern: ${pattern}`));
+    }
+    return lines.join("\n") + "\n";
+}

package/dist/traceability.d.ts

@@ -10,6 +10,8 @@ export type TraceabilityGraph = {
     schema: 1;
     links: TraceabilityLink[];
 };
+/** Parse a serialized traceability graph (round-trips serializeTraceabilityGraph). */
+export declare function parseTraceabilityGraph(content: string): TraceabilityGraph;
 type TraceabilityInput = {
     specification: string;
     plan: string;

package/dist/traceability.js

@@ -1,3 +1,38 @@
+/** Parse a serialized traceability graph (round-trips serializeTraceabilityGraph). */
+export function parseTraceabilityGraph(content) {
+    if (!content.trim())
+        throw new Error("Traceability graph is empty.");
+    const schema = content.match(/^schema: (\d+)$/m)?.[1];
+    if (schema !== "1")
+        throw new Error("Traceability graph schema must be version 1.");
+    const links = [];
+    for (const match of content.matchAll(/^ {2}- requirement: (REQ-\d+)$/gm)) {
+        const requirement = match[1];
+        const blockStart = (match.index ?? 0) + match[0].length;
+        const next = content.slice(blockStart).match(/^ {2}- requirement: /m);
+        const block = content.slice(blockStart, next?.index === undefined ? undefined : blockStart + next.index);
+        const listOf = (field) => {
+            const raw = block?.match(new RegExp(`^ {4}${field}: (.+)$`, "m"))?.[1] ?? "[]";
+            if (raw === "[]")
+                return [];
+            return raw
+                .slice(1, -1)
+                .split(/,\s*/)
+                .map((value) => (value.startsWith('"') && value.endsWith('"') ? JSON.parse(value) : value));
+        };
+        links.push({
+            requirement,
+            decisions: listOf("decisions"),
+            tasks: listOf("tasks"),
+            code: listOf("code"),
+            tests: listOf("tests"),
+            evidence: listOf("evidence"),
+        });
+    }
+    if (!links.length)
+        throw new Error("Traceability graph has no requirement links.");
+    return { schema: 1, links };
+}
 const uniqueSorted = (values) => [...new Set(values)].sort();
 function valuesForField(block, field) {
     const expression = new RegExp(`^\\s*-?\\s*${field}:\\s*(.+)$`, "gim");

package/dist/usage-sidecar.d.ts

@@ -0,0 +1,56 @@
+/**
+ * Usage sidecar (switchboard pattern): per-run token records with billing
+ * tiers, measurement labels, and harness attribution, stored alongside
+ * workflow checkpoints.
+ *
+ * Honest-measurement rules (docs/token-efficiency-study-notes.md):
+ * - every entry carries `measured` or `estimated`; aggregates propagate the
+ *   label pessimistically (any estimated component makes the total estimated);
+ * - no timestamps: entries are ordered by a monotonic sequence number, so
+ *   unchanged input produces byte-identical output;
+ * - naive `tokens x price` accounting overstates savings on cache-heavy
+ *   agents, so tiers are kept separate and cache hit rate is a first-class
+ *   metric. Cost conversion is deliberately not built in: provider prices
+ *   rot, and unlabeled dollars would be dishonest by default.
+ */
+export declare const USAGE_SIDECAR_SCHEMA = 1;
+export declare const USAGE_SIDECAR_PATH = "checkpoints/usage-sidecar.yml";
+export declare const USAGE_SIDECAR_CAP = 500;
+export declare const USAGE_TIERS: readonly ["cache-read", "fresh-input", "cache-write", "output"];
+export type UsageTier = (typeof USAGE_TIERS)[number];
+export type UsageLabel = "measured" | "estimated";
+export type UsageSidecarEntry = {
+    seq: number;
+    command: string;
+    label: UsageLabel;
+    tiers: Record<UsageTier, number>;
+    model?: string;
+    harness?: string;
+    harnessVersion?: string;
+    requirement?: string;
+    /** Correlates entries with one SDD task run (`aiw task start --id=` or ticket), start to merged PR. */
+    workflow?: string;
+};
+export type UsageSidecarSummary = {
+    entries: number;
+    label: UsageLabel;
+    tiers: Record<UsageTier, number>;
+    cacheHitRate?: number;
+    byModel: {
+        model: string;
+        tokens: number;
+        label: UsageLabel;
+    }[];
+    byHarness: {
+        harness: string;
+        tokens: number;
+        label: UsageLabel;
+    }[];
+};
+export declare function validateUsageSidecarEntry(entry: UsageSidecarEntry): UsageSidecarEntry;
+export declare function parseUsageSidecar(content: string): UsageSidecarEntry[];
+export declare function serializeUsageSidecar(entries: UsageSidecarEntry[]): string;
+/** Append an entry, assigning the next sequence number. Drops nothing: the ledger is capped by the caller. */
+export declare function appendUsageSidecarEntry(entries: UsageSidecarEntry[], entry: Omit<UsageSidecarEntry, "seq">): UsageSidecarEntry[];
+export declare function summarizeUsageSidecar(entries: UsageSidecarEntry[]): UsageSidecarSummary;
+export declare function serializeUsageSidecarSummary(summary: UsageSidecarSummary): string;

package/dist/usage-sidecar.js

@@ -0,0 +1,187 @@
+import { isSeq, parseDocument } from "yaml";
+/**
+ * Usage sidecar (switchboard pattern): per-run token records with billing
+ * tiers, measurement labels, and harness attribution, stored alongside
+ * workflow checkpoints.
+ *
+ * Honest-measurement rules (docs/token-efficiency-study-notes.md):
+ * - every entry carries `measured` or `estimated`; aggregates propagate the
+ *   label pessimistically (any estimated component makes the total estimated);
+ * - no timestamps: entries are ordered by a monotonic sequence number, so
+ *   unchanged input produces byte-identical output;
+ * - naive `tokens x price` accounting overstates savings on cache-heavy
+ *   agents, so tiers are kept separate and cache hit rate is a first-class
+ *   metric. Cost conversion is deliberately not built in: provider prices
+ *   rot, and unlabeled dollars would be dishonest by default.
+ */
+export const USAGE_SIDECAR_SCHEMA = 1;
+export const USAGE_SIDECAR_PATH = "checkpoints/usage-sidecar.yml";
+export const USAGE_SIDECAR_CAP = 500;
+export const USAGE_TIERS = ["cache-read", "fresh-input", "cache-write", "output"];
+export function validateUsageSidecarEntry(entry) {
+    if (!Number.isInteger(entry.seq) || entry.seq < 1)
+        throw new Error("Usage sidecar seq must be a positive integer.");
+    if (!entry.command.trim())
+        throw new Error("Usage sidecar command must not be empty.");
+    for (const tier of USAGE_TIERS) {
+        const value = entry.tiers[tier];
+        if (!Number.isInteger(value) || value < 0)
+            throw new Error(`Usage sidecar tier ${tier} must be a non-negative integer.`);
+    }
+    if (!USAGE_TIERS.some((tier) => entry.tiers[tier] > 0))
+        throw new Error("Usage sidecar entry needs at least one non-zero tier.");
+    if (entry.model !== undefined && !entry.model.trim())
+        throw new Error("Usage sidecar model must be a non-empty string when present.");
+    if (entry.requirement !== undefined && !/^REQ-\d+$/.test(entry.requirement))
+        throw new Error("Usage sidecar requirement must use the REQ-### identifier format.");
+    if (entry.workflow !== undefined && !/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(entry.workflow))
+        throw new Error("Usage sidecar workflow must be a workflow id or ticket identifier without spaces.");
+    for (const [name, value] of [
+        ["harness", entry.harness],
+        ["harnessVersion", entry.harnessVersion],
+    ])
+        if (value !== undefined && !value.trim())
+            throw new Error(`Usage sidecar ${name} must be a non-empty string when present.`);
+    return entry;
+}
+export function parseUsageSidecar(content) {
+    if (!content.trim())
+        return [];
+    const document = parseDocument(content);
+    if (document.errors.length)
+        throw new Error("Usage sidecar is not valid YAML.");
+    const node = document.get("entries");
+    if (node === null || node === undefined)
+        throw new Error("Usage sidecar is missing entries.");
+    if (!isSeq(node))
+        throw new Error("Usage sidecar entries must be a list.");
+    const schema = document.get("schema");
+    if (schema !== USAGE_SIDECAR_SCHEMA)
+        throw new Error(`Usage sidecar schema must be version ${USAGE_SIDECAR_SCHEMA}.`);
+    const entries = node.items.map((raw) => {
+        const pick = (key) => typeof raw.get === "function"
+            ? raw.get(key)
+            : raw[key];
+        const tiers = {};
+        const rawTiers = pick("tiers");
+        if (rawTiers === null || typeof rawTiers !== "object")
+            throw new Error("Usage sidecar entry is missing tiers.");
+        const readTier = (tier) => typeof rawTiers.get === "function"
+            ? rawTiers.get(tier)
+            : rawTiers[tier];
+        for (const tier of USAGE_TIERS) {
+            const value = readTier(tier);
+            if (value !== null && value !== undefined)
+                tiers[tier] = value;
+        }
+        const optional = (key) => {
+            const value = pick(key);
+            return value === null || value === undefined ? undefined : String(value);
+        };
+        return validateUsageSidecarEntry({
+            seq: pick("seq"),
+            command: String(pick("command")),
+            label: pick("label"),
+            tiers: Object.fromEntries(USAGE_TIERS.map((tier) => [tier, tiers[tier] ?? 0])),
+            ...(optional("model") === undefined ? {} : { model: optional("model") }),
+            ...(optional("harness") === undefined ? {} : { harness: optional("harness") }),
+            ...(optional("harness_version") === undefined
+                ? {}
+                : { harnessVersion: optional("harness_version") }),
+            ...(optional("requirement") === undefined ? {} : { requirement: optional("requirement") }),
+            ...(optional("workflow") === undefined ? {} : { workflow: optional("workflow") }),
+        });
+    });
+    const sequences = entries.map((entry) => entry.seq);
+    if (new Set(sequences).size !== sequences.length)
+        throw new Error("Usage sidecar sequence numbers must be unique.");
+    return entries;
+}
+export function serializeUsageSidecar(entries) {
+    const lines = [`schema: ${USAGE_SIDECAR_SCHEMA}`, "entries:"];
+    for (const entry of entries) {
+        lines.push(`  - seq: ${entry.seq}`, `    command: ${entry.command}`, `    label: ${entry.label}`, "    tiers:", ...USAGE_TIERS.map((tier) => `      ${tier}: ${entry.tiers[tier]}`), ...(entry.model === undefined ? [] : [`    model: ${entry.model}`]), ...(entry.harness === undefined ? [] : [`    harness: ${entry.harness}`]), ...(entry.harnessVersion === undefined
+            ? []
+            : [`    harness_version: ${entry.harnessVersion}`]), ...(entry.requirement === undefined ? [] : [`    requirement: ${entry.requirement}`]), ...(entry.workflow === undefined ? [] : [`    workflow: ${entry.workflow}`]));
+    }
+    return lines.join("\n") + "\n";
+}
+/** Append an entry, assigning the next sequence number. Drops nothing: the ledger is capped by the caller. */
+export function appendUsageSidecarEntry(entries, entry) {
+    validateUsageSidecarEntry({ ...entry, seq: 1 });
+    const nextSeq = entries.reduce((max, existing) => Math.max(max, existing.seq), 0) + 1;
+    const recorded = [...entries, { ...entry, seq: nextSeq }];
+    return recorded.length > USAGE_SIDECAR_CAP
+        ? recorded.slice(recorded.length - USAGE_SIDECAR_CAP)
+        : recorded;
+}
+const totalTokens = (entry) => USAGE_TIERS.reduce((total, tier) => total + entry.tiers[tier], 0);
+export function summarizeUsageSidecar(entries) {
+    if (!entries.length)
+        return {
+            entries: 0,
+            label: "measured",
+            tiers: Object.fromEntries(USAGE_TIERS.map((tier) => [tier, 0])),
+            cacheHitRate: undefined,
+            byModel: [],
+            byHarness: [],
+        };
+    const tiers = Object.fromEntries(USAGE_TIERS.map((tier) => [
+        tier,
+        entries.reduce((total, entry) => total + entry.tiers[tier], 0),
+    ]));
+    const billableInput = tiers["cache-read"] + tiers["fresh-input"];
+    const group = (key) => {
+        const map = new Map();
+        for (const entry of entries) {
+            const name = entry[key] ?? "unattributed";
+            const bucket = map.get(name) ?? { tokens: 0, measured: true };
+            bucket.tokens += totalTokens(entry);
+            bucket.measured = bucket.measured && entry.label === "measured";
+            map.set(name, bucket);
+        }
+        return [...map]
+            .map(([name, { tokens, measured }]) => ({
+            name,
+            tokens,
+            label: measured ? "measured" : "estimated",
+        }))
+            .sort((a, b) => b.tokens - a.tokens || a.name.localeCompare(b.name));
+    };
+    return {
+        entries: entries.length,
+        label: entries.every((entry) => entry.label === "measured") ? "measured" : "estimated",
+        tiers,
+        ...(billableInput > 0 ? { cacheHitRate: tiers["cache-read"] / billableInput } : {}),
+        byModel: group("model").map(({ name, tokens, label }) => ({ model: name, tokens, label })),
+        byHarness: group("harness").map(({ name, tokens, label }) => ({
+            harness: name,
+            tokens,
+            label,
+        })),
+    };
+}
+export function serializeUsageSidecarSummary(summary) {
+    if (!summary.entries)
+        return "No usage sidecar entries recorded.";
+    const percent = summary.cacheHitRate === undefined ? undefined : `${Math.round(summary.cacheHitRate * 100)}%`;
+    return [
+        `entries: ${summary.entries} (label: ${summary.label})`,
+        ...USAGE_TIERS.map((tier) => `  ${tier}: ${summary.tiers[tier].toLocaleString("en-US")} tokens`),
+        ...(percent === undefined
+            ? []
+            : [`  cache_hit_rate: ${percent} (cache-read / billable input)`]),
+        ...(summary.byModel.length
+            ? [
+                "by model:",
+                ...summary.byModel.map(({ model, tokens, label }) => `  - ${model}: ${tokens.toLocaleString("en-US")} (${label})`),
+            ]
+            : []),
+        ...(summary.byHarness.length
+            ? [
+                "by harness:",
+                ...summary.byHarness.map(({ harness, tokens, label }) => `  - ${harness}: ${tokens.toLocaleString("en-US")} (${label})`),
+            ]
+            : []),
+    ].join("\n");
+}