@workflow-cannon/workspace-kit 0.6.0 → 0.8.0

package/dist/modules/documentation/runtime.js

@@ -1,6 +1,6 @@
 import { mkdir, readFile, writeFile } from "node:fs/promises";
 import { existsSync } from "node:fs";
-import { resolve, sep } from "node:path";
+import { dirname, resolve, sep } from "node:path";
 import { readdir } from "node:fs/promises";
 function isPathWithinRoot(path, root) {
     return path === root || path.startsWith(`${root}${sep}`);
@@ -30,14 +30,336 @@ async function loadRuntimeConfig(workspacePath) {
         maxValidationAttempts: Number.isFinite(maxValidationAttempts) ? maxValidationAttempts : 3
     };
 }
-function validateAiSchema(aiOutput) {
+function parseAiRecordLine(line) {
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith("#"))
+        return null;
+    const parts = trimmed.split("|");
+    // Record format is `type|token|token...`. Ignore non-record markdown lines.
+    if (parts.length < 2)
+        return null;
+    const type = parts[0] ?? "";
+    if (!type)
+        return null;
+    const positional = [];
+    const kv = {};
+    for (const token of parts.slice(1)) {
+        if (!token)
+            continue;
+        const idx = token.indexOf("=");
+        if (idx >= 0) {
+            const k = token.slice(0, idx).trim();
+            const v = token.slice(idx + 1).trim();
+            if (!k)
+                continue;
+            kv[k] = v;
+        }
+        else {
+            positional.push(token);
+        }
+    }
+    return { type, positional, kv, raw: line };
+}
+function isAllowedMetaDoc(doc) {
+    return (doc === "rules" ||
+        doc === "runbook" ||
+        doc === "workbook" ||
+        doc === "generator" ||
+        doc === "map" ||
+        doc === "workflows" ||
+        doc === "commands" ||
+        doc === "decisions" ||
+        doc === "glossary" ||
+        doc === "observed" ||
+        doc === "planned" ||
+        doc === "checks" ||
+        doc === "manifest");
+}
+function validateAiSchema(aiOutput, ctx) {
     const issues = [];
-    const lines = aiOutput.split("\n").filter((line) => line.trim().length > 0);
-    if (!lines[0]?.startsWith("meta|v=")) {
+    const lines = aiOutput.split("\n").map((l) => l.trim()).filter((l) => l.length > 0);
+    if (lines.length === 0) {
+        return [
+            {
+                check: "schema",
+                message: "AI output is empty",
+                resolved: false,
+            }
+        ];
+    }
+    const metaLine = lines[0];
+    const meta = parseAiRecordLine(metaLine);
+    if (!meta || meta.type !== "meta") {
+        return [
+            {
+                check: "schema",
+                message: "AI output must start with a meta record",
+                resolved: false,
+            }
+        ];
+    }
+    const v = meta.kv["v"];
+    const doc = meta.kv["doc"];
+    const truth = meta.kv["truth"];
+    const st = meta.kv["st"];
+    if (v !== "1") {
         issues.push({
             check: "schema",
-            message: "AI output must start with a meta record",
-            resolved: false
+            message: "AI meta schemaVersion must be v=1",
+            resolved: false,
+        });
+    }
+    if (!doc || !isAllowedMetaDoc(doc)) {
+        issues.push({
+            check: "schema",
+            message: `Unsupported meta.doc '${doc ?? ""}'`,
+            resolved: false,
+        });
+    }
+    if (!truth || truth.length === 0) {
+        issues.push({
+            check: "schema",
+            message: "AI meta.truth is required",
+            resolved: false,
+        });
+    }
+    if (!st || st.length === 0) {
+        issues.push({
+            check: "schema",
+            message: "AI meta.st is required",
+            resolved: false,
+        });
+    }
+    if (ctx.expectedDoc && doc && ctx.expectedDoc !== doc) {
+        issues.push({
+            check: "schema",
+            message: `meta.doc '${doc}' does not match expected doc family for '${ctx.expectedDoc}'`,
+            resolved: !ctx.strict,
+        });
+    }
+    const requireActiveRecords = st === "active";
+    const allowedTypes = new Set([
+        // Global ai record families used across .ai/*.
+        "project",
+        "stack",
+        "prio",
+        "ref",
+        "rule",
+        "check",
+        "path",
+        "role",
+        "has",
+        "xhas",
+        "deps",
+        "xdeps",
+        "module",
+        "wf",
+        "cmd",
+        "decision",
+        "term",
+        "observed",
+        "planned",
+        "map",
+        // Runbooks
+        "runbook",
+        "intent",
+        "chain",
+        "artifact",
+        "state",
+        "transition",
+        "promotion",
+        "rollback",
+        // Workbooks
+        "workbook",
+        "scope",
+        "command",
+        "config",
+        "cadence",
+        "guardrail",
+    ]);
+    const presentByType = {};
+    const missingRequired = [];
+    for (const line of lines.slice(1)) {
+        const rec = parseAiRecordLine(line);
+        if (!rec)
+            continue;
+        presentByType[rec.type] = true;
+        if (!allowedTypes.has(rec.type)) {
+            issues.push({
+                check: "schema",
+                message: `Unknown AI record type '${rec.type}'`,
+                resolved: !ctx.strict,
+            });
+            continue;
+        }
+        // Minimal record-level validation for current runbook/workbook families.
+        if (rec.type === "ref") {
+            const p = rec.kv["path"];
+            const n = rec.kv["name"];
+            if (!p || !n) {
+                issues.push({
+                    check: "schema",
+                    message: "ref records require both 'name' and 'path'",
+                    resolved: !ctx.strict,
+                });
+            }
+            else {
+                const abs = resolve(ctx.workspacePath, p);
+                const ok = existsSync(abs);
+                if (!ok) {
+                    issues.push({
+                        check: "schema",
+                        message: `ref.path does not exist: '${p}'`,
+                        resolved: !ctx.strict,
+                    });
+                }
+            }
+            continue;
+        }
+        if (rec.type === "rule") {
+            const rid = rec.positional[0];
+            const lvl = rec.positional[1] ?? rec.kv["lvl"];
+            const directive = (() => {
+                // rule lines can be either:
+                // rule|RID|lvl|scope|directive|...
+                // or the scope can be omitted:
+                // rule|RID|lvl|directive|...
+                const nonKey = rec.positional.slice(2);
+                return nonKey[nonKey.length - 1];
+            })();
+            if (!rid || !/^R\d{3,}$/.test(rid)) {
+                issues.push({
+                    check: "schema",
+                    message: "rule records require RID formatted like R### or R####",
+                    resolved: !ctx.strict,
+                });
+            }
+            if (!lvl || !["must", "must_not", "should", "may"].includes(lvl)) {
+                issues.push({
+                    check: "schema",
+                    message: `rule lvl is invalid: '${lvl ?? ""}'`,
+                    resolved: !ctx.strict,
+                });
+            }
+            if (!directive || directive.length < 2) {
+                issues.push({
+                    check: "schema",
+                    message: "rule directive cannot be empty",
+                    resolved: !ctx.strict,
+                });
+            }
+            continue;
+        }
+        if (rec.type === "runbook") {
+            if (!rec.kv["name"] || !rec.kv["scope"]) {
+                issues.push({
+                    check: "schema",
+                    message: "runbook records require at least name and scope",
+                    resolved: !ctx.strict,
+                });
+            }
+            continue;
+        }
+        if (rec.type === "workbook") {
+            if (!rec.kv["name"]) {
+                issues.push({
+                    check: "schema",
+                    message: "workbook records require 'name'",
+                    resolved: !ctx.strict,
+                });
+            }
+            continue;
+        }
+        if (rec.type === "chain") {
+            const step = rec.kv["step"];
+            const command = rec.kv["command"];
+            const expect = rec.kv["expect_exit"];
+            if (!step || !command || expect === undefined) {
+                issues.push({
+                    check: "schema",
+                    message: "chain records require step, command, and expect_exit",
+                    resolved: !ctx.strict,
+                });
+            }
+            continue;
+        }
+        if (rec.type === "transition") {
+            if (!rec.kv["from"] || !rec.kv["to"] || !rec.kv["requires"]) {
+                issues.push({
+                    check: "schema",
+                    message: "transition records require from, to, requires",
+                    resolved: !ctx.strict,
+                });
+            }
+            continue;
+        }
+        if (rec.type === "state") {
+            if (!rec.kv["name"]) {
+                issues.push({
+                    check: "schema",
+                    message: "state records require name",
+                    resolved: !ctx.strict,
+                });
+            }
+            continue;
+        }
+        if (rec.type === "artifact") {
+            if (!rec.kv["path"] || !rec.kv["schema"]) {
+                issues.push({
+                    check: "schema",
+                    message: "artifact records require path and schema",
+                    resolved: !ctx.strict,
+                });
+            }
+            continue;
+        }
+        if (rec.type === "command") {
+            if (!rec.kv["name"]) {
+                issues.push({
+                    check: "schema",
+                    message: "command records require name",
+                    resolved: !ctx.strict,
+                });
+            }
+            continue;
+        }
+        if (rec.type === "config") {
+            if (!rec.kv["key"]) {
+                issues.push({
+                    check: "schema",
+                    message: "config records require key",
+                    resolved: !ctx.strict,
+                });
+            }
+            continue;
+        }
+    }
+    // Per-doc required record sets.
+    if (requireActiveRecords) {
+        if (doc === "runbook") {
+            if (!presentByType["runbook"])
+                missingRequired.push("runbook| record");
+            if (!presentByType["rule"] && !presentByType["chain"])
+                missingRequired.push("at least one rule| or chain| record");
+        }
+        if (doc === "workbook") {
+            if (!presentByType["workbook"])
+                missingRequired.push("workbook| record");
+            if (!presentByType["command"])
+                missingRequired.push("at least one command| record");
+            if (!presentByType["config"])
+                missingRequired.push("at least one config| record");
+        }
+        if (doc === "rules") {
+            if (!presentByType["rule"] && !presentByType["check"])
+                missingRequired.push("at least one rule| or check| record");
+        }
+    }
+    if (missingRequired.length > 0) {
+        issues.push({
+            check: "schema",
+            message: `Missing required AI records for doc family '${doc}': ${missingRequired.join(", ")}`,
+            resolved: !ctx.strict,
         });
     }
     return issues;
@@ -109,6 +431,8 @@ export async function generateDocument(args, ctx) {
         };
     }
     const options = args.options ?? {};
+    const canOverwriteAi = options.overwriteAi ?? options.overwrite ?? true;
+    const canOverwriteHuman = options.overwriteHuman ?? options.overwrite ?? true;
     const config = await loadRuntimeConfig(ctx.workspacePath);
     const filesRead = [];
     const filesWritten = [];
@@ -174,20 +498,49 @@ export async function generateDocument(args, ctx) {
         filesRead.push(schemaPath);
         await readFile(schemaPath, "utf8");
     }
-    let aiOutput = `meta|v=1|doc=rules|truth=canonical|st=draft\nproject|name=workflow-cannon|type=generated_doc|scope=${documentType}`;
+    function resolveExpectedDocFamily(docType) {
+        if (docType.includes("runbooks/") || docType.startsWith("runbooks/"))
+            return "runbook";
+        if (docType.includes("workbooks/") || docType.startsWith("workbooks/"))
+            return "workbook";
+        return "rules";
+    }
+    const expectedDoc = resolveExpectedDocFamily(documentType);
+    // Default AI output for draft generation. When AI files already exist and overwriteAi is false,
+    // we validate and preserve the existing AI surface content instead of using this stub.
+    let aiOutput = `meta|v=1|doc=${expectedDoc}|truth=canonical|st=draft\nproject|name=workflow-cannon|type=generated_doc|scope=${documentType}`;
     let attemptsUsed = 0;
     const maxAttempts = options.maxValidationAttempts ?? config.maxValidationAttempts;
+    const strict = options.strict !== false;
+    if (existsSync(aiOutputPath) && !canOverwriteAi) {
+        // Preserve existing AI docs: validate them instead of validating the stub.
+        // This avoids schema regressions from breaking doc regeneration when AI docs are already curated.
+        aiOutput = await readFile(aiOutputPath, "utf8");
+    }
     while (attemptsUsed < maxAttempts) {
         attemptsUsed += 1;
-        const schemaIssues = validateAiSchema(aiOutput);
+        const schemaIssues = validateAiSchema(aiOutput, {
+            strict,
+            workspacePath: ctx.workspacePath,
+            expectedDoc,
+        });
         if (schemaIssues.length === 0) {
             break;
         }
+        const hasUnresolved = schemaIssues.some((i) => !i.resolved);
         validationIssues.push(...schemaIssues);
+        if (!hasUnresolved) {
+            // In advisory mode, schema warnings should not block generation.
+            break;
+        }
         aiOutput = autoResolveAiSchema(aiOutput);
     }
-    const aiFinalIssues = validateAiSchema(aiOutput);
-    if (aiFinalIssues.length > 0) {
+    const aiFinalIssues = validateAiSchema(aiOutput, {
+        strict,
+        workspacePath: ctx.workspacePath,
+        expectedDoc,
+    });
+    if (aiFinalIssues.some((i) => !i.resolved)) {
         validationIssues.push(...aiFinalIssues);
         return {
             ok: false,
@@ -249,8 +602,6 @@ export async function generateDocument(args, ctx) {
         };
     }
     if (!options.dryRun) {
-        const canOverwriteAi = options.overwriteAi ?? options.overwrite ?? true;
-        const canOverwriteHuman = options.overwriteHuman ?? options.overwrite ?? true;
         const aiExists = existsSync(aiOutputPath);
         const humanExists = existsSync(humanOutputPath);
         if ((!canOverwriteAi && aiExists) && (!canOverwriteHuman && humanExists)) {
@@ -277,6 +628,8 @@ export async function generateDocument(args, ctx) {
         }
         await mkdir(aiRoot, { recursive: true });
         await mkdir(humanRoot, { recursive: true });
+        await mkdir(dirname(aiOutputPath), { recursive: true });
+        await mkdir(dirname(humanOutputPath), { recursive: true });
         if (canOverwriteAi || !aiExists) {
             await writeFile(aiOutputPath, `${aiOutput}\n`, "utf8");
             filesWritten.push(aiOutputPath);
@@ -311,10 +664,26 @@ export async function generateDocument(args, ctx) {
 export async function generateAllDocuments(args, ctx) {
     const config = await loadRuntimeConfig(ctx.workspacePath);
     const templatesDir = resolve(ctx.workspacePath, config.templatesRoot);
+    async function listTemplateFiles(dir, baseDir) {
+        const entries = await readdir(dir, { withFileTypes: true });
+        const files = [];
+        for (const entry of entries) {
+            const absPath = resolve(dir, entry.name);
+            if (entry.isDirectory()) {
+                files.push(...(await listTemplateFiles(absPath, baseDir)));
+                continue;
+            }
+            if (!entry.isFile() || !entry.name.endsWith(".md")) {
+                continue;
+            }
+            const relPath = absPath.slice(baseDir.length + 1).split("\\").join("/");
+            files.push(relPath);
+        }
+        return files;
+    }
     let templateFiles = [];
     try {
-        const entries = await readdir(templatesDir);
-        templateFiles = entries.filter((f) => f.endsWith(".md")).sort();
+        templateFiles = (await listTemplateFiles(templatesDir, templatesDir)).sort();
     }
     catch {
         return {

package/dist/modules/improvement/generate-recommendations-runtime.d.ts

@@ -1,4 +1,5 @@
 import type { ModuleLifecycleContext } from "../../contracts/module-contract.js";
+export declare function getMaxRecommendationCandidatesPerRun(ctx: ModuleLifecycleContext): number;
 export type GenerateRecommendationsArgs = {
     /** Directory relative to workspace containing agent `*.jsonl` transcripts (default: agent-transcripts). */
     transcriptsRoot?: string;
@@ -7,7 +8,13 @@ export type GenerateRecommendationsArgs = {
     toTag?: string;
 };
 export declare function runGenerateRecommendations(ctx: ModuleLifecycleContext, args: GenerateRecommendationsArgs): Promise<{
+    runId: string;
     created: string[];
     skipped: number;
     candidates: number;
+    dedupe: {
+        skippedDuplicateEvidenceKey: number;
+        skippedExistingTaskId: number;
+        cappedRemaining: number;
+    };
 }>;

package/dist/modules/improvement/generate-recommendations-runtime.js

@@ -1,3 +1,4 @@
+import { randomUUID } from "node:crypto";
 import { TaskStore } from "../task-engine/store.js";
 import { appendLineageEvent } from "../../core/lineage-store.js";
 import { loadImprovementState, saveImprovementState } from "./improvement-state.js";
@@ -19,13 +20,38 @@ function hasEvidenceKey(tasks, key) {
         return m.evidenceKey === key;
     });
 }
+function resolveTranscriptArchivePath(ctx, args) {
+    if (typeof args.transcriptsRoot === "string" && args.transcriptsRoot.trim().length > 0) {
+        return args.transcriptsRoot.trim();
+    }
+    const improvement = ctx.effectiveConfig?.improvement && typeof ctx.effectiveConfig.improvement === "object"
+        ? ctx.effectiveConfig.improvement
+        : {};
+    const transcripts = improvement.transcripts && typeof improvement.transcripts === "object"
+        ? improvement.transcripts
+        : {};
+    const archivePath = typeof transcripts.archivePath === "string" ? transcripts.archivePath.trim() : "";
+    return archivePath || "agent-transcripts";
+}
+export function getMaxRecommendationCandidatesPerRun(ctx) {
+    const improvement = ctx.effectiveConfig?.improvement && typeof ctx.effectiveConfig.improvement === "object"
+        ? ctx.effectiveConfig.improvement
+        : {};
+    const cadence = improvement.cadence && typeof improvement.cadence === "object"
+        ? improvement.cadence
+        : {};
+    const raw = cadence.maxRecommendationCandidatesPerRun;
+    if (typeof raw === "number" && Number.isFinite(raw)) {
+        return Math.max(1, Math.floor(raw));
+    }
+    return 500;
+}
 export async function runGenerateRecommendations(ctx, args) {
+    const runId = randomUUID();
     const store = new TaskStore(ctx.workspacePath, taskStoreRelativePath(ctx));
     await store.load();
     const state = await loadImprovementState(ctx.workspacePath);
-    const transcriptsRoot = typeof args.transcriptsRoot === "string" && args.transcriptsRoot.trim()
-        ? args.transcriptsRoot.trim()
-        : "agent-transcripts";
+    const transcriptsRoot = resolveTranscriptArchivePath(ctx, args);
     const fromTag = typeof args.fromTag === "string" ? args.fromTag.trim() : undefined;
     const toTag = typeof args.toTag === "string" ? args.toTag.trim() : undefined;
     const candidates = [];
@@ -40,16 +66,23 @@ export async function runGenerateRecommendations(ctx, args) {
     }
     const allTasks = store.getAllTasks();
     const created = [];
-    let skipped = 0;
+    let skippedDuplicateEvidenceKey = 0;
+    let skippedExistingTaskId = 0;
+    let cappedRemaining = 0;
     const now = new Date().toISOString();
+    const maxCreates = getMaxRecommendationCandidatesPerRun(ctx);
     for (const c of candidates) {
         if (hasEvidenceKey(allTasks, c.evidenceKey)) {
-            skipped += 1;
+            skippedDuplicateEvidenceKey += 1;
             continue;
         }
         const id = taskIdForEvidenceKey(c.evidenceKey);
         if (store.getTask(id)) {
-            skipped += 1;
+            skippedExistingTaskId += 1;
+            continue;
+        }
+        if (created.length >= maxCreates) {
+            cappedRemaining += 1;
             continue;
         }
         const task = {
@@ -88,5 +121,16 @@ export async function runGenerateRecommendations(ctx, args) {
     }
     await store.save();
     await saveImprovementState(ctx.workspacePath, state);
-    return { created, skipped, candidates: candidates.length };
+    const skipped = skippedDuplicateEvidenceKey + skippedExistingTaskId;
+    return {
+        runId,
+        created,
+        skipped,
+        candidates: candidates.length,
+        dedupe: {
+            skippedDuplicateEvidenceKey,
+            skippedExistingTaskId,
+            cappedRemaining
+        }
+    };
 }

package/dist/modules/improvement/improvement-state.d.ts

@@ -1,10 +1,21 @@
-export declare const IMPROVEMENT_STATE_SCHEMA_VERSION: 1;
+export declare const IMPROVEMENT_STATE_SCHEMA_VERSION: 2;
+export type TranscriptRetryEntry = {
+    relativePath: string;
+    attempts: number;
+    lastErrorCode: string;
+    lastErrorMessage: string;
+    nextRetryAt: string;
+};
 export type ImprovementStateDocument = {
     schemaVersion: typeof IMPROVEMENT_STATE_SCHEMA_VERSION;
     policyTraceLineCursor: number;
     mutationLineCursor: number;
     transitionLogLengthCursor: number;
     transcriptLineCursors: Record<string, number>;
+    lastSyncRunAt: string | null;
+    lastIngestRunAt: string | null;
+    /** Bounded queue of transcript files that failed to copy; retried on subsequent syncs. */
+    transcriptRetryQueue: TranscriptRetryEntry[];
 };
 export declare function emptyImprovementState(): ImprovementStateDocument;
 export declare function loadImprovementState(workspacePath: string): Promise<ImprovementStateDocument>;

package/dist/modules/improvement/improvement-state.js

@@ -1,6 +1,6 @@
 import fs from "node:fs/promises";
 import path from "node:path";
-export const IMPROVEMENT_STATE_SCHEMA_VERSION = 1;
+export const IMPROVEMENT_STATE_SCHEMA_VERSION = 2;
 const DEFAULT_REL = ".workspace-kit/improvement/state.json";
 function statePath(workspacePath) {
     return path.join(workspacePath, DEFAULT_REL);
@@ -11,21 +11,48 @@ export function emptyImprovementState() {
         policyTraceLineCursor: 0,
         mutationLineCursor: 0,
         transitionLogLengthCursor: 0,
-        transcriptLineCursors: {}
+        transcriptLineCursors: {},
+        lastSyncRunAt: null,
+        lastIngestRunAt: null,
+        transcriptRetryQueue: []
+    };
+}
+function migrateFromV1(raw) {
+    const base = emptyImprovementState();
+    return {
+        ...base,
+        policyTraceLineCursor: typeof raw.policyTraceLineCursor === "number" ? raw.policyTraceLineCursor : 0,
+        mutationLineCursor: typeof raw.mutationLineCursor === "number" ? raw.mutationLineCursor : 0,
+        transitionLogLengthCursor: typeof raw.transitionLogLengthCursor === "number" ? raw.transitionLogLengthCursor : 0,
+        transcriptLineCursors: raw.transcriptLineCursors && typeof raw.transcriptLineCursors === "object" && raw.transcriptLineCursors !== null
+            ? raw.transcriptLineCursors
+            : {},
+        lastSyncRunAt: typeof raw.lastSyncRunAt === "string" ? raw.lastSyncRunAt : null,
+        lastIngestRunAt: typeof raw.lastIngestRunAt === "string" ? raw.lastIngestRunAt : null
     };
 }
 export async function loadImprovementState(workspacePath) {
     const fp = statePath(workspacePath);
     try {
-        const raw = await fs.readFile(fp, "utf8");
-        const doc = JSON.parse(raw);
-        if (doc.schemaVersion !== IMPROVEMENT_STATE_SCHEMA_VERSION) {
+        const rawText = await fs.readFile(fp, "utf8");
+        const raw = JSON.parse(rawText);
+        const ver = raw.schemaVersion;
+        if (ver === 1) {
+            return migrateFromV1(raw);
+        }
+        if (ver !== IMPROVEMENT_STATE_SCHEMA_VERSION) {
             return emptyImprovementState();
         }
+        const doc = raw;
         return {
             ...emptyImprovementState(),
             ...doc,
-            transcriptLineCursors: doc.transcriptLineCursors ?? {}
+            transcriptLineCursors: doc.transcriptLineCursors ?? {},
+            transcriptRetryQueue: Array.isArray(doc.transcriptRetryQueue)
+                ? doc.transcriptRetryQueue.filter((e) => e !== null &&
+                    typeof e === "object" &&
+                    typeof e.relativePath === "string")
+                : []
         };
     }
     catch (e) {
@@ -38,5 +65,9 @@ export async function loadImprovementState(workspacePath) {
 export async function saveImprovementState(workspacePath, doc) {
     const fp = statePath(workspacePath);
     await fs.mkdir(path.dirname(fp), { recursive: true });
-    await fs.writeFile(fp, `${JSON.stringify(doc, null, 2)}\n`, "utf8");
+    const out = {
+        ...doc,
+        schemaVersion: IMPROVEMENT_STATE_SCHEMA_VERSION
+    };
+    await fs.writeFile(fp, `${JSON.stringify(out, null, 2)}\n`, "utf8");
 }