cclaw-cli 0.26.0 → 0.27.0

package/dist/eval/verifiers/workflow-consistency.js

@@ -0,0 +1,225 @@
+const DEFAULT_PLACEHOLDERS = ["TBD", "TODO", "placeholder"];
+export function verifyWorkflowConsistency(artifacts, expected) {
+    if (!expected)
+        return [];
+    const out = [];
+    if (expected.idsFlow) {
+        for (const rule of expected.idsFlow) {
+            out.push(...checkIdsFlow(artifacts, rule));
+        }
+    }
+    if (expected.placeholderFree) {
+        out.push(...checkPlaceholderFree(artifacts, expected.placeholderFree.stages, expected.placeholderFree.phrases && expected.placeholderFree.phrases.length > 0
+            ? expected.placeholderFree.phrases
+            : DEFAULT_PLACEHOLDERS));
+    }
+    if (expected.noContradictions) {
+        for (const rule of expected.noContradictions) {
+            out.push(...checkNoContradiction(artifacts, rule));
+        }
+    }
+    return out;
+}
+function slug(value) {
+    return value
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, "-")
+        .replace(/^-+|-+$/g, "")
+        .slice(0, 48);
+}
+function missingStage(artifacts, stage, verifierId, label) {
+    if (artifacts.has(stage))
+        return undefined;
+    return {
+        kind: "consistency",
+        id: verifierId,
+        ok: false,
+        score: 0,
+        message: `Workflow artifact for stage "${stage}" is missing (${label}).`,
+        details: { stage, missing: true }
+    };
+}
+function extractIds(text, pattern, flags) {
+    const normalized = flags.includes("g") ? flags : `${flags}g`;
+    const regex = new RegExp(pattern, normalized);
+    const hits = new Set();
+    let match;
+    while ((match = regex.exec(text)) !== null) {
+        hits.add(match[0]);
+        if (regex.lastIndex === match.index)
+            regex.lastIndex += 1;
+    }
+    return [...hits].sort((a, b) => a.localeCompare(b));
+}
+function checkIdsFlow(artifacts, rule) {
+    const idTag = slug(rule.idPattern);
+    const baseId = `consistency:ids-flow:${rule.from}:${idTag}`;
+    const results = [];
+    const missingFrom = missingStage(artifacts, rule.from, `${baseId}:source-missing`, "ids-flow source");
+    if (missingFrom) {
+        results.push(missingFrom);
+        return results;
+    }
+    const source = artifacts.get(rule.from);
+    let sourceIds;
+    try {
+        sourceIds = extractIds(source, rule.idPattern, rule.idFlags ?? "g");
+    }
+    catch (err) {
+        results.push({
+            kind: "consistency",
+            id: `${baseId}:regex`,
+            ok: false,
+            score: 0,
+            message: `Invalid id regex "${rule.idPattern}": ${err instanceof Error ? err.message : String(err)}`,
+            details: { from: rule.from }
+        });
+        return results;
+    }
+    if (sourceIds.length === 0) {
+        results.push({
+            kind: "consistency",
+            id: `${baseId}:source-empty`,
+            ok: false,
+            score: 0,
+            message: `No ids matched "${rule.idPattern}" in stage "${rule.from}".`,
+            details: { from: rule.from, pattern: rule.idPattern }
+        });
+        return results;
+    }
+    for (const target of rule.to) {
+        const missingTarget = missingStage(artifacts, target, `${baseId}:${target}:target-missing`, "ids-flow target");
+        if (missingTarget) {
+            results.push(missingTarget);
+            continue;
+        }
+        const body = artifacts.get(target);
+        const missing = sourceIds.filter((id) => !body.includes(id));
+        const verifierId = `${baseId}:${target}`;
+        if (missing.length === 0) {
+            results.push({
+                kind: "consistency",
+                id: verifierId,
+                ok: true,
+                score: 1,
+                message: `All ${sourceIds.length} id(s) from "${rule.from}" appear in "${target}".`,
+                details: { from: rule.from, to: target, ids: sourceIds }
+            });
+        }
+        else {
+            results.push({
+                kind: "consistency",
+                id: verifierId,
+                ok: false,
+                score: 0,
+                message: `Missing in "${target}": ${missing.slice(0, 5).join(", ")}` +
+                    (missing.length > 5 ? ` (+${missing.length - 5} more)` : ""),
+                details: {
+                    from: rule.from,
+                    to: target,
+                    ids: sourceIds,
+                    missing
+                }
+            });
+        }
+    }
+    return results;
+}
+function checkPlaceholderFree(artifacts, stages, phrases) {
+    const results = [];
+    for (const stage of stages) {
+        const verifierId = `consistency:placeholder-free:${stage}`;
+        const missing = missingStage(artifacts, stage, verifierId, "placeholder-free");
+        if (missing) {
+            results.push(missing);
+            continue;
+        }
+        const body = artifacts.get(stage);
+        const lower = body.toLowerCase();
+        const hits = phrases.filter((p) => lower.includes(p.toLowerCase()));
+        if (hits.length === 0) {
+            results.push({
+                kind: "consistency",
+                id: verifierId,
+                ok: true,
+                score: 1,
+                message: `No placeholder phrases found in "${stage}".`,
+                details: { stage, phrases }
+            });
+        }
+        else {
+            results.push({
+                kind: "consistency",
+                id: verifierId,
+                ok: false,
+                score: 0,
+                message: `Placeholder phrases in "${stage}": ${hits.join(", ")}.`,
+                details: { stage, phrases, hits }
+            });
+        }
+    }
+    return results;
+}
+function checkNoContradiction(artifacts, rule) {
+    const tag = `${slug(rule.must)}-vs-${slug(rule.forbid)}`;
+    const baseId = `consistency:no-contradiction:${rule.stage}:${tag}`;
+    const results = [];
+    const missingAnchor = missingStage(artifacts, rule.stage, `${baseId}:anchor-missing`, "no-contradiction anchor");
+    if (missingAnchor) {
+        results.push(missingAnchor);
+        return results;
+    }
+    const anchorText = artifacts.get(rule.stage);
+    if (!anchorText.toLowerCase().includes(rule.must.toLowerCase())) {
+        // The declaring stage doesn't actually assert `must`, so the rule is vacuously satisfied.
+        results.push({
+            kind: "consistency",
+            id: `${baseId}:anchor-inactive`,
+            ok: true,
+            score: 1,
+            message: `Anchor "${rule.must}" not present in "${rule.stage}"; contradiction check skipped.`,
+            details: { stage: rule.stage, anchor: rule.must, skipped: true }
+        });
+        return results;
+    }
+    for (const target of rule.stages) {
+        const verifierId = `${baseId}:${target}`;
+        const missingTarget = missingStage(artifacts, target, `${verifierId}:target-missing`, "no-contradiction target");
+        if (missingTarget) {
+            results.push(missingTarget);
+            continue;
+        }
+        const body = artifacts.get(target);
+        if (body.toLowerCase().includes(rule.forbid.toLowerCase())) {
+            results.push({
+                kind: "consistency",
+                id: verifierId,
+                ok: false,
+                score: 0,
+                message: `"${rule.stage}" asserts "${rule.must}" but "${target}" contains "${rule.forbid}".`,
+                details: {
+                    stage: rule.stage,
+                    anchor: rule.must,
+                    forbid: rule.forbid,
+                    target
+                }
+            });
+        }
+        else {
+            results.push({
+                kind: "consistency",
+                id: verifierId,
+                ok: true,
+                score: 1,
+                message: `"${target}" does not contradict "${rule.stage}" on "${rule.must}".`,
+                details: {
+                    stage: rule.stage,
+                    anchor: rule.must,
+                    forbid: rule.forbid,
+                    target
+                }
+            });
+        }
+    }
+    return results;
+}

package/dist/eval/workflow-corpus.d.ts

@@ -0,0 +1,7 @@
+import type { WorkflowCase } from "./types.js";
+/**
+ * Load every Tier C workflow case under
+ * `.cclaw/evals/corpus/workflows/*.yaml`. Returns an empty array when the
+ * directory is missing — a fresh `cclaw init` has no Tier C corpus yet.
+ */
+export declare function loadWorkflowCorpus(projectRoot: string): Promise<WorkflowCase[]>;

package/dist/eval/workflow-corpus.js

@@ -0,0 +1,207 @@
+/**
+ * Workflow corpus loader for Tier C.
+ *
+ * Tier C cases live under `.cclaw/evals/corpus/workflows/<id>.yaml` and
+ * describe a multi-stage run that chains the with-tools agent across
+ * `brainstorm → scope → design → spec → plan`. Unlike single-stage
+ * cases (which are keyed by stage folder), workflow cases ship as a
+ * single YAML that embeds each stage's prompt + expectations.
+ *
+ * The loader is intentionally separate from `loadCorpus` so the
+ * structural / rules CI paths never walk the workflow directory — those
+ * paths are single-stage only.
+ */
+import fs from "node:fs/promises";
+import path from "node:path";
+import { parse } from "yaml";
+import { EVALS_ROOT } from "../constants.js";
+import { exists } from "../fs-utils.js";
+import { WORKFLOW_STAGES } from "./types.js";
+const WORKFLOW_STAGE_SET = new Set(WORKFLOW_STAGES);
+function workflowCorpusError(filePath, reason) {
+    return new Error(`Invalid workflow case at ${filePath}: ${reason}\n` +
+        `Supported workflow stages: ${WORKFLOW_STAGES.join(", ")}`);
+}
+function isRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function readStringArray(filePath, context, value) {
+    if (value === undefined)
+        return undefined;
+    if (!Array.isArray(value) || value.some((item) => typeof item !== "string")) {
+        throw workflowCorpusError(filePath, `"${context}" must be an array of strings`);
+    }
+    return value;
+}
+function parseStageName(filePath, context, value) {
+    if (typeof value !== "string" || !WORKFLOW_STAGE_SET.has(value)) {
+        throw workflowCorpusError(filePath, `"${context}" must be one of: ${WORKFLOW_STAGES.join(", ")}`);
+    }
+    return value;
+}
+function parseStageArray(filePath, context, value) {
+    if (!Array.isArray(value) || value.length === 0) {
+        throw workflowCorpusError(filePath, `"${context}" must be a non-empty array of stage names`);
+    }
+    return value.map((entry, index) => parseStageName(filePath, `${context}[${index}]`, entry));
+}
+function parseStageStep(filePath, index, raw) {
+    if (!isRecord(raw)) {
+        throw workflowCorpusError(filePath, `stages[${index}] must be a mapping`);
+    }
+    const name = parseStageName(filePath, `stages[${index}].name`, raw.name);
+    const inputPrompt = raw.input_prompt ?? raw.inputPrompt;
+    if (typeof inputPrompt !== "string" || inputPrompt.trim().length === 0) {
+        throw workflowCorpusError(filePath, `stages[${index}].input_prompt must be a non-empty string`);
+    }
+    const step = { name, inputPrompt: inputPrompt.trim() };
+    if (raw.rubric !== undefined) {
+        if (typeof raw.rubric !== "string" || raw.rubric.trim().length === 0) {
+            throw workflowCorpusError(filePath, `stages[${index}].rubric must be a non-empty string`);
+        }
+        step.rubric = raw.rubric.trim();
+    }
+    const requiredChecks = readStringArray(filePath, `stages[${index}].required_checks`, raw.required_checks ?? raw.requiredChecks);
+    if (requiredChecks)
+        step.requiredChecks = requiredChecks;
+    const minScoresRaw = raw.minimum_scores ?? raw.minimumScores;
+    if (minScoresRaw !== undefined) {
+        if (!isRecord(minScoresRaw)) {
+            throw workflowCorpusError(filePath, `stages[${index}].minimum_scores must be a mapping of check id → number`);
+        }
+        const minimumScores = {};
+        for (const [key, val] of Object.entries(minScoresRaw)) {
+            if (typeof val !== "number" || !Number.isFinite(val) || val < 1 || val > 5) {
+                throw workflowCorpusError(filePath, `stages[${index}].minimum_scores.${key} must be a number in [1,5]`);
+            }
+            minimumScores[key] = val;
+        }
+        step.minimumScores = minimumScores;
+    }
+    return step;
+}
+function parseConsistency(filePath, raw) {
+    if (raw === undefined)
+        return undefined;
+    if (!isRecord(raw)) {
+        throw workflowCorpusError(filePath, `"consistency" must be a mapping`);
+    }
+    const out = {};
+    const idsFlowRaw = raw.ids_flow ?? raw.idsFlow;
+    if (idsFlowRaw !== undefined) {
+        if (!Array.isArray(idsFlowRaw)) {
+            throw workflowCorpusError(filePath, `"consistency.ids_flow" must be an array`);
+        }
+        out.idsFlow = idsFlowRaw.map((entry, index) => {
+            if (!isRecord(entry)) {
+                throw workflowCorpusError(filePath, `consistency.ids_flow[${index}] must be a mapping`);
+            }
+            const idPattern = entry.id_pattern ?? entry.idPattern;
+            if (typeof idPattern !== "string" || idPattern.length === 0) {
+                throw workflowCorpusError(filePath, `consistency.ids_flow[${index}].id_pattern must be a non-empty regex source`);
+            }
+            const idFlags = entry.id_flags ?? entry.idFlags;
+            if (idFlags !== undefined && typeof idFlags !== "string") {
+                throw workflowCorpusError(filePath, `consistency.ids_flow[${index}].id_flags must be a string`);
+            }
+            const from = parseStageName(filePath, `consistency.ids_flow[${index}].from`, entry.from);
+            const to = parseStageArray(filePath, `consistency.ids_flow[${index}].to`, entry.to);
+            const result = { idPattern, from, to };
+            if (idFlags !== undefined)
+                result.idFlags = idFlags;
+            return result;
+        });
+    }
+    const placeholderRaw = raw.placeholder_free ?? raw.placeholderFree;
+    if (placeholderRaw !== undefined) {
+        if (!isRecord(placeholderRaw)) {
+            throw workflowCorpusError(filePath, `"consistency.placeholder_free" must be a mapping`);
+        }
+        const stages = parseStageArray(filePath, "consistency.placeholder_free.stages", placeholderRaw.stages);
+        const phrases = readStringArray(filePath, "consistency.placeholder_free.phrases", placeholderRaw.phrases);
+        const block = { stages };
+        if (phrases)
+            block.phrases = phrases;
+        out.placeholderFree = block;
+    }
+    const noContradictionsRaw = raw.no_contradictions ?? raw.noContradictions;
+    if (noContradictionsRaw !== undefined) {
+        if (!Array.isArray(noContradictionsRaw)) {
+            throw workflowCorpusError(filePath, `"consistency.no_contradictions" must be an array`);
+        }
+        out.noContradictions = noContradictionsRaw.map((entry, index) => {
+            if (!isRecord(entry)) {
+                throw workflowCorpusError(filePath, `consistency.no_contradictions[${index}] must be a mapping`);
+            }
+            const stage = parseStageName(filePath, `consistency.no_contradictions[${index}].stage`, entry.stage);
+            if (typeof entry.must !== "string" || entry.must.length === 0) {
+                throw workflowCorpusError(filePath, `consistency.no_contradictions[${index}].must must be a non-empty string`);
+            }
+            if (typeof entry.forbid !== "string" || entry.forbid.length === 0) {
+                throw workflowCorpusError(filePath, `consistency.no_contradictions[${index}].forbid must be a non-empty string`);
+            }
+            const stages = parseStageArray(filePath, `consistency.no_contradictions[${index}].stages`, entry.stages);
+            return {
+                stage,
+                must: entry.must,
+                forbid: entry.forbid,
+                stages
+            };
+        });
+    }
+    return Object.keys(out).length === 0 ? undefined : out;
+}
+function validateWorkflowCase(filePath, raw) {
+    if (!isRecord(raw)) {
+        throw workflowCorpusError(filePath, `top-level value must be a mapping`);
+    }
+    const id = raw.id;
+    if (typeof id !== "string" || id.trim().length === 0) {
+        throw workflowCorpusError(filePath, `"id" must be a non-empty string`);
+    }
+    const stagesRaw = raw.stages;
+    if (!Array.isArray(stagesRaw) || stagesRaw.length === 0) {
+        throw workflowCorpusError(filePath, `"stages" must be a non-empty array`);
+    }
+    const stages = stagesRaw.map((entry, index) => parseStageStep(filePath, index, entry));
+    const contextFiles = readStringArray(filePath, "context_files", raw.context_files ?? raw.contextFiles);
+    const consistency = parseConsistency(filePath, raw.consistency);
+    const description = typeof raw.description === "string" ? raw.description.trim() : undefined;
+    const out = { id: id.trim(), stages };
+    if (description)
+        out.description = description;
+    if (contextFiles)
+        out.contextFiles = contextFiles;
+    if (consistency)
+        out.consistency = consistency;
+    return out;
+}
+/**
+ * Load every Tier C workflow case under
+ * `.cclaw/evals/corpus/workflows/*.yaml`. Returns an empty array when the
+ * directory is missing — a fresh `cclaw init` has no Tier C corpus yet.
+ */
+export async function loadWorkflowCorpus(projectRoot) {
+    const dir = path.join(projectRoot, EVALS_ROOT, "corpus", "workflows");
+    if (!(await exists(dir)))
+        return [];
+    const entries = await fs.readdir(dir, { withFileTypes: true });
+    const out = [];
+    for (const entry of entries) {
+        if (!entry.isFile())
+            continue;
+        if (!entry.name.endsWith(".yaml") && !entry.name.endsWith(".yml"))
+            continue;
+        const filePath = path.join(dir, entry.name);
+        let parsed;
+        try {
+            parsed = parse(await fs.readFile(filePath, "utf8"));
+        }
+        catch (err) {
+            throw workflowCorpusError(filePath, err instanceof Error ? err.message : String(err));
+        }
+        out.push(validateWorkflowCase(filePath, parsed));
+    }
+    out.sort((a, b) => a.id.localeCompare(b.id));
+    return out;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cclaw-cli",
-  "version": "0.26.0",
+  "version": "0.27.0",
   "description": "Installer-first flow toolkit for coding agents",
   "type": "module",
   "bin": {