@smartmemory/compose 0.2.58-beta → 0.3.0

package/lib/vocabulary-compliance.js

@@ -0,0 +1,268 @@
+/**
+ * vocabulary-compliance.js — deterministic Compose-side vocabulary checks.
+ *
+ * E3/F5: the TS engine's `judged:` ensure cannot see the changed files or the
+ * vocabulary (the judge receives only {result, input} and fails when evidence is
+ * missing), so it is unevaluable as a merge guard. v1 vocabulary enforcement is
+ * therefore moved here: compose evaluates it over the ACTUAL changed files at the
+ * review_merge step and, on violation, sends a FAILURE step_done envelope so the
+ * engine's attempts/retry lifecycle governs (never a throw past the step handler).
+ *
+ * Inert when contracts/vocabulary.yaml is missing / empty / comments-only. Message
+ * Stable message formats let tagVocabularyViolations classify them.
+ */
+import { existsSync, statSync, readFileSync } from 'node:fs';
+import { normalize, isAbsolute, join } from 'node:path';
+import { execFileSync } from 'node:child_process';
+import YAML from 'yaml';
+
+const IDENTIFIER_RE = /^[A-Za-z_][A-Za-z0-9_]*$/;
+const VOCAB_SIZE_LIMIT = 10 * 1024 * 1024; // 10 MB
+
+/** Raised by loadVocabulary; `.violations` mirrors the Python ValueError([...]). */
+export class VocabularyError extends Error {
+  constructor(violations) {
+    super(Array.isArray(violations) ? violations.join('; ') : String(violations));
+    this.name = 'VocabularyError';
+    this.violations = Array.isArray(violations) ? violations : [String(violations)];
+  }
+}
+
+/**
+ * Load and validate vocabulary.yaml. Returns {} for missing/empty/comments-only
+ * files (the no-op case). Throws VocabularyError([messages]) on malformed YAML or
+ * schema errors.
+ * @returns {Record<string, {reject: string[], reason: string}>}
+ */
+export function loadVocabulary(path) {
+  if (!existsSync(path) || !statSync(path).isFile()) return {};
+
+  let raw;
+  try {
+    raw = readFileSync(path, 'utf-8');
+  } catch (exc) {
+    throw new VocabularyError([`vocabulary.yaml malformed: cannot read ${path}: ${exc.message}`]);
+  }
+
+  let parsed;
+  try {
+    parsed = YAML.parse(raw);
+  } catch (exc) {
+    throw new VocabularyError([`vocabulary.yaml malformed: ${exc.message}`]);
+  }
+
+  // Empty file or comments-only → null/undefined → treat as empty.
+  if (parsed === null || parsed === undefined) return {};
+
+  if (typeof parsed !== 'object' || Array.isArray(parsed)) {
+    throw new VocabularyError(['vocabulary.yaml schema error: top-level must be a mapping']);
+  }
+
+  const validated = {};
+  for (const [canonical, entry] of Object.entries(parsed)) {
+    if (typeof canonical !== 'string' || !IDENTIFIER_RE.test(canonical)) {
+      throw new VocabularyError([
+        `vocabulary.yaml schema error: canonical name ${JSON.stringify(canonical)} `
+        + `must match identifier syntax (letters, digits, underscore; starts with letter or underscore)`,
+      ]);
+    }
+    if (typeof entry !== 'object' || entry === null || Array.isArray(entry)) {
+      throw new VocabularyError([
+        `vocabulary.yaml schema error: entry for ${JSON.stringify(canonical)} `
+        + `must be a mapping, got ${Array.isArray(entry) ? 'array' : typeof entry}`,
+      ]);
+    }
+
+    const allowedFields = new Set(['reject', 'reason']);
+    const unknown = Object.keys(entry).filter((k) => !allowedFields.has(k)).sort();
+    if (unknown.length > 0) {
+      throw new VocabularyError([
+        `vocabulary.yaml schema error: entry for ${JSON.stringify(canonical)} `
+        + `has unknown fields: ${JSON.stringify(unknown)}. Allowed: ["reject","reason"]`,
+      ]);
+    }
+
+    const reject = entry.reject;
+    if (!Array.isArray(reject) || reject.length === 0) {
+      throw new VocabularyError([
+        `vocabulary.yaml schema error: entry for ${JSON.stringify(canonical)} must have non-empty 'reject' list`,
+      ]);
+    }
+
+    for (const alias of reject) {
+      if (typeof alias !== 'string' || !IDENTIFIER_RE.test(alias)) {
+        throw new VocabularyError([
+          `vocabulary.yaml schema error: alias ${JSON.stringify(alias)} in ${JSON.stringify(canonical)} `
+          + `must match identifier syntax`,
+        ]);
+      }
+      if (alias === canonical) {
+        throw new VocabularyError([
+          `vocabulary.yaml schema error: canonical ${JSON.stringify(canonical)} cannot reject itself`,
+        ]);
+      }
+    }
+
+    const reason = entry.reason ?? '';
+    if (reason !== null && typeof reason !== 'string') {
+      throw new VocabularyError([
+        `vocabulary.yaml schema error: reason for ${JSON.stringify(canonical)} must be a string`,
+      ]);
+    }
+
+    validated[canonical] = { reject: [...reject], reason: reason || '' };
+  }
+
+  // Cross-entry validation: no duplicate aliases, no canonical-as-alias.
+  const canonicals = new Set(Object.keys(validated));
+  const aliasToCanonical = new Map();
+  for (const [canonical, entry] of Object.entries(validated)) {
+    for (const alias of entry.reject) {
+      if (canonicals.has(alias)) {
+        throw new VocabularyError([
+          `vocabulary.yaml schema error: ${JSON.stringify(alias)} is both a canonical `
+          + `and a rejected alias for ${JSON.stringify(canonical)}`,
+        ]);
+      }
+      if (aliasToCanonical.has(alias)) {
+        throw new VocabularyError([
+          `vocabulary.yaml schema error: alias ${JSON.stringify(alias)} is rejected by `
+          + `multiple canonicals (${aliasToCanonical.get(alias)}, ${canonical})`,
+        ]);
+      }
+      aliasToCanonical.set(alias, canonical);
+    }
+  }
+
+  return validated;
+}
+
+/** git diff --name-only <base> plus untracked files. Returns null on any failure. */
+function gitChangedFiles(base, cwd) {
+  try {
+    const diff = execFileSync('git', ['diff', '--name-only', base], {
+      cwd, encoding: 'utf-8', timeout: 5000, stdio: ['ignore', 'pipe', 'ignore'],
+    });
+    const tracked = diff.split('\n').filter(Boolean);
+    let untracked = [];
+    try {
+      const others = execFileSync('git', ['ls-files', '--others', '--exclude-standard'], {
+        cwd, encoding: 'utf-8', timeout: 5000, stdio: ['ignore', 'pipe', 'ignore'],
+      });
+      untracked = others.split('\n').filter(Boolean);
+    } catch { /* untracked listing best-effort */ }
+    const seen = new Set();
+    const combined = [];
+    for (const f of [...tracked, ...untracked]) {
+      if (!seen.has(f)) { seen.add(f); combined.push(f); }
+    }
+    return combined;
+  } catch {
+    return null;
+  }
+}
+
+function scanFileForAliases(displayPath, absPath, aliasToCanonical, canonicalToReason, matchers) {
+  const violations = [];
+  let size;
+  try {
+    size = statSync(absPath).size;
+  } catch {
+    return violations; // disappeared between checks
+  }
+  if (size > VOCAB_SIZE_LIMIT) return violations; // skip large files silently
+
+  let lines;
+  try {
+    lines = readFileSync(absPath, 'utf-8').split(/\r?\n/);
+  } catch {
+    return violations;
+  }
+
+  for (let i = 0; i < lines.length; i += 1) {
+    const lineNo = i + 1;
+    for (const [alias, pattern] of matchers) {
+      // G5: emit one violation per OCCURRENCE (python parity — the builtin uses
+      // finditer), not one per line. matchAll requires the global flag on pattern.
+      for (const _match of lines[i].matchAll(pattern)) {
+        void _match;
+        const canonical = aliasToCanonical.get(alias);
+        const reason = canonicalToReason.get(canonical) ?? '';
+        let msg = `vocabulary violation: ${displayPath}:${lineNo} uses '${alias}' — canonical is '${canonical}'`;
+        if (reason) msg += ` (reason: ${reason})`;
+        violations.push(msg);
+      }
+    }
+  }
+  return violations;
+}
+
+/**
+ * Scan `filesChanged` for rejected vocabulary aliases. Returns an array of
+ * violation strings — EMPTY means compliant or inert (no vocabulary, nothing to
+ * scan). Never throws for violations: schema/malformed vocab errors are returned
+ * as (blocking) violation strings too, so the caller converts them into a failure
+ * envelope rather than crashing the step handler.
+ *
+ * @param {string}   vocabPath   path to vocabulary.yaml (resolved against cwd)
+ * @param {string[]} filesChanged authoritative list of touched files
+ * @param {object}   [opts]
+ * @param {boolean}  [opts.gitFallback=false] scan `git diff <base>` when filesChanged is empty
+ * @param {string}   [opts.base='HEAD']
+ * @param {string}   [opts.cwd=process.cwd()]
+ * @returns {string[]}
+ */
+export function vocabularyCompliance(vocabPath, filesChanged, { gitFallback = false, base = 'HEAD', cwd = process.cwd() } = {}) {
+  let vocab;
+  try {
+    vocab = loadVocabulary(vocabPath);
+  } catch (err) {
+    // A malformed / invalid vocabulary blocks the step (matches the Python builtin
+    // raising) — surface the schema errors as violations.
+    return err instanceof VocabularyError ? err.violations : [String(err?.message ?? err)];
+  }
+  if (!vocab || Object.keys(vocab).length === 0) return []; // no vocabulary → nothing to check
+
+  const aliasToCanonical = new Map();
+  const canonicalToReason = new Map();
+  for (const [canonical, entry] of Object.entries(vocab)) {
+    canonicalToReason.set(canonical, entry.reason);
+    for (const alias of entry.reject) aliasToCanonical.set(alias, canonical);
+  }
+
+  let fileList;
+  if (Array.isArray(filesChanged) && filesChanged.length > 0) {
+    fileList = filesChanged;
+  } else if (gitFallback) {
+    const fallback = gitChangedFiles(base, cwd);
+    if (fallback === null) return []; // git failed; nothing we can do
+    fileList = fallback;
+  } else {
+    return []; // no files to scan
+  }
+
+  // Normalize + dedupe; keep only existing regular files.
+  const seen = new Set();
+  const files = []; // [displayPath, absPath]
+  for (const f of fileList) {
+    const disp = normalize(f);
+    if (seen.has(disp)) continue;
+    seen.add(disp);
+    const abs = isAbsolute(disp) ? disp : join(cwd, disp);
+    try {
+      if (statSync(abs).isFile()) files.push([disp, abs]);
+    } catch { /* missing/deleted — skip */ }
+  }
+  if (files.length === 0) return [];
+
+  const matchers = new Map();
+  for (const alias of aliasToCanonical.keys()) {
+    matchers.set(alias, new RegExp(`\\b${alias.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\b`, 'g'));
+  }
+
+  const violations = [];
+  for (const [disp, abs] of files) {
+    violations.push(...scanFileForAliases(disp, abs, aliasToCanonical, canonicalToReason, matchers));
+  }
+  return violations;
+}

package/lib/vocabulary-inject.js

@@ -1,11 +1,9 @@
 /**
  * STRAT-VOCAB-3 — Compose integration for vocabulary enforcement.
  *
- * Wires the already-shipped Stratum `vocabulary_compliance` ensure builtin
- * (stratum-mcp/src/stratum_mcp/spec.py) into the compose lifecycle:
+ * Wires Compose's deterministic vocabulary compliance check into the lifecycle:
  *   - VOCABULARY_TEMPLATE        : starter contracts/vocabulary.yaml (compose init)
  *   - vocabularyEnabled()        : gate — capability not disabled AND a vocab file exists
- *   - injectVocabularyEnsure()   : append the vocab ensure to the build flow's `review` step
  *   - tagVocabularyViolations()  : mark vocab violation strings as must-fix for display
  *
  * Design: docs/features/STRAT-VOCAB-3/design.md
@@ -16,15 +14,6 @@ import { join } from 'node:path';
 /** Project-relative location of the vocabulary file (also the path baked into the ensure). */
 export const VOCABULARY_FILE = 'contracts/vocabulary.yaml';
 
-/**
- * The ensure expression injected into the build flow's `review` step.
- * Empty-list literal + git_fallback=True: never references a possibly-missing
- * `result.*` attribute; scans the uncommitted working-tree diff vs HEAD (the
- * implementation changes are merged but not yet committed at review time).
- * Python-evaluated by the Stratum executor — hence `[]` and `True`.
- */
-export const VOCABULARY_ENSURE = `vocabulary_compliance('${VOCABULARY_FILE}', [], True)`;
-
 /** Starter vocabulary file: all comments, so `_load_vocabulary` returns {} (inert) until edited. */
 export const VOCABULARY_TEMPLATE = `# contracts/vocabulary.yaml — project naming vocabulary (STRAT-VOCAB)
 #
@@ -58,30 +47,6 @@ export function vocabularyEnabled(cwd, composeConfig) {
   return existsSync(join(cwd, VOCABULARY_FILE));
 }
 
-/**
- * Append the vocabulary ensure to the EXECUTED flow's `review` step (idempotent).
- * Mutates and returns specObj. No-op when that flow has no `review` step.
- *
- * `flowName` must be the flow Stratum will actually run (build.js resolves it via
- * extractFlowName). Targeting it precisely matters because a sub-flow
- * (`review_check`) ALSO has a step id'd `review`; injecting there would be wrong.
- * When `flowName` is omitted/unknown, fall back to the `build` flow (or the first),
- * matching the shipped templates.
- */
-export function injectVocabularyEnsure(specObj, flowName) {
-  const flows = specObj?.flows ?? {};
-  const keys = Object.keys(flows);
-  const flowKey = flowName && keys.includes(flowName)
-    ? flowName
-    : (keys.includes('build') ? 'build' : keys[0]);
-  const steps = flows[flowKey]?.steps ?? [];
-  const review = steps.find((s) => s?.id === 'review');
-  if (!review) return specObj;
-  if (!Array.isArray(review.ensure)) review.ensure = [];
-  if (!review.ensure.includes(VOCABULARY_ENSURE)) review.ensure.push(VOCABULARY_ENSURE);
-  return specObj;
-}
-
 /**
  * Tag vocabulary failure strings as must-fix for the findings display.
  * The cli-progress parser classifies a string by keyword (defaults to `nit`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@smartmemory/compose",
-  "version": "0.2.58-beta",
+  "version": "0.3.0",
   "description": "Structured AI dev pipeline — goal-to-product orchestration with gates, iteration loops, and feature lifecycle management.",
   "author": "SmartMemory",
   "license": "MIT",
@@ -85,6 +85,7 @@
     "@radix-ui/react-toggle": "^1.1.10",
     "@radix-ui/react-toggle-group": "^1.1.11",
     "@radix-ui/react-tooltip": "^1.2.8",
+    "@smartmemory/stratum": "0.3.2",
     "@tanstack/react-virtual": "^3.13.23",
     "ajv": "^8.18.0",
     "ajv-formats": "^3.0.1",

package/pipelines/build-quick.stratum.yaml

@@ -112,7 +112,6 @@ flows:
       task:               {type: string}
       blueprint:          {type: string}
       diff:               {type: string}
-      prior_dirty_lenses: {type: array, optional: true}
     output: ReviewResult
     steps:
       - id: triage
@@ -120,19 +119,9 @@ flows:
         intent: >
           Decide which review lenses to activate.
 
-          RETRY PATH — If the file .compose/prior_dirty_lenses.json exists, read it.
-          It contains a JSON array of lens IDs that had actionable findings in the prior
-          review run. In this case:
-            - Activate all lenses listed in that array.
-            - Always also include diff-quality and contract-compliance (baseline lenses —
-              they re-run on every retry to catch regressions introduced by the fix, even
-              if they passed clean last time).
-            - Skip all other lenses — they already passed clean.
-
-          FIRST RUN PATH — If .compose/prior_dirty_lenses.json does not exist:
-            - Always include diff-quality and contract-compliance.
-            - Add security if files touch auth, crypto, SQL, HTTP handlers.
-            - Add framework if detected framework files (React, Express, Next.js, etc).
+          Always include diff-quality and contract-compliance.
+          Add security if files touch auth, crypto, SQL, HTTP handlers.
+          Add framework if detected framework files (React, Express, Next.js, etc).
 
           Return JSON: { "tasks": LensTask[] } where each task has id, lens_name,
           lens_focus, confidence_gate, exclusions.
@@ -314,8 +303,7 @@ flows:
         depends_on: [decompose]
 
       # Sub-flow: Review (parallel multi-lens review; falls back to review_check)
-      # Parent-step ensure drives the fix loop: ensure_failed -> build.js fix ->
-      # retry the whole parallel_review sub-flow, which re-runs fresh triage/lenses/merge.
+      # Parent-step ensure retries the whole review flow after a failed result.
       # Quick path: review reference is the design doc (no blueprint step exists).
       - id: review
         flow: parallel_review
@@ -337,7 +325,7 @@ flows:
           - "result.clean == True"
         depends_on: [review]
 
-      # Sub-flow: Coverage (claude runs tests; build.js dispatches fix on ensure_failed)
+      # Sub-flow: Coverage (claude runs tests)
       - id: coverage
         flow: coverage_check
         inputs:

package/pipelines/build.profiles.json

@@ -0,0 +1,11 @@
+{
+  "_comment": "D6 (STRAT-TS-FANOUT-CONSUMER): compose-owned agent profiles for build.stratum.yaml. The TS engine accepts only the literal claude|codex agent, so the full profile strings (tool restrictions + model tier) that the v0.3->v1 conversion stripped live here, keyed by step id, and are applied compose-side at invocation (lib/build.js loadPipelineProfiles). The isolation:none review_lenses fanout MUST bind the read-only-reviewer restriction.",
+  "_reduceSteps": ["review_merge"],
+  "_reduceSteps_comment": "H1: ReviewResult-out steps that MERGE/deduplicate rather than review. They get review normalization (confidence handling) but NOT the reviewer scaffold. This is python's stripped reduce_mode input, restored compose-side.",
+  "blueprint": "claude::critical",
+  "review_triage": "claude:orchestrator",
+  "review_lenses": "claude:read-only-reviewer",
+  "review_merge": "claude:orchestrator",
+  "run_tests": "claude::fast",
+  "ship": "claude::critical"
+}