@smartmemory/compose 0.1.0

package/lib/step-prompt.js

@@ -0,0 +1,260 @@
+/**
+ * Step Prompt Builder — constructs agent prompts from Stratum step dispatch responses.
+ */
+
+import { readdirSync, readFileSync, existsSync } from 'node:fs';
+import { join } from 'node:path';
+import { checkStaleness } from './staleness.js';
+
+// ---------------------------------------------------------------------------
+// Ambient context cache — loaded once per build, keyed by contextDir path.
+// Cleared between builds by passing context.contextDir on first call.
+// ---------------------------------------------------------------------------
+
+const _contextCache = new Map();
+
+/**
+ * Load and concatenate all .md files from docs/context/ (or the configured
+ * contextDir). Returns the combined text or null if the directory is absent.
+ * Results are cached so disk reads happen once per build context dir.
+ *
+ * @param {string} contextDir - Absolute path to the context directory
+ * @returns {string|null}
+ */
+export function loadAmbientContext(contextDir) {
+  if (!contextDir || !existsSync(contextDir)) return null;
+  if (_contextCache.has(contextDir)) return _contextCache.get(contextDir);
+
+  let files;
+  try {
+    files = readdirSync(contextDir).filter(f => f.endsWith('.md')).sort();
+  } catch {
+    return null;
+  }
+
+  const parts = [];
+  for (const filename of files) {
+    try {
+      const content = readFileSync(join(contextDir, filename), 'utf-8').trimEnd();
+      if (content) parts.push(content);
+    } catch {
+      // skip unreadable files
+    }
+  }
+
+  const combined = parts.length > 0 ? parts.join('\n\n') : null;
+  _contextCache.set(contextDir, combined);
+  return combined;
+}
+
+/**
+ * Clear the ambient context cache for a given contextDir (call at build start).
+ *
+ * @param {string} contextDir
+ */
+export function clearAmbientContextCache(contextDir) {
+  if (contextDir) _contextCache.delete(contextDir);
+}
+
+/**
+ * Build an agent prompt from a step dispatch and execution context.
+ *
+ * @param {object} stepDispatch - Stratum step dispatch (step_id, intent, inputs, output_fields, ensure)
+ * @param {object} context      - Execution context (cwd, featureCode, contextDir?)
+ * @returns {string}
+ */
+export function buildStepPrompt(stepDispatch, context) {
+  const sections = [];
+
+  sections.push(`You are executing step "${stepDispatch.step_id}" in a Stratum workflow.`);
+
+  sections.push(`## Intent\n${stepDispatch.intent}`);
+
+  sections.push(`## Inputs\n${JSON.stringify(stepDispatch.inputs, null, 2)}`);
+
+  if (Array.isArray(stepDispatch.output_fields) && stepDispatch.output_fields.length > 0) {
+    const fieldLines = stepDispatch.output_fields
+      .map(f => `- ${f.name} (${f.type})`)
+      .join('\n');
+    sections.push(`## Expected Output\nReturn a JSON object with these fields:\n${fieldLines}`);
+  }
+
+  if (Array.isArray(stepDispatch.ensure) && stepDispatch.ensure.length > 0) {
+    const ensureLines = stepDispatch.ensure.map(e => `- ${e}`).join('\n');
+    sections.push(`## Postconditions\nYour result must satisfy:\n${ensureLines}`);
+  }
+
+  // Inject ambient project context (docs/context/*.md) — cached per build
+  if (context.contextDir) {
+    const ambient = loadAmbientContext(context.contextDir);
+    if (ambient) {
+      sections.push(`## Project Context\n${ambient}`);
+    }
+  }
+
+  const ctxLines = [
+    `Working directory: ${context.cwd}`,
+    `Feature: ${context.featureCode}`,
+  ];
+  if (context.featureDir) {
+    ctxLines.push(`Feature docs: ${context.featureDir}`);
+  }
+  sections.push(`## Context\n${ctxLines.join('\n')}`);
+
+  // Inject prior step results so the agent doesn't re-explore from scratch
+  if (Array.isArray(context.stepHistory) && context.stepHistory.length > 0) {
+    const historyLines = context.stepHistory.map(h => {
+      let line = `- **${h.stepId}**: ${h.summary}`;
+      if (h.artifact) line += ` → \`${h.artifact}\``;
+      return line;
+    });
+    sections.push(`## Prior Steps\n${historyLines.join('\n')}`);
+
+    // If any prior step captured a file manifest, include it for downstream steps
+    // (context.filesChanged is maintained as a pre-deduplicated array in build.js)
+    if (context.filesChanged?.length > 0) {
+      sections.push(`## Files Changed by This Feature\n${context.filesChanged.map(f => '- ' + f).join('\n')}`);
+    }
+  }
+
+  return sections.join('\n\n');
+}
+
+/**
+ * Build a "File Ownership Conflicts" section for decompose-step retry prompts.
+ *
+ * @param {Array<{task_a: string, task_b: string, files: string[]}>} conflicts
+ * @returns {string}
+ */
+function buildConflictSection(conflicts) {
+  const lines = [
+    '## File Ownership Conflicts — Resolution Required',
+    '',
+    'The following task pairs share `files_owned` entries but have no `depends_on`',
+    'relationship. Independent tasks may not both claim the same file.',
+    'Add a `depends_on` edge from the later task to the earlier task to resolve each conflict:',
+    '',
+  ];
+
+  for (const { task_a, task_b, files } of conflicts) {
+    lines.push(`- **${task_a}** and **${task_b}** both own:`);
+    for (const f of files) lines.push(`    - \`${f}\``);
+    lines.push(`  → Add \`depends_on: [${task_a}]\` to \`${task_b}\` (or vice versa).`);
+    lines.push('');
+  }
+
+  return lines.join('\n');
+}
+
+/**
+ * Build a retry prompt when postconditions failed.
+ *
+ * @param {object}   stepDispatch - Original step dispatch
+ * @param {string[]} violations   - List of postcondition violations
+ * @param {object}   context      - Execution context
+ * @param {Array<{task_a, task_b, files}>} [conflicts] - Structured file conflicts (optional)
+ * @returns {string}
+ */
+export function buildRetryPrompt(stepDispatch, violations, context, conflicts) {
+  const violationLines = violations.map(v => `- ${v}`).join('\n');
+  const header = `RETRY — Previous attempt failed postconditions:\n${violationLines}\n\nFix these issues and try again.`;
+
+  const sections = [header, buildStepPrompt(stepDispatch, context)];
+
+  if (conflicts && conflicts.length > 0) {
+    sections.push(buildConflictSection(conflicts));
+  }
+
+  return sections.join('\n\n');
+}
+
+/**
+ * Build a prompt for a child flow step within a larger workflow.
+ *
+ * @param {object} flowDispatch - Flow dispatch (child_flow_name, child_step)
+ * @param {object} context      - Execution context
+ * @returns {string}
+ */
+export function buildFlowStepPrompt(flowDispatch, context) {
+  const header = `You are executing a sub-workflow "${flowDispatch.child_flow_name}" as part of a larger workflow.`;
+  return `${header}\n\n${buildStepPrompt(flowDispatch.child_step, context)}`;
+}
+
+/**
+ * Build context preamble for a gate Q&A agent.
+ *
+ * Assembles the same execution context that regular steps get so the agent
+ * answering gate questions knows what feature is being built, what just
+ * completed, what files were touched, and what the gate controls.
+ *
+ * @param {object} gateDispatch - Stratum gate dispatch (step_id, on_approve, on_revise, on_kill)
+ * @param {object} context      - Execution context (cwd, featureCode, featureDir, stepHistory, filesChanged)
+ * @param {object} [gateExtras] - Optional enrichment (fromPhase, toPhase, summary)
+ * @returns {string}
+ */
+export function buildGateContext(gateDispatch, context, gateExtras) {
+  const sections = [];
+
+  sections.push(
+    `You are answering questions about a gate review in a Compose build workflow.\n` +
+    `Gate: "${gateDispatch.step_id}"`,
+  );
+
+  // Feature identity
+  const ctxLines = [
+    `Working directory: ${context.cwd}`,
+    `Feature: ${context.featureCode}`,
+  ];
+  if (context.featureDir) {
+    ctxLines.push(`Feature docs: ${context.featureDir}`);
+  }
+  sections.push(`## Feature\n${ctxLines.join('\n')}`);
+
+  // Phase transition
+  if (gateExtras?.fromPhase || gateExtras?.toPhase) {
+    const from = gateExtras.fromPhase ?? '(unknown)';
+    const to = gateExtras.toPhase ?? '(unknown)';
+    sections.push(`## Phase Transition\n${from} → ${to}`);
+  }
+
+  // Gate summary (from stratum dispatch enrichment)
+  if (gateExtras?.summary) {
+    sections.push(`## Gate Summary\n${gateExtras.summary}`);
+  }
+
+  // Routing — what happens on each decision
+  const routing = [];
+  routing.push(`- **Approve** → ${gateDispatch.on_approve ?? '(complete flow)'}`);
+  routing.push(`- **Revise** → re-run from \`${gateDispatch.on_revise ?? '(kill)'}\``);
+  routing.push(`- **Kill** → ${gateDispatch.on_kill ?? '(terminate flow)'}`);
+  sections.push(`## Gate Routing\n${routing.join('\n')}`);
+
+  // Prior step history
+  if (Array.isArray(context.stepHistory) && context.stepHistory.length > 0) {
+    const historyLines = context.stepHistory.map(h => {
+      let line = `- **${h.stepId}**: ${h.summary}`;
+      if (h.artifact) line += ` → \`${h.artifact}\``;
+      return line;
+    });
+    sections.push(`## Prior Steps\n${historyLines.join('\n')}`);
+  }
+
+  // Files changed
+  if (context.filesChanged?.length > 0) {
+    sections.push(`## Files Changed by This Feature\n${context.filesChanged.map(f => '- ' + f).join('\n')}`);
+  }
+
+  // Staleness warnings — flag artifacts that belong to an earlier phase
+  if (context.featureDir && gateExtras?.toPhase) {
+    const staleArtifacts = checkStaleness(context.featureDir, gateExtras.toPhase);
+    const stale = staleArtifacts.filter(a => a.stale);
+    if (stale.length > 0) {
+      const lines = stale.map(a =>
+        `- **${a.file}** was written in phase \`${a.writtenPhase}\` but feature is now in \`${a.currentPhase}\``
+      );
+      sections.push(`## Stale Artifacts\nThe following artifacts may be outdated:\n${lines.join('\n')}`);
+    }
+  }
+
+  return sections.join('\n\n');
+}

package/lib/step-validator.js

@@ -0,0 +1,49 @@
+/**
+ * step-validator.js — Agent-as-validator for pipeline step outputs.
+ *
+ * After a step writes its artifact, dispatches a lightweight agent call
+ * to read the artifact and validate it against criteria. Returns
+ * { valid, issues } so the dispatch loop can retry if needed.
+ */
+
+import { runAndNormalize } from './result-normalizer.js';
+
+/**
+ * Run a validation agent call for a completed step.
+ *
+ * @param {object} opts
+ * @param {string}   opts.artifact  - File path to validate (relative to cwd)
+ * @param {string[]} opts.criteria  - List of things to check
+ * @param {string}   opts.stepId    - Step ID (for logging)
+ * @param {object}   opts.connector - Agent connector to use
+ * @returns {Promise<{ valid: boolean, issues: string[] }>}
+ */
+export async function validateStep({ artifact, criteria, stepId, connector }) {
+  const prompt =
+    `You are a validator. Read the file "${artifact}" and check the following criteria:\n\n` +
+    criteria.map((c, i) => `${i + 1}. ${c}`).join('\n') + '\n\n' +
+    `Return ONLY a JSON code block — no other text:\n` +
+    '```json\n' +
+    '{ "valid": true, "issues": [] }\n' +
+    '```\n' +
+    `Set valid to false and list issues if any criterion is not met.`;
+
+  // Minimal dispatch descriptor — only output_fields needed for JSON extraction
+  const dispatch = {
+    step_id: `validate_${stepId}`,
+    output_fields: {
+      valid: 'boolean',
+      issues: 'array',
+    },
+  };
+
+  const { result } = await runAndNormalize(connector, prompt, dispatch);
+
+  if (!result || typeof result.valid !== 'boolean') {
+    // Extraction failed — assume valid (optimistic fallback)
+    process.stderr.write(`    ⚠ Validator returned no structured result for ${stepId}, assuming valid\n`);
+    return { valid: true, issues: [] };
+  }
+
+  return { valid: result.valid, issues: result.issues ?? [] };
+}

package/lib/stratum-mcp-client.js

@@ -0,0 +1,365 @@
+/**
+ * stratum-mcp-client.js — MCP protocol client for stratum-mcp.
+ *
+ * Spawns `stratum-mcp` (no subcommand) as a child process and communicates
+ * via the MCP SDK over stdio. This is for the build runner's plan/step_done
+ * loop — distinct from server/stratum-client.js which uses CLI subcommands.
+ *
+ * Usage:
+ *   const client = new StratumMcpClient();
+ *   await client.connect();
+ *   const dispatch = await client.plan(specPath, 'build', { featureCode: 'FEAT-1' });
+ *   const next = await client.stepDone(dispatch.flow_id, 'step1', { phase: 'design' });
+ *   await client.close();
+ */
+
+import { execFileSync } from 'node:child_process';
+import { Client } from '@modelcontextprotocol/sdk/client/index.js';
+import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js';
+
+export class StratumError extends Error {
+  constructor(code, message, detail) {
+    super(message);
+    this.name = 'StratumError';
+    this.code = code;
+    this.detail = detail;
+  }
+}
+
+export class StratumMcpClient {
+  #client = null;
+  #transport = null;
+  #connected = false;
+
+  /**
+   * Spawn stratum-mcp and establish MCP connection.
+   * @param {object} [opts]
+   * @param {string} [opts.command] - Override binary (for testing)
+   * @param {string[]} [opts.args] - Override args
+   * @param {string}   [opts.cwd]  - Working directory for the subprocess
+   */
+  async connect(opts = {}) {
+    if (this.#connected) return;
+
+    const command = opts.command ?? 'stratum-mcp';
+    const args = opts.args ?? [];
+
+    // Pre-flight: verify binary exists on $PATH (skip for test overrides)
+    if (command === 'stratum-mcp') {
+      try {
+        execFileSync('which', [command], { stdio: 'pipe', timeout: 3000 });
+      } catch {
+        throw new Error(
+          'stratum-mcp not found on $PATH. Install with: pip install stratum-mcp'
+        );
+      }
+    }
+
+    const transportOpts = { command, args, stderr: 'pipe' };
+    if (opts.cwd) transportOpts.cwd = opts.cwd;
+    this.#transport = new StdioClientTransport(transportOpts);
+
+    this.#client = new Client(
+      { name: 'compose-build', version: '1.0.0' },
+      { capabilities: {} }
+    );
+
+    await this.#client.connect(this.#transport);
+    this.#connected = true;
+  }
+
+  /** Kill subprocess and clean up. */
+  async close() {
+    if (!this.#connected) return;
+    try {
+      await this.#client.close();
+    } catch {
+      // Ignore close errors — process may already be dead
+    }
+    this.#client = null;
+    this.#transport = null;
+    this.#connected = false;
+  }
+
+  /**
+   * Call an MCP tool and return the parsed JSON result.
+   * @param {string} toolName
+   * @param {object} args
+   * @returns {Promise<any>}
+   */
+  async #callTool(toolName, args) {
+    // Allow test-injected client to bypass real connection requirement.
+    // Gated on NODE_ENV=test so production code cannot accidentally redirect calls.
+    const client = (process.env.NODE_ENV === 'test' && this._testClient) || null;
+    if (!client && !this.#connected) {
+      throw new Error('StratumMcpClient not connected. Call connect() first.');
+    }
+
+    const result = await (client ?? this.#client).callTool({
+      name: toolName,
+      arguments: args,
+    });
+
+    // MCP tool results come back as content array; extract text content
+    const textContent = result.content?.find(c => c.type === 'text');
+    if (!textContent) {
+      throw new StratumError('EMPTY_RESPONSE', `Tool ${toolName} returned no text content`, '');
+    }
+
+    // MCP isError flag indicates tool-level failure
+    if (result.isError) {
+      throw new StratumError('TOOL_ERROR', textContent.text, '');
+    }
+
+    let parsed;
+    try {
+      parsed = JSON.parse(textContent.text);
+    } catch {
+      // Try to extract JSON from text that may have surrounding prose
+      const jsonMatch = textContent.text.match(/\{[\s\S]*\}/);
+      if (jsonMatch) {
+        try {
+          parsed = JSON.parse(jsonMatch[0]);
+        } catch {
+          throw new StratumError('PARSE_ERROR', `Tool ${toolName} returned invalid JSON`, textContent.text);
+        }
+      } else {
+        throw new StratumError('PARSE_ERROR', `Tool ${toolName} returned invalid JSON`, textContent.text);
+      }
+    }
+
+    // Check for Stratum error envelope
+    if (parsed.status === 'error' || parsed.error) {
+      const err = parsed.error ?? parsed;
+      throw new StratumError(
+        err.code ?? 'STRATUM_ERROR',
+        err.message ?? 'Stratum tool call failed',
+        err.detail ?? ''
+      );
+    }
+
+    return parsed;
+  }
+
+  /**
+   * Start a flow. Returns the first step dispatch.
+   * @param {string} spec - Inline YAML spec content (not a file path)
+   * @param {string} flow - Flow name within the spec
+   * @param {object} inputs - Flow input values
+   * @returns {Promise<object>} Step dispatch response
+   */
+  async plan(spec, flow, inputs) {
+    return this.#callTool('stratum_plan', { spec, flow, inputs });
+  }
+
+  /**
+   * Resume an in-progress flow. Returns the current step dispatch.
+   * @param {string} flowId
+   * @returns {Promise<object>} Step dispatch response (same format as plan/stepDone)
+   */
+  async resume(flowId) {
+    return this.#callTool('stratum_resume', { flow_id: flowId });
+  }
+
+  /**
+   * Report step completion. Returns next step dispatch or completion.
+   * @param {string} flowId
+   * @param {string} stepId
+   * @param {object} result - Step result (must match output_contract)
+   * @returns {Promise<object>}
+   */
+  async stepDone(flowId, stepId, result) {
+    return this.#callTool('stratum_step_done', {
+      flow_id: flowId,
+      step_id: stepId,
+      result,
+    });
+  }
+
+  /**
+   * Resolve a gate step.
+   * @param {string} flowId
+   * @param {string} stepId
+   * @param {'approve'|'revise'|'kill'} outcome
+   * @param {string} rationale
+   * @param {'human'|'agent'|'system'} resolvedBy
+   * @returns {Promise<object>}
+   */
+  async gateResolve(flowId, stepId, outcome, rationale, resolvedBy = 'human') {
+    return this.#callTool('stratum_gate_resolve', {
+      flow_id: flowId,
+      step_id: stepId,
+      outcome,
+      rationale,
+      resolved_by: resolvedBy,
+    });
+  }
+
+  /**
+   * Skip the current step with a recorded reason.
+   * @param {string} flowId
+   * @param {string} stepId
+   * @param {string} reason
+   * @returns {Promise<object>}
+   */
+  async skipStep(flowId, stepId, reason) {
+    return this.#callTool('stratum_skip_step', {
+      flow_id: flowId,
+      step_id: stepId,
+      reason,
+    });
+  }
+
+  /**
+   * Get the full execution trace.
+   * @param {string} flowId
+   * @returns {Promise<object>}
+   */
+  async audit(flowId) {
+    return this.#callTool('stratum_audit', { flow_id: flowId });
+  }
+
+  /**
+   * Start a counted iteration loop on a step.
+   * @param {string} flowId
+   * @param {string} stepId
+   * @returns {Promise<object>}
+   */
+  async iterationStart(flowId, stepId) {
+    return this.#callTool('stratum_iteration_start', {
+      flow_id: flowId,
+      step_id: stepId,
+    });
+  }
+
+  /**
+   * Report one iteration result.
+   * @param {string} flowId
+   * @param {string} stepId
+   * @param {object} result
+   * @returns {Promise<object>}
+   */
+  async iterationReport(flowId, stepId, result) {
+    return this.#callTool('stratum_iteration_report', {
+      flow_id: flowId,
+      step_id: stepId,
+      result,
+    });
+  }
+
+  /**
+   * Abort an iteration loop early.
+   * @param {string} flowId
+   * @param {string} stepId
+   * @param {string} reason
+   * @returns {Promise<object>}
+   */
+  async iterationAbort(flowId, stepId, reason) {
+    return this.#callTool('stratum_iteration_abort', {
+      flow_id: flowId,
+      step_id: stepId,
+      reason,
+    });
+  }
+
+  /**
+   * Validate a spec without executing.
+   * @param {string} spec - Inline YAML spec content
+   * @returns {Promise<{valid: boolean, errors?: string[]}>}
+   */
+  async validate(spec) {
+    return this.#callTool('stratum_validate', { spec });
+  }
+
+  /**
+   * Create a named checkpoint.
+   * @param {string} flowId
+   * @param {string} label
+   * @returns {Promise<object>}
+   */
+  async commit(flowId, label) {
+    return this.#callTool('stratum_commit', {
+      flow_id: flowId,
+      label,
+    });
+  }
+
+  /**
+   * Roll back to a checkpoint.
+   * @param {string} flowId
+   * @param {string} label
+   * @returns {Promise<object>}
+   */
+  async revert(flowId, label) {
+    return this.#callTool('stratum_revert', {
+      flow_id: flowId,
+      label,
+    });
+  }
+
+  /**
+   * Report batch task results for a parallel_dispatch step.
+   * @param {string} flowId
+   * @param {string} stepId
+   * @param {Array<{task_id: string, status: string, result?: object, error?: string}>} taskResults
+   * @param {'clean'|'conflict'|'fallback'|'manual_required'} mergeStatus
+   * @returns {Promise<object>} Next dispatch response
+   */
+  async parallelDone(flowId, stepId, taskResults, mergeStatus) {
+    return this.#callTool('stratum_parallel_done', {
+      flow_id:      flowId,
+      step_id:      stepId,
+      task_results: taskResults,
+      merge_status: mergeStatus,
+    });
+  }
+
+  /**
+   * Start server-side execution of a parallel_dispatch step (T2-F5-COMPOSE-MIGRATE).
+   * Returns {status: 'started', ...} on success or {error, message} on known error.
+   * @param {string} flowId
+   * @param {string} stepId
+   * @returns {Promise<object>}
+   */
+  async parallelStart(flowId, stepId) {
+    return this.#callTool('stratum_parallel_start', {
+      flow_id: flowId,
+      step_id: stepId,
+    });
+  }
+
+  /**
+   * Poll state of a server-dispatched parallel_dispatch step (T2-F5-COMPOSE-MIGRATE).
+   * Returns {summary, tasks, require_satisfied, can_advance, outcome}.
+   * Break on `outcome != null`, not `can_advance` — see design doc §3.
+   * @param {string} flowId
+   * @param {string} stepId
+   * @returns {Promise<object>}
+   */
+  async parallelPoll(flowId, stepId) {
+    return this.#callTool('stratum_parallel_poll', {
+      flow_id: flowId,
+      step_id: stepId,
+    });
+  }
+
+  /**
+   * Consumer-driven advance for parallel_dispatch steps with defer_advance:true.
+   * Call after observing outcome.status === 'awaiting_consumer_advance' from
+   * parallelPoll. Feeds merge_status back to Stratum which runs
+   * _evaluate_parallel_results + _advance_after_parallel and returns the real
+   * advance outcome.
+   *
+   * @param {string} flowId
+   * @param {string} stepId
+   * @param {'clean'|'conflict'} mergeStatus
+   * @returns {Promise<object>}
+   */
+  async parallelAdvance(flowId, stepId, mergeStatus) {
+    return this.#callTool('stratum_parallel_advance', {
+      flow_id: flowId,
+      step_id: stepId,
+      merge_status: mergeStatus,
+    });
+  }
+}