@smartmemory/compose 0.1.0

package/lib/result-normalizer.js

@@ -0,0 +1,349 @@
+/**
+ * Result Normalizer — bridges connector text streams to structured step results
+ * for the headless build runner.
+ *
+ * Converts flat Stratum output_fields to JSON Schema, runs a connector,
+ * accumulates text, and extracts structured JSON from the response.
+ */
+
+import { injectSchema } from '../server/connectors/agent-connector.js';
+import { CliProgress } from './cli-progress.js';
+import { calculateCost } from './model-pricing.js';
+
+// ---------------------------------------------------------------------------
+// Error classes
+// ---------------------------------------------------------------------------
+
+export class ResultParseError extends Error {
+  /**
+   * @param {string} message
+   * @param {string} rawText  The raw connector output that could not be parsed
+   */
+  constructor(message, rawText) {
+    super(message);
+    this.name = 'ResultParseError';
+    this.rawText = rawText;
+  }
+}
+
+export class AgentError extends Error {
+  constructor(message) {
+    super(message);
+    this.name = 'AgentError';
+  }
+}
+
+export class UserInterruptError extends Error {
+  /** @param {string} stepId @param {'skip'|'retry'} action */
+  constructor(stepId, action) {
+    super(`User requested ${action} for step "${stepId}"`);
+    this.name = 'UserInterruptError';
+    this.stepId = stepId;
+    this.action = action;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Schema conversion
+// ---------------------------------------------------------------------------
+
+/** Map from Stratum flat type names to JSON Schema property descriptors. */
+const TYPE_MAP = {
+  string:  { type: 'string' },
+  boolean: { type: 'boolean' },
+  integer: { type: 'integer' },
+  number:  { type: 'number' },
+  array:   { type: 'array' },
+  object:  { type: 'object' },
+};
+
+/**
+ * Convert Stratum's flat output_fields type map to a JSON Schema object.
+ *
+ * @param {Record<string, string>} outputFields  e.g. { "clean": "boolean", "findings": "array" }
+ * @returns {object} A JSON Schema object with type, required, and properties.
+ */
+export function outputFieldsToJsonSchema(outputFields) {
+  const properties = {};
+  const required = Object.keys(outputFields);
+
+  for (const [key, typeStr] of Object.entries(outputFields)) {
+    const lower = typeStr.toLowerCase();
+    properties[key] = TYPE_MAP[lower] ?? {}; // any/unknown → unconstrained
+  }
+
+  return {
+    type: 'object',
+    required,
+    properties,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// JSON extraction helpers
+// ---------------------------------------------------------------------------
+
+/**
+ * Try to extract a JSON object from text using multiple strategies.
+ *
+ * @param {string} text
+ * @returns {object|null} Parsed JSON or null if all strategies fail.
+ */
+function extractJson(text) {
+  // Strategy A: full text is valid JSON
+  try {
+    return JSON.parse(text);
+  } catch { /* continue */ }
+
+  // Strategy B: fenced ```json ... ``` block
+  const fenceMatch = text.match(/```json\s*\n?([\s\S]*?)```/);
+  if (fenceMatch) {
+    try {
+      return JSON.parse(fenceMatch[1].trim());
+    } catch { /* continue */ }
+  }
+
+  // Strategy C: first balanced { ... } substring
+  const startIdx = text.indexOf('{');
+  if (startIdx !== -1) {
+    let depth = 0;
+    for (let i = startIdx; i < text.length; i++) {
+      if (text[i] === '{') depth++;
+      else if (text[i] === '}') depth--;
+      if (depth === 0) {
+        try {
+          return JSON.parse(text.slice(startIdx, i + 1));
+        } catch { /* continue */ }
+        break;
+      }
+    }
+  }
+
+  return null;
+}
+
+// ---------------------------------------------------------------------------
+// Main entry point
+// ---------------------------------------------------------------------------
+
+/**
+ * Run a connector and normalize its output to a structured result.
+ *
+ * @param {object} connector         Object with a run(prompt, opts) async generator method.
+ * @param {string} prompt            The prompt to send to the connector.
+ * @param {object} stepDispatch      Step dispatch descriptor.
+ * @param {Record<string, string>} [stepDispatch.output_fields]  Expected output fields.
+ * @param {object} [opts]
+ * @param {CliProgress} [opts.progress]  CLI progress renderer.
+ * @returns {Promise<{ text: string, result: object|null }>}
+ */
+export class AgentTimeoutError extends Error {
+  constructor(stepId, durationMs) {
+    super(`Agent timed out on step "${stepId}" after ${Math.round(durationMs / 1000)}s`);
+    this.name = 'AgentTimeoutError';
+    this.stepId = stepId;
+    this.durationMs = durationMs;
+  }
+}
+
+export async function runAndNormalize(connector, prompt, stepDispatch, opts = {}) {
+  const progress = opts.progress;
+  const streamWriter = opts.streamWriter;
+  const onToolUse = opts.onToolUse ?? null; // passive tap — Item 193
+  const maxDurationMs = opts.maxDurationMs ?? null; // null = no timeout
+  const outputFields = stepDispatch.output_fields;
+  const hasSchema = outputFields && typeof outputFields === 'object' && Object.keys(outputFields).length > 0;
+
+  let actualPrompt = prompt;
+  let schema = null;
+
+  if (hasSchema) {
+    schema = outputFieldsToJsonSchema(outputFields);
+    actualPrompt = injectSchema(prompt, schema);
+  }
+
+  const textParts = [];
+  const startTime = Date.now();
+
+  // Accumulate usage events across all connector events
+  const usageTotals = {
+    input_tokens: 0,
+    output_tokens: 0,
+    cache_creation_input_tokens: 0,
+    cache_read_input_tokens: 0,
+    cost_usd: 0,
+    model: null,
+  };
+
+  // Set up timeout timer if configured
+  let timeoutTimer = null;
+  let timedOut = false;
+  if (maxDurationMs) {
+    timeoutTimer = setTimeout(() => {
+      timedOut = true;
+      try { connector.interrupt(); } catch { /* best effort */ }
+    }, maxDurationMs);
+  }
+
+  // Wire up skip/retry interrupt from CLI key press
+  let userInterrupted = false;
+  const onInterrupt = () => {
+    userInterrupted = true;
+    try { connector.interrupt(); } catch { /* best effort */ }
+  };
+  if (progress?.on) progress.on('interrupt', onInterrupt);
+
+  try {
+  for await (const event of connector.run(actualPrompt, {})) {
+    // Check for timeout after each event
+    if (timedOut) {
+      const stepId = stepDispatch.step_id ?? 'unknown';
+      throw new AgentTimeoutError(stepId, Date.now() - startTime);
+    }
+    // Check for user interrupt (skip/retry)
+    if (userInterrupted) {
+      const stepId = stepDispatch.step_id ?? 'unknown';
+      throw new UserInterruptError(stepId, progress?.consumeAction() ?? 'skip');
+    }
+    if (progress) {
+      progress.debug(`event type=${event.type} keys=${Object.keys(event).join(',')}`);
+    } else if (process.env.COMPOSE_DEBUG) {
+      process.stderr.write(`  [event] type=${event.type} keys=${Object.keys(event).join(',')}\n`);
+    }
+    if (event.type === 'error') {
+      // build_error is written by build.js catch blocks, not here — avoids duplicate events
+      throw new AgentError(event.message);
+    }
+    if (event.type === 'assistant' && event.content) {
+      textParts.push(event.content);
+      if (streamWriter) {
+        streamWriter.write({ type: 'assistant', content: event.content });
+      }
+    }
+    if (event.type === 'result' && event.content) {
+      // Result contains the final aggregated text — use it if we got nothing from blocks
+      if (textParts.length === 0) {
+        textParts.push(event.content);
+      }
+    }
+    // Forward tool_use to build stream (before progress logging)
+    if (streamWriter && event.type === 'tool_use' && event.tool) {
+      streamWriter.write({ type: 'tool_use', tool: event.tool, input: event.input });
+    }
+    // Passive tap — notify caller of each tool_use without altering event flow (Item 193)
+    if (onToolUse && event.type === 'tool_use' && event.tool) {
+      onToolUse({ tool: event.tool, input: event.input, timestamp: Date.now() });
+    }
+    // Progress logging — show tool calls so the user sees activity
+    if (event.type === 'tool_use' && event.tool) {
+      const detail = event.input?.command
+        ?? event.input?.pattern
+        ?? event.input?.query
+        ?? event.input?.file_path
+        ?? '';
+      if (progress) {
+        progress.toolUse(event.tool, detail);
+      } else {
+        const short = typeof detail === 'string' && detail.length > 60
+          ? detail.slice(0, 57) + '...'
+          : detail;
+        process.stderr.write(`    ↳ ${event.tool}${short ? ': ' + short : ''}\n`);
+      }
+    }
+    // Forward tool_use_summary to build stream (COMP-OBS-STREAM)
+    if (streamWriter && event.type === 'tool_use_summary') {
+      streamWriter.write({ type: 'tool_use_summary', summary: event.summary, output: event.output });
+    }
+    if (event.type === 'tool_use_summary' && event.summary) {
+      if (progress) {
+        progress.toolSummary(event.summary);
+      } else {
+        const short = event.summary.length > 80
+          ? event.summary.slice(0, 77) + '...'
+          : event.summary;
+        process.stderr.write(`    ↳ ${short}\n`);
+      }
+    }
+    // Forward tool_progress to build stream (COMP-OBS-STREAM)
+    if (streamWriter && event.type === 'tool_progress') {
+      streamWriter.write({ type: 'tool_progress', tool: event.tool, elapsed: event.elapsed });
+    }
+    if (event.type === 'tool_progress' && event.tool) {
+      if (progress) {
+        progress.toolProgress(event.tool, event.elapsed);
+      } else {
+        process.stderr.write(`    ↳ ${event.tool} (${Math.round(event.elapsed)}s)\n`);
+      }
+    }
+    // Accumulate usage events (COMP-OBS-COST)
+    if (event.type === 'usage') {
+      usageTotals.input_tokens += event.input_tokens ?? 0;
+      usageTotals.output_tokens += event.output_tokens ?? 0;
+      usageTotals.cache_creation_input_tokens += event.cache_creation_input_tokens ?? 0;
+      usageTotals.cache_read_input_tokens += event.cache_read_input_tokens ?? 0;
+      if (event.model) usageTotals.model = event.model;
+      // If the connector already computed cost_usd, use it; otherwise calculate
+      const stepCost = event.cost_usd != null
+        ? event.cost_usd
+        : calculateCost(
+            event.model,
+            event.input_tokens ?? 0,
+            event.output_tokens ?? 0,
+            event.cache_creation_input_tokens ?? 0,
+            event.cache_read_input_tokens ?? 0,
+          );
+      usageTotals.cost_usd += stepCost;
+      // Forward per-step usage to build stream
+      if (streamWriter) {
+        streamWriter.write({
+          type: 'usage',
+          input_tokens: event.input_tokens ?? 0,
+          output_tokens: event.output_tokens ?? 0,
+          cache_creation_input_tokens: event.cache_creation_input_tokens ?? 0,
+          cache_read_input_tokens: event.cache_read_input_tokens ?? 0,
+          cost_usd: stepCost,
+          model: event.model ?? null,
+        });
+      }
+    }
+  }
+  } finally {
+    if (timeoutTimer) clearTimeout(timeoutTimer);
+    if (progress?.removeListener) progress.removeListener('interrupt', onInterrupt);
+  }
+
+  // If we broke out of the loop due to timeout, throw
+  if (timedOut) {
+    const stepId = stepDispatch.step_id ?? 'unknown';
+    throw new AgentTimeoutError(stepId, Date.now() - startTime);
+  }
+
+  const text = textParts.join('');
+
+  if (progress) {
+    progress.debug(`normalizer: textParts=${textParts.length}, text length=${text.length}`);
+    if (text.length > 0) progress.debug(`text preview: ${text.slice(0, 300)}`);
+  } else if (process.env.COMPOSE_DEBUG) {
+    process.stderr.write(`  [normalizer] textParts count: ${textParts.length}, text length: ${text.length}\n`);
+    if (text.length > 0) process.stderr.write(`  [normalizer] text preview: ${text.slice(0, 300)}\n`);
+  }
+
+  if (!hasSchema) {
+    return { text, result: null, usage: usageTotals };
+  }
+
+  const result = extractJson(text);
+  if (result === null) {
+    // Agent did its work but didn't return structured JSON.
+    // Log a warning and return a fallback — don't crash the pipeline.
+    if (progress) {
+      progress.warn('Could not extract JSON from agent output, using fallback');
+    } else {
+      process.stderr.write('    ⚠ Could not extract JSON from agent output, using fallback\n');
+    }
+    const summary = text.slice(0, 200).replace(/\n/g, ' ').trim();
+    return { text, result: { summary: summary || 'Step complete' }, usage: usageTotals };
+  }
+
+  return { text, result, usage: usageTotals };
+}

package/lib/review-lenses.js

@@ -0,0 +1,166 @@
+/**
+ * review-lenses.js — Review lens definitions and triage logic for STRAT-REV.
+ *
+ * Each lens is a specialized reviewer that focuses on one concern.
+ * The triage function decides which lenses to activate based on the file list.
+ */
+
+/** Baseline lenses — always run */
+export const BASELINE_LENSES = ['diff-quality', 'contract-compliance', 'debug-discipline'];
+
+/** Full lens definitions */
+export const LENS_DEFINITIONS = {
+  'diff-quality': {
+    id: 'diff-quality',
+    lens_name: 'diff-quality',
+    lens_focus: 'Code style, test coverage gaps, dead code, naming, duplication. Read the git diff only.',
+    confidence_gate: 6,
+    exclusions: 'Style-only nits without functional impact',
+    reasoning_template: {
+      require_citations: true,
+      sections: [
+        { id: 'premises', label: 'Premises', description: 'List each changed function/block from the diff. Cite file:line for each.' },
+        { id: 'trace', label: 'Quality Trace', description: 'For each premise, evaluate: naming clarity, duplication, error handling, dead code. Reference premises by ID.' },
+        { id: 'findings', label: 'Findings', description: 'List LensFinding items. Each must reference the premise it came from.' },
+      ],
+    },
+  },
+  'contract-compliance': {
+    id: 'contract-compliance',
+    lens_name: 'contract-compliance',
+    lens_focus: 'Does the implementation match the blueprint? Missing acceptance criteria, wrong file paths, unimplemented items. Read blueprint + implementation.',
+    confidence_gate: 7,
+    exclusions: 'Items explicitly deferred in the plan',
+    reasoning_template: {
+      require_citations: true,
+      sections: [
+        { id: 'premises', label: 'Premises', description: 'List each blueprint requirement and the file:line that implements it. List any blueprint item with NO matching implementation.' },
+        { id: 'trace', label: 'Compliance Trace', description: 'For each premise pair (requirement <-> implementation), verify: correct path, correct signature, correct behavior. For unmatched items, confirm they are truly missing.' },
+        { id: 'findings', label: 'Findings', description: 'List compliance gaps as LensFinding items. Each must cite the blueprint requirement [P<n>] and the implementation (or lack thereof).' },
+      ],
+    },
+  },
+  'security': {
+    id: 'security',
+    lens_name: 'security',
+    lens_focus: 'OWASP top 10, injection, secrets in code, insecure defaults. Focus on concrete, exploitable issues.',
+    confidence_gate: 8,
+    exclusions: 'DoS/rate-limiting, memory safety in memory-safe languages, theoretical risks without concrete exploit path',
+    reasoning_template: {
+      require_citations: true,
+      sections: [
+        { id: 'premises', label: 'Premises', description: 'List each security-sensitive operation in the diff: auth checks, SQL queries, user input handling, crypto, secrets. Cite file:line.' },
+        { id: 'trace', label: 'Threat Trace', description: 'For each premise, trace the data flow from source to sink. Identify: is input validated? Is output escaped? Are secrets hardcoded? Reference premises by ID.' },
+        { id: 'findings', label: 'Findings', description: 'List vulnerabilities as LensFinding items with OWASP category. Each must trace back to a specific premise and data flow.' },
+      ],
+    },
+  },
+  'debug-discipline': {
+    id: 'debug-discipline',
+    lens_name: 'debug-discipline',
+    lens_focus: 'Fix-chain detection: are multiple iterations patching the same function? ' +
+      'Trace evidence: was actual data inspected before the fix? ' +
+      'Cross-layer completeness: does a migration/rename address ALL references? ' +
+      'Type contracts: are there isinstance(x, dict) gates hiding type ambiguity?',
+    confidence_gate: 7,
+    exclusions: 'First-attempt fixes with trace evidence, pure refactors',
+    reasoning_template: {
+      require_citations: true,
+      sections: [
+        { id: 'premises', label: 'Premises', description: 'List each fix-iteration file change, trace evidence artifact, cross-layer reference, and type boundary check in the diff. Cite file:line for each.' },
+        { id: 'trace', label: 'Discipline Trace', description: 'For each premise, evaluate: was this a repeated fix to the same location? Was trace evidence produced before the fix? Were all cross-layer references addressed? Are type boundaries explicit?' },
+        { id: 'findings', label: 'Findings', description: 'List discipline violations as LensFinding items. Each must reference the specific premise and the anti-pattern it represents.' },
+      ],
+    },
+  },
+  'framework': {
+    id: 'framework',
+    lens_name: 'framework',
+    lens_focus: 'Framework-specific anti-patterns, deprecated APIs, performance pitfalls for the detected framework.',
+    confidence_gate: 6,
+    exclusions: 'Opinions without measurable impact',
+    reasoning_template: {
+      require_citations: true,
+      sections: [
+        { id: 'premises', label: 'Premises', description: 'List each framework API call or pattern used in the diff. Cite file:line. Note the framework version from package.json/requirements.txt.' },
+        { id: 'trace', label: 'Pattern Trace', description: 'For each premise, check: is this API deprecated? Is there a preferred alternative? Does usage match framework conventions for this version?' },
+        { id: 'findings', label: 'Findings', description: 'List anti-patterns and deprecations as LensFinding items. Each must reference the specific API call [P<n>] and the framework docs justification.' },
+      ],
+    },
+  },
+};
+
+/** File patterns that trigger the security lens */
+const SECURITY_PATTERNS = [
+  /auth/i, /login/i, /session/i, /token/i, /crypt/i, /password/i, /secret/i,
+  /\.sql$/i, /query/i, /sanitiz/i, /escape/i, /middleware/i,
+  /routes?\.(js|ts|jsx|tsx)$/i, /handler/i, /endpoint/i,
+  /api\//i, /server\//i,
+];
+
+/** File patterns that trigger the framework lens */
+const FRAMEWORK_PATTERNS = [
+  /\.(jsx|tsx)$/i,                          // React
+  /next\.config/i, /pages\//i, /app\//i,   // Next.js
+  /express/i, /router/i, /middleware/i,     // Express
+  /\.vue$/i, /nuxt/i,                       // Vue/Nuxt
+  /angular/i,                               // Angular
+];
+
+// ---------------------------------------------------------------------------
+// Diff-size classification (STRAT-REV-7)
+// ---------------------------------------------------------------------------
+
+/**
+ * Classify a diff by number of changed files.
+ *
+ * @param {string[]} filesChanged - list of changed file paths
+ * @returns {'small'|'medium'|'large'}
+ */
+export function classifyDiffSize(filesChanged) {
+  const count = Array.isArray(filesChanged) ? filesChanged.length : 0;
+  if (count <= 2) return 'small';
+  if (count <= 8) return 'medium';
+  return 'large';
+}
+
+/**
+ * Whether cross-model (Codex) review should run for this diff.
+ * Only triggers for large diffs (≥9 files).
+ *
+ * @param {string[]} filesChanged - list of changed file paths
+ * @returns {boolean}
+ */
+export function shouldRunCrossModel(filesChanged) {
+  return classifyDiffSize(filesChanged) === 'large';
+}
+
+// ---------------------------------------------------------------------------
+// Triage
+// ---------------------------------------------------------------------------
+
+/**
+ * Triage: decide which lenses to activate.
+ *
+ * @param {string[]} fileList - list of changed file paths
+ * @returns {Array<object>} LensTask[] for parallel_dispatch
+ */
+export function triageLenses(fileList) {
+  const activeLensIds = [...BASELINE_LENSES];
+
+  const hasSecurityFiles = fileList.some(f =>
+    SECURITY_PATTERNS.some(p => p.test(f))
+  );
+  if (hasSecurityFiles) activeLensIds.push('security');
+
+  const hasFrameworkFiles = fileList.some(f =>
+    FRAMEWORK_PATTERNS.some(p => p.test(f))
+  );
+  if (hasFrameworkFiles) activeLensIds.push('framework');
+
+  return activeLensIds.map(id => {
+    const def = LENS_DEFINITIONS[id];
+    if (!def) throw new Error(`Unknown lens: ${id}`);
+    return { ...def };
+  });
+}