@smartmemory/compose 0.2.52-beta → 0.2.54-beta

package/lib/smartmemory-client.js

@@ -0,0 +1,139 @@
+/**
+ * smartmemory-client.js — COMP-SMARTMEMORY-INGEST S02
+ *
+ * Raw HTTP client for the SmartMemory ingest/search wire contract. Global
+ * `fetch` + `AbortController` timeout. No SDK, no new dependency.
+ */
+
+/**
+ * Thrown on non-2xx from ingest/search, OR on a 2xx whose body doesn't match
+ * the expected shape (non-JSON, or missing the field the caller depends on —
+ * `status` for ingest, `results` for search). The latter case sets
+ * `kind: 'malformed-response'` so callers can tell "the service said no" from
+ * "the service said something we can't trust" if they want to, while both
+ * still surface as one failure type upstream (sync: `failed`; emitter:
+ * counts toward the circuit breaker).
+ */
+export class SmartmemoryHttpError extends Error {
+  constructor(message, status, kind) {
+    super(message);
+    this.name = 'SmartmemoryHttpError';
+    this.status = status;
+    this.kind = kind;
+  }
+}
+
+/**
+ * Build a client bound to a resolved config. The API key is read from
+ * process.env[cfg.apiKeyEnv] at call time (missing ⇒ treated as unreachable).
+ * @param {{ baseUrl: string, apiKeyEnv?: string, timeoutMs?: number }} cfg
+ * @returns {{ health(): Promise<{ok:boolean,status?:number}>, ingest(content:string,ctx:object): Promise<{status:string,unchanged:boolean,raw:object}>, search(query:string,opts?:object): Promise<object> }}
+ */
+export function createSmartmemoryClient(cfg) {
+  const baseUrl = cfg.baseUrl;
+  const timeoutMs = cfg.timeoutMs ?? 3000;
+
+  async function withTimeout(fn) {
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeoutMs);
+    try {
+      return await fn(controller.signal);
+    } finally {
+      clearTimeout(timer);
+    }
+  }
+
+  function authHeader() {
+    const key = cfg.apiKeyEnv ? process.env[cfg.apiKeyEnv] : undefined;
+    if (!key) {
+      throw new SmartmemoryHttpError('smartmemory: missing api key', 0);
+    }
+    return `Bearer ${key}`;
+  }
+
+  async function health() {
+    try {
+      const res = await withTimeout((signal) => fetch(`${baseUrl}/health`, { signal }));
+      return { ok: res.ok, status: res.status };
+    } catch {
+      return { ok: false };
+    }
+  }
+
+  async function ingest(content, ctx) {
+    const auth = authHeader(); // throws BEFORE any fetch when key is missing
+    let res;
+    try {
+      res = await withTimeout((signal) => fetch(`${baseUrl}/memory/ingest?mode=sync`, {
+        method: 'POST',
+        signal,
+        headers: {
+          'Content-Type': 'application/json',
+          Authorization: auth,
+        },
+        body: JSON.stringify({ content, context: ctx }),
+      }));
+    } catch (err) {
+      throw new SmartmemoryHttpError(`smartmemory: ingest request failed: ${err.message}`, 0);
+    }
+    if (!res.ok) {
+      throw new SmartmemoryHttpError(`smartmemory: ingest failed (HTTP ${res.status})`, res.status);
+    }
+    let raw;
+    try {
+      raw = await res.json();
+    } catch {
+      throw new SmartmemoryHttpError(
+        `smartmemory: ingest returned a 2xx (HTTP ${res.status}) with a non-JSON body`,
+        res.status, 'malformed-response',
+      );
+    }
+    if (typeof raw?.status !== 'string') {
+      throw new SmartmemoryHttpError(
+        `smartmemory: ingest returned a 2xx (HTTP ${res.status}) body missing a "status" field`,
+        res.status, 'malformed-response',
+      );
+    }
+    const unchanged = raw.status === 'unchanged' || raw.unchanged === true;
+    return { status: raw.status, unchanged, raw };
+  }
+
+  async function search(query, opts = {}) {
+    const auth = authHeader();
+    let res;
+    try {
+      res = await withTimeout((signal) => fetch(`${baseUrl}/memory/search`, {
+        method: 'POST',
+        signal,
+        headers: {
+          'Content-Type': 'application/json',
+          Authorization: auth,
+        },
+        body: JSON.stringify({ query, ...opts }),
+      }));
+    } catch (err) {
+      throw new SmartmemoryHttpError(`smartmemory: search request failed: ${err.message}`, 0);
+    }
+    if (!res.ok) {
+      throw new SmartmemoryHttpError(`smartmemory: search failed (HTTP ${res.status})`, res.status);
+    }
+    let raw;
+    try {
+      raw = await res.json();
+    } catch {
+      throw new SmartmemoryHttpError(
+        `smartmemory: search returned a 2xx (HTTP ${res.status}) with a non-JSON body`,
+        res.status, 'malformed-response',
+      );
+    }
+    if (!Array.isArray(raw?.results)) {
+      throw new SmartmemoryHttpError(
+        `smartmemory: search returned a 2xx (HTTP ${res.status}) body missing a "results" array`,
+        res.status, 'malformed-response',
+      );
+    }
+    return raw;
+  }
+
+  return { health, ingest, search };
+}

package/lib/smartmemory-config.js

@@ -0,0 +1,50 @@
+/**
+ * smartmemory-config.js — COMP-SMARTMEMORY-INGEST S01
+ *
+ * Shared reader + provenance helpers for the optional SmartMemory coupling.
+ * Leaf module: no fetch, no compose-state writes → safe to import eagerly at
+ * both hook sites (lib/feature-events.js, server/gate-log-store.js).
+ */
+
+import { readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { deriveId } from './discover-workspaces.js';
+
+/**
+ * Read `.compose/compose.json` → `smartmemory` block. Uncached direct read,
+ * try/catch → {} on missing/malformed. Returns the raw block; consumers gate
+ * on `.enabled === true`. Does NOT resolve the API key.
+ * @param {string} cwd
+ * @returns {{ enabled?: boolean, baseUrl?: string, apiKeyEnv?: string, timeoutMs?: number }}
+ */
+export function getSmartmemoryConfig(cwd) {
+  const cfgPath = join(cwd, '.compose', 'compose.json');
+  try {
+    const cfg = JSON.parse(readFileSync(cfgPath, 'utf-8'));
+    return cfg.smartmemory ?? {};
+  } catch {
+    return {};
+  }
+}
+
+/**
+ * Canonical per-project provenance tag. Wraps deriveId: compose.json
+ * #workspaceId when valid, else basename(cwd). Same value for emitters, sync,
+ * and RECALL → exact string-equality badge comparison end-to-end.
+ * @param {string} cwd
+ * @returns {string}
+ */
+export function resolveProjectTag(cwd) {
+  return deriveId({ root: cwd }).id;
+}
+
+/**
+ * Deterministic provenance path: `compose/<project>/<repoRel>`. Used for
+ * every source_path (events: provenance only; files: dedupe key). Pure/total.
+ * @param {string} projectTag
+ * @param {string} repoRel  repo-relative path (forward slashes)
+ * @returns {string}
+ */
+export function sourcePathFor(projectTag, repoRel) {
+  return `compose/${projectTag}/${repoRel}`;
+}

package/lib/smartmemory-ingest.js

@@ -0,0 +1,115 @@
+/**
+ * smartmemory-ingest.js — COMP-SMARTMEMORY-INGEST S03
+ *
+ * Live emitter + shared pure renderers/context builders. Lazy-imported by the
+ * two hook sites (lib/feature-events.js, server/gate-log-store.js) only when
+ * smartmemory.enabled === true, so the fetch-capable module never loads when
+ * the feature is off.
+ *
+ * Fail-open: emitFeatureEvent/emitGateLogEntry never throw and never affect
+ * the caller — the local durable write has already completed by the time
+ * this runs. A circuit breaker disables further attempts after 3 consecutive
+ * failures so a dead service doesn't spam logs for the rest of the process
+ * lifetime.
+ */
+
+import { getSmartmemoryConfig, resolveProjectTag, sourcePathFor } from './smartmemory-config.js';
+import { createSmartmemoryClient } from './smartmemory-client.js';
+
+// ── module state ──
+let consecutiveFailures = 0;
+let disabled = false;
+const pending = new Set();
+
+/** PURE. One deterministic line per feature-event row. Shared with sync. */
+export function renderFeatureEventContent(row, projectTag) {
+  return `[compose:${projectTag}] ${row.ts} ${row.tool} ${row.code ?? row.feature_code ?? '-'} by ${row.actor ?? 'mcp:agent'}`;
+}
+
+/** PURE. One deterministic line per gate-log entry. Shared with sync. */
+export function renderGateLogContent(entry, projectTag) {
+  return `[compose:${projectTag}] ${entry.timestamp} gate:${entry.decision} ${entry.feature_code ?? '-'} ${entry.id}`;
+}
+
+/** PURE. context for an event item (cli:compose regime — content-hash dedupe). */
+export function buildFeatureEventContext(projectTag, row) {
+  return {
+    origin: 'cli:compose',
+    project: projectTag,
+    source_path: sourcePathFor(projectTag, '.compose/data/feature-events.jsonl'),
+    event: row,
+  };
+}
+
+export function buildGateLogContext(projectTag, entry) {
+  return {
+    origin: 'cli:compose',
+    project: projectTag,
+    source_path: sourcePathFor(projectTag, '.compose/data/gate-log.jsonl'),
+    event: entry,
+  };
+}
+
+function onFailure(err) {
+  consecutiveFailures++;
+  if (consecutiveFailures === 1) {
+    console.warn('[smartmemory] ingest failed (fail-open): ' + err.message);
+  }
+  if (consecutiveFailures >= 3) {
+    disabled = true;
+  }
+}
+
+function emit(cwd, item, render, buildContext) {
+  if (disabled) return;
+  let cfg;
+  try {
+    cfg = getSmartmemoryConfig(cwd);
+  } catch {
+    return;
+  }
+  if (cfg.enabled !== true) return; // no fetch — byte-identity when off
+
+  let p;
+  try {
+    const tag = resolveProjectTag(cwd);
+    const content = render(item, tag);
+    const ctx = buildContext(tag, item);
+    const client = createSmartmemoryClient(cfg);
+    p = client.ingest(content, ctx);
+  } catch (err) {
+    // Synchronous throw (e.g. missing api key) — treat as a failure, never rethrow.
+    onFailure(err);
+    return;
+  }
+
+  pending.add(p);
+  p.then(
+    () => { consecutiveFailures = 0; },
+    (err) => onFailure(err),
+  ).finally(() => pending.delete(p));
+}
+
+/** Fire-and-forget live emit after a durable local append. Never throws. */
+export function emitFeatureEvent(cwd, row) {
+  emit(cwd, row, renderFeatureEventContent, buildFeatureEventContext);
+}
+
+export function emitGateLogEntry(cwd, entry) {
+  emit(cwd, entry, renderGateLogContent, buildGateLogContext);
+}
+
+/** Await in-flight POSTs (bounded by timeoutMs). Drain point for callers that exit. */
+export async function flushPending(timeoutMs = 3000) {
+  if (pending.size === 0) return;
+  const settleAll = Promise.allSettled([...pending]);
+  const timeout = new Promise((resolve) => setTimeout(resolve, timeoutMs));
+  await Promise.race([settleAll, timeout]);
+}
+
+/** Test hook: clear circuit breaker + pending between cases. */
+export function _resetEmitterState() {
+  consecutiveFailures = 0;
+  disabled = false;
+  pending.clear();
+}

package/lib/smartmemory-sync.js

@@ -0,0 +1,275 @@
+/**
+ * smartmemory-sync.js — COMP-SMARTMEMORY-INGEST S05
+ *
+ * Idempotent backfill/re-sync walker: `compose smartmemory sync`. Walks the
+ * four persisted surfaces (feature-events, gate-log, journal entries,
+ * per-feature artifacts), in order, and ingests each as one SmartMemory item.
+ * Re-running is safe (Decision 4 dedupe regimes): events dedupe by content
+ * hash, files dedupe by source_path.
+ */
+
+import { existsSync, readFileSync, readdirSync } from 'node:fs';
+import { join } from 'node:path';
+
+import { getSmartmemoryConfig, resolveProjectTag, sourcePathFor } from './smartmemory-config.js';
+import { createSmartmemoryClient, SmartmemoryHttpError } from './smartmemory-client.js';
+import {
+  renderFeatureEventContent, renderGateLogContent,
+  buildFeatureEventContext, buildGateLogContext,
+} from './smartmemory-ingest.js';
+import { readEvents } from './feature-events.js';
+import { readGateLog } from '../server/gate-log-store.js';
+import { parseJournalEntry } from './journal-writer.js';
+import { listFeatures } from './feature-json.js';
+import { loadFeaturesDir, resolveFeaturesPath, resolveJournalPath, relForDisplay } from './project-paths.js';
+
+const ARTIFACT_FILES = [
+  { file: 'design.md', kind: 'design' },
+  { file: 'blueprint.md', kind: 'blueprint' },
+  { file: 'plan.md', kind: 'plan' },
+  { file: 'report.md', kind: 'report' },
+  { file: 'audit.json', kind: 'audit' },
+];
+
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function gateLogPathFor(cwd) {
+  return process.env.COMPOSE_GATE_LOG || join(cwd, '.compose', 'data', 'gate-log.jsonl');
+}
+
+/**
+ * Surface 1: feature-events.jsonl — malformed lines are already skipped by
+ * readEvents. May throw (naked fs read) — caller wraps per-surface.
+ */
+function collectFeatureEventItems(cwd, feature, tag) {
+  const items = [];
+  const events = readEvents(cwd);
+  for (const row of events) {
+    // OR semantics (blueprint.md ~line 176): a row carrying BOTH fields must
+    // still match on `feature_code` even when `code` differs — nullish
+    // coalescing (`code ?? feature_code`) would wrongly ignore `feature_code`
+    // whenever `code` is present at all.
+    if (feature && row.code !== feature && row.feature_code !== feature) continue;
+    items.push({ content: renderFeatureEventContent(row, tag), context: buildFeatureEventContext(tag, row) });
+  }
+  return items;
+}
+
+/**
+ * Surface 2: gate-log.jsonl. May throw (naked fs read) — caller wraps
+ * per-surface.
+ */
+function collectGateLogItems(cwd, feature, tag) {
+  const items = [];
+  const gateEntries = readGateLog({ featureCode: feature ?? undefined, logPath: gateLogPathFor(cwd) });
+  for (const entry of gateEntries) {
+    items.push({ content: renderGateLogContent(entry, tag), context: buildGateLogContext(tag, entry) });
+  }
+  return items;
+}
+
+/**
+ * Surface 3: journal entries — file regime (import:compose, source_path =
+ * dedupe key). Direct directory walk, NOT getJournalEntries: that reader
+ * hard-caps at MAX_LIMIT=500 (wrong for a backfill walker, which must walk
+ * everything) and silently drops unreadable/unparseable files instead of
+ * accounting for them, which would break this walker's skipped-count
+ * contract.
+ *
+ * A journal entry is defined by filename shape AND parse success (the same
+ * contract the canonical reader enforces) — not just "any *.md file". A
+ * stray non-entry .md file is simply not a candidate: it is never ingested
+ * and never counted as skipped. Per-file read/parse failures increment
+ * `skipped` via the returned counter (not thrown) so one broken entry never
+ * takes down the rest of the surface; a failure to even list the directory
+ * (e.g. it's not a directory at all) DOES throw — the caller wraps that.
+ */
+function collectJournalItems(cwd, feature, tag) {
+  const items = [];
+  let skipped = 0;
+
+  const journalDir = resolveJournalPath(cwd);
+  if (!existsSync(journalDir)) return { items, skipped };
+
+  // Exact filename pattern from lib/journal-writer.js:44 (FILENAME_RE, not
+  // exported — replicated here rather than editing that module).
+  const JOURNAL_FILENAME_RE = /^(\d{4}-\d{2}-\d{2})-session-(\d+)-([a-z0-9][a-z0-9-]*[a-z0-9])\.md$/;
+
+  const candidates = [];
+  for (const name of readdirSync(journalDir)) {
+    const m = name.match(JOURNAL_FILENAME_RE);
+    if (!m) continue; // not a candidate at all — not counted
+    candidates.push({ name, date: m[1], session_number: parseInt(m[2], 10) });
+  }
+
+  // Newest-first, mirroring lib/journal-writer.js:920-923 exactly (date desc,
+  // then session_number desc) — matters because runSync stops on quota 429,
+  // so a partial sync must ingest the newest entries first.
+  candidates.sort((a, b) => {
+    if (b.date !== a.date) return b.date.localeCompare(a.date);
+    return b.session_number - a.session_number;
+  });
+
+  for (const { name } of candidates) {
+    const absPath = join(journalDir, name);
+    let text;
+    try {
+      text = readFileSync(absPath, 'utf-8');
+    } catch {
+      skipped++;
+      continue;
+    }
+
+    // Always parse — a candidate that fails to parse is skipped, not ingested.
+    let parsed;
+    try {
+      parsed = parseJournalEntry(text);
+    } catch {
+      skipped++;
+      continue;
+    }
+    if (feature && parsed.feature_code !== feature) continue;
+
+    items.push({
+      content: text,
+      context: {
+        origin: 'import:compose',
+        project: tag,
+        source_path: sourcePathFor(tag, relForDisplay(cwd, absPath)),
+      },
+    });
+  }
+
+  return { items, skipped };
+}
+
+/**
+ * Surface 4: per-feature artifacts — file regime. Per-file read failures
+ * increment `skipped` (not thrown); a failure to list features or the
+ * features directory itself DOES throw — the caller wraps that.
+ */
+function collectArtifactItems(cwd, feature, tag) {
+  const items = [];
+  let skipped = 0;
+
+  const allFeatures = listFeatures(cwd, loadFeaturesDir(cwd));
+  const targetFeatures = feature ? allFeatures.filter((f) => f.code === feature) : allFeatures;
+  const featuresAbsDir = resolveFeaturesPath(cwd);
+  for (const f of targetFeatures) {
+    const folder = join(featuresAbsDir, f.code);
+    for (const { file, kind } of ARTIFACT_FILES) {
+      const absPath = join(folder, file);
+      if (!existsSync(absPath)) continue;
+      let text;
+      try {
+        text = readFileSync(absPath, 'utf-8');
+      } catch {
+        skipped++;
+        continue;
+      }
+      items.push({
+        content: text,
+        context: {
+          origin: 'import:compose',
+          project: tag,
+          source_path: sourcePathFor(tag, relForDisplay(cwd, absPath)),
+          artifact: { feature_code: f.code, kind },
+        },
+      });
+    }
+  }
+
+  return { items, skipped };
+}
+
+/**
+ * Walk the four surfaces and build the flat, ordered list of {content,
+ * context} items to ingest, honoring the `--feature CODE` filter (Decision 5
+ * semantics): entries with no feature association are excluded when `feature`
+ * is set, included in a full sync.
+ *
+ * Each surface is collected independently, wrapped in its own try/catch: a
+ * surface whose top-level read throws (e.g. a corrupted directory in its
+ * path) is reported (one `skipped` increment + a console.warn naming the
+ * surface and the error) and does NOT abort the other three surfaces.
+ */
+function collectItems(cwd, feature, tag) {
+  const items = [];
+  let skipped = 0;
+
+  const surfaces = [
+    { name: 'feature-events', collect: () => ({ items: collectFeatureEventItems(cwd, feature, tag), skipped: 0 }) },
+    { name: 'gate-log', collect: () => ({ items: collectGateLogItems(cwd, feature, tag), skipped: 0 }) },
+    { name: 'journal', collect: () => collectJournalItems(cwd, feature, tag) },
+    { name: 'artifacts', collect: () => collectArtifactItems(cwd, feature, tag) },
+  ];
+
+  for (const surface of surfaces) {
+    try {
+      const result = surface.collect();
+      items.push(...result.items);
+      skipped += result.skipped;
+    } catch (err) {
+      skipped++;
+      console.warn(`[smartmemory-sync] ${surface.name} surface read failed (skipping this surface): ${err.message}`);
+    }
+  }
+
+  return { items, skipped };
+}
+
+/**
+ * Walk the four persisted surfaces and ingest each item (idempotent).
+ * @param {{ cwd: string, dryRun?: boolean, feature?: string|null }} opts
+ * @returns {Promise<{ ingested:number, unchanged:number, skipped:number, failed:number, stoppedOnQuota:boolean }>}
+ */
+export async function runSync({ cwd, dryRun = false, feature = null }) {
+  const cfg = getSmartmemoryConfig(cwd);
+  if (cfg.enabled !== true && !dryRun) {
+    throw new Error('smartmemory not enabled');
+  }
+
+  const tag = resolveProjectTag(cwd);
+  const { items, skipped } = collectItems(cwd, feature, tag);
+
+  let ingested = 0;
+  let unchanged = 0;
+  let failed = 0;
+  let stoppedOnQuota = false;
+
+  if (dryRun) {
+    return { ingested: items.length, unchanged: 0, skipped, failed: 0, stoppedOnQuota: false };
+  }
+
+  const client = createSmartmemoryClient(cfg);
+
+  for (const item of items) {
+    let result;
+    try {
+      result = await client.ingest(item.content, item.context);
+    } catch (err) {
+      if (err instanceof SmartmemoryHttpError && err.status === 429) {
+        await sleep(1000);
+        try {
+          result = await client.ingest(item.content, item.context);
+        } catch (err2) {
+          if (err2 instanceof SmartmemoryHttpError && err2.status === 429) {
+            stoppedOnQuota = true;
+            break;
+          }
+          failed++;
+          continue;
+        }
+      } else {
+        failed++;
+        continue;
+      }
+    }
+    if (result.unchanged) unchanged++;
+    else ingested++;
+  }
+
+  return { ingested, unchanged, skipped, failed, stoppedOnQuota };
+}

package/lib/stratum-mcp-client.js

@@ -139,6 +139,15 @@ export class StratumMcpClient {
 
     const transportOpts = { command, args, stderr: 'pipe' };
     if (opts.cwd) transportOpts.cwd = opts.cwd;
+    // Inherit the parent environment. Given no `env`, StdioClientTransport
+    // supplies only the MCP SDK's 6-var default allowlist
+    // (HOME/LOGNAME/PATH/SHELL/TERM/USER), which strips the auth context the
+    // spawned stratum-mcp → claude agent needs. Once claude credentials moved
+    // to the macOS keychain, that stripped env stopped authenticating and every
+    // agent step failed with `403 forbidden / "Request not allowed"`. Pass the
+    // full env through; the stratum connector scrubs SENSITIVE_ENV_VARS
+    // (API keys, CLAUDECODE) before it spawns the agent.
+    transportOpts.env = opts.env ?? { ...process.env };
     this.#transport = new StdioClientTransport(transportOpts);
 
     this.#client = new Client(

package/lib/triage.js

@@ -232,7 +232,9 @@ export async function runTriage(featureCode, opts = {}) {
  *
  * Returns true if:
  *   - feature.json has no triageTimestamp
- *   - any file in the feature folder has an mtime newer than triageTimestamp
+ *   - any file in the feature folder (EXCEPT feature.json itself) has an mtime
+ *     newer than triageTimestamp. feature.json is excluded because the stamp it
+ *     is compared against lives inside it, so its own mtime would self-stale.
  *
  * @param {string} cwd - Project root
  * @param {string} featureCode - Feature code
@@ -263,6 +265,10 @@ export function isTriageStale(cwd, featureCode, featuresDir = resolveFeaturesPat
     const entries = readdirSync(featureDir, { withFileTypes: true });
     for (const entry of entries) {
       if (!entry.isFile()) continue;
+      // feature.json holds the triageTimestamp we compare against — counting its
+      // own mtime is circular: writing the stamp would self-invalidate the cache
+      // (COMP-ROADMAP-PLAN C14). Exclude it from the staleness scan.
+      if (entry.name === 'feature.json') continue;
       const filePath = join(featureDir, entry.name);
       try {
         const stat = statSync(filePath);