@smartmemory/compose 0.2.53-beta → 0.2.54-beta

package/lib/smartmemory-ingest.js

@@ -0,0 +1,115 @@
+/**
+ * smartmemory-ingest.js — COMP-SMARTMEMORY-INGEST S03
+ *
+ * Live emitter + shared pure renderers/context builders. Lazy-imported by the
+ * two hook sites (lib/feature-events.js, server/gate-log-store.js) only when
+ * smartmemory.enabled === true, so the fetch-capable module never loads when
+ * the feature is off.
+ *
+ * Fail-open: emitFeatureEvent/emitGateLogEntry never throw and never affect
+ * the caller — the local durable write has already completed by the time
+ * this runs. A circuit breaker disables further attempts after 3 consecutive
+ * failures so a dead service doesn't spam logs for the rest of the process
+ * lifetime.
+ */
+
+import { getSmartmemoryConfig, resolveProjectTag, sourcePathFor } from './smartmemory-config.js';
+import { createSmartmemoryClient } from './smartmemory-client.js';
+
+// ── module state ──
+let consecutiveFailures = 0;
+let disabled = false;
+const pending = new Set();
+
+/** PURE. One deterministic line per feature-event row. Shared with sync. */
+export function renderFeatureEventContent(row, projectTag) {
+  return `[compose:${projectTag}] ${row.ts} ${row.tool} ${row.code ?? row.feature_code ?? '-'} by ${row.actor ?? 'mcp:agent'}`;
+}
+
+/** PURE. One deterministic line per gate-log entry. Shared with sync. */
+export function renderGateLogContent(entry, projectTag) {
+  return `[compose:${projectTag}] ${entry.timestamp} gate:${entry.decision} ${entry.feature_code ?? '-'} ${entry.id}`;
+}
+
+/** PURE. context for an event item (cli:compose regime — content-hash dedupe). */
+export function buildFeatureEventContext(projectTag, row) {
+  return {
+    origin: 'cli:compose',
+    project: projectTag,
+    source_path: sourcePathFor(projectTag, '.compose/data/feature-events.jsonl'),
+    event: row,
+  };
+}
+
+export function buildGateLogContext(projectTag, entry) {
+  return {
+    origin: 'cli:compose',
+    project: projectTag,
+    source_path: sourcePathFor(projectTag, '.compose/data/gate-log.jsonl'),
+    event: entry,
+  };
+}
+
+function onFailure(err) {
+  consecutiveFailures++;
+  if (consecutiveFailures === 1) {
+    console.warn('[smartmemory] ingest failed (fail-open): ' + err.message);
+  }
+  if (consecutiveFailures >= 3) {
+    disabled = true;
+  }
+}
+
+function emit(cwd, item, render, buildContext) {
+  if (disabled) return;
+  let cfg;
+  try {
+    cfg = getSmartmemoryConfig(cwd);
+  } catch {
+    return;
+  }
+  if (cfg.enabled !== true) return; // no fetch — byte-identity when off
+
+  let p;
+  try {
+    const tag = resolveProjectTag(cwd);
+    const content = render(item, tag);
+    const ctx = buildContext(tag, item);
+    const client = createSmartmemoryClient(cfg);
+    p = client.ingest(content, ctx);
+  } catch (err) {
+    // Synchronous throw (e.g. missing api key) — treat as a failure, never rethrow.
+    onFailure(err);
+    return;
+  }
+
+  pending.add(p);
+  p.then(
+    () => { consecutiveFailures = 0; },
+    (err) => onFailure(err),
+  ).finally(() => pending.delete(p));
+}
+
+/** Fire-and-forget live emit after a durable local append. Never throws. */
+export function emitFeatureEvent(cwd, row) {
+  emit(cwd, row, renderFeatureEventContent, buildFeatureEventContext);
+}
+
+export function emitGateLogEntry(cwd, entry) {
+  emit(cwd, entry, renderGateLogContent, buildGateLogContext);
+}
+
+/** Await in-flight POSTs (bounded by timeoutMs). Drain point for callers that exit. */
+export async function flushPending(timeoutMs = 3000) {
+  if (pending.size === 0) return;
+  const settleAll = Promise.allSettled([...pending]);
+  const timeout = new Promise((resolve) => setTimeout(resolve, timeoutMs));
+  await Promise.race([settleAll, timeout]);
+}
+
+/** Test hook: clear circuit breaker + pending between cases. */
+export function _resetEmitterState() {
+  consecutiveFailures = 0;
+  disabled = false;
+  pending.clear();
+}

package/lib/smartmemory-sync.js

@@ -0,0 +1,275 @@
+/**
+ * smartmemory-sync.js — COMP-SMARTMEMORY-INGEST S05
+ *
+ * Idempotent backfill/re-sync walker: `compose smartmemory sync`. Walks the
+ * four persisted surfaces (feature-events, gate-log, journal entries,
+ * per-feature artifacts), in order, and ingests each as one SmartMemory item.
+ * Re-running is safe (Decision 4 dedupe regimes): events dedupe by content
+ * hash, files dedupe by source_path.
+ */
+
+import { existsSync, readFileSync, readdirSync } from 'node:fs';
+import { join } from 'node:path';
+
+import { getSmartmemoryConfig, resolveProjectTag, sourcePathFor } from './smartmemory-config.js';
+import { createSmartmemoryClient, SmartmemoryHttpError } from './smartmemory-client.js';
+import {
+  renderFeatureEventContent, renderGateLogContent,
+  buildFeatureEventContext, buildGateLogContext,
+} from './smartmemory-ingest.js';
+import { readEvents } from './feature-events.js';
+import { readGateLog } from '../server/gate-log-store.js';
+import { parseJournalEntry } from './journal-writer.js';
+import { listFeatures } from './feature-json.js';
+import { loadFeaturesDir, resolveFeaturesPath, resolveJournalPath, relForDisplay } from './project-paths.js';
+
+const ARTIFACT_FILES = [
+  { file: 'design.md', kind: 'design' },
+  { file: 'blueprint.md', kind: 'blueprint' },
+  { file: 'plan.md', kind: 'plan' },
+  { file: 'report.md', kind: 'report' },
+  { file: 'audit.json', kind: 'audit' },
+];
+
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function gateLogPathFor(cwd) {
+  return process.env.COMPOSE_GATE_LOG || join(cwd, '.compose', 'data', 'gate-log.jsonl');
+}
+
+/**
+ * Surface 1: feature-events.jsonl — malformed lines are already skipped by
+ * readEvents. May throw (naked fs read) — caller wraps per-surface.
+ */
+function collectFeatureEventItems(cwd, feature, tag) {
+  const items = [];
+  const events = readEvents(cwd);
+  for (const row of events) {
+    // OR semantics (blueprint.md ~line 176): a row carrying BOTH fields must
+    // still match on `feature_code` even when `code` differs — nullish
+    // coalescing (`code ?? feature_code`) would wrongly ignore `feature_code`
+    // whenever `code` is present at all.
+    if (feature && row.code !== feature && row.feature_code !== feature) continue;
+    items.push({ content: renderFeatureEventContent(row, tag), context: buildFeatureEventContext(tag, row) });
+  }
+  return items;
+}
+
+/**
+ * Surface 2: gate-log.jsonl. May throw (naked fs read) — caller wraps
+ * per-surface.
+ */
+function collectGateLogItems(cwd, feature, tag) {
+  const items = [];
+  const gateEntries = readGateLog({ featureCode: feature ?? undefined, logPath: gateLogPathFor(cwd) });
+  for (const entry of gateEntries) {
+    items.push({ content: renderGateLogContent(entry, tag), context: buildGateLogContext(tag, entry) });
+  }
+  return items;
+}
+
+/**
+ * Surface 3: journal entries — file regime (import:compose, source_path =
+ * dedupe key). Direct directory walk, NOT getJournalEntries: that reader
+ * hard-caps at MAX_LIMIT=500 (wrong for a backfill walker, which must walk
+ * everything) and silently drops unreadable/unparseable files instead of
+ * accounting for them, which would break this walker's skipped-count
+ * contract.
+ *
+ * A journal entry is defined by filename shape AND parse success (the same
+ * contract the canonical reader enforces) — not just "any *.md file". A
+ * stray non-entry .md file is simply not a candidate: it is never ingested
+ * and never counted as skipped. Per-file read/parse failures increment
+ * `skipped` via the returned counter (not thrown) so one broken entry never
+ * takes down the rest of the surface; a failure to even list the directory
+ * (e.g. it's not a directory at all) DOES throw — the caller wraps that.
+ */
+function collectJournalItems(cwd, feature, tag) {
+  const items = [];
+  let skipped = 0;
+
+  const journalDir = resolveJournalPath(cwd);
+  if (!existsSync(journalDir)) return { items, skipped };
+
+  // Exact filename pattern from lib/journal-writer.js:44 (FILENAME_RE, not
+  // exported — replicated here rather than editing that module).
+  const JOURNAL_FILENAME_RE = /^(\d{4}-\d{2}-\d{2})-session-(\d+)-([a-z0-9][a-z0-9-]*[a-z0-9])\.md$/;
+
+  const candidates = [];
+  for (const name of readdirSync(journalDir)) {
+    const m = name.match(JOURNAL_FILENAME_RE);
+    if (!m) continue; // not a candidate at all — not counted
+    candidates.push({ name, date: m[1], session_number: parseInt(m[2], 10) });
+  }
+
+  // Newest-first, mirroring lib/journal-writer.js:920-923 exactly (date desc,
+  // then session_number desc) — matters because runSync stops on quota 429,
+  // so a partial sync must ingest the newest entries first.
+  candidates.sort((a, b) => {
+    if (b.date !== a.date) return b.date.localeCompare(a.date);
+    return b.session_number - a.session_number;
+  });
+
+  for (const { name } of candidates) {
+    const absPath = join(journalDir, name);
+    let text;
+    try {
+      text = readFileSync(absPath, 'utf-8');
+    } catch {
+      skipped++;
+      continue;
+    }
+
+    // Always parse — a candidate that fails to parse is skipped, not ingested.
+    let parsed;
+    try {
+      parsed = parseJournalEntry(text);
+    } catch {
+      skipped++;
+      continue;
+    }
+    if (feature && parsed.feature_code !== feature) continue;
+
+    items.push({
+      content: text,
+      context: {
+        origin: 'import:compose',
+        project: tag,
+        source_path: sourcePathFor(tag, relForDisplay(cwd, absPath)),
+      },
+    });
+  }
+
+  return { items, skipped };
+}
+
+/**
+ * Surface 4: per-feature artifacts — file regime. Per-file read failures
+ * increment `skipped` (not thrown); a failure to list features or the
+ * features directory itself DOES throw — the caller wraps that.
+ */
+function collectArtifactItems(cwd, feature, tag) {
+  const items = [];
+  let skipped = 0;
+
+  const allFeatures = listFeatures(cwd, loadFeaturesDir(cwd));
+  const targetFeatures = feature ? allFeatures.filter((f) => f.code === feature) : allFeatures;
+  const featuresAbsDir = resolveFeaturesPath(cwd);
+  for (const f of targetFeatures) {
+    const folder = join(featuresAbsDir, f.code);
+    for (const { file, kind } of ARTIFACT_FILES) {
+      const absPath = join(folder, file);
+      if (!existsSync(absPath)) continue;
+      let text;
+      try {
+        text = readFileSync(absPath, 'utf-8');
+      } catch {
+        skipped++;
+        continue;
+      }
+      items.push({
+        content: text,
+        context: {
+          origin: 'import:compose',
+          project: tag,
+          source_path: sourcePathFor(tag, relForDisplay(cwd, absPath)),
+          artifact: { feature_code: f.code, kind },
+        },
+      });
+    }
+  }
+
+  return { items, skipped };
+}
+
+/**
+ * Walk the four surfaces and build the flat, ordered list of {content,
+ * context} items to ingest, honoring the `--feature CODE` filter (Decision 5
+ * semantics): entries with no feature association are excluded when `feature`
+ * is set, included in a full sync.
+ *
+ * Each surface is collected independently, wrapped in its own try/catch: a
+ * surface whose top-level read throws (e.g. a corrupted directory in its
+ * path) is reported (one `skipped` increment + a console.warn naming the
+ * surface and the error) and does NOT abort the other three surfaces.
+ */
+function collectItems(cwd, feature, tag) {
+  const items = [];
+  let skipped = 0;
+
+  const surfaces = [
+    { name: 'feature-events', collect: () => ({ items: collectFeatureEventItems(cwd, feature, tag), skipped: 0 }) },
+    { name: 'gate-log', collect: () => ({ items: collectGateLogItems(cwd, feature, tag), skipped: 0 }) },
+    { name: 'journal', collect: () => collectJournalItems(cwd, feature, tag) },
+    { name: 'artifacts', collect: () => collectArtifactItems(cwd, feature, tag) },
+  ];
+
+  for (const surface of surfaces) {
+    try {
+      const result = surface.collect();
+      items.push(...result.items);
+      skipped += result.skipped;
+    } catch (err) {
+      skipped++;
+      console.warn(`[smartmemory-sync] ${surface.name} surface read failed (skipping this surface): ${err.message}`);
+    }
+  }
+
+  return { items, skipped };
+}
+
+/**
+ * Walk the four persisted surfaces and ingest each item (idempotent).
+ * @param {{ cwd: string, dryRun?: boolean, feature?: string|null }} opts
+ * @returns {Promise<{ ingested:number, unchanged:number, skipped:number, failed:number, stoppedOnQuota:boolean }>}
+ */
+export async function runSync({ cwd, dryRun = false, feature = null }) {
+  const cfg = getSmartmemoryConfig(cwd);
+  if (cfg.enabled !== true && !dryRun) {
+    throw new Error('smartmemory not enabled');
+  }
+
+  const tag = resolveProjectTag(cwd);
+  const { items, skipped } = collectItems(cwd, feature, tag);
+
+  let ingested = 0;
+  let unchanged = 0;
+  let failed = 0;
+  let stoppedOnQuota = false;
+
+  if (dryRun) {
+    return { ingested: items.length, unchanged: 0, skipped, failed: 0, stoppedOnQuota: false };
+  }
+
+  const client = createSmartmemoryClient(cfg);
+
+  for (const item of items) {
+    let result;
+    try {
+      result = await client.ingest(item.content, item.context);
+    } catch (err) {
+      if (err instanceof SmartmemoryHttpError && err.status === 429) {
+        await sleep(1000);
+        try {
+          result = await client.ingest(item.content, item.context);
+        } catch (err2) {
+          if (err2 instanceof SmartmemoryHttpError && err2.status === 429) {
+            stoppedOnQuota = true;
+            break;
+          }
+          failed++;
+          continue;
+        }
+      } else {
+        failed++;
+        continue;
+      }
+    }
+    if (result.unchanged) unchanged++;
+    else ingested++;
+  }
+
+  return { ingested, unchanged, skipped, failed, stoppedOnQuota };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@smartmemory/compose",
-  "version": "0.2.53-beta",
+  "version": "0.2.54-beta",
   "description": "Structured AI dev pipeline — goal-to-product orchestration with gates, iteration loops, and feature lifecycle management.",
   "author": "SmartMemory",
   "license": "MIT",

package/server/gate-log-store.js

@@ -21,7 +21,8 @@
 
 import { appendFileSync, readFileSync, existsSync, mkdirSync } from 'node:fs';
 import { join } from 'node:path';
-import { getDataDir } from './project-root.js';
+import { getDataDir, getTargetRoot } from './project-root.js';
+import { getSmartmemoryConfig } from '../lib/smartmemory-config.js';
 
 // Gate log is project-scoped (mirrors sessions.json, active-build.json etc).
 // COMPOSE_GATE_LOG env var overrides the path — read dynamically so tests can inject it.
@@ -65,6 +66,15 @@ export function appendGateLogEntry(entry) {
   }
 
   appendFileSync(filePath, JSON.stringify(entry) + '\n', 'utf8');
+  // COMP-SMARTMEMORY-INGEST: fail-open live emit after the durable append.
+  try {
+    const cwd = getTargetRoot();
+    if (getSmartmemoryConfig(cwd).enabled === true) {
+      import('../lib/smartmemory-ingest.js')
+        .then((m) => m.emitGateLogEntry(cwd, entry))
+        .catch(() => {});
+    }
+  } catch { /* fail-open */ }
 }
 
 /**

package/server/smartmemory-routes.js

@@ -0,0 +1,113 @@
+/**
+ * smartmemory-routes.js — COMP-SMARTMEMORY-RECALL S01.
+ *
+ * Route:
+ *   GET /api/smartmemory/recall?featureCode=<CODE> — ranked prior-context
+ *     lookup for the cockpit's Recall tab, backed by INGEST's SmartMemory
+ *     client + config (lib/smartmemory-client.js, lib/smartmemory-config.js).
+ *
+ * Degrade-never-fail (the qa-scope-routes.js model): every branch returns a
+ * shaped 200, never a throw. Flag OFF (or no workspace root) answers from
+ * config alone — zero client construction, zero SmartMemory-bound traffic.
+ *
+ * Not on the auth allowlist (server/index.js) — auth-gated in remote mode
+ * like the vision routes, since results can carry local project context.
+ */
+import { getSmartmemoryConfig, resolveProjectTag as defaultResolveProjectTag } from '../lib/smartmemory-config.js';
+import { createSmartmemoryClient } from '../lib/smartmemory-client.js';
+import { readFeature as defaultReadFeature } from '../lib/feature-json.js';
+import { resolveFeaturesPath as defaultResolveFeaturesPath } from '../lib/project-paths.js';
+
+function defaultCreateClient(cfg) {
+  return createSmartmemoryClient(cfg);
+}
+
+function shortReason(e) {
+  return e?.message?.slice(0, 200) || 'unreachable';
+}
+
+function safeProjectTag(resolveProjectTag, root) {
+  try {
+    return resolveProjectTag(root);
+  } catch {
+    return null;
+  }
+}
+
+/** Defensive adapter over the SmartMemory hit shape (multiple field-name fallbacks). */
+function mapHit(hit) {
+  const content = hit?.content ?? hit?.text ?? '';
+  return {
+    id: hit?.id ?? hit?.item_id ?? null,
+    snippet: content.length > 280 ? content.slice(0, 280) + '…' : content,
+    score: hit?.score ?? null,
+    memoryType: hit?.memory_type ?? hit?.memoryType ?? null,
+    ts: hit?.context?.event?.ts ?? hit?.context?.ts ?? hit?.ts ?? null,
+    project: hit?.context?.project ?? null,
+  };
+}
+
+/**
+ * @param {import('express').Express} app
+ * @param {object} [deps]
+ * @param {Function} [deps.getConfig]           — (cwd) => {} | {enabled,baseUrl,apiKeyEnv,timeoutMs}
+ * @param {Function} [deps.createClient]         — (cfg) => { search(query, opts) }
+ * @param {Function} [deps.readFeature]          — (cwd, code, featuresDir) => FeatureJson|null
+ * @param {Function} [deps.resolveFeaturesPath]  — (cwd) => absolute features dir
+ * @param {Function} [deps.resolveProjectTag]    — (cwd) => canonical project tag
+ */
+export function attachSmartmemoryRoutes(app, {
+  getConfig = getSmartmemoryConfig,
+  createClient = defaultCreateClient,
+  readFeature = defaultReadFeature,
+  resolveFeaturesPath = defaultResolveFeaturesPath,
+  resolveProjectTag = defaultResolveProjectTag,
+} = {}) {
+  app.get('/api/smartmemory/recall', async (req, res) => {
+    const root = req.workspace?.root;
+    if (!root) {
+      return res.json({ enabled: false });
+    }
+
+    let cfg;
+    try {
+      cfg = getConfig(root);
+    } catch {
+      cfg = {};
+    }
+    if (cfg?.enabled !== true) {
+      return res.json({ enabled: false });
+    }
+
+    const featureCode = (req.query.featureCode || '').toString().trim();
+    if (!featureCode) {
+      return res.json({ enabled: true, available: true, results: [], invalidFeatureCode: true });
+    }
+
+    let feature;
+    try {
+      feature = readFeature(root, featureCode, resolveFeaturesPath(root));
+    } catch {
+      feature = null;
+    }
+    const desc = feature?.description || '';
+    const query = desc ? `${featureCode} ${desc}` : featureCode;
+
+    let hits;
+    try {
+      const client = createClient(cfg);
+      hits = await client.search(query, { top_k: 10 });
+    } catch (e) {
+      return res.json({ enabled: true, available: false, error: shortReason(e) });
+    }
+
+    const results = (Array.isArray(hits) ? hits : (hits?.results ?? [])).map(mapHit);
+    return res.json({
+      enabled: true,
+      available: true,
+      featureCode,
+      project: safeProjectTag(resolveProjectTag, root),
+      results,
+    });
+  });
+}

package/server/vision-server.js

@@ -30,6 +30,7 @@ import { attachSettingsRoutes } from './settings-routes.js';
 import { attachHealthRoutes } from './health-routes.js';
 import { attachValidateRoutes } from './validate-routes.js';
 import { attachQaScopeRoutes } from './qa-scope-routes.js';
+import { attachSmartmemoryRoutes } from './smartmemory-routes.js';
 import { attachDesignRoutes } from './design-routes.js';
 import { DesignSessionManager } from './design-session.js';
 import { attachPipelineRoutes } from './pipeline-routes.js';
@@ -102,6 +103,11 @@ export class VisionServer {
     // Read-only GET /api/qa-scope?featureCode=…; reads off req.workspace.
     attachQaScopeRoutes(app);
 
+    // ── SmartMemory recall (COMP-SMARTMEMORY-RECALL, opt-in) ───────────────
+    // Read-only GET /api/smartmemory/recall?featureCode=…; degrade-never-fail,
+    // answers {enabled:false} from config alone when the flag is OFF.
+    attachSmartmemoryRoutes(app);
+
     // ── Vision CRUD + lifecycle routes ─────────────────────────────────────
     attachVisionRoutes(app, {
       store: this.store,

package/dist/assets/channel-DQJPWkb_.js

@@ -1 +0,0 @@
-import{ag as o,ah as n}from"./App-Bu9KMtTa.js";const t=(a,r)=>o.lang.round(n.parse(a)[r]);export{t as c};

package/dist/assets/classDiagram-OUVF2IWQ-CTLbAiUK.js

@@ -1 +0,0 @@
-import{s as a,c as s,a as e,C as t}from"./chunk-V7JOEXUC-Cy4Ixww6.js";import{_ as i}from"./App-Bu9KMtTa.js";import"./chunk-5VM5RSS4-D-f_Jn3G.js";import"./chunk-XXDRQBXY-uE-zN_vc.js";import"./chunk-VR4S4FIN-CFexfU_a.js";import"./chunk-32BRIVSS-CarSmVlX.js";import"./mobile-CNLMhdFP.js";import"./index-B7-HQenC.js";import"./graph-xkel59g2.js";var b={parser:e,get db(){return new t},renderer:s,styles:a,init:i(r=>{r.class||(r.class={}),r.class.arrowMarkerAbsolute=r.arrowMarkerAbsolute},"init")};export{b as diagram};

package/dist/assets/classDiagram-v2-EOCWNBFH-CTLbAiUK.js

@@ -1 +0,0 @@
-import{s as a,c as s,a as e,C as t}from"./chunk-V7JOEXUC-Cy4Ixww6.js";import{_ as i}from"./App-Bu9KMtTa.js";import"./chunk-5VM5RSS4-D-f_Jn3G.js";import"./chunk-XXDRQBXY-uE-zN_vc.js";import"./chunk-VR4S4FIN-CFexfU_a.js";import"./chunk-32BRIVSS-CarSmVlX.js";import"./mobile-CNLMhdFP.js";import"./index-B7-HQenC.js";import"./graph-xkel59g2.js";var b={parser:e,get db(){return new t},renderer:s,styles:a,init:i(r=>{r.class||(r.class={}),r.class.arrowMarkerAbsolute=r.arrowMarkerAbsolute},"init")};export{b as diagram};