@geraldmaron/construct 1.0.5 → 1.0.7

package/lib/knowledge/graph.mjs

@@ -0,0 +1,213 @@
+/**
+ * lib/knowledge/graph.mjs — GraphRAG-style entity index + global query.
+ *
+ * Phase C9 foundations:
+ *   - buildGraph(rootDir): derive nodes + edges from `.cx/observations/entities.json`.
+ *   - detectCommunities(graph): pure-JS label propagation. Deterministic
+ *     (sorted neighbor scan, lowest-id tiebreak). O(V+E) per iteration.
+ *   - summarizeCommunity(community, entities): extractive summary over the
+ *     member entities, ranked by degree centrality. Requires no LLM call,
+ *     so the primitive works in solo mode without a provider configured.
+ *   - askGlobal({ query, rootDir }): rank communities by overlap with the
+ *     query terms + their members' search text, return the top communities
+ *     with member entities and the question they best answer.
+ *
+ * Full Leiden (modularity optimization + refinement) would need an external
+ * graph library and is blocked by ADR-0001 (zero-npm-core). Label propagation
+ * is the deterministic alternative. Produces useful coarse communities on
+ * the small graphs Construct sessions accumulate (typically <500 entities).
+ * Swap implementations later by replacing `detectCommunities` only.
+ */
+import { existsSync, readFileSync } from 'node:fs';
+import path from 'node:path';
+import { rankByBm25 } from '../storage/embeddings.mjs';
+
+const OBS_DIR = '.cx/observations';
+const ENTITIES_FILE = 'entities.json';
+
+const MAX_LP_ITERATIONS = 30;
+const MAX_COMMUNITY_TOP_TERMS = 8;
+const MIN_COMMUNITY_SIZE = 2;
+
+function readEntities(rootDir) {
+  const p = path.join(rootDir, OBS_DIR, ENTITIES_FILE);
+  if (!existsSync(p)) return [];
+  try { return JSON.parse(readFileSync(p, 'utf8')); } catch { return []; }
+}
+
+/**
+ * Build an undirected graph from the entity store. Nodes are entity names;
+ * edges come from `relatedEntities[]`. Returns:
+ *   { nodes: Map<name, entity>, adj: Map<name, Set<name>> }
+ */
+export function buildGraph(rootDir) {
+  const entities = readEntities(rootDir);
+  const nodes = new Map();
+  const adj = new Map();
+  for (const e of entities) {
+    if (!e?.name) continue;
+    nodes.set(e.name, e);
+    if (!adj.has(e.name)) adj.set(e.name, new Set());
+  }
+  for (const e of entities) {
+    if (!e?.name) continue;
+    for (const other of e.relatedEntities || []) {
+      if (!nodes.has(other)) continue;
+      adj.get(e.name).add(other);
+      adj.get(other).add(e.name);
+    }
+  }
+  return { nodes, adj };
+}
+
+/**
+ * Run synchronous label propagation. Each node starts with its own label;
+ * iteratively adopt the most frequent label among neighbors, with a sorted
+ * neighbor scan and lowest-id tiebreak so the result is deterministic.
+ *
+ * Returns: Map<name, communityId> plus a stable grouping.
+ */
+export function detectCommunities(graph) {
+  const labels = new Map();
+  const sortedNodes = [...graph.nodes.keys()].sort();
+  for (const n of sortedNodes) labels.set(n, n);
+
+  for (let iter = 0; iter < MAX_LP_ITERATIONS; iter++) {
+    let changed = false;
+    for (const node of sortedNodes) {
+      const neighbors = [...(graph.adj.get(node) || [])].sort();
+      if (neighbors.length === 0) continue;
+
+      const counts = new Map();
+      for (const nb of neighbors) {
+        const lbl = labels.get(nb);
+        counts.set(lbl, (counts.get(lbl) || 0) + 1);
+      }
+
+      // highest-count label, lowest-id tiebreak
+      let bestLabel = labels.get(node);
+      let bestCount = -1;
+      const sortedLabels = [...counts.entries()].sort((a, b) => {
+        if (b[1] !== a[1]) return b[1] - a[1];
+        return a[0] < b[0] ? -1 : 1;
+      });
+      if (sortedLabels.length > 0) {
+        [bestLabel, bestCount] = sortedLabels[0];
+      }
+
+      if (bestCount > 0 && bestLabel !== labels.get(node)) {
+        labels.set(node, bestLabel);
+        changed = true;
+      }
+    }
+    if (!changed) break;
+  }
+
+  const communities = new Map();
+  for (const [node, label] of labels) {
+    if (!communities.has(label)) communities.set(label, []);
+    communities.get(label).push(node);
+  }
+  // Stable order: by size desc, then alphabetical on canonical label
+  const grouped = [...communities.entries()]
+    .map(([id, members]) => ({ id, members: members.sort() }))
+    .sort((a, b) => {
+      if (b.members.length !== a.members.length) return b.members.length - a.members.length;
+      return a.id < b.id ? -1 : 1;
+    });
+  return { labels, communities: grouped };
+}
+
+/**
+ * Extractive community summary. Picks the top-N members by intra-community
+ * degree (the "central" nodes of the cluster) and harvests their summaries
+ * into a short text block. Deterministic and dependency-free.
+ */
+export function summarizeCommunity(group, graph) {
+  if (!group?.members?.length) return { id: group?.id, size: 0, topMembers: [], summary: '' };
+  const memberSet = new Set(group.members);
+  const scored = group.members.map((name) => {
+    const neighbors = graph.adj.get(name) || new Set();
+    let degree = 0;
+    for (const nb of neighbors) if (memberSet.has(nb)) degree++;
+    return { name, degree };
+  }).sort((a, b) => {
+    if (b.degree !== a.degree) return b.degree - a.degree;
+    return a.name < b.name ? -1 : 1;
+  });
+
+  const topMembers = scored.slice(0, MAX_COMMUNITY_TOP_TERMS);
+  const lines = [];
+  for (const { name } of topMembers) {
+    const entity = graph.nodes.get(name);
+    if (!entity) continue;
+    const summary = (entity.summary || '').trim();
+    if (summary) lines.push(`- ${entity.name}: ${summary}`);
+    else lines.push(`- ${entity.name}`);
+  }
+  return {
+    id: group.id,
+    size: group.members.length,
+    topMembers: topMembers.map((m) => m.name),
+    summary: lines.join('\n'),
+  };
+}
+
+/**
+ * Global query over the entity graph. Builds the graph, detects communities,
+ * scores each community by BM25 over its concatenated member summaries, and
+ * returns the top communities with members.
+ *
+ * Intentionally returns structured data instead of a synthesized prose
+ * answer: the LM-based response synthesis is the next layer up and belongs
+ * in the persona prompt, not in this retrieval primitive. Solo mode can
+ * consume the raw result without calling an external model.
+ *
+ * @param {object} args
+ * @param {string} args.query  - natural-language question
+ * @param {string} args.rootDir - project root containing `.cx/observations/`
+ * @param {number} [args.topK=5] - max communities to return
+ * @param {number} [args.minSize=MIN_COMMUNITY_SIZE] - skip singleton communities
+ * @returns {{ query, communities: Array<{ id, size, topMembers, summary, score }>, totalEntities, totalCommunities }}
+ */
+export function askGlobal({ query, rootDir, topK = 5, minSize = MIN_COMMUNITY_SIZE } = {}) {
+  if (!query || typeof query !== 'string') {
+    return { query: query || '', communities: [], totalEntities: 0, totalCommunities: 0 };
+  }
+  const graph = buildGraph(rootDir);
+  if (graph.nodes.size === 0) {
+    return { query, communities: [], totalEntities: 0, totalCommunities: 0 };
+  }
+
+  const { communities } = detectCommunities(graph);
+  const eligible = communities.filter((g) => g.members.length >= minSize);
+  if (eligible.length === 0) {
+    return {
+      query,
+      communities: [],
+      totalEntities: graph.nodes.size,
+      totalCommunities: communities.length,
+    };
+  }
+
+  const summaries = eligible.map((group) => summarizeCommunity(group, graph));
+  const docs = summaries.map((s, i) => ({
+    id: String(i),
+    text: `${s.topMembers.join(' ')}\n${s.summary}`,
+  }));
+  const ranked = rankByBm25(docs, query, { limit: topK });
+
+  const out = [];
+  for (const r of ranked) {
+    const summary = summaries[Number(r.id)];
+    if (!summary) continue;
+    out.push({ ...summary, score: r.score });
+  }
+
+  return {
+    query,
+    communities: out,
+    totalEntities: graph.nodes.size,
+    totalCommunities: communities.length,
+  };
+}

package/lib/knowledge/research-store.mjs

@@ -19,6 +19,7 @@ import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
 import { dirname, join } from 'node:path';
 import { syncFileStateToSql } from '../storage/sync.mjs';
 import { resolveActiveProfile } from '../profiles/loader.mjs';
+import { ensureCxDir } from '../project-init-shared.mjs';
 
 const ROOT = '.cx/knowledge/external/research';
 const MAX_BYTES = 50 * 1024;
@@ -68,6 +69,7 @@ export async function addResearchFinding({
   const expiresAt = new Date(now.getTime() + ttlDays * 24 * 60 * 60 * 1000).toISOString();
 
   const outDir = join(cwd, ROOT);
+  ensureCxDir(cwd);
   mkdirSync(outDir, { recursive: true });
   const outPath = join(outDir, `${slug}.md`);
 

package/lib/maintenance/cleanup.mjs

@@ -0,0 +1,315 @@
+/**
+ * lib/maintenance/cleanup.mjs — Construct's self-maintenance primitives.
+ *
+ * Every artifact Construct writes has a lifecycle. This module collects the
+ * primitives that enforce that lifecycle: rotate oversized logs, prune
+ * rotated segments, age-out cache directories. Each primitive is pure and
+ * returns a structured summary; runFullCleanup orchestrates them with sane
+ * defaults.
+ *
+ * Triggered from two entry points:
+ *   - `construct cleanup`            — manual sweep, optionally --dry-run
+ *   - bin/construct startup check    — automatic when the installed version
+ *                                       differs from ~/.construct/.cleanup-stamp,
+ *                                       so an upgrade reclaims disk without
+ *                                       requiring a manual command.
+ *
+ * Scope is deliberately narrow: only artifacts whose retention is owned by
+ * the runtime (logs, caches, rotated segments). Knowledge stores
+ * (.cx/observations/, .cx/handoffs/, .cx/intake/) require explicit retention
+ * policy and are left for a follow-up after policy is agreed.
+ */
+
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+
+const HARD_BUDGET_MS = 5000;
+
+// Size caps in bytes. Each entry: [filename relative to ~/.cx/, maxBytes].
+// Files that exceed maxBytes are truncated (head-preserving rotation is not
+// useful for append-only event logs — the most-recent events are what matter,
+// not the oldest; tail-preserving truncation keeps the last N% by recopying).
+const JSONL_LOG_CAPS_BYTES = {
+  'audit.jsonl': 25 * 1024 * 1024,
+  'contract-violations.jsonl': 25 * 1024 * 1024,
+  'doctor-log.jsonl': 25 * 1024 * 1024,
+  'role-pending.jsonl': 5 * 1024 * 1024,
+  'role-events.jsonl': 25 * 1024 * 1024,
+};
+
+const EMBED_LOG_DEFAULT_MAX_MB = 50;
+const EMBED_LOG_DEFAULT_KEEP = 3;
+const EMBED_LOG_HARD_CAP_MAX_MB = 500;
+const EMBED_LOG_HARD_CAP_KEEP = 20;
+
+function fileSizeOrZero(p) {
+  try { return fs.statSync(p).size; } catch { return 0; }
+}
+
+function rmSafe(p, dryRun) {
+  if (dryRun) return;
+  try { fs.rmSync(p, { force: true }); } catch (err) {
+    process.stderr.write(`[cleanup] failed to remove ${p}: ${err.message}\n`);
+  }
+}
+
+/**
+ * Tail-preserving truncate: keeps the last `keepBytes` of `file`, drops the
+ * rest. Used for append-only event logs where recent events are valuable but
+ * historical bloat is not.
+ */
+function tailTruncate(file, keepBytes, dryRun) {
+  if (dryRun) return;
+  let stat;
+  try { stat = fs.statSync(file); } catch { return; }
+  if (stat.size <= keepBytes) return;
+  const start = stat.size - keepBytes;
+  const buf = Buffer.alloc(keepBytes);
+  const fd = fs.openSync(file, 'r');
+  try { fs.readSync(fd, buf, 0, keepBytes, start); } finally { fs.closeSync(fd); }
+  // Find the first newline so we don't start mid-record.
+  const nl = buf.indexOf(0x0a);
+  const payload = nl >= 0 && nl < buf.length - 1 ? buf.subarray(nl + 1) : buf;
+  fs.writeFileSync(file, payload);
+}
+
+function rotationConfig(env = process.env) {
+  const maxMbRaw = Number.parseInt(env.CONSTRUCT_EMBED_LOG_MAX_MB ?? '', 10);
+  const keepRaw = Number.parseInt(env.CONSTRUCT_EMBED_LOG_KEEP ?? '', 10);
+  const maxMb = Number.isFinite(maxMbRaw) && maxMbRaw > 0 ? Math.min(maxMbRaw, EMBED_LOG_HARD_CAP_MAX_MB) : EMBED_LOG_DEFAULT_MAX_MB;
+  const keep = Number.isFinite(keepRaw) && keepRaw >= 0 ? Math.min(keepRaw, EMBED_LOG_HARD_CAP_KEEP) : EMBED_LOG_DEFAULT_KEEP;
+  return { maxBytes: maxMb * 1024 * 1024, keep };
+}
+
+/**
+ * Walk the embed log + every rotated segment (.1, .2, ...) and drop the ones
+ * that exceed the per-segment cap OR sit beyond the keep count.
+ *
+ * Unlike rotateEmbedLogIfNeeded in lib/embed/cli.mjs (which is called at
+ * daemon spawn time and only rotates the live log), this is the post-upgrade
+ * cleanup: it handles the case where an existing install has segments that
+ * were oversized before the cap existed.
+ */
+export function cleanupEmbedLog({ homeDir = os.homedir(), env = process.env, dryRun = false } = {}) {
+  const dir = path.join(homeDir, '.cx', 'runtime');
+  const baseLog = path.join(dir, 'embed-daemon.log');
+  const { maxBytes, keep } = rotationConfig(env);
+
+  const summary = { removed: [], truncated: [], freedBytes: 0, baseLog };
+
+  // Every existing segment (.log, .log.1, .log.2, ...)
+  if (!fs.existsSync(dir)) return summary;
+  const entries = fs.readdirSync(dir).filter(n => n === 'embed-daemon.log' || n.startsWith('embed-daemon.log.'));
+
+  for (const name of entries) {
+    const full = path.join(dir, name);
+    const size = fileSizeOrZero(full);
+
+    // Drop segments beyond the keep horizon entirely.
+    const dotMatch = name.match(/^embed-daemon\.log\.(\d+)$/);
+    if (dotMatch && Number.parseInt(dotMatch[1], 10) > keep) {
+      summary.removed.push({ path: full, size });
+      summary.freedBytes += size;
+      rmSafe(full, dryRun);
+      continue;
+    }
+
+    // The live log: if oversized, truncate to 0 (it will rotate cleanly on next
+    // daemon spawn via rotateEmbedLogIfNeeded). Truncating in place keeps any
+    // open daemon FDs valid (writes resume at byte 0 on next O_APPEND write).
+    if (name === 'embed-daemon.log' && size > maxBytes) {
+      summary.truncated.push({ path: full, sizeWas: size });
+      summary.freedBytes += size;
+      if (!dryRun) {
+        try {
+          fs.truncateSync(full, 0);
+        } catch (err) {
+          process.stderr.write(`[cleanup] failed to truncate ${full}: ${err.message}\n`);
+        }
+      }
+      continue;
+    }
+
+    // Rotated segments that are individually oversized: drop them. They are
+    // archived noise; keeping them around defeats the cap.
+    if (dotMatch && size > maxBytes) {
+      summary.removed.push({ path: full, size });
+      summary.freedBytes += size;
+      rmSafe(full, dryRun);
+    }
+  }
+
+  return summary;
+}
+
+/**
+ * Apply per-file size caps to the user-level JSONL event logs at ~/.cx/.
+ * Truncates oldest events rather than removing the whole file so the most
+ * recent records survive.
+ */
+export function cleanupJsonlLogs({ homeDir = os.homedir(), dryRun = false } = {}) {
+  const dir = path.join(homeDir, '.cx');
+  const summary = { truncated: [], freedBytes: 0 };
+  if (!fs.existsSync(dir)) return summary;
+
+  for (const [name, capBytes] of Object.entries(JSONL_LOG_CAPS_BYTES)) {
+    const file = path.join(dir, name);
+    const size = fileSizeOrZero(file);
+    if (size <= capBytes) continue;
+    // Retain the most recent half-cap so headroom exists before next truncate.
+    const keepBytes = Math.floor(capBytes / 2);
+    summary.truncated.push({ path: file, sizeWas: size, sizeNow: keepBytes });
+    summary.freedBytes += size - keepBytes;
+    tailTruncate(file, keepBytes, dryRun);
+  }
+
+  return summary;
+}
+
+/**
+ * Age out ~/.construct/cache/ entries older than `maxAgeDays`. Cache hits are
+ * idempotent and re-fetched on demand, so pruning is safe.
+ */
+export function cleanupCacheDir({ homeDir = os.homedir(), maxAgeDays = 30, dryRun = false } = {}) {
+  const dir = path.join(homeDir, '.construct', 'cache');
+  const summary = { removed: [], freedBytes: 0 };
+  if (!fs.existsSync(dir)) return summary;
+
+  const cutoff = Date.now() - (maxAgeDays * 24 * 60 * 60 * 1000);
+  const walk = (current) => {
+    let entries;
+    try { entries = fs.readdirSync(current, { withFileTypes: true }); } catch { return; }
+    for (const entry of entries) {
+      const full = path.join(current, entry.name);
+      if (entry.isDirectory()) { walk(full); continue; }
+      let stat;
+      try { stat = fs.statSync(full); } catch { continue; }
+      if (stat.mtimeMs < cutoff) {
+        summary.removed.push({ path: full, size: stat.size });
+        summary.freedBytes += stat.size;
+        rmSafe(full, dryRun);
+      }
+    }
+  };
+  walk(dir);
+  return summary;
+}
+
+/**
+ * Orchestrator. Runs each primitive under a hard time budget and returns an
+ * aggregated summary. Individual failures are non-fatal: a primitive that
+ * throws is recorded as an error in the summary but does not block the rest.
+ */
+export function runFullCleanup({ homeDir = os.homedir(), env = process.env, dryRun = false } = {}) {
+  const startedAt = Date.now();
+  const summary = {
+    dryRun,
+    startedAt: new Date(startedAt).toISOString(),
+    durationMs: 0,
+    freedBytes: 0,
+    embedLog: null,
+    jsonlLogs: null,
+    cacheDir: null,
+    errors: [],
+  };
+
+  const safelyRun = (label, fn) => {
+    if (Date.now() - startedAt > HARD_BUDGET_MS) {
+      summary.errors.push({ step: label, error: 'time budget exceeded; skipped' });
+      return null;
+    }
+    try { return fn(); }
+    catch (err) { summary.errors.push({ step: label, error: err.message }); return null; }
+  };
+
+  summary.embedLog = safelyRun('embedLog', () => cleanupEmbedLog({ homeDir, env, dryRun }));
+  summary.jsonlLogs = safelyRun('jsonlLogs', () => cleanupJsonlLogs({ homeDir, dryRun }));
+  summary.cacheDir = safelyRun('cacheDir', () => cleanupCacheDir({ homeDir, dryRun }));
+
+  for (const part of [summary.embedLog, summary.jsonlLogs, summary.cacheDir]) {
+    if (part?.freedBytes) summary.freedBytes += part.freedBytes;
+  }
+  summary.durationMs = Date.now() - startedAt;
+  return summary;
+}
+
+/**
+ * Format bytes for one-line operator output. 1234567 → "1.2 MB".
+ */
+export function formatBytes(bytes) {
+  if (!bytes || bytes < 0) return '0 B';
+  const units = ['B', 'KB', 'MB', 'GB', 'TB'];
+  let n = bytes;
+  let unit = 0;
+  while (n >= 1024 && unit < units.length - 1) { n /= 1024; unit += 1; }
+  return `${n.toFixed(unit === 0 ? 0 : 1)} ${units[unit]}`;
+}
+
+// ---------------------------------------------------------------------------
+// Version stamp — drives automatic cleanup on upgrade.
+//
+// ~/.construct/.cleanup-stamp records the last version that ran cleanup.
+// On every CLI invocation, bin/construct compares the current package
+// version to the stamp. A mismatch triggers a single cleanup pass and
+// writes the new version into the stamp. The first ever invocation also
+// triggers cleanup (no stamp present) — this is what unblocks existing
+// installs that already have 34 GB logs sitting around.
+// ---------------------------------------------------------------------------
+
+const STAMP_RELATIVE = path.join('.construct', '.cleanup-stamp');
+
+export function stampPath(homeDir = os.homedir()) {
+  return path.join(homeDir, STAMP_RELATIVE);
+}
+
+export function readVersionStamp({ homeDir = os.homedir() } = {}) {
+  const p = stampPath(homeDir);
+  try { return JSON.parse(fs.readFileSync(p, 'utf8')); }
+  catch { return null; }
+}
+
+export function writeVersionStamp({ homeDir = os.homedir(), version, summary } = {}) {
+  const p = stampPath(homeDir);
+  fs.mkdirSync(path.dirname(p), { recursive: true });
+  const payload = {
+    version,
+    ranAt: new Date().toISOString(),
+    freedBytes: summary?.freedBytes ?? 0,
+    durationMs: summary?.durationMs ?? 0,
+  };
+  fs.writeFileSync(p, JSON.stringify(payload, null, 2));
+}
+
+/**
+ * Run cleanup if and only if the installed version differs from the stamp.
+ * Returns { ran: bool, reason, summary?, previousVersion? }.
+ *
+ * Designed to be called at CLI startup. Never throws — cleanup failures are
+ * surfaced via the returned summary and never block the actual command.
+ */
+export function maybeRunCleanupOnUpgrade({
+  homeDir = os.homedir(),
+  env = process.env,
+  currentVersion,
+  force = false,
+} = {}) {
+  if (env.CONSTRUCT_DISABLE_AUTO_CLEANUP === '1') {
+    return { ran: false, reason: 'disabled via CONSTRUCT_DISABLE_AUTO_CLEANUP' };
+  }
+  const stamp = readVersionStamp({ homeDir });
+  if (!force && stamp && stamp.version === currentVersion) {
+    return { ran: false, reason: 'version matches stamp' };
+  }
+  let summary;
+  try { summary = runFullCleanup({ homeDir, env, dryRun: false }); }
+  catch (err) { return { ran: false, reason: `cleanup threw: ${err.message}` }; }
+  try { writeVersionStamp({ homeDir, version: currentVersion, summary }); }
+  catch (err) { /* stamp write failure is non-fatal */ void err; }
+  return {
+    ran: true,
+    reason: stamp ? 'version changed' : 'first run',
+    previousVersion: stamp?.version ?? null,
+    summary,
+  };
+}