@geraldmaron/construct 1.0.0 → 1.0.2

package/lib/embed/inbox.mjs

@@ -0,0 +1,343 @@
+/**
+ * lib/embed/inbox.mjs — filesystem inbox watcher for the embed daemon.
+ *
+ * Watches one or more local directories for new ingestable files and
+ * processes them into the observation store. Agnostic to content type —
+ * handles specs, ADRs, meeting notes, internal docs, research, PDFs,
+ * Office files, plain text, code, or any format supported by document-extract.
+ *
+ * Watch dirs (in priority order):
+ *   1. CX_INBOX_DIRS env var — colon-separated absolute paths
+ *   2. <rootDir>/.cx/inbox/  — per-project drop folder (always included)
+ *   3. <rootDir>/docs/intake/ — project intake drop zone when docs init is used
+ *
+ * State tracking:
+ *   <rootDir>/.cx/runtime/inbox-state.json — maps filePath → processedAt
+ *   Prevents re-processing the same file across daemon restarts.
+ *
+ * Usage:
+ *   const watcher = new InboxWatcher({ rootDir, env });
+ *   const result  = await watcher.poll();
+ *   // result: { processed: [...], skipped: number, errors: [...] }
+ */
+
+import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
+import { join, resolve, isAbsolute } from 'node:path';
+import { isExtractableDocumentPath } from '../document-extract.mjs';
+import { ingestDocuments } from '../document-ingest.mjs';
+import { addObservation } from '../observation-store.mjs';
+import { inferKnowledgeTarget, knowledgeDirForCategory } from '../knowledge/layout.mjs';
+import { docLaneDir, suggestDocsLaneForFile } from '../docs-routing.mjs';
+import { shouldCreateCx } from '../project-detection.mjs';
+import { prepareIntakeForIngestedFile } from '../intake/prepare.mjs';
+import { loadIntakeConfig, INTAKE_DEFAULT_MAX_DEPTH } from '../intake/intake-config.mjs';
+
+const STATE_FILE = '.cx/runtime/inbox-state.json';
+const DEFAULT_INBOX_SUBDIR = '.cx/inbox';
+const DOCS_INTAKE_SUBDIR = 'docs/intake';
+const MAX_FILE_SIZE_BYTES = 50 * 1024 * 1024; // 50 MB hard cap
+
+function nextAvailablePath(targetPath) {
+  if (!existsSync(targetPath)) return targetPath;
+  const dot = targetPath.lastIndexOf('.');
+  const base = dot === -1 ? targetPath : targetPath.slice(0, dot);
+  const ext = dot === -1 ? '' : targetPath.slice(dot);
+  let index = 2;
+  while (true) {
+    const candidate = `${base}-${index}${ext}`;
+    if (!existsSync(candidate)) return candidate;
+    index += 1;
+  }
+}
+
+// ─── State helpers ────────────────────────────────────────────────────────────
+
+function statePath(rootDir) {
+  return join(rootDir, STATE_FILE);
+}
+
+function readState(rootDir) {
+  const p = statePath(rootDir);
+  if (!existsSync(p)) return {};
+  try { return JSON.parse(readFileSync(p, 'utf8')); } catch (err) { process.stderr.write('[inbox.mjs] readState: ' + (err?.message ?? String(err)) + '\n'); return {}; }
+}
+
+function writeState(rootDir, state) {
+  const p = statePath(rootDir);
+  
+  // Only create .cx directories if this is an initialized project
+  if (shouldCreateCx(rootDir)) {
+    mkdirSync(join(rootDir, '.cx', 'runtime'), { recursive: true });
+    writeFileSync(p, JSON.stringify(state, null, 2) + '\n');
+  }
+  // If not an initialized project, don't create .cx - state will be lost
+  // but that's okay since we shouldn't be processing files in uninitialized dirs
+}
+
+// ─── Directory resolution ─────────────────────────────────────────────────────
+
+/**
+ * Resolve inbox watch directories from the persisted intake-config.
+ * Always includes <rootDir>/.cx/inbox/ (created on first use) when the
+ * config has includeProjectInbox: true (default).
+ * Includes <rootDir>/docs/intake/ when it exists and includeDocsIntake is true.
+ * `parentDirs` from intake-config.json plus CX_INBOX_DIRS env var add extras.
+ */
+export function resolveInboxDirs(rootDir, env = process.env) {
+  const config = loadIntakeConfig(rootDir, env);
+  const dirs = [];
+
+  if (config.includeProjectInbox) {
+    const projectInbox = join(rootDir, DEFAULT_INBOX_SUBDIR);
+    mkdirSync(projectInbox, { recursive: true });
+    dirs.push(projectInbox);
+  }
+
+  if (config.includeDocsIntake) {
+    const docsIntake = join(rootDir, DOCS_INTAKE_SUBDIR);
+    if (existsSync(docsIntake)) dirs.push(docsIntake);
+  }
+
+  for (const candidate of config.parentDirs) {
+    if (!candidate) continue;
+    if (!existsSync(candidate)) continue;
+    if (dirs.includes(candidate)) continue;
+    dirs.push(candidate);
+  }
+
+  return dirs;
+}
+
+// ─── File scanning ────────────────────────────────────────────────────────────
+
+/**
+ * Walk `dir` up to `maxDepth` subdirectory levels.
+ * maxDepth=0 → only files in `dir` itself (no subdirs).
+ * Directories are tracked with their depth so the walk stops at the limit.
+ */
+function scanDir(dir, maxDepth = INTAKE_DEFAULT_MAX_DEPTH) {
+  const results = [];
+  const stack = [{ path: dir, depth: 0 }];
+  try {
+    while (stack.length > 0) {
+      const { path: current, depth } = stack.pop();
+      const entries = readdirSync(current, { withFileTypes: true });
+      for (const entry of entries) {
+        if (entry.name.startsWith('.')) continue;
+        const full = join(current, entry.name);
+        if (entry.isDirectory()) {
+          if (depth < maxDepth) stack.push({ path: full, depth: depth + 1 });
+          continue;
+        }
+        if (!entry.isFile()) continue;
+        if (!isExtractableDocumentPath(full)) continue;
+        try {
+          const st = statSync(full);
+          if (st.size === 0 || st.size > MAX_FILE_SIZE_BYTES) continue;
+          results.push({ path: full, size: st.size, mtimeMs: st.mtimeMs });
+        } catch (err) { process.stderr.write('[inbox.mjs] scan-file-stat: ' + (err?.message ?? String(err)) + '\n'); }
+      }
+    }
+  } catch (err) { process.stderr.write('[inbox.mjs] scan-dir: ' + (err?.message ?? String(err)) + '\n'); }
+  return results;
+}
+
+// ─── Observation extraction ───────────────────────────────────────────────────
+
+/**
+ * Derive observation category and knowledge subdir from a file path.
+ * Uses inferKnowledgeTarget for routing, then maps subdir → obs category.
+ */
+function inferFileClassification(filePath) {
+  const knowledgeSubdir = inferKnowledgeTarget(filePath);
+  // Map subdir to observation category
+  let category;
+  switch (knowledgeSubdir) {
+    case 'decisions': category = 'decision';     break;
+    case 'how-tos':   category = 'pattern';      break;
+    case 'reference': category = 'insight';      break;
+    case 'external':  category = 'insight';      break;
+    default:          category = 'insight';
+  }
+  // Postmortem / incident files deserve anti-pattern category regardless of subdir
+  const name = filePath.toLowerCase();
+  if (/\bpost.?mortem\b|\bincident\b|\brca\b/.test(name)) category = 'anti-pattern';
+  return { category, knowledgeSubdir };
+}
+
+/**
+ * Write an observation summarising the ingested document.
+ * Summary = first non-empty line of extracted text (≤120 chars).
+ * Content = first 1500 chars of extracted text for RAG searchability.
+ */
+function recordInboxObservation(rootDir, { sourcePath, outputPath, characters, extractedText, category, knowledgeSubdir }) {
+  const lines = (extractedText ?? '').split('\n').map((l) => l.trim()).filter(Boolean);
+  const firstLine = (lines[0] ?? '').slice(0, 120);
+  const filename = sourcePath.split('/').pop();
+  const summary = firstLine
+    ? `[inbox] ${filename}: ${firstLine}`
+    : `[inbox] Ingested document: ${filename}`;
+  const content = (extractedText ?? '').slice(0, 1500);
+  const tags = ['inbox', 'ingested-doc', category, `knowledge:${knowledgeSubdir}`];
+
+  addObservation(rootDir, {
+    role: 'construct',
+    category,
+    summary,
+    content: `source: ${sourcePath}\noutput: ${outputPath}\nknowledge-subdir: ${knowledgeSubdir}\ncharacters: ${characters}\n\n${content}`,
+    tags,
+    confidence: 0.75,
+    source: 'inbox-watcher',
+  });
+}
+
+function maybePromoteToDocs(rootDir, { sourcePath, extractedText }) {
+  const suggestedLane = suggestDocsLaneForFile(sourcePath, extractedText);
+  if (!suggestedLane || suggestedLane === 'intake') return null;
+
+  const laneDir = join(rootDir, 'docs', docLaneDir(suggestedLane));
+  if (!existsSync(laneDir)) return null;
+
+  const sourceName = sourcePath.split('/').pop() ?? 'intake-doc';
+  const targetPath = nextAvailablePath(join(laneDir, `${sourceName}.md`));
+  const title = sourceName.replace(/\.[^.]+$/, '').replace(/[-_]+/g, ' ').trim() || 'Promoted intake document';
+  const markdown = [
+    '# ' + title,
+    '',
+    '> Promoted from intake for review and incorporation into this docs lane.',
+    '',
+    `- Source: \`${sourcePath}\``,
+    `- Suggested lane: \`${suggestedLane}\``,
+    '',
+    '## Extracted Content',
+    '',
+    extractedText || '',
+    '',
+  ].join('\n');
+
+  writeFileSync(targetPath, markdown);
+  return targetPath;
+}
+
+// ─── InboxWatcher class ───────────────────────────────────────────────────────
+
+export class InboxWatcher {
+  #rootDir;
+  #env;
+  #cwd;
+
+  /**
+   * @param {object} opts
+   * @param {string} opts.rootDir - Root dir for observation store + state file
+   * @param {object} [opts.env]   - Env override (default: process.env)
+   * @param {string} [opts.cwd]   - Working dir for ingest output (default: rootDir)
+   */
+  constructor({ rootDir, env = process.env, cwd } = {}) {
+    this.#rootDir = rootDir;
+    this.#env = env;
+    this.#cwd = cwd ?? rootDir;
+  }
+
+  /**
+   * Scan all inbox dirs, ingest new files, record observations.
+   * Returns { processed: [...], skipped: number, errors: [...] }
+   */
+  async poll() {
+    const config = loadIntakeConfig(this.#rootDir, this.#env);
+    const dirs = resolveInboxDirs(this.#rootDir, this.#env);
+    const state = readState(this.#rootDir);
+    const processed = [];
+    const errors = [];
+    let skipped = 0;
+
+    for (const dir of dirs) {
+      const candidates = scanDir(dir, config.maxDepth);
+      for (const candidate of candidates) {
+        const key = candidate.path;
+
+        // Skip if already processed (state key = path + mtime for change detection)
+        const stateKey = `${key}:${candidate.mtimeMs}`;
+        if (state[key] && state[key].mtimeMs === candidate.mtimeMs) {
+          skipped += 1;
+          continue;
+        }
+
+        try {
+          const { category, knowledgeSubdir } = inferFileClassification(candidate.path);
+          const result = await ingestDocuments([candidate.path], {
+            cwd: this.#cwd,
+            target: `knowledge/${knowledgeSubdir}`,
+            sync: false,
+            env: this.#env,
+          });
+
+          const fileResult = result.files?.[0];
+          if (fileResult) {
+            // Best-effort: read back the extracted text from the written markdown
+            let extractedText = '';
+            try {
+              const md = readFileSync(fileResult.outputPath, 'utf8');
+              const contentIdx = md.indexOf('## Extracted Content');
+              if (contentIdx !== -1) extractedText = md.slice(contentIdx + 20).trim();
+            } catch (err) { process.stderr.write('[inbox.mjs] read-extracted-text: ' + (err?.message ?? String(err)) + '\n'); }
+
+            recordInboxObservation(this.#rootDir, {
+              sourcePath: candidate.path,
+              outputPath: fileResult.outputPath,
+              characters: fileResult.characters,
+              extractedText,
+              category,
+              knowledgeSubdir,
+            });
+
+            const docsPath = maybePromoteToDocs(this.#rootDir, {
+              sourcePath: candidate.path,
+              extractedText,
+            });
+
+            // Queue an intake packet so the agent can triage the new
+            // signal against the existing corpus. Failure here is
+            // non-fatal — ingest already succeeded; the queue is a
+            // best-effort handoff, not the ingestion's source of truth.
+            let intakeId = null;
+            try {
+              const intakeResult = await prepareIntakeForIngestedFile({
+                rootDir: this.#rootDir,
+                ingestedFile: {
+                  sourcePath: candidate.path,
+                  outputPath: fileResult.outputPath,
+                  characters: fileResult.characters,
+                  knowledgeSubdir,
+                },
+                env: this.#env,
+              });
+              intakeId = intakeResult?.id || null;
+            } catch (err) {
+              if (this.#env.CONSTRUCT_DEBUG_INTAKE === '1') {
+                process.stderr.write(`[inbox] intake queue write failed for ${candidate.path}: ${err.message}\n`);
+              }
+            }
+
+            state[key] = { mtimeMs: candidate.mtimeMs, processedAt: new Date().toISOString(), outputPath: fileResult.outputPath };
+            processed.push({ path: candidate.path, outputPath: fileResult.outputPath, docsPath, characters: fileResult.characters, knowledgeSubdir, intakeId });
+          }
+        } catch (err) {
+          errors.push({ path: candidate.path, error: err.message });
+        }
+      }
+    }
+
+    if (processed.length || errors.length) {
+      writeState(this.#rootDir, state);
+    }
+
+    return { processed, skipped, errors, dirs };
+  }
+
+  /**
+   * Return the configured inbox directories (creates project inbox if missing).
+   */
+  dirs() {
+    return resolveInboxDirs(this.#rootDir, this.#env);
+  }
+}

package/lib/embed/intake-metrics.mjs

@@ -0,0 +1,190 @@
+/**
+ * lib/embed/intake-metrics.mjs — Intake pipeline observability.
+ *
+ * Aggregates volume, velocity, processing time, and acceptance rate from
+ * the filesystem intake queue. Designed for the dashboard /api/insights
+ * endpoint and the `construct intake metrics` CLI.
+ *
+ * Data sources:
+ *   .cx/intake/pending/   — JSON files waiting for triage
+ *   .cx/intake/processed/  — JSON files already handled
+ *   .cx/intake/skipped/     — JSON files intentionally skipped
+ *   ~/.cx/intake/recommendations-index.json — recommendations
+ *
+ * All read-only. Never writes to the intake queue.
+ */
+
+import { existsSync, readFileSync, readdirSync, statSync } from 'node:fs';
+import { join } from 'node:path';
+import { homedir } from 'node:os';
+
+/**
+ * Scan an intake directory and return file metadata.
+ */
+function scanDir(dir) {
+  if (!existsSync(dir)) return [];
+  try {
+    return readdirSync(dir)
+      .filter(f => f.endsWith('.json'))
+      .map(f => {
+        const fp = join(dir, f);
+        try {
+          const stat = statSync(fp);
+          const content = readFileSync(fp, 'utf8');
+          const parsed = JSON.parse(content);
+          return {
+            id: f.replace('.json', ''),
+            createdAt: parsed.createdAt || parsed.triage?.createdAt || stat.birthtime.toISOString(),
+            size: stat.size,
+            mtimeMs: stat.mtimeMs,
+            triage: parsed.triage?.intakeType || 'unknown',
+            owner: parsed.triage?.primaryOwner || '—',
+            action: parsed.triage?.recommendedAction || '—',
+            customers: parsed.customers?.length || 0,
+          };
+        } catch {
+          return null;
+        }
+      })
+      .filter(Boolean)
+      .sort((a, b) => new Date(b.createdAt) - new Date(a.createdAt));
+  } catch {
+    return [];
+  }
+}
+
+/**
+ * Compute metrics for an intake queue.
+ *
+ * @param {object} [opts]
+ * @param {string} [opts.rootDir] - Project root (default: cwd)
+ * @returns {object} Metrics summary
+ */
+export function computeIntakeMetrics({ rootDir } = {}) {
+  const root = rootDir || process.cwd();
+  const pendingDir = join(root, '.cx', 'intake', 'pending');
+  const processedDir = join(root, '.cx', 'intake', 'processed');
+  const skippedDir = join(root, '.cx', 'intake', 'skipped');
+
+  const pending = scanDir(pendingDir);
+  const processed = scanDir(processedDir);
+  const skipped = scanDir(skippedDir);
+
+  // Volume
+  const volume = {
+    pending: pending.length,
+    processed: processed.length,
+    skipped: skipped.length,
+    total: pending.length + processed.length + skipped.length,
+  };
+
+  // Velocity — items per day over trailing 7d window
+  const now = Date.now();
+  const SEVEN_DAYS = 7 * 24 * 60 * 60 * 1000;
+  const allItems = [...pending, ...processed, ...skipped];
+  const recentItems = allItems.filter(i => (now - new Date(i.createdAt).getTime()) < SEVEN_DAYS);
+  const velocity = Number((recentItems.length / 7).toFixed(1));
+
+  // Processing time — how long between creation and processing
+  const processingTimes = [];
+  for (const item of processed) {
+    try {
+      const fp = join(processedDir, `${item.id}.json`);
+      const raw = JSON.parse(readFileSync(fp, 'utf8'));
+      if (raw.createdAt && raw.processedAt) {
+        const ms = new Date(raw.processedAt) - new Date(raw.createdAt);
+        if (ms > 0) processingTimes.push(ms);
+      }
+    } catch { /* skip */ }
+  }
+  const avgProcessingTimeMs = processingTimes.length
+    ? Math.round(processingTimes.reduce((s, t) => s + t, 0) / processingTimes.length)
+    : null;
+
+  // — By type breakdown
+  const byType = {};
+  for (const item of allItems) {
+    byType[item.triage] = (byType[item.triage] || 0) + 1;
+  }
+
+  // — By owner breakdown
+  const byOwner = {};
+  for (const item of allItems) {
+    byOwner[item.owner] = (byOwner[item.owner] || 0) + 1;
+  }
+
+  // — Acceptance rate (processed / total actionable)
+  const actionable = volume.processed + volume.skipped;
+  const acceptRate = actionable > 0
+    ? Number((volume.processed / actionable * 100).toFixed(1))
+    : null;
+
+  // — Customer-linked items
+  const customerLinked = allItems.filter(i => i.customers > 0).length;
+
+  // — Metrics from recommendation store
+  let recStats = { active: 0, dismissed: 0, total: 0 };
+  try {
+    const recIndexPath = join(homedir(), '.cx', 'intake', 'recommendations-index.json');
+    if (existsSync(recIndexPath)) {
+      const index = JSON.parse(readFileSync(recIndexPath, 'utf8'));
+      const entries = Object.values(index);
+      recStats = {
+        total: entries.length,
+        active: entries.filter(e => !e.dismissedAt && !e.supersededAt).length,
+        dismissed: entries.filter(e => e.dismissedAt).length,
+      };
+    }
+  } catch { /* skip */ }
+
+  return {
+    volume,
+    velocity,
+    recentItems: recentItems.length,
+    avgProcessingTimeMs,
+    avgProcessingTimeFormatted: avgProcessingTimeMs
+      ? formatDuration(avgProcessingTimeMs)
+      : null,
+    byType,
+    byOwner,
+    acceptRate,
+    customerLinked,
+    recommendations: recStats,
+    sampledAt: new Date().toISOString(),
+  };
+}
+
+function formatDuration(ms) {
+  if (ms < 1000) return `${ms}ms`;
+  if (ms < 60000) return `${(ms / 1000).toFixed(1)}s`;
+  const mins = Math.floor(ms / 60000);
+  const secs = Math.round((ms % 60000) / 1000);
+  return `${mins}m ${secs}s`;
+}
+
+/**
+ * Get the age of the oldest pending item.
+ *
+ * @param {object} [opts]
+ * @param {string} [opts.rootDir]
+ * @returns {{ oldest: string|null, ageHours: number|null, count: number }}
+ */
+export function pendingAge({ rootDir } = {}) {
+  const root = rootDir || process.cwd();
+  const pendingDir = join(root, '.cx', 'intake', 'pending');
+  const items = scanDir(pendingDir);
+  if (!items.length) return { oldest: null, ageHours: null, count: 0 };
+
+  const oldest = items.reduce((a, b) =>
+    new Date(a.createdAt) < new Date(b.createdAt) ? a : b
+  );
+  const ageHours = (Date.now() - new Date(oldest.createdAt).getTime()) / (60 * 60 * 1000);
+
+  return {
+    oldest: oldest.id,
+    oldestCreatedAt: oldest.createdAt,
+    ageHours: Number(ageHours.toFixed(1)),
+    count: items.length,
+    oldItemsCount: items.filter(i => (Date.now() - new Date(i.createdAt).getTime()) > 24 * 60 * 60 * 1000).length,
+  };
+}