composto-ai 0.3.0 → 0.4.1

package/dist/memory/pool.js

@@ -0,0 +1,57 @@
+#!/usr/bin/env node
+
+// src/memory/pool.ts
+import { Worker } from "worker_threads";
+import { fileURLToPath } from "url";
+import { dirname, join } from "path";
+function resolveWorkerPath() {
+  const here = dirname(fileURLToPath(import.meta.url));
+  if (here.endsWith("/memory") || here.endsWith("\\memory")) {
+    return join(here, "worker.js");
+  }
+  return join(here, "memory", "worker.js");
+}
+var WorkerPool = class {
+  workers = [];
+  nextJobId = 1;
+  pending = /* @__PURE__ */ new Map();
+  constructor(opts = {}) {
+    const size = Math.max(1, opts.size ?? 1);
+    for (let i = 0; i < size; i++) this.spawn();
+  }
+  spawn() {
+    const worker = new Worker(resolveWorkerPath());
+    worker.on("message", (msg) => {
+      const job = this.pending.get(msg.jobId);
+      if (!job) return;
+      this.pending.delete(msg.jobId);
+      if (msg.type === "ingest_done") {
+        job.resolve({ status: "done", commits: msg.commits });
+      } else if (msg.type === "ingest_error") {
+        job.reject(new Error(msg.message));
+      }
+    });
+    worker.on("error", (err) => {
+      const error = err instanceof Error ? err : new Error(String(err));
+      for (const job of this.pending.values()) job.reject(error);
+      this.pending.clear();
+    });
+    this.workers.push(worker);
+  }
+  runIngest(args) {
+    const jobId = this.nextJobId++;
+    const worker = this.workers[jobId % this.workers.length];
+    return new Promise((resolve, reject) => {
+      this.pending.set(jobId, { resolve, reject });
+      worker.postMessage({ type: "ingest", jobId, ...args });
+    });
+  }
+  async close() {
+    await Promise.all(this.workers.map((w) => w.terminate()));
+    this.workers = [];
+    this.pending.clear();
+  }
+};
+export {
+  WorkerPool
+};

package/dist/memory/worker.js

@@ -0,0 +1,448 @@
+#!/usr/bin/env node
+
+// src/memory/worker.ts
+import { parentPort } from "worker_threads";
+
+// src/memory/db.ts
+import Database from "better-sqlite3";
+import { mkdirSync } from "fs";
+import { dirname } from "path";
+function openDatabase(path) {
+  mkdirSync(dirname(path), { recursive: true });
+  const db = new Database(path);
+  db.pragma("journal_mode = WAL");
+  db.pragma("synchronous = NORMAL");
+  db.pragma("foreign_keys = ON");
+  return db;
+}
+
+// src/memory/schema.ts
+var CURRENT_VERSION = 1;
+var V1_SQL = `
+CREATE TABLE IF NOT EXISTS index_state (
+  key   TEXT PRIMARY KEY,
+  value TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS commits (
+  sha         TEXT PRIMARY KEY,
+  parent_sha  TEXT,
+  author      TEXT NOT NULL,
+  timestamp   INTEGER NOT NULL,
+  subject     TEXT NOT NULL,
+  is_fix      INTEGER NOT NULL,
+  is_revert   INTEGER NOT NULL,
+  reverts_sha TEXT,
+  FOREIGN KEY (reverts_sha) REFERENCES commits(sha)
+);
+CREATE INDEX IF NOT EXISTS idx_commits_timestamp ON commits(timestamp);
+CREATE INDEX IF NOT EXISTS idx_commits_is_fix    ON commits(is_fix) WHERE is_fix = 1;
+
+CREATE TABLE IF NOT EXISTS file_touches (
+  commit_sha    TEXT NOT NULL,
+  file_path     TEXT NOT NULL,
+  adds          INTEGER NOT NULL,
+  dels          INTEGER NOT NULL,
+  change_type   TEXT NOT NULL,
+  renamed_from  TEXT,
+  PRIMARY KEY (commit_sha, file_path),
+  FOREIGN KEY (commit_sha) REFERENCES commits(sha)
+);
+CREATE INDEX IF NOT EXISTS idx_ft_file ON file_touches(file_path);
+
+CREATE TABLE IF NOT EXISTS symbols (
+  id              INTEGER PRIMARY KEY,
+  file_path       TEXT NOT NULL,
+  kind            TEXT NOT NULL,
+  qualified_name  TEXT NOT NULL,
+  first_seen_sha  TEXT NOT NULL,
+  last_seen_sha   TEXT,
+  UNIQUE (file_path, kind, qualified_name)
+);
+CREATE INDEX IF NOT EXISTS idx_symbols_file ON symbols(file_path);
+
+CREATE TABLE IF NOT EXISTS symbol_touches (
+  commit_sha    TEXT NOT NULL,
+  symbol_id     INTEGER NOT NULL,
+  change_type   TEXT NOT NULL,
+  PRIMARY KEY (commit_sha, symbol_id),
+  FOREIGN KEY (commit_sha) REFERENCES commits(sha),
+  FOREIGN KEY (symbol_id)  REFERENCES symbols(id)
+);
+CREATE INDEX IF NOT EXISTS idx_st_symbol ON symbol_touches(symbol_id);
+
+CREATE TABLE IF NOT EXISTS fix_links (
+  fix_commit_sha       TEXT NOT NULL,
+  suspected_break_sha  TEXT NOT NULL,
+  evidence_type        TEXT NOT NULL,
+  confidence           REAL NOT NULL,
+  window_hours         INTEGER,
+  PRIMARY KEY (fix_commit_sha, suspected_break_sha, evidence_type),
+  FOREIGN KEY (fix_commit_sha)      REFERENCES commits(sha),
+  FOREIGN KEY (suspected_break_sha) REFERENCES commits(sha)
+);
+CREATE INDEX IF NOT EXISTS idx_fl_break ON fix_links(suspected_break_sha);
+
+CREATE TABLE IF NOT EXISTS signal_calibration (
+  signal_type        TEXT PRIMARY KEY,
+  precision          REAL NOT NULL,
+  sample_size        INTEGER NOT NULL,
+  last_computed_sha  TEXT NOT NULL,
+  computed_at        INTEGER NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS file_index_state (
+  file_path            TEXT PRIMARY KEY,
+  last_commit_indexed  TEXT NOT NULL,
+  last_blob_indexed    TEXT,
+  indexed_at           INTEGER NOT NULL,
+  parse_failed         INTEGER NOT NULL DEFAULT 0,
+  FOREIGN KEY (last_commit_indexed) REFERENCES commits(sha)
+);
+`;
+function runMigrations(db) {
+  const current = db.pragma("user_version", { simple: true });
+  if (current >= CURRENT_VERSION) return;
+  db.exec("BEGIN");
+  try {
+    db.exec(V1_SQL);
+    db.pragma(`user_version = ${CURRENT_VERSION}`);
+    db.exec("COMMIT");
+  } catch (err) {
+    db.exec("ROLLBACK");
+    throw err;
+  }
+}
+
+// src/memory/git.ts
+import { execSync } from "child_process";
+function logRange(cwd, from, to, timeoutMs = 6e4) {
+  const range = from ? `${from}..${to}` : to;
+  const fmt = "--format=%x1e%H%x00%P%x00%an%x00%at%x00%s%x00%b%x1f";
+  const cmd = `git log ${fmt} --numstat --no-renames ${range}`;
+  return execSync(cmd, { cwd, encoding: "utf-8", timeout: timeoutMs, maxBuffer: 256 * 1024 * 1024 });
+}
+
+// src/memory/commit-parser.ts
+var FIX_PATTERNS = [
+  /\bfix(es|ed|ing)?\b/i,
+  /\bbugfix\b/i,
+  /\bhotfix\b/i,
+  /\bpatch\b/i,
+  /\bbug\b/i,
+  /closes?\s+#\d+/i,
+  /resolves?\s+#\d+/i
+];
+var REVERT_SUBJECT = /^\s*revert\b/i;
+var REVERT_BODY_SHA = /This reverts commit ([0-9a-f]{7,40})/i;
+function parseCommit(subject, body) {
+  const is_revert = REVERT_SUBJECT.test(subject);
+  const match = is_revert ? body.match(REVERT_BODY_SHA) : null;
+  const reverts_sha = match ? match[1] : null;
+  const is_fix = !is_revert && FIX_PATTERNS.some((re) => re.test(subject));
+  return { is_fix, is_revert, reverts_sha };
+}
+
+// src/memory/ingest/fix-links.ts
+var FOLLOWUP_WINDOW_HOURS = 72;
+var CHAIN_WINDOW_DAYS = 14;
+var CHAIN_MIN = 3;
+function deriveFixLinks(db) {
+  const insert = db.prepare(`
+    INSERT OR IGNORE INTO fix_links
+      (fix_commit_sha, suspected_break_sha, evidence_type, confidence, window_hours)
+    VALUES (?, ?, ?, ?, ?)
+  `);
+  const reverts = db.prepare(`
+    SELECT sha, reverts_sha FROM commits
+    WHERE is_revert = 1 AND reverts_sha IS NOT NULL
+  `).all();
+  for (const r of reverts) {
+    const exists = db.prepare("SELECT 1 FROM commits WHERE sha = ?").get(r.reverts_sha);
+    if (exists) insert.run(r.sha, r.reverts_sha, "revert_marker", 1, null);
+  }
+  const fixes = db.prepare(`
+    SELECT sha, timestamp FROM commits WHERE is_fix = 1
+  `).all();
+  const priorByFile = db.prepare(`
+    SELECT c.sha AS prior_sha
+      FROM file_touches ft_fix
+      JOIN file_touches ft_prior ON ft_prior.file_path = ft_fix.file_path
+      JOIN commits c ON c.sha = ft_prior.commit_sha
+     WHERE ft_fix.commit_sha = ?
+       AND c.timestamp < ?
+       AND c.timestamp >= ?
+       AND c.sha != ?
+       AND c.is_fix = 0
+       AND c.is_revert = 0
+  `);
+  for (const f of fixes) {
+    const lowerBound = f.timestamp - FOLLOWUP_WINDOW_HOURS * 3600;
+    const priors = priorByFile.all(f.sha, f.timestamp, lowerBound, f.sha);
+    const unique = new Set(priors.map((p) => p.prior_sha));
+    for (const prior of unique) {
+      insert.run(f.sha, prior, "short_followup_fix", 0.7, FOLLOWUP_WINDOW_HOURS);
+    }
+  }
+  const filesWithFixes = db.prepare(`
+    SELECT DISTINCT ft.file_path
+      FROM file_touches ft
+      JOIN commits c ON c.sha = ft.commit_sha
+     WHERE c.is_fix = 1
+  `).all();
+  const fixesOnFile = db.prepare(`
+    SELECT c.sha, c.timestamp
+      FROM commits c
+      JOIN file_touches ft ON ft.commit_sha = c.sha
+     WHERE ft.file_path = ? AND c.is_fix = 1
+     ORDER BY c.timestamp ASC
+  `);
+  const priorNonFixOnFile = db.prepare(`
+    SELECT c.sha
+      FROM commits c
+      JOIN file_touches ft ON ft.commit_sha = c.sha
+     WHERE ft.file_path = ?
+       AND c.timestamp < ?
+       AND c.is_fix = 0 AND c.is_revert = 0
+     ORDER BY c.timestamp DESC
+     LIMIT 1
+  `);
+  for (const { file_path } of filesWithFixes) {
+    const rows = fixesOnFile.all(file_path);
+    const windowSec = CHAIN_WINDOW_DAYS * 86400;
+    for (let i = 0; i + CHAIN_MIN - 1 < rows.length; i++) {
+      const windowEnd = rows[i + CHAIN_MIN - 1];
+      if (windowEnd.timestamp - rows[i].timestamp > windowSec) continue;
+      const prior = priorNonFixOnFile.get(file_path, rows[i].timestamp);
+      if (!prior) continue;
+      for (let j = i; j < rows.length && rows[j].timestamp - rows[i].timestamp <= windowSec; j++) {
+        insert.run(rows[j].sha, prior.sha, "same_region_fix_chain", 0.4, CHAIN_WINDOW_DAYS * 24);
+      }
+    }
+  }
+}
+
+// src/memory/calibration.ts
+var LOOKAHEAD_SECONDS = 14 * 86400;
+var REFRESH_AFTER_SECONDS = 7 * 86400;
+function validateRevertMatch(db) {
+  const total = db.prepare(`SELECT COUNT(*) AS n FROM fix_links`).get().n;
+  const hits = db.prepare(`
+    SELECT COUNT(*) AS n
+    FROM fix_links fl
+    JOIN commits fix_c ON fix_c.sha = fl.fix_commit_sha
+    JOIN commits break_c ON break_c.sha = fl.suspected_break_sha
+    WHERE fix_c.timestamp - break_c.timestamp <= ?
+  `).get(LOOKAHEAD_SECONDS).n;
+  return { total, hits };
+}
+function validateHotspot(db) {
+  const total = db.prepare(`
+    SELECT COUNT(DISTINCT file_path) AS n FROM file_touches
+  `).get().n;
+  const hits = db.prepare(`
+    SELECT COUNT(DISTINCT ft.file_path) AS n
+    FROM file_touches ft
+    JOIN fix_links fl ON fl.suspected_break_sha = ft.commit_sha
+  `).get().n;
+  return { total, hits };
+}
+function validateFixRatio(db) {
+  const total = db.prepare(`
+    SELECT COUNT(DISTINCT file_path) AS n FROM file_touches
+  `).get().n;
+  const hits = db.prepare(`
+    SELECT COUNT(DISTINCT ft.file_path) AS n
+    FROM file_touches ft
+    JOIN fix_links fl ON fl.suspected_break_sha = ft.commit_sha
+  `).get().n;
+  return { total, hits };
+}
+function validateCoverageDecline(_db) {
+  return { total: 0, hits: 0 };
+}
+function validateAuthorChurn(db) {
+  const total = db.prepare(`SELECT COUNT(*) AS n FROM file_touches`).get().n;
+  const hits = db.prepare(`SELECT COUNT(*) AS n FROM fix_links`).get().n;
+  return { total, hits };
+}
+var VALIDATORS = {
+  revert_match: validateRevertMatch,
+  hotspot: validateHotspot,
+  fix_ratio: validateFixRatio,
+  coverage_decline: validateCoverageDecline,
+  author_churn: validateAuthorChurn
+};
+function refreshCalibration(db, headSha) {
+  const now = Math.floor(Date.now() / 1e3);
+  const upsert = db.prepare(`
+    INSERT INTO signal_calibration (signal_type, precision, sample_size, last_computed_sha, computed_at)
+    VALUES (?, ?, ?, ?, ?)
+    ON CONFLICT(signal_type) DO UPDATE SET
+      precision = excluded.precision,
+      sample_size = excluded.sample_size,
+      last_computed_sha = excluded.last_computed_sha,
+      computed_at = excluded.computed_at
+  `);
+  for (const [type, validator] of Object.entries(VALIDATORS)) {
+    const v = validator(db);
+    const precision = v.total === 0 ? 0 : v.hits / v.total;
+    upsert.run(type, precision, v.total, headSha, now);
+  }
+  db.prepare(`
+    INSERT INTO index_state (key, value) VALUES ('calibration_last_refreshed_at', ?)
+    ON CONFLICT(key) DO UPDATE SET value = excluded.value
+  `).run(String(now));
+}
+function shouldRefresh(db, currentHeadSha) {
+  const lastTimeRow = db.prepare(`SELECT value FROM index_state WHERE key = 'calibration_last_refreshed_at'`).get();
+  if (!lastTimeRow) return true;
+  const now = Math.floor(Date.now() / 1e3);
+  const lastTime = parseInt(lastTimeRow.value, 10);
+  if (now - lastTime >= REFRESH_AFTER_SECONDS) return true;
+  const anyCal = db.prepare(`SELECT last_computed_sha FROM signal_calibration LIMIT 1`).get();
+  if (!anyCal) return true;
+  return anyCal.last_computed_sha !== currentHeadSha;
+}
+
+// src/memory/ingest/tier1.ts
+var RECORD_SEP = "";
+var FIELD_SEP = "\0";
+var RECORD_END = "";
+function parseNumstatLine(line) {
+  const parts = line.split("	");
+  if (parts.length < 3) return null;
+  const adds = parts[0] === "-" ? 0 : parseInt(parts[0], 10);
+  const dels = parts[1] === "-" ? 0 : parseInt(parts[1], 10);
+  if (Number.isNaN(adds) || Number.isNaN(dels)) return null;
+  return { adds, dels, path: parts.slice(2).join("	").trim() };
+}
+function parseLogOutput(output) {
+  const commits = [];
+  const records = output.split(RECORD_SEP).slice(1);
+  for (const rec of records) {
+    const endIdx = rec.indexOf(RECORD_END);
+    if (endIdx === -1) continue;
+    const content = rec.slice(0, endIdx);
+    const tail = rec.slice(endIdx + 1);
+    const fields = content.split(FIELD_SEP);
+    if (fields.length < 6) continue;
+    const [sha, parent, author, tsStr, subject, ...rest] = fields;
+    const body = rest.join(FIELD_SEP);
+    const touches = [];
+    for (const line of tail.split("\n")) {
+      const trimmed = line.trim();
+      if (!trimmed) continue;
+      const parsed = parseNumstatLine(trimmed);
+      if (!parsed) continue;
+      touches.push({
+        file_path: parsed.path,
+        adds: parsed.adds,
+        dels: parsed.dels,
+        change_type: parsed.adds > 0 && parsed.dels === 0 ? "A" : parsed.dels > 0 && parsed.adds === 0 ? "D" : "M"
+      });
+    }
+    commits.push({
+      sha,
+      parent_sha: parent ? parent.split(" ")[0] : null,
+      author,
+      timestamp: parseInt(tsStr, 10),
+      subject,
+      body,
+      touches
+    });
+  }
+  return commits;
+}
+function resolveRevertsSha(raw, knownShas) {
+  if (!raw) return null;
+  if (knownShas.has(raw)) return raw;
+  if (raw.length < 40) {
+    for (const sha of knownShas) {
+      if (sha.startsWith(raw)) return sha;
+    }
+  }
+  return null;
+}
+function ingestRange(db, repoPath, range) {
+  const raw = logRange(repoPath, range.from, range.to);
+  const commits = parseLogOutput(raw);
+  commits.sort((a, b) => a.timestamp - b.timestamp);
+  const knownShas = new Set(commits.map((c) => c.sha));
+  for (const existing of db.prepare(`SELECT sha FROM commits`).all()) {
+    knownShas.add(existing.sha);
+  }
+  const insertCommit = db.prepare(`
+    INSERT OR IGNORE INTO commits
+      (sha, parent_sha, author, timestamp, subject, is_fix, is_revert, reverts_sha)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+  `);
+  const insertTouch = db.prepare(`
+    INSERT OR IGNORE INTO file_touches
+      (commit_sha, file_path, adds, dels, change_type, renamed_from)
+    VALUES (?, ?, ?, ?, ?, NULL)
+  `);
+  const upsertState = db.prepare(`
+    INSERT INTO index_state (key, value) VALUES (?, ?)
+    ON CONFLICT(key) DO UPDATE SET value = excluded.value
+  `);
+  const tx = db.transaction((batch) => {
+    for (const c of batch) {
+      const parsed = parseCommit(c.subject, c.body);
+      insertCommit.run(
+        c.sha,
+        c.parent_sha,
+        c.author,
+        c.timestamp,
+        c.subject,
+        parsed.is_fix ? 1 : 0,
+        parsed.is_revert ? 1 : 0,
+        resolveRevertsSha(parsed.reverts_sha, knownShas)
+      );
+      for (const t of c.touches) {
+        insertTouch.run(c.sha, t.file_path, t.adds, t.dels, t.change_type);
+      }
+    }
+  });
+  db.pragma("foreign_keys = OFF");
+  try {
+    const BATCH = 1e3;
+    for (let i = 0; i < commits.length; i += BATCH) {
+      tx(commits.slice(i, i + BATCH));
+    }
+  } finally {
+    db.pragma("foreign_keys = ON");
+  }
+  deriveFixLinks(db);
+  if (shouldRefresh(db, range.to)) {
+    refreshCalibration(db, range.to);
+  }
+  upsertState.run("last_indexed_sha", range.to);
+  const totalRow = db.prepare("SELECT COUNT(*) AS n FROM commits").get();
+  upsertState.run("indexed_commits_total", String(totalRow.n));
+  return commits.length;
+}
+
+// src/memory/worker.ts
+if (!parentPort) {
+  throw new Error("memory/worker.ts must run inside a Worker");
+}
+parentPort.on("message", (msg) => {
+  if (msg.type === "ingest") {
+    try {
+      const db = openDatabase(msg.dbPath);
+      runMigrations(db);
+      const n = ingestRange(db, msg.repoPath, msg.range);
+      db.close();
+      const out = { type: "ingest_done", jobId: msg.jobId, commits: n };
+      parentPort.postMessage(out);
+    } catch (err) {
+      const out = {
+        type: "ingest_error",
+        jobId: msg.jobId,
+        message: err instanceof Error ? err.message : String(err)
+      };
+      parentPort.postMessage(out);
+    }
+  }
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "composto-ai",
-  "version": "0.3.0",
+  "version": "0.4.1",
   "description": "Proactive AI team companion — less tokens, more insight",
   "type": "module",
   "bin": {
@@ -32,6 +32,7 @@
   "license": "MIT",
   "packageManager": "pnpm@10.30.1",
   "devDependencies": {
+    "@types/better-sqlite3": "^7.6.13",
     "@types/node": "^25.5.2",
     "@types/picomatch": "^4.0.3",
     "tsup": "^8.5.1",
@@ -42,6 +43,8 @@
   "dependencies": {
     "@anthropic-ai/sdk": "^0.87.0",
     "@modelcontextprotocol/sdk": "^1.29.0",
+    "better-sqlite3": "^11.10.0",
+    "ignore": "^7.0.5",
     "picomatch": "^4.0.4",
     "web-tree-sitter": "^0.26.8",
     "yaml": "^2.8.3",