@vortex-os/ontos 0.1.0 → 0.2.0

package/scripts/enrich-ontos.mjs

@@ -0,0 +1,182 @@
+#!/usr/bin/env node
+// Agent-mediated LLM enrichment (Phase 1 §4 step 3) — the thin I/O wrapper over
+// the pure enrich layer. ontos has no LLM of its own: this EMITs extraction
+// requests for the running agent to fulfil, then INGESTs the JSON the agent
+// returns (validated → tiered → stored as PROPOSALS, never into the trusted graph).
+//
+// Two-step, agent in the middle:
+//   1. enrich-ontos --emit requests.json     # ontos -> agent : what to extract
+//   2. (the agent reads requests.json, does the extraction + self/cross-check,
+//       writes proposals.json: [{ recordPath, proposals: [RawProposal...] }])
+//   3. enrich-ontos --ingest proposals.json  # agent -> ontos : store proposals
+//
+// Read-only on your markdown in BOTH steps; writes only the sqlite index (ingest).
+// Usage: enrich-ontos [--data-dir <dir>] [--db <path>] (--emit [file] | --ingest <file>)
+import { readFileSync, readdirSync, writeFileSync, existsSync } from "node:fs";
+import { join, resolve, sep } from "node:path";
+import { OntosStore } from "../dist/sqlite/index.js";
+import { extractDeterministic } from "../dist/extract/index.js";
+import {
+  buildEnrichmentRequest,
+  needsEnrichment,
+  extractionHash,
+  ingestProposals,
+} from "../dist/enrich/index.js";
+import { parseFrontmatter } from "../dist/internal/frontmatter.js";
+
+const args = process.argv.slice(2);
+let dataDir = resolve(process.cwd(), "data");
+let dbPath = null;
+let mode = null; // "emit" | "ingest"
+let ioFile = null;
+for (let i = 0; i < args.length; i++) {
+  const a = args[i];
+  if (a === "--data-dir" && args[i + 1]) dataDir = resolve(args[++i]);
+  else if (a === "--db" && args[i + 1]) dbPath = resolve(args[++i]);
+  else if (a === "--emit") {
+    mode = "emit";
+    if (args[i + 1] && !args[i + 1].startsWith("--")) ioFile = resolve(args[++i]);
+  } else if (a === "--ingest" && args[i + 1]) {
+    mode = "ingest";
+    ioFile = resolve(args[++i]);
+  } else if (a === "--gc") {
+    mode = "gc";
+  }
+}
+if (!dbPath) dbPath = join(dataDir, "_indexes", "ontos.sqlite");
+if (!mode) {
+  console.error("usage: enrich-ontos [--data-dir <dir>] [--db <path>] (--emit [file] | --ingest <file> | --gc)");
+  process.exit(2);
+}
+
+// Prose-heavy sources where the LLM gap-filler earns its keep. worklog is NESTED
+// (YYYY/MM/…), so it is walked recursively; decision-log and _memory are flat.
+// _failures is frontmatter-structured (deterministic handles it) — not enriched.
+const SOURCES = [
+  { dir: join(dataDir, "worklog"), category: "worklog", recursive: true },
+  { dir: join(dataDir, "decision-log"), category: "decision", recursive: false },
+  { dir: join(dataDir, "_memory"), category: "memory", recursive: false },
+];
+
+function listMd(dir, recursive) {
+  let entries;
+  try {
+    entries = readdirSync(dir, { withFileTypes: true });
+  } catch {
+    return []; // dir absent — skip
+  }
+  const out = [];
+  for (const e of entries) {
+    if (e.isDirectory()) {
+      if (recursive) out.push(...listMd(join(dir, e.name), true));
+    } else if (e.isFile() && e.name.endsWith(".md") && e.name !== "_INDEX.md") {
+      out.push(join(dir, e.name));
+    }
+  }
+  return out;
+}
+
+function* records() {
+  for (const { dir, category, recursive } of SOURCES) {
+    for (const path of listMd(dir, recursive)) {
+      const { frontmatter, body } = parseFrontmatter(readFileSync(path, "utf8"));
+      yield { path, category, frontmatter, body };
+    }
+  }
+}
+
+const store = new OntosStore(dbPath);
+try {
+  if (mode === "emit") {
+    const known = store.allObjectIds();
+    const requests = [];
+    let scanned = 0;
+    for (const rec of records()) {
+      scanned++;
+      if (!needsEnrichment(store, rec)) continue; // unchanged since last enrichment
+      requests.push(buildEnrichmentRequest(rec, extractDeterministic(rec), known));
+    }
+    const payload = JSON.stringify(requests, null, 2);
+    if (ioFile) writeFileSync(ioFile, payload);
+    else process.stdout.write(payload + "\n");
+    console.error(
+      `[enrich-ontos] emit: ${requests.length} record(s) need enrichment (of ${scanned} scanned)${ioFile ? ` -> ${ioFile}` : ""}`,
+    );
+  } else if (mode === "gc") {
+    // Reconcile: drop enrichment_state + UNCONFIRMED proposals whose source record
+    // vanished (rename/delete). Liveness is an fs check here (the store is pure
+    // sqlite); confirmed/rejected proposals are human decisions and are kept.
+    const dead = (p) => !existsSync(resolve(p));
+    let states = 0;
+    for (const rp of store.allEnrichmentPaths()) {
+      if (dead(rp) && store.deleteEnrichmentState(rp)) states++;
+    }
+    let props = 0;
+    for (const ev of store.pendingProposalEvidencePaths()) {
+      if (dead(ev)) props += store.dropPendingProposalsByEvidence(ev);
+    }
+    console.error(`[enrich-ontos] gc: dropped ${states} stale enrichment_state, ${props} orphaned proposal(s)`);
+  } else {
+    // ingest: [{ recordPath, proposals: [RawProposal...] }]
+    const parsed = JSON.parse(readFileSync(ioFile, "utf8"));
+    if (!Array.isArray(parsed)) {
+      console.error("[enrich-ontos] ingest: expected a top-level JSON array of { recordPath, proposals }");
+      process.exit(2);
+    }
+    const at = new Date().toISOString().slice(0, 10);
+    const dataRoot = resolve(dataDir);
+    let accepted = 0;
+    let rejected = 0;
+    let records_ = 0;
+    let skipped = 0;
+    for (const item of parsed) {
+      const recordPath = item?.recordPath;
+      // Malformed shape (no path, or proposals not an array) -> skip; do NOT stamp
+      // state, or a bad agent reply would suppress future --emit for that record.
+      if (typeof recordPath !== "string" || !Array.isArray(item?.proposals)) {
+        skipped++;
+        continue;
+      }
+      // Bound the path to the data dir: the agent only ever echoes back a path ontos
+      // emitted, so anything outside scope is malformed/hostile — refuse to read it
+      // (defense against path traversal, even though we only hash the content).
+      const abs = resolve(recordPath);
+      if (abs !== dataRoot && !abs.startsWith(dataRoot + sep)) {
+        skipped++;
+        continue;
+      }
+      // Re-read the record to compute its current extraction hash (the change-detection
+      // stamp), so a later text edit re-emits it. Read-only. Unreadable -> skip (do not
+      // ingest proposals for a record we cannot verify, and do not stamp).
+      let hash;
+      try {
+        const { body } = parseFrontmatter(readFileSync(abs, "utf8"));
+        hash = extractionHash({ body });
+      } catch {
+        skipped++;
+        continue;
+      }
+      records_++;
+      // Anchor each proposal's evidence to the record's ABSOLUTE path (all proposals
+      // in this item come from this record). A raw/relative evidencePath would make
+      // GC's existsSync(resolve(...)) cwd-dependent and mis-classify orphans; the
+      // specific quote is preserved in evidenceSpan (Codex review).
+      const proposals = item.proposals.map((p) => ({ ...p, evidencePath: abs }));
+      const report = ingestProposals(store, proposals, { sourceRecordHash: hash, proposedAt: at });
+      accepted += report.accepted;
+      rejected += report.rejected.length;
+      // Stamp the change-detection state ONLY when something was actually captured.
+      // An empty or all-rejected reply (a failed/empty agent run) must NOT mark the
+      // record done, or it would silently suppress future --emit (Codex round 2).
+      // Trade-off: a record with genuinely no extractable relations re-emits each
+      // run; a later "examined, nothing found" marker can suppress that — TODO.
+      if (report.accepted > 0) store.setExtractionState(abs, hash, at);
+    }
+    console.error(
+      `[enrich-ontos] ingest: ${accepted} accepted, ${rejected} rejected, ${skipped} skipped, across ${records_} record(s)`,
+    );
+  }
+  console.error(`[enrich-ontos] db: ${store.dbPath}`);
+} finally {
+  store.close();
+}

package/scripts/rebuild-ontos.mjs

@@ -25,6 +25,7 @@ const SOURCES = [
   { dir: join(dataDir, "_failures"), category: "failure" },
   { dir: join(dataDir, "decision-log"), category: "decision" },
   { dir: join(dataDir, "_memory"), category: "memory" },
+  { dir: join(dataDir, "runbooks"), category: "runbook" },
 ];
 
 // FLAT listing (non-recursive): the extracted categories (_failures, decision-log,