@remnic/coding-graph 9.3.759

package/src/graph-schema.ts

@@ -0,0 +1,383 @@
+/**
+ * Coding-graph SQLite schema — versioned meta + tables + FTS5 virtual table.
+ *
+ * Issue #1552 PR1 (Track B, Phase 1). The schema + write pipeline only;
+ * traversal, search, dead-code and the openCypher subset land in PR2/PR3.
+ *
+ * PR2 additive table (`node_attributes`): tracks per-node exclusion flags
+ * consumed by `deadCode()` — `is_exported`, `is_route_handler`. Added in
+ * PR2 (issue #1552 step 5) as a SEPARATE table rather than ALTER TABLE on
+ * `nodes`, so existing v1 databases gain the table via the same
+ * `CREATE TABLE IF NOT EXISTS` pass without a schema-version bump or a
+ * migration (rule 23 — additive, characterized before moving).
+ *
+ * Design anchors:
+ *   - `packages/remnic-core/src/lcm/schema.ts` (versioning + pragmas) —
+ *     copied VERBATIM for the WAL / busy_timeout / synchronous pragmas and
+ *     the meta-table version row. Do not invent a new pattern (rule 23/38).
+ *   - `packages/remnic-core/src/runtime/better-sqlite.ts` (`openBetterSqlite3`)
+ *     — the shared opener; native-binding lifecycle is paid for once there.
+ *   - issue://1552 "Design" — node ids hash sorted key material; file
+ *     contents are NEVER stored (only spans + content hashes); FTS5 covers
+ *     node names; provenance CHECK enforces the four-value enum.
+ *
+ * Schema versioning:
+ *   The schema_version row lives in `meta` (a generic key/value table, not
+ *   lcm_meta — independent store, separate version namespace). Fresh DB →
+ *   schema_version=1. Upgrade stub: meta v0 → v1 (rule 23, characterize
+ *   before moving).
+ *
+ * Dangling-edge policy (PR1 decision):
+ *   When `upsertFileBatch` deletes a file's prior nodes + edges, cross-file
+ *   edges whose `dst` was a node owned by the deleted file become dangling.
+ *   We DROP them. They are counted in `UpsertResult.droppedDanglingEdges`
+ *   so callers can surface the loss. Keeping them with a `dst_unresolved`
+ *   marker would leak orphans and bias `traverse()` results — drop is the
+ *   conservative choice for a write pipeline whose caller knows the
+ *   canonical file set on each batch (rule 11, 40).
+ */
+import type { BetterSqlite3Database } from "@remnic/core/runtime/better-sqlite";
+
+import { expectRow, expectRows } from "./row-types.js";
+
+export const CODING_GRAPH_SCHEMA_VERSION = 1;
+
+/**
+ * FTS5 rowid derived from a deterministic node id. FTS5 rowids are
+ * signed 64-bit integers; we slice the leading 16 hex chars (= 64 bits)
+ * of the sha256 id and mask to the int64 positive range so SQLite
+ * accepts it. The full 64-bit space is large enough that collisions
+ * across distinct node ids are negligible. Contentless FTS5
+ * (`content=''`) does NOT store UNINDEXED column values, so the only
+ * reliable key into the virtual table is the rowid.
+ *
+ * Lives in graph-schema (not graph-store) so the schema migration path
+ * can rebuild FTS rows from existing `nodes` without importing the
+ * store module (which would create a circular dependency — graph-store
+ * imports graph-schema).
+ */
+export function ftsRowidForNodeId(nodeId: string): bigint {
+  return BigInt(`0x${nodeId.slice(0, 16)}`) & BigInt("0x7fffffffffffffff");
+}
+
+/**
+ * Provenance enum for edges — mirrors #1552's `heuristic|lsp|trace|semantic`
+ * whitelist. The CHECK constraint rejects writes outside this set so a
+ * buggy resolver can't sneak in unknown values (rule 23).
+ */
+export const EDGE_PROVENANCE_VALUES = [
+  "heuristic",
+  "lsp",
+  "trace",
+  "semantic",
+] as const;
+
+export type EdgeProvenance = (typeof EDGE_PROVENANCE_VALUES)[number];
+
+export function isEdgeProvenance(value: unknown): value is EdgeProvenance {
+  return (
+    typeof value === "string" &&
+    (EDGE_PROVENANCE_VALUES as readonly string[]).includes(value)
+  );
+}
+
+/**
+ * Apply (or upgrade) the coding-graph schema on an already-open SQLite
+ * handle. Public so test seams and migration tools can bootstrap an
+ * in-memory database without going through {@link openCodingGraphDatabase}.
+ *
+ * Mirrors `applyLcmSchema` in `packages/remnic-core/src/lcm/schema.ts` —
+ * distinct function, same shape, separate version namespace.
+ */
+export function applyCodingGraphSchema(db: BetterSqlite3Database): void {
+  const versionRow = expectRow<{ name: string }>(
+    db
+      .prepare(
+        "SELECT name FROM sqlite_master WHERE type='table' AND name='meta'",
+      )
+      .get(),
+    ["name"],
+  );
+
+  if (!versionRow) {
+    // Fresh DB — create every table and stamp the current version.
+    createTables(db);
+    writeSchemaVersion(db, CODING_GRAPH_SCHEMA_VERSION);
+    return;
+  }
+
+  const meta = expectRow<{ value: string }>(
+    db
+      .prepare("SELECT value FROM meta WHERE key = 'schema_version'")
+      .get(),
+    ["value"],
+  );
+  const currentVersion = meta ? parseInt(meta.value, 10) : 0;
+
+  // Only run createTables when the on-disk version is at or below this
+  // code's version. For an AT-OR-BELOW DB (fresh-ish v0/v1) the pass is
+  // additive: every core statement is CREATE TABLE IF NOT EXISTS, so the
+  // PR2 `node_attributes` table appears on existing v1 databases without
+  // a version bump (chatgpt-codex-connector P1: 'Create node_attributes
+  // for existing v1 stores'), and a v0 DB is upgraded.
+  //
+  // A NEWER DB (currentVersion > CODING_GRAPH_SCHEMA_VERSION — older
+  // code opening a future-version DB after a downgrade, or a parallel
+  // install) must be left UNTOUCHED: createTables is NOT purely
+  // additive because its FTS migration drops + recreates `nodes_fts`
+  // when the stored CREATE SQL lacks `contentless_delete=1`. Running
+  // that against a future schema that legitimately changed or removed
+  // that table would mutate the newer schema while preserving its
+  // version marker — silent corruption. Skip createTables AND the
+  // version write for newer DBs (chatgpt-codex-connector P2: 'Skip
+  // destructive DDL for future schema versions').
+  if (currentVersion > CODING_GRAPH_SCHEMA_VERSION) {
+    return;
+  }
+  createTables(db);
+  writeSchemaVersion(db, CODING_GRAPH_SCHEMA_VERSION);
+}
+
+function createTables(db: BetterSqlite3Database): void {
+  // Provenance whitelist must match EDGE_PROVENANCE_VALUES. SQLite CHECK
+  // constraints are re-checked against every INSERT/UPDATE; bypassing this
+  // gate would require raw exec, which we never do (rule 51).
+  const provenanceList = EDGE_PROVENANCE_VALUES.map((v) => `'${v}'`).join(", ");
+
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS meta (
+      key   TEXT PRIMARY KEY,
+      value TEXT NOT NULL
+    );
+
+    CREATE TABLE IF NOT EXISTS files (
+      id           INTEGER PRIMARY KEY AUTOINCREMENT,
+      path         TEXT NOT NULL UNIQUE,
+      lang         TEXT NOT NULL,
+      content_hash TEXT NOT NULL
+    );
+    CREATE INDEX IF NOT EXISTS idx_files_path ON files(path);
+
+    CREATE TABLE IF NOT EXISTS nodes (
+      id             TEXT PRIMARY KEY,
+      label          TEXT NOT NULL,
+      name           TEXT NOT NULL,
+      qualified_name TEXT NOT NULL,
+      file_id        INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE,
+      span_start     INTEGER NOT NULL,
+      span_end       INTEGER NOT NULL,
+      lang           TEXT NOT NULL
+    );
+    CREATE INDEX IF NOT EXISTS idx_nodes_file ON nodes(file_id);
+    CREATE INDEX IF NOT EXISTS idx_nodes_qname ON nodes(qualified_name);
+    CREATE INDEX IF NOT EXISTS idx_nodes_label ON nodes(label);
+
+    CREATE TABLE IF NOT EXISTS edges (
+      src        TEXT NOT NULL REFERENCES nodes(id) ON DELETE CASCADE,
+      dst        TEXT NOT NULL REFERENCES nodes(id) ON DELETE CASCADE,
+      type       TEXT NOT NULL,
+      confidence REAL NOT NULL CHECK (confidence >= 0.0 AND confidence <= 1.0),
+      provenance TEXT NOT NULL CHECK (provenance IN (${provenanceList})),
+      UNIQUE (src, dst, type)
+    );
+    -- Destination-leading index. The UNIQUE(src,dst,type) key is
+    -- src-leading, so pruneFileNodes()'s 'WHERE dst IN (...)' count and
+    -- the ON DELETE CASCADE that follows a node delete (SQLite must
+    -- locate child edges by 'dst') would otherwise scan the whole edges
+    -- table. A dst-leading index turns ordinary symbol deletion into an
+    -- index lookup instead of a full scan
+    -- (chatgpt-codex-connector P2: 'Add an index for edge destination
+    -- lookups').
+    CREATE INDEX IF NOT EXISTS idx_edges_dst ON edges(dst);
+  `);
+  // PR2 (issue #1552 step 5): per-node exclusion flags consumed by
+  // `GraphStore.deadCode()`. Kept in a SEPARATE table rather than an
+  // ALTER TABLE on `nodes` so existing v1 databases gain the table via
+  // the same `CREATE TABLE IF NOT EXISTS` pass without a schema-version
+  // bump or a data migration (rule 23 — additive, characterized before
+  // moving). ON DELETE CASCADE on `nodes(id)` keeps the table in lockstep
+  // with node lifetimes; the foreign_keys=ON pragma set in
+  // `GraphStore.open()` enforces it.
+  //
+  // `is_exported`: 1 when the symbol's `name` matches an entry in the
+  //   FileIR's `exports` list (matched per-file at write time). The
+  //   dead-code query treats exported symbols as not-dead even with
+  //   zero inbound CALLS/USES_TYPE edges — they form the package's
+  //   public surface and may be called by external consumers the graph
+  //   cannot see.
+  // `is_route_handler`: 1 when the symbol's `qualified_name` matches a
+  //   route's `handlerQualifiedName` in the FileIR's `routes` list.
+  //   Route handlers are reachable from HTTP requests regardless of
+  //   whether any other node CALLS them inside the indexed codebase.
+  //
+  // Both columns are NOT NULL with CHECK IN (0,1): a missing row means
+  // "neither flag set" (the LEFT JOIN in deadCode() COALESCEs to 0).
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS node_attributes (
+      node_id           TEXT PRIMARY KEY REFERENCES nodes(id) ON DELETE CASCADE,
+      is_exported       INTEGER NOT NULL CHECK (is_exported IN (0, 1)),
+      is_route_handler  INTEGER NOT NULL CHECK (is_route_handler IN (0, 1))
+    );
+    CREATE INDEX IF NOT EXISTS idx_node_attributes_exported
+      ON node_attributes(is_exported) WHERE is_exported = 1;
+    CREATE INDEX IF NOT EXISTS idx_node_attributes_route
+      ON node_attributes(is_route_handler) WHERE is_route_handler = 1;
+  `);
+  // PR3 (issue #1553): co-change edges — file-level relationships mined
+  // from git history. Stored separately from the symbol-level `edges`
+  // table because co-change is a file-to-file concern, not symbol-to-
+  // symbol. Additive to v1 (CREATE TABLE IF NOT EXISTS — same pattern
+  // as `node_attributes` in PR2, rule 23).
+  //
+  // UNIQUE(file_a, file_b) ensures idempotent upserts; the mining
+  // pipeline clears + repopulates each run so stale edges from history
+  // changes are pruned automatically.
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS co_changes (
+      file_a     TEXT NOT NULL,
+      file_b     TEXT NOT NULL,
+      support    INTEGER NOT NULL CHECK (support >= 0),
+      confidence REAL NOT NULL CHECK (confidence >= 0.0 AND confidence <= 1.0),
+      UNIQUE (file_a, file_b)
+    );
+    CREATE INDEX IF NOT EXISTS idx_co_changes_a ON co_changes(file_a);
+    CREATE INDEX IF NOT EXISTS idx_co_changes_b ON co_changes(file_b);
+  `);
+  // Semantic layer (issue #1556): symbol embedding vectors. Additive to v1
+  // (CREATE TABLE IF NOT EXISTS — same rule-23 pattern as node_attributes).
+  // One row per (node_id, model_id): a node re-embedded under a different
+  // provider/model gets a distinct row so a provider swap does not
+  // overwrite the prior vectors (the cache invalidation test covers this).
+  // content_hash is the canonical-text hash (rule 37) — a re-index compares
+  // it to decide whether to re-embed. CASCADE on nodes(id) keeps the table
+  // in lockstep with node lifetimes (foreign_keys=ON is set in GraphStore.open).
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS symbol_vectors (
+      node_id      TEXT NOT NULL REFERENCES nodes(id) ON DELETE CASCADE,
+      model_id     TEXT NOT NULL,
+      content_hash TEXT NOT NULL,
+      dims         INTEGER NOT NULL CHECK (dims > 0),
+      vector       BLOB NOT NULL,
+      PRIMARY KEY (node_id, model_id)
+    );
+    CREATE INDEX IF NOT EXISTS idx_symbol_vectors_model
+      ON symbol_vectors(model_id);
+  `);
+  // repopulate both tables in lockstep without ordering concerns.
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS fts_index (
+      fts_rowid INTEGER PRIMARY KEY,
+      node_id   TEXT NOT NULL UNIQUE
+    );
+    CREATE INDEX IF NOT EXISTS idx_fts_index_node ON fts_index(node_id);
+  `);
+  // FTS5 virtual table over node names + qualified_names for PR2's
+  // name search. Contentless-delete mode (`content=''` plus
+  // `contentless_delete=1`, SQLite 3.43+) lets us run standard
+  // `DELETE` and `INSERT OR REPLACE` statements against the virtual
+  // table — the write pipeline's only requirement. The `id UNINDEXED`
+  // column mirrors the deterministic node id for human-readable
+  // inspection; the write pipeline's actual key is the rowid derived
+  // from the node-id hash (see `ftsRowidForNodeId` below).
+  //
+  // Migration: databases created before the contentless-FTS5 fix have
+  // an OLD `nodes_fts` table whose CREATE SQL used `content=nodes`
+  // (external-content) and lacks `contentless_delete=1`. The write
+  // pipeline's `DELETE FROM nodes_fts WHERE rowid = ?` fails on such
+  // a table while the source `nodes` row still exists, throwing and
+  // aborting the whole batch (kilo WARNING: 'Contentless FTS5 assumes
+  // fresh schema — old databases are not migrated'). We detect this
+  // by inspecting sqlite_master for the `contentless_delete=1` token;
+  // if absent, we DROP + RECREATE the virtual table and rebuild its
+  // rows from the surviving `nodes` table so the index is immediately
+  // usable after migration (rule 23: characterize before moving).
+  const ftsCreateSql = expectRow<{ sql: string }>(
+    db
+      .prepare(
+        "SELECT sql FROM sqlite_master WHERE type='table' AND name='nodes_fts'",
+      )
+      .get(),
+    ["sql"],
+  );
+  const needsFtsRecreate =
+    !ftsCreateSql || !ftsCreateSql.sql.includes("contentless_delete=1");
+  if (needsFtsRecreate) {
+    db.exec("DROP TABLE IF EXISTS nodes_fts;");
+    db.exec(`
+      CREATE VIRTUAL TABLE nodes_fts USING fts5(
+        name,
+        qualified_name,
+        id UNINDEXED,
+        content='',
+        contentless_delete=1,
+        tokenize='unicode61 remove_diacritics 2'
+      );
+    `);
+    // Rebuild FTS + fts_index from surviving nodes so both are populated
+    // after migration. Fresh databases have zero rows so this is a no-op.
+    const survivingNodes = expectRows<{
+      id: string;
+      name: string;
+      qualified_name: string;
+    }>(
+      db.prepare("SELECT id, name, qualified_name FROM nodes").all(),
+      ["id", "name", "qualified_name"],
+    );
+    if (survivingNodes.length > 0) {
+      const insertFts = db.prepare(
+        "INSERT INTO nodes_fts (rowid, name, qualified_name) VALUES (?, ?, ?)",
+      );
+      const upsertFtsIndex = db.prepare(
+        "INSERT OR REPLACE INTO fts_index (fts_rowid, node_id) VALUES (?, ?)",
+      );
+      for (const n of survivingNodes) {
+        const rowid = ftsRowidForNodeId(n.id);
+        insertFts.run(rowid, n.name, n.qualified_name);
+        upsertFtsIndex.run(rowid, n.id);
+      }
+    }
+  }
+  // NOTE: the schema_version marker is written by the caller
+  // (applyCodingGraphSchema / writeSchemaVersion), NOT here. createTables
+  // only owns additive DDL; the version-write concern (never downgrade a
+  // newer DB) lives at the apply layer where currentVersion is known.
+}
+
+/**
+ * Stamp the schema_version meta row. INSERT OR REPLACE so the upgrade
+ * path can rewrite the row in place (rule 23 — one canonical form).
+ * Callers MUST gate this on `currentVersion <= CODING_GRAPH_SCHEMA_VERSION`
+ * to avoid downgrading a newer DB.
+ */
+function writeSchemaVersion(
+  db: BetterSqlite3Database,
+  version: number,
+): void {
+  db.prepare(
+    "INSERT OR REPLACE INTO meta (key, value) VALUES ('schema_version', ?)",
+  ).run(String(version));
+}
+
+/**
+ * Read the schema_version row. Returns 0 when the table is missing or the
+ * row has not been written yet — used by the upgrade stub test.
+ *
+ * Mirrors the LCM helper's tolerance: a missing `meta` table is not an
+ * error condition here, it just means the schema has never been applied.
+ */
+export function readSchemaVersion(db: BetterSqlite3Database): number {
+  const metaTable = expectRow<{ name: string }>(
+    db
+      .prepare(
+        "SELECT name FROM sqlite_master WHERE type='table' AND name='meta'",
+      )
+      .get(),
+    ["name"],
+  );
+  if (!metaTable) return 0;
+  const meta = expectRow<{ value: string }>(
+    db.prepare("SELECT value FROM meta WHERE key = 'schema_version'").get(),
+    ["value"],
+  );
+  return meta ? parseInt(meta.value, 10) : 0;
+}