@inerrata-corporation/errata 2.0.22-dev.4173 → 2.0.22-dev.4180

package/consolidate-worker.mjs

@@ -16010,20 +16010,20 @@ function openDatabase(path2, opts) {
     } catch {
     }
   }
-  const wrapStmt = (stmt) => ({
+  const wrapStmt = (stmt2) => ({
     run: (...params) => {
-      const r = stmt.run(...coerceParams(params));
+      const r = stmt2.run(...coerceParams(params));
       return {
         changes: Number(r.changes),
         lastInsertRowid: Number(r.lastInsertRowid)
       };
     },
     get: (...params) => {
-      const r = stmt.get(...coerceParams(params));
+      const r = stmt2.get(...coerceParams(params));
       return r;
     },
     all: (...params) => {
-      const r = stmt.all(...coerceParams(params));
+      const r = stmt2.all(...coerceParams(params));
       return r;
     }
   });
@@ -17083,6 +17083,169 @@ function textIndexReady(db) {
   return db.prepare("SELECT value FROM store_meta WHERE key = ?").get(TEXT_INDEX_KEY)?.["value"] === "installed";
 }
 
+// ../../packages/local-graph/src/lineage-integrity.ts
+var statements = /* @__PURE__ */ new WeakMap();
+function stmt(db, sql) {
+  let byDb = statements.get(db);
+  if (!byDb) statements.set(db, byDb = /* @__PURE__ */ new Map());
+  let s = byDb.get(sql);
+  if (!s) byDb.set(sql, s = db.prepare(sql));
+  return s;
+}
+var HISTORY_EDGE_TYPES = [
+  "SUPERSEDED_BY",
+  "CONTINUES",
+  "FORKED_FROM",
+  "REGRESSION_OF",
+  "SPLIT_INTO",
+  "REVEALED_BY"
+];
+var LINEAGE_INTEGRITY_KEY = "lineage-integrity-v1";
+var HISTORY_SQL = HISTORY_EDGE_TYPES.map((t) => `'${t}'`).join(",");
+var NOT_HISTORY = `type NOT IN (${HISTORY_SQL})`;
+var NOT_HISTORICAL_ROW = `json_type(attrs_json, '$.historical') IS NOT 'true'`;
+var RETAINS_EDGES = (n) => `(${n}.label = 'Problem'
+  AND json_type(${n}.attrs_json, '$.resolvedAt') IS NOT NULL
+  AND COALESCE(json_extract(${n}.attrs_json, '$.resolvedAs'), '') NOT IN ('duplicate', 'false_positive'))`;
+var PRESENT = (n) => `(${n}.valid_to IS NULL OR ${RETAINS_EDGES(n)})`;
+var STRANDED_EDGE_PREDICATE = `e.valid_to IS NULL AND e.${NOT_HISTORY}
+  AND (NOT EXISTS (SELECT 1 FROM nodes f WHERE f.id = e.from_id AND ${PRESENT("f")})
+    OR NOT EXISTS (SELECT 1 FROM nodes t WHERE t.id = e.to_id AND ${PRESENT("t")}))`;
+function countStrandedEdges(db) {
+  const rows = db.prepare(`SELECT e.type AS type, count(*) AS n FROM edges e WHERE ${STRANDED_EDGE_PREDICATE} GROUP BY e.type ORDER BY n DESC`).all();
+  const byType = {};
+  let total = 0;
+  for (const r of rows) {
+    byType[r.type] = Number(r.n);
+    total += Number(r.n);
+  }
+  return { total, byType };
+}
+function countLiveHeadConflicts(db) {
+  const row = db.prepare(
+    `SELECT count(*) AS n FROM (
+         SELECT COALESCE(logical_id, id) AS lid FROM nodes
+         WHERE valid_to IS NULL AND ${NOT_HISTORICAL_ROW}
+         GROUP BY lid HAVING count(*) > 1)`
+  ).get();
+  return Number(row?.n ?? 0);
+}
+function closeIncidentEdges(db, id, t) {
+  let closed = 0;
+  for (const side of ["from_id", "to_id"]) {
+    const r = stmt(db, `UPDATE edges SET valid_to = MAX(valid_from, ?) WHERE ${side} = ? AND valid_to IS NULL AND ${NOT_HISTORY}`).run(t, id);
+    closed += Number(r.changes);
+  }
+  return closed;
+}
+function reopenCascadeClosedEdges(db, id, closedAt) {
+  let reopened = 0;
+  for (const [side, other] of [["from_id", "to_id"], ["to_id", "from_id"]]) {
+    const r = stmt(
+      db,
+      `UPDATE edges SET valid_to = NULL
+       WHERE ${side} = ? AND valid_to = ? AND ${NOT_HISTORY}
+         AND EXISTS (SELECT 1 FROM nodes o WHERE o.id = edges.${other} AND o.valid_to IS NULL)`
+    ).run(id, closedAt);
+    reopened += Number(r.changes);
+  }
+  return reopened;
+}
+function collapseLiveHeads(db, logicalIds, t) {
+  const headsOf = db.prepare(
+    `SELECT id, version, valid_from FROM (
+       SELECT id, version, valid_from, attrs_json FROM nodes WHERE logical_id = ?1 AND valid_to IS NULL
+       UNION
+       SELECT id, version, valid_from, attrs_json FROM nodes WHERE id = ?1 AND logical_id IS NULL AND valid_to IS NULL
+     ) WHERE ${NOT_HISTORICAL_ROW}`
+  );
+  const continuesInto = db.prepare(
+    `SELECT to_id FROM edges WHERE from_id = ? AND type = 'CONTINUES' AND valid_to IS NULL`
+  );
+  const rekeyHead = db.prepare(`UPDATE nodes SET logical_id = ?1 WHERE id = ?1`);
+  const rekeyCopies = db.prepare(
+    `UPDATE nodes SET logical_id = ?1 WHERE logical_id = ?2 AND substr(id, 1, length(?1) + 2) = ?1 || '@v'`
+  );
+  const lineageMembers = db.prepare(
+    `SELECT id FROM nodes WHERE logical_id = ?1 UNION SELECT id FROM nodes WHERE id = ?1`
+  );
+  const closeContinues = db.prepare(
+    `UPDATE edges SET valid_to = MAX(valid_from, ?) WHERE from_id = ? AND to_id = ? AND type = 'CONTINUES' AND valid_to IS NULL`
+  );
+  let split = 0;
+  const queue = [...new Set(logicalIds)];
+  for (let i = 0; i < queue.length; i++) {
+    const lid = queue[i];
+    const heads = headsOf.all(lid);
+    if (heads.length < 2) continue;
+    const members = new Set(lineageMembers.all(lid).map((r) => r.id));
+    const splitOff = (h) => {
+      for (const { to_id } of continuesInto.all(h.id)) {
+        if (members.has(to_id)) closeContinues.run(t, h.id, to_id);
+      }
+      rekeyHead.run(h.id);
+      rekeyCopies.run(h.id, lid);
+      queue.push(h.id);
+      split++;
+    };
+    const kept = new Set(heads.map((h) => h.id));
+    for (const h of heads) {
+      if (h.id === lid) continue;
+      const refuted = continuesInto.all(h.id).some(
+        (e) => e.to_id !== h.id && kept.has(e.to_id)
+      );
+      if (refuted && kept.size > 1) {
+        splitOff(h);
+        kept.delete(h.id);
+      }
+    }
+    let remaining = heads.filter((h) => kept.has(h.id));
+    if (remaining.length < 2) continue;
+    remaining = [...remaining].sort(
+      (a, b) => Number(b.id === lid) - Number(a.id === lid) || Number(b.version) - Number(a.version) || Number(b.valid_from) - Number(a.valid_from) || (a.id < b.id ? -1 : 1)
+    );
+    for (const h of remaining.slice(1)) splitOff(h);
+  }
+  return split;
+}
+function repairLineageIntegrity(db, now) {
+  const historicalRowsClosed = Number(
+    db.prepare(
+      `UPDATE nodes SET valid_to = MAX(valid_from, COALESCE(
+           (SELECT s.valid_from FROM edges s WHERE s.id = 'edge_superseded_' || nodes.id), ?))
+         WHERE valid_to IS NULL AND json_type(attrs_json, '$.historical') = 'true'`
+    ).run(now).changes
+  );
+  const conflicted = db.prepare(
+    `SELECT COALESCE(logical_id, id) AS lid FROM nodes
+         WHERE valid_to IS NULL AND ${NOT_HISTORICAL_ROW}
+         GROUP BY lid HAVING count(*) > 1`
+  ).all().map((r) => r.lid);
+  const headsSplit = collapseLiveHeads(db, conflicted, now);
+  const strandedEdgesClosed = Number(
+    db.prepare(
+      `UPDATE edges SET valid_to = MAX(valid_from, COALESCE(
+           (SELECT min(v) FROM (
+              SELECT f.valid_to AS v FROM nodes f WHERE f.id = edges.from_id AND NOT ${PRESENT("f")}
+              UNION ALL
+              SELECT t.valid_to FROM nodes t WHERE t.id = edges.to_id AND NOT ${PRESENT("t")})), ?))
+         WHERE id IN (SELECT e.id FROM edges e WHERE ${STRANDED_EDGE_PREDICATE})`
+    ).run(now).changes
+  );
+  return { historicalRowsClosed, headsSplit, strandedEdgesClosed };
+}
+function installLineageIntegrityRepair(db, now = Date.now()) {
+  if (db.prepare("SELECT value FROM store_meta WHERE key = ?").get(LINEAGE_INTEGRITY_KEY)) return null;
+  return db.transaction(() => {
+    const report = repairLineageIntegrity(db, now);
+    db.prepare("INSERT OR REPLACE INTO store_meta (key, value) VALUES (?, ?)").run(
+      LINEAGE_INTEGRITY_KEY,
+      JSON.stringify({ at: now, ...report })
+    );
+    return report;
+  });
+}
+
 // ../../packages/math/src/constants.ts
 var NEMORI_DEDUP_COSINE = 0.88;
 
@@ -17706,6 +17869,7 @@ var SqliteGraphStore = class {
         this.db.prepare("INSERT OR REPLACE INTO store_meta (key, value) VALUES (?, 'done')").run(SESSION_REPAIR_KEY);
       });
     }
+    installLineageIntegrityRepair(this.db);
     this.stmts = {
       insertNode: this.db.prepare(`
         INSERT INTO nodes (
@@ -17916,6 +18080,12 @@ var SqliteGraphStore = class {
         AND (t.id IS NULL OR t.valid_to IS NOT NULL)
     `).all();
   }
+  strandedEdges() {
+    return countStrandedEdges(this.db);
+  }
+  liveHeadConflicts() {
+    return countLiveHeadConflicts(this.db);
+  }
   fileIndexStates(ids) {
     if (ids.length === 0) return [];
     if (ids.length > 500) throw new RangeError("fileIndexStates accepts at most 500 ids per read");
@@ -18084,15 +18254,15 @@ var SqliteGraphStore = class {
   findNodeFieldsByLabel(label, attrKeys) {
     for (const k of attrKeys) if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(k)) throw new Error(`findNodeFieldsByLabel: bad attr key ${k}`);
     const cacheKey = attrKeys.join(",");
-    let stmt = this.fieldStmts.get(cacheKey);
-    if (!stmt) {
+    let stmt2 = this.fieldStmts.get(cacheKey);
+    if (!stmt2) {
       const cols = attrKeys.map((k, i) => `, ${attrValueSql(`'$.${k}'`)} AS a${i}, json_type(attrs_json, '$.${k}') AS t${i}`).join("");
-      stmt = this.db.prepare(
+      stmt2 = this.db.prepare(
         `SELECT id, description, community, extraction_confidence${cols} FROM nodes WHERE label = ? AND valid_to IS NULL`
       );
-      this.fieldStmts.set(cacheKey, stmt);
+      this.fieldStmts.set(cacheKey, stmt2);
     }
-    return stmt.all(label).map((r) => {
+    return stmt2.all(label).map((r) => {
       const attrs = {};
       attrKeys.forEach((k, i) => {
         const v = decodeAttr(r[`a${i}`], r[`t${i}`]);
@@ -18212,14 +18382,14 @@ var SqliteGraphStore = class {
   }
   nodeLabels(ids) {
     const width = NODE_LABELS_CHUNK;
-    const stmt = this.typedEdgeStmt(
+    const stmt2 = this.typedEdgeStmt(
       `SELECT id, label FROM nodes WHERE id IN (${Array.from({ length: width }, () => "?").join(",")})`
     );
     const labels = /* @__PURE__ */ new Map();
     for (let i = 0; i < ids.length; i += width) {
       const chunk = ids.slice(i, i + width);
       while (chunk.length < width) chunk.push(null);
-      for (const row of stmt.all(...chunk)) labels.set(row.id, row.label);
+      for (const row of stmt2.all(...chunk)) labels.set(row.id, row.label);
     }
     return labels;
   }
@@ -18418,12 +18588,12 @@ var SqliteGraphStore = class {
     return scale;
   }
   typedEdgeStmt(sql) {
-    let stmt = this.typedEdgeStmts.get(sql);
-    if (!stmt) {
-      stmt = this.db.prepare(sql);
-      this.typedEdgeStmts.set(sql, stmt);
+    let stmt2 = this.typedEdgeStmts.get(sql);
+    if (!stmt2) {
+      stmt2 = this.db.prepare(sql);
+      this.typedEdgeStmts.set(sql, stmt2);
     }
-    return stmt;
+    return stmt2;
   }
   outEdges(nodeId, types) {
     if (!types || types.length === 0) {
@@ -18624,22 +18794,28 @@ var SqliteGraphStore = class {
     }
   }
   reviveReobserved(relPaths, workspaceId, since) {
-    let revived = 0;
+    const revived = [];
     const CHUNK = 400;
     for (let i = 0; i < relPaths.length; i += CHUNK) {
       const slice = relPaths.slice(i, i + CHUNK);
       if (slice.length === 0) continue;
-      const r = this.db.prepare(
-        `UPDATE nodes SET valid_to = NULL
+      const rows = this.db.prepare(
+        `SELECT id, valid_to AS closedAt, COALESCE(logical_id, id) AS lid FROM nodes
              WHERE valid_to IS NOT NULL
                AND last_updated_at >= ?
                AND json_extract(attrs_json, '$.workspaceId') = ?
-               AND json_extract(attrs_json, '$.relPath') IN (${slice.map(() => "?").join(",")})`
-      ).run(since, workspaceId, ...slice);
-      revived += Number(r.changes);
+               AND json_extract(attrs_json, '$.relPath') IN (${slice.map(() => "?").join(",")})
+               AND json_type(attrs_json, '$.historical') IS NOT 'true'`
+      ).all(since, workspaceId, ...slice);
+      revived.push(...rows);
     }
-    if (revived > 0) this.mutations++;
-    return revived;
+    if (revived.length === 0) return 0;
+    const reopen = this.db.prepare("UPDATE nodes SET valid_to = NULL WHERE id = ? AND valid_to IS NOT NULL");
+    for (const r of revived) reopen.run(r.id);
+    for (const r of revived) reopenCascadeClosedEdges(this.db, r.id, Number(r.closedAt));
+    collapseLiveHeads(this.db, revived.map((r) => r.lid), since);
+    this.mutations++;
+    return revived.length;
   }
   incomingEdgeCoverage(label, edgeTypes, cloudOnly) {
     if (edgeTypes.length === 0) return { count: 0, total: 0 };
@@ -18785,8 +18961,9 @@ var SqliteGraphStore = class {
     this.stmts.advanceLive.run({ live_id: liveId, t });
     return frozenId;
   }
-  closeNode(id, t) {
-    this.stmts.closeNode.run({ id, t });
+  closeNode(id, t, opts = {}) {
+    const r = this.stmts.closeNode.run({ id, t });
+    if (Number(r.changes) > 0 && !opts.retainEdges) closeIncidentEdges(this.db, id, t);
   }
   closeEdge(id, t) {
     this.stmts.closeEdge.run({ id, t });