@inerrata-corporation/errata 2.0.22-dev.4641 → 2.0.22-dev.4657
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/consolidate-worker.mjs +190 -22
- package/errata.mjs +192 -24
- package/generalize-worker.mjs +145 -14
- package/package.json +1 -1
- package/pass-worker.mjs +90 -14
- package/resources/context-plugin/.codex-plugin/plugin.json +1 -1
- package/resources/context-plugin/SHA256SUMS +1 -1
package/consolidate-worker.mjs
CHANGED
|
@@ -17126,6 +17126,7 @@ var EXPOSURE_SCOPE = "exposure";
|
|
|
17126
17126
|
// ../../packages/local-graph/src/change-feed.ts
|
|
17127
17127
|
var CHANGE_FEED_KEY = "change-feed";
|
|
17128
17128
|
var ANY_CHANGE = "*";
|
|
17129
|
+
var nodeScope = (label) => `node:${label}`;
|
|
17129
17130
|
var CHANGE_LOG_KEEP = 25e4;
|
|
17130
17131
|
var PRUNED_TO_KEY = "change-log-pruned-to";
|
|
17131
17132
|
function installChangeFeed(db) {
|
|
@@ -17596,8 +17597,10 @@ function column(c) {
|
|
|
17596
17597
|
function compileNodeWhere(q) {
|
|
17597
17598
|
if (q.labels.length === 0) throw new Error("selectNodes: no labels");
|
|
17598
17599
|
const params = [...q.labels];
|
|
17599
|
-
const
|
|
17600
|
-
|
|
17600
|
+
const byId = (q.where ?? []).some((w) => "idIn" in w);
|
|
17601
|
+
const unindexed = byId ? "+" : "";
|
|
17602
|
+
const conds = [`${unindexed}label IN (${q.labels.map(() => "?").join(", ")})`];
|
|
17603
|
+
if (!q.includeClosed) conds.push(`${unindexed}valid_to IS NULL`);
|
|
17601
17604
|
for (const w of q.where ?? []) {
|
|
17602
17605
|
if ("embedding" in w) {
|
|
17603
17606
|
conds.push(`${EMBEDDING_DIM_SQL} ${w.embedding === "present" ? ">" : "="} 0`);
|
|
@@ -17649,6 +17652,14 @@ function compileNodeWhere(q) {
|
|
|
17649
17652
|
function attrValueSql(p) {
|
|
17650
17653
|
return `CASE json_type(attrs_json, ${p}) WHEN 'integer' THEN CAST(json_extract(attrs_json, ${p}) AS REAL) ELSE json_extract(attrs_json, ${p}) END`;
|
|
17651
17654
|
}
|
|
17655
|
+
function nodeSelectFrom(q) {
|
|
17656
|
+
if (q.indexedBy) return `nodes INDEXED BY ${selectIndex(q.indexedBy)}`;
|
|
17657
|
+
const where = q.where ?? [];
|
|
17658
|
+
if (!q.includeClosed && where.some((w) => "descriptionIn" in w && w.descriptionIn.length > 0) && !where.some((w) => "idIn" in w)) {
|
|
17659
|
+
return "nodes INDEXED BY nodes_live_description";
|
|
17660
|
+
}
|
|
17661
|
+
return "nodes";
|
|
17662
|
+
}
|
|
17652
17663
|
function compileNodeSelect(q) {
|
|
17653
17664
|
const { where, params } = compileNodeWhere(q);
|
|
17654
17665
|
const attrs = q.attrs ?? [];
|
|
@@ -17666,7 +17677,7 @@ function compileNodeSelect(q) {
|
|
|
17666
17677
|
}
|
|
17667
17678
|
const orderParams = !q.orderBy && q.labels.length > 1 ? [...q.labels] : [];
|
|
17668
17679
|
const limit = q.limit !== void 0 ? ` LIMIT ${Math.max(0, Math.floor(q.limit))}` : "";
|
|
17669
|
-
const sql = `SELECT id, label, description, created_at, last_updated_at, page_rank, cumulative_hits, extraction_confidence, extraction_source, community, created_at_seq, last_reinforced_at_seq, valid_to, logical_id, version, ${EMBEDDING_DIM_SQL} AS embedding_dim${q.withEmbedding ? ", embedding_blob, embedding_json" : ""}${attrCols} FROM
|
|
17680
|
+
const sql = `SELECT id, label, description, created_at, last_updated_at, page_rank, cumulative_hits, extraction_confidence, extraction_source, community, created_at_seq, last_reinforced_at_seq, valid_to, logical_id, version, ${EMBEDDING_DIM_SQL} AS embedding_dim${q.withEmbedding ? ", embedding_blob, embedding_json" : ""}${attrCols} FROM ${nodeSelectFrom(q)} WHERE ${where} ORDER BY ${order}${limit}`;
|
|
17670
17681
|
return { sql, params: [...params, ...orderParams] };
|
|
17671
17682
|
}
|
|
17672
17683
|
function decodeAttr(value, type) {
|
|
@@ -17903,6 +17914,18 @@ CREATE INDEX IF NOT EXISTS edges_live_prior_tag ON edges(created_at) WHERE valid
|
|
|
17903
17914
|
-- incremental reindex can fetch just the changed files' symbols instead of
|
|
17904
17915
|
-- scanning every versioned node. Partial: only live rows are ever queried.
|
|
17905
17916
|
CREATE INDEX IF NOT EXISTS nodes_relpath ON nodes(json_extract(attrs_json, '$.relPath')) WHERE valid_to IS NULL;
|
|
17917
|
+
-- Symbols by qname and doc comments by the symbol they attach to: the drain's
|
|
17918
|
+
-- symbol lexicon resolves the names in the text it ships one at a time, instead
|
|
17919
|
+
-- of reading every Function/Method/Class/Interface/Const and Comment row on each
|
|
17920
|
+
-- live push (~300 MB of a 1.1 GB store per push, 2026-10-20).
|
|
17921
|
+
CREATE INDEX IF NOT EXISTS nodes_live_qname ON nodes(json_extract(attrs_json, '$.qname')) WHERE valid_to IS NULL;
|
|
17922
|
+
CREATE INDEX IF NOT EXISTS nodes_live_attached_to ON nodes(json_extract(attrs_json, '$.attachedTo')) WHERE valid_to IS NULL AND label = 'Comment';
|
|
17923
|
+
-- The attrs the cloud drain gates every instance row on, so it reads them from
|
|
17924
|
+
-- the index alone: pulled cloud rows are most of these labels (7,105 of 7,708
|
|
17925
|
+
-- Solutions on the live store) at ~10 KB of attrs each, and five full scans for
|
|
17926
|
+
-- two attrs read ~220 MB per push to find the few local rows that can ship.
|
|
17927
|
+
-- created_at leads so the index order is findNodeFieldsByLabel's.
|
|
17928
|
+
CREATE INDEX IF NOT EXISTS nodes_live_instance_gate ON nodes(label, created_at, json_extract(attrs_json, '$.source'), json_extract(attrs_json, '$.privateFinding'), json_extract(attrs_json, '$.resolvedAs'), id) WHERE valid_to IS NULL AND label IN ('Problem', 'Vulnerability', 'Solution', 'RootCause');
|
|
17906
17929
|
-- Closed rows by relPath, for reviveReobserved (every incremental reindex): the
|
|
17907
17930
|
-- live-only index above cannot serve it, so it scanned the whole table (~275 MB
|
|
17908
17931
|
-- of a 1.1 GB store per reindex, 2026-10-20).
|
|
@@ -18018,6 +18041,7 @@ var SqliteGraphStore = class {
|
|
|
18018
18041
|
initialPageRankScale;
|
|
18019
18042
|
constructor(opts) {
|
|
18020
18043
|
this.db = openDatabase(opts.path);
|
|
18044
|
+
this.db.exec("PRAGMA cache_size = -32768");
|
|
18021
18045
|
this.db.exec(SCHEMA_SQL);
|
|
18022
18046
|
const versionRow = this.db.prepare("SELECT version FROM schema_version LIMIT 1").get();
|
|
18023
18047
|
if (!versionRow) {
|
|
@@ -18082,6 +18106,7 @@ var SqliteGraphStore = class {
|
|
|
18082
18106
|
attrs_json = excluded.attrs_json
|
|
18083
18107
|
`),
|
|
18084
18108
|
getNode: this.db.prepare("SELECT * FROM nodes WHERE id = ?"),
|
|
18109
|
+
labelOf: this.db.prepare("SELECT label FROM nodes WHERE id = ?"),
|
|
18085
18110
|
// Single-column fast paths for the nightly pass — writing 100k pageRank
|
|
18086
18111
|
// values through the 18-column COALESCE updateNode took ~30s; these are ~10x
|
|
18087
18112
|
// cheaper (one bound column, no COALESCE evaluation per row).
|
|
@@ -18269,6 +18294,36 @@ var SqliteGraphStore = class {
|
|
|
18269
18294
|
}
|
|
18270
18295
|
return out;
|
|
18271
18296
|
}
|
|
18297
|
+
firstLiveByNameOrQname(labels, name) {
|
|
18298
|
+
if (labels.length === 0) return null;
|
|
18299
|
+
const inLabels = labels.map(() => "?").join(",");
|
|
18300
|
+
const probe = (index, match) => this.db.prepare(
|
|
18301
|
+
`SELECT rowid AS rid, created_at AS at, label, description, json_extract(attrs_json, '$.qname') AS qname FROM nodes INDEXED BY ${index}
|
|
18302
|
+
WHERE ${match} = ? AND label IN (${inLabels}) AND valid_to IS NULL`
|
|
18303
|
+
).all(name, ...labels);
|
|
18304
|
+
const rank = (r) => [labels.indexOf(r.label), r.at, r.rid];
|
|
18305
|
+
const before = (a, b) => a[0] < b[0] || a[0] === b[0] && (a[1] < b[1] || a[1] === b[1] && a[2] < b[2]);
|
|
18306
|
+
let best;
|
|
18307
|
+
for (const r of [...probe("nodes_live_description", "description"), ...probe("nodes_live_qname", "json_extract(attrs_json, '$.qname')")]) {
|
|
18308
|
+
if (!best || before(rank(r), rank(best))) best = r;
|
|
18309
|
+
}
|
|
18310
|
+
return best ? { label: best.label, description: best.description, qname: typeof best.qname === "string" ? best.qname : null } : null;
|
|
18311
|
+
}
|
|
18312
|
+
instanceGateRows(label) {
|
|
18313
|
+
return this.db.prepare(
|
|
18314
|
+
`SELECT id, json_extract(attrs_json, '$.source') AS source, json_extract(attrs_json, '$.privateFinding') AS privateFinding,
|
|
18315
|
+
json_extract(attrs_json, '$.resolvedAs') AS resolvedAs
|
|
18316
|
+
FROM nodes INDEXED BY nodes_live_instance_gate
|
|
18317
|
+
WHERE label IN ('Problem', 'Vulnerability', 'Solution', 'RootCause') AND label = ? AND valid_to IS NULL
|
|
18318
|
+
ORDER BY label, created_at, rowid`
|
|
18319
|
+
).all(label);
|
|
18320
|
+
}
|
|
18321
|
+
liveCommentsAttachedTo(name) {
|
|
18322
|
+
return this.db.prepare(
|
|
18323
|
+
`SELECT json_extract(attrs_json, '$.kind') AS kind, json_extract(attrs_json, '$.text') AS text, description FROM nodes INDEXED BY nodes_live_attached_to
|
|
18324
|
+
WHERE label = 'Comment' AND valid_to IS NULL AND json_extract(attrs_json, '$.attachedTo') = ? ORDER BY created_at, rowid`
|
|
18325
|
+
).all(name);
|
|
18326
|
+
}
|
|
18272
18327
|
strandedEdges() {
|
|
18273
18328
|
return countStrandedEdges(this.db);
|
|
18274
18329
|
}
|
|
@@ -18447,7 +18502,11 @@ var SqliteGraphStore = class {
|
|
|
18447
18502
|
if (!stmt2) {
|
|
18448
18503
|
const cols = attrKeys.map((k, i) => `, ${attrValueSql(`'$.${k}'`)} AS a${i}, json_type(attrs_json, '$.${k}') AS t${i}`).join("");
|
|
18449
18504
|
stmt2 = this.db.prepare(
|
|
18450
|
-
|
|
18505
|
+
// Ordered, not left to the planner: readers take "first row wins" (the
|
|
18506
|
+
// symbol lexicon's name winner and doc pick), and the order otherwise
|
|
18507
|
+
// followed whichever partial index the planner chose. nodes_live_created
|
|
18508
|
+
// serves this order without a sort.
|
|
18509
|
+
`SELECT id, description, community, extraction_confidence${cols} FROM nodes WHERE label = ? AND valid_to IS NULL ORDER BY created_at, rowid`
|
|
18451
18510
|
);
|
|
18452
18511
|
this.fieldStmts.set(cacheKey, stmt2);
|
|
18453
18512
|
}
|
|
@@ -18473,7 +18532,7 @@ var SqliteGraphStore = class {
|
|
|
18473
18532
|
}
|
|
18474
18533
|
countNodes(q) {
|
|
18475
18534
|
const { where, params } = compileNodeWhere(q);
|
|
18476
|
-
return Number(this.db.prepare(`SELECT COUNT(*) AS n FROM
|
|
18535
|
+
return Number(this.db.prepare(`SELECT COUNT(*) AS n FROM ${nodeSelectFrom(q)} WHERE ${where}`).get(...params).n);
|
|
18477
18536
|
}
|
|
18478
18537
|
setCommunity(id, community) {
|
|
18479
18538
|
this.stmts.setCommunity.run({ id, v: community });
|
|
@@ -18547,11 +18606,13 @@ var SqliteGraphStore = class {
|
|
|
18547
18606
|
}
|
|
18548
18607
|
}
|
|
18549
18608
|
edgeRuleViolation(edge) {
|
|
18550
|
-
return localEdgeViolation(
|
|
18551
|
-
|
|
18552
|
-
|
|
18553
|
-
|
|
18554
|
-
|
|
18609
|
+
return localEdgeViolation(this.labelOf(edge.from), edge.type, this.labelOf(edge.to));
|
|
18610
|
+
}
|
|
18611
|
+
/** An endpoint's label, any version, as `getNode(id)?.label` reads it. Every
|
|
18612
|
+
* edge write checks both endpoints, and hydrating them (attrs + vector) to
|
|
18613
|
+
* read one leading column was most of an incremental reindex's reads. */
|
|
18614
|
+
labelOf(id) {
|
|
18615
|
+
return this.stmts.labelOf.get(id)?.label;
|
|
18555
18616
|
}
|
|
18556
18617
|
updateEdge(id, patch) {
|
|
18557
18618
|
this.stmts.updateEdge.run({
|
|
@@ -18595,7 +18656,21 @@ var SqliteGraphStore = class {
|
|
|
18595
18656
|
* per project per request; the materializing form held the daemon's event
|
|
18596
18657
|
* loop for seconds at scale. */
|
|
18597
18658
|
countNodesByLabel(label) {
|
|
18598
|
-
return Number(this.stmts.countByLabel.get(label).n);
|
|
18659
|
+
return this.countAt(`label:${label}`, [nodeScope(label)], () => Number(this.stmts.countByLabel.get(label).n));
|
|
18660
|
+
}
|
|
18661
|
+
/** Counts asked for on every status poll (/health, `errata status`, the
|
|
18662
|
+
* dashboards): a live-node COUNT(*) reads the whole liveness index, ~3 MB per
|
|
18663
|
+
* call on a 1.1 GB store, to recount a set that moves only on a write. Each is
|
|
18664
|
+
* kept with the change-feed reading of the rows it counts — triggers bump it on
|
|
18665
|
+
* every write from any connection, so a hit is exact, not a TTL guess. */
|
|
18666
|
+
counts = /* @__PURE__ */ new Map();
|
|
18667
|
+
countAt(key, scopes, count) {
|
|
18668
|
+
const seq = readChangeSeq(this.db, scopes);
|
|
18669
|
+
const hit = this.counts.get(key);
|
|
18670
|
+
if (hit && hit.seq === seq) return hit.n;
|
|
18671
|
+
const n = count();
|
|
18672
|
+
this.counts.set(key, { seq, n });
|
|
18673
|
+
return n;
|
|
18599
18674
|
}
|
|
18600
18675
|
/** Live rows for a label with attrs and creation stamps only — for scans that
|
|
18601
18676
|
* read attrs and never the vector. `findNodesByLabel` decodes every embedding
|
|
@@ -18862,7 +18937,7 @@ var SqliteGraphStore = class {
|
|
|
18862
18937
|
return out;
|
|
18863
18938
|
}
|
|
18864
18939
|
nodeCount() {
|
|
18865
|
-
return Number(this.stmts.nodeCount.get().n);
|
|
18940
|
+
return this.countAt("nodes", [ANY_CHANGE], () => Number(this.stmts.nodeCount.get().n));
|
|
18866
18941
|
}
|
|
18867
18942
|
mutationCount() {
|
|
18868
18943
|
return this.mutations;
|
|
@@ -18906,7 +18981,7 @@ var SqliteGraphStore = class {
|
|
|
18906
18981
|
return this.ingestSeq;
|
|
18907
18982
|
}
|
|
18908
18983
|
edgeCount() {
|
|
18909
|
-
return Number(this.stmts.edgeCount.get().n);
|
|
18984
|
+
return this.countAt("edges", [ANY_CHANGE], () => Number(this.stmts.edgeCount.get().n));
|
|
18910
18985
|
}
|
|
18911
18986
|
// Lightweight whole-graph scans — select only the columns the caller needs so a
|
|
18912
18987
|
// 100k-node pass skips the per-row attrs JSON.parse + embedding decode rowToNode
|
|
@@ -20893,6 +20968,62 @@ function buildSymbolLexicon(store) {
|
|
|
20893
20968
|
}
|
|
20894
20969
|
return lex;
|
|
20895
20970
|
}
|
|
20971
|
+
function lazySymbolLexicon(store) {
|
|
20972
|
+
if (!store.firstLiveByNameOrQname || !store.liveCommentsAttachedTo) return null;
|
|
20973
|
+
const labels = SYMBOL_LABELS.map(([label]) => label);
|
|
20974
|
+
const kindOf = new Map(SYMBOL_LABELS.map(([label, kind]) => [label, kind]));
|
|
20975
|
+
const members = /* @__PURE__ */ new Map();
|
|
20976
|
+
const docs = /* @__PURE__ */ new Map();
|
|
20977
|
+
const entries = /* @__PURE__ */ new Map();
|
|
20978
|
+
const member = (name) => {
|
|
20979
|
+
if (!members.has(name)) {
|
|
20980
|
+
const hit = isDistinctiveIdentifier(name) ? store.firstLiveByNameOrQname(labels, name) : null;
|
|
20981
|
+
members.set(name, hit ? { kind: kindOf.get(hit.label), description: hit.description, qname: hit.qname } : null);
|
|
20982
|
+
}
|
|
20983
|
+
return members.get(name);
|
|
20984
|
+
};
|
|
20985
|
+
const docFor = (name) => {
|
|
20986
|
+
if (!docs.has(name)) {
|
|
20987
|
+
let found;
|
|
20988
|
+
for (const c of store.liveCommentsAttachedTo(name)) {
|
|
20989
|
+
if (typeof c.kind !== "string" || !DOC_COMMENT_KINDS.has(c.kind)) continue;
|
|
20990
|
+
const text = typeof c.text === "string" ? c.text : c.description;
|
|
20991
|
+
if (!text?.trim()) continue;
|
|
20992
|
+
found = text.trim().slice(0, MAX_DOC_CHARS);
|
|
20993
|
+
break;
|
|
20994
|
+
}
|
|
20995
|
+
docs.set(name, found);
|
|
20996
|
+
}
|
|
20997
|
+
return docs.get(name);
|
|
20998
|
+
};
|
|
20999
|
+
const isKnown = (tok) => member(tok) !== null;
|
|
21000
|
+
let anySymbol;
|
|
21001
|
+
return {
|
|
21002
|
+
has: isKnown,
|
|
21003
|
+
get(name) {
|
|
21004
|
+
if (!entries.has(name)) {
|
|
21005
|
+
const m = member(name);
|
|
21006
|
+
let entry = null;
|
|
21007
|
+
if (m) {
|
|
21008
|
+
entry = { kind: m.kind };
|
|
21009
|
+
const doc = (m.description ? docFor(m.description) : void 0) ?? (m.qname ? docFor(m.qname) : void 0);
|
|
21010
|
+
for (const phrase of summaryCandidates(name, m.kind, doc)) {
|
|
21011
|
+
if (summaryRejectionReason(phrase, isKnown) === null) {
|
|
21012
|
+
entry.summary = phrase;
|
|
21013
|
+
break;
|
|
21014
|
+
}
|
|
21015
|
+
}
|
|
21016
|
+
}
|
|
21017
|
+
entries.set(name, entry);
|
|
21018
|
+
}
|
|
21019
|
+
return entries.get(name) ?? void 0;
|
|
21020
|
+
},
|
|
21021
|
+
get size() {
|
|
21022
|
+
anySymbol ??= labels.some((label) => store.countNodesByLabel(label) > 0);
|
|
21023
|
+
return anySymbol ? 1 : 0;
|
|
21024
|
+
}
|
|
21025
|
+
};
|
|
21026
|
+
}
|
|
20896
21027
|
var LEXICON_SYMBOL_LABELS = ["Function", "Method", "Class", "Interface", "Const", "Comment"];
|
|
20897
21028
|
function codeGraphKey(store) {
|
|
20898
21029
|
const files = store.labelFingerprint?.(["File"]);
|
|
@@ -20904,7 +21035,7 @@ function lexiconsFor(store) {
|
|
|
20904
21035
|
const key = codeGraphKey(store);
|
|
20905
21036
|
const hit = key !== null ? lexiconCache.get(store) : void 0;
|
|
20906
21037
|
if (hit && hit.key === key) return hit;
|
|
20907
|
-
const built = { symbols: buildSymbolLexicon(store), files: buildFileLexicon(store) };
|
|
21038
|
+
const built = { symbols: lazySymbolLexicon(store) ?? buildSymbolLexicon(store), files: buildFileLexicon(store) };
|
|
20908
21039
|
if (key !== null) lexiconCache.set(store, { key, ...built });
|
|
20909
21040
|
return built;
|
|
20910
21041
|
}
|
|
@@ -21143,6 +21274,38 @@ function stripCodebaseScope(scope) {
|
|
|
21143
21274
|
delete s["codebase"];
|
|
21144
21275
|
return s;
|
|
21145
21276
|
}
|
|
21277
|
+
function instanceGateRows(store, label) {
|
|
21278
|
+
return store.instanceGateRows?.(label) ?? store.findNodeFieldsByLabel(label, ["source", "privateFinding", "resolvedAs"]).map((r) => ({
|
|
21279
|
+
id: r.id,
|
|
21280
|
+
source: r.attrs["source"],
|
|
21281
|
+
privateFinding: r.attrs["privateFinding"],
|
|
21282
|
+
resolvedAs: r.attrs["resolvedAs"]
|
|
21283
|
+
}));
|
|
21284
|
+
}
|
|
21285
|
+
var GATE_VIEW_ATTRS = [
|
|
21286
|
+
"privateFinding",
|
|
21287
|
+
"source",
|
|
21288
|
+
"sources",
|
|
21289
|
+
"resolvedAs",
|
|
21290
|
+
"instanceReceipts",
|
|
21291
|
+
"cloudWithdrawn",
|
|
21292
|
+
"withdrawnAtCorrectionSeq",
|
|
21293
|
+
"contributedAtSeq",
|
|
21294
|
+
"cloudNodeId"
|
|
21295
|
+
];
|
|
21296
|
+
var GATE_VIEW_CHUNK = 500;
|
|
21297
|
+
var gateView = /* @__PURE__ */ new WeakSet();
|
|
21298
|
+
function gateViews(store, label, ids) {
|
|
21299
|
+
const byId = /* @__PURE__ */ new Map();
|
|
21300
|
+
for (let i = 0; i < ids.length; i += GATE_VIEW_CHUNK) {
|
|
21301
|
+
for (const r of store.selectNodes({ labels: [label], attrs: GATE_VIEW_ATTRS, where: [{ idIn: ids.slice(i, i + GATE_VIEW_CHUNK) }] })) {
|
|
21302
|
+
const view = { ...r, embedding: [] };
|
|
21303
|
+
gateView.add(view);
|
|
21304
|
+
byId.set(r.id, view);
|
|
21305
|
+
}
|
|
21306
|
+
}
|
|
21307
|
+
return ids.flatMap((id) => byId.get(id) ?? []);
|
|
21308
|
+
}
|
|
21146
21309
|
function contributedWireId(n, destinationKey) {
|
|
21147
21310
|
const cloudId = instanceReceipt(n, destinationKey)["cloudNodeId"];
|
|
21148
21311
|
return typeof cloudId === "string" && cloudId ? cloudId : n.id;
|
|
@@ -21301,7 +21464,7 @@ function buildInstanceIngest(store, profile, daemonVersion, ignorePatterns = [],
|
|
|
21301
21464
|
edgesHeldDropped: 0
|
|
21302
21465
|
};
|
|
21303
21466
|
const sharedCauses = /* @__PURE__ */ new Map();
|
|
21304
|
-
const causalQueue = store
|
|
21467
|
+
const causalQueue = instanceGateRows(store, "Problem").filter((r) => r.source !== "cloud" && r.resolvedAs !== PROBLEM_RESOLUTION.FALSE_POSITIVE).map((r) => r.id).filter((id) => ignorePatterns.length === 0 || !ignored(store.getNode(id)?.description ?? ""));
|
|
21305
21468
|
const causalSeen = new Set(causalQueue);
|
|
21306
21469
|
for (let i = 0; i < causalQueue.length; i++) {
|
|
21307
21470
|
const id = causalQueue[i];
|
|
@@ -21317,21 +21480,21 @@ function buildInstanceIngest(store, profile, daemonVersion, ignorePatterns = [],
|
|
|
21317
21480
|
for (const label of INSTANCE_LABELS) {
|
|
21318
21481
|
const ref = [];
|
|
21319
21482
|
includedByLabel.set(label, ref);
|
|
21320
|
-
const
|
|
21321
|
-
for (const row of store
|
|
21322
|
-
if (row.
|
|
21323
|
-
|
|
21324
|
-
if (n) local.push(n);
|
|
21483
|
+
const localIds = [];
|
|
21484
|
+
for (const row of instanceGateRows(store, label)) {
|
|
21485
|
+
if (row.source !== "cloud") {
|
|
21486
|
+
localIds.push(row.id);
|
|
21325
21487
|
continue;
|
|
21326
21488
|
}
|
|
21327
21489
|
if (seen.has(row.id) || walked.has(row.id)) continue;
|
|
21328
21490
|
walked.add(row.id);
|
|
21329
21491
|
composition.candidates++;
|
|
21330
|
-
if ((row.
|
|
21492
|
+
if ((row.privateFinding || label === "Vulnerability") && !opts.project) composition.privateTargetMissing++;
|
|
21331
21493
|
else composition.cloudSourced++;
|
|
21332
21494
|
}
|
|
21495
|
+
const local = gateViews(store, label, localIds);
|
|
21333
21496
|
const population = label === "RootCause" && opts.sharedStore ? [...local, ...sharedCauses.values()] : local;
|
|
21334
|
-
for (
|
|
21497
|
+
for (let n of population) {
|
|
21335
21498
|
if (seen.has(n.id)) continue;
|
|
21336
21499
|
if (walked.has(n.id)) continue;
|
|
21337
21500
|
walked.add(n.id);
|
|
@@ -21367,6 +21530,11 @@ function buildInstanceIngest(store, profile, daemonVersion, ignorePatterns = [],
|
|
|
21367
21530
|
semanticEndpoints.set(n.id, contributedWireId(n, opts.destinationKey));
|
|
21368
21531
|
continue;
|
|
21369
21532
|
}
|
|
21533
|
+
if (gateView.has(n)) {
|
|
21534
|
+
const full = store.getNode(n.id);
|
|
21535
|
+
if (!full) continue;
|
|
21536
|
+
n = full;
|
|
21537
|
+
}
|
|
21370
21538
|
if (isEditPlaceholderSolution(n)) {
|
|
21371
21539
|
composition.autoMint++;
|
|
21372
21540
|
continue;
|
package/errata.mjs
CHANGED
|
@@ -20393,8 +20393,10 @@ function column(c) {
|
|
|
20393
20393
|
function compileNodeWhere(q) {
|
|
20394
20394
|
if (q.labels.length === 0) throw new Error("selectNodes: no labels");
|
|
20395
20395
|
const params = [...q.labels];
|
|
20396
|
-
const
|
|
20397
|
-
|
|
20396
|
+
const byId = (q.where ?? []).some((w) => "idIn" in w);
|
|
20397
|
+
const unindexed = byId ? "+" : "";
|
|
20398
|
+
const conds = [`${unindexed}label IN (${q.labels.map(() => "?").join(", ")})`];
|
|
20399
|
+
if (!q.includeClosed) conds.push(`${unindexed}valid_to IS NULL`);
|
|
20398
20400
|
for (const w of q.where ?? []) {
|
|
20399
20401
|
if ("embedding" in w) {
|
|
20400
20402
|
conds.push(`${EMBEDDING_DIM_SQL} ${w.embedding === "present" ? ">" : "="} 0`);
|
|
@@ -20446,6 +20448,14 @@ function compileNodeWhere(q) {
|
|
|
20446
20448
|
function attrValueSql(p) {
|
|
20447
20449
|
return `CASE json_type(attrs_json, ${p}) WHEN 'integer' THEN CAST(json_extract(attrs_json, ${p}) AS REAL) ELSE json_extract(attrs_json, ${p}) END`;
|
|
20448
20450
|
}
|
|
20451
|
+
function nodeSelectFrom(q) {
|
|
20452
|
+
if (q.indexedBy) return `nodes INDEXED BY ${selectIndex(q.indexedBy)}`;
|
|
20453
|
+
const where = q.where ?? [];
|
|
20454
|
+
if (!q.includeClosed && where.some((w) => "descriptionIn" in w && w.descriptionIn.length > 0) && !where.some((w) => "idIn" in w)) {
|
|
20455
|
+
return "nodes INDEXED BY nodes_live_description";
|
|
20456
|
+
}
|
|
20457
|
+
return "nodes";
|
|
20458
|
+
}
|
|
20449
20459
|
function compileNodeSelect(q) {
|
|
20450
20460
|
const { where, params } = compileNodeWhere(q);
|
|
20451
20461
|
const attrs = q.attrs ?? [];
|
|
@@ -20463,7 +20473,7 @@ function compileNodeSelect(q) {
|
|
|
20463
20473
|
}
|
|
20464
20474
|
const orderParams = !q.orderBy && q.labels.length > 1 ? [...q.labels] : [];
|
|
20465
20475
|
const limit = q.limit !== void 0 ? ` LIMIT ${Math.max(0, Math.floor(q.limit))}` : "";
|
|
20466
|
-
const sql = `SELECT id, label, description, created_at, last_updated_at, page_rank, cumulative_hits, extraction_confidence, extraction_source, community, created_at_seq, last_reinforced_at_seq, valid_to, logical_id, version, ${EMBEDDING_DIM_SQL} AS embedding_dim${q.withEmbedding ? ", embedding_blob, embedding_json" : ""}${attrCols} FROM
|
|
20476
|
+
const sql = `SELECT id, label, description, created_at, last_updated_at, page_rank, cumulative_hits, extraction_confidence, extraction_source, community, created_at_seq, last_reinforced_at_seq, valid_to, logical_id, version, ${EMBEDDING_DIM_SQL} AS embedding_dim${q.withEmbedding ? ", embedding_blob, embedding_json" : ""}${attrCols} FROM ${nodeSelectFrom(q)} WHERE ${where} ORDER BY ${order}${limit}`;
|
|
20467
20477
|
return { sql, params: [...params, ...orderParams] };
|
|
20468
20478
|
}
|
|
20469
20479
|
function decodeAttr(value, type) {
|
|
@@ -20812,6 +20822,18 @@ CREATE INDEX IF NOT EXISTS edges_live_prior_tag ON edges(created_at) WHERE valid
|
|
|
20812
20822
|
-- incremental reindex can fetch just the changed files' symbols instead of
|
|
20813
20823
|
-- scanning every versioned node. Partial: only live rows are ever queried.
|
|
20814
20824
|
CREATE INDEX IF NOT EXISTS nodes_relpath ON nodes(json_extract(attrs_json, '$.relPath')) WHERE valid_to IS NULL;
|
|
20825
|
+
-- Symbols by qname and doc comments by the symbol they attach to: the drain's
|
|
20826
|
+
-- symbol lexicon resolves the names in the text it ships one at a time, instead
|
|
20827
|
+
-- of reading every Function/Method/Class/Interface/Const and Comment row on each
|
|
20828
|
+
-- live push (~300 MB of a 1.1 GB store per push, 2026-10-20).
|
|
20829
|
+
CREATE INDEX IF NOT EXISTS nodes_live_qname ON nodes(json_extract(attrs_json, '$.qname')) WHERE valid_to IS NULL;
|
|
20830
|
+
CREATE INDEX IF NOT EXISTS nodes_live_attached_to ON nodes(json_extract(attrs_json, '$.attachedTo')) WHERE valid_to IS NULL AND label = 'Comment';
|
|
20831
|
+
-- The attrs the cloud drain gates every instance row on, so it reads them from
|
|
20832
|
+
-- the index alone: pulled cloud rows are most of these labels (7,105 of 7,708
|
|
20833
|
+
-- Solutions on the live store) at ~10 KB of attrs each, and five full scans for
|
|
20834
|
+
-- two attrs read ~220 MB per push to find the few local rows that can ship.
|
|
20835
|
+
-- created_at leads so the index order is findNodeFieldsByLabel's.
|
|
20836
|
+
CREATE INDEX IF NOT EXISTS nodes_live_instance_gate ON nodes(label, created_at, json_extract(attrs_json, '$.source'), json_extract(attrs_json, '$.privateFinding'), json_extract(attrs_json, '$.resolvedAs'), id) WHERE valid_to IS NULL AND label IN ('Problem', 'Vulnerability', 'Solution', 'RootCause');
|
|
20815
20837
|
-- Closed rows by relPath, for reviveReobserved (every incremental reindex): the
|
|
20816
20838
|
-- live-only index above cannot serve it, so it scanned the whole table (~275 MB
|
|
20817
20839
|
-- of a 1.1 GB store per reindex, 2026-10-20).
|
|
@@ -20870,6 +20892,7 @@ CREATE INDEX IF NOT EXISTS nodes_revisit_true ON nodes(valid_from) WHERE json_ty
|
|
|
20870
20892
|
initialPageRankScale;
|
|
20871
20893
|
constructor(opts) {
|
|
20872
20894
|
this.db = openDatabase(opts.path);
|
|
20895
|
+
this.db.exec("PRAGMA cache_size = -32768");
|
|
20873
20896
|
this.db.exec(SCHEMA_SQL);
|
|
20874
20897
|
const versionRow = this.db.prepare("SELECT version FROM schema_version LIMIT 1").get();
|
|
20875
20898
|
if (!versionRow) {
|
|
@@ -20934,6 +20957,7 @@ CREATE INDEX IF NOT EXISTS nodes_revisit_true ON nodes(valid_from) WHERE json_ty
|
|
|
20934
20957
|
attrs_json = excluded.attrs_json
|
|
20935
20958
|
`),
|
|
20936
20959
|
getNode: this.db.prepare("SELECT * FROM nodes WHERE id = ?"),
|
|
20960
|
+
labelOf: this.db.prepare("SELECT label FROM nodes WHERE id = ?"),
|
|
20937
20961
|
// Single-column fast paths for the nightly pass — writing 100k pageRank
|
|
20938
20962
|
// values through the 18-column COALESCE updateNode took ~30s; these are ~10x
|
|
20939
20963
|
// cheaper (one bound column, no COALESCE evaluation per row).
|
|
@@ -21121,6 +21145,36 @@ CREATE INDEX IF NOT EXISTS nodes_revisit_true ON nodes(valid_from) WHERE json_ty
|
|
|
21121
21145
|
}
|
|
21122
21146
|
return out2;
|
|
21123
21147
|
}
|
|
21148
|
+
firstLiveByNameOrQname(labels, name2) {
|
|
21149
|
+
if (labels.length === 0) return null;
|
|
21150
|
+
const inLabels = labels.map(() => "?").join(",");
|
|
21151
|
+
const probe = (index, match2) => this.db.prepare(
|
|
21152
|
+
`SELECT rowid AS rid, created_at AS at, label, description, json_extract(attrs_json, '$.qname') AS qname FROM nodes INDEXED BY ${index}
|
|
21153
|
+
WHERE ${match2} = ? AND label IN (${inLabels}) AND valid_to IS NULL`
|
|
21154
|
+
).all(name2, ...labels);
|
|
21155
|
+
const rank = (r) => [labels.indexOf(r.label), r.at, r.rid];
|
|
21156
|
+
const before = (a, b) => a[0] < b[0] || a[0] === b[0] && (a[1] < b[1] || a[1] === b[1] && a[2] < b[2]);
|
|
21157
|
+
let best;
|
|
21158
|
+
for (const r of [...probe("nodes_live_description", "description"), ...probe("nodes_live_qname", "json_extract(attrs_json, '$.qname')")]) {
|
|
21159
|
+
if (!best || before(rank(r), rank(best))) best = r;
|
|
21160
|
+
}
|
|
21161
|
+
return best ? { label: best.label, description: best.description, qname: typeof best.qname === "string" ? best.qname : null } : null;
|
|
21162
|
+
}
|
|
21163
|
+
instanceGateRows(label) {
|
|
21164
|
+
return this.db.prepare(
|
|
21165
|
+
`SELECT id, json_extract(attrs_json, '$.source') AS source, json_extract(attrs_json, '$.privateFinding') AS privateFinding,
|
|
21166
|
+
json_extract(attrs_json, '$.resolvedAs') AS resolvedAs
|
|
21167
|
+
FROM nodes INDEXED BY nodes_live_instance_gate
|
|
21168
|
+
WHERE label IN ('Problem', 'Vulnerability', 'Solution', 'RootCause') AND label = ? AND valid_to IS NULL
|
|
21169
|
+
ORDER BY label, created_at, rowid`
|
|
21170
|
+
).all(label);
|
|
21171
|
+
}
|
|
21172
|
+
liveCommentsAttachedTo(name2) {
|
|
21173
|
+
return this.db.prepare(
|
|
21174
|
+
`SELECT json_extract(attrs_json, '$.kind') AS kind, json_extract(attrs_json, '$.text') AS text, description FROM nodes INDEXED BY nodes_live_attached_to
|
|
21175
|
+
WHERE label = 'Comment' AND valid_to IS NULL AND json_extract(attrs_json, '$.attachedTo') = ? ORDER BY created_at, rowid`
|
|
21176
|
+
).all(name2);
|
|
21177
|
+
}
|
|
21124
21178
|
strandedEdges() {
|
|
21125
21179
|
return countStrandedEdges(this.db);
|
|
21126
21180
|
}
|
|
@@ -21299,7 +21353,11 @@ CREATE INDEX IF NOT EXISTS nodes_revisit_true ON nodes(valid_from) WHERE json_ty
|
|
|
21299
21353
|
if (!stmt2) {
|
|
21300
21354
|
const cols = attrKeys.map((k, i2) => `, ${attrValueSql(`'$.${k}'`)} AS a${i2}, json_type(attrs_json, '$.${k}') AS t${i2}`).join("");
|
|
21301
21355
|
stmt2 = this.db.prepare(
|
|
21302
|
-
|
|
21356
|
+
// Ordered, not left to the planner: readers take "first row wins" (the
|
|
21357
|
+
// symbol lexicon's name winner and doc pick), and the order otherwise
|
|
21358
|
+
// followed whichever partial index the planner chose. nodes_live_created
|
|
21359
|
+
// serves this order without a sort.
|
|
21360
|
+
`SELECT id, description, community, extraction_confidence${cols} FROM nodes WHERE label = ? AND valid_to IS NULL ORDER BY created_at, rowid`
|
|
21303
21361
|
);
|
|
21304
21362
|
this.fieldStmts.set(cacheKey2, stmt2);
|
|
21305
21363
|
}
|
|
@@ -21325,7 +21383,7 @@ CREATE INDEX IF NOT EXISTS nodes_revisit_true ON nodes(valid_from) WHERE json_ty
|
|
|
21325
21383
|
}
|
|
21326
21384
|
countNodes(q) {
|
|
21327
21385
|
const { where, params } = compileNodeWhere(q);
|
|
21328
|
-
return Number(this.db.prepare(`SELECT COUNT(*) AS n FROM
|
|
21386
|
+
return Number(this.db.prepare(`SELECT COUNT(*) AS n FROM ${nodeSelectFrom(q)} WHERE ${where}`).get(...params).n);
|
|
21329
21387
|
}
|
|
21330
21388
|
setCommunity(id, community) {
|
|
21331
21389
|
this.stmts.setCommunity.run({ id, v: community });
|
|
@@ -21399,11 +21457,13 @@ CREATE INDEX IF NOT EXISTS nodes_revisit_true ON nodes(valid_from) WHERE json_ty
|
|
|
21399
21457
|
}
|
|
21400
21458
|
}
|
|
21401
21459
|
edgeRuleViolation(edge2) {
|
|
21402
|
-
return localEdgeViolation(
|
|
21403
|
-
|
|
21404
|
-
|
|
21405
|
-
|
|
21406
|
-
|
|
21460
|
+
return localEdgeViolation(this.labelOf(edge2.from), edge2.type, this.labelOf(edge2.to));
|
|
21461
|
+
}
|
|
21462
|
+
/** An endpoint's label, any version, as `getNode(id)?.label` reads it. Every
|
|
21463
|
+
* edge write checks both endpoints, and hydrating them (attrs + vector) to
|
|
21464
|
+
* read one leading column was most of an incremental reindex's reads. */
|
|
21465
|
+
labelOf(id) {
|
|
21466
|
+
return this.stmts.labelOf.get(id)?.label;
|
|
21407
21467
|
}
|
|
21408
21468
|
updateEdge(id, patch) {
|
|
21409
21469
|
this.stmts.updateEdge.run({
|
|
@@ -21447,7 +21507,21 @@ CREATE INDEX IF NOT EXISTS nodes_revisit_true ON nodes(valid_from) WHERE json_ty
|
|
|
21447
21507
|
* per project per request; the materializing form held the daemon's event
|
|
21448
21508
|
* loop for seconds at scale. */
|
|
21449
21509
|
countNodesByLabel(label) {
|
|
21450
|
-
return Number(this.stmts.countByLabel.get(label).n);
|
|
21510
|
+
return this.countAt(`label:${label}`, [nodeScope(label)], () => Number(this.stmts.countByLabel.get(label).n));
|
|
21511
|
+
}
|
|
21512
|
+
/** Counts asked for on every status poll (/health, `errata status`, the
|
|
21513
|
+
* dashboards): a live-node COUNT(*) reads the whole liveness index, ~3 MB per
|
|
21514
|
+
* call on a 1.1 GB store, to recount a set that moves only on a write. Each is
|
|
21515
|
+
* kept with the change-feed reading of the rows it counts — triggers bump it on
|
|
21516
|
+
* every write from any connection, so a hit is exact, not a TTL guess. */
|
|
21517
|
+
counts = /* @__PURE__ */ new Map();
|
|
21518
|
+
countAt(key, scopes, count) {
|
|
21519
|
+
const seq = readChangeSeq(this.db, scopes);
|
|
21520
|
+
const hit = this.counts.get(key);
|
|
21521
|
+
if (hit && hit.seq === seq) return hit.n;
|
|
21522
|
+
const n = count();
|
|
21523
|
+
this.counts.set(key, { seq, n });
|
|
21524
|
+
return n;
|
|
21451
21525
|
}
|
|
21452
21526
|
/** Live rows for a label with attrs and creation stamps only — for scans that
|
|
21453
21527
|
* read attrs and never the vector. `findNodesByLabel` decodes every embedding
|
|
@@ -21714,7 +21788,7 @@ CREATE INDEX IF NOT EXISTS nodes_revisit_true ON nodes(valid_from) WHERE json_ty
|
|
|
21714
21788
|
return out2;
|
|
21715
21789
|
}
|
|
21716
21790
|
nodeCount() {
|
|
21717
|
-
return Number(this.stmts.nodeCount.get().n);
|
|
21791
|
+
return this.countAt("nodes", [ANY_CHANGE], () => Number(this.stmts.nodeCount.get().n));
|
|
21718
21792
|
}
|
|
21719
21793
|
mutationCount() {
|
|
21720
21794
|
return this.mutations;
|
|
@@ -21758,7 +21832,7 @@ CREATE INDEX IF NOT EXISTS nodes_revisit_true ON nodes(valid_from) WHERE json_ty
|
|
|
21758
21832
|
return this.ingestSeq;
|
|
21759
21833
|
}
|
|
21760
21834
|
edgeCount() {
|
|
21761
|
-
return Number(this.stmts.edgeCount.get().n);
|
|
21835
|
+
return this.countAt("edges", [ANY_CHANGE], () => Number(this.stmts.edgeCount.get().n));
|
|
21762
21836
|
}
|
|
21763
21837
|
// Lightweight whole-graph scans — select only the columns the caller needs so a
|
|
21764
21838
|
// 100k-node pass skips the per-row attrs JSON.parse + embedding decode rowToNode
|
|
@@ -37302,6 +37376,62 @@ function buildSymbolLexicon(store) {
|
|
|
37302
37376
|
}
|
|
37303
37377
|
return lex;
|
|
37304
37378
|
}
|
|
37379
|
+
function lazySymbolLexicon(store) {
|
|
37380
|
+
if (!store.firstLiveByNameOrQname || !store.liveCommentsAttachedTo) return null;
|
|
37381
|
+
const labels = SYMBOL_LABELS2.map(([label]) => label);
|
|
37382
|
+
const kindOf = new Map(SYMBOL_LABELS2.map(([label, kind]) => [label, kind]));
|
|
37383
|
+
const members = /* @__PURE__ */ new Map();
|
|
37384
|
+
const docs = /* @__PURE__ */ new Map();
|
|
37385
|
+
const entries = /* @__PURE__ */ new Map();
|
|
37386
|
+
const member = (name2) => {
|
|
37387
|
+
if (!members.has(name2)) {
|
|
37388
|
+
const hit = isDistinctiveIdentifier(name2) ? store.firstLiveByNameOrQname(labels, name2) : null;
|
|
37389
|
+
members.set(name2, hit ? { kind: kindOf.get(hit.label), description: hit.description, qname: hit.qname } : null);
|
|
37390
|
+
}
|
|
37391
|
+
return members.get(name2);
|
|
37392
|
+
};
|
|
37393
|
+
const docFor = (name2) => {
|
|
37394
|
+
if (!docs.has(name2)) {
|
|
37395
|
+
let found;
|
|
37396
|
+
for (const c of store.liveCommentsAttachedTo(name2)) {
|
|
37397
|
+
if (typeof c.kind !== "string" || !DOC_COMMENT_KINDS.has(c.kind)) continue;
|
|
37398
|
+
const text = typeof c.text === "string" ? c.text : c.description;
|
|
37399
|
+
if (!text?.trim()) continue;
|
|
37400
|
+
found = text.trim().slice(0, MAX_DOC_CHARS);
|
|
37401
|
+
break;
|
|
37402
|
+
}
|
|
37403
|
+
docs.set(name2, found);
|
|
37404
|
+
}
|
|
37405
|
+
return docs.get(name2);
|
|
37406
|
+
};
|
|
37407
|
+
const isKnown = (tok) => member(tok) !== null;
|
|
37408
|
+
let anySymbol;
|
|
37409
|
+
return {
|
|
37410
|
+
has: isKnown,
|
|
37411
|
+
get(name2) {
|
|
37412
|
+
if (!entries.has(name2)) {
|
|
37413
|
+
const m = member(name2);
|
|
37414
|
+
let entry = null;
|
|
37415
|
+
if (m) {
|
|
37416
|
+
entry = { kind: m.kind };
|
|
37417
|
+
const doc = (m.description ? docFor(m.description) : void 0) ?? (m.qname ? docFor(m.qname) : void 0);
|
|
37418
|
+
for (const phrase2 of summaryCandidates(name2, m.kind, doc)) {
|
|
37419
|
+
if (summaryRejectionReason(phrase2, isKnown) === null) {
|
|
37420
|
+
entry.summary = phrase2;
|
|
37421
|
+
break;
|
|
37422
|
+
}
|
|
37423
|
+
}
|
|
37424
|
+
}
|
|
37425
|
+
entries.set(name2, entry);
|
|
37426
|
+
}
|
|
37427
|
+
return entries.get(name2) ?? void 0;
|
|
37428
|
+
},
|
|
37429
|
+
get size() {
|
|
37430
|
+
anySymbol ??= labels.some((label) => store.countNodesByLabel(label) > 0);
|
|
37431
|
+
return anySymbol ? 1 : 0;
|
|
37432
|
+
}
|
|
37433
|
+
};
|
|
37434
|
+
}
|
|
37305
37435
|
function codeGraphKey(store) {
|
|
37306
37436
|
const files = store.labelFingerprint?.(["File"]);
|
|
37307
37437
|
if (files === void 0) return null;
|
|
@@ -37311,7 +37441,7 @@ function lexiconsFor(store) {
|
|
|
37311
37441
|
const key = codeGraphKey(store);
|
|
37312
37442
|
const hit = key !== null ? lexiconCache.get(store) : void 0;
|
|
37313
37443
|
if (hit && hit.key === key) return hit;
|
|
37314
|
-
const built = { symbols: buildSymbolLexicon(store), files: buildFileLexicon(store) };
|
|
37444
|
+
const built = { symbols: lazySymbolLexicon(store) ?? buildSymbolLexicon(store), files: buildFileLexicon(store) };
|
|
37315
37445
|
if (key !== null) lexiconCache.set(store, { key, ...built });
|
|
37316
37446
|
return built;
|
|
37317
37447
|
}
|
|
@@ -39491,9 +39621,10 @@ async function runIndexer(store, opts) {
|
|
|
39491
39621
|
if (ei > 0) await new Promise((r) => setImmediate(r));
|
|
39492
39622
|
const slice = pendingEdges.slice(ei, ei + EDGE_BATCH);
|
|
39493
39623
|
store.transaction(() => {
|
|
39624
|
+
const present = store.nodeLabels([...new Set(slice.flatMap((e) => [e.from, e.to]))]);
|
|
39494
39625
|
for (const e of slice) {
|
|
39495
39626
|
if (e.from === e.to) continue;
|
|
39496
|
-
if (!
|
|
39627
|
+
if (!present.has(e.from) || !present.has(e.to)) continue;
|
|
39497
39628
|
upsertEdge(
|
|
39498
39629
|
store,
|
|
39499
39630
|
e.from,
|
|
@@ -86894,7 +87025,7 @@ function createWatchBreaker(deps) {
|
|
|
86894
87025
|
}
|
|
86895
87026
|
|
|
86896
87027
|
// src/engine.ts
|
|
86897
|
-
var DAEMON_VERSION = true ? "2.0.22-dev.
|
|
87028
|
+
var DAEMON_VERSION = true ? "2.0.22-dev.4657" : "2.0.0-alpha.0";
|
|
86898
87029
|
var IGNORED_PATH = /[\\/](?:\.git|node_modules|\.errata|\.claude|\.codex|\.cursor|\.turbo|\.next|dist|coverage|test-results|playwright-report|__pycache__)(?:[\\/]|$)/;
|
|
86899
87030
|
var IGNORED_NOISE = /castalia\.db|eventlog\.sqlite|turn-cursors/;
|
|
86900
87031
|
var GIT_OP_MUTE_MS = 4e3;
|
|
@@ -90732,6 +90863,38 @@ function compositionCounters(c) {
|
|
|
90732
90863
|
...Object.fromEntries(buckets.map((b) => [`sidecarDrop_${b}`, sidecarDropReasons[b] ?? 0]))
|
|
90733
90864
|
};
|
|
90734
90865
|
}
|
|
90866
|
+
function instanceGateRows(store, label) {
|
|
90867
|
+
return store.instanceGateRows?.(label) ?? store.findNodeFieldsByLabel(label, ["source", "privateFinding", "resolvedAs"]).map((r) => ({
|
|
90868
|
+
id: r.id,
|
|
90869
|
+
source: r.attrs["source"],
|
|
90870
|
+
privateFinding: r.attrs["privateFinding"],
|
|
90871
|
+
resolvedAs: r.attrs["resolvedAs"]
|
|
90872
|
+
}));
|
|
90873
|
+
}
|
|
90874
|
+
var GATE_VIEW_ATTRS = [
|
|
90875
|
+
"privateFinding",
|
|
90876
|
+
"source",
|
|
90877
|
+
"sources",
|
|
90878
|
+
"resolvedAs",
|
|
90879
|
+
"instanceReceipts",
|
|
90880
|
+
"cloudWithdrawn",
|
|
90881
|
+
"withdrawnAtCorrectionSeq",
|
|
90882
|
+
"contributedAtSeq",
|
|
90883
|
+
"cloudNodeId"
|
|
90884
|
+
];
|
|
90885
|
+
var GATE_VIEW_CHUNK = 500;
|
|
90886
|
+
var gateView = /* @__PURE__ */ new WeakSet();
|
|
90887
|
+
function gateViews(store, label, ids) {
|
|
90888
|
+
const byId = /* @__PURE__ */ new Map();
|
|
90889
|
+
for (let i2 = 0; i2 < ids.length; i2 += GATE_VIEW_CHUNK) {
|
|
90890
|
+
for (const r of store.selectNodes({ labels: [label], attrs: GATE_VIEW_ATTRS, where: [{ idIn: ids.slice(i2, i2 + GATE_VIEW_CHUNK) }] })) {
|
|
90891
|
+
const view = { ...r, embedding: [] };
|
|
90892
|
+
gateView.add(view);
|
|
90893
|
+
byId.set(r.id, view);
|
|
90894
|
+
}
|
|
90895
|
+
}
|
|
90896
|
+
return ids.flatMap((id) => byId.get(id) ?? []);
|
|
90897
|
+
}
|
|
90735
90898
|
function contributedWireId(n, destinationKey) {
|
|
90736
90899
|
const cloudId = instanceReceipt(n, destinationKey)["cloudNodeId"];
|
|
90737
90900
|
return typeof cloudId === "string" && cloudId ? cloudId : n.id;
|
|
@@ -90890,7 +91053,7 @@ function buildInstanceIngest(store, profile, daemonVersion, ignorePatterns = [],
|
|
|
90890
91053
|
edgesHeldDropped: 0
|
|
90891
91054
|
};
|
|
90892
91055
|
const sharedCauses = /* @__PURE__ */ new Map();
|
|
90893
|
-
const causalQueue = store
|
|
91056
|
+
const causalQueue = instanceGateRows(store, "Problem").filter((r) => r.source !== "cloud" && r.resolvedAs !== PROBLEM_RESOLUTION.FALSE_POSITIVE).map((r) => r.id).filter((id) => ignorePatterns.length === 0 || !ignored(store.getNode(id)?.description ?? ""));
|
|
90894
91057
|
const causalSeen = new Set(causalQueue);
|
|
90895
91058
|
for (let i2 = 0; i2 < causalQueue.length; i2++) {
|
|
90896
91059
|
const id = causalQueue[i2];
|
|
@@ -90906,21 +91069,21 @@ function buildInstanceIngest(store, profile, daemonVersion, ignorePatterns = [],
|
|
|
90906
91069
|
for (const label of INSTANCE_LABELS) {
|
|
90907
91070
|
const ref = [];
|
|
90908
91071
|
includedByLabel.set(label, ref);
|
|
90909
|
-
const
|
|
90910
|
-
for (const row of store
|
|
90911
|
-
if (row.
|
|
90912
|
-
|
|
90913
|
-
if (n) local.push(n);
|
|
91072
|
+
const localIds = [];
|
|
91073
|
+
for (const row of instanceGateRows(store, label)) {
|
|
91074
|
+
if (row.source !== "cloud") {
|
|
91075
|
+
localIds.push(row.id);
|
|
90914
91076
|
continue;
|
|
90915
91077
|
}
|
|
90916
91078
|
if (seen.has(row.id) || walked.has(row.id)) continue;
|
|
90917
91079
|
walked.add(row.id);
|
|
90918
91080
|
composition.candidates++;
|
|
90919
|
-
if ((row.
|
|
91081
|
+
if ((row.privateFinding || label === "Vulnerability") && !opts.project) composition.privateTargetMissing++;
|
|
90920
91082
|
else composition.cloudSourced++;
|
|
90921
91083
|
}
|
|
91084
|
+
const local = gateViews(store, label, localIds);
|
|
90922
91085
|
const population = label === "RootCause" && opts.sharedStore ? [...local, ...sharedCauses.values()] : local;
|
|
90923
|
-
for (
|
|
91086
|
+
for (let n of population) {
|
|
90924
91087
|
if (seen.has(n.id)) continue;
|
|
90925
91088
|
if (walked.has(n.id)) continue;
|
|
90926
91089
|
walked.add(n.id);
|
|
@@ -90956,6 +91119,11 @@ function buildInstanceIngest(store, profile, daemonVersion, ignorePatterns = [],
|
|
|
90956
91119
|
semanticEndpoints.set(n.id, contributedWireId(n, opts.destinationKey));
|
|
90957
91120
|
continue;
|
|
90958
91121
|
}
|
|
91122
|
+
if (gateView.has(n)) {
|
|
91123
|
+
const full = store.getNode(n.id);
|
|
91124
|
+
if (!full) continue;
|
|
91125
|
+
n = full;
|
|
91126
|
+
}
|
|
90959
91127
|
if (isEditPlaceholderSolution(n)) {
|
|
90960
91128
|
composition.autoMint++;
|
|
90961
91129
|
continue;
|
package/generalize-worker.mjs
CHANGED
|
@@ -15700,6 +15700,7 @@ var EXPOSURE_SCOPE = "exposure";
|
|
|
15700
15700
|
// ../../packages/local-graph/src/change-feed.ts
|
|
15701
15701
|
var CHANGE_FEED_KEY = "change-feed";
|
|
15702
15702
|
var ANY_CHANGE = "*";
|
|
15703
|
+
var nodeScope = (label) => `node:${label}`;
|
|
15703
15704
|
var CHANGE_LOG_KEEP = 25e4;
|
|
15704
15705
|
var PRUNED_TO_KEY = "change-log-pruned-to";
|
|
15705
15706
|
function installChangeFeed(db) {
|
|
@@ -15993,8 +15994,10 @@ function column(c) {
|
|
|
15993
15994
|
function compileNodeWhere(q) {
|
|
15994
15995
|
if (q.labels.length === 0) throw new Error("selectNodes: no labels");
|
|
15995
15996
|
const params = [...q.labels];
|
|
15996
|
-
const
|
|
15997
|
-
|
|
15997
|
+
const byId = (q.where ?? []).some((w) => "idIn" in w);
|
|
15998
|
+
const unindexed = byId ? "+" : "";
|
|
15999
|
+
const conds = [`${unindexed}label IN (${q.labels.map(() => "?").join(", ")})`];
|
|
16000
|
+
if (!q.includeClosed) conds.push(`${unindexed}valid_to IS NULL`);
|
|
15998
16001
|
for (const w of q.where ?? []) {
|
|
15999
16002
|
if ("embedding" in w) {
|
|
16000
16003
|
conds.push(`${EMBEDDING_DIM_SQL} ${w.embedding === "present" ? ">" : "="} 0`);
|
|
@@ -16046,6 +16049,14 @@ function compileNodeWhere(q) {
|
|
|
16046
16049
|
function attrValueSql(p) {
|
|
16047
16050
|
return `CASE json_type(attrs_json, ${p}) WHEN 'integer' THEN CAST(json_extract(attrs_json, ${p}) AS REAL) ELSE json_extract(attrs_json, ${p}) END`;
|
|
16048
16051
|
}
|
|
16052
|
+
function nodeSelectFrom(q) {
|
|
16053
|
+
if (q.indexedBy) return `nodes INDEXED BY ${selectIndex(q.indexedBy)}`;
|
|
16054
|
+
const where = q.where ?? [];
|
|
16055
|
+
if (!q.includeClosed && where.some((w) => "descriptionIn" in w && w.descriptionIn.length > 0) && !where.some((w) => "idIn" in w)) {
|
|
16056
|
+
return "nodes INDEXED BY nodes_live_description";
|
|
16057
|
+
}
|
|
16058
|
+
return "nodes";
|
|
16059
|
+
}
|
|
16049
16060
|
function compileNodeSelect(q) {
|
|
16050
16061
|
const { where, params } = compileNodeWhere(q);
|
|
16051
16062
|
const attrs = q.attrs ?? [];
|
|
@@ -16063,7 +16074,7 @@ function compileNodeSelect(q) {
|
|
|
16063
16074
|
}
|
|
16064
16075
|
const orderParams = !q.orderBy && q.labels.length > 1 ? [...q.labels] : [];
|
|
16065
16076
|
const limit = q.limit !== void 0 ? ` LIMIT ${Math.max(0, Math.floor(q.limit))}` : "";
|
|
16066
|
-
const sql = `SELECT id, label, description, created_at, last_updated_at, page_rank, cumulative_hits, extraction_confidence, extraction_source, community, created_at_seq, last_reinforced_at_seq, valid_to, logical_id, version, ${EMBEDDING_DIM_SQL} AS embedding_dim${q.withEmbedding ? ", embedding_blob, embedding_json" : ""}${attrCols} FROM
|
|
16077
|
+
const sql = `SELECT id, label, description, created_at, last_updated_at, page_rank, cumulative_hits, extraction_confidence, extraction_source, community, created_at_seq, last_reinforced_at_seq, valid_to, logical_id, version, ${EMBEDDING_DIM_SQL} AS embedding_dim${q.withEmbedding ? ", embedding_blob, embedding_json" : ""}${attrCols} FROM ${nodeSelectFrom(q)} WHERE ${where} ORDER BY ${order}${limit}`;
|
|
16067
16078
|
return { sql, params: [...params, ...orderParams] };
|
|
16068
16079
|
}
|
|
16069
16080
|
function decodeAttr(value, type) {
|
|
@@ -16300,6 +16311,18 @@ CREATE INDEX IF NOT EXISTS edges_live_prior_tag ON edges(created_at) WHERE valid
|
|
|
16300
16311
|
-- incremental reindex can fetch just the changed files' symbols instead of
|
|
16301
16312
|
-- scanning every versioned node. Partial: only live rows are ever queried.
|
|
16302
16313
|
CREATE INDEX IF NOT EXISTS nodes_relpath ON nodes(json_extract(attrs_json, '$.relPath')) WHERE valid_to IS NULL;
|
|
16314
|
+
-- Symbols by qname and doc comments by the symbol they attach to: the drain's
|
|
16315
|
+
-- symbol lexicon resolves the names in the text it ships one at a time, instead
|
|
16316
|
+
-- of reading every Function/Method/Class/Interface/Const and Comment row on each
|
|
16317
|
+
-- live push (~300 MB of a 1.1 GB store per push, 2026-10-20).
|
|
16318
|
+
CREATE INDEX IF NOT EXISTS nodes_live_qname ON nodes(json_extract(attrs_json, '$.qname')) WHERE valid_to IS NULL;
|
|
16319
|
+
CREATE INDEX IF NOT EXISTS nodes_live_attached_to ON nodes(json_extract(attrs_json, '$.attachedTo')) WHERE valid_to IS NULL AND label = 'Comment';
|
|
16320
|
+
-- The attrs the cloud drain gates every instance row on, so it reads them from
|
|
16321
|
+
-- the index alone: pulled cloud rows are most of these labels (7,105 of 7,708
|
|
16322
|
+
-- Solutions on the live store) at ~10 KB of attrs each, and five full scans for
|
|
16323
|
+
-- two attrs read ~220 MB per push to find the few local rows that can ship.
|
|
16324
|
+
-- created_at leads so the index order is findNodeFieldsByLabel's.
|
|
16325
|
+
CREATE INDEX IF NOT EXISTS nodes_live_instance_gate ON nodes(label, created_at, json_extract(attrs_json, '$.source'), json_extract(attrs_json, '$.privateFinding'), json_extract(attrs_json, '$.resolvedAs'), id) WHERE valid_to IS NULL AND label IN ('Problem', 'Vulnerability', 'Solution', 'RootCause');
|
|
16303
16326
|
-- Closed rows by relPath, for reviveReobserved (every incremental reindex): the
|
|
16304
16327
|
-- live-only index above cannot serve it, so it scanned the whole table (~275 MB
|
|
16305
16328
|
-- of a 1.1 GB store per reindex, 2026-10-20).
|
|
@@ -16415,6 +16438,7 @@ var SqliteGraphStore = class {
|
|
|
16415
16438
|
initialPageRankScale;
|
|
16416
16439
|
constructor(opts) {
|
|
16417
16440
|
this.db = openDatabase(opts.path);
|
|
16441
|
+
this.db.exec("PRAGMA cache_size = -32768");
|
|
16418
16442
|
this.db.exec(SCHEMA_SQL);
|
|
16419
16443
|
const versionRow = this.db.prepare("SELECT version FROM schema_version LIMIT 1").get();
|
|
16420
16444
|
if (!versionRow) {
|
|
@@ -16479,6 +16503,7 @@ var SqliteGraphStore = class {
|
|
|
16479
16503
|
attrs_json = excluded.attrs_json
|
|
16480
16504
|
`),
|
|
16481
16505
|
getNode: this.db.prepare("SELECT * FROM nodes WHERE id = ?"),
|
|
16506
|
+
labelOf: this.db.prepare("SELECT label FROM nodes WHERE id = ?"),
|
|
16482
16507
|
// Single-column fast paths for the nightly pass — writing 100k pageRank
|
|
16483
16508
|
// values through the 18-column COALESCE updateNode took ~30s; these are ~10x
|
|
16484
16509
|
// cheaper (one bound column, no COALESCE evaluation per row).
|
|
@@ -16666,6 +16691,36 @@ var SqliteGraphStore = class {
|
|
|
16666
16691
|
}
|
|
16667
16692
|
return out;
|
|
16668
16693
|
}
|
|
16694
|
+
firstLiveByNameOrQname(labels, name) {
|
|
16695
|
+
if (labels.length === 0) return null;
|
|
16696
|
+
const inLabels = labels.map(() => "?").join(",");
|
|
16697
|
+
const probe = (index, match) => this.db.prepare(
|
|
16698
|
+
`SELECT rowid AS rid, created_at AS at, label, description, json_extract(attrs_json, '$.qname') AS qname FROM nodes INDEXED BY ${index}
|
|
16699
|
+
WHERE ${match} = ? AND label IN (${inLabels}) AND valid_to IS NULL`
|
|
16700
|
+
).all(name, ...labels);
|
|
16701
|
+
const rank = (r) => [labels.indexOf(r.label), r.at, r.rid];
|
|
16702
|
+
const before = (a, b) => a[0] < b[0] || a[0] === b[0] && (a[1] < b[1] || a[1] === b[1] && a[2] < b[2]);
|
|
16703
|
+
let best;
|
|
16704
|
+
for (const r of [...probe("nodes_live_description", "description"), ...probe("nodes_live_qname", "json_extract(attrs_json, '$.qname')")]) {
|
|
16705
|
+
if (!best || before(rank(r), rank(best))) best = r;
|
|
16706
|
+
}
|
|
16707
|
+
return best ? { label: best.label, description: best.description, qname: typeof best.qname === "string" ? best.qname : null } : null;
|
|
16708
|
+
}
|
|
16709
|
+
instanceGateRows(label) {
|
|
16710
|
+
return this.db.prepare(
|
|
16711
|
+
`SELECT id, json_extract(attrs_json, '$.source') AS source, json_extract(attrs_json, '$.privateFinding') AS privateFinding,
|
|
16712
|
+
json_extract(attrs_json, '$.resolvedAs') AS resolvedAs
|
|
16713
|
+
FROM nodes INDEXED BY nodes_live_instance_gate
|
|
16714
|
+
WHERE label IN ('Problem', 'Vulnerability', 'Solution', 'RootCause') AND label = ? AND valid_to IS NULL
|
|
16715
|
+
ORDER BY label, created_at, rowid`
|
|
16716
|
+
).all(label);
|
|
16717
|
+
}
|
|
16718
|
+
liveCommentsAttachedTo(name) {
|
|
16719
|
+
return this.db.prepare(
|
|
16720
|
+
`SELECT json_extract(attrs_json, '$.kind') AS kind, json_extract(attrs_json, '$.text') AS text, description FROM nodes INDEXED BY nodes_live_attached_to
|
|
16721
|
+
WHERE label = 'Comment' AND valid_to IS NULL AND json_extract(attrs_json, '$.attachedTo') = ? ORDER BY created_at, rowid`
|
|
16722
|
+
).all(name);
|
|
16723
|
+
}
|
|
16669
16724
|
strandedEdges() {
|
|
16670
16725
|
return countStrandedEdges(this.db);
|
|
16671
16726
|
}
|
|
@@ -16844,7 +16899,11 @@ var SqliteGraphStore = class {
|
|
|
16844
16899
|
if (!stmt2) {
|
|
16845
16900
|
const cols = attrKeys.map((k, i) => `, ${attrValueSql(`'$.${k}'`)} AS a${i}, json_type(attrs_json, '$.${k}') AS t${i}`).join("");
|
|
16846
16901
|
stmt2 = this.db.prepare(
|
|
16847
|
-
|
|
16902
|
+
// Ordered, not left to the planner: readers take "first row wins" (the
|
|
16903
|
+
// symbol lexicon's name winner and doc pick), and the order otherwise
|
|
16904
|
+
// followed whichever partial index the planner chose. nodes_live_created
|
|
16905
|
+
// serves this order without a sort.
|
|
16906
|
+
`SELECT id, description, community, extraction_confidence${cols} FROM nodes WHERE label = ? AND valid_to IS NULL ORDER BY created_at, rowid`
|
|
16848
16907
|
);
|
|
16849
16908
|
this.fieldStmts.set(cacheKey, stmt2);
|
|
16850
16909
|
}
|
|
@@ -16870,7 +16929,7 @@ var SqliteGraphStore = class {
|
|
|
16870
16929
|
}
|
|
16871
16930
|
countNodes(q) {
|
|
16872
16931
|
const { where, params } = compileNodeWhere(q);
|
|
16873
|
-
return Number(this.db.prepare(`SELECT COUNT(*) AS n FROM
|
|
16932
|
+
return Number(this.db.prepare(`SELECT COUNT(*) AS n FROM ${nodeSelectFrom(q)} WHERE ${where}`).get(...params).n);
|
|
16874
16933
|
}
|
|
16875
16934
|
setCommunity(id, community) {
|
|
16876
16935
|
this.stmts.setCommunity.run({ id, v: community });
|
|
@@ -16944,11 +17003,13 @@ var SqliteGraphStore = class {
|
|
|
16944
17003
|
}
|
|
16945
17004
|
}
|
|
16946
17005
|
edgeRuleViolation(edge) {
|
|
16947
|
-
return localEdgeViolation(
|
|
16948
|
-
|
|
16949
|
-
|
|
16950
|
-
|
|
16951
|
-
|
|
17006
|
+
return localEdgeViolation(this.labelOf(edge.from), edge.type, this.labelOf(edge.to));
|
|
17007
|
+
}
|
|
17008
|
+
/** An endpoint's label, any version, as `getNode(id)?.label` reads it. Every
|
|
17009
|
+
* edge write checks both endpoints, and hydrating them (attrs + vector) to
|
|
17010
|
+
* read one leading column was most of an incremental reindex's reads. */
|
|
17011
|
+
labelOf(id) {
|
|
17012
|
+
return this.stmts.labelOf.get(id)?.label;
|
|
16952
17013
|
}
|
|
16953
17014
|
updateEdge(id, patch) {
|
|
16954
17015
|
this.stmts.updateEdge.run({
|
|
@@ -16992,7 +17053,21 @@ var SqliteGraphStore = class {
|
|
|
16992
17053
|
* per project per request; the materializing form held the daemon's event
|
|
16993
17054
|
* loop for seconds at scale. */
|
|
16994
17055
|
countNodesByLabel(label) {
|
|
16995
|
-
return Number(this.stmts.countByLabel.get(label).n);
|
|
17056
|
+
return this.countAt(`label:${label}`, [nodeScope(label)], () => Number(this.stmts.countByLabel.get(label).n));
|
|
17057
|
+
}
|
|
17058
|
+
/** Counts asked for on every status poll (/health, `errata status`, the
|
|
17059
|
+
* dashboards): a live-node COUNT(*) reads the whole liveness index, ~3 MB per
|
|
17060
|
+
* call on a 1.1 GB store, to recount a set that moves only on a write. Each is
|
|
17061
|
+
* kept with the change-feed reading of the rows it counts — triggers bump it on
|
|
17062
|
+
* every write from any connection, so a hit is exact, not a TTL guess. */
|
|
17063
|
+
counts = /* @__PURE__ */ new Map();
|
|
17064
|
+
countAt(key, scopes, count) {
|
|
17065
|
+
const seq = readChangeSeq(this.db, scopes);
|
|
17066
|
+
const hit = this.counts.get(key);
|
|
17067
|
+
if (hit && hit.seq === seq) return hit.n;
|
|
17068
|
+
const n = count();
|
|
17069
|
+
this.counts.set(key, { seq, n });
|
|
17070
|
+
return n;
|
|
16996
17071
|
}
|
|
16997
17072
|
/** Live rows for a label with attrs and creation stamps only — for scans that
|
|
16998
17073
|
* read attrs and never the vector. `findNodesByLabel` decodes every embedding
|
|
@@ -17259,7 +17334,7 @@ var SqliteGraphStore = class {
|
|
|
17259
17334
|
return out;
|
|
17260
17335
|
}
|
|
17261
17336
|
nodeCount() {
|
|
17262
|
-
return Number(this.stmts.nodeCount.get().n);
|
|
17337
|
+
return this.countAt("nodes", [ANY_CHANGE], () => Number(this.stmts.nodeCount.get().n));
|
|
17263
17338
|
}
|
|
17264
17339
|
mutationCount() {
|
|
17265
17340
|
return this.mutations;
|
|
@@ -17303,7 +17378,7 @@ var SqliteGraphStore = class {
|
|
|
17303
17378
|
return this.ingestSeq;
|
|
17304
17379
|
}
|
|
17305
17380
|
edgeCount() {
|
|
17306
|
-
return Number(this.stmts.edgeCount.get().n);
|
|
17381
|
+
return this.countAt("edges", [ANY_CHANGE], () => Number(this.stmts.edgeCount.get().n));
|
|
17307
17382
|
}
|
|
17308
17383
|
// Lightweight whole-graph scans — select only the columns the caller needs so a
|
|
17309
17384
|
// 100k-node pass skips the per-row attrs JSON.parse + embedding decode rowToNode
|
|
@@ -18350,6 +18425,62 @@ function buildSymbolLexicon(store) {
|
|
|
18350
18425
|
}
|
|
18351
18426
|
return lex;
|
|
18352
18427
|
}
|
|
18428
|
+
function lazySymbolLexicon(store) {
|
|
18429
|
+
if (!store.firstLiveByNameOrQname || !store.liveCommentsAttachedTo) return null;
|
|
18430
|
+
const labels = SYMBOL_LABELS.map(([label]) => label);
|
|
18431
|
+
const kindOf = new Map(SYMBOL_LABELS.map(([label, kind]) => [label, kind]));
|
|
18432
|
+
const members = /* @__PURE__ */ new Map();
|
|
18433
|
+
const docs = /* @__PURE__ */ new Map();
|
|
18434
|
+
const entries = /* @__PURE__ */ new Map();
|
|
18435
|
+
const member = (name) => {
|
|
18436
|
+
if (!members.has(name)) {
|
|
18437
|
+
const hit = isDistinctiveIdentifier(name) ? store.firstLiveByNameOrQname(labels, name) : null;
|
|
18438
|
+
members.set(name, hit ? { kind: kindOf.get(hit.label), description: hit.description, qname: hit.qname } : null);
|
|
18439
|
+
}
|
|
18440
|
+
return members.get(name);
|
|
18441
|
+
};
|
|
18442
|
+
const docFor = (name) => {
|
|
18443
|
+
if (!docs.has(name)) {
|
|
18444
|
+
let found;
|
|
18445
|
+
for (const c of store.liveCommentsAttachedTo(name)) {
|
|
18446
|
+
if (typeof c.kind !== "string" || !DOC_COMMENT_KINDS.has(c.kind)) continue;
|
|
18447
|
+
const text = typeof c.text === "string" ? c.text : c.description;
|
|
18448
|
+
if (!text?.trim()) continue;
|
|
18449
|
+
found = text.trim().slice(0, MAX_DOC_CHARS);
|
|
18450
|
+
break;
|
|
18451
|
+
}
|
|
18452
|
+
docs.set(name, found);
|
|
18453
|
+
}
|
|
18454
|
+
return docs.get(name);
|
|
18455
|
+
};
|
|
18456
|
+
const isKnown = (tok) => member(tok) !== null;
|
|
18457
|
+
let anySymbol;
|
|
18458
|
+
return {
|
|
18459
|
+
has: isKnown,
|
|
18460
|
+
get(name) {
|
|
18461
|
+
if (!entries.has(name)) {
|
|
18462
|
+
const m = member(name);
|
|
18463
|
+
let entry = null;
|
|
18464
|
+
if (m) {
|
|
18465
|
+
entry = { kind: m.kind };
|
|
18466
|
+
const doc = (m.description ? docFor(m.description) : void 0) ?? (m.qname ? docFor(m.qname) : void 0);
|
|
18467
|
+
for (const phrase of summaryCandidates(name, m.kind, doc)) {
|
|
18468
|
+
if (summaryRejectionReason(phrase, isKnown) === null) {
|
|
18469
|
+
entry.summary = phrase;
|
|
18470
|
+
break;
|
|
18471
|
+
}
|
|
18472
|
+
}
|
|
18473
|
+
}
|
|
18474
|
+
entries.set(name, entry);
|
|
18475
|
+
}
|
|
18476
|
+
return entries.get(name) ?? void 0;
|
|
18477
|
+
},
|
|
18478
|
+
get size() {
|
|
18479
|
+
anySymbol ??= labels.some((label) => store.countNodesByLabel(label) > 0);
|
|
18480
|
+
return anySymbol ? 1 : 0;
|
|
18481
|
+
}
|
|
18482
|
+
};
|
|
18483
|
+
}
|
|
18353
18484
|
var LEXICON_SYMBOL_LABELS = ["Function", "Method", "Class", "Interface", "Const", "Comment"];
|
|
18354
18485
|
function codeGraphKey(store) {
|
|
18355
18486
|
const files = store.labelFingerprint?.(["File"]);
|
|
@@ -18361,7 +18492,7 @@ function lexiconsFor(store) {
|
|
|
18361
18492
|
const key = codeGraphKey(store);
|
|
18362
18493
|
const hit = key !== null ? lexiconCache.get(store) : void 0;
|
|
18363
18494
|
if (hit && hit.key === key) return hit;
|
|
18364
|
-
const built = { symbols: buildSymbolLexicon(store), files: buildFileLexicon(store) };
|
|
18495
|
+
const built = { symbols: lazySymbolLexicon(store) ?? buildSymbolLexicon(store), files: buildFileLexicon(store) };
|
|
18365
18496
|
if (key !== null) lexiconCache.set(store, { key, ...built });
|
|
18366
18497
|
return built;
|
|
18367
18498
|
}
|
package/package.json
CHANGED
package/pass-worker.mjs
CHANGED
|
@@ -18280,9 +18280,10 @@ async function runIndexer(store2, opts) {
|
|
|
18280
18280
|
if (ei > 0) await new Promise((r) => setImmediate(r));
|
|
18281
18281
|
const slice = pendingEdges.slice(ei, ei + EDGE_BATCH);
|
|
18282
18282
|
store2.transaction(() => {
|
|
18283
|
+
const present = store2.nodeLabels([...new Set(slice.flatMap((e) => [e.from, e.to]))]);
|
|
18283
18284
|
for (const e of slice) {
|
|
18284
18285
|
if (e.from === e.to) continue;
|
|
18285
|
-
if (!
|
|
18286
|
+
if (!present.has(e.from) || !present.has(e.to)) continue;
|
|
18286
18287
|
upsertEdge(
|
|
18287
18288
|
store2,
|
|
18288
18289
|
e.from,
|
|
@@ -26142,6 +26143,7 @@ var EXPOSURE_SCOPE = "exposure";
|
|
|
26142
26143
|
// ../../packages/local-graph/src/change-feed.ts
|
|
26143
26144
|
var CHANGE_FEED_KEY = "change-feed";
|
|
26144
26145
|
var ANY_CHANGE = "*";
|
|
26146
|
+
var nodeScope = (label) => `node:${label}`;
|
|
26145
26147
|
var CHANGE_LOG_KEEP = 25e4;
|
|
26146
26148
|
var PRUNED_TO_KEY = "change-log-pruned-to";
|
|
26147
26149
|
function installChangeFeed(db) {
|
|
@@ -26871,8 +26873,10 @@ function column(c) {
|
|
|
26871
26873
|
function compileNodeWhere(q) {
|
|
26872
26874
|
if (q.labels.length === 0) throw new Error("selectNodes: no labels");
|
|
26873
26875
|
const params = [...q.labels];
|
|
26874
|
-
const
|
|
26875
|
-
|
|
26876
|
+
const byId = (q.where ?? []).some((w) => "idIn" in w);
|
|
26877
|
+
const unindexed = byId ? "+" : "";
|
|
26878
|
+
const conds = [`${unindexed}label IN (${q.labels.map(() => "?").join(", ")})`];
|
|
26879
|
+
if (!q.includeClosed) conds.push(`${unindexed}valid_to IS NULL`);
|
|
26876
26880
|
for (const w of q.where ?? []) {
|
|
26877
26881
|
if ("embedding" in w) {
|
|
26878
26882
|
conds.push(`${EMBEDDING_DIM_SQL} ${w.embedding === "present" ? ">" : "="} 0`);
|
|
@@ -26924,6 +26928,14 @@ function compileNodeWhere(q) {
|
|
|
26924
26928
|
function attrValueSql(p) {
|
|
26925
26929
|
return `CASE json_type(attrs_json, ${p}) WHEN 'integer' THEN CAST(json_extract(attrs_json, ${p}) AS REAL) ELSE json_extract(attrs_json, ${p}) END`;
|
|
26926
26930
|
}
|
|
26931
|
+
function nodeSelectFrom(q) {
|
|
26932
|
+
if (q.indexedBy) return `nodes INDEXED BY ${selectIndex(q.indexedBy)}`;
|
|
26933
|
+
const where = q.where ?? [];
|
|
26934
|
+
if (!q.includeClosed && where.some((w) => "descriptionIn" in w && w.descriptionIn.length > 0) && !where.some((w) => "idIn" in w)) {
|
|
26935
|
+
return "nodes INDEXED BY nodes_live_description";
|
|
26936
|
+
}
|
|
26937
|
+
return "nodes";
|
|
26938
|
+
}
|
|
26927
26939
|
function compileNodeSelect(q) {
|
|
26928
26940
|
const { where, params } = compileNodeWhere(q);
|
|
26929
26941
|
const attrs = q.attrs ?? [];
|
|
@@ -26941,7 +26953,7 @@ function compileNodeSelect(q) {
|
|
|
26941
26953
|
}
|
|
26942
26954
|
const orderParams = !q.orderBy && q.labels.length > 1 ? [...q.labels] : [];
|
|
26943
26955
|
const limit = q.limit !== void 0 ? ` LIMIT ${Math.max(0, Math.floor(q.limit))}` : "";
|
|
26944
|
-
const sql = `SELECT id, label, description, created_at, last_updated_at, page_rank, cumulative_hits, extraction_confidence, extraction_source, community, created_at_seq, last_reinforced_at_seq, valid_to, logical_id, version, ${EMBEDDING_DIM_SQL} AS embedding_dim${q.withEmbedding ? ", embedding_blob, embedding_json" : ""}${attrCols} FROM
|
|
26956
|
+
const sql = `SELECT id, label, description, created_at, last_updated_at, page_rank, cumulative_hits, extraction_confidence, extraction_source, community, created_at_seq, last_reinforced_at_seq, valid_to, logical_id, version, ${EMBEDDING_DIM_SQL} AS embedding_dim${q.withEmbedding ? ", embedding_blob, embedding_json" : ""}${attrCols} FROM ${nodeSelectFrom(q)} WHERE ${where} ORDER BY ${order}${limit}`;
|
|
26945
26957
|
return { sql, params: [...params, ...orderParams] };
|
|
26946
26958
|
}
|
|
26947
26959
|
function decodeAttr(value, type) {
|
|
@@ -27178,6 +27190,18 @@ CREATE INDEX IF NOT EXISTS edges_live_prior_tag ON edges(created_at) WHERE valid
|
|
|
27178
27190
|
-- incremental reindex can fetch just the changed files' symbols instead of
|
|
27179
27191
|
-- scanning every versioned node. Partial: only live rows are ever queried.
|
|
27180
27192
|
CREATE INDEX IF NOT EXISTS nodes_relpath ON nodes(json_extract(attrs_json, '$.relPath')) WHERE valid_to IS NULL;
|
|
27193
|
+
-- Symbols by qname and doc comments by the symbol they attach to: the drain's
|
|
27194
|
+
-- symbol lexicon resolves the names in the text it ships one at a time, instead
|
|
27195
|
+
-- of reading every Function/Method/Class/Interface/Const and Comment row on each
|
|
27196
|
+
-- live push (~300 MB of a 1.1 GB store per push, 2026-10-20).
|
|
27197
|
+
CREATE INDEX IF NOT EXISTS nodes_live_qname ON nodes(json_extract(attrs_json, '$.qname')) WHERE valid_to IS NULL;
|
|
27198
|
+
CREATE INDEX IF NOT EXISTS nodes_live_attached_to ON nodes(json_extract(attrs_json, '$.attachedTo')) WHERE valid_to IS NULL AND label = 'Comment';
|
|
27199
|
+
-- The attrs the cloud drain gates every instance row on, so it reads them from
|
|
27200
|
+
-- the index alone: pulled cloud rows are most of these labels (7,105 of 7,708
|
|
27201
|
+
-- Solutions on the live store) at ~10 KB of attrs each, and five full scans for
|
|
27202
|
+
-- two attrs read ~220 MB per push to find the few local rows that can ship.
|
|
27203
|
+
-- created_at leads so the index order is findNodeFieldsByLabel's.
|
|
27204
|
+
CREATE INDEX IF NOT EXISTS nodes_live_instance_gate ON nodes(label, created_at, json_extract(attrs_json, '$.source'), json_extract(attrs_json, '$.privateFinding'), json_extract(attrs_json, '$.resolvedAs'), id) WHERE valid_to IS NULL AND label IN ('Problem', 'Vulnerability', 'Solution', 'RootCause');
|
|
27181
27205
|
-- Closed rows by relPath, for reviveReobserved (every incremental reindex): the
|
|
27182
27206
|
-- live-only index above cannot serve it, so it scanned the whole table (~275 MB
|
|
27183
27207
|
-- of a 1.1 GB store per reindex, 2026-10-20).
|
|
@@ -27293,6 +27317,7 @@ var SqliteGraphStore = class {
|
|
|
27293
27317
|
initialPageRankScale;
|
|
27294
27318
|
constructor(opts) {
|
|
27295
27319
|
this.db = openDatabase(opts.path);
|
|
27320
|
+
this.db.exec("PRAGMA cache_size = -32768");
|
|
27296
27321
|
this.db.exec(SCHEMA_SQL);
|
|
27297
27322
|
const versionRow = this.db.prepare("SELECT version FROM schema_version LIMIT 1").get();
|
|
27298
27323
|
if (!versionRow) {
|
|
@@ -27357,6 +27382,7 @@ var SqliteGraphStore = class {
|
|
|
27357
27382
|
attrs_json = excluded.attrs_json
|
|
27358
27383
|
`),
|
|
27359
27384
|
getNode: this.db.prepare("SELECT * FROM nodes WHERE id = ?"),
|
|
27385
|
+
labelOf: this.db.prepare("SELECT label FROM nodes WHERE id = ?"),
|
|
27360
27386
|
// Single-column fast paths for the nightly pass — writing 100k pageRank
|
|
27361
27387
|
// values through the 18-column COALESCE updateNode took ~30s; these are ~10x
|
|
27362
27388
|
// cheaper (one bound column, no COALESCE evaluation per row).
|
|
@@ -27544,6 +27570,36 @@ var SqliteGraphStore = class {
|
|
|
27544
27570
|
}
|
|
27545
27571
|
return out2;
|
|
27546
27572
|
}
|
|
27573
|
+
firstLiveByNameOrQname(labels, name2) {
|
|
27574
|
+
if (labels.length === 0) return null;
|
|
27575
|
+
const inLabels = labels.map(() => "?").join(",");
|
|
27576
|
+
const probe = (index, match) => this.db.prepare(
|
|
27577
|
+
`SELECT rowid AS rid, created_at AS at, label, description, json_extract(attrs_json, '$.qname') AS qname FROM nodes INDEXED BY ${index}
|
|
27578
|
+
WHERE ${match} = ? AND label IN (${inLabels}) AND valid_to IS NULL`
|
|
27579
|
+
).all(name2, ...labels);
|
|
27580
|
+
const rank = (r) => [labels.indexOf(r.label), r.at, r.rid];
|
|
27581
|
+
const before = (a, b) => a[0] < b[0] || a[0] === b[0] && (a[1] < b[1] || a[1] === b[1] && a[2] < b[2]);
|
|
27582
|
+
let best;
|
|
27583
|
+
for (const r of [...probe("nodes_live_description", "description"), ...probe("nodes_live_qname", "json_extract(attrs_json, '$.qname')")]) {
|
|
27584
|
+
if (!best || before(rank(r), rank(best))) best = r;
|
|
27585
|
+
}
|
|
27586
|
+
return best ? { label: best.label, description: best.description, qname: typeof best.qname === "string" ? best.qname : null } : null;
|
|
27587
|
+
}
|
|
27588
|
+
instanceGateRows(label) {
|
|
27589
|
+
return this.db.prepare(
|
|
27590
|
+
`SELECT id, json_extract(attrs_json, '$.source') AS source, json_extract(attrs_json, '$.privateFinding') AS privateFinding,
|
|
27591
|
+
json_extract(attrs_json, '$.resolvedAs') AS resolvedAs
|
|
27592
|
+
FROM nodes INDEXED BY nodes_live_instance_gate
|
|
27593
|
+
WHERE label IN ('Problem', 'Vulnerability', 'Solution', 'RootCause') AND label = ? AND valid_to IS NULL
|
|
27594
|
+
ORDER BY label, created_at, rowid`
|
|
27595
|
+
).all(label);
|
|
27596
|
+
}
|
|
27597
|
+
liveCommentsAttachedTo(name2) {
|
|
27598
|
+
return this.db.prepare(
|
|
27599
|
+
`SELECT json_extract(attrs_json, '$.kind') AS kind, json_extract(attrs_json, '$.text') AS text, description FROM nodes INDEXED BY nodes_live_attached_to
|
|
27600
|
+
WHERE label = 'Comment' AND valid_to IS NULL AND json_extract(attrs_json, '$.attachedTo') = ? ORDER BY created_at, rowid`
|
|
27601
|
+
).all(name2);
|
|
27602
|
+
}
|
|
27547
27603
|
strandedEdges() {
|
|
27548
27604
|
return countStrandedEdges(this.db);
|
|
27549
27605
|
}
|
|
@@ -27722,7 +27778,11 @@ var SqliteGraphStore = class {
|
|
|
27722
27778
|
if (!stmt2) {
|
|
27723
27779
|
const cols = attrKeys.map((k, i2) => `, ${attrValueSql(`'$.${k}'`)} AS a${i2}, json_type(attrs_json, '$.${k}') AS t${i2}`).join("");
|
|
27724
27780
|
stmt2 = this.db.prepare(
|
|
27725
|
-
|
|
27781
|
+
// Ordered, not left to the planner: readers take "first row wins" (the
|
|
27782
|
+
// symbol lexicon's name winner and doc pick), and the order otherwise
|
|
27783
|
+
// followed whichever partial index the planner chose. nodes_live_created
|
|
27784
|
+
// serves this order without a sort.
|
|
27785
|
+
`SELECT id, description, community, extraction_confidence${cols} FROM nodes WHERE label = ? AND valid_to IS NULL ORDER BY created_at, rowid`
|
|
27726
27786
|
);
|
|
27727
27787
|
this.fieldStmts.set(cacheKey, stmt2);
|
|
27728
27788
|
}
|
|
@@ -27748,7 +27808,7 @@ var SqliteGraphStore = class {
|
|
|
27748
27808
|
}
|
|
27749
27809
|
countNodes(q) {
|
|
27750
27810
|
const { where, params } = compileNodeWhere(q);
|
|
27751
|
-
return Number(this.db.prepare(`SELECT COUNT(*) AS n FROM
|
|
27811
|
+
return Number(this.db.prepare(`SELECT COUNT(*) AS n FROM ${nodeSelectFrom(q)} WHERE ${where}`).get(...params).n);
|
|
27752
27812
|
}
|
|
27753
27813
|
setCommunity(id, community) {
|
|
27754
27814
|
this.stmts.setCommunity.run({ id, v: community });
|
|
@@ -27822,11 +27882,13 @@ var SqliteGraphStore = class {
|
|
|
27822
27882
|
}
|
|
27823
27883
|
}
|
|
27824
27884
|
edgeRuleViolation(edge) {
|
|
27825
|
-
return localEdgeViolation(
|
|
27826
|
-
|
|
27827
|
-
|
|
27828
|
-
|
|
27829
|
-
|
|
27885
|
+
return localEdgeViolation(this.labelOf(edge.from), edge.type, this.labelOf(edge.to));
|
|
27886
|
+
}
|
|
27887
|
+
/** An endpoint's label, any version, as `getNode(id)?.label` reads it. Every
|
|
27888
|
+
* edge write checks both endpoints, and hydrating them (attrs + vector) to
|
|
27889
|
+
* read one leading column was most of an incremental reindex's reads. */
|
|
27890
|
+
labelOf(id) {
|
|
27891
|
+
return this.stmts.labelOf.get(id)?.label;
|
|
27830
27892
|
}
|
|
27831
27893
|
updateEdge(id, patch) {
|
|
27832
27894
|
this.stmts.updateEdge.run({
|
|
@@ -27870,7 +27932,21 @@ var SqliteGraphStore = class {
|
|
|
27870
27932
|
* per project per request; the materializing form held the daemon's event
|
|
27871
27933
|
* loop for seconds at scale. */
|
|
27872
27934
|
countNodesByLabel(label) {
|
|
27873
|
-
return Number(this.stmts.countByLabel.get(label).n);
|
|
27935
|
+
return this.countAt(`label:${label}`, [nodeScope(label)], () => Number(this.stmts.countByLabel.get(label).n));
|
|
27936
|
+
}
|
|
27937
|
+
/** Counts asked for on every status poll (/health, `errata status`, the
|
|
27938
|
+
* dashboards): a live-node COUNT(*) reads the whole liveness index, ~3 MB per
|
|
27939
|
+
* call on a 1.1 GB store, to recount a set that moves only on a write. Each is
|
|
27940
|
+
* kept with the change-feed reading of the rows it counts — triggers bump it on
|
|
27941
|
+
* every write from any connection, so a hit is exact, not a TTL guess. */
|
|
27942
|
+
counts = /* @__PURE__ */ new Map();
|
|
27943
|
+
countAt(key, scopes, count) {
|
|
27944
|
+
const seq = readChangeSeq(this.db, scopes);
|
|
27945
|
+
const hit = this.counts.get(key);
|
|
27946
|
+
if (hit && hit.seq === seq) return hit.n;
|
|
27947
|
+
const n = count();
|
|
27948
|
+
this.counts.set(key, { seq, n });
|
|
27949
|
+
return n;
|
|
27874
27950
|
}
|
|
27875
27951
|
/** Live rows for a label with attrs and creation stamps only — for scans that
|
|
27876
27952
|
* read attrs and never the vector. `findNodesByLabel` decodes every embedding
|
|
@@ -28137,7 +28213,7 @@ var SqliteGraphStore = class {
|
|
|
28137
28213
|
return out2;
|
|
28138
28214
|
}
|
|
28139
28215
|
nodeCount() {
|
|
28140
|
-
return Number(this.stmts.nodeCount.get().n);
|
|
28216
|
+
return this.countAt("nodes", [ANY_CHANGE], () => Number(this.stmts.nodeCount.get().n));
|
|
28141
28217
|
}
|
|
28142
28218
|
mutationCount() {
|
|
28143
28219
|
return this.mutations;
|
|
@@ -28181,7 +28257,7 @@ var SqliteGraphStore = class {
|
|
|
28181
28257
|
return this.ingestSeq;
|
|
28182
28258
|
}
|
|
28183
28259
|
edgeCount() {
|
|
28184
|
-
return Number(this.stmts.edgeCount.get().n);
|
|
28260
|
+
return this.countAt("edges", [ANY_CHANGE], () => Number(this.stmts.edgeCount.get().n));
|
|
28185
28261
|
}
|
|
28186
28262
|
// Lightweight whole-graph scans — select only the columns the caller needs so a
|
|
28187
28263
|
// 100k-node pass skips the per-row attrs JSON.parse + embedding decode rowToNode
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
|
|
1
|
+
e7f2b11ac106aa4f1ad1c492a48adfed53759ae7c7d5cba5f087ff791f67f48d .codex-plugin/plugin.json
|
|
2
2
|
a7bebae9fb01c838d6c75fc784129e28be571233fa66ff5ee8039b574f35ccf7 assets/icon.png
|
|
3
3
|
fc38b64e4df1e0b72c35764bef6c4c05430d97c9009ad0ecc8a9e75b4deab232 configure-local.mjs
|
|
4
4
|
73dc6028b3b070257c7c31f252d11e55a9e096f4aed0f187f10cb21716e1e6d7 dist/client.js
|