codesift-mcp 0.13.1 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (131) hide show
  1. package/dist/cli/commands.d.ts.map +1 -1
  2. package/dist/cli/commands.js +14 -1
  3. package/dist/cli/commands.js.map +1 -1
  4. package/dist/cli/hooks/pre-tool-use.d.ts +1 -0
  5. package/dist/cli/hooks/pre-tool-use.d.ts.map +1 -1
  6. package/dist/cli/hooks/pre-tool-use.js +11 -2
  7. package/dist/cli/hooks/pre-tool-use.js.map +1 -1
  8. package/dist/config.d.ts +13 -0
  9. package/dist/config.d.ts.map +1 -1
  10. package/dist/config.js +21 -0
  11. package/dist/config.js.map +1 -1
  12. package/dist/formatters-core.d.ts +7 -0
  13. package/dist/formatters-core.d.ts.map +1 -1
  14. package/dist/formatters-core.js +14 -3
  15. package/dist/formatters-core.js.map +1 -1
  16. package/dist/parser/extractors/hono-inline-analyzer.js +2 -2
  17. package/dist/parser/extractors/hono-inline-analyzer.js.map +1 -1
  18. package/dist/register-tool-groups/core/search.d.ts.map +1 -1
  19. package/dist/register-tool-groups/core/search.js +14 -0
  20. package/dist/register-tool-groups/core/search.js.map +1 -1
  21. package/dist/register-tool-groups/core/symbols.d.ts.map +1 -1
  22. package/dist/register-tool-groups/core/symbols.js +50 -5
  23. package/dist/register-tool-groups/core/symbols.js.map +1 -1
  24. package/dist/server.d.ts.map +1 -1
  25. package/dist/server.js +54 -2
  26. package/dist/server.js.map +1 -1
  27. package/dist/storage/_shared.d.ts +21 -0
  28. package/dist/storage/_shared.d.ts.map +1 -1
  29. package/dist/storage/_shared.js +36 -0
  30. package/dist/storage/_shared.js.map +1 -1
  31. package/dist/storage/index-footprint.d.ts +12 -0
  32. package/dist/storage/index-footprint.d.ts.map +1 -0
  33. package/dist/storage/index-footprint.js +41 -0
  34. package/dist/storage/index-footprint.js.map +1 -0
  35. package/dist/storage/index-store.d.ts +30 -3
  36. package/dist/storage/index-store.d.ts.map +1 -1
  37. package/dist/storage/index-store.js +122 -9
  38. package/dist/storage/index-store.js.map +1 -1
  39. package/dist/storage/registry.d.ts.map +1 -1
  40. package/dist/storage/registry.js +64 -1
  41. package/dist/storage/registry.js.map +1 -1
  42. package/dist/storage/sqlite/accessors.d.ts +34 -0
  43. package/dist/storage/sqlite/accessors.d.ts.map +1 -0
  44. package/dist/storage/sqlite/accessors.js +178 -0
  45. package/dist/storage/sqlite/accessors.js.map +1 -0
  46. package/dist/storage/sqlite/connection.d.ts +43 -0
  47. package/dist/storage/sqlite/connection.d.ts.map +1 -0
  48. package/dist/storage/sqlite/connection.js +241 -0
  49. package/dist/storage/sqlite/connection.js.map +1 -0
  50. package/dist/storage/sqlite/errors.d.ts +38 -0
  51. package/dist/storage/sqlite/errors.d.ts.map +1 -0
  52. package/dist/storage/sqlite/errors.js +150 -0
  53. package/dist/storage/sqlite/errors.js.map +1 -0
  54. package/dist/storage/sqlite/index-io.d.ts +49 -0
  55. package/dist/storage/sqlite/index-io.d.ts.map +1 -0
  56. package/dist/storage/sqlite/index-io.js +268 -0
  57. package/dist/storage/sqlite/index-io.js.map +1 -0
  58. package/dist/storage/sqlite/meta.d.ts +41 -0
  59. package/dist/storage/sqlite/meta.d.ts.map +1 -0
  60. package/dist/storage/sqlite/meta.js +28 -0
  61. package/dist/storage/sqlite/meta.js.map +1 -0
  62. package/dist/storage/sqlite/rows.d.ts +64 -0
  63. package/dist/storage/sqlite/rows.d.ts.map +1 -0
  64. package/dist/storage/sqlite/rows.js +186 -0
  65. package/dist/storage/sqlite/rows.js.map +1 -0
  66. package/dist/storage/sqlite/runtime.d.ts +6 -0
  67. package/dist/storage/sqlite/runtime.d.ts.map +1 -0
  68. package/dist/storage/sqlite/runtime.js +31 -0
  69. package/dist/storage/sqlite/runtime.js.map +1 -0
  70. package/dist/storage/sqlite/schema.d.ts +34 -0
  71. package/dist/storage/sqlite/schema.d.ts.map +1 -0
  72. package/dist/storage/sqlite/schema.js +98 -0
  73. package/dist/storage/sqlite/schema.js.map +1 -0
  74. package/dist/storage/sqlite-index-store.d.ts +40 -80
  75. package/dist/storage/sqlite-index-store.d.ts.map +1 -1
  76. package/dist/storage/sqlite-index-store.js +32 -577
  77. package/dist/storage/sqlite-index-store.js.map +1 -1
  78. package/dist/storage/telemetry/config.d.ts.map +1 -1
  79. package/dist/storage/telemetry/config.js +14 -3
  80. package/dist/storage/telemetry/config.js.map +1 -1
  81. package/dist/storage/telemetry/payload.d.ts.map +1 -1
  82. package/dist/storage/telemetry/payload.js +4 -1
  83. package/dist/storage/telemetry/payload.js.map +1 -1
  84. package/dist/storage/telemetry/retro-aggregator.d.ts +27 -0
  85. package/dist/storage/telemetry/retro-aggregator.d.ts.map +1 -0
  86. package/dist/storage/telemetry/retro-aggregator.js +179 -0
  87. package/dist/storage/telemetry/retro-aggregator.js.map +1 -0
  88. package/dist/storage/telemetry/sanitizer.d.ts +30 -0
  89. package/dist/storage/telemetry/sanitizer.d.ts.map +1 -1
  90. package/dist/storage/telemetry/sanitizer.js +23 -0
  91. package/dist/storage/telemetry/sanitizer.js.map +1 -1
  92. package/dist/storage/telemetry/uploader.d.ts.map +1 -1
  93. package/dist/storage/telemetry/uploader.js +6 -0
  94. package/dist/storage/telemetry/uploader.js.map +1 -1
  95. package/dist/storage/workspace-resolver.js +1 -1
  96. package/dist/storage/workspace-resolver.js.map +1 -1
  97. package/dist/tools/index-tools/folder-indexer.d.ts.map +1 -1
  98. package/dist/tools/index-tools/folder-indexer.js +4 -1
  99. package/dist/tools/index-tools/folder-indexer.js.map +1 -1
  100. package/dist/tools/index-tools/registry.d.ts +16 -0
  101. package/dist/tools/index-tools/registry.d.ts.map +1 -1
  102. package/dist/tools/index-tools/registry.js +79 -1
  103. package/dist/tools/index-tools/registry.js.map +1 -1
  104. package/dist/tools/index-tools.d.ts +2 -2
  105. package/dist/tools/index-tools.d.ts.map +1 -1
  106. package/dist/tools/index-tools.js +2 -2
  107. package/dist/tools/index-tools.js.map +1 -1
  108. package/dist/tools/plan-turn/stale-index.d.ts +32 -0
  109. package/dist/tools/plan-turn/stale-index.d.ts.map +1 -1
  110. package/dist/tools/plan-turn/stale-index.js +42 -5
  111. package/dist/tools/plan-turn/stale-index.js.map +1 -1
  112. package/dist/tools/search-tools/types.d.ts +20 -0
  113. package/dist/tools/search-tools/types.d.ts.map +1 -1
  114. package/dist/tools/search-tools/zero-hit.d.ts.map +1 -1
  115. package/dist/tools/search-tools/zero-hit.js +0 -0
  116. package/dist/tools/search-tools/zero-hit.js.map +1 -1
  117. package/dist/tools/status-tools.d.ts +10 -0
  118. package/dist/tools/status-tools.d.ts.map +1 -1
  119. package/dist/tools/status-tools.js +14 -4
  120. package/dist/tools/status-tools.js.map +1 -1
  121. package/dist/tools/symbol-tools.d.ts +95 -1
  122. package/dist/tools/symbol-tools.d.ts.map +1 -1
  123. package/dist/tools/symbol-tools.js +182 -26
  124. package/dist/tools/symbol-tools.js.map +1 -1
  125. package/dist/types.d.ts +6 -0
  126. package/dist/types.d.ts.map +1 -1
  127. package/dist/utils/git-head.d.ts +12 -0
  128. package/dist/utils/git-head.d.ts.map +1 -1
  129. package/dist/utils/git-head.js +25 -0
  130. package/dist/utils/git-head.js.map +1 -1
  131. package/package.json +6 -4
@@ -0,0 +1,268 @@
1
+ import { recordIndexFootprint } from "../index-footprint.js";
2
+ import { openIndexDb, openReadConnection, readMetaValue, rollbackQuietly, writeMetaValue, } from "./connection.js";
3
+ import { rethrowOperational } from "./errors.js";
4
+ import { readMetaExtras } from "./meta.js";
5
+ import { INSERT_FILE_SQL, INSERT_SYMBOL_SQL, fileEntryToRow, fileRowBytes, rowToFileEntry, rowToSymbol, symbolRowBytes, symbolToRow, } from "./rows.js";
6
+ import { SCHEMA_VERSION } from "./schema.js";
7
+ /**
8
+ * Whole-index read and write: the paths that touch every row.
9
+ *
10
+ * These are the operations the narrow accessors exist to avoid, kept together because they share
11
+ * the same two invariants — one transaction per logical read, and `repo` written last so a
12
+ * half-written index reads as empty rather than as a complete one.
13
+ */
14
+ /**
15
+ * Replace the entire index. Used by full (re)indexing and by JSON->SQLite migration.
16
+ *
17
+ * `sourceComplete` means every symbol in `index` came from parsing the file it belongs to during
18
+ * THIS run — the only condition under which the `lossy_v1_migration` marker can honestly be
19
+ * cleared. It defaults to false, and the default is the point: rewriting every row is NOT the
20
+ * same as re-reading every file. `index_folder` reuses cached symbols for files whose mtime has
21
+ * not moved, and `enqueueIndexMutation` folds a single-file edit into a whole-index rewrite that
22
+ * arrives here indistinguishable from a real reindex. An earlier version cleared the marker on
23
+ * any call and so erased the warning on the next keystroke after an upgrade.
24
+ *
25
+ * Defaulting to "still lossy" errs toward over-warning, which a reindex clears, rather than
26
+ * toward silence, which nothing does.
27
+ */
28
+ export async function saveIndexSqlite(dbPath, index, opts) {
29
+ const db = await openIndexDb(dbPath);
30
+ db.exec("BEGIN");
31
+ try {
32
+ writeIndexRows(db, index);
33
+ if (opts?.sourceComplete === true) {
34
+ db.prepare("DELETE FROM meta WHERE key = ?").run("lossy_v1_migration");
35
+ }
36
+ db.exec("COMMIT");
37
+ }
38
+ catch (err) {
39
+ rollbackQuietly(db);
40
+ // Classified here, not left raw. `saveIndex`/`saveIncremental` have no classifying boundary of
41
+ // their own, so a SQLITE_BUSY or a full disk during a write used to reach the tool layer as an
42
+ // anonymous Error — carrying the same information as a bug in our own code.
43
+ rethrowOperational(err, dbPath);
44
+ }
45
+ }
46
+ /**
47
+ * True when this database was upgraded from the v1 schema, whose PRIMARY KEY on a non-unique
48
+ * `symbols.id` silently discarded colliding rows. The rebuild preserved everything still stored,
49
+ * but cannot recover what was already gone — so a short symbol list here is a fact about the
50
+ * migration, not about the code. Cleared by a full reindex.
51
+ */
52
+ export async function wasLossilyMigrated(dbPath) {
53
+ const db = await openIndexDb(dbPath);
54
+ return readMetaValue(db, "lossy_v1_migration") !== undefined;
55
+ }
56
+ /**
57
+ * Replace all rows + meta. Caller owns the transaction.
58
+ *
59
+ * `repo` is written here, inside that transaction, and `loadIndexSqlite` treats its absence
60
+ * as "no index" — so a write killed midway rolls back and the db reads as empty rather than
61
+ * as a partially-populated index that later looks complete.
62
+ */
63
+ function writeIndexRows(db, index) {
64
+ db.exec("DELETE FROM symbols");
65
+ db.exec("DELETE FROM files");
66
+ const insSym = db.prepare(INSERT_SYMBOL_SQL);
67
+ for (const sym of index.symbols)
68
+ insSym.run(...symbolToRow(sym));
69
+ const insFile = db.prepare(INSERT_FILE_SQL);
70
+ for (const file of index.files)
71
+ insFile.run(...fileEntryToRow(file));
72
+ writeMetaValue(db, "repo", index.repo);
73
+ writeMetaValue(db, "root", index.root);
74
+ writeMetaValue(db, "created_at", String(index.created_at));
75
+ writeMetaValue(db, "updated_at", String(index.updated_at));
76
+ writeMetaValue(db, "extractor_version", JSON.stringify(index.extractor_version ?? null));
77
+ writeMetaValue(db, "workspaces", JSON.stringify(index.workspaces ?? null));
78
+ writeMetaValue(db, "schema_version", String(SCHEMA_VERSION));
79
+ }
80
+ /**
81
+ * Import a legacy JSON index, but only if this db is still empty — decided and written
82
+ * inside ONE `BEGIN IMMEDIATE` transaction.
83
+ *
84
+ * The check-then-act cannot be done in JS: `codesift postindex-file` is a fresh process per
85
+ * edit, so two of them (or one plus the server) can both observe an empty db, both read the
86
+ * same legacy JSON, and the slower writer then overwrites an incremental update the faster
87
+ * one already committed. An in-process guard cannot see the other process at all.
88
+ *
89
+ * `BEGIN IMMEDIATE` takes the write lock before the emptiness check, so the second writer
90
+ * either waits and then sees the committed import (returns false), or fails on busy_timeout —
91
+ * never silently clobbers. Returns true when this call performed the import.
92
+ */
93
+ export async function importLegacyIndexIfEmpty(dbPath, index) {
94
+ const db = await openIndexDb(dbPath);
95
+ db.exec("BEGIN IMMEDIATE");
96
+ try {
97
+ if (readMetaValue(db, "repo") !== undefined) {
98
+ db.exec("ROLLBACK");
99
+ return false; // another process got there first
100
+ }
101
+ writeIndexRows(db, index);
102
+ db.exec("COMMIT");
103
+ return true;
104
+ }
105
+ catch (err) {
106
+ rollbackQuietly(db);
107
+ rethrowOperational(err, dbPath);
108
+ }
109
+ }
110
+ /**
111
+ * Read a whole table into objects a page at a time, yielding between pages.
112
+ *
113
+ * Materializing an index is the single longest CPU burst in the process, and under the shared
114
+ * daemon it is not the caller's own time being spent — every other client's request is stalled
115
+ * behind it. Measured on this machine's largest index (240,133 symbols): a plain `.map()` blocked
116
+ * the loop for 4.8 s with ZERO timer ticks in that window, so `/health` stopped answering and one
117
+ * client's first cold request looked to everyone else like the daemon had died.
118
+ *
119
+ * The block to remove is `.all()`, not the `.map()` after it: it materializes every row into JS
120
+ * before any of our code runs, so chunking only the mapping still leaves one unbroken stall.
121
+ *
122
+ * A/B'd in one process, alternating, 3 runs each, on the largest index here (240,133
123
+ * symbols) — separate processes are useless for this, the page cache moved the same
124
+ * measurement by 2x:
125
+ *
126
+ * .all() + map median 5609 ms 0 timer ticks — the loop never ran at all
127
+ * paged median 4635 ms 12 ticks/run
128
+ *
129
+ * So this is not the usual responsiveness-for-throughput trade: paging is BOTH faster and
130
+ * interruptible, because a page is one bulk `.all()` over a rowid range and SQLite never
131
+ * has to hold the whole result set. `iterate()` also fixes the stall but gives up the bulk
132
+ * fetch per row.
133
+ *
134
+ * `setImmediate` rather than a microtask: a promise continuation would run before I/O and timers,
135
+ * which is exactly the work being starved. The chunk size is what keeps the yield itself cheap —
136
+ * per-item yielding turned the same load into minutes.
137
+ *
138
+ * The page is sized by TIME, not rows, because yielding is not sufficient on its own — an
139
+ * HTTP server only answers between pages, so the page length is the latency floor. Measured
140
+ * against a live server polled every 50 ms while pages of a given length ran:
141
+ *
142
+ * page 400 ms 6 answered, 8 timed out, worst latency 2182 ms
143
+ * page 200 ms 19 answered, 0 timed out, worst latency 489 ms
144
+ * page 50 ms 78 answered, 0 timed out, worst latency 226 ms
145
+ *
146
+ * and the row count that lands on 50 ms is not a constant: profiling the daemon mid-freeze
147
+ * put 8,016 of 10,113 samples in `StatementSync::All` with GC on the rest, because the same
148
+ * page that costs milliseconds on a fresh heap costs seconds on a daemon already holding
149
+ * several indexes. Hence the feedback loop rather than a tuned constant.
150
+ *
151
+ * Keyset pagination on rowid, never OFFSET: OFFSET re-scans the skipped rows on every page,
152
+ * turning a linear read into a quadratic one exactly on the biggest indexes this exists for.
153
+ */
154
+ const PAGE_TARGET_MS = 50;
155
+ const PAGE_MIN_ROWS = 500;
156
+ const PAGE_MAX_ROWS = 20_000;
157
+ async function readTablePaged(db, table, fn) {
158
+ const stmt = db.prepare(`SELECT rowid AS _rid, * FROM ${table} WHERE rowid > ? ORDER BY rowid LIMIT ?`);
159
+ const out = [];
160
+ let cursor = 0;
161
+ let rows = 4_000;
162
+ for (;;) {
163
+ const started = Date.now();
164
+ const page = stmt.all(cursor, rows);
165
+ if (page.length === 0)
166
+ return out;
167
+ for (const row of page)
168
+ out.push(fn(row));
169
+ const elapsed = Date.now() - started;
170
+ cursor = page[page.length - 1]._rid;
171
+ if (page.length < rows)
172
+ return out;
173
+ // Aim the NEXT page at the time budget rather than a row count. Per-row cost is not a
174
+ // property of the data: the same page costs milliseconds on a fresh heap and seconds
175
+ // under a daemon already holding several indexes, because allocating a few thousand
176
+ // objects there lands in incremental marking. A fixed row count is therefore tuned for
177
+ // one heap and wrong for the other; a fixed time budget holds in both.
178
+ rows = Math.max(PAGE_MIN_ROWS, Math.min(PAGE_MAX_ROWS, Math.round(rows * (PAGE_TARGET_MS / Math.max(elapsed, 1)))));
179
+ await new Promise((r) => setImmediate(r));
180
+ }
181
+ }
182
+ /** Materialise the whole index. Returns null when the db has never been written to. */
183
+ export async function loadIndexSqlite(dbPath) {
184
+ // Called for its side effects, not its handle: this is what creates the schema and runs the
185
+ // v1->v2 migration. The read below deliberately uses its own connection instead (see the
186
+ // snapshot comment), so there is nothing here to bind.
187
+ await openIndexDb(dbPath);
188
+ let repo;
189
+ let root;
190
+ let symbols = [];
191
+ let files = [];
192
+ let meta = {};
193
+ let reader;
194
+ let footprint = 0;
195
+ try {
196
+ // EVERYTHING that goes into the returned index is read inside ONE transaction on a private
197
+ // connection, so every field describes the same instant. The paged reader yields between
198
+ // pages; without the snapshot a write arriving during a yield would be half-included, and
199
+ // `symbols` could disagree with `files` in the very object callers treat as consistent.
200
+ //
201
+ // The meta reads belong inside it for the same reason and are easy to leave out — the first
202
+ // version of this did exactly that, taking `repo`/`root` before the transaction opened and
203
+ // `created_at`/`updated_at`/`extractor_version`/`workspaces` after it committed. That is a
204
+ // narrower window, not a closed one: a concurrent `saveIndexSqlite` landing in either gap
205
+ // yields metadata from one revision paired with rows from another, which is worse than an
206
+ // obviously stale index because every field is individually valid.
207
+ reader = await openReadConnection(dbPath);
208
+ reader.exec("BEGIN");
209
+ try {
210
+ repo = readMetaValue(reader, "repo");
211
+ root = readMetaValue(reader, "root");
212
+ if (repo === undefined || root === undefined) {
213
+ reader.exec("COMMIT");
214
+ return null; // genuinely empty — not a fault
215
+ }
216
+ const owner = repo;
217
+ for (const key of [
218
+ "created_at",
219
+ "updated_at",
220
+ "extractor_version",
221
+ "workspaces",
222
+ "lossy_v1_migration",
223
+ ]) {
224
+ meta[key] = readMetaValue(reader, key);
225
+ }
226
+ symbols = await readTablePaged(reader, "symbols", (row) => {
227
+ footprint += symbolRowBytes(row);
228
+ return rowToSymbol(row, owner);
229
+ });
230
+ files = await readTablePaged(reader, "files", (row) => {
231
+ footprint += fileRowBytes(row);
232
+ return rowToFileEntry(row);
233
+ });
234
+ reader.exec("COMMIT");
235
+ }
236
+ catch (err) {
237
+ rollbackQuietly(reader);
238
+ throw err;
239
+ }
240
+ }
241
+ catch (err) {
242
+ // Corruption often only surfaces on the first page read, not at open time.
243
+ rethrowOperational(err, dbPath);
244
+ }
245
+ finally {
246
+ try {
247
+ reader?.close();
248
+ }
249
+ catch { /* best-effort; the snapshot is done with either way */ }
250
+ }
251
+ const index = {
252
+ repo,
253
+ root,
254
+ symbols,
255
+ files,
256
+ created_at: Number(meta["created_at"] ?? 0),
257
+ updated_at: Number(meta["updated_at"] ?? 0),
258
+ symbol_count: symbols.length,
259
+ file_count: files.length,
260
+ };
261
+ // `lossy_migration` rides on the index itself rather than behind an accessor callers must know
262
+ // to call: the first version of that marker was written on migration, exported, and consulted by
263
+ // nobody, so a lossy index still answered like a complete one.
264
+ Object.assign(index, readMetaExtras((k) => meta[k]));
265
+ recordIndexFootprint(index, footprint);
266
+ return index;
267
+ }
268
+ //# sourceMappingURL=index-io.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index-io.js","sourceRoot":"","sources":["../../../src/storage/sqlite/index-io.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAC7D,OAAO,EACL,WAAW,EACX,kBAAkB,EAClB,aAAa,EACb,eAAe,EACf,cAAc,GACf,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAC3C,OAAO,EACL,eAAe,EACf,iBAAiB,EACjB,cAAc,EACd,YAAY,EACZ,cAAc,EACd,WAAW,EACX,cAAc,EACd,WAAW,GAGZ,MAAM,WAAW,CAAC;AACnB,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAE7C;;;;;;GAMG;AAEH;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,MAAc,EACd,KAAgB,EAChB,IAAmC;IAEnC,MAAM,EAAE,GAAG,MAAM,WAAW,CAAC,MAAM,CAAC,CAAC;IAErC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACjB,IAAI,CAAC;QACH,cAAc,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;QAC1B,IAAI,IAAI,EAAE,cAAc,KAAK,IAAI,EAAE,CAAC;YAClC,EAAE,CAAC,OAAO,CAAC,gCAAgC,CAAC,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;QACzE,CAAC;QACD,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACpB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,eAAe,CAAC,EAAE,CAAC,CAAC;QACpB,+FAA+F;QAC/F,+FAA+F;QAC/F,4EAA4E;QAC5E,kBAAkB,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IAClC,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,MAAc;IACrD,MAAM,EAAE,GAAG,MAAM,WAAW,CAAC,MAAM,CAAC,CAAC;IACrC,OAAO,aAAa,CAAC,EAAE,EAAE,oBAAoB,CAAC,KAAK,SAAS,CAAC;AAC/D,CAAC;AAED;;;;;;GAMG;AACH,SAAS,cAAc,CAAC,EAAoB,EAAE,KAAgB;IAC5D,EAAE,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;IAC/B,EAAE,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;IAE7B,MAAM,MAAM,GAAG,EAAE,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC;IAC7C,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,OAAO;QAAE,MAAM,CAAC,GAAG,CAAC,GAAI,WAAW,CAAC,GAAG,CAAa,CAAC,CAAC;IAE9E,MAAM,OAAO,GAAG,EAAE,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;IAC5C,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK;QAAE,OAAO,CAAC,GAAG,CAAC,GAAI,cAAc,CAAC,IAAI,CAAa,CAAC,CAAC;IAElF,cAAc,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;IACvC,cAAc,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;IACvC,cAAc,CAAC,EAAE,EAAE,YAAY,EAAE,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC;IAC3D,cAAc,CAAC,EAAE,EAAE,YAAY,EAAE,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC;IAC3D,cAAc,CAAC,EAAE,EAAE,mBAAmB,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,iBAAiB,IAAI,IAAI,CAAC,CAAC,CAAC;IACzF,cAAc,CAAC,EAAE,EAAE,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,UAAU,IAAI,IAAI,CAAC,CAAC,CAAC;IAC3E,cAAc,CAAC,EAAE,EAAE,gBAAgB,EAAE,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC;AAC/D,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,MAAc,EACd,KAAgB;IAEhB,MAAM,EAAE,GAAG,MAAM,WAAW,CAAC,MAAM,CAAC,CAAC;IAErC,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IAC3B,IAAI,CAAC;QACH,IAAI,aAAa,CAAC,EAAE,EAAE,MAAM,CAAC,KAAK,SAAS,EAAE,CAAC;YAC5C,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACpB,OAAO,KAAK,CAAC,CAAC,kCAAkC;QAClD,CAAC;QACD,cAAc,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;QAC1B,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAClB,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,eAAe,CAAC,EAAE,CAAC,CAAC;QACpB,kBAAkB,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IAClC,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2CG;AACH,MAAM,cAAc,GAAG,EAAE,CAAC;AAC1B,MAAM,aAAa,GAAG,GAAG,CAAC;AAC1B,MAAM,aAAa,GAAG,MAAM,CAAC;AAE7B,KAAK,UAAU,cAAc,CAC3B,EAAoB,EACpB,KAAa,EACb,EAAiB;IAEjB,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CACrB,gCAAgC,KAAK,yCAAyC,CAC/E,CAAC;IACF,MAAM,GAAG,GAAQ,EAAE,CAAC;IACpB,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,IAAI,GAAG,KAAK,CAAC;IACjB,SAAS,CAAC;QACR,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAA2C,CAAC;QAC9E,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,GAAG,CAAC;QAClC,KAAK,MAAM,GAAG,IAAI,IAAI;YAAE,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QAC1C,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;QACrC,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC,IAAI,CAAC;QACrC,IAAI,IAAI,CAAC,MAAM,GAAG,IAAI;YAAE,OAAO,GAAG,CAAC;QACnC,sFAAsF;QACtF,qFAAqF;QACrF,oFAAoF;QACpF,uFAAuF;QACvF,uEAAuE;QACvE,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,aAAa,EAAE,IAAI,CAAC,GAAG,CAAC,aAAa,EACnD,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/D,MAAM,IAAI,OAAO,CAAO,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC;IAClD,CAAC;AACH,CAAC;AAED,uFAAuF;AACvF,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,MAAc;IAClD,4FAA4F;IAC5F,yFAAyF;IACzF,uDAAuD;IACvD,MAAM,WAAW,CAAC,MAAM,CAAC,CAAC;IAE1B,IAAI,IAAwB,CAAC;IAC7B,IAAI,IAAwB,CAAC;IAC7B,IAAI,OAAO,GAAiB,EAAE,CAAC;IAC/B,IAAI,KAAK,GAAgB,EAAE,CAAC;IAC5B,IAAI,IAAI,GAAuC,EAAE,CAAC;IAClD,IAAI,MAAoC,CAAC;IACzC,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,CAAC;QACH,2FAA2F;QAC3F,yFAAyF;QACzF,0FAA0F;QAC1F,wFAAwF;QACxF,EAAE;QACF,4FAA4F;QAC5F,2FAA2F;QAC3F,2FAA2F;QAC3F,0FAA0F;QAC1F,0FAA0F;QAC1F,mEAAmE;QACnE,MAAM,GAAG,MAAM,kBAAkB,CAAC,MAAM,CAAC,CAAC;QAC1C,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACrB,IAAI,CAAC;YACH,IAAI,GAAG,aAAa,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;YACrC,IAAI,GAAG,aAAa,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;YACrC,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;gBAC7C,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACtB,OAAO,IAAI,CAAC,CAAC,gCAAgC;YAC/C,CAAC;YACD,MAAM,KAAK,GAAG,IAAI,CAAC;YACnB,KAAK,MAAM,GAAG,IAAI;gBAChB,YAAY;gBACZ,YAAY;gBACZ,mBAAmB;gBACnB,YAAY;gBACZ,oBAAoB;aACrB,EAAE,CAAC;gBACF,IAAI,CAAC,GAAG,CAAC,GAAG,aAAa,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;YACzC,CAAC;YACD,OAAO,GAAG,MAAM,cAAc,CAAwB,MAAM,EAAE,SAAS,EAAE,CAAC,GAAG,EAAE,EAAE;gBAC/E,SAAS,IAAI,cAAc,CAAC,GAAG,CAAC,CAAC;gBACjC,OAAO,WAAW,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YACjC,CAAC,CAAC,CAAC;YACH,KAAK,GAAG,MAAM,cAAc,CAAqB,MAAM,EAAE,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;gBACxE,SAAS,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC;gBAC/B,OAAO,cAAc,CAAC,GAAG,CAAC,CAAC;YAC7B,CAAC,CAAC,CAAC;YACH,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACxB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,eAAe,CAAC,MAAM,CAAC,CAAC;YACxB,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,2EAA2E;QAC3E,kBAAkB,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IAClC,CAAC;YAAS,CAAC;QACT,IAAI,CAAC;YAAC,MAAM,EAAE,KAAK,EAAE,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC,CAAC,uDAAuD,CAAC,CAAC;IAC5F,CAAC;IAED,MAAM,KAAK,GAAc;QACvB,IAAI;QACJ,IAAI;QACJ,OAAO;QACP,KAAK;QACL,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QAC3C,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QAC3C,YAAY,EAAE,OAAO,CAAC,MAAM;QAC5B,UAAU,EAAE,KAAK,CAAC,MAAM;KACzB,CAAC;IAEF,+FAA+F;IAC/F,iGAAiG;IACjG,+DAA+D;IAC/D,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,cAAc,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAErD,oBAAoB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IACvC,OAAO,KAAK,CAAC;AACf,CAAC"}
@@ -0,0 +1,41 @@
1
+ import type { FileEntry, Workspace } from "../../types.js";
2
+ /**
3
+ * The optional metadata both readers reconstruct, and the summary shape built on it.
4
+ *
5
+ * This is its own module rather than living beside `loadIndexSummarySqlite` because BOTH readers
6
+ * need it — `index-io.ts` for the full index and `accessors.ts` for the summary. Putting it in
7
+ * either one would make that reader a dependency of the other for no reason beyond where the code
8
+ * happened to be written.
9
+ */
10
+ /**
11
+ * Everything about an index EXCEPT its symbols (ADR-004 stage 2).
12
+ *
13
+ * Deliberately not a `CodeIndex` with an empty `symbols` array. That shape would be a lie a caller
14
+ * cannot detect: iterating it yields nothing and reads as "this repo has no symbols", which is the
15
+ * empty-because-not-loaded answer this codebase treats as the worst one available. With no such
16
+ * field at all, a consumer that needs symbols fails to compile instead of failing silently.
17
+ *
18
+ * `symbol_count` is counted in SQL rather than derived from an array, so it stays a real count.
19
+ */
20
+ export interface IndexSummary {
21
+ repo: string;
22
+ root: string;
23
+ files: FileEntry[];
24
+ created_at: number;
25
+ updated_at: number;
26
+ symbol_count: number;
27
+ file_count: number;
28
+ extractor_version?: Record<string, string>;
29
+ workspaces?: Workspace[];
30
+ lossy_migration?: boolean;
31
+ }
32
+ /**
33
+ * The three optional meta fields both readers reconstruct, parsed once.
34
+ *
35
+ * `loadIndexSqlite` and `loadIndexSummarySqlite` had the same JSON-parse-and-null-check block
36
+ * duplicated verbatim. Two copies of a parser is two places for a schema addition to be applied
37
+ * to only one, and the readers would then disagree about the same database — the class of
38
+ * divergence this file has already been bitten by twice.
39
+ */
40
+ export declare function readMetaExtras(meta: (key: string) => string | undefined): Pick<IndexSummary, "extractor_version" | "workspaces" | "lossy_migration">;
41
+ //# sourceMappingURL=meta.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"meta.d.ts","sourceRoot":"","sources":["../../../src/storage/sqlite/meta.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAE3D;;;;;;;GAOG;AAEH;;;;;;;;;GASG;AACH,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,SAAS,EAAE,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,iBAAiB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC3C,UAAU,CAAC,EAAE,SAAS,EAAE,CAAC;IACzB,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B;AAED;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAC5B,IAAI,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,MAAM,GAAG,SAAS,GACxC,IAAI,CAAC,YAAY,EAAE,mBAAmB,GAAG,YAAY,GAAG,iBAAiB,CAAC,CAmB5E"}
@@ -0,0 +1,28 @@
1
+ /**
2
+ * The three optional meta fields both readers reconstruct, parsed once.
3
+ *
4
+ * `loadIndexSqlite` and `loadIndexSummarySqlite` had the same JSON-parse-and-null-check block
5
+ * duplicated verbatim. Two copies of a parser is two places for a schema addition to be applied
6
+ * to only one, and the readers would then disagree about the same database — the class of
7
+ * divergence this file has already been bitten by twice.
8
+ */
9
+ export function readMetaExtras(meta) {
10
+ const out = {};
11
+ const extractorRaw = meta("extractor_version");
12
+ if (extractorRaw !== undefined) {
13
+ const parsed = JSON.parse(extractorRaw);
14
+ if (parsed !== null)
15
+ out.extractor_version = parsed;
16
+ }
17
+ const workspacesRaw = meta("workspaces");
18
+ if (workspacesRaw !== undefined) {
19
+ const parsed = JSON.parse(workspacesRaw);
20
+ if (parsed !== null)
21
+ out.workspaces = parsed;
22
+ }
23
+ // Presence IS the marker: the writer only ever stores "1" and clears by deleting the key.
24
+ if (meta("lossy_v1_migration") !== undefined)
25
+ out.lossy_migration = true;
26
+ return out;
27
+ }
28
+ //# sourceMappingURL=meta.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"meta.js","sourceRoot":"","sources":["../../../src/storage/sqlite/meta.ts"],"names":[],"mappings":"AAkCA;;;;;;;GAOG;AACH,MAAM,UAAU,cAAc,CAC5B,IAAyC;IAEzC,MAAM,GAAG,GAA+E,EAAE,CAAC;IAE3F,MAAM,YAAY,GAAG,IAAI,CAAC,mBAAmB,CAAC,CAAC;IAC/C,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAkC,CAAC;QACzE,IAAI,MAAM,KAAK,IAAI;YAAE,GAAG,CAAC,iBAAiB,GAAG,MAAM,CAAC;IACtD,CAAC;IAED,MAAM,aAAa,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC;IACzC,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;QAChC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAuB,CAAC;QAC/D,IAAI,MAAM,KAAK,IAAI;YAAE,GAAG,CAAC,UAAU,GAAG,MAAM,CAAC;IAC/C,CAAC;IAED,0FAA0F;IAC1F,IAAI,IAAI,CAAC,oBAAoB,CAAC,KAAK,SAAS;QAAE,GAAG,CAAC,eAAe,GAAG,IAAI,CAAC;IAEzE,OAAO,GAAG,CAAC;AACb,CAAC"}
@@ -0,0 +1,64 @@
1
+ import type { CodeSymbol, FileEntry } from "../../types.js";
2
+ export type SymbolRow = {
3
+ id: string;
4
+ file: string;
5
+ name: string;
6
+ kind: string;
7
+ start_line: number;
8
+ end_line: number;
9
+ start_col: number | null;
10
+ end_col: number | null;
11
+ start_byte: number | null;
12
+ end_byte: number | null;
13
+ signature: string | null;
14
+ docstring: string | null;
15
+ source: string | null;
16
+ parent: string | null;
17
+ is_async: number | null;
18
+ is_exported: number | null;
19
+ extras: string | null;
20
+ };
21
+ export declare function symbolToRow(sym: CodeSymbol): unknown[];
22
+ /** `repo` is identical for every symbol in an index, so it lives in `meta` and is stamped
23
+ * back on here instead of being written 100k times. */
24
+ export declare function rowToSymbol(row: SymbolRow, repo: string): CodeSymbol;
25
+ export type FileRow = {
26
+ path: string;
27
+ language: string;
28
+ symbol_count: number;
29
+ last_modified: number;
30
+ mtime_ms: number | null;
31
+ stale: number | null;
32
+ };
33
+ export declare function rowToFileEntry(row: FileRow): FileEntry;
34
+ export declare function fileEntryToRow(entry: FileEntry): unknown[];
35
+ export declare const INSERT_SYMBOL_SQL = "\nINSERT INTO symbols (\n id, file, name, kind, start_line, end_line, start_col, end_col,\n start_byte, end_byte, signature, docstring, source, parent,\n is_async, is_exported, extras\n) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)\n";
36
+ export declare const INSERT_FILE_SQL = "\nINSERT INTO files (path, language, symbol_count, last_modified, mtime_ms, stale)\nVALUES (?,?,?,?,?,?)\nON CONFLICT(path) DO UPDATE SET\n language=excluded.language, symbol_count=excluded.symbol_count,\n last_modified=excluded.last_modified, mtime_ms=excluded.mtime_ms,\n stale=excluded.stale\n";
37
+ /**
38
+ * Fixed heap cost of one materialised symbol, on top of its text.
39
+ *
40
+ * Calibrated against the real tgm-survey-platform index (240,137 symbols): loading it moves
41
+ * `heapUsed` by 349 MB, of which the summed string lengths account for ~221 MB, leaving ~560 B
42
+ * per symbol for the object header, property slots and array pointers. Counting only the text
43
+ * would therefore under-report the cache by more than a third.
44
+ *
45
+ * Rounded UP from the fitted 560, deliberately. The constant is fitted to one index and other
46
+ * repos will differ, so the residual error should fall on the safe side: over-reporting evicts a
47
+ * little sooner than strictly necessary, whereas under-reporting lets the cache quietly exceed
48
+ * the budget it exists to enforce. Measured overshoot on the calibration index is ~11%.
49
+ */
50
+ export declare const SYMBOL_OBJECT_OVERHEAD_BYTES = 700;
51
+ export declare const FILE_OBJECT_OVERHEAD_BYTES = 200;
52
+ /**
53
+ * Byte cost of one materialised file entry.
54
+ *
55
+ * Counted with `textBytes` and including `language`, not `path.length` alone. The caller used to
56
+ * do the latter inline, which under-reported on exactly the two axes the comment above rules out:
57
+ * a non-ASCII path counts half its bytes, and `language` counted zero. Small per row — but a
58
+ * budget that is wrong in the permissive direction is the failure mode the budget exists to stop,
59
+ * and having one of the two footprint helpers contradict the other's stated rule is worse than
60
+ * either number.
61
+ */
62
+ export declare function fileRowBytes(row: FileRow): number;
63
+ export declare function symbolRowBytes(row: SymbolRow): number;
64
+ //# sourceMappingURL=rows.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"rows.d.ts","sourceRoot":"","sources":["../../../src/storage/sqlite/rows.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAwB5D,MAAM,MAAM,SAAS,GAAG;IACtB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;CACvB,CAAC;AAEF,wBAAgB,WAAW,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,EAAE,CA4BtD;AAED;wDACwD;AACxD,wBAAgB,WAAW,CAAC,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,GAAG,UAAU,CAgCpE;AAED,MAAM,MAAM,OAAO,GAAG;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CACtB,CAAC;AAEF,wBAAgB,cAAc,CAAC,GAAG,EAAE,OAAO,GAAG,SAAS,CAUtD;AAED,wBAAgB,cAAc,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,EAAE,CAS1D;AAED,eAAO,MAAM,iBAAiB,+OAM7B,CAAC;AAMF,eAAO,MAAM,eAAe,gTAO3B,CAAC;AAEF;;;;;;;;;;;;GAYG;AACH,eAAO,MAAM,4BAA4B,MAAM,CAAC;AAChD,eAAO,MAAM,0BAA0B,MAAM,CAAC;AAoB9C;;;;;;;;;GASG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,OAAO,GAAG,MAAM,CAEjD;AAED,wBAAgB,cAAc,CAAC,GAAG,EAAE,SAAS,GAAG,MAAM,CAkBrD"}
@@ -0,0 +1,186 @@
1
+ export function symbolToRow(sym) {
2
+ const extras = {};
3
+ if (sym.tokens !== undefined)
4
+ extras.tokens = sym.tokens;
5
+ if (sym.decorators !== undefined)
6
+ extras.decorators = sym.decorators;
7
+ if (sym.extends !== undefined)
8
+ extras.extends = sym.extends;
9
+ if (sym.implements !== undefined)
10
+ extras.implements = sym.implements;
11
+ if (sym.meta !== undefined)
12
+ extras.meta = sym.meta;
13
+ const hasExtras = Object.keys(extras).length > 0;
14
+ return [
15
+ sym.id,
16
+ sym.file,
17
+ sym.name,
18
+ sym.kind,
19
+ sym.start_line,
20
+ sym.end_line,
21
+ sym.start_col ?? null,
22
+ sym.end_col ?? null,
23
+ sym.start_byte ?? null,
24
+ sym.end_byte ?? null,
25
+ sym.signature ?? null,
26
+ sym.docstring ?? null,
27
+ sym.source ?? null,
28
+ sym.parent ?? null,
29
+ sym.is_async === undefined ? null : sym.is_async ? 1 : 0,
30
+ sym.is_exported === undefined ? null : sym.is_exported ? 1 : 0,
31
+ hasExtras ? JSON.stringify(extras) : null,
32
+ ];
33
+ }
34
+ /** `repo` is identical for every symbol in an index, so it lives in `meta` and is stamped
35
+ * back on here instead of being written 100k times. */
36
+ export function rowToSymbol(row, repo) {
37
+ const sym = {
38
+ id: row.id,
39
+ repo,
40
+ name: row.name,
41
+ kind: row.kind,
42
+ file: row.file,
43
+ start_line: row.start_line,
44
+ end_line: row.end_line,
45
+ };
46
+ if (row.start_col !== null)
47
+ sym.start_col = row.start_col;
48
+ if (row.end_col !== null)
49
+ sym.end_col = row.end_col;
50
+ if (row.start_byte !== null)
51
+ sym.start_byte = row.start_byte;
52
+ if (row.end_byte !== null)
53
+ sym.end_byte = row.end_byte;
54
+ if (row.signature !== null)
55
+ sym.signature = row.signature;
56
+ if (row.docstring !== null)
57
+ sym.docstring = row.docstring;
58
+ if (row.source !== null)
59
+ sym.source = row.source;
60
+ if (row.parent !== null)
61
+ sym.parent = row.parent;
62
+ if (row.is_async !== null)
63
+ sym.is_async = row.is_async === 1;
64
+ if (row.is_exported !== null)
65
+ sym.is_exported = row.is_exported === 1;
66
+ if (row.extras !== null) {
67
+ const extras = JSON.parse(row.extras);
68
+ if (extras.tokens !== undefined)
69
+ sym.tokens = extras.tokens;
70
+ if (extras.decorators !== undefined)
71
+ sym.decorators = extras.decorators;
72
+ if (extras.extends !== undefined)
73
+ sym.extends = extras.extends;
74
+ if (extras.implements !== undefined)
75
+ sym.implements = extras.implements;
76
+ if (extras.meta !== undefined)
77
+ sym.meta = extras.meta;
78
+ }
79
+ return sym;
80
+ }
81
+ export function rowToFileEntry(row) {
82
+ const entry = {
83
+ path: row.path,
84
+ language: row.language,
85
+ symbol_count: row.symbol_count,
86
+ last_modified: row.last_modified,
87
+ };
88
+ if (row.mtime_ms !== null)
89
+ entry.mtime_ms = row.mtime_ms;
90
+ if (row.stale !== null)
91
+ entry.stale = row.stale === 1;
92
+ return entry;
93
+ }
94
+ export function fileEntryToRow(entry) {
95
+ return [
96
+ entry.path,
97
+ entry.language,
98
+ entry.symbol_count,
99
+ entry.last_modified,
100
+ entry.mtime_ms ?? null,
101
+ entry.stale === undefined ? null : entry.stale ? 1 : 0,
102
+ ];
103
+ }
104
+ export const INSERT_SYMBOL_SQL = `
105
+ INSERT INTO symbols (
106
+ id, file, name, kind, start_line, end_line, start_col, end_col,
107
+ start_byte, end_byte, signature, docstring, source, parent,
108
+ is_async, is_exported, extras
109
+ ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
110
+ `;
111
+ // No upsert: both writers (`writeIndexRows`, `saveIncrementalSqlite`) DELETE the rows they
112
+ // are about to replace, so the only thing an ON CONFLICT clause ever resolved was a
113
+ // collision WITHIN one payload — two distinct symbols sharing a non-unique id — by throwing
114
+ // one of them away. See SCHEMA_VERSION.
115
+ export const INSERT_FILE_SQL = `
116
+ INSERT INTO files (path, language, symbol_count, last_modified, mtime_ms, stale)
117
+ VALUES (?,?,?,?,?,?)
118
+ ON CONFLICT(path) DO UPDATE SET
119
+ language=excluded.language, symbol_count=excluded.symbol_count,
120
+ last_modified=excluded.last_modified, mtime_ms=excluded.mtime_ms,
121
+ stale=excluded.stale
122
+ `;
123
+ /**
124
+ * Fixed heap cost of one materialised symbol, on top of its text.
125
+ *
126
+ * Calibrated against the real tgm-survey-platform index (240,137 symbols): loading it moves
127
+ * `heapUsed` by 349 MB, of which the summed string lengths account for ~221 MB, leaving ~560 B
128
+ * per symbol for the object header, property slots and array pointers. Counting only the text
129
+ * would therefore under-report the cache by more than a third.
130
+ *
131
+ * Rounded UP from the fitted 560, deliberately. The constant is fitted to one index and other
132
+ * repos will differ, so the residual error should fall on the safe side: over-reporting evicts a
133
+ * little sooner than strictly necessary, whereas under-reporting lets the cache quietly exceed
134
+ * the budget it exists to enforce. Measured overshoot on the calibration index is ~11%.
135
+ */
136
+ export const SYMBOL_OBJECT_OVERHEAD_BYTES = 700;
137
+ export const FILE_OBJECT_OVERHEAD_BYTES = 200;
138
+ /**
139
+ * Byte cost of a string field that may hold prose.
140
+ *
141
+ * `String.length` counts UTF-16 code units, not bytes. V8 stores a string one byte per character
142
+ * only while every character fits Latin1; one non-Latin1 character anywhere makes the whole string
143
+ * two-byte. So `.length` under-reports CJK, Cyrillic and emoji-bearing text by about half — and
144
+ * under-reporting is the one direction the calibration comment below rules out, because it lets
145
+ * the cache exceed the budget it exists to enforce. `source` alone is 45% of the footprint
146
+ * (ADR-004), so a repo commented in Chinese would quietly blow through the cap.
147
+ *
148
+ * `Buffer.byteLength` over-reports instead (3 UTF-8 bytes per CJK character against V8's 2), which
149
+ * is the acceptable side.
150
+ */
151
+ function textBytes(value) {
152
+ if (value === null || value === undefined || value === "")
153
+ return 0;
154
+ return Buffer.byteLength(value, "utf8");
155
+ }
156
+ /**
157
+ * Byte cost of one materialised file entry.
158
+ *
159
+ * Counted with `textBytes` and including `language`, not `path.length` alone. The caller used to
160
+ * do the latter inline, which under-reported on exactly the two axes the comment above rules out:
161
+ * a non-ASCII path counts half its bytes, and `language` counted zero. Small per row — but a
162
+ * budget that is wrong in the permissive direction is the failure mode the budget exists to stop,
163
+ * and having one of the two footprint helpers contradict the other's stated rule is worse than
164
+ * either number.
165
+ */
166
+ export function fileRowBytes(row) {
167
+ return FILE_OBJECT_OVERHEAD_BYTES + textBytes(row.path) + textBytes(row.language);
168
+ }
169
+ export function symbolRowBytes(row) {
170
+ return (SYMBOL_OBJECT_OVERHEAD_BYTES +
171
+ // Identifiers and paths: overwhelmingly ASCII, and there are 240k of them — `.length` is
172
+ // exact for ASCII and skips a scan per row on the fields least likely to need one.
173
+ row.id.length +
174
+ row.file.length +
175
+ row.name.length +
176
+ row.kind.length +
177
+ // Prose and code text: the fields that actually carry non-Latin1 content. `parent` is a
178
+ // symbol name and so is ASCII in practice, but it is measured the same way as the rest —
179
+ // splitting it off saved one scan per row and cost a reader having to ask why.
180
+ textBytes(row.parent) +
181
+ textBytes(row.signature) +
182
+ textBytes(row.docstring) +
183
+ textBytes(row.source) +
184
+ textBytes(row.extras));
185
+ }
186
+ //# sourceMappingURL=rows.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"rows.js","sourceRoot":"","sources":["../../../src/storage/sqlite/rows.ts"],"names":[],"mappings":"AA4CA,MAAM,UAAU,WAAW,CAAC,GAAe;IACzC,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS;QAAE,MAAM,CAAC,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC;IACzD,IAAI,GAAG,CAAC,UAAU,KAAK,SAAS;QAAE,MAAM,CAAC,UAAU,GAAG,GAAG,CAAC,UAAU,CAAC;IACrE,IAAI,GAAG,CAAC,OAAO,KAAK,SAAS;QAAE,MAAM,CAAC,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC;IAC5D,IAAI,GAAG,CAAC,UAAU,KAAK,SAAS;QAAE,MAAM,CAAC,UAAU,GAAG,GAAG,CAAC,UAAU,CAAC;IACrE,IAAI,GAAG,CAAC,IAAI,KAAK,SAAS;QAAE,MAAM,CAAC,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IACnD,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IAEjD,OAAO;QACL,GAAG,CAAC,EAAE;QACN,GAAG,CAAC,IAAI;QACR,GAAG,CAAC,IAAI;QACR,GAAG,CAAC,IAAI;QACR,GAAG,CAAC,UAAU;QACd,GAAG,CAAC,QAAQ;QACZ,GAAG,CAAC,SAAS,IAAI,IAAI;QACrB,GAAG,CAAC,OAAO,IAAI,IAAI;QACnB,GAAG,CAAC,UAAU,IAAI,IAAI;QACtB,GAAG,CAAC,QAAQ,IAAI,IAAI;QACpB,GAAG,CAAC,SAAS,IAAI,IAAI;QACrB,GAAG,CAAC,SAAS,IAAI,IAAI;QACrB,GAAG,CAAC,MAAM,IAAI,IAAI;QAClB,GAAG,CAAC,MAAM,IAAI,IAAI;QAClB,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACxD,GAAG,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9D,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI;KAC1C,CAAC;AACJ,CAAC;AAED;wDACwD;AACxD,MAAM,UAAU,WAAW,CAAC,GAAc,EAAE,IAAY;IACtD,MAAM,GAAG,GAAe;QACtB,EAAE,EAAE,GAAG,CAAC,EAAE;QACV,IAAI;QACJ,IAAI,EAAE,GAAG,CAAC,IAAI;QACd,IAAI,EAAE,GAAG,CAAC,IAA0B;QACpC,IAAI,EAAE,GAAG,CAAC,IAAI;QACd,UAAU,EAAE,GAAG,CAAC,UAAU;QAC1B,QAAQ,EAAE,GAAG,CAAC,QAAQ;KACvB,CAAC;IAEF,IAAI,GAAG,CAAC,SAAS,KAAK,IAAI;QAAE,GAAG,CAAC,SAAS,GAAG,GAAG,CAAC,SAAS,CAAC;IAC1D,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI;QAAE,GAAG,CAAC,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC;IACpD,IAAI,GAAG,CAAC,UAAU,KAAK,IAAI;QAAE,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC,UAAU,CAAC;IAC7D,IAAI,GAAG,CAAC,QAAQ,KAAK,IAAI;QAAE,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC;IACvD,IAAI,GAAG,CAAC,SAAS,KAAK,IAAI;QAAE,GAAG,CAAC,SAAS,GAAG,GAAG,CAAC,SAAS,CAAC;IAC1D,IAAI,GAAG,CAAC,SAAS,KAAK,IAAI;QAAE,GAAG,CAAC,SAAS,GAAG,GAAG,CAAC,SAAS,CAAC;IAC1D,IAAI,GAAG,CAAC,MAAM,KAAK,IAAI;QAAE,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC;IACjD,IAAI,GAAG,CAAC,MAAM,KAAK,IAAI;QAAE,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC;IACjD,IAAI,GAAG,CAAC,QAAQ,KAAK,IAAI;QAAE,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,QAAQ,KAAK,CAAC,CAAC;IAC7D,IAAI,GAAG,CAAC,WAAW,KAAK,IAAI;QAAE,GAAG,CAAC,WAAW,GAAG,GAAG,CAAC,WAAW,KAAK,CAAC,CAAC;IAEtE,IAAI,GAAG,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC;QACxB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAiB,CAAC;QACtD,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS;YAAE,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAC5D,IAAI,MAAM,CAAC,UAAU,KAAK,SAAS;YAAE,GAAG,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;QACxE,IAAI,MAAM,CAAC,OAAO,KAAK,SAAS;YAAE,GAAG,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QAC/D,IAAI,MAAM,CAAC,UAAU,KAAK,SAAS;YAAE,GAAG,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;QACxE,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS;YAAE,GAAG,CAAC,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;IACxD,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC;AAWD,MAAM,UAAU,cAAc,CAAC,GAAY;IACzC,MAAM,KAAK,GAAc;QACvB,IAAI,EAAE,GAAG,CAAC,IAAI;QACd,QAAQ,EAAE,GAAG,CAAC,QAAQ;QACtB,YAAY,EAAE,GAAG,CAAC,YAAY;QAC9B,aAAa,EAAE,GAAG,CAAC,aAAa;KACjC,CAAC;IACF,IAAI,GAAG,CAAC,QAAQ,KAAK,IAAI;QAAE,KAAK,CAAC,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC;IACzD,IAAI,GAAG,CAAC,KAAK,KAAK,IAAI;QAAE,KAAK,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,KAAK,CAAC,CAAC;IACtD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,KAAgB;IAC7C,OAAO;QACL,KAAK,CAAC,IAAI;QACV,KAAK,CAAC,QAAQ;QACd,KAAK,CAAC,YAAY;QAClB,KAAK,CAAC,aAAa;QACnB,KAAK,CAAC,QAAQ,IAAI,IAAI;QACtB,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;KACvD,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,MAAM,iBAAiB,GAAG;;;;;;CAMhC,CAAC;AACF,2FAA2F;AAC3F,oFAAoF;AACpF,4FAA4F;AAC5F,wCAAwC;AAExC,MAAM,CAAC,MAAM,eAAe,GAAG;;;;;;;CAO9B,CAAC;AAEF;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,MAAM,4BAA4B,GAAG,GAAG,CAAC;AAChD,MAAM,CAAC,MAAM,0BAA0B,GAAG,GAAG,CAAC;AAE9C;;;;;;;;;;;;GAYG;AACH,SAAS,SAAS,CAAC,KAAgC;IACjD,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,EAAE;QAAE,OAAO,CAAC,CAAC;IACpE,OAAO,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;AAC1C,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,YAAY,CAAC,GAAY;IACvC,OAAO,0BAA0B,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;AACpF,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,GAAc;IAC3C,OAAO,CACL,4BAA4B;QAC5B,yFAAyF;QACzF,mFAAmF;QACnF,GAAG,CAAC,EAAE,CAAC,MAAM;QACb,GAAG,CAAC,IAAI,CAAC,MAAM;QACf,GAAG,CAAC,IAAI,CAAC,MAAM;QACf,GAAG,CAAC,IAAI,CAAC,MAAM;QACf,wFAAwF;QACxF,yFAAyF;QACzF,+EAA+E;QAC/E,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC;QACrB,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC;QACxB,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC;QACxB,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC;QACrB,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CACtB,CAAC;AACJ,CAAC"}
@@ -0,0 +1,6 @@
1
+ import type { DatabaseSync as DatabaseSyncType } from "node:sqlite";
2
+ export declare function loadSqliteCtor(): Promise<typeof DatabaseSyncType | null>;
3
+ export declare function isSqliteAvailable(): Promise<boolean>;
4
+ /** Tests need to prove the JSON fallback still works on a runtime that has sqlite. */
5
+ export declare function setSqliteCtorForTesting(ctor: typeof DatabaseSyncType | null | undefined): void;
6
+ //# sourceMappingURL=runtime.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"runtime.d.ts","sourceRoot":"","sources":["../../../src/storage/sqlite/runtime.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,IAAI,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAepE,wBAAsB,cAAc,IAAI,OAAO,CAAC,OAAO,gBAAgB,GAAG,IAAI,CAAC,CAS9E;AAED,wBAAsB,iBAAiB,IAAI,OAAO,CAAC,OAAO,CAAC,CAE1D;AAED,sFAAsF;AACtF,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,OAAO,gBAAgB,GAAG,IAAI,GAAG,SAAS,GAAG,IAAI,CAE9F"}