@colbymchenry/codegraph-darwin-arm64 1.3.1 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/dist/db/index.d.ts +47 -0
- package/lib/dist/db/index.d.ts.map +1 -1
- package/lib/dist/db/index.js +114 -12
- package/lib/dist/db/index.js.map +1 -1
- package/lib/dist/db/wal-valve.d.ts +106 -0
- package/lib/dist/db/wal-valve.d.ts.map +1 -0
- package/lib/dist/db/wal-valve.js +208 -0
- package/lib/dist/db/wal-valve.js.map +1 -0
- package/lib/dist/extraction/grammars.d.ts +16 -1
- package/lib/dist/extraction/grammars.d.ts.map +1 -1
- package/lib/dist/extraction/grammars.js +85 -43
- package/lib/dist/extraction/grammars.js.map +1 -1
- package/lib/dist/extraction/index.d.ts +1 -1
- package/lib/dist/extraction/index.d.ts.map +1 -1
- package/lib/dist/extraction/index.js +29 -6
- package/lib/dist/extraction/index.js.map +1 -1
- package/lib/dist/extraction/parse-pool.d.ts +27 -1
- package/lib/dist/extraction/parse-pool.d.ts.map +1 -1
- package/lib/dist/extraction/parse-pool.js +75 -7
- package/lib/dist/extraction/parse-pool.js.map +1 -1
- package/lib/dist/extraction/parse-worker.js +9 -2
- package/lib/dist/extraction/parse-worker.js.map +1 -1
- package/lib/dist/index.d.ts.map +1 -1
- package/lib/dist/index.js +49 -1
- package/lib/dist/index.js.map +1 -1
- package/lib/dist/resolution/callback-synthesizer.d.ts.map +1 -1
- package/lib/dist/resolution/callback-synthesizer.js +59 -8
- package/lib/dist/resolution/callback-synthesizer.js.map +1 -1
- package/lib/node_modules/.package-lock.json +1 -1
- package/lib/package.json +1 -1
- package/package.json +1 -1
package/lib/dist/db/index.d.ts
CHANGED
|
@@ -68,6 +68,44 @@ export declare class DatabaseConnection {
|
|
|
68
68
|
* Get database file size in bytes
|
|
69
69
|
*/
|
|
70
70
|
getSize(): number;
|
|
71
|
+
/**
|
|
72
|
+
* Size of the `-wal` sidecar file in bytes. 0 when it doesn't exist (non-WAL
|
|
73
|
+
* journal mode, in-memory DB, or no write since the last checkpoint+reset).
|
|
74
|
+
*/
|
|
75
|
+
getWalSizeBytes(): number;
|
|
76
|
+
/** Current `wal_autocheckpoint` interval in pages (0 = disabled). */
|
|
77
|
+
getWalAutocheckpoint(): number;
|
|
78
|
+
/**
|
|
79
|
+
* Set the connection's `wal_autocheckpoint` interval (pages; 0 disables).
|
|
80
|
+
* Bulk indexing defers checkpoints entirely (#1231): the default 1000-page
|
|
81
|
+
* auto-checkpoint re-writes hot B-tree/FTS pages into the main DB file over
|
|
82
|
+
* and over — measured at ~95% of ALL disk I/O during a bulk index, and the
|
|
83
|
+
* difference between 45s and 19+ minutes on HDD-class storage. During
|
|
84
|
+
* deferral a {@link WalCheckpointValve} bounds WAL growth off-thread.
|
|
85
|
+
*/
|
|
86
|
+
setWalAutocheckpoint(pages: number): void;
|
|
87
|
+
/**
|
|
88
|
+
* `PRAGMA wal_checkpoint(PASSIVE)` on a worker thread with its own
|
|
89
|
+
* connection. PASSIVE never blocks the writer, and running it off-thread
|
|
90
|
+
* means the main thread — and the #850 watchdog heartbeat — keep turning
|
|
91
|
+
* even when the backfill is minutes of I/O on slow storage (a synchronous
|
|
92
|
+
* checkpoint that exceeds the watchdog's 60s window gets a healthy index
|
|
93
|
+
* SIGKILLed — observed in the #1231 repro).
|
|
94
|
+
*
|
|
95
|
+
* Returns SQLite's checkpoint result row — `log === checkpointed` with
|
|
96
|
+
* `busy === 0` means the ENTIRE WAL was backfilled, so the writer's next
|
|
97
|
+
* commit restarts the WAL from the top and the file stops growing. The
|
|
98
|
+
* WAL valve needs that signal because a WAL file's SIZE never shrinks:
|
|
99
|
+
* after the first wrap, raw file size says nothing about the un-backfilled
|
|
100
|
+
* backlog. Best-effort: returns null on any failure (including worker
|
|
101
|
+
* threads being unavailable — a potentially minutes-long checkpoint must
|
|
102
|
+
* never run inline on the main thread).
|
|
103
|
+
*/
|
|
104
|
+
checkpointWalPassive(): Promise<{
|
|
105
|
+
busy: number;
|
|
106
|
+
log: number;
|
|
107
|
+
checkpointed: number;
|
|
108
|
+
} | null>;
|
|
71
109
|
/**
|
|
72
110
|
* Optimize database (vacuum and analyze)
|
|
73
111
|
*/
|
|
@@ -102,6 +140,15 @@ export declare class DatabaseConnection {
|
|
|
102
140
|
* CLI has already disarmed its watchdog).
|
|
103
141
|
*/
|
|
104
142
|
runMaintenance(): Promise<void>;
|
|
143
|
+
/**
|
|
144
|
+
* Run pragmas on a worker thread against its own connection to this DB
|
|
145
|
+
* (shared machinery for {@link runMaintenance} and
|
|
146
|
+
* {@link checkpointWalPassive}). Each pragma is individually best-effort;
|
|
147
|
+
* the whole call is best-effort. `inlineFallback` (if any) runs on THIS
|
|
148
|
+
* connection only when worker threads are unavailable — keep it to pragmas
|
|
149
|
+
* that are safe to run synchronously on the main thread.
|
|
150
|
+
*/
|
|
151
|
+
private runPragmasOffThread;
|
|
105
152
|
/**
|
|
106
153
|
* Close the database connection
|
|
107
154
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/db/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,cAAc,EAAE,aAAa,EAAkB,MAAM,kBAAkB,CAAC;AAGjF,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAIzC,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AA0BjE;;GAEG;AACH,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,EAAE,CAAiB;IAC3B,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,OAAO,CAAgB;IAC/B;;;;;;;OAOG;IACH,OAAO,CAAC,WAAW,CAAgB;IAEnC,OAAO;IAOP;;OAEG;IACH,MAAM,CAAC,UAAU,CAAC,MAAM,EAAE,MAAM,GAAG,kBAAkB;IA4BrD;;OAEG;IACH,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,kBAAkB;IAoB/C;;OAEG;IACH,KAAK,IAAI,cAAc;IAIvB;;;;OAIG;IACH,UAAU,IAAI,aAAa;IAI3B;;OAEG;IACH,OAAO,IAAI,MAAM;IAIjB;;;;;;;;;OASG;IACH,cAAc,IAAI,MAAM;IASxB;;OAEG;IACH,gBAAgB,IAAI,aAAa,GAAG,IAAI;IAcxC;;OAEG;IACH,WAAW,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC;IAI9B;;OAEG;IACH,OAAO,IAAI,MAAM;IAKjB;;OAEG;IACH,QAAQ,IAAI,IAAI;IAKhB;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;IACG,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/db/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,cAAc,EAAE,aAAa,EAAkB,MAAM,kBAAkB,CAAC;AAGjF,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAIzC,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AA0BjE;;GAEG;AACH,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,EAAE,CAAiB;IAC3B,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,OAAO,CAAgB;IAC/B;;;;;;;OAOG;IACH,OAAO,CAAC,WAAW,CAAgB;IAEnC,OAAO;IAOP;;OAEG;IACH,MAAM,CAAC,UAAU,CAAC,MAAM,EAAE,MAAM,GAAG,kBAAkB;IA4BrD;;OAEG;IACH,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,kBAAkB;IAoB/C;;OAEG;IACH,KAAK,IAAI,cAAc;IAIvB;;;;OAIG;IACH,UAAU,IAAI,aAAa;IAI3B;;OAEG;IACH,OAAO,IAAI,MAAM;IAIjB;;;;;;;;;OASG;IACH,cAAc,IAAI,MAAM;IASxB;;OAEG;IACH,gBAAgB,IAAI,aAAa,GAAG,IAAI;IAcxC;;OAEG;IACH,WAAW,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC;IAI9B;;OAEG;IACH,OAAO,IAAI,MAAM;IAKjB;;;OAGG;IACH,eAAe,IAAI,MAAM;IASzB,qEAAqE;IACrE,oBAAoB,IAAI,MAAM;IAM9B;;;;;;;OAOG;IACH,oBAAoB,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAIzC;;;;;;;;;;;;;;;;OAgBG;IACG,oBAAoB,IAAI,OAAO,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;IA2CjG;;OAEG;IACH,QAAQ,IAAI,IAAI;IAKhB;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;IACG,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;IAcrC;;;;;;;OAOG;YACW,mBAAmB;IAkCjC;;OAEG;IACH,KAAK,IAAI,IAAI;IAIb;;OAEG;IACH,MAAM,IAAI,OAAO;IAIjB;;;;;;;;;;OAUG;IACH,gBAAgB,IAAI,OAAO;CAK5B;AAmBD;;GAEG;AACH,eAAO,MAAM,iBAAiB,iBAAiB,CAAC;AAShD;;GAEG;AACH,wBAAgB,eAAe,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM,CAE3D;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAWxD"}
|
package/lib/dist/db/index.js
CHANGED
|
@@ -197,6 +197,100 @@ class DatabaseConnection {
|
|
|
197
197
|
const stats = fs.statSync(this.dbPath);
|
|
198
198
|
return stats.size;
|
|
199
199
|
}
|
|
200
|
+
/**
|
|
201
|
+
* Size of the `-wal` sidecar file in bytes. 0 when it doesn't exist (non-WAL
|
|
202
|
+
* journal mode, in-memory DB, or no write since the last checkpoint+reset).
|
|
203
|
+
*/
|
|
204
|
+
getWalSizeBytes() {
|
|
205
|
+
if (!this.dbPath || this.dbPath === ':memory:')
|
|
206
|
+
return 0;
|
|
207
|
+
try {
|
|
208
|
+
return fs.statSync(`${this.dbPath}-wal`).size;
|
|
209
|
+
}
|
|
210
|
+
catch {
|
|
211
|
+
return 0;
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
/** Current `wal_autocheckpoint` interval in pages (0 = disabled). */
|
|
215
|
+
getWalAutocheckpoint() {
|
|
216
|
+
const v = this.db.pragma('wal_autocheckpoint', { simple: true });
|
|
217
|
+
const n = Number(v);
|
|
218
|
+
return Number.isFinite(n) ? n : 0;
|
|
219
|
+
}
|
|
220
|
+
/**
|
|
221
|
+
* Set the connection's `wal_autocheckpoint` interval (pages; 0 disables).
|
|
222
|
+
* Bulk indexing defers checkpoints entirely (#1231): the default 1000-page
|
|
223
|
+
* auto-checkpoint re-writes hot B-tree/FTS pages into the main DB file over
|
|
224
|
+
* and over — measured at ~95% of ALL disk I/O during a bulk index, and the
|
|
225
|
+
* difference between 45s and 19+ minutes on HDD-class storage. During
|
|
226
|
+
* deferral a {@link WalCheckpointValve} bounds WAL growth off-thread.
|
|
227
|
+
*/
|
|
228
|
+
setWalAutocheckpoint(pages) {
|
|
229
|
+
this.db.pragma(`wal_autocheckpoint = ${Math.max(0, Math.floor(pages))}`);
|
|
230
|
+
}
|
|
231
|
+
/**
|
|
232
|
+
* `PRAGMA wal_checkpoint(PASSIVE)` on a worker thread with its own
|
|
233
|
+
* connection. PASSIVE never blocks the writer, and running it off-thread
|
|
234
|
+
* means the main thread — and the #850 watchdog heartbeat — keep turning
|
|
235
|
+
* even when the backfill is minutes of I/O on slow storage (a synchronous
|
|
236
|
+
* checkpoint that exceeds the watchdog's 60s window gets a healthy index
|
|
237
|
+
* SIGKILLed — observed in the #1231 repro).
|
|
238
|
+
*
|
|
239
|
+
* Returns SQLite's checkpoint result row — `log === checkpointed` with
|
|
240
|
+
* `busy === 0` means the ENTIRE WAL was backfilled, so the writer's next
|
|
241
|
+
* commit restarts the WAL from the top and the file stops growing. The
|
|
242
|
+
* WAL valve needs that signal because a WAL file's SIZE never shrinks:
|
|
243
|
+
* after the first wrap, raw file size says nothing about the un-backfilled
|
|
244
|
+
* backlog. Best-effort: returns null on any failure (including worker
|
|
245
|
+
* threads being unavailable — a potentially minutes-long checkpoint must
|
|
246
|
+
* never run inline on the main thread).
|
|
247
|
+
*/
|
|
248
|
+
async checkpointWalPassive() {
|
|
249
|
+
if (!this.dbPath || this.dbPath === ':memory:') {
|
|
250
|
+
try {
|
|
251
|
+
const row = this.db.prepare('PRAGMA wal_checkpoint(PASSIVE)').get();
|
|
252
|
+
return row ? { busy: Number(row.busy), log: Number(row.log), checkpointed: Number(row.checkpointed) } : null;
|
|
253
|
+
}
|
|
254
|
+
catch {
|
|
255
|
+
return null;
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
try {
|
|
259
|
+
const { Worker } = await Promise.resolve().then(() => __importStar(require('node:worker_threads')));
|
|
260
|
+
const workerSource = `
|
|
261
|
+
const { workerData, parentPort } = require('node:worker_threads');
|
|
262
|
+
let row = null;
|
|
263
|
+
try {
|
|
264
|
+
const { DatabaseSync } = require('node:sqlite');
|
|
265
|
+
const db = new DatabaseSync(workerData.dbPath);
|
|
266
|
+
try { row = db.prepare('PRAGMA wal_checkpoint(PASSIVE)').get(); } catch {}
|
|
267
|
+
try { db.close(); } catch {}
|
|
268
|
+
} catch {}
|
|
269
|
+
parentPort.postMessage({ row });
|
|
270
|
+
`;
|
|
271
|
+
return await new Promise((resolve) => {
|
|
272
|
+
let settled = false;
|
|
273
|
+
const finish = (row) => {
|
|
274
|
+
if (settled)
|
|
275
|
+
return;
|
|
276
|
+
settled = true;
|
|
277
|
+
resolve(row ? { busy: Number(row.busy), log: Number(row.log), checkpointed: Number(row.checkpointed) } : null);
|
|
278
|
+
};
|
|
279
|
+
try {
|
|
280
|
+
const worker = new Worker(workerSource, { eval: true, workerData: { dbPath: this.dbPath } });
|
|
281
|
+
worker.once('message', (m) => { void worker.terminate(); finish(m?.row ?? null); });
|
|
282
|
+
worker.once('error', () => { void worker.terminate(); finish(null); });
|
|
283
|
+
worker.once('exit', () => finish(null));
|
|
284
|
+
}
|
|
285
|
+
catch {
|
|
286
|
+
finish(null);
|
|
287
|
+
}
|
|
288
|
+
});
|
|
289
|
+
}
|
|
290
|
+
catch {
|
|
291
|
+
return null;
|
|
292
|
+
}
|
|
293
|
+
}
|
|
200
294
|
/**
|
|
201
295
|
* Optimize database (vacuum and analyze)
|
|
202
296
|
*/
|
|
@@ -246,6 +340,19 @@ class DatabaseConnection {
|
|
|
246
340
|
catch { /* ignore */ }
|
|
247
341
|
return;
|
|
248
342
|
}
|
|
343
|
+
await this.runPragmasOffThread(['PRAGMA analysis_limit=1000', 'PRAGMA optimize', 'PRAGMA wal_checkpoint(PASSIVE)'],
|
|
344
|
+
// Worker threads unavailable — bounded in-line fallback, no checkpoint.
|
|
345
|
+
['PRAGMA analysis_limit=1000', 'PRAGMA optimize']);
|
|
346
|
+
}
|
|
347
|
+
/**
|
|
348
|
+
* Run pragmas on a worker thread against its own connection to this DB
|
|
349
|
+
* (shared machinery for {@link runMaintenance} and
|
|
350
|
+
* {@link checkpointWalPassive}). Each pragma is individually best-effort;
|
|
351
|
+
* the whole call is best-effort. `inlineFallback` (if any) runs on THIS
|
|
352
|
+
* connection only when worker threads are unavailable — keep it to pragmas
|
|
353
|
+
* that are safe to run synchronously on the main thread.
|
|
354
|
+
*/
|
|
355
|
+
async runPragmasOffThread(pragmas, inlineFallback = []) {
|
|
249
356
|
try {
|
|
250
357
|
const { Worker } = await Promise.resolve().then(() => __importStar(require('node:worker_threads')));
|
|
251
358
|
const workerSource = `
|
|
@@ -253,9 +360,7 @@ class DatabaseConnection {
|
|
|
253
360
|
try {
|
|
254
361
|
const { DatabaseSync } = require('node:sqlite');
|
|
255
362
|
const db = new DatabaseSync(workerData.dbPath);
|
|
256
|
-
try { db.exec(
|
|
257
|
-
try { db.exec('PRAGMA optimize'); } catch {}
|
|
258
|
-
try { db.exec('PRAGMA wal_checkpoint(PASSIVE)'); } catch {}
|
|
363
|
+
for (const p of workerData.pragmas) { try { db.exec(p); } catch {} }
|
|
259
364
|
try { db.close(); } catch {}
|
|
260
365
|
} catch {}
|
|
261
366
|
parentPort.postMessage('done');
|
|
@@ -269,7 +374,7 @@ class DatabaseConnection {
|
|
|
269
374
|
}
|
|
270
375
|
};
|
|
271
376
|
try {
|
|
272
|
-
const worker = new Worker(workerSource, { eval: true, workerData: { dbPath: this.dbPath } });
|
|
377
|
+
const worker = new Worker(workerSource, { eval: true, workerData: { dbPath: this.dbPath, pragmas } });
|
|
273
378
|
worker.once('message', () => { void worker.terminate(); finish(); });
|
|
274
379
|
worker.once('error', () => { void worker.terminate(); finish(); });
|
|
275
380
|
worker.once('exit', finish);
|
|
@@ -280,15 +385,12 @@ class DatabaseConnection {
|
|
|
280
385
|
});
|
|
281
386
|
}
|
|
282
387
|
catch {
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
try {
|
|
289
|
-
this.db.exec('PRAGMA optimize');
|
|
388
|
+
for (const p of inlineFallback) {
|
|
389
|
+
try {
|
|
390
|
+
this.db.exec(p);
|
|
391
|
+
}
|
|
392
|
+
catch { /* ignore */ }
|
|
290
393
|
}
|
|
291
|
-
catch { /* ignore */ }
|
|
292
394
|
}
|
|
293
395
|
}
|
|
294
396
|
/**
|
package/lib/dist/db/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/db/index.ts"],"names":[],"mappings":";AAAA;;;;GAIG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/db/index.ts"],"names":[],"mappings":";AAAA;;;;GAIG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAobH,0CAEC;AAqBD,kDAWC;AApdD,qDAAiF;AACjF,uCAAyB;AACzB,2CAA6B;AAE7B,6CAAwF;AACxF,4CAA+C;AAI/C;;;;;;;;;;;;;GAaG;AACH,SAAS,mBAAmB,CAAC,EAAkB;IAC7C,EAAE,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC,CAAM,4BAA4B;IACnE,EAAE,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC;IAC/B,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC,CAAO,6CAA6C;IACpF,EAAE,CAAC,MAAM,CAAC,sBAAsB,CAAC,CAAC,CAAK,qBAAqB;IAC5D,EAAE,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC,CAAM,mBAAmB;IAC1D,EAAE,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC,CAAM,wBAAwB;IAC/D,EAAE,CAAC,MAAM,CAAC,uBAAuB,CAAC,CAAC,CAAI,2BAA2B;AACpE,CAAC;AAED;;GAEG;AACH,MAAa,kBAAkB;IACrB,EAAE,CAAiB;IACnB,MAAM,CAAS;IACf,OAAO,CAAgB;IAC/B;;;;;;;OAOG;IACK,WAAW,CAAgB;IAEnC,YAAoB,EAAkB,EAAE,MAAc,EAAE,OAAsB;QAC5E,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,WAAW,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;IACvC,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,UAAU,CAAC,MAAc;QAC9B,iCAAiC;QACjC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YACxB,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACzC,CAAC;QAED,gCAAgC;QAChC,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,GAAG,IAAA,+BAAc,EAAC,MAAM,CAAC,CAAC;QAE/C,mBAAmB,CAAC,EAAE,CAAC,CAAC;QAExB,4BAA4B;QAC5B,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;QACtD,MAAM,MAAM,GAAG,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QACpD,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEhB,wEAAwE;QACxE,MAAM,cAAc,GAAG,IAAA,8BAAiB,EAAC,EAAE,CAAC,CAAC;QAC7C,IAAI,cAAc,GAAG,mCAAsB,EAAE,CAAC;YAC5C,EAAE,CAAC,OAAO,CACR,2FAA2F,CAC5F,CAAC,GAAG,CAAC,mCAAsB,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,wCAAwC,CAAC,CAAC;QACtF,CAAC;QAED,OAAO,IAAI,kBAAkB,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;IACrD,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,IAAI,CAAC,MAAc;QACxB,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,uBAAuB,MAAM,EAAE,CAAC,CAAC;QACnD,CAAC;QAED,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,GAAG,IAAA,+BAAc,EAAC,MAAM,CAAC,CAAC;QAE/C,mBAAmB,CAAC,EAAE,CAAC,CAAC;QAExB,qCAAqC;QACrC,MAAM,IAAI,GAAG,IAAI,kBAAkB,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;QACzD,MAAM,cAAc,GAAG,IAAA,8BAAiB,EAAC,EAAE,CAAC,CAAC;QAE7C,IAAI,cAAc,GAAG,mCAAsB,EAAE,CAAC;YAC5C,IAAA,0BAAa,EAAC,EAAE,EAAE,cAAc,CAAC,CAAC;QACpC,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACH,KAAK;QACH,OAAO,IAAI,CAAC,EAAE,CAAC;IACjB,CAAC;IAED;;;;OAIG;IACH,UAAU;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED;;OAEG;IACH,OAAO;QACL,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED;;;;;;;;;OASG;IACH,cAAc;QACZ,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;QAC3C,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;QAC9C,MAAM,IAAI,GAAG,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ;YACzC,CAAC,CAAE,GAA+B,CAAC,YAAY;YAC/C,CAAC,CAAC,GAAG,CAAC;QACR,OAAO,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;IAC1C,CAAC;IAED;;OAEG;IACH,gBAAgB;QACd,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE;aAChB,OAAO,CAAC,4FAA4F,CAAC;aACrG,GAAG,EAAqF,CAAC;QAE5F,IAAI,CAAC,GAAG;YAAE,OAAO,IAAI,CAAC;QAEtB,OAAO;YACL,OAAO,EAAE,GAAG,CAAC,OAAO;YACpB,SAAS,EAAE,GAAG,CAAC,UAAU;YACzB,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,SAAS;SAC1C,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,WAAW,CAAI,EAAW;QACxB,OAAO,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,EAAE,CAAC,EAAE,CAAC;IACnC,CAAC;IAED;;OAEG;IACH,OAAO;QACL,MAAM,KAAK,GAAG,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACvC,OAAO,KAAK,CAAC,IAAI,CAAC;IACpB,CAAC;IAED;;;OAGG;IACH,eAAe;QACb,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,KAAK,UAAU;YAAE,OAAO,CAAC,CAAC;QACzD,IAAI,CAAC;YACH,OAAO,EAAE,CAAC,QAAQ,CAAC,GAAG,IAAI,CAAC,MAAM,MAAM,CAAC,CAAC,IAAI,CAAC;QAChD,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,CAAC,CAAC;QACX,CAAC;IACH,CAAC;IAED,qEAAqE;IACrE,oBAAoB;QAClB,MAAM,CAAC,GAAG,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,oBAAoB,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;QACjE,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QACpB,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACpC,CAAC;IAED;;;;;;;OAOG;IACH,oBAAoB,CAAC,KAAa;QAChC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,wBAAwB,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;IAC3E,CAAC;IAED;;;;;;;;;;;;;;;;OAgBG;IACH,KAAK,CAAC,oBAAoB;QACxB,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;YAC/C,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,gCAAgC,CAAC,CAAC,GAAG,EAAwC,CAAC;gBAC1G,OAAO,GAAG,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,YAAY,EAAE,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;YAC/G,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QACD,IAAI,CAAC;YACH,MAAM,EAAE,MAAM,EAAE,GAAG,wDAAa,qBAAqB,GAAC,CAAC;YACvD,MAAM,YAAY,GAAG;;;;;;;;;;OAUpB,CAAC;YACF,OAAO,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;gBACnC,IAAI,OAAO,GAAG,KAAK,CAAC;gBACpB,MAAM,MAAM,GAAG,CAAC,GAAmC,EAAQ,EAAE;oBAC3D,IAAI,OAAO;wBAAE,OAAO;oBACpB,OAAO,GAAG,IAAI,CAAC;oBACf,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,YAAY,EAAE,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBACjH,CAAC,CAAC;gBACF,IAAI,CAAC;oBACH,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,YAAY,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;oBAC7F,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,CAA0C,EAAE,EAAE,GAAG,KAAK,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC7H,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,GAAG,KAAK,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;oBACvE,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;gBAC1C,CAAC;gBAAC,MAAM,CAAC;oBACP,MAAM,CAAC,IAAI,CAAC,CAAC;gBACf,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED;;OAEG;IACH,QAAQ;QACN,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACvB,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC1B,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;IACH,KAAK,CAAC,cAAc;QAClB,iEAAiE;QACjE,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;YAC/C,IAAI,CAAC;gBAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;YAAC,CAAC;YAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;YAC/D,IAAI,CAAC;gBAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAC;YAAC,CAAC;YAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;YAC9E,OAAO;QACT,CAAC;QACD,MAAM,IAAI,CAAC,mBAAmB,CAC5B,CAAC,4BAA4B,EAAE,iBAAiB,EAAE,gCAAgC,CAAC;QACnF,wEAAwE;QACxE,CAAC,4BAA4B,EAAE,iBAAiB,CAAC,CAClD,CAAC;IACJ,CAAC;IAED;;;;;;;OAOG;IACK,KAAK,CAAC,mBAAmB,CAAC,OAAiB,EAAE,iBAA2B,EAAE;QAChF,IAAI,CAAC;YACH,MAAM,EAAE,MAAM,EAAE,GAAG,wDAAa,qBAAqB,GAAC,CAAC;YACvD,MAAM,YAAY,GAAG;;;;;;;;;OASpB,CAAC;YACF,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;gBAClC,IAAI,OAAO,GAAG,KAAK,CAAC;gBACpB,MAAM,MAAM,GAAG,GAAS,EAAE;oBACxB,IAAI,CAAC,OAAO,EAAE,CAAC;wBAAC,OAAO,GAAG,IAAI,CAAC;wBAAC,OAAO,EAAE,CAAC;oBAAC,CAAC;gBAC9C,CAAC,CAAC;gBACF,IAAI,CAAC;oBACH,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,YAAY,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC;oBACtG,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,EAAE,GAAG,KAAK,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;oBACrE,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,GAAG,KAAK,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;oBACnE,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gBAC9B,CAAC;gBAAC,MAAM,CAAC;oBACP,MAAM,EAAE,CAAC;gBACX,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,KAAK,MAAM,CAAC,IAAI,cAAc,EAAE,CAAC;gBAC/B,IAAI,CAAC;oBAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAAC,CAAC;gBAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;YACjD,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK;QACH,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;IAClB,CAAC;IAED;;OAEG;IACH,MAAM;QACJ,OAAO,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC;IACtB,CAAC;IAED;;;;;;;;;;OAUG;IACH,gBAAgB;QACd,IAAI,IAAI,CAAC,WAAW,KAAK,IAAI;YAAE,OAAO,KAAK,CAAC;QAC5C,MAAM,OAAO,GAAG,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACvC,OAAO,OAAO,KAAK,IAAI,IAAI,OAAO,KAAK,IAAI,CAAC,WAAW,CAAC;IAC1D,CAAC;CACF;AA5WD,gDA4WC;AAED;;;;;;GAMG;AACH,SAAS,SAAS,CAAC,CAAS;IAC1B,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO;QAAE,OAAO,IAAI,CAAC;IAC9C,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;QACzB,OAAO,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,GAAG,EAAE,CAAC;IAC7B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;GAEG;AACU,QAAA,iBAAiB,GAAG,cAAc,CAAC;AAEhD;;;;GAIG;AACH,MAAM,oBAAoB,GAAG,CAAC,MAAM,EAAE,MAAM,CAAU,CAAC;AAEvD;;GAEG;AACH,SAAgB,eAAe,CAAC,WAAmB;IACjD,OAAO,IAAI,CAAC,IAAI,CAAC,IAAA,2BAAe,EAAC,WAAW,CAAC,EAAE,yBAAiB,CAAC,CAAC;AACpE,CAAC;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,SAAgB,mBAAmB,CAAC,MAAc;IAChD,8EAA8E;IAC9E,8EAA8E;IAC9E,EAAE,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACnC,KAAK,MAAM,MAAM,IAAI,oBAAoB,EAAE,CAAC;QAC1C,IAAI,CAAC;YACH,EAAE,CAAC,MAAM,CAAC,MAAM,GAAG,MAAM,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9C,CAAC;QAAC,MAAM,CAAC;YACP,wEAAwE;QAC1E,CAAC;IACH,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* WAL checkpoint valve — bounds WAL growth while auto-checkpointing is
|
|
3
|
+
* deferred during a bulk index (#1231).
|
|
4
|
+
*
|
|
5
|
+
* Why deferral: SQLite's default `wal_autocheckpoint` (1000 pages) re-writes
|
|
6
|
+
* hot B-tree/FTS pages into the main DB file over and over during a bulk
|
|
7
|
+
* index — measured at ~95% of ALL disk I/O, and the difference between 45s
|
|
8
|
+
* and 19+ minutes on HDD-class storage (150 random IOPS). Deferring
|
|
9
|
+
* checkpoints turns the store into pure sequential WAL appends; each backfill
|
|
10
|
+
* pass writes distinct pages once, in page order (≈ sequential).
|
|
11
|
+
*
|
|
12
|
+
* Why a valve: unbounded deferral is its own failure mode, both measured in
|
|
13
|
+
* the #1231 repro. The WAL duplicates hot pages per COMMIT, so it grows far
|
|
14
|
+
* faster than the DB (5.9GB WAL for a ~340MB DB on a 3.3k-file index) —
|
|
15
|
+
* filling the disk, and poisoning every subsequent read that must page
|
|
16
|
+
* through it (the first resolution-phase read blocked the main thread >60s
|
|
17
|
+
* and the #850 liveness watchdog killed the healthy index). The valve
|
|
18
|
+
* watches WAL growth on a timer and, past a soft threshold, backfills with
|
|
19
|
+
* `PRAGMA wal_checkpoint(PASSIVE)` on a worker-thread connection — PASSIVE
|
|
20
|
+
* never blocks the writer, and off-thread means the main thread (and the
|
|
21
|
+
* watchdog heartbeat) keep turning regardless of how long a backfill takes.
|
|
22
|
+
*
|
|
23
|
+
* The load-bearing subtlety: a WAL file's SIZE never shrinks. After a full
|
|
24
|
+
* backfill, the writer's next commit RESTARTS the WAL from the top and the
|
|
25
|
+
* frames recycle inside the same file — so raw size says nothing about the
|
|
26
|
+
* un-backfilled backlog, and a size-triggered valve degenerates into firing
|
|
27
|
+
* (and pausing the writer) forever once the file passes its threshold
|
|
28
|
+
* (measured: guava crawled at ~9min per 160 files). Instead the valve
|
|
29
|
+
* tracks `sizeAtLastFullBackfill` — refreshed whenever a checkpoint reports
|
|
30
|
+
* `log === checkpointed` (everything backfilled) — and triggers on GROWTH
|
|
31
|
+
* beyond that baseline, which only happens when genuinely un-backfilled
|
|
32
|
+
* frames push past the file's high-water mark.
|
|
33
|
+
*
|
|
34
|
+
* Backpressure: if the writer outruns the checkpointer past a hard cap of
|
|
35
|
+
* growth (2× soft), {@link backpressure} pauses the writer (at a safe,
|
|
36
|
+
* between-transactions boundary) until a FULL backfill lands. One in-flight
|
|
37
|
+
* pass is not enough: on a disk saturated by the writer, every concurrent
|
|
38
|
+
* PASSIVE pass is already stale by the time it finishes (the writer appended
|
|
39
|
+
* past its snapshot), so neither SQLite's WAL wrap nor the baseline ever
|
|
40
|
+
* trigger and the WAL grows without bound (measured: 5.9GB on guava at 150
|
|
41
|
+
* IOPS, then a >60s read stall and a watchdog kill). With the writer parked,
|
|
42
|
+
* the next pass covers everything, the WAL wraps on the following commit,
|
|
43
|
+
* and the pause is the disk's honest catch-up cost — the correct terminal
|
|
44
|
+
* mode when hardware genuinely can't keep up with the append rate.
|
|
45
|
+
*/
|
|
46
|
+
import type { DatabaseConnection } from './index';
|
|
47
|
+
/**
|
|
48
|
+
* Resolve the valve's soft threshold from the `CODEGRAPH_WAL_VALVE_MB`
|
|
49
|
+
* override; non-numeric / non-positive values fall back to the default.
|
|
50
|
+
*/
|
|
51
|
+
export declare function resolveWalValveMb(envVal: string | undefined): number;
|
|
52
|
+
export declare class WalCheckpointValve {
|
|
53
|
+
private readonly db;
|
|
54
|
+
private readonly intervalMs;
|
|
55
|
+
private readonly log;
|
|
56
|
+
private timer;
|
|
57
|
+
private inflight;
|
|
58
|
+
/** Writer pause in progress (hard cap breached): passes loop until a full backfill. */
|
|
59
|
+
private pause;
|
|
60
|
+
/**
|
|
61
|
+
* WAL file size observed when a checkpoint last reported the ENTIRE WAL
|
|
62
|
+
* backfilled. Growth is measured against this baseline — see the header
|
|
63
|
+
* comment for why absolute size cannot be used.
|
|
64
|
+
*/
|
|
65
|
+
private sizeAtLastFullBackfill;
|
|
66
|
+
private readonly softBytes;
|
|
67
|
+
private readonly hardBytes;
|
|
68
|
+
constructor(db: DatabaseConnection, softMb?: number, intervalMs?: number, log?: (msg: string) => void);
|
|
69
|
+
private mb;
|
|
70
|
+
/** Un-backfilled growth estimate: bytes the WAL has grown past the last full backfill. */
|
|
71
|
+
private growthBytes;
|
|
72
|
+
/** Begin watching the WAL. Idempotent; the timer never holds the loop open. */
|
|
73
|
+
start(): void;
|
|
74
|
+
/** Stop watching. Any in-flight checkpoint keeps running — await drain(). */
|
|
75
|
+
stop(): void;
|
|
76
|
+
/** One poll: fire an off-thread passive checkpoint when growth passes the soft threshold. */
|
|
77
|
+
check(): void;
|
|
78
|
+
/**
|
|
79
|
+
* Writer-side backstop, called at a between-transactions boundary. Returns
|
|
80
|
+
* null (no wait) while growth is under the hard cap; past it, returns a
|
|
81
|
+
* promise that resolves only once a FULL backfill has landed — see the
|
|
82
|
+
* header comment for why a single pass is not enough on a saturated disk.
|
|
83
|
+
*/
|
|
84
|
+
backpressure(): Promise<void> | null;
|
|
85
|
+
/** Await any in-flight checkpoint and writer pause. */
|
|
86
|
+
drain(): Promise<void>;
|
|
87
|
+
/**
|
|
88
|
+
* Phase-boundary fold: backfill the ENTIRE WAL now (off-thread, awaited).
|
|
89
|
+
* Called between bulk phases — e.g. after parsing, before resolution's
|
|
90
|
+
* first reads — so the next phase never pages a bulk-write-sized WAL on
|
|
91
|
+
* the main thread (the post-parse read against a multi-GB WAL is what
|
|
92
|
+
* blew the #850 watchdog's 60s window in the #1231 repro). The await
|
|
93
|
+
* keeps the event loop (and the watchdog heartbeat) turning.
|
|
94
|
+
*/
|
|
95
|
+
foldNow(): Promise<void>;
|
|
96
|
+
/**
|
|
97
|
+
* With the writer parked on the returned promise, loop passive passes until
|
|
98
|
+
* one reports the entire WAL backfilled (typically the second: the first
|
|
99
|
+
* drains the pass that was already running against a stale snapshot). Gives
|
|
100
|
+
* up after a bounded number of passes — e.g. a reader pinning the WAL —
|
|
101
|
+
* because unbounded WAL growth degrades; a wedged writer never recovers.
|
|
102
|
+
*/
|
|
103
|
+
private backfillFully;
|
|
104
|
+
private fire;
|
|
105
|
+
}
|
|
106
|
+
//# sourceMappingURL=wal-valve.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"wal-valve.d.ts","sourceRoot":"","sources":["../../src/db/wal-valve.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4CG;AAEH,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAC;AAWlD;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,CAMpE;AAED,qBAAa,kBAAkB;IAe3B,OAAO,CAAC,QAAQ,CAAC,EAAE;IAEnB,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,GAAG;IAjBtB,OAAO,CAAC,KAAK,CAA+C;IAC5D,OAAO,CAAC,QAAQ,CAA8B;IAC9C,uFAAuF;IACvF,OAAO,CAAC,KAAK,CAA8B;IAC3C;;;;OAIG;IACH,OAAO,CAAC,sBAAsB,CAAK;IACnC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;gBAGhB,EAAE,EAAE,kBAAkB,EACvC,MAAM,GAAE,MAA8D,EACrD,UAAU,GAAE,MAA0B,EACtC,GAAG,GAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAe;IAMxD,OAAO,CAAC,EAAE;IAIV,0FAA0F;IAC1F,OAAO,CAAC,WAAW;IAInB,+EAA+E;IAC/E,KAAK,IAAI,IAAI;IAMb,6EAA6E;IAC7E,IAAI,IAAI,IAAI;IAOZ,6FAA6F;IAC7F,KAAK,IAAI,IAAI;IAIb;;;;;OAKG;IACH,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI;IAYpC,uDAAuD;IACjD,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAO5B;;;;;;;OAOG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAQ9B;;;;;;OAMG;YACW,aAAa;IAc3B,OAAO,CAAC,IAAI;CAoBb"}
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* WAL checkpoint valve — bounds WAL growth while auto-checkpointing is
|
|
4
|
+
* deferred during a bulk index (#1231).
|
|
5
|
+
*
|
|
6
|
+
* Why deferral: SQLite's default `wal_autocheckpoint` (1000 pages) re-writes
|
|
7
|
+
* hot B-tree/FTS pages into the main DB file over and over during a bulk
|
|
8
|
+
* index — measured at ~95% of ALL disk I/O, and the difference between 45s
|
|
9
|
+
* and 19+ minutes on HDD-class storage (150 random IOPS). Deferring
|
|
10
|
+
* checkpoints turns the store into pure sequential WAL appends; each backfill
|
|
11
|
+
* pass writes distinct pages once, in page order (≈ sequential).
|
|
12
|
+
*
|
|
13
|
+
* Why a valve: unbounded deferral is its own failure mode, both measured in
|
|
14
|
+
* the #1231 repro. The WAL duplicates hot pages per COMMIT, so it grows far
|
|
15
|
+
* faster than the DB (5.9GB WAL for a ~340MB DB on a 3.3k-file index) —
|
|
16
|
+
* filling the disk, and poisoning every subsequent read that must page
|
|
17
|
+
* through it (the first resolution-phase read blocked the main thread >60s
|
|
18
|
+
* and the #850 liveness watchdog killed the healthy index). The valve
|
|
19
|
+
* watches WAL growth on a timer and, past a soft threshold, backfills with
|
|
20
|
+
* `PRAGMA wal_checkpoint(PASSIVE)` on a worker-thread connection — PASSIVE
|
|
21
|
+
* never blocks the writer, and off-thread means the main thread (and the
|
|
22
|
+
* watchdog heartbeat) keep turning regardless of how long a backfill takes.
|
|
23
|
+
*
|
|
24
|
+
* The load-bearing subtlety: a WAL file's SIZE never shrinks. After a full
|
|
25
|
+
* backfill, the writer's next commit RESTARTS the WAL from the top and the
|
|
26
|
+
* frames recycle inside the same file — so raw size says nothing about the
|
|
27
|
+
* un-backfilled backlog, and a size-triggered valve degenerates into firing
|
|
28
|
+
* (and pausing the writer) forever once the file passes its threshold
|
|
29
|
+
* (measured: guava crawled at ~9min per 160 files). Instead the valve
|
|
30
|
+
* tracks `sizeAtLastFullBackfill` — refreshed whenever a checkpoint reports
|
|
31
|
+
* `log === checkpointed` (everything backfilled) — and triggers on GROWTH
|
|
32
|
+
* beyond that baseline, which only happens when genuinely un-backfilled
|
|
33
|
+
* frames push past the file's high-water mark.
|
|
34
|
+
*
|
|
35
|
+
* Backpressure: if the writer outruns the checkpointer past a hard cap of
|
|
36
|
+
* growth (2× soft), {@link backpressure} pauses the writer (at a safe,
|
|
37
|
+
* between-transactions boundary) until a FULL backfill lands. One in-flight
|
|
38
|
+
* pass is not enough: on a disk saturated by the writer, every concurrent
|
|
39
|
+
* PASSIVE pass is already stale by the time it finishes (the writer appended
|
|
40
|
+
* past its snapshot), so neither SQLite's WAL wrap nor the baseline ever
|
|
41
|
+
* trigger and the WAL grows without bound (measured: 5.9GB on guava at 150
|
|
42
|
+
* IOPS, then a >60s read stall and a watchdog kill). With the writer parked,
|
|
43
|
+
* the next pass covers everything, the WAL wraps on the following commit,
|
|
44
|
+
* and the pause is the disk's honest catch-up cost — the correct terminal
|
|
45
|
+
* mode when hardware genuinely can't keep up with the append rate.
|
|
46
|
+
*/
|
|
47
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
48
|
+
exports.WalCheckpointValve = void 0;
|
|
49
|
+
exports.resolveWalValveMb = resolveWalValveMb;
|
|
50
|
+
/** Soft WAL-growth threshold (MB) that triggers an off-thread passive checkpoint. */
|
|
51
|
+
const DEFAULT_WAL_VALVE_MB = 256;
|
|
52
|
+
/** Hard cap = this × soft threshold; past it the writer pauses for a full backfill. */
|
|
53
|
+
const HARD_CAP_MULTIPLIER = 2;
|
|
54
|
+
/** Passes attempted per writer pause before giving up (a pinned reader could stall forever). */
|
|
55
|
+
const MAX_PAUSED_BACKFILL_PASSES = 20;
|
|
56
|
+
/** How often the timer looks at the WAL file size. */
|
|
57
|
+
const CHECK_INTERVAL_MS = 2000;
|
|
58
|
+
/**
|
|
59
|
+
* Resolve the valve's soft threshold from the `CODEGRAPH_WAL_VALVE_MB`
|
|
60
|
+
* override; non-numeric / non-positive values fall back to the default.
|
|
61
|
+
*/
|
|
62
|
+
function resolveWalValveMb(envVal) {
|
|
63
|
+
if (envVal !== undefined && envVal !== '') {
|
|
64
|
+
const n = Number(envVal);
|
|
65
|
+
if (Number.isFinite(n) && n > 0)
|
|
66
|
+
return Math.floor(n);
|
|
67
|
+
}
|
|
68
|
+
return DEFAULT_WAL_VALVE_MB;
|
|
69
|
+
}
|
|
70
|
+
class WalCheckpointValve {
|
|
71
|
+
db;
|
|
72
|
+
intervalMs;
|
|
73
|
+
log;
|
|
74
|
+
timer = null;
|
|
75
|
+
inflight = null;
|
|
76
|
+
/** Writer pause in progress (hard cap breached): passes loop until a full backfill. */
|
|
77
|
+
pause = null;
|
|
78
|
+
/**
|
|
79
|
+
* WAL file size observed when a checkpoint last reported the ENTIRE WAL
|
|
80
|
+
* backfilled. Growth is measured against this baseline — see the header
|
|
81
|
+
* comment for why absolute size cannot be used.
|
|
82
|
+
*/
|
|
83
|
+
sizeAtLastFullBackfill = 0;
|
|
84
|
+
softBytes;
|
|
85
|
+
hardBytes;
|
|
86
|
+
constructor(db, softMb = resolveWalValveMb(process.env.CODEGRAPH_WAL_VALVE_MB), intervalMs = CHECK_INTERVAL_MS, log = () => { }) {
|
|
87
|
+
this.db = db;
|
|
88
|
+
this.intervalMs = intervalMs;
|
|
89
|
+
this.log = log;
|
|
90
|
+
this.softBytes = softMb * 1024 * 1024;
|
|
91
|
+
this.hardBytes = this.softBytes * HARD_CAP_MULTIPLIER;
|
|
92
|
+
}
|
|
93
|
+
mb(n) {
|
|
94
|
+
return `${Math.round(n / 1024 / 1024)}MB`;
|
|
95
|
+
}
|
|
96
|
+
/** Un-backfilled growth estimate: bytes the WAL has grown past the last full backfill. */
|
|
97
|
+
growthBytes() {
|
|
98
|
+
return this.db.getWalSizeBytes() - this.sizeAtLastFullBackfill;
|
|
99
|
+
}
|
|
100
|
+
/** Begin watching the WAL. Idempotent; the timer never holds the loop open. */
|
|
101
|
+
start() {
|
|
102
|
+
if (this.timer)
|
|
103
|
+
return;
|
|
104
|
+
this.timer = setInterval(() => this.check(), this.intervalMs);
|
|
105
|
+
this.timer.unref?.();
|
|
106
|
+
}
|
|
107
|
+
/** Stop watching. Any in-flight checkpoint keeps running — await drain(). */
|
|
108
|
+
stop() {
|
|
109
|
+
if (this.timer) {
|
|
110
|
+
clearInterval(this.timer);
|
|
111
|
+
this.timer = null;
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
/** One poll: fire an off-thread passive checkpoint when growth passes the soft threshold. */
|
|
115
|
+
check() {
|
|
116
|
+
if (!this.pause && !this.inflight && this.growthBytes() > this.softBytes)
|
|
117
|
+
this.fire();
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* Writer-side backstop, called at a between-transactions boundary. Returns
|
|
121
|
+
* null (no wait) while growth is under the hard cap; past it, returns a
|
|
122
|
+
* promise that resolves only once a FULL backfill has landed — see the
|
|
123
|
+
* header comment for why a single pass is not enough on a saturated disk.
|
|
124
|
+
*/
|
|
125
|
+
backpressure() {
|
|
126
|
+
if (this.pause)
|
|
127
|
+
return this.pause;
|
|
128
|
+
if (this.growthBytes() <= this.hardBytes)
|
|
129
|
+
return null;
|
|
130
|
+
this.log(`backpressure: wal=${this.mb(this.db.getWalSizeBytes())} baseline=${this.mb(this.sizeAtLastFullBackfill)} — pausing writer for full backfill`);
|
|
131
|
+
const t0 = Date.now();
|
|
132
|
+
this.pause = this.backfillFully().finally(() => {
|
|
133
|
+
this.pause = null;
|
|
134
|
+
this.log(`backpressure released after ${Date.now() - t0}ms: wal=${this.mb(this.db.getWalSizeBytes())} baseline=${this.mb(this.sizeAtLastFullBackfill)}`);
|
|
135
|
+
});
|
|
136
|
+
return this.pause;
|
|
137
|
+
}
|
|
138
|
+
/** Await any in-flight checkpoint and writer pause. */
|
|
139
|
+
async drain() {
|
|
140
|
+
while (this.pause || this.inflight) {
|
|
141
|
+
if (this.pause)
|
|
142
|
+
await this.pause;
|
|
143
|
+
if (this.inflight)
|
|
144
|
+
await this.inflight;
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
/**
|
|
148
|
+
* Phase-boundary fold: backfill the ENTIRE WAL now (off-thread, awaited).
|
|
149
|
+
* Called between bulk phases — e.g. after parsing, before resolution's
|
|
150
|
+
* first reads — so the next phase never pages a bulk-write-sized WAL on
|
|
151
|
+
* the main thread (the post-parse read against a multi-GB WAL is what
|
|
152
|
+
* blew the #850 watchdog's 60s window in the #1231 repro). The await
|
|
153
|
+
* keeps the event loop (and the watchdog heartbeat) turning.
|
|
154
|
+
*/
|
|
155
|
+
async foldNow() {
|
|
156
|
+
await this.drain();
|
|
157
|
+
if (this.growthBytes() <= 0)
|
|
158
|
+
return;
|
|
159
|
+
this.log(`foldNow: wal=${this.mb(this.db.getWalSizeBytes())} baseline=${this.mb(this.sizeAtLastFullBackfill)}`);
|
|
160
|
+
this.pause = this.backfillFully().finally(() => { this.pause = null; });
|
|
161
|
+
await this.pause;
|
|
162
|
+
}
|
|
163
|
+
/**
|
|
164
|
+
* With the writer parked on the returned promise, loop passive passes until
|
|
165
|
+
* one reports the entire WAL backfilled (typically the second: the first
|
|
166
|
+
* drains the pass that was already running against a stale snapshot). Gives
|
|
167
|
+
* up after a bounded number of passes — e.g. a reader pinning the WAL —
|
|
168
|
+
* because unbounded WAL growth degrades; a wedged writer never recovers.
|
|
169
|
+
*/
|
|
170
|
+
async backfillFully() {
|
|
171
|
+
for (let i = 0; i < MAX_PAUSED_BACKFILL_PASSES; i++) {
|
|
172
|
+
if (this.inflight)
|
|
173
|
+
await this.inflight; // fold in the stale in-flight pass first
|
|
174
|
+
const res = await this.db.checkpointWalPassive();
|
|
175
|
+
if (!res)
|
|
176
|
+
return; // checkpoint machinery unavailable — don't spin
|
|
177
|
+
this.log(`backfill pass ${i + 1}: busy=${res.busy} log=${res.log} checkpointed=${res.checkpointed} wal=${this.mb(this.db.getWalSizeBytes())}`);
|
|
178
|
+
if (res.busy === 0 && res.log === res.checkpointed) {
|
|
179
|
+
this.sizeAtLastFullBackfill = this.db.getWalSizeBytes();
|
|
180
|
+
return;
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
this.log(`backfill gave up after ${MAX_PAUSED_BACKFILL_PASSES} passes — WAL stays unbounded this cycle`);
|
|
184
|
+
}
|
|
185
|
+
fire() {
|
|
186
|
+
const p = this.db
|
|
187
|
+
.checkpointWalPassive()
|
|
188
|
+
.then((res) => {
|
|
189
|
+
// Full backfill (busy 0, every log frame checkpointed) ⇒ the writer's
|
|
190
|
+
// next commit wraps the WAL; the file's current size becomes the new
|
|
191
|
+
// growth baseline. A partial pass (writer appended during it, or a
|
|
192
|
+
// read transaction pinned frames) leaves the baseline alone, so the
|
|
193
|
+
// next tick fires again and copies the remainder. In non-WAL mode
|
|
194
|
+
// SQLite reports log = checkpointed = -1, which is harmless here.
|
|
195
|
+
if (res && res.busy === 0 && res.log === res.checkpointed) {
|
|
196
|
+
this.sizeAtLastFullBackfill = this.db.getWalSizeBytes();
|
|
197
|
+
}
|
|
198
|
+
})
|
|
199
|
+
.catch(() => { })
|
|
200
|
+
.finally(() => {
|
|
201
|
+
if (this.inflight === p)
|
|
202
|
+
this.inflight = null;
|
|
203
|
+
});
|
|
204
|
+
this.inflight = p;
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
exports.WalCheckpointValve = WalCheckpointValve;
|
|
208
|
+
//# sourceMappingURL=wal-valve.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"wal-valve.js","sourceRoot":"","sources":["../../src/db/wal-valve.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4CG;;;AAiBH,8CAMC;AAnBD,qFAAqF;AACrF,MAAM,oBAAoB,GAAG,GAAG,CAAC;AACjC,uFAAuF;AACvF,MAAM,mBAAmB,GAAG,CAAC,CAAC;AAC9B,gGAAgG;AAChG,MAAM,0BAA0B,GAAG,EAAE,CAAC;AACtC,sDAAsD;AACtD,MAAM,iBAAiB,GAAG,IAAI,CAAC;AAE/B;;;GAGG;AACH,SAAgB,iBAAiB,CAAC,MAA0B;IAC1D,IAAI,MAAM,KAAK,SAAS,IAAI,MAAM,KAAK,EAAE,EAAE,CAAC;QAC1C,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;QACzB,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACxD,CAAC;IACD,OAAO,oBAAoB,CAAC;AAC9B,CAAC;AAED,MAAa,kBAAkB;IAeV;IAEA;IACA;IAjBX,KAAK,GAA0C,IAAI,CAAC;IACpD,QAAQ,GAAyB,IAAI,CAAC;IAC9C,uFAAuF;IAC/E,KAAK,GAAyB,IAAI,CAAC;IAC3C;;;;OAIG;IACK,sBAAsB,GAAG,CAAC,CAAC;IAClB,SAAS,CAAS;IAClB,SAAS,CAAS;IAEnC,YACmB,EAAsB,EACvC,SAAiB,iBAAiB,CAAC,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,EACrD,aAAqB,iBAAiB,EACtC,MAA6B,GAAG,EAAE,GAAE,CAAC;QAHrC,OAAE,GAAF,EAAE,CAAoB;QAEtB,eAAU,GAAV,UAAU,CAA4B;QACtC,QAAG,GAAH,GAAG,CAAkC;QAEtD,IAAI,CAAC,SAAS,GAAG,MAAM,GAAG,IAAI,GAAG,IAAI,CAAC;QACtC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,GAAG,mBAAmB,CAAC;IACxD,CAAC;IAEO,EAAE,CAAC,CAAS;QAClB,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;IAC5C,CAAC;IAED,0FAA0F;IAClF,WAAW;QACjB,OAAO,IAAI,CAAC,EAAE,CAAC,eAAe,EAAE,GAAG,IAAI,CAAC,sBAAsB,CAAC;IACjE,CAAC;IAED,+EAA+E;IAC/E,KAAK;QACH,IAAI,IAAI,CAAC,KAAK;YAAE,OAAO;QACvB,IAAI,CAAC,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QAC9D,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;IACvB,CAAC;IAED,6EAA6E;IAC7E,IAAI;QACF,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC1B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QACpB,CAAC;IACH,CAAC;IAED,6FAA6F;IAC7F,KAAK;QACH,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,WAAW,EAAE,GAAG,IAAI,CAAC,SAAS;YAAE,IAAI,CAAC,IAAI,EAAE,CAAC;IACxF,CAAC;IAED;;;;;OAKG;IACH,YAAY;QACV,IAAI,IAAI,CAAC,KAAK;YAAE,OAAO,IAAI,CAAC,KAAK,CAAC;QAClC,IAAI,IAAI,CAAC,WAAW,EAAE,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO,IAAI,CAAC;QACtD,IAAI,CAAC,GAAG,CAAC,qBAAqB,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,eAAe,EAAE,CAAC,aAAa,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,sBAAsB,CAAC,qCAAqC,CAAC,CAAC;QACxJ,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACtB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE;YAC7C,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;YAClB,IAAI,CAAC,GAAG,CAAC,+BAA+B,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,WAAW,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,eAAe,EAAE,CAAC,aAAa,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,sBAAsB,CAAC,EAAE,CAAC,CAAC;QAC3J,CAAC,CAAC,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED,uDAAuD;IACvD,KAAK,CAAC,KAAK;QACT,OAAO,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnC,IAAI,IAAI,CAAC,KAAK;gBAAE,MAAM,IAAI,CAAC,KAAK,CAAC;YACjC,IAAI,IAAI,CAAC,QAAQ;gBAAE,MAAM,IAAI,CAAC,QAAQ,CAAC;QACzC,CAAC;IACH,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,OAAO;QACX,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACnB,IAAI,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC;YAAE,OAAO;QACpC,IAAI,CAAC,GAAG,CAAC,gBAAgB,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,eAAe,EAAE,CAAC,aAAa,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,sBAAsB,CAAC,EAAE,CAAC,CAAC;QAChH,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QACxE,MAAM,IAAI,CAAC,KAAK,CAAC;IACnB,CAAC;IAED;;;;;;OAMG;IACK,KAAK,CAAC,aAAa;QACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,0BAA0B,EAAE,CAAC,EAAE,EAAE,CAAC;YACpD,IAAI,IAAI,CAAC,QAAQ;gBAAE,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC,yCAAyC;YACjF,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,oBAAoB,EAAE,CAAC;YACjD,IAAI,CAAC,GAAG;gBAAE,OAAO,CAAC,gDAAgD;YAClE,IAAI,CAAC,GAAG,CAAC,iBAAiB,CAAC,GAAG,CAAC,UAAU,GAAG,CAAC,IAAI,QAAQ,GAAG,CAAC,GAAG,iBAAiB,GAAG,CAAC,YAAY,QAAQ,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,eAAe,EAAE,CAAC,EAAE,CAAC,CAAC;YAC/I,IAAI,GAAG,CAAC,IAAI,KAAK,CAAC,IAAI,GAAG,CAAC,GAAG,KAAK,GAAG,CAAC,YAAY,EAAE,CAAC;gBACnD,IAAI,CAAC,sBAAsB,GAAG,IAAI,CAAC,EAAE,CAAC,eAAe,EAAE,CAAC;gBACxD,OAAO;YACT,CAAC;QACH,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,0BAA0B,0BAA0B,0CAA0C,CAAC,CAAC;IAC3G,CAAC;IAEO,IAAI;QACV,MAAM,CAAC,GAAG,IAAI,CAAC,EAAE;aACd,oBAAoB,EAAE;aACtB,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE;YACZ,sEAAsE;YACtE,qEAAqE;YACrE,mEAAmE;YACnE,oEAAoE;YACpE,kEAAkE;YAClE,kEAAkE;YAClE,IAAI,GAAG,IAAI,GAAG,CAAC,IAAI,KAAK,CAAC,IAAI,GAAG,CAAC,GAAG,KAAK,GAAG,CAAC,YAAY,EAAE,CAAC;gBAC1D,IAAI,CAAC,sBAAsB,GAAG,IAAI,CAAC,EAAE,CAAC,eAAe,EAAE,CAAC;YAC1D,CAAC;QACH,CAAC,CAAC;aACD,KAAK,CAAC,GAAG,EAAE,GAAqB,CAAC,CAAC;aAClC,OAAO,CAAC,GAAG,EAAE;YACZ,IAAI,IAAI,CAAC,QAAQ,KAAK,CAAC;gBAAE,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QAChD,CAAC,CAAC,CAAC;QACL,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;IACpB,CAAC;CACF;AAxID,gDAwIC"}
|
|
@@ -49,12 +49,27 @@ export declare function isPlayRoutesFile(filePath: string): boolean;
|
|
|
49
49
|
* Idempotent — safe to call multiple times.
|
|
50
50
|
*/
|
|
51
51
|
export declare function initGrammars(): Promise<void>;
|
|
52
|
+
/**
|
|
53
|
+
* Pre-read the grammar WASM bytes for an index set, keyed by language. The
|
|
54
|
+
* orchestrator reads each grammar ONCE and hands the bytes to every parse
|
|
55
|
+
* worker via its `load-grammars` message, so worker spawns/respawns load
|
|
56
|
+
* grammars from memory instead of re-reading them from disk — on slow storage
|
|
57
|
+
* (HDD, issue #1231) each respawn's grammar re-read otherwise amplifies the
|
|
58
|
+
* I/O contention that caused the respawn. Best-effort: a language whose WASM
|
|
59
|
+
* can't be read here is simply omitted, and the worker falls back to its own
|
|
60
|
+
* disk load (which surfaces the real error/warning path).
|
|
61
|
+
*/
|
|
62
|
+
export declare function readGrammarWasmBytes(languages: Language[]): Promise<Record<string, Uint8Array>>;
|
|
52
63
|
/**
|
|
53
64
|
* Load grammar WASM files for specific languages only.
|
|
54
65
|
* Skips languages that are already loaded or have no WASM grammar.
|
|
55
66
|
* Must be called after initGrammars().
|
|
67
|
+
*
|
|
68
|
+
* `wasmBytes` (optional) holds pre-read grammar bytes keyed by language (from
|
|
69
|
+
* {@link readGrammarWasmBytes}, forwarded through the parse pool); when a
|
|
70
|
+
* language's bytes are present they're loaded from memory instead of disk.
|
|
56
71
|
*/
|
|
57
|
-
export declare function loadGrammarsForLanguages(languages: Language[]): Promise<void>;
|
|
72
|
+
export declare function loadGrammarsForLanguages(languages: Language[], wasmBytes?: Record<string, Uint8Array>): Promise<void>;
|
|
58
73
|
/**
|
|
59
74
|
* Load ALL grammar WASM files. Convenience function for tests and
|
|
60
75
|
* backward compatibility. Prefer loadGrammarsForLanguages() in production.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"grammars.d.ts","sourceRoot":"","sources":["../../src/extraction/grammars.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;
|
|
1
|
+
{"version":3,"file":"grammars.d.ts","sourceRoot":"","sources":["../../src/extraction/grammars.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAIH,OAAO,EAAE,MAAM,EAA4B,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AAEpC,MAAM,MAAM,eAAe,GAAG,OAAO,CAAC,QAAQ,EAAE,QAAQ,GAAG,KAAK,GAAG,OAAO,GAAG,QAAQ,GAAG,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,KAAK,GAAG,YAAY,GAAG,SAAS,CAAC,CAAC;AAyCtJ;;GAEG;AACH,eAAO,MAAM,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,CAmHlD,CAAC;AAEF;;;;;;;;GAQG;AACH,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,GAAG,OAAO,CAQ5F;AAED;;;;GAIG;AACH,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAI7D;AAED;;;;;;;GAOG;AACH,wBAAgB,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAEzD;AAED;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAM1D;AAWD;;;;GAIG;AACH,wBAAsB,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC,CAMlD;AAyDD;;;;;;;;;GASG;AACH,wBAAsB,oBAAoB,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CAarG;AAED;;;;;;;;GAQG;AACH,wBAAsB,wBAAwB,CAAC,SAAS,EAAE,QAAQ,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CA4B3H;AAED;;;GAGG;AACH,wBAAsB,eAAe,IAAI,OAAO,CAAC,IAAI,CAAC,CAGrD;AAED;;GAEG;AACH,wBAAgB,qBAAqB,IAAI,OAAO,CAE/C;AAED;;;GAGG;AACH,wBAAgB,SAAS,CAAC,QAAQ,EAAE,QAAQ,GAAG,MAAM,GAAG,IAAI,CAc3D;AAED;;;;;;GAMG;AACH,wBAAgB,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,GAAG,QAAQ,CAoBhH;AA4BD;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAY/D;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAK3D;AAED;;;;;;;;GAQG;AACH,wBAAgB,uBAAuB,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAEnE;AAED;;GAEG;AACH,wBAAgB,qBAAqB,IAAI,QAAQ,EAAE,CAElD;AAED;;;;;;GAMG;AACH,wBAAgB,WAAW,CAAC,QAAQ,EAAE,QAAQ,GAAG,IAAI,CAMpD;AAED;;GAEG;AACH,wBAAgB,gBAAgB,IAAI,IAAI,CAQvC;AAED;;GAEG;AACH,wBAAgB,2BAA2B,IAAI,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAM/E;AAED;;GAEG;AACH,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,QAAQ,GAAG,MAAM,CA8CjE"}
|