gitnexus 1.6.13-rc.71 → 1.6.13-rc.73
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/embeddings-sync.js +10 -2
- package/dist/core/embeddings/staged-embedding-recovery-child.d.ts +1 -0
- package/dist/core/embeddings/staged-embedding-recovery-child.js +170 -0
- package/dist/core/embeddings/staged-embedding-recovery.d.ts +23 -0
- package/dist/core/embeddings/staged-embedding-recovery.js +183 -0
- package/dist/core/run-analyze.js +145 -15
- package/dist/server/api.js +10 -2
- package/dist/storage/embedding-recovery.d.ts +15 -0
- package/dist/storage/embedding-recovery.js +146 -0
- package/dist/storage/index-lock.d.ts +3 -2
- package/dist/storage/index-lock.js +6 -3
- package/dist/storage/repo-meta.d.ts +13 -0
- package/package.json +1 -1
|
@@ -4,7 +4,7 @@ import { LBUG_DIRECTORY } from '../storage/storage-constants.js';
|
|
|
4
4
|
import path from 'node:path';
|
|
5
5
|
import { cliInfo } from './cli-message.js';
|
|
6
6
|
import { getGitRoot } from '../storage/git.js';
|
|
7
|
-
import { acquireIndexLock, requireExclusiveIndexLock } from '../storage/index-lock.js';
|
|
7
|
+
import { acquireIndexLock, requireExclusiveIndexLock, sweepStagingArtifacts, } from '../storage/index-lock.js';
|
|
8
8
|
import { getStoragePaths, loadMeta, saveMeta } from '../storage/repo-manager.js';
|
|
9
9
|
import { closeLbug, executeQuery, executeWithReusedStatement, fetchExistingEmbeddingHashes, initLbug, } from '../core/lbug/lbug-adapter.js';
|
|
10
10
|
import { decideEmbeddingResume, mintInterruptedCheckpoint, mintPartialCheckpoint, mintUnverifiedCountCheckpoint, } from '../core/embedding-checkpoint.js';
|
|
@@ -24,9 +24,17 @@ export const embeddingsSyncCommand = async (inputPath) => {
|
|
|
24
24
|
// Writes go to the slot's own graph. A shared-store checkout that reads an
|
|
25
25
|
// immutable commit graph (#3352) takes a private copy first.
|
|
26
26
|
const lbugPath = path.join(metaDir, LBUG_DIRECTORY);
|
|
27
|
-
const lock = await acquireIndexLock(metaDir);
|
|
27
|
+
const lock = await acquireIndexLock(metaDir, { sweep: false });
|
|
28
28
|
try {
|
|
29
29
|
requireExclusiveIndexLock(lock, `Cannot acquire the index lock at ${metaDir}; refusing an unlocked embeddings sync.`);
|
|
30
|
+
// Sync writes the published graph and cannot recover a staged generation.
|
|
31
|
+
// Reject even malformed receipts before sweeping staging files or writing.
|
|
32
|
+
const recoveryCheckpoint = (await loadMeta(metaDir))?.embeddingCheckpoint;
|
|
33
|
+
if (recoveryCheckpoint && Object.hasOwn(recoveryCheckpoint, 'recovery')) {
|
|
34
|
+
throw new Error('Cannot sync embeddings: the index checkpoint references staged embeddings. ' +
|
|
35
|
+
'Run `gitnexus analyze` to recover them first.');
|
|
36
|
+
}
|
|
37
|
+
sweepStagingArtifacts(metaDir);
|
|
30
38
|
if (!(await ensurePrivateSharedGraph(metaDir, (m) => console.log(` ${m}`)))) {
|
|
31
39
|
throw new Error('The shared graph this checkout reads is gone. Run gitnexus analyze first.');
|
|
32
40
|
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
/** Isolated strict native reader. Never import this entrypoint into analyze. */
|
|
2
|
+
import fs from 'node:fs';
|
|
3
|
+
import path from 'node:path';
|
|
4
|
+
import lbug from '@ladybugdb/core';
|
|
5
|
+
import { createLbugDatabase, toNativeSafePath } from '../lbug/lbug-config.js';
|
|
6
|
+
import { FAMILY_SUFFIXES } from '../../storage/embedding-recovery.js';
|
|
7
|
+
import { abortCachedEmbeddingsBuilder, createCachedEmbeddingsBuilder, finalizeCachedEmbeddingsSnapshot, ingestCachedEmbeddingRow, } from './embedding-restore-spill.js';
|
|
8
|
+
/** Native replay and close may checkpoint; only give them disposable copies. */
|
|
9
|
+
function copyRecoveryFamily(dbPath, exportDir) {
|
|
10
|
+
const replayDir = fs.mkdtempSync(path.join(exportDir, 'replay-'));
|
|
11
|
+
const replayPath = path.join(replayDir, path.basename(dbPath));
|
|
12
|
+
const noFollow = fs.constants.O_NOFOLLOW ?? 0;
|
|
13
|
+
const flags = fs.constants.O_RDONLY | noFollow | (fs.constants.O_NONBLOCK ?? 0);
|
|
14
|
+
const buffer = Buffer.allocUnsafe(1024 * 1024);
|
|
15
|
+
for (const suffix of FAMILY_SUFFIXES) {
|
|
16
|
+
const sourcePath = dbPath + suffix;
|
|
17
|
+
let entry;
|
|
18
|
+
try {
|
|
19
|
+
entry = fs.lstatSync(sourcePath, { bigint: true });
|
|
20
|
+
}
|
|
21
|
+
catch (error) {
|
|
22
|
+
if (suffix && error.code === 'ENOENT')
|
|
23
|
+
continue;
|
|
24
|
+
throw error;
|
|
25
|
+
}
|
|
26
|
+
if (!entry.isFile())
|
|
27
|
+
throw new Error('staged embedding family is not a regular file');
|
|
28
|
+
const source = fs.openSync(sourcePath, flags);
|
|
29
|
+
let destination;
|
|
30
|
+
try {
|
|
31
|
+
const opened = fs.fstatSync(source, { bigint: true });
|
|
32
|
+
const current = fs.lstatSync(sourcePath, { bigint: true });
|
|
33
|
+
if (!opened.isFile() ||
|
|
34
|
+
!current.isFile() ||
|
|
35
|
+
(noFollow === 0 && opened.ino === 0n) ||
|
|
36
|
+
opened.dev !== entry.dev ||
|
|
37
|
+
opened.ino !== entry.ino ||
|
|
38
|
+
opened.dev !== current.dev ||
|
|
39
|
+
opened.ino !== current.ino) {
|
|
40
|
+
throw new Error('staged embedding family changed while opening');
|
|
41
|
+
}
|
|
42
|
+
destination = fs.openSync(replayPath + suffix, 'wx', 0o600);
|
|
43
|
+
let copied = 0n;
|
|
44
|
+
for (;;) {
|
|
45
|
+
const bytesRead = fs.readSync(source, buffer, 0, buffer.length, null);
|
|
46
|
+
if (bytesRead === 0)
|
|
47
|
+
break;
|
|
48
|
+
fs.writeFileSync(destination, buffer.subarray(0, bytesRead));
|
|
49
|
+
copied += BigInt(bytesRead);
|
|
50
|
+
}
|
|
51
|
+
const after = fs.fstatSync(source, { bigint: true });
|
|
52
|
+
if (copied !== opened.size ||
|
|
53
|
+
after.size !== opened.size ||
|
|
54
|
+
after.mtimeNs !== opened.mtimeNs ||
|
|
55
|
+
after.ctimeNs !== opened.ctimeNs) {
|
|
56
|
+
throw new Error('staged embedding family changed while copying');
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
finally {
|
|
60
|
+
try {
|
|
61
|
+
if (destination !== undefined)
|
|
62
|
+
fs.closeSync(destination);
|
|
63
|
+
}
|
|
64
|
+
finally {
|
|
65
|
+
fs.closeSync(source);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
return replayPath;
|
|
70
|
+
}
|
|
71
|
+
async function extract() {
|
|
72
|
+
const [dbPath, exportDir, dimensionsArg] = process.argv.slice(2);
|
|
73
|
+
const dimensions = Number(dimensionsArg);
|
|
74
|
+
if (!dbPath || !exportDir || !Number.isInteger(dimensions) || dimensions <= 0) {
|
|
75
|
+
throw new Error('invalid staged embedding extraction arguments');
|
|
76
|
+
}
|
|
77
|
+
// The parent owns exportDir and reclaims it even after killing this child.
|
|
78
|
+
const replayPath = copyRecoveryFamily(dbPath, exportDir);
|
|
79
|
+
const builder = createCachedEmbeddingsBuilder({ inMemoryRowLimit: 0, spillDir: exportDir });
|
|
80
|
+
const rejectedNodeIds = new Set();
|
|
81
|
+
let db;
|
|
82
|
+
let conn;
|
|
83
|
+
try {
|
|
84
|
+
// Avoid openLbugConnection's test-fixture lock sweep: a recovery source must
|
|
85
|
+
// never have its WAL removed, even when an external slot resembles a fixture.
|
|
86
|
+
db = createLbugDatabase(lbug, toNativeSafePath(replayPath), { throwOnWalReplayFailure: true });
|
|
87
|
+
conn = new lbug.Connection(db);
|
|
88
|
+
const queried = await conn.query('MATCH (e:CodeEmbedding) RETURN e.nodeId AS nodeId, e.chunkIndex AS chunkIndex, e.startLine AS startLine, e.endLine AS endLine, e.embedding AS embedding, e.contentHash AS contentHash');
|
|
89
|
+
const results = Array.isArray(queried) ? queried : [queried];
|
|
90
|
+
try {
|
|
91
|
+
if (results.length !== 1)
|
|
92
|
+
throw new Error('unexpected staged embedding query result');
|
|
93
|
+
const result = results[0];
|
|
94
|
+
while (await result.hasNext()) {
|
|
95
|
+
const raw = await result.getNext();
|
|
96
|
+
const rec = raw;
|
|
97
|
+
const nodeId = rec.nodeId ?? rec[0];
|
|
98
|
+
if (typeof nodeId !== 'string' || !nodeId)
|
|
99
|
+
throw new Error('invalid staged embedding node id');
|
|
100
|
+
const chunkIndex = rec.chunkIndex ?? rec[1];
|
|
101
|
+
const startLine = rec.startLine ?? rec[2];
|
|
102
|
+
const endLine = rec.endLine ?? rec[3];
|
|
103
|
+
const embedding = rec.embedding ?? rec[4];
|
|
104
|
+
const contentHash = rec.contentHash ?? rec[5];
|
|
105
|
+
const vector = Array.isArray(embedding) ||
|
|
106
|
+
(ArrayBuffer.isView(embedding) && !(embedding instanceof DataView))
|
|
107
|
+
? Array.from(embedding)
|
|
108
|
+
: undefined;
|
|
109
|
+
if (!Number.isInteger(chunkIndex) ||
|
|
110
|
+
Number(chunkIndex) < 0 ||
|
|
111
|
+
!Number.isInteger(startLine) ||
|
|
112
|
+
Number(startLine) < 0 ||
|
|
113
|
+
!Number.isInteger(endLine) ||
|
|
114
|
+
Number(endLine) < Number(startLine) ||
|
|
115
|
+
typeof contentHash !== 'string' ||
|
|
116
|
+
!contentHash ||
|
|
117
|
+
!vector ||
|
|
118
|
+
vector.length !== dimensions ||
|
|
119
|
+
vector.some((value) => typeof value !== 'number' ||
|
|
120
|
+
!Number.isFinite(value) ||
|
|
121
|
+
!Number.isFinite(Math.fround(value)))) {
|
|
122
|
+
rejectedNodeIds.add(nodeId);
|
|
123
|
+
continue;
|
|
124
|
+
}
|
|
125
|
+
ingestCachedEmbeddingRow(builder, { nodeId, chunkIndex, startLine, endLine, embedding: vector, contentHash }, true);
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
finally {
|
|
129
|
+
for (const result of results)
|
|
130
|
+
await result.close();
|
|
131
|
+
}
|
|
132
|
+
// Both closes must succeed. Suppressed native teardown errors are unsafe.
|
|
133
|
+
await conn.close();
|
|
134
|
+
await db.close();
|
|
135
|
+
const snapshot = finalizeCachedEmbeddingsSnapshot(builder);
|
|
136
|
+
if (snapshot.spill)
|
|
137
|
+
fs.renameSync(snapshot.spill.path, path.join(exportDir, 'vectors.bin'));
|
|
138
|
+
const manifest = {
|
|
139
|
+
version: 1,
|
|
140
|
+
dimensions,
|
|
141
|
+
rows: snapshot.rows,
|
|
142
|
+
rejectedNodeIds: [...rejectedNodeIds],
|
|
143
|
+
};
|
|
144
|
+
fs.writeFileSync(path.join(exportDir, 'manifest.json'), JSON.stringify(manifest), {
|
|
145
|
+
flag: 'wx',
|
|
146
|
+
mode: 0o600,
|
|
147
|
+
});
|
|
148
|
+
}
|
|
149
|
+
catch (err) {
|
|
150
|
+
abortCachedEmbeddingsBuilder(builder);
|
|
151
|
+
// Cleanup is best effort on a rejected source, never used to approve output.
|
|
152
|
+
try {
|
|
153
|
+
await conn?.close();
|
|
154
|
+
}
|
|
155
|
+
catch {
|
|
156
|
+
/* rejected */
|
|
157
|
+
}
|
|
158
|
+
try {
|
|
159
|
+
await db?.close();
|
|
160
|
+
}
|
|
161
|
+
catch {
|
|
162
|
+
/* rejected */
|
|
163
|
+
}
|
|
164
|
+
throw err;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
extract().catch((err) => {
|
|
168
|
+
process.stderr.write(`${err instanceof Error ? err.message : String(err)}\n`);
|
|
169
|
+
process.exitCode = 1;
|
|
170
|
+
});
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { type CachedEmbeddingMeta, type CachedEmbeddingsSnapshot } from './embedding-restore-spill.js';
|
|
2
|
+
export interface StagedEmbeddingRecoveryOptions {
|
|
3
|
+
dimensions: number;
|
|
4
|
+
/** Active checkpoint window and any incomplete inherited restore groups. */
|
|
5
|
+
excludedNodeIds?: Iterable<string>;
|
|
6
|
+
timeoutMs?: number;
|
|
7
|
+
}
|
|
8
|
+
export interface StagedEmbeddingExport {
|
|
9
|
+
version: 1;
|
|
10
|
+
dimensions: number;
|
|
11
|
+
rows: CachedEmbeddingMeta[];
|
|
12
|
+
rejectedNodeIds: string[];
|
|
13
|
+
}
|
|
14
|
+
/** A contiguous prefix is safe only when its node is outside every unsafe window. */
|
|
15
|
+
export declare function validateRecoveredNodeGroups(rows: readonly CachedEmbeddingMeta[], excludedNodeIds?: ReadonlySet<string>): Set<string>;
|
|
16
|
+
/** Whole recovered nodes replace whole published groups; vectors stay in bounded batches. */
|
|
17
|
+
export declare function mergeRecoveredEmbeddings(live: CachedEmbeddingsSnapshot, recovered: CachedEmbeddingsSnapshot): CachedEmbeddingsSnapshot;
|
|
18
|
+
/**
|
|
19
|
+
* Caller must validate checkpoint identity, schema, exact generation, and hold the
|
|
20
|
+
* index lock. A subprocess contains native WAL replay/query/destructor failures.
|
|
21
|
+
* A failed strict open is never retried with WAL removed or validation disabled.
|
|
22
|
+
*/
|
|
23
|
+
export declare function recoverStagedEmbeddings(dbPath: string, options: StagedEmbeddingRecoveryOptions): Promise<CachedEmbeddingsSnapshot>;
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
/** Recover paid embedding rows without opening an interrupted native DB in analyze. */
|
|
2
|
+
import { spawn } from 'node:child_process';
|
|
3
|
+
import fs from 'node:fs';
|
|
4
|
+
import os from 'node:os';
|
|
5
|
+
import path from 'node:path';
|
|
6
|
+
import { fileURLToPath } from 'node:url';
|
|
7
|
+
import { abortCachedEmbeddingsBuilder, createCachedEmbeddingsBuilder, EmbeddingSpillReader, emptyCachedEmbeddingsSnapshot, finalizeCachedEmbeddingsSnapshot, ingestCachedEmbeddingRow, materializeCachedEmbeddings, } from './embedding-restore-spill.js';
|
|
8
|
+
const RESTORE_BATCH_SIZE = 200;
|
|
9
|
+
const DEFAULT_EXTRACTION_TIMEOUT_MS = 120_000;
|
|
10
|
+
/** A contiguous prefix is safe only when its node is outside every unsafe window. */
|
|
11
|
+
export function validateRecoveredNodeGroups(rows, excludedNodeIds = new Set()) {
|
|
12
|
+
const groups = new Map();
|
|
13
|
+
for (const row of rows) {
|
|
14
|
+
let group = groups.get(row.nodeId);
|
|
15
|
+
if (!group) {
|
|
16
|
+
group = { hash: row.contentHash, ordinals: new Set(), invalid: false };
|
|
17
|
+
groups.set(row.nodeId, group);
|
|
18
|
+
}
|
|
19
|
+
if (typeof row.nodeId !== 'string' ||
|
|
20
|
+
!row.nodeId ||
|
|
21
|
+
typeof row.contentHash !== 'string' ||
|
|
22
|
+
!row.contentHash ||
|
|
23
|
+
row.contentHash !== group.hash ||
|
|
24
|
+
!Number.isInteger(row.chunkIndex) ||
|
|
25
|
+
row.chunkIndex < 0 ||
|
|
26
|
+
group.ordinals.has(row.chunkIndex) ||
|
|
27
|
+
!Number.isInteger(row.startLine) ||
|
|
28
|
+
row.startLine < 0 ||
|
|
29
|
+
!Number.isInteger(row.endLine) ||
|
|
30
|
+
row.endLine < row.startLine)
|
|
31
|
+
group.invalid = true;
|
|
32
|
+
group.ordinals.add(row.chunkIndex);
|
|
33
|
+
}
|
|
34
|
+
const accepted = new Set();
|
|
35
|
+
for (const [nodeId, group] of groups) {
|
|
36
|
+
if (group.invalid || excludedNodeIds.has(nodeId))
|
|
37
|
+
continue;
|
|
38
|
+
// Unique ordinals with max n-1 and zero present have no holes.
|
|
39
|
+
if (!group.ordinals.has(0))
|
|
40
|
+
continue;
|
|
41
|
+
if ([...group.ordinals].some((ordinal) => ordinal >= group.ordinals.size))
|
|
42
|
+
continue;
|
|
43
|
+
accepted.add(nodeId);
|
|
44
|
+
}
|
|
45
|
+
return accepted;
|
|
46
|
+
}
|
|
47
|
+
/** Whole recovered nodes replace whole published groups; vectors stay in bounded batches. */
|
|
48
|
+
export function mergeRecoveredEmbeddings(live, recovered) {
|
|
49
|
+
const builder = createCachedEmbeddingsBuilder({ inMemoryRowLimit: 0 });
|
|
50
|
+
try {
|
|
51
|
+
appendSnapshotRows(live, live.rows.filter((row) => !recovered.embeddingNodeIds.has(row.nodeId)), builder);
|
|
52
|
+
appendSnapshotRows(recovered, recovered.rows, builder);
|
|
53
|
+
return finalizeCachedEmbeddingsSnapshot(builder);
|
|
54
|
+
}
|
|
55
|
+
catch (err) {
|
|
56
|
+
abortCachedEmbeddingsBuilder(builder);
|
|
57
|
+
throw err;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
function appendSnapshotRows(snapshot, rows, builder) {
|
|
61
|
+
const reader = snapshot.spill ? new EmbeddingSpillReader(snapshot.spill) : undefined;
|
|
62
|
+
try {
|
|
63
|
+
for (let i = 0; i < rows.length; i += RESTORE_BATCH_SIZE) {
|
|
64
|
+
const batch = materializeCachedEmbeddings(snapshot, rows.slice(i, i + RESTORE_BATCH_SIZE), reader);
|
|
65
|
+
for (const row of batch)
|
|
66
|
+
ingestCachedEmbeddingRow(builder, row, true);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
finally {
|
|
70
|
+
reader?.close();
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Caller must validate checkpoint identity, schema, exact generation, and hold the
|
|
75
|
+
* index lock. A subprocess contains native WAL replay/query/destructor failures.
|
|
76
|
+
* A failed strict open is never retried with WAL removed or validation disabled.
|
|
77
|
+
*/
|
|
78
|
+
export async function recoverStagedEmbeddings(dbPath, options) {
|
|
79
|
+
if (!Number.isInteger(options.dimensions) || options.dimensions <= 0) {
|
|
80
|
+
throw new Error('invalid staged embedding dimensions');
|
|
81
|
+
}
|
|
82
|
+
const dbStat = fs.lstatSync(dbPath);
|
|
83
|
+
if (!dbStat.isFile() || dbStat.isSymbolicLink())
|
|
84
|
+
throw new Error('staged embedding DB is not a regular file');
|
|
85
|
+
const exportDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-stage-export-'));
|
|
86
|
+
try {
|
|
87
|
+
await runExtractionChild(dbPath, exportDir, options);
|
|
88
|
+
const manifestPath = path.join(exportDir, 'manifest.json');
|
|
89
|
+
const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));
|
|
90
|
+
if (manifest.version !== 1 ||
|
|
91
|
+
manifest.dimensions !== options.dimensions ||
|
|
92
|
+
!Array.isArray(manifest.rows) ||
|
|
93
|
+
!Array.isArray(manifest.rejectedNodeIds) ||
|
|
94
|
+
manifest.rejectedNodeIds.some((id) => typeof id !== 'string') ||
|
|
95
|
+
manifest.rows.some((row, i) => !row || row.vectorIndex !== i))
|
|
96
|
+
throw new Error('invalid staged embedding export manifest');
|
|
97
|
+
if (manifest.rows.length === 0)
|
|
98
|
+
return emptyCachedEmbeddingsSnapshot();
|
|
99
|
+
const spill = {
|
|
100
|
+
path: path.join(exportDir, 'vectors.bin'),
|
|
101
|
+
dims: options.dimensions,
|
|
102
|
+
rowCount: manifest.rows.length,
|
|
103
|
+
};
|
|
104
|
+
const vectorStat = fs.lstatSync(spill.path);
|
|
105
|
+
if (!vectorStat.isFile() ||
|
|
106
|
+
vectorStat.isSymbolicLink() ||
|
|
107
|
+
vectorStat.size !== 12 + spill.rowCount * spill.dims * 4) {
|
|
108
|
+
throw new Error('invalid staged embedding export size');
|
|
109
|
+
}
|
|
110
|
+
const excluded = new Set(options.excludedNodeIds ?? []);
|
|
111
|
+
for (const nodeId of manifest.rejectedNodeIds)
|
|
112
|
+
excluded.add(nodeId);
|
|
113
|
+
const accepted = validateRecoveredNodeGroups(manifest.rows, excluded);
|
|
114
|
+
const exported = {
|
|
115
|
+
rows: manifest.rows,
|
|
116
|
+
embeddings: [],
|
|
117
|
+
embeddingNodeIds: accepted,
|
|
118
|
+
spill,
|
|
119
|
+
};
|
|
120
|
+
const builder = createCachedEmbeddingsBuilder({ inMemoryRowLimit: 0 });
|
|
121
|
+
const reader = new EmbeddingSpillReader(spill);
|
|
122
|
+
try {
|
|
123
|
+
const rows = manifest.rows.filter((row) => accepted.has(row.nodeId));
|
|
124
|
+
for (let i = 0; i < rows.length; i += RESTORE_BATCH_SIZE) {
|
|
125
|
+
for (const row of materializeCachedEmbeddings(exported, rows.slice(i, i + RESTORE_BATCH_SIZE), reader)) {
|
|
126
|
+
if (row.embedding.length !== options.dimensions ||
|
|
127
|
+
row.embedding.some((value) => !Number.isFinite(value))) {
|
|
128
|
+
throw new Error('invalid staged embedding vector');
|
|
129
|
+
}
|
|
130
|
+
ingestCachedEmbeddingRow(builder, row, true);
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
return finalizeCachedEmbeddingsSnapshot(builder);
|
|
134
|
+
}
|
|
135
|
+
catch (err) {
|
|
136
|
+
abortCachedEmbeddingsBuilder(builder);
|
|
137
|
+
throw err;
|
|
138
|
+
}
|
|
139
|
+
finally {
|
|
140
|
+
reader.close();
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
finally {
|
|
144
|
+
fs.rmSync(exportDir, { recursive: true, force: true });
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
function runExtractionChild(dbPath, exportDir, options) {
|
|
148
|
+
const compiledPath = fileURLToPath(new URL('./staged-embedding-recovery-child.js', import.meta.url));
|
|
149
|
+
const sourcePath = compiledPath.replace(/\.js$/, '.ts');
|
|
150
|
+
const childPath = fs.existsSync(compiledPath) ? compiledPath : sourcePath;
|
|
151
|
+
const args = childPath.endsWith('.ts')
|
|
152
|
+
? ['--import', import.meta.resolve('tsx'), childPath]
|
|
153
|
+
: [childPath];
|
|
154
|
+
args.push(dbPath, exportDir, String(options.dimensions));
|
|
155
|
+
return new Promise((resolve, reject) => {
|
|
156
|
+
const child = spawn(process.execPath, args, {
|
|
157
|
+
stdio: ['ignore', 'ignore', 'pipe'],
|
|
158
|
+
windowsHide: true,
|
|
159
|
+
});
|
|
160
|
+
let stderr = '';
|
|
161
|
+
let timedOut = false;
|
|
162
|
+
child.stderr.on('data', (chunk) => {
|
|
163
|
+
if (stderr.length < 4096)
|
|
164
|
+
stderr += chunk.toString().slice(0, 4096 - stderr.length);
|
|
165
|
+
});
|
|
166
|
+
const timer = setTimeout(() => {
|
|
167
|
+
timedOut = true;
|
|
168
|
+
// A process boundary is safe to kill even while native code is executing.
|
|
169
|
+
child.kill('SIGKILL');
|
|
170
|
+
}, options.timeoutMs ?? DEFAULT_EXTRACTION_TIMEOUT_MS);
|
|
171
|
+
child.once('error', (err) => {
|
|
172
|
+
clearTimeout(timer);
|
|
173
|
+
reject(err);
|
|
174
|
+
});
|
|
175
|
+
child.once('close', (code, signal) => {
|
|
176
|
+
clearTimeout(timer);
|
|
177
|
+
if (code === 0 && !signal && !timedOut)
|
|
178
|
+
resolve();
|
|
179
|
+
else
|
|
180
|
+
reject(new Error(`staged embedding extraction failed${timedOut ? ' (timeout)' : ` (${signal ?? code})`}${stderr ? `: ${stderr.trim()}` : ''}`));
|
|
181
|
+
});
|
|
182
|
+
});
|
|
183
|
+
}
|
package/dist/core/run-analyze.js
CHANGED
|
@@ -17,7 +17,8 @@ import fs from 'fs/promises';
|
|
|
17
17
|
import { constants as fsConstants, existsSync } from 'node:fs';
|
|
18
18
|
import { randomUUID } from 'node:crypto';
|
|
19
19
|
import { retryRename } from '../storage/fs-atomic.js';
|
|
20
|
-
import { acquireIndexLock, requireExclusiveIndexLock } from '../storage/index-lock.js';
|
|
20
|
+
import { acquireIndexLock, requireExclusiveIndexLock, sweepStagingArtifacts, } from '../storage/index-lock.js';
|
|
21
|
+
import { resolveEmbeddingRecovery } from '../storage/embedding-recovery.js';
|
|
21
22
|
import { invalidateNodeWorkspacePackages } from './ingestion/import-resolvers/node-workspace-packages.js';
|
|
22
23
|
import { logNameFallbackSummary, summarizeNameFallback, countCallsByLanguage, } from './ingestion/scope-resolution/name-fallback-summary.js';
|
|
23
24
|
import { runPipelineFromRepo } from './ingestion/pipeline.js';
|
|
@@ -657,6 +658,8 @@ export async function runFullAnalysis(repoPath, options, callbacks, runnerIdenti
|
|
|
657
658
|
const log = (msg) => callbacks.onLog?.(stripControlCharacters(msg));
|
|
658
659
|
const acquireOpts = {
|
|
659
660
|
log,
|
|
661
|
+
// Resolve and validate the canonical slot under the lock before cleanup.
|
|
662
|
+
sweep: false,
|
|
660
663
|
onWaitStart: () => callbacks.onProgress('lock', 0, 'Waiting for another analyze to finish on this index…'),
|
|
661
664
|
};
|
|
662
665
|
let writeTarget = await resolveWriteTarget(repoPath, options);
|
|
@@ -702,6 +705,7 @@ export async function runFullAnalysis(repoPath, options, callbacks, runnerIdenti
|
|
|
702
705
|
const flatShared = writeTarget.placement.branch ? undefined : writeTarget.sharedStore;
|
|
703
706
|
if (flatShared)
|
|
704
707
|
await seedSharedSlot(flatShared, repoPath, log);
|
|
708
|
+
sweepStagingArtifacts(writeTarget.metaDir, log);
|
|
705
709
|
const slotToLeave = options.noShare ? await optedInSlotToLeave(repoPath) : undefined;
|
|
706
710
|
const result = await runFullAnalysisInner(repoPath, options, callbacks, writeTarget, contentRetention, runnerIdentityAtBootstrap);
|
|
707
711
|
if (flatShared) {
|
|
@@ -1144,7 +1148,11 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, c
|
|
|
1144
1148
|
}
|
|
1145
1149
|
if (decision.action === 'abort')
|
|
1146
1150
|
throw new Error(decision.error);
|
|
1147
|
-
log(decision.
|
|
1151
|
+
log(decision.action === 'resume' && checkpoint.recovery
|
|
1152
|
+
? `Previous analyze recorded an embedding checkpoint (${checkpoint.nodesProcessed}/` +
|
|
1153
|
+
`${checkpoint.totalNodes} nodes); validating staged vectors before retrying ` +
|
|
1154
|
+
`${decision.pendingNodeIds.size} pending node(s).`
|
|
1155
|
+
: decision.log);
|
|
1148
1156
|
if (options.dropEmbeddings) {
|
|
1149
1157
|
// --drop-embeddings has always implied a rebuild here; the decision only
|
|
1150
1158
|
// covers the marker.
|
|
@@ -1877,6 +1885,68 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, c
|
|
|
1877
1885
|
}
|
|
1878
1886
|
}
|
|
1879
1887
|
}
|
|
1888
|
+
// A checkpoint's pending decision and its paid, complete vectors are
|
|
1889
|
+
// independent: --force discards the former but can still reuse the latter.
|
|
1890
|
+
// Select only the explicitly referenced generation, never an orphan by age.
|
|
1891
|
+
const stagedRecovery = resolveEmbeddingRecovery(metaDir, existingMeta?.embeddingCheckpoint);
|
|
1892
|
+
const stagedCheckpoint = existingMeta?.embeddingCheckpoint;
|
|
1893
|
+
const inheritedUnsafeNodeIds = new Set([
|
|
1894
|
+
...pendingEmbeddingNodeIds,
|
|
1895
|
+
...(stagedRecovery?.unsafeNodeIds ?? []),
|
|
1896
|
+
]);
|
|
1897
|
+
if (shouldLoadCache && !options.dropEmbeddings && stagedRecovery && stagedCheckpoint) {
|
|
1898
|
+
if (!embeddingIdentityForRun) {
|
|
1899
|
+
const { resolveEmbeddingIdentity } = await import('./embeddings/embedding-identity.js');
|
|
1900
|
+
embeddingIdentityForRun = resolveEmbeddingIdentity();
|
|
1901
|
+
}
|
|
1902
|
+
const marker = stagedCheckpoint;
|
|
1903
|
+
const matchesIdentity = marker.model === embeddingIdentityForRun.model &&
|
|
1904
|
+
marker.dimensions === embeddingIdentityForRun.dimensions &&
|
|
1905
|
+
marker.provider === embeddingIdentityForRun.provider;
|
|
1906
|
+
if (matchesIdentity && stagedRecovery.schemaFingerprint === SCHEMA_FINGERPRINT) {
|
|
1907
|
+
// A force-discarded pending decision must not turn a known incomplete
|
|
1908
|
+
// inherited group into a reusable cache merely because its hash matches.
|
|
1909
|
+
if (inheritedUnsafeNodeIds.size > 0) {
|
|
1910
|
+
const rows = cachedSnapshot.rows.filter((row) => !inheritedUnsafeNodeIds.has(row.nodeId));
|
|
1911
|
+
cachedSnapshot = {
|
|
1912
|
+
...cachedSnapshot,
|
|
1913
|
+
rows,
|
|
1914
|
+
embeddingNodeIds: new Set(rows.map((row) => row.nodeId)),
|
|
1915
|
+
};
|
|
1916
|
+
}
|
|
1917
|
+
let recovered;
|
|
1918
|
+
try {
|
|
1919
|
+
const { recoverStagedEmbeddings, mergeRecoveredEmbeddings } = await import('./embeddings/staged-embedding-recovery.js');
|
|
1920
|
+
recovered = await recoverStagedEmbeddings(stagedRecovery.dbPath, {
|
|
1921
|
+
dimensions: embeddingIdentityForRun.dimensions,
|
|
1922
|
+
excludedNodeIds: stagedRecovery.unsafeNodeIds,
|
|
1923
|
+
});
|
|
1924
|
+
const liveCacheDims = snapshotEmbeddingDims(cachedSnapshot);
|
|
1925
|
+
if (liveCacheDims !== undefined && liveCacheDims !== embeddingIdentityForRun.dimensions) {
|
|
1926
|
+
log(`Embedding dimensions changed (${liveCacheDims}d -> ` +
|
|
1927
|
+
`${embeddingIdentityForRun.dimensions}d), discarding published cache`);
|
|
1928
|
+
discardCachedEmbeddings();
|
|
1929
|
+
}
|
|
1930
|
+
if (recovered.rows.length > 0) {
|
|
1931
|
+
const merged = mergeRecoveredEmbeddings(cachedSnapshot, recovered);
|
|
1932
|
+
disposeEmbeddingSpill(cachedSnapshot.spill);
|
|
1933
|
+
adoptCachedEmbeddings(merged);
|
|
1934
|
+
}
|
|
1935
|
+
log(`Recovered ${recovered.rows.length} complete staged embedding chunk(s) ` +
|
|
1936
|
+
`for ${recovered.embeddingNodeIds.size} node(s); unchanged content can reuse them.`);
|
|
1937
|
+
}
|
|
1938
|
+
catch (err) {
|
|
1939
|
+
log(`Warning: could not recover staged embeddings (${err.message}); ` +
|
|
1940
|
+
'the retry will regenerate missing chunks.');
|
|
1941
|
+
}
|
|
1942
|
+
finally {
|
|
1943
|
+
disposeEmbeddingSpill(recovered?.spill);
|
|
1944
|
+
}
|
|
1945
|
+
}
|
|
1946
|
+
else {
|
|
1947
|
+
log('Staged embedding identity or schema changed; its vectors will not be reused.');
|
|
1948
|
+
}
|
|
1949
|
+
}
|
|
1880
1950
|
// ── Load incremental parse cache ──────────────────────────────────
|
|
1881
1951
|
// Content-addressed: `--force` reuses parser shards; `useParseCache: false`
|
|
1882
1952
|
// stages a new generation under a run-unique parse-rebuild.* dir and publishes
|
|
@@ -3505,6 +3575,7 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, c
|
|
|
3505
3575
|
const vectorIndexReady = await buildVectorIndex();
|
|
3506
3576
|
semanticMode = vectorIndexReady ? 'vector-index' : 'exact-scan';
|
|
3507
3577
|
}
|
|
3578
|
+
let stagedCheckpointEmbeddingCount;
|
|
3508
3579
|
if (!embeddingSkipped) {
|
|
3509
3580
|
const { isHttpMode } = await import('./embeddings/http-client.js');
|
|
3510
3581
|
const httpMode = isHttpMode();
|
|
@@ -3515,6 +3586,14 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, c
|
|
|
3515
3586
|
embeddingIdentityForRun = resolveEmbeddingIdentity();
|
|
3516
3587
|
}
|
|
3517
3588
|
const embeddingIdentity = embeddingIdentityForRun;
|
|
3589
|
+
const stagedRecoveryEnabled = useAtomicSwap && isManualCheckpointEnabled();
|
|
3590
|
+
if (useAtomicSwap && !stagedRecoveryEnabled) {
|
|
3591
|
+
log('Manual WAL checkpoints are disabled; new staged work cannot be recovered after interruption. ' +
|
|
3592
|
+
'Any previous durable recovery source is retained until publication.');
|
|
3593
|
+
}
|
|
3594
|
+
const unsafeRecoveryNodeIds = new Set([...inheritedUnsafeNodeIds, ...restoreFailedNodeIds]);
|
|
3595
|
+
let activeWindowNodeIds = [];
|
|
3596
|
+
let recoveryGenerationDurable = false;
|
|
3518
3597
|
// Build a Map<nodeId, contentHash> from cached embeddings for incremental mode
|
|
3519
3598
|
let existingEmbeddings;
|
|
3520
3599
|
if (cachedSnapshot.embeddingNodeIds.size > 0) {
|
|
@@ -3550,13 +3629,22 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, c
|
|
|
3550
3629
|
// /api/embed checkpoint writer in server/api.ts already uses, which also
|
|
3551
3630
|
// keeps a concurrent writer's update from being reverted by a stale
|
|
3552
3631
|
// snapshot) and replace ONLY `embeddingCheckpoint` — plus
|
|
3553
|
-
// `stats.embeddings` when the caller actually MEASURED the
|
|
3554
|
-
// (the post-window `onCheckpoint`
|
|
3555
|
-
//
|
|
3556
|
-
//
|
|
3557
|
-
// had just written.
|
|
3632
|
+
// `stats.embeddings` when the caller actually MEASURED the published
|
|
3633
|
+
// index (the post-window `onCheckpoint` on an in-place build). A staging
|
|
3634
|
+
// build's count is not published until the atomic swap succeeds. The
|
|
3635
|
+
// window-start callback passes nothing, preserving the latest count.
|
|
3558
3636
|
const saveEmbeddingCheckpoint = async (checkpoint, pendingNodeIds, embeddings) => {
|
|
3637
|
+
if (embeddings !== undefined && buildPath !== lbugPath) {
|
|
3638
|
+
stagedCheckpointEmbeddingCount = embeddings;
|
|
3639
|
+
}
|
|
3559
3640
|
const latestMeta = (await loadMeta(metaDir)) ?? existingMeta;
|
|
3641
|
+
// An in-place write or manual-checkpoint opt-out cannot create a new
|
|
3642
|
+
// recoverable staged generation. Keep the complete previous receipt:
|
|
3643
|
+
// updated progress or unsafe nodes would describe different source bytes.
|
|
3644
|
+
const preservedRecoveryCheckpoint = !stagedRecoveryEnabled &&
|
|
3645
|
+
resolveEmbeddingRecovery(metaDir, latestMeta?.embeddingCheckpoint)
|
|
3646
|
+
? latestMeta?.embeddingCheckpoint
|
|
3647
|
+
: undefined;
|
|
3560
3648
|
// First-ever analyze of this repo: no meta exists on disk yet (the
|
|
3561
3649
|
// pre-wipe dirty stamp only fires when one does). Mint the minimum
|
|
3562
3650
|
// RepoMeta requires, with `lastCommit: ''` — never `currentCommit` —
|
|
@@ -3567,12 +3655,26 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, c
|
|
|
3567
3655
|
lastCommit: '',
|
|
3568
3656
|
indexedAt: new Date().toISOString(),
|
|
3569
3657
|
};
|
|
3658
|
+
const interrupted = mintInterruptedCheckpoint(embeddingIdentity, checkpoint, pendingNodeIds);
|
|
3570
3659
|
await saveMeta(metaDir, {
|
|
3571
3660
|
...base,
|
|
3572
|
-
...(embeddings === undefined
|
|
3661
|
+
...(embeddings === undefined || buildPath !== lbugPath
|
|
3662
|
+
? {}
|
|
3663
|
+
: { stats: { ...base.stats, embeddings } }),
|
|
3573
3664
|
// Written by a run that is still IN FLIGHT — see the `kind` doc in
|
|
3574
3665
|
// repo-manager.ts.
|
|
3575
|
-
embeddingCheckpoint:
|
|
3666
|
+
embeddingCheckpoint: preservedRecoveryCheckpoint ?? {
|
|
3667
|
+
...interrupted,
|
|
3668
|
+
...(stagedRecoveryEnabled
|
|
3669
|
+
? {
|
|
3670
|
+
recovery: {
|
|
3671
|
+
stagingFile: path.basename(buildPath),
|
|
3672
|
+
schemaFingerprint: SCHEMA_FINGERPRINT,
|
|
3673
|
+
unsafeNodeIds: [...unsafeRecoveryNodeIds],
|
|
3674
|
+
},
|
|
3675
|
+
}
|
|
3676
|
+
: {}),
|
|
3677
|
+
},
|
|
3576
3678
|
});
|
|
3577
3679
|
};
|
|
3578
3680
|
embeddingResult = await runEmbeddingPipeline(executeQuery, executeWithReusedStatement, (p) => {
|
|
@@ -3586,7 +3688,21 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, c
|
|
|
3586
3688
|
}, {}, cachedSnapshot.embeddingNodeIds.size > 0 ? cachedSnapshot.embeddingNodeIds : undefined, existingEmbeddings, {
|
|
3587
3689
|
forceReembedNodeIds: pendingEmbeddingNodeIds,
|
|
3588
3690
|
onCheckpointWindowStart: async ({ nodeIds, ...checkpoint }) => {
|
|
3691
|
+
const handoff = stagedRecoveryEnabled && !recoveryGenerationDurable;
|
|
3692
|
+
if (handoff) {
|
|
3693
|
+
if (!(await checkpointOnce())) {
|
|
3694
|
+
throw new Error('Could not checkpoint restored embeddings before recovery handoff.');
|
|
3695
|
+
}
|
|
3696
|
+
recoveryGenerationDurable = true;
|
|
3697
|
+
}
|
|
3698
|
+
activeWindowNodeIds = nodeIds;
|
|
3699
|
+
for (const id of nodeIds)
|
|
3700
|
+
unsafeRecoveryNodeIds.add(id);
|
|
3589
3701
|
await saveEmbeddingCheckpoint(checkpoint, nodeIds);
|
|
3702
|
+
// Reclaim the old source only after the new durable generation's
|
|
3703
|
+
// reference is saved. Later windows retain this same generation.
|
|
3704
|
+
if (handoff)
|
|
3705
|
+
sweepStagingArtifacts(metaDir, log);
|
|
3590
3706
|
},
|
|
3591
3707
|
// ── The mid-run count is a DIAGNOSTIC, not a gate (#2790) ──────
|
|
3592
3708
|
// This used to run the count query bare. THIS callback's rejection
|
|
@@ -3600,7 +3716,13 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, c
|
|
|
3600
3716
|
// touch stats.embeddings" signal — so the checkpoint still lands,
|
|
3601
3717
|
// with whatever count is already on disk left alone.
|
|
3602
3718
|
onCheckpoint: async (checkpoint) => {
|
|
3603
|
-
await checkpointOnce();
|
|
3719
|
+
const durable = await checkpointOnce();
|
|
3720
|
+
if (stagedRecoveryEnabled && !durable) {
|
|
3721
|
+
throw new Error('Could not checkpoint the completed embedding window for recovery.');
|
|
3722
|
+
}
|
|
3723
|
+
for (const id of activeWindowNodeIds)
|
|
3724
|
+
unsafeRecoveryNodeIds.delete(id);
|
|
3725
|
+
activeWindowNodeIds = [];
|
|
3604
3726
|
const measured = await measurePersistedEmbeddingCount(executeQuery);
|
|
3605
3727
|
if (measured.kind === 'unknown') {
|
|
3606
3728
|
log(`Warning: could not measure persisted embeddings at the embedding checkpoint ` +
|
|
@@ -3726,11 +3848,10 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, c
|
|
|
3726
3848
|
// already written to disk: prior meta says 0, a clean run inserts
|
|
3727
3849
|
// embeddings and checkpoints the real count, the final probe is
|
|
3728
3850
|
// unavailable, and finalization carries the stale 0 forward while reporting
|
|
3729
|
-
// success.
|
|
3730
|
-
//
|
|
3731
|
-
// discipline applied to the same field.
|
|
3851
|
+
// success. For a staged build, use its last measured count only in the
|
|
3852
|
+
// final meta, written after the swap; never publish it at a checkpoint.
|
|
3732
3853
|
const latestMetaForCount = embeddingCount === undefined ? ((await loadMeta(metaDir)) ?? existingMeta) : undefined;
|
|
3733
|
-
const persistedEmbeddingCount = resolvePersistedEmbeddingCount(measuredEmbeddingCount, latestMetaForCount?.stats?.embeddings);
|
|
3854
|
+
const persistedEmbeddingCount = resolvePersistedEmbeddingCount(measuredEmbeddingCount, stagedCheckpointEmbeddingCount ?? latestMetaForCount?.stats?.embeddings);
|
|
3734
3855
|
const { getRuntimeCapabilities } = await import('./platform/capabilities.js');
|
|
3735
3856
|
const runtimeCapabilities = getRuntimeCapabilities();
|
|
3736
3857
|
// `semanticMode` is authoritative when set (Phase 4 reported what it
|
|
@@ -4018,6 +4139,7 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, c
|
|
|
4018
4139
|
// is a crash-safety improvement: a failed swap leaves the previous index
|
|
4019
4140
|
// live and the next run recovers via the full-rebuild path.
|
|
4020
4141
|
await saveMeta(metaDir, meta);
|
|
4142
|
+
sweepStagingArtifacts(metaDir, log);
|
|
4021
4143
|
// Registry freshness is published only after the graph and its metadata.
|
|
4022
4144
|
// A failed close, swap, or metadata save must leave the previous registry
|
|
4023
4145
|
// receipt intact, just as it leaves the cache unpublished.
|
|
@@ -4213,7 +4335,11 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, c
|
|
|
4213
4335
|
// rethrow below is the surface, and the lock's sweep remains the backstop.
|
|
4214
4336
|
if (useAtomicSwap && buildPath !== lbugPath) {
|
|
4215
4337
|
try {
|
|
4216
|
-
await
|
|
4338
|
+
const recovery = resolveEmbeddingRecovery(metaDir, (await loadMeta(metaDir))?.embeddingCheckpoint);
|
|
4339
|
+
// Both paths belong to this locked slot. The validated generation
|
|
4340
|
+
// basename identifies the same file even through a directory alias.
|
|
4341
|
+
if (recovery?.stagingFile !== path.basename(buildPath))
|
|
4342
|
+
await wipeLbugDbFiles(buildPath);
|
|
4217
4343
|
}
|
|
4218
4344
|
catch {
|
|
4219
4345
|
/* swallow — orphan reclamation must never mask the real failure */
|
|
@@ -4226,6 +4352,10 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, c
|
|
|
4226
4352
|
// IndexLockTimeoutError and other domain failures with `instanceof`.
|
|
4227
4353
|
recordLiveIndexMutationRisk(err);
|
|
4228
4354
|
}
|
|
4355
|
+
if (/max(?:imum)?(?: database| db)? size|database size limit|maxDBSize/i.test(String(err))) {
|
|
4356
|
+
log('The database size limit was reached. Set GITNEXUS_LBUG_MAX_DB_SIZE to a larger ' +
|
|
4357
|
+
'byte limit before retrying analyze; retained complete embeddings can be reused.');
|
|
4358
|
+
}
|
|
4229
4359
|
throw err;
|
|
4230
4360
|
}
|
|
4231
4361
|
}
|
package/dist/server/api.js
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
* Security: binds to localhost by default (use --host to override).
|
|
8
8
|
* CORS is restricted to localhost, private/LAN networks, and the deployed site.
|
|
9
9
|
*/
|
|
10
|
-
import { acquireIndexLock, IndexLockTimeoutError, requireExclusiveIndexLock, } from '../storage/index-lock.js';
|
|
10
|
+
import { acquireIndexLock, IndexLockTimeoutError, requireExclusiveIndexLock, sweepStagingArtifacts, } from '../storage/index-lock.js';
|
|
11
11
|
import { ensurePrivateSharedGraph } from '../core/shared-store-analyze.js';
|
|
12
12
|
import { resolveGraphPath } from '../storage/shared-store.js';
|
|
13
13
|
import { reclaimAfterSlotRemoval, removeCheckoutStorage, } from '../storage/shared-store-lifecycle.js';
|
|
@@ -1797,8 +1797,16 @@ export const createServer = async (port, host = '127.0.0.1') => {
|
|
|
1797
1797
|
// for the whole embedding write, released in the finally below.
|
|
1798
1798
|
let slotLock;
|
|
1799
1799
|
try {
|
|
1800
|
-
slotLock = await acquireIndexLock(storagePath);
|
|
1800
|
+
slotLock = await acquireIndexLock(storagePath, { sweep: false });
|
|
1801
1801
|
requireExclusiveIndexLock(slotLock, `Cannot acquire the index lock at ${storagePath}; refusing an unlocked embedding run.`);
|
|
1802
|
+
// This writer cannot recover staged generations. Preserve their
|
|
1803
|
+
// receipts, including malformed ones, before sweeping or writing.
|
|
1804
|
+
const recoveryCheckpoint = (await loadMeta(storagePath))?.embeddingCheckpoint;
|
|
1805
|
+
if (recoveryCheckpoint && Object.hasOwn(recoveryCheckpoint, 'recovery')) {
|
|
1806
|
+
throw new Error('Cannot generate embeddings: the index checkpoint references staged embeddings. ' +
|
|
1807
|
+
'Run `gitnexus analyze` to recover them first.');
|
|
1808
|
+
}
|
|
1809
|
+
sweepStagingArtifacts(storagePath);
|
|
1802
1810
|
// Writes go to the slot's own graph; a shared-store checkout
|
|
1803
1811
|
// reading an immutable commit graph (#3352) takes a private copy.
|
|
1804
1812
|
if (!(await ensurePrivateSharedGraph(storagePath, () => { }))) {
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { EmbeddingRecoveryReference } from './repo-meta.js';
|
|
2
|
+
export declare const FAMILY_SUFFIXES: readonly ['', '.wal', '.shadow', '.wal.checkpoint', '.lock', '.checkpoint.intent.lock', '.checkpoint.apply.lock'];
|
|
3
|
+
export interface ResolvedEmbeddingRecovery extends EmbeddingRecoveryReference {
|
|
4
|
+
dbPath: string;
|
|
5
|
+
/** Exact basenames, never a prefix match that can preserve another generation. */
|
|
6
|
+
familyFiles: string[];
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* Resolve only an explicit interrupted-generation receipt in this canonical
|
|
10
|
+
* slot. This validates provenance, not native contents or current identity;
|
|
11
|
+
* those checks must pass separately before any rows can be reused.
|
|
12
|
+
*/
|
|
13
|
+
export declare const resolveEmbeddingRecovery: (lockDir: string, checkpoint: unknown) => ResolvedEmbeddingRecovery | undefined;
|
|
14
|
+
/** Synchronous mirror of loadMeta's primary-first, absent-only fallback rule. */
|
|
15
|
+
export declare const readEmbeddingRecovery: (lockDir: string) => ResolvedEmbeddingRecovery | undefined;
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Filesystem-only staged embedding provenance. Keep this independent of native
|
|
3
|
+
* and model imports: every index-lock caller needs the retention decision.
|
|
4
|
+
*/
|
|
5
|
+
import { closeSync, constants, fstatSync, lstatSync, openSync, readFileSync, realpathSync, } from 'node:fs';
|
|
6
|
+
import path from 'node:path';
|
|
7
|
+
import { INDEX_METADATA_FILE, LEGACY_METADATA_FILE } from './storage-constants.js';
|
|
8
|
+
const STAGING_FILENAME = /^lbug\.staging\.[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
|
|
9
|
+
export const FAMILY_SUFFIXES = [
|
|
10
|
+
'',
|
|
11
|
+
'.wal',
|
|
12
|
+
'.shadow',
|
|
13
|
+
'.wal.checkpoint',
|
|
14
|
+
'.lock',
|
|
15
|
+
'.checkpoint.intent.lock',
|
|
16
|
+
'.checkpoint.apply.lock',
|
|
17
|
+
];
|
|
18
|
+
const isRecord = (value) => value !== null && typeof value === 'object' && !Array.isArray(value);
|
|
19
|
+
const isNonemptyString = (value) => typeof value === 'string' && value.length > 0;
|
|
20
|
+
const isNodeIds = (value) => Array.isArray(value) && value.every(isNonemptyString);
|
|
21
|
+
const isCount = (value) => typeof value === 'number' && Number.isSafeInteger(value) && value >= 0;
|
|
22
|
+
/**
|
|
23
|
+
* Resolve only an explicit interrupted-generation receipt in this canonical
|
|
24
|
+
* slot. This validates provenance, not native contents or current identity;
|
|
25
|
+
* those checks must pass separately before any rows can be reused.
|
|
26
|
+
*/
|
|
27
|
+
export const resolveEmbeddingRecovery = (lockDir, checkpoint) => {
|
|
28
|
+
if (!isRecord(checkpoint) || !isRecord(checkpoint.recovery))
|
|
29
|
+
return undefined;
|
|
30
|
+
if (checkpoint.kind !== undefined && checkpoint.kind !== 'interrupted')
|
|
31
|
+
return undefined;
|
|
32
|
+
if (!isNonemptyString(checkpoint.at) ||
|
|
33
|
+
!Number.isFinite(Date.parse(checkpoint.at)) ||
|
|
34
|
+
!isCount(checkpoint.nodesProcessed) ||
|
|
35
|
+
!isCount(checkpoint.totalNodes) ||
|
|
36
|
+
checkpoint.nodesProcessed > checkpoint.totalNodes ||
|
|
37
|
+
!isCount(checkpoint.chunksProcessed) ||
|
|
38
|
+
!isNonemptyString(checkpoint.model) ||
|
|
39
|
+
!isCount(checkpoint.dimensions) ||
|
|
40
|
+
checkpoint.dimensions === 0 ||
|
|
41
|
+
!isNonemptyString(checkpoint.provider) ||
|
|
42
|
+
(checkpoint.pendingNodeIds !== undefined && !isNodeIds(checkpoint.pendingNodeIds))) {
|
|
43
|
+
return undefined;
|
|
44
|
+
}
|
|
45
|
+
const recovery = checkpoint.recovery;
|
|
46
|
+
if (!isNonemptyString(recovery.stagingFile) ||
|
|
47
|
+
!STAGING_FILENAME.test(recovery.stagingFile) ||
|
|
48
|
+
!isNonemptyString(recovery.schemaFingerprint) ||
|
|
49
|
+
!isNodeIds(recovery.unsafeNodeIds)) {
|
|
50
|
+
return undefined;
|
|
51
|
+
}
|
|
52
|
+
const unsafeNodeIds = new Set(recovery.unsafeNodeIds);
|
|
53
|
+
if (isNodeIds(checkpoint.pendingNodeIds) &&
|
|
54
|
+
checkpoint.pendingNodeIds.some((nodeId) => !unsafeNodeIds.has(nodeId))) {
|
|
55
|
+
return undefined;
|
|
56
|
+
}
|
|
57
|
+
try {
|
|
58
|
+
const canonicalDir = realpathSync(lockDir);
|
|
59
|
+
if (!lstatSync(canonicalDir).isDirectory())
|
|
60
|
+
return undefined;
|
|
61
|
+
const familyFiles = FAMILY_SUFFIXES.map((suffix) => recovery.stagingFile + suffix);
|
|
62
|
+
for (const [index, filename] of familyFiles.entries()) {
|
|
63
|
+
try {
|
|
64
|
+
// lstat refuses both live and dangling symlinks, without following one
|
|
65
|
+
// to a database outside the slot. The base file must exist.
|
|
66
|
+
if (!lstatSync(path.join(canonicalDir, filename)).isFile())
|
|
67
|
+
return undefined;
|
|
68
|
+
}
|
|
69
|
+
catch (error) {
|
|
70
|
+
if (index > 0 && error.code === 'ENOENT')
|
|
71
|
+
continue;
|
|
72
|
+
return undefined;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
return {
|
|
76
|
+
dbPath: path.join(canonicalDir, recovery.stagingFile),
|
|
77
|
+
stagingFile: recovery.stagingFile,
|
|
78
|
+
schemaFingerprint: recovery.schemaFingerprint,
|
|
79
|
+
unsafeNodeIds: [...unsafeNodeIds],
|
|
80
|
+
familyFiles,
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
catch {
|
|
84
|
+
return undefined;
|
|
85
|
+
}
|
|
86
|
+
};
|
|
87
|
+
/** Synchronous mirror of loadMeta's primary-first, absent-only fallback rule. */
|
|
88
|
+
export const readEmbeddingRecovery = (lockDir) => {
|
|
89
|
+
let metadataPath = path.join(lockDir, INDEX_METADATA_FILE);
|
|
90
|
+
let descriptor;
|
|
91
|
+
const noFollow = constants.O_NOFOLLOW ?? 0;
|
|
92
|
+
const flags = constants.O_RDONLY | noFollow | (constants.O_NONBLOCK ?? 0);
|
|
93
|
+
try {
|
|
94
|
+
try {
|
|
95
|
+
descriptor = openSync(metadataPath, flags);
|
|
96
|
+
}
|
|
97
|
+
catch (error) {
|
|
98
|
+
const code = error.code;
|
|
99
|
+
if (code !== 'ENOENT' && code !== 'ENOTDIR')
|
|
100
|
+
return undefined;
|
|
101
|
+
// Windows cannot open with O_NOFOLLOW: an open of a dangling symlink
|
|
102
|
+
// reports ENOENT, but that existing primary entry must prevent fallback.
|
|
103
|
+
try {
|
|
104
|
+
lstatSync(metadataPath);
|
|
105
|
+
return undefined;
|
|
106
|
+
}
|
|
107
|
+
catch (statError) {
|
|
108
|
+
const statCode = statError.code;
|
|
109
|
+
if (statCode !== 'ENOENT' && statCode !== 'ENOTDIR')
|
|
110
|
+
return undefined;
|
|
111
|
+
}
|
|
112
|
+
metadataPath = path.join(lockDir, LEGACY_METADATA_FILE);
|
|
113
|
+
descriptor = openSync(metadataPath, flags);
|
|
114
|
+
}
|
|
115
|
+
const opened = fstatSync(descriptor, { bigint: true });
|
|
116
|
+
const entry = lstatSync(metadataPath, { bigint: true });
|
|
117
|
+
// Check the opened file itself and match the current non-symlink entry.
|
|
118
|
+
// This also refuses replacement on platforms without O_NOFOLLOW.
|
|
119
|
+
if (!opened.isFile() ||
|
|
120
|
+
!entry.isFile() ||
|
|
121
|
+
(noFollow === 0 && opened.ino === 0n) ||
|
|
122
|
+
opened.dev !== entry.dev ||
|
|
123
|
+
opened.ino !== entry.ino) {
|
|
124
|
+
return undefined;
|
|
125
|
+
}
|
|
126
|
+
const meta = JSON.parse(readFileSync(descriptor, 'utf8'));
|
|
127
|
+
if (!isRecord(meta))
|
|
128
|
+
return undefined;
|
|
129
|
+
// storagePath describes the flat/cache root, including in branch-slot
|
|
130
|
+
// metadata. The current locked directory is the generation boundary.
|
|
131
|
+
return resolveEmbeddingRecovery(lockDir, meta.embeddingCheckpoint);
|
|
132
|
+
}
|
|
133
|
+
catch {
|
|
134
|
+
return undefined;
|
|
135
|
+
}
|
|
136
|
+
finally {
|
|
137
|
+
if (descriptor !== undefined) {
|
|
138
|
+
try {
|
|
139
|
+
closeSync(descriptor);
|
|
140
|
+
}
|
|
141
|
+
catch {
|
|
142
|
+
/* best-effort */
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
};
|
|
@@ -95,8 +95,9 @@ export declare const isLockUnwritableCode: (code: string | undefined) => boolean
|
|
|
95
95
|
/**
|
|
96
96
|
* Delete orphaned build/staging artifacts left in the lock directory by a
|
|
97
97
|
* crashed prior writer. Safe precisely because we hold the exclusive lock: no
|
|
98
|
-
* other writer can be creating these here right now
|
|
99
|
-
*
|
|
98
|
+
* other writer can be creating these here right now. A checkpoint-referenced
|
|
99
|
+
* generation is retained for embedding recovery; all other stages are orphans.
|
|
100
|
+
* Matches this slot's staging files ONLY — never `lbug` itself,
|
|
100
101
|
* never `lbug.wal`/`lbug.shadow` (the LIVE index's own sidecars), and never a
|
|
101
102
|
* `branches/<slug>/` sub-slot (which owns its own lock + sweep). Non-recursive.
|
|
102
103
|
*/
|
|
@@ -52,6 +52,7 @@ import path from 'node:path';
|
|
|
52
52
|
import os from 'node:os';
|
|
53
53
|
import { randomBytes, randomUUID, createHash } from 'node:crypto';
|
|
54
54
|
import { isProcessAlive } from '../utils/process-identity.js';
|
|
55
|
+
import { readEmbeddingRecovery } from './embedding-recovery.js';
|
|
55
56
|
const LOCK_FILENAME = 'analyze.lock';
|
|
56
57
|
const LOCK_RECORD_VERSION = 1;
|
|
57
58
|
const lockGuardPath = (lockPath) => `${lockPath}.guard`;
|
|
@@ -304,8 +305,9 @@ const deniedCreateHandle = (lockPath, record, error) => {
|
|
|
304
305
|
/**
|
|
305
306
|
* Delete orphaned build/staging artifacts left in the lock directory by a
|
|
306
307
|
* crashed prior writer. Safe precisely because we hold the exclusive lock: no
|
|
307
|
-
* other writer can be creating these here right now
|
|
308
|
-
*
|
|
308
|
+
* other writer can be creating these here right now. A checkpoint-referenced
|
|
309
|
+
* generation is retained for embedding recovery; all other stages are orphans.
|
|
310
|
+
* Matches this slot's staging files ONLY — never `lbug` itself,
|
|
309
311
|
* never `lbug.wal`/`lbug.shadow` (the LIVE index's own sidecars), and never a
|
|
310
312
|
* `branches/<slug>/` sub-slot (which owns its own lock + sweep). Non-recursive.
|
|
311
313
|
*/
|
|
@@ -313,6 +315,7 @@ export const sweepStagingArtifacts = (lockDir, log) => {
|
|
|
313
315
|
// Matches `lbug.new`, `lbug.new.wal`, `lbug.staging.<id>`, `lbug.staging.<id>.wal`, …
|
|
314
316
|
// Does NOT match `lbug`, `lbug.wal`, `lbug.shadow`.
|
|
315
317
|
const stagingRe = /^lbug\.(staging\..+|new(\..+)?)$/;
|
|
318
|
+
const retained = new Set(readEmbeddingRecovery(lockDir)?.familyFiles ?? []);
|
|
316
319
|
let removed = 0;
|
|
317
320
|
let entries;
|
|
318
321
|
try {
|
|
@@ -322,7 +325,7 @@ export const sweepStagingArtifacts = (lockDir, log) => {
|
|
|
322
325
|
return;
|
|
323
326
|
}
|
|
324
327
|
for (const name of entries) {
|
|
325
|
-
if (!stagingRe.test(name))
|
|
328
|
+
if (!stagingRe.test(name) || retained.has(name))
|
|
326
329
|
continue;
|
|
327
330
|
try {
|
|
328
331
|
unlinkSync(path.join(lockDir, name));
|
|
@@ -35,6 +35,14 @@ export { GITNEXUS_DIR, INDEX_METADATA_FILE, LEGACY_METADATA_FILE } from './stora
|
|
|
35
35
|
export type ContentRetention = 'full' | 'symbol' | 'none';
|
|
36
36
|
export type FtsProfile = 'full' | 'symbol-no-file-content' | 'name-only';
|
|
37
37
|
export declare const CONTENT_RETENTION_SCHEMA_VERSION = 1;
|
|
38
|
+
/** Exact staged generation whose completed embedding groups can survive a retry. */
|
|
39
|
+
export interface EmbeddingRecoveryReference {
|
|
40
|
+
/** A run-minted basename within this metadata file's index slot. */
|
|
41
|
+
stagingFile: string;
|
|
42
|
+
schemaFingerprint: string;
|
|
43
|
+
/** Active-window and inherited incomplete groups: never reusable until completed. */
|
|
44
|
+
unsafeNodeIds: string[];
|
|
45
|
+
}
|
|
38
46
|
/**
|
|
39
47
|
* Versioned receipt for the analyzer process that produced an index.
|
|
40
48
|
*
|
|
@@ -511,6 +519,11 @@ export interface RepoMeta {
|
|
|
511
519
|
* subset of their chunks; for `'partial'` they hold none.
|
|
512
520
|
*/
|
|
513
521
|
pendingNodeIds?: string[];
|
|
522
|
+
/**
|
|
523
|
+
* Interrupted atomic builds only. This does not publish the staged graph
|
|
524
|
+
* or advance live embedding statistics; it identifies a recovery source.
|
|
525
|
+
*/
|
|
526
|
+
recovery?: EmbeddingRecoveryReference;
|
|
514
527
|
};
|
|
515
528
|
/**
|
|
516
529
|
* Name of the git branch this index represents (#2106). Absent for the
|
package/package.json
CHANGED