gitnexus 1.6.12-rc.31 → 1.6.12-rc.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -7,7 +7,8 @@ import { getStoragePaths, loadMeta, saveMeta } from '../storage/repo-manager.js'
|
|
|
7
7
|
import { closeLbug, executeQuery, executeWithReusedStatement, fetchExistingEmbeddingHashes, initLbug, } from '../core/lbug/lbug-adapter.js';
|
|
8
8
|
import { runEmbeddingPipeline } from '../core/embeddings/embedding-pipeline.js';
|
|
9
9
|
import { resolveEmbeddingIdentity } from '../core/embeddings/embedding-identity.js';
|
|
10
|
-
import {
|
|
10
|
+
import { decideEmbeddingResume, mintInterruptedCheckpoint, mintPartialCheckpoint, mintUnverifiedCountCheckpoint, } from '../core/embedding-checkpoint.js';
|
|
11
|
+
import { EMBEDDING_DIMS, embeddingDimsMismatch } from '../core/lbug/schema.js';
|
|
11
12
|
import { measurePersistedEmbeddingCount, persistedEmbeddingCountOrUndefined, } from '../core/embedding-count.js';
|
|
12
13
|
/** Add missing embeddings directly to a healthy index, checkpointing periodically. */
|
|
13
14
|
export const embeddingsSyncCommand = async (inputPath) => {
|
|
@@ -45,10 +46,16 @@ export const embeddingsSyncCommand = async (inputPath) => {
|
|
|
45
46
|
const identityDiffers = checkpoint.provider !== identity.provider ||
|
|
46
47
|
checkpoint.model !== identity.model ||
|
|
47
48
|
checkpoint.dimensions !== identity.dimensions;
|
|
48
|
-
// `abandon` on a
|
|
49
|
-
//
|
|
50
|
-
//
|
|
51
|
-
|
|
49
|
+
// `abandon` on a foreign identity drops the pending set only. Existing
|
|
50
|
+
// rows stay; sync would then embed the holes under the new identity and
|
|
51
|
+
// mix vector spaces. Fail closed — rebuild via analyze.
|
|
52
|
+
//
|
|
53
|
+
// Every kind is gated, `unverified-count` included. Exempting it looked
|
|
54
|
+
// safe because that kind only records "the count could not be read", but
|
|
55
|
+
// `decideEmbeddingResume` returns `abandon` for it BEFORE comparing
|
|
56
|
+
// identity, so the exemption was the only thing standing between a
|
|
57
|
+
// foreign identity and a silently mixed table.
|
|
58
|
+
if (identityDiffers) {
|
|
52
59
|
throw new Error(`Cannot sync embeddings: the index checkpoint was written by ${checkpoint.model} ` +
|
|
53
60
|
`(${checkpoint.provider}) at ${checkpoint.dimensions} dimensions, but this run ` +
|
|
54
61
|
`resolves ${identity.model} (${identity.provider}) at ${identity.dimensions}. ` +
|
|
@@ -60,19 +67,34 @@ export const embeddingsSyncCommand = async (inputPath) => {
|
|
|
60
67
|
resumedFrom = decision.resumedFrom;
|
|
61
68
|
}
|
|
62
69
|
}
|
|
70
|
+
// The vector column is FLOAT[N] fixed when the index was built, and the
|
|
71
|
+
// pipeline deletes each batch's stale rows immediately before inserting the
|
|
72
|
+
// replacements — so a width change here deletes rows it cannot re-insert.
|
|
73
|
+
// `analyze` forces a full rebuild on the same mismatch; only a rebuild can
|
|
74
|
+
// retype the column, so this writer refuses instead. An absent recorded
|
|
75
|
+
// width is not a mismatch (see `embeddingDimsMismatch`).
|
|
76
|
+
if (embeddingDimsMismatch(meta.embeddingDims, EMBEDDING_DIMS)) {
|
|
77
|
+
throw new Error(`Cannot sync embeddings: this index stores FLOAT[${meta.embeddingDims}] vectors, ` +
|
|
78
|
+
`but this run embeds at ${EMBEDDING_DIMS} dimensions. ` +
|
|
79
|
+
'Run `gitnexus analyze --embeddings --force` to rebuild the column at the new width.');
|
|
80
|
+
}
|
|
63
81
|
await initLbug(lbugPath);
|
|
64
82
|
try {
|
|
65
83
|
const existing = await fetchExistingEmbeddingHashes(executeQuery);
|
|
66
84
|
let lastPercent = -1;
|
|
67
85
|
const countEmbeddings = async () => persistedEmbeddingCountOrUndefined(await measurePersistedEmbeddingCount(executeQuery));
|
|
68
|
-
|
|
86
|
+
// One write path for every meta update this command makes. The re-read
|
|
87
|
+
// happens immediately before each save so a concurrent writer's fields
|
|
88
|
+
// survive. #2790 traced two production drifts to hand-copied writers of
|
|
89
|
+
// these exact fields, so this file keeps one copy instead of three.
|
|
90
|
+
const persistMeta = async (patch) => {
|
|
69
91
|
const latest = (await loadMeta(metaDir)) ?? meta;
|
|
70
|
-
await saveMeta(metaDir, {
|
|
71
|
-
...latest,
|
|
72
|
-
...(embeddings === undefined ? {} : { stats: { ...latest.stats, embeddings } }),
|
|
73
|
-
embeddingCheckpoint: mintInterruptedCheckpoint(identity, checkpoint, pendingNodeIds),
|
|
74
|
-
});
|
|
92
|
+
await saveMeta(metaDir, { ...latest, ...patch(latest) });
|
|
75
93
|
};
|
|
94
|
+
const saveCheckpoint = async (checkpoint, pendingNodeIds, embeddings) => persistMeta((latest) => ({
|
|
95
|
+
...(embeddings === undefined ? {} : { stats: { ...latest.stats, embeddings } }),
|
|
96
|
+
embeddingCheckpoint: mintInterruptedCheckpoint(identity, checkpoint, pendingNodeIds),
|
|
97
|
+
}));
|
|
76
98
|
cliInfo(`Embedding ${repoPath}`);
|
|
77
99
|
cliInfo(`Checkpointed nodes already present: ${existing?.size ?? 0}`);
|
|
78
100
|
const result = await runEmbeddingPipeline(executeQuery, executeWithReusedStatement, (progress) => {
|
|
@@ -91,13 +113,11 @@ export const embeddingsSyncCommand = async (inputPath) => {
|
|
|
91
113
|
},
|
|
92
114
|
});
|
|
93
115
|
const embeddings = await countEmbeddings();
|
|
94
|
-
const latest = (await loadMeta(metaDir)) ?? meta;
|
|
95
116
|
if (embeddings === undefined) {
|
|
96
117
|
// Keep last-known stats.embeddings. An interrupted window marker would
|
|
97
118
|
// fail the identity gate on the next run even though this run finished;
|
|
98
119
|
// unverified-count is the recovery kind that forces a recount (#2790).
|
|
99
|
-
await
|
|
100
|
-
...latest,
|
|
120
|
+
await persistMeta(() => ({
|
|
101
121
|
embeddingCheckpoint: result.failedNodeIds.length
|
|
102
122
|
? mintPartialCheckpoint(identity, result, resumedFrom)
|
|
103
123
|
: mintUnverifiedCountCheckpoint(identity, {
|
|
@@ -105,16 +125,15 @@ export const embeddingsSyncCommand = async (inputPath) => {
|
|
|
105
125
|
totalNodes: result.nodesProcessed,
|
|
106
126
|
chunksProcessed: result.chunksProcessed,
|
|
107
127
|
}),
|
|
108
|
-
});
|
|
128
|
+
}));
|
|
109
129
|
throw new Error('Could not verify persisted embedding count.');
|
|
110
130
|
}
|
|
111
|
-
await
|
|
112
|
-
...latest,
|
|
131
|
+
await persistMeta((latest) => ({
|
|
113
132
|
stats: { ...latest.stats, embeddings },
|
|
114
133
|
embeddingCheckpoint: result.failedNodeIds.length
|
|
115
134
|
? mintPartialCheckpoint(identity, result, resumedFrom)
|
|
116
135
|
: undefined,
|
|
117
|
-
});
|
|
136
|
+
}));
|
|
118
137
|
cliInfo(`Embeddings ready: ${embeddings}`);
|
|
119
138
|
}
|
|
120
139
|
finally {
|
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
* via `AbortSignal.timeout` on the underlying fetch.
|
|
12
12
|
*/
|
|
13
13
|
import { chunk } from '../../lib/utils.js';
|
|
14
|
+
import { parseTruthyEnv } from '../ingestion/utils/env.js';
|
|
14
15
|
import { CircuitOpenError, ResilientFetchExhaustedError, isTerminalNetworkError, resilientFetch, } from '../../_shared/index.js';
|
|
15
16
|
const DEFAULT_HTTP_TIMEOUT_MS = 180_000;
|
|
16
17
|
const MAX_HTTP_TIMEOUT_MS = 300_000;
|
|
@@ -157,7 +158,12 @@ const readConfig = () => {
|
|
|
157
158
|
retryCapMs: parsePositiveIntegerEnv('GITNEXUS_EMBEDDING_RETRY_CAP_MS', HTTP_RETRY_CAP_MS, 300_000),
|
|
158
159
|
minIntervalMs: parseNonNegativeIntegerEnv('GITNEXUS_EMBEDDING_MIN_INTERVAL_MS', 0, 300_000),
|
|
159
160
|
timeoutMs: parsePositiveIntegerEnv(HTTP_TIMEOUT_ENV, DEFAULT_HTTP_TIMEOUT_MS, MAX_HTTP_TIMEOUT_MS),
|
|
160
|
-
|
|
161
|
+
// A boolean toggle, so it takes the repo's truthy convention (`1`/`true`/
|
|
162
|
+
// `yes`) and falls back to the documented default on anything else. The
|
|
163
|
+
// integer parser this used throws on a non-digit, which turned the
|
|
164
|
+
// conventional `=true` into a hard failure of every embedding call rather
|
|
165
|
+
// than either enabling the flag or leaving it off.
|
|
166
|
+
retryTimeouts: parseTruthyEnv(process.env.GITNEXUS_EMBEDDING_RETRY_TIMEOUTS),
|
|
161
167
|
requestDimensions,
|
|
162
168
|
};
|
|
163
169
|
};
|
package/package.json
CHANGED