gitnexus 1.6.10-rc.28 → 1.6.10-rc.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -1
- package/dist/_shared/integrations/resilient-fetch.d.ts +3 -1
- package/dist/_shared/integrations/resilient-fetch.d.ts.map +1 -1
- package/dist/_shared/integrations/resilient-fetch.js +4 -3
- package/dist/_shared/integrations/resilient-fetch.js.map +1 -1
- package/dist/cli/tool.js +11 -0
- package/dist/core/embeddings/embedder.d.ts +3 -2
- package/dist/core/embeddings/embedder.js +8 -5
- package/dist/core/embeddings/embedding-identity.d.ts +11 -0
- package/dist/core/embeddings/embedding-identity.js +24 -0
- package/dist/core/embeddings/embedding-pipeline.d.ts +16 -1
- package/dist/core/embeddings/embedding-pipeline.js +86 -6
- package/dist/core/embeddings/http-client.d.ts +5 -2
- package/dist/core/embeddings/http-client.js +109 -13
- package/dist/core/ingestion/pipeline-phases/parse-impl.js +12 -1
- package/dist/core/lbug/lbug-adapter.js +7 -5
- package/dist/core/run-analyze.js +92 -4
- package/dist/server/analyze-job.d.ts +3 -0
- package/dist/server/analyze-job.js +17 -0
- package/dist/server/api.js +51 -11
- package/dist/storage/parse-cache.js +1 -1
- package/dist/storage/parsedfile-store.d.ts +8 -0
- package/dist/storage/parsedfile-store.js +12 -0
- package/dist/storage/repo-manager.d.ts +24 -0
- package/package.json +1 -1
- package/web/assets/{index-BirNeEtL.js → index-CRLoSOyy.js} +1 -1
- package/web/index.html +1 -1
package/README.md
CHANGED
|
@@ -283,10 +283,13 @@ export GITNEXUS_EMBEDDING_URL=http://your-server:8080/v1
|
|
|
283
283
|
export GITNEXUS_EMBEDDING_MODEL=BAAI/bge-large-en-v1.5
|
|
284
284
|
export GITNEXUS_EMBEDDING_DIMS=1024 # optional, default 384
|
|
285
285
|
export GITNEXUS_EMBEDDING_API_KEY=your-key # optional, default: "unused"
|
|
286
|
+
export GITNEXUS_EMBEDDING_MAX_ATTEMPTS=3 # optional, total attempts (1-20)
|
|
287
|
+
export GITNEXUS_EMBEDDING_RETRY_CAP_MS=5000 # optional, maximum retry delay
|
|
288
|
+
export GITNEXUS_EMBEDDING_MIN_INTERVAL_MS=0 # optional, minimum request spacing
|
|
286
289
|
gitnexus analyze . --embeddings
|
|
287
290
|
```
|
|
288
291
|
|
|
289
|
-
Works with Infinity, vLLM, TEI, llama.cpp, Ollama, LM Studio, or OpenAI. When unset, local embeddings are used unchanged.
|
|
292
|
+
Works with Infinity, vLLM, TEI, llama.cpp, Ollama, LM Studio, or OpenAI. Retry and pacing settings are provider-neutral; provider-specific limits should be supplied through configuration. When unset, local embeddings are used unchanged.
|
|
290
293
|
|
|
291
294
|
## Multi-Repo Support
|
|
292
295
|
|
|
@@ -25,6 +25,8 @@ export interface ResilientFetchOptions {
|
|
|
25
25
|
breakerOptions?: CircuitBreakerOptions;
|
|
26
26
|
/** Tuning knobs for the retry helper. */
|
|
27
27
|
retry?: Partial<Pick<RetryOptions, 'maxAttempts' | 'baseDelayMs' | 'capDelayMs'>> & {
|
|
28
|
+
/** Upper bound on a single Retry-After wait. Defaults to RETRY_AFTER_CAP_MS. */
|
|
29
|
+
retryAfterCapMs?: number;
|
|
28
30
|
sleep?: RetryOptions['sleep'];
|
|
29
31
|
random?: RetryOptions['random'];
|
|
30
32
|
};
|
|
@@ -64,7 +66,7 @@ export declare function classifyOutcome(result: {
|
|
|
64
66
|
} | {
|
|
65
67
|
kind: 'response';
|
|
66
68
|
resp: Response;
|
|
67
|
-
}, now: () => number): Outcome;
|
|
69
|
+
}, now: () => number, retryAfterCapMs?: number): Outcome;
|
|
68
70
|
/** Final error thrown when retries are exhausted on a 5xx / 429. */
|
|
69
71
|
export declare class ResilientFetchExhaustedError extends Error {
|
|
70
72
|
readonly response: Response;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"resilient-fetch.d.ts","sourceRoot":"","sources":["../../src/integrations/resilient-fetch.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EACL,cAAc,EACd,gBAAgB,EAEhB,KAAK,qBAAqB,EAC3B,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAoB,KAAK,YAAY,EAAE,MAAM,YAAY,CAAC;AAEjE,OAAO,EAAE,gBAAgB,EAAE,CAAC;AAE5B,MAAM,WAAW,qBAAqB;IACpC,8EAA8E;IAC9E,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IACzB;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,2EAA2E;IAC3E,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,kEAAkE;IAClE,cAAc,CAAC,EAAE,qBAAqB,CAAC;IACvC,yCAAyC;IACzC,KAAK,CAAC,EAAE,OAAO,CAAC,IAAI,CAAC,YAAY,EAAE,aAAa,GAAG,aAAa,GAAG,YAAY,CAAC,CAAC,GAAG;QAClF,KAAK,CAAC,EAAE,YAAY,CAAC,OAAO,CAAC,CAAC;QAC9B,MAAM,CAAC,EAAE,YAAY,CAAC,QAAQ,CAAC,CAAC;KACjC,CAAC;IACF,6EAA6E;IAC7E,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;CACpB;AAED,+EAA+E;AAC/E,eAAO,MAAM,kBAAkB,QAAS,CAAC;AAQzC;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,EAAE,GAAG,GAAE,MAAM,MAAiB,GAAG,MAAM,GAAG,IAAI,CAejG;AAED,wEAAwE;AACxE,KAAK,OAAO,GACR;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,IAAI,EAAE,QAAQ,CAAA;CAAE,GACnC;IAAE,IAAI,EAAE,iBAAiB,CAAC;IAAC,IAAI,EAAE,QAAQ,CAAA;CAAE,GAC3C;IAAE,IAAI,EAAE,kBAAkB,CAAC;IAAC,IAAI,EAAE,QAAQ,CAAC;IAAC,OAAO,EAAE,MAAM,GAAG,SAAS,CAAA;CAAE,GACzE;IAAE,IAAI,EAAE,kBAAkB,CAAC;IAAC,GAAG,EAAE,OAAO,CAAA;CAAE,GAC1C;IAAE,IAAI,EAAE,mBAAmB,CAAC;IAAC,GAAG,EAAE,OAAO,CAAA;CAAE,CAAC;AAEhD,+BAA+B;AAC/B,wBAAgB,eAAe,CAC7B,MAAM,EAAE;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,GAAG,EAAE,OAAO,CAAA;CAAE,GAAG;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,IAAI,EAAE,QAAQ,CAAA;CAAE,EAC9E,GAAG,EAAE,MAAM,MAAM,
|
|
1
|
+
{"version":3,"file":"resilient-fetch.d.ts","sourceRoot":"","sources":["../../src/integrations/resilient-fetch.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EACL,cAAc,EACd,gBAAgB,EAEhB,KAAK,qBAAqB,EAC3B,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAoB,KAAK,YAAY,EAAE,MAAM,YAAY,CAAC;AAEjE,OAAO,EAAE,gBAAgB,EAAE,CAAC;AAE5B,MAAM,WAAW,qBAAqB;IACpC,8EAA8E;IAC9E,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IACzB;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,2EAA2E;IAC3E,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,kEAAkE;IAClE,cAAc,CAAC,EAAE,qBAAqB,CAAC;IACvC,yCAAyC;IACzC,KAAK,CAAC,EAAE,OAAO,CAAC,IAAI,CAAC,YAAY,EAAE,aAAa,GAAG,aAAa,GAAG,YAAY,CAAC,CAAC,GAAG;QAClF,gFAAgF;QAChF,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,KAAK,CAAC,EAAE,YAAY,CAAC,OAAO,CAAC,CAAC;QAC9B,MAAM,CAAC,EAAE,YAAY,CAAC,QAAQ,CAAC,CAAC;KACjC,CAAC;IACF,6EAA6E;IAC7E,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;CACpB;AAED,+EAA+E;AAC/E,eAAO,MAAM,kBAAkB,QAAS,CAAC;AAQzC;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,EAAE,GAAG,GAAE,MAAM,MAAiB,GAAG,MAAM,GAAG,IAAI,CAejG;AAED,wEAAwE;AACxE,KAAK,OAAO,GACR;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,IAAI,EAAE,QAAQ,CAAA;CAAE,GACnC;IAAE,IAAI,EAAE,iBAAiB,CAAC;IAAC,IAAI,EAAE,QAAQ,CAAA;CAAE,GAC3C;IAAE,IAAI,EAAE,kBAAkB,CAAC;IAAC,IAAI,EAAE,QAAQ,CAAC;IAAC,OAAO,EAAE,MAAM,GAAG,SAAS,CAAA;CAAE,GACzE;IAAE,IAAI,EAAE,kBAAkB,CAAC;IAAC,GAAG,EAAE,OAAO,CAAA;CAAE,GAC1C;IAAE,IAAI,EAAE,mBAAmB,CAAC;IAAC,GAAG,EAAE,OAAO,CAAA;CAAE,CAAC;AAEhD,+BAA+B;AAC/B,wBAAgB,eAAe,CAC7B,MAAM,EAAE;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,GAAG,EAAE,OAAO,CAAA;CAAE,GAAG;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,IAAI,EAAE,QAAQ,CAAA;CAAE,EAC9E,GAAG,EAAE,MAAM,MAAM,EACjB,eAAe,SAAqB,GACnC,OAAO,CAiCT;AAcD,oEAAoE;AACpE,qBAAa,4BAA6B,SAAQ,KAAK;aAEzB,QAAQ,EAAE,QAAQ;IAD9C,SAAkB,IAAI,kCAAkC;gBAC5B,QAAQ,EAAE,QAAQ;CAG/C;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAsB,cAAc,CAClC,KAAK,EAAE,MAAM,GAAG,GAAG,EACnB,IAAI,EAAE,WAAW,GAAG,SAAS,EAC7B,IAAI,GAAE,qBAA0B,GAC/B,OAAO,CAAC,QAAQ,CAAC,CA+GnB"}
|
|
@@ -41,7 +41,7 @@ export function parseRetryAfter(value, now = Date.now) {
|
|
|
41
41
|
return delta >= 0 ? delta : 0;
|
|
42
42
|
}
|
|
43
43
|
/** Exported for unit tests. */
|
|
44
|
-
export function classifyOutcome(result, now) {
|
|
44
|
+
export function classifyOutcome(result, now, retryAfterCapMs = RETRY_AFTER_CAP_MS) {
|
|
45
45
|
if (result.kind === 'error') {
|
|
46
46
|
// Both timer-fired aborts (`AbortSignal.timeout()` → `TimeoutError`)
|
|
47
47
|
// and caller-driven aborts (`AbortController.abort()` → `AbortError`)
|
|
@@ -67,7 +67,7 @@ export function classifyOutcome(result, now) {
|
|
|
67
67
|
return {
|
|
68
68
|
kind: 'retryable-status',
|
|
69
69
|
resp,
|
|
70
|
-
afterMs: parsed !== null ? Math.min(parsed,
|
|
70
|
+
afterMs: parsed !== null ? Math.min(parsed, retryAfterCapMs) : undefined,
|
|
71
71
|
};
|
|
72
72
|
}
|
|
73
73
|
if (resp.status >= 500)
|
|
@@ -125,6 +125,7 @@ export async function resilientFetch(input, init, opts = {}) {
|
|
|
125
125
|
maxAttempts: opts.retry?.maxAttempts ?? DEFAULT_RETRY.maxAttempts,
|
|
126
126
|
baseDelayMs: opts.retry?.baseDelayMs ?? DEFAULT_RETRY.baseDelayMs,
|
|
127
127
|
capDelayMs: opts.retry?.capDelayMs ?? DEFAULT_RETRY.capDelayMs,
|
|
128
|
+
retryAfterCapMs: opts.retry?.retryAfterCapMs ?? RETRY_AFTER_CAP_MS,
|
|
128
129
|
};
|
|
129
130
|
const sleep = opts.retry?.sleep ?? defaultSleep;
|
|
130
131
|
const random = opts.retry?.random ?? Math.random;
|
|
@@ -149,7 +150,7 @@ export async function resilientFetch(input, init, opts = {}) {
|
|
|
149
150
|
catch (err) {
|
|
150
151
|
result = { kind: 'error', err };
|
|
151
152
|
}
|
|
152
|
-
const outcome = classifyOutcome(result, now);
|
|
153
|
+
const outcome = classifyOutcome(result, now, retryConfig.retryAfterCapMs);
|
|
153
154
|
switch (outcome.kind) {
|
|
154
155
|
case 'success':
|
|
155
156
|
breaker.recordSuccess();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"resilient-fetch.js","sourceRoot":"","sources":["../../src/integrations/resilient-fetch.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAEL,gBAAgB,EAChB,UAAU,GAEX,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAqB,MAAM,YAAY,CAAC;AAEjE,OAAO,EAAE,gBAAgB,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"resilient-fetch.js","sourceRoot":"","sources":["../../src/integrations/resilient-fetch.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAEL,gBAAgB,EAChB,UAAU,GAEX,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAqB,MAAM,YAAY,CAAC;AAEjE,OAAO,EAAE,gBAAgB,EAAE,CAAC;AA0B5B,+EAA+E;AAC/E,MAAM,CAAC,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAEzC,MAAM,aAAa,GAAG;IACpB,WAAW,EAAE,CAAC;IACd,WAAW,EAAE,GAAG;IAChB,UAAU,EAAE,KAAK;CAClB,CAAC;AAEF;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,KAAoB,EAAE,MAAoB,IAAI,CAAC,GAAG;IAChF,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IACxB,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;IAC7B,IAAI,OAAO,KAAK,EAAE;QAAE,OAAO,IAAI,CAAC;IAEhC,IAAI,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC7B,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QACtC,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,OAAO,GAAG,CAAC;YAAE,OAAO,IAAI,CAAC;QACtD,OAAO,OAAO,GAAG,IAAI,CAAC;IACxB,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACnC,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC;QAAE,OAAO,IAAI,CAAC;IACtC,MAAM,KAAK,GAAG,MAAM,GAAG,GAAG,EAAE,CAAC;IAC7B,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAUD,+BAA+B;AAC/B,MAAM,UAAU,eAAe,CAC7B,MAA8E,EAC9E,GAAiB,EACjB,eAAe,GAAG,kBAAkB;IAEpC,IAAI,MAAM,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;QAC5B,qEAAqE;QACrE,sEAAsE;QACtE,iEAAiE;QACjE,+DAA+D;QAC/D,yDAAyD;QACzD,IACE,MAAM,CAAC,GAAG,YAAY,YAAY;YAClC,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,KAAK,cAAc,IAAI,MAAM,CAAC,GAAG,CAAC,IAAI,KAAK,YAAY,CAAC,EACxE,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,kBAAkB,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC;QACvD,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,mBAAmB,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC;IACxD,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;IACzB,IAAI,IAAI,CAAC,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG;QAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;IAC9E,IAAI,IAAI,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;QACxB,mEAAmE;QACnE,kEAAkE;QAClE,mEAAmE;QACnE,kDAAkD;QAClD,MAAM,gBAAgB,GACpB,OAAO,IAAI,CAAC,OAAO,EAAE,GAAG,KAAK,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACnF,MAAM,MAAM,GAAG,eAAe,CAAC,gBAAgB,EAAE,GAAG,CAAC,CAAC;QACtD,OAAO;YACL,IAAI,EAAE,kBAAkB;YACxB,IAAI;YACJ,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC,CAAC,CAAC,SAAS;SACzE,CAAC;IACJ,CAAC;IACD,IAAI,IAAI,CAAC,MAAM,IAAI,GAAG;QAAE,OAAO,EAAE,IAAI,EAAE,kBAAkB,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;IACtF,OAAO,EAAE,IAAI,EAAE,iBAAiB,EAAE,IAAI,EAAE,CAAC;AAC3C,CAAC;AAED,MAAM,YAAY,GAAG,CAAC,EAAU,EAAiB,EAAE,CACjD,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAEpD,SAAS,iBAAiB,CAAC,KAAmB;IAC5C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QAC/D,OAAO,GAAG,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,QAAQ,EAAE,CAAC;IACtC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;IACvB,CAAC;AACH,CAAC;AAED,oEAAoE;AACpE,MAAM,OAAO,4BAA6B,SAAQ,KAAK;IAEzB;IADV,IAAI,GAAG,8BAA8B,CAAC;IACxD,YAA4B,QAAkB;QAC5C,KAAK,CAAC,sCAAsC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;QADtC,aAAQ,GAAR,QAAQ,CAAU;IAE9C,CAAC;CACF;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,KAAmB,EACnB,IAA6B,EAC7B,OAA8B,EAAE;IAEhC,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,UAAU,CAAC,KAAK,CAAC;IACrD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IAC3C,MAAM,OAAO,GACX,IAAI,CAAC,OAAO,IAAI,UAAU,CAAC,IAAI,CAAC,UAAU,IAAI,iBAAiB,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC;IAE/F,MAAM,WAAW,GAAG;QAClB,WAAW,EAAE,IAAI,CAAC,KAAK,EAAE,WAAW,IAAI,aAAa,CAAC,WAAW;QACjE,WAAW,EAAE,IAAI,CAAC,KAAK,EAAE,WAAW,IAAI,aAAa,CAAC,WAAW;QACjE,UAAU,EAAE,IAAI,CAAC,KAAK,EAAE,UAAU,IAAI,aAAa,CAAC,UAAU;QAC9D,eAAe,EAAE,IAAI,CAAC,KAAK,EAAE,eAAe,IAAI,kBAAkB;KACnE,CAAC;IACF,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,EAAE,KAAK,IAAI,YAAY,CAAC;IAChD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,EAAE,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC;IAEjD,uDAAuD;IACvD,OAAO,CAAC,KAAK,EAAE,CAAC;IAEhB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,WAAW,CAAC,WAAW,EAAE,OAAO,EAAE,EAAE,CAAC;QACnE,IAAI,MAA8E,CAAC;QACnF,IAAI,CAAC;YACH,kEAAkE;YAClE,+DAA+D;YAC/D,oEAAoE;YACpE,6DAA6D;YAC7D,gEAAgE;YAChE,4DAA4D;YAC5D,+DAA+D;YAC/D,2DAA2D;YAC3D,uCAAuC;YACvC,yCAAyC;YACzC,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;YAC1C,MAAM,GAAG,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;QACtC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC;QAClC,CAAC;QAED,MAAM,OAAO,GAAG,eAAe,CAAC,MAAM,EAAE,GAAG,EAAE,WAAW,CAAC,eAAe,CAAC,CAAC;QAE1E,QAAQ,OAAO,CAAC,IAAI,EAAE,CAAC;YACrB,KAAK,SAAS;gBACZ,OAAO,CAAC,aAAa,EAAE,CAAC;gBACxB,OAAO,OAAO,CAAC,IAAI,CAAC;YAEtB,KAAK,iBAAiB;gBACpB,8DAA8D;gBAC9D,4DAA4D;gBAC5D,2DAA2D;gBAC3D,yDAAyD;gBACzD,6DAA6D;gBAC7D,OAAO,CAAC,aAAa,EAAE,CAAC;gBACxB,OAAO,OAAO,CAAC,IAAI,CAAC;YAEtB,KAAK,kBAAkB;gBACrB,8DAA8D;gBAC9D,+DAA+D;gBAC/D,yDAAyD;gBACzD,4DAA4D;gBAC5D,wDAAwD;gBACxD,OAAO,CAAC,aAAa,EAAE,CAAC;gBACxB,MAAM,OAAO,CAAC,GAAG,CAAC;YAEpB,KAAK,kBAAkB;gBACrB,IAAI,OAAO,GAAG,CAAC,IAAI,WAAW,CAAC,WAAW,EAAE,CAAC;oBAC3C,OAAO,CAAC,aAAa,EAAE,CAAC;oBACxB,MAAM,IAAI,4BAA4B,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;gBACvD,CAAC;gBACD,MAAM,KAAK,CACT,gBAAgB,CACd,OAAO,EACP,WAAW,CAAC,WAAW,EACvB,WAAW,CAAC,UAAU,EACtB,OAAO,CAAC,OAAO,EACf,MAAM,CACP,CACF,CAAC;gBACF,MAAM;YAER,KAAK,mBAAmB;gBACtB,IAAI,OAAO,GAAG,CAAC,IAAI,WAAW,CAAC,WAAW,EAAE,CAAC;oBAC3C,OAAO,CAAC,aAAa,EAAE,CAAC;oBACxB,MAAM,OAAO,CAAC,GAAG,CAAC;gBACpB,CAAC;gBACD,MAAM,KAAK,CACT,gBAAgB,CACd,OAAO,EACP,WAAW,CAAC,WAAW,EACvB,WAAW,CAAC,UAAU,EACtB,SAAS,EACT,MAAM,CACP,CACF,CAAC;gBACF,MAAM;YAER,OAAO,CAAC,CAAC,CAAC;gBACR,8DAA8D;gBAC9D,6DAA6D;gBAC7D,yDAAyD;gBACzD,iEAAiE;gBACjE,MAAM,WAAW,GAAU,OAAO,CAAC;gBACnC,MAAM,IAAI,KAAK,CAAC,qCAAqC,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;YACtF,CAAC;QACH,CAAC;IACH,CAAC;IAED,mEAAmE;IACnE,sEAAsE;IACtE,sEAAsE;IACtE,8EAA8E;IAC9E,sBAAsB;IACtB,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;AACxE,CAAC"}
|
package/dist/cli/tool.js
CHANGED
|
@@ -54,6 +54,17 @@ function output(data) {
|
|
|
54
54
|
// Fallback: stderr (previous behavior, works on all platforms)
|
|
55
55
|
process.stderr.write(text + '\n');
|
|
56
56
|
}
|
|
57
|
+
// Backend failures come back as `{ error }` payloads rather than throws
|
|
58
|
+
// (#2469). Every tool command routes its result through here, so this is
|
|
59
|
+
// the one place that keeps scripted callers honest: print the payload,
|
|
60
|
+
// then exit non-zero.
|
|
61
|
+
if (data &&
|
|
62
|
+
typeof data === 'object' &&
|
|
63
|
+
'error' in data &&
|
|
64
|
+
typeof data.error === 'string' &&
|
|
65
|
+
data.error.trim().length > 0) {
|
|
66
|
+
process.exitCode = 1;
|
|
67
|
+
}
|
|
57
68
|
}
|
|
58
69
|
/**
|
|
59
70
|
* Parse a `--limit` CLI option into a positive row cap, or `undefined` when the
|
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
*/
|
|
9
9
|
import type { FeatureExtractionPipeline } from '@huggingface/transformers';
|
|
10
10
|
import { type EmbeddingConfig, type ModelProgress } from './types.js';
|
|
11
|
+
import { type EmbeddingRequestOptions } from './http-client.js';
|
|
11
12
|
/**
|
|
12
13
|
* Progress callback type for model loading
|
|
13
14
|
*/
|
|
@@ -45,7 +46,7 @@ export declare const getEmbedder: () => FeatureExtractionPipeline;
|
|
|
45
46
|
* @param text - Text to embed
|
|
46
47
|
* @returns Float32Array of embedding vector
|
|
47
48
|
*/
|
|
48
|
-
export declare const embedText: (text: string) => Promise<Float32Array>;
|
|
49
|
+
export declare const embedText: (text: string, options?: EmbeddingRequestOptions) => Promise<Float32Array>;
|
|
49
50
|
/**
|
|
50
51
|
* Embed multiple texts in a single batch
|
|
51
52
|
* More efficient than calling embedText multiple times
|
|
@@ -53,7 +54,7 @@ export declare const embedText: (text: string) => Promise<Float32Array>;
|
|
|
53
54
|
* @param texts - Array of texts to embed
|
|
54
55
|
* @returns Array of Float32Array embedding vectors
|
|
55
56
|
*/
|
|
56
|
-
export declare const embedBatch: (texts: string[]) => Promise<Float32Array[]>;
|
|
57
|
+
export declare const embedBatch: (texts: string[], options?: EmbeddingRequestOptions) => Promise<Float32Array[]>;
|
|
57
58
|
/**
|
|
58
59
|
* Convert Float32Array to regular number array (for LadybugDB storage)
|
|
59
60
|
*/
|
|
@@ -13,7 +13,7 @@ if (!process.env.ORT_LOG_LEVEL) {
|
|
|
13
13
|
process.env.ORT_LOG_LEVEL = '3';
|
|
14
14
|
}
|
|
15
15
|
import { DEFAULT_EMBEDDING_CONFIG } from './types.js';
|
|
16
|
-
import { isHttpMode, getHttpDimensions, httpEmbed } from './http-client.js';
|
|
16
|
+
import { isHttpMode, getHttpDimensions, httpEmbed, } from './http-client.js';
|
|
17
17
|
import { resolveEmbeddingConfig } from './config.js';
|
|
18
18
|
import { applyHfEnvOverrides, isHfDownloadFailure, withHfDownloadRetry } from './hf-env.js';
|
|
19
19
|
import { getLocalEmbeddingRuntimeBlocker, getMissingLocalEmbeddingStackMessage, } from './runtime-support.js';
|
|
@@ -243,9 +243,10 @@ export const getEmbedder = () => {
|
|
|
243
243
|
* @param text - Text to embed
|
|
244
244
|
* @returns Float32Array of embedding vector
|
|
245
245
|
*/
|
|
246
|
-
export const embedText = async (text) => {
|
|
246
|
+
export const embedText = async (text, options = {}) => {
|
|
247
|
+
options.signal?.throwIfAborted();
|
|
247
248
|
if (isHttpMode()) {
|
|
248
|
-
const [vec] = await httpEmbed([text]);
|
|
249
|
+
const [vec] = await httpEmbed([text], options);
|
|
249
250
|
return vec;
|
|
250
251
|
}
|
|
251
252
|
const embedder = getEmbedder();
|
|
@@ -263,12 +264,13 @@ export const embedText = async (text) => {
|
|
|
263
264
|
* @param texts - Array of texts to embed
|
|
264
265
|
* @returns Array of Float32Array embedding vectors
|
|
265
266
|
*/
|
|
266
|
-
export const embedBatch = async (texts) => {
|
|
267
|
+
export const embedBatch = async (texts, options = {}) => {
|
|
268
|
+
options.signal?.throwIfAborted();
|
|
267
269
|
if (texts.length === 0) {
|
|
268
270
|
return [];
|
|
269
271
|
}
|
|
270
272
|
if (isHttpMode()) {
|
|
271
|
-
return httpEmbed(texts);
|
|
273
|
+
return httpEmbed(texts, options);
|
|
272
274
|
}
|
|
273
275
|
const embedder = getEmbedder();
|
|
274
276
|
// Process batch
|
|
@@ -276,6 +278,7 @@ export const embedBatch = async (texts) => {
|
|
|
276
278
|
pooling: 'mean',
|
|
277
279
|
normalize: true,
|
|
278
280
|
});
|
|
281
|
+
options.signal?.throwIfAborted();
|
|
279
282
|
// Result shape is [batch_size, dimensions]
|
|
280
283
|
// Need to split into individual vectors
|
|
281
284
|
const data = result.data;
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export interface EmbeddingIdentity {
|
|
2
|
+
model: string;
|
|
3
|
+
dimensions: number;
|
|
4
|
+
provider: string;
|
|
5
|
+
}
|
|
6
|
+
/**
|
|
7
|
+
* Identify the vector space strongly enough to resume without mixing providers.
|
|
8
|
+
* The HTTP fingerprint excludes URL credentials and query parameters before
|
|
9
|
+
* hashing, so metadata contains neither an endpoint nor a secret-derived hash.
|
|
10
|
+
*/
|
|
11
|
+
export declare function resolveEmbeddingIdentity(): EmbeddingIdentity;
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import { getEmbeddingDimensions } from './embedder.js';
|
|
3
|
+
import { resolveEmbeddingConfig } from './config.js';
|
|
4
|
+
import { isHttpMode, safeUrl } from './http-client.js';
|
|
5
|
+
/**
|
|
6
|
+
* Identify the vector space strongly enough to resume without mixing providers.
|
|
7
|
+
* The HTTP fingerprint excludes URL credentials and query parameters before
|
|
8
|
+
* hashing, so metadata contains neither an endpoint nor a secret-derived hash.
|
|
9
|
+
*/
|
|
10
|
+
export function resolveEmbeddingIdentity() {
|
|
11
|
+
const httpMode = isHttpMode();
|
|
12
|
+
const provider = httpMode
|
|
13
|
+
? `http:${createHash('sha256')
|
|
14
|
+
.update(safeUrl(process.env.GITNEXUS_EMBEDDING_URL ?? ''))
|
|
15
|
+
.digest('hex')}`
|
|
16
|
+
: 'local';
|
|
17
|
+
return {
|
|
18
|
+
model: httpMode
|
|
19
|
+
? process.env.GITNEXUS_EMBEDDING_MODEL
|
|
20
|
+
: resolveEmbeddingConfig().modelId,
|
|
21
|
+
dimensions: getEmbeddingDimensions(),
|
|
22
|
+
provider,
|
|
23
|
+
};
|
|
24
|
+
}
|
|
@@ -77,6 +77,21 @@ export interface EmbeddingPipelineResult {
|
|
|
77
77
|
vectorIndexReady: boolean;
|
|
78
78
|
semanticMode: 'vector-index' | 'exact-scan';
|
|
79
79
|
}
|
|
80
|
+
export interface EmbeddingPipelineCheckpoint {
|
|
81
|
+
nodesProcessed: number;
|
|
82
|
+
totalNodes: number;
|
|
83
|
+
chunksProcessed: number;
|
|
84
|
+
}
|
|
85
|
+
export interface EmbeddingPipelineCheckpointWindow extends EmbeddingPipelineCheckpoint {
|
|
86
|
+
nodeIds: string[];
|
|
87
|
+
}
|
|
88
|
+
export interface EmbeddingPipelineOptions {
|
|
89
|
+
signal?: AbortSignal;
|
|
90
|
+
checkpointEveryNodes?: number;
|
|
91
|
+
forceReembedNodeIds?: ReadonlySet<string>;
|
|
92
|
+
onCheckpointWindowStart?: (window: EmbeddingPipelineCheckpointWindow) => Promise<void>;
|
|
93
|
+
onCheckpoint?: (checkpoint: EmbeddingPipelineCheckpoint) => Promise<void>;
|
|
94
|
+
}
|
|
80
95
|
/**
|
|
81
96
|
* Run the embedding pipeline
|
|
82
97
|
*
|
|
@@ -89,7 +104,7 @@ export interface EmbeddingPipelineResult {
|
|
|
89
104
|
* Nodes whose hash matches are skipped; nodes with a changed hash are DELETE'd
|
|
90
105
|
* and re-embedded; nodes not in the map are embedded fresh.
|
|
91
106
|
*/
|
|
92
|
-
export declare const runEmbeddingPipeline: (executeQuery: (cypher: string) => Promise<any[]>, executeWithReusedStatement: (cypher: string, paramsList: Array<Record<string, any>>) => Promise<void>, onProgress: EmbeddingProgressCallback, config?: Partial<EmbeddingConfig>, skipNodeIds?: Set<string>, existingEmbeddings?: Map<string, string
|
|
107
|
+
export declare const runEmbeddingPipeline: (executeQuery: (cypher: string) => Promise<any[]>, executeWithReusedStatement: (cypher: string, paramsList: Array<Record<string, any>>) => Promise<void>, onProgress: EmbeddingProgressCallback, config?: Partial<EmbeddingConfig>, skipNodeIds?: Set<string>, existingEmbeddings?: Map<string, string>, pipelineOptions?: EmbeddingPipelineOptions) => Promise<EmbeddingPipelineResult>;
|
|
93
108
|
/**
|
|
94
109
|
* Perform semantic search using the vector index with chunk deduplication
|
|
95
110
|
*/
|
|
@@ -135,7 +135,44 @@ const queryEmbeddableNodes = async (executeQuery) => {
|
|
|
135
135
|
}
|
|
136
136
|
}
|
|
137
137
|
}
|
|
138
|
-
return allNodes;
|
|
138
|
+
return allNodes.length > 0 ? allNodes : queryFallbackFileNodes(executeQuery);
|
|
139
|
+
};
|
|
140
|
+
/**
|
|
141
|
+
* Static and documentation repositories may contain no code symbols while
|
|
142
|
+
* still persisting useful text on File nodes. Keep File embeddings as a
|
|
143
|
+
* zero-symbol fallback so code repositories retain symbol-first selection.
|
|
144
|
+
*/
|
|
145
|
+
const queryFallbackFileNodes = async (executeQuery) => {
|
|
146
|
+
try {
|
|
147
|
+
const rows = await executeQuery(`
|
|
148
|
+
MATCH (n:File)
|
|
149
|
+
RETURN n.id AS id, n.name AS name, 'File' AS label,
|
|
150
|
+
n.filePath AS filePath, n.content AS content
|
|
151
|
+
`);
|
|
152
|
+
return rows
|
|
153
|
+
.map((row) => {
|
|
154
|
+
const content = row.content ?? row[4] ?? '';
|
|
155
|
+
return {
|
|
156
|
+
id: row.id ?? row[0],
|
|
157
|
+
name: row.name ?? row[1],
|
|
158
|
+
label: row.label ?? row[2] ?? 'File',
|
|
159
|
+
filePath: row.filePath ?? row[3],
|
|
160
|
+
content,
|
|
161
|
+
startLine: 1,
|
|
162
|
+
endLine: Math.max(1, content.split('\n').length),
|
|
163
|
+
};
|
|
164
|
+
})
|
|
165
|
+
.filter((node) => node.id &&
|
|
166
|
+
node.filePath &&
|
|
167
|
+
node.content.trim() &&
|
|
168
|
+
node.content !== '[Binary file - content not stored]');
|
|
169
|
+
}
|
|
170
|
+
catch (error) {
|
|
171
|
+
if (isDev) {
|
|
172
|
+
logger.warn({ error }, 'Fallback File-node embedding query failed:');
|
|
173
|
+
}
|
|
174
|
+
return [];
|
|
175
|
+
}
|
|
139
176
|
};
|
|
140
177
|
/**
|
|
141
178
|
* Batch INSERT chunk-aware embeddings into CodeEmbedding table
|
|
@@ -231,11 +268,18 @@ const deleteStaleEmbeddingRows = async (executeWithReusedStatement, nodeIds) =>
|
|
|
231
268
|
* Nodes whose hash matches are skipped; nodes with a changed hash are DELETE'd
|
|
232
269
|
* and re-embedded; nodes not in the map are embedded fresh.
|
|
233
270
|
*/
|
|
234
|
-
export const runEmbeddingPipeline = async (executeQuery, executeWithReusedStatement, onProgress, config = {}, skipNodeIds, existingEmbeddings) => {
|
|
271
|
+
export const runEmbeddingPipeline = async (executeQuery, executeWithReusedStatement, onProgress, config = {}, skipNodeIds, existingEmbeddings, pipelineOptions = {}) => {
|
|
235
272
|
const finalConfig = resolveEmbeddingConfig(config);
|
|
236
273
|
let totalChunks = 0;
|
|
274
|
+
const checkpointEveryNodes = pipelineOptions.checkpointEveryNodes ?? 5_000;
|
|
275
|
+
if (!Number.isSafeInteger(checkpointEveryNodes) || checkpointEveryNodes <= 0) {
|
|
276
|
+
throw new Error('checkpointEveryNodes must be a positive integer');
|
|
277
|
+
}
|
|
278
|
+
const throwIfCancelled = () => pipelineOptions.signal?.throwIfAborted();
|
|
237
279
|
try {
|
|
280
|
+
throwIfCancelled();
|
|
238
281
|
const vectorAvailable = await ensureVectorExtensionAvailable();
|
|
282
|
+
throwIfCancelled();
|
|
239
283
|
if (!vectorAvailable) {
|
|
240
284
|
logger.warn(vectorUnavailableMessage);
|
|
241
285
|
}
|
|
@@ -254,6 +298,7 @@ export const runEmbeddingPipeline = async (executeQuery, executeWithReusedStatem
|
|
|
254
298
|
modelDownloadPercent: downloadPercent,
|
|
255
299
|
});
|
|
256
300
|
}, finalConfig);
|
|
301
|
+
throwIfCancelled();
|
|
257
302
|
}
|
|
258
303
|
onProgress({
|
|
259
304
|
phase: 'loading-model',
|
|
@@ -265,6 +310,8 @@ export const runEmbeddingPipeline = async (executeQuery, executeWithReusedStatem
|
|
|
265
310
|
}
|
|
266
311
|
// Phase 2: Query embeddable nodes
|
|
267
312
|
let nodes = await queryEmbeddableNodes(executeQuery);
|
|
313
|
+
throwIfCancelled();
|
|
314
|
+
const embeddableNodeIds = new Set(nodes.map((node) => node.id));
|
|
268
315
|
// Incremental mode: compare content hashes, delete stale rows, skip fresh ones.
|
|
269
316
|
// Computed hashes for stale nodes are cached so batchInsertEmbeddings can reuse them
|
|
270
317
|
// (avoids double computation).
|
|
@@ -273,16 +320,18 @@ export const runEmbeddingPipeline = async (executeQuery, executeWithReusedStatem
|
|
|
273
320
|
// than all up front — see U6 / KTD7. `staleNodeIds` is consulted inside the
|
|
274
321
|
// batch loop; it stays empty in full (non-incremental) mode so no deletes fire.
|
|
275
322
|
const staleNodeIds = new Set();
|
|
276
|
-
|
|
323
|
+
const forceReembedNodeIds = pipelineOptions.forceReembedNodeIds;
|
|
324
|
+
if ((existingEmbeddings && existingEmbeddings.size > 0) ||
|
|
325
|
+
(forceReembedNodeIds && forceReembedNodeIds.size > 0)) {
|
|
277
326
|
const beforeCount = nodes.length;
|
|
278
327
|
nodes = nodes.filter((n) => {
|
|
279
|
-
const existingHash = existingEmbeddings
|
|
328
|
+
const existingHash = existingEmbeddings?.get(n.id);
|
|
280
329
|
if (existingHash === undefined) {
|
|
281
330
|
// New node — needs embedding
|
|
282
331
|
return true;
|
|
283
332
|
}
|
|
284
333
|
const currentHash = contentHashForNode(n, finalConfig);
|
|
285
|
-
if (currentHash !== existingHash) {
|
|
334
|
+
if (currentHash !== existingHash || forceReembedNodeIds?.has(n.id)) {
|
|
286
335
|
// Content changed — cache hash for reuse during insert, mark for DELETE + re-embed
|
|
287
336
|
computedStaleHashes.set(n.id, currentHash);
|
|
288
337
|
staleNodeIds.add(n.id);
|
|
@@ -295,11 +344,17 @@ export const runEmbeddingPipeline = async (executeQuery, executeWithReusedStatem
|
|
|
295
344
|
logger.info(`📦 Incremental embeddings: ${beforeCount} total, ${existingEmbeddings.size} cached, ${staleNodeIds.size} stale, ${nodes.length} to embed`);
|
|
296
345
|
}
|
|
297
346
|
}
|
|
347
|
+
if (forceReembedNodeIds && forceReembedNodeIds.size > 0) {
|
|
348
|
+
const removedPendingNodeIds = [...forceReembedNodeIds].filter((nodeId) => !embeddableNodeIds.has(nodeId));
|
|
349
|
+
await deleteStaleEmbeddingRows(executeWithReusedStatement, removedPendingNodeIds);
|
|
350
|
+
throwIfCancelled();
|
|
351
|
+
}
|
|
298
352
|
const totalNodes = nodes.length;
|
|
299
353
|
if (isDev) {
|
|
300
354
|
logger.info(`📊 Found ${totalNodes} embeddable nodes`);
|
|
301
355
|
}
|
|
302
356
|
if (totalNodes === 0) {
|
|
357
|
+
throwIfCancelled();
|
|
303
358
|
// Ensure the vector index exists even when no new nodes need embedding.
|
|
304
359
|
// A prior crash or first-time incremental run may have left CodeEmbedding
|
|
305
360
|
// rows without ever reaching index creation.
|
|
@@ -321,6 +376,7 @@ export const runEmbeddingPipeline = async (executeQuery, executeWithReusedStatem
|
|
|
321
376
|
const batchSize = finalConfig.batchSize;
|
|
322
377
|
const chunkSize = finalConfig.chunkSize;
|
|
323
378
|
const overlap = finalConfig.overlap;
|
|
379
|
+
const checkpointWindowNodeCount = Math.max(batchSize, Math.ceil(checkpointEveryNodes / batchSize) * batchSize);
|
|
324
380
|
let processedNodes = 0;
|
|
325
381
|
onProgress({
|
|
326
382
|
phase: 'embedding',
|
|
@@ -332,6 +388,18 @@ export const runEmbeddingPipeline = async (executeQuery, executeWithReusedStatem
|
|
|
332
388
|
});
|
|
333
389
|
// Process in batches of nodes
|
|
334
390
|
for (let batchIndex = 0; batchIndex < totalNodes; batchIndex += batchSize) {
|
|
391
|
+
throwIfCancelled();
|
|
392
|
+
if (pipelineOptions.onCheckpointWindowStart && batchIndex % checkpointWindowNodeCount === 0) {
|
|
393
|
+
await pipelineOptions.onCheckpointWindowStart({
|
|
394
|
+
nodesProcessed: processedNodes,
|
|
395
|
+
totalNodes,
|
|
396
|
+
chunksProcessed: totalChunks,
|
|
397
|
+
nodeIds: nodes
|
|
398
|
+
.slice(batchIndex, batchIndex + checkpointWindowNodeCount)
|
|
399
|
+
.map((node) => node.id),
|
|
400
|
+
});
|
|
401
|
+
throwIfCancelled();
|
|
402
|
+
}
|
|
335
403
|
const batch = nodes.slice(batchIndex, batchIndex + batchSize);
|
|
336
404
|
// Chunk each node and generate text
|
|
337
405
|
const allTexts = [];
|
|
@@ -387,6 +455,7 @@ export const runEmbeddingPipeline = async (executeQuery, executeWithReusedStatem
|
|
|
387
455
|
// Preserves Kuzu's required DELETE-before-INSERT for vector-indexed rows.
|
|
388
456
|
const batchStaleIds = batch.filter((n) => staleNodeIds.has(n.id)).map((n) => n.id);
|
|
389
457
|
await deleteStaleEmbeddingRows(executeWithReusedStatement, batchStaleIds);
|
|
458
|
+
throwIfCancelled();
|
|
390
459
|
// Embed chunk texts in sub-batches to control memory
|
|
391
460
|
const EMBED_SUB_BATCH = finalConfig.subBatchSize;
|
|
392
461
|
for (let si = 0; si < allTexts.length; si += EMBED_SUB_BATCH) {
|
|
@@ -394,7 +463,7 @@ export const runEmbeddingPipeline = async (executeQuery, executeWithReusedStatem
|
|
|
394
463
|
const subUpdates = allUpdates.slice(si, si + EMBED_SUB_BATCH);
|
|
395
464
|
let embeddings;
|
|
396
465
|
try {
|
|
397
|
-
embeddings = await embedBatch(subTexts);
|
|
466
|
+
embeddings = await embedBatch(subTexts, { signal: pipelineOptions.signal });
|
|
398
467
|
}
|
|
399
468
|
catch (embedErr) {
|
|
400
469
|
logger.error({ embedErr }, `❌ embedBatch failed for ${subTexts.length} texts (first: "${subTexts[0]?.substring(0, 80)}..."):`);
|
|
@@ -405,6 +474,7 @@ export const runEmbeddingPipeline = async (executeQuery, executeWithReusedStatem
|
|
|
405
474
|
embedding: embeddingToArray(embeddings[i]),
|
|
406
475
|
}));
|
|
407
476
|
await batchInsertEmbeddings(executeWithReusedStatement, dbUpdates);
|
|
477
|
+
throwIfCancelled();
|
|
408
478
|
}
|
|
409
479
|
processedNodes += batch.length;
|
|
410
480
|
totalChunks += allUpdates.length;
|
|
@@ -417,8 +487,18 @@ export const runEmbeddingPipeline = async (executeQuery, executeWithReusedStatem
|
|
|
417
487
|
currentBatch: Math.floor(batchIndex / batchSize) + 1,
|
|
418
488
|
totalBatches: Math.ceil(totalNodes / batchSize),
|
|
419
489
|
});
|
|
490
|
+
if (pipelineOptions.onCheckpoint &&
|
|
491
|
+
(processedNodes % checkpointWindowNodeCount === 0 || processedNodes === totalNodes)) {
|
|
492
|
+
await pipelineOptions.onCheckpoint({
|
|
493
|
+
nodesProcessed: processedNodes,
|
|
494
|
+
totalNodes,
|
|
495
|
+
chunksProcessed: totalChunks,
|
|
496
|
+
});
|
|
497
|
+
throwIfCancelled();
|
|
498
|
+
}
|
|
420
499
|
}
|
|
421
500
|
// Phase 4: Create vector index
|
|
501
|
+
throwIfCancelled();
|
|
422
502
|
onProgress({
|
|
423
503
|
phase: 'indexing',
|
|
424
504
|
percent: 90,
|
|
@@ -10,6 +10,9 @@
|
|
|
10
10
|
* fails fast on a flapping endpoint. Per-attempt timeout is enforced
|
|
11
11
|
* via `AbortSignal.timeout` on the underlying fetch.
|
|
12
12
|
*/
|
|
13
|
+
export interface EmbeddingRequestOptions {
|
|
14
|
+
signal?: AbortSignal;
|
|
15
|
+
}
|
|
13
16
|
/**
|
|
14
17
|
* Whether HTTP embedding mode is active — i.e. both `GITNEXUS_EMBEDDING_URL` and
|
|
15
18
|
* `GITNEXUS_EMBEDDING_MODEL` are set. A pure presence probe: it deliberately does
|
|
@@ -60,7 +63,7 @@ export declare class HttpEmbeddingError extends Error {
|
|
|
60
63
|
* @param texts - Array of texts to embed
|
|
61
64
|
* @returns Array of Float32Array embedding vectors
|
|
62
65
|
*/
|
|
63
|
-
export declare const httpEmbed: (texts: string[]) => Promise<Float32Array[]>;
|
|
66
|
+
export declare const httpEmbed: (texts: string[], requestOptions?: EmbeddingRequestOptions) => Promise<Float32Array[]>;
|
|
64
67
|
/**
|
|
65
68
|
* Embed a single query text via the HTTP backend.
|
|
66
69
|
* Convenience for MCP search where only one vector is needed.
|
|
@@ -68,4 +71,4 @@ export declare const httpEmbed: (texts: string[]) => Promise<Float32Array[]>;
|
|
|
68
71
|
* @param text - Query text to embed
|
|
69
72
|
* @returns Embedding vector as number array
|
|
70
73
|
*/
|
|
71
|
-
export declare const httpEmbedQuery: (text: string) => Promise<number[]>;
|
|
74
|
+
export declare const httpEmbedQuery: (text: string, requestOptions?: EmbeddingRequestOptions) => Promise<number[]>;
|