opencode-rag-plugin 2.4.1 → 2.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -64,4 +64,4 @@ export declare function probeEmbeddingDimension(embedder: EmbeddingProvider): Pr
|
|
|
64
64
|
* @returns A promise resolving to a flat array of embedding vectors (one per input text);
|
|
65
65
|
* failed batches return empty arrays
|
|
66
66
|
*/
|
|
67
|
-
export declare function embedBatch(embedder: EmbeddingProvider, texts: string[], batchSize?: number, purpose?: "query" | "document", concurrency?: number, onProgress?: (completed: number, total: number) => void, retryMax?: number, retryBaseDelayMs?: number): Promise<number[][]>;
|
|
67
|
+
export declare function embedBatch(embedder: EmbeddingProvider, texts: string[], batchSize?: number, purpose?: "query" | "document", concurrency?: number, onProgress?: (completed: number, total: number) => void, retryMax?: number, retryBaseDelayMs?: number, maxConsecutiveFailures?: number): Promise<number[][]>;
|
package/dist/embedder/factory.js
CHANGED
|
@@ -94,6 +94,29 @@ const TIMEOUT_RE = /timed out|timeout/i;
|
|
|
94
94
|
function isTimeoutError(err) {
|
|
95
95
|
return TIMEOUT_RE.test(err instanceof Error ? err.message : String(err));
|
|
96
96
|
}
|
|
97
|
+
/**
|
|
98
|
+
* HTTP 502/503/504 — the request never got a usable answer from the model
|
|
99
|
+
* backend. A reverse proxy or model router (llama-swap, nginx, …) returns these
|
|
100
|
+
* when the backend is down, over capacity, or crashing on the input — most
|
|
101
|
+
* commonly an embedding server whose context window is smaller than the chunk
|
|
102
|
+
* we send (it accepts a short probe, so preflight passes, then 502s on every
|
|
103
|
+
* real chunk). Unlike a per-text context rejection, shrinking the batch cannot
|
|
104
|
+
* help: the server itself is the problem. So gateway errors are retried at the
|
|
105
|
+
* top level (a brief blip recovers) but never split — splitting only multiplies
|
|
106
|
+
* load on an unhealthy backend — and repeated ones trip the circuit breaker.
|
|
107
|
+
*/
|
|
108
|
+
const GATEWAY_STATUS_RE = /\(?(502|503|504)\)?/;
|
|
109
|
+
function isGatewayError(err) {
|
|
110
|
+
return GATEWAY_STATUS_RE.test(err instanceof Error ? err.message : String(err));
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* How many CONSECUTIVELY failed batches stop all further embedding requests in a
|
|
114
|
+
* single `embedBatch` call (the counter is reset by any batch that succeeds).
|
|
115
|
+
* Bounds the damage when the provider is down or undersized so a run fails fast
|
|
116
|
+
* and reports the provider as unavailable, instead of silently grinding through
|
|
117
|
+
* every batch for minutes — which reads as a hang. `0` disables the breaker.
|
|
118
|
+
*/
|
|
119
|
+
const DEFAULT_MAX_CONSECUTIVE_BATCH_FAILURES = 6;
|
|
97
120
|
/** Resolve the workspace debug log file used for batch failure diagnostics. */
|
|
98
121
|
function getBatchLogFilePath() {
|
|
99
122
|
return path.resolve(process.cwd(), ".opencode", "opencode-rag.log");
|
|
@@ -127,7 +150,7 @@ function getBatchLogFilePath() {
|
|
|
127
150
|
* @returns A promise resolving to a flat array of embedding vectors (one per input text);
|
|
128
151
|
* failed batches return empty arrays
|
|
129
152
|
*/
|
|
130
|
-
export async function embedBatch(embedder, texts, batchSize = 10, purpose, concurrency = 1, onProgress, retryMax = 3, retryBaseDelayMs = 1000) {
|
|
153
|
+
export async function embedBatch(embedder, texts, batchSize = 10, purpose, concurrency = 1, onProgress, retryMax = 3, retryBaseDelayMs = 1000, maxConsecutiveFailures = DEFAULT_MAX_CONSECUTIVE_BATCH_FAILURES) {
|
|
131
154
|
if (texts.length === 0)
|
|
132
155
|
return [];
|
|
133
156
|
const batches = [];
|
|
@@ -184,8 +207,16 @@ export async function embedBatch(embedder, texts, batchSize = 10, purpose, concu
|
|
|
184
207
|
// counted by the caller. A context-limit rejection carries a permanent 400
|
|
185
208
|
// status, so it needs its own predicate; timeouts are already non-permanent
|
|
186
209
|
// (belt and braces). Genuine permanent errors never split — halving the
|
|
187
|
-
// batch would fail the same way, only slower.
|
|
188
|
-
|
|
210
|
+
// batch would fail the same way, only slower. Neither do gateway errors
|
|
211
|
+
// (502/503/504): they come from the SERVER (down, overloaded, or a context
|
|
212
|
+
// smaller than our chunks), not a bad text, so a smaller batch fails
|
|
213
|
+
// identically. Top-level retry already handles a transient one; a persistent
|
|
214
|
+
// one is left to the circuit breaker below rather than split into an
|
|
215
|
+
// exponential retry tree that looks exactly like a hang.
|
|
216
|
+
const splittable = (!isPermanentError(lastError) ||
|
|
217
|
+
isContextLimitError(lastError) ||
|
|
218
|
+
isTimeoutError(lastError)) &&
|
|
219
|
+
!isGatewayError(lastError);
|
|
189
220
|
if (batchTexts.length > 1 && splittable) {
|
|
190
221
|
const mid = Math.ceil(batchTexts.length / 2);
|
|
191
222
|
appendDebugLog(getBatchLogFilePath(), {
|
|
@@ -203,10 +234,50 @@ export async function embedBatch(embedder, texts, batchSize = 10, purpose, concu
|
|
|
203
234
|
}
|
|
204
235
|
return null;
|
|
205
236
|
}
|
|
237
|
+
// ── Circuit breaker ────────────────────────────────────────────────────
|
|
238
|
+
// A down / overloaded / undersized embedding server fails EVERY batch. Without
|
|
239
|
+
// a break we keep firing batch after batch at it for the whole run — a silent,
|
|
240
|
+
// minutes-long grind indistinguishable from a hang. After this many
|
|
241
|
+
// CONSECUTIVELY failed batches (reset by any success) we stop calling the
|
|
242
|
+
// provider entirely and short-circuit the rest to empty vectors, so the pass
|
|
243
|
+
// fails fast and the pipeline reports the provider as unavailable.
|
|
244
|
+
let consecutiveFailedBatches = 0;
|
|
245
|
+
let circuitTripped = false;
|
|
246
|
+
function recordBatchResult(succeeded) {
|
|
247
|
+
if (succeeded) {
|
|
248
|
+
consecutiveFailedBatches = 0;
|
|
249
|
+
return;
|
|
250
|
+
}
|
|
251
|
+
consecutiveFailedBatches++;
|
|
252
|
+
if (!circuitTripped &&
|
|
253
|
+
maxConsecutiveFailures > 0 &&
|
|
254
|
+
consecutiveFailedBatches >= maxConsecutiveFailures) {
|
|
255
|
+
circuitTripped = true;
|
|
256
|
+
appendDebugLog(getBatchLogFilePath(), {
|
|
257
|
+
scope: "embedder.batch",
|
|
258
|
+
severity: "warn",
|
|
259
|
+
message: `Embedding circuit breaker tripped after ${consecutiveFailedBatches} consecutive failed ` +
|
|
260
|
+
"batches — the embedding server is unavailable, overloaded, or rejecting the input (an " +
|
|
261
|
+
"over-large / too-small context is the usual cause). Stopping further requests; un-embedded " +
|
|
262
|
+
"chunks are kept for the next pass. Fix the provider (e.g. raise its context window) and " +
|
|
263
|
+
"run the index again.",
|
|
264
|
+
});
|
|
265
|
+
}
|
|
266
|
+
}
|
|
206
267
|
if (concurrency <= 1 || batches.length <= 1) {
|
|
207
268
|
const results = [];
|
|
208
269
|
for (const batch of batches) {
|
|
270
|
+
if (circuitTripped) {
|
|
271
|
+
// Breaker open: emit empty vectors so caller index→vector alignment is
|
|
272
|
+
// preserved, advance progress, and never touch the provider again.
|
|
273
|
+
for (let i = 0; i < batch.texts.length; i++) {
|
|
274
|
+
results.push([]);
|
|
275
|
+
}
|
|
276
|
+
onProgress?.(results.length, texts.length);
|
|
277
|
+
continue;
|
|
278
|
+
}
|
|
209
279
|
const embeddings = await embedWithRetry(batch.texts);
|
|
280
|
+
recordBatchResult(embeddings !== null);
|
|
210
281
|
if (embeddings) {
|
|
211
282
|
results.push(...embeddings);
|
|
212
283
|
}
|
|
@@ -222,7 +293,13 @@ export async function embedBatch(embedder, texts, batchSize = 10, purpose, concu
|
|
|
222
293
|
let completedCount = 0;
|
|
223
294
|
const limit = pLimit(concurrency);
|
|
224
295
|
const batchResults = await Promise.all(batches.map((batch) => limit(async () => {
|
|
296
|
+
if (circuitTripped) {
|
|
297
|
+
completedCount += batch.texts.length;
|
|
298
|
+
onProgress?.(completedCount, texts.length);
|
|
299
|
+
return { index: batch.index, embeddings: batch.texts.map(() => []) };
|
|
300
|
+
}
|
|
225
301
|
const embeddings = await embedWithRetry(batch.texts);
|
|
302
|
+
recordBatchResult(embeddings !== null);
|
|
226
303
|
const flatResult = embeddings ?? batch.texts.map(() => []);
|
|
227
304
|
// Count processed TEXTS (not returned vectors) so failed batches
|
|
228
305
|
// still advance progress towards the total.
|