opencode-rag-plugin 1.19.4 → 1.19.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunker/base.js +19 -5
- package/dist/chunker/factory.js +27 -9
- package/dist/chunker/grammar.d.ts +18 -1
- package/dist/chunker/grammar.js +48 -10
- package/dist/chunker/image.js +5 -0
- package/dist/chunker/pdf.js +30 -14
- package/dist/cli/commands/backend-detect.d.ts +61 -0
- package/dist/cli/commands/backend-detect.js +119 -0
- package/dist/cli/commands/index-command.js +11 -0
- package/dist/cli/commands/init-helpers.d.ts +4 -1
- package/dist/cli/commands/init-helpers.js +21 -4
- package/dist/cli/commands/init.js +61 -24
- package/dist/cli/commands/quirk.js +9 -3
- package/dist/cli/commands/setup.js +12 -3
- package/dist/cli/commands/status.js +14 -5
- package/dist/cli/commands/ui.js +23 -9
- package/dist/cli/commands/update.js +4 -5
- package/dist/cli/format.d.ts +5 -2
- package/dist/cli/format.js +14 -5
- package/dist/content/image.js +33 -11
- package/dist/content/reader.js +74 -13
- package/dist/core/bootstrap.js +10 -3
- package/dist/core/config.d.ts +27 -1
- package/dist/core/config.js +41 -2
- package/dist/core/desc-cache.d.ts +8 -2
- package/dist/core/desc-cache.js +10 -3
- package/dist/core/doc-progress.js +5 -2
- package/dist/core/interfaces.d.ts +32 -4
- package/dist/core/manifest.js +1 -1
- package/dist/core/provider-defaults.d.ts +2 -0
- package/dist/core/provider-defaults.js +19 -4
- package/dist/core/runtime-overrides.d.ts +0 -6
- package/dist/core/version-check.d.ts +5 -0
- package/dist/core/version-check.js +8 -2
- package/dist/describer/anthropic.d.ts +2 -2
- package/dist/describer/anthropic.js +19 -5
- package/dist/describer/describer.d.ts +20 -0
- package/dist/describer/describer.js +132 -16
- package/dist/describer/gemini.js +25 -10
- package/dist/describer/shared.d.ts +28 -0
- package/dist/describer/shared.js +60 -0
- package/dist/embedder/factory.d.ts +5 -3
- package/dist/embedder/factory.js +42 -9
- package/dist/embedder/health.js +19 -19
- package/dist/embedder/http.d.ts +14 -1
- package/dist/embedder/http.js +60 -6
- package/dist/embedder/ollama.d.ts +3 -1
- package/dist/embedder/ollama.js +12 -2
- package/dist/eval/session-logger.js +7 -0
- package/dist/eval/storage.js +8 -0
- package/dist/indexer/git-diff.d.ts +1 -1
- package/dist/indexer/git-diff.js +5 -1
- package/dist/indexer/pipeline.js +511 -346
- package/dist/indexer/stats.d.ts +2 -0
- package/dist/indexer/stats.js +1 -0
- package/dist/indexer/watch.js +8 -1
- package/dist/indexer/worker.js +21 -0
- package/dist/mcp/cli.js +4 -0
- package/dist/mcp/handlers.js +16 -5
- package/dist/mcp/server.js +3 -0
- package/dist/opencode/create-read-tool.js +14 -3
- package/dist/opencode/tool-args.js +23 -1
- package/dist/plugin.js +66 -152
- package/dist/quirks/auto-capture.js +5 -0
- package/dist/quirks/quirk-store.d.ts +1 -1
- package/dist/quirks/quirk-store.js +56 -17
- package/dist/retriever/context-optimizer.js +18 -4
- package/dist/retriever/keyword-index.d.ts +2 -0
- package/dist/retriever/keyword-index.js +38 -4
- package/dist/retriever/retriever.js +6 -1
- package/dist/tui.js +41 -4
- package/dist/vectorstore/lancedb.d.ts +104 -8
- package/dist/vectorstore/lancedb.js +345 -71
- package/dist/vectorstore/memory.d.ts +8 -3
- package/dist/vectorstore/memory.js +27 -4
- package/dist/watcher.d.ts +8 -0
- package/dist/watcher.js +237 -85
- package/dist/web/api.d.ts +5 -1
- package/dist/web/api.js +195 -69
- package/dist/web/server.d.ts +2 -0
- package/dist/web/server.js +66 -28
- package/dist/web/static.d.ts +5 -2
- package/dist/web/static.js +9 -5
- package/dist/web/ui/assets/index-BDPYdtA1.js +3 -0
- package/dist/web/ui/index.html +1 -1
- package/package.json +1 -1
- package/dist/web/ui/assets/index-CJBvt6e0.js +0 -3
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import { postJson } from "../embedder/http.js";
|
|
2
|
-
import { buildUserMessage, sleep } from "./shared.js";
|
|
2
|
+
import { buildUserMessage, buildBatchUserMessage, parseBatchDescriptions, sleep } from "./shared.js";
|
|
3
3
|
import pLimit from "p-limit";
|
|
4
4
|
/** HTTP status codes that are safe to retry on. */
|
|
5
5
|
const RETRYABLE_STATUSES = new Set([408, 429, 500, 502, 503, 504]);
|
|
6
|
+
/** Consecutive failed batch attempts after which batching is disabled for the rest of the run. */
|
|
7
|
+
const BATCH_MAX_STREAK = 2;
|
|
6
8
|
/**
|
|
7
9
|
* Description provider that works with any OpenAI-compatible chat API (including Ollama).
|
|
8
10
|
*
|
|
@@ -11,6 +13,10 @@ const RETRYABLE_STATUSES = new Set([408, 429, 500, 502, 503, 504]);
|
|
|
11
13
|
*/
|
|
12
14
|
export class LlmDescriptionProvider {
|
|
13
15
|
config;
|
|
16
|
+
/** Consecutive batches that needed individual fallback; disables batching past BATCH_MAX_STREAK. */
|
|
17
|
+
batchFailStreak = 0;
|
|
18
|
+
/** Whether multi-chunk batching is still active (adaptive, per provider instance / index run). */
|
|
19
|
+
adaptiveBatchActive = true;
|
|
14
20
|
/**
|
|
15
21
|
* @param config - Configuration for the LLM provider, including base URL, model, API key, proxy, and retry settings.
|
|
16
22
|
*/
|
|
@@ -37,28 +43,125 @@ export class LlmDescriptionProvider {
|
|
|
37
43
|
async generateBatchDescriptions(chunks, logger, opts) {
|
|
38
44
|
const log = logger ?? { info: (msg) => process.stderr.write(`${msg}\n`), warn: (msg) => process.stderr.write(`${msg}\n`), debug: (msg) => process.stderr.write(`${msg}\n`) };
|
|
39
45
|
const concurrency = this.config.batchConcurrency ?? 3;
|
|
46
|
+
// Batching is an EXPERIMENTAL opt-in (description.batchEnabled). Off by
|
|
47
|
+
// default — a batch size of 1 routes every group through the individual
|
|
48
|
+
// request path below, which is exactly the per-chunk behavior.
|
|
49
|
+
const batchEnabled = this.config.batchEnabled === true;
|
|
50
|
+
const batchMaxChunks = batchEnabled ? (this.config.batchMaxChunks ?? 25) : 1;
|
|
40
51
|
const total = chunks.length;
|
|
41
|
-
log.debug(`[describer] Generating descriptions for ${total} chunks via ${this.config.provider}/${this.config.model} (concurrency: ${concurrency})`);
|
|
52
|
+
log.debug(`[describer] Generating descriptions for ${total} chunks via ${this.config.provider}/${this.config.model} (concurrency: ${concurrency}, batch: ${batchEnabled ? batchMaxChunks : "off"})`);
|
|
42
53
|
const result = new Map();
|
|
43
54
|
const limit = pLimit(concurrency);
|
|
44
55
|
let completed = 0;
|
|
45
|
-
|
|
46
|
-
const userMsg = buildUserMessage(chunk, this.config.maxContentChars);
|
|
47
|
-
log.debug(`[describer] REQUEST chunk ${chunk.id} (${chunk.metadata.filePath}:${chunk.metadata.startLine}):\n${userMsg}`);
|
|
48
|
-
try {
|
|
49
|
-
const desc = await this.generateDescription(chunk);
|
|
50
|
-
result.set(chunk.id, desc);
|
|
51
|
-
log.debug(`[describer] RESPONSE chunk ${chunk.id}: ${desc}`);
|
|
52
|
-
}
|
|
53
|
-
catch (err) {
|
|
54
|
-
log.warn(`[describer] Failed to describe chunk ${chunk.id} (${chunk.metadata.filePath}:${chunk.metadata.startLine}): ${err instanceof Error ? err.message : String(err)}`);
|
|
55
|
-
}
|
|
56
|
+
const emitProgress = (chunk) => {
|
|
56
57
|
completed++;
|
|
57
58
|
opts?.onProgress?.(chunk, completed, opts.total ?? total);
|
|
59
|
+
};
|
|
60
|
+
// Group chunks so that up to batchMaxChunks share a single LLM request,
|
|
61
|
+
// cutting round trips (one prefill + one generation per group instead of
|
|
62
|
+
// per chunk). Groups with a single chunk use the individual path for
|
|
63
|
+
// consistent error handling.
|
|
64
|
+
const groups = [];
|
|
65
|
+
for (let i = 0; i < chunks.length; i += batchMaxChunks) {
|
|
66
|
+
groups.push(chunks.slice(i, i + batchMaxChunks));
|
|
67
|
+
}
|
|
68
|
+
await Promise.all(groups.map((group) => limit(async () => {
|
|
69
|
+
if (group.length === 1) {
|
|
70
|
+
const chunk = group[0];
|
|
71
|
+
try {
|
|
72
|
+
const desc = await this.generateDescription(chunk);
|
|
73
|
+
result.set(chunk.id, desc);
|
|
74
|
+
}
|
|
75
|
+
catch (err) {
|
|
76
|
+
log.warn(`[describer] Failed to describe chunk ${chunk.id} (${chunk.metadata.filePath}:${chunk.metadata.startLine}): ${err instanceof Error ? err.message : String(err)}`);
|
|
77
|
+
}
|
|
78
|
+
emitProgress(chunk);
|
|
79
|
+
return;
|
|
80
|
+
}
|
|
81
|
+
// Small models are unreliable at structured multi-item output — if a
|
|
82
|
+
// batch response can't be parsed, fall back to individual requests
|
|
83
|
+
// and degrade to per-chunk mode for the rest of the run after
|
|
84
|
+
// BATCH_MAX_STREAK consecutive failures.
|
|
85
|
+
let resolved;
|
|
86
|
+
let batchFailed = false;
|
|
87
|
+
if (this.adaptiveBatchActive) {
|
|
88
|
+
try {
|
|
89
|
+
resolved = await this.batchDescribe(group, log);
|
|
90
|
+
}
|
|
91
|
+
catch (err) {
|
|
92
|
+
log.warn(`[describer] Batch description failed for ${group.length} chunks, falling back to individual: ${err instanceof Error ? err.message : String(err)}`);
|
|
93
|
+
resolved = new Map();
|
|
94
|
+
batchFailed = true;
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
else {
|
|
98
|
+
resolved = new Map();
|
|
99
|
+
}
|
|
100
|
+
// Batch labels are ordinals (1..N); map back to chunks by position.
|
|
101
|
+
const missing = [];
|
|
102
|
+
for (let i = 0; i < group.length; i++) {
|
|
103
|
+
const chunk = group[i];
|
|
104
|
+
const desc = resolved.get(String(i + 1));
|
|
105
|
+
if (desc && desc.trim().length > 0) {
|
|
106
|
+
result.set(chunk.id, desc.trim());
|
|
107
|
+
emitProgress(chunk);
|
|
108
|
+
}
|
|
109
|
+
else {
|
|
110
|
+
missing.push(chunk);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
if (batchFailed || missing.length > 0) {
|
|
114
|
+
this.batchFailStreak++;
|
|
115
|
+
if (this.adaptiveBatchActive && this.batchFailStreak >= BATCH_MAX_STREAK) {
|
|
116
|
+
this.adaptiveBatchActive = false;
|
|
117
|
+
log.warn(`[describer] Batch descriptions unreliable (${this.batchFailStreak} consecutive failures), switching to individual requests for the rest of the run`);
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
else {
|
|
121
|
+
this.batchFailStreak = 0;
|
|
122
|
+
}
|
|
123
|
+
// Fall back to individual requests for chunks the batch did not cover.
|
|
124
|
+
for (const chunk of missing) {
|
|
125
|
+
try {
|
|
126
|
+
const desc = await this.generateDescription(chunk);
|
|
127
|
+
result.set(chunk.id, desc);
|
|
128
|
+
}
|
|
129
|
+
catch (err) {
|
|
130
|
+
log.warn(`[describer] Failed to describe chunk ${chunk.id} (${chunk.metadata.filePath}:${chunk.metadata.startLine}): ${err instanceof Error ? err.message : String(err)}`);
|
|
131
|
+
}
|
|
132
|
+
emitProgress(chunk);
|
|
133
|
+
}
|
|
58
134
|
})));
|
|
59
135
|
log.debug(`[describer] Descriptions generated: ${result.size}/${total}`);
|
|
60
136
|
return result;
|
|
61
137
|
}
|
|
138
|
+
/**
|
|
139
|
+
* Describe a group of chunks in a single LLM request and parse the response.
|
|
140
|
+
*
|
|
141
|
+
* Builds one chat request whose user message contains all chunks wrapped in
|
|
142
|
+
* `[CHUNK <n>]` markers and expects a `<n>: <description>` line per chunk.
|
|
143
|
+
* Labels are ordinals and are mapped back to chunks by position in the
|
|
144
|
+
* calling group; labels outside the group's range (hallucinations) are
|
|
145
|
+
* naturally dropped, prompting the caller to fall back to individual
|
|
146
|
+
* requests for the missing chunks.
|
|
147
|
+
*
|
|
148
|
+
* @param group - Chunks to describe in one request (length > 1)
|
|
149
|
+
* @param log - Logger for diagnostic messages
|
|
150
|
+
* @returns Map of ordinal label to description (may be partial or empty)
|
|
151
|
+
* @throws When the LLM request itself fails (caller falls back per chunk)
|
|
152
|
+
*/
|
|
153
|
+
async batchDescribe(group, log) {
|
|
154
|
+
const messages = [
|
|
155
|
+
{ role: "system", content: this.config.systemPrompt },
|
|
156
|
+
{ role: "user", content: buildBatchUserMessage(group, this.config.maxContentChars) },
|
|
157
|
+
];
|
|
158
|
+
const timeoutMs = this.config.batchTimeoutMs ?? this.config.timeoutMs ?? 120000;
|
|
159
|
+
log.debug(`[describer] BATCH REQUEST ${group.length} chunks (${group[0].metadata.filePath}):\n${messages[1].content}`);
|
|
160
|
+
const content = await this.chatRequest(messages, timeoutMs);
|
|
161
|
+
const parsed = parseBatchDescriptions(content);
|
|
162
|
+
log.debug(`[describer] BATCH RESPONSE parsed ${parsed.size}/${group.length} chunks`);
|
|
163
|
+
return parsed;
|
|
164
|
+
}
|
|
62
165
|
/**
|
|
63
166
|
* Sends a chat completion request to the LLM API with retry and exponential backoff.
|
|
64
167
|
* For Ollama, uses the `/api/chat` endpoint with streaming disabled; otherwise uses the standard `/v1/chat/completions` endpoint.
|
|
@@ -75,7 +178,7 @@ export class LlmDescriptionProvider {
|
|
|
75
178
|
? `${baseUrl}/chat`
|
|
76
179
|
: `${baseUrl}${baseUrl.endsWith("/v1") ? "" : "/v1"}/chat/completions`;
|
|
77
180
|
const body = isOllama
|
|
78
|
-
? { model: this.config.model, messages, stream: false, think: this.config.think ?? false, options: { num_ctx: this.config.numCtx } }
|
|
181
|
+
? { model: this.config.model, messages, stream: false, think: this.config.think ?? false, options: { num_ctx: this.config.numCtx }, keep_alive: this.config.keepAlive }
|
|
79
182
|
: { model: this.config.model, messages };
|
|
80
183
|
const headers = {};
|
|
81
184
|
if (this.config.apiKey) {
|
|
@@ -85,7 +188,20 @@ export class LlmDescriptionProvider {
|
|
|
85
188
|
const retryBaseDelayMs = this.config.retryBaseDelayMs ?? 1000;
|
|
86
189
|
let lastError;
|
|
87
190
|
for (let attempt = 0; attempt <= retryMax; attempt++) {
|
|
88
|
-
|
|
191
|
+
let response;
|
|
192
|
+
try {
|
|
193
|
+
response = await postJson(url, body, headers, timeoutMs, this.config.proxy);
|
|
194
|
+
}
|
|
195
|
+
catch (err) {
|
|
196
|
+
// Network-level failures (ECONNREFUSED, socket timeouts) are transient —
|
|
197
|
+
// treat them like retryable HTTP statuses.
|
|
198
|
+
lastError = err instanceof Error ? err : new Error(String(err));
|
|
199
|
+
if (attempt === retryMax)
|
|
200
|
+
throw lastError;
|
|
201
|
+
const delayMs = retryBaseDelayMs * Math.pow(2, attempt) * (0.8 + Math.random() * 0.4);
|
|
202
|
+
await sleep(delayMs);
|
|
203
|
+
continue;
|
|
204
|
+
}
|
|
89
205
|
if (response.ok) {
|
|
90
206
|
const json = (await response.json());
|
|
91
207
|
return extractResponseText(json, isOllama);
|
|
@@ -96,7 +212,7 @@ export class LlmDescriptionProvider {
|
|
|
96
212
|
throw error;
|
|
97
213
|
}
|
|
98
214
|
lastError = error;
|
|
99
|
-
const delayMs = retryBaseDelayMs * Math.pow(2, attempt);
|
|
215
|
+
const delayMs = retryBaseDelayMs * Math.pow(2, attempt) * (0.8 + Math.random() * 0.4);
|
|
100
216
|
await sleep(delayMs);
|
|
101
217
|
}
|
|
102
218
|
throw lastError ?? new Error("Description LLM request failed: unknown error");
|
package/dist/describer/gemini.js
CHANGED
|
@@ -57,24 +57,39 @@ export class GeminiDescriptionProvider {
|
|
|
57
57
|
const apiKey = this.config.apiKey ?? "";
|
|
58
58
|
const model = this.config.model;
|
|
59
59
|
const systemPrompt = systemOverride ?? this.config.systemPrompt;
|
|
60
|
-
const allParts = [{ text: systemPrompt }];
|
|
61
|
-
for (const c of contents) {
|
|
62
|
-
allParts.push(...c.parts);
|
|
63
|
-
}
|
|
64
60
|
const body = {
|
|
65
|
-
|
|
61
|
+
// The system prompt goes into the native systemInstruction field, never
|
|
62
|
+
// into the same content parts as the chunk text.
|
|
63
|
+
systemInstruction: { parts: [{ text: systemPrompt }] },
|
|
64
|
+
contents: contents.map((c) => ({ role: c.role, parts: c.parts })),
|
|
66
65
|
};
|
|
67
66
|
const headers = {
|
|
68
67
|
"Content-Type": "application/json",
|
|
69
68
|
};
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
69
|
+
// Send the key via header so it never appears in URLs (which are echoed in
|
|
70
|
+
// redirect-limit error messages) and cannot be mangled by URL encoding.
|
|
71
|
+
if (apiKey) {
|
|
72
|
+
headers["x-goog-api-key"] = apiKey;
|
|
73
|
+
}
|
|
74
|
+
const url = `${baseUrl}/models/${model}:generateContent`;
|
|
73
75
|
const retryMax = this.config.retryMax ?? 3;
|
|
74
76
|
const retryBaseDelayMs = this.config.retryBaseDelayMs ?? 1000;
|
|
75
77
|
let lastError;
|
|
76
78
|
for (let attempt = 0; attempt <= retryMax; attempt++) {
|
|
77
|
-
|
|
79
|
+
let response;
|
|
80
|
+
try {
|
|
81
|
+
response = await postJson(url, body, headers, timeoutMs, this.config.proxy);
|
|
82
|
+
}
|
|
83
|
+
catch (err) {
|
|
84
|
+
// Network-level failures (ECONNREFUSED, socket timeouts) are transient —
|
|
85
|
+
// treat them like retryable HTTP statuses.
|
|
86
|
+
lastError = err instanceof Error ? err : new Error(String(err));
|
|
87
|
+
if (attempt === retryMax)
|
|
88
|
+
throw lastError;
|
|
89
|
+
const delayMs = retryBaseDelayMs * Math.pow(2, attempt) * (0.8 + Math.random() * 0.4);
|
|
90
|
+
await sleep(delayMs);
|
|
91
|
+
continue;
|
|
92
|
+
}
|
|
78
93
|
if (response.ok) {
|
|
79
94
|
const json = (await response.json());
|
|
80
95
|
const text = json.candidates?.[0]?.content?.parts?.[0]?.text;
|
|
@@ -89,7 +104,7 @@ export class GeminiDescriptionProvider {
|
|
|
89
104
|
throw error;
|
|
90
105
|
}
|
|
91
106
|
lastError = error;
|
|
92
|
-
const delayMs = retryBaseDelayMs * Math.pow(2, attempt);
|
|
107
|
+
const delayMs = retryBaseDelayMs * Math.pow(2, attempt) * (0.8 + Math.random() * 0.4);
|
|
93
108
|
await sleep(delayMs);
|
|
94
109
|
}
|
|
95
110
|
throw lastError ?? new Error("Gemini LLM request failed: unknown error");
|
|
@@ -15,3 +15,31 @@ import type { Chunk } from "../core/interfaces.js";
|
|
|
15
15
|
export declare function buildUserMessage(chunk: Chunk, maxContentChars?: number): string;
|
|
16
16
|
/** Promise-based delay for use with async/await. */
|
|
17
17
|
export declare function sleep(ms: number): Promise<void>;
|
|
18
|
+
/**
|
|
19
|
+
* Build a user message describing multiple chunks in a single LLM request.
|
|
20
|
+
*
|
|
21
|
+
* Each chunk is wrapped in `[CHUNK <n>] ... [END CHUNK <n>]` markers using
|
|
22
|
+
* short ordinal labels (1-based position in the group) rather than chunk
|
|
23
|
+
* UUIDs — small models reliably reproduce `n: <description>` lines but often
|
|
24
|
+
* mangle long random ids. The model is asked to reply with exactly one line
|
|
25
|
+
* per chunk in `<n>: <one-line description>` format (see
|
|
26
|
+
* `parseBatchDescriptions`); labels map back to chunks by position.
|
|
27
|
+
*
|
|
28
|
+
* @param chunks - The chunks to describe in one request
|
|
29
|
+
* @param maxContentChars - Optional per-chunk content truncation limit
|
|
30
|
+
* @returns Formatted message string for the batched LLM request
|
|
31
|
+
*/
|
|
32
|
+
export declare function buildBatchUserMessage(chunks: Chunk[], maxContentChars?: number): string;
|
|
33
|
+
/**
|
|
34
|
+
* Parse a batched description response into a `label → description` map.
|
|
35
|
+
*
|
|
36
|
+
* Tolerant parser: only lines matching `<label>: <text>` are kept, so
|
|
37
|
+
* preamble, trailing prose, or markdown fences are ignored. Labels are the
|
|
38
|
+
* short ordinals used in `buildBatchUserMessage` (e.g. `1: handles auth`);
|
|
39
|
+
* they are mapped back to chunk IDs by position in the calling group.
|
|
40
|
+
* Duplicate labels keep the first occurrence.
|
|
41
|
+
*
|
|
42
|
+
* @param content - Raw response text from the LLM
|
|
43
|
+
* @returns Map of label to description (may be empty or partial)
|
|
44
|
+
*/
|
|
45
|
+
export declare function parseBatchDescriptions(content: string): Map<string, string>;
|
package/dist/describer/shared.js
CHANGED
|
@@ -31,4 +31,64 @@ export function buildUserMessage(chunk, maxContentChars) {
|
|
|
31
31
|
export function sleep(ms) {
|
|
32
32
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
33
33
|
}
|
|
34
|
+
/**
|
|
35
|
+
* Build a user message describing multiple chunks in a single LLM request.
|
|
36
|
+
*
|
|
37
|
+
* Each chunk is wrapped in `[CHUNK <n>] ... [END CHUNK <n>]` markers using
|
|
38
|
+
* short ordinal labels (1-based position in the group) rather than chunk
|
|
39
|
+
* UUIDs — small models reliably reproduce `n: <description>` lines but often
|
|
40
|
+
* mangle long random ids. The model is asked to reply with exactly one line
|
|
41
|
+
* per chunk in `<n>: <one-line description>` format (see
|
|
42
|
+
* `parseBatchDescriptions`); labels map back to chunks by position.
|
|
43
|
+
*
|
|
44
|
+
* @param chunks - The chunks to describe in one request
|
|
45
|
+
* @param maxContentChars - Optional per-chunk content truncation limit
|
|
46
|
+
* @returns Formatted message string for the batched LLM request
|
|
47
|
+
*/
|
|
48
|
+
export function buildBatchUserMessage(chunks, maxContentChars) {
|
|
49
|
+
const parts = [
|
|
50
|
+
"Describe each code chunk below in ONE short sentence (max 20 words) each.",
|
|
51
|
+
"Do not repeat code in your descriptions.",
|
|
52
|
+
"Reply with EXACTLY one line per chunk, in this format:",
|
|
53
|
+
"<number>: <one-line description>",
|
|
54
|
+
"",
|
|
55
|
+
];
|
|
56
|
+
chunks.forEach((chunk, index) => {
|
|
57
|
+
const label = index + 1;
|
|
58
|
+
parts.push(`[CHUNK ${label}]`);
|
|
59
|
+
parts.push(buildUserMessage(chunk, maxContentChars));
|
|
60
|
+
parts.push(`[END CHUNK ${label}]`);
|
|
61
|
+
parts.push("");
|
|
62
|
+
});
|
|
63
|
+
return parts.join("\n").trim();
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Parse a batched description response into a `label → description` map.
|
|
67
|
+
*
|
|
68
|
+
* Tolerant parser: only lines matching `<label>: <text>` are kept, so
|
|
69
|
+
* preamble, trailing prose, or markdown fences are ignored. Labels are the
|
|
70
|
+
* short ordinals used in `buildBatchUserMessage` (e.g. `1: handles auth`);
|
|
71
|
+
* they are mapped back to chunk IDs by position in the calling group.
|
|
72
|
+
* Duplicate labels keep the first occurrence.
|
|
73
|
+
*
|
|
74
|
+
* @param content - Raw response text from the LLM
|
|
75
|
+
* @returns Map of label to description (may be empty or partial)
|
|
76
|
+
*/
|
|
77
|
+
export function parseBatchDescriptions(content) {
|
|
78
|
+
const result = new Map();
|
|
79
|
+
const lineRe = /^([A-Za-z0-9_.-]+):\s*(.+)$/;
|
|
80
|
+
for (const rawLine of content.split(/\r?\n/)) {
|
|
81
|
+
const line = rawLine.trim();
|
|
82
|
+
if (!line)
|
|
83
|
+
continue;
|
|
84
|
+
const match = lineRe.exec(line);
|
|
85
|
+
if (match && match[1] && match[2] && match[2].trim().length > 0) {
|
|
86
|
+
const id = match[1];
|
|
87
|
+
if (!result.has(id)) {
|
|
88
|
+
result.set(id, match[2].trim());
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
return result;
|
|
93
|
+
}
|
|
34
94
|
//# sourceMappingURL=shared.js.map
|
|
@@ -22,9 +22,11 @@ export declare function createEmbedder(config: RagConfig): EmbeddingProvider;
|
|
|
22
22
|
* (or concurrently when `concurrency > 1`). When concurrency is limited, uses
|
|
23
23
|
* `p-limit` to cap the number of in-flight requests.
|
|
24
24
|
*
|
|
25
|
-
* Each batch is retried up to `retryMax` times with exponential backoff
|
|
26
|
-
*
|
|
27
|
-
*
|
|
25
|
+
* Each batch is retried up to `retryMax` times with exponential backoff (only
|
|
26
|
+
* for transient failures — auth/validation errors are not retried). If all
|
|
27
|
+
* retries are exhausted or the provider returns a mismatched embedding count,
|
|
28
|
+
* the batch is skipped and empty arrays are returned for those texts so the
|
|
29
|
+
* caller can still process successfully embedded batches.
|
|
28
30
|
*
|
|
29
31
|
* @param embedder - The embedding provider to use
|
|
30
32
|
* @param texts - Array of text strings to embed
|
package/dist/embedder/factory.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { isOpenAiCompatible } from "../core/provider-defaults.js";
|
|
1
|
+
import { isOpenAiCompatible, supportsEmbedding } from "../core/provider-defaults.js";
|
|
2
2
|
import { OllamaProvider } from "./ollama.js";
|
|
3
3
|
import { OpenAIProvider } from "./openai.js";
|
|
4
4
|
import { CohereProvider } from "./cohere.js";
|
|
@@ -17,8 +17,14 @@ import pLimit from "p-limit";
|
|
|
17
17
|
export function createEmbedder(config) {
|
|
18
18
|
const { provider, baseUrl, model, apiKey, proxy, timeoutMs } = config.embedding;
|
|
19
19
|
const effectiveTimeoutMs = timeoutMs ?? 120000;
|
|
20
|
+
// Fail fast for chat-only providers (groq, deepseek, anthropic, google)
|
|
21
|
+
// — proceeding would surface a cryptic failure at index time.
|
|
22
|
+
if (!supportsEmbedding(provider)) {
|
|
23
|
+
throw new Error(`Provider "${provider}" does not support embeddings. ` +
|
|
24
|
+
"Use an embedding-capable provider (ollama, openai, cohere, nvidia, azure, mistral, together, fireworks).");
|
|
25
|
+
}
|
|
20
26
|
if (provider === "ollama") {
|
|
21
|
-
return new OllamaProvider(baseUrl, model, apiKey, effectiveTimeoutMs, proxy, config.logging.level);
|
|
27
|
+
return new OllamaProvider(baseUrl, model, apiKey, effectiveTimeoutMs, proxy, config.logging.level, config.embedding.keepAlive);
|
|
22
28
|
}
|
|
23
29
|
if (provider === "cohere") {
|
|
24
30
|
if (!apiKey) {
|
|
@@ -34,6 +40,15 @@ export function createEmbedder(config) {
|
|
|
34
40
|
}
|
|
35
41
|
throw new Error(`Unknown embedding provider: ${provider}`);
|
|
36
42
|
}
|
|
43
|
+
/**
|
|
44
|
+
* HTTP statuses that indicate a permanent failure — retrying cannot help
|
|
45
|
+
* (auth errors, bad requests, missing resources). Providers raise these as
|
|
46
|
+
* Error objects whose messages contain the status code.
|
|
47
|
+
*/
|
|
48
|
+
const PERMANENT_STATUS_RE = /\(?(400|401|403|404|422)\)?/;
|
|
49
|
+
function isPermanentError(err) {
|
|
50
|
+
return PERMANENT_STATUS_RE.test(err instanceof Error ? err.message : String(err));
|
|
51
|
+
}
|
|
37
52
|
/**
|
|
38
53
|
* Embed a list of texts in batches with optional concurrency control and per-batch retry.
|
|
39
54
|
*
|
|
@@ -41,9 +56,11 @@ export function createEmbedder(config) {
|
|
|
41
56
|
* (or concurrently when `concurrency > 1`). When concurrency is limited, uses
|
|
42
57
|
* `p-limit` to cap the number of in-flight requests.
|
|
43
58
|
*
|
|
44
|
-
* Each batch is retried up to `retryMax` times with exponential backoff
|
|
45
|
-
*
|
|
46
|
-
*
|
|
59
|
+
* Each batch is retried up to `retryMax` times with exponential backoff (only
|
|
60
|
+
* for transient failures — auth/validation errors are not retried). If all
|
|
61
|
+
* retries are exhausted or the provider returns a mismatched embedding count,
|
|
62
|
+
* the batch is skipped and empty arrays are returned for those texts so the
|
|
63
|
+
* caller can still process successfully embedded batches.
|
|
47
64
|
*
|
|
48
65
|
* @param embedder - The embedding provider to use
|
|
49
66
|
* @param texts - Array of text strings to embed
|
|
@@ -67,13 +84,27 @@ export async function embedBatch(embedder, texts, batchSize = 10, purpose, concu
|
|
|
67
84
|
async function embedWithRetry(batchTexts) {
|
|
68
85
|
for (let attempt = 0; attempt <= retryMax; attempt++) {
|
|
69
86
|
try {
|
|
70
|
-
|
|
87
|
+
const embeddings = await embedder.embed(batchTexts, purpose);
|
|
88
|
+
// Validate the response shape: a count mismatch would silently attach
|
|
89
|
+
// vectors to the wrong chunks downstream; a dimension mismatch would
|
|
90
|
+
// poison the store. Treat both as a retryable batch failure.
|
|
91
|
+
if (embeddings.length !== batchTexts.length) {
|
|
92
|
+
throw new Error(`Embedding provider returned ${embeddings.length} vectors for ${batchTexts.length} texts`);
|
|
93
|
+
}
|
|
94
|
+
const dims = new Set(embeddings.map((v) => v.length));
|
|
95
|
+
if (dims.size > 1 || dims.has(0)) {
|
|
96
|
+
throw new Error(`Embedding provider returned inconsistent vector dimensions: ${[...dims].join(", ")}`);
|
|
97
|
+
}
|
|
98
|
+
return embeddings;
|
|
71
99
|
}
|
|
72
100
|
catch (err) {
|
|
73
|
-
if (attempt < retryMax) {
|
|
74
|
-
const delay = retryBaseDelayMs * Math.pow(2, attempt);
|
|
101
|
+
if (attempt < retryMax && !isPermanentError(err)) {
|
|
102
|
+
const delay = retryBaseDelayMs * Math.pow(2, attempt) * (0.8 + Math.random() * 0.4);
|
|
75
103
|
await new Promise(resolve => setTimeout(resolve, delay));
|
|
76
104
|
}
|
|
105
|
+
else if (attempt >= retryMax || isPermanentError(err)) {
|
|
106
|
+
return null;
|
|
107
|
+
}
|
|
77
108
|
}
|
|
78
109
|
}
|
|
79
110
|
return null;
|
|
@@ -99,7 +130,9 @@ export async function embedBatch(embedder, texts, batchSize = 10, purpose, concu
|
|
|
99
130
|
const batchResults = await Promise.all(batches.map((batch) => limit(async () => {
|
|
100
131
|
const embeddings = await embedWithRetry(batch.texts);
|
|
101
132
|
const flatResult = embeddings ?? batch.texts.map(() => []);
|
|
102
|
-
|
|
133
|
+
// Count processed TEXTS (not returned vectors) so failed batches
|
|
134
|
+
// still advance progress towards the total.
|
|
135
|
+
completedCount += batch.texts.length;
|
|
103
136
|
onProgress?.(completedCount, texts.length);
|
|
104
137
|
return { index: batch.index, embeddings: flatResult };
|
|
105
138
|
})));
|
package/dist/embedder/health.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { postJson } from "./http.js";
|
|
1
|
+
import { fetchWithProxy, postJson } from "./http.js";
|
|
2
2
|
/**
|
|
3
3
|
* Check connectivity and model availability for all configured providers.
|
|
4
4
|
* Returns one result per configured model (embedding + description + image_description if enabled).
|
|
@@ -42,10 +42,10 @@ async function checkDescriptionModel(config, _timeoutMs) {
|
|
|
42
42
|
return checkOllamaChat(baseUrl, model, descTimeout, desc.proxy);
|
|
43
43
|
}
|
|
44
44
|
if (provider === "anthropic") {
|
|
45
|
-
return checkAnthropicChat(baseUrl, model, apiKey, descTimeout);
|
|
45
|
+
return checkAnthropicChat(baseUrl, model, apiKey, descTimeout, desc.proxy);
|
|
46
46
|
}
|
|
47
47
|
if (provider === "google") {
|
|
48
|
-
return checkGoogleChat(baseUrl, model, apiKey, descTimeout);
|
|
48
|
+
return checkGoogleChat(baseUrl, model, apiKey, descTimeout, desc.proxy);
|
|
49
49
|
}
|
|
50
50
|
// OpenAI-compatible chat endpoint
|
|
51
51
|
return checkOpenAiChat(baseUrl, model, apiKey, descTimeout, desc.proxy);
|
|
@@ -62,10 +62,10 @@ async function checkImageDescriptionModel(config, _timeoutMs) {
|
|
|
62
62
|
return checkOllamaChat(baseUrl, model, imgTimeout, img.proxy, "image_description");
|
|
63
63
|
}
|
|
64
64
|
if (provider === "anthropic") {
|
|
65
|
-
return checkAnthropicChat(baseUrl, model, apiKey, imgTimeout, "image_description");
|
|
65
|
+
return checkAnthropicChat(baseUrl, model, apiKey, imgTimeout, img.proxy, "image_description");
|
|
66
66
|
}
|
|
67
67
|
if (provider === "google") {
|
|
68
|
-
return checkGoogleChat(baseUrl, model, apiKey, imgTimeout, "image_description");
|
|
68
|
+
return checkGoogleChat(baseUrl, model, apiKey, imgTimeout, img.proxy, "image_description");
|
|
69
69
|
}
|
|
70
70
|
// OpenAI-compatible chat endpoint
|
|
71
71
|
return checkOpenAiChat(baseUrl, model, apiKey, imgTimeout, img.proxy, "image_description");
|
|
@@ -125,16 +125,16 @@ async function checkOllamaChat(baseUrl, model, timeoutMs, proxy, type = "descrip
|
|
|
125
125
|
* Check OpenAI-compatible embedding endpoint via the /models endpoint
|
|
126
126
|
* to validate the API key without consuming embedding tokens.
|
|
127
127
|
*/
|
|
128
|
-
async function checkOpenAiEmbed(baseUrl, model, apiKey, timeoutMs,
|
|
128
|
+
async function checkOpenAiEmbed(baseUrl, model, apiKey, timeoutMs, proxy) {
|
|
129
129
|
if (!apiKey) {
|
|
130
130
|
return { provider: "openai", model, type: "embedding", status: "error", error: "No API key configured" };
|
|
131
131
|
}
|
|
132
132
|
const url = `${baseUrl.replace(/\/+$/, "")}/models`;
|
|
133
133
|
try {
|
|
134
|
-
const response = await
|
|
134
|
+
const response = await fetchWithProxy(url, {
|
|
135
135
|
headers: { Authorization: `Bearer ${apiKey}` },
|
|
136
136
|
signal: AbortSignal.timeout(Math.min(timeoutMs ?? 15000, 15000)),
|
|
137
|
-
});
|
|
137
|
+
}, proxy);
|
|
138
138
|
if (response.ok) {
|
|
139
139
|
return { provider: "openai", model, type: "embedding", status: "ok" };
|
|
140
140
|
}
|
|
@@ -150,16 +150,16 @@ async function checkOpenAiEmbed(baseUrl, model, apiKey, timeoutMs, _proxy) {
|
|
|
150
150
|
}
|
|
151
151
|
}
|
|
152
152
|
/** Check OpenAI-compatible chat endpoint via the /models endpoint. */
|
|
153
|
-
async function checkOpenAiChat(baseUrl, model, apiKey, timeoutMs,
|
|
153
|
+
async function checkOpenAiChat(baseUrl, model, apiKey, timeoutMs, proxy, type = "description") {
|
|
154
154
|
if (!apiKey) {
|
|
155
155
|
return { provider: "openai", model, type, status: "error", error: "No API key configured" };
|
|
156
156
|
}
|
|
157
157
|
const url = `${baseUrl.replace(/\/+$/, "")}/models`;
|
|
158
158
|
try {
|
|
159
|
-
const response = await
|
|
159
|
+
const response = await fetchWithProxy(url, {
|
|
160
160
|
headers: { Authorization: `Bearer ${apiKey}` },
|
|
161
161
|
signal: AbortSignal.timeout(Math.min(timeoutMs ?? 15000, 15000)),
|
|
162
|
-
});
|
|
162
|
+
}, proxy);
|
|
163
163
|
if (response.ok) {
|
|
164
164
|
return { provider: "openai", model, type, status: "ok" };
|
|
165
165
|
}
|
|
@@ -196,13 +196,13 @@ async function checkCohereEmbed(baseUrl, model, apiKey, timeoutMs, proxy) {
|
|
|
196
196
|
}
|
|
197
197
|
// ── Anthropic check ────────────────────────────────────────────
|
|
198
198
|
/** Check Anthropic chat endpoint by sending a minimal message. */
|
|
199
|
-
async function checkAnthropicChat(baseUrl, model, apiKey, timeoutMs, type = "description") {
|
|
199
|
+
async function checkAnthropicChat(baseUrl, model, apiKey, timeoutMs, proxy, type = "description") {
|
|
200
200
|
if (!apiKey) {
|
|
201
201
|
return { provider: "anthropic", model, type, status: "error", error: "No API key configured" };
|
|
202
202
|
}
|
|
203
203
|
const url = `${baseUrl.replace(/\/+$/, "")}/messages`;
|
|
204
204
|
try {
|
|
205
|
-
const response = await
|
|
205
|
+
const response = await fetchWithProxy(url, {
|
|
206
206
|
method: "POST",
|
|
207
207
|
headers: {
|
|
208
208
|
"Content-Type": "application/json",
|
|
@@ -215,7 +215,7 @@ async function checkAnthropicChat(baseUrl, model, apiKey, timeoutMs, type = "des
|
|
|
215
215
|
messages: [{ role: "user", content: "hi" }],
|
|
216
216
|
}),
|
|
217
217
|
signal: AbortSignal.timeout(Math.min(timeoutMs ?? 15000, 15000)),
|
|
218
|
-
});
|
|
218
|
+
}, proxy);
|
|
219
219
|
if (response.ok) {
|
|
220
220
|
return { provider: "anthropic", model, type, status: "ok" };
|
|
221
221
|
}
|
|
@@ -232,20 +232,20 @@ async function checkAnthropicChat(baseUrl, model, apiKey, timeoutMs, type = "des
|
|
|
232
232
|
}
|
|
233
233
|
// ── Google Gemini check ────────────────────────────────────────
|
|
234
234
|
/** Check Google Gemini chat endpoint by sending a minimal generateContent request. */
|
|
235
|
-
async function checkGoogleChat(baseUrl, model, apiKey, timeoutMs, type = "description") {
|
|
235
|
+
async function checkGoogleChat(baseUrl, model, apiKey, timeoutMs, proxy, type = "description") {
|
|
236
236
|
if (!apiKey) {
|
|
237
237
|
return { provider: "google", model, type, status: "error", error: "No API key configured" };
|
|
238
238
|
}
|
|
239
|
-
const url = `${baseUrl.replace(/\/+$/, "")}/models/${model}:generateContent
|
|
239
|
+
const url = `${baseUrl.replace(/\/+$/, "")}/models/${model}:generateContent`;
|
|
240
240
|
try {
|
|
241
|
-
const response = await
|
|
241
|
+
const response = await fetchWithProxy(url, {
|
|
242
242
|
method: "POST",
|
|
243
|
-
headers: { "Content-Type": "application/json" },
|
|
243
|
+
headers: { "Content-Type": "application/json", "x-goog-api-key": apiKey },
|
|
244
244
|
body: JSON.stringify({
|
|
245
245
|
contents: [{ parts: [{ text: "hi" }] }],
|
|
246
246
|
}),
|
|
247
247
|
signal: AbortSignal.timeout(Math.min(timeoutMs ?? 15000, 15000)),
|
|
248
|
-
});
|
|
248
|
+
}, proxy);
|
|
249
249
|
if (response.ok) {
|
|
250
250
|
return { provider: "google", model, type, status: "ok" };
|
|
251
251
|
}
|
package/dist/embedder/http.d.ts
CHANGED
|
@@ -12,6 +12,19 @@ export interface HttpResponseLike {
|
|
|
12
12
|
/** Destroy all pooled TCP/TLS sockets and clear the connection pool. */
|
|
13
13
|
declare function destroyAllPooledConnections(): void;
|
|
14
14
|
export { destroyAllPooledConnections };
|
|
15
|
+
/**
|
|
16
|
+
* Fetch a URL through the configured proxy, applying the proxy to
|
|
17
|
+
* HTTP(S)_PROXY env vars for the duration of the request (serialized).
|
|
18
|
+
*
|
|
19
|
+
* Used by health checks and other one-off requests that do not go
|
|
20
|
+
* through {@link postJson}.
|
|
21
|
+
*/
|
|
22
|
+
export declare function fetchWithProxy(urlString: string, init: {
|
|
23
|
+
method?: string;
|
|
24
|
+
headers?: Record<string, string>;
|
|
25
|
+
body?: string;
|
|
26
|
+
signal?: AbortSignal;
|
|
27
|
+
}, proxy?: ProxyConfig): Promise<Response>;
|
|
15
28
|
/**
|
|
16
29
|
* Check whether a hostname refers to the local machine.
|
|
17
30
|
*
|
|
@@ -43,7 +56,7 @@ export declare function matchesNoProxy(hostname: string, noProxy?: string): bool
|
|
|
43
56
|
* @param redirectCount - Internal redirect counter (starts at 0)
|
|
44
57
|
* @returns A promise resolving to an HttpResponseLike
|
|
45
58
|
*/
|
|
46
|
-
export declare function directRequest(url: URL, body: unknown, headers: Record<string, string>, timeoutMs: number, redirectCount?: number): Promise<HttpResponseLike>;
|
|
59
|
+
export declare function directRequest(url: URL, body: unknown, headers: Record<string, string>, timeoutMs: number, redirectCount?: number, signal?: AbortSignal): Promise<HttpResponseLike>;
|
|
47
60
|
/**
|
|
48
61
|
* POST JSON to a URL, optionally routing through a proxy.
|
|
49
62
|
*
|