devsplain 2.3.1 → 2.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/bin/cli.js +128 -11
- package/lib/config.js +81 -3
- package/lib/llm.js +174 -27
- package/package.json +1 -1
package/lib/llm.js
CHANGED
|
@@ -1,15 +1,14 @@
|
|
|
1
1
|
/** [ds]
|
|
2
|
-
*
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
* @returns {Promise<Response>} - Returns the successful Response object.
|
|
8
|
-
* @throws {Error} Throws a RateLimitError (with .isRateLimit=true) on 429, or the last error after exhausting retries.
|
|
2
|
+
* Fetches a URL with automatic retry using exponential backoff.
|
|
3
|
+
* Rate limit (429) errors are re-thrown immediately so callers can apply
|
|
4
|
+
* higher-level throttling; 4xx client errors are returned as-is (no retry),
|
|
5
|
+
* while 5xx server errors and network/timeout failures are retried up to
|
|
6
|
+
* maxRetries times. A 45s AbortController timeout guards each attempt.
|
|
9
7
|
*/
|
|
10
8
|
async function fetchWithRetry(url, options, maxRetries = 3, initialDelay = 1000) {
|
|
11
9
|
let lastError;
|
|
12
10
|
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
11
|
+
// Fresh AbortController per attempt so a prior timeout does not poison retries [ds]
|
|
13
12
|
const controller = new AbortController();
|
|
14
13
|
const timeoutId = setTimeout(() => controller.abort(), 45000);
|
|
15
14
|
try {
|
|
@@ -22,14 +21,25 @@ async function fetchWithRetry(url, options, maxRetries = 3, initialDelay = 1000)
|
|
|
22
21
|
throw new Error("No response received from fetch");
|
|
23
22
|
}
|
|
24
23
|
if (response.ok) {
|
|
24
|
+
// Opportunistically harvest provider-advertised token limits from response headers to enable dynamic tier scaling [ds]
|
|
25
|
+
if (response.headers && typeof response.headers.get === 'function') {
|
|
26
|
+
const limitTokensHeader = response.headers.get('x-ratelimit-limit-tokens');
|
|
27
|
+
if (limitTokensHeader) {
|
|
28
|
+
const parsedLimit = parseInt(limitTokensHeader, 10);
|
|
29
|
+
if (!isNaN(parsedLimit) && parsedLimit > 0) {
|
|
30
|
+
_discoveredTierLimit = parsedLimit;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
}
|
|
25
34
|
return response;
|
|
26
35
|
}
|
|
27
|
-
//
|
|
36
|
+
// 429 is propagated to the caller (not retried here) so the concurrency controller can globally throttle [ds]
|
|
28
37
|
if (response.status === 429) {
|
|
29
38
|
const err = new Error(`HTTP Error 429: ${response.statusText || 'Too Many Requests'}`);
|
|
30
39
|
err.isRateLimit = true;
|
|
31
40
|
throw err;
|
|
32
41
|
}
|
|
42
|
+
// Retry only transient server errors; other 4xx responses are terminal and returned to caller [ds]
|
|
33
43
|
if (response.status >= 500 && response.status < 600) {
|
|
34
44
|
lastError = new Error(`HTTP Error ${response.status}: ${response.statusText}`);
|
|
35
45
|
} else {
|
|
@@ -37,7 +47,6 @@ async function fetchWithRetry(url, options, maxRetries = 3, initialDelay = 1000)
|
|
|
37
47
|
}
|
|
38
48
|
} catch (err) {
|
|
39
49
|
clearTimeout(timeoutId);
|
|
40
|
-
// Propagate rate limit errors immediately without retrying [ds]
|
|
41
50
|
if (err.isRateLimit) {
|
|
42
51
|
throw err;
|
|
43
52
|
}
|
|
@@ -48,6 +57,7 @@ async function fetchWithRetry(url, options, maxRetries = 3, initialDelay = 1000)
|
|
|
48
57
|
}
|
|
49
58
|
}
|
|
50
59
|
|
|
60
|
+
// Exponential backoff (initialDelay * 2^attempt); skip delay after the final attempt [ds]
|
|
51
61
|
if (attempt < maxRetries - 1) {
|
|
52
62
|
const backoffDelay = initialDelay * Math.pow(2, attempt);
|
|
53
63
|
console.warn(`[devsplain] AI request failed. Retrying in ${backoffDelay}ms... (Attempt ${attempt + 1}/${maxRetries})`);
|
|
@@ -57,13 +67,73 @@ async function fetchWithRetry(url, options, maxRetries = 3, initialDelay = 1000)
|
|
|
57
67
|
throw lastError;
|
|
58
68
|
}
|
|
59
69
|
|
|
60
|
-
// ─── Chunking
|
|
70
|
+
// ─── Provider Profiles for Model-Adaptive Chunking ────────────────────────────
|
|
71
|
+
// Baseline free-tier safe defaults: preserves token limits for free/low tiers,
|
|
72
|
+
// while expanding window sizes for high-capacity providers (DeepSeek, Gemini, Claude).
|
|
73
|
+
/** [ds]
|
|
74
|
+
* Per-provider chunking profiles tuned to each API's safe free-tier limits.
|
|
75
|
+
* Size/overlap are measured in lines; threshold triggers a fresh chunk;
|
|
76
|
+
* maxTokens caps the model context window for each provider.
|
|
77
|
+
*/
|
|
78
|
+
const PROVIDER_PROFILES = {
|
|
79
|
+
groq: { size: 200, overlap: 20, threshold: 250, maxTokens: 1000 },
|
|
80
|
+
deepseek: { size: 600, overlap: 50, threshold: 750, maxTokens: 8192 },
|
|
81
|
+
gemini: { size: 800, overlap: 60, threshold: 1000, maxTokens: 8192 },
|
|
82
|
+
claude: { size: 600, overlap: 50, threshold: 750, maxTokens: 8192 },
|
|
83
|
+
openai: { size: 600, overlap: 50, threshold: 750, maxTokens: 4096 },
|
|
84
|
+
default: { size: 250, overlap: 25, threshold: 300, maxTokens: 4096 }
|
|
85
|
+
};
|
|
86
|
+
|
|
87
|
+
// Tracks discovered rate limits from response headers (e.g. x-ratelimit-limit-tokens)
|
|
88
|
+
let _discoveredTierLimit = null;
|
|
89
|
+
|
|
90
|
+
// Baseline constants exported for backward compatibility
|
|
61
91
|
const CHUNK_SIZE = 200;
|
|
62
92
|
const CHUNK_OVERLAP = 20;
|
|
63
93
|
const CHUNK_THRESHOLD = 250;
|
|
64
94
|
|
|
95
|
+
/**
|
|
96
|
+
* Returns the effective chunking parameters for a provider.
|
|
97
|
+
* Allows override via config (e.g., config.chunkSize from --chunk-size flag)
|
|
98
|
+
* or scaled up dynamically if response headers reveal high tier capacity.
|
|
99
|
+
* @param {object} config - Provider configuration object.
|
|
100
|
+
* @returns {object} { size, overlap, threshold, maxTokens }
|
|
101
|
+
*/
|
|
102
|
+
/** [ds]
|
|
103
|
+
* Resolves effective chunk parameters for a provider, honoring explicit
|
|
104
|
+
* config overrides. When response headers previously revealed a high-tier
|
|
105
|
+
* limit (>100k TPM), chunk sizes are scaled up 1.5x (capped) unless the
|
|
106
|
+
* user explicitly pinned chunkSize.
|
|
107
|
+
*/
|
|
108
|
+
function getChunkConfig(config = {}) {
|
|
109
|
+
const provider = (config.provider || '').toLowerCase();
|
|
110
|
+
const base = PROVIDER_PROFILES[provider] || PROVIDER_PROFILES.default;
|
|
111
|
+
|
|
112
|
+
let size = config.chunkSize || base.size;
|
|
113
|
+
let overlap = config.chunkOverlap || base.overlap;
|
|
114
|
+
let threshold = config.chunkThreshold || base.threshold;
|
|
115
|
+
const maxTokens = base.maxTokens;
|
|
116
|
+
|
|
117
|
+
// Dynamic Tier Scaling: if response headers revealed a high-tier token limit (>100k TPM),
|
|
118
|
+
// scale up chunk size safely by 1.5x up to 1000 lines max
|
|
119
|
+
// Only auto-scale when the user hasn't manually overridden the chunk size [ds]
|
|
120
|
+
if (!config.chunkSize && _discoveredTierLimit && _discoveredTierLimit > 100000) {
|
|
121
|
+
size = Math.min(1000, Math.round(size * 1.5));
|
|
122
|
+
threshold = Math.min(1200, Math.round(threshold * 1.5));
|
|
123
|
+
overlap = Math.min(100, Math.round(overlap * 1.5));
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
return { size, overlap, threshold, maxTokens };
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
function setDiscoveredTierLimit(limit) {
|
|
130
|
+
_discoveredTierLimit = limit;
|
|
131
|
+
}
|
|
132
|
+
|
|
65
133
|
// ─── Adaptive Concurrency Controller ──────────────────────────────────────────
|
|
66
|
-
//
|
|
134
|
+
// Module-level adaptive state: shared across all runWithConcurrency invocations
|
|
135
|
+
// within a single run so a 429 on one task throttles all subsequent tasks.
|
|
136
|
+
// Shared adaptive state: a 429 anywhere flips the whole run into serial mode [ds]
|
|
67
137
|
let _concurrencyLimit = 2;
|
|
68
138
|
let _hitRateLimit = false;
|
|
69
139
|
|
|
@@ -74,6 +144,7 @@ let _hitRateLimit = false;
|
|
|
74
144
|
function resetConcurrency(initialLimit = 2) {
|
|
75
145
|
_concurrencyLimit = initialLimit;
|
|
76
146
|
_hitRateLimit = false;
|
|
147
|
+
_discoveredTierLimit = null;
|
|
77
148
|
}
|
|
78
149
|
|
|
79
150
|
/**
|
|
@@ -84,6 +155,12 @@ function resetConcurrency(initialLimit = 2) {
|
|
|
84
155
|
* @param {Function} taskFn - Async function to run per item.
|
|
85
156
|
* @returns {Promise<Array>} - Resolved results in order.
|
|
86
157
|
*/
|
|
158
|
+
/** [ds]
|
|
159
|
+
* Runs async tasks through a bounded pool. On a 429 RateLimitError, the
|
|
160
|
+
* controller permanently reduces concurrency to 1 for the rest of the run
|
|
161
|
+
* and retries the failed task with exponential backoff (jittered).
|
|
162
|
+
* Preserves input ordering in the returned results array.
|
|
163
|
+
*/
|
|
87
164
|
async function runWithConcurrency(items, taskFn) {
|
|
88
165
|
const results = [];
|
|
89
166
|
const executing = new Set();
|
|
@@ -91,10 +168,12 @@ async function runWithConcurrency(items, taskFn) {
|
|
|
91
168
|
for (let i = 0; i < items.length; i++) {
|
|
92
169
|
const item = items[i];
|
|
93
170
|
|
|
171
|
+
// Backpressure: wait until a slot frees up before launching the next task [ds]
|
|
94
172
|
while (executing.size >= _concurrencyLimit) {
|
|
95
173
|
await Promise.race(executing);
|
|
96
174
|
}
|
|
97
175
|
|
|
176
|
+
// Self-retry loop wraps each task so a single 429 doesn't abort the whole batch [ds]
|
|
98
177
|
const task = (async () => {
|
|
99
178
|
const maxRetries = 3;
|
|
100
179
|
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
@@ -104,9 +183,11 @@ async function runWithConcurrency(items, taskFn) {
|
|
|
104
183
|
if (err.isRateLimit && attempt < maxRetries - 1) {
|
|
105
184
|
if (!_hitRateLimit) {
|
|
106
185
|
_hitRateLimit = true;
|
|
186
|
+
// Global throttle: first 429 in the run drops concurrency to serial for all remaining tasks [ds]
|
|
107
187
|
_concurrencyLimit = 1;
|
|
108
188
|
console.warn(`[devsplain] Rate limit hit — switching to serial mode.`);
|
|
109
189
|
}
|
|
190
|
+
// Use a short backoff under test to keep suites fast; add jitter to avoid thundering herd [ds]
|
|
110
191
|
const backoff = (process.env.NODE_ENV === 'test' ? 50 : 2000) * Math.pow(2, attempt) + Math.random() * 500;
|
|
111
192
|
console.warn(`[devsplain] Backing off for ${Math.round(backoff)}ms...`);
|
|
112
193
|
await new Promise(r => setTimeout(r, backoff));
|
|
@@ -118,9 +199,11 @@ async function runWithConcurrency(items, taskFn) {
|
|
|
118
199
|
})();
|
|
119
200
|
|
|
120
201
|
let tracked;
|
|
202
|
+
// Attach the removal handler to the tracked promise so the slot frees once the task settles [ds]
|
|
121
203
|
tracked = task.finally(() => {
|
|
122
204
|
executing.delete(tracked);
|
|
123
205
|
});
|
|
206
|
+
// Swallow rejection on the tracked promise to avoid unhandled rejection warnings; callers see it via results [ds]
|
|
124
207
|
tracked.catch(() => {});
|
|
125
208
|
executing.add(tracked);
|
|
126
209
|
results.push(tracked);
|
|
@@ -137,14 +220,22 @@ async function runWithConcurrency(items, taskFn) {
|
|
|
137
220
|
* @param {string} mode - Documentation mode ('default', 'light', 'full').
|
|
138
221
|
* @returns {string} The assembled prompt string.
|
|
139
222
|
*/
|
|
140
|
-
|
|
223
|
+
/** [ds]
|
|
224
|
+
* Assembles a mode-aware prompt instructing the LLM to emit strictly a JSON
|
|
225
|
+
* array of { line, comment } objects, using language-appropriate comment
|
|
226
|
+
* tokens. Includes anti-triviality rules and optional context to disambiguate.
|
|
227
|
+
*/
|
|
228
|
+
function buildPrompt(numberedCode, language, mode, contextSnippet = '') {
|
|
229
|
+
// Derive language from file extension since callers may pass a path or a bare identifier. [ds]
|
|
141
230
|
const extMatch = language.match(/\.[0-9a-z]+$/i);
|
|
142
231
|
const ext = extMatch ? extMatch[0].toLowerCase() : '';
|
|
143
232
|
const isPython = ext === '.py';
|
|
233
|
+
// Ruby and shell both share the '#' comment syntax [ds]
|
|
144
234
|
const isRubyOrShell = ['.rb', '.sh'].includes(ext);
|
|
145
235
|
const isHTML = ['.html', '.vue', '.svelte'].includes(ext);
|
|
146
236
|
const isCss = ['.css', '.scss'].includes(ext);
|
|
147
237
|
const isSql = ext === '.sql';
|
|
238
|
+
// Default comment syntax assumes C-style languages; overridden per detected extension below. [ds]
|
|
148
239
|
let singleLineToken = '//';
|
|
149
240
|
let blockExample = '/** Calculates the total price */';
|
|
150
241
|
let inlineExample = '// Check for null values';
|
|
@@ -167,6 +258,7 @@ function buildPrompt(numberedCode, language, mode) {
|
|
|
167
258
|
inlineExample = '-- Check for null values';
|
|
168
259
|
}
|
|
169
260
|
|
|
261
|
+
// 'light' and 'full' modes reuse the same scaffolding but swap the core instruction string. [ds]
|
|
170
262
|
let instruction = `Provide block comments above functions and sparse inline comments for complex logic.`;
|
|
171
263
|
if (mode === 'light') {
|
|
172
264
|
instruction = `Provide ONLY block comments above functions. Keep it minimal.`;
|
|
@@ -174,12 +266,13 @@ function buildPrompt(numberedCode, language, mode) {
|
|
|
174
266
|
instruction = `Provide highly detailed block comments above functions, and exhaustive step-by-step inline comments explaining every conditional branch, loop, variable assignment, and logical block inside function bodies. Do not be sparse; explain the code's execution flow in detail.`;
|
|
175
267
|
}
|
|
176
268
|
|
|
269
|
+
// Rule 5 must be language-aware: CSS forbids //, while most other languages forbid # or <!-- as a primary comment marker. [ds]
|
|
177
270
|
let rule5 = `5. IMPORTANT: Use ONLY ${singleLineToken} for comments. DO NOT use docstrings or multi-line string literals like """ or ''' for comments.`;
|
|
271
|
+
// SCSS technically allows //, but we force /* */ to keep the prompt safe for plain CSS output. [ds]
|
|
178
272
|
if (isCss) {
|
|
179
273
|
rule5 = `5. IMPORTANT: In CSS/SCSS, you MUST use /* ... */ for comments. DO NOT use // comments under any circumstances.`;
|
|
180
274
|
}
|
|
181
275
|
|
|
182
|
-
// Anti-triviality negative constraints to eliminate syntax-narrating clutter [ds]
|
|
183
276
|
const antiTrivialityRules = `
|
|
184
277
|
ANTI-TRIVIALITY RULES (STRICTLY ENFORCED):
|
|
185
278
|
6. NEVER write comments that merely narrate the syntax (e.g. NEVER write "// Loop over items" above a for loop, "// Return result" above a return, "// Increment i" above i++, or "// Define variable" above a declaration).
|
|
@@ -190,6 +283,12 @@ ANTI-TRIVIALITY RULES (STRICTLY ENFORCED):
|
|
|
190
283
|
- A tricky formula, index manipulation (e.g. 0-indexed vs 1-indexed), or protocol-specific behavior occurs.
|
|
191
284
|
9. Prefer comprehensive function-level block comments over cluttered inline comments. Quality over quantity.`;
|
|
192
285
|
|
|
286
|
+
// Context lines are included for semantic grounding only; the model is instructed not to emit comments for them. [ds]
|
|
287
|
+
const contextBlock = contextSnippet ? `
|
|
288
|
+
Top-of-file context (for semantic reference only; DO NOT generate comments for these reference lines):
|
|
289
|
+
${contextSnippet}
|
|
290
|
+
` : '';
|
|
291
|
+
|
|
193
292
|
const prompt = `
|
|
194
293
|
You are a code documentation engine. Analyze the following ${language} code which has line numbers prepended to it.
|
|
195
294
|
${instruction}
|
|
@@ -201,7 +300,7 @@ CRITICAL RULES:
|
|
|
201
300
|
4. If no comments are needed, return an empty array: [].
|
|
202
301
|
${rule5}
|
|
203
302
|
${antiTrivialityRules}
|
|
204
|
-
|
|
303
|
+
${contextBlock}
|
|
205
304
|
Example Output:
|
|
206
305
|
[
|
|
207
306
|
{ "line": 4, "comment": "${blockExample}" },
|
|
@@ -223,10 +322,16 @@ ${numberedCode}
|
|
|
223
322
|
* @param {object} config - Provider config (provider, model, apiKey, baseUrl).
|
|
224
323
|
* @returns {Promise<string>} Raw text response from the AI.
|
|
225
324
|
*/
|
|
325
|
+
/** [ds]
|
|
326
|
+
* Dispatches the prompt to the active provider (Gemini, Claude, or OpenAI-compatible)
|
|
327
|
+
* and normalizes the provider-specific response envelope into a single string.
|
|
328
|
+
* Rate-limit errors are re-thrown unmodified so callers can apply backoff.
|
|
329
|
+
*/
|
|
226
330
|
async function fetchFromProvider(prompt, config) {
|
|
227
331
|
let textResponse = "";
|
|
228
332
|
|
|
229
333
|
if (config.provider === 'gemini') {
|
|
334
|
+
// Gemini is the only provider that takes the API key in the query string rather than a header. [ds]
|
|
230
335
|
const url = `https://generativelanguage.googleapis.com/v1beta/models/${config.model}:generateContent?key=${config.apiKey}`;
|
|
231
336
|
let data;
|
|
232
337
|
try {
|
|
@@ -241,20 +346,24 @@ async function fetchFromProvider(prompt, config) {
|
|
|
241
346
|
});
|
|
242
347
|
data = await response.json();
|
|
243
348
|
} catch (error) {
|
|
244
|
-
//
|
|
349
|
+
// Preserve rate-limit errors so upstream retry logic can detect and handle them. [ds]
|
|
245
350
|
if (error.isRateLimit) throw error;
|
|
246
351
|
throw new Error(`AI Provider Request Failed: ${error.message}`);
|
|
247
352
|
}
|
|
353
|
+
// Gemini may return errors either as a structured object or a raw string; normalize both. [ds]
|
|
248
354
|
if (data.error) {
|
|
249
355
|
const msg = data.error.message || (typeof data.error === 'string' ? data.error : JSON.stringify(data.error));
|
|
250
356
|
throw new Error(`API Error: ${msg}`);
|
|
251
357
|
}
|
|
358
|
+
// Defensive deep-path check: Gemini can return 200 with empty candidates (e.g. safety blocks). [ds]
|
|
252
359
|
if (!data.candidates || !data.candidates[0] || !data.candidates[0].content || !data.candidates[0].content.parts || !data.candidates[0].content.parts[0]) {
|
|
360
|
+
// finishReason distinguishes safety blocks from genuinely empty responses for better diagnostics. [ds]
|
|
253
361
|
const reason = data.candidates?.[0]?.finishReason || 'Unknown error';
|
|
254
362
|
throw new Error(`AI Provider returned no content (finish reason: ${reason})`);
|
|
255
363
|
}
|
|
256
364
|
textResponse = data.candidates[0].content.parts[0].text;
|
|
257
365
|
} else if (config.provider === 'claude') {
|
|
366
|
+
// Anthropic endpoint and API version are pinned to the stable Messages API. [ds]
|
|
258
367
|
const url = `${config.baseUrl}/v1/messages`;
|
|
259
368
|
let data;
|
|
260
369
|
try {
|
|
@@ -267,6 +376,7 @@ async function fetchFromProvider(prompt, config) {
|
|
|
267
376
|
},
|
|
268
377
|
body: JSON.stringify({
|
|
269
378
|
"model": config.model,
|
|
379
|
+
// Anthropic requires an explicit max_tokens; 8192 covers large multi-function chunks. [ds]
|
|
270
380
|
"max_tokens": 8192,
|
|
271
381
|
"messages": [{
|
|
272
382
|
"role": "user",
|
|
@@ -289,6 +399,7 @@ async function fetchFromProvider(prompt, config) {
|
|
|
289
399
|
textResponse = data.content[0].text;
|
|
290
400
|
}
|
|
291
401
|
else {
|
|
402
|
+
// Fallback path assumes OpenAI-compatible /chat/completions schema (OpenAI, Groq, Ollama, etc.). [ds]
|
|
292
403
|
const url = `${config.baseUrl}/v1/chat/completions`;
|
|
293
404
|
let data;
|
|
294
405
|
|
|
@@ -299,11 +410,9 @@ async function fetchFromProvider(prompt, config) {
|
|
|
299
410
|
"content": prompt
|
|
300
411
|
}]
|
|
301
412
|
};
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
reqBody.max_tokens = 8192;
|
|
306
|
-
}
|
|
413
|
+
// Provider-specific token limits vary; getChunkConfig caps output to avoid truncation mid-JSON. [ds]
|
|
414
|
+
const chunkConfig = getChunkConfig(config);
|
|
415
|
+
reqBody.max_tokens = chunkConfig.maxTokens;
|
|
307
416
|
|
|
308
417
|
try {
|
|
309
418
|
const response = await fetchWithRetry(url, {
|
|
@@ -323,6 +432,7 @@ async function fetchFromProvider(prompt, config) {
|
|
|
323
432
|
const msg = data.error.message || (typeof data.error === 'string' ? data.error : JSON.stringify(data.error));
|
|
324
433
|
throw new Error(`API Error: ${msg}`);
|
|
325
434
|
}
|
|
435
|
+
// Verify the nested response shape before accessing content, since some providers return null choices. [ds]
|
|
326
436
|
if (!data.choices || !data.choices[0] || !data.choices[0].message || typeof data.choices[0].message.content !== 'string') {
|
|
327
437
|
throw new Error(`AI Provider returned an unexpected response structure: ${JSON.stringify(data)}`);
|
|
328
438
|
}
|
|
@@ -340,14 +450,24 @@ async function fetchFromProvider(prompt, config) {
|
|
|
340
450
|
* @param {string} mode - Documentation mode.
|
|
341
451
|
* @returns {Array} Validated array of comment objects.
|
|
342
452
|
*/
|
|
453
|
+
/** [ds]
|
|
454
|
+
* Parses raw LLM output into a validated comment array. LLMs frequently wrap JSON in prose or
|
|
455
|
+
* markdown fences, so we defensively locate the outermost '[' ... ']' substring before parsing.
|
|
456
|
+
* If the closing ']' is missing (truncated output), we attempt to repair by appending ']' after
|
|
457
|
+
* the last '}' — a heuristic that recovers the common case of a cut-off JSON array.
|
|
458
|
+
* Also enforces that every returned comment is actually comment-shaped (starts with a known
|
|
459
|
+
* comment marker), preventing prompt-injection payloads from being injected as executable code.
|
|
460
|
+
*/
|
|
343
461
|
function parseAndValidate(textResponse, mode) {
|
|
344
462
|
let cleanText = textResponse.trim();
|
|
463
|
+
// Heuristic extraction: bracketed substring allows recovery even when LLM prefixes/suffixes with prose [ds]
|
|
345
464
|
const start = cleanText.indexOf('[');
|
|
346
465
|
const end = cleanText.lastIndexOf(']');
|
|
347
466
|
if (start !== -1) {
|
|
348
467
|
if (end !== -1 && end >= start) {
|
|
349
468
|
cleanText = cleanText.substring(start, end + 1);
|
|
350
469
|
} else {
|
|
470
|
+
// Truncated-array repair: closing bracket was cut off mid-stream; synthesize one after the final object [ds]
|
|
351
471
|
const lastBrace = cleanText.lastIndexOf('}');
|
|
352
472
|
if (lastBrace > start) {
|
|
353
473
|
cleanText = cleanText.substring(start, lastBrace + 1) + ']';
|
|
@@ -384,6 +504,7 @@ function parseAndValidate(textResponse, mode) {
|
|
|
384
504
|
}
|
|
385
505
|
|
|
386
506
|
const trimmedComment = item.comment.trim();
|
|
507
|
+
// Multi-line comment validation state machine: detect entering/exiting /* ... */ and <!-- ... --> blocks [ds]
|
|
387
508
|
const commentLines = trimmedComment.split(/\r?\n/);
|
|
388
509
|
let inBlock = false;
|
|
389
510
|
for (const cl of commentLines) {
|
|
@@ -395,6 +516,7 @@ function parseAndValidate(textResponse, mode) {
|
|
|
395
516
|
}
|
|
396
517
|
continue;
|
|
397
518
|
}
|
|
519
|
+
// Whitelist of comment-start markers; anything else is treated as an injection attempt [ds]
|
|
398
520
|
const startsWithMarker =
|
|
399
521
|
tcl.startsWith('//') ||
|
|
400
522
|
tcl.startsWith('/*') ||
|
|
@@ -405,6 +527,7 @@ function parseAndValidate(textResponse, mode) {
|
|
|
405
527
|
if (!startsWithMarker) {
|
|
406
528
|
throw new Error(`Security Error: Comment on line ${item.line} contains invalid non-comment line: "${tcl}"`);
|
|
407
529
|
}
|
|
530
|
+
// Block-comment openers without a same-line closer enter block mode; subsequent lines are exempt from marker checks [ds]
|
|
408
531
|
if ((tcl.startsWith('/*') && !tcl.includes('*/')) || (tcl.startsWith('<!--') && !tcl.includes('-->'))) {
|
|
409
532
|
inBlock = true;
|
|
410
533
|
}
|
|
@@ -428,10 +551,22 @@ function parseAndValidate(textResponse, mode) {
|
|
|
428
551
|
* @param {string} mode - Documentation mode ('default', 'light', 'full', 'clean').
|
|
429
552
|
* @returns {Promise<Array>} Array of validated comment objects with global line numbers.
|
|
430
553
|
*/
|
|
554
|
+
/** [ds]
|
|
555
|
+
* Entry point: obtains AI-generated comments for an entire source file.
|
|
556
|
+
* Files under CHUNK_THRESHOLD are sent in a single request. Larger files are split into
|
|
557
|
+
* overlapping windows (CHUNK_SIZE with CHUNK_OVERLAP) so that cross-boundary context is
|
|
558
|
+
* preserved at chunk seams. Each chunk is processed with global line numbers baked into the
|
|
559
|
+
* prompt, then results are filtered to the chunk's own range and deduplicated by line —
|
|
560
|
+
* the dedup step is required because overlap means a comment may be returned by two chunks.
|
|
561
|
+
*/
|
|
431
562
|
async function getComments(code, language, config, mode = 'default') {
|
|
432
563
|
const lines = code.split(/\r?\n/);
|
|
564
|
+
const { size: CHUNK_SIZE, overlap: CHUNK_OVERLAP, threshold: CHUNK_THRESHOLD } = getChunkConfig(config);
|
|
565
|
+
|
|
566
|
+
// Prepend the file header to every non-first chunk as additional context, capped at 25 lines to bound prompt size [ds]
|
|
567
|
+
const contextLinesCount = Math.min(25, lines.length);
|
|
568
|
+
const contextSnippet = lines.slice(0, contextLinesCount).map((line, i) => `${i + 1}: ${line}`).join('\n');
|
|
433
569
|
|
|
434
|
-
// Small files: single-shot processing (no chunking overhead) [ds]
|
|
435
570
|
if (lines.length <= CHUNK_THRESHOLD) {
|
|
436
571
|
const numberedCode = lines.map((line, i) => `${i + 1}: ${line}`).join('\n');
|
|
437
572
|
const prompt = buildPrompt(numberedCode, language, mode);
|
|
@@ -439,25 +574,27 @@ async function getComments(code, language, config, mode = 'default') {
|
|
|
439
574
|
return parseAndValidate(textResponse, mode);
|
|
440
575
|
}
|
|
441
576
|
|
|
442
|
-
// Large files: slice into overlapping chunks with global line numbers [ds]
|
|
443
577
|
const chunks = [];
|
|
578
|
+
// Stride is (size - overlap) so consecutive windows share CHUNK_OVERLAP lines at their boundaries [ds]
|
|
444
579
|
for (let start = 0; start < lines.length; start += (CHUNK_SIZE - CHUNK_OVERLAP)) {
|
|
445
580
|
const end = Math.min(start + CHUNK_SIZE, lines.length);
|
|
446
581
|
chunks.push({ start, end });
|
|
447
582
|
if (end >= lines.length) break;
|
|
448
583
|
}
|
|
449
584
|
|
|
450
|
-
// Process chunks through the adaptive concurrency pool [ds]
|
|
451
585
|
const chunkResults = await runWithConcurrency(chunks, async (chunk) => {
|
|
452
586
|
const chunkLines = lines.slice(chunk.start, chunk.end);
|
|
587
|
+
// Line numbers in the prompt are absolute (1-indexed) so returned comments need no offset translation [ds]
|
|
453
588
|
const startLineNum = chunk.start + 1;
|
|
454
589
|
const numberedCode = chunkLines.map((line, i) => `${startLineNum + i}: ${line}`).join('\n');
|
|
455
|
-
const
|
|
590
|
+
const contextForChunk = chunk.start > 0 ? contextSnippet : '';
|
|
591
|
+
const prompt = buildPrompt(numberedCode, language, mode, contextForChunk);
|
|
456
592
|
const textResponse = await fetchFromProvider(prompt, config);
|
|
457
|
-
|
|
593
|
+
const parsed = parseAndValidate(textResponse, mode);
|
|
594
|
+
// Discard hallucinated lines outside this chunk's range; overlap duplicates are resolved during dedup below [ds]
|
|
595
|
+
return parsed.filter(c => c.line >= startLineNum && c.line < startLineNum + chunkLines.length);
|
|
458
596
|
});
|
|
459
597
|
|
|
460
|
-
// Merge and deduplicate: first comment wins for overlapping line numbers [ds]
|
|
461
598
|
const seenLines = new Set();
|
|
462
599
|
const allComments = [];
|
|
463
600
|
for (const chunkComments of chunkResults) {
|
|
@@ -472,4 +609,14 @@ async function getComments(code, language, config, mode = 'default') {
|
|
|
472
609
|
return allComments;
|
|
473
610
|
}
|
|
474
611
|
|
|
475
|
-
module.exports = {
|
|
612
|
+
module.exports = {
|
|
613
|
+
getComments,
|
|
614
|
+
runWithConcurrency,
|
|
615
|
+
resetConcurrency,
|
|
616
|
+
getChunkConfig,
|
|
617
|
+
PROVIDER_PROFILES,
|
|
618
|
+
setDiscoveredTierLimit,
|
|
619
|
+
CHUNK_SIZE,
|
|
620
|
+
CHUNK_OVERLAP,
|
|
621
|
+
CHUNK_THRESHOLD
|
|
622
|
+
};
|
package/package.json
CHANGED