devsplain 2.2.4 → 2.3.0

package/bin/cli.js

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
 
-const { getComments } = require('../lib/llm.js');
+const { getComments, runWithConcurrency, resetConcurrency } = require('../lib/llm.js');
 const { getConfig } = require('../lib/config.js');
 const fs = require('fs');
 const path = require('path');
@@ -817,7 +817,7 @@ Options:
     };
 
     let filepath = '.';
-    const flagKeys = ['--provider', '--model', '--api-key', '--base-url'];
+    const flagKeys = ['--provider', '--model', '--api-key', '--base-url', '--concurrency'];
     for (let i = 0; i < args.length; i++) {
         const arg = args[i];
         if (arg.startsWith('--')) {
@@ -877,6 +877,10 @@ Options:
 
     const isOverwrite = (hasOverwriteFlag || config.autoPrune) && !hasKeepFlag;
 
+    // Parse --concurrency flag (default: 2, max: 5, min: 1) [ds]
+    const cliConcurrency = parseInt(getArgValue('--concurrency'), 10);
+    const concurrencyLevel = (cliConcurrency && cliConcurrency >= 1 && cliConcurrency <= 5) ? cliConcurrency : 2;
+
     let userIgnorePatterns = [];
     try {
         const ignorePath = path.join(process.cwd(), '.devsplainignore');
@@ -908,80 +912,95 @@ Options:
         return false;
     }
 
-    async function processPath(targetPath) {
+    const validExtensions = [
+        '.js', '.jsx', '.ts', '.tsx', '.html', '.css', '.scss', '.vue', '.svelte',
+        '.py', '.java', '.c', '.cpp', '.cs', '.go', '.rb', '.php', '.rs', 
+        '.swift', '.kt', '.dart', '.sh', '.sql'
+    ];
+
+    // Separate file discovery from processing for concurrency support [ds]
+    function collectFiles(targetPath) {
+        const collected = [];
         const stats = fs.statSync(targetPath);
 
-        if (isPathIgnored(targetPath)) {
-            return;
-        }
+        if (isPathIgnored(targetPath)) return collected;
 
         if (stats.isDirectory()) {
             console.log(`\n Scanning directory: ${targetPath}`);
             const items = fs.readdirSync(targetPath);
             for (const item of items) {
-                const fullPath = path.join(targetPath, item);
-                await processPath(fullPath); 
+                collected.push(...collectFiles(path.join(targetPath, item)));
             }
-        } 
-        else if (stats.isFile()) {
+        } else if (stats.isFile()) {
             const ext = path.extname(targetPath).toLowerCase();
-            const validExtensions = [
-                '.js', '.jsx', '.ts', '.tsx', '.html', '.css', '.scss', '.vue', '.svelte',
-                '.py', '.java', '.c', '.cpp', '.cs', '.go', '.rb', '.php', '.rs', 
-                '.swift', '.kt', '.dart', '.sh', '.sql'
-            ];
-
-            if (!validExtensions.includes(ext)) {
-                return;
-            }
+            if (!validExtensions.includes(ext)) return collected;
 
-            const filename = path.basename(targetPath);
             const data = fs.readFileSync(targetPath, 'utf-8');
             if (data.trim() === '') {
-                console.log(` Skipping ${filename} (Empty File)`);
-                return;
+                console.log(` Skipping ${path.basename(targetPath)} (Empty File)`);
+                return collected;
             }
+            collected.push(targetPath);
+        }
+        return collected;
+    }
 
-            console.log(` Analyzing ${filename} in ${mode} mode...`);
-            try {
-                let comments = [];
-                let commentedCode;
-                if (mode !== 'clean' && mode !== 'prune') {
-                    const preProcessMode = isOverwrite ? 'prune' : 'clean';
-                    const cleanData = spliceComments(data, [], preProcessMode, ext);
-                    comments = await getComments(cleanData, filename, config, mode);
-                    commentedCode = spliceComments(cleanData, comments, mode, ext);
-                } else {
-                    commentedCode = spliceComments(data, [], mode, ext);
-                }
-                if (isDryRun) {
-                    console.log(`\n --- DRY RUN PREVIEW: ${filename} ---`);
-                    console.log(commentedCode);
-                    console.log(`---------------------------------------\n`);
-                    const answer = await askQuestion("Type 'write' to save to file, or press any key to discard: ");
-                    if (answer.toLowerCase() === 'write') {
-                        const tempPath = targetPath + '.tmp';
-                        fs.writeFileSync(tempPath, commentedCode, 'utf8');
-                        fs.renameSync(tempPath, targetPath);
-                        console.log(` Successfully saved ${targetPath}`);
-                    } else {
-                        console.log(` Skipped ${targetPath}`);
-                    }
-                } else {
+    async function processSingleFile(targetPath) {
+        const filename = path.basename(targetPath);
+        const ext = path.extname(targetPath).toLowerCase();
+        const data = fs.readFileSync(targetPath, 'utf-8');
+
+        console.log(` Analyzing ${filename} in ${mode} mode...`);
+        try {
+            let comments = [];
+            let commentedCode;
+            if (mode !== 'clean' && mode !== 'prune') {
+                const preProcessMode = isOverwrite ? 'prune' : 'clean';
+                const cleanData = spliceComments(data, [], preProcessMode, ext);
+                comments = await getComments(cleanData, filename, config, mode);
+                commentedCode = spliceComments(cleanData, comments, mode, ext);
+            } else {
+                commentedCode = spliceComments(data, [], mode, ext);
+            }
+            if (isDryRun) {
+                console.log(`\n --- DRY RUN PREVIEW: ${filename} ---`);
+                console.log(commentedCode);
+                console.log(`---------------------------------------\n`);
+                const answer = await askQuestion("Type 'write' to save to file, or press any key to discard: ");
+                if (answer.toLowerCase() === 'write') {
                     const tempPath = targetPath + '.tmp';
                     fs.writeFileSync(tempPath, commentedCode, 'utf8');
                     fs.renameSync(tempPath, targetPath);
-                    console.log(` Successfully commented ${targetPath}`);
+                    console.log(` Successfully saved ${targetPath}`);
+                } else {
+                    console.log(` Skipped ${targetPath}`);
                 }
-                successCount++;
-            } catch (err) {
-                console.error(` Error processing ${filename}: ${err.message}`);
-                failCount++;
+            } else {
+                const tempPath = targetPath + '.tmp';
+                fs.writeFileSync(tempPath, commentedCode, 'utf8');
+                fs.renameSync(tempPath, targetPath);
+                console.log(` Successfully commented ${targetPath}`);
             }
+            successCount++;
+        } catch (err) {
+            console.error(` Error processing ${filename}: ${err.message}`);
+            failCount++;
         }
     }
 
-    await processPath(filepath);
+    // Collect all eligible files, then process with adaptive concurrency [ds]
+    const filesToProcess = collectFiles(filepath);
+
+    // Dry-run mode processes files serially to allow interactive prompts [ds]
+    if (isDryRun || mode === 'clean' || mode === 'prune') {
+        for (const file of filesToProcess) {
+            await processSingleFile(file);
+        }
+    } else {
+        resetConcurrency(concurrencyLevel);
+        console.log(`\n Processing ${filesToProcess.length} file(s) with concurrency: ${concurrencyLevel}`);
+        await runWithConcurrency(filesToProcess, processSingleFile);
+    }
 
     if (failCount > 0 && successCount === 0) {
         console.error("\nFailed: No files were successfully commented.");

package/lib/llm.js

@@ -1,432 +1,473 @@
-/** [ds]
- * Sends an HTTP request with automatic retry logic and timeout management.
- * @param {string} url - The target URL.
- * @param {object} options - Fetch options including method, headers, and body.
- * @param {number} maxRetries - Maximum number of retry attempts.
- * @param {number} initialDelay - Initial delay in milliseconds for exponential backoff.
- * @returns {Promise<Response>} - Returns the successful Response object.
-*/
-async function fetchWithRetry(url, options, maxRetries = 3, initialDelay = 1000) {
-    // Initialize a variable to store the last encountered error for potential re-throwing [ds]
-    let lastError;
-    // Enter a retry loop that executes up to 'maxRetries' times [ds]
-    for (let attempt = 0; attempt < maxRetries; attempt++) {
-        // Create an AbortController to manually trigger request timeouts [ds]
-        const controller = new AbortController();
-        // Schedule an abort action after 45 seconds to prevent hanging connections [ds]
-        const timeoutId = setTimeout(() => controller.abort(), 45000);
-        // Begin a try block to handle potential network or processing exceptions [ds]
-        try {
-            // Execute the fetch request, passing the URL and options with the abort signal [ds]
-            const response = await fetch(url, {
-                // Spread existing options into the fetch configuration [ds]
-                ...options,
-                // Attach the abort signal to allow external cancellation [ds]
-                signal: controller.signal
-            });
-            // Clear the timeout timer to clean up the abort signal since the request completed [ds]
-            clearTimeout(timeoutId);
-            // Validate that a response object was actually received [ds]
-            if (!response) {
-                // Throw a custom error if the response is missing or null [ds]
-                throw new Error("No response received from fetch");
-            }
-            // Check if the HTTP response indicates success (2xx status) [ds]
-            if (response.ok) {
-                // Return the response immediately upon successful HTTP completion [ds]
-                return response;
-            }
-            // Identify transient errors: 429 (Too Many Requests) or 5xx (Server Errors) [ds]
-            if (response.status === 429 || (response.status >= 500 && response.status < 600)) {
-                // Store the transient error to trigger a retry in the next loop iteration [ds]
-                lastError = new Error(`HTTP Error ${response.status}: ${response.statusText}`);
-            } else {
-                // For non-transient errors (4xx), return the response directly to let caller handle [ds]
-                return response;
-            }
-        // Catch any synchronous errors thrown during the await or setup [ds]
-        } catch (err) {
-            // Clear the timeout timer to prevent memory leaks on failure [ds]
-            clearTimeout(timeoutId);
-            // Check if the error was caused by the manual timeout abort [ds]
-            if (err.name === 'AbortError') {
-                // Convert the AbortError into a descriptive timeout message [ds]
-                lastError = new Error("Request timed out after 45 seconds");
-            } else {
-                // Store any other standard errors for potential retry logic [ds]
-                lastError = err;
-            }
-        }
-
-        // Check if more retry attempts remain before the final try [ds]
-        if (attempt < maxRetries - 1) {
-            // Calculate exponential backoff delay: base delay multiplied by 2 to the power of attempts [ds]
-            const backoffDelay = initialDelay * Math.pow(2, attempt);
-            // Log a warning to the console indicating a retry is occurring [ds]
-            console.warn(`[devsplain] AI request failed. Retrying in ${backoffDelay}ms... (Attempt ${attempt + 1}/${maxRetries})`);
-            // Pause execution for the calculated backoff duration before retrying [ds]
-            await new Promise(resolve => setTimeout(resolve, backoffDelay));
-        }
-    }
-    // Re-throw the last error if all retry attempts have been exhausted [ds]
-    throw lastError;
-}
-
-/** [ds]
- * Orchestrate code documentation by calling various AI providers.
- * Handles prompt construction, provider-specific API calls, and response parsing.
- * @param {string} code - The source code to analyze.
- * @param {string} language - The programming language and file extension.
- * @param {object} config - Provider configuration details.
- * @param {string} mode - Documentation density mode ('default', 'light', 'full', 'clean').
- * @returns {Promise<Array>} - Array of comment insertion objects.
-*/
-async function getComments(code, language, config, mode = 'default') {
-    // Split source code into an array of lines, handling Windows and Unix newline styles [ds]
-    const lines = code.split(/\r?\n/);
-    // Prepend line numbers to each line to help the AI map comments to specific locations [ds]
-    const numberedCode = lines.map((line, index) => `${index + 1}: ${line}`).join('\n');
-
-    // Extract the file extension using a regular expression [ds]
-    const extMatch = language.match(/\.[0-9a-z]+$/i);
-    // Normalize the extension to lowercase or default to empty string if missing [ds]
-    const ext = extMatch ? extMatch[0].toLowerCase() : '';
-    // Determine if the language is Python [ds]
-    const isPython = ext === '.py';
-    // Determine if the language is Ruby or Shell Script [ds]
-    const isRubyOrShell = ['.rb', '.sh'].includes(ext);
-    // Determine if the language is HTML-based (HTML, Vue, Svelte) [ds]
-    const isHTML = ['.html', '.vue', '.svelte'].includes(ext);
-    // Determine if the language is CSS or SCSS [ds]
-    const isCss = ['.css', '.scss'].includes(ext);
-    // Determine if the language is SQL [ds]
-    const isSql = ext === '.sql';
-    // Initialize comment syntax tokens for standard C-like languages [ds]
-    let singleLineToken = '//';
-    let blockExample = '/** Calculates the total price */';
-    let inlineExample = '// Check for null values';
-
-    // Adjust comment syntax for Python and Shell-based languages [ds]
-    if (isPython || isRubyOrShell) {
-        singleLineToken = '#';
-        blockExample = '# Calculates the total price';
-        inlineExample = '# Check for null values';
-    // Adjust comment syntax for HTML template languages [ds]
-    } else if (isHTML) {
-        singleLineToken = '<!--';
-        blockExample = '<!-- Calculates the total price -->';
-        inlineExample = '<!-- Check for null values -->';
-    // Adjust comment syntax for CSS stylesheets [ds]
-    } else if (isCss) {
-        singleLineToken = '/*';
-        blockExample = '/* Calculates the total price */';
-        inlineExample = '/* Check for null values */';
-    // Adjust comment syntax for SQL queries [ds]
-    } else if (isSql) {
-        singleLineToken = '--';
-        blockExample = '-- Calculates the total price';
-        inlineExample = '-- Check for null values';
-    }
-
-    // Set default prompting instruction for standard documentation [ds]
-    let instruction = `Provide block comments above functions and sparse inline comments for complex logic.`;
-    // Modify instruction if minimal documentation is requested [ds]
-    if (mode === 'light') {
-        instruction = `Provide ONLY block comments above functions. Keep it minimal.`;
-    // Modify instruction for exhaustive step-by-step documentation mode [ds]
-    } else if (mode === 'full') {
-        instruction = `Provide highly detailed block comments above functions, and exhaustive step-by-step inline comments explaining every conditional branch, loop, variable assignment, and logical block inside function bodies. Do not be sparse; explain the code's execution flow in detail.`;
-    }
-
-    // Define a placeholder rule for comment syntax restrictions [ds]
-    let rule5 = `5. IMPORTANT: Use ONLY ${singleLineToken} for comments. DO NOT use docstrings or multi-line string literals like """ or ''' for comments.`;
-    // Override the syntax rule specifically for CSS files [ds]
-    if (isCss) {
-        rule5 = `5. IMPORTANT: In CSS/SCSS, you MUST use /* ... */ for comments. DO NOT use // comments under any circumstances.`;
-    }
-
-    // Construct the full prompt combining rules, examples, and the numbered source code [ds]
-    let prompt = `
-You are a code documentation engine. Analyze the following ${language} code which has line numbers prepended to it.
-${instruction}
-
-CRITICAL RULES:
-1. You MUST respond with ONLY a raw, valid JSON array of objects. NO markdown formatting, NO backticks, NO explanations, NO text before or after the JSON.
-2. Each object must have exactly two properties: "line" (the integer line number where the comment should be inserted ABOVE) and "comment" (the text of the comment itself).
-3. Do NOT include the original code in your response.
-4. If no comments are needed, return an empty array: [].
-${rule5}
-
-Example Output:
-[
-  { "line": 4, "comment": "${blockExample}" },
-  { "line": 12, "comment": "${inlineExample}" }
-]
-
-Here is the source code:
-${numberedCode}
-    `.trim();
-
-    // Initialize a variable to hold the raw text output from the AI provider [ds]
-    let textResponse = "";
-
-    // Branch to handle the Google Gemini API provider [ds]
-    if (config.provider === 'gemini') {
-        // Construct the Google Gemini API endpoint URL with model and API key [ds]
-        const url = `https://generativelanguage.googleapis.com/v1beta/models/${config.model}:generateContent?key=${config.apiKey}`;
-        // Declare a variable to store the parsed API response [ds]
-        let data;
-        // Begin error handling for the Gemini API call [ds]
-        try {
-            // Send the POST request to the Gemini API with the constructed prompt [ds]
-            const response = await fetchWithRetry(url, {
-                method: 'POST',
-                headers: { 
-                    'Content-Type': 'application/json' 
-                },
-                body: JSON.stringify({
-                    "contents": [{ "parts": [{ "text": prompt }] }]
-                })
-            });
-            // Parse the JSON response body from the Gemini service [ds]
-            data = await response.json();
-        // Wrap network or fetch errors in a standard descriptive error [ds]
-        } catch (error) {
-            throw new Error(`AI Provider Request Failed: ${error.message}`);
-        }
-        // Check if the Gemini response contains an error object [ds]
-        if (data.error) {
-            // Extract the human-readable error message from the Google-specific structure [ds]
-            const msg = data.error.message || (typeof data.error === 'string' ? data.error : JSON.stringify(data.error));
-            // Throw a normalized error based on the API error details [ds]
-            throw new Error(`API Error: ${msg}`);
-        }
-        // Validate that the Gemini response structure contains the expected content [ds]
-        if (!data.candidates || !data.candidates[0] || !data.candidates[0].content || !data.candidates[0].content.parts || !data.candidates[0].content.parts[0]) {
-            // Capture the finish reason to provide context for empty content errors [ds]
-            const reason = data.candidates?.[0]?.finishReason || 'Unknown error';
-            // Throw an error indicating the AI returned no usable content [ds]
-            throw new Error(`AI Provider returned no content (finish reason: ${reason})`);
-        }
-        // Extract the raw text content from the Gemini response candidates [ds]
-        textResponse = data.candidates[0].content.parts[0].text;
-    // Branch to handle the Anthropic Claude API provider [ds]
-    } else if (config.provider === 'claude') {
-        // Construct the Anthropic API endpoint URL [ds]
-        const url = `${config.baseUrl}/v1/messages`;
-        // Declare a variable to store the parsed Anthropic API response [ds]
-        let data;
-        // Begin error handling for the Anthropic API call [ds]
-        try {
-            // Send the POST request to Anthropic with version headers and API key [ds]
-            const response = await fetchWithRetry(url, {
-                method: 'POST',
-                headers: {
-                    'Content-Type': 'application/json',
-                    'x-api-key': config.apiKey,
-                    'anthropic-version': '2023-06-01'
-                },
-                body: JSON.stringify({
-                    "model": config.model,
-                    "max_tokens": 8192,
-                    "messages": [{
-                        "role": "user",
-                        "content": prompt
-                    }]
-                })
-            });
-            // Parse the JSON response body from the Anthropic service [ds]
-            data = await response.json();
-        // Wrap network or fetch errors in a standard descriptive error [ds]
-        } catch (error) {
-            throw new Error(`AI Provider Request Failed: ${error.message}`);
-        }
-        // Check if the Anthropic response contains an error object [ds]
-        if (data.error) {
-            // Extract the human-readable error message from the Anthropic response [ds]
-            const msg = data.error.message || (typeof data.error === 'string' ? data.error : JSON.stringify(data.error));
-            // Throw a normalized error based on the API error details [ds]
-            throw new Error(`API Error: ${msg}`);
-        }
-        // Validate that the Anthropic response structure contains valid text content [ds]
-        if (!data.content || !data.content[0] || typeof data.content[0].text !== 'string') {
-            // Throw an error with the raw JSON if the structure is unexpected [ds]
-            throw new Error(`AI Provider returned an unexpected response structure: ${JSON.stringify(data)}`);
-        }
-        // Extract the text content from the first message block [ds]
-        textResponse = data.content[0].text;
-    } 
-    // Default branch for OpenAI-compatible APIs (OpenAI, Groq, etc.) [ds]
-    else {
-        // Construct the OpenAI-compatible chat completions endpoint [ds]
-        const url = `${config.baseUrl}/v1/chat/completions`;
-        // Declare a variable to store the parsed OpenAI-compatible response [ds]
-        let data;
-
-        // Initialize the request body structure with model and message content [ds]
-        const reqBody = { 
-            "model": config.model, 
-            "messages": [{ 
-                "role": "user", 
-                "content": prompt 
-            }] 
-        };
-        // Apply specific token limits for Groq provider to avoid issues [ds]
-        if (config.provider === 'groq') {
-            reqBody.max_tokens = 1000;
-        // Apply default high token limit for other OpenAI-compatible providers [ds]
-        } else {
-            reqBody.max_tokens = 8192;
-        }
-
-        // Begin error handling for the OpenAI-compatible API call [ds]
-        try {
-            // Send the POST request with Bearer token authentication [ds]
-            const response = await fetchWithRetry(url, {
-                method: 'POST',
-                headers: { 
-                    'Content-Type': 'application/json', 
-                    'Authorization': `Bearer ${config.apiKey}` 
-                },
-                body: JSON.stringify(reqBody)
-            });
-            // Parse the JSON response body from the OpenAI-compatible service [ds]
-            data = await response.json();
-        // Wrap network or fetch errors in a standard descriptive error [ds]
-        } catch (error) {
-            throw new Error(`AI Provider Request Failed: ${error.message}`);
-        }
-        // Check if the OpenAI-compatible response contains an error object [ds]
-        if (data.error) {
-            // Extract the human-readable error message from the OpenAI error structure [ds]
-            const msg = data.error.message || (typeof data.error === 'string' ? data.error : JSON.stringify(data.error));
-            // Throw a normalized error based on the API error details [ds]
-            throw new Error(`API Error: ${msg}`);
-        }
-        // Validate that the OpenAI response structure contains a valid message object [ds]
-        if (!data.choices || !data.choices[0] || !data.choices[0].message || typeof data.choices[0].message.content !== 'string') {
-            // Throw an error with the raw JSON if the structure is unexpected [ds]
-            throw new Error(`AI Provider returned an unexpected response structure: ${JSON.stringify(data)}`);
-        }
-        // Extract the text content from the first choice's message [ds]
-        textResponse = data.choices[0].message.content;
-    }
-
-    // Remove leading and trailing whitespace from the AI text output [ds]
-    let cleanText = textResponse.trim();
-    // Locate the starting index of the JSON array [ds]
-    const start = cleanText.indexOf('[');
-    // Locate the ending index of the JSON array [ds]
-    const end = cleanText.lastIndexOf(']');
-    // Check if a JSON array was actually found in the text [ds]
-    if (start !== -1) {
-        // Validate that the end index exists and comes after the start index [ds]
-        if (end !== -1 && end >= start) {
-            // Slice the text to contain only the JSON array portion [ds]
-            cleanText = cleanText.substring(start, end + 1);
-        } else {
-            // If ']' is missing, look for the last closing brace to fix truncated JSON [ds]
-            const lastBrace = cleanText.lastIndexOf('}');
-            // Ensure the found brace is actually within the current text segment [ds]
-            if (lastBrace > start) {
-                // Manually append the missing closing bracket to make valid JSON [ds]
-                cleanText = cleanText.substring(start, lastBrace + 1) + ']';
-            }
-        }
-    }
-
-    // Declare a variable to store the parsed JSON object [ds]
-    let parsed;
-    // Attempt to parse the cleaned text string into a JavaScript object [ds]
-    try {
-        // Execute the JSON parsing operation [ds]
-        parsed = JSON.parse(cleanText);
-    // Catch parsing errors if the text is malformed [ds]
-    } catch (e) {
-        // Throw a descriptive error including the raw response for debugging [ds]
-        throw new Error(`Parsing Error: Failed to parse LLM response as JSON. Raw response was:\n${textResponse}`);
-    }
-
-    // Validate that the parsed result is a JavaScript array [ds]
-    if (!Array.isArray(parsed)) {
-        // Throw an error if the top-level JSON structure is not an array [ds]
-        throw new Error("Schema Error: LLM response is not a JSON array.");
-    }
-
-    // Iterate over each comment object in the parsed array for validation [ds]
-    for (const item of parsed) {
-        // Verify that each item is a real object and not null or a primitive [ds]
-        if (typeof item !== 'object' || item === null) {
-            // Throw a schema error for invalid object types [ds]
-            throw new Error("Schema Error: Array elements must be objects.");
-        }
-        // Validate that the 'line' property is a positive integer [ds]
-        if (!Number.isInteger(item.line) || item.line <= 0) {
-            // Throw a schema error for invalid line numbers [ds]
-            throw new Error("Schema Error: 'line' must be a positive integer.");
-        }
-
-        // Enforce specific schema rules for data cleaning mode [ds]
-        if (mode === 'clean') {
-            // Ensure the 'action' property is 'delete' if in clean mode [ds]
-            if (item.action !== 'delete') {
-                // Throw a schema error for invalid actions in clean mode [ds]
-                throw new Error("Schema Error: 'action' must be 'delete' in clean mode.");
-            }
-        // Handle standard documentation modes that require comment text [ds]
-        } else {
-            // Verify that the 'comment' property exists and is a string [ds]
-            if (typeof item.comment !== 'string') {
-                // Throw a schema error if comment text is missing or not a string [ds]
-                throw new Error("Schema Error: 'comment' must be a string.");
-            }
-
-            // Normalize the comment text by trimming whitespace [ds]
-            const trimmedComment = item.comment.trim();
-            // Split the comment into lines to validate syntax per line [ds]
-            const commentLines = trimmedComment.split(/\r?\n/);
-            // Track whether we are currently inside a block comment [ds]
-            let inBlock = false;
-            // Loop through each line of the comment string [ds]
-            for (const cl of commentLines) {
-                // Trim leading/trailing whitespace from the current comment line [ds]
-                const tcl = cl.trim();
-                // Skip empty lines during validation [ds]
-                if (!tcl) continue;
-                // Check if this line is part of the continuation of a block comment [ds]
-                if (inBlock) {
-                    // Detect if the line closes a block comment (CSS/HTML style) [ds]
-                    if (tcl.includes('*/') || tcl.includes('-->')) {
-                        // Mark block comment state as false if a closing marker is found [ds]
-                        inBlock = false;
-                    }
-                    // Skip validation for lines inside a block comment [ds]
-                    continue;
-                }
-                // Check if the line starts with a valid comment marker [ds]
-                const startsWithMarker = 
-                    tcl.startsWith('//') || 
-                    tcl.startsWith('/*') || 
-                    tcl.startsWith('*') || 
-                    tcl.startsWith('#') || 
-                    tcl.startsWith('<!--') || 
-                    tcl.startsWith('--');
-                // Throw a strict security error if a line lacks a comment prefix [ds]
-                if (!startsWithMarker) {
-                    throw new Error(`Security Error: Comment on line ${item.line} contains invalid non-comment line: "${tcl}"`);
-                }
-                // Detect if a line opens a block comment without closing it [ds]
-                if ((tcl.startsWith('/*') && !tcl.includes('*/')) || (tcl.startsWith('<!--') && !tcl.includes('-->'))) {
-                    // Set the block comment state to true for subsequent line processing [ds]
-                    inBlock = true;
-                }
-            }
-        }
-    }
-
-    // Return the fully validated and parsed array of comments [ds]
-    return parsed;
-}
-
-// Export the main function for use by external modules [ds]
-module.exports = { getComments };
+/** [ds]
+ * Sends an HTTP request with automatic retry logic and timeout management.
+ * @param {string} url - The target URL.
+ * @param {object} options - Fetch options including method, headers, and body.
+ * @param {number} maxRetries - Maximum number of retry attempts.
+ * @param {number} initialDelay - Initial delay in milliseconds for exponential backoff.
+ * @returns {Promise<Response>} - Returns the successful Response object.
+ * @throws {Error} Throws a RateLimitError (with .isRateLimit=true) on 429, or the last error after exhausting retries.
+*/
+async function fetchWithRetry(url, options, maxRetries = 3, initialDelay = 1000) {
+    let lastError;
+    for (let attempt = 0; attempt < maxRetries; attempt++) {
+        const controller = new AbortController();
+        const timeoutId = setTimeout(() => controller.abort(), 45000);
+        try {
+            const response = await fetch(url, {
+                ...options,
+                signal: controller.signal
+            });
+            clearTimeout(timeoutId);
+            if (!response) {
+                throw new Error("No response received from fetch");
+            }
+            if (response.ok) {
+                return response;
+            }
+            // Surface 429 rate limits immediately so the adaptive controller can react [ds]
+            if (response.status === 429) {
+                const err = new Error(`HTTP Error 429: ${response.statusText || 'Too Many Requests'}`);
+                err.isRateLimit = true;
+                throw err;
+            }
+            if (response.status >= 500 && response.status < 600) {
+                lastError = new Error(`HTTP Error ${response.status}: ${response.statusText}`);
+            } else {
+                return response;
+            }
+        } catch (err) {
+            clearTimeout(timeoutId);
+            // Propagate rate limit errors immediately without retrying [ds]
+            if (err.isRateLimit) {
+                throw err;
+            }
+            if (err.name === 'AbortError') {
+                lastError = new Error("Request timed out after 45 seconds");
+            } else {
+                lastError = err;
+            }
+        }
+
+        if (attempt < maxRetries - 1) {
+            const backoffDelay = initialDelay * Math.pow(2, attempt);
+            console.warn(`[devsplain] AI request failed. Retrying in ${backoffDelay}ms... (Attempt ${attempt + 1}/${maxRetries})`);
+            await new Promise(resolve => setTimeout(resolve, backoffDelay));
+        }
+    }
+    throw lastError;
+}
+
+// ─── Chunking Constants ───────────────────────────────────────────────────────
+const CHUNK_SIZE = 200;
+const CHUNK_OVERLAP = 20;
+const CHUNK_THRESHOLD = 250;
+
+// ─── Adaptive Concurrency Controller ──────────────────────────────────────────
+// Shared mutable state: starts at the requested concurrency and drops to 1 on 429 [ds]
+let _concurrencyLimit = 2;
+let _hitRateLimit = false;
+
+/**
+ * Reset the adaptive concurrency controller for a new run.
+ * @param {number} initialLimit - Starting concurrency level.
+ */
+function resetConcurrency(initialLimit = 2) {
+    _concurrencyLimit = initialLimit;
+    _hitRateLimit = false;
+}
+
+/**
+ * Zero-dependency promise pool that respects the adaptive concurrency limit.
+ * If a task throws a 429 RateLimitError, concurrency is reduced to 1 for
+ * the remainder of the run, and the failed task is retried after a backoff.
+ * @param {Array} items - Items to process.
+ * @param {Function} taskFn - Async function to run per item.
+ * @returns {Promise<Array>} - Resolved results in order.
+ */
+async function runWithConcurrency(items, taskFn) {
+    const results = [];
+    const executing = new Set();
+
+    for (let i = 0; i < items.length; i++) {
+        const item = items[i];
+        const task = (async () => {
+            try {
+                return await taskFn(item);
+            } catch (err) {
+                // On 429, downshift to serial and retry the item after backoff [ds]
+                if (err.isRateLimit) {
+                    if (!_hitRateLimit) {
+                        _hitRateLimit = true;
+                        _concurrencyLimit = 1;
+                        console.warn(`[devsplain] Rate limit hit — switching to serial mode.`);
+                    }
+                    // Wait for all in-flight tasks to settle before retrying [ds]
+                    await Promise.allSettled([...executing]);
+                    const backoff = 2000 + Math.random() * 1000;
+                    console.warn(`[devsplain] Backing off for ${Math.round(backoff)}ms...`);
+                    await new Promise(r => setTimeout(r, backoff));
+                    return await taskFn(item);
+                }
+                throw err;
+            }
+        })();
+
+        const tracked = task.then(
+            val => { executing.delete(tracked); return val; },
+            err => { executing.delete(tracked); throw err; }
+        );
+        executing.add(tracked);
+        results.push(tracked);
+
+        if (executing.size >= _concurrencyLimit) {
+            await Promise.race(executing);
+        }
+    }
+    return Promise.all(results);
+}
+
+// ─── Prompt Builder ───────────────────────────────────────────────────────────
+
+/**
+ * Build the LLM prompt for a block of numbered code lines.
+ * @param {string} numberedCode - Code with line numbers prepended.
+ * @param {string} language - Filename or language identifier.
+ * @param {string} mode - Documentation mode ('default', 'light', 'full').
+ * @returns {string} The assembled prompt string.
+ */
+function buildPrompt(numberedCode, language, mode) {
+    const extMatch = language.match(/\.[0-9a-z]+$/i);
+    const ext = extMatch ? extMatch[0].toLowerCase() : '';
+    const isPython = ext === '.py';
+    const isRubyOrShell = ['.rb', '.sh'].includes(ext);
+    const isHTML = ['.html', '.vue', '.svelte'].includes(ext);
+    const isCss = ['.css', '.scss'].includes(ext);
+    const isSql = ext === '.sql';
+    let singleLineToken = '//';
+    let blockExample = '/** Calculates the total price */';
+    let inlineExample = '// Check for null values';
+
+    if (isPython || isRubyOrShell) {
+        singleLineToken = '#';
+        blockExample = '# Calculates the total price';
+        inlineExample = '# Check for null values';
+    } else if (isHTML) {
+        singleLineToken = '<!--';
+        blockExample = '<!-- Calculates the total price -->';
+        inlineExample = '<!-- Check for null values -->';
+    } else if (isCss) {
+        singleLineToken = '/*';
+        blockExample = '/* Calculates the total price */';
+        inlineExample = '/* Check for null values */';
+    } else if (isSql) {
+        singleLineToken = '--';
+        blockExample = '-- Calculates the total price';
+        inlineExample = '-- Check for null values';
+    }
+
+    let instruction = `Provide block comments above functions and sparse inline comments for complex logic.`;
+    if (mode === 'light') {
+        instruction = `Provide ONLY block comments above functions. Keep it minimal.`;
+    } else if (mode === 'full') {
+        instruction = `Provide highly detailed block comments above functions, and exhaustive step-by-step inline comments explaining every conditional branch, loop, variable assignment, and logical block inside function bodies. Do not be sparse; explain the code's execution flow in detail.`;
+    }
+
+    let rule5 = `5. IMPORTANT: Use ONLY ${singleLineToken} for comments. DO NOT use docstrings or multi-line string literals like """ or ''' for comments.`;
+    if (isCss) {
+        rule5 = `5. IMPORTANT: In CSS/SCSS, you MUST use /* ... */ for comments. DO NOT use // comments under any circumstances.`;
+    }
+
+    // Anti-triviality negative constraints to eliminate syntax-narrating clutter [ds]
+    const antiTrivialityRules = `
+ANTI-TRIVIALITY RULES (STRICTLY ENFORCED):
+6. NEVER write comments that merely narrate the syntax (e.g. NEVER write "// Loop over items" above a for loop, "// Return result" above a return, "// Increment i" above i++, or "// Define variable" above a declaration).
+7. NEVER comment standard variable initializations, obvious assignments, or self-describing code.
+8. ONLY write comments where:
+   - The WHY or architectural intent is non-obvious.
+   - An edge case, security workaround, or regex heuristic is being handled.
+   - A tricky formula, index manipulation (e.g. 0-indexed vs 1-indexed), or protocol-specific behavior occurs.
+9. Prefer comprehensive function-level block comments over cluttered inline comments. Quality over quantity.`;
+
+    const prompt = `
+You are a code documentation engine. Analyze the following ${language} code which has line numbers prepended to it.
+${instruction}
+
+CRITICAL RULES:
+1. You MUST respond with ONLY a raw, valid JSON array of objects. NO markdown formatting, NO backticks, NO explanations, NO text before or after the JSON.
+2. Each object must have exactly two properties: "line" (the integer line number where the comment should be inserted ABOVE) and "comment" (the text of the comment itself).
+3. Do NOT include the original code in your response.
+4. If no comments are needed, return an empty array: [].
+${rule5}
+${antiTrivialityRules}
+
+Example Output:
+[
+  { "line": 4, "comment": "${blockExample}" },
+  { "line": 12, "comment": "${inlineExample}" }
+]
+
+Here is the source code:
+${numberedCode}
+    `.trim();
+
+    return prompt;
+}
+
+// ─── Single-Chunk Comment Fetcher ─────────────────────────────────────────────
+
+/**
+ * Fetch comments for a single chunk of code from the configured AI provider.
+ * @param {string} prompt - The assembled prompt.
+ * @param {object} config - Provider config (provider, model, apiKey, baseUrl).
+ * @returns {Promise<string>} Raw text response from the AI.
+ */
+async function fetchFromProvider(prompt, config) {
+    let textResponse = "";
+
+    if (config.provider === 'gemini') {
+        const url = `https://generativelanguage.googleapis.com/v1beta/models/${config.model}:generateContent?key=${config.apiKey}`;
+        let data;
+        try {
+            const response = await fetchWithRetry(url, {
+                method: 'POST',
+                headers: { 
+                    'Content-Type': 'application/json' 
+                },
+                body: JSON.stringify({
+                    "contents": [{ "parts": [{ "text": prompt }] }]
+                })
+            });
+            data = await response.json();
+        } catch (error) {
+            // Re-throw rate limit errors so the concurrency controller can catch them [ds]
+            if (error.isRateLimit) throw error;
+            throw new Error(`AI Provider Request Failed: ${error.message}`);
+        }
+        if (data.error) {
+            const msg = data.error.message || (typeof data.error === 'string' ? data.error : JSON.stringify(data.error));
+            throw new Error(`API Error: ${msg}`);
+        }
+        if (!data.candidates || !data.candidates[0] || !data.candidates[0].content || !data.candidates[0].content.parts || !data.candidates[0].content.parts[0]) {
+            const reason = data.candidates?.[0]?.finishReason || 'Unknown error';
+            throw new Error(`AI Provider returned no content (finish reason: ${reason})`);
+        }
+        textResponse = data.candidates[0].content.parts[0].text;
+    } else if (config.provider === 'claude') {
+        const url = `${config.baseUrl}/v1/messages`;
+        let data;
+        try {
+            const response = await fetchWithRetry(url, {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json',
+                    'x-api-key': config.apiKey,
+                    'anthropic-version': '2023-06-01'
+                },
+                body: JSON.stringify({
+                    "model": config.model,
+                    "max_tokens": 8192,
+                    "messages": [{
+                        "role": "user",
+                        "content": prompt
+                    }]
+                })
+            });
+            data = await response.json();
+        } catch (error) {
+            if (error.isRateLimit) throw error;
+            throw new Error(`AI Provider Request Failed: ${error.message}`);
+        }
+        if (data.error) {
+            const msg = data.error.message || (typeof data.error === 'string' ? data.error : JSON.stringify(data.error));
+            throw new Error(`API Error: ${msg}`);
+        }
+        if (!data.content || !data.content[0] || typeof data.content[0].text !== 'string') {
+            throw new Error(`AI Provider returned an unexpected response structure: ${JSON.stringify(data)}`);
+        }
+        textResponse = data.content[0].text;
+    } 
+    else {
+        const url = `${config.baseUrl}/v1/chat/completions`;
+        let data;
+
+        const reqBody = { 
+            "model": config.model, 
+            "messages": [{ 
+                "role": "user", 
+                "content": prompt 
+            }] 
+        };
+        if (config.provider === 'groq') {
+            reqBody.max_tokens = 1000;
+        } else {
+            reqBody.max_tokens = 8192;
+        }
+
+        try {
+            const response = await fetchWithRetry(url, {
+                method: 'POST',
+                headers: { 
+                    'Content-Type': 'application/json', 
+                    'Authorization': `Bearer ${config.apiKey}` 
+                },
+                body: JSON.stringify(reqBody)
+            });
+            data = await response.json();
+        } catch (error) {
+            if (error.isRateLimit) throw error;
+            throw new Error(`AI Provider Request Failed: ${error.message}`);
+        }
+        if (data.error) {
+            const msg = data.error.message || (typeof data.error === 'string' ? data.error : JSON.stringify(data.error));
+            throw new Error(`API Error: ${msg}`);
+        }
+        if (!data.choices || !data.choices[0] || !data.choices[0].message || typeof data.choices[0].message.content !== 'string') {
+            throw new Error(`AI Provider returned an unexpected response structure: ${JSON.stringify(data)}`);
+        }
+        textResponse = data.choices[0].message.content;
+    }
+
+    return textResponse;
+}
+
+// ─── Response Parser & Validator ──────────────────────────────────────────────
+
+/**
+ * Parse, validate, and sanitize the raw text response from the AI provider.
+ * @param {string} textResponse - Raw text from the AI.
+ * @param {string} mode - Documentation mode.
+ * @returns {Array} Validated array of comment objects.
+ */
+function parseAndValidate(textResponse, mode) {
+    let cleanText = textResponse.trim();
+    const start = cleanText.indexOf('[');
+    const end = cleanText.lastIndexOf(']');
+    if (start !== -1) {
+        if (end !== -1 && end >= start) {
+            cleanText = cleanText.substring(start, end + 1);
+        } else {
+            const lastBrace = cleanText.lastIndexOf('}');
+            if (lastBrace > start) {
+                cleanText = cleanText.substring(start, lastBrace + 1) + ']';
+            }
+        }
+    }
+
+    let parsed;
+    try {
+        parsed = JSON.parse(cleanText);
+    } catch (e) {
+        throw new Error(`Parsing Error: Failed to parse LLM response as JSON. Raw response was:\n${textResponse}`);
+    }
+
+    if (!Array.isArray(parsed)) {
+        throw new Error("Schema Error: LLM response is not a JSON array.");
+    }
+
+    for (const item of parsed) {
+        if (typeof item !== 'object' || item === null) {
+            throw new Error("Schema Error: Array elements must be objects.");
+        }
+        if (!Number.isInteger(item.line) || item.line <= 0) {
+            throw new Error("Schema Error: 'line' must be a positive integer.");
+        }
+
+        if (mode === 'clean') {
+            if (item.action !== 'delete') {
+                throw new Error("Schema Error: 'action' must be 'delete' in clean mode.");
+            }
+        } else {
+            if (typeof item.comment !== 'string') {
+                throw new Error("Schema Error: 'comment' must be a string.");
+            }
+
+            const trimmedComment = item.comment.trim();
+            const commentLines = trimmedComment.split(/\r?\n/);
+            let inBlock = false;
+            for (const cl of commentLines) {
+                const tcl = cl.trim();
+                if (!tcl) continue;
+                if (inBlock) {
+                    if (tcl.includes('*/') || tcl.includes('-->')) {
+                        inBlock = false;
+                    }
+                    continue;
+                }
+                const startsWithMarker = 
+                    tcl.startsWith('//') || 
+                    tcl.startsWith('/*') || 
+                    tcl.startsWith('*') || 
+                    tcl.startsWith('#') || 
+                    tcl.startsWith('<!--') || 
+                    tcl.startsWith('--');
+                if (!startsWithMarker) {
+                    throw new Error(`Security Error: Comment on line ${item.line} contains invalid non-comment line: "${tcl}"`);
+                }
+                if ((tcl.startsWith('/*') && !tcl.includes('*/')) || (tcl.startsWith('<!--') && !tcl.includes('-->'))) {
+                    inBlock = true;
+                }
+            }
+        }
+    }
+
+    return parsed;
+}
+
+// ─── Core Public API ──────────────────────────────────────────────────────────
+
+/**
+ * Fetch and validate AI-generated comments for a source file.
+ * Automatically chunks files exceeding CHUNK_THRESHOLD lines into
+ * overlapping windows, processes them with adaptive concurrency,
+ * and deduplicates comments across chunk boundaries.
+ * @param {string} code - Full source code of the file.
+ * @param {string} language - Filename or language identifier.
+ * @param {object} config - Provider configuration.
+ * @param {string} mode - Documentation mode ('default', 'light', 'full', 'clean').
+ * @returns {Promise<Array>} Array of validated comment objects with global line numbers.
+ */
+async function getComments(code, language, config, mode = 'default') {
+    const lines = code.split(/\r?\n/);
+
+    // Small files: single-shot processing (no chunking overhead) [ds]
+    if (lines.length <= CHUNK_THRESHOLD) {
+        const numberedCode = lines.map((line, i) => `${i + 1}: ${line}`).join('\n');
+        const prompt = buildPrompt(numberedCode, language, mode);
+        const textResponse = await fetchFromProvider(prompt, config);
+        return parseAndValidate(textResponse, mode);
+    }
+
+    // Large files: slice into overlapping chunks with global line numbers [ds]
+    const chunks = [];
+    for (let start = 0; start < lines.length; start += (CHUNK_SIZE - CHUNK_OVERLAP)) {
+        const end = Math.min(start + CHUNK_SIZE, lines.length);
+        chunks.push({ start, end });
+        if (end >= lines.length) break;
+    }
+
+    // Process chunks through the adaptive concurrency pool [ds]
+    const chunkResults = await runWithConcurrency(chunks, async (chunk) => {
+        const chunkLines = lines.slice(chunk.start, chunk.end);
+        const startLineNum = chunk.start + 1;
+        const numberedCode = chunkLines.map((line, i) => `${startLineNum + i}: ${line}`).join('\n');
+        const prompt = buildPrompt(numberedCode, language, mode);
+        const textResponse = await fetchFromProvider(prompt, config);
+        return parseAndValidate(textResponse, mode);
+    });
+
+    // Merge and deduplicate: first comment wins for overlapping line numbers [ds]
+    const seenLines = new Set();
+    const allComments = [];
+    for (const chunkComments of chunkResults) {
+        for (const c of chunkComments) {
+            if (!seenLines.has(c.line)) {
+                seenLines.add(c.line);
+                allComments.push(c);
+            }
+        }
+    }
+
+    return allComments;
+}
+
+module.exports = { getComments, runWithConcurrency, resetConcurrency, CHUNK_SIZE, CHUNK_OVERLAP, CHUNK_THRESHOLD };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "devsplain",
-  "version": "2.2.4",
+  "version": "2.3.0",
   "description": "An agent-agnostic CLI tool that automatically adds JSDoc and inline comments to your code using free LLMs.",
   "author": "mwahaj36",
   "license": "MIT",