byterover-cli 1.8.0 → 2.0.0

package/dist/server/infra/executor/curate-executor.js

@@ -9,8 +9,8 @@ import { validateFileForCurate } from '../../utils/file-validator.js';
  * It's an Executor that wraps agent.execute() with curate-specific options.
  *
  * Architecture:
- * - TaskProcessor injects the long-lived CipherAgent
- * - Event streaming is handled by agent-worker (subscribes to agentEventBus)
+ * - AgentProcess injects the long-lived CipherAgent
+ * - Event streaming is handled by agent-process (subscribes to agentEventBus)
  * - Transport handles task lifecycle (task:started, task:completed, task:error)
  * - Executor focuses solely on curate execution
  */
@@ -23,24 +23,65 @@ export class CurateExecutor {
     static MAX_LINES_PER_FILE = 2000;
     /** Maximum pages to extract for PDFs */
     static MAX_PDF_PAGES = 50;
+    /** Last curation status — available for future status-check command */
+    lastStatus;
     fileContentReader;
     constructor(fileContentReader) {
         this.fileContentReader = fileContentReader ?? createFileContentReader();
     }
     async executeWithAgent(agent, options) {
         const { clientCwd, content, files, taskId } = options;
-        // Process file references - now reads file contents directly
+        // Create per-task session for parallel isolation (own sandbox + history + LLM service)
+        const taskSessionId = await agent.createTaskSession(taskId, 'curate');
+        // Process file references - reads file contents directly
         const fileReferenceInstructions = await this.processFileReferences(files ?? [], clientCwd);
-        // Build prompt with optional file reference instructions
-        const prompt = fileReferenceInstructions ? `${content}\n${fileReferenceInstructions}` : content;
-        // Execute with curate commandType
-        // Agent uses its default session (created during start())
-        // Task lifecycle is managed by Transport (task:started, task:completed, task:error)
-        const response = await agent.execute(prompt, {
-            executionContext: { commandType: 'curate' },
-            taskId,
-        });
-        return response;
+        // Build full context (content + optional file references)
+        const fullContext = fileReferenceInstructions ? `${content}\n${fileReferenceInstructions}` : content;
+        // Task-scoped variable names for RLM pattern.
+        // Replace hyphens with underscores: UUIDs have hyphens which are invalid in JS identifiers,
+        // so the LLM would naturally use underscores when writing code-exec calls — causing a
+        // ReferenceError if the variable was stored under the hyphen version.
+        const taskIdSafe = taskId.replaceAll('-', '_');
+        const ctxVar = `__curate_ctx_${taskIdSafe}`;
+        const histVar = `__curate_hist_${taskIdSafe}`;
+        const metaVar = `__curate_meta_${taskIdSafe}`;
+        // Compute context metadata (RLM pattern — LM sees metadata, not raw content)
+        const contextLines = fullContext.split('\n');
+        const metadata = {
+            charCount: fullContext.length,
+            lineCount: contextLines.length,
+            messageCount: (fullContext.match(/\n\n\[(USER|ASSISTANT)\]:/g) || []).length,
+            preview: fullContext.slice(0, 500),
+            type: 'string',
+        };
+        // Inject context, metadata, and empty history into the TASK session's sandbox
+        agent.setSandboxVariableOnSession(taskSessionId, ctxVar, fullContext);
+        agent.setSandboxVariableOnSession(taskSessionId, histVar, { entries: [], totalProcessed: 0 });
+        agent.setSandboxVariableOnSession(taskSessionId, metaVar, metadata);
+        // Prompt with metadata guidance (RLM pattern: LM sees metadata first, peeks via slicing)
+        const prompt = [
+            `Curate using RLM approach.`,
+            `Context variable: ${ctxVar} (${metadata.charCount} chars, ${metadata.lineCount} lines, ${metadata.messageCount} messages)`,
+            `History variable: ${histVar}`,
+            `Metadata variable: ${metaVar}`,
+            `IMPORTANT: Do NOT print raw context. Use slicing to peek at sections (e.g., ${ctxVar}.slice(0, 3000)).`,
+            `Use silent mode (silent: true) for variable assignments. Use tools.agentQuery() for chunk processing.`,
+        ].join('\n');
+        try {
+            // Execute on the task session (isolated sandbox + history)
+            // Task lifecycle is managed by Transport (task:started, task:completed, task:error)
+            const response = await agent.executeOnSession(taskSessionId, prompt, {
+                executionContext: { clearHistory: true, commandType: 'curate', maxIterations: 50 },
+                taskId,
+            });
+            // Parse curation status from agent response for status tracking
+            this.lastStatus = this.parseCurationStatus(taskId, response);
+            return response;
+        }
+        finally {
+            // Clean up entire task session (sandbox + history) in one call
+            await agent.deleteTaskSession(taskSessionId);
+        }
     }
     /**
      * Format file contents for inclusion in the prompt.
@@ -83,6 +124,39 @@ export class CurateExecutor {
             }
         }
     }
+    /**
+     * Parse curation status from the agent response.
+     * Extracts JSON status block if present, otherwise infers from response text.
+     */
+    parseCurationStatus(taskId, response) {
+        const defaultSummary = { added: 0, deleted: 0, failed: 0, merged: 0, updated: 0 };
+        const defaultVerification = { checked: 0, confirmed: 0, missing: [] };
+        // Try to extract JSON status block from response (agent instructed to include it)
+        try {
+            const jsonMatch = /```json\n([\S\s]*?)\n```/.exec(response);
+            if (jsonMatch) {
+                const parsed = JSON.parse(jsonMatch[1]);
+                return {
+                    completedAt: new Date().toISOString(),
+                    status: parsed.summary?.failed > 0 ? 'partial' : 'success',
+                    summary: parsed.summary ?? defaultSummary,
+                    taskId,
+                    verification: parsed.verification ?? defaultVerification,
+                };
+            }
+        }
+        catch {
+            // Ignore parse errors — fall through to heuristic
+        }
+        // Fallback: infer from response text
+        return {
+            completedAt: new Date().toISOString(),
+            status: response.includes('failed') ? 'failed' : 'success',
+            summary: defaultSummary,
+            taskId,
+            verification: defaultVerification,
+        };
+    }
     /**
      * Process file paths from --files flag.
      * Now reads file contents directly using FileContentReader.

package/dist/server/infra/executor/direct-search-responder.d.ts

@@ -1,9 +1,9 @@
-/** Score at which the result is so strong that dominance check is skipped */
-export declare const DIRECT_RESPONSE_HIGH_CONFIDENCE_THRESHOLD = 15;
-/** Minimum score for the top result to qualify for a direct (no-LLM) response */
-export declare const DIRECT_RESPONSE_SCORE_THRESHOLD = 8;
-/** Top result must be N times the second result's score to be considered dominant */
-export declare const DIRECT_RESPONSE_DOMINANCE_RATIO = 2;
+/** Normalized score at which the result is so strong that dominance check is skipped */
+export declare const DIRECT_RESPONSE_HIGH_CONFIDENCE_THRESHOLD = 0.93;
+/** Minimum normalized score for the top result to qualify for a direct (no-LLM) response */
+export declare const DIRECT_RESPONSE_SCORE_THRESHOLD = 0.85;
+/** Minimum gap between top and second result's normalized score to be considered dominant */
+export declare const DIRECT_RESPONSE_MIN_GAP = 0.08;
 /**
  * A search result with full document content for direct response formatting.
  */
@@ -17,10 +17,13 @@ export interface DirectSearchResult {
  * Determines if search results are confident enough for a direct response
  * without involving the LLM.
  *
- * Requires:
+ * Uses normalized [0, 1) scores with a gap-based dominance model:
  * 1. Top result score >= DIRECT_RESPONSE_SCORE_THRESHOLD (minimum confidence)
  * 2. Either: top score >= HIGH_CONFIDENCE_THRESHOLD (strong enough to skip dominance check)
- *    Or: top result dominates other results (score >= 2x the second result)
+ *    Or: gap between top and #2 >= DIRECT_RESPONSE_MIN_GAP (clear separation)
+ *
+ * Gap-based instead of ratio-based because normalized scores cluster in [0.8, 0.95],
+ * making ratio checks (e.g., 2x) impossible to satisfy.
  *
  * @param results - Sorted search results (highest score first)
  * @returns true if a direct response can be served

package/dist/server/infra/executor/direct-search-responder.js

@@ -1,21 +1,24 @@
-/** Score at which the result is so strong that dominance check is skipped */
-export const DIRECT_RESPONSE_HIGH_CONFIDENCE_THRESHOLD = 15;
-/** Minimum score for the top result to qualify for a direct (no-LLM) response */
-export const DIRECT_RESPONSE_SCORE_THRESHOLD = 8;
-/** Top result must be N times the second result's score to be considered dominant */
-export const DIRECT_RESPONSE_DOMINANCE_RATIO = 2;
+/** Normalized score at which the result is so strong that dominance check is skipped */
+export const DIRECT_RESPONSE_HIGH_CONFIDENCE_THRESHOLD = 0.93;
+/** Minimum normalized score for the top result to qualify for a direct (no-LLM) response */
+export const DIRECT_RESPONSE_SCORE_THRESHOLD = 0.85;
+/** Minimum gap between top and second result's normalized score to be considered dominant */
+export const DIRECT_RESPONSE_MIN_GAP = 0.08;
 /** Maximum content length per document in the direct response */
-const MAX_CONTENT_LENGTH = 1500;
+const MAX_CONTENT_LENGTH = 5000;
 /** Maximum number of documents to include in the direct response */
-const MAX_DOCS = 3;
+const MAX_DOCS = 5;
 /**
  * Determines if search results are confident enough for a direct response
  * without involving the LLM.
  *
- * Requires:
+ * Uses normalized [0, 1) scores with a gap-based dominance model:
  * 1. Top result score >= DIRECT_RESPONSE_SCORE_THRESHOLD (minimum confidence)
  * 2. Either: top score >= HIGH_CONFIDENCE_THRESHOLD (strong enough to skip dominance check)
- *    Or: top result dominates other results (score >= 2x the second result)
+ *    Or: gap between top and #2 >= DIRECT_RESPONSE_MIN_GAP (clear separation)
+ *
+ * Gap-based instead of ratio-based because normalized scores cluster in [0.8, 0.95],
+ * making ratio checks (e.g., 2x) impossible to satisfy.
  *
  * @param results - Sorted search results (highest score first)
  * @returns true if a direct response can be served
@@ -32,10 +35,9 @@ export function canRespondDirectly(results) {
     // High-confidence path: score so strong that dominance is irrelevant
     if (topResult.score >= DIRECT_RESPONSE_HIGH_CONFIDENCE_THRESHOLD)
         return true;
-    const secondScore = results[1].score;
-    if (secondScore === 0)
-        return true;
-    return topResult.score / secondScore >= DIRECT_RESPONSE_DOMINANCE_RATIO;
+    // Gap-based dominance: top must be clearly separated from #2
+    const gap = topResult.score - results[1].score;
+    return gap >= DIRECT_RESPONSE_MIN_GAP;
 }
 /**
  * Format a direct response from search results (no LLM involved).

package/dist/server/infra/executor/folder-pack-executor.js

@@ -1,5 +1,17 @@
+import { appendFileSync } from 'node:fs';
 import fs from 'node:fs/promises';
 import path from 'node:path';
+const LOG_PATH = process.env.BRV_SESSION_LOG;
+function folderPackLog(message) {
+    if (!LOG_PATH)
+        return;
+    try {
+        appendFileSync(LOG_PATH, `${new Date().toISOString()} [folder-pack-executor] ${message}\n`);
+    }
+    catch {
+        // ignore — never block on logging
+    }
+}
 /**
  * FolderPackExecutor - Executes folder pack + curate tasks with an injected CipherAgent.
  *
@@ -23,6 +35,9 @@ export class FolderPackExecutor {
     }
     async executeWithAgent(agent, options) {
         const { clientCwd, content, folderPath, taskId } = options;
+        if (!folderPath) {
+            throw new Error('folderPath is required for curate-folder tasks');
+        }
         // Resolve folder path
         const basePath = clientCwd ?? process.cwd();
         const absoluteFolderPath = path.isAbsolute(folderPath) ? folderPath : path.resolve(basePath, folderPath);
@@ -35,7 +50,7 @@ export class FolderPackExecutor {
         // Use iterative extraction strategy (inspired by rlm)
         // Stores packed folder in sandbox environment and lets agent iteratively query/extract
         // This avoids token limits entirely - works for folders of any size
-        return this.executeIterative(agent, packResult, content, absoluteFolderPath, taskId);
+        return this.executeIterative(agent, packResult, content, absoluteFolderPath, taskId, basePath);
     }
     /**
      * Build iterative extraction prompt with file-based access.
@@ -468,7 +483,7 @@ When curating knowledge from source files, you MUST preserve the exact, complete
    - Detect: fenced blocks with mermaid/plantuml tags, @startuml/@enduml, box-drawing characters
 
 8. **For Tables:**
-   - Copy complete tables with ALL rows into \`narrative.structure\` or \`narrative.features\`
+   - Copy complete tables with ALL rows into \`narrative.structure\` or \`narrative.highlights\`
    - Preserve column headers and every data row - do not summarize
 
 ### What "Preserve" Means:
@@ -526,7 +541,7 @@ content: {
 **Secondary fields for structured details:**
 - \`narrative.rules\` - Exact rule/constraint text from docs
 - \`narrative.examples\` - Complete example code with full context
-- \`narrative.features\` - Full feature descriptions with all details
+- \`narrative.highlights\` - Key highlights, capabilities, deliverables, or notable outcomes
 - \`narrative.structure\` - Complete structural documentation
 - \`narrative.dependencies\` - Full dependency information
 - \`rawConcept.patterns\` - All patterns with complete regex/validation strings
@@ -558,7 +573,7 @@ Extract ALL of the following - COMPLETE and VERBATIM:
 8. **Examples** - Copy full example code with all context
 9. **Metadata** - Capture authors, versions, dates from files
 10. **Diagrams** - Mermaid diagrams, PlantUML, ASCII art flow charts, sequence diagrams (use \`narrative.diagrams\` with type and content - preserve verbatim)
-11. **Tables** - Data tables with ALL rows preserved (use \`narrative.structure\` or \`narrative.features\`)
+11. **Tables** - Data tables with ALL rows preserved (use \`narrative.structure\` or \`narrative.highlights\`)
 12. **Procedures** - Step-by-step instructions, numbered workflows (use \`narrative.rules\`)
 
 ## Curation Process
@@ -740,41 +755,67 @@ await tools.curate([{
      * This avoids token limits entirely - data is stored in REPL, not in prompt.
      */
     // eslint-disable-next-line max-params
-    async executeIterative(agent, packResult, userContext, folderPath, taskId) {
+    async executeIterative(agent, packResult, userContext, folderPath, taskId, projectRoot) {
         // Step 1: Generate repomix-style XML (single string with all file contents)
         const packedXml = this.folderPackService.generateXml(packResult);
         const xmlSizeInMB = (packedXml.length / (1024 * 1024)).toFixed(2);
-        console.log(`[FolderPackExecutor] Generated XML: ${xmlSizeInMB} MB for ${packResult.fileCount} files`);
+        folderPackLog(`Generated XML: ${xmlSizeInMB} MB for ${packResult.fileCount} files`);
         // Step 2: Write XML to temporary file (avoids token limits, works with any agent)
         // This approach: file path (~50 bytes) sent to LLM, data stays on disk
-        // IMPORTANT: Write to CWD (not /tmp) so sandbox can access it
-        const tmpFilePath = path.join(process.cwd(), `.byterover-curate-${taskId}.xml`);
-        console.log(`[FolderPackExecutor] Writing folder data to temp file: ${tmpFilePath}`);
+        // IMPORTANT: Write to project root (not /tmp) so sandbox can access it
+        const tmpFilePath = path.join(projectRoot, `.byterover-curate-${taskId}.xml`);
+        folderPackLog(`Writing folder data to temp file: ${tmpFilePath}`);
         try {
             await fs.writeFile(tmpFilePath, packedXml, 'utf8');
-            console.log(`[FolderPackExecutor] Successfully wrote ${xmlSizeInMB} MB to temp file`);
+            folderPackLog(`Successfully wrote ${xmlSizeInMB} MB to temp file`);
         }
         catch (error) {
             throw new Error(`Failed to write temp file: ${error instanceof Error ? error.message : String(error)}`);
         }
-        // Step 3: Execute main curation task with file path reference
-        const prompt = this.buildIterativePromptWithFileAccess(userContext, folderPath, tmpFilePath, packResult.fileCount, packResult.totalLines);
+        // Create per-task session for parallel isolation (own sandbox + history + LLM service)
+        const taskSessionId = await agent.createTaskSession(taskId, 'curate');
+        // Step 3: Store full instructions as sandbox variable (lazy prompt loading).
+        // This saves ~12-15K tokens by keeping the massive instruction set out of the prompt.
+        // The LLM reads instructions on-demand via code_exec.
+        const fullInstructions = this.buildIterativePromptWithFileAccess(userContext, folderPath, tmpFilePath, packResult.fileCount, packResult.totalLines);
+        // Replace hyphens with underscores: UUIDs have hyphens which are invalid in JS identifiers.
+        // The LLM uses underscores when writing code-exec calls — matching curate-executor pattern.
+        const taskIdSafe = taskId.replaceAll('-', '_');
+        const instructionsVar = `__curate_instructions_${taskIdSafe}`;
+        agent.setSandboxVariableOnSession(taskSessionId, instructionsVar, fullInstructions);
+        // Compact prompt with variable reference and essential metadata
+        const contextSection = userContext?.trim() ? `\nUser context: ${userContext}\n` : '';
+        const compactPrompt = [
+            `# Folder Curation Task`,
+            ``,
+            `Folder: ${folderPath} (${packResult.fileCount} files, ${packResult.totalLines} lines)`,
+            `Data file: \`${tmpFilePath}\` (repomix-style XML format)`,
+            `Full instructions: variable \`${instructionsVar}\``,
+            contextSection,
+            `**Start by reading instructions**: Use code_exec to read \`${instructionsVar}.slice(0, 5000)\` for the strategy section, then \`${instructionsVar}.slice(5000, 10000)\` for content rules.`,
+            `Use \`tools.readFile()\` and \`tools.grep()\` inside code_exec to process the XML data file.`,
+            `Use \`tools.curate()\` to create knowledge topics. Use \`setFinalResult()\` when done.`,
+        ]
+            .filter(Boolean)
+            .join('\n');
         let response;
         try {
-            response = await agent.execute(prompt, {
+            response = await agent.executeOnSession(taskSessionId, compactPrompt, {
                 executionContext: { commandType: 'curate' },
                 taskId,
             });
         }
         finally {
-            // Step 4: Clean up - delete temp file
-            console.log(`[FolderPackExecutor] Cleaning up temp file`);
+            // Clean up task session (sandbox + history)
+            await agent.deleteTaskSession(taskSessionId);
+            // Clean up temp file
+            folderPackLog(`Cleaning up temp file: ${tmpFilePath}`);
             try {
                 await fs.unlink(tmpFilePath);
-                console.log(`[FolderPackExecutor] Temp file cleanup successful`);
+                folderPackLog(`Temp file cleanup successful`);
             }
             catch (error) {
-                console.warn(`[FolderPackExecutor] Temp file cleanup warning (non-fatal):`, error);
+                folderPackLog(`Temp file cleanup warning (non-fatal): ${error instanceof Error ? error.message : String(error)}`);
             }
         }
         return response;

package/dist/server/infra/executor/query-executor.d.ts

@@ -21,8 +21,8 @@ export interface QueryExecutorDeps {
  * It's an Executor that wraps agent.execute() with query-specific options.
  *
  * Architecture:
- * - TaskProcessor injects the long-lived CipherAgent
- * - Event streaming is handled by agent-worker (subscribes to agentEventBus)
+ * - AgentProcess injects the long-lived CipherAgent
+ * - Event streaming is handled by agent-process (subscribes to agentEventBus)
  * - Transport handles task lifecycle (task:started, task:completed, task:error)
  * - Executor focuses solely on query execution
  *
@@ -48,11 +48,13 @@ export declare class QueryExecutor implements IQueryExecutor {
      */
     private buildPrefetchedContext;
     /**
-     * Build a streamlined query prompt optimized for fast, accurate responses.
+     * Build a query prompt using RLM pattern (variable references, not embedded data).
      *
-     * When pre-fetched context is available, the prompt instructs the LLM to answer
-     * directly from the provided context (reducing LLM calls from 2+ to 1).
-     * When no context is available, falls back to tool-based search.
+     * Search results are pre-loaded into sandbox variables. The prompt references
+     * variable names so the LLM accesses results via code_exec, not via the prompt.
+     *
+     * @param query - User query
+     * @param options - Prompt options with variable names and metadata
      */
     private buildQueryPrompt;
     /**
@@ -61,6 +63,21 @@ export declare class QueryExecutor implements IQueryExecutor {
      * the fingerprint changes and cached results are invalidated.
      */
     private computeContextTreeFingerprint;
+    /**
+     * Extract key entities from a query for supplementary searches.
+     * Simple heuristic: split query, filter stopwords, keep significant terms.
+     */
+    private extractQueryEntities;
+    /**
+     * Run supplementary entity-based searches to improve recall.
+     * Extracts key entities from the query and searches for each independently,
+     * then merges unique results (by path) into the original search result.
+     *
+     * @param query - Original user query
+     * @param searchResult - Initial search result to supplement
+     * @returns Merged search result with additional entity-based matches
+     */
+    private supplementEntitySearches;
     /**
      * Attempt to produce a direct response from search results without LLM.
      * Returns formatted response if high-confidence dominant match found, undefined otherwise.