byterover-cli 1.7.1 → 1.8.0

package/dist/server/core/domain/knowledge/markdown-writer.js

@@ -1,23 +1,37 @@
 import { generateRelationsSection, parseRelations } from './relation-parser.js';
+/**
+ * Normalizes newline characters in text.
+ * Converts literal \n strings to actual newlines.
+ */
+function normalizeNewlines(text) {
+    return text.replaceAll(String.raw `\n`, '\n');
+}
 function generateRawConceptSection(rawConcept) {
     if (!rawConcept) {
         return '';
     }
     const parts = [];
     if (rawConcept.task) {
-        parts.push(`**Task:**\n${rawConcept.task}`);
+        parts.push(`**Task:**\n${normalizeNewlines(rawConcept.task)}`);
     }
     if (rawConcept.changes && rawConcept.changes.length > 0) {
-        parts.push(`**Changes:**\n${rawConcept.changes.map(c => `- ${c}`).join('\n')}`);
+        parts.push(`**Changes:**\n${rawConcept.changes.map(c => `- ${normalizeNewlines(c)}`).join('\n')}`);
     }
     if (rawConcept.files && rawConcept.files.length > 0) {
-        parts.push(`**Files:**\n${rawConcept.files.map(f => `- ${f}`).join('\n')}`);
+        parts.push(`**Files:**\n${rawConcept.files.map(f => `- ${normalizeNewlines(f)}`).join('\n')}`);
     }
     if (rawConcept.flow) {
-        parts.push(`**Flow:**\n${rawConcept.flow}`);
+        parts.push(`**Flow:**\n${normalizeNewlines(rawConcept.flow)}`);
     }
     if (rawConcept.timestamp) {
-        parts.push(`**Timestamp:** ${rawConcept.timestamp}`);
+        parts.push(`**Timestamp:** ${normalizeNewlines(rawConcept.timestamp)}`);
+    }
+    if (rawConcept.author) {
+        parts.push(`**Author:** ${rawConcept.author}`);
+    }
+    if (rawConcept.patterns && rawConcept.patterns.length > 0) {
+        const patternsText = rawConcept.patterns.map(p => `- \`${p.pattern}\`${p.flags ? ` (flags: ${p.flags})` : ''} - ${p.description}`).join('\n');
+        parts.push(`**Patterns:**\n${patternsText}`);
     }
     if (parts.length === 0) {
         return '';
@@ -30,13 +44,27 @@ function generateNarrativeSection(narrative) {
     }
     const parts = [];
     if (narrative.structure) {
-        parts.push(`### Structure\n${narrative.structure}`);
+        parts.push(`### Structure\n${normalizeNewlines(narrative.structure)}`);
     }
     if (narrative.dependencies) {
-        parts.push(`### Dependencies\n${narrative.dependencies}`);
+        parts.push(`### Dependencies\n${normalizeNewlines(narrative.dependencies)}`);
     }
     if (narrative.features) {
-        parts.push(`### Features\n${narrative.features}`);
+        parts.push(`### Features\n${normalizeNewlines(narrative.features)}`);
+    }
+    if (narrative.rules) {
+        parts.push(`### Rules\n${narrative.rules}`);
+    }
+    if (narrative.examples) {
+        parts.push(`### Examples\n${narrative.examples}`);
+    }
+    if (narrative.diagrams && narrative.diagrams.length > 0) {
+        const diagramParts = narrative.diagrams.map(d => {
+            const lang = d.type === 'ascii' ? '' : d.type;
+            const titleLine = d.title ? `**${d.title}**\n` : '';
+            return `${titleLine}\`\`\`${lang}\n${d.content}\n\`\`\``;
+        });
+        parts.push(`### Diagrams\n${diagramParts.join('\n\n')}`);
     }
     if (parts.length === 0) {
         return '';
@@ -79,6 +107,31 @@ function parseRawConceptSection(content) {
     if (timestampMatch) {
         rawConcept.timestamp = timestampMatch[1].trim();
     }
+    // Author can be inline
+    const authorMatch = sectionContent.match(/\*\*\s*Author\s*:\s*\*\*\s*(.+)/i);
+    if (authorMatch) {
+        rawConcept.author = authorMatch[1].trim();
+    }
+    // Patterns is multi-line with list items
+    const patternsMatch = sectionContent.match(/\*\*\s*Patterns\s*:\s*\*\*\s*\n([\s\S]*?)(?=\n\*\*|\n##|$)/i);
+    if (patternsMatch) {
+        const patterns = [];
+        for (const line of patternsMatch[1]
+            .split('\n')
+            .filter(line => line.trim().startsWith('- `'))) {
+            const match = line.match(/- `(.+?)`(?:\s*\(flags:\s*(.+?)\))?\s*-\s*(.+)/);
+            if (match) {
+                patterns.push({
+                    description: match[3].trim(),
+                    pattern: match[1],
+                    ...(match[2] ? { flags: match[2] } : {})
+                });
+            }
+        }
+        if (patterns.length > 0) {
+            rawConcept.patterns = patterns;
+        }
+    }
     if (Object.keys(rawConcept).length === 0) {
         return undefined;
     }
@@ -105,6 +158,30 @@ function parseNarrativeSection(content) {
     if (featuresMatch) {
         narrative.features = featuresMatch[1].trim();
     }
+    const rulesMatch = sectionContent.match(/###\s*Rules\s*\n([\s\S]*?)(?=\n###\s|\n##\s|$)/i);
+    if (rulesMatch) {
+        narrative.rules = rulesMatch[1].trim();
+    }
+    const examplesMatch = sectionContent.match(/###\s*Examples\s*\n([\s\S]*?)(?=\n###\s|\n##\s|$)/i);
+    if (examplesMatch) {
+        narrative.examples = examplesMatch[1].trim();
+    }
+    const diagramsMatch = sectionContent.match(/###\s*Diagrams\s*\n([\s\S]*?)(?=\n###\s|\n##\s|$)/i);
+    if (diagramsMatch) {
+        const diagrams = [];
+        const blockRegex = /(?:\*\*(.+?)\*\*\n)?```(\w*)\n([\s\S]*?)```/g;
+        let match;
+        while ((match = blockRegex.exec(diagramsMatch[1])) !== null) {
+            diagrams.push({
+                content: match[3].trimEnd(),
+                ...(match[1] ? { title: match[1] } : {}),
+                type: match[2] || 'ascii',
+            });
+        }
+        if (diagrams.length > 0) {
+            narrative.diagrams = diagrams;
+        }
+    }
     if (Object.keys(narrative).length === 0) {
         return undefined;
     }
@@ -159,6 +236,7 @@ function mergeRawConcepts(source, target) {
     merged.task = source.task || target.task;
     merged.flow = source.flow || target.flow;
     merged.timestamp = source.timestamp || target.timestamp;
+    merged.author = source.author || target.author;
     // Arrays: concatenate and deduplicate (target first, then source)
     const allChanges = [...(target.changes || []), ...(source.changes || [])];
     if (allChanges.length > 0) {
@@ -168,6 +246,18 @@ function mergeRawConcepts(source, target) {
     if (allFiles.length > 0) {
         merged.files = [...new Set(allFiles)];
     }
+    // Patterns: concatenate and deduplicate by pattern+flags
+    const allPatterns = [...(target.patterns || []), ...(source.patterns || [])];
+    if (allPatterns.length > 0) {
+        const seen = new Set();
+        merged.patterns = allPatterns.filter(p => {
+            const key = p.pattern + (p.flags || '');
+            if (seen.has(key))
+                return false;
+            seen.add(key);
+            return true;
+        });
+    }
     if (Object.keys(merged).length === 0) {
         return undefined;
     }
@@ -194,6 +284,24 @@ function mergeNarratives(source, target) {
         const parts = [target.features, source.features].filter(Boolean);
         merged.features = parts.join('\n\n');
     }
+    if (source.rules || target.rules) {
+        const parts = [target.rules, source.rules].filter(Boolean);
+        merged.rules = parts.join('\n\n');
+    }
+    if (source.examples || target.examples) {
+        const parts = [target.examples, source.examples].filter(Boolean);
+        merged.examples = parts.join('\n\n');
+    }
+    if (source.diagrams || target.diagrams) {
+        const allDiagrams = [...(target.diagrams || []), ...(source.diagrams || [])];
+        const seen = new Set();
+        merged.diagrams = allDiagrams.filter(d => {
+            if (seen.has(d.content))
+                return false;
+            seen.add(d.content);
+            return true;
+        });
+    }
     if (Object.keys(merged).length === 0) {
         return undefined;
     }

package/dist/server/infra/executor/curate-executor.js

@@ -61,7 +61,7 @@ export class CurateExecutor {
         if (failedReads.length > 0 || skippedFiles.length > 0) {
             instructions.push('### Files that could not be read:', ...failedReads.map((r) => `- ${path.relative(projectRoot, r.filePath)}: ${r.error}`), ...skippedFiles.map((f) => `- ${f.path}: ${f.reason}`), '', '**Note:** You may use `read_file` or `grep_content` tools to find additional context if needed.', '');
         }
-        instructions.push('**INSTRUCTIONS:**', '- The file contents above have been pre-loaded for you', '- Use this content to understand the context and create comprehensive knowledge topics', '- DO NOT use read_file tool for the files above - the content is already provided', '- Proceed with the normal workflow: detect domains, find existing knowledge, create/update topics', '');
+        instructions.push('**INSTRUCTIONS:**', '- The file contents above have been pre-loaded for you', '- Use this content to understand the context and create comprehensive knowledge topics', '- DO NOT use read_file tool for the files above - the content is already provided', '- Proceed with the normal workflow: detect domains, find existing knowledge, create/update topics', '- PRESERVE all diagrams (Mermaid, PlantUML, ASCII art) verbatim using narrative.diagrams array', '- PRESERVE all tables with every row - do not summarize table data', '- PRESERVE exact code examples, API signatures, and interface definitions', '- PRESERVE step-by-step procedures and numbered instructions in narrative.rules', '');
         return instructions.join('\n');
     }
     /**

package/dist/server/infra/executor/direct-search-responder.d.ts

@@ -0,0 +1,45 @@
+/** Score at which the result is so strong that dominance check is skipped */
+export declare const DIRECT_RESPONSE_HIGH_CONFIDENCE_THRESHOLD = 15;
+/** Minimum score for the top result to qualify for a direct (no-LLM) response */
+export declare const DIRECT_RESPONSE_SCORE_THRESHOLD = 8;
+/** Top result must be N times the second result's score to be considered dominant */
+export declare const DIRECT_RESPONSE_DOMINANCE_RATIO = 2;
+/**
+ * A search result with full document content for direct response formatting.
+ */
+export interface DirectSearchResult {
+    content: string;
+    path: string;
+    score: number;
+    title: string;
+}
+/**
+ * Determines if search results are confident enough for a direct response
+ * without involving the LLM.
+ *
+ * Requires:
+ * 1. Top result score >= DIRECT_RESPONSE_SCORE_THRESHOLD (minimum confidence)
+ * 2. Either: top score >= HIGH_CONFIDENCE_THRESHOLD (strong enough to skip dominance check)
+ *    Or: top result dominates other results (score >= 2x the second result)
+ *
+ * @param results - Sorted search results (highest score first)
+ * @returns true if a direct response can be served
+ */
+export declare function canRespondDirectly(results: DirectSearchResult[]): boolean;
+/**
+ * Format a direct response from search results (no LLM involved).
+ * Uses a structured template matching the existing query response format.
+ *
+ * @param query - Original user query
+ * @param results - Search results with full content
+ * @returns Formatted response string
+ */
+export declare function formatDirectResponse(query: string, results: DirectSearchResult[]): string;
+/**
+ * Format a "not found" response when OOD detection determines
+ * the query topic is not covered in the knowledge base.
+ *
+ * @param query - Original user query
+ * @returns Formatted not-found response string
+ */
+export declare function formatNotFoundResponse(query: string): string;

package/dist/server/infra/executor/direct-search-responder.js

@@ -0,0 +1,86 @@
+/** Score at which the result is so strong that dominance check is skipped */
+export const DIRECT_RESPONSE_HIGH_CONFIDENCE_THRESHOLD = 15;
+/** Minimum score for the top result to qualify for a direct (no-LLM) response */
+export const DIRECT_RESPONSE_SCORE_THRESHOLD = 8;
+/** Top result must be N times the second result's score to be considered dominant */
+export const DIRECT_RESPONSE_DOMINANCE_RATIO = 2;
+/** Maximum content length per document in the direct response */
+const MAX_CONTENT_LENGTH = 1500;
+/** Maximum number of documents to include in the direct response */
+const MAX_DOCS = 3;
+/**
+ * Determines if search results are confident enough for a direct response
+ * without involving the LLM.
+ *
+ * Requires:
+ * 1. Top result score >= DIRECT_RESPONSE_SCORE_THRESHOLD (minimum confidence)
+ * 2. Either: top score >= HIGH_CONFIDENCE_THRESHOLD (strong enough to skip dominance check)
+ *    Or: top result dominates other results (score >= 2x the second result)
+ *
+ * @param results - Sorted search results (highest score first)
+ * @returns true if a direct response can be served
+ */
+export function canRespondDirectly(results) {
+    if (results.length === 0)
+        return false;
+    const topResult = results[0];
+    if (topResult.score < DIRECT_RESPONSE_SCORE_THRESHOLD)
+        return false;
+    // Single result that passes threshold
+    if (results.length === 1)
+        return true;
+    // High-confidence path: score so strong that dominance is irrelevant
+    if (topResult.score >= DIRECT_RESPONSE_HIGH_CONFIDENCE_THRESHOLD)
+        return true;
+    const secondScore = results[1].score;
+    if (secondScore === 0)
+        return true;
+    return topResult.score / secondScore >= DIRECT_RESPONSE_DOMINANCE_RATIO;
+}
+/**
+ * Format a direct response from search results (no LLM involved).
+ * Uses a structured template matching the existing query response format.
+ *
+ * @param query - Original user query
+ * @param results - Search results with full content
+ * @returns Formatted response string
+ */
+export function formatDirectResponse(query, results) {
+    const topResults = results.slice(0, MAX_DOCS);
+    const summary = topResults.length === 1
+        ? `Based on the curated knowledge, here is information about "${query}":`
+        : `Found ${topResults.length} relevant topics for "${query}":`;
+    const details = topResults
+        .map((r) => {
+        const truncatedContent = r.content.length > MAX_CONTENT_LENGTH ? `${r.content.slice(0, MAX_CONTENT_LENGTH).trim()}...` : r.content;
+        return `### ${r.title}\n\n${truncatedContent}`;
+    })
+        .join('\n\n---\n\n');
+    const sources = topResults.map((r) => `- \`.brv/context-tree/${r.path}\``).join('\n');
+    return `**Summary**: ${summary}
+
+**Details**:
+
+${details}
+
+**Sources**:
+${sources}
+
+**Gaps**: This is a direct match from the context tree. For deeper analysis or cross-topic synthesis, try a more specific question.`;
+}
+/**
+ * Format a "not found" response when OOD detection determines
+ * the query topic is not covered in the knowledge base.
+ *
+ * @param query - Original user query
+ * @returns Formatted not-found response string
+ */
+export function formatNotFoundResponse(query) {
+    return `**Summary**: No matching knowledge found for "${query}".
+
+**Details**: The topic does not appear to be covered in the context tree. This could mean the topic hasn't been curated yet.
+
+**Sources**: None
+
+**Gaps**: Try rephrasing your query with different terms, or use /curate to add knowledge about this topic.`;
+}

package/dist/server/infra/executor/folder-pack-executor.d.ts

@@ -6,22 +6,30 @@ import type { FolderPackExecuteOptions, IFolderPackExecutor } from '../../core/i
  *
  * This executor:
  * 1. Packs the folder using FolderPackService
- * 2. Generates XML from the pack result
- * 3. Builds a prompt for the agent to analyze and curate the folder
- * 4. Executes with the agent
+ * 2. Stores packed data in sandbox environment as context variable
+ * 3. Guides agent to iteratively query and extract knowledge
+ * 4. Agent curates extracted pieces using tools.curate()
  *
  * Architecture:
  * - TaskProcessor injects the long-lived CipherAgent
  * - Event streaming is handled by agent-worker (subscribes to agentEventBus)
  * - Transport handles task lifecycle (task:started, task:completed, task:error)
  * - Executor focuses solely on folder pack + curate execution
+ * - Uses iterative extraction strategy (inspired by rlm) to avoid token limits
  */
 export declare class FolderPackExecutor implements IFolderPackExecutor {
     private readonly folderPackService;
     constructor(folderPackService: IFolderPackService);
     executeWithAgent(agent: ICipherAgent, options: FolderPackExecuteOptions): Promise<string>;
     /**
-     * Build the analysis prompt for the agent.
+     * Build iterative extraction prompt with file-based access.
+     * Folder data is stored in a temporary file to avoid token limits.
      */
-    private buildAnalysisPrompt;
+    private buildIterativePromptWithFileAccess;
+    /**
+     * Execute folder curation using iterative extraction strategy.
+     * Pre-loads folder data into REPL environment, then guides agent to iterate and curate.
+     * This avoids token limits entirely - data is stored in REPL, not in prompt.
+     */
+    private executeIterative;
 }