byterover-cli 2.0.0 → 2.1.0

package/dist/agent/infra/map/worker-pool.js

@@ -0,0 +1,73 @@
+import { randomUUID } from 'node:crypto';
+// ── Worker Pool ──────────────────────────────────────────────────────────────
+/**
+ * In-memory parallel worker pool for map operations.
+ *
+ * N workers run in parallel via Promise.all(). Each worker claims items by
+ * incrementing a shared index counter (safe because JS is single-threaded
+ * for synchronous code — no cross-process races).
+ *
+ * Replaces the previous FileMapStore-backed implementation which used atomic
+ * file renames for item claiming. That pattern was ported from VoltCode's
+ * multi-process PostgreSQL architecture but is unnecessary for byterover-cli's
+ * single-process execution model.
+ *
+ * @returns Summary of the map run including a results Map (index → result)
+ */
+export async function runMapWorkerPool(options) {
+    const { abortSignal, concurrency, items, onProgress, processItem } = options;
+    const mapId = `map-${Date.now()}-${randomUUID().slice(0, 8)}`;
+    const total = items.length;
+    const results = new Map();
+    let succeededCount = 0;
+    let failedCount = 0;
+    let runningCount = 0;
+    let nextIndex = 0;
+    function emitProgress() {
+        onProgress?.({
+            failed: failedCount,
+            mapId,
+            running: runningCount,
+            succeeded: succeededCount,
+            total,
+        });
+    }
+    /**
+     * Single worker: claims items by incrementing the shared index until
+     * the queue is exhausted or the abort signal fires.
+     */
+    async function runWorker() {
+        while (!abortSignal?.aborted) {
+            const idx = nextIndex++;
+            if (idx >= items.length) {
+                break;
+            }
+            runningCount++;
+            emitProgress();
+            try {
+                // eslint-disable-next-line no-await-in-loop
+                const result = await processItem(idx, items[idx]);
+                results.set(idx, result);
+                succeededCount++;
+            }
+            catch {
+                // Store null placeholder so output JSONL maintains 1:1 line mapping with input
+                results.set(idx, null);
+                failedCount++;
+            }
+            runningCount--;
+            emitProgress();
+        }
+    }
+    // Launch worker pool — N workers run in parallel
+    const workerCount = Math.min(concurrency, total || concurrency);
+    const workers = Array.from({ length: workerCount }, () => runWorker());
+    await Promise.all(workers);
+    return {
+        failed: failedCount,
+        mapId,
+        results,
+        succeeded: succeededCount,
+        total,
+    };
+}

package/dist/agent/infra/sandbox/curation-helpers.d.ts

@@ -0,0 +1,62 @@
+/**
+ * Curation REPL Library — Pre-built helpers for curation workflow.
+ *
+ * Injected into the sandbox as `tools.curation.*` so the LLM calls
+ * these functions instead of generating identical infrastructure code
+ * (chunking loops, metadata inspection, deduplication) every curation run.
+ *
+ * All functions operate on values passed in, not variable names.
+ * - recon, chunk, detectMessageBoundaries, groupBySubject, dedup: stateless (no mutation, no I/O)
+ * - recordProgress: intentionally mutating (pushes entry into history object)
+ */
+/** Threshold below which chunking is skipped — derived from shared constant */
+export declare const SINGLE_PASS_CHAR_THRESHOLD = 20000;
+/** Valid categories — mirrors CurateFact.category from i-curate-service.ts:51 */
+export type CurationCategory = 'convention' | 'environment' | 'other' | 'personal' | 'preference' | 'project' | 'team';
+export declare const VALID_CATEGORIES: Set<string>;
+export interface CurationFact {
+    category?: CurationCategory;
+    statement: string;
+    subject?: string;
+}
+export interface ReconResult {
+    headPreview: string;
+    history: {
+        domains: Record<string, string[]>;
+        totalProcessed: number;
+    };
+    meta: {
+        charCount: number;
+        lineCount: number;
+        messageCount: number;
+    };
+    suggestedChunkCount: number;
+    suggestedMode: 'chunked' | 'single-pass';
+    tailPreview: string;
+}
+export interface ChunkResult {
+    boundaries: Array<{
+        end: number;
+        start: number;
+    }>;
+    chunks: string[];
+    totalChunks: number;
+}
+export interface MessageBoundary {
+    index: number;
+    offset: number;
+    role: string;
+}
+export declare function recon(context: string, meta: Record<string, unknown>, history: Record<string, unknown>): ReconResult;
+export declare function chunk(context: string, options?: {
+    overlap?: number;
+    size?: number;
+}): ChunkResult;
+export declare function detectMessageBoundaries(context: string): MessageBoundary[];
+export declare function groupBySubject(facts: CurationFact[]): Record<string, CurationFact[]>;
+export declare function dedup(facts: CurationFact[], threshold?: number): CurationFact[];
+export declare function recordProgress(history: Record<string, unknown>, entry: {
+    domain: string;
+    keyFacts: string[];
+    title: string;
+}): void;

package/dist/agent/infra/sandbox/curation-helpers.js

@@ -0,0 +1,219 @@
+/**
+ * Curation REPL Library — Pre-built helpers for curation workflow.
+ *
+ * Injected into the sandbox as `tools.curation.*` so the LLM calls
+ * these functions instead of generating identical infrastructure code
+ * (chunking loops, metadata inspection, deduplication) every curation run.
+ *
+ * All functions operate on values passed in, not variable names.
+ * - recon, chunk, detectMessageBoundaries, groupBySubject, dedup: stateless (no mutation, no I/O)
+ * - recordProgress: intentionally mutating (pushes entry into history object)
+ */
+import { CURATION_CHAR_THRESHOLD } from '../../../shared/constants/curation.js';
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+/** Threshold below which chunking is skipped — derived from shared constant */
+export const SINGLE_PASS_CHAR_THRESHOLD = CURATION_CHAR_THRESHOLD;
+export const VALID_CATEGORIES = new Set([
+    'convention', 'environment', 'other', 'personal', 'preference', 'project', 'team',
+]);
+// ---------------------------------------------------------------------------
+// recon — combines Steps 0-2 into one call
+// ---------------------------------------------------------------------------
+export function recon(context, meta, history) {
+    const charCount = context.length;
+    const lines = context.split('\n');
+    const lineCount = lines.length;
+    const messageCount = (context.match(/\n\n\[(USER|ASSISTANT)\]:/g) || []).length;
+    // Summarize history domains
+    const histEntries = history.entries ?? [];
+    const domains = {};
+    for (const entry of histEntries) {
+        const domain = entry.domain ?? 'unknown';
+        if (!domains[domain]) {
+            domains[domain] = [];
+        }
+        if (entry.title) {
+            domains[domain].push(entry.title);
+        }
+    }
+    const totalProcessed = history.totalProcessed ?? 0;
+    const suggestedChunkCount = Math.ceil(charCount / 8000);
+    const suggestedMode = charCount < SINGLE_PASS_CHAR_THRESHOLD ? 'single-pass' : 'chunked';
+    return {
+        headPreview: context.slice(0, 3000),
+        history: { domains, totalProcessed },
+        meta: { charCount, lineCount, messageCount },
+        suggestedChunkCount,
+        suggestedMode,
+        tailPreview: context.slice(-1000),
+    };
+}
+// ---------------------------------------------------------------------------
+// chunk — intelligent boundary-aware text splitting
+// ---------------------------------------------------------------------------
+const CODE_FENCE_REGEX = /^```/;
+export function chunk(context, options) {
+    const chunkSize = options?.size ?? 8000;
+    const overlap = options?.overlap ?? 200;
+    if (!context || context.length === 0) {
+        return { boundaries: [], chunks: [], totalChunks: 0 };
+    }
+    if (context.length <= chunkSize) {
+        return {
+            boundaries: [{ end: context.length, start: 0 }],
+            chunks: [context],
+            totalChunks: 1,
+        };
+    }
+    const chunks = [];
+    const boundaries = [];
+    let offset = 0;
+    while (offset < context.length) {
+        let end = Math.min(offset + chunkSize, context.length);
+        // If not at the end of the string, try to find a good boundary
+        if (end < context.length) {
+            end = findChunkBoundary(context, offset, end);
+        }
+        chunks.push(context.slice(offset, end));
+        boundaries.push({ end, start: offset });
+        // Advance with overlap (but never go backwards)
+        const nextOffset = end - overlap;
+        offset = nextOffset > offset ? nextOffset : end;
+        // Safety: ensure we always advance (prevents infinite loops on pathological input)
+        if (offset <= boundaries.at(-1).start) {
+            offset = end;
+        }
+    }
+    return { boundaries, chunks, totalChunks: chunks.length };
+}
+/**
+ * Find the best chunk boundary near `end` without going past `offset + maxSize`.
+ * Priority: \n\n (paragraph) > [USER]:/[ASSISTANT]: marker > \n (line) > hard cut.
+ * Never splits inside ``` code fences.
+ */
+function findChunkBoundary(context, offset, end) {
+    const searchStart = Math.max(offset + Math.floor((end - offset) * 0.5), offset);
+    const region = context.slice(searchStart, end);
+    // Check if we're inside a code fence and try to close it
+    const fencesBefore = countCodeFences(context.slice(offset, end));
+    if (fencesBefore % 2 !== 0) {
+        // Inside a code fence — look for closing fence after end
+        const closingFence = context.indexOf('```', end);
+        if (closingFence !== -1 && closingFence - offset <= (end - offset) * 1.2) {
+            // Extend to include closing fence + newline
+            const afterFence = context.indexOf('\n', closingFence);
+            return afterFence === -1 ? closingFence + 3 : afterFence + 1;
+        }
+    }
+    // Try paragraph boundary (\n\n)
+    const paraBreak = region.lastIndexOf('\n\n');
+    if (paraBreak !== -1) {
+        return searchStart + paraBreak + 2;
+    }
+    // Try message boundary ([USER]: or [ASSISTANT]:)
+    const msgPattern = /\n\[(USER|ASSISTANT)\]:/g;
+    let lastMsgMatch = null;
+    let match = null;
+    while ((match = msgPattern.exec(region)) !== null) {
+        lastMsgMatch = match;
+    }
+    if (lastMsgMatch) {
+        return searchStart + lastMsgMatch.index + 1;
+    }
+    // Try line boundary (\n)
+    const lineBreak = region.lastIndexOf('\n');
+    if (lineBreak !== -1) {
+        return searchStart + lineBreak + 1;
+    }
+    // Hard cut — guarantees forward progress
+    return end;
+}
+function countCodeFences(text) {
+    let count = 0;
+    for (const line of text.split('\n')) {
+        if (CODE_FENCE_REGEX.test(line.trim())) {
+            count++;
+        }
+    }
+    return count;
+}
+// ---------------------------------------------------------------------------
+// detectMessageBoundaries
+// ---------------------------------------------------------------------------
+const MESSAGE_BOUNDARY_REGEX = /\n\[(USER|ASSISTANT)\]:/g;
+export function detectMessageBoundaries(context) {
+    const results = [];
+    let match = null;
+    let index = 0;
+    MESSAGE_BOUNDARY_REGEX.lastIndex = 0;
+    while ((match = MESSAGE_BOUNDARY_REGEX.exec(context)) !== null) {
+        results.push({
+            index: index++,
+            offset: match.index + 1, // skip the leading \n
+            role: match[1].toLowerCase(),
+        });
+    }
+    return results;
+}
+// ---------------------------------------------------------------------------
+// groupBySubject
+// ---------------------------------------------------------------------------
+export function groupBySubject(facts) {
+    const groups = {};
+    for (const fact of facts) {
+        const key = fact.subject ?? fact.category ?? 'uncategorized';
+        if (!groups[key]) {
+            groups[key] = [];
+        }
+        groups[key].push(fact);
+    }
+    return groups;
+}
+// ---------------------------------------------------------------------------
+// dedup — word-overlap Jaccard similarity
+// ---------------------------------------------------------------------------
+export function dedup(facts, threshold = 0.85) {
+    if (facts.length <= 1) {
+        return facts;
+    }
+    const tokenized = facts.map((f) => tokenize(f.statement));
+    const keep = Array.from({ length: facts.length }, () => true);
+    for (let i = 0; i < facts.length; i++) {
+        if (!keep[i])
+            continue;
+        for (let j = i + 1; j < facts.length; j++) {
+            if (!keep[j])
+                continue;
+            if (jaccardSimilarity(tokenized[i], tokenized[j]) >= threshold) {
+                keep[j] = false;
+            }
+        }
+    }
+    return facts.filter((_, i) => keep[i]);
+}
+function tokenize(text) {
+    return new Set(text.toLowerCase().split(/\s+/).filter(Boolean));
+}
+function jaccardSimilarity(a, b) {
+    if (a.size === 0 && b.size === 0)
+        return 1;
+    let intersection = 0;
+    for (const word of a) {
+        if (b.has(word))
+            intersection++;
+    }
+    const union = a.size + b.size - intersection;
+    return union === 0 ? 0 : intersection / union;
+}
+// ---------------------------------------------------------------------------
+// recordProgress — intentionally mutating
+// ---------------------------------------------------------------------------
+export function recordProgress(history, entry) {
+    const entries = (history.entries ?? []);
+    entries.push(entry);
+    history.entries = entries;
+    const current = (history.totalProcessed ?? 0);
+    history.totalProcessed = current + 1;
+}

package/dist/agent/infra/sandbox/sandbox-service.d.ts

@@ -1,5 +1,6 @@
 import type { EnvironmentContext } from '../../core/domain/environment/types.js';
 import type { REPLResult, SandboxConfig } from '../../core/domain/sandbox/types.js';
+import type { IContentGenerator } from '../../core/interfaces/i-content-generator.js';
 import type { ICurateService } from '../../core/interfaces/i-curate-service.js';
 import type { IFileSystem } from '../../core/interfaces/i-file-system.js';
 import type { ISandboxService } from '../../core/interfaces/i-sandbox-service.js';
@@ -12,6 +13,8 @@ import type { ISearchKnowledgeService } from './tools-sdk.js';
 export declare class SandboxService implements ISandboxService {
     /** Collector wrapping curateService — captures curate() results per executeCode() call */
     private collector?;
+    /** Content generator for parallel LLM operations (mapExtract) */
+    private contentGenerator?;
     /** Curate service for Tools SDK */
     private curateService?;
     /** Environment context for sandbox injection */
@@ -20,6 +23,8 @@ export declare class SandboxService implements ISandboxService {
     private fileSystem?;
     /** Variables buffered before sandbox creation, keyed by sessionId */
     private pendingVariables;
+    /** Command type used to build each sandbox's ToolsSDK, keyed by sessionId */
+    private sandboxCommandTypes;
     /** Map of agent sessionId to LocalSandbox instance */
     private sandboxes;
     /** Search knowledge service for Tools SDK */
@@ -53,6 +58,13 @@ export declare class SandboxService implements ISandboxService {
      * @returns Execution result
      */
     executeCode(code: string, sessionId: string, config?: SandboxConfig): Promise<REPLResult>;
+    /**
+     * Set the content generator for parallel LLM operations (mapExtract).
+     * When set, new sandboxes will have access to `tools.curation.mapExtract()`.
+     *
+     * @param contentGenerator - Content generator instance
+     */
+    setContentGenerator(contentGenerator: IContentGenerator): void;
     /**
      * Set the curate service for Tools SDK injection.
      * When set, new sandboxes will have access to curate operations via `tools.curate()`.

package/dist/agent/infra/sandbox/sandbox-service.js

@@ -8,6 +8,8 @@ import { createToolsSDK } from './tools-sdk.js';
 export class SandboxService {
     /** Collector wrapping curateService — captures curate() results per executeCode() call */
     collector;
+    /** Content generator for parallel LLM operations (mapExtract) */
+    contentGenerator;
     /** Curate service for Tools SDK */
     curateService;
     /** Environment context for sandbox injection */
@@ -16,6 +18,8 @@ export class SandboxService {
     fileSystem;
     /** Variables buffered before sandbox creation, keyed by sessionId */
     pendingVariables = new Map();
+    /** Command type used to build each sandbox's ToolsSDK, keyed by sessionId */
+    sandboxCommandTypes = new Map();
     /** Map of agent sessionId to LocalSandbox instance */
     sandboxes = new Map();
     /** Search knowledge service for Tools SDK */
@@ -27,6 +31,7 @@ export class SandboxService {
      */
     async cleanup() {
         this.sandboxes.clear();
+        this.sandboxCommandTypes.clear();
         this.pendingVariables.clear();
     }
     /**
@@ -36,6 +41,7 @@ export class SandboxService {
      */
     async clearSession(sessionId) {
         this.sandboxes.delete(sessionId);
+        this.sandboxCommandTypes.delete(sessionId);
         this.pendingVariables.delete(sessionId);
     }
     /**
@@ -66,7 +72,22 @@ export class SandboxService {
     async executeCode(code, sessionId, config) {
         // Get or create sandbox for this agent session
         let sandbox = this.sandboxes.get(sessionId);
-        if (!sandbox) {
+        if (sandbox) {
+            // Hot-swap ToolsSDK if commandType changed (security: enforce read-only on transition)
+            const previousCommandType = this.sandboxCommandTypes.get(sessionId);
+            if (config?.commandType !== previousCommandType) {
+                const newToolsSDK = this.buildToolsSDK(sessionId, config?.commandType);
+                if (newToolsSDK) {
+                    sandbox.updateContext({ tools: newToolsSDK });
+                }
+                this.sandboxCommandTypes.set(sessionId, config?.commandType);
+            }
+            // Update context if provided
+            if (config?.contextPayload) {
+                sandbox.updateContext({ context: config.contextPayload });
+            }
+        }
+        else {
             // First execution for this session - create new sandbox
             const initialContext = {};
             if (config?.contextPayload) {
@@ -79,17 +100,14 @@ export class SandboxService {
                 this.pendingVariables.delete(sessionId);
             }
             // Build per-session ToolsSDK (includes agentQuery bound to this sessionId)
-            const sessionToolsSDK = this.buildToolsSDK(sessionId);
+            const sessionToolsSDK = this.buildToolsSDK(sessionId, config?.commandType);
             sandbox = new LocalSandbox({
                 environmentContext: this.environmentContext,
                 initialContext,
                 toolsSDK: sessionToolsSDK,
             });
             this.sandboxes.set(sessionId, sandbox);
-        }
-        else if (config?.contextPayload) {
-            // Update context if provided
-            sandbox.updateContext({ context: config.contextPayload });
+            this.sandboxCommandTypes.set(sessionId, config?.commandType);
         }
         if (this.collector) {
             const { curateResults, result } = await this.collector.collect(() => sandbox.execute(code, config));
@@ -97,6 +115,16 @@ export class SandboxService {
         }
         return sandbox.execute(code, config);
     }
+    /**
+     * Set the content generator for parallel LLM operations (mapExtract).
+     * When set, new sandboxes will have access to `tools.curation.mapExtract()`.
+     *
+     * @param contentGenerator - Content generator instance
+     */
+    setContentGenerator(contentGenerator) {
+        this.contentGenerator = contentGenerator;
+        this.invalidateSandboxes();
+    }
     /**
      * Set the curate service for Tools SDK injection.
      * When set, new sandboxes will have access to curate operations via `tools.curate()`.
@@ -118,6 +146,7 @@ export class SandboxService {
         this.environmentContext = environmentContext;
         // Clear existing sandboxes so new ones get the updated environment
         this.sandboxes.clear();
+        this.sandboxCommandTypes.clear();
     }
     /**
      * Set the file system service for Tools SDK injection.
@@ -176,11 +205,13 @@ export class SandboxService {
      * Build a Tools SDK instance for a specific session.
      * Includes `agentQuery` bound to the session's ID for sub-agent delegation.
      */
-    buildToolsSDK(sessionId) {
+    buildToolsSDK(sessionId, commandType) {
         if (!this.fileSystem) {
             return undefined;
         }
         return createToolsSDK({
+            commandType,
+            contentGenerator: this.contentGenerator,
             curateService: this.curateService,
             fileSystem: this.fileSystem,
             parentSessionId: sessionId,
@@ -196,6 +227,7 @@ export class SandboxService {
     invalidateSandboxes() {
         if (this.fileSystem) {
             this.sandboxes.clear();
+            this.sandboxCommandTypes.clear();
         }
     }
 }

package/dist/agent/infra/sandbox/tools-sdk.d.ts

@@ -1,8 +1,10 @@
 import type { FileContent, GlobResult, ListDirectoryResult, SearchResult, WriteResult } from '../../core/domain/file-system/types.js';
+import type { IContentGenerator } from '../../core/interfaces/i-content-generator.js';
 import type { CurateOperation, CurateOptions, CurateResult, DetectDomainsInput, DetectDomainsResult, ICurateService } from '../../core/interfaces/i-curate-service.js';
 import type { IFileSystem } from '../../core/interfaces/i-file-system.js';
 import type { ISandboxService } from '../../core/interfaces/i-sandbox-service.js';
 import type { SessionManager } from '../session/session-manager.js';
+import { type ChunkResult, type CurationFact, type MessageBoundary, type ReconResult } from './curation-helpers.js';
 /**
  * Options for glob operation in ToolsSDK.
  */
@@ -67,6 +69,8 @@ export interface SearchKnowledgeOptions {
 export interface SearchKnowledgeResult {
     message: string;
     results: Array<{
+        /** For archive stubs: path to full content (for expand_knowledge tool) */
+        archiveFullPath?: string;
         /** Number of other memories that reference this one */
         backlinkCount?: number;
         excerpt: string;
@@ -74,7 +78,7 @@ export interface SearchKnowledgeResult {
         /** Top backlink source paths (max 3) */
         relatedPaths?: string[];
         score: number;
-        /** Symbol kind: 'domain' | 'topic' | 'subtopic' | 'context' */
+        /** Symbol kind: 'domain' | 'topic' | 'subtopic' | 'context' | 'archive_stub' */
         symbolKind?: string;
         /** Resolved hierarchical path in the symbol tree */
         symbolPath?: string;
@@ -100,6 +104,7 @@ export interface ToolsSDK {
      * Only the final response string flows back.
      * @param prompt - Prompt for the sub-agent
      * @param options - Optional limits
+     * @param options.contextData - Optional key/value context injected into child session sandbox
      * @param options.maxIterations - Maximum agentic iterations (default: 5)
      * @returns Promise resolving to the sub-agent's final response
      */
@@ -115,6 +120,44 @@ export interface ToolsSDK {
      * @returns Promise resolving to curate result with applied operations and summary
      */
     curate(operations: CurateOperation[], options?: CurateOptions): Promise<CurateResult>;
+    /**
+     * Pre-built curation helpers — reduces LLM iteration overhead.
+     * Stateless functions except recordProgress (intentionally mutating).
+     */
+    readonly curation: {
+        /** Intelligent boundary-aware text splitting */
+        chunk(context: string, options?: {
+            overlap?: number;
+            size?: number;
+        }): ChunkResult;
+        /** Remove near-duplicate facts using Jaccard word-overlap similarity */
+        dedup(facts: CurationFact[], threshold?: number): CurationFact[];
+        /** Find [USER]: and [ASSISTANT]: markers with offsets */
+        detectMessageBoundaries(context: string): MessageBoundary[];
+        /** Group facts by subject, with fallback to category */
+        groupBySubject(facts: CurationFact[]): Record<string, CurationFact[]>;
+        /** Parallel LLM extraction over chunked context. Curate mode only. */
+        mapExtract(context: string, options: {
+            chunkSize?: number;
+            concurrency?: number;
+            maxContextTokens?: number;
+            prompt: string;
+            taskId?: string;
+        }): Promise<{
+            facts: CurationFact[];
+            failed: number;
+            succeeded: number;
+            total: number;
+        }>;
+        /** Combine Steps 0-2 into one call: metadata + history + preview + mode recommendation */
+        recon(context: string, meta: Record<string, unknown>, history: Record<string, unknown>): ReconResult;
+        /** Push entry into history and increment totalProcessed (intentionally mutating) */
+        recordProgress(history: Record<string, unknown>, entry: {
+            domain: string;
+            keyFacts: string[];
+            title: string;
+        }): void;
+    };
     /**
      * Detect and validate domains from input data.
      * Use this to analyze text and categorize it into knowledge domains.
@@ -170,6 +213,10 @@ export interface ToolsSDK {
  * Options for creating a Tools SDK instance.
  */
 export interface CreateToolsSDKOptions {
+    /** Command type — when 'query', mutating APIs (curate, writeFile) are disabled */
+    commandType?: string;
+    /** Content generator for parallel LLM operations (mapExtract) */
+    contentGenerator?: IContentGenerator;
     /** Curate service for knowledge curation */
     curateService?: ICurateService;
     /** File system service for file operations */