byterover-cli 1.8.0 → 2.1.0

package/dist/agent/infra/map/agentic-map-service.d.ts

@@ -0,0 +1,97 @@
+import type { ICipherAgent } from '../../core/interfaces/i-cipher-agent.js';
+import type { ILogger } from '../../core/interfaces/i-logger.js';
+import type { ContextTreeStore } from './context-tree-store.js';
+import { type AgenticMapParameters } from './map-shared.js';
+import { type MapProgress, type MapRunResult } from './worker-pool.js';
+/** Absolute depth ceiling — cannot be exceeded regardless of LLM-supplied max_depth */
+export declare const HARD_MAX_DEPTH = 3;
+interface NestingRecord {
+    absoluteMaxDepth: number;
+    ancestorInputPaths: ReadonlySet<string>;
+    isRootCaller: boolean;
+    mapRunId: string;
+    nestingDepth: number;
+    /** Agent instance that registered this root-eligible session (undefined for sub-sessions) */
+    ownerId?: string;
+}
+/** Read-only accessor — raw map is not exported */
+export declare function getNestingRecord(sessionId: string): Readonly<NestingRecord> | undefined;
+/**
+ * Register a session as root-eligible for write-enabled agentic_map calls.
+ * Must be called at session creation time (e.g., CipherAgent.start/createSession).
+ * Idempotent for already-registered root callers with the same ownerId.
+ * Throws if the session is already registered as a sub-session (isRootCaller: false) —
+ * this indicates a bug (task-session ID passed to a root-eligible creation path).
+ * Throws if the session is already registered to a DIFFERENT ownerId — this
+ * indicates a cross-agent session ID collision and must fail fast.
+ *
+ * @param sessionId - Session to register
+ * @param ownerId - Unique identifier for the agent instance that owns this registration.
+ *   Used to scope deregistration: only the owning agent can remove the record.
+ */
+export declare function registerRootEligibleSession(sessionId: string, ownerId: string): void;
+/**
+ * Remove a root-eligible session registration.
+ * Call when a session registered via registerRootEligibleSession() is deleted.
+ * Only deletes if the record's ownerId matches the caller — prevents one agent
+ * instance from removing another agent's live record.
+ * Safe to call for unregistered sessions (no-op).
+ *
+ * @param sessionId - Session to deregister
+ * @param ownerId - Must match the ownerId used at registration time
+ */
+export declare function deregisterRootEligibleSession(sessionId: string, ownerId: string): void;
+/** FOR TEST USE ONLY — do not call in production code */
+export declare function _resetNestingRegistryForTests(): void;
+/** FOR TEST USE ONLY — inject a sub-session record into the nesting registry */
+export declare function _setNestingRecordForTests(sessionId: string, record: {
+    absoluteMaxDepth: number;
+    ancestorInputPaths: ReadonlySet<string>;
+    isRootCaller: boolean;
+    mapRunId: string;
+    nestingDepth: number;
+}): void;
+/** FOR TEST USE ONLY — inject entries into the runSessionIndex */
+export declare function _setRunSessionIndexForTests(mapRunId: string, sessionIds: Set<string>): void;
+/** FOR TEST USE ONLY — expose cleanupMapRun for registry lifecycle tests */
+export declare function _cleanupMapRunForTests(mapRunId: string): void;
+export interface AgenticMapServiceOptions {
+    /** Abort signal for cancellation */
+    abortSignal?: AbortSignal;
+    /** The cipher agent instance for creating sub-sessions */
+    agent: ICipherAgent;
+    /** Internal: canonical input paths of all ancestor calls (anti-cycle) */
+    ancestorInputPaths?: ReadonlySet<string>;
+    /** Optional context tree store for result aggregation */
+    contextTreeStore?: ContextTreeStore;
+    /** Internal: root's clamped max depth, inherited by descendants */
+    effectiveMaxDepth?: number;
+    /** Optional logger for fail-open warnings */
+    logger?: ILogger;
+    /** Internal: root run ID for bulk sub-session cleanup */
+    mapRunId?: string;
+    /** Internal: nesting depth of this call (0 = root) */
+    nestingDepth?: number;
+    /** Progress callback */
+    onProgress?: (progress: MapProgress) => void;
+    /** Tool parameters from the LLM */
+    params: AgenticMapParameters;
+    /** Task ID for event routing */
+    taskId?: string;
+    /** Working directory (project root) */
+    workingDirectory: string;
+}
+/**
+ * Execute an Agentic-Map: parallel sub-agent sessions over a JSONL file.
+ *
+ * For each item (line), spawns a full agent session with tool access.
+ * The sub-agent must output a JSON value that validates against the provided
+ * output schema.
+ *
+ * Ported from VoltCode's agentic-map.ts, adapted for byterover-cli:
+ * - Uses agent.createTaskSession() / agent.executeOnSession() instead of Session.create()
+ * - Uses in-memory worker pool (no FileMapStore / PostgreSQL)
+ * - Concurrency capped at 4 (CLI runs on user machines)
+ */
+export declare function executeAgenticMap(options: AgenticMapServiceOptions): Promise<MapRunResult>;
+export {};

package/dist/agent/infra/map/agentic-map-service.js

@@ -0,0 +1,309 @@
+import { randomUUID } from 'node:crypto';
+import { writeFile } from 'node:fs/promises';
+import { buildRecursiveCompositionGuidance, buildUserMessage, canonicalizePath, parseJsonlFile, resolveAndValidatePath, validateAgainstSchema, withTimeout, } from './map-shared.js';
+import { runMapWorkerPool } from './worker-pool.js';
+// ── Constants ────────────────────────────────────────────────────────────────
+/** Absolute depth ceiling — cannot be exceeded regardless of LLM-supplied max_depth */
+export const HARD_MAX_DEPTH = 3;
+/** Max parallel sub-agent sessions (lower than VoltCode's 16 for CLI machines) */
+const DEFAULT_CONCURRENCY = 4;
+const nestingRegistry = new Map();
+const runSessionIndex = new Map();
+/** Read-only accessor — raw map is not exported */
+export function getNestingRecord(sessionId) {
+    return nestingRegistry.get(sessionId);
+}
+/**
+ * Register a session as root-eligible for write-enabled agentic_map calls.
+ * Must be called at session creation time (e.g., CipherAgent.start/createSession).
+ * Idempotent for already-registered root callers with the same ownerId.
+ * Throws if the session is already registered as a sub-session (isRootCaller: false) —
+ * this indicates a bug (task-session ID passed to a root-eligible creation path).
+ * Throws if the session is already registered to a DIFFERENT ownerId — this
+ * indicates a cross-agent session ID collision and must fail fast.
+ *
+ * @param sessionId - Session to register
+ * @param ownerId - Unique identifier for the agent instance that owns this registration.
+ *   Used to scope deregistration: only the owning agent can remove the record.
+ */
+export function registerRootEligibleSession(sessionId, ownerId) {
+    const existing = nestingRegistry.get(sessionId);
+    if (existing !== undefined) {
+        if (!existing.isRootCaller) {
+            throw new Error(`registerRootEligibleSession: session "${sessionId}" is already registered as a ` +
+                `sub-session. Cannot promote to root-eligible. This is a bug.`);
+        }
+        if (existing.ownerId !== ownerId) {
+            throw new Error(`registerRootEligibleSession: session "${sessionId}" is already owned by a different ` +
+                `agent instance. Refusing to share root-eligible registration across owners.`);
+        }
+        return; // already root-eligible for same owner — idempotent no-op
+    }
+    nestingRegistry.set(sessionId, {
+        absoluteMaxDepth: 0, // not used for root callers
+        ancestorInputPaths: new Set(),
+        isRootCaller: true,
+        mapRunId: '', // not used; re-derived per invocation
+        nestingDepth: 0,
+        ownerId,
+    });
+}
+/**
+ * Remove a root-eligible session registration.
+ * Call when a session registered via registerRootEligibleSession() is deleted.
+ * Only deletes if the record's ownerId matches the caller — prevents one agent
+ * instance from removing another agent's live record.
+ * Safe to call for unregistered sessions (no-op).
+ *
+ * @param sessionId - Session to deregister
+ * @param ownerId - Must match the ownerId used at registration time
+ */
+export function deregisterRootEligibleSession(sessionId, ownerId) {
+    const record = nestingRegistry.get(sessionId);
+    if (record?.isRootCaller && record.ownerId === ownerId) {
+        nestingRegistry.delete(sessionId);
+    }
+}
+/** FOR TEST USE ONLY — do not call in production code */
+export function _resetNestingRegistryForTests() {
+    nestingRegistry.clear();
+    runSessionIndex.clear();
+}
+/** FOR TEST USE ONLY — inject a sub-session record into the nesting registry */
+export function _setNestingRecordForTests(sessionId, record) {
+    nestingRegistry.set(sessionId, record);
+}
+/** FOR TEST USE ONLY — inject entries into the runSessionIndex */
+export function _setRunSessionIndexForTests(mapRunId, sessionIds) {
+    runSessionIndex.set(mapRunId, sessionIds);
+}
+/** FOR TEST USE ONLY — expose cleanupMapRun for registry lifecycle tests */
+export function _cleanupMapRunForTests(mapRunId) {
+    cleanupMapRun(mapRunId);
+}
+/** Bulk cleanup for a completed root run. Called ONLY at nestingDepth === 0. */
+function cleanupMapRun(mapRunId) {
+    const sids = runSessionIndex.get(mapRunId);
+    if (sids) {
+        for (const sid of sids) {
+            nestingRegistry.delete(sid); // all entries here are isRootCaller: false
+        }
+        runSessionIndex.delete(mapRunId);
+    }
+}
+// ── Agentic-Map Service ──────────────────────────────────────────────────────
+/**
+ * Execute an Agentic-Map: parallel sub-agent sessions over a JSONL file.
+ *
+ * For each item (line), spawns a full agent session with tool access.
+ * The sub-agent must output a JSON value that validates against the provided
+ * output schema.
+ *
+ * Ported from VoltCode's agentic-map.ts, adapted for byterover-cli:
+ * - Uses agent.createTaskSession() / agent.executeOnSession() instead of Session.create()
+ * - Uses in-memory worker pool (no FileMapStore / PostgreSQL)
+ * - Concurrency capped at 4 (CLI runs on user machines)
+ */
+export async function executeAgenticMap(options) {
+    const { abortSignal, agent, onProgress, params, taskId, workingDirectory, } = options;
+    const { input_path: inputPath, max_attempts: maxAttempts = 3, output_path: outputPath, output_schema: outputSchema, prompt, read_only: readOnly = true, timeout_seconds: timeoutSeconds = 300, } = params;
+    // Destructure internal nesting options
+    const nestingDepth = options.nestingDepth ?? 0;
+    // Always clamp to HARD_MAX_DEPTH — defense-in-depth even if caller supplies a higher value
+    const effectiveMaxDepth = Math.min(options.effectiveMaxDepth ?? Math.min(params.max_depth ?? 1, HARD_MAX_DEPTH), HARD_MAX_DEPTH);
+    const ancestorInputPaths = options.ancestorInputPaths ?? new Set();
+    const mapRunId = options.mapRunId ?? randomUUID();
+    // Resolve paths relative to working directory and validate they don't escape it
+    const resolvedInputPath = resolveAndValidatePath(workingDirectory, inputPath);
+    const resolvedOutputPath = resolveAndValidatePath(workingDirectory, outputPath);
+    const canonicalInputPath = canonicalizePath(resolvedInputPath);
+    // Anti-cycle — unconditional (defense-in-depth regardless of read_only and allowlist state)
+    if (ancestorInputPaths.has(canonicalInputPath)) {
+        throw new Error(`agentic_map: cycle — input_path "${inputPath}" is already being processed ` +
+            `by an ancestor call. Use a distinct JSONL file at each nesting level.`);
+    }
+    // Depth — unconditional
+    if (nestingDepth >= effectiveMaxDepth) {
+        throw new Error(`agentic_map: nesting depth ${nestingDepth} has reached max_depth ${effectiveMaxDepth} ` +
+            `(hard ceiling: ${HARD_MAX_DEPTH}).`);
+    }
+    // Depth-aware concurrency
+    const concurrency = Math.max(1, Math.floor(DEFAULT_CONCURRENCY / (nestingDepth + 1)));
+    // Always include current input path in child ancestor set (covers read_only branches too)
+    const childAncestorPaths = new Set([canonicalInputPath, ...ancestorInputPaths]);
+    // 1. Parse input JSONL
+    const items = await parseJsonlFile(resolvedInputPath);
+    if (items.length === 0) {
+        await writeFile(resolvedOutputPath, '', 'utf8');
+        return { failed: 0, mapId: 'empty', succeeded: 0, total: 0 };
+    }
+    // 2. Prepare run metadata
+    const runStartedAt = new Date().toISOString();
+    // Track created sessions for cleanup
+    const sessionIds = [];
+    try {
+        // 3. Define per-item processing function
+        async function processItem(itemIndex, item) {
+            // Create a per-item task session
+            // When read_only, use 'query' command type to restrict sub-agent to read-only tools
+            const itemTaskId = `map-item-${itemIndex}-${randomUUID().slice(0, 8)}`;
+            const sessionCommandType = readOnly ? 'query' : 'curate';
+            // mapRootEligible intentionally omitted — sub-sessions are NOT root-eligible.
+            // The isRootCaller: false record is set directly below.
+            const sessionId = await agent.createTaskSession(itemTaskId, sessionCommandType);
+            sessionIds.push(sessionId);
+            // Register ALL sub-sessions for depth tracking (defense against future allowlist changes)
+            nestingRegistry.set(sessionId, {
+                absoluteMaxDepth: effectiveMaxDepth,
+                ancestorInputPaths: childAncestorPaths,
+                isRootCaller: false,
+                mapRunId,
+                nestingDepth: nestingDepth + 1,
+            });
+            if (!runSessionIndex.has(mapRunId))
+                runSessionIndex.set(mapRunId, new Set());
+            runSessionIndex.get(mapRunId).add(sessionId);
+            // Build user message
+            let userMessageText = buildUserMessage(prompt, 'pending', runStartedAt, itemIndex, item, outputSchema);
+            // Append guidance when further nesting is possible (write-enabled only)
+            const canNest = !readOnly && (nestingDepth + 1) < effectiveMaxDepth;
+            if (canNest) {
+                userMessageText += '\n\n' + buildRecursiveCompositionGuidance(nestingDepth + 1, effectiveMaxDepth);
+            }
+            // Per-item timeout
+            const timeoutController = new AbortController();
+            const timeoutHandle = setTimeout(() => {
+                timeoutController.abort();
+            }, timeoutSeconds * 1000);
+            try {
+                let attemptsUsed = 1;
+                // Full agentic prompt (with tool access, multi-step reasoning)
+                let result = await withTimeout(agent.executeOnSession(sessionId, userMessageText, {
+                    executionContext: {
+                        clearHistory: true,
+                        commandType: sessionCommandType,
+                        maxIterations: readOnly ? 10 : 20,
+                    },
+                    taskId: taskId ?? randomUUID(),
+                }), timeoutController.signal);
+                // Validation loop with retry
+                while (true) {
+                    // Extract JSON from the response
+                    let parsed;
+                    let lastError = '';
+                    // Try to parse the entire response as JSON first
+                    try {
+                        parsed = JSON.parse(result);
+                    }
+                    catch {
+                        // Try to extract JSON block from response
+                        const jsonMatch = result.match(/```json\s*\n([\s\S]*?)\n```/);
+                        if (jsonMatch) {
+                            try {
+                                parsed = JSON.parse(jsonMatch[1]);
+                            }
+                            catch (error) {
+                                lastError = `JSON parse error from code block: ${error instanceof Error ? error.message : String(error)}`;
+                            }
+                        }
+                        else {
+                            // Try to find JSON object/array in the response
+                            const jsonObjMatch = result.match(/(\{[\s\S]*\}|\[[\s\S]*\])/);
+                            if (jsonObjMatch) {
+                                try {
+                                    parsed = JSON.parse(jsonObjMatch[1]);
+                                }
+                                catch (error) {
+                                    lastError = `JSON parse error from extraction: ${error instanceof Error ? error.message : String(error)}`;
+                                }
+                            }
+                            else {
+                                lastError = 'No JSON found in response';
+                            }
+                        }
+                    }
+                    // Validate against schema
+                    if (parsed !== undefined) {
+                        const validation = validateAgainstSchema(parsed, outputSchema);
+                        if (validation.valid) {
+                            // Fail-open: store result in context tree if available
+                            if (options.contextTreeStore) {
+                                try {
+                                    options.contextTreeStore.store(itemIndex, JSON.stringify(parsed));
+                                }
+                                catch (storeError) {
+                                    options.logger?.warn('Context tree store failed', { error: String(storeError), itemIndex });
+                                }
+                            }
+                            return parsed;
+                        }
+                        lastError = `Schema validation failed: ${validation.error}`;
+                    }
+                    // Check retry budget
+                    if (attemptsUsed >= maxAttempts) {
+                        throw new Error(`Failed after ${attemptsUsed} attempts. Last error: ${lastError}`);
+                    }
+                    // Check abort
+                    if (abortSignal?.aborted || timeoutController.signal.aborted) {
+                        throw new Error('Aborted or timed out');
+                    }
+                    // Retry by sending validation error back to the SAME session
+                    attemptsUsed++;
+                    const retryPrompt = [
+                        `Validation failed: ${lastError}`,
+                        '',
+                        'Respond with corrected JSON only. No explanations, no markdown fences.',
+                    ].join('\n');
+                    // eslint-disable-next-line no-await-in-loop
+                    result = await withTimeout(agent.executeOnSession(sessionId, retryPrompt, {
+                        executionContext: { commandType: sessionCommandType, maxIterations: 5 },
+                        taskId: taskId ?? randomUUID(),
+                    }), timeoutController.signal);
+                }
+            }
+            finally {
+                clearTimeout(timeoutHandle);
+                // Do NOT delete sessions or registry here.
+                // Session deletion is handled by the outer finally (Promise.allSettled)
+                // to avoid a double-dispose race with concurrent deleteTaskSession calls.
+                // Registry cleanup is done exclusively by cleanupMapRun at root,
+                // ensuring orphaned sessions (withTimeout race) retain their depth records
+                // until the root run has fully completed.
+            }
+        }
+        // 4. Run in-memory worker pool
+        const result = await runMapWorkerPool({
+            abortSignal,
+            concurrency,
+            items,
+            onProgress,
+            processItem,
+        });
+        // 5. Compact context tree and attach summaryHandle (fail-open)
+        if (options.contextTreeStore) {
+            try {
+                await options.contextTreeStore.compact();
+                result.summaryHandle = options.contextTreeStore.getSummaryHandle();
+            }
+            catch (compactError) {
+                options.logger?.warn('Context tree compaction failed', { error: String(compactError) });
+            }
+        }
+        // 6. Write output JSONL from in-memory results (sorted by index)
+        const sorted = [...result.results.entries()].sort(([a], [b]) => a - b);
+        const outputContent = sorted.map(([, r]) => JSON.stringify(r)).join('\n');
+        await writeFile(resolvedOutputPath, outputContent, 'utf8');
+        return result;
+    }
+    finally {
+        // Await all session deletions before cleaning registry.
+        // Reduces the race window where orphaned sessions could call agentic_map
+        // after their records are wiped.
+        await Promise.allSettled(sessionIds.map((sid) => agent.deleteTaskSession(sid)));
+        // Bulk cleanup only at root — nested levels must NOT call this,
+        // as it would wipe sibling sessions still running at the same depth.
+        if (nestingDepth === 0) {
+            cleanupMapRun(mapRunId);
+        }
+    }
+}

package/dist/agent/infra/map/context-tree-store.d.ts

@@ -0,0 +1,94 @@
+/**
+ * Context Tree Store.
+ *
+ * In-memory store implementing bounded buffer with automatic compaction.
+ * Two-tier architecture: synchronous buffering with bounded eviction + async final compaction.
+ *
+ * Hot path (store): synchronous, no LLM calls. If buffer exceeds τ_hard,
+ * entries are evicted, summaries consolidated, and single summaries truncated
+ * via deterministic truncation in a loop until totalTokens ≤ τ_hard or no
+ * further reduction is possible (single entry with no summary).
+ *
+ * Cold path (compact): called ONCE after worker pool completes.
+ * Runs full 3-level escalation to produce high-quality summary.
+ *
+ * Memory bound: after store() completes, totalTokens ≤ τ_hard + maxSingleLabeledEntrySize
+ * where labeled entry size = countTokens("[Item N]: content").
+ * The single-entry overshoot covers the item that triggered eviction.
+ */
+import type { IContentGenerator } from '../../core/interfaces/i-content-generator.js';
+import type { ITokenizer } from '../../core/interfaces/i-tokenizer.js';
+/**
+ * Options for ContextTreeStore.
+ */
+export interface ContextTreeStoreOptions {
+    /** IContentGenerator for LLM summarization in compact() */
+    generator: IContentGenerator;
+    /** Maximum compaction rounds to prevent infinite loops (default: 10) */
+    maxCompactionRounds?: number;
+    /** Maximum tokens for the final summaryHandle (default: 2000) */
+    summaryBudget?: number;
+    /** Hard limit triggering synchronous eviction */
+    tauHard: number;
+    /** Tokenizer for token counting */
+    tokenizer: ITokenizer;
+}
+/**
+ * Context Tree Store with bounded buffer and 3-level compaction.
+ */
+export declare class ContextTreeStore {
+    private readonly entries;
+    private readonly generator;
+    private readonly maxCompactionRounds;
+    private readonly summaries;
+    private readonly summaryBudget;
+    private summaryHandle;
+    private summaryTokens;
+    private readonly tauHard;
+    private readonly tokenizer;
+    private totalTokens;
+    constructor(options: ContextTreeStoreOptions);
+    /**
+     * Canonical entry format — single source of truth for labeled entries.
+     * Used in store(), evictOldest(), and compact().
+     */
+    private static formatEntry;
+    /**
+     * Cold path — called ONCE after worker pool completes.
+     * Runs full 3-level escalation (may involve LLM calls) to produce
+     * high-quality summary from remaining entries + prior summaries.
+     */
+    compact(): Promise<void>;
+    /**
+     * Returns compact summary text, bounded to summaryBudget tokens.
+     * Must call compact() first for LLM-quality output.
+     */
+    getSummaryHandle(): string | undefined;
+    /**
+     * Hot path — called from processItem(). Synchronous, no LLM calls.
+     * If buffer exceeds τ_hard, evicts entries and consolidates summaries
+     * until totalTokens ≤ τ_hard or no further reduction is possible.
+     */
+    store(index: number, content: string): void;
+    /**
+     * Consolidate all summaries into a single truncated summary.
+     * Called when entries alone can't bring totalTokens below τ_hard.
+     * Synchronous — uses deterministic truncation only.
+     */
+    private consolidateSummaries;
+    /**
+     * Synchronous eviction of oldest entries via deterministic truncation.
+     * Truncates oldest half of entries into a compact summary chunk.
+     */
+    private evictOldest;
+    /**
+     * Truncate a single remaining summary to fit within budget.
+     * Called when entries.size <= 1 and summaries.length === 1 but still over τ_hard.
+     * Halves the summary via deterministic truncation each round.
+     */
+    private truncateSingleSummary;
+    /**
+     * Attempt LLM summarization for compact().
+     */
+    private tryLlmSummarization;
+}