byterover-cli 2.0.0 → 2.1.0

package/dist/agent/infra/storage/file-key-storage.js

@@ -0,0 +1,572 @@
+var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
+    if (value !== null && value !== void 0) {
+        if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
+        var dispose, inner;
+        if (async) {
+            if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
+            dispose = value[Symbol.asyncDispose];
+        }
+        if (dispose === void 0) {
+            if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
+            dispose = value[Symbol.dispose];
+            if (async) inner = dispose;
+        }
+        if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
+        if (inner) dispose = function() { try { inner.call(this); } catch (e) { return Promise.reject(e); } };
+        env.stack.push({ value: value, dispose: dispose, async: async });
+    }
+    else if (async) {
+        env.stack.push({ async: true });
+    }
+    return value;
+};
+var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
+    return function (env) {
+        function fail(e) {
+            env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
+            env.hasError = true;
+        }
+        var r, s = 0;
+        function next() {
+            while (r = env.stack.pop()) {
+                try {
+                    if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
+                    if (r.dispose) {
+                        var result = r.dispose.call(r.value);
+                        if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
+                    }
+                    else s |= 1;
+                }
+                catch (e) {
+                    fail(e);
+                }
+            }
+            if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
+            if (env.hasError) throw env.error;
+        }
+        return next();
+    };
+})(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
+    var e = new Error(message);
+    return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
+});
+import { randomUUID } from 'node:crypto';
+import * as fs from 'node:fs/promises';
+import { join } from 'node:path';
+import { lockKeyFromStorageKey, RWLock } from '../llm/context/rw-lock.js';
+/**
+ * File-based key storage implementation.
+ *
+ * Stores key-value pairs as individual JSON files on the filesystem.
+ * Composite keys map to directory paths:
+ *   ["message", sessionId, msgId] → {storageDir}/keystore/message/{sessionId}/{msgId}.json
+ *
+ * Features:
+ * - One file per entity (O(1) read/write/delete)
+ * - Atomic writes via write-to-temp + rename
+ * - Reader-writer locks for concurrent access (reuses global RWLock)
+ * - In-memory mode for fast unit tests
+ * - Hierarchical prefix listing via readdir
+ */
+export class FileKeyStorage {
+    baseDir;
+    initialized = false;
+    inMemory;
+    memoryStore = null;
+    storageDir;
+    constructor(config) {
+        this.inMemory = config?.inMemory ?? false;
+        this.storageDir = config?.storageDir ?? '';
+        if (!this.inMemory && !this.storageDir) {
+            throw new Error('FileKeyStorage: storageDir is required when inMemory is false');
+        }
+        this.baseDir = join(this.storageDir, 'keystore');
+    }
+    /**
+     * Execute batch operations.
+     * Operations are executed sequentially with write-temp-rename per operation.
+     * Best-effort: not ACID across multiple files, but each individual
+     * file write is atomic via rename.
+     */
+    async batch(operations) {
+        const env_1 = { stack: [], error: void 0, hasError: false };
+        try {
+            this.ensureInitialized();
+            if (operations.length === 0) {
+                return;
+            }
+            const lockKey = 'batch:global';
+            // eslint-disable-next-line @typescript-eslint/no-unused-vars
+            const _lock = __addDisposableResource(env_1, await RWLock.write(lockKey), false);
+            try {
+                for (const op of operations) {
+                    if (op.type === 'set') {
+                        const now = Date.now();
+                        // eslint-disable-next-line no-await-in-loop
+                        const existing = await this.readEnvelope(op.key);
+                        const envelope = {
+                            createdAt: existing?.createdAt ?? now,
+                            updatedAt: now,
+                            value: op.value,
+                        };
+                        // eslint-disable-next-line no-await-in-loop
+                        await this.writeEnvelope(op.key, envelope);
+                    }
+                    else if (op.type === 'delete') {
+                        // eslint-disable-next-line no-await-in-loop
+                        await this.removeFile(op.key);
+                    }
+                }
+            }
+            catch (error) {
+                throw new Error(`Batch operation failed: ${error instanceof Error ? error.message : String(error)}`);
+            }
+        }
+        catch (e_1) {
+            env_1.error = e_1;
+            env_1.hasError = true;
+        }
+        finally {
+            __disposeResources(env_1);
+        }
+    }
+    /**
+     * Close the storage. No-op for file-based storage.
+     */
+    close() {
+        if (this.inMemory) {
+            this.memoryStore = null;
+        }
+        this.initialized = false;
+    }
+    /**
+     * Delete a value by its composite key.
+     */
+    async delete(key) {
+        const env_2 = { stack: [], error: void 0, hasError: false };
+        try {
+            this.ensureInitialized();
+            this.validateKey(key);
+            const lockKey = lockKeyFromStorageKey(key);
+            // eslint-disable-next-line @typescript-eslint/no-unused-vars
+            const _lock = __addDisposableResource(env_2, await RWLock.write(lockKey), false);
+            return this.removeFile(key);
+        }
+        catch (e_2) {
+            env_2.error = e_2;
+            env_2.hasError = true;
+        }
+        finally {
+            __disposeResources(env_2);
+        }
+    }
+    /**
+     * Check if a key exists.
+     */
+    async exists(key) {
+        const env_3 = { stack: [], error: void 0, hasError: false };
+        try {
+            this.ensureInitialized();
+            this.validateKey(key);
+            const lockKey = lockKeyFromStorageKey(key);
+            // eslint-disable-next-line @typescript-eslint/no-unused-vars
+            const _lock = __addDisposableResource(env_3, await RWLock.read(lockKey), false);
+            if (this.inMemory) {
+                return this.memoryStore.has(this.serializeKey(key));
+            }
+            try {
+                await fs.access(this.keyToPath(key));
+                return true;
+            }
+            catch {
+                return false;
+            }
+        }
+        catch (e_3) {
+            env_3.error = e_3;
+            env_3.hasError = true;
+        }
+        finally {
+            __disposeResources(env_3);
+        }
+    }
+    /**
+     * Get a value by its composite key.
+     */
+    async get(key) {
+        const env_4 = { stack: [], error: void 0, hasError: false };
+        try {
+            this.ensureInitialized();
+            this.validateKey(key);
+            const lockKey = lockKeyFromStorageKey(key);
+            // eslint-disable-next-line @typescript-eslint/no-unused-vars
+            const _lock = __addDisposableResource(env_4, await RWLock.read(lockKey), false);
+            try {
+                const envelope = await this.readEnvelope(key);
+                return envelope?.value;
+            }
+            catch (error) {
+                throw new Error(`Failed to get key ${this.serializeKey(key)}: ${error instanceof Error ? error.message : String(error)}`);
+            }
+        }
+        catch (e_4) {
+            env_4.error = e_4;
+            env_4.hasError = true;
+        }
+        finally {
+            __disposeResources(env_4);
+        }
+    }
+    /**
+     * Initialize the storage backend.
+     * Creates the base keystore directory.
+     */
+    async initialize() {
+        if (this.initialized) {
+            return;
+        }
+        try {
+            if (this.inMemory) {
+                this.memoryStore = new Map();
+            }
+            else {
+                await fs.mkdir(this.baseDir, { recursive: true });
+            }
+            this.initialized = true;
+        }
+        catch (error) {
+            throw new Error(`Failed to initialize file key storage: ${error instanceof Error ? error.message : String(error)}`);
+        }
+    }
+    /**
+     * List all keys matching a prefix.
+     */
+    async list(prefix) {
+        const env_5 = { stack: [], error: void 0, hasError: false };
+        try {
+            this.ensureInitialized();
+            const lockKey = lockKeyFromStorageKey(prefix);
+            // eslint-disable-next-line @typescript-eslint/no-unused-vars
+            const _lock = __addDisposableResource(env_5, await RWLock.read(lockKey), false);
+            try {
+                if (this.inMemory) {
+                    return this.listInMemory(prefix);
+                }
+                return this.listFromDisk(prefix);
+            }
+            catch (error) {
+                throw new Error(`Failed to list keys with prefix ${this.serializeKey(prefix)}: ${error instanceof Error ? error.message : String(error)}`);
+            }
+        }
+        catch (e_5) {
+            env_5.error = e_5;
+            env_5.hasError = true;
+        }
+        finally {
+            __disposeResources(env_5);
+        }
+    }
+    /**
+     * List all key-value pairs matching a prefix.
+     * More efficient than list() followed by individual get() calls.
+     */
+    async listWithValues(prefix) {
+        const env_6 = { stack: [], error: void 0, hasError: false };
+        try {
+            this.ensureInitialized();
+            const lockKey = lockKeyFromStorageKey(prefix);
+            // eslint-disable-next-line @typescript-eslint/no-unused-vars
+            const _lock = __addDisposableResource(env_6, await RWLock.read(lockKey), false);
+            try {
+                if (this.inMemory) {
+                    return this.listWithValuesInMemory(prefix);
+                }
+                return this.listWithValuesFromDisk(prefix);
+            }
+            catch (error) {
+                throw new Error(`Failed to list keys with values for prefix ${this.serializeKey(prefix)}: ${error instanceof Error ? error.message : String(error)}`);
+            }
+        }
+        catch (e_6) {
+            env_6.error = e_6;
+            env_6.hasError = true;
+        }
+        finally {
+            __disposeResources(env_6);
+        }
+    }
+    /**
+     * Set a value at a composite key.
+     */
+    async set(key, value) {
+        const env_7 = { stack: [], error: void 0, hasError: false };
+        try {
+            this.ensureInitialized();
+            this.validateKey(key);
+            const lockKey = lockKeyFromStorageKey(key);
+            // eslint-disable-next-line @typescript-eslint/no-unused-vars
+            const _lock = __addDisposableResource(env_7, await RWLock.write(lockKey), false);
+            const now = Date.now();
+            try {
+                // Preserve createdAt if key already exists
+                const existing = await this.readEnvelope(key);
+                const envelope = {
+                    createdAt: existing?.createdAt ?? now,
+                    updatedAt: now,
+                    value,
+                };
+                await this.writeEnvelope(key, envelope);
+            }
+            catch (error) {
+                throw new Error(`Failed to set key ${this.serializeKey(key)}: ${error instanceof Error ? error.message : String(error)}`);
+            }
+        }
+        catch (e_7) {
+            env_7.error = e_7;
+            env_7.hasError = true;
+        }
+        finally {
+            __disposeResources(env_7);
+        }
+    }
+    /**
+     * Atomic update with optimistic locking.
+     */
+    async update(key, updater) {
+        const env_8 = { stack: [], error: void 0, hasError: false };
+        try {
+            this.ensureInitialized();
+            this.validateKey(key);
+            const lockKey = lockKeyFromStorageKey(key);
+            // eslint-disable-next-line @typescript-eslint/no-unused-vars
+            const _lock = __addDisposableResource(env_8, await RWLock.write(lockKey), false);
+            try {
+                const existing = await this.readEnvelope(key);
+                const currentValue = existing?.value;
+                const newValue = updater(currentValue);
+                const now = Date.now();
+                const envelope = {
+                    createdAt: existing?.createdAt ?? now,
+                    updatedAt: now,
+                    value: newValue,
+                };
+                await this.writeEnvelope(key, envelope);
+                return newValue;
+            }
+            catch (error) {
+                throw new Error(`Failed to update key ${this.serializeKey(key)}: ${error instanceof Error ? error.message : String(error)}`);
+            }
+        }
+        catch (e_8) {
+            env_8.error = e_8;
+            env_8.hasError = true;
+        }
+        finally {
+            __disposeResources(env_8);
+        }
+    }
+    // ---------------------------------------------------------------------------
+    // Private helpers
+    // ---------------------------------------------------------------------------
+    /**
+     * Deserialize a colon-separated string back to a StorageKey.
+     */
+    deserializeKey(keyStr) {
+        return keyStr.split(':');
+    }
+    ensureInitialized() {
+        if (!this.initialized) {
+            throw new Error('FileKeyStorage not initialized. Call initialize() first.');
+        }
+    }
+    /**
+     * Convert a StorageKey to a filesystem path.
+     * ["message", "abc", "msg1"] → {baseDir}/message/abc/msg1.json
+     */
+    keyToPath(key) {
+        const segments = [...key];
+        const last = segments.pop();
+        const dir = segments.length > 0 ? join(this.baseDir, ...segments) : this.baseDir;
+        return join(dir, `${last}.json`);
+    }
+    /**
+     * List keys from disk by scanning the prefix directory.
+     */
+    async listFromDisk(prefix) {
+        const prefixDir = join(this.baseDir, ...prefix);
+        const results = [];
+        try {
+            await fs.access(prefixDir);
+        }
+        catch {
+            return results;
+        }
+        await this.scanDirectory(prefixDir, prefix, results);
+        results.sort((a, b) => this.serializeKey(a).localeCompare(this.serializeKey(b)));
+        return results;
+    }
+    /**
+     * List keys from in-memory store matching a prefix.
+     */
+    listInMemory(prefix) {
+        const prefixStr = this.serializeKey(prefix);
+        const results = [];
+        for (const storedKey of this.memoryStore.keys()) {
+            if (storedKey.startsWith(prefixStr)) {
+                results.push(this.deserializeKey(storedKey));
+            }
+        }
+        results.sort((a, b) => this.serializeKey(a).localeCompare(this.serializeKey(b)));
+        return results;
+    }
+    /**
+     * List keys with values from disk.
+     */
+    async listWithValuesFromDisk(prefix) {
+        const keys = await this.listFromDisk(prefix);
+        const results = [];
+        for (const key of keys) {
+            try {
+                // eslint-disable-next-line no-await-in-loop
+                const envelope = await this.readEnvelope(key);
+                if (envelope) {
+                    results.push({ key, value: envelope.value });
+                }
+            }
+            catch {
+                // Skip corrupt or unreadable files
+            }
+        }
+        return results;
+    }
+    /**
+     * List keys with values from in-memory store.
+     */
+    listWithValuesInMemory(prefix) {
+        const prefixStr = this.serializeKey(prefix);
+        const results = [];
+        for (const [storedKey, envelope] of this.memoryStore.entries()) {
+            if (storedKey.startsWith(prefixStr)) {
+                results.push({ key: this.deserializeKey(storedKey), value: envelope.value });
+            }
+        }
+        results.sort((a, b) => this.serializeKey(a.key).localeCompare(this.serializeKey(b.key)));
+        return results;
+    }
+    /**
+     * Read the stored envelope for a key. Returns undefined if not found.
+     */
+    async readEnvelope(key) {
+        if (this.inMemory) {
+            return this.memoryStore.get(this.serializeKey(key));
+        }
+        const filePath = this.keyToPath(key);
+        try {
+            const content = await fs.readFile(filePath, 'utf8');
+            return JSON.parse(content);
+        }
+        catch (error) {
+            if (error.code === 'ENOENT') {
+                return undefined;
+            }
+            throw error;
+        }
+    }
+    /**
+     * Remove a file for the given key. Returns true if it existed.
+     */
+    async removeFile(key) {
+        if (this.inMemory) {
+            return this.memoryStore.delete(this.serializeKey(key));
+        }
+        const filePath = this.keyToPath(key);
+        try {
+            await fs.unlink(filePath);
+            return true;
+        }
+        catch (error) {
+            if (error.code === 'ENOENT') {
+                return false;
+            }
+            throw error;
+        }
+    }
+    /**
+     * Recursively scan a directory to collect all storage keys.
+     */
+    async scanDirectory(dir, prefix, results) {
+        let entries;
+        try {
+            entries = await fs.readdir(dir, { withFileTypes: true });
+        }
+        catch {
+            return;
+        }
+        for (const entry of entries) {
+            // Skip temp files from atomic writes
+            if (entry.name.includes('.tmp.')) {
+                continue;
+            }
+            const fullPath = join(dir, entry.name);
+            if (entry.isDirectory()) {
+                // eslint-disable-next-line no-await-in-loop
+                await this.scanDirectory(fullPath, [...prefix, entry.name], results);
+            }
+            else if (entry.isFile() && entry.name.endsWith('.json')) {
+                const segment = entry.name.slice(0, -5); // Remove .json
+                results.push([...prefix, segment]);
+            }
+        }
+    }
+    /**
+     * Serialize a StorageKey to a string for in-memory store lookups.
+     * Uses ':' separator (same as lockKeyFromStorageKey).
+     */
+    serializeKey(key) {
+        return key.join(':');
+    }
+    /**
+     * Validate key segments.
+     */
+    validateKey(key) {
+        if (key.length === 0) {
+            throw new Error('Storage key cannot be empty');
+        }
+        for (const segment of key) {
+            if (!segment) {
+                throw new Error('Key segment cannot be empty');
+            }
+            if (segment.includes(':')) {
+                throw new Error(`Key segment cannot contain ':': ${segment}`);
+            }
+            if (segment.includes('/') || segment.includes('\\')) {
+                throw new Error(`Key segment cannot contain path separators: ${segment}`);
+            }
+            if (segment === '..' || segment === '.') {
+                throw new Error(`Key segment cannot be '${segment}'`);
+            }
+        }
+    }
+    /**
+     * Write an envelope to disk using atomic write-to-temp + rename.
+     */
+    async writeEnvelope(key, envelope) {
+        if (this.inMemory) {
+            this.memoryStore.set(this.serializeKey(key), envelope);
+            return;
+        }
+        const filePath = this.keyToPath(key);
+        const dir = join(filePath, '..');
+        await fs.mkdir(dir, { recursive: true });
+        const tmpPath = `${filePath}.tmp.${randomUUID()}`;
+        const content = JSON.stringify(envelope, null, 2);
+        await fs.writeFile(tmpPath, content, 'utf8');
+        await fs.rename(tmpPath, filePath);
+    }
+}
+/**
+ * Factory function to create FileKeyStorage with common defaults.
+ */
+export function createFileKeyStorage(config) {
+    return new FileKeyStorage(config);
+}

package/dist/agent/infra/storage/granular-history-storage.d.ts

@@ -10,7 +10,7 @@ import { MessageStorageService } from './message-storage-service.js';
  * - Selective tool output pruning
  * - Compaction boundary markers
  *
- * This is used for NEW sessions only. Existing sessions use BlobHistoryStorage.
+ * This is the primary history storage implementation.
  */
 export declare class GranularHistoryStorage implements IHistoryStorage {
     private readonly messageStorage;

package/dist/agent/infra/storage/granular-history-storage.js

@@ -6,7 +6,7 @@
  * - Selective tool output pruning
  * - Compaction boundary markers
  *
- * This is used for NEW sessions only. Existing sessions use BlobHistoryStorage.
+ * This is the primary history storage implementation.
  */
 export class GranularHistoryStorage {
     messageStorage;

package/dist/agent/infra/system-prompt/contributors/context-tree-structure-contributor.d.ts

@@ -63,5 +63,9 @@ export declare class ContextTreeStructureContributor implements SystemPromptCont
      * @returns Instructions for using search_knowledge tool
      */
     private buildSearchKnowledgeInstructions;
+    /**
+     * Count .stub.md files inside an _archived/ directory (recursive).
+     */
+    private countArchivedEntries;
     private traverseContextTree;
 }

package/dist/agent/infra/system-prompt/contributors/context-tree-structure-contributor.js

@@ -92,7 +92,7 @@ export class ContextTreeStructureContributor {
         if (truncatedCount.value > 0) {
             parts.push('', `[${truncatedCount.value} additional entries not shown]`);
         }
-        parts.push('', '## Structure Guide', '- Each top-level folder is a **domain** (dynamically created based on content)', '- Inside domains are **topics** as `.md` files or subfolders with `context.md`', '- `context.md` files contain the curated knowledge content', '', '## Dynamic Domains', '- Domains are created dynamically based on the semantics of curated content', '- Domain names should be descriptive, use snake_case (e.g., `architecture`, `market_trends`, `risk_analysis`)', '- Before creating a new domain, check if existing domains could accommodate the content', '', '## Usage', '- **Query commands**: Search ONLY within this context tree structure', '- **Curate commands**: Check existing domains/topics before creating new ones', '</context-tree-structure>');
+        parts.push('', '## Structure Guide', '- Each top-level folder is a **domain** (dynamically created based on content)', '- Inside domains are **topics** as `.md` files or subfolders with `context.md`', '- `context.md` files contain the curated knowledge content', '- `_index.md` files are auto-generated summaries at each directory level', '- `_archived/` contains archived low-importance entries (use `expand_knowledge` to drill into them)', '', '## Dynamic Domains', '- Domains are created dynamically based on the semantics of curated content', '- Domain names should be descriptive, use snake_case (e.g., `architecture`, `market_trends`, `risk_analysis`)', '- Before creating a new domain, check if existing domains could accommodate the content', '', '## Usage', '- **Query commands**: Search ONLY within this context tree structure', '- **Curate commands**: Check existing domains/topics before creating new ones', '</context-tree-structure>');
         return parts.join('\n');
     }
     /**
@@ -168,6 +168,18 @@ export class ContextTreeStructureContributor {
             '</context-tree-structure>',
         ].join('\n');
     }
+    /**
+     * Count .stub.md files inside an _archived/ directory (recursive).
+     */
+    async countArchivedEntries(archivedDir) {
+        try {
+            const entries = await fs.readdir(archivedDir, { recursive: true });
+            return entries.filter((e) => String(e).endsWith('.stub.md')).length;
+        }
+        catch {
+            return 0;
+        }
+    }
     async traverseContextTree(options) {
         const { currentDepth, dir, entriesCount, lines, maxDepth, maxEntries, relativePath, truncatedCount } = options;
         if (currentDepth >= maxDepth) {
@@ -198,21 +210,37 @@ export class ContextTreeStructureContributor {
             entriesCount.value++;
             const entryRelativePath = relativePath ? `${relativePath}/${entry.name}` : entry.name;
             if (entry.isDirectory()) {
-                lines.push(`${indent}${entry.name}/`);
-                // eslint-disable-next-line no-await-in-loop -- Sequential traversal required for ordered output
-                await this.traverseContextTree({
-                    currentDepth: currentDepth + 1,
-                    dir: path.join(dir, entry.name),
-                    entriesCount,
-                    lines,
-                    maxDepth,
-                    maxEntries,
-                    relativePath: entryRelativePath,
-                    truncatedCount,
-                });
+                // For _archived/ directories, show entry count instead of recursing
+                if (entry.name === '_archived') {
+                    // eslint-disable-next-line no-await-in-loop -- Sequential traversal required for ordered output
+                    const archivedCount = await this.countArchivedEntries(path.join(dir, entry.name));
+                    if (archivedCount > 0) {
+                        lines.push(`${indent}_archived/ (${archivedCount} archived entries)`);
+                    }
+                }
+                else {
+                    lines.push(`${indent}${entry.name}/`);
+                    // eslint-disable-next-line no-await-in-loop -- Sequential traversal required for ordered output
+                    await this.traverseContextTree({
+                        currentDepth: currentDepth + 1,
+                        dir: path.join(dir, entry.name),
+                        entriesCount,
+                        lines,
+                        maxDepth,
+                        maxEntries,
+                        relativePath: entryRelativePath,
+                        truncatedCount,
+                    });
+                }
             }
             else {
-                const annotation = entry.name === 'context.md' ? ' (knowledge content)' : '';
+                let annotation = '';
+                if (entry.name === 'context.md') {
+                    annotation = ' (knowledge content)';
+                }
+                else if (entry.name === '_index.md') {
+                    annotation = ' (summary)';
+                }
                 lines.push(`${indent}${entry.name}${annotation}`);
             }
         }