byterover-cli 2.0.0 → 2.1.0

package/dist/server/infra/context-tree/file-context-tree-manifest-service.js

@@ -0,0 +1,278 @@
+/* eslint-disable camelcase */
+/**
+ * File-based implementation of IContextTreeManifestService.
+ *
+ * Builds and reads the context manifest (_manifest.json) which allocates
+ * context tree entries into three lanes (summaries, contexts, stubs)
+ * with token budgets for efficient query context injection.
+ *
+ * Freshness check uses source_fingerprint (hash of sorted path:mtime:size)
+ * to detect additions, modifications, and deletions. Stat-only, no file reads.
+ *
+ * Tradeoff: rare false-fresh cases where content changes but mtime+size
+ * are preserved. Acceptable because writeFile() updates mtime and the next
+ * curate run will rebuild the manifest anyway.
+ */
+import { readdir, readFile, stat, writeFile } from 'node:fs/promises';
+import { join, relative } from 'node:path';
+import { ARCHIVE_DIR, BRV_DIR, CONTEXT_FILE_EXTENSION, CONTEXT_TREE_DIR, MANIFEST_FILE, STUB_EXTENSION, SUMMARY_INDEX_FILE, } from '../../constants.js';
+import { parseFrontmatterScoring } from '../../core/domain/knowledge/markdown-writer.js';
+import { DEFAULT_LANE_BUDGETS } from '../../core/domain/knowledge/summary-types.js';
+import { estimateTokens } from '../executor/pre-compaction/compaction-escalation.js';
+import { isArchiveStub, isDerivedArtifact } from './derived-artifact.js';
+import { computeContentHash } from './hash-utils.js';
+import { toUnixPath } from './path-utils.js';
+import { parseSummaryFrontmatter } from './summary-frontmatter.js';
+export class FileContextTreeManifestService {
+    config;
+    constructor(config = {}) {
+        this.config = config;
+    }
+    async buildManifest(directory, laneBudgets) {
+        const baseDir = directory ?? this.config.baseDirectory ?? process.cwd();
+        const contextTreeDir = join(baseDir, BRV_DIR, CONTEXT_TREE_DIR);
+        const budgets = laneBudgets ?? DEFAULT_LANE_BUDGETS;
+        // Scan all entries
+        const summaries = [];
+        const contexts = [];
+        const stubs = [];
+        await this.scanForManifest(contextTreeDir, contextTreeDir, summaries, contexts, stubs);
+        // Lane allocation with prioritized fill
+        const activeSummaries = this.allocateLane(summaries.sort((a, b) => (b.order ?? 0) - (a.order ?? 0)), budgets.summaries);
+        const activeContexts = this.allocateLane(contexts.sort((a, b) => (b.importance ?? 50) - (a.importance ?? 50)), budgets.contexts);
+        const activeStubs = this.allocateLane(stubs, budgets.stubs);
+        const activeContext = [...activeSummaries, ...activeContexts, ...activeStubs];
+        const totalTokens = activeContext.reduce((sum, e) => sum + e.tokens, 0);
+        // Compute source fingerprint (stat-only, no file reads)
+        const sourceFingerprint = await this.computeSourceFingerprint(contextTreeDir);
+        const manifest = {
+            active_context: activeContext,
+            generated_at: new Date().toISOString(),
+            lane_tokens: {
+                contexts: activeContexts.reduce((sum, e) => sum + e.tokens, 0),
+                stubs: activeStubs.reduce((sum, e) => sum + e.tokens, 0),
+                summaries: activeSummaries.reduce((sum, e) => sum + e.tokens, 0),
+            },
+            source_fingerprint: sourceFingerprint,
+            total_tokens: totalTokens,
+            version: 1,
+        };
+        // Write _manifest.json
+        const manifestPath = join(contextTreeDir, MANIFEST_FILE);
+        await writeFile(manifestPath, JSON.stringify(manifest, null, 2), 'utf8');
+        return manifest;
+    }
+    async readManifest(directory) {
+        const baseDir = directory ?? this.config.baseDirectory ?? process.cwd();
+        const manifestPath = join(baseDir, BRV_DIR, CONTEXT_TREE_DIR, MANIFEST_FILE);
+        try {
+            const content = await readFile(manifestPath, 'utf8');
+            return JSON.parse(content);
+        }
+        catch {
+            return null;
+        }
+    }
+    async readManifestIfFresh(directory) {
+        const manifest = await this.readManifest(directory);
+        if (!manifest)
+            return null;
+        const baseDir = directory ?? this.config.baseDirectory ?? process.cwd();
+        const contextTreeDir = join(baseDir, BRV_DIR, CONTEXT_TREE_DIR);
+        // Compare stored fingerprint against current state
+        const currentFingerprint = await this.computeSourceFingerprint(contextTreeDir);
+        if (currentFingerprint !== manifest.source_fingerprint)
+            return null;
+        return manifest;
+    }
+    async resolveForInjection(manifest, _query, directory) {
+        const baseDir = directory ?? this.config.baseDirectory ?? process.cwd();
+        const contextTreeDir = join(baseDir, BRV_DIR, CONTEXT_TREE_DIR);
+        const resolved = [];
+        // Order: summaries (broadest first) → contexts → stubs
+        const ordered = [
+            ...manifest.active_context.filter((e) => e.type === 'summary'),
+            ...manifest.active_context.filter((e) => e.type === 'context'),
+            ...manifest.active_context.filter((e) => e.type === 'stub'),
+        ];
+        /* eslint-disable no-await-in-loop */
+        for (const entry of ordered) {
+            try {
+                const fullPath = join(contextTreeDir, entry.path);
+                const content = await readFile(fullPath, 'utf8');
+                resolved.push({
+                    content,
+                    path: entry.path,
+                    tokens: entry.tokens,
+                    type: entry.type,
+                });
+            }
+            catch {
+                // Skip unreadable files
+            }
+        }
+        /* eslint-enable no-await-in-loop */
+        return resolved;
+    }
+    /**
+     * Allocate entries into a lane respecting the token budget.
+     * Entries are already sorted by priority (caller responsibility).
+     */
+    allocateLane(entries, budget) {
+        const allocated = [];
+        let remaining = budget;
+        for (const entry of entries) {
+            if (entry.tokens <= remaining) {
+                allocated.push(entry);
+                remaining -= entry.tokens;
+            }
+        }
+        return allocated;
+    }
+    /**
+     * Compute source fingerprint from stat data (path:mtime:size).
+     * Stat-only — no file reads required.
+     */
+    async computeSourceFingerprint(contextTreeDir) {
+        const entries = [];
+        await this.scanSourceStats(contextTreeDir, contextTreeDir, entries);
+        const input = entries
+            .sort((a, b) => a.path.localeCompare(b.path))
+            .map((e) => `${e.path}:${e.mtime}:${e.size}`)
+            .join('\n');
+        return computeContentHash(input);
+    }
+    /**
+     * Recursively scan _archived/ directory for .stub.md files.
+     */
+    async scanArchivedStubs(currentDir, contextTreeDir, stubs) {
+        let entries;
+        try {
+            entries = await readdir(currentDir, { withFileTypes: true });
+        }
+        catch {
+            return;
+        }
+        /* eslint-disable no-await-in-loop */
+        for (const entry of entries) {
+            const entryName = entry.name;
+            const fullPath = join(currentDir, entryName);
+            if (entry.isDirectory()) {
+                await this.scanArchivedStubs(fullPath, contextTreeDir, stubs);
+            }
+            else if (entry.isFile() && entryName.endsWith(STUB_EXTENSION)) {
+                const relativePath = toUnixPath(relative(contextTreeDir, fullPath));
+                try {
+                    const content = await readFile(fullPath, 'utf8');
+                    stubs.push({
+                        path: relativePath,
+                        tokens: estimateTokens(content),
+                        type: 'stub',
+                    });
+                }
+                catch {
+                    // Skip unreadable stubs
+                }
+            }
+        }
+        /* eslint-enable no-await-in-loop */
+    }
+    /**
+     * Recursively scan context tree, collecting entries for manifest building.
+     */
+    async scanForManifest(currentDir, contextTreeDir, summaries, contexts, stubs) {
+        let entries;
+        try {
+            entries = await readdir(currentDir, { withFileTypes: true });
+        }
+        catch {
+            return;
+        }
+        /* eslint-disable no-await-in-loop */
+        for (const entry of entries) {
+            const entryName = entry.name;
+            const fullPath = join(currentDir, entryName);
+            if (entry.isDirectory()) {
+                // Scan _archived/ for .stub.md files only; recurse otherwise
+                await (entryName === ARCHIVE_DIR
+                    ? this.scanArchivedStubs(fullPath, contextTreeDir, stubs)
+                    : this.scanForManifest(fullPath, contextTreeDir, summaries, contexts, stubs));
+            }
+            else if (entry.isFile() && entryName.endsWith(CONTEXT_FILE_EXTENSION)) {
+                const relativePath = toUnixPath(relative(contextTreeDir, fullPath));
+                if (entryName === SUMMARY_INDEX_FILE) {
+                    // Summary entry — read frontmatter for condensation_order and token_count
+                    try {
+                        const content = await readFile(fullPath, 'utf8');
+                        const fm = parseSummaryFrontmatter(content);
+                        summaries.push({
+                            order: fm?.condensation_order,
+                            path: relativePath,
+                            tokens: fm?.token_count ?? estimateTokens(content),
+                            type: 'summary',
+                        });
+                    }
+                    catch {
+                        // Skip unreadable summaries
+                    }
+                }
+                else if (!isDerivedArtifact(relativePath) && !isArchiveStub(relativePath)) {
+                    // Regular context entry — extract importance from frontmatter
+                    try {
+                        const content = await readFile(fullPath, 'utf8');
+                        const scoring = parseFrontmatterScoring(content);
+                        contexts.push({
+                            importance: scoring?.importance ?? 50,
+                            path: relativePath,
+                            tokens: estimateTokens(content),
+                            type: 'context',
+                        });
+                    }
+                    catch {
+                        // Skip unreadable files
+                    }
+                }
+            }
+        }
+        /* eslint-enable no-await-in-loop */
+    }
+    /**
+     * Recursively collect stat data for all source files (for fingerprint).
+     * Excludes derived artifacts.
+     */
+    async scanSourceStats(currentDir, contextTreeDir, entries) {
+        let dirEntries;
+        try {
+            dirEntries = await readdir(currentDir, { withFileTypes: true });
+        }
+        catch {
+            return;
+        }
+        /* eslint-disable no-await-in-loop */
+        for (const entry of dirEntries) {
+            const entryName = entry.name;
+            const fullPath = join(currentDir, entryName);
+            if (entry.isDirectory()) {
+                await this.scanSourceStats(fullPath, contextTreeDir, entries);
+            }
+            else if (entry.isFile() && entryName.endsWith(CONTEXT_FILE_EXTENSION)) {
+                const relativePath = toUnixPath(relative(contextTreeDir, fullPath));
+                // Include all non-derived files (contexts + stubs) in fingerprint
+                if (isDerivedArtifact(relativePath))
+                    continue;
+                try {
+                    const fileStat = await stat(fullPath);
+                    entries.push({
+                        mtime: fileStat.mtimeMs,
+                        path: relativePath,
+                        size: fileStat.size,
+                    });
+                }
+                catch {
+                    // Skip unreadable files
+                }
+            }
+        }
+        /* eslint-enable no-await-in-loop */
+    }
+}

package/dist/server/infra/context-tree/file-context-tree-merger.js

@@ -1,6 +1,7 @@
 import { copyFile, mkdir, opendir, readFile, rename, rm, stat, unlink, writeFile } from 'node:fs/promises';
 import { dirname, extname, join } from 'node:path';
 import { BRV_DIR, CONTEXT_TREE_BACKUP_DIR, CONTEXT_TREE_CONFLICT_DIR, CONTEXT_TREE_DIR } from '../../constants.js';
+import { isExcludedFromSync } from './derived-artifact.js';
 import { computeContentHash } from './hash-utils.js';
 import { toUnixPath } from './path-utils.js';
 /**
@@ -148,6 +149,9 @@ export class FileContextTreeMerger {
         const remoteFileStates = new Map();
         /* eslint-disable no-await-in-loop */
         for (const [normalPath, file] of remoteFilesMap) {
+            // Skip derived artifacts (_index.md, _archived/*, _manifest.json)
+            if (isExcludedFromSync(normalPath))
+                continue;
             const targetPath = join(contextTreeDir, normalPath);
             const remoteHash = computeContentHash(file.decodedContent);
             const snapshotHash = snapshotState.get(normalPath)?.hash;

package/dist/server/infra/context-tree/file-context-tree-snapshot-service.js

@@ -1,7 +1,8 @@
 import { mkdir, readdir, readFile, stat, writeFile } from 'node:fs/promises';
 import { dirname, join, relative } from 'node:path';
-import { BRV_DIR, CONTEXT_FILE_EXTENSION, CONTEXT_TREE_DIR, README_FILE, SNAPSHOT_FILE } from '../../constants.js';
+import { ARCHIVE_DIR, BRV_DIR, CONTEXT_FILE_EXTENSION, CONTEXT_TREE_DIR, README_FILE, SNAPSHOT_FILE } from '../../constants.js';
 import { ContextTreeSnapshot, } from '../../core/domain/entities/context-tree-snapshot.js';
+import { isExcludedFromSync } from './derived-artifact.js';
 import { computeContentHash } from './hash-utils.js';
 import { toUnixPath } from './path-utils.js';
 /**
@@ -117,15 +118,22 @@ export class FileContextTreeSnapshotService {
                 continue;
             }
             if (entry.isDirectory()) {
+                // Skip _archived/ directory (contains derived artifacts)
+                if (entry.name === ARCHIVE_DIR)
+                    continue;
                 tasks.push(this.scanDirectory(fullPath, rootDir, files));
             }
             else if (entry.isFile() && entry.name.endsWith(CONTEXT_FILE_EXTENSION)) {
                 // Only ignore README.md at root level, track it in subdirectories
                 const isRoot = currentDir === rootDir;
                 const isReadmeAtRoot = entry.name === README_FILE && isRoot;
-                if (!isReadmeAtRoot) {
-                    tasks.push(this.processFile(fullPath, rootDir, files));
-                }
+                if (isReadmeAtRoot)
+                    continue;
+                // Skip derived artifacts (_index.md, _manifest.json, .stub.md, .full.md)
+                const relativePath = toUnixPath(relative(rootDir, fullPath));
+                if (isExcludedFromSync(relativePath))
+                    continue;
+                tasks.push(this.processFile(fullPath, rootDir, files));
             }
         }
         await Promise.all(tasks);

package/dist/server/infra/context-tree/file-context-tree-summary-service.d.ts

@@ -0,0 +1,44 @@
+/**
+ * File-based implementation of IContextTreeSummaryService.
+ *
+ * Manages hierarchical summary nodes (_index.md) in the context tree.
+ * Uses three-tier escalation (normal → aggressive → deterministic fallback)
+ * following the same pattern as PreCompactionService.
+ *
+ * Fail-open: any error returns { actionTaken: false } — never blocks curation.
+ */
+import type { ICipherAgent } from '../../../agent/core/interfaces/i-cipher-agent.js';
+import type { StalenessCheckResult, SummaryGenerationResult } from '../../core/domain/knowledge/summary-types.js';
+import type { IContextTreeSummaryService } from '../../core/interfaces/context-tree/i-context-tree-summary-service.js';
+export declare class FileContextTreeSummaryService implements IContextTreeSummaryService {
+    checkStaleness(directoryPath: string, directory?: string): Promise<StalenessCheckResult>;
+    generateSummary(directoryPath: string, agent: ICipherAgent, directory?: string): Promise<SummaryGenerationResult>;
+    hasSummary(directoryPath: string, directory?: string): Promise<boolean>;
+    propagateStaleness(changedPaths: string[], agent: ICipherAgent, directory?: string): Promise<SummaryGenerationResult[]>;
+    /**
+     * Collect inputs for a summary (the summary input set invariant):
+     * - Leaf .md files in the directory (excluding _index.md, _archived/)
+     * - Child directory _index.md files (summaries of subdirectories)
+     */
+    private collectInputs;
+    /**
+     * Determine condensation order from directory depth relative to context tree root.
+     * Root = d3, domain = d2, topic = d1, subtopic = d0
+     */
+    private depthToCondensationOrder;
+    /**
+     * Execute a single summary generation pass via the agent.
+     */
+    private executeSummaryPass;
+    /**
+     * Three-tier escalation for summary generation.
+     * Follows the same pattern as PreCompactionService.compact().
+     */
+    private generateWithEscalation;
+    /**
+     * Check if any descendant of dirPath was stopped due to LLM/IO error.
+     * Since we process bottom-up (deepest first), a stopped child means
+     * its parent should also stop to prevent regenerating from stale state.
+     */
+    private hasStoppedDescendant;
+}