byterover-cli 3.3.0 → 3.5.0

package/dist/agent/infra/swarm/adapters/local-markdown-adapter.js

@@ -0,0 +1,256 @@
+import MiniSearch from 'minisearch';
+import { existsSync, readdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
+import { join, relative } from 'node:path';
+import { ADAPTER_CONTENT_LIMIT, applyGapRatio, POST_EXPANSION_GAP_RATIO, searchWithPrecision } from '../search-precision.js';
+/** Wikilink decay factor for graph-expanded results */
+const WIKILINK_DECAY = 0.7;
+/**
+ * Extract `[[target]]` and `[[target|alias]]` wikilinks from markdown content.
+ */
+function extractWikilinks(content) {
+    const links = [];
+    const regex = /\[\[([^\]|]+)(?:\|[^\]]+)?\]\]/g;
+    let match;
+    while ((match = regex.exec(content)) !== null) {
+        links.push(match[1].trim());
+    }
+    return links;
+}
+function findMarkdownFiles(dirPath, basePath) {
+    const results = [];
+    try {
+        const entries = readdirSync(dirPath, { withFileTypes: true });
+        for (const entry of entries) {
+            if (entry.name.startsWith('.'))
+                continue;
+            const fullPath = join(dirPath, entry.name);
+            if (entry.isDirectory()) {
+                results.push(...findMarkdownFiles(fullPath, basePath));
+            }
+            else if (entry.name.endsWith('.md')) {
+                const stats = statSync(fullPath);
+                results.push({
+                    fullPath,
+                    relativePath: relative(basePath, fullPath),
+                    signature: `${stats.mtimeMs}:${stats.size}`,
+                });
+            }
+        }
+    }
+    catch {
+        // Skip inaccessible directories
+    }
+    return results;
+}
+/**
+ * Derive a filename from note content (first heading or fallback).
+ */
+function deriveFilename(content) {
+    const titleMatch = content.match(/^#\s+(.+)$/m);
+    const title = titleMatch?.[1] ?? `note-${Date.now()}`;
+    const slug = title
+        .toLowerCase()
+        .replaceAll(/[^\w-]+/g, '-')
+        .replaceAll(/-+/g, '-')
+        .replaceAll(/^-+|-+$/g, '');
+    return `${slug || `note-${Date.now()}`}.md`;
+}
+function buildIndexSignature(files) {
+    return files
+        .map((file) => `${file.relativePath}:${file.signature}`)
+        .sort()
+        .join('|');
+}
+function resolveUniqueFilename(folderPath, preferredFilename) {
+    const baseName = preferredFilename.replace(/\.md$/u, '');
+    const MAX_SUFFIX = 10_000;
+    let suffix = 0;
+    while (suffix <= MAX_SUFFIX) {
+        const candidate = suffix === 0 ? `${baseName}.md` : `${baseName}-${suffix}.md`;
+        if (!existsSync(join(folderPath, candidate))) {
+            return candidate;
+        }
+        suffix++;
+    }
+    return `${baseName}-${Date.now()}.md`;
+}
+/**
+ * Local markdown folder adapter — indexes .md files via MiniSearch,
+ * optionally follows [[wikilinks]] one hop, supports write unless read-only.
+ *
+ * Each folder instance has a unique id: `local-markdown:{name}`.
+ */
+export class LocalMarkdownAdapter {
+    folderPath;
+    name;
+    capabilities;
+    id;
+    type = 'local-markdown';
+    documents = [];
+    followWikilinks;
+    index;
+    indexSignature;
+    pathToDoc = new Map();
+    readOnly;
+    watchForChanges;
+    constructor(folderPath, name, options) {
+        this.folderPath = folderPath;
+        this.name = name;
+        this.id = `local-markdown:${name}`;
+        this.readOnly = options?.readOnly ?? false;
+        this.followWikilinks = options?.followWikilinks ?? true;
+        this.watchForChanges = options?.watchForChanges ?? true;
+        this.capabilities = {
+            avgLatencyMs: 80,
+            graphTraversal: this.followWikilinks,
+            keywordSearch: true,
+            localOnly: true,
+            maxTokensPerQuery: 6000,
+            semanticSearch: false,
+            temporalQuery: false,
+            userModeling: false,
+            writeSupported: !this.readOnly,
+        };
+    }
+    async delete(_id) {
+        throw new Error('Local markdown delete not implemented.');
+    }
+    estimateCost(_request) {
+        return {
+            estimatedCostCents: 0,
+            estimatedLatencyMs: this.capabilities.avgLatencyMs,
+            estimatedTokens: 0,
+        };
+    }
+    async healthCheck() {
+        return { available: existsSync(this.folderPath) };
+    }
+    async query(request) {
+        this.ensureIndex();
+        const maxResults = request.maxResults ?? 10;
+        if (!this.index)
+            return [];
+        // T1/T2/T3: Precision-filtered search (stop words, AND-first, score floor, gap ratio)
+        const precisionResults = searchWithPrecision(this.index, request.query, { maxResults });
+        if (precisionResults.length === 0)
+            return [];
+        // Collect direct matches
+        const resultMap = new Map();
+        for (const pr of precisionResults) {
+            const doc = this.documents[pr.id];
+            if (!doc)
+                continue;
+            resultMap.set(doc.path, { doc, matchType: 'keyword', score: pr.normalizedScore });
+        }
+        // Without wikilink expansion: return direct matches
+        if (!this.followWikilinks) {
+            const sorted = [...resultMap.values()].sort((a, b) => b.score - a.score).slice(0, maxResults);
+            return sorted.map((entry, index) => ({
+                content: entry.doc.content.slice(0, ADAPTER_CONTENT_LIMIT),
+                id: `local-md-${this.name}-${index}`,
+                metadata: {
+                    matchType: entry.matchType,
+                    path: entry.doc.path,
+                    source: entry.doc.path,
+                },
+                provider: this.id,
+                providerType: 'local-markdown',
+                score: entry.score,
+            }));
+        }
+        // Expand wikilinks one hop from direct matches
+        for (const [, entry] of resultMap) {
+            for (const linkTarget of entry.doc.wikilinks) {
+                const candidates = [
+                    `${linkTarget}.md`,
+                    linkTarget,
+                    ...[...this.pathToDoc.keys()].filter((p) => p.toLowerCase().endsWith(`${linkTarget.toLowerCase()}.md`) ||
+                        p.toLowerCase() === linkTarget.toLowerCase()),
+                ];
+                for (const candidate of candidates) {
+                    const linkedDoc = this.pathToDoc.get(candidate);
+                    if (linkedDoc && !resultMap.has(linkedDoc.path)) {
+                        resultMap.set(linkedDoc.path, {
+                            doc: linkedDoc,
+                            matchType: 'graph',
+                            score: entry.score * WIKILINK_DECAY,
+                        });
+                        break;
+                    }
+                }
+            }
+        }
+        // Second gap-ratio pass on combined direct + expanded results (T3 only)
+        const combined = [...resultMap.entries()].map(([path, entry]) => ({
+            doc: entry.doc,
+            matchType: entry.matchType,
+            normalizedScore: entry.score,
+            path,
+        }));
+        const gapFiltered = applyGapRatio(combined.map((c) => ({ id: c.path, normalizedScore: c.normalizedScore, queryTerms: [], rawScore: 0 })), POST_EXPANSION_GAP_RATIO);
+        const keptPaths = new Set(gapFiltered.map((r) => r.id));
+        const sorted = combined
+            .filter((c) => keptPaths.has(c.path))
+            .sort((a, b) => b.normalizedScore - a.normalizedScore)
+            .slice(0, maxResults);
+        return sorted.map((entry, index) => ({
+            content: entry.doc.content.slice(0, ADAPTER_CONTENT_LIMIT),
+            id: `local-md-${this.name}-${index}`,
+            metadata: {
+                matchType: entry.matchType,
+                path: entry.path,
+                source: entry.path,
+            },
+            provider: this.id,
+            providerType: 'local-markdown',
+            score: entry.normalizedScore,
+        }));
+    }
+    async store(entry) {
+        if (this.readOnly) {
+            throw new Error(`Local markdown folder '${this.name}' is read-only.`);
+        }
+        const filename = resolveUniqueFilename(this.folderPath, deriveFilename(entry.content));
+        const filePath = join(this.folderPath, filename);
+        writeFileSync(filePath, entry.content);
+        // Invalidate index so next query picks up the new file
+        this.index = undefined;
+        this.indexSignature = undefined;
+        return { id: filename, provider: this.id, success: true };
+    }
+    async update(_id, _entry) {
+        throw new Error('Local markdown update not implemented.');
+    }
+    ensureIndex() {
+        // When watchForChanges is false, reuse the existing index after initial build
+        if (this.index && !this.watchForChanges) {
+            return;
+        }
+        const files = findMarkdownFiles(this.folderPath, this.folderPath);
+        const nextSignature = buildIndexSignature(files);
+        if (this.index && this.indexSignature === nextSignature) {
+            return;
+        }
+        this.documents = files.map((file, index) => {
+            const content = readFileSync(file.fullPath, 'utf8');
+            const titleMatch = content.match(/^#\s+(.+)$/m);
+            return {
+                content,
+                id: index,
+                path: file.relativePath,
+                title: titleMatch?.[1] ?? file.relativePath,
+                wikilinks: extractWikilinks(content),
+            };
+        });
+        this.index = new MiniSearch({
+            fields: ['title', 'content'],
+            storeFields: ['title', 'path'],
+        });
+        this.index.addAll(this.documents);
+        this.pathToDoc.clear();
+        for (const doc of this.documents) {
+            this.pathToDoc.set(doc.path, doc);
+        }
+        this.indexSignature = nextSignature;
+    }
+}

package/dist/agent/infra/swarm/adapters/memory-wiki-adapter.d.ts

@@ -0,0 +1,29 @@
+import type { CostEstimate, HealthStatus, MemoryEntry, ProviderCapabilities, QueryRequest, QueryResult, StoreResult } from '../../../core/domain/swarm/types.js';
+import type { IMemoryProvider } from '../../../core/interfaces/i-memory-provider.js';
+export interface MemoryWikiAdapterOptions {
+    boostFresh?: boolean;
+    vaultPath: string;
+    writePageType?: 'concept' | 'entity';
+}
+export declare class MemoryWikiAdapter implements IMemoryProvider {
+    readonly capabilities: ProviderCapabilities;
+    readonly id = "memory-wiki";
+    readonly type: "memory-wiki";
+    private readonly boostFresh;
+    private digest;
+    private documents;
+    private index;
+    private indexSignature;
+    private readonly vaultPath;
+    private readonly writePageType;
+    constructor(options: MemoryWikiAdapterOptions);
+    delete(_id: string): Promise<void>;
+    estimateCost(_request: QueryRequest): CostEstimate;
+    healthCheck(): Promise<HealthStatus>;
+    query(request: QueryRequest): Promise<QueryResult[]>;
+    store(entry: MemoryEntry): Promise<StoreResult>;
+    update(_id: string, _entry: Partial<MemoryEntry>): Promise<void>;
+    private ensureIndex;
+    private loadDigest;
+    private scanPages;
+}

package/dist/agent/infra/swarm/adapters/memory-wiki-adapter.js

@@ -0,0 +1,244 @@
+import MiniSearch from 'minisearch';
+import { existsSync, mkdirSync, readdirSync, readFileSync, statSync } from 'node:fs';
+import { writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { searchWithPrecision } from '../search-precision.js';
+/** Directories to scan for wiki pages */
+const PAGE_DIRS = ['entities', 'concepts', 'syntheses', 'sources'];
+/** Score boost for fresh pages */
+const FRESH_BOOST = 1.2;
+/** Score boost for entities/concepts over sources */
+const KIND_BOOST = {
+    concept: 1.3,
+    entity: 1.4,
+    source: 1,
+    synthesis: 1.2,
+};
+function slugify(text) {
+    return text
+        .toLowerCase()
+        .replaceAll(/[^\w\s-]/g, '')
+        .trim()
+        .replaceAll(/\s+/g, '_');
+}
+function buildIndexSignature(vaultPath) {
+    const digestPath = join(vaultPath, '.openclaw-wiki', 'cache', 'agent-digest.json');
+    try {
+        const stat = statSync(digestPath);
+        return `${stat.mtimeMs}:${stat.size}`;
+    }
+    catch {
+        return '';
+    }
+}
+function extractContentSection(fullContent) {
+    // Extract content from openclaw:wiki:content markers (written by store())
+    const wikiMarkerMatch = fullContent.match(/<!-- openclaw:wiki:content:start -->\n([\s\S]*?)<!-- openclaw:wiki:content:end -->/);
+    if (wikiMarkerMatch) {
+        return wikiMarkerMatch[1].trim();
+    }
+    // Extract from ```text code block (wiki source page format)
+    const codeBlockMatch = fullContent.match(/```text\n([\s\S]*?)```/);
+    if (codeBlockMatch) {
+        return codeBlockMatch[1].trim();
+    }
+    // Fallback: strip frontmatter and return everything
+    const withoutFrontmatter = fullContent.replace(/^---[\s\S]*?---\n*/, '');
+    return withoutFrontmatter.trim();
+}
+export class MemoryWikiAdapter {
+    capabilities = {
+        avgLatencyMs: 60,
+        graphTraversal: false,
+        keywordSearch: true,
+        localOnly: true,
+        maxTokensPerQuery: 8000,
+        semanticSearch: false,
+        temporalQuery: false,
+        userModeling: false,
+        writeSupported: true,
+    };
+    id = 'memory-wiki';
+    type = 'memory-wiki';
+    boostFresh;
+    digest = null;
+    documents = [];
+    index = null;
+    indexSignature = '';
+    vaultPath;
+    writePageType;
+    constructor(options) {
+        this.vaultPath = options.vaultPath;
+        this.boostFresh = options.boostFresh ?? true;
+        this.writePageType = options.writePageType ?? 'concept';
+    }
+    async delete(_id) {
+        throw new Error('Memory Wiki pages should be managed via wiki_apply.');
+    }
+    estimateCost(_request) {
+        return { estimatedCostCents: 0, estimatedLatencyMs: 60, estimatedTokens: 0 };
+    }
+    async healthCheck() {
+        try {
+            if (!existsSync(this.vaultPath)) {
+                return { available: false, error: `Wiki vault not found at ${this.vaultPath}` };
+            }
+            return { available: true };
+        }
+        catch (error) {
+            return { available: false, error: error instanceof Error ? error.message : String(error) };
+        }
+    }
+    async query(request) {
+        this.ensureIndex();
+        // Safety net: ensureIndex() could leave this.index null if file reads fail
+        if (!this.index) {
+            return [];
+        }
+        const maxResults = request.maxResults ?? 10;
+        const precisionResults = searchWithPrecision(this.index, request.query, { maxResults });
+        if (precisionResults.length === 0) {
+            return [];
+        }
+        const mapped = [];
+        for (const pr of precisionResults) {
+            const doc = this.documents[pr.id];
+            if (!doc)
+                continue;
+            let score = pr.normalizedScore;
+            // Freshness boost
+            if (this.boostFresh && doc.freshnessLevel === 'fresh') {
+                score *= FRESH_BOOST;
+            }
+            // Kind boost
+            score *= KIND_BOOST[doc.kind] ?? 1;
+            mapped.push({
+                content: extractContentSection(doc.content).slice(0, 5000),
+                id: doc.pageId,
+                metadata: {
+                    matchType: 'keyword',
+                    path: doc.path,
+                    source: doc.path,
+                },
+                provider: 'memory-wiki',
+                providerType: 'memory-wiki',
+                score,
+            });
+        }
+        return mapped.sort((a, b) => b.score - a.score).slice(0, maxResults);
+    }
+    async store(entry) {
+        const { content } = entry;
+        const titleMatch = content.match(/^#\s+(.+)$/m);
+        const title = titleMatch?.[1] ?? content.slice(0, 60).trim();
+        const slug = slugify(title);
+        const pageType = this.writePageType;
+        const dir = pageType === 'entity' ? 'entities' : 'concepts';
+        const dirPath = join(this.vaultPath, dir);
+        mkdirSync(dirPath, { recursive: true });
+        const now = new Date().toISOString();
+        // Resolve unique filename
+        const MAX_SUFFIX = 10_000;
+        let filename = `${slug}.md`;
+        let filePath = join(dirPath, filename);
+        let suffix = 1;
+        while (existsSync(filePath) && suffix <= MAX_SUFFIX) {
+            filename = `${slug}-${suffix}.md`;
+            filePath = join(dirPath, filename);
+            suffix++;
+        }
+        if (existsSync(filePath)) {
+            filename = `${slug}-${Date.now()}.md`;
+            filePath = join(dirPath, filename);
+        }
+        const pageId = `${pageType}.swarm.${slug}`;
+        const pageContent = [
+            '---',
+            `pageType: ${pageType}`,
+            `id: ${pageId}`,
+            `title: ${JSON.stringify(title)}`,
+            'status: active',
+            `updatedAt: "${now}"`,
+            'sourceType: swarm-curate',
+            '---',
+            '',
+            '<!-- openclaw:wiki:content:start -->',
+            content,
+            '<!-- openclaw:wiki:content:end -->',
+            '',
+            '<!-- openclaw:human:start -->',
+            '<!-- openclaw:human:end -->',
+            '',
+        ].join('\n');
+        await writeFile(filePath, pageContent);
+        // Invalidate index so next query picks up the new page
+        this.index = null;
+        this.indexSignature = '';
+        return { id: pageId, provider: 'memory-wiki', success: true };
+    }
+    async update(_id, _entry) {
+        throw new Error('Memory Wiki pages should be managed via wiki_apply.');
+    }
+    ensureIndex() {
+        const nextSignature = buildIndexSignature(this.vaultPath);
+        if (this.index && this.indexSignature === nextSignature) {
+            return;
+        }
+        this.digest = this.loadDigest();
+        this.documents = this.scanPages();
+        this.index = new MiniSearch({
+            fields: ['title', 'content'],
+            idField: 'id',
+            storeFields: ['title', 'path'],
+        });
+        this.index.addAll(this.documents);
+        this.indexSignature = nextSignature;
+    }
+    loadDigest() {
+        const digestPath = join(this.vaultPath, '.openclaw-wiki', 'cache', 'agent-digest.json');
+        try {
+            return JSON.parse(readFileSync(digestPath, 'utf8'));
+        }
+        catch {
+            return null;
+        }
+    }
+    scanPages() {
+        const digestMap = new Map();
+        if (this.digest) {
+            for (const page of this.digest.pages) {
+                digestMap.set(page.path, page);
+            }
+        }
+        const docs = [];
+        let docId = 0;
+        for (const dir of PAGE_DIRS) {
+            const dirPath = join(this.vaultPath, dir);
+            if (!existsSync(dirPath))
+                continue;
+            for (const entry of readdirSync(dirPath)) {
+                if (!entry.endsWith('.md') || entry === 'index.md')
+                    continue;
+                try {
+                    const content = readFileSync(join(dirPath, entry), 'utf8');
+                    const relPath = `${dir}/${entry}`;
+                    const digestPage = digestMap.get(relPath);
+                    const titleMatch = content.match(/^title:\s*"?(.+?)"?\s*$/m) ?? content.match(/^#\s+(.+)$/m);
+                    docs.push({
+                        content,
+                        freshnessLevel: digestPage?.freshnessLevel ?? 'unknown',
+                        id: docId++,
+                        kind: digestPage?.kind ?? dir.replace(/s$/, ''),
+                        pageId: digestPage?.id ?? `${dir}.${entry.replace(/\.md$/, '')}`,
+                        path: relPath,
+                        title: titleMatch?.[1] ?? entry.replace(/\.md$/, ''),
+                    });
+                }
+                catch {
+                    // Skip unreadable files
+                }
+            }
+        }
+        return docs;
+    }
+}

package/dist/agent/infra/swarm/adapters/obsidian-adapter.d.ts

@@ -0,0 +1,37 @@
+import type { CostEstimate, HealthStatus, MemoryEntry, ProviderCapabilities, QueryRequest, QueryResult, StoreResult } from '../../../core/domain/swarm/types.js';
+import type { IMemoryProvider } from '../../../core/interfaces/i-memory-provider.js';
+/**
+ * Options for the Obsidian adapter.
+ */
+export interface ObsidianAdapterOptions {
+    /** Additional folder names to ignore when scanning the vault */
+    ignorePatterns?: string[];
+    /** Whether to rescan files on each query (default: true). When false, the index is built once and frozen. */
+    watchForChanges?: boolean;
+}
+/**
+ * Obsidian vault adapter — indexes .md files via MiniSearch,
+ * follows [[wikilinks]] one hop for graph expansion.
+ *
+ * Read-only by default (does not modify the user's vault).
+ */
+export declare class ObsidianAdapter implements IMemoryProvider {
+    private readonly vaultPath;
+    readonly capabilities: ProviderCapabilities;
+    readonly id = "obsidian";
+    readonly type: "obsidian";
+    private documents;
+    private readonly ignoreSet;
+    private index?;
+    private indexSignature?;
+    private pathToDoc;
+    private readonly watchForChanges;
+    constructor(vaultPath: string, options?: ObsidianAdapterOptions);
+    delete(_id: string): Promise<void>;
+    estimateCost(_request: QueryRequest): CostEstimate;
+    healthCheck(): Promise<HealthStatus>;
+    query(request: QueryRequest): Promise<QueryResult[]>;
+    store(_entry: MemoryEntry): Promise<StoreResult>;
+    update(_id: string, _entry: Partial<MemoryEntry>): Promise<void>;
+    private ensureIndex;
+}