ihow-memory 0.1.0-alpha.0

package/dist/core.js

@@ -0,0 +1,85 @@
+import fs from 'node:fs';
+import { ensureWorkspace, resolveWorkspace } from './workspace.js';
+import { readMemoryFile } from './store/files.js';
+import { durablePromoteCandidate, promoteCandidate, writeCandidate } from './governance.js';
+import { countIndexedDocuments } from './engine/fts.js';
+import { engineStatus, indexWithEngineFallback, resolveEngineConfig, searchWithEngineFallback } from './engine/retrieval.js';
+function excerpt(content, max = 300) {
+    const compact = content.replace(/\s+/g, ' ').trim();
+    return compact.length > max ? `${compact.slice(0, max - 3)}...` : compact;
+}
+export async function openCore(options = {}) {
+    const workspace = await ensureWorkspace(resolveWorkspace(options));
+    const engineConfig = resolveEngineConfig(options);
+    return {
+        workspace,
+        async search (query, opts = {}) {
+            if (typeof query !== 'string' || !query.trim()) return [];
+            return (await searchWithEngineFallback(workspace, engineConfig, query, {
+                limit: opts.limit
+            })).hits;
+        },
+        async read (ref) {
+            const result = await readMemoryFile(workspace, ref);
+            const snippet = excerpt(result.content);
+            return {
+                path: result.path,
+                content: result.content,
+                snippet,
+                source: 'markdown',
+                citation: {
+                    path: result.path,
+                    snippet
+                }
+            };
+        },
+        async write_candidate (payload) {
+            const result = await writeCandidate(workspace, payload);
+            await indexWithEngineFallback(workspace, engineConfig);
+            return result;
+        },
+        async promote (candidate, target = {}) {
+            const result = await promoteCandidate(workspace, candidate, target);
+            await indexWithEngineFallback(workspace, engineConfig);
+            return result;
+        },
+        async durable_promote (candidate, promoteOptions) {
+            const result = await durablePromoteCandidate(workspace, candidate, promoteOptions);
+            if (result.status === 'promoted') await indexWithEngineFallback(workspace, engineConfig);
+            return result;
+        },
+        async status () {
+            const exists = fs.existsSync(workspace.indexPath);
+            const documents = await countIndexedDocuments(workspace);
+            const providerStatus = await engineStatus(workspace, engineConfig);
+            return {
+                ok: true,
+                workspace: {
+                    root: workspace.root,
+                    space: workspace.space,
+                    path: workspace.spaceDir,
+                    mode: workspace.mode,
+                    memoryRoot: workspace.memoryDir
+                },
+                index: {
+                    path: workspace.indexPath,
+                    manifestPath: workspace.indexManifestPath,
+                    providerId: providerStatus.provider.id,
+                    status: exists ? 'ready' : 'missing',
+                    documents,
+                    lastError: providerStatus.manifestLastError
+                },
+                provider: providerStatus.provider,
+                sync: {
+                    enabled: false
+                }
+            };
+        },
+        async rebuild () {
+            return await indexWithEngineFallback(workspace, engineConfig);
+        }
+    };
+}
+
+
+//# sourceURL=core.ts

package/dist/engine/fts.js

@@ -0,0 +1,210 @@
+import fs from 'node:fs';
+import fsp from 'node:fs/promises';
+import path from 'node:path';
+import { createRequire } from 'node:module';
+import { listMarkdownFiles } from '../store/files.js';
+import { withWorkspaceLock } from '../store/lock.js';
+import { relativeToSpace } from '../workspace.js';
+import { defaultFtsManifest, writeProviderManifest } from './manifest.js';
+const BUSY_TIMEOUT_MS = 10000;
+const requireBuiltin = createRequire(import.meta.url);
+let databaseSyncConstructor;
+function sqliteErrorMessage(error) {
+    const message = error instanceof Error ? error.message : String(error);
+    return message.replace(/\s+/g, ' ').slice(0, 300);
+}
+export function loadDatabaseSync() {
+    if (databaseSyncConstructor) return databaseSyncConstructor;
+    try {
+        const sqlite = requireBuiltin('node:sqlite');
+        if (!sqlite.DatabaseSync) throw new Error('DatabaseSync export missing');
+        databaseSyncConstructor = sqlite.DatabaseSync;
+        return databaseSyncConstructor;
+    } catch (error) {
+        throw new Error(`sqlite_unavailable:${sqliteErrorMessage(error)}`);
+    }
+}
+export function sqliteRuntimeStatus() {
+    try {
+        loadDatabaseSync();
+        return {
+            ok: true,
+            detail: 'node:sqlite DatabaseSync available'
+        };
+    } catch (error) {
+        return {
+            ok: false,
+            detail: sqliteErrorMessage(error)
+        };
+    }
+}
+function openDatabase(workspace, opts = {}) {
+    fs.mkdirSync(path.dirname(workspace.indexPath), {
+        recursive: true
+    });
+    const DatabaseSync = loadDatabaseSync();
+    const db = new DatabaseSync(workspace.indexPath, {
+        timeout: BUSY_TIMEOUT_MS
+    });
+    db.exec(`PRAGMA busy_timeout = ${BUSY_TIMEOUT_MS};`);
+    if (opts.initialize !== false) {
+        db.exec('PRAGMA journal_mode = WAL;');
+        db.exec('PRAGMA synchronous = NORMAL;');
+        db.exec(`
+      CREATE VIRTUAL TABLE IF NOT EXISTS memory_fts
+      USING fts5(path UNINDEXED, content, tokenize = 'unicode61');
+    `);
+    }
+    return db;
+}
+function cjkSegment(text) {
+    if (typeof text !== 'string' || !text) return text;
+    return text.replace(/[㐀-鿿豈-﫿]/g, (char)=>` ${char} `);
+}
+function queryToFts(query) {
+    const terms = cjkSegment(query).match(/[\p{L}\p{N}_-]+/gu) || [];
+    if (terms.length === 0) return '""';
+    return terms.slice(0, 12).map((term)=>`"${term.replace(/"/g, '""')}"`).join(' OR ');
+}
+async function collectDocuments(workspace) {
+    const files = await listMarkdownFiles(workspace.memoryDir);
+    const documents = [];
+    for (const filePath of files){
+        const relative = relativeToSpace(workspace, filePath);
+        if (relative.startsWith('memory/candidate/')) continue;
+        if (relative.startsWith('memory/_mcp/_events/')) continue;
+        if (relative.startsWith('memory/_mcp/history/')) continue;
+        const content = await fsp.readFile(filePath, 'utf8');
+        documents.push({
+            path: relative,
+            content
+        });
+    }
+    return documents;
+}
+async function rebuildFtsIndexUnlocked(workspace) {
+    const documents = await collectDocuments(workspace);
+    const db = openDatabase(workspace);
+    try {
+        db.exec('BEGIN');
+        db.exec('DELETE FROM memory_fts');
+        const insert = db.prepare('INSERT INTO memory_fts(path, content) VALUES (?, ?)');
+        for (const document of documents){
+            insert.run(document.path, cjkSegment(document.content));
+        }
+        db.exec('COMMIT');
+    } catch (error) {
+        try {
+            db.exec('ROLLBACK');
+        } catch  {}
+        throw error;
+    } finally{
+        db.close();
+    }
+    await writeProviderManifest(workspace, defaultFtsManifest('ready'));
+    return documents.length;
+}
+async function hasUsableIndex(workspace) {
+    if (!fs.existsSync(workspace.indexPath)) return false;
+    const db = openDatabase(workspace, {
+        initialize: false
+    });
+    try {
+        db.prepare('SELECT rowid FROM memory_fts LIMIT 1').all();
+        return true;
+    } catch  {
+        return false;
+    } finally{
+        db.close();
+    }
+}
+async function ensureFtsIndex(workspace) {
+    if (await hasUsableIndex(workspace)) return;
+    await withWorkspaceLock(workspace, async ()=>{
+        if (await hasUsableIndex(workspace)) return;
+        await rebuildFtsIndexUnlocked(workspace);
+    });
+}
+export async function rebuildFtsIndex(workspace) {
+    return await withWorkspaceLock(workspace, async ()=>await rebuildFtsIndexUnlocked(workspace));
+}
+export async function searchFts(workspace, query, opts = {}) {
+    if (opts.rebuild === true) {
+        await rebuildFtsIndex(workspace);
+    } else {
+        await ensureFtsIndex(workspace);
+    }
+    const limit = Math.max(1, Math.min(Number(opts.limit || 5), 25));
+    const db = openDatabase(workspace, {
+        initialize: false
+    });
+    try {
+        const rows = db.prepare(`
+        SELECT
+          path,
+          snippet(memory_fts, 1, '[', ']', '...', 24) AS snippet,
+          bm25(memory_fts) AS rank
+        FROM memory_fts
+        WHERE memory_fts MATCH ?
+        ORDER BY rank
+        LIMIT ?
+      `).all(queryToFts(query), limit);
+        return rows.map((row)=>({
+                path: row.path,
+                snippet: row.snippet,
+                score: Number(row.rank),
+                source: 'fts',
+                citation: {
+                    path: row.path,
+                    snippet: row.snippet
+                }
+            }));
+    } finally{
+        db.close();
+    }
+}
+export async function countIndexedDocuments(workspace) {
+    if (!fs.existsSync(workspace.indexPath)) return 0;
+    let db;
+    try {
+        db = openDatabase(workspace, {
+            initialize: false
+        });
+    } catch  {
+        return 0;
+    }
+    try {
+        const row = db.prepare('SELECT count(*) AS count FROM memory_fts').get();
+        return Number(row.count || 0);
+    } catch  {
+        return 0;
+    } finally{
+        db.close();
+    }
+}
+export const ftsEngine = {
+    id: 'fts',
+    capabilities: {
+        lexical: true,
+        semantic: false
+    },
+    async index (workspace) {
+        return {
+            indexed: await rebuildFtsIndex(workspace)
+        };
+    },
+    async search (workspace, query, opts = {}) {
+        return await searchFts(workspace, query, opts);
+    },
+    async status () {
+        return {
+            id: 'fts',
+            model: null,
+            ready: true,
+            cloud: false
+        };
+    }
+};
+
+
+//# sourceURL=engine/fts.ts

package/dist/engine/manifest.js

@@ -0,0 +1,45 @@
+import fs from 'node:fs/promises';
+export function defaultFtsManifest(status = 'ready') {
+    return {
+        providerId: 'fts',
+        modelId: null,
+        dims: null,
+        createdAt: new Date().toISOString(),
+        corpusFingerprint: null,
+        status,
+        ready: status === 'ready',
+        cloud: false,
+        activeProviderId: 'fts',
+        providers: {
+            fts: {
+                id: 'fts',
+                model: null,
+                ready: status === 'ready',
+                cloud: false,
+                capabilities: {
+                    lexical: true,
+                    semantic: false
+                }
+            }
+        }
+    };
+}
+export async function readProviderManifest(workspace) {
+    try {
+        return JSON.parse(await fs.readFile(workspace.indexManifestPath, 'utf8'));
+    } catch (error) {
+        if (error.code === 'ENOENT') return null;
+        return null;
+    }
+}
+export async function writeProviderManifest(workspace, manifest) {
+    const existing = await readProviderManifest(workspace);
+    await fs.writeFile(workspace.indexManifestPath, `${JSON.stringify({
+        createdAt: existing?.createdAt || manifest.createdAt || new Date().toISOString(),
+        ...manifest,
+        updatedAt: new Date().toISOString()
+    }, null, 2)}\n`, 'utf8');
+}
+
+
+//# sourceURL=engine/manifest.ts

package/dist/engine/retrieval.js

@@ -0,0 +1,324 @@
+import { spawn } from 'node:child_process';
+import { countIndexedDocuments, ftsEngine } from './fts.js';
+import { readProviderManifest, writeProviderManifest } from './manifest.js';
+const DEFAULT_VECTOR_TIMEOUT_MS = 1500;
+function stringEnv(name) {
+    const value = process.env[name];
+    return value && value.trim() ? value.trim() : undefined;
+}
+function parseTimeout(value) {
+    const parsed = Number(value);
+    if (!Number.isFinite(parsed) || parsed <= 0) return DEFAULT_VECTOR_TIMEOUT_MS;
+    return Math.max(100, Math.min(parsed, 30000));
+}
+function requestedEngineId(options) {
+    const requested = options.engine || stringEnv('IHOW_MEMORY_ENGINE') || stringEnv('IHOW_MEMORY_PROVIDER') || 'fts';
+    const normalized = requested.trim().toLowerCase();
+    if ([
+        'vector',
+        'semantic',
+        'vector-process',
+        'vector-gguf'
+    ].includes(normalized)) return 'vector-gguf';
+    return normalized || 'fts';
+}
+export function resolveEngineConfig(options = {}) {
+    return {
+        requestedId: requestedEngineId(options),
+        vectorProviderCommand: options.vectorProviderCommand || stringEnv('IHOW_MEMORY_VECTOR_PROVIDER_COMMAND'),
+        vectorModel: options.vectorModel || stringEnv('IHOW_MEMORY_VECTOR_MODEL') || null || undefined,
+        vectorTimeoutMs: parseTimeout(options.vectorTimeoutMs || stringEnv('IHOW_MEMORY_VECTOR_TIMEOUT_MS'))
+    };
+}
+function safeErrorMessage(error) {
+    const raw = error instanceof Error ? error.message : String(error);
+    return raw.replace(/\b(Bearer\s+)[A-Za-z0-9._~+/=-]{8,}/gi, '$1[redacted]').replace(/\b(sk-[A-Za-z0-9_-]{8,})\b/g, '[redacted]').replace(/\b(token|password|secret|api[_-]?key)=\S+/gi, '$1=[redacted]').slice(0, 500);
+}
+function splitCommand(input) {
+    return input.match(/(?:[^\s"']+|"[^"]*"|'[^']*')+/g)?.map((part)=>part.replace(/^["']|["']$/g, '')) || [];
+}
+class VectorProcessEngine {
+    id = 'vector-gguf';
+    capabilities = {
+        lexical: false,
+        semantic: true
+    };
+    config;
+    constructor(config){
+        this.config = config;
+    }
+    async index(workspace) {
+        return await this.callProvider('index', workspace);
+    }
+    async search(workspace, query, opts = {}) {
+        const result = await this.callProvider('search', workspace, {
+            query,
+            opts
+        });
+        const hits = result.hits || result.results || [];
+        return hits.map((hit)=>({
+                ...hit,
+                source: hit.source || this.id,
+                citation: hit.citation || {
+                    path: hit.path,
+                    snippet: hit.snippet
+                }
+            }));
+    }
+    async status(workspace) {
+        const status = await this.callProvider('status', workspace);
+        return {
+            id: String(status.id || this.id),
+            model: typeof status.model === 'string' ? status.model : this.config.vectorModel || null,
+            ready: status.ready === true,
+            cloud: status.cloud === true,
+            lastError: typeof status.lastError === 'string' ? status.lastError : undefined
+        };
+    }
+    async callProvider(method, workspace, payload = {}) {
+        if (!this.config.vectorProviderCommand) throw new Error('vector_provider_unconfigured');
+        const parts = splitCommand(this.config.vectorProviderCommand);
+        const [command, ...baseArgs] = parts;
+        if (!command) throw new Error('vector_provider_unconfigured');
+        const request = {
+            method,
+            workspace: {
+                root: workspace.root,
+                space: workspace.space,
+                memoryDir: workspace.memoryDir,
+                indexPath: workspace.indexPath,
+                indexManifestPath: workspace.indexManifestPath
+            },
+            provider: {
+                id: this.id,
+                model: this.config.vectorModel || null
+            },
+            ...payload
+        };
+        return await new Promise((resolve, reject)=>{
+            const child = spawn(command, [
+                ...baseArgs,
+                method
+            ], {
+                stdio: [
+                    'pipe',
+                    'pipe',
+                    'pipe'
+                ]
+            });
+            let stdout = '';
+            let stderr = '';
+            const timer = setTimeout(()=>{
+                child.kill('SIGTERM');
+                reject(new Error(`vector_provider_timeout:${method}`));
+            }, this.config.vectorTimeoutMs);
+            child.stdout.on('data', (chunk)=>{
+                stdout += String(chunk);
+            });
+            child.stderr.on('data', (chunk)=>{
+                stderr += String(chunk);
+            });
+            child.on('error', (error)=>{
+                clearTimeout(timer);
+                reject(error);
+            });
+            child.on('close', (code)=>{
+                clearTimeout(timer);
+                if (code !== 0) {
+                    reject(new Error(`vector_provider_exit_${code}:${stderr.trim() || method}`));
+                    return;
+                }
+                try {
+                    resolve(JSON.parse(stdout.trim() || '{}'));
+                } catch  {
+                    reject(new Error(`vector_provider_invalid_json:${method}`));
+                }
+            });
+            child.stdin.end(`${JSON.stringify(request)}\n`);
+        });
+    }
+}
+function isVectorRequested(config) {
+    return config.requestedId === 'vector-gguf';
+}
+function vectorEngine(config) {
+    return new VectorProcessEngine(config);
+}
+async function writeReadyManifest(workspace, status, documents) {
+    await writeProviderManifest(workspace, {
+        providerId: status.id,
+        modelId: status.model,
+        dims: null,
+        createdAt: new Date().toISOString(),
+        corpusFingerprint: null,
+        status: status.ready ? 'ready' : 'error',
+        ready: status.ready,
+        cloud: status.cloud,
+        activeProviderId: status.id,
+        lastError: status.lastError,
+        providers: {
+            fts: {
+                id: 'fts',
+                model: null,
+                ready: documents === undefined ? true : documents >= 0,
+                cloud: false,
+                capabilities: ftsEngine.capabilities
+            },
+            [status.id]: {
+                id: status.id,
+                model: status.model,
+                ready: status.ready,
+                cloud: status.cloud,
+                lastError: status.lastError,
+                capabilities: {
+                    semantic: true
+                }
+            }
+        }
+    });
+}
+async function writeFallbackManifest(workspace, fallback) {
+    const documents = await countIndexedDocuments(workspace);
+    await writeProviderManifest(workspace, {
+        providerId: 'fts',
+        modelId: null,
+        dims: null,
+        createdAt: new Date().toISOString(),
+        corpusFingerprint: null,
+        status: 'fallback',
+        ready: true,
+        cloud: false,
+        activeProviderId: 'fts',
+        fallbackFrom: fallback.from,
+        fallbackTo: fallback.to,
+        lastError: fallback.reason,
+        providers: {
+            fts: {
+                id: 'fts',
+                model: null,
+                ready: true,
+                cloud: false,
+                capabilities: ftsEngine.capabilities
+            },
+            [fallback.from]: {
+                id: fallback.from,
+                model: null,
+                ready: false,
+                cloud: false,
+                lastError: fallback.reason,
+                capabilities: {
+                    semantic: true
+                }
+            }
+        }
+    });
+    if (documents < 0) {
+        throw new Error('unreachable_index_document_count');
+    }
+}
+export async function indexWithEngineFallback(workspace, config) {
+    const ftsIndexed = (await ftsEngine.index(workspace)).indexed;
+    if (!isVectorRequested(config)) return ftsIndexed;
+    try {
+        const requested = vectorEngine(config);
+        const status = await requested.status(workspace);
+        if (!status.ready) throw new Error(status.lastError || 'vector_provider_not_ready');
+        await requested.index(workspace);
+        await writeReadyManifest(workspace, status, ftsIndexed);
+        return ftsIndexed;
+    } catch (error) {
+        await writeFallbackManifest(workspace, {
+            from: 'vector-gguf',
+            to: 'fts',
+            reason: safeErrorMessage(error)
+        });
+        return ftsIndexed;
+    }
+}
+export async function searchWithEngineFallback(workspace, config, query, opts = {}) {
+    if (!isVectorRequested(config)) {
+        return {
+            hits: await ftsEngine.search(workspace, query, opts)
+        };
+    }
+    const requested = vectorEngine(config);
+    try {
+        const status = await requested.status(workspace);
+        if (!status.ready) throw new Error(status.lastError || 'vector_provider_not_ready');
+        const hits = await requested.search(workspace, query, opts);
+        await writeReadyManifest(workspace, status);
+        return {
+            hits
+        };
+    } catch (error) {
+        const fallback = {
+            from: requested.id,
+            to: 'fts',
+            reason: safeErrorMessage(error)
+        };
+        const hits = await ftsEngine.search(workspace, query, opts);
+        await writeFallbackManifest(workspace, fallback);
+        return {
+            hits: hits.map((hit)=>({
+                    ...hit,
+                    fallback
+                })),
+            fallback
+        };
+    }
+}
+export async function engineStatus(workspace, config) {
+    if (!isVectorRequested(config)) {
+        const manifest = await readProviderManifest(workspace);
+        return {
+            provider: {
+                id: 'fts',
+                model: null,
+                ready: true,
+                cloud: false,
+                lastError: manifest?.providerId === 'fts' ? manifest.lastError : undefined
+            },
+            manifestLastError: manifest?.lastError
+        };
+    }
+    const requested = vectorEngine(config);
+    try {
+        const requestedStatus = await requested.status(workspace);
+        if (!requestedStatus.ready) throw new Error(requestedStatus.lastError || 'vector_provider_not_ready');
+        await writeReadyManifest(workspace, requestedStatus);
+        return {
+            provider: requestedStatus,
+            manifestLastError: requestedStatus.lastError
+        };
+    } catch (error) {
+        const fallback = {
+            from: requested.id,
+            to: 'fts',
+            reason: safeErrorMessage(error)
+        };
+        await writeFallbackManifest(workspace, fallback);
+        const requestedStatus = {
+            id: requested.id,
+            model: config.vectorModel || null,
+            ready: false,
+            cloud: false,
+            lastError: fallback.reason
+        };
+        return {
+            provider: {
+                id: 'fts',
+                model: null,
+                ready: true,
+                cloud: false,
+                lastError: fallback.reason,
+                fallback: true,
+                fallbackFrom: requested.id,
+                requested: requestedStatus
+            },
+            manifestLastError: fallback.reason
+        };
+    }
+}
+
+
+//# sourceURL=engine/retrieval.ts