@spark-agents/engram 0.1.0

package/dist/sync.js

@@ -0,0 +1,516 @@
+import fs from "node:fs";
+import path from "node:path";
+import { createHash } from "node:crypto";
+import { watch } from "chokidar";
+import { chunkMarkdown } from "./chunker.js";
+import { MEDIA_EXTENSIONS, MEDIA_MIME_TYPES, } from "./types.js";
+const DEFAULT_MAX_MEDIA_FILE_BYTES = 10 * 1024 * 1024;
+function toRelPath(baseDir, absPath) {
+    return path.relative(baseDir, absPath).split(path.sep).join("/");
+}
+function walkDir(dir, baseDir, out) {
+    const entries = fs.readdirSync(dir, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name));
+    for (const entry of entries) {
+        const abs = path.join(dir, entry.name);
+        if (entry.isDirectory()) {
+            walkDir(abs, baseDir, out);
+        }
+        else if (entry.isFile() && entry.name.endsWith(".md")) {
+            out.push(toRelPath(baseDir, abs));
+        }
+    }
+}
+function discoverMemoryFiles(workspaceDir) {
+    const files = [];
+    if (fs.existsSync(workspaceDir) && fs.statSync(workspaceDir).isDirectory()) {
+        const rootEntries = new Set(fs.readdirSync(workspaceDir));
+        for (const name of ["MEMORY.md", "memory.md"]) {
+            if (rootEntries.has(name)) {
+                files.push(name);
+            }
+        }
+    }
+    const memoryDir = path.join(workspaceDir, "memory");
+    if (fs.existsSync(memoryDir) && fs.statSync(memoryDir).isDirectory()) {
+        walkDir(memoryDir, workspaceDir, files);
+    }
+    return Array.from(new Set(files)).sort((a, b) => a.localeCompare(b));
+}
+function walkMediaDir(dir, baseDir, extensions, maxFileBytes, out) {
+    const entries = fs.readdirSync(dir, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name));
+    for (const entry of entries) {
+        const abs = path.join(dir, entry.name);
+        if (entry.isDirectory()) {
+            walkMediaDir(abs, baseDir, extensions, maxFileBytes, out);
+            continue;
+        }
+        if (!entry.isFile()) {
+            continue;
+        }
+        const ext = path.extname(entry.name).toLowerCase();
+        if (!extensions.has(ext)) {
+            continue;
+        }
+        const stat = fs.statSync(abs);
+        if (stat.size > maxFileBytes) {
+            continue;
+        }
+        out.push(toRelPath(baseDir, abs));
+    }
+}
+function discoverMediaFiles(workspaceDir, modalities, maxFileBytes) {
+    const files = [];
+    const extensions = new Set();
+    for (const modality of modalities) {
+        for (const ext of MEDIA_EXTENSIONS[modality] ?? []) {
+            extensions.add(ext.toLowerCase());
+        }
+    }
+    if (extensions.size === 0) {
+        return files;
+    }
+    if (fs.existsSync(workspaceDir) && fs.statSync(workspaceDir).isDirectory()) {
+        const rootEntries = fs
+            .readdirSync(workspaceDir, { withFileTypes: true })
+            .sort((a, b) => a.name.localeCompare(b.name));
+        for (const entry of rootEntries) {
+            if (!entry.isFile()) {
+                continue;
+            }
+            const ext = path.extname(entry.name).toLowerCase();
+            if (!extensions.has(ext)) {
+                continue;
+            }
+            const abs = path.join(workspaceDir, entry.name);
+            const stat = fs.statSync(abs);
+            if (stat.size > maxFileBytes) {
+                continue;
+            }
+            files.push(toRelPath(workspaceDir, abs));
+        }
+    }
+    const memoryDir = path.join(workspaceDir, "memory");
+    if (fs.existsSync(memoryDir) && fs.statSync(memoryDir).isDirectory()) {
+        walkMediaDir(memoryDir, workspaceDir, extensions, maxFileBytes, files);
+    }
+    return Array.from(new Set(files)).sort((a, b) => a.localeCompare(b));
+}
+function discoverSessionFiles(sessionsDir) {
+    if (!(fs.existsSync(sessionsDir) && fs.statSync(sessionsDir).isDirectory())) {
+        return [];
+    }
+    const files = fs
+        .readdirSync(sessionsDir, { withFileTypes: true })
+        .sort((a, b) => a.name.localeCompare(b.name))
+        .filter((entry) => entry.isFile() && entry.name.endsWith(".jsonl"))
+        .map((entry) => entry.name);
+    return files;
+}
+function hashContent(content) {
+    return createHash("sha256").update(content).digest("hex").slice(0, 16);
+}
+function hashBuffer(content) {
+    return createHash("sha256").update(content).digest("hex").slice(0, 16);
+}
+function mediaLabelPrefix(mimeType) {
+    if (mimeType.startsWith("image/")) {
+        return "Image";
+    }
+    if (mimeType.startsWith("audio/")) {
+        return "Audio";
+    }
+    return "Media";
+}
+function buildWatchPatterns(multimodalEnabled, modalities) {
+    const patterns = new Set(["MEMORY.md", "memory.md", "memory/**/*.md"]);
+    if (!multimodalEnabled) {
+        return Array.from(patterns);
+    }
+    for (const modality of modalities) {
+        for (const ext of MEDIA_EXTENSIONS[modality] ?? []) {
+            const normalizedExt = ext.toLowerCase();
+            patterns.add(`*${normalizedExt}`);
+            patterns.add(`memory/**/*${normalizedExt}`);
+        }
+    }
+    return Array.from(patterns);
+}
+function flattenSessionContent(raw) {
+    if (typeof raw === "string") {
+        const normalized = raw.replace(/\r?\n+/g, " ").trim();
+        return normalized.length > 0 ? normalized : null;
+    }
+    if (!Array.isArray(raw)) {
+        return null;
+    }
+    const parts = [];
+    for (const block of raw) {
+        if (typeof block !== "object" || block === null) {
+            continue;
+        }
+        const type = "type" in block ? block.type : undefined;
+        const text = "text" in block ? block.text : undefined;
+        if (type === "text" && typeof text === "string") {
+            const normalized = text.replace(/\r?\n+/g, " ").trim();
+            if (normalized.length > 0) {
+                parts.push(normalized);
+            }
+        }
+    }
+    if (parts.length === 0) {
+        return null;
+    }
+    return parts.join(" ");
+}
+export function flattenSessionJsonl(content) {
+    if (content.trim().length === 0) {
+        return null;
+    }
+    const flattenedLines = [];
+    const lineMap = [];
+    const lines = content.split(/\r?\n/);
+    for (let i = 0; i < lines.length; i += 1) {
+        const rawLine = lines[i];
+        if (rawLine.trim().length === 0) {
+            continue;
+        }
+        let parsed;
+        try {
+            parsed = JSON.parse(rawLine);
+        }
+        catch {
+            continue;
+        }
+        if (typeof parsed !== "object" || parsed === null) {
+            continue;
+        }
+        // Support both formats:
+        // 1. { role: "user", content: ... }  (flat format)
+        // 2. { type: "message", message: { role: "user", content: ... } }  (OpenClaw session format)
+        const inner = "message" in parsed && typeof parsed.message === "object" && parsed.message !== null
+            ? parsed.message
+            : parsed;
+        const role = "role" in inner ? inner.role : undefined;
+        if (role === "system") {
+            continue;
+        }
+        let prefix = null;
+        if (role === "user") {
+            prefix = "User: ";
+        }
+        else if (role === "assistant") {
+            prefix = "Assistant: ";
+        }
+        if (prefix === null) {
+            continue;
+        }
+        const rawMessage = "content" in inner ? inner.content : undefined;
+        const message = flattenSessionContent(rawMessage);
+        if (message === null) {
+            continue;
+        }
+        flattenedLines.push(`${prefix}${message}`);
+        lineMap.push(i + 1);
+    }
+    if (flattenedLines.length === 0) {
+        return null;
+    }
+    return {
+        text: flattenedLines.join("\n"),
+        lineMap,
+    };
+}
+export function remapChunkLines(chunks, lineMap) {
+    for (const chunk of chunks) {
+        const startIdx = chunk.startLine - 1;
+        const endIdx = chunk.endLine - 1;
+        if (startIdx >= 0 && startIdx < lineMap.length) {
+            chunk.startLine = lineMap[startIdx];
+        }
+        if (endIdx >= 0 && endIdx < lineMap.length) {
+            chunk.endLine = lineMap[endIdx];
+        }
+    }
+}
+export function createSyncManager(params) {
+    const { workspaceDir, index, embedding, chunkTokens, chunkOverlap, sessionsDir } = params;
+    const multimodalEnabled = params.multimodal?.enabled === true;
+    const multimodalModalities = params.multimodal?.modalities ?? [];
+    const maxMediaFileBytes = Math.max(1, Math.trunc(params.multimodal?.maxFileBytes ?? DEFAULT_MAX_MEDIA_FILE_BYTES));
+    const trackedFiles = new Set();
+    const warmedSessions = new Set();
+    let dirty = true;
+    let closed = false;
+    let watcher = null;
+    let sessionWatcher = null;
+    let syncTimer = null;
+    let intervalTimer = null;
+    let debounceMs = 1000;
+    function markDirty() {
+        dirty = true;
+    }
+    function isDirty() {
+        return dirty;
+    }
+    function clearSyncTimer() {
+        if (syncTimer !== null) {
+            clearTimeout(syncTimer);
+            syncTimer = null;
+        }
+    }
+    function clearIntervalTimer() {
+        if (intervalTimer !== null) {
+            clearInterval(intervalTimer);
+            intervalTimer = null;
+        }
+    }
+    function closeWatcher() {
+        if (watcher !== null) {
+            void watcher.close().catch(() => { });
+            watcher = null;
+        }
+        if (sessionWatcher !== null) {
+            void sessionWatcher.close().catch(() => { });
+            sessionWatcher = null;
+        }
+    }
+    async function sync(opts) {
+        if (closed) {
+            return;
+        }
+        const progress = opts?.progress;
+        const force = opts?.force === true;
+        const memoryFiles = discoverMemoryFiles(workspaceDir);
+        const mediaFiles = multimodalEnabled && multimodalModalities.length > 0
+            ? discoverMediaFiles(workspaceDir, multimodalModalities, maxMediaFileBytes)
+            : [];
+        const sessionFiles = sessionsDir ? discoverSessionFiles(sessionsDir) : [];
+        const seenFiles = new Set();
+        let completed = 0;
+        const total = memoryFiles.length + mediaFiles.length + sessionFiles.length;
+        if (total === 0) {
+            progress?.({ completed: 0, total: 0, label: "No memory/session/media files found" });
+        }
+        try {
+            for (const relPath of memoryFiles) {
+                seenFiles.add(relPath);
+                trackedFiles.add(relPath);
+                const absPath = path.join(workspaceDir, relPath);
+                const content = fs.readFileSync(absPath, "utf8");
+                const contentHash = hashContent(content);
+                const existingHash = index.getFileHash(relPath);
+                if (!force && existingHash === contentHash) {
+                    completed += 1;
+                    progress?.({ completed, total, label: `Skipped ${relPath}` });
+                    continue;
+                }
+                const chunks = chunkMarkdown(content, {
+                    maxTokens: chunkTokens,
+                    overlapRatio: chunkOverlap,
+                });
+                const texts = chunks.map((chunk) => chunk.text);
+                const vectors = texts.length === 0 ? [] : await embedding.embedBatch(texts, "RETRIEVAL_DOCUMENT");
+                const file = {
+                    fileKey: relPath,
+                    contentHash,
+                    source: "memory",
+                    indexedAt: Date.now(),
+                };
+                index.indexFile(file, chunks, vectors);
+                completed += 1;
+                progress?.({ completed, total, label: `Indexed ${relPath}` });
+            }
+            if (mediaFiles.length > 0) {
+                if (!embedding.supportsMultimodal || embedding.embedMedia === undefined) {
+                    console.warn("Engram: multimodal indexing is enabled, but the current embedding client does not support media. Skipping media files.");
+                    for (const relPath of mediaFiles) {
+                        seenFiles.add(relPath);
+                        trackedFiles.add(relPath);
+                        completed += 1;
+                        progress?.({ completed, total, label: `Skipped ${relPath} (multimodal unsupported)` });
+                    }
+                }
+                else {
+                    for (const relPath of mediaFiles) {
+                        seenFiles.add(relPath);
+                        trackedFiles.add(relPath);
+                        const absPath = path.join(workspaceDir, relPath);
+                        const ext = path.extname(relPath).toLowerCase();
+                        const mimeType = MEDIA_MIME_TYPES[ext];
+                        if (!mimeType) {
+                            completed += 1;
+                            progress?.({ completed, total, label: `Skipped ${relPath} (unknown mime type)` });
+                            continue;
+                        }
+                        // Enforce size limits before reading file contents.
+                        const stat = fs.statSync(absPath);
+                        if (stat.size > maxMediaFileBytes) {
+                            completed += 1;
+                            progress?.({ completed, total, label: `Skipped ${relPath} (file too large)` });
+                            continue;
+                        }
+                        const content = fs.readFileSync(absPath);
+                        const contentHash = hashBuffer(content);
+                        const existingHash = index.getFileHash(relPath);
+                        if (!force && existingHash === contentHash) {
+                            completed += 1;
+                            progress?.({ completed, total, label: `Skipped ${relPath}` });
+                            continue;
+                        }
+                        const vector = await embedding.embedMedia(content, mimeType);
+                        const prefix = mediaLabelPrefix(mimeType);
+                        const chunks = [
+                            {
+                                text: `${prefix} file: ${relPath}`,
+                                startLine: 1,
+                                endLine: 1,
+                                hash: contentHash,
+                            },
+                        ];
+                        const file = {
+                            fileKey: relPath,
+                            contentHash,
+                            source: "memory",
+                            indexedAt: Date.now(),
+                        };
+                        index.indexFile(file, chunks, [vector]);
+                        completed += 1;
+                        progress?.({ completed, total, label: `Indexed ${relPath}` });
+                    }
+                }
+            }
+            for (const sessionName of sessionFiles) {
+                const relPath = path.posix.join("sessions", sessionName);
+                seenFiles.add(relPath);
+                trackedFiles.add(relPath);
+                const absPath = path.join(sessionsDir ?? "", sessionName);
+                const content = fs.readFileSync(absPath, "utf8");
+                const contentHash = hashContent(content);
+                const existingHash = index.getFileHash(relPath);
+                if (!force && existingHash === contentHash) {
+                    completed += 1;
+                    progress?.({ completed, total, label: `Skipped ${relPath}` });
+                    continue;
+                }
+                const flattened = flattenSessionJsonl(content);
+                let chunks = [];
+                if (flattened !== null) {
+                    chunks = chunkMarkdown(flattened.text, {
+                        maxTokens: chunkTokens,
+                        overlapRatio: chunkOverlap,
+                    });
+                    remapChunkLines(chunks, flattened.lineMap);
+                }
+                const texts = chunks.map((chunk) => chunk.text);
+                const vectors = texts.length === 0 ? [] : await embedding.embedBatch(texts, "RETRIEVAL_DOCUMENT");
+                const file = {
+                    fileKey: relPath,
+                    contentHash,
+                    source: "sessions",
+                    indexedAt: Date.now(),
+                };
+                index.indexFile(file, chunks, vectors);
+                completed += 1;
+                progress?.({ completed, total, label: `Indexed ${relPath}` });
+            }
+            for (const relPath of Array.from(trackedFiles)) {
+                if (seenFiles.has(relPath)) {
+                    continue;
+                }
+                index.removeFile(relPath);
+                trackedFiles.delete(relPath);
+            }
+            dirty = false;
+        }
+        catch (error) {
+            dirty = true;
+            throw error;
+        }
+    }
+    function scheduleDebouncedSync() {
+        clearSyncTimer();
+        syncTimer = setTimeout(() => {
+            void sync({ reason: "file-change" }).catch(() => { });
+        }, debounceMs);
+    }
+    return {
+        sync,
+        markDirty,
+        isDirty,
+        startWatching(opts) {
+            if (closed) {
+                return;
+            }
+            closeWatcher();
+            clearSyncTimer();
+            clearIntervalTimer();
+            debounceMs = Math.max(0, Math.trunc(opts?.debounceMs ?? 1000));
+            const intervalMinutes = Math.max(0, Math.trunc(opts?.intervalMinutes ?? 5));
+            const watchPatterns = buildWatchPatterns(multimodalEnabled, multimodalModalities);
+            watcher = watch(watchPatterns, {
+                cwd: workspaceDir,
+                ignoreInitial: true,
+                awaitWriteFinish: {
+                    stabilityThreshold: debounceMs,
+                    pollInterval: 100,
+                },
+            });
+            const onFsUpdate = () => {
+                markDirty();
+                scheduleDebouncedSync();
+            };
+            watcher.on("add", onFsUpdate);
+            watcher.on("change", onFsUpdate);
+            watcher.on("unlink", onFsUpdate);
+            if (sessionsDir && fs.existsSync(sessionsDir)) {
+                sessionWatcher = watch("*.jsonl", {
+                    cwd: sessionsDir,
+                    ignoreInitial: true,
+                    awaitWriteFinish: {
+                        stabilityThreshold: debounceMs,
+                        pollInterval: 100,
+                    },
+                });
+                sessionWatcher.on("add", onFsUpdate);
+                sessionWatcher.on("change", onFsUpdate);
+                sessionWatcher.on("unlink", onFsUpdate);
+            }
+            if (intervalMinutes > 0) {
+                intervalTimer = setInterval(() => {
+                    if (isDirty()) {
+                        void sync({ reason: "interval" }).catch(() => { });
+                    }
+                }, intervalMinutes * 60 * 1000);
+            }
+        },
+        async warmSession(sessionKey) {
+            if (closed) {
+                return;
+            }
+            const key = sessionKey ?? "__default__";
+            if (warmedSessions.has(key)) {
+                return;
+            }
+            warmedSessions.add(key);
+            if (isDirty()) {
+                await sync({ reason: `warm-session:${key}` });
+            }
+        },
+        syncIfDirty() {
+            if (closed) {
+                return;
+            }
+            if (isDirty()) {
+                void sync({ reason: "on-search" }).catch(() => { });
+            }
+        },
+        close() {
+            closed = true;
+            closeWatcher();
+            clearSyncTimer();
+            clearIntervalTimer();
+            trackedFiles.clear();
+            warmedSessions.clear();
+        },
+    };
+}

package/dist/types.d.ts

@@ -0,0 +1,111 @@
+export type MediaModality = "image" | "audio";
+export declare const MEDIA_EXTENSIONS: Record<MediaModality, string[]>;
+export declare const MEDIA_MIME_TYPES: Record<string, string>;
+export interface EngramConfig {
+    geminiApiKey?: string;
+    dimensions?: 768 | 1536 | 3072;
+    chunkTokens?: number;
+    chunkOverlap?: number;
+    reranking?: boolean;
+    timeDecay?: {
+        enabled?: boolean;
+        halfLifeDays?: number;
+    };
+    maxSessionShare?: number;
+    multimodal?: {
+        enabled?: boolean;
+        modalities?: MediaModality[];
+        maxFileBytes?: number;
+    };
+}
+export declare const DEFAULT_CONFIG: Required<Pick<EngramConfig, "dimensions" | "chunkTokens" | "chunkOverlap" | "maxSessionShare">> & {
+    timeDecay: Required<NonNullable<EngramConfig["timeDecay"]>>;
+};
+export type GeminiTaskType = "RETRIEVAL_DOCUMENT" | "RETRIEVAL_QUERY";
+export interface EmbeddingClient {
+    embedText(text: string, taskType: GeminiTaskType): Promise<Float32Array>;
+    embedBatch(texts: string[], taskType: GeminiTaskType): Promise<Float32Array[]>;
+    /** Embed a binary file (image, audio, PDF). Returns a normalized Float32Array. */
+    embedMedia?(data: Buffer, mimeType: string): Promise<Float32Array>;
+    /** Whether this client supports multimodal embedding. */
+    readonly supportsMultimodal: boolean;
+    dimensions: number;
+    model: string;
+}
+export interface Chunk {
+    text: string;
+    startLine: number;
+    endLine: number;
+    hash: string;
+    headingContext?: string;
+}
+export interface ChunkerOptions {
+    maxTokens: number;
+    overlapRatio: number;
+}
+export type MemorySource = "memory" | "sessions";
+export interface StoredFile {
+    fileKey: string;
+    contentHash: string;
+    source: MemorySource;
+    indexedAt: number;
+}
+export interface ScoredChunk {
+    fileKey: string;
+    startLine: number;
+    endLine: number;
+    text: string;
+    score: number;
+    source: MemorySource;
+    headingContext?: string;
+    indexedAt?: number;
+}
+export interface IndexStats {
+    files: number;
+    chunks: number;
+    sources: Array<{
+        source: MemorySource;
+        files: number;
+        chunks: number;
+    }>;
+    dbPath: string;
+    vectorDims: number;
+}
+export interface IndexManager {
+    indexFile(file: StoredFile, chunks: Chunk[], vectors: Float32Array[]): void;
+    removeFile(fileKey: string): void;
+    searchBM25(query: string, topK: number): ScoredChunk[];
+    searchVector(queryVec: Float32Array, topK: number): ScoredChunk[];
+    getFileHash(fileKey: string): string | null;
+    readFileContent(relPath: string, from?: number, lines?: number): {
+        text: string;
+        path: string;
+    } | null;
+    stats(): IndexStats;
+    close(): void;
+}
+export interface SearchOptions {
+    maxResults?: number;
+    minScore?: number;
+    sessionKey?: string;
+}
+export interface SearchResult {
+    path: string;
+    startLine: number;
+    endLine: number;
+    score: number;
+    snippet: string;
+    source: MemorySource;
+    citation?: string;
+}
+export interface SyncProgress {
+    completed: number;
+    total: number;
+    label?: string;
+}
+export interface SyncOptions {
+    reason?: string;
+    force?: boolean;
+    sessionFiles?: string[];
+    progress?: (update: SyncProgress) => void;
+}

package/dist/types.js

@@ -0,0 +1,28 @@
+export const MEDIA_EXTENSIONS = {
+    image: [".jpg", ".jpeg", ".png", ".webp", ".gif"],
+    audio: [".mp3", ".wav", ".ogg", ".opus", ".m4a", ".aac", ".flac"],
+};
+export const MEDIA_MIME_TYPES = {
+    ".jpg": "image/jpeg",
+    ".jpeg": "image/jpeg",
+    ".png": "image/png",
+    ".webp": "image/webp",
+    ".gif": "image/gif",
+    ".mp3": "audio/mpeg",
+    ".wav": "audio/wav",
+    ".ogg": "audio/ogg",
+    ".opus": "audio/opus",
+    ".m4a": "audio/mp4",
+    ".aac": "audio/aac",
+    ".flac": "audio/flac",
+};
+export const DEFAULT_CONFIG = {
+    dimensions: 768,
+    chunkTokens: 1024,
+    chunkOverlap: 0.15,
+    maxSessionShare: 0.4,
+    timeDecay: {
+        enabled: true,
+        halfLifeDays: 30,
+    },
+};