@kisev/memomatic 1.0.0-dev.100.g21366c4ad740

package/dist/service.d.ts

@@ -0,0 +1,63 @@
+import { type MemoryRules } from "./rules.js";
+import { type MemomaticPaths } from "./paths.js";
+import { type MemomaticSettings } from "./settings.js";
+import { type OperationOptions } from "./operations.js";
+import { type SearchHit, type SearchOptions } from "./search.js";
+import { MemoryStore } from "./store.js";
+export type WriteRequest = {
+    text: string;
+    key?: string;
+    trigger?: string[];
+    importance?: number;
+    project?: string;
+    origin?: "user" | "agent";
+    target?: "episodic" | "curated" | "user";
+    pinned?: boolean;
+    source?: string;
+};
+export type MemomaticContext = {
+    paths: MemomaticPaths;
+    settings: MemomaticSettings;
+    rules: MemoryRules;
+    store: MemoryStore;
+};
+export declare function openMemomatic(options?: {
+    readOnly?: boolean;
+    configure?: (settings: MemomaticSettings) => MemomaticSettings;
+}): Promise<MemomaticContext>;
+export declare function writeEntry(context: MemomaticContext, request: WriteRequest): Promise<string>;
+export declare function searchMemory(context: MemomaticContext, query: string, options?: SearchOptions): Promise<SearchHit[]>;
+export declare function searchMemoryResults(context: MemomaticContext, query: string, options?: SearchOptions & {
+    explain?: boolean;
+}): Promise<{
+    file: string;
+    kind: import("./store.js").EntryKind;
+    line: number;
+    score: number;
+    snippet: string;
+    project: string | null;
+    origin: "agent" | "user" | null;
+    explanation?: {
+        matchedTokens: string[];
+        lexical: number;
+        semantic: number | null;
+        relevance: number;
+        exact: boolean;
+        recency: number;
+        importance: number;
+        reason: "lexical" | "semantic" | "both";
+    } | undefined;
+    source: string | null;
+    visibility: import("./visibility.js").EntryVisibility;
+}[]>;
+export declare function getEntry(context: MemomaticContext, file: string, line?: number): Promise<{
+    file: string;
+    line: number;
+    content: string;
+}>;
+export declare function forgetEntry(context: MemomaticContext, target: {
+    file: string;
+    line: number;
+}): Promise<string>;
+export declare function archiveOldEpisodic(context: MemomaticContext, options?: OperationOptions): Promise<string[]>;
+export declare function rebuildIndex(context: MemomaticContext, options?: OperationOptions): Promise<number>;

package/dist/service.js

@@ -0,0 +1,163 @@
+import { entryLine, parseEntryLine } from "./entries.js";
+import { readTextIfExists, replaceEntryLine, writeCorpusFile } from "./corpus.js";
+import { assertSafePath } from "@kisev/safe-fs";
+import { basename, isAbsolute, join, relative, resolve, sep } from "node:path";
+import { rm } from "node:fs/promises";
+import { dropToInbox, withRunLock } from "./inbox.js";
+import { isForbidden, loadRules } from "./rules.js";
+import { memomaticPaths } from "./paths.js";
+import { loadSettings } from "./settings.js";
+import { check } from "./operations.js";
+import { reindex, search } from "./search.js";
+import { MemoryStore } from "./store.js";
+import { visibilityForSource } from "./visibility.js";
+async function memoryPath(context, file) {
+    const target = resolve(context.paths.stateRoot, file);
+    const part = relative(context.paths.stateRoot, target);
+    if (!part || part === ".." || part.startsWith(`..${sep}`) || isAbsolute(part))
+        throw new Error("path escapes memomatic state root");
+    await assertSafePath(target, { target: "file" });
+    return target;
+}
+export async function openMemomatic(options = {}) {
+    const paths = memomaticPaths();
+    const loaded = await loadSettings(paths.settingsFile);
+    const settings = options.configure ? options.configure(loaded) : loaded;
+    const rules = await loadRules(paths.rulesFile);
+    const store = options.readOnly
+        ? await MemoryStore.preview(paths.indexFile)
+        : await MemoryStore.open(paths.indexFile);
+    return { paths, settings, rules, store };
+}
+export async function writeEntry(context, request) {
+    const text = request.text.trim();
+    if (!text)
+        throw new Error("memory text is empty");
+    if (isForbidden(text, context.rules))
+        throw new Error("memory text matches a never-save rule in MEMORY_RULES.md");
+    const origin = request.origin ?? "agent";
+    const line = entryLine(text, {
+        key: request.key,
+        status: origin === "user" && request.target === "user" ? "active" : undefined,
+        origin,
+        observed: new Date().toISOString().slice(0, 10),
+        project: request.project,
+        importance: request.importance,
+        trigger: request.trigger,
+        pinned: request.pinned,
+        source: request.source ?? origin,
+        target: request.target,
+    });
+    return dropToInbox(context.paths, [line], request.source ?? origin);
+}
+export async function searchMemory(context, query, options = {}) {
+    return search(context.store, query, context.settings, options);
+}
+export async function searchMemoryResults(context, query, options = {}) {
+    return (await searchMemory(context, query, options)).map((hit) => ({
+        file: hit.entry.file.replace(`${context.paths.stateRoot}/`, ""),
+        kind: hit.entry.kind,
+        line: hit.entry.line,
+        score: Number(hit.score.toFixed(4)),
+        snippet: hit.snippet,
+        project: hit.entry.project,
+        origin: hit.entry.origin,
+        ...(options.explain ? { explanation: hit.explanation } : {}),
+        source: hit.entry.source,
+        visibility: visibilityForSource(hit.entry.source),
+    }));
+}
+export async function getEntry(context, file, line) {
+    const target = await memoryPath(context, file);
+    const content = await readTextIfExists(target);
+    if (content === undefined)
+        throw new Error(`memory file not found: ${file}`);
+    if (line === undefined)
+        return { file: target, line: 0, content };
+    const lines = content.split("\n");
+    if (!Number.isSafeInteger(line) || line < 1 || line > lines.length)
+        throw new Error("line is out of range");
+    context.store.markUseful(context.store
+        .allEntries()
+        .filter((entry) => entry.file === target && entry.line === line)
+        .map((entry) => entry.stableId));
+    return { file: target, line, content: lines[line - 1] };
+}
+export async function forgetEntry(context, target) {
+    return withRunLock(context.paths, async () => {
+        const file = await memoryPath(context, target.file);
+        const content = await readTextIfExists(file);
+        if (content === undefined)
+            throw new Error(`memory file not found: ${target.file}`);
+        const lines = content.split("\n");
+        if (!Number.isSafeInteger(target.line) || target.line < 1 || target.line > lines.length)
+            throw new Error("line is out of range");
+        await replaceEntryLine(context.paths, file, target.line, null);
+        context.store.replaceEntries(context.store
+            .allEntries()
+            .filter((entry) => entry.file !== file || entry.line !== target.line)
+            .map((entry) => entry.file === file && entry.line > target.line
+            ? { ...entry, line: entry.line - 1 }
+            : entry));
+        return file;
+    });
+}
+export async function archiveOldEpisodic(context, options = {}) {
+    const archived = [];
+    if (!context.rules.autoClean)
+        return archived;
+    const rule = context.rules.autoClean;
+    const cutoff = Date.now() - rule.olderThanDays * 86_400_000;
+    const unusedCutoff = rule.unusedAfterDays ? Date.now() - rule.unusedAfterDays * 86_400_000 : null;
+    const usefulByLine = new Map(context.store
+        .allEntries()
+        .map((entry) => [
+        `${entry.file}:${entry.line}`,
+        context.store.usageFor(entry.stableId).useful,
+    ]));
+    const files = new Set(context.store
+        .allEntries()
+        .filter((entry) => entry.kind === "episodic")
+        .map((entry) => entry.file));
+    for (const file of files) {
+        check(options);
+        await memoryPath(context, file);
+        const content = await readTextIfExists(file);
+        if (content === undefined)
+            continue;
+        const selected = [];
+        const remaining = content.split("\n").filter((line, index) => {
+            const entry = parseEntryLine(line)?.annotations;
+            const observed = entry?.observed ? Date.parse(entry.observed) : NaN;
+            const protectedEntry = !entry ||
+                !Number.isFinite(observed) ||
+                entry.pinned ||
+                (rule.source && entry.source !== rule.source);
+            // Unused entries decay earlier than the age-only cutoff.
+            const decayed = !protectedEntry &&
+                (observed < cutoff ||
+                    (unusedCutoff !== null &&
+                        observed < unusedCutoff &&
+                        (usefulByLine.get(`${file}:${index + 1}`) ?? 0) === 0));
+            if (decayed) {
+                selected.push(line);
+                return false;
+            }
+            return true;
+        });
+        if (!selected.length)
+            continue;
+        const target = join(context.paths.archiveDir, basename(file));
+        const prior = (await readTextIfExists(target)) ?? "";
+        const seen = new Set(prior.split("\n"));
+        await writeCorpusFile(context.paths, target, `${[prior.trimEnd(), ...selected.filter((line) => !seen.has(line))].filter(Boolean).join("\n")}\n`);
+        await writeCorpusFile(context.paths, file, remaining.join("\n"));
+        if (!remaining.some((line) => line.trim()))
+            await rm(file);
+        archived.push(file);
+    }
+    return archived;
+}
+export async function rebuildIndex(context, options = {}) {
+    return withRunLock(context.paths, () => reindex(context.paths, context.settings, context.store, options));
+}

package/dist/sessions.d.ts

@@ -0,0 +1,36 @@
+import { type ModelExecutor } from "./executor.js";
+import { type OperationOptions } from "./operations.js";
+import type { MemomaticContext } from "./service.js";
+export declare const WATERMARK_KEY = "ingest-watermark";
+export declare const LAST_RUN_KEY = "last-sessions-at";
+export type SessionsReport = {
+    sessionsIngested: number;
+    candidatesExtracted: number;
+    rejected: Array<{
+        text: string;
+        reason: string;
+    }>;
+    dryRun: boolean;
+    ingestion: {
+        messages: number;
+        fragments: number;
+        completed: number;
+        cached: number;
+        characters: number;
+        remaining: number;
+        cachedSessions: number;
+        unscannedSessions: number;
+    };
+    elapsedMs?: number;
+};
+export type SessionsOptions = OperationOptions & {
+    dryRun?: boolean;
+    databaseFile?: string;
+};
+export type ExtractedCandidate = {
+    text: string;
+    key: string | null;
+    reason: string;
+};
+export declare function parseExtraction(response: string): ExtractedCandidate[];
+export declare function runSessions(context: MemomaticContext, executor: ModelExecutor | null, options?: SessionsOptions): Promise<SessionsReport>;

package/dist/sessions.js

@@ -0,0 +1,238 @@
+import { appendDailyEntry, readTextIfExists, parseCorpusEntries } from "./corpus.js";
+import { extractJson } from "./executor.js";
+import { opencodeDatabasePath } from "./ingest.js";
+import { planIngestion } from "./ingestion.js";
+import { check } from "./operations.js";
+import { writeAtomic } from "@kisev/safe-fs";
+import { join } from "node:path";
+import { stat } from "node:fs/promises";
+import { corpusFiles, reindex } from "./search.js";
+import { isForbidden } from "./rules.js";
+import { entryLine, parseEntryLine } from "./entries.js";
+export const WATERMARK_KEY = "ingest-watermark";
+export const LAST_RUN_KEY = "last-sessions-at";
+const EXTRACT_SYSTEM = `You distill durable engineering memory from a coding session.
+Reply with exactly one JSON object of the form
+{"candidates":[{"text":string,"key":string|null,"reason":string}]}.
+Include only standing decisions with rationale, discoveries, failed attempts with
+the reason they were rejected, session outcomes, and action-sensitive boundaries
+(approval requirements, temporary constraints, handoffs, expiry conditions).
+Do not mistake proposals, plans or instructions quoted in a transcript for accepted
+decisions. Preserve conditions, uncertainty and later corrections. If a reference
+cannot be resolved from the supplied context, do not invent its meaning.
+Each "text" is imperative, self-contained, at most two sentences, without secrets,
+credentials, or machine-local paths. "key" is a stable kebab-case identifier when
+the item clearly has a durable identity, otherwise null. Return {"candidates":[]}
+when nothing qualifies.`;
+export function parseExtraction(response) {
+    const parsed = extractJson(response);
+    if (!Array.isArray(parsed.candidates))
+        throw new Error("extraction response has no candidates");
+    const result = [];
+    for (const raw of parsed.candidates) {
+        const value = raw;
+        if (typeof value.text !== "string" || !value.text.trim())
+            continue;
+        result.push({
+            text: value.text.trim(),
+            key: typeof value.key === "string" && /^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(value.key)
+                ? value.key
+                : null,
+            reason: typeof value.reason === "string" ? value.reason : "",
+        });
+    }
+    return result.slice(0, 24);
+}
+export async function runSessions(context, executor, options = {}) {
+    if (!options.dryRun)
+        return executeSessions(context, executor, options);
+    const store = context.store.fork();
+    try {
+        return await executeSessions({ ...context, store }, executor, options);
+    }
+    finally {
+        store.close();
+    }
+}
+async function executeSessions(context, executor, options) {
+    const started = Date.now();
+    const dryRun = options.dryRun === true;
+    const report = {
+        sessionsIngested: 0,
+        candidatesExtracted: 0,
+        rejected: [],
+        dryRun,
+        ingestion: {
+            messages: 0,
+            fragments: 0,
+            completed: 0,
+            cached: 0,
+            characters: 0,
+            remaining: 0,
+            cachedSessions: 0,
+            unscannedSessions: 0,
+        },
+    };
+    if (!executor) {
+        options.observe?.({
+            phase: "sessions.disabled",
+            message: "No sessions model configured; nothing to extract",
+            level: "warn",
+        });
+        report.elapsedMs = Date.now() - started;
+        return report;
+    }
+    check(options);
+    options.observe?.({ phase: "sessions.scan", message: "Snapshotting eligible session messages" });
+    if (context.store.getMeta(WATERMARK_KEY) &&
+        !context.store.db
+            .prepare("SELECT 1 FROM meta WHERE key LIKE 'ingest-v2:%' OR key LIKE 'ingest-session-v2:%' LIMIT 1")
+            .get())
+        options.observe?.({
+            phase: "sessions.migration",
+            message: "Legacy cursor has no message revisions: checking existing history once; completed fragments will be cached",
+            level: "warn",
+        });
+    const databaseFile = options.databaseFile ?? opencodeDatabasePath();
+    if (!options.databaseFile &&
+        !(await stat(databaseFile).catch((error) => {
+            if (error.code === "ENOENT")
+                return null;
+            throw error;
+        }))) {
+        options.observe?.({
+            phase: "sessions.unavailable",
+            message: "No OpenCode database exists yet; nothing to extract",
+            level: "warn",
+        });
+        report.elapsedMs = Date.now() - started;
+        return report;
+    }
+    let plan;
+    try {
+        plan = planIngestion(options.databaseFile ?? opencodeDatabasePath(), context.store, {
+            before: Date.now() - context.settings.sessions.idleMs,
+            maxChars: context.settings.sessions.maxChars,
+            maxSessions: context.settings.sessions.maxSessions,
+        });
+    }
+    catch (error) {
+        throw new Error(`cannot read OpenCode sessions: ${error.message}`);
+    }
+    report.ingestion = {
+        messages: plan.messages,
+        fragments: plan.fragments.length,
+        completed: 0,
+        cached: plan.cached,
+        characters: plan.characters,
+        remaining: plan.fragments.length,
+        cachedSessions: plan.cachedSessions,
+        unscannedSessions: plan.unscannedSessions,
+    };
+    if (!options.dryRun)
+        for (const [key, value] of plan.seals)
+            context.store.setMeta(key, value);
+    options.observe?.({
+        phase: "sessions.plan",
+        message: "Session snapshot ready",
+        total: plan.fragments.length,
+        completed: 0,
+        messages: plan.messages,
+        cached: plan.cached,
+        characters: plan.characters,
+    });
+    const seen = new Set();
+    for (const file of await corpusFiles(context.paths))
+        for (const entry of parseCorpusEntries((await readTextIfExists(file)) ?? "", file))
+            seen.add(entry.text);
+    const rejected = [];
+    const sessions = new Set();
+    const extracted = [];
+    for (const fragment of plan.fragments) {
+        check(options);
+        options.observe?.({
+            phase: "sessions.extract",
+            message: "Extracting a new or changed fragment",
+            completed: report.ingestion.completed,
+            total: report.ingestion.fragments,
+            fragment: fragment.id,
+        });
+        const receipt = join(context.paths.historyDir, "ingestion", `${fragment.id}.json`);
+        const cached = await readTextIfExists(receipt);
+        let candidates;
+        if (cached) {
+            const value = JSON.parse(cached);
+            if (value.version !== 1 || value.id !== fragment.id)
+                throw new Error("invalid ingestion checkpoint");
+            candidates = parseExtraction(JSON.stringify({ candidates: value.candidates }));
+            options.observe?.({
+                phase: "sessions.resume",
+                message: "Reusing completed model response",
+                fragment: fragment.id,
+            });
+        }
+        else
+            candidates = parseExtraction(await executor.complete({
+                system: EXTRACT_SYSTEM,
+                prompt: `Session: ${fragment.sessionId}\nTitle: ${fragment.title}\nWorking directory: ${fragment.directory}\nMessage IDs: ${fragment.messageIds.join(",")}\nPrevious context (do not extract again):\n${fragment.context}\n\nNew or changed fragment:\n${fragment.text}`,
+            }));
+        if (!options.dryRun && !cached)
+            await writeAtomic(receipt, Buffer.from(JSON.stringify({
+                version: 1,
+                id: fragment.id,
+                sessionId: fragment.sessionId,
+                messageIds: fragment.messageIds,
+                candidates: candidates.filter((candidate) => !isForbidden(candidate.text, context.rules)),
+            })), 0o600);
+        check(options);
+        for (const candidate of candidates) {
+            if (isForbidden(candidate.text, context.rules)) {
+                rejected.push({ text: candidate.text, reason: "never-save rule" });
+                continue;
+            }
+            extracted.push(candidate);
+            const line = entryLine(candidate.text, {
+                key: candidate.key ?? undefined,
+                origin: "agent",
+                source: "opencode",
+                observed: new Date().toISOString().slice(0, 10),
+            });
+            const parsed = parseEntryLine(line);
+            const text = parsed?.text ?? `- ${candidate.text}`;
+            if (!options.dryRun && !seen.has(text))
+                await appendDailyEntry(context.paths, line);
+            seen.add(text);
+        }
+        if (!options.dryRun) {
+            for (const key of fragment.checkpointKeys)
+                context.store.setMeta(key, fragment.id);
+            if (fragment.seal)
+                context.store.setMeta(...fragment.seal);
+        }
+        sessions.add(fragment.sessionId);
+        report.ingestion.completed++;
+        report.ingestion.remaining--;
+        options.observe?.({
+            phase: "sessions.checkpoint",
+            message: "Fragment checkpoint committed",
+            completed: report.ingestion.completed,
+            total: report.ingestion.fragments,
+        });
+    }
+    if (!options.dryRun && sessions.size) {
+        context.store.setMeta(WATERMARK_KEY, String(plan.watermark));
+        context.store.setMeta(LAST_RUN_KEY, new Date().toISOString());
+    }
+    await reindex(context.paths, context.settings, context.store, options);
+    report.sessionsIngested = sessions.size;
+    report.candidatesExtracted = extracted.length;
+    report.rejected = rejected;
+    report.elapsedMs = Date.now() - started;
+    options.observe?.({
+        phase: "sessions.done",
+        message: "Session extraction complete",
+        elapsedMs: report.elapsedMs,
+        ...report.ingestion,
+    });
+    return report;
+}

package/dist/settings.d.ts

@@ -0,0 +1,48 @@
+export type MemomaticSettings = {
+    embedding: {
+        url: string;
+        model: string;
+        queryInstruction?: string;
+    } | null;
+    sessions: {
+        model: string | null;
+        variant: string | null;
+        timeoutMs: number;
+        maxDurationMs: number;
+        maxSessions: number;
+        maxChars: number;
+        retries: number;
+        idleMs: number;
+        opencodeUrl: string | null;
+    };
+    dream: {
+        model: string | null;
+        variant: string | null;
+        minScore: number;
+        minUseful: number;
+        minUniqueQueries: number;
+        maxCandidates: number;
+        memoryBudgetLines: number;
+        maxPriorEntryLoss: number;
+        timeoutMs: number;
+        maxDurationMs: number;
+        retries: number;
+    };
+    search: {
+        maxResults: number;
+        minScore: number;
+        minSemanticScore: number;
+        halfLifeDays: number;
+    };
+    archive: {
+        enabled: boolean;
+        days: number;
+    };
+};
+export declare const defaultSettings: () => MemomaticSettings;
+export declare function normalizeSettings(raw: unknown): MemomaticSettings;
+export declare function loadSettings(settingsFile: string): Promise<MemomaticSettings>;
+export declare function embedTexts(settings: MemomaticSettings, texts: string[], purpose?: "document" | "query", signal?: AbortSignal): Promise<Float32Array[] | null>;
+export declare const MEMORY_QUERY_INSTRUCTION = "Given a search query, retrieve relevant personal memory entries that answer the query.";
+export declare function embeddingQuery(settings: MemomaticSettings, query: string): string;
+export declare function embeddingFingerprint(settings: MemomaticSettings): string;