@kisev/memomatic 1.0.0-dev.100.g21366c4ad740

package/dist/corpus.js

@@ -0,0 +1,138 @@
+import { createHash } from "node:crypto";
+import { lstat, mkdir, open, readFile, rm } from "node:fs/promises";
+import { dirname, join, relative } from "node:path";
+import { writeAtomic } from "@kisev/safe-fs";
+import { parseEntryLine } from "./entries.js";
+const CURATED_FILES = ["MEMORY.md", "USER.md"];
+function inside(root, target) {
+    const value = relative(root, target);
+    return value === "" || (!value.startsWith("..") && value !== "..");
+}
+async function safeDirectory(path, boundary, create) {
+    if (!inside(boundary, path))
+        throw new Error("memomatic path escapes its root");
+    if (create)
+        await mkdir(path, { mode: 0o700, recursive: true });
+    let current = boundary;
+    for (const piece of relative(current, path).split("/").filter(Boolean)) {
+        current = join(current, piece);
+        const info = await lstat(current);
+        if (!info.isDirectory() || info.isSymbolicLink())
+            throw new Error(`memomatic directory is unsafe: ${current}`);
+    }
+}
+async function writePreImage(paths, source, previous) {
+    await safeDirectory(paths.historyDir, paths.stateRoot, true);
+    const name = `${createHash("sha256").update(previous).digest("hex")}.md`;
+    const target = join(paths.historyDir, name);
+    try {
+        const file = await open(target, "wx", 0o600);
+        try {
+            await file.writeFile(previous);
+            await file.sync();
+        }
+        finally {
+            await file.close();
+        }
+    }
+    catch (error) {
+        if (error.code !== "EEXIST")
+            throw error;
+        const info = await lstat(target);
+        if (!info.isFile() || info.isSymbolicLink() || !(await readFile(target)).equals(previous))
+            throw new Error("memomatic history was changed");
+    }
+}
+export async function readTextIfExists(path) {
+    try {
+        return await readFile(path, "utf8");
+    }
+    catch (error) {
+        if (error.code === "ENOENT")
+            return undefined;
+        throw error;
+    }
+}
+export async function writeCorpusFile(paths, file, next) {
+    if (!inside(paths.stateRoot, file))
+        throw new Error("memomatic path escapes its root");
+    const previous = await readFile(file).catch(() => undefined);
+    if (previous !== undefined && previous.toString("utf8") === next)
+        return;
+    await safeDirectory(dirname(file), paths.stateRoot, true);
+    if (previous !== undefined)
+        await writePreImage(paths, file, previous);
+    await writeAtomic(file, Buffer.from(next), 0o600);
+}
+export function dailyNotePath(paths, date = new Date()) {
+    const iso = date.toISOString().slice(0, 10);
+    return join(paths.dailyDir, `${iso}.md`);
+}
+export function entryKind(file) {
+    const name = file.slice(file.lastIndexOf("/") + 1);
+    if (name === "MEMORY.md")
+        return "curated";
+    if (name === "USER.md")
+        return "user";
+    return "episodic";
+}
+export function parseCorpusEntries(markdown, file) {
+    const entries = [];
+    const lines = markdown.split("\n");
+    for (let index = 0; index < lines.length; index += 1) {
+        const parsed = parseEntryLine(lines[index]);
+        if (!parsed)
+            continue;
+        entries.push({ file, line: index + 1, text: parsed.text, annotations: parsed.annotations });
+    }
+    return entries;
+}
+export async function appendDailyEntry(paths, line, date = new Date()) {
+    const file = dailyNotePath(paths, date);
+    const current = (await readTextIfExists(file)) ?? "";
+    const next = current.length ? `${current.trimEnd()}\n${line}\n` : `${line}\n`;
+    await writeCorpusFile(paths, file, next);
+    return file;
+}
+export async function replaceEntryLine(paths, file, line, replacement) {
+    const content = await readFile(file, "utf8");
+    const lines = content.split("\n");
+    if (line < 1 || line > lines.length)
+        throw new Error("entry line is out of range");
+    if (replacement === null) {
+        lines.splice(line - 1, 1);
+        const rest = lines.join("\n");
+        await writeCorpusFile(paths, file, rest.length ? `${rest}` : "");
+        return;
+    }
+    lines[line - 1] = replacement;
+    await writeCorpusFile(paths, file, lines.join("\n"));
+}
+export async function appendDreams(paths, report) {
+    const stamp = new Date().toISOString().replace("T", " ").slice(0, 16);
+    const current = (await readTextIfExists(paths.dreamsFile)) ?? "";
+    const block = `## ${stamp}\n\n${report.trim()}\n\n`;
+    const next = `${current}${current.length ? "\n" : ""}${block}`;
+    await writeCorpusFile(paths, paths.dreamsFile, next);
+}
+export async function archiveFile(paths, file) {
+    if (!inside(paths.stateRoot, file))
+        throw new Error("memomatic path escapes its root");
+    const name = file.slice(file.lastIndexOf("/") + 1);
+    const target = join(paths.archiveDir, `${name}`);
+    await safeDirectory(paths.archiveDir, paths.stateRoot, true);
+    const content = await readFile(file);
+    const existing = await readFile(target).catch(() => undefined);
+    if (existing !== undefined) {
+        const merged = `${existing.toString("utf8").trimEnd()}\n${content.toString("utf8").trimEnd()}\n`;
+        await writeAtomic(target, Buffer.from(merged), 0o600);
+    }
+    else {
+        await writeAtomic(target, content, 0o600);
+    }
+    await rm(file);
+    return target;
+}
+export function isCuratedFile(file) {
+    return CURATED_FILES.some((name) => file.endsWith(name));
+}

package/dist/dream.d.ts

@@ -0,0 +1,31 @@
+import { type InboxReport } from "./inbox.js";
+import { type ModelExecutor } from "./executor.js";
+import { type OperationOptions } from "./operations.js";
+import type { MemomaticContext } from "./service.js";
+export declare const LAST_RUN_KEY = "last-dream-at";
+export type DreamReport = {
+    inbox: InboxReport;
+    promoted: string[];
+    superseded: string[];
+    dropped: string[];
+    archived: string[];
+    appendOnlyFallback: boolean;
+    dryRun: boolean;
+    elapsedMs?: number;
+};
+export type DreamOptions = OperationOptions & {
+    dryRun?: boolean;
+};
+export type ConsolidationOperation = {
+    op: "add";
+    line: string;
+} | {
+    op: "supersede";
+    key: string;
+    line: string;
+} | {
+    op: "drop";
+    key: string;
+};
+export declare function parseConsolidation(response: string): ConsolidationOperation[];
+export declare function runDream(context: MemomaticContext, executor: ModelExecutor | null, options?: DreamOptions): Promise<DreamReport>;

package/dist/dream.js

@@ -0,0 +1,234 @@
+import { entryLine, parseEntryLine } from "./entries.js";
+import { appendDreams, readTextIfExists, writeCorpusFile } from "./corpus.js";
+import { processInbox } from "./inbox.js";
+import { promotionCandidates, relevanceByFts } from "./gates.js";
+import { extractJson } from "./executor.js";
+import { check } from "./operations.js";
+import { createHash } from "node:crypto";
+import { reindex } from "./search.js";
+import { archiveOldEpisodic } from "./service.js";
+export const LAST_RUN_KEY = "last-dream-at";
+const CONSOLIDATE_SYSTEM = `You consolidate a long-term memory file.
+Reply with exactly one JSON object of the form
+{"operations":[{"op":"add","line":string},{"op":"supersede","key":string,"line":string},{"op":"drop","key":string}]}.
+Merge duplicates, supersede outdated entries sharing a key instead of appending,
+keep every line compact (at most two sentences), keep or assign "key" annotations
+in the form <!-- key: ... -->, and add <!-- trigger: ... --> phrases only when a
+clear activation context exists. Never invent keys for "supersede" or "drop" that
+are not present in the current file.`;
+export function parseConsolidation(response) {
+    const parsed = extractJson(response);
+    if (!Array.isArray(parsed.operations))
+        throw new Error("consolidation response has no operations");
+    const result = [];
+    for (const raw of parsed.operations) {
+        const value = raw;
+        if (value.op === "add" && typeof value.line === "string") {
+            const line = value.line.trim();
+            if (parseEntryLine(line))
+                result.push({ op: "add", line });
+        }
+        else if (value.op === "supersede" &&
+            typeof value.key === "string" &&
+            typeof value.line === "string") {
+            const line = value.line.trim();
+            if (parseEntryLine(line))
+                result.push({ op: "supersede", key: value.key, line });
+        }
+        else if (value.op === "drop" && typeof value.key === "string") {
+            result.push({ op: "drop", key: value.key });
+        }
+    }
+    return result.slice(0, 32);
+}
+export async function runDream(context, executor, options = {}) {
+    if (!options.dryRun)
+        return executeDream(context, executor, options);
+    const store = context.store.fork();
+    try {
+        return await executeDream({ ...context, store }, executor, options);
+    }
+    finally {
+        store.close();
+    }
+}
+async function executeDream(context, executor, options) {
+    const started = Date.now();
+    const dryRun = options.dryRun === true;
+    const report = {
+        inbox: {
+            filesProcessed: 0,
+            filesRejected: 0,
+            entriesAppended: 0,
+            entriesSuperseded: 0,
+            entriesDuplicated: 0,
+            linesForbidden: 0,
+            linesInvalid: 0,
+            dryRun,
+        },
+        promoted: [],
+        superseded: [],
+        dropped: [],
+        archived: [],
+        appendOnlyFallback: false,
+        dryRun,
+    };
+    options.observe?.({
+        phase: "dream.start",
+        message: dryRun ? "Previewing Dream without persistent writes" : "Starting Dream",
+    });
+    report.inbox = await processInbox(context, { ...options, dryRun, deferIndex: true });
+    await reindex(context.paths, context.settings, context.store, options);
+    const relevance = relevanceByFts(context.store, context.settings);
+    const finalCandidates = promotionCandidates(context.store, context.settings, relevance);
+    if (executor && finalCandidates.length) {
+        check(options);
+        options.observe?.({
+            phase: "consolidation",
+            message: "Consolidating eligible memory",
+            candidates: finalCandidates.length,
+        });
+        const current = (await readTextIfExists(context.paths.memoryFile)) ?? "";
+        const consolidationKey = `consolidation-v1:${createHash("sha256")
+            .update(JSON.stringify([
+            current,
+            finalCandidates.map((candidate) => [candidate.entry.stableId, candidate.entry.text]),
+            context.settings.dream.model,
+            context.settings.dream.variant,
+        ]))
+            .digest("hex")}`;
+        const cachedResponse = context.store.getMeta(consolidationKey);
+        const response = cachedResponse ??
+            (await executor.complete({
+                system: CONSOLIDATE_SYSTEM,
+                prompt: `Current MEMORY.md:\n${current || "(empty)"}\n\nCandidates:\n${finalCandidates
+                    .map((candidate) => `- ${candidate.entry.text} <!-- key: ${candidate.entry.key ?? candidate.entry.stableId} -->`)
+                    .join("\n")}`,
+            }));
+        let operations;
+        let invalidResponse = false;
+        try {
+            operations = parseConsolidation(response);
+        }
+        catch {
+            invalidResponse = true;
+            operations = finalCandidates.map(({ entry }) => ({
+                op: "add",
+                line: entryLine(entry.text.replace(/^- /, ""), {
+                    key: entry.key ?? undefined,
+                    origin: entry.origin ?? "agent",
+                    source: entry.source ?? undefined,
+                }),
+            }));
+        }
+        check(options);
+        const outcome = await applyConsolidation(context, operations, current, dryRun);
+        if (!dryRun)
+            context.store.setMeta(consolidationKey, response);
+        if (cachedResponse)
+            options.observe?.({
+                phase: "consolidation.cached",
+                message: "Reused unchanged consolidation input",
+            });
+        report.promoted = outcome.promoted;
+        report.superseded = outcome.superseded;
+        report.dropped = outcome.dropped;
+        report.appendOnlyFallback = invalidResponse || outcome.appendOnlyFallback;
+    }
+    if (!dryRun) {
+        report.archived = await archiveOldEpisodic(context, options);
+        context.store.setMeta(LAST_RUN_KEY, new Date().toISOString());
+    }
+    if (report.promoted.length || report.superseded.length || report.archived.length)
+        await reindex(context.paths, context.settings, context.store, options);
+    const summary = [
+        `- inbox files processed: ${report.inbox.filesProcessed}`,
+        `- inbox entries appended: ${report.inbox.entriesAppended}`,
+        `- inbox entries superseded: ${report.inbox.entriesSuperseded}`,
+        `- inbox files rejected: ${report.inbox.filesRejected}`,
+        `- promoted: ${report.promoted.length}${report.promoted.length ? ` (${report.promoted.join(", ")})` : ""}`,
+        `- superseded: ${report.superseded.length ? report.superseded.join(", ") : "none"}`,
+        `- dropped: ${report.dropped.length ? report.dropped.join(", ") : "none"}`,
+        `- archived files: ${report.archived.length ? report.archived.join(", ") : "none"}`,
+        `- append-only fallback: ${report.appendOnlyFallback}`,
+        `- dry run: ${report.dryRun}`,
+    ].join("\n");
+    if (!dryRun)
+        await appendDreams(context.paths, summary);
+    report.elapsedMs = Date.now() - started;
+    options.observe?.({
+        phase: "dream.done",
+        message: "Dream complete",
+        elapsedMs: report.elapsedMs,
+    });
+    return report;
+}
+async function applyConsolidation(context, operations, currentMemory, dryRun) {
+    const result = {
+        promoted: [],
+        superseded: [],
+        dropped: [],
+        appendOnlyFallback: false,
+    };
+    if (!operations.length)
+        return result;
+    const lines = currentMemory.split("\n").filter((line) => line.trim().length > 0);
+    const existingKeys = new Set(lines
+        .map((line) => parseEntryLine(line)?.annotations.key)
+        .filter((key) => Boolean(key)));
+    const removals = operations.filter((operation) => operation.op !== "add");
+    const lossRatio = lines.length ? removals.length / lines.length : 0;
+    const adds = operations.filter((operation) => operation.op === "add");
+    const supersessions = operations.filter((operation) => operation.op === "supersede" && existingKeys.has(operation.key));
+    const drops = operations.filter((operation) => operation.op === "drop" && existingKeys.has(operation.key));
+    // Consolidation never grants permission to delete curated memory.
+    if (drops.length)
+        result.appendOnlyFallback = true;
+    if (removals.length !== supersessions.length + drops.length)
+        result.appendOnlyFallback = true;
+    if (!result.appendOnlyFallback && lossRatio > context.settings.dream.maxPriorEntryLoss)
+        result.appendOnlyFallback = true;
+    if (result.appendOnlyFallback) {
+        const budget = context.settings.dream.memoryBudgetLines;
+        if (lines.length + adds.length > budget)
+            return result;
+        const next = `${lines.join("\n")}\n${adds.map((operation) => operation.line).join("\n")}\n`;
+        if (!dryRun)
+            await writeCorpusFile(context.paths, context.paths.memoryFile, next);
+        result.promoted = adds.map((operation) => operation.line);
+        return result;
+    }
+    const nextLines = [...lines];
+    for (const operation of supersessions) {
+        const index = nextLines.findIndex((line) => parseEntryLine(line)?.annotations.key === operation.key);
+        if (index === -1)
+            continue;
+        nextLines[index] = operation.line;
+        result.superseded.push(operation.key);
+    }
+    for (const operation of drops) {
+        const index = nextLines.findIndex((line) => parseEntryLine(line)?.annotations.key === operation.key);
+        if (index === -1)
+            continue;
+        nextLines.splice(index, 1);
+        result.dropped.push(operation.key);
+    }
+    for (const operation of adds) {
+        const key = parseEntryLine(operation.line)?.annotations.key;
+        if (key && nextLines.some((line) => parseEntryLine(line)?.annotations.key === key))
+            continue;
+        nextLines.push(operation.line);
+        result.promoted.push(operation.line);
+    }
+    if (nextLines.length > context.settings.dream.memoryBudgetLines) {
+        result.appendOnlyFallback = true;
+        result.promoted = [];
+        result.superseded = [];
+        result.dropped = [];
+        return result;
+    }
+    const next = `${nextLines.join("\n")}\n`;
+    if (!dryRun)
+        await writeCorpusFile(context.paths, context.paths.memoryFile, next);
+    return result;
+}

package/dist/entries.d.ts

@@ -0,0 +1,31 @@
+export type EntryAnnotations = {
+    key?: string;
+    status?: "active" | "superseded";
+    origin?: "user" | "agent";
+    observed?: string;
+    project?: string;
+    importance?: number;
+    trigger?: string[];
+    pinned?: boolean;
+    source?: string;
+    target?: "episodic" | "curated" | "user";
+};
+export type CorpusEntry = {
+    file: string;
+    line: number;
+    text: string;
+    annotations: EntryAnnotations;
+};
+export declare function parseEntryLine(line: string): {
+    text: string;
+    annotations: EntryAnnotations;
+} | null;
+export declare function serializeAnnotations(annotations: EntryAnnotations, options?: {
+    dropTarget?: boolean;
+}): string;
+export declare function entryLine(text: string, annotations: EntryAnnotations, options?: {
+    dropTarget?: boolean;
+}): string;
+export declare function entryKey(entry: CorpusEntry): string | null;
+export declare function entryImportance(entry: CorpusEntry): number;
+export declare function entryObservedAt(entry: CorpusEntry): number;

package/dist/entries.js

@@ -0,0 +1,119 @@
+const SOURCE_PATTERN = /^[a-z][a-z0-9-]{0,63}$/;
+const TRAILING_ANNOTATION = /\s*<!--\s*([a-z-]+):\s*([^<]*?)\s*-->\s*$/;
+export function parseEntryLine(line) {
+    let value = line.trimEnd();
+    if (!value.startsWith("- "))
+        return null;
+    const annotations = {};
+    for (let index = 0; index < 8; index += 1) {
+        const match = TRAILING_ANNOTATION.exec(value);
+        if (!match)
+            break;
+        const name = match[1];
+        const raw = match[2];
+        let consumed = true;
+        if (name === "key" || name === "project" || name === "observed" || name === "source") {
+            if (name === "observed")
+                annotations.observed = raw;
+            else if (name === "project")
+                annotations.project = raw;
+            else if (name === "source") {
+                if (SOURCE_PATTERN.test(raw))
+                    annotations.source = raw;
+                else
+                    consumed = false;
+            }
+            else
+                annotations.key = raw;
+        }
+        else if (name === "target") {
+            if (raw === "episodic" || raw === "curated" || raw === "user")
+                annotations.target = raw;
+            else
+                consumed = false;
+        }
+        else if (name === "status") {
+            if (raw === "active" || raw === "superseded")
+                annotations.status = raw;
+            else
+                consumed = false;
+        }
+        else if (name === "origin") {
+            if (raw === "user" || raw === "agent")
+                annotations.origin = raw;
+            else
+                consumed = false;
+        }
+        else if (name === "importance") {
+            const parsed = Number.parseInt(raw, 10);
+            if (Number.isFinite(parsed) && parsed >= 1 && parsed <= 10)
+                annotations.importance = parsed;
+            else
+                consumed = false;
+        }
+        else if (name === "trigger") {
+            annotations.trigger = raw
+                .split(/[;,]\s*/)
+                .map((item) => item.trim())
+                .filter(Boolean);
+            if (!annotations.trigger.length)
+                consumed = false;
+        }
+        else if (name === "pinned") {
+            if (raw === "true")
+                annotations.pinned = true;
+            else
+                consumed = false;
+        }
+        else
+            consumed = false;
+        if (!consumed)
+            break;
+        value = value.slice(0, match.index).trimEnd();
+    }
+    return { text: value, annotations };
+}
+export function serializeAnnotations(annotations, options = {}) {
+    const parts = [];
+    if (annotations.key)
+        parts.push(`key: ${annotations.key}`);
+    if (annotations.status)
+        parts.push(`status: ${annotations.status}`);
+    if (annotations.origin)
+        parts.push(`origin: ${annotations.origin}`);
+    if (annotations.observed)
+        parts.push(`observed: ${annotations.observed}`);
+    if (annotations.project)
+        parts.push(`project: ${annotations.project}`);
+    if (annotations.importance !== undefined)
+        parts.push(`importance: ${annotations.importance}`);
+    if (annotations.trigger?.length)
+        parts.push(`trigger: ${annotations.trigger.join("; ")}`);
+    if (annotations.pinned)
+        parts.push("pinned: true");
+    if (annotations.source)
+        parts.push(`source: ${annotations.source}`);
+    if (!options.dropTarget && annotations.target)
+        parts.push(`target: ${annotations.target}`);
+    return parts.map((part) => `<!-- ${part} -->`).join(" ");
+}
+export function entryLine(text, annotations, options = {}) {
+    const suffix = serializeAnnotations(annotations, options);
+    return `- ${text}${suffix ? ` ${suffix}` : ""}`;
+}
+export function entryKey(entry) {
+    if (entry.annotations.key)
+        return entry.annotations.key;
+    return null;
+}
+export function entryImportance(entry) {
+    return entry.annotations.importance ?? 5;
+}
+export function entryObservedAt(entry) {
+    if (entry.annotations.observed) {
+        const parsed = Date.parse(entry.annotations.observed);
+        if (Number.isFinite(parsed))
+            return parsed;
+    }
+    return Number.NaN;
+}

package/dist/executor.d.ts

@@ -0,0 +1,45 @@
+import type { OperationOptions } from "./operations.js";
+export type ModelExecutor = {
+    name: string;
+    complete(request: {
+        system: string;
+        prompt: string;
+    }): Promise<string>;
+    close?: () => Promise<void>;
+};
+export declare function extractJson(response: string): unknown;
+type ExecutorOptions = OperationOptions & {
+    model: string | null;
+    variant: string | null;
+    binary?: string;
+    url?: string | null;
+    timeoutMs?: number;
+    retries?: number;
+};
+export declare class OpenCodeExecutor implements ModelExecutor {
+    private readonly options;
+    readonly name = "opencode";
+    readonly usage: {
+        calls: number;
+        retries: number;
+        inputTokens: number;
+        outputTokens: number;
+        cacheReadTokens: number;
+        cacheWriteTokens: number;
+        usageAvailable: boolean;
+        charactersSent: number;
+    };
+    private child?;
+    private directory?;
+    private url?;
+    private authorization?;
+    constructor(options: ExecutorOptions);
+    private initialize;
+    private request;
+    complete(request: {
+        system: string;
+        prompt: string;
+    }): Promise<string>;
+    close(): Promise<void>;
+}
+export {};