@kisev/memomatic 1.0.0-dev.100.g21366c4ad740

package/dist/executor.js

@@ -0,0 +1,267 @@
+import { spawn } from "node:child_process";
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { randomBytes } from "node:crypto";
+import { isolatedProviderConfig } from "./opencode-config.js";
+export function extractJson(response) {
+    const fenced = /```(?:json)?\s*([\s\S]*?)```/i.exec(response);
+    for (const candidate of [fenced?.[1], response]) {
+        if (!candidate)
+            continue;
+        const start = candidate.indexOf("{");
+        const end = candidate.lastIndexOf("}");
+        if (start < 0 || end <= start)
+            continue;
+        try {
+            return JSON.parse(candidate.slice(start, end + 1));
+        }
+        catch {
+            continue;
+        }
+    }
+    throw new Error("model response contains no JSON object");
+}
+export class OpenCodeExecutor {
+    options;
+    name = "opencode";
+    usage = {
+        calls: 0,
+        retries: 0,
+        inputTokens: 0,
+        outputTokens: 0,
+        cacheReadTokens: 0,
+        cacheWriteTokens: 0,
+        usageAvailable: false,
+        charactersSent: 0,
+    };
+    child;
+    directory;
+    url;
+    authorization;
+    constructor(options) {
+        this.options = options;
+    }
+    async initialize() {
+        if (this.url)
+            return;
+        if (this.options.url) {
+            const url = new URL(this.options.url);
+            if (!["http:", "https:"].includes(url.protocol) || url.username || url.password)
+                throw new Error("invalid OpenCode URL");
+            this.url = url.href.replace(/\/$/, "");
+            const password = process.env.OPENCODE_PASSWORD ?? process.env.OPENCODE_SERVER_PASSWORD;
+            if (password)
+                this.authorization = `Basic ${Buffer.from(`opencode:${password}`).toString("base64")}`;
+            return;
+        }
+        const inherited = await isolatedProviderConfig();
+        this.directory = await mkdtemp(join(tmpdir(), "memomatic-opencode-"));
+        const configFile = join(this.directory, "opencode.json");
+        await writeFile(configFile, JSON.stringify({
+            ...inherited,
+            plugins: ["-opencode.skill", "-opencode.config.skill", "-opencode.tool.skill"],
+            permissions: [{ action: "*", resource: "*", effect: "deny" }],
+            agents: {
+                memomatic: {
+                    mode: "primary",
+                    description: "Bounded memory extraction",
+                    system: "Return only the requested JSON. Do not use tools.",
+                    permissions: [{ action: "*", resource: "*", effect: "deny" }],
+                },
+            },
+        }), { mode: 0o600 });
+        const password = randomBytes(24).toString("hex");
+        this.authorization = `Basic ${Buffer.from(`opencode:${password}`).toString("base64")}`;
+        this.options.observe?.({
+            phase: "opencode.start",
+            message: "Starting one isolated OpenCode server for this run",
+        });
+        const child = spawn(this.options.binary ?? "opencode", ["serve", "--hostname", "127.0.0.1", "--port", "0"], {
+            cwd: this.directory,
+            detached: process.platform !== "win32",
+            stdio: ["ignore", "pipe", "pipe"],
+            env: {
+                ...process.env,
+                OPENCODE_SERVER_USERNAME: "opencode",
+                OPENCODE_SERVER_PASSWORD: password,
+                OPENCODE_CONFIG_PROJECT_DISABLE: "1",
+                OPENCODE_CONFIG_DIR: this.directory,
+                OPENCODE_CONFIG: configFile,
+                OPENCODE_CONFIG_CONTENT: "{}",
+                OPENCODE_PASSWORD: password,
+            },
+        });
+        this.child = child;
+        this.url = await new Promise((resolve, reject) => {
+            let output = "";
+            const timeout = setTimeout(() => done(new Error("OpenCode startup timed out")), 30000);
+            const abort = () => done(this.options.signal?.reason instanceof Error
+                ? this.options.signal.reason
+                : new Error("interrupted"));
+            const failed = (error) => done(error);
+            const closed = () => done(new Error("OpenCode exited before becoming ready"));
+            const data = (chunk) => {
+                output = (output + chunk.toString()).slice(-8192);
+                const match = /https?:\/\/127\.0\.0\.1:\d+/.exec(output);
+                if (match)
+                    done(undefined, match[0]);
+            };
+            const done = (error, url) => {
+                clearTimeout(timeout);
+                child.stdout?.off("data", data);
+                child.stderr?.off("data", data);
+                child.off("error", failed);
+                child.off("exit", closed);
+                this.options.signal?.removeEventListener("abort", abort);
+                if (error)
+                    reject(error);
+                else
+                    resolve(url);
+            };
+            child.stdout?.on("data", data);
+            child.stderr?.on("data", data);
+            child.once("error", failed);
+            child.once("exit", closed);
+            this.options.signal?.addEventListener("abort", abort, { once: true });
+            if (this.options.signal?.aborted)
+                abort();
+        });
+        // Drain logs without retaining provider output, credentials or prompts.
+        child.stdout?.resume();
+        child.stderr?.resume();
+        this.options.observe?.({ phase: "opencode.ready", message: "OpenCode server ready" });
+    }
+    async request(path, body, signal, method = "POST") {
+        const response = await fetch(`${this.url}${path}`, {
+            method,
+            headers: {
+                "content-type": "application/json",
+                ...(this.authorization ? { authorization: this.authorization } : {}),
+            },
+            body: JSON.stringify(body),
+            signal,
+        });
+        if (!response.ok)
+            throw new Error(`OpenCode ${path.includes("generate") ? "model request" : "session request"} failed: HTTP ${response.status}`);
+        return response.status === 204 ? {} : (await response.json());
+    }
+    async complete(request) {
+        this.options.signal?.throwIfAborted();
+        await this.initialize();
+        for (let attempt = 0; attempt <= (this.options.retries ?? 1); attempt++) {
+            this.options.signal?.throwIfAborted();
+            const signal = AbortSignal.any([
+                AbortSignal.timeout(this.options.timeoutMs ?? 180000),
+                ...(this.options.signal ? [this.options.signal] : []),
+            ]);
+            let id;
+            const started = Date.now();
+            const heartbeat = setInterval(() => this.options.observe?.({
+                phase: "model.wait",
+                message: "Waiting for OpenCode/model response",
+                elapsedMs: Date.now() - started,
+                attempt: attempt + 1,
+            }), 15000);
+            heartbeat.unref();
+            try {
+                const slash = this.options.model?.indexOf("/") ?? -1;
+                if (this.options.model && slash < 1)
+                    throw new Error("model must use provider/model format");
+                const session = await this.request("/api/session", {
+                    title: "[memomatic-internal] memory extraction",
+                    agent: this.options.url ? "build" : "memomatic",
+                    ...(this.directory ? { location: { directory: this.directory } } : {}),
+                    permissions: [{ action: "*", resource: "*", effect: "deny" }],
+                    ...(this.options.model
+                        ? {
+                            model: {
+                                providerID: this.options.model.slice(0, slash),
+                                id: this.options.model.slice(slash + 1),
+                                ...(this.options.variant ? { variant: this.options.variant } : {}),
+                            },
+                        }
+                        : {}),
+                }, signal);
+                const data = session.data;
+                if (typeof data?.id !== "string")
+                    throw new Error("OpenCode session response is malformed");
+                id = data.id;
+                await this.request(`/api/experimental/session/${encodeURIComponent(id)}/instructions/entries/memomatic`, { value: request.system }, signal, "PUT");
+                this.usage.calls++;
+                this.usage.charactersSent += request.system.length + request.prompt.length;
+                this.options.observe?.({
+                    phase: "model.request",
+                    message: "Sending prepared fragment to OpenCode",
+                    characters: request.prompt.length,
+                    attempt: attempt + 1,
+                });
+                const response = await this.request(`/api/session/${encodeURIComponent(id)}/generate`, { prompt: `[memomatic-internal] ${request.prompt}` }, signal);
+                const generated = response.data;
+                if (typeof generated?.text !== "string")
+                    throw new Error("OpenCode model response is malformed");
+                // V2 transient generation returns text only, not exact token usage.
+                const parsed = extractJson(generated.text);
+                this.options.observe?.({
+                    phase: "model.done",
+                    message: "Model response received",
+                    elapsedMs: Date.now() - started,
+                    ...this.usage,
+                });
+                return JSON.stringify(parsed);
+            }
+            catch (error) {
+                if (id)
+                    await this.request(`/api/session/${encodeURIComponent(id)}/interrupt?resume=false`, {}, AbortSignal.timeout(3000)).catch(() => undefined);
+                if (this.options.signal?.aborted)
+                    throw this.options.signal.reason;
+                if (attempt === (this.options.retries ?? 1))
+                    throw new Error(`OpenCode request failed after ${attempt + 1} attempt(s): ${error.message}`);
+                this.usage.retries++;
+                this.options.observe?.({
+                    phase: "model.retry",
+                    message: "Retrying failed model request",
+                    level: "warn",
+                    attempt: attempt + 2,
+                    reason: error.message,
+                });
+            }
+            finally {
+                clearInterval(heartbeat);
+            }
+        }
+        throw new Error("unreachable");
+    }
+    async close() {
+        const child = this.child;
+        if (child?.pid && child.exitCode === null && child.signalCode === null) {
+            await new Promise((resolve) => {
+                const kill = (signal) => {
+                    try {
+                        if (process.platform !== "win32" && child.pid)
+                            process.kill(-child.pid, signal);
+                        else
+                            child.kill(signal);
+                    }
+                    catch { }
+                };
+                const timer = setTimeout(() => kill("SIGKILL"), 1000);
+                timer.unref();
+                child.once("close", () => {
+                    clearTimeout(timer);
+                    resolve();
+                });
+                kill("SIGTERM");
+            });
+        }
+        if (child?.pid && process.platform !== "win32") {
+            try {
+                process.kill(-child.pid, "SIGKILL");
+            }
+            catch { }
+        }
+        this.child = undefined;
+        if (this.directory)
+            await rm(this.directory, { recursive: true, force: true });
+    }
+}

package/dist/gates.d.ts

@@ -0,0 +1,27 @@
+import type { MemomaticSettings } from "./settings.js";
+import type { IndexedEntry, MemoryStore } from "./store.js";
+export type PromotionCandidate = {
+    entry: IndexedEntry;
+    score: number;
+    signals: {
+        relevance: number;
+        frequency: number;
+        diversity: number;
+        recency: number;
+        consolidation: number;
+        richness: number;
+    };
+    eligible: boolean;
+    reason: string | null;
+};
+export declare function scoreCandidate(entry: IndexedEntry, usage: {
+    useful: number;
+    queries: string[];
+    days: string[];
+}, settings: MemomaticSettings, relevance: number, now?: number): PromotionCandidate;
+export declare function promotionCandidates(store: MemoryStore, settings: MemomaticSettings, relevanceByStableId: Map<string, number>, now?: number): PromotionCandidate[];
+/**
+ * Self-relevance of episodic entries that already passed the useful-signal
+ * gate: an entry matching its own text through FTS is consolidation-relevant.
+ */
+export declare function relevanceByFts(store: MemoryStore, settings: MemomaticSettings): Map<string, number>;

package/dist/gates.js

@@ -0,0 +1,69 @@
+export function scoreCandidate(entry, usage, settings, relevance, now = Date.now()) {
+    const signals = {
+        relevance: Math.max(0, Math.min(1, relevance)),
+        frequency: Math.min(1, usage.useful / 10),
+        diversity: Math.min(1, usage.queries.length / 5),
+        recency: entry.observedAt
+            ? Math.pow(0.5, Math.max(0, now - entry.observedAt) / 86_400_000 / settings.search.halfLifeDays)
+            : 0.5,
+        consolidation: Math.min(1, usage.useful / Math.max(1, settings.dream.minUseful)),
+        richness: Math.min(1, entry.text.length / 400),
+    };
+    const score = 0.3 * signals.relevance +
+        0.24 * signals.frequency +
+        0.15 * signals.diversity +
+        0.15 * signals.recency +
+        0.1 * signals.consolidation +
+        0.06 * signals.richness;
+    let eligible = true;
+    let reason = null;
+    if (entry.kind !== "episodic") {
+        eligible = false;
+        reason = "not episodic";
+    }
+    else if (entry.origin === null) {
+        eligible = false;
+        reason = "unknown origin";
+    }
+    else if (usage.useful < settings.dream.minUseful) {
+        eligible = false;
+        reason = `useful ${usage.useful} < ${settings.dream.minUseful}`;
+    }
+    else if (usage.queries.length < settings.dream.minUniqueQueries) {
+        eligible = false;
+        reason = `queries ${usage.queries.length} < ${settings.dream.minUniqueQueries}`;
+    }
+    else if (score < settings.dream.minScore) {
+        eligible = false;
+        reason = `score ${score.toFixed(3)} < ${settings.dream.minScore}`;
+    }
+    return { entry, score, signals, eligible, reason };
+}
+export function promotionCandidates(store, settings, relevanceByStableId, now = Date.now()) {
+    const candidates = [];
+    for (const entry of store.allEntries()) {
+        const usage = store.usageFor(entry.stableId);
+        candidates.push(scoreCandidate(entry, usage, settings, relevanceByStableId.get(entry.stableId) ?? 0, now));
+    }
+    return candidates
+        .filter((candidate) => candidate.eligible)
+        .sort((left, right) => right.score - left.score)
+        .slice(0, settings.dream.maxCandidates);
+}
+/**
+ * Self-relevance of episodic entries that already passed the useful-signal
+ * gate: an entry matching its own text through FTS is consolidation-relevant.
+ */
+export function relevanceByFts(store, settings) {
+    const relevance = new Map();
+    for (const entry of store.allEntries()) {
+        if (entry.kind !== "episodic")
+            continue;
+        if (store.usageFor(entry.stableId).useful < settings.dream.minUseful)
+            continue;
+        const match = store.ftsSearch(entry.text, 10).get(entry.stableId);
+        if (match)
+            relevance.set(entry.stableId, match);
+    }
+    return relevance;
+}

package/dist/generated/cli.d.ts

@@ -0,0 +1,28 @@
+import { Command, Option, CommanderError, InvalidArgumentError } from "commander";
+export { Command, Option, CommanderError, InvalidArgumentError };
+export declare function integer(value: string): number;
+export declare function duration(value: string): number;
+export declare function program(name: string, description: string, version: string): Command;
+export declare function common(command: Command, prefix: string): Command;
+export declare function configFile(path: string | undefined): Promise<Record<string, unknown>>;
+export declare function configFileSync(path: string | undefined): Record<string, unknown>;
+export declare function applyConfig(command: Command, config: Record<string, unknown>): void;
+export type Event = {
+    phase: string;
+    message: string;
+    level?: "debug" | "info" | "warn" | "error";
+    completed?: number;
+    total?: number;
+    elapsedMs?: number;
+    [key: string]: unknown;
+};
+export type Observe = (event: Event) => void;
+export declare function reporter(options?: Record<string, unknown>): {
+    emit: Observe;
+    close: () => void;
+};
+export declare function cancellation(): {
+    signal: AbortSignal;
+    dispose: () => void;
+};
+export declare function fail(name: string, error: unknown, json?: boolean): void;

package/dist/generated/cli.js

@@ -0,0 +1,181 @@
+import { Command, Option, CommanderError, InvalidArgumentError } from "commander";
+import { readFile } from "node:fs/promises";
+import { readFileSync } from "node:fs";
+export { Command, Option, CommanderError, InvalidArgumentError };
+export function integer(value) {
+    if (!/^\d+$/.test(value) || !Number.isSafeInteger(Number(value)))
+        throw new InvalidArgumentError("expected a non-negative integer");
+    return Number(value);
+}
+export function duration(value) {
+    const match = /^(\d+)(ms|s|m|h)?$/.exec(value);
+    if (!match)
+        throw new InvalidArgumentError("expected a duration such as 500ms, 30s or 5m");
+    const result = Number(match[1]) * ({ ms: 1, s: 1000, m: 60000, h: 3600000 }[match[2] ?? "ms"] ?? 1);
+    if (!Number.isSafeInteger(result) || result < 0)
+        throw new InvalidArgumentError("duration is out of range");
+    return result;
+}
+export function program(name, description, version) {
+    return new Command(name)
+        .description(description)
+        .version(version)
+        .exitOverride()
+        .configureHelp({ showGlobalOptions: true })
+        .showHelpAfterError()
+        .configureOutput({ outputError: () => { } });
+}
+export function common(command, prefix) {
+    return command
+        .addOption(new Option("--config <file>", "JSON configuration file (CLI > env > file > defaults)").env(`${prefix}_CONFIG`))
+        .addOption(new Option("--json", "machine-readable result on stdout").env(`${prefix}_JSON`))
+        .addOption(new Option("--log-level <level>", "diagnostic verbosity on stderr")
+        .choices(["debug", "info", "warn", "error", "silent"])
+        .env(`${prefix}_LOG_LEVEL`))
+        .addOption(new Option("--log-format <format>", "diagnostic format on stderr")
+        .choices(["text", "json"])
+        .env(`${prefix}_LOG_FORMAT`))
+        .addOption(new Option("--progress <mode>", "terminal progress; never written to stdout")
+        .choices(["auto", "always", "never"])
+        .env(`${prefix}_PROGRESS`))
+        .addOption(new Option("--color <mode>", "color policy (NO_COLOR disables color)")
+        .choices(["auto", "always", "never"])
+        .env(`${prefix}_COLOR`));
+}
+export async function configFile(path) {
+    if (!path)
+        return {};
+    const value = JSON.parse(await readFile(path, "utf8"));
+    if (!value || typeof value !== "object" || Array.isArray(value))
+        throw new InvalidArgumentError("configuration must be a JSON object");
+    return value;
+}
+export function configFileSync(path) {
+    if (!path)
+        return {};
+    const value = JSON.parse(readFileSync(path, "utf8"));
+    if (!value || typeof value !== "object" || Array.isArray(value))
+        throw new InvalidArgumentError("configuration must be a JSON object");
+    return value;
+}
+export function applyConfig(command, config) {
+    for (const option of command.options) {
+        const key = option.attributeName();
+        if (!(key in config) || ["cli", "env"].includes(command.getOptionValueSource(key) ?? ""))
+            continue;
+        if (["yes", "config"].includes(key))
+            continue;
+        const value = config[key];
+        if (option.required || option.optional) {
+            const raw = String(value);
+            if (option.argChoices && !option.argChoices.includes(raw))
+                throw new InvalidArgumentError(`invalid configuration value for ${key}`);
+            command.setOptionValueWithSource(key, option.parseArg ? option.parseArg(raw, undefined) : raw, "config");
+        }
+        else {
+            if (typeof value !== "boolean")
+                throw new InvalidArgumentError(`${key} must be boolean`);
+            command.setOptionValueWithSource(key, value, "config");
+        }
+    }
+}
+export function reporter(options = {}) {
+    const started = Date.now();
+    const level = String(options.logLevel ?? "info");
+    const levels = ["debug", "info", "warn", "error", "silent"];
+    const json = options.logFormat === "json";
+    const interactive = !json &&
+        options.progress !== "never" &&
+        (options.progress === "always" || process.stderr.isTTY);
+    let current;
+    let painted = false;
+    let frame = 0;
+    let meter;
+    const bar = () => {
+        if (!meter || meter.total <= 0)
+            return "";
+        const ratio = Math.min(1, Math.max(0, meter.completed / meter.total));
+        const filled = Math.floor(ratio * 16);
+        return ` | ${meter.phase} [${"█".repeat(filled)}${"░".repeat(16 - filled)}] ${Math.floor(ratio * 100)}%`;
+    };
+    const clean = (text) => Array.from(text, (char) => {
+        const code = char.codePointAt(0);
+        return code < 32 || (code >= 127 && code <= 159) ? " " : char;
+    }).join("");
+    const line = (event) => `${event.phase}: ${event.message}${event.total !== undefined ? ` [${event.completed ?? 0}/${event.total}]` : ""} (${Math.floor((Date.now() - started) / 1000)}s)`;
+    const clear = () => {
+        if (painted) {
+            process.stderr.write("\r\x1b[2K");
+            painted = false;
+        }
+    };
+    const timer = interactive && level !== "silent"
+        ? setInterval(() => {
+            if (current) {
+                const color = process.env.NO_COLOR === undefined &&
+                    (options.color === "always" || (options.color !== "never" && process.stderr.isTTY));
+                process.stderr.write(`\r\x1b[2K${color ? "\x1b[36m" : ""}${["⠋", "⠙", "⠹", "⠸"][frame++ % 4]} ${clean(line(current))}${bar()}${color ? "\x1b[0m" : ""}`);
+                painted = true;
+            }
+        }, 120)
+        : undefined;
+    timer?.unref();
+    const emit = (event) => {
+        const severity = event.level ?? "info";
+        if (levels.indexOf(severity) < levels.indexOf(level))
+            return;
+        current = event;
+        if (event.total !== undefined)
+            meter =
+                event.total > 0
+                    ? {
+                        phase: event.phase.split(".")[0],
+                        completed: event.completed ?? 0,
+                        total: event.total,
+                    }
+                    : undefined;
+        else if (meter && meter.completed >= meter.total)
+            meter = undefined;
+        clear();
+        process.stderr.write(json
+            ? `${JSON.stringify({ at: new Date().toISOString(), elapsedMs: Date.now() - started, ...event })}\n`
+            : `${clean(line(event))}\n`);
+    };
+    return {
+        emit,
+        close: () => {
+            if (timer)
+                clearInterval(timer);
+            clear();
+        },
+    };
+}
+export function cancellation() {
+    const controller = new AbortController();
+    const interrupt = () => controller.abort(new Error("interrupted"));
+    process.on("SIGINT", interrupt);
+    process.on("SIGTERM", interrupt);
+    process.on("SIGHUP", interrupt);
+    return {
+        signal: controller.signal,
+        dispose: () => {
+            process.off("SIGINT", interrupt);
+            process.off("SIGTERM", interrupt);
+            process.off("SIGHUP", interrupt);
+        },
+    };
+}
+export function fail(name, error, json = false) {
+    if (error instanceof CommanderError && error.exitCode === 0)
+        return;
+    const message = error instanceof Error ? error.message : String(error);
+    const interrupted = /interrupted|aborted/i.test(message);
+    process.stderr.write(json ? `${JSON.stringify({ status: "error", message })}\n` : `${name}: ${message}\n`);
+    process.exitCode = /timeout|timed out|deadline/i.test(message)
+        ? 124
+        : interrupted
+            ? 130
+            : error instanceof CommanderError
+                ? 2
+                : 1;
+}

package/dist/inbox.d.ts

@@ -0,0 +1,34 @@
+import { type OperationOptions } from "./operations.js";
+import type { MemomaticContext } from "./service.js";
+import type { MemomaticPaths } from "./paths.js";
+export type InboxReport = {
+    filesProcessed: number;
+    filesRejected: number;
+    entriesAppended: number;
+    entriesSuperseded: number;
+    entriesDuplicated: number;
+    linesForbidden: number;
+    linesInvalid: number;
+    dryRun: boolean;
+};
+export declare function inboxFileName(source: string, now?: Date): string;
+/**
+ * Append one inbox drop file. Producers write Markdown entry lines that the
+ * deterministic `process` pass later validates, gates, and moves into the
+ * corpus. Nothing here touches corpus files.
+ */
+export declare function dropToInbox(paths: MemomaticPaths, lines: string[], source: string): Promise<string>;
+/**
+ * Deterministic inbox pass: validate entry lines, enforce never-save rules,
+ * deduplicate exact texts, supersede entries sharing a key, route user-origin
+ * targets, then rebuild the index with embeddings. No model turns.
+ */
+export declare function processInbox(context: MemomaticContext, options?: OperationOptions & {
+    dryRun?: boolean;
+    deferIndex?: boolean;
+}): Promise<InboxReport>;
+/**
+ * Serialize whole runs (dream or process) with a stale-tolerant lock file so a
+ * nightly sweep and a manual CLI invocation never mutate the corpus together.
+ */
+export declare function withRunLock<T>(paths: MemomaticPaths, work: () => Promise<T>): Promise<T>;