om-memory-system 4.13.3 → 4.14.1

package/dist/importer/web-import-group.js

@@ -0,0 +1,355 @@
+import { randomUUID } from "node:crypto";
+import { CONFIG } from "../config.js";
+import { hostLabel } from "../types/host-label.js";
+import { parseHistoryImportArgs, webImportTokens, } from "./import-args.js";
+import { importBlockedReason } from "./import-readiness.js";
+import { resolveImportSelection } from "./import-sessions.js";
+import { importSourceKey, readImportSourceToken, } from "./import-sources.js";
+import { historyImportFailed, ImportAlreadyRunningError, runHistoryImport, summarizeHistoryImportReport, } from "./run-import.js";
+import { safeHealthError } from "./settings-health.js";
+import { ImportJobError } from "./web-import-jobs.js";
+import { profileEstimate, readWaitingProfilePrompts, } from "./web-import-profile-estimate.js";
+import { profileFailureCode } from "../core/profile-failure.js";
+async function prepareModels(choice, directory, signal) {
+    if (signal.aborted)
+        return {};
+    let models;
+    if (choice === "external") {
+        const { selectImportModel } = await import("./model-selection.js");
+        models = selectImportModel({});
+    }
+    else {
+        const [providerID = "", ...id] = choice.split("/");
+        const { getOpencodeHostModels } = await import("./backfill-controls.js");
+        const opencode = getOpencodeHostModels();
+        if (!opencode?.isProviderConnected(providerID))
+            throw new Error("OpenCode provider is not connected");
+        models = await opencode.createImportModels({ providerID, modelID: id.join("/") }, directory);
+    }
+    if (!signal.aborted) {
+        const { memoryClient } = await import("../services/client.js");
+        await memoryClient.warmup();
+    }
+    return models;
+}
+async function discardSelection(identity) {
+    if (identity.kind !== "opencode-db")
+        return;
+    const { opencodeSnapshots } = await import("./opencode-snapshot.js");
+    opencodeSnapshots.discard(importSourceKey(identity));
+}
+function matchOptions(child, directory, signal) {
+    return {
+        host: child.input.host,
+        scope: child.args.scope,
+        ...(child.args.project ? { project: child.args.project } : {}),
+        pathMaps: child.args.pathMaps,
+        cwd: directory,
+        allowEmpty: true,
+        signal,
+    };
+}
+function hostError(host, error) {
+    return new ImportJobError(`${hostLabel(host)}: ${safeHealthError(error, [CONFIG.memoryApiKey, CONFIG.embeddingApiKey])}`, error.status ?? 400);
+}
+function emptySummary(dryRun) {
+    return {
+        dryRun,
+        sessionsDiscovered: 0,
+        sessionsLoaded: 0,
+        sessionsFilteredOut: 0,
+        unitsTotal: 0,
+        unitsImported: 0,
+        unitsWouldImport: 0,
+        unitsSkipped: 0,
+        unitsFailed: 0,
+        unitsAlreadyHandled: 0,
+        unitsHeldBack: 0,
+        unitsUntimed: 0,
+        projects: 0,
+        unresolved: 0,
+        loadErrors: 0,
+    };
+}
+function updateTotals(job) {
+    job.sessions = job.hosts.reduce((n, child) => n + child.sessions, 0);
+    job.processed = job.hosts.reduce((n, child) => n + child.processed, 0);
+    job.total = job.hosts.reduce((n, child) => n + child.total, 0);
+    const reports = job.hosts.flatMap((child) => (child.summary ? [child.summary] : []));
+    const sum = {
+        ...(reports[0] ?? emptySummary(job.dryRun)),
+        profile: undefined,
+    };
+    for (const key of Object.keys(sum)) {
+        if (typeof sum[key] !== "number")
+            continue;
+        sum[key] = reports.reduce((n, report) => n + report[key], 0);
+    }
+    const profiles = reports.flatMap((report) => (report.profile ? [report.profile] : []));
+    if (profiles.length)
+        sum.profile = {
+            promptsRecorded: profiles.reduce((n, p) => n + p.promptsRecorded, 0),
+            promptsWouldRecord: profiles.reduce((n, p) => n + p.promptsWouldRecord, 0),
+            promptsAlreadyHandled: profiles.reduce((n, p) => n + p.promptsAlreadyHandled, 0),
+            batchesBuilt: profiles.reduce((n, p) => n + p.batchesBuilt, 0),
+            // Every child drains the same queue. Its last reading is the remaining backlog.
+            remaining: profiles.at(-1).remaining,
+            failed: profiles.some((p) => p.failed),
+        };
+    job.summary = sum;
+}
+/** Preflight every child before launching a bounded sequential group in the existing slot. */
+export async function startImportGroup(input, directory, deps, publish) {
+    const controller = new AbortController();
+    const job = {
+        id: randomUUID(),
+        dryRun: Boolean(input.options.dryRun),
+        state: "running",
+        hosts: input.hosts.map(({ host }) => ({
+            host,
+            state: "queued",
+            sessions: 0,
+            phase: "preparing",
+            processed: 0,
+            total: 0,
+            profileProcessed: 0,
+            profileTotal: 0,
+        })),
+        sessions: 0,
+        processed: 0,
+        total: 0,
+    };
+    const children = input.hosts.map((child, index) => {
+        const options = {
+            ...input.options,
+            ...(child.modelChoice && child.modelChoice !== "external"
+                ? { model: child.modelChoice }
+                : {}),
+        };
+        const args = parseHistoryImportArgs(webImportTokens(options, child.host), {
+            host: child.host,
+            surface: "web",
+        });
+        if (args.errors.length)
+            throw hostError(child.host, new Error(args.errors.join("; ")));
+        return { input: child, args, row: job.hosts[index] };
+    });
+    publish(job, controller);
+    const cleanup = async () => {
+        for (const child of children) {
+            // A failed resolver may already have opened a snapshot for its signed token.
+            let identity = child.selection?.identity;
+            if (!identity) {
+                try {
+                    identity = readImportSourceToken(child.input.source, child.input.host);
+                }
+                catch {
+                    continue;
+                }
+            }
+            await (deps.discardSelection ?? discardSelection)(identity).catch(() => { });
+        }
+    };
+    try {
+        const readiness = await deps.readiness();
+        for (const child of children) {
+            if (controller.signal.aborted)
+                break;
+            const host = child.input.host;
+            try {
+                const readerBlocked = importBlockedReason(readiness, { host, needsModel: false });
+                if (readerBlocked)
+                    throw new Error(readerBlocked);
+                child.row.blocker =
+                    importBlockedReason(readiness, {
+                        host,
+                        needsModel: true,
+                        modelChoice: child.input.modelChoice,
+                    }) ?? undefined;
+                if (!job.dryRun && child.row.blocker)
+                    throw new Error(child.row.blocker);
+                child.selection = await deps.resolveSelection(child.input.source, child.input.selection, matchOptions(child, directory, controller.signal));
+                child.row.sessions = child.selection.keys.length;
+                if (child.selection.unresolvedCount !== undefined)
+                    child.row.summary = {
+                        ...emptySummary(job.dryRun),
+                        unresolved: child.selection.unresolvedCount,
+                    };
+                child.args.source = child.selection.identity.realPath;
+                if (!child.row.sessions)
+                    child.row.state = "no-work";
+            }
+            catch (error) {
+                if (controller.signal.aborted)
+                    break;
+                const refused = hostError(host, error);
+                if (!job.dryRun)
+                    throw refused;
+                child.row.state = "failed";
+                child.row.error = refused.message;
+            }
+        }
+        updateTotals(job);
+        if (controller.signal.aborted) {
+            for (const child of job.hosts)
+                if (child.state === "queued")
+                    child.state = "cancelled";
+            await cleanup();
+            job.state = "cancelled";
+            return structuredClone(job);
+        }
+        const waiting = job.dryRun && !input.options.skipProfile
+            ? await (deps.waitingPrompts ?? readWaitingProfilePrompts)()
+            : new Set();
+        void (async () => {
+            let outcome = "failed";
+            try {
+                outcome = await executeGroup(children, job, controller.signal, directory, deps, waiting);
+            }
+            catch {
+                outcome = controller.signal.aborted ? "cancelled" : "failed";
+                job.error = "Import preparation failed. Refresh the preview and retry.";
+            }
+            finally {
+                delete job.activeHost;
+                if (!job.dryRun || outcome !== "done")
+                    await cleanup();
+                job.state = controller.signal.aborted ? "cancelled" : outcome;
+            }
+        })();
+        return structuredClone(job);
+    }
+    catch (error) {
+        await cleanup();
+        if (controller.signal.aborted) {
+            for (const child of job.hosts)
+                if (child.state === "queued")
+                    child.state = "cancelled";
+            job.state = "cancelled";
+            return structuredClone(job);
+        }
+        throw error;
+    }
+}
+async function executeGroup(children, job, signal, directory, deps, waiting) {
+    const history = new Set();
+    if (job.dryRun && !children[0].args.skipProfile)
+        job.profileEstimate = profileEstimate(history, waiting, children[0].args.profileBatch ?? 50);
+    let failed = job.hosts.some((row) => row.state === "failed");
+    for (const child of children) {
+        if (child.row.state !== "queued")
+            continue;
+        if (signal.aborted || (failed && !job.dryRun)) {
+            child.row.state = signal.aborted ? "cancelled" : "not-run";
+            continue;
+        }
+        const host = child.input.host;
+        job.activeHost = host;
+        child.row.state = "running";
+        try {
+            // Reuse the signed source and original selection. Never refresh a queued child.
+            const current = await deps.resolveSelection(child.input.source, child.input.selection, matchOptions(child, directory, signal));
+            const pinned = child.selection;
+            if (importSourceKey(current.identity) !== importSourceKey(pinned.identity) ||
+                current.cutoff !== pinned.cutoff ||
+                JSON.stringify([...current.keys].sort()) !== JSON.stringify([...pinned.keys].sort()))
+                throw new Error("The session list is out of date. Refresh the list and try again.");
+            if (signal.aborted) {
+                child.row.state = "cancelled";
+                continue;
+            }
+            // Reader/model readiness may change while earlier children finish their profile work.
+            const blocked = importBlockedReason(await deps.readiness(), {
+                host,
+                needsModel: !job.dryRun,
+                modelChoice: child.input.modelChoice,
+            });
+            if (blocked)
+                throw new Error(blocked);
+            const models = job.dryRun
+                ? {}
+                : await (deps.prepareModels ?? prepareModels)(child.input.modelChoice, directory, signal, host);
+            if (signal.aborted) {
+                child.row.state = "cancelled";
+                continue;
+            }
+            const eligible = new Set();
+            child.row.phase = child.args.skipMemories ? "profile" : "memory";
+            const report = await deps.runner(host, child.args, {
+                cwd: directory,
+                models,
+                signal,
+                track: { surface: "web" },
+                selection: {
+                    keys: pinned.keys,
+                    cutoff: pinned.cutoff,
+                    ...(pinned.identity.kind === "opencode-db"
+                        ? { snapshotKey: importSourceKey(pinned.identity) }
+                        : {}),
+                },
+                onProgress: (processed, total) => {
+                    child.row.processed = processed;
+                    child.row.total = total;
+                    updateTotals(job);
+                },
+                onProfileProgress: (done, total) => {
+                    child.row.phase = "profile";
+                    child.row.profileProcessed = done;
+                    child.row.profileTotal = total;
+                },
+                ...(job.dryRun && !child.args.skipProfile
+                    ? {
+                        onProfilePrompt: (identity) => {
+                            eligible.add(identity);
+                            history.add(identity);
+                        },
+                    }
+                    : {}),
+            });
+            const summary = summarizeHistoryImportReport(report);
+            child.row.summary = {
+                ...summary,
+                // Source resolution sees missing folders omitted from the pinned runner keys.
+                unresolved: pinned.unresolvedCount ??
+                    (report.unresolvedProjects ?? []).reduce((n, project) => n + project.sessions, 0) +
+                        report.unresolvableSessions.length,
+                profile: summary.profile
+                    ? { ...summary.profile, promptsAlreadyHandled: report.profile.promptsAlreadyHandled }
+                    : undefined,
+            };
+            if (job.dryRun && !child.args.skipProfile) {
+                child.row.profileEstimate = profileEstimate(eligible, waiting, child.args.profileBatch ?? 50);
+                job.profileEstimate = profileEstimate(history, waiting, child.args.profileBatch ?? 50);
+            }
+            if (signal.aborted)
+                child.row.state = "cancelled";
+            else if (historyImportFailed(report)) {
+                child.row.state = "failed";
+                failed = true;
+                child.row.error = `${hostLabel(host)}: ${report.unitsFailed ? "Memory work failed" : "Profile learning failed"}. Refresh the preview and retry.`;
+                job.error = child.row.error;
+            }
+            else {
+                const pending = report.unitsImported +
+                    report.unitsWouldImport +
+                    (report.profile?.promptsRecorded ?? 0) +
+                    (report.profile?.promptsWouldRecord ?? 0) +
+                    (report.profile?.batchesBuilt ?? 0) +
+                    (child.row.profileEstimate?.totalPrompts ?? 0);
+                child.row.state = pending ? "done" : "no-work";
+            }
+            updateTotals(job);
+        }
+        catch (error) {
+            child.row.state = signal.aborted ? "cancelled" : "failed";
+            // Model exceptions can contain raw replies. Return a reason code, never their text.
+            const reason = error instanceof ImportAlreadyRunningError || child.row.phase === "preparing"
+                ? safeHealthError(error, [CONFIG.memoryApiKey, CONFIG.embeddingApiKey])
+                : `${profileFailureCode(error)}. Refresh the preview and retry.`;
+            child.row.error = `${hostLabel(host)}: ${reason}`;
+            job.error = child.row.error;
+            failed = true;
+        }
+    }
+    return signal.aborted ? "cancelled" : failed ? "failed" : "done";
+}

package/dist/importer/web-import-jobs.d.ts

@@ -2,6 +2,7 @@ import { type ImportHost, type WebImportOptions } from "./import-args.js";
 import { type ImportReadiness } from "./import-readiness.js";
 import { resolveImportSelection, type ImportSelection } from "./import-sessions.js";
 import { runHistoryImport, summarizeHistoryImportReport } from "./run-import.js";
+import { type GroupImportJob, type GroupImportDeps } from "./web-import-group.js";
 type JobState = "running" | "cancelling" | "cancelled" | "done" | "failed";
 type Job = {
     id: string;
@@ -16,7 +17,7 @@ type Job = {
     summary?: ReturnType<typeof summarizeHistoryImportReport>;
     error?: string;
 };
-type Request = {
+export type WebImportRequest = {
     host: ImportHost;
     source: unknown;
     selection: ImportSelection;
@@ -24,13 +25,17 @@ type Request = {
     modelChoice?: string;
 };
 type Runner = typeof runHistoryImport;
+export type GroupImportRequest = {
+    hosts: Omit<WebImportRequest, "options">[];
+    options: WebImportOptions;
+};
 /** A refused job; `status` is the HTTP status the page receives. */
 export declare class ImportJobError extends Error {
     readonly status: number;
     constructor(message: string, status?: number);
 }
-export declare function validateWebImportRequest(value: unknown): Request;
-export interface ImportJobDeps {
+export declare function validateWebImportRequest(value: unknown): WebImportRequest | GroupImportRequest;
+export interface ImportJobDeps extends GroupImportDeps {
     runner?: Runner;
     resolveSelection?: typeof resolveImportSelection;
     readiness?: () => Promise<ImportReadiness>;
@@ -38,16 +43,17 @@ export interface ImportJobDeps {
 /** One in-memory slot per web server, shared by previews and imports. */
 export declare class SettingsImportJobs {
     private job?;
+    private readonly deps;
     private controller?;
     private starting;
     private readonly runner;
     private readonly resolveSelection;
     private readonly readiness;
     constructor(deps?: ImportJobDeps);
-    current(): Job | null;
+    current(): Job | GroupImportJob | null;
     private busy;
-    start(request: unknown, directory: string): Promise<Job>;
+    start(request: unknown, directory: string): Promise<Job | GroupImportJob>;
     private begin;
-    cancel(): Job;
+    cancel(): Job | GroupImportJob;
 }
 export {};

package/dist/importer/web-import-jobs.js

@@ -7,6 +7,7 @@ import { resolveImportSelection, validateImportSelection, } from "./import-sessi
 import { importSourceKey } from "./import-sources.js";
 import { formatHistoryImportReport, runHistoryImport, summarizeHistoryImportReport, } from "./run-import.js";
 import { safeHealthError } from "./settings-health.js";
+import { startImportGroup } from "./web-import-group.js";
 /** A refused job; `status` is the HTTP status the page receives. */
 export class ImportJobError extends Error {
     status;
@@ -33,6 +34,28 @@ export function validateWebImportRequest(value) {
     if (!value || typeof value !== "object" || Array.isArray(value))
         throw new Error("Invalid import request");
     const input = value;
+    if ("hosts" in input) {
+        if (Object.keys(input).some((key) => key !== "hosts" && key !== "options"))
+            throw new Error("Grouped imports accept hosts and shared options only");
+        if (!Array.isArray(input.hosts) || input.hosts.length === 0 || input.hosts.length > 3)
+            throw new Error("Choose one or more distinct hosts");
+        const hosts = input.hosts.map((child) => {
+            if (!child ||
+                typeof child !== "object" ||
+                Array.isArray(child) ||
+                Object.keys(child).some((key) => !["host", "source", "selection", "modelChoice"].includes(key)))
+                throw new Error("Invalid host import options; use shared options");
+            return validateWebImportRequest({ ...child, options: input.options });
+        });
+        if (new Set(hosts.map((child) => child.host)).size !== hosts.length)
+            throw new Error("Choose distinct hosts");
+        const order = ["pi", "opencode", "claude-code"];
+        hosts.sort((a, b) => order.indexOf(a.host) - order.indexOf(b.host));
+        return {
+            hosts: hosts.map(({ options: _options, ...child }) => child),
+            options: hosts[0].options,
+        };
+    }
     if (input.host !== "pi" && input.host !== "opencode" && input.host !== "claude-code") {
         throw new Error("Choose Pi, OpenCode, or Claude Code");
     }
@@ -69,6 +92,8 @@ export function validateWebImportRequest(value) {
             throw new Error("The project must be an absolute path");
         }
     }
+    if (options.skipMemories === true && options.skipProfile === true)
+        throw new Error("Choose Project memories or User profile");
     if (input.modelChoice !== undefined &&
         (typeof input.modelChoice !== "string" ||
             (input.modelChoice !== "external" && !/^[^/]+\/.+$/.test(input.modelChoice))))
@@ -90,18 +115,20 @@ export function validateWebImportRequest(value) {
 /** One in-memory slot per web server, shared by previews and imports. */
 export class SettingsImportJobs {
     job;
+    deps;
     controller;
     starting = false;
     runner;
     resolveSelection;
     readiness;
     constructor(deps = {}) {
+        this.deps = deps;
         this.runner = deps.runner ?? runHistoryImport;
         this.resolveSelection = deps.resolveSelection ?? resolveImportSelection;
         this.readiness = deps.readiness ?? (() => importReadiness());
     }
     current() {
-        return this.job ? { ...this.job } : null;
+        return this.job ? structuredClone(this.job) : null;
     }
     busy() {
         return this.starting || this.job?.state === "running" || this.job?.state === "cancelling";
@@ -125,6 +152,25 @@ export class SettingsImportJobs {
         catch (error) {
             throw new ImportJobError(error.message);
         }
+        if ("hosts" in input) {
+            const previous = this.job;
+            try {
+                return await startImportGroup(input, directory, {
+                    ...this.deps,
+                    runner: this.runner,
+                    resolveSelection: this.resolveSelection,
+                    readiness: this.readiness,
+                }, (job, controller) => {
+                    this.job = job;
+                    this.controller = controller;
+                });
+            }
+            catch (error) {
+                this.job = previous;
+                this.controller = undefined;
+                throw error;
+            }
+        }
         const options = { ...input.options };
         if (input.modelChoice && input.modelChoice !== "external")
             options.model = input.modelChoice;
@@ -229,6 +275,6 @@ export class SettingsImportJobs {
             throw new Error("No running import");
         this.controller.abort();
         this.job.state = "cancelling";
-        return { ...this.job };
+        return structuredClone(this.job);
     }
 }

package/dist/importer/web-import-profile-estimate.d.ts

@@ -0,0 +1,14 @@
+export interface ProfileEstimate {
+    /** Eligible history prompts absent from the shared waiting queue. */
+    historyPrompts: number;
+    waitingPrompts: number;
+    totalPrompts: number;
+    /** Analysis calls only; matching, deduplication and retries can add calls. */
+    analysisCalls: number;
+}
+/** Use the prompt store's existing session/message identity without changing ledger keys. */
+export declare function profilePromptIdentity(sessionId: string, messageId: string): string;
+/** Count the union of history and waiting prompts without counting overlapping identities twice. */
+export declare function profileEstimate(history: Set<string>, waiting: Set<string>, batchSize: number): ProfileEstimate;
+/** Read waiting identities only. Never initialise the prompt store or mark trivial prompts learned. */
+export declare function readWaitingProfilePrompts(): Promise<Set<string>>;

package/dist/importer/web-import-profile-estimate.js

@@ -0,0 +1,37 @@
+import { existsSync } from "node:fs";
+import { join } from "node:path";
+import { CONFIG } from "../config.js";
+import { isTrivialPrompt } from "../core/trivial-prompt.js";
+import { stripPrivateContent } from "../services/privacy.js";
+/** Use the prompt store's existing session/message identity without changing ledger keys. */
+export function profilePromptIdentity(sessionId, messageId) {
+    return JSON.stringify([sessionId, messageId]);
+}
+/** Count the union of history and waiting prompts without counting overlapping identities twice. */
+export function profileEstimate(history, waiting, batchSize) {
+    const historyPrompts = [...history].filter((key) => !waiting.has(key)).length;
+    const totalPrompts = historyPrompts + waiting.size;
+    return {
+        historyPrompts,
+        waitingPrompts: waiting.size,
+        totalPrompts,
+        analysisCalls: Math.ceil(totalPrompts / batchSize),
+    };
+}
+/** Read waiting identities only. Never initialise the prompt store or mark trivial prompts learned. */
+export async function readWaitingProfilePrompts() {
+    const path = join(CONFIG.storagePath, "user-prompts.db");
+    if (!existsSync(path))
+        return new Set();
+    const { tursoConnectionManager } = await import("../services/turso/connection-manager.js");
+    const db = await tursoConnectionManager.getConnection(path);
+    if (!(await db.get("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'user_prompts'")))
+        return new Set();
+    const rows = await db.all("SELECT session_id, message_id, content FROM user_prompts WHERE user_learning_captured = 0");
+    return new Set(rows
+        .filter((row) => {
+        const text = stripPrivateContent(String(row.content)).trim();
+        return text.length > 0 && !isTrivialPrompt(text);
+    })
+        .map((row) => profilePromptIdentity(String(row.session_id), String(row.message_id))));
+}

package/dist/services/web-server.js

@@ -823,7 +823,7 @@ export class WebServer {
                     instance: PROCESS_INSTANCE,
                 });
             }
-            if (path === "/" || path === "/index.html" || path === "/settings") {
+            if (path === "/" || path === "/index.html" || path === "/settings" || path === "/memory") {
                 return this.serveStaticFile("index.html", "text/html");
             }
             if (path === "/favicon.ico") {