om-memory-system 3.2.0-next.9 → 3.3.0-next.18

package/dist/importer/importer.js

@@ -1,12 +1,14 @@
 import { existsSync, realpathSync } from "node:fs";
 import { captureConversation } from "../core/capture.js";
 import { memoryClient } from "../services/client.js";
+import { findLiveCapturedEntryIds } from "../services/live-captured-entries.js";
 import { extractScopeFromContainerTag } from "../services/memory-scope.js";
 import { getTags } from "../services/tags.js";
 import { tursoConnectionManager } from "../services/turso/connection-manager.js";
 import { tursoShardManager } from "../services/turso/shard-manager.js";
 import { extractPiConversationWindows, } from "../adapters/pi/conversation.js";
 import { discoverPiSessions } from "./discovery.js";
+import { resolveImportProject } from "./import-project.js";
 import { PiImportLedger, importLedgerDbPath } from "./ledger.js";
 export function buildImportKey(sessionId, window, host = "pi") {
     const terminalEntryId = window.sourceEntryIds?.[window.sourceEntryIds.length - 1];
@@ -46,13 +48,6 @@ export function projectFilterTag(directory) {
     }
     return getTags(real).project.tag;
 }
-function resolveSessionDirectory(cwd, pathMaps) {
-    if (!cwd)
-        return null;
-    const mapped = pathMaps.find((map) => map.from === cwd)?.to;
-    const directory = mapped ?? cwd;
-    return existsSync(directory) ? directory : null;
-}
 function preview(text) {
     const trimmed = text.trim().replace(/\s+/g, " ");
     return trimmed.length > 60 ? `${trimmed.slice(0, 60)}...` : trimmed;
@@ -66,6 +61,38 @@ function windowWithinDateRange(window, filters) {
         return false;
     return true;
 }
+/**
+ * Keep the windows a run would import: inside the date range and not newer
+ * than the web listing cutoff. Held-back windows are only counted; they never
+ * reach the ledger, so a later run imports them under the same keys.
+ */
+export function selectImportWindows(windows, filters) {
+    const kept = [];
+    let heldBack = 0;
+    let untimed = 0;
+    for (const window of windows) {
+        if (!windowWithinDateRange(window, filters))
+            continue;
+        if (filters.cutoff !== undefined &&
+            window.userTimestamp !== undefined &&
+            window.userTimestamp > filters.cutoff) {
+            heldBack++;
+            continue;
+        }
+        if (window.userTimestamp === undefined)
+            untimed++;
+        kept.push(window);
+    }
+    return { kept, heldBack, untimed };
+}
+/** Units of one session that `importHistorySource` would consider, for a counting pass. */
+export function countImportableUnits(session, host, filters) {
+    return session.units.filter((window) => windowWithinDateRange(window, filters) &&
+        buildImportKey(session.sessionId, window, host) !== null).length;
+}
+function isLazySource(source) {
+    return typeof source.open === "function";
+}
 async function lookupExistingMemory(projectHash, key, options) {
     if (!options.storageAccessible)
         return null;
@@ -91,21 +118,34 @@ export async function importHistorySource(source, host, deps, filters, report) {
             throw new Error(`Memory system is not ready for import: ${embeddingInitError}. Start the extension and retry.`);
         }
     }
-    const candidates = [];
-    for await (const session of source) {
+    const sessionCandidates = (session) => {
         const { hash } = extractScopeFromContainerTag(getTags(session.directory).project.tag);
-        for (const window of session.units) {
-            if (!windowWithinDateRange(window, filters))
-                continue;
-            if (!buildImportKey(session.sessionId, window, host))
-                continue;
-            candidates.push({ ...session, hash, window });
-        }
+        return session.units
+            .filter((window) => windowWithinDateRange(window, filters) &&
+            buildImportKey(session.sessionId, window, host) !== null)
+            .map((window) => ({ ...session, hash, window }));
+    };
+    // A lazy source is read session by session; an eager one is buffered as before.
+    let candidates;
+    if (isLazySource(source)) {
+        report.unitsTotal = source.total;
+        candidates = (async function* () {
+            for await (const session of source.open())
+                yield* sessionCandidates(session);
+        })();
+    }
+    else {
+        const all = [];
+        for await (const session of source)
+            all.push(...sessionCandidates(session));
+        report.unitsTotal = all.length;
+        candidates = all;
     }
-    report.unitsTotal = candidates.length;
     const storageAccessible = ledgerFileExists;
     let processed = 0;
-    for (const candidate of candidates) {
+    for await (const candidate of candidates) {
+        if (deps.signal?.aborted)
+            break;
         const { sessionId, directory, sourceFile, hash, window } = candidate;
         const key = buildImportKey(sessionId, window, host);
         processed++;
@@ -132,6 +172,19 @@ export async function importHistorySource(source, host, deps, filters, report) {
             report.units.push(unit);
             continue;
         }
+        const liveEntryIds = await findLiveCapturedEntryIds(hash, host, sessionId);
+        if (liveEntryIds.has(window.userEntryId)) {
+            if (!dryRun) {
+                await ledger.begin({ key, sessionId, sourceFile, projectHash: hash });
+                await ledger.skip(key, "live-captured");
+            }
+            unit.status = "skipped";
+            unit.reason = "live-captured";
+            report.unitsSkipped++;
+            report.skipReasons["live-captured"] = (report.skipReasons["live-captured"] ?? 0) + 1;
+            report.units.push(unit);
+            continue;
+        }
         // Crash reconciliation: the memory landed but the ledger never committed.
         const storedMemoryId = await lookupExistingMemory(hash, key, { storageAccessible });
         if (storedMemoryId) {
@@ -223,10 +276,19 @@ export async function importPiHistory(deps, filters) {
         skipReasons: {},
         projects: [],
         units: [],
+        unitsHeldBack: 0,
+        unitsUntimed: 0,
     };
+    const selected = filters.selectionKeys ? new Set(filters.selectionKeys) : null;
     const projectAggregates = new Map();
     const candidates = [];
     for (const discovered of discovery.sessions) {
+        if (deps.signal?.aborted)
+            break;
+        if (selected && !selected.has(discovered.key)) {
+            report.sessionsFilteredOut++;
+            continue;
+        }
         if (filters.session) {
             const wanted = filters.session;
             if (discovered.sessionId !== wanted && discovered.file !== wanted) {
@@ -245,7 +307,16 @@ export async function importPiHistory(deps, filters) {
             });
             continue;
         }
-        const directory = resolveSessionDirectory(session.cwd, pathMaps);
+        // The header ID names the session in the page's list; a file that loads as
+        // another session would be imported under keys the user never saw.
+        if (discovered.sessionId && discovered.sessionId !== session.sessionId) {
+            report.loadErrors.push({
+                file: discovered.file,
+                error: "Loaded session ID does not match the file header",
+            });
+            continue;
+        }
+        const directory = resolveImportProject(session.cwd, pathMaps).directory;
         if (!directory) {
             report.unresolvableSessions.push({ file: session.sourceFile, cwd: session.cwd });
             continue;
@@ -269,10 +340,10 @@ export async function importPiHistory(deps, filters) {
             projectAggregates.set(tags.project.tag, aggregate);
         }
         aggregate.sessions.add(session.sessionId);
-        const windows = extractPiConversationWindows(session.branch);
-        for (const window of windows) {
-            if (!windowWithinDateRange(window, filters))
-                continue;
+        const windows = selectImportWindows(extractPiConversationWindows(session.branch), filters);
+        report.unitsHeldBack += windows.heldBack;
+        report.unitsUntimed += windows.untimed;
+        for (const window of windows.kept) {
             const key = buildImportKey(session.sessionId, window);
             if (!key)
                 continue;
@@ -311,10 +382,11 @@ export async function importPiHistory(deps, filters) {
         sessions: aggregate.sessions.size,
         units: aggregate.units,
     }));
-    if (deps.profile) {
+    if (deps.profile && !deps.signal?.aborted) {
         const { importProfileFromHistory } = await import("./profile-import.js");
         report.profile = await importProfileFromHistory(sourceSessions, {
             host: "pi",
+            signal: deps.signal,
             dryRun,
             model: deps.profile.model,
             batchSize: deps.profile.batchSize,

package/dist/importer/manual-import-guard.d.ts

@@ -0,0 +1,4 @@
+import type { BackfillHost } from "./backfill-model.js";
+/** Keep an automatic run from starting during a manual session import. */
+export declare function beginManualImport(host: BackfillHost): () => void;
+export declare function isManualImportRunning(host: BackfillHost): boolean;

package/dist/importer/manual-import-guard.js

@@ -0,0 +1,9 @@
+const running = new Set();
+/** Keep an automatic run from starting during a manual session import. */
+export function beginManualImport(host) {
+    running.add(host);
+    return () => running.delete(host);
+}
+export function isManualImportRunning(host) {
+    return running.has(host);
+}

package/dist/importer/opencode-import.d.ts

@@ -1,14 +1,22 @@
 import type { CaptureSummaryProvider } from "../core/host.js";
 import type { ModelPort } from "../core/profile-analysis.js";
 import { type ImportPathMap, type ImportReport } from "./importer.js";
-import { type UnresolvedProject } from "./opencode-project.js";
+import type { UnresolvedProject } from "./opencode-project.js";
+import { DEFAULT_OPENCODE_DB, type OpencodeOpenOptions } from "./opencode-reader.js";
 import { type ProfileImportReport } from "./profile-import.js";
+export { DEFAULT_OPENCODE_DB };
 export interface OpencodeImportOptions {
     dbPath?: string;
     dryRun?: boolean;
     since?: number;
     until?: number;
     session?: string;
+    /** Web selection by session ID; never combined with `session` or `maxSessions`. */
+    selectionKeys?: string[];
+    /** Web selection: user turns after this epoch ms are held back for a later run. */
+    cutoff?: number;
+    /** Web jobs share the listing's snapshot of a live database. */
+    snapshot?: OpencodeOpenOptions["shared"];
     project?: string;
     maxSessions?: number;
     pathMaps?: ImportPathMap[];
@@ -18,6 +26,8 @@ export interface OpencodeImportOptions {
     profileBatch?: number;
     provider?: CaptureSummaryProvider;
     profileModel?: ModelPort;
+    onProgress?: (processed: number, total: number, promptPreview: string) => void;
+    signal?: AbortSignal;
 }
 export interface OpencodeImportReport extends ImportReport {
     childSessionsFolded: number;

package/dist/importer/opencode-import.js

@@ -1,11 +1,12 @@
-import { homedir } from "node:os";
-import { join } from "node:path";
 import { getTags } from "../services/tags.js";
 import { extractScopeFromContainerTag } from "../services/memory-scope.js";
-import { importHistorySource, projectFilterTag, } from "./importer.js";
-import { resolveOpencodeSessions } from "./opencode-project.js";
-import { readOpencodeHistory } from "./opencode-reader.js";
+import { countImportableUnits, importHistorySource, projectFilterTag, selectImportWindows, } from "./importer.js";
+import { resolveImportProject } from "./import-project.js";
+import { DEFAULT_OPENCODE_DB, openOpencodeHistory, } from "./opencode-reader.js";
 import { importProfileFromHistory } from "./profile-import.js";
+export { DEFAULT_OPENCODE_DB };
+/** Let the web server and live capture run between sessions of a long read. */
+const yieldToEventLoop = () => new Promise((resolve) => setImmediate(resolve));
 /** Import resolved OpenCode history through the shared memory and profile steps. */
 export async function importOpencodeHistory(options) {
     if (!options.dryRun && !options.skipMemories && !options.provider) {
@@ -14,74 +15,137 @@ export async function importOpencodeHistory(options) {
     if (!options.dryRun && !options.skipProfile && !options.profileModel) {
         throw new Error("OpenCode history import needs a profile model");
     }
-    const dbPath = options.dbPath ?? join(homedir(), ".local/share/opencode/opencode.db");
-    // Resolve the filter before opening the reader: the reader only closes once iterated.
+    const dbPath = options.dbPath ?? DEFAULT_OPENCODE_DB;
     const projectTag = options.project ? projectFilterTag(options.project) : null;
-    const reader = readOpencodeHistory(dbPath, options);
-    const unresolved = new Map();
-    const sessions = [];
-    const projects = new Map();
-    let filtered = 0;
-    for await (const session of resolveOpencodeSessions(reader.sessions, options.pathMaps ?? [], unresolved)) {
-        const info = getTags(session.directory).project;
-        if (projectTag && projectTag !== projectFilterTag(session.directory)) {
-            filtered++;
-            continue;
+    const history = await openOpencodeHistory(dbPath, {
+        ...(options.since !== undefined ? { since: options.since } : {}),
+        ...(options.until !== undefined ? { until: options.until } : {}),
+        ...(options.session ? { session: options.session } : {}),
+        ...(options.selectionKeys ? { sessionIds: options.selectionKeys } : {}),
+        ...(options.maxSessions ? { maxSessions: options.maxSessions } : {}),
+    }, {
+        ...(options.snapshot ? { shared: options.snapshot } : {}),
+        ...(options.signal ? { signal: options.signal } : {}),
+    });
+    try {
+        const unresolved = new Map();
+        const projects = new Map();
+        const selected = [];
+        let filtered = 0;
+        for (const session of history.sessions) {
+            const { directory } = resolveImportProject(session.recordedDirectory, options.pathMaps ?? [], session.projectWorktree);
+            if (!directory) {
+                const entry = unresolved.get(session.recordedDirectory) ?? {
+                    directory: session.recordedDirectory,
+                    sessions: 0,
+                    units: 0,
+                };
+                entry.sessions++;
+                entry.units += history.loadUnits(session.sessionId).length;
+                unresolved.set(entry.directory, entry);
+                continue;
+            }
+            if (projectTag && projectTag !== projectFilterTag(directory)) {
+                filtered++;
+                continue;
+            }
+            selected.push({ ...session, directory });
         }
-        const project = projects.get(info.tag) ?? {
-            tag: info.tag,
-            hash: extractScopeFromContainerTag(info.tag).hash,
-            directory: session.directory,
-            sessions: 0,
-            units: 0,
-            sessionIds: new Set(),
+        const build = (session) => {
+            const windows = selectImportWindows(history.loadUnits(session.sessionId), options);
+            const source = {
+                sessionId: session.sessionId,
+                directory: session.directory,
+                sourceFile: dbPath,
+                units: windows.kept,
+            };
+            return { source, windows };
         };
-        project.sessionIds.add(session.sessionId);
-        project.sessions = project.sessionIds.size;
-        project.units += session.units.length;
-        projects.set(info.tag, project);
-        sessions.push(session);
-    }
-    const report = {
-        dryRun: Boolean(options.dryRun),
-        root: dbPath,
-        sessionsDiscovered: reader.topLevelSessions,
-        sessionsLoaded: sessions.length,
-        sessionsFilteredOut: filtered,
-        sessionsUnrecognized: 0,
-        unresolvableSessions: [],
-        loadErrors: [],
-        unitsTotal: 0,
-        unitsImported: 0,
-        unitsWouldImport: 0,
-        unitsSkipped: 0,
-        unitsFailed: 0,
-        unitsAlreadyHandled: 0,
-        skipReasons: {},
-        units: [],
-        projects: [...projects.values()].map(({ sessionIds: _ids, ...project }) => project),
-        childSessionsFolded: reader.childSessions,
-        unresolvedProjects: [...unresolved.values()],
-    };
-    if (!options.skipMemories) {
-        await importHistorySource(sessions, "opencode", {
-            provider: options.provider ?? {
-                summarize: async () => {
-                    throw new Error("A dry-run cannot call the capture model");
+        // Counting pass: totals for progress and the report, one session in memory at a time.
+        let total = 0;
+        let kept = 0;
+        let heldBack = 0;
+        let untimed = 0;
+        for (const session of selected) {
+            if (options.signal?.aborted)
+                break;
+            const { source, windows } = build(session);
+            const info = getTags(session.directory).project;
+            const project = projects.get(info.tag) ?? {
+                tag: info.tag,
+                hash: extractScopeFromContainerTag(info.tag).hash,
+                directory: session.directory,
+                sessions: 0,
+                units: 0,
+                sessionIds: new Set(),
+            };
+            project.sessionIds.add(session.sessionId);
+            project.sessions = project.sessionIds.size;
+            project.units += windows.kept.length;
+            projects.set(info.tag, project);
+            total += countImportableUnits(source, "opencode", options);
+            kept += windows.kept.length;
+            heldBack += windows.heldBack;
+            untimed += windows.untimed;
+            await yieldToEventLoop();
+        }
+        const open = () => (async function* () {
+            for (const session of selected) {
+                if (options.signal?.aborted)
+                    return;
+                await yieldToEventLoop();
+                yield build(session).source;
+            }
+        })();
+        const report = {
+            dryRun: Boolean(options.dryRun),
+            root: dbPath,
+            sessionsDiscovered: history.topLevelSessions,
+            sessionsLoaded: selected.length,
+            sessionsFilteredOut: filtered,
+            sessionsUnrecognized: 0,
+            unresolvableSessions: [],
+            loadErrors: [],
+            unitsTotal: 0,
+            unitsImported: 0,
+            unitsWouldImport: 0,
+            unitsSkipped: 0,
+            unitsFailed: 0,
+            unitsAlreadyHandled: 0,
+            unitsHeldBack: heldBack,
+            unitsUntimed: untimed,
+            skipReasons: {},
+            units: [],
+            projects: [...projects.values()].map(({ sessionIds: _ids, ...project }) => project),
+            childSessionsFolded: history.childSessions,
+            unresolvedProjects: [...unresolved.values()],
+        };
+        if (!options.skipMemories) {
+            await importHistorySource({ total, open }, "opencode", {
+                provider: options.provider ?? {
+                    summarize: async () => {
+                        throw new Error("A dry-run cannot call the capture model");
+                    },
                 },
-            },
-        }, options, report);
-    }
-    else {
-        report.unitsTotal = sessions.reduce((total, session) => total + session.units.length, 0);
+                ...(options.signal ? { signal: options.signal } : {}),
+                ...(options.onProgress ? { onProgress: options.onProgress } : {}),
+            }, options, report);
+        }
+        else {
+            report.unitsTotal = kept;
+        }
+        if (!options.skipProfile && !options.signal?.aborted) {
+            report.profile = await importProfileFromHistory(open(), {
+                host: "opencode",
+                signal: options.signal,
+                dryRun: options.dryRun,
+                batchSize: options.profileBatch,
+                model: options.profileModel,
+            });
+        }
+        return report;
     }
-    if (!options.skipProfile) {
-        report.profile = await importProfileFromHistory(sessions, {
-            host: "opencode",
-            dryRun: options.dryRun,
-            batchSize: options.profileBatch,
-            model: options.profileModel,
-        });
+    finally {
+        await history.close();
     }
-    return report;
 }

package/dist/importer/opencode-project.d.ts

@@ -1,11 +1,8 @@
 import type { ImportPathMap } from "./importer.js";
-import type { OpencodeSourceSession } from "./opencode-reader.js";
-/** Prefer the recorded directory, then an explicit map, then the project root. */
+/** A directory map first, then the recorded directory, then the project root. */
 export declare function resolveOpencodeProject(recordedDirectory: string, projectWorktree: string | null, maps?: ImportPathMap[]): string | null;
 export interface UnresolvedProject {
     directory: string;
     sessions: number;
     units: number;
 }
-/** Resolve each session without buffering its conversation windows. */
-export declare function resolveOpencodeSessions(sessions: AsyncIterable<OpencodeSourceSession>, maps: ImportPathMap[], unresolved: Map<string, UnresolvedProject>): AsyncGenerator<OpencodeSourceSession>;

package/dist/importer/opencode-project.js

@@ -1,38 +1,5 @@
-import { statSync } from "node:fs";
-function isDirectory(path) {
-    if (!path)
-        return false;
-    try {
-        return statSync(path).isDirectory();
-    }
-    catch {
-        return false;
-    }
-}
-/** Prefer the recorded directory, then an explicit map, then the project root. */
+import { resolveImportProject } from "./import-project.js";
+/** A directory map first, then the recorded directory, then the project root. */
 export function resolveOpencodeProject(recordedDirectory, projectWorktree, maps = []) {
-    if (isDirectory(recordedDirectory))
-        return recordedDirectory;
-    const mapped = maps.find((item) => item.from === recordedDirectory)?.to;
-    if (isDirectory(mapped))
-        return mapped;
-    return projectWorktree !== "/" && isDirectory(projectWorktree) ? projectWorktree : null;
-}
-/** Resolve each session without buffering its conversation windows. */
-export async function* resolveOpencodeSessions(sessions, maps, unresolved) {
-    for await (const session of sessions) {
-        const directory = resolveOpencodeProject(session.recordedDirectory, session.projectWorktree, maps);
-        if (directory) {
-            yield { ...session, directory };
-            continue;
-        }
-        const entry = unresolved.get(session.recordedDirectory) ?? {
-            directory: session.recordedDirectory,
-            sessions: 0,
-            units: 0,
-        };
-        entry.sessions++;
-        entry.units += session.units.length;
-        unresolved.set(entry.directory, entry);
-    }
+    return resolveImportProject(recordedDirectory, maps, projectWorktree).directory;
 }

package/dist/importer/opencode-reader.d.ts

@@ -1,19 +1,39 @@
-import type { ImportSourceSession } from "./importer.js";
-export interface OpencodeSourceSession extends ImportSourceSession {
+import type { ImportWindow } from "./importer.js";
+export declare const DEFAULT_OPENCODE_DB: string;
+/** One top-level session's metadata; its turns are built on demand. */
+export interface OpencodeSessionMeta {
+    sessionId: string;
     recordedDirectory: string;
     projectWorktree: string | null;
     timeCreated: number;
 }
-export interface OpencodeReader {
+export interface OpencodeHistory {
     childSessions: number;
     topLevelSessions: number;
-    sessions: AsyncIterable<OpencodeSourceSession>;
+    /** Top-level sessions matching the filters, oldest first. */
+    sessions: OpencodeSessionMeta[];
+    /** Build one session's work units; call only before `close`. */
+    loadUnits: (sessionId: string) => ImportWindow[];
+    close: () => Promise<void>;
 }
 export interface OpencodeReaderFilters {
     session?: string;
+    /** Web selection; never combined with `session` or `maxSessions`. */
+    sessionIds?: string[];
     maxSessions?: number;
     since?: number;
     until?: number;
 }
+export interface OpencodeOpenOptions {
+    /**
+     * Share the snapshot with the web listing and jobs under this source key.
+     * `fresh` copies again, `reuse` fails when no copy exists, `any` does either.
+     */
+    shared?: {
+        key: string;
+        mode: "fresh" | "reuse" | "any";
+    };
+    signal?: AbortSignal;
+}
 /** Read V1 OpenCode history without changing its database or WAL sidecars. */
-export declare function readOpencodeHistory(dbPath: string, filters?: OpencodeReaderFilters): OpencodeReader;
+export declare function openOpencodeHistory(dbPath: string, filters?: OpencodeReaderFilters, options?: OpencodeOpenOptions): Promise<OpencodeHistory>;