om-memory-system 5.5.0 → 5.6.0-next.216

package/dist/services/spaces/transfer-service.js

@@ -0,0 +1,408 @@
+/**
+ * The transfer service (task 3.5): it wires the pure engine (`transfer.ts`) to
+ * two space stores, resolves the spaces through the registry, refuses the
+ * transfers the spec forbids, and logs one record per outcome with counts,
+ * space ids and error codes only. The caller passes every setting in place
+ * of CONFIG, like `registry.ts` expects.
+ */
+import { existsSync } from "node:fs";
+import { log } from "../logger.js";
+import { tursoVectorSearch } from "../turso/vector-search.js";
+import { parseExtractedVector } from "../turso/vector-utils.js";
+import { openSpaceStore } from "./space-store.js";
+import { findSpace, readRegistryFile, resolveSpaceFolder } from "./registry.js";
+import { applyTransfer, previewTransfer } from "./transfer.js";
+import { tagTextKey } from "./transfer-types.js";
+const ID_CHUNK = 500;
+/** The insert writes is_pinned 0, so a pinned row sets its flag with this. */
+const PINNED_SQL = `UPDATE memories SET is_pinned = 1 WHERE id = ?`;
+function refusal(code, error) {
+    return { ok: false, code, error };
+}
+function readAccessOf(store) {
+    return {
+        listShards: (scope) => store.mode === "write" ? store.shards.getAllShards(scope, "") : store.listShards(scope, ""),
+        open: (dbPath) => store.mode === "write" ? store.connections.getConnection(dbPath) : store.openShard(dbPath),
+    };
+}
+/** Registered shards whose file still exists; a vanished file holds no rows. */
+async function existingShards(access, scope) {
+    return (await access.listShards(scope)).filter((shard) => existsSync(shard.dbPath));
+}
+async function eachShard(access, use) {
+    for (const scope of ["user", "project"]) {
+        for (const shard of await existingShards(access, scope)) {
+            await use(await access.open(shard.dbPath), shard);
+        }
+    }
+}
+function optionalString(value) {
+    return value === null || value === undefined ? undefined : String(value);
+}
+function rowToTransferRow(raw, shard) {
+    return {
+        id: String(raw.id),
+        text: String(raw.content),
+        containerTag: String(raw.container_tag),
+        tags: optionalString(raw.tags),
+        type: optionalString(raw.type),
+        createdAt: Number(raw.created_at),
+        updatedAt: Number(raw.updated_at),
+        metadata: optionalString(raw.metadata),
+        isPinned: Number(raw.is_pinned) === 1,
+        projectName: optionalString(raw.project_name),
+        vector: parseExtractedVector(raw.vector_json) ?? undefined,
+        tagsVector: parseExtractedVector(raw.tags_vector_json) ?? undefined,
+        shardKey: String(shard.id),
+    };
+}
+async function readRowsAcrossShards(access, filter) {
+    const tagClause = filter.projectTag === undefined ? "" : " AND container_tag = ?";
+    const args = [filter.since, filter.until];
+    if (filter.projectTag !== undefined)
+        args.push(filter.projectTag);
+    const rows = [];
+    await eachShard(access, async (db, shard) => {
+        const raw = await db.all(`
+        SELECT id, content, container_tag, tags, type, created_at, updated_at, metadata,
+               project_name, is_pinned,
+               vector_extract(vector) AS vector_json,
+               CASE WHEN tags_vector IS NULL THEN NULL ELSE vector_extract(tags_vector) END
+                 AS tags_vector_json
+        FROM memories
+        WHERE created_at >= ? AND created_at <= ?${tagClause}
+      `, args);
+        for (const item of raw)
+            rows.push(rowToTransferRow(item, shard));
+    });
+    return rows;
+}
+/**
+ * The settings a space's shards record. All shards must agree, or the vectors
+ * are not provably reusable and this reads as undefined.
+ */
+async function readEmbeddingInfo(access) {
+    let agreed;
+    for (const scope of ["user", "project"]) {
+        for (const shard of await existingShards(access, scope)) {
+            const db = await access.open(shard.dbPath);
+            const meta = await db.all(`SELECT key, value FROM shard_metadata WHERE key IN ('embedding_model', 'embedding_dimensions')`);
+            const model = meta.find((row) => String(row.key) === "embedding_model")?.value;
+            const dimensions = meta.find((row) => String(row.key) === "embedding_dimensions")?.value;
+            if (typeof model !== "string" || dimensions === undefined)
+                return undefined;
+            const entry = { model, dimensions: Number(dimensions) };
+            if (agreed && (agreed.model !== entry.model || agreed.dimensions !== entry.dimensions)) {
+                return undefined;
+            }
+            agreed ??= entry;
+        }
+    }
+    return agreed;
+}
+async function readAllIds(access) {
+    const ids = new Set();
+    await eachShard(access, async (db) => {
+        for (const row of await db.all(`SELECT id FROM memories`))
+            ids.add(String(row.id));
+    });
+    return ids;
+}
+async function findIdsIn(access, ids) {
+    const found = new Set();
+    await eachShard(access, async (db) => {
+        for (const chunk of chunkIds(ids)) {
+            const sql = `SELECT id FROM memories WHERE id IN (${chunk.map(() => "?").join(", ")})`;
+            for (const row of await db.all(sql, chunk))
+                found.add(String(row.id));
+        }
+    });
+    return found;
+}
+async function readAllTagTexts(access) {
+    const keys = new Set();
+    await eachShard(access, async (db) => {
+        for (const row of await db.all(`SELECT container_tag, content FROM memories`)) {
+            keys.add(tagTextKey(String(row.container_tag), String(row.content)));
+        }
+    });
+    return keys;
+}
+function chunkIds(ids) {
+    return Array.from({ length: Math.ceil(ids.length / ID_CHUNK) }, (_, index) => ids.slice(index * ID_CHUNK, (index + 1) * ID_CHUNK));
+}
+/** The source side of the engine: a read store, plus a write store for a move. */
+class SpaceSourceReader {
+    access;
+    write;
+    constructor(access, write) {
+        this.access = access;
+        this.write = write;
+    }
+    readRows(filter) {
+        return readRowsAcrossShards(this.access, filter);
+    }
+    getEmbeddingInfo() {
+        return readEmbeddingInfo(this.access);
+    }
+    async getShardCounts(shardKeys) {
+        const write = this.requireWrite();
+        const counts = new Map();
+        for (const key of shardKeys) {
+            const shard = await write.shards.getShardById(Number(key));
+            counts.set(key, shard?.vectorCount ?? 0);
+        }
+        return counts;
+    }
+    async deleteFromShard(shardKey, ids) {
+        const write = this.requireWrite();
+        const shard = await write.shards.getShardById(Number(shardKey));
+        if (!shard)
+            throw new Error(`Source shard ${shardKey} is not registered.`);
+        await write.shards.withScopeWriteLock(shard.scope, shard.scopeHash, async () => {
+            const db = await write.connections.getConnection(shard.dbPath);
+            await db.transaction("write", async (tx) => {
+                for (const chunk of chunkIds(ids)) {
+                    await tx.execute({
+                        sql: `DELETE FROM memories WHERE id IN (${chunk.map(() => "?").join(", ")})`,
+                        args: chunk,
+                    });
+                }
+            });
+            // A recount, not a delta, keeps the count right if another writer raced.
+            await write.shards.setVectorCount(shard.id, await tursoVectorSearch.countAllVectors(db));
+        });
+    }
+    async setShardCount(shardKey, _count) {
+        const write = this.requireWrite();
+        const shard = await write.shards.getShardById(Number(shardKey));
+        if (!shard)
+            return;
+        const db = await write.connections.getConnection(shard.dbPath);
+        await write.shards.setVectorCount(shard.id, await tursoVectorSearch.countAllVectors(db));
+    }
+    requireWrite() {
+        if (!this.write)
+            throw new Error("Only a move reaches this step; a copy source is read-only.");
+        return this.write;
+    }
+}
+function toMemoryRecord(row) {
+    return {
+        id: row.id,
+        content: row.text,
+        vector: row.vector,
+        tagsVector: row.tagsVector,
+        containerTag: row.containerTag,
+        tags: row.tags,
+        type: row.type,
+        createdAt: row.createdAt,
+        updatedAt: row.updatedAt,
+        metadata: row.metadata,
+    };
+}
+class SpaceTargetWriter {
+    store;
+    embedding;
+    shardInfo = new Map();
+    constructor(store, embedding) {
+        this.store = store;
+        this.embedding = embedding;
+    }
+    loadExistingIds() {
+        return readAllIds(readAccessOf(this.store));
+    }
+    loadExistingTagTexts() {
+        return readAllTagTexts(readAccessOf(this.store));
+    }
+    getEmbeddingSettings() {
+        return Promise.resolve(this.embedding);
+    }
+    async getWriteShard(scope, scopeHash) {
+        const shard = await this.store.shards.getWriteShard(scope, scopeHash);
+        this.shardInfo.set(String(shard.id), shard);
+        return { key: String(shard.id), vectorCount: shard.vectorCount };
+    }
+    async insertBatch(shard, rows) {
+        const info = this.offeredShard(shard.key);
+        await this.store.shards.withScopeWriteLock(info.scope, info.scopeHash, async () => {
+            const db = await this.store.connections.getConnection(info.dbPath);
+            await db.transaction("write", async (tx) => {
+                for (const row of rows) {
+                    await tursoVectorSearch.insertVectorInTransaction(tx, toMemoryRecord(row));
+                    // The insert writes is_pinned 0, so a pinned row sets its flag after.
+                    if (row.isPinned)
+                        await tx.execute({ sql: PINNED_SQL, args: [row.id] });
+                }
+            });
+            // A recount, not a delta, keeps the count right if another writer raced.
+            await this.store.shards.setVectorCount(info.id, await tursoVectorSearch.countAllVectors(db));
+        });
+    }
+    async setShardCount(shardKey, _count) {
+        const info = this.offeredShard(shardKey);
+        const db = await this.store.connections.getConnection(info.dbPath);
+        await this.store.shards.setVectorCount(info.id, await tursoVectorSearch.countAllVectors(db));
+    }
+    findExistingIds(ids) {
+        return findIdsIn(readAccessOf(this.store), ids);
+    }
+    offeredShard(shardKey) {
+        const info = this.shardInfo.get(shardKey);
+        if (!info)
+            throw new Error(`Target shard ${shardKey} was never offered for writing.`);
+        return info;
+    }
+}
+/**
+ * The read-only target of a preview: reads from the read store, every write
+ * refused. Opening an empty target in write mode would create its metadata.db.
+ */
+class ReadOnlyTarget {
+    access;
+    constructor(access) {
+        this.access = access;
+    }
+    loadExistingIds() {
+        return readAllIds(this.access);
+    }
+    loadExistingTagTexts() {
+        return readAllTagTexts(this.access);
+    }
+    getEmbeddingSettings() {
+        return this.refuse();
+    }
+    getWriteShard() {
+        return this.refuse();
+    }
+    insertBatch() {
+        return this.refuse();
+    }
+    setShardCount() {
+        return this.refuse();
+    }
+    findExistingIds(ids) {
+        return findIdsIn(this.access, ids);
+    }
+    refuse = () => Promise.reject(new Error("A preview must not change the target."));
+}
+function defaultEmbed(text) {
+    return import("../embedding.js").then(({ embeddingService }) => embeddingService.embedWithTimeout(text, { task: "document" }));
+}
+/** Resolves both spaces and applies the refusals the spec demands, in order. */
+function resolveSpaces(request, context) {
+    const read = readRegistryFile(context.files.registryPath);
+    if (read.problem)
+        return refusal("registry-problem", read.problem);
+    const source = findSpace(read.registry, request.from);
+    const target = findSpace(read.registry, request.to);
+    if (!source)
+        return refusal("unknown-space", `No space named "${request.from}".`);
+    if (!target)
+        return refusal("unknown-space", `No space named "${request.to}".`);
+    if (source.id === target.id) {
+        return refusal("same-space", `The source and the target must differ; both are "${source.name}".`);
+    }
+    if (source.archived) {
+        return refusal("archived-space", `The source space "${source.name}" is archived.`);
+    }
+    if (target.archived) {
+        return refusal("archived-space", `The target space "${target.name}" is archived.`);
+    }
+    return {
+        ok: true,
+        sourceId: source.id,
+        targetId: target.id,
+        sourceFolder: resolveSpaceFolder(source.id, context.paths),
+        targetFolder: resolveSpaceFolder(target.id, context.paths),
+    };
+}
+function errorName(error) {
+    return error instanceof Error ? error.name : "unknown error";
+}
+/** Previews a transfer without changing either space: read stores, no locks. */
+export async function previewSpaceTransfer(request, context) {
+    const resolved = resolveSpaces(request, context);
+    if (!resolved.ok) {
+        logRefusal(request, resolved.code);
+        return resolved;
+    }
+    const ids = { from: resolved.sourceId, to: resolved.targetId };
+    const source = openSpaceStore(resolved.sourceFolder, "read");
+    const target = openSpaceStore(resolved.targetFolder, "read");
+    try {
+        const preview = await previewTransfer({
+            source: new SpaceSourceReader(readAccessOf(source)),
+            target: new ReadOnlyTarget(readAccessOf(target)),
+            filter: request.filter,
+        });
+        log("space transfer preview", {
+            ...ids,
+            matched: preview.matched,
+            duplicates: preview.duplicates,
+            toCopy: preview.toCopy,
+        });
+        return { ok: true, preview };
+    }
+    catch (error) {
+        log("space transfer preview failed", { ...ids, error: errorName(error) }, "error");
+        throw error;
+    }
+    finally {
+        await source.close();
+        await target.close();
+    }
+}
+/**
+ * Runs a copy or a move. The target opens in write mode; the source opens in
+ * write mode only for a move. Engine failures land in `result.error`; the call
+ * throws only on an unexpected store failure, after logging its name.
+ */
+export async function applySpaceTransfer(request, context) {
+    const resolved = resolveSpaces(request, context);
+    if (!resolved.ok) {
+        logRefusal(request, resolved.code);
+        return resolved;
+    }
+    const ids = { from: resolved.sourceId, to: resolved.targetId };
+    const move = request.mode === "move";
+    const sourceStore = move
+        ? openSpaceStore(resolved.sourceFolder, "write")
+        : openSpaceStore(resolved.sourceFolder, "read");
+    const targetStore = openSpaceStore(resolved.targetFolder, "write");
+    try {
+        const write = move ? sourceStore : undefined;
+        const source = new SpaceSourceReader(readAccessOf(sourceStore), write);
+        const result = await applyTransfer({
+            source,
+            target: new SpaceTargetWriter(targetStore, context.embedding),
+            filter: request.filter,
+            mode: request.mode,
+            maxVectorsPerShard: context.maxVectorsPerShard,
+            embed: context.embed ?? defaultEmbed,
+            ...(request.previewMatched === undefined ? {} : { previewMatched: request.previewMatched }),
+        });
+        log("space transfer", {
+            ...ids,
+            mode: request.mode,
+            ok: result.ok,
+            copied: result.copied,
+            skipped: result.skipped,
+            removed: result.removed,
+            changedFromPreview: result.changedFromPreview,
+            ...(result.error ? { step: result.error.step } : {}),
+        });
+        return { ok: true, result };
+    }
+    catch (error) {
+        log("space transfer failed", { ...ids, mode: request.mode, error: errorName(error) }, "error");
+        throw error;
+    }
+    finally {
+        await sourceStore.close();
+        await targetStore.close();
+    }
+}
+function logRefusal(request, code) {
+    log("space transfer refused", { from: request.from, to: request.to, code }, "warn");
+}

package/dist/services/spaces/transfer-types.d.ts

@@ -0,0 +1,163 @@
+/**
+ * Types for the space transfer engine (`transfer.ts`). The engine is pure: it
+ * works only through the `SourceReader` and `TargetWriter` interfaces below,
+ * never imports CONFIG and never opens a file. The wiring layer implements both
+ * interfaces against the space stores.
+ */
+export type TransferScope = "user" | "project";
+export type TransferMode = "copy" | "move";
+/**
+ * The embedding settings of a space or shard. A missing model means legacy
+ * shards that record none, which the engine treats as not provably compatible.
+ */
+export interface EmbeddingInfo {
+    model?: string;
+    dimensions: number;
+}
+/** One source row, shaped after the `memories` table of a shard. */
+export interface TransferRow {
+    id: string;
+    /** Memory text (the `content` column). */
+    text: string;
+    /** Scope tag: `omms_project_<hash>` or `omms_user_<hash>`. */
+    containerTag: string;
+    /** Comma-separated tags, exactly as stored. */
+    tags?: string;
+    type?: string;
+    /** Creation time, epoch milliseconds. */
+    createdAt: number;
+    /** Last update time, epoch milliseconds. */
+    updatedAt: number;
+    /** Stored metadata as the raw JSON string, exactly as stored. */
+    metadata?: string;
+    isPinned: boolean;
+    /** The `project_name` column; drives the preview's project groups. */
+    projectName?: string;
+    /** The stored content vector, when the source shard can supply it. */
+    vector?: Float32Array;
+    /** The stored tags vector, when the row has one. */
+    tagsVector?: Float32Array;
+    /** The source shard the row was read from; drives a move's per-shard deletes. */
+    shardKey: string;
+}
+/** Selection filter. Both bounds on `created_at` are inclusive. */
+export interface TransferFilter {
+    since: number;
+    until: number;
+    /** Only rows with this container tag. Omit for every project plus Personal. */
+    projectTag?: string;
+}
+/** One project group of a preview. User-scope rows group under "Personal". */
+export interface TransferGroup {
+    name: string;
+    tag: string;
+    count: number;
+}
+/**
+ * A preview example: the id and the text cut to 160 characters. For the screen
+ * only; callers must never log it.
+ */
+export interface TransferExample {
+    id: string;
+    text: string;
+}
+export interface TransferPreview {
+    /** Source rows that match the filter. */
+    matched: number;
+    /** Rows a copy would write: matched minus exact duplicates. */
+    toCopy: number;
+    /** Exact duplicates the target already holds. */
+    duplicates: number;
+    groups: TransferGroup[];
+    examples: TransferExample[];
+}
+/** A row ready to write: every stored field plus a usable content vector. */
+export interface TransferWriteRow {
+    id: string;
+    text: string;
+    containerTag: string;
+    tags?: string;
+    type?: string;
+    createdAt: number;
+    updatedAt: number;
+    metadata?: string;
+    isPinned: boolean;
+    vector: Float32Array;
+    tagsVector?: Float32Array;
+}
+/** A shard the target offers for writes, with the rows it already holds. */
+export interface TargetShard {
+    key: string;
+    vectorCount: number;
+}
+/** The key of a container tag and text pair, for the duplicate check. */
+export declare function tagTextKey(tag: string, text: string): string;
+/**
+ * Reads one space as the transfer source. Read methods must not change the
+ * space; the delete and count methods run only in a move's remove step.
+ */
+export interface SourceReader {
+    /**
+     * Read the source rows for a filter. The reader may pre-filter in SQL, but
+     * the engine applies the exact inclusive bounds and the project filter
+     * itself, so the reader must return at least every matching row. Rows whose
+     * container tag OMMS cannot parse are left alone by the engine.
+     */
+    readRows(filter: TransferFilter): Promise<TransferRow[]>;
+    /**
+     * The model and dimensions recorded in the source shards. Undefined when the
+     * shards record no model; the engine then embeds again instead of reusing.
+     */
+    getEmbeddingInfo(): Promise<EmbeddingInfo | undefined>;
+    /** Row counts of the given shards, used to fix counts after a move's deletes. */
+    getShardCounts(shardKeys: string[]): Promise<Map<string, number>>;
+    /** Delete the ids from one shard in a single transaction. */
+    deleteFromShard(shardKey: string, ids: string[]): Promise<void>;
+    /** Store a shard's vector count after the engine changed the shard. */
+    setShardCount(shardKey: string, count: number): Promise<void>;
+}
+/** Writes one space as the transfer target. */
+export interface TargetWriter {
+    /** Every id the target holds, loaded once for the duplicate check. */
+    loadExistingIds(): Promise<Set<string>>;
+    /** Every container tag and text pair the target holds, as `tagTextKey` keys, loaded once. */
+    loadExistingTagTexts(): Promise<Set<string>>;
+    /** The target space's embedding settings. */
+    getEmbeddingSettings(): Promise<EmbeddingInfo>;
+    /**
+     * The shard that takes the next rows of one scope group, creating a shard
+     * when the scope has none. Must not return a shard with no room left under
+     * the capacity the engine was given.
+     */
+    getWriteShard(scope: TransferScope, scopeHash: string): Promise<TargetShard>;
+    /** Insert the rows into the shard in one transaction: all rows or none. */
+    insertBatch(shard: TargetShard, rows: TransferWriteRow[]): Promise<void>;
+    /** Store the shard's vector count after the engine wrote to it. */
+    setShardCount(shardKey: string, count: number): Promise<void>;
+    /** Read back which of the ids the target holds; called once per move. */
+    findExistingIds(ids: string[]): Promise<Set<string>>;
+}
+export type TransferFailureStep = "select" | "write" | "check" | "remove";
+/** A failed step. The message holds counts, shard keys and ids, never memory text. */
+export interface TransferFailure {
+    step: TransferFailureStep;
+    message: string;
+}
+/** The outcome of an apply. Counts say how far the transfer got. */
+export interface TransferApplyResult {
+    ok: boolean;
+    mode: TransferMode;
+    /** Rows written to the target before any failure. */
+    copied: number;
+    /** Exact duplicates skipped in the target's favour. */
+    skipped: number;
+    /** Rows deleted from the source (move only, always after a passed check). */
+    removed: number;
+    /** How many more (or fewer) rows matched at apply time than at preview time. */
+    changedFromPreview: number;
+    /** True when at least one row was embedded again instead of reusing its vector. */
+    embeddedAgain: boolean;
+    /** Copied ids the check could not find in the target (move only). */
+    missingIds: string[];
+    error?: TransferFailure;
+}

package/dist/services/spaces/transfer-types.js

@@ -0,0 +1,10 @@
+/**
+ * Types for the space transfer engine (`transfer.ts`). The engine is pure: it
+ * works only through the `SourceReader` and `TargetWriter` interfaces below,
+ * never imports CONFIG and never opens a file. The wiring layer implements both
+ * interfaces against the space stores.
+ */
+/** The key of a container tag and text pair, for the duplicate check. */
+export function tagTextKey(tag, text) {
+    return `${tag}\u0000${text}`;
+}

package/dist/services/spaces/transfer.d.ts

@@ -0,0 +1,31 @@
+import type { SourceReader, TargetWriter, TransferApplyResult, TransferFilter, TransferMode, TransferPreview } from "./transfer-types.js";
+export * from "./transfer-types.js";
+type EmbedFn = (text: string) => Promise<Float32Array>;
+/**
+ * Preview a transfer: count, project groups, at most five examples (text cut
+ * to 160 characters) and the duplicate count. Reads only; never writes.
+ */
+export declare function previewTransfer(args: {
+    source: SourceReader;
+    target: TargetWriter;
+    filter: TransferFilter;
+}): Promise<TransferPreview>;
+export interface TransferApplyArgs {
+    source: SourceReader;
+    target: TargetWriter;
+    filter: TransferFilter;
+    mode: TransferMode;
+    /** Shard capacity; writes are split so no shard receives more rows than it has room for. */
+    maxVectorsPerShard: number;
+    /** Embeds one text with the target's model. Needed only when the stored vectors cannot be reused. */
+    embed?: EmbedFn;
+    /** The preview's matched count, so the apply can report how the source changed. */
+    previewMatched?: number;
+}
+/**
+ * Run a transfer: select and skip duplicates at apply time, write to the
+ * target, and for a move check every copied id before removing it from the
+ * source. Failures are reported in the result, never thrown: the counts say
+ * how far the transfer got.
+ */
+export declare function applyTransfer(args: TransferApplyArgs): Promise<TransferApplyResult>;