om-memory-system 5.5.0 → 5.7.0-next.218

package/dist/services/spaces/export-space.js

@@ -0,0 +1,367 @@
+import { existsSync, mkdirSync, readFileSync, renameSync, unlinkSync, writeFileSync, } from "node:fs";
+import { dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { isFullyPrivate } from "../privacy.js";
+import { scrubIdentifiers } from "./identifier-scrub.js";
+import { openSpaceStore } from "./space-store.js";
+import { SPACE_EXPORT_FORMAT, SPACE_EXPORT_SCHEMA_VERSION, SpaceExportDocumentSchema, } from "../../shared/api/space-export-schemas.js";
+/**
+ * The profile database of a space folder. Kept as a literal because importing
+ * the profile manager for its constant would run the manager's constructor at
+ * import time; see `USER_PROFILES_DB_NAME` in user-profile-manager.ts.
+ */
+const PROFILE_DB_NAME = "user-profiles.db";
+/** Identity columns of a memory row: dropped whole, never exported. */
+const IDENTITY_COLUMNS = [
+    "display_name",
+    "user_name",
+    "user_email",
+    "project_path",
+    "project_name",
+    "git_repo_url",
+];
+/** Every column the export reads. The vector columns are absent by design. */
+const MEMORY_COLUMNS = "id, content, container_tag, tags, type, created_at, updated_at, metadata, " +
+    "display_name, user_name, user_email, project_path, project_name, git_repo_url, is_pinned";
+/** The version of this OMMS build, read from package.json; "unknown" when unreadable. */
+export function getOmmsVersion() {
+    try {
+        const pkgPath = join(dirname(fileURLToPath(import.meta.url)), "../../../package.json");
+        const pkg = JSON.parse(readFileSync(pkgPath, "utf-8"));
+        return pkg.version ?? "unknown";
+    }
+    catch {
+        return "unknown";
+    }
+}
+/** The free-text limit the spec requires every export result to state. */
+const FREE_TEXT_WARNING = "Free-text names in memory text are not detected. Check the file before you share it.";
+/** Scrub one string and add its replacement counts to the totals. */
+function scrub(text, counts) {
+    const result = scrubIdentifiers(text);
+    counts.pathsReplaced += result.counts.pathsReplaced;
+    counts.emailsReplaced += result.counts.emailsReplaced;
+    return result.text;
+}
+/** True when a whole metadata value is itself a folder or file path. */
+function isPathValue(value) {
+    return (value.startsWith("/") ||
+        value.startsWith("~/") ||
+        value === "~" ||
+        /^[a-zA-Z]:[\\/]/.test(value) ||
+        value.startsWith("\\\\"));
+}
+/** The identity values of one row: any metadata value equal to one is dropped. */
+function identityValues(row) {
+    const values = new Set();
+    for (const column of IDENTITY_COLUMNS) {
+        const value = row[column];
+        if (typeof value === "string" && value.trim())
+            values.add(value.trim());
+    }
+    return values;
+}
+/**
+ * Clean one metadata value. A string that is a path or equals a row identity
+ * value returns `undefined`, which drops its key; every other string is
+ * scrubbed. Objects and arrays are cleaned recursively.
+ */
+function cleanMetadataValue(value, identities, counts) {
+    if (typeof value === "string") {
+        const trimmed = value.trim();
+        if (identities.has(trimmed) || isPathValue(trimmed))
+            return undefined;
+        return scrub(value, counts);
+    }
+    if (Array.isArray(value)) {
+        if (value.length === 0)
+            return value;
+        const items = value
+            .map((item) => cleanMetadataValue(item, identities, counts))
+            .filter((item) => item !== undefined);
+        return items.length > 0 ? items : undefined;
+    }
+    if (value !== null && typeof value === "object") {
+        const cleaned = {};
+        for (const [key, child] of Object.entries(value)) {
+            const result = cleanMetadataValue(child, identities, counts);
+            if (result !== undefined)
+                cleaned[key] = result;
+        }
+        return Object.keys(cleaned).length > 0 ? cleaned : undefined;
+    }
+    return value;
+}
+/** Clean the metadata JSON of one row; `undefined` when nothing survives. */
+function cleanMetadata(raw, row, counts) {
+    if (!raw || typeof raw !== "string")
+        return undefined;
+    let parsed;
+    try {
+        parsed = JSON.parse(raw);
+    }
+    catch {
+        return undefined;
+    }
+    if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed))
+        return undefined;
+    const cleaned = cleanMetadataValue(parsed, identityValues(row), counts);
+    if (cleaned === undefined || typeof cleaned !== "object")
+        return undefined;
+    return cleaned;
+}
+/** Split and scrub the comma-joined tags column; `undefined` when nothing survives. */
+function cleanTags(raw, counts) {
+    if (!raw || typeof raw !== "string")
+        return undefined;
+    const tags = raw
+        .split(",")
+        .map((tag) => scrub(tag.trim(), counts))
+        .filter(Boolean);
+    return tags.length > 0 ? tags : undefined;
+}
+/** Read every memory of one scope that passes the date range. */
+async function readScopeMemories(store, scope, options, counts) {
+    const memories = [];
+    for (const shard of await store.listShards(scope, "")) {
+        if (!existsSync(shard.dbPath))
+            continue;
+        const db = await store.openShard(shard.dbPath);
+        const rows = await db.all(`SELECT ${MEMORY_COLUMNS} FROM memories`);
+        for (const row of rows) {
+            const content = String(row.content ?? "");
+            if (isFullyPrivate(content)) {
+                counts.privateSkipped++;
+                continue;
+            }
+            const createdAt = Number(row.created_at);
+            if (options.since !== undefined && createdAt < options.since)
+                continue;
+            if (options.until !== undefined && createdAt > options.until)
+                continue;
+            const scrubbedContent = scrub(content, counts);
+            if (!scrubbedContent.trim()) {
+                counts.privateSkipped++;
+                continue;
+            }
+            const metadata = cleanMetadata(row.metadata, row, counts);
+            const tags = cleanTags(row.tags, counts);
+            memories.push({
+                id: String(row.id),
+                content: scrubbedContent,
+                scope,
+                ...(scope === "project" ? { projectTag: String(row.container_tag) } : {}),
+                ...(row.type ? { type: String(row.type) } : {}),
+                ...(tags ? { tags } : {}),
+                createdAt,
+                updatedAt: Number(row.updated_at ?? createdAt),
+                isPinned: Number(row.is_pinned ?? 0) === 1,
+                ...(metadata ? { metadata } : {}),
+            });
+            if (scope === "project")
+                counts.projectMemories++;
+            else
+                counts.userMemories++;
+        }
+    }
+    return memories;
+}
+/** Map one learned item onto the export shape: no centroid, anchor or sampling fields. */
+function mapProfileItem(item, counts, withSteps) {
+    if (item === null || typeof item !== "object")
+        return undefined;
+    const source = item;
+    const description = scrub(String(source.description ?? ""), counts);
+    if (!description.trim())
+        return undefined;
+    return {
+        ...(source.category !== undefined ? { category: scrub(String(source.category), counts) } : {}),
+        description,
+        ...(withSteps && Array.isArray(source.steps)
+            ? { steps: source.steps.map((step) => scrub(String(step), counts)) }
+            : {}),
+        confidence: Number(source.confidence ?? 0),
+        frequency: Number(source.frequency ?? 0),
+        evidence: Array.isArray(source.evidence)
+            ? source.evidence.map((entry) => scrub(String(entry), counts))
+            : [],
+        lastSeen: Number(source.lastSeen ?? 0),
+        ...(source.firstSeen !== undefined ? { firstSeen: Number(source.firstSeen) } : {}),
+    };
+}
+/** Map every learned item of one kind, dropping anything without a description. */
+function mapProfileItems(items, counts, withSteps) {
+    if (!Array.isArray(items))
+        return [];
+    return items
+        .map((item) => mapProfileItem(item, counts, withSteps))
+        .filter((item) => item !== undefined);
+}
+/** Map one learning path: three scrubbed strings, nothing else. */
+function mapLearningPath(path, counts) {
+    if (path === null || typeof path !== "object")
+        return undefined;
+    const source = path;
+    const topic = scrub(String(source.topic ?? ""), counts);
+    const description = scrub(String(source.description ?? ""), counts);
+    if (!topic.trim() || !description.trim())
+        return undefined;
+    return {
+        topic,
+        chain: Array.isArray(source.chain)
+            ? source.chain.map((step) => scrub(String(step), counts))
+            : [],
+        description,
+    };
+}
+/**
+ * Read the newest active profile of the space folder, read-only. The identity
+ * columns of the profile row (user id, names, email) are never read, so they
+ * cannot reach the file.
+ */
+async function readProfile(store, counts) {
+    const dbPath = join(store.folder, PROFILE_DB_NAME);
+    if (!existsSync(dbPath))
+        return undefined;
+    const db = await store.openShard(dbPath);
+    const table = await db.get(`SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'user_profiles'`);
+    if (!table)
+        return undefined;
+    const row = await db.get(`SELECT profile_data, version, last_analyzed_at, total_prompts_analyzed
+       FROM user_profiles WHERE is_active = 1 ORDER BY last_analyzed_at DESC LIMIT 1`);
+    if (!row)
+        return undefined;
+    let data;
+    try {
+        data = JSON.parse(String(row.profile_data ?? "{}"));
+    }
+    catch {
+        return undefined;
+    }
+    const source = (data ?? {});
+    const preferences = mapProfileItems(source.preferences, counts, false);
+    const patterns = mapProfileItems(source.patterns, counts, false);
+    const workflows = mapProfileItems(source.workflows, counts, true);
+    counts.profileItems = preferences.length + patterns.length + workflows.length;
+    const learningPaths = (Array.isArray(source.learning_paths) ? source.learning_paths : [])
+        .map((path) => mapLearningPath(path, counts))
+        .filter((path) => path !== undefined);
+    return {
+        preferences,
+        patterns,
+        workflows,
+        ...(learningPaths.length > 0 ? { learningPaths } : {}),
+        version: Number(row.version ?? 1),
+        lastAnalyzedAt: Number(row.last_analyzed_at ?? 0),
+        totalPromptsAnalyzed: Number(row.total_prompts_analyzed ?? 0),
+    };
+}
+/** Home-folder shapes of the scrubber, re-checked on the finished file. */
+const SCAN_POSIX_HOME = /(?<![\w~])\/(?:Users|home)\//;
+const SCAN_WINDOWS_HOME = /(?<![\w~])[a-z]:[\\/]users[\\/]/i;
+/** Wider than the scrubber's email rule: catches quoted local parts and the like. */
+const SCAN_EMAIL = /\S*@\S+\.\S+/;
+function escapeRegExp(text) {
+    return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+/**
+ * The final safety scan (design D7): fail the export when a home-folder path
+ * or an email address survived the per-string scrubbing. System paths and
+ * URLs are deliberate non-matches, because memories often quote them.
+ */
+export function scanExportContent(serialised, homePath) {
+    const findings = [];
+    const home = homePath.replace(/[\\/]+$/, "");
+    if (home && new RegExp(`${escapeRegExp(home)}(?![\\w])`).test(serialised)) {
+        findings.push("the home folder path of the exporting user");
+    }
+    if (SCAN_POSIX_HOME.test(serialised))
+        findings.push("a POSIX home-folder path");
+    if (SCAN_WINDOWS_HOME.test(serialised))
+        findings.push("a Windows home-folder path");
+    if (SCAN_EMAIL.test(serialised))
+        findings.push("an email address");
+    if (findings.length > 0) {
+        throw new Error(`Space export identifier scan failed: ${findings.join("; ")} survived the scrubber, so no file was written.`);
+    }
+}
+/** Write atomically: a temporary file first, then one rename into place. */
+function writeAtomic(outPath, serialised) {
+    const target = resolve(outPath);
+    mkdirSync(dirname(target), { recursive: true });
+    const tmpPath = `${target}.tmp-${process.pid}-${Date.now()}`;
+    try {
+        writeFileSync(tmpPath, serialised, "utf-8");
+        renameSync(tmpPath, target);
+    }
+    catch (error) {
+        if (existsSync(tmpPath)) {
+            try {
+                unlinkSync(tmpPath);
+            }
+            catch {
+                // Best-effort cleanup; the failure below is the real error.
+            }
+        }
+        throw error;
+    }
+}
+/**
+ * Export one space to a JSON file (design D7). Reads the space through a
+ * read-only store, so the space is never changed. Throws when the final
+ * identifier scan fails; returns `written: false` for an empty selection.
+ */
+export async function exportSpace(options) {
+    const counts = {
+        projectMemories: 0,
+        userMemories: 0,
+        privateSkipped: 0,
+        profileItems: 0,
+        pathsReplaced: 0,
+        emailsReplaced: 0,
+    };
+    const warnings = [FREE_TEXT_WARNING];
+    const store = openSpaceStore(options.folder, "read");
+    let memories;
+    let profile;
+    try {
+        memories = await readScopeMemories(store, "project", options, counts);
+        if (options.includeProfile) {
+            memories.push(...(await readScopeMemories(store, "user", options, counts)));
+            profile = await readProfile(store, counts);
+            if (!profile)
+                warnings.push("The space holds no learned profile.");
+        }
+    }
+    finally {
+        await store.close();
+    }
+    if (memories.length === 0) {
+        return {
+            written: false,
+            reason: "Nothing to export: the selection holds no memory.",
+            counts,
+            warnings,
+        };
+    }
+    memories.sort((a, b) => b.createdAt - a.createdAt);
+    const document = {
+        format: SPACE_EXPORT_FORMAT,
+        schemaVersion: SPACE_EXPORT_SCHEMA_VERSION,
+        exportedAt: new Date().toISOString(),
+        ommsVersion: options.ommsVersion,
+        embedding: {
+            model: options.embeddingModel,
+            dimensions: options.embeddingDimensions,
+        },
+        memories,
+        ...(profile ? { profile } : {}),
+    };
+    // Validate our own output before it reaches the disk, then run the final
+    // identifier scan on exactly the bytes that would be written.
+    SpaceExportDocumentSchema.parse(document);
+    const serialised = JSON.stringify(document, null, 2);
+    scanExportContent(serialised, options.homePath);
+    writeAtomic(options.outPath, serialised);
+    return { written: true, outputPath: resolve(options.outPath), counts, warnings };
+}

package/dist/services/spaces/identifier-scrub.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Replacement counts for one scrubbed string, so an export can report how
+ * much it changed.
+ */
+export interface ScrubCounts {
+    /** Home-folder paths rewritten to the `~` form. */
+    readonly pathsReplaced: number;
+    /** Email addresses rewritten to `<email>`. */
+    readonly emailsReplaced: number;
+}
+/** The scrubbed string plus what the scrubber replaced in it. */
+export interface ScrubResult {
+    readonly text: string;
+    readonly counts: ScrubCounts;
+}
+/**
+ * Replace the parts of a string that identify a person: home-folder paths
+ * become `~` plus the rest of the path, email addresses become `<email>`,
+ * and `<private>` regions go first (via `stripPrivateContent`), so anything
+ * inside them is neither kept nor counted.
+ *
+ * Deliberately narrow: system paths (`/usr/local`) and URLs stay, because
+ * memories often quote them. The exporter scans the finished file and fails
+ * the export when an identifier survives anyway.
+ */
+export declare function scrubIdentifiers(text: string): ScrubResult;

package/dist/services/spaces/identifier-scrub.js

@@ -0,0 +1,40 @@
+import { stripPrivateContent } from "../privacy.js";
+// Characters that end a path token in free text. A path cut at one of them
+// still loses its user name, which is the part that identifies a person.
+const STOP = "\\s\"'`<>()[\\]{},;:!?";
+// `/Users/<name>/...` (macOS) and `/home/<name>/...` (Linux). The lookbehind
+// keeps the path part of a URL (`example.com/home/jo`) out of the match.
+const POSIX_HOME = new RegExp(`(?<!\\w)\\/(?:Users|home)\\/([^/${STOP}]+)([^${STOP}]*)`, "g");
+// `<drive>:\Users\<name>\...` (Windows), any drive letter, either separator.
+const WINDOWS_HOME = new RegExp(`(?<!\\w)[a-z]:[\\\\/]users[\\\\/]([^\\\\/${STOP}]+)([^${STOP}]*)`, "gi");
+const EMAIL = /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g;
+/**
+ * Replace the parts of a string that identify a person: home-folder paths
+ * become `~` plus the rest of the path, email addresses become `<email>`,
+ * and `<private>` regions go first (via `stripPrivateContent`), so anything
+ * inside them is neither kept nor counted.
+ *
+ * Deliberately narrow: system paths (`/usr/local`) and URLs stay, because
+ * memories often quote them. The exporter scans the finished file and fails
+ * the export when an identifier survives anyway.
+ */
+export function scrubIdentifiers(text) {
+    const withoutPrivate = stripPrivateContent(text);
+    // Windows first: its forward-slash form ends in `/Users/...`, which the
+    // POSIX pass would otherwise rewrite and leave a stray `C:~` behind.
+    let pathsReplaced = 0;
+    const toTilde = (_match, _name, rest) => {
+        pathsReplaced++;
+        return `~${rest}`;
+    };
+    const afterPaths = withoutPrivate.replace(WINDOWS_HOME, toTilde).replace(POSIX_HOME, toTilde);
+    let emailsReplaced = 0;
+    const scrubbed = afterPaths.replace(EMAIL, () => {
+        emailsReplaced++;
+        return "<email>";
+    });
+    return {
+        text: scrubbed,
+        counts: { pathsReplaced, emailsReplaced },
+    };
+}

package/dist/services/spaces/import-space.d.ts

@@ -0,0 +1,85 @@
+import { type SpaceExportDocument } from "../../shared/api/space-export-schemas.js";
+import { type SpaceFiles } from "./registry-store.js";
+/** The import size cap of the spec: a file larger than this is refused. */
+export declare const IMPORT_MAX_FILE_BYTES: number;
+/** The embedding settings of the importing machine; `CONFIG` satisfies this. */
+export interface ImportEmbeddingSettings {
+    model: string;
+    dimensions: number;
+}
+/** The importing user: user-scope memories take `userTag`. */
+export interface ImportingUser {
+    /** The importing user's container tag, `omms_user_<hash>`. */
+    userTag: string;
+    displayName?: string;
+    userName?: string;
+    userEmail?: string;
+}
+/** Embeds one text with the current model; `EmbeddingService.embed` satisfies this. */
+export type ImportEmbedder = (text: string) => Promise<Float32Array>;
+/** Everything the importer needs; nothing is read from CONFIG here. */
+export interface ImportSpaceOptions {
+    /** Path of the export file. Give this or `document`. */
+    filePath?: string;
+    /** A parsed export document, as the web upload check route passes it. */
+    document?: unknown;
+    /** The name of the new space. A taken name is refused. */
+    spaceName: string;
+    /** Registry and folders of the OMMS store, normally `spaceFiles(ommsHomeDir())`. */
+    files: SpaceFiles;
+    /** Optional project folder: every project memory takes that folder's tag. */
+    retagToProject?: string;
+    embedding: ImportEmbeddingSettings;
+    embed: ImportEmbedder;
+    user: ImportingUser;
+    /** Size cap for `filePath`; defaults to {@link IMPORT_MAX_FILE_BYTES}. */
+    maxFileBytes?: number;
+}
+/** Totals of one import, for the CLI and web dialogs to report. */
+export interface ImportSpaceCounts {
+    projectMemories: number;
+    userMemories: number;
+    /** Profile items written through the rooted profile manager. */
+    profileItems: number;
+    /** Distinct project tags in the file: what a `retagToProject` merges into one. */
+    fileProjects: number;
+}
+export interface ImportSpaceResult {
+    ok: boolean;
+    /** Why the import was refused or failed; set only when `ok` is false. */
+    reason?: string;
+    /** Id of the created space; set only when `ok` is true. */
+    spaceId?: string;
+    spaceName?: string;
+    /** The created space folder; set only when `ok` is true. */
+    folder?: string;
+    counts: ImportSpaceCounts;
+    /** Human-facing notes, never blocking. The model warning comes first. */
+    warnings: string[];
+}
+/**
+ * The model and dimension warning of the spec: names the file's settings and
+ * the current ones, and never blocks. Undefined when both match.
+ */
+export declare function embeddingWarning(file: ImportEmbeddingSettings, current: ImportEmbeddingSettings): string | undefined;
+export type ValidatedImport = {
+    ok: false;
+    reason: string;
+} | {
+    ok: true;
+    document: SpaceExportDocument;
+    counts: ImportSpaceCounts;
+    warning?: string;
+};
+/**
+ * Validates one parsed export document against the spec rules: the schema
+ * version, the schema itself, duplicate ids and the shape of every container
+ * tag. Pure, so the web check route can run it without creating a space.
+ */
+export declare function validateSpaceImportDocument(raw: unknown, embedding: ImportEmbeddingSettings, userTag: string): ValidatedImport;
+/**
+ * Imports one export file into a new, user-named space (design D7). Every
+ * refusal runs before any space exists; a failure after the space exists
+ * removes the space and its folder. The active space never changes.
+ */
+export declare function importSpace(options: ImportSpaceOptions): Promise<ImportSpaceResult>;