om-memory-system 4.17.0-next.124 → 4.19.1

package/dist/services/user-profile/category-cleanup.js

@@ -9,7 +9,7 @@ import { applySafeExtraParams } from "../ai/providers/base-provider.js";
 import { completeStructured, resolveHostProfileModel } from "./profile-model.js";
 import { CATEGORY_GROUP_THRESHOLD, collectCategoryLabels, groupCategoryLabels, groupId, recordRenames, restoreRenames, } from "./category-groups.js";
 import { canonicaliseCategories, categoryKey, tidyCategory } from "./profile-category.js";
-import { latestOpenCleanup, markCleanupUndone, recordCleanup } from "./category-cleanup-store.js";
+import { latestOpenCleanup, loadVerdicts, markCleanupUndone, pruneVerdicts, recordCleanup, saveVerdicts, } from "./category-cleanup-store.js";
 export const GROUPS_PER_MODEL_CALL = 20;
 export const MAX_EXAMPLES_PER_LABEL = 3;
 /**
@@ -83,23 +83,43 @@ function applyReply(group, answer) {
         : labels[0].label;
     return { id: groupId(labels), labels, name, checked: true };
 }
+function verdictOf(answer) {
+    if (!answer || !Array.isArray(answer.keep))
+        return null;
+    return {
+        keep: answer.keep.filter((name) => typeof name === "string"),
+        name: typeof answer.name === "string" ? answer.name : null,
+    };
+}
 /**
  * The profile model checks each group in batches: it removes names that do not belong
- * and proposes one name. A failed batch, a missing reply, or no model falls back to the
- * most used label and marks the group unchecked. The log holds counts and codes only.
+ * and proposes one name. A group with a saved answer skips the model. A failed batch, a
+ * missing reply, or no model falls back to the most used label, marks the group unchecked
+ * and saves nothing, so a later preview retries it. The log holds counts and codes only.
  */
 export async function checkAndNameGroups(groups, examples, model, 
-/** Called with batches done and the total, before the first batch and after each one. */
-onProgress) {
+/** Called with batches to run and the total, before the first batch and after each one. */
+onProgress, cache) {
+    const named = new Map();
+    const pending = [];
+    for (const group of groups) {
+        const saved = cache?.saved.get(group.id);
+        if (saved)
+            named.set(group.id, applyReply(group, saved));
+        else
+            pending.push(group);
+    }
     const batches = [];
-    for (let start = 0; start < groups.length; start += GROUPS_PER_MODEL_CALL) {
-        batches.push(groups.slice(start, start + GROUPS_PER_MODEL_CALL));
+    for (let start = 0; start < pending.length; start += GROUPS_PER_MODEL_CALL) {
+        batches.push(pending.slice(start, start + GROUPS_PER_MODEL_CALL));
     }
     let failedBatches = 0;
-    let dropped = 0;
     const checkBatch = async (batch) => {
-        if (!model)
-            return batch.map(fallback);
+        if (!model) {
+            for (const group of batch)
+                named.set(group.id, fallback(group));
+            return;
+        }
         let replies;
         try {
             replies = readReply(await model(SYSTEM_PROMPT, buildPrompt(batch, examples)));
@@ -110,34 +130,50 @@ onProgress) {
                 groups: batch.length,
                 code: error instanceof Error ? error.name : "unknown",
             });
-            return batch.map(fallback);
+            for (const group of batch)
+                named.set(group.id, fallback(group));
+            return;
         }
-        const results = [];
+        const verdicts = new Map();
         for (const group of batch) {
-            const result = applyReply(group, replies.get(group.id));
-            if (result)
-                results.push(result);
-            else
-                dropped++;
+            const answer = replies.get(group.id);
+            named.set(group.id, applyReply(group, answer));
+            const verdict = verdictOf(answer);
+            if (verdict)
+                verdicts.set(group.id, verdict);
+        }
+        if (cache && verdicts.size > 0) {
+            try {
+                await cache.save(verdicts);
+            }
+            catch (error) {
+                log("category cleanup: verdict save failed", {
+                    groups: verdicts.size,
+                    code: error instanceof Error ? error.name : "unknown",
+                });
+            }
         }
-        return results;
     };
     // One call at a time: a provider that refuses parallel calls still finishes (TDR-044).
-    const named = [];
     onProgress?.(0, batches.length);
     for (const [index, batch] of batches.entries()) {
-        named.push(...(await checkBatch(batch)));
+        await checkBatch(batch);
         onProgress?.(index + 1, batches.length);
     }
+    const result = groups
+        .map((group) => named.get(group.id))
+        .filter((group) => !!group);
     log("category cleanup: groups checked", {
         groups: groups.length,
-        named: named.length,
-        dropped,
-        unchecked: named.filter((group) => !group.checked).length,
+        reused: groups.length - pending.length,
+        sent: pending.length,
+        named: result.length,
+        dropped: groups.length - result.length,
+        unchecked: result.filter((group) => !group.checked).length,
         failedBatches,
         model: model ? "available" : "none",
     });
-    return named;
+    return result;
 }
 /** The host model when one is registered, else the external API, else none. */
 export async function resolveCleanupModel() {
@@ -219,28 +255,53 @@ function countAfter(labels, groups) {
     }
     return new Set(labels.map((entry) => categoryKey(rename.get(entry.label) ?? entry.label))).size;
 }
+/** Opens the saved model answers for the groups; without the store the preview just asks the model. */
+async function openVerdictCache(profileId, groupIds) {
+    try {
+        await pruneVerdicts(profileId, groupIds);
+        return {
+            saved: await loadVerdicts(profileId),
+            save: (verdicts) => saveVerdicts(profileId, verdicts),
+        };
+    }
+    catch (error) {
+        log("category cleanup: verdict store unavailable", {
+            code: error instanceof Error ? error.name : "unknown",
+        });
+        return undefined;
+    }
+}
 /** Builds the groups for review. Changes nothing in the stored profile. */
 export async function previewCategoryCleanup(userId, deps = {}) {
     const manager = await managerOf(deps);
     const profile = await manager.getActiveProfile(userId);
     if (!profile)
         return null;
+    const started = Date.now();
     const data = readProfile(profile.profileData);
     const labels = collectCategoryLabels(data);
     const embed = deps.embed ?? (await defaultEmbed());
     const vectors = [];
     for (const entry of labels)
         vectors.push({ ...entry, vector: await embed(entry.label) });
+    const embedded = Date.now();
     const groups = groupCategoryLabels(vectors, deps.threshold ?? CATEGORY_GROUP_THRESHOLD);
+    const grouped = Date.now();
+    const cache = await openVerdictCache(profile.id, groups.map((group) => group.id));
     const model = deps.model === undefined ? await resolveCleanupModel() : deps.model;
-    const named = await checkAndNameGroups(groups, examplesByLabel(data), model, deps.onProgress);
+    const named = await checkAndNameGroups(groups, examplesByLabel(data), model, deps.onProgress, cache);
+    const reused = groups.filter((group) => cache?.saved.has(group.id)).length;
     log("category cleanup: preview built", {
         labels: labels.length,
         groups: named.length,
+        reusedGroups: reused,
+        sentGroups: groups.length - reused,
+        embedMs: embedded - started,
+        groupMs: grouped - embedded,
+        checkMs: Date.now() - grouped,
     });
     return {
         profileId: profile.id,
-        version: profile.version,
         before: labels.length,
         after: countAfter(labels, named),
         groups: named,
@@ -255,16 +316,18 @@ function validGroups(groups) {
             typeof group.name === "string" &&
             group.name.trim().length > 0));
 }
-/** Renames the categories of the accepted groups; refuses when the profile changed since the preview. */
+/**
+ * Renames the categories of the accepted groups on the profile as stored now. Learning may
+ * have saved newer versions since the preview; the rename goes by label, so that is safe.
+ * It refuses only when a save lands between the read and the write below.
+ */
 export async function applyCategoryCleanup(userId, input, deps = {}) {
-    if (!Number.isInteger(input?.version) || !validGroups(input.groups))
+    if (!validGroups(input?.groups))
         return { ok: false, error: "invalid" };
     const manager = await managerOf(deps);
     const profile = await manager.getActiveProfile(userId);
     if (!profile)
         return { ok: false, error: "no_profile" };
-    if (profile.version !== input.version)
-        return { ok: false, error: "stale" };
     const data = readProfile(profile.profileData);
     const rename = new Map();
     for (const group of input.groups) {
@@ -281,7 +344,7 @@ export async function applyCategoryCleanup(userId, input, deps = {}) {
     if (renames.length === 0)
         return { ok: true, renamed: 0 };
     const summary = `Category cleanup: ${renames.length} items renamed in ${input.groups.length} groups`;
-    if (!(await manager.updateProfile(profile.id, next, 0, summary, input.version))) {
+    if (!(await manager.updateProfile(profile.id, next, 0, summary, profile.version))) {
         return { ok: false, error: "stale" };
     }
     await recordCleanup(profile.id, renames);

package/dist/services/user-profile/merge-and-save.d.ts

@@ -0,0 +1,25 @@
+import type { UserProfile, UserProfileData } from "./types.js";
+/** Learning makes this many save attempts before it leaves the prompts for a later run. */
+export declare const MAX_PROFILE_SAVE_ATTEMPTS = 3;
+/** The profile store operations that a save needs. */
+export interface ProfileSaveStore {
+    getProfileById(id: string): Promise<UserProfile | null>;
+    mergeProfileData(existing: UserProfileData, updates: Partial<UserProfileData>, embedService?: undefined, profileId?: string): Promise<UserProfileData>;
+    updateProfile(profileId: string, profileData: UserProfileData, additionalPromptsAnalyzed: number, changeSummary: string, expectedVersion: number): Promise<boolean>;
+}
+type ReadProfile = Pick<UserProfile, "id" | "version">;
+/**
+ * Saves `data` over the profile as the caller read it. Returns false, and changes nothing,
+ * when another save landed since. The log holds a code only, never profile content.
+ */
+export declare function saveProfile(profile: ReadProfile, data: UserProfileData, summary: string, promptsAnalyzed?: number, store?: Pick<ProfileSaveStore, "updateProfile">): Promise<boolean>;
+/**
+ * Saves the merged result of a profile analysis. When another save landed first, it loads
+ * the newer profile, merges the same model result into it, and saves again. It never calls
+ * the model. Returns false when there is nothing to save or every attempt met a newer version.
+ */
+export declare function saveMergedProfile(store: ProfileSaveStore, profile: ReadProfile, analysis: {
+    raw: Partial<UserProfileData>;
+    merged: UserProfileData | null;
+}, promptsAnalyzed: number, summary: string): Promise<boolean>;
+export {};

package/dist/services/user-profile/merge-and-save.js

@@ -0,0 +1,38 @@
+import { log } from "../logger.js";
+/** Learning makes this many save attempts before it leaves the prompts for a later run. */
+export const MAX_PROFILE_SAVE_ATTEMPTS = 3;
+/**
+ * Saves `data` over the profile as the caller read it. Returns false, and changes nothing,
+ * when another save landed since. The log holds a code only, never profile content.
+ */
+export async function saveProfile(profile, data, summary, promptsAnalyzed = 0, store) {
+    const target = store ?? (await import("./user-profile-manager.js")).userProfileManager;
+    const saved = await target.updateProfile(profile.id, data, promptsAnalyzed, summary, profile.version);
+    if (!saved)
+        log("profile save skipped", { code: "version-changed" });
+    return saved;
+}
+/**
+ * Saves the merged result of a profile analysis. When another save landed first, it loads
+ * the newer profile, merges the same model result into it, and saves again. It never calls
+ * the model. Returns false when there is nothing to save or every attempt met a newer version.
+ */
+export async function saveMergedProfile(store, profile, analysis, promptsAnalyzed, summary) {
+    let merged = analysis.merged;
+    let version = profile.version;
+    for (let attempt = 1; merged && attempt <= MAX_PROFILE_SAVE_ATTEMPTS; attempt++) {
+        if (await store.updateProfile(profile.id, merged, promptsAnalyzed, summary, version)) {
+            return true;
+        }
+        log("profile save conflict", { attempt, of: MAX_PROFILE_SAVE_ATTEMPTS });
+        if (attempt === MAX_PROFILE_SAVE_ATTEMPTS)
+            break;
+        const fresh = await store.getProfileById(profile.id);
+        if (!fresh)
+            return false;
+        const current = JSON.parse(fresh.profileData);
+        merged = await store.mergeProfileData(current, analysis.raw, undefined, fresh.id);
+        version = fresh.version;
+    }
+    return false;
+}

package/dist/services/user-profile/user-profile-manager.d.ts

@@ -20,8 +20,8 @@ export declare class UserProfileManager {
     getActiveProfile(userId: string): Promise<UserProfile | null>;
     createProfile(userId: string, displayName: string, userName: string, userEmail: string, profileData: UserProfileData, promptsAnalyzed: number): Promise<string>;
     updateProfile(profileId: string, profileData: UserProfileData, additionalPromptsAnalyzed: number, changeSummary: string, 
-    /** When given, refuse the save unless the stored version still equals it. */
-    expectedVersion?: number): Promise<boolean>;
+    /** The version the caller read. The save is refused when the stored version differs. */
+    expectedVersion: number): Promise<boolean>;
     private addChangelog;
     private cleanupOldChangelogs;
     getProfileChangelogs(profileId: string, limit?: number): Promise<UserProfileChangelog[]>;

package/dist/services/user-profile/user-profile-manager.js

@@ -274,7 +274,7 @@ export class UserProfileManager {
         return id;
     }
     async updateProfile(profileId, profileData, additionalPromptsAnalyzed, changeSummary, 
-    /** When given, refuse the save unless the stored version still equals it. */
+    /** The version the caller read. The save is refused when the stored version differs. */
     expectedVersion) {
         const db = await this.ready();
         const now = Date.now();
@@ -286,7 +286,7 @@ export class UserProfileManager {
         });
         const versionRow = await db.get(`SELECT version FROM user_profiles WHERE id = ?`, [profileId]);
         const currentVersion = Number(versionRow?.version ?? 0);
-        if (expectedVersion !== undefined && currentVersion !== expectedVersion)
+        if (currentVersion !== expectedVersion)
             return false;
         const newVersion = currentVersion + 1;
         const changes = await db.run(`