om-memory-system 4.19.1 → 4.20.0

package/dist/services/user-profile/ai-cleanup-batches.js

@@ -0,0 +1,177 @@
+import { log } from "../logger.js";
+import { addIdsToProfile, buildAICleanupPrompt, buildItemIndex, callAICleanup, generateDiff, rebuildProfileUsing, } from "./ai-cleanup.js";
+/**
+ * A large selection sent in one model request runs past the request time limit. The web
+ * cleanup therefore sends the selection in batches, one request each, one after another.
+ */
+const SECTIONS = ["preferences", "patterns", "workflows"];
+/** Indexes the full profile once, so ids are profile positions, and keeps the selected items. */
+export function selectForCleanup(profileData, includeIds) {
+    const idSet = new Set(includeIds);
+    const indexed = addIdsToProfile(profileData);
+    return {
+        preferences: indexed.preferences.filter((p) => idSet.has(p.id)),
+        patterns: indexed.patterns.filter((p) => idSet.has(p.id)),
+        workflows: indexed.workflows.filter((p) => idSet.has(p.id)),
+    };
+}
+/**
+ * Packs each section on its own, sorted by category, so a category of up to `size` items
+ * stays in one batch. Likely duplicates share a category, and the model merges only
+ * inside a section.
+ */
+export function splitIntoBatches(indexed, size) {
+    const batches = [];
+    for (const section of SECTIONS) {
+        const flush = (items) => {
+            if (items.length > 0)
+                batches.push({ preferences: [], patterns: [], workflows: [], [section]: items });
+        };
+        let current = [];
+        for (const group of groupByCategory(indexed[section])) {
+            if (group.length > size) {
+                flush(current);
+                current = [];
+                for (let i = 0; i < group.length; i += size) {
+                    const chunk = group.slice(i, i + size);
+                    if (chunk.length === size)
+                        flush(chunk);
+                    else
+                        current = chunk;
+                }
+            }
+            else if (current.length + group.length > size) {
+                flush(current);
+                current = [...group];
+            }
+            else {
+                current.push(...group);
+            }
+        }
+        flush(current);
+    }
+    return batches;
+}
+function groupByCategory(items) {
+    const groups = new Map();
+    for (const item of items) {
+        const key = (item.category ?? "").toLowerCase();
+        const group = groups.get(key);
+        if (group)
+            group.push(item);
+        else
+            groups.set(key, [item]);
+    }
+    return [...groups.keys()].sort().map((key) => groups.get(key));
+}
+/** One model request for one batch. */
+export async function cleanupBatch(indexed) {
+    const t0 = Date.now();
+    const prompt = buildAICleanupPrompt(indexed);
+    log("AI cleanup: batch prompt built", { items: countItems(indexed), promptLen: prompt.length });
+    const result = await callAICleanup(prompt);
+    log("AI cleanup: batch reply received", { callMs: Date.now() - t0 });
+    const cleanedById = buildItemIndex(result.profile);
+    const originalById = buildItemIndex(indexed);
+    const mapping = restrictToBatch(result.mapping, originalById);
+    return {
+        diff: generateDiff(indexed, mapping, cleanedById),
+        mergedItems: buildMergedItems(mapping, cleanedById, originalById),
+    };
+}
+// The model may return an id that is not in the batch. Apply looks ids up in the whole
+// profile, so such an id could change an item the user never selected.
+function restrictToBatch(mapping, originalById) {
+    const inBatch = (id) => originalById.has(id);
+    return {
+        kept: mapping.kept.filter(inBatch),
+        merged: mapping.merged.map((group) => group.filter(inBatch)).filter((g) => g.length > 1),
+        removed: mapping.removed.filter(inBatch),
+    };
+}
+// The merged item of each group, keyed by its target id, so Apply can put a ticked merge
+// into a newer profile without the rest of the cleaned batch.
+function buildMergedItems(mapping, cleanedById, originalById) {
+    const items = {};
+    for (const group of mapping.merged.filter((g) => g.length > 1)) {
+        const originals = new Map();
+        for (const id of group) {
+            const original = originalById.get(id);
+            if (original)
+                originals.set(id, original);
+        }
+        const one = rebuildProfileUsing({ kept: [], merged: [group], removed: [] }, cleanedById, originals);
+        const item = one.preferences[0] ?? one.patterns[0] ?? one.workflows[0];
+        if (item)
+            items[group[0]] = item;
+    }
+    return items;
+}
+/**
+ * Runs the batches one after another and joins their results. A failed batch adds
+ * nothing, so its items stay as they are. Throws the first error when every batch
+ * fails. Returns null when stopped; the stop flag is read before and after each call.
+ */
+export async function runInBatches(batches, { callBatch = cleanupBatch, onProgress, isStopped = () => false } = {}) {
+    const t0 = Date.now();
+    const joined = {
+        diff: { kept: [], merged: [], removed: [] },
+        mergedItems: {},
+        failedBatches: 0,
+        failedItems: 0,
+    };
+    let firstError;
+    for (const [index, batch] of batches.entries()) {
+        if (isStopped())
+            return null;
+        try {
+            const result = await callBatch(batch);
+            if (isStopped())
+                return null;
+            joined.diff.kept.push(...result.diff.kept);
+            joined.diff.merged.push(...result.diff.merged);
+            joined.diff.removed.push(...result.diff.removed);
+            Object.assign(joined.mergedItems, result.mergedItems);
+        }
+        catch (error) {
+            if (isStopped())
+                return null;
+            if (joined.failedBatches === 0)
+                firstError = error;
+            joined.failedBatches++;
+            joined.failedItems += countItems(batch);
+            log("AI cleanup: batch failed", {
+                batch: index + 1,
+                of: batches.length,
+                items: countItems(batch),
+                reason: failureReason(error),
+            });
+        }
+        onProgress?.(index + 1, batches.length, joined.failedBatches);
+    }
+    if (batches.length > 0 && joined.failedBatches === batches.length)
+        throw firstError;
+    log("AI cleanup: batches finished", {
+        batches: batches.length,
+        failedBatches: joined.failedBatches,
+        totalMs: Date.now() - t0,
+    });
+    return joined;
+}
+/** A fixed reason code for the log. Never the error text, which may quote the reply. */
+export function failureReason(error) {
+    if (error instanceof Error) {
+        if (error.name === "TimeoutError" || error.name === "AbortError")
+            return "timeout";
+        const status = /API request failed: (\d{3})/.exec(error.message)?.[1];
+        if (status)
+            return `http-${status}`;
+        if (error instanceof SyntaxError || /valid JSON|No content/.test(error.message)) {
+            return "invalid-reply";
+        }
+    }
+    return "error";
+}
+function countItems(indexed) {
+    return indexed.preferences.length + indexed.patterns.length + indexed.workflows.length;
+}

package/dist/services/user-profile/ai-cleanup-job.d.ts

@@ -0,0 +1,52 @@
+import type { CleanupDiff } from "./ai-cleanup.js";
+import type { CleanupJobResult } from "./ai-cleanup-apply.js";
+import { type RunInBatchesOptions } from "./ai-cleanup-batches.js";
+import type { UserProfileData } from "./types.js";
+/**
+ * A large AI cleanup takes many minutes, longer than a web request may stay open. The web
+ * app therefore runs it as a background job per user, and the dialog polls its state.
+ * The job also holds the finished result until Apply. Jobs live in this process only; a
+ * restart drops them, and nothing is saved before Apply.
+ */
+export type AiCleanupJobStatus = {
+    state: "running";
+    done: number;
+    total: number;
+    failed: number;
+} | {
+    state: "done";
+    result: {
+        old: UserProfileData;
+        changes: CleanupDiff;
+        failedBatches: number;
+        failedItems: number;
+    };
+} | {
+    state: "failed";
+    error: string;
+};
+export interface AiCleanupJobView {
+    id: string;
+    startedAt: number;
+    selection: string[];
+    status: AiCleanupJobStatus;
+}
+export interface AiCleanupJobInput {
+    /** The profile as the dialog lists it, so item ids match. */
+    profileData: UserProfileData;
+    /** An empty or missing selection means every item. */
+    includeIds?: string[];
+}
+export interface AiCleanupJobDeps {
+    callBatch?: RunInBatchesOptions["callBatch"];
+    batchSize?: number;
+    now?: () => number;
+}
+/** Starts a cleanup job for the user, or returns the one already running. */
+export declare function startAiCleanupJob(userId: string, input: AiCleanupJobInput, deps?: AiCleanupJobDeps): AiCleanupJobView;
+/** The user's job, or null when none was started or a finished one expired. */
+export declare function getAiCleanupJob(userId: string, now?: number): AiCleanupJobView | null;
+/** The finished result that Apply needs, or null when the job is not done. */
+export declare function getAiCleanupResult(userId: string, now?: number): CleanupJobResult | null;
+/** Stops a running job, or discards a finished one. A stopped job writes nothing. */
+export declare function stopAiCleanupJob(userId: string): void;

package/dist/services/user-profile/ai-cleanup-job.js

@@ -0,0 +1,112 @@
+import { log } from "../logger.js";
+import { AI_CLEANUP_BATCH_SIZE } from "../request-timeouts.js";
+import { failureReason, runInBatches, selectForCleanup, splitIntoBatches, } from "./ai-cleanup-batches.js";
+/** A finished job is forgotten this long after it finished, as the old pending result was. */
+const FINISHED_JOB_TTL_MS = 30 * 60 * 1000;
+const jobs = new Map();
+let nextId = 1;
+/** Starts a cleanup job for the user, or returns the one already running. */
+export function startAiCleanupJob(userId, input, deps = {}) {
+    const current = jobs.get(userId);
+    if (current?.status.state === "running")
+        return view(current);
+    const now = deps.now ?? Date.now;
+    const selection = input.includeIds?.length ? input.includeIds : allIds(input.profileData);
+    const batches = splitIntoBatches(selectForCleanup(input.profileData, selection), deps.batchSize ?? AI_CLEANUP_BATCH_SIZE);
+    const job = {
+        id: String(nextId++),
+        startedAt: now(),
+        selection,
+        status: { state: "running", done: 0, total: batches.length, failed: 0 },
+        stopped: false,
+        mergedItems: {},
+    };
+    jobs.set(userId, job);
+    void (async () => {
+        try {
+            const result = await runInBatches(batches, {
+                callBatch: deps.callBatch,
+                isStopped: () => job.stopped,
+                onProgress: (done, total, failed) => {
+                    job.status = { state: "running", done, total, failed };
+                },
+            });
+            if (!result)
+                return;
+            job.mergedItems = result.mergedItems;
+            job.status = {
+                state: "done",
+                result: {
+                    old: input.profileData,
+                    changes: result.diff,
+                    failedBatches: result.failedBatches,
+                    failedItems: result.failedItems,
+                },
+            };
+        }
+        catch (error) {
+            if (job.stopped)
+                return;
+            const reason = failureReason(error);
+            log("AI cleanup job: failed", { reason });
+            job.status = {
+                state: "failed",
+                error: reason === "timeout"
+                    ? "The model did not answer in time."
+                    : error instanceof Error
+                        ? error.message
+                        : String(error),
+            };
+        }
+        finally {
+            job.finishedAt = now();
+        }
+    })();
+    return view(job);
+}
+/** The user's job, or null when none was started or a finished one expired. */
+export function getAiCleanupJob(userId, now = Date.now()) {
+    const job = liveJob(userId, now);
+    return job ? view(job) : null;
+}
+/** The finished result that Apply needs, or null when the job is not done. */
+export function getAiCleanupResult(userId, now = Date.now()) {
+    const job = liveJob(userId, now);
+    if (job?.status.state !== "done")
+        return null;
+    return {
+        old: job.status.result.old,
+        changes: job.status.result.changes,
+        mergedItems: job.mergedItems,
+    };
+}
+/** Stops a running job, or discards a finished one. A stopped job writes nothing. */
+export function stopAiCleanupJob(userId) {
+    const job = jobs.get(userId);
+    if (!job)
+        return;
+    job.stopped = true;
+    jobs.delete(userId);
+}
+function liveJob(userId, now) {
+    const job = jobs.get(userId);
+    if (!job)
+        return null;
+    if (job.status.state !== "running" && job.finishedAt !== undefined) {
+        if (now - job.finishedAt > FINISHED_JOB_TTL_MS) {
+            jobs.delete(userId);
+            return null;
+        }
+    }
+    return job;
+}
+function view(job) {
+    return { id: job.id, startedAt: job.startedAt, selection: job.selection, status: job.status };
+}
+function allIds(profile) {
+    return [
+        ...profile.preferences.map((_, i) => `pref_${i}`),
+        ...profile.patterns.map((_, i) => `pat_${i}`),
+        ...profile.workflows.map((_, i) => `wf_${i}`),
+    ];
+}

package/dist/services/user-profile/ai-cleanup.d.ts

@@ -15,9 +15,7 @@ export interface CleanupDiff {
     }>;
 }
 export declare function aiCleanupProfile(profileData: UserProfileData): Promise<AICleanupResult>;
-export declare function aiCleanupProfileFromIndexed(indexed: IndexedProfile): Promise<AICleanupResult>;
-export declare function filterProfileForCleanup(profileData: UserProfileData, includeIds: string[]): IndexedProfile;
-interface IndexedProfileItem {
+export interface IndexedProfileItem {
     id: string;
     category?: string;
     description: string;
@@ -25,18 +23,25 @@ interface IndexedProfileItem {
     frequency?: number;
     [key: string]: unknown;
 }
-interface IndexedProfile {
+export interface IndexedProfile {
     preferences: IndexedProfileItem[];
     patterns: IndexedProfileItem[];
     workflows: IndexedProfileItem[];
 }
-interface AIMapping {
+export interface AIMapping {
     kept: string[];
     merged: string[][];
     removed: string[];
 }
+export declare function addIdsToProfile(profile: UserProfileData): IndexedProfile;
+export declare function buildAICleanupPrompt(profile: IndexedProfile): string;
+export declare function callAICleanup(prompt: string): Promise<{
+    profile: IndexedProfile;
+    mapping: AIMapping;
+}>;
+export declare function buildItemIndex(profile: IndexedProfile): Map<string, IndexedProfileItem>;
 export declare function rebuildProfileUsing(mapping: AIMapping, cleanedById: Map<string, IndexedProfileItem>, originalById: Map<string, IndexedProfileItem>, counters?: {
     cleaned: number;
     original: number;
 }): UserProfileData;
-export {};
+export declare function generateDiff(original: IndexedProfile, mapping: AIMapping, cleanedById: Map<string, IndexedProfileItem>): CleanupDiff;

package/dist/services/user-profile/ai-cleanup.js

@@ -43,41 +43,6 @@ export async function aiCleanupProfile(profileData) {
     });
     return { cleaned, diff };
 }
-export async function aiCleanupProfileFromIndexed(indexed) {
-    const t0 = Date.now();
-    const prompt = buildAICleanupPrompt(indexed);
-    log("AI cleanup: prompt built (filtered)", {
-        prefCount: indexed.preferences.length,
-        patCount: indexed.patterns.length,
-        wfCount: indexed.workflows.length,
-        promptLen: prompt.length,
-        buildMs: Date.now() - t0,
-    });
-    const aiStart = Date.now();
-    const result = await callAICleanup(prompt);
-    log("AI cleanup: AI response received (filtered)", { callMs: Date.now() - aiStart });
-    const cleanedById = buildItemIndex(result.profile);
-    const originalById = buildItemIndex(indexed);
-    const counters = { cleaned: 0, original: 0 };
-    const cleaned = rebuildProfileUsing(result.mapping, cleanedById, originalById, counters);
-    const diff = generateDiff(indexed, result.mapping, cleanedById);
-    log("AI cleanup: complete (filtered)", {
-        totalMs: Date.now() - t0,
-        kept: diff.kept.length,
-        merged: diff.merged.length,
-        removed: diff.removed.length,
-    });
-    return { cleaned, diff };
-}
-export function filterProfileForCleanup(profileData, includeIds) {
-    const idSet = new Set(includeIds);
-    const indexed = addIdsToProfile(profileData);
-    return {
-        preferences: indexed.preferences.filter((p) => idSet.has(p.id)),
-        patterns: indexed.patterns.filter((p) => idSet.has(p.id)),
-        workflows: indexed.workflows.filter((p) => idSet.has(p.id)),
-    };
-}
 // The model controls this JSON; it may omit fields or return wrong types. Coerce to a
 // well-formed AIMapping so rebuildProfileUsing/generateDiff never call .map/.filter/.includes
 // on undefined. A missing or malformed mapping degrades to a no-op cleanup (all originals are
@@ -100,7 +65,7 @@ function normalizeAIMapping(raw) {
     const removed = Array.isArray(raw.removed) ? raw.removed.filter(isStr) : [];
     return { kept, merged, removed };
 }
-function addIdsToProfile(profile) {
+export function addIdsToProfile(profile) {
     const items = {
         preferences: profile.preferences.map((p, i) => ({ ...p, id: `pref_${i}` })),
         patterns: profile.patterns.map((p, i) => ({ ...p, id: `pat_${i}` })),
@@ -108,7 +73,7 @@ function addIdsToProfile(profile) {
     };
     return items;
 }
-function buildAICleanupPrompt(profile) {
+export function buildAICleanupPrompt(profile) {
     const profileJSON = JSON.stringify({
         preferences: profile.preferences.map(formatForAI),
         patterns: profile.patterns.map(formatForAI),
@@ -147,7 +112,7 @@ function formatForAI(item) {
     const { id, category, description, frequency } = item;
     return { id, category, description, frequency };
 }
-async function callAICleanup(prompt) {
+export async function callAICleanup(prompt) {
     // Use the host model when the host registered one (same pattern as auto-capture).
     // When it is available, surface its errors instead of masking them as
     // "No AI provider configured" via a silent fallback (#177).
@@ -215,7 +180,7 @@ async function callViaHostModel(model, prompt) {
         mapping: normalizeAIMapping(parsed.mapping),
     };
 }
-function buildItemIndex(profile) {
+export function buildItemIndex(profile) {
     const map = new Map();
     for (const item of profile.preferences)
         map.set(item.id, item);
@@ -393,7 +358,7 @@ export function rebuildProfileUsing(mapping, cleanedById, originalById, counters
     }
     return result;
 }
-function generateDiff(original, mapping, cleanedById) {
+export function generateDiff(original, mapping, cleanedById) {
     const index = buildItemIndex(original);
     const diff = {
         kept: mapping.kept.map((id) => index.get(id)?.description || id),

package/dist/services/web-server.js

@@ -12,7 +12,8 @@ import { isOlderVersion } from "./version-compare.js";
 import { AUTH_HEADER, getOrCreateAuthToken, isAuthorizedApiRequest } from "./auth-token.js";
 import { WebAuth } from "./web-auth.js";
 import { NODE_HTTP_IDLE_TIMEOUT_MS } from "./request-timeouts.js";
-import { handleListTags, handleListKeywords, handleListMemories, handleAddMemory, handleDeleteMemory, handleBulkDelete, handleUpdateMemory, handleSearch, handleStats, handlePinMemory, handleUnpinMemory, handleRunCleanup, handleRunDeduplication, handleDetectMigration, handleRunMigration, handleDetectTagMigration, handleRunTagMigrationBatch, handleGetTagMigrationProgress, handleDeletePrompt, handleBulkDeletePrompts, handleGetUserProfile, handleGetProfileChangelog, handleGetProfileSnapshot, handleRefreshProfile, handleAICleanup, handleApplyCleanup, handleCategoryCleanupApply, handleCategoryCleanupStart, handleCategoryCleanupStatus, handleCategoryCleanupUndo, handleUpdateProfileItem, } from "./api-handlers.js";
+import { handleListTags, handleListKeywords, handleListMemories, handleAddMemory, handleDeleteMemory, handleBulkDelete, handleUpdateMemory, handleSearch, handleStats, handlePinMemory, handleUnpinMemory, handleRunCleanup, handleRunDeduplication, handleDetectMigration, handleRunMigration, handleDetectTagMigration, handleRunTagMigrationBatch, handleGetTagMigrationProgress, handleDeletePrompt, handleBulkDeletePrompts, handleGetUserProfile, handleGetProfileChangelog, handleGetProfileSnapshot, handleRefreshProfile, handleCategoryCleanupApply, handleCategoryCleanupStart, handleCategoryCleanupStatus, handleCategoryCleanupUndo, handleUpdateProfileItem, } from "./api-handlers.js";
+import * as aiCleanup from "./ai-cleanup-handlers.js";
 const isBun = typeof globalThis.Bun !== "undefined";
 const MIN_FAILED_TAKEOVERS = 3;
 export function attachNodeDisconnectHandlers(req, res, onDisconnect) {
@@ -1343,19 +1344,16 @@ export class WebServer {
             }
             if (path === "/api/user-profile/ai-cleanup" && method === "POST") {
                 const body = (await req.json().catch(() => ({})));
-                const userId = body.userId || undefined;
-                const includeIds = Array.isArray(body.includeIds)
-                    ? body.includeIds
-                    : undefined;
-                const profileVersion = typeof body.profileVersion === "number" ? body.profileVersion : undefined;
-                const result = await handleAICleanup(userId, includeIds, profileVersion);
-                return this.jsonResponse(result);
+                return this.jsonResponse(await aiCleanup.handleAiCleanupStart(body.userId, body));
+            }
+            if (path === "/api/user-profile/ai-cleanup" && (method === "GET" || method === "DELETE")) {
+                const userId = url.searchParams.get("userId") || undefined;
+                const handle = method === "GET" ? aiCleanup.handleAiCleanupStatus : aiCleanup.handleAiCleanupStop;
+                return this.jsonResponse(await handle(userId));
             }
             if (path === "/api/user-profile/ai-cleanup/apply" && method === "POST") {
                 const body = (await req.json().catch(() => ({})));
-                const userId = body.userId || undefined;
-                const result = await handleApplyCleanup(userId, body);
-                return this.jsonResponse(result);
+                return this.jsonResponse(await aiCleanup.handleAiCleanupApply(body.userId, body));
             }
             if (path === "/api/user-profile/category-cleanup" && method === "POST") {
                 const body = (await req.json().catch(() => ({})));