om-memory-system 4.14.1 → 4.16.1

package/dist/services/user-profile/category-cleanup-job.js

@@ -0,0 +1,50 @@
+import { log } from "../logger.js";
+import { previewCategoryCleanup, } from "./category-cleanup.js";
+const jobs = new Map();
+let nextId = 1;
+function running(job) {
+    return !!job && (job.status.state === "grouping" || job.status.state === "checking");
+}
+/** Starts a preview job for the user, or returns the one already running. */
+export function startCategoryCleanupJob(userId, deps = {}) {
+    const current = jobs.get(userId);
+    if (running(current))
+        return current;
+    const job = {
+        id: String(nextId++),
+        startedAt: Date.now(),
+        status: { state: "grouping" },
+    };
+    jobs.set(userId, job);
+    void (async () => {
+        try {
+            const preview = await previewCategoryCleanup(userId, {
+                ...deps,
+                onProgress: (done, total) => {
+                    job.status = { state: "checking", done, total };
+                },
+            });
+            job.status = preview
+                ? { state: "done", preview }
+                : { state: "failed", error: "No profile found to clean up" };
+        }
+        catch (error) {
+            log("category cleanup job: failed", {
+                code: error instanceof Error ? error.name : "unknown",
+            });
+            job.status = {
+                state: "failed",
+                error: error instanceof Error ? error.message : String(error),
+            };
+        }
+    })();
+    return job;
+}
+/** Forgets the user's finished job, so the next preview reflects the changed profile. */
+export function clearCategoryCleanupJob(userId) {
+    if (!running(jobs.get(userId)))
+        jobs.delete(userId);
+}
+export function getCategoryCleanupJob(userId) {
+    return jobs.get(userId) ?? null;
+}

package/dist/services/user-profile/category-cleanup-store.d.ts

@@ -0,0 +1,8 @@
+import type { CategoryRename } from "./category-groups.js";
+export declare function recordCleanup(profileId: string, renames: CategoryRename[]): Promise<void>;
+/** The newest cleanup of a profile that has not been undone. */
+export declare function latestOpenCleanup(profileId: string): Promise<{
+    id: number;
+    renames: CategoryRename[];
+} | null>;
+export declare function markCleanupUndone(id: number): Promise<void>;

package/dist/services/user-profile/category-cleanup-store.js

@@ -0,0 +1,35 @@
+import { join } from "node:path";
+import { CONFIG } from "../../config.js";
+import { tursoConnectionManager } from "../turso/connection-manager.js";
+import { USER_PROFILES_DB_NAME } from "./user-profile-manager.js";
+/** Undo records for category cleanups. The profile changelog keeps too few entries for undo. */
+async function connection() {
+    const db = await tursoConnectionManager.getConnection(join(CONFIG.storagePath || "", USER_PROFILES_DB_NAME));
+    await db.execute(`CREATE TABLE IF NOT EXISTS profile_category_cleanups (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    profile_id TEXT NOT NULL,
+    created_at INTEGER NOT NULL,
+    renames TEXT NOT NULL,
+    undone_at INTEGER
+  )`);
+    return db;
+}
+export async function recordCleanup(profileId, renames) {
+    const db = await connection();
+    await db.execute("INSERT INTO profile_category_cleanups (profile_id, created_at, renames) VALUES (?, ?, ?)", [profileId, Date.now(), JSON.stringify(renames)]);
+}
+/** The newest cleanup of a profile that has not been undone. */
+export async function latestOpenCleanup(profileId) {
+    const db = await connection();
+    const row = await db.get("SELECT id, renames FROM profile_category_cleanups WHERE profile_id = ? AND undone_at IS NULL ORDER BY id DESC LIMIT 1", [profileId]);
+    if (!row)
+        return null;
+    return { id: Number(row.id), renames: JSON.parse(String(row.renames)) };
+}
+export async function markCleanupUndone(id) {
+    const db = await connection();
+    await db.execute("UPDATE profile_category_cleanups SET undone_at = ? WHERE id = ?", [
+        Date.now(),
+        id,
+    ]);
+}

package/dist/services/user-profile/category-cleanup.d.ts

@@ -0,0 +1,74 @@
+import { type CategoryGroup } from "./category-groups.js";
+import type { UserProfileManager } from "./user-profile-manager.js";
+export declare const GROUPS_PER_MODEL_CALL = 20;
+export declare const MAX_EXAMPLES_PER_LABEL = 3;
+/**
+ * Time limit per model call. A reasoning model took about 110 s per batch on the
+ * live profile (TDR-044), so the 60 s profile cleanup limit timed out every batch.
+ */
+export declare const CATEGORY_CLEANUP_MODEL_TIMEOUT_MS = 180000;
+/** Rejects with the message when the work takes longer than the limit, and always clears its timer. */
+export declare function withTimeout<T>(work: Promise<T>, ms: number, message: string): Promise<T>;
+/** Sends one prompt and returns the parsed JSON reply; throws on any failure. */
+export type CleanupModel = (systemPrompt: string, userPrompt: string) => Promise<unknown>;
+export interface NamedGroup extends CategoryGroup {
+    name: string;
+    /** False when the model did not check this group and the most used label was chosen. */
+    checked: boolean;
+}
+/**
+ * The profile model checks each group in batches: it removes names that do not belong
+ * and proposes one name. A failed batch, a missing reply, or no model falls back to the
+ * most used label and marks the group unchecked. The log holds counts and codes only.
+ */
+export declare function checkAndNameGroups(groups: CategoryGroup[], examples: Map<string, string[]>, model: CleanupModel | null, 
+/** Called with batches done and the total, before the first batch and after each one. */
+onProgress?: (done: number, total: number) => void): Promise<NamedGroup[]>;
+/** The host model when one is registered, else the external API, else none. */
+export declare function resolveCleanupModel(): Promise<CleanupModel | null>;
+export interface CleanupDeps {
+    manager?: UserProfileManager;
+    /** Embeds one category name; defaults to the configured embedder. */
+    embed?: (text: string) => Promise<number[]>;
+    /** Undefined resolves the profile model; null runs without one. */
+    model?: CleanupModel | null;
+    threshold?: number;
+    /** Reports model batches done and the total while a preview is built. */
+    onProgress?: (done: number, total: number) => void;
+}
+export interface CategoryCleanupPreview {
+    profileId: string;
+    version: number;
+    /** Distinct categories now, and if every group were accepted. */
+    before: number;
+    after: number;
+    groups: NamedGroup[];
+}
+export interface AcceptedGroup {
+    labels: string[];
+    name: string;
+}
+export type ApplyResult = {
+    ok: true;
+    renamed: number;
+} | {
+    ok: false;
+    error: "stale" | "no_profile" | "invalid";
+};
+export type UndoResult = {
+    ok: true;
+    restored: number;
+    notFound: number;
+} | {
+    ok: false;
+    error: "nothing_to_undo" | "stale";
+};
+/** Builds the groups for review. Changes nothing in the stored profile. */
+export declare function previewCategoryCleanup(userId: string, deps?: CleanupDeps): Promise<CategoryCleanupPreview | null>;
+/** Renames the categories of the accepted groups; refuses when the profile changed since the preview. */
+export declare function applyCategoryCleanup(userId: string, input: {
+    version: number;
+    groups: AcceptedGroup[];
+}, deps?: CleanupDeps): Promise<ApplyResult>;
+/** Restores the categories of the newest cleanup that is not undone yet. */
+export declare function undoCategoryCleanup(userId: string, deps?: CleanupDeps): Promise<UndoResult>;

package/dist/services/user-profile/category-cleanup.js

@@ -0,0 +1,310 @@
+/**
+ * Smart category cleanup: group profile category names by meaning, let the profile
+ * model check and name each group, then apply a rename-only change the user accepted.
+ * See openspec/changes/smart-category-cleanup.
+ */
+import { CONFIG } from "../../config.js";
+import { log } from "../logger.js";
+import { applySafeExtraParams } from "../ai/providers/base-provider.js";
+import { completeStructured, resolveHostProfileModel } from "./profile-model.js";
+import { CATEGORY_GROUP_THRESHOLD, collectCategoryLabels, groupCategoryLabels, groupId, recordRenames, restoreRenames, } from "./category-groups.js";
+import { canonicaliseCategories, categoryKey, tidyCategory } from "./profile-category.js";
+import { latestOpenCleanup, markCleanupUndone, recordCleanup } from "./category-cleanup-store.js";
+export const GROUPS_PER_MODEL_CALL = 20;
+export const MAX_EXAMPLES_PER_LABEL = 3;
+/**
+ * Time limit per model call. A reasoning model took about 110 s per batch on the
+ * live profile (TDR-044), so the 60 s profile cleanup limit timed out every batch.
+ */
+export const CATEGORY_CLEANUP_MODEL_TIMEOUT_MS = 180_000;
+const MAX_NAME_LENGTH = 60;
+/** Rejects with the message when the work takes longer than the limit, and always clears its timer. */
+export async function withTimeout(work, ms, message) {
+    let timer;
+    try {
+        return await Promise.race([
+            work,
+            new Promise((_, reject) => {
+                timer = setTimeout(() => reject(new Error(message)), ms);
+            }),
+        ]);
+    }
+    finally {
+        clearTimeout(timer);
+    }
+}
+const SYSTEM_PROMPT = "You tidy the category names of a user profile. Return only JSON without markdown wrapping.";
+function buildPrompt(groups, examples) {
+    const payload = groups.map((group) => ({
+        group: group.id,
+        labels: group.labels.map((entry) => ({
+            name: entry.label,
+            count: entry.count,
+            examples: (examples.get(entry.label) ?? []).slice(0, MAX_EXAMPLES_PER_LABEL),
+        })),
+    }));
+    return `Each group below holds category names that may mean the same thing. The examples show memories filed under each name.
+
+For each group:
+1. "keep": the names that mean the same thing as each other. Leave out any name with a different meaning.
+2. "name": one short category name for the kept names. Prefer an existing name when one fits.
+
+Groups:
+${JSON.stringify(payload)}
+
+Return JSON only: { "groups": [ { "group": "<id>", "keep": ["<name>", "..."], "name": "<name>" } ] }`;
+}
+function fallback(group) {
+    return { ...group, name: group.labels[0].label, checked: false };
+}
+function readReply(reply) {
+    const entries = reply?.groups;
+    const byId = new Map();
+    if (!Array.isArray(entries))
+        return byId;
+    for (const entry of entries) {
+        if (entry && typeof entry === "object" && typeof entry.group === "string") {
+            byId.set(entry.group, { keep: entry.keep, name: entry.name });
+        }
+    }
+    return byId;
+}
+function applyReply(group, answer) {
+    if (!answer || !Array.isArray(answer.keep))
+        return fallback(group);
+    const keep = new Set(answer.keep.filter((name) => typeof name === "string"));
+    const labels = group.labels.filter((entry) => keep.has(entry.label));
+    if (labels.length < 2)
+        return null;
+    const name = typeof answer.name === "string" &&
+        answer.name.trim() &&
+        answer.name.trim().length <= MAX_NAME_LENGTH
+        ? answer.name.trim()
+        : labels[0].label;
+    return { id: groupId(labels), labels, name, checked: true };
+}
+/**
+ * The profile model checks each group in batches: it removes names that do not belong
+ * and proposes one name. A failed batch, a missing reply, or no model falls back to the
+ * most used label and marks the group unchecked. The log holds counts and codes only.
+ */
+export async function checkAndNameGroups(groups, examples, model, 
+/** Called with batches done and the total, before the first batch and after each one. */
+onProgress) {
+    const batches = [];
+    for (let start = 0; start < groups.length; start += GROUPS_PER_MODEL_CALL) {
+        batches.push(groups.slice(start, start + GROUPS_PER_MODEL_CALL));
+    }
+    let failedBatches = 0;
+    let dropped = 0;
+    const checkBatch = async (batch) => {
+        if (!model)
+            return batch.map(fallback);
+        let replies;
+        try {
+            replies = readReply(await model(SYSTEM_PROMPT, buildPrompt(batch, examples)));
+        }
+        catch (error) {
+            failedBatches++;
+            log("category cleanup: model batch failed", {
+                groups: batch.length,
+                code: error instanceof Error ? error.name : "unknown",
+            });
+            return batch.map(fallback);
+        }
+        const results = [];
+        for (const group of batch) {
+            const result = applyReply(group, replies.get(group.id));
+            if (result)
+                results.push(result);
+            else
+                dropped++;
+        }
+        return results;
+    };
+    // One call at a time: a provider that refuses parallel calls still finishes (TDR-044).
+    const named = [];
+    onProgress?.(0, batches.length);
+    for (const [index, batch] of batches.entries()) {
+        named.push(...(await checkBatch(batch)));
+        onProgress?.(index + 1, batches.length);
+    }
+    log("category cleanup: groups checked", {
+        groups: groups.length,
+        named: named.length,
+        dropped,
+        unchecked: named.filter((group) => !group.checked).length,
+        failedBatches,
+        model: model ? "available" : "none",
+    });
+    return named;
+}
+/** The host model when one is registered, else the external API, else none. */
+export async function resolveCleanupModel() {
+    const hostModel = await resolveHostProfileModel();
+    if (hostModel) {
+        const { z } = await import("zod");
+        const schema = z.object({
+            groups: z.array(z.object({ group: z.string(), keep: z.array(z.string()), name: z.string() })),
+        });
+        return (systemPrompt, userPrompt) => withTimeout(completeStructured(hostModel, systemPrompt, userPrompt, schema), CATEGORY_CLEANUP_MODEL_TIMEOUT_MS, "category cleanup timeout");
+    }
+    if (CONFIG.memoryModel && CONFIG.memoryApiUrl) {
+        return async (systemPrompt, userPrompt) => {
+            const requestBody = {};
+            if (CONFIG.memoryExtraParams)
+                applySafeExtraParams(requestBody, CONFIG.memoryExtraParams);
+            Object.assign(requestBody, {
+                model: CONFIG.memoryModel,
+                messages: [
+                    { role: "system", content: systemPrompt },
+                    { role: "user", content: userPrompt },
+                ],
+                temperature: 0,
+                response_format: { type: "json_object" },
+            });
+            const response = await fetch(`${CONFIG.memoryApiUrl}/chat/completions`, {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                    Authorization: `Bearer ${CONFIG.memoryApiKey || ""}`,
+                },
+                body: JSON.stringify(requestBody),
+                signal: AbortSignal.timeout(CATEGORY_CLEANUP_MODEL_TIMEOUT_MS),
+            });
+            if (!response.ok)
+                throw new Error(`API request failed: ${response.status}`);
+            const data = await response.json();
+            const content = data.choices?.[0]?.message?.content;
+            if (!content)
+                throw new Error("No content in API response");
+            return JSON.parse(content);
+        };
+    }
+    return null;
+}
+async function managerOf(deps) {
+    return deps.manager ?? (await import("./user-profile-manager.js")).userProfileManager;
+}
+async function defaultEmbed() {
+    const { EmbeddingService } = await import("../embedding.js");
+    const service = EmbeddingService.getInstance();
+    return async (text) => Array.from(await service.embed(text, { task: "document" }));
+}
+function readProfile(profileData) {
+    const parsed = JSON.parse(profileData);
+    return canonicaliseCategories({
+        ...parsed,
+        preferences: Array.isArray(parsed.preferences) ? parsed.preferences : [],
+        patterns: Array.isArray(parsed.patterns) ? parsed.patterns : [],
+        workflows: Array.isArray(parsed.workflows) ? parsed.workflows : [],
+    });
+}
+function examplesByLabel(data) {
+    const items = [...data.preferences, ...data.patterns].sort((a, b) => (b.frequency ?? 0) - (a.frequency ?? 0));
+    const examples = new Map();
+    for (const item of items) {
+        const list = examples.get(item.category) ?? [];
+        if (list.length < MAX_EXAMPLES_PER_LABEL)
+            list.push(item.description);
+        examples.set(item.category, list);
+    }
+    return examples;
+}
+function countAfter(labels, groups) {
+    const rename = new Map();
+    for (const group of groups) {
+        for (const entry of group.labels)
+            rename.set(entry.label, tidyCategory(group.name));
+    }
+    return new Set(labels.map((entry) => categoryKey(rename.get(entry.label) ?? entry.label))).size;
+}
+/** Builds the groups for review. Changes nothing in the stored profile. */
+export async function previewCategoryCleanup(userId, deps = {}) {
+    const manager = await managerOf(deps);
+    const profile = await manager.getActiveProfile(userId);
+    if (!profile)
+        return null;
+    const data = readProfile(profile.profileData);
+    const labels = collectCategoryLabels(data);
+    const embed = deps.embed ?? (await defaultEmbed());
+    const vectors = [];
+    for (const entry of labels)
+        vectors.push({ ...entry, vector: await embed(entry.label) });
+    const groups = groupCategoryLabels(vectors, deps.threshold ?? CATEGORY_GROUP_THRESHOLD);
+    const model = deps.model === undefined ? await resolveCleanupModel() : deps.model;
+    const named = await checkAndNameGroups(groups, examplesByLabel(data), model, deps.onProgress);
+    log("category cleanup: preview built", {
+        labels: labels.length,
+        groups: named.length,
+    });
+    return {
+        profileId: profile.id,
+        version: profile.version,
+        before: labels.length,
+        after: countAfter(labels, named),
+        groups: named,
+    };
+}
+function validGroups(groups) {
+    return (Array.isArray(groups) &&
+        groups.every((group) => group &&
+            Array.isArray(group.labels) &&
+            group.labels.length >= 2 &&
+            group.labels.every((label) => typeof label === "string") &&
+            typeof group.name === "string" &&
+            group.name.trim().length > 0));
+}
+/** Renames the categories of the accepted groups; refuses when the profile changed since the preview. */
+export async function applyCategoryCleanup(userId, input, deps = {}) {
+    if (!Number.isInteger(input?.version) || !validGroups(input.groups))
+        return { ok: false, error: "invalid" };
+    const manager = await managerOf(deps);
+    const profile = await manager.getActiveProfile(userId);
+    if (!profile)
+        return { ok: false, error: "no_profile" };
+    if (profile.version !== input.version)
+        return { ok: false, error: "stale" };
+    const data = readProfile(profile.profileData);
+    const rename = new Map();
+    for (const group of input.groups) {
+        for (const label of group.labels)
+            rename.set(label, tidyCategory(group.name));
+    }
+    const retag = (items) => items.map((item) => rename.has(item.category) ? { ...item, category: rename.get(item.category) } : item);
+    const next = {
+        ...data,
+        preferences: retag(data.preferences),
+        patterns: retag(data.patterns),
+    };
+    const renames = recordRenames(data, canonicaliseCategories(next));
+    if (renames.length === 0)
+        return { ok: true, renamed: 0 };
+    const summary = `Category cleanup: ${renames.length} items renamed in ${input.groups.length} groups`;
+    if (!(await manager.updateProfile(profile.id, next, 0, summary, input.version))) {
+        return { ok: false, error: "stale" };
+    }
+    await recordCleanup(profile.id, renames);
+    log("category cleanup: applied", { groups: input.groups.length, renamed: renames.length });
+    return { ok: true, renamed: renames.length };
+}
+/** Restores the categories of the newest cleanup that is not undone yet. */
+export async function undoCategoryCleanup(userId, deps = {}) {
+    const manager = await managerOf(deps);
+    const profile = await manager.getActiveProfile(userId);
+    if (!profile)
+        return { ok: false, error: "nothing_to_undo" };
+    const record = await latestOpenCleanup(profile.id);
+    if (!record)
+        return { ok: false, error: "nothing_to_undo" };
+    const restored = restoreRenames(readProfile(profile.profileData), record.renames);
+    if (restored.restored > 0) {
+        const summary = `Category cleanup undone: ${restored.restored} items restored`;
+        if (!(await manager.updateProfile(profile.id, restored.profile, 0, summary, profile.version))) {
+            return { ok: false, error: "stale" };
+        }
+    }
+    await markCleanupUndone(record.id);
+    log("category cleanup: undone", { restored: restored.restored, notFound: restored.notFound });
+    return { ok: true, restored: restored.restored, notFound: restored.notFound };
+}

package/dist/services/user-profile/category-groups.d.ts

@@ -0,0 +1,60 @@
+/** Average cosine similarity at which two groups of category names count as one meaning. */
+export declare const CATEGORY_GROUP_THRESHOLD = 0.8;
+export interface CategoryLabelCount {
+    label: string;
+    count: number;
+}
+export interface CategoryLabelVector extends CategoryLabelCount {
+    vector: number[];
+}
+export interface CategoryGroup {
+    /** Stable for the same set of labels, so a preview and its apply agree. */
+    id: string;
+    labels: CategoryLabelCount[];
+}
+type CategoryProfile = {
+    preferences: {
+        category: string;
+    }[];
+    patterns: {
+        category: string;
+    }[];
+};
+/** Distinct labels across preferences and patterns after the spelling rules, most used first. */
+export declare function collectCategoryLabels(profile: CategoryProfile): CategoryLabelCount[];
+export declare function groupId(labels: CategoryLabelCount[]): string;
+/**
+ * Groups category names that mean the same thing. Only labels joined by at least one
+ * pair at or above the threshold can share a group, so clustering runs per connected
+ * component of that graph. A group needs two labels; `explicit` never joins one.
+ */
+export declare function groupCategoryLabels(labels: CategoryLabelVector[], threshold?: number): CategoryGroup[];
+export type CategorySection = "preferences" | "patterns";
+/** One item renamed by a cleanup. The description is stored as a hash, never as text. */
+export interface CategoryRename {
+    section: CategorySection;
+    descriptionSha256: string;
+    from: string;
+    to: string;
+}
+type DescribedItem = {
+    category: string;
+    description: string;
+};
+type DescribedProfile = {
+    preferences: DescribedItem[];
+    patterns: DescribedItem[];
+};
+export declare function descriptionHash(description: string): string;
+/** Renames between two versions of one profile whose items are in the same order. */
+export declare function recordRenames(before: DescribedProfile, after: DescribedProfile): CategoryRename[];
+/**
+ * Puts the old category back on each renamed item that still exists with the
+ * category the cleanup gave it. Items added or merged later are left alone.
+ */
+export declare function restoreRenames<T extends DescribedProfile>(current: T, renames: CategoryRename[]): {
+    profile: T;
+    restored: number;
+    notFound: number;
+};
+export {};