om-memory-system 4.18.0 → 4.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/opencode/profile-learning.js +7 -6
- package/dist/adapters/pi/profile.js +2 -1
- package/dist/cli/profile-normalise-categories-command.js +1 -1
- package/dist/core/memory-operations.js +10 -2
- package/dist/importer/claude-hook-api.js +2 -2
- package/dist/importer/profile-backlog.d.ts +1 -1
- package/dist/importer/profile-backlog.js +3 -2
- package/dist/importer/profile-import.d.ts +1 -1
- package/dist/services/ai-cleanup-handlers.d.ts +31 -0
- package/dist/services/ai-cleanup-handlers.js +126 -0
- package/dist/services/api-handlers.d.ts +0 -22
- package/dist/services/api-handlers.js +3 -242
- package/dist/services/profile-admin.d.ts +1 -1
- package/dist/services/profile-admin.js +1 -1
- package/dist/services/request-timeouts.d.ts +2 -0
- package/dist/services/request-timeouts.js +2 -0
- package/dist/services/user-profile/ai-cleanup-apply.d.ts +27 -0
- package/dist/services/user-profile/ai-cleanup-apply.js +94 -0
- package/dist/services/user-profile/ai-cleanup-batches.d.ts +34 -0
- package/dist/services/user-profile/ai-cleanup-batches.js +177 -0
- package/dist/services/user-profile/ai-cleanup-job.d.ts +52 -0
- package/dist/services/user-profile/ai-cleanup-job.js +112 -0
- package/dist/services/user-profile/ai-cleanup.d.ts +11 -6
- package/dist/services/user-profile/ai-cleanup.js +5 -40
- package/dist/services/user-profile/merge-and-save.d.ts +25 -0
- package/dist/services/user-profile/merge-and-save.js +38 -0
- package/dist/services/user-profile/user-profile-manager.d.ts +2 -2
- package/dist/services/user-profile/user-profile-manager.js +2 -2
- package/dist/services/web-server.js +9 -11
- package/dist/web/assets/index-CYcRfxr4.css +2 -0
- package/dist/web/assets/index-DvNL0vOb.js +127 -0
- package/dist/web/index.html +2 -2
- package/package.json +1 -1
- package/dist/web/assets/index-BktRr9cC.css +0 -2
- package/dist/web/assets/index-CIODmA62.js +0 -127
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
export declare const EXTERNAL_PROFILE_CLEANUP_TIMEOUT_MS = 60000;
|
|
2
2
|
export declare const OPENCODE_PROFILE_CLEANUP_TIMEOUT_MS = 300000;
|
|
3
|
+
/** The most items one web AI cleanup request sends, so each request ends inside its time limit. */
|
|
4
|
+
export declare const AI_CLEANUP_BATCH_SIZE = 100;
|
|
3
5
|
export declare const NODE_HTTP_IDLE_TIMEOUT_MS: number;
|
|
4
6
|
/**
|
|
5
7
|
* Upper bound an OpenCode v2 model request waits for the per-prompt memory
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
export const EXTERNAL_PROFILE_CLEANUP_TIMEOUT_MS = 60_000;
|
|
2
2
|
export const OPENCODE_PROFILE_CLEANUP_TIMEOUT_MS = 300_000;
|
|
3
|
+
/** The most items one web AI cleanup request sends, so each request ends inside its time limit. */
|
|
4
|
+
export const AI_CLEANUP_BATCH_SIZE = 100;
|
|
3
5
|
const HTTP_TIMEOUT_GRACE_MS = 30_000;
|
|
4
6
|
export const NODE_HTTP_IDLE_TIMEOUT_MS = Math.max(EXTERNAL_PROFILE_CLEANUP_TIMEOUT_MS, OPENCODE_PROFILE_CLEANUP_TIMEOUT_MS) +
|
|
5
7
|
HTTP_TIMEOUT_GRACE_MS;
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import type { CleanupDiff } from "./ai-cleanup.js";
|
|
2
|
+
import type { UserProfileData } from "./types.js";
|
|
3
|
+
/** What a finished cleanup job keeps for Apply. Ids are positions in `old`. */
|
|
4
|
+
export interface CleanupJobResult {
|
|
5
|
+
old: UserProfileData;
|
|
6
|
+
changes: CleanupDiff;
|
|
7
|
+
mergedItems: Record<string, unknown>;
|
|
8
|
+
}
|
|
9
|
+
export interface AcceptedCleanup {
|
|
10
|
+
merged?: string[][];
|
|
11
|
+
removed?: string[];
|
|
12
|
+
}
|
|
13
|
+
export interface AppliedCleanup {
|
|
14
|
+
profile: UserProfileData;
|
|
15
|
+
applied: number;
|
|
16
|
+
skipped: number;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Puts the ticked changes of a job into `current`, which may be newer than the profile
|
|
20
|
+
* the job read. Each item is found by section, category, description and steps, so
|
|
21
|
+
* learning's counters do not break a match. A change is skipped when one of its items is
|
|
22
|
+
* gone. Every other item stays exactly as stored. Ticked ids that are not changes of
|
|
23
|
+
* the job are ignored.
|
|
24
|
+
*/
|
|
25
|
+
export declare function applyCleanupEdits(current: UserProfileData, job: CleanupJobResult, accepted: AcceptedCleanup): AppliedCleanup;
|
|
26
|
+
/** The fields that identify a profile item. Counters and vectors are left out. */
|
|
27
|
+
export declare function profileItemIdentityKey(item: any): string;
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
const SECTION_BY_PREFIX = {
|
|
2
|
+
pref: "preferences",
|
|
3
|
+
pat: "patterns",
|
|
4
|
+
wf: "workflows",
|
|
5
|
+
};
|
|
6
|
+
/**
|
|
7
|
+
* Puts the ticked changes of a job into `current`, which may be newer than the profile
|
|
8
|
+
* the job read. Each item is found by section, category, description and steps, so
|
|
9
|
+
* learning's counters do not break a match. A change is skipped when one of its items is
|
|
10
|
+
* gone. Every other item stays exactly as stored. Ticked ids that are not changes of
|
|
11
|
+
* the job are ignored.
|
|
12
|
+
*/
|
|
13
|
+
export function applyCleanupEdits(current, job, accepted) {
|
|
14
|
+
const profile = {
|
|
15
|
+
...current,
|
|
16
|
+
preferences: [...current.preferences],
|
|
17
|
+
patterns: [...current.patterns],
|
|
18
|
+
workflows: [...current.workflows],
|
|
19
|
+
};
|
|
20
|
+
const merges = new Map(job.changes.merged.map((m) => [m.ids.join(","), m.ids]));
|
|
21
|
+
const removals = new Set(job.changes.removed.map((r) => r.id));
|
|
22
|
+
const used = new Set();
|
|
23
|
+
let applied = 0;
|
|
24
|
+
let skipped = 0;
|
|
25
|
+
for (const ticked of accepted.merged ?? []) {
|
|
26
|
+
const ids = Array.isArray(ticked) ? merges.get(ticked.join(",")) : undefined;
|
|
27
|
+
if (!ids || ids.length < 2)
|
|
28
|
+
continue;
|
|
29
|
+
const mergedItem = job.mergedItems[ids[0]];
|
|
30
|
+
const found = mergedItem ? locate(profile, job.old, ids, used) : null;
|
|
31
|
+
if (!found) {
|
|
32
|
+
skipped++;
|
|
33
|
+
continue;
|
|
34
|
+
}
|
|
35
|
+
const items = profile[found.section];
|
|
36
|
+
items[found.indices[0]] = { ...mergedItem };
|
|
37
|
+
for (const index of found.indices.slice(1).sort((a, b) => b - a))
|
|
38
|
+
items.splice(index, 1);
|
|
39
|
+
for (const id of ids)
|
|
40
|
+
used.add(id);
|
|
41
|
+
applied++;
|
|
42
|
+
}
|
|
43
|
+
for (const id of accepted.removed ?? []) {
|
|
44
|
+
if (typeof id !== "string" || !removals.has(id))
|
|
45
|
+
continue;
|
|
46
|
+
const found = locate(profile, job.old, [id], used);
|
|
47
|
+
if (!found) {
|
|
48
|
+
skipped++;
|
|
49
|
+
continue;
|
|
50
|
+
}
|
|
51
|
+
profile[found.section].splice(found.indices[0], 1);
|
|
52
|
+
used.add(id);
|
|
53
|
+
applied++;
|
|
54
|
+
}
|
|
55
|
+
return { profile, applied, skipped };
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Finds each id's item from `old` in `profile`, at a distinct position, all in one
|
|
59
|
+
* section. Returns null when an item is gone or an earlier change already used it.
|
|
60
|
+
*/
|
|
61
|
+
function locate(profile, old, ids, used) {
|
|
62
|
+
const section = sectionOf(ids[0]);
|
|
63
|
+
if (!section)
|
|
64
|
+
return null;
|
|
65
|
+
const items = profile[section];
|
|
66
|
+
const indices = [];
|
|
67
|
+
for (const id of ids) {
|
|
68
|
+
if (used.has(id) || sectionOf(id) !== section)
|
|
69
|
+
return null;
|
|
70
|
+
const source = old[section][positionOf(id)];
|
|
71
|
+
if (!source)
|
|
72
|
+
return null;
|
|
73
|
+
const key = profileItemIdentityKey(source);
|
|
74
|
+
const index = items.findIndex((item, i) => !indices.includes(i) && profileItemIdentityKey(item) === key);
|
|
75
|
+
if (index < 0)
|
|
76
|
+
return null;
|
|
77
|
+
indices.push(index);
|
|
78
|
+
}
|
|
79
|
+
return { section, indices };
|
|
80
|
+
}
|
|
81
|
+
function sectionOf(id) {
|
|
82
|
+
return SECTION_BY_PREFIX[id.split("_")[0] ?? ""];
|
|
83
|
+
}
|
|
84
|
+
function positionOf(id) {
|
|
85
|
+
return Number.parseInt(id.split("_")[1] ?? "", 10);
|
|
86
|
+
}
|
|
87
|
+
/** The fields that identify a profile item. Counters and vectors are left out. */
|
|
88
|
+
export function profileItemIdentityKey(item) {
|
|
89
|
+
return JSON.stringify({
|
|
90
|
+
category: item.category ?? null,
|
|
91
|
+
description: item.description ?? null,
|
|
92
|
+
steps: Array.isArray(item.steps) ? item.steps : null,
|
|
93
|
+
});
|
|
94
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import type { UserProfileData } from "./types.js";
|
|
2
|
+
import { type CleanupDiff, type IndexedProfile } from "./ai-cleanup.js";
|
|
3
|
+
/** What one batch adds to the review: its changes, and the merged item of each merge. */
|
|
4
|
+
export interface BatchCleanupResult {
|
|
5
|
+
diff: CleanupDiff;
|
|
6
|
+
mergedItems: Record<string, unknown>;
|
|
7
|
+
}
|
|
8
|
+
export interface BatchRunResult extends BatchCleanupResult {
|
|
9
|
+
failedBatches: number;
|
|
10
|
+
failedItems: number;
|
|
11
|
+
}
|
|
12
|
+
export interface RunInBatchesOptions {
|
|
13
|
+
callBatch?: (batch: IndexedProfile) => Promise<BatchCleanupResult>;
|
|
14
|
+
onProgress?: (done: number, total: number, failed: number) => void;
|
|
15
|
+
isStopped?: () => boolean;
|
|
16
|
+
}
|
|
17
|
+
/** Indexes the full profile once, so ids are profile positions, and keeps the selected items. */
|
|
18
|
+
export declare function selectForCleanup(profileData: UserProfileData, includeIds: string[]): IndexedProfile;
|
|
19
|
+
/**
|
|
20
|
+
* Packs each section on its own, sorted by category, so a category of up to `size` items
|
|
21
|
+
* stays in one batch. Likely duplicates share a category, and the model merges only
|
|
22
|
+
* inside a section.
|
|
23
|
+
*/
|
|
24
|
+
export declare function splitIntoBatches(indexed: IndexedProfile, size: number): IndexedProfile[];
|
|
25
|
+
/** One model request for one batch. */
|
|
26
|
+
export declare function cleanupBatch(indexed: IndexedProfile): Promise<BatchCleanupResult>;
|
|
27
|
+
/**
|
|
28
|
+
* Runs the batches one after another and joins their results. A failed batch adds
|
|
29
|
+
* nothing, so its items stay as they are. Throws the first error when every batch
|
|
30
|
+
* fails. Returns null when stopped; the stop flag is read before and after each call.
|
|
31
|
+
*/
|
|
32
|
+
export declare function runInBatches(batches: IndexedProfile[], { callBatch, onProgress, isStopped }?: RunInBatchesOptions): Promise<BatchRunResult | null>;
|
|
33
|
+
/** A fixed reason code for the log. Never the error text, which may quote the reply. */
|
|
34
|
+
export declare function failureReason(error: unknown): string;
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
import { log } from "../logger.js";
|
|
2
|
+
import { addIdsToProfile, buildAICleanupPrompt, buildItemIndex, callAICleanup, generateDiff, rebuildProfileUsing, } from "./ai-cleanup.js";
|
|
3
|
+
/**
|
|
4
|
+
* A large selection sent in one model request runs past the request time limit. The web
|
|
5
|
+
* cleanup therefore sends the selection in batches, one request each, one after another.
|
|
6
|
+
*/
|
|
7
|
+
const SECTIONS = ["preferences", "patterns", "workflows"];
|
|
8
|
+
/** Indexes the full profile once, so ids are profile positions, and keeps the selected items. */
|
|
9
|
+
export function selectForCleanup(profileData, includeIds) {
|
|
10
|
+
const idSet = new Set(includeIds);
|
|
11
|
+
const indexed = addIdsToProfile(profileData);
|
|
12
|
+
return {
|
|
13
|
+
preferences: indexed.preferences.filter((p) => idSet.has(p.id)),
|
|
14
|
+
patterns: indexed.patterns.filter((p) => idSet.has(p.id)),
|
|
15
|
+
workflows: indexed.workflows.filter((p) => idSet.has(p.id)),
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Packs each section on its own, sorted by category, so a category of up to `size` items
|
|
20
|
+
* stays in one batch. Likely duplicates share a category, and the model merges only
|
|
21
|
+
* inside a section.
|
|
22
|
+
*/
|
|
23
|
+
export function splitIntoBatches(indexed, size) {
|
|
24
|
+
const batches = [];
|
|
25
|
+
for (const section of SECTIONS) {
|
|
26
|
+
const flush = (items) => {
|
|
27
|
+
if (items.length > 0)
|
|
28
|
+
batches.push({ preferences: [], patterns: [], workflows: [], [section]: items });
|
|
29
|
+
};
|
|
30
|
+
let current = [];
|
|
31
|
+
for (const group of groupByCategory(indexed[section])) {
|
|
32
|
+
if (group.length > size) {
|
|
33
|
+
flush(current);
|
|
34
|
+
current = [];
|
|
35
|
+
for (let i = 0; i < group.length; i += size) {
|
|
36
|
+
const chunk = group.slice(i, i + size);
|
|
37
|
+
if (chunk.length === size)
|
|
38
|
+
flush(chunk);
|
|
39
|
+
else
|
|
40
|
+
current = chunk;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
else if (current.length + group.length > size) {
|
|
44
|
+
flush(current);
|
|
45
|
+
current = [...group];
|
|
46
|
+
}
|
|
47
|
+
else {
|
|
48
|
+
current.push(...group);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
flush(current);
|
|
52
|
+
}
|
|
53
|
+
return batches;
|
|
54
|
+
}
|
|
55
|
+
function groupByCategory(items) {
|
|
56
|
+
const groups = new Map();
|
|
57
|
+
for (const item of items) {
|
|
58
|
+
const key = (item.category ?? "").toLowerCase();
|
|
59
|
+
const group = groups.get(key);
|
|
60
|
+
if (group)
|
|
61
|
+
group.push(item);
|
|
62
|
+
else
|
|
63
|
+
groups.set(key, [item]);
|
|
64
|
+
}
|
|
65
|
+
return [...groups.keys()].sort().map((key) => groups.get(key));
|
|
66
|
+
}
|
|
67
|
+
/** One model request for one batch. */
|
|
68
|
+
export async function cleanupBatch(indexed) {
|
|
69
|
+
const t0 = Date.now();
|
|
70
|
+
const prompt = buildAICleanupPrompt(indexed);
|
|
71
|
+
log("AI cleanup: batch prompt built", { items: countItems(indexed), promptLen: prompt.length });
|
|
72
|
+
const result = await callAICleanup(prompt);
|
|
73
|
+
log("AI cleanup: batch reply received", { callMs: Date.now() - t0 });
|
|
74
|
+
const cleanedById = buildItemIndex(result.profile);
|
|
75
|
+
const originalById = buildItemIndex(indexed);
|
|
76
|
+
const mapping = restrictToBatch(result.mapping, originalById);
|
|
77
|
+
return {
|
|
78
|
+
diff: generateDiff(indexed, mapping, cleanedById),
|
|
79
|
+
mergedItems: buildMergedItems(mapping, cleanedById, originalById),
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
// The model may return an id that is not in the batch. Apply looks ids up in the whole
|
|
83
|
+
// profile, so such an id could change an item the user never selected.
|
|
84
|
+
function restrictToBatch(mapping, originalById) {
|
|
85
|
+
const inBatch = (id) => originalById.has(id);
|
|
86
|
+
return {
|
|
87
|
+
kept: mapping.kept.filter(inBatch),
|
|
88
|
+
merged: mapping.merged.map((group) => group.filter(inBatch)).filter((g) => g.length > 1),
|
|
89
|
+
removed: mapping.removed.filter(inBatch),
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
// The merged item of each group, keyed by its target id, so Apply can put a ticked merge
|
|
93
|
+
// into a newer profile without the rest of the cleaned batch.
|
|
94
|
+
function buildMergedItems(mapping, cleanedById, originalById) {
|
|
95
|
+
const items = {};
|
|
96
|
+
for (const group of mapping.merged.filter((g) => g.length > 1)) {
|
|
97
|
+
const originals = new Map();
|
|
98
|
+
for (const id of group) {
|
|
99
|
+
const original = originalById.get(id);
|
|
100
|
+
if (original)
|
|
101
|
+
originals.set(id, original);
|
|
102
|
+
}
|
|
103
|
+
const one = rebuildProfileUsing({ kept: [], merged: [group], removed: [] }, cleanedById, originals);
|
|
104
|
+
const item = one.preferences[0] ?? one.patterns[0] ?? one.workflows[0];
|
|
105
|
+
if (item)
|
|
106
|
+
items[group[0]] = item;
|
|
107
|
+
}
|
|
108
|
+
return items;
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* Runs the batches one after another and joins their results. A failed batch adds
|
|
112
|
+
* nothing, so its items stay as they are. Throws the first error when every batch
|
|
113
|
+
* fails. Returns null when stopped; the stop flag is read before and after each call.
|
|
114
|
+
*/
|
|
115
|
+
export async function runInBatches(batches, { callBatch = cleanupBatch, onProgress, isStopped = () => false } = {}) {
|
|
116
|
+
const t0 = Date.now();
|
|
117
|
+
const joined = {
|
|
118
|
+
diff: { kept: [], merged: [], removed: [] },
|
|
119
|
+
mergedItems: {},
|
|
120
|
+
failedBatches: 0,
|
|
121
|
+
failedItems: 0,
|
|
122
|
+
};
|
|
123
|
+
let firstError;
|
|
124
|
+
for (const [index, batch] of batches.entries()) {
|
|
125
|
+
if (isStopped())
|
|
126
|
+
return null;
|
|
127
|
+
try {
|
|
128
|
+
const result = await callBatch(batch);
|
|
129
|
+
if (isStopped())
|
|
130
|
+
return null;
|
|
131
|
+
joined.diff.kept.push(...result.diff.kept);
|
|
132
|
+
joined.diff.merged.push(...result.diff.merged);
|
|
133
|
+
joined.diff.removed.push(...result.diff.removed);
|
|
134
|
+
Object.assign(joined.mergedItems, result.mergedItems);
|
|
135
|
+
}
|
|
136
|
+
catch (error) {
|
|
137
|
+
if (isStopped())
|
|
138
|
+
return null;
|
|
139
|
+
if (joined.failedBatches === 0)
|
|
140
|
+
firstError = error;
|
|
141
|
+
joined.failedBatches++;
|
|
142
|
+
joined.failedItems += countItems(batch);
|
|
143
|
+
log("AI cleanup: batch failed", {
|
|
144
|
+
batch: index + 1,
|
|
145
|
+
of: batches.length,
|
|
146
|
+
items: countItems(batch),
|
|
147
|
+
reason: failureReason(error),
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
onProgress?.(index + 1, batches.length, joined.failedBatches);
|
|
151
|
+
}
|
|
152
|
+
if (batches.length > 0 && joined.failedBatches === batches.length)
|
|
153
|
+
throw firstError;
|
|
154
|
+
log("AI cleanup: batches finished", {
|
|
155
|
+
batches: batches.length,
|
|
156
|
+
failedBatches: joined.failedBatches,
|
|
157
|
+
totalMs: Date.now() - t0,
|
|
158
|
+
});
|
|
159
|
+
return joined;
|
|
160
|
+
}
|
|
161
|
+
/** A fixed reason code for the log. Never the error text, which may quote the reply. */
|
|
162
|
+
export function failureReason(error) {
|
|
163
|
+
if (error instanceof Error) {
|
|
164
|
+
if (error.name === "TimeoutError" || error.name === "AbortError")
|
|
165
|
+
return "timeout";
|
|
166
|
+
const status = /API request failed: (\d{3})/.exec(error.message)?.[1];
|
|
167
|
+
if (status)
|
|
168
|
+
return `http-${status}`;
|
|
169
|
+
if (error instanceof SyntaxError || /valid JSON|No content/.test(error.message)) {
|
|
170
|
+
return "invalid-reply";
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
return "error";
|
|
174
|
+
}
|
|
175
|
+
function countItems(indexed) {
|
|
176
|
+
return indexed.preferences.length + indexed.patterns.length + indexed.workflows.length;
|
|
177
|
+
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import type { CleanupDiff } from "./ai-cleanup.js";
|
|
2
|
+
import type { CleanupJobResult } from "./ai-cleanup-apply.js";
|
|
3
|
+
import { type RunInBatchesOptions } from "./ai-cleanup-batches.js";
|
|
4
|
+
import type { UserProfileData } from "./types.js";
|
|
5
|
+
/**
|
|
6
|
+
* A large AI cleanup takes many minutes, longer than a web request may stay open. The web
|
|
7
|
+
* app therefore runs it as a background job per user, and the dialog polls its state.
|
|
8
|
+
* The job also holds the finished result until Apply. Jobs live in this process only; a
|
|
9
|
+
* restart drops them, and nothing is saved before Apply.
|
|
10
|
+
*/
|
|
11
|
+
export type AiCleanupJobStatus = {
|
|
12
|
+
state: "running";
|
|
13
|
+
done: number;
|
|
14
|
+
total: number;
|
|
15
|
+
failed: number;
|
|
16
|
+
} | {
|
|
17
|
+
state: "done";
|
|
18
|
+
result: {
|
|
19
|
+
old: UserProfileData;
|
|
20
|
+
changes: CleanupDiff;
|
|
21
|
+
failedBatches: number;
|
|
22
|
+
failedItems: number;
|
|
23
|
+
};
|
|
24
|
+
} | {
|
|
25
|
+
state: "failed";
|
|
26
|
+
error: string;
|
|
27
|
+
};
|
|
28
|
+
export interface AiCleanupJobView {
|
|
29
|
+
id: string;
|
|
30
|
+
startedAt: number;
|
|
31
|
+
selection: string[];
|
|
32
|
+
status: AiCleanupJobStatus;
|
|
33
|
+
}
|
|
34
|
+
export interface AiCleanupJobInput {
|
|
35
|
+
/** The profile as the dialog lists it, so item ids match. */
|
|
36
|
+
profileData: UserProfileData;
|
|
37
|
+
/** An empty or missing selection means every item. */
|
|
38
|
+
includeIds?: string[];
|
|
39
|
+
}
|
|
40
|
+
export interface AiCleanupJobDeps {
|
|
41
|
+
callBatch?: RunInBatchesOptions["callBatch"];
|
|
42
|
+
batchSize?: number;
|
|
43
|
+
now?: () => number;
|
|
44
|
+
}
|
|
45
|
+
/** Starts a cleanup job for the user, or returns the one already running. */
|
|
46
|
+
export declare function startAiCleanupJob(userId: string, input: AiCleanupJobInput, deps?: AiCleanupJobDeps): AiCleanupJobView;
|
|
47
|
+
/** The user's job, or null when none was started or a finished one expired. */
|
|
48
|
+
export declare function getAiCleanupJob(userId: string, now?: number): AiCleanupJobView | null;
|
|
49
|
+
/** The finished result that Apply needs, or null when the job is not done. */
|
|
50
|
+
export declare function getAiCleanupResult(userId: string, now?: number): CleanupJobResult | null;
|
|
51
|
+
/** Stops a running job, or discards a finished one. A stopped job writes nothing. */
|
|
52
|
+
export declare function stopAiCleanupJob(userId: string): void;
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
import { log } from "../logger.js";
|
|
2
|
+
import { AI_CLEANUP_BATCH_SIZE } from "../request-timeouts.js";
|
|
3
|
+
import { failureReason, runInBatches, selectForCleanup, splitIntoBatches, } from "./ai-cleanup-batches.js";
|
|
4
|
+
/** A finished job is forgotten this long after it finished, as the old pending result was. */
|
|
5
|
+
const FINISHED_JOB_TTL_MS = 30 * 60 * 1000;
|
|
6
|
+
const jobs = new Map();
|
|
7
|
+
let nextId = 1;
|
|
8
|
+
/** Starts a cleanup job for the user, or returns the one already running. */
|
|
9
|
+
export function startAiCleanupJob(userId, input, deps = {}) {
|
|
10
|
+
const current = jobs.get(userId);
|
|
11
|
+
if (current?.status.state === "running")
|
|
12
|
+
return view(current);
|
|
13
|
+
const now = deps.now ?? Date.now;
|
|
14
|
+
const selection = input.includeIds?.length ? input.includeIds : allIds(input.profileData);
|
|
15
|
+
const batches = splitIntoBatches(selectForCleanup(input.profileData, selection), deps.batchSize ?? AI_CLEANUP_BATCH_SIZE);
|
|
16
|
+
const job = {
|
|
17
|
+
id: String(nextId++),
|
|
18
|
+
startedAt: now(),
|
|
19
|
+
selection,
|
|
20
|
+
status: { state: "running", done: 0, total: batches.length, failed: 0 },
|
|
21
|
+
stopped: false,
|
|
22
|
+
mergedItems: {},
|
|
23
|
+
};
|
|
24
|
+
jobs.set(userId, job);
|
|
25
|
+
void (async () => {
|
|
26
|
+
try {
|
|
27
|
+
const result = await runInBatches(batches, {
|
|
28
|
+
callBatch: deps.callBatch,
|
|
29
|
+
isStopped: () => job.stopped,
|
|
30
|
+
onProgress: (done, total, failed) => {
|
|
31
|
+
job.status = { state: "running", done, total, failed };
|
|
32
|
+
},
|
|
33
|
+
});
|
|
34
|
+
if (!result)
|
|
35
|
+
return;
|
|
36
|
+
job.mergedItems = result.mergedItems;
|
|
37
|
+
job.status = {
|
|
38
|
+
state: "done",
|
|
39
|
+
result: {
|
|
40
|
+
old: input.profileData,
|
|
41
|
+
changes: result.diff,
|
|
42
|
+
failedBatches: result.failedBatches,
|
|
43
|
+
failedItems: result.failedItems,
|
|
44
|
+
},
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
catch (error) {
|
|
48
|
+
if (job.stopped)
|
|
49
|
+
return;
|
|
50
|
+
const reason = failureReason(error);
|
|
51
|
+
log("AI cleanup job: failed", { reason });
|
|
52
|
+
job.status = {
|
|
53
|
+
state: "failed",
|
|
54
|
+
error: reason === "timeout"
|
|
55
|
+
? "The model did not answer in time."
|
|
56
|
+
: error instanceof Error
|
|
57
|
+
? error.message
|
|
58
|
+
: String(error),
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
finally {
|
|
62
|
+
job.finishedAt = now();
|
|
63
|
+
}
|
|
64
|
+
})();
|
|
65
|
+
return view(job);
|
|
66
|
+
}
|
|
67
|
+
/** The user's job, or null when none was started or a finished one expired. */
|
|
68
|
+
export function getAiCleanupJob(userId, now = Date.now()) {
|
|
69
|
+
const job = liveJob(userId, now);
|
|
70
|
+
return job ? view(job) : null;
|
|
71
|
+
}
|
|
72
|
+
/** The finished result that Apply needs, or null when the job is not done. */
|
|
73
|
+
export function getAiCleanupResult(userId, now = Date.now()) {
|
|
74
|
+
const job = liveJob(userId, now);
|
|
75
|
+
if (job?.status.state !== "done")
|
|
76
|
+
return null;
|
|
77
|
+
return {
|
|
78
|
+
old: job.status.result.old,
|
|
79
|
+
changes: job.status.result.changes,
|
|
80
|
+
mergedItems: job.mergedItems,
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
/** Stops a running job, or discards a finished one. A stopped job writes nothing. */
|
|
84
|
+
export function stopAiCleanupJob(userId) {
|
|
85
|
+
const job = jobs.get(userId);
|
|
86
|
+
if (!job)
|
|
87
|
+
return;
|
|
88
|
+
job.stopped = true;
|
|
89
|
+
jobs.delete(userId);
|
|
90
|
+
}
|
|
91
|
+
function liveJob(userId, now) {
|
|
92
|
+
const job = jobs.get(userId);
|
|
93
|
+
if (!job)
|
|
94
|
+
return null;
|
|
95
|
+
if (job.status.state !== "running" && job.finishedAt !== undefined) {
|
|
96
|
+
if (now - job.finishedAt > FINISHED_JOB_TTL_MS) {
|
|
97
|
+
jobs.delete(userId);
|
|
98
|
+
return null;
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
return job;
|
|
102
|
+
}
|
|
103
|
+
function view(job) {
|
|
104
|
+
return { id: job.id, startedAt: job.startedAt, selection: job.selection, status: job.status };
|
|
105
|
+
}
|
|
106
|
+
function allIds(profile) {
|
|
107
|
+
return [
|
|
108
|
+
...profile.preferences.map((_, i) => `pref_${i}`),
|
|
109
|
+
...profile.patterns.map((_, i) => `pat_${i}`),
|
|
110
|
+
...profile.workflows.map((_, i) => `wf_${i}`),
|
|
111
|
+
];
|
|
112
|
+
}
|
|
@@ -15,9 +15,7 @@ export interface CleanupDiff {
|
|
|
15
15
|
}>;
|
|
16
16
|
}
|
|
17
17
|
export declare function aiCleanupProfile(profileData: UserProfileData): Promise<AICleanupResult>;
|
|
18
|
-
export
|
|
19
|
-
export declare function filterProfileForCleanup(profileData: UserProfileData, includeIds: string[]): IndexedProfile;
|
|
20
|
-
interface IndexedProfileItem {
|
|
18
|
+
export interface IndexedProfileItem {
|
|
21
19
|
id: string;
|
|
22
20
|
category?: string;
|
|
23
21
|
description: string;
|
|
@@ -25,18 +23,25 @@ interface IndexedProfileItem {
|
|
|
25
23
|
frequency?: number;
|
|
26
24
|
[key: string]: unknown;
|
|
27
25
|
}
|
|
28
|
-
interface IndexedProfile {
|
|
26
|
+
export interface IndexedProfile {
|
|
29
27
|
preferences: IndexedProfileItem[];
|
|
30
28
|
patterns: IndexedProfileItem[];
|
|
31
29
|
workflows: IndexedProfileItem[];
|
|
32
30
|
}
|
|
33
|
-
interface AIMapping {
|
|
31
|
+
export interface AIMapping {
|
|
34
32
|
kept: string[];
|
|
35
33
|
merged: string[][];
|
|
36
34
|
removed: string[];
|
|
37
35
|
}
|
|
36
|
+
export declare function addIdsToProfile(profile: UserProfileData): IndexedProfile;
|
|
37
|
+
export declare function buildAICleanupPrompt(profile: IndexedProfile): string;
|
|
38
|
+
export declare function callAICleanup(prompt: string): Promise<{
|
|
39
|
+
profile: IndexedProfile;
|
|
40
|
+
mapping: AIMapping;
|
|
41
|
+
}>;
|
|
42
|
+
export declare function buildItemIndex(profile: IndexedProfile): Map<string, IndexedProfileItem>;
|
|
38
43
|
export declare function rebuildProfileUsing(mapping: AIMapping, cleanedById: Map<string, IndexedProfileItem>, originalById: Map<string, IndexedProfileItem>, counters?: {
|
|
39
44
|
cleaned: number;
|
|
40
45
|
original: number;
|
|
41
46
|
}): UserProfileData;
|
|
42
|
-
export
|
|
47
|
+
export declare function generateDiff(original: IndexedProfile, mapping: AIMapping, cleanedById: Map<string, IndexedProfileItem>): CleanupDiff;
|
|
@@ -43,41 +43,6 @@ export async function aiCleanupProfile(profileData) {
|
|
|
43
43
|
});
|
|
44
44
|
return { cleaned, diff };
|
|
45
45
|
}
|
|
46
|
-
export async function aiCleanupProfileFromIndexed(indexed) {
|
|
47
|
-
const t0 = Date.now();
|
|
48
|
-
const prompt = buildAICleanupPrompt(indexed);
|
|
49
|
-
log("AI cleanup: prompt built (filtered)", {
|
|
50
|
-
prefCount: indexed.preferences.length,
|
|
51
|
-
patCount: indexed.patterns.length,
|
|
52
|
-
wfCount: indexed.workflows.length,
|
|
53
|
-
promptLen: prompt.length,
|
|
54
|
-
buildMs: Date.now() - t0,
|
|
55
|
-
});
|
|
56
|
-
const aiStart = Date.now();
|
|
57
|
-
const result = await callAICleanup(prompt);
|
|
58
|
-
log("AI cleanup: AI response received (filtered)", { callMs: Date.now() - aiStart });
|
|
59
|
-
const cleanedById = buildItemIndex(result.profile);
|
|
60
|
-
const originalById = buildItemIndex(indexed);
|
|
61
|
-
const counters = { cleaned: 0, original: 0 };
|
|
62
|
-
const cleaned = rebuildProfileUsing(result.mapping, cleanedById, originalById, counters);
|
|
63
|
-
const diff = generateDiff(indexed, result.mapping, cleanedById);
|
|
64
|
-
log("AI cleanup: complete (filtered)", {
|
|
65
|
-
totalMs: Date.now() - t0,
|
|
66
|
-
kept: diff.kept.length,
|
|
67
|
-
merged: diff.merged.length,
|
|
68
|
-
removed: diff.removed.length,
|
|
69
|
-
});
|
|
70
|
-
return { cleaned, diff };
|
|
71
|
-
}
|
|
72
|
-
export function filterProfileForCleanup(profileData, includeIds) {
|
|
73
|
-
const idSet = new Set(includeIds);
|
|
74
|
-
const indexed = addIdsToProfile(profileData);
|
|
75
|
-
return {
|
|
76
|
-
preferences: indexed.preferences.filter((p) => idSet.has(p.id)),
|
|
77
|
-
patterns: indexed.patterns.filter((p) => idSet.has(p.id)),
|
|
78
|
-
workflows: indexed.workflows.filter((p) => idSet.has(p.id)),
|
|
79
|
-
};
|
|
80
|
-
}
|
|
81
46
|
// The model controls this JSON; it may omit fields or return wrong types. Coerce to a
|
|
82
47
|
// well-formed AIMapping so rebuildProfileUsing/generateDiff never call .map/.filter/.includes
|
|
83
48
|
// on undefined. A missing or malformed mapping degrades to a no-op cleanup (all originals are
|
|
@@ -100,7 +65,7 @@ function normalizeAIMapping(raw) {
|
|
|
100
65
|
const removed = Array.isArray(raw.removed) ? raw.removed.filter(isStr) : [];
|
|
101
66
|
return { kept, merged, removed };
|
|
102
67
|
}
|
|
103
|
-
function addIdsToProfile(profile) {
|
|
68
|
+
export function addIdsToProfile(profile) {
|
|
104
69
|
const items = {
|
|
105
70
|
preferences: profile.preferences.map((p, i) => ({ ...p, id: `pref_${i}` })),
|
|
106
71
|
patterns: profile.patterns.map((p, i) => ({ ...p, id: `pat_${i}` })),
|
|
@@ -108,7 +73,7 @@ function addIdsToProfile(profile) {
|
|
|
108
73
|
};
|
|
109
74
|
return items;
|
|
110
75
|
}
|
|
111
|
-
function buildAICleanupPrompt(profile) {
|
|
76
|
+
export function buildAICleanupPrompt(profile) {
|
|
112
77
|
const profileJSON = JSON.stringify({
|
|
113
78
|
preferences: profile.preferences.map(formatForAI),
|
|
114
79
|
patterns: profile.patterns.map(formatForAI),
|
|
@@ -147,7 +112,7 @@ function formatForAI(item) {
|
|
|
147
112
|
const { id, category, description, frequency } = item;
|
|
148
113
|
return { id, category, description, frequency };
|
|
149
114
|
}
|
|
150
|
-
async function callAICleanup(prompt) {
|
|
115
|
+
export async function callAICleanup(prompt) {
|
|
151
116
|
// Use the host model when the host registered one (same pattern as auto-capture).
|
|
152
117
|
// When it is available, surface its errors instead of masking them as
|
|
153
118
|
// "No AI provider configured" via a silent fallback (#177).
|
|
@@ -215,7 +180,7 @@ async function callViaHostModel(model, prompt) {
|
|
|
215
180
|
mapping: normalizeAIMapping(parsed.mapping),
|
|
216
181
|
};
|
|
217
182
|
}
|
|
218
|
-
function buildItemIndex(profile) {
|
|
183
|
+
export function buildItemIndex(profile) {
|
|
219
184
|
const map = new Map();
|
|
220
185
|
for (const item of profile.preferences)
|
|
221
186
|
map.set(item.id, item);
|
|
@@ -393,7 +358,7 @@ export function rebuildProfileUsing(mapping, cleanedById, originalById, counters
|
|
|
393
358
|
}
|
|
394
359
|
return result;
|
|
395
360
|
}
|
|
396
|
-
function generateDiff(original, mapping, cleanedById) {
|
|
361
|
+
export function generateDiff(original, mapping, cleanedById) {
|
|
397
362
|
const index = buildItemIndex(original);
|
|
398
363
|
const diff = {
|
|
399
364
|
kept: mapping.kept.map((id) => index.get(id)?.description || id),
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type { UserProfile, UserProfileData } from "./types.js";
|
|
2
|
+
/** Learning makes this many save attempts before it leaves the prompts for a later run. */
|
|
3
|
+
export declare const MAX_PROFILE_SAVE_ATTEMPTS = 3;
|
|
4
|
+
/** The profile store operations that a save needs. */
|
|
5
|
+
export interface ProfileSaveStore {
|
|
6
|
+
getProfileById(id: string): Promise<UserProfile | null>;
|
|
7
|
+
mergeProfileData(existing: UserProfileData, updates: Partial<UserProfileData>, embedService?: undefined, profileId?: string): Promise<UserProfileData>;
|
|
8
|
+
updateProfile(profileId: string, profileData: UserProfileData, additionalPromptsAnalyzed: number, changeSummary: string, expectedVersion: number): Promise<boolean>;
|
|
9
|
+
}
|
|
10
|
+
type ReadProfile = Pick<UserProfile, "id" | "version">;
|
|
11
|
+
/**
|
|
12
|
+
* Saves `data` over the profile as the caller read it. Returns false, and changes nothing,
|
|
13
|
+
* when another save landed since. The log holds a code only, never profile content.
|
|
14
|
+
*/
|
|
15
|
+
export declare function saveProfile(profile: ReadProfile, data: UserProfileData, summary: string, promptsAnalyzed?: number, store?: Pick<ProfileSaveStore, "updateProfile">): Promise<boolean>;
|
|
16
|
+
/**
|
|
17
|
+
* Saves the merged result of a profile analysis. When another save landed first, it loads
|
|
18
|
+
* the newer profile, merges the same model result into it, and saves again. It never calls
|
|
19
|
+
* the model. Returns false when there is nothing to save or every attempt met a newer version.
|
|
20
|
+
*/
|
|
21
|
+
export declare function saveMergedProfile(store: ProfileSaveStore, profile: ReadProfile, analysis: {
|
|
22
|
+
raw: Partial<UserProfileData>;
|
|
23
|
+
merged: UserProfileData | null;
|
|
24
|
+
}, promptsAnalyzed: number, summary: string): Promise<boolean>;
|
|
25
|
+
export {};
|