@xopcai/xopc 0.0.25 → 0.0.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/extensions/telegram/xopc.extension.json +1 -1
- package/dist/gateway/static/root/assets/agents-w8_jzuiX.js +216 -0
- package/dist/gateway/static/root/assets/agents-w8_jzuiX.js.map +1 -0
- package/dist/gateway/static/root/assets/{apps-page-DbzO48lg.js → apps-page-CBBh_Ww8.js} +2 -2
- package/dist/gateway/static/root/assets/{apps-page-DbzO48lg.js.map → apps-page-CBBh_Ww8.js.map} +1 -1
- package/dist/gateway/static/root/assets/channels-settings-DUKRPC7C.js +9 -0
- package/dist/gateway/static/root/assets/{channels-settings-CeGoU9v8.js.map → channels-settings-DUKRPC7C.js.map} +1 -1
- package/dist/gateway/static/root/assets/cron-page-S18t1yG-.js +2 -0
- package/dist/gateway/static/root/assets/{cron-page-DpEYUvxB.js.map → cron-page-S18t1yG-.js.map} +1 -1
- package/dist/gateway/static/root/assets/{cron-utils-Cvv0F3pa.js → cron-utils-08gdQfl9.js} +2 -2
- package/dist/gateway/static/root/assets/{cron-utils-Cvv0F3pa.js.map → cron-utils-08gdQfl9.js.map} +1 -1
- package/dist/gateway/static/root/assets/dist-C1MrygQH.js +2 -0
- package/dist/gateway/static/root/assets/{dist-C41N3YrO.js.map → dist-C1MrygQH.js.map} +1 -1
- package/dist/gateway/static/root/assets/{extension-debug-page-CkkYZjNP.js → extension-debug-page-DN3HKUGS.js} +2 -2
- package/dist/gateway/static/root/assets/{extension-debug-page-CkkYZjNP.js.map → extension-debug-page-DN3HKUGS.js.map} +1 -1
- package/dist/gateway/static/root/assets/{extension-page-BjUIPVNG.js → extension-page-CoFDHZtZ.js} +2 -2
- package/dist/gateway/static/root/assets/{extension-page-BjUIPVNG.js.map → extension-page-CoFDHZtZ.js.map} +1 -1
- package/dist/gateway/static/root/assets/{extension-settings-page-CwuFDOdk.js → extension-settings-page-BcPCu_Go.js} +2 -2
- package/dist/gateway/static/root/assets/{extension-settings-page-CwuFDOdk.js.map → extension-settings-page-BcPCu_Go.js.map} +1 -1
- package/dist/gateway/static/root/assets/index-OT4cGzon.css +1 -0
- package/dist/gateway/static/root/assets/index-PfkB8N37.js +4734 -0
- package/dist/gateway/static/root/assets/index-PfkB8N37.js.map +1 -0
- package/dist/gateway/static/root/assets/logs-page-DoWe1GWy.js +2 -0
- package/dist/gateway/static/root/assets/{logs-page-BtwGPuw2.js.map → logs-page-DoWe1GWy.js.map} +1 -1
- package/dist/gateway/static/root/assets/sessions-page-2uOYwEwd.js +2 -0
- package/dist/gateway/static/root/assets/{sessions-page-4rKFDn2k.js.map → sessions-page-2uOYwEwd.js.map} +1 -1
- package/dist/gateway/static/root/assets/settings-page-fQWswCuq.js +2 -0
- package/dist/gateway/static/root/assets/settings-page-fQWswCuq.js.map +1 -0
- package/dist/gateway/static/root/assets/skills-page-BmBDCEbY.js +3 -0
- package/dist/gateway/static/root/assets/{skills-page-_siDuHeF.js.map → skills-page-BmBDCEbY.js.map} +1 -1
- package/dist/gateway/static/root/index.html +2 -2
- package/dist/package.js +1 -1
- package/dist/src/agent/memory/dreaming/config.d.ts +37 -9
- package/dist/src/agent/memory/dreaming/config.js +60 -18
- package/dist/src/agent/memory/dreaming/config.js.map +1 -1
- package/dist/src/agent/memory/dreaming/constants.d.ts +22 -1
- package/dist/src/agent/memory/dreaming/constants.js +26 -2
- package/dist/src/agent/memory/dreaming/constants.js.map +1 -1
- package/dist/src/agent/memory/dreaming/deep-promotion.d.ts +5 -6
- package/dist/src/agent/memory/dreaming/deep-promotion.js +90 -156
- package/dist/src/agent/memory/dreaming/deep-promotion.js.map +1 -1
- package/dist/src/agent/memory/dreaming/events.d.ts +36 -0
- package/dist/src/agent/memory/dreaming/events.js +44 -0
- package/dist/src/agent/memory/dreaming/events.js.map +1 -0
- package/dist/src/agent/memory/dreaming/last-run.d.ts +80 -0
- package/dist/src/agent/memory/dreaming/last-run.js +98 -0
- package/dist/src/agent/memory/dreaming/last-run.js.map +1 -0
- package/dist/src/agent/memory/dreaming/light-sweep.d.ts +19 -0
- package/dist/src/agent/memory/dreaming/light-sweep.js +328 -0
- package/dist/src/agent/memory/dreaming/light-sweep.js.map +1 -0
- package/dist/src/agent/memory/dreaming/preview.d.ts +3 -1
- package/dist/src/agent/memory/dreaming/preview.js +11 -90
- package/dist/src/agent/memory/dreaming/preview.js.map +1 -1
- package/dist/src/agent/memory/dreaming/rem-patterns.d.ts +21 -0
- package/dist/src/agent/memory/dreaming/rem-patterns.js +286 -0
- package/dist/src/agent/memory/dreaming/rem-patterns.js.map +1 -0
- package/dist/src/agent/memory/dreaming/short-term-store.d.ts +20 -0
- package/dist/src/agent/memory/dreaming/short-term-store.js +25 -15
- package/dist/src/agent/memory/dreaming/short-term-store.js.map +1 -1
- package/dist/src/agent/memory/dreaming/utils.d.ts +42 -0
- package/dist/src/agent/memory/dreaming/utils.js +141 -0
- package/dist/src/agent/memory/dreaming/utils.js.map +1 -0
- package/dist/src/agent/orchestration/agent-orchestrator.js +54 -12
- package/dist/src/agent/orchestration/agent-orchestrator.js.map +1 -1
- package/dist/src/agent/service.js +54 -28
- package/dist/src/agent/service.js.map +1 -1
- package/dist/src/config/schema.d.ts +54 -0
- package/dist/src/config/schema.js +34 -8
- package/dist/src/config/schema.js.map +1 -1
- package/dist/src/gateway/hono/lib/config-payload.d.ts +18 -0
- package/dist/src/gateway/hono/routes/dreaming.js +105 -15
- package/dist/src/gateway/hono/routes/dreaming.js.map +1 -1
- package/dist/src/gateway/hono/routes/models.js +26 -1
- package/dist/src/gateway/hono/routes/models.js.map +1 -1
- package/dist/src/gateway/hono/routes/public-gateway.js +1 -0
- package/dist/src/gateway/hono/routes/public-gateway.js.map +1 -1
- package/package.json +1 -1
- package/dist/gateway/static/root/assets/agents-C_bPhtBs.js +0 -216
- package/dist/gateway/static/root/assets/agents-C_bPhtBs.js.map +0 -1
- package/dist/gateway/static/root/assets/channels-settings-CeGoU9v8.js +0 -9
- package/dist/gateway/static/root/assets/cron-page-DpEYUvxB.js +0 -2
- package/dist/gateway/static/root/assets/dist-C41N3YrO.js +0 -2
- package/dist/gateway/static/root/assets/index-DwzwDCjW.js +0 -150
- package/dist/gateway/static/root/assets/index-DwzwDCjW.js.map +0 -1
- package/dist/gateway/static/root/assets/index-dhtHG1nU.css +0 -1
- package/dist/gateway/static/root/assets/logs-page-BtwGPuw2.js +0 -2
- package/dist/gateway/static/root/assets/sessions-page-4rKFDn2k.js +0 -2
- package/dist/gateway/static/root/assets/settings-page-iYLSxQYc.js +0 -2
- package/dist/gateway/static/root/assets/settings-page-iYLSxQYc.js.map +0 -1
- package/dist/gateway/static/root/assets/skills-page-_siDuHeF.js +0 -3
|
@@ -0,0 +1,328 @@
|
|
|
1
|
+
import { createLogger } from "../../../utils/logger/index.js";
|
|
2
|
+
import { init_logger } from "../../../utils/logger.js";
|
|
3
|
+
import { DREAMING_DIR_RELATIVE } from "./constants.js";
|
|
4
|
+
import { buildEntryKey, isoDay, normalizeMemoryPath, normalizeSnippetForHash, snippetHash } from "./utils.js";
|
|
5
|
+
import { bumpEntryPhaseSignal, loadDreamingStore, saveDreamingStore } from "./short-term-store.js";
|
|
6
|
+
import "./last-run.js";
|
|
7
|
+
import path from "node:path";
|
|
8
|
+
import fs from "node:fs/promises";
|
|
9
|
+
//#region src/agent/memory/dreaming/light-sweep.ts
|
|
10
|
+
init_logger();
|
|
11
|
+
const log = createLogger("Dreaming:Light");
|
|
12
|
+
function isDailyMemoryFile(filename) {
|
|
13
|
+
return /^\d{4}-\d{2}-\d{2}\.md$/i.test(filename);
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Cosine-ish similarity via character n-gram overlap.
|
|
17
|
+
* Fast and deterministic — no embeddings required.
|
|
18
|
+
*/
|
|
19
|
+
function trigramSimilarity(textA, textB) {
|
|
20
|
+
const normalizedA = normalizeSnippetForHash(textA);
|
|
21
|
+
const normalizedB = normalizeSnippetForHash(textB);
|
|
22
|
+
if (!normalizedA || !normalizedB) return 0;
|
|
23
|
+
if (normalizedA === normalizedB) return 1;
|
|
24
|
+
const gramsA = buildTrigramSet(normalizedA);
|
|
25
|
+
const gramsB = buildTrigramSet(normalizedB);
|
|
26
|
+
if (gramsA.size === 0 || gramsB.size === 0) return 0;
|
|
27
|
+
let intersection = 0;
|
|
28
|
+
for (const gram of gramsA) if (gramsB.has(gram)) intersection += 1;
|
|
29
|
+
return intersection / Math.sqrt(gramsA.size * gramsB.size);
|
|
30
|
+
}
|
|
31
|
+
function buildTrigramSet(text) {
|
|
32
|
+
const grams = /* @__PURE__ */ new Set();
|
|
33
|
+
for (let i = 0; i <= text.length - 3; i++) grams.add(text.slice(i, i + 3));
|
|
34
|
+
return grams;
|
|
35
|
+
}
|
|
36
|
+
function resolveConfig(overrides) {
|
|
37
|
+
return {
|
|
38
|
+
enabled: overrides?.enabled === true,
|
|
39
|
+
cron: typeof overrides?.cron === "string" ? overrides.cron : "0 */6 * * *",
|
|
40
|
+
lookbackDays: Math.max(1, Math.floor(Number(overrides?.lookbackDays) || 2)),
|
|
41
|
+
limit: Math.max(0, Math.floor(Number(overrides?.limit) || 100)),
|
|
42
|
+
dedupeSimilarity: Math.max(0, Math.min(1, Number(overrides?.dedupeSimilarity) || .9))
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Scan recent daily memory files (memory/YYYY-MM-DD.md) and collect signal
|
|
47
|
+
* entries into the short-term store. Deduplicates near-identical snippets
|
|
48
|
+
* using trigram similarity.
|
|
49
|
+
*
|
|
50
|
+
* This is the "light sleep" phase: fast, frequent, cheap.
|
|
51
|
+
*/
|
|
52
|
+
async function runLightSweep(params) {
|
|
53
|
+
const cfg = resolveConfig(params.config);
|
|
54
|
+
const now = params.now ?? /* @__PURE__ */ new Date();
|
|
55
|
+
const startedAt = now.toISOString();
|
|
56
|
+
const runId = `light:${startedAt}:${process.pid}`;
|
|
57
|
+
const startMs = Date.now();
|
|
58
|
+
if (!cfg.enabled) {
|
|
59
|
+
await writeLastRun(params.workspaceDir, {
|
|
60
|
+
runId,
|
|
61
|
+
startedAt,
|
|
62
|
+
cfg,
|
|
63
|
+
ok: true,
|
|
64
|
+
reason: "light sweep disabled",
|
|
65
|
+
startMs,
|
|
66
|
+
light: {
|
|
67
|
+
scannedEntries: 0,
|
|
68
|
+
newSignals: 0,
|
|
69
|
+
deduped: 0
|
|
70
|
+
}
|
|
71
|
+
});
|
|
72
|
+
return {
|
|
73
|
+
ok: true,
|
|
74
|
+
reason: "light sweep disabled",
|
|
75
|
+
scannedEntries: 0,
|
|
76
|
+
newSignals: 0,
|
|
77
|
+
deduped: 0
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
try {
|
|
81
|
+
const recentFiles = await listRecentDailyFiles(path.join(params.workspaceDir, "memory"), cfg.lookbackDays, now);
|
|
82
|
+
if (recentFiles.length === 0) {
|
|
83
|
+
await writeLastRun(params.workspaceDir, {
|
|
84
|
+
runId,
|
|
85
|
+
startedAt,
|
|
86
|
+
cfg,
|
|
87
|
+
ok: true,
|
|
88
|
+
reason: "no recent daily files",
|
|
89
|
+
startMs,
|
|
90
|
+
light: {
|
|
91
|
+
scannedEntries: 0,
|
|
92
|
+
newSignals: 0,
|
|
93
|
+
deduped: 0
|
|
94
|
+
}
|
|
95
|
+
});
|
|
96
|
+
return {
|
|
97
|
+
ok: true,
|
|
98
|
+
reason: "no recent daily files",
|
|
99
|
+
scannedEntries: 0,
|
|
100
|
+
newSignals: 0,
|
|
101
|
+
deduped: 0
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
const allChunks = [];
|
|
105
|
+
for (const filePath of recentFiles) {
|
|
106
|
+
const chunks = await parseFileChunks(filePath, params.workspaceDir);
|
|
107
|
+
allChunks.push(...chunks);
|
|
108
|
+
}
|
|
109
|
+
if (allChunks.length === 0) {
|
|
110
|
+
await writeLastRun(params.workspaceDir, {
|
|
111
|
+
runId,
|
|
112
|
+
startedAt,
|
|
113
|
+
cfg,
|
|
114
|
+
ok: true,
|
|
115
|
+
reason: "no chunks found in daily files",
|
|
116
|
+
startMs,
|
|
117
|
+
light: {
|
|
118
|
+
scannedEntries: 0,
|
|
119
|
+
newSignals: 0,
|
|
120
|
+
deduped: 0
|
|
121
|
+
}
|
|
122
|
+
});
|
|
123
|
+
return {
|
|
124
|
+
ok: true,
|
|
125
|
+
reason: "no chunks found in daily files",
|
|
126
|
+
scannedEntries: 0,
|
|
127
|
+
newSignals: 0,
|
|
128
|
+
deduped: 0
|
|
129
|
+
};
|
|
130
|
+
}
|
|
131
|
+
const { unique, deduped } = deduplicateChunks(allChunks, cfg.dedupeSimilarity);
|
|
132
|
+
const capped = unique.slice(0, cfg.limit);
|
|
133
|
+
const { store } = await loadDreamingStore({ workspaceDir: params.workspaceDir });
|
|
134
|
+
let newSignals = 0;
|
|
135
|
+
const dayBucket = isoDay(now);
|
|
136
|
+
const nowIso = now.toISOString();
|
|
137
|
+
for (const chunk of capped) {
|
|
138
|
+
const key = buildEntryKey({
|
|
139
|
+
path: chunk.path,
|
|
140
|
+
startLine: chunk.startLine,
|
|
141
|
+
endLine: chunk.endLine
|
|
142
|
+
});
|
|
143
|
+
const existing = store.entries[key];
|
|
144
|
+
if (existing) {
|
|
145
|
+
bumpEntryPhaseSignal(existing, "lightHits");
|
|
146
|
+
bumpEntryPhaseSignal(existing, "dailyCount");
|
|
147
|
+
existing.snippet = chunk.text.slice(0, 360);
|
|
148
|
+
existing.lastRecalledAt = nowIso;
|
|
149
|
+
} else {
|
|
150
|
+
store.entries[key] = {
|
|
151
|
+
key,
|
|
152
|
+
path: chunk.path,
|
|
153
|
+
startLine: chunk.startLine,
|
|
154
|
+
endLine: chunk.endLine,
|
|
155
|
+
snippet: chunk.text.slice(0, 360),
|
|
156
|
+
recallCount: 0,
|
|
157
|
+
dailyCount: 1,
|
|
158
|
+
groundedCount: 0,
|
|
159
|
+
lightHits: 1,
|
|
160
|
+
remHits: 0,
|
|
161
|
+
phaseHitCount: 1,
|
|
162
|
+
totalSignalCount: 1,
|
|
163
|
+
totalScore: 0,
|
|
164
|
+
maxScore: 0,
|
|
165
|
+
queryHashes: [],
|
|
166
|
+
recallDays: [dayBucket],
|
|
167
|
+
firstRecalledAt: nowIso,
|
|
168
|
+
lastRecalledAt: nowIso
|
|
169
|
+
};
|
|
170
|
+
newSignals += 1;
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
store.updatedAt = nowIso;
|
|
174
|
+
await saveDreamingStore({
|
|
175
|
+
workspaceDir: params.workspaceDir,
|
|
176
|
+
store
|
|
177
|
+
});
|
|
178
|
+
log.info({
|
|
179
|
+
workspaceDir: params.workspaceDir,
|
|
180
|
+
scanned: allChunks.length,
|
|
181
|
+
newSignals,
|
|
182
|
+
deduped
|
|
183
|
+
}, "Light sweep complete");
|
|
184
|
+
await writeLastRun(params.workspaceDir, {
|
|
185
|
+
runId,
|
|
186
|
+
startedAt,
|
|
187
|
+
cfg,
|
|
188
|
+
ok: true,
|
|
189
|
+
reason: "light sweep complete",
|
|
190
|
+
startMs,
|
|
191
|
+
light: {
|
|
192
|
+
scannedEntries: allChunks.length,
|
|
193
|
+
newSignals,
|
|
194
|
+
deduped
|
|
195
|
+
}
|
|
196
|
+
});
|
|
197
|
+
return {
|
|
198
|
+
ok: true,
|
|
199
|
+
reason: "light sweep complete",
|
|
200
|
+
scannedEntries: allChunks.length,
|
|
201
|
+
newSignals,
|
|
202
|
+
deduped
|
|
203
|
+
};
|
|
204
|
+
} catch (err) {
|
|
205
|
+
const errorMessage = err instanceof Error ? err.message : String(err);
|
|
206
|
+
log.error({
|
|
207
|
+
err,
|
|
208
|
+
errorMessage,
|
|
209
|
+
workspaceDir: params.workspaceDir
|
|
210
|
+
}, `Light sweep failed: ${errorMessage}`);
|
|
211
|
+
await writeLastRun(params.workspaceDir, {
|
|
212
|
+
runId,
|
|
213
|
+
startedAt,
|
|
214
|
+
cfg,
|
|
215
|
+
ok: false,
|
|
216
|
+
reason: `light sweep error: ${errorMessage}`,
|
|
217
|
+
startMs,
|
|
218
|
+
light: {
|
|
219
|
+
scannedEntries: 0,
|
|
220
|
+
newSignals: 0,
|
|
221
|
+
deduped: 0
|
|
222
|
+
},
|
|
223
|
+
errorMessage
|
|
224
|
+
}).catch(() => void 0);
|
|
225
|
+
return {
|
|
226
|
+
ok: false,
|
|
227
|
+
reason: errorMessage,
|
|
228
|
+
scannedEntries: 0,
|
|
229
|
+
newSignals: 0,
|
|
230
|
+
deduped: 0
|
|
231
|
+
};
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
async function listRecentDailyFiles(memoryDir, lookbackDays, now) {
|
|
235
|
+
let entries;
|
|
236
|
+
try {
|
|
237
|
+
entries = await fs.readdir(memoryDir);
|
|
238
|
+
} catch (err) {
|
|
239
|
+
if (err?.code === "ENOENT") return [];
|
|
240
|
+
throw err;
|
|
241
|
+
}
|
|
242
|
+
const cutoff = new Date(now);
|
|
243
|
+
cutoff.setDate(cutoff.getDate() - lookbackDays);
|
|
244
|
+
const cutoffDay = isoDay(cutoff);
|
|
245
|
+
return entries.filter((name) => isDailyMemoryFile(name)).filter((name) => {
|
|
246
|
+
return name.replace(/\.md$/i, "") >= cutoffDay;
|
|
247
|
+
}).sort().map((name) => path.join(memoryDir, name));
|
|
248
|
+
}
|
|
249
|
+
async function parseFileChunks(filePath, workspaceDir) {
|
|
250
|
+
let raw;
|
|
251
|
+
try {
|
|
252
|
+
raw = await fs.readFile(filePath, "utf-8");
|
|
253
|
+
} catch {
|
|
254
|
+
return [];
|
|
255
|
+
}
|
|
256
|
+
const relativePath = normalizeMemoryPath(path.relative(workspaceDir, filePath));
|
|
257
|
+
const lines = raw.split(/\r?\n/);
|
|
258
|
+
const chunks = [];
|
|
259
|
+
let chunkStartLine = 1;
|
|
260
|
+
let currentLines = [];
|
|
261
|
+
for (let i = 0; i < lines.length; i++) {
|
|
262
|
+
const trimmed = lines[i].trim();
|
|
263
|
+
if (trimmed.length === 0) {
|
|
264
|
+
if (currentLines.length > 0) {
|
|
265
|
+
const text = currentLines.join(" ").replace(/\s+/g, " ").trim();
|
|
266
|
+
if (text.length >= 20) chunks.push({
|
|
267
|
+
path: relativePath,
|
|
268
|
+
startLine: chunkStartLine,
|
|
269
|
+
endLine: chunkStartLine + currentLines.length - 1,
|
|
270
|
+
text: text.slice(0, 360),
|
|
271
|
+
hash: snippetHash(text)
|
|
272
|
+
});
|
|
273
|
+
currentLines = [];
|
|
274
|
+
}
|
|
275
|
+
chunkStartLine = i + 2;
|
|
276
|
+
} else {
|
|
277
|
+
if (currentLines.length === 0) chunkStartLine = i + 1;
|
|
278
|
+
currentLines.push(trimmed);
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
if (currentLines.length > 0) {
|
|
282
|
+
const text = currentLines.join(" ").replace(/\s+/g, " ").trim();
|
|
283
|
+
if (text.length >= 20) chunks.push({
|
|
284
|
+
path: relativePath,
|
|
285
|
+
startLine: chunkStartLine,
|
|
286
|
+
endLine: chunkStartLine + currentLines.length - 1,
|
|
287
|
+
text: text.slice(0, 360),
|
|
288
|
+
hash: snippetHash(text)
|
|
289
|
+
});
|
|
290
|
+
}
|
|
291
|
+
return chunks;
|
|
292
|
+
}
|
|
293
|
+
function deduplicateChunks(chunks, threshold) {
|
|
294
|
+
const unique = [];
|
|
295
|
+
let deduped = 0;
|
|
296
|
+
for (const chunk of chunks) if (unique.some((existing) => existing.hash === chunk.hash || trigramSimilarity(existing.text, chunk.text) >= threshold)) deduped += 1;
|
|
297
|
+
else unique.push(chunk);
|
|
298
|
+
return {
|
|
299
|
+
unique,
|
|
300
|
+
deduped
|
|
301
|
+
};
|
|
302
|
+
}
|
|
303
|
+
async function writeLastRun(workspaceDir, params) {
|
|
304
|
+
const finishedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
305
|
+
const durationMs = Math.max(0, Date.now() - params.startMs);
|
|
306
|
+
const lastRun = {
|
|
307
|
+
version: 2,
|
|
308
|
+
phase: "light",
|
|
309
|
+
runId: params.runId,
|
|
310
|
+
startedAt: params.startedAt,
|
|
311
|
+
finishedAt,
|
|
312
|
+
durationMs,
|
|
313
|
+
ok: params.ok,
|
|
314
|
+
reason: params.reason,
|
|
315
|
+
config: params.cfg,
|
|
316
|
+
light: params.light,
|
|
317
|
+
...params.errorMessage ? { errorMessage: params.errorMessage } : {}
|
|
318
|
+
};
|
|
319
|
+
const lastRunPath = path.join(workspaceDir, DREAMING_DIR_RELATIVE, "last-run-light.json");
|
|
320
|
+
await fs.mkdir(path.dirname(lastRunPath), { recursive: true });
|
|
321
|
+
const tmp = `${lastRunPath}.${process.pid}.${Date.now()}.tmp`;
|
|
322
|
+
await fs.writeFile(tmp, `${JSON.stringify(lastRun, null, 2)}\n`, "utf-8");
|
|
323
|
+
await fs.rename(tmp, lastRunPath);
|
|
324
|
+
}
|
|
325
|
+
//#endregion
|
|
326
|
+
export { runLightSweep };
|
|
327
|
+
|
|
328
|
+
//# sourceMappingURL=light-sweep.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"light-sweep.js","names":[],"sources":["../../../../../src/agent/memory/dreaming/light-sweep.ts"],"sourcesContent":["import fs from 'node:fs/promises';\nimport path from 'node:path';\n\nimport { createLogger } from '../../../utils/logger.js';\nimport { DREAMING_DIR_RELATIVE } from './constants.js';\nimport type { DreamingLightConfig } from './config.js';\nimport {\n bumpEntryPhaseSignal,\n loadDreamingStore,\n saveDreamingStore,\n} from './short-term-store.js';\nimport {\n buildEntryKey,\n isoDay,\n normalizeMemoryPath,\n normalizeSnippetForHash,\n snippetHash,\n} from './utils.js';\nimport {\n DREAMING_LAST_RUN_FORMAT_VERSION,\n type DreamingLightLastRun,\n} from './last-run.js';\n\nconst log = createLogger('Dreaming:Light');\n\n// ── Helpers ────────────────────────────────────────────────────────────\n\nfunction isDailyMemoryFile(filename: string): boolean {\n return /^\\d{4}-\\d{2}-\\d{2}\\.md$/i.test(filename);\n}\n\n/**\n * Cosine-ish similarity via character n-gram overlap.\n * Fast and deterministic — no embeddings required.\n */\nfunction trigramSimilarity(textA: string, textB: string): number {\n const normalizedA = normalizeSnippetForHash(textA);\n const normalizedB = normalizeSnippetForHash(textB);\n if (!normalizedA || !normalizedB) return 0;\n if (normalizedA === normalizedB) return 1;\n\n const gramsA = buildTrigramSet(normalizedA);\n const gramsB = buildTrigramSet(normalizedB);\n if (gramsA.size === 0 || gramsB.size === 0) return 0;\n\n let intersection = 0;\n for (const gram of gramsA) {\n if (gramsB.has(gram)) intersection += 1;\n }\n return intersection / Math.sqrt(gramsA.size * gramsB.size);\n}\n\nfunction buildTrigramSet(text: string): Set<string> {\n const grams = new Set<string>();\n for (let i = 0; i <= text.length - 3; i++) {\n grams.add(text.slice(i, i + 3));\n }\n return grams;\n}\n\n// ── Light sweep config defaults ────────────────────────────────────────\n\nfunction resolveConfig(overrides?: Partial<DreamingLightConfig>): DreamingLightConfig {\n return {\n enabled: overrides?.enabled === true,\n cron: typeof overrides?.cron === 'string' ? overrides.cron : '0 */6 * * *',\n lookbackDays: Math.max(1, Math.floor(Number(overrides?.lookbackDays) || 2)),\n limit: Math.max(0, Math.floor(Number(overrides?.limit) || 100)),\n dedupeSimilarity: Math.max(0, Math.min(1, Number(overrides?.dedupeSimilarity) || 0.9)),\n };\n}\n\n// ── Core light sweep ───────────────────────────────────────────────────\n\ntype LightSweepChunk = {\n path: string;\n startLine: number;\n endLine: number;\n text: string;\n hash: string;\n};\n\n/**\n * Scan recent daily memory files (memory/YYYY-MM-DD.md) and collect signal\n * entries into the short-term store. Deduplicates near-identical snippets\n * using trigram similarity.\n *\n * This is the \"light sleep\" phase: fast, frequent, cheap.\n */\nexport async function runLightSweep(params: {\n workspaceDir: string;\n config?: Partial<DreamingLightConfig>;\n now?: Date;\n}): Promise<{\n ok: boolean;\n reason: string;\n scannedEntries: number;\n newSignals: number;\n deduped: number;\n}> {\n const cfg = resolveConfig(params.config);\n const now = params.now ?? new Date();\n const startedAt = now.toISOString();\n const runId = `light:${startedAt}:${process.pid}`;\n const startMs = Date.now();\n\n if (!cfg.enabled) {\n await writeLastRun(params.workspaceDir, {\n runId, startedAt, cfg, ok: true, reason: 'light sweep disabled', startMs,\n light: { scannedEntries: 0, newSignals: 0, deduped: 0 },\n });\n return { ok: true, reason: 'light sweep disabled', scannedEntries: 0, newSignals: 0, deduped: 0 };\n }\n\n try {\n const memoryDir = path.join(params.workspaceDir, 'memory');\n const recentFiles = await listRecentDailyFiles(memoryDir, cfg.lookbackDays, now);\n\n if (recentFiles.length === 0) {\n await writeLastRun(params.workspaceDir, {\n runId, startedAt, cfg, ok: true, reason: 'no recent daily files', startMs,\n light: { scannedEntries: 0, newSignals: 0, deduped: 0 },\n });\n return { ok: true, reason: 'no recent daily files', scannedEntries: 0, newSignals: 0, deduped: 0 };\n }\n\n // Parse chunks from daily files.\n const allChunks: LightSweepChunk[] = [];\n for (const filePath of recentFiles) {\n const chunks = await parseFileChunks(filePath, params.workspaceDir);\n allChunks.push(...chunks);\n }\n\n if (allChunks.length === 0) {\n await writeLastRun(params.workspaceDir, {\n runId, startedAt, cfg, ok: true, reason: 'no chunks found in daily files', startMs,\n light: { scannedEntries: 0, newSignals: 0, deduped: 0 },\n });\n return { ok: true, reason: 'no chunks found in daily files', scannedEntries: 0, newSignals: 0, deduped: 0 };\n }\n\n // Deduplicate chunks by trigram similarity.\n const { unique, deduped } = deduplicateChunks(allChunks, cfg.dedupeSimilarity);\n const capped = unique.slice(0, cfg.limit);\n\n // Merge into the short-term store, bumping lightHits + dailyCount.\n const { store } = await loadDreamingStore({ workspaceDir: params.workspaceDir });\n let newSignals = 0;\n const dayBucket = isoDay(now);\n const nowIso = now.toISOString();\n\n for (const chunk of capped) {\n const key = buildEntryKey({ path: chunk.path, startLine: chunk.startLine, endLine: chunk.endLine });\n const existing = store.entries[key];\n\n if (existing) {\n bumpEntryPhaseSignal(existing, 'lightHits');\n bumpEntryPhaseSignal(existing, 'dailyCount');\n existing.snippet = chunk.text.slice(0, 360);\n existing.lastRecalledAt = nowIso;\n } else {\n store.entries[key] = {\n key,\n path: chunk.path,\n startLine: chunk.startLine,\n endLine: chunk.endLine,\n snippet: chunk.text.slice(0, 360),\n recallCount: 0,\n dailyCount: 1,\n groundedCount: 0,\n lightHits: 1,\n remHits: 0,\n phaseHitCount: 1,\n totalSignalCount: 1,\n totalScore: 0,\n maxScore: 0,\n queryHashes: [],\n recallDays: [dayBucket],\n firstRecalledAt: nowIso,\n lastRecalledAt: nowIso,\n };\n newSignals += 1;\n }\n }\n\n store.updatedAt = nowIso;\n await saveDreamingStore({ workspaceDir: params.workspaceDir, store });\n\n log.info(\n { workspaceDir: params.workspaceDir, scanned: allChunks.length, newSignals, deduped },\n 'Light sweep complete',\n );\n\n await writeLastRun(params.workspaceDir, {\n runId, startedAt, cfg, ok: true, reason: 'light sweep complete', startMs,\n light: { scannedEntries: allChunks.length, newSignals, deduped },\n });\n\n return { ok: true, reason: 'light sweep complete', scannedEntries: allChunks.length, newSignals, deduped };\n } catch (err) {\n const errorMessage = err instanceof Error ? err.message : String(err);\n log.error({ err, errorMessage, workspaceDir: params.workspaceDir }, `Light sweep failed: ${errorMessage}`);\n await writeLastRun(params.workspaceDir, {\n runId, startedAt, cfg, ok: false, reason: `light sweep error: ${errorMessage}`, startMs,\n light: { scannedEntries: 0, newSignals: 0, deduped: 0 }, errorMessage,\n }).catch(() => undefined);\n return { ok: false, reason: errorMessage, scannedEntries: 0, newSignals: 0, deduped: 0 };\n }\n}\n\n// ── File scanning ──────────────────────────────────────────────────────\n\nasync function listRecentDailyFiles(memoryDir: string, lookbackDays: number, now: Date): Promise<string[]> {\n let entries: string[];\n try {\n entries = await fs.readdir(memoryDir);\n } catch (err) {\n if ((err as NodeJS.ErrnoException)?.code === 'ENOENT') return [];\n throw err;\n }\n\n const cutoff = new Date(now);\n cutoff.setDate(cutoff.getDate() - lookbackDays);\n const cutoffDay = isoDay(cutoff);\n\n return entries\n .filter((name) => isDailyMemoryFile(name))\n .filter((name) => {\n const day = name.replace(/\\.md$/i, '');\n return day >= cutoffDay;\n })\n .sort()\n .map((name) => path.join(memoryDir, name));\n}\n\nasync function parseFileChunks(filePath: string, workspaceDir: string): Promise<LightSweepChunk[]> {\n let raw: string;\n try {\n raw = await fs.readFile(filePath, 'utf-8');\n } catch {\n return [];\n }\n\n const relativePath = normalizeMemoryPath(path.relative(workspaceDir, filePath));\n const lines = raw.split(/\\r?\\n/);\n const chunks: LightSweepChunk[] = [];\n\n // Split into paragraph-level chunks (separated by blank lines).\n let chunkStartLine = 1;\n let currentLines: string[] = [];\n\n for (let i = 0; i < lines.length; i++) {\n const trimmed = lines[i]!.trim();\n if (trimmed.length === 0) {\n if (currentLines.length > 0) {\n const text = currentLines.join(' ').replace(/\\s+/g, ' ').trim();\n if (text.length >= 20) {\n chunks.push({\n path: relativePath,\n startLine: chunkStartLine,\n endLine: chunkStartLine + currentLines.length - 1,\n text: text.slice(0, 360),\n hash: snippetHash(text),\n });\n }\n currentLines = [];\n }\n chunkStartLine = i + 2; // next non-blank line (1-indexed)\n } else {\n if (currentLines.length === 0) chunkStartLine = i + 1;\n currentLines.push(trimmed);\n }\n }\n\n // Flush remaining.\n if (currentLines.length > 0) {\n const text = currentLines.join(' ').replace(/\\s+/g, ' ').trim();\n if (text.length >= 20) {\n chunks.push({\n path: relativePath,\n startLine: chunkStartLine,\n endLine: chunkStartLine + currentLines.length - 1,\n text: text.slice(0, 360),\n hash: snippetHash(text),\n });\n }\n }\n\n return chunks;\n}\n\n// ── Deduplication ──────────────────────────────────────────────────────\n\nfunction deduplicateChunks(\n chunks: LightSweepChunk[],\n threshold: number,\n): { unique: LightSweepChunk[]; deduped: number } {\n const unique: LightSweepChunk[] = [];\n let deduped = 0;\n\n for (const chunk of chunks) {\n const isDuplicate = unique.some(\n (existing) => existing.hash === chunk.hash || trigramSimilarity(existing.text, chunk.text) >= threshold,\n );\n if (isDuplicate) {\n deduped += 1;\n } else {\n unique.push(chunk);\n }\n }\n\n return { unique, deduped };\n}\n\n// ── Last-run writer ────────────────────────────────────────────────────\n\nasync function writeLastRun(\n workspaceDir: string,\n params: {\n runId: string;\n startedAt: string;\n cfg: DreamingLightConfig;\n ok: boolean;\n reason: string;\n startMs: number;\n light: DreamingLightLastRun['light'];\n errorMessage?: string;\n },\n): Promise<void> {\n const finishedAt = new Date().toISOString();\n const durationMs = Math.max(0, Date.now() - params.startMs);\n\n const lastRun: DreamingLightLastRun = {\n version: DREAMING_LAST_RUN_FORMAT_VERSION,\n phase: 'light',\n runId: params.runId,\n startedAt: params.startedAt,\n finishedAt,\n durationMs,\n ok: params.ok,\n reason: params.reason,\n config: params.cfg,\n light: params.light,\n ...(params.errorMessage ? { errorMessage: params.errorMessage } : {}),\n };\n\n const lastRunPath = path.join(workspaceDir, DREAMING_DIR_RELATIVE, 'last-run-light.json');\n await fs.mkdir(path.dirname(lastRunPath), { recursive: true });\n const tmp = `${lastRunPath}.${process.pid}.${Date.now()}.tmp`;\n await fs.writeFile(tmp, `${JSON.stringify(lastRun, null, 2)}\\n`, 'utf-8');\n await fs.rename(tmp, lastRunPath);\n}\n"],"mappings":";;;;;;;;;aAGwD;AAoBxD,MAAM,MAAM,aAAa,iBAAiB;AAI1C,SAAS,kBAAkB,UAA2B;AACpD,QAAO,2BAA2B,KAAK,SAAS;;;;;;AAOlD,SAAS,kBAAkB,OAAe,OAAuB;CAC/D,MAAM,cAAc,wBAAwB,MAAM;CAClD,MAAM,cAAc,wBAAwB,MAAM;AAClD,KAAI,CAAC,eAAe,CAAC,YAAa,QAAO;AACzC,KAAI,gBAAgB,YAAa,QAAO;CAExC,MAAM,SAAS,gBAAgB,YAAY;CAC3C,MAAM,SAAS,gBAAgB,YAAY;AAC3C,KAAI,OAAO,SAAS,KAAK,OAAO,SAAS,EAAG,QAAO;CAEnD,IAAI,eAAe;AACnB,MAAK,MAAM,QAAQ,OACjB,KAAI,OAAO,IAAI,KAAK,CAAE,iBAAgB;AAExC,QAAO,eAAe,KAAK,KAAK,OAAO,OAAO,OAAO,KAAK;;AAG5D,SAAS,gBAAgB,MAA2B;CAClD,MAAM,wBAAQ,IAAI,KAAa;AAC/B,MAAK,IAAI,IAAI,GAAG,KAAK,KAAK,SAAS,GAAG,IACpC,OAAM,IAAI,KAAK,MAAM,GAAG,IAAI,EAAE,CAAC;AAEjC,QAAO;;AAKT,SAAS,cAAc,WAA+D;AACpF,QAAO;EACL,SAAS,WAAW,YAAY;EAChC,MAAM,OAAO,WAAW,SAAS,WAAW,UAAU,OAAO;EAC7D,cAAc,KAAK,IAAI,GAAG,KAAK,MAAM,OAAO,WAAW,aAAa,IAAI,EAAE,CAAC;EAC3E,OAAO,KAAK,IAAI,GAAG,KAAK,MAAM,OAAO,WAAW,MAAM,IAAI,IAAI,CAAC;EAC/D,kBAAkB,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,OAAO,WAAW,iBAAiB,IAAI,GAAI,CAAC;EACvF;;;;;;;;;AAoBH,eAAsB,cAAc,QAUjC;CACD,MAAM,MAAM,cAAc,OAAO,OAAO;CACxC,MAAM,MAAM,OAAO,uBAAO,IAAI,MAAM;CACpC,MAAM,YAAY,IAAI,aAAa;CACnC,MAAM,QAAQ,SAAS,UAAU,GAAG,QAAQ;CAC5C,MAAM,UAAU,KAAK,KAAK;AAE1B,KAAI,CAAC,IAAI,SAAS;AAChB,QAAM,aAAa,OAAO,cAAc;GACtC;GAAO;GAAW;GAAK,IAAI;GAAM,QAAQ;GAAwB;GACjE,OAAO;IAAE,gBAAgB;IAAG,YAAY;IAAG,SAAS;IAAG;GACxD,CAAC;AACF,SAAO;GAAE,IAAI;GAAM,QAAQ;GAAwB,gBAAgB;GAAG,YAAY;GAAG,SAAS;GAAG;;AAGnG,KAAI;EAEF,MAAM,cAAc,MAAM,qBADR,KAAK,KAAK,OAAO,cAAc,SACO,EAAE,IAAI,cAAc,IAAI;AAEhF,MAAI,YAAY,WAAW,GAAG;AAC5B,SAAM,aAAa,OAAO,cAAc;IACtC;IAAO;IAAW;IAAK,IAAI;IAAM,QAAQ;IAAyB;IAClE,OAAO;KAAE,gBAAgB;KAAG,YAAY;KAAG,SAAS;KAAG;IACxD,CAAC;AACF,UAAO;IAAE,IAAI;IAAM,QAAQ;IAAyB,gBAAgB;IAAG,YAAY;IAAG,SAAS;IAAG;;EAIpG,MAAM,YAA+B,EAAE;AACvC,OAAK,MAAM,YAAY,aAAa;GAClC,MAAM,SAAS,MAAM,gBAAgB,UAAU,OAAO,aAAa;AACnE,aAAU,KAAK,GAAG,OAAO;;AAG3B,MAAI,UAAU,WAAW,GAAG;AAC1B,SAAM,aAAa,OAAO,cAAc;IACtC;IAAO;IAAW;IAAK,IAAI;IAAM,QAAQ;IAAkC;IAC3E,OAAO;KAAE,gBAAgB;KAAG,YAAY;KAAG,SAAS;KAAG;IACxD,CAAC;AACF,UAAO;IAAE,IAAI;IAAM,QAAQ;IAAkC,gBAAgB;IAAG,YAAY;IAAG,SAAS;IAAG;;EAI7G,MAAM,EAAE,QAAQ,YAAY,kBAAkB,WAAW,IAAI,iBAAiB;EAC9E,MAAM,SAAS,OAAO,MAAM,GAAG,IAAI,MAAM;EAGzC,MAAM,EAAE,UAAU,MAAM,kBAAkB,EAAE,cAAc,OAAO,cAAc,CAAC;EAChF,IAAI,aAAa;EACjB,MAAM,YAAY,OAAO,IAAI;EAC7B,MAAM,SAAS,IAAI,aAAa;AAEhC,OAAK,MAAM,SAAS,QAAQ;GAC1B,MAAM,MAAM,cAAc;IAAE,MAAM,MAAM;IAAM,WAAW,MAAM;IAAW,SAAS,MAAM;IAAS,CAAC;GACnG,MAAM,WAAW,MAAM,QAAQ;AAE/B,OAAI,UAAU;AACZ,yBAAqB,UAAU,YAAY;AAC3C,yBAAqB,UAAU,aAAa;AAC5C,aAAS,UAAU,MAAM,KAAK,MAAM,GAAG,IAAI;AAC3C,aAAS,iBAAiB;UACrB;AACL,UAAM,QAAQ,OAAO;KACnB;KACA,MAAM,MAAM;KACZ,WAAW,MAAM;KACjB,SAAS,MAAM;KACf,SAAS,MAAM,KAAK,MAAM,GAAG,IAAI;KACjC,aAAa;KACb,YAAY;KACZ,eAAe;KACf,WAAW;KACX,SAAS;KACT,eAAe;KACf,kBAAkB;KAClB,YAAY;KACZ,UAAU;KACV,aAAa,EAAE;KACf,YAAY,CAAC,UAAU;KACvB,iBAAiB;KACjB,gBAAgB;KACjB;AACD,kBAAc;;;AAIlB,QAAM,YAAY;AAClB,QAAM,kBAAkB;GAAE,cAAc,OAAO;GAAc;GAAO,CAAC;AAErE,MAAI,KACF;GAAE,cAAc,OAAO;GAAc,SAAS,UAAU;GAAQ;GAAY;GAAS,EACrF,uBACD;AAED,QAAM,aAAa,OAAO,cAAc;GACtC;GAAO;GAAW;GAAK,IAAI;GAAM,QAAQ;GAAwB;GACjE,OAAO;IAAE,gBAAgB,UAAU;IAAQ;IAAY;IAAS;GACjE,CAAC;AAEF,SAAO;GAAE,IAAI;GAAM,QAAQ;GAAwB,gBAAgB,UAAU;GAAQ;GAAY;GAAS;UACnG,KAAK;EACZ,MAAM,eAAe,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI;AACrE,MAAI,MAAM;GAAE;GAAK;GAAc,cAAc,OAAO;GAAc,EAAE,uBAAuB,eAAe;AAC1G,QAAM,aAAa,OAAO,cAAc;GACtC;GAAO;GAAW;GAAK,IAAI;GAAO,QAAQ,sBAAsB;GAAgB;GAChF,OAAO;IAAE,gBAAgB;IAAG,YAAY;IAAG,SAAS;IAAG;GAAE;GAC1D,CAAC,CAAC,YAAY,KAAA,EAAU;AACzB,SAAO;GAAE,IAAI;GAAO,QAAQ;GAAc,gBAAgB;GAAG,YAAY;GAAG,SAAS;GAAG;;;AAM5F,eAAe,qBAAqB,WAAmB,cAAsB,KAA8B;CACzG,IAAI;AACJ,KAAI;AACF,YAAU,MAAM,GAAG,QAAQ,UAAU;UAC9B,KAAK;AACZ,MAAK,KAA+B,SAAS,SAAU,QAAO,EAAE;AAChE,QAAM;;CAGR,MAAM,SAAS,IAAI,KAAK,IAAI;AAC5B,QAAO,QAAQ,OAAO,SAAS,GAAG,aAAa;CAC/C,MAAM,YAAY,OAAO,OAAO;AAEhC,QAAO,QACJ,QAAQ,SAAS,kBAAkB,KAAK,CAAC,CACzC,QAAQ,SAAS;AAEhB,SADY,KAAK,QAAQ,UAAU,GACzB,IAAI;GACd,CACD,MAAM,CACN,KAAK,SAAS,KAAK,KAAK,WAAW,KAAK,CAAC;;AAG9C,eAAe,gBAAgB,UAAkB,cAAkD;CACjG,IAAI;AACJ,KAAI;AACF,QAAM,MAAM,GAAG,SAAS,UAAU,QAAQ;SACpC;AACN,SAAO,EAAE;;CAGX,MAAM,eAAe,oBAAoB,KAAK,SAAS,cAAc,SAAS,CAAC;CAC/E,MAAM,QAAQ,IAAI,MAAM,QAAQ;CAChC,MAAM,SAA4B,EAAE;CAGpC,IAAI,iBAAiB;CACrB,IAAI,eAAyB,EAAE;AAE/B,MAAK,IAAI,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;EACrC,MAAM,UAAU,MAAM,GAAI,MAAM;AAChC,MAAI,QAAQ,WAAW,GAAG;AACxB,OAAI,aAAa,SAAS,GAAG;IAC3B,MAAM,OAAO,aAAa,KAAK,IAAI,CAAC,QAAQ,QAAQ,IAAI,CAAC,MAAM;AAC/D,QAAI,KAAK,UAAU,GACjB,QAAO,KAAK;KACV,MAAM;KACN,WAAW;KACX,SAAS,iBAAiB,aAAa,SAAS;KAChD,MAAM,KAAK,MAAM,GAAG,IAAI;KACxB,MAAM,YAAY,KAAK;KACxB,CAAC;AAEJ,mBAAe,EAAE;;AAEnB,oBAAiB,IAAI;SAChB;AACL,OAAI,aAAa,WAAW,EAAG,kBAAiB,IAAI;AACpD,gBAAa,KAAK,QAAQ;;;AAK9B,KAAI,aAAa,SAAS,GAAG;EAC3B,MAAM,OAAO,aAAa,KAAK,IAAI,CAAC,QAAQ,QAAQ,IAAI,CAAC,MAAM;AAC/D,MAAI,KAAK,UAAU,GACjB,QAAO,KAAK;GACV,MAAM;GACN,WAAW;GACX,SAAS,iBAAiB,aAAa,SAAS;GAChD,MAAM,KAAK,MAAM,GAAG,IAAI;GACxB,MAAM,YAAY,KAAK;GACxB,CAAC;;AAIN,QAAO;;AAKT,SAAS,kBACP,QACA,WACgD;CAChD,MAAM,SAA4B,EAAE;CACpC,IAAI,UAAU;AAEd,MAAK,MAAM,SAAS,OAIlB,KAHoB,OAAO,MACxB,aAAa,SAAS,SAAS,MAAM,QAAQ,kBAAkB,SAAS,MAAM,MAAM,KAAK,IAAI,UAEjF,CACb,YAAW;KAEX,QAAO,KAAK,MAAM;AAItB,QAAO;EAAE;EAAQ;EAAS;;AAK5B,eAAe,aACb,cACA,QAUe;CACf,MAAM,8BAAa,IAAI,MAAM,EAAC,aAAa;CAC3C,MAAM,aAAa,KAAK,IAAI,GAAG,KAAK,KAAK,GAAG,OAAO,QAAQ;CAE3D,MAAM,UAAgC;EACpC,SAAA;EACA,OAAO;EACP,OAAO,OAAO;EACd,WAAW,OAAO;EAClB;EACA;EACA,IAAI,OAAO;EACX,QAAQ,OAAO;EACf,QAAQ,OAAO;EACf,OAAO,OAAO;EACd,GAAI,OAAO,eAAe,EAAE,cAAc,OAAO,cAAc,GAAG,EAAE;EACrE;CAED,MAAM,cAAc,KAAK,KAAK,cAAc,uBAAuB,sBAAsB;AACzF,OAAM,GAAG,MAAM,KAAK,QAAQ,YAAY,EAAE,EAAE,WAAW,MAAM,CAAC;CAC9D,MAAM,MAAM,GAAG,YAAY,GAAG,QAAQ,IAAI,GAAG,KAAK,KAAK,CAAC;AACxD,OAAM,GAAG,UAAU,KAAK,GAAG,KAAK,UAAU,SAAS,MAAM,EAAE,CAAC,KAAK,QAAQ;AACzE,OAAM,GAAG,OAAO,KAAK,YAAY"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { DreamingDeepConfig } from './
|
|
1
|
+
import type { DreamingDeepConfig } from './config.js';
|
|
2
2
|
type PreviewItem = {
|
|
3
3
|
key: string;
|
|
4
4
|
hash: string;
|
|
@@ -9,6 +9,7 @@ type PreviewItem = {
|
|
|
9
9
|
score: number;
|
|
10
10
|
avgScore: number;
|
|
11
11
|
recallCount: number;
|
|
12
|
+
recencyDecay: number;
|
|
12
13
|
alreadyPromotedByKey: boolean;
|
|
13
14
|
alreadyPromotedByHash: boolean;
|
|
14
15
|
skippedReason: string | null;
|
|
@@ -17,6 +18,7 @@ export declare function previewDreamingDeepPromotion(params: {
|
|
|
17
18
|
workspaceDir: string;
|
|
18
19
|
config?: Partial<DreamingDeepConfig>;
|
|
19
20
|
limit?: number;
|
|
21
|
+
now?: Date;
|
|
20
22
|
}): Promise<{
|
|
21
23
|
ok: boolean;
|
|
22
24
|
reason: string;
|
|
@@ -1,95 +1,11 @@
|
|
|
1
1
|
import { MEMORY_MD_FILENAME } from "./constants.js";
|
|
2
|
+
import { clamp01, compareCandidatesByScore, computeCandidateScore, extractPromotionMarkers, isContaminatedSnippet, isExpiredEntry, readFileLines, resolveDeepDefaults, sliceRange, snippetHash } from "./utils.js";
|
|
2
3
|
import { loadDreamingStore } from "./short-term-store.js";
|
|
3
4
|
import path from "node:path";
|
|
4
5
|
import fs from "node:fs/promises";
|
|
5
|
-
import { createHash } from "node:crypto";
|
|
6
6
|
//#region src/agent/memory/dreaming/preview.ts
|
|
7
|
-
function clamp01(value) {
|
|
8
|
-
if (!Number.isFinite(value)) return 0;
|
|
9
|
-
return Math.max(0, Math.min(1, value));
|
|
10
|
-
}
|
|
11
|
-
function compare(a, b) {
|
|
12
|
-
if (b.score !== a.score) return b.score - a.score;
|
|
13
|
-
if (b.recallCount !== a.recallCount) return b.recallCount - a.recallCount;
|
|
14
|
-
const aMs = Date.parse(a.lastRecalledAt);
|
|
15
|
-
const bMs = Date.parse(b.lastRecalledAt);
|
|
16
|
-
if (Number.isFinite(aMs) || Number.isFinite(bMs)) {
|
|
17
|
-
if (bMs !== aMs) return bMs - aMs;
|
|
18
|
-
}
|
|
19
|
-
return a.path.localeCompare(b.path);
|
|
20
|
-
}
|
|
21
|
-
async function readFileLines(fullPath) {
|
|
22
|
-
try {
|
|
23
|
-
return (await fs.readFile(fullPath, "utf-8")).split(/\r?\n/);
|
|
24
|
-
} catch (err) {
|
|
25
|
-
if (err?.code === "ENOENT") return null;
|
|
26
|
-
throw err;
|
|
27
|
-
}
|
|
28
|
-
}
|
|
29
|
-
function sliceRange(lines, startLine, endLine) {
|
|
30
|
-
const startIdx = Math.max(0, startLine - 1);
|
|
31
|
-
const endIdx = Math.min(lines.length, endLine);
|
|
32
|
-
if (startIdx >= endIdx) return "";
|
|
33
|
-
return lines.slice(startIdx, endIdx).map((l) => l.trim()).filter(Boolean).join(" ").replace(/\s+/g, " ").trim().slice(0, 360);
|
|
34
|
-
}
|
|
35
|
-
function normalizeSnippetForHash(snippet) {
|
|
36
|
-
return snippet.trim().replace(/\s+/g, " ").replace(/[“”]/g, "\"").replace(/[‘’]/g, "'").toLowerCase().slice(0, 512);
|
|
37
|
-
}
|
|
38
|
-
function snippetHash(snippet) {
|
|
39
|
-
return createHash("sha1").update(normalizeSnippetForHash(snippet)).digest("hex").slice(0, 12);
|
|
40
|
-
}
|
|
41
|
-
function extractPromotionMarkers(memoryText) {
|
|
42
|
-
const keys = /* @__PURE__ */ new Set();
|
|
43
|
-
const hashes = /* @__PURE__ */ new Set();
|
|
44
|
-
for (const match of memoryText.matchAll(/<!--\s*xopc-memory-promotion\b([\s\S]*?)-->/gi)) {
|
|
45
|
-
const body = match[1] ?? "";
|
|
46
|
-
const k = body.match(/\bkey\s*=\s*"([^"]+)"/i)?.[1]?.trim();
|
|
47
|
-
const h = body.match(/\bhash\s*=\s*"([^"]+)"/i)?.[1]?.trim();
|
|
48
|
-
if (k) keys.add(k);
|
|
49
|
-
if (h) hashes.add(h);
|
|
50
|
-
}
|
|
51
|
-
for (const match of memoryText.matchAll(/<!--\s*xopc-memory-promotion:([^\n]+?)\s*-->/gi)) {
|
|
52
|
-
const key = match[1]?.trim();
|
|
53
|
-
if (key) keys.add(key);
|
|
54
|
-
}
|
|
55
|
-
return {
|
|
56
|
-
keys,
|
|
57
|
-
hashes
|
|
58
|
-
};
|
|
59
|
-
}
|
|
60
|
-
function isContaminatedSnippet(snippet) {
|
|
61
|
-
const s = snippet.trim();
|
|
62
|
-
if (!s) return true;
|
|
63
|
-
const lower = s.toLowerCase();
|
|
64
|
-
if ([
|
|
65
|
-
/\b(system|assistant|tool)\s*:/i,
|
|
66
|
-
/<\s*(system|assistant|tool)\b/i,
|
|
67
|
-
/<!--\s*xopc-memory-promotion\b/i,
|
|
68
|
-
/tool_call_id|toolcallid|function_call|arguments\s*:\s*\{/i,
|
|
69
|
-
/"tool"\s*:\s*|\btool_name\b|\btool\b\s*results?/i,
|
|
70
|
-
/you are (an|a)\s+(ai|assistant)|follow these instructions/i,
|
|
71
|
-
/begin\s+(system prompt|instructions)|end\s+(system prompt|instructions)/i,
|
|
72
|
-
/```/i,
|
|
73
|
-
/\b__xopc_/i
|
|
74
|
-
].some((p) => p.test(s))) return true;
|
|
75
|
-
if ((s.match(/[{}[\]]/g) ?? []).length >= 12) return true;
|
|
76
|
-
if ((lower.match(/https?:\/\//g) ?? []).length >= 3) return true;
|
|
77
|
-
return false;
|
|
78
|
-
}
|
|
79
|
-
function resolveDefaultConfig(overrides) {
|
|
80
|
-
const enabled = overrides?.enabled === true;
|
|
81
|
-
const minScore = typeof overrides?.minScore === "number" ? overrides.minScore : .8;
|
|
82
|
-
const minRecallCount = typeof overrides?.minRecallCount === "number" ? overrides.minRecallCount : 3;
|
|
83
|
-
const limit = typeof overrides?.limit === "number" ? overrides.limit : 10;
|
|
84
|
-
return {
|
|
85
|
-
enabled,
|
|
86
|
-
minScore: clamp01(minScore),
|
|
87
|
-
minRecallCount: Math.max(1, Math.floor(minRecallCount)),
|
|
88
|
-
limit: Math.max(0, Math.floor(limit))
|
|
89
|
-
};
|
|
90
|
-
}
|
|
91
7
|
async function previewDreamingDeepPromotion(params) {
|
|
92
|
-
const cfg =
|
|
8
|
+
const cfg = resolveDeepDefaults(params.config);
|
|
93
9
|
const memoryPath = path.join(params.workspaceDir, MEMORY_MD_FILENAME);
|
|
94
10
|
if (!cfg.enabled) return {
|
|
95
11
|
ok: true,
|
|
@@ -98,21 +14,24 @@ async function previewDreamingDeepPromotion(params) {
|
|
|
98
14
|
memoryPath
|
|
99
15
|
};
|
|
100
16
|
const { store } = await loadDreamingStore({ workspaceDir: params.workspaceDir });
|
|
17
|
+
const nowMs = (params.now ?? /* @__PURE__ */ new Date()).getTime();
|
|
101
18
|
const ranked = Object.values(store.entries ?? {}).filter((e) => {
|
|
102
19
|
if (!e || typeof e !== "object") return false;
|
|
103
20
|
if (e.promotedAt) return false;
|
|
104
21
|
if (!e.path || !e.path.startsWith("memory/")) return false;
|
|
105
22
|
if (e.recallCount < cfg.minRecallCount) return false;
|
|
23
|
+
if ((e.queryHashes?.length ?? 0) < cfg.minUniqueQueries) return false;
|
|
24
|
+
if (isExpiredEntry(e.lastRecalledAt, nowMs, cfg.maxAgeDays)) return false;
|
|
106
25
|
return clamp01(e.recallCount > 0 ? e.totalScore / e.recallCount : 0) >= cfg.minScore;
|
|
107
26
|
}).map((e) => {
|
|
108
|
-
const avgScore
|
|
109
|
-
const score = clamp01(avgScore + clamp01(Math.log1p(e.recallCount) / Math.log1p(10)) * .12);
|
|
27
|
+
const { avgScore, score, recencyDecay } = computeCandidateScore(e, nowMs, cfg.recencyHalfLifeDays);
|
|
110
28
|
return {
|
|
111
29
|
...e,
|
|
112
30
|
avgScore,
|
|
113
|
-
score
|
|
31
|
+
score,
|
|
32
|
+
recencyDecay
|
|
114
33
|
};
|
|
115
|
-
}).sort(
|
|
34
|
+
}).sort(compareCandidatesByScore);
|
|
116
35
|
const markers = extractPromotionMarkers(await fs.readFile(memoryPath, "utf-8").catch((err) => {
|
|
117
36
|
if (err?.code === "ENOENT") return "";
|
|
118
37
|
throw err;
|
|
@@ -132,6 +51,7 @@ async function previewDreamingDeepPromotion(params) {
|
|
|
132
51
|
score: candidate.score,
|
|
133
52
|
avgScore: candidate.avgScore,
|
|
134
53
|
recallCount: candidate.recallCount,
|
|
54
|
+
recencyDecay: candidate.recencyDecay,
|
|
135
55
|
alreadyPromotedByKey: true,
|
|
136
56
|
alreadyPromotedByHash: false,
|
|
137
57
|
skippedReason: "already promoted (key)"
|
|
@@ -157,6 +77,7 @@ async function previewDreamingDeepPromotion(params) {
|
|
|
157
77
|
score: candidate.score,
|
|
158
78
|
avgScore: candidate.avgScore,
|
|
159
79
|
recallCount: candidate.recallCount,
|
|
80
|
+
recencyDecay: candidate.recencyDecay,
|
|
160
81
|
alreadyPromotedByKey: false,
|
|
161
82
|
alreadyPromotedByHash,
|
|
162
83
|
skippedReason: alreadyPromotedByHash ? "already promoted (hash)" : null
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"preview.js","names":[],"sources":["../../../../../src/agent/memory/dreaming/preview.ts"],"sourcesContent":["import fs from 'node:fs/promises';\nimport path from 'node:path';\nimport { createHash } from 'node:crypto';\n\nimport type { DreamingDeepConfig } from './deep-promotion.js';\nimport { loadDreamingStore, type DreamingStoreEntry } from './short-term-store.js';\nimport { MEMORY_MD_FILENAME } from './constants.js';\n\ntype PreviewItem = {\n key: string;\n hash: string;\n snippet: string;\n path: string;\n startLine: number;\n endLine: number;\n score: number;\n avgScore: number;\n recallCount: number;\n alreadyPromotedByKey: boolean;\n alreadyPromotedByHash: boolean;\n skippedReason: string | null;\n};\n\nfunction clamp01(value: number): number {\n if (!Number.isFinite(value)) return 0;\n return Math.max(0, Math.min(1, value));\n}\n\nfunction compare(a: { score: number; recallCount: number; lastRecalledAt: string; path: string }, b: { score: number; recallCount: number; lastRecalledAt: string; path: string }): number {\n if (b.score !== a.score) return b.score - a.score;\n if (b.recallCount !== a.recallCount) return b.recallCount - a.recallCount;\n const aMs = Date.parse(a.lastRecalledAt);\n const bMs = Date.parse(b.lastRecalledAt);\n if (Number.isFinite(aMs) || Number.isFinite(bMs)) {\n if (bMs !== aMs) return bMs - aMs;\n }\n return a.path.localeCompare(b.path);\n}\n\nasync function readFileLines(fullPath: string): Promise<string[] | null> {\n try {\n const raw = await fs.readFile(fullPath, 'utf-8');\n return raw.split(/\\r?\\n/);\n } catch (err) {\n const code = (err as NodeJS.ErrnoException | undefined)?.code;\n if (code === 'ENOENT') return null;\n throw err;\n }\n}\n\nfunction sliceRange(lines: string[], startLine: number, endLine: number): string {\n const startIdx = Math.max(0, startLine - 1);\n const endIdx = Math.min(lines.length, endLine);\n if (startIdx >= endIdx) return '';\n return lines\n .slice(startIdx, endIdx)\n .map((l) => l.trim())\n .filter(Boolean)\n .join(' ')\n .replace(/\\s+/g, ' ')\n .trim()\n .slice(0, 360);\n}\n\nfunction normalizeSnippetForHash(snippet: string): string {\n return snippet\n .trim()\n .replace(/\\s+/g, ' ')\n .replace(/[“”]/g, '\"')\n .replace(/[‘’]/g, \"'\")\n .toLowerCase()\n .slice(0, 512);\n}\n\nfunction snippetHash(snippet: string): string {\n return createHash('sha1').update(normalizeSnippetForHash(snippet)).digest('hex').slice(0, 12);\n}\n\nfunction extractPromotionMarkers(memoryText: string): { keys: Set<string>; hashes: Set<string> } {\n const keys = new Set<string>();\n const hashes = new Set<string>();\n for (const match of memoryText.matchAll(/<!--\\s*xopc-memory-promotion\\b([\\s\\S]*?)-->/gi)) {\n const body = match[1] ?? '';\n const k = body.match(/\\bkey\\s*=\\s*\"([^\"]+)\"/i)?.[1]?.trim();\n const h = body.match(/\\bhash\\s*=\\s*\"([^\"]+)\"/i)?.[1]?.trim();\n if (k) keys.add(k);\n if (h) hashes.add(h);\n }\n for (const match of memoryText.matchAll(/<!--\\s*xopc-memory-promotion:([^\\n]+?)\\s*-->/gi)) {\n const key = match[1]?.trim();\n if (key) keys.add(key);\n }\n return { keys, hashes };\n}\n\nfunction isContaminatedSnippet(snippet: string): boolean {\n const s = snippet.trim();\n if (!s) return true;\n const lower = s.toLowerCase();\n const patterns: RegExp[] = [\n /\\b(system|assistant|tool)\\s*:/i,\n /<\\s*(system|assistant|tool)\\b/i,\n /<!--\\s*xopc-memory-promotion\\b/i,\n /tool_call_id|toolcallid|function_call|arguments\\s*:\\s*\\{/i,\n /\"tool\"\\s*:\\s*|\\btool_name\\b|\\btool\\b\\s*results?/i,\n /you are (an|a)\\s+(ai|assistant)|follow these instructions/i,\n /begin\\s+(system prompt|instructions)|end\\s+(system prompt|instructions)/i,\n /```/i,\n /\\b__xopc_/i,\n ];\n if (patterns.some((p) => p.test(s))) return true;\n const braceCount = (s.match(/[{}[\\]]/g) ?? []).length;\n if (braceCount >= 12) return true;\n const urlCount = (lower.match(/https?:\\/\\//g) ?? []).length;\n if (urlCount >= 3) return true;\n return false;\n}\n\nfunction resolveDefaultConfig(overrides?: Partial<DreamingDeepConfig>): DreamingDeepConfig {\n const enabled = overrides?.enabled === true;\n const minScore = typeof overrides?.minScore === 'number' ? overrides.minScore : 0.8;\n const minRecallCount = typeof overrides?.minRecallCount === 'number' ? overrides.minRecallCount : 3;\n const limit = typeof overrides?.limit === 'number' ? overrides.limit : 10;\n return {\n enabled,\n minScore: clamp01(minScore),\n minRecallCount: Math.max(1, Math.floor(minRecallCount)),\n limit: Math.max(0, Math.floor(limit)),\n };\n}\n\nexport async function previewDreamingDeepPromotion(params: {\n workspaceDir: string;\n config?: Partial<DreamingDeepConfig>;\n limit?: number;\n}): Promise<{ ok: boolean; reason: string; items: PreviewItem[]; memoryPath: string }> {\n const cfg = resolveDefaultConfig(params.config);\n const memoryPath = path.join(params.workspaceDir, MEMORY_MD_FILENAME);\n if (!cfg.enabled) return { ok: true, reason: 'dreaming disabled', items: [], memoryPath };\n\n const { store } = await loadDreamingStore({ workspaceDir: params.workspaceDir });\n const all = Object.values(store.entries ?? {}).filter((e): e is DreamingStoreEntry => {\n if (!e || typeof e !== 'object') return false;\n if (e.promotedAt) return false;\n if (!e.path || !e.path.startsWith('memory/')) return false;\n if (e.recallCount < cfg.minRecallCount) return false;\n const avg = e.recallCount > 0 ? e.totalScore / e.recallCount : 0;\n return clamp01(avg) >= cfg.minScore;\n });\n\n const ranked = all\n .map((e) => {\n const avgScore = e.recallCount > 0 ? clamp01(e.totalScore / e.recallCount) : 0;\n const reinforcement = clamp01(Math.log1p(e.recallCount) / Math.log1p(10)) * 0.12;\n const score = clamp01(avgScore + reinforcement);\n return { ...e, avgScore, score };\n })\n .sort(compare);\n\n const existing = await fs.readFile(memoryPath, 'utf-8').catch((err: unknown) => {\n if ((err as NodeJS.ErrnoException | undefined)?.code === 'ENOENT') return '';\n throw err;\n });\n const markers = extractPromotionMarkers(existing);\n\n const limit = Math.min(Math.max(params.limit ?? 20, 1), 50);\n const out: PreviewItem[] = [];\n\n // Scan more than limit so filtering doesn’t yield empty previews.\n const scanCap = Math.min(ranked.length, Math.max(limit * 3, limit));\n for (const candidate of ranked.slice(0, scanCap)) {\n const alreadyPromotedByKey = markers.keys.has(candidate.key);\n if (alreadyPromotedByKey) {\n out.push({\n key: candidate.key,\n hash: '',\n snippet: '',\n path: candidate.path,\n startLine: candidate.startLine,\n endLine: candidate.endLine,\n score: candidate.score,\n avgScore: candidate.avgScore,\n recallCount: candidate.recallCount,\n alreadyPromotedByKey: true,\n alreadyPromotedByHash: false,\n skippedReason: 'already promoted (key)',\n });\n continue;\n }\n\n const fullPath = path.join(params.workspaceDir, candidate.path);\n const lines = await readFileLines(fullPath);\n if (!lines) continue;\n const startLine = Math.max(1, Math.floor(candidate.startLine));\n const endLine = Math.max(startLine, Math.floor(candidate.endLine));\n const snippet = sliceRange(lines, startLine, endLine);\n if (!snippet) continue;\n if (isContaminatedSnippet(snippet)) continue;\n const hash = snippetHash(snippet);\n const alreadyPromotedByHash = markers.hashes.has(hash);\n out.push({\n key: candidate.key,\n hash,\n snippet,\n path: candidate.path,\n startLine,\n endLine,\n score: candidate.score,\n avgScore: candidate.avgScore,\n recallCount: candidate.recallCount,\n alreadyPromotedByKey: false,\n alreadyPromotedByHash,\n skippedReason: alreadyPromotedByHash ? 'already promoted (hash)' : null,\n });\n if (out.filter((x) => !x.skippedReason).length >= limit) break;\n }\n\n return { ok: true, reason: 'ok', items: out, memoryPath };\n}\n\n"],"mappings":";;;;;;AAuBA,SAAS,QAAQ,OAAuB;AACtC,KAAI,CAAC,OAAO,SAAS,MAAM,CAAE,QAAO;AACpC,QAAO,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,MAAM,CAAC;;AAGxC,SAAS,QAAQ,GAAiF,GAAyF;AACzL,KAAI,EAAE,UAAU,EAAE,MAAO,QAAO,EAAE,QAAQ,EAAE;AAC5C,KAAI,EAAE,gBAAgB,EAAE,YAAa,QAAO,EAAE,cAAc,EAAE;CAC9D,MAAM,MAAM,KAAK,MAAM,EAAE,eAAe;CACxC,MAAM,MAAM,KAAK,MAAM,EAAE,eAAe;AACxC,KAAI,OAAO,SAAS,IAAI,IAAI,OAAO,SAAS,IAAI;MAC1C,QAAQ,IAAK,QAAO,MAAM;;AAEhC,QAAO,EAAE,KAAK,cAAc,EAAE,KAAK;;AAGrC,eAAe,cAAc,UAA4C;AACvE,KAAI;AAEF,UAAO,MADW,GAAG,SAAS,UAAU,QAAQ,EACrC,MAAM,QAAQ;UAClB,KAAK;AAEZ,MADc,KAA2C,SAC5C,SAAU,QAAO;AAC9B,QAAM;;;AAIV,SAAS,WAAW,OAAiB,WAAmB,SAAyB;CAC/E,MAAM,WAAW,KAAK,IAAI,GAAG,YAAY,EAAE;CAC3C,MAAM,SAAS,KAAK,IAAI,MAAM,QAAQ,QAAQ;AAC9C,KAAI,YAAY,OAAQ,QAAO;AAC/B,QAAO,MACJ,MAAM,UAAU,OAAO,CACvB,KAAK,MAAM,EAAE,MAAM,CAAC,CACpB,OAAO,QAAQ,CACf,KAAK,IAAI,CACT,QAAQ,QAAQ,IAAI,CACpB,MAAM,CACN,MAAM,GAAG,IAAI;;AAGlB,SAAS,wBAAwB,SAAyB;AACxD,QAAO,QACJ,MAAM,CACN,QAAQ,QAAQ,IAAI,CACpB,QAAQ,SAAS,KAAI,CACrB,QAAQ,SAAS,IAAI,CACrB,aAAa,CACb,MAAM,GAAG,IAAI;;AAGlB,SAAS,YAAY,SAAyB;AAC5C,QAAO,WAAW,OAAO,CAAC,OAAO,wBAAwB,QAAQ,CAAC,CAAC,OAAO,MAAM,CAAC,MAAM,GAAG,GAAG;;AAG/F,SAAS,wBAAwB,YAAgE;CAC/F,MAAM,uBAAO,IAAI,KAAa;CAC9B,MAAM,yBAAS,IAAI,KAAa;AAChC,MAAK,MAAM,SAAS,WAAW,SAAS,gDAAgD,EAAE;EACxF,MAAM,OAAO,MAAM,MAAM;EACzB,MAAM,IAAI,KAAK,MAAM,yBAAyB,GAAG,IAAI,MAAM;EAC3D,MAAM,IAAI,KAAK,MAAM,0BAA0B,GAAG,IAAI,MAAM;AAC5D,MAAI,EAAG,MAAK,IAAI,EAAE;AAClB,MAAI,EAAG,QAAO,IAAI,EAAE;;AAEtB,MAAK,MAAM,SAAS,WAAW,SAAS,iDAAiD,EAAE;EACzF,MAAM,MAAM,MAAM,IAAI,MAAM;AAC5B,MAAI,IAAK,MAAK,IAAI,IAAI;;AAExB,QAAO;EAAE;EAAM;EAAQ;;AAGzB,SAAS,sBAAsB,SAA0B;CACvD,MAAM,IAAI,QAAQ,MAAM;AACxB,KAAI,CAAC,EAAG,QAAO;CACf,MAAM,QAAQ,EAAE,aAAa;AAY7B,KAAI;EAVF;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EAEU,CAAC,MAAM,MAAM,EAAE,KAAK,EAAE,CAAC,CAAE,QAAO;AAE5C,MADoB,EAAE,MAAM,WAAW,IAAI,EAAE,EAAE,UAC7B,GAAI,QAAO;AAE7B,MADkB,MAAM,MAAM,eAAe,IAAI,EAAE,EAAE,UACrC,EAAG,QAAO;AAC1B,QAAO;;AAGT,SAAS,qBAAqB,WAA6D;CACzF,MAAM,UAAU,WAAW,YAAY;CACvC,MAAM,WAAW,OAAO,WAAW,aAAa,WAAW,UAAU,WAAW;CAChF,MAAM,iBAAiB,OAAO,WAAW,mBAAmB,WAAW,UAAU,iBAAiB;CAClG,MAAM,QAAQ,OAAO,WAAW,UAAU,WAAW,UAAU,QAAQ;AACvE,QAAO;EACL;EACA,UAAU,QAAQ,SAAS;EAC3B,gBAAgB,KAAK,IAAI,GAAG,KAAK,MAAM,eAAe,CAAC;EACvD,OAAO,KAAK,IAAI,GAAG,KAAK,MAAM,MAAM,CAAC;EACtC;;AAGH,eAAsB,6BAA6B,QAIoC;CACrF,MAAM,MAAM,qBAAqB,OAAO,OAAO;CAC/C,MAAM,aAAa,KAAK,KAAK,OAAO,cAAc,mBAAmB;AACrE,KAAI,CAAC,IAAI,QAAS,QAAO;EAAE,IAAI;EAAM,QAAQ;EAAqB,OAAO,EAAE;EAAE;EAAY;CAEzF,MAAM,EAAE,UAAU,MAAM,kBAAkB,EAAE,cAAc,OAAO,cAAc,CAAC;CAUhF,MAAM,SATM,OAAO,OAAO,MAAM,WAAW,EAAE,CAAC,CAAC,QAAQ,MAA+B;AACpF,MAAI,CAAC,KAAK,OAAO,MAAM,SAAU,QAAO;AACxC,MAAI,EAAE,WAAY,QAAO;AACzB,MAAI,CAAC,EAAE,QAAQ,CAAC,EAAE,KAAK,WAAW,UAAU,CAAE,QAAO;AACrD,MAAI,EAAE,cAAc,IAAI,eAAgB,QAAO;AAE/C,SAAO,QADK,EAAE,cAAc,IAAI,EAAE,aAAa,EAAE,cAAc,EAC5C,IAAI,IAAI;GAGX,CACf,KAAK,MAAM;EACV,MAAM,WAAW,EAAE,cAAc,IAAI,QAAQ,EAAE,aAAa,EAAE,YAAY,GAAG;EAE7E,MAAM,QAAQ,QAAQ,WADA,QAAQ,KAAK,MAAM,EAAE,YAAY,GAAG,KAAK,MAAM,GAAG,CAAC,GAAG,IAC7B;AAC/C,SAAO;GAAE,GAAG;GAAG;GAAU;GAAO;GAChC,CACD,KAAK,QAAQ;CAMhB,MAAM,UAAU,wBAAwB,MAJjB,GAAG,SAAS,YAAY,QAAQ,CAAC,OAAO,QAAiB;AAC9E,MAAK,KAA2C,SAAS,SAAU,QAAO;AAC1E,QAAM;GACN,CAC+C;CAEjD,MAAM,QAAQ,KAAK,IAAI,KAAK,IAAI,OAAO,SAAS,IAAI,EAAE,EAAE,GAAG;CAC3D,MAAM,MAAqB,EAAE;CAG7B,MAAM,UAAU,KAAK,IAAI,OAAO,QAAQ,KAAK,IAAI,QAAQ,GAAG,MAAM,CAAC;AACnE,MAAK,MAAM,aAAa,OAAO,MAAM,GAAG,QAAQ,EAAE;AAEhD,MAD6B,QAAQ,KAAK,IAAI,UAAU,IAChC,EAAE;AACxB,OAAI,KAAK;IACP,KAAK,UAAU;IACf,MAAM;IACN,SAAS;IACT,MAAM,UAAU;IAChB,WAAW,UAAU;IACrB,SAAS,UAAU;IACnB,OAAO,UAAU;IACjB,UAAU,UAAU;IACpB,aAAa,UAAU;IACvB,sBAAsB;IACtB,uBAAuB;IACvB,eAAe;IAChB,CAAC;AACF;;EAIF,MAAM,QAAQ,MAAM,cADH,KAAK,KAAK,OAAO,cAAc,UAAU,KAChB,CAAC;AAC3C,MAAI,CAAC,MAAO;EACZ,MAAM,YAAY,KAAK,IAAI,GAAG,KAAK,MAAM,UAAU,UAAU,CAAC;EAC9D,MAAM,UAAU,KAAK,IAAI,WAAW,KAAK,MAAM,UAAU,QAAQ,CAAC;EAClE,MAAM,UAAU,WAAW,OAAO,WAAW,QAAQ;AACrD,MAAI,CAAC,QAAS;AACd,MAAI,sBAAsB,QAAQ,CAAE;EACpC,MAAM,OAAO,YAAY,QAAQ;EACjC,MAAM,wBAAwB,QAAQ,OAAO,IAAI,KAAK;AACtD,MAAI,KAAK;GACP,KAAK,UAAU;GACf;GACA;GACA,MAAM,UAAU;GAChB;GACA;GACA,OAAO,UAAU;GACjB,UAAU,UAAU;GACpB,aAAa,UAAU;GACvB,sBAAsB;GACtB;GACA,eAAe,wBAAwB,4BAA4B;GACpE,CAAC;AACF,MAAI,IAAI,QAAQ,MAAM,CAAC,EAAE,cAAc,CAAC,UAAU,MAAO;;AAG3D,QAAO;EAAE,IAAI;EAAM,QAAQ;EAAM,OAAO;EAAK;EAAY"}
|
|
1
|
+
{"version":3,"file":"preview.js","names":[],"sources":["../../../../../src/agent/memory/dreaming/preview.ts"],"sourcesContent":["import fs from 'node:fs/promises';\nimport path from 'node:path';\n\nimport type { DreamingDeepConfig } from './config.js';\nimport { MEMORY_MD_FILENAME } from './constants.js';\nimport { loadDreamingStore, type DreamingStoreEntry } from './short-term-store.js';\nimport {\n clamp01,\n compareCandidatesByScore,\n computeCandidateScore,\n extractPromotionMarkers,\n isContaminatedSnippet,\n isExpiredEntry,\n readFileLines,\n resolveDeepDefaults,\n sliceRange,\n snippetHash,\n} from './utils.js';\n\ntype PreviewItem = {\n key: string;\n hash: string;\n snippet: string;\n path: string;\n startLine: number;\n endLine: number;\n score: number;\n avgScore: number;\n recallCount: number;\n recencyDecay: number;\n alreadyPromotedByKey: boolean;\n alreadyPromotedByHash: boolean;\n skippedReason: string | null;\n};\n\nexport async function previewDreamingDeepPromotion(params: {\n workspaceDir: string;\n config?: Partial<DreamingDeepConfig>;\n limit?: number;\n now?: Date;\n}): Promise<{ ok: boolean; reason: string; items: PreviewItem[]; memoryPath: string }> {\n const cfg = resolveDeepDefaults(params.config);\n const memoryPath = path.join(params.workspaceDir, MEMORY_MD_FILENAME);\n if (!cfg.enabled) return { ok: true, reason: 'dreaming disabled', items: [], memoryPath };\n\n const { store } = await loadDreamingStore({ workspaceDir: params.workspaceDir });\n const nowMs = (params.now ?? new Date()).getTime();\n\n const all = Object.values(store.entries ?? {}).filter((e): e is DreamingStoreEntry => {\n if (!e || typeof e !== 'object') return false;\n if (e.promotedAt) return false;\n if (!e.path || !e.path.startsWith('memory/')) return false;\n if (e.recallCount < cfg.minRecallCount) return false;\n if ((e.queryHashes?.length ?? 0) < cfg.minUniqueQueries) return false;\n if (isExpiredEntry(e.lastRecalledAt, nowMs, cfg.maxAgeDays)) return false;\n const avg = e.recallCount > 0 ? e.totalScore / e.recallCount : 0;\n return clamp01(avg) >= cfg.minScore;\n });\n\n const ranked = all\n .map((e) => {\n const { avgScore, score, recencyDecay } = computeCandidateScore(e, nowMs, cfg.recencyHalfLifeDays);\n return { ...e, avgScore, score, recencyDecay };\n })\n .sort(compareCandidatesByScore);\n\n const existing = await fs.readFile(memoryPath, 'utf-8').catch((err: unknown) => {\n if ((err as NodeJS.ErrnoException | undefined)?.code === 'ENOENT') return '';\n throw err;\n });\n const markers = extractPromotionMarkers(existing);\n\n const limit = Math.min(Math.max(params.limit ?? 20, 1), 50);\n const out: PreviewItem[] = [];\n\n // Scan more than limit so filtering doesn't yield empty previews.\n const scanCap = Math.min(ranked.length, Math.max(limit * 3, limit));\n for (const candidate of ranked.slice(0, scanCap)) {\n const alreadyPromotedByKey = markers.keys.has(candidate.key);\n if (alreadyPromotedByKey) {\n out.push({\n key: candidate.key,\n hash: '',\n snippet: '',\n path: candidate.path,\n startLine: candidate.startLine,\n endLine: candidate.endLine,\n score: candidate.score,\n avgScore: candidate.avgScore,\n recallCount: candidate.recallCount,\n recencyDecay: candidate.recencyDecay,\n alreadyPromotedByKey: true,\n alreadyPromotedByHash: false,\n skippedReason: 'already promoted (key)',\n });\n continue;\n }\n\n const fullPath = path.join(params.workspaceDir, candidate.path);\n const lines = await readFileLines(fullPath);\n if (!lines) continue;\n const startLine = Math.max(1, Math.floor(candidate.startLine));\n const endLine = Math.max(startLine, Math.floor(candidate.endLine));\n const snippet = sliceRange(lines, startLine, endLine);\n if (!snippet) continue;\n if (isContaminatedSnippet(snippet)) continue;\n const hash = snippetHash(snippet);\n const alreadyPromotedByHash = markers.hashes.has(hash);\n out.push({\n key: candidate.key,\n hash,\n snippet,\n path: candidate.path,\n startLine,\n endLine,\n score: candidate.score,\n avgScore: candidate.avgScore,\n recallCount: candidate.recallCount,\n recencyDecay: candidate.recencyDecay,\n alreadyPromotedByKey: false,\n alreadyPromotedByHash,\n skippedReason: alreadyPromotedByHash ? 'already promoted (hash)' : null,\n });\n if (out.filter((x) => !x.skippedReason).length >= limit) break;\n }\n\n return { ok: true, reason: 'ok', items: out, memoryPath };\n}\n"],"mappings":";;;;;;AAmCA,eAAsB,6BAA6B,QAKoC;CACrF,MAAM,MAAM,oBAAoB,OAAO,OAAO;CAC9C,MAAM,aAAa,KAAK,KAAK,OAAO,cAAc,mBAAmB;AACrE,KAAI,CAAC,IAAI,QAAS,QAAO;EAAE,IAAI;EAAM,QAAQ;EAAqB,OAAO,EAAE;EAAE;EAAY;CAEzF,MAAM,EAAE,UAAU,MAAM,kBAAkB,EAAE,cAAc,OAAO,cAAc,CAAC;CAChF,MAAM,SAAS,OAAO,uBAAO,IAAI,MAAM,EAAE,SAAS;CAalD,MAAM,SAXM,OAAO,OAAO,MAAM,WAAW,EAAE,CAAC,CAAC,QAAQ,MAA+B;AACpF,MAAI,CAAC,KAAK,OAAO,MAAM,SAAU,QAAO;AACxC,MAAI,EAAE,WAAY,QAAO;AACzB,MAAI,CAAC,EAAE,QAAQ,CAAC,EAAE,KAAK,WAAW,UAAU,CAAE,QAAO;AACrD,MAAI,EAAE,cAAc,IAAI,eAAgB,QAAO;AAC/C,OAAK,EAAE,aAAa,UAAU,KAAK,IAAI,iBAAkB,QAAO;AAChE,MAAI,eAAe,EAAE,gBAAgB,OAAO,IAAI,WAAW,CAAE,QAAO;AAEpE,SAAO,QADK,EAAE,cAAc,IAAI,EAAE,aAAa,EAAE,cAAc,EAC5C,IAAI,IAAI;GAGX,CACf,KAAK,MAAM;EACV,MAAM,EAAE,UAAU,OAAO,iBAAiB,sBAAsB,GAAG,OAAO,IAAI,oBAAoB;AAClG,SAAO;GAAE,GAAG;GAAG;GAAU;GAAO;GAAc;GAC9C,CACD,KAAK,yBAAyB;CAMjC,MAAM,UAAU,wBAAwB,MAJjB,GAAG,SAAS,YAAY,QAAQ,CAAC,OAAO,QAAiB;AAC9E,MAAK,KAA2C,SAAS,SAAU,QAAO;AAC1E,QAAM;GACN,CAC+C;CAEjD,MAAM,QAAQ,KAAK,IAAI,KAAK,IAAI,OAAO,SAAS,IAAI,EAAE,EAAE,GAAG;CAC3D,MAAM,MAAqB,EAAE;CAG7B,MAAM,UAAU,KAAK,IAAI,OAAO,QAAQ,KAAK,IAAI,QAAQ,GAAG,MAAM,CAAC;AACnE,MAAK,MAAM,aAAa,OAAO,MAAM,GAAG,QAAQ,EAAE;AAEhD,MAD6B,QAAQ,KAAK,IAAI,UAAU,IAChC,EAAE;AACxB,OAAI,KAAK;IACP,KAAK,UAAU;IACf,MAAM;IACN,SAAS;IACT,MAAM,UAAU;IAChB,WAAW,UAAU;IACrB,SAAS,UAAU;IACnB,OAAO,UAAU;IACjB,UAAU,UAAU;IACpB,aAAa,UAAU;IACvB,cAAc,UAAU;IACxB,sBAAsB;IACtB,uBAAuB;IACvB,eAAe;IAChB,CAAC;AACF;;EAIF,MAAM,QAAQ,MAAM,cADH,KAAK,KAAK,OAAO,cAAc,UAAU,KAChB,CAAC;AAC3C,MAAI,CAAC,MAAO;EACZ,MAAM,YAAY,KAAK,IAAI,GAAG,KAAK,MAAM,UAAU,UAAU,CAAC;EAC9D,MAAM,UAAU,KAAK,IAAI,WAAW,KAAK,MAAM,UAAU,QAAQ,CAAC;EAClE,MAAM,UAAU,WAAW,OAAO,WAAW,QAAQ;AACrD,MAAI,CAAC,QAAS;AACd,MAAI,sBAAsB,QAAQ,CAAE;EACpC,MAAM,OAAO,YAAY,QAAQ;EACjC,MAAM,wBAAwB,QAAQ,OAAO,IAAI,KAAK;AACtD,MAAI,KAAK;GACP,KAAK,UAAU;GACf;GACA;GACA,MAAM,UAAU;GAChB;GACA;GACA,OAAO,UAAU;GACjB,UAAU,UAAU;GACpB,aAAa,UAAU;GACvB,cAAc,UAAU;GACxB,sBAAsB;GACtB;GACA,eAAe,wBAAwB,4BAA4B;GACpE,CAAC;AACF,MAAI,IAAI,QAAQ,MAAM,CAAC,EAAE,cAAc,CAAC,UAAU,MAAO;;AAG3D,QAAO;EAAE,IAAI;EAAM,QAAQ;EAAM,OAAO;EAAK;EAAY"}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { DreamingRemConfig } from './config.js';
|
|
2
|
+
/**
|
|
3
|
+
* REM phase: cross-session pattern discovery.
|
|
4
|
+
*
|
|
5
|
+
* Scans the short-term store for entries that share query hashes or
|
|
6
|
+
* appear across multiple daily files, then clusters them to identify
|
|
7
|
+
* recurring themes/patterns. Bumps `remHits` on touched entries and
|
|
8
|
+
* optionally writes a pattern summary to DREAMS.md.
|
|
9
|
+
*
|
|
10
|
+
* Runs weekly; expensive but insightful.
|
|
11
|
+
*/
|
|
12
|
+
export declare function runRemPatterns(params: {
|
|
13
|
+
workspaceDir: string;
|
|
14
|
+
config?: Partial<DreamingRemConfig>;
|
|
15
|
+
now?: Date;
|
|
16
|
+
}): Promise<{
|
|
17
|
+
ok: boolean;
|
|
18
|
+
reason: string;
|
|
19
|
+
patternsDiscovered: number;
|
|
20
|
+
entriesAnalyzed: number;
|
|
21
|
+
}>;
|