dsh-tdai-memory 0.4.2 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +8 -0
- package/README.zh.md +8 -0
- package/THIRD-PARTY-NOTICES.md +20 -20
- package/cordis.patch.yml +8 -8
- package/package.json +103 -100
- package/vendor/tdai/adapters/index.js +15 -15
- package/vendor/tdai/adapters/openclaw/host-adapter.js +83 -83
- package/vendor/tdai/adapters/openclaw/index.js +5 -5
- package/vendor/tdai/adapters/openclaw/llm-runner.js +69 -69
- package/vendor/tdai/adapters/standalone/host-adapter.js +58 -58
- package/vendor/tdai/adapters/standalone/index.js +5 -5
- package/vendor/tdai/core/conversation/l0-recorder.js +420 -420
- package/vendor/tdai/core/hooks/auto-capture.js +236 -236
- package/vendor/tdai/core/hooks/auto-recall.js +640 -640
- package/vendor/tdai/core/index.js +8 -8
- package/vendor/tdai/core/persona/persona-generator.js +184 -184
- package/vendor/tdai/core/persona/persona-trigger.js +94 -94
- package/vendor/tdai/core/profile/profile-sync.js +200 -200
- package/vendor/tdai/core/prompts/l1-dedup.js +72 -72
- package/vendor/tdai/core/prompts/l1-extraction.js +34 -34
- package/vendor/tdai/core/prompts/persona-generation.js +35 -35
- package/vendor/tdai/core/prompts/scene-extraction.js +45 -45
- package/vendor/tdai/core/record/l1-dedup.js +318 -318
- package/vendor/tdai/core/record/l1-extractor.js +379 -379
- package/vendor/tdai/core/record/l1-reader.js +178 -178
- package/vendor/tdai/core/record/l1-writer.js +141 -141
- package/vendor/tdai/core/report/reporter.js +86 -86
- package/vendor/tdai/core/scene/filename-normalizer.js +161 -161
- package/vendor/tdai/core/scene/scene-extractor.js +410 -410
- package/vendor/tdai/core/scene/scene-format.js +53 -53
- package/vendor/tdai/core/scene/scene-index.js +82 -82
- package/vendor/tdai/core/scene/scene-navigation.js +57 -57
- package/vendor/tdai/core/seed/input.js +404 -404
- package/vendor/tdai/core/seed/types.js +9 -9
- package/vendor/tdai/core/store/bm25-client.js +117 -117
- package/vendor/tdai/core/store/bm25-local.js +68 -68
- package/vendor/tdai/core/store/embedding.js +632 -632
- package/vendor/tdai/core/store/factory.js +95 -95
- package/vendor/tdai/core/store/search-utils.js +54 -54
- package/vendor/tdai/core/store/sqlite.js +1632 -1632
- package/vendor/tdai/core/store/tcvdb-client.js +221 -221
- package/vendor/tdai/core/store/tcvdb.js +1047 -1047
- package/vendor/tdai/core/store/types.js +17 -17
- package/vendor/tdai/core/tools/conversation-search.js +199 -199
- package/vendor/tdai/core/tools/memory-search.js +207 -207
- package/vendor/tdai/core/types.js +12 -12
- package/vendor/tdai/utils/backup.js +189 -189
- package/vendor/tdai/utils/checkpoint.js +360 -360
- package/vendor/tdai/utils/clean-context-runner.js +472 -472
- package/vendor/tdai/utils/ensure-hook-policy.js +213 -213
- package/vendor/tdai/utils/env.js +13 -13
- package/vendor/tdai/utils/managed-timer.js +127 -127
- package/vendor/tdai/utils/manifest.js +91 -91
- package/vendor/tdai/utils/memory-cleaner.js +300 -300
- package/vendor/tdai/utils/no-think-fetch.js +109 -109
- package/vendor/tdai/utils/openclaw-state-dir.js +26 -26
- package/vendor/tdai/utils/pipeline-factory.js +517 -517
- package/vendor/tdai/utils/pipeline-manager.js +892 -892
- package/vendor/tdai/utils/sanitize.js +360 -360
- package/vendor/tdai/utils/serial-queue.js +103 -103
- package/vendor/tdai/utils/session-filter.js +88 -88
- package/vendor/tdai/utils/text-utils.js +29 -29
- package/vendor/tdai/utils/time.js +252 -252
|
@@ -1,300 +1,300 @@
|
|
|
1
|
-
import fs from "node:fs/promises";
|
|
2
|
-
import path from "node:path";
|
|
3
|
-
import { ManagedTimer } from "./managed-timer.js";
|
|
4
|
-
import { formatLocalDateTime, startOfLocalDay } from "./time.js";
|
|
5
|
-
const TAG = "[memory-tdai][cleaner]";
|
|
6
|
-
const L0_DIR_NAME = "conversations";
|
|
7
|
-
const L1_DIR_NAME = "records";
|
|
8
|
-
/** Minimum records to retain — skip deletion if total is at or below this threshold. */
|
|
9
|
-
const MIN_RETAIN_L0 = 50;
|
|
10
|
-
const MIN_RETAIN_L1 = 20;
|
|
11
|
-
export class LocalMemoryCleaner {
|
|
12
|
-
opts;
|
|
13
|
-
timer;
|
|
14
|
-
destroyed = false;
|
|
15
|
-
vectorStore;
|
|
16
|
-
constructor(opts) {
|
|
17
|
-
this.opts = opts;
|
|
18
|
-
this.timer = new ManagedTimer("memory-tdai-cleaner", () => this.destroyed);
|
|
19
|
-
this.vectorStore = opts.vectorStore;
|
|
20
|
-
}
|
|
21
|
-
setVectorStore(vectorStore) {
|
|
22
|
-
this.vectorStore = vectorStore;
|
|
23
|
-
}
|
|
24
|
-
start() {
|
|
25
|
-
if (this.destroyed)
|
|
26
|
-
return;
|
|
27
|
-
const now = new Date();
|
|
28
|
-
const tz = Intl.DateTimeFormat().resolvedOptions().timeZone || "unknown";
|
|
29
|
-
const utcOffset = formatUtcOffset(-now.getTimezoneOffset());
|
|
30
|
-
this.opts.logger?.debug?.(`${TAG} Enabled: retentionDays=${this.opts.retentionDays}, cleanTime=${this.opts.cleanTime}, dirs=[${L0_DIR_NAME}, ${L1_DIR_NAME}]`);
|
|
31
|
-
this.opts.logger?.debug?.(`${TAG} Runtime clock: nowLocal=${formatLocalDateTime(now)}, nowIso=${now.toISOString()}, tz=${tz}, utcOffset=${utcOffset}`);
|
|
32
|
-
this.scheduleNext();
|
|
33
|
-
}
|
|
34
|
-
destroy() {
|
|
35
|
-
if (this.destroyed)
|
|
36
|
-
return;
|
|
37
|
-
this.destroyed = true;
|
|
38
|
-
this.timer.cancel();
|
|
39
|
-
this.opts.logger?.info(`${TAG} Stopped`);
|
|
40
|
-
}
|
|
41
|
-
async runOnce(nowMs = Date.now()) {
|
|
42
|
-
if (this.destroyed)
|
|
43
|
-
return;
|
|
44
|
-
const retentionDays = this.opts.retentionDays;
|
|
45
|
-
if (!(retentionDays > 0)) {
|
|
46
|
-
this.opts.logger?.debug?.(`${TAG} Skip run: invalid retentionDays=${retentionDays}`);
|
|
47
|
-
return;
|
|
48
|
-
}
|
|
49
|
-
// 按"本地自然日"保留策略计算截止时间。
|
|
50
|
-
// 例如 retentionDays=2,今天是 03-15,则保留 03-14/03-15,删除早于 03-14 00:00:00.000 的记录。
|
|
51
|
-
let cutoffMs;
|
|
52
|
-
try {
|
|
53
|
-
cutoffMs = computeCutoffMsByLocalDay(nowMs, retentionDays);
|
|
54
|
-
}
|
|
55
|
-
catch (err) {
|
|
56
|
-
this.opts.logger?.error(`${TAG} ${err instanceof Error ? err.message : String(err)}`);
|
|
57
|
-
return;
|
|
58
|
-
}
|
|
59
|
-
const targetDirs = [
|
|
60
|
-
path.join(this.opts.baseDir, L0_DIR_NAME),
|
|
61
|
-
path.join(this.opts.baseDir, L1_DIR_NAME),
|
|
62
|
-
];
|
|
63
|
-
const total = {
|
|
64
|
-
scannedFiles: 0,
|
|
65
|
-
changedFiles: 0,
|
|
66
|
-
skippedNonShardFiles: 0,
|
|
67
|
-
deleteFailedFiles: 0,
|
|
68
|
-
};
|
|
69
|
-
for (const dirPath of targetDirs) {
|
|
70
|
-
const stats = await this.cleanDirectory(dirPath, cutoffMs);
|
|
71
|
-
total.scannedFiles += stats.scannedFiles;
|
|
72
|
-
total.changedFiles += stats.changedFiles;
|
|
73
|
-
total.skippedNonShardFiles += stats.skippedNonShardFiles;
|
|
74
|
-
total.deleteFailedFiles += stats.deleteFailedFiles;
|
|
75
|
-
}
|
|
76
|
-
if (this.vectorStore) {
|
|
77
|
-
const vectorStore = this.vectorStore;
|
|
78
|
-
const cutoffIso = new Date(cutoffMs).toISOString();
|
|
79
|
-
const startMs = Date.now();
|
|
80
|
-
// ── Pre-delete: count totals and decide whether to proceed ──
|
|
81
|
-
let totalL0 = 0;
|
|
82
|
-
let totalL1 = 0;
|
|
83
|
-
try {
|
|
84
|
-
totalL0 = await vectorStore.countL0();
|
|
85
|
-
}
|
|
86
|
-
catch { /* non-fatal */ }
|
|
87
|
-
try {
|
|
88
|
-
totalL1 = await vectorStore.countL1();
|
|
89
|
-
}
|
|
90
|
-
catch { /* non-fatal */ }
|
|
91
|
-
this.opts.logger?.info(`${TAG} [Pre-delete] cutoffIso=${cutoffIso}, retentionDays=${retentionDays}, totalL0=${totalL0}, totalL1=${totalL1}`);
|
|
92
|
-
let removedL0 = 0;
|
|
93
|
-
let removedL1 = 0;
|
|
94
|
-
let skippedL0 = false;
|
|
95
|
-
let skippedL1 = false;
|
|
96
|
-
let failedL0DbCleanup = 0;
|
|
97
|
-
let failedL1DbCleanup = 0;
|
|
98
|
-
// ── L0 cleanup with minimum-retention guard ──
|
|
99
|
-
if (totalL0 <= MIN_RETAIN_L0) {
|
|
100
|
-
skippedL0 = true;
|
|
101
|
-
this.opts.logger?.info(`${TAG} [L0-delete] SKIPPED: totalL0=${totalL0} <= minRetain=${MIN_RETAIN_L0}`);
|
|
102
|
-
}
|
|
103
|
-
else {
|
|
104
|
-
try {
|
|
105
|
-
removedL0 = await vectorStore.deleteL0Expired(cutoffIso);
|
|
106
|
-
}
|
|
107
|
-
catch (err) {
|
|
108
|
-
failedL0DbCleanup = 1;
|
|
109
|
-
this.opts.logger?.warn(`${TAG} [L0-delete] FAILED: ${err instanceof Error ? err.message : String(err)}`);
|
|
110
|
-
}
|
|
111
|
-
}
|
|
112
|
-
// ── L1 cleanup with minimum-retention guard ──
|
|
113
|
-
if (totalL1 <= MIN_RETAIN_L1) {
|
|
114
|
-
skippedL1 = true;
|
|
115
|
-
this.opts.logger?.info(`${TAG} [L1-delete] SKIPPED: totalL1=${totalL1} <= minRetain=${MIN_RETAIN_L1}`);
|
|
116
|
-
}
|
|
117
|
-
else {
|
|
118
|
-
try {
|
|
119
|
-
removedL1 = await vectorStore.deleteL1Expired(cutoffIso);
|
|
120
|
-
}
|
|
121
|
-
catch (err) {
|
|
122
|
-
failedL1DbCleanup = 1;
|
|
123
|
-
this.opts.logger?.warn(`${TAG} [L1-delete] FAILED: ${err instanceof Error ? err.message : String(err)}`);
|
|
124
|
-
}
|
|
125
|
-
}
|
|
126
|
-
if (removedL1 > 0 || removedL0 > 0) {
|
|
127
|
-
total.changedFiles += 1;
|
|
128
|
-
}
|
|
129
|
-
// ── Post-delete: audit summary ──
|
|
130
|
-
const durationMs = Date.now() - startMs;
|
|
131
|
-
const remainingL0 = totalL0 - removedL0;
|
|
132
|
-
const remainingL1 = totalL1 - removedL1;
|
|
133
|
-
const summary = {
|
|
134
|
-
event: "cleaner_summary",
|
|
135
|
-
cutoffIso,
|
|
136
|
-
retentionDays,
|
|
137
|
-
l0: { total: totalL0, expired: removedL0, remaining: remainingL0, skipped: skippedL0, failed: failedL0DbCleanup > 0 },
|
|
138
|
-
l1: { total: totalL1, expired: removedL1, remaining: remainingL1, skipped: skippedL1, failed: failedL1DbCleanup > 0 },
|
|
139
|
-
durationMs,
|
|
140
|
-
};
|
|
141
|
-
this.opts.logger?.info(`${TAG} ${JSON.stringify(summary)}`);
|
|
142
|
-
}
|
|
143
|
-
this.opts.logger?.info(`${TAG} Cleanup done: scannedFiles=${total.scannedFiles}, changedFiles=${total.changedFiles}, skippedNonShardFiles=${total.skippedNonShardFiles}, deleteFailedFiles=${total.deleteFailedFiles}`);
|
|
144
|
-
}
|
|
145
|
-
scheduleNext() {
|
|
146
|
-
const nowMs = Date.now();
|
|
147
|
-
const now = new Date(nowMs);
|
|
148
|
-
const next = nextRunAt(this.opts.cleanTime, nowMs);
|
|
149
|
-
const targetToday = buildTodayRunTime(this.opts.cleanTime, nowMs);
|
|
150
|
-
const passedToday = targetToday <= nowMs;
|
|
151
|
-
const delayMs = Math.max(0, next - nowMs);
|
|
152
|
-
this.opts.logger?.debug?.(`${TAG} Schedule next run: nowLocal=${formatLocalDateTime(now)}, cleanTime=${this.opts.cleanTime}, targetTodayLocal=${formatLocalDateTime(new Date(targetToday))}, passedToday=${passedToday}, nextRunLocal=${formatLocalDateTime(new Date(next))}, nextRunIso=${new Date(next).toISOString()}, delayMs=${delayMs}`);
|
|
153
|
-
this.timer.scheduleAt(next, () => {
|
|
154
|
-
const firedAtMs = Date.now();
|
|
155
|
-
this.opts.logger?.info(`${TAG} Timer fired: scheduledLocal=${formatLocalDateTime(new Date(next))}, firedLocal=${formatLocalDateTime(new Date(firedAtMs))}, driftMs=${firedAtMs - next}`);
|
|
156
|
-
void this.runAndReschedule();
|
|
157
|
-
});
|
|
158
|
-
}
|
|
159
|
-
async runAndReschedule() {
|
|
160
|
-
if (this.destroyed)
|
|
161
|
-
return;
|
|
162
|
-
const runStart = new Date();
|
|
163
|
-
this.opts.logger?.info(`${TAG} Cleanup tick start: nowLocal=${formatLocalDateTime(runStart)}, nowIso=${runStart.toISOString()}`);
|
|
164
|
-
try {
|
|
165
|
-
await this.runOnce();
|
|
166
|
-
}
|
|
167
|
-
catch (err) {
|
|
168
|
-
this.opts.logger?.error(`${TAG} Cleanup failed: ${err instanceof Error ? err.stack ?? err.message : String(err)}`);
|
|
169
|
-
}
|
|
170
|
-
finally {
|
|
171
|
-
if (!this.destroyed) {
|
|
172
|
-
this.scheduleNext();
|
|
173
|
-
}
|
|
174
|
-
}
|
|
175
|
-
}
|
|
176
|
-
async cleanDirectory(dirPath, cutoffMs) {
|
|
177
|
-
const stats = {
|
|
178
|
-
scannedFiles: 0,
|
|
179
|
-
changedFiles: 0,
|
|
180
|
-
skippedNonShardFiles: 0,
|
|
181
|
-
deleteFailedFiles: 0,
|
|
182
|
-
};
|
|
183
|
-
let entries;
|
|
184
|
-
try {
|
|
185
|
-
entries = await fs.readdir(dirPath, { withFileTypes: true });
|
|
186
|
-
}
|
|
187
|
-
catch {
|
|
188
|
-
this.opts.logger?.debug?.(`${TAG} Directory not found, skip: ${dirPath}`);
|
|
189
|
-
return stats;
|
|
190
|
-
}
|
|
191
|
-
for (const entry of entries) {
|
|
192
|
-
if (!entry.isFile())
|
|
193
|
-
continue;
|
|
194
|
-
if (!isJsonLikeFile(entry.name))
|
|
195
|
-
continue;
|
|
196
|
-
const filePath = path.join(dirPath, entry.name);
|
|
197
|
-
stats.scannedFiles += 1;
|
|
198
|
-
// 仅支持日期分片文件:YYYY-MM-DD(.jsonl/.json)
|
|
199
|
-
const shard = extractShardDateFromFileName(entry.name);
|
|
200
|
-
if (!shard) {
|
|
201
|
-
stats.skippedNonShardFiles += 1;
|
|
202
|
-
this.opts.logger?.debug?.(`${TAG} Skip non-shard file: ${filePath}`);
|
|
203
|
-
continue;
|
|
204
|
-
}
|
|
205
|
-
const dayEndMs = localDayEndMs(shard.year, shard.month, shard.day);
|
|
206
|
-
if (dayEndMs < cutoffMs) {
|
|
207
|
-
try {
|
|
208
|
-
await fs.unlink(filePath);
|
|
209
|
-
stats.changedFiles += 1;
|
|
210
|
-
this.opts.logger?.info(`${TAG} Removed expired file by name: ${filePath}`);
|
|
211
|
-
}
|
|
212
|
-
catch (err) {
|
|
213
|
-
stats.deleteFailedFiles += 1;
|
|
214
|
-
this.opts.logger?.warn(`${TAG} Failed to delete expired shard file ${filePath}: ${err instanceof Error ? err.message : String(err)}`);
|
|
215
|
-
}
|
|
216
|
-
}
|
|
217
|
-
else {
|
|
218
|
-
this.opts.logger?.debug?.(`${TAG} Keep shard file by name: ${filePath}`);
|
|
219
|
-
}
|
|
220
|
-
}
|
|
221
|
-
return stats;
|
|
222
|
-
}
|
|
223
|
-
}
|
|
224
|
-
function isJsonLikeFile(name) {
|
|
225
|
-
return name.endsWith(".jsonl") || name.endsWith(".json");
|
|
226
|
-
}
|
|
227
|
-
function extractShardDateFromFileName(fileName) {
|
|
228
|
-
// Supported format: YYYY-MM-DD.jsonl | YYYY-MM-DD.json
|
|
229
|
-
const m = /^(\d{4})-(\d{2})-(\d{2})\.(?:jsonl|json)$/.exec(fileName);
|
|
230
|
-
if (!m)
|
|
231
|
-
return undefined;
|
|
232
|
-
const year = Number(m[1]);
|
|
233
|
-
const month = Number(m[2]);
|
|
234
|
-
const day = Number(m[3]);
|
|
235
|
-
if (!Number.isInteger(year) || !Number.isInteger(month) || !Number.isInteger(day)) {
|
|
236
|
-
return undefined;
|
|
237
|
-
}
|
|
238
|
-
if (month < 1 || month > 12 || day < 1 || day > 31) {
|
|
239
|
-
return undefined;
|
|
240
|
-
}
|
|
241
|
-
const probe = new Date(year, month - 1, day);
|
|
242
|
-
if (probe.getFullYear() !== year
|
|
243
|
-
|| probe.getMonth() !== month - 1
|
|
244
|
-
|| probe.getDate() !== day) {
|
|
245
|
-
return undefined;
|
|
246
|
-
}
|
|
247
|
-
return { year, month, day };
|
|
248
|
-
}
|
|
249
|
-
function localDayEndMs(year, month, day) {
|
|
250
|
-
// End of day = start of next day minus 1ms (in configured timezone)
|
|
251
|
-
const nextDay = new Date(Date.UTC(year, month - 1, day + 1));
|
|
252
|
-
const nextDayStartMs = startOfLocalDay(nextDay);
|
|
253
|
-
return nextDayStartMs - 1;
|
|
254
|
-
}
|
|
255
|
-
function formatUtcOffset(offsetMinutes) {
|
|
256
|
-
const sign = offsetMinutes >= 0 ? "+" : "-";
|
|
257
|
-
const abs = Math.abs(offsetMinutes);
|
|
258
|
-
const hh = String(Math.floor(abs / 60)).padStart(2, "0");
|
|
259
|
-
const mm = String(abs % 60).padStart(2, "0");
|
|
260
|
-
return `${sign}${hh}:${mm}`;
|
|
261
|
-
}
|
|
262
|
-
function computeCutoffMsByLocalDay(nowMs, retentionDays) {
|
|
263
|
-
// 自然日策略,保留"今天 + 往前 retentionDays-1 天"
|
|
264
|
-
// 删除阈值为 keepStart 当天 00:00:00.000(配置时区)
|
|
265
|
-
const now = new Date(nowMs);
|
|
266
|
-
const todayStartMs = startOfLocalDay(now);
|
|
267
|
-
const cutoffMs = todayStartMs - (retentionDays - 1) * 24 * 60 * 60 * 1000;
|
|
268
|
-
// Sanity check: cutoff must be strictly in the past
|
|
269
|
-
if (cutoffMs >= nowMs) {
|
|
270
|
-
throw new Error(`cutoff sanity failed: cutoff (${cutoffMs}) >= now (${nowMs}), ` +
|
|
271
|
-
`possible clock skew or invalid retentionDays=${retentionDays}`);
|
|
272
|
-
}
|
|
273
|
-
// Sanity check: gap between now and cutoff must be at least 24h
|
|
274
|
-
const MIN_GAP_MS = 24 * 60 * 60 * 1000;
|
|
275
|
-
if (nowMs - cutoffMs < MIN_GAP_MS) {
|
|
276
|
-
throw new Error(`cutoff sanity failed: gap ${nowMs - cutoffMs}ms < 24h, ` +
|
|
277
|
-
`retentionDays=${retentionDays}, possible clock skew`);
|
|
278
|
-
}
|
|
279
|
-
return cutoffMs;
|
|
280
|
-
}
|
|
281
|
-
function buildTodayRunTime(cleanTime, nowMs) {
|
|
282
|
-
const [hRaw, mRaw] = cleanTime.split(":");
|
|
283
|
-
const hour = Number(hRaw);
|
|
284
|
-
const minute = Number(mRaw);
|
|
285
|
-
const target = new Date(nowMs);
|
|
286
|
-
target.setHours(hour, minute, 0, 0);
|
|
287
|
-
return target.getTime();
|
|
288
|
-
}
|
|
289
|
-
function nextRunAt(cleanTime, nowMs) {
|
|
290
|
-
const [hRaw, mRaw] = cleanTime.split(":");
|
|
291
|
-
const hour = Number(hRaw);
|
|
292
|
-
const minute = Number(mRaw);
|
|
293
|
-
const now = new Date(nowMs);
|
|
294
|
-
const next = new Date(nowMs);
|
|
295
|
-
next.setHours(hour, minute, 0, 0);
|
|
296
|
-
if (next.getTime() <= now.getTime()) {
|
|
297
|
-
next.setDate(next.getDate() + 1);
|
|
298
|
-
}
|
|
299
|
-
return next.getTime();
|
|
300
|
-
}
|
|
1
|
+
import fs from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { ManagedTimer } from "./managed-timer.js";
|
|
4
|
+
import { formatLocalDateTime, startOfLocalDay } from "./time.js";
|
|
5
|
+
const TAG = "[memory-tdai][cleaner]";
|
|
6
|
+
const L0_DIR_NAME = "conversations";
|
|
7
|
+
const L1_DIR_NAME = "records";
|
|
8
|
+
/** Minimum records to retain — skip deletion if total is at or below this threshold. */
|
|
9
|
+
const MIN_RETAIN_L0 = 50;
|
|
10
|
+
const MIN_RETAIN_L1 = 20;
|
|
11
|
+
export class LocalMemoryCleaner {
|
|
12
|
+
opts;
|
|
13
|
+
timer;
|
|
14
|
+
destroyed = false;
|
|
15
|
+
vectorStore;
|
|
16
|
+
constructor(opts) {
|
|
17
|
+
this.opts = opts;
|
|
18
|
+
this.timer = new ManagedTimer("memory-tdai-cleaner", () => this.destroyed);
|
|
19
|
+
this.vectorStore = opts.vectorStore;
|
|
20
|
+
}
|
|
21
|
+
setVectorStore(vectorStore) {
|
|
22
|
+
this.vectorStore = vectorStore;
|
|
23
|
+
}
|
|
24
|
+
start() {
|
|
25
|
+
if (this.destroyed)
|
|
26
|
+
return;
|
|
27
|
+
const now = new Date();
|
|
28
|
+
const tz = Intl.DateTimeFormat().resolvedOptions().timeZone || "unknown";
|
|
29
|
+
const utcOffset = formatUtcOffset(-now.getTimezoneOffset());
|
|
30
|
+
this.opts.logger?.debug?.(`${TAG} Enabled: retentionDays=${this.opts.retentionDays}, cleanTime=${this.opts.cleanTime}, dirs=[${L0_DIR_NAME}, ${L1_DIR_NAME}]`);
|
|
31
|
+
this.opts.logger?.debug?.(`${TAG} Runtime clock: nowLocal=${formatLocalDateTime(now)}, nowIso=${now.toISOString()}, tz=${tz}, utcOffset=${utcOffset}`);
|
|
32
|
+
this.scheduleNext();
|
|
33
|
+
}
|
|
34
|
+
destroy() {
|
|
35
|
+
if (this.destroyed)
|
|
36
|
+
return;
|
|
37
|
+
this.destroyed = true;
|
|
38
|
+
this.timer.cancel();
|
|
39
|
+
this.opts.logger?.info(`${TAG} Stopped`);
|
|
40
|
+
}
|
|
41
|
+
async runOnce(nowMs = Date.now()) {
|
|
42
|
+
if (this.destroyed)
|
|
43
|
+
return;
|
|
44
|
+
const retentionDays = this.opts.retentionDays;
|
|
45
|
+
if (!(retentionDays > 0)) {
|
|
46
|
+
this.opts.logger?.debug?.(`${TAG} Skip run: invalid retentionDays=${retentionDays}`);
|
|
47
|
+
return;
|
|
48
|
+
}
|
|
49
|
+
// 按"本地自然日"保留策略计算截止时间。
|
|
50
|
+
// 例如 retentionDays=2,今天是 03-15,则保留 03-14/03-15,删除早于 03-14 00:00:00.000 的记录。
|
|
51
|
+
let cutoffMs;
|
|
52
|
+
try {
|
|
53
|
+
cutoffMs = computeCutoffMsByLocalDay(nowMs, retentionDays);
|
|
54
|
+
}
|
|
55
|
+
catch (err) {
|
|
56
|
+
this.opts.logger?.error(`${TAG} ${err instanceof Error ? err.message : String(err)}`);
|
|
57
|
+
return;
|
|
58
|
+
}
|
|
59
|
+
const targetDirs = [
|
|
60
|
+
path.join(this.opts.baseDir, L0_DIR_NAME),
|
|
61
|
+
path.join(this.opts.baseDir, L1_DIR_NAME),
|
|
62
|
+
];
|
|
63
|
+
const total = {
|
|
64
|
+
scannedFiles: 0,
|
|
65
|
+
changedFiles: 0,
|
|
66
|
+
skippedNonShardFiles: 0,
|
|
67
|
+
deleteFailedFiles: 0,
|
|
68
|
+
};
|
|
69
|
+
for (const dirPath of targetDirs) {
|
|
70
|
+
const stats = await this.cleanDirectory(dirPath, cutoffMs);
|
|
71
|
+
total.scannedFiles += stats.scannedFiles;
|
|
72
|
+
total.changedFiles += stats.changedFiles;
|
|
73
|
+
total.skippedNonShardFiles += stats.skippedNonShardFiles;
|
|
74
|
+
total.deleteFailedFiles += stats.deleteFailedFiles;
|
|
75
|
+
}
|
|
76
|
+
if (this.vectorStore) {
|
|
77
|
+
const vectorStore = this.vectorStore;
|
|
78
|
+
const cutoffIso = new Date(cutoffMs).toISOString();
|
|
79
|
+
const startMs = Date.now();
|
|
80
|
+
// ── Pre-delete: count totals and decide whether to proceed ──
|
|
81
|
+
let totalL0 = 0;
|
|
82
|
+
let totalL1 = 0;
|
|
83
|
+
try {
|
|
84
|
+
totalL0 = await vectorStore.countL0();
|
|
85
|
+
}
|
|
86
|
+
catch { /* non-fatal */ }
|
|
87
|
+
try {
|
|
88
|
+
totalL1 = await vectorStore.countL1();
|
|
89
|
+
}
|
|
90
|
+
catch { /* non-fatal */ }
|
|
91
|
+
this.opts.logger?.info(`${TAG} [Pre-delete] cutoffIso=${cutoffIso}, retentionDays=${retentionDays}, totalL0=${totalL0}, totalL1=${totalL1}`);
|
|
92
|
+
let removedL0 = 0;
|
|
93
|
+
let removedL1 = 0;
|
|
94
|
+
let skippedL0 = false;
|
|
95
|
+
let skippedL1 = false;
|
|
96
|
+
let failedL0DbCleanup = 0;
|
|
97
|
+
let failedL1DbCleanup = 0;
|
|
98
|
+
// ── L0 cleanup with minimum-retention guard ──
|
|
99
|
+
if (totalL0 <= MIN_RETAIN_L0) {
|
|
100
|
+
skippedL0 = true;
|
|
101
|
+
this.opts.logger?.info(`${TAG} [L0-delete] SKIPPED: totalL0=${totalL0} <= minRetain=${MIN_RETAIN_L0}`);
|
|
102
|
+
}
|
|
103
|
+
else {
|
|
104
|
+
try {
|
|
105
|
+
removedL0 = await vectorStore.deleteL0Expired(cutoffIso);
|
|
106
|
+
}
|
|
107
|
+
catch (err) {
|
|
108
|
+
failedL0DbCleanup = 1;
|
|
109
|
+
this.opts.logger?.warn(`${TAG} [L0-delete] FAILED: ${err instanceof Error ? err.message : String(err)}`);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
// ── L1 cleanup with minimum-retention guard ──
|
|
113
|
+
if (totalL1 <= MIN_RETAIN_L1) {
|
|
114
|
+
skippedL1 = true;
|
|
115
|
+
this.opts.logger?.info(`${TAG} [L1-delete] SKIPPED: totalL1=${totalL1} <= minRetain=${MIN_RETAIN_L1}`);
|
|
116
|
+
}
|
|
117
|
+
else {
|
|
118
|
+
try {
|
|
119
|
+
removedL1 = await vectorStore.deleteL1Expired(cutoffIso);
|
|
120
|
+
}
|
|
121
|
+
catch (err) {
|
|
122
|
+
failedL1DbCleanup = 1;
|
|
123
|
+
this.opts.logger?.warn(`${TAG} [L1-delete] FAILED: ${err instanceof Error ? err.message : String(err)}`);
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
if (removedL1 > 0 || removedL0 > 0) {
|
|
127
|
+
total.changedFiles += 1;
|
|
128
|
+
}
|
|
129
|
+
// ── Post-delete: audit summary ──
|
|
130
|
+
const durationMs = Date.now() - startMs;
|
|
131
|
+
const remainingL0 = totalL0 - removedL0;
|
|
132
|
+
const remainingL1 = totalL1 - removedL1;
|
|
133
|
+
const summary = {
|
|
134
|
+
event: "cleaner_summary",
|
|
135
|
+
cutoffIso,
|
|
136
|
+
retentionDays,
|
|
137
|
+
l0: { total: totalL0, expired: removedL0, remaining: remainingL0, skipped: skippedL0, failed: failedL0DbCleanup > 0 },
|
|
138
|
+
l1: { total: totalL1, expired: removedL1, remaining: remainingL1, skipped: skippedL1, failed: failedL1DbCleanup > 0 },
|
|
139
|
+
durationMs,
|
|
140
|
+
};
|
|
141
|
+
this.opts.logger?.info(`${TAG} ${JSON.stringify(summary)}`);
|
|
142
|
+
}
|
|
143
|
+
this.opts.logger?.info(`${TAG} Cleanup done: scannedFiles=${total.scannedFiles}, changedFiles=${total.changedFiles}, skippedNonShardFiles=${total.skippedNonShardFiles}, deleteFailedFiles=${total.deleteFailedFiles}`);
|
|
144
|
+
}
|
|
145
|
+
scheduleNext() {
|
|
146
|
+
const nowMs = Date.now();
|
|
147
|
+
const now = new Date(nowMs);
|
|
148
|
+
const next = nextRunAt(this.opts.cleanTime, nowMs);
|
|
149
|
+
const targetToday = buildTodayRunTime(this.opts.cleanTime, nowMs);
|
|
150
|
+
const passedToday = targetToday <= nowMs;
|
|
151
|
+
const delayMs = Math.max(0, next - nowMs);
|
|
152
|
+
this.opts.logger?.debug?.(`${TAG} Schedule next run: nowLocal=${formatLocalDateTime(now)}, cleanTime=${this.opts.cleanTime}, targetTodayLocal=${formatLocalDateTime(new Date(targetToday))}, passedToday=${passedToday}, nextRunLocal=${formatLocalDateTime(new Date(next))}, nextRunIso=${new Date(next).toISOString()}, delayMs=${delayMs}`);
|
|
153
|
+
this.timer.scheduleAt(next, () => {
|
|
154
|
+
const firedAtMs = Date.now();
|
|
155
|
+
this.opts.logger?.info(`${TAG} Timer fired: scheduledLocal=${formatLocalDateTime(new Date(next))}, firedLocal=${formatLocalDateTime(new Date(firedAtMs))}, driftMs=${firedAtMs - next}`);
|
|
156
|
+
void this.runAndReschedule();
|
|
157
|
+
});
|
|
158
|
+
}
|
|
159
|
+
async runAndReschedule() {
|
|
160
|
+
if (this.destroyed)
|
|
161
|
+
return;
|
|
162
|
+
const runStart = new Date();
|
|
163
|
+
this.opts.logger?.info(`${TAG} Cleanup tick start: nowLocal=${formatLocalDateTime(runStart)}, nowIso=${runStart.toISOString()}`);
|
|
164
|
+
try {
|
|
165
|
+
await this.runOnce();
|
|
166
|
+
}
|
|
167
|
+
catch (err) {
|
|
168
|
+
this.opts.logger?.error(`${TAG} Cleanup failed: ${err instanceof Error ? err.stack ?? err.message : String(err)}`);
|
|
169
|
+
}
|
|
170
|
+
finally {
|
|
171
|
+
if (!this.destroyed) {
|
|
172
|
+
this.scheduleNext();
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
async cleanDirectory(dirPath, cutoffMs) {
|
|
177
|
+
const stats = {
|
|
178
|
+
scannedFiles: 0,
|
|
179
|
+
changedFiles: 0,
|
|
180
|
+
skippedNonShardFiles: 0,
|
|
181
|
+
deleteFailedFiles: 0,
|
|
182
|
+
};
|
|
183
|
+
let entries;
|
|
184
|
+
try {
|
|
185
|
+
entries = await fs.readdir(dirPath, { withFileTypes: true });
|
|
186
|
+
}
|
|
187
|
+
catch {
|
|
188
|
+
this.opts.logger?.debug?.(`${TAG} Directory not found, skip: ${dirPath}`);
|
|
189
|
+
return stats;
|
|
190
|
+
}
|
|
191
|
+
for (const entry of entries) {
|
|
192
|
+
if (!entry.isFile())
|
|
193
|
+
continue;
|
|
194
|
+
if (!isJsonLikeFile(entry.name))
|
|
195
|
+
continue;
|
|
196
|
+
const filePath = path.join(dirPath, entry.name);
|
|
197
|
+
stats.scannedFiles += 1;
|
|
198
|
+
// 仅支持日期分片文件:YYYY-MM-DD(.jsonl/.json)
|
|
199
|
+
const shard = extractShardDateFromFileName(entry.name);
|
|
200
|
+
if (!shard) {
|
|
201
|
+
stats.skippedNonShardFiles += 1;
|
|
202
|
+
this.opts.logger?.debug?.(`${TAG} Skip non-shard file: ${filePath}`);
|
|
203
|
+
continue;
|
|
204
|
+
}
|
|
205
|
+
const dayEndMs = localDayEndMs(shard.year, shard.month, shard.day);
|
|
206
|
+
if (dayEndMs < cutoffMs) {
|
|
207
|
+
try {
|
|
208
|
+
await fs.unlink(filePath);
|
|
209
|
+
stats.changedFiles += 1;
|
|
210
|
+
this.opts.logger?.info(`${TAG} Removed expired file by name: ${filePath}`);
|
|
211
|
+
}
|
|
212
|
+
catch (err) {
|
|
213
|
+
stats.deleteFailedFiles += 1;
|
|
214
|
+
this.opts.logger?.warn(`${TAG} Failed to delete expired shard file ${filePath}: ${err instanceof Error ? err.message : String(err)}`);
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
else {
|
|
218
|
+
this.opts.logger?.debug?.(`${TAG} Keep shard file by name: ${filePath}`);
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
return stats;
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
function isJsonLikeFile(name) {
|
|
225
|
+
return name.endsWith(".jsonl") || name.endsWith(".json");
|
|
226
|
+
}
|
|
227
|
+
function extractShardDateFromFileName(fileName) {
|
|
228
|
+
// Supported format: YYYY-MM-DD.jsonl | YYYY-MM-DD.json
|
|
229
|
+
const m = /^(\d{4})-(\d{2})-(\d{2})\.(?:jsonl|json)$/.exec(fileName);
|
|
230
|
+
if (!m)
|
|
231
|
+
return undefined;
|
|
232
|
+
const year = Number(m[1]);
|
|
233
|
+
const month = Number(m[2]);
|
|
234
|
+
const day = Number(m[3]);
|
|
235
|
+
if (!Number.isInteger(year) || !Number.isInteger(month) || !Number.isInteger(day)) {
|
|
236
|
+
return undefined;
|
|
237
|
+
}
|
|
238
|
+
if (month < 1 || month > 12 || day < 1 || day > 31) {
|
|
239
|
+
return undefined;
|
|
240
|
+
}
|
|
241
|
+
const probe = new Date(year, month - 1, day);
|
|
242
|
+
if (probe.getFullYear() !== year
|
|
243
|
+
|| probe.getMonth() !== month - 1
|
|
244
|
+
|| probe.getDate() !== day) {
|
|
245
|
+
return undefined;
|
|
246
|
+
}
|
|
247
|
+
return { year, month, day };
|
|
248
|
+
}
|
|
249
|
+
function localDayEndMs(year, month, day) {
|
|
250
|
+
// End of day = start of next day minus 1ms (in configured timezone)
|
|
251
|
+
const nextDay = new Date(Date.UTC(year, month - 1, day + 1));
|
|
252
|
+
const nextDayStartMs = startOfLocalDay(nextDay);
|
|
253
|
+
return nextDayStartMs - 1;
|
|
254
|
+
}
|
|
255
|
+
function formatUtcOffset(offsetMinutes) {
|
|
256
|
+
const sign = offsetMinutes >= 0 ? "+" : "-";
|
|
257
|
+
const abs = Math.abs(offsetMinutes);
|
|
258
|
+
const hh = String(Math.floor(abs / 60)).padStart(2, "0");
|
|
259
|
+
const mm = String(abs % 60).padStart(2, "0");
|
|
260
|
+
return `${sign}${hh}:${mm}`;
|
|
261
|
+
}
|
|
262
|
+
function computeCutoffMsByLocalDay(nowMs, retentionDays) {
|
|
263
|
+
// 自然日策略,保留"今天 + 往前 retentionDays-1 天"
|
|
264
|
+
// 删除阈值为 keepStart 当天 00:00:00.000(配置时区)
|
|
265
|
+
const now = new Date(nowMs);
|
|
266
|
+
const todayStartMs = startOfLocalDay(now);
|
|
267
|
+
const cutoffMs = todayStartMs - (retentionDays - 1) * 24 * 60 * 60 * 1000;
|
|
268
|
+
// Sanity check: cutoff must be strictly in the past
|
|
269
|
+
if (cutoffMs >= nowMs) {
|
|
270
|
+
throw new Error(`cutoff sanity failed: cutoff (${cutoffMs}) >= now (${nowMs}), ` +
|
|
271
|
+
`possible clock skew or invalid retentionDays=${retentionDays}`);
|
|
272
|
+
}
|
|
273
|
+
// Sanity check: gap between now and cutoff must be at least 24h
|
|
274
|
+
const MIN_GAP_MS = 24 * 60 * 60 * 1000;
|
|
275
|
+
if (nowMs - cutoffMs < MIN_GAP_MS) {
|
|
276
|
+
throw new Error(`cutoff sanity failed: gap ${nowMs - cutoffMs}ms < 24h, ` +
|
|
277
|
+
`retentionDays=${retentionDays}, possible clock skew`);
|
|
278
|
+
}
|
|
279
|
+
return cutoffMs;
|
|
280
|
+
}
|
|
281
|
+
function buildTodayRunTime(cleanTime, nowMs) {
|
|
282
|
+
const [hRaw, mRaw] = cleanTime.split(":");
|
|
283
|
+
const hour = Number(hRaw);
|
|
284
|
+
const minute = Number(mRaw);
|
|
285
|
+
const target = new Date(nowMs);
|
|
286
|
+
target.setHours(hour, minute, 0, 0);
|
|
287
|
+
return target.getTime();
|
|
288
|
+
}
|
|
289
|
+
function nextRunAt(cleanTime, nowMs) {
|
|
290
|
+
const [hRaw, mRaw] = cleanTime.split(":");
|
|
291
|
+
const hour = Number(hRaw);
|
|
292
|
+
const minute = Number(mRaw);
|
|
293
|
+
const now = new Date(nowMs);
|
|
294
|
+
const next = new Date(nowMs);
|
|
295
|
+
next.setHours(hour, minute, 0, 0);
|
|
296
|
+
if (next.getTime() <= now.getTime()) {
|
|
297
|
+
next.setDate(next.getDate() + 1);
|
|
298
|
+
}
|
|
299
|
+
return next.getTime();
|
|
300
|
+
}
|