dsh-tdai-memory 0.3.2 → 0.3.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. package/LICENSE +21 -21
  2. package/README.md +9 -1
  3. package/README.zh.md +8 -1
  4. package/THIRD-PARTY-NOTICES.md +20 -20
  5. package/cordis.patch.yml +8 -8
  6. package/package.json +16 -4
  7. package/recall-inject.js +86 -86
  8. package/vendor/tdai/adapters/index.js +15 -15
  9. package/vendor/tdai/adapters/openclaw/host-adapter.js +83 -83
  10. package/vendor/tdai/adapters/openclaw/index.js +5 -5
  11. package/vendor/tdai/adapters/openclaw/llm-runner.js +69 -69
  12. package/vendor/tdai/adapters/standalone/host-adapter.js +58 -58
  13. package/vendor/tdai/adapters/standalone/index.js +5 -5
  14. package/vendor/tdai/core/conversation/l0-recorder.js +420 -420
  15. package/vendor/tdai/core/hooks/auto-capture.js +236 -236
  16. package/vendor/tdai/core/hooks/auto-recall.js +640 -640
  17. package/vendor/tdai/core/index.js +8 -8
  18. package/vendor/tdai/core/persona/persona-generator.js +184 -184
  19. package/vendor/tdai/core/persona/persona-trigger.js +94 -94
  20. package/vendor/tdai/core/profile/profile-sync.js +200 -200
  21. package/vendor/tdai/core/prompts/l1-dedup.js +72 -72
  22. package/vendor/tdai/core/prompts/l1-extraction.js +34 -34
  23. package/vendor/tdai/core/prompts/persona-generation.js +35 -35
  24. package/vendor/tdai/core/prompts/scene-extraction.js +45 -45
  25. package/vendor/tdai/core/record/l1-dedup.js +318 -318
  26. package/vendor/tdai/core/record/l1-extractor.js +379 -379
  27. package/vendor/tdai/core/record/l1-reader.js +178 -178
  28. package/vendor/tdai/core/record/l1-writer.js +141 -141
  29. package/vendor/tdai/core/report/reporter.js +86 -86
  30. package/vendor/tdai/core/scene/filename-normalizer.js +161 -161
  31. package/vendor/tdai/core/scene/scene-extractor.js +410 -410
  32. package/vendor/tdai/core/scene/scene-format.js +53 -53
  33. package/vendor/tdai/core/scene/scene-index.js +82 -82
  34. package/vendor/tdai/core/scene/scene-navigation.js +57 -57
  35. package/vendor/tdai/core/seed/input.js +404 -404
  36. package/vendor/tdai/core/seed/types.js +9 -9
  37. package/vendor/tdai/core/store/bm25-client.js +117 -117
  38. package/vendor/tdai/core/store/bm25-local.js +68 -68
  39. package/vendor/tdai/core/store/embedding.js +632 -632
  40. package/vendor/tdai/core/store/factory.js +95 -95
  41. package/vendor/tdai/core/store/search-utils.js +54 -54
  42. package/vendor/tdai/core/store/sqlite.js +1632 -1632
  43. package/vendor/tdai/core/store/tcvdb-client.js +221 -221
  44. package/vendor/tdai/core/store/tcvdb.js +1047 -1047
  45. package/vendor/tdai/core/store/types.js +17 -17
  46. package/vendor/tdai/core/tools/conversation-search.js +199 -199
  47. package/vendor/tdai/core/tools/memory-search.js +207 -207
  48. package/vendor/tdai/core/types.js +12 -12
  49. package/vendor/tdai/utils/backup.js +189 -189
  50. package/vendor/tdai/utils/checkpoint.js +360 -360
  51. package/vendor/tdai/utils/clean-context-runner.js +472 -472
  52. package/vendor/tdai/utils/ensure-hook-policy.js +213 -213
  53. package/vendor/tdai/utils/env.js +13 -13
  54. package/vendor/tdai/utils/managed-timer.js +127 -127
  55. package/vendor/tdai/utils/manifest.js +91 -91
  56. package/vendor/tdai/utils/memory-cleaner.js +300 -300
  57. package/vendor/tdai/utils/no-think-fetch.js +109 -109
  58. package/vendor/tdai/utils/openclaw-state-dir.js +26 -26
  59. package/vendor/tdai/utils/pipeline-factory.js +517 -517
  60. package/vendor/tdai/utils/pipeline-manager.js +892 -892
  61. package/vendor/tdai/utils/sanitize.js +360 -360
  62. package/vendor/tdai/utils/serial-queue.js +103 -103
  63. package/vendor/tdai/utils/session-filter.js +88 -88
  64. package/vendor/tdai/utils/text-utils.js +29 -29
  65. package/vendor/tdai/utils/time.js +252 -252
@@ -1,300 +1,300 @@
1
- import fs from "node:fs/promises";
2
- import path from "node:path";
3
- import { ManagedTimer } from "./managed-timer.js";
4
- import { formatLocalDateTime, startOfLocalDay } from "./time.js";
5
- const TAG = "[memory-tdai][cleaner]";
6
- const L0_DIR_NAME = "conversations";
7
- const L1_DIR_NAME = "records";
8
- /** Minimum records to retain — skip deletion if total is at or below this threshold. */
9
- const MIN_RETAIN_L0 = 50;
10
- const MIN_RETAIN_L1 = 20;
11
- export class LocalMemoryCleaner {
12
- opts;
13
- timer;
14
- destroyed = false;
15
- vectorStore;
16
- constructor(opts) {
17
- this.opts = opts;
18
- this.timer = new ManagedTimer("memory-tdai-cleaner", () => this.destroyed);
19
- this.vectorStore = opts.vectorStore;
20
- }
21
- setVectorStore(vectorStore) {
22
- this.vectorStore = vectorStore;
23
- }
24
- start() {
25
- if (this.destroyed)
26
- return;
27
- const now = new Date();
28
- const tz = Intl.DateTimeFormat().resolvedOptions().timeZone || "unknown";
29
- const utcOffset = formatUtcOffset(-now.getTimezoneOffset());
30
- this.opts.logger?.debug?.(`${TAG} Enabled: retentionDays=${this.opts.retentionDays}, cleanTime=${this.opts.cleanTime}, dirs=[${L0_DIR_NAME}, ${L1_DIR_NAME}]`);
31
- this.opts.logger?.debug?.(`${TAG} Runtime clock: nowLocal=${formatLocalDateTime(now)}, nowIso=${now.toISOString()}, tz=${tz}, utcOffset=${utcOffset}`);
32
- this.scheduleNext();
33
- }
34
- destroy() {
35
- if (this.destroyed)
36
- return;
37
- this.destroyed = true;
38
- this.timer.cancel();
39
- this.opts.logger?.info(`${TAG} Stopped`);
40
- }
41
- async runOnce(nowMs = Date.now()) {
42
- if (this.destroyed)
43
- return;
44
- const retentionDays = this.opts.retentionDays;
45
- if (!(retentionDays > 0)) {
46
- this.opts.logger?.debug?.(`${TAG} Skip run: invalid retentionDays=${retentionDays}`);
47
- return;
48
- }
49
- // 按"本地自然日"保留策略计算截止时间。
50
- // 例如 retentionDays=2,今天是 03-15,则保留 03-14/03-15,删除早于 03-14 00:00:00.000 的记录。
51
- let cutoffMs;
52
- try {
53
- cutoffMs = computeCutoffMsByLocalDay(nowMs, retentionDays);
54
- }
55
- catch (err) {
56
- this.opts.logger?.error(`${TAG} ${err instanceof Error ? err.message : String(err)}`);
57
- return;
58
- }
59
- const targetDirs = [
60
- path.join(this.opts.baseDir, L0_DIR_NAME),
61
- path.join(this.opts.baseDir, L1_DIR_NAME),
62
- ];
63
- const total = {
64
- scannedFiles: 0,
65
- changedFiles: 0,
66
- skippedNonShardFiles: 0,
67
- deleteFailedFiles: 0,
68
- };
69
- for (const dirPath of targetDirs) {
70
- const stats = await this.cleanDirectory(dirPath, cutoffMs);
71
- total.scannedFiles += stats.scannedFiles;
72
- total.changedFiles += stats.changedFiles;
73
- total.skippedNonShardFiles += stats.skippedNonShardFiles;
74
- total.deleteFailedFiles += stats.deleteFailedFiles;
75
- }
76
- if (this.vectorStore) {
77
- const vectorStore = this.vectorStore;
78
- const cutoffIso = new Date(cutoffMs).toISOString();
79
- const startMs = Date.now();
80
- // ── Pre-delete: count totals and decide whether to proceed ──
81
- let totalL0 = 0;
82
- let totalL1 = 0;
83
- try {
84
- totalL0 = await vectorStore.countL0();
85
- }
86
- catch { /* non-fatal */ }
87
- try {
88
- totalL1 = await vectorStore.countL1();
89
- }
90
- catch { /* non-fatal */ }
91
- this.opts.logger?.info(`${TAG} [Pre-delete] cutoffIso=${cutoffIso}, retentionDays=${retentionDays}, totalL0=${totalL0}, totalL1=${totalL1}`);
92
- let removedL0 = 0;
93
- let removedL1 = 0;
94
- let skippedL0 = false;
95
- let skippedL1 = false;
96
- let failedL0DbCleanup = 0;
97
- let failedL1DbCleanup = 0;
98
- // ── L0 cleanup with minimum-retention guard ──
99
- if (totalL0 <= MIN_RETAIN_L0) {
100
- skippedL0 = true;
101
- this.opts.logger?.info(`${TAG} [L0-delete] SKIPPED: totalL0=${totalL0} <= minRetain=${MIN_RETAIN_L0}`);
102
- }
103
- else {
104
- try {
105
- removedL0 = await vectorStore.deleteL0Expired(cutoffIso);
106
- }
107
- catch (err) {
108
- failedL0DbCleanup = 1;
109
- this.opts.logger?.warn(`${TAG} [L0-delete] FAILED: ${err instanceof Error ? err.message : String(err)}`);
110
- }
111
- }
112
- // ── L1 cleanup with minimum-retention guard ──
113
- if (totalL1 <= MIN_RETAIN_L1) {
114
- skippedL1 = true;
115
- this.opts.logger?.info(`${TAG} [L1-delete] SKIPPED: totalL1=${totalL1} <= minRetain=${MIN_RETAIN_L1}`);
116
- }
117
- else {
118
- try {
119
- removedL1 = await vectorStore.deleteL1Expired(cutoffIso);
120
- }
121
- catch (err) {
122
- failedL1DbCleanup = 1;
123
- this.opts.logger?.warn(`${TAG} [L1-delete] FAILED: ${err instanceof Error ? err.message : String(err)}`);
124
- }
125
- }
126
- if (removedL1 > 0 || removedL0 > 0) {
127
- total.changedFiles += 1;
128
- }
129
- // ── Post-delete: audit summary ──
130
- const durationMs = Date.now() - startMs;
131
- const remainingL0 = totalL0 - removedL0;
132
- const remainingL1 = totalL1 - removedL1;
133
- const summary = {
134
- event: "cleaner_summary",
135
- cutoffIso,
136
- retentionDays,
137
- l0: { total: totalL0, expired: removedL0, remaining: remainingL0, skipped: skippedL0, failed: failedL0DbCleanup > 0 },
138
- l1: { total: totalL1, expired: removedL1, remaining: remainingL1, skipped: skippedL1, failed: failedL1DbCleanup > 0 },
139
- durationMs,
140
- };
141
- this.opts.logger?.info(`${TAG} ${JSON.stringify(summary)}`);
142
- }
143
- this.opts.logger?.info(`${TAG} Cleanup done: scannedFiles=${total.scannedFiles}, changedFiles=${total.changedFiles}, skippedNonShardFiles=${total.skippedNonShardFiles}, deleteFailedFiles=${total.deleteFailedFiles}`);
144
- }
145
- scheduleNext() {
146
- const nowMs = Date.now();
147
- const now = new Date(nowMs);
148
- const next = nextRunAt(this.opts.cleanTime, nowMs);
149
- const targetToday = buildTodayRunTime(this.opts.cleanTime, nowMs);
150
- const passedToday = targetToday <= nowMs;
151
- const delayMs = Math.max(0, next - nowMs);
152
- this.opts.logger?.debug?.(`${TAG} Schedule next run: nowLocal=${formatLocalDateTime(now)}, cleanTime=${this.opts.cleanTime}, targetTodayLocal=${formatLocalDateTime(new Date(targetToday))}, passedToday=${passedToday}, nextRunLocal=${formatLocalDateTime(new Date(next))}, nextRunIso=${new Date(next).toISOString()}, delayMs=${delayMs}`);
153
- this.timer.scheduleAt(next, () => {
154
- const firedAtMs = Date.now();
155
- this.opts.logger?.info(`${TAG} Timer fired: scheduledLocal=${formatLocalDateTime(new Date(next))}, firedLocal=${formatLocalDateTime(new Date(firedAtMs))}, driftMs=${firedAtMs - next}`);
156
- void this.runAndReschedule();
157
- });
158
- }
159
- async runAndReschedule() {
160
- if (this.destroyed)
161
- return;
162
- const runStart = new Date();
163
- this.opts.logger?.info(`${TAG} Cleanup tick start: nowLocal=${formatLocalDateTime(runStart)}, nowIso=${runStart.toISOString()}`);
164
- try {
165
- await this.runOnce();
166
- }
167
- catch (err) {
168
- this.opts.logger?.error(`${TAG} Cleanup failed: ${err instanceof Error ? err.stack ?? err.message : String(err)}`);
169
- }
170
- finally {
171
- if (!this.destroyed) {
172
- this.scheduleNext();
173
- }
174
- }
175
- }
176
- async cleanDirectory(dirPath, cutoffMs) {
177
- const stats = {
178
- scannedFiles: 0,
179
- changedFiles: 0,
180
- skippedNonShardFiles: 0,
181
- deleteFailedFiles: 0,
182
- };
183
- let entries;
184
- try {
185
- entries = await fs.readdir(dirPath, { withFileTypes: true });
186
- }
187
- catch {
188
- this.opts.logger?.debug?.(`${TAG} Directory not found, skip: ${dirPath}`);
189
- return stats;
190
- }
191
- for (const entry of entries) {
192
- if (!entry.isFile())
193
- continue;
194
- if (!isJsonLikeFile(entry.name))
195
- continue;
196
- const filePath = path.join(dirPath, entry.name);
197
- stats.scannedFiles += 1;
198
- // 仅支持日期分片文件:YYYY-MM-DD(.jsonl/.json)
199
- const shard = extractShardDateFromFileName(entry.name);
200
- if (!shard) {
201
- stats.skippedNonShardFiles += 1;
202
- this.opts.logger?.debug?.(`${TAG} Skip non-shard file: ${filePath}`);
203
- continue;
204
- }
205
- const dayEndMs = localDayEndMs(shard.year, shard.month, shard.day);
206
- if (dayEndMs < cutoffMs) {
207
- try {
208
- await fs.unlink(filePath);
209
- stats.changedFiles += 1;
210
- this.opts.logger?.info(`${TAG} Removed expired file by name: ${filePath}`);
211
- }
212
- catch (err) {
213
- stats.deleteFailedFiles += 1;
214
- this.opts.logger?.warn(`${TAG} Failed to delete expired shard file ${filePath}: ${err instanceof Error ? err.message : String(err)}`);
215
- }
216
- }
217
- else {
218
- this.opts.logger?.debug?.(`${TAG} Keep shard file by name: ${filePath}`);
219
- }
220
- }
221
- return stats;
222
- }
223
- }
224
- function isJsonLikeFile(name) {
225
- return name.endsWith(".jsonl") || name.endsWith(".json");
226
- }
227
- function extractShardDateFromFileName(fileName) {
228
- // Supported format: YYYY-MM-DD.jsonl | YYYY-MM-DD.json
229
- const m = /^(\d{4})-(\d{2})-(\d{2})\.(?:jsonl|json)$/.exec(fileName);
230
- if (!m)
231
- return undefined;
232
- const year = Number(m[1]);
233
- const month = Number(m[2]);
234
- const day = Number(m[3]);
235
- if (!Number.isInteger(year) || !Number.isInteger(month) || !Number.isInteger(day)) {
236
- return undefined;
237
- }
238
- if (month < 1 || month > 12 || day < 1 || day > 31) {
239
- return undefined;
240
- }
241
- const probe = new Date(year, month - 1, day);
242
- if (probe.getFullYear() !== year
243
- || probe.getMonth() !== month - 1
244
- || probe.getDate() !== day) {
245
- return undefined;
246
- }
247
- return { year, month, day };
248
- }
249
- function localDayEndMs(year, month, day) {
250
- // End of day = start of next day minus 1ms (in configured timezone)
251
- const nextDay = new Date(Date.UTC(year, month - 1, day + 1));
252
- const nextDayStartMs = startOfLocalDay(nextDay);
253
- return nextDayStartMs - 1;
254
- }
255
- function formatUtcOffset(offsetMinutes) {
256
- const sign = offsetMinutes >= 0 ? "+" : "-";
257
- const abs = Math.abs(offsetMinutes);
258
- const hh = String(Math.floor(abs / 60)).padStart(2, "0");
259
- const mm = String(abs % 60).padStart(2, "0");
260
- return `${sign}${hh}:${mm}`;
261
- }
262
- function computeCutoffMsByLocalDay(nowMs, retentionDays) {
263
- // 自然日策略,保留"今天 + 往前 retentionDays-1 天"
264
- // 删除阈值为 keepStart 当天 00:00:00.000(配置时区)
265
- const now = new Date(nowMs);
266
- const todayStartMs = startOfLocalDay(now);
267
- const cutoffMs = todayStartMs - (retentionDays - 1) * 24 * 60 * 60 * 1000;
268
- // Sanity check: cutoff must be strictly in the past
269
- if (cutoffMs >= nowMs) {
270
- throw new Error(`cutoff sanity failed: cutoff (${cutoffMs}) >= now (${nowMs}), ` +
271
- `possible clock skew or invalid retentionDays=${retentionDays}`);
272
- }
273
- // Sanity check: gap between now and cutoff must be at least 24h
274
- const MIN_GAP_MS = 24 * 60 * 60 * 1000;
275
- if (nowMs - cutoffMs < MIN_GAP_MS) {
276
- throw new Error(`cutoff sanity failed: gap ${nowMs - cutoffMs}ms < 24h, ` +
277
- `retentionDays=${retentionDays}, possible clock skew`);
278
- }
279
- return cutoffMs;
280
- }
281
- function buildTodayRunTime(cleanTime, nowMs) {
282
- const [hRaw, mRaw] = cleanTime.split(":");
283
- const hour = Number(hRaw);
284
- const minute = Number(mRaw);
285
- const target = new Date(nowMs);
286
- target.setHours(hour, minute, 0, 0);
287
- return target.getTime();
288
- }
289
- function nextRunAt(cleanTime, nowMs) {
290
- const [hRaw, mRaw] = cleanTime.split(":");
291
- const hour = Number(hRaw);
292
- const minute = Number(mRaw);
293
- const now = new Date(nowMs);
294
- const next = new Date(nowMs);
295
- next.setHours(hour, minute, 0, 0);
296
- if (next.getTime() <= now.getTime()) {
297
- next.setDate(next.getDate() + 1);
298
- }
299
- return next.getTime();
300
- }
1
+ import fs from "node:fs/promises";
2
+ import path from "node:path";
3
+ import { ManagedTimer } from "./managed-timer.js";
4
+ import { formatLocalDateTime, startOfLocalDay } from "./time.js";
5
+ const TAG = "[memory-tdai][cleaner]";
6
+ const L0_DIR_NAME = "conversations";
7
+ const L1_DIR_NAME = "records";
8
+ /** Minimum records to retain — skip deletion if total is at or below this threshold. */
9
+ const MIN_RETAIN_L0 = 50;
10
+ const MIN_RETAIN_L1 = 20;
11
+ export class LocalMemoryCleaner {
12
+ opts;
13
+ timer;
14
+ destroyed = false;
15
+ vectorStore;
16
+ constructor(opts) {
17
+ this.opts = opts;
18
+ this.timer = new ManagedTimer("memory-tdai-cleaner", () => this.destroyed);
19
+ this.vectorStore = opts.vectorStore;
20
+ }
21
+ setVectorStore(vectorStore) {
22
+ this.vectorStore = vectorStore;
23
+ }
24
+ start() {
25
+ if (this.destroyed)
26
+ return;
27
+ const now = new Date();
28
+ const tz = Intl.DateTimeFormat().resolvedOptions().timeZone || "unknown";
29
+ const utcOffset = formatUtcOffset(-now.getTimezoneOffset());
30
+ this.opts.logger?.debug?.(`${TAG} Enabled: retentionDays=${this.opts.retentionDays}, cleanTime=${this.opts.cleanTime}, dirs=[${L0_DIR_NAME}, ${L1_DIR_NAME}]`);
31
+ this.opts.logger?.debug?.(`${TAG} Runtime clock: nowLocal=${formatLocalDateTime(now)}, nowIso=${now.toISOString()}, tz=${tz}, utcOffset=${utcOffset}`);
32
+ this.scheduleNext();
33
+ }
34
+ destroy() {
35
+ if (this.destroyed)
36
+ return;
37
+ this.destroyed = true;
38
+ this.timer.cancel();
39
+ this.opts.logger?.info(`${TAG} Stopped`);
40
+ }
41
+ async runOnce(nowMs = Date.now()) {
42
+ if (this.destroyed)
43
+ return;
44
+ const retentionDays = this.opts.retentionDays;
45
+ if (!(retentionDays > 0)) {
46
+ this.opts.logger?.debug?.(`${TAG} Skip run: invalid retentionDays=${retentionDays}`);
47
+ return;
48
+ }
49
+ // 按"本地自然日"保留策略计算截止时间。
50
+ // 例如 retentionDays=2,今天是 03-15,则保留 03-14/03-15,删除早于 03-14 00:00:00.000 的记录。
51
+ let cutoffMs;
52
+ try {
53
+ cutoffMs = computeCutoffMsByLocalDay(nowMs, retentionDays);
54
+ }
55
+ catch (err) {
56
+ this.opts.logger?.error(`${TAG} ${err instanceof Error ? err.message : String(err)}`);
57
+ return;
58
+ }
59
+ const targetDirs = [
60
+ path.join(this.opts.baseDir, L0_DIR_NAME),
61
+ path.join(this.opts.baseDir, L1_DIR_NAME),
62
+ ];
63
+ const total = {
64
+ scannedFiles: 0,
65
+ changedFiles: 0,
66
+ skippedNonShardFiles: 0,
67
+ deleteFailedFiles: 0,
68
+ };
69
+ for (const dirPath of targetDirs) {
70
+ const stats = await this.cleanDirectory(dirPath, cutoffMs);
71
+ total.scannedFiles += stats.scannedFiles;
72
+ total.changedFiles += stats.changedFiles;
73
+ total.skippedNonShardFiles += stats.skippedNonShardFiles;
74
+ total.deleteFailedFiles += stats.deleteFailedFiles;
75
+ }
76
+ if (this.vectorStore) {
77
+ const vectorStore = this.vectorStore;
78
+ const cutoffIso = new Date(cutoffMs).toISOString();
79
+ const startMs = Date.now();
80
+ // ── Pre-delete: count totals and decide whether to proceed ──
81
+ let totalL0 = 0;
82
+ let totalL1 = 0;
83
+ try {
84
+ totalL0 = await vectorStore.countL0();
85
+ }
86
+ catch { /* non-fatal */ }
87
+ try {
88
+ totalL1 = await vectorStore.countL1();
89
+ }
90
+ catch { /* non-fatal */ }
91
+ this.opts.logger?.info(`${TAG} [Pre-delete] cutoffIso=${cutoffIso}, retentionDays=${retentionDays}, totalL0=${totalL0}, totalL1=${totalL1}`);
92
+ let removedL0 = 0;
93
+ let removedL1 = 0;
94
+ let skippedL0 = false;
95
+ let skippedL1 = false;
96
+ let failedL0DbCleanup = 0;
97
+ let failedL1DbCleanup = 0;
98
+ // ── L0 cleanup with minimum-retention guard ──
99
+ if (totalL0 <= MIN_RETAIN_L0) {
100
+ skippedL0 = true;
101
+ this.opts.logger?.info(`${TAG} [L0-delete] SKIPPED: totalL0=${totalL0} <= minRetain=${MIN_RETAIN_L0}`);
102
+ }
103
+ else {
104
+ try {
105
+ removedL0 = await vectorStore.deleteL0Expired(cutoffIso);
106
+ }
107
+ catch (err) {
108
+ failedL0DbCleanup = 1;
109
+ this.opts.logger?.warn(`${TAG} [L0-delete] FAILED: ${err instanceof Error ? err.message : String(err)}`);
110
+ }
111
+ }
112
+ // ── L1 cleanup with minimum-retention guard ──
113
+ if (totalL1 <= MIN_RETAIN_L1) {
114
+ skippedL1 = true;
115
+ this.opts.logger?.info(`${TAG} [L1-delete] SKIPPED: totalL1=${totalL1} <= minRetain=${MIN_RETAIN_L1}`);
116
+ }
117
+ else {
118
+ try {
119
+ removedL1 = await vectorStore.deleteL1Expired(cutoffIso);
120
+ }
121
+ catch (err) {
122
+ failedL1DbCleanup = 1;
123
+ this.opts.logger?.warn(`${TAG} [L1-delete] FAILED: ${err instanceof Error ? err.message : String(err)}`);
124
+ }
125
+ }
126
+ if (removedL1 > 0 || removedL0 > 0) {
127
+ total.changedFiles += 1;
128
+ }
129
+ // ── Post-delete: audit summary ──
130
+ const durationMs = Date.now() - startMs;
131
+ const remainingL0 = totalL0 - removedL0;
132
+ const remainingL1 = totalL1 - removedL1;
133
+ const summary = {
134
+ event: "cleaner_summary",
135
+ cutoffIso,
136
+ retentionDays,
137
+ l0: { total: totalL0, expired: removedL0, remaining: remainingL0, skipped: skippedL0, failed: failedL0DbCleanup > 0 },
138
+ l1: { total: totalL1, expired: removedL1, remaining: remainingL1, skipped: skippedL1, failed: failedL1DbCleanup > 0 },
139
+ durationMs,
140
+ };
141
+ this.opts.logger?.info(`${TAG} ${JSON.stringify(summary)}`);
142
+ }
143
+ this.opts.logger?.info(`${TAG} Cleanup done: scannedFiles=${total.scannedFiles}, changedFiles=${total.changedFiles}, skippedNonShardFiles=${total.skippedNonShardFiles}, deleteFailedFiles=${total.deleteFailedFiles}`);
144
+ }
145
+ scheduleNext() {
146
+ const nowMs = Date.now();
147
+ const now = new Date(nowMs);
148
+ const next = nextRunAt(this.opts.cleanTime, nowMs);
149
+ const targetToday = buildTodayRunTime(this.opts.cleanTime, nowMs);
150
+ const passedToday = targetToday <= nowMs;
151
+ const delayMs = Math.max(0, next - nowMs);
152
+ this.opts.logger?.debug?.(`${TAG} Schedule next run: nowLocal=${formatLocalDateTime(now)}, cleanTime=${this.opts.cleanTime}, targetTodayLocal=${formatLocalDateTime(new Date(targetToday))}, passedToday=${passedToday}, nextRunLocal=${formatLocalDateTime(new Date(next))}, nextRunIso=${new Date(next).toISOString()}, delayMs=${delayMs}`);
153
+ this.timer.scheduleAt(next, () => {
154
+ const firedAtMs = Date.now();
155
+ this.opts.logger?.info(`${TAG} Timer fired: scheduledLocal=${formatLocalDateTime(new Date(next))}, firedLocal=${formatLocalDateTime(new Date(firedAtMs))}, driftMs=${firedAtMs - next}`);
156
+ void this.runAndReschedule();
157
+ });
158
+ }
159
+ async runAndReschedule() {
160
+ if (this.destroyed)
161
+ return;
162
+ const runStart = new Date();
163
+ this.opts.logger?.info(`${TAG} Cleanup tick start: nowLocal=${formatLocalDateTime(runStart)}, nowIso=${runStart.toISOString()}`);
164
+ try {
165
+ await this.runOnce();
166
+ }
167
+ catch (err) {
168
+ this.opts.logger?.error(`${TAG} Cleanup failed: ${err instanceof Error ? err.stack ?? err.message : String(err)}`);
169
+ }
170
+ finally {
171
+ if (!this.destroyed) {
172
+ this.scheduleNext();
173
+ }
174
+ }
175
+ }
176
+ async cleanDirectory(dirPath, cutoffMs) {
177
+ const stats = {
178
+ scannedFiles: 0,
179
+ changedFiles: 0,
180
+ skippedNonShardFiles: 0,
181
+ deleteFailedFiles: 0,
182
+ };
183
+ let entries;
184
+ try {
185
+ entries = await fs.readdir(dirPath, { withFileTypes: true });
186
+ }
187
+ catch {
188
+ this.opts.logger?.debug?.(`${TAG} Directory not found, skip: ${dirPath}`);
189
+ return stats;
190
+ }
191
+ for (const entry of entries) {
192
+ if (!entry.isFile())
193
+ continue;
194
+ if (!isJsonLikeFile(entry.name))
195
+ continue;
196
+ const filePath = path.join(dirPath, entry.name);
197
+ stats.scannedFiles += 1;
198
+ // 仅支持日期分片文件:YYYY-MM-DD(.jsonl/.json)
199
+ const shard = extractShardDateFromFileName(entry.name);
200
+ if (!shard) {
201
+ stats.skippedNonShardFiles += 1;
202
+ this.opts.logger?.debug?.(`${TAG} Skip non-shard file: ${filePath}`);
203
+ continue;
204
+ }
205
+ const dayEndMs = localDayEndMs(shard.year, shard.month, shard.day);
206
+ if (dayEndMs < cutoffMs) {
207
+ try {
208
+ await fs.unlink(filePath);
209
+ stats.changedFiles += 1;
210
+ this.opts.logger?.info(`${TAG} Removed expired file by name: ${filePath}`);
211
+ }
212
+ catch (err) {
213
+ stats.deleteFailedFiles += 1;
214
+ this.opts.logger?.warn(`${TAG} Failed to delete expired shard file ${filePath}: ${err instanceof Error ? err.message : String(err)}`);
215
+ }
216
+ }
217
+ else {
218
+ this.opts.logger?.debug?.(`${TAG} Keep shard file by name: ${filePath}`);
219
+ }
220
+ }
221
+ return stats;
222
+ }
223
+ }
224
+ function isJsonLikeFile(name) {
225
+ return name.endsWith(".jsonl") || name.endsWith(".json");
226
+ }
227
+ function extractShardDateFromFileName(fileName) {
228
+ // Supported format: YYYY-MM-DD.jsonl | YYYY-MM-DD.json
229
+ const m = /^(\d{4})-(\d{2})-(\d{2})\.(?:jsonl|json)$/.exec(fileName);
230
+ if (!m)
231
+ return undefined;
232
+ const year = Number(m[1]);
233
+ const month = Number(m[2]);
234
+ const day = Number(m[3]);
235
+ if (!Number.isInteger(year) || !Number.isInteger(month) || !Number.isInteger(day)) {
236
+ return undefined;
237
+ }
238
+ if (month < 1 || month > 12 || day < 1 || day > 31) {
239
+ return undefined;
240
+ }
241
+ const probe = new Date(year, month - 1, day);
242
+ if (probe.getFullYear() !== year
243
+ || probe.getMonth() !== month - 1
244
+ || probe.getDate() !== day) {
245
+ return undefined;
246
+ }
247
+ return { year, month, day };
248
+ }
249
+ function localDayEndMs(year, month, day) {
250
+ // End of day = start of next day minus 1ms (in configured timezone)
251
+ const nextDay = new Date(Date.UTC(year, month - 1, day + 1));
252
+ const nextDayStartMs = startOfLocalDay(nextDay);
253
+ return nextDayStartMs - 1;
254
+ }
255
+ function formatUtcOffset(offsetMinutes) {
256
+ const sign = offsetMinutes >= 0 ? "+" : "-";
257
+ const abs = Math.abs(offsetMinutes);
258
+ const hh = String(Math.floor(abs / 60)).padStart(2, "0");
259
+ const mm = String(abs % 60).padStart(2, "0");
260
+ return `${sign}${hh}:${mm}`;
261
+ }
262
+ function computeCutoffMsByLocalDay(nowMs, retentionDays) {
263
+ // 自然日策略,保留"今天 + 往前 retentionDays-1 天"
264
+ // 删除阈值为 keepStart 当天 00:00:00.000(配置时区)
265
+ const now = new Date(nowMs);
266
+ const todayStartMs = startOfLocalDay(now);
267
+ const cutoffMs = todayStartMs - (retentionDays - 1) * 24 * 60 * 60 * 1000;
268
+ // Sanity check: cutoff must be strictly in the past
269
+ if (cutoffMs >= nowMs) {
270
+ throw new Error(`cutoff sanity failed: cutoff (${cutoffMs}) >= now (${nowMs}), ` +
271
+ `possible clock skew or invalid retentionDays=${retentionDays}`);
272
+ }
273
+ // Sanity check: gap between now and cutoff must be at least 24h
274
+ const MIN_GAP_MS = 24 * 60 * 60 * 1000;
275
+ if (nowMs - cutoffMs < MIN_GAP_MS) {
276
+ throw new Error(`cutoff sanity failed: gap ${nowMs - cutoffMs}ms < 24h, ` +
277
+ `retentionDays=${retentionDays}, possible clock skew`);
278
+ }
279
+ return cutoffMs;
280
+ }
281
+ function buildTodayRunTime(cleanTime, nowMs) {
282
+ const [hRaw, mRaw] = cleanTime.split(":");
283
+ const hour = Number(hRaw);
284
+ const minute = Number(mRaw);
285
+ const target = new Date(nowMs);
286
+ target.setHours(hour, minute, 0, 0);
287
+ return target.getTime();
288
+ }
289
+ function nextRunAt(cleanTime, nowMs) {
290
+ const [hRaw, mRaw] = cleanTime.split(":");
291
+ const hour = Number(hRaw);
292
+ const minute = Number(mRaw);
293
+ const now = new Date(nowMs);
294
+ const next = new Date(nowMs);
295
+ next.setHours(hour, minute, 0, 0);
296
+ if (next.getTime() <= now.getTime()) {
297
+ next.setDate(next.getDate() + 1);
298
+ }
299
+ return next.getTime();
300
+ }