dsh-tdai-memory 0.4.2 → 0.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/LICENSE +21 -21
  2. package/README.md +8 -0
  3. package/README.zh.md +8 -0
  4. package/THIRD-PARTY-NOTICES.md +20 -20
  5. package/cordis.patch.yml +8 -8
  6. package/package.json +103 -100
  7. package/vendor/tdai/adapters/index.js +15 -15
  8. package/vendor/tdai/adapters/openclaw/host-adapter.js +83 -83
  9. package/vendor/tdai/adapters/openclaw/index.js +5 -5
  10. package/vendor/tdai/adapters/openclaw/llm-runner.js +69 -69
  11. package/vendor/tdai/adapters/standalone/host-adapter.js +58 -58
  12. package/vendor/tdai/adapters/standalone/index.js +5 -5
  13. package/vendor/tdai/core/conversation/l0-recorder.js +420 -420
  14. package/vendor/tdai/core/hooks/auto-capture.js +236 -236
  15. package/vendor/tdai/core/hooks/auto-recall.js +640 -640
  16. package/vendor/tdai/core/index.js +8 -8
  17. package/vendor/tdai/core/persona/persona-generator.js +184 -184
  18. package/vendor/tdai/core/persona/persona-trigger.js +94 -94
  19. package/vendor/tdai/core/profile/profile-sync.js +200 -200
  20. package/vendor/tdai/core/prompts/l1-dedup.js +72 -72
  21. package/vendor/tdai/core/prompts/l1-extraction.js +34 -34
  22. package/vendor/tdai/core/prompts/persona-generation.js +35 -35
  23. package/vendor/tdai/core/prompts/scene-extraction.js +45 -45
  24. package/vendor/tdai/core/record/l1-dedup.js +318 -318
  25. package/vendor/tdai/core/record/l1-extractor.js +379 -379
  26. package/vendor/tdai/core/record/l1-reader.js +178 -178
  27. package/vendor/tdai/core/record/l1-writer.js +141 -141
  28. package/vendor/tdai/core/report/reporter.js +86 -86
  29. package/vendor/tdai/core/scene/filename-normalizer.js +161 -161
  30. package/vendor/tdai/core/scene/scene-extractor.js +410 -410
  31. package/vendor/tdai/core/scene/scene-format.js +53 -53
  32. package/vendor/tdai/core/scene/scene-index.js +82 -82
  33. package/vendor/tdai/core/scene/scene-navigation.js +57 -57
  34. package/vendor/tdai/core/seed/input.js +404 -404
  35. package/vendor/tdai/core/seed/types.js +9 -9
  36. package/vendor/tdai/core/store/bm25-client.js +117 -117
  37. package/vendor/tdai/core/store/bm25-local.js +68 -68
  38. package/vendor/tdai/core/store/embedding.js +632 -632
  39. package/vendor/tdai/core/store/factory.js +95 -95
  40. package/vendor/tdai/core/store/search-utils.js +54 -54
  41. package/vendor/tdai/core/store/sqlite.js +1632 -1632
  42. package/vendor/tdai/core/store/tcvdb-client.js +221 -221
  43. package/vendor/tdai/core/store/tcvdb.js +1047 -1047
  44. package/vendor/tdai/core/store/types.js +17 -17
  45. package/vendor/tdai/core/tools/conversation-search.js +199 -199
  46. package/vendor/tdai/core/tools/memory-search.js +207 -207
  47. package/vendor/tdai/core/types.js +12 -12
  48. package/vendor/tdai/utils/backup.js +189 -189
  49. package/vendor/tdai/utils/checkpoint.js +360 -360
  50. package/vendor/tdai/utils/clean-context-runner.js +472 -472
  51. package/vendor/tdai/utils/ensure-hook-policy.js +213 -213
  52. package/vendor/tdai/utils/env.js +13 -13
  53. package/vendor/tdai/utils/managed-timer.js +127 -127
  54. package/vendor/tdai/utils/manifest.js +91 -91
  55. package/vendor/tdai/utils/memory-cleaner.js +300 -300
  56. package/vendor/tdai/utils/no-think-fetch.js +109 -109
  57. package/vendor/tdai/utils/openclaw-state-dir.js +26 -26
  58. package/vendor/tdai/utils/pipeline-factory.js +517 -517
  59. package/vendor/tdai/utils/pipeline-manager.js +892 -892
  60. package/vendor/tdai/utils/sanitize.js +360 -360
  61. package/vendor/tdai/utils/serial-queue.js +103 -103
  62. package/vendor/tdai/utils/session-filter.js +88 -88
  63. package/vendor/tdai/utils/text-utils.js +29 -29
  64. package/vendor/tdai/utils/time.js +252 -252
@@ -1,300 +1,300 @@
1
- import fs from "node:fs/promises";
2
- import path from "node:path";
3
- import { ManagedTimer } from "./managed-timer.js";
4
- import { formatLocalDateTime, startOfLocalDay } from "./time.js";
5
- const TAG = "[memory-tdai][cleaner]";
6
- const L0_DIR_NAME = "conversations";
7
- const L1_DIR_NAME = "records";
8
- /** Minimum records to retain — skip deletion if total is at or below this threshold. */
9
- const MIN_RETAIN_L0 = 50;
10
- const MIN_RETAIN_L1 = 20;
11
- export class LocalMemoryCleaner {
12
- opts;
13
- timer;
14
- destroyed = false;
15
- vectorStore;
16
- constructor(opts) {
17
- this.opts = opts;
18
- this.timer = new ManagedTimer("memory-tdai-cleaner", () => this.destroyed);
19
- this.vectorStore = opts.vectorStore;
20
- }
21
- setVectorStore(vectorStore) {
22
- this.vectorStore = vectorStore;
23
- }
24
- start() {
25
- if (this.destroyed)
26
- return;
27
- const now = new Date();
28
- const tz = Intl.DateTimeFormat().resolvedOptions().timeZone || "unknown";
29
- const utcOffset = formatUtcOffset(-now.getTimezoneOffset());
30
- this.opts.logger?.debug?.(`${TAG} Enabled: retentionDays=${this.opts.retentionDays}, cleanTime=${this.opts.cleanTime}, dirs=[${L0_DIR_NAME}, ${L1_DIR_NAME}]`);
31
- this.opts.logger?.debug?.(`${TAG} Runtime clock: nowLocal=${formatLocalDateTime(now)}, nowIso=${now.toISOString()}, tz=${tz}, utcOffset=${utcOffset}`);
32
- this.scheduleNext();
33
- }
34
- destroy() {
35
- if (this.destroyed)
36
- return;
37
- this.destroyed = true;
38
- this.timer.cancel();
39
- this.opts.logger?.info(`${TAG} Stopped`);
40
- }
41
- async runOnce(nowMs = Date.now()) {
42
- if (this.destroyed)
43
- return;
44
- const retentionDays = this.opts.retentionDays;
45
- if (!(retentionDays > 0)) {
46
- this.opts.logger?.debug?.(`${TAG} Skip run: invalid retentionDays=${retentionDays}`);
47
- return;
48
- }
49
- // 按"本地自然日"保留策略计算截止时间。
50
- // 例如 retentionDays=2,今天是 03-15,则保留 03-14/03-15,删除早于 03-14 00:00:00.000 的记录。
51
- let cutoffMs;
52
- try {
53
- cutoffMs = computeCutoffMsByLocalDay(nowMs, retentionDays);
54
- }
55
- catch (err) {
56
- this.opts.logger?.error(`${TAG} ${err instanceof Error ? err.message : String(err)}`);
57
- return;
58
- }
59
- const targetDirs = [
60
- path.join(this.opts.baseDir, L0_DIR_NAME),
61
- path.join(this.opts.baseDir, L1_DIR_NAME),
62
- ];
63
- const total = {
64
- scannedFiles: 0,
65
- changedFiles: 0,
66
- skippedNonShardFiles: 0,
67
- deleteFailedFiles: 0,
68
- };
69
- for (const dirPath of targetDirs) {
70
- const stats = await this.cleanDirectory(dirPath, cutoffMs);
71
- total.scannedFiles += stats.scannedFiles;
72
- total.changedFiles += stats.changedFiles;
73
- total.skippedNonShardFiles += stats.skippedNonShardFiles;
74
- total.deleteFailedFiles += stats.deleteFailedFiles;
75
- }
76
- if (this.vectorStore) {
77
- const vectorStore = this.vectorStore;
78
- const cutoffIso = new Date(cutoffMs).toISOString();
79
- const startMs = Date.now();
80
- // ── Pre-delete: count totals and decide whether to proceed ──
81
- let totalL0 = 0;
82
- let totalL1 = 0;
83
- try {
84
- totalL0 = await vectorStore.countL0();
85
- }
86
- catch { /* non-fatal */ }
87
- try {
88
- totalL1 = await vectorStore.countL1();
89
- }
90
- catch { /* non-fatal */ }
91
- this.opts.logger?.info(`${TAG} [Pre-delete] cutoffIso=${cutoffIso}, retentionDays=${retentionDays}, totalL0=${totalL0}, totalL1=${totalL1}`);
92
- let removedL0 = 0;
93
- let removedL1 = 0;
94
- let skippedL0 = false;
95
- let skippedL1 = false;
96
- let failedL0DbCleanup = 0;
97
- let failedL1DbCleanup = 0;
98
- // ── L0 cleanup with minimum-retention guard ──
99
- if (totalL0 <= MIN_RETAIN_L0) {
100
- skippedL0 = true;
101
- this.opts.logger?.info(`${TAG} [L0-delete] SKIPPED: totalL0=${totalL0} <= minRetain=${MIN_RETAIN_L0}`);
102
- }
103
- else {
104
- try {
105
- removedL0 = await vectorStore.deleteL0Expired(cutoffIso);
106
- }
107
- catch (err) {
108
- failedL0DbCleanup = 1;
109
- this.opts.logger?.warn(`${TAG} [L0-delete] FAILED: ${err instanceof Error ? err.message : String(err)}`);
110
- }
111
- }
112
- // ── L1 cleanup with minimum-retention guard ──
113
- if (totalL1 <= MIN_RETAIN_L1) {
114
- skippedL1 = true;
115
- this.opts.logger?.info(`${TAG} [L1-delete] SKIPPED: totalL1=${totalL1} <= minRetain=${MIN_RETAIN_L1}`);
116
- }
117
- else {
118
- try {
119
- removedL1 = await vectorStore.deleteL1Expired(cutoffIso);
120
- }
121
- catch (err) {
122
- failedL1DbCleanup = 1;
123
- this.opts.logger?.warn(`${TAG} [L1-delete] FAILED: ${err instanceof Error ? err.message : String(err)}`);
124
- }
125
- }
126
- if (removedL1 > 0 || removedL0 > 0) {
127
- total.changedFiles += 1;
128
- }
129
- // ── Post-delete: audit summary ──
130
- const durationMs = Date.now() - startMs;
131
- const remainingL0 = totalL0 - removedL0;
132
- const remainingL1 = totalL1 - removedL1;
133
- const summary = {
134
- event: "cleaner_summary",
135
- cutoffIso,
136
- retentionDays,
137
- l0: { total: totalL0, expired: removedL0, remaining: remainingL0, skipped: skippedL0, failed: failedL0DbCleanup > 0 },
138
- l1: { total: totalL1, expired: removedL1, remaining: remainingL1, skipped: skippedL1, failed: failedL1DbCleanup > 0 },
139
- durationMs,
140
- };
141
- this.opts.logger?.info(`${TAG} ${JSON.stringify(summary)}`);
142
- }
143
- this.opts.logger?.info(`${TAG} Cleanup done: scannedFiles=${total.scannedFiles}, changedFiles=${total.changedFiles}, skippedNonShardFiles=${total.skippedNonShardFiles}, deleteFailedFiles=${total.deleteFailedFiles}`);
144
- }
145
- scheduleNext() {
146
- const nowMs = Date.now();
147
- const now = new Date(nowMs);
148
- const next = nextRunAt(this.opts.cleanTime, nowMs);
149
- const targetToday = buildTodayRunTime(this.opts.cleanTime, nowMs);
150
- const passedToday = targetToday <= nowMs;
151
- const delayMs = Math.max(0, next - nowMs);
152
- this.opts.logger?.debug?.(`${TAG} Schedule next run: nowLocal=${formatLocalDateTime(now)}, cleanTime=${this.opts.cleanTime}, targetTodayLocal=${formatLocalDateTime(new Date(targetToday))}, passedToday=${passedToday}, nextRunLocal=${formatLocalDateTime(new Date(next))}, nextRunIso=${new Date(next).toISOString()}, delayMs=${delayMs}`);
153
- this.timer.scheduleAt(next, () => {
154
- const firedAtMs = Date.now();
155
- this.opts.logger?.info(`${TAG} Timer fired: scheduledLocal=${formatLocalDateTime(new Date(next))}, firedLocal=${formatLocalDateTime(new Date(firedAtMs))}, driftMs=${firedAtMs - next}`);
156
- void this.runAndReschedule();
157
- });
158
- }
159
- async runAndReschedule() {
160
- if (this.destroyed)
161
- return;
162
- const runStart = new Date();
163
- this.opts.logger?.info(`${TAG} Cleanup tick start: nowLocal=${formatLocalDateTime(runStart)}, nowIso=${runStart.toISOString()}`);
164
- try {
165
- await this.runOnce();
166
- }
167
- catch (err) {
168
- this.opts.logger?.error(`${TAG} Cleanup failed: ${err instanceof Error ? err.stack ?? err.message : String(err)}`);
169
- }
170
- finally {
171
- if (!this.destroyed) {
172
- this.scheduleNext();
173
- }
174
- }
175
- }
176
- async cleanDirectory(dirPath, cutoffMs) {
177
- const stats = {
178
- scannedFiles: 0,
179
- changedFiles: 0,
180
- skippedNonShardFiles: 0,
181
- deleteFailedFiles: 0,
182
- };
183
- let entries;
184
- try {
185
- entries = await fs.readdir(dirPath, { withFileTypes: true });
186
- }
187
- catch {
188
- this.opts.logger?.debug?.(`${TAG} Directory not found, skip: ${dirPath}`);
189
- return stats;
190
- }
191
- for (const entry of entries) {
192
- if (!entry.isFile())
193
- continue;
194
- if (!isJsonLikeFile(entry.name))
195
- continue;
196
- const filePath = path.join(dirPath, entry.name);
197
- stats.scannedFiles += 1;
198
- // 仅支持日期分片文件:YYYY-MM-DD(.jsonl/.json)
199
- const shard = extractShardDateFromFileName(entry.name);
200
- if (!shard) {
201
- stats.skippedNonShardFiles += 1;
202
- this.opts.logger?.debug?.(`${TAG} Skip non-shard file: ${filePath}`);
203
- continue;
204
- }
205
- const dayEndMs = localDayEndMs(shard.year, shard.month, shard.day);
206
- if (dayEndMs < cutoffMs) {
207
- try {
208
- await fs.unlink(filePath);
209
- stats.changedFiles += 1;
210
- this.opts.logger?.info(`${TAG} Removed expired file by name: ${filePath}`);
211
- }
212
- catch (err) {
213
- stats.deleteFailedFiles += 1;
214
- this.opts.logger?.warn(`${TAG} Failed to delete expired shard file ${filePath}: ${err instanceof Error ? err.message : String(err)}`);
215
- }
216
- }
217
- else {
218
- this.opts.logger?.debug?.(`${TAG} Keep shard file by name: ${filePath}`);
219
- }
220
- }
221
- return stats;
222
- }
223
- }
224
- function isJsonLikeFile(name) {
225
- return name.endsWith(".jsonl") || name.endsWith(".json");
226
- }
227
- function extractShardDateFromFileName(fileName) {
228
- // Supported format: YYYY-MM-DD.jsonl | YYYY-MM-DD.json
229
- const m = /^(\d{4})-(\d{2})-(\d{2})\.(?:jsonl|json)$/.exec(fileName);
230
- if (!m)
231
- return undefined;
232
- const year = Number(m[1]);
233
- const month = Number(m[2]);
234
- const day = Number(m[3]);
235
- if (!Number.isInteger(year) || !Number.isInteger(month) || !Number.isInteger(day)) {
236
- return undefined;
237
- }
238
- if (month < 1 || month > 12 || day < 1 || day > 31) {
239
- return undefined;
240
- }
241
- const probe = new Date(year, month - 1, day);
242
- if (probe.getFullYear() !== year
243
- || probe.getMonth() !== month - 1
244
- || probe.getDate() !== day) {
245
- return undefined;
246
- }
247
- return { year, month, day };
248
- }
249
- function localDayEndMs(year, month, day) {
250
- // End of day = start of next day minus 1ms (in configured timezone)
251
- const nextDay = new Date(Date.UTC(year, month - 1, day + 1));
252
- const nextDayStartMs = startOfLocalDay(nextDay);
253
- return nextDayStartMs - 1;
254
- }
255
- function formatUtcOffset(offsetMinutes) {
256
- const sign = offsetMinutes >= 0 ? "+" : "-";
257
- const abs = Math.abs(offsetMinutes);
258
- const hh = String(Math.floor(abs / 60)).padStart(2, "0");
259
- const mm = String(abs % 60).padStart(2, "0");
260
- return `${sign}${hh}:${mm}`;
261
- }
262
- function computeCutoffMsByLocalDay(nowMs, retentionDays) {
263
- // 自然日策略,保留"今天 + 往前 retentionDays-1 天"
264
- // 删除阈值为 keepStart 当天 00:00:00.000(配置时区)
265
- const now = new Date(nowMs);
266
- const todayStartMs = startOfLocalDay(now);
267
- const cutoffMs = todayStartMs - (retentionDays - 1) * 24 * 60 * 60 * 1000;
268
- // Sanity check: cutoff must be strictly in the past
269
- if (cutoffMs >= nowMs) {
270
- throw new Error(`cutoff sanity failed: cutoff (${cutoffMs}) >= now (${nowMs}), ` +
271
- `possible clock skew or invalid retentionDays=${retentionDays}`);
272
- }
273
- // Sanity check: gap between now and cutoff must be at least 24h
274
- const MIN_GAP_MS = 24 * 60 * 60 * 1000;
275
- if (nowMs - cutoffMs < MIN_GAP_MS) {
276
- throw new Error(`cutoff sanity failed: gap ${nowMs - cutoffMs}ms < 24h, ` +
277
- `retentionDays=${retentionDays}, possible clock skew`);
278
- }
279
- return cutoffMs;
280
- }
281
- function buildTodayRunTime(cleanTime, nowMs) {
282
- const [hRaw, mRaw] = cleanTime.split(":");
283
- const hour = Number(hRaw);
284
- const minute = Number(mRaw);
285
- const target = new Date(nowMs);
286
- target.setHours(hour, minute, 0, 0);
287
- return target.getTime();
288
- }
289
- function nextRunAt(cleanTime, nowMs) {
290
- const [hRaw, mRaw] = cleanTime.split(":");
291
- const hour = Number(hRaw);
292
- const minute = Number(mRaw);
293
- const now = new Date(nowMs);
294
- const next = new Date(nowMs);
295
- next.setHours(hour, minute, 0, 0);
296
- if (next.getTime() <= now.getTime()) {
297
- next.setDate(next.getDate() + 1);
298
- }
299
- return next.getTime();
300
- }
1
+ import fs from "node:fs/promises";
2
+ import path from "node:path";
3
+ import { ManagedTimer } from "./managed-timer.js";
4
+ import { formatLocalDateTime, startOfLocalDay } from "./time.js";
5
+ const TAG = "[memory-tdai][cleaner]";
6
+ const L0_DIR_NAME = "conversations";
7
+ const L1_DIR_NAME = "records";
8
+ /** Minimum records to retain — skip deletion if total is at or below this threshold. */
9
+ const MIN_RETAIN_L0 = 50;
10
+ const MIN_RETAIN_L1 = 20;
11
+ export class LocalMemoryCleaner {
12
+ opts;
13
+ timer;
14
+ destroyed = false;
15
+ vectorStore;
16
+ constructor(opts) {
17
+ this.opts = opts;
18
+ this.timer = new ManagedTimer("memory-tdai-cleaner", () => this.destroyed);
19
+ this.vectorStore = opts.vectorStore;
20
+ }
21
+ setVectorStore(vectorStore) {
22
+ this.vectorStore = vectorStore;
23
+ }
24
+ start() {
25
+ if (this.destroyed)
26
+ return;
27
+ const now = new Date();
28
+ const tz = Intl.DateTimeFormat().resolvedOptions().timeZone || "unknown";
29
+ const utcOffset = formatUtcOffset(-now.getTimezoneOffset());
30
+ this.opts.logger?.debug?.(`${TAG} Enabled: retentionDays=${this.opts.retentionDays}, cleanTime=${this.opts.cleanTime}, dirs=[${L0_DIR_NAME}, ${L1_DIR_NAME}]`);
31
+ this.opts.logger?.debug?.(`${TAG} Runtime clock: nowLocal=${formatLocalDateTime(now)}, nowIso=${now.toISOString()}, tz=${tz}, utcOffset=${utcOffset}`);
32
+ this.scheduleNext();
33
+ }
34
+ destroy() {
35
+ if (this.destroyed)
36
+ return;
37
+ this.destroyed = true;
38
+ this.timer.cancel();
39
+ this.opts.logger?.info(`${TAG} Stopped`);
40
+ }
41
+ async runOnce(nowMs = Date.now()) {
42
+ if (this.destroyed)
43
+ return;
44
+ const retentionDays = this.opts.retentionDays;
45
+ if (!(retentionDays > 0)) {
46
+ this.opts.logger?.debug?.(`${TAG} Skip run: invalid retentionDays=${retentionDays}`);
47
+ return;
48
+ }
49
+ // 按"本地自然日"保留策略计算截止时间。
50
+ // 例如 retentionDays=2,今天是 03-15,则保留 03-14/03-15,删除早于 03-14 00:00:00.000 的记录。
51
+ let cutoffMs;
52
+ try {
53
+ cutoffMs = computeCutoffMsByLocalDay(nowMs, retentionDays);
54
+ }
55
+ catch (err) {
56
+ this.opts.logger?.error(`${TAG} ${err instanceof Error ? err.message : String(err)}`);
57
+ return;
58
+ }
59
+ const targetDirs = [
60
+ path.join(this.opts.baseDir, L0_DIR_NAME),
61
+ path.join(this.opts.baseDir, L1_DIR_NAME),
62
+ ];
63
+ const total = {
64
+ scannedFiles: 0,
65
+ changedFiles: 0,
66
+ skippedNonShardFiles: 0,
67
+ deleteFailedFiles: 0,
68
+ };
69
+ for (const dirPath of targetDirs) {
70
+ const stats = await this.cleanDirectory(dirPath, cutoffMs);
71
+ total.scannedFiles += stats.scannedFiles;
72
+ total.changedFiles += stats.changedFiles;
73
+ total.skippedNonShardFiles += stats.skippedNonShardFiles;
74
+ total.deleteFailedFiles += stats.deleteFailedFiles;
75
+ }
76
+ if (this.vectorStore) {
77
+ const vectorStore = this.vectorStore;
78
+ const cutoffIso = new Date(cutoffMs).toISOString();
79
+ const startMs = Date.now();
80
+ // ── Pre-delete: count totals and decide whether to proceed ──
81
+ let totalL0 = 0;
82
+ let totalL1 = 0;
83
+ try {
84
+ totalL0 = await vectorStore.countL0();
85
+ }
86
+ catch { /* non-fatal */ }
87
+ try {
88
+ totalL1 = await vectorStore.countL1();
89
+ }
90
+ catch { /* non-fatal */ }
91
+ this.opts.logger?.info(`${TAG} [Pre-delete] cutoffIso=${cutoffIso}, retentionDays=${retentionDays}, totalL0=${totalL0}, totalL1=${totalL1}`);
92
+ let removedL0 = 0;
93
+ let removedL1 = 0;
94
+ let skippedL0 = false;
95
+ let skippedL1 = false;
96
+ let failedL0DbCleanup = 0;
97
+ let failedL1DbCleanup = 0;
98
+ // ── L0 cleanup with minimum-retention guard ──
99
+ if (totalL0 <= MIN_RETAIN_L0) {
100
+ skippedL0 = true;
101
+ this.opts.logger?.info(`${TAG} [L0-delete] SKIPPED: totalL0=${totalL0} <= minRetain=${MIN_RETAIN_L0}`);
102
+ }
103
+ else {
104
+ try {
105
+ removedL0 = await vectorStore.deleteL0Expired(cutoffIso);
106
+ }
107
+ catch (err) {
108
+ failedL0DbCleanup = 1;
109
+ this.opts.logger?.warn(`${TAG} [L0-delete] FAILED: ${err instanceof Error ? err.message : String(err)}`);
110
+ }
111
+ }
112
+ // ── L1 cleanup with minimum-retention guard ──
113
+ if (totalL1 <= MIN_RETAIN_L1) {
114
+ skippedL1 = true;
115
+ this.opts.logger?.info(`${TAG} [L1-delete] SKIPPED: totalL1=${totalL1} <= minRetain=${MIN_RETAIN_L1}`);
116
+ }
117
+ else {
118
+ try {
119
+ removedL1 = await vectorStore.deleteL1Expired(cutoffIso);
120
+ }
121
+ catch (err) {
122
+ failedL1DbCleanup = 1;
123
+ this.opts.logger?.warn(`${TAG} [L1-delete] FAILED: ${err instanceof Error ? err.message : String(err)}`);
124
+ }
125
+ }
126
+ if (removedL1 > 0 || removedL0 > 0) {
127
+ total.changedFiles += 1;
128
+ }
129
+ // ── Post-delete: audit summary ──
130
+ const durationMs = Date.now() - startMs;
131
+ const remainingL0 = totalL0 - removedL0;
132
+ const remainingL1 = totalL1 - removedL1;
133
+ const summary = {
134
+ event: "cleaner_summary",
135
+ cutoffIso,
136
+ retentionDays,
137
+ l0: { total: totalL0, expired: removedL0, remaining: remainingL0, skipped: skippedL0, failed: failedL0DbCleanup > 0 },
138
+ l1: { total: totalL1, expired: removedL1, remaining: remainingL1, skipped: skippedL1, failed: failedL1DbCleanup > 0 },
139
+ durationMs,
140
+ };
141
+ this.opts.logger?.info(`${TAG} ${JSON.stringify(summary)}`);
142
+ }
143
+ this.opts.logger?.info(`${TAG} Cleanup done: scannedFiles=${total.scannedFiles}, changedFiles=${total.changedFiles}, skippedNonShardFiles=${total.skippedNonShardFiles}, deleteFailedFiles=${total.deleteFailedFiles}`);
144
+ }
145
+ scheduleNext() {
146
+ const nowMs = Date.now();
147
+ const now = new Date(nowMs);
148
+ const next = nextRunAt(this.opts.cleanTime, nowMs);
149
+ const targetToday = buildTodayRunTime(this.opts.cleanTime, nowMs);
150
+ const passedToday = targetToday <= nowMs;
151
+ const delayMs = Math.max(0, next - nowMs);
152
+ this.opts.logger?.debug?.(`${TAG} Schedule next run: nowLocal=${formatLocalDateTime(now)}, cleanTime=${this.opts.cleanTime}, targetTodayLocal=${formatLocalDateTime(new Date(targetToday))}, passedToday=${passedToday}, nextRunLocal=${formatLocalDateTime(new Date(next))}, nextRunIso=${new Date(next).toISOString()}, delayMs=${delayMs}`);
153
+ this.timer.scheduleAt(next, () => {
154
+ const firedAtMs = Date.now();
155
+ this.opts.logger?.info(`${TAG} Timer fired: scheduledLocal=${formatLocalDateTime(new Date(next))}, firedLocal=${formatLocalDateTime(new Date(firedAtMs))}, driftMs=${firedAtMs - next}`);
156
+ void this.runAndReschedule();
157
+ });
158
+ }
159
+ async runAndReschedule() {
160
+ if (this.destroyed)
161
+ return;
162
+ const runStart = new Date();
163
+ this.opts.logger?.info(`${TAG} Cleanup tick start: nowLocal=${formatLocalDateTime(runStart)}, nowIso=${runStart.toISOString()}`);
164
+ try {
165
+ await this.runOnce();
166
+ }
167
+ catch (err) {
168
+ this.opts.logger?.error(`${TAG} Cleanup failed: ${err instanceof Error ? err.stack ?? err.message : String(err)}`);
169
+ }
170
+ finally {
171
+ if (!this.destroyed) {
172
+ this.scheduleNext();
173
+ }
174
+ }
175
+ }
176
+ async cleanDirectory(dirPath, cutoffMs) {
177
+ const stats = {
178
+ scannedFiles: 0,
179
+ changedFiles: 0,
180
+ skippedNonShardFiles: 0,
181
+ deleteFailedFiles: 0,
182
+ };
183
+ let entries;
184
+ try {
185
+ entries = await fs.readdir(dirPath, { withFileTypes: true });
186
+ }
187
+ catch {
188
+ this.opts.logger?.debug?.(`${TAG} Directory not found, skip: ${dirPath}`);
189
+ return stats;
190
+ }
191
+ for (const entry of entries) {
192
+ if (!entry.isFile())
193
+ continue;
194
+ if (!isJsonLikeFile(entry.name))
195
+ continue;
196
+ const filePath = path.join(dirPath, entry.name);
197
+ stats.scannedFiles += 1;
198
+ // 仅支持日期分片文件:YYYY-MM-DD(.jsonl/.json)
199
+ const shard = extractShardDateFromFileName(entry.name);
200
+ if (!shard) {
201
+ stats.skippedNonShardFiles += 1;
202
+ this.opts.logger?.debug?.(`${TAG} Skip non-shard file: ${filePath}`);
203
+ continue;
204
+ }
205
+ const dayEndMs = localDayEndMs(shard.year, shard.month, shard.day);
206
+ if (dayEndMs < cutoffMs) {
207
+ try {
208
+ await fs.unlink(filePath);
209
+ stats.changedFiles += 1;
210
+ this.opts.logger?.info(`${TAG} Removed expired file by name: ${filePath}`);
211
+ }
212
+ catch (err) {
213
+ stats.deleteFailedFiles += 1;
214
+ this.opts.logger?.warn(`${TAG} Failed to delete expired shard file ${filePath}: ${err instanceof Error ? err.message : String(err)}`);
215
+ }
216
+ }
217
+ else {
218
+ this.opts.logger?.debug?.(`${TAG} Keep shard file by name: ${filePath}`);
219
+ }
220
+ }
221
+ return stats;
222
+ }
223
+ }
224
+ function isJsonLikeFile(name) {
225
+ return name.endsWith(".jsonl") || name.endsWith(".json");
226
+ }
227
+ function extractShardDateFromFileName(fileName) {
228
+ // Supported format: YYYY-MM-DD.jsonl | YYYY-MM-DD.json
229
+ const m = /^(\d{4})-(\d{2})-(\d{2})\.(?:jsonl|json)$/.exec(fileName);
230
+ if (!m)
231
+ return undefined;
232
+ const year = Number(m[1]);
233
+ const month = Number(m[2]);
234
+ const day = Number(m[3]);
235
+ if (!Number.isInteger(year) || !Number.isInteger(month) || !Number.isInteger(day)) {
236
+ return undefined;
237
+ }
238
+ if (month < 1 || month > 12 || day < 1 || day > 31) {
239
+ return undefined;
240
+ }
241
+ const probe = new Date(year, month - 1, day);
242
+ if (probe.getFullYear() !== year
243
+ || probe.getMonth() !== month - 1
244
+ || probe.getDate() !== day) {
245
+ return undefined;
246
+ }
247
+ return { year, month, day };
248
+ }
249
+ function localDayEndMs(year, month, day) {
250
+ // End of day = start of next day minus 1ms (in configured timezone)
251
+ const nextDay = new Date(Date.UTC(year, month - 1, day + 1));
252
+ const nextDayStartMs = startOfLocalDay(nextDay);
253
+ return nextDayStartMs - 1;
254
+ }
255
+ function formatUtcOffset(offsetMinutes) {
256
+ const sign = offsetMinutes >= 0 ? "+" : "-";
257
+ const abs = Math.abs(offsetMinutes);
258
+ const hh = String(Math.floor(abs / 60)).padStart(2, "0");
259
+ const mm = String(abs % 60).padStart(2, "0");
260
+ return `${sign}${hh}:${mm}`;
261
+ }
262
+ function computeCutoffMsByLocalDay(nowMs, retentionDays) {
263
+ // 自然日策略,保留"今天 + 往前 retentionDays-1 天"
264
+ // 删除阈值为 keepStart 当天 00:00:00.000(配置时区)
265
+ const now = new Date(nowMs);
266
+ const todayStartMs = startOfLocalDay(now);
267
+ const cutoffMs = todayStartMs - (retentionDays - 1) * 24 * 60 * 60 * 1000;
268
+ // Sanity check: cutoff must be strictly in the past
269
+ if (cutoffMs >= nowMs) {
270
+ throw new Error(`cutoff sanity failed: cutoff (${cutoffMs}) >= now (${nowMs}), ` +
271
+ `possible clock skew or invalid retentionDays=${retentionDays}`);
272
+ }
273
+ // Sanity check: gap between now and cutoff must be at least 24h
274
+ const MIN_GAP_MS = 24 * 60 * 60 * 1000;
275
+ if (nowMs - cutoffMs < MIN_GAP_MS) {
276
+ throw new Error(`cutoff sanity failed: gap ${nowMs - cutoffMs}ms < 24h, ` +
277
+ `retentionDays=${retentionDays}, possible clock skew`);
278
+ }
279
+ return cutoffMs;
280
+ }
281
+ function buildTodayRunTime(cleanTime, nowMs) {
282
+ const [hRaw, mRaw] = cleanTime.split(":");
283
+ const hour = Number(hRaw);
284
+ const minute = Number(mRaw);
285
+ const target = new Date(nowMs);
286
+ target.setHours(hour, minute, 0, 0);
287
+ return target.getTime();
288
+ }
289
+ function nextRunAt(cleanTime, nowMs) {
290
+ const [hRaw, mRaw] = cleanTime.split(":");
291
+ const hour = Number(hRaw);
292
+ const minute = Number(mRaw);
293
+ const now = new Date(nowMs);
294
+ const next = new Date(nowMs);
295
+ next.setHours(hour, minute, 0, 0);
296
+ if (next.getTime() <= now.getTime()) {
297
+ next.setDate(next.getDate() + 1);
298
+ }
299
+ return next.getTime();
300
+ }