dsh-tdai-memory 0.2.13 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/LICENSE +21 -21
  2. package/README.md +15 -13
  3. package/README.zh.md +9 -7
  4. package/THIRD-PARTY-NOTICES.md +20 -20
  5. package/client.js +21 -4
  6. package/cordis.patch.yml +8 -8
  7. package/index.js +36 -14
  8. package/package.json +77 -78
  9. package/recall-inject.js +86 -86
  10. package/vendor/tdai/adapters/index.js +15 -15
  11. package/vendor/tdai/adapters/openclaw/host-adapter.js +83 -83
  12. package/vendor/tdai/adapters/openclaw/index.js +5 -5
  13. package/vendor/tdai/adapters/openclaw/llm-runner.js +69 -69
  14. package/vendor/tdai/adapters/standalone/host-adapter.js +58 -58
  15. package/vendor/tdai/adapters/standalone/index.js +5 -5
  16. package/vendor/tdai/adapters/standalone/llm-runner.js +34 -6
  17. package/vendor/tdai/config.js +3 -0
  18. package/vendor/tdai/core/conversation/l0-recorder.js +420 -420
  19. package/vendor/tdai/core/hooks/auto-capture.js +236 -236
  20. package/vendor/tdai/core/hooks/auto-recall.js +640 -640
  21. package/vendor/tdai/core/index.js +8 -8
  22. package/vendor/tdai/core/persona/persona-generator.js +184 -184
  23. package/vendor/tdai/core/persona/persona-trigger.js +94 -94
  24. package/vendor/tdai/core/profile/profile-sync.js +200 -200
  25. package/vendor/tdai/core/prompts/l1-dedup.js +72 -72
  26. package/vendor/tdai/core/prompts/l1-extraction.js +34 -34
  27. package/vendor/tdai/core/prompts/persona-generation.js +35 -35
  28. package/vendor/tdai/core/prompts/scene-extraction.js +45 -45
  29. package/vendor/tdai/core/record/l1-dedup.js +318 -318
  30. package/vendor/tdai/core/record/l1-extractor.js +379 -379
  31. package/vendor/tdai/core/record/l1-reader.js +178 -178
  32. package/vendor/tdai/core/record/l1-writer.js +141 -141
  33. package/vendor/tdai/core/report/reporter.js +86 -86
  34. package/vendor/tdai/core/scene/filename-normalizer.js +161 -161
  35. package/vendor/tdai/core/scene/scene-extractor.js +410 -410
  36. package/vendor/tdai/core/scene/scene-format.js +53 -53
  37. package/vendor/tdai/core/scene/scene-index.js +82 -82
  38. package/vendor/tdai/core/scene/scene-navigation.js +57 -57
  39. package/vendor/tdai/core/seed/input.js +404 -404
  40. package/vendor/tdai/core/seed/seed-runtime.js +4 -0
  41. package/vendor/tdai/core/seed/types.js +9 -9
  42. package/vendor/tdai/core/store/bm25-client.js +117 -117
  43. package/vendor/tdai/core/store/bm25-local.js +68 -68
  44. package/vendor/tdai/core/store/embedding.js +632 -632
  45. package/vendor/tdai/core/store/factory.js +95 -95
  46. package/vendor/tdai/core/store/search-utils.js +54 -54
  47. package/vendor/tdai/core/store/sqlite.js +1632 -1632
  48. package/vendor/tdai/core/store/tcvdb-client.js +221 -221
  49. package/vendor/tdai/core/store/tcvdb.js +1047 -1047
  50. package/vendor/tdai/core/store/types.js +17 -17
  51. package/vendor/tdai/core/tdai-core.js +4 -0
  52. package/vendor/tdai/core/tools/conversation-search.js +199 -199
  53. package/vendor/tdai/core/tools/memory-search.js +207 -207
  54. package/vendor/tdai/core/types.js +12 -12
  55. package/vendor/tdai/utils/backup.js +189 -189
  56. package/vendor/tdai/utils/checkpoint.js +360 -360
  57. package/vendor/tdai/utils/clean-context-runner.js +472 -472
  58. package/vendor/tdai/utils/ensure-hook-policy.js +213 -213
  59. package/vendor/tdai/utils/env.js +13 -13
  60. package/vendor/tdai/utils/managed-timer.js +127 -127
  61. package/vendor/tdai/utils/manifest.js +91 -91
  62. package/vendor/tdai/utils/memory-cleaner.js +300 -300
  63. package/vendor/tdai/utils/no-think-fetch.js +109 -109
  64. package/vendor/tdai/utils/openclaw-state-dir.js +26 -26
  65. package/vendor/tdai/utils/pipeline-factory.js +517 -517
  66. package/vendor/tdai/utils/pipeline-manager.js +892 -892
  67. package/vendor/tdai/utils/sanitize.js +360 -360
  68. package/vendor/tdai/utils/serial-queue.js +103 -103
  69. package/vendor/tdai/utils/session-filter.js +88 -88
  70. package/vendor/tdai/utils/session-header-fetch.js +88 -0
  71. package/vendor/tdai/utils/text-utils.js +29 -29
  72. package/vendor/tdai/utils/time.js +252 -252
@@ -1,300 +1,300 @@
1
- import fs from "node:fs/promises";
2
- import path from "node:path";
3
- import { ManagedTimer } from "./managed-timer.js";
4
- import { formatLocalDateTime, startOfLocalDay } from "./time.js";
5
- const TAG = "[memory-tdai][cleaner]";
6
- const L0_DIR_NAME = "conversations";
7
- const L1_DIR_NAME = "records";
8
- /** Minimum records to retain — skip deletion if total is at or below this threshold. */
9
- const MIN_RETAIN_L0 = 50;
10
- const MIN_RETAIN_L1 = 20;
11
- export class LocalMemoryCleaner {
12
- opts;
13
- timer;
14
- destroyed = false;
15
- vectorStore;
16
- constructor(opts) {
17
- this.opts = opts;
18
- this.timer = new ManagedTimer("memory-tdai-cleaner", () => this.destroyed);
19
- this.vectorStore = opts.vectorStore;
20
- }
21
- setVectorStore(vectorStore) {
22
- this.vectorStore = vectorStore;
23
- }
24
- start() {
25
- if (this.destroyed)
26
- return;
27
- const now = new Date();
28
- const tz = Intl.DateTimeFormat().resolvedOptions().timeZone || "unknown";
29
- const utcOffset = formatUtcOffset(-now.getTimezoneOffset());
30
- this.opts.logger?.debug?.(`${TAG} Enabled: retentionDays=${this.opts.retentionDays}, cleanTime=${this.opts.cleanTime}, dirs=[${L0_DIR_NAME}, ${L1_DIR_NAME}]`);
31
- this.opts.logger?.debug?.(`${TAG} Runtime clock: nowLocal=${formatLocalDateTime(now)}, nowIso=${now.toISOString()}, tz=${tz}, utcOffset=${utcOffset}`);
32
- this.scheduleNext();
33
- }
34
- destroy() {
35
- if (this.destroyed)
36
- return;
37
- this.destroyed = true;
38
- this.timer.cancel();
39
- this.opts.logger?.info(`${TAG} Stopped`);
40
- }
41
- async runOnce(nowMs = Date.now()) {
42
- if (this.destroyed)
43
- return;
44
- const retentionDays = this.opts.retentionDays;
45
- if (!(retentionDays > 0)) {
46
- this.opts.logger?.debug?.(`${TAG} Skip run: invalid retentionDays=${retentionDays}`);
47
- return;
48
- }
49
- // 按"本地自然日"保留策略计算截止时间。
50
- // 例如 retentionDays=2,今天是 03-15,则保留 03-14/03-15,删除早于 03-14 00:00:00.000 的记录。
51
- let cutoffMs;
52
- try {
53
- cutoffMs = computeCutoffMsByLocalDay(nowMs, retentionDays);
54
- }
55
- catch (err) {
56
- this.opts.logger?.error(`${TAG} ${err instanceof Error ? err.message : String(err)}`);
57
- return;
58
- }
59
- const targetDirs = [
60
- path.join(this.opts.baseDir, L0_DIR_NAME),
61
- path.join(this.opts.baseDir, L1_DIR_NAME),
62
- ];
63
- const total = {
64
- scannedFiles: 0,
65
- changedFiles: 0,
66
- skippedNonShardFiles: 0,
67
- deleteFailedFiles: 0,
68
- };
69
- for (const dirPath of targetDirs) {
70
- const stats = await this.cleanDirectory(dirPath, cutoffMs);
71
- total.scannedFiles += stats.scannedFiles;
72
- total.changedFiles += stats.changedFiles;
73
- total.skippedNonShardFiles += stats.skippedNonShardFiles;
74
- total.deleteFailedFiles += stats.deleteFailedFiles;
75
- }
76
- if (this.vectorStore) {
77
- const vectorStore = this.vectorStore;
78
- const cutoffIso = new Date(cutoffMs).toISOString();
79
- const startMs = Date.now();
80
- // ── Pre-delete: count totals and decide whether to proceed ──
81
- let totalL0 = 0;
82
- let totalL1 = 0;
83
- try {
84
- totalL0 = await vectorStore.countL0();
85
- }
86
- catch { /* non-fatal */ }
87
- try {
88
- totalL1 = await vectorStore.countL1();
89
- }
90
- catch { /* non-fatal */ }
91
- this.opts.logger?.info(`${TAG} [Pre-delete] cutoffIso=${cutoffIso}, retentionDays=${retentionDays}, totalL0=${totalL0}, totalL1=${totalL1}`);
92
- let removedL0 = 0;
93
- let removedL1 = 0;
94
- let skippedL0 = false;
95
- let skippedL1 = false;
96
- let failedL0DbCleanup = 0;
97
- let failedL1DbCleanup = 0;
98
- // ── L0 cleanup with minimum-retention guard ──
99
- if (totalL0 <= MIN_RETAIN_L0) {
100
- skippedL0 = true;
101
- this.opts.logger?.info(`${TAG} [L0-delete] SKIPPED: totalL0=${totalL0} <= minRetain=${MIN_RETAIN_L0}`);
102
- }
103
- else {
104
- try {
105
- removedL0 = await vectorStore.deleteL0Expired(cutoffIso);
106
- }
107
- catch (err) {
108
- failedL0DbCleanup = 1;
109
- this.opts.logger?.warn(`${TAG} [L0-delete] FAILED: ${err instanceof Error ? err.message : String(err)}`);
110
- }
111
- }
112
- // ── L1 cleanup with minimum-retention guard ──
113
- if (totalL1 <= MIN_RETAIN_L1) {
114
- skippedL1 = true;
115
- this.opts.logger?.info(`${TAG} [L1-delete] SKIPPED: totalL1=${totalL1} <= minRetain=${MIN_RETAIN_L1}`);
116
- }
117
- else {
118
- try {
119
- removedL1 = await vectorStore.deleteL1Expired(cutoffIso);
120
- }
121
- catch (err) {
122
- failedL1DbCleanup = 1;
123
- this.opts.logger?.warn(`${TAG} [L1-delete] FAILED: ${err instanceof Error ? err.message : String(err)}`);
124
- }
125
- }
126
- if (removedL1 > 0 || removedL0 > 0) {
127
- total.changedFiles += 1;
128
- }
129
- // ── Post-delete: audit summary ──
130
- const durationMs = Date.now() - startMs;
131
- const remainingL0 = totalL0 - removedL0;
132
- const remainingL1 = totalL1 - removedL1;
133
- const summary = {
134
- event: "cleaner_summary",
135
- cutoffIso,
136
- retentionDays,
137
- l0: { total: totalL0, expired: removedL0, remaining: remainingL0, skipped: skippedL0, failed: failedL0DbCleanup > 0 },
138
- l1: { total: totalL1, expired: removedL1, remaining: remainingL1, skipped: skippedL1, failed: failedL1DbCleanup > 0 },
139
- durationMs,
140
- };
141
- this.opts.logger?.info(`${TAG} ${JSON.stringify(summary)}`);
142
- }
143
- this.opts.logger?.info(`${TAG} Cleanup done: scannedFiles=${total.scannedFiles}, changedFiles=${total.changedFiles}, skippedNonShardFiles=${total.skippedNonShardFiles}, deleteFailedFiles=${total.deleteFailedFiles}`);
144
- }
145
- scheduleNext() {
146
- const nowMs = Date.now();
147
- const now = new Date(nowMs);
148
- const next = nextRunAt(this.opts.cleanTime, nowMs);
149
- const targetToday = buildTodayRunTime(this.opts.cleanTime, nowMs);
150
- const passedToday = targetToday <= nowMs;
151
- const delayMs = Math.max(0, next - nowMs);
152
- this.opts.logger?.debug?.(`${TAG} Schedule next run: nowLocal=${formatLocalDateTime(now)}, cleanTime=${this.opts.cleanTime}, targetTodayLocal=${formatLocalDateTime(new Date(targetToday))}, passedToday=${passedToday}, nextRunLocal=${formatLocalDateTime(new Date(next))}, nextRunIso=${new Date(next).toISOString()}, delayMs=${delayMs}`);
153
- this.timer.scheduleAt(next, () => {
154
- const firedAtMs = Date.now();
155
- this.opts.logger?.info(`${TAG} Timer fired: scheduledLocal=${formatLocalDateTime(new Date(next))}, firedLocal=${formatLocalDateTime(new Date(firedAtMs))}, driftMs=${firedAtMs - next}`);
156
- void this.runAndReschedule();
157
- });
158
- }
159
- async runAndReschedule() {
160
- if (this.destroyed)
161
- return;
162
- const runStart = new Date();
163
- this.opts.logger?.info(`${TAG} Cleanup tick start: nowLocal=${formatLocalDateTime(runStart)}, nowIso=${runStart.toISOString()}`);
164
- try {
165
- await this.runOnce();
166
- }
167
- catch (err) {
168
- this.opts.logger?.error(`${TAG} Cleanup failed: ${err instanceof Error ? err.stack ?? err.message : String(err)}`);
169
- }
170
- finally {
171
- if (!this.destroyed) {
172
- this.scheduleNext();
173
- }
174
- }
175
- }
176
- async cleanDirectory(dirPath, cutoffMs) {
177
- const stats = {
178
- scannedFiles: 0,
179
- changedFiles: 0,
180
- skippedNonShardFiles: 0,
181
- deleteFailedFiles: 0,
182
- };
183
- let entries;
184
- try {
185
- entries = await fs.readdir(dirPath, { withFileTypes: true });
186
- }
187
- catch {
188
- this.opts.logger?.debug?.(`${TAG} Directory not found, skip: ${dirPath}`);
189
- return stats;
190
- }
191
- for (const entry of entries) {
192
- if (!entry.isFile())
193
- continue;
194
- if (!isJsonLikeFile(entry.name))
195
- continue;
196
- const filePath = path.join(dirPath, entry.name);
197
- stats.scannedFiles += 1;
198
- // 仅支持日期分片文件:YYYY-MM-DD(.jsonl/.json)
199
- const shard = extractShardDateFromFileName(entry.name);
200
- if (!shard) {
201
- stats.skippedNonShardFiles += 1;
202
- this.opts.logger?.debug?.(`${TAG} Skip non-shard file: ${filePath}`);
203
- continue;
204
- }
205
- const dayEndMs = localDayEndMs(shard.year, shard.month, shard.day);
206
- if (dayEndMs < cutoffMs) {
207
- try {
208
- await fs.unlink(filePath);
209
- stats.changedFiles += 1;
210
- this.opts.logger?.info(`${TAG} Removed expired file by name: ${filePath}`);
211
- }
212
- catch (err) {
213
- stats.deleteFailedFiles += 1;
214
- this.opts.logger?.warn(`${TAG} Failed to delete expired shard file ${filePath}: ${err instanceof Error ? err.message : String(err)}`);
215
- }
216
- }
217
- else {
218
- this.opts.logger?.debug?.(`${TAG} Keep shard file by name: ${filePath}`);
219
- }
220
- }
221
- return stats;
222
- }
223
- }
224
- function isJsonLikeFile(name) {
225
- return name.endsWith(".jsonl") || name.endsWith(".json");
226
- }
227
- function extractShardDateFromFileName(fileName) {
228
- // Supported format: YYYY-MM-DD.jsonl | YYYY-MM-DD.json
229
- const m = /^(\d{4})-(\d{2})-(\d{2})\.(?:jsonl|json)$/.exec(fileName);
230
- if (!m)
231
- return undefined;
232
- const year = Number(m[1]);
233
- const month = Number(m[2]);
234
- const day = Number(m[3]);
235
- if (!Number.isInteger(year) || !Number.isInteger(month) || !Number.isInteger(day)) {
236
- return undefined;
237
- }
238
- if (month < 1 || month > 12 || day < 1 || day > 31) {
239
- return undefined;
240
- }
241
- const probe = new Date(year, month - 1, day);
242
- if (probe.getFullYear() !== year
243
- || probe.getMonth() !== month - 1
244
- || probe.getDate() !== day) {
245
- return undefined;
246
- }
247
- return { year, month, day };
248
- }
249
- function localDayEndMs(year, month, day) {
250
- // End of day = start of next day minus 1ms (in configured timezone)
251
- const nextDay = new Date(Date.UTC(year, month - 1, day + 1));
252
- const nextDayStartMs = startOfLocalDay(nextDay);
253
- return nextDayStartMs - 1;
254
- }
255
- function formatUtcOffset(offsetMinutes) {
256
- const sign = offsetMinutes >= 0 ? "+" : "-";
257
- const abs = Math.abs(offsetMinutes);
258
- const hh = String(Math.floor(abs / 60)).padStart(2, "0");
259
- const mm = String(abs % 60).padStart(2, "0");
260
- return `${sign}${hh}:${mm}`;
261
- }
262
- function computeCutoffMsByLocalDay(nowMs, retentionDays) {
263
- // 自然日策略,保留"今天 + 往前 retentionDays-1 天"
264
- // 删除阈值为 keepStart 当天 00:00:00.000(配置时区)
265
- const now = new Date(nowMs);
266
- const todayStartMs = startOfLocalDay(now);
267
- const cutoffMs = todayStartMs - (retentionDays - 1) * 24 * 60 * 60 * 1000;
268
- // Sanity check: cutoff must be strictly in the past
269
- if (cutoffMs >= nowMs) {
270
- throw new Error(`cutoff sanity failed: cutoff (${cutoffMs}) >= now (${nowMs}), ` +
271
- `possible clock skew or invalid retentionDays=${retentionDays}`);
272
- }
273
- // Sanity check: gap between now and cutoff must be at least 24h
274
- const MIN_GAP_MS = 24 * 60 * 60 * 1000;
275
- if (nowMs - cutoffMs < MIN_GAP_MS) {
276
- throw new Error(`cutoff sanity failed: gap ${nowMs - cutoffMs}ms < 24h, ` +
277
- `retentionDays=${retentionDays}, possible clock skew`);
278
- }
279
- return cutoffMs;
280
- }
281
- function buildTodayRunTime(cleanTime, nowMs) {
282
- const [hRaw, mRaw] = cleanTime.split(":");
283
- const hour = Number(hRaw);
284
- const minute = Number(mRaw);
285
- const target = new Date(nowMs);
286
- target.setHours(hour, minute, 0, 0);
287
- return target.getTime();
288
- }
289
- function nextRunAt(cleanTime, nowMs) {
290
- const [hRaw, mRaw] = cleanTime.split(":");
291
- const hour = Number(hRaw);
292
- const minute = Number(mRaw);
293
- const now = new Date(nowMs);
294
- const next = new Date(nowMs);
295
- next.setHours(hour, minute, 0, 0);
296
- if (next.getTime() <= now.getTime()) {
297
- next.setDate(next.getDate() + 1);
298
- }
299
- return next.getTime();
300
- }
1
+ import fs from "node:fs/promises";
2
+ import path from "node:path";
3
+ import { ManagedTimer } from "./managed-timer.js";
4
+ import { formatLocalDateTime, startOfLocalDay } from "./time.js";
5
+ const TAG = "[memory-tdai][cleaner]";
6
+ const L0_DIR_NAME = "conversations";
7
+ const L1_DIR_NAME = "records";
8
+ /** Minimum records to retain — skip deletion if total is at or below this threshold. */
9
+ const MIN_RETAIN_L0 = 50;
10
+ const MIN_RETAIN_L1 = 20;
11
+ export class LocalMemoryCleaner {
12
+ opts;
13
+ timer;
14
+ destroyed = false;
15
+ vectorStore;
16
+ constructor(opts) {
17
+ this.opts = opts;
18
+ this.timer = new ManagedTimer("memory-tdai-cleaner", () => this.destroyed);
19
+ this.vectorStore = opts.vectorStore;
20
+ }
21
+ setVectorStore(vectorStore) {
22
+ this.vectorStore = vectorStore;
23
+ }
24
+ start() {
25
+ if (this.destroyed)
26
+ return;
27
+ const now = new Date();
28
+ const tz = Intl.DateTimeFormat().resolvedOptions().timeZone || "unknown";
29
+ const utcOffset = formatUtcOffset(-now.getTimezoneOffset());
30
+ this.opts.logger?.debug?.(`${TAG} Enabled: retentionDays=${this.opts.retentionDays}, cleanTime=${this.opts.cleanTime}, dirs=[${L0_DIR_NAME}, ${L1_DIR_NAME}]`);
31
+ this.opts.logger?.debug?.(`${TAG} Runtime clock: nowLocal=${formatLocalDateTime(now)}, nowIso=${now.toISOString()}, tz=${tz}, utcOffset=${utcOffset}`);
32
+ this.scheduleNext();
33
+ }
34
+ destroy() {
35
+ if (this.destroyed)
36
+ return;
37
+ this.destroyed = true;
38
+ this.timer.cancel();
39
+ this.opts.logger?.info(`${TAG} Stopped`);
40
+ }
41
+ async runOnce(nowMs = Date.now()) {
42
+ if (this.destroyed)
43
+ return;
44
+ const retentionDays = this.opts.retentionDays;
45
+ if (!(retentionDays > 0)) {
46
+ this.opts.logger?.debug?.(`${TAG} Skip run: invalid retentionDays=${retentionDays}`);
47
+ return;
48
+ }
49
+ // 按"本地自然日"保留策略计算截止时间。
50
+ // 例如 retentionDays=2,今天是 03-15,则保留 03-14/03-15,删除早于 03-14 00:00:00.000 的记录。
51
+ let cutoffMs;
52
+ try {
53
+ cutoffMs = computeCutoffMsByLocalDay(nowMs, retentionDays);
54
+ }
55
+ catch (err) {
56
+ this.opts.logger?.error(`${TAG} ${err instanceof Error ? err.message : String(err)}`);
57
+ return;
58
+ }
59
+ const targetDirs = [
60
+ path.join(this.opts.baseDir, L0_DIR_NAME),
61
+ path.join(this.opts.baseDir, L1_DIR_NAME),
62
+ ];
63
+ const total = {
64
+ scannedFiles: 0,
65
+ changedFiles: 0,
66
+ skippedNonShardFiles: 0,
67
+ deleteFailedFiles: 0,
68
+ };
69
+ for (const dirPath of targetDirs) {
70
+ const stats = await this.cleanDirectory(dirPath, cutoffMs);
71
+ total.scannedFiles += stats.scannedFiles;
72
+ total.changedFiles += stats.changedFiles;
73
+ total.skippedNonShardFiles += stats.skippedNonShardFiles;
74
+ total.deleteFailedFiles += stats.deleteFailedFiles;
75
+ }
76
+ if (this.vectorStore) {
77
+ const vectorStore = this.vectorStore;
78
+ const cutoffIso = new Date(cutoffMs).toISOString();
79
+ const startMs = Date.now();
80
+ // ── Pre-delete: count totals and decide whether to proceed ──
81
+ let totalL0 = 0;
82
+ let totalL1 = 0;
83
+ try {
84
+ totalL0 = await vectorStore.countL0();
85
+ }
86
+ catch { /* non-fatal */ }
87
+ try {
88
+ totalL1 = await vectorStore.countL1();
89
+ }
90
+ catch { /* non-fatal */ }
91
+ this.opts.logger?.info(`${TAG} [Pre-delete] cutoffIso=${cutoffIso}, retentionDays=${retentionDays}, totalL0=${totalL0}, totalL1=${totalL1}`);
92
+ let removedL0 = 0;
93
+ let removedL1 = 0;
94
+ let skippedL0 = false;
95
+ let skippedL1 = false;
96
+ let failedL0DbCleanup = 0;
97
+ let failedL1DbCleanup = 0;
98
+ // ── L0 cleanup with minimum-retention guard ──
99
+ if (totalL0 <= MIN_RETAIN_L0) {
100
+ skippedL0 = true;
101
+ this.opts.logger?.info(`${TAG} [L0-delete] SKIPPED: totalL0=${totalL0} <= minRetain=${MIN_RETAIN_L0}`);
102
+ }
103
+ else {
104
+ try {
105
+ removedL0 = await vectorStore.deleteL0Expired(cutoffIso);
106
+ }
107
+ catch (err) {
108
+ failedL0DbCleanup = 1;
109
+ this.opts.logger?.warn(`${TAG} [L0-delete] FAILED: ${err instanceof Error ? err.message : String(err)}`);
110
+ }
111
+ }
112
+ // ── L1 cleanup with minimum-retention guard ──
113
+ if (totalL1 <= MIN_RETAIN_L1) {
114
+ skippedL1 = true;
115
+ this.opts.logger?.info(`${TAG} [L1-delete] SKIPPED: totalL1=${totalL1} <= minRetain=${MIN_RETAIN_L1}`);
116
+ }
117
+ else {
118
+ try {
119
+ removedL1 = await vectorStore.deleteL1Expired(cutoffIso);
120
+ }
121
+ catch (err) {
122
+ failedL1DbCleanup = 1;
123
+ this.opts.logger?.warn(`${TAG} [L1-delete] FAILED: ${err instanceof Error ? err.message : String(err)}`);
124
+ }
125
+ }
126
+ if (removedL1 > 0 || removedL0 > 0) {
127
+ total.changedFiles += 1;
128
+ }
129
+ // ── Post-delete: audit summary ──
130
+ const durationMs = Date.now() - startMs;
131
+ const remainingL0 = totalL0 - removedL0;
132
+ const remainingL1 = totalL1 - removedL1;
133
+ const summary = {
134
+ event: "cleaner_summary",
135
+ cutoffIso,
136
+ retentionDays,
137
+ l0: { total: totalL0, expired: removedL0, remaining: remainingL0, skipped: skippedL0, failed: failedL0DbCleanup > 0 },
138
+ l1: { total: totalL1, expired: removedL1, remaining: remainingL1, skipped: skippedL1, failed: failedL1DbCleanup > 0 },
139
+ durationMs,
140
+ };
141
+ this.opts.logger?.info(`${TAG} ${JSON.stringify(summary)}`);
142
+ }
143
+ this.opts.logger?.info(`${TAG} Cleanup done: scannedFiles=${total.scannedFiles}, changedFiles=${total.changedFiles}, skippedNonShardFiles=${total.skippedNonShardFiles}, deleteFailedFiles=${total.deleteFailedFiles}`);
144
+ }
145
+ scheduleNext() {
146
+ const nowMs = Date.now();
147
+ const now = new Date(nowMs);
148
+ const next = nextRunAt(this.opts.cleanTime, nowMs);
149
+ const targetToday = buildTodayRunTime(this.opts.cleanTime, nowMs);
150
+ const passedToday = targetToday <= nowMs;
151
+ const delayMs = Math.max(0, next - nowMs);
152
+ this.opts.logger?.debug?.(`${TAG} Schedule next run: nowLocal=${formatLocalDateTime(now)}, cleanTime=${this.opts.cleanTime}, targetTodayLocal=${formatLocalDateTime(new Date(targetToday))}, passedToday=${passedToday}, nextRunLocal=${formatLocalDateTime(new Date(next))}, nextRunIso=${new Date(next).toISOString()}, delayMs=${delayMs}`);
153
+ this.timer.scheduleAt(next, () => {
154
+ const firedAtMs = Date.now();
155
+ this.opts.logger?.info(`${TAG} Timer fired: scheduledLocal=${formatLocalDateTime(new Date(next))}, firedLocal=${formatLocalDateTime(new Date(firedAtMs))}, driftMs=${firedAtMs - next}`);
156
+ void this.runAndReschedule();
157
+ });
158
+ }
159
+ async runAndReschedule() {
160
+ if (this.destroyed)
161
+ return;
162
+ const runStart = new Date();
163
+ this.opts.logger?.info(`${TAG} Cleanup tick start: nowLocal=${formatLocalDateTime(runStart)}, nowIso=${runStart.toISOString()}`);
164
+ try {
165
+ await this.runOnce();
166
+ }
167
+ catch (err) {
168
+ this.opts.logger?.error(`${TAG} Cleanup failed: ${err instanceof Error ? err.stack ?? err.message : String(err)}`);
169
+ }
170
+ finally {
171
+ if (!this.destroyed) {
172
+ this.scheduleNext();
173
+ }
174
+ }
175
+ }
176
+ async cleanDirectory(dirPath, cutoffMs) {
177
+ const stats = {
178
+ scannedFiles: 0,
179
+ changedFiles: 0,
180
+ skippedNonShardFiles: 0,
181
+ deleteFailedFiles: 0,
182
+ };
183
+ let entries;
184
+ try {
185
+ entries = await fs.readdir(dirPath, { withFileTypes: true });
186
+ }
187
+ catch {
188
+ this.opts.logger?.debug?.(`${TAG} Directory not found, skip: ${dirPath}`);
189
+ return stats;
190
+ }
191
+ for (const entry of entries) {
192
+ if (!entry.isFile())
193
+ continue;
194
+ if (!isJsonLikeFile(entry.name))
195
+ continue;
196
+ const filePath = path.join(dirPath, entry.name);
197
+ stats.scannedFiles += 1;
198
+ // 仅支持日期分片文件:YYYY-MM-DD(.jsonl/.json)
199
+ const shard = extractShardDateFromFileName(entry.name);
200
+ if (!shard) {
201
+ stats.skippedNonShardFiles += 1;
202
+ this.opts.logger?.debug?.(`${TAG} Skip non-shard file: ${filePath}`);
203
+ continue;
204
+ }
205
+ const dayEndMs = localDayEndMs(shard.year, shard.month, shard.day);
206
+ if (dayEndMs < cutoffMs) {
207
+ try {
208
+ await fs.unlink(filePath);
209
+ stats.changedFiles += 1;
210
+ this.opts.logger?.info(`${TAG} Removed expired file by name: ${filePath}`);
211
+ }
212
+ catch (err) {
213
+ stats.deleteFailedFiles += 1;
214
+ this.opts.logger?.warn(`${TAG} Failed to delete expired shard file ${filePath}: ${err instanceof Error ? err.message : String(err)}`);
215
+ }
216
+ }
217
+ else {
218
+ this.opts.logger?.debug?.(`${TAG} Keep shard file by name: ${filePath}`);
219
+ }
220
+ }
221
+ return stats;
222
+ }
223
+ }
224
+ function isJsonLikeFile(name) {
225
+ return name.endsWith(".jsonl") || name.endsWith(".json");
226
+ }
227
+ function extractShardDateFromFileName(fileName) {
228
+ // Supported format: YYYY-MM-DD.jsonl | YYYY-MM-DD.json
229
+ const m = /^(\d{4})-(\d{2})-(\d{2})\.(?:jsonl|json)$/.exec(fileName);
230
+ if (!m)
231
+ return undefined;
232
+ const year = Number(m[1]);
233
+ const month = Number(m[2]);
234
+ const day = Number(m[3]);
235
+ if (!Number.isInteger(year) || !Number.isInteger(month) || !Number.isInteger(day)) {
236
+ return undefined;
237
+ }
238
+ if (month < 1 || month > 12 || day < 1 || day > 31) {
239
+ return undefined;
240
+ }
241
+ const probe = new Date(year, month - 1, day);
242
+ if (probe.getFullYear() !== year
243
+ || probe.getMonth() !== month - 1
244
+ || probe.getDate() !== day) {
245
+ return undefined;
246
+ }
247
+ return { year, month, day };
248
+ }
249
+ function localDayEndMs(year, month, day) {
250
+ // End of day = start of next day minus 1ms (in configured timezone)
251
+ const nextDay = new Date(Date.UTC(year, month - 1, day + 1));
252
+ const nextDayStartMs = startOfLocalDay(nextDay);
253
+ return nextDayStartMs - 1;
254
+ }
255
+ function formatUtcOffset(offsetMinutes) {
256
+ const sign = offsetMinutes >= 0 ? "+" : "-";
257
+ const abs = Math.abs(offsetMinutes);
258
+ const hh = String(Math.floor(abs / 60)).padStart(2, "0");
259
+ const mm = String(abs % 60).padStart(2, "0");
260
+ return `${sign}${hh}:${mm}`;
261
+ }
262
+ function computeCutoffMsByLocalDay(nowMs, retentionDays) {
263
+ // 自然日策略,保留"今天 + 往前 retentionDays-1 天"
264
+ // 删除阈值为 keepStart 当天 00:00:00.000(配置时区)
265
+ const now = new Date(nowMs);
266
+ const todayStartMs = startOfLocalDay(now);
267
+ const cutoffMs = todayStartMs - (retentionDays - 1) * 24 * 60 * 60 * 1000;
268
+ // Sanity check: cutoff must be strictly in the past
269
+ if (cutoffMs >= nowMs) {
270
+ throw new Error(`cutoff sanity failed: cutoff (${cutoffMs}) >= now (${nowMs}), ` +
271
+ `possible clock skew or invalid retentionDays=${retentionDays}`);
272
+ }
273
+ // Sanity check: gap between now and cutoff must be at least 24h
274
+ const MIN_GAP_MS = 24 * 60 * 60 * 1000;
275
+ if (nowMs - cutoffMs < MIN_GAP_MS) {
276
+ throw new Error(`cutoff sanity failed: gap ${nowMs - cutoffMs}ms < 24h, ` +
277
+ `retentionDays=${retentionDays}, possible clock skew`);
278
+ }
279
+ return cutoffMs;
280
+ }
281
+ function buildTodayRunTime(cleanTime, nowMs) {
282
+ const [hRaw, mRaw] = cleanTime.split(":");
283
+ const hour = Number(hRaw);
284
+ const minute = Number(mRaw);
285
+ const target = new Date(nowMs);
286
+ target.setHours(hour, minute, 0, 0);
287
+ return target.getTime();
288
+ }
289
+ function nextRunAt(cleanTime, nowMs) {
290
+ const [hRaw, mRaw] = cleanTime.split(":");
291
+ const hour = Number(hRaw);
292
+ const minute = Number(mRaw);
293
+ const now = new Date(nowMs);
294
+ const next = new Date(nowMs);
295
+ next.setHours(hour, minute, 0, 0);
296
+ if (next.getTime() <= now.getTime()) {
297
+ next.setDate(next.getDate() + 1);
298
+ }
299
+ return next.getTime();
300
+ }