dsh-tdai-memory 0.2.13 → 0.2.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/THIRD-PARTY-NOTICES.md +20 -20
- package/client.js +8 -4
- package/cordis.patch.yml +8 -8
- package/index.js +414 -414
- package/package.json +1 -1
- package/recall-inject.js +86 -86
- package/vendor/tdai/adapters/index.js +15 -15
- package/vendor/tdai/adapters/openclaw/host-adapter.js +83 -83
- package/vendor/tdai/adapters/openclaw/index.js +5 -5
- package/vendor/tdai/adapters/openclaw/llm-runner.js +69 -69
- package/vendor/tdai/adapters/standalone/host-adapter.js +58 -58
- package/vendor/tdai/adapters/standalone/index.js +5 -5
- package/vendor/tdai/adapters/standalone/llm-runner.js +246 -246
- package/vendor/tdai/config.js +363 -363
- package/vendor/tdai/core/conversation/l0-recorder.js +420 -420
- package/vendor/tdai/core/hooks/auto-capture.js +236 -236
- package/vendor/tdai/core/hooks/auto-recall.js +653 -653
- package/vendor/tdai/core/index.js +8 -8
- package/vendor/tdai/core/persona/persona-generator.js +184 -184
- package/vendor/tdai/core/persona/persona-trigger.js +94 -94
- package/vendor/tdai/core/profile/profile-sync.js +200 -200
- package/vendor/tdai/core/prompts/l1-dedup.js +136 -136
- package/vendor/tdai/core/prompts/l1-extraction.js +132 -132
- package/vendor/tdai/core/prompts/persona-generation.js +160 -160
- package/vendor/tdai/core/prompts/scene-extraction.js +269 -269
- package/vendor/tdai/core/record/l1-dedup.js +318 -318
- package/vendor/tdai/core/record/l1-extractor.js +379 -379
- package/vendor/tdai/core/record/l1-reader.js +178 -178
- package/vendor/tdai/core/record/l1-writer.js +141 -141
- package/vendor/tdai/core/report/reporter.js +86 -86
- package/vendor/tdai/core/scene/filename-normalizer.js +161 -161
- package/vendor/tdai/core/scene/scene-extractor.js +410 -410
- package/vendor/tdai/core/scene/scene-format.js +53 -53
- package/vendor/tdai/core/scene/scene-index.js +82 -82
- package/vendor/tdai/core/scene/scene-navigation.js +60 -60
- package/vendor/tdai/core/seed/input.js +404 -404
- package/vendor/tdai/core/seed/seed-runtime.js +314 -314
- package/vendor/tdai/core/seed/types.js +9 -9
- package/vendor/tdai/core/store/bm25-client.js +117 -117
- package/vendor/tdai/core/store/bm25-local.js +68 -68
- package/vendor/tdai/core/store/embedding.js +632 -632
- package/vendor/tdai/core/store/factory.js +95 -95
- package/vendor/tdai/core/store/search-utils.js +54 -54
- package/vendor/tdai/core/store/sqlite.js +1822 -1822
- package/vendor/tdai/core/store/tcvdb-client.js +221 -221
- package/vendor/tdai/core/store/tcvdb.js +1047 -1047
- package/vendor/tdai/core/store/types.js +17 -17
- package/vendor/tdai/core/tdai-core.js +446 -446
- package/vendor/tdai/core/tools/conversation-search.js +199 -199
- package/vendor/tdai/core/tools/memory-search.js +207 -207
- package/vendor/tdai/core/types.js +12 -12
- package/vendor/tdai/utils/backup.js +189 -189
- package/vendor/tdai/utils/checkpoint.js +360 -360
- package/vendor/tdai/utils/clean-context-runner.js +472 -472
- package/vendor/tdai/utils/ensure-hook-policy.js +213 -213
- package/vendor/tdai/utils/env.js +13 -13
- package/vendor/tdai/utils/managed-timer.js +127 -127
- package/vendor/tdai/utils/manifest.js +91 -91
- package/vendor/tdai/utils/memory-cleaner.js +300 -300
- package/vendor/tdai/utils/no-think-fetch.js +109 -109
- package/vendor/tdai/utils/openclaw-state-dir.js +26 -26
- package/vendor/tdai/utils/pipeline-factory.js +517 -517
- package/vendor/tdai/utils/pipeline-manager.js +892 -892
- package/vendor/tdai/utils/sanitize.js +360 -360
- package/vendor/tdai/utils/serial-queue.js +103 -103
- package/vendor/tdai/utils/session-filter.js +88 -88
- package/vendor/tdai/utils/text-utils.js +29 -29
- package/vendor/tdai/utils/time.js +252 -252
|
@@ -1,86 +1,86 @@
|
|
|
1
|
-
import { randomUUID } from "node:crypto";
|
|
2
|
-
import fs from "node:fs/promises";
|
|
3
|
-
import path from "node:path";
|
|
4
|
-
export const REPORT_CONST = {
|
|
5
|
-
PLUGIN: "plugin",
|
|
6
|
-
};
|
|
7
|
-
// ── Singleton ──
|
|
8
|
-
let _reporter;
|
|
9
|
-
export function initReporter(opts) {
|
|
10
|
-
if (_reporter)
|
|
11
|
-
return;
|
|
12
|
-
if (!opts.enabled)
|
|
13
|
-
return;
|
|
14
|
-
switch (opts.type) {
|
|
15
|
-
case "local":
|
|
16
|
-
_reporter = new LocalReporter(opts.logger, opts.instanceId, opts.pluginVersion);
|
|
17
|
-
break;
|
|
18
|
-
// TODO: add new reporter type
|
|
19
|
-
default:
|
|
20
|
-
opts.logger.debug?.(`[memory-tdai] Unknown reporter type "${opts.type}", disabled reporting`);
|
|
21
|
-
break;
|
|
22
|
-
}
|
|
23
|
-
}
|
|
24
|
-
export function setReporter(reporter) {
|
|
25
|
-
_reporter = reporter;
|
|
26
|
-
}
|
|
27
|
-
/**
|
|
28
|
-
* Reset the reporter singleton so that the next `initReporter` call takes effect.
|
|
29
|
-
* Must be called at plugin re-registration (hot-reload) to pick up config changes.
|
|
30
|
-
*/
|
|
31
|
-
export function resetReporter() {
|
|
32
|
-
_reporter = undefined;
|
|
33
|
-
}
|
|
34
|
-
export function report(event, data) {
|
|
35
|
-
if (!_reporter)
|
|
36
|
-
return;
|
|
37
|
-
try {
|
|
38
|
-
_reporter.reportFunc(REPORT_CONST.PLUGIN, { event, ...data });
|
|
39
|
-
}
|
|
40
|
-
catch { /* never block business logic */ }
|
|
41
|
-
}
|
|
42
|
-
// ── LocalReporter (default) ──
|
|
43
|
-
class LocalReporter {
|
|
44
|
-
logger;
|
|
45
|
-
instanceId;
|
|
46
|
-
pluginVersion;
|
|
47
|
-
constructor(logger, instanceId, pluginVersion) {
|
|
48
|
-
this.logger = logger;
|
|
49
|
-
this.instanceId = instanceId;
|
|
50
|
-
this.pluginVersion = pluginVersion;
|
|
51
|
-
}
|
|
52
|
-
reportFunc(category, payload) {
|
|
53
|
-
try {
|
|
54
|
-
this.logger.info(JSON.stringify({
|
|
55
|
-
tag: "METRIC",
|
|
56
|
-
category,
|
|
57
|
-
plugin: "memory-tdai",
|
|
58
|
-
instanceId: this.instanceId,
|
|
59
|
-
pluginVersion: this.pluginVersion,
|
|
60
|
-
ts: new Date().toISOString(),
|
|
61
|
-
...payload,
|
|
62
|
-
}));
|
|
63
|
-
}
|
|
64
|
-
catch { /* swallow */ }
|
|
65
|
-
}
|
|
66
|
-
}
|
|
67
|
-
// ── Instance ID (persisted per-install) ──
|
|
68
|
-
let _instanceIdCache;
|
|
69
|
-
export async function getOrCreateInstanceId(pluginDataDir) {
|
|
70
|
-
if (_instanceIdCache)
|
|
71
|
-
return _instanceIdCache;
|
|
72
|
-
const idFile = path.join(pluginDataDir, ".metadata", "instance_id");
|
|
73
|
-
try {
|
|
74
|
-
const existing = (await fs.readFile(idFile, "utf-8")).trim();
|
|
75
|
-
if (existing) {
|
|
76
|
-
_instanceIdCache = existing;
|
|
77
|
-
return existing;
|
|
78
|
-
}
|
|
79
|
-
}
|
|
80
|
-
catch { /* file doesn't exist */ }
|
|
81
|
-
const newId = randomUUID();
|
|
82
|
-
await fs.mkdir(path.dirname(idFile), { recursive: true });
|
|
83
|
-
await fs.writeFile(idFile, newId, "utf-8");
|
|
84
|
-
_instanceIdCache = newId;
|
|
85
|
-
return newId;
|
|
86
|
-
}
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import fs from "node:fs/promises";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
export const REPORT_CONST = {
|
|
5
|
+
PLUGIN: "plugin",
|
|
6
|
+
};
|
|
7
|
+
// ── Singleton ──
|
|
8
|
+
let _reporter;
|
|
9
|
+
export function initReporter(opts) {
|
|
10
|
+
if (_reporter)
|
|
11
|
+
return;
|
|
12
|
+
if (!opts.enabled)
|
|
13
|
+
return;
|
|
14
|
+
switch (opts.type) {
|
|
15
|
+
case "local":
|
|
16
|
+
_reporter = new LocalReporter(opts.logger, opts.instanceId, opts.pluginVersion);
|
|
17
|
+
break;
|
|
18
|
+
// TODO: add new reporter type
|
|
19
|
+
default:
|
|
20
|
+
opts.logger.debug?.(`[memory-tdai] Unknown reporter type "${opts.type}", disabled reporting`);
|
|
21
|
+
break;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
export function setReporter(reporter) {
|
|
25
|
+
_reporter = reporter;
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Reset the reporter singleton so that the next `initReporter` call takes effect.
|
|
29
|
+
* Must be called at plugin re-registration (hot-reload) to pick up config changes.
|
|
30
|
+
*/
|
|
31
|
+
export function resetReporter() {
|
|
32
|
+
_reporter = undefined;
|
|
33
|
+
}
|
|
34
|
+
export function report(event, data) {
|
|
35
|
+
if (!_reporter)
|
|
36
|
+
return;
|
|
37
|
+
try {
|
|
38
|
+
_reporter.reportFunc(REPORT_CONST.PLUGIN, { event, ...data });
|
|
39
|
+
}
|
|
40
|
+
catch { /* never block business logic */ }
|
|
41
|
+
}
|
|
42
|
+
// ── LocalReporter (default) ──
|
|
43
|
+
class LocalReporter {
|
|
44
|
+
logger;
|
|
45
|
+
instanceId;
|
|
46
|
+
pluginVersion;
|
|
47
|
+
constructor(logger, instanceId, pluginVersion) {
|
|
48
|
+
this.logger = logger;
|
|
49
|
+
this.instanceId = instanceId;
|
|
50
|
+
this.pluginVersion = pluginVersion;
|
|
51
|
+
}
|
|
52
|
+
reportFunc(category, payload) {
|
|
53
|
+
try {
|
|
54
|
+
this.logger.info(JSON.stringify({
|
|
55
|
+
tag: "METRIC",
|
|
56
|
+
category,
|
|
57
|
+
plugin: "memory-tdai",
|
|
58
|
+
instanceId: this.instanceId,
|
|
59
|
+
pluginVersion: this.pluginVersion,
|
|
60
|
+
ts: new Date().toISOString(),
|
|
61
|
+
...payload,
|
|
62
|
+
}));
|
|
63
|
+
}
|
|
64
|
+
catch { /* swallow */ }
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
// ── Instance ID (persisted per-install) ──
|
|
68
|
+
let _instanceIdCache;
|
|
69
|
+
export async function getOrCreateInstanceId(pluginDataDir) {
|
|
70
|
+
if (_instanceIdCache)
|
|
71
|
+
return _instanceIdCache;
|
|
72
|
+
const idFile = path.join(pluginDataDir, ".metadata", "instance_id");
|
|
73
|
+
try {
|
|
74
|
+
const existing = (await fs.readFile(idFile, "utf-8")).trim();
|
|
75
|
+
if (existing) {
|
|
76
|
+
_instanceIdCache = existing;
|
|
77
|
+
return existing;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
catch { /* file doesn't exist */ }
|
|
81
|
+
const newId = randomUUID();
|
|
82
|
+
await fs.mkdir(path.dirname(idFile), { recursive: true });
|
|
83
|
+
await fs.writeFile(idFile, newId, "utf-8");
|
|
84
|
+
_instanceIdCache = newId;
|
|
85
|
+
return newId;
|
|
86
|
+
}
|
|
@@ -1,161 +1,161 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Scene filename normalizer.
|
|
3
|
-
*
|
|
4
|
-
* Defensive engineering layer that runs *after* the LLM writes scene_blocks/*.md
|
|
5
|
-
* and *before* syncSceneIndex(). Even though the prompt forbids spaces and
|
|
6
|
-
* punctuation in filenames, LLMs occasionally produce names like
|
|
7
|
-
* `Daily Rhythm in Shanghai.md`. Such names break:
|
|
8
|
-
* - Markdown navigation refs that downstream tools parse with `\S+\.md`
|
|
9
|
-
* (e.g. health-checker's scene reference detection).
|
|
10
|
-
* - Shell-based tools that iterate scene files without quoting.
|
|
11
|
-
* - URL/path encoding consumers (COS object keys etc).
|
|
12
|
-
*
|
|
13
|
-
* This module renames offenders to a canonical form on disk and lets every
|
|
14
|
-
* other consumer (PersonaGenerator, recall, profile-sync) read the already
|
|
15
|
-
* sanitized name from scene_index.json — no additional changes needed.
|
|
16
|
-
*/
|
|
17
|
-
import fs from "node:fs/promises";
|
|
18
|
-
import path from "node:path";
|
|
19
|
-
/**
|
|
20
|
-
* Normalize a single scene filename.
|
|
21
|
-
*
|
|
22
|
-
* Rules:
|
|
23
|
-
* - Preserves the `.md` extension (case-insensitive match, lowercased).
|
|
24
|
-
* - Whitespace runs (spaces / tabs) → single hyphen.
|
|
25
|
-
* - Strips quotes, brackets, and ASCII punctuation that breaks shell/markdown.
|
|
26
|
-
* - Collapses consecutive separators (`-`, `_`, `.`).
|
|
27
|
-
* - Trims leading / trailing separators.
|
|
28
|
-
* - Falls back to `"scene"` if the stem becomes empty.
|
|
29
|
-
*
|
|
30
|
-
* Allowed character set after normalization (informally):
|
|
31
|
-
* Unicode letters/numbers, hyphen, underscore, dot.
|
|
32
|
-
*
|
|
33
|
-
* Examples:
|
|
34
|
-
* "Daily Rhythm in Shanghai.md" → "Daily-Rhythm-in-Shanghai.md"
|
|
35
|
-
* "日常生活 健康管理.md" → "日常生活-健康管理.md"
|
|
36
|
-
* "Coffee (Yirgacheffe).md" → "Coffee-Yirgacheffe.md"
|
|
37
|
-
* " spaced .md" → "spaced.md"
|
|
38
|
-
* ".MD" → "scene.md"
|
|
39
|
-
* "已经规范.md" → "已经规范.md" (no-op)
|
|
40
|
-
*/
|
|
41
|
-
export function normalizeSceneFilename(name) {
|
|
42
|
-
if (!name)
|
|
43
|
-
return "scene.md";
|
|
44
|
-
// Strip directory components defensively — we only normalize the basename.
|
|
45
|
-
const base = name.replace(/^.*[\\/]/, "");
|
|
46
|
-
// Detect & strip `.md` (case-insensitive). Always re-emit lowercase `.md`.
|
|
47
|
-
const lower = base.toLowerCase();
|
|
48
|
-
const hasMd = lower.endsWith(".md");
|
|
49
|
-
const stem = hasMd ? base.slice(0, -3) : base;
|
|
50
|
-
const safe = stem
|
|
51
|
-
// Replace whitespace runs (incl. NBSP, full-width space) with `-`
|
|
52
|
-
.replace(/[\s\u00A0\u3000]+/g, "-")
|
|
53
|
-
// Drop quotes, brackets, and punctuation known to break shells/markdown.
|
|
54
|
-
// Keep Unicode letters/numbers and the safe separators `-`, `_`, `.`.
|
|
55
|
-
.replace(/[()[\]{}<>'"`,;:!?*|/\\=&%$#@^~+]/g, "")
|
|
56
|
-
// Collapse consecutive separators.
|
|
57
|
-
.replace(/-{2,}/g, "-")
|
|
58
|
-
.replace(/_{2,}/g, "_")
|
|
59
|
-
.replace(/\.{2,}/g, ".")
|
|
60
|
-
// Trim leading / trailing separators.
|
|
61
|
-
.replace(/^[-_.]+|[-_.]+$/g, "");
|
|
62
|
-
return (safe || "scene") + ".md";
|
|
63
|
-
}
|
|
64
|
-
/**
|
|
65
|
-
* Return whether a filename already matches its normalized form.
|
|
66
|
-
* Faster than computing the normalized form when callers only need a yes/no.
|
|
67
|
-
*/
|
|
68
|
-
export function isNormalizedSceneFilename(name) {
|
|
69
|
-
return normalizeSceneFilename(name) === name;
|
|
70
|
-
}
|
|
71
|
-
/**
|
|
72
|
-
* Resolve a non-conflicting target path inside `dir` for the desired filename.
|
|
73
|
-
*
|
|
74
|
-
* If `desired` (e.g. `Daily-Rhythm.md`) already exists in `dir`, append a
|
|
75
|
-
* numeric suffix `-2`, `-3`, ... before the `.md` extension until a free slot
|
|
76
|
-
* is found. Caller may also pass `excludePath` to ignore a known existing file
|
|
77
|
-
* (e.g. the source path of an in-flight rename, when source != target).
|
|
78
|
-
*/
|
|
79
|
-
export async function resolveUniqueScenePath(dir, desired, excludePath) {
|
|
80
|
-
const target = path.join(dir, desired);
|
|
81
|
-
if (!(await pathExists(target)) || target === excludePath)
|
|
82
|
-
return target;
|
|
83
|
-
const ext = ".md";
|
|
84
|
-
const stem = desired.endsWith(ext) ? desired.slice(0, -ext.length) : desired;
|
|
85
|
-
// Bound the search to keep this defensive (LLMs rarely produce hundreds of
|
|
86
|
-
// colliding names; if they do, surface the failure rather than spin).
|
|
87
|
-
for (let i = 2; i < 1000; i++) {
|
|
88
|
-
const candidate = path.join(dir, `${stem}-${i}${ext}`);
|
|
89
|
-
if (!(await pathExists(candidate)) || candidate === excludePath) {
|
|
90
|
-
return candidate;
|
|
91
|
-
}
|
|
92
|
-
}
|
|
93
|
-
throw new Error(`resolveUniqueScenePath: could not find a free slot for ${desired} in ${dir} after 1000 attempts`);
|
|
94
|
-
}
|
|
95
|
-
async function pathExists(p) {
|
|
96
|
-
try {
|
|
97
|
-
await fs.access(p);
|
|
98
|
-
return true;
|
|
99
|
-
}
|
|
100
|
-
catch {
|
|
101
|
-
return false;
|
|
102
|
-
}
|
|
103
|
-
}
|
|
104
|
-
/**
|
|
105
|
-
* Walk a scene_blocks directory and rename any `.md` file whose basename does
|
|
106
|
-
* not match `normalizeSceneFilename(basename)`.
|
|
107
|
-
*
|
|
108
|
-
* Safe to call multiple times: subsequent invocations are no-ops once names
|
|
109
|
-
* have stabilized.
|
|
110
|
-
*
|
|
111
|
-
* Notes:
|
|
112
|
-
* - Non-`.md` files are ignored (the LLM tool surface is restricted to .md,
|
|
113
|
-
* but the directory may contain transient artifacts).
|
|
114
|
-
* - Empty / soft-deleted files are not pre-filtered here; the SceneExtractor
|
|
115
|
-
* cleanup pass handles those before / after this call as appropriate.
|
|
116
|
-
* - Failures on individual entries are logged via the optional logger and
|
|
117
|
-
* do not abort the loop — index sync should still see the remaining files.
|
|
118
|
-
*/
|
|
119
|
-
export async function normalizeSceneFilenames(blocksDir, logger) {
|
|
120
|
-
const result = { renamed: 0, skipped: 0, renames: [] };
|
|
121
|
-
let entries;
|
|
122
|
-
try {
|
|
123
|
-
entries = (await fs.readdir(blocksDir)).filter((f) => f.endsWith(".md"));
|
|
124
|
-
}
|
|
125
|
-
catch {
|
|
126
|
-
return result;
|
|
127
|
-
}
|
|
128
|
-
for (const file of entries) {
|
|
129
|
-
const normalized = normalizeSceneFilename(file);
|
|
130
|
-
if (normalized === file) {
|
|
131
|
-
result.skipped++;
|
|
132
|
-
continue;
|
|
133
|
-
}
|
|
134
|
-
const from = path.join(blocksDir, file);
|
|
135
|
-
let to;
|
|
136
|
-
try {
|
|
137
|
-
to = await resolveUniqueScenePath(blocksDir, normalized, from);
|
|
138
|
-
}
|
|
139
|
-
catch (err) {
|
|
140
|
-
logger?.warn?.(`[filename-normalizer] could not resolve unique target for ${file}: ${err instanceof Error ? err.message : String(err)}`);
|
|
141
|
-
result.skipped++;
|
|
142
|
-
continue;
|
|
143
|
-
}
|
|
144
|
-
if (to === from) {
|
|
145
|
-
// Filesystem already matched (e.g. case-insensitive FS where source and
|
|
146
|
-
// target collapse to the same inode); treat as a no-op.
|
|
147
|
-
result.skipped++;
|
|
148
|
-
continue;
|
|
149
|
-
}
|
|
150
|
-
try {
|
|
151
|
-
await fs.rename(from, to);
|
|
152
|
-
result.renamed++;
|
|
153
|
-
result.renames.push({ from: file, to: path.basename(to) });
|
|
154
|
-
logger?.debug?.(`[filename-normalizer] renamed: ${file} → ${path.basename(to)}`);
|
|
155
|
-
}
|
|
156
|
-
catch (err) {
|
|
157
|
-
logger?.warn?.(`[filename-normalizer] rename failed (${file} → ${path.basename(to)}): ${err instanceof Error ? err.message : String(err)}`);
|
|
158
|
-
}
|
|
159
|
-
}
|
|
160
|
-
return result;
|
|
161
|
-
}
|
|
1
|
+
/**
|
|
2
|
+
* Scene filename normalizer.
|
|
3
|
+
*
|
|
4
|
+
* Defensive engineering layer that runs *after* the LLM writes scene_blocks/*.md
|
|
5
|
+
* and *before* syncSceneIndex(). Even though the prompt forbids spaces and
|
|
6
|
+
* punctuation in filenames, LLMs occasionally produce names like
|
|
7
|
+
* `Daily Rhythm in Shanghai.md`. Such names break:
|
|
8
|
+
* - Markdown navigation refs that downstream tools parse with `\S+\.md`
|
|
9
|
+
* (e.g. health-checker's scene reference detection).
|
|
10
|
+
* - Shell-based tools that iterate scene files without quoting.
|
|
11
|
+
* - URL/path encoding consumers (COS object keys etc).
|
|
12
|
+
*
|
|
13
|
+
* This module renames offenders to a canonical form on disk and lets every
|
|
14
|
+
* other consumer (PersonaGenerator, recall, profile-sync) read the already
|
|
15
|
+
* sanitized name from scene_index.json — no additional changes needed.
|
|
16
|
+
*/
|
|
17
|
+
import fs from "node:fs/promises";
|
|
18
|
+
import path from "node:path";
|
|
19
|
+
/**
|
|
20
|
+
* Normalize a single scene filename.
|
|
21
|
+
*
|
|
22
|
+
* Rules:
|
|
23
|
+
* - Preserves the `.md` extension (case-insensitive match, lowercased).
|
|
24
|
+
* - Whitespace runs (spaces / tabs) → single hyphen.
|
|
25
|
+
* - Strips quotes, brackets, and ASCII punctuation that breaks shell/markdown.
|
|
26
|
+
* - Collapses consecutive separators (`-`, `_`, `.`).
|
|
27
|
+
* - Trims leading / trailing separators.
|
|
28
|
+
* - Falls back to `"scene"` if the stem becomes empty.
|
|
29
|
+
*
|
|
30
|
+
* Allowed character set after normalization (informally):
|
|
31
|
+
* Unicode letters/numbers, hyphen, underscore, dot.
|
|
32
|
+
*
|
|
33
|
+
* Examples:
|
|
34
|
+
* "Daily Rhythm in Shanghai.md" → "Daily-Rhythm-in-Shanghai.md"
|
|
35
|
+
* "日常生活 健康管理.md" → "日常生活-健康管理.md"
|
|
36
|
+
* "Coffee (Yirgacheffe).md" → "Coffee-Yirgacheffe.md"
|
|
37
|
+
* " spaced .md" → "spaced.md"
|
|
38
|
+
* ".MD" → "scene.md"
|
|
39
|
+
* "已经规范.md" → "已经规范.md" (no-op)
|
|
40
|
+
*/
|
|
41
|
+
export function normalizeSceneFilename(name) {
|
|
42
|
+
if (!name)
|
|
43
|
+
return "scene.md";
|
|
44
|
+
// Strip directory components defensively — we only normalize the basename.
|
|
45
|
+
const base = name.replace(/^.*[\\/]/, "");
|
|
46
|
+
// Detect & strip `.md` (case-insensitive). Always re-emit lowercase `.md`.
|
|
47
|
+
const lower = base.toLowerCase();
|
|
48
|
+
const hasMd = lower.endsWith(".md");
|
|
49
|
+
const stem = hasMd ? base.slice(0, -3) : base;
|
|
50
|
+
const safe = stem
|
|
51
|
+
// Replace whitespace runs (incl. NBSP, full-width space) with `-`
|
|
52
|
+
.replace(/[\s\u00A0\u3000]+/g, "-")
|
|
53
|
+
// Drop quotes, brackets, and punctuation known to break shells/markdown.
|
|
54
|
+
// Keep Unicode letters/numbers and the safe separators `-`, `_`, `.`.
|
|
55
|
+
.replace(/[()[\]{}<>'"`,;:!?*|/\\=&%$#@^~+]/g, "")
|
|
56
|
+
// Collapse consecutive separators.
|
|
57
|
+
.replace(/-{2,}/g, "-")
|
|
58
|
+
.replace(/_{2,}/g, "_")
|
|
59
|
+
.replace(/\.{2,}/g, ".")
|
|
60
|
+
// Trim leading / trailing separators.
|
|
61
|
+
.replace(/^[-_.]+|[-_.]+$/g, "");
|
|
62
|
+
return (safe || "scene") + ".md";
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Return whether a filename already matches its normalized form.
|
|
66
|
+
* Faster than computing the normalized form when callers only need a yes/no.
|
|
67
|
+
*/
|
|
68
|
+
export function isNormalizedSceneFilename(name) {
|
|
69
|
+
return normalizeSceneFilename(name) === name;
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Resolve a non-conflicting target path inside `dir` for the desired filename.
|
|
73
|
+
*
|
|
74
|
+
* If `desired` (e.g. `Daily-Rhythm.md`) already exists in `dir`, append a
|
|
75
|
+
* numeric suffix `-2`, `-3`, ... before the `.md` extension until a free slot
|
|
76
|
+
* is found. Caller may also pass `excludePath` to ignore a known existing file
|
|
77
|
+
* (e.g. the source path of an in-flight rename, when source != target).
|
|
78
|
+
*/
|
|
79
|
+
export async function resolveUniqueScenePath(dir, desired, excludePath) {
|
|
80
|
+
const target = path.join(dir, desired);
|
|
81
|
+
if (!(await pathExists(target)) || target === excludePath)
|
|
82
|
+
return target;
|
|
83
|
+
const ext = ".md";
|
|
84
|
+
const stem = desired.endsWith(ext) ? desired.slice(0, -ext.length) : desired;
|
|
85
|
+
// Bound the search to keep this defensive (LLMs rarely produce hundreds of
|
|
86
|
+
// colliding names; if they do, surface the failure rather than spin).
|
|
87
|
+
for (let i = 2; i < 1000; i++) {
|
|
88
|
+
const candidate = path.join(dir, `${stem}-${i}${ext}`);
|
|
89
|
+
if (!(await pathExists(candidate)) || candidate === excludePath) {
|
|
90
|
+
return candidate;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
throw new Error(`resolveUniqueScenePath: could not find a free slot for ${desired} in ${dir} after 1000 attempts`);
|
|
94
|
+
}
|
|
95
|
+
async function pathExists(p) {
|
|
96
|
+
try {
|
|
97
|
+
await fs.access(p);
|
|
98
|
+
return true;
|
|
99
|
+
}
|
|
100
|
+
catch {
|
|
101
|
+
return false;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
/**
|
|
105
|
+
* Walk a scene_blocks directory and rename any `.md` file whose basename does
|
|
106
|
+
* not match `normalizeSceneFilename(basename)`.
|
|
107
|
+
*
|
|
108
|
+
* Safe to call multiple times: subsequent invocations are no-ops once names
|
|
109
|
+
* have stabilized.
|
|
110
|
+
*
|
|
111
|
+
* Notes:
|
|
112
|
+
* - Non-`.md` files are ignored (the LLM tool surface is restricted to .md,
|
|
113
|
+
* but the directory may contain transient artifacts).
|
|
114
|
+
* - Empty / soft-deleted files are not pre-filtered here; the SceneExtractor
|
|
115
|
+
* cleanup pass handles those before / after this call as appropriate.
|
|
116
|
+
* - Failures on individual entries are logged via the optional logger and
|
|
117
|
+
* do not abort the loop — index sync should still see the remaining files.
|
|
118
|
+
*/
|
|
119
|
+
export async function normalizeSceneFilenames(blocksDir, logger) {
|
|
120
|
+
const result = { renamed: 0, skipped: 0, renames: [] };
|
|
121
|
+
let entries;
|
|
122
|
+
try {
|
|
123
|
+
entries = (await fs.readdir(blocksDir)).filter((f) => f.endsWith(".md"));
|
|
124
|
+
}
|
|
125
|
+
catch {
|
|
126
|
+
return result;
|
|
127
|
+
}
|
|
128
|
+
for (const file of entries) {
|
|
129
|
+
const normalized = normalizeSceneFilename(file);
|
|
130
|
+
if (normalized === file) {
|
|
131
|
+
result.skipped++;
|
|
132
|
+
continue;
|
|
133
|
+
}
|
|
134
|
+
const from = path.join(blocksDir, file);
|
|
135
|
+
let to;
|
|
136
|
+
try {
|
|
137
|
+
to = await resolveUniqueScenePath(blocksDir, normalized, from);
|
|
138
|
+
}
|
|
139
|
+
catch (err) {
|
|
140
|
+
logger?.warn?.(`[filename-normalizer] could not resolve unique target for ${file}: ${err instanceof Error ? err.message : String(err)}`);
|
|
141
|
+
result.skipped++;
|
|
142
|
+
continue;
|
|
143
|
+
}
|
|
144
|
+
if (to === from) {
|
|
145
|
+
// Filesystem already matched (e.g. case-insensitive FS where source and
|
|
146
|
+
// target collapse to the same inode); treat as a no-op.
|
|
147
|
+
result.skipped++;
|
|
148
|
+
continue;
|
|
149
|
+
}
|
|
150
|
+
try {
|
|
151
|
+
await fs.rename(from, to);
|
|
152
|
+
result.renamed++;
|
|
153
|
+
result.renames.push({ from: file, to: path.basename(to) });
|
|
154
|
+
logger?.debug?.(`[filename-normalizer] renamed: ${file} → ${path.basename(to)}`);
|
|
155
|
+
}
|
|
156
|
+
catch (err) {
|
|
157
|
+
logger?.warn?.(`[filename-normalizer] rename failed (${file} → ${path.basename(to)}): ${err instanceof Error ? err.message : String(err)}`);
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
return result;
|
|
161
|
+
}
|