dsh-memorytree-rule 0.0.0-stage → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +42 -0
- package/LICENSE +21 -0
- package/README.md +144 -3
- package/bin/mt.mjs +589 -0
- package/client.js +227 -0
- package/cordis.patch.yml +16 -0
- package/docs/DETAIL.md +69 -0
- package/docs/DETAIL.svg +85 -0
- package/docs/EMOTION.md +101 -0
- package/examples/AGENTS.demo.md +22 -0
- package/examples/AGENTS.template.md +212 -0
- package/icon.svg +17 -0
- package/index.js +418 -0
- package/lib/board.mjs +1113 -0
- package/lib/cards.mjs +56 -0
- package/lib/daily.mjs +140 -0
- package/lib/db.mjs +84 -0
- package/lib/demo.mjs +161 -0
- package/lib/diary.mjs +92 -0
- package/lib/emotion.mjs +474 -0
- package/lib/import-legacy.mjs +167 -0
- package/lib/ingest.mjs +279 -0
- package/lib/levels.mjs +40 -0
- package/lib/paths.mjs +85 -0
- package/lib/report.mjs +2130 -0
- package/lib/rules.mjs +635 -0
- package/lib/runtime.mjs +232 -0
- package/lib/schema.mjs +363 -0
- package/lib/search.mjs +264 -0
- package/lib/store.mjs +1334 -0
- package/lib/text.mjs +161 -0
- package/lib/tools.mjs +962 -0
- package/lib/vector.mjs +51 -0
- package/locale/en.json +7 -0
- package/locale/zh.json +7 -0
- package/package.json +81 -6
- package/tests/emotion.test.mjs +230 -0
- package/tests/memory-tree.test.mjs +620 -0
- package/tests/regressions.test.mjs +834 -0
- package/tests/renumber.test.mjs +276 -0
- package/tests/trunk-bucket.test.mjs +225 -0
package/lib/text.mjs
ADDED
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
// lib/text.mjs — 归一化、分词、脱敏、摘要、校验和。
|
|
2
|
+
//
|
|
3
|
+
// 为什么不用 jieba/FTS5:
|
|
4
|
+
// - 依赖越少越能独立安装;FTS5 的 unicode61 不切中文(整段当一个词),trigram 又不支持 2 字查询。
|
|
5
|
+
// - 中文用「字 + 字二元组(bigram)」在小语料上召回稳定,2 个字也能查("备份" → b:备份)。
|
|
6
|
+
// - 英文/路径用 ASCII 词;权重:词 1.0、bigram 1.0、单字 0.3(单字只做弱匹配,避免噪声)。
|
|
7
|
+
import { createHash } from "node:crypto";
|
|
8
|
+
|
|
9
|
+
const CJK_RUN = /[\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff]+/g;
|
|
10
|
+
const SINGLE_CJK = /[\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff]/;
|
|
11
|
+
const ASCII_WORD = /[a-z0-9][a-z0-9_.\-]{1,}/g;
|
|
12
|
+
|
|
13
|
+
// 高频虚词:单字直接丢;bigram 两侧都是虚词才丢("备份"里的"份"不是虚词,不受影响)。
|
|
14
|
+
const STOP_ZH = new Set(
|
|
15
|
+
"的了是在和与就都也而及或我你他她它这那个们有不上下中为以到由把被让从对于之其并很更最会可要说用做能还没好又再只吗呢吧啊哦嗯".split(""),
|
|
16
|
+
);
|
|
17
|
+
const STOP_EN = new Set([
|
|
18
|
+
"the", "and", "for", "with", "that", "this", "from", "are", "was", "were", "has", "have", "had",
|
|
19
|
+
"not", "but", "you", "your", "they", "them", "will", "can", "all", "any", "our", "out", "its",
|
|
20
|
+
"of", "to", "in", "on", "at", "by", "as", "is", "be", "or", "if", "we", "he", "she", "do", "does",
|
|
21
|
+
"did", "so", "no", "up", "my", "me", "his", "her", "it", "an", "a", "i",
|
|
22
|
+
]);
|
|
23
|
+
|
|
24
|
+
export function normalize(s) {
|
|
25
|
+
return String(s ?? "")
|
|
26
|
+
.normalize("NFKC")
|
|
27
|
+
.replace(/\u200b|\u200c|\u200d|\ufeff/g, "")
|
|
28
|
+
.replace(/\r\n?/g, "\n")
|
|
29
|
+
.toLowerCase();
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function bump(map, key, w) {
|
|
33
|
+
map.set(key, (map.get(key) ?? 0) + w);
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* 分词:返回 Map<term, weight>。
|
|
38
|
+
* term 前缀 w:(ASCII 词)/ b:(中文 bigram)/ u:(中文单字)。
|
|
39
|
+
*/
|
|
40
|
+
export function tokenize(text) {
|
|
41
|
+
const out = new Map();
|
|
42
|
+
const s = normalize(text);
|
|
43
|
+
if (s === "") return out;
|
|
44
|
+
for (const m of s.matchAll(ASCII_WORD)) {
|
|
45
|
+
const raw = m[0].replace(/[._\-]+$/, "");
|
|
46
|
+
if (raw.length < 2 || STOP_EN.has(raw)) continue;
|
|
47
|
+
bump(out, "w:" + raw, 1);
|
|
48
|
+
// 形如 restart_dsh.ps1 的复合串再拆一次,方便按 restart 查到
|
|
49
|
+
const parts = raw.split(/[._\-]+/).filter((x) => x.length >= 2 && !STOP_EN.has(x));
|
|
50
|
+
if (parts.length > 1) for (const p of parts) bump(out, "w:" + p, 0.8);
|
|
51
|
+
}
|
|
52
|
+
for (const m of s.matchAll(CJK_RUN)) {
|
|
53
|
+
const run = m[0];
|
|
54
|
+
for (let i = 0; i < run.length; i++) {
|
|
55
|
+
if (!STOP_ZH.has(run[i])) bump(out, "u:" + run[i], 0.3);
|
|
56
|
+
}
|
|
57
|
+
for (let i = 0; i + 1 < run.length; i++) {
|
|
58
|
+
const bg = run.slice(i, i + 2);
|
|
59
|
+
if (STOP_ZH.has(bg[0]) && STOP_ZH.has(bg[1])) continue;
|
|
60
|
+
bump(out, "b:" + bg, 1);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
return out;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** 查询词里的「可读片段」:用于高亮/片段窗口,不含前缀。 */
|
|
67
|
+
export function queryFragments(query) {
|
|
68
|
+
const s = normalize(query).trim();
|
|
69
|
+
const frags = [];
|
|
70
|
+
for (const m of s.matchAll(CJK_RUN)) if (m[0].length >= 2) frags.push(m[0]);
|
|
71
|
+
for (const m of s.matchAll(ASCII_WORD)) if (m[0].length >= 2) frags.push(m[0]);
|
|
72
|
+
if (frags.length === 0 && s.length >= 1) frags.push(s);
|
|
73
|
+
return Array.from(new Set(frags));
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** 取包含查询片段的一小段;没命中就取开头。 */
|
|
77
|
+
export function snippet(text, query, width = 180) {
|
|
78
|
+
const s = String(text ?? "").replace(/\s+/g, " ").trim();
|
|
79
|
+
if (width <= 0 || s.length <= width) return s;
|
|
80
|
+
const frags = queryFragments(query);
|
|
81
|
+
let at = -1;
|
|
82
|
+
for (const f of frags) {
|
|
83
|
+
const i = s.toLowerCase().indexOf(f.toLowerCase());
|
|
84
|
+
if (i >= 0 && (at < 0 || i < at)) at = i;
|
|
85
|
+
}
|
|
86
|
+
if (at < 0) return s.slice(0, width) + "…";
|
|
87
|
+
const start = Math.max(0, at - Math.floor(width / 3));
|
|
88
|
+
const end = Math.min(s.length, start + width);
|
|
89
|
+
return (start > 0 ? "…" : "") + s.slice(start, end) + (end < s.length ? "…" : "");
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
const SECRET_PATTERNS = [
|
|
93
|
+
[/\bsk-[A-Za-z0-9_\-]{16,}\b/g, "sk-***"],
|
|
94
|
+
[/\bsk_(?:live|test)_[A-Za-z0-9]{16,}\b/g, "sk-***"],
|
|
95
|
+
[/\brk_(?:live|test)_[A-Za-z0-9]{16,}\b/g, "rk-***"],
|
|
96
|
+
[/\bgh[pousr]_[A-Za-z0-9]{20,}\b/g, "gh-***"],
|
|
97
|
+
[/\bgithub_pat_[A-Za-z0-9_]{20,}\b/g, "gh-***"],
|
|
98
|
+
[/\bglpat-[A-Za-z0-9_\-]{20,}\b/g, "glpat-***"],
|
|
99
|
+
[/\bxox[baprs]-[A-Za-z0-9\-]{10,}\b/g, "xox-***"],
|
|
100
|
+
[/\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/g, "AKIA***"],
|
|
101
|
+
[/\bAIza[0-9A-Za-z_\-]{35}\b/g, "AIza***"],
|
|
102
|
+
[/\bBearer\s+[A-Za-z0-9._\-]{16,}\b/gi, "Bearer ***"],
|
|
103
|
+
[/\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b/g, "jwt-***"],
|
|
104
|
+
[/-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g, "[私钥已移除]"],
|
|
105
|
+
[/((?:password|passwd|pwd|secret|token|api[_-]?key)\s*[:=]\s*)(["']?)[^\s"',;]{6,}\2/gi, "$1***"],
|
|
106
|
+
[/\b[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}\b/g, "***@***"],
|
|
107
|
+
// 只认中国大陆手机号与明确的 QQ/微信字段;不再把所有 11 位以上数字一律打码(会误伤时间戳、路径、ID)。
|
|
108
|
+
[/\b1[3-9]\d{9}\b/g, "***"],
|
|
109
|
+
[/\b\d{15,19}\b/g, "***"],
|
|
110
|
+
[/(?<![A-Za-z0-9_])(openid|open_id|qq|微信)\s*[:=]\s*[A-Za-z0-9]{5,}\b/gi, "$1=***"],
|
|
111
|
+
[/(?:https?:\/\/)?(?:hooks\.slack\.com\/services|discord(?:app)?\.com\/api\/webhooks)\/[A-Za-z0-9/_-]+/gi, "[webhook 已移除]"],
|
|
112
|
+
];
|
|
113
|
+
|
|
114
|
+
/** 写入前脱敏。返回 {text, redactions},redactions 是人能看懂的计数说明。 */
|
|
115
|
+
export function redact(text) {
|
|
116
|
+
let out = String(text ?? "");
|
|
117
|
+
const hits = [];
|
|
118
|
+
for (const [re, repl] of SECRET_PATTERNS) {
|
|
119
|
+
let n = 0;
|
|
120
|
+
out = out.replace(re, (...args) => {
|
|
121
|
+
n += 1;
|
|
122
|
+
return repl.replace(/\$(\d)/g, (m, i) => String(args[Number(i)] ?? ""));
|
|
123
|
+
});
|
|
124
|
+
if (n > 0) hits.push(re.source.slice(0, 24) + "×" + n);
|
|
125
|
+
}
|
|
126
|
+
return { text: out, redactions: hits };
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
export function checksum(s) {
|
|
130
|
+
return createHash("sha1").update(normalize(s)).digest("hex").slice(0, 16);
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/** 粗估 token(中文 1 字≈1,英文 4 字符≈1),只用于预算提示,不作为计费依据。 */
|
|
134
|
+
export function estimateTokens(s) {
|
|
135
|
+
const t = String(s ?? "");
|
|
136
|
+
let cjk = 0;
|
|
137
|
+
for (const ch of t) if (SINGLE_CJK.test(ch)) cjk += 1;
|
|
138
|
+
return Math.ceil(cjk + (t.length - cjk) / 4);
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
export function truncate(s, max) {
|
|
142
|
+
const t = String(s ?? "");
|
|
143
|
+
return t.length <= max ? t : t.slice(0, Math.max(0, max - 1)) + "…";
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/**
|
|
147
|
+
* 把工具 render 的返回值规范成宿主要求的「内容块数组」。
|
|
148
|
+
* 宿主(cordis-host-runner 的 sandboxDefineTool)只接受 [{ type: "text", text }];
|
|
149
|
+
* 直接返回字符串会在下游 content.some 处报错。
|
|
150
|
+
*/
|
|
151
|
+
export function contentBlocks(value) {
|
|
152
|
+
if (Array.isArray(value)) return value;
|
|
153
|
+
return [{ type: "text", text: String(value ?? "") }];
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/** 稳定的 JSON 字符串(键排序),用于去重比较与导出。 */
|
|
157
|
+
export function stableStringify(v) {
|
|
158
|
+
if (v === null || typeof v !== "object") return JSON.stringify(v);
|
|
159
|
+
if (Array.isArray(v)) return "[" + v.map(stableStringify).join(",") + "]";
|
|
160
|
+
return "{" + Object.keys(v).sort().map((k) => JSON.stringify(k) + ":" + stableStringify(v[k])).join(",") + "}";
|
|
161
|
+
}
|