dsh-macro-memory 0.0.0-stage → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +506 -2
- package/cordis.patch.yml +6 -0
- package/lib/client.js +3623 -0
- package/lib/index.js +502 -0
- package/lib/macro/brand.js +82 -0
- package/lib/macro/bridge.js +158 -0
- package/lib/macro/chain.js +206 -0
- package/lib/macro/contract.js +471 -0
- package/lib/macro/cost.js +164 -0
- package/lib/macro/defs/admin.js +442 -0
- package/lib/macro/defs/dream.js +24 -0
- package/lib/macro/defs/link.js +257 -0
- package/lib/macro/defs/project.js +25 -0
- package/lib/macro/defs/read.js +160 -0
- package/lib/macro/defs/remember.js +411 -0
- package/lib/macro/defs/search.js +871 -0
- package/lib/macro/defs/update.js +161 -0
- package/lib/macro/dreamledger.js +1398 -0
- package/lib/macro/facade.js +135 -0
- package/lib/macro/fusion.js +182 -0
- package/lib/macro/linkflow.js +391 -0
- package/lib/macro/links.js +448 -0
- package/lib/macro/neighbor.js +209 -0
- package/lib/macro/panel.js +96 -0
- package/lib/macro/prompt.js +139 -0
- package/lib/macro/provenance.js +148 -0
- package/lib/macro/render.js +35 -0
- package/lib/macro/revisions.js +253 -0
- package/lib/macro/shared.js +113 -0
- package/lib/macro/store.js +103 -0
- package/lib/macro/supersede.js +197 -0
- package/lib/macro/tokencost.js +90 -0
- package/lib/macro/tools.js +67 -0
- package/package.json +54 -4
- package/vendor/VENDOR.json +23 -0
- package/vendor/dsh-memory-nebula/CONTRIBUTING.md +48 -0
- package/vendor/dsh-memory-nebula/LICENSE +21 -0
- package/vendor/dsh-memory-nebula/README.en.md +203 -0
- package/vendor/dsh-memory-nebula/README.md +253 -0
- package/vendor/dsh-memory-nebula/SECURITY.md +43 -0
- package/vendor/dsh-memory-nebula/SUPPORT.md +33 -0
- package/vendor/dsh-memory-nebula/cordis.patch.yml +4 -0
- package/vendor/dsh-memory-nebula/lib/client.js +105 -0
- package/vendor/dsh-memory-nebula/lib/host/adapters/index.js +279 -0
- package/vendor/dsh-memory-nebula/lib/host/adapters/ingest.js +230 -0
- package/vendor/dsh-memory-nebula/lib/host/adapters/meow.js +320 -0
- package/vendor/dsh-memory-nebula/lib/host/adapters/notes.js +305 -0
- package/vendor/dsh-memory-nebula/lib/host/adapters/sqlite.js +287 -0
- package/vendor/dsh-memory-nebula/lib/host/adapters/util.js +231 -0
- package/vendor/dsh-memory-nebula/lib/host/bridge.js +351 -0
- package/vendor/dsh-memory-nebula/lib/host/compress.js +97 -0
- package/vendor/dsh-memory-nebula/lib/host/demo.js +357 -0
- package/vendor/dsh-memory-nebula/lib/host/diagnostics.js +125 -0
- package/vendor/dsh-memory-nebula/lib/host/doctor.js +270 -0
- package/vendor/dsh-memory-nebula/lib/host/entities.js +230 -0
- package/vendor/dsh-memory-nebula/lib/host/graph.js +1762 -0
- package/vendor/dsh-memory-nebula/lib/host/graphcache.js +290 -0
- package/vendor/dsh-memory-nebula/lib/host/http.js +993 -0
- package/vendor/dsh-memory-nebula/lib/host/injectcfg.js +95 -0
- package/vendor/dsh-memory-nebula/lib/host/ledger.js +175 -0
- package/vendor/dsh-memory-nebula/lib/host/live.js +204 -0
- package/vendor/dsh-memory-nebula/lib/host/nebula.js +2105 -0
- package/vendor/dsh-memory-nebula/lib/host/page.js +75 -0
- package/vendor/dsh-memory-nebula/lib/host/path.js +83 -0
- package/vendor/dsh-memory-nebula/lib/host/recall.js +524 -0
- package/vendor/dsh-memory-nebula/lib/host/share.js +633 -0
- package/vendor/dsh-memory-nebula/lib/host/sources.js +266 -0
- package/vendor/dsh-memory-nebula/lib/host/store.js +333 -0
- package/vendor/dsh-memory-nebula/lib/host/text.js +410 -0
- package/vendor/dsh-memory-nebula/lib/host/transcripts.js +396 -0
- package/vendor/dsh-memory-nebula/lib/index.js +782 -0
- package/vendor/dsh-memory-nebula/lib/standalone.mjs +315 -0
- package/vendor/dsh-memory-nebula/lib/web/api.js +426 -0
- package/vendor/dsh-memory-nebula/lib/web/app.js +4172 -0
- package/vendor/dsh-memory-nebula/lib/web/i18n.js +580 -0
- package/vendor/dsh-memory-nebula/lib/web/index.html +174 -0
- package/vendor/dsh-memory-nebula/lib/web/nav.js +132 -0
- package/vendor/dsh-memory-nebula/lib/web/render.js +1890 -0
- package/vendor/dsh-memory-nebula/lib/web/space3d.js +1883 -0
- package/vendor/dsh-memory-nebula/lib/web/style.css +1402 -0
- package/vendor/dsh-memory-nebula/locale/en.json +7 -0
- package/vendor/dsh-memory-nebula/locale/zh.json +7 -0
- package/vendor/dsh-memory-nebula/market/assets/icon-180.png +0 -0
- package/vendor/dsh-memory-nebula/market/assets/icon.svg +40 -0
- package/vendor/dsh-memory-nebula/market/index.json +101 -0
- package/vendor/dsh-memory-nebula/package.json +87 -0
- package/vendor/dsh-memory-nebula/sidecar.ps1 +119 -0
- package/vendor/meow-memory/LICENSE +21 -0
- package/vendor/meow-memory/README.en.md +192 -0
- package/vendor/meow-memory/README.md +174 -0
- package/vendor/meow-memory/README.pt-br.md +180 -0
- package/vendor/meow-memory/cordis.patch.yml +7 -0
- package/vendor/meow-memory/lib/client.js +3344 -0
- package/vendor/meow-memory/lib/client.js.map +7 -0
- package/vendor/meow-memory/lib/index.js +9802 -0
- package/vendor/meow-memory/lib/index.js.map +7 -0
- package/vendor/meow-memory/lib/prompts/README.md +65 -0
- package/vendor/meow-memory/lib/prompts/en/dream-atomic.md +36 -0
- package/vendor/meow-memory/lib/prompts/en/dream-header.md +9 -0
- package/vendor/meow-memory/lib/prompts/en/dream-project-summary.md +12 -0
- package/vendor/meow-memory/lib/prompts/en/dream-topic.md +27 -0
- package/vendor/meow-memory/lib/prompts/en/external-memory-notice.md +8 -0
- package/vendor/meow-memory/lib/prompts/en/first-create-notice.md +8 -0
- package/vendor/meow-memory/lib/prompts/en/labels.md +98 -0
- package/vendor/meow-memory/lib/prompts/en/reflect.md +31 -0
- package/vendor/meow-memory/lib/prompts/en/system-guide.md +121 -0
- package/vendor/meow-memory/lib/prompts/en/tools.md +83 -0
- package/vendor/meow-memory/lib/prompts/en/welcome-guide.md +12 -0
- package/vendor/meow-memory/lib/prompts/pt-br/dream-atomic.md +36 -0
- package/vendor/meow-memory/lib/prompts/pt-br/dream-header.md +9 -0
- package/vendor/meow-memory/lib/prompts/pt-br/dream-project-summary.md +12 -0
- package/vendor/meow-memory/lib/prompts/pt-br/dream-topic.md +27 -0
- package/vendor/meow-memory/lib/prompts/pt-br/external-memory-notice.md +8 -0
- package/vendor/meow-memory/lib/prompts/pt-br/first-create-notice.md +8 -0
- package/vendor/meow-memory/lib/prompts/pt-br/labels.md +98 -0
- package/vendor/meow-memory/lib/prompts/pt-br/reflect.md +31 -0
- package/vendor/meow-memory/lib/prompts/pt-br/system-guide.md +121 -0
- package/vendor/meow-memory/lib/prompts/pt-br/tools.md +83 -0
- package/vendor/meow-memory/lib/prompts/pt-br/welcome-guide.md +12 -0
- package/vendor/meow-memory/lib/prompts/zh/dream-atomic.md +36 -0
- package/vendor/meow-memory/lib/prompts/zh/dream-header.md +9 -0
- package/vendor/meow-memory/lib/prompts/zh/dream-project-summary.md +19 -0
- package/vendor/meow-memory/lib/prompts/zh/dream-topic.md +27 -0
- package/vendor/meow-memory/lib/prompts/zh/external-memory-notice.md +8 -0
- package/vendor/meow-memory/lib/prompts/zh/first-create-notice.md +8 -0
- package/vendor/meow-memory/lib/prompts/zh/labels.md +98 -0
- package/vendor/meow-memory/lib/prompts/zh/reflect.md +30 -0
- package/vendor/meow-memory/lib/prompts/zh/system-guide.md +119 -0
- package/vendor/meow-memory/lib/prompts/zh/tools.md +83 -0
- package/vendor/meow-memory/lib/prompts/zh/welcome-guide.md +12 -0
- package/vendor/meow-memory/package.json +64 -0
|
@@ -0,0 +1,410 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 文本工具箱:分词、实体抽取、哈希、停用词。
|
|
3
|
+
*
|
|
4
|
+
* 这里的所有函数都是纯函数、无 IO,方便单独跑测试。
|
|
5
|
+
* 设计目标:中文靠 bigram(不需要分词词典),英文/代码靠标识符,
|
|
6
|
+
* 然后在「稀有度」上做加权 —— 越少的词,越能证明两条记忆真有关联。
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
/** CJK 统一表意文字区间。 */
|
|
10
|
+
export const CJK_RE = /[\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff]/
|
|
11
|
+
|
|
12
|
+
/** 英文停用词:太常见的词没有区分度,进关联只会制造误连。 */
|
|
13
|
+
const EN_STOP = new Set([
|
|
14
|
+
'the', 'and', 'for', 'with', 'this', 'that', 'from', 'are', 'was', 'were', 'been',
|
|
15
|
+
'but', 'not', 'you', 'your', 'they', 'them', 'their', 'have', 'has', 'had', 'will',
|
|
16
|
+
'would', 'could', 'should', 'can', 'may', 'might', 'must', 'shall', 'than', 'then',
|
|
17
|
+
'there', 'here', 'when', 'where', 'what', 'which', 'who', 'why', 'how', 'all', 'any',
|
|
18
|
+
'each', 'both', 'more', 'most', 'some', 'such', 'only', 'own', 'same', 'other',
|
|
19
|
+
'into', 'over', 'under', 'about', 'after', 'before', 'between', 'during', 'through',
|
|
20
|
+
'true', 'false', 'null', 'undefined', 'none', 'void', 'this.', 'that.',
|
|
21
|
+
'function', 'return', 'const', 'let', 'var', 'import', 'export', 'default', 'class',
|
|
22
|
+
'async', 'await', 'type', 'interface', 'extends', 'implements', 'typeof', 'instanceof',
|
|
23
|
+
'string', 'number', 'boolean', 'object', 'array', 'error', 'value', 'values', 'data',
|
|
24
|
+
'name', 'names', 'item', 'items', 'list', 'index', 'length', 'node', 'npm', 'http',
|
|
25
|
+
'https', 'www', 'com', 'org', 'html', 'json', 'test', 'tests', 'spec', 'true', 'false',
|
|
26
|
+
'like', 'just', 'also', 'even', 'still', 'very', 'much', 'many', 'one', 'two', 'new',
|
|
27
|
+
'use', 'used', 'using', 'get', 'set', 'add', 'put', 'run', 'make', 'made', 'see',
|
|
28
|
+
'now', 'out', 'off', 'per', 'via', 'etc', 'eg', 'ie', 'yes', 'no', 'ok', 'okay',
|
|
29
|
+
])
|
|
30
|
+
|
|
31
|
+
/** 中文停用 bigram / 短语:同理,命中了也不算关联证据。 */
|
|
32
|
+
const ZH_STOP = new Set([
|
|
33
|
+
'这个', '那个', '一个', '一些', '我们', '你们', '他们', '她们', '它们', '什么', '怎么',
|
|
34
|
+
'可以', '因为', '所以', '但是', '然后', '现在', '已经', '如果', '还是', '或者', '以及',
|
|
35
|
+
'关于', '没有', '不是', '就是', '这样', '那样', '时候', '一直', '还有', '而且', '并且',
|
|
36
|
+
'之后', '之前', '里面', '上面', '下面', '比较', '非常', '特别', '应该', '可能', '觉得',
|
|
37
|
+
'知道', '看到', '一下', '一点', '自己', '大家', '东西', '事情', '问题', '地方', '需要',
|
|
38
|
+
'进行', '使用', '通过', '对于', '由于', '不过', '其实', '当然', '虽然', '然而', '于是',
|
|
39
|
+
'于是', '这是', '那是', '这些', '那些', '这里', '那里', '本次', '目前', '同时', '另外',
|
|
40
|
+
'用户', '系统', '功能', '内容', '方式', '情况', '部分', '方面', '结果', '过程', '时候',
|
|
41
|
+
'主人', '小代', '码酱', '助手', '模型', '项目', '工作', '文件', '数据',
|
|
42
|
+
])
|
|
43
|
+
|
|
44
|
+
/** 这些词太泛,就算出现在两条记忆里也不该单独成边。 */
|
|
45
|
+
export const WEAK_KEYWORDS = new Set([
|
|
46
|
+
'dsh', 'deepseek', 'harness', 'plugin', '插件', '技能', '配置', '测试', '代码', '实现',
|
|
47
|
+
'修复', '问题', '功能', '优化', '设计', '开发', '运行', '更新', '修改', '需求', '任务',
|
|
48
|
+
])
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* 把一段文本切成 token 集合:英文/数字标识符 + 中文 bigram + 中文短词。
|
|
52
|
+
* 全部小写;不做词形还原(收益小于风险)。
|
|
53
|
+
* @param {unknown} text
|
|
54
|
+
* @returns {Set<string>}
|
|
55
|
+
*/
|
|
56
|
+
export function tokenize(text) {
|
|
57
|
+
const out = new Set()
|
|
58
|
+
if (text === null || text === undefined) return out
|
|
59
|
+
const lower = String(text).toLowerCase()
|
|
60
|
+
const len = lower.length
|
|
61
|
+
if (len === 0) return out
|
|
62
|
+
|
|
63
|
+
// 单次扫描代替三条 matchAll 正则:中文内容动辄几百字,regex 的匹配开销
|
|
64
|
+
// 在 1 万条记忆上会被放大到秒级。这里按字符区间手动切,行为与旧实现一致:
|
|
65
|
+
// 英文标识符(含 _+#.-)、连续数字 ≥3 位、中文 bigram 与 2–8 字短词。
|
|
66
|
+
let i = 0
|
|
67
|
+
while (i < len) {
|
|
68
|
+
const c = lower.charCodeAt(i)
|
|
69
|
+
|
|
70
|
+
// a-z
|
|
71
|
+
if (c >= 97 && c <= 122) {
|
|
72
|
+
let j = i + 1
|
|
73
|
+
while (j < len) {
|
|
74
|
+
const d = lower.charCodeAt(j)
|
|
75
|
+
if ((d >= 97 && d <= 122) || (d >= 48 && d <= 57) || d === 95 || d === 43 || d === 35 || d === 46 || d === 45) j++
|
|
76
|
+
else break
|
|
77
|
+
}
|
|
78
|
+
let w = lower.slice(i, j)
|
|
79
|
+
w = w.replace(/[.\-+#]+$/, '')
|
|
80
|
+
if (w.length >= 2 && !EN_STOP.has(w)) out.add(w)
|
|
81
|
+
i = j
|
|
82
|
+
continue
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
// 0-9
|
|
86
|
+
if (c >= 48 && c <= 57) {
|
|
87
|
+
let j = i + 1
|
|
88
|
+
while (j < len) {
|
|
89
|
+
const d = lower.charCodeAt(j)
|
|
90
|
+
if (d >= 48 && d <= 57) j++
|
|
91
|
+
else break
|
|
92
|
+
}
|
|
93
|
+
if (j - i >= 3) out.add(lower.slice(i, j))
|
|
94
|
+
i = j
|
|
95
|
+
continue
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
// CJK:㐀-䶿 / 一-鿿 / 豈-
|
|
99
|
+
if ((c >= 0x3400 && c <= 0x4dbf) || (c >= 0x4e00 && c <= 0x9fff) || (c >= 0xf900 && c <= 0xfaff)) {
|
|
100
|
+
let j = i + 1
|
|
101
|
+
while (j < len) {
|
|
102
|
+
const d = lower.charCodeAt(j)
|
|
103
|
+
if ((d >= 0x3400 && d <= 0x4dbf) || (d >= 0x4e00 && d <= 0x9fff) || (d >= 0xf900 && d <= 0xfaff)) j++
|
|
104
|
+
else break
|
|
105
|
+
}
|
|
106
|
+
const run = lower.slice(i, j)
|
|
107
|
+
for (let k = 0; k + 1 < run.length; k++) {
|
|
108
|
+
const bg = run.slice(k, k + 2)
|
|
109
|
+
if (!ZH_STOP.has(bg)) out.add(bg)
|
|
110
|
+
}
|
|
111
|
+
if (run.length >= 2 && run.length <= 8 && !ZH_STOP.has(run)) out.add('«' + run + '»')
|
|
112
|
+
i = j
|
|
113
|
+
continue
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
i++
|
|
117
|
+
}
|
|
118
|
+
return out
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* 关键词集合归一化:去空、去停用弱词、小写、去重。
|
|
123
|
+
* @param {Iterable<unknown>} list
|
|
124
|
+
* @returns {Set<string>}
|
|
125
|
+
*/
|
|
126
|
+
export function normalizeKeywords(list) {
|
|
127
|
+
const out = new Set()
|
|
128
|
+
if (!list) return out
|
|
129
|
+
for (const raw of list) {
|
|
130
|
+
if (raw === null || raw === undefined) continue
|
|
131
|
+
const k = String(raw).trim().toLowerCase().replace(/\s+/g, ' ')
|
|
132
|
+
if (k.length < 2 || k.length > 40) continue
|
|
133
|
+
if (WEAK_KEYWORDS.has(k)) continue
|
|
134
|
+
if (EN_STOP.has(k)) continue
|
|
135
|
+
out.add(k)
|
|
136
|
+
}
|
|
137
|
+
return out
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* meow-memory 的 keywords 列是 JSON 数组文本,容错解析。
|
|
142
|
+
* @param {unknown} value
|
|
143
|
+
* @returns {string[]}
|
|
144
|
+
*/
|
|
145
|
+
export function parseKeywords(value) {
|
|
146
|
+
if (value === null || value === undefined || value === '') return []
|
|
147
|
+
if (Array.isArray(value)) return value.map((x) => String(x))
|
|
148
|
+
const s = String(value).trim()
|
|
149
|
+
if (s.startsWith('[') || s.startsWith('{')) {
|
|
150
|
+
try {
|
|
151
|
+
const parsed = JSON.parse(s)
|
|
152
|
+
if (Array.isArray(parsed)) return parsed.map((x) => String(x))
|
|
153
|
+
} catch {
|
|
154
|
+
/* 落回逗号分隔 */
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
return s.split(/[,,;;\n]/).map((x) => x.trim()).filter((x) => x.length > 0)
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/** 实体类型权重:越「硬」的标识越能证明关联。 */
|
|
161
|
+
const ENT_WEIGHT = {
|
|
162
|
+
path: 1.0,
|
|
163
|
+
url: 0.95,
|
|
164
|
+
uuid: 1.0,
|
|
165
|
+
hex: 0.95,
|
|
166
|
+
ref: 0.95,
|
|
167
|
+
file: 0.8,
|
|
168
|
+
code: 0.7,
|
|
169
|
+
quote: 0.65,
|
|
170
|
+
email: 0.9,
|
|
171
|
+
ip: 0.85,
|
|
172
|
+
version: 0.35,
|
|
173
|
+
number: 0.2,
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* 实体抽取规则;顺序有意义:先抽长的,避免被短规则截断。
|
|
178
|
+
* 第三条是「这条规则可能命中的最便宜判据」:一段正文里连冒号都没有,就
|
|
179
|
+
* 不必为路径规则扫 13 遍正则。判据必须**宁可放过**——判错会漏实体,
|
|
180
|
+
* 判宽只是没省到时间,所以每条都要能证明「命中必然含这个字符」。
|
|
181
|
+
*/
|
|
182
|
+
const ENT_RULES = [
|
|
183
|
+
['path', /[a-z]:[\\/][^\s"'`<>|,;,。;、))\]】]+/gi, (s) => s.includes(':')],
|
|
184
|
+
['path', /(?:^|\s)(?:\.{1,2}\/|\/)(?:[\w.@-]+\/)+[\w.@-]+/g, (s) => s.includes('/') || s.includes('\\')],
|
|
185
|
+
['url', /\bhttps?:\/\/[^\s"'`<>)\]]+/gi, (s) => s.includes('http')],
|
|
186
|
+
['uuid', /\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b/gi, (s) => s.includes('-')],
|
|
187
|
+
['hex', /\b[0-9a-f]{12,64}\b/gi, null],
|
|
188
|
+
['ref', /\bm\d{5}\b/g, null],
|
|
189
|
+
['file', /\b[\w@.-]{1,64}\.(?:js|mjs|cjs|ts|tsx|jsx|json|jsonc|md|mdx|py|rs|go|java|kt|cs|cpp|cc|h|hpp|db|sqlite|sql|yml|yaml|toml|ini|cfg|sh|ps1|bat|cmd|exe|dll|so|dylib|zip|tar|gz|png|jpe?g|webp|gif|svg|css|scss|html|htm|txt|csv|tsv|xlsx|xls|docx|pptx|pdf|apk|ipa|dylib|uproject|uasset|unity|smali|dex|elf|bin|log)\b/gi, (s) => s.includes('.')],
|
|
190
|
+
['email', /\b[\w.+-]+@[\w-]+\.[\w.-]{2,}\b/g, (s) => s.includes('@')],
|
|
191
|
+
['ip', /\b\d{1,3}(?:\.\d{1,3}){3}(?::\d{2,5})?\b/g, (s) => s.includes('.')],
|
|
192
|
+
['version', /\bv?\d+\.\d+\.\d+(?:-[0-9a-z.]+)?\b/gi, (s) => s.includes('.')],
|
|
193
|
+
]
|
|
194
|
+
|
|
195
|
+
/** 实体结尾要剪掉的标点:原来用两条正则,一条要在 20 万个命中上各跑一遍。 */
|
|
196
|
+
function isTailChar(c) {
|
|
197
|
+
return c === 46 || c === 44 || c === 59 || c === 58 || c === 41 || c === 93 || c === 125 || c === 34 || c === 39 || c === 96 || c === 92
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
function stripEntityTail(v) {
|
|
201
|
+
let end = v.length
|
|
202
|
+
while (end > 0 && isTailChar(v.charCodeAt(end - 1))) end--
|
|
203
|
+
return end === v.length ? v : v.slice(0, end)
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
/**
|
|
207
|
+
* 从文本里抽「硬实体」。这些是记忆之间最可信的连接理由。
|
|
208
|
+
* @param {unknown} text
|
|
209
|
+
* @returns {Map<string,string>} entity -> type
|
|
210
|
+
*/
|
|
211
|
+
export function extractEntities(text) {
|
|
212
|
+
const out = new Map()
|
|
213
|
+
if (text === null || text === undefined) return out
|
|
214
|
+
const s = String(text)
|
|
215
|
+
if (s.length === 0) return out
|
|
216
|
+
const lower = s.toLowerCase()
|
|
217
|
+
|
|
218
|
+
for (const [type, re, gate] of ENT_RULES) {
|
|
219
|
+
if (gate && !gate(lower)) continue
|
|
220
|
+
re.lastIndex = 0
|
|
221
|
+
for (const m of lower.matchAll(re)) {
|
|
222
|
+
let v = stripEntityTail(String(m[0]).trim())
|
|
223
|
+
if (v.length < 4) continue
|
|
224
|
+
if (type !== 'version' && v.length >= 5 && v.charCodeAt(v.length - 2) === 46 && /^\d+\.\d+\.\d+$/.test(v)) continue
|
|
225
|
+
const key = type + '|' + v
|
|
226
|
+
if (!out.has(key)) out.set(key, type)
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
// 反引号里的代码片段:`xxx` —— 人写记忆时用来标「专有名词」
|
|
230
|
+
if (s.includes('`')) {
|
|
231
|
+
for (const m of s.matchAll(/`([^`\n]{2,60})`/g)) {
|
|
232
|
+
const v = m[1].trim().toLowerCase()
|
|
233
|
+
if (v.length >= 2 && /[a-z\u4e00-\u9fff]/i.test(v)) out.set('code|' + v, 'code')
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
// 「」引号里的专名
|
|
237
|
+
if (s.includes('「') || s.includes('『')) {
|
|
238
|
+
for (const m of s.matchAll(/[「『]([^」』\n]{2,24})[」』]/g)) {
|
|
239
|
+
const v = m[1].trim().toLowerCase()
|
|
240
|
+
if (v.length >= 2) out.set('quote|' + v, 'quote')
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
return out
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
/**
|
|
247
|
+
* 拆出实体的类型(key 的前缀)与裸值。
|
|
248
|
+
* @param {string} key
|
|
249
|
+
*/
|
|
250
|
+
export function splitEntity(key) {
|
|
251
|
+
const i = key.indexOf('|')
|
|
252
|
+
return i < 0 ? ['', key] : [key.slice(0, i), key.slice(i + 1)]
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
/**
|
|
256
|
+
* 实体对关联的贡献权重:稀有 + 类型硬 = 强证据。
|
|
257
|
+
* @param {string} key
|
|
258
|
+
* @param {number} df 该实体在语料里出现过的记忆条数
|
|
259
|
+
* @param {number} n 语料总量
|
|
260
|
+
*/
|
|
261
|
+
export function entityStrength(key, df, n) {
|
|
262
|
+
const [type] = splitEntity(key)
|
|
263
|
+
const base = ENT_WEIGHT[type] ?? 0.4
|
|
264
|
+
const rarity = 1 / (1 + Math.log2(1 + Math.max(1, df)))
|
|
265
|
+
const coverage = Math.max(0.05, 1 - Math.min(1, df / Math.max(8, n * 0.35)))
|
|
266
|
+
return base * (0.55 + 0.45 * rarity) * (0.4 + 0.6 * coverage)
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
/** 32 位 FNV-1a:稳定哈希,用于配色与初始布局。 */
|
|
270
|
+
export function hash32(str) {
|
|
271
|
+
let h = 0x811c9dc5
|
|
272
|
+
const s = String(str)
|
|
273
|
+
for (let i = 0; i < s.length; i++) {
|
|
274
|
+
h ^= s.charCodeAt(i)
|
|
275
|
+
h = Math.imul(h, 0x01000193)
|
|
276
|
+
}
|
|
277
|
+
return h >>> 0
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
/**
|
|
281
|
+
* 由字符串派生一个稳定的色相(0-360)。
|
|
282
|
+
* @param {string} str
|
|
283
|
+
*/
|
|
284
|
+
export function hueOf(str) {
|
|
285
|
+
return hash32(str) % 360
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
/**
|
|
289
|
+
* 把哈希打成 [0,1) 的伪随机序列,用于稳定的布局种子。
|
|
290
|
+
* @param {string} seed
|
|
291
|
+
*/
|
|
292
|
+
export function rngFrom(seed) {
|
|
293
|
+
let a = hash32(seed) || 1
|
|
294
|
+
return function next() {
|
|
295
|
+
a ^= a << 13
|
|
296
|
+
a >>>= 0
|
|
297
|
+
a ^= a >> 17
|
|
298
|
+
a ^= a << 5
|
|
299
|
+
a >>>= 0
|
|
300
|
+
return a / 4294967296
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
/**
|
|
305
|
+
* 干净地截断,不劈开「一个字」。
|
|
306
|
+
*
|
|
307
|
+
* JS 的字符串里,一个 emoji 常常占两个 code unit;直接 slice 会把代理对切成两半,
|
|
308
|
+
* 那半个字符写进 UTF-8 就变成 U+FFFD(界面上一个方块问号),导出报告里也一样。
|
|
309
|
+
* 这里退一步:切点落代理对中间就少切一个;落在 ZWJ / 变体选择符 / 肤色修饰符后面
|
|
310
|
+
* 就继续往回退,别让「👨👩👧」「👋🏽」变成半截。
|
|
311
|
+
*
|
|
312
|
+
* @param {unknown} text
|
|
313
|
+
* @param {number} max 保留多少个 code unit(与 String#length 同口径)
|
|
314
|
+
*/
|
|
315
|
+
export function cut(text, max = 160) {
|
|
316
|
+
const s = String(text ?? '')
|
|
317
|
+
const limit = Math.max(0, Math.floor(Number(max) || 0))
|
|
318
|
+
let out = s
|
|
319
|
+
if (s.length > limit) {
|
|
320
|
+
let end = limit
|
|
321
|
+
if (end > 0) {
|
|
322
|
+
const last = s.charCodeAt(end - 1)
|
|
323
|
+
if (last >= 0xd800 && last <= 0xdbff) end -= 1
|
|
324
|
+
// ZWJ(连字)、变体选择符 16、肤色修饰符:都是贴在别的字符上的零件。
|
|
325
|
+
// 既不留在结尾变孤儿,也不把它挂着的本体留成半截。
|
|
326
|
+
const isPart = (cp) => cp === 0x200d || cp === 0xfe0f || (cp >= 0x1f3fb && cp <= 0x1f3ff)
|
|
327
|
+
const cpBefore = (i) => {
|
|
328
|
+
if (i <= 0) return -1
|
|
329
|
+
const low = s.charCodeAt(i - 1)
|
|
330
|
+
if (low >= 0xdc00 && low <= 0xdfff && i >= 2) return s.codePointAt(i - 2)
|
|
331
|
+
return low
|
|
332
|
+
}
|
|
333
|
+
const cpAt = (i) => (i < s.length ? s.codePointAt(i) : -1)
|
|
334
|
+
for (;;) {
|
|
335
|
+
if (end <= 0) break
|
|
336
|
+
if (!isPart(cpBefore(end)) && !isPart(cpAt(end))) break
|
|
337
|
+
end -= 1
|
|
338
|
+
const back = s.charCodeAt(end - 1)
|
|
339
|
+
if (back >= 0xd800 && back <= 0xdbff) end -= 1
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
out = s.slice(0, end)
|
|
343
|
+
}
|
|
344
|
+
// 数据本身就可能带半个字符(别人写进来的),顺手清掉,别传给下游
|
|
345
|
+
if (/[\ud800-\udfff]/.test(out)) {
|
|
346
|
+
return out.replace(/[\ud800-\udbff](?![\udc00-\udfff])|(?<![\ud800-\udbff])[\udc00-\udfff]/g, '')
|
|
347
|
+
}
|
|
348
|
+
return out
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
/**
|
|
352
|
+
* 截断长文本(按字符数),保留结尾。
|
|
353
|
+
* @param {unknown} text
|
|
354
|
+
* @param {number} max
|
|
355
|
+
*/
|
|
356
|
+
export function clip(text, max = 160) {
|
|
357
|
+
const width = Math.max(0, Math.floor(Number(max) || 0))
|
|
358
|
+
const s = String(text ?? '').replace(/\s+/g, ' ').trim()
|
|
359
|
+
if (s.length <= width) return s
|
|
360
|
+
return cut(s, Math.max(0, width - 1)) + '…'
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
/**
|
|
364
|
+
* IDF 与余弦相似度用的小工具:给定文档频次表算 idf。
|
|
365
|
+
* @param {number} df
|
|
366
|
+
* @param {number} n
|
|
367
|
+
*/
|
|
368
|
+
export function idf(df, n) {
|
|
369
|
+
return Math.log(1 + (n - df + 0.5) / (df + 0.5))
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
/**
|
|
373
|
+
* 两集合的 Jaccard。
|
|
374
|
+
* @param {Set<string>} a
|
|
375
|
+
* @param {Set<string>} b
|
|
376
|
+
*/
|
|
377
|
+
export function jaccard(a, b) {
|
|
378
|
+
if (a.size === 0 || b.size === 0) return 0
|
|
379
|
+
let inter = 0
|
|
380
|
+
const [small, big] = a.size <= b.size ? [a, b] : [b, a]
|
|
381
|
+
for (const x of small) if (big.has(x)) inter++
|
|
382
|
+
if (inter === 0) return 0
|
|
383
|
+
return inter / (a.size + b.size - inter)
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
/**
|
|
387
|
+
* 两个 Set 的交集大小。
|
|
388
|
+
* @param {Set<string>} a
|
|
389
|
+
* @param {Set<string>} b
|
|
390
|
+
*/
|
|
391
|
+
export function overlapSize(a, b) {
|
|
392
|
+
if (a.size === 0 || b.size === 0) return 0
|
|
393
|
+
let n = 0
|
|
394
|
+
const [small, big] = a.size <= b.size ? [a, b] : [b, a]
|
|
395
|
+
for (const x of small) if (big.has(x)) n++
|
|
396
|
+
return n
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
/**
|
|
400
|
+
* 取交集。
|
|
401
|
+
* @param {Set<string>} a
|
|
402
|
+
* @param {Set<string>} b
|
|
403
|
+
* @returns {string[]}
|
|
404
|
+
*/
|
|
405
|
+
export function intersect(a, b) {
|
|
406
|
+
const out = []
|
|
407
|
+
const [small, big] = a.size <= b.size ? [a, b] : [b, a]
|
|
408
|
+
for (const x of small) if (big.has(x)) out.push(x)
|
|
409
|
+
return out
|
|
410
|
+
}
|