lume-dsh-plugin 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +238 -0
- package/assets/personalities/butler-corpus.jsonl +30 -0
- package/assets/personalities/butler.txt +12 -0
- package/assets/personalities/loli-corpus.jsonl +30 -0
- package/assets/personalities/loli.txt +12 -0
- package/assets/personalities/none-corpus.jsonl +0 -0
- package/assets/personalities/none.txt +0 -0
- package/assets/personalities/senpai-corpus.jsonl +30 -0
- package/assets/personalities/senpai.txt +12 -0
- package/assets/personalities/tsundere-corpus.jsonl +30 -0
- package/assets/personalities/tsundere.txt +12 -0
- package/assets/personalities.json +47 -0
- package/cordis.patch.yml +5 -0
- package/lib/client.js +2395 -0
- package/lib/core/card.js +101 -0
- package/lib/core/dialogue-mining.js +409 -0
- package/lib/core/leak-detector.js +41 -0
- package/lib/core/manifest.js +60 -0
- package/lib/core/persona-text.js +30 -0
- package/lib/core/retrieval.js +100 -0
- package/lib/core/sampling.js +46 -0
- package/lib/core/text.js +17 -0
- package/lib/host/boundary.js +32 -0
- package/lib/host/distill.js +520 -0
- package/lib/host/extraction.js +150 -0
- package/lib/host/identity.js +217 -0
- package/lib/host/injection.js +100 -0
- package/lib/host/personalities.js +49 -0
- package/lib/host/reflection.js +150 -0
- package/lib/host/registry.js +62 -0
- package/lib/host/rpc.js +284 -0
- package/lib/host/session-runtime.js +48 -0
- package/lib/host/store.js +123 -0
- package/lib/index.js +737 -0
- package/package.json +99 -0
package/lib/core/card.js
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 人设卡导出/导入的纯函数层:序列化、解析与归一化。
|
|
3
|
+
*
|
|
4
|
+
* 卡片格式:自包含 JSON,含契约 + 语料 + 风格约定 + 记忆(可选)+ 签名。
|
|
5
|
+
* 导入侧校验键名合法性、内置名保护、字段截断上限,但不对 promptText
|
|
6
|
+
* 做语义校验(那是蒸馏管线的职责)。
|
|
7
|
+
*/
|
|
8
|
+
import { BUILTIN_PERSONA_NAMES, CORPUS_CAP, CORPUS_LINE_CAP, MEMORY_CAP, STYLE_CAP, sanitizeCorpus } from "../host/identity.js";
|
|
9
|
+
export const CARD_FORMAT = "lume-persona-card";
|
|
10
|
+
export const CARD_VERSION = 1;
|
|
11
|
+
/** 序列化一张卡片。 */
|
|
12
|
+
export function serializeCard(bundle) {
|
|
13
|
+
return JSON.stringify(bundle, null, 2) + "\n";
|
|
14
|
+
}
|
|
15
|
+
/** 解析一段 JSON 文本;结构/格式/版本不合法返回带 error 的 ParseErr。 */
|
|
16
|
+
export function parseCard(text) {
|
|
17
|
+
let raw;
|
|
18
|
+
try {
|
|
19
|
+
raw = JSON.parse(text);
|
|
20
|
+
}
|
|
21
|
+
catch {
|
|
22
|
+
return { ok: false, error: "JSON 解析失败:文件不是合法的 JSON。" };
|
|
23
|
+
}
|
|
24
|
+
if (typeof raw !== "object" || raw === null) {
|
|
25
|
+
return { ok: false, error: "卡片格式错误:根节点必须是对象。" };
|
|
26
|
+
}
|
|
27
|
+
const r = raw;
|
|
28
|
+
if (r.format !== CARD_FORMAT) {
|
|
29
|
+
return { ok: false, error: `不支持的文件格式:期望 "${CARD_FORMAT}",实际 "${String(r.format)}"。` };
|
|
30
|
+
}
|
|
31
|
+
if (typeof r.version !== "number" || r.version !== CARD_VERSION) {
|
|
32
|
+
return { ok: false, error: `不支持的卡片版本:期望 ${CARD_VERSION},实际 ${String(r.version)}。` };
|
|
33
|
+
}
|
|
34
|
+
const p = r.persona;
|
|
35
|
+
if (typeof p !== "object" || p === null) {
|
|
36
|
+
return { ok: false, error: "卡片缺少 persona 字段。" };
|
|
37
|
+
}
|
|
38
|
+
const persona = p;
|
|
39
|
+
if (typeof persona.name !== "string" || !persona.name) {
|
|
40
|
+
return { ok: false, error: "卡片 persona.name 为空或不是字符串。" };
|
|
41
|
+
}
|
|
42
|
+
if (typeof persona.displayName !== "string" || !persona.displayName) {
|
|
43
|
+
return { ok: false, error: "卡片 persona.displayName 为空或不是字符串。" };
|
|
44
|
+
}
|
|
45
|
+
if (typeof persona.promptText !== "string" || !persona.promptText) {
|
|
46
|
+
return { ok: false, error: "卡片 persona.promptText 为空或不是字符串。" };
|
|
47
|
+
}
|
|
48
|
+
return {
|
|
49
|
+
ok: true,
|
|
50
|
+
value: {
|
|
51
|
+
format: CARD_FORMAT,
|
|
52
|
+
version: CARD_VERSION,
|
|
53
|
+
persona: {
|
|
54
|
+
name: persona.name,
|
|
55
|
+
displayName: persona.displayName,
|
|
56
|
+
description: typeof persona.description === "string" ? persona.description : "",
|
|
57
|
+
promptText: persona.promptText,
|
|
58
|
+
corpus: sanitizeCorpus(persona.corpus),
|
|
59
|
+
profileName: typeof persona.profileName === "string" && persona.profileName ? persona.profileName : null,
|
|
60
|
+
styleRules: Array.isArray(persona.styleRules)
|
|
61
|
+
? persona.styleRules
|
|
62
|
+
.filter((r) => typeof r?.rule === "string" && r.rule)
|
|
63
|
+
.slice(-STYLE_CAP)
|
|
64
|
+
: [],
|
|
65
|
+
memory: Array.isArray(persona.memory)
|
|
66
|
+
? persona.memory
|
|
67
|
+
.filter((m) => typeof m?.text === "string" && m.text)
|
|
68
|
+
.slice(-MEMORY_CAP)
|
|
69
|
+
: undefined,
|
|
70
|
+
signatureWords: Array.isArray(persona.signatureWords) ? persona.signatureWords.filter((w) => typeof w === "string" && w.length > 0) : undefined,
|
|
71
|
+
},
|
|
72
|
+
},
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* 归一化并校验:键名合法性、内置名保护、字段截断。
|
|
77
|
+
* 拒绝覆盖内置人设;返回规范化后的卡片(供导入写入)。
|
|
78
|
+
*/
|
|
79
|
+
export function normalizeCard(card) {
|
|
80
|
+
const name = card.name.toLowerCase().replace(/[^a-z0-9-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 32);
|
|
81
|
+
if (!/^[a-z][a-z0-9-]*$/.test(name)) {
|
|
82
|
+
return { ok: false, error: `人设键名 "${name}" 不合法:必须以小写字母开头,只含小写字母/数字/连字符。` };
|
|
83
|
+
}
|
|
84
|
+
if (BUILTIN_PERSONA_NAMES.has(name)) {
|
|
85
|
+
return { ok: false, error: `"${name}" 是内置人设,不可覆盖。` };
|
|
86
|
+
}
|
|
87
|
+
return {
|
|
88
|
+
ok: true,
|
|
89
|
+
value: {
|
|
90
|
+
name,
|
|
91
|
+
displayName: card.displayName.trim().slice(0, 12),
|
|
92
|
+
description: (card.description ?? "").trim().slice(0, 60),
|
|
93
|
+
promptText: card.promptText.trim().slice(0, 2000),
|
|
94
|
+
corpus: sanitizeCorpus(card.corpus),
|
|
95
|
+
profileName: card.profileName?.trim() || null,
|
|
96
|
+
styleRules: (card.styleRules ?? []).slice(-STYLE_CAP),
|
|
97
|
+
memory: card.memory?.slice(-MEMORY_CAP),
|
|
98
|
+
signatureWords: card.signatureWords,
|
|
99
|
+
},
|
|
100
|
+
};
|
|
101
|
+
}
|
|
@@ -0,0 +1,409 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 对话挖掘(纯函数层):从小说/剧本/设定文档等素材里抽取目标角色的台词样本
|
|
3
|
+
* 与叙述线索。蒸馏管线的第 0 步,零 token 成本;产出交给 LLM 做契约与语料合成。
|
|
4
|
+
*
|
|
5
|
+
* 两种素材形态:
|
|
6
|
+
* - 剧本行(`角色名:台词`)——归属明确,直接按名字统计;
|
|
7
|
+
* - 小说引号台词(「…」「…」)——用引号前的「XX说/道/问」归属;归属线索不足时
|
|
8
|
+
* (设定文档/独白类素材)视为单一声音,全部台词归目标角色。
|
|
9
|
+
*/
|
|
10
|
+
export const MAX_MINED_LINES = 48;
|
|
11
|
+
export const MAX_OTHER_LINES = 12;
|
|
12
|
+
export const NARRATIVE_CAP = 1600;
|
|
13
|
+
export const MIN_SCRIPT_LINES = 3;
|
|
14
|
+
/** 引号台词:中文直角/弯引号 + 英文双引号。 */
|
|
15
|
+
const QUOTE_RE = /[「『“"]([^」』”"]{2,120})[」』”"]/g;
|
|
16
|
+
/** 剧本行:行首(可带 - • 序号)短名字 + 冒号 + 台词。 */
|
|
17
|
+
const SCRIPT_LINE_RE = /^\s*(?:[-*•]\s*)?(?:\d+[.、]\s*)?([^\s::,。!?、"'「」『』()()]{1,12})\s*[::]\s*(\S.{1,200})$/;
|
|
18
|
+
/** 说话引导动词:引号前窗口内的归属线索。捕获组是动作发出者;代词不算有效归属。
|
|
19
|
+
* 名字组非贪婪 + 复合动词(又问/再说等)入表,保证「噜噜又问」解析为 名字=噜噜 动词=又问。 */
|
|
20
|
+
const SAID_RE = /([\u4e00-\u9fffA-Za-z0-9·]{1,8}?)(?:小声道|轻声道|冷冷道|淡淡地?道|笑道|哭道|喊道|问道|答道|说道|叫道|骂道|嘀咕|嘟囔|反驳道?|回答道?|补充道?|开口道?|低声道?|追问|反问道?|又问|又说|又道|再说|再道|接着说|接着道|道|说|问|喊)/;
|
|
21
|
+
const PRONOUNS = new Set(["她", "他", "它", "你", "我"]);
|
|
22
|
+
/** 均匀取样:n 超限时按索引等距抽取,保持时序。 */
|
|
23
|
+
function evenSample(items, n) {
|
|
24
|
+
if (items.length <= n)
|
|
25
|
+
return items;
|
|
26
|
+
const out = [];
|
|
27
|
+
for (let i = 0; i < n; i++) {
|
|
28
|
+
out.push(items[Math.floor((i * items.length) / n)]);
|
|
29
|
+
}
|
|
30
|
+
return out;
|
|
31
|
+
}
|
|
32
|
+
/** 剧本行挖掘。 */
|
|
33
|
+
function mineScriptLines(text) {
|
|
34
|
+
const out = [];
|
|
35
|
+
for (const raw of text.split("\n")) {
|
|
36
|
+
const match = SCRIPT_LINE_RE.exec(raw);
|
|
37
|
+
if (match)
|
|
38
|
+
out.push({ speaker: match[1].trim(), line: match[2].trim() });
|
|
39
|
+
}
|
|
40
|
+
return out;
|
|
41
|
+
}
|
|
42
|
+
/** 归属窗口:引号开始处往前的字符数 / 引号结束处往后的字符数。 */
|
|
43
|
+
const ATTRIBUTION_WINDOW = 30;
|
|
44
|
+
const POST_ATTRIBUTION_WINDOW = 12;
|
|
45
|
+
/** 名字有效性:代词(含代词开头的误捕获,如「她淡淡」)不算说话人。 */
|
|
46
|
+
function cleanName(raw) {
|
|
47
|
+
return raw && !PRONOUNS.has(raw[0]) ? raw.trim() : null;
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* 引号台词挖掘 + 归属(汉语小说惯例):
|
|
51
|
+
* - 「X说:」紧贴引号前且带冒号 → 描述当前引号;
|
|
52
|
+
* - 「…」X笑道 紧贴引号后 → 描述当前引号;
|
|
53
|
+
* - 两引号之间的裸标签(无冒号)属于前一个引号,不算当前归属。
|
|
54
|
+
*/
|
|
55
|
+
function mineQuotedLines(text) {
|
|
56
|
+
const out = [];
|
|
57
|
+
let prevEnd = 0;
|
|
58
|
+
for (const match of text.matchAll(QUOTE_RE)) {
|
|
59
|
+
const line = match[1].trim();
|
|
60
|
+
const start = match.index ?? 0;
|
|
61
|
+
const end = start + match[0].length;
|
|
62
|
+
let speaker = null;
|
|
63
|
+
const preWindow = text.slice(Math.max(prevEnd, start - ATTRIBUTION_WINDOW), start);
|
|
64
|
+
const pre = SAID_RE.exec(preWindow);
|
|
65
|
+
if (pre) {
|
|
66
|
+
const afterVerb = preWindow.slice(pre.index + pre[0].length, pre.index + pre[0].length + 1);
|
|
67
|
+
if (afterVerb === ":" || afterVerb === ":")
|
|
68
|
+
speaker = cleanName(pre[1]);
|
|
69
|
+
}
|
|
70
|
+
// 前窗没有「X说:」形态时,再看引号后是否紧跟「X笑道」——两个来源独立尝试
|
|
71
|
+
if (!speaker) {
|
|
72
|
+
const post = SAID_RE.exec(text.slice(end, end + POST_ATTRIBUTION_WINDOW));
|
|
73
|
+
if (post)
|
|
74
|
+
speaker = cleanName(post[1]);
|
|
75
|
+
}
|
|
76
|
+
out.push({ speaker, line });
|
|
77
|
+
prevEnd = end;
|
|
78
|
+
}
|
|
79
|
+
return out;
|
|
80
|
+
}
|
|
81
|
+
function countBySpeaker(lines) {
|
|
82
|
+
const counts = new Map();
|
|
83
|
+
for (const { speaker } of lines) {
|
|
84
|
+
if (!speaker)
|
|
85
|
+
continue;
|
|
86
|
+
counts.set(speaker, (counts.get(speaker) ?? 0) + 1);
|
|
87
|
+
}
|
|
88
|
+
return counts;
|
|
89
|
+
}
|
|
90
|
+
function topSpeaker(counts, hint) {
|
|
91
|
+
if (hint) {
|
|
92
|
+
const wanted = [...counts.keys()].find((name) => name === hint.trim() || name.includes(hint.trim()) || hint.trim().includes(name));
|
|
93
|
+
if (wanted)
|
|
94
|
+
return wanted;
|
|
95
|
+
}
|
|
96
|
+
let best = null;
|
|
97
|
+
let bestCount = 0;
|
|
98
|
+
for (const [name, count] of counts) {
|
|
99
|
+
if (count > bestCount) {
|
|
100
|
+
best = name;
|
|
101
|
+
bestCount = count;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
return best;
|
|
105
|
+
}
|
|
106
|
+
/** 台词剥离 → 压缩空白 → 截断。 */
|
|
107
|
+
function condenseNarrative(text, cap = NARRATIVE_CAP) {
|
|
108
|
+
const stripped = text.replace(QUOTE_RE, "□").replace(/[ \t]+/g, " ");
|
|
109
|
+
const lines = stripped
|
|
110
|
+
.split("\n")
|
|
111
|
+
.map((line) => line.trim())
|
|
112
|
+
.filter(Boolean);
|
|
113
|
+
let out = "";
|
|
114
|
+
for (const line of lines) {
|
|
115
|
+
if (out.length + line.length + 1 > cap)
|
|
116
|
+
break;
|
|
117
|
+
out += (out ? "\n" : "") + line;
|
|
118
|
+
}
|
|
119
|
+
return out;
|
|
120
|
+
}
|
|
121
|
+
// ── 聊天记录解析(微信/QQ 导出格式)─────────────────────────────────────
|
|
122
|
+
//
|
|
123
|
+
// 微信「多选→复制」与第三方导出工具的常见形态:
|
|
124
|
+
// 昵称A
|
|
125
|
+
// 2026年08月31日 00:40
|
|
126
|
+
// 消息内容(可能多行)
|
|
127
|
+
//
|
|
128
|
+
// 昵称B
|
|
129
|
+
// 2026年08月31日 00:41
|
|
130
|
+
// [语音] 3" / [图片] xxx.jpg / [动画表情] / [无语] / 
|
|
131
|
+
//
|
|
132
|
+
// 结构特征:每个昵称独占一行,紧跟时间戳行。时间戳行是可靠的分块锚点,
|
|
133
|
+
// 据此可以切出「谁 → 说了什么」。导出不含「谁是自己」的标记——目标说话人
|
|
134
|
+
// 由 UI 点选(hint)传入,其余全部归为用户侧。
|
|
135
|
+
/** 聊天记录时间戳行:2026年08月31日 00:40(也兼容 2026-08-31 00:40)。 */
|
|
136
|
+
const CHAT_TS_RE = /^\d{4}[-年]\d{1,2}[-月]\d{1,2}日?\s+\d{1,2}:\d{2}/;
|
|
137
|
+
/** 非文本消息占位:[语音] 3" / [图片] 微信图片_xxx.jpg / [动画表情]。 */
|
|
138
|
+
const CHAT_PLACEHOLDER_RE = /^\[(语音|图片|视频|动画表情|表情|文件|链接|转账|红包|位置|名片|小程序|引用|音乐|语音通话|视频通话|接龙|笔记|收藏)/;
|
|
139
|
+
/** 纯方括号短占位(QQ 表情名如 [无语]、[捂脸])。 */
|
|
140
|
+
const CHAT_EMOJI_RE = /^\[[^\]\s]{1,8}\]$/;
|
|
141
|
+
/** 对象替换符(微信复制时图片/表情的残留)。 */
|
|
142
|
+
const OBJ_REPLACEMENT = "\uFFFC";
|
|
143
|
+
/**
|
|
144
|
+
* 解析聊天记录导出文本。识别不出聊天结构(时间戳锚点不足 / 说话人单一)
|
|
145
|
+
* 返回 null——调用方回退到小说/剧本挖掘。
|
|
146
|
+
*
|
|
147
|
+
* 结构锚点:说话人独占一行,紧跟时间戳行。解析用前瞻——若某行的下一行是
|
|
148
|
+
* 时间戳,则该行是说话人,内容从再下一行起,直到下一个说话人行。
|
|
149
|
+
*/
|
|
150
|
+
export function parseChatLog(text) {
|
|
151
|
+
const lines = text.split("\n");
|
|
152
|
+
const messages = [];
|
|
153
|
+
const counts = new Map();
|
|
154
|
+
let i = 0;
|
|
155
|
+
while (i < lines.length - 1) {
|
|
156
|
+
// 前瞻:下一行是时间戳 → 当前行是说话人
|
|
157
|
+
if (!CHAT_TS_RE.test(lines[i + 1].trim())) {
|
|
158
|
+
i++;
|
|
159
|
+
continue;
|
|
160
|
+
}
|
|
161
|
+
const speaker = lines[i].trim();
|
|
162
|
+
if (!speaker) {
|
|
163
|
+
i++;
|
|
164
|
+
continue;
|
|
165
|
+
}
|
|
166
|
+
const timestamp = lines[i + 1].trim();
|
|
167
|
+
const content = [];
|
|
168
|
+
let j = i + 2;
|
|
169
|
+
while (j < lines.length) {
|
|
170
|
+
// 下一行若是时间戳且当前行非空 → 新的说话人行,内容到此为止
|
|
171
|
+
if (j + 1 < lines.length && lines[j].trim() && CHAT_TS_RE.test(lines[j + 1].trim()))
|
|
172
|
+
break;
|
|
173
|
+
content.push(lines[j]);
|
|
174
|
+
j++;
|
|
175
|
+
}
|
|
176
|
+
const cleaned = cleanChatContent(content);
|
|
177
|
+
if (cleaned) {
|
|
178
|
+
messages.push({ speaker, text: cleaned, timestamp });
|
|
179
|
+
counts.set(speaker, (counts.get(speaker) ?? 0) + 1);
|
|
180
|
+
}
|
|
181
|
+
i = j;
|
|
182
|
+
}
|
|
183
|
+
if (messages.length < 4 || counts.size < 2)
|
|
184
|
+
return null;
|
|
185
|
+
const speakers = [...counts.entries()].sort((a, b) => b[1] - a[1]).map(([name]) => name);
|
|
186
|
+
return { messages, speakers };
|
|
187
|
+
}
|
|
188
|
+
/** 清洗一条消息的原始行:剔除占位符、对象替换符、空行,合并多行。 */
|
|
189
|
+
function cleanChatContent(lines) {
|
|
190
|
+
const kept = [];
|
|
191
|
+
for (const raw of lines) {
|
|
192
|
+
const line = raw.replaceAll(OBJ_REPLACEMENT, "").trim();
|
|
193
|
+
if (!line)
|
|
194
|
+
continue;
|
|
195
|
+
if (CHAT_PLACEHOLDER_RE.test(line))
|
|
196
|
+
continue;
|
|
197
|
+
if (CHAT_EMOJI_RE.test(line))
|
|
198
|
+
continue;
|
|
199
|
+
kept.push(line);
|
|
200
|
+
}
|
|
201
|
+
return kept.join(" ").trim();
|
|
202
|
+
}
|
|
203
|
+
/** 聊天记录模式挖掘:目标说话人(hint 或最高频)为台词主源,其余归用户侧。 */
|
|
204
|
+
export function mineChatLog(chat, hint) {
|
|
205
|
+
const target = hint?.trim() ? (chat.speakers.find((s) => s === hint.trim() || s.includes(hint.trim()) || hint.trim().includes(s)) ?? null) : null;
|
|
206
|
+
const speaker = target ?? chat.speakers[0] ?? null;
|
|
207
|
+
const targetLines = chat.messages.filter((m) => m.speaker === speaker).map((m) => m.text);
|
|
208
|
+
const otherLines = chat.messages.filter((m) => m.speaker !== speaker).map((m) => m.text);
|
|
209
|
+
// 真实对话对:允许用户连续发多条时,合并成一个上下文,避免只保留最后一句。
|
|
210
|
+
const pairs = [];
|
|
211
|
+
let pendingUser = [];
|
|
212
|
+
let pendingAt;
|
|
213
|
+
const toMillis = (stamp) => {
|
|
214
|
+
if (!stamp)
|
|
215
|
+
return null;
|
|
216
|
+
const normalized = stamp.replace(/年|月/g, "-").replace(/日/g, "");
|
|
217
|
+
const value = Date.parse(normalized.replace(/\s+/g, "T"));
|
|
218
|
+
return Number.isFinite(value) ? value : null;
|
|
219
|
+
};
|
|
220
|
+
for (const m of chat.messages) {
|
|
221
|
+
if (m.speaker === speaker) {
|
|
222
|
+
const gap = pendingAt && m.timestamp ? (toMillis(m.timestamp) - toMillis(pendingAt)) : null;
|
|
223
|
+
// 超过 6 小时视为新话题,不能把前一天的闲聊拼成当前回复的上下文。
|
|
224
|
+
if (pendingUser.length > 0 && m.text.length <= 240 && (gap === null || (gap >= 0 && gap <= 6 * 60 * 60 * 1000))) {
|
|
225
|
+
pairs.push({ user: pendingUser.join(" ").slice(-240), assistant: m.text });
|
|
226
|
+
}
|
|
227
|
+
pendingUser = [];
|
|
228
|
+
pendingAt = undefined;
|
|
229
|
+
}
|
|
230
|
+
else {
|
|
231
|
+
if (pendingUser.length === 0)
|
|
232
|
+
pendingAt = m.timestamp;
|
|
233
|
+
pendingUser.push(m.text);
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
// 契约不能只看目标单边台词:保留每条目标消息前后的小窗口,学习触发条件、
|
|
237
|
+
// 关系距离和情绪转折。窗口只进蒸馏 prompt,不会污染最终 few-shot 语料。
|
|
238
|
+
const targetIndexes = chat.messages.map((m, i) => m.speaker === speaker ? i : -1).filter((i) => i >= 0);
|
|
239
|
+
const contexts = [];
|
|
240
|
+
for (const i of evenSample(targetIndexes, 16)) {
|
|
241
|
+
const start = Math.max(0, i - 3);
|
|
242
|
+
const end = Math.min(chat.messages.length, i + 2);
|
|
243
|
+
const window = chat.messages.slice(start, end).map((m) => `${m.speaker === speaker ? "目标" : "用户"}:${m.text}`).join("\n");
|
|
244
|
+
contexts.push(window.slice(0, 700));
|
|
245
|
+
}
|
|
246
|
+
const targetTexts = targetLines;
|
|
247
|
+
const lengths = targetTexts.map((s) => s.length);
|
|
248
|
+
const avg = lengths.length ? Math.round(lengths.reduce((a, b) => a + b, 0) / lengths.length) : 0;
|
|
249
|
+
const sorted = [...lengths].sort((a, b) => a - b);
|
|
250
|
+
const median = sorted.length ? sorted[Math.floor(sorted.length / 2)] : 0;
|
|
251
|
+
const punct = (re) => targetTexts.reduce((n, s) => n + (s.match(re)?.length ?? 0), 0);
|
|
252
|
+
const emojiCount = punct(/[😀-🙏🌀-]/gu);
|
|
253
|
+
const questionCount = punct(/[??]/g);
|
|
254
|
+
const exclaimCount = punct(/[!!]/g);
|
|
255
|
+
const ellipsisCount = punct(/[…。]{2,}|\.\.\./g);
|
|
256
|
+
const stats = `样本数 ${targetTexts.length};平均 ${avg} 字;中位数 ${median} 字;含问号 ${questionCount} 条;含感叹号 ${exclaimCount} 条;含省略号 ${ellipsisCount} 条;emoji 总数 ${emojiCount}。`;
|
|
257
|
+
// 记忆点候选:真实事件类消息(生日/纪念/共同经历/对方的事实/约定)
|
|
258
|
+
const memoryPoints = chat.messages
|
|
259
|
+
.filter((m) => MEMORY_EVENT_RE.test(m.text))
|
|
260
|
+
.map((m) => m.text.slice(0, 160))
|
|
261
|
+
.slice(0, 12);
|
|
262
|
+
// 关系称呼:双方谁扮演什么角色。微信「备注名/对方昵称」本身即关系线索——
|
|
263
|
+
// 用户给 TA 的备注是「老公」→ 用户叫 TA 老公(userToTarget);TA 对用户的称呼
|
|
264
|
+
// 只能看 TA 消息里的称呼词(「老婆,…」)。消息内容中的称呼词同样计入另一侧。
|
|
265
|
+
const otherSpeakers = chat.speakers.filter((s) => s !== speaker);
|
|
266
|
+
const userLines = chat.messages.filter((m) => m.speaker !== speaker).map((m) => m.text);
|
|
267
|
+
const targetLinesAll = targetLines;
|
|
268
|
+
const greetingOf = (text) => {
|
|
269
|
+
if (!text)
|
|
270
|
+
return null;
|
|
271
|
+
for (const word of RELATION_WORDS) {
|
|
272
|
+
if (text.includes(word))
|
|
273
|
+
return word;
|
|
274
|
+
}
|
|
275
|
+
return null;
|
|
276
|
+
};
|
|
277
|
+
// 用户给目标的备注(如「老公」)= 用户如何称呼目标;目标消息里的称呼词(如「老婆」)= 目标如何称呼用户
|
|
278
|
+
const userToTarget = [...new Set([...RELATION_WORDS.filter((w) => speaker !== null && speaker.includes(w)), ...userLines.map(greetingOf).filter((w) => Boolean(w))])].slice(0, 6);
|
|
279
|
+
const targetToUser = [...new Set(targetLinesAll.map(greetingOf).filter((w) => Boolean(w)))].slice(0, 6);
|
|
280
|
+
const relationship = { userToTarget, targetToUser };
|
|
281
|
+
return {
|
|
282
|
+
speaker,
|
|
283
|
+
lines: evenSample(targetLines, MAX_MINED_LINES),
|
|
284
|
+
otherLines: evenSample(otherLines, MAX_OTHER_LINES),
|
|
285
|
+
narrative: "",
|
|
286
|
+
kind: "chat",
|
|
287
|
+
mixed: false,
|
|
288
|
+
// 不取最早的 12 组:按时间均匀覆盖全聊天,避免语料被开头某个话题垄断。
|
|
289
|
+
pairs: evenSample(pairs, 12),
|
|
290
|
+
contexts,
|
|
291
|
+
styleStats: stats,
|
|
292
|
+
memoryPoints,
|
|
293
|
+
relationship,
|
|
294
|
+
flow: buildChatFlow(chat.messages, speaker),
|
|
295
|
+
};
|
|
296
|
+
}
|
|
297
|
+
/** 关系称呼词:出现在对话中即揭示双方关系定位。 */
|
|
298
|
+
const RELATION_WORDS = [
|
|
299
|
+
"老公", "老婆", "媳妇", "弟妹", "宝宝", "宝贝", "亲爱的",
|
|
300
|
+
"爸爸", "妈妈", "爸", "妈", "爹", "娘",
|
|
301
|
+
"哥", "姐", "弟", "妹", "哥哥", "姐姐", "弟弟", "妹妹",
|
|
302
|
+
"师傅", "师父", "老板", "同事", "闺蜜", "兄弟", "哥们", "姐妹",
|
|
303
|
+
"女神", "男神", "前男/女友", "前男友", "前女友",
|
|
304
|
+
];
|
|
305
|
+
/** 真实事件信号:生日/纪念/年份/岁数/共同经历/对方身份事实/约定。 */
|
|
306
|
+
const MEMORY_EVENT_RE = /生日|纪念|周年|过完生日|周岁|去[^。,]{0,12}(过|去|玩|旅游)|第一次|那一年|去年|前年|过年|春节|中秋|国庆|跨年|毕业|结婚|认识[^。,]{0,10}年|领养|搬[^。,]{0,6}家|换工作|辞职|入职|生[了过][^。,]{0,8}(孩子|小孩|女儿|儿子)|考[上完研][^。,]{0,8}|我做|我是[^。,]{0,10}(医生|老师|老师|程序员|设计师)|我[在学过][^。,]{0,10}(编程|画画|钢琴|吉他)/;
|
|
307
|
+
export const MEMORY_POINT_CAP = 12;
|
|
308
|
+
/** 对话流单条消息字数上限(超限截断,保留开头)。 */
|
|
309
|
+
const FLOW_LINE_CAP = 160;
|
|
310
|
+
/** 对话流总字数预算:聊天记录可达 20 万字,记忆提炼 prompt 吃不下全文;
|
|
311
|
+
* 超出时按「一问一答」成对等距抽样,事件信号消息必留,保证关键事实不丢。 */
|
|
312
|
+
const FLOW_CHAR_BUDGET = 6000;
|
|
313
|
+
/**
|
|
314
|
+
* 构建记忆提炼用的完整对话流:双方消息按时间顺序、带归属(me)保留。
|
|
315
|
+
* 抽样策略:事件信号消息(生日/入职/搬家…)一律保留;其余消息在超预算时
|
|
316
|
+
* 按「一方的消息 + 紧随的另一方消息」成对等距抽取,保住一问一答的语境。
|
|
317
|
+
*/
|
|
318
|
+
function buildChatFlow(messages, target) {
|
|
319
|
+
const tagged = messages.map((m) => ({
|
|
320
|
+
me: m.speaker === target,
|
|
321
|
+
text: m.text.length > FLOW_LINE_CAP ? m.text.slice(0, FLOW_LINE_CAP) + "…" : m.text,
|
|
322
|
+
event: MEMORY_EVENT_RE.test(m.text),
|
|
323
|
+
}));
|
|
324
|
+
const total = tagged.reduce((sum, m) => sum + m.text.length, 0);
|
|
325
|
+
if (total <= FLOW_CHAR_BUDGET)
|
|
326
|
+
return tagged.map(({ me, text }) => ({ me, text }));
|
|
327
|
+
const keep = new Set();
|
|
328
|
+
let eventChars = 0;
|
|
329
|
+
tagged.forEach((m, i) => {
|
|
330
|
+
if (m.event) {
|
|
331
|
+
keep.add(i);
|
|
332
|
+
eventChars += m.text.length;
|
|
333
|
+
}
|
|
334
|
+
});
|
|
335
|
+
// 剩余预算按「对方消息+我的消息」成对抽样(对话天然是问↔答结构)
|
|
336
|
+
const pairBudget = Math.max(0, FLOW_CHAR_BUDGET - eventChars);
|
|
337
|
+
const pairs = [];
|
|
338
|
+
for (let i = 0; i < tagged.length - 1; i++) {
|
|
339
|
+
if (keep.has(i) || keep.has(i + 1))
|
|
340
|
+
continue;
|
|
341
|
+
if (tagged[i].me !== tagged[i + 1].me)
|
|
342
|
+
pairs.push(i);
|
|
343
|
+
}
|
|
344
|
+
const targetPairs = Math.max(1, Math.floor(pairBudget / 80)); // 每对约 80 字
|
|
345
|
+
for (const start of evenSample(pairs, targetPairs)) {
|
|
346
|
+
keep.add(start);
|
|
347
|
+
keep.add(start + 1);
|
|
348
|
+
}
|
|
349
|
+
return tagged.filter((_, i) => keep.has(i)).map(({ me, text }) => ({ me, text }));
|
|
350
|
+
}
|
|
351
|
+
/** 探测文本是否为聊天记录导出;是则返回说话人列表(供 UI 点选),否则 null。 */
|
|
352
|
+
export function detectChatLog(text) {
|
|
353
|
+
const chat = parseChatLog(text);
|
|
354
|
+
return chat ? chat.speakers : null;
|
|
355
|
+
}
|
|
356
|
+
/**
|
|
357
|
+
* 主入口:剧本格式(≥ MIN_SCRIPT_LINES 行)按名字归属;否则引号模式——
|
|
358
|
+
* 归属线索足够(≥3 条有归属)按说话人分,不足则视为单一声音素材全归目标。
|
|
359
|
+
*/
|
|
360
|
+
export function mineDialogue(text, hint) {
|
|
361
|
+
const normalized = text.replace(/\r\n?/g, "\n");
|
|
362
|
+
const chat = parseChatLog(normalized);
|
|
363
|
+
if (chat)
|
|
364
|
+
return mineChatLog(chat, hint);
|
|
365
|
+
const scriptLines = mineScriptLines(normalized);
|
|
366
|
+
if (scriptLines.length >= MIN_SCRIPT_LINES) {
|
|
367
|
+
const counts = countBySpeaker(scriptLines);
|
|
368
|
+
const target = topSpeaker(counts, hint) ?? [...counts.keys()][0] ?? null;
|
|
369
|
+
const targetLines = target ? scriptLines.filter((l) => l.speaker === target).map((l) => l.line) : [];
|
|
370
|
+
const otherLines = scriptLines.filter((l) => l.speaker !== target && l.speaker !== null).map((l) => l.line);
|
|
371
|
+
return {
|
|
372
|
+
speaker: target,
|
|
373
|
+
lines: evenSample(targetLines, MAX_MINED_LINES),
|
|
374
|
+
otherLines: evenSample(otherLines, MAX_OTHER_LINES),
|
|
375
|
+
narrative: condenseNarrative(normalized),
|
|
376
|
+
kind: "script",
|
|
377
|
+
mixed: false,
|
|
378
|
+
};
|
|
379
|
+
}
|
|
380
|
+
const quoted = mineQuotedLines(normalized);
|
|
381
|
+
if (quoted.length === 0) {
|
|
382
|
+
return { speaker: hint ?? null, lines: [], otherLines: [], narrative: condenseNarrative(normalized), kind: "none", mixed: true };
|
|
383
|
+
}
|
|
384
|
+
const attributed = quoted.filter((l) => l.speaker !== null);
|
|
385
|
+
if (attributed.length >= 3) {
|
|
386
|
+
// 归属足够:按说话人切分目标与他人
|
|
387
|
+
const counts = countBySpeaker(quoted);
|
|
388
|
+
const speaker = topSpeaker(counts, hint);
|
|
389
|
+
const targetLines = speaker ? quoted.filter((l) => l.speaker === speaker).map((l) => l.line) : [];
|
|
390
|
+
const otherLines = quoted.filter((l) => l.speaker !== null && l.speaker !== speaker).map((l) => l.line);
|
|
391
|
+
return {
|
|
392
|
+
speaker: speaker ?? hint ?? null,
|
|
393
|
+
lines: evenSample(targetLines, MAX_MINED_LINES),
|
|
394
|
+
otherLines: evenSample(otherLines, MAX_OTHER_LINES),
|
|
395
|
+
narrative: condenseNarrative(normalized),
|
|
396
|
+
kind: "quote",
|
|
397
|
+
mixed: false,
|
|
398
|
+
};
|
|
399
|
+
}
|
|
400
|
+
// 归属不足(小说多声部 / 独白 / 设定文档):全部台词交 LLM 甄别
|
|
401
|
+
return {
|
|
402
|
+
speaker: hint ?? null,
|
|
403
|
+
lines: evenSample(quoted.map((l) => l.line), MAX_MINED_LINES),
|
|
404
|
+
otherLines: [],
|
|
405
|
+
narrative: condenseNarrative(normalized),
|
|
406
|
+
kind: "quote",
|
|
407
|
+
mixed: true,
|
|
408
|
+
};
|
|
409
|
+
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 风格泄漏检测(纯函数):切换人设后,检查助手回复是否仍带着旧人设的声音。
|
|
3
|
+
*
|
|
4
|
+
* 背景:切换播报只在边界窗口(默认 2 个用户轮)内注入,窗口关闭后注入里只剩
|
|
5
|
+
* 新契约;长对话中旧人设的历史语气会压过新契约——需要持续、零成本的纠偏信号。
|
|
6
|
+
*
|
|
7
|
+
* 检测是词法的:旧人设的签名词(自称/称呼/口头禅,manifest.signatureWords)
|
|
8
|
+
* 在回复中出现得足够频繁即判为泄漏。误报的代价只是多注入一次边界提示,故
|
|
9
|
+
* 阈值取保守(多词命中或单词多次),宁可温和也不误伤。
|
|
10
|
+
*/
|
|
11
|
+
/** 剥去 fenced 代码块与行内代码——代码内容不属于「说话方式」。 */
|
|
12
|
+
export function stripCode(text) {
|
|
13
|
+
return text
|
|
14
|
+
.replace(/```[\s\S]*?```/g, " ")
|
|
15
|
+
.replace(/`[^`\n]*`/g, " ");
|
|
16
|
+
}
|
|
17
|
+
export const DEFAULT_LEAK_THRESHOLD = { distinctWords: 2, singleWordCount: 3 };
|
|
18
|
+
/**
|
|
19
|
+
* 判定回复是否泄漏旧人设的声音。
|
|
20
|
+
* 规则:命中 ≥ distinctWords 个不同签名词,或任一签名词出现 ≥ singleWordCount 次。
|
|
21
|
+
*/
|
|
22
|
+
export function detectLeak(replyText, signatureWords, threshold = DEFAULT_LEAK_THRESHOLD) {
|
|
23
|
+
const clean = stripCode(replyText);
|
|
24
|
+
const hits = [];
|
|
25
|
+
for (const word of signatureWords) {
|
|
26
|
+
if (!word)
|
|
27
|
+
continue;
|
|
28
|
+
let count = 0;
|
|
29
|
+
let idx = clean.indexOf(word);
|
|
30
|
+
while (idx !== -1) {
|
|
31
|
+
count++;
|
|
32
|
+
idx = clean.indexOf(word, idx + word.length);
|
|
33
|
+
}
|
|
34
|
+
if (count > 0)
|
|
35
|
+
hits.push({ word, count });
|
|
36
|
+
}
|
|
37
|
+
hits.sort((a, b) => b.count - a.count);
|
|
38
|
+
const leaked = hits.filter((h) => h.count >= threshold.singleWordCount).length > 0 ||
|
|
39
|
+
hits.filter((h) => h.count >= 1).length >= threshold.distinctWords;
|
|
40
|
+
return { leaked, hits };
|
|
41
|
+
}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 人设清单与语料的纯解析层。
|
|
3
|
+
*
|
|
4
|
+
* 解析与 IO 分离:这里没有任何 fs 依赖,坏行跳过、坏文件由宿主侧
|
|
5
|
+
* loadPersonalities 兜底,所有容错行为可直接单测。
|
|
6
|
+
*/
|
|
7
|
+
/** 解析 manifest 文本;结构不合法时抛错,由调用方兜底。 */
|
|
8
|
+
export function parseManifest(raw) {
|
|
9
|
+
const data = JSON.parse(raw);
|
|
10
|
+
const list = data?.personalities;
|
|
11
|
+
if (!Array.isArray(list))
|
|
12
|
+
throw new TypeError("manifest.personalities must be an array");
|
|
13
|
+
const out = [];
|
|
14
|
+
for (const item of list) {
|
|
15
|
+
const entry = item;
|
|
16
|
+
if (typeof entry.name !== "string" || !entry.name) {
|
|
17
|
+
throw new TypeError("manifest entry missing string `name`");
|
|
18
|
+
}
|
|
19
|
+
out.push({
|
|
20
|
+
name: entry.name,
|
|
21
|
+
displayName: typeof entry.displayName === "string" ? entry.displayName : entry.name,
|
|
22
|
+
description: typeof entry.description === "string" ? entry.description : "",
|
|
23
|
+
defaultName: typeof entry.defaultName === "string" && entry.defaultName ? entry.defaultName : undefined,
|
|
24
|
+
signatureWords: Array.isArray(entry.signatureWords) ? entry.signatureWords.filter((w) => typeof w === "string" && w.length > 0) : undefined,
|
|
25
|
+
promptFile: typeof entry.promptFile === "string" ? entry.promptFile : `${entry.name}.txt`,
|
|
26
|
+
corpusFile: typeof entry.corpusFile === "string" ? entry.corpusFile : `${entry.name}-corpus.jsonl`,
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
return out;
|
|
30
|
+
}
|
|
31
|
+
/** 解析单行语料;不合法返回 null(跳过而非塞占位对象)。 */
|
|
32
|
+
export function parseCorpusLine(line) {
|
|
33
|
+
const trimmed = line.trim();
|
|
34
|
+
if (!trimmed)
|
|
35
|
+
return null;
|
|
36
|
+
let data;
|
|
37
|
+
try {
|
|
38
|
+
data = JSON.parse(trimmed);
|
|
39
|
+
}
|
|
40
|
+
catch {
|
|
41
|
+
return null;
|
|
42
|
+
}
|
|
43
|
+
const rec = data;
|
|
44
|
+
if (typeof rec?.assistant !== "string")
|
|
45
|
+
return null;
|
|
46
|
+
return {
|
|
47
|
+
user: typeof rec.user === "string" ? rec.user : "",
|
|
48
|
+
assistant: rec.assistant,
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
/** 解析整个语料文件;坏行跳过。 */
|
|
52
|
+
export function parseCorpus(raw) {
|
|
53
|
+
const out = [];
|
|
54
|
+
for (const line of raw.split("\n")) {
|
|
55
|
+
const sample = parseCorpusLine(line);
|
|
56
|
+
if (sample)
|
|
57
|
+
out.push(sample);
|
|
58
|
+
}
|
|
59
|
+
return out;
|
|
60
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 人设提示词组装:基础风格契约 + 会话级稳定的语料示例。
|
|
3
|
+
*
|
|
4
|
+
* 「不使用人设」(none)与缺失人设的 promptText 均为空串,组装结果
|
|
5
|
+
* 为空字符串 —— 宿主侧对空串不注册任何注入内容。
|
|
6
|
+
*/
|
|
7
|
+
import { sampleForSession } from "./sampling.js";
|
|
8
|
+
/** 组装一段人设注入文本;sessionId 只用作采样种子,保证会话内稳定。 */
|
|
9
|
+
export function buildPersonaText(persona, sampleCount, sessionId) {
|
|
10
|
+
if (!persona)
|
|
11
|
+
return "";
|
|
12
|
+
const promptText = persona.promptText.trim();
|
|
13
|
+
const samples = sampleForSession(persona.corpus, sampleCount, sessionId, persona.name);
|
|
14
|
+
const corpusLines = samples
|
|
15
|
+
.map((entry) => {
|
|
16
|
+
const user = entry.user ?? "";
|
|
17
|
+
const assistant = entry.assistant ?? "";
|
|
18
|
+
if (user && assistant)
|
|
19
|
+
return `用户: ${user}\n回复: ${assistant}`;
|
|
20
|
+
if (assistant)
|
|
21
|
+
return `回复: ${assistant}`;
|
|
22
|
+
return "";
|
|
23
|
+
})
|
|
24
|
+
.filter(Boolean)
|
|
25
|
+
.join("\n\n");
|
|
26
|
+
const parts = [promptText];
|
|
27
|
+
if (corpusLines)
|
|
28
|
+
parts.push(`参考对话示例:\n(只模仿说话方式,不要把示例中的时间、地点、正在做什么或其他事实当成当前事实)\n${corpusLines}`);
|
|
29
|
+
return parts.filter(Boolean).join("\n\n");
|
|
30
|
+
}
|