@team-harness/memory-algorithms 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +27 -0
- package/README.md +203 -0
- package/dist/contracts.d.ts +178 -0
- package/dist/contracts.js +1 -0
- package/dist/index.d.ts +14 -0
- package/dist/index.js +6 -0
- package/dist/runtime/documents.d.ts +15 -0
- package/dist/runtime/documents.js +209 -0
- package/dist/runtime/l1.d.ts +10 -0
- package/dist/runtime/l1.js +172 -0
- package/dist/runtime/run.d.ts +44 -0
- package/dist/runtime/run.js +191 -0
- package/dist/runtime/skill-workspace.d.ts +70 -0
- package/dist/runtime/skill-workspace.js +156 -0
- package/dist/runtime/skills.d.ts +9 -0
- package/dist/runtime/skills.js +48 -0
- package/dist/runtime/telemetry.d.ts +10 -0
- package/dist/runtime/telemetry.js +5 -0
- package/dist/runtime/tools.d.ts +15 -0
- package/dist/runtime/tools.js +5 -0
- package/dist/upstream/config.d.ts +1 -0
- package/dist/upstream/config.js +1 -0
- package/dist/upstream/core/conversation/l0-recorder.d.ts +6 -0
- package/dist/upstream/core/conversation/l0-recorder.js +1 -0
- package/dist/upstream/core/memory-prompt/composer.d.ts +6 -0
- package/dist/upstream/core/memory-prompt/composer.js +33 -0
- package/dist/upstream/core/memory-prompt/types.d.ts +103 -0
- package/dist/upstream/core/memory-prompt/types.js +21 -0
- package/dist/upstream/core/prompts/l1-dedup.d.ts +33 -0
- package/dist/upstream/core/prompts/l1-dedup.js +202 -0
- package/dist/upstream/core/prompts/l1-extraction.d.ts +24 -0
- package/dist/upstream/core/prompts/l1-extraction.js +400 -0
- package/dist/upstream/core/prompts/persona-generation.d.ts +29 -0
- package/dist/upstream/core/prompts/persona-generation.js +284 -0
- package/dist/upstream/core/prompts/scene-extraction.d.ts +40 -0
- package/dist/upstream/core/prompts/scene-extraction.js +534 -0
- package/dist/upstream/core/record/l1-dedup.d.ts +10 -0
- package/dist/upstream/core/record/l1-dedup.js +108 -0
- package/dist/upstream/core/record/l1-extractor.d.ts +33 -0
- package/dist/upstream/core/record/l1-extractor.js +128 -0
- package/dist/upstream/core/record/l1-writer.d.ts +95 -0
- package/dist/upstream/core/record/l1-writer.js +1 -0
- package/dist/upstream/core/scene/filename-normalizer.d.ts +6 -0
- package/dist/upstream/core/scene/filename-normalizer.js +30 -0
- package/dist/upstream/core/scene/scene-format.d.ts +26 -0
- package/dist/upstream/core/scene/scene-format.js +53 -0
- package/dist/upstream/core/scene/scene-index.d.ts +7 -0
- package/dist/upstream/core/scene/scene-index.js +1 -0
- package/dist/upstream/core/scene/scene-navigation.d.ts +66 -0
- package/dist/upstream/core/scene/scene-navigation.js +107 -0
- package/dist/upstream/core/skill/conversation-add/message-compressor.d.ts +47 -0
- package/dist/upstream/core/skill/conversation-add/message-compressor.js +58 -0
- package/dist/upstream/core/skill/conversation-add/oversize-strategy.d.ts +41 -0
- package/dist/upstream/core/skill/conversation-add/oversize-strategy.js +100 -0
- package/dist/upstream/core/skill/prompts/skill-review-prompt.d.ts +39 -0
- package/dist/upstream/core/skill/prompts/skill-review-prompt.js +197 -0
- package/dist/upstream/core/skill/skill-extractor.d.ts +146 -0
- package/dist/upstream/core/skill/skill-extractor.js +432 -0
- package/dist/upstream/core/skill/skill-format.d.ts +46 -0
- package/dist/upstream/core/skill/skill-format.js +191 -0
- package/dist/upstream/core/skill/skill-tools.d.ts +75 -0
- package/dist/upstream/core/skill/skill-tools.js +193 -0
- package/dist/upstream/core/skill/types.d.ts +324 -0
- package/dist/upstream/core/skill/types.js +7 -0
- package/dist/upstream/utils/sanitize.d.ts +96 -0
- package/dist/upstream/utils/sanitize.js +359 -0
- package/package.json +28 -0
- package/upstream/baseline.json +426 -0
- package/upstream/changes.md +81 -0
|
@@ -0,0 +1,432 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SkillExtractor — 接受结构化 ExtractMessage[] 的抽取入口
|
|
3
|
+
*
|
|
4
|
+
* 与旧 SkillExtractor 的差异:
|
|
5
|
+
* - 入参 messages 必须是 `ExtractMessage[]`,不再接受裸字符串
|
|
6
|
+
* - 内部把 messages 串成 transcript(保留 role 标记)
|
|
7
|
+
* - 工具调用走 SkillToolsV2(操作 SkillCore)
|
|
8
|
+
* - 候选返回 ExtractedSkillCandidate 形态(含 skill_id / version)
|
|
9
|
+
*
|
|
10
|
+
* 每次调用都走 LLM,不做任何对话级去重/缓存 —— 缓存机制已移除。
|
|
11
|
+
*/
|
|
12
|
+
import { createSkillTools } from "./skill-tools.js";
|
|
13
|
+
import { metricProducer, trace, obsLogger } from "../../../runtime/telemetry.js";
|
|
14
|
+
const TAG = "[skill-extractor]";
|
|
15
|
+
export class SkillExtractor {
|
|
16
|
+
core;
|
|
17
|
+
runner;
|
|
18
|
+
systemPrompt;
|
|
19
|
+
maxIterations;
|
|
20
|
+
headChars;
|
|
21
|
+
tailChars;
|
|
22
|
+
maxTokens;
|
|
23
|
+
prefixSkillsLimit;
|
|
24
|
+
logger;
|
|
25
|
+
constructor(opts) {
|
|
26
|
+
this.core = opts.core;
|
|
27
|
+
this.runner = opts.runner;
|
|
28
|
+
this.systemPrompt = opts.systemPrompt ?? "You are a Skill Review Agent. Use tools to look at existing skills, decide what to add/improve, and call skill_create / skill_update / skill_patch / skill_files_write to persist.";
|
|
29
|
+
this.maxIterations = opts.maxIterations ?? 16;
|
|
30
|
+
this.headChars = opts.headChars ?? 8000;
|
|
31
|
+
this.tailChars = opts.tailChars ?? 32000;
|
|
32
|
+
this.maxTokens = opts.maxTokens;
|
|
33
|
+
// 构造器默认 0 (关闭前缀注入 → 不会触发额外的 query-gen LLM 调用);
|
|
34
|
+
// 生产 wiring 会显式传入 resolved.extraction.prefixSkillsLimit (默认 20)。
|
|
35
|
+
// <0 或非数字回落到 0 (等价于关闭), 而不是静默改到 20 —— 不想让配置错误
|
|
36
|
+
// 变成"意外多花一次 LLM"。
|
|
37
|
+
const rawLimit = opts.prefixSkillsLimit;
|
|
38
|
+
this.prefixSkillsLimit = rawLimit === undefined
|
|
39
|
+
? 0
|
|
40
|
+
: (Number.isFinite(rawLimit) && rawLimit >= 0 ? Math.floor(rawLimit) : 0);
|
|
41
|
+
this.logger = opts.logger;
|
|
42
|
+
}
|
|
43
|
+
async extract(input) {
|
|
44
|
+
const { messages } = input;
|
|
45
|
+
if (!Array.isArray(messages) || messages.length === 0) {
|
|
46
|
+
throw new Error("ExtractV2: messages must be a non-empty array of ExtractMessage");
|
|
47
|
+
}
|
|
48
|
+
// [obs] 一次抽取一条汇总事件;LLM 内部 iteration 走 langfuse trace(trace.report
|
|
49
|
+
// → OTel Span → Langfuse SpanProcessor 过滤上报),这里只做「入口→出口」
|
|
50
|
+
// 的耗时 / 候选数量汇总,用 task_id 做 anchor 跟 worker 段对齐。obsLogger
|
|
51
|
+
// 内部 try/catch + FileLogger + 后端降级,logger 挂了也不影响抽取本身。
|
|
52
|
+
const t0 = Date.now();
|
|
53
|
+
const transcript = formatTranscript(messages);
|
|
54
|
+
const truncated = truncateHeadTail(transcript, this.headChars, this.tailChars);
|
|
55
|
+
// 预检索 skill 列表, 塞在 user prompt 前面, 让 review agent 一进场就能看到
|
|
56
|
+
// agent 自己已经拥有哪些 skill (避免盲目 skill_create 撞 SKILL_NAME_DUPLICATE)。
|
|
57
|
+
//
|
|
58
|
+
// 三种模式, 由 prefixSkillsLimit + 该 agent 拥有的 skill 总数联合决定:
|
|
59
|
+
// full — total ≤ limit: 全部注入; 不花 query-gen LLM。
|
|
60
|
+
// relevant — total > limit + query-gen 成功 + BM25 命中 ≥1: 相关性优先。
|
|
61
|
+
// recent — total > limit + relevant 失败 (query-gen 抛 / 空 / 命中 0):
|
|
62
|
+
// 退回按 updated_at DESC 的 top-N + "还有 X 条未显示" 提示。
|
|
63
|
+
// none — prefixSkillsLimit=0 或 total=0。
|
|
64
|
+
//
|
|
65
|
+
// 关键: 一次 core.list({ limit }) 同时拿到 items + total, 之后所有分支复用,
|
|
66
|
+
// 不再二次查库; total ≤ limit 的场景直接跳过 query-gen, 覆盖率反而更高。
|
|
67
|
+
let prefixBlock = "";
|
|
68
|
+
let prefixMode = "none";
|
|
69
|
+
let prefixQuery;
|
|
70
|
+
if (this.prefixSkillsLimit > 0) {
|
|
71
|
+
let recentPage = null;
|
|
72
|
+
try {
|
|
73
|
+
recentPage = await this.core.list({
|
|
74
|
+
user_id: input.user_id,
|
|
75
|
+
team_id: input.team_id,
|
|
76
|
+
agent_id: input.agent_id,
|
|
77
|
+
pagination: { limit: this.prefixSkillsLimit },
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
catch (e) {
|
|
81
|
+
// list 挂 → 后续分支也没数据可用, 直接进 none 分支。
|
|
82
|
+
this.logger?.warn(`${TAG} prefix list failed: ${e.message}`);
|
|
83
|
+
}
|
|
84
|
+
if (recentPage && recentPage.items.length > 0) {
|
|
85
|
+
if (recentPage.total <= this.prefixSkillsLimit) {
|
|
86
|
+
// Case full: 拿到全部, 直接铺开; query-gen 完全省掉。
|
|
87
|
+
prefixBlock = renderFullSkillsBlock(recentPage.items);
|
|
88
|
+
prefixMode = "full";
|
|
89
|
+
}
|
|
90
|
+
else {
|
|
91
|
+
// Case relevant: total 超 limit, 花一次 LLM query-gen + BM25 找最相关的。
|
|
92
|
+
try {
|
|
93
|
+
const relevant = await this.buildRelevantSkillsBlock(input, truncated);
|
|
94
|
+
if (relevant) {
|
|
95
|
+
prefixBlock = relevant.block;
|
|
96
|
+
prefixMode = "relevant";
|
|
97
|
+
prefixQuery = relevant.query;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
catch (e) {
|
|
101
|
+
this.logger?.warn(`${TAG} buildRelevantSkillsBlock failed: ${e.message}`);
|
|
102
|
+
}
|
|
103
|
+
if (!prefixBlock) {
|
|
104
|
+
// Case recent: relevant 失败, 用一开始拿到的 top-N 兜底, 附带
|
|
105
|
+
// "还有 X 条未显示" 提示让 LLM 知道自己可以 skill_list 手动补拉。
|
|
106
|
+
prefixBlock = renderRecentSkillsBlock(recentPage.items, recentPage.total);
|
|
107
|
+
prefixMode = "recent";
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
let prompt = prefixBlock ? `${prefixBlock}\n\n---\n\n${truncated}` : truncated;
|
|
113
|
+
// 主 Agent 注入抽取提示(reason 非空时放在 prompt 最前面)
|
|
114
|
+
if (input.reason && input.reason.trim().length > 0) {
|
|
115
|
+
const hintBlock = [
|
|
116
|
+
"## 主 Agent 的抽取提示",
|
|
117
|
+
"以下是主 Agent 对本次对话的说明,请重点参考其意图进行抽取:",
|
|
118
|
+
input.reason,
|
|
119
|
+
].join("\n");
|
|
120
|
+
prompt = `${hintBlock}\n\n---\n\n${prompt}`;
|
|
121
|
+
}
|
|
122
|
+
const auditSink = [];
|
|
123
|
+
if (!this.runner) {
|
|
124
|
+
// No runner injected (test environment / disabled) → 返回空候选
|
|
125
|
+
this.logger?.info(`${TAG} no runner provided; returning empty candidates`);
|
|
126
|
+
obsLogger.info("skill.extractor.extract", {
|
|
127
|
+
task_id: input.task_id,
|
|
128
|
+
dur_ms: Date.now() - t0,
|
|
129
|
+
msg_count: messages.length,
|
|
130
|
+
candidates: 0,
|
|
131
|
+
skipped: "no_runner",
|
|
132
|
+
});
|
|
133
|
+
return { candidates: [] };
|
|
134
|
+
}
|
|
135
|
+
const tools = createSkillTools({
|
|
136
|
+
core: this.core,
|
|
137
|
+
user_id: input.user_id,
|
|
138
|
+
team_id: input.team_id,
|
|
139
|
+
agent_id: input.agent_id,
|
|
140
|
+
task_id: input.task_id,
|
|
141
|
+
auditSink,
|
|
142
|
+
logger: this.logger,
|
|
143
|
+
});
|
|
144
|
+
let text;
|
|
145
|
+
try {
|
|
146
|
+
text = await this.runner.run({
|
|
147
|
+
prompt,
|
|
148
|
+
systemPrompt: this.systemPrompt,
|
|
149
|
+
tools,
|
|
150
|
+
enableTools: true,
|
|
151
|
+
maxIterations: input.options?.max_iterations ?? this.maxIterations,
|
|
152
|
+
maxTokens: this.maxTokens,
|
|
153
|
+
taskId: `skill-extract-${input.task_id ?? "unknown"}`,
|
|
154
|
+
// Langfuse trace 语义:让此次抽取在 Langfuse UI 有稳定 name / 可筛选 tags。
|
|
155
|
+
// 详见 core/types.ts LLMRunParams 的 traceName/tags/sessionId/userId 注释。
|
|
156
|
+
traceName: "skill.extract",
|
|
157
|
+
tags: [
|
|
158
|
+
"skill-extract",
|
|
159
|
+
`team:${input.team_id}`,
|
|
160
|
+
`agent:${input.agent_id}`,
|
|
161
|
+
],
|
|
162
|
+
sessionId: input.session_id,
|
|
163
|
+
userId: input.user_id,
|
|
164
|
+
instanceId: input.space_id,
|
|
165
|
+
});
|
|
166
|
+
}
|
|
167
|
+
catch (e) {
|
|
168
|
+
// 一条 warn 汇总失败,包含 task_id / err_name / dur —— Worker 侧再按分类
|
|
169
|
+
// (transient / permanent) 决定 requeue 还是 DLQ。这里不吞异常。
|
|
170
|
+
const dur = Date.now() - t0;
|
|
171
|
+
obsLogger.warn("skill.extractor.extract", {
|
|
172
|
+
task_id: input.task_id,
|
|
173
|
+
dur_ms: dur,
|
|
174
|
+
msg_count: messages.length,
|
|
175
|
+
candidates: auditSink.length,
|
|
176
|
+
err_name: e.name,
|
|
177
|
+
err_msg: e.message,
|
|
178
|
+
});
|
|
179
|
+
try {
|
|
180
|
+
trace.report("skill.extractor.extract", {
|
|
181
|
+
task_id: input.task_id,
|
|
182
|
+
team_id: input.team_id,
|
|
183
|
+
agent_id: input.agent_id,
|
|
184
|
+
session_id: input.session_id,
|
|
185
|
+
msg_count: messages.length,
|
|
186
|
+
candidates: auditSink.length,
|
|
187
|
+
dur_ms: dur,
|
|
188
|
+
success: false,
|
|
189
|
+
error: e.message,
|
|
190
|
+
});
|
|
191
|
+
}
|
|
192
|
+
catch { /* noop */ }
|
|
193
|
+
throw e;
|
|
194
|
+
}
|
|
195
|
+
try {
|
|
196
|
+
metricProducer.send({ metric: "skill.extract.candidates", instanceId: input.team_id, value: auditSink.length });
|
|
197
|
+
}
|
|
198
|
+
catch { /* noop */ }
|
|
199
|
+
const dur = Date.now() - t0;
|
|
200
|
+
obsLogger.info("skill.extractor.extract", {
|
|
201
|
+
task_id: input.task_id,
|
|
202
|
+
dur_ms: dur,
|
|
203
|
+
msg_count: messages.length,
|
|
204
|
+
candidates: auditSink.length,
|
|
205
|
+
prompt_chars: prompt.length,
|
|
206
|
+
prefix_mode: prefixMode,
|
|
207
|
+
// 只截前 60 字符 (够识别关键词; 长了对 obs 无用)。
|
|
208
|
+
prefix_query: prefixQuery ? prefixQuery.slice(0, 60) : undefined,
|
|
209
|
+
});
|
|
210
|
+
try {
|
|
211
|
+
trace.report("skill.extractor.extract", {
|
|
212
|
+
task_id: input.task_id,
|
|
213
|
+
team_id: input.team_id,
|
|
214
|
+
agent_id: input.agent_id,
|
|
215
|
+
session_id: input.session_id,
|
|
216
|
+
msg_count: messages.length,
|
|
217
|
+
candidates: auditSink.length,
|
|
218
|
+
prompt_chars: prompt.length,
|
|
219
|
+
dur_ms: dur,
|
|
220
|
+
success: true,
|
|
221
|
+
});
|
|
222
|
+
}
|
|
223
|
+
catch { /* noop */ }
|
|
224
|
+
return { candidates: auditSink, text };
|
|
225
|
+
}
|
|
226
|
+
/**
|
|
227
|
+
* "可能与本对话相关"的 skill 预检索: 让 runner 用一次轻量 LLM 调用从
|
|
228
|
+
* transcript 里挤 2-5 个 BM25 关键词, 再走 core.search 拿 top-N (受
|
|
229
|
+
* prefixSkillsLimit 限制)。成功且非空 → 返回 { block, query }; query 为空
|
|
230
|
+
* / search 空命中 / runner 缺失都返回 null 让上游走 recent 降级。
|
|
231
|
+
*
|
|
232
|
+
* 拆两步(而不是直接把 relevant 检索合进 review agent 的迭代里)的原因:
|
|
233
|
+
* 1. 让 review agent 一进场就有个具体的候选池, 而不是先摸黑扫一遍再检索;
|
|
234
|
+
* 2. query-gen 是短输出 (≤100 tokens), 不占 review agent 上下文;
|
|
235
|
+
* 3. 失败可精确降级; review agent 内跑 skill_list 只会退化到"全 owner",
|
|
236
|
+
* 本函数则可选 relevant 或 recent。
|
|
237
|
+
*/
|
|
238
|
+
async buildRelevantSkillsBlock(input, transcript) {
|
|
239
|
+
if (!this.runner)
|
|
240
|
+
return null;
|
|
241
|
+
const query = await this.generateSearchQueryFromTranscript(input, transcript);
|
|
242
|
+
if (!query)
|
|
243
|
+
return null;
|
|
244
|
+
let hits;
|
|
245
|
+
try {
|
|
246
|
+
hits = await this.core.search({
|
|
247
|
+
user_id: input.user_id,
|
|
248
|
+
team_id: input.team_id,
|
|
249
|
+
agent_id: input.agent_id,
|
|
250
|
+
query,
|
|
251
|
+
top_k: this.prefixSkillsLimit,
|
|
252
|
+
});
|
|
253
|
+
}
|
|
254
|
+
catch (e) {
|
|
255
|
+
this.logger?.warn(`${TAG} relevant search failed for query='${query}': ${e.message}`);
|
|
256
|
+
return null;
|
|
257
|
+
}
|
|
258
|
+
if (!hits.length)
|
|
259
|
+
return null;
|
|
260
|
+
const lines = hits.map((h) => formatSkillLine(h.skill.name, h.skill.description));
|
|
261
|
+
const block = [
|
|
262
|
+
`## Skills possibly relevant to this conversation (query='${query}', BM25 prefetched)`,
|
|
263
|
+
"These were retrieved by pre-running skill_list(query=...) against your own skill pool.",
|
|
264
|
+
"If any of these already covers the topic, prefer `skill_update` / `skill_patch` over creating a duplicate.",
|
|
265
|
+
...lines,
|
|
266
|
+
].join("\n");
|
|
267
|
+
return { block, query };
|
|
268
|
+
}
|
|
269
|
+
/**
|
|
270
|
+
* 让 runner 用一次无工具的短 LLM 调用从 transcript 里抽 2-5 个 BM25 关键词。
|
|
271
|
+
* 返回值经过 sanitize (换行 / FTS5 保留词打成空格, 空串视为失败)。runner
|
|
272
|
+
* 抛异常一律往外扔, 上游 (buildRelevantSkillsBlock) 会 catch 并降级。
|
|
273
|
+
*/
|
|
274
|
+
async generateSearchQueryFromTranscript(input, transcript) {
|
|
275
|
+
if (!this.runner)
|
|
276
|
+
return "";
|
|
277
|
+
const raw = await this.runner.run({
|
|
278
|
+
systemPrompt: QUERY_GEN_SYSTEM_PROMPT,
|
|
279
|
+
prompt: [
|
|
280
|
+
"Below is a past conversation. Extract 2-5 short keywords (Chinese or English) that",
|
|
281
|
+
"capture what the user was trying to do. These will feed a BM25 search over an",
|
|
282
|
+
"existing skill library. Output the keywords on a single line, space-separated,",
|
|
283
|
+
"no punctuation, no labels, no explanation.",
|
|
284
|
+
"",
|
|
285
|
+
"<<transcript>>",
|
|
286
|
+
transcript,
|
|
287
|
+
"<<end-of-transcript>>",
|
|
288
|
+
].join("\n"),
|
|
289
|
+
enableTools: false,
|
|
290
|
+
// 让 runner 别真跑 tool loop, 就当一次普通 completion 用。
|
|
291
|
+
maxIterations: 1,
|
|
292
|
+
// 关键词很短; 32 token 足够 5 词 ×~6 字符 CJK, 也帮 runner 快速返回。
|
|
293
|
+
maxTokens: 64,
|
|
294
|
+
taskId: `skill-extract-query-${input.task_id ?? "unknown"}`,
|
|
295
|
+
// Langfuse 上可按此 traceName 单独筛这类 query-gen call, 跟主 skill.extract 分开。
|
|
296
|
+
traceName: "skill.extract.query-gen",
|
|
297
|
+
tags: [
|
|
298
|
+
"skill-extract",
|
|
299
|
+
"skill-extract-query-gen",
|
|
300
|
+
`team:${input.team_id}`,
|
|
301
|
+
`agent:${input.agent_id}`,
|
|
302
|
+
],
|
|
303
|
+
sessionId: input.session_id,
|
|
304
|
+
userId: input.user_id,
|
|
305
|
+
instanceId: input.space_id,
|
|
306
|
+
});
|
|
307
|
+
return sanitizeGeneratedQuery(raw);
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
// ═════════════════════════════════════════════════════════════════════
|
|
311
|
+
// helpers
|
|
312
|
+
// ═════════════════════════════════════════════════════════════════════
|
|
313
|
+
/**
|
|
314
|
+
* 把 ExtractMessage[] 序列化成给 Skill Review Agent 看的 transcript。
|
|
315
|
+
*
|
|
316
|
+
* 关键设计(对齐 SKILL_REVIEW_PROMPT 的 "Role isolation" 段):
|
|
317
|
+
* - role 前缀用非自然的 `<<past-xxx>>` 双尖括号 tag,而不是 `[user]` / `[assistant]`。
|
|
318
|
+
* 后者是 chat completion 的原生 role signal,模型会本能把 transcript 尾部的
|
|
319
|
+
* `[assistant]` 当成"该我续写下一 turn"的锚点,直接接着写主 agent 的回复。
|
|
320
|
+
* `<<past-user>>` 这类非典型 tag 打破这个暗示,让模型明确知道"这些是我在
|
|
321
|
+
* review 的历史内容,不是我要扮演的角色"。
|
|
322
|
+
* - 末尾追加 `<<end-of-transcript>>` 锚点 + 一句"现在按 system prompt 里的
|
|
323
|
+
* output contract 决策"。没有这个锚点时,模型看到 transcript 直接结束在
|
|
324
|
+
* 一段 assistant 长回复上,很容易走"再续写一段类似风格"的路径。
|
|
325
|
+
*
|
|
326
|
+
* 相关背景:docs/2026-07-28 skill extractor role-capture 分析(trace
|
|
327
|
+
* f546ab8c-c7c5-4598-a310-6b2162372e7c,抽取 LLM 完全忽略 SKILL_REVIEW_PROMPT
|
|
328
|
+
* 契约,产出 1235 tokens 的主对话续写,零 tool call、零 "Nothing to save.")。
|
|
329
|
+
*/
|
|
330
|
+
function formatTranscript(messages) {
|
|
331
|
+
const body = messages.map((m) => `<<past-${m.role}>>\n${m.content}`).join("\n\n");
|
|
332
|
+
return `${body}\n\n<<end-of-transcript>>\nAbove is the past conversation to review. Now decide, and respond only per the output contract in the system prompt.`;
|
|
333
|
+
}
|
|
334
|
+
function truncateHeadTail(s, head, tail) {
|
|
335
|
+
if (s.length <= head + tail)
|
|
336
|
+
return s;
|
|
337
|
+
return `${s.slice(0, head)}\n\n... [truncated ${s.length - head - tail} chars] ...\n\n${s.slice(-tail)}`;
|
|
338
|
+
}
|
|
339
|
+
/**
|
|
340
|
+
* "从对话里挤 2-5 个 BM25 关键词" 的 system prompt。
|
|
341
|
+
* 关键点:
|
|
342
|
+
* 1. 明确说这些关键词后面要塞进 BM25 → 让 LLM 挑高信号词, 别造完整句子;
|
|
343
|
+
* 2. 中英都行, jieba 会正确切;
|
|
344
|
+
* 3. 单行输出 + 无标点 + 无标签; 后置 sanitize 也会强制这个 shape。
|
|
345
|
+
*/
|
|
346
|
+
const QUERY_GEN_SYSTEM_PROMPT = [
|
|
347
|
+
"You are helping build a BM25 keyword query.",
|
|
348
|
+
"Given a past user↔assistant conversation, output 2-5 short high-signal keywords",
|
|
349
|
+
"(Chinese or English mixed as they appear in the transcript) that best represent",
|
|
350
|
+
"what the user was trying to accomplish.",
|
|
351
|
+
"",
|
|
352
|
+
"Rules:",
|
|
353
|
+
"- Output ONLY the keywords, on a single line, separated by single spaces.",
|
|
354
|
+
"- No punctuation, no quotes, no bullets, no labels, no explanation.",
|
|
355
|
+
"- Do NOT invent topics not present in the transcript.",
|
|
356
|
+
"- Prefer nouns / product names / verbs; drop filler words (the, a, 一下, 帮我).",
|
|
357
|
+
"- If the transcript is empty or has no clear intent, output an empty line.",
|
|
358
|
+
].join("\n");
|
|
359
|
+
/**
|
|
360
|
+
* 生成的 query 有效化:
|
|
361
|
+
* - 只取第一行 (LLM 有时会先说 "Here are the keywords:" 再一行)
|
|
362
|
+
* - 去 FTS5 保留词 (AND/OR/NOT/NEAR) 直接删掉 (BM25 层再走 buildFtsQuery 兜底)
|
|
363
|
+
* - 去掉常见标点 (标点会让 BM25 tokenizer 抖动)
|
|
364
|
+
* - 折叠空白, trim
|
|
365
|
+
* - 保守长度: 上限 120 字符 (5 词 × ~24 char)
|
|
366
|
+
* 输出空串 = 认为 LLM 没抽到; 上游走 recent 降级。
|
|
367
|
+
*/
|
|
368
|
+
export function sanitizeGeneratedQuery(raw) {
|
|
369
|
+
if (typeof raw !== "string")
|
|
370
|
+
return "";
|
|
371
|
+
// A₂ 派 thinking 模型 (minimax-m3 / GLM-4-thinking / qwq-32b / 部分 vLLM
|
|
372
|
+
// DeepSeek-R1) 会把 reasoning 内嵌为 `<think>…</think>` 塞在 content 里。
|
|
373
|
+
// 如果不先剥掉,`raw.split(/\r?\n/)` 的首个非空行会是 `<think>` 或 think 内容,
|
|
374
|
+
// 后续标点清洗会把 `<` `>` 打成空格 → BM25 拿到脏 query 命中 0 → 上游降级到
|
|
375
|
+
// recent。业务不断,但预检索静默失效。跟 l1-extractor.ts@ccaa5dc3 同一思路:
|
|
376
|
+
// - 非贪婪 `*?` + 强制 </think> 闭合:截断的 think tag (max_tokens 卡住)
|
|
377
|
+
// 不 match → 保留原样 → sanitize 后续把它当噪声处理 → recent 降级
|
|
378
|
+
// - `g` flag:多段 think 全剥
|
|
379
|
+
// - A₁ 派 (minimax-m2.7 / deepseek-v4-pro / o1 / o3) content 无 <think>,
|
|
380
|
+
// replace 是 no-op → byte-for-byte 等价,backward-compatible。
|
|
381
|
+
raw = raw.replace(/<think>[\s\S]*?<\/think>\s*/g, "");
|
|
382
|
+
// 只取首个非空行 —— LLM 偶尔会在关键词前后加一行元数据。
|
|
383
|
+
const firstLine = raw.split(/\r?\n/).map((l) => l.trim()).find((l) => l.length > 0) ?? "";
|
|
384
|
+
if (!firstLine)
|
|
385
|
+
return "";
|
|
386
|
+
// 剔常见标点 / FTS5 保留词。用空格替换而非删除, 避免 "foo,bar" 变成 "foobar"。
|
|
387
|
+
const noPunct = firstLine
|
|
388
|
+
.replace(/[,;:!?"'`()\[\]{}<>|/\\*+=~@#$%^&]/g, " ")
|
|
389
|
+
.replace(/[,。;:!?、()【】《》「」『』]/g, " ")
|
|
390
|
+
.replace(/\b(AND|OR|NOT|NEAR)\b/gi, " ");
|
|
391
|
+
const collapsed = noPunct.replace(/\s+/g, " ").trim();
|
|
392
|
+
if (!collapsed)
|
|
393
|
+
return "";
|
|
394
|
+
return collapsed.length > 120 ? collapsed.slice(0, 120).trimEnd() : collapsed;
|
|
395
|
+
}
|
|
396
|
+
/**
|
|
397
|
+
* 拼一行 "- name — 描述截断到 100 char"。空描述回退到 "- name"。
|
|
398
|
+
* 抽 recent / relevant / full 三个块共用, 避免行格式漂移。
|
|
399
|
+
*/
|
|
400
|
+
function formatSkillLine(name, description) {
|
|
401
|
+
const desc = (description ?? "").trim().replace(/\s+/g, " ");
|
|
402
|
+
const short = desc.length > 100 ? `${desc.slice(0, 100)}…` : desc;
|
|
403
|
+
return short ? `- ${name} — ${short}` : `- ${name}`;
|
|
404
|
+
}
|
|
405
|
+
/**
|
|
406
|
+
* "全量" 前缀块 —— 该 agent 的 skill 总数 ≤ prefixSkillsLimit, 直接铺开。
|
|
407
|
+
* 语义上告诉 LLM "这就是你所有的 skill, 无遗漏", 相比 recent 段更强的先验。
|
|
408
|
+
* 复用时无 truncated 提示 (因为没截断)。
|
|
409
|
+
*/
|
|
410
|
+
function renderFullSkillsBlock(items) {
|
|
411
|
+
const lines = items.map((s) => formatSkillLine(s.name, s.description));
|
|
412
|
+
return [
|
|
413
|
+
`## Skills you (this agent) own (${items.length} total — full list, no truncation)`,
|
|
414
|
+
"This is your entire skill inventory. Consider `skill_update` / `skill_patch` on an existing skill before creating a new one.",
|
|
415
|
+
...lines,
|
|
416
|
+
].join("\n");
|
|
417
|
+
}
|
|
418
|
+
/**
|
|
419
|
+
* "最近更新" 前缀块 —— relevant 分支失败时的兜底。附带 "还有 X 条未显示"
|
|
420
|
+
* 提示, 让 LLM 知道可以主动 skill_list(query=...) 拉更多。
|
|
421
|
+
*/
|
|
422
|
+
function renderRecentSkillsBlock(items, total) {
|
|
423
|
+
const lines = items.map((s) => formatSkillLine(s.name, s.description));
|
|
424
|
+
const omitted = total - items.length;
|
|
425
|
+
const header = `## Skills you (this agent) own (${items.length} most-recently-updated of ${total} total, prefetched via skill_list)`;
|
|
426
|
+
const hint = omitted > 0
|
|
427
|
+
? `Most recent first. ${omitted} more not shown — call skill_list(query=...) to search the rest. Consider \`skill_update\` / \`skill_patch\` on an existing skill before creating a new one.`
|
|
428
|
+
: "Most recent first. Consider `skill_update` / `skill_patch` on an existing skill before creating a new one.";
|
|
429
|
+
return [header, hint, ...lines].join("\n");
|
|
430
|
+
}
|
|
431
|
+
// A02: expose the canonical formatter for coverage accounting in the host-neutral runtime.
|
|
432
|
+
export { formatTranscript };
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* skill-format — SKILL.md ↔ SkillFile (de)serialization + validation.
|
|
3
|
+
*
|
|
4
|
+
* Why a dedicated module:
|
|
5
|
+
* - Keep frontmatter parsing rules in one place (the LLM Reviewer in M6
|
|
6
|
+
* emits SKILL.md text; the Gateway accepts SKILL.md text in
|
|
7
|
+
* /v3/skill/create; both flow through here).
|
|
8
|
+
* - Validation lives next to parsing so callers do `parse → validate`
|
|
9
|
+
* atomically.
|
|
10
|
+
*
|
|
11
|
+
* Frontmatter contract (SKILL_ENGINEERING_DESIGN §4.1 / §11.1):
|
|
12
|
+
* - YAML between leading `---\n` ... `\n---` fences (closing `---`
|
|
13
|
+
* followed by either newline or EOF).
|
|
14
|
+
* - REQUIRED: name, description.
|
|
15
|
+
* - OPTIONAL: category, created_at, updated_at, source, resources[].
|
|
16
|
+
*
|
|
17
|
+
* Limits (locked in M3):
|
|
18
|
+
* - name : 1..64 chars, ^[a-z0-9][a-z0-9-]*$
|
|
19
|
+
* - description: 1..1024 chars
|
|
20
|
+
* - body : ≤50_000 chars
|
|
21
|
+
* - resources[*].type ∈ {text, executable, binary}
|
|
22
|
+
*/
|
|
23
|
+
import type { SkillFile } from "./types.js";
|
|
24
|
+
/**
|
|
25
|
+
* Parse a SKILL.md raw string into a SkillFile.
|
|
26
|
+
*
|
|
27
|
+
* Accepts CRLF or LF line endings. Throws with a clear message when:
|
|
28
|
+
* - the file does not start with a `---\n` (no frontmatter)
|
|
29
|
+
* - the closing `---` fence is missing
|
|
30
|
+
* - the YAML body cannot be parsed
|
|
31
|
+
* - the parsed YAML is missing `name` or `description`
|
|
32
|
+
*
|
|
33
|
+
* No length / regex checks here — call `validateSkillFile` next.
|
|
34
|
+
*/
|
|
35
|
+
export declare function parseSkillFile(raw: string): SkillFile;
|
|
36
|
+
/**
|
|
37
|
+
* Throw if the SkillFile violates any contract limit.
|
|
38
|
+
* Mutates nothing; pure assertion.
|
|
39
|
+
*/
|
|
40
|
+
export declare function validateSkillFile(file: SkillFile): void;
|
|
41
|
+
/**
|
|
42
|
+
* Serialize a SkillFile back to canonical SKILL.md text (frontmatter + body).
|
|
43
|
+
* Round-trip safe: parseSkillFile(formatSkillFile(f)) yields a file with the
|
|
44
|
+
* same logical fields and body.
|
|
45
|
+
*/
|
|
46
|
+
export declare function formatSkillFile(file: SkillFile): string;
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* skill-format — SKILL.md ↔ SkillFile (de)serialization + validation.
|
|
3
|
+
*
|
|
4
|
+
* Why a dedicated module:
|
|
5
|
+
* - Keep frontmatter parsing rules in one place (the LLM Reviewer in M6
|
|
6
|
+
* emits SKILL.md text; the Gateway accepts SKILL.md text in
|
|
7
|
+
* /v3/skill/create; both flow through here).
|
|
8
|
+
* - Validation lives next to parsing so callers do `parse → validate`
|
|
9
|
+
* atomically.
|
|
10
|
+
*
|
|
11
|
+
* Frontmatter contract (SKILL_ENGINEERING_DESIGN §4.1 / §11.1):
|
|
12
|
+
* - YAML between leading `---\n` ... `\n---` fences (closing `---`
|
|
13
|
+
* followed by either newline or EOF).
|
|
14
|
+
* - REQUIRED: name, description.
|
|
15
|
+
* - OPTIONAL: category, created_at, updated_at, source, resources[].
|
|
16
|
+
*
|
|
17
|
+
* Limits (locked in M3):
|
|
18
|
+
* - name : 1..64 chars, ^[a-z0-9][a-z0-9-]*$
|
|
19
|
+
* - description: 1..1024 chars
|
|
20
|
+
* - body : ≤50_000 chars
|
|
21
|
+
* - resources[*].type ∈ {text, executable, binary}
|
|
22
|
+
*/
|
|
23
|
+
import { parse as parseYaml, stringify as stringifyYaml } from "yaml";
|
|
24
|
+
const NAME_MAX = 64;
|
|
25
|
+
const DESCRIPTION_MAX = 1024;
|
|
26
|
+
const BODY_MAX = 50_000;
|
|
27
|
+
const NAME_REGEX = /^[a-z0-9][a-z0-9-]*$/;
|
|
28
|
+
const RESOURCE_TYPES = new Set(["text", "executable", "binary"]);
|
|
29
|
+
/**
|
|
30
|
+
* Parse a SKILL.md raw string into a SkillFile.
|
|
31
|
+
*
|
|
32
|
+
* Accepts CRLF or LF line endings. Throws with a clear message when:
|
|
33
|
+
* - the file does not start with a `---\n` (no frontmatter)
|
|
34
|
+
* - the closing `---` fence is missing
|
|
35
|
+
* - the YAML body cannot be parsed
|
|
36
|
+
* - the parsed YAML is missing `name` or `description`
|
|
37
|
+
*
|
|
38
|
+
* No length / regex checks here — call `validateSkillFile` next.
|
|
39
|
+
*/
|
|
40
|
+
export function parseSkillFile(raw) {
|
|
41
|
+
// Normalize line endings so the fence matcher is simpler.
|
|
42
|
+
const text = raw.replace(/\r\n?/g, "\n");
|
|
43
|
+
if (!text.startsWith("---\n") && text !== "---\n") {
|
|
44
|
+
throw new Error(`[skill][format] missing frontmatter — file must start with '---\\n'`);
|
|
45
|
+
}
|
|
46
|
+
// Find the closing fence. Must be on its own line, after the opening fence.
|
|
47
|
+
// We search starting from index 4 (past the opening "---\n") for "\n---" followed
|
|
48
|
+
// by either "\n" or EOF.
|
|
49
|
+
const start = 4;
|
|
50
|
+
let close = -1;
|
|
51
|
+
let endOfClose = -1;
|
|
52
|
+
for (let i = start; i < text.length - 3; i++) {
|
|
53
|
+
if (text[i] === "\n" && text[i + 1] === "-" && text[i + 2] === "-" && text[i + 3] === "-") {
|
|
54
|
+
// Must be followed by newline or EOF.
|
|
55
|
+
const after = text[i + 4];
|
|
56
|
+
if (after === undefined || after === "\n") {
|
|
57
|
+
close = i + 1; // index of the closing "---"
|
|
58
|
+
endOfClose = i + 4 + (after === "\n" ? 1 : 0);
|
|
59
|
+
break;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
if (close === -1) {
|
|
64
|
+
throw new Error(`[skill][format] missing closing '---' fence in frontmatter`);
|
|
65
|
+
}
|
|
66
|
+
const yamlText = text.slice(start, close);
|
|
67
|
+
let fm;
|
|
68
|
+
try {
|
|
69
|
+
fm = parseYaml(yamlText) ?? {};
|
|
70
|
+
}
|
|
71
|
+
catch (err) {
|
|
72
|
+
throw new Error(`[skill][format] YAML parse error: ${err instanceof Error ? err.message : String(err)}`);
|
|
73
|
+
}
|
|
74
|
+
// Coerce non-string values to string so YAML's type inference (null / true /
|
|
75
|
+
// 123 / comment-like #…) doesn't cause confusing "missing required field"
|
|
76
|
+
// errors when the key is clearly present. An empty value after coercion is
|
|
77
|
+
// still an error — every SKILL.md MUST carry a non-empty description.
|
|
78
|
+
//
|
|
79
|
+
// Distinguish between "key absent" (undefined) and "key present but non-string"
|
|
80
|
+
// (null / true / 123). The former is still an error; the latter is coerced.
|
|
81
|
+
const nameRaw = fm.name;
|
|
82
|
+
const name = typeof nameRaw === "string"
|
|
83
|
+
? nameRaw
|
|
84
|
+
: nameRaw === undefined || nameRaw === null
|
|
85
|
+
? ""
|
|
86
|
+
: String(nameRaw);
|
|
87
|
+
if (name.trim() === "") {
|
|
88
|
+
throw new Error(`[skill][format] frontmatter missing required field 'name'`);
|
|
89
|
+
}
|
|
90
|
+
fm.name = name;
|
|
91
|
+
const descRaw = fm.description;
|
|
92
|
+
const description = typeof descRaw === "string"
|
|
93
|
+
? descRaw
|
|
94
|
+
: descRaw === undefined || descRaw === null
|
|
95
|
+
? ""
|
|
96
|
+
: String(descRaw);
|
|
97
|
+
if (description.trim() === "") {
|
|
98
|
+
throw new Error(`[skill][format] frontmatter missing required field 'description'`);
|
|
99
|
+
}
|
|
100
|
+
fm.description = description;
|
|
101
|
+
// body is everything after the closing fence; strip a single leading newline
|
|
102
|
+
// so SKILL.md authors can leave a blank line between fence and body.
|
|
103
|
+
let body = text.slice(endOfClose);
|
|
104
|
+
if (body.startsWith("\n"))
|
|
105
|
+
body = body.slice(1);
|
|
106
|
+
return {
|
|
107
|
+
frontmatter: {
|
|
108
|
+
name,
|
|
109
|
+
description,
|
|
110
|
+
category: typeof fm.category === "string" ? fm.category : undefined,
|
|
111
|
+
created_at: typeof fm.created_at === "string" ? fm.created_at : undefined,
|
|
112
|
+
updated_at: typeof fm.updated_at === "string" ? fm.updated_at : undefined,
|
|
113
|
+
source: fm.source === "auto" || fm.source === "manual" ? fm.source : undefined,
|
|
114
|
+
resources: parseResources(fm.resources),
|
|
115
|
+
},
|
|
116
|
+
body,
|
|
117
|
+
raw,
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
/**
|
|
121
|
+
* Throw if the SkillFile violates any contract limit.
|
|
122
|
+
* Mutates nothing; pure assertion.
|
|
123
|
+
*/
|
|
124
|
+
export function validateSkillFile(file) {
|
|
125
|
+
const { name, description, resources } = file.frontmatter;
|
|
126
|
+
if (!NAME_REGEX.test(name)) {
|
|
127
|
+
throw new Error(`[skill][format] invalid name '${name}' — must match ^[a-z0-9][a-z0-9-]*$`);
|
|
128
|
+
}
|
|
129
|
+
if (name.length > NAME_MAX) {
|
|
130
|
+
throw new Error(`[skill][format] name length ${name.length} exceeds max ${NAME_MAX}`);
|
|
131
|
+
}
|
|
132
|
+
if (description.length > DESCRIPTION_MAX) {
|
|
133
|
+
throw new Error(`[skill][format] description length ${description.length} exceeds max ${DESCRIPTION_MAX}`);
|
|
134
|
+
}
|
|
135
|
+
if (file.body.length > BODY_MAX) {
|
|
136
|
+
throw new Error(`[skill][format] body length ${file.body.length} exceeds max ${BODY_MAX}`);
|
|
137
|
+
}
|
|
138
|
+
if (resources) {
|
|
139
|
+
for (const r of resources) {
|
|
140
|
+
if (!RESOURCE_TYPES.has(r.type)) {
|
|
141
|
+
throw new Error(`[skill][format] resource type '${r.type}' invalid — must be one of text/executable/binary`);
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
/**
|
|
147
|
+
* Serialize a SkillFile back to canonical SKILL.md text (frontmatter + body).
|
|
148
|
+
* Round-trip safe: parseSkillFile(formatSkillFile(f)) yields a file with the
|
|
149
|
+
* same logical fields and body.
|
|
150
|
+
*/
|
|
151
|
+
export function formatSkillFile(file) {
|
|
152
|
+
const fm = {
|
|
153
|
+
name: file.frontmatter.name,
|
|
154
|
+
description: file.frontmatter.description,
|
|
155
|
+
};
|
|
156
|
+
if (file.frontmatter.category !== undefined)
|
|
157
|
+
fm.category = file.frontmatter.category;
|
|
158
|
+
if (file.frontmatter.created_at !== undefined)
|
|
159
|
+
fm.created_at = file.frontmatter.created_at;
|
|
160
|
+
if (file.frontmatter.updated_at !== undefined)
|
|
161
|
+
fm.updated_at = file.frontmatter.updated_at;
|
|
162
|
+
if (file.frontmatter.source !== undefined)
|
|
163
|
+
fm.source = file.frontmatter.source;
|
|
164
|
+
if (file.frontmatter.resources && file.frontmatter.resources.length > 0) {
|
|
165
|
+
fm.resources = file.frontmatter.resources;
|
|
166
|
+
}
|
|
167
|
+
const yamlBlock = stringifyYaml(fm).replace(/\n+$/, "");
|
|
168
|
+
return `---\n${yamlBlock}\n---\n\n${file.body}`;
|
|
169
|
+
}
|
|
170
|
+
// ============================
|
|
171
|
+
// Internal helpers
|
|
172
|
+
// ============================
|
|
173
|
+
function parseResources(raw) {
|
|
174
|
+
if (!Array.isArray(raw))
|
|
175
|
+
return undefined;
|
|
176
|
+
const out = [];
|
|
177
|
+
for (const item of raw) {
|
|
178
|
+
if (typeof item !== "object" || item == null)
|
|
179
|
+
continue;
|
|
180
|
+
const o = item;
|
|
181
|
+
if (typeof o.path !== "string" || typeof o.type !== "string")
|
|
182
|
+
continue;
|
|
183
|
+
// Note: type is preserved verbatim; full enum check happens in validateSkillFile
|
|
184
|
+
// so callers see a clean error message rather than a silent drop.
|
|
185
|
+
out.push({
|
|
186
|
+
path: o.path,
|
|
187
|
+
type: o.type,
|
|
188
|
+
});
|
|
189
|
+
}
|
|
190
|
+
return out;
|
|
191
|
+
}
|