mocode-ai 1.2.6 → 1.2.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/core.js +72 -32
- package/dist/agent/index.js +70 -34
- package/dist/agent/spawn.js +4 -0
- package/dist/commands/skill.js +230 -0
- package/dist/config/index.js +66 -4
- package/dist/config/presets.js +29 -7
- package/dist/context/artifacts.js +20 -0
- package/dist/context/budget.js +20 -7
- package/dist/context/index.js +1 -1
- package/dist/host/stdio.js +10 -1
- package/dist/llm/index.js +31 -6
- package/dist/llm/providers/anthropic.js +370 -0
- package/dist/repl/index.js +85 -40
- package/dist/session/compact.js +121 -34
- package/dist/session/persist.js +2 -2
- package/dist/session/scheduler.js +4 -4
- package/dist/skills/skill-eval.js +345 -0
- package/dist/skills/skill-improve.js +221 -0
- package/dist/skills/stats.js +102 -0
- package/dist/tools/constants.js +14 -0
- package/dist/tools/registry.js +5 -1
- package/dist/ui/layout.js +11 -0
- package/dist/verification/prompt.js +55 -0
- package/package.json +2 -2
package/dist/session/compact.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { chat, chatTools, correctTokenEstimate, estimatePromptTokens, estimateTokens, } from '../llm/index.js';
|
|
2
2
|
import { config } from '../config/index.js';
|
|
3
|
-
import { MAX_HISTORY_RESULT, MAX_MEMORY_RESULT, MAX_OLD_TOOL_STUB, MAX_SKILL_RESULT } from '../tools/constants.js';
|
|
3
|
+
import { MAX_HISTORY_RESULT, MAX_MEMORY_RESULT, MAX_OLD_TOOL_STUB, MAX_SKILL_RESULT, SUMMARY_MSG_MAX_CHARS, SUMMARY_OUTPUT_MAX_CHARS, SUMMARY_TRANSCRIPT_WINDOW_RATIO, } from '../tools/constants.js';
|
|
4
4
|
import { ui } from '../ui/theme.js';
|
|
5
5
|
import { Spinner } from '../ui/spinner.js';
|
|
6
6
|
import * as layout from '../ui/layout.js';
|
|
@@ -221,45 +221,105 @@ function microcompactGroup(g) {
|
|
|
221
221
|
return done;
|
|
222
222
|
}
|
|
223
223
|
// ── 默认摘要器:复用 chat(),空 handlers 不打印 ──────────────────────────
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
224
|
+
/** 单条消息封顶:中间段省略、保头 + 尾(头有路径/意图,尾有结论/报错),总长 ≤ max。 */
|
|
225
|
+
function capMessageText(text, max) {
|
|
226
|
+
if (text.length <= max)
|
|
227
|
+
return text;
|
|
228
|
+
const removed = text.length - max;
|
|
229
|
+
const marker = `\n…[中间 ${removed} 字符已省略]…\n`;
|
|
230
|
+
let remain = max - marker.length;
|
|
231
|
+
if (remain <= 0)
|
|
232
|
+
return text.slice(0, Math.max(0, max)) + marker;
|
|
233
|
+
const head = Math.ceil(remain * 0.6);
|
|
234
|
+
return text.slice(0, head) + marker + text.slice(text.length - (remain - head));
|
|
235
|
+
}
|
|
236
|
+
/** tool_call 参数封顶:只保头(路径/目标等关键信息在开头,长正文对摘要无信息量)。 */
|
|
237
|
+
function capArgsHead(args, max) {
|
|
238
|
+
if (args.length <= max)
|
|
239
|
+
return args;
|
|
240
|
+
return args.slice(0, max) + ` …[另有 ${args.length - max} 字符已省略]`;
|
|
241
|
+
}
|
|
242
|
+
/** 摘要输出硬上限:按最后换行处裁断(不留半句),防模型不听话产出巨长摘要撑大 history。 */
|
|
243
|
+
function capSummaryOutput(summary) {
|
|
244
|
+
if (summary.length <= SUMMARY_OUTPUT_MAX_CHARS)
|
|
245
|
+
return summary;
|
|
246
|
+
const cut = summary.lastIndexOf('\n', SUMMARY_OUTPUT_MAX_CHARS);
|
|
247
|
+
const at = cut > SUMMARY_OUTPUT_MAX_CHARS * 0.5 ? cut : SUMMARY_OUTPUT_MAX_CHARS;
|
|
248
|
+
return summary.slice(0, at) + '\n[摘要输出超长,已截断]';
|
|
249
|
+
}
|
|
250
|
+
/** 按封顶配额拼转录。scale ∈ (0,1] 等比缩小各角色配额(总预算不足时用)。 */
|
|
251
|
+
function buildTranscript(stripped, scale) {
|
|
252
|
+
const caps = SUMMARY_MSG_MAX_CHARS;
|
|
253
|
+
const capFor = (role) => Math.max(200, Math.floor(scale *
|
|
254
|
+
(role === 'user'
|
|
255
|
+
? caps.user
|
|
256
|
+
: role === 'assistant'
|
|
257
|
+
? caps.assistant
|
|
258
|
+
: role === 'tool'
|
|
259
|
+
? caps.tool
|
|
260
|
+
: caps.other)));
|
|
261
|
+
return stripped
|
|
228
262
|
.map((m) => {
|
|
229
|
-
const
|
|
230
|
-
let line = `${role}: ${toText(m.content)}`;
|
|
263
|
+
const cap = capFor(m.role);
|
|
264
|
+
let line = `${m.role}: ${capMessageText(toText(m.content), cap)}`;
|
|
231
265
|
const tcs = m.tool_calls;
|
|
232
266
|
if (tcs) {
|
|
233
267
|
for (const tc of tcs) {
|
|
234
|
-
|
|
268
|
+
const args = tc?.function?.arguments ?? '';
|
|
269
|
+
line += `\n [tool_call ${tc?.function?.name}] ${capArgsHead(args, Math.max(200, Math.floor(scale * caps.tool)))}`;
|
|
235
270
|
}
|
|
236
271
|
}
|
|
237
272
|
return line;
|
|
238
273
|
})
|
|
239
274
|
.join('\n');
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
275
|
+
}
|
|
276
|
+
async function defaultSummarize(older, focus) {
|
|
277
|
+
// 摘要前剥离多模态 user 消息里的图片(base64 会撑爆摘要 prompt;image 对摘要无信息量)。
|
|
278
|
+
const stripped = older.map(stripImagesForSummary);
|
|
279
|
+
// 触发压缩时旧区约占窗口 50-60%,摘要器共享同一窗口——逐条封顶后旧区原文即可装入,
|
|
280
|
+
// 不需要"先压一遍再摘要"。封顶策略:先按满额封顶;总量仍超窗口 55% 预算时等比缩小
|
|
281
|
+
// 配额重拼(优先保条数/每轮都有代表,其次保单条长度);再超才整段中截兜底(罕见)。
|
|
282
|
+
let transcript = buildTranscript(stripped, 1);
|
|
283
|
+
const tokenBudget = Math.floor(config.contextWindowTokens * SUMMARY_TRANSCRIPT_WINDOW_RATIO);
|
|
284
|
+
let tokens = estimateTokens(transcript);
|
|
285
|
+
if (tokens > tokenBudget) {
|
|
286
|
+
transcript = buildTranscript(stripped, tokenBudget / tokens);
|
|
287
|
+
tokens = estimateTokens(transcript);
|
|
288
|
+
if (tokens > Math.floor(config.contextWindowTokens * 0.6)) {
|
|
289
|
+
transcript = truncateMid(transcript, Math.floor(config.contextWindowTokens * 0.5));
|
|
290
|
+
}
|
|
244
291
|
}
|
|
245
292
|
const sysMsg = {
|
|
246
293
|
role: 'system',
|
|
247
|
-
content: 'You are
|
|
294
|
+
content: 'You are an aggressive session compressor writing a handoff note for an agent that lost its context. ' +
|
|
295
|
+
'The agent will continue the task with ONLY your summary plus a few most-recent messages, so your summary is the sole memory of everything older.\n' +
|
|
296
|
+
'Output ONLY the summary body in this exact structure (omit empty sections):\n' +
|
|
297
|
+
'## Objective — the user\'s core request(s); cover EVERY distinct user request in the transcript, in order, noting which are completed vs pending.\n' +
|
|
298
|
+
'## Completed — what is already done: files created/modified (exact paths), key change per file, commands run and their outcomes (pass/fail, key numbers), decisions made and why.\n' +
|
|
299
|
+
'## In Progress — what is being worked on right now and exactly where it stopped (e.g. "edit applied to foo.ts, test not yet run").\n' +
|
|
300
|
+
'## Next Steps — the concrete next actions in order.\n' +
|
|
301
|
+
'## Key Facts — only what later steps cannot work without: exact paths, symbols/API shapes, artifact IDs/hashes, constraints, open questions, failed approaches and errors to avoid repeating.\n' +
|
|
302
|
+
'What counts as important (keep): user requests and constraints; final state of each modified file; conclusions and results, not the steps that led there; decisions with reasons; precise references (paths, symbols, hashes, commands) that later steps must cite; failures and what was tried, so mistakes are not repeated.\n' +
|
|
303
|
+
'What to drop: verbatim file contents and tool-output dumps, step-by-step recaps, exploration dead-ends that led nowhere, polite chatter, anything re-derivable by re-reading files.\n' +
|
|
304
|
+
'Rules: total ≤ 400 words. State conclusions and locations (path:line where useful), never paste content. ' +
|
|
305
|
+
'Never invent facts, paths, or results that are not in the transcript; if unsure whether something happened, omit it.',
|
|
248
306
|
};
|
|
249
307
|
const userMsg = {
|
|
250
308
|
role: 'user',
|
|
251
309
|
content: focus
|
|
252
|
-
?
|
|
253
|
-
:
|
|
310
|
+
? `请将以下会话历史压缩成交接摘要,重点保留与「${focus}」相关的事实/决策/文件改动:\n\n${transcript}\n\n摘要:`
|
|
311
|
+
: `请将以下会话历史压缩成交接摘要:\n\n${transcript}\n\n摘要:`,
|
|
254
312
|
};
|
|
255
313
|
const spinner = new Spinner((msg, frame) => layout.setStatus(msg, frame ?? undefined));
|
|
256
314
|
spinner.start('压缩中');
|
|
257
315
|
try {
|
|
258
|
-
|
|
316
|
+
// 空 handlers:不打印、不外显流式;tools=[] 不带工具表——摘要纯文本任务,
|
|
317
|
+
// 全量工具 schema 白占几千 token 窗口,还诱导幻觉工具调用。
|
|
318
|
+
const r = await chat([sysMsg, userMsg], {}, undefined, []);
|
|
259
319
|
// 推理模型可能只返 reasoning_content(content 为 null),或幻觉出 tool_calls → 视为失败
|
|
260
320
|
if (r.toolCalls.length > 0 || !r.content)
|
|
261
321
|
return null;
|
|
262
|
-
return r.content;
|
|
322
|
+
return capSummaryOutput(r.content);
|
|
263
323
|
}
|
|
264
324
|
finally {
|
|
265
325
|
spinner.stop();
|
|
@@ -277,7 +337,12 @@ export async function compactHistory(history, opts) {
|
|
|
277
337
|
state.lastEstimate = estimateBefore;
|
|
278
338
|
const groups = groupFromEnd(history);
|
|
279
339
|
// 保近期:按策略中的 token 比例累积(至少保 1 组),永不劈开 group。
|
|
280
|
-
|
|
340
|
+
// force(手动 /compact 默认、硬闸触发)用更激进的保留比例;再加绝对上限,
|
|
341
|
+
// 防大窗口(如 256k)下保留区按比例仍过大——压缩目标 = 摘要 + 最小续工上下文。
|
|
342
|
+
const keepRatio = opts.force
|
|
343
|
+
? DEFAULT_BUDGET_POLICY.compactForceKeepRatio
|
|
344
|
+
: DEFAULT_BUDGET_POLICY.compactKeepRatio;
|
|
345
|
+
const keepBudget = Math.min(Math.floor(opts.window * keepRatio), DEFAULT_BUDGET_POLICY.compactKeepMaxTokens);
|
|
281
346
|
const kept = [];
|
|
282
347
|
let keptTokens = 0;
|
|
283
348
|
for (let k = groups.length - 1; k >= 0; k--) {
|
|
@@ -292,10 +357,22 @@ export async function compactHistory(history, opts) {
|
|
|
292
357
|
let oldGroups = groups.slice(0, groups.length - kept.length);
|
|
293
358
|
// force(硬闸/手动强压):保护区不豁免——常规切分无旧区时只保最后一组,
|
|
294
359
|
// 其余全部进可压区(首轮/当前轮也一样)。仍按 group 边界切,不破坏 tool_call 配对。
|
|
360
|
+
// **必须保留最早 user 所在 group**:LLM API(OpenAI / Anthropic)要求 messages 至少
|
|
361
|
+
// 含一条非空 user 消息,否则 400。force 旧实现把所有 user 丢进摘要 → 重建后 history
|
|
362
|
+
// 无 user → 下一轮 chat() 被后端拒绝。保最早 user(而非最后一个)因为它是最原始的
|
|
363
|
+
// 请求上下文,摘要器已覆盖后续交互。
|
|
295
364
|
if (oldGroups.length === 0 && opts.force && groups.length >= 2) {
|
|
296
365
|
kept.length = 0;
|
|
297
|
-
|
|
298
|
-
|
|
366
|
+
const lastIdx = groups.length - 1;
|
|
367
|
+
const firstUserIdx = groups.findIndex((g) => g.assistant?.role === 'user');
|
|
368
|
+
if (firstUserIdx >= 0 && firstUserIdx !== lastIdx) {
|
|
369
|
+
kept.push(groups[firstUserIdx], groups[lastIdx]);
|
|
370
|
+
oldGroups = groups.filter((_, i) => i !== firstUserIdx && i !== lastIdx);
|
|
371
|
+
}
|
|
372
|
+
else {
|
|
373
|
+
kept.push(groups[lastIdx]);
|
|
374
|
+
oldGroups = groups.slice(0, groups.length - 1);
|
|
375
|
+
}
|
|
299
376
|
}
|
|
300
377
|
const noop = {
|
|
301
378
|
compacted: false,
|
|
@@ -319,6 +396,8 @@ export async function compactHistory(history, opts) {
|
|
|
319
396
|
const estimateAfter = estimatePromptTokens(history, activeTools, state.correction);
|
|
320
397
|
state.lastEstimate = estimateAfter;
|
|
321
398
|
state.lastUsage = undefined;
|
|
399
|
+
if (!layout.isLastContentRowBlank())
|
|
400
|
+
layout.contentWrite('\n');
|
|
322
401
|
layout.contentWrite(` ${ui.bold}${ui.accent}●${ui.reset} ${ui.accent}强制微压缩(单组)${ui.reset} ${ui.dim}${estimateBefore} → ${estimateAfter} tokens${ui.reset}\n`);
|
|
323
402
|
return {
|
|
324
403
|
compacted: true,
|
|
@@ -339,23 +418,17 @@ export async function compactHistory(history, opts) {
|
|
|
339
418
|
}
|
|
340
419
|
// history 有内容但全在保护区(系统 + 当前轮)
|
|
341
420
|
if (estimateBefore >= opts.threshold * opts.window) {
|
|
421
|
+
if (!layout.isLastContentRowBlank())
|
|
422
|
+
layout.contentWrite('\n');
|
|
342
423
|
layout.contentWrite(` ${ui.yellow}●${ui.reset} ${ui.yellow}上下文已超阈但无可压缩项(全在保护区),建议 /clear 或缩短输入。${ui.reset}\n`);
|
|
343
424
|
return { ...noop, reason: 'noop-shrunk-too-large', protectedRatio };
|
|
344
425
|
}
|
|
345
426
|
return { ...noop, reason: 'noop-protected', protectedRatio };
|
|
346
427
|
}
|
|
347
|
-
//
|
|
348
|
-
//
|
|
349
|
-
//
|
|
350
|
-
//
|
|
351
|
-
// 替换为 "<N 字符,已省略>"(保 path + JSON 合法,见 stubToolCallArguments);
|
|
352
|
-
// ③ 旧 assistant 正文 content(模型长解释,回看价值低)。
|
|
353
|
-
let microcompactDone = false;
|
|
354
|
-
for (const g of oldGroups) {
|
|
355
|
-
if (microcompactGroup(g))
|
|
356
|
-
microcompactDone = true;
|
|
357
|
-
}
|
|
358
|
-
// 第二层:摘要——把旧区(微压缩后)压成一条 system 摘要
|
|
428
|
+
// 主层:摘要——把旧区【原始内容】压成一条 system 摘要。
|
|
429
|
+
// 触发压缩时旧区约占窗口 50-60%,摘要请求与摘要器共享窗口,逐条封顶后即装得下;
|
|
430
|
+
// 不在这里预跑微压缩——预截断会让摘要模型看到 600 字符残片,等于弄瞎它,
|
|
431
|
+
// 且摘要成功后旧区整体丢弃,预截断本身也无收益。
|
|
359
432
|
const older = flattenGroups(oldGroups);
|
|
360
433
|
const summarizeFn = opts.summarize ?? defaultSummarize;
|
|
361
434
|
let summary = null;
|
|
@@ -363,7 +436,7 @@ export async function compactHistory(history, opts) {
|
|
|
363
436
|
summary = await summarizeFn(older, opts.focus);
|
|
364
437
|
}
|
|
365
438
|
catch {
|
|
366
|
-
summary = null; // 摘要失败 →
|
|
439
|
+
summary = null; // 摘要失败 → 回退微压缩兜底,不崩
|
|
367
440
|
}
|
|
368
441
|
if (summary) {
|
|
369
442
|
const artifactRefs = collectArtifactRefs(older);
|
|
@@ -383,6 +456,10 @@ export async function compactHistory(history, opts) {
|
|
|
383
456
|
const estimateAfter = estimatePromptTokens(history, activeTools, state.correction);
|
|
384
457
|
state.lastEstimate = estimateAfter;
|
|
385
458
|
state.lastUsage = undefined; // 压缩后旧 usage 失效,/context 改用校正估算
|
|
459
|
+
// 压缩行与上一个工具批次摘要行之间补空行分隔(compact 在 core step 循环顶部触发,
|
|
460
|
+
// 上一步的 batch 可能尚未 flush,缓冲末行仍是 ● 工具摘要行 → 两行黏在一起)。
|
|
461
|
+
if (!layout.isLastContentRowBlank())
|
|
462
|
+
layout.contentWrite('\n');
|
|
386
463
|
layout.contentWrite(` ${ui.bold}${ui.accent}●${ui.reset} ${ui.accent}压缩上下文${ui.reset} ${ui.dim}${estimateBefore} → ${estimateAfter} tokens${ui.reset}\n`);
|
|
387
464
|
// 抖动保护:压缩后仍超阈 → 提示 /clear,不死循环
|
|
388
465
|
if (estimateAfter >= opts.threshold * opts.window) {
|
|
@@ -397,11 +474,21 @@ export async function compactHistory(history, opts) {
|
|
|
397
474
|
reason: 'summarize',
|
|
398
475
|
};
|
|
399
476
|
}
|
|
400
|
-
//
|
|
477
|
+
// 摘要失败兜底:微压缩——旧区原地截短(保 tool_call_id,无 LLM 调用),结构不动。
|
|
478
|
+
// 覆盖三类大字段,均只裁模型/工具产物,不动 user 原话与 system:
|
|
479
|
+
// ① tool 结果 content;② 旧 assistant 的 tool_calls.arguments(provenance stub,
|
|
480
|
+
// 整体超长才进,大字段替换为 "<N 字符,已省略>",保 path + JSON 合法);③ 旧 assistant 正文。
|
|
481
|
+
let microcompactDone = false;
|
|
482
|
+
for (const g of oldGroups) {
|
|
483
|
+
if (microcompactGroup(g))
|
|
484
|
+
microcompactDone = true;
|
|
485
|
+
}
|
|
401
486
|
const estimateAfter = estimatePromptTokens(history, activeTools, state.correction);
|
|
402
487
|
state.lastEstimate = estimateAfter;
|
|
403
488
|
state.lastUsage = undefined; // token 数已变,旧 usage 失效
|
|
404
489
|
if (microcompactDone) {
|
|
490
|
+
if (!layout.isLastContentRowBlank())
|
|
491
|
+
layout.contentWrite('\n');
|
|
405
492
|
layout.contentWrite(` ${ui.bold}${ui.accent}●${ui.reset} ${ui.accent}微压缩旧工具结果${ui.reset} ${ui.dim}${estimateBefore} → ${estimateAfter} tokens${ui.reset}\n`);
|
|
406
493
|
return {
|
|
407
494
|
compacted: true,
|
package/dist/session/persist.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { existsSync, mkdirSync, readFileSync, readdirSync, unlinkSync, writeFileSync, } from 'node:fs';
|
|
2
2
|
import path from 'node:path';
|
|
3
|
-
import { config } from '../config/index.js';
|
|
3
|
+
import { config, getActiveModel } from '../config/index.js';
|
|
4
4
|
import { truncateDisplay } from '../ui/render.js';
|
|
5
5
|
/** 会话目录(确保存在)。 */
|
|
6
6
|
export function sessionDir() {
|
|
@@ -49,7 +49,7 @@ export function saveSession(history, id, queryHistory = []) {
|
|
|
49
49
|
const meta = {
|
|
50
50
|
id,
|
|
51
51
|
createdAt: idToIso(id),
|
|
52
|
-
model:
|
|
52
|
+
model: getActiveModel(),
|
|
53
53
|
firstUser: history.length > 1
|
|
54
54
|
? firstUserOf(history)
|
|
55
55
|
: truncateDisplay((queryHistory[0] ?? '').replace(/\n/g, ' ').trim(), 40),
|
|
@@ -25,13 +25,13 @@ function emptyPressure(report) {
|
|
|
25
25
|
}
|
|
26
26
|
/** One scheduler instance is owned by one agent run. */
|
|
27
27
|
export function createBudgetScheduler(state = contextState) {
|
|
28
|
-
const evaluate = (history, step, activeTools) => evaluateBudget(history, config.contextWindowTokens, step, state.correction, activeTools);
|
|
28
|
+
const evaluate = (history, step, activeTools, ephemeralTokens) => evaluateBudget(history, config.contextWindowTokens, step, state.correction, activeTools, ephemeralTokens);
|
|
29
29
|
const scheduler = {
|
|
30
30
|
lastRunLog: null,
|
|
31
|
-
async runStep(history, step, activeTools = chatTools) {
|
|
31
|
+
async runStep(history, step, activeTools = chatTools, ephemeralTokens = 0) {
|
|
32
32
|
// External file changes and mutations only update artifact metadata here.
|
|
33
33
|
refreshArtifactFreshness(state, history);
|
|
34
|
-
const report = evaluate(history, step, activeTools);
|
|
34
|
+
const report = evaluate(history, step, activeTools, ephemeralTokens);
|
|
35
35
|
const pressure = emptyPressure(report);
|
|
36
36
|
pressure.triggered = atPressure(report);
|
|
37
37
|
if (pressure.triggered) {
|
|
@@ -46,7 +46,7 @@ export function createBudgetScheduler(state = contextState) {
|
|
|
46
46
|
const ageAware = createAgeAwareEncodingState(history);
|
|
47
47
|
pressure.encodedLogsAndSearches = ageAware.sweepPressure(history, report.hotBoundary);
|
|
48
48
|
}
|
|
49
|
-
pressure.after = evaluate(history, step, activeTools).total;
|
|
49
|
+
pressure.after = evaluate(history, step, activeTools, ephemeralTokens).total;
|
|
50
50
|
}
|
|
51
51
|
// Use the trigger report intentionally: cleanup may reduce the current estimate,
|
|
52
52
|
// but crossing 80% commits this step to compacting for maximum token savings.
|
|
@@ -0,0 +1,345 @@
|
|
|
1
|
+
// skill 触发评测引擎(自进化 Phase 1a)。
|
|
2
|
+
//
|
|
3
|
+
// 设计(docs/skill-self-evolution-research.md Part 3 Phase 1):
|
|
4
|
+
// - 评测对象是 skill 的 description(触发器),不是正文——触发不准是最常见痛点。
|
|
5
|
+
// - 载体:<skill-dir>/evals/trigger.json = [{ "query": "...", "should_trigger": true|false }, ...]
|
|
6
|
+
// - 判定:mocode 没有 Claude Code 的原生 Skill 触发事件,等价信号 = 该 query 的单轮运行里
|
|
7
|
+
// 模型是否调用了 use_skill(name) / run_skill(name)。用 onToolOutcome hook 判定(拿完整 args)。
|
|
8
|
+
// - 隔离:系统提示只含该 skill 的 L0 行(name + description),工具表只给这两个工具,
|
|
9
|
+
// 排除其他 skill / 工具干扰。保真度折扣(无历史上下文)在报告里如实标注(见调研 Q1)。
|
|
10
|
+
// - 噪声处理(调研共识):runs-per-query 多次运行取触发率,禁止单次跑分定生死。
|
|
11
|
+
//
|
|
12
|
+
// 依赖 runAgentCore:与 evals/coding/runner.ts 同构(隔离状态 + 临时沙箱 + 限步 + 超时)。
|
|
13
|
+
// 纯函数(splitTriggerSet / scoreTriggerResults / applyDescription / parseTriggerEvalSet)
|
|
14
|
+
// 与执行解耦,可单测。
|
|
15
|
+
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
|
16
|
+
import { tmpdir } from 'node:os';
|
|
17
|
+
import path from 'node:path';
|
|
18
|
+
import { runAgentCore } from '../agent/core.js';
|
|
19
|
+
import { config } from '../config/index.js';
|
|
20
|
+
import { beginTurn, resetState } from '../rollback/index.js';
|
|
21
|
+
import { setSandboxRoot } from '../sandbox/root.js';
|
|
22
|
+
import { createContextState } from '../session/compact.js';
|
|
23
|
+
import { tools } from '../tools/registry.js';
|
|
24
|
+
import { findSkill, buildSkillSectionFor } from './index.js';
|
|
25
|
+
/** eval 集路径(与 skill 同目录,随 skill 分发;trust hash 不覆盖它——evals 不是执行面)。 */
|
|
26
|
+
export function triggerEvalPath(skill) {
|
|
27
|
+
return path.join(skill.dir, 'evals', 'trigger.json');
|
|
28
|
+
}
|
|
29
|
+
/** 解析并校验 eval 集;文件不存在返 null;格式错误抛 Error(带诊断信息)。 */
|
|
30
|
+
export function parseTriggerEvalSet(skill) {
|
|
31
|
+
const p = triggerEvalPath(skill);
|
|
32
|
+
if (!existsSync(p))
|
|
33
|
+
return null;
|
|
34
|
+
let raw;
|
|
35
|
+
try {
|
|
36
|
+
raw = JSON.parse(readFileSync(p, 'utf8'));
|
|
37
|
+
}
|
|
38
|
+
catch (e) {
|
|
39
|
+
throw new Error(`evals/trigger.json 不是合法 JSON: ${e instanceof Error ? e.message : String(e)}`);
|
|
40
|
+
}
|
|
41
|
+
if (!Array.isArray(raw) || raw.length === 0) {
|
|
42
|
+
throw new Error('evals/trigger.json 必须是非空数组: [{ "query": "...", "should_trigger": true|false }, ...]');
|
|
43
|
+
}
|
|
44
|
+
const out = [];
|
|
45
|
+
for (let i = 0; i < raw.length; i++) {
|
|
46
|
+
const item = raw[i];
|
|
47
|
+
if (typeof item?.query !== 'string' || !item.query.trim()) {
|
|
48
|
+
throw new Error(`evals/trigger.json 第 ${i + 1} 项缺 "query"(非空字符串)`);
|
|
49
|
+
}
|
|
50
|
+
if (typeof item?.should_trigger !== 'boolean') {
|
|
51
|
+
throw new Error(`evals/trigger.json 第 ${i + 1} 项缺 "should_trigger"(true/false)`);
|
|
52
|
+
}
|
|
53
|
+
out.push({ query: item.query, should_trigger: item.should_trigger });
|
|
54
|
+
}
|
|
55
|
+
return out;
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* 分层切 train/holdout(按 should_trigger 分组各切,防过拟合)。
|
|
59
|
+
* holdout ∈ (0,1);0 表示禁用(全 train)。seeded Fisher-Yates,结果可复现。
|
|
60
|
+
*/
|
|
61
|
+
export function splitTriggerSet(evalSet, holdout, seed = 42) {
|
|
62
|
+
if (holdout <= 0)
|
|
63
|
+
return { train: evalSet, holdout: [] };
|
|
64
|
+
const frac = Math.min(Math.max(holdout, 0), 0.9);
|
|
65
|
+
const trigger = shuffleSeeded(evalSet.filter((e) => e.should_trigger), seed);
|
|
66
|
+
const noTrigger = shuffleSeeded(evalSet.filter((e) => !e.should_trigger), seed + 1);
|
|
67
|
+
const nT = Math.max(1, Math.round(trigger.length * frac));
|
|
68
|
+
const nN = Math.max(1, Math.round(noTrigger.length * frac));
|
|
69
|
+
return {
|
|
70
|
+
train: [
|
|
71
|
+
...trigger.slice(nT, trigger.length),
|
|
72
|
+
...noTrigger.slice(nN, noTrigger.length),
|
|
73
|
+
],
|
|
74
|
+
holdout: [...trigger.slice(0, nT), ...noTrigger.slice(0, nN)],
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
function shuffleSeeded(arr, seed) {
|
|
78
|
+
const a = [...arr];
|
|
79
|
+
let s = seed >>> 0;
|
|
80
|
+
const rand = () => {
|
|
81
|
+
// mulberry32:小而确定,跨平台一致
|
|
82
|
+
s = (s + 0x6d2b79f5) >>> 0;
|
|
83
|
+
let t = s;
|
|
84
|
+
t = Math.imul(t ^ (t >>> 15), t | 1);
|
|
85
|
+
t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
|
|
86
|
+
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
|
87
|
+
};
|
|
88
|
+
for (let i = a.length - 1; i > 0; i--) {
|
|
89
|
+
const j = Math.floor(rand() * (i + 1));
|
|
90
|
+
const tmp = a[i];
|
|
91
|
+
a[i] = a[j];
|
|
92
|
+
a[j] = tmp;
|
|
93
|
+
}
|
|
94
|
+
return a;
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* 纯评分:每 query 触发率 vs 阈值。should_trigger 需 rate ≥ threshold;
|
|
98
|
+
* 否则需 rate < threshold(边界 < 而非 ≤:恰在阈值上视为误触发,保守)。
|
|
99
|
+
*/
|
|
100
|
+
export function scoreTriggerResults(evalSet, runResults, threshold) {
|
|
101
|
+
return evalSet.map((e) => {
|
|
102
|
+
const triggers = runResults.get(e.query) ?? [];
|
|
103
|
+
const rate = triggers.length ? triggers.filter(Boolean).length / triggers.length : 0;
|
|
104
|
+
const pass = e.should_trigger ? rate >= threshold : rate < threshold;
|
|
105
|
+
return {
|
|
106
|
+
query: e.query,
|
|
107
|
+
should_trigger: e.should_trigger,
|
|
108
|
+
triggers: triggers.filter(Boolean).length,
|
|
109
|
+
runs: triggers.length,
|
|
110
|
+
trigger_rate: rate,
|
|
111
|
+
pass,
|
|
112
|
+
};
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
export function summarizeTriggerResults(results) {
|
|
116
|
+
const passed = results.filter((r) => r.pass).length;
|
|
117
|
+
return { total: results.length, passed, failed: results.length - passed, passRate: results.length ? passed / results.length : 0 };
|
|
118
|
+
}
|
|
119
|
+
/** 描述长度上限(开放标准:与 when_to_use 合并 1536 字;单字段取同值,保守)。 */
|
|
120
|
+
export const MAX_DESCRIPTION_CHARS = 1536;
|
|
121
|
+
/**
|
|
122
|
+
* 把新 description 写进 SKILL.md 的 frontmatter(纯函数,不落盘)。
|
|
123
|
+
* 支持:标量 `key: value` / 引号标量 / 块标量 `key: |`(多行)三种现状形态,
|
|
124
|
+
* 统一替换为单行标量(前提:新值不含换行与 `: `,由调用方校验)。
|
|
125
|
+
* 找不到 frontmatter 或 description 键 → 返 null(调用方生成诊断)。
|
|
126
|
+
*/
|
|
127
|
+
export function applyDescription(content, newDescription) {
|
|
128
|
+
const lines = content.replace(/\r\n/g, '\n').split('\n');
|
|
129
|
+
let i = 0;
|
|
130
|
+
while (i < lines.length && lines[i].trim() === '')
|
|
131
|
+
i++;
|
|
132
|
+
if (i >= lines.length || lines[i].trim() !== '---')
|
|
133
|
+
return null;
|
|
134
|
+
const start = i;
|
|
135
|
+
i++;
|
|
136
|
+
let end = -1;
|
|
137
|
+
let descLine = -1;
|
|
138
|
+
let blockEnd = -1;
|
|
139
|
+
while (i < lines.length) {
|
|
140
|
+
if (lines[i].trim() === '---') {
|
|
141
|
+
end = i;
|
|
142
|
+
break;
|
|
143
|
+
}
|
|
144
|
+
const trimmed = lines[i].trim();
|
|
145
|
+
if (trimmed.startsWith('description:') && descLine === -1) {
|
|
146
|
+
descLine = i;
|
|
147
|
+
const value = lines[i].slice(lines[i].indexOf(':') + 1).trim();
|
|
148
|
+
if (value === '|' || value === '|-' || value === '>' || value === '>-') {
|
|
149
|
+
let j = i + 1;
|
|
150
|
+
while (j < lines.length && (lines[j].trim() === '' || /^\s+/.test(lines[j])) && lines[j].trim() !== '---') {
|
|
151
|
+
j++;
|
|
152
|
+
}
|
|
153
|
+
blockEnd = j; // 块标量消费到块结束(不含)
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
i++;
|
|
157
|
+
}
|
|
158
|
+
if (end === -1 || descLine === -1)
|
|
159
|
+
return null;
|
|
160
|
+
const next = lines.slice();
|
|
161
|
+
next[descLine] = `description: ${newDescription}`;
|
|
162
|
+
// 原为块标量:删除被消费的块行(倒序删避免位移)
|
|
163
|
+
if (blockEnd !== -1) {
|
|
164
|
+
next.splice(descLine + 1, blockEnd - (descLine + 1));
|
|
165
|
+
}
|
|
166
|
+
void start;
|
|
167
|
+
return next.join('\n');
|
|
168
|
+
}
|
|
169
|
+
// ── 执行:单 query 触发探测 + 批量评测 ─────────────────────────────────────
|
|
170
|
+
const TRIGGER_SESSION_ID = 'skill-trigger-eval';
|
|
171
|
+
/** 隔离评测系统提示:身份 + 该 skill 的唯一 L0 行。确定性模板(不用 t(),保证跨语言可比)。 */
|
|
172
|
+
function buildHarnessPrompt(skill, description) {
|
|
173
|
+
const section = buildSkillSectionFor([
|
|
174
|
+
{ ...skill, description, modelInvocable: true },
|
|
175
|
+
]);
|
|
176
|
+
return ('You are mocode, a terminal coding agent.\n' +
|
|
177
|
+
'A user request follows. Decide whether the single skill below is relevant to it.\n' +
|
|
178
|
+
'- If it is relevant, you MUST call use_skill with exactly this skill name, then stop and briefly say the skill instructions are loaded.\n' +
|
|
179
|
+
'- If it is not relevant, answer the request normally with plain text; do NOT call any tool.\n' +
|
|
180
|
+
'- Do not use any tool other than use_skill / run_skill.\n' +
|
|
181
|
+
section);
|
|
182
|
+
}
|
|
183
|
+
/**
|
|
184
|
+
* 单 query 跑一轮:返回是否触发(该轮内出现 use_skill/run_skill 且 name 匹配)。
|
|
185
|
+
* 隔离:临时沙箱根、禁权限、独立 contextState、限步、超时 abort。状态在 finally 复原。
|
|
186
|
+
* fork skill 归一为 inline 形态评测:harness 只测「模型据 description 选不选这个 skill」,
|
|
187
|
+
* 与 use_skill 返回引导语还是正文无关,归一保证评测语义一致且判定路径唯一。
|
|
188
|
+
*/
|
|
189
|
+
async function runSingleQuery(skill, description, query, opts) {
|
|
190
|
+
const root = mkdtempSync(path.join(tmpdir(), `mocode-skill-eval-${skill.name}-`));
|
|
191
|
+
const previousCwd = process.cwd();
|
|
192
|
+
const previousRoot = setSandboxRoot(root);
|
|
193
|
+
const previousPermission = config.permissionEnabled;
|
|
194
|
+
const previousEvalFlag = process.env.MOCODE_SKILL_EVAL;
|
|
195
|
+
process.env.MOCODE_SKILL_EVAL = '1'; // 评测内的人工调用不记使用台账(见 stats.ts)
|
|
196
|
+
config.permissionEnabled = false; // 隔离评测:run_skill 等 confirm 工具不弹面板(非 TTY fail-closed 会全拒)
|
|
197
|
+
resetState();
|
|
198
|
+
const turnId = beginTurn(query);
|
|
199
|
+
const controller = new AbortController();
|
|
200
|
+
const timer = setTimeout(() => controller.abort(), opts.timeoutMs ?? 60_000);
|
|
201
|
+
// 外部中断透传(AbortSignal.any 需 Node ≥20,这里手动桥接,兼容 engines>=18)
|
|
202
|
+
const externalAbort = () => controller.abort();
|
|
203
|
+
if (opts.signal) {
|
|
204
|
+
if (opts.signal.aborted)
|
|
205
|
+
controller.abort();
|
|
206
|
+
else
|
|
207
|
+
opts.signal.addEventListener('abort', externalAbort, { once: true });
|
|
208
|
+
}
|
|
209
|
+
let triggered = false;
|
|
210
|
+
try {
|
|
211
|
+
const systemPrompt = buildHarnessPrompt({ ...skill, context: 'inline' }, description);
|
|
212
|
+
const history = [{ role: 'system', content: systemPrompt }];
|
|
213
|
+
const isTriggerCall = (tool, args) => (tool === 'use_skill' || tool === 'run_skill') &&
|
|
214
|
+
String(args?.name ?? '').trim() === skill.name;
|
|
215
|
+
await runAgentCore({
|
|
216
|
+
history,
|
|
217
|
+
userInput: query,
|
|
218
|
+
signal: controller.signal,
|
|
219
|
+
hooks: {},
|
|
220
|
+
maxSteps: opts.maxSteps ?? 4,
|
|
221
|
+
contextState: createContextState(),
|
|
222
|
+
// 工具面收窄到两个 skill 入口:排除其余工具对触发的干扰,也避免误触发的副作用面。
|
|
223
|
+
toolsOverride: tools
|
|
224
|
+
.filter((t) => t.name === 'use_skill' || t.name === 'run_skill')
|
|
225
|
+
.map((t) => ({
|
|
226
|
+
type: 'function',
|
|
227
|
+
function: {
|
|
228
|
+
name: t.name,
|
|
229
|
+
description: t.description,
|
|
230
|
+
parameters: t.parameters,
|
|
231
|
+
},
|
|
232
|
+
})),
|
|
233
|
+
onTrace: () => { },
|
|
234
|
+
onToolOutcome: (tool, args) => {
|
|
235
|
+
if (isTriggerCall(tool, args)) {
|
|
236
|
+
triggered = true;
|
|
237
|
+
// 触发已判定:立即终止,不再跑后续步(省 token;工具执行本身被 abort 取消)。
|
|
238
|
+
controller.abort();
|
|
239
|
+
}
|
|
240
|
+
},
|
|
241
|
+
traceContext: { sessionId: TRIGGER_SESSION_ID, turnId },
|
|
242
|
+
suppressOpeningAnalysis: true,
|
|
243
|
+
});
|
|
244
|
+
}
|
|
245
|
+
catch (e) {
|
|
246
|
+
// abort(超时/外部中断/触发后早停)或 LLM 失败:未触发即未触发,不改变判定。
|
|
247
|
+
void e;
|
|
248
|
+
}
|
|
249
|
+
finally {
|
|
250
|
+
clearTimeout(timer);
|
|
251
|
+
opts.signal?.removeEventListener('abort', externalAbort);
|
|
252
|
+
process.chdir(previousCwd);
|
|
253
|
+
setSandboxRoot(previousRoot);
|
|
254
|
+
if (previousEvalFlag === undefined)
|
|
255
|
+
delete process.env.MOCODE_SKILL_EVAL;
|
|
256
|
+
else
|
|
257
|
+
process.env.MOCODE_SKILL_EVAL = previousEvalFlag;
|
|
258
|
+
config.permissionEnabled = previousPermission;
|
|
259
|
+
resetState();
|
|
260
|
+
rmSync(root, { recursive: true, force: true });
|
|
261
|
+
}
|
|
262
|
+
return triggered;
|
|
263
|
+
}
|
|
264
|
+
/**
|
|
265
|
+
* 对 eval 集批量评测一个 description:每条 query 跑 runsPerQuery 次(串行;
|
|
266
|
+
* 触发评测单轮短小,并行收益小且会放大 LLM 限流面),输出 TriggerReport。
|
|
267
|
+
*/
|
|
268
|
+
export async function runTriggerEval(skill, description, evalSet, runsPerQuery, threshold, opts = {}) {
|
|
269
|
+
const runResults = new Map();
|
|
270
|
+
let done = 0;
|
|
271
|
+
const total = evalSet.length * runsPerQuery;
|
|
272
|
+
for (const e of evalSet) {
|
|
273
|
+
const bools = [];
|
|
274
|
+
for (let r = 0; r < runsPerQuery; r++) {
|
|
275
|
+
if (opts.signal?.aborted)
|
|
276
|
+
throw new Error('评测被中断');
|
|
277
|
+
const hit = await runSingleQuery(skill, description, e.query, opts);
|
|
278
|
+
bools.push(hit ? 1 : 0);
|
|
279
|
+
done++;
|
|
280
|
+
opts.onProgress?.(done, total, ` [${done}/${total}] ${hit ? '✓ 触发' : '· 未触发'} ${e.query.slice(0, 60)}`);
|
|
281
|
+
}
|
|
282
|
+
runResults.set(e.query, bools);
|
|
283
|
+
}
|
|
284
|
+
const results = scoreTriggerResults(evalSet, runResults, threshold);
|
|
285
|
+
return { skill: skill.name, description, results, summary: summarizeTriggerResults(results) };
|
|
286
|
+
}
|
|
287
|
+
// ── CLI 输出渲染 ───────────────────────────────────────────────────────────
|
|
288
|
+
/** 终端报告(人类可读)。 */
|
|
289
|
+
export function renderTriggerReport(report, extra) {
|
|
290
|
+
const lines = [];
|
|
291
|
+
lines.push(`skill: ${report.skill}`);
|
|
292
|
+
lines.push(`description: ${report.description}`);
|
|
293
|
+
lines.push('');
|
|
294
|
+
for (const r of report.results) {
|
|
295
|
+
const status = r.pass ? 'PASS' : 'FAIL';
|
|
296
|
+
const expect = r.should_trigger ? '期望触发' : '期望不触发';
|
|
297
|
+
lines.push(` [${status}] ${r.triggers}/${r.runs} (${expect}) ${r.query.slice(0, 70)}`);
|
|
298
|
+
}
|
|
299
|
+
const s = report.summary;
|
|
300
|
+
lines.push('');
|
|
301
|
+
lines.push(`汇总: ${s.passed}/${s.total} 通过 (阈值 ${extra.threshold}, 每 query ${extra.runsPerQuery} 次)`);
|
|
302
|
+
lines.push('注: 隔离评测(单轮 + 仅该 skill 的 L0 行),与完整生产上下文的触发保真度存在已知折扣。');
|
|
303
|
+
return lines.join('\n');
|
|
304
|
+
}
|
|
305
|
+
// ── 便捷入口:按 skill 名定位 + 读取 eval 集(带诊断)────────────────────
|
|
306
|
+
export function loadSkillForEval(name) {
|
|
307
|
+
const skill = findSkill(name);
|
|
308
|
+
if (!skill)
|
|
309
|
+
throw new Error(`未找到 skill "${name}"(用 /skills 或 mocode skill eval 的帮助查看列表)`);
|
|
310
|
+
if (skill.dir === 'builtin') {
|
|
311
|
+
throw new Error(`内置 skill "${name}" 没有磁盘载体,无法做触发评测(进化对象仅限 ~/.mocode/skills 与 <cwd>/.mocode/skills)`);
|
|
312
|
+
}
|
|
313
|
+
return skill;
|
|
314
|
+
}
|
|
315
|
+
/** 读 eval 集;不存在时返回 null(调用方按「缺文件」处理:打印模板 + 退出)。 */
|
|
316
|
+
export function loadTriggerEvalSet(skill) {
|
|
317
|
+
return parseTriggerEvalSet(skill);
|
|
318
|
+
}
|
|
319
|
+
/** 生成 trigger.json 模板内容(供 CLI 提示)。 */
|
|
320
|
+
export function triggerEvalTemplate(skill) {
|
|
321
|
+
const cases = [
|
|
322
|
+
{ query: `TODO: 一条应该触发 "${skill.name}" 的真实请求`, should_trigger: true },
|
|
323
|
+
{ query: 'TODO: 一条不应触发的相近请求', should_trigger: false },
|
|
324
|
+
];
|
|
325
|
+
return JSON.stringify(cases, null, 2);
|
|
326
|
+
}
|
|
327
|
+
/** 评测结果落盘目录(<cwd>/.mocode/skill-eval/);返回写入路径。 */
|
|
328
|
+
export function saveTriggerReport(report) {
|
|
329
|
+
const dir = path.join(process.cwd(), '.mocode', 'skill-eval');
|
|
330
|
+
mkdirSync(dir, { recursive: true });
|
|
331
|
+
const stamp = new Date().toISOString().replace(/[:.]/g, '-');
|
|
332
|
+
const p = path.join(dir, `${report.skill}-${stamp}.json`);
|
|
333
|
+
writeFileSync(p, JSON.stringify(report, null, 2), 'utf8');
|
|
334
|
+
return p;
|
|
335
|
+
}
|
|
336
|
+
/** 供 CLI 校验 runsPerQuery / threshold 参数(纯函数,单测覆盖)。 */
|
|
337
|
+
export function validateEvalParams(runsPerQuery, threshold) {
|
|
338
|
+
if (!Number.isInteger(runsPerQuery) || runsPerQuery < 1 || runsPerQuery > 10) {
|
|
339
|
+
return 'runsPerQuery 必须是 1..10 的整数';
|
|
340
|
+
}
|
|
341
|
+
if (!(threshold > 0 && threshold <= 1)) {
|
|
342
|
+
return 'threshold 必须是 (0, 1] 的数';
|
|
343
|
+
}
|
|
344
|
+
return null;
|
|
345
|
+
}
|