mocode-ai 1.2.8 → 1.2.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/core.js +126 -43
- package/dist/config/index.js +31 -6
- package/dist/config/presets.js +36 -1
- package/dist/context/budget.js +0 -15
- package/dist/llm/index.js +33 -0
- package/dist/repl/index.js +57 -13
- package/dist/session/compact.js +237 -13
- package/dist/session/notes.js +63 -0
- package/dist/session/scheduler.js +5 -9
- package/dist/tools/constants.js +7 -0
- package/package.json +1 -1
- package/dist/__trace_manual_test__.js +0 -1
- package/dist/agent/core.d.ts +0 -112
- package/dist/agent/index.d.ts +0 -19
- package/dist/agent/mode.d.ts +0 -29
- package/dist/agent/retry-classifier.js +0 -64
- package/dist/agent/spawn.d.ts +0 -36
- package/dist/attachments/image.d.ts +0 -22
- package/dist/commands/config.d.ts +0 -8
- package/dist/commands/skill.js +0 -230
- package/dist/config/file.d.ts +0 -23
- package/dist/config/index.d.ts +0 -114
- package/dist/config/presets.d.ts +0 -57
- package/dist/context/budget.d.ts +0 -79
- package/dist/context/classifier.d.ts +0 -14
- package/dist/context/encoders/_util.d.ts +0 -11
- package/dist/context/encoders/code.d.ts +0 -2
- package/dist/context/encoders/doc.d.ts +0 -14
- package/dist/context/encoders/graph.d.ts +0 -11
- package/dist/context/encoders/graph.js +0 -81
- package/dist/context/encoders/index.d.ts +0 -2
- package/dist/context/encoders/log.d.ts +0 -2
- package/dist/context/encoders/log.js +0 -2
- package/dist/context/encoders/memory.d.ts +0 -2
- package/dist/context/encoders/passthrough.d.ts +0 -11
- package/dist/context/encoders/search.d.ts +0 -2
- package/dist/context/encoders/summary.d.ts +0 -12
- package/dist/context/encoders/table.d.ts +0 -2
- package/dist/context/encoders/tree.d.ts +0 -2
- package/dist/context/index.d.ts +0 -6
- package/dist/context/lifecycle.d.ts +0 -51
- package/dist/context/pipeline.d.ts +0 -9
- package/dist/context/registry.d.ts +0 -9
- package/dist/context/relevance.d.ts +0 -65
- package/dist/context/types.d.ts +0 -36
- package/dist/foo.js +0 -28
- package/dist/foo.test.js +0 -12
- package/dist/index.d.ts +0 -1
- package/dist/llm/capabilities.d.ts +0 -19
- package/dist/llm/index.d.ts +0 -111
- package/dist/memory/discover.d.ts +0 -15
- package/dist/memory/index.d.ts +0 -13
- package/dist/memory/reflect.d.ts +0 -34
- package/dist/memory/store.d.ts +0 -124
- package/dist/pet/bridge.d.ts +0 -73
- package/dist/pet/protocol.d.ts +0 -83
- package/dist/pet/state.d.ts +0 -28
- package/dist/plan/active.d.ts +0 -23
- package/dist/plan/active.js +0 -45
- package/dist/plan/index.d.ts +0 -13
- package/dist/plan/index.js +0 -33
- package/dist/plan/state.d.ts +0 -2
- package/dist/plan/state.js +0 -11
- package/dist/plan/store.d.ts +0 -78
- package/dist/plan/store.js +0 -429
- package/dist/project-skill/index.js +0 -95
- package/dist/project-skill/initializer.js +0 -178
- package/dist/project-snapshot/index.js +0 -85
- package/dist/project-snapshot/llm-snapshot.js +0 -150
- package/dist/project-snapshot/static-files.js +0 -87
- package/dist/repl/index.d.ts +0 -22
- package/dist/rollback/index.d.ts +0 -79
- package/dist/sandbox/command.d.ts +0 -3
- package/dist/sandbox/index.d.ts +0 -4
- package/dist/sandbox/jail.d.ts +0 -17
- package/dist/sandbox/policy.d.ts +0 -21
- package/dist/sandbox/root.d.ts +0 -5
- package/dist/session/compact.d.ts +0 -92
- package/dist/session/index.d.ts +0 -13
- package/dist/session/notes-plan.js +0 -41
- package/dist/session/persist.d.ts +0 -30
- package/dist/session/scheduler.d.ts +0 -53
- package/dist/skills/discover.d.ts +0 -33
- package/dist/skills/index.d.ts +0 -13
- package/dist/skills/skill-eval.js +0 -345
- package/dist/skills/skill-improve.js +0 -221
- package/dist/skills/stats.js +0 -102
- package/dist/tools/builtins/apply-patch.js +0 -174
- package/dist/tools/builtins/ask-human.d.ts +0 -9
- package/dist/tools/builtins/codegraph.d.ts +0 -2
- package/dist/tools/builtins/codegraph.js +0 -128
- package/dist/tools/builtins/edit-file.d.ts +0 -2
- package/dist/tools/builtins/glob.d.ts +0 -2
- package/dist/tools/builtins/grep.d.ts +0 -2
- package/dist/tools/builtins/index.d.ts +0 -2
- package/dist/tools/builtins/memory-forget.d.ts +0 -2
- package/dist/tools/builtins/memory-list.d.ts +0 -2
- package/dist/tools/builtins/memory-save.d.ts +0 -2
- package/dist/tools/builtins/memory-search.d.ts +0 -2
- package/dist/tools/builtins/memory-update.d.ts +0 -2
- package/dist/tools/builtins/project-skill-update.js +0 -102
- package/dist/tools/builtins/read-file.d.ts +0 -2
- package/dist/tools/builtins/run-command.d.ts +0 -2
- package/dist/tools/builtins/switch-mode.d.ts +0 -2
- package/dist/tools/builtins/switch-mode.js +0 -44
- package/dist/tools/builtins/task.d.ts +0 -2
- package/dist/tools/builtins/todolist.d.ts +0 -2
- package/dist/tools/builtins/todolist.js +0 -411
- package/dist/tools/builtins/use-skill.d.ts +0 -2
- package/dist/tools/builtins/web-fetch.d.ts +0 -2
- package/dist/tools/builtins/web-search.d.ts +0 -2
- package/dist/tools/builtins/write-file.d.ts +0 -2
- package/dist/tools/constants.d.ts +0 -52
- package/dist/tools/registry.d.ts +0 -18
- package/dist/tools/retry.js +0 -137
- package/dist/tools/types.d.ts +0 -42
- package/dist/ui/batch.d.ts +0 -67
- package/dist/ui/clipboard.d.ts +0 -9
- package/dist/ui/content.d.ts +0 -55
- package/dist/ui/diff.d.ts +0 -34
- package/dist/ui/intervention.d.ts +0 -46
- package/dist/ui/layout.d.ts +0 -225
- package/dist/ui/markdown.d.ts +0 -46
- package/dist/ui/mouse.d.ts +0 -29
- package/dist/ui/prompt.d.ts +0 -61
- package/dist/ui/render.d.ts +0 -60
- package/dist/ui/spinner.d.ts +0 -26
- package/dist/ui/theme.d.ts +0 -47
- package/dist/ui/thinking.js +0 -80
|
@@ -1,53 +0,0 @@
|
|
|
1
|
-
import { evaluateBudget, scheduleActions, formatReport, type BudgetReport, type ScheduleAction } from '../context/budget.js';
|
|
2
|
-
import type { ChatMessage } from '../llm/index.js';
|
|
3
|
-
/** 一次调度的执行日志(供 /context 命令显示与调试)。 */
|
|
4
|
-
export interface SchedulerRunLog {
|
|
5
|
-
step: number;
|
|
6
|
-
report: BudgetReport;
|
|
7
|
-
actions: ScheduleAction[];
|
|
8
|
-
/** 实际触发的闸('compact_history' 调过 maybeCompact → true;其它已在 push-time 自动跑)。 */
|
|
9
|
-
compactHistoryCalled: boolean;
|
|
10
|
-
ts: number;
|
|
11
|
-
}
|
|
12
|
-
/** Scheduler 实例状态:压一份最近日志,方便 /context 看到上一次决策。 */
|
|
13
|
-
export interface BudgetScheduler {
|
|
14
|
-
observePush: (history: ChatMessage[], idx: number) => void;
|
|
15
|
-
runStep: (history: ChatMessage[], step: number) => Promise<void>;
|
|
16
|
-
/** 暴露最近一次决策(供 /context)。 */
|
|
17
|
-
lastRunLog: SchedulerRunLog | null;
|
|
18
|
-
}
|
|
19
|
-
/** compact_history action 的执行明细(供 repl 文案展示"为什么没压")。 */
|
|
20
|
-
export interface CompactHistoryDetail {
|
|
21
|
-
reason: 'microcompact' | 'summarize' | 'noop-empty' | 'noop-protected' | 'noop-ml-only' | 'noop-shrunk-too-large' | 'noop-noold-noop';
|
|
22
|
-
estimateBefore: number;
|
|
23
|
-
estimateAfter: number;
|
|
24
|
-
/** 保护区占比 0-1(系统 + 当前轮 / 总 history)。 */
|
|
25
|
-
protectedRatio?: number;
|
|
26
|
-
/** 旧区可压组数。 */
|
|
27
|
-
oldGroupCount?: number;
|
|
28
|
-
/** focus 透传。 */
|
|
29
|
-
focus?: string;
|
|
30
|
-
}
|
|
31
|
-
/** 创建 runAgentCore 闭包持有的 scheduler(每次 agent 启动一个新实例)。
|
|
32
|
-
* observePush 当前只是占位:真正 L1/L2/L3 已由 cap / pruner / lifecycle 在 push 时跑;
|
|
33
|
-
* 保留接口为后续「调度器注入 hotBoundary 给 lifecycle」演进留接缝。 */
|
|
34
|
-
export declare function createBudgetScheduler(): BudgetScheduler;
|
|
35
|
-
/** 便捷:agent/core.ts 不需要每次 createBudgetScheduler,直接 runScheduler(history, step)。 */
|
|
36
|
-
export declare function runScheduler(history: ChatMessage[], step: number): Promise<void>;
|
|
37
|
-
/** 手动 /compact 入口(repl):与自动路径完全一致——五区 ROI 调度,但 history 摘要强制执行。
|
|
38
|
-
* 即便 layers.history.overBudget=false 或 totalOver=false,manual 仍产 compact_history action
|
|
39
|
-
* 把 focus 透传给 LLM 摘要 prompt。其它 ROI 决策(cold tools / cap hot / warn)按 scheduleActions 走。
|
|
40
|
-
*
|
|
41
|
-
* 关系:runScheduler 是「自动触发」,manualCompact 是「用户显式触发」,二者共享 scheduleActions。
|
|
42
|
-
*
|
|
43
|
-
* force=true:即便 oldGroups 空(history 全在保护区)也强行把早期消息降级压一次。
|
|
44
|
-
* 适合"history 太长,自动阈值从未触发,但用户想强制压"的场景。
|
|
45
|
-
*
|
|
46
|
-
* 退化:config.contextBudget === false 时直接调 compactHistory(history, { focus }),与改造前等价。
|
|
47
|
-
* 返回 SchedulerRunLog + compactDetail 字段,供 repl 文案展示"为什么没压"。 */
|
|
48
|
-
export declare function manualCompact(history: ChatMessage[], focus?: string, opts?: {
|
|
49
|
-
force?: boolean;
|
|
50
|
-
}): Promise<SchedulerRunLog & {
|
|
51
|
-
compactDetail?: CompactHistoryDetail;
|
|
52
|
-
}>;
|
|
53
|
-
export { evaluateBudget, scheduleActions, formatReport };
|
|
@@ -1,33 +0,0 @@
|
|
|
1
|
-
export interface Skill {
|
|
2
|
-
name: string;
|
|
3
|
-
description: string;
|
|
4
|
-
version?: string;
|
|
5
|
-
license?: string;
|
|
6
|
-
dir: string;
|
|
7
|
-
skillMdPath: string;
|
|
8
|
-
}
|
|
9
|
-
/**
|
|
10
|
-
* 解析 skill 目录列表。
|
|
11
|
-
* env SKILLS_DIRS 设则覆盖默认(用 path.delimiter 切分,与 PATH 同语义:
|
|
12
|
-
* win32 用 ';',POSIX 用 ':',规避盘符冒号问题);未设则默认三目录,
|
|
13
|
-
* 按优先级升序(低→高):~/.claude/skills → ~/.mocode/skills → <cwd>/.mocode/skills。
|
|
14
|
-
*/
|
|
15
|
-
export declare function resolveSkillsDirs(): string[];
|
|
16
|
-
/**
|
|
17
|
-
* 极简 YAML frontmatter 解析(不引入 yaml 依赖)。
|
|
18
|
-
* 仅支持单行键值:按首个冒号切 key/value,trim,去首尾配对引号。
|
|
19
|
-
* 不支持块标量(| / >)与多行值——v1 限制(skill 的 description 实际多为单行)。
|
|
20
|
-
*
|
|
21
|
-
* 返回 { meta, body }:无 frontmatter 时 meta 为空、body 为原文。
|
|
22
|
-
*/
|
|
23
|
-
export declare function parseFrontmatter(content: string): {
|
|
24
|
-
meta: Record<string, string>;
|
|
25
|
-
body: string;
|
|
26
|
-
};
|
|
27
|
-
/**
|
|
28
|
-
* 扫描所有 skill 目录,解析 frontmatter,按目录优先级去重。
|
|
29
|
-
* 同步 fs + 全程静默容错(目录不存在 / 读失败 / 解析失败 → 跳过,不抛),
|
|
30
|
-
* 风格对齐 src/session/persist.ts。按 resolveSkillsDirs 升序遍历,
|
|
31
|
-
* Map.set 后设覆盖先设 → 项目级优先。
|
|
32
|
-
*/
|
|
33
|
-
export declare function discoverSkills(): Skill[];
|
package/dist/skills/index.d.ts
DELETED
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
import { type Skill } from './discover.js';
|
|
2
|
-
export type { Skill };
|
|
3
|
-
/** 已发现的 skill 列表(懒加载,首次调用触发扫描;启动期 repl 调一次)。 */
|
|
4
|
-
export declare function listSkills(): Skill[];
|
|
5
|
-
/**
|
|
6
|
-
* 读取某 skill 的 SKILL.md 正文(去掉 frontmatter)。
|
|
7
|
-
* 纯函数:找不到 / 读失败返 null(错误字符串交给调用方工具层生成)。
|
|
8
|
-
*/
|
|
9
|
-
export declare function getSkillBody(name: string): string | null;
|
|
10
|
-
/** 拼进系统提示的 skill 段;无 skill 返空串(零行为变化)。 */
|
|
11
|
-
export declare function buildSkillsSection(): string;
|
|
12
|
-
/** base 系统提示 + skill 段;无 skill 时 === base。 */
|
|
13
|
-
export declare function effectiveSystemPrompt(base: string): string;
|
|
@@ -1,345 +0,0 @@
|
|
|
1
|
-
// skill 触发评测引擎(自进化 Phase 1a)。
|
|
2
|
-
//
|
|
3
|
-
// 设计(docs/skill-self-evolution-research.md Part 3 Phase 1):
|
|
4
|
-
// - 评测对象是 skill 的 description(触发器),不是正文——触发不准是最常见痛点。
|
|
5
|
-
// - 载体:<skill-dir>/evals/trigger.json = [{ "query": "...", "should_trigger": true|false }, ...]
|
|
6
|
-
// - 判定:mocode 没有 Claude Code 的原生 Skill 触发事件,等价信号 = 该 query 的单轮运行里
|
|
7
|
-
// 模型是否调用了 use_skill(name) / run_skill(name)。用 onToolOutcome hook 判定(拿完整 args)。
|
|
8
|
-
// - 隔离:系统提示只含该 skill 的 L0 行(name + description),工具表只给这两个工具,
|
|
9
|
-
// 排除其他 skill / 工具干扰。保真度折扣(无历史上下文)在报告里如实标注(见调研 Q1)。
|
|
10
|
-
// - 噪声处理(调研共识):runs-per-query 多次运行取触发率,禁止单次跑分定生死。
|
|
11
|
-
//
|
|
12
|
-
// 依赖 runAgentCore:与 evals/coding/runner.ts 同构(隔离状态 + 临时沙箱 + 限步 + 超时)。
|
|
13
|
-
// 纯函数(splitTriggerSet / scoreTriggerResults / applyDescription / parseTriggerEvalSet)
|
|
14
|
-
// 与执行解耦,可单测。
|
|
15
|
-
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
|
16
|
-
import { tmpdir } from 'node:os';
|
|
17
|
-
import path from 'node:path';
|
|
18
|
-
import { runAgentCore } from '../agent/core.js';
|
|
19
|
-
import { config } from '../config/index.js';
|
|
20
|
-
import { beginTurn, resetState } from '../rollback/index.js';
|
|
21
|
-
import { setSandboxRoot } from '../sandbox/root.js';
|
|
22
|
-
import { createContextState } from '../session/compact.js';
|
|
23
|
-
import { tools } from '../tools/registry.js';
|
|
24
|
-
import { findSkill, buildSkillSectionFor } from './index.js';
|
|
25
|
-
/** eval 集路径(与 skill 同目录,随 skill 分发;trust hash 不覆盖它——evals 不是执行面)。 */
|
|
26
|
-
export function triggerEvalPath(skill) {
|
|
27
|
-
return path.join(skill.dir, 'evals', 'trigger.json');
|
|
28
|
-
}
|
|
29
|
-
/** 解析并校验 eval 集;文件不存在返 null;格式错误抛 Error(带诊断信息)。 */
|
|
30
|
-
export function parseTriggerEvalSet(skill) {
|
|
31
|
-
const p = triggerEvalPath(skill);
|
|
32
|
-
if (!existsSync(p))
|
|
33
|
-
return null;
|
|
34
|
-
let raw;
|
|
35
|
-
try {
|
|
36
|
-
raw = JSON.parse(readFileSync(p, 'utf8'));
|
|
37
|
-
}
|
|
38
|
-
catch (e) {
|
|
39
|
-
throw new Error(`evals/trigger.json 不是合法 JSON: ${e instanceof Error ? e.message : String(e)}`);
|
|
40
|
-
}
|
|
41
|
-
if (!Array.isArray(raw) || raw.length === 0) {
|
|
42
|
-
throw new Error('evals/trigger.json 必须是非空数组: [{ "query": "...", "should_trigger": true|false }, ...]');
|
|
43
|
-
}
|
|
44
|
-
const out = [];
|
|
45
|
-
for (let i = 0; i < raw.length; i++) {
|
|
46
|
-
const item = raw[i];
|
|
47
|
-
if (typeof item?.query !== 'string' || !item.query.trim()) {
|
|
48
|
-
throw new Error(`evals/trigger.json 第 ${i + 1} 项缺 "query"(非空字符串)`);
|
|
49
|
-
}
|
|
50
|
-
if (typeof item?.should_trigger !== 'boolean') {
|
|
51
|
-
throw new Error(`evals/trigger.json 第 ${i + 1} 项缺 "should_trigger"(true/false)`);
|
|
52
|
-
}
|
|
53
|
-
out.push({ query: item.query, should_trigger: item.should_trigger });
|
|
54
|
-
}
|
|
55
|
-
return out;
|
|
56
|
-
}
|
|
57
|
-
/**
|
|
58
|
-
* 分层切 train/holdout(按 should_trigger 分组各切,防过拟合)。
|
|
59
|
-
* holdout ∈ (0,1);0 表示禁用(全 train)。seeded Fisher-Yates,结果可复现。
|
|
60
|
-
*/
|
|
61
|
-
export function splitTriggerSet(evalSet, holdout, seed = 42) {
|
|
62
|
-
if (holdout <= 0)
|
|
63
|
-
return { train: evalSet, holdout: [] };
|
|
64
|
-
const frac = Math.min(Math.max(holdout, 0), 0.9);
|
|
65
|
-
const trigger = shuffleSeeded(evalSet.filter((e) => e.should_trigger), seed);
|
|
66
|
-
const noTrigger = shuffleSeeded(evalSet.filter((e) => !e.should_trigger), seed + 1);
|
|
67
|
-
const nT = Math.max(1, Math.round(trigger.length * frac));
|
|
68
|
-
const nN = Math.max(1, Math.round(noTrigger.length * frac));
|
|
69
|
-
return {
|
|
70
|
-
train: [
|
|
71
|
-
...trigger.slice(nT, trigger.length),
|
|
72
|
-
...noTrigger.slice(nN, noTrigger.length),
|
|
73
|
-
],
|
|
74
|
-
holdout: [...trigger.slice(0, nT), ...noTrigger.slice(0, nN)],
|
|
75
|
-
};
|
|
76
|
-
}
|
|
77
|
-
function shuffleSeeded(arr, seed) {
|
|
78
|
-
const a = [...arr];
|
|
79
|
-
let s = seed >>> 0;
|
|
80
|
-
const rand = () => {
|
|
81
|
-
// mulberry32:小而确定,跨平台一致
|
|
82
|
-
s = (s + 0x6d2b79f5) >>> 0;
|
|
83
|
-
let t = s;
|
|
84
|
-
t = Math.imul(t ^ (t >>> 15), t | 1);
|
|
85
|
-
t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
|
|
86
|
-
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
|
87
|
-
};
|
|
88
|
-
for (let i = a.length - 1; i > 0; i--) {
|
|
89
|
-
const j = Math.floor(rand() * (i + 1));
|
|
90
|
-
const tmp = a[i];
|
|
91
|
-
a[i] = a[j];
|
|
92
|
-
a[j] = tmp;
|
|
93
|
-
}
|
|
94
|
-
return a;
|
|
95
|
-
}
|
|
96
|
-
/**
|
|
97
|
-
* 纯评分:每 query 触发率 vs 阈值。should_trigger 需 rate ≥ threshold;
|
|
98
|
-
* 否则需 rate < threshold(边界 < 而非 ≤:恰在阈值上视为误触发,保守)。
|
|
99
|
-
*/
|
|
100
|
-
export function scoreTriggerResults(evalSet, runResults, threshold) {
|
|
101
|
-
return evalSet.map((e) => {
|
|
102
|
-
const triggers = runResults.get(e.query) ?? [];
|
|
103
|
-
const rate = triggers.length ? triggers.filter(Boolean).length / triggers.length : 0;
|
|
104
|
-
const pass = e.should_trigger ? rate >= threshold : rate < threshold;
|
|
105
|
-
return {
|
|
106
|
-
query: e.query,
|
|
107
|
-
should_trigger: e.should_trigger,
|
|
108
|
-
triggers: triggers.filter(Boolean).length,
|
|
109
|
-
runs: triggers.length,
|
|
110
|
-
trigger_rate: rate,
|
|
111
|
-
pass,
|
|
112
|
-
};
|
|
113
|
-
});
|
|
114
|
-
}
|
|
115
|
-
export function summarizeTriggerResults(results) {
|
|
116
|
-
const passed = results.filter((r) => r.pass).length;
|
|
117
|
-
return { total: results.length, passed, failed: results.length - passed, passRate: results.length ? passed / results.length : 0 };
|
|
118
|
-
}
|
|
119
|
-
/** 描述长度上限(开放标准:与 when_to_use 合并 1536 字;单字段取同值,保守)。 */
|
|
120
|
-
export const MAX_DESCRIPTION_CHARS = 1536;
|
|
121
|
-
/**
|
|
122
|
-
* 把新 description 写进 SKILL.md 的 frontmatter(纯函数,不落盘)。
|
|
123
|
-
* 支持:标量 `key: value` / 引号标量 / 块标量 `key: |`(多行)三种现状形态,
|
|
124
|
-
* 统一替换为单行标量(前提:新值不含换行与 `: `,由调用方校验)。
|
|
125
|
-
* 找不到 frontmatter 或 description 键 → 返 null(调用方生成诊断)。
|
|
126
|
-
*/
|
|
127
|
-
export function applyDescription(content, newDescription) {
|
|
128
|
-
const lines = content.replace(/\r\n/g, '\n').split('\n');
|
|
129
|
-
let i = 0;
|
|
130
|
-
while (i < lines.length && lines[i].trim() === '')
|
|
131
|
-
i++;
|
|
132
|
-
if (i >= lines.length || lines[i].trim() !== '---')
|
|
133
|
-
return null;
|
|
134
|
-
const start = i;
|
|
135
|
-
i++;
|
|
136
|
-
let end = -1;
|
|
137
|
-
let descLine = -1;
|
|
138
|
-
let blockEnd = -1;
|
|
139
|
-
while (i < lines.length) {
|
|
140
|
-
if (lines[i].trim() === '---') {
|
|
141
|
-
end = i;
|
|
142
|
-
break;
|
|
143
|
-
}
|
|
144
|
-
const trimmed = lines[i].trim();
|
|
145
|
-
if (trimmed.startsWith('description:') && descLine === -1) {
|
|
146
|
-
descLine = i;
|
|
147
|
-
const value = lines[i].slice(lines[i].indexOf(':') + 1).trim();
|
|
148
|
-
if (value === '|' || value === '|-' || value === '>' || value === '>-') {
|
|
149
|
-
let j = i + 1;
|
|
150
|
-
while (j < lines.length && (lines[j].trim() === '' || /^\s+/.test(lines[j])) && lines[j].trim() !== '---') {
|
|
151
|
-
j++;
|
|
152
|
-
}
|
|
153
|
-
blockEnd = j; // 块标量消费到块结束(不含)
|
|
154
|
-
}
|
|
155
|
-
}
|
|
156
|
-
i++;
|
|
157
|
-
}
|
|
158
|
-
if (end === -1 || descLine === -1)
|
|
159
|
-
return null;
|
|
160
|
-
const next = lines.slice();
|
|
161
|
-
next[descLine] = `description: ${newDescription}`;
|
|
162
|
-
// 原为块标量:删除被消费的块行(倒序删避免位移)
|
|
163
|
-
if (blockEnd !== -1) {
|
|
164
|
-
next.splice(descLine + 1, blockEnd - (descLine + 1));
|
|
165
|
-
}
|
|
166
|
-
void start;
|
|
167
|
-
return next.join('\n');
|
|
168
|
-
}
|
|
169
|
-
// ── 执行:单 query 触发探测 + 批量评测 ─────────────────────────────────────
|
|
170
|
-
const TRIGGER_SESSION_ID = 'skill-trigger-eval';
|
|
171
|
-
/** 隔离评测系统提示:身份 + 该 skill 的唯一 L0 行。确定性模板(不用 t(),保证跨语言可比)。 */
|
|
172
|
-
function buildHarnessPrompt(skill, description) {
|
|
173
|
-
const section = buildSkillSectionFor([
|
|
174
|
-
{ ...skill, description, modelInvocable: true },
|
|
175
|
-
]);
|
|
176
|
-
return ('You are mocode, a terminal coding agent.\n' +
|
|
177
|
-
'A user request follows. Decide whether the single skill below is relevant to it.\n' +
|
|
178
|
-
'- If it is relevant, you MUST call use_skill with exactly this skill name, then stop and briefly say the skill instructions are loaded.\n' +
|
|
179
|
-
'- If it is not relevant, answer the request normally with plain text; do NOT call any tool.\n' +
|
|
180
|
-
'- Do not use any tool other than use_skill / run_skill.\n' +
|
|
181
|
-
section);
|
|
182
|
-
}
|
|
183
|
-
/**
|
|
184
|
-
* 单 query 跑一轮:返回是否触发(该轮内出现 use_skill/run_skill 且 name 匹配)。
|
|
185
|
-
* 隔离:临时沙箱根、禁权限、独立 contextState、限步、超时 abort。状态在 finally 复原。
|
|
186
|
-
* fork skill 归一为 inline 形态评测:harness 只测「模型据 description 选不选这个 skill」,
|
|
187
|
-
* 与 use_skill 返回引导语还是正文无关,归一保证评测语义一致且判定路径唯一。
|
|
188
|
-
*/
|
|
189
|
-
async function runSingleQuery(skill, description, query, opts) {
|
|
190
|
-
const root = mkdtempSync(path.join(tmpdir(), `mocode-skill-eval-${skill.name}-`));
|
|
191
|
-
const previousCwd = process.cwd();
|
|
192
|
-
const previousRoot = setSandboxRoot(root);
|
|
193
|
-
const previousPermission = config.permissionEnabled;
|
|
194
|
-
const previousEvalFlag = process.env.MOCODE_SKILL_EVAL;
|
|
195
|
-
process.env.MOCODE_SKILL_EVAL = '1'; // 评测内的人工调用不记使用台账(见 stats.ts)
|
|
196
|
-
config.permissionEnabled = false; // 隔离评测:run_skill 等 confirm 工具不弹面板(非 TTY fail-closed 会全拒)
|
|
197
|
-
resetState();
|
|
198
|
-
const turnId = beginTurn(query);
|
|
199
|
-
const controller = new AbortController();
|
|
200
|
-
const timer = setTimeout(() => controller.abort(), opts.timeoutMs ?? 60_000);
|
|
201
|
-
// 外部中断透传(AbortSignal.any 需 Node ≥20,这里手动桥接,兼容 engines>=18)
|
|
202
|
-
const externalAbort = () => controller.abort();
|
|
203
|
-
if (opts.signal) {
|
|
204
|
-
if (opts.signal.aborted)
|
|
205
|
-
controller.abort();
|
|
206
|
-
else
|
|
207
|
-
opts.signal.addEventListener('abort', externalAbort, { once: true });
|
|
208
|
-
}
|
|
209
|
-
let triggered = false;
|
|
210
|
-
try {
|
|
211
|
-
const systemPrompt = buildHarnessPrompt({ ...skill, context: 'inline' }, description);
|
|
212
|
-
const history = [{ role: 'system', content: systemPrompt }];
|
|
213
|
-
const isTriggerCall = (tool, args) => (tool === 'use_skill' || tool === 'run_skill') &&
|
|
214
|
-
String(args?.name ?? '').trim() === skill.name;
|
|
215
|
-
await runAgentCore({
|
|
216
|
-
history,
|
|
217
|
-
userInput: query,
|
|
218
|
-
signal: controller.signal,
|
|
219
|
-
hooks: {},
|
|
220
|
-
maxSteps: opts.maxSteps ?? 4,
|
|
221
|
-
contextState: createContextState(),
|
|
222
|
-
// 工具面收窄到两个 skill 入口:排除其余工具对触发的干扰,也避免误触发的副作用面。
|
|
223
|
-
toolsOverride: tools
|
|
224
|
-
.filter((t) => t.name === 'use_skill' || t.name === 'run_skill')
|
|
225
|
-
.map((t) => ({
|
|
226
|
-
type: 'function',
|
|
227
|
-
function: {
|
|
228
|
-
name: t.name,
|
|
229
|
-
description: t.description,
|
|
230
|
-
parameters: t.parameters,
|
|
231
|
-
},
|
|
232
|
-
})),
|
|
233
|
-
onTrace: () => { },
|
|
234
|
-
onToolOutcome: (tool, args) => {
|
|
235
|
-
if (isTriggerCall(tool, args)) {
|
|
236
|
-
triggered = true;
|
|
237
|
-
// 触发已判定:立即终止,不再跑后续步(省 token;工具执行本身被 abort 取消)。
|
|
238
|
-
controller.abort();
|
|
239
|
-
}
|
|
240
|
-
},
|
|
241
|
-
traceContext: { sessionId: TRIGGER_SESSION_ID, turnId },
|
|
242
|
-
suppressOpeningAnalysis: true,
|
|
243
|
-
});
|
|
244
|
-
}
|
|
245
|
-
catch (e) {
|
|
246
|
-
// abort(超时/外部中断/触发后早停)或 LLM 失败:未触发即未触发,不改变判定。
|
|
247
|
-
void e;
|
|
248
|
-
}
|
|
249
|
-
finally {
|
|
250
|
-
clearTimeout(timer);
|
|
251
|
-
opts.signal?.removeEventListener('abort', externalAbort);
|
|
252
|
-
process.chdir(previousCwd);
|
|
253
|
-
setSandboxRoot(previousRoot);
|
|
254
|
-
if (previousEvalFlag === undefined)
|
|
255
|
-
delete process.env.MOCODE_SKILL_EVAL;
|
|
256
|
-
else
|
|
257
|
-
process.env.MOCODE_SKILL_EVAL = previousEvalFlag;
|
|
258
|
-
config.permissionEnabled = previousPermission;
|
|
259
|
-
resetState();
|
|
260
|
-
rmSync(root, { recursive: true, force: true });
|
|
261
|
-
}
|
|
262
|
-
return triggered;
|
|
263
|
-
}
|
|
264
|
-
/**
|
|
265
|
-
* 对 eval 集批量评测一个 description:每条 query 跑 runsPerQuery 次(串行;
|
|
266
|
-
* 触发评测单轮短小,并行收益小且会放大 LLM 限流面),输出 TriggerReport。
|
|
267
|
-
*/
|
|
268
|
-
export async function runTriggerEval(skill, description, evalSet, runsPerQuery, threshold, opts = {}) {
|
|
269
|
-
const runResults = new Map();
|
|
270
|
-
let done = 0;
|
|
271
|
-
const total = evalSet.length * runsPerQuery;
|
|
272
|
-
for (const e of evalSet) {
|
|
273
|
-
const bools = [];
|
|
274
|
-
for (let r = 0; r < runsPerQuery; r++) {
|
|
275
|
-
if (opts.signal?.aborted)
|
|
276
|
-
throw new Error('评测被中断');
|
|
277
|
-
const hit = await runSingleQuery(skill, description, e.query, opts);
|
|
278
|
-
bools.push(hit ? 1 : 0);
|
|
279
|
-
done++;
|
|
280
|
-
opts.onProgress?.(done, total, ` [${done}/${total}] ${hit ? '✓ 触发' : '· 未触发'} ${e.query.slice(0, 60)}`);
|
|
281
|
-
}
|
|
282
|
-
runResults.set(e.query, bools);
|
|
283
|
-
}
|
|
284
|
-
const results = scoreTriggerResults(evalSet, runResults, threshold);
|
|
285
|
-
return { skill: skill.name, description, results, summary: summarizeTriggerResults(results) };
|
|
286
|
-
}
|
|
287
|
-
// ── CLI 输出渲染 ───────────────────────────────────────────────────────────
|
|
288
|
-
/** 终端报告(人类可读)。 */
|
|
289
|
-
export function renderTriggerReport(report, extra) {
|
|
290
|
-
const lines = [];
|
|
291
|
-
lines.push(`skill: ${report.skill}`);
|
|
292
|
-
lines.push(`description: ${report.description}`);
|
|
293
|
-
lines.push('');
|
|
294
|
-
for (const r of report.results) {
|
|
295
|
-
const status = r.pass ? 'PASS' : 'FAIL';
|
|
296
|
-
const expect = r.should_trigger ? '期望触发' : '期望不触发';
|
|
297
|
-
lines.push(` [${status}] ${r.triggers}/${r.runs} (${expect}) ${r.query.slice(0, 70)}`);
|
|
298
|
-
}
|
|
299
|
-
const s = report.summary;
|
|
300
|
-
lines.push('');
|
|
301
|
-
lines.push(`汇总: ${s.passed}/${s.total} 通过 (阈值 ${extra.threshold}, 每 query ${extra.runsPerQuery} 次)`);
|
|
302
|
-
lines.push('注: 隔离评测(单轮 + 仅该 skill 的 L0 行),与完整生产上下文的触发保真度存在已知折扣。');
|
|
303
|
-
return lines.join('\n');
|
|
304
|
-
}
|
|
305
|
-
// ── 便捷入口:按 skill 名定位 + 读取 eval 集(带诊断)────────────────────
|
|
306
|
-
export function loadSkillForEval(name) {
|
|
307
|
-
const skill = findSkill(name);
|
|
308
|
-
if (!skill)
|
|
309
|
-
throw new Error(`未找到 skill "${name}"(用 /skills 或 mocode skill eval 的帮助查看列表)`);
|
|
310
|
-
if (skill.dir === 'builtin') {
|
|
311
|
-
throw new Error(`内置 skill "${name}" 没有磁盘载体,无法做触发评测(进化对象仅限 ~/.mocode/skills 与 <cwd>/.mocode/skills)`);
|
|
312
|
-
}
|
|
313
|
-
return skill;
|
|
314
|
-
}
|
|
315
|
-
/** 读 eval 集;不存在时返回 null(调用方按「缺文件」处理:打印模板 + 退出)。 */
|
|
316
|
-
export function loadTriggerEvalSet(skill) {
|
|
317
|
-
return parseTriggerEvalSet(skill);
|
|
318
|
-
}
|
|
319
|
-
/** 生成 trigger.json 模板内容(供 CLI 提示)。 */
|
|
320
|
-
export function triggerEvalTemplate(skill) {
|
|
321
|
-
const cases = [
|
|
322
|
-
{ query: `TODO: 一条应该触发 "${skill.name}" 的真实请求`, should_trigger: true },
|
|
323
|
-
{ query: 'TODO: 一条不应触发的相近请求', should_trigger: false },
|
|
324
|
-
];
|
|
325
|
-
return JSON.stringify(cases, null, 2);
|
|
326
|
-
}
|
|
327
|
-
/** 评测结果落盘目录(<cwd>/.mocode/skill-eval/);返回写入路径。 */
|
|
328
|
-
export function saveTriggerReport(report) {
|
|
329
|
-
const dir = path.join(process.cwd(), '.mocode', 'skill-eval');
|
|
330
|
-
mkdirSync(dir, { recursive: true });
|
|
331
|
-
const stamp = new Date().toISOString().replace(/[:.]/g, '-');
|
|
332
|
-
const p = path.join(dir, `${report.skill}-${stamp}.json`);
|
|
333
|
-
writeFileSync(p, JSON.stringify(report, null, 2), 'utf8');
|
|
334
|
-
return p;
|
|
335
|
-
}
|
|
336
|
-
/** 供 CLI 校验 runsPerQuery / threshold 参数(纯函数,单测覆盖)。 */
|
|
337
|
-
export function validateEvalParams(runsPerQuery, threshold) {
|
|
338
|
-
if (!Number.isInteger(runsPerQuery) || runsPerQuery < 1 || runsPerQuery > 10) {
|
|
339
|
-
return 'runsPerQuery 必须是 1..10 的整数';
|
|
340
|
-
}
|
|
341
|
-
if (!(threshold > 0 && threshold <= 1)) {
|
|
342
|
-
return 'threshold 必须是 (0, 1] 的数';
|
|
343
|
-
}
|
|
344
|
-
return null;
|
|
345
|
-
}
|
|
@@ -1,221 +0,0 @@
|
|
|
1
|
-
// skill description 进化循环(自进化 Phase 1b)。
|
|
2
|
-
//
|
|
3
|
-
// 协议移植自 skill-creator 的 run_loop.py(调研见 docs/skill-self-evolution-research.md Part 1.2):
|
|
4
|
-
// - 分层切 train/holdout(默认 0.4)防过拟合
|
|
5
|
-
// - 每轮:LLM 读失败案例 → 提案新 description → 评测 → 接受门
|
|
6
|
-
// - 接受门(保守双条件):train 严格提升 且 holdout 不降;holdout 为空时退化为全集严格提升
|
|
7
|
-
// - 拒绝缓冲:被拒的候选记下来,后续提案明确告知「这些已试过且更差」,防重复打转
|
|
8
|
-
// - 迭代上限(默认 5);全程记录 history,返回最佳 description
|
|
9
|
-
//
|
|
10
|
-
// 安全边界(调研 Part 3):
|
|
11
|
-
// - 默认 dry-run(只打印不落盘);--apply 才写回 SKILL.md,且写前再显式确认一次
|
|
12
|
-
// - builtin skill 无磁盘载体,拒绝(入口在 skill-eval.loadSkillForEval)
|
|
13
|
-
// - 进化对象仅 description 一行;正文/脚本/执行面字段不在自动编辑范围
|
|
14
|
-
//
|
|
15
|
-
// 依赖:skill-eval.ts(评测引擎 + 纯函数)+ llm.chat(提案,单轮无工具)。
|
|
16
|
-
import { readFileSync, writeFileSync } from 'node:fs';
|
|
17
|
-
import { chat } from '../llm/index.js';
|
|
18
|
-
import { runTriggerEval, splitTriggerSet, applyDescription, MAX_DESCRIPTION_CHARS, } from './skill-eval.js';
|
|
19
|
-
/** 提案 LLM 的系统指令:确定性模板(不用 t(),保证跨语言可比、可单测断言)。 */
|
|
20
|
-
export function buildImprovePrompt(skill, currentDescription, failingCases, rejected) {
|
|
21
|
-
const body = (skill.body ?? '').slice(0, 1500);
|
|
22
|
-
const failLines = failingCases
|
|
23
|
-
.map((c) => {
|
|
24
|
-
const expect = c.should_trigger ? 'SHOULD trigger' : 'should NOT trigger';
|
|
25
|
-
return `- "${c.query}" → ${expect}, actual trigger rate ${c.trigger_rate.toFixed(2)}`;
|
|
26
|
-
})
|
|
27
|
-
.join('\n');
|
|
28
|
-
const rejectedBlock = rejected.length
|
|
29
|
-
? `\nThese descriptions were already tried and performed WORSE. Do not propose any of them or near-duplicates:\n${rejected
|
|
30
|
-
.map((d) => `- ${d}`)
|
|
31
|
-
.join('\n')}\n`
|
|
32
|
-
: '';
|
|
33
|
-
return [
|
|
34
|
-
`You are optimizing the frontmatter "description" field of an agent skill named "${skill.name}".`,
|
|
35
|
-
'The description is the ONLY text the model sees when deciding whether to trigger the skill.',
|
|
36
|
-
`The skill body (what it actually does) begins with:\n---\n${body}\n---`,
|
|
37
|
-
`Current description:\n${currentDescription}`,
|
|
38
|
-
'Test results with the current description (failures to fix):',
|
|
39
|
-
failLines,
|
|
40
|
-
rejectedBlock,
|
|
41
|
-
'Propose ONE improved description that makes the skill trigger for the SHOULD-trigger cases and stay silent for the should-NOT-trigger cases.',
|
|
42
|
-
'Rules:',
|
|
43
|
-
'- Output ONLY the description text, nothing else (no quotes, no "description:", no markdown).',
|
|
44
|
-
'- One line only, 100-400 characters.',
|
|
45
|
-
'- Write it as a routing rule: state WHAT it does and WHEN to use it (concrete triggers/phrases), not as documentation.',
|
|
46
|
-
'- Preserve the language of the current description (English stays English, Chinese stays Chinese).',
|
|
47
|
-
].join('\n');
|
|
48
|
-
}
|
|
49
|
-
/**
|
|
50
|
-
* 从 LLM 输出里提取候选 description(纯函数):
|
|
51
|
-
* 去首尾空白 → 若以 "description:" 开头则剥掉 → 只取第一行 → 去配对引号 → 校验长度。
|
|
52
|
-
* 非法(空 / 超长)返 null。
|
|
53
|
-
*/
|
|
54
|
-
export function extractDescriptionProposal(text, maxChars = MAX_DESCRIPTION_CHARS) {
|
|
55
|
-
let t = (text ?? '').trim();
|
|
56
|
-
if (!t)
|
|
57
|
-
return null;
|
|
58
|
-
t = t.replace(/^description\s*[::]\s*/i, '');
|
|
59
|
-
let first = t.split(/\r?\n/)[0].trim();
|
|
60
|
-
if (!first)
|
|
61
|
-
return null;
|
|
62
|
-
if (first.length >= 2 && (first[0] === '"' || first[0] === "'") && first[first.length - 1] === first[0]) {
|
|
63
|
-
first = first.slice(1, -1).trim();
|
|
64
|
-
}
|
|
65
|
-
if (!first || first.length > maxChars)
|
|
66
|
-
return null;
|
|
67
|
-
return first;
|
|
68
|
-
}
|
|
69
|
-
/** 接受门(纯函数):train 严格提升 且 holdout 不降;holdout 为空时全集严格提升。 */
|
|
70
|
-
export function shouldAcceptCandidate(base, cand) {
|
|
71
|
-
const trainUp = cand.trainPass > base.trainPass;
|
|
72
|
-
if (!trainUp) {
|
|
73
|
-
return { accept: false, reason: `train 未提升 (${cand.trainPass}/${cand.trainTotal} ≤ ${base.trainPass}/${base.trainTotal})` };
|
|
74
|
-
}
|
|
75
|
-
if (base.holdoutPass === null || cand.holdoutPass === null) {
|
|
76
|
-
return { accept: true, reason: `train 提升且无 holdout(${base.trainPass}→${cand.trainPass})` };
|
|
77
|
-
}
|
|
78
|
-
if (cand.holdoutPass < base.holdoutPass) {
|
|
79
|
-
return { accept: false, reason: `holdout 回退 (${cand.holdoutPass} < ${base.holdoutPass}),过拟合 train` };
|
|
80
|
-
}
|
|
81
|
-
return { accept: true, reason: `train ${base.trainPass}→${cand.trainPass},holdout ${base.holdoutPass}→${cand.holdoutPass}(不降)` };
|
|
82
|
-
}
|
|
83
|
-
/** 把报告按 query 集合拆成 train / holdout 的分项(纯函数)。 */
|
|
84
|
-
function partitionReport(report, trainQueries) {
|
|
85
|
-
const train = report.results.filter((r) => trainQueries.has(r.query));
|
|
86
|
-
const holdout = report.results.filter((r) => !trainQueries.has(r.query));
|
|
87
|
-
return {
|
|
88
|
-
trainPass: train.filter((r) => r.pass).length,
|
|
89
|
-
trainTotal: train.length,
|
|
90
|
-
holdoutPass: holdout.length ? holdout.filter((r) => r.pass).length : null,
|
|
91
|
-
holdoutTotal: holdout.length,
|
|
92
|
-
};
|
|
93
|
-
}
|
|
94
|
-
function failingCases(report) {
|
|
95
|
-
return report.results
|
|
96
|
-
.filter((r) => !r.pass)
|
|
97
|
-
.map((r) => ({ query: r.query, should_trigger: r.should_trigger, trigger_rate: r.trigger_rate }));
|
|
98
|
-
}
|
|
99
|
-
/** 调 LLM 提案一次(单轮、无工具、超时 abort)。失败抛错由调用方捕获。 */
|
|
100
|
-
async function proposeDescription(prompt, timeoutMs, signal) {
|
|
101
|
-
const controller = new AbortController();
|
|
102
|
-
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
103
|
-
// AbortSignal.any 需 Node ≥20;engines>=18,手动桥接外部 signal。
|
|
104
|
-
const externalAbort = () => controller.abort();
|
|
105
|
-
if (signal) {
|
|
106
|
-
if (signal.aborted)
|
|
107
|
-
controller.abort();
|
|
108
|
-
else
|
|
109
|
-
signal.addEventListener('abort', externalAbort, { once: true });
|
|
110
|
-
}
|
|
111
|
-
try {
|
|
112
|
-
const messages = [{ role: 'user', content: prompt }];
|
|
113
|
-
const res = await chat(messages, {}, controller.signal);
|
|
114
|
-
return extractDescriptionProposal(res.content ?? '');
|
|
115
|
-
}
|
|
116
|
-
finally {
|
|
117
|
-
clearTimeout(timer);
|
|
118
|
-
signal?.removeEventListener('abort', externalAbort);
|
|
119
|
-
}
|
|
120
|
-
}
|
|
121
|
-
/**
|
|
122
|
-
* 进化主循环。evalSet 必须非空(调用方保证;无 evals 的 skill 在 CLI 层拒绝——
|
|
123
|
-
* 「没验证门的 skill 禁止自动落盘」,调研共识 1)。
|
|
124
|
-
*/
|
|
125
|
-
export async function runImproveLoop(skill, evalSet, opts = {}) {
|
|
126
|
-
const maxIterations = opts.maxIterations ?? 5;
|
|
127
|
-
const runsPerQuery = opts.runsPerQuery ?? 3;
|
|
128
|
-
const threshold = opts.threshold ?? 0.5;
|
|
129
|
-
const { train, holdout } = splitTriggerSet(evalSet, opts.holdout ?? 0.4, opts.seed ?? 42);
|
|
130
|
-
if (train.length === 0)
|
|
131
|
-
throw new Error('train 集为空:eval 集太小,无法进化(至少需要 2 条且覆盖触发/不触发)');
|
|
132
|
-
const trainQueries = new Set(train.map((t) => t.query));
|
|
133
|
-
const evalOpts = { runsPerQuery, threshold, timeoutMs: opts.timeoutMs, signal: opts.signal };
|
|
134
|
-
let baseDescription = skill.description;
|
|
135
|
-
let baseReport = await runTriggerEval(skill, baseDescription, evalSet, runsPerQuery, threshold, evalOpts);
|
|
136
|
-
let baseParts = partitionReport(baseReport, trainQueries);
|
|
137
|
-
const iterations = [];
|
|
138
|
-
const rejected = [];
|
|
139
|
-
let bestDescription = baseDescription;
|
|
140
|
-
let bestKey = [baseParts.trainPass, baseParts.holdoutPass ?? 0];
|
|
141
|
-
const earlyExit = () => baseParts.trainTotal > 0 &&
|
|
142
|
-
baseParts.trainPass === baseParts.trainTotal &&
|
|
143
|
-
(baseParts.holdoutPass === null || (baseParts.holdoutTotal > 0 && baseParts.holdoutPass === baseParts.holdoutTotal));
|
|
144
|
-
if (earlyExit()) {
|
|
145
|
-
opts.onIteration?.(0, maxIterations, '当前 description 在 train + holdout 全通过,无需进化。');
|
|
146
|
-
}
|
|
147
|
-
for (let iter = 1; iter <= maxIterations && !earlyExit(); iter++) {
|
|
148
|
-
opts.onIteration?.(iter, maxIterations, `第 ${iter}/${maxIterations} 轮:提案新 description…`);
|
|
149
|
-
const prompt = buildImprovePrompt(skill, baseDescription, failingCases(baseReport), rejected);
|
|
150
|
-
let candidate;
|
|
151
|
-
try {
|
|
152
|
-
candidate = await proposeDescription(prompt, opts.proposeTimeoutMs ?? 120_000, opts.signal);
|
|
153
|
-
}
|
|
154
|
-
catch (e) {
|
|
155
|
-
const msg = e instanceof Error ? e.message : String(e);
|
|
156
|
-
opts.onIteration?.(iter, maxIterations, `提案失败(${msg.slice(0, 120)}),终止。`);
|
|
157
|
-
break;
|
|
158
|
-
}
|
|
159
|
-
if (!candidate || candidate === baseDescription) {
|
|
160
|
-
opts.onIteration?.(iter, maxIterations, '提案无效(空/超长/与当前相同),终止。');
|
|
161
|
-
break;
|
|
162
|
-
}
|
|
163
|
-
const candReport = await runTriggerEval(skill, candidate, evalSet, runsPerQuery, threshold, evalOpts);
|
|
164
|
-
const candParts = partitionReport(candReport, trainQueries);
|
|
165
|
-
const gate = shouldAcceptCandidate(baseParts, candParts);
|
|
166
|
-
const iteration = {
|
|
167
|
-
iteration: iter,
|
|
168
|
-
description: candidate,
|
|
169
|
-
trainPass: candParts.trainPass,
|
|
170
|
-
trainTotal: candParts.trainTotal,
|
|
171
|
-
holdoutPass: candParts.holdoutPass,
|
|
172
|
-
holdoutTotal: candParts.holdoutTotal,
|
|
173
|
-
accepted: gate.accept,
|
|
174
|
-
reason: gate.reason,
|
|
175
|
-
};
|
|
176
|
-
iterations.push(iteration);
|
|
177
|
-
opts.onIteration?.(iter, maxIterations, `${gate.accept ? '✓ 接受' : '· 拒绝'}: train ${candParts.trainPass}/${candParts.trainTotal}, holdout ${candParts.holdoutPass ?? '-'}/${candParts.holdoutTotal} — ${gate.reason}`);
|
|
178
|
-
if (gate.accept) {
|
|
179
|
-
baseDescription = candidate;
|
|
180
|
-
baseReport = candReport;
|
|
181
|
-
baseParts = candParts;
|
|
182
|
-
const key = [candParts.trainPass, candParts.holdoutPass ?? 0];
|
|
183
|
-
if (key[0] > bestKey[0] || (key[0] === bestKey[0] && key[1] > bestKey[1])) {
|
|
184
|
-
bestKey = key;
|
|
185
|
-
bestDescription = candidate;
|
|
186
|
-
}
|
|
187
|
-
}
|
|
188
|
-
else {
|
|
189
|
-
rejected.push(candidate);
|
|
190
|
-
if (rejected.length > 5)
|
|
191
|
-
rejected.shift();
|
|
192
|
-
}
|
|
193
|
-
}
|
|
194
|
-
return {
|
|
195
|
-
skill: skill.name,
|
|
196
|
-
originalDescription: skill.description,
|
|
197
|
-
bestDescription,
|
|
198
|
-
improved: bestDescription !== skill.description,
|
|
199
|
-
iterations,
|
|
200
|
-
};
|
|
201
|
-
}
|
|
202
|
-
/**
|
|
203
|
-
* 把进化结果写回 SKILL.md 的 description(唯一的落盘点;CLI --apply 时才调)。
|
|
204
|
-
* 返回写后的文件内容;frontmatter 无法安全替换时抛错(不落盘)。
|
|
205
|
-
*/
|
|
206
|
-
export function applyImprovedDescription(skill, description) {
|
|
207
|
-
if (skill.dir === 'builtin')
|
|
208
|
-
throw new Error('内置 skill 不可修改');
|
|
209
|
-
if (description.length > MAX_DESCRIPTION_CHARS)
|
|
210
|
-
throw new Error(`description 超长(>${MAX_DESCRIPTION_CHARS})`);
|
|
211
|
-
const original = readFileSync(skill.skillMdPath, 'utf8');
|
|
212
|
-
const next = applyDescription(original, description);
|
|
213
|
-
if (next === null) {
|
|
214
|
-
throw new Error('无法定位 SKILL.md frontmatter 的 description 字段,拒绝写回(请手工编辑)');
|
|
215
|
-
}
|
|
216
|
-
if (next === original) {
|
|
217
|
-
throw new Error('写回无变化(description 与当前一致)');
|
|
218
|
-
}
|
|
219
|
-
writeFileSync(skill.skillMdPath, next, 'utf8');
|
|
220
|
-
return next;
|
|
221
|
-
}
|