modelfunction-cli 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +118 -2
- package/fixtures/models.json +39 -0
- package/fixtures/suite.json +150 -0
- package/fixtures/tools/resume_score.json +37 -0
- package/lib/ablate.js +120 -0
- package/lib/adapter-import.js +24 -0
- package/lib/adapters/mock.js +143 -0
- package/lib/adapters/openai.js +95 -0
- package/lib/cases.js +188 -0
- package/lib/compare.js +220 -0
- package/lib/eval-exec.js +140 -0
- package/lib/exec.js +265 -0
- package/lib/fingerprint.js +52 -0
- package/lib/fsutil.js +7 -0
- package/lib/lint.js +110 -0
- package/lib/mcp-probe.js +152 -0
- package/lib/mcp-server.js +222 -0
- package/lib/report.js +119 -0
- package/lib/runtime/harness.mjs +145 -0
- package/lib/runtime/harness.py +373 -0
- package/lib/sandbox-check.js +166 -0
- package/lib/scan.js +510 -0
- package/lib/score.js +137 -0
- package/lib/suite.js +117 -0
- package/mf.js +638 -0
- package/package.json +38 -4
- package/skills/mf-cases/SKILL.md +59 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Shanghai Yunzerui Technology Co., Ltd.
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,119 @@
|
|
|
1
|
-
#
|
|
1
|
+
# modelfunction-cli
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
`mf` — 测你的模型会不会选对 function、参数填得对不对,输出多模型评分卡。本地运行,不开机自启、不驻留、不上传代码。
|
|
4
|
+
|
|
5
|
+
## 安装
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
npx -y modelfunction-cli@0.1.1 --version
|
|
9
|
+
# 或
|
|
10
|
+
npm i -D modelfunction-cli
|
|
11
|
+
npx mf --version
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
## 五步上手
|
|
15
|
+
|
|
16
|
+
```bash
|
|
17
|
+
mf scan --root . # 找 function.json / OpenAPI / MCP 配置 / SKILL.md
|
|
18
|
+
mf scan --root . --probe # 连 stdio MCP server 拉 tools/list
|
|
19
|
+
mf coverage --suite mf.suite.json # 看每个工具还缺多少用例
|
|
20
|
+
mf eval --suite mf.suite.json --models deepseek,qwen --show-cases
|
|
21
|
+
mf compare before.json after.json # 跨版本比较(内容指纹判定可比性)
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
被测模型用环境变量提供 Key:
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
DEEPSEEK_API_KEY=... mf eval --models deepseek
|
|
28
|
+
DASHSCOPE_API_KEY=... mf eval --models qwen
|
|
29
|
+
ZHIPU_API_KEY=... mf eval --models glm
|
|
30
|
+
SILICONFLOW_API_KEY=... mf eval --models dsv4-flash
|
|
31
|
+
OPENAI_API_KEY=... mf eval --models openai
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
### 自建 / 内网网关
|
|
35
|
+
|
|
36
|
+
企业内网基本都是 OpenAI 兼容协议,写一个 models-file 即可,Key 只走环境变量、不落盘:
|
|
37
|
+
|
|
38
|
+
```json
|
|
39
|
+
{
|
|
40
|
+
"internal-ds": {
|
|
41
|
+
"type": "openai",
|
|
42
|
+
"baseUrl": "https://llm.example.internal/v1",
|
|
43
|
+
"model": "deepseek-v4-flash",
|
|
44
|
+
"apiKeyEnv": "INTERNAL_LLM_KEY"
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
```bash
|
|
50
|
+
INTERNAL_LLM_KEY=... mf eval --suite mf.suite.json \
|
|
51
|
+
--models-file my-models.json --models internal-ds
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
`pricing` 可省(省了就不输出成本);要算成本请按自家价目填写,默认 registry 里的价格只是参考值。
|
|
55
|
+
|
|
56
|
+
## 命令
|
|
57
|
+
|
|
58
|
+
| 命令 | 作用 |
|
|
59
|
+
| --- | --- |
|
|
60
|
+
| `mf scan` | 扫描工具定义;`--probe` 连 MCP server;`--from` 导入运行时导出的工具数组 |
|
|
61
|
+
| `mf eval` | 跑评分卡;`--fail-under` 阈值;`--baseline` + `--fail-on-regression` 回归门禁 |
|
|
62
|
+
| `mf case add` | 写入用例(默认 `source=agent`、`reviewed=false`) |
|
|
63
|
+
| `mf coverage` | 每工具用例数、参数覆盖、负例比例、疑似混淆对 |
|
|
64
|
+
| `mf lint` | 静态检查描述与参数 schema |
|
|
65
|
+
| `mf compare` | 跨版本比较、干扰归因、混淆矩阵 |
|
|
66
|
+
| `mf ablate` | 留一法:去掉某个工具再跑一遍,量化它是否干扰其他工具 |
|
|
67
|
+
| `mf exec` | 单工具执行 + 指标(Node / Python),支持 `--allow-read/write/net`、超时、sideEffects 准入 |
|
|
68
|
+
| `mf eval --execute` | 一次跑完选择 + 参数 + 运行断言(`toolInput` / `expect`) |
|
|
69
|
+
| `mf sandbox check` | 探测本机可用隔离引擎与降级建议 |
|
|
70
|
+
| `mf mcp` | stdio MCP server,供 Claude Code / Cursor / Kimi Code 调用 |
|
|
71
|
+
|
|
72
|
+
## MCP 配置
|
|
73
|
+
|
|
74
|
+
```json
|
|
75
|
+
{
|
|
76
|
+
"mcpServers": {
|
|
77
|
+
"modelfunction": {
|
|
78
|
+
"command": "npx",
|
|
79
|
+
"args": ["-y", "modelfunction-cli@0.1.1", "mcp"]
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
暴露:`scan_tools`、`list_suite`、`coverage`、`add_cases`、`run_eval`。
|
|
86
|
+
|
|
87
|
+
## 数据边界
|
|
88
|
+
|
|
89
|
+
- 本地运行,无遥测、无服务端地址
|
|
90
|
+
- 唯一出站:你配置的模型端点,以及你自己的 MCP server(`--probe` 时)
|
|
91
|
+
- 评测会把用例输入与工具描述发给被测模型供应商——敏感数据请选境内供应商或本地模型
|
|
92
|
+
|
|
93
|
+
## 用例从哪来
|
|
94
|
+
|
|
95
|
+
由你或你的编码 Agent 撰写,规则模板见包内 `skills/mf-cases/SKILL.md`。未复核用例(`reviewed:false`)不计入正式分数。
|
|
96
|
+
|
|
97
|
+
工具一多就别手工誊抄,直接让编码 Agent 驱动 `mf`:
|
|
98
|
+
|
|
99
|
+
```bash
|
|
100
|
+
# 1. 把 mf 挂成 MCP server
|
|
101
|
+
claude mcp add modelfunction -- npx -y modelfunction-cli@0.1.1 mcp
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
```text
|
|
105
|
+
# 2. 让 Agent 扫仓库并写出套件草稿
|
|
106
|
+
用 modelfunction 的 scan_tools 扫 .,probe=true,writeSuite=mf.suite.json
|
|
107
|
+
|
|
108
|
+
# 3. 让 Agent 补用例
|
|
109
|
+
按 skills/mf-cases/SKILL.md 的规则,为每个工具补 3 条正例和 1 条负例,用 add_cases
|
|
110
|
+
写入 mf.suite.json,然后跑 coverage 看还缺哪些参数和负例
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
```bash
|
|
114
|
+
# 4. 人工复核:把认可的用例 reviewed 改为 true,再跑正式评分
|
|
115
|
+
mf coverage --suite mf.suite.json
|
|
116
|
+
mf eval --suite mf.suite.json --models internal-ds
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
License: MIT
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
{
|
|
2
|
+
"_note": "pricing 为人民币/百万 tokens 的参考值,运行前请按各家最新价目核对",
|
|
3
|
+
"mock": { "type": "mock" },
|
|
4
|
+
"deepseek": {
|
|
5
|
+
"type": "openai",
|
|
6
|
+
"baseUrl": "https://api.deepseek.com/v1",
|
|
7
|
+
"model": "deepseek-chat",
|
|
8
|
+
"apiKeyEnv": "DEEPSEEK_API_KEY",
|
|
9
|
+
"pricing": { "input": 2, "output": 8 }
|
|
10
|
+
},
|
|
11
|
+
"dsv4-flash": {
|
|
12
|
+
"type": "openai",
|
|
13
|
+
"baseUrl": "https://api.siliconflow.cn/v1",
|
|
14
|
+
"model": "deepseek-ai/DeepSeek-V4-Flash",
|
|
15
|
+
"apiKeyEnv": "SILICONFLOW_API_KEY",
|
|
16
|
+
"pricing": { "input": 1.0, "cachedInput": 0.02, "output": 2.0 }
|
|
17
|
+
},
|
|
18
|
+
"qwen": {
|
|
19
|
+
"type": "openai",
|
|
20
|
+
"baseUrl": "https://dashscope.aliyuncs.com/compatible-mode/v1",
|
|
21
|
+
"model": "qwen-plus",
|
|
22
|
+
"apiKeyEnv": "DASHSCOPE_API_KEY",
|
|
23
|
+
"pricing": { "input": 0.8, "output": 2 }
|
|
24
|
+
},
|
|
25
|
+
"glm": {
|
|
26
|
+
"type": "openai",
|
|
27
|
+
"baseUrl": "https://open.bigmodel.cn/api/paas/v4",
|
|
28
|
+
"model": "glm-4-air",
|
|
29
|
+
"apiKeyEnv": "ZHIPU_API_KEY",
|
|
30
|
+
"pricing": { "input": 0.5, "output": 0.5 }
|
|
31
|
+
},
|
|
32
|
+
"openai": {
|
|
33
|
+
"type": "openai",
|
|
34
|
+
"baseUrl": "https://api.openai.com/v1",
|
|
35
|
+
"model": "gpt-4o-mini",
|
|
36
|
+
"apiKeyEnv": "OPENAI_API_KEY",
|
|
37
|
+
"pricing": { "input": 1.1, "output": 4.3 }
|
|
38
|
+
}
|
|
39
|
+
}
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "recruiting-agent",
|
|
3
|
+
"description": "招聘 Agent 的 function-calling 评测集:简历评分、面试题、薪资区间、JD 改写。",
|
|
4
|
+
"tools": [
|
|
5
|
+
{ "ref": "./tools/resume_score.json" },
|
|
6
|
+
{
|
|
7
|
+
"name": "interview_question",
|
|
8
|
+
"description": "根据岗位和候选人背景生成针对性的模拟面试题。",
|
|
9
|
+
"parameters": {
|
|
10
|
+
"type": "object",
|
|
11
|
+
"required": ["role"],
|
|
12
|
+
"properties": {
|
|
13
|
+
"role": { "type": "string", "description": "岗位名称,如后端工程师" },
|
|
14
|
+
"background": { "type": "string", "description": "候选人简历或背景摘要" },
|
|
15
|
+
"count": { "type": "number", "description": "题目数量,默认 5" }
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
},
|
|
19
|
+
{
|
|
20
|
+
"name": "salary_band",
|
|
21
|
+
"description": "根据城市、岗位和年限给出薪资区间参考。",
|
|
22
|
+
"parameters": {
|
|
23
|
+
"type": "object",
|
|
24
|
+
"required": ["city", "role"],
|
|
25
|
+
"properties": {
|
|
26
|
+
"city": { "type": "string", "description": "城市,如上海" },
|
|
27
|
+
"role": { "type": "string", "description": "岗位名称" },
|
|
28
|
+
"years": { "type": "number", "description": "工作年限" }
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
},
|
|
32
|
+
{
|
|
33
|
+
"name": "jd_rewrite",
|
|
34
|
+
"description": "把原始职位描述改写得更有吸引力,保留关键要求。",
|
|
35
|
+
"parameters": {
|
|
36
|
+
"type": "object",
|
|
37
|
+
"required": ["jd"],
|
|
38
|
+
"properties": {
|
|
39
|
+
"jd": { "type": "string", "description": "原始职位描述文本" },
|
|
40
|
+
"tone": { "type": "string", "enum": ["专业", "活泼", "简洁"], "description": "语气风格" }
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
],
|
|
45
|
+
"cases": [
|
|
46
|
+
{
|
|
47
|
+
"id": "r01",
|
|
48
|
+
"input": "帮我给这个候选人打分,看看和岗位的匹配度。\n\n简历:张三,5 年 Python 后端经验,做过 RAG 检索增强和 LLM 应用,熟悉 MySQL、Redis、Docker,带过 3 人小组。\n\n职位描述:招聘 AI 应用后端工程师,3 年以上经验,要求熟悉 Python、MySQL、Redis、Docker、Kubernetes,有 LLM 与 RAG 项目经验优先。",
|
|
49
|
+
"expectedTool": "resume_score",
|
|
50
|
+
"expectedArgs": {
|
|
51
|
+
"resume": { "$contains": "张三" },
|
|
52
|
+
"jd": { "$contains": "Kubernetes" }
|
|
53
|
+
}
|
|
54
|
+
},
|
|
55
|
+
{
|
|
56
|
+
"id": "r02",
|
|
57
|
+
"input": "Do these match?\n\nResume: Bob, 6 years of Python and FastAPI experience, built RAG pipelines with Redis and Docker.\n\nJD: Backend engineer, 3+ years Python, Redis, Docker, Kubernetes, LLM experience preferred.",
|
|
58
|
+
"expectedTool": "resume_score",
|
|
59
|
+
"expectedArgs": {
|
|
60
|
+
"resume": { "$contains": "Bob" },
|
|
61
|
+
"jd": { "$contains": "Kubernetes" }
|
|
62
|
+
}
|
|
63
|
+
},
|
|
64
|
+
{
|
|
65
|
+
"id": "r03",
|
|
66
|
+
"input": "看看这个候选人和岗位的匹配度怎么样?",
|
|
67
|
+
"expectedTool": null,
|
|
68
|
+
"note": "没有简历或 JD 内容,应追问,而不是用编造的参数调用工具"
|
|
69
|
+
},
|
|
70
|
+
{
|
|
71
|
+
"id": "q01",
|
|
72
|
+
"input": "候选人简历:李四,3 年前端开发,做过 React 和 Vue 项目,主导过一个低代码平台。\n岗位:高级前端工程师\n帮我出 5 道面试题。",
|
|
73
|
+
"expectedTool": "interview_question",
|
|
74
|
+
"expectedArgs": {
|
|
75
|
+
"background": { "$contains": "李四" },
|
|
76
|
+
"role": { "$contains": "前端" },
|
|
77
|
+
"count": 5
|
|
78
|
+
}
|
|
79
|
+
},
|
|
80
|
+
{
|
|
81
|
+
"id": "q02",
|
|
82
|
+
"input": "给我出几道关于 Redis 和消息队列的面试题。",
|
|
83
|
+
"expectedTool": "interview_question",
|
|
84
|
+
"expectedArgs": {},
|
|
85
|
+
"note": "只测工具选择:没有岗位和背景信息"
|
|
86
|
+
},
|
|
87
|
+
{
|
|
88
|
+
"id": "s01",
|
|
89
|
+
"input": "城市:上海\n岗位:后端工程师\n年限:5\n这个岗位大概能给多少薪水?",
|
|
90
|
+
"expectedTool": "salary_band",
|
|
91
|
+
"expectedArgs": {
|
|
92
|
+
"city": { "$contains": "上海" },
|
|
93
|
+
"role": { "$contains": "后端" },
|
|
94
|
+
"years": 5
|
|
95
|
+
}
|
|
96
|
+
},
|
|
97
|
+
{
|
|
98
|
+
"id": "s02",
|
|
99
|
+
"input": "我们在杭州招一个算法工程师,3 年经验,预算大概多少?",
|
|
100
|
+
"expectedTool": "salary_band",
|
|
101
|
+
"expectedArgs": {
|
|
102
|
+
"city": { "$contains": "杭州" },
|
|
103
|
+
"role": { "$contains": "算法" },
|
|
104
|
+
"years": 3
|
|
105
|
+
}
|
|
106
|
+
},
|
|
107
|
+
{
|
|
108
|
+
"id": "j01",
|
|
109
|
+
"input": "岗位:AI 产品经理\n职位描述:负责 AI 产品规划与落地,要求 3 年 B 端产品经验,熟悉大模型能力边界。把它改写得更吸引人,语气活泼一点。",
|
|
110
|
+
"expectedTool": "jd_rewrite",
|
|
111
|
+
"expectedArgs": {
|
|
112
|
+
"jd": { "$contains": "AI 产品规划" },
|
|
113
|
+
"tone": "活泼"
|
|
114
|
+
}
|
|
115
|
+
},
|
|
116
|
+
{
|
|
117
|
+
"id": "j02",
|
|
118
|
+
"input": "帮我把这段职位描述润色得更简洁。\n职位描述:我们寻找一位熟悉 Kubernetes 和 Prometheus 的 SRE 工程师。",
|
|
119
|
+
"expectedTool": "jd_rewrite",
|
|
120
|
+
"expectedArgs": {
|
|
121
|
+
"jd": { "$contains": "Kubernetes" }
|
|
122
|
+
}
|
|
123
|
+
},
|
|
124
|
+
{
|
|
125
|
+
"id": "n01",
|
|
126
|
+
"input": "你好,你能做什么?",
|
|
127
|
+
"expectedTool": null
|
|
128
|
+
},
|
|
129
|
+
{
|
|
130
|
+
"id": "n02",
|
|
131
|
+
"input": "你们怎么收费?",
|
|
132
|
+
"expectedTool": null
|
|
133
|
+
},
|
|
134
|
+
{
|
|
135
|
+
"id": "n03",
|
|
136
|
+
"input": "帮我写一封邮件给候选人,说明面试安排。",
|
|
137
|
+
"expectedTool": null,
|
|
138
|
+
"note": "没有邮件工具,不应误调用面试题工具"
|
|
139
|
+
},
|
|
140
|
+
{
|
|
141
|
+
"id": "x01",
|
|
142
|
+
"input": "简历里有 5 年经验,这个岗位在上海能给多少薪水?",
|
|
143
|
+
"expectedTool": "salary_band",
|
|
144
|
+
"expectedArgs": {
|
|
145
|
+
"city": { "$contains": "上海" }
|
|
146
|
+
},
|
|
147
|
+
"note": "有“简历”字样,但真实意图是薪资"
|
|
148
|
+
}
|
|
149
|
+
]
|
|
150
|
+
}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "resume_score",
|
|
3
|
+
"description": "简历与 JD 匹配评分",
|
|
4
|
+
"version": "0.1.0",
|
|
5
|
+
"input": {
|
|
6
|
+
"type": "object",
|
|
7
|
+
"required": ["resume", "jd"],
|
|
8
|
+
"properties": {
|
|
9
|
+
"resume": {
|
|
10
|
+
"type": "string",
|
|
11
|
+
"minLength": 20,
|
|
12
|
+
"description": "候选人简历文本"
|
|
13
|
+
},
|
|
14
|
+
"jd": {
|
|
15
|
+
"type": "string",
|
|
16
|
+
"minLength": 20,
|
|
17
|
+
"description": "职位描述文本"
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
},
|
|
21
|
+
"output": {
|
|
22
|
+
"type": "object",
|
|
23
|
+
"properties": {
|
|
24
|
+
"score": { "type": "number", "minimum": 0, "maximum": 100 },
|
|
25
|
+
"reasons": { "type": "array", "items": { "type": "string" } },
|
|
26
|
+
"matched": { "type": "array", "items": { "type": "string" } },
|
|
27
|
+
"missing": { "type": "array", "items": { "type": "string" } },
|
|
28
|
+
"years": { "type": "number" }
|
|
29
|
+
}
|
|
30
|
+
},
|
|
31
|
+
"sideEffects": "none",
|
|
32
|
+
"model": "mock-keyword-v1",
|
|
33
|
+
"price": { "amount": 0.02, "currency": "CNY", "unit": "call" },
|
|
34
|
+
"mcp": true,
|
|
35
|
+
"auth": { "type": "api_key" },
|
|
36
|
+
"rateLimit": { "requests": 60, "window": "1m" }
|
|
37
|
+
}
|
package/lib/ablate.js
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
export function analyzeAblation(model, full, reduced) {
|
|
2
|
+
const fullRows = new Map((full.rows ?? []).map((row) => [row.id, row]));
|
|
3
|
+
const reducedRows = new Map((reduced.rows ?? []).map((row) => [row.id, row]));
|
|
4
|
+
const comparedIds = [...reducedRows.keys()].filter((id) => fullRows.has(id));
|
|
5
|
+
|
|
6
|
+
const improved = [];
|
|
7
|
+
const worsened = [];
|
|
8
|
+
const perTool = new Map();
|
|
9
|
+
|
|
10
|
+
for (const id of comparedIds) {
|
|
11
|
+
const rowFull = fullRows.get(id);
|
|
12
|
+
const rowReduced = reducedRows.get(id);
|
|
13
|
+
if (rowFull.reviewed === false || rowReduced.reviewed === false) continue;
|
|
14
|
+
|
|
15
|
+
const label = rowFull.expectedTool ?? "(none)";
|
|
16
|
+
const bucket = perTool.get(label) ?? { tool: label, total: 0, fullCorrect: 0, reducedCorrect: 0 };
|
|
17
|
+
bucket.total += 1;
|
|
18
|
+
if (rowFull.selectionCorrect) bucket.fullCorrect += 1;
|
|
19
|
+
if (rowReduced.selectionCorrect) bucket.reducedCorrect += 1;
|
|
20
|
+
perTool.set(label, bucket);
|
|
21
|
+
|
|
22
|
+
if (rowFull.selectionCorrect && !rowReduced.selectionCorrect) {
|
|
23
|
+
worsened.push({
|
|
24
|
+
id,
|
|
25
|
+
expected: rowFull.expectedTool,
|
|
26
|
+
fullTool: rowFull.actualTool ?? null,
|
|
27
|
+
reducedTool: rowReduced.actualTool ?? null
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
if (!rowFull.selectionCorrect && rowReduced.selectionCorrect) {
|
|
31
|
+
improved.push({
|
|
32
|
+
id,
|
|
33
|
+
expected: rowFull.expectedTool,
|
|
34
|
+
fullTool: rowFull.actualTool ?? null,
|
|
35
|
+
reducedTool: rowReduced.actualTool ?? null
|
|
36
|
+
});
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
const tools = [...perTool.values()].map((bucket) => {
|
|
41
|
+
const fullAccuracy = ratio(bucket.fullCorrect, bucket.total);
|
|
42
|
+
const reducedAccuracy = ratio(bucket.reducedCorrect, bucket.total);
|
|
43
|
+
return {
|
|
44
|
+
...bucket,
|
|
45
|
+
fullAccuracy,
|
|
46
|
+
reducedAccuracy,
|
|
47
|
+
delta: fullAccuracy === null || reducedAccuracy === null ? null : reducedAccuracy - fullAccuracy
|
|
48
|
+
};
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
return {
|
|
52
|
+
model,
|
|
53
|
+
compared: comparedIds.length,
|
|
54
|
+
improved,
|
|
55
|
+
worsened,
|
|
56
|
+
tools,
|
|
57
|
+
verdict:
|
|
58
|
+
improved.length > worsened.length
|
|
59
|
+
? "interference"
|
|
60
|
+
: worsened.length > improved.length
|
|
61
|
+
? "helpful"
|
|
62
|
+
: "neutral"
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export function formatAblation(payload) {
|
|
67
|
+
const lines = [
|
|
68
|
+
`ablate: ${payload.tool}(suite ${payload.suite},去掉后期望该工具的用例 ${payload.droppedCases} 条)`
|
|
69
|
+
];
|
|
70
|
+
|
|
71
|
+
for (const analysis of payload.models) {
|
|
72
|
+
lines.push("", `[${analysis.model}] 比较用例 ${analysis.compared} 条`);
|
|
73
|
+
lines.push(" tool full reduced delta");
|
|
74
|
+
lines.push(" ---------------------- ----- -------- -----");
|
|
75
|
+
for (const bucket of analysis.tools) {
|
|
76
|
+
lines.push(
|
|
77
|
+
` ${bucket.tool.padEnd(22)} ${percent(bucket.fullAccuracy).padStart(5)} ${percent(bucket.reducedAccuracy).padStart(8)} ${signed(bucket.delta).padStart(5)}`
|
|
78
|
+
);
|
|
79
|
+
}
|
|
80
|
+
if (analysis.improved.length > 0) {
|
|
81
|
+
lines.push(" 去掉后变对的用例:");
|
|
82
|
+
for (const item of analysis.improved) {
|
|
83
|
+
lines.push(
|
|
84
|
+
` - ${item.id}: ${item.fullTool ?? "(none)"} → ${item.reducedTool ?? "(none)"}(期望 ${item.expected ?? "(none)"})`
|
|
85
|
+
);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
if (analysis.worsened.length > 0) {
|
|
89
|
+
lines.push(" 去掉后变错的用例:");
|
|
90
|
+
for (const item of analysis.worsened) {
|
|
91
|
+
lines.push(
|
|
92
|
+
` - ${item.id}: ${item.fullTool ?? "(none)"} → ${item.reducedTool ?? "(none)"}(期望 ${item.expected ?? "(none)"})`
|
|
93
|
+
);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
const verdict =
|
|
97
|
+
analysis.verdict === "interference"
|
|
98
|
+
? `存在干扰:去掉 ${payload.tool} 后改善 ${analysis.improved.length} 条、变差 ${analysis.worsened.length} 条`
|
|
99
|
+
: analysis.verdict === "helpful"
|
|
100
|
+
? `有正向作用:去掉后变差 ${analysis.worsened.length} 条多于改善 ${analysis.improved.length} 条`
|
|
101
|
+
: `影响中性:改善 ${analysis.improved.length} 条、变差 ${analysis.worsened.length} 条`;
|
|
102
|
+
lines.push(` 结论: ${verdict}`);
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
return lines.join("\n");
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
function ratio(numerator, denominator) {
|
|
109
|
+
return denominator ? numerator / denominator : null;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
function percent(value) {
|
|
113
|
+
return value === null ? "n/a" : `${Math.round(value * 100)}%`;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
function signed(value) {
|
|
117
|
+
if (value === null) return "n/a";
|
|
118
|
+
const rounded = Math.round(value * 100);
|
|
119
|
+
return `${rounded > 0 ? "+" : ""}${rounded}%`;
|
|
120
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { resolve } from "node:path";
|
|
2
|
+
import { pathToFileURL } from "node:url";
|
|
3
|
+
import { normalizeTool } from "./scan.js";
|
|
4
|
+
|
|
5
|
+
export async function loadAdapterTools(filePath) {
|
|
6
|
+
const abs = resolve(filePath);
|
|
7
|
+
let mod;
|
|
8
|
+
try {
|
|
9
|
+
mod = await import(`${pathToFileURL(abs).href}?t=${Date.now()}`);
|
|
10
|
+
} catch (error) {
|
|
11
|
+
throw new Error(`adapter import failed (${abs}): ${error.message}`);
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
const raw = mod.default ?? mod.tools ?? mod.functions;
|
|
15
|
+
if (!Array.isArray(raw)) {
|
|
16
|
+
throw new Error(`${abs}: 需要 export default [...] 或 export const tools = [...]`);
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
const tools = raw.map(normalizeTool).filter(Boolean);
|
|
20
|
+
if (tools.length === 0) {
|
|
21
|
+
throw new Error(`${abs}: 没有解析出任何工具(需要 name + description + input/parameters)`);
|
|
22
|
+
}
|
|
23
|
+
return tools;
|
|
24
|
+
}
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
const INTENTS = [
|
|
2
|
+
{ tool: "resume_score", words: ["简历", "匹配", "打分", "评分", "resume", "match"] },
|
|
3
|
+
{ tool: "interview_question", words: ["面试", "提问", "interview"] },
|
|
4
|
+
{ tool: "salary_band", words: ["薪资", "薪水", "工资", "salary", "pay", "多少"] },
|
|
5
|
+
{ tool: "jd_rewrite", words: ["改写", "润色", "重写", "rewrite", "jd"] }
|
|
6
|
+
];
|
|
7
|
+
|
|
8
|
+
const PARAM_HINTS = {
|
|
9
|
+
resume: ["简历", "resume"],
|
|
10
|
+
jd: ["职位描述", "岗位描述", "jd"],
|
|
11
|
+
role: ["岗位", "职位", "role"],
|
|
12
|
+
city: ["城市", "city"],
|
|
13
|
+
level: ["级别", "职级", "level"],
|
|
14
|
+
years: ["年限", "经验", "years"]
|
|
15
|
+
};
|
|
16
|
+
|
|
17
|
+
const NAME_WEIGHT = 3;
|
|
18
|
+
const DESCRIPTION_WEIGHT = 1.5;
|
|
19
|
+
const PARAM_WEIGHT = 0.5;
|
|
20
|
+
const INTENT_WEIGHT = 2.5;
|
|
21
|
+
const THRESHOLD = 2.5;
|
|
22
|
+
|
|
23
|
+
export function createMockAdapter() {
|
|
24
|
+
return {
|
|
25
|
+
label: "mock-router-v1",
|
|
26
|
+
async call({ input, tools }) {
|
|
27
|
+
const started = performance.now();
|
|
28
|
+
const picked = pickTool(input, tools.map((tool) => tool.function ?? tool));
|
|
29
|
+
const toolCalls = picked ? [{ name: picked.name, arguments: fillArgs(picked, input) }] : [];
|
|
30
|
+
const toolNames = tools.map((tool) => (tool.function ?? tool).name).join(" ");
|
|
31
|
+
const promptChars = input.length + toolNames.length + JSON.stringify(tools).length;
|
|
32
|
+
const completionChars = JSON.stringify(toolCalls).length;
|
|
33
|
+
return {
|
|
34
|
+
ok: true,
|
|
35
|
+
toolCalls,
|
|
36
|
+
usage: {
|
|
37
|
+
input: Math.ceil(promptChars / 2),
|
|
38
|
+
output: Math.ceil(completionChars / 2),
|
|
39
|
+
estimated: true
|
|
40
|
+
},
|
|
41
|
+
latencyMs: Math.round(performance.now() - started)
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
function pickTool(input, tools) {
|
|
48
|
+
const queryTokens = tokenize(input);
|
|
49
|
+
let best = null;
|
|
50
|
+
for (const tool of tools) {
|
|
51
|
+
const score = scoreTool(tool, queryTokens, input);
|
|
52
|
+
if (!best || score > best.score) best = { tool, score };
|
|
53
|
+
}
|
|
54
|
+
return best && best.score >= THRESHOLD ? best.tool : null;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function scoreTool(tool, queryTokens, input) {
|
|
58
|
+
const lowered = input.toLowerCase();
|
|
59
|
+
let score = overlap(queryTokens, tokenize(tool.name)) * NAME_WEIGHT;
|
|
60
|
+
score += overlap(queryTokens, tokenize(tool.description)) * DESCRIPTION_WEIGHT;
|
|
61
|
+
|
|
62
|
+
const props = tool.parameters?.properties ?? {};
|
|
63
|
+
for (const [name, schema] of Object.entries(props)) {
|
|
64
|
+
score += overlap(queryTokens, tokenize(`${name} ${schema?.description ?? ""}`)) * PARAM_WEIGHT;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
for (const intent of INTENTS) {
|
|
68
|
+
if (intent.tool !== tool.name) continue;
|
|
69
|
+
if (intent.words.some((word) => lowered.includes(word.toLowerCase()))) {
|
|
70
|
+
score += INTENT_WEIGHT;
|
|
71
|
+
break;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
return score;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function overlap(a, b) {
|
|
78
|
+
if (a.size === 0 || b.size === 0) return 0;
|
|
79
|
+
let hits = 0;
|
|
80
|
+
for (const token of b) if (a.has(token)) hits += 1;
|
|
81
|
+
return hits;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function tokenize(text) {
|
|
85
|
+
const tokens = new Set();
|
|
86
|
+
const lowered = String(text ?? "").toLowerCase();
|
|
87
|
+
for (const word of lowered.match(/[a-z][a-z0-9_]{1,}/g) ?? []) tokens.add(word);
|
|
88
|
+
for (const chunk of lowered.match(/[一-鿿]+/g) ?? []) {
|
|
89
|
+
if (chunk.length === 1) tokens.add(chunk);
|
|
90
|
+
for (let i = 0; i + 1 < chunk.length; i += 1) tokens.add(chunk.slice(i, i + 2));
|
|
91
|
+
}
|
|
92
|
+
return tokens;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function fillArgs(tool, input) {
|
|
96
|
+
const sections = parseSections(input);
|
|
97
|
+
const args = {};
|
|
98
|
+
const props = tool.parameters?.properties ?? {};
|
|
99
|
+
for (const name of Object.keys(props)) {
|
|
100
|
+
const value = findSectionValue(name, sections);
|
|
101
|
+
if (value !== undefined) args[name] = value;
|
|
102
|
+
}
|
|
103
|
+
return args;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function parseSections(text) {
|
|
107
|
+
const lines = String(text).split(/\r?\n/);
|
|
108
|
+
const sections = [];
|
|
109
|
+
for (let i = 0; i < lines.length; i += 1) {
|
|
110
|
+
const match = lines[i].match(/^\s*([^\s::]{1,20})\s*[::]\s*(.*)$/);
|
|
111
|
+
if (!match) continue;
|
|
112
|
+
const header = match[1].trim();
|
|
113
|
+
let value = match[2].trim();
|
|
114
|
+
if (value === "") {
|
|
115
|
+
const body = [];
|
|
116
|
+
let j = i + 1;
|
|
117
|
+
while (j < lines.length) {
|
|
118
|
+
const line = lines[j].trim();
|
|
119
|
+
if (line === "" || /^[^\s::]{1,20}\s*[::]/.test(line)) break;
|
|
120
|
+
body.push(line);
|
|
121
|
+
j += 1;
|
|
122
|
+
}
|
|
123
|
+
value = body.join("\n");
|
|
124
|
+
}
|
|
125
|
+
if (value !== "") sections.push({ header, value });
|
|
126
|
+
}
|
|
127
|
+
return sections;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
function findSectionValue(paramName, sections) {
|
|
131
|
+
const hints = PARAM_HINTS[paramName] ?? [paramName];
|
|
132
|
+
const loweredHints = hints.map((hint) => hint.toLowerCase());
|
|
133
|
+
const hit = sections.find((section) => {
|
|
134
|
+
const header = section.header.toLowerCase();
|
|
135
|
+
return loweredHints.some((hint) => header.includes(hint));
|
|
136
|
+
});
|
|
137
|
+
if (!hit) return undefined;
|
|
138
|
+
if (paramName === "years") {
|
|
139
|
+
const number = hit.value.match(/\d+(?:\.\d+)?/);
|
|
140
|
+
return number ? Number(number[0]) : undefined;
|
|
141
|
+
}
|
|
142
|
+
return hit.value;
|
|
143
|
+
}
|