oh-my-knowledge 0.52.3 → 0.53.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -1
- package/README.zh.md +13 -1
- package/dist/assets/agent-skills/omk/SKILL.md +4 -0
- package/dist/assets/agent-skills/omk/references/commands.md +1 -1
- package/dist/authoring/evolver.js +1 -1
- package/dist/authoring/generator.js +1 -1
- package/dist/cli/commands/eval/index.js +7 -6
- package/dist/cli/commands/install.js +5 -1
- package/dist/cli/lib/generation-failure-hint.js +6 -8
- package/dist/cli/lib/runtime-defaults.d.ts +2 -0
- package/dist/cli/lib/runtime-defaults.js +7 -4
- package/dist/dsh-plugin/cordis.patch.yml +3 -0
- package/dist/dsh-plugin/host-executor.d.ts +93 -0
- package/dist/dsh-plugin/host-executor.js +232 -0
- package/dist/dsh-plugin/index.d.ts +28 -0
- package/dist/dsh-plugin/index.js +156 -0
- package/dist/dsh-plugin/protocol.d.ts +21 -0
- package/dist/dsh-plugin/protocol.js +229 -0
- package/dist/eval-core/comparability.js +3 -0
- package/dist/eval-core/evaluation-execution.js +5 -13
- package/dist/eval-core/evaluation-reporting.d.ts +7 -4
- package/dist/eval-core/evaluation-reporting.js +39 -18
- package/dist/eval-core/judge-independence.d.ts +1 -1
- package/dist/eval-core/judge-independence.js +1 -1
- package/dist/eval-core/report-document.js +6 -0
- package/dist/eval-core/resume-compatibility.d.ts +1 -0
- package/dist/eval-core/resume-compatibility.js +5 -4
- package/dist/eval-workflows/batch-evaluation-workflow.js +1 -1
- package/dist/eval-workflows/evaluation-pipeline/preflight-warnings.js +4 -2
- package/dist/eval-workflows/evaluation-pipeline.d.ts +3 -1
- package/dist/eval-workflows/evaluation-pipeline.js +7 -3
- package/dist/eval-workflows/run-evaluation.d.ts +4 -2
- package/dist/eval-workflows/run-evaluation.js +7 -3
- package/dist/executors/{anthropic-api.d.ts → anthropic/api.d.ts} +1 -1
- package/dist/executors/{anthropic-api.js → anthropic/api.js} +4 -2
- package/dist/executors/{claude-cli.d.ts → anthropic/claude/cli.d.ts} +1 -1
- package/dist/executors/{claude-cli.js → anthropic/claude/cli.js} +5 -3
- package/dist/executors/anthropic/claude/protocol.d.ts +87 -0
- package/dist/executors/{claude-protocol.js → anthropic/claude/protocol.js} +6 -6
- package/dist/executors/{claude-sdk.d.ts → anthropic/claude/sdk.d.ts} +2 -2
- package/dist/executors/{claude-sdk.js → anthropic/claude/sdk.js} +8 -5
- package/dist/executors/anthropic/claude/trace.d.ts +9 -0
- package/dist/executors/{claude-sdk-trace.js → anthropic/claude/trace.js} +5 -5
- package/dist/executors/{capabilities.d.ts → core/capabilities.d.ts} +3 -5
- package/dist/executors/{capabilities.js → core/capabilities.js} +4 -11
- package/dist/executors/core/http.d.ts +6 -0
- package/dist/executors/core/http.js +19 -0
- package/dist/executors/core/limits.d.ts +2 -0
- package/dist/executors/core/limits.js +2 -0
- package/dist/executors/core/optional-dependencies.d.ts +7 -0
- package/dist/executors/core/optional-dependencies.js +35 -0
- package/dist/executors/core/registry.d.ts +145 -0
- package/dist/executors/core/registry.js +127 -0
- package/dist/executors/core/runtime-fingerprint.d.ts +13 -0
- package/dist/executors/{runtime-fingerprint.js → core/runtime-fingerprint.js} +119 -59
- package/dist/executors/core/runtime.d.ts +12 -0
- package/dist/executors/core/runtime.js +61 -0
- package/dist/executors/core/subprocess.d.ts +44 -0
- package/dist/executors/{shared.js → core/subprocess.js} +20 -156
- package/dist/executors/index.d.ts +4 -4
- package/dist/executors/index.js +26 -15
- package/dist/executors/{openai-api.d.ts → openai/api.d.ts} +1 -1
- package/dist/executors/{openai-api.js → openai/api.js} +4 -2
- package/dist/executors/{codex-cli.d.ts → openai/codex/cli.d.ts} +3 -3
- package/dist/executors/{codex-cli.js → openai/codex/cli.js} +5 -3
- package/dist/executors/openai/codex/protocol.d.ts +72 -0
- package/dist/executors/{codex-protocol.js → openai/codex/protocol.js} +33 -2
- package/dist/executors/{codex-sdk.d.ts → openai/codex/sdk.d.ts} +18 -4
- package/dist/executors/{codex-sdk.js → openai/codex/sdk.js} +7 -4
- package/dist/executors/{codex-cli-trace.d.ts → openai/codex/trace.d.ts} +2 -2
- package/dist/executors/{codex-cli-trace.js → openai/codex/trace.js} +4 -4
- package/dist/executors/{script.d.ts → script/index.d.ts} +1 -1
- package/dist/executors/{script.js → script/index.js} +6 -4
- package/dist/grading/judge.d.ts +1 -1
- package/dist/grading/judge.js +1 -1
- package/dist/renderer/html-renderer.js +30 -11
- package/dist/types/executor.d.ts +13 -2
- package/dist/types/judge.d.ts +2 -2
- package/package.json +21 -5
- package/dist/executors/claude-protocol.d.ts +0 -28
- package/dist/executors/claude-sdk-trace.d.ts +0 -9
- package/dist/executors/codex-protocol.d.ts +0 -24
- package/dist/executors/gemini.d.ts +0 -2
- package/dist/executors/gemini.js +0 -156
- package/dist/executors/runtime-fingerprint.d.ts +0 -6
- package/dist/executors/shared.d.ts +0 -226
- /package/dist/executors/{script-command.d.ts → script/command.d.ts} +0 -0
- /package/dist/executors/{script-command.js → script/command.js} +0 -0
|
@@ -1,155 +1,19 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import { delimiter, join } from 'node:path';
|
|
4
|
-
import { promisify } from 'node:util';
|
|
5
|
-
export const execFileAsync = promisify(execFile);
|
|
6
|
-
// 系统默认任务执行模型。用 alias 'sonnet' — 性价比最优,eval/sample/evolve 统一默认。
|
|
7
|
-
// 需要更高质量时显式传 --model opus。report.meta.executorRuntime 记录实际模型 ID。
|
|
8
|
-
export const DEFAULT_MODEL = 'sonnet';
|
|
9
|
-
export const JUDGE_MODEL = 'haiku';
|
|
10
|
-
export const DEFAULT_TIMEOUT_MS = 600_000;
|
|
11
|
-
export const MAX_BUFFER = 10 * 1024 * 1024;
|
|
12
|
-
// 把 executor 名归到模型厂商家族,用于评委独立性判断(同厂商评委 = 自我偏好敞口)。
|
|
13
|
-
// codex 是 OpenAI 的代码模型,故 codex* 归 OpenAI。自定义 script executor 无法归类 → unknown
|
|
14
|
-
// (调用方据此跳过判断,不误报)。
|
|
15
|
-
const EXECUTOR_VENDOR = {
|
|
16
|
-
claude: 'anthropic',
|
|
17
|
-
'claude-sdk': 'anthropic',
|
|
18
|
-
'anthropic-api': 'anthropic',
|
|
19
|
-
codex: 'openai',
|
|
20
|
-
'codex-sdk': 'openai',
|
|
21
|
-
'openai-api': 'openai',
|
|
22
|
-
gemini: 'google',
|
|
23
|
-
};
|
|
24
|
-
export function executorVendor(executor) {
|
|
25
|
-
return EXECUTOR_VENDOR[executor] ?? 'unknown';
|
|
26
|
-
}
|
|
27
|
-
/**
|
|
28
|
-
* Translate Codex's external event shape into omk's internal protocol model.
|
|
29
|
-
* Codex currently calls file-change discriminators `kind`; omk reserves bare
|
|
30
|
-
* `kind` for ArtifactKind, so the raw field is qualified at the boundary.
|
|
31
|
-
*/
|
|
32
|
-
export function normalizeCodexProtocolEvent(value) {
|
|
33
|
-
if (typeof value !== 'object' || value === null || Array.isArray(value))
|
|
34
|
-
return null;
|
|
35
|
-
const event = value;
|
|
36
|
-
const rawItem = event.item;
|
|
37
|
-
if (typeof rawItem !== 'object' || rawItem === null || Array.isArray(rawItem)) {
|
|
38
|
-
return event;
|
|
39
|
-
}
|
|
40
|
-
const item = rawItem;
|
|
41
|
-
const rawChanges = item.changes;
|
|
42
|
-
const normalizedItem = {
|
|
43
|
-
...item,
|
|
44
|
-
...(Array.isArray(rawChanges) && {
|
|
45
|
-
changes: rawChanges.flatMap((change) => {
|
|
46
|
-
if (typeof change !== 'object' || change === null || Array.isArray(change))
|
|
47
|
-
return [];
|
|
48
|
-
const rawChange = change;
|
|
49
|
-
return [{
|
|
50
|
-
...(typeof rawChange.path === 'string' && { path: rawChange.path }),
|
|
51
|
-
...(typeof rawChange.kind === 'string' && { changeKind: rawChange.kind }),
|
|
52
|
-
}];
|
|
53
|
-
}),
|
|
54
|
-
}),
|
|
55
|
-
};
|
|
56
|
-
return { ...event, item: normalizedItem };
|
|
57
|
-
}
|
|
58
|
-
export function asErrorLike(err) {
|
|
59
|
-
return typeof err === 'object' && err !== null ? err : {};
|
|
60
|
-
}
|
|
61
|
-
export function errorMessage(err, fallback = 'unknown error') {
|
|
62
|
-
const details = asErrorLike(err);
|
|
63
|
-
return details.message || fallback;
|
|
64
|
-
}
|
|
65
|
-
export function parseJson(content) {
|
|
66
|
-
return JSON.parse(content);
|
|
67
|
-
}
|
|
68
|
-
export async function readJsonResponse(response) {
|
|
69
|
-
const rawBody = await response.text();
|
|
70
|
-
if (!rawBody.trim())
|
|
71
|
-
return { data: null, rawBody };
|
|
72
|
-
try {
|
|
73
|
-
return { data: JSON.parse(rawBody), rawBody };
|
|
74
|
-
}
|
|
75
|
-
catch {
|
|
76
|
-
return { data: null, rawBody };
|
|
77
|
-
}
|
|
78
|
-
}
|
|
79
|
-
export function responseBodyPreview(rawBody, maxLength = 500) {
|
|
80
|
-
const normalized = rawBody.replace(/\s+/g, ' ').trim();
|
|
81
|
-
if (!normalized)
|
|
82
|
-
return '';
|
|
83
|
-
return normalized.length > maxLength
|
|
84
|
-
? `${normalized.slice(0, maxLength)}...`
|
|
85
|
-
: normalized;
|
|
86
|
-
}
|
|
87
|
-
export function buildExecEnv(skillDir) {
|
|
88
|
-
const proxyUrl = process.env.CCV_PROXY_URL || undefined;
|
|
89
|
-
const env = proxyUrl
|
|
90
|
-
? { ...process.env, ANTHROPIC_BASE_URL: proxyUrl }
|
|
91
|
-
: { ...process.env };
|
|
92
|
-
if (skillDir) {
|
|
93
|
-
const nodeBin = join(skillDir, 'node_modules', '.bin');
|
|
94
|
-
if (existsSync(nodeBin)) {
|
|
95
|
-
env.PATH = `${nodeBin}${env.PATH ? delimiter + env.PATH : ''}`;
|
|
96
|
-
}
|
|
97
|
-
}
|
|
98
|
-
return env;
|
|
99
|
-
}
|
|
100
|
-
export function timeoutExecResult(timeoutMs, durationMs) {
|
|
101
|
-
return {
|
|
102
|
-
ok: false,
|
|
103
|
-
error: `execution timed out after ${timeoutMs / 1000}s`,
|
|
104
|
-
durationMs,
|
|
105
|
-
durationApiMs: 0,
|
|
106
|
-
inputTokens: 0,
|
|
107
|
-
outputTokens: 0,
|
|
108
|
-
cacheReadTokens: 0,
|
|
109
|
-
cacheCreationTokens: 0,
|
|
110
|
-
tokenUsageReportedByExecutor: false,
|
|
111
|
-
costUSD: 0,
|
|
112
|
-
costReportedByExecutor: false,
|
|
113
|
-
output: null,
|
|
114
|
-
stopReason: 'timeout',
|
|
115
|
-
numTurns: 0,
|
|
116
|
-
};
|
|
117
|
-
}
|
|
118
|
-
// SIGINT-killed child(用户在 host 按 Ctrl+C,parent 收到 SIGINT 后广播 SIGTERM 给 children)
|
|
119
|
-
// 跟 timeout-killed 区分:stopReason='interrupted',跟 'timeout' / 'error' 都不一样,
|
|
120
|
-
// caller / renderer 可以据此显示不同的语义("用户中断" vs "超时" vs "执行错误")。
|
|
121
|
-
export function interruptedExecResult(durationMs) {
|
|
122
|
-
return {
|
|
123
|
-
ok: false,
|
|
124
|
-
error: 'execution interrupted (SIGINT)',
|
|
125
|
-
durationMs,
|
|
126
|
-
durationApiMs: 0,
|
|
127
|
-
inputTokens: 0,
|
|
128
|
-
outputTokens: 0,
|
|
129
|
-
cacheReadTokens: 0,
|
|
130
|
-
cacheCreationTokens: 0,
|
|
131
|
-
tokenUsageReportedByExecutor: false,
|
|
132
|
-
costUSD: 0,
|
|
133
|
-
costReportedByExecutor: false,
|
|
134
|
-
output: null,
|
|
135
|
-
stopReason: 'interrupted',
|
|
136
|
-
numTurns: 0,
|
|
137
|
-
};
|
|
138
|
-
}
|
|
1
|
+
import { spawn } from 'node:child_process';
|
|
2
|
+
import { MAX_BUFFER } from './limits.js';
|
|
139
3
|
// ===========================================================================
|
|
140
|
-
// SIGINT propagation
|
|
141
|
-
// 避免 host CLI
|
|
4
|
+
// SIGINT propagation:统一让 spawn 出来的 child 在 parent 收到 SIGINT 时被 kill,
|
|
5
|
+
// 避免 host CLI(codex / claude code)Ctrl+C 后内层子进程成为 orphan。
|
|
142
6
|
// ===========================================================================
|
|
143
7
|
//
|
|
144
|
-
//
|
|
145
|
-
// - 单一进程级 process.on('SIGINT', ...) listener
|
|
146
|
-
// - 模块级 Set<ChildProcess> registry
|
|
147
|
-
// - SIGTERM
|
|
148
|
-
// - handler 用具名 function + process.removeListener(self)
|
|
8
|
+
// 设计要点:
|
|
9
|
+
// - 单一进程级 process.on('SIGINT', ...) listener,惰性首次 spawn 时安装一次(零 leak)
|
|
10
|
+
// - 模块级 Set<ChildProcess> registry,child 退出 / 出错时立即 delete
|
|
11
|
+
// - SIGTERM 先发(child 一般会做 telemetry flush),500ms grace,然后 SIGKILL fallback
|
|
12
|
+
// - handler 用具名 function + process.removeListener(self),不动 host 自己的 listener;
|
|
149
13
|
// 然后 process.kill(pid, 'SIGINT') re-raise 让 default action / host listener 接管
|
|
150
|
-
// - shuttingDown flag
|
|
151
|
-
//
|
|
152
|
-
// - 同时支持 timeout 跟 abortSignal 两条 kill
|
|
14
|
+
// - shuttingDown flag:第一次 SIGINT 进 handler 后置 true,第二次按 Ctrl+C 走 default action
|
|
15
|
+
// 立即退出(软关 → 硬关两段式)
|
|
16
|
+
// - 同时支持 timeout 跟 abortSignal 两条 kill 路径,跟 SIGINT 共用 grace 逻辑
|
|
153
17
|
const activeChildren = new Set();
|
|
154
18
|
const sigintSubscribers = new Set();
|
|
155
19
|
let sigintListenerInstalled = false;
|
|
@@ -178,7 +42,7 @@ function broadcastShutdown() {
|
|
|
178
42
|
catch { /* */ }
|
|
179
43
|
}, SIGTERM_GRACE_MS).unref();
|
|
180
44
|
}
|
|
181
|
-
//
|
|
45
|
+
// 卸载自己,re-raise SIGINT 让 host listener / default action(exit code 130)接管
|
|
182
46
|
process.removeListener('SIGINT', sigintHandler);
|
|
183
47
|
sigintListenerInstalled = false;
|
|
184
48
|
process.kill(process.pid, 'SIGINT');
|
|
@@ -209,7 +73,7 @@ export function registerSigintSubscriber(subscriber) {
|
|
|
209
73
|
sigintSubscribers.delete(subscriber);
|
|
210
74
|
};
|
|
211
75
|
}
|
|
212
|
-
// test-only
|
|
76
|
+
// test-only:重置模块级状态,让 vitest 之间互不污染
|
|
213
77
|
export function __resetSigintRegistryForTest() {
|
|
214
78
|
activeChildren.clear();
|
|
215
79
|
sigintSubscribers.clear();
|
|
@@ -221,10 +85,10 @@ export function __resetSigintRegistryForTest() {
|
|
|
221
85
|
}
|
|
222
86
|
/**
|
|
223
87
|
* spawn child + 注册到全局 SIGINT registry。返回 { child, done } 让 caller 自己
|
|
224
|
-
* 操作 stdin
|
|
88
|
+
* 操作 stdin(写入 / 关闭),通过 done 等结果。
|
|
225
89
|
*
|
|
226
|
-
*
|
|
227
|
-
* AbortSignal.timeout
|
|
90
|
+
* 适用:claude / codex / script CLI 子进程。HTTP executor(*-api)用 fetch +
|
|
91
|
+
* AbortSignal.timeout,自带 abort,不走这个。claude-sdk in-process 也不走。
|
|
228
92
|
*/
|
|
229
93
|
export function spawnWithSigintPropagation(command, args, options = {}) {
|
|
230
94
|
const { cwd, env, timeoutMs, maxBuffer = MAX_BUFFER, abortSignal } = options;
|
|
@@ -245,7 +109,7 @@ export function spawnWithSigintPropagation(command, args, options = {}) {
|
|
|
245
109
|
let graceTimer = null;
|
|
246
110
|
let timeoutTimer = null;
|
|
247
111
|
function killWithGrace(reason) {
|
|
248
|
-
//
|
|
112
|
+
// 优先级:abort 跟 timeout 互不覆盖(谁先设谁赢);buffer 不重写已有 reason
|
|
249
113
|
if (reason === 'timeout' && !killedBySignalReason)
|
|
250
114
|
killedByTimeout = true;
|
|
251
115
|
if (reason === 'abort' && !killedByTimeout)
|
|
@@ -256,7 +120,7 @@ export function spawnWithSigintPropagation(command, args, options = {}) {
|
|
|
256
120
|
catch { /* already dead */ }
|
|
257
121
|
if (graceTimer)
|
|
258
122
|
return;
|
|
259
|
-
// 即使 child trap SIGTERM
|
|
123
|
+
// 即使 child trap SIGTERM 不退出,500ms 后强 SIGKILL 兜底
|
|
260
124
|
graceTimer = setTimeout(() => {
|
|
261
125
|
try {
|
|
262
126
|
if (child.exitCode === null && child.signalCode === null)
|
|
@@ -357,7 +221,7 @@ export function spawnWithSigintPropagation(command, args, options = {}) {
|
|
|
357
221
|
reject(e);
|
|
358
222
|
return;
|
|
359
223
|
}
|
|
360
|
-
// code === null && signal === null
|
|
224
|
+
// code === null && signal === null:罕见,fallback resolve
|
|
361
225
|
resolve(result);
|
|
362
226
|
});
|
|
363
227
|
});
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { ExecutorFn } from '../types/index.js';
|
|
2
|
-
import {
|
|
3
|
-
import { createScriptExecutor } from './script.js';
|
|
4
|
-
export {
|
|
5
|
-
export { assertExecutorInputCapabilities, assertSamplesCompatibleWithExecutor, executorSupportsSampleMocks, getExecutorCapabilities, } from './capabilities.js';
|
|
2
|
+
import { extractClaudeTrace } from './anthropic/claude/trace.js';
|
|
3
|
+
import { createScriptExecutor } from './script/index.js';
|
|
4
|
+
export { extractClaudeTrace, createScriptExecutor };
|
|
5
|
+
export { assertExecutorInputCapabilities, assertSamplesCompatibleWithExecutor, executorSupportsSampleMocks, getExecutorCapabilities, } from './core/capabilities.js';
|
|
6
6
|
export declare function createExecutor(name: string): ExecutorFn;
|
package/dist/executors/index.js
CHANGED
|
@@ -1,31 +1,42 @@
|
|
|
1
|
-
import { anthropicApiExecutor } from './anthropic
|
|
2
|
-
import { claudeCliExecutor } from './claude
|
|
3
|
-
import { claudeSdkExecutor } from './claude
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
1
|
+
import { anthropicApiExecutor } from './anthropic/api.js';
|
|
2
|
+
import { claudeCliExecutor } from './anthropic/claude/cli.js';
|
|
3
|
+
import { claudeSdkExecutor } from './anthropic/claude/sdk.js';
|
|
4
|
+
import { extractClaudeTrace } from './anthropic/claude/trace.js';
|
|
5
|
+
import { openAiApiExecutor } from './openai/api.js';
|
|
6
|
+
import { codexCliExecutor } from './openai/codex/cli.js';
|
|
7
|
+
import { codexSdkExecutor } from './openai/codex/sdk.js';
|
|
8
|
+
import { createScriptExecutor } from './script/index.js';
|
|
9
|
+
import { enforceExecutorCapabilities } from './core/capabilities.js';
|
|
10
|
+
import { getExecutorDescriptor, } from './core/registry.js';
|
|
11
|
+
import { assertOptionalExecutorDependency } from './core/optional-dependencies.js';
|
|
11
12
|
// 命名一致性:provider HTTP 路径统一用 `<vendor>-api`(`anthropic-api` / `openai-api`),
|
|
12
13
|
// vendor coding agent CLI 用 vendor 名(`claude` / `codex`)。`openai` 这个不带 -api 后缀的
|
|
13
14
|
// 旧 alias 历史上指 openai-cli 子进程实现,删除后不再设别名 — 用 `--executor openai-api`。
|
|
14
|
-
const
|
|
15
|
+
const EXECUTOR_FACTORIES = {
|
|
15
16
|
claude: claudeCliExecutor,
|
|
16
17
|
'claude-sdk': claudeSdkExecutor,
|
|
17
18
|
codex: codexCliExecutor,
|
|
18
19
|
'codex-sdk': codexSdkExecutor,
|
|
19
|
-
gemini: geminiExecutor,
|
|
20
20
|
'anthropic-api': anthropicApiExecutor,
|
|
21
21
|
'openai-api': openAiApiExecutor,
|
|
22
22
|
};
|
|
23
|
-
export {
|
|
24
|
-
export { assertExecutorInputCapabilities, assertSamplesCompatibleWithExecutor, executorSupportsSampleMocks, getExecutorCapabilities, } from './capabilities.js';
|
|
23
|
+
export { extractClaudeTrace, createScriptExecutor };
|
|
24
|
+
export { assertExecutorInputCapabilities, assertSamplesCompatibleWithExecutor, executorSupportsSampleMocks, getExecutorCapabilities, } from './core/capabilities.js';
|
|
25
25
|
export function createExecutor(name) {
|
|
26
26
|
if (name.trim().length === 0) {
|
|
27
27
|
throw new Error('executor name or script command is required');
|
|
28
28
|
}
|
|
29
|
-
|
|
29
|
+
if (name === 'gemini') {
|
|
30
|
+
throw new Error('内置 gemini 执行器已移除;请改用自定义执行器适配脚本,并显式填写脚本命令。');
|
|
31
|
+
}
|
|
32
|
+
const descriptor = getExecutorDescriptor(name);
|
|
33
|
+
if (descriptor?.execution === 'host-only') {
|
|
34
|
+
throw new Error('dsh-host 仅供 DeepSeek Harness 宿主插件内部使用;请在 DSH 中运行 /omk eval。');
|
|
35
|
+
}
|
|
36
|
+
if (descriptor)
|
|
37
|
+
assertOptionalExecutorDependency(descriptor.name);
|
|
38
|
+
const executor = descriptor
|
|
39
|
+
? EXECUTOR_FACTORIES[descriptor.name]
|
|
40
|
+
: createScriptExecutor(name);
|
|
30
41
|
return enforceExecutorCapabilities(name, executor);
|
|
31
42
|
}
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import type { ExecResult, ExecutorInput } from '
|
|
1
|
+
import type { ExecResult, ExecutorInput } from '../../types/index.js';
|
|
2
2
|
export declare function openAiApiExecutor({ model, system, prompt, timeoutMs }: ExecutorInput): Promise<ExecResult>;
|
|
@@ -1,5 +1,7 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { optionalTokenCount, splitInclusiveInputTokens } from '../../shared/token-usage.js';
|
|
2
|
+
import { DEFAULT_TIMEOUT_MS } from '../core/limits.js';
|
|
3
|
+
import { readJsonResponse, responseBodyPreview } from '../core/http.js';
|
|
4
|
+
import { asErrorLike, errorMessage } from '../core/runtime.js';
|
|
3
5
|
export async function openAiApiExecutor({ model, system, prompt, timeoutMs = DEFAULT_TIMEOUT_MS }) {
|
|
4
6
|
const apiKey = process.env.OPENAI_API_KEY;
|
|
5
7
|
if (!apiKey)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import type { ExecResult, ExecutorInput } from '
|
|
2
|
-
import type
|
|
3
|
-
export { extractCodexFinalOutput, extractCodexProtocolError, extractCodexStopReason, extractCodexUsage, sumCodexElapsed, } from './
|
|
1
|
+
import type { ExecResult, ExecutorInput } from '../../../types/index.js';
|
|
2
|
+
import { type CodexEvent } from './protocol.js';
|
|
3
|
+
export { extractCodexFinalOutput, extractCodexProtocolError, extractCodexStopReason, extractCodexUsage, sumCodexElapsed, } from './protocol.js';
|
|
4
4
|
export declare function isolateCodexCwd(allowedSkills: string[] | undefined, cwd: string | null | undefined, executorName?: string): void;
|
|
5
5
|
export interface CodexJsonlParseResult {
|
|
6
6
|
events: CodexEvent[];
|
|
@@ -1,6 +1,8 @@
|
|
|
1
|
-
import { buildCodexResult } from './
|
|
2
|
-
import {
|
|
3
|
-
|
|
1
|
+
import { buildCodexResult, normalizeCodexProtocolEvent, } from './protocol.js';
|
|
2
|
+
import { DEFAULT_TIMEOUT_MS, MAX_BUFFER } from '../../core/limits.js';
|
|
3
|
+
import { buildExecEnv, errorMessage, interruptedExecResult, timeoutExecResult, } from '../../core/runtime.js';
|
|
4
|
+
import { spawnWithSigintPropagation } from '../../core/subprocess.js';
|
|
5
|
+
export { extractCodexFinalOutput, extractCodexProtocolError, extractCodexStopReason, extractCodexUsage, sumCodexElapsed, } from './protocol.js';
|
|
4
6
|
// codex CLI(0.125)隔离能力对比 claude-cli:
|
|
5
7
|
// Claude:三条 channel(SDK skills auto-discovery / subagent Skill 工具 /
|
|
6
8
|
// cwd 文件系统),`--disable-slash-commands` + `--disallowedTools Skill`
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import type { ExecResult } from '../../../types/index.js';
|
|
2
|
+
export interface CodexEvent {
|
|
3
|
+
type?: string;
|
|
4
|
+
turn_id?: string;
|
|
5
|
+
usage?: {
|
|
6
|
+
input_tokens?: number;
|
|
7
|
+
cached_input_tokens?: number;
|
|
8
|
+
output_tokens?: number;
|
|
9
|
+
reasoning_output_tokens?: number;
|
|
10
|
+
};
|
|
11
|
+
elapsed_ms?: number;
|
|
12
|
+
stop_reason?: string;
|
|
13
|
+
item?: {
|
|
14
|
+
id?: string;
|
|
15
|
+
type?: string;
|
|
16
|
+
text?: string;
|
|
17
|
+
command?: string;
|
|
18
|
+
aggregated_output?: string;
|
|
19
|
+
exit_code?: number | null;
|
|
20
|
+
status?: string;
|
|
21
|
+
path?: string;
|
|
22
|
+
content?: string;
|
|
23
|
+
query?: string;
|
|
24
|
+
results?: unknown[];
|
|
25
|
+
changes?: Array<{
|
|
26
|
+
path?: string;
|
|
27
|
+
changeKind?: string;
|
|
28
|
+
}>;
|
|
29
|
+
server?: string;
|
|
30
|
+
tool?: string;
|
|
31
|
+
name?: string;
|
|
32
|
+
arguments?: unknown;
|
|
33
|
+
result?: unknown;
|
|
34
|
+
message?: string;
|
|
35
|
+
error?: {
|
|
36
|
+
message?: string;
|
|
37
|
+
};
|
|
38
|
+
};
|
|
39
|
+
error?: {
|
|
40
|
+
message?: string;
|
|
41
|
+
};
|
|
42
|
+
message?: string;
|
|
43
|
+
ts?: number;
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Translate Codex's external event shape into omk's internal protocol model.
|
|
47
|
+
* Codex currently calls file-change discriminators `kind`; omk reserves bare
|
|
48
|
+
* `kind` for ArtifactKind, so the raw field is qualified at the boundary.
|
|
49
|
+
*/
|
|
50
|
+
export declare function normalizeCodexProtocolEvent(value: unknown): CodexEvent | null;
|
|
51
|
+
export declare function extractCodexUsage(events: CodexEvent[]): {
|
|
52
|
+
input: number;
|
|
53
|
+
cached: number;
|
|
54
|
+
output: number;
|
|
55
|
+
};
|
|
56
|
+
/**
|
|
57
|
+
* Match @openai/codex-sdk's `finalResponse`: the latest completed
|
|
58
|
+
* agent_message is the answer. Earlier messages remain available in `turns`.
|
|
59
|
+
*/
|
|
60
|
+
export declare function extractCodexFinalOutput(events: CodexEvent[]): string;
|
|
61
|
+
export declare function extractCodexProtocolError(events: CodexEvent[]): string | undefined;
|
|
62
|
+
export declare function validateCodexProtocol(events: CodexEvent[]): string | undefined;
|
|
63
|
+
export declare function extractCodexStopReason(events: CodexEvent[]): string;
|
|
64
|
+
export declare function sumCodexElapsed(resultEvents: CodexEvent[], wallClock: number): number;
|
|
65
|
+
export interface BuildCodexResultOptions {
|
|
66
|
+
events: CodexEvent[];
|
|
67
|
+
wallClockDurationMs: number;
|
|
68
|
+
source: 'codex --json' | 'codex-sdk';
|
|
69
|
+
malformedLineCount?: number;
|
|
70
|
+
forcedError?: string;
|
|
71
|
+
}
|
|
72
|
+
export declare function buildCodexResult({ events, wallClockDurationMs, source, malformedLineCount, forcedError, }: BuildCodexResultOptions): ExecResult;
|
|
@@ -1,5 +1,36 @@
|
|
|
1
|
-
import { checkedSumTokenCounts, optionalTokenCount, splitInclusiveInputTokens, sumTokenCounts, } from '
|
|
2
|
-
import { extractCodexTrace, isCodexResultEvent } from './
|
|
1
|
+
import { checkedSumTokenCounts, optionalTokenCount, splitInclusiveInputTokens, sumTokenCounts, } from '../../../shared/token-usage.js';
|
|
2
|
+
import { extractCodexTrace, isCodexResultEvent } from './trace.js';
|
|
3
|
+
/**
|
|
4
|
+
* Translate Codex's external event shape into omk's internal protocol model.
|
|
5
|
+
* Codex currently calls file-change discriminators `kind`; omk reserves bare
|
|
6
|
+
* `kind` for ArtifactKind, so the raw field is qualified at the boundary.
|
|
7
|
+
*/
|
|
8
|
+
export function normalizeCodexProtocolEvent(value) {
|
|
9
|
+
if (typeof value !== 'object' || value === null || Array.isArray(value))
|
|
10
|
+
return null;
|
|
11
|
+
const event = value;
|
|
12
|
+
const rawItem = event.item;
|
|
13
|
+
if (typeof rawItem !== 'object' || rawItem === null || Array.isArray(rawItem)) {
|
|
14
|
+
return event;
|
|
15
|
+
}
|
|
16
|
+
const item = rawItem;
|
|
17
|
+
const rawChanges = item.changes;
|
|
18
|
+
const normalizedItem = {
|
|
19
|
+
...item,
|
|
20
|
+
...(Array.isArray(rawChanges) && {
|
|
21
|
+
changes: rawChanges.flatMap((change) => {
|
|
22
|
+
if (typeof change !== 'object' || change === null || Array.isArray(change))
|
|
23
|
+
return [];
|
|
24
|
+
const rawChange = change;
|
|
25
|
+
return [{
|
|
26
|
+
...(typeof rawChange.path === 'string' && { path: rawChange.path }),
|
|
27
|
+
...(typeof rawChange.kind === 'string' && { changeKind: rawChange.kind }),
|
|
28
|
+
}];
|
|
29
|
+
}),
|
|
30
|
+
}),
|
|
31
|
+
};
|
|
32
|
+
return { ...event, item: normalizedItem };
|
|
33
|
+
}
|
|
3
34
|
const CODEX_EVENT_TYPES = new Set([
|
|
4
35
|
'thread.started',
|
|
5
36
|
'turn.started',
|
|
@@ -1,5 +1,18 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
|
|
1
|
+
import type { ExecResult, ExecutorInput } from '../../../types/index.js';
|
|
2
|
+
interface CodexSdkThread {
|
|
3
|
+
runStreamed(input: string, options?: {
|
|
4
|
+
signal?: AbortSignal;
|
|
5
|
+
}): Promise<{
|
|
6
|
+
events: AsyncIterable<unknown>;
|
|
7
|
+
}>;
|
|
8
|
+
}
|
|
9
|
+
export interface CodexSdkClient {
|
|
10
|
+
startThread(options: ReturnType<typeof buildCodexSdkThreadOptions>): CodexSdkThread;
|
|
11
|
+
}
|
|
12
|
+
export interface CodexSdkClientOptions {
|
|
13
|
+
env: Record<string, string>;
|
|
14
|
+
codexPathOverride?: string;
|
|
15
|
+
}
|
|
3
16
|
export declare function getIsolatedCodexHome(): Promise<string>;
|
|
4
17
|
export declare function __resetCodexSdkStateForTest(): void;
|
|
5
18
|
export declare function buildCodexSdkThreadOptions({ model, cwd }: {
|
|
@@ -12,6 +25,7 @@ export declare function buildCodexSdkThreadOptions({ model, cwd }: {
|
|
|
12
25
|
skipGitRepoCheck: true;
|
|
13
26
|
approvalPolicy: 'never';
|
|
14
27
|
};
|
|
15
|
-
export declare function buildCodexSdkClientOptions(env: NodeJS.ProcessEnv):
|
|
16
|
-
export declare function createCodexSdkClient(env: NodeJS.ProcessEnv): Promise<
|
|
28
|
+
export declare function buildCodexSdkClientOptions(env: NodeJS.ProcessEnv): CodexSdkClientOptions;
|
|
29
|
+
export declare function createCodexSdkClient(env: NodeJS.ProcessEnv): Promise<CodexSdkClient>;
|
|
17
30
|
export declare function codexSdkExecutor({ model, system, prompt, cwd, skillDir, timeoutMs, allowedSkills, verbose }: ExecutorInput): Promise<ExecResult>;
|
|
31
|
+
export {};
|
|
@@ -2,15 +2,18 @@ import { existsSync } from 'node:fs';
|
|
|
2
2
|
import { copyFile, mkdtemp, rm } from 'node:fs/promises';
|
|
3
3
|
import { homedir, tmpdir } from 'node:os';
|
|
4
4
|
import { join } from 'node:path';
|
|
5
|
-
import { buildCodexResult } from './
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
5
|
+
import { buildCodexResult, normalizeCodexProtocolEvent, } from './protocol.js';
|
|
6
|
+
import { DEFAULT_TIMEOUT_MS } from '../../core/limits.js';
|
|
7
|
+
import { asErrorLike, buildExecEnv, errorMessage, interruptedExecResult, timeoutExecResult, } from '../../core/runtime.js';
|
|
8
|
+
import { registerSigintSubscriber } from '../../core/subprocess.js';
|
|
9
|
+
import { isolateCodexCwd } from './cli.js';
|
|
10
|
+
const CODEX_SDK_PACKAGE = '@openai/codex-sdk';
|
|
8
11
|
let CodexCtor = null;
|
|
9
12
|
let hasWarnedSystem = false;
|
|
10
13
|
let hasWarnedCost = false;
|
|
11
14
|
async function getCodexCtor() {
|
|
12
15
|
if (!CodexCtor) {
|
|
13
|
-
const sdk = await import(
|
|
16
|
+
const sdk = await import(CODEX_SDK_PACKAGE);
|
|
14
17
|
CodexCtor = sdk.Codex;
|
|
15
18
|
}
|
|
16
19
|
return CodexCtor;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import type { ToolCallInfo, TurnInfo } from '
|
|
2
|
-
import type { CodexEvent } from './
|
|
1
|
+
import type { ToolCallInfo, TurnInfo } from '../../../types/index.js';
|
|
2
|
+
import type { CodexEvent } from './protocol.js';
|
|
3
3
|
export declare function isCodexResultEvent(event: CodexEvent): boolean;
|
|
4
4
|
export declare function extractCodexTrace(events: CodexEvent[]): {
|
|
5
5
|
turns: TurnInfo[];
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { safeSliceForJson } from '
|
|
2
|
-
import { normalizeToolIdentity } from '
|
|
3
|
-
// Codex CLI(codex 0.125)`exec --json`
|
|
4
|
-
// 跟
|
|
1
|
+
import { safeSliceForJson } from '../../../util/safe-slice.js';
|
|
2
|
+
import { normalizeToolIdentity } from '../../../shared/tool-identity.js';
|
|
3
|
+
// Codex CLI(codex 0.125)`exec --json` 与 SDK ThreadEvent 共用的 omk trace 抽取器。
|
|
4
|
+
// 跟 Claude message trace 不同源:
|
|
5
5
|
// Claude SDK 是 message[block] 嵌套结构,有 tool_use / tool_result 配对;
|
|
6
6
|
// Codex 是事件流(turn.started → turn.completed,夹杂 item.* 事件),每个
|
|
7
7
|
// item.completed 事件直接生成完整 ToolCallInfo,**不需要 use/result 配对**。
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import type { ExecutorFn } from '
|
|
1
|
+
import type { ExecutorFn } from '../../types/index.js';
|
|
2
2
|
export declare function createScriptExecutor(command: string): ExecutorFn;
|
|
@@ -1,7 +1,9 @@
|
|
|
1
|
-
import { materializeForCliConfigDir } from '
|
|
2
|
-
import { executorResultValidationError, normalizeExecResultToolIdentities, } from '
|
|
3
|
-
import { resolveScriptCommand } from './
|
|
4
|
-
import { DEFAULT_TIMEOUT_MS
|
|
1
|
+
import { materializeForCliConfigDir } from '../../eval-core/mocks-runtime.js';
|
|
2
|
+
import { executorResultValidationError, normalizeExecResultToolIdentities, } from '../../shared/executor-result.js';
|
|
3
|
+
import { resolveScriptCommand } from './command.js';
|
|
4
|
+
import { DEFAULT_TIMEOUT_MS } from '../core/limits.js';
|
|
5
|
+
import { interruptedExecResult, timeoutExecResult, } from '../core/runtime.js';
|
|
6
|
+
import { spawnWithSigintPropagation } from '../core/subprocess.js';
|
|
5
7
|
// script executor 由用户自定义,omk 无法保证它实现 skill 隔离。
|
|
6
8
|
// 任何 allowedSkills(包括 [])下都 stderr 一次性 warn,不阻塞执行,
|
|
7
9
|
// 让用户知道 strict-baseline / 显式 allowedSkills 在 script executor 下静默无效。
|
package/dist/grading/judge.d.ts
CHANGED
|
@@ -53,7 +53,7 @@ export declare function judgeId(config: JudgeConfig): string;
|
|
|
53
53
|
export declare function computeJudgeAgreement(judgeScores: number[][]): JudgeAgreement;
|
|
54
54
|
/**
|
|
55
55
|
* Judge a single (output, rubric) pair with N judge models in parallel. Each judge
|
|
56
|
-
* may use a different executor (e.g. claude:opus + openai-api:gpt-4o
|
|
56
|
+
* may use a different executor (e.g. claude:opus + openai-api:gpt-4o). Each
|
|
57
57
|
* judge can also be repeated `judgeRepeat` times — final per-judge score is its mean.
|
|
58
58
|
*
|
|
59
59
|
* Returns: aggregate DimensionResult (score = mean across judges; this is the "consensus"
|
package/dist/grading/judge.js
CHANGED
|
@@ -342,7 +342,7 @@ export function computeJudgeAgreement(judgeScores) {
|
|
|
342
342
|
}
|
|
343
343
|
/**
|
|
344
344
|
* Judge a single (output, rubric) pair with N judge models in parallel. Each judge
|
|
345
|
-
* may use a different executor (e.g. claude:opus + openai-api:gpt-4o
|
|
345
|
+
* may use a different executor (e.g. claude:opus + openai-api:gpt-4o). Each
|
|
346
346
|
* judge can also be repeated `judgeRepeat` times — final per-judge score is its mean.
|
|
347
347
|
*
|
|
348
348
|
* Returns: aggregate DimensionResult (score = mean across judges; this is the "consensus"
|