oh-my-knowledge 0.47.0 → 0.49.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +50 -18
- package/README.zh.md +55 -23
- package/dist/analysis/coverage-analyzer.d.ts +1 -0
- package/dist/analysis/coverage-analyzer.js +125 -62
- package/dist/analysis/failure-clusterer.js +2 -1
- package/dist/analysis/gap-analyzer.d.ts +2 -2
- package/dist/analysis/gap-analyzer.js +13 -3
- package/dist/analysis/hedging-classifier.d.ts +2 -2
- package/dist/analysis/hedging-classifier.js +3 -4
- package/dist/analysis/report-diagnostics.js +9 -7
- package/dist/analysis/sample-diagnostics.js +6 -6
- package/dist/artifact-graph/doctor.js +15 -7
- package/dist/assets/agent-skills/omk/SKILL.md +27 -7
- package/dist/assets/agent-skills/omk/references/commands.md +19 -17
- package/dist/authoring/evolver.d.ts +10 -6
- package/dist/authoring/evolver.js +496 -83
- package/dist/authoring/generator.d.ts +3 -3
- package/dist/authoring/generator.js +5 -10
- package/dist/authoring/sample-fixer.d.ts +8 -6
- package/dist/authoring/sample-fixer.js +76 -5
- package/dist/cli/commands/doctor.js +31 -14
- package/dist/cli/commands/eval/index.d.ts +3 -0
- package/dist/cli/commands/eval/index.js +163 -17
- package/dist/cli/commands/evolve.d.ts +4 -4
- package/dist/cli/commands/evolve.js +27 -13
- package/dist/cli/commands/init.js +16 -3
- package/dist/cli/commands/observe/inbox.js +44 -21
- package/dist/cli/commands/observe/index.js +20 -11
- package/dist/cli/commands/observe/ingest.d.ts +3 -0
- package/dist/cli/commands/observe/ingest.js +35 -4
- package/dist/cli/commands/sample.d.ts +9 -3
- package/dist/cli/commands/sample.js +91 -74
- package/dist/cli/lib/cmd-flags.d.ts +1 -0
- package/dist/cli/lib/codex-model-hint.d.ts +9 -0
- package/dist/cli/lib/codex-model-hint.js +45 -0
- package/dist/cli/lib/generation-failure-hint.d.ts +2 -0
- package/dist/cli/lib/generation-failure-hint.js +61 -0
- package/dist/cli/lib/i18n-dict/common.d.ts +1 -1
- package/dist/cli/lib/i18n-dict/common.js +4 -0
- package/dist/cli/lib/i18n-dict/gen.d.ts +1 -1
- package/dist/cli/lib/i18n-dict/gen.js +38 -6
- package/dist/cli/lib/i18n-dict/help.js +6 -6
- package/dist/cli/lib/i18n-dict/init.d.ts +1 -1
- package/dist/cli/lib/i18n-dict/init.js +13 -9
- package/dist/cli/lib/i18n-dict/run.d.ts +1 -1
- package/dist/cli/lib/i18n-dict/run.js +34 -2
- package/dist/cli/lib/llm-failure-classifier.d.ts +2 -0
- package/dist/cli/lib/llm-failure-classifier.js +8 -0
- package/dist/cli/lib/parse-run-config.d.ts +6 -5
- package/dist/cli/lib/parse-run-config.js +16 -9
- package/dist/cli/lib/runtime-defaults.d.ts +21 -0
- package/dist/cli/lib/runtime-defaults.js +79 -0
- package/dist/diagnosis/observe-mapper.js +14 -15
- package/dist/diagnosis/observe-producer.js +3 -1
- package/dist/diagnosis/studio-projection.js +14 -7
- package/dist/diagnosis/types.d.ts +2 -0
- package/dist/diagnosis/types.js +12 -0
- package/dist/doctor/endpoint-rule.js +2 -1
- package/dist/eval-core/artifact-file-names.js +18 -1
- package/dist/eval-core/artifact-index.d.ts +7 -11
- package/dist/eval-core/artifact-index.js +139 -80
- package/dist/eval-core/cache.d.ts +12 -3
- package/dist/eval-core/cache.js +89 -29
- package/dist/eval-core/comparability.js +10 -6
- package/dist/eval-core/evaluation-execution.d.ts +2 -1
- package/dist/eval-core/evaluation-execution.js +122 -37
- package/dist/eval-core/evaluation-job.d.ts +4 -1
- package/dist/eval-core/evaluation-job.js +4 -1
- package/dist/eval-core/evaluation-reporting.d.ts +15 -13
- package/dist/eval-core/evaluation-reporting.js +54 -52
- package/dist/eval-core/execution-strategy.d.ts +2 -0
- package/dist/eval-core/execution-strategy.js +11 -9
- package/dist/eval-core/fact-checker.js +15 -7
- package/dist/eval-core/holdout.js +3 -2
- package/dist/eval-core/judge-independence.d.ts +2 -2
- package/dist/eval-core/mock-hook.cjs +23 -6
- package/dist/eval-core/mocks-runtime.js +30 -8
- package/dist/eval-core/report-document.d.ts +12 -0
- package/dist/eval-core/report-document.js +1151 -0
- package/dist/eval-core/report-extensions.d.ts +4 -0
- package/dist/eval-core/report-extensions.js +500 -0
- package/dist/eval-core/report-file-migration.js +7 -2
- package/dist/eval-core/resume-compatibility.d.ts +31 -0
- package/dist/eval-core/resume-compatibility.js +141 -0
- package/dist/eval-core/sample-fingerprint.d.ts +12 -0
- package/dist/eval-core/sample-fingerprint.js +193 -0
- package/dist/eval-core/schema.js +86 -31
- package/dist/eval-core/verdict.d.ts +8 -4
- package/dist/eval-core/verdict.js +24 -10
- package/dist/eval-workflows/batch-evaluation-workflow.d.ts +2 -1
- package/dist/eval-workflows/batch-evaluation-workflow.js +25 -12
- package/dist/eval-workflows/evaluation-pipeline/preflight-warnings.d.ts +10 -5
- package/dist/eval-workflows/evaluation-pipeline/preflight-warnings.js +58 -21
- package/dist/eval-workflows/evaluation-pipeline/report-finalize.js +3 -1
- package/dist/eval-workflows/evaluation-pipeline/run-state.d.ts +4 -1
- package/dist/eval-workflows/evaluation-pipeline/run-state.js +4 -1
- package/dist/eval-workflows/evaluation-pipeline/test-set-hash.d.ts +6 -5
- package/dist/eval-workflows/evaluation-pipeline/test-set-hash.js +17 -10
- package/dist/eval-workflows/evaluation-pipeline.js +12 -7
- package/dist/eval-workflows/run-evaluation.d.ts +9 -7
- package/dist/eval-workflows/run-evaluation.js +79 -51
- package/dist/executors/anthropic-api.js +65 -9
- package/dist/executors/claude-cli.js +16 -79
- package/dist/executors/claude-protocol.d.ts +28 -0
- package/dist/executors/claude-protocol.js +180 -0
- package/dist/executors/claude-sdk-trace.js +56 -28
- package/dist/executors/claude-sdk.d.ts +1 -0
- package/dist/executors/claude-sdk.js +39 -93
- package/dist/executors/codex-cli-trace.js +166 -31
- package/dist/executors/codex-cli.d.ts +6 -8
- package/dist/executors/codex-cli.js +49 -151
- package/dist/executors/codex-protocol.d.ts +24 -0
- package/dist/executors/codex-protocol.js +234 -0
- package/dist/executors/codex-sdk.js +68 -120
- package/dist/executors/gemini.js +88 -13
- package/dist/executors/index.d.ts +2 -3
- package/dist/executors/index.js +5 -3
- package/dist/executors/openai-api.js +70 -9
- package/dist/executors/runtime-fingerprint.js +88 -11
- package/dist/executors/script-command.d.ts +8 -0
- package/dist/executors/script-command.js +87 -0
- package/dist/executors/script.js +202 -29
- package/dist/executors/shared.d.ts +35 -3
- package/dist/executors/shared.js +113 -15
- package/dist/grading/assertions.d.ts +1 -1
- package/dist/grading/assertions.js +19 -9
- package/dist/grading/diagnostic.d.ts +9 -2
- package/dist/grading/diagnostic.js +25 -2
- package/dist/grading/index.js +10 -4
- package/dist/grading/judge.js +19 -6
- package/dist/grading/layered-scores.d.ts +2 -3
- package/dist/grading/layered-scores.js +2 -3
- package/dist/inputs/load-samples.d.ts +1 -2
- package/dist/inputs/load-samples.js +23 -1
- package/dist/inputs/mcp-resolver.js +6 -3
- package/dist/inputs/sample-document.d.ts +11 -0
- package/dist/inputs/sample-document.js +96 -0
- package/dist/managed/evidence.d.ts +1 -0
- package/dist/managed/evidence.js +1 -1
- package/dist/managed/store.js +200 -91
- package/dist/observability/codex-trace-adapter.d.ts +5 -0
- package/dist/observability/codex-trace-adapter.js +850 -0
- package/dist/observability/experience.d.ts +32 -6
- package/dist/observability/experience.js +2695 -459
- package/dist/observability/feedback-matchers.js +16 -1
- package/dist/observability/inbox-view-model.d.ts +2 -1
- package/dist/observability/inbox-view-model.js +20 -14
- package/dist/observability/inbox.d.ts +7 -1
- package/dist/observability/inbox.js +632 -124
- package/dist/observability/problem-patterns.js +2 -0
- package/dist/observability/review-state.d.ts +6 -0
- package/dist/observability/review-state.js +235 -63
- package/dist/observability/skill-chain-advisories.js +1 -1
- package/dist/observability/skill-chain.js +17 -4
- package/dist/observability/skill-health-analyzer.d.ts +32 -7
- package/dist/observability/skill-health-analyzer.js +194 -121
- package/dist/observability/skill-health-report.d.ts +10 -0
- package/dist/observability/skill-health-report.js +620 -0
- package/dist/observability/soft-standards/constants.d.ts +0 -1
- package/dist/observability/soft-standards/constants.js +0 -1
- package/dist/observability/soft-standards/index.d.ts +1 -1
- package/dist/observability/soft-standards/index.js +1 -1
- package/dist/observability/soft-standards/llm-extractor.js +8 -10
- package/dist/observability/soft-standards/skill-standards-store.d.ts +2 -1
- package/dist/observability/soft-standards/skill-standards-store.js +59 -18
- package/dist/observability/soft-standards/types.d.ts +2 -2
- package/dist/observability/trace-adapter.d.ts +12 -7
- package/dist/observability/trace-adapter.js +11 -9
- package/dist/observability/trace-attribution.d.ts +13 -5
- package/dist/observability/trace-attribution.js +315 -21
- package/dist/observability/trace-ingestion.d.ts +9 -0
- package/dist/observability/trace-ingestion.js +80 -0
- package/dist/observability/trace-ir.d.ts +113 -0
- package/dist/observability/trace-ir.js +87 -0
- package/dist/observability/trace-segmenter.d.ts +19 -6
- package/dist/observability/trace-segmenter.js +377 -196
- package/dist/observability/trace-session-index.d.ts +19 -0
- package/dist/observability/trace-session-index.js +68 -0
- package/dist/observability/trace-source.d.ts +12 -4
- package/dist/observability/trace-source.js +939 -215
- package/dist/renderer/html-renderer.js +37 -6
- package/dist/renderer/icons.js +3 -0
- package/dist/renderer/observation-inbox-renderer.js +227 -91
- package/dist/renderer/skill-detail-renderer.js +452 -109
- package/dist/renderer/skill-health-renderer.js +69 -12
- package/dist/renderer/summary.js +28 -7
- package/dist/renderer/table.js +21 -4
- package/dist/renderer/test-view.d.ts +1 -0
- package/dist/renderer/test-view.js +44 -9
- package/dist/server/indexed-report-store.js +14 -18
- package/dist/server/job-store.js +64 -26
- package/dist/server/report-server.js +190 -78
- package/dist/server/report-store.js +57 -80
- package/dist/server/skill-index.js +143 -49
- package/dist/server/skill-insights.js +44 -5
- package/dist/shared/artifact-graph.d.ts +3 -0
- package/dist/shared/artifact-graph.js +224 -0
- package/dist/shared/assertion-types.d.ts +8 -0
- package/dist/shared/assertion-types.js +46 -0
- package/dist/shared/atomic-json.d.ts +8 -0
- package/dist/shared/atomic-json.js +33 -0
- package/dist/shared/diagnosis-schema.d.ts +9 -0
- package/dist/shared/diagnosis-schema.js +181 -0
- package/dist/shared/doctor-report.d.ts +3 -0
- package/dist/shared/doctor-report.js +103 -0
- package/dist/shared/evaluation-job.d.ts +6 -0
- package/dist/shared/evaluation-job.js +217 -0
- package/dist/shared/executor-result.d.ts +17 -0
- package/dist/shared/executor-result.js +221 -0
- package/dist/shared/file-lock.d.ts +12 -0
- package/dist/shared/file-lock.js +129 -0
- package/dist/shared/json-value.d.ts +5 -0
- package/dist/shared/json-value.js +36 -0
- package/dist/shared/keyed-mutex.d.ts +7 -0
- package/dist/shared/keyed-mutex.js +24 -0
- package/dist/shared/record-count.d.ts +8 -0
- package/dist/shared/record-count.js +43 -0
- package/dist/shared/sample-contract.d.ts +3 -0
- package/dist/shared/sample-contract.js +332 -0
- package/dist/shared/shell-quote.d.ts +2 -0
- package/dist/shared/shell-quote.js +7 -0
- package/dist/shared/timestamp.d.ts +6 -0
- package/dist/shared/timestamp.js +64 -0
- package/dist/shared/token-usage.d.ts +19 -0
- package/dist/shared/token-usage.js +50 -0
- package/dist/shared/tool-call-status.d.ts +8 -0
- package/dist/shared/tool-call-status.js +28 -0
- package/dist/shared/tool-identity.d.ts +21 -0
- package/dist/shared/tool-identity.js +84 -0
- package/dist/shared/tool-search.js +73 -16
- package/dist/shared/trace-projection.d.ts +5 -0
- package/dist/shared/trace-projection.js +20 -0
- package/dist/shared/trace-source-kind.d.ts +3 -0
- package/dist/shared/trace-source-kind.js +12 -0
- package/dist/types/diagnosis.d.ts +2 -0
- package/dist/types/eval.d.ts +4 -0
- package/dist/types/executor.d.ts +32 -5
- package/dist/types/index.d.ts +1 -0
- package/dist/types/index.js +1 -0
- package/dist/types/judge.d.ts +2 -0
- package/dist/types/observability.d.ts +116 -9
- package/dist/types/report.d.ts +58 -6
- package/dist/types/skill-index.d.ts +7 -0
- package/dist/types/trace.d.ts +2 -0
- package/dist/types/trace.js +1 -0
- package/package.json +9 -5
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
const BUILTIN_ALIASES = new Map([
|
|
2
|
+
['bash', 'Bash'],
|
|
3
|
+
['shell', 'Bash'],
|
|
4
|
+
['exec_command', 'Bash'],
|
|
5
|
+
['command_execution', 'Bash'],
|
|
6
|
+
['read', 'Read'],
|
|
7
|
+
['file_read', 'Read'],
|
|
8
|
+
['grep', 'Grep'],
|
|
9
|
+
['edit', 'Edit'],
|
|
10
|
+
['apply_patch', 'Edit'],
|
|
11
|
+
['file_change', 'Edit'],
|
|
12
|
+
['write', 'Write'],
|
|
13
|
+
['file_write', 'Write'],
|
|
14
|
+
['view_image', 'ViewImage'],
|
|
15
|
+
['viewimage', 'ViewImage'],
|
|
16
|
+
['write_stdin', 'WriteStdin'],
|
|
17
|
+
['writestdin', 'WriteStdin'],
|
|
18
|
+
['web_search', 'WebSearch'],
|
|
19
|
+
['websearch', 'WebSearch'],
|
|
20
|
+
]);
|
|
21
|
+
/**
|
|
22
|
+
* Normalize runtime-specific tool labels into one comparison namespace while
|
|
23
|
+
* retaining source identity for audit and future protocol migrations.
|
|
24
|
+
*/
|
|
25
|
+
export function normalizeToolIdentity(input) {
|
|
26
|
+
const sourceName = input.sourceName.trim() || 'unknown';
|
|
27
|
+
const namespace = nonEmpty(input.namespace) ?? inferredMcpNamespace(sourceName);
|
|
28
|
+
const authoritativeName = nonEmpty(input.authoritativeName);
|
|
29
|
+
const isMcp = Boolean(authoritativeName
|
|
30
|
+
|| sourceName.startsWith('mcp__')
|
|
31
|
+
|| namespace?.startsWith('mcp__'));
|
|
32
|
+
if (isMcp) {
|
|
33
|
+
const sourceLeaf = sourceName.split('__').filter(Boolean).at(-1) ?? sourceName;
|
|
34
|
+
const leaf = authoritativeName ?? sourceLeaf;
|
|
35
|
+
const provider = nonEmpty(input.provider) ?? providerFromNamespace(namespace);
|
|
36
|
+
const name = provider && !leaf.startsWith(`${provider}.`)
|
|
37
|
+
? `${provider}.${leaf}`
|
|
38
|
+
: leaf;
|
|
39
|
+
return {
|
|
40
|
+
name,
|
|
41
|
+
...(sourceName !== name ? { sourceName } : {}),
|
|
42
|
+
...(namespace ? { namespace } : {}),
|
|
43
|
+
...(provider ? { provider } : {}),
|
|
44
|
+
displayName: name,
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
if (namespace) {
|
|
48
|
+
const name = sourceName.startsWith(`${namespace}.`)
|
|
49
|
+
? sourceName
|
|
50
|
+
: `${namespace}.${sourceName}`;
|
|
51
|
+
return {
|
|
52
|
+
name,
|
|
53
|
+
...(sourceName !== name ? { sourceName } : {}),
|
|
54
|
+
namespace,
|
|
55
|
+
provider: nonEmpty(input.provider) ?? namespace,
|
|
56
|
+
displayName: name,
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
const name = BUILTIN_ALIASES.get(sourceName.toLowerCase()) ?? sourceName;
|
|
60
|
+
return {
|
|
61
|
+
name,
|
|
62
|
+
...(sourceName !== name ? { sourceName } : {}),
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
function inferredMcpNamespace(sourceName) {
|
|
66
|
+
const parts = sourceName.split('__').filter(Boolean);
|
|
67
|
+
return parts[0] === 'mcp' && parts.length > 2
|
|
68
|
+
? parts.slice(0, -1).join('__')
|
|
69
|
+
: undefined;
|
|
70
|
+
}
|
|
71
|
+
function providerFromNamespace(namespace) {
|
|
72
|
+
if (!namespace)
|
|
73
|
+
return undefined;
|
|
74
|
+
const parts = namespace.replace(/^mcp__/, '').split('__').filter(Boolean);
|
|
75
|
+
// codex_apps is a transport namespace; the following component is the
|
|
76
|
+
// actual connector/provider and matches Codex MCP end-event metadata.
|
|
77
|
+
if (parts[0] === 'codex_apps' && parts.length > 1)
|
|
78
|
+
parts.shift();
|
|
79
|
+
return parts.length > 0 ? parts.join('.') : undefined;
|
|
80
|
+
}
|
|
81
|
+
function nonEmpty(value) {
|
|
82
|
+
const normalized = value?.trim();
|
|
83
|
+
return normalized ? normalized : undefined;
|
|
84
|
+
}
|
|
@@ -1,11 +1,19 @@
|
|
|
1
|
+
import { isToolCallCancelled, isToolCallFailure, isToolCallSuccess } from './tool-call-status.js';
|
|
1
2
|
export function toolCallQuery(tc) {
|
|
2
3
|
const input = (tc.input && typeof tc.input === 'object') ? tc.input : {};
|
|
4
|
+
const legacyInput = typeof tc.input === 'string' ? tc.input : undefined;
|
|
3
5
|
if (tc.tool === 'Grep')
|
|
4
6
|
return { query: String(input.pattern ?? ''), path: String(input.path ?? '') };
|
|
5
7
|
if (tc.tool === 'Read')
|
|
6
|
-
return { path: String(input.file_path ?? '') };
|
|
8
|
+
return { path: String(input.file_path ?? legacyInput ?? '') };
|
|
7
9
|
if (tc.tool === 'Bash')
|
|
8
|
-
return bashStructuredQuery(String(input.command ?? ''));
|
|
10
|
+
return bashStructuredQuery(String(input.command ?? legacyInput ?? ''));
|
|
11
|
+
if (isWebSearchTool(tc.tool)) {
|
|
12
|
+
return {
|
|
13
|
+
query: typeof input.query === 'string' ? input.query : legacyInput,
|
|
14
|
+
path: typeof input.url === 'string' ? input.url : undefined,
|
|
15
|
+
};
|
|
16
|
+
}
|
|
9
17
|
return {};
|
|
10
18
|
}
|
|
11
19
|
/**
|
|
@@ -57,15 +65,46 @@ function bashStructuredQuery(command) {
|
|
|
57
65
|
const findName = /\bfind\b[^|;&]*?-(?:i?name|path|wholename)\s+['"]?([^'"\s]+)/.exec(command);
|
|
58
66
|
if (findName)
|
|
59
67
|
return { query: findName[1] };
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
// - 允许字母/数字/`.`/`/`/`~` 开头的 path token (foo / ./foo / /foo / ~/foo)
|
|
64
|
-
const pathOnly = /\b(?:ls|cat|test|head|tail|wc|file|stat|du|tree)\b\s+(?:-\w+\s+)*([\w./~][^\s|;&]*)/.exec(command);
|
|
65
|
-
if (pathOnly)
|
|
66
|
-
return { path: pathOnly[1] };
|
|
68
|
+
const path = shellReaderPath(command);
|
|
69
|
+
if (path)
|
|
70
|
+
return { path };
|
|
67
71
|
return {};
|
|
68
72
|
}
|
|
73
|
+
function shellReaderPath(command) {
|
|
74
|
+
const match = /\b(ls|cat|test|head|tail|wc|file|stat|du|tree|sed|awk|less|more|bat)\b\s+([^|;&]+)/.exec(command);
|
|
75
|
+
if (!match)
|
|
76
|
+
return undefined;
|
|
77
|
+
const tool = match[1];
|
|
78
|
+
const tokens = shellWords(match[2]);
|
|
79
|
+
const positional = [];
|
|
80
|
+
for (let index = 0; index < tokens.length; index += 1) {
|
|
81
|
+
const token = tokens[index];
|
|
82
|
+
if (!token.startsWith('-') || token === '-') {
|
|
83
|
+
positional.push(token);
|
|
84
|
+
continue;
|
|
85
|
+
}
|
|
86
|
+
if ((tool === 'head' || tool === 'tail')
|
|
87
|
+
&& (token === '-n' || token === '-c')
|
|
88
|
+
&& index + 1 < tokens.length) {
|
|
89
|
+
index += 1;
|
|
90
|
+
continue;
|
|
91
|
+
}
|
|
92
|
+
if (tool === 'sed' && (token === '-e' || token === '-f') && index + 1 < tokens.length) {
|
|
93
|
+
index += 1;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
const candidate = tool === 'sed' || tool === 'awk'
|
|
97
|
+
? positional.at(-1)
|
|
98
|
+
: positional[0];
|
|
99
|
+
if (!candidate || /^\d+$/.test(candidate))
|
|
100
|
+
return undefined;
|
|
101
|
+
return /^[\w./~]/.test(candidate) ? candidate : undefined;
|
|
102
|
+
}
|
|
103
|
+
function shellWords(value) {
|
|
104
|
+
return [...value.matchAll(/"((?:\\.|[^"])*)"|'((?:\\.|[^'])*)'|([^\s]+)/g)]
|
|
105
|
+
.map((match) => match[1] ?? match[2] ?? match[3])
|
|
106
|
+
.filter((token) => Boolean(token));
|
|
107
|
+
}
|
|
69
108
|
export function bashSearchOrProbeCommand(command) {
|
|
70
109
|
if (/\b(grep|rg|find)\b/.test(command))
|
|
71
110
|
return true;
|
|
@@ -76,28 +115,46 @@ export function bashSearchOrProbeCommand(command) {
|
|
|
76
115
|
export function isSearchToolCall(tc) {
|
|
77
116
|
if (tc.tool === 'Read' || tc.tool === 'Grep')
|
|
78
117
|
return true;
|
|
118
|
+
if (isWebSearchTool(tc.tool))
|
|
119
|
+
return true;
|
|
79
120
|
if (tc.tool !== 'Bash')
|
|
80
121
|
return false;
|
|
81
122
|
const input = (tc.input && typeof tc.input === 'object') ? tc.input : {};
|
|
82
|
-
|
|
123
|
+
const command = input.command ?? (typeof tc.input === 'string' ? tc.input : '');
|
|
124
|
+
return bashSearchOrProbeCommand(String(command));
|
|
83
125
|
}
|
|
84
126
|
export function isFailedSearchToolCall(tc) {
|
|
85
127
|
const output = typeof tc.output === 'string' ? tc.output : String(tc.output ?? '');
|
|
86
|
-
|
|
128
|
+
// Runtime cancellation is authoritative. Some runtimes retain partial or
|
|
129
|
+
// stale output text on cancellation; it must not become knowledge-gap evidence.
|
|
130
|
+
if (isToolCallCancelled(tc))
|
|
131
|
+
return false;
|
|
132
|
+
const explicitlyEmpty = /No matches found/i.test(output);
|
|
133
|
+
// An unresolved call also has an empty output. That is missing evidence,
|
|
134
|
+
// not evidence that a search completed with zero matches.
|
|
135
|
+
const completedEmpty = output.trim() === '' && isToolCallSuccess(tc);
|
|
87
136
|
if (tc.tool === 'Read') {
|
|
88
|
-
return tc
|
|
137
|
+
return isToolCallFailure(tc);
|
|
89
138
|
}
|
|
90
139
|
if (tc.tool === 'Grep') {
|
|
91
|
-
if (tc
|
|
140
|
+
if (isToolCallFailure(tc))
|
|
92
141
|
return true;
|
|
93
|
-
return
|
|
142
|
+
return explicitlyEmpty || completedEmpty;
|
|
143
|
+
}
|
|
144
|
+
if (isWebSearchTool(tc.tool)) {
|
|
145
|
+
return isToolCallFailure(tc);
|
|
94
146
|
}
|
|
95
147
|
if (tc.tool === 'Bash') {
|
|
96
148
|
if (!isSearchToolCall(tc))
|
|
97
149
|
return false;
|
|
98
|
-
if (tc
|
|
150
|
+
if (isToolCallFailure(tc))
|
|
99
151
|
return true;
|
|
100
|
-
return
|
|
152
|
+
return explicitlyEmpty || completedEmpty;
|
|
101
153
|
}
|
|
102
154
|
return false;
|
|
103
155
|
}
|
|
156
|
+
function isWebSearchTool(tool) {
|
|
157
|
+
// `WebSearch` is the Trace IR canonical identity. Keep the old persisted spelling
|
|
158
|
+
// readable because report schemas before v5 could contain runtime-native names.
|
|
159
|
+
return tool === 'WebSearch' || tool === 'web_search';
|
|
160
|
+
}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { ToolCallInfo, TurnInfo } from '../types/index.js';
|
|
2
|
+
export declare const MAX_PERSISTED_TURN_CONTENT = 2000;
|
|
3
|
+
export declare const MAX_PERSISTED_TOOL_OUTPUT = 1000;
|
|
4
|
+
export declare function truncateToolCallsForPersistence(toolCalls: ToolCallInfo[]): ToolCallInfo[];
|
|
5
|
+
export declare function truncateTurnsForPersistence(turns: TurnInfo[]): TurnInfo[];
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { safeSliceForJson } from '../util/safe-slice.js';
|
|
2
|
+
export const MAX_PERSISTED_TURN_CONTENT = 2000;
|
|
3
|
+
export const MAX_PERSISTED_TOOL_OUTPUT = 1000;
|
|
4
|
+
export function truncateToolCallsForPersistence(toolCalls) {
|
|
5
|
+
return toolCalls.map((toolCall) => ({
|
|
6
|
+
...toolCall,
|
|
7
|
+
output: typeof toolCall.output === 'string'
|
|
8
|
+
? safeSliceForJson(toolCall.output, MAX_PERSISTED_TOOL_OUTPUT)
|
|
9
|
+
: toolCall.output,
|
|
10
|
+
}));
|
|
11
|
+
}
|
|
12
|
+
export function truncateTurnsForPersistence(turns) {
|
|
13
|
+
return turns.map((turn) => ({
|
|
14
|
+
...turn,
|
|
15
|
+
content: safeSliceForJson(turn.content, MAX_PERSISTED_TURN_CONTENT),
|
|
16
|
+
...(turn.toolCalls && {
|
|
17
|
+
toolCalls: truncateToolCallsForPersistence(turn.toolCalls),
|
|
18
|
+
}),
|
|
19
|
+
}));
|
|
20
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
const TRACE_SOURCE_KINDS = new Set([
|
|
2
|
+
'claude',
|
|
3
|
+
'codex',
|
|
4
|
+
'openclaw',
|
|
5
|
+
'markdown_log',
|
|
6
|
+
'unknown',
|
|
7
|
+
]);
|
|
8
|
+
/** Runtime validator for the persisted Trace IR source identity protocol. */
|
|
9
|
+
export function isTraceSourceKind(value) {
|
|
10
|
+
return typeof value === 'string'
|
|
11
|
+
&& TRACE_SOURCE_KINDS.has(value);
|
|
12
|
+
}
|
|
@@ -12,6 +12,7 @@ export interface DiagnosisScope {
|
|
|
12
12
|
primary: 'skill' | 'definition' | 'session' | 'sample';
|
|
13
13
|
refs: {
|
|
14
14
|
skillName: string;
|
|
15
|
+
traceId?: string;
|
|
15
16
|
sessionId?: string;
|
|
16
17
|
invocationId?: string;
|
|
17
18
|
sampleId?: string;
|
|
@@ -23,6 +24,7 @@ export interface DiagnosisScope {
|
|
|
23
24
|
export interface DiagnosisEvidenceRef {
|
|
24
25
|
id: string;
|
|
25
26
|
kind: string;
|
|
27
|
+
traceId?: string;
|
|
26
28
|
sourceTrace?: string;
|
|
27
29
|
sessionId?: string;
|
|
28
30
|
messageIndex?: number;
|
package/dist/types/eval.d.ts
CHANGED
|
@@ -305,6 +305,10 @@ export interface EvaluationRequest {
|
|
|
305
305
|
strictBaseline?: boolean;
|
|
306
306
|
/** Reasoning effort for executor LLM。透传到 ExecutorInput.effort。 */
|
|
307
307
|
effort?: 'low' | 'medium' | 'high' | 'xhigh' | 'max';
|
|
308
|
+
/** 每个执行任务失败后的最大重试次数。缺省 / 0 表示不重试。 */
|
|
309
|
+
retry?: number;
|
|
310
|
+
/** true 时关闭 assertion 失败后的诊断 LLM 调用。缺省等价于 false。 */
|
|
311
|
+
noDiagnostic?: boolean;
|
|
308
312
|
}
|
|
309
313
|
export type EvaluationJobStatus = 'queued' | 'running' | 'succeeded' | 'failed' | 'cancelled';
|
|
310
314
|
export type EvaluationErrorCategory = 'user' | 'executor' | 'judge' | 'system';
|
package/dist/types/executor.d.ts
CHANGED
|
@@ -1,19 +1,35 @@
|
|
|
1
|
+
import type { TraceSourceKind } from './trace.js';
|
|
2
|
+
export type ToolCallStatus = 'success' | 'failure' | 'cancelled' | 'unknown';
|
|
3
|
+
export type ToolCallStatusSource = 'runtime' | 'tool-output' | 'inferred' | 'unknown';
|
|
1
4
|
export interface ToolCallInfo {
|
|
5
|
+
/** Source-neutral tool identity used by assertions and aggregate reports. */
|
|
2
6
|
tool: string;
|
|
7
|
+
/** Runtime-native identity retained when `tool` was normalized. */
|
|
8
|
+
sourceTool?: string;
|
|
9
|
+
toolNamespace?: string;
|
|
10
|
+
toolProvider?: string;
|
|
3
11
|
input: unknown;
|
|
4
12
|
output: unknown;
|
|
13
|
+
/**
|
|
14
|
+
* Source-neutral completion state. Older reports only have `success`; readers
|
|
15
|
+
* must fall back to that boolean when this field is absent.
|
|
16
|
+
*/
|
|
17
|
+
status?: ToolCallStatus;
|
|
18
|
+
statusSource?: ToolCallStatusSource;
|
|
5
19
|
success: boolean;
|
|
6
20
|
messageIndex?: number;
|
|
7
21
|
messageUuid?: string;
|
|
22
|
+
/** Source-neutral identity for one concrete call occurrence. */
|
|
23
|
+
callInstanceId?: string;
|
|
8
24
|
toolUseId?: string;
|
|
9
25
|
timestamp?: string;
|
|
10
26
|
sourceTrace?: string;
|
|
11
|
-
sourceKind?:
|
|
27
|
+
sourceKind?: TraceSourceKind;
|
|
12
28
|
traceRole?: 'standalone' | 'main' | 'subagent';
|
|
13
29
|
traceLabel?: string;
|
|
14
30
|
}
|
|
15
31
|
export interface TurnInfo {
|
|
16
|
-
role: 'assistant' | 'tool';
|
|
32
|
+
role: 'user' | 'assistant' | 'tool';
|
|
17
33
|
content: string;
|
|
18
34
|
toolCalls?: ToolCallInfo[];
|
|
19
35
|
durationMs?: number;
|
|
@@ -27,14 +43,23 @@ export interface ExecResult {
|
|
|
27
43
|
outputTokens: number;
|
|
28
44
|
cacheReadTokens: number;
|
|
29
45
|
cacheCreationTokens: number;
|
|
46
|
+
/**
|
|
47
|
+
* Whether the token counters came from the underlying runtime.
|
|
48
|
+
* `false` means the numeric fields are compatibility placeholders and must
|
|
49
|
+
* not be treated as measured zero. Missing/true preserves legacy executors.
|
|
50
|
+
*/
|
|
51
|
+
tokenUsageReportedByExecutor?: boolean;
|
|
30
52
|
costUSD: number;
|
|
31
53
|
/** Whether the underlying executor binary/SDK reported a USD cost figure.
|
|
32
|
-
* - undefined / true : `costUSD` is authoritative
|
|
33
|
-
* - false : executor 不报 cost
|
|
34
|
-
* 目前只有 codex-cli executor 设 false (codex 0.125 binary 不输出 cost)。 */
|
|
54
|
+
* - undefined / true : `costUSD` is authoritative
|
|
55
|
+
* - false : executor 不报 cost,`costUSD` 是占位 0,renderer 应显示「未报告」/「—」。 */
|
|
35
56
|
costReportedByExecutor?: boolean;
|
|
36
57
|
stopReason: string;
|
|
37
58
|
numTurns: number;
|
|
59
|
+
/** Number of executor attempts represented by this result. Missing means one.
|
|
60
|
+
* Output, token counters, and trace come from the final attempt; costUSD
|
|
61
|
+
* includes every attempt because all of them may have incurred spend. */
|
|
62
|
+
attemptCount?: number;
|
|
38
63
|
fullNumTurns?: number;
|
|
39
64
|
numSubAgents?: number;
|
|
40
65
|
error?: string;
|
|
@@ -138,6 +163,8 @@ export interface ExecutorRuntimeBinary {
|
|
|
138
163
|
source: 'path' | 'bundled' | 'none' | 'unknown';
|
|
139
164
|
version?: string;
|
|
140
165
|
path?: string;
|
|
166
|
+
/** Content identity for local script/custom executor inputs. */
|
|
167
|
+
contentHash?: string;
|
|
141
168
|
package?: ExecutorRuntimePackage;
|
|
142
169
|
error?: string;
|
|
143
170
|
}
|
package/dist/types/index.d.ts
CHANGED
package/dist/types/index.js
CHANGED
package/dist/types/judge.d.ts
CHANGED
|
@@ -118,6 +118,8 @@ export interface DiagnosticResult {
|
|
|
118
118
|
};
|
|
119
119
|
/** Diagnostic 自身花费(USD)。 */
|
|
120
120
|
costUSD?: number;
|
|
121
|
+
/** False 表示底层 executor 没有报告 diagnostic 的真实 USD 成本。 */
|
|
122
|
+
costReportedByExecutor?: boolean;
|
|
121
123
|
/** Diagnostic 是否成功(LLM JSON parse 失败时为 false,fallback 显示原始文本)。 */
|
|
122
124
|
ok: boolean;
|
|
123
125
|
/** Diagnostic 失败时的错误信息(ok=false 时使用)。 */
|