oh-my-knowledge 0.48.0 → 0.49.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +50 -18
- package/README.zh.md +55 -23
- package/dist/analysis/coverage-analyzer.d.ts +1 -0
- package/dist/analysis/coverage-analyzer.js +125 -62
- package/dist/analysis/failure-clusterer.js +2 -1
- package/dist/analysis/gap-analyzer.d.ts +2 -2
- package/dist/analysis/gap-analyzer.js +13 -3
- package/dist/analysis/hedging-classifier.d.ts +2 -2
- package/dist/analysis/hedging-classifier.js +3 -4
- package/dist/analysis/report-diagnostics.js +9 -7
- package/dist/analysis/sample-diagnostics.js +6 -6
- package/dist/artifact-graph/doctor.js +15 -7
- package/dist/assets/agent-skills/omk/SKILL.md +27 -7
- package/dist/assets/agent-skills/omk/references/commands.md +18 -17
- package/dist/authoring/evolver.d.ts +10 -6
- package/dist/authoring/evolver.js +496 -83
- package/dist/authoring/generator.d.ts +3 -3
- package/dist/authoring/generator.js +5 -10
- package/dist/authoring/sample-fixer.d.ts +8 -6
- package/dist/authoring/sample-fixer.js +76 -5
- package/dist/cli/commands/doctor.js +31 -14
- package/dist/cli/commands/eval/index.d.ts +3 -0
- package/dist/cli/commands/eval/index.js +163 -17
- package/dist/cli/commands/evolve.d.ts +4 -4
- package/dist/cli/commands/evolve.js +27 -13
- package/dist/cli/commands/init.js +16 -3
- package/dist/cli/commands/observe/inbox.js +28 -21
- package/dist/cli/commands/observe/index.js +20 -11
- package/dist/cli/commands/observe/ingest.d.ts +3 -0
- package/dist/cli/commands/observe/ingest.js +30 -2
- package/dist/cli/commands/sample.d.ts +6 -3
- package/dist/cli/commands/sample.js +72 -68
- package/dist/cli/lib/codex-model-hint.d.ts +9 -0
- package/dist/cli/lib/codex-model-hint.js +45 -0
- package/dist/cli/lib/generation-failure-hint.d.ts +2 -0
- package/dist/cli/lib/generation-failure-hint.js +61 -0
- package/dist/cli/lib/i18n-dict/common.d.ts +1 -1
- package/dist/cli/lib/i18n-dict/common.js +4 -0
- package/dist/cli/lib/i18n-dict/gen.d.ts +1 -1
- package/dist/cli/lib/i18n-dict/gen.js +38 -6
- package/dist/cli/lib/i18n-dict/help.js +6 -6
- package/dist/cli/lib/i18n-dict/init.d.ts +1 -1
- package/dist/cli/lib/i18n-dict/init.js +13 -9
- package/dist/cli/lib/i18n-dict/run.d.ts +1 -1
- package/dist/cli/lib/i18n-dict/run.js +34 -2
- package/dist/cli/lib/llm-failure-classifier.d.ts +2 -0
- package/dist/cli/lib/llm-failure-classifier.js +8 -0
- package/dist/cli/lib/parse-run-config.d.ts +6 -5
- package/dist/cli/lib/parse-run-config.js +16 -9
- package/dist/cli/lib/runtime-defaults.d.ts +21 -0
- package/dist/cli/lib/runtime-defaults.js +79 -0
- package/dist/diagnosis/observe-mapper.js +14 -15
- package/dist/diagnosis/observe-producer.js +3 -1
- package/dist/diagnosis/studio-projection.js +14 -7
- package/dist/diagnosis/types.d.ts +2 -0
- package/dist/diagnosis/types.js +12 -0
- package/dist/doctor/endpoint-rule.js +2 -1
- package/dist/eval-core/artifact-file-names.js +18 -1
- package/dist/eval-core/artifact-index.d.ts +7 -11
- package/dist/eval-core/artifact-index.js +139 -80
- package/dist/eval-core/cache.d.ts +12 -3
- package/dist/eval-core/cache.js +89 -29
- package/dist/eval-core/comparability.js +10 -6
- package/dist/eval-core/evaluation-execution.d.ts +2 -1
- package/dist/eval-core/evaluation-execution.js +122 -37
- package/dist/eval-core/evaluation-job.d.ts +4 -1
- package/dist/eval-core/evaluation-job.js +4 -1
- package/dist/eval-core/evaluation-reporting.d.ts +15 -13
- package/dist/eval-core/evaluation-reporting.js +54 -52
- package/dist/eval-core/execution-strategy.d.ts +2 -0
- package/dist/eval-core/execution-strategy.js +11 -9
- package/dist/eval-core/fact-checker.js +15 -7
- package/dist/eval-core/holdout.js +3 -2
- package/dist/eval-core/judge-independence.d.ts +2 -2
- package/dist/eval-core/mock-hook.cjs +23 -6
- package/dist/eval-core/mocks-runtime.js +30 -8
- package/dist/eval-core/report-document.d.ts +12 -0
- package/dist/eval-core/report-document.js +1151 -0
- package/dist/eval-core/report-extensions.d.ts +4 -0
- package/dist/eval-core/report-extensions.js +500 -0
- package/dist/eval-core/report-file-migration.js +7 -2
- package/dist/eval-core/resume-compatibility.d.ts +31 -0
- package/dist/eval-core/resume-compatibility.js +141 -0
- package/dist/eval-core/sample-fingerprint.d.ts +12 -0
- package/dist/eval-core/sample-fingerprint.js +193 -0
- package/dist/eval-core/schema.js +86 -31
- package/dist/eval-core/verdict.d.ts +8 -4
- package/dist/eval-core/verdict.js +24 -10
- package/dist/eval-workflows/batch-evaluation-workflow.d.ts +2 -1
- package/dist/eval-workflows/batch-evaluation-workflow.js +25 -12
- package/dist/eval-workflows/evaluation-pipeline/preflight-warnings.d.ts +10 -5
- package/dist/eval-workflows/evaluation-pipeline/preflight-warnings.js +58 -21
- package/dist/eval-workflows/evaluation-pipeline/report-finalize.js +3 -1
- package/dist/eval-workflows/evaluation-pipeline/run-state.d.ts +4 -1
- package/dist/eval-workflows/evaluation-pipeline/run-state.js +4 -1
- package/dist/eval-workflows/evaluation-pipeline/test-set-hash.d.ts +6 -5
- package/dist/eval-workflows/evaluation-pipeline/test-set-hash.js +17 -10
- package/dist/eval-workflows/evaluation-pipeline.js +12 -7
- package/dist/eval-workflows/run-evaluation.d.ts +9 -7
- package/dist/eval-workflows/run-evaluation.js +79 -51
- package/dist/executors/anthropic-api.js +65 -9
- package/dist/executors/claude-cli.js +16 -79
- package/dist/executors/claude-protocol.d.ts +28 -0
- package/dist/executors/claude-protocol.js +180 -0
- package/dist/executors/claude-sdk-trace.js +56 -28
- package/dist/executors/claude-sdk.d.ts +1 -0
- package/dist/executors/claude-sdk.js +39 -93
- package/dist/executors/codex-cli-trace.js +166 -31
- package/dist/executors/codex-cli.d.ts +6 -8
- package/dist/executors/codex-cli.js +49 -151
- package/dist/executors/codex-protocol.d.ts +24 -0
- package/dist/executors/codex-protocol.js +234 -0
- package/dist/executors/codex-sdk.js +68 -120
- package/dist/executors/gemini.js +88 -13
- package/dist/executors/index.d.ts +2 -3
- package/dist/executors/index.js +5 -3
- package/dist/executors/openai-api.js +70 -9
- package/dist/executors/runtime-fingerprint.js +88 -11
- package/dist/executors/script-command.d.ts +8 -0
- package/dist/executors/script-command.js +87 -0
- package/dist/executors/script.js +202 -29
- package/dist/executors/shared.d.ts +35 -3
- package/dist/executors/shared.js +113 -15
- package/dist/grading/assertions.d.ts +1 -1
- package/dist/grading/assertions.js +19 -9
- package/dist/grading/diagnostic.d.ts +9 -2
- package/dist/grading/diagnostic.js +25 -2
- package/dist/grading/index.js +10 -4
- package/dist/grading/judge.js +19 -6
- package/dist/grading/layered-scores.d.ts +2 -3
- package/dist/grading/layered-scores.js +2 -3
- package/dist/inputs/load-samples.d.ts +1 -2
- package/dist/inputs/load-samples.js +23 -1
- package/dist/inputs/mcp-resolver.js +6 -3
- package/dist/inputs/sample-document.d.ts +11 -0
- package/dist/inputs/sample-document.js +96 -0
- package/dist/managed/evidence.d.ts +1 -0
- package/dist/managed/evidence.js +1 -1
- package/dist/managed/store.js +200 -91
- package/dist/observability/codex-trace-adapter.d.ts +5 -0
- package/dist/observability/codex-trace-adapter.js +850 -0
- package/dist/observability/experience.d.ts +32 -6
- package/dist/observability/experience.js +2695 -459
- package/dist/observability/feedback-matchers.js +16 -1
- package/dist/observability/inbox-view-model.d.ts +1 -1
- package/dist/observability/inbox-view-model.js +19 -14
- package/dist/observability/inbox.d.ts +7 -1
- package/dist/observability/inbox.js +632 -124
- package/dist/observability/problem-patterns.js +2 -0
- package/dist/observability/review-state.d.ts +6 -0
- package/dist/observability/review-state.js +235 -63
- package/dist/observability/skill-chain-advisories.js +1 -1
- package/dist/observability/skill-chain.js +17 -4
- package/dist/observability/skill-health-analyzer.d.ts +32 -7
- package/dist/observability/skill-health-analyzer.js +194 -121
- package/dist/observability/skill-health-report.d.ts +10 -0
- package/dist/observability/skill-health-report.js +620 -0
- package/dist/observability/soft-standards/constants.d.ts +0 -1
- package/dist/observability/soft-standards/constants.js +0 -1
- package/dist/observability/soft-standards/index.d.ts +1 -1
- package/dist/observability/soft-standards/index.js +1 -1
- package/dist/observability/soft-standards/llm-extractor.js +8 -10
- package/dist/observability/soft-standards/skill-standards-store.d.ts +2 -1
- package/dist/observability/soft-standards/skill-standards-store.js +59 -18
- package/dist/observability/soft-standards/types.d.ts +2 -2
- package/dist/observability/trace-adapter.d.ts +12 -7
- package/dist/observability/trace-adapter.js +11 -9
- package/dist/observability/trace-attribution.d.ts +13 -5
- package/dist/observability/trace-attribution.js +315 -21
- package/dist/observability/trace-ingestion.d.ts +9 -0
- package/dist/observability/trace-ingestion.js +80 -0
- package/dist/observability/trace-ir.d.ts +113 -0
- package/dist/observability/trace-ir.js +87 -0
- package/dist/observability/trace-segmenter.d.ts +19 -6
- package/dist/observability/trace-segmenter.js +377 -196
- package/dist/observability/trace-session-index.d.ts +19 -0
- package/dist/observability/trace-session-index.js +68 -0
- package/dist/observability/trace-source.d.ts +12 -4
- package/dist/observability/trace-source.js +939 -215
- package/dist/renderer/html-renderer.js +37 -6
- package/dist/renderer/icons.js +3 -0
- package/dist/renderer/observation-inbox-renderer.js +208 -90
- package/dist/renderer/skill-detail-renderer.js +452 -109
- package/dist/renderer/skill-health-renderer.js +69 -12
- package/dist/renderer/summary.js +28 -7
- package/dist/renderer/table.js +21 -4
- package/dist/renderer/test-view.d.ts +1 -0
- package/dist/renderer/test-view.js +44 -9
- package/dist/server/indexed-report-store.js +14 -18
- package/dist/server/job-store.js +64 -26
- package/dist/server/report-server.js +190 -78
- package/dist/server/report-store.js +57 -80
- package/dist/server/skill-index.js +143 -49
- package/dist/server/skill-insights.js +44 -5
- package/dist/shared/artifact-graph.d.ts +3 -0
- package/dist/shared/artifact-graph.js +224 -0
- package/dist/shared/assertion-types.d.ts +8 -0
- package/dist/shared/assertion-types.js +46 -0
- package/dist/shared/atomic-json.d.ts +8 -0
- package/dist/shared/atomic-json.js +33 -0
- package/dist/shared/diagnosis-schema.d.ts +9 -0
- package/dist/shared/diagnosis-schema.js +181 -0
- package/dist/shared/doctor-report.d.ts +3 -0
- package/dist/shared/doctor-report.js +103 -0
- package/dist/shared/evaluation-job.d.ts +6 -0
- package/dist/shared/evaluation-job.js +217 -0
- package/dist/shared/executor-result.d.ts +17 -0
- package/dist/shared/executor-result.js +221 -0
- package/dist/shared/file-lock.d.ts +12 -0
- package/dist/shared/file-lock.js +129 -0
- package/dist/shared/json-value.d.ts +5 -0
- package/dist/shared/json-value.js +36 -0
- package/dist/shared/keyed-mutex.d.ts +7 -0
- package/dist/shared/keyed-mutex.js +24 -0
- package/dist/shared/record-count.d.ts +8 -0
- package/dist/shared/record-count.js +43 -0
- package/dist/shared/sample-contract.d.ts +3 -0
- package/dist/shared/sample-contract.js +332 -0
- package/dist/shared/timestamp.d.ts +6 -0
- package/dist/shared/timestamp.js +64 -0
- package/dist/shared/token-usage.d.ts +19 -0
- package/dist/shared/token-usage.js +50 -0
- package/dist/shared/tool-call-status.d.ts +8 -0
- package/dist/shared/tool-call-status.js +28 -0
- package/dist/shared/tool-identity.d.ts +21 -0
- package/dist/shared/tool-identity.js +84 -0
- package/dist/shared/tool-search.js +73 -16
- package/dist/shared/trace-projection.d.ts +5 -0
- package/dist/shared/trace-projection.js +20 -0
- package/dist/shared/trace-source-kind.d.ts +3 -0
- package/dist/shared/trace-source-kind.js +12 -0
- package/dist/types/diagnosis.d.ts +2 -0
- package/dist/types/eval.d.ts +4 -0
- package/dist/types/executor.d.ts +32 -5
- package/dist/types/index.d.ts +1 -0
- package/dist/types/index.js +1 -0
- package/dist/types/judge.d.ts +2 -0
- package/dist/types/observability.d.ts +116 -9
- package/dist/types/report.d.ts +58 -6
- package/dist/types/skill-index.d.ts +7 -0
- package/dist/types/trace.d.ts +2 -0
- package/dist/types/trace.js +1 -0
- package/package.json +9 -5
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
* - layout.ts 的整体 HTML skeleton + CSS (ki-card / ki-col / ki-bar 全部现成)
|
|
12
12
|
* - 报告视觉和 eval HTML 保持一致,读者无学习成本切换
|
|
13
13
|
*/
|
|
14
|
-
import { confidenceOf } from '../observability/skill-health-analyzer.js';
|
|
14
|
+
import { confidenceOf, toolStabilityOf } from '../observability/skill-health-analyzer.js';
|
|
15
15
|
import { COLORS, e, t } from './layout.js';
|
|
16
16
|
import { icon } from './icons.js';
|
|
17
17
|
import { reportShell } from './report-shell.js';
|
|
@@ -79,11 +79,32 @@ function renderSkillCard(skill, variantColor, lang) {
|
|
|
79
79
|
// ─── 右栏: gap signals ────────────────────────
|
|
80
80
|
const softShare = gapPct - weightedPct;
|
|
81
81
|
const failureRatePct = Math.round(skill.toolFailureRate * 100);
|
|
82
|
-
const
|
|
82
|
+
const resolvedToolCalls = skill.toolResolvedCount
|
|
83
|
+
?? Math.max(0, skill.toolCallCount - (skill.toolUnknownCount ?? 0));
|
|
84
|
+
const cancelledToolCalls = skill.toolCancelledCount ?? 0;
|
|
85
|
+
const comparableToolCalls = Math.max(0, resolvedToolCalls - cancelledToolCalls);
|
|
86
|
+
const unknownToolCalls = skill.toolUnknownCount ?? Math.max(0, skill.toolCallCount - resolvedToolCalls);
|
|
87
|
+
const stability = toolStabilityOf(skill.toolFailureRate, comparableToolCalls, skill.toolCallCount);
|
|
88
|
+
const stabilityNote = stability === 'very-unstable'
|
|
83
89
|
? ` · <span style="color:var(--red)">${lang === 'zh' ? `失败率 ${failureRatePct}%,gap 可能是环境问题` : `failure rate ${failureRatePct}%, gap likely env issue`}</span>`
|
|
84
|
-
:
|
|
90
|
+
: stability === 'unstable'
|
|
85
91
|
? ` · <span style="color:var(--yellow)">${lang === 'zh' ? `失败率 ${failureRatePct}%,gap 可能含噪声` : `failure rate ${failureRatePct}%, gap may be noisy`}</span>`
|
|
86
|
-
: ''
|
|
92
|
+
: stability === 'unknown'
|
|
93
|
+
? ` · <span style="color:var(--text-faint)">${cancelledToolCalls > 0
|
|
94
|
+
? (lang === 'zh' ? '没有可比较结果,工具调用已取消' : 'no comparable outcomes; calls cancelled')
|
|
95
|
+
: (lang === 'zh' ? '工具结果状态不可测' : 'tool outcomes unavailable')}</span>`
|
|
96
|
+
: '';
|
|
97
|
+
const outcomeEvidenceNote = stability === 'unknown'
|
|
98
|
+
? ''
|
|
99
|
+
: cancelledToolCalls > 0 || unknownToolCalls > 0
|
|
100
|
+
? ` · <span style="color:var(--text-faint)">${lang === 'zh'
|
|
101
|
+
? `${comparableToolCalls}/${skill.toolCallCount} 次结果可比较${cancelledToolCalls > 0 ? `,${cancelledToolCalls} 次取消` : ''}`
|
|
102
|
+
: `${comparableToolCalls}/${skill.toolCallCount} outcomes comparable${cancelledToolCalls > 0 ? `, ${cancelledToolCalls} cancelled` : ''}`}</span>`
|
|
103
|
+
: comparableToolCalls > 0 && comparableToolCalls < 5
|
|
104
|
+
? ` · <span style="color:var(--text-faint)">${lang === 'zh'
|
|
105
|
+
? `仅 ${comparableToolCalls} 次结果,结论置信度低`
|
|
106
|
+
: `only ${comparableToolCalls} outcomes, low confidence`}</span>`
|
|
107
|
+
: '';
|
|
87
108
|
const weightedHintBase = softShare >= 10
|
|
88
109
|
? `<strong>${lang === 'zh' ? '加权盲区' : 'weighted gap'} ${weightedPct}%</strong> · ${softShare}% ${lang === 'zh' ? '为软信号(建议复核)' : 'soft signals (review)'}`
|
|
89
110
|
: `<strong>${lang === 'zh' ? '加权盲区' : 'weighted gap'} ${weightedPct}%</strong> · ${lang === 'zh' ? '以硬证据为主' : 'mostly hard evidence'}`;
|
|
@@ -92,7 +113,7 @@ function renderSkillCard(skill, variantColor, lang) {
|
|
|
92
113
|
? `${skill.segmentCount} 段,可信度${confidence === 'underpowered' ? '不足,仅供参考' : '偏低'}`
|
|
93
114
|
: `N=${skill.segmentCount}, ${confidence} confidence`}</span>`
|
|
94
115
|
: '';
|
|
95
|
-
const weightedHint = weightedHintBase +
|
|
116
|
+
const weightedHint = weightedHintBase + stabilityNote + outcomeEvidenceNote + confNote;
|
|
96
117
|
const signalBadges = Object.entries(gap.byType)
|
|
97
118
|
.filter(([, n]) => n > 0)
|
|
98
119
|
.map(([k, n]) => {
|
|
@@ -118,19 +139,25 @@ function renderSkillCard(skill, variantColor, lang) {
|
|
|
118
139
|
${signalBadges ? `<div>${signalBadges}</div>` : ''}
|
|
119
140
|
`;
|
|
120
141
|
// 失败率 badge: 用 skill.stability 决定颜色
|
|
121
|
-
const stabilityColor =
|
|
142
|
+
const stabilityColor = stability === 'very-unstable'
|
|
122
143
|
? 'var(--red)'
|
|
123
|
-
:
|
|
144
|
+
: stability === 'unstable'
|
|
124
145
|
? 'var(--yellow)'
|
|
125
146
|
: 'var(--text-muted)';
|
|
126
|
-
const failureLabel = skill.toolCallCount > 0
|
|
127
|
-
?
|
|
128
|
-
|
|
147
|
+
const failureLabel = skill.toolCallCount > 0 && comparableToolCalls === 0
|
|
148
|
+
? cancelledToolCalls > 0 && unknownToolCalls === 0
|
|
149
|
+
? `${cancelledToolCalls} ${lang === 'zh' ? '次取消' : 'cancelled'}`
|
|
150
|
+
: cancelledToolCalls > 0
|
|
151
|
+
? `${cancelledToolCalls} ${lang === 'zh' ? '次取消' : 'cancelled'} · ${unknownToolCalls} ${lang === 'zh' ? '次状态未知' : 'unknown'}`
|
|
152
|
+
: `${skill.toolCallCount} ${lang === 'zh' ? '次状态未知' : 'unknown outcomes'}`
|
|
153
|
+
: comparableToolCalls > 0
|
|
154
|
+
? `${skill.toolFailureCount}/${comparableToolCalls} ${lang === 'zh' ? '失败' : 'failed'} (${failureRatePct}%)${cancelledToolCalls > 0 ? ` · ${cancelledToolCalls} ${lang === 'zh' ? '取消' : 'cancelled'}` : ''}`
|
|
155
|
+
: `0 ${lang === 'zh' ? '次工具调用' : 'tool calls'}`;
|
|
129
156
|
// ─── 成本/耗时行(第四轴,skill 维度聚合) ─────────
|
|
130
157
|
// 旧 JSON (加 usage 字段前生成的) 缺此字段,降级为全 0
|
|
131
158
|
const u = skill.usage ?? {
|
|
132
159
|
inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheCreationTokens: 0,
|
|
133
|
-
totalTokens: 0, durationMs: 0, numTurns: 0,
|
|
160
|
+
totalTokens: 0, tokenObservedSegmentCount: 0, tokenCoverage: 0, durationMs: 0, numTurns: 0,
|
|
134
161
|
avgTokensPerSegment: 0, avgDurationMsPerSegment: 0,
|
|
135
162
|
};
|
|
136
163
|
const billableTokens = (u.inputTokens ?? 0) + (u.outputTokens ?? 0);
|
|
@@ -140,7 +167,11 @@ function renderSkillCard(skill, variantColor, lang) {
|
|
|
140
167
|
const durSec = (u.durationMs / 1000).toFixed(1);
|
|
141
168
|
const avgDurSec = (u.avgDurationMsPerSegment / 1000).toFixed(1);
|
|
142
169
|
const cachedSeg = cachedTokens > 0 ? ` + ${cachedK}k ${lang === 'zh' ? '缓存' : 'cached'}` : '';
|
|
143
|
-
const
|
|
170
|
+
const tokenCoverage = u.tokenCoverage ?? 0;
|
|
171
|
+
const tokenText = tokenCoverage > 0
|
|
172
|
+
? `${billK}k tokens${cachedSeg}${tokenCoverage < 1 ? ` (${Math.round(tokenCoverage * 100)}% ${lang === 'zh' ? '覆盖' : 'coverage'})` : ''}`
|
|
173
|
+
: lang === 'zh' ? 'tokens —(未观测)' : 'tokens — (unobserved)';
|
|
174
|
+
const usageLine = `${tokenText} · ${durSec}s (${lang === 'zh' ? '均' : 'avg'} ${avgDurSec}s/${lang === 'zh' ? '段' : 'seg'}) · ${u.numTurns} ${lang === 'zh' ? '轮次' : 'turns'}`;
|
|
144
175
|
// ─── Card 结构 ───────────────────────────────
|
|
145
176
|
return `
|
|
146
177
|
<div class="ki-card" style="border-left:3px solid ${variantColor}">
|
|
@@ -231,8 +262,34 @@ export function renderSkillHealthReport(report, lang = 'zh') {
|
|
|
231
262
|
<div class="rs-stat rs-stat--total"><span class="rs-stat-num">${meta.toolCallCount}</span><span class="rs-stat-label">${zh ? '工具调用' : 'Tool calls'}</span></div>
|
|
232
263
|
<div class="rs-stat ${overall.weightedGapRate >= 0.3 ? 'rs-stat--fail' : overall.weightedGapRate >= 0.1 ? 'rs-stat--warn' : 'rs-stat--pass'}"><span class="rs-stat-num">${Math.round(overall.weightedGapRate * 100)}%</span><span class="rs-stat-label">${zh ? '加权盲区' : 'Weighted gap'}</span></div>
|
|
233
264
|
</div>`;
|
|
265
|
+
const ingestion = meta.ingestion;
|
|
266
|
+
const ingestionNotice = ingestion && (ingestion.malformedRecordCount > 0
|
|
267
|
+
|| ingestion.ignoredValueCount > 0
|
|
268
|
+
|| ingestion.unknownEventCount > 0)
|
|
269
|
+
? `<div style="margin:12px 0;padding:10px 12px;border:1px solid var(--yellow);border-radius:8px;background:var(--yellow-bg);color:var(--text-secondary);font-size:13px">
|
|
270
|
+
<strong style="color:var(--yellow)">${zh ? '观测输入需要复核' : 'Observation input needs review'}</strong>
|
|
271
|
+
<span style="margin-left:8px">${zh
|
|
272
|
+
? `${ingestion.malformedRecordCount} 条格式损坏记录,${ingestion.ignoredValueCount} 个非对象值,${ingestion.unknownEventCount} 个未识别事件。`
|
|
273
|
+
: `${ingestion.malformedRecordCount} malformed records, ${ingestion.ignoredValueCount} non-object values, ${ingestion.unknownEventCount} unrecognized events.`}</span>
|
|
274
|
+
</div>`
|
|
275
|
+
: '';
|
|
276
|
+
const timestampedSegmentCount = meta.timestampedSegmentCount
|
|
277
|
+
?? (meta.timeRange.from && meta.timeRange.to ? meta.segmentCount : 0);
|
|
278
|
+
const timestampCoverage = meta.timestampCoverage
|
|
279
|
+
?? (meta.segmentCount > 0 ? timestampedSegmentCount / meta.segmentCount : 1);
|
|
280
|
+
const excludedUntimestampedSegmentCount = meta.excludedUntimestampedSegmentCount ?? 0;
|
|
281
|
+
const timestampNotice = timestampCoverage < 1 || excludedUntimestampedSegmentCount > 0
|
|
282
|
+
? `<div style="margin:12px 0;padding:10px 12px;border:1px solid var(--yellow);border-radius:8px;background:var(--yellow-bg);color:var(--text-secondary);font-size:13px">
|
|
283
|
+
<strong style="color:var(--yellow)">${zh ? '时间范围不完整' : 'Incomplete time coverage'}</strong>
|
|
284
|
+
<span style="margin-left:8px">${zh
|
|
285
|
+
? `${meta.segmentCount} 个入选片段中,${timestampedSegmentCount} 个具有可观测时间;时间筛选另排除了 ${excludedUntimestampedSegmentCount} 个无时间戳片段。`
|
|
286
|
+
: `${timestampedSegmentCount} of ${meta.segmentCount} selected segments have observed timestamps; time filtering excluded ${excludedUntimestampedSegmentCount} additional untimestamped segments.`}</span>
|
|
287
|
+
</div>`
|
|
288
|
+
: '';
|
|
234
289
|
const body = [
|
|
235
290
|
statsBar,
|
|
291
|
+
ingestionNotice,
|
|
292
|
+
timestampNotice,
|
|
236
293
|
renderWatermark(report, lang),
|
|
237
294
|
`<div class="rs-panel">
|
|
238
295
|
<div class="rs-panel-title">${zh ? '各 skill 健康度' : 'Per-skill health'} <small>${zh ? '按使用量降序' : 'sorted by usage'}</small></div>
|
package/dist/renderer/summary.js
CHANGED
|
@@ -4,6 +4,7 @@ import { generateAnalysisSummary } from '../analysis/report-diagnostics.js';
|
|
|
4
4
|
import { pValueCategory } from '../eval-core/statistics.js';
|
|
5
5
|
import { ciLevelLabel } from '../eval-core/bootstrap.js';
|
|
6
6
|
import { computeVerdict, medianStabilityCV, STABILITY_UNSTABLE_CV, ENSEMBLE_STRONG_PEARSON, ENSEMBLE_DISSENT_PEARSON } from '../eval-core/verdict.js';
|
|
7
|
+
import { ownRecordValue, setOwnRecordValue } from '../shared/record-count.js';
|
|
7
8
|
/**
|
|
8
9
|
* Verdict pill — sticky banner at the top of the HTML report giving the same
|
|
9
10
|
* one-line conclusion as the `omk eval` CLI. Both surfaces share the
|
|
@@ -522,9 +523,13 @@ export function renderSummaryCards(variants, summary, lang, variance) {
|
|
|
522
523
|
// Cost — 只算 execution cost(judge cost 是工具开销,不算 skill 成本)。
|
|
523
524
|
const execCost = s.totalExecCostUSD || 0;
|
|
524
525
|
const costReported = s.execCostReported !== false;
|
|
525
|
-
const
|
|
526
|
+
const tokenCoverage = s.tokenUsageCoverageRate ?? 1;
|
|
527
|
+
const hasMeasuredTokens = tokenCoverage > 0;
|
|
528
|
+
const hasCost = (s.totalSamples || 0) > 0;
|
|
526
529
|
const costPerSample = (s.totalSamples || 0) > 0 ? execCost / s.totalSamples : execCost;
|
|
527
|
-
const tokenDetail =
|
|
530
|
+
const tokenDetail = hasMeasuredTokens
|
|
531
|
+
? `${fmtNum(s.avgTotalTokens)} tokens/${t('tokPerReq', lang).replace('tokens/', '')}${tokenCoverage < 1 ? ` · ${Math.round(tokenCoverage * 100)}% ${lang === 'zh' ? '覆盖' : 'coverage'}` : ''}`
|
|
532
|
+
: lang === 'zh' ? 'tokens 未观测' : 'tokens unobserved';
|
|
528
533
|
const costDim = hasCost
|
|
529
534
|
? { label: dimText('dimCost', lang), value: costReported ? fmtCost(execCost) : fmtCost(0, false), valueColor: costReported ? 'var(--text-primary)' : 'var(--text-muted)', primary: false, bg: '', detailLines: [tokenDetail], title: costReported ? undefined : costUnreportedTooltip, icon: 'cost' }
|
|
530
535
|
: { label: dimText('dimCost', lang), value: 'N/A', valueColor: 'var(--text-muted)', primary: false, bg: '', detailLines: [], icon: 'cost' };
|
|
@@ -1671,8 +1676,22 @@ export function renderAgentOverview(variants, summary, lang) {
|
|
|
1671
1676
|
return '';
|
|
1672
1677
|
const color = COLORS[i % COLORS.length];
|
|
1673
1678
|
const avgTools = stats.avgToolCalls ?? 0;
|
|
1674
|
-
const
|
|
1675
|
-
|
|
1679
|
+
const outcomeSuffix = [
|
|
1680
|
+
stats.avgToolCancelled
|
|
1681
|
+
? lang === 'zh' ? `${stats.avgToolCancelled} 次取消` : `${stats.avgToolCancelled} cancelled`
|
|
1682
|
+
: '',
|
|
1683
|
+
stats.avgToolUnknown
|
|
1684
|
+
? lang === 'zh' ? `${stats.avgToolUnknown} 次状态未知` : `${stats.avgToolUnknown} unknown`
|
|
1685
|
+
: '',
|
|
1686
|
+
].filter(Boolean).map((value) => ` · ${value}`).join('');
|
|
1687
|
+
const successRate = stats.toolSuccessRate != null
|
|
1688
|
+
? `${(stats.toolSuccessRate * 100).toFixed(0)}% OK${outcomeSuffix}`
|
|
1689
|
+
: stats.avgToolCancelled || stats.avgToolUnknown
|
|
1690
|
+
? outcomeSuffix.replace(/^ · /, '')
|
|
1691
|
+
: lang === 'zh' ? '状态未知' : 'unknown';
|
|
1692
|
+
const successRateColor = stats.toolSuccessRate == null
|
|
1693
|
+
? 'var(--text-muted)'
|
|
1694
|
+
: stats.toolSuccessRate >= 0.8 ? 'var(--green)' : 'var(--red)';
|
|
1676
1695
|
const turns = stats.avgFullNumTurns ?? stats.avgNumTurns ?? 0;
|
|
1677
1696
|
const distributionEntries = Object.entries(stats.toolDistribution || {}).sort((a, b) => b[1] - a[1]);
|
|
1678
1697
|
const distSummary = distributionEntries.length > 0
|
|
@@ -1686,7 +1705,7 @@ export function renderAgentOverview(variants, summary, lang) {
|
|
|
1686
1705
|
<span class="ctx-sep">·</span>
|
|
1687
1706
|
<span>${avgTools} ${lang === 'zh' ? '工具/次' : 'tools/req'}</span>
|
|
1688
1707
|
<span class="ctx-sep">·</span>
|
|
1689
|
-
<span style="color:${successRateColor}">${successRate}
|
|
1708
|
+
<span style="color:${successRateColor}">${successRate}</span>
|
|
1690
1709
|
</span>
|
|
1691
1710
|
</div>`;
|
|
1692
1711
|
}).join('');
|
|
@@ -1726,7 +1745,8 @@ export function renderCoverageSection(coverage, lang) {
|
|
|
1726
1745
|
const uncoveredByType = {};
|
|
1727
1746
|
for (const entry of knowledgeCoverage.entries.filter((item) => !item.accessed)) {
|
|
1728
1747
|
const category = entry.path.startsWith('repos/') ? 'code' : entry.type;
|
|
1729
|
-
(uncoveredByType
|
|
1748
|
+
const files = ownRecordValue(uncoveredByType, category) ?? setOwnRecordValue(uncoveredByType, category, []);
|
|
1749
|
+
files.push(entry.path);
|
|
1730
1750
|
}
|
|
1731
1751
|
const hintLines = [];
|
|
1732
1752
|
const typeLabels = lang === 'zh'
|
|
@@ -1850,7 +1870,8 @@ export function renderKnowledgeInteractionSection(coverage, gapReports, lang) {
|
|
|
1850
1870
|
const uncoveredByType = {};
|
|
1851
1871
|
for (const entry of cov.entries.filter((item) => !item.accessed)) {
|
|
1852
1872
|
const category = entry.path.startsWith('repos/') ? 'code' : entry.type;
|
|
1853
|
-
(uncoveredByType
|
|
1873
|
+
const files = ownRecordValue(uncoveredByType, category) ?? setOwnRecordValue(uncoveredByType, category, []);
|
|
1874
|
+
files.push(entry.path);
|
|
1854
1875
|
}
|
|
1855
1876
|
const typeLabels = lang === 'zh'
|
|
1856
1877
|
? { principle: '原则文件', semantic: '语义索引', design: '设计文档', code: '代码路径', script: '脚本工具', other: '其他知识' }
|
package/dist/renderer/table.js
CHANGED
|
@@ -113,11 +113,28 @@ function renderVariantBlock(d, variantName, comparedTo, lang) {
|
|
|
113
113
|
const totalMs = d.timing?.totalMs || d.durationMs;
|
|
114
114
|
const metaParts = [];
|
|
115
115
|
if (d.numToolCalls != null && d.numToolCalls > 0) {
|
|
116
|
-
const
|
|
117
|
-
const
|
|
118
|
-
|
|
116
|
+
const hasResolvedOutcome = d.toolSuccessRate != null;
|
|
117
|
+
const srColor = hasResolvedOutcome
|
|
118
|
+
? d.toolSuccessRate >= 0.8 ? 'var(--green)' : 'var(--red)'
|
|
119
|
+
: 'var(--text-muted)';
|
|
120
|
+
const outcomeSuffix = [
|
|
121
|
+
(d.numToolCancelled ?? 0) > 0
|
|
122
|
+
? lang === 'zh' ? `${d.numToolCancelled} 次取消` : `${d.numToolCancelled} cancelled`
|
|
123
|
+
: '',
|
|
124
|
+
(d.numToolUnknown ?? 0) > 0
|
|
125
|
+
? lang === 'zh' ? `${d.numToolUnknown} 次状态未知` : `${d.numToolUnknown} unknown`
|
|
126
|
+
: '',
|
|
127
|
+
].filter(Boolean).map((value) => ` · ${value}`).join('');
|
|
128
|
+
const outcomeText = hasResolvedOutcome
|
|
129
|
+
? `${(d.toolSuccessRate * 100).toFixed(0)}% OK${outcomeSuffix}`
|
|
130
|
+
: outcomeSuffix
|
|
131
|
+
? outcomeSuffix.replace(/^ · /, '')
|
|
132
|
+
: lang === 'zh'
|
|
133
|
+
? `${d.numToolCalls} 次状态未知`
|
|
134
|
+
: `${d.numToolCalls} unknown`;
|
|
135
|
+
metaParts.push(`<div class="sc-meta-row"><span class="sc-meta-name">${lang === 'zh' ? '工具调用' : 'Tools'}</span>${d.numToolCalls} <span style="color:${srColor}">(${outcomeText})</span></div>`);
|
|
119
136
|
}
|
|
120
|
-
metaParts.push(`<div class="sc-meta-row"><span class="sc-meta-name">${lang === 'zh' ? 'Tokens' : 'Tokens'}</span>${fmtNum(d.totalTokens)}</div>`);
|
|
137
|
+
metaParts.push(`<div class="sc-meta-row"><span class="sc-meta-name">${lang === 'zh' ? 'Tokens' : 'Tokens'}</span>${d.tokenUsageReportedByExecutor === false ? '—' : fmtNum(d.totalTokens)}</div>`);
|
|
121
138
|
metaParts.push(`<div class="sc-meta-row"><span class="sc-meta-name">${lang === 'zh' ? '耗时' : 'Duration'}</span>${fmtDuration(totalMs)}</div>`);
|
|
122
139
|
// Error message
|
|
123
140
|
const errorHtml = !d.ok && d.error
|
|
@@ -49,6 +49,7 @@ export declare function sampleVerdict(details: AssertionDetail[] | undefined): '
|
|
|
49
49
|
* 避免使用者误以为是要修的真 bug。
|
|
50
50
|
*/
|
|
51
51
|
export declare function isTripwireSample(snapshot: SampleSnapshot | undefined, variant: VariantResult): boolean;
|
|
52
|
+
export declare function renderTurnContent(turn: TurnInfo, lang: Lang): string;
|
|
52
53
|
export declare function renderTestView(report: EvaluationReport, lang: Lang): string;
|
|
53
54
|
/** 单测视角的 CSS。
|
|
54
55
|
*
|
|
@@ -15,6 +15,7 @@
|
|
|
15
15
|
*/
|
|
16
16
|
import { e, t } from './layout.js';
|
|
17
17
|
import { icon } from './icons.js';
|
|
18
|
+
import { isToolCallFailure, toolCallStatus } from '../shared/tool-call-status.js';
|
|
18
19
|
// ──────────────────────────────────────────────────────────────────
|
|
19
20
|
// Assertion → Turn 绑定:扫执行轨迹找断言匹配的 turn 编号
|
|
20
21
|
// ──────────────────────────────────────────────────────────────────
|
|
@@ -182,10 +183,30 @@ function renderDesignSection(snapshot, lang) {
|
|
|
182
183
|
<div class="tv-mocks">${snapshot.mocks.map((m, i) => renderMock(m, i + 1)).join('')}</div>
|
|
183
184
|
</details>`
|
|
184
185
|
: '';
|
|
186
|
+
const executionContract = snapshot.cwd
|
|
187
|
+
|| snapshot.environment
|
|
188
|
+
|| snapshot.mocksStrict !== undefined
|
|
189
|
+
|| (snapshot.allowedTools && snapshot.allowedTools.length > 0)
|
|
190
|
+
|| (snapshot.expectedTools && snapshot.expectedTools.length > 0)
|
|
191
|
+
? `<details class="tv-collapsible"><summary>${lang === 'zh' ? '执行契约' : 'Execution contract'}</summary>
|
|
192
|
+
<pre class="tv-pre">${e(JSON.stringify({
|
|
193
|
+
...(snapshot.cwd ? { cwd: snapshot.cwd } : {}),
|
|
194
|
+
...(snapshot.environment ? { environment: snapshot.environment } : {}),
|
|
195
|
+
...(snapshot.mocksStrict !== undefined ? { mocksStrict: snapshot.mocksStrict } : {}),
|
|
196
|
+
...(snapshot.allowedTools ? { allowedTools: snapshot.allowedTools } : {}),
|
|
197
|
+
...(snapshot.expectedTools ? { expectedTools: snapshot.expectedTools } : {}),
|
|
198
|
+
}, null, 2))}</pre>
|
|
199
|
+
</details>`
|
|
200
|
+
: '';
|
|
201
|
+
const dimensionsBlock = snapshot.dimensions && Object.keys(snapshot.dimensions).length > 0
|
|
202
|
+
? `<details class="tv-collapsible"><summary>${lang === 'zh' ? '评分维度' : 'Scoring dimensions'}</summary><pre class="tv-pre">${e(JSON.stringify(snapshot.dimensions, null, 2))}</pre></details>`
|
|
203
|
+
: '';
|
|
185
204
|
return `<div class="tv-section--design-body">
|
|
186
205
|
<div class="tv-prompt"><strong>${lang === 'zh' ? '提示词(Prompt):' : 'Prompt:'}</strong> <span>${e(snapshot.prompt)}</span></div>
|
|
187
206
|
${snapshot.rubric ? `<div class="tv-rubric"><strong>${lang === 'zh' ? '评分标准(Rubric):' : 'Rubric:'}</strong> <span>${e(snapshot.rubric)}</span></div>` : ''}
|
|
188
207
|
${snapshot.context ? `<details class="tv-collapsible"><summary>${lang === 'zh' ? '上下文(Context)' : 'Context'}</summary><pre class="tv-pre">${e(snapshot.context)}</pre></details>` : ''}
|
|
208
|
+
${executionContract}
|
|
209
|
+
${dimensionsBlock}
|
|
189
210
|
${meta.length > 0 ? `<div class="tv-meta-pills">${meta.join(' ')}</div>` : ''}
|
|
190
211
|
${expectedAssertions ? `<div class="tv-expected">
|
|
191
212
|
<div class="tv-expected-h">${lang === 'zh' ? '期望(assertions):' : 'Expected (assertions):'}</div>
|
|
@@ -270,17 +291,14 @@ function renderExecutionTrace(variant, detailsByTurn, matchedMockNumbersByTurn,
|
|
|
270
291
|
}
|
|
271
292
|
const stepsHtml = turns.map((turn, i) => {
|
|
272
293
|
const turnNum = i + 1;
|
|
273
|
-
const
|
|
274
|
-
const
|
|
275
|
-
? `<details class="tv-thinking"><summary>${lang === 'zh' ? '展开 thinking' : 'show thinking'} (${turn.content.length} ${lang === 'zh' ? '字' : 'chars'})</summary><div class="tv-thinking-body">${e(turn.content)}</div></details>`
|
|
276
|
-
: '';
|
|
277
|
-
const calls = (turn.toolCalls ?? []).map((tc) => renderToolCall(tc, turnNum, matchedMockNumbersByTurn.get(turnNum) ?? [])).join('');
|
|
294
|
+
const contentBlock = renderTurnContent(turn, lang);
|
|
295
|
+
const calls = (turn.toolCalls ?? []).map((tc) => renderToolCall(tc, turnNum, matchedMockNumbersByTurn.get(turnNum) ?? [], lang)).join('');
|
|
278
296
|
const assertsForTurn = (detailsByTurn.get(turnNum) ?? []);
|
|
279
297
|
const turnAssertsHtml = assertsForTurn.map((d) => `<div class="tv-trace-assert ${d.passed ? 'tv-assert--pass' : 'tv-assert--fail'}">${d.passed ? '✅' : '❌'} <code>${e(d.type)}</code> <code class="tv-assert-value">${e(typeof d.value === 'object' ? JSON.stringify(d.value) : String(d.value))}</code></div>`).join('');
|
|
280
298
|
const durationTag = turn.durationMs ? `<span class="tv-turn-dur">${(turn.durationMs / 1000).toFixed(1)}s</span>` : '';
|
|
281
299
|
return `<div class="tv-turn">
|
|
282
300
|
<div class="tv-turn-h">Turn ${turnNum} · <code>${e(turn.role)}</code> ${durationTag}</div>
|
|
283
|
-
${
|
|
301
|
+
${contentBlock}
|
|
284
302
|
${calls}
|
|
285
303
|
${turnAssertsHtml}
|
|
286
304
|
</div>`;
|
|
@@ -294,14 +312,31 @@ function renderExecutionTrace(variant, detailsByTurn, matchedMockNumbersByTurn,
|
|
|
294
312
|
${finalOutput}
|
|
295
313
|
</div>`;
|
|
296
314
|
}
|
|
297
|
-
function
|
|
315
|
+
export function renderTurnContent(turn, lang) {
|
|
316
|
+
if (!turn.content)
|
|
317
|
+
return '';
|
|
318
|
+
if (turn.role === 'assistant') {
|
|
319
|
+
return `<details class="tv-thinking"><summary>${lang === 'zh' ? '展开 thinking' : 'show thinking'} (${turn.content.length} ${lang === 'zh' ? '字' : 'chars'})</summary><div class="tv-thinking-body">${e(turn.content)}</div></details>`;
|
|
320
|
+
}
|
|
321
|
+
const label = turn.role === 'user'
|
|
322
|
+
? lang === 'zh' ? '用户输入' : 'User input'
|
|
323
|
+
: lang === 'zh' ? '工具结果' : 'Tool result';
|
|
324
|
+
return `<div class="tv-turn-content"><div class="tv-turn-content-h">${label}</div><pre class="tv-pre">${e(turn.content)}</pre></div>`;
|
|
325
|
+
}
|
|
326
|
+
function renderToolCall(tc, _turnNum, mockHitsHere, lang) {
|
|
298
327
|
const inputRepr = typeof tc.input === 'string' ? tc.input : JSON.stringify(tc.input ?? '');
|
|
299
328
|
const outputRepr = typeof tc.output === 'string' ? tc.output : JSON.stringify(tc.output ?? '');
|
|
300
329
|
const mockBadge = mockHitsHere.length > 0
|
|
301
330
|
? `<span class="tv-mock-badge" title="matched mock #${mockHitsHere.join(',#')}">✓ mock#${mockHitsHere.join(',#')}</span>`
|
|
302
331
|
: '';
|
|
303
|
-
|
|
304
|
-
|
|
332
|
+
const status = toolCallStatus(tc);
|
|
333
|
+
const statusLabel = status === 'unknown'
|
|
334
|
+
? `<span class="tv-status">${e(lang === 'zh' ? '状态未知' : 'unknown status')}</span>`
|
|
335
|
+
: status === 'cancelled'
|
|
336
|
+
? `<span class="tv-status">${e(lang === 'zh' ? '已取消' : 'cancelled')}</span>`
|
|
337
|
+
: '';
|
|
338
|
+
return `<div class="tv-call ${isToolCallFailure(tc) ? 'tv-call--err' : ''}">
|
|
339
|
+
<div class="tv-call-h">🔧 <code>${e(tc.tool)}</code> ${statusLabel} ${mockBadge}</div>
|
|
305
340
|
<details><summary>input</summary><pre class="tv-pre">${e(inputRepr.length > 1000 ? inputRepr.slice(0, 1000) + '…' : inputRepr)}</pre></details>
|
|
306
341
|
<details><summary>output</summary><pre class="tv-pre">${e(outputRepr.length > 1000 ? outputRepr.slice(0, 1000) + '…' : outputRepr)}</pre></details>
|
|
307
342
|
</div>`;
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* 三数据源合并、按 id dedup(live 盖卡片,优先级 项目 > 全局 > 卡片):
|
|
5
5
|
* - live(当前项目 `.omk/reports`) + live(全局 `~/.oh-my-knowledge/reports`):完整真身,永远新鲜(无需 backfill);
|
|
6
6
|
* - 索引卡片(state/artifact-index/report/):覆盖**别的项目**的报告(当前 studio 的 cwd live 扫不到),
|
|
7
|
-
*
|
|
7
|
+
* 卡片只用于发现 `card.path`;列表与详情都回源并严格解析真身。
|
|
8
8
|
*
|
|
9
9
|
* 与 createOverlayReportStore(项目盖全局、记录优先)的区别:这里是**机器级 merge**(看全本机),不是项目优先;
|
|
10
10
|
* 可比性红线在比较层(verdict / findByVariant 锁同口径)而非视图层,故跨项目 merge 列表合法。
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
import { existsSync, statSync, readdirSync } from 'node:fs';
|
|
14
14
|
import { dirname, join } from 'node:path';
|
|
15
15
|
import { createFileStore } from './report-store.js';
|
|
16
|
-
import { artifactIndexDir, listReportCards, listLiveReportCards,
|
|
16
|
+
import { artifactIndexDir, listReportCards, listLiveReportCards, removeReportCard, cardTargetSentinel } from '../eval-core/artifact-index.js';
|
|
17
17
|
function isEvaluation(r) {
|
|
18
18
|
return r.kind === 'evaluation';
|
|
19
19
|
}
|
|
@@ -47,19 +47,20 @@ export function createIndexedReportStore({ projectDir, globalDir }) {
|
|
|
47
47
|
return 'none';
|
|
48
48
|
}
|
|
49
49
|
}
|
|
50
|
-
function cardDocs() {
|
|
50
|
+
async function cardDocs() {
|
|
51
51
|
const fp = cardFingerprint();
|
|
52
52
|
if (fp === cachedFp && cachedCards)
|
|
53
|
-
return cachedCards;
|
|
54
|
-
//
|
|
55
|
-
//
|
|
56
|
-
const
|
|
53
|
+
return structuredClone(cachedCards);
|
|
54
|
+
// 卡片只是发现指针。机器级 list 也必须回源并走 createFileStore 的 canonical parser,
|
|
55
|
+
// 不能把索引里的 meta/summary 壳伪装成完整 ReportDocument。
|
|
56
|
+
const loaded = await Promise.all(listLiveReportCards().map((card) => createFileStore(dirname(card.path)).get(card.id)));
|
|
57
|
+
const docs = loaded.filter((doc) => doc !== null);
|
|
57
58
|
cachedFp = fp;
|
|
58
59
|
cachedCards = docs;
|
|
59
|
-
return docs;
|
|
60
|
+
return structuredClone(docs);
|
|
60
61
|
}
|
|
61
62
|
async function list() {
|
|
62
|
-
const [proj, glob] = await Promise.all([project.list(), global.list()]);
|
|
63
|
+
const [proj, glob, indexed] = await Promise.all([project.list(), global.list(), cardDocs()]);
|
|
63
64
|
const seen = new Set();
|
|
64
65
|
const merged = [];
|
|
65
66
|
for (const r of proj) {
|
|
@@ -74,7 +75,7 @@ export function createIndexedReportStore({ projectDir, globalDir }) {
|
|
|
74
75
|
merged.push(r);
|
|
75
76
|
}
|
|
76
77
|
}
|
|
77
|
-
for (const r of
|
|
78
|
+
for (const r of indexed) {
|
|
78
79
|
if (!seen.has(r.id)) {
|
|
79
80
|
seen.add(r.id);
|
|
80
81
|
merged.push(r);
|
|
@@ -88,19 +89,14 @@ export function createIndexedReportStore({ projectDir, globalDir }) {
|
|
|
88
89
|
const live = (await project.get(id)) ?? (await global.get(id));
|
|
89
90
|
if (live)
|
|
90
91
|
return live;
|
|
91
|
-
// 别项目:按卡片 path
|
|
92
|
+
// 别项目:按卡片 path 读真身。悬空或损坏都 fail closed,不从 scratch 卡片伪造报告壳。
|
|
92
93
|
const card = listReportCards().find((c) => c.id === id);
|
|
93
94
|
if (!card)
|
|
94
95
|
return null;
|
|
95
|
-
|
|
96
|
-
return fromFile ?? cardToReportDocument(card);
|
|
96
|
+
return createFileStore(dirname(card.path)).get(id);
|
|
97
97
|
}
|
|
98
98
|
async function exists(id) {
|
|
99
|
-
|
|
100
|
-
return true;
|
|
101
|
-
if (await global.exists(id))
|
|
102
|
-
return true;
|
|
103
|
-
return listReportCards().some((c) => c.id === id);
|
|
99
|
+
return (await get(id)) !== null;
|
|
104
100
|
}
|
|
105
101
|
async function save(id, report) {
|
|
106
102
|
return project.save(id, report);
|
package/dist/server/job-store.js
CHANGED
|
@@ -1,11 +1,18 @@
|
|
|
1
1
|
import { promises as fs } from 'node:fs';
|
|
2
2
|
import { existsSync, mkdirSync } from 'node:fs';
|
|
3
3
|
import { join } from 'node:path';
|
|
4
|
+
import { writeJsonFileAtomicAsync } from '../shared/atomic-json.js';
|
|
5
|
+
import { isSafeEvaluationJobId, isValidEvaluationJob, } from '../shared/evaluation-job.js';
|
|
6
|
+
import { KeyedMutex } from '../shared/keyed-mutex.js';
|
|
4
7
|
async function listJsonFiles(dir) {
|
|
5
8
|
try {
|
|
6
9
|
const entries = await fs.readdir(dir, { withFileTypes: true });
|
|
7
10
|
return entries
|
|
8
|
-
.filter((entry) =>
|
|
11
|
+
.filter((entry) => {
|
|
12
|
+
if (!entry.isFile() || !entry.name.endsWith('.json'))
|
|
13
|
+
return false;
|
|
14
|
+
return isSafeJobId(entry.name.slice(0, -'.json'.length));
|
|
15
|
+
})
|
|
9
16
|
.map((entry) => entry.name);
|
|
10
17
|
}
|
|
11
18
|
catch {
|
|
@@ -13,53 +20,84 @@ async function listJsonFiles(dir) {
|
|
|
13
20
|
}
|
|
14
21
|
}
|
|
15
22
|
export function createFileJobStore(dir) {
|
|
23
|
+
const mutations = new KeyedMutex();
|
|
16
24
|
if (!existsSync(dir)) {
|
|
17
25
|
mkdirSync(dir, { recursive: true });
|
|
18
26
|
}
|
|
19
27
|
async function list() {
|
|
20
28
|
const files = await listJsonFiles(dir);
|
|
21
|
-
const jobs = await Promise.all(files.map(async (file) =>
|
|
29
|
+
const jobs = (await Promise.all(files.map(async (file) => {
|
|
30
|
+
const id = file.slice(0, -'.json'.length);
|
|
31
|
+
try {
|
|
32
|
+
return parseJob(await fs.readFile(join(dir, file), 'utf-8'), id);
|
|
33
|
+
}
|
|
34
|
+
catch {
|
|
35
|
+
return null;
|
|
36
|
+
}
|
|
37
|
+
}))).filter((job) => job !== null);
|
|
22
38
|
return jobs.sort((a, b) => (b.createdAt || '').localeCompare(a.createdAt || ''));
|
|
23
39
|
}
|
|
24
40
|
async function get(id) {
|
|
41
|
+
const path = jobFilePath(dir, id);
|
|
42
|
+
if (!path)
|
|
43
|
+
return null;
|
|
25
44
|
try {
|
|
26
|
-
return
|
|
45
|
+
return parseJob(await fs.readFile(path, 'utf-8'), id);
|
|
27
46
|
}
|
|
28
47
|
catch {
|
|
29
48
|
return null;
|
|
30
49
|
}
|
|
31
50
|
}
|
|
51
|
+
async function saveUnlocked(id, job) {
|
|
52
|
+
const path = jobFilePath(dir, id);
|
|
53
|
+
if (!path || !isValidEvaluationJob(job, id))
|
|
54
|
+
throw new Error('invalid job');
|
|
55
|
+
await writeJsonFileAtomicAsync(path, job);
|
|
56
|
+
}
|
|
32
57
|
async function save(id, job) {
|
|
33
|
-
await
|
|
58
|
+
await mutations.run(id, () => saveUnlocked(id, job));
|
|
34
59
|
}
|
|
35
60
|
async function update(id, mutator) {
|
|
36
|
-
|
|
37
|
-
if (!current)
|
|
61
|
+
if (!isSafeJobId(id))
|
|
38
62
|
return null;
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
63
|
+
return mutations.run(id, async () => {
|
|
64
|
+
const current = await get(id);
|
|
65
|
+
if (!current)
|
|
66
|
+
return null;
|
|
67
|
+
const updated = mutator(current);
|
|
68
|
+
await saveUnlocked(id, updated);
|
|
69
|
+
return updated;
|
|
70
|
+
});
|
|
42
71
|
}
|
|
43
72
|
async function remove(id) {
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
return
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
73
|
+
const path = jobFilePath(dir, id);
|
|
74
|
+
if (!path)
|
|
75
|
+
return false;
|
|
76
|
+
return mutations.run(id, async () => {
|
|
77
|
+
try {
|
|
78
|
+
await fs.unlink(path);
|
|
79
|
+
return true;
|
|
80
|
+
}
|
|
81
|
+
catch (err) {
|
|
82
|
+
const fsError = err;
|
|
83
|
+
if (fsError.code === 'ENOENT')
|
|
84
|
+
return false;
|
|
85
|
+
throw err;
|
|
86
|
+
}
|
|
87
|
+
});
|
|
54
88
|
}
|
|
55
89
|
async function exists(id) {
|
|
56
|
-
|
|
57
|
-
await fs.access(join(dir, `${id}.json`));
|
|
58
|
-
return true;
|
|
59
|
-
}
|
|
60
|
-
catch {
|
|
61
|
-
return false;
|
|
62
|
-
}
|
|
90
|
+
return (await get(id)) !== null;
|
|
63
91
|
}
|
|
64
92
|
return { list, get, save, update, remove, exists };
|
|
65
93
|
}
|
|
94
|
+
function isSafeJobId(id) {
|
|
95
|
+
return isSafeEvaluationJobId(id);
|
|
96
|
+
}
|
|
97
|
+
function jobFilePath(dir, id) {
|
|
98
|
+
return isSafeJobId(id) ? join(dir, `${id}.json`) : null;
|
|
99
|
+
}
|
|
100
|
+
function parseJob(raw, expectedId) {
|
|
101
|
+
const value = JSON.parse(raw);
|
|
102
|
+
return isValidEvaluationJob(value, expectedId) ? value : null;
|
|
103
|
+
}
|