oh-my-knowledge 0.47.0 → 0.49.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +50 -18
- package/README.zh.md +55 -23
- package/dist/analysis/coverage-analyzer.d.ts +1 -0
- package/dist/analysis/coverage-analyzer.js +125 -62
- package/dist/analysis/failure-clusterer.js +2 -1
- package/dist/analysis/gap-analyzer.d.ts +2 -2
- package/dist/analysis/gap-analyzer.js +13 -3
- package/dist/analysis/hedging-classifier.d.ts +2 -2
- package/dist/analysis/hedging-classifier.js +3 -4
- package/dist/analysis/report-diagnostics.js +9 -7
- package/dist/analysis/sample-diagnostics.js +6 -6
- package/dist/artifact-graph/doctor.js +15 -7
- package/dist/assets/agent-skills/omk/SKILL.md +27 -7
- package/dist/assets/agent-skills/omk/references/commands.md +19 -17
- package/dist/authoring/evolver.d.ts +10 -6
- package/dist/authoring/evolver.js +496 -83
- package/dist/authoring/generator.d.ts +3 -3
- package/dist/authoring/generator.js +5 -10
- package/dist/authoring/sample-fixer.d.ts +8 -6
- package/dist/authoring/sample-fixer.js +76 -5
- package/dist/cli/commands/doctor.js +31 -14
- package/dist/cli/commands/eval/index.d.ts +3 -0
- package/dist/cli/commands/eval/index.js +163 -17
- package/dist/cli/commands/evolve.d.ts +4 -4
- package/dist/cli/commands/evolve.js +27 -13
- package/dist/cli/commands/init.js +16 -3
- package/dist/cli/commands/observe/inbox.js +44 -21
- package/dist/cli/commands/observe/index.js +20 -11
- package/dist/cli/commands/observe/ingest.d.ts +3 -0
- package/dist/cli/commands/observe/ingest.js +35 -4
- package/dist/cli/commands/sample.d.ts +9 -3
- package/dist/cli/commands/sample.js +91 -74
- package/dist/cli/lib/cmd-flags.d.ts +1 -0
- package/dist/cli/lib/codex-model-hint.d.ts +9 -0
- package/dist/cli/lib/codex-model-hint.js +45 -0
- package/dist/cli/lib/generation-failure-hint.d.ts +2 -0
- package/dist/cli/lib/generation-failure-hint.js +61 -0
- package/dist/cli/lib/i18n-dict/common.d.ts +1 -1
- package/dist/cli/lib/i18n-dict/common.js +4 -0
- package/dist/cli/lib/i18n-dict/gen.d.ts +1 -1
- package/dist/cli/lib/i18n-dict/gen.js +38 -6
- package/dist/cli/lib/i18n-dict/help.js +6 -6
- package/dist/cli/lib/i18n-dict/init.d.ts +1 -1
- package/dist/cli/lib/i18n-dict/init.js +13 -9
- package/dist/cli/lib/i18n-dict/run.d.ts +1 -1
- package/dist/cli/lib/i18n-dict/run.js +34 -2
- package/dist/cli/lib/llm-failure-classifier.d.ts +2 -0
- package/dist/cli/lib/llm-failure-classifier.js +8 -0
- package/dist/cli/lib/parse-run-config.d.ts +6 -5
- package/dist/cli/lib/parse-run-config.js +16 -9
- package/dist/cli/lib/runtime-defaults.d.ts +21 -0
- package/dist/cli/lib/runtime-defaults.js +79 -0
- package/dist/diagnosis/observe-mapper.js +14 -15
- package/dist/diagnosis/observe-producer.js +3 -1
- package/dist/diagnosis/studio-projection.js +14 -7
- package/dist/diagnosis/types.d.ts +2 -0
- package/dist/diagnosis/types.js +12 -0
- package/dist/doctor/endpoint-rule.js +2 -1
- package/dist/eval-core/artifact-file-names.js +18 -1
- package/dist/eval-core/artifact-index.d.ts +7 -11
- package/dist/eval-core/artifact-index.js +139 -80
- package/dist/eval-core/cache.d.ts +12 -3
- package/dist/eval-core/cache.js +89 -29
- package/dist/eval-core/comparability.js +10 -6
- package/dist/eval-core/evaluation-execution.d.ts +2 -1
- package/dist/eval-core/evaluation-execution.js +122 -37
- package/dist/eval-core/evaluation-job.d.ts +4 -1
- package/dist/eval-core/evaluation-job.js +4 -1
- package/dist/eval-core/evaluation-reporting.d.ts +15 -13
- package/dist/eval-core/evaluation-reporting.js +54 -52
- package/dist/eval-core/execution-strategy.d.ts +2 -0
- package/dist/eval-core/execution-strategy.js +11 -9
- package/dist/eval-core/fact-checker.js +15 -7
- package/dist/eval-core/holdout.js +3 -2
- package/dist/eval-core/judge-independence.d.ts +2 -2
- package/dist/eval-core/mock-hook.cjs +23 -6
- package/dist/eval-core/mocks-runtime.js +30 -8
- package/dist/eval-core/report-document.d.ts +12 -0
- package/dist/eval-core/report-document.js +1151 -0
- package/dist/eval-core/report-extensions.d.ts +4 -0
- package/dist/eval-core/report-extensions.js +500 -0
- package/dist/eval-core/report-file-migration.js +7 -2
- package/dist/eval-core/resume-compatibility.d.ts +31 -0
- package/dist/eval-core/resume-compatibility.js +141 -0
- package/dist/eval-core/sample-fingerprint.d.ts +12 -0
- package/dist/eval-core/sample-fingerprint.js +193 -0
- package/dist/eval-core/schema.js +86 -31
- package/dist/eval-core/verdict.d.ts +8 -4
- package/dist/eval-core/verdict.js +24 -10
- package/dist/eval-workflows/batch-evaluation-workflow.d.ts +2 -1
- package/dist/eval-workflows/batch-evaluation-workflow.js +25 -12
- package/dist/eval-workflows/evaluation-pipeline/preflight-warnings.d.ts +10 -5
- package/dist/eval-workflows/evaluation-pipeline/preflight-warnings.js +58 -21
- package/dist/eval-workflows/evaluation-pipeline/report-finalize.js +3 -1
- package/dist/eval-workflows/evaluation-pipeline/run-state.d.ts +4 -1
- package/dist/eval-workflows/evaluation-pipeline/run-state.js +4 -1
- package/dist/eval-workflows/evaluation-pipeline/test-set-hash.d.ts +6 -5
- package/dist/eval-workflows/evaluation-pipeline/test-set-hash.js +17 -10
- package/dist/eval-workflows/evaluation-pipeline.js +12 -7
- package/dist/eval-workflows/run-evaluation.d.ts +9 -7
- package/dist/eval-workflows/run-evaluation.js +79 -51
- package/dist/executors/anthropic-api.js +65 -9
- package/dist/executors/claude-cli.js +16 -79
- package/dist/executors/claude-protocol.d.ts +28 -0
- package/dist/executors/claude-protocol.js +180 -0
- package/dist/executors/claude-sdk-trace.js +56 -28
- package/dist/executors/claude-sdk.d.ts +1 -0
- package/dist/executors/claude-sdk.js +39 -93
- package/dist/executors/codex-cli-trace.js +166 -31
- package/dist/executors/codex-cli.d.ts +6 -8
- package/dist/executors/codex-cli.js +49 -151
- package/dist/executors/codex-protocol.d.ts +24 -0
- package/dist/executors/codex-protocol.js +234 -0
- package/dist/executors/codex-sdk.js +68 -120
- package/dist/executors/gemini.js +88 -13
- package/dist/executors/index.d.ts +2 -3
- package/dist/executors/index.js +5 -3
- package/dist/executors/openai-api.js +70 -9
- package/dist/executors/runtime-fingerprint.js +88 -11
- package/dist/executors/script-command.d.ts +8 -0
- package/dist/executors/script-command.js +87 -0
- package/dist/executors/script.js +202 -29
- package/dist/executors/shared.d.ts +35 -3
- package/dist/executors/shared.js +113 -15
- package/dist/grading/assertions.d.ts +1 -1
- package/dist/grading/assertions.js +19 -9
- package/dist/grading/diagnostic.d.ts +9 -2
- package/dist/grading/diagnostic.js +25 -2
- package/dist/grading/index.js +10 -4
- package/dist/grading/judge.js +19 -6
- package/dist/grading/layered-scores.d.ts +2 -3
- package/dist/grading/layered-scores.js +2 -3
- package/dist/inputs/load-samples.d.ts +1 -2
- package/dist/inputs/load-samples.js +23 -1
- package/dist/inputs/mcp-resolver.js +6 -3
- package/dist/inputs/sample-document.d.ts +11 -0
- package/dist/inputs/sample-document.js +96 -0
- package/dist/managed/evidence.d.ts +1 -0
- package/dist/managed/evidence.js +1 -1
- package/dist/managed/store.js +200 -91
- package/dist/observability/codex-trace-adapter.d.ts +5 -0
- package/dist/observability/codex-trace-adapter.js +850 -0
- package/dist/observability/experience.d.ts +32 -6
- package/dist/observability/experience.js +2695 -459
- package/dist/observability/feedback-matchers.js +16 -1
- package/dist/observability/inbox-view-model.d.ts +2 -1
- package/dist/observability/inbox-view-model.js +20 -14
- package/dist/observability/inbox.d.ts +7 -1
- package/dist/observability/inbox.js +632 -124
- package/dist/observability/problem-patterns.js +2 -0
- package/dist/observability/review-state.d.ts +6 -0
- package/dist/observability/review-state.js +235 -63
- package/dist/observability/skill-chain-advisories.js +1 -1
- package/dist/observability/skill-chain.js +17 -4
- package/dist/observability/skill-health-analyzer.d.ts +32 -7
- package/dist/observability/skill-health-analyzer.js +194 -121
- package/dist/observability/skill-health-report.d.ts +10 -0
- package/dist/observability/skill-health-report.js +620 -0
- package/dist/observability/soft-standards/constants.d.ts +0 -1
- package/dist/observability/soft-standards/constants.js +0 -1
- package/dist/observability/soft-standards/index.d.ts +1 -1
- package/dist/observability/soft-standards/index.js +1 -1
- package/dist/observability/soft-standards/llm-extractor.js +8 -10
- package/dist/observability/soft-standards/skill-standards-store.d.ts +2 -1
- package/dist/observability/soft-standards/skill-standards-store.js +59 -18
- package/dist/observability/soft-standards/types.d.ts +2 -2
- package/dist/observability/trace-adapter.d.ts +12 -7
- package/dist/observability/trace-adapter.js +11 -9
- package/dist/observability/trace-attribution.d.ts +13 -5
- package/dist/observability/trace-attribution.js +315 -21
- package/dist/observability/trace-ingestion.d.ts +9 -0
- package/dist/observability/trace-ingestion.js +80 -0
- package/dist/observability/trace-ir.d.ts +113 -0
- package/dist/observability/trace-ir.js +87 -0
- package/dist/observability/trace-segmenter.d.ts +19 -6
- package/dist/observability/trace-segmenter.js +377 -196
- package/dist/observability/trace-session-index.d.ts +19 -0
- package/dist/observability/trace-session-index.js +68 -0
- package/dist/observability/trace-source.d.ts +12 -4
- package/dist/observability/trace-source.js +939 -215
- package/dist/renderer/html-renderer.js +37 -6
- package/dist/renderer/icons.js +3 -0
- package/dist/renderer/observation-inbox-renderer.js +227 -91
- package/dist/renderer/skill-detail-renderer.js +452 -109
- package/dist/renderer/skill-health-renderer.js +69 -12
- package/dist/renderer/summary.js +28 -7
- package/dist/renderer/table.js +21 -4
- package/dist/renderer/test-view.d.ts +1 -0
- package/dist/renderer/test-view.js +44 -9
- package/dist/server/indexed-report-store.js +14 -18
- package/dist/server/job-store.js +64 -26
- package/dist/server/report-server.js +190 -78
- package/dist/server/report-store.js +57 -80
- package/dist/server/skill-index.js +143 -49
- package/dist/server/skill-insights.js +44 -5
- package/dist/shared/artifact-graph.d.ts +3 -0
- package/dist/shared/artifact-graph.js +224 -0
- package/dist/shared/assertion-types.d.ts +8 -0
- package/dist/shared/assertion-types.js +46 -0
- package/dist/shared/atomic-json.d.ts +8 -0
- package/dist/shared/atomic-json.js +33 -0
- package/dist/shared/diagnosis-schema.d.ts +9 -0
- package/dist/shared/diagnosis-schema.js +181 -0
- package/dist/shared/doctor-report.d.ts +3 -0
- package/dist/shared/doctor-report.js +103 -0
- package/dist/shared/evaluation-job.d.ts +6 -0
- package/dist/shared/evaluation-job.js +217 -0
- package/dist/shared/executor-result.d.ts +17 -0
- package/dist/shared/executor-result.js +221 -0
- package/dist/shared/file-lock.d.ts +12 -0
- package/dist/shared/file-lock.js +129 -0
- package/dist/shared/json-value.d.ts +5 -0
- package/dist/shared/json-value.js +36 -0
- package/dist/shared/keyed-mutex.d.ts +7 -0
- package/dist/shared/keyed-mutex.js +24 -0
- package/dist/shared/record-count.d.ts +8 -0
- package/dist/shared/record-count.js +43 -0
- package/dist/shared/sample-contract.d.ts +3 -0
- package/dist/shared/sample-contract.js +332 -0
- package/dist/shared/shell-quote.d.ts +2 -0
- package/dist/shared/shell-quote.js +7 -0
- package/dist/shared/timestamp.d.ts +6 -0
- package/dist/shared/timestamp.js +64 -0
- package/dist/shared/token-usage.d.ts +19 -0
- package/dist/shared/token-usage.js +50 -0
- package/dist/shared/tool-call-status.d.ts +8 -0
- package/dist/shared/tool-call-status.js +28 -0
- package/dist/shared/tool-identity.d.ts +21 -0
- package/dist/shared/tool-identity.js +84 -0
- package/dist/shared/tool-search.js +73 -16
- package/dist/shared/trace-projection.d.ts +5 -0
- package/dist/shared/trace-projection.js +20 -0
- package/dist/shared/trace-source-kind.d.ts +3 -0
- package/dist/shared/trace-source-kind.js +12 -0
- package/dist/types/diagnosis.d.ts +2 -0
- package/dist/types/eval.d.ts +4 -0
- package/dist/types/executor.d.ts +32 -5
- package/dist/types/index.d.ts +1 -0
- package/dist/types/index.js +1 -0
- package/dist/types/judge.d.ts +2 -0
- package/dist/types/observability.d.ts +116 -9
- package/dist/types/report.d.ts +58 -6
- package/dist/types/skill-index.d.ts +7 -0
- package/dist/types/trace.d.ts +2 -0
- package/dist/types/trace.js +1 -0
- package/package.json +9 -5
|
@@ -196,12 +196,14 @@ function evidenceRefFromEvent(event) {
|
|
|
196
196
|
return {
|
|
197
197
|
id: event.id,
|
|
198
198
|
kind: event.kind,
|
|
199
|
+
traceId: event.traceId,
|
|
199
200
|
sourceTrace: event.sourceTrace,
|
|
200
201
|
sessionId: event.sessionId,
|
|
201
202
|
messageIndex: event.messageIndex,
|
|
202
203
|
logicalMessageIndex: event.logicalMessageIndex ?? event.messageIndex,
|
|
203
204
|
sourceLineIndex: event.sourceLineIndex ?? event.messageIndex,
|
|
204
205
|
messageUuid: event.messageUuid,
|
|
206
|
+
callInstanceId: event.callInstanceId,
|
|
205
207
|
toolUseId: event.toolUseId,
|
|
206
208
|
timestamp: event.timestamp,
|
|
207
209
|
role: event.role,
|
|
@@ -1,17 +1,23 @@
|
|
|
1
1
|
import type { ObservationMetricKey, ObservationMetricScope, ObservationReviewState, ObservationReviewStateEntry, ObservationReviewStateUpdate, ObservationReviewTargetType, ObservationReviewVerdict } from '../types/index.js';
|
|
2
2
|
export type { ObservationMetricKey, ObservationMetricScope, ObservationReviewState, ObservationReviewStateEntry, ObservationReviewStateUpdate, ObservationReviewTargetType, ObservationReviewVerdict, };
|
|
3
|
+
export declare class ObservationReviewStateValidationError extends Error {
|
|
4
|
+
readonly name = "ObservationReviewStateValidationError";
|
|
5
|
+
}
|
|
3
6
|
export declare function observationReviewStateKey(targetType: ObservationReviewTargetType, targetId: string): string;
|
|
4
7
|
export declare function observationMetricScopeFor(metricKey: ObservationMetricKey): ObservationMetricScope;
|
|
5
8
|
export declare function observationMetricAnnotationTargetId(ref: {
|
|
6
9
|
id?: string;
|
|
10
|
+
traceId?: string;
|
|
7
11
|
sourceTrace?: string;
|
|
8
12
|
sessionId?: string;
|
|
9
13
|
messageIndex?: number;
|
|
10
14
|
messageUuid?: string;
|
|
15
|
+
callInstanceId?: string;
|
|
11
16
|
toolUseId?: string;
|
|
12
17
|
metricScopeId?: string;
|
|
13
18
|
}, metricKey: ObservationMetricKey): string;
|
|
14
19
|
export declare function observationMetricAnnotationVerdict(state: ObservationReviewState | undefined, ref: Parameters<typeof observationMetricAnnotationTargetId>[0], metricKey: ObservationMetricKey): 'confirmed' | 'rejected' | undefined;
|
|
20
|
+
export declare function observationMetricAnnotationEntry(state: ObservationReviewState | undefined, ref: Parameters<typeof observationMetricAnnotationTargetId>[0], metricKey: ObservationMetricKey): ObservationReviewStateEntry | undefined;
|
|
15
21
|
export declare function observationReviewStatePath(observationsDir: string): string;
|
|
16
22
|
export declare function emptyObservationReviewState(now?: string): ObservationReviewState;
|
|
17
23
|
export declare function loadObservationReviewState(observationsDir: string): ObservationReviewState;
|
|
@@ -1,6 +1,12 @@
|
|
|
1
1
|
import { createHash } from 'node:crypto';
|
|
2
|
-
import { existsSync,
|
|
2
|
+
import { existsSync, readFileSync, } from 'node:fs';
|
|
3
3
|
import { join } from 'node:path';
|
|
4
|
+
import { writeJsonFileAtomic } from '../shared/atomic-json.js';
|
|
5
|
+
import { withFileLock } from '../shared/file-lock.js';
|
|
6
|
+
import { normalizeRfc3339Timestamp } from '../shared/timestamp.js';
|
|
7
|
+
export class ObservationReviewStateValidationError extends Error {
|
|
8
|
+
name = 'ObservationReviewStateValidationError';
|
|
9
|
+
}
|
|
4
10
|
export function observationReviewStateKey(targetType, targetId) {
|
|
5
11
|
return `${targetType}:${targetId}`;
|
|
6
12
|
}
|
|
@@ -14,36 +20,69 @@ export function observationMetricScopeFor(metricKey) {
|
|
|
14
20
|
: 'message';
|
|
15
21
|
}
|
|
16
22
|
export function observationMetricAnnotationTargetId(ref, metricKey) {
|
|
17
|
-
const hasStableTraceRef = Boolean(ref.
|
|
23
|
+
const hasStableTraceRef = Boolean(ref.traceId
|
|
24
|
+
|| ref.sourceTrace
|
|
25
|
+
|| ref.sessionId
|
|
26
|
+
|| ref.messageIndex !== undefined
|
|
27
|
+
|| ref.messageUuid
|
|
28
|
+
|| ref.callInstanceId
|
|
29
|
+
|| ref.toolUseId);
|
|
18
30
|
const scope = observationMetricScopeFor(metricKey);
|
|
19
31
|
const stable = [
|
|
20
32
|
metricKey,
|
|
21
33
|
...(scope === 'skill_segment' ? [scope, ref.metricScopeId ?? ''] : []),
|
|
34
|
+
ref.traceId ?? '',
|
|
22
35
|
ref.sourceTrace ?? '',
|
|
23
36
|
ref.sessionId ?? '',
|
|
24
37
|
ref.messageIndex === undefined ? '' : String(ref.messageIndex),
|
|
25
38
|
ref.messageUuid ?? '',
|
|
39
|
+
ref.callInstanceId ?? '',
|
|
26
40
|
ref.toolUseId ?? '',
|
|
27
41
|
hasStableTraceRef ? '' : ref.id ?? '',
|
|
28
42
|
].join('\u0000');
|
|
29
43
|
return `metric:${metricKey}:${createHash('sha256').update(stable).digest('hex').slice(0, 16)}`;
|
|
30
44
|
}
|
|
31
45
|
export function observationMetricAnnotationVerdict(state, ref, metricKey) {
|
|
46
|
+
const entry = observationMetricAnnotationEntry(state, ref, metricKey);
|
|
47
|
+
return entry?.verdict === 'confirmed' || entry?.verdict === 'rejected'
|
|
48
|
+
? entry.verdict
|
|
49
|
+
: undefined;
|
|
50
|
+
}
|
|
51
|
+
export function observationMetricAnnotationEntry(state, ref, metricKey) {
|
|
32
52
|
if (!state)
|
|
33
53
|
return undefined;
|
|
34
|
-
const
|
|
35
|
-
|
|
36
|
-
|
|
54
|
+
const candidates = [
|
|
55
|
+
ref,
|
|
56
|
+
...(ref.callInstanceId ? [{ ...ref, callInstanceId: undefined }] : []),
|
|
57
|
+
...(ref.traceId ? [{ ...ref, traceId: undefined }] : []),
|
|
58
|
+
...(ref.traceId && ref.callInstanceId
|
|
59
|
+
? [{ ...ref, traceId: undefined, callInstanceId: undefined }]
|
|
60
|
+
: []),
|
|
61
|
+
];
|
|
62
|
+
const seen = new Set();
|
|
63
|
+
for (const candidate of candidates) {
|
|
64
|
+
const targetId = observationMetricAnnotationTargetId(candidate, metricKey);
|
|
65
|
+
if (seen.has(targetId))
|
|
66
|
+
continue;
|
|
67
|
+
seen.add(targetId);
|
|
68
|
+
const entry = state.entries[observationReviewStateKey('evidence_metric', targetId)];
|
|
69
|
+
if (entry)
|
|
70
|
+
return entry;
|
|
71
|
+
}
|
|
72
|
+
return undefined;
|
|
37
73
|
}
|
|
38
74
|
export function observationReviewStatePath(observationsDir) {
|
|
39
75
|
return join(observationsDir, 'review-state.json');
|
|
40
76
|
}
|
|
41
77
|
const OBSERVATION_REVIEW_STATE_SCHEMA_VERSION = 2;
|
|
42
78
|
export function emptyObservationReviewState(now = new Date().toISOString()) {
|
|
79
|
+
const updatedAt = normalizedTimestamp(now);
|
|
80
|
+
if (!updatedAt)
|
|
81
|
+
throw new ObservationReviewStateValidationError('invalid updatedAt');
|
|
43
82
|
return {
|
|
44
83
|
kind: 'observe-review-state',
|
|
45
84
|
schemaVersion: OBSERVATION_REVIEW_STATE_SCHEMA_VERSION,
|
|
46
|
-
updatedAt
|
|
85
|
+
updatedAt,
|
|
47
86
|
entries: {},
|
|
48
87
|
};
|
|
49
88
|
}
|
|
@@ -51,39 +90,57 @@ export function loadObservationReviewState(observationsDir) {
|
|
|
51
90
|
const path = observationReviewStatePath(observationsDir);
|
|
52
91
|
if (!existsSync(path))
|
|
53
92
|
return emptyObservationReviewState();
|
|
93
|
+
let parsed;
|
|
54
94
|
try {
|
|
55
|
-
|
|
56
|
-
return normalizeObservationReviewState(parsed) ?? emptyObservationReviewState();
|
|
95
|
+
parsed = JSON.parse(readFileSync(path, 'utf-8'));
|
|
57
96
|
}
|
|
58
|
-
catch {
|
|
59
|
-
|
|
97
|
+
catch (cause) {
|
|
98
|
+
throw new Error(`无法读取观测复核状态:${path}`, { cause });
|
|
60
99
|
}
|
|
100
|
+
const normalized = normalizeObservationReviewState(parsed);
|
|
101
|
+
if (!normalized)
|
|
102
|
+
throw new Error(`观测复核状态格式无效:${path}`);
|
|
103
|
+
return normalized;
|
|
61
104
|
}
|
|
62
|
-
export function updateObservationReviewState(observationsDir, update, now
|
|
105
|
+
export function updateObservationReviewState(observationsDir, update, now) {
|
|
63
106
|
assertReviewStateUpdate(update);
|
|
64
|
-
const
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
107
|
+
const path = observationReviewStatePath(observationsDir);
|
|
108
|
+
return withFileLock(`${path}.lock`, () => {
|
|
109
|
+
const reviewedAt = normalizedTimestamp(now ?? new Date().toISOString());
|
|
110
|
+
if (!reviewedAt) {
|
|
111
|
+
throw new ObservationReviewStateValidationError('invalid reviewedAt');
|
|
112
|
+
}
|
|
113
|
+
const hadPersistedState = existsSync(path);
|
|
114
|
+
const state = loadObservationReviewState(observationsDir);
|
|
115
|
+
if (hadPersistedState && reviewedAt < state.updatedAt) {
|
|
116
|
+
throw new ObservationReviewStateValidationError('reviewedAt cannot precede state updatedAt');
|
|
117
|
+
}
|
|
118
|
+
const key = observationReviewStateKey(update.targetType, update.targetId);
|
|
119
|
+
state.entries[key] = {
|
|
120
|
+
targetType: update.targetType,
|
|
121
|
+
targetId: update.targetId,
|
|
122
|
+
verdict: update.verdict,
|
|
123
|
+
reviewedAt,
|
|
124
|
+
...(update.note ? { note: update.note.slice(0, 500) } : {}),
|
|
125
|
+
...(update.reason ? { reason: update.reason.slice(0, 500) } : {}),
|
|
126
|
+
...(update.metricKey ? { metricKey: update.metricKey } : {}),
|
|
127
|
+
...(update.metricKey ? {
|
|
128
|
+
metricScope: update.metricScope ?? observationMetricScopeFor(update.metricKey),
|
|
129
|
+
} : {}),
|
|
130
|
+
...(update.metricScopeId ? { metricScopeId: update.metricScopeId.slice(0, 200) } : {}),
|
|
131
|
+
...(update.traceId ? { traceId: update.traceId } : {}),
|
|
132
|
+
...(update.sourceTrace ? { sourceTrace: update.sourceTrace } : {}),
|
|
133
|
+
...(update.sessionId ? { sessionId: update.sessionId } : {}),
|
|
134
|
+
...(update.messageIndex !== undefined ? { messageIndex: update.messageIndex } : {}),
|
|
135
|
+
...(update.messageUuid ? { messageUuid: update.messageUuid } : {}),
|
|
136
|
+
...(update.callInstanceId ? { callInstanceId: update.callInstanceId } : {}),
|
|
137
|
+
...(update.toolUseId ? { toolUseId: update.toolUseId } : {}),
|
|
138
|
+
...(update.snippet ? { snippet: update.snippet.slice(0, 500) } : {}),
|
|
139
|
+
};
|
|
140
|
+
state.updatedAt = reviewedAt;
|
|
141
|
+
writeObservationReviewState(observationsDir, state);
|
|
142
|
+
return state;
|
|
143
|
+
}, { label: 'observation review state' });
|
|
87
144
|
}
|
|
88
145
|
function normalizeObservationReviewState(value) {
|
|
89
146
|
if (!value || typeof value !== 'object')
|
|
@@ -94,50 +151,165 @@ function normalizeObservationReviewState(value) {
|
|
|
94
151
|
return null;
|
|
95
152
|
if (parsed.schemaVersion !== OBSERVATION_REVIEW_STATE_SCHEMA_VERSION)
|
|
96
153
|
return null;
|
|
97
|
-
if (!parsed.entries || typeof parsed.entries !== 'object')
|
|
154
|
+
if (!parsed.entries || typeof parsed.entries !== 'object' || Array.isArray(parsed.entries))
|
|
155
|
+
return null;
|
|
156
|
+
const updatedAt = normalizedTimestamp(parsed.updatedAt);
|
|
157
|
+
if (!updatedAt)
|
|
158
|
+
return null;
|
|
159
|
+
const entries = {};
|
|
160
|
+
for (const [key, rawEntry] of Object.entries(parsed.entries)) {
|
|
161
|
+
const entry = normalizeReviewStateEntry(rawEntry);
|
|
162
|
+
if (!entry || key !== observationReviewStateKey(entry.targetType, entry.targetId))
|
|
163
|
+
return null;
|
|
164
|
+
entries[key] = entry;
|
|
165
|
+
}
|
|
166
|
+
if (Object.values(entries).some((entry) => entry.reviewedAt > updatedAt))
|
|
98
167
|
return null;
|
|
99
168
|
return {
|
|
100
169
|
kind: 'observe-review-state',
|
|
101
170
|
schemaVersion: OBSERVATION_REVIEW_STATE_SCHEMA_VERSION,
|
|
102
|
-
updatedAt
|
|
103
|
-
entries
|
|
171
|
+
updatedAt,
|
|
172
|
+
entries,
|
|
104
173
|
};
|
|
105
174
|
}
|
|
106
|
-
export function deleteObservationReviewState(observationsDir, targetType, targetId, now
|
|
175
|
+
export function deleteObservationReviewState(observationsDir, targetType, targetId, now) {
|
|
107
176
|
if (!isReviewTargetType(targetType))
|
|
108
|
-
throw new
|
|
177
|
+
throw new ObservationReviewStateValidationError('invalid review targetType');
|
|
109
178
|
if (typeof targetId !== 'string' || targetId.trim() === '')
|
|
110
|
-
throw new
|
|
111
|
-
const
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
179
|
+
throw new ObservationReviewStateValidationError('invalid review targetId');
|
|
180
|
+
const path = observationReviewStatePath(observationsDir);
|
|
181
|
+
return withFileLock(`${path}.lock`, () => {
|
|
182
|
+
const updatedAt = normalizedTimestamp(now ?? new Date().toISOString());
|
|
183
|
+
if (!updatedAt) {
|
|
184
|
+
throw new ObservationReviewStateValidationError('invalid updatedAt');
|
|
185
|
+
}
|
|
186
|
+
const hadPersistedState = existsSync(path);
|
|
187
|
+
const state = loadObservationReviewState(observationsDir);
|
|
188
|
+
if (hadPersistedState && updatedAt < state.updatedAt) {
|
|
189
|
+
throw new ObservationReviewStateValidationError('updatedAt cannot precede state updatedAt');
|
|
190
|
+
}
|
|
191
|
+
const key = observationReviewStateKey(targetType, targetId);
|
|
192
|
+
delete state.entries[key];
|
|
193
|
+
state.updatedAt = updatedAt;
|
|
194
|
+
writeObservationReviewState(observationsDir, state);
|
|
195
|
+
return state;
|
|
196
|
+
}, { label: 'observation review state' });
|
|
118
197
|
}
|
|
119
198
|
function assertReviewStateUpdate(value) {
|
|
120
199
|
if (!isReviewTargetType(value.targetType))
|
|
121
|
-
throw new
|
|
200
|
+
throw new ObservationReviewStateValidationError('invalid review targetType');
|
|
122
201
|
if (typeof value.targetId !== 'string' || value.targetId.trim() === '')
|
|
123
|
-
throw new
|
|
202
|
+
throw new ObservationReviewStateValidationError('invalid review targetId');
|
|
124
203
|
if (!isReviewVerdict(value.verdict))
|
|
125
|
-
throw new
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
204
|
+
throw new ObservationReviewStateValidationError('invalid review verdict');
|
|
205
|
+
const optionalStrings = [
|
|
206
|
+
value.note,
|
|
207
|
+
value.reason,
|
|
208
|
+
value.metricScopeId,
|
|
209
|
+
value.traceId,
|
|
210
|
+
value.sourceTrace,
|
|
211
|
+
value.sessionId,
|
|
212
|
+
value.messageUuid,
|
|
213
|
+
value.callInstanceId,
|
|
214
|
+
value.toolUseId,
|
|
215
|
+
value.snippet,
|
|
216
|
+
];
|
|
217
|
+
if (!optionalStrings.every(optionalString))
|
|
218
|
+
throw new ObservationReviewStateValidationError('invalid review metadata');
|
|
219
|
+
if (value.messageIndex !== undefined
|
|
220
|
+
&& (!Number.isSafeInteger(value.messageIndex) || value.messageIndex < 0))
|
|
221
|
+
throw new ObservationReviewStateValidationError('invalid messageIndex');
|
|
222
|
+
if (value.targetType === 'evidence_metric') {
|
|
223
|
+
if (!isObservationMetricKey(value.metricKey))
|
|
224
|
+
throw new ObservationReviewStateValidationError('invalid metricKey');
|
|
225
|
+
if (value.verdict !== 'confirmed' && value.verdict !== 'rejected') {
|
|
226
|
+
throw new ObservationReviewStateValidationError('evidence_metric requires confirmed or rejected verdict');
|
|
227
|
+
}
|
|
228
|
+
const expectedScope = observationMetricScopeFor(value.metricKey);
|
|
229
|
+
if (value.metricScope !== undefined && value.metricScope !== expectedScope) {
|
|
230
|
+
throw new ObservationReviewStateValidationError('metricScope does not match metricKey');
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
else {
|
|
234
|
+
if (value.verdict === 'confirmed' || value.verdict === 'rejected') {
|
|
235
|
+
throw new ObservationReviewStateValidationError('metric verdict requires evidence_metric targetType');
|
|
236
|
+
}
|
|
237
|
+
if (value.metricKey !== undefined || value.metricScope !== undefined || value.metricScopeId !== undefined) {
|
|
238
|
+
throw new ObservationReviewStateValidationError('metric metadata requires evidence_metric targetType');
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
function normalizeReviewStateEntry(value) {
|
|
134
243
|
if (!value || typeof value !== 'object')
|
|
135
|
-
return
|
|
244
|
+
return null;
|
|
136
245
|
const entry = value;
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
246
|
+
const reviewedAt = normalizedTimestamp(entry.reviewedAt);
|
|
247
|
+
if (!isReviewTargetType(entry.targetType)
|
|
248
|
+
|| typeof entry.targetId !== 'string'
|
|
249
|
+
|| entry.targetId.trim() === ''
|
|
250
|
+
|| !isReviewVerdict(entry.verdict)
|
|
251
|
+
|| !reviewedAt
|
|
252
|
+
|| !optionalString(entry.note)
|
|
253
|
+
|| !optionalString(entry.reason)
|
|
254
|
+
|| !optionalString(entry.metricScopeId)
|
|
255
|
+
|| !optionalString(entry.traceId)
|
|
256
|
+
|| !optionalString(entry.sourceTrace)
|
|
257
|
+
|| !optionalString(entry.sessionId)
|
|
258
|
+
|| !optionalString(entry.messageUuid)
|
|
259
|
+
|| !optionalString(entry.callInstanceId)
|
|
260
|
+
|| !optionalString(entry.toolUseId)
|
|
261
|
+
|| !optionalString(entry.snippet)
|
|
262
|
+
|| (entry.messageIndex !== undefined
|
|
263
|
+
&& (!Number.isSafeInteger(entry.messageIndex) || entry.messageIndex < 0))
|
|
264
|
+
|| (entry.metricKey !== undefined && !isObservationMetricKey(entry.metricKey))
|
|
265
|
+
|| (entry.metricScope !== undefined && !isObservationMetricScope(entry.metricScope)))
|
|
266
|
+
return null;
|
|
267
|
+
if (entry.targetType === 'evidence_metric') {
|
|
268
|
+
if (!isObservationMetricKey(entry.metricKey)
|
|
269
|
+
|| (entry.verdict !== 'confirmed' && entry.verdict !== 'rejected'))
|
|
270
|
+
return null;
|
|
271
|
+
}
|
|
272
|
+
else if (entry.verdict === 'confirmed'
|
|
273
|
+
|| entry.verdict === 'rejected'
|
|
274
|
+
|| entry.metricKey !== undefined
|
|
275
|
+
|| entry.metricScope !== undefined
|
|
276
|
+
|| entry.metricScopeId !== undefined) {
|
|
277
|
+
return null;
|
|
278
|
+
}
|
|
279
|
+
const metricScope = entry.metricKey
|
|
280
|
+
? entry.metricScope ?? observationMetricScopeFor(entry.metricKey)
|
|
281
|
+
: undefined;
|
|
282
|
+
if (entry.metricKey
|
|
283
|
+
&& metricScope !== observationMetricScopeFor(entry.metricKey))
|
|
284
|
+
return null;
|
|
285
|
+
return {
|
|
286
|
+
targetType: entry.targetType,
|
|
287
|
+
targetId: entry.targetId,
|
|
288
|
+
verdict: entry.verdict,
|
|
289
|
+
reviewedAt,
|
|
290
|
+
...(entry.note !== undefined ? { note: entry.note } : {}),
|
|
291
|
+
...(entry.reason !== undefined ? { reason: entry.reason } : {}),
|
|
292
|
+
...(entry.metricKey !== undefined ? { metricKey: entry.metricKey, metricScope } : {}),
|
|
293
|
+
...(entry.metricScopeId !== undefined ? { metricScopeId: entry.metricScopeId } : {}),
|
|
294
|
+
...(entry.traceId !== undefined ? { traceId: entry.traceId } : {}),
|
|
295
|
+
...(entry.sourceTrace !== undefined ? { sourceTrace: entry.sourceTrace } : {}),
|
|
296
|
+
...(entry.sessionId !== undefined ? { sessionId: entry.sessionId } : {}),
|
|
297
|
+
...(entry.messageIndex !== undefined ? { messageIndex: entry.messageIndex } : {}),
|
|
298
|
+
...(entry.messageUuid !== undefined ? { messageUuid: entry.messageUuid } : {}),
|
|
299
|
+
...(entry.callInstanceId !== undefined ? { callInstanceId: entry.callInstanceId } : {}),
|
|
300
|
+
...(entry.toolUseId !== undefined ? { toolUseId: entry.toolUseId } : {}),
|
|
301
|
+
...(entry.snippet !== undefined ? { snippet: entry.snippet } : {}),
|
|
302
|
+
};
|
|
303
|
+
}
|
|
304
|
+
function normalizedTimestamp(value) {
|
|
305
|
+
return normalizeRfc3339Timestamp(value) ?? null;
|
|
306
|
+
}
|
|
307
|
+
function optionalString(value) {
|
|
308
|
+
return value === undefined || typeof value === 'string';
|
|
309
|
+
}
|
|
310
|
+
function writeObservationReviewState(observationsDir, state) {
|
|
311
|
+
const path = observationReviewStatePath(observationsDir);
|
|
312
|
+
writeJsonFileAtomic(path, state);
|
|
141
313
|
}
|
|
142
314
|
function isReviewTargetType(value) {
|
|
143
315
|
return value === 'experience_session'
|
|
@@ -50,7 +50,7 @@ const ADVISORIES = {
|
|
|
50
50
|
code: 'skill_md_not_found',
|
|
51
51
|
shortLabel: '本地未找到 SKILL.md',
|
|
52
52
|
message: '本地常见 skill 目录里没有这个 skill 的 SKILL.md,inbox 跑不了静态/运行时审计。如果你想看 LLM 健康度审计(触发边界 / 文档 / 依赖 / 安全等 7 维),可以试试 omk doctor;inbox 本身不依赖它。',
|
|
53
|
-
commandTemplate: 'omk doctor
|
|
53
|
+
commandTemplate: 'omk doctor /path/to/${skillName}/SKILL.md',
|
|
54
54
|
},
|
|
55
55
|
};
|
|
56
56
|
export function getSkillChainAdvisory(code) {
|
|
@@ -2,6 +2,7 @@ import { existsSync, readFileSync, readdirSync } from 'node:fs';
|
|
|
2
2
|
import { homedir } from 'node:os';
|
|
3
3
|
import { join } from 'node:path';
|
|
4
4
|
import { extractMarkdownStepWorkflows, validateSkillHardRules, validateSkillWorkflows, } from '../shared/hard-rules.js';
|
|
5
|
+
import { incrementRecordCount } from '../shared/record-count.js';
|
|
5
6
|
const MAX_SKILL_DEFINITION_CHARS = 12000;
|
|
6
7
|
export function buildObservationSkillChains(skillNames, cwd = process.cwd(), experienceReports = []) {
|
|
7
8
|
const uniqueSkillNames = Array.from(new Set(skillNames.filter(Boolean))).sort();
|
|
@@ -199,7 +200,7 @@ function runtimeEvidence(invocations) {
|
|
|
199
200
|
const events = invocations.flatMap((invocation) => invocation.timeline ?? []);
|
|
200
201
|
for (const invocation of invocations) {
|
|
201
202
|
for (const [tool, count] of Object.entries(invocation.toolCounts ?? {})) {
|
|
202
|
-
toolCounts
|
|
203
|
+
incrementRecordCount(toolCounts, tool, count);
|
|
203
204
|
}
|
|
204
205
|
}
|
|
205
206
|
return {
|
|
@@ -207,8 +208,14 @@ function runtimeEvidence(invocations) {
|
|
|
207
208
|
toolCallCount: invocations.reduce((sum, invocation) => sum + invocation.indicators.toolCallCount, 0),
|
|
208
209
|
toolFailureCount: invocations.reduce((sum, invocation) => sum + invocation.indicators.toolFailureCount, 0),
|
|
209
210
|
events,
|
|
210
|
-
|
|
211
|
-
.filter((event) => event.kind === 'tool_use' && event.role === 'assistant' && event.
|
|
211
|
+
assistantCallInstanceIds: new Set(events
|
|
212
|
+
.filter((event) => event.kind === 'tool_use' && event.role === 'assistant' && event.callInstanceId)
|
|
213
|
+
.map((event) => event.callInstanceId)),
|
|
214
|
+
legacyAssistantToolUseIds: new Set(events
|
|
215
|
+
.filter((event) => event.kind === 'tool_use'
|
|
216
|
+
&& event.role === 'assistant'
|
|
217
|
+
&& !event.callInstanceId
|
|
218
|
+
&& event.toolUseId)
|
|
212
219
|
.map((event) => event.toolUseId)),
|
|
213
220
|
};
|
|
214
221
|
}
|
|
@@ -351,6 +358,7 @@ function eventToEvidencePackRef(event) {
|
|
|
351
358
|
messageIndex: event.messageIndex,
|
|
352
359
|
logicalMessageIndex: event.logicalMessageIndex,
|
|
353
360
|
sourceLineIndex: event.sourceLineIndex,
|
|
361
|
+
callInstanceId: event.callInstanceId,
|
|
354
362
|
toolUseId: event.toolUseId,
|
|
355
363
|
timestamp: event.timestamp,
|
|
356
364
|
role: event.role,
|
|
@@ -384,7 +392,12 @@ function isAssistantAuthoredUploadEvent(event, evidence, text) {
|
|
|
384
392
|
return false;
|
|
385
393
|
if (event.kind === 'tool_use' && event.role === 'assistant')
|
|
386
394
|
return true;
|
|
387
|
-
if (event.kind === 'tool_result'
|
|
395
|
+
if (event.kind === 'tool_result'
|
|
396
|
+
&& event.role === 'tool'
|
|
397
|
+
&& (event.callInstanceId
|
|
398
|
+
? evidence.assistantCallInstanceIds.has(event.callInstanceId)
|
|
399
|
+
: Boolean(event.toolUseId)
|
|
400
|
+
&& evidence.legacyAssistantToolUseIds.has(event.toolUseId)))
|
|
388
401
|
return true;
|
|
389
402
|
return false;
|
|
390
403
|
}
|
|
@@ -6,28 +6,33 @@
|
|
|
6
6
|
* 定位是"真实使用 trace 的 skill 维度观察",不是通用 APM / 生产监控。
|
|
7
7
|
*
|
|
8
8
|
* 分析流水线:
|
|
9
|
-
* 1.
|
|
9
|
+
* 1. tracesToResultEntries(path) → segments + ResultEntry[]
|
|
10
10
|
* 2. 时间窗 / skill 白名单过滤
|
|
11
11
|
* 3. 按 skill name (variant key) 分别 computeCoverage + computeGapReport
|
|
12
12
|
* 4. 聚合 overall 指标 + 健康度色带
|
|
13
13
|
*/
|
|
14
14
|
import { type CoverageReport } from '../analysis/coverage-analyzer.js';
|
|
15
|
-
import type { GapReport } from '../types/index.js';
|
|
16
|
-
import { type CcSession, type SkillSegment } from './trace-adapter.js';
|
|
15
|
+
import type { GapReport, TraceIngestionSummary } from '../types/index.js';
|
|
16
|
+
import { type CcSession, type TraceSession, type SkillSegment } from './trace-adapter.js';
|
|
17
17
|
export interface SkillHealth {
|
|
18
18
|
skillName: string;
|
|
19
19
|
segmentCount: number;
|
|
20
20
|
toolCallCount: number;
|
|
21
21
|
toolFailureCount: number;
|
|
22
|
-
|
|
22
|
+
toolCancelledCount?: number;
|
|
23
|
+
toolUnknownCount?: number;
|
|
24
|
+
toolResolvedCount?: number;
|
|
25
|
+
toolOutcomeCoverage?: number;
|
|
26
|
+
/** 失败率 = toolFailureCount / 可比较结果数;取消与未知状态不进入分母。 */
|
|
23
27
|
toolFailureRate: number;
|
|
24
28
|
/**
|
|
25
29
|
* 执行稳定性标签。阈值:
|
|
26
30
|
* - very-unstable: failureRate >= 0.4 (gap 信号极可能是环境问题,不是真知识缺口)
|
|
27
31
|
* - unstable: failureRate >= 0.2 (建议排查环境后再看 gap)
|
|
32
|
+
* - unknown: 有工具调用,但没有可比较的成功 / 失败结果
|
|
28
33
|
* - stable: 否则
|
|
29
34
|
*/
|
|
30
|
-
stability: 'stable' | 'unstable' | 'very-unstable';
|
|
35
|
+
stability: 'stable' | 'unstable' | 'very-unstable' | 'unknown';
|
|
31
36
|
/**
|
|
32
37
|
* 统计可信度(按 segment 数)。守护「1 段 + 一次失败就判 red」的过度自信:
|
|
33
38
|
* - underpowered: segmentCount < 5 (样本太少,色带 / gap rate 不可信)
|
|
@@ -43,6 +48,8 @@ export interface SkillHealth {
|
|
|
43
48
|
cacheReadTokens: number;
|
|
44
49
|
cacheCreationTokens: number;
|
|
45
50
|
totalTokens: number;
|
|
51
|
+
tokenObservedSegmentCount: number;
|
|
52
|
+
tokenCoverage: number;
|
|
46
53
|
durationMs: number;
|
|
47
54
|
numTurns: number;
|
|
48
55
|
avgTokensPerSegment: number;
|
|
@@ -61,13 +68,21 @@ export interface SkillHealthReport {
|
|
|
61
68
|
sessionCount: number;
|
|
62
69
|
segmentCount: number;
|
|
63
70
|
messageCount: number;
|
|
71
|
+
timestampedSegmentCount?: number;
|
|
72
|
+
timestampCoverage?: number;
|
|
73
|
+
excludedUntimestampedSegmentCount?: number;
|
|
64
74
|
toolCallCount: number;
|
|
75
|
+
toolCancelledCount?: number;
|
|
76
|
+
toolUnknownCount?: number;
|
|
77
|
+
toolResolvedCount?: number;
|
|
78
|
+
toolOutcomeCoverage?: number;
|
|
65
79
|
toolFailureRate: number;
|
|
66
80
|
timeRange: {
|
|
67
81
|
from: string;
|
|
68
82
|
to: string;
|
|
69
83
|
};
|
|
70
84
|
generatedAt: string;
|
|
85
|
+
ingestion?: TraceIngestionSummary;
|
|
71
86
|
};
|
|
72
87
|
bySkill: Record<string, SkillHealth>;
|
|
73
88
|
overall: {
|
|
@@ -89,15 +104,25 @@ export interface AnalyzeOptions {
|
|
|
89
104
|
* #235 受管反哺时 observe CLI 用本函数逐 skill 算 band 传入 managed —— 阈值单一来源,managed 不复制。
|
|
90
105
|
*/
|
|
91
106
|
export declare function healthBandOf(weightedGapRate: number): 'green' | 'yellow' | 'red';
|
|
107
|
+
/**
|
|
108
|
+
* 按 per-skill 失败率判定执行稳定性。阈值见 SkillHealth.stability。
|
|
109
|
+
*/
|
|
110
|
+
export declare function toolStabilityOf(toolFailureRate: number, comparableToolCalls: number, totalToolCalls: number): SkillHealth['stability'];
|
|
92
111
|
/** 统计可信度护栏:按 segment 数判 high / low / underpowered,守护「1 段就判 red」。
|
|
93
112
|
* 导出供 renderer / CLI / server 给历史报告(缺 confidence 字段)做 segmentCount 兜底。 */
|
|
94
113
|
export declare function confidenceOf(segmentCount: number): SkillHealth['confidence'];
|
|
95
114
|
/**
|
|
96
|
-
*
|
|
115
|
+
* 推断 KB root: 没传 --kb 时,取第一个 assistant record 的 cwd。
|
|
116
|
+
* 如果跨多个 cwd,取第一个并 warn。
|
|
117
|
+
*/
|
|
118
|
+
type HealthSession = TraceSession | CcSession;
|
|
119
|
+
/**
|
|
120
|
+
* 主入口:从受支持的 trace 输入生成 SkillHealthReport。
|
|
97
121
|
*/
|
|
98
122
|
export declare function computeSkillHealthReport(tracePath: string, opts?: AnalyzeOptions): SkillHealthReport;
|
|
99
123
|
/**
|
|
100
124
|
* 便利入口:直接从已准备好的 segments(跳过 loadCcSessions)算 report。
|
|
101
125
|
* 用于测试 / 已手工组装过 segments 的场景。
|
|
102
126
|
*/
|
|
103
|
-
export declare function computeSkillHealthFromSegments(segments: SkillSegment[], sessions:
|
|
127
|
+
export declare function computeSkillHealthFromSegments(segments: SkillSegment[], sessions: HealthSession[], tracePath: string, opts?: AnalyzeOptions, ingestion?: TraceIngestionSummary): SkillHealthReport;
|
|
128
|
+
export {};
|