oh-my-knowledge 0.47.0 → 0.49.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (246) hide show
  1. package/README.md +50 -18
  2. package/README.zh.md +55 -23
  3. package/dist/analysis/coverage-analyzer.d.ts +1 -0
  4. package/dist/analysis/coverage-analyzer.js +125 -62
  5. package/dist/analysis/failure-clusterer.js +2 -1
  6. package/dist/analysis/gap-analyzer.d.ts +2 -2
  7. package/dist/analysis/gap-analyzer.js +13 -3
  8. package/dist/analysis/hedging-classifier.d.ts +2 -2
  9. package/dist/analysis/hedging-classifier.js +3 -4
  10. package/dist/analysis/report-diagnostics.js +9 -7
  11. package/dist/analysis/sample-diagnostics.js +6 -6
  12. package/dist/artifact-graph/doctor.js +15 -7
  13. package/dist/assets/agent-skills/omk/SKILL.md +27 -7
  14. package/dist/assets/agent-skills/omk/references/commands.md +19 -17
  15. package/dist/authoring/evolver.d.ts +10 -6
  16. package/dist/authoring/evolver.js +496 -83
  17. package/dist/authoring/generator.d.ts +3 -3
  18. package/dist/authoring/generator.js +5 -10
  19. package/dist/authoring/sample-fixer.d.ts +8 -6
  20. package/dist/authoring/sample-fixer.js +76 -5
  21. package/dist/cli/commands/doctor.js +31 -14
  22. package/dist/cli/commands/eval/index.d.ts +3 -0
  23. package/dist/cli/commands/eval/index.js +163 -17
  24. package/dist/cli/commands/evolve.d.ts +4 -4
  25. package/dist/cli/commands/evolve.js +27 -13
  26. package/dist/cli/commands/init.js +16 -3
  27. package/dist/cli/commands/observe/inbox.js +44 -21
  28. package/dist/cli/commands/observe/index.js +20 -11
  29. package/dist/cli/commands/observe/ingest.d.ts +3 -0
  30. package/dist/cli/commands/observe/ingest.js +35 -4
  31. package/dist/cli/commands/sample.d.ts +9 -3
  32. package/dist/cli/commands/sample.js +91 -74
  33. package/dist/cli/lib/cmd-flags.d.ts +1 -0
  34. package/dist/cli/lib/codex-model-hint.d.ts +9 -0
  35. package/dist/cli/lib/codex-model-hint.js +45 -0
  36. package/dist/cli/lib/generation-failure-hint.d.ts +2 -0
  37. package/dist/cli/lib/generation-failure-hint.js +61 -0
  38. package/dist/cli/lib/i18n-dict/common.d.ts +1 -1
  39. package/dist/cli/lib/i18n-dict/common.js +4 -0
  40. package/dist/cli/lib/i18n-dict/gen.d.ts +1 -1
  41. package/dist/cli/lib/i18n-dict/gen.js +38 -6
  42. package/dist/cli/lib/i18n-dict/help.js +6 -6
  43. package/dist/cli/lib/i18n-dict/init.d.ts +1 -1
  44. package/dist/cli/lib/i18n-dict/init.js +13 -9
  45. package/dist/cli/lib/i18n-dict/run.d.ts +1 -1
  46. package/dist/cli/lib/i18n-dict/run.js +34 -2
  47. package/dist/cli/lib/llm-failure-classifier.d.ts +2 -0
  48. package/dist/cli/lib/llm-failure-classifier.js +8 -0
  49. package/dist/cli/lib/parse-run-config.d.ts +6 -5
  50. package/dist/cli/lib/parse-run-config.js +16 -9
  51. package/dist/cli/lib/runtime-defaults.d.ts +21 -0
  52. package/dist/cli/lib/runtime-defaults.js +79 -0
  53. package/dist/diagnosis/observe-mapper.js +14 -15
  54. package/dist/diagnosis/observe-producer.js +3 -1
  55. package/dist/diagnosis/studio-projection.js +14 -7
  56. package/dist/diagnosis/types.d.ts +2 -0
  57. package/dist/diagnosis/types.js +12 -0
  58. package/dist/doctor/endpoint-rule.js +2 -1
  59. package/dist/eval-core/artifact-file-names.js +18 -1
  60. package/dist/eval-core/artifact-index.d.ts +7 -11
  61. package/dist/eval-core/artifact-index.js +139 -80
  62. package/dist/eval-core/cache.d.ts +12 -3
  63. package/dist/eval-core/cache.js +89 -29
  64. package/dist/eval-core/comparability.js +10 -6
  65. package/dist/eval-core/evaluation-execution.d.ts +2 -1
  66. package/dist/eval-core/evaluation-execution.js +122 -37
  67. package/dist/eval-core/evaluation-job.d.ts +4 -1
  68. package/dist/eval-core/evaluation-job.js +4 -1
  69. package/dist/eval-core/evaluation-reporting.d.ts +15 -13
  70. package/dist/eval-core/evaluation-reporting.js +54 -52
  71. package/dist/eval-core/execution-strategy.d.ts +2 -0
  72. package/dist/eval-core/execution-strategy.js +11 -9
  73. package/dist/eval-core/fact-checker.js +15 -7
  74. package/dist/eval-core/holdout.js +3 -2
  75. package/dist/eval-core/judge-independence.d.ts +2 -2
  76. package/dist/eval-core/mock-hook.cjs +23 -6
  77. package/dist/eval-core/mocks-runtime.js +30 -8
  78. package/dist/eval-core/report-document.d.ts +12 -0
  79. package/dist/eval-core/report-document.js +1151 -0
  80. package/dist/eval-core/report-extensions.d.ts +4 -0
  81. package/dist/eval-core/report-extensions.js +500 -0
  82. package/dist/eval-core/report-file-migration.js +7 -2
  83. package/dist/eval-core/resume-compatibility.d.ts +31 -0
  84. package/dist/eval-core/resume-compatibility.js +141 -0
  85. package/dist/eval-core/sample-fingerprint.d.ts +12 -0
  86. package/dist/eval-core/sample-fingerprint.js +193 -0
  87. package/dist/eval-core/schema.js +86 -31
  88. package/dist/eval-core/verdict.d.ts +8 -4
  89. package/dist/eval-core/verdict.js +24 -10
  90. package/dist/eval-workflows/batch-evaluation-workflow.d.ts +2 -1
  91. package/dist/eval-workflows/batch-evaluation-workflow.js +25 -12
  92. package/dist/eval-workflows/evaluation-pipeline/preflight-warnings.d.ts +10 -5
  93. package/dist/eval-workflows/evaluation-pipeline/preflight-warnings.js +58 -21
  94. package/dist/eval-workflows/evaluation-pipeline/report-finalize.js +3 -1
  95. package/dist/eval-workflows/evaluation-pipeline/run-state.d.ts +4 -1
  96. package/dist/eval-workflows/evaluation-pipeline/run-state.js +4 -1
  97. package/dist/eval-workflows/evaluation-pipeline/test-set-hash.d.ts +6 -5
  98. package/dist/eval-workflows/evaluation-pipeline/test-set-hash.js +17 -10
  99. package/dist/eval-workflows/evaluation-pipeline.js +12 -7
  100. package/dist/eval-workflows/run-evaluation.d.ts +9 -7
  101. package/dist/eval-workflows/run-evaluation.js +79 -51
  102. package/dist/executors/anthropic-api.js +65 -9
  103. package/dist/executors/claude-cli.js +16 -79
  104. package/dist/executors/claude-protocol.d.ts +28 -0
  105. package/dist/executors/claude-protocol.js +180 -0
  106. package/dist/executors/claude-sdk-trace.js +56 -28
  107. package/dist/executors/claude-sdk.d.ts +1 -0
  108. package/dist/executors/claude-sdk.js +39 -93
  109. package/dist/executors/codex-cli-trace.js +166 -31
  110. package/dist/executors/codex-cli.d.ts +6 -8
  111. package/dist/executors/codex-cli.js +49 -151
  112. package/dist/executors/codex-protocol.d.ts +24 -0
  113. package/dist/executors/codex-protocol.js +234 -0
  114. package/dist/executors/codex-sdk.js +68 -120
  115. package/dist/executors/gemini.js +88 -13
  116. package/dist/executors/index.d.ts +2 -3
  117. package/dist/executors/index.js +5 -3
  118. package/dist/executors/openai-api.js +70 -9
  119. package/dist/executors/runtime-fingerprint.js +88 -11
  120. package/dist/executors/script-command.d.ts +8 -0
  121. package/dist/executors/script-command.js +87 -0
  122. package/dist/executors/script.js +202 -29
  123. package/dist/executors/shared.d.ts +35 -3
  124. package/dist/executors/shared.js +113 -15
  125. package/dist/grading/assertions.d.ts +1 -1
  126. package/dist/grading/assertions.js +19 -9
  127. package/dist/grading/diagnostic.d.ts +9 -2
  128. package/dist/grading/diagnostic.js +25 -2
  129. package/dist/grading/index.js +10 -4
  130. package/dist/grading/judge.js +19 -6
  131. package/dist/grading/layered-scores.d.ts +2 -3
  132. package/dist/grading/layered-scores.js +2 -3
  133. package/dist/inputs/load-samples.d.ts +1 -2
  134. package/dist/inputs/load-samples.js +23 -1
  135. package/dist/inputs/mcp-resolver.js +6 -3
  136. package/dist/inputs/sample-document.d.ts +11 -0
  137. package/dist/inputs/sample-document.js +96 -0
  138. package/dist/managed/evidence.d.ts +1 -0
  139. package/dist/managed/evidence.js +1 -1
  140. package/dist/managed/store.js +200 -91
  141. package/dist/observability/codex-trace-adapter.d.ts +5 -0
  142. package/dist/observability/codex-trace-adapter.js +850 -0
  143. package/dist/observability/experience.d.ts +32 -6
  144. package/dist/observability/experience.js +2695 -459
  145. package/dist/observability/feedback-matchers.js +16 -1
  146. package/dist/observability/inbox-view-model.d.ts +2 -1
  147. package/dist/observability/inbox-view-model.js +20 -14
  148. package/dist/observability/inbox.d.ts +7 -1
  149. package/dist/observability/inbox.js +632 -124
  150. package/dist/observability/problem-patterns.js +2 -0
  151. package/dist/observability/review-state.d.ts +6 -0
  152. package/dist/observability/review-state.js +235 -63
  153. package/dist/observability/skill-chain-advisories.js +1 -1
  154. package/dist/observability/skill-chain.js +17 -4
  155. package/dist/observability/skill-health-analyzer.d.ts +32 -7
  156. package/dist/observability/skill-health-analyzer.js +194 -121
  157. package/dist/observability/skill-health-report.d.ts +10 -0
  158. package/dist/observability/skill-health-report.js +620 -0
  159. package/dist/observability/soft-standards/constants.d.ts +0 -1
  160. package/dist/observability/soft-standards/constants.js +0 -1
  161. package/dist/observability/soft-standards/index.d.ts +1 -1
  162. package/dist/observability/soft-standards/index.js +1 -1
  163. package/dist/observability/soft-standards/llm-extractor.js +8 -10
  164. package/dist/observability/soft-standards/skill-standards-store.d.ts +2 -1
  165. package/dist/observability/soft-standards/skill-standards-store.js +59 -18
  166. package/dist/observability/soft-standards/types.d.ts +2 -2
  167. package/dist/observability/trace-adapter.d.ts +12 -7
  168. package/dist/observability/trace-adapter.js +11 -9
  169. package/dist/observability/trace-attribution.d.ts +13 -5
  170. package/dist/observability/trace-attribution.js +315 -21
  171. package/dist/observability/trace-ingestion.d.ts +9 -0
  172. package/dist/observability/trace-ingestion.js +80 -0
  173. package/dist/observability/trace-ir.d.ts +113 -0
  174. package/dist/observability/trace-ir.js +87 -0
  175. package/dist/observability/trace-segmenter.d.ts +19 -6
  176. package/dist/observability/trace-segmenter.js +377 -196
  177. package/dist/observability/trace-session-index.d.ts +19 -0
  178. package/dist/observability/trace-session-index.js +68 -0
  179. package/dist/observability/trace-source.d.ts +12 -4
  180. package/dist/observability/trace-source.js +939 -215
  181. package/dist/renderer/html-renderer.js +37 -6
  182. package/dist/renderer/icons.js +3 -0
  183. package/dist/renderer/observation-inbox-renderer.js +227 -91
  184. package/dist/renderer/skill-detail-renderer.js +452 -109
  185. package/dist/renderer/skill-health-renderer.js +69 -12
  186. package/dist/renderer/summary.js +28 -7
  187. package/dist/renderer/table.js +21 -4
  188. package/dist/renderer/test-view.d.ts +1 -0
  189. package/dist/renderer/test-view.js +44 -9
  190. package/dist/server/indexed-report-store.js +14 -18
  191. package/dist/server/job-store.js +64 -26
  192. package/dist/server/report-server.js +190 -78
  193. package/dist/server/report-store.js +57 -80
  194. package/dist/server/skill-index.js +143 -49
  195. package/dist/server/skill-insights.js +44 -5
  196. package/dist/shared/artifact-graph.d.ts +3 -0
  197. package/dist/shared/artifact-graph.js +224 -0
  198. package/dist/shared/assertion-types.d.ts +8 -0
  199. package/dist/shared/assertion-types.js +46 -0
  200. package/dist/shared/atomic-json.d.ts +8 -0
  201. package/dist/shared/atomic-json.js +33 -0
  202. package/dist/shared/diagnosis-schema.d.ts +9 -0
  203. package/dist/shared/diagnosis-schema.js +181 -0
  204. package/dist/shared/doctor-report.d.ts +3 -0
  205. package/dist/shared/doctor-report.js +103 -0
  206. package/dist/shared/evaluation-job.d.ts +6 -0
  207. package/dist/shared/evaluation-job.js +217 -0
  208. package/dist/shared/executor-result.d.ts +17 -0
  209. package/dist/shared/executor-result.js +221 -0
  210. package/dist/shared/file-lock.d.ts +12 -0
  211. package/dist/shared/file-lock.js +129 -0
  212. package/dist/shared/json-value.d.ts +5 -0
  213. package/dist/shared/json-value.js +36 -0
  214. package/dist/shared/keyed-mutex.d.ts +7 -0
  215. package/dist/shared/keyed-mutex.js +24 -0
  216. package/dist/shared/record-count.d.ts +8 -0
  217. package/dist/shared/record-count.js +43 -0
  218. package/dist/shared/sample-contract.d.ts +3 -0
  219. package/dist/shared/sample-contract.js +332 -0
  220. package/dist/shared/shell-quote.d.ts +2 -0
  221. package/dist/shared/shell-quote.js +7 -0
  222. package/dist/shared/timestamp.d.ts +6 -0
  223. package/dist/shared/timestamp.js +64 -0
  224. package/dist/shared/token-usage.d.ts +19 -0
  225. package/dist/shared/token-usage.js +50 -0
  226. package/dist/shared/tool-call-status.d.ts +8 -0
  227. package/dist/shared/tool-call-status.js +28 -0
  228. package/dist/shared/tool-identity.d.ts +21 -0
  229. package/dist/shared/tool-identity.js +84 -0
  230. package/dist/shared/tool-search.js +73 -16
  231. package/dist/shared/trace-projection.d.ts +5 -0
  232. package/dist/shared/trace-projection.js +20 -0
  233. package/dist/shared/trace-source-kind.d.ts +3 -0
  234. package/dist/shared/trace-source-kind.js +12 -0
  235. package/dist/types/diagnosis.d.ts +2 -0
  236. package/dist/types/eval.d.ts +4 -0
  237. package/dist/types/executor.d.ts +32 -5
  238. package/dist/types/index.d.ts +1 -0
  239. package/dist/types/index.js +1 -0
  240. package/dist/types/judge.d.ts +2 -0
  241. package/dist/types/observability.d.ts +116 -9
  242. package/dist/types/report.d.ts +58 -6
  243. package/dist/types/skill-index.d.ts +7 -0
  244. package/dist/types/trace.d.ts +2 -0
  245. package/dist/types/trace.js +1 -0
  246. package/package.json +9 -5
@@ -196,12 +196,14 @@ function evidenceRefFromEvent(event) {
196
196
  return {
197
197
  id: event.id,
198
198
  kind: event.kind,
199
+ traceId: event.traceId,
199
200
  sourceTrace: event.sourceTrace,
200
201
  sessionId: event.sessionId,
201
202
  messageIndex: event.messageIndex,
202
203
  logicalMessageIndex: event.logicalMessageIndex ?? event.messageIndex,
203
204
  sourceLineIndex: event.sourceLineIndex ?? event.messageIndex,
204
205
  messageUuid: event.messageUuid,
206
+ callInstanceId: event.callInstanceId,
205
207
  toolUseId: event.toolUseId,
206
208
  timestamp: event.timestamp,
207
209
  role: event.role,
@@ -1,17 +1,23 @@
1
1
  import type { ObservationMetricKey, ObservationMetricScope, ObservationReviewState, ObservationReviewStateEntry, ObservationReviewStateUpdate, ObservationReviewTargetType, ObservationReviewVerdict } from '../types/index.js';
2
2
  export type { ObservationMetricKey, ObservationMetricScope, ObservationReviewState, ObservationReviewStateEntry, ObservationReviewStateUpdate, ObservationReviewTargetType, ObservationReviewVerdict, };
3
+ export declare class ObservationReviewStateValidationError extends Error {
4
+ readonly name = "ObservationReviewStateValidationError";
5
+ }
3
6
  export declare function observationReviewStateKey(targetType: ObservationReviewTargetType, targetId: string): string;
4
7
  export declare function observationMetricScopeFor(metricKey: ObservationMetricKey): ObservationMetricScope;
5
8
  export declare function observationMetricAnnotationTargetId(ref: {
6
9
  id?: string;
10
+ traceId?: string;
7
11
  sourceTrace?: string;
8
12
  sessionId?: string;
9
13
  messageIndex?: number;
10
14
  messageUuid?: string;
15
+ callInstanceId?: string;
11
16
  toolUseId?: string;
12
17
  metricScopeId?: string;
13
18
  }, metricKey: ObservationMetricKey): string;
14
19
  export declare function observationMetricAnnotationVerdict(state: ObservationReviewState | undefined, ref: Parameters<typeof observationMetricAnnotationTargetId>[0], metricKey: ObservationMetricKey): 'confirmed' | 'rejected' | undefined;
20
+ export declare function observationMetricAnnotationEntry(state: ObservationReviewState | undefined, ref: Parameters<typeof observationMetricAnnotationTargetId>[0], metricKey: ObservationMetricKey): ObservationReviewStateEntry | undefined;
15
21
  export declare function observationReviewStatePath(observationsDir: string): string;
16
22
  export declare function emptyObservationReviewState(now?: string): ObservationReviewState;
17
23
  export declare function loadObservationReviewState(observationsDir: string): ObservationReviewState;
@@ -1,6 +1,12 @@
1
1
  import { createHash } from 'node:crypto';
2
- import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
2
+ import { existsSync, readFileSync, } from 'node:fs';
3
3
  import { join } from 'node:path';
4
+ import { writeJsonFileAtomic } from '../shared/atomic-json.js';
5
+ import { withFileLock } from '../shared/file-lock.js';
6
+ import { normalizeRfc3339Timestamp } from '../shared/timestamp.js';
7
+ export class ObservationReviewStateValidationError extends Error {
8
+ name = 'ObservationReviewStateValidationError';
9
+ }
4
10
  export function observationReviewStateKey(targetType, targetId) {
5
11
  return `${targetType}:${targetId}`;
6
12
  }
@@ -14,36 +20,69 @@ export function observationMetricScopeFor(metricKey) {
14
20
  : 'message';
15
21
  }
16
22
  export function observationMetricAnnotationTargetId(ref, metricKey) {
17
- const hasStableTraceRef = Boolean(ref.sourceTrace || ref.sessionId || ref.messageIndex !== undefined || ref.messageUuid || ref.toolUseId);
23
+ const hasStableTraceRef = Boolean(ref.traceId
24
+ || ref.sourceTrace
25
+ || ref.sessionId
26
+ || ref.messageIndex !== undefined
27
+ || ref.messageUuid
28
+ || ref.callInstanceId
29
+ || ref.toolUseId);
18
30
  const scope = observationMetricScopeFor(metricKey);
19
31
  const stable = [
20
32
  metricKey,
21
33
  ...(scope === 'skill_segment' ? [scope, ref.metricScopeId ?? ''] : []),
34
+ ref.traceId ?? '',
22
35
  ref.sourceTrace ?? '',
23
36
  ref.sessionId ?? '',
24
37
  ref.messageIndex === undefined ? '' : String(ref.messageIndex),
25
38
  ref.messageUuid ?? '',
39
+ ref.callInstanceId ?? '',
26
40
  ref.toolUseId ?? '',
27
41
  hasStableTraceRef ? '' : ref.id ?? '',
28
42
  ].join('\u0000');
29
43
  return `metric:${metricKey}:${createHash('sha256').update(stable).digest('hex').slice(0, 16)}`;
30
44
  }
31
45
  export function observationMetricAnnotationVerdict(state, ref, metricKey) {
46
+ const entry = observationMetricAnnotationEntry(state, ref, metricKey);
47
+ return entry?.verdict === 'confirmed' || entry?.verdict === 'rejected'
48
+ ? entry.verdict
49
+ : undefined;
50
+ }
51
+ export function observationMetricAnnotationEntry(state, ref, metricKey) {
32
52
  if (!state)
33
53
  return undefined;
34
- const targetId = observationMetricAnnotationTargetId(ref, metricKey);
35
- const entry = state.entries[observationReviewStateKey('evidence_metric', targetId)];
36
- return entry?.verdict === 'confirmed' || entry?.verdict === 'rejected' ? entry.verdict : undefined;
54
+ const candidates = [
55
+ ref,
56
+ ...(ref.callInstanceId ? [{ ...ref, callInstanceId: undefined }] : []),
57
+ ...(ref.traceId ? [{ ...ref, traceId: undefined }] : []),
58
+ ...(ref.traceId && ref.callInstanceId
59
+ ? [{ ...ref, traceId: undefined, callInstanceId: undefined }]
60
+ : []),
61
+ ];
62
+ const seen = new Set();
63
+ for (const candidate of candidates) {
64
+ const targetId = observationMetricAnnotationTargetId(candidate, metricKey);
65
+ if (seen.has(targetId))
66
+ continue;
67
+ seen.add(targetId);
68
+ const entry = state.entries[observationReviewStateKey('evidence_metric', targetId)];
69
+ if (entry)
70
+ return entry;
71
+ }
72
+ return undefined;
37
73
  }
38
74
  export function observationReviewStatePath(observationsDir) {
39
75
  return join(observationsDir, 'review-state.json');
40
76
  }
41
77
  const OBSERVATION_REVIEW_STATE_SCHEMA_VERSION = 2;
42
78
  export function emptyObservationReviewState(now = new Date().toISOString()) {
79
+ const updatedAt = normalizedTimestamp(now);
80
+ if (!updatedAt)
81
+ throw new ObservationReviewStateValidationError('invalid updatedAt');
43
82
  return {
44
83
  kind: 'observe-review-state',
45
84
  schemaVersion: OBSERVATION_REVIEW_STATE_SCHEMA_VERSION,
46
- updatedAt: now,
85
+ updatedAt,
47
86
  entries: {},
48
87
  };
49
88
  }
@@ -51,39 +90,57 @@ export function loadObservationReviewState(observationsDir) {
51
90
  const path = observationReviewStatePath(observationsDir);
52
91
  if (!existsSync(path))
53
92
  return emptyObservationReviewState();
93
+ let parsed;
54
94
  try {
55
- const parsed = JSON.parse(readFileSync(path, 'utf-8'));
56
- return normalizeObservationReviewState(parsed) ?? emptyObservationReviewState();
95
+ parsed = JSON.parse(readFileSync(path, 'utf-8'));
57
96
  }
58
- catch {
59
- return emptyObservationReviewState();
97
+ catch (cause) {
98
+ throw new Error(`无法读取观测复核状态:${path}`, { cause });
60
99
  }
100
+ const normalized = normalizeObservationReviewState(parsed);
101
+ if (!normalized)
102
+ throw new Error(`观测复核状态格式无效:${path}`);
103
+ return normalized;
61
104
  }
62
- export function updateObservationReviewState(observationsDir, update, now = new Date().toISOString()) {
105
+ export function updateObservationReviewState(observationsDir, update, now) {
63
106
  assertReviewStateUpdate(update);
64
- const state = loadObservationReviewState(observationsDir);
65
- const key = observationReviewStateKey(update.targetType, update.targetId);
66
- state.entries[key] = {
67
- targetType: update.targetType,
68
- targetId: update.targetId,
69
- verdict: update.verdict,
70
- reviewedAt: now,
71
- ...(update.note ? { note: update.note.slice(0, 500) } : {}),
72
- ...(update.reason ? { reason: update.reason.slice(0, 500) } : {}),
73
- ...(update.metricKey ? { metricKey: update.metricKey } : {}),
74
- ...(update.metricKey ? { metricScope: update.metricScope ?? observationMetricScopeFor(update.metricKey) } : {}),
75
- ...(update.metricScopeId ? { metricScopeId: update.metricScopeId.slice(0, 200) } : {}),
76
- ...(update.sourceTrace ? { sourceTrace: update.sourceTrace } : {}),
77
- ...(update.sessionId ? { sessionId: update.sessionId } : {}),
78
- ...(update.messageIndex !== undefined ? { messageIndex: update.messageIndex } : {}),
79
- ...(update.messageUuid ? { messageUuid: update.messageUuid } : {}),
80
- ...(update.toolUseId ? { toolUseId: update.toolUseId } : {}),
81
- ...(update.snippet ? { snippet: update.snippet.slice(0, 500) } : {}),
82
- };
83
- state.updatedAt = now;
84
- mkdirSync(observationsDir, { recursive: true });
85
- writeFileSync(observationReviewStatePath(observationsDir), JSON.stringify(state, null, 2));
86
- return state;
107
+ const path = observationReviewStatePath(observationsDir);
108
+ return withFileLock(`${path}.lock`, () => {
109
+ const reviewedAt = normalizedTimestamp(now ?? new Date().toISOString());
110
+ if (!reviewedAt) {
111
+ throw new ObservationReviewStateValidationError('invalid reviewedAt');
112
+ }
113
+ const hadPersistedState = existsSync(path);
114
+ const state = loadObservationReviewState(observationsDir);
115
+ if (hadPersistedState && reviewedAt < state.updatedAt) {
116
+ throw new ObservationReviewStateValidationError('reviewedAt cannot precede state updatedAt');
117
+ }
118
+ const key = observationReviewStateKey(update.targetType, update.targetId);
119
+ state.entries[key] = {
120
+ targetType: update.targetType,
121
+ targetId: update.targetId,
122
+ verdict: update.verdict,
123
+ reviewedAt,
124
+ ...(update.note ? { note: update.note.slice(0, 500) } : {}),
125
+ ...(update.reason ? { reason: update.reason.slice(0, 500) } : {}),
126
+ ...(update.metricKey ? { metricKey: update.metricKey } : {}),
127
+ ...(update.metricKey ? {
128
+ metricScope: update.metricScope ?? observationMetricScopeFor(update.metricKey),
129
+ } : {}),
130
+ ...(update.metricScopeId ? { metricScopeId: update.metricScopeId.slice(0, 200) } : {}),
131
+ ...(update.traceId ? { traceId: update.traceId } : {}),
132
+ ...(update.sourceTrace ? { sourceTrace: update.sourceTrace } : {}),
133
+ ...(update.sessionId ? { sessionId: update.sessionId } : {}),
134
+ ...(update.messageIndex !== undefined ? { messageIndex: update.messageIndex } : {}),
135
+ ...(update.messageUuid ? { messageUuid: update.messageUuid } : {}),
136
+ ...(update.callInstanceId ? { callInstanceId: update.callInstanceId } : {}),
137
+ ...(update.toolUseId ? { toolUseId: update.toolUseId } : {}),
138
+ ...(update.snippet ? { snippet: update.snippet.slice(0, 500) } : {}),
139
+ };
140
+ state.updatedAt = reviewedAt;
141
+ writeObservationReviewState(observationsDir, state);
142
+ return state;
143
+ }, { label: 'observation review state' });
87
144
  }
88
145
  function normalizeObservationReviewState(value) {
89
146
  if (!value || typeof value !== 'object')
@@ -94,50 +151,165 @@ function normalizeObservationReviewState(value) {
94
151
  return null;
95
152
  if (parsed.schemaVersion !== OBSERVATION_REVIEW_STATE_SCHEMA_VERSION)
96
153
  return null;
97
- if (!parsed.entries || typeof parsed.entries !== 'object')
154
+ if (!parsed.entries || typeof parsed.entries !== 'object' || Array.isArray(parsed.entries))
155
+ return null;
156
+ const updatedAt = normalizedTimestamp(parsed.updatedAt);
157
+ if (!updatedAt)
158
+ return null;
159
+ const entries = {};
160
+ for (const [key, rawEntry] of Object.entries(parsed.entries)) {
161
+ const entry = normalizeReviewStateEntry(rawEntry);
162
+ if (!entry || key !== observationReviewStateKey(entry.targetType, entry.targetId))
163
+ return null;
164
+ entries[key] = entry;
165
+ }
166
+ if (Object.values(entries).some((entry) => entry.reviewedAt > updatedAt))
98
167
  return null;
99
168
  return {
100
169
  kind: 'observe-review-state',
101
170
  schemaVersion: OBSERVATION_REVIEW_STATE_SCHEMA_VERSION,
102
- updatedAt: typeof parsed.updatedAt === 'string' ? parsed.updatedAt : new Date().toISOString(),
103
- entries: Object.fromEntries(Object.entries(parsed.entries).filter(([, entry]) => isReviewStateEntry(entry))),
171
+ updatedAt,
172
+ entries,
104
173
  };
105
174
  }
106
- export function deleteObservationReviewState(observationsDir, targetType, targetId, now = new Date().toISOString()) {
175
+ export function deleteObservationReviewState(observationsDir, targetType, targetId, now) {
107
176
  if (!isReviewTargetType(targetType))
108
- throw new Error('invalid review targetType');
177
+ throw new ObservationReviewStateValidationError('invalid review targetType');
109
178
  if (typeof targetId !== 'string' || targetId.trim() === '')
110
- throw new Error('invalid review targetId');
111
- const state = loadObservationReviewState(observationsDir);
112
- const key = observationReviewStateKey(targetType, targetId);
113
- delete state.entries[key];
114
- state.updatedAt = now;
115
- mkdirSync(observationsDir, { recursive: true });
116
- writeFileSync(observationReviewStatePath(observationsDir), JSON.stringify(state, null, 2));
117
- return state;
179
+ throw new ObservationReviewStateValidationError('invalid review targetId');
180
+ const path = observationReviewStatePath(observationsDir);
181
+ return withFileLock(`${path}.lock`, () => {
182
+ const updatedAt = normalizedTimestamp(now ?? new Date().toISOString());
183
+ if (!updatedAt) {
184
+ throw new ObservationReviewStateValidationError('invalid updatedAt');
185
+ }
186
+ const hadPersistedState = existsSync(path);
187
+ const state = loadObservationReviewState(observationsDir);
188
+ if (hadPersistedState && updatedAt < state.updatedAt) {
189
+ throw new ObservationReviewStateValidationError('updatedAt cannot precede state updatedAt');
190
+ }
191
+ const key = observationReviewStateKey(targetType, targetId);
192
+ delete state.entries[key];
193
+ state.updatedAt = updatedAt;
194
+ writeObservationReviewState(observationsDir, state);
195
+ return state;
196
+ }, { label: 'observation review state' });
118
197
  }
119
198
  function assertReviewStateUpdate(value) {
120
199
  if (!isReviewTargetType(value.targetType))
121
- throw new Error('invalid review targetType');
200
+ throw new ObservationReviewStateValidationError('invalid review targetType');
122
201
  if (typeof value.targetId !== 'string' || value.targetId.trim() === '')
123
- throw new Error('invalid review targetId');
202
+ throw new ObservationReviewStateValidationError('invalid review targetId');
124
203
  if (!isReviewVerdict(value.verdict))
125
- throw new Error('invalid review verdict');
126
- if (value.targetType === 'evidence_metric' && !isObservationMetricKey(value.metricKey))
127
- throw new Error('invalid metricKey');
128
- if (value.metricScope !== undefined && !isObservationMetricScope(value.metricScope))
129
- throw new Error('invalid metricScope');
130
- if ((value.verdict === 'confirmed' || value.verdict === 'rejected') && value.targetType !== 'evidence_metric')
131
- throw new Error('metric verdict requires evidence_metric targetType');
132
- }
133
- function isReviewStateEntry(value) {
204
+ throw new ObservationReviewStateValidationError('invalid review verdict');
205
+ const optionalStrings = [
206
+ value.note,
207
+ value.reason,
208
+ value.metricScopeId,
209
+ value.traceId,
210
+ value.sourceTrace,
211
+ value.sessionId,
212
+ value.messageUuid,
213
+ value.callInstanceId,
214
+ value.toolUseId,
215
+ value.snippet,
216
+ ];
217
+ if (!optionalStrings.every(optionalString))
218
+ throw new ObservationReviewStateValidationError('invalid review metadata');
219
+ if (value.messageIndex !== undefined
220
+ && (!Number.isSafeInteger(value.messageIndex) || value.messageIndex < 0))
221
+ throw new ObservationReviewStateValidationError('invalid messageIndex');
222
+ if (value.targetType === 'evidence_metric') {
223
+ if (!isObservationMetricKey(value.metricKey))
224
+ throw new ObservationReviewStateValidationError('invalid metricKey');
225
+ if (value.verdict !== 'confirmed' && value.verdict !== 'rejected') {
226
+ throw new ObservationReviewStateValidationError('evidence_metric requires confirmed or rejected verdict');
227
+ }
228
+ const expectedScope = observationMetricScopeFor(value.metricKey);
229
+ if (value.metricScope !== undefined && value.metricScope !== expectedScope) {
230
+ throw new ObservationReviewStateValidationError('metricScope does not match metricKey');
231
+ }
232
+ }
233
+ else {
234
+ if (value.verdict === 'confirmed' || value.verdict === 'rejected') {
235
+ throw new ObservationReviewStateValidationError('metric verdict requires evidence_metric targetType');
236
+ }
237
+ if (value.metricKey !== undefined || value.metricScope !== undefined || value.metricScopeId !== undefined) {
238
+ throw new ObservationReviewStateValidationError('metric metadata requires evidence_metric targetType');
239
+ }
240
+ }
241
+ }
242
+ function normalizeReviewStateEntry(value) {
134
243
  if (!value || typeof value !== 'object')
135
- return false;
244
+ return null;
136
245
  const entry = value;
137
- return isReviewTargetType(entry.targetType)
138
- && typeof entry.targetId === 'string'
139
- && isReviewVerdict(entry.verdict)
140
- && typeof entry.reviewedAt === 'string';
246
+ const reviewedAt = normalizedTimestamp(entry.reviewedAt);
247
+ if (!isReviewTargetType(entry.targetType)
248
+ || typeof entry.targetId !== 'string'
249
+ || entry.targetId.trim() === ''
250
+ || !isReviewVerdict(entry.verdict)
251
+ || !reviewedAt
252
+ || !optionalString(entry.note)
253
+ || !optionalString(entry.reason)
254
+ || !optionalString(entry.metricScopeId)
255
+ || !optionalString(entry.traceId)
256
+ || !optionalString(entry.sourceTrace)
257
+ || !optionalString(entry.sessionId)
258
+ || !optionalString(entry.messageUuid)
259
+ || !optionalString(entry.callInstanceId)
260
+ || !optionalString(entry.toolUseId)
261
+ || !optionalString(entry.snippet)
262
+ || (entry.messageIndex !== undefined
263
+ && (!Number.isSafeInteger(entry.messageIndex) || entry.messageIndex < 0))
264
+ || (entry.metricKey !== undefined && !isObservationMetricKey(entry.metricKey))
265
+ || (entry.metricScope !== undefined && !isObservationMetricScope(entry.metricScope)))
266
+ return null;
267
+ if (entry.targetType === 'evidence_metric') {
268
+ if (!isObservationMetricKey(entry.metricKey)
269
+ || (entry.verdict !== 'confirmed' && entry.verdict !== 'rejected'))
270
+ return null;
271
+ }
272
+ else if (entry.verdict === 'confirmed'
273
+ || entry.verdict === 'rejected'
274
+ || entry.metricKey !== undefined
275
+ || entry.metricScope !== undefined
276
+ || entry.metricScopeId !== undefined) {
277
+ return null;
278
+ }
279
+ const metricScope = entry.metricKey
280
+ ? entry.metricScope ?? observationMetricScopeFor(entry.metricKey)
281
+ : undefined;
282
+ if (entry.metricKey
283
+ && metricScope !== observationMetricScopeFor(entry.metricKey))
284
+ return null;
285
+ return {
286
+ targetType: entry.targetType,
287
+ targetId: entry.targetId,
288
+ verdict: entry.verdict,
289
+ reviewedAt,
290
+ ...(entry.note !== undefined ? { note: entry.note } : {}),
291
+ ...(entry.reason !== undefined ? { reason: entry.reason } : {}),
292
+ ...(entry.metricKey !== undefined ? { metricKey: entry.metricKey, metricScope } : {}),
293
+ ...(entry.metricScopeId !== undefined ? { metricScopeId: entry.metricScopeId } : {}),
294
+ ...(entry.traceId !== undefined ? { traceId: entry.traceId } : {}),
295
+ ...(entry.sourceTrace !== undefined ? { sourceTrace: entry.sourceTrace } : {}),
296
+ ...(entry.sessionId !== undefined ? { sessionId: entry.sessionId } : {}),
297
+ ...(entry.messageIndex !== undefined ? { messageIndex: entry.messageIndex } : {}),
298
+ ...(entry.messageUuid !== undefined ? { messageUuid: entry.messageUuid } : {}),
299
+ ...(entry.callInstanceId !== undefined ? { callInstanceId: entry.callInstanceId } : {}),
300
+ ...(entry.toolUseId !== undefined ? { toolUseId: entry.toolUseId } : {}),
301
+ ...(entry.snippet !== undefined ? { snippet: entry.snippet } : {}),
302
+ };
303
+ }
304
+ function normalizedTimestamp(value) {
305
+ return normalizeRfc3339Timestamp(value) ?? null;
306
+ }
307
+ function optionalString(value) {
308
+ return value === undefined || typeof value === 'string';
309
+ }
310
+ function writeObservationReviewState(observationsDir, state) {
311
+ const path = observationReviewStatePath(observationsDir);
312
+ writeJsonFileAtomic(path, state);
141
313
  }
142
314
  function isReviewTargetType(value) {
143
315
  return value === 'experience_session'
@@ -50,7 +50,7 @@ const ADVISORIES = {
50
50
  code: 'skill_md_not_found',
51
51
  shortLabel: '本地未找到 SKILL.md',
52
52
  message: '本地常见 skill 目录里没有这个 skill 的 SKILL.md,inbox 跑不了静态/运行时审计。如果你想看 LLM 健康度审计(触发边界 / 文档 / 依赖 / 安全等 7 维),可以试试 omk doctor;inbox 本身不依赖它。',
53
- commandTemplate: 'omk doctor ~/.claude/skills/${skillName}/SKILL.md',
53
+ commandTemplate: 'omk doctor /path/to/${skillName}/SKILL.md',
54
54
  },
55
55
  };
56
56
  export function getSkillChainAdvisory(code) {
@@ -2,6 +2,7 @@ import { existsSync, readFileSync, readdirSync } from 'node:fs';
2
2
  import { homedir } from 'node:os';
3
3
  import { join } from 'node:path';
4
4
  import { extractMarkdownStepWorkflows, validateSkillHardRules, validateSkillWorkflows, } from '../shared/hard-rules.js';
5
+ import { incrementRecordCount } from '../shared/record-count.js';
5
6
  const MAX_SKILL_DEFINITION_CHARS = 12000;
6
7
  export function buildObservationSkillChains(skillNames, cwd = process.cwd(), experienceReports = []) {
7
8
  const uniqueSkillNames = Array.from(new Set(skillNames.filter(Boolean))).sort();
@@ -199,7 +200,7 @@ function runtimeEvidence(invocations) {
199
200
  const events = invocations.flatMap((invocation) => invocation.timeline ?? []);
200
201
  for (const invocation of invocations) {
201
202
  for (const [tool, count] of Object.entries(invocation.toolCounts ?? {})) {
202
- toolCounts[tool] = (toolCounts[tool] ?? 0) + count;
203
+ incrementRecordCount(toolCounts, tool, count);
203
204
  }
204
205
  }
205
206
  return {
@@ -207,8 +208,14 @@ function runtimeEvidence(invocations) {
207
208
  toolCallCount: invocations.reduce((sum, invocation) => sum + invocation.indicators.toolCallCount, 0),
208
209
  toolFailureCount: invocations.reduce((sum, invocation) => sum + invocation.indicators.toolFailureCount, 0),
209
210
  events,
210
- assistantToolUseIds: new Set(events
211
- .filter((event) => event.kind === 'tool_use' && event.role === 'assistant' && event.toolUseId)
211
+ assistantCallInstanceIds: new Set(events
212
+ .filter((event) => event.kind === 'tool_use' && event.role === 'assistant' && event.callInstanceId)
213
+ .map((event) => event.callInstanceId)),
214
+ legacyAssistantToolUseIds: new Set(events
215
+ .filter((event) => event.kind === 'tool_use'
216
+ && event.role === 'assistant'
217
+ && !event.callInstanceId
218
+ && event.toolUseId)
212
219
  .map((event) => event.toolUseId)),
213
220
  };
214
221
  }
@@ -351,6 +358,7 @@ function eventToEvidencePackRef(event) {
351
358
  messageIndex: event.messageIndex,
352
359
  logicalMessageIndex: event.logicalMessageIndex,
353
360
  sourceLineIndex: event.sourceLineIndex,
361
+ callInstanceId: event.callInstanceId,
354
362
  toolUseId: event.toolUseId,
355
363
  timestamp: event.timestamp,
356
364
  role: event.role,
@@ -384,7 +392,12 @@ function isAssistantAuthoredUploadEvent(event, evidence, text) {
384
392
  return false;
385
393
  if (event.kind === 'tool_use' && event.role === 'assistant')
386
394
  return true;
387
- if (event.kind === 'tool_result' && event.role === 'tool' && event.toolUseId && evidence.assistantToolUseIds.has(event.toolUseId))
395
+ if (event.kind === 'tool_result'
396
+ && event.role === 'tool'
397
+ && (event.callInstanceId
398
+ ? evidence.assistantCallInstanceIds.has(event.callInstanceId)
399
+ : Boolean(event.toolUseId)
400
+ && evidence.legacyAssistantToolUseIds.has(event.toolUseId)))
388
401
  return true;
389
402
  return false;
390
403
  }
@@ -6,28 +6,33 @@
6
6
  * 定位是"真实使用 trace 的 skill 维度观察",不是通用 APM / 生产监控。
7
7
  *
8
8
  * 分析流水线:
9
- * 1. ccTracesToResultEntries(path) → segments + ResultEntry[]
9
+ * 1. tracesToResultEntries(path) → segments + ResultEntry[]
10
10
  * 2. 时间窗 / skill 白名单过滤
11
11
  * 3. 按 skill name (variant key) 分别 computeCoverage + computeGapReport
12
12
  * 4. 聚合 overall 指标 + 健康度色带
13
13
  */
14
14
  import { type CoverageReport } from '../analysis/coverage-analyzer.js';
15
- import type { GapReport } from '../types/index.js';
16
- import { type CcSession, type SkillSegment } from './trace-adapter.js';
15
+ import type { GapReport, TraceIngestionSummary } from '../types/index.js';
16
+ import { type CcSession, type TraceSession, type SkillSegment } from './trace-adapter.js';
17
17
  export interface SkillHealth {
18
18
  skillName: string;
19
19
  segmentCount: number;
20
20
  toolCallCount: number;
21
21
  toolFailureCount: number;
22
- /** 失败率 = toolFailureCount / toolCallCount; toolCallCount=0 时为 0 */
22
+ toolCancelledCount?: number;
23
+ toolUnknownCount?: number;
24
+ toolResolvedCount?: number;
25
+ toolOutcomeCoverage?: number;
26
+ /** 失败率 = toolFailureCount / 可比较结果数;取消与未知状态不进入分母。 */
23
27
  toolFailureRate: number;
24
28
  /**
25
29
  * 执行稳定性标签。阈值:
26
30
  * - very-unstable: failureRate >= 0.4 (gap 信号极可能是环境问题,不是真知识缺口)
27
31
  * - unstable: failureRate >= 0.2 (建议排查环境后再看 gap)
32
+ * - unknown: 有工具调用,但没有可比较的成功 / 失败结果
28
33
  * - stable: 否则
29
34
  */
30
- stability: 'stable' | 'unstable' | 'very-unstable';
35
+ stability: 'stable' | 'unstable' | 'very-unstable' | 'unknown';
31
36
  /**
32
37
  * 统计可信度(按 segment 数)。守护「1 段 + 一次失败就判 red」的过度自信:
33
38
  * - underpowered: segmentCount < 5 (样本太少,色带 / gap rate 不可信)
@@ -43,6 +48,8 @@ export interface SkillHealth {
43
48
  cacheReadTokens: number;
44
49
  cacheCreationTokens: number;
45
50
  totalTokens: number;
51
+ tokenObservedSegmentCount: number;
52
+ tokenCoverage: number;
46
53
  durationMs: number;
47
54
  numTurns: number;
48
55
  avgTokensPerSegment: number;
@@ -61,13 +68,21 @@ export interface SkillHealthReport {
61
68
  sessionCount: number;
62
69
  segmentCount: number;
63
70
  messageCount: number;
71
+ timestampedSegmentCount?: number;
72
+ timestampCoverage?: number;
73
+ excludedUntimestampedSegmentCount?: number;
64
74
  toolCallCount: number;
75
+ toolCancelledCount?: number;
76
+ toolUnknownCount?: number;
77
+ toolResolvedCount?: number;
78
+ toolOutcomeCoverage?: number;
65
79
  toolFailureRate: number;
66
80
  timeRange: {
67
81
  from: string;
68
82
  to: string;
69
83
  };
70
84
  generatedAt: string;
85
+ ingestion?: TraceIngestionSummary;
71
86
  };
72
87
  bySkill: Record<string, SkillHealth>;
73
88
  overall: {
@@ -89,15 +104,25 @@ export interface AnalyzeOptions {
89
104
  * #235 受管反哺时 observe CLI 用本函数逐 skill 算 band 传入 managed —— 阈值单一来源,managed 不复制。
90
105
  */
91
106
  export declare function healthBandOf(weightedGapRate: number): 'green' | 'yellow' | 'red';
107
+ /**
108
+ * 按 per-skill 失败率判定执行稳定性。阈值见 SkillHealth.stability。
109
+ */
110
+ export declare function toolStabilityOf(toolFailureRate: number, comparableToolCalls: number, totalToolCalls: number): SkillHealth['stability'];
92
111
  /** 统计可信度护栏:按 segment 数判 high / low / underpowered,守护「1 段就判 red」。
93
112
  * 导出供 renderer / CLI / server 给历史报告(缺 confidence 字段)做 segmentCount 兜底。 */
94
113
  export declare function confidenceOf(segmentCount: number): SkillHealth['confidence'];
95
114
  /**
96
- * 主入口:从 cc trace 目录生成 SkillHealthReport。
115
+ * 推断 KB root: 没传 --kb 时,取第一个 assistant record 的 cwd。
116
+ * 如果跨多个 cwd,取第一个并 warn。
117
+ */
118
+ type HealthSession = TraceSession | CcSession;
119
+ /**
120
+ * 主入口:从受支持的 trace 输入生成 SkillHealthReport。
97
121
  */
98
122
  export declare function computeSkillHealthReport(tracePath: string, opts?: AnalyzeOptions): SkillHealthReport;
99
123
  /**
100
124
  * 便利入口:直接从已准备好的 segments(跳过 loadCcSessions)算 report。
101
125
  * 用于测试 / 已手工组装过 segments 的场景。
102
126
  */
103
- export declare function computeSkillHealthFromSegments(segments: SkillSegment[], sessions: CcSession[], tracePath: string, opts?: AnalyzeOptions): SkillHealthReport;
127
+ export declare function computeSkillHealthFromSegments(segments: SkillSegment[], sessions: HealthSession[], tracePath: string, opts?: AnalyzeOptions, ingestion?: TraceIngestionSummary): SkillHealthReport;
128
+ export {};