oh-my-knowledge 0.48.0 → 0.49.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (243) hide show
  1. package/README.md +50 -18
  2. package/README.zh.md +55 -23
  3. package/dist/analysis/coverage-analyzer.d.ts +1 -0
  4. package/dist/analysis/coverage-analyzer.js +125 -62
  5. package/dist/analysis/failure-clusterer.js +2 -1
  6. package/dist/analysis/gap-analyzer.d.ts +2 -2
  7. package/dist/analysis/gap-analyzer.js +13 -3
  8. package/dist/analysis/hedging-classifier.d.ts +2 -2
  9. package/dist/analysis/hedging-classifier.js +3 -4
  10. package/dist/analysis/report-diagnostics.js +9 -7
  11. package/dist/analysis/sample-diagnostics.js +6 -6
  12. package/dist/artifact-graph/doctor.js +15 -7
  13. package/dist/assets/agent-skills/omk/SKILL.md +27 -7
  14. package/dist/assets/agent-skills/omk/references/commands.md +18 -17
  15. package/dist/authoring/evolver.d.ts +10 -6
  16. package/dist/authoring/evolver.js +496 -83
  17. package/dist/authoring/generator.d.ts +3 -3
  18. package/dist/authoring/generator.js +5 -10
  19. package/dist/authoring/sample-fixer.d.ts +8 -6
  20. package/dist/authoring/sample-fixer.js +76 -5
  21. package/dist/cli/commands/doctor.js +31 -14
  22. package/dist/cli/commands/eval/index.d.ts +3 -0
  23. package/dist/cli/commands/eval/index.js +163 -17
  24. package/dist/cli/commands/evolve.d.ts +4 -4
  25. package/dist/cli/commands/evolve.js +27 -13
  26. package/dist/cli/commands/init.js +16 -3
  27. package/dist/cli/commands/observe/inbox.js +28 -21
  28. package/dist/cli/commands/observe/index.js +20 -11
  29. package/dist/cli/commands/observe/ingest.d.ts +3 -0
  30. package/dist/cli/commands/observe/ingest.js +30 -2
  31. package/dist/cli/commands/sample.d.ts +6 -3
  32. package/dist/cli/commands/sample.js +72 -68
  33. package/dist/cli/lib/codex-model-hint.d.ts +9 -0
  34. package/dist/cli/lib/codex-model-hint.js +45 -0
  35. package/dist/cli/lib/generation-failure-hint.d.ts +2 -0
  36. package/dist/cli/lib/generation-failure-hint.js +61 -0
  37. package/dist/cli/lib/i18n-dict/common.d.ts +1 -1
  38. package/dist/cli/lib/i18n-dict/common.js +4 -0
  39. package/dist/cli/lib/i18n-dict/gen.d.ts +1 -1
  40. package/dist/cli/lib/i18n-dict/gen.js +38 -6
  41. package/dist/cli/lib/i18n-dict/help.js +6 -6
  42. package/dist/cli/lib/i18n-dict/init.d.ts +1 -1
  43. package/dist/cli/lib/i18n-dict/init.js +13 -9
  44. package/dist/cli/lib/i18n-dict/run.d.ts +1 -1
  45. package/dist/cli/lib/i18n-dict/run.js +34 -2
  46. package/dist/cli/lib/llm-failure-classifier.d.ts +2 -0
  47. package/dist/cli/lib/llm-failure-classifier.js +8 -0
  48. package/dist/cli/lib/parse-run-config.d.ts +6 -5
  49. package/dist/cli/lib/parse-run-config.js +16 -9
  50. package/dist/cli/lib/runtime-defaults.d.ts +21 -0
  51. package/dist/cli/lib/runtime-defaults.js +79 -0
  52. package/dist/diagnosis/observe-mapper.js +14 -15
  53. package/dist/diagnosis/observe-producer.js +3 -1
  54. package/dist/diagnosis/studio-projection.js +14 -7
  55. package/dist/diagnosis/types.d.ts +2 -0
  56. package/dist/diagnosis/types.js +12 -0
  57. package/dist/doctor/endpoint-rule.js +2 -1
  58. package/dist/eval-core/artifact-file-names.js +18 -1
  59. package/dist/eval-core/artifact-index.d.ts +7 -11
  60. package/dist/eval-core/artifact-index.js +139 -80
  61. package/dist/eval-core/cache.d.ts +12 -3
  62. package/dist/eval-core/cache.js +89 -29
  63. package/dist/eval-core/comparability.js +10 -6
  64. package/dist/eval-core/evaluation-execution.d.ts +2 -1
  65. package/dist/eval-core/evaluation-execution.js +122 -37
  66. package/dist/eval-core/evaluation-job.d.ts +4 -1
  67. package/dist/eval-core/evaluation-job.js +4 -1
  68. package/dist/eval-core/evaluation-reporting.d.ts +15 -13
  69. package/dist/eval-core/evaluation-reporting.js +54 -52
  70. package/dist/eval-core/execution-strategy.d.ts +2 -0
  71. package/dist/eval-core/execution-strategy.js +11 -9
  72. package/dist/eval-core/fact-checker.js +15 -7
  73. package/dist/eval-core/holdout.js +3 -2
  74. package/dist/eval-core/judge-independence.d.ts +2 -2
  75. package/dist/eval-core/mock-hook.cjs +23 -6
  76. package/dist/eval-core/mocks-runtime.js +30 -8
  77. package/dist/eval-core/report-document.d.ts +12 -0
  78. package/dist/eval-core/report-document.js +1151 -0
  79. package/dist/eval-core/report-extensions.d.ts +4 -0
  80. package/dist/eval-core/report-extensions.js +500 -0
  81. package/dist/eval-core/report-file-migration.js +7 -2
  82. package/dist/eval-core/resume-compatibility.d.ts +31 -0
  83. package/dist/eval-core/resume-compatibility.js +141 -0
  84. package/dist/eval-core/sample-fingerprint.d.ts +12 -0
  85. package/dist/eval-core/sample-fingerprint.js +193 -0
  86. package/dist/eval-core/schema.js +86 -31
  87. package/dist/eval-core/verdict.d.ts +8 -4
  88. package/dist/eval-core/verdict.js +24 -10
  89. package/dist/eval-workflows/batch-evaluation-workflow.d.ts +2 -1
  90. package/dist/eval-workflows/batch-evaluation-workflow.js +25 -12
  91. package/dist/eval-workflows/evaluation-pipeline/preflight-warnings.d.ts +10 -5
  92. package/dist/eval-workflows/evaluation-pipeline/preflight-warnings.js +58 -21
  93. package/dist/eval-workflows/evaluation-pipeline/report-finalize.js +3 -1
  94. package/dist/eval-workflows/evaluation-pipeline/run-state.d.ts +4 -1
  95. package/dist/eval-workflows/evaluation-pipeline/run-state.js +4 -1
  96. package/dist/eval-workflows/evaluation-pipeline/test-set-hash.d.ts +6 -5
  97. package/dist/eval-workflows/evaluation-pipeline/test-set-hash.js +17 -10
  98. package/dist/eval-workflows/evaluation-pipeline.js +12 -7
  99. package/dist/eval-workflows/run-evaluation.d.ts +9 -7
  100. package/dist/eval-workflows/run-evaluation.js +79 -51
  101. package/dist/executors/anthropic-api.js +65 -9
  102. package/dist/executors/claude-cli.js +16 -79
  103. package/dist/executors/claude-protocol.d.ts +28 -0
  104. package/dist/executors/claude-protocol.js +180 -0
  105. package/dist/executors/claude-sdk-trace.js +56 -28
  106. package/dist/executors/claude-sdk.d.ts +1 -0
  107. package/dist/executors/claude-sdk.js +39 -93
  108. package/dist/executors/codex-cli-trace.js +166 -31
  109. package/dist/executors/codex-cli.d.ts +6 -8
  110. package/dist/executors/codex-cli.js +49 -151
  111. package/dist/executors/codex-protocol.d.ts +24 -0
  112. package/dist/executors/codex-protocol.js +234 -0
  113. package/dist/executors/codex-sdk.js +68 -120
  114. package/dist/executors/gemini.js +88 -13
  115. package/dist/executors/index.d.ts +2 -3
  116. package/dist/executors/index.js +5 -3
  117. package/dist/executors/openai-api.js +70 -9
  118. package/dist/executors/runtime-fingerprint.js +88 -11
  119. package/dist/executors/script-command.d.ts +8 -0
  120. package/dist/executors/script-command.js +87 -0
  121. package/dist/executors/script.js +202 -29
  122. package/dist/executors/shared.d.ts +35 -3
  123. package/dist/executors/shared.js +113 -15
  124. package/dist/grading/assertions.d.ts +1 -1
  125. package/dist/grading/assertions.js +19 -9
  126. package/dist/grading/diagnostic.d.ts +9 -2
  127. package/dist/grading/diagnostic.js +25 -2
  128. package/dist/grading/index.js +10 -4
  129. package/dist/grading/judge.js +19 -6
  130. package/dist/grading/layered-scores.d.ts +2 -3
  131. package/dist/grading/layered-scores.js +2 -3
  132. package/dist/inputs/load-samples.d.ts +1 -2
  133. package/dist/inputs/load-samples.js +23 -1
  134. package/dist/inputs/mcp-resolver.js +6 -3
  135. package/dist/inputs/sample-document.d.ts +11 -0
  136. package/dist/inputs/sample-document.js +96 -0
  137. package/dist/managed/evidence.d.ts +1 -0
  138. package/dist/managed/evidence.js +1 -1
  139. package/dist/managed/store.js +200 -91
  140. package/dist/observability/codex-trace-adapter.d.ts +5 -0
  141. package/dist/observability/codex-trace-adapter.js +850 -0
  142. package/dist/observability/experience.d.ts +32 -6
  143. package/dist/observability/experience.js +2695 -459
  144. package/dist/observability/feedback-matchers.js +16 -1
  145. package/dist/observability/inbox-view-model.d.ts +1 -1
  146. package/dist/observability/inbox-view-model.js +19 -14
  147. package/dist/observability/inbox.d.ts +7 -1
  148. package/dist/observability/inbox.js +632 -124
  149. package/dist/observability/problem-patterns.js +2 -0
  150. package/dist/observability/review-state.d.ts +6 -0
  151. package/dist/observability/review-state.js +235 -63
  152. package/dist/observability/skill-chain-advisories.js +1 -1
  153. package/dist/observability/skill-chain.js +17 -4
  154. package/dist/observability/skill-health-analyzer.d.ts +32 -7
  155. package/dist/observability/skill-health-analyzer.js +194 -121
  156. package/dist/observability/skill-health-report.d.ts +10 -0
  157. package/dist/observability/skill-health-report.js +620 -0
  158. package/dist/observability/soft-standards/constants.d.ts +0 -1
  159. package/dist/observability/soft-standards/constants.js +0 -1
  160. package/dist/observability/soft-standards/index.d.ts +1 -1
  161. package/dist/observability/soft-standards/index.js +1 -1
  162. package/dist/observability/soft-standards/llm-extractor.js +8 -10
  163. package/dist/observability/soft-standards/skill-standards-store.d.ts +2 -1
  164. package/dist/observability/soft-standards/skill-standards-store.js +59 -18
  165. package/dist/observability/soft-standards/types.d.ts +2 -2
  166. package/dist/observability/trace-adapter.d.ts +12 -7
  167. package/dist/observability/trace-adapter.js +11 -9
  168. package/dist/observability/trace-attribution.d.ts +13 -5
  169. package/dist/observability/trace-attribution.js +315 -21
  170. package/dist/observability/trace-ingestion.d.ts +9 -0
  171. package/dist/observability/trace-ingestion.js +80 -0
  172. package/dist/observability/trace-ir.d.ts +113 -0
  173. package/dist/observability/trace-ir.js +87 -0
  174. package/dist/observability/trace-segmenter.d.ts +19 -6
  175. package/dist/observability/trace-segmenter.js +377 -196
  176. package/dist/observability/trace-session-index.d.ts +19 -0
  177. package/dist/observability/trace-session-index.js +68 -0
  178. package/dist/observability/trace-source.d.ts +12 -4
  179. package/dist/observability/trace-source.js +939 -215
  180. package/dist/renderer/html-renderer.js +37 -6
  181. package/dist/renderer/icons.js +3 -0
  182. package/dist/renderer/observation-inbox-renderer.js +208 -90
  183. package/dist/renderer/skill-detail-renderer.js +452 -109
  184. package/dist/renderer/skill-health-renderer.js +69 -12
  185. package/dist/renderer/summary.js +28 -7
  186. package/dist/renderer/table.js +21 -4
  187. package/dist/renderer/test-view.d.ts +1 -0
  188. package/dist/renderer/test-view.js +44 -9
  189. package/dist/server/indexed-report-store.js +14 -18
  190. package/dist/server/job-store.js +64 -26
  191. package/dist/server/report-server.js +190 -78
  192. package/dist/server/report-store.js +57 -80
  193. package/dist/server/skill-index.js +143 -49
  194. package/dist/server/skill-insights.js +44 -5
  195. package/dist/shared/artifact-graph.d.ts +3 -0
  196. package/dist/shared/artifact-graph.js +224 -0
  197. package/dist/shared/assertion-types.d.ts +8 -0
  198. package/dist/shared/assertion-types.js +46 -0
  199. package/dist/shared/atomic-json.d.ts +8 -0
  200. package/dist/shared/atomic-json.js +33 -0
  201. package/dist/shared/diagnosis-schema.d.ts +9 -0
  202. package/dist/shared/diagnosis-schema.js +181 -0
  203. package/dist/shared/doctor-report.d.ts +3 -0
  204. package/dist/shared/doctor-report.js +103 -0
  205. package/dist/shared/evaluation-job.d.ts +6 -0
  206. package/dist/shared/evaluation-job.js +217 -0
  207. package/dist/shared/executor-result.d.ts +17 -0
  208. package/dist/shared/executor-result.js +221 -0
  209. package/dist/shared/file-lock.d.ts +12 -0
  210. package/dist/shared/file-lock.js +129 -0
  211. package/dist/shared/json-value.d.ts +5 -0
  212. package/dist/shared/json-value.js +36 -0
  213. package/dist/shared/keyed-mutex.d.ts +7 -0
  214. package/dist/shared/keyed-mutex.js +24 -0
  215. package/dist/shared/record-count.d.ts +8 -0
  216. package/dist/shared/record-count.js +43 -0
  217. package/dist/shared/sample-contract.d.ts +3 -0
  218. package/dist/shared/sample-contract.js +332 -0
  219. package/dist/shared/timestamp.d.ts +6 -0
  220. package/dist/shared/timestamp.js +64 -0
  221. package/dist/shared/token-usage.d.ts +19 -0
  222. package/dist/shared/token-usage.js +50 -0
  223. package/dist/shared/tool-call-status.d.ts +8 -0
  224. package/dist/shared/tool-call-status.js +28 -0
  225. package/dist/shared/tool-identity.d.ts +21 -0
  226. package/dist/shared/tool-identity.js +84 -0
  227. package/dist/shared/tool-search.js +73 -16
  228. package/dist/shared/trace-projection.d.ts +5 -0
  229. package/dist/shared/trace-projection.js +20 -0
  230. package/dist/shared/trace-source-kind.d.ts +3 -0
  231. package/dist/shared/trace-source-kind.js +12 -0
  232. package/dist/types/diagnosis.d.ts +2 -0
  233. package/dist/types/eval.d.ts +4 -0
  234. package/dist/types/executor.d.ts +32 -5
  235. package/dist/types/index.d.ts +1 -0
  236. package/dist/types/index.js +1 -0
  237. package/dist/types/judge.d.ts +2 -0
  238. package/dist/types/observability.d.ts +116 -9
  239. package/dist/types/report.d.ts +58 -6
  240. package/dist/types/skill-index.d.ts +7 -0
  241. package/dist/types/trace.d.ts +2 -0
  242. package/dist/types/trace.js +1 -0
  243. package/package.json +9 -5
@@ -0,0 +1,217 @@
1
+ import { isRfc3339Timestamp } from './timestamp.js';
2
+ function isRecord(value) {
3
+ return Boolean(value) && typeof value === 'object' && !Array.isArray(value);
4
+ }
5
+ function isOptionalString(value) {
6
+ return value === undefined || typeof value === 'string';
7
+ }
8
+ function isOptionalBoolean(value) {
9
+ return value === undefined || typeof value === 'boolean';
10
+ }
11
+ function isOptionalPositiveInteger(value) {
12
+ return value === undefined || (Number.isSafeInteger(value) && value > 0);
13
+ }
14
+ function isOptionalTimestamp(value) {
15
+ return value === undefined || isRfc3339Timestamp(value);
16
+ }
17
+ function isOptionalBudget(value) {
18
+ if (value === undefined)
19
+ return true;
20
+ if (!isRecord(value))
21
+ return false;
22
+ return [
23
+ value.totalUSD,
24
+ value.perSampleUSD,
25
+ value.perSampleMs,
26
+ ].every((metric) => metric === undefined
27
+ || (typeof metric === 'number'
28
+ && Number.isFinite(metric)
29
+ && metric >= 0
30
+ && metric <= Number.MAX_SAFE_INTEGER));
31
+ }
32
+ function isArtifact(value) {
33
+ if (!isRecord(value))
34
+ return false;
35
+ return typeof value.name === 'string'
36
+ && value.name.length > 0
37
+ && (value.kind === 'baseline'
38
+ || value.kind === 'skill'
39
+ || value.kind === 'prompt'
40
+ || value.kind === 'agent'
41
+ || value.kind === 'workflow')
42
+ && (value.source === 'baseline'
43
+ || value.source === 'variant-name'
44
+ || value.source === 'file-path'
45
+ || value.source === 'git'
46
+ || value.source === 'inline'
47
+ || value.source === 'custom')
48
+ && (value.content === null || typeof value.content === 'string')
49
+ && [
50
+ value.contentHash,
51
+ value.locator,
52
+ value.ref,
53
+ value.resolvedCommit,
54
+ value.cwd,
55
+ value.skillRoot,
56
+ value.execRoot,
57
+ ].every(isOptionalString)
58
+ && (value.experimentRole === undefined
59
+ || value.experimentRole === 'control'
60
+ || value.experimentRole === 'treatment')
61
+ && (value.allowedSkills === undefined
62
+ || (Array.isArray(value.allowedSkills)
63
+ && value.allowedSkills.every((skill) => typeof skill === 'string' && skill.length > 0)
64
+ && new Set(value.allowedSkills).size === value.allowedSkills.length))
65
+ && (value.metadata === undefined || isRecord(value.metadata));
66
+ }
67
+ export function isSafeEvaluationJobId(value) {
68
+ return typeof value === 'string'
69
+ && /^[A-Za-z0-9][A-Za-z0-9._-]{0,255}$/.test(value);
70
+ }
71
+ export function isValidEvaluationRequest(value) {
72
+ if (!isRecord(value))
73
+ return false;
74
+ return typeof value.samplesPath === 'string'
75
+ && typeof value.skillDir === 'string'
76
+ && Array.isArray(value.artifacts)
77
+ && value.artifacts.every(isArtifact)
78
+ && new Set(value.artifacts.map((artifact) => artifact.name)).size === value.artifacts.length
79
+ && typeof value.model === 'string'
80
+ && value.model.trim().length > 0
81
+ && typeof value.executor === 'string'
82
+ && value.executor.trim().length > 0
83
+ && typeof value.noJudge === 'boolean'
84
+ && Number.isSafeInteger(value.concurrency)
85
+ && value.concurrency > 0
86
+ && typeof value.noCache === 'boolean'
87
+ && typeof value.dryRun === 'boolean'
88
+ && Array.isArray(value.judgeModels)
89
+ && value.judgeModels.length > 0
90
+ && value.judgeModels.every((judge) => isRecord(judge)
91
+ && typeof judge.executor === 'string'
92
+ && judge.executor.trim().length > 0
93
+ && typeof judge.model === 'string'
94
+ && judge.model.trim().length > 0)
95
+ && isOptionalString(value.project)
96
+ && isOptionalString(value.owner)
97
+ && (value.tags === undefined
98
+ || (Array.isArray(value.tags)
99
+ && value.tags.every((tag) => typeof tag === 'string')))
100
+ && isOptionalPositiveInteger(value.timeoutMs)
101
+ && isOptionalPositiveInteger(value.repeat)
102
+ && (value.holdoutRatio === undefined
103
+ || (typeof value.holdoutRatio === 'number'
104
+ && Number.isFinite(value.holdoutRatio)
105
+ && value.holdoutRatio > 0
106
+ && value.holdoutRatio < 1))
107
+ && isOptionalBoolean(value.batch)
108
+ && isOptionalPositiveInteger(value.judgeRepeat)
109
+ && isOptionalBoolean(value.bootstrap)
110
+ && isOptionalPositiveInteger(value.bootstrapSamples)
111
+ && isOptionalBoolean(value.lengthDebias)
112
+ && isOptionalBudget(value.budget)
113
+ && isOptionalBoolean(value.strictBaseline)
114
+ && (value.retry === undefined
115
+ || (Number.isSafeInteger(value.retry) && value.retry >= 0))
116
+ && isOptionalBoolean(value.noDiagnostic)
117
+ && (value.effort === undefined
118
+ || value.effort === 'low'
119
+ || value.effort === 'medium'
120
+ || value.effort === 'high'
121
+ || value.effort === 'xhigh'
122
+ || value.effort === 'max');
123
+ }
124
+ export function isValidEvaluationRun(value, expectedRunId) {
125
+ if (!isRecord(value)
126
+ || typeof value.runId !== 'string'
127
+ || value.runId.length === 0
128
+ || (expectedRunId !== undefined && value.runId !== expectedRunId)
129
+ || !isRfc3339Timestamp(value.startedAt)
130
+ || !isOptionalTimestamp(value.finishedAt)
131
+ || (value.status !== 'running'
132
+ && value.status !== 'succeeded'
133
+ && value.status !== 'failed'
134
+ && value.status !== 'cancelled'))
135
+ return false;
136
+ if (value.status === 'running')
137
+ return value.finishedAt === undefined;
138
+ return value.finishedAt !== undefined
139
+ && Date.parse(value.finishedAt) >= Date.parse(value.startedAt);
140
+ }
141
+ function jobStateIsConsistent(job) {
142
+ if (job.status === 'queued') {
143
+ return job.startedAt === undefined
144
+ && job.finishedAt === undefined
145
+ && job.runId === undefined
146
+ && job.resultReportId === undefined
147
+ && job.error === undefined
148
+ && job.errorCategory === undefined;
149
+ }
150
+ if (job.status === 'running') {
151
+ return job.startedAt !== undefined
152
+ && job.finishedAt === undefined
153
+ && typeof job.runId === 'string'
154
+ && job.runId.length > 0
155
+ && job.resultReportId === undefined
156
+ && job.error === undefined
157
+ && job.errorCategory === undefined;
158
+ }
159
+ if (job.status === 'succeeded') {
160
+ return job.startedAt !== undefined
161
+ && job.finishedAt !== undefined
162
+ && typeof job.runId === 'string'
163
+ && job.runId.length > 0
164
+ && typeof job.resultReportId === 'string'
165
+ && job.resultReportId.length > 0
166
+ && job.error === undefined
167
+ && job.errorCategory === undefined;
168
+ }
169
+ if (job.status === 'failed') {
170
+ return job.finishedAt !== undefined
171
+ && typeof job.error === 'string'
172
+ && job.error.length > 0
173
+ && job.errorCategory !== undefined
174
+ && job.resultReportId === undefined;
175
+ }
176
+ return job.finishedAt !== undefined && job.resultReportId === undefined;
177
+ }
178
+ export function isValidEvaluationJob(value, expectedJobId) {
179
+ if (!isRecord(value)
180
+ || !isSafeEvaluationJobId(value.jobId)
181
+ || (expectedJobId !== undefined && value.jobId !== expectedJobId)
182
+ || (value.status !== 'queued'
183
+ && value.status !== 'running'
184
+ && value.status !== 'succeeded'
185
+ && value.status !== 'failed'
186
+ && value.status !== 'cancelled')
187
+ || !isRfc3339Timestamp(value.createdAt)
188
+ || !isValidEvaluationRequest(value.request)
189
+ || !isOptionalTimestamp(value.updatedAt)
190
+ || !isOptionalTimestamp(value.startedAt)
191
+ || !isOptionalTimestamp(value.finishedAt)
192
+ || !isOptionalString(value.runId)
193
+ || !isOptionalString(value.resultReportId)
194
+ || !isOptionalString(value.error)
195
+ || (value.errorCategory !== undefined
196
+ && value.errorCategory !== 'user'
197
+ && value.errorCategory !== 'executor'
198
+ && value.errorCategory !== 'judge'
199
+ && value.errorCategory !== 'system'))
200
+ return false;
201
+ const createdAtMs = Date.parse(value.createdAt);
202
+ const startedAtMs = value.startedAt === undefined ? undefined : Date.parse(value.startedAt);
203
+ const finishedAtMs = value.finishedAt === undefined ? undefined : Date.parse(value.finishedAt);
204
+ const updatedAtMs = value.updatedAt === undefined ? undefined : Date.parse(value.updatedAt);
205
+ if ((startedAtMs !== undefined && startedAtMs < createdAtMs)
206
+ || (finishedAtMs !== undefined && finishedAtMs < createdAtMs)
207
+ || (startedAtMs !== undefined
208
+ && finishedAtMs !== undefined
209
+ && finishedAtMs < startedAtMs)
210
+ || (updatedAtMs !== undefined
211
+ && updatedAtMs < Math.max(createdAtMs, startedAtMs ?? createdAtMs, finishedAtMs ?? createdAtMs)))
212
+ return false;
213
+ return jobStateIsConsistent(value);
214
+ }
215
+ export function evaluationRequestsEqual(left, right) {
216
+ return JSON.stringify(left) === JSON.stringify(right);
217
+ }
@@ -0,0 +1,17 @@
1
+ import type { ExecResult, ToolCallInfo, TurnInfo } from '../types/index.js';
2
+ /**
3
+ * Validate the source-neutral executor result contract at process/cache
4
+ * boundaries. Provider payloads and custom executors are untrusted input; a
5
+ * malformed metric must fail the sample instead of entering a report.
6
+ */
7
+ export declare function executorResultValidationError(value: unknown): string | undefined;
8
+ export declare function parseExecResult(value: unknown): ExecResult | null;
9
+ /**
10
+ * Canonicalize every execution trace after structural validation. This keeps
11
+ * programmatic/custom executors and restored caches in the same comparison
12
+ * namespace as built-in provider adapters.
13
+ */
14
+ export declare function normalizeExecResultToolIdentities(result: ExecResult): ExecResult;
15
+ export declare function isValidTurnInfo(value: unknown): value is TurnInfo;
16
+ export declare function isValidToolCallInfo(value: unknown): value is ToolCallInfo;
17
+ export declare function isValidMockStats(value: unknown): boolean;
@@ -0,0 +1,221 @@
1
+ import { checkedSumTokenCounts, optionalTokenCount } from './token-usage.js';
2
+ import { normalizeRfc3339Timestamp } from './timestamp.js';
3
+ import { normalizeToolIdentity } from './tool-identity.js';
4
+ import { isTraceSourceKind } from './trace-source-kind.js';
5
+ import { isJsonValue } from './json-value.js';
6
+ const TOOL_STATUSES = new Set(['success', 'failure', 'cancelled', 'unknown']);
7
+ const TOOL_STATUS_SOURCES = new Set(['runtime', 'tool-output', 'inferred', 'unknown']);
8
+ const TRACE_ROLES = new Set(['standalone', 'main', 'subagent']);
9
+ const TURN_ROLES = new Set(['user', 'assistant', 'tool']);
10
+ /**
11
+ * Validate the source-neutral executor result contract at process/cache
12
+ * boundaries. Provider payloads and custom executors are untrusted input; a
13
+ * malformed metric must fail the sample instead of entering a report.
14
+ */
15
+ export function executorResultValidationError(value) {
16
+ if (!isRecord(value))
17
+ return 'executor result must be an object';
18
+ if (typeof value.ok !== 'boolean')
19
+ return '"ok" must be boolean';
20
+ if (value.output !== null && typeof value.output !== 'string') {
21
+ return '"output" must be string or null';
22
+ }
23
+ if (value.ok && (typeof value.output !== 'string' || value.output.trim() === '')) {
24
+ return 'successful executor result must contain model output';
25
+ }
26
+ for (const field of ['durationMs', 'durationApiMs']) {
27
+ if (!isNonNegativeSafeMetric(value[field])) {
28
+ return `"${field}" must be a non-negative safe number`;
29
+ }
30
+ }
31
+ for (const field of [
32
+ 'inputTokens',
33
+ 'outputTokens',
34
+ 'cacheReadTokens',
35
+ 'cacheCreationTokens',
36
+ 'numTurns',
37
+ ]) {
38
+ if (optionalTokenCount(value[field]) === undefined) {
39
+ return `"${field}" must be a non-negative safe integer`;
40
+ }
41
+ }
42
+ if (checkedSumTokenCounts(value.inputTokens, value.outputTokens, value.cacheReadTokens, value.cacheCreationTokens) === undefined) {
43
+ return 'token usage aggregate exceeds safe integer';
44
+ }
45
+ if (!isNonNegativeSafeMetric(value.costUSD)) {
46
+ return '"costUSD" must be a non-negative safe number';
47
+ }
48
+ if (typeof value.stopReason !== 'string')
49
+ return '"stopReason" must be string';
50
+ if (!isOptionalBoolean(value.costReportedByExecutor)) {
51
+ return '"costReportedByExecutor" must be boolean when present';
52
+ }
53
+ if (!isOptionalBoolean(value.tokenUsageReportedByExecutor)) {
54
+ return '"tokenUsageReportedByExecutor" must be boolean when present';
55
+ }
56
+ if (!isOptionalBoolean(value.cached))
57
+ return '"cached" must be boolean when present';
58
+ if (!isOptionalString(value.error))
59
+ return '"error" must be string when present';
60
+ if (!isOptionalCount(value.fullNumTurns)) {
61
+ return '"fullNumTurns" must be a non-negative safe integer when present';
62
+ }
63
+ if (!isOptionalCount(value.numSubAgents)) {
64
+ return '"numSubAgents" must be a non-negative safe integer when present';
65
+ }
66
+ if (value.attemptCount !== undefined
67
+ && (typeof value.attemptCount !== 'number'
68
+ || !Number.isSafeInteger(value.attemptCount)
69
+ || value.attemptCount < 1)) {
70
+ return '"attemptCount" must be a positive safe integer when present';
71
+ }
72
+ if (value.turns !== undefined) {
73
+ if (!Array.isArray(value.turns) || !value.turns.every(isValidTurnInfo)) {
74
+ return '"turns" contains an invalid trace entry';
75
+ }
76
+ }
77
+ if (value.toolCalls !== undefined) {
78
+ if (!Array.isArray(value.toolCalls) || !value.toolCalls.every(isValidToolCallInfo)) {
79
+ return '"toolCalls" contains an invalid trace entry';
80
+ }
81
+ }
82
+ if (value.mockStats !== undefined && !isValidMockStats(value.mockStats)) {
83
+ return '"mockStats" contains invalid counters';
84
+ }
85
+ return undefined;
86
+ }
87
+ export function parseExecResult(value) {
88
+ return executorResultValidationError(value)
89
+ ? null
90
+ : normalizeExecResultToolIdentities(value);
91
+ }
92
+ /**
93
+ * Canonicalize every execution trace after structural validation. This keeps
94
+ * programmatic/custom executors and restored caches in the same comparison
95
+ * namespace as built-in provider adapters.
96
+ */
97
+ export function normalizeExecResultToolIdentities(result) {
98
+ const normalizeCall = (call) => {
99
+ const sourceName = call.sourceTool ?? call.tool;
100
+ const isMcpIdentity = sourceName.startsWith('mcp__')
101
+ || sourceName === 'mcp_tool_call'
102
+ || call.toolNamespace?.startsWith('mcp__');
103
+ const identity = normalizeToolIdentity({
104
+ sourceName,
105
+ namespace: call.toolNamespace,
106
+ provider: call.toolProvider,
107
+ authoritativeName: isMcpIdentity && call.tool !== sourceName
108
+ ? call.tool
109
+ : undefined,
110
+ });
111
+ const normalizedCall = {
112
+ ...call,
113
+ tool: identity.name,
114
+ };
115
+ if (identity.sourceName)
116
+ normalizedCall.sourceTool = identity.sourceName;
117
+ else
118
+ delete normalizedCall.sourceTool;
119
+ if (identity.namespace)
120
+ normalizedCall.toolNamespace = identity.namespace;
121
+ else
122
+ delete normalizedCall.toolNamespace;
123
+ if (identity.provider)
124
+ normalizedCall.toolProvider = identity.provider;
125
+ else
126
+ delete normalizedCall.toolProvider;
127
+ return normalizedCall;
128
+ };
129
+ return {
130
+ ...result,
131
+ ...(result.toolCalls && {
132
+ toolCalls: result.toolCalls.map(normalizeCall),
133
+ }),
134
+ ...(result.turns && {
135
+ turns: result.turns.map((turn) => ({
136
+ ...turn,
137
+ ...(turn.toolCalls && {
138
+ toolCalls: turn.toolCalls.map(normalizeCall),
139
+ }),
140
+ })),
141
+ }),
142
+ };
143
+ }
144
+ export function isValidTurnInfo(value) {
145
+ return isRecord(value)
146
+ && typeof value.role === 'string'
147
+ && TURN_ROLES.has(value.role)
148
+ && typeof value.content === 'string'
149
+ && (value.toolCalls === undefined
150
+ || (Array.isArray(value.toolCalls) && value.toolCalls.every(isValidToolCallInfo)))
151
+ && (value.durationMs === undefined
152
+ || isNonNegativeSafeMetric(value.durationMs));
153
+ }
154
+ export function isValidToolCallInfo(value) {
155
+ if (!isRecord(value)
156
+ || typeof value.tool !== 'string'
157
+ || value.tool.trim() === ''
158
+ || !isJsonValue(value.input)
159
+ || !isJsonValue(value.output)
160
+ || typeof value.success !== 'boolean'
161
+ || !isOptionalCount(value.messageIndex))
162
+ return false;
163
+ if (value.status !== undefined
164
+ && (typeof value.status !== 'string'
165
+ || !TOOL_STATUSES.has(value.status)
166
+ || value.success !== (value.status === 'success')))
167
+ return false;
168
+ if (value.statusSource !== undefined
169
+ && (typeof value.statusSource !== 'string'
170
+ || !TOOL_STATUS_SOURCES.has(value.statusSource)))
171
+ return false;
172
+ if (value.sourceKind !== undefined
173
+ && !isTraceSourceKind(value.sourceKind))
174
+ return false;
175
+ if (value.traceRole !== undefined
176
+ && (typeof value.traceRole !== 'string'
177
+ || !TRACE_ROLES.has(value.traceRole)))
178
+ return false;
179
+ if (value.timestamp !== undefined
180
+ && normalizeRfc3339Timestamp(value.timestamp) === undefined)
181
+ return false;
182
+ return [
183
+ value.sourceTool,
184
+ value.toolNamespace,
185
+ value.toolProvider,
186
+ value.messageUuid,
187
+ value.callInstanceId,
188
+ value.toolUseId,
189
+ value.sourceTrace,
190
+ value.traceLabel,
191
+ ].every(isOptionalString);
192
+ }
193
+ export function isValidMockStats(value) {
194
+ if (!isRecord(value)
195
+ || optionalTokenCount(value.hits) === undefined
196
+ || optionalTokenCount(value.misses) === undefined
197
+ || !isRecord(value.perMock))
198
+ return false;
199
+ const perMock = Object.values(value.perMock);
200
+ if (perMock.some((count) => optionalTokenCount(count) === undefined))
201
+ return false;
202
+ return checkedSumTokenCounts(...perMock) === value.hits;
203
+ }
204
+ function isNonNegativeSafeMetric(value) {
205
+ return typeof value === 'number'
206
+ && Number.isFinite(value)
207
+ && value >= 0
208
+ && value <= Number.MAX_SAFE_INTEGER;
209
+ }
210
+ function isOptionalCount(value) {
211
+ return value === undefined || optionalTokenCount(value) !== undefined;
212
+ }
213
+ function isOptionalString(value) {
214
+ return value === undefined || typeof value === 'string';
215
+ }
216
+ function isOptionalBoolean(value) {
217
+ return value === undefined || typeof value === 'boolean';
218
+ }
219
+ function isRecord(value) {
220
+ return Boolean(value) && typeof value === 'object' && !Array.isArray(value);
221
+ }
@@ -0,0 +1,12 @@
1
+ export interface FileLockOptions {
2
+ timeoutMs?: number;
3
+ staleMs?: number;
4
+ retryMs?: number;
5
+ label?: string;
6
+ }
7
+ /**
8
+ * Serialize a short synchronous read-modify-write transaction across
9
+ * processes. The owner token prevents an old holder from deleting a lock that
10
+ * was recovered and reacquired by another process.
11
+ */
12
+ export declare function withFileLock<T>(lockPath: string, operation: () => T, options?: FileLockOptions): T;
@@ -0,0 +1,129 @@
1
+ import { randomUUID } from 'node:crypto';
2
+ import { closeSync, mkdirSync, openSync, readFileSync, statSync, unlinkSync, writeFileSync, } from 'node:fs';
3
+ import { hostname } from 'node:os';
4
+ import { dirname } from 'node:path';
5
+ const LOCK_SLEEP = new Int32Array(new SharedArrayBuffer(4));
6
+ /**
7
+ * Serialize a short synchronous read-modify-write transaction across
8
+ * processes. The owner token prevents an old holder from deleting a lock that
9
+ * was recovered and reacquired by another process.
10
+ */
11
+ export function withFileLock(lockPath, operation, options = {}) {
12
+ const timeoutMs = options.timeoutMs ?? 5_000;
13
+ const staleMs = options.staleMs ?? 30_000;
14
+ const retryMs = options.retryMs ?? 10;
15
+ const label = options.label ?? lockPath;
16
+ const owner = {
17
+ owner: randomUUID(),
18
+ pid: process.pid,
19
+ hostname: hostname(),
20
+ acquiredAt: new Date().toISOString(),
21
+ };
22
+ mkdirSync(dirname(lockPath), { recursive: true });
23
+ const deadline = Date.now() + timeoutMs;
24
+ let descriptor;
25
+ while (descriptor === undefined) {
26
+ try {
27
+ descriptor = openSync(lockPath, 'wx');
28
+ writeFileSync(descriptor, JSON.stringify(owner));
29
+ }
30
+ catch (error) {
31
+ if (descriptor !== undefined) {
32
+ closeSync(descriptor);
33
+ descriptor = undefined;
34
+ try {
35
+ unlinkSync(lockPath);
36
+ }
37
+ catch (unlinkError) {
38
+ if (unlinkError.code !== 'ENOENT') {
39
+ throw unlinkError;
40
+ }
41
+ }
42
+ }
43
+ const code = error.code;
44
+ if (code !== 'EEXIST')
45
+ throw error;
46
+ if (isRecoverableStaleLock(lockPath, staleMs)) {
47
+ try {
48
+ unlinkSync(lockPath);
49
+ }
50
+ catch (unlinkError) {
51
+ if (unlinkError.code !== 'ENOENT') {
52
+ throw unlinkError;
53
+ }
54
+ }
55
+ continue;
56
+ }
57
+ if (Date.now() >= deadline) {
58
+ throw new Error(`timed out waiting for file lock: ${label}`);
59
+ }
60
+ Atomics.wait(LOCK_SLEEP, 0, 0, retryMs);
61
+ }
62
+ }
63
+ try {
64
+ return operation();
65
+ }
66
+ finally {
67
+ closeSync(descriptor);
68
+ removeOwnedLock(lockPath, owner.owner);
69
+ }
70
+ }
71
+ function isRecoverableStaleLock(lockPath, staleMs) {
72
+ let ageMs;
73
+ try {
74
+ ageMs = Date.now() - statSync(lockPath).mtimeMs;
75
+ }
76
+ catch (error) {
77
+ if (error.code === 'ENOENT')
78
+ return true;
79
+ throw error;
80
+ }
81
+ if (ageMs <= staleMs)
82
+ return false;
83
+ const owner = readLockOwner(lockPath);
84
+ if (!owner)
85
+ return true;
86
+ // A remote PID cannot be checked safely. On shared filesystems, deleting a
87
+ // stale-looking lock owned by another host could break an active writer.
88
+ if (owner.hostname !== hostname())
89
+ return false;
90
+ return !isProcessAlive(owner.pid);
91
+ }
92
+ function readLockOwner(lockPath) {
93
+ try {
94
+ const value = JSON.parse(readFileSync(lockPath, 'utf-8'));
95
+ if (!value
96
+ || typeof value !== 'object'
97
+ || typeof value.owner !== 'string'
98
+ || !Number.isSafeInteger(value.pid)
99
+ || value.pid <= 0
100
+ || typeof value.hostname !== 'string'
101
+ || typeof value.acquiredAt !== 'string')
102
+ return null;
103
+ return value;
104
+ }
105
+ catch {
106
+ return null;
107
+ }
108
+ }
109
+ function isProcessAlive(pid) {
110
+ try {
111
+ process.kill(pid, 0);
112
+ return true;
113
+ }
114
+ catch (error) {
115
+ return error.code !== 'ESRCH';
116
+ }
117
+ }
118
+ function removeOwnedLock(lockPath, owner) {
119
+ const current = readLockOwner(lockPath);
120
+ if (!current || current.owner !== owner)
121
+ return;
122
+ try {
123
+ unlinkSync(lockPath);
124
+ }
125
+ catch (error) {
126
+ if (error.code !== 'ENOENT')
127
+ throw error;
128
+ }
129
+ }
@@ -0,0 +1,5 @@
1
+ /**
2
+ * Runtime guard for data that can cross OMK's JSON persistence boundaries
3
+ * without coercion, property loss, or cycle errors.
4
+ */
5
+ export declare function isJsonValue(value: unknown, seen?: Set<object>, depth?: number): boolean;
@@ -0,0 +1,36 @@
1
+ /**
2
+ * Runtime guard for data that can cross OMK's JSON persistence boundaries
3
+ * without coercion, property loss, or cycle errors.
4
+ */
5
+ export function isJsonValue(value, seen = new Set(), depth = 0) {
6
+ if (value === null
7
+ || typeof value === 'string'
8
+ || typeof value === 'boolean')
9
+ return true;
10
+ if (typeof value === 'number')
11
+ return Number.isFinite(value);
12
+ if (typeof value !== 'object' || depth >= 32 || seen.has(value))
13
+ return false;
14
+ try {
15
+ if (Array.isArray(value)) {
16
+ if (Object.keys(value).length !== value.length
17
+ || value.some((_, index) => !Object.hasOwn(value, index)))
18
+ return false;
19
+ }
20
+ else {
21
+ const prototype = Object.getPrototypeOf(value);
22
+ if (prototype !== Object.prototype && prototype !== null)
23
+ return false;
24
+ }
25
+ seen.add(value);
26
+ const valid = Array.isArray(value)
27
+ ? value.every((entry) => isJsonValue(entry, seen, depth + 1))
28
+ : Object.values(value).every((entry) => isJsonValue(entry, seen, depth + 1));
29
+ seen.delete(value);
30
+ return valid;
31
+ }
32
+ catch {
33
+ seen.delete(value);
34
+ return false;
35
+ }
36
+ }
@@ -0,0 +1,7 @@
1
+ /**
2
+ * Serialize asynchronous mutations by key without coupling unrelated keys.
3
+ */
4
+ export declare class KeyedMutex {
5
+ private readonly tails;
6
+ run<T>(key: string, operation: () => Promise<T>): Promise<T>;
7
+ }
@@ -0,0 +1,24 @@
1
+ /**
2
+ * Serialize asynchronous mutations by key without coupling unrelated keys.
3
+ */
4
+ export class KeyedMutex {
5
+ tails = new Map();
6
+ async run(key, operation) {
7
+ const previous = this.tails.get(key) ?? Promise.resolve();
8
+ let release;
9
+ const current = new Promise((resolve) => {
10
+ release = resolve;
11
+ });
12
+ this.tails.set(key, current);
13
+ await previous;
14
+ try {
15
+ return await operation();
16
+ }
17
+ finally {
18
+ release();
19
+ if (this.tails.get(key) === current) {
20
+ this.tails.delete(key);
21
+ }
22
+ }
23
+ }
24
+ }
@@ -0,0 +1,8 @@
1
+ export declare function ownRecordValue<T>(record: Record<string, T>, key: string): T | undefined;
2
+ export declare function setOwnRecordValue<T>(record: Record<string, T>, key: string, value: T): T;
3
+ /**
4
+ * Increment an arbitrary string-keyed counter without reading inherited
5
+ * properties such as `constructor` or invoking the legacy `__proto__` setter.
6
+ */
7
+ export declare function incrementRecordCount(record: Record<string, number>, key: string, amount?: number): number;
8
+ export declare function sumRecordCounts(...values: number[]): number;