oh-my-knowledge 0.47.0 → 0.49.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (246) hide show
  1. package/README.md +50 -18
  2. package/README.zh.md +55 -23
  3. package/dist/analysis/coverage-analyzer.d.ts +1 -0
  4. package/dist/analysis/coverage-analyzer.js +125 -62
  5. package/dist/analysis/failure-clusterer.js +2 -1
  6. package/dist/analysis/gap-analyzer.d.ts +2 -2
  7. package/dist/analysis/gap-analyzer.js +13 -3
  8. package/dist/analysis/hedging-classifier.d.ts +2 -2
  9. package/dist/analysis/hedging-classifier.js +3 -4
  10. package/dist/analysis/report-diagnostics.js +9 -7
  11. package/dist/analysis/sample-diagnostics.js +6 -6
  12. package/dist/artifact-graph/doctor.js +15 -7
  13. package/dist/assets/agent-skills/omk/SKILL.md +27 -7
  14. package/dist/assets/agent-skills/omk/references/commands.md +19 -17
  15. package/dist/authoring/evolver.d.ts +10 -6
  16. package/dist/authoring/evolver.js +496 -83
  17. package/dist/authoring/generator.d.ts +3 -3
  18. package/dist/authoring/generator.js +5 -10
  19. package/dist/authoring/sample-fixer.d.ts +8 -6
  20. package/dist/authoring/sample-fixer.js +76 -5
  21. package/dist/cli/commands/doctor.js +31 -14
  22. package/dist/cli/commands/eval/index.d.ts +3 -0
  23. package/dist/cli/commands/eval/index.js +163 -17
  24. package/dist/cli/commands/evolve.d.ts +4 -4
  25. package/dist/cli/commands/evolve.js +27 -13
  26. package/dist/cli/commands/init.js +16 -3
  27. package/dist/cli/commands/observe/inbox.js +44 -21
  28. package/dist/cli/commands/observe/index.js +20 -11
  29. package/dist/cli/commands/observe/ingest.d.ts +3 -0
  30. package/dist/cli/commands/observe/ingest.js +35 -4
  31. package/dist/cli/commands/sample.d.ts +9 -3
  32. package/dist/cli/commands/sample.js +91 -74
  33. package/dist/cli/lib/cmd-flags.d.ts +1 -0
  34. package/dist/cli/lib/codex-model-hint.d.ts +9 -0
  35. package/dist/cli/lib/codex-model-hint.js +45 -0
  36. package/dist/cli/lib/generation-failure-hint.d.ts +2 -0
  37. package/dist/cli/lib/generation-failure-hint.js +61 -0
  38. package/dist/cli/lib/i18n-dict/common.d.ts +1 -1
  39. package/dist/cli/lib/i18n-dict/common.js +4 -0
  40. package/dist/cli/lib/i18n-dict/gen.d.ts +1 -1
  41. package/dist/cli/lib/i18n-dict/gen.js +38 -6
  42. package/dist/cli/lib/i18n-dict/help.js +6 -6
  43. package/dist/cli/lib/i18n-dict/init.d.ts +1 -1
  44. package/dist/cli/lib/i18n-dict/init.js +13 -9
  45. package/dist/cli/lib/i18n-dict/run.d.ts +1 -1
  46. package/dist/cli/lib/i18n-dict/run.js +34 -2
  47. package/dist/cli/lib/llm-failure-classifier.d.ts +2 -0
  48. package/dist/cli/lib/llm-failure-classifier.js +8 -0
  49. package/dist/cli/lib/parse-run-config.d.ts +6 -5
  50. package/dist/cli/lib/parse-run-config.js +16 -9
  51. package/dist/cli/lib/runtime-defaults.d.ts +21 -0
  52. package/dist/cli/lib/runtime-defaults.js +79 -0
  53. package/dist/diagnosis/observe-mapper.js +14 -15
  54. package/dist/diagnosis/observe-producer.js +3 -1
  55. package/dist/diagnosis/studio-projection.js +14 -7
  56. package/dist/diagnosis/types.d.ts +2 -0
  57. package/dist/diagnosis/types.js +12 -0
  58. package/dist/doctor/endpoint-rule.js +2 -1
  59. package/dist/eval-core/artifact-file-names.js +18 -1
  60. package/dist/eval-core/artifact-index.d.ts +7 -11
  61. package/dist/eval-core/artifact-index.js +139 -80
  62. package/dist/eval-core/cache.d.ts +12 -3
  63. package/dist/eval-core/cache.js +89 -29
  64. package/dist/eval-core/comparability.js +10 -6
  65. package/dist/eval-core/evaluation-execution.d.ts +2 -1
  66. package/dist/eval-core/evaluation-execution.js +122 -37
  67. package/dist/eval-core/evaluation-job.d.ts +4 -1
  68. package/dist/eval-core/evaluation-job.js +4 -1
  69. package/dist/eval-core/evaluation-reporting.d.ts +15 -13
  70. package/dist/eval-core/evaluation-reporting.js +54 -52
  71. package/dist/eval-core/execution-strategy.d.ts +2 -0
  72. package/dist/eval-core/execution-strategy.js +11 -9
  73. package/dist/eval-core/fact-checker.js +15 -7
  74. package/dist/eval-core/holdout.js +3 -2
  75. package/dist/eval-core/judge-independence.d.ts +2 -2
  76. package/dist/eval-core/mock-hook.cjs +23 -6
  77. package/dist/eval-core/mocks-runtime.js +30 -8
  78. package/dist/eval-core/report-document.d.ts +12 -0
  79. package/dist/eval-core/report-document.js +1151 -0
  80. package/dist/eval-core/report-extensions.d.ts +4 -0
  81. package/dist/eval-core/report-extensions.js +500 -0
  82. package/dist/eval-core/report-file-migration.js +7 -2
  83. package/dist/eval-core/resume-compatibility.d.ts +31 -0
  84. package/dist/eval-core/resume-compatibility.js +141 -0
  85. package/dist/eval-core/sample-fingerprint.d.ts +12 -0
  86. package/dist/eval-core/sample-fingerprint.js +193 -0
  87. package/dist/eval-core/schema.js +86 -31
  88. package/dist/eval-core/verdict.d.ts +8 -4
  89. package/dist/eval-core/verdict.js +24 -10
  90. package/dist/eval-workflows/batch-evaluation-workflow.d.ts +2 -1
  91. package/dist/eval-workflows/batch-evaluation-workflow.js +25 -12
  92. package/dist/eval-workflows/evaluation-pipeline/preflight-warnings.d.ts +10 -5
  93. package/dist/eval-workflows/evaluation-pipeline/preflight-warnings.js +58 -21
  94. package/dist/eval-workflows/evaluation-pipeline/report-finalize.js +3 -1
  95. package/dist/eval-workflows/evaluation-pipeline/run-state.d.ts +4 -1
  96. package/dist/eval-workflows/evaluation-pipeline/run-state.js +4 -1
  97. package/dist/eval-workflows/evaluation-pipeline/test-set-hash.d.ts +6 -5
  98. package/dist/eval-workflows/evaluation-pipeline/test-set-hash.js +17 -10
  99. package/dist/eval-workflows/evaluation-pipeline.js +12 -7
  100. package/dist/eval-workflows/run-evaluation.d.ts +9 -7
  101. package/dist/eval-workflows/run-evaluation.js +79 -51
  102. package/dist/executors/anthropic-api.js +65 -9
  103. package/dist/executors/claude-cli.js +16 -79
  104. package/dist/executors/claude-protocol.d.ts +28 -0
  105. package/dist/executors/claude-protocol.js +180 -0
  106. package/dist/executors/claude-sdk-trace.js +56 -28
  107. package/dist/executors/claude-sdk.d.ts +1 -0
  108. package/dist/executors/claude-sdk.js +39 -93
  109. package/dist/executors/codex-cli-trace.js +166 -31
  110. package/dist/executors/codex-cli.d.ts +6 -8
  111. package/dist/executors/codex-cli.js +49 -151
  112. package/dist/executors/codex-protocol.d.ts +24 -0
  113. package/dist/executors/codex-protocol.js +234 -0
  114. package/dist/executors/codex-sdk.js +68 -120
  115. package/dist/executors/gemini.js +88 -13
  116. package/dist/executors/index.d.ts +2 -3
  117. package/dist/executors/index.js +5 -3
  118. package/dist/executors/openai-api.js +70 -9
  119. package/dist/executors/runtime-fingerprint.js +88 -11
  120. package/dist/executors/script-command.d.ts +8 -0
  121. package/dist/executors/script-command.js +87 -0
  122. package/dist/executors/script.js +202 -29
  123. package/dist/executors/shared.d.ts +35 -3
  124. package/dist/executors/shared.js +113 -15
  125. package/dist/grading/assertions.d.ts +1 -1
  126. package/dist/grading/assertions.js +19 -9
  127. package/dist/grading/diagnostic.d.ts +9 -2
  128. package/dist/grading/diagnostic.js +25 -2
  129. package/dist/grading/index.js +10 -4
  130. package/dist/grading/judge.js +19 -6
  131. package/dist/grading/layered-scores.d.ts +2 -3
  132. package/dist/grading/layered-scores.js +2 -3
  133. package/dist/inputs/load-samples.d.ts +1 -2
  134. package/dist/inputs/load-samples.js +23 -1
  135. package/dist/inputs/mcp-resolver.js +6 -3
  136. package/dist/inputs/sample-document.d.ts +11 -0
  137. package/dist/inputs/sample-document.js +96 -0
  138. package/dist/managed/evidence.d.ts +1 -0
  139. package/dist/managed/evidence.js +1 -1
  140. package/dist/managed/store.js +200 -91
  141. package/dist/observability/codex-trace-adapter.d.ts +5 -0
  142. package/dist/observability/codex-trace-adapter.js +850 -0
  143. package/dist/observability/experience.d.ts +32 -6
  144. package/dist/observability/experience.js +2695 -459
  145. package/dist/observability/feedback-matchers.js +16 -1
  146. package/dist/observability/inbox-view-model.d.ts +2 -1
  147. package/dist/observability/inbox-view-model.js +20 -14
  148. package/dist/observability/inbox.d.ts +7 -1
  149. package/dist/observability/inbox.js +632 -124
  150. package/dist/observability/problem-patterns.js +2 -0
  151. package/dist/observability/review-state.d.ts +6 -0
  152. package/dist/observability/review-state.js +235 -63
  153. package/dist/observability/skill-chain-advisories.js +1 -1
  154. package/dist/observability/skill-chain.js +17 -4
  155. package/dist/observability/skill-health-analyzer.d.ts +32 -7
  156. package/dist/observability/skill-health-analyzer.js +194 -121
  157. package/dist/observability/skill-health-report.d.ts +10 -0
  158. package/dist/observability/skill-health-report.js +620 -0
  159. package/dist/observability/soft-standards/constants.d.ts +0 -1
  160. package/dist/observability/soft-standards/constants.js +0 -1
  161. package/dist/observability/soft-standards/index.d.ts +1 -1
  162. package/dist/observability/soft-standards/index.js +1 -1
  163. package/dist/observability/soft-standards/llm-extractor.js +8 -10
  164. package/dist/observability/soft-standards/skill-standards-store.d.ts +2 -1
  165. package/dist/observability/soft-standards/skill-standards-store.js +59 -18
  166. package/dist/observability/soft-standards/types.d.ts +2 -2
  167. package/dist/observability/trace-adapter.d.ts +12 -7
  168. package/dist/observability/trace-adapter.js +11 -9
  169. package/dist/observability/trace-attribution.d.ts +13 -5
  170. package/dist/observability/trace-attribution.js +315 -21
  171. package/dist/observability/trace-ingestion.d.ts +9 -0
  172. package/dist/observability/trace-ingestion.js +80 -0
  173. package/dist/observability/trace-ir.d.ts +113 -0
  174. package/dist/observability/trace-ir.js +87 -0
  175. package/dist/observability/trace-segmenter.d.ts +19 -6
  176. package/dist/observability/trace-segmenter.js +377 -196
  177. package/dist/observability/trace-session-index.d.ts +19 -0
  178. package/dist/observability/trace-session-index.js +68 -0
  179. package/dist/observability/trace-source.d.ts +12 -4
  180. package/dist/observability/trace-source.js +939 -215
  181. package/dist/renderer/html-renderer.js +37 -6
  182. package/dist/renderer/icons.js +3 -0
  183. package/dist/renderer/observation-inbox-renderer.js +227 -91
  184. package/dist/renderer/skill-detail-renderer.js +452 -109
  185. package/dist/renderer/skill-health-renderer.js +69 -12
  186. package/dist/renderer/summary.js +28 -7
  187. package/dist/renderer/table.js +21 -4
  188. package/dist/renderer/test-view.d.ts +1 -0
  189. package/dist/renderer/test-view.js +44 -9
  190. package/dist/server/indexed-report-store.js +14 -18
  191. package/dist/server/job-store.js +64 -26
  192. package/dist/server/report-server.js +190 -78
  193. package/dist/server/report-store.js +57 -80
  194. package/dist/server/skill-index.js +143 -49
  195. package/dist/server/skill-insights.js +44 -5
  196. package/dist/shared/artifact-graph.d.ts +3 -0
  197. package/dist/shared/artifact-graph.js +224 -0
  198. package/dist/shared/assertion-types.d.ts +8 -0
  199. package/dist/shared/assertion-types.js +46 -0
  200. package/dist/shared/atomic-json.d.ts +8 -0
  201. package/dist/shared/atomic-json.js +33 -0
  202. package/dist/shared/diagnosis-schema.d.ts +9 -0
  203. package/dist/shared/diagnosis-schema.js +181 -0
  204. package/dist/shared/doctor-report.d.ts +3 -0
  205. package/dist/shared/doctor-report.js +103 -0
  206. package/dist/shared/evaluation-job.d.ts +6 -0
  207. package/dist/shared/evaluation-job.js +217 -0
  208. package/dist/shared/executor-result.d.ts +17 -0
  209. package/dist/shared/executor-result.js +221 -0
  210. package/dist/shared/file-lock.d.ts +12 -0
  211. package/dist/shared/file-lock.js +129 -0
  212. package/dist/shared/json-value.d.ts +5 -0
  213. package/dist/shared/json-value.js +36 -0
  214. package/dist/shared/keyed-mutex.d.ts +7 -0
  215. package/dist/shared/keyed-mutex.js +24 -0
  216. package/dist/shared/record-count.d.ts +8 -0
  217. package/dist/shared/record-count.js +43 -0
  218. package/dist/shared/sample-contract.d.ts +3 -0
  219. package/dist/shared/sample-contract.js +332 -0
  220. package/dist/shared/shell-quote.d.ts +2 -0
  221. package/dist/shared/shell-quote.js +7 -0
  222. package/dist/shared/timestamp.d.ts +6 -0
  223. package/dist/shared/timestamp.js +64 -0
  224. package/dist/shared/token-usage.d.ts +19 -0
  225. package/dist/shared/token-usage.js +50 -0
  226. package/dist/shared/tool-call-status.d.ts +8 -0
  227. package/dist/shared/tool-call-status.js +28 -0
  228. package/dist/shared/tool-identity.d.ts +21 -0
  229. package/dist/shared/tool-identity.js +84 -0
  230. package/dist/shared/tool-search.js +73 -16
  231. package/dist/shared/trace-projection.d.ts +5 -0
  232. package/dist/shared/trace-projection.js +20 -0
  233. package/dist/shared/trace-source-kind.d.ts +3 -0
  234. package/dist/shared/trace-source-kind.js +12 -0
  235. package/dist/types/diagnosis.d.ts +2 -0
  236. package/dist/types/eval.d.ts +4 -0
  237. package/dist/types/executor.d.ts +32 -5
  238. package/dist/types/index.d.ts +1 -0
  239. package/dist/types/index.js +1 -0
  240. package/dist/types/judge.d.ts +2 -0
  241. package/dist/types/observability.d.ts +116 -9
  242. package/dist/types/report.d.ts +58 -6
  243. package/dist/types/skill-index.d.ts +7 -0
  244. package/dist/types/trace.d.ts +2 -0
  245. package/dist/types/trace.js +1 -0
  246. package/package.json +9 -5
@@ -0,0 +1,43 @@
1
+ export function ownRecordValue(record, key) {
2
+ return Object.hasOwn(record, key) ? record[key] : undefined;
3
+ }
4
+ export function setOwnRecordValue(record, key, value) {
5
+ Object.defineProperty(record, key, {
6
+ value,
7
+ enumerable: true,
8
+ configurable: true,
9
+ writable: true,
10
+ });
11
+ return value;
12
+ }
13
+ /**
14
+ * Increment an arbitrary string-keyed counter without reading inherited
15
+ * properties such as `constructor` or invoking the legacy `__proto__` setter.
16
+ */
17
+ export function incrementRecordCount(record, key, amount = 1) {
18
+ if (!Number.isSafeInteger(amount) || amount < 0) {
19
+ throw new TypeError(`Record count increment must be a non-negative integer, got ${String(amount)}`);
20
+ }
21
+ const ownValue = ownRecordValue(record, key);
22
+ const current = typeof ownValue === 'number' && Number.isSafeInteger(ownValue) && ownValue >= 0
23
+ ? ownValue
24
+ : 0;
25
+ const next = current + amount;
26
+ if (!Number.isSafeInteger(next)) {
27
+ throw new RangeError(`Record count for "${key}" exceeds Number.MAX_SAFE_INTEGER`);
28
+ }
29
+ return setOwnRecordValue(record, key, next);
30
+ }
31
+ export function sumRecordCounts(...values) {
32
+ let total = 0;
33
+ for (const value of values) {
34
+ if (!Number.isSafeInteger(value) || value < 0) {
35
+ throw new TypeError(`Record count must be a non-negative safe integer, got ${String(value)}`);
36
+ }
37
+ total += value;
38
+ if (!Number.isSafeInteger(total)) {
39
+ throw new RangeError('Record count sum exceeds Number.MAX_SAFE_INTEGER');
40
+ }
41
+ }
42
+ return total;
43
+ }
@@ -0,0 +1,3 @@
1
+ export declare function assertionContractValidationError(value: unknown, depth?: number, insideAssertSet?: boolean): string | undefined;
2
+ export declare function dependencyRequirementsValidationError(value: unknown): string | undefined;
3
+ export declare function sampleContractValidationError(value: unknown, expectedId?: string): string | undefined;
@@ -0,0 +1,332 @@
1
+ import { isJsonValue } from './json-value.js';
2
+ import { ASYNC_ASSERTION_TYPES, SUPPORTED_ASSERTION_TYPES, } from './assertion-types.js';
3
+ function isRecord(value) {
4
+ return Boolean(value) && typeof value === 'object' && !Array.isArray(value);
5
+ }
6
+ function isFiniteNumber(value) {
7
+ return typeof value === 'number'
8
+ && Number.isFinite(value)
9
+ && Math.abs(value) <= Number.MAX_SAFE_INTEGER;
10
+ }
11
+ function isNonEmptyString(value) {
12
+ return typeof value === 'string' && value.trim().length > 0;
13
+ }
14
+ function isStringArray(value) {
15
+ return Array.isArray(value) && value.every(isNonEmptyString);
16
+ }
17
+ function hasOnlyKeys(value, allowed) {
18
+ const allowedKeys = new Set(allowed);
19
+ return Object.keys(value).every((key) => allowedKeys.has(key));
20
+ }
21
+ const STRING_VALUE_ASSERTIONS = new Set([
22
+ 'contains',
23
+ 'not_contains',
24
+ 'starts_with',
25
+ 'ends_with',
26
+ 'equals',
27
+ 'not_equals',
28
+ 'tool_output_contains',
29
+ 'tool_input_contains',
30
+ 'tool_input_not_contains',
31
+ 'mock_hit',
32
+ ]);
33
+ const NUMBER_VALUE_ASSERTIONS = new Set([
34
+ 'min_length',
35
+ 'max_length',
36
+ 'word_count_min',
37
+ 'word_count_max',
38
+ 'cost_max',
39
+ 'latency_max',
40
+ 'turns_max',
41
+ 'turns_min',
42
+ 'tools_count_max',
43
+ 'tools_count_min',
44
+ 'levenshtein_max',
45
+ ]);
46
+ const NON_NEGATIVE_VALUE_ASSERTIONS = new Set([
47
+ ...NUMBER_VALUE_ASSERTIONS,
48
+ ]);
49
+ const STRING_VALUES_ASSERTIONS = new Set([
50
+ 'contains_all',
51
+ 'contains_any',
52
+ 'tools_called',
53
+ 'tools_not_called',
54
+ ]);
55
+ const REFERENCE_ASSERTIONS = new Set([
56
+ 'semantic_similarity',
57
+ 'rouge_n_min',
58
+ 'levenshtein_max',
59
+ 'bleu_min',
60
+ ]);
61
+ export function assertionContractValidationError(value, depth = 0, insideAssertSet = false) {
62
+ if (depth >= 16)
63
+ return 'assertion nesting exceeds 16 levels';
64
+ if (!isRecord(value))
65
+ return 'assertion must be an object';
66
+ if (!isNonEmptyString(value.type))
67
+ return 'assertion "type" must be a non-empty string';
68
+ if (!SUPPORTED_ASSERTION_TYPES.has(value.type)) {
69
+ return `unsupported assertion type: ${JSON.stringify(value.type)}`;
70
+ }
71
+ if (insideAssertSet && ASYNC_ASSERTION_TYPES.has(value.type)) {
72
+ return `async assertion type ${JSON.stringify(value.type)} cannot be nested in "assert-set"`;
73
+ }
74
+ if (value.value !== undefined
75
+ && typeof value.value !== 'string'
76
+ && !isFiniteNumber(value.value))
77
+ return '"value" must be a finite number or string when present';
78
+ if (value.values !== undefined && !isStringArray(value.values)) {
79
+ return '"values" must be an array of non-empty strings when present';
80
+ }
81
+ if (value.pattern !== undefined && typeof value.pattern !== 'string') {
82
+ return '"pattern" must be a string when present';
83
+ }
84
+ if (value.flags !== undefined && typeof value.flags !== 'string') {
85
+ return '"flags" must be a string when present';
86
+ }
87
+ if (value.schema !== undefined
88
+ && (!isRecord(value.schema) || !isJsonValue(value.schema)))
89
+ return '"schema" must be a JSON object when present';
90
+ if (value.weight !== undefined && (!isFiniteNumber(value.weight) || value.weight <= 0)) {
91
+ return '"weight" must be a positive finite number when present';
92
+ }
93
+ if (value.fn !== undefined && typeof value.fn !== 'string') {
94
+ return '"fn" must be a string when present';
95
+ }
96
+ if (value.reference !== undefined && typeof value.reference !== 'string') {
97
+ return '"reference" must be a string when present';
98
+ }
99
+ if (value.threshold !== undefined && !isFiniteNumber(value.threshold)) {
100
+ return '"threshold" must be a finite number when present';
101
+ }
102
+ if (value.not !== undefined && typeof value.not !== 'boolean') {
103
+ return '"not" must be boolean when present';
104
+ }
105
+ if (value.mode !== undefined
106
+ && value.mode !== 'any'
107
+ && value.mode !== 'all')
108
+ return '"mode" must be "any" or "all" when present';
109
+ if (value.n !== undefined
110
+ && (!Number.isSafeInteger(value.n) || value.n <= 0))
111
+ return '"n" must be a positive integer when present';
112
+ if (STRING_VALUE_ASSERTIONS.has(value.type) && !isNonEmptyString(value.value)) {
113
+ return `${JSON.stringify(value.type)} requires a non-empty string "value"`;
114
+ }
115
+ if (NUMBER_VALUE_ASSERTIONS.has(value.type) && !isFiniteNumber(value.value)) {
116
+ return `${JSON.stringify(value.type)} requires a finite numeric "value"`;
117
+ }
118
+ if (NON_NEGATIVE_VALUE_ASSERTIONS.has(value.type)
119
+ && isFiniteNumber(value.value)
120
+ && value.value < 0)
121
+ return `${JSON.stringify(value.type)} requires a non-negative "value"`;
122
+ if (STRING_VALUES_ASSERTIONS.has(value.type)
123
+ && (!isStringArray(value.values) || value.values.length === 0))
124
+ return `${JSON.stringify(value.type)} requires non-empty string "values"`;
125
+ if (REFERENCE_ASSERTIONS.has(value.type) && !isNonEmptyString(value.reference)) {
126
+ return `${JSON.stringify(value.type)} requires a non-empty "reference"`;
127
+ }
128
+ if (value.type === 'regex') {
129
+ if (typeof value.pattern !== 'string')
130
+ return '"regex" requires a string "pattern"';
131
+ try {
132
+ new RegExp(value.pattern, typeof value.flags === 'string' ? value.flags : 'i');
133
+ }
134
+ catch {
135
+ return '"regex" contains an invalid pattern or flags';
136
+ }
137
+ }
138
+ if (value.type === 'json_schema' && (!isRecord(value.schema) || !isJsonValue(value.schema))) {
139
+ return '"json_schema" requires a JSON object "schema"';
140
+ }
141
+ if (value.type === 'custom' && !isNonEmptyString(value.fn)) {
142
+ return '"custom" requires a non-empty "fn"';
143
+ }
144
+ if ((value.type === 'rouge_n_min' || value.type === 'bleu_min')
145
+ && value.threshold !== undefined
146
+ && (value.threshold < 0 || value.threshold > 1))
147
+ return `${JSON.stringify(value.type)} "threshold" must be within [0, 1]`;
148
+ if ((value.type === 'semantic_similarity'
149
+ || value.type === 'faithfulness'
150
+ || value.type === 'answer_relevancy'
151
+ || value.type === 'context_recall')
152
+ && value.threshold !== undefined
153
+ && (value.threshold < 1 || value.threshold > 5))
154
+ return `${JSON.stringify(value.type)} "threshold" must be within [1, 5]`;
155
+ if (value.type === 'mock_hit'
156
+ && typeof value.value === 'string'
157
+ && !/^[^:]+:[1-9]\d*$/.test(value.value))
158
+ return '"mock_hit" value must use "Tool:N" with a positive 1-based ordinal';
159
+ if (value.type === 'assert-set'
160
+ && (!Array.isArray(value.children) || value.children.length === 0))
161
+ return '"assert-set" requires non-empty "children"';
162
+ if (value.children !== undefined) {
163
+ if (!Array.isArray(value.children))
164
+ return '"children" must be an array when present';
165
+ for (const [index, child] of value.children.entries()) {
166
+ const error = assertionContractValidationError(child, depth + 1, value.type === 'assert-set');
167
+ if (error)
168
+ return `children[${index}]: ${error}`;
169
+ }
170
+ }
171
+ return undefined;
172
+ }
173
+ function isMockReturn(value) {
174
+ return typeof value === 'string'
175
+ || (isRecord(value) && isJsonValue(value));
176
+ }
177
+ function mockValidationError(value) {
178
+ if (!isRecord(value))
179
+ return 'mock must be an object';
180
+ if (!isNonEmptyString(value.tool))
181
+ return '"tool" must be a non-empty string';
182
+ if (value.return !== undefined && !isMockReturn(value.return)) {
183
+ return '"return" must be a string or JSON object';
184
+ }
185
+ if (value.return_file !== undefined && !isNonEmptyString(value.return_file)) {
186
+ return '"return_file" must be a non-empty string';
187
+ }
188
+ if (value.return_seq !== undefined
189
+ && (!Array.isArray(value.return_seq)
190
+ || value.return_seq.length === 0
191
+ || !value.return_seq.every(isMockReturn)))
192
+ return '"return_seq" must be a non-empty array of mock returns';
193
+ if (value.return === undefined
194
+ && value.return_file === undefined
195
+ && value.return_seq === undefined)
196
+ return 'mock requires "return", "return_file", or "return_seq"';
197
+ if (value.match === undefined)
198
+ return undefined;
199
+ if (!isRecord(value.match))
200
+ return '"match" must be an object when present';
201
+ const matchFields = [
202
+ 'file_path',
203
+ 'file_path_endswith',
204
+ 'url',
205
+ 'url_glob',
206
+ 'command_glob',
207
+ 'input_contains',
208
+ ];
209
+ if (!hasOnlyKeys(value.match, [...matchFields, 'input'])) {
210
+ return '"match" contains an unsupported field';
211
+ }
212
+ for (const field of matchFields) {
213
+ if (value.match[field] !== undefined && !isNonEmptyString(value.match[field])) {
214
+ return `"match.${field}" must be a non-empty string when present`;
215
+ }
216
+ }
217
+ if (value.match.input !== undefined
218
+ && (!isRecord(value.match.input) || !isJsonValue(value.match.input)))
219
+ return '"match.input" must be a JSON object when present';
220
+ return undefined;
221
+ }
222
+ export function dependencyRequirementsValidationError(value) {
223
+ if (!isRecord(value))
224
+ return '"requires" must be an object';
225
+ if (!hasOnlyKeys(value, ['tools', 'files', 'env', 'preflight'])) {
226
+ return '"requires" contains an unsupported field';
227
+ }
228
+ for (const field of ['tools', 'files', 'env', 'preflight']) {
229
+ if (value[field] !== undefined && !isStringArray(value[field])) {
230
+ return `"requires.${field}" must be an array of non-empty strings`;
231
+ }
232
+ }
233
+ return undefined;
234
+ }
235
+ export function sampleContractValidationError(value, expectedId) {
236
+ if (!isRecord(value))
237
+ return 'sample must be an object';
238
+ if (!isNonEmptyString(value.sample_id)
239
+ || (expectedId !== undefined && value.sample_id !== expectedId))
240
+ return '"sample_id" must be a non-empty matching string';
241
+ if (!isNonEmptyString(value.prompt))
242
+ return '"prompt" must be a non-empty string';
243
+ for (const field of ['cwd', 'rubric', 'context']) {
244
+ if (value[field] !== undefined && typeof value[field] !== 'string') {
245
+ return `"${field}" must be a string when present`;
246
+ }
247
+ }
248
+ if (value.dimensions !== undefined
249
+ && (!isRecord(value.dimensions)
250
+ || !Object.keys(value.dimensions).every(isNonEmptyString)
251
+ || !Object.values(value.dimensions).every(isNonEmptyString)))
252
+ return '"dimensions" must map non-empty names to non-empty string rubrics';
253
+ if (value.assertions !== undefined) {
254
+ if (!Array.isArray(value.assertions))
255
+ return '"assertions" must be an array';
256
+ for (const [index, assertion] of value.assertions.entries()) {
257
+ const error = assertionContractValidationError(assertion);
258
+ if (error)
259
+ return `"assertions[${index}]": ${error}`;
260
+ }
261
+ for (const assertion of value.assertions) {
262
+ if (isRecord(assertion)
263
+ && (assertion.type === 'faithfulness' || assertion.type === 'context_recall')
264
+ && !isNonEmptyString(assertion.reference)
265
+ && !isNonEmptyString(value.context)) {
266
+ return `${JSON.stringify(assertion.type)} requires sample "context" or assertion "reference"`;
267
+ }
268
+ }
269
+ }
270
+ if (value.mocks !== undefined) {
271
+ if (!Array.isArray(value.mocks))
272
+ return '"mocks" must be an array';
273
+ for (const [index, mock] of value.mocks.entries()) {
274
+ const error = mockValidationError(mock);
275
+ if (error)
276
+ return `"mocks[${index}]": ${error}`;
277
+ }
278
+ }
279
+ if (value.mocksStrict !== undefined && typeof value.mocksStrict !== 'boolean') {
280
+ return '"mocksStrict" must be boolean when present';
281
+ }
282
+ for (const field of ['allowedTools', 'expectedTools', 'capability']) {
283
+ if (value[field] !== undefined && !isStringArray(value[field])) {
284
+ return `"${field}" must be an array of non-empty strings`;
285
+ }
286
+ }
287
+ if (value.environment !== undefined
288
+ && (!isRecord(value.environment)
289
+ || !hasOnlyKeys(value.environment, ['cli_available', 'files_available', 'notes'])
290
+ || (value.environment.cli_available !== undefined
291
+ && !isStringArray(value.environment.cli_available))
292
+ || (value.environment.files_available !== undefined
293
+ && !isStringArray(value.environment.files_available))
294
+ || (value.environment.notes !== undefined
295
+ && typeof value.environment.notes !== 'string')))
296
+ return '"environment" has an invalid shape';
297
+ if (value.difficulty !== undefined
298
+ && value.difficulty !== 'easy'
299
+ && value.difficulty !== 'medium'
300
+ && value.difficulty !== 'hard')
301
+ return '"difficulty" has an unsupported value';
302
+ if (value.construct !== undefined && typeof value.construct !== 'string') {
303
+ return '"construct" must be string when present';
304
+ }
305
+ if (value.provenance !== undefined
306
+ && value.provenance !== 'human'
307
+ && value.provenance !== 'llm-generated'
308
+ && value.provenance !== 'production-trace')
309
+ return '"provenance" has an unsupported value';
310
+ if (value.tripwire !== undefined && typeof value.tripwire !== 'boolean') {
311
+ return '"tripwire" must be boolean when present';
312
+ }
313
+ if (value.covers === undefined)
314
+ return undefined;
315
+ const targetKinds = new Set([
316
+ 'skill',
317
+ 'skill_file',
318
+ 'frontmatter',
319
+ 'reference',
320
+ 'script',
321
+ 'hard_rule',
322
+ 'workflow',
323
+ 'workflow_node',
324
+ ]);
325
+ return Array.isArray(value.covers)
326
+ && value.covers.every((target) => isRecord(target)
327
+ && typeof target.targetKind === 'string'
328
+ && targetKinds.has(target.targetKind)
329
+ && isNonEmptyString(target.ref))
330
+ ? undefined
331
+ : '"covers" contains an invalid coverage target';
332
+ }
@@ -0,0 +1,2 @@
1
+ /** Format one POSIX shell argument for user-facing copy/paste commands. */
2
+ export declare function shellQuoteArg(value: string): string;
@@ -0,0 +1,7 @@
1
+ const SAFE_SHELL_ARG_RE = /^[A-Za-z0-9_./:@%+=,-]+$/;
2
+ /** Format one POSIX shell argument for user-facing copy/paste commands. */
3
+ export function shellQuoteArg(value) {
4
+ if (value && SAFE_SHELL_ARG_RE.test(value))
5
+ return value;
6
+ return `'${value.replace(/'/g, "'\\''")}'`;
7
+ }
@@ -0,0 +1,6 @@
1
+ /**
2
+ * Normalize an absolute RFC 3339 timestamp without accepting JavaScript's
3
+ * environment-dependent date shortcuts or calendar rollover.
4
+ */
5
+ export declare function normalizeRfc3339Timestamp(value: unknown): string | undefined;
6
+ export declare function isRfc3339Timestamp(value: unknown): value is string;
@@ -0,0 +1,64 @@
1
+ const RFC3339_TIMESTAMP_RE = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.(\d{1,9}))?(Z|[+-]\d{2}:\d{2})$/i;
2
+ /**
3
+ * Normalize an absolute RFC 3339 timestamp without accepting JavaScript's
4
+ * environment-dependent date shortcuts or calendar rollover.
5
+ */
6
+ export function normalizeRfc3339Timestamp(value) {
7
+ if (typeof value !== 'string')
8
+ return undefined;
9
+ const candidate = value.trim();
10
+ const match = RFC3339_TIMESTAMP_RE.exec(candidate);
11
+ if (!match)
12
+ return undefined;
13
+ const year = Number(match[1]);
14
+ const month = Number(match[2]);
15
+ const day = Number(match[3]);
16
+ const hour = Number(match[4]);
17
+ const minute = Number(match[5]);
18
+ const second = Number(match[6]);
19
+ if (month < 1
20
+ || month > 12
21
+ || day < 1
22
+ || day > daysInMonth(year, month)
23
+ || hour > 23
24
+ || minute > 59
25
+ || second > 60)
26
+ return undefined;
27
+ const offset = match[8].toUpperCase();
28
+ // RFC 3339 reserves -00:00 for an unknown local offset. It is syntactically
29
+ // valid but does not identify an absolute instant, so measurement code must
30
+ // not silently reinterpret it as UTC.
31
+ if (offset === '-00:00')
32
+ return undefined;
33
+ if (offset !== 'Z') {
34
+ const offsetHour = Number(offset.slice(1, 3));
35
+ const offsetMinute = Number(offset.slice(4, 6));
36
+ if (offsetHour > 23 || offsetMinute > 59)
37
+ return undefined;
38
+ }
39
+ const parseableCandidate = second === 60
40
+ ? `${match[1]}-${match[2]}-${match[3]}T${match[4]}:${match[5]}:59${match[7] ? `.${match[7]}` : ''}${offset}`
41
+ : candidate;
42
+ const parsedMs = Date.parse(parseableCandidate);
43
+ if (second === 60) {
44
+ const beforeLeap = new Date(parsedMs);
45
+ if (beforeLeap.getUTCHours() !== 23
46
+ || beforeLeap.getUTCMinutes() !== 59
47
+ || beforeLeap.getUTCSeconds() !== 59
48
+ || beforeLeap.getUTCDate()
49
+ !== daysInMonth(beforeLeap.getUTCFullYear(), beforeLeap.getUTCMonth() + 1))
50
+ return undefined;
51
+ }
52
+ const epochMs = second === 60 ? parsedMs + 1000 : parsedMs;
53
+ return Number.isFinite(epochMs) ? new Date(epochMs).toISOString() : undefined;
54
+ }
55
+ export function isRfc3339Timestamp(value) {
56
+ return normalizeRfc3339Timestamp(value) !== undefined;
57
+ }
58
+ function daysInMonth(year, month) {
59
+ if (month === 2) {
60
+ const leap = year % 4 === 0 && (year % 100 !== 0 || year % 400 === 0);
61
+ return leap ? 29 : 28;
62
+ }
63
+ return month === 4 || month === 6 || month === 9 || month === 11 ? 30 : 31;
64
+ }
@@ -0,0 +1,19 @@
1
+ export interface ExclusiveInputTokenUsage {
2
+ inputTokens: number;
3
+ cacheReadTokens: number;
4
+ cacheCreationTokens: number;
5
+ }
6
+ /** Provider counters are untrusted protocol input. Token metrics are integer counts. */
7
+ export declare function tokenCount(value: unknown): number;
8
+ export declare function optionalTokenCount(value: unknown): number | undefined;
9
+ /** Sum untrusted provider counters without exposing an unsafe integer. */
10
+ export declare function sumTokenCounts(...values: unknown[]): number;
11
+ /** Validate and sum provider counters while preserving invalid/overflow as unknown. */
12
+ export declare function checkedSumTokenCounts(...values: unknown[]): number | undefined;
13
+ export declare function nonNegativeMetric(value: unknown): number | undefined;
14
+ /**
15
+ * Split providers whose input total includes cache reads/writes into mutually
16
+ * exclusive buckets. Invalid cache counters are bounded by the authoritative
17
+ * input total so one malformed response cannot inflate cross-runtime totals.
18
+ */
19
+ export declare function splitInclusiveInputTokens(rawInput: unknown, rawCacheRead: unknown, rawCacheCreation?: unknown): ExclusiveInputTokenUsage;
@@ -0,0 +1,50 @@
1
+ /** Provider counters are untrusted protocol input. Token metrics are integer counts. */
2
+ export function tokenCount(value) {
3
+ return typeof value === 'number'
4
+ && Number.isSafeInteger(value)
5
+ && value >= 0
6
+ ? value
7
+ : 0;
8
+ }
9
+ export function optionalTokenCount(value) {
10
+ return typeof value === 'number'
11
+ && Number.isSafeInteger(value)
12
+ && value >= 0
13
+ ? value
14
+ : undefined;
15
+ }
16
+ /** Sum untrusted provider counters without exposing an unsafe integer. */
17
+ export function sumTokenCounts(...values) {
18
+ return checkedSumTokenCounts(...values) ?? 0;
19
+ }
20
+ /** Validate and sum provider counters while preserving invalid/overflow as unknown. */
21
+ export function checkedSumTokenCounts(...values) {
22
+ let total = 0;
23
+ for (const value of values) {
24
+ const count = optionalTokenCount(value);
25
+ if (count === undefined || total > Number.MAX_SAFE_INTEGER - count)
26
+ return undefined;
27
+ total += count;
28
+ }
29
+ return total;
30
+ }
31
+ export function nonNegativeMetric(value) {
32
+ return typeof value === 'number' && Number.isFinite(value) && value >= 0
33
+ ? value
34
+ : undefined;
35
+ }
36
+ /**
37
+ * Split providers whose input total includes cache reads/writes into mutually
38
+ * exclusive buckets. Invalid cache counters are bounded by the authoritative
39
+ * input total so one malformed response cannot inflate cross-runtime totals.
40
+ */
41
+ export function splitInclusiveInputTokens(rawInput, rawCacheRead, rawCacheCreation = 0) {
42
+ const totalInput = tokenCount(rawInput);
43
+ const cacheReadTokens = Math.min(tokenCount(rawCacheRead), totalInput);
44
+ const cacheCreationTokens = Math.min(tokenCount(rawCacheCreation), totalInput - cacheReadTokens);
45
+ return {
46
+ inputTokens: totalInput - cacheReadTokens - cacheCreationTokens,
47
+ cacheReadTokens,
48
+ cacheCreationTokens,
49
+ };
50
+ }
@@ -0,0 +1,8 @@
1
+ import type { ToolCallInfo, ToolCallStatus } from '../types/index.js';
2
+ type ToolCallOutcomeInput = Partial<Pick<ToolCallInfo, 'status' | 'success'>>;
3
+ export declare function toolCallStatus(call: ToolCallOutcomeInput): ToolCallStatus;
4
+ export declare function isToolCallSuccess(call: ToolCallOutcomeInput): boolean;
5
+ export declare function isToolCallFailure(call: ToolCallOutcomeInput): boolean;
6
+ export declare function isToolCallCancelled(call: ToolCallOutcomeInput): boolean;
7
+ export declare function isToolCallUnknown(call: ToolCallOutcomeInput): boolean;
8
+ export {};
@@ -0,0 +1,28 @@
1
+ const TOOL_CALL_STATUSES = new Set([
2
+ 'success',
3
+ 'failure',
4
+ 'cancelled',
5
+ 'unknown',
6
+ ]);
7
+ export function toolCallStatus(call) {
8
+ if (call.status !== undefined) {
9
+ return TOOL_CALL_STATUSES.has(call.status) ? call.status : 'unknown';
10
+ }
11
+ if (call.success === true)
12
+ return 'success';
13
+ if (call.success === false)
14
+ return 'failure';
15
+ return 'unknown';
16
+ }
17
+ export function isToolCallSuccess(call) {
18
+ return toolCallStatus(call) === 'success';
19
+ }
20
+ export function isToolCallFailure(call) {
21
+ return toolCallStatus(call) === 'failure';
22
+ }
23
+ export function isToolCallCancelled(call) {
24
+ return toolCallStatus(call) === 'cancelled';
25
+ }
26
+ export function isToolCallUnknown(call) {
27
+ return toolCallStatus(call) === 'unknown';
28
+ }
@@ -0,0 +1,21 @@
1
+ export interface NormalizedToolIdentity {
2
+ /** Source-neutral name consumed by assertions and aggregate reports. */
3
+ name: string;
4
+ /** Runtime-native name, retained when normalization changes it. */
5
+ sourceName?: string;
6
+ namespace?: string;
7
+ provider?: string;
8
+ displayName?: string;
9
+ }
10
+ export interface ToolIdentityInput {
11
+ sourceName: string;
12
+ namespace?: string;
13
+ provider?: string;
14
+ /** Runtime-authoritative leaf name, such as an MCP end event's tool. */
15
+ authoritativeName?: string;
16
+ }
17
+ /**
18
+ * Normalize runtime-specific tool labels into one comparison namespace while
19
+ * retaining source identity for audit and future protocol migrations.
20
+ */
21
+ export declare function normalizeToolIdentity(input: ToolIdentityInput): NormalizedToolIdentity;