oh-my-knowledge 0.22.0 → 0.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (184) hide show
  1. package/README.md +63 -63
  2. package/README.zh.md +64 -62
  3. package/dist/src/analysis/report-diagnostics.d.ts +28 -3
  4. package/dist/src/analysis/report-diagnostics.d.ts.map +1 -1
  5. package/dist/src/analysis/report-diagnostics.js +201 -81
  6. package/dist/src/analysis/report-diagnostics.js.map +1 -1
  7. package/dist/src/analysis/sample-diagnostics.d.ts +9 -2
  8. package/dist/src/analysis/sample-diagnostics.d.ts.map +1 -1
  9. package/dist/src/analysis/sample-diagnostics.js +224 -25
  10. package/dist/src/analysis/sample-diagnostics.js.map +1 -1
  11. package/dist/src/authoring/evolver.d.ts +7 -2
  12. package/dist/src/authoring/evolver.d.ts.map +1 -1
  13. package/dist/src/authoring/evolver.js +43 -9
  14. package/dist/src/authoring/evolver.js.map +1 -1
  15. package/dist/src/authoring/generator.d.ts +24 -0
  16. package/dist/src/authoring/generator.d.ts.map +1 -1
  17. package/dist/src/authoring/generator.js +67 -5
  18. package/dist/src/authoring/generator.js.map +1 -1
  19. package/dist/src/cli/coverage-renderer.d.ts +15 -0
  20. package/dist/src/cli/coverage-renderer.d.ts.map +1 -0
  21. package/dist/src/cli/coverage-renderer.js +74 -0
  22. package/dist/src/cli/coverage-renderer.js.map +1 -0
  23. package/dist/src/cli/i18n-dict.d.ts +1 -1
  24. package/dist/src/cli/i18n-dict.d.ts.map +1 -1
  25. package/dist/src/cli/i18n-dict.js +62 -40
  26. package/dist/src/cli/i18n-dict.js.map +1 -1
  27. package/dist/src/cli/index.d.ts +3 -0
  28. package/dist/src/cli/index.d.ts.map +1 -0
  29. package/dist/src/{cli.js → cli/index.js} +123 -384
  30. package/dist/src/cli/index.js.map +1 -0
  31. package/dist/src/cli/parse-run-config.d.ts +56 -0
  32. package/dist/src/cli/parse-run-config.d.ts.map +1 -0
  33. package/dist/src/cli/parse-run-config.js +195 -0
  34. package/dist/src/cli/parse-run-config.js.map +1 -0
  35. package/dist/src/cli/progress.d.ts +25 -0
  36. package/dist/src/cli/progress.d.ts.map +1 -0
  37. package/dist/src/cli/progress.js +62 -0
  38. package/dist/src/cli/progress.js.map +1 -0
  39. package/dist/src/cli/update-check.d.ts +3 -0
  40. package/dist/src/cli/update-check.d.ts.map +1 -0
  41. package/dist/src/cli/update-check.js +37 -0
  42. package/dist/src/cli/update-check.js.map +1 -0
  43. package/dist/src/eval-core/cache.d.ts +7 -5
  44. package/dist/src/eval-core/cache.d.ts.map +1 -1
  45. package/dist/src/eval-core/cache.js +11 -7
  46. package/dist/src/eval-core/cache.js.map +1 -1
  47. package/dist/src/eval-core/comparability.d.ts +11 -0
  48. package/dist/src/eval-core/comparability.d.ts.map +1 -0
  49. package/dist/src/eval-core/comparability.js +271 -0
  50. package/dist/src/eval-core/comparability.js.map +1 -0
  51. package/dist/src/eval-core/dependency-checker.d.ts +1 -1
  52. package/dist/src/eval-core/dependency-checker.js +1 -1
  53. package/dist/src/eval-core/evaluation-execution.d.ts +5 -2
  54. package/dist/src/eval-core/evaluation-execution.d.ts.map +1 -1
  55. package/dist/src/eval-core/evaluation-execution.js +29 -21
  56. package/dist/src/eval-core/evaluation-execution.js.map +1 -1
  57. package/dist/src/eval-core/evaluation-job.d.ts +2 -2
  58. package/dist/src/eval-core/evaluation-job.d.ts.map +1 -1
  59. package/dist/src/eval-core/evaluation-job.js +2 -2
  60. package/dist/src/eval-core/evaluation-job.js.map +1 -1
  61. package/dist/src/eval-core/evaluation-reporting.d.ts +3 -1
  62. package/dist/src/eval-core/evaluation-reporting.d.ts.map +1 -1
  63. package/dist/src/eval-core/evaluation-reporting.js +65 -5
  64. package/dist/src/eval-core/evaluation-reporting.js.map +1 -1
  65. package/dist/src/eval-core/execution-strategy.js +6 -6
  66. package/dist/src/eval-core/execution-strategy.js.map +1 -1
  67. package/dist/src/eval-core/schema.d.ts.map +1 -1
  68. package/dist/src/eval-core/schema.js +14 -1
  69. package/dist/src/eval-core/schema.js.map +1 -1
  70. package/dist/src/eval-core/verdict.d.ts +3 -3
  71. package/dist/src/eval-core/verdict.js +3 -3
  72. package/dist/src/eval-workflows/batch-evaluation-workflow.d.ts +111 -0
  73. package/dist/src/eval-workflows/batch-evaluation-workflow.d.ts.map +1 -0
  74. package/dist/src/eval-workflows/batch-evaluation-workflow.js +215 -0
  75. package/dist/src/eval-workflows/batch-evaluation-workflow.js.map +1 -0
  76. package/dist/src/eval-workflows/evaluation-pipeline.d.ts +7 -5
  77. package/dist/src/eval-workflows/evaluation-pipeline.d.ts.map +1 -1
  78. package/dist/src/eval-workflows/evaluation-pipeline.js +13 -8
  79. package/dist/src/eval-workflows/evaluation-pipeline.js.map +1 -1
  80. package/dist/src/eval-workflows/evaluation-preparation.d.ts +10 -8
  81. package/dist/src/eval-workflows/evaluation-preparation.d.ts.map +1 -1
  82. package/dist/src/eval-workflows/evaluation-preparation.js +8 -4
  83. package/dist/src/eval-workflows/evaluation-preparation.js.map +1 -1
  84. package/dist/src/eval-workflows/run-evaluation.d.ts +23 -20
  85. package/dist/src/eval-workflows/run-evaluation.d.ts.map +1 -1
  86. package/dist/src/eval-workflows/run-evaluation.js +34 -25
  87. package/dist/src/eval-workflows/run-evaluation.js.map +1 -1
  88. package/dist/src/executors/claude-cli.d.ts.map +1 -1
  89. package/dist/src/executors/claude-cli.js +12 -7
  90. package/dist/src/executors/claude-cli.js.map +1 -1
  91. package/dist/src/executors/claude-sdk.d.ts +1 -1
  92. package/dist/src/executors/claude-sdk.js +1 -1
  93. package/dist/src/executors/codex-cli-trace.d.ts +10 -0
  94. package/dist/src/executors/codex-cli-trace.d.ts.map +1 -0
  95. package/dist/src/executors/codex-cli-trace.js +123 -0
  96. package/dist/src/executors/codex-cli-trace.js.map +1 -0
  97. package/dist/src/executors/codex-cli.d.ts +18 -0
  98. package/dist/src/executors/codex-cli.d.ts.map +1 -0
  99. package/dist/src/executors/codex-cli.js +254 -0
  100. package/dist/src/executors/codex-cli.js.map +1 -0
  101. package/dist/src/executors/codex-sdk.d.ts +18 -0
  102. package/dist/src/executors/codex-sdk.d.ts.map +1 -0
  103. package/dist/src/executors/codex-sdk.js +214 -0
  104. package/dist/src/executors/codex-sdk.js.map +1 -0
  105. package/dist/src/executors/gemini.d.ts.map +1 -1
  106. package/dist/src/executors/gemini.js +28 -24
  107. package/dist/src/executors/gemini.js.map +1 -1
  108. package/dist/src/executors/index.d.ts.map +1 -1
  109. package/dist/src/executors/index.js +7 -2
  110. package/dist/src/executors/index.js.map +1 -1
  111. package/dist/src/executors/runtime-fingerprint.d.ts +7 -0
  112. package/dist/src/executors/runtime-fingerprint.d.ts.map +1 -0
  113. package/dist/src/executors/runtime-fingerprint.js +277 -0
  114. package/dist/src/executors/runtime-fingerprint.js.map +1 -0
  115. package/dist/src/executors/script.d.ts.map +1 -1
  116. package/dist/src/executors/script.js +48 -56
  117. package/dist/src/executors/script.js.map +1 -1
  118. package/dist/src/executors/shared.d.ts +78 -1
  119. package/dist/src/executors/shared.d.ts.map +1 -1
  120. package/dist/src/executors/shared.js +203 -1
  121. package/dist/src/executors/shared.js.map +1 -1
  122. package/dist/src/grading/assertions.d.ts.map +1 -1
  123. package/dist/src/grading/assertions.js +21 -6
  124. package/dist/src/grading/assertions.js.map +1 -1
  125. package/dist/src/grading/gold-dataset.d.ts +1 -1
  126. package/dist/src/grading/gold-dataset.js +1 -1
  127. package/dist/src/grading/index.d.ts.map +1 -1
  128. package/dist/src/grading/index.js +11 -0
  129. package/dist/src/grading/index.js.map +1 -1
  130. package/dist/src/grading/judge.d.ts.map +1 -1
  131. package/dist/src/grading/judge.js +75 -6
  132. package/dist/src/grading/judge.js.map +1 -1
  133. package/dist/src/inputs/eval-config.js +2 -2
  134. package/dist/src/inputs/eval-config.js.map +1 -1
  135. package/dist/src/inputs/load-samples.d.ts.map +1 -1
  136. package/dist/src/inputs/load-samples.js +30 -4
  137. package/dist/src/inputs/load-samples.js.map +1 -1
  138. package/dist/src/inputs/skill-loader.d.ts +2 -2
  139. package/dist/src/inputs/skill-loader.d.ts.map +1 -1
  140. package/dist/src/inputs/skill-loader.js +2 -2
  141. package/dist/src/inputs/skill-loader.js.map +1 -1
  142. package/dist/src/renderer/html-renderer.d.ts +5 -4
  143. package/dist/src/renderer/html-renderer.d.ts.map +1 -1
  144. package/dist/src/renderer/html-renderer.js +217 -93
  145. package/dist/src/renderer/html-renderer.js.map +1 -1
  146. package/dist/src/renderer/layout.d.ts +2 -1
  147. package/dist/src/renderer/layout.d.ts.map +1 -1
  148. package/dist/src/renderer/layout.js +30 -42
  149. package/dist/src/renderer/layout.js.map +1 -1
  150. package/dist/src/renderer/summary.d.ts +3 -3
  151. package/dist/src/renderer/summary.d.ts.map +1 -1
  152. package/dist/src/renderer/summary.js +232 -55
  153. package/dist/src/renderer/summary.js.map +1 -1
  154. package/dist/src/renderer/trends.d.ts.map +1 -1
  155. package/dist/src/renderer/trends.js +5 -3
  156. package/dist/src/renderer/trends.js.map +1 -1
  157. package/dist/src/server/report-server.js +4 -4
  158. package/dist/src/server/report-server.js.map +1 -1
  159. package/dist/src/server/report-store.d.ts +7 -5
  160. package/dist/src/server/report-store.d.ts.map +1 -1
  161. package/dist/src/server/report-store.js +39 -11
  162. package/dist/src/server/report-store.js.map +1 -1
  163. package/dist/src/types/eval.d.ts +27 -5
  164. package/dist/src/types/eval.d.ts.map +1 -1
  165. package/dist/src/types/executor.d.ts +5 -0
  166. package/dist/src/types/executor.d.ts.map +1 -1
  167. package/dist/src/types/judge.d.ts +10 -0
  168. package/dist/src/types/judge.d.ts.map +1 -1
  169. package/dist/src/types/report.d.ts +151 -29
  170. package/dist/src/types/report.d.ts.map +1 -1
  171. package/dist/src/types/storage.d.ts +7 -7
  172. package/dist/src/types/storage.d.ts.map +1 -1
  173. package/package.json +14 -5
  174. package/dist/src/cli.d.ts +0 -3
  175. package/dist/src/cli.d.ts.map +0 -1
  176. package/dist/src/cli.js.map +0 -1
  177. package/dist/src/eval-workflows/each-evaluation-workflow.d.ts +0 -153
  178. package/dist/src/eval-workflows/each-evaluation-workflow.d.ts.map +0 -1
  179. package/dist/src/eval-workflows/each-evaluation-workflow.js +0 -178
  180. package/dist/src/eval-workflows/each-evaluation-workflow.js.map +0 -1
  181. package/dist/src/executors/openai-cli.d.ts +0 -3
  182. package/dist/src/executors/openai-cli.d.ts.map +0 -1
  183. package/dist/src/executors/openai-cli.js +0 -60
  184. package/dist/src/executors/openai-cli.js.map +0 -1
@@ -1,60 +0,0 @@
1
- import { asErrorLike, DEFAULT_TIMEOUT_MS, errorMessage, execFileAsync, MAX_BUFFER, parseJson, timeoutExecResult, } from './shared.js';
2
- export async function openAiCliExecutor({ model, system, prompt, timeoutMs = DEFAULT_TIMEOUT_MS }) {
3
- const args = ['api', 'chat.completions.create', '-m', model || 'gpt-4o'];
4
- if (system) {
5
- args.push('-g', 'system', system);
6
- }
7
- args.push('-g', 'user', prompt);
8
- const start = Date.now();
9
- try {
10
- const { stdout } = await execFileAsync('openai', args, {
11
- maxBuffer: MAX_BUFFER,
12
- timeout: timeoutMs,
13
- env: { ...process.env },
14
- });
15
- const durationMs = Date.now() - start;
16
- const data = parseJson(stdout);
17
- const usage = data.usage || {};
18
- const content = data.choices?.[0]?.message?.content || '';
19
- const finishReason = data.choices?.[0]?.finish_reason || 'unknown';
20
- return {
21
- ok: true,
22
- durationMs,
23
- durationApiMs: 0,
24
- inputTokens: usage.prompt_tokens || 0,
25
- outputTokens: usage.completion_tokens || 0,
26
- cacheReadTokens: usage.prompt_tokens_details?.cached_tokens || 0,
27
- cacheCreationTokens: 0,
28
- costUSD: 0,
29
- output: content,
30
- stopReason: finishReason,
31
- numTurns: 1,
32
- };
33
- }
34
- catch (err) {
35
- const durationMs = Date.now() - start;
36
- const details = asErrorLike(err);
37
- if (details.killed)
38
- return timeoutExecResult(timeoutMs, durationMs);
39
- let parsed = null;
40
- try {
41
- parsed = parseJson(details.stdout || '');
42
- }
43
- catch { }
44
- return {
45
- ok: false,
46
- error: parsed?.error?.message || errorMessage(err),
47
- durationMs,
48
- durationApiMs: 0,
49
- inputTokens: 0,
50
- outputTokens: 0,
51
- cacheReadTokens: 0,
52
- cacheCreationTokens: 0,
53
- costUSD: 0,
54
- output: null,
55
- stopReason: 'error',
56
- numTurns: 0,
57
- };
58
- }
59
- }
60
- //# sourceMappingURL=openai-cli.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"openai-cli.js","sourceRoot":"","sources":["../../../src/executors/openai-cli.ts"],"names":[],"mappings":"AACA,OAAO,EACL,WAAW,EACX,kBAAkB,EAClB,YAAY,EACZ,aAAa,EACb,UAAU,EAEV,SAAS,EACT,iBAAiB,GAClB,MAAM,aAAa,CAAC;AAErB,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,GAAG,kBAAkB,EAAiB;IAC9G,MAAM,IAAI,GAAG,CAAC,KAAK,EAAE,yBAAyB,EAAE,IAAI,EAAE,KAAK,IAAI,QAAQ,CAAC,CAAC;IACzE,IAAI,MAAM,EAAE,CAAC;QACX,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;IACpC,CAAC;IACD,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAEhC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,IAAI,CAAC;QACH,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,aAAa,CAAC,QAAQ,EAAE,IAAI,EAAE;YACrD,SAAS,EAAE,UAAU;YACrB,OAAO,EAAE,SAAS;YAClB,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE;SACxB,CAAC,CAAC;QACH,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QACtC,MAAM,IAAI,GAAG,SAAS,CAAiB,MAAM,CAAC,CAAC;QAC/C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;QAC/B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,IAAI,EAAE,CAAC;QAC1D,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,aAAa,IAAI,SAAS,CAAC;QACnE,OAAO;YACL,EAAE,EAAE,IAAI;YACR,UAAU;YACV,aAAa,EAAE,CAAC;YAChB,WAAW,EAAE,KAAK,CAAC,aAAa,IAAI,CAAC;YACrC,YAAY,EAAE,KAAK,CAAC,iBAAiB,IAAI,CAAC;YAC1C,eAAe,EAAE,KAAK,CAAC,qBAAqB,EAAE,aAAa,IAAI,CAAC;YAChE,mBAAmB,EAAE,CAAC;YACtB,OAAO,EAAE,CAAC;YACV,MAAM,EAAE,OAAO;YACf,UAAU,EAAE,YAAY;YACxB,QAAQ,EAAE,CAAC;SACZ,CAAC;IACJ,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QACtC,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;QACjC,IAAI,OAAO,CAAC,MAAM;YAAE,OAAO,iBAAiB,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;QACpE,IAAI,MAAM,GAA0B,IAAI,CAAC;QACzC,IAAI,CAAC;YAAC,MAAM,GAAG,SAAS,CAAiB,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC,CAAA,CAAC;QAC1E,OAAO;YACL,EAAE,EAAE,KAAK;YACT,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,IAAI,YAAY,CAAC,GAAG,CAAC;YAClD,UAAU;YACV,aAAa,EAAE,CAAC;YAChB,WAAW,EAAE,CAAC;YACd,YAAY,EAAE,CAAC;YACf,eAAe,EAAE,CAAC;YAClB,mBAAmB,EAAE,CAAC;YACtB,OAAO,EAAE,CAAC;YACV,MAAM,EAAE,IAAI;YACZ,UAAU,EAAE,OAAO;YACnB,QAAQ,EAAE,CAAC;SACZ,CAAC;IACJ,CAAC;AACH,CAAC"}