oh-my-knowledge 0.22.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +63 -63
- package/README.zh.md +64 -62
- package/dist/src/analysis/report-diagnostics.d.ts +28 -3
- package/dist/src/analysis/report-diagnostics.d.ts.map +1 -1
- package/dist/src/analysis/report-diagnostics.js +201 -81
- package/dist/src/analysis/report-diagnostics.js.map +1 -1
- package/dist/src/analysis/sample-diagnostics.d.ts +9 -2
- package/dist/src/analysis/sample-diagnostics.d.ts.map +1 -1
- package/dist/src/analysis/sample-diagnostics.js +224 -25
- package/dist/src/analysis/sample-diagnostics.js.map +1 -1
- package/dist/src/authoring/evolver.d.ts +7 -2
- package/dist/src/authoring/evolver.d.ts.map +1 -1
- package/dist/src/authoring/evolver.js +43 -9
- package/dist/src/authoring/evolver.js.map +1 -1
- package/dist/src/authoring/generator.d.ts +24 -0
- package/dist/src/authoring/generator.d.ts.map +1 -1
- package/dist/src/authoring/generator.js +67 -5
- package/dist/src/authoring/generator.js.map +1 -1
- package/dist/src/cli/coverage-renderer.d.ts +15 -0
- package/dist/src/cli/coverage-renderer.d.ts.map +1 -0
- package/dist/src/cli/coverage-renderer.js +74 -0
- package/dist/src/cli/coverage-renderer.js.map +1 -0
- package/dist/src/cli/i18n-dict.d.ts +1 -1
- package/dist/src/cli/i18n-dict.d.ts.map +1 -1
- package/dist/src/cli/i18n-dict.js +62 -40
- package/dist/src/cli/i18n-dict.js.map +1 -1
- package/dist/src/cli/index.d.ts +3 -0
- package/dist/src/cli/index.d.ts.map +1 -0
- package/dist/src/{cli.js → cli/index.js} +123 -384
- package/dist/src/cli/index.js.map +1 -0
- package/dist/src/cli/parse-run-config.d.ts +56 -0
- package/dist/src/cli/parse-run-config.d.ts.map +1 -0
- package/dist/src/cli/parse-run-config.js +195 -0
- package/dist/src/cli/parse-run-config.js.map +1 -0
- package/dist/src/cli/progress.d.ts +25 -0
- package/dist/src/cli/progress.d.ts.map +1 -0
- package/dist/src/cli/progress.js +62 -0
- package/dist/src/cli/progress.js.map +1 -0
- package/dist/src/cli/update-check.d.ts +3 -0
- package/dist/src/cli/update-check.d.ts.map +1 -0
- package/dist/src/cli/update-check.js +37 -0
- package/dist/src/cli/update-check.js.map +1 -0
- package/dist/src/eval-core/cache.d.ts +7 -5
- package/dist/src/eval-core/cache.d.ts.map +1 -1
- package/dist/src/eval-core/cache.js +11 -7
- package/dist/src/eval-core/cache.js.map +1 -1
- package/dist/src/eval-core/comparability.d.ts +11 -0
- package/dist/src/eval-core/comparability.d.ts.map +1 -0
- package/dist/src/eval-core/comparability.js +271 -0
- package/dist/src/eval-core/comparability.js.map +1 -0
- package/dist/src/eval-core/dependency-checker.d.ts +1 -1
- package/dist/src/eval-core/dependency-checker.js +1 -1
- package/dist/src/eval-core/evaluation-execution.d.ts +5 -2
- package/dist/src/eval-core/evaluation-execution.d.ts.map +1 -1
- package/dist/src/eval-core/evaluation-execution.js +29 -21
- package/dist/src/eval-core/evaluation-execution.js.map +1 -1
- package/dist/src/eval-core/evaluation-job.d.ts +2 -2
- package/dist/src/eval-core/evaluation-job.d.ts.map +1 -1
- package/dist/src/eval-core/evaluation-job.js +2 -2
- package/dist/src/eval-core/evaluation-job.js.map +1 -1
- package/dist/src/eval-core/evaluation-reporting.d.ts +3 -1
- package/dist/src/eval-core/evaluation-reporting.d.ts.map +1 -1
- package/dist/src/eval-core/evaluation-reporting.js +65 -5
- package/dist/src/eval-core/evaluation-reporting.js.map +1 -1
- package/dist/src/eval-core/execution-strategy.js +6 -6
- package/dist/src/eval-core/execution-strategy.js.map +1 -1
- package/dist/src/eval-core/schema.d.ts.map +1 -1
- package/dist/src/eval-core/schema.js +14 -1
- package/dist/src/eval-core/schema.js.map +1 -1
- package/dist/src/eval-core/verdict.d.ts +3 -3
- package/dist/src/eval-core/verdict.js +3 -3
- package/dist/src/eval-workflows/batch-evaluation-workflow.d.ts +111 -0
- package/dist/src/eval-workflows/batch-evaluation-workflow.d.ts.map +1 -0
- package/dist/src/eval-workflows/batch-evaluation-workflow.js +215 -0
- package/dist/src/eval-workflows/batch-evaluation-workflow.js.map +1 -0
- package/dist/src/eval-workflows/evaluation-pipeline.d.ts +7 -5
- package/dist/src/eval-workflows/evaluation-pipeline.d.ts.map +1 -1
- package/dist/src/eval-workflows/evaluation-pipeline.js +13 -8
- package/dist/src/eval-workflows/evaluation-pipeline.js.map +1 -1
- package/dist/src/eval-workflows/evaluation-preparation.d.ts +10 -8
- package/dist/src/eval-workflows/evaluation-preparation.d.ts.map +1 -1
- package/dist/src/eval-workflows/evaluation-preparation.js +8 -4
- package/dist/src/eval-workflows/evaluation-preparation.js.map +1 -1
- package/dist/src/eval-workflows/run-evaluation.d.ts +23 -20
- package/dist/src/eval-workflows/run-evaluation.d.ts.map +1 -1
- package/dist/src/eval-workflows/run-evaluation.js +34 -25
- package/dist/src/eval-workflows/run-evaluation.js.map +1 -1
- package/dist/src/executors/claude-cli.d.ts.map +1 -1
- package/dist/src/executors/claude-cli.js +12 -7
- package/dist/src/executors/claude-cli.js.map +1 -1
- package/dist/src/executors/claude-sdk.d.ts +1 -1
- package/dist/src/executors/claude-sdk.js +1 -1
- package/dist/src/executors/codex-cli-trace.d.ts +10 -0
- package/dist/src/executors/codex-cli-trace.d.ts.map +1 -0
- package/dist/src/executors/codex-cli-trace.js +123 -0
- package/dist/src/executors/codex-cli-trace.js.map +1 -0
- package/dist/src/executors/codex-cli.d.ts +18 -0
- package/dist/src/executors/codex-cli.d.ts.map +1 -0
- package/dist/src/executors/codex-cli.js +254 -0
- package/dist/src/executors/codex-cli.js.map +1 -0
- package/dist/src/executors/codex-sdk.d.ts +18 -0
- package/dist/src/executors/codex-sdk.d.ts.map +1 -0
- package/dist/src/executors/codex-sdk.js +214 -0
- package/dist/src/executors/codex-sdk.js.map +1 -0
- package/dist/src/executors/gemini.d.ts.map +1 -1
- package/dist/src/executors/gemini.js +28 -24
- package/dist/src/executors/gemini.js.map +1 -1
- package/dist/src/executors/index.d.ts.map +1 -1
- package/dist/src/executors/index.js +7 -2
- package/dist/src/executors/index.js.map +1 -1
- package/dist/src/executors/runtime-fingerprint.d.ts +7 -0
- package/dist/src/executors/runtime-fingerprint.d.ts.map +1 -0
- package/dist/src/executors/runtime-fingerprint.js +277 -0
- package/dist/src/executors/runtime-fingerprint.js.map +1 -0
- package/dist/src/executors/script.d.ts.map +1 -1
- package/dist/src/executors/script.js +48 -56
- package/dist/src/executors/script.js.map +1 -1
- package/dist/src/executors/shared.d.ts +78 -1
- package/dist/src/executors/shared.d.ts.map +1 -1
- package/dist/src/executors/shared.js +203 -1
- package/dist/src/executors/shared.js.map +1 -1
- package/dist/src/grading/assertions.d.ts.map +1 -1
- package/dist/src/grading/assertions.js +21 -6
- package/dist/src/grading/assertions.js.map +1 -1
- package/dist/src/grading/gold-dataset.d.ts +1 -1
- package/dist/src/grading/gold-dataset.js +1 -1
- package/dist/src/grading/index.d.ts.map +1 -1
- package/dist/src/grading/index.js +11 -0
- package/dist/src/grading/index.js.map +1 -1
- package/dist/src/grading/judge.d.ts.map +1 -1
- package/dist/src/grading/judge.js +75 -6
- package/dist/src/grading/judge.js.map +1 -1
- package/dist/src/inputs/eval-config.js +2 -2
- package/dist/src/inputs/eval-config.js.map +1 -1
- package/dist/src/inputs/load-samples.d.ts.map +1 -1
- package/dist/src/inputs/load-samples.js +30 -4
- package/dist/src/inputs/load-samples.js.map +1 -1
- package/dist/src/inputs/skill-loader.d.ts +2 -2
- package/dist/src/inputs/skill-loader.d.ts.map +1 -1
- package/dist/src/inputs/skill-loader.js +2 -2
- package/dist/src/inputs/skill-loader.js.map +1 -1
- package/dist/src/renderer/html-renderer.d.ts +5 -4
- package/dist/src/renderer/html-renderer.d.ts.map +1 -1
- package/dist/src/renderer/html-renderer.js +217 -93
- package/dist/src/renderer/html-renderer.js.map +1 -1
- package/dist/src/renderer/layout.d.ts +2 -1
- package/dist/src/renderer/layout.d.ts.map +1 -1
- package/dist/src/renderer/layout.js +30 -42
- package/dist/src/renderer/layout.js.map +1 -1
- package/dist/src/renderer/summary.d.ts +3 -3
- package/dist/src/renderer/summary.d.ts.map +1 -1
- package/dist/src/renderer/summary.js +232 -55
- package/dist/src/renderer/summary.js.map +1 -1
- package/dist/src/renderer/trends.d.ts.map +1 -1
- package/dist/src/renderer/trends.js +5 -3
- package/dist/src/renderer/trends.js.map +1 -1
- package/dist/src/server/report-server.js +4 -4
- package/dist/src/server/report-server.js.map +1 -1
- package/dist/src/server/report-store.d.ts +7 -5
- package/dist/src/server/report-store.d.ts.map +1 -1
- package/dist/src/server/report-store.js +39 -11
- package/dist/src/server/report-store.js.map +1 -1
- package/dist/src/types/eval.d.ts +27 -5
- package/dist/src/types/eval.d.ts.map +1 -1
- package/dist/src/types/executor.d.ts +5 -0
- package/dist/src/types/executor.d.ts.map +1 -1
- package/dist/src/types/judge.d.ts +10 -0
- package/dist/src/types/judge.d.ts.map +1 -1
- package/dist/src/types/report.d.ts +151 -29
- package/dist/src/types/report.d.ts.map +1 -1
- package/dist/src/types/storage.d.ts +7 -7
- package/dist/src/types/storage.d.ts.map +1 -1
- package/package.json +14 -5
- package/dist/src/cli.d.ts +0 -3
- package/dist/src/cli.d.ts.map +0 -1
- package/dist/src/cli.js.map +0 -1
- package/dist/src/eval-workflows/each-evaluation-workflow.d.ts +0 -153
- package/dist/src/eval-workflows/each-evaluation-workflow.d.ts.map +0 -1
- package/dist/src/eval-workflows/each-evaluation-workflow.js +0 -178
- package/dist/src/eval-workflows/each-evaluation-workflow.js.map +0 -1
- package/dist/src/executors/openai-cli.d.ts +0 -3
- package/dist/src/executors/openai-cli.d.ts.map +0 -1
- package/dist/src/executors/openai-cli.js +0 -60
- package/dist/src/executors/openai-cli.js.map +0 -1
|
@@ -1,60 +0,0 @@
|
|
|
1
|
-
import { asErrorLike, DEFAULT_TIMEOUT_MS, errorMessage, execFileAsync, MAX_BUFFER, parseJson, timeoutExecResult, } from './shared.js';
|
|
2
|
-
export async function openAiCliExecutor({ model, system, prompt, timeoutMs = DEFAULT_TIMEOUT_MS }) {
|
|
3
|
-
const args = ['api', 'chat.completions.create', '-m', model || 'gpt-4o'];
|
|
4
|
-
if (system) {
|
|
5
|
-
args.push('-g', 'system', system);
|
|
6
|
-
}
|
|
7
|
-
args.push('-g', 'user', prompt);
|
|
8
|
-
const start = Date.now();
|
|
9
|
-
try {
|
|
10
|
-
const { stdout } = await execFileAsync('openai', args, {
|
|
11
|
-
maxBuffer: MAX_BUFFER,
|
|
12
|
-
timeout: timeoutMs,
|
|
13
|
-
env: { ...process.env },
|
|
14
|
-
});
|
|
15
|
-
const durationMs = Date.now() - start;
|
|
16
|
-
const data = parseJson(stdout);
|
|
17
|
-
const usage = data.usage || {};
|
|
18
|
-
const content = data.choices?.[0]?.message?.content || '';
|
|
19
|
-
const finishReason = data.choices?.[0]?.finish_reason || 'unknown';
|
|
20
|
-
return {
|
|
21
|
-
ok: true,
|
|
22
|
-
durationMs,
|
|
23
|
-
durationApiMs: 0,
|
|
24
|
-
inputTokens: usage.prompt_tokens || 0,
|
|
25
|
-
outputTokens: usage.completion_tokens || 0,
|
|
26
|
-
cacheReadTokens: usage.prompt_tokens_details?.cached_tokens || 0,
|
|
27
|
-
cacheCreationTokens: 0,
|
|
28
|
-
costUSD: 0,
|
|
29
|
-
output: content,
|
|
30
|
-
stopReason: finishReason,
|
|
31
|
-
numTurns: 1,
|
|
32
|
-
};
|
|
33
|
-
}
|
|
34
|
-
catch (err) {
|
|
35
|
-
const durationMs = Date.now() - start;
|
|
36
|
-
const details = asErrorLike(err);
|
|
37
|
-
if (details.killed)
|
|
38
|
-
return timeoutExecResult(timeoutMs, durationMs);
|
|
39
|
-
let parsed = null;
|
|
40
|
-
try {
|
|
41
|
-
parsed = parseJson(details.stdout || '');
|
|
42
|
-
}
|
|
43
|
-
catch { }
|
|
44
|
-
return {
|
|
45
|
-
ok: false,
|
|
46
|
-
error: parsed?.error?.message || errorMessage(err),
|
|
47
|
-
durationMs,
|
|
48
|
-
durationApiMs: 0,
|
|
49
|
-
inputTokens: 0,
|
|
50
|
-
outputTokens: 0,
|
|
51
|
-
cacheReadTokens: 0,
|
|
52
|
-
cacheCreationTokens: 0,
|
|
53
|
-
costUSD: 0,
|
|
54
|
-
output: null,
|
|
55
|
-
stopReason: 'error',
|
|
56
|
-
numTurns: 0,
|
|
57
|
-
};
|
|
58
|
-
}
|
|
59
|
-
}
|
|
60
|
-
//# sourceMappingURL=openai-cli.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"openai-cli.js","sourceRoot":"","sources":["../../../src/executors/openai-cli.ts"],"names":[],"mappings":"AACA,OAAO,EACL,WAAW,EACX,kBAAkB,EAClB,YAAY,EACZ,aAAa,EACb,UAAU,EAEV,SAAS,EACT,iBAAiB,GAClB,MAAM,aAAa,CAAC;AAErB,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,GAAG,kBAAkB,EAAiB;IAC9G,MAAM,IAAI,GAAG,CAAC,KAAK,EAAE,yBAAyB,EAAE,IAAI,EAAE,KAAK,IAAI,QAAQ,CAAC,CAAC;IACzE,IAAI,MAAM,EAAE,CAAC;QACX,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;IACpC,CAAC;IACD,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAEhC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,IAAI,CAAC;QACH,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,aAAa,CAAC,QAAQ,EAAE,IAAI,EAAE;YACrD,SAAS,EAAE,UAAU;YACrB,OAAO,EAAE,SAAS;YAClB,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE;SACxB,CAAC,CAAC;QACH,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QACtC,MAAM,IAAI,GAAG,SAAS,CAAiB,MAAM,CAAC,CAAC;QAC/C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;QAC/B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,IAAI,EAAE,CAAC;QAC1D,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,aAAa,IAAI,SAAS,CAAC;QACnE,OAAO;YACL,EAAE,EAAE,IAAI;YACR,UAAU;YACV,aAAa,EAAE,CAAC;YAChB,WAAW,EAAE,KAAK,CAAC,aAAa,IAAI,CAAC;YACrC,YAAY,EAAE,KAAK,CAAC,iBAAiB,IAAI,CAAC;YAC1C,eAAe,EAAE,KAAK,CAAC,qBAAqB,EAAE,aAAa,IAAI,CAAC;YAChE,mBAAmB,EAAE,CAAC;YACtB,OAAO,EAAE,CAAC;YACV,MAAM,EAAE,OAAO;YACf,UAAU,EAAE,YAAY;YACxB,QAAQ,EAAE,CAAC;SACZ,CAAC;IACJ,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QACtC,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;QACjC,IAAI,OAAO,CAAC,MAAM;YAAE,OAAO,iBAAiB,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;QACpE,IAAI,MAAM,GAA0B,IAAI,CAAC;QACzC,IAAI,CAAC;YAAC,MAAM,GAAG,SAAS,CAAiB,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC,CAAA,CAAC;QAC1E,OAAO;YACL,EAAE,EAAE,KAAK;YACT,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,IAAI,YAAY,CAAC,GAAG,CAAC;YAClD,UAAU;YACV,aAAa,EAAE,CAAC;YAChB,WAAW,EAAE,CAAC;YACd,YAAY,EAAE,CAAC;YACf,eAAe,EAAE,CAAC;YAClB,mBAAmB,EAAE,CAAC;YACtB,OAAO,EAAE,CAAC;YACV,MAAM,EAAE,IAAI;YACZ,UAAU,EAAE,OAAO;YACnB,QAAQ,EAAE,CAAC;SACZ,CAAC;IACJ,CAAC;AACH,CAAC"}
|