@oscharko-dev/keiko-cli 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -0
- package/dist/context.d.ts +3 -0
- package/dist/context.d.ts.map +1 -0
- package/dist/context.js +103 -0
- package/dist/doctor.d.ts +24 -0
- package/dist/doctor.d.ts.map +1 -0
- package/dist/doctor.js +108 -0
- package/dist/evaluate.d.ts +8 -0
- package/dist/evaluate.d.ts.map +1 -0
- package/dist/evaluate.js +270 -0
- package/dist/evidence.d.ts +9 -0
- package/dist/evidence.d.ts.map +1 -0
- package/dist/evidence.js +129 -0
- package/dist/gateway-config.d.ts +12 -0
- package/dist/gateway-config.d.ts.map +1 -0
- package/dist/gateway-config.js +19 -0
- package/dist/gen-tests.d.ts +8 -0
- package/dist/gen-tests.d.ts.map +1 -0
- package/dist/gen-tests.js +216 -0
- package/dist/index.d.ts +18 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +26 -0
- package/dist/init.d.ts +9 -0
- package/dist/init.d.ts.map +1 -0
- package/dist/init.js +122 -0
- package/dist/install-layout.d.ts +19 -0
- package/dist/install-layout.d.ts.map +1 -0
- package/dist/install-layout.js +76 -0
- package/dist/investigate.d.ts +9 -0
- package/dist/investigate.d.ts.map +1 -0
- package/dist/investigate.js +249 -0
- package/dist/launcher-paths.d.ts +4 -0
- package/dist/launcher-paths.d.ts.map +1 -0
- package/dist/launcher-paths.js +69 -0
- package/dist/launcher-platforms.d.ts +25 -0
- package/dist/launcher-platforms.d.ts.map +1 -0
- package/dist/launcher-platforms.js +131 -0
- package/dist/launcher-state.d.ts +25 -0
- package/dist/launcher-state.d.ts.map +1 -0
- package/dist/launcher-state.js +228 -0
- package/dist/launcher.d.ts +21 -0
- package/dist/launcher.d.ts.map +1 -0
- package/dist/launcher.js +439 -0
- package/dist/lifecycle.d.ts +22 -0
- package/dist/lifecycle.d.ts.map +1 -0
- package/dist/lifecycle.js +425 -0
- package/dist/memory.d.ts +14 -0
- package/dist/memory.d.ts.map +1 -0
- package/dist/memory.js +290 -0
- package/dist/models.d.ts +4 -0
- package/dist/models.d.ts.map +1 -0
- package/dist/models.js +62 -0
- package/dist/prompt-enhancer.d.ts +13 -0
- package/dist/prompt-enhancer.d.ts.map +1 -0
- package/dist/prompt-enhancer.js +261 -0
- package/dist/repair.d.ts +10 -0
- package/dist/repair.d.ts.map +1 -0
- package/dist/repair.js +402 -0
- package/dist/run.d.ts +10 -0
- package/dist/run.d.ts.map +1 -0
- package/dist/run.js +269 -0
- package/dist/runner.d.ts +7 -0
- package/dist/runner.d.ts.map +1 -0
- package/dist/runner.js +108 -0
- package/dist/state-paths.d.ts +43 -0
- package/dist/state-paths.d.ts.map +1 -0
- package/dist/state-paths.js +396 -0
- package/dist/ui.d.ts +39 -0
- package/dist/ui.d.ts.map +1 -0
- package/dist/ui.js +450 -0
- package/dist/uninstall.d.ts +10 -0
- package/dist/uninstall.d.ts.map +1 -0
- package/dist/uninstall.js +345 -0
- package/dist/verify.d.ts +3 -0
- package/dist/verify.d.ts.map +1 -0
- package/dist/verify.js +108 -0
- package/package.json +42 -0
package/dist/doctor.js
ADDED
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
2
|
+
import { isAbsolute, join, resolve } from "node:path";
|
|
3
|
+
import { SDK_VERSION } from "@oscharko-dev/keiko-sdk";
|
|
4
|
+
import { hasBuiltKeikoLayout, localPackageRoot, resolvePreferredInstallLayout, } from "./install-layout.js";
|
|
5
|
+
function readVersion(packageJsonPath) {
|
|
6
|
+
if (!existsSync(packageJsonPath))
|
|
7
|
+
return undefined;
|
|
8
|
+
try {
|
|
9
|
+
const parsed = JSON.parse(readFileSync(packageJsonPath, "utf8"));
|
|
10
|
+
return typeof parsed.version === "string" ? parsed.version : undefined;
|
|
11
|
+
}
|
|
12
|
+
catch {
|
|
13
|
+
return undefined;
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
function resolveLocalPackageInstall(cwd) {
|
|
17
|
+
const packageRoot = localPackageRoot(cwd);
|
|
18
|
+
const cliEntry = resolve(packageRoot, "dist", "cli", "index.js");
|
|
19
|
+
const version = readVersion(join(packageRoot, "package.json"));
|
|
20
|
+
if (!hasBuiltKeikoLayout(packageRoot) || version === undefined)
|
|
21
|
+
return undefined;
|
|
22
|
+
return { packageRoot, cliEntry, version };
|
|
23
|
+
}
|
|
24
|
+
function resolveRunningEntry(argv) {
|
|
25
|
+
const entry = argv?.[1];
|
|
26
|
+
if (typeof entry !== "string")
|
|
27
|
+
return undefined;
|
|
28
|
+
const windowsAbsolute = /^[A-Za-z]:[\\/]/.test(entry);
|
|
29
|
+
if (!isAbsolute(entry) && !windowsAbsolute)
|
|
30
|
+
return undefined;
|
|
31
|
+
return entry;
|
|
32
|
+
}
|
|
33
|
+
function staleBinaryWarning(report) {
|
|
34
|
+
const runningEntry = report.runningEntry;
|
|
35
|
+
if (runningEntry === undefined)
|
|
36
|
+
return undefined;
|
|
37
|
+
if (report.localBuildBin !== undefined && runningEntry !== report.localBuildBin) {
|
|
38
|
+
return ("You are running Keiko from a different binary than this checkout's built CLI entry.\n" +
|
|
39
|
+
` running: ${runningEntry}\n` +
|
|
40
|
+
` local build: ${report.localBuildBin}\n` +
|
|
41
|
+
"Use `npm run keiko:start` or `node ./dist/cli/index.js start` from this checkout.");
|
|
42
|
+
}
|
|
43
|
+
const localInstall = report.localPackageInstall;
|
|
44
|
+
if (localInstall !== undefined && runningEntry !== localInstall.cliEntry) {
|
|
45
|
+
return ("You are running a different Keiko binary than the package installed in this workspace.\n" +
|
|
46
|
+
` running: ${report.runningVersion} (${runningEntry})\n` +
|
|
47
|
+
` local install: ${localInstall.version} (${localInstall.cliEntry})\n` +
|
|
48
|
+
"Use the local package script (`npm run keiko:start`) or remove the stale global install.");
|
|
49
|
+
}
|
|
50
|
+
return undefined;
|
|
51
|
+
}
|
|
52
|
+
export function collectDoctorReport(deps = {}) {
|
|
53
|
+
const cwd = deps.cwd ?? process.cwd();
|
|
54
|
+
const runningEntry = resolveRunningEntry(deps.argv ?? process.argv);
|
|
55
|
+
const localBuildBin = resolvePreferredInstallLayout(cwd)?.binPath;
|
|
56
|
+
const localPackageInstall = resolveLocalPackageInstall(cwd);
|
|
57
|
+
const report = {
|
|
58
|
+
cwd,
|
|
59
|
+
runningEntry,
|
|
60
|
+
runningVersion: SDK_VERSION,
|
|
61
|
+
localBuildBin,
|
|
62
|
+
localPackageInstall,
|
|
63
|
+
warning: undefined,
|
|
64
|
+
};
|
|
65
|
+
return { ...report, warning: staleBinaryWarning(report) };
|
|
66
|
+
}
|
|
67
|
+
function windowsRemediation() {
|
|
68
|
+
return [
|
|
69
|
+
"Windows remediation:",
|
|
70
|
+
" 1. Run `where.exe keiko` to see which global shim wins on PATH.",
|
|
71
|
+
" 2. Prefer `npm run keiko:start` from the project checkout.",
|
|
72
|
+
" 3. If a stale global install is found, run `npm uninstall -g @oscharko-dev/keiko`.",
|
|
73
|
+
" 4. For a direct local launch, run `node .\\dist\\cli\\index.js start`.",
|
|
74
|
+
].join("\n");
|
|
75
|
+
}
|
|
76
|
+
function posixRemediation() {
|
|
77
|
+
return [
|
|
78
|
+
"macOS/Linux remediation:",
|
|
79
|
+
" 1. Run `which keiko` to see which global binary wins on PATH.",
|
|
80
|
+
" 2. Prefer `npm run keiko:start` from the project checkout.",
|
|
81
|
+
" 3. If a stale global install is found, run `npm uninstall -g @oscharko-dev/keiko`.",
|
|
82
|
+
" 4. For a direct local launch, run `node ./dist/cli/index.js start`.",
|
|
83
|
+
].join("\n");
|
|
84
|
+
}
|
|
85
|
+
export function emitDoctorWarning(io, deps = {}) {
|
|
86
|
+
const report = collectDoctorReport(deps);
|
|
87
|
+
if (report.warning === undefined)
|
|
88
|
+
return;
|
|
89
|
+
io.err(`keiko warning: stale launch path detected.\n${report.warning}\n`);
|
|
90
|
+
}
|
|
91
|
+
export function runDoctorCli(_args, io, _env, deps = {}) {
|
|
92
|
+
const report = collectDoctorReport(deps);
|
|
93
|
+
io.out(`Keiko doctor\n`);
|
|
94
|
+
io.out(` cwd: ${report.cwd}\n`);
|
|
95
|
+
io.out(` running version: ${report.runningVersion}\n`);
|
|
96
|
+
io.out(` running entry: ${report.runningEntry ?? "(unavailable)"}\n`);
|
|
97
|
+
io.out(` local built CLI: ${report.localBuildBin ?? "(not found)"}\n`);
|
|
98
|
+
io.out(` local installed package: ${report.localPackageInstall?.cliEntry ?? "(not found)"}\n`);
|
|
99
|
+
io.out(` local installed version: ${report.localPackageInstall?.version ?? "(not found)"}\n`);
|
|
100
|
+
if (report.warning !== undefined) {
|
|
101
|
+
io.out(`\nDiagnosis:\n${report.warning}\n\n`);
|
|
102
|
+
}
|
|
103
|
+
else {
|
|
104
|
+
io.out("\nDiagnosis:\nNo stale-launch mismatch detected.\n\n");
|
|
105
|
+
}
|
|
106
|
+
io.out(`${process.platform === "win32" ? windowsRemediation() : posixRemediation()}\n`);
|
|
107
|
+
return 0;
|
|
108
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { type EnvSource } from "@oscharko-dev/keiko-model-gateway";
|
|
2
|
+
import { type EvalRunnerDeps } from "@oscharko-dev/keiko-evaluations";
|
|
3
|
+
import type { CliIo } from "./runner.js";
|
|
4
|
+
export interface EvaluateDeps {
|
|
5
|
+
readonly runner?: EvalRunnerDeps | undefined;
|
|
6
|
+
}
|
|
7
|
+
export declare function runEvaluateCli(args: readonly string[], io: CliIo, env?: EnvSource, deps?: EvaluateDeps): Promise<number>;
|
|
8
|
+
//# sourceMappingURL=evaluate.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"evaluate.d.ts","sourceRoot":"","sources":["../src/evaluate.ts"],"names":[],"mappings":"AASA,OAAO,EAOL,KAAK,SAAS,EAGf,MAAM,mCAAmC,CAAC;AAI3C,OAAO,EAML,KAAK,cAAc,EAGpB,MAAM,iCAAiC,CAAC;AAIzC,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AAWzC,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,MAAM,CAAC,EAAE,cAAc,GAAG,SAAS,CAAC;CAC9C;AA+ID,wBAAsB,cAAc,CAClC,IAAI,EAAE,SAAS,MAAM,EAAE,EACvB,EAAE,EAAE,KAAK,EACT,GAAG,GAAE,SAAc,EACnB,IAAI,GAAE,YAAiB,GACtB,OAAO,CAAC,MAAM,CAAC,CAqBjB"}
|
package/dist/evaluate.js
ADDED
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
// `keiko evaluate` — runs the evaluation harness (ADR-0012 D10). Offline (default, deterministic, no
|
|
2
|
+
// network) replays each fixture's scripted transcript; --live builds a GatewayModelPort and fails
|
|
3
|
+
// CLOSED (exit 1, names the required env vars) when no config/credentials resolve — it NEVER silently
|
|
4
|
+
// falls back to offline. Dry-run-safe by construction: fixtures choose their own apply mode. Mirrors
|
|
5
|
+
// runGenTestsCli structurally (injected CliIo + deps, testable without process.*). Exit 0 when all
|
|
6
|
+
// applicable dimensions pass AND surface parity passes; 1 on dimension/parity failure or runtime
|
|
7
|
+
// error; 2 on usage error (unknown flag, mutual exclusion, unknown suite/fixture name).
|
|
8
|
+
import { writeFileSync } from "node:fs";
|
|
9
|
+
import { ConfigInvalidError, GatewayError, assertConfiguredModel, findConfiguredCapability, listConfiguredCapabilities, redact, } from "@oscharko-dev/keiko-model-gateway";
|
|
10
|
+
import { createAuditRedactor, deepRedactStrings } from "@oscharko-dev/keiko-evidence";
|
|
11
|
+
import { keikoApiKeySecretValues } from "@oscharko-dev/keiko-security";
|
|
12
|
+
import { parseRunRequest } from "@oscharko-dev/keiko-server";
|
|
13
|
+
import { fixtureByName, fixturesForSuite, isSuiteName, renderEvalSummary, runEvaluationSuite, } from "@oscharko-dev/keiko-evaluations";
|
|
14
|
+
import { runGenTestsCli } from "./gen-tests.js";
|
|
15
|
+
import { loadGatewayConfigFromFile } from "./gateway-config.js";
|
|
16
|
+
import { runInvestigateCli } from "./investigate.js";
|
|
17
|
+
const USAGE = `Usage:
|
|
18
|
+
keiko evaluate [--suite <unit-tests|bug-investigation|all>] [--fixture <name>]
|
|
19
|
+
[--live] [--model <id>] [--config PATH] [--json] [--output <path>]
|
|
20
|
+
|
|
21
|
+
Runs the evaluation harness against the built-in fixtures. Offline by default
|
|
22
|
+
(deterministic, no network); pass --live to evaluate against a configured model.
|
|
23
|
+
--suite and --fixture are mutually exclusive.
|
|
24
|
+
`;
|
|
25
|
+
function flagValue(args, name) {
|
|
26
|
+
const i = args.indexOf(name);
|
|
27
|
+
if (i === -1) {
|
|
28
|
+
return undefined;
|
|
29
|
+
}
|
|
30
|
+
const value = args[i + 1];
|
|
31
|
+
return value === undefined || value.startsWith("--") ? null : value;
|
|
32
|
+
}
|
|
33
|
+
const VALUE_FLAGS = ["--suite", "--fixture", "--model", "--config", "--output"];
|
|
34
|
+
const BOOLEAN_FLAGS = ["--live", "--json"];
|
|
35
|
+
function readValueFlags(args) {
|
|
36
|
+
const values = {};
|
|
37
|
+
for (const flag of VALUE_FLAGS) {
|
|
38
|
+
const value = flagValue(args, flag);
|
|
39
|
+
if (value === null) {
|
|
40
|
+
return null;
|
|
41
|
+
}
|
|
42
|
+
values[flag] = value;
|
|
43
|
+
}
|
|
44
|
+
return values;
|
|
45
|
+
}
|
|
46
|
+
function isValueFlag(value) {
|
|
47
|
+
return VALUE_FLAGS.includes(value);
|
|
48
|
+
}
|
|
49
|
+
function isBooleanFlag(value) {
|
|
50
|
+
return BOOLEAN_FLAGS.includes(value);
|
|
51
|
+
}
|
|
52
|
+
function findUsageError(args) {
|
|
53
|
+
for (let i = 0; i < args.length; i += 1) {
|
|
54
|
+
const arg = args[i];
|
|
55
|
+
if (arg === undefined) {
|
|
56
|
+
continue;
|
|
57
|
+
}
|
|
58
|
+
if (isValueFlag(arg)) {
|
|
59
|
+
const value = args[i + 1];
|
|
60
|
+
if (value === undefined || value.startsWith("--")) {
|
|
61
|
+
return `missing value for ${arg}`;
|
|
62
|
+
}
|
|
63
|
+
i += 1;
|
|
64
|
+
continue;
|
|
65
|
+
}
|
|
66
|
+
if (isBooleanFlag(arg)) {
|
|
67
|
+
continue;
|
|
68
|
+
}
|
|
69
|
+
return arg.startsWith("--") ? `unknown flag ${arg}` : `unexpected argument ${arg}`;
|
|
70
|
+
}
|
|
71
|
+
return undefined;
|
|
72
|
+
}
|
|
73
|
+
function parseArgs(args) {
|
|
74
|
+
const values = readValueFlags(args);
|
|
75
|
+
if (values === null) {
|
|
76
|
+
return null;
|
|
77
|
+
}
|
|
78
|
+
return {
|
|
79
|
+
suite: values["--suite"],
|
|
80
|
+
fixture: values["--fixture"],
|
|
81
|
+
live: args.includes("--live"),
|
|
82
|
+
model: values["--model"],
|
|
83
|
+
config: values["--config"],
|
|
84
|
+
json: args.includes("--json"),
|
|
85
|
+
output: values["--output"],
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
// Resolves the fixture set from --suite / --fixture, enforcing mutual exclusion and name validity.
|
|
89
|
+
function selectFixtures(parsed) {
|
|
90
|
+
if (parsed.suite !== undefined && parsed.fixture !== undefined) {
|
|
91
|
+
return { usageError: "Error: --suite and --fixture are mutually exclusive.\n" };
|
|
92
|
+
}
|
|
93
|
+
if (parsed.fixture !== undefined) {
|
|
94
|
+
const fixture = fixtureByName(parsed.fixture);
|
|
95
|
+
return fixture === undefined
|
|
96
|
+
? { usageError: `Error: unknown fixture "${parsed.fixture}".\n` }
|
|
97
|
+
: { fixtures: [fixture] };
|
|
98
|
+
}
|
|
99
|
+
const suite = parsed.suite ?? "all";
|
|
100
|
+
if (!isSuiteName(suite)) {
|
|
101
|
+
return { usageError: `Error: unknown suite "${suite}".\n` };
|
|
102
|
+
}
|
|
103
|
+
return { fixtures: fixturesForSuite(suite) };
|
|
104
|
+
}
|
|
105
|
+
// In live mode, deep-redact the scorecard before serialization so that any model content that
|
|
106
|
+
// leaked into workflow report fields (e.g. fixture reasons) is scrubbed by the same audit
|
|
107
|
+
// redactor applied at evidence-persist time. Offline scorecard is static harness text — safe as-is.
|
|
108
|
+
function redactedScorecard(scorecard, live, env) {
|
|
109
|
+
if (!live) {
|
|
110
|
+
return scorecard;
|
|
111
|
+
}
|
|
112
|
+
const redactFn = createAuditRedactor({ additionalSecrets: keikoApiKeySecretValues(env) }, env);
|
|
113
|
+
return deepRedactStrings(scorecard, redactFn);
|
|
114
|
+
}
|
|
115
|
+
function writeScorecard(path, output) {
|
|
116
|
+
writeFileSync(path, `${JSON.stringify(output, null, 2)}\n`, { encoding: "utf8", flag: "wx" });
|
|
117
|
+
}
|
|
118
|
+
function emit(scorecard, parsed, io, env) {
|
|
119
|
+
const output = redactedScorecard(scorecard, parsed.live, env);
|
|
120
|
+
// Emit --json to stdout BEFORE attempting the file write so that a file-already-exists
|
|
121
|
+
// error (EEXIST from writeScorecard) does not silently suppress the JSON output.
|
|
122
|
+
if (parsed.json) {
|
|
123
|
+
io.out(`${JSON.stringify(output, null, 2)}\n`);
|
|
124
|
+
}
|
|
125
|
+
if (parsed.output !== undefined) {
|
|
126
|
+
writeScorecard(parsed.output, output);
|
|
127
|
+
}
|
|
128
|
+
if (!parsed.json) {
|
|
129
|
+
io.out(`${renderEvalSummary(scorecard)}\n`);
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
// Exit 0 only when every scored dimension passed (zero failures) AND surface parity passed.
|
|
133
|
+
function exitCodeFor(scorecard) {
|
|
134
|
+
if (!scorecard.surfaceParity.allPassed) {
|
|
135
|
+
return 1;
|
|
136
|
+
}
|
|
137
|
+
return scorecard.dimensions.some((d) => d.failCount > 0) ? 1 : 0;
|
|
138
|
+
}
|
|
139
|
+
export async function runEvaluateCli(args, io, env = {}, deps = {}) {
|
|
140
|
+
if (args.includes("--help")) {
|
|
141
|
+
io.out(USAGE);
|
|
142
|
+
return 0;
|
|
143
|
+
}
|
|
144
|
+
const usageError = findUsageError(args);
|
|
145
|
+
if (usageError !== undefined) {
|
|
146
|
+
io.err(`Error: ${usageError}.\n${USAGE}`);
|
|
147
|
+
return 2;
|
|
148
|
+
}
|
|
149
|
+
const parsed = parseArgs(args);
|
|
150
|
+
if (parsed === null) {
|
|
151
|
+
io.err(USAGE);
|
|
152
|
+
return 2;
|
|
153
|
+
}
|
|
154
|
+
const selection = selectFixtures(parsed);
|
|
155
|
+
if ("usageError" in selection) {
|
|
156
|
+
io.err(selection.usageError);
|
|
157
|
+
return 2;
|
|
158
|
+
}
|
|
159
|
+
return runSuite(parsed, selection.fixtures, io, env, deps);
|
|
160
|
+
}
|
|
161
|
+
async function runSuite(parsed, fixtures, io, env, deps) {
|
|
162
|
+
try {
|
|
163
|
+
const liveModelId = resolveLiveModelId(parsed, io, env);
|
|
164
|
+
if (typeof liveModelId === "number") {
|
|
165
|
+
return liveModelId;
|
|
166
|
+
}
|
|
167
|
+
const scorecard = await runEvaluationSuite({
|
|
168
|
+
mode: parsed.live ? "live" : "offline",
|
|
169
|
+
fixtures,
|
|
170
|
+
...(liveModelId === undefined ? {} : { modelIdOverride: liveModelId }),
|
|
171
|
+
...(parsed.config === undefined ? {} : { configPath: parsed.config }),
|
|
172
|
+
},
|
|
173
|
+
// Provide Date.now as the default wall-clock so a real `keiko evaluate` prints the actual
|
|
174
|
+
// current time. Tests override this via deps.runner.now for deterministic evaluatedAt.
|
|
175
|
+
{
|
|
176
|
+
env,
|
|
177
|
+
now: Date.now,
|
|
178
|
+
configLoader: loadGatewayConfigFromFile,
|
|
179
|
+
surfaceParity: {
|
|
180
|
+
runGenTestsCli,
|
|
181
|
+
runInvestigateCli,
|
|
182
|
+
parseRunRequest,
|
|
183
|
+
},
|
|
184
|
+
...deps.runner,
|
|
185
|
+
});
|
|
186
|
+
emit(scorecard, parsed, io, env);
|
|
187
|
+
return exitCodeFor(scorecard);
|
|
188
|
+
}
|
|
189
|
+
catch (error) {
|
|
190
|
+
if (isOutputAlreadyExistsError(error)) {
|
|
191
|
+
io.err(`Error: output file already exists: ${parsed.output ?? "<unknown>"}\n`);
|
|
192
|
+
return 1;
|
|
193
|
+
}
|
|
194
|
+
return handleRunError(error, parsed, io);
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
function resolveLiveModelId(parsed, io, env) {
|
|
198
|
+
if (!parsed.live) {
|
|
199
|
+
return parsed.model;
|
|
200
|
+
}
|
|
201
|
+
try {
|
|
202
|
+
const path = parsed.config ?? env.KEIKO_CONFIG_FILE;
|
|
203
|
+
if (path === undefined) {
|
|
204
|
+
throw new ConfigInvalidError("no config source; pass --config PATH or set KEIKO_CONFIG_FILE");
|
|
205
|
+
}
|
|
206
|
+
const config = loadGatewayConfigFromFile(path, env);
|
|
207
|
+
if (parsed.model !== undefined) {
|
|
208
|
+
assertLiveEvaluationModel(config, parsed.model);
|
|
209
|
+
return parsed.model;
|
|
210
|
+
}
|
|
211
|
+
const modelId = selectLiveEvaluationModel(config);
|
|
212
|
+
if (modelId === undefined) {
|
|
213
|
+
io.err("Error: no configured workflow-capable chat model is available.\n");
|
|
214
|
+
return 1;
|
|
215
|
+
}
|
|
216
|
+
return modelId;
|
|
217
|
+
}
|
|
218
|
+
catch (error) {
|
|
219
|
+
if (error instanceof GatewayError) {
|
|
220
|
+
io.err(`Error: model gateway configuration problem — ${redact(error.message)}\n` +
|
|
221
|
+
`Provide a gateway config with --config PATH or KEIKO_CONFIG_FILE.\n`);
|
|
222
|
+
return 1;
|
|
223
|
+
}
|
|
224
|
+
throw error;
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
function isLiveEvaluationCapable(capability) {
|
|
228
|
+
return (capability?.kind === "chat" &&
|
|
229
|
+
capability.workflowEligible &&
|
|
230
|
+
capability.toolCalling &&
|
|
231
|
+
capability.structuredOutput);
|
|
232
|
+
}
|
|
233
|
+
const COST_RANK = { low: 0, medium: 1, high: 2 };
|
|
234
|
+
function selectLiveEvaluationModel(config) {
|
|
235
|
+
let best;
|
|
236
|
+
for (const capability of listConfiguredCapabilities(config)) {
|
|
237
|
+
if (!isLiveEvaluationCapable(capability)) {
|
|
238
|
+
continue;
|
|
239
|
+
}
|
|
240
|
+
if (best === undefined || COST_RANK[capability.costClass] < COST_RANK[best.costClass]) {
|
|
241
|
+
best = capability;
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
return best?.id;
|
|
245
|
+
}
|
|
246
|
+
function assertLiveEvaluationModel(config, modelId) {
|
|
247
|
+
assertConfiguredModel(config, modelId);
|
|
248
|
+
if (!isLiveEvaluationCapable(findConfiguredCapability(config, modelId))) {
|
|
249
|
+
throw new ConfigInvalidError(`model '${modelId}' is not workflow-capable; live evaluation requires chat + tool-calling + structured-output`);
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
function isOutputAlreadyExistsError(error) {
|
|
253
|
+
return (typeof error === "object" &&
|
|
254
|
+
error !== null &&
|
|
255
|
+
"code" in error &&
|
|
256
|
+
error.code === "EEXIST");
|
|
257
|
+
}
|
|
258
|
+
// Live-mode fail-closed: a GatewayError (incl. ConfigInvalidError) means no resolvable config or
|
|
259
|
+
// credentials. Name the required env vars and exit 1 — never fall back to offline silently.
|
|
260
|
+
function handleRunError(error, parsed, io) {
|
|
261
|
+
if (error instanceof GatewayError) {
|
|
262
|
+
io.err(`Error: model gateway configuration problem — ${redact(error.message)}\n` +
|
|
263
|
+
(parsed.live
|
|
264
|
+
? "Live evaluation requires a configured provider. Pass --config PATH or set " +
|
|
265
|
+
"KEIKO_CONFIG_FILE.\n"
|
|
266
|
+
: ""));
|
|
267
|
+
return 1;
|
|
268
|
+
}
|
|
269
|
+
throw error;
|
|
270
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { type EvidenceStore } from "@oscharko-dev/keiko-evidence";
|
|
2
|
+
import type { EnvSource } from "@oscharko-dev/keiko-model-gateway";
|
|
3
|
+
import type { CliIo } from "./runner.js";
|
|
4
|
+
export interface EvidenceCliDeps {
|
|
5
|
+
readonly store?: EvidenceStore | undefined;
|
|
6
|
+
readonly env?: EnvSource | undefined;
|
|
7
|
+
}
|
|
8
|
+
export declare function runEvidenceCli(args: readonly string[], io: CliIo, deps?: EvidenceCliDeps): number;
|
|
9
|
+
//# sourceMappingURL=evidence.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"evidence.d.ts","sourceRoot":"","sources":["../src/evidence.ts"],"names":[],"mappings":"AAQA,OAAO,EAQL,KAAK,aAAa,EAGnB,MAAM,8BAA8B,CAAC;AACtC,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mCAAmC,CAAC;AACnE,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AAWzC,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,KAAK,CAAC,EAAE,aAAa,GAAG,SAAS,CAAC;IAC3C,QAAQ,CAAC,GAAG,CAAC,EAAE,SAAS,GAAG,SAAS,CAAC;CACtC;AAmID,wBAAgB,cAAc,CAC5B,IAAI,EAAE,SAAS,MAAM,EAAE,EACvB,EAAE,EAAE,KAAK,EACT,IAAI,GAAE,eAAoB,GACzB,MAAM,CA2BR"}
|
package/dist/evidence.js
ADDED
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
// `keiko evidence` — inspects previously written evidence manifests (ADR-0010 D9). `list` prints the
|
|
2
|
+
// EvidenceListEntry[] (text or --json); `show <runId>` prints one EvidenceReport / full manifest
|
|
3
|
+
// (--json). It reads ONLY the contained base dir via the EvidenceStore (default $KEIKO_EVIDENCE_DIR
|
|
4
|
+
// or ./.keiko/evidence, overridable with --evidence-dir). Because manifests are redacted by
|
|
5
|
+
// construction there is no un-redaction path. Exit 0 on success, 1 on a missing runId / read error,
|
|
6
|
+
// 2 on usage (unknown or missing subcommand, invalid runId). Tests inject an in-memory store via deps
|
|
7
|
+
// so no disk is touched.
|
|
8
|
+
import { buildEvidenceReport, renderEvidenceReport, listEvidence, loadEvidence, createNodeEvidenceStore, resolveEvidenceDir, AuditError, InvalidRunIdError, } from "@oscharko-dev/keiko-evidence";
|
|
9
|
+
const USAGE = `Usage:
|
|
10
|
+
keiko evidence list [--evidence-dir PATH] [--json]
|
|
11
|
+
keiko evidence show <runId> [--evidence-dir PATH] [--json]
|
|
12
|
+
|
|
13
|
+
Lists or shows redacted evidence manifests written by \`keiko run\`. Reads only the
|
|
14
|
+
evidence base dir (default $KEIKO_EVIDENCE_DIR or ./.keiko/evidence; override with --evidence-dir).
|
|
15
|
+
`;
|
|
16
|
+
function parseSubcommand(value) {
|
|
17
|
+
return value === "list" || value === "show" ? value : undefined;
|
|
18
|
+
}
|
|
19
|
+
function parseFlags(args, startIndex) {
|
|
20
|
+
let json = false;
|
|
21
|
+
let evidenceDir;
|
|
22
|
+
let index = startIndex;
|
|
23
|
+
while (index < args.length) {
|
|
24
|
+
const arg = args[index];
|
|
25
|
+
if (arg === "--json") {
|
|
26
|
+
json = true;
|
|
27
|
+
index += 1;
|
|
28
|
+
continue;
|
|
29
|
+
}
|
|
30
|
+
if (arg !== "--evidence-dir") {
|
|
31
|
+
return { ok: false };
|
|
32
|
+
}
|
|
33
|
+
const value = args[index + 1];
|
|
34
|
+
if (value === undefined || value.startsWith("--")) {
|
|
35
|
+
return { ok: false };
|
|
36
|
+
}
|
|
37
|
+
evidenceDir = value;
|
|
38
|
+
index += 2;
|
|
39
|
+
}
|
|
40
|
+
return { ok: true, json, evidenceDir };
|
|
41
|
+
}
|
|
42
|
+
function invalidArgs(message) {
|
|
43
|
+
return { ok: false, error: message };
|
|
44
|
+
}
|
|
45
|
+
function resolveStore(evidenceDir, deps) {
|
|
46
|
+
if (deps.store !== undefined) {
|
|
47
|
+
return deps.store;
|
|
48
|
+
}
|
|
49
|
+
return createNodeEvidenceStore(resolveEvidenceDir(evidenceDir, deps.env));
|
|
50
|
+
}
|
|
51
|
+
function parseEvidenceArgs(args) {
|
|
52
|
+
const subcommand = parseSubcommand(args[0]);
|
|
53
|
+
if (subcommand === undefined) {
|
|
54
|
+
const sub = args[0];
|
|
55
|
+
return invalidArgs(sub === undefined ? USAGE : `keiko evidence: unknown subcommand: ${sub}\n${USAGE}`);
|
|
56
|
+
}
|
|
57
|
+
let index = 1;
|
|
58
|
+
let runId;
|
|
59
|
+
if (subcommand === "show") {
|
|
60
|
+
runId = args[index];
|
|
61
|
+
if (runId === undefined || runId.startsWith("--")) {
|
|
62
|
+
return invalidArgs(`keiko evidence: show requires a <runId>.\n${USAGE}`);
|
|
63
|
+
}
|
|
64
|
+
index += 1;
|
|
65
|
+
}
|
|
66
|
+
const flags = parseFlags(args, index);
|
|
67
|
+
if (!flags.ok) {
|
|
68
|
+
return invalidArgs(`keiko evidence: invalid arguments.\n${USAGE}`);
|
|
69
|
+
}
|
|
70
|
+
return {
|
|
71
|
+
ok: true,
|
|
72
|
+
error: "",
|
|
73
|
+
parsed: { subcommand, runId, evidenceDir: flags.evidenceDir, json: flags.json === true },
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
function renderListText(entries) {
|
|
77
|
+
if (entries.length === 0) {
|
|
78
|
+
return "No evidence manifests found.\n";
|
|
79
|
+
}
|
|
80
|
+
const rows = entries.map((e) => `${e.runId} ${e.taskType} ${e.outcome} started=${String(e.startedAt)} finished=${String(e.finishedAt)}`);
|
|
81
|
+
return `${rows.join("\n")}\n`;
|
|
82
|
+
}
|
|
83
|
+
function runList(store, json, io) {
|
|
84
|
+
const entries = listEvidence(store);
|
|
85
|
+
io.out(json ? `${JSON.stringify(entries, null, 2)}\n` : renderListText(entries));
|
|
86
|
+
return 0;
|
|
87
|
+
}
|
|
88
|
+
function runShow(store, runId, json, io) {
|
|
89
|
+
const manifest = loadEvidence(store, runId);
|
|
90
|
+
if (manifest === undefined) {
|
|
91
|
+
io.err(`keiko evidence: no manifest for runId: ${runId}\n`);
|
|
92
|
+
return 1;
|
|
93
|
+
}
|
|
94
|
+
if (json) {
|
|
95
|
+
io.out(`${JSON.stringify(manifest, null, 2)}\n`);
|
|
96
|
+
return 0;
|
|
97
|
+
}
|
|
98
|
+
io.out(renderEvidenceReport(buildEvidenceReport(manifest, store.location?.(runId) ?? `${runId}.json`)));
|
|
99
|
+
return 0;
|
|
100
|
+
}
|
|
101
|
+
// Maps a thrown AuditError to an exit code: an invalid runId is a usage error (2), any other
|
|
102
|
+
// audit/read failure is a runtime error (1). Messages are already redacted at construction.
|
|
103
|
+
function exitForAuditError(error, io) {
|
|
104
|
+
io.err(`keiko evidence: ${error.message}\n`);
|
|
105
|
+
return error instanceof InvalidRunIdError ? 2 : 1;
|
|
106
|
+
}
|
|
107
|
+
export function runEvidenceCli(args, io, deps = {}) {
|
|
108
|
+
const parsed = parseEvidenceArgs(args);
|
|
109
|
+
if (!parsed.ok || parsed.parsed === undefined) {
|
|
110
|
+
io.err(parsed.error);
|
|
111
|
+
return 2;
|
|
112
|
+
}
|
|
113
|
+
try {
|
|
114
|
+
if (parsed.parsed.subcommand === "list") {
|
|
115
|
+
return runList(resolveStore(parsed.parsed.evidenceDir, deps), parsed.parsed.json, io);
|
|
116
|
+
}
|
|
117
|
+
if (parsed.parsed.runId === undefined) {
|
|
118
|
+
io.err(`keiko evidence: show requires a <runId>.\n${USAGE}`);
|
|
119
|
+
return 2;
|
|
120
|
+
}
|
|
121
|
+
return runShow(resolveStore(parsed.parsed.evidenceDir, deps), parsed.parsed.runId, parsed.parsed.json, io);
|
|
122
|
+
}
|
|
123
|
+
catch (error) {
|
|
124
|
+
if (error instanceof AuditError) {
|
|
125
|
+
return exitForAuditError(error, io);
|
|
126
|
+
}
|
|
127
|
+
throw error;
|
|
128
|
+
}
|
|
129
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { type EnvSource, type GatewayConfig } from "@oscharko-dev/keiko-model-gateway";
|
|
2
|
+
export type ConfigPathResolution = {
|
|
3
|
+
readonly kind: "path";
|
|
4
|
+
readonly path: string;
|
|
5
|
+
} | {
|
|
6
|
+
readonly kind: "missing-value";
|
|
7
|
+
} | {
|
|
8
|
+
readonly kind: "not-configured";
|
|
9
|
+
};
|
|
10
|
+
export declare function resolveConfigPathFromArgs(args: readonly string[], env: EnvSource): ConfigPathResolution;
|
|
11
|
+
export declare function loadGatewayConfigFromFile(path: string, env: EnvSource): GatewayConfig;
|
|
12
|
+
//# sourceMappingURL=gateway-config.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"gateway-config.d.ts","sourceRoot":"","sources":["../src/gateway-config.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,KAAK,SAAS,EACd,KAAK,aAAa,EACnB,MAAM,mCAAmC,CAAC;AAG3C,MAAM,MAAM,oBAAoB,GAC5B;IAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAChD;IAAE,QAAQ,CAAC,IAAI,EAAE,eAAe,CAAA;CAAE,GAClC;IAAE,QAAQ,CAAC,IAAI,EAAE,gBAAgB,CAAA;CAAE,CAAC;AAExC,wBAAgB,yBAAyB,CACvC,IAAI,EAAE,SAAS,MAAM,EAAE,EACvB,GAAG,EAAE,SAAS,GACb,oBAAoB,CAWtB;AAED,wBAAgB,yBAAyB,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,SAAS,GAAG,aAAa,CAIrF"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { loadConfigFromFile, } from "@oscharko-dev/keiko-model-gateway";
|
|
2
|
+
import { createProviderSecretResolver } from "@oscharko-dev/keiko-server/credential-vault";
|
|
3
|
+
export function resolveConfigPathFromArgs(args, env) {
|
|
4
|
+
const flagIndex = args.indexOf("--config");
|
|
5
|
+
if (flagIndex !== -1) {
|
|
6
|
+
const value = args[flagIndex + 1];
|
|
7
|
+
return value === undefined || value.startsWith("--")
|
|
8
|
+
? { kind: "missing-value" }
|
|
9
|
+
: { kind: "path", path: value };
|
|
10
|
+
}
|
|
11
|
+
return env.KEIKO_CONFIG_FILE === undefined
|
|
12
|
+
? { kind: "not-configured" }
|
|
13
|
+
: { kind: "path", path: env.KEIKO_CONFIG_FILE };
|
|
14
|
+
}
|
|
15
|
+
export function loadGatewayConfigFromFile(path, env) {
|
|
16
|
+
return loadConfigFromFile(path, env, {
|
|
17
|
+
secretResolver: createProviderSecretResolver({ configPath: path, env }),
|
|
18
|
+
});
|
|
19
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { type EnvSource } from "@oscharko-dev/keiko-model-gateway";
|
|
2
|
+
import { type ModelPort } from "@oscharko-dev/keiko-harness";
|
|
3
|
+
import type { CliIo } from "./runner.js";
|
|
4
|
+
export interface GenTestsDeps {
|
|
5
|
+
readonly model?: ModelPort | undefined;
|
|
6
|
+
}
|
|
7
|
+
export declare function runGenTestsCli(args: readonly string[], io: CliIo, env?: EnvSource, deps?: GenTestsDeps): Promise<number>;
|
|
8
|
+
//# sourceMappingURL=gen-tests.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"gen-tests.d.ts","sourceRoot":"","sources":["../src/gen-tests.ts"],"names":[],"mappings":"AAQA,OAAO,EAEL,KAAK,SAAS,EAMf,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAAoB,KAAK,SAAS,EAAE,MAAM,6BAA6B,CAAC;AAK/E,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AAWzC,MAAM,WAAW,YAAY;IAE3B,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,GAAG,SAAS,CAAC;CACxC;AAkMD,wBAAsB,cAAc,CAClC,IAAI,EAAE,SAAS,MAAM,EAAE,EACvB,EAAE,EAAE,KAAK,EACT,GAAG,GAAE,SAAc,EACnB,IAAI,GAAE,YAAiB,GACtB,OAAO,CAAC,MAAM,CAAC,CAuCjB"}
|