@relayflows/sdk 2.0.42 → 2.0.43
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/base.d.ts +12 -1
- package/dist/adapters/base.d.ts.map +1 -1
- package/dist/adapters/base.js +11 -1
- package/dist/adapters/base.js.map +1 -1
- package/dist/adapters/claude.d.ts.map +1 -1
- package/dist/adapters/claude.js +8 -4
- package/dist/adapters/claude.js.map +1 -1
- package/dist/adapters/codex.d.ts.map +1 -1
- package/dist/adapters/codex.js +15 -6
- package/dist/adapters/codex.js.map +1 -1
- package/dist/agent-resume.d.ts +8 -0
- package/dist/agent-resume.d.ts.map +1 -0
- package/dist/agent-resume.js +37 -0
- package/dist/agent-resume.js.map +1 -0
- package/dist/agent-transcript.d.ts.map +1 -1
- package/dist/agent-transcript.js +5 -3
- package/dist/agent-transcript.js.map +1 -1
- package/dist/authored-flow-error.d.ts +10 -2
- package/dist/authored-flow-error.d.ts.map +1 -1
- package/dist/authored-flow-error.js.map +1 -1
- package/dist/authored-flow-executor.d.ts.map +1 -1
- package/dist/authored-flow-executor.js +16 -2
- package/dist/authored-flow-executor.js.map +1 -1
- package/dist/authored-flow-loader.d.ts +6 -0
- package/dist/authored-flow-loader.d.ts.map +1 -1
- package/dist/authored-flow-loader.js +18 -6
- package/dist/authored-flow-loader.js.map +1 -1
- package/dist/authored-helper-effect.js +3 -2
- package/dist/authored-helper-effect.js.map +1 -1
- package/dist/authored-human.d.ts +2 -0
- package/dist/authored-human.d.ts.map +1 -1
- package/dist/authored-human.js +4 -0
- package/dist/authored-human.js.map +1 -1
- package/dist/authored-memory.d.ts +47 -2
- package/dist/authored-memory.d.ts.map +1 -1
- package/dist/authored-memory.js +394 -15
- package/dist/authored-memory.js.map +1 -1
- package/dist/authored-node-entry.js +6 -1
- package/dist/authored-node-entry.js.map +1 -1
- package/dist/authored-node-runner.d.ts +1 -1
- package/dist/authored-node-runner.d.ts.map +1 -1
- package/dist/authored-node-runner.js +6 -3
- package/dist/authored-node-runner.js.map +1 -1
- package/dist/authored-root-readback.d.ts +21 -0
- package/dist/authored-root-readback.d.ts.map +1 -0
- package/dist/authored-root-readback.js +90 -0
- package/dist/authored-root-readback.js.map +1 -0
- package/dist/authored-root.d.ts.map +1 -1
- package/dist/authored-root.js +53 -74
- package/dist/authored-root.js.map +1 -1
- package/dist/authored-step-output.d.ts +2 -0
- package/dist/authored-step-output.d.ts.map +1 -1
- package/dist/authored-step-output.js +1 -0
- package/dist/authored-step-output.js.map +1 -1
- package/dist/authored-worker-step.d.ts.map +1 -1
- package/dist/authored-worker-step.js +26 -4
- package/dist/authored-worker-step.js.map +1 -1
- package/dist/cli/check-helper-body.d.ts +2 -2
- package/dist/cli/check-helper-body.d.ts.map +1 -1
- package/dist/cli/check-helper-body.js +7 -2
- package/dist/cli/check-helper-body.js.map +1 -1
- package/dist/cli/check-helper-surface.d.ts +8 -0
- package/dist/cli/check-helper-surface.d.ts.map +1 -0
- package/dist/cli/check-helper-surface.js +37 -0
- package/dist/cli/check-helper-surface.js.map +1 -0
- package/dist/cli/check-triggers.d.ts +4 -3
- package/dist/cli/check-triggers.d.ts.map +1 -1
- package/dist/cli/check-triggers.js +7 -3
- package/dist/cli/check-triggers.js.map +1 -1
- package/dist/cli/check.d.ts +7 -1
- package/dist/cli/check.d.ts.map +1 -1
- package/dist/cli/check.js +35 -8
- package/dist/cli/check.js.map +1 -1
- package/dist/cli/cloud-answer.d.ts +9 -0
- package/dist/cli/cloud-answer.d.ts.map +1 -0
- package/dist/cli/cloud-answer.js +39 -0
- package/dist/cli/cloud-answer.js.map +1 -0
- package/dist/cli/cloud-connect-cli.d.ts +2 -1
- package/dist/cli/cloud-connect-cli.d.ts.map +1 -1
- package/dist/cli/cloud-connect-cli.js +10 -2
- package/dist/cli/cloud-connect-cli.js.map +1 -1
- package/dist/cli/cloud-human.d.ts +7 -0
- package/dist/cli/cloud-human.d.ts.map +1 -0
- package/dist/cli/cloud-human.js +22 -0
- package/dist/cli/cloud-human.js.map +1 -0
- package/dist/cli/cloud-live.d.ts +2 -2
- package/dist/cli/cloud-live.d.ts.map +1 -1
- package/dist/cli/cloud-live.js +9 -2
- package/dist/cli/cloud-live.js.map +1 -1
- package/dist/cli/cloud-mirror-session.d.ts.map +1 -1
- package/dist/cli/cloud-mirror-session.js +3 -2
- package/dist/cli/cloud-mirror-session.js.map +1 -1
- package/dist/cli/cloud-run.d.ts +1 -1
- package/dist/cli/cloud-run.d.ts.map +1 -1
- package/dist/cli/cloud-run.js +12 -0
- package/dist/cli/cloud-run.js.map +1 -1
- package/dist/cli/detached-record.d.ts +22 -0
- package/dist/cli/detached-record.d.ts.map +1 -0
- package/dist/cli/detached-record.js +57 -0
- package/dist/cli/detached-record.js.map +1 -0
- package/dist/cli/detached-run.d.ts +28 -0
- package/dist/cli/detached-run.d.ts.map +1 -0
- package/dist/cli/detached-run.js +122 -0
- package/dist/cli/detached-run.js.map +1 -0
- package/dist/cli/diagnostic-fold.d.ts +19 -0
- package/dist/cli/diagnostic-fold.d.ts.map +1 -0
- package/dist/cli/diagnostic-fold.js +19 -0
- package/dist/cli/diagnostic-fold.js.map +1 -0
- package/dist/cli/direct-run.d.ts.map +1 -1
- package/dist/cli/direct-run.js +16 -2
- package/dist/cli/direct-run.js.map +1 -1
- package/dist/cli/eval.d.ts +22 -0
- package/dist/cli/eval.d.ts.map +1 -0
- package/dist/cli/eval.js +131 -0
- package/dist/cli/eval.js.map +1 -0
- package/dist/cli/journal-timeout.d.ts +12 -0
- package/dist/cli/journal-timeout.d.ts.map +1 -0
- package/dist/cli/journal-timeout.js +42 -0
- package/dist/cli/journal-timeout.js.map +1 -0
- package/dist/cli/local-agent-remedy.d.ts.map +1 -1
- package/dist/cli/local-agent-remedy.js +5 -4
- package/dist/cli/local-agent-remedy.js.map +1 -1
- package/dist/cli/observer-session.d.ts +1 -0
- package/dist/cli/observer-session.d.ts.map +1 -1
- package/dist/cli/observer-session.js +8 -1
- package/dist/cli/observer-session.js.map +1 -1
- package/dist/cli/reuse.d.ts +8 -0
- package/dist/cli/reuse.d.ts.map +1 -1
- package/dist/cli/reuse.js +24 -0
- package/dist/cli/reuse.js.map +1 -1
- package/dist/cli/run.d.ts +8 -2
- package/dist/cli/run.d.ts.map +1 -1
- package/dist/cli/run.js +72 -56
- package/dist/cli/run.js.map +1 -1
- package/dist/cli/running-step.d.ts +9 -0
- package/dist/cli/running-step.d.ts.map +1 -0
- package/dist/cli/running-step.js +156 -0
- package/dist/cli/running-step.js.map +1 -0
- package/dist/cli-adapter.d.ts +3 -1
- package/dist/cli-adapter.d.ts.map +1 -1
- package/dist/cli-adapter.js +6 -2
- package/dist/cli-adapter.js.map +1 -1
- package/dist/cli-commands.d.ts +77 -11
- package/dist/cli-commands.d.ts.map +1 -1
- package/dist/cli-commands.js +41 -7
- package/dist/cli-commands.js.map +1 -1
- package/dist/cli-watch.d.ts +6 -2
- package/dist/cli-watch.d.ts.map +1 -1
- package/dist/cli-watch.js +7 -6
- package/dist/cli-watch.js.map +1 -1
- package/dist/cli.d.ts +22 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +154 -50
- package/dist/cli.js.map +1 -1
- package/dist/cloud-answer.d.ts +35 -0
- package/dist/cloud-answer.d.ts.map +1 -0
- package/dist/cloud-answer.js +194 -0
- package/dist/cloud-answer.js.map +1 -0
- package/dist/cloud-auth-store.d.ts +46 -0
- package/dist/cloud-auth-store.d.ts.map +1 -0
- package/dist/cloud-auth-store.js +171 -0
- package/dist/cloud-auth-store.js.map +1 -0
- package/dist/cloud-http.d.ts +7 -16
- package/dist/cloud-http.d.ts.map +1 -1
- package/dist/cloud-http.js +59 -40
- package/dist/cloud-http.js.map +1 -1
- package/dist/cloud-human-wait.d.ts +10 -0
- package/dist/cloud-human-wait.d.ts.map +1 -0
- package/dist/cloud-human-wait.js +17 -0
- package/dist/cloud-human-wait.js.map +1 -0
- package/dist/cloud-mirror-transport.d.ts.map +1 -1
- package/dist/cloud-mirror-transport.js +1 -0
- package/dist/cloud-mirror-transport.js.map +1 -1
- package/dist/cloud-run-record.d.ts +7 -2
- package/dist/cloud-run-record.d.ts.map +1 -1
- package/dist/cloud-run-record.js +13 -5
- package/dist/cloud-run-record.js.map +1 -1
- package/dist/cloud-run.d.ts +9 -0
- package/dist/cloud-run.d.ts.map +1 -1
- package/dist/cloud-run.js +36 -2
- package/dist/cloud-run.js.map +1 -1
- package/dist/communication/local.d.ts.map +1 -1
- package/dist/communication/local.js +6 -1
- package/dist/communication/local.js.map +1 -1
- package/dist/communication/worker.d.ts.map +1 -1
- package/dist/communication/worker.js +12 -2
- package/dist/communication/worker.js.map +1 -1
- package/dist/compile.d.ts.map +1 -1
- package/dist/compile.js +5 -3
- package/dist/compile.js.map +1 -1
- package/dist/daemon-spec-validation.d.ts +31 -0
- package/dist/daemon-spec-validation.d.ts.map +1 -0
- package/dist/daemon-spec-validation.js +115 -0
- package/dist/daemon-spec-validation.js.map +1 -0
- package/dist/failure-kinds.d.ts +6 -4
- package/dist/failure-kinds.d.ts.map +1 -1
- package/dist/failure-kinds.js +17 -1
- package/dist/failure-kinds.js.map +1 -1
- package/dist/flow-eval-cloud.d.ts +25 -0
- package/dist/flow-eval-cloud.d.ts.map +1 -0
- package/dist/flow-eval-cloud.js +108 -0
- package/dist/flow-eval-cloud.js.map +1 -0
- package/dist/flow-eval-error.d.ts +7 -0
- package/dist/flow-eval-error.d.ts.map +1 -0
- package/dist/flow-eval-error.js +10 -0
- package/dist/flow-eval-error.js.map +1 -0
- package/dist/flow-eval-gate.d.ts +15 -0
- package/dist/flow-eval-gate.d.ts.map +1 -0
- package/dist/flow-eval-gate.js +102 -0
- package/dist/flow-eval-gate.js.map +1 -0
- package/dist/flow-eval-local.d.ts +54 -0
- package/dist/flow-eval-local.d.ts.map +1 -0
- package/dist/flow-eval-local.js +220 -0
- package/dist/flow-eval-local.js.map +1 -0
- package/dist/flow-eval-outcome.d.ts +22 -0
- package/dist/flow-eval-outcome.d.ts.map +1 -0
- package/dist/flow-eval-outcome.js +37 -0
- package/dist/flow-eval-outcome.js.map +1 -0
- package/dist/flow-eval-report.d.ts +106 -0
- package/dist/flow-eval-report.d.ts.map +1 -0
- package/dist/flow-eval-report.js +4 -0
- package/dist/flow-eval-report.js.map +1 -0
- package/dist/flow-eval-suite.d.ts +46 -0
- package/dist/flow-eval-suite.d.ts.map +1 -0
- package/dist/flow-eval-suite.js +144 -0
- package/dist/flow-eval-suite.js.map +1 -0
- package/dist/flow-eval-version.d.ts +59 -0
- package/dist/flow-eval-version.d.ts.map +1 -0
- package/dist/flow-eval-version.js +303 -0
- package/dist/flow-eval-version.js.map +1 -0
- package/dist/flow-eval.d.ts +41 -0
- package/dist/flow-eval.d.ts.map +1 -0
- package/dist/flow-eval.js +201 -0
- package/dist/flow-eval.js.map +1 -0
- package/dist/helper-operation-use.d.ts +18 -0
- package/dist/helper-operation-use.d.ts.map +1 -0
- package/dist/helper-operation-use.js +336 -0
- package/dist/helper-operation-use.js.map +1 -0
- package/dist/helper-preflight.d.ts.map +1 -1
- package/dist/helper-preflight.js +16 -2
- package/dist/helper-preflight.js.map +1 -1
- package/dist/helper-reference.d.ts +33 -0
- package/dist/helper-reference.d.ts.map +1 -1
- package/dist/helper-reference.js +5 -5
- package/dist/helper-reference.js.map +1 -1
- package/dist/index.d.ts +6 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -1
- package/dist/index.js.map +1 -1
- package/dist/journal-budgeted-reads.d.ts +41 -0
- package/dist/journal-budgeted-reads.d.ts.map +1 -0
- package/dist/journal-budgeted-reads.js +142 -0
- package/dist/journal-budgeted-reads.js.map +1 -0
- package/dist/journal-client.d.ts +34 -29
- package/dist/journal-client.d.ts.map +1 -1
- package/dist/journal-client.js +45 -151
- package/dist/journal-client.js.map +1 -1
- package/dist/journal-connection.d.ts +42 -0
- package/dist/journal-connection.d.ts.map +1 -0
- package/dist/journal-connection.js +209 -0
- package/dist/journal-connection.js.map +1 -0
- package/dist/journal-read-policy.d.ts +38 -0
- package/dist/journal-read-policy.d.ts.map +1 -0
- package/dist/journal-read-policy.js +126 -0
- package/dist/journal-read-policy.js.map +1 -0
- package/dist/named-gate-lowering.d.ts.map +1 -1
- package/dist/named-gate-lowering.js +2 -6
- package/dist/named-gate-lowering.js.map +1 -1
- package/dist/re2js-embedded.d.ts +3 -0
- package/dist/re2js-embedded.d.ts.map +1 -0
- package/dist/re2js-embedded.js +4 -0
- package/dist/re2js-embedded.js.map +1 -0
- package/dist/spec.d.ts +10 -0
- package/dist/spec.d.ts.map +1 -1
- package/dist/spec.js.map +1 -1
- package/dist/step-fields.d.ts +1 -1
- package/dist/step-fields.js +1 -1
- package/dist/step-fields.js.map +1 -1
- package/dist/validate.d.ts.map +1 -1
- package/dist/validate.js +15 -0
- package/dist/validate.js.map +1 -1
- package/dist/worker-cli.d.ts +3 -1
- package/dist/worker-cli.d.ts.map +1 -1
- package/dist/worker-cli.js +15 -5
- package/dist/worker-cli.js.map +1 -1
- package/dist/worker-lease.d.ts.map +1 -1
- package/dist/worker-lease.js +11 -3
- package/dist/worker-lease.js.map +1 -1
- package/dist/worker.js +1 -1
- package/dist/worker.js.map +1 -1
- package/package.json +5 -3
- package/src/adapters/base.ts +16 -1
- package/src/adapters/claude.ts +8 -4
- package/src/adapters/codex.ts +15 -6
- package/src/agent-resume.ts +36 -0
- package/src/agent-transcript.ts +5 -3
- package/src/authored-flow-error.ts +9 -0
- package/src/authored-flow-executor.ts +18 -1
- package/src/authored-flow-loader.ts +27 -10
- package/src/authored-helper-effect.ts +3 -2
- package/src/authored-human.ts +5 -0
- package/src/authored-memory.ts +445 -15
- package/src/authored-node-entry.ts +6 -1
- package/src/authored-node-runner.ts +6 -3
- package/src/authored-root-readback.ts +100 -0
- package/src/authored-root.ts +58 -85
- package/src/authored-step-output.ts +3 -0
- package/src/authored-worker-step.ts +29 -4
- package/src/cli/check-helper-body.ts +9 -3
- package/src/cli/check-helper-surface.ts +43 -0
- package/src/cli/check-triggers.ts +9 -4
- package/src/cli/check.ts +37 -7
- package/src/cli/cloud-answer.ts +40 -0
- package/src/cli/cloud-connect-cli.ts +9 -2
- package/src/cli/cloud-human.ts +24 -0
- package/src/cli/cloud-live.ts +10 -5
- package/src/cli/cloud-mirror-session.ts +3 -2
- package/src/cli/cloud-run.ts +10 -1
- package/src/cli/detached-record.ts +71 -0
- package/src/cli/detached-run.ts +131 -0
- package/src/cli/diagnostic-fold.ts +28 -0
- package/src/cli/direct-run.ts +14 -1
- package/src/cli/eval.ts +141 -0
- package/src/cli/journal-timeout.ts +49 -0
- package/src/cli/local-agent-remedy.ts +6 -4
- package/src/cli/observer-session.ts +7 -1
- package/src/cli/reuse.ts +25 -0
- package/src/cli/run.ts +75 -67
- package/src/cli/running-step.ts +140 -0
- package/src/cli-adapter.ts +7 -1
- package/src/cli-commands.ts +43 -7
- package/src/cli-watch.ts +8 -5
- package/src/cli.ts +158 -51
- package/src/cloud-answer.ts +202 -0
- package/src/cloud-auth-store.ts +185 -0
- package/src/cloud-http.ts +61 -50
- package/src/cloud-human-wait.ts +23 -0
- package/src/cloud-mirror-transport.ts +1 -0
- package/src/cloud-run-record.ts +14 -5
- package/src/cloud-run.ts +46 -2
- package/src/communication/local.ts +6 -1
- package/src/communication/worker.ts +11 -3
- package/src/compile.ts +5 -3
- package/src/daemon-spec-validation.ts +126 -0
- package/src/failure-kinds.ts +17 -1
- package/src/flow-eval-cloud.ts +114 -0
- package/src/flow-eval-error.ts +10 -0
- package/src/flow-eval-gate.ts +114 -0
- package/src/flow-eval-local.ts +236 -0
- package/src/flow-eval-outcome.ts +43 -0
- package/src/flow-eval-report.ts +102 -0
- package/src/flow-eval-suite.ts +169 -0
- package/src/flow-eval-version.ts +316 -0
- package/src/flow-eval.ts +234 -0
- package/src/helper-operation-use.ts +285 -0
- package/src/helper-preflight.ts +17 -2
- package/src/helper-reference.ts +6 -6
- package/src/index.ts +15 -1
- package/src/journal-budgeted-reads.ts +134 -0
- package/src/journal-client.ts +59 -166
- package/src/journal-connection.ts +222 -0
- package/src/journal-read-policy.ts +123 -0
- package/src/named-gate-lowering.ts +2 -6
- package/src/re2js-embedded.ts +3 -0
- package/src/spec.ts +10 -0
- package/src/step-fields.ts +1 -1
- package/src/validate.ts +14 -0
- package/src/worker-cli.ts +14 -3
- package/src/worker-lease.ts +11 -3
- package/src/worker.ts +1 -1
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import { spawnSync } from 'node:child_process';
|
|
2
|
+
import { canonicalize } from './canonical.js';
|
|
3
|
+
import { CompileError, toKernelSpec } from './compile.js';
|
|
4
|
+
import { PROTOCOL_VERSION } from './protocol.js';
|
|
5
|
+
import { RelayflowdNotFoundError, resolveRelayflowdBinary } from './relayflowd-path.js';
|
|
6
|
+
// Cache discovery, never verdicts. Changed resolution inputs get a fresh lookup.
|
|
7
|
+
const binaries = new Map();
|
|
8
|
+
const defaults = {
|
|
9
|
+
resolveBinary() {
|
|
10
|
+
const key = JSON.stringify([process.env['RELAYFLOWD_BIN'], process.env['PATH'], process.cwd(), process.argv[1]]);
|
|
11
|
+
let found = binaries.get(key);
|
|
12
|
+
if (found === undefined) {
|
|
13
|
+
try {
|
|
14
|
+
found = resolveRelayflowdBinary();
|
|
15
|
+
}
|
|
16
|
+
catch (error) {
|
|
17
|
+
found = error instanceof Error ? error : new Error(String(error));
|
|
18
|
+
}
|
|
19
|
+
binaries.set(key, found);
|
|
20
|
+
}
|
|
21
|
+
if (found instanceof Error)
|
|
22
|
+
throw found;
|
|
23
|
+
return found;
|
|
24
|
+
},
|
|
25
|
+
run: (binary, input) => spawnSync(binary, ['validate-spec'], {
|
|
26
|
+
input, encoding: 'utf8', timeout: 5000, killSignal: 'SIGKILL', maxBuffer: 1024 * 1024,
|
|
27
|
+
stdio: ['pipe', 'pipe', 'pipe'],
|
|
28
|
+
}),
|
|
29
|
+
};
|
|
30
|
+
export function unavailableValidation(mode, reason, detail = '', binary) {
|
|
31
|
+
const message = `${binary === undefined ? 'relayflowd' : `relayflowd at ${binary}`}: ${detail || reason.replaceAll('_', ' ')}. `
|
|
32
|
+
+ 'Daemon acceptance is unproven; an already attached daemon from another build is not checked.';
|
|
33
|
+
return {
|
|
34
|
+
validation: { mode: 'local', reason, ...(binary === undefined ? {} : { binary }) },
|
|
35
|
+
diagnostics: mode === 'required'
|
|
36
|
+
? [{ severity: 'refusal', kind: 'daemon_validation_unavailable', message }]
|
|
37
|
+
: reason === 'disabled' || reason === 'not_reached' || reason === 'relayflowd_not_found'
|
|
38
|
+
? [] : [{ severity: 'warning', kind: 'daemon_unvalidated', message }],
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
/** Same spec value run.start submits, in canonical form; no socket or run. */
|
|
42
|
+
export function validateSpecWithDaemon(flow, mode, deps = defaults) {
|
|
43
|
+
if (mode === 'off')
|
|
44
|
+
return unavailableValidation(mode, 'disabled');
|
|
45
|
+
let submitted;
|
|
46
|
+
let input;
|
|
47
|
+
try {
|
|
48
|
+
submitted = toKernelSpec(flow);
|
|
49
|
+
input = canonicalize(submitted);
|
|
50
|
+
}
|
|
51
|
+
catch (error) {
|
|
52
|
+
// run refuses this lowering too: degrading to local would fail open.
|
|
53
|
+
const errors = error instanceof CompileError ? error.errors : [String(error)];
|
|
54
|
+
return { validation: { mode: 'local', reason: 'not_reached' },
|
|
55
|
+
diagnostics: [{ severity: 'refusal', kind: 'invalid_spec', message: errors.join('; '), errors }] };
|
|
56
|
+
}
|
|
57
|
+
let binary;
|
|
58
|
+
try {
|
|
59
|
+
binary = deps.resolveBinary();
|
|
60
|
+
}
|
|
61
|
+
catch (error) {
|
|
62
|
+
const reason = error instanceof RelayflowdNotFoundError
|
|
63
|
+
? error.attempts[0]?.startsWith('RELAYFLOWD_BIN=') ? 'relayflowd_bin_invalid' : 'relayflowd_not_found'
|
|
64
|
+
: 'spawn_failed';
|
|
65
|
+
return unavailableValidation(mode, reason, String(error));
|
|
66
|
+
}
|
|
67
|
+
try {
|
|
68
|
+
const result = deps.run(binary, input);
|
|
69
|
+
if (result.error)
|
|
70
|
+
return unavailableValidation(mode, result.error.code === 'ETIMEDOUT' ? 'timed_out' : 'spawn_failed', result.error.message, binary);
|
|
71
|
+
let verdict;
|
|
72
|
+
try {
|
|
73
|
+
verdict = JSON.parse(result.stdout);
|
|
74
|
+
}
|
|
75
|
+
catch { /* Old binaries have no envelope. */ }
|
|
76
|
+
if (!isVerdict(verdict))
|
|
77
|
+
return unavailableValidation(mode, 'no_verdict', 'no validation verdict (the binary may predate validate-spec)', binary);
|
|
78
|
+
if (verdict.protocol !== PROTOCOL_VERSION)
|
|
79
|
+
return unavailableValidation(mode, 'protocol_mismatch', `validator protocol ${verdict.protocol}, SDK protocol ${PROTOCOL_VERSION}`, binary);
|
|
80
|
+
const validation = { mode: 'daemon', binary,
|
|
81
|
+
protocol: verdict.protocol, specVersion: verdict.spec_version };
|
|
82
|
+
if (verdict.ok) {
|
|
83
|
+
// Acceptance is the envelope AND exit 0; a validator that fails or is
|
|
84
|
+
// killed after printing it has not completed validation.
|
|
85
|
+
if (result.status !== 0)
|
|
86
|
+
return unavailableValidation(mode, 'abnormal_exit', `validator ${result.status === null ? `was terminated by ${result.signal ?? 'a signal'}` : `exited ${result.status}`} after an accepting verdict`, binary);
|
|
87
|
+
return { validation, diagnostics: [] };
|
|
88
|
+
}
|
|
89
|
+
// A refusal fails closed whatever the exit status, so it is reported as-is.
|
|
90
|
+
const message = verdict.error.message;
|
|
91
|
+
const index = /steps\[(\d+)\]/.exec(message)?.[1];
|
|
92
|
+
const stepId = index === undefined ? undefined : submitted.steps[Number(index)]?.id;
|
|
93
|
+
const generated = stepId !== undefined && !flow.steps.some(step => step.id === stepId);
|
|
94
|
+
return { validation, diagnostics: [{ severity: 'refusal', kind: 'invalid_spec',
|
|
95
|
+
message: message + (generated ? `\nStep "${stepId}" is generated by this compiler's gate lowering, not declared in the flow; `
|
|
96
|
+
+ 'the installed relayflowd and this @relayflows/sdk disagree (SDK lowering bug). Report this rather than editing the flow.' : ''),
|
|
97
|
+
errors: [message], ...(stepId === undefined ? {} : { stepId }) }] };
|
|
98
|
+
}
|
|
99
|
+
catch (error) {
|
|
100
|
+
return unavailableValidation(mode, 'spawn_failed', String(error), binary);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
function isVerdict(value) {
|
|
104
|
+
if (typeof value !== 'object' || value === null)
|
|
105
|
+
return false;
|
|
106
|
+
const v = value;
|
|
107
|
+
return Number.isInteger(v.protocol) && typeof v.spec_version === 'string'
|
|
108
|
+
&& (v.ok === true || (v.ok === false && v.error?.code === 'invalid_spec' && typeof v.error.message === 'string'));
|
|
109
|
+
}
|
|
110
|
+
export function validationAnnotation(validation) {
|
|
111
|
+
return validation?.mode === 'daemon'
|
|
112
|
+
? ` [validated by relayflowd at ${validation.binary}, protocol ${validation.protocol}]`
|
|
113
|
+
: ` [local only: ${(validation?.reason ?? 'not_reached').replaceAll('_', ' ')}, so daemon acceptance is unproven]`;
|
|
114
|
+
}
|
|
115
|
+
//# sourceMappingURL=daemon-spec-validation.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"daemon-spec-validation.js","sourceRoot":"","sources":["../src/daemon-spec-validation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC1D,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,EAAE,uBAAuB,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAqBxF,iFAAiF;AACjF,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA0B,CAAC;AACnD,MAAM,QAAQ,GAAyB;IACrC,aAAa;QACX,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,EAAE,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,GAAG,EAAE,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACjH,IAAI,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC9B,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,IAAI,CAAC;gBAAC,KAAK,GAAG,uBAAuB,EAAE,CAAC;YAAC,CAAC;YAC1C,OAAO,KAAK,EAAE,CAAC;gBAAC,KAAK,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YAAC,CAAC;YACpF,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC3B,CAAC;QACD,IAAI,KAAK,YAAY,KAAK;YAAE,MAAM,KAAK,CAAC;QACxC,OAAO,KAAK,CAAC;IACf,CAAC;IACD,GAAG,EAAE,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,eAAe,CAAC,EAAE;QAC3D,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,SAAS,EAAE,SAAS,EAAE,IAAI,GAAG,IAAI;QACrF,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;KAChC,CAAC;CACH,CAAC;AAEF,MAAM,UAAU,qBAAqB,CACnC,IAA0B,EAAE,MAAmB,EAAE,MAAM,GAAG,EAAE,EAAE,MAAe;IAE7E,MAAM,OAAO,GAAG,GAAG,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,iBAAiB,MAAM,EAAE,KAAK,MAAM,IAAI,MAAM,CAAC,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI;UAC5H,8FAA8F,CAAC;IACnG,OAAO;QACL,UAAU,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;QAClF,WAAW,EAAE,IAAI,KAAK,UAAU;YAC9B,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,+BAA+B,EAAE,OAAO,EAAE,CAAC;YAC3E,CAAC,CAAC,MAAM,KAAK,UAAU,IAAI,MAAM,KAAK,aAAa,IAAI,MAAM,KAAK,sBAAsB;gBACtF,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,oBAAoB,EAAE,OAAO,EAAE,CAAC;KAC1E,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,MAAM,UAAU,sBAAsB,CACpC,IAAc,EAAE,IAA0B,EAAE,OAA6B,QAAQ;IAEjF,IAAI,IAAI,KAAK,KAAK;QAAE,OAAO,qBAAqB,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;IACnE,IAAI,SAA0C,CAAC;IAC/C,IAAI,KAAa,CAAC;IAClB,IAAI,CAAC;QAAC,SAAS,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;QAAC,KAAK,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC;IAAC,CAAC;IACxE,OAAO,KAAK,EAAE,CAAC;QACb,qEAAqE;QACrE,MAAM,MAAM,GAAG,KAAK,YAAY,YAAY,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QAC9E,OAAO,EAAE,UAAU,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE;YAC3D,WAAW,EAAE,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;IACvG,CAAC;IACD,IAAI,MAAc,CAAC;IACnB,IAAI,CAAC;QAAC,MAAM,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;IAAC,CAAC;IACtC,OAAO,KAAK,EAAE,CAAC;QACb,MAAM,MAAM,GAAG,KAAK,YAAY,uBAAuB;YACrD,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC,sBAAsB;YACtG,CAAC,CAAC,cAAc,CAAC;QACnB,OAAO,qBAAqB,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IAC5D,CAAC;IACD,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QACvC,IAAI,MAAM,CAAC,KAAK;YAAE,OAAO,qBAAqB,CAAC,IAAI,EAChD,MAAM,CAAC,KAA+B,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,cAAc,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC7H,IAAI,OAAgB,CAAC;QACrB,IAAI,CAAC;YAAC,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC,CAAC,oCAAoC,CAAC,CAAC;QAC3F,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;YAAE,OAAO,qBAAqB,CAAC,IAAI,EAAE,YAAY,EACtE,8DAA8D,EAAE,MAAM,CAAC,CAAC;QAC1E,IAAI,OAAO,CAAC,QAAQ,KAAK,gBAAgB;YAAE,OAAO,qBAAqB,CAAC,IAAI,EAAE,mBAAmB,EAC/F,sBAAsB,OAAO,CAAC,QAAQ,kBAAkB,gBAAgB,EAAE,EAAE,MAAM,CAAC,CAAC;QACtF,MAAM,UAAU,GAAqB,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM;YAC3D,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,WAAW,EAAE,OAAO,CAAC,YAAY,EAAE,CAAC;QAClE,IAAI,OAAO,CAAC,EAAE,EAAE,CAAC;YACf,sEAAsE;YACtE,yDAAyD;YACzD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,qBAAqB,CAAC,IAAI,EAAE,eAAe,EACzE,aAAa,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,qBAAqB,MAAM,CAAC,MAAM,IAAI,UAAU,EAAE,CAAC,CAAC,CAAC,UAAU,MAAM,CAAC,MAAM,EAAE,6BAA6B,EAAE,MAAM,CAAC,CAAC;YAC7J,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC;QACzC,CAAC;QACD,4EAA4E;QAC5E,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC;QACtC,MAAM,KAAK,GAAG,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAClD,MAAM,MAAM,GAAG,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;QACpF,MAAM,SAAS,GAAG,MAAM,KAAK,SAAS,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,KAAK,MAAM,CAAC,CAAC;QACvF,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,cAAc;oBAC5E,OAAO,EAAE,OAAO,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,MAAM,6EAA6E;0BAC1H,0HAA0H,CAAC,CAAC,CAAC,EAAE,CAAC;oBACpI,MAAM,EAAE,CAAC,OAAO,CAAC,EAAE,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC;IACxE,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAAC,OAAO,qBAAqB,CAAC,IAAI,EAAE,cAAc,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC,CAAC;IAAC,CAAC;AAChG,CAAC;AAID,SAAS,SAAS,CAAC,KAAc;IAC/B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,KAAK,CAAC;IAC9D,MAAM,CAAC,GAAG,KAAyB,CAAC;IACpC,OAAO,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,OAAO,CAAC,CAAC,YAAY,KAAK,QAAQ;WACpE,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,KAAK,IAAI,CAAC,CAAC,KAAK,EAAE,IAAI,KAAK,cAAc,IAAI,OAAO,CAAC,CAAC,KAAK,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC;AACtH,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,UAA6B;IAChE,OAAO,UAAU,EAAE,IAAI,KAAK,QAAQ;QAClC,CAAC,CAAC,gCAAgC,UAAU,CAAC,MAAM,cAAc,UAAU,CAAC,QAAQ,GAAG;QACvF,CAAC,CAAC,iBAAiB,CAAC,UAAU,EAAE,MAAM,IAAI,aAAa,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC,qCAAqC,CAAC;AACvH,CAAC"}
|
package/dist/failure-kinds.d.ts
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
export declare const PREFLIGHT_FAILURE_KINDS: readonly ["invalid_spec", "unknown_gate_kind", "gate_pattern_invalid", "gate_command_missing", "gate_bound_invalid", "gate_path_invalid", "gate_host_unsupported", "gate_ids_invalid", "gate_source_unsupported", "plugin_unknown", "plugin_unlisted", "plugin_install_failed", "plugin_manifest_missing", "plugin_manifest_invalid", "plugin_preflight_missing", "plugin_verb_unknown_primitive", "plugin_unsupported", "plugin_credential_missing", "plugin_server_unreachable", "plugin_kind_invalid", "plugin_incompatible", "plugin_event_unroutable", "plugin_source_invalid", "plugin_event_ambiguous", "plugin_source_unresolved", "plugin_fetch_failed", "plugin_path_invalid", "plugin_too_large", "plugin_source_drift", "plugin_lock_invalid", "helper_provider.mount_required", "helper_provider.unsupported", "helper_slack.credential_missing", "helper_slack.mount_required", "helper_mount_required", "mcp_undeclared_server", "mcp_unreachable", "budget_syntax_invalid", "cli_missing", "cli_outdated", "cli_unauthenticated", "cli_unresolved", "cli_unsupported", "command_missing", "model_unavailable", "model_unknown", "memory_unreachable", "no_executor", "probe_failed", "provider_usage_limited", "scope_syntax_invalid", "mount_unknown", "scope_ungrantable"];
|
|
3
3
|
/** Input/command refusals emitted before the pure preflight predicates run. */
|
|
4
4
|
export declare const CHECK_INPUT_FAILURE_KINDS: readonly ["config_invalid", "input_invalid", "input_missing", "input_too_large", "input_unreadable", "invalid_invocation", "invalid_spec"];
|
|
5
|
-
export declare const CHECK_FAILURE_KINDS: readonly ["config_invalid", "input_invalid", "input_missing", "input_too_large", "input_unreadable", "invalid_invocation", "invalid_spec", "unknown_gate_kind", "gate_pattern_invalid", "gate_command_missing", "gate_bound_invalid", "gate_path_invalid", "gate_host_unsupported", "gate_ids_invalid", "gate_source_unsupported", "plugin_unknown", "plugin_unlisted", "plugin_install_failed", "plugin_manifest_missing", "plugin_manifest_invalid", "plugin_preflight_missing", "plugin_verb_unknown_primitive", "plugin_unsupported", "plugin_credential_missing", "plugin_server_unreachable", "plugin_kind_invalid", "plugin_incompatible", "plugin_event_unroutable", "plugin_source_invalid", "plugin_event_ambiguous", "plugin_source_unresolved", "plugin_fetch_failed", "plugin_path_invalid", "plugin_too_large", "plugin_source_drift", "plugin_lock_invalid", "helper_provider.mount_required", "helper_provider.unsupported", "helper_slack.credential_missing", "helper_slack.mount_required", "helper_mount_required", "mcp_undeclared_server", "mcp_unreachable", "budget_syntax_invalid", "cli_missing", "cli_outdated", "cli_unauthenticated", "cli_unresolved", "cli_unsupported", "command_missing", "model_unavailable", "model_unknown", "memory_unreachable", "no_executor", "probe_failed", "provider_usage_limited", "scope_syntax_invalid", "mount_unknown", "scope_ungrantable"];
|
|
5
|
+
export declare const CHECK_FAILURE_KINDS: readonly ["daemon_validation_unavailable", "config_invalid", "input_invalid", "input_missing", "input_too_large", "input_unreadable", "invalid_invocation", "invalid_spec", "unknown_gate_kind", "gate_pattern_invalid", "gate_command_missing", "gate_bound_invalid", "gate_path_invalid", "gate_host_unsupported", "gate_ids_invalid", "gate_source_unsupported", "plugin_unknown", "plugin_unlisted", "plugin_install_failed", "plugin_manifest_missing", "plugin_manifest_invalid", "plugin_preflight_missing", "plugin_verb_unknown_primitive", "plugin_unsupported", "plugin_credential_missing", "plugin_server_unreachable", "plugin_kind_invalid", "plugin_incompatible", "plugin_event_unroutable", "plugin_source_invalid", "plugin_event_ambiguous", "plugin_source_unresolved", "plugin_fetch_failed", "plugin_path_invalid", "plugin_too_large", "plugin_source_drift", "plugin_lock_invalid", "helper_provider.mount_required", "helper_provider.unsupported", "helper_slack.credential_missing", "helper_slack.mount_required", "helper_mount_required", "mcp_undeclared_server", "mcp_unreachable", "budget_syntax_invalid", "cli_missing", "cli_outdated", "cli_unauthenticated", "cli_unresolved", "cli_unsupported", "command_missing", "model_unavailable", "model_unknown", "memory_unreachable", "no_executor", "probe_failed", "provider_usage_limited", "scope_syntax_invalid", "mount_unknown", "scope_ungrantable"];
|
|
6
6
|
/**
|
|
7
7
|
* Warnings never refuse. Covenant 2 asks preflight to refuse *or warn* on
|
|
8
8
|
* anything it cannot prove, so a deterministic step always leaves exactly one
|
|
@@ -54,14 +54,14 @@ export declare const PREFLIGHT_WARNING_KINDS: readonly ["unprovable_effects", "m
|
|
|
54
54
|
* members is pre-existing — renaming it is a separate change, and neither this
|
|
55
55
|
* list nor `RunDiagnostic` is exported from the package index.
|
|
56
56
|
*/
|
|
57
|
-
export declare const RUN_FAILURE_KINDS: readonly ["bucket_unconfigured", "bucket_unreachable", "bundle_signature_invalid", "bundle_unsupported", "human_influenced_run", "reuse_spec_mismatch", "reuse_run_not_found", "reuse_journal_read_failed", "daemon_unreachable", "daemon_protocol_mismatch", "daemon_start_failed", "daemon_start_timeout", "relayflowd_not_found", "protocol_error", "run_parked", "run_declined", "run_unavailable", "gate_failed", "human_wait_unknown", "local_agent_unavailable", "root_lease_lost"];
|
|
57
|
+
export declare const RUN_FAILURE_KINDS: readonly ["bucket_unconfigured", "bucket_unreachable", "bundle_signature_invalid", "bundle_unsupported", "human_influenced_run", "reuse_spec_mismatch", "reuse_run_not_found", "reuse_journal_read_failed", "daemon_unreachable", "daemon_protocol_mismatch", "daemon_start_failed", "daemon_start_timeout", "relayflowd_not_found", "protocol_error", "run_parked", "run_declined", "run_unavailable", "gate_failed", "human_wait_unknown", "memory_finding_invalid", "memory_unwritable", "memory_unjournaled", "memory_unreachable", "local_agent_unavailable", "daemon_unresponsive", "root_lease_lost"];
|
|
58
58
|
/**
|
|
59
59
|
* Non-refusing outcomes of the attach step. `connection_file_stale` is
|
|
60
60
|
* DAEMON-LIFECYCLE.md §2 row 2: the socket answered while `connection.json`
|
|
61
61
|
* described a process that is gone. The socket is the authority, so this
|
|
62
62
|
* attaches — but it says so rather than passing in silence.
|
|
63
63
|
*/
|
|
64
|
-
export declare const RUN_WARNING_KINDS: readonly ["connection_file_stale"];
|
|
64
|
+
export declare const RUN_WARNING_KINDS: readonly ["connection_file_stale", "reuse_summary_unavailable", "result_unreadable", "inspection_interrupted"];
|
|
65
65
|
/**
|
|
66
66
|
* Warnings emitted by flows check, outside pure preflight.
|
|
67
67
|
*
|
|
@@ -72,7 +72,9 @@ export declare const RUN_WARNING_KINDS: readonly ["connection_file_stale"];
|
|
|
72
72
|
* daemon-free, can see none either — so it opts in, while `flows run`, `flows
|
|
73
73
|
* build` and SDK submissions do not (cli/check-worker-surface.ts).
|
|
74
74
|
*/
|
|
75
|
-
|
|
75
|
+
/** helper_credential_unresolved describes a local mount gap during inspection;
|
|
76
|
+
* Cloud validates workspace integrations at submit, and local execution refuses. */
|
|
77
|
+
export declare const CHECK_WARNING_KINDS: readonly ["editor_schema_missing", "daemon_unvalidated", "agent_worker_unresolved", "helper_credential_unresolved"];
|
|
76
78
|
export type CheckWarningKind = (typeof CHECK_WARNING_KINDS)[number];
|
|
77
79
|
export type PreflightFailureKind = (typeof PREFLIGHT_FAILURE_KINDS)[number];
|
|
78
80
|
export type CheckFailureKind = (typeof CHECK_FAILURE_KINDS)[number];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"failure-kinds.d.ts","sourceRoot":"","sources":["../src/failure-kinds.ts"],"names":[],"mappings":"AAiCA,qEAAqE;AACrE,eAAO,MAAM,uBAAuB,8tCAG1B,CAAC;AAEX,+EAA+E;AAC/E,eAAO,MAAM,yBAAyB,4IAQ5B,CAAC;
|
|
1
|
+
{"version":3,"file":"failure-kinds.d.ts","sourceRoot":"","sources":["../src/failure-kinds.ts"],"names":[],"mappings":"AAiCA,qEAAqE;AACrE,eAAO,MAAM,uBAAuB,8tCAG1B,CAAC;AAEX,+EAA+E;AAC/E,eAAO,MAAM,yBAAyB,4IAQ5B,CAAC;AAGX,eAAO,MAAM,mBAAmB,g3CAItB,CAAC;AAEX;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,eAAO,MAAM,uBAAuB,4LAS1B,CAAC;AAEX;;;;;;;;;;;;;;;;GAgBG;AACH,eAAO,MAAM,iBAAiB,8kBA6CpB,CAAC;AAEX;;;;;GAKG;AACH,eAAO,MAAM,iBAAiB,gHAKpB,CAAC;AAEX;;;;;;;;;GASG;AACH;oFACoF;AACpF,eAAO,MAAM,mBAAmB,qHAAsH,CAAC;AACvJ,MAAM,MAAM,gBAAgB,GAAG,CAAC,OAAO,mBAAmB,CAAC,CAAC,MAAM,CAAC,CAAC;AAEpE,MAAM,MAAM,oBAAoB,GAAG,CAAC,OAAO,uBAAuB,CAAC,CAAC,MAAM,CAAC,CAAC;AAC5E,MAAM,MAAM,gBAAgB,GAAG,CAAC,OAAO,mBAAmB,CAAC,CAAC,MAAM,CAAC,CAAC;AACpE,MAAM,MAAM,oBAAoB,GAAG,CAAC,OAAO,uBAAuB,CAAC,CAAC,MAAM,CAAC,CAAC;AAC5E,MAAM,MAAM,cAAc,GAAG,CAAC,OAAO,iBAAiB,CAAC,CAAC,MAAM,CAAC,CAAC;AAChE,MAAM,MAAM,cAAc,GAAG,CAAC,OAAO,iBAAiB,CAAC,CAAC,MAAM,CAAC,CAAC;AAEhE;;;;;;;;;;;;;GAaG;AACH,MAAM,WAAW,kBAAkB;IACjC;;;;OAIG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,uEAAuE;IACvE,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,mFAAmF;IACnF,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,sEAAsE;IACtE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,gEAAgE;IAChE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,gEAAgE;IAChE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,+EAA+E;IAC/E,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,sEAAsE;IACtE,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAED;;;;;;;;;;GAUG;AACH,MAAM,MAAM,yBAAyB,GAAG,SAAS,GAAG,WAAW,GAAG,SAAS,CAAC;AAE5E;;;;;;;;;;;GAWG;AACH,MAAM,MAAM,SAAS,GAAG,oBAAoB,GAAG,aAAa,CAAC;AAE7D;;;;;;;;;;;GAWG;AACH,MAAM,WAAW,iBAAiB;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,0EAA0E;IAC1E,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,8EAA8E;IAC9E,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B;;;;;OAKG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,0EAA0E;IAC1E,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B;;;;;;OAMG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B;;;;;;;;OAQG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,4EAA4E;IAC5E,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;;OAIG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,0FAA0F;IAC1F,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,kBAAkB,EAAE,CAAC;IAChC,oEAAoE;IACpE,eAAe,CAAC,EAAE,yBAAyB,CAAC;IAC5C,yDAAyD;IACzD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oEAAoE;IACpE,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAKD,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,KAAK,IAAI,gBAAgB,CAE3E;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,KAAK,IAAI,cAAc,CAEvE"}
|
package/dist/failure-kinds.js
CHANGED
|
@@ -44,7 +44,9 @@ export const CHECK_INPUT_FAILURE_KINDS = [
|
|
|
44
44
|
'invalid_invocation',
|
|
45
45
|
...SHARED_SPEC_FAILURE_KINDS,
|
|
46
46
|
];
|
|
47
|
+
// run never enables daemon validation; fromCheckReport needs a mapping if that changes.
|
|
47
48
|
export const CHECK_FAILURE_KINDS = [
|
|
49
|
+
'daemon_validation_unavailable',
|
|
48
50
|
...CHECK_INPUT_FAILURE_KINDS,
|
|
49
51
|
...PREFLIGHT_ENVIRONMENT_FAILURE_KINDS,
|
|
50
52
|
];
|
|
@@ -130,6 +132,14 @@ export const RUN_FAILURE_KINDS = [
|
|
|
130
132
|
'gate_failed',
|
|
131
133
|
/** `flows answer` named a wait the run is not asking: unknown, or already answered. */
|
|
132
134
|
'human_wait_unknown',
|
|
135
|
+
/** `f.memory.learn` was given a finding without non-empty string fields. */
|
|
136
|
+
'memory_finding_invalid',
|
|
137
|
+
/** `f.memory.learn` could not persist the finding to script memory. */
|
|
138
|
+
'memory_unwritable',
|
|
139
|
+
/** `f.memory.learn` ran without a durable root run to journal the finding in. */
|
|
140
|
+
'memory_unjournaled',
|
|
141
|
+
/** `f.memory` lost its database, or found a damaged learned finding, after the run started. */
|
|
142
|
+
'memory_unreachable',
|
|
133
143
|
/**
|
|
134
144
|
* `--local-agent` cannot be honoured for this invocation, so it is refused
|
|
135
145
|
* rather than accepted and ignored. A local agent worker is admitted at run
|
|
@@ -143,6 +153,7 @@ export const RUN_FAILURE_KINDS = [
|
|
|
143
153
|
* reach this process. The run is NOT failed (`status: running`): its
|
|
144
154
|
* completed steps are journaled and `flows resume <rootRunId>` continues it.
|
|
145
155
|
*/
|
|
156
|
+
'daemon_unresponsive',
|
|
146
157
|
'root_lease_lost',
|
|
147
158
|
];
|
|
148
159
|
/**
|
|
@@ -153,6 +164,9 @@ export const RUN_FAILURE_KINDS = [
|
|
|
153
164
|
*/
|
|
154
165
|
export const RUN_WARNING_KINDS = [
|
|
155
166
|
'connection_file_stale',
|
|
167
|
+
'reuse_summary_unavailable',
|
|
168
|
+
'result_unreadable',
|
|
169
|
+
'inspection_interrupted',
|
|
156
170
|
];
|
|
157
171
|
/**
|
|
158
172
|
* Warnings emitted by flows check, outside pure preflight.
|
|
@@ -164,7 +178,9 @@ export const RUN_WARNING_KINDS = [
|
|
|
164
178
|
* daemon-free, can see none either — so it opts in, while `flows run`, `flows
|
|
165
179
|
* build` and SDK submissions do not (cli/check-worker-surface.ts).
|
|
166
180
|
*/
|
|
167
|
-
|
|
181
|
+
/** helper_credential_unresolved describes a local mount gap during inspection;
|
|
182
|
+
* Cloud validates workspace integrations at submit, and local execution refuses. */
|
|
183
|
+
export const CHECK_WARNING_KINDS = ['editor_schema_missing', 'daemon_unvalidated', 'agent_worker_unresolved', 'helper_credential_unresolved'];
|
|
168
184
|
const CHECK_FAILURE_KIND_SET = new Set(CHECK_FAILURE_KINDS);
|
|
169
185
|
const RUN_FAILURE_KIND_SET = new Set(RUN_FAILURE_KINDS);
|
|
170
186
|
export function isCheckFailureKind(value) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"failure-kinds.js","sourceRoot":"","sources":["../src/failure-kinds.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,EAAE,wBAAwB,EAAE,MAAM,kBAAkB,CAAC;AAC5D,MAAM,yBAAyB,GAAG,CAAC,cAAc,CAAU,CAAC;AAE5D,yEAAyE;AACzE,MAAM,mCAAmC,GAAG;IAC1C,GAAG,wBAAwB;IAC3B,GAAG,oBAAoB;IACvB,gCAAgC;IAChC,6BAA6B;IAC7B,iCAAiC;IACjC,6BAA6B;IAC7B,uBAAuB;IACvB,uBAAuB;IACvB,iBAAiB;IACjB,uBAAuB;IACvB,aAAa;IACb,cAAc;IACd,qBAAqB;IACrB,gBAAgB;IAChB,iBAAiB;IACjB,iBAAiB;IACjB,mBAAmB;IACnB,eAAe;IACf,oBAAoB;IACpB,aAAa;IACb,cAAc;IACd,wBAAwB;IACxB,sBAAsB;IACtB,eAAe;IACf,mBAAmB;CACX,CAAC;AAEX,qEAAqE;AACrE,MAAM,CAAC,MAAM,uBAAuB,GAAG;IACrC,GAAG,yBAAyB;IAC5B,GAAG,mCAAmC;CAC9B,CAAC;AAEX,+EAA+E;AAC/E,MAAM,CAAC,MAAM,yBAAyB,GAAG;IACvC,gBAAgB;IAChB,eAAe;IACf,eAAe;IACf,iBAAiB;IACjB,kBAAkB;IAClB,oBAAoB;IACpB,GAAG,yBAAyB;CACpB,CAAC;AAEX,MAAM,CAAC,MAAM,mBAAmB,GAAG;IACjC,GAAG,yBAAyB;IAC5B,GAAG,mCAAmC;CAC9B,CAAC;AAEX;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG;IACrC,oBAAoB;IACpB,wBAAwB;IACxB,oBAAoB;IACpB,oBAAoB;IACpB,cAAc;IACd,kBAAkB;IAClB,wBAAwB;IACxB,qBAAqB;CACb,CAAC;AAEX;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG;IAC/B,qBAAqB;IACrB,oBAAoB;IACpB,0BAA0B;IAC1B,oBAAoB;IACpB,sBAAsB;IACtB,qBAAqB;IACrB,qBAAqB;IACrB,2BAA2B;IAC3B,oBAAoB;IACpB,0BAA0B;IAC1B,qBAAqB;IACrB,sBAAsB;IACtB,sBAAsB;IACtB,gBAAgB;IAChB,YAAY;IACZ,cAAc;IACd,iBAAiB;IACjB,uFAAuF;IACvF,aAAa;IACb,uFAAuF;IACvF,oBAAoB;IACpB;;;;;;OAMG;IACH,yBAAyB;IACzB;;;;OAIG;IACH,iBAAiB;CACT,CAAC;AAEX;;;;;GAKG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG;IAC/B,uBAAuB;
|
|
1
|
+
{"version":3,"file":"failure-kinds.js","sourceRoot":"","sources":["../src/failure-kinds.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,EAAE,wBAAwB,EAAE,MAAM,kBAAkB,CAAC;AAC5D,MAAM,yBAAyB,GAAG,CAAC,cAAc,CAAU,CAAC;AAE5D,yEAAyE;AACzE,MAAM,mCAAmC,GAAG;IAC1C,GAAG,wBAAwB;IAC3B,GAAG,oBAAoB;IACvB,gCAAgC;IAChC,6BAA6B;IAC7B,iCAAiC;IACjC,6BAA6B;IAC7B,uBAAuB;IACvB,uBAAuB;IACvB,iBAAiB;IACjB,uBAAuB;IACvB,aAAa;IACb,cAAc;IACd,qBAAqB;IACrB,gBAAgB;IAChB,iBAAiB;IACjB,iBAAiB;IACjB,mBAAmB;IACnB,eAAe;IACf,oBAAoB;IACpB,aAAa;IACb,cAAc;IACd,wBAAwB;IACxB,sBAAsB;IACtB,eAAe;IACf,mBAAmB;CACX,CAAC;AAEX,qEAAqE;AACrE,MAAM,CAAC,MAAM,uBAAuB,GAAG;IACrC,GAAG,yBAAyB;IAC5B,GAAG,mCAAmC;CAC9B,CAAC;AAEX,+EAA+E;AAC/E,MAAM,CAAC,MAAM,yBAAyB,GAAG;IACvC,gBAAgB;IAChB,eAAe;IACf,eAAe;IACf,iBAAiB;IACjB,kBAAkB;IAClB,oBAAoB;IACpB,GAAG,yBAAyB;CACpB,CAAC;AAEX,wFAAwF;AACxF,MAAM,CAAC,MAAM,mBAAmB,GAAG;IACjC,+BAA+B;IAC/B,GAAG,yBAAyB;IAC5B,GAAG,mCAAmC;CAC9B,CAAC;AAEX;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG;IACrC,oBAAoB;IACpB,wBAAwB;IACxB,oBAAoB;IACpB,oBAAoB;IACpB,cAAc;IACd,kBAAkB;IAClB,wBAAwB;IACxB,qBAAqB;CACb,CAAC;AAEX;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG;IAC/B,qBAAqB;IACrB,oBAAoB;IACpB,0BAA0B;IAC1B,oBAAoB;IACpB,sBAAsB;IACtB,qBAAqB;IACrB,qBAAqB;IACrB,2BAA2B;IAC3B,oBAAoB;IACpB,0BAA0B;IAC1B,qBAAqB;IACrB,sBAAsB;IACtB,sBAAsB;IACtB,gBAAgB;IAChB,YAAY;IACZ,cAAc;IACd,iBAAiB;IACjB,uFAAuF;IACvF,aAAa;IACb,uFAAuF;IACvF,oBAAoB;IACpB,4EAA4E;IAC5E,wBAAwB;IACxB,uEAAuE;IACvE,mBAAmB;IACnB,iFAAiF;IACjF,oBAAoB;IACpB,+FAA+F;IAC/F,oBAAoB;IACpB;;;;;;OAMG;IACH,yBAAyB;IACzB;;;;OAIG;IACH,qBAAqB;IACrB,iBAAiB;CACT,CAAC;AAEX;;;;;GAKG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG;IAC/B,uBAAuB;IACvB,2BAA2B;IAC3B,mBAAmB;IACnB,wBAAwB;CAChB,CAAC;AAEX;;;;;;;;;GASG;AACH;oFACoF;AACpF,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,uBAAuB,EAAE,oBAAoB,EAAE,yBAAyB,EAAE,8BAA8B,CAAU,CAAC;AAkKvJ,MAAM,sBAAsB,GAAwB,IAAI,GAAG,CAAC,mBAAmB,CAAC,CAAC;AACjF,MAAM,oBAAoB,GAAwB,IAAI,GAAG,CAAC,iBAAiB,CAAC,CAAC;AAE7E,MAAM,UAAU,kBAAkB,CAAC,KAAa;IAC9C,OAAO,sBAAsB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;AAC3C,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,KAAa;IAC5C,OAAO,oBAAoB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;AACzC,CAAC"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { type CloudStep } from './cloud-read.js';
|
|
2
|
+
import { type RunInCloudOptions } from './cloud-run.js';
|
|
3
|
+
import { type CloudConnectionOptions } from './cloud-http.js';
|
|
4
|
+
import type { FlowEvalExecutor, FlowEvalRun } from './flow-eval-report.js';
|
|
5
|
+
export interface CloudFlowEvalExecutorOptions extends CloudConnectionOptions {
|
|
6
|
+
workspaceId?: string;
|
|
7
|
+
syncCode?: RunInCloudOptions['syncCode'];
|
|
8
|
+
pollIntervalMs?: number;
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Execute each case as a fresh hosted run and wait for it. Needs a token with
|
|
12
|
+
* `workflow:invoke:write` and `runs:read`; inside a Cloud step that means an
|
|
13
|
+
* injected workspace token, since a run-scoped sandbox token reads only its own run.
|
|
14
|
+
*/
|
|
15
|
+
export declare function cloudFlowEvalExecutor(options?: CloudFlowEvalExecutorOptions): FlowEvalExecutor;
|
|
16
|
+
/**
|
|
17
|
+
* Map a terminal hosted run record onto the eval vocabulary: the validated run
|
|
18
|
+
* state decides parked/failed/canceled, and the stored report (the same
|
|
19
|
+
* RunReport a local run prints) supplies `declined` and the completion detail.
|
|
20
|
+
* Exported for tests.
|
|
21
|
+
*/
|
|
22
|
+
export declare function cloudRunOutcome(record: unknown, runId: string): Pick<FlowEvalRun, 'completionReason' | 'completionDetail'>;
|
|
23
|
+
/** Sum hosted step spend; any step with unknown cost makes the total unknown. Exported for tests. */
|
|
24
|
+
export declare function cloudSpend(steps: readonly CloudStep[] | undefined): Pick<FlowEvalRun, 'costUsd' | 'tokensIn' | 'tokensOut' | 'steps'>;
|
|
25
|
+
//# sourceMappingURL=flow-eval-cloud.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"flow-eval-cloud.d.ts","sourceRoot":"","sources":["../src/flow-eval-cloud.ts"],"names":[],"mappings":"AAIA,OAAO,EAAoB,KAAK,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAAmC,KAAK,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAEzF,OAAO,EAA+B,KAAK,sBAAsB,EAAE,MAAM,iBAAiB,CAAC;AAE3F,OAAO,KAAK,EAAE,gBAAgB,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AA6B3E,MAAM,WAAW,4BAA6B,SAAQ,sBAAsB;IAC1E,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,iBAAiB,CAAC,UAAU,CAAC,CAAC;IACzC,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;;;GAIG;AACH,wBAAgB,qBAAqB,CAAC,OAAO,GAAE,4BAAiC,GAAG,gBAAgB,CAgClG;AAED;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC,WAAW,EAAE,kBAAkB,GAAG,kBAAkB,CAAC,CAU1H;AAED,qGAAqG;AACrG,wBAAgB,UAAU,CAAC,KAAK,EAAE,SAAS,SAAS,EAAE,GAAG,SAAS,GAAG,IAAI,CAAC,WAAW,EAAE,SAAS,GAAG,UAAU,GAAG,WAAW,GAAG,OAAO,CAAC,CAWrI"}
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
// The Cloud executor for `evaluateFlow`: one fresh hosted run per case,
|
|
2
|
+
// waited on, with its verdict read from the hosted run report and its spend
|
|
3
|
+
// from the run's steps.
|
|
4
|
+
import { getCloudRunSteps } from './cloud-read.js';
|
|
5
|
+
import { runInCloud, waitForCloudFlowRun } from './cloud-run.js';
|
|
6
|
+
import { cloudRunState } from './cloud-run-record.js';
|
|
7
|
+
import { cloudRequest, isCloudRecord } from './cloud-http.js';
|
|
8
|
+
import { isAuthoredFlowPath } from './direct-input.js';
|
|
9
|
+
import { runReportOutcome } from './flow-eval-outcome.js';
|
|
10
|
+
import { flowEvalSources, isPathTarget } from './flow-eval-version.js';
|
|
11
|
+
const MODULE = /\.(?:[mc]?[jt]s)$/u;
|
|
12
|
+
const EXTENSION_STORE = /(?:^|\/)\.flows\/plugins\//u;
|
|
13
|
+
/** Manifests and locks the version pins but Cloud does not need as imports. */
|
|
14
|
+
const PROJECT_METADATA = /(?:^|\/)(?:package(?:-lock)?|flows(?:\.lock)?)\.json$/u;
|
|
15
|
+
/**
|
|
16
|
+
* Cloud receives the entry file's source alone (cloud-run.ts): a relative
|
|
17
|
+
* import is not resolved by the hosted runner, even with `syncCode`. A flow
|
|
18
|
+
* whose version spans local modules would therefore fail only on Cloud, or run
|
|
19
|
+
* something other than what was judged. Refuse it before submitting anything.
|
|
20
|
+
*/
|
|
21
|
+
async function refuseLocalImports(flow) {
|
|
22
|
+
if (!isPathTarget(flow) || !MODULE.test(flow.path))
|
|
23
|
+
return;
|
|
24
|
+
const { files } = await flowEvalSources(flow);
|
|
25
|
+
const entry = files.find(file => !file.includes('/') && flow.path.endsWith(file));
|
|
26
|
+
// Locked extension payloads are not imports: Cloud receives them separately
|
|
27
|
+
// (`extensions` in the submission), so they never make a flow non-self-contained.
|
|
28
|
+
const modules = files.filter(file => (MODULE.test(file) || /\.json$/u.test(file)) && file !== entry
|
|
29
|
+
&& !EXTENSION_STORE.test(file) && !PROJECT_METADATA.test(file));
|
|
30
|
+
if (modules.length > 0) {
|
|
31
|
+
throw new Error(`Cloud evaluation runs the entry file alone, and this flow imports local modules (${modules.join(', ')}). `
|
|
32
|
+
+ 'Evaluate it locally, or inline those modules.');
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Execute each case as a fresh hosted run and wait for it. Needs a token with
|
|
37
|
+
* `workflow:invoke:write` and `runs:read`; inside a Cloud step that means an
|
|
38
|
+
* injected workspace token, since a run-scoped sandbox token reads only its own run.
|
|
39
|
+
*/
|
|
40
|
+
export function cloudFlowEvalExecutor(options = {}) {
|
|
41
|
+
return async ({ flow, input, signal }) => {
|
|
42
|
+
await refuseLocalImports(flow);
|
|
43
|
+
const connection = {
|
|
44
|
+
...(options.apiUrl === undefined ? {} : { apiUrl: options.apiUrl }),
|
|
45
|
+
...(options.token === undefined ? {} : { token: options.token }),
|
|
46
|
+
...(options.requestTimeoutMs === undefined ? {} : { requestTimeoutMs: options.requestTimeoutMs }),
|
|
47
|
+
...(signal === undefined ? {} : { signal }),
|
|
48
|
+
};
|
|
49
|
+
const receipt = await runInCloud(flow, {
|
|
50
|
+
...connection,
|
|
51
|
+
...(options.workspaceId === undefined ? {} : { workspaceId: options.workspaceId }),
|
|
52
|
+
...(options.syncCode === undefined ? {} : { syncCode: options.syncCode }),
|
|
53
|
+
// Cloud requires an explicit input for authored source; `{}` is the
|
|
54
|
+
// documented "no fields", matching the local executor's default.
|
|
55
|
+
...(input !== undefined ? { input } : 'path' in flow && isAuthoredFlowPath(flow.path) ? { input: {} } : {}),
|
|
56
|
+
});
|
|
57
|
+
await waitForCloudFlowRun(receipt.runId, {
|
|
58
|
+
...connection, ...(options.pollIntervalMs === undefined ? {} : { pollIntervalMs: options.pollIntervalMs }),
|
|
59
|
+
});
|
|
60
|
+
// Re-read the terminal record for its stored report: the run state alone
|
|
61
|
+
// carries the kernel's reason, which says `success` for a declined body
|
|
62
|
+
// and has no completion detail.
|
|
63
|
+
const record = await cloudRequest(`/api/v1/workflows/runs/${receipt.runId}`, connection);
|
|
64
|
+
let steps;
|
|
65
|
+
try {
|
|
66
|
+
steps = await getCloudRunSteps(receipt.runId, connection);
|
|
67
|
+
}
|
|
68
|
+
catch {
|
|
69
|
+
// As locally: spend is evidence, not the verdict.
|
|
70
|
+
}
|
|
71
|
+
return { runId: receipt.runId, ...cloudRunOutcome(record, receipt.runId), ...cloudSpend(steps) };
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Map a terminal hosted run record onto the eval vocabulary: the validated run
|
|
76
|
+
* state decides parked/failed/canceled, and the stored report (the same
|
|
77
|
+
* RunReport a local run prints) supplies `declined` and the completion detail.
|
|
78
|
+
* Exported for tests.
|
|
79
|
+
*/
|
|
80
|
+
export function cloudRunOutcome(record, runId) {
|
|
81
|
+
const state = cloudRunState(record, runId);
|
|
82
|
+
const report = isCloudRecord(record) && isCloudRecord(record.result) ? record.result : {};
|
|
83
|
+
const mapped = runReportOutcome(report);
|
|
84
|
+
// A park with nothing attached to run it is the same fact on both sides.
|
|
85
|
+
if (mapped.completionReason === 'worker_unavailable')
|
|
86
|
+
return mapped;
|
|
87
|
+
if (state.status === 'needs_human')
|
|
88
|
+
return { ...mapped, completionReason: 'needs_human' };
|
|
89
|
+
if (!('completionReason' in state))
|
|
90
|
+
return { ...mapped, completionReason: state.status };
|
|
91
|
+
if (state.completionReason === 'success')
|
|
92
|
+
return mapped.completionReason === 'declined' ? mapped : { ...mapped, completionReason: 'success' };
|
|
93
|
+
return { ...mapped, completionReason: state.completionReason };
|
|
94
|
+
}
|
|
95
|
+
/** Sum hosted step spend; any step with unknown cost makes the total unknown. Exported for tests. */
|
|
96
|
+
export function cloudSpend(steps) {
|
|
97
|
+
if (steps === undefined)
|
|
98
|
+
return { costUsd: null, tokensIn: null, tokensOut: null };
|
|
99
|
+
const known = (values) => values.some(v => v === null || !Number.isFinite(v) || v < 0) ? null : values.reduce((sum, v) => sum + v, 0);
|
|
100
|
+
const costed = steps.filter(step => step.step_type === 'agent' || step.step_type === 'llm' || step.cost_usd !== null);
|
|
101
|
+
return {
|
|
102
|
+
costUsd: known(costed.map(step => step.cost_usd)),
|
|
103
|
+
tokensIn: known(costed.map(step => step.tokens_in)),
|
|
104
|
+
tokensOut: known(costed.map(step => step.tokens_out)),
|
|
105
|
+
steps: steps.map(step => ({ id: step.step_name, status: step.status, durationMs: step.duration_ms, costUsd: step.cost_usd })),
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
//# sourceMappingURL=flow-eval-cloud.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"flow-eval-cloud.js","sourceRoot":"","sources":["../src/flow-eval-cloud.ts"],"names":[],"mappings":"AAAA,wEAAwE;AACxE,4EAA4E;AAC5E,wBAAwB;AAExB,OAAO,EAAE,gBAAgB,EAAkB,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAAE,UAAU,EAAE,mBAAmB,EAA0B,MAAM,gBAAgB,CAAC;AACzF,OAAO,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AACtD,OAAO,EAAE,YAAY,EAAE,aAAa,EAA+B,MAAM,iBAAiB,CAAC;AAC3F,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAEvD,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAE,eAAe,EAAE,YAAY,EAAuB,MAAM,wBAAwB,CAAC;AAE5F,MAAM,MAAM,GAAG,oBAAoB,CAAC;AACpC,MAAM,eAAe,GAAG,6BAA6B,CAAC;AACtD,+EAA+E;AAC/E,MAAM,gBAAgB,GAAG,wDAAwD,CAAC;AAElF;;;;;GAKG;AACH,KAAK,UAAU,kBAAkB,CAAC,IAAoB;IACpD,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO;IAC3D,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,eAAe,CAAC,IAAI,CAAC,CAAC;IAC9C,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;IAClF,4EAA4E;IAC5E,kFAAkF;IAClF,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,KAAK,KAAK;WAC9F,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IAClE,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,oFAAoF,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK;cACvH,+CAA+C,CAAC,CAAC;IACvD,CAAC;AACH,CAAC;AAQD;;;;GAIG;AACH,MAAM,UAAU,qBAAqB,CAAC,UAAwC,EAAE;IAC9E,OAAO,KAAK,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE;QACvC,MAAM,kBAAkB,CAAC,IAAI,CAAC,CAAC;QAC/B,MAAM,UAAU,GAA2B;YACzC,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;YACnE,GAAG,CAAC,OAAO,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC;YAChE,GAAG,CAAC,OAAO,CAAC,gBAAgB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,EAAE,CAAC;YACjG,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC;SAC5C,CAAC;QACF,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,IAAI,EAAE;YACrC,GAAG,UAAU;YACb,GAAG,CAAC,OAAO,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC;YAClF,GAAG,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC;YACzE,oEAAoE;YACpE,iEAAiE;YACjE,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,IAAI,IAAI,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC5G,CAAC,CAAC;QACH,MAAM,mBAAmB,CAAC,OAAO,CAAC,KAAK,EAAE;YACvC,GAAG,UAAU,EAAE,GAAG,CAAC,OAAO,CAAC,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,OAAO,CAAC,cAAc,EAAE,CAAC;SAC3G,CAAC,CAAC;QACH,yEAAyE;QACzE,wEAAwE;QACxE,gCAAgC;QAChC,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,0BAA0B,OAAO,CAAC,KAAK,EAAE,EAAE,UAAU,CAAC,CAAC;QACzF,IAAI,KAA8B,CAAC;QACnC,IAAI,CAAC;YACH,KAAK,GAAG,MAAM,gBAAgB,CAAC,OAAO,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QAC5D,CAAC;QAAC,MAAM,CAAC;YACP,kDAAkD;QACpD,CAAC;QACD,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,GAAG,eAAe,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,CAAC,EAAE,GAAG,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;IACnG,CAAC,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,eAAe,CAAC,MAAe,EAAE,KAAa;IAC5D,MAAM,KAAK,GAAG,aAAa,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IAC3C,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;IAC1F,MAAM,MAAM,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;IACxC,yEAAyE;IACzE,IAAI,MAAM,CAAC,gBAAgB,KAAK,oBAAoB;QAAE,OAAO,MAAM,CAAC;IACpE,IAAI,KAAK,CAAC,MAAM,KAAK,aAAa;QAAE,OAAO,EAAE,GAAG,MAAM,EAAE,gBAAgB,EAAE,aAAa,EAAE,CAAC;IAC1F,IAAI,CAAC,CAAC,kBAAkB,IAAI,KAAK,CAAC;QAAE,OAAO,EAAE,GAAG,MAAM,EAAE,gBAAgB,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC;IACzF,IAAI,KAAK,CAAC,gBAAgB,KAAK,SAAS;QAAE,OAAO,MAAM,CAAC,gBAAgB,KAAK,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,GAAG,MAAM,EAAE,gBAAgB,EAAE,SAAS,EAAE,CAAC;IAC9I,OAAO,EAAE,GAAG,MAAM,EAAE,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,EAAE,CAAC;AACjE,CAAC;AAED,qGAAqG;AACrG,MAAM,UAAU,UAAU,CAAC,KAAuC;IAChE,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;IACnF,MAAM,KAAK,GAAG,CAAC,MAA4B,EAAiB,EAAE,CAC5D,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAS,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAE,EAAE,CAAC,CAAC,CAAC;IACvH,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,SAAS,KAAK,OAAO,IAAI,IAAI,CAAC,SAAS,KAAK,KAAK,IAAI,IAAI,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC;IACtH,OAAO;QACL,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACjD,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACnD,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACrD,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,UAAU,EAAE,IAAI,CAAC,WAAW,EAAE,OAAO,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;KAC9H,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
export type FlowEvalErrorCode = 'invalid_suite' | 'invalid_baseline' | 'version_mismatch' | 'suite_mismatch' | 'invalid_options' | 'unreadable_flow';
|
|
2
|
+
/** The evaluation itself is invalid. Case failures are verdicts in the report, never this. */
|
|
3
|
+
export declare class FlowEvalError extends Error {
|
|
4
|
+
readonly code: FlowEvalErrorCode;
|
|
5
|
+
constructor(code: FlowEvalErrorCode, message: string);
|
|
6
|
+
}
|
|
7
|
+
//# sourceMappingURL=flow-eval-error.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"flow-eval-error.d.ts","sourceRoot":"","sources":["../src/flow-eval-error.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,iBAAiB,GACzB,eAAe,GAAG,kBAAkB,GAAG,kBAAkB,GAAG,gBAAgB,GAAG,iBAAiB,GAAG,iBAAiB,CAAC;AAEzH,8FAA8F;AAC9F,qBAAa,aAAc,SAAQ,KAAK;IAC1B,QAAQ,CAAC,IAAI,EAAE,iBAAiB;gBAAvB,IAAI,EAAE,iBAAiB,EAAE,OAAO,EAAE,MAAM;CAI9D"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/** The evaluation itself is invalid. Case failures are verdicts in the report, never this. */
|
|
2
|
+
export class FlowEvalError extends Error {
|
|
3
|
+
code;
|
|
4
|
+
constructor(code, message) {
|
|
5
|
+
super(message);
|
|
6
|
+
this.code = code;
|
|
7
|
+
this.name = 'FlowEvalError';
|
|
8
|
+
}
|
|
9
|
+
}
|
|
10
|
+
//# sourceMappingURL=flow-eval-error.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"flow-eval-error.js","sourceRoot":"","sources":["../src/flow-eval-error.ts"],"names":[],"mappings":"AAGA,8FAA8F;AAC9F,MAAM,OAAO,aAAc,SAAQ,KAAK;IACjB;IAArB,YAAqB,IAAuB,EAAE,OAAe;QAC3D,KAAK,CAAC,OAAO,CAAC,CAAC;QADI,SAAI,GAAJ,IAAI,CAAmB;QAE1C,IAAI,CAAC,IAAI,GAAG,eAAe,CAAC;IAC9B,CAAC;CACF"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { FlowEvalCaseResult, FlowEvalOutcome, FlowEvalReport } from './flow-eval-report.js';
|
|
2
|
+
import { type FlowEvalSuite } from './flow-eval-suite.js';
|
|
3
|
+
/** The part of a prior report a regression check reads, validated. */
|
|
4
|
+
export interface FlowEvalBaseline {
|
|
5
|
+
suiteSha256: string;
|
|
6
|
+
outcomes: ReadonlyMap<string, FlowEvalOutcome>;
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* Validate an untrusted prior report (parsed JSON) before any case runs, so a
|
|
10
|
+
* malformed baseline is refused before an evaluation spends anything.
|
|
11
|
+
*/
|
|
12
|
+
export declare function parseFlowEvalBaseline(value: unknown): FlowEvalBaseline;
|
|
13
|
+
export declare function summarizeFlowEval(results: readonly FlowEvalCaseResult[]): FlowEvalReport['summary'];
|
|
14
|
+
export declare function decideFlowEvalGate(suite: FlowEvalSuite, results: readonly FlowEvalCaseResult[], summary: FlowEvalReport['summary'], baseline: FlowEvalBaseline | undefined): FlowEvalReport['gate'];
|
|
15
|
+
//# sourceMappingURL=flow-eval-gate.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"flow-eval-gate.d.ts","sourceRoot":"","sources":["../src/flow-eval-gate.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,kBAAkB,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACjG,OAAO,EAAY,KAAK,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAEpE,sEAAsE;AACtE,MAAM,WAAW,gBAAgB;IAC/B,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,WAAW,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;CAChD;AAID;;;GAGG;AACH,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,OAAO,GAAG,gBAAgB,CAiBtE;AAED,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,SAAS,kBAAkB,EAAE,GAAG,cAAc,CAAC,SAAS,CAAC,CAYnG;AAED,wBAAgB,kBAAkB,CAChC,KAAK,EAAE,aAAa,EACpB,OAAO,EAAE,SAAS,kBAAkB,EAAE,EACtC,OAAO,EAAE,cAAc,CAAC,SAAS,CAAC,EAClC,QAAQ,EAAE,gBAAgB,GAAG,SAAS,GACrC,cAAc,CAAC,MAAM,CAAC,CAkCxB"}
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
// The verdict of a flow evaluation: case results aggregated into a summary,
|
|
2
|
+
// and the summary judged against the suite's thresholds and a baseline.
|
|
3
|
+
import { FlowEvalError } from './flow-eval-error.js';
|
|
4
|
+
import { isRecord } from './flow-eval-suite.js';
|
|
5
|
+
const OUTCOMES = ['pass', 'fail', 'error'];
|
|
6
|
+
/**
|
|
7
|
+
* Validate an untrusted prior report (parsed JSON) before any case runs, so a
|
|
8
|
+
* malformed baseline is refused before an evaluation spends anything.
|
|
9
|
+
*/
|
|
10
|
+
export function parseFlowEvalBaseline(value) {
|
|
11
|
+
const fail = (message) => { throw new FlowEvalError('invalid_baseline', message); };
|
|
12
|
+
if (!isRecord(value) || value['kind'] !== 'flows.eval.report')
|
|
13
|
+
fail('Baseline is not a flows eval report.');
|
|
14
|
+
const report = value;
|
|
15
|
+
const suite = report['suite'];
|
|
16
|
+
if (!isRecord(suite) || typeof suite['sha256'] !== 'string')
|
|
17
|
+
fail('Baseline report names no suite sha256.');
|
|
18
|
+
if (!Array.isArray(report['cases']))
|
|
19
|
+
fail('Baseline report has no `cases` array.');
|
|
20
|
+
const outcomes = new Map();
|
|
21
|
+
for (const [index, entry] of report['cases'].entries()) {
|
|
22
|
+
if (!isRecord(entry) || typeof entry['id'] !== 'string' || !OUTCOMES.includes(entry['outcome'])) {
|
|
23
|
+
fail(`Baseline cases[${index}] needs a string \`id\` and an \`outcome\` of pass, fail or error.`);
|
|
24
|
+
}
|
|
25
|
+
const item = entry;
|
|
26
|
+
if (outcomes.has(item['id']))
|
|
27
|
+
fail(`Baseline case id "${item['id']}" is not unique.`);
|
|
28
|
+
outcomes.set(item['id'], item['outcome']);
|
|
29
|
+
}
|
|
30
|
+
return { suiteSha256: suite['sha256'], outcomes };
|
|
31
|
+
}
|
|
32
|
+
export function summarizeFlowEval(results) {
|
|
33
|
+
const passed = results.filter(r => r.outcome === 'pass').length;
|
|
34
|
+
const failed = results.filter(r => r.outcome === 'fail').length;
|
|
35
|
+
const errored = results.filter(r => r.outcome === 'error').length;
|
|
36
|
+
const costs = results.map(r => r.costUsd);
|
|
37
|
+
const latencies = results.map(r => r.latencyMs).sort((a, b) => a - b);
|
|
38
|
+
return {
|
|
39
|
+
total: results.length, passed, failed, errored,
|
|
40
|
+
passRate: results.length === 0 ? 0 : passed / results.length,
|
|
41
|
+
totalCostUsd: costs.some(c => c === null) ? null : fromNanos(sumNanos(costs)),
|
|
42
|
+
latencyMs: { p50: percentile(latencies, 0.5), p95: percentile(latencies, 0.95), max: latencies.at(-1) ?? 0 },
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
export function decideFlowEvalGate(suite, results, summary, baseline) {
|
|
46
|
+
const reasons = [];
|
|
47
|
+
const thresholds = suite.thresholds ?? {};
|
|
48
|
+
const minPassRate = thresholds.minPassRate ?? 1;
|
|
49
|
+
if (summary.passRate < minPassRate) {
|
|
50
|
+
reasons.push(`pass rate ${formatRate(summary.passRate)} is below ${formatRate(minPassRate)} (${summary.passed}/${summary.total})`);
|
|
51
|
+
}
|
|
52
|
+
// Fails closed regardless of minPassRate: an errored case was never judged,
|
|
53
|
+
// so a tolerance for quality misses must not absorb it.
|
|
54
|
+
if (summary.errored > 0)
|
|
55
|
+
reasons.push(`${summary.errored} case(s) errored before a verdict`);
|
|
56
|
+
if (thresholds.maxTotalCostUsd !== undefined) {
|
|
57
|
+
// Decimal, not binary: costs and the ceiling are compared in whole
|
|
58
|
+
// nanodollars, so 0.1 + 0.2 meets a 0.3 ceiling exactly and a total a
|
|
59
|
+
// fraction of a cent over it still fails. No display rounding is involved.
|
|
60
|
+
const costs = results.map(r => r.costUsd);
|
|
61
|
+
if (costs.some(c => c === null))
|
|
62
|
+
reasons.push(`total cost is unknown; the suite caps it at $${thresholds.maxTotalCostUsd}`);
|
|
63
|
+
else {
|
|
64
|
+
const total = sumNanos(costs);
|
|
65
|
+
if (total > toNanos(thresholds.maxTotalCostUsd)) {
|
|
66
|
+
reasons.push(`total cost $${fromNanos(total)} exceeds $${thresholds.maxTotalCostUsd}`);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
if (thresholds.maxP95LatencyMs !== undefined && summary.latencyMs.p95 > thresholds.maxP95LatencyMs) {
|
|
71
|
+
reasons.push(`p95 latency ${summary.latencyMs.p95}ms exceeds ${thresholds.maxP95LatencyMs}ms`);
|
|
72
|
+
}
|
|
73
|
+
const regressions = [];
|
|
74
|
+
if (baseline !== undefined) {
|
|
75
|
+
for (const result of results) {
|
|
76
|
+
if (baseline.outcomes.get(result.id) === 'pass' && result.outcome !== 'pass')
|
|
77
|
+
regressions.push(result.id);
|
|
78
|
+
}
|
|
79
|
+
if (regressions.length > 0)
|
|
80
|
+
reasons.push(`${regressions.length} case(s) regressed against the baseline: ${regressions.join(', ')}`);
|
|
81
|
+
}
|
|
82
|
+
return { pass: reasons.length === 0, reasons, regressions };
|
|
83
|
+
}
|
|
84
|
+
function percentile(sorted, p) {
|
|
85
|
+
if (sorted.length === 0)
|
|
86
|
+
return 0;
|
|
87
|
+
return sorted[Math.min(sorted.length - 1, Math.max(0, Math.ceil(p * sorted.length) - 1))];
|
|
88
|
+
}
|
|
89
|
+
/** Dollars as whole nanodollars: exact for any cost a run reports, far below a cent. */
|
|
90
|
+
function toNanos(usd) {
|
|
91
|
+
return BigInt(Math.round(usd * 1e9));
|
|
92
|
+
}
|
|
93
|
+
function sumNanos(costs) {
|
|
94
|
+
return costs.reduce((sum, cost) => sum + toNanos(cost), 0n);
|
|
95
|
+
}
|
|
96
|
+
function fromNanos(nanos) {
|
|
97
|
+
return Number(nanos) / 1e9;
|
|
98
|
+
}
|
|
99
|
+
function formatRate(rate) {
|
|
100
|
+
return `${Math.round(rate * 1000) / 10}%`;
|
|
101
|
+
}
|
|
102
|
+
//# sourceMappingURL=flow-eval-gate.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"flow-eval-gate.js","sourceRoot":"","sources":["../src/flow-eval-gate.ts"],"names":[],"mappings":"AAAA,4EAA4E;AAC5E,wEAAwE;AAExE,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAErD,OAAO,EAAE,QAAQ,EAAsB,MAAM,sBAAsB,CAAC;AAQpE,MAAM,QAAQ,GAAsB,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;AAE9D;;;GAGG;AACH,MAAM,UAAU,qBAAqB,CAAC,KAAc;IAClD,MAAM,IAAI,GAAG,CAAC,OAAe,EAAS,EAAE,GAAG,MAAM,IAAI,aAAa,CAAC,kBAAkB,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IACnG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,mBAAmB;QAAE,IAAI,CAAC,sCAAsC,CAAC,CAAC;IAC5G,MAAM,MAAM,GAAG,KAAgC,CAAC;IAChD,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;IAC9B,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,QAAQ,CAAC,KAAK,QAAQ;QAAE,IAAI,CAAC,wCAAwC,CAAC,CAAC;IAC5G,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAAE,IAAI,CAAC,uCAAuC,CAAC,CAAC;IACnF,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA2B,CAAC;IACpD,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAK,MAAM,CAAC,OAAO,CAAe,CAAC,OAAO,EAAE,EAAE,CAAC;QACtE,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,QAAQ,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,SAAS,CAAW,CAAC,EAAE,CAAC;YAC1G,IAAI,CAAC,kBAAkB,KAAK,oEAAoE,CAAC,CAAC;QACpG,CAAC;QACD,MAAM,IAAI,GAAG,KAAgC,CAAC;QAC9C,IAAI,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAW,CAAC;YAAE,IAAI,CAAC,qBAAqB,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAChG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAW,EAAE,IAAI,CAAC,SAAS,CAAoB,CAAC,CAAC;IACzE,CAAC;IACD,OAAO,EAAE,WAAW,EAAG,KAAiC,CAAC,QAAQ,CAAW,EAAE,QAAQ,EAAE,CAAC;AAC3F,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,OAAsC;IACtE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;IAChE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;IAChE,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,OAAO,CAAC,CAAC,MAAM,CAAC;IAClE,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IAC1C,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACtE,OAAO;QACL,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO;QAC9C,QAAQ,EAAE,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM;QAC5D,YAAY,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAiB,CAAC,CAAC;QACzF,SAAS,EAAE,EAAE,GAAG,EAAE,UAAU,CAAC,SAAS,EAAE,GAAG,CAAC,EAAE,GAAG,EAAE,UAAU,CAAC,SAAS,EAAE,IAAI,CAAC,EAAE,GAAG,EAAE,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE;KAC7G,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,kBAAkB,CAChC,KAAoB,EACpB,OAAsC,EACtC,OAAkC,EAClC,QAAsC;IAEtC,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,UAAU,GAAG,KAAK,CAAC,UAAU,IAAI,EAAE,CAAC;IAC1C,MAAM,WAAW,GAAG,UAAU,CAAC,WAAW,IAAI,CAAC,CAAC;IAChD,IAAI,OAAO,CAAC,QAAQ,GAAG,WAAW,EAAE,CAAC;QACnC,OAAO,CAAC,IAAI,CAAC,aAAa,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,aAAa,UAAU,CAAC,WAAW,CAAC,KAAK,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,KAAK,GAAG,CAAC,CAAC;IACrI,CAAC;IACD,4EAA4E;IAC5E,wDAAwD;IACxD,IAAI,OAAO,CAAC,OAAO,GAAG,CAAC;QAAE,OAAO,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,OAAO,mCAAmC,CAAC,CAAC;IAC7F,IAAI,UAAU,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;QAC7C,mEAAmE;QACnE,sEAAsE;QACtE,2EAA2E;QAC3E,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAC1C,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC;YAAE,OAAO,CAAC,IAAI,CAAC,gDAAgD,UAAU,CAAC,eAAe,EAAE,CAAC,CAAC;aACvH,CAAC;YACJ,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAiB,CAAC,CAAC;YAC1C,IAAI,KAAK,GAAG,OAAO,CAAC,UAAU,CAAC,eAAe,CAAC,EAAE,CAAC;gBAChD,OAAO,CAAC,IAAI,CAAC,eAAe,SAAS,CAAC,KAAK,CAAC,aAAa,UAAU,CAAC,eAAe,EAAE,CAAC,CAAC;YACzF,CAAC;QACH,CAAC;IACH,CAAC;IACD,IAAI,UAAU,CAAC,eAAe,KAAK,SAAS,IAAI,OAAO,CAAC,SAAS,CAAC,GAAG,GAAG,UAAU,CAAC,eAAe,EAAE,CAAC;QACnG,OAAO,CAAC,IAAI,CAAC,eAAe,OAAO,CAAC,SAAS,CAAC,GAAG,cAAc,UAAU,CAAC,eAAe,IAAI,CAAC,CAAC;IACjG,CAAC;IACD,MAAM,WAAW,GAAa,EAAE,CAAC;IACjC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,MAAM,IAAI,MAAM,CAAC,OAAO,KAAK,MAAM;gBAAE,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAC5G,CAAC;QACD,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,MAAM,4CAA4C,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACtI,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;AAC9D,CAAC;AAED,SAAS,UAAU,CAAC,MAAyB,EAAE,CAAS;IACtD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAClC,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC;AAC7F,CAAC;AAED,wFAAwF;AACxF,SAAS,OAAO,CAAC,GAAW;IAC1B,OAAO,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC;AACvC,CAAC;AAED,SAAS,QAAQ,CAAC,KAAwB;IACxC,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;AAC9D,CAAC;AAED,SAAS,SAAS,CAAC,KAAa;IAC9B,OAAO,MAAM,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC;AAC7B,CAAC;AAED,SAAS,UAAU,CAAC,IAAY;IAC9B,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC;AAC5C,CAAC"}
|