@parcha/agentrun-dsl 0.1.0-beta.3 → 0.1.0-beta.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +165 -21
- package/dist/author-references.d.ts +8 -0
- package/dist/author-references.js +8 -0
- package/dist/author.d.ts +16 -3
- package/dist/author.js +40 -16
- package/dist/authoring.js +2 -2
- package/dist/cli.js +17 -12
- package/dist/code-exec.d.ts +6 -0
- package/dist/code-exec.js +126 -0
- package/dist/decision-provider.d.ts +157 -0
- package/dist/decision-provider.js +230 -0
- package/dist/decision-receipts.d.ts +143 -0
- package/dist/decision-receipts.js +93 -0
- package/dist/dry-run.d.ts +14 -1
- package/dist/dry-run.js +58 -15
- package/dist/graph.d.ts +169 -0
- package/dist/graph.js +433 -0
- package/dist/index.d.ts +15 -8
- package/dist/index.js +8 -4
- package/dist/inspection.d.ts +5 -0
- package/dist/inspection.js +16 -3
- package/dist/job.d.ts +14 -0
- package/dist/job.js +5 -0
- package/dist/long-timer.d.ts +7 -0
- package/dist/long-timer.js +22 -0
- package/dist/observer-snapshot.d.ts +3 -0
- package/dist/observer-snapshot.js +46 -0
- package/dist/predicates.d.ts +10 -0
- package/dist/predicates.js +18 -0
- package/dist/recovery/binding.d.ts +10 -0
- package/dist/recovery/binding.js +43 -0
- package/dist/recovery/call-key.d.ts +13 -0
- package/dist/recovery/call-key.js +24 -0
- package/dist/recovery/canonical-hash.d.ts +7 -0
- package/dist/recovery/canonical-hash.js +22 -0
- package/dist/recovery/driver.d.ts +238 -0
- package/dist/recovery/driver.js +1337 -0
- package/dist/recovery/effect-failure.d.ts +34 -0
- package/dist/recovery/effect-failure.js +62 -0
- package/dist/recovery/errors.d.ts +31 -0
- package/dist/recovery/errors.js +51 -0
- package/dist/recovery/execution-path.d.ts +53 -0
- package/dist/recovery/execution-path.js +167 -0
- package/dist/recovery/file-store.d.ts +7 -0
- package/dist/recovery/file-store.js +131 -0
- package/dist/recovery/frozen-snapshot.d.ts +274 -0
- package/dist/recovery/frozen-snapshot.js +187 -0
- package/dist/recovery/handoff-block.d.ts +36 -0
- package/dist/recovery/handoff-block.js +148 -0
- package/dist/recovery/handoff-digest.d.ts +58 -0
- package/dist/recovery/handoff-digest.js +195 -0
- package/dist/recovery/handoff-files.d.ts +54 -0
- package/dist/recovery/handoff-files.js +162 -0
- package/dist/recovery/handoff-receipts.d.ts +42 -0
- package/dist/recovery/handoff-receipts.js +53 -0
- package/dist/recovery/handoff-redact.d.ts +12 -0
- package/dist/recovery/handoff-redact.js +44 -0
- package/dist/recovery/handoff.d.ts +44 -0
- package/dist/recovery/handoff.js +46 -0
- package/dist/recovery/index.d.ts +19 -0
- package/dist/recovery/index.js +17 -0
- package/dist/recovery/item-state.d.ts +48 -0
- package/dist/recovery/item-state.js +89 -0
- package/dist/recovery/journal.d.ts +28 -0
- package/dist/recovery/journal.js +139 -0
- package/dist/recovery/memory-store.d.ts +4 -0
- package/dist/recovery/memory-store.js +20 -0
- package/dist/recovery/store.d.ts +105 -0
- package/dist/recovery/store.js +6 -0
- package/dist/recovery/testing/conformance.d.ts +4 -0
- package/dist/recovery/testing/conformance.js +268 -0
- package/dist/recovery/testing/crash-points.d.ts +18 -0
- package/dist/recovery/testing/crash-points.js +43 -0
- package/dist/recovery/testing/crash-sweep.d.ts +65 -0
- package/dist/recovery/testing/crash-sweep.js +146 -0
- package/dist/recovery/testing/index.d.ts +3 -0
- package/dist/recovery/testing/index.js +4 -0
- package/dist/recovery/testing/preload.d.ts +1 -0
- package/dist/recovery/testing/preload.js +3 -0
- package/dist/recovery/transport-facts.d.ts +23 -0
- package/dist/recovery/transport-facts.js +46 -0
- package/dist/recovery/with-recovery.d.ts +50 -0
- package/dist/recovery/with-recovery.js +142 -0
- package/dist/submit-envelope.d.ts +9 -0
- package/dist/submit-envelope.js +40 -0
- package/dist/system-one.d.ts +40 -1
- package/dist/system-one.js +141 -5
- package/dist/testing/decision-provider.d.ts +32 -0
- package/dist/testing/decision-provider.js +183 -0
- package/dist/testing/index.d.ts +1 -0
- package/dist/testing/index.js +2 -0
- package/dist/vocabulary.d.ts +22 -3
- package/dist/vocabulary.js +33 -10
- package/dist/workflow-shape.d.ts +7 -0
- package/dist/workflow-shape.js +115 -67
- package/dist/workflow.d.ts +269 -20
- package/dist/workflow.js +1251 -341
- package/package.json +25 -2
- package/schema/workflow.schema.json +506 -12
- package/skills/author/SKILL.md +6 -1
- package/skills/author/references/jev-decisions.md +58 -6
- package/skills/author/references/language.md +28 -19
- package/skills/author/references/sdk-authoring.md +2 -0
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import { type WorkflowDeps } from "../workflow.js";
|
|
2
|
+
import type { openRecovery, RecoveryEffectParams } from "./driver.js";
|
|
3
|
+
import { DECISION_INTERRUPTED } from "../decision-receipts.js";
|
|
4
|
+
export { DECISION_INTERRUPTED };
|
|
5
|
+
/** A recovery driver, as `openRecovery` returns it. */
|
|
6
|
+
export type RecoveryDriver = Awaited<ReturnType<typeof openRecovery>>;
|
|
7
|
+
/** The attempt of an LLM step a node runner is asked to run, as the driver admitted it. One attempt is one session:
|
|
8
|
+
* a runner that keeps what an attempt delivered keys it by `sessionId`. */
|
|
9
|
+
export type RecoveryStep = {
|
|
10
|
+
sessionId: string;
|
|
11
|
+
attempt: number;
|
|
12
|
+
earlierSessionIds: readonly string[];
|
|
13
|
+
};
|
|
14
|
+
/** What a host's node runner receives under the driver: the interpreter's parameters, and the attempt it runs. A
|
|
15
|
+
* runner marks a failure no second attempt should follow (one that ended on a bound of its own) with `final: true`
|
|
16
|
+
* on what it throws. */
|
|
17
|
+
export type RecoveryNodeParams = Parameters<NonNullable<WorkflowDeps["runNode"]>>[0] & {
|
|
18
|
+
step?: RecoveryStep;
|
|
19
|
+
};
|
|
20
|
+
/** The host's adapters as `withRecovery` takes them: the interpreter's, with the two that learn more under the driver. */
|
|
21
|
+
export type RecoveryAdapters = Omit<WorkflowDeps, "runEffect" | "runNode"> & {
|
|
22
|
+
runEffect?: (params: RecoveryEffectParams) => Promise<unknown>;
|
|
23
|
+
runNode?: (params: RecoveryNodeParams) => Promise<unknown>;
|
|
24
|
+
};
|
|
25
|
+
export type WithRecoveryOptions = {
|
|
26
|
+
/** The node runner keeps what each attempt delivered: asked again for a `sessionId` whose record it holds, it
|
|
27
|
+
* returns that record without running anything. A step found delivered is then asked of it again as the same
|
|
28
|
+
* attempt, and nothing is spent. Absent, such a step has spent its attempt.
|
|
29
|
+
* A record the interpreter refuses is therefore returned, and refused, again at every later open: the step neither
|
|
30
|
+
* advances nor spends, and the run fails the same way each time until the host ends it (an escalation, a new run). */
|
|
31
|
+
durableNodes?: boolean;
|
|
32
|
+
};
|
|
33
|
+
/** The dependencies to run a workflow with under `driver`. The interpreter's `recovery` hooks cover nodes; three
|
|
34
|
+
* things they have no slot for are applied to the host's adapters here:
|
|
35
|
+
* - `runEffect`: each effect is admitted before it is dispatched, a completed one is answered from its receipt, and
|
|
36
|
+
* one admitted and never completed is refused;
|
|
37
|
+
* - `runJudge`: each request is committed (`decision.request`) before the judge is asked, and ends with one outcome:
|
|
38
|
+
* answered, or failed in its own process (asked again as its next attempt). A later open that finds it asked and
|
|
39
|
+
* never answered (the process died, or a cancel cut the call) books it once at an unknown cost and ends the run
|
|
40
|
+
* escalated (`decision_interrupted`) instead of asking again. A route's answer is committed as its decision before
|
|
41
|
+
* any branch step runs, and a resume inside the branch follows it without asking again; every answer's spend is
|
|
42
|
+
* committed with the driver's state;
|
|
43
|
+
* - `runNode`: an LLM step's record is committed before the step runs, with two attempts, each handed to the
|
|
44
|
+
* runner as its `step`. A failed attempt is followed at once by the second, unless the runner marked the
|
|
45
|
+
* failure final; a step that spent both is refused at every later open. A process that dies inside a step spends
|
|
46
|
+
* no attempt. A step that delivered a submission and whose node never committed (the interpreter refused it, or
|
|
47
|
+
* the process died first) is answered again from the runner's own record under `durableNodes`, and has spent its
|
|
48
|
+
* attempt otherwise.
|
|
49
|
+
* The run stops through the driver's signal, together with the host's own. */
|
|
50
|
+
export declare function withRecovery(driver: RecoveryDriver, deps: RecoveryAdapters, options?: WithRecoveryOptions): WorkflowDeps;
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
import { chargedCost, DecisionProviderError } from "../decision-provider.js";
|
|
2
|
+
import { validateAnswers } from "../system-one.js";
|
|
3
|
+
import { EscalationSignal } from "../workflow.js";
|
|
4
|
+
import { canonicalHash } from "./canonical-hash.js";
|
|
5
|
+
import { DECISION_INTERRUPTED } from "../decision-receipts.js";
|
|
6
|
+
export { DECISION_INTERRUPTED };
|
|
7
|
+
/** The dependencies to run a workflow with under `driver`. The interpreter's `recovery` hooks cover nodes; three
|
|
8
|
+
* things they have no slot for are applied to the host's adapters here:
|
|
9
|
+
* - `runEffect`: each effect is admitted before it is dispatched, a completed one is answered from its receipt, and
|
|
10
|
+
* one admitted and never completed is refused;
|
|
11
|
+
* - `runJudge`: each request is committed (`decision.request`) before the judge is asked, and ends with one outcome:
|
|
12
|
+
* answered, or failed in its own process (asked again as its next attempt). A later open that finds it asked and
|
|
13
|
+
* never answered (the process died, or a cancel cut the call) books it once at an unknown cost and ends the run
|
|
14
|
+
* escalated (`decision_interrupted`) instead of asking again. A route's answer is committed as its decision before
|
|
15
|
+
* any branch step runs, and a resume inside the branch follows it without asking again; every answer's spend is
|
|
16
|
+
* committed with the driver's state;
|
|
17
|
+
* - `runNode`: an LLM step's record is committed before the step runs, with two attempts, each handed to the
|
|
18
|
+
* runner as its `step`. A failed attempt is followed at once by the second, unless the runner marked the
|
|
19
|
+
* failure final; a step that spent both is refused at every later open. A process that dies inside a step spends
|
|
20
|
+
* no attempt. A step that delivered a submission and whose node never committed (the interpreter refused it, or
|
|
21
|
+
* the process died first) is answered again from the runner's own record under `durableNodes`, and has spent its
|
|
22
|
+
* attempt otherwise.
|
|
23
|
+
* The run stops through the driver's signal, together with the host's own. */
|
|
24
|
+
export function withRecovery(driver, deps, options = {}) {
|
|
25
|
+
const { runEffect, runJudge, runNode } = deps;
|
|
26
|
+
return {
|
|
27
|
+
...deps,
|
|
28
|
+
recovery: driver.recovery,
|
|
29
|
+
signal: deps.signal ? AbortSignal.any([deps.signal, driver.signal]) : driver.signal,
|
|
30
|
+
runEffect: runEffect && driver.wrapEffect(runEffect),
|
|
31
|
+
runJudge: runJudge && (async (params) => {
|
|
32
|
+
const decided = params.kind === "route" ? driver.routeDecision(params.executionPath) : undefined;
|
|
33
|
+
if (decided)
|
|
34
|
+
return decided.result;
|
|
35
|
+
await driver.seam();
|
|
36
|
+
const decisionId = params.decisionId ?? canonicalHash(["decision-request", params.executionPath, params.kind, params.questions, params.state]);
|
|
37
|
+
const open = driver.admitDecision(params.executionPath, { decision_id: decisionId, kind: params.kind, label: params.label, model: params.model ?? null,
|
|
38
|
+
request_sha256: canonicalHash({ kind: params.kind, label: params.label, state: params.state, questions: params.questions }) });
|
|
39
|
+
if (open) {
|
|
40
|
+
// An earlier process asked and its answer never reached a receipt: never asked again, booked once at an unknown
|
|
41
|
+
// cost, and the run goes to review. The unknown cost is counted with the outcome in one commit, the way the
|
|
42
|
+
// answered and failed paths commit theirs, so the run's spend reads incomplete and never omits it.
|
|
43
|
+
if (open.status === "requested")
|
|
44
|
+
driver.decisionEnded(params.executionPath, decisionId, "interrupted", null, () => driver.recordQuestionSpend(null));
|
|
45
|
+
await driver.flush();
|
|
46
|
+
throw new EscalationSignal({ kind: DECISION_INTERRUPTED, stage: "decision", executionPath: params.executionPath, label: params.label, state: driver.lastState(),
|
|
47
|
+
summary: `The decision "${params.label}" was asked by an earlier process and its answer never reached a receipt; it is not asked again.` });
|
|
48
|
+
}
|
|
49
|
+
try {
|
|
50
|
+
// The request is durable before the judge is asked.
|
|
51
|
+
await driver.flush();
|
|
52
|
+
let result;
|
|
53
|
+
try {
|
|
54
|
+
result = await runJudge(params);
|
|
55
|
+
}
|
|
56
|
+
catch (error) {
|
|
57
|
+
// A request sent whose outcome was lost may have been answered and billed: it is interrupted, never asked again,
|
|
58
|
+
// and its estimate (or an unknown cost, which makes the spend read incomplete) is booked with that outcome.
|
|
59
|
+
const charge = chargedCost(error);
|
|
60
|
+
if (error instanceof DecisionProviderError && error.code === "outcome_unknown") {
|
|
61
|
+
driver.decisionEnded(params.executionPath, decisionId, "interrupted", charge ?? null, () => driver.recordQuestionSpend(charge ?? null));
|
|
62
|
+
throw error;
|
|
63
|
+
}
|
|
64
|
+
// An answer rejected after it was paid for failed, and its cost (or an unknown one) is booked with that outcome:
|
|
65
|
+
// the next process asks again.
|
|
66
|
+
if (charge !== undefined) {
|
|
67
|
+
driver.decisionEnded(params.executionPath, decisionId, "failed", charge, () => driver.recordQuestionSpend(charge));
|
|
68
|
+
throw error;
|
|
69
|
+
}
|
|
70
|
+
// A cut call (a cancel, the run's failure) has no outcome: a later open finds it asked and unanswered. A call
|
|
71
|
+
// that ended on its own failed, and the next process asks again.
|
|
72
|
+
if (!driver.signal.aborted && !params.signal?.aborted)
|
|
73
|
+
driver.decisionEnded(params.executionPath, decisionId, "failed", null);
|
|
74
|
+
throw error;
|
|
75
|
+
}
|
|
76
|
+
const cost = typeof result.cost_usd === "number" && Number.isFinite(result.cost_usd) ? result.cost_usd : null;
|
|
77
|
+
// An answer the interpreter would refuse is spent at its price and is never a decision: the question is
|
|
78
|
+
// asked again by the next process.
|
|
79
|
+
try {
|
|
80
|
+
validateAnswers(params.questions, result.answers);
|
|
81
|
+
}
|
|
82
|
+
catch (error) {
|
|
83
|
+
driver.decisionEnded(params.executionPath, decisionId, "failed", cost, () => driver.recordQuestionSpend(cost));
|
|
84
|
+
throw error;
|
|
85
|
+
}
|
|
86
|
+
// The outcome, the spend and a route's decision are one commit.
|
|
87
|
+
driver.decisionEnded(params.executionPath, decisionId, "answered", cost, () => driver.recordQuestionSpend(cost, params.kind === "route"
|
|
88
|
+
? { executionPath: params.executionPath, label: params.label, result: result, receipts: null } : undefined));
|
|
89
|
+
// The decision is durable before the interpreter acts on the answer, and a stop taken meanwhile ends the run here.
|
|
90
|
+
await driver.flush();
|
|
91
|
+
driver.checkStop();
|
|
92
|
+
return result;
|
|
93
|
+
}
|
|
94
|
+
finally {
|
|
95
|
+
driver.decisionSettled();
|
|
96
|
+
}
|
|
97
|
+
}),
|
|
98
|
+
runNode: runNode && (async (params) => {
|
|
99
|
+
const admit = () => driver.stepSession(params.executionPath, params.label, { attemptsAllowed: 2 });
|
|
100
|
+
let admission = admit();
|
|
101
|
+
for (;;) {
|
|
102
|
+
// An attempt that delivered a submission is asked for again only when its node never committed. A runner that
|
|
103
|
+
// keeps what it delivered answers it again below, as the same attempt; with any other runner it is spent.
|
|
104
|
+
if (admission.status === "submitted" && !options.durableNodes) {
|
|
105
|
+
driver.stepAttemptFailed(params.executionPath);
|
|
106
|
+
admission = admit();
|
|
107
|
+
}
|
|
108
|
+
// A step that failed on an item failure is refused with it, so the item it lies in ends on the same failure.
|
|
109
|
+
if (admission.status === "failed") {
|
|
110
|
+
throw admission.failure ? Object.assign(new Error(admission.failure.message), { code: admission.failure.code, itemFailure: true })
|
|
111
|
+
: new Error(`${params.label} failed after ${admission.attemptsAllowed} attempts`);
|
|
112
|
+
}
|
|
113
|
+
// The step's record is durable before the attempt runs. A stop or a caller's cancellation that landed
|
|
114
|
+
// meanwhile ends the step here: the runner is not called, and no attempt is spent. A pause waiting for an
|
|
115
|
+
// effect in flight holds the step here until it is taken.
|
|
116
|
+
await driver.flush();
|
|
117
|
+
await driver.seam();
|
|
118
|
+
if (params.signal?.aborted)
|
|
119
|
+
throw params.signal.reason;
|
|
120
|
+
try {
|
|
121
|
+
const submission = await runNode({ ...params, step: { sessionId: admission.sessionId, attempt: admission.attempt, earlierSessionIds: admission.earlierSessionIds } });
|
|
122
|
+
driver.stepSubmitted(params.executionPath);
|
|
123
|
+
return submission;
|
|
124
|
+
}
|
|
125
|
+
catch (error) {
|
|
126
|
+
// A stop the driver took, or a sibling's failure, ends the step where it is: no attempt is spent.
|
|
127
|
+
if (driver.signal.aborted || params.signal?.aborted)
|
|
128
|
+
throw error;
|
|
129
|
+
// A failure the runner marked final (an attempt that ended on a bound of its own) gets no second attempt: it
|
|
130
|
+
// would spend the same again.
|
|
131
|
+
const final = error?.final === true;
|
|
132
|
+
driver.stepAttemptFailed(params.executionPath, final, error);
|
|
133
|
+
if (final)
|
|
134
|
+
throw error;
|
|
135
|
+
admission = admit();
|
|
136
|
+
if (admission.status !== "running")
|
|
137
|
+
throw error;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
}),
|
|
141
|
+
};
|
|
142
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/** The input schema of a node runner's `submit` tool: the node's output schema under `value`, as
|
|
2
|
+
* `{ value: <schema> }`. A node schema is not a valid tool input as it stands (a root `$ref` or `$defs`
|
|
3
|
+
* is not an object schema), and embedding it moves the document root its local pointers resolve
|
|
4
|
+
* against, so each local `$ref` (`#`, `#/...`) is relocated under `#/properties/value`. A subschema
|
|
5
|
+
* that sets its own `$id` is a separate resource: its local references do not move. Every runner that
|
|
6
|
+
* delivers a node's record through a tool call uses this one envelope, so a schema behaves the same on
|
|
7
|
+
* every harness. A keyword of the DSL's own (`criteria`) is left out of every schema in it, never a property or
|
|
8
|
+
* definition of that name; the submission is still checked against the node's schema as written. */
|
|
9
|
+
export declare function submitParameters(schema: unknown): Record<string, unknown>;
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
// DSL vocabulary that may sit on a schema but is not JSON Schema: System One's question keyword. The model never sees it.
|
|
2
|
+
const DSL_ONLY_SCHEMA_KEYWORDS = new Set(["criteria"]);
|
|
3
|
+
/** The input schema of a node runner's `submit` tool: the node's output schema under `value`, as
|
|
4
|
+
* `{ value: <schema> }`. A node schema is not a valid tool input as it stands (a root `$ref` or `$defs`
|
|
5
|
+
* is not an object schema), and embedding it moves the document root its local pointers resolve
|
|
6
|
+
* against, so each local `$ref` (`#`, `#/...`) is relocated under `#/properties/value`. A subschema
|
|
7
|
+
* that sets its own `$id` is a separate resource: its local references do not move. Every runner that
|
|
8
|
+
* delivers a node's record through a tool call uses this one envelope, so a schema behaves the same on
|
|
9
|
+
* every harness. A keyword of the DSL's own (`criteria`) is left out of every schema in it, never a property or
|
|
10
|
+
* definition of that name; the submission is still checked against the node's schema as written. */
|
|
11
|
+
export function submitParameters(schema) {
|
|
12
|
+
const maps = new Set(["properties", "patternProperties", "$defs", "definitions", "dependentSchemas"]);
|
|
13
|
+
const singles = new Set(["additionalProperties", "unevaluatedProperties", "propertyNames", "items", "additionalItems", "contains", "unevaluatedItems", "not", "if", "then", "else", "contentSchema"]);
|
|
14
|
+
const arrays = new Set(["allOf", "anyOf", "oneOf", "prefixItems"]);
|
|
15
|
+
const relocate = (node, resourceScope = false) => {
|
|
16
|
+
if (!node || typeof node !== "object" || Array.isArray(node))
|
|
17
|
+
return structuredClone(node);
|
|
18
|
+
// $id establishes a separate schema resource: its local references do not move.
|
|
19
|
+
resourceScope ||= typeof node.$id === "string" && node.$id !== "";
|
|
20
|
+
const child = (value) => relocate(value, resourceScope);
|
|
21
|
+
return Object.fromEntries(Object.entries(node).filter(([key]) => !DSL_ONLY_SCHEMA_KEYWORDS.has(key)).map(([key, value]) => {
|
|
22
|
+
if (!resourceScope && key === "$ref" && typeof value === "string" && (value === "#" || value.startsWith("#/"))) {
|
|
23
|
+
return [key, `#/properties/value${value.slice(1)}`];
|
|
24
|
+
}
|
|
25
|
+
if (maps.has(key) && value && typeof value === "object" && !Array.isArray(value)) {
|
|
26
|
+
return [key, Object.fromEntries(Object.entries(value).map(([name, value]) => [name, child(value)]))];
|
|
27
|
+
}
|
|
28
|
+
if (arrays.has(key) && Array.isArray(value))
|
|
29
|
+
return [key, value.map(child)];
|
|
30
|
+
if (singles.has(key))
|
|
31
|
+
return [key, Array.isArray(value) ? value.map(child) : child(value)];
|
|
32
|
+
if (key === "dependencies" && value && typeof value === "object" && !Array.isArray(value)) {
|
|
33
|
+
return [key, Object.fromEntries(Object.entries(value).map(([name, value]) => [name, Array.isArray(value) ? structuredClone(value) : child(value)]))];
|
|
34
|
+
}
|
|
35
|
+
// Annotations and literal values are data, not schemas (including a literal $ref).
|
|
36
|
+
return [key, structuredClone(value)];
|
|
37
|
+
}));
|
|
38
|
+
};
|
|
39
|
+
return { type: "object", properties: { value: relocate(schema) }, required: ["value"], additionalProperties: false };
|
|
40
|
+
}
|
package/dist/system-one.d.ts
CHANGED
|
@@ -13,6 +13,18 @@ export type SystemOneQuestion = {
|
|
|
13
13
|
type: "score";
|
|
14
14
|
instructions: string;
|
|
15
15
|
criteria: string[];
|
|
16
|
+
}
|
|
17
|
+
/** Locates what its instructions describe in the request's image. A host-side question: no workflow node declares one, and it needs an image sent with the request. */
|
|
18
|
+
| {
|
|
19
|
+
type: "box";
|
|
20
|
+
instructions: string;
|
|
21
|
+
};
|
|
22
|
+
/** A box's edges as fractions of the image's height (`top`, `bottom`) and width (`left`, `right`), from the top left corner. */
|
|
23
|
+
export type SystemOneBox = {
|
|
24
|
+
top: number;
|
|
25
|
+
bottom: number;
|
|
26
|
+
left: number;
|
|
27
|
+
right: number;
|
|
16
28
|
};
|
|
17
29
|
export type SystemOneAnswer = {
|
|
18
30
|
type: "choice";
|
|
@@ -28,6 +40,14 @@ export type SystemOneAnswer = {
|
|
|
28
40
|
legend: Record<string, string>;
|
|
29
41
|
probabilities: Record<string, number>;
|
|
30
42
|
confidence: number;
|
|
43
|
+
} | {
|
|
44
|
+
type: "box";
|
|
45
|
+
box: SystemOneBox;
|
|
46
|
+
confidence: number | null;
|
|
47
|
+
}
|
|
48
|
+
/** The provider declined to answer this question. It is never decoded as a value. */
|
|
49
|
+
| {
|
|
50
|
+
type: "refusal";
|
|
31
51
|
};
|
|
32
52
|
export type SystemOneResult = {
|
|
33
53
|
answers: Record<string, SystemOneAnswer>;
|
|
@@ -65,7 +85,7 @@ export type AnswersSidecar = {
|
|
|
65
85
|
export declare function answersSidecar(answers: Record<string, SystemOneAnswer>): AnswersSidecar;
|
|
66
86
|
export declare function synthesizeAnswers(questions: Record<string, SystemOneQuestion>): Record<string, SystemOneAnswer>;
|
|
67
87
|
export type SystemOneRetryClass = "http_429" | "http_5xx" | "connection" | "timeout";
|
|
68
|
-
declare const responseReasons: readonly ["answers_shape", "answer_keys", "answer_shape", "answer_type", "noul_probability", "confidence", "probabilities", "probability_keys", "probability_mass", "score_legend", "score_consistency", "choice_option", "score_range"];
|
|
88
|
+
declare const responseReasons: readonly ["answers_shape", "answer_keys", "answer_shape", "answer_type", "noul_probability", "confidence", "box_coordinates", "probabilities", "probability_keys", "probability_mass", "score_legend", "score_consistency", "choice_option", "score_range", "refusal"];
|
|
69
89
|
export type SystemOneResponseReason = typeof responseReasons[number];
|
|
70
90
|
export declare function isSystemOneResponseReason(value: unknown): value is SystemOneResponseReason;
|
|
71
91
|
export declare class SystemOneError extends Error {
|
|
@@ -75,4 +95,23 @@ export declare class SystemOneError extends Error {
|
|
|
75
95
|
constructor(message: string, retryClass: SystemOneRetryClass | null, status?: number | undefined, responseReason?: SystemOneResponseReason);
|
|
76
96
|
}
|
|
77
97
|
export declare function validateAnswers(questions: Record<string, SystemOneQuestion>, answers: unknown): asserts answers is Record<string, SystemOneAnswer>;
|
|
98
|
+
/** The questions whose answer is a refusal, in answer order. */
|
|
99
|
+
export declare function refusedQuestions(answers: Record<string, unknown>): string[];
|
|
100
|
+
/** What `answersToValue` throws for a refused answer: the backstop. A decision node never gets this far, since `decide()`
|
|
101
|
+
* escalates the run for review (DECISION_REFUSED) on a refusal first. */
|
|
102
|
+
export declare function refusalError(ids: string[]): SystemOneError;
|
|
103
|
+
/** The confidence TypeSafe documents for Jev (docs.typesafe.ai/confidence), computed from an answer's probabilities: for a
|
|
104
|
+
* choice of n options, (p - 1/n) / (1 - 1/n) for the chosen option's p, floored at 0; for a score of n levels, 1 minus
|
|
105
|
+
* the probability-weighted distance from the most likely level (the lowest, on a tie) over that distance for an even
|
|
106
|
+
* spread, floored at 0. Null when the probabilities are missing or are not a distribution over the answer's options. */
|
|
107
|
+
export declare function deriveConfidence(answer: unknown): number | null;
|
|
108
|
+
/** Where an answer's confidence came from: the provider's response, or derived from its probabilities. */
|
|
109
|
+
export type ConfidenceSource = "provider" | "derived";
|
|
110
|
+
/** The answers with every choice and score carrying a confidence: the provider's own when it gave one in [0, 1], else
|
|
111
|
+
* one derived from the probabilities (deriveConfidence), else 0, which validation then refuses with the probabilities
|
|
112
|
+
* that left nothing to derive from. `confidence_source` names each choice and score answer's source. */
|
|
113
|
+
export declare function completeConfidence(questions: Record<string, SystemOneQuestion>, answers: Record<string, unknown>): {
|
|
114
|
+
answers: Record<string, unknown>;
|
|
115
|
+
confidence_source: Record<string, ConfidenceSource>;
|
|
116
|
+
};
|
|
78
117
|
export {};
|
package/dist/system-one.js
CHANGED
|
@@ -84,6 +84,9 @@ export function compileQuestions(schema, definitions = {}) {
|
|
|
84
84
|
}
|
|
85
85
|
export function answersToValue(answers, questions) {
|
|
86
86
|
const out = Object.create(null);
|
|
87
|
+
const refused = refusedQuestions(answers).filter((id) => Object.hasOwn(questions, id));
|
|
88
|
+
if (refused.length)
|
|
89
|
+
throw refusalError(refused);
|
|
87
90
|
for (const [id, q] of Object.entries(questions)) {
|
|
88
91
|
const a = answers[id];
|
|
89
92
|
if (!a)
|
|
@@ -94,6 +97,8 @@ export function answersToValue(answers, questions) {
|
|
|
94
97
|
out[id] = a.noul >= 0.5;
|
|
95
98
|
else if (q.type === "score" && a.type === "score")
|
|
96
99
|
out[id] = Math.max(0, Math.round(a.score));
|
|
100
|
+
else if (q.type === "box" && a.type === "box")
|
|
101
|
+
out[id] = { ...a.box };
|
|
97
102
|
else
|
|
98
103
|
throw new Error(`System One answered question "${id}" with type "${a.type}", expected ${q.type}`);
|
|
99
104
|
}
|
|
@@ -102,8 +107,11 @@ export function answersToValue(answers, questions) {
|
|
|
102
107
|
export function answerConfidence(answer) {
|
|
103
108
|
if (!answer || typeof answer !== "object")
|
|
104
109
|
throw new SystemOneError("System One answer missing for a question that was asked", null);
|
|
110
|
+
if (answer.type === "refusal")
|
|
111
|
+
return 0;
|
|
105
112
|
if (answer.type === "noul")
|
|
106
113
|
return Math.abs(answer.noul - 0.5) * 2;
|
|
114
|
+
// A box the provider reported no confidence for counts as none: unknown is never read as sure.
|
|
107
115
|
return typeof answer.confidence === "number" ? answer.confidence : 0;
|
|
108
116
|
}
|
|
109
117
|
export function answersSidecar(answers) {
|
|
@@ -124,6 +132,8 @@ export function synthesizeAnswers(questions) {
|
|
|
124
132
|
const options = Object.keys(q.criteria);
|
|
125
133
|
out[id] = { type: "choice", choice: options[0], probabilities: Object.fromEntries(options.map((o) => [o, 1 / options.length])), confidence: 0 };
|
|
126
134
|
}
|
|
135
|
+
else if (q.type === "box")
|
|
136
|
+
out[id] = { type: "box", box: { top: 0, bottom: 1, left: 0, right: 1 }, confidence: 0 };
|
|
127
137
|
else if (q.type === "noul")
|
|
128
138
|
out[id] = { type: "noul", noul: 0.5 };
|
|
129
139
|
else
|
|
@@ -132,7 +142,7 @@ export function synthesizeAnswers(questions) {
|
|
|
132
142
|
return { ...out };
|
|
133
143
|
}
|
|
134
144
|
const responseReasons = ["answers_shape", "answer_keys", "answer_shape", "answer_type", "noul_probability",
|
|
135
|
-
"confidence", "probabilities", "probability_keys", "probability_mass", "score_legend", "score_consistency", "choice_option", "score_range"];
|
|
145
|
+
"confidence", "box_coordinates", "probabilities", "probability_keys", "probability_mass", "score_legend", "score_consistency", "choice_option", "score_range", "refusal"];
|
|
136
146
|
export function isSystemOneResponseReason(value) {
|
|
137
147
|
return typeof value === "string" && responseReasons.includes(value);
|
|
138
148
|
}
|
|
@@ -150,6 +160,56 @@ export class SystemOneError extends Error {
|
|
|
150
160
|
}
|
|
151
161
|
}
|
|
152
162
|
const unit = (v) => typeof v === "number" && Number.isFinite(v) && v >= 0 && v <= 1;
|
|
163
|
+
// System One reports probabilities rounded, Jev to two places and some OpenRouter models to four (Cloudflare's clef,
|
|
164
|
+
// live: 0.2967 + 0.6661 + 0.0371 = 0.9999), so the reported sum drifts from 1 by up to half a step per option
|
|
165
|
+
// (3 × 0.33 = 0.99; 8 × 0.13 = 1.04). A distribution is accepted when it could be that rounding of one summing to 1: a
|
|
166
|
+
// value on the two-, three- or four-place grid may be off by half that grid's step (within [0, 1]), the coarsest grid it
|
|
167
|
+
// lies on; any other value is taken as exact. With no grid values this is |sum − 1| ≤ 1e-5, the check for
|
|
168
|
+
// full-precision distributions. Answers are accepted as reported, never rescaled.
|
|
169
|
+
const roundingHalfStep = (p) => {
|
|
170
|
+
for (const places of [2, 3, 4]) {
|
|
171
|
+
const scale = 10 ** places;
|
|
172
|
+
if (Math.abs(p * scale - Math.round(p * scale)) < 1e-9 * scale / 100)
|
|
173
|
+
return 0.5 / scale;
|
|
174
|
+
}
|
|
175
|
+
return 0;
|
|
176
|
+
};
|
|
177
|
+
function roundedFromUnitMass(probabilities) {
|
|
178
|
+
let low = 0, high = 0;
|
|
179
|
+
for (const p of probabilities) {
|
|
180
|
+
const step = roundingHalfStep(p);
|
|
181
|
+
low += Math.max(0, p - step);
|
|
182
|
+
high += Math.min(1, p + step);
|
|
183
|
+
}
|
|
184
|
+
return low <= 1 + 1e-5 && high >= 1 - 1e-5;
|
|
185
|
+
}
|
|
186
|
+
// A Score's weighted level drifts the same way. The unrounded probabilities lie within their rounding
|
|
187
|
+
// intervals and sum to 1, so Σ level·p lies between two bounds: start every level at its interval's
|
|
188
|
+
// low end, then give the rest of the unit mass to the lowest levels (the minimum) or the highest (the
|
|
189
|
+
// maximum), each up to its interval's high end. A score on a grid is itself off by up to half its step. The
|
|
190
|
+
// score is accepted when its interval meets [min, max]; with no grid values this is
|
|
191
|
+
// |score − Σ level·p| ≤ 1e-5. A score that is not a number contradicts nothing here and is left to
|
|
192
|
+
// the range check, as before.
|
|
193
|
+
function scoreContradictsDistribution(score, probabilities) {
|
|
194
|
+
const levels = Object.entries(probabilities).map(([level, p]) => {
|
|
195
|
+
const step = roundingHalfStep(p);
|
|
196
|
+
return { level: Number(level), low: Math.max(0, p - step), high: Math.min(1, p + step) };
|
|
197
|
+
}).sort((a, b) => a.level - b.level);
|
|
198
|
+
const floor = levels.reduce((sum, l) => sum + l.level * l.low, 0);
|
|
199
|
+
const spare = Math.max(0, 1 - levels.reduce((sum, l) => sum + l.low, 0));
|
|
200
|
+
const fill = (order) => {
|
|
201
|
+
let rest = spare, weighted = floor;
|
|
202
|
+
for (const l of order) {
|
|
203
|
+
const add = Math.min(rest, l.high - l.low);
|
|
204
|
+
weighted += l.level * add;
|
|
205
|
+
rest -= add;
|
|
206
|
+
}
|
|
207
|
+
return weighted;
|
|
208
|
+
};
|
|
209
|
+
const min = fill(levels), max = fill([...levels].reverse());
|
|
210
|
+
const step = roundingHalfStep(score);
|
|
211
|
+
return score - step > max + 1e-5 || score + step < min - 1e-5;
|
|
212
|
+
}
|
|
153
213
|
export function validateAnswers(questions, answers) {
|
|
154
214
|
if (!answers || typeof answers !== "object" || Array.isArray(answers))
|
|
155
215
|
throw new SystemOneError("System One response carries no answers", null, undefined, "answers_shape");
|
|
@@ -160,8 +220,24 @@ export function validateAnswers(questions, answers) {
|
|
|
160
220
|
const a = answers[id];
|
|
161
221
|
if (!a || typeof a !== "object")
|
|
162
222
|
throw new SystemOneError("System One returned no answer for a question", null, undefined, "answer_shape");
|
|
223
|
+
if (a.type === "refusal") {
|
|
224
|
+
if (Object.keys(a).length !== 1)
|
|
225
|
+
throw new SystemOneError("System One answer: a refusal carries nothing but its type", null, undefined, "answer_shape");
|
|
226
|
+
continue;
|
|
227
|
+
}
|
|
163
228
|
if (a.type !== q.type)
|
|
164
229
|
throw new SystemOneError("System One answer type does not match its question", null, undefined, "answer_type");
|
|
230
|
+
if (q.type === "box") {
|
|
231
|
+
const box = a.box;
|
|
232
|
+
if (!box || typeof box !== "object" || Array.isArray(box) || Object.keys(box).length !== 4 || !["top", "bottom", "left", "right"].every(edge => unit(box[edge])))
|
|
233
|
+
throw new SystemOneError("System One answer: a box's edges are not fractions in [0, 1]", null, undefined, "box_coordinates");
|
|
234
|
+
// An edge pair that runs backwards is a box with negative width or height: not a box.
|
|
235
|
+
if (box.top > box.bottom || box.left > box.right)
|
|
236
|
+
throw new SystemOneError("System One answer: a box's top and left edges cannot lie past its bottom and right", null, undefined, "box_coordinates");
|
|
237
|
+
if (!(a.confidence === null || unit(a.confidence)))
|
|
238
|
+
throw new SystemOneError("System One answer: confidence is not in [0, 1]", null, undefined, "confidence");
|
|
239
|
+
continue;
|
|
240
|
+
}
|
|
165
241
|
if (q.type === "noul") {
|
|
166
242
|
if (!unit(a.noul))
|
|
167
243
|
throw new SystemOneError("System One answer: noul is not a probability", null, undefined, "noul_probability");
|
|
@@ -175,14 +251,12 @@ export function validateAnswers(questions, answers) {
|
|
|
175
251
|
const probabilityKeys = Object.keys(a.probabilities);
|
|
176
252
|
if (Array.isArray(a.probabilities) || probabilityKeys.length !== expectedKeys.length || expectedKeys.some(key => !Object.hasOwn(a.probabilities, key)))
|
|
177
253
|
throw new SystemOneError("System One answer: probabilities must contain exactly the declared options", null, undefined, "probability_keys");
|
|
178
|
-
|
|
179
|
-
if (Math.abs(mass - 1) > 1e-5)
|
|
254
|
+
if (!roundedFromUnitMass(Object.values(a.probabilities)))
|
|
180
255
|
throw new SystemOneError("System One answer: probabilities must sum to 1", null, undefined, "probability_mass");
|
|
181
256
|
if (q.type === "score") {
|
|
182
257
|
if (!a.legend || typeof a.legend !== "object" || Array.isArray(a.legend) || Object.keys(a.legend).length !== expectedKeys.length || q.criteria.some((text, index) => a.legend[String(index)] !== text))
|
|
183
258
|
throw new SystemOneError("System One answer: legend must match the score criteria", null, undefined, "score_legend");
|
|
184
|
-
|
|
185
|
-
if (Math.abs(weighted - a.score) > 1e-5)
|
|
259
|
+
if (scoreContradictsDistribution(a.score, a.probabilities))
|
|
186
260
|
throw new SystemOneError("System One answer: score must match its weighted distribution", null, undefined, "score_consistency");
|
|
187
261
|
}
|
|
188
262
|
if (q.type === "choice" && !(typeof a.choice === "string" && Object.prototype.hasOwnProperty.call(q.criteria, a.choice)))
|
|
@@ -191,3 +265,65 @@ export function validateAnswers(questions, answers) {
|
|
|
191
265
|
throw new SystemOneError("System One answer: score is outside its levels", null, undefined, "score_range");
|
|
192
266
|
}
|
|
193
267
|
}
|
|
268
|
+
/** The questions whose answer is a refusal, in answer order. */
|
|
269
|
+
export function refusedQuestions(answers) {
|
|
270
|
+
return Object.entries(answers).filter(([, a]) => isRecord(a) && a.type === "refusal").map(([id]) => id);
|
|
271
|
+
}
|
|
272
|
+
/** What `answersToValue` throws for a refused answer: the backstop. A decision node never gets this far, since `decide()`
|
|
273
|
+
* escalates the run for review (DECISION_REFUSED) on a refusal first. */
|
|
274
|
+
export function refusalError(ids) {
|
|
275
|
+
return new SystemOneError(`System One refused ${ids.length === 1 ? "question" : "questions"} ${ids.map((id) => `"${id}"`).join(", ")}`, null, undefined, "refusal");
|
|
276
|
+
}
|
|
277
|
+
const isRecord = (v) => v !== null && typeof v === "object" && !Array.isArray(v);
|
|
278
|
+
/** The confidence TypeSafe documents for Jev (docs.typesafe.ai/confidence), computed from an answer's probabilities: for a
|
|
279
|
+
* choice of n options, (p - 1/n) / (1 - 1/n) for the chosen option's p, floored at 0; for a score of n levels, 1 minus
|
|
280
|
+
* the probability-weighted distance from the most likely level (the lowest, on a tie) over that distance for an even
|
|
281
|
+
* spread, floored at 0. Null when the probabilities are missing or are not a distribution over the answer's options. */
|
|
282
|
+
export function deriveConfidence(answer) {
|
|
283
|
+
if (!isRecord(answer) || !isRecord(answer.probabilities))
|
|
284
|
+
return null;
|
|
285
|
+
const probabilities = answer.probabilities;
|
|
286
|
+
const values = Object.values(probabilities);
|
|
287
|
+
const n = values.length;
|
|
288
|
+
if (n < 2 || !values.every(unit))
|
|
289
|
+
return null;
|
|
290
|
+
if (answer.type === "choice") {
|
|
291
|
+
if (typeof answer.choice !== "string" || !Object.hasOwn(probabilities, answer.choice))
|
|
292
|
+
return null;
|
|
293
|
+
return Math.min(1, Math.max(0, (probabilities[answer.choice] - 1 / n) / (1 - 1 / n)));
|
|
294
|
+
}
|
|
295
|
+
if (answer.type === "score") {
|
|
296
|
+
const levels = Array.from({ length: n }, (_, i) => probabilities[String(i)]);
|
|
297
|
+
if (!levels.every(unit))
|
|
298
|
+
return null;
|
|
299
|
+
const p = levels;
|
|
300
|
+
const mode = p.indexOf(Math.max(...p));
|
|
301
|
+
const spread = p.reduce((sum, pi, i) => sum + pi * Math.abs(i - mode), 0);
|
|
302
|
+
const even = p.reduce((sum, _, i) => sum + Math.abs(i - (n - 1) / 2), 0) / n;
|
|
303
|
+
return Math.min(1, Math.max(0, 1 - spread / even));
|
|
304
|
+
}
|
|
305
|
+
return null;
|
|
306
|
+
}
|
|
307
|
+
/** The answers with every choice and score carrying a confidence: the provider's own when it gave one in [0, 1], else
|
|
308
|
+
* one derived from the probabilities (deriveConfidence), else 0, which validation then refuses with the probabilities
|
|
309
|
+
* that left nothing to derive from. `confidence_source` names each choice and score answer's source. */
|
|
310
|
+
export function completeConfidence(questions, answers) {
|
|
311
|
+
// Null-prototype records: a question may be named __proto__, which on a plain object would set its prototype.
|
|
312
|
+
const out = Object.create(null);
|
|
313
|
+
const confidence_source = Object.create(null);
|
|
314
|
+
for (const [id, answer] of Object.entries(answers)) {
|
|
315
|
+
const asked = Object.hasOwn(questions, id) ? questions[id] : undefined;
|
|
316
|
+
if (!isRecord(answer) || (answer.type !== "choice" && answer.type !== "score") || asked?.type !== answer.type) {
|
|
317
|
+
out[id] = answer;
|
|
318
|
+
continue;
|
|
319
|
+
}
|
|
320
|
+
if (unit(answer.confidence)) {
|
|
321
|
+
out[id] = answer;
|
|
322
|
+
confidence_source[id] = "provider";
|
|
323
|
+
continue;
|
|
324
|
+
}
|
|
325
|
+
out[id] = { ...answer, confidence: deriveConfidence(answer) ?? 0 };
|
|
326
|
+
confidence_source[id] = "derived";
|
|
327
|
+
}
|
|
328
|
+
return { answers: { ...out }, confidence_source: { ...confidence_source } };
|
|
329
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { type DecisionEvidence, type DecisionProvider } from "../decision-provider.js";
|
|
2
|
+
/** What the kit's fake transport answers one call with. An `answer` carries answers in the DSL's shape (and, for a
|
|
3
|
+
* provider with evidence, sources), which the kit sends in the provider's own wire format. `lost` is a connection that
|
|
4
|
+
* drops after the request was sent: whether it was served is not known. */
|
|
5
|
+
export type KitReply = {
|
|
6
|
+
kind: "answer";
|
|
7
|
+
answers: Record<string, unknown>;
|
|
8
|
+
evidence?: DecisionEvidence[];
|
|
9
|
+
} | {
|
|
10
|
+
kind: "status";
|
|
11
|
+
status: number;
|
|
12
|
+
} | {
|
|
13
|
+
kind: "malformed";
|
|
14
|
+
} | {
|
|
15
|
+
kind: "lost";
|
|
16
|
+
} | {
|
|
17
|
+
kind: "hang";
|
|
18
|
+
};
|
|
19
|
+
export interface DecisionProviderKit {
|
|
20
|
+
/** A new provider under test whose transport answers each call with the next of `replies` (the last repeats), and
|
|
21
|
+
* the number of calls its transport has taken. Retry delays should be short: the suite waits for them. */
|
|
22
|
+
build(replies: KitReply[]): {
|
|
23
|
+
provider: DecisionProvider;
|
|
24
|
+
calls(): number;
|
|
25
|
+
};
|
|
26
|
+
/** The key the kit configures the provider with: it must never appear in a result or an error. */
|
|
27
|
+
canary: string;
|
|
28
|
+
/** Whether the provider's wire format can carry a refusal. */
|
|
29
|
+
refuses: boolean;
|
|
30
|
+
}
|
|
31
|
+
/** Registers the suite as `node:test` tests named under `name`. */
|
|
32
|
+
export declare function registerDecisionProviderConformance(name: string, kit: DecisionProviderKit): void;
|