@parcha/agentrun-dsl 0.1.0-beta.3 → 0.1.0-beta.30

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/README.md +165 -21
  2. package/dist/author-references.d.ts +8 -0
  3. package/dist/author-references.js +8 -0
  4. package/dist/author.d.ts +16 -3
  5. package/dist/author.js +40 -16
  6. package/dist/authoring.js +2 -2
  7. package/dist/cli.js +17 -12
  8. package/dist/code-exec.d.ts +6 -0
  9. package/dist/code-exec.js +126 -0
  10. package/dist/decision-provider.d.ts +157 -0
  11. package/dist/decision-provider.js +230 -0
  12. package/dist/decision-receipts.d.ts +143 -0
  13. package/dist/decision-receipts.js +93 -0
  14. package/dist/dry-run.d.ts +14 -1
  15. package/dist/dry-run.js +58 -15
  16. package/dist/graph.d.ts +169 -0
  17. package/dist/graph.js +433 -0
  18. package/dist/index.d.ts +15 -8
  19. package/dist/index.js +8 -4
  20. package/dist/inspection.d.ts +5 -0
  21. package/dist/inspection.js +16 -3
  22. package/dist/job.d.ts +14 -0
  23. package/dist/job.js +5 -0
  24. package/dist/long-timer.d.ts +7 -0
  25. package/dist/long-timer.js +22 -0
  26. package/dist/observer-snapshot.d.ts +3 -0
  27. package/dist/observer-snapshot.js +46 -0
  28. package/dist/predicates.d.ts +10 -0
  29. package/dist/predicates.js +18 -0
  30. package/dist/recovery/binding.d.ts +10 -0
  31. package/dist/recovery/binding.js +43 -0
  32. package/dist/recovery/call-key.d.ts +13 -0
  33. package/dist/recovery/call-key.js +24 -0
  34. package/dist/recovery/canonical-hash.d.ts +7 -0
  35. package/dist/recovery/canonical-hash.js +22 -0
  36. package/dist/recovery/driver.d.ts +238 -0
  37. package/dist/recovery/driver.js +1337 -0
  38. package/dist/recovery/effect-failure.d.ts +34 -0
  39. package/dist/recovery/effect-failure.js +62 -0
  40. package/dist/recovery/errors.d.ts +31 -0
  41. package/dist/recovery/errors.js +51 -0
  42. package/dist/recovery/execution-path.d.ts +53 -0
  43. package/dist/recovery/execution-path.js +167 -0
  44. package/dist/recovery/file-store.d.ts +7 -0
  45. package/dist/recovery/file-store.js +131 -0
  46. package/dist/recovery/frozen-snapshot.d.ts +274 -0
  47. package/dist/recovery/frozen-snapshot.js +187 -0
  48. package/dist/recovery/handoff-block.d.ts +36 -0
  49. package/dist/recovery/handoff-block.js +148 -0
  50. package/dist/recovery/handoff-digest.d.ts +58 -0
  51. package/dist/recovery/handoff-digest.js +195 -0
  52. package/dist/recovery/handoff-files.d.ts +54 -0
  53. package/dist/recovery/handoff-files.js +162 -0
  54. package/dist/recovery/handoff-receipts.d.ts +42 -0
  55. package/dist/recovery/handoff-receipts.js +53 -0
  56. package/dist/recovery/handoff-redact.d.ts +12 -0
  57. package/dist/recovery/handoff-redact.js +44 -0
  58. package/dist/recovery/handoff.d.ts +44 -0
  59. package/dist/recovery/handoff.js +46 -0
  60. package/dist/recovery/index.d.ts +19 -0
  61. package/dist/recovery/index.js +17 -0
  62. package/dist/recovery/item-state.d.ts +48 -0
  63. package/dist/recovery/item-state.js +89 -0
  64. package/dist/recovery/journal.d.ts +28 -0
  65. package/dist/recovery/journal.js +139 -0
  66. package/dist/recovery/memory-store.d.ts +4 -0
  67. package/dist/recovery/memory-store.js +20 -0
  68. package/dist/recovery/store.d.ts +105 -0
  69. package/dist/recovery/store.js +6 -0
  70. package/dist/recovery/testing/conformance.d.ts +4 -0
  71. package/dist/recovery/testing/conformance.js +268 -0
  72. package/dist/recovery/testing/crash-points.d.ts +18 -0
  73. package/dist/recovery/testing/crash-points.js +43 -0
  74. package/dist/recovery/testing/crash-sweep.d.ts +65 -0
  75. package/dist/recovery/testing/crash-sweep.js +146 -0
  76. package/dist/recovery/testing/index.d.ts +3 -0
  77. package/dist/recovery/testing/index.js +4 -0
  78. package/dist/recovery/testing/preload.d.ts +1 -0
  79. package/dist/recovery/testing/preload.js +3 -0
  80. package/dist/recovery/transport-facts.d.ts +23 -0
  81. package/dist/recovery/transport-facts.js +46 -0
  82. package/dist/recovery/with-recovery.d.ts +50 -0
  83. package/dist/recovery/with-recovery.js +142 -0
  84. package/dist/submit-envelope.d.ts +9 -0
  85. package/dist/submit-envelope.js +40 -0
  86. package/dist/system-one.d.ts +40 -1
  87. package/dist/system-one.js +141 -5
  88. package/dist/testing/decision-provider.d.ts +32 -0
  89. package/dist/testing/decision-provider.js +183 -0
  90. package/dist/testing/index.d.ts +1 -0
  91. package/dist/testing/index.js +2 -0
  92. package/dist/vocabulary.d.ts +22 -3
  93. package/dist/vocabulary.js +33 -10
  94. package/dist/workflow-shape.d.ts +7 -0
  95. package/dist/workflow-shape.js +115 -67
  96. package/dist/workflow.d.ts +269 -20
  97. package/dist/workflow.js +1251 -341
  98. package/package.json +25 -2
  99. package/schema/workflow.schema.json +506 -12
  100. package/skills/author/SKILL.md +6 -1
  101. package/skills/author/references/jev-decisions.md +58 -6
  102. package/skills/author/references/language.md +28 -19
  103. package/skills/author/references/sdk-authoring.md +2 -0
@@ -0,0 +1,50 @@
1
+ import { type WorkflowDeps } from "../workflow.js";
2
+ import type { openRecovery, RecoveryEffectParams } from "./driver.js";
3
+ import { DECISION_INTERRUPTED } from "../decision-receipts.js";
4
+ export { DECISION_INTERRUPTED };
5
+ /** A recovery driver, as `openRecovery` returns it. */
6
+ export type RecoveryDriver = Awaited<ReturnType<typeof openRecovery>>;
7
+ /** The attempt of an LLM step a node runner is asked to run, as the driver admitted it. One attempt is one session:
8
+ * a runner that keeps what an attempt delivered keys it by `sessionId`. */
9
+ export type RecoveryStep = {
10
+ sessionId: string;
11
+ attempt: number;
12
+ earlierSessionIds: readonly string[];
13
+ };
14
+ /** What a host's node runner receives under the driver: the interpreter's parameters, and the attempt it runs. A
15
+ * runner marks a failure no second attempt should follow (one that ended on a bound of its own) with `final: true`
16
+ * on what it throws. */
17
+ export type RecoveryNodeParams = Parameters<NonNullable<WorkflowDeps["runNode"]>>[0] & {
18
+ step?: RecoveryStep;
19
+ };
20
+ /** The host's adapters as `withRecovery` takes them: the interpreter's, with the two that learn more under the driver. */
21
+ export type RecoveryAdapters = Omit<WorkflowDeps, "runEffect" | "runNode"> & {
22
+ runEffect?: (params: RecoveryEffectParams) => Promise<unknown>;
23
+ runNode?: (params: RecoveryNodeParams) => Promise<unknown>;
24
+ };
25
+ export type WithRecoveryOptions = {
26
+ /** The node runner keeps what each attempt delivered: asked again for a `sessionId` whose record it holds, it
27
+ * returns that record without running anything. A step found delivered is then asked of it again as the same
28
+ * attempt, and nothing is spent. Absent, such a step has spent its attempt.
29
+ * A record the interpreter refuses is therefore returned, and refused, again at every later open: the step neither
30
+ * advances nor spends, and the run fails the same way each time until the host ends it (an escalation, a new run). */
31
+ durableNodes?: boolean;
32
+ };
33
+ /** The dependencies to run a workflow with under `driver`. The interpreter's `recovery` hooks cover nodes; three
34
+ * things they have no slot for are applied to the host's adapters here:
35
+ * - `runEffect`: each effect is admitted before it is dispatched, a completed one is answered from its receipt, and
36
+ * one admitted and never completed is refused;
37
+ * - `runJudge`: each request is committed (`decision.request`) before the judge is asked, and ends with one outcome:
38
+ * answered, or failed in its own process (asked again as its next attempt). A later open that finds it asked and
39
+ * never answered (the process died, or a cancel cut the call) books it once at an unknown cost and ends the run
40
+ * escalated (`decision_interrupted`) instead of asking again. A route's answer is committed as its decision before
41
+ * any branch step runs, and a resume inside the branch follows it without asking again; every answer's spend is
42
+ * committed with the driver's state;
43
+ * - `runNode`: an LLM step's record is committed before the step runs, with two attempts, each handed to the
44
+ * runner as its `step`. A failed attempt is followed at once by the second, unless the runner marked the
45
+ * failure final; a step that spent both is refused at every later open. A process that dies inside a step spends
46
+ * no attempt. A step that delivered a submission and whose node never committed (the interpreter refused it, or
47
+ * the process died first) is answered again from the runner's own record under `durableNodes`, and has spent its
48
+ * attempt otherwise.
49
+ * The run stops through the driver's signal, together with the host's own. */
50
+ export declare function withRecovery(driver: RecoveryDriver, deps: RecoveryAdapters, options?: WithRecoveryOptions): WorkflowDeps;
@@ -0,0 +1,142 @@
1
+ import { chargedCost, DecisionProviderError } from "../decision-provider.js";
2
+ import { validateAnswers } from "../system-one.js";
3
+ import { EscalationSignal } from "../workflow.js";
4
+ import { canonicalHash } from "./canonical-hash.js";
5
+ import { DECISION_INTERRUPTED } from "../decision-receipts.js";
6
+ export { DECISION_INTERRUPTED };
7
+ /** The dependencies to run a workflow with under `driver`. The interpreter's `recovery` hooks cover nodes; three
8
+ * things they have no slot for are applied to the host's adapters here:
9
+ * - `runEffect`: each effect is admitted before it is dispatched, a completed one is answered from its receipt, and
10
+ * one admitted and never completed is refused;
11
+ * - `runJudge`: each request is committed (`decision.request`) before the judge is asked, and ends with one outcome:
12
+ * answered, or failed in its own process (asked again as its next attempt). A later open that finds it asked and
13
+ * never answered (the process died, or a cancel cut the call) books it once at an unknown cost and ends the run
14
+ * escalated (`decision_interrupted`) instead of asking again. A route's answer is committed as its decision before
15
+ * any branch step runs, and a resume inside the branch follows it without asking again; every answer's spend is
16
+ * committed with the driver's state;
17
+ * - `runNode`: an LLM step's record is committed before the step runs, with two attempts, each handed to the
18
+ * runner as its `step`. A failed attempt is followed at once by the second, unless the runner marked the
19
+ * failure final; a step that spent both is refused at every later open. A process that dies inside a step spends
20
+ * no attempt. A step that delivered a submission and whose node never committed (the interpreter refused it, or
21
+ * the process died first) is answered again from the runner's own record under `durableNodes`, and has spent its
22
+ * attempt otherwise.
23
+ * The run stops through the driver's signal, together with the host's own. */
24
+ export function withRecovery(driver, deps, options = {}) {
25
+ const { runEffect, runJudge, runNode } = deps;
26
+ return {
27
+ ...deps,
28
+ recovery: driver.recovery,
29
+ signal: deps.signal ? AbortSignal.any([deps.signal, driver.signal]) : driver.signal,
30
+ runEffect: runEffect && driver.wrapEffect(runEffect),
31
+ runJudge: runJudge && (async (params) => {
32
+ const decided = params.kind === "route" ? driver.routeDecision(params.executionPath) : undefined;
33
+ if (decided)
34
+ return decided.result;
35
+ await driver.seam();
36
+ const decisionId = params.decisionId ?? canonicalHash(["decision-request", params.executionPath, params.kind, params.questions, params.state]);
37
+ const open = driver.admitDecision(params.executionPath, { decision_id: decisionId, kind: params.kind, label: params.label, model: params.model ?? null,
38
+ request_sha256: canonicalHash({ kind: params.kind, label: params.label, state: params.state, questions: params.questions }) });
39
+ if (open) {
40
+ // An earlier process asked and its answer never reached a receipt: never asked again, booked once at an unknown
41
+ // cost, and the run goes to review. The unknown cost is counted with the outcome in one commit, the way the
42
+ // answered and failed paths commit theirs, so the run's spend reads incomplete and never omits it.
43
+ if (open.status === "requested")
44
+ driver.decisionEnded(params.executionPath, decisionId, "interrupted", null, () => driver.recordQuestionSpend(null));
45
+ await driver.flush();
46
+ throw new EscalationSignal({ kind: DECISION_INTERRUPTED, stage: "decision", executionPath: params.executionPath, label: params.label, state: driver.lastState(),
47
+ summary: `The decision "${params.label}" was asked by an earlier process and its answer never reached a receipt; it is not asked again.` });
48
+ }
49
+ try {
50
+ // The request is durable before the judge is asked.
51
+ await driver.flush();
52
+ let result;
53
+ try {
54
+ result = await runJudge(params);
55
+ }
56
+ catch (error) {
57
+ // A request sent whose outcome was lost may have been answered and billed: it is interrupted, never asked again,
58
+ // and its estimate (or an unknown cost, which makes the spend read incomplete) is booked with that outcome.
59
+ const charge = chargedCost(error);
60
+ if (error instanceof DecisionProviderError && error.code === "outcome_unknown") {
61
+ driver.decisionEnded(params.executionPath, decisionId, "interrupted", charge ?? null, () => driver.recordQuestionSpend(charge ?? null));
62
+ throw error;
63
+ }
64
+ // An answer rejected after it was paid for failed, and its cost (or an unknown one) is booked with that outcome:
65
+ // the next process asks again.
66
+ if (charge !== undefined) {
67
+ driver.decisionEnded(params.executionPath, decisionId, "failed", charge, () => driver.recordQuestionSpend(charge));
68
+ throw error;
69
+ }
70
+ // A cut call (a cancel, the run's failure) has no outcome: a later open finds it asked and unanswered. A call
71
+ // that ended on its own failed, and the next process asks again.
72
+ if (!driver.signal.aborted && !params.signal?.aborted)
73
+ driver.decisionEnded(params.executionPath, decisionId, "failed", null);
74
+ throw error;
75
+ }
76
+ const cost = typeof result.cost_usd === "number" && Number.isFinite(result.cost_usd) ? result.cost_usd : null;
77
+ // An answer the interpreter would refuse is spent at its price and is never a decision: the question is
78
+ // asked again by the next process.
79
+ try {
80
+ validateAnswers(params.questions, result.answers);
81
+ }
82
+ catch (error) {
83
+ driver.decisionEnded(params.executionPath, decisionId, "failed", cost, () => driver.recordQuestionSpend(cost));
84
+ throw error;
85
+ }
86
+ // The outcome, the spend and a route's decision are one commit.
87
+ driver.decisionEnded(params.executionPath, decisionId, "answered", cost, () => driver.recordQuestionSpend(cost, params.kind === "route"
88
+ ? { executionPath: params.executionPath, label: params.label, result: result, receipts: null } : undefined));
89
+ // The decision is durable before the interpreter acts on the answer, and a stop taken meanwhile ends the run here.
90
+ await driver.flush();
91
+ driver.checkStop();
92
+ return result;
93
+ }
94
+ finally {
95
+ driver.decisionSettled();
96
+ }
97
+ }),
98
+ runNode: runNode && (async (params) => {
99
+ const admit = () => driver.stepSession(params.executionPath, params.label, { attemptsAllowed: 2 });
100
+ let admission = admit();
101
+ for (;;) {
102
+ // An attempt that delivered a submission is asked for again only when its node never committed. A runner that
103
+ // keeps what it delivered answers it again below, as the same attempt; with any other runner it is spent.
104
+ if (admission.status === "submitted" && !options.durableNodes) {
105
+ driver.stepAttemptFailed(params.executionPath);
106
+ admission = admit();
107
+ }
108
+ // A step that failed on an item failure is refused with it, so the item it lies in ends on the same failure.
109
+ if (admission.status === "failed") {
110
+ throw admission.failure ? Object.assign(new Error(admission.failure.message), { code: admission.failure.code, itemFailure: true })
111
+ : new Error(`${params.label} failed after ${admission.attemptsAllowed} attempts`);
112
+ }
113
+ // The step's record is durable before the attempt runs. A stop or a caller's cancellation that landed
114
+ // meanwhile ends the step here: the runner is not called, and no attempt is spent. A pause waiting for an
115
+ // effect in flight holds the step here until it is taken.
116
+ await driver.flush();
117
+ await driver.seam();
118
+ if (params.signal?.aborted)
119
+ throw params.signal.reason;
120
+ try {
121
+ const submission = await runNode({ ...params, step: { sessionId: admission.sessionId, attempt: admission.attempt, earlierSessionIds: admission.earlierSessionIds } });
122
+ driver.stepSubmitted(params.executionPath);
123
+ return submission;
124
+ }
125
+ catch (error) {
126
+ // A stop the driver took, or a sibling's failure, ends the step where it is: no attempt is spent.
127
+ if (driver.signal.aborted || params.signal?.aborted)
128
+ throw error;
129
+ // A failure the runner marked final (an attempt that ended on a bound of its own) gets no second attempt: it
130
+ // would spend the same again.
131
+ const final = error?.final === true;
132
+ driver.stepAttemptFailed(params.executionPath, final, error);
133
+ if (final)
134
+ throw error;
135
+ admission = admit();
136
+ if (admission.status !== "running")
137
+ throw error;
138
+ }
139
+ }
140
+ }),
141
+ };
142
+ }
@@ -0,0 +1,9 @@
1
+ /** The input schema of a node runner's `submit` tool: the node's output schema under `value`, as
2
+ * `{ value: <schema> }`. A node schema is not a valid tool input as it stands (a root `$ref` or `$defs`
3
+ * is not an object schema), and embedding it moves the document root its local pointers resolve
4
+ * against, so each local `$ref` (`#`, `#/...`) is relocated under `#/properties/value`. A subschema
5
+ * that sets its own `$id` is a separate resource: its local references do not move. Every runner that
6
+ * delivers a node's record through a tool call uses this one envelope, so a schema behaves the same on
7
+ * every harness. A keyword of the DSL's own (`criteria`) is left out of every schema in it, never a property or
8
+ * definition of that name; the submission is still checked against the node's schema as written. */
9
+ export declare function submitParameters(schema: unknown): Record<string, unknown>;
@@ -0,0 +1,40 @@
1
+ // DSL vocabulary that may sit on a schema but is not JSON Schema: System One's question keyword. The model never sees it.
2
+ const DSL_ONLY_SCHEMA_KEYWORDS = new Set(["criteria"]);
3
+ /** The input schema of a node runner's `submit` tool: the node's output schema under `value`, as
4
+ * `{ value: <schema> }`. A node schema is not a valid tool input as it stands (a root `$ref` or `$defs`
5
+ * is not an object schema), and embedding it moves the document root its local pointers resolve
6
+ * against, so each local `$ref` (`#`, `#/...`) is relocated under `#/properties/value`. A subschema
7
+ * that sets its own `$id` is a separate resource: its local references do not move. Every runner that
8
+ * delivers a node's record through a tool call uses this one envelope, so a schema behaves the same on
9
+ * every harness. A keyword of the DSL's own (`criteria`) is left out of every schema in it, never a property or
10
+ * definition of that name; the submission is still checked against the node's schema as written. */
11
+ export function submitParameters(schema) {
12
+ const maps = new Set(["properties", "patternProperties", "$defs", "definitions", "dependentSchemas"]);
13
+ const singles = new Set(["additionalProperties", "unevaluatedProperties", "propertyNames", "items", "additionalItems", "contains", "unevaluatedItems", "not", "if", "then", "else", "contentSchema"]);
14
+ const arrays = new Set(["allOf", "anyOf", "oneOf", "prefixItems"]);
15
+ const relocate = (node, resourceScope = false) => {
16
+ if (!node || typeof node !== "object" || Array.isArray(node))
17
+ return structuredClone(node);
18
+ // $id establishes a separate schema resource: its local references do not move.
19
+ resourceScope ||= typeof node.$id === "string" && node.$id !== "";
20
+ const child = (value) => relocate(value, resourceScope);
21
+ return Object.fromEntries(Object.entries(node).filter(([key]) => !DSL_ONLY_SCHEMA_KEYWORDS.has(key)).map(([key, value]) => {
22
+ if (!resourceScope && key === "$ref" && typeof value === "string" && (value === "#" || value.startsWith("#/"))) {
23
+ return [key, `#/properties/value${value.slice(1)}`];
24
+ }
25
+ if (maps.has(key) && value && typeof value === "object" && !Array.isArray(value)) {
26
+ return [key, Object.fromEntries(Object.entries(value).map(([name, value]) => [name, child(value)]))];
27
+ }
28
+ if (arrays.has(key) && Array.isArray(value))
29
+ return [key, value.map(child)];
30
+ if (singles.has(key))
31
+ return [key, Array.isArray(value) ? value.map(child) : child(value)];
32
+ if (key === "dependencies" && value && typeof value === "object" && !Array.isArray(value)) {
33
+ return [key, Object.fromEntries(Object.entries(value).map(([name, value]) => [name, Array.isArray(value) ? structuredClone(value) : child(value)]))];
34
+ }
35
+ // Annotations and literal values are data, not schemas (including a literal $ref).
36
+ return [key, structuredClone(value)];
37
+ }));
38
+ };
39
+ return { type: "object", properties: { value: relocate(schema) }, required: ["value"], additionalProperties: false };
40
+ }
@@ -13,6 +13,18 @@ export type SystemOneQuestion = {
13
13
  type: "score";
14
14
  instructions: string;
15
15
  criteria: string[];
16
+ }
17
+ /** Locates what its instructions describe in the request's image. A host-side question: no workflow node declares one, and it needs an image sent with the request. */
18
+ | {
19
+ type: "box";
20
+ instructions: string;
21
+ };
22
+ /** A box's edges as fractions of the image's height (`top`, `bottom`) and width (`left`, `right`), from the top left corner. */
23
+ export type SystemOneBox = {
24
+ top: number;
25
+ bottom: number;
26
+ left: number;
27
+ right: number;
16
28
  };
17
29
  export type SystemOneAnswer = {
18
30
  type: "choice";
@@ -28,6 +40,14 @@ export type SystemOneAnswer = {
28
40
  legend: Record<string, string>;
29
41
  probabilities: Record<string, number>;
30
42
  confidence: number;
43
+ } | {
44
+ type: "box";
45
+ box: SystemOneBox;
46
+ confidence: number | null;
47
+ }
48
+ /** The provider declined to answer this question. It is never decoded as a value. */
49
+ | {
50
+ type: "refusal";
31
51
  };
32
52
  export type SystemOneResult = {
33
53
  answers: Record<string, SystemOneAnswer>;
@@ -65,7 +85,7 @@ export type AnswersSidecar = {
65
85
  export declare function answersSidecar(answers: Record<string, SystemOneAnswer>): AnswersSidecar;
66
86
  export declare function synthesizeAnswers(questions: Record<string, SystemOneQuestion>): Record<string, SystemOneAnswer>;
67
87
  export type SystemOneRetryClass = "http_429" | "http_5xx" | "connection" | "timeout";
68
- declare const responseReasons: readonly ["answers_shape", "answer_keys", "answer_shape", "answer_type", "noul_probability", "confidence", "probabilities", "probability_keys", "probability_mass", "score_legend", "score_consistency", "choice_option", "score_range"];
88
+ declare const responseReasons: readonly ["answers_shape", "answer_keys", "answer_shape", "answer_type", "noul_probability", "confidence", "box_coordinates", "probabilities", "probability_keys", "probability_mass", "score_legend", "score_consistency", "choice_option", "score_range", "refusal"];
69
89
  export type SystemOneResponseReason = typeof responseReasons[number];
70
90
  export declare function isSystemOneResponseReason(value: unknown): value is SystemOneResponseReason;
71
91
  export declare class SystemOneError extends Error {
@@ -75,4 +95,23 @@ export declare class SystemOneError extends Error {
75
95
  constructor(message: string, retryClass: SystemOneRetryClass | null, status?: number | undefined, responseReason?: SystemOneResponseReason);
76
96
  }
77
97
  export declare function validateAnswers(questions: Record<string, SystemOneQuestion>, answers: unknown): asserts answers is Record<string, SystemOneAnswer>;
98
+ /** The questions whose answer is a refusal, in answer order. */
99
+ export declare function refusedQuestions(answers: Record<string, unknown>): string[];
100
+ /** What `answersToValue` throws for a refused answer: the backstop. A decision node never gets this far, since `decide()`
101
+ * escalates the run for review (DECISION_REFUSED) on a refusal first. */
102
+ export declare function refusalError(ids: string[]): SystemOneError;
103
+ /** The confidence TypeSafe documents for Jev (docs.typesafe.ai/confidence), computed from an answer's probabilities: for a
104
+ * choice of n options, (p - 1/n) / (1 - 1/n) for the chosen option's p, floored at 0; for a score of n levels, 1 minus
105
+ * the probability-weighted distance from the most likely level (the lowest, on a tie) over that distance for an even
106
+ * spread, floored at 0. Null when the probabilities are missing or are not a distribution over the answer's options. */
107
+ export declare function deriveConfidence(answer: unknown): number | null;
108
+ /** Where an answer's confidence came from: the provider's response, or derived from its probabilities. */
109
+ export type ConfidenceSource = "provider" | "derived";
110
+ /** The answers with every choice and score carrying a confidence: the provider's own when it gave one in [0, 1], else
111
+ * one derived from the probabilities (deriveConfidence), else 0, which validation then refuses with the probabilities
112
+ * that left nothing to derive from. `confidence_source` names each choice and score answer's source. */
113
+ export declare function completeConfidence(questions: Record<string, SystemOneQuestion>, answers: Record<string, unknown>): {
114
+ answers: Record<string, unknown>;
115
+ confidence_source: Record<string, ConfidenceSource>;
116
+ };
78
117
  export {};
@@ -84,6 +84,9 @@ export function compileQuestions(schema, definitions = {}) {
84
84
  }
85
85
  export function answersToValue(answers, questions) {
86
86
  const out = Object.create(null);
87
+ const refused = refusedQuestions(answers).filter((id) => Object.hasOwn(questions, id));
88
+ if (refused.length)
89
+ throw refusalError(refused);
87
90
  for (const [id, q] of Object.entries(questions)) {
88
91
  const a = answers[id];
89
92
  if (!a)
@@ -94,6 +97,8 @@ export function answersToValue(answers, questions) {
94
97
  out[id] = a.noul >= 0.5;
95
98
  else if (q.type === "score" && a.type === "score")
96
99
  out[id] = Math.max(0, Math.round(a.score));
100
+ else if (q.type === "box" && a.type === "box")
101
+ out[id] = { ...a.box };
97
102
  else
98
103
  throw new Error(`System One answered question "${id}" with type "${a.type}", expected ${q.type}`);
99
104
  }
@@ -102,8 +107,11 @@ export function answersToValue(answers, questions) {
102
107
  export function answerConfidence(answer) {
103
108
  if (!answer || typeof answer !== "object")
104
109
  throw new SystemOneError("System One answer missing for a question that was asked", null);
110
+ if (answer.type === "refusal")
111
+ return 0;
105
112
  if (answer.type === "noul")
106
113
  return Math.abs(answer.noul - 0.5) * 2;
114
+ // A box the provider reported no confidence for counts as none: unknown is never read as sure.
107
115
  return typeof answer.confidence === "number" ? answer.confidence : 0;
108
116
  }
109
117
  export function answersSidecar(answers) {
@@ -124,6 +132,8 @@ export function synthesizeAnswers(questions) {
124
132
  const options = Object.keys(q.criteria);
125
133
  out[id] = { type: "choice", choice: options[0], probabilities: Object.fromEntries(options.map((o) => [o, 1 / options.length])), confidence: 0 };
126
134
  }
135
+ else if (q.type === "box")
136
+ out[id] = { type: "box", box: { top: 0, bottom: 1, left: 0, right: 1 }, confidence: 0 };
127
137
  else if (q.type === "noul")
128
138
  out[id] = { type: "noul", noul: 0.5 };
129
139
  else
@@ -132,7 +142,7 @@ export function synthesizeAnswers(questions) {
132
142
  return { ...out };
133
143
  }
134
144
  const responseReasons = ["answers_shape", "answer_keys", "answer_shape", "answer_type", "noul_probability",
135
- "confidence", "probabilities", "probability_keys", "probability_mass", "score_legend", "score_consistency", "choice_option", "score_range"];
145
+ "confidence", "box_coordinates", "probabilities", "probability_keys", "probability_mass", "score_legend", "score_consistency", "choice_option", "score_range", "refusal"];
136
146
  export function isSystemOneResponseReason(value) {
137
147
  return typeof value === "string" && responseReasons.includes(value);
138
148
  }
@@ -150,6 +160,56 @@ export class SystemOneError extends Error {
150
160
  }
151
161
  }
152
162
  const unit = (v) => typeof v === "number" && Number.isFinite(v) && v >= 0 && v <= 1;
163
+ // System One reports probabilities rounded, Jev to two places and some OpenRouter models to four (Cloudflare's clef,
164
+ // live: 0.2967 + 0.6661 + 0.0371 = 0.9999), so the reported sum drifts from 1 by up to half a step per option
165
+ // (3 × 0.33 = 0.99; 8 × 0.13 = 1.04). A distribution is accepted when it could be that rounding of one summing to 1: a
166
+ // value on the two-, three- or four-place grid may be off by half that grid's step (within [0, 1]), the coarsest grid it
167
+ // lies on; any other value is taken as exact. With no grid values this is |sum − 1| ≤ 1e-5, the check for
168
+ // full-precision distributions. Answers are accepted as reported, never rescaled.
169
+ const roundingHalfStep = (p) => {
170
+ for (const places of [2, 3, 4]) {
171
+ const scale = 10 ** places;
172
+ if (Math.abs(p * scale - Math.round(p * scale)) < 1e-9 * scale / 100)
173
+ return 0.5 / scale;
174
+ }
175
+ return 0;
176
+ };
177
+ function roundedFromUnitMass(probabilities) {
178
+ let low = 0, high = 0;
179
+ for (const p of probabilities) {
180
+ const step = roundingHalfStep(p);
181
+ low += Math.max(0, p - step);
182
+ high += Math.min(1, p + step);
183
+ }
184
+ return low <= 1 + 1e-5 && high >= 1 - 1e-5;
185
+ }
186
+ // A Score's weighted level drifts the same way. The unrounded probabilities lie within their rounding
187
+ // intervals and sum to 1, so Σ level·p lies between two bounds: start every level at its interval's
188
+ // low end, then give the rest of the unit mass to the lowest levels (the minimum) or the highest (the
189
+ // maximum), each up to its interval's high end. A score on a grid is itself off by up to half its step. The
190
+ // score is accepted when its interval meets [min, max]; with no grid values this is
191
+ // |score − Σ level·p| ≤ 1e-5. A score that is not a number contradicts nothing here and is left to
192
+ // the range check, as before.
193
+ function scoreContradictsDistribution(score, probabilities) {
194
+ const levels = Object.entries(probabilities).map(([level, p]) => {
195
+ const step = roundingHalfStep(p);
196
+ return { level: Number(level), low: Math.max(0, p - step), high: Math.min(1, p + step) };
197
+ }).sort((a, b) => a.level - b.level);
198
+ const floor = levels.reduce((sum, l) => sum + l.level * l.low, 0);
199
+ const spare = Math.max(0, 1 - levels.reduce((sum, l) => sum + l.low, 0));
200
+ const fill = (order) => {
201
+ let rest = spare, weighted = floor;
202
+ for (const l of order) {
203
+ const add = Math.min(rest, l.high - l.low);
204
+ weighted += l.level * add;
205
+ rest -= add;
206
+ }
207
+ return weighted;
208
+ };
209
+ const min = fill(levels), max = fill([...levels].reverse());
210
+ const step = roundingHalfStep(score);
211
+ return score - step > max + 1e-5 || score + step < min - 1e-5;
212
+ }
153
213
  export function validateAnswers(questions, answers) {
154
214
  if (!answers || typeof answers !== "object" || Array.isArray(answers))
155
215
  throw new SystemOneError("System One response carries no answers", null, undefined, "answers_shape");
@@ -160,8 +220,24 @@ export function validateAnswers(questions, answers) {
160
220
  const a = answers[id];
161
221
  if (!a || typeof a !== "object")
162
222
  throw new SystemOneError("System One returned no answer for a question", null, undefined, "answer_shape");
223
+ if (a.type === "refusal") {
224
+ if (Object.keys(a).length !== 1)
225
+ throw new SystemOneError("System One answer: a refusal carries nothing but its type", null, undefined, "answer_shape");
226
+ continue;
227
+ }
163
228
  if (a.type !== q.type)
164
229
  throw new SystemOneError("System One answer type does not match its question", null, undefined, "answer_type");
230
+ if (q.type === "box") {
231
+ const box = a.box;
232
+ if (!box || typeof box !== "object" || Array.isArray(box) || Object.keys(box).length !== 4 || !["top", "bottom", "left", "right"].every(edge => unit(box[edge])))
233
+ throw new SystemOneError("System One answer: a box's edges are not fractions in [0, 1]", null, undefined, "box_coordinates");
234
+ // An edge pair that runs backwards is a box with negative width or height: not a box.
235
+ if (box.top > box.bottom || box.left > box.right)
236
+ throw new SystemOneError("System One answer: a box's top and left edges cannot lie past its bottom and right", null, undefined, "box_coordinates");
237
+ if (!(a.confidence === null || unit(a.confidence)))
238
+ throw new SystemOneError("System One answer: confidence is not in [0, 1]", null, undefined, "confidence");
239
+ continue;
240
+ }
165
241
  if (q.type === "noul") {
166
242
  if (!unit(a.noul))
167
243
  throw new SystemOneError("System One answer: noul is not a probability", null, undefined, "noul_probability");
@@ -175,14 +251,12 @@ export function validateAnswers(questions, answers) {
175
251
  const probabilityKeys = Object.keys(a.probabilities);
176
252
  if (Array.isArray(a.probabilities) || probabilityKeys.length !== expectedKeys.length || expectedKeys.some(key => !Object.hasOwn(a.probabilities, key)))
177
253
  throw new SystemOneError("System One answer: probabilities must contain exactly the declared options", null, undefined, "probability_keys");
178
- const mass = Object.values(a.probabilities).reduce((sum, p) => sum + p, 0);
179
- if (Math.abs(mass - 1) > 1e-5)
254
+ if (!roundedFromUnitMass(Object.values(a.probabilities)))
180
255
  throw new SystemOneError("System One answer: probabilities must sum to 1", null, undefined, "probability_mass");
181
256
  if (q.type === "score") {
182
257
  if (!a.legend || typeof a.legend !== "object" || Array.isArray(a.legend) || Object.keys(a.legend).length !== expectedKeys.length || q.criteria.some((text, index) => a.legend[String(index)] !== text))
183
258
  throw new SystemOneError("System One answer: legend must match the score criteria", null, undefined, "score_legend");
184
- const weighted = expectedKeys.reduce((sum, key) => sum + Number(key) * a.probabilities[key], 0);
185
- if (Math.abs(weighted - a.score) > 1e-5)
259
+ if (scoreContradictsDistribution(a.score, a.probabilities))
186
260
  throw new SystemOneError("System One answer: score must match its weighted distribution", null, undefined, "score_consistency");
187
261
  }
188
262
  if (q.type === "choice" && !(typeof a.choice === "string" && Object.prototype.hasOwnProperty.call(q.criteria, a.choice)))
@@ -191,3 +265,65 @@ export function validateAnswers(questions, answers) {
191
265
  throw new SystemOneError("System One answer: score is outside its levels", null, undefined, "score_range");
192
266
  }
193
267
  }
268
+ /** The questions whose answer is a refusal, in answer order. */
269
+ export function refusedQuestions(answers) {
270
+ return Object.entries(answers).filter(([, a]) => isRecord(a) && a.type === "refusal").map(([id]) => id);
271
+ }
272
+ /** What `answersToValue` throws for a refused answer: the backstop. A decision node never gets this far, since `decide()`
273
+ * escalates the run for review (DECISION_REFUSED) on a refusal first. */
274
+ export function refusalError(ids) {
275
+ return new SystemOneError(`System One refused ${ids.length === 1 ? "question" : "questions"} ${ids.map((id) => `"${id}"`).join(", ")}`, null, undefined, "refusal");
276
+ }
277
+ const isRecord = (v) => v !== null && typeof v === "object" && !Array.isArray(v);
278
+ /** The confidence TypeSafe documents for Jev (docs.typesafe.ai/confidence), computed from an answer's probabilities: for a
279
+ * choice of n options, (p - 1/n) / (1 - 1/n) for the chosen option's p, floored at 0; for a score of n levels, 1 minus
280
+ * the probability-weighted distance from the most likely level (the lowest, on a tie) over that distance for an even
281
+ * spread, floored at 0. Null when the probabilities are missing or are not a distribution over the answer's options. */
282
+ export function deriveConfidence(answer) {
283
+ if (!isRecord(answer) || !isRecord(answer.probabilities))
284
+ return null;
285
+ const probabilities = answer.probabilities;
286
+ const values = Object.values(probabilities);
287
+ const n = values.length;
288
+ if (n < 2 || !values.every(unit))
289
+ return null;
290
+ if (answer.type === "choice") {
291
+ if (typeof answer.choice !== "string" || !Object.hasOwn(probabilities, answer.choice))
292
+ return null;
293
+ return Math.min(1, Math.max(0, (probabilities[answer.choice] - 1 / n) / (1 - 1 / n)));
294
+ }
295
+ if (answer.type === "score") {
296
+ const levels = Array.from({ length: n }, (_, i) => probabilities[String(i)]);
297
+ if (!levels.every(unit))
298
+ return null;
299
+ const p = levels;
300
+ const mode = p.indexOf(Math.max(...p));
301
+ const spread = p.reduce((sum, pi, i) => sum + pi * Math.abs(i - mode), 0);
302
+ const even = p.reduce((sum, _, i) => sum + Math.abs(i - (n - 1) / 2), 0) / n;
303
+ return Math.min(1, Math.max(0, 1 - spread / even));
304
+ }
305
+ return null;
306
+ }
307
+ /** The answers with every choice and score carrying a confidence: the provider's own when it gave one in [0, 1], else
308
+ * one derived from the probabilities (deriveConfidence), else 0, which validation then refuses with the probabilities
309
+ * that left nothing to derive from. `confidence_source` names each choice and score answer's source. */
310
+ export function completeConfidence(questions, answers) {
311
+ // Null-prototype records: a question may be named __proto__, which on a plain object would set its prototype.
312
+ const out = Object.create(null);
313
+ const confidence_source = Object.create(null);
314
+ for (const [id, answer] of Object.entries(answers)) {
315
+ const asked = Object.hasOwn(questions, id) ? questions[id] : undefined;
316
+ if (!isRecord(answer) || (answer.type !== "choice" && answer.type !== "score") || asked?.type !== answer.type) {
317
+ out[id] = answer;
318
+ continue;
319
+ }
320
+ if (unit(answer.confidence)) {
321
+ out[id] = answer;
322
+ confidence_source[id] = "provider";
323
+ continue;
324
+ }
325
+ out[id] = { ...answer, confidence: deriveConfidence(answer) ?? 0 };
326
+ confidence_source[id] = "derived";
327
+ }
328
+ return { answers: { ...out }, confidence_source: { ...confidence_source } };
329
+ }
@@ -0,0 +1,32 @@
1
+ import { type DecisionEvidence, type DecisionProvider } from "../decision-provider.js";
2
+ /** What the kit's fake transport answers one call with. An `answer` carries answers in the DSL's shape (and, for a
3
+ * provider with evidence, sources), which the kit sends in the provider's own wire format. `lost` is a connection that
4
+ * drops after the request was sent: whether it was served is not known. */
5
+ export type KitReply = {
6
+ kind: "answer";
7
+ answers: Record<string, unknown>;
8
+ evidence?: DecisionEvidence[];
9
+ } | {
10
+ kind: "status";
11
+ status: number;
12
+ } | {
13
+ kind: "malformed";
14
+ } | {
15
+ kind: "lost";
16
+ } | {
17
+ kind: "hang";
18
+ };
19
+ export interface DecisionProviderKit {
20
+ /** A new provider under test whose transport answers each call with the next of `replies` (the last repeats), and
21
+ * the number of calls its transport has taken. Retry delays should be short: the suite waits for them. */
22
+ build(replies: KitReply[]): {
23
+ provider: DecisionProvider;
24
+ calls(): number;
25
+ };
26
+ /** The key the kit configures the provider with: it must never appear in a result or an error. */
27
+ canary: string;
28
+ /** Whether the provider's wire format can carry a refusal. */
29
+ refuses: boolean;
30
+ }
31
+ /** Registers the suite as `node:test` tests named under `name`. */
32
+ export declare function registerDecisionProviderConformance(name: string, kit: DecisionProviderKit): void;