@melra/runtime-core 0.3.0-alpha.12 → 0.3.0-alpha.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -2
- package/dist/task-controller.d.ts +81 -3
- package/dist/task-controller.d.ts.map +1 -1
- package/dist/task-controller.js +298 -52
- package/dist/task-controller.js.map +1 -1
- package/dist/workflow-controller.d.ts +19 -1
- package/dist/workflow-controller.d.ts.map +1 -1
- package/dist/workflow-controller.js +357 -91
- package/dist/workflow-controller.js.map +1 -1
- package/dist/workflow-graph.d.ts.map +1 -1
- package/dist/workflow-graph.js +25 -1
- package/dist/workflow-graph.js.map +1 -1
- package/package.json +6 -6
package/README.md
CHANGED
|
@@ -10,8 +10,9 @@ npm install @melra/runtime-core
|
|
|
10
10
|
|
|
11
11
|
`TaskController` owns the pipeline. `plan()` classifies the operation, evaluates
|
|
12
12
|
policy, persists a `TaskRecord`, and returns any approval challenge — it never
|
|
13
|
-
executes. `execute()` re-evaluates policy, validates the approval,
|
|
14
|
-
|
|
13
|
+
executes. `execute()` re-evaluates policy, validates the approval (or, in
|
|
14
|
+
enforced mode, requires an operator's out-of-band decision), runs the adapter
|
|
15
|
+
under an `AbortSignal` armed with the duration budget, then verifies.
|
|
15
16
|
|
|
16
17
|
Verification decides success, not the adapter. A task reaches
|
|
17
18
|
`verified_success` only when the adapter succeeded *and* every declared evidence
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { ApprovalResponse, EffectContract, EvidencePredicate, Operation, PolicyDecision, TaskRecord, TaskRequest } from "@melra/protocol";
|
|
1
|
+
import type { ApprovalDecision, ApprovalResponse, EffectContract, EvidencePredicate, Operation, PolicyDecision, TaskRecord, TaskRequest } from "@melra/protocol";
|
|
2
2
|
import { type CapabilityUsageReader, type LocalPolicy } from "@melra/policy-core";
|
|
3
3
|
import { type ActionReceipt, type EvidenceItem, type ExecutionCertificate } from "@melra/receipt-schema";
|
|
4
4
|
import { SqliteStore } from "@melra/storage-sqlite";
|
|
@@ -18,6 +18,12 @@ export interface ExecutionResult {
|
|
|
18
18
|
export interface TaskPlanOptions {
|
|
19
19
|
idempotencyKey?: string;
|
|
20
20
|
attempt?: number;
|
|
21
|
+
/**
|
|
22
|
+
* Planned again from a workflow definition stored when it was accepted.
|
|
23
|
+
* Checks that only govern new work (an unused browser list) are not applied
|
|
24
|
+
* again, so a run accepted before a check existed does not stop mid-way.
|
|
25
|
+
*/
|
|
26
|
+
stored?: boolean;
|
|
21
27
|
}
|
|
22
28
|
/**
|
|
23
29
|
* A grant's durable draw-down, the way policy wants to read it.
|
|
@@ -35,6 +41,8 @@ export declare class TaskController {
|
|
|
35
41
|
private readonly payloadCipher;
|
|
36
42
|
private readonly active;
|
|
37
43
|
private readonly breaker;
|
|
44
|
+
/** Whether approvals come only from an operator, out of band. */
|
|
45
|
+
get enforced(): boolean;
|
|
38
46
|
constructor(store: SqliteStore, policy: LocalPolicy, executor: OperationExecutor, verifier: Verifier, payloadCipher: PayloadCipher, breaker?: CircuitBreaker);
|
|
39
47
|
/**
|
|
40
48
|
* The durable draw-down on each grant, as policy sees it.
|
|
@@ -45,10 +53,24 @@ export declare class TaskController {
|
|
|
45
53
|
* with a way around it.
|
|
46
54
|
*/
|
|
47
55
|
private readonly capabilityUsage;
|
|
56
|
+
/**
|
|
57
|
+
* Draws by tasks between their re-check and their record, per grant and
|
|
58
|
+
* task. The durable draw is written only once work is verified, so without
|
|
59
|
+
* this two tasks running at once both saw the grant unspent: two operations
|
|
60
|
+
* against `maxOperations: 1`, a parallel workflow node overspending
|
|
61
|
+
* `dailyMax`.
|
|
62
|
+
*
|
|
63
|
+
* ponytail: held in this process. Two servers sharing one database can
|
|
64
|
+
* still both pass a re-check at the same moment; a durable reservation row
|
|
65
|
+
* would close that if it ever matters.
|
|
66
|
+
*/
|
|
67
|
+
private readonly reserved;
|
|
48
68
|
plan(request: TaskRequest, options?: TaskPlanOptions): TaskRecord & {
|
|
49
69
|
contract: EffectContract;
|
|
50
70
|
};
|
|
51
|
-
preflight(request: TaskRequest
|
|
71
|
+
preflight(request: TaskRequest, options?: {
|
|
72
|
+
stored?: boolean;
|
|
73
|
+
}): PolicyDecision;
|
|
52
74
|
status(taskId: string): TaskRecord;
|
|
53
75
|
execute(taskId: string, approval?: ApprovalResponse): Promise<ExecutionResult>;
|
|
54
76
|
cancel(taskId: string): TaskRecord;
|
|
@@ -83,7 +105,57 @@ export declare class TaskController {
|
|
|
83
105
|
* or an effect settled this way would be free to run again and free to spend.
|
|
84
106
|
*/
|
|
85
107
|
private resolveAsVerified;
|
|
86
|
-
|
|
108
|
+
/**
|
|
109
|
+
* With `failed`, each task that could not be recovered is recorded there by
|
|
110
|
+
* id instead of the first failure being thrown at the end.
|
|
111
|
+
*/
|
|
112
|
+
recoverInterrupted(failed?: Map<string, unknown>): Promise<TaskRecord[]>;
|
|
113
|
+
private recoverTask;
|
|
114
|
+
/**
|
|
115
|
+
* What an approver is shown: the effect contract, minus the challenge and
|
|
116
|
+
* the policy verdict, which a recheck at execution rewrites. Its digest is
|
|
117
|
+
* what a decision is bound to, so a request that changed after it was
|
|
118
|
+
* shown cannot run on that decision.
|
|
119
|
+
*/
|
|
120
|
+
private approvalRender;
|
|
121
|
+
/** A pending challenge as an operator surface shows it. */
|
|
122
|
+
approvalView(taskId: string): {
|
|
123
|
+
contract: {
|
|
124
|
+
contractVersion: string;
|
|
125
|
+
taskId: string;
|
|
126
|
+
identity: import("@melra/protocol").Identity;
|
|
127
|
+
capability: string;
|
|
128
|
+
operation: Operation;
|
|
129
|
+
effect: import("@melra/protocol").Effect;
|
|
130
|
+
risk: import("@melra/protocol").Risk;
|
|
131
|
+
target: string;
|
|
132
|
+
traits: import("@melra/protocol").CapabilityTrait[];
|
|
133
|
+
executionGuarantee: import("@melra/protocol").ExecutionGuarantee;
|
|
134
|
+
forbiddenEffects: import("@melra/protocol").Effect[];
|
|
135
|
+
postconditions: EvidencePredicate[];
|
|
136
|
+
budget: import("@melra/protocol").TaskBudget;
|
|
137
|
+
idempotencyKey?: string;
|
|
138
|
+
metadata: {
|
|
139
|
+
goal: string;
|
|
140
|
+
};
|
|
141
|
+
};
|
|
142
|
+
renderDigest: string;
|
|
143
|
+
task: TaskRecord;
|
|
144
|
+
};
|
|
145
|
+
/**
|
|
146
|
+
* Records an approver's yes to exactly the render they were shown. Never
|
|
147
|
+
* executes, and never touches the task row: the decision lives in its own
|
|
148
|
+
* insert-once table, and `execute` reads it. `out_of_band` only in enforced
|
|
149
|
+
* mode, where nothing the agent saw could have produced it.
|
|
150
|
+
*/
|
|
151
|
+
decide(taskId: string, renderDigest: string, approver: string): ApprovalDecision;
|
|
152
|
+
/** The recorded decision for a pending task, while it can still be used. */
|
|
153
|
+
approvalDecision(task: TaskRecord): ApprovalDecision | undefined;
|
|
154
|
+
/**
|
|
155
|
+
* Made before the challenge expired, and used within a TTL of being made:
|
|
156
|
+
* an answer that came in time is not lost to the time it takes to act on.
|
|
157
|
+
*/
|
|
158
|
+
private usable;
|
|
87
159
|
receipts(input: {
|
|
88
160
|
taskId?: string;
|
|
89
161
|
receiptId?: string;
|
|
@@ -91,6 +163,12 @@ export declare class TaskController {
|
|
|
91
163
|
receipts: ActionReceipt[];
|
|
92
164
|
certificate?: ExecutionCertificate;
|
|
93
165
|
};
|
|
166
|
+
/**
|
|
167
|
+
* The mode a task's effect ran under: recorded when it started running, or,
|
|
168
|
+
* for a task that started before that was recorded, the mode on its first
|
|
169
|
+
* receipt.
|
|
170
|
+
*/
|
|
171
|
+
private runMode;
|
|
94
172
|
private createReceipt;
|
|
95
173
|
private loadRequest;
|
|
96
174
|
private executeWithRetries;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"task-controller.d.ts","sourceRoot":"","sources":["../src/task-controller.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EACV,gBAAgB,EAChB,cAAc,EACd,iBAAiB,EACjB,SAAS,EACT,cAAc,EACd,UAAU,EACV,WAAW,EACZ,MAAM,iBAAiB,CAAC;AAUzB,OAAO,
|
|
1
|
+
{"version":3,"file":"task-controller.d.ts","sourceRoot":"","sources":["../src/task-controller.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EACV,gBAAgB,EAChB,gBAAgB,EAChB,cAAc,EACd,iBAAiB,EACjB,SAAS,EACT,cAAc,EACd,UAAU,EACV,WAAW,EACZ,MAAM,iBAAiB,CAAC;AAUzB,OAAO,EAML,KAAK,qBAAqB,EAC1B,KAAK,WAAW,EAEjB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAOL,KAAK,aAAa,EAElB,KAAK,YAAY,EACjB,KAAK,oBAAoB,EAC1B,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACpD,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAChD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEpD,MAAM,WAAW,iBAAiB;IAChC,YAAY,CAAC,IAAI,WAAW,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;IAChD,OAAO,CACL,SAAS,EAAE,SAAS,EACpB,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;CACrC;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,UAAU,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjC,OAAO,CAAC,EAAE,aAAa,CAAC;IACxB,WAAW,CAAC,EAAE,oBAAoB,CAAC;CACpC;AAED,MAAM,WAAW,eAAe;IAC9B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;;OAIG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAuED;;;;;;GAMG;AACH,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,WAAW,GAAG,qBAAqB,CAM/E;AASD,qBAAa,cAAc;IAWvB,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,OAAO,CAAC,QAAQ,CAAC,aAAa;IAdhC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAsC;IAE7D,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAiB;IAEzC,iEAAiE;IACjE,IAAI,QAAQ,IAAI,OAAO,CAEtB;gBAGkB,KAAK,EAAE,WAAW,EAClB,MAAM,EAAE,WAAW,EACnB,QAAQ,EAAE,iBAAiB,EAC3B,QAAQ,EAAE,QAAQ,EAClB,aAAa,EAAE,aAAa,EAC7C,OAAO,CAAC,EAAE,cAAc;IAY1B;;;;;;;OAOG;IACH,OAAO,CAAC,QAAQ,CAAC,eAAe,CAO9B;IAEF;;;;;;;;;;OAUG;IACH,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA0C;IAEnE,IAAI,CACF,OAAO,EAAE,WAAW,EACpB,OAAO,GAAE,eAAoB,GAC5B,UAAU,GAAG;QAAE,QAAQ,EAAE,cAAc,CAAA;KAAE;IA8D5C,SAAS,CAAC,OAAO,EAAE,WAAW,EAAE,OAAO,GAAE;QAAE,MAAM,CAAC,EAAE,OAAO,CAAA;KAAO,GAAG,cAAc;IAqCnF,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,UAAU;IAM5B,OAAO,CACX,MAAM,EAAE,MAAM,EACd,QAAQ,CAAC,EAAE,gBAAgB,GAC1B,OAAO,CAAC,eAAe,CAAC;IAwS3B,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,UAAU;IAe5B,eAAe,CACnB,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,iBAAiB,EAAE,GAC9B,OAAO,CAAC;QAAE,QAAQ,EAAE,OAAO,CAAC;QAAC,QAAQ,EAAE,YAAY,EAAE,CAAA;KAAE,CAAC;IAerD,gBAAgB,CACpB,UAAU,EAAE,iBAAiB,EAAE,GAC9B,OAAO,CAAC;QAAE,QAAQ,EAAE,OAAO,CAAC;QAAC,QAAQ,EAAE,YAAY,EAAE,CAAA;KAAE,CAAC;IAI3D;;;;;;;;;;;;OAYG;IACG,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IA6DzD;;;;;;;OAOG;IACH,OAAO,CAAC,iBAAiB;IAuDzB;;;OAGG;IACG,kBAAkB,CAAC,MAAM,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC;YAkChE,WAAW;IAkDzB;;;;;OAKG;IACH,OAAO,CAAC,cAAc;IAQtB,2DAA2D;IAC3D,YAAY,CAAC,MAAM,EAAE,MAAM;;;;;;;;;;;;;;;;;;;;;;;IAQ3B;;;;;OAKG;IACH,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,gBAAgB;IAqBhF,4EAA4E;IAC5E,gBAAgB,CAAC,IAAI,EAAE,UAAU,GAAG,gBAAgB,GAAG,SAAS;IAUhE;;;OAGG;IACH,OAAO,CAAC,MAAM;IASd,QAAQ,CAAC,KAAK,EAAE;QACd,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,GAAG;QACF,QAAQ,EAAE,aAAa,EAAE,CAAC;QAC1B,WAAW,CAAC,EAAE,oBAAoB,CAAC;KACpC;IAmBD;;;;OAIG;IACH,OAAO,CAAC,OAAO;IAMf,OAAO,CAAC,aAAa;IAkErB,OAAO,CAAC,WAAW;YAQL,kBAAkB;IAsChC,OAAO,CAAC,wBAAwB;IAkChC,OAAO,CAAC,oBAAoB;CAO7B"}
|
package/dist/task-controller.js
CHANGED
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
// SPDX-License-Identifier: Apache-2.0
|
|
3
3
|
import { randomUUID } from "node:crypto";
|
|
4
4
|
import { allowsRetry, delegationChain, effectContract, executionGuaranteeFor, isOutcomeUnknown, LOCAL_IDENTITY, TaskRequestSchema, } from "@melra/protocol";
|
|
5
|
-
import { CAPABILITY_DAY_MS, classifyOperation, defaultEvidenceFor, evaluatePolicy, validateApproval, } from "@melra/policy-core";
|
|
6
|
-
import { createCertificate, createReceiptId, evidenceStrength, redactStructuredValue, sha256, } from "@melra/receipt-schema";
|
|
5
|
+
import { approvalTtl, CAPABILITY_DAY_MS, classifyOperation, defaultEvidenceFor, evaluatePolicy, validateApproval, } from "@melra/policy-core";
|
|
6
|
+
import { createCertificate, createReceiptId, evidenceStrength, redactSecrets, redactStructuredValue, sha256, } from "@melra/receipt-schema";
|
|
7
7
|
import { CircuitBreaker } from "./circuit-breaker.js";
|
|
8
8
|
function now() {
|
|
9
9
|
return new Date().toISOString();
|
|
@@ -83,6 +83,12 @@ export function capabilityUsageReader(store) {
|
|
|
83
83
|
return { operations: drawn.operations, amountToday: drawn.amountInWindow };
|
|
84
84
|
};
|
|
85
85
|
}
|
|
86
|
+
/** The one browser action that reads each list. */
|
|
87
|
+
const BROWSER_LIST_ACTIONS = [
|
|
88
|
+
["values", "select"],
|
|
89
|
+
["fields", "fill_form"],
|
|
90
|
+
["filePaths", "upload"],
|
|
91
|
+
];
|
|
86
92
|
export class TaskController {
|
|
87
93
|
store;
|
|
88
94
|
policy;
|
|
@@ -91,6 +97,10 @@ export class TaskController {
|
|
|
91
97
|
payloadCipher;
|
|
92
98
|
active = new Map();
|
|
93
99
|
breaker;
|
|
100
|
+
/** Whether approvals come only from an operator, out of band. */
|
|
101
|
+
get enforced() {
|
|
102
|
+
return this.policy.mode === "enforced";
|
|
103
|
+
}
|
|
94
104
|
constructor(store, policy, executor, verifier, payloadCipher, breaker) {
|
|
95
105
|
this.store = store;
|
|
96
106
|
this.policy = policy;
|
|
@@ -112,7 +122,26 @@ export class TaskController {
|
|
|
112
122
|
* preflight — a bound that only one of the three enforced would be a bound
|
|
113
123
|
* with a way around it.
|
|
114
124
|
*/
|
|
115
|
-
capabilityUsage = (grantId) =>
|
|
125
|
+
capabilityUsage = (grantId) => {
|
|
126
|
+
const drawn = capabilityUsageReader(this.store)(grantId);
|
|
127
|
+
const held = [...(this.reserved.get(grantId)?.values() ?? [])];
|
|
128
|
+
return {
|
|
129
|
+
operations: drawn.operations + held.length,
|
|
130
|
+
amountToday: drawn.amountToday + held.reduce((total, amount) => total + amount, 0),
|
|
131
|
+
};
|
|
132
|
+
};
|
|
133
|
+
/**
|
|
134
|
+
* Draws by tasks between their re-check and their record, per grant and
|
|
135
|
+
* task. The durable draw is written only once work is verified, so without
|
|
136
|
+
* this two tasks running at once both saw the grant unspent: two operations
|
|
137
|
+
* against `maxOperations: 1`, a parallel workflow node overspending
|
|
138
|
+
* `dailyMax`.
|
|
139
|
+
*
|
|
140
|
+
* ponytail: held in this process. Two servers sharing one database can
|
|
141
|
+
* still both pass a re-check at the same moment; a durable reservation row
|
|
142
|
+
* would close that if it ever matters.
|
|
143
|
+
*/
|
|
144
|
+
reserved = new Map();
|
|
116
145
|
plan(request, options = {}) {
|
|
117
146
|
const parsedRequest = withDefaultEvidence(TaskRequestSchema.parse(request));
|
|
118
147
|
if (options.idempotencyKey !== undefined &&
|
|
@@ -123,7 +152,7 @@ export class TaskController {
|
|
|
123
152
|
(!Number.isInteger(options.attempt) || options.attempt < 1)) {
|
|
124
153
|
throw new Error("idempotency_attempt_invalid");
|
|
125
154
|
}
|
|
126
|
-
this.preflight(parsedRequest);
|
|
155
|
+
this.preflight(parsedRequest, options);
|
|
127
156
|
const id = randomUUID();
|
|
128
157
|
const policy = evaluatePolicy(id, parsedRequest, this.policy, this.capabilityUsage);
|
|
129
158
|
const timestamp = now();
|
|
@@ -160,13 +189,22 @@ export class TaskController {
|
|
|
160
189
|
contract: effectContract(planned, classifyOperation(parsedRequest.operation)),
|
|
161
190
|
};
|
|
162
191
|
}
|
|
163
|
-
preflight(request) {
|
|
192
|
+
preflight(request, options = {}) {
|
|
164
193
|
const parsed = withDefaultEvidence(TaskRequestSchema.parse(request));
|
|
165
194
|
const capabilities = this.executor.capabilities?.();
|
|
166
195
|
if (capabilities !== undefined &&
|
|
167
196
|
!capabilities.has(parsed.operation.kind)) {
|
|
168
197
|
throw new Error(`operation_capability_unavailable:${parsed.operation.kind}`);
|
|
169
198
|
}
|
|
199
|
+
// A list the browser action never reads would sit in the approval as a
|
|
200
|
+
// decoy: shown to the person approving, doing nothing.
|
|
201
|
+
if (parsed.operation.kind === "browser" && options.stored !== true) {
|
|
202
|
+
for (const [field, action] of BROWSER_LIST_ACTIONS) {
|
|
203
|
+
if (parsed.operation[field] !== undefined && parsed.operation.action !== action) {
|
|
204
|
+
throw new Error(`browser_field_unused:${field}`);
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
}
|
|
170
208
|
// Refused before anything is approved: a mutation run against evidence no
|
|
171
209
|
// registered verifier can check would end `partial` after the effect.
|
|
172
210
|
for (const predicate of parsed.requiredEvidence) {
|
|
@@ -225,22 +263,71 @@ export class TaskController {
|
|
|
225
263
|
task.approval?.actionDigest !== rechecked.challenge.actionDigest) {
|
|
226
264
|
throw new Error("approval_action_digest_mismatch");
|
|
227
265
|
}
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
266
|
+
// Enforced mode asserts the agent holds no credential, so anything it
|
|
267
|
+
// sends is refused rather than ignored, and only an operator's
|
|
268
|
+
// out-of-band decision lets the effect run.
|
|
269
|
+
if (this.policy.mode === "enforced" && approval !== undefined) {
|
|
270
|
+
throw new Error("approval_in_band_refused");
|
|
271
|
+
}
|
|
272
|
+
// A decision an operator recorded wins over anything the caller sent.
|
|
273
|
+
const render = this.approvalRender(task, request).renderDigest;
|
|
274
|
+
const decision = this.store.getApprovalDecision(task.approval.approvalId);
|
|
275
|
+
// Waiting is only worth it while an operator could still say yes:
|
|
276
|
+
// `decide` refuses an expired challenge, so past it the task is re-planned.
|
|
277
|
+
if (this.policy.mode === "enforced" && decision?.channel !== "out_of_band") {
|
|
278
|
+
throw new Error(Date.parse(task.approval.expiresAt) <= Date.now() ? "approval_expired" : "approval_awaiting_operator");
|
|
279
|
+
}
|
|
280
|
+
if (decision !== undefined) {
|
|
281
|
+
if (decision.renderDigest !== render)
|
|
282
|
+
throw new Error("approval_render_mismatch");
|
|
283
|
+
if (!this.usable(task, decision))
|
|
284
|
+
throw new Error("approval_expired");
|
|
285
|
+
// Out of band is this kernel's claim to make, not the row's: a
|
|
286
|
+
// developer-mode kernel cannot vouch that nothing the agent touched
|
|
287
|
+
// wrote it, whatever mode the process that recorded it was in.
|
|
288
|
+
task.approvedBy = {
|
|
289
|
+
...decision,
|
|
290
|
+
channel: decision.channel === "out_of_band" && this.policy.mode === "enforced"
|
|
291
|
+
? "out_of_band"
|
|
292
|
+
: "in_band",
|
|
293
|
+
};
|
|
294
|
+
}
|
|
295
|
+
else {
|
|
296
|
+
const approvalResult = validateApproval(task.approval, approval);
|
|
297
|
+
if (!approvalResult.ok) {
|
|
298
|
+
throw new Error(approvalResult.reason);
|
|
299
|
+
}
|
|
300
|
+
// The phrase is derivable by anyone who knows the task, so an echo of
|
|
301
|
+
// it says only that something echoed it.
|
|
302
|
+
task.approvedBy = {
|
|
303
|
+
approvalId: task.approval.approvalId,
|
|
304
|
+
taskId: task.id,
|
|
305
|
+
renderDigest: render,
|
|
306
|
+
channel: "in_band",
|
|
307
|
+
approver: "phrase",
|
|
308
|
+
decidedAt: now(),
|
|
309
|
+
};
|
|
231
310
|
}
|
|
232
311
|
}
|
|
233
312
|
task.policyDecision = rechecked.decision;
|
|
234
313
|
const controller = new AbortController();
|
|
235
314
|
this.active.set(task.id, controller);
|
|
315
|
+
this.store.inFlight.add(task.id);
|
|
236
316
|
task.status = "running";
|
|
317
|
+
task.mode = this.policy.mode;
|
|
237
318
|
task.updatedAt = now();
|
|
238
|
-
|
|
319
|
+
// Status and owner land in one write, so no running row is ever unowned.
|
|
320
|
+
this.store.saveTask(task, this.store.ownerId());
|
|
239
321
|
const startedAt = now();
|
|
240
322
|
const classified = classifyOperation(request.operation);
|
|
241
323
|
let timeout;
|
|
242
324
|
let budgetExhausted = false;
|
|
243
325
|
let shortCircuited = false;
|
|
326
|
+
if (rechecked.grantId !== undefined) {
|
|
327
|
+
const held = this.reserved.get(rechecked.grantId) ?? new Map();
|
|
328
|
+
held.set(task.id, classified.spend?.amount ?? 0);
|
|
329
|
+
this.reserved.set(rechecked.grantId, held);
|
|
330
|
+
}
|
|
244
331
|
try {
|
|
245
332
|
const opened = this.breaker.check(classified.target);
|
|
246
333
|
if (opened !== undefined) {
|
|
@@ -295,7 +382,7 @@ export class TaskController {
|
|
|
295
382
|
rechecked.grantId !== undefined) {
|
|
296
383
|
this.store.recordCapabilityUse(rechecked.grantId, task.id, classified.spend?.amount ?? 0, now());
|
|
297
384
|
}
|
|
298
|
-
const receipt = this.createReceipt(task, request, classified.capability, classified.target, classified.effect, startedAt, actionSucceeded, result, evidence
|
|
385
|
+
const receipt = this.createReceipt(task, request, classified.capability, classified.target, classified.effect, startedAt, actionSucceeded, result, evidence);
|
|
299
386
|
this.store.saveReceipt(receipt);
|
|
300
387
|
task.receiptIds.push(receipt.receiptId);
|
|
301
388
|
if (task.status !== "cancelled") {
|
|
@@ -352,7 +439,7 @@ export class TaskController {
|
|
|
352
439
|
: message,
|
|
353
440
|
},
|
|
354
441
|
];
|
|
355
|
-
const receipt = this.createReceipt(task, request, classified.capability, classified.target, classified.effect, startedAt, false, {}, evidence,
|
|
442
|
+
const receipt = this.createReceipt(task, request, classified.capability, classified.target, classified.effect, startedAt, false, {}, evidence, task.error);
|
|
356
443
|
this.store.saveReceipt(receipt);
|
|
357
444
|
task.receiptIds.push(receipt.receiptId);
|
|
358
445
|
this.store.saveTask(task);
|
|
@@ -363,6 +450,9 @@ export class TaskController {
|
|
|
363
450
|
if (timeout !== undefined)
|
|
364
451
|
clearTimeout(timeout);
|
|
365
452
|
this.active.delete(task.id);
|
|
453
|
+
this.store.inFlight.delete(task.id);
|
|
454
|
+
if (rechecked.grantId !== undefined)
|
|
455
|
+
this.reserved.get(rechecked.grantId)?.delete(task.id);
|
|
366
456
|
}
|
|
367
457
|
}
|
|
368
458
|
cancel(taskId) {
|
|
@@ -436,7 +526,7 @@ export class TaskController {
|
|
|
436
526
|
task.error = "reconciliation_confirms_effect_not_applied";
|
|
437
527
|
task.updatedAt = now();
|
|
438
528
|
this.store.saveTask(task);
|
|
439
|
-
const receipt = this.createReceipt(task, request, classified.capability, classified.target, classified.effect, task.updatedAt, false, { reconciliation: "provider_state_denies_effect" }, verification.evidence,
|
|
529
|
+
const receipt = this.createReceipt(task, request, classified.capability, classified.target, classified.effect, task.updatedAt, false, { reconciliation: "provider_state_denies_effect" }, verification.evidence, task.error);
|
|
440
530
|
this.store.saveReceipt(receipt);
|
|
441
531
|
task.receiptIds.push(receipt.receiptId);
|
|
442
532
|
const certificate = this.createAndSaveCertificate(task, verification.evidence);
|
|
@@ -456,15 +546,18 @@ export class TaskController {
|
|
|
456
546
|
this.store.commitIdempotency(task.idempotencyKey, task.id, task.attempt ?? 1, now());
|
|
457
547
|
}
|
|
458
548
|
// Read for the grant id only. The work is already done, so this is
|
|
459
|
-
// accounting rather than permission
|
|
460
|
-
// pays for what it authorised.
|
|
461
|
-
|
|
549
|
+
// accounting rather than permission: a grant that has since run out still
|
|
550
|
+
// pays for what it authorised. Asked as if nothing were drawn, because
|
|
551
|
+
// policy names no grant at all for one that is now exhausted, and the
|
|
552
|
+
// effect then went uncharged.
|
|
553
|
+
const { grantId } = evaluatePolicy(task.id, request, this.policy, () => ({
|
|
554
|
+
operations: 0,
|
|
555
|
+
amountToday: 0,
|
|
556
|
+
}));
|
|
462
557
|
if (grantId !== undefined) {
|
|
463
558
|
this.store.recordCapabilityUse(grantId, task.id, classified.spend?.amount ?? 0, now());
|
|
464
559
|
}
|
|
465
|
-
const receipt = this.createReceipt(task, request, classified.capability, classified.target, classified.effect, task.updatedAt, true, observed, evidence
|
|
466
|
-
? undefined
|
|
467
|
-
: { approvalId: task.approval.approvalId, phrase: "recovered" });
|
|
560
|
+
const receipt = this.createReceipt(task, request, classified.capability, classified.target, classified.effect, task.updatedAt, true, observed, evidence);
|
|
468
561
|
this.store.saveReceipt(receipt);
|
|
469
562
|
task.receiptIds.push(receipt.receiptId);
|
|
470
563
|
task.status = "verified_success";
|
|
@@ -473,43 +566,156 @@ export class TaskController {
|
|
|
473
566
|
const certificate = this.createAndSaveCertificate(task, evidence);
|
|
474
567
|
return { task, receipt, certificate };
|
|
475
568
|
}
|
|
476
|
-
|
|
569
|
+
/**
|
|
570
|
+
* With `failed`, each task that could not be recovered is recorded there by
|
|
571
|
+
* id instead of the first failure being thrown at the end.
|
|
572
|
+
*/
|
|
573
|
+
async recoverInterrupted(failed) {
|
|
477
574
|
const recovered = [];
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
575
|
+
const failures = failed ?? new Map();
|
|
576
|
+
for (const listed of this.store.listInterruptedTasks()) {
|
|
577
|
+
// Only tasks whose owning process is gone, and only once: a live peer's
|
|
578
|
+
// work is left to finish. This process's own rows are taken too when
|
|
579
|
+
// nothing here has them in flight, so a task whose final write failed,
|
|
580
|
+
// or whose recovery threw after the claim, is retried by the next pass
|
|
581
|
+
// instead of waiting for a restart.
|
|
582
|
+
const task = this.store.claimInterruptedTask(listed.id);
|
|
583
|
+
if (task === undefined)
|
|
584
|
+
continue;
|
|
585
|
+
this.store.inFlight.add(task.id);
|
|
586
|
+
try {
|
|
587
|
+
recovered.push(await this.recoverTask(task));
|
|
588
|
+
}
|
|
589
|
+
catch (error) {
|
|
590
|
+
// One task that cannot be recovered must not keep the rest waiting;
|
|
591
|
+
// the failure is still reported once every task has had its turn.
|
|
592
|
+
failures.set(task.id, error);
|
|
593
|
+
// Given back rather than kept: held by this live process, no other
|
|
594
|
+
// could take it, so an advance elsewhere could never learn why its
|
|
595
|
+
// run is stuck. Whoever claims it next meets the same failure.
|
|
596
|
+
try {
|
|
597
|
+
this.store.releaseTaskClaim(task.id);
|
|
598
|
+
}
|
|
599
|
+
catch {
|
|
600
|
+
// Still ours, then; this process retakes it on its next pass.
|
|
492
601
|
}
|
|
493
602
|
}
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
task.error =
|
|
497
|
-
effect === "read"
|
|
498
|
-
? "interrupted_read_ready_for_retry"
|
|
499
|
-
: "interrupted_mutation_requires_reconciliation";
|
|
500
|
-
task.updatedAt = now();
|
|
501
|
-
this.store.saveTask(task);
|
|
502
|
-
// A crash mid-flight is the same unknown a dropped reply is, so a task
|
|
503
|
-
// that declared how to settle one gets asked here rather than waiting for
|
|
504
|
-
// someone to notice it. Still parked if the provider cannot answer.
|
|
505
|
-
if (task.status === "recovery_required" && request.reconciliation.length > 0) {
|
|
506
|
-
recovered.push((await this.reconcile(task.id)).task);
|
|
507
|
-
continue;
|
|
603
|
+
finally {
|
|
604
|
+
this.store.inFlight.delete(task.id);
|
|
508
605
|
}
|
|
509
|
-
recovered.push(task);
|
|
510
606
|
}
|
|
607
|
+
if (failed === undefined && failures.size > 0)
|
|
608
|
+
throw failures.values().next().value;
|
|
511
609
|
return recovered;
|
|
512
610
|
}
|
|
611
|
+
async recoverTask(task) {
|
|
612
|
+
const request = this.loadRequest(task.id);
|
|
613
|
+
const classified = classifyOperation(request.operation);
|
|
614
|
+
if (task.status === "verifying" &&
|
|
615
|
+
classified.effect !== "read" &&
|
|
616
|
+
request.requiredEvidence.length > 0 &&
|
|
617
|
+
request.requiredEvidence.every((predicate) => ["file_exists", "file_absent", "file_hash"].includes(predicate.type))) {
|
|
618
|
+
// What the adapter reported before the crash, when it got that far.
|
|
619
|
+
// Evidence alone is not success: a build that exited 1 beside a dist/
|
|
620
|
+
// left by an earlier one would otherwise recover as verified.
|
|
621
|
+
const sealed = this.store.getTaskResult(task.id);
|
|
622
|
+
const reported = sealed === undefined
|
|
623
|
+
? undefined
|
|
624
|
+
: this.payloadCipher.open(sealed, `task:${task.id}:result`);
|
|
625
|
+
const actionSucceeded = reported?.success !== false;
|
|
626
|
+
const verification = await this.verifier.verify(request.requiredEvidence, {});
|
|
627
|
+
if (actionSucceeded && verification.verified) {
|
|
628
|
+
this.resolveAsVerified(task, request, classified, verification.evidence, {
|
|
629
|
+
recovery: "independent_reobservation",
|
|
630
|
+
});
|
|
631
|
+
return task;
|
|
632
|
+
}
|
|
633
|
+
}
|
|
634
|
+
const effect = classified.effect;
|
|
635
|
+
task.status = effect === "read" ? "planned" : "recovery_required";
|
|
636
|
+
task.error =
|
|
637
|
+
effect === "read"
|
|
638
|
+
? "interrupted_read_ready_for_retry"
|
|
639
|
+
: "interrupted_mutation_requires_reconciliation";
|
|
640
|
+
task.updatedAt = now();
|
|
641
|
+
this.store.saveTask(task);
|
|
642
|
+
// A crash mid-flight is the same unknown a dropped reply is, so a task
|
|
643
|
+
// that declared how to settle one gets asked here rather than waiting for
|
|
644
|
+
// someone to notice it. Still parked if the provider cannot answer.
|
|
645
|
+
if (task.status === "recovery_required" && request.reconciliation.length > 0) {
|
|
646
|
+
return (await this.reconcile(task.id)).task;
|
|
647
|
+
}
|
|
648
|
+
return task;
|
|
649
|
+
}
|
|
650
|
+
/**
|
|
651
|
+
* What an approver is shown: the effect contract, minus the challenge and
|
|
652
|
+
* the policy verdict, which a recheck at execution rewrites. Its digest is
|
|
653
|
+
* what a decision is bound to, so a request that changed after it was
|
|
654
|
+
* shown cannot run on that decision.
|
|
655
|
+
*/
|
|
656
|
+
approvalRender(task, request) {
|
|
657
|
+
const { authorization: _authorization, policy: _policy, ...contract } = effectContract({ ...task, request }, classifyOperation(request.operation));
|
|
658
|
+
return { contract, renderDigest: sha256(contract) };
|
|
659
|
+
}
|
|
660
|
+
/** A pending challenge as an operator surface shows it. */
|
|
661
|
+
approvalView(taskId) {
|
|
662
|
+
const task = this.status(taskId);
|
|
663
|
+
if (task.status !== "awaiting_approval" || task.approval === undefined) {
|
|
664
|
+
throw new Error(`task_not_awaiting_approval:${task.status}`);
|
|
665
|
+
}
|
|
666
|
+
return { task, ...this.approvalRender(task, this.loadRequest(taskId)) };
|
|
667
|
+
}
|
|
668
|
+
/**
|
|
669
|
+
* Records an approver's yes to exactly the render they were shown. Never
|
|
670
|
+
* executes, and never touches the task row: the decision lives in its own
|
|
671
|
+
* insert-once table, and `execute` reads it. `out_of_band` only in enforced
|
|
672
|
+
* mode, where nothing the agent saw could have produced it.
|
|
673
|
+
*/
|
|
674
|
+
decide(taskId, renderDigest, approver) {
|
|
675
|
+
const { task, renderDigest: current } = this.approvalView(taskId);
|
|
676
|
+
// No phrase means an enforced kernel planned it, and would refuse this
|
|
677
|
+
// in-band decision while the insert-once row kept anyone from correcting it.
|
|
678
|
+
if (this.policy.mode !== "enforced" && task.approval.phrase === undefined) {
|
|
679
|
+
throw new Error("approval_needs_enforced_mode");
|
|
680
|
+
}
|
|
681
|
+
if (Date.parse(task.approval.expiresAt) <= Date.now())
|
|
682
|
+
throw new Error("approval_expired");
|
|
683
|
+
if (renderDigest !== current)
|
|
684
|
+
throw new Error("approval_render_mismatch");
|
|
685
|
+
const decision = {
|
|
686
|
+
approvalId: task.approval.approvalId,
|
|
687
|
+
taskId,
|
|
688
|
+
renderDigest,
|
|
689
|
+
channel: this.policy.mode === "enforced" ? "out_of_band" : "in_band",
|
|
690
|
+
approver,
|
|
691
|
+
decidedAt: now(),
|
|
692
|
+
};
|
|
693
|
+
if (!this.store.recordApprovalDecision(decision))
|
|
694
|
+
throw new Error("approval_already_decided");
|
|
695
|
+
return decision;
|
|
696
|
+
}
|
|
697
|
+
/** The recorded decision for a pending task, while it can still be used. */
|
|
698
|
+
approvalDecision(task) {
|
|
699
|
+
if (task.approval === undefined)
|
|
700
|
+
return undefined;
|
|
701
|
+
const decision = this.store.getApprovalDecision(task.approval.approvalId);
|
|
702
|
+
if (decision === undefined || !this.usable(task, decision))
|
|
703
|
+
return undefined;
|
|
704
|
+
// In enforced mode an in-band row is no decision at all, so a workflow node
|
|
705
|
+
// waits for an operator and is re-planned at expiry instead of failing on
|
|
706
|
+
// every advance.
|
|
707
|
+
return this.policy.mode === "enforced" && decision.channel !== "out_of_band" ? undefined : decision;
|
|
708
|
+
}
|
|
709
|
+
/**
|
|
710
|
+
* Made before the challenge expired, and used within a TTL of being made:
|
|
711
|
+
* an answer that came in time is not lost to the time it takes to act on.
|
|
712
|
+
*/
|
|
713
|
+
usable(task, decision) {
|
|
714
|
+
const decidedAt = Date.parse(decision.decidedAt);
|
|
715
|
+
return (task.approval !== undefined &&
|
|
716
|
+
decidedAt <= Date.parse(task.approval.expiresAt) &&
|
|
717
|
+
Date.now() < decidedAt + approvalTtl(this.policy));
|
|
718
|
+
}
|
|
513
719
|
receipts(input) {
|
|
514
720
|
if (input.receiptId !== undefined) {
|
|
515
721
|
const receipt = this.store.getReceipt(input.receiptId);
|
|
@@ -530,7 +736,23 @@ export class TaskController {
|
|
|
530
736
|
...(certificate === undefined ? {} : { certificate }),
|
|
531
737
|
};
|
|
532
738
|
}
|
|
533
|
-
|
|
739
|
+
/**
|
|
740
|
+
* The mode a task's effect ran under: recorded when it started running, or,
|
|
741
|
+
* for a task that started before that was recorded, the mode on its first
|
|
742
|
+
* receipt.
|
|
743
|
+
*/
|
|
744
|
+
runMode(task) {
|
|
745
|
+
if (task.mode !== undefined)
|
|
746
|
+
return task.mode;
|
|
747
|
+
const [first] = task.receiptIds;
|
|
748
|
+
return first === undefined ? undefined : this.store.getReceipt(first)?.mode;
|
|
749
|
+
}
|
|
750
|
+
createReceipt(task, request, capability, target, effect, startedAt, success, result, evidence, error) {
|
|
751
|
+
// From the task, not the caller: recovery and reconciliation write
|
|
752
|
+
// receipts with no caller in sight. A task running since before
|
|
753
|
+
// `approvedBy` existed still names the challenge it was planned under.
|
|
754
|
+
const approvalId = task.approvedBy?.approvalId ??
|
|
755
|
+
(task.policyDecision.outcome === "confirm" ? task.approval?.approvalId : undefined);
|
|
534
756
|
const sanitized = redactStructuredValue(result);
|
|
535
757
|
const adapterRedactions = Array.isArray(result.redactions)
|
|
536
758
|
? result.redactions.filter((item) => typeof item === "string")
|
|
@@ -546,13 +768,25 @@ export class TaskController {
|
|
|
546
768
|
principal: delegationChain(request.identity ?? LOCAL_IDENTITY),
|
|
547
769
|
effect,
|
|
548
770
|
executionGuarantee: executionGuaranteeFor(effect, request.reconciliation.length > 0),
|
|
549
|
-
mode: this.policy.mode,
|
|
771
|
+
mode: this.runMode(task) ?? this.policy.mode,
|
|
550
772
|
policyDecision: {
|
|
551
773
|
outcome: task.policyDecision.outcome,
|
|
552
774
|
policyVersion: task.policyDecision.policyVersion,
|
|
553
|
-
...(
|
|
775
|
+
...(approvalId === undefined ? {} : { approvalId }),
|
|
776
|
+
...(task.approvedBy === undefined
|
|
777
|
+
? {}
|
|
778
|
+
: {
|
|
779
|
+
approval: {
|
|
780
|
+
channel: task.approvedBy.channel,
|
|
781
|
+
renderDigest: task.approvedBy.renderDigest,
|
|
782
|
+
approver: task.approvedBy.approver,
|
|
783
|
+
decidedAt: task.approvedBy.decidedAt,
|
|
784
|
+
},
|
|
785
|
+
}),
|
|
554
786
|
},
|
|
555
|
-
target,
|
|
787
|
+
// The target is built from the request (a URL, a memory key) and stored
|
|
788
|
+
// as it is, so it takes the same secret shapes out as everything else.
|
|
789
|
+
target: redactSecrets(target).value,
|
|
556
790
|
inputDigest: sha256(request.operation),
|
|
557
791
|
startedAt,
|
|
558
792
|
endedAt: now(),
|
|
@@ -603,6 +837,16 @@ export class TaskController {
|
|
|
603
837
|
: new Error("execution_failed_without_error");
|
|
604
838
|
}
|
|
605
839
|
createAndSaveCertificate(task, evidence) {
|
|
840
|
+
const approvalChannel = task.approvedBy?.channel ??
|
|
841
|
+
(task.policyDecision.outcome === "allow"
|
|
842
|
+
? task.policyDecision.effect === "read"
|
|
843
|
+
? "not_required"
|
|
844
|
+
: "bypassed"
|
|
845
|
+
: undefined);
|
|
846
|
+
// Sealed, so never guessed: a task that ran says how it ran, one that
|
|
847
|
+
// never did was decided here, and one that ran before the mode was
|
|
848
|
+
// recorded and left no receipt saying so goes without.
|
|
849
|
+
const mode = this.runMode(task) ?? (task.receiptIds.length === 0 ? this.policy.mode : undefined);
|
|
606
850
|
const certificate = createCertificate({
|
|
607
851
|
taskId: task.id,
|
|
608
852
|
goal: task.request.goal,
|
|
@@ -611,6 +855,8 @@ export class TaskController {
|
|
|
611
855
|
receiptIds: task.receiptIds,
|
|
612
856
|
evidence,
|
|
613
857
|
createdAt: now(),
|
|
858
|
+
...(mode === undefined ? {} : { mode }),
|
|
859
|
+
...(approvalChannel === undefined ? {} : { approvalChannel }),
|
|
614
860
|
});
|
|
615
861
|
task.certificateId = certificate.certificateId;
|
|
616
862
|
task.updatedAt = now();
|