@melra/runtime-core 0.3.0-alpha.14 → 0.3.0-alpha.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -2
- package/dist/task-controller.d.ts +94 -4
- package/dist/task-controller.d.ts.map +1 -1
- package/dist/task-controller.js +384 -85
- package/dist/task-controller.js.map +1 -1
- package/dist/workflow-controller.d.ts +87 -0
- package/dist/workflow-controller.d.ts.map +1 -1
- package/dist/workflow-controller.js +1062 -258
- package/dist/workflow-controller.js.map +1 -1
- package/package.json +9 -6
package/dist/task-controller.js
CHANGED
|
@@ -15,9 +15,9 @@ function now() {
|
|
|
15
15
|
* what the provider was asked to do is the key MELRA itself sent. Shaped like
|
|
16
16
|
* an adapter result so one predicate type reads the same on both paths.
|
|
17
17
|
*/
|
|
18
|
-
function reconciliationFacts(
|
|
18
|
+
function reconciliationFacts(taskId, operation) {
|
|
19
19
|
return {
|
|
20
|
-
taskId
|
|
20
|
+
taskId,
|
|
21
21
|
...(operation.kind === "http"
|
|
22
22
|
? {
|
|
23
23
|
url: operation.url,
|
|
@@ -59,9 +59,10 @@ function certificateResult(task) {
|
|
|
59
59
|
* Applied once, where the request is normalized, so the derived predicates flow
|
|
60
60
|
* identically into the policy decision, the sealed payload, the persisted task
|
|
61
61
|
* record, and verification. A caller that declared its own evidence is left
|
|
62
|
-
* untouched — this only replaces an empty list.
|
|
62
|
+
* untouched — this only replaces an empty list. Exported for `melra policy
|
|
63
|
+
* test`, whose preview must decide on the request the server would.
|
|
63
64
|
*/
|
|
64
|
-
function withDefaultEvidence(request) {
|
|
65
|
+
export function withDefaultEvidence(request) {
|
|
65
66
|
if (request.requiredEvidence.length > 0)
|
|
66
67
|
return request;
|
|
67
68
|
const derived = defaultEvidenceFor(request.operation);
|
|
@@ -83,6 +84,12 @@ export function capabilityUsageReader(store) {
|
|
|
83
84
|
return { operations: drawn.operations, amountToday: drawn.amountInWindow };
|
|
84
85
|
};
|
|
85
86
|
}
|
|
87
|
+
/** Statuses a task leaves only once: by starting, by a deny, or by a refusal. */
|
|
88
|
+
const UNSTARTED = ["planned", "awaiting_approval"];
|
|
89
|
+
/** Whether a task waits on a question only someone outside MELRA answers (#54). */
|
|
90
|
+
function asksSemantic(request) {
|
|
91
|
+
return request.requiredEvidence.some((predicate) => predicate.type === "semantic");
|
|
92
|
+
}
|
|
86
93
|
/** The one browser action that reads each list. */
|
|
87
94
|
const BROWSER_LIST_ACTIONS = [
|
|
88
95
|
["values", "select"],
|
|
@@ -96,6 +103,8 @@ export class TaskController {
|
|
|
96
103
|
verifier;
|
|
97
104
|
payloadCipher;
|
|
98
105
|
active = new Map();
|
|
106
|
+
/** Each task running here, settled once it has ended (see `settled`). */
|
|
107
|
+
running = new Map();
|
|
99
108
|
breaker;
|
|
100
109
|
/** Whether approvals come only from an operator, out of band. */
|
|
101
110
|
get enforced() {
|
|
@@ -132,10 +141,12 @@ export class TaskController {
|
|
|
132
141
|
};
|
|
133
142
|
/**
|
|
134
143
|
* Draws by tasks between their re-check and their record, per grant and
|
|
135
|
-
* task. The durable draw is written only once
|
|
136
|
-
* this two tasks running at once both saw the grant unspent: two
|
|
137
|
-
* against `maxOperations: 1`, a parallel workflow node
|
|
138
|
-
* `dailyMax`.
|
|
144
|
+
* task. The durable draw is written only once the adapter has answered, so
|
|
145
|
+
* without this two tasks running at once both saw the grant unspent: two
|
|
146
|
+
* operations against `maxOperations: 1`, a parallel workflow node
|
|
147
|
+
* overspending `dailyMax`. Let go once the draw is written or the effect is
|
|
148
|
+
* known not to have run; one whose draw failed to write is held until
|
|
149
|
+
* recovery writes it (`drawGrant`).
|
|
139
150
|
*
|
|
140
151
|
* ponytail: held in this process. Two servers sharing one database can
|
|
141
152
|
* still both pass a re-check at the same moment; a durable reservation row
|
|
@@ -206,13 +217,39 @@ export class TaskController {
|
|
|
206
217
|
}
|
|
207
218
|
}
|
|
208
219
|
// Refused before anything is approved: a mutation run against evidence no
|
|
209
|
-
// registered verifier can check would end `partial` after the effect
|
|
210
|
-
|
|
220
|
+
// registered verifier can check would end `partial` after the effect, and
|
|
221
|
+
// one that could not be reconciled would stay parked for good. A stored
|
|
222
|
+
// step's reconciliation was never checked, and refusing it now would stop
|
|
223
|
+
// a run whose earlier steps already ran.
|
|
224
|
+
this.assertVerifiable(options.stored === true
|
|
225
|
+
? parsed.requiredEvidence
|
|
226
|
+
: [...parsed.requiredEvidence, ...parsed.reconciliation], options);
|
|
227
|
+
// Read only against the facts known before the call, so a URL token they
|
|
228
|
+
// cannot fill (`{{json.id}}`: no response is read on this path) is
|
|
229
|
+
// refused every time it is asked, and the task would stay parked.
|
|
230
|
+
if (options.stored !== true) {
|
|
231
|
+
const facts = reconciliationFacts("00000000-0000-4000-8000-000000000000", parsed.operation);
|
|
232
|
+
for (const predicate of parsed.reconciliation) {
|
|
233
|
+
const refused = this.verifier.unfilledToken(predicate, facts);
|
|
234
|
+
if (refused !== undefined)
|
|
235
|
+
throw new Error(`reconciliation_unanswerable:${refused}`);
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
return evaluatePolicy("00000000-0000-4000-8000-000000000000", parsed, this.policy, this.capabilityUsage).decision;
|
|
239
|
+
}
|
|
240
|
+
/** Refuses a predicate no registered verifier can check, or that it would refuse to. */
|
|
241
|
+
assertVerifiable(predicates, options = {}) {
|
|
242
|
+
for (const predicate of predicates) {
|
|
211
243
|
if (!this.verifier.supports(predicate)) {
|
|
212
244
|
throw new Error(`evidence_verifier_unavailable:${predicate.type === "custom" ? predicate.verifier : predicate.type}`);
|
|
213
245
|
}
|
|
246
|
+
// A file path the verifier refuses however often it is asked. New work
|
|
247
|
+
// only, like the reconciliation check: a run accepted before this
|
|
248
|
+
// check existed is not stopped at the step.
|
|
249
|
+
if (options.stored !== true && this.verifier.outsideWorkspace(predicate)) {
|
|
250
|
+
throw new Error("evidence_path_outside_workspace");
|
|
251
|
+
}
|
|
214
252
|
}
|
|
215
|
-
return evaluatePolicy("00000000-0000-4000-8000-000000000000", parsed, this.policy, this.capabilityUsage).decision;
|
|
216
253
|
}
|
|
217
254
|
status(taskId) {
|
|
218
255
|
const task = this.store.getTask(taskId);
|
|
@@ -231,7 +268,7 @@ export class TaskController {
|
|
|
231
268
|
if (task.status === "recovery_required") {
|
|
232
269
|
return await this.reconcile(taskId);
|
|
233
270
|
}
|
|
234
|
-
if (!
|
|
271
|
+
if (!UNSTARTED.includes(task.status)) {
|
|
235
272
|
throw new Error(`task_not_executable:${task.status}`);
|
|
236
273
|
}
|
|
237
274
|
if (task.idempotencyKey !== undefined &&
|
|
@@ -239,7 +276,7 @@ export class TaskController {
|
|
|
239
276
|
task.status = "cancelled";
|
|
240
277
|
task.error = "duplicate_attempt_prevented";
|
|
241
278
|
task.updatedAt = now();
|
|
242
|
-
this.
|
|
279
|
+
this.leaveUnstarted(task);
|
|
243
280
|
return this.finishWithoutReceipt(task, [
|
|
244
281
|
{
|
|
245
282
|
type: "idempotency",
|
|
@@ -255,7 +292,7 @@ export class TaskController {
|
|
|
255
292
|
task.status = "policy_blocked";
|
|
256
293
|
task.policyDecision = rechecked.decision;
|
|
257
294
|
task.updatedAt = now();
|
|
258
|
-
this.
|
|
295
|
+
this.leaveUnstarted(task);
|
|
259
296
|
return this.finishWithoutReceipt(task, []);
|
|
260
297
|
}
|
|
261
298
|
if (rechecked.decision.outcome === "confirm") {
|
|
@@ -310,24 +347,44 @@ export class TaskController {
|
|
|
310
347
|
}
|
|
311
348
|
}
|
|
312
349
|
task.policyDecision = rechecked.decision;
|
|
313
|
-
const controller = new AbortController();
|
|
314
|
-
this.active.set(task.id, controller);
|
|
315
|
-
this.store.inFlight.add(task.id);
|
|
316
350
|
task.status = "running";
|
|
317
351
|
task.mode = this.policy.mode;
|
|
318
352
|
task.updatedAt = now();
|
|
319
353
|
// Status and owner land in one write, so no running row is ever unowned.
|
|
320
|
-
this.
|
|
354
|
+
this.leaveUnstarted(task, this.store.ownerId());
|
|
355
|
+
const controller = new AbortController();
|
|
356
|
+
let ended;
|
|
357
|
+
this.running.set(task.id, new Promise((resolve) => (ended = resolve)));
|
|
358
|
+
this.active.set(task.id, controller);
|
|
359
|
+
this.store.inFlight.add(task.id);
|
|
321
360
|
const startedAt = now();
|
|
322
361
|
const classified = classifyOperation(request.operation);
|
|
323
362
|
let timeout;
|
|
324
363
|
let budgetExhausted = false;
|
|
325
364
|
let shortCircuited = false;
|
|
365
|
+
let effectReturned = false;
|
|
366
|
+
let settled;
|
|
326
367
|
if (rechecked.grantId !== undefined) {
|
|
327
368
|
const held = this.reserved.get(rechecked.grantId) ?? new Map();
|
|
328
369
|
held.set(task.id, classified.spend?.amount ?? 0);
|
|
329
370
|
this.reserved.set(rechecked.grantId, held);
|
|
330
371
|
}
|
|
372
|
+
// Charged whenever the adapter may have done the work, whatever the
|
|
373
|
+
// evidence says of it: `verified_success` is the evidence verdict only.
|
|
374
|
+
// Read off what the adapter gave back. It returned, whatever its `success`
|
|
375
|
+
// (a request answered 500 reached the far end, a command that exited 1
|
|
376
|
+
// ran), or it threw an outcome-unknown error (`OUTCOME_UNKNOWN_PREFIX`).
|
|
377
|
+
// Any other throw is an adapter saying nothing went out, and costs
|
|
378
|
+
// nothing; so does a refusal before it was called. Counting an effect
|
|
379
|
+
// that did not happen is preferred to a grant funding more than it says.
|
|
380
|
+
const charge = (ran) => {
|
|
381
|
+
if (rechecked.grantId === undefined)
|
|
382
|
+
return;
|
|
383
|
+
if (ran) {
|
|
384
|
+
this.store.recordCapabilityUse(rechecked.grantId, task.id, classified.spend?.amount ?? 0, now());
|
|
385
|
+
}
|
|
386
|
+
this.reserved.get(rechecked.grantId)?.delete(task.id);
|
|
387
|
+
};
|
|
331
388
|
try {
|
|
332
389
|
const opened = this.breaker.check(classified.target);
|
|
333
390
|
if (opened !== undefined) {
|
|
@@ -342,6 +399,7 @@ export class TaskController {
|
|
|
342
399
|
}, request.budget.maxDurationMs);
|
|
343
400
|
timeout.unref();
|
|
344
401
|
const result = await this.executeWithRetries(task, request, classified.effect, controller.signal);
|
|
402
|
+
effectReturned = true;
|
|
345
403
|
const sanitizedResult = redactStructuredValue(result);
|
|
346
404
|
task.status = "verifying";
|
|
347
405
|
task.result = sanitizedResult.value;
|
|
@@ -367,52 +425,90 @@ export class TaskController {
|
|
|
367
425
|
: verification.evidence;
|
|
368
426
|
const verified = actionSucceeded &&
|
|
369
427
|
(request.requiredEvidence.length === 0 || verification.verified);
|
|
370
|
-
|
|
428
|
+
// Every check the kernel can make held; what is left is a question only
|
|
429
|
+
// someone outside answers (#54).
|
|
430
|
+
const awaitingJudgement = actionSucceeded &&
|
|
431
|
+
evidence.some((item) => item.type === "semantic") &&
|
|
432
|
+
evidence.every((item) => item.passed || item.type === "semantic");
|
|
433
|
+
// The effect ran and passed every kernel check, so it is committed now:
|
|
434
|
+
// a key that ran must not run again, even while it waits for its verdict.
|
|
435
|
+
const checksHeld = verified || awaitingJudgement;
|
|
436
|
+
if (checksHeld &&
|
|
371
437
|
task.idempotencyKey !== undefined &&
|
|
372
438
|
!this.store.commitIdempotency(task.idempotencyKey, task.id, task.attempt ?? 1, now())) {
|
|
373
439
|
task.status = "cancelled";
|
|
374
440
|
task.error = "duplicate_attempt_prevented";
|
|
375
441
|
}
|
|
376
|
-
//
|
|
377
|
-
//
|
|
378
|
-
|
|
379
|
-
// none of them reach this line, and none of them cost the caller a draw.
|
|
380
|
-
if (verified &&
|
|
381
|
-
task.status !== "cancelled" &&
|
|
382
|
-
rechecked.grantId !== undefined) {
|
|
383
|
-
this.store.recordCapabilityUse(rechecked.grantId, task.id, classified.spend?.amount ?? 0, now());
|
|
384
|
-
}
|
|
442
|
+
// The adapter returned, so the effect ran: drawn down for a `partial`,
|
|
443
|
+
// a `failed` or a duplicate collapsed just above as for a verified task.
|
|
444
|
+
charge(true);
|
|
385
445
|
const receipt = this.createReceipt(task, request, classified.capability, classified.target, classified.effect, startedAt, actionSucceeded, result, evidence);
|
|
386
446
|
this.store.saveReceipt(receipt);
|
|
387
447
|
task.receiptIds.push(receipt.receiptId);
|
|
388
448
|
if (task.status !== "cancelled") {
|
|
389
449
|
task.status = verified
|
|
390
450
|
? "verified_success"
|
|
391
|
-
:
|
|
392
|
-
? "
|
|
393
|
-
:
|
|
451
|
+
: awaitingJudgement
|
|
452
|
+
? "waiting_user"
|
|
453
|
+
: actionSucceeded
|
|
454
|
+
? "partial"
|
|
455
|
+
: "failed";
|
|
456
|
+
}
|
|
457
|
+
// Written in the same save as the status, so a waiting row always says
|
|
458
|
+
// what it waits on, and until when.
|
|
459
|
+
if (task.status === "waiting_user") {
|
|
460
|
+
const expiresAt = new Date(Date.now() + approvalTtl(this.policy)).toISOString();
|
|
461
|
+
task.judgements = request.requiredEvidence.flatMap((predicate, predicateIndex) => predicate.type === "semantic" ? [{ judgementId: randomUUID(), predicateIndex, expiresAt }] : []);
|
|
394
462
|
}
|
|
395
463
|
task.updatedAt = now();
|
|
464
|
+
// A wait lands with its certificate in one transaction: written apart, a
|
|
465
|
+
// peer's expiry between the two was overwritten by the second. Either
|
|
466
|
+
// failing writes neither, so `settled` stays unset for the catch.
|
|
467
|
+
if (task.status === "waiting_user") {
|
|
468
|
+
const certificate = this.store.transaction(() => {
|
|
469
|
+
this.store.saveTask(task);
|
|
470
|
+
return this.createAndSaveCertificate(task, evidence, true);
|
|
471
|
+
});
|
|
472
|
+
return { task, output: result, receipt, certificate };
|
|
473
|
+
}
|
|
396
474
|
this.store.saveTask(task);
|
|
397
|
-
|
|
398
|
-
|
|
475
|
+
settled = { task, output: result, receipt };
|
|
476
|
+
const certificate = this.createAndSaveCertificate(task, evidence, true);
|
|
477
|
+
return { ...settled, certificate };
|
|
399
478
|
}
|
|
400
479
|
catch (error) {
|
|
480
|
+
// The outcome is recorded, beside its receipt and idempotency commit,
|
|
481
|
+
// and only the certificate summarising it failed to write. Rewriting it
|
|
482
|
+
// would contradict them, and an error would read as the effect not
|
|
483
|
+
// having happened, so the caller gets what was recorded. A workflow
|
|
484
|
+
// step's certificate is written by the run's next advance (`certify`).
|
|
485
|
+
if (settled !== undefined)
|
|
486
|
+
return settled;
|
|
401
487
|
const rawMessage = error instanceof Error ? error.message : String(error);
|
|
402
|
-
|
|
488
|
+
// Capped after redaction, which can lengthen it, at the 10,000 characters
|
|
489
|
+
// a workflow node's `error` holds: a longer one failed the node's write
|
|
490
|
+
// after the effect ran, and the next advance ran the step again.
|
|
491
|
+
const message = redactStructuredValue(rawMessage).value.slice(0, 10_000);
|
|
403
492
|
const aborted = controller.signal.aborted;
|
|
404
493
|
// A refusal by the breaker is not fresh evidence about the target, and an
|
|
405
494
|
// operator's cancellation is not evidence about it at all. A budget that
|
|
406
|
-
// ran out is: the target did not answer in time.
|
|
407
|
-
|
|
495
|
+
// ran out is: the target did not answer in time. Nor is anything after
|
|
496
|
+
// the adapter returned, which the target already answered.
|
|
497
|
+
if (!shortCircuited && !effectReturned && (!aborted || budgetExhausted)) {
|
|
408
498
|
this.breaker.recordFailure(classified.target);
|
|
409
499
|
}
|
|
410
|
-
//
|
|
411
|
-
//
|
|
412
|
-
//
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
500
|
+
// Paid for before the outcome is written, read or change alike. A draw
|
|
501
|
+
// that fails to write leaves the reservation held, and the task running
|
|
502
|
+
// for recovery to charge.
|
|
503
|
+
const ran = effectReturned || isOutcomeUnknown(rawMessage);
|
|
504
|
+
charge(ran);
|
|
505
|
+
// Ahead of every other outcome, because it is the strongest thing known:
|
|
506
|
+
// the effect may already have happened, either because the adapter says
|
|
507
|
+
// so or because it returned and a write after it failed. `failed`,
|
|
508
|
+
// `cancelled` and `budget_exhausted` all assert more than that. Reads
|
|
509
|
+
// are exempt — a read that may or may not have run changed nothing
|
|
510
|
+
// either way, so there is nothing to reconcile.
|
|
511
|
+
const unknownOutcome = classified.effect !== "read" && ran;
|
|
416
512
|
task.status = unknownOutcome
|
|
417
513
|
? "recovery_required"
|
|
418
514
|
: aborted
|
|
@@ -443,7 +539,7 @@ export class TaskController {
|
|
|
443
539
|
this.store.saveReceipt(receipt);
|
|
444
540
|
task.receiptIds.push(receipt.receiptId);
|
|
445
541
|
this.store.saveTask(task);
|
|
446
|
-
const certificate = this.createAndSaveCertificate(task, evidence);
|
|
542
|
+
const certificate = this.createAndSaveCertificate(task, evidence, true);
|
|
447
543
|
return { task, receipt, certificate };
|
|
448
544
|
}
|
|
449
545
|
finally {
|
|
@@ -451,10 +547,18 @@ export class TaskController {
|
|
|
451
547
|
clearTimeout(timeout);
|
|
452
548
|
this.active.delete(task.id);
|
|
453
549
|
this.store.inFlight.delete(task.id);
|
|
454
|
-
|
|
455
|
-
|
|
550
|
+
this.running.delete(task.id);
|
|
551
|
+
ended();
|
|
456
552
|
}
|
|
457
553
|
}
|
|
554
|
+
/**
|
|
555
|
+
* Resolves once a task this controller is running has ended, its outcome
|
|
556
|
+
* recorded: a cancel only signals the adapter to stop. At once for one it
|
|
557
|
+
* is not running.
|
|
558
|
+
*/
|
|
559
|
+
settled(taskId) {
|
|
560
|
+
return this.running.get(taskId) ?? Promise.resolve();
|
|
561
|
+
}
|
|
458
562
|
cancel(taskId) {
|
|
459
563
|
const task = this.status(taskId);
|
|
460
564
|
const controller = this.active.get(taskId);
|
|
@@ -462,13 +566,26 @@ export class TaskController {
|
|
|
462
566
|
controller.abort(new Error("task_cancelled"));
|
|
463
567
|
return task;
|
|
464
568
|
}
|
|
465
|
-
if (
|
|
569
|
+
if (UNSTARTED.includes(task.status)) {
|
|
466
570
|
task.status = "cancelled";
|
|
467
571
|
task.updatedAt = now();
|
|
468
|
-
|
|
572
|
+
// Started meanwhile by another process: it runs, so the caller is told
|
|
573
|
+
// what the task is now rather than that it was cancelled.
|
|
574
|
+
if (!this.store.saveTaskIf(task, UNSTARTED))
|
|
575
|
+
return this.status(taskId);
|
|
469
576
|
}
|
|
470
577
|
return task;
|
|
471
578
|
}
|
|
579
|
+
/**
|
|
580
|
+
* Writes a task out of planned or awaiting_approval, unless a deny or an
|
|
581
|
+
* execute in another process got there after this one read it: that one
|
|
582
|
+
* stands, and this one is refused as it would have been had it read later.
|
|
583
|
+
*/
|
|
584
|
+
leaveUnstarted(task, owner) {
|
|
585
|
+
if (!this.store.saveTaskIf(task, UNSTARTED, owner)) {
|
|
586
|
+
throw new Error(`task_not_executable:${this.status(task.id).status}`);
|
|
587
|
+
}
|
|
588
|
+
}
|
|
472
589
|
async verifyPersisted(taskId, predicates) {
|
|
473
590
|
const sealed = this.store.getTaskResult(taskId);
|
|
474
591
|
const result = sealed === undefined
|
|
@@ -502,35 +619,67 @@ export class TaskController {
|
|
|
502
619
|
}
|
|
503
620
|
const request = this.loadRequest(taskId);
|
|
504
621
|
const classified = classifyOperation(request.operation);
|
|
505
|
-
const park = (reason) => {
|
|
622
|
+
const park = (reason) => this.settleParked(taskId, () => {
|
|
506
623
|
task.error = reason;
|
|
507
624
|
task.updatedAt = now();
|
|
508
625
|
this.store.saveTask(task);
|
|
509
626
|
return { task };
|
|
510
|
-
};
|
|
627
|
+
});
|
|
628
|
+
// Its own commit already answers: every kernel check held, and only a
|
|
629
|
+
// write after it failed. Parked for good otherwise, with nothing declared.
|
|
630
|
+
const committed = this.settleParked(taskId, () => this.settleCommitted(task, request, classified));
|
|
631
|
+
if (committed !== undefined)
|
|
632
|
+
return committed;
|
|
633
|
+
// A provider can say whether an effect happened, never whether it is
|
|
634
|
+
// right: a task that declared a semantic predicate stays parked with its
|
|
635
|
+
// cause unchanged (#54).
|
|
636
|
+
if (asksSemantic(request))
|
|
637
|
+
return { task };
|
|
511
638
|
if (request.reconciliation.length === 0) {
|
|
512
639
|
return park("reconciliation_not_declared");
|
|
513
640
|
}
|
|
514
|
-
const verification = await this.verifier.verify(request.reconciliation, reconciliationFacts(task, request.operation));
|
|
641
|
+
const verification = await this.verifier.verify(request.reconciliation, reconciliationFacts(task.id, request.operation));
|
|
642
|
+
// Refused by the kernel itself, it is refused again however often it is
|
|
643
|
+
// asked: the declaration needs fixing, which waiting on the provider will
|
|
644
|
+
// not do. Named, so a caller can tell the two apart.
|
|
645
|
+
const unanswerable = verification.evidence.filter((item) => item.unanswerable === true);
|
|
646
|
+
if (unanswerable.length > 0) {
|
|
647
|
+
const reason = `reconciliation_unanswerable:${unanswerable.map((item) => item.summary).join("; ")}`;
|
|
648
|
+
return park(redactStructuredValue(reason).value.slice(0, 10_000));
|
|
649
|
+
}
|
|
515
650
|
// An unreachable provider did not say no. Resolving on this would be the
|
|
516
651
|
// same lie as the `failed` this whole path exists to avoid.
|
|
517
652
|
if (verification.evidence.some((item) => item.inconclusive === true)) {
|
|
518
653
|
return park("reconciliation_inconclusive");
|
|
519
654
|
}
|
|
520
655
|
if (verification.verified) {
|
|
521
|
-
return this.resolveAsVerified(task, request, classified, verification.evidence, { reconciliation: "provider_state_confirms_effect" });
|
|
656
|
+
return this.settleParked(taskId, () => this.resolveAsVerified(task, request, classified, verification.evidence, { reconciliation: "provider_state_confirms_effect" }));
|
|
522
657
|
}
|
|
523
658
|
// The provider answered, and the answer is that it never happened. Only now
|
|
524
659
|
// is `failed` a claim MELRA is entitled to make.
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
660
|
+
return this.settleParked(taskId, () => {
|
|
661
|
+
task.status = "failed";
|
|
662
|
+
task.error = "reconciliation_confirms_effect_not_applied";
|
|
663
|
+
task.updatedAt = now();
|
|
664
|
+
this.store.saveTask(task);
|
|
665
|
+
const receipt = this.createReceipt(task, request, classified.capability, classified.target, classified.effect, task.updatedAt, false, { reconciliation: "provider_state_denies_effect" }, verification.evidence, task.error);
|
|
666
|
+
this.store.saveReceipt(receipt);
|
|
667
|
+
task.receiptIds.push(receipt.receiptId);
|
|
668
|
+
const certificate = this.createAndSaveCertificate(task, verification.evidence);
|
|
669
|
+
return { task, receipt, certificate };
|
|
670
|
+
});
|
|
671
|
+
}
|
|
672
|
+
/**
|
|
673
|
+
* Writes a reconcile's answer in one transaction, and only while the task
|
|
674
|
+
* still waits for one. Two reconciles of one task (two melra_execute calls,
|
|
675
|
+
* or the sweep's recovery beside an agent's) each read it parked and ask
|
|
676
|
+
* the provider; written unconditionally, the slower answer, even one that
|
|
677
|
+
* could not reach the provider, overwrote a settle the other had recorded
|
|
678
|
+
* and dropped its receipt. Now the first stands, and the other writes
|
|
679
|
+
* nothing, not even a receipt, and returns the task as it now stands.
|
|
680
|
+
*/
|
|
681
|
+
settleParked(taskId, write) {
|
|
682
|
+
return this.store.transaction(() => this.store.getTask(taskId)?.status === "recovery_required" ? write() : { task: this.status(taskId) });
|
|
534
683
|
}
|
|
535
684
|
/**
|
|
536
685
|
* Settle a task as done on evidence gathered after the fact.
|
|
@@ -545,27 +694,78 @@ export class TaskController {
|
|
|
545
694
|
this.store.getIdempotencyCommit(task.idempotencyKey) === undefined) {
|
|
546
695
|
this.store.commitIdempotency(task.idempotencyKey, task.id, task.attempt ?? 1, now());
|
|
547
696
|
}
|
|
548
|
-
|
|
549
|
-
// accounting rather than permission: a grant that has since run out still
|
|
550
|
-
// pays for what it authorised. Asked as if nothing were drawn, because
|
|
551
|
-
// policy names no grant at all for one that is now exhausted, and the
|
|
552
|
-
// effect then went uncharged.
|
|
553
|
-
const { grantId } = evaluatePolicy(task.id, request, this.policy, () => ({
|
|
554
|
-
operations: 0,
|
|
555
|
-
amountToday: 0,
|
|
556
|
-
}));
|
|
557
|
-
if (grantId !== undefined) {
|
|
558
|
-
this.store.recordCapabilityUse(grantId, task.id, classified.spend?.amount ?? 0, now());
|
|
559
|
-
}
|
|
697
|
+
this.drawGrant(task, request, classified);
|
|
560
698
|
const receipt = this.createReceipt(task, request, classified.capability, classified.target, classified.effect, task.updatedAt, true, observed, evidence);
|
|
561
699
|
this.store.saveReceipt(receipt);
|
|
562
700
|
task.receiptIds.push(receipt.receiptId);
|
|
701
|
+
// Evidence gathered after the fact answers no question about meaning: a
|
|
702
|
+
// semantic task, its effect done and receipted, settles closed as at
|
|
703
|
+
// expiry (#54).
|
|
704
|
+
if (asksSemantic(request))
|
|
705
|
+
return { ...this.settleUnanswered(task), receipt };
|
|
563
706
|
task.status = "verified_success";
|
|
564
707
|
delete task.error;
|
|
565
708
|
task.updatedAt = now();
|
|
566
709
|
const certificate = this.createAndSaveCertificate(task, evidence);
|
|
567
710
|
return { task, receipt, certificate };
|
|
568
711
|
}
|
|
712
|
+
/**
|
|
713
|
+
* Draws the grant behind an effect already done, once per task however
|
|
714
|
+
* often asked. Policy is read for the grant id only: this is accounting
|
|
715
|
+
* rather than permission, so a grant that has since run out still pays for
|
|
716
|
+
* what it authorised. Asked as if nothing were drawn, because policy names
|
|
717
|
+
* no grant at all for one that is now exhausted, and the effect then went
|
|
718
|
+
* uncharged. A task that drew one already is left alone: policy read now
|
|
719
|
+
* names the first grant that matches, not the one execute drew once that
|
|
720
|
+
* was spent, and charging it too drew the task twice. Drawn, it lets go
|
|
721
|
+
* of a reservation `execute` held when its own draw failed to write.
|
|
722
|
+
*/
|
|
723
|
+
drawGrant(task, request, classified) {
|
|
724
|
+
const grantId = this.store.hasCapabilityUse(task.id)
|
|
725
|
+
? undefined
|
|
726
|
+
: evaluatePolicy(task.id, request, this.policy, () => ({
|
|
727
|
+
operations: 0,
|
|
728
|
+
amountToday: 0,
|
|
729
|
+
})).grantId;
|
|
730
|
+
if (grantId !== undefined) {
|
|
731
|
+
this.store.recordCapabilityUse(grantId, task.id, classified.spend?.amount ?? 0, now());
|
|
732
|
+
}
|
|
733
|
+
for (const held of this.reserved.values())
|
|
734
|
+
held.delete(task.id);
|
|
735
|
+
}
|
|
736
|
+
/**
|
|
737
|
+
* Settle a task whose idempotency key is committed to it. A key is
|
|
738
|
+
* committed only once every kernel check held, so whatever stopped the
|
|
739
|
+
* task came after the effect: an interruption, or a write that failed. One
|
|
740
|
+
* with a semantic question commits on entering `waiting_user`, so its
|
|
741
|
+
* commit never says the question was answered: it settles closed, as at
|
|
742
|
+
* expiry (#54).
|
|
743
|
+
*/
|
|
744
|
+
settleCommitted(task, request, classified) {
|
|
745
|
+
if (task.idempotencyKey === undefined ||
|
|
746
|
+
this.store.getIdempotencyCommit(task.idempotencyKey)?.taskId !== task.id) {
|
|
747
|
+
return undefined;
|
|
748
|
+
}
|
|
749
|
+
if (asksSemantic(request)) {
|
|
750
|
+
// Settled already, it is left as it stands rather than certified again.
|
|
751
|
+
// Asked of the stored row: a peer may have settled it since `task` was
|
|
752
|
+
// read, and its status says `recovery_required` either way.
|
|
753
|
+
const stored = this.status(task.id);
|
|
754
|
+
if (stored.error === "semantic_judgement_expired")
|
|
755
|
+
return { task: stored };
|
|
756
|
+
// Cut off before it reached `waiting_user`, maybe before its draw too.
|
|
757
|
+
this.drawGrant(task, request, classified);
|
|
758
|
+
return this.settleUnanswered(task);
|
|
759
|
+
}
|
|
760
|
+
return this.resolveAsVerified(task, request, classified, [
|
|
761
|
+
{
|
|
762
|
+
type: "idempotency",
|
|
763
|
+
passed: true,
|
|
764
|
+
strength: evidenceStrength("idempotency"),
|
|
765
|
+
summary: "verified and committed before the interruption",
|
|
766
|
+
},
|
|
767
|
+
], { recovery: "committed_before_interruption" });
|
|
768
|
+
}
|
|
569
769
|
/**
|
|
570
770
|
* With `failed`, each task that could not be recovered is recorded there by
|
|
571
771
|
* id instead of the first failure being thrown at the end.
|
|
@@ -604,17 +804,64 @@ export class TaskController {
|
|
|
604
804
|
this.store.inFlight.delete(task.id);
|
|
605
805
|
}
|
|
606
806
|
}
|
|
807
|
+
this.expireJudgements(failures);
|
|
607
808
|
if (failed === undefined && failures.size > 0)
|
|
608
809
|
throw failures.values().next().value;
|
|
609
810
|
return recovered;
|
|
610
811
|
}
|
|
812
|
+
/**
|
|
813
|
+
* A waiting task whose judgements are past their `expiresAt` settles
|
|
814
|
+
* closed. Re-read and written in one transaction, so a peer settling the
|
|
815
|
+
* same task wins once.
|
|
816
|
+
*
|
|
817
|
+
* ponytail: runs on the startup pass and the 30 s recovery sweep, so a wait
|
|
818
|
+
* can outlive its `expiresAt` by one interval, or until a runtime next opens
|
|
819
|
+
* this data directory; a verdict tool must check `expiresAt` itself.
|
|
820
|
+
*/
|
|
821
|
+
expireJudgements(failures) {
|
|
822
|
+
for (const listed of this.store.listExpiredJudgements(now())) {
|
|
823
|
+
try {
|
|
824
|
+
this.store.transaction(() => {
|
|
825
|
+
const task = this.store.getTask(listed.id);
|
|
826
|
+
if (task?.status === "waiting_user")
|
|
827
|
+
this.settleUnanswered(task);
|
|
828
|
+
});
|
|
829
|
+
}
|
|
830
|
+
catch (error) {
|
|
831
|
+
failures.set(listed.id, error);
|
|
832
|
+
}
|
|
833
|
+
}
|
|
834
|
+
}
|
|
835
|
+
/**
|
|
836
|
+
* No answer is not a yes (#54). A task whose semantic question went
|
|
837
|
+
* unanswered, at expiry or cut off after its commit, settles closed: its
|
|
838
|
+
* effect ran, so a change is `recovery_required` (never `failed`) and a
|
|
839
|
+
* read is `partial`. Nothing runs here, so it writes no receipt: the
|
|
840
|
+
* certificate, replaced, carries the evidence of the task's last one, the
|
|
841
|
+
* live path's or the one recovery wrote.
|
|
842
|
+
*/
|
|
843
|
+
settleUnanswered(task) {
|
|
844
|
+
const last = task.receiptIds[task.receiptIds.length - 1];
|
|
845
|
+
const receipt = last === undefined ? undefined : this.store.getReceipt(last);
|
|
846
|
+
const evidence = (receipt?.evidence ?? []).map((item) => item.type === "semantic"
|
|
847
|
+
? { ...item, passed: false, inconclusive: true, summary: "semantic_judgement_expired" }
|
|
848
|
+
: item);
|
|
849
|
+
task.status = task.policyDecision.effect === "read" ? "partial" : "recovery_required";
|
|
850
|
+
task.error = "semantic_judgement_expired";
|
|
851
|
+
task.updatedAt = now();
|
|
852
|
+
return { task, certificate: this.createAndSaveCertificate(task, evidence) };
|
|
853
|
+
}
|
|
611
854
|
async recoverTask(task) {
|
|
612
855
|
const request = this.loadRequest(task.id);
|
|
613
856
|
const classified = classifyOperation(request.operation);
|
|
857
|
+
// A semantic question cannot be observed again, and resolveAsVerified
|
|
858
|
+
// settles it closed; the rest must hold as for any other task, so a
|
|
859
|
+
// change with no key to commit is charged here too.
|
|
860
|
+
const observable = request.requiredEvidence.filter((predicate) => predicate.type !== "semantic");
|
|
614
861
|
if (task.status === "verifying" &&
|
|
615
862
|
classified.effect !== "read" &&
|
|
616
|
-
|
|
617
|
-
|
|
863
|
+
observable.length > 0 &&
|
|
864
|
+
observable.every((predicate) => ["file_exists", "file_absent", "file_hash"].includes(predicate.type))) {
|
|
618
865
|
// What the adapter reported before the crash, when it got that far.
|
|
619
866
|
// Evidence alone is not success: a build that exited 1 beside a dist/
|
|
620
867
|
// left by an earlier one would otherwise recover as verified.
|
|
@@ -623,14 +870,28 @@ export class TaskController {
|
|
|
623
870
|
? undefined
|
|
624
871
|
: this.payloadCipher.open(sealed, `task:${task.id}:result`);
|
|
625
872
|
const actionSucceeded = reported?.success !== false;
|
|
626
|
-
|
|
627
|
-
|
|
873
|
+
// Against what the adapter reported, as the live path verifies: a
|
|
874
|
+
// move's own result is what tells a case-only rename's source is gone.
|
|
875
|
+
// Its semantic item, never passed, rides along unanswered as on the
|
|
876
|
+
// live receipt, and is the one item not required to pass.
|
|
877
|
+
const verification = await this.verifier.verify(request.requiredEvidence, reported ?? {});
|
|
878
|
+
if (actionSucceeded &&
|
|
879
|
+
verification.evidence.every((item) => item.passed || item.type === "semantic")) {
|
|
628
880
|
this.resolveAsVerified(task, request, classified, verification.evidence, {
|
|
629
881
|
recovery: "independent_reobservation",
|
|
630
882
|
});
|
|
631
883
|
return task;
|
|
632
884
|
}
|
|
633
885
|
}
|
|
886
|
+
// A read that verified and committed its key before the interruption is
|
|
887
|
+
// done. Run again, it would only be cancelled as a duplicate of itself,
|
|
888
|
+
// and a workflow reads a cancelled step as a denial. A task with a
|
|
889
|
+
// semantic question that committed settles closed, change or read.
|
|
890
|
+
if (classified.effect === "read" || asksSemantic(request)) {
|
|
891
|
+
const committed = this.settleCommitted(task, request, classified);
|
|
892
|
+
if (committed !== undefined)
|
|
893
|
+
return committed.task;
|
|
894
|
+
}
|
|
634
895
|
const effect = classified.effect;
|
|
635
896
|
task.status = effect === "read" ? "planned" : "recovery_required";
|
|
636
897
|
task.error =
|
|
@@ -638,6 +899,10 @@ export class TaskController {
|
|
|
638
899
|
? "interrupted_read_ready_for_retry"
|
|
639
900
|
: "interrupted_mutation_requires_reconciliation";
|
|
640
901
|
task.updatedAt = now();
|
|
902
|
+
// Cut off mid-flight, read or change, it may have run, so it is paid for
|
|
903
|
+
// before it is parked, as `execute` pays for one. A read run again is the
|
|
904
|
+
// same task, so it draws nothing more, but its re-check counts this draw.
|
|
905
|
+
this.drawGrant(task, request, classified);
|
|
641
906
|
this.store.saveTask(task);
|
|
642
907
|
// A crash mid-flight is the same unknown a dropped reply is, so a task
|
|
643
908
|
// that declared how to settle one gets asked here rather than waiting for
|
|
@@ -736,6 +1001,31 @@ export class TaskController {
|
|
|
736
1001
|
...(certificate === undefined ? {} : { certificate }),
|
|
737
1002
|
};
|
|
738
1003
|
}
|
|
1004
|
+
/**
|
|
1005
|
+
* A task's certificate, written now for a `verified_success` task whose
|
|
1006
|
+
* certificate failed to write after it was recorded: a workflow counts a
|
|
1007
|
+
* step done only on its certificate. For a writer of the run, never a
|
|
1008
|
+
* read. Only that status, because every other path to it writes the
|
|
1009
|
+
* certificate before the task, and `execute`, which does not, writes it
|
|
1010
|
+
* only if none is stored: whoever writes first, both hand out the same one.
|
|
1011
|
+
* A write that fails again is left for the next writer.
|
|
1012
|
+
*/
|
|
1013
|
+
certify(taskId) {
|
|
1014
|
+
const certificate = this.store.getCertificateForTask(taskId);
|
|
1015
|
+
if (certificate !== undefined)
|
|
1016
|
+
return certificate;
|
|
1017
|
+
const task = this.store.getTask(taskId);
|
|
1018
|
+
const last = task?.receiptIds.at(-1);
|
|
1019
|
+
const receipt = last === undefined ? undefined : this.store.getReceipt(last);
|
|
1020
|
+
if (task?.status !== "verified_success" || receipt === undefined)
|
|
1021
|
+
return undefined;
|
|
1022
|
+
try {
|
|
1023
|
+
return this.createAndSaveCertificate(task, receipt.evidence, true);
|
|
1024
|
+
}
|
|
1025
|
+
catch {
|
|
1026
|
+
return undefined;
|
|
1027
|
+
}
|
|
1028
|
+
}
|
|
739
1029
|
/**
|
|
740
1030
|
* The mode a task's effect ran under: recorded when it started running, or,
|
|
741
1031
|
* for a task that started before that was recorded, the mode on its first
|
|
@@ -754,11 +1044,13 @@ export class TaskController {
|
|
|
754
1044
|
const approvalId = task.approvedBy?.approvalId ??
|
|
755
1045
|
(task.policyDecision.outcome === "confirm" ? task.approval?.approvalId : undefined);
|
|
756
1046
|
const sanitized = redactStructuredValue(result);
|
|
1047
|
+
// A verifier's summary and source are stored text like any other.
|
|
1048
|
+
const sanitizedEvidence = redactStructuredValue(evidence);
|
|
757
1049
|
const adapterRedactions = Array.isArray(result.redactions)
|
|
758
1050
|
? result.redactions.filter((item) => typeof item === "string")
|
|
759
1051
|
: [];
|
|
760
1052
|
const redactions = [
|
|
761
|
-
...new Set([...adapterRedactions, ...sanitized.redactions]),
|
|
1053
|
+
...new Set([...adapterRedactions, ...sanitized.redactions, ...sanitizedEvidence.redactions]),
|
|
762
1054
|
];
|
|
763
1055
|
return {
|
|
764
1056
|
schemaVersion: "1.0.0",
|
|
@@ -792,7 +1084,7 @@ export class TaskController {
|
|
|
792
1084
|
endedAt: now(),
|
|
793
1085
|
success,
|
|
794
1086
|
observedEffect: sanitized.value,
|
|
795
|
-
evidence,
|
|
1087
|
+
evidence: sanitizedEvidence.value,
|
|
796
1088
|
redactions,
|
|
797
1089
|
...(error === undefined ? {} : { error }),
|
|
798
1090
|
};
|
|
@@ -827,16 +1119,20 @@ export class TaskController {
|
|
|
827
1119
|
};
|
|
828
1120
|
}
|
|
829
1121
|
catch (error) {
|
|
830
|
-
|
|
1122
|
+
// An attempt that may have reached the far end outranks a later one
|
|
1123
|
+
// saying nothing went out: the read may have run, so it is charged.
|
|
1124
|
+
lastError = lastError instanceof Error && isOutcomeUnknown(lastError.message) ? lastError : error;
|
|
831
1125
|
if (attempt >= maximumAttempts || signal.aborted)
|
|
832
|
-
throw
|
|
1126
|
+
throw lastError;
|
|
833
1127
|
}
|
|
834
1128
|
}
|
|
835
1129
|
throw lastError instanceof Error
|
|
836
1130
|
? lastError
|
|
837
1131
|
: new Error("execution_failed_without_error");
|
|
838
1132
|
}
|
|
839
|
-
createAndSaveCertificate(task, evidence
|
|
1133
|
+
createAndSaveCertificate(task, evidence,
|
|
1134
|
+
// A task's first: one another writer stored meanwhile stays, and is used.
|
|
1135
|
+
first = false) {
|
|
840
1136
|
const approvalChannel = task.approvedBy?.channel ??
|
|
841
1137
|
(task.policyDecision.outcome === "allow"
|
|
842
1138
|
? task.policyDecision.effect === "read"
|
|
@@ -853,16 +1149,19 @@ export class TaskController {
|
|
|
853
1149
|
result: certificateResult(task),
|
|
854
1150
|
policyVersion: task.policyDecision.policyVersion,
|
|
855
1151
|
receiptIds: task.receiptIds,
|
|
856
|
-
|
|
1152
|
+
// Redacted as a receipt's is, before the digest is taken. Redacting
|
|
1153
|
+
// again changes nothing, so one written from a stored receipt's items
|
|
1154
|
+
// (`certify`) says what one written from the verifier's would.
|
|
1155
|
+
evidence: redactStructuredValue(evidence).value,
|
|
857
1156
|
createdAt: now(),
|
|
858
1157
|
...(mode === undefined ? {} : { mode }),
|
|
859
1158
|
...(approvalChannel === undefined ? {} : { approvalChannel }),
|
|
860
1159
|
});
|
|
861
|
-
|
|
1160
|
+
const stored = this.store.saveCertificate(certificate, { ifAbsent: first });
|
|
1161
|
+
task.certificateId = stored.certificateId;
|
|
862
1162
|
task.updatedAt = now();
|
|
863
|
-
this.store.saveCertificate(certificate);
|
|
864
1163
|
this.store.saveTask(task);
|
|
865
|
-
return
|
|
1164
|
+
return stored;
|
|
866
1165
|
}
|
|
867
1166
|
finishWithoutReceipt(task, evidence) {
|
|
868
1167
|
const certificate = this.createAndSaveCertificate(task, evidence);
|