@melra/runtime-core 0.3.0-alpha.14 → 0.3.0-alpha.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -15,9 +15,9 @@ function now() {
15
15
  * what the provider was asked to do is the key MELRA itself sent. Shaped like
16
16
  * an adapter result so one predicate type reads the same on both paths.
17
17
  */
18
- function reconciliationFacts(task, operation) {
18
+ function reconciliationFacts(taskId, operation) {
19
19
  return {
20
- taskId: task.id,
20
+ taskId,
21
21
  ...(operation.kind === "http"
22
22
  ? {
23
23
  url: operation.url,
@@ -59,9 +59,10 @@ function certificateResult(task) {
59
59
  * Applied once, where the request is normalized, so the derived predicates flow
60
60
  * identically into the policy decision, the sealed payload, the persisted task
61
61
  * record, and verification. A caller that declared its own evidence is left
62
- * untouched — this only replaces an empty list.
62
+ * untouched — this only replaces an empty list. Exported for `melra policy
63
+ * test`, whose preview must decide on the request the server would.
63
64
  */
64
- function withDefaultEvidence(request) {
65
+ export function withDefaultEvidence(request) {
65
66
  if (request.requiredEvidence.length > 0)
66
67
  return request;
67
68
  const derived = defaultEvidenceFor(request.operation);
@@ -83,6 +84,12 @@ export function capabilityUsageReader(store) {
83
84
  return { operations: drawn.operations, amountToday: drawn.amountInWindow };
84
85
  };
85
86
  }
87
+ /** Statuses a task leaves only once: by starting, by a deny, or by a refusal. */
88
+ const UNSTARTED = ["planned", "awaiting_approval"];
89
+ /** Whether a task waits on a question only someone outside MELRA answers (#54). */
90
+ function asksSemantic(request) {
91
+ return request.requiredEvidence.some((predicate) => predicate.type === "semantic");
92
+ }
86
93
  /** The one browser action that reads each list. */
87
94
  const BROWSER_LIST_ACTIONS = [
88
95
  ["values", "select"],
@@ -96,6 +103,8 @@ export class TaskController {
96
103
  verifier;
97
104
  payloadCipher;
98
105
  active = new Map();
106
+ /** Each task running here, settled once it has ended (see `settled`). */
107
+ running = new Map();
99
108
  breaker;
100
109
  /** Whether approvals come only from an operator, out of band. */
101
110
  get enforced() {
@@ -132,10 +141,12 @@ export class TaskController {
132
141
  };
133
142
  /**
134
143
  * Draws by tasks between their re-check and their record, per grant and
135
- * task. The durable draw is written only once work is verified, so without
136
- * this two tasks running at once both saw the grant unspent: two operations
137
- * against `maxOperations: 1`, a parallel workflow node overspending
138
- * `dailyMax`.
144
+ * task. The durable draw is written only once the adapter has answered, so
145
+ * without this two tasks running at once both saw the grant unspent: two
146
+ * operations against `maxOperations: 1`, a parallel workflow node
147
+ * overspending `dailyMax`. Let go once the draw is written or the effect is
148
+ * known not to have run; one whose draw failed to write is held until
149
+ * recovery writes it (`drawGrant`).
139
150
  *
140
151
  * ponytail: held in this process. Two servers sharing one database can
141
152
  * still both pass a re-check at the same moment; a durable reservation row
@@ -206,13 +217,39 @@ export class TaskController {
206
217
  }
207
218
  }
208
219
  // Refused before anything is approved: a mutation run against evidence no
209
- // registered verifier can check would end `partial` after the effect.
210
- for (const predicate of parsed.requiredEvidence) {
220
+ // registered verifier can check would end `partial` after the effect, and
221
+ // one that could not be reconciled would stay parked for good. A stored
222
+ // step's reconciliation was never checked, and refusing it now would stop
223
+ // a run whose earlier steps already ran.
224
+ this.assertVerifiable(options.stored === true
225
+ ? parsed.requiredEvidence
226
+ : [...parsed.requiredEvidence, ...parsed.reconciliation], options);
227
+ // Read only against the facts known before the call, so a URL token they
228
+ // cannot fill (`{{json.id}}`: no response is read on this path) is
229
+ // refused every time it is asked, and the task would stay parked.
230
+ if (options.stored !== true) {
231
+ const facts = reconciliationFacts("00000000-0000-4000-8000-000000000000", parsed.operation);
232
+ for (const predicate of parsed.reconciliation) {
233
+ const refused = this.verifier.unfilledToken(predicate, facts);
234
+ if (refused !== undefined)
235
+ throw new Error(`reconciliation_unanswerable:${refused}`);
236
+ }
237
+ }
238
+ return evaluatePolicy("00000000-0000-4000-8000-000000000000", parsed, this.policy, this.capabilityUsage).decision;
239
+ }
240
+ /** Refuses a predicate no registered verifier can check, or that it would refuse to. */
241
+ assertVerifiable(predicates, options = {}) {
242
+ for (const predicate of predicates) {
211
243
  if (!this.verifier.supports(predicate)) {
212
244
  throw new Error(`evidence_verifier_unavailable:${predicate.type === "custom" ? predicate.verifier : predicate.type}`);
213
245
  }
246
+ // A file path the verifier refuses however often it is asked. New work
247
+ // only, like the reconciliation check: a run accepted before this
248
+ // check existed is not stopped at the step.
249
+ if (options.stored !== true && this.verifier.outsideWorkspace(predicate)) {
250
+ throw new Error("evidence_path_outside_workspace");
251
+ }
214
252
  }
215
- return evaluatePolicy("00000000-0000-4000-8000-000000000000", parsed, this.policy, this.capabilityUsage).decision;
216
253
  }
217
254
  status(taskId) {
218
255
  const task = this.store.getTask(taskId);
@@ -231,7 +268,7 @@ export class TaskController {
231
268
  if (task.status === "recovery_required") {
232
269
  return await this.reconcile(taskId);
233
270
  }
234
- if (!["planned", "awaiting_approval"].includes(task.status)) {
271
+ if (!UNSTARTED.includes(task.status)) {
235
272
  throw new Error(`task_not_executable:${task.status}`);
236
273
  }
237
274
  if (task.idempotencyKey !== undefined &&
@@ -239,7 +276,7 @@ export class TaskController {
239
276
  task.status = "cancelled";
240
277
  task.error = "duplicate_attempt_prevented";
241
278
  task.updatedAt = now();
242
- this.store.saveTask(task);
279
+ this.leaveUnstarted(task);
243
280
  return this.finishWithoutReceipt(task, [
244
281
  {
245
282
  type: "idempotency",
@@ -255,7 +292,7 @@ export class TaskController {
255
292
  task.status = "policy_blocked";
256
293
  task.policyDecision = rechecked.decision;
257
294
  task.updatedAt = now();
258
- this.store.saveTask(task);
295
+ this.leaveUnstarted(task);
259
296
  return this.finishWithoutReceipt(task, []);
260
297
  }
261
298
  if (rechecked.decision.outcome === "confirm") {
@@ -310,24 +347,44 @@ export class TaskController {
310
347
  }
311
348
  }
312
349
  task.policyDecision = rechecked.decision;
313
- const controller = new AbortController();
314
- this.active.set(task.id, controller);
315
- this.store.inFlight.add(task.id);
316
350
  task.status = "running";
317
351
  task.mode = this.policy.mode;
318
352
  task.updatedAt = now();
319
353
  // Status and owner land in one write, so no running row is ever unowned.
320
- this.store.saveTask(task, this.store.ownerId());
354
+ this.leaveUnstarted(task, this.store.ownerId());
355
+ const controller = new AbortController();
356
+ let ended;
357
+ this.running.set(task.id, new Promise((resolve) => (ended = resolve)));
358
+ this.active.set(task.id, controller);
359
+ this.store.inFlight.add(task.id);
321
360
  const startedAt = now();
322
361
  const classified = classifyOperation(request.operation);
323
362
  let timeout;
324
363
  let budgetExhausted = false;
325
364
  let shortCircuited = false;
365
+ let effectReturned = false;
366
+ let settled;
326
367
  if (rechecked.grantId !== undefined) {
327
368
  const held = this.reserved.get(rechecked.grantId) ?? new Map();
328
369
  held.set(task.id, classified.spend?.amount ?? 0);
329
370
  this.reserved.set(rechecked.grantId, held);
330
371
  }
372
+ // Charged whenever the adapter may have done the work, whatever the
373
+ // evidence says of it: `verified_success` is the evidence verdict only.
374
+ // Read off what the adapter gave back. It returned, whatever its `success`
375
+ // (a request answered 500 reached the far end, a command that exited 1
376
+ // ran), or it threw an outcome-unknown error (`OUTCOME_UNKNOWN_PREFIX`).
377
+ // Any other throw is an adapter saying nothing went out, and costs
378
+ // nothing; so does a refusal before it was called. Counting an effect
379
+ // that did not happen is preferred to a grant funding more than it says.
380
+ const charge = (ran) => {
381
+ if (rechecked.grantId === undefined)
382
+ return;
383
+ if (ran) {
384
+ this.store.recordCapabilityUse(rechecked.grantId, task.id, classified.spend?.amount ?? 0, now());
385
+ }
386
+ this.reserved.get(rechecked.grantId)?.delete(task.id);
387
+ };
331
388
  try {
332
389
  const opened = this.breaker.check(classified.target);
333
390
  if (opened !== undefined) {
@@ -342,6 +399,7 @@ export class TaskController {
342
399
  }, request.budget.maxDurationMs);
343
400
  timeout.unref();
344
401
  const result = await this.executeWithRetries(task, request, classified.effect, controller.signal);
402
+ effectReturned = true;
345
403
  const sanitizedResult = redactStructuredValue(result);
346
404
  task.status = "verifying";
347
405
  task.result = sanitizedResult.value;
@@ -367,52 +425,90 @@ export class TaskController {
367
425
  : verification.evidence;
368
426
  const verified = actionSucceeded &&
369
427
  (request.requiredEvidence.length === 0 || verification.verified);
370
- if (verified &&
428
+ // Every check the kernel can make held; what is left is a question only
429
+ // someone outside answers (#54).
430
+ const awaitingJudgement = actionSucceeded &&
431
+ evidence.some((item) => item.type === "semantic") &&
432
+ evidence.every((item) => item.passed || item.type === "semantic");
433
+ // The effect ran and passed every kernel check, so it is committed now:
434
+ // a key that ran must not run again, even while it waits for its verdict.
435
+ const checksHeld = verified || awaitingJudgement;
436
+ if (checksHeld &&
371
437
  task.idempotencyKey !== undefined &&
372
438
  !this.store.commitIdempotency(task.idempotencyKey, task.id, task.attempt ?? 1, now())) {
373
439
  task.status = "cancelled";
374
440
  task.error = "duplicate_attempt_prevented";
375
441
  }
376
- // Metered here and nowhere else: the same point idempotency commits, so a
377
- // grant is drawn down by work that actually happened and was verified.
378
- // A refusal, a failed verification, a duplicate collapsed just above —
379
- // none of them reach this line, and none of them cost the caller a draw.
380
- if (verified &&
381
- task.status !== "cancelled" &&
382
- rechecked.grantId !== undefined) {
383
- this.store.recordCapabilityUse(rechecked.grantId, task.id, classified.spend?.amount ?? 0, now());
384
- }
442
+ // The adapter returned, so the effect ran: drawn down for a `partial`,
443
+ // a `failed` or a duplicate collapsed just above as for a verified task.
444
+ charge(true);
385
445
  const receipt = this.createReceipt(task, request, classified.capability, classified.target, classified.effect, startedAt, actionSucceeded, result, evidence);
386
446
  this.store.saveReceipt(receipt);
387
447
  task.receiptIds.push(receipt.receiptId);
388
448
  if (task.status !== "cancelled") {
389
449
  task.status = verified
390
450
  ? "verified_success"
391
- : actionSucceeded
392
- ? "partial"
393
- : "failed";
451
+ : awaitingJudgement
452
+ ? "waiting_user"
453
+ : actionSucceeded
454
+ ? "partial"
455
+ : "failed";
456
+ }
457
+ // Written in the same save as the status, so a waiting row always says
458
+ // what it waits on, and until when.
459
+ if (task.status === "waiting_user") {
460
+ const expiresAt = new Date(Date.now() + approvalTtl(this.policy)).toISOString();
461
+ task.judgements = request.requiredEvidence.flatMap((predicate, predicateIndex) => predicate.type === "semantic" ? [{ judgementId: randomUUID(), predicateIndex, expiresAt }] : []);
394
462
  }
395
463
  task.updatedAt = now();
464
+ // A wait lands with its certificate in one transaction: written apart, a
465
+ // peer's expiry between the two was overwritten by the second. Either
466
+ // failing writes neither, so `settled` stays unset for the catch.
467
+ if (task.status === "waiting_user") {
468
+ const certificate = this.store.transaction(() => {
469
+ this.store.saveTask(task);
470
+ return this.createAndSaveCertificate(task, evidence, true);
471
+ });
472
+ return { task, output: result, receipt, certificate };
473
+ }
396
474
  this.store.saveTask(task);
397
- const certificate = this.createAndSaveCertificate(task, evidence);
398
- return { task, output: result, receipt, certificate };
475
+ settled = { task, output: result, receipt };
476
+ const certificate = this.createAndSaveCertificate(task, evidence, true);
477
+ return { ...settled, certificate };
399
478
  }
400
479
  catch (error) {
480
+ // The outcome is recorded, beside its receipt and idempotency commit,
481
+ // and only the certificate summarising it failed to write. Rewriting it
482
+ // would contradict them, and an error would read as the effect not
483
+ // having happened, so the caller gets what was recorded. A workflow
484
+ // step's certificate is written by the run's next advance (`certify`).
485
+ if (settled !== undefined)
486
+ return settled;
401
487
  const rawMessage = error instanceof Error ? error.message : String(error);
402
- const message = redactStructuredValue(rawMessage).value;
488
+ // Capped after redaction, which can lengthen it, at the 10,000 characters
489
+ // a workflow node's `error` holds: a longer one failed the node's write
490
+ // after the effect ran, and the next advance ran the step again.
491
+ const message = redactStructuredValue(rawMessage).value.slice(0, 10_000);
403
492
  const aborted = controller.signal.aborted;
404
493
  // A refusal by the breaker is not fresh evidence about the target, and an
405
494
  // operator's cancellation is not evidence about it at all. A budget that
406
- // ran out is: the target did not answer in time.
407
- if (!shortCircuited && (!aborted || budgetExhausted)) {
495
+ // ran out is: the target did not answer in time. Nor is anything after
496
+ // the adapter returned, which the target already answered.
497
+ if (!shortCircuited && !effectReturned && (!aborted || budgetExhausted)) {
408
498
  this.breaker.recordFailure(classified.target);
409
499
  }
410
- // Checked on the raw message and ahead of every other outcome, because it
411
- // is the strongest thing known: the adapter says the effect may already
412
- // have happened. `failed`, `cancelled` and `budget_exhausted` all assert
413
- // more than that. Reads are exempt — a read that may or may not have run
414
- // changed nothing either way, so there is nothing to reconcile.
415
- const unknownOutcome = classified.effect !== "read" && isOutcomeUnknown(rawMessage);
500
+ // Paid for before the outcome is written, read or change alike. A draw
501
+ // that fails to write leaves the reservation held, and the task running
502
+ // for recovery to charge.
503
+ const ran = effectReturned || isOutcomeUnknown(rawMessage);
504
+ charge(ran);
505
+ // Ahead of every other outcome, because it is the strongest thing known:
506
+ // the effect may already have happened, either because the adapter says
507
+ // so or because it returned and a write after it failed. `failed`,
508
+ // `cancelled` and `budget_exhausted` all assert more than that. Reads
509
+ // are exempt — a read that may or may not have run changed nothing
510
+ // either way, so there is nothing to reconcile.
511
+ const unknownOutcome = classified.effect !== "read" && ran;
416
512
  task.status = unknownOutcome
417
513
  ? "recovery_required"
418
514
  : aborted
@@ -443,7 +539,7 @@ export class TaskController {
443
539
  this.store.saveReceipt(receipt);
444
540
  task.receiptIds.push(receipt.receiptId);
445
541
  this.store.saveTask(task);
446
- const certificate = this.createAndSaveCertificate(task, evidence);
542
+ const certificate = this.createAndSaveCertificate(task, evidence, true);
447
543
  return { task, receipt, certificate };
448
544
  }
449
545
  finally {
@@ -451,10 +547,18 @@ export class TaskController {
451
547
  clearTimeout(timeout);
452
548
  this.active.delete(task.id);
453
549
  this.store.inFlight.delete(task.id);
454
- if (rechecked.grantId !== undefined)
455
- this.reserved.get(rechecked.grantId)?.delete(task.id);
550
+ this.running.delete(task.id);
551
+ ended();
456
552
  }
457
553
  }
554
+ /**
555
+ * Resolves once a task this controller is running has ended, its outcome
556
+ * recorded: a cancel only signals the adapter to stop. At once for one it
557
+ * is not running.
558
+ */
559
+ settled(taskId) {
560
+ return this.running.get(taskId) ?? Promise.resolve();
561
+ }
458
562
  cancel(taskId) {
459
563
  const task = this.status(taskId);
460
564
  const controller = this.active.get(taskId);
@@ -462,13 +566,26 @@ export class TaskController {
462
566
  controller.abort(new Error("task_cancelled"));
463
567
  return task;
464
568
  }
465
- if (["planned", "awaiting_approval"].includes(task.status)) {
569
+ if (UNSTARTED.includes(task.status)) {
466
570
  task.status = "cancelled";
467
571
  task.updatedAt = now();
468
- this.store.saveTask(task);
572
+ // Started meanwhile by another process: it runs, so the caller is told
573
+ // what the task is now rather than that it was cancelled.
574
+ if (!this.store.saveTaskIf(task, UNSTARTED))
575
+ return this.status(taskId);
469
576
  }
470
577
  return task;
471
578
  }
579
+ /**
580
+ * Writes a task out of planned or awaiting_approval, unless a deny or an
581
+ * execute in another process got there after this one read it: that one
582
+ * stands, and this one is refused as it would have been had it read later.
583
+ */
584
+ leaveUnstarted(task, owner) {
585
+ if (!this.store.saveTaskIf(task, UNSTARTED, owner)) {
586
+ throw new Error(`task_not_executable:${this.status(task.id).status}`);
587
+ }
588
+ }
472
589
  async verifyPersisted(taskId, predicates) {
473
590
  const sealed = this.store.getTaskResult(taskId);
474
591
  const result = sealed === undefined
@@ -502,35 +619,67 @@ export class TaskController {
502
619
  }
503
620
  const request = this.loadRequest(taskId);
504
621
  const classified = classifyOperation(request.operation);
505
- const park = (reason) => {
622
+ const park = (reason) => this.settleParked(taskId, () => {
506
623
  task.error = reason;
507
624
  task.updatedAt = now();
508
625
  this.store.saveTask(task);
509
626
  return { task };
510
- };
627
+ });
628
+ // Its own commit already answers: every kernel check held, and only a
629
+ // write after it failed. Parked for good otherwise, with nothing declared.
630
+ const committed = this.settleParked(taskId, () => this.settleCommitted(task, request, classified));
631
+ if (committed !== undefined)
632
+ return committed;
633
+ // A provider can say whether an effect happened, never whether it is
634
+ // right: a task that declared a semantic predicate stays parked with its
635
+ // cause unchanged (#54).
636
+ if (asksSemantic(request))
637
+ return { task };
511
638
  if (request.reconciliation.length === 0) {
512
639
  return park("reconciliation_not_declared");
513
640
  }
514
- const verification = await this.verifier.verify(request.reconciliation, reconciliationFacts(task, request.operation));
641
+ const verification = await this.verifier.verify(request.reconciliation, reconciliationFacts(task.id, request.operation));
642
+ // Refused by the kernel itself, it is refused again however often it is
643
+ // asked: the declaration needs fixing, which waiting on the provider will
644
+ // not do. Named, so a caller can tell the two apart.
645
+ const unanswerable = verification.evidence.filter((item) => item.unanswerable === true);
646
+ if (unanswerable.length > 0) {
647
+ const reason = `reconciliation_unanswerable:${unanswerable.map((item) => item.summary).join("; ")}`;
648
+ return park(redactStructuredValue(reason).value.slice(0, 10_000));
649
+ }
515
650
  // An unreachable provider did not say no. Resolving on this would be the
516
651
  // same lie as the `failed` this whole path exists to avoid.
517
652
  if (verification.evidence.some((item) => item.inconclusive === true)) {
518
653
  return park("reconciliation_inconclusive");
519
654
  }
520
655
  if (verification.verified) {
521
- return this.resolveAsVerified(task, request, classified, verification.evidence, { reconciliation: "provider_state_confirms_effect" });
656
+ return this.settleParked(taskId, () => this.resolveAsVerified(task, request, classified, verification.evidence, { reconciliation: "provider_state_confirms_effect" }));
522
657
  }
523
658
  // The provider answered, and the answer is that it never happened. Only now
524
659
  // is `failed` a claim MELRA is entitled to make.
525
- task.status = "failed";
526
- task.error = "reconciliation_confirms_effect_not_applied";
527
- task.updatedAt = now();
528
- this.store.saveTask(task);
529
- const receipt = this.createReceipt(task, request, classified.capability, classified.target, classified.effect, task.updatedAt, false, { reconciliation: "provider_state_denies_effect" }, verification.evidence, task.error);
530
- this.store.saveReceipt(receipt);
531
- task.receiptIds.push(receipt.receiptId);
532
- const certificate = this.createAndSaveCertificate(task, verification.evidence);
533
- return { task, receipt, certificate };
660
+ return this.settleParked(taskId, () => {
661
+ task.status = "failed";
662
+ task.error = "reconciliation_confirms_effect_not_applied";
663
+ task.updatedAt = now();
664
+ this.store.saveTask(task);
665
+ const receipt = this.createReceipt(task, request, classified.capability, classified.target, classified.effect, task.updatedAt, false, { reconciliation: "provider_state_denies_effect" }, verification.evidence, task.error);
666
+ this.store.saveReceipt(receipt);
667
+ task.receiptIds.push(receipt.receiptId);
668
+ const certificate = this.createAndSaveCertificate(task, verification.evidence);
669
+ return { task, receipt, certificate };
670
+ });
671
+ }
672
+ /**
673
+ * Writes a reconcile's answer in one transaction, and only while the task
674
+ * still waits for one. Two reconciles of one task (two melra_execute calls,
675
+ * or the sweep's recovery beside an agent's) each read it parked and ask
676
+ * the provider; written unconditionally, the slower answer, even one that
677
+ * could not reach the provider, overwrote a settle the other had recorded
678
+ * and dropped its receipt. Now the first stands, and the other writes
679
+ * nothing, not even a receipt, and returns the task as it now stands.
680
+ */
681
+ settleParked(taskId, write) {
682
+ return this.store.transaction(() => this.store.getTask(taskId)?.status === "recovery_required" ? write() : { task: this.status(taskId) });
534
683
  }
535
684
  /**
536
685
  * Settle a task as done on evidence gathered after the fact.
@@ -545,27 +694,78 @@ export class TaskController {
545
694
  this.store.getIdempotencyCommit(task.idempotencyKey) === undefined) {
546
695
  this.store.commitIdempotency(task.idempotencyKey, task.id, task.attempt ?? 1, now());
547
696
  }
548
- // Read for the grant id only. The work is already done, so this is
549
- // accounting rather than permission: a grant that has since run out still
550
- // pays for what it authorised. Asked as if nothing were drawn, because
551
- // policy names no grant at all for one that is now exhausted, and the
552
- // effect then went uncharged.
553
- const { grantId } = evaluatePolicy(task.id, request, this.policy, () => ({
554
- operations: 0,
555
- amountToday: 0,
556
- }));
557
- if (grantId !== undefined) {
558
- this.store.recordCapabilityUse(grantId, task.id, classified.spend?.amount ?? 0, now());
559
- }
697
+ this.drawGrant(task, request, classified);
560
698
  const receipt = this.createReceipt(task, request, classified.capability, classified.target, classified.effect, task.updatedAt, true, observed, evidence);
561
699
  this.store.saveReceipt(receipt);
562
700
  task.receiptIds.push(receipt.receiptId);
701
+ // Evidence gathered after the fact answers no question about meaning: a
702
+ // semantic task, its effect done and receipted, settles closed as at
703
+ // expiry (#54).
704
+ if (asksSemantic(request))
705
+ return { ...this.settleUnanswered(task), receipt };
563
706
  task.status = "verified_success";
564
707
  delete task.error;
565
708
  task.updatedAt = now();
566
709
  const certificate = this.createAndSaveCertificate(task, evidence);
567
710
  return { task, receipt, certificate };
568
711
  }
712
+ /**
713
+ * Draws the grant behind an effect already done, once per task however
714
+ * often asked. Policy is read for the grant id only: this is accounting
715
+ * rather than permission, so a grant that has since run out still pays for
716
+ * what it authorised. Asked as if nothing were drawn, because policy names
717
+ * no grant at all for one that is now exhausted, and the effect then went
718
+ * uncharged. A task that drew one already is left alone: policy read now
719
+ * names the first grant that matches, not the one execute drew once that
720
+ * was spent, and charging it too drew the task twice. Drawn, it lets go
721
+ * of a reservation `execute` held when its own draw failed to write.
722
+ */
723
+ drawGrant(task, request, classified) {
724
+ const grantId = this.store.hasCapabilityUse(task.id)
725
+ ? undefined
726
+ : evaluatePolicy(task.id, request, this.policy, () => ({
727
+ operations: 0,
728
+ amountToday: 0,
729
+ })).grantId;
730
+ if (grantId !== undefined) {
731
+ this.store.recordCapabilityUse(grantId, task.id, classified.spend?.amount ?? 0, now());
732
+ }
733
+ for (const held of this.reserved.values())
734
+ held.delete(task.id);
735
+ }
736
+ /**
737
+ * Settle a task whose idempotency key is committed to it. A key is
738
+ * committed only once every kernel check held, so whatever stopped the
739
+ * task came after the effect: an interruption, or a write that failed. One
740
+ * with a semantic question commits on entering `waiting_user`, so its
741
+ * commit never says the question was answered: it settles closed, as at
742
+ * expiry (#54).
743
+ */
744
+ settleCommitted(task, request, classified) {
745
+ if (task.idempotencyKey === undefined ||
746
+ this.store.getIdempotencyCommit(task.idempotencyKey)?.taskId !== task.id) {
747
+ return undefined;
748
+ }
749
+ if (asksSemantic(request)) {
750
+ // Settled already, it is left as it stands rather than certified again.
751
+ // Asked of the stored row: a peer may have settled it since `task` was
752
+ // read, and its status says `recovery_required` either way.
753
+ const stored = this.status(task.id);
754
+ if (stored.error === "semantic_judgement_expired")
755
+ return { task: stored };
756
+ // Cut off before it reached `waiting_user`, maybe before its draw too.
757
+ this.drawGrant(task, request, classified);
758
+ return this.settleUnanswered(task);
759
+ }
760
+ return this.resolveAsVerified(task, request, classified, [
761
+ {
762
+ type: "idempotency",
763
+ passed: true,
764
+ strength: evidenceStrength("idempotency"),
765
+ summary: "verified and committed before the interruption",
766
+ },
767
+ ], { recovery: "committed_before_interruption" });
768
+ }
569
769
  /**
570
770
  * With `failed`, each task that could not be recovered is recorded there by
571
771
  * id instead of the first failure being thrown at the end.
@@ -604,17 +804,64 @@ export class TaskController {
604
804
  this.store.inFlight.delete(task.id);
605
805
  }
606
806
  }
807
+ this.expireJudgements(failures);
607
808
  if (failed === undefined && failures.size > 0)
608
809
  throw failures.values().next().value;
609
810
  return recovered;
610
811
  }
812
+ /**
813
+ * A waiting task whose judgements are past their `expiresAt` settles
814
+ * closed. Re-read and written in one transaction, so a peer settling the
815
+ * same task wins once.
816
+ *
817
+ * ponytail: runs on the startup pass and the 30 s recovery sweep, so a wait
818
+ * can outlive its `expiresAt` by one interval, or until a runtime next opens
819
+ * this data directory; a verdict tool must check `expiresAt` itself.
820
+ */
821
+ expireJudgements(failures) {
822
+ for (const listed of this.store.listExpiredJudgements(now())) {
823
+ try {
824
+ this.store.transaction(() => {
825
+ const task = this.store.getTask(listed.id);
826
+ if (task?.status === "waiting_user")
827
+ this.settleUnanswered(task);
828
+ });
829
+ }
830
+ catch (error) {
831
+ failures.set(listed.id, error);
832
+ }
833
+ }
834
+ }
835
+ /**
836
+ * No answer is not a yes (#54). A task whose semantic question went
837
+ * unanswered, at expiry or cut off after its commit, settles closed: its
838
+ * effect ran, so a change is `recovery_required` (never `failed`) and a
839
+ * read is `partial`. Nothing runs here, so it writes no receipt: the
840
+ * certificate, replaced, carries the evidence of the task's last one, the
841
+ * live path's or the one recovery wrote.
842
+ */
843
+ settleUnanswered(task) {
844
+ const last = task.receiptIds[task.receiptIds.length - 1];
845
+ const receipt = last === undefined ? undefined : this.store.getReceipt(last);
846
+ const evidence = (receipt?.evidence ?? []).map((item) => item.type === "semantic"
847
+ ? { ...item, passed: false, inconclusive: true, summary: "semantic_judgement_expired" }
848
+ : item);
849
+ task.status = task.policyDecision.effect === "read" ? "partial" : "recovery_required";
850
+ task.error = "semantic_judgement_expired";
851
+ task.updatedAt = now();
852
+ return { task, certificate: this.createAndSaveCertificate(task, evidence) };
853
+ }
611
854
  async recoverTask(task) {
612
855
  const request = this.loadRequest(task.id);
613
856
  const classified = classifyOperation(request.operation);
857
+ // A semantic question cannot be observed again, and resolveAsVerified
858
+ // settles it closed; the rest must hold as for any other task, so a
859
+ // change with no key to commit is charged here too.
860
+ const observable = request.requiredEvidence.filter((predicate) => predicate.type !== "semantic");
614
861
  if (task.status === "verifying" &&
615
862
  classified.effect !== "read" &&
616
- request.requiredEvidence.length > 0 &&
617
- request.requiredEvidence.every((predicate) => ["file_exists", "file_absent", "file_hash"].includes(predicate.type))) {
863
+ observable.length > 0 &&
864
+ observable.every((predicate) => ["file_exists", "file_absent", "file_hash"].includes(predicate.type))) {
618
865
  // What the adapter reported before the crash, when it got that far.
619
866
  // Evidence alone is not success: a build that exited 1 beside a dist/
620
867
  // left by an earlier one would otherwise recover as verified.
@@ -623,14 +870,28 @@ export class TaskController {
623
870
  ? undefined
624
871
  : this.payloadCipher.open(sealed, `task:${task.id}:result`);
625
872
  const actionSucceeded = reported?.success !== false;
626
- const verification = await this.verifier.verify(request.requiredEvidence, {});
627
- if (actionSucceeded && verification.verified) {
873
+ // Against what the adapter reported, as the live path verifies: a
874
+ // move's own result is what tells a case-only rename's source is gone.
875
+ // Its semantic item, never passed, rides along unanswered as on the
876
+ // live receipt, and is the one item not required to pass.
877
+ const verification = await this.verifier.verify(request.requiredEvidence, reported ?? {});
878
+ if (actionSucceeded &&
879
+ verification.evidence.every((item) => item.passed || item.type === "semantic")) {
628
880
  this.resolveAsVerified(task, request, classified, verification.evidence, {
629
881
  recovery: "independent_reobservation",
630
882
  });
631
883
  return task;
632
884
  }
633
885
  }
886
+ // A read that verified and committed its key before the interruption is
887
+ // done. Run again, it would only be cancelled as a duplicate of itself,
888
+ // and a workflow reads a cancelled step as a denial. A task with a
889
+ // semantic question that committed settles closed, change or read.
890
+ if (classified.effect === "read" || asksSemantic(request)) {
891
+ const committed = this.settleCommitted(task, request, classified);
892
+ if (committed !== undefined)
893
+ return committed.task;
894
+ }
634
895
  const effect = classified.effect;
635
896
  task.status = effect === "read" ? "planned" : "recovery_required";
636
897
  task.error =
@@ -638,6 +899,10 @@ export class TaskController {
638
899
  ? "interrupted_read_ready_for_retry"
639
900
  : "interrupted_mutation_requires_reconciliation";
640
901
  task.updatedAt = now();
902
+ // Cut off mid-flight, read or change, it may have run, so it is paid for
903
+ // before it is parked, as `execute` pays for one. A read run again is the
904
+ // same task, so it draws nothing more, but its re-check counts this draw.
905
+ this.drawGrant(task, request, classified);
641
906
  this.store.saveTask(task);
642
907
  // A crash mid-flight is the same unknown a dropped reply is, so a task
643
908
  // that declared how to settle one gets asked here rather than waiting for
@@ -736,6 +1001,31 @@ export class TaskController {
736
1001
  ...(certificate === undefined ? {} : { certificate }),
737
1002
  };
738
1003
  }
1004
+ /**
1005
+ * A task's certificate, written now for a `verified_success` task whose
1006
+ * certificate failed to write after it was recorded: a workflow counts a
1007
+ * step done only on its certificate. For a writer of the run, never a
1008
+ * read. Only that status, because every other path to it writes the
1009
+ * certificate before the task, and `execute`, which does not, writes it
1010
+ * only if none is stored: whoever writes first, both hand out the same one.
1011
+ * A write that fails again is left for the next writer.
1012
+ */
1013
+ certify(taskId) {
1014
+ const certificate = this.store.getCertificateForTask(taskId);
1015
+ if (certificate !== undefined)
1016
+ return certificate;
1017
+ const task = this.store.getTask(taskId);
1018
+ const last = task?.receiptIds.at(-1);
1019
+ const receipt = last === undefined ? undefined : this.store.getReceipt(last);
1020
+ if (task?.status !== "verified_success" || receipt === undefined)
1021
+ return undefined;
1022
+ try {
1023
+ return this.createAndSaveCertificate(task, receipt.evidence, true);
1024
+ }
1025
+ catch {
1026
+ return undefined;
1027
+ }
1028
+ }
739
1029
  /**
740
1030
  * The mode a task's effect ran under: recorded when it started running, or,
741
1031
  * for a task that started before that was recorded, the mode on its first
@@ -754,11 +1044,13 @@ export class TaskController {
754
1044
  const approvalId = task.approvedBy?.approvalId ??
755
1045
  (task.policyDecision.outcome === "confirm" ? task.approval?.approvalId : undefined);
756
1046
  const sanitized = redactStructuredValue(result);
1047
+ // A verifier's summary and source are stored text like any other.
1048
+ const sanitizedEvidence = redactStructuredValue(evidence);
757
1049
  const adapterRedactions = Array.isArray(result.redactions)
758
1050
  ? result.redactions.filter((item) => typeof item === "string")
759
1051
  : [];
760
1052
  const redactions = [
761
- ...new Set([...adapterRedactions, ...sanitized.redactions]),
1053
+ ...new Set([...adapterRedactions, ...sanitized.redactions, ...sanitizedEvidence.redactions]),
762
1054
  ];
763
1055
  return {
764
1056
  schemaVersion: "1.0.0",
@@ -792,7 +1084,7 @@ export class TaskController {
792
1084
  endedAt: now(),
793
1085
  success,
794
1086
  observedEffect: sanitized.value,
795
- evidence,
1087
+ evidence: sanitizedEvidence.value,
796
1088
  redactions,
797
1089
  ...(error === undefined ? {} : { error }),
798
1090
  };
@@ -827,16 +1119,20 @@ export class TaskController {
827
1119
  };
828
1120
  }
829
1121
  catch (error) {
830
- lastError = error;
1122
+ // An attempt that may have reached the far end outranks a later one
1123
+ // saying nothing went out: the read may have run, so it is charged.
1124
+ lastError = lastError instanceof Error && isOutcomeUnknown(lastError.message) ? lastError : error;
831
1125
  if (attempt >= maximumAttempts || signal.aborted)
832
- throw error;
1126
+ throw lastError;
833
1127
  }
834
1128
  }
835
1129
  throw lastError instanceof Error
836
1130
  ? lastError
837
1131
  : new Error("execution_failed_without_error");
838
1132
  }
839
- createAndSaveCertificate(task, evidence) {
1133
+ createAndSaveCertificate(task, evidence,
1134
+ // A task's first: one another writer stored meanwhile stays, and is used.
1135
+ first = false) {
840
1136
  const approvalChannel = task.approvedBy?.channel ??
841
1137
  (task.policyDecision.outcome === "allow"
842
1138
  ? task.policyDecision.effect === "read"
@@ -853,16 +1149,19 @@ export class TaskController {
853
1149
  result: certificateResult(task),
854
1150
  policyVersion: task.policyDecision.policyVersion,
855
1151
  receiptIds: task.receiptIds,
856
- evidence,
1152
+ // Redacted as a receipt's is, before the digest is taken. Redacting
1153
+ // again changes nothing, so one written from a stored receipt's items
1154
+ // (`certify`) says what one written from the verifier's would.
1155
+ evidence: redactStructuredValue(evidence).value,
857
1156
  createdAt: now(),
858
1157
  ...(mode === undefined ? {} : { mode }),
859
1158
  ...(approvalChannel === undefined ? {} : { approvalChannel }),
860
1159
  });
861
- task.certificateId = certificate.certificateId;
1160
+ const stored = this.store.saveCertificate(certificate, { ifAbsent: first });
1161
+ task.certificateId = stored.certificateId;
862
1162
  task.updatedAt = now();
863
- this.store.saveCertificate(certificate);
864
1163
  this.store.saveTask(task);
865
- return certificate;
1164
+ return stored;
866
1165
  }
867
1166
  finishWithoutReceipt(task, evidence) {
868
1167
  const certificate = this.createAndSaveCertificate(task, evidence);