@melra/runtime-core 0.3.0-alpha.14 → 0.3.0-alpha.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -2
- package/dist/task-controller.d.ts +89 -1
- package/dist/task-controller.d.ts.map +1 -1
- package/dist/task-controller.js +351 -73
- package/dist/task-controller.js.map +1 -1
- package/dist/workflow-controller.d.ts +87 -0
- package/dist/workflow-controller.d.ts.map +1 -1
- package/dist/workflow-controller.js +1062 -258
- package/dist/workflow-controller.js.map +1 -1
- package/package.json +9 -6
package/dist/task-controller.js
CHANGED
|
@@ -15,9 +15,9 @@ function now() {
|
|
|
15
15
|
* what the provider was asked to do is the key MELRA itself sent. Shaped like
|
|
16
16
|
* an adapter result so one predicate type reads the same on both paths.
|
|
17
17
|
*/
|
|
18
|
-
function reconciliationFacts(
|
|
18
|
+
function reconciliationFacts(taskId, operation) {
|
|
19
19
|
return {
|
|
20
|
-
taskId
|
|
20
|
+
taskId,
|
|
21
21
|
...(operation.kind === "http"
|
|
22
22
|
? {
|
|
23
23
|
url: operation.url,
|
|
@@ -59,9 +59,10 @@ function certificateResult(task) {
|
|
|
59
59
|
* Applied once, where the request is normalized, so the derived predicates flow
|
|
60
60
|
* identically into the policy decision, the sealed payload, the persisted task
|
|
61
61
|
* record, and verification. A caller that declared its own evidence is left
|
|
62
|
-
* untouched — this only replaces an empty list.
|
|
62
|
+
* untouched — this only replaces an empty list. Exported for `melra policy
|
|
63
|
+
* test`, whose preview must decide on the request the server would.
|
|
63
64
|
*/
|
|
64
|
-
function withDefaultEvidence(request) {
|
|
65
|
+
export function withDefaultEvidence(request) {
|
|
65
66
|
if (request.requiredEvidence.length > 0)
|
|
66
67
|
return request;
|
|
67
68
|
const derived = defaultEvidenceFor(request.operation);
|
|
@@ -83,6 +84,12 @@ export function capabilityUsageReader(store) {
|
|
|
83
84
|
return { operations: drawn.operations, amountToday: drawn.amountInWindow };
|
|
84
85
|
};
|
|
85
86
|
}
|
|
87
|
+
/** Statuses a task leaves only once: by starting, by a deny, or by a refusal. */
|
|
88
|
+
const UNSTARTED = ["planned", "awaiting_approval"];
|
|
89
|
+
/** Whether a task waits on a question only someone outside MELRA answers (#54). */
|
|
90
|
+
function asksSemantic(request) {
|
|
91
|
+
return request.requiredEvidence.some((predicate) => predicate.type === "semantic");
|
|
92
|
+
}
|
|
86
93
|
/** The one browser action that reads each list. */
|
|
87
94
|
const BROWSER_LIST_ACTIONS = [
|
|
88
95
|
["values", "select"],
|
|
@@ -96,6 +103,8 @@ export class TaskController {
|
|
|
96
103
|
verifier;
|
|
97
104
|
payloadCipher;
|
|
98
105
|
active = new Map();
|
|
106
|
+
/** Each task running here, settled once it has ended (see `settled`). */
|
|
107
|
+
running = new Map();
|
|
99
108
|
breaker;
|
|
100
109
|
/** Whether approvals come only from an operator, out of band. */
|
|
101
110
|
get enforced() {
|
|
@@ -132,7 +141,8 @@ export class TaskController {
|
|
|
132
141
|
};
|
|
133
142
|
/**
|
|
134
143
|
* Draws by tasks between their re-check and their record, per grant and
|
|
135
|
-
* task. The durable draw is written only once work
|
|
144
|
+
* task. The durable draw is written only once work has passed every check
|
|
145
|
+
* the kernel can make (#54: verified, or waiting on a semantic verdict), so without
|
|
136
146
|
* this two tasks running at once both saw the grant unspent: two operations
|
|
137
147
|
* against `maxOperations: 1`, a parallel workflow node overspending
|
|
138
148
|
* `dailyMax`.
|
|
@@ -206,13 +216,39 @@ export class TaskController {
|
|
|
206
216
|
}
|
|
207
217
|
}
|
|
208
218
|
// Refused before anything is approved: a mutation run against evidence no
|
|
209
|
-
// registered verifier can check would end `partial` after the effect
|
|
210
|
-
|
|
219
|
+
// registered verifier can check would end `partial` after the effect, and
|
|
220
|
+
// one that could not be reconciled would stay parked for good. A stored
|
|
221
|
+
// step's reconciliation was never checked, and refusing it now would stop
|
|
222
|
+
// a run whose earlier steps already ran.
|
|
223
|
+
this.assertVerifiable(options.stored === true
|
|
224
|
+
? parsed.requiredEvidence
|
|
225
|
+
: [...parsed.requiredEvidence, ...parsed.reconciliation], options);
|
|
226
|
+
// Read only against the facts known before the call, so a URL token they
|
|
227
|
+
// cannot fill (`{{json.id}}`: no response is read on this path) is
|
|
228
|
+
// refused every time it is asked, and the task would stay parked.
|
|
229
|
+
if (options.stored !== true) {
|
|
230
|
+
const facts = reconciliationFacts("00000000-0000-4000-8000-000000000000", parsed.operation);
|
|
231
|
+
for (const predicate of parsed.reconciliation) {
|
|
232
|
+
const refused = this.verifier.unfilledToken(predicate, facts);
|
|
233
|
+
if (refused !== undefined)
|
|
234
|
+
throw new Error(`reconciliation_unanswerable:${refused}`);
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
return evaluatePolicy("00000000-0000-4000-8000-000000000000", parsed, this.policy, this.capabilityUsage).decision;
|
|
238
|
+
}
|
|
239
|
+
/** Refuses a predicate no registered verifier can check, or that it would refuse to. */
|
|
240
|
+
assertVerifiable(predicates, options = {}) {
|
|
241
|
+
for (const predicate of predicates) {
|
|
211
242
|
if (!this.verifier.supports(predicate)) {
|
|
212
243
|
throw new Error(`evidence_verifier_unavailable:${predicate.type === "custom" ? predicate.verifier : predicate.type}`);
|
|
213
244
|
}
|
|
245
|
+
// A file path the verifier refuses however often it is asked. New work
|
|
246
|
+
// only, like the reconciliation check: a run accepted before this
|
|
247
|
+
// check existed is not stopped at the step.
|
|
248
|
+
if (options.stored !== true && this.verifier.outsideWorkspace(predicate)) {
|
|
249
|
+
throw new Error("evidence_path_outside_workspace");
|
|
250
|
+
}
|
|
214
251
|
}
|
|
215
|
-
return evaluatePolicy("00000000-0000-4000-8000-000000000000", parsed, this.policy, this.capabilityUsage).decision;
|
|
216
252
|
}
|
|
217
253
|
status(taskId) {
|
|
218
254
|
const task = this.store.getTask(taskId);
|
|
@@ -231,7 +267,7 @@ export class TaskController {
|
|
|
231
267
|
if (task.status === "recovery_required") {
|
|
232
268
|
return await this.reconcile(taskId);
|
|
233
269
|
}
|
|
234
|
-
if (!
|
|
270
|
+
if (!UNSTARTED.includes(task.status)) {
|
|
235
271
|
throw new Error(`task_not_executable:${task.status}`);
|
|
236
272
|
}
|
|
237
273
|
if (task.idempotencyKey !== undefined &&
|
|
@@ -239,7 +275,7 @@ export class TaskController {
|
|
|
239
275
|
task.status = "cancelled";
|
|
240
276
|
task.error = "duplicate_attempt_prevented";
|
|
241
277
|
task.updatedAt = now();
|
|
242
|
-
this.
|
|
278
|
+
this.leaveUnstarted(task);
|
|
243
279
|
return this.finishWithoutReceipt(task, [
|
|
244
280
|
{
|
|
245
281
|
type: "idempotency",
|
|
@@ -255,7 +291,7 @@ export class TaskController {
|
|
|
255
291
|
task.status = "policy_blocked";
|
|
256
292
|
task.policyDecision = rechecked.decision;
|
|
257
293
|
task.updatedAt = now();
|
|
258
|
-
this.
|
|
294
|
+
this.leaveUnstarted(task);
|
|
259
295
|
return this.finishWithoutReceipt(task, []);
|
|
260
296
|
}
|
|
261
297
|
if (rechecked.decision.outcome === "confirm") {
|
|
@@ -310,19 +346,23 @@ export class TaskController {
|
|
|
310
346
|
}
|
|
311
347
|
}
|
|
312
348
|
task.policyDecision = rechecked.decision;
|
|
313
|
-
const controller = new AbortController();
|
|
314
|
-
this.active.set(task.id, controller);
|
|
315
|
-
this.store.inFlight.add(task.id);
|
|
316
349
|
task.status = "running";
|
|
317
350
|
task.mode = this.policy.mode;
|
|
318
351
|
task.updatedAt = now();
|
|
319
352
|
// Status and owner land in one write, so no running row is ever unowned.
|
|
320
|
-
this.
|
|
353
|
+
this.leaveUnstarted(task, this.store.ownerId());
|
|
354
|
+
const controller = new AbortController();
|
|
355
|
+
let ended;
|
|
356
|
+
this.running.set(task.id, new Promise((resolve) => (ended = resolve)));
|
|
357
|
+
this.active.set(task.id, controller);
|
|
358
|
+
this.store.inFlight.add(task.id);
|
|
321
359
|
const startedAt = now();
|
|
322
360
|
const classified = classifyOperation(request.operation);
|
|
323
361
|
let timeout;
|
|
324
362
|
let budgetExhausted = false;
|
|
325
363
|
let shortCircuited = false;
|
|
364
|
+
let effectReturned = false;
|
|
365
|
+
let settled;
|
|
326
366
|
if (rechecked.grantId !== undefined) {
|
|
327
367
|
const held = this.reserved.get(rechecked.grantId) ?? new Map();
|
|
328
368
|
held.set(task.id, classified.spend?.amount ?? 0);
|
|
@@ -342,6 +382,7 @@ export class TaskController {
|
|
|
342
382
|
}, request.budget.maxDurationMs);
|
|
343
383
|
timeout.unref();
|
|
344
384
|
const result = await this.executeWithRetries(task, request, classified.effect, controller.signal);
|
|
385
|
+
effectReturned = true;
|
|
345
386
|
const sanitizedResult = redactStructuredValue(result);
|
|
346
387
|
task.status = "verifying";
|
|
347
388
|
task.result = sanitizedResult.value;
|
|
@@ -367,17 +408,27 @@ export class TaskController {
|
|
|
367
408
|
: verification.evidence;
|
|
368
409
|
const verified = actionSucceeded &&
|
|
369
410
|
(request.requiredEvidence.length === 0 || verification.verified);
|
|
370
|
-
|
|
411
|
+
// Every check the kernel can make held; what is left is a question only
|
|
412
|
+
// someone outside answers (#54).
|
|
413
|
+
const awaitingJudgement = actionSucceeded &&
|
|
414
|
+
evidence.some((item) => item.type === "semantic") &&
|
|
415
|
+
evidence.every((item) => item.passed || item.type === "semantic");
|
|
416
|
+
// The effect ran and passed every kernel check, so it is committed and
|
|
417
|
+
// metered now: a key that ran must not run again, and a grant must not
|
|
418
|
+
// fund a second effect while the first waits for its verdict.
|
|
419
|
+
const checksHeld = verified || awaitingJudgement;
|
|
420
|
+
if (checksHeld &&
|
|
371
421
|
task.idempotencyKey !== undefined &&
|
|
372
422
|
!this.store.commitIdempotency(task.idempotencyKey, task.id, task.attempt ?? 1, now())) {
|
|
373
423
|
task.status = "cancelled";
|
|
374
424
|
task.error = "duplicate_attempt_prevented";
|
|
375
425
|
}
|
|
376
426
|
// Metered here and nowhere else: the same point idempotency commits, so a
|
|
377
|
-
// grant is drawn down by work that actually happened and
|
|
378
|
-
//
|
|
379
|
-
//
|
|
380
|
-
|
|
427
|
+
// grant is drawn down by work that actually happened and passed every
|
|
428
|
+
// check the kernel can make (verified, or waiting on a semantic verdict).
|
|
429
|
+
// A refusal, a failed check, a duplicate collapsed just above — none of
|
|
430
|
+
// them reach this line, and none of them cost the caller a draw.
|
|
431
|
+
if (checksHeld &&
|
|
381
432
|
task.status !== "cancelled" &&
|
|
382
433
|
rechecked.grantId !== undefined) {
|
|
383
434
|
this.store.recordCapabilityUse(rechecked.grantId, task.id, classified.spend?.amount ?? 0, now());
|
|
@@ -388,31 +439,62 @@ export class TaskController {
|
|
|
388
439
|
if (task.status !== "cancelled") {
|
|
389
440
|
task.status = verified
|
|
390
441
|
? "verified_success"
|
|
391
|
-
:
|
|
392
|
-
? "
|
|
393
|
-
:
|
|
442
|
+
: awaitingJudgement
|
|
443
|
+
? "waiting_user"
|
|
444
|
+
: actionSucceeded
|
|
445
|
+
? "partial"
|
|
446
|
+
: "failed";
|
|
447
|
+
}
|
|
448
|
+
// Written in the same save as the status, so a waiting row always says
|
|
449
|
+
// what it waits on, and until when.
|
|
450
|
+
if (task.status === "waiting_user") {
|
|
451
|
+
const expiresAt = new Date(Date.now() + approvalTtl(this.policy)).toISOString();
|
|
452
|
+
task.judgements = request.requiredEvidence.flatMap((predicate, predicateIndex) => predicate.type === "semantic" ? [{ judgementId: randomUUID(), predicateIndex, expiresAt }] : []);
|
|
394
453
|
}
|
|
395
454
|
task.updatedAt = now();
|
|
455
|
+
// A wait lands with its certificate in one transaction: written apart, a
|
|
456
|
+
// peer's expiry between the two was overwritten by the second. Either
|
|
457
|
+
// failing writes neither, so `settled` stays unset for the catch.
|
|
458
|
+
if (task.status === "waiting_user") {
|
|
459
|
+
const certificate = this.store.transaction(() => {
|
|
460
|
+
this.store.saveTask(task);
|
|
461
|
+
return this.createAndSaveCertificate(task, evidence, true);
|
|
462
|
+
});
|
|
463
|
+
return { task, output: result, receipt, certificate };
|
|
464
|
+
}
|
|
396
465
|
this.store.saveTask(task);
|
|
397
|
-
|
|
398
|
-
|
|
466
|
+
settled = { task, output: result, receipt };
|
|
467
|
+
const certificate = this.createAndSaveCertificate(task, evidence, true);
|
|
468
|
+
return { ...settled, certificate };
|
|
399
469
|
}
|
|
400
470
|
catch (error) {
|
|
471
|
+
// The outcome is recorded, beside its receipt and idempotency commit,
|
|
472
|
+
// and only the certificate summarising it failed to write. Rewriting it
|
|
473
|
+
// would contradict them, and an error would read as the effect not
|
|
474
|
+
// having happened, so the caller gets what was recorded. A workflow
|
|
475
|
+
// step's certificate is written by the run's next advance (`certify`).
|
|
476
|
+
if (settled !== undefined)
|
|
477
|
+
return settled;
|
|
401
478
|
const rawMessage = error instanceof Error ? error.message : String(error);
|
|
402
|
-
|
|
479
|
+
// Capped after redaction, which can lengthen it, at the 10,000 characters
|
|
480
|
+
// a workflow node's `error` holds: a longer one failed the node's write
|
|
481
|
+
// after the effect ran, and the next advance ran the step again.
|
|
482
|
+
const message = redactStructuredValue(rawMessage).value.slice(0, 10_000);
|
|
403
483
|
const aborted = controller.signal.aborted;
|
|
404
484
|
// A refusal by the breaker is not fresh evidence about the target, and an
|
|
405
485
|
// operator's cancellation is not evidence about it at all. A budget that
|
|
406
|
-
// ran out is: the target did not answer in time.
|
|
407
|
-
|
|
486
|
+
// ran out is: the target did not answer in time. Nor is anything after
|
|
487
|
+
// the adapter returned, which the target already answered.
|
|
488
|
+
if (!shortCircuited && !effectReturned && (!aborted || budgetExhausted)) {
|
|
408
489
|
this.breaker.recordFailure(classified.target);
|
|
409
490
|
}
|
|
410
|
-
//
|
|
411
|
-
//
|
|
412
|
-
//
|
|
413
|
-
//
|
|
414
|
-
//
|
|
415
|
-
|
|
491
|
+
// Ahead of every other outcome, because it is the strongest thing known:
|
|
492
|
+
// the effect may already have happened, either because the adapter says
|
|
493
|
+
// so or because it returned and a write after it failed. `failed`,
|
|
494
|
+
// `cancelled` and `budget_exhausted` all assert more than that. Reads
|
|
495
|
+
// are exempt — a read that may or may not have run changed nothing
|
|
496
|
+
// either way, so there is nothing to reconcile.
|
|
497
|
+
const unknownOutcome = classified.effect !== "read" && (effectReturned || isOutcomeUnknown(rawMessage));
|
|
416
498
|
task.status = unknownOutcome
|
|
417
499
|
? "recovery_required"
|
|
418
500
|
: aborted
|
|
@@ -443,7 +525,7 @@ export class TaskController {
|
|
|
443
525
|
this.store.saveReceipt(receipt);
|
|
444
526
|
task.receiptIds.push(receipt.receiptId);
|
|
445
527
|
this.store.saveTask(task);
|
|
446
|
-
const certificate = this.createAndSaveCertificate(task, evidence);
|
|
528
|
+
const certificate = this.createAndSaveCertificate(task, evidence, true);
|
|
447
529
|
return { task, receipt, certificate };
|
|
448
530
|
}
|
|
449
531
|
finally {
|
|
@@ -453,8 +535,18 @@ export class TaskController {
|
|
|
453
535
|
this.store.inFlight.delete(task.id);
|
|
454
536
|
if (rechecked.grantId !== undefined)
|
|
455
537
|
this.reserved.get(rechecked.grantId)?.delete(task.id);
|
|
538
|
+
this.running.delete(task.id);
|
|
539
|
+
ended();
|
|
456
540
|
}
|
|
457
541
|
}
|
|
542
|
+
/**
|
|
543
|
+
* Resolves once a task this controller is running has ended, its outcome
|
|
544
|
+
* recorded: a cancel only signals the adapter to stop. At once for one it
|
|
545
|
+
* is not running.
|
|
546
|
+
*/
|
|
547
|
+
settled(taskId) {
|
|
548
|
+
return this.running.get(taskId) ?? Promise.resolve();
|
|
549
|
+
}
|
|
458
550
|
cancel(taskId) {
|
|
459
551
|
const task = this.status(taskId);
|
|
460
552
|
const controller = this.active.get(taskId);
|
|
@@ -462,13 +554,26 @@ export class TaskController {
|
|
|
462
554
|
controller.abort(new Error("task_cancelled"));
|
|
463
555
|
return task;
|
|
464
556
|
}
|
|
465
|
-
if (
|
|
557
|
+
if (UNSTARTED.includes(task.status)) {
|
|
466
558
|
task.status = "cancelled";
|
|
467
559
|
task.updatedAt = now();
|
|
468
|
-
|
|
560
|
+
// Started meanwhile by another process: it runs, so the caller is told
|
|
561
|
+
// what the task is now rather than that it was cancelled.
|
|
562
|
+
if (!this.store.saveTaskIf(task, UNSTARTED))
|
|
563
|
+
return this.status(taskId);
|
|
469
564
|
}
|
|
470
565
|
return task;
|
|
471
566
|
}
|
|
567
|
+
/**
|
|
568
|
+
* Writes a task out of planned or awaiting_approval, unless a deny or an
|
|
569
|
+
* execute in another process got there after this one read it: that one
|
|
570
|
+
* stands, and this one is refused as it would have been had it read later.
|
|
571
|
+
*/
|
|
572
|
+
leaveUnstarted(task, owner) {
|
|
573
|
+
if (!this.store.saveTaskIf(task, UNSTARTED, owner)) {
|
|
574
|
+
throw new Error(`task_not_executable:${this.status(task.id).status}`);
|
|
575
|
+
}
|
|
576
|
+
}
|
|
472
577
|
async verifyPersisted(taskId, predicates) {
|
|
473
578
|
const sealed = this.store.getTaskResult(taskId);
|
|
474
579
|
const result = sealed === undefined
|
|
@@ -502,35 +607,67 @@ export class TaskController {
|
|
|
502
607
|
}
|
|
503
608
|
const request = this.loadRequest(taskId);
|
|
504
609
|
const classified = classifyOperation(request.operation);
|
|
505
|
-
const park = (reason) => {
|
|
610
|
+
const park = (reason) => this.settleParked(taskId, () => {
|
|
506
611
|
task.error = reason;
|
|
507
612
|
task.updatedAt = now();
|
|
508
613
|
this.store.saveTask(task);
|
|
509
614
|
return { task };
|
|
510
|
-
};
|
|
615
|
+
});
|
|
616
|
+
// Its own commit already answers: every kernel check held, and only a
|
|
617
|
+
// write after it failed. Parked for good otherwise, with nothing declared.
|
|
618
|
+
const committed = this.settleParked(taskId, () => this.settleCommitted(task, request, classified));
|
|
619
|
+
if (committed !== undefined)
|
|
620
|
+
return committed;
|
|
621
|
+
// A provider can say whether an effect happened, never whether it is
|
|
622
|
+
// right: a task that declared a semantic predicate stays parked with its
|
|
623
|
+
// cause unchanged (#54).
|
|
624
|
+
if (asksSemantic(request))
|
|
625
|
+
return { task };
|
|
511
626
|
if (request.reconciliation.length === 0) {
|
|
512
627
|
return park("reconciliation_not_declared");
|
|
513
628
|
}
|
|
514
|
-
const verification = await this.verifier.verify(request.reconciliation, reconciliationFacts(task, request.operation));
|
|
629
|
+
const verification = await this.verifier.verify(request.reconciliation, reconciliationFacts(task.id, request.operation));
|
|
630
|
+
// Refused by the kernel itself, it is refused again however often it is
|
|
631
|
+
// asked: the declaration needs fixing, which waiting on the provider will
|
|
632
|
+
// not do. Named, so a caller can tell the two apart.
|
|
633
|
+
const unanswerable = verification.evidence.filter((item) => item.unanswerable === true);
|
|
634
|
+
if (unanswerable.length > 0) {
|
|
635
|
+
const reason = `reconciliation_unanswerable:${unanswerable.map((item) => item.summary).join("; ")}`;
|
|
636
|
+
return park(redactStructuredValue(reason).value.slice(0, 10_000));
|
|
637
|
+
}
|
|
515
638
|
// An unreachable provider did not say no. Resolving on this would be the
|
|
516
639
|
// same lie as the `failed` this whole path exists to avoid.
|
|
517
640
|
if (verification.evidence.some((item) => item.inconclusive === true)) {
|
|
518
641
|
return park("reconciliation_inconclusive");
|
|
519
642
|
}
|
|
520
643
|
if (verification.verified) {
|
|
521
|
-
return this.resolveAsVerified(task, request, classified, verification.evidence, { reconciliation: "provider_state_confirms_effect" });
|
|
644
|
+
return this.settleParked(taskId, () => this.resolveAsVerified(task, request, classified, verification.evidence, { reconciliation: "provider_state_confirms_effect" }));
|
|
522
645
|
}
|
|
523
646
|
// The provider answered, and the answer is that it never happened. Only now
|
|
524
647
|
// is `failed` a claim MELRA is entitled to make.
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
648
|
+
return this.settleParked(taskId, () => {
|
|
649
|
+
task.status = "failed";
|
|
650
|
+
task.error = "reconciliation_confirms_effect_not_applied";
|
|
651
|
+
task.updatedAt = now();
|
|
652
|
+
this.store.saveTask(task);
|
|
653
|
+
const receipt = this.createReceipt(task, request, classified.capability, classified.target, classified.effect, task.updatedAt, false, { reconciliation: "provider_state_denies_effect" }, verification.evidence, task.error);
|
|
654
|
+
this.store.saveReceipt(receipt);
|
|
655
|
+
task.receiptIds.push(receipt.receiptId);
|
|
656
|
+
const certificate = this.createAndSaveCertificate(task, verification.evidence);
|
|
657
|
+
return { task, receipt, certificate };
|
|
658
|
+
});
|
|
659
|
+
}
|
|
660
|
+
/**
|
|
661
|
+
* Writes a reconcile's answer in one transaction, and only while the task
|
|
662
|
+
* still waits for one. Two reconciles of one task (two melra_execute calls,
|
|
663
|
+
* or the sweep's recovery beside an agent's) each read it parked and ask
|
|
664
|
+
* the provider; written unconditionally, the slower answer, even one that
|
|
665
|
+
* could not reach the provider, overwrote a settle the other had recorded
|
|
666
|
+
* and dropped its receipt. Now the first stands, and the other writes
|
|
667
|
+
* nothing, not even a receipt, and returns the task as it now stands.
|
|
668
|
+
*/
|
|
669
|
+
settleParked(taskId, write) {
|
|
670
|
+
return this.store.transaction(() => this.store.getTask(taskId)?.status === "recovery_required" ? write() : { task: this.status(taskId) });
|
|
534
671
|
}
|
|
535
672
|
/**
|
|
536
673
|
* Settle a task as done on evidence gathered after the fact.
|
|
@@ -545,27 +682,75 @@ export class TaskController {
|
|
|
545
682
|
this.store.getIdempotencyCommit(task.idempotencyKey) === undefined) {
|
|
546
683
|
this.store.commitIdempotency(task.idempotencyKey, task.id, task.attempt ?? 1, now());
|
|
547
684
|
}
|
|
548
|
-
|
|
549
|
-
// accounting rather than permission: a grant that has since run out still
|
|
550
|
-
// pays for what it authorised. Asked as if nothing were drawn, because
|
|
551
|
-
// policy names no grant at all for one that is now exhausted, and the
|
|
552
|
-
// effect then went uncharged.
|
|
553
|
-
const { grantId } = evaluatePolicy(task.id, request, this.policy, () => ({
|
|
554
|
-
operations: 0,
|
|
555
|
-
amountToday: 0,
|
|
556
|
-
}));
|
|
557
|
-
if (grantId !== undefined) {
|
|
558
|
-
this.store.recordCapabilityUse(grantId, task.id, classified.spend?.amount ?? 0, now());
|
|
559
|
-
}
|
|
685
|
+
this.drawGrant(task, request, classified);
|
|
560
686
|
const receipt = this.createReceipt(task, request, classified.capability, classified.target, classified.effect, task.updatedAt, true, observed, evidence);
|
|
561
687
|
this.store.saveReceipt(receipt);
|
|
562
688
|
task.receiptIds.push(receipt.receiptId);
|
|
689
|
+
// Evidence gathered after the fact answers no question about meaning: a
|
|
690
|
+
// semantic task, its effect done and receipted, settles closed as at
|
|
691
|
+
// expiry (#54).
|
|
692
|
+
if (asksSemantic(request))
|
|
693
|
+
return { ...this.settleUnanswered(task), receipt };
|
|
563
694
|
task.status = "verified_success";
|
|
564
695
|
delete task.error;
|
|
565
696
|
task.updatedAt = now();
|
|
566
697
|
const certificate = this.createAndSaveCertificate(task, evidence);
|
|
567
698
|
return { task, receipt, certificate };
|
|
568
699
|
}
|
|
700
|
+
/**
|
|
701
|
+
* Draws the grant behind an effect already done, once per task however
|
|
702
|
+
* often asked. Policy is read for the grant id only: this is accounting
|
|
703
|
+
* rather than permission, so a grant that has since run out still pays for
|
|
704
|
+
* what it authorised. Asked as if nothing were drawn, because policy names
|
|
705
|
+
* no grant at all for one that is now exhausted, and the effect then went
|
|
706
|
+
* uncharged. A task that drew one already is left alone: policy read now
|
|
707
|
+
* names the first grant that matches, not the one execute drew once that
|
|
708
|
+
* was spent, and charging it too drew the task twice.
|
|
709
|
+
*/
|
|
710
|
+
drawGrant(task, request, classified) {
|
|
711
|
+
if (this.store.hasCapabilityUse(task.id))
|
|
712
|
+
return;
|
|
713
|
+
const { grantId } = evaluatePolicy(task.id, request, this.policy, () => ({
|
|
714
|
+
operations: 0,
|
|
715
|
+
amountToday: 0,
|
|
716
|
+
}));
|
|
717
|
+
if (grantId !== undefined) {
|
|
718
|
+
this.store.recordCapabilityUse(grantId, task.id, classified.spend?.amount ?? 0, now());
|
|
719
|
+
}
|
|
720
|
+
}
|
|
721
|
+
/**
|
|
722
|
+
* Settle a task whose idempotency key is committed to it. A key is
|
|
723
|
+
* committed only once every kernel check held, so whatever stopped the
|
|
724
|
+
* task came after the effect: an interruption, or a write that failed. One
|
|
725
|
+
* with a semantic question commits on entering `waiting_user`, so its
|
|
726
|
+
* commit never says the question was answered: it settles closed, as at
|
|
727
|
+
* expiry (#54).
|
|
728
|
+
*/
|
|
729
|
+
settleCommitted(task, request, classified) {
|
|
730
|
+
if (task.idempotencyKey === undefined ||
|
|
731
|
+
this.store.getIdempotencyCommit(task.idempotencyKey)?.taskId !== task.id) {
|
|
732
|
+
return undefined;
|
|
733
|
+
}
|
|
734
|
+
if (asksSemantic(request)) {
|
|
735
|
+
// Settled already, it is left as it stands rather than certified again.
|
|
736
|
+
// Asked of the stored row: a peer may have settled it since `task` was
|
|
737
|
+
// read, and its status says `recovery_required` either way.
|
|
738
|
+
const stored = this.status(task.id);
|
|
739
|
+
if (stored.error === "semantic_judgement_expired")
|
|
740
|
+
return { task: stored };
|
|
741
|
+
// Cut off before it reached `waiting_user`, maybe before its draw too.
|
|
742
|
+
this.drawGrant(task, request, classified);
|
|
743
|
+
return this.settleUnanswered(task);
|
|
744
|
+
}
|
|
745
|
+
return this.resolveAsVerified(task, request, classified, [
|
|
746
|
+
{
|
|
747
|
+
type: "idempotency",
|
|
748
|
+
passed: true,
|
|
749
|
+
strength: evidenceStrength("idempotency"),
|
|
750
|
+
summary: "verified and committed before the interruption",
|
|
751
|
+
},
|
|
752
|
+
], { recovery: "committed_before_interruption" });
|
|
753
|
+
}
|
|
569
754
|
/**
|
|
570
755
|
* With `failed`, each task that could not be recovered is recorded there by
|
|
571
756
|
* id instead of the first failure being thrown at the end.
|
|
@@ -604,17 +789,64 @@ export class TaskController {
|
|
|
604
789
|
this.store.inFlight.delete(task.id);
|
|
605
790
|
}
|
|
606
791
|
}
|
|
792
|
+
this.expireJudgements(failures);
|
|
607
793
|
if (failed === undefined && failures.size > 0)
|
|
608
794
|
throw failures.values().next().value;
|
|
609
795
|
return recovered;
|
|
610
796
|
}
|
|
797
|
+
/**
|
|
798
|
+
* A waiting task whose judgements are past their `expiresAt` settles
|
|
799
|
+
* closed. Re-read and written in one transaction, so a peer settling the
|
|
800
|
+
* same task wins once.
|
|
801
|
+
*
|
|
802
|
+
* ponytail: runs on the startup pass and the 30 s recovery sweep, so a wait
|
|
803
|
+
* can outlive its `expiresAt` by one interval, or until a runtime next opens
|
|
804
|
+
* this data directory; a verdict tool must check `expiresAt` itself.
|
|
805
|
+
*/
|
|
806
|
+
expireJudgements(failures) {
|
|
807
|
+
for (const listed of this.store.listExpiredJudgements(now())) {
|
|
808
|
+
try {
|
|
809
|
+
this.store.transaction(() => {
|
|
810
|
+
const task = this.store.getTask(listed.id);
|
|
811
|
+
if (task?.status === "waiting_user")
|
|
812
|
+
this.settleUnanswered(task);
|
|
813
|
+
});
|
|
814
|
+
}
|
|
815
|
+
catch (error) {
|
|
816
|
+
failures.set(listed.id, error);
|
|
817
|
+
}
|
|
818
|
+
}
|
|
819
|
+
}
|
|
820
|
+
/**
|
|
821
|
+
* No answer is not a yes (#54). A task whose semantic question went
|
|
822
|
+
* unanswered, at expiry or cut off after its commit, settles closed: its
|
|
823
|
+
* effect ran, so a change is `recovery_required` (never `failed`) and a
|
|
824
|
+
* read is `partial`. Nothing runs here, so it writes no receipt: the
|
|
825
|
+
* certificate, replaced, carries the evidence of the task's last one, the
|
|
826
|
+
* live path's or the one recovery wrote.
|
|
827
|
+
*/
|
|
828
|
+
settleUnanswered(task) {
|
|
829
|
+
const last = task.receiptIds[task.receiptIds.length - 1];
|
|
830
|
+
const receipt = last === undefined ? undefined : this.store.getReceipt(last);
|
|
831
|
+
const evidence = (receipt?.evidence ?? []).map((item) => item.type === "semantic"
|
|
832
|
+
? { ...item, passed: false, inconclusive: true, summary: "semantic_judgement_expired" }
|
|
833
|
+
: item);
|
|
834
|
+
task.status = task.policyDecision.effect === "read" ? "partial" : "recovery_required";
|
|
835
|
+
task.error = "semantic_judgement_expired";
|
|
836
|
+
task.updatedAt = now();
|
|
837
|
+
return { task, certificate: this.createAndSaveCertificate(task, evidence) };
|
|
838
|
+
}
|
|
611
839
|
async recoverTask(task) {
|
|
612
840
|
const request = this.loadRequest(task.id);
|
|
613
841
|
const classified = classifyOperation(request.operation);
|
|
842
|
+
// A semantic question cannot be observed again, and resolveAsVerified
|
|
843
|
+
// settles it closed; the rest must hold as for any other task, so a
|
|
844
|
+
// change with no key to commit is charged here too.
|
|
845
|
+
const observable = request.requiredEvidence.filter((predicate) => predicate.type !== "semantic");
|
|
614
846
|
if (task.status === "verifying" &&
|
|
615
847
|
classified.effect !== "read" &&
|
|
616
|
-
|
|
617
|
-
|
|
848
|
+
observable.length > 0 &&
|
|
849
|
+
observable.every((predicate) => ["file_exists", "file_absent", "file_hash"].includes(predicate.type))) {
|
|
618
850
|
// What the adapter reported before the crash, when it got that far.
|
|
619
851
|
// Evidence alone is not success: a build that exited 1 beside a dist/
|
|
620
852
|
// left by an earlier one would otherwise recover as verified.
|
|
@@ -623,14 +855,28 @@ export class TaskController {
|
|
|
623
855
|
? undefined
|
|
624
856
|
: this.payloadCipher.open(sealed, `task:${task.id}:result`);
|
|
625
857
|
const actionSucceeded = reported?.success !== false;
|
|
626
|
-
|
|
627
|
-
|
|
858
|
+
// Against what the adapter reported, as the live path verifies: a
|
|
859
|
+
// move's own result is what tells a case-only rename's source is gone.
|
|
860
|
+
// Its semantic item, never passed, rides along unanswered as on the
|
|
861
|
+
// live receipt, and is the one item not required to pass.
|
|
862
|
+
const verification = await this.verifier.verify(request.requiredEvidence, reported ?? {});
|
|
863
|
+
if (actionSucceeded &&
|
|
864
|
+
verification.evidence.every((item) => item.passed || item.type === "semantic")) {
|
|
628
865
|
this.resolveAsVerified(task, request, classified, verification.evidence, {
|
|
629
866
|
recovery: "independent_reobservation",
|
|
630
867
|
});
|
|
631
868
|
return task;
|
|
632
869
|
}
|
|
633
870
|
}
|
|
871
|
+
// A read that verified and committed its key before the interruption is
|
|
872
|
+
// done. Run again, it would only be cancelled as a duplicate of itself,
|
|
873
|
+
// and a workflow reads a cancelled step as a denial. A task with a
|
|
874
|
+
// semantic question that committed settles closed, change or read.
|
|
875
|
+
if (classified.effect === "read" || asksSemantic(request)) {
|
|
876
|
+
const committed = this.settleCommitted(task, request, classified);
|
|
877
|
+
if (committed !== undefined)
|
|
878
|
+
return committed.task;
|
|
879
|
+
}
|
|
634
880
|
const effect = classified.effect;
|
|
635
881
|
task.status = effect === "read" ? "planned" : "recovery_required";
|
|
636
882
|
task.error =
|
|
@@ -736,6 +982,31 @@ export class TaskController {
|
|
|
736
982
|
...(certificate === undefined ? {} : { certificate }),
|
|
737
983
|
};
|
|
738
984
|
}
|
|
985
|
+
/**
|
|
986
|
+
* A task's certificate, written now for a `verified_success` task whose
|
|
987
|
+
* certificate failed to write after it was recorded: a workflow counts a
|
|
988
|
+
* step done only on its certificate. For a writer of the run, never a
|
|
989
|
+
* read. Only that status, because every other path to it writes the
|
|
990
|
+
* certificate before the task, and `execute`, which does not, writes it
|
|
991
|
+
* only if none is stored: whoever writes first, both hand out the same one.
|
|
992
|
+
* A write that fails again is left for the next writer.
|
|
993
|
+
*/
|
|
994
|
+
certify(taskId) {
|
|
995
|
+
const certificate = this.store.getCertificateForTask(taskId);
|
|
996
|
+
if (certificate !== undefined)
|
|
997
|
+
return certificate;
|
|
998
|
+
const task = this.store.getTask(taskId);
|
|
999
|
+
const last = task?.receiptIds.at(-1);
|
|
1000
|
+
const receipt = last === undefined ? undefined : this.store.getReceipt(last);
|
|
1001
|
+
if (task?.status !== "verified_success" || receipt === undefined)
|
|
1002
|
+
return undefined;
|
|
1003
|
+
try {
|
|
1004
|
+
return this.createAndSaveCertificate(task, receipt.evidence, true);
|
|
1005
|
+
}
|
|
1006
|
+
catch {
|
|
1007
|
+
return undefined;
|
|
1008
|
+
}
|
|
1009
|
+
}
|
|
739
1010
|
/**
|
|
740
1011
|
* The mode a task's effect ran under: recorded when it started running, or,
|
|
741
1012
|
* for a task that started before that was recorded, the mode on its first
|
|
@@ -754,11 +1025,13 @@ export class TaskController {
|
|
|
754
1025
|
const approvalId = task.approvedBy?.approvalId ??
|
|
755
1026
|
(task.policyDecision.outcome === "confirm" ? task.approval?.approvalId : undefined);
|
|
756
1027
|
const sanitized = redactStructuredValue(result);
|
|
1028
|
+
// A verifier's summary and source are stored text like any other.
|
|
1029
|
+
const sanitizedEvidence = redactStructuredValue(evidence);
|
|
757
1030
|
const adapterRedactions = Array.isArray(result.redactions)
|
|
758
1031
|
? result.redactions.filter((item) => typeof item === "string")
|
|
759
1032
|
: [];
|
|
760
1033
|
const redactions = [
|
|
761
|
-
...new Set([...adapterRedactions, ...sanitized.redactions]),
|
|
1034
|
+
...new Set([...adapterRedactions, ...sanitized.redactions, ...sanitizedEvidence.redactions]),
|
|
762
1035
|
];
|
|
763
1036
|
return {
|
|
764
1037
|
schemaVersion: "1.0.0",
|
|
@@ -792,7 +1065,7 @@ export class TaskController {
|
|
|
792
1065
|
endedAt: now(),
|
|
793
1066
|
success,
|
|
794
1067
|
observedEffect: sanitized.value,
|
|
795
|
-
evidence,
|
|
1068
|
+
evidence: sanitizedEvidence.value,
|
|
796
1069
|
redactions,
|
|
797
1070
|
...(error === undefined ? {} : { error }),
|
|
798
1071
|
};
|
|
@@ -836,7 +1109,9 @@ export class TaskController {
|
|
|
836
1109
|
? lastError
|
|
837
1110
|
: new Error("execution_failed_without_error");
|
|
838
1111
|
}
|
|
839
|
-
createAndSaveCertificate(task, evidence
|
|
1112
|
+
createAndSaveCertificate(task, evidence,
|
|
1113
|
+
// A task's first: one another writer stored meanwhile stays, and is used.
|
|
1114
|
+
first = false) {
|
|
840
1115
|
const approvalChannel = task.approvedBy?.channel ??
|
|
841
1116
|
(task.policyDecision.outcome === "allow"
|
|
842
1117
|
? task.policyDecision.effect === "read"
|
|
@@ -853,16 +1128,19 @@ export class TaskController {
|
|
|
853
1128
|
result: certificateResult(task),
|
|
854
1129
|
policyVersion: task.policyDecision.policyVersion,
|
|
855
1130
|
receiptIds: task.receiptIds,
|
|
856
|
-
|
|
1131
|
+
// Redacted as a receipt's is, before the digest is taken. Redacting
|
|
1132
|
+
// again changes nothing, so one written from a stored receipt's items
|
|
1133
|
+
// (`certify`) says what one written from the verifier's would.
|
|
1134
|
+
evidence: redactStructuredValue(evidence).value,
|
|
857
1135
|
createdAt: now(),
|
|
858
1136
|
...(mode === undefined ? {} : { mode }),
|
|
859
1137
|
...(approvalChannel === undefined ? {} : { approvalChannel }),
|
|
860
1138
|
});
|
|
861
|
-
|
|
1139
|
+
const stored = this.store.saveCertificate(certificate, { ifAbsent: first });
|
|
1140
|
+
task.certificateId = stored.certificateId;
|
|
862
1141
|
task.updatedAt = now();
|
|
863
|
-
this.store.saveCertificate(certificate);
|
|
864
1142
|
this.store.saveTask(task);
|
|
865
|
-
return
|
|
1143
|
+
return stored;
|
|
866
1144
|
}
|
|
867
1145
|
finishWithoutReceipt(task, evidence) {
|
|
868
1146
|
const certificate = this.createAndSaveCertificate(task, evidence);
|