@agentplat/mesh-sim 0.3.0-beta.2 → 0.3.0-beta.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +447 -2
- package/dist/collective-closed-loop-action.d.ts +80 -0
- package/dist/collective-closed-loop-action.d.ts.map +1 -0
- package/dist/collective-closed-loop-action.js +471 -0
- package/dist/collective-closed-loop-action.js.map +1 -0
- package/dist/collective-closed-loop-contracts.d.ts +75 -0
- package/dist/collective-closed-loop-contracts.d.ts.map +1 -0
- package/dist/collective-closed-loop-contracts.js +420 -0
- package/dist/collective-closed-loop-contracts.js.map +1 -0
- package/dist/collective-closed-loop-fault-matrix.d.ts +101 -0
- package/dist/collective-closed-loop-fault-matrix.d.ts.map +1 -0
- package/dist/collective-closed-loop-fault-matrix.js +253 -0
- package/dist/collective-closed-loop-fault-matrix.js.map +1 -0
- package/dist/collective-closed-loop-recovery.d.ts +70 -0
- package/dist/collective-closed-loop-recovery.d.ts.map +1 -0
- package/dist/collective-closed-loop-recovery.js +823 -0
- package/dist/collective-closed-loop-recovery.js.map +1 -0
- package/dist/collective-closed-loop-reference.d.ts +54 -0
- package/dist/collective-closed-loop-reference.d.ts.map +1 -0
- package/dist/collective-closed-loop-reference.js +1696 -0
- package/dist/collective-closed-loop-reference.js.map +1 -0
- package/dist/collective-closed-loop-replanning.d.ts +27 -0
- package/dist/collective-closed-loop-replanning.d.ts.map +1 -0
- package/dist/collective-closed-loop-replanning.js +151 -0
- package/dist/collective-closed-loop-replanning.js.map +1 -0
- package/dist/collective-closed-loop-resilience-campaign.d.ts +22 -0
- package/dist/collective-closed-loop-resilience-campaign.d.ts.map +1 -0
- package/dist/collective-closed-loop-resilience-campaign.js +196 -0
- package/dist/collective-closed-loop-resilience-campaign.js.map +1 -0
- package/dist/collective-closed-loop-resilience-contracts.d.ts +138 -0
- package/dist/collective-closed-loop-resilience-contracts.d.ts.map +1 -0
- package/dist/collective-closed-loop-resilience-contracts.js +677 -0
- package/dist/collective-closed-loop-resilience-contracts.js.map +1 -0
- package/dist/collective-closed-loop-runner.d.ts +133 -0
- package/dist/collective-closed-loop-runner.d.ts.map +1 -0
- package/dist/collective-closed-loop-runner.js +1429 -0
- package/dist/collective-closed-loop-runner.js.map +1 -0
- package/dist/collective-closed-loop-runtime.d.ts +95 -0
- package/dist/collective-closed-loop-runtime.d.ts.map +1 -0
- package/dist/collective-closed-loop-runtime.js +1192 -0
- package/dist/collective-closed-loop-runtime.js.map +1 -0
- package/dist/collective-effect-provenance.d.ts +5 -0
- package/dist/collective-effect-provenance.d.ts.map +1 -0
- package/dist/collective-effect-provenance.js +32 -0
- package/dist/collective-effect-provenance.js.map +1 -0
- package/dist/collective-environment.d.ts +49 -0
- package/dist/collective-environment.d.ts.map +1 -0
- package/dist/collective-environment.js +907 -0
- package/dist/collective-environment.js.map +1 -0
- package/dist/collective-progressive-scale-overlay.d.ts +39 -0
- package/dist/collective-progressive-scale-overlay.d.ts.map +1 -0
- package/dist/collective-progressive-scale-overlay.js +147 -0
- package/dist/collective-progressive-scale-overlay.js.map +1 -0
- package/dist/collective-progressive-scale.d.ts +202 -0
- package/dist/collective-progressive-scale.d.ts.map +1 -0
- package/dist/collective-progressive-scale.js +755 -0
- package/dist/collective-progressive-scale.js.map +1 -0
- package/dist/collective-statistical-campaign-aggregation.d.ts +97 -0
- package/dist/collective-statistical-campaign-aggregation.d.ts.map +1 -0
- package/dist/collective-statistical-campaign-aggregation.js +446 -0
- package/dist/collective-statistical-campaign-aggregation.js.map +1 -0
- package/dist/collective-statistical-campaign-artifact-stream.d.ts +66 -0
- package/dist/collective-statistical-campaign-artifact-stream.d.ts.map +1 -0
- package/dist/collective-statistical-campaign-artifact-stream.js +207 -0
- package/dist/collective-statistical-campaign-artifact-stream.js.map +1 -0
- package/dist/collective-statistical-campaign-bundle.d.ts +151 -0
- package/dist/collective-statistical-campaign-bundle.d.ts.map +1 -0
- package/dist/collective-statistical-campaign-bundle.js +885 -0
- package/dist/collective-statistical-campaign-bundle.js.map +1 -0
- package/dist/collective-statistical-campaign-executor.d.ts +131 -0
- package/dist/collective-statistical-campaign-executor.d.ts.map +1 -0
- package/dist/collective-statistical-campaign-executor.js +856 -0
- package/dist/collective-statistical-campaign-executor.js.map +1 -0
- package/dist/collective-statistical-campaign-normative-analysis.d.ts +85 -0
- package/dist/collective-statistical-campaign-normative-analysis.d.ts.map +1 -0
- package/dist/collective-statistical-campaign-normative-analysis.js +483 -0
- package/dist/collective-statistical-campaign-normative-analysis.js.map +1 -0
- package/dist/collective-statistical-campaign-normative-operation.d.ts +89 -0
- package/dist/collective-statistical-campaign-normative-operation.d.ts.map +1 -0
- package/dist/collective-statistical-campaign-normative-operation.js +266 -0
- package/dist/collective-statistical-campaign-normative-operation.js.map +1 -0
- package/dist/collective-statistical-campaign-normative-registry.d.ts +24 -0
- package/dist/collective-statistical-campaign-normative-registry.d.ts.map +1 -0
- package/dist/collective-statistical-campaign-normative-registry.js +163 -0
- package/dist/collective-statistical-campaign-normative-registry.js.map +1 -0
- package/dist/collective-statistical-campaign-registered-adapter.d.ts +16 -0
- package/dist/collective-statistical-campaign-registered-adapter.d.ts.map +1 -0
- package/dist/collective-statistical-campaign-registered-adapter.js +518 -0
- package/dist/collective-statistical-campaign-registered-adapter.js.map +1 -0
- package/dist/collective-statistical-campaign-scale.d.ts +85 -0
- package/dist/collective-statistical-campaign-scale.d.ts.map +1 -0
- package/dist/collective-statistical-campaign-scale.js +349 -0
- package/dist/collective-statistical-campaign-scale.js.map +1 -0
- package/dist/collective-trace-journal.d.ts +20 -0
- package/dist/collective-trace-journal.d.ts.map +1 -0
- package/dist/collective-trace-journal.js +105 -0
- package/dist/collective-trace-journal.js.map +1 -0
- package/dist/index.d.ts +25 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +25 -0
- package/dist/index.js.map +1 -1
- package/dist/multi-domain-environment-contracts.d.ts +122 -0
- package/dist/multi-domain-environment-contracts.d.ts.map +1 -0
- package/dist/multi-domain-environment-contracts.js +14 -0
- package/dist/multi-domain-environment-contracts.js.map +1 -0
- package/dist/multi-domain-environment-runtime.d.ts +25 -0
- package/dist/multi-domain-environment-runtime.d.ts.map +1 -0
- package/dist/multi-domain-environment-runtime.js +493 -0
- package/dist/multi-domain-environment-runtime.js.map +1 -0
- package/dist/multi-domain-environment-validation.d.ts +33 -0
- package/dist/multi-domain-environment-validation.d.ts.map +1 -0
- package/dist/multi-domain-environment-validation.js +415 -0
- package/dist/multi-domain-environment-validation.js.map +1 -0
- package/dist/multi-domain-environment.d.ts +4 -0
- package/dist/multi-domain-environment.d.ts.map +1 -0
- package/dist/multi-domain-environment.js +4 -0
- package/dist/multi-domain-environment.js.map +1 -0
- package/dist/reducer-scenario.d.ts.map +1 -1
- package/dist/reducer-scenario.js +67 -19
- package/dist/reducer-scenario.js.map +1 -1
- package/dist/scalable-evaluation-contracts.d.ts +835 -0
- package/dist/scalable-evaluation-contracts.d.ts.map +1 -0
- package/dist/scalable-evaluation-contracts.js +22 -0
- package/dist/scalable-evaluation-contracts.js.map +1 -0
- package/dist/scalable-evaluation-durability.d.ts +16 -0
- package/dist/scalable-evaluation-durability.d.ts.map +1 -0
- package/dist/scalable-evaluation-durability.js +339 -0
- package/dist/scalable-evaluation-durability.js.map +1 -0
- package/dist/scalable-evaluation-evidence.d.ts +93 -0
- package/dist/scalable-evaluation-evidence.d.ts.map +1 -0
- package/dist/scalable-evaluation-evidence.js +470 -0
- package/dist/scalable-evaluation-evidence.js.map +1 -0
- package/dist/scalable-evaluation-reference-integrated.d.ts +87 -0
- package/dist/scalable-evaluation-reference-integrated.d.ts.map +1 -0
- package/dist/scalable-evaluation-reference-integrated.js +1318 -0
- package/dist/scalable-evaluation-reference-integrated.js.map +1 -0
- package/dist/scalable-evaluation-runner.d.ts +8 -0
- package/dist/scalable-evaluation-runner.d.ts.map +1 -0
- package/dist/scalable-evaluation-runner.js +1810 -0
- package/dist/scalable-evaluation-runner.js.map +1 -0
- package/dist/scalable-evaluation-runtime.d.ts +86 -0
- package/dist/scalable-evaluation-runtime.d.ts.map +1 -0
- package/dist/scalable-evaluation-runtime.js +1310 -0
- package/dist/scalable-evaluation-runtime.js.map +1 -0
- package/dist/scalable-evaluation-validation.d.ts +59 -0
- package/dist/scalable-evaluation-validation.d.ts.map +1 -0
- package/dist/scalable-evaluation-validation.js +377 -0
- package/dist/scalable-evaluation-validation.js.map +1 -0
- package/dist/scalable-evaluation.d.ts +8 -0
- package/dist/scalable-evaluation.d.ts.map +1 -0
- package/dist/scalable-evaluation.js +8 -0
- package/dist/scalable-evaluation.js.map +1 -0
- package/dist/sharded-simulation-contracts.d.ts +240 -0
- package/dist/sharded-simulation-contracts.d.ts.map +1 -0
- package/dist/sharded-simulation-contracts.js +199 -0
- package/dist/sharded-simulation-contracts.js.map +1 -0
- package/dist/sharded-simulation-runtime.d.ts +64 -0
- package/dist/sharded-simulation-runtime.d.ts.map +1 -0
- package/dist/sharded-simulation-runtime.js +755 -0
- package/dist/sharded-simulation-runtime.js.map +1 -0
- package/dist/sharded-simulation-validation.d.ts +10 -0
- package/dist/sharded-simulation-validation.d.ts.map +1 -0
- package/dist/sharded-simulation-validation.js +250 -0
- package/dist/sharded-simulation-validation.js.map +1 -0
- package/dist/sharded-simulation.d.ts +46 -0
- package/dist/sharded-simulation.d.ts.map +1 -0
- package/dist/sharded-simulation.js +32 -0
- package/dist/sharded-simulation.js.map +1 -0
- package/dist/verified-benchmark-contracts.d.ts +132 -0
- package/dist/verified-benchmark-contracts.d.ts.map +1 -0
- package/dist/verified-benchmark-contracts.js +2 -0
- package/dist/verified-benchmark-contracts.js.map +1 -0
- package/dist/verified-benchmark-runtime.d.ts +20 -0
- package/dist/verified-benchmark-runtime.d.ts.map +1 -0
- package/dist/verified-benchmark-runtime.js +199 -0
- package/dist/verified-benchmark-runtime.js.map +1 -0
- package/dist/verified-benchmark-validation.d.ts +64 -0
- package/dist/verified-benchmark-validation.d.ts.map +1 -0
- package/dist/verified-benchmark-validation.js +481 -0
- package/dist/verified-benchmark-validation.js.map +1 -0
- package/dist/verified-benchmark.d.ts +4 -0
- package/dist/verified-benchmark.d.ts.map +1 -0
- package/dist/verified-benchmark.js +4 -0
- package/dist/verified-benchmark.js.map +1 -0
- package/package.json +18 -8
|
@@ -0,0 +1,823 @@
|
|
|
1
|
+
import { createWorkContractFromMeshV1 } from "@agentplat/collective-control/mesh";
|
|
2
|
+
import { validateCollectiveProtectedEffectAttemptV1, validateCollectiveProtectedEffectReceiptV1, } from "@agentplat/collective-planning/evaluation";
|
|
3
|
+
import { createMeshAllocationInboundProcessor, createMeshAllocationInboundRuntimeState, createMeshAllocationRuntimeState, evaluateMeshAllocationCommand, evaluateMeshAllocationTimer, evaluateVerifiedMeshAllocationEnvelope, restoreMeshDiscoveryState, } from "@agentplat/mesh/coordination";
|
|
4
|
+
import { verifyMeshEnvelope } from "@agentplat/mesh-crypto";
|
|
5
|
+
import { canonicalizeMeshPayload, validateSignedMeshEnvelope, } from "@agentplat/mesh-protocol";
|
|
6
|
+
import { assertCollectiveEffectReceiptProvenanceV1 } from "./collective-effect-provenance.js";
|
|
7
|
+
// Recovery is part of the same registered closed-loop runtime as the
|
|
8
|
+
// definition and execution contracts. Keep its admission bound aligned with
|
|
9
|
+
// their public 500-peer ceiling so a declared large-scale resilient scenario
|
|
10
|
+
// cannot fail only when the recovery path is exercised.
|
|
11
|
+
const MAXIMUM_RECOVERY_ROSTER_SIZE = 500;
|
|
12
|
+
const WIRE_VERSION = 0;
|
|
13
|
+
const RECOVERY_LEASE_MS = 120_000;
|
|
14
|
+
const RECOVERY_ENVELOPE_TTL_MS = 59_000;
|
|
15
|
+
/**
|
|
16
|
+
* Applies a real, certificate-backed reassignment to an Increment 5
|
|
17
|
+
* pre-effect checkpoint. Every state change below passes through the public
|
|
18
|
+
* Mesh allocation reducers; this module only signs and routes the records.
|
|
19
|
+
*/
|
|
20
|
+
export async function recoverCollectiveClosedLoopAssignmentV1(rawInput) {
|
|
21
|
+
const input = validateInput(rawInput);
|
|
22
|
+
const context = {
|
|
23
|
+
input,
|
|
24
|
+
peersById: new Map(input.peers.map((peer) => [peer.peerId, peer])),
|
|
25
|
+
sequences: new Map(),
|
|
26
|
+
nextMessage: 1,
|
|
27
|
+
interactionCount: input.preEffect.interactionCount,
|
|
28
|
+
};
|
|
29
|
+
let states = {
|
|
30
|
+
...input.preEffect.meshStates,
|
|
31
|
+
};
|
|
32
|
+
// Admission is local transport configuration, not assignment authority. The
|
|
33
|
+
// nominal star topology only admits owner<->peer traffic, while certificate
|
|
34
|
+
// witnesses must receive signed copies from the original assignee. Expand
|
|
35
|
+
// the already configured in-memory admission set to the closed peer roster;
|
|
36
|
+
// every subsequent authority transition still goes through Mesh reducers.
|
|
37
|
+
states = extendRecoveryAdmission(context, states);
|
|
38
|
+
const ownerPeerId = input.preEffect.workContract.assignment.ownerPeerId;
|
|
39
|
+
const failed = input.preEffect.workContract.assignment.assignedPeerId;
|
|
40
|
+
if (failed !== input.failedWinnerPeerId)
|
|
41
|
+
throw new TypeError("closed_loop_recovery_failed_winner_mismatch");
|
|
42
|
+
const owner = peer(context, ownerPeerId);
|
|
43
|
+
const replacement = peer(context, input.replacementPeerId);
|
|
44
|
+
if (replacement.peerId === failed || replacement.peerId === owner.peerId)
|
|
45
|
+
throw new TypeError("closed_loop_recovery_replacement_invalid");
|
|
46
|
+
const oldAward = states[failed]?.allocation.receivedAwards[input.preEffect.execution.awardId]
|
|
47
|
+
?.envelope;
|
|
48
|
+
const oldAcceptance = states[failed]?.allocation.localAssignmentResponses[input.preEffect.execution.awardId]?.envelope;
|
|
49
|
+
const oldCheckpoint = input.preEffect.checkpoint.envelope;
|
|
50
|
+
if (!oldAward || !oldAcceptance || !oldCheckpoint)
|
|
51
|
+
throw new Error("closed_loop_recovery_original_evidence_missing");
|
|
52
|
+
const objective = states[owner.peerId]?.objectives.objectives[input.missionIntent.objective.objectiveId];
|
|
53
|
+
if (!objective)
|
|
54
|
+
throw new Error("closed_loop_recovery_objective_missing");
|
|
55
|
+
const witnesses = objective.recoveryWitnessPeerIds.filter((peerId) => peerId !== failed && context.peersById.has(peerId));
|
|
56
|
+
if (witnesses.length < objective.recoveryWitnessThreshold)
|
|
57
|
+
throw new Error("closed_loop_recovery_witness_quorum_unavailable");
|
|
58
|
+
// A witness needs exact direct copies, not a reconstructed authority. The
|
|
59
|
+
// source records remain signed by their original issuer and are routed again
|
|
60
|
+
// to each configured in-mesh witness.
|
|
61
|
+
// The selected replacement also needs the original signed evidence. It may
|
|
62
|
+
// be an eligible bidder without being part of the voting witness quorum.
|
|
63
|
+
// Preserve direct copies rather than reconstructing authority from state.
|
|
64
|
+
const evidenceRecipients = [...new Set([...witnesses, replacement.peerId])]
|
|
65
|
+
.filter((peerId) => peerId !== failed)
|
|
66
|
+
.sort();
|
|
67
|
+
for (const recipientPeerId of evidenceRecipients) {
|
|
68
|
+
const awardCopy = await copyForWitness(context, oldAward, recipientPeerId);
|
|
69
|
+
states = await deliver(context, states, recipientPeerId, awardCopy, input.preEffect.logicalTimeMs + 1, oldAward.sentAt);
|
|
70
|
+
const acceptanceCopy = await copyForWitness(context, oldAcceptance, recipientPeerId, awardCopy.messageId);
|
|
71
|
+
states = await deliver(context, states, recipientPeerId, acceptanceCopy, input.preEffect.logicalTimeMs + 1, oldAcceptance.sentAt);
|
|
72
|
+
const checkpointCopy = await copyForWitness(context, oldCheckpoint, recipientPeerId, acceptanceCopy.messageId);
|
|
73
|
+
states = await deliver(context, states, recipientPeerId, checkpointCopy, input.preEffect.logicalTimeMs + 1, oldCheckpoint.sentAt);
|
|
74
|
+
}
|
|
75
|
+
const recoveryTime = expireLeases(states, [owner.peerId, failed, ...witnesses], input.faultLogicalTimeMs);
|
|
76
|
+
states = recoveryTime.states;
|
|
77
|
+
let logicalTimeMs = recoveryTime.logicalTimeMs;
|
|
78
|
+
const oldLease = states[failed]?.allocation.leaseHeads[input.preEffect.execution.executionScopeKey];
|
|
79
|
+
if (!oldLease || oldLease.status !== "expired")
|
|
80
|
+
throw new Error("closed_loop_recovery_expiry_missing");
|
|
81
|
+
const recoveryAt = Math.max(logicalTimeMs, oldLease.currentLeaseExpiresAtLogical + objective.recoveryGraceMs);
|
|
82
|
+
logicalTimeMs = recoveryAt;
|
|
83
|
+
const proposalId = "recovery:takeover:1";
|
|
84
|
+
const replacementWitness = Object.values(states[replacement.peerId].allocation.witnessAssignments).find((assignment) => assignment.leaseHead?.status === "expired");
|
|
85
|
+
const proposalCausationId = replacementWitness?.acceptanceEnvelope?.messageId;
|
|
86
|
+
if (!proposalCausationId)
|
|
87
|
+
throw new Error("closed_loop_recovery_replacement_witness_missing");
|
|
88
|
+
const proposalPayload = {
|
|
89
|
+
type: "lease.takeover_proposal",
|
|
90
|
+
takeoverProposalId: proposalId,
|
|
91
|
+
proposalAuthority: "candidate",
|
|
92
|
+
proposerPeerId: replacement.peerId,
|
|
93
|
+
proposedAssigneePeerId: replacement.peerId,
|
|
94
|
+
proposedAssignmentEpoch: input.preEffect.execution.assignmentEpoch + 1,
|
|
95
|
+
objectiveId: input.preEffect.execution.objectiveId,
|
|
96
|
+
objectiveDocumentId: input.preEffect.execution.objectiveDocumentId,
|
|
97
|
+
objectiveRevision: input.preEffect.execution.objectiveRevision,
|
|
98
|
+
workItemId: input.preEffect.execution.workItemId,
|
|
99
|
+
workItemRevision: input.preEffect.execution.workItemRevision,
|
|
100
|
+
ownerPeerId,
|
|
101
|
+
ownerEpoch: input.preEffect.execution.ownerEpoch,
|
|
102
|
+
assigneePeerId: failed,
|
|
103
|
+
awardId: input.preEffect.execution.awardId,
|
|
104
|
+
acceptanceId: input.preEffect.execution.acceptanceId,
|
|
105
|
+
assignmentEpoch: input.preEffect.execution.assignmentEpoch,
|
|
106
|
+
assignmentAuthorityId: input.preEffect.execution.assignmentAuthorityId,
|
|
107
|
+
fencingToken: input.preEffect.execution.fencingToken,
|
|
108
|
+
leaseExpiresAt: oldLease.currentLeaseExpiresAt,
|
|
109
|
+
leaseRenewalSequence: oldLease.leaseRenewalSequence,
|
|
110
|
+
...(oldLease.latestLeaseRenewalId === undefined
|
|
111
|
+
? {}
|
|
112
|
+
: { latestLeaseRenewalId: oldLease.latestLeaseRenewalId }),
|
|
113
|
+
};
|
|
114
|
+
const proposalRecipients = await recoveryRecipients(context, replacement.peerId, proposalPayload, proposalCausationId, logicalTimeMs);
|
|
115
|
+
states = applyLocalRecovery(context, states, replacement.peerId, proposalRecipients, logicalTimeMs);
|
|
116
|
+
states = await deliverRecoveryEffects(context, states, replacement.peerId, proposalRecipients, logicalTimeMs);
|
|
117
|
+
const voteIds = [];
|
|
118
|
+
for (const witnessPeerId of witnesses.sort()) {
|
|
119
|
+
const voteId = `recovery:vote:${voteIds.length + 1}`;
|
|
120
|
+
voteIds.push(voteId);
|
|
121
|
+
logicalTimeMs += 1;
|
|
122
|
+
const proposalForWitness = states[witnessPeerId]?.allocation.takeoverProposals[proposalId]?.envelope;
|
|
123
|
+
if (!proposalForWitness)
|
|
124
|
+
throw new Error("closed_loop_recovery_witness_proposal_missing");
|
|
125
|
+
const votePayload = {
|
|
126
|
+
type: "lease.vote",
|
|
127
|
+
leaseVoteId: voteId,
|
|
128
|
+
takeoverProposalId: proposalId,
|
|
129
|
+
witnessPeerId,
|
|
130
|
+
objectiveId: input.preEffect.execution.objectiveId,
|
|
131
|
+
};
|
|
132
|
+
const recipients = await recoveryRecipients(context, witnessPeerId, votePayload, proposalForWitness.messageId, logicalTimeMs);
|
|
133
|
+
states = applyLocalRecovery(context, states, witnessPeerId, recipients, logicalTimeMs);
|
|
134
|
+
states = await deliverRecoveryEffects(context, states, witnessPeerId, recipients, logicalTimeMs);
|
|
135
|
+
}
|
|
136
|
+
logicalTimeMs += 1;
|
|
137
|
+
const certificateId = "recovery:certificate:1";
|
|
138
|
+
const ownerProposal = states[owner.peerId]?.allocation.takeoverProposals[proposalId]?.envelope;
|
|
139
|
+
if (!ownerProposal)
|
|
140
|
+
throw new Error("closed_loop_recovery_owner_proposal_missing");
|
|
141
|
+
const certificatePayload = {
|
|
142
|
+
type: "lease.certificate",
|
|
143
|
+
certificateId,
|
|
144
|
+
certificateAssemblerPeerId: owner.peerId,
|
|
145
|
+
takeoverProposalId: proposalId,
|
|
146
|
+
leaseVoteIds: [...voteIds].sort(),
|
|
147
|
+
objectiveId: input.preEffect.execution.objectiveId,
|
|
148
|
+
};
|
|
149
|
+
const certificateRecipients = await recoveryRecipients(context, owner.peerId, certificatePayload, ownerProposal.messageId, logicalTimeMs);
|
|
150
|
+
states = applyLocalRecovery(context, states, owner.peerId, certificateRecipients, logicalTimeMs);
|
|
151
|
+
states = await deliverRecoveryEffects(context, states, owner.peerId, certificateRecipients, logicalTimeMs);
|
|
152
|
+
logicalTimeMs += 1;
|
|
153
|
+
const ownerState = states[owner.peerId];
|
|
154
|
+
const replacementBid = Object.values(ownerState.allocation.bidHeads).find((bid) => bid.bidderPeerId === replacement.peerId);
|
|
155
|
+
if (!replacementBid)
|
|
156
|
+
throw new Error("closed_loop_recovery_replacement_bid_missing");
|
|
157
|
+
const ownerOffer = ownerState.allocation.localOffers[replacementBid.offerId];
|
|
158
|
+
if (!ownerOffer)
|
|
159
|
+
throw new Error("closed_loop_recovery_offer_missing");
|
|
160
|
+
const certificateForReplacement = ownerState.allocation.recoveryCertificates[certificateId]
|
|
161
|
+
?.recipientEnvelopes?.[replacement.peerId];
|
|
162
|
+
if (!certificateForReplacement)
|
|
163
|
+
throw new Error("closed_loop_recovery_certificate_delivery_missing");
|
|
164
|
+
const recoveryAwardId = "recovery:award:1";
|
|
165
|
+
const awardSentAt = wallTime(input.missionIntent, logicalTimeMs);
|
|
166
|
+
const leaseExpiresAt = earlierTimestamp(input.preEffect.workContract.assignment.workDeadline, wallTime(input.missionIntent, logicalTimeMs + RECOVERY_LEASE_MS));
|
|
167
|
+
const acceptanceDeadline = wallTime(input.missionIntent, logicalTimeMs + 30_000);
|
|
168
|
+
const award = await signEnvelope(context, owner.peerId, "work.award", {
|
|
169
|
+
type: "work.award",
|
|
170
|
+
awardId: recoveryAwardId,
|
|
171
|
+
offerId: ownerOffer.offerId,
|
|
172
|
+
bidId: replacementBid.bidId,
|
|
173
|
+
bidRevision: replacementBid.bidRevision,
|
|
174
|
+
objectiveId: input.preEffect.execution.objectiveId,
|
|
175
|
+
objectiveDocumentId: input.preEffect.execution.objectiveDocumentId,
|
|
176
|
+
objectiveRevision: input.preEffect.execution.objectiveRevision,
|
|
177
|
+
workItemId: input.preEffect.execution.workItemId,
|
|
178
|
+
workItemRevision: input.preEffect.execution.workItemRevision,
|
|
179
|
+
ownerPeerId,
|
|
180
|
+
ownerEpoch: input.preEffect.execution.ownerEpoch,
|
|
181
|
+
offerAttempt: ownerOffer.offerAttempt,
|
|
182
|
+
assigneePeerId: replacement.peerId,
|
|
183
|
+
assignmentEpoch: input.preEffect.execution.assignmentEpoch + 1,
|
|
184
|
+
authorityKind: "recovery_certificate",
|
|
185
|
+
recoveryCertificateId: certificateId,
|
|
186
|
+
assignmentAuthorityId: certificateId,
|
|
187
|
+
fencingToken: certificateId,
|
|
188
|
+
budgetReservationUnits: ownerOffer.work.budgetReservationUnits,
|
|
189
|
+
workDeadline: ownerOffer.work.workDeadline,
|
|
190
|
+
resumeCheckpointId: input.preEffect.checkpoint.recordId,
|
|
191
|
+
leaseStartsAt: awardSentAt,
|
|
192
|
+
leaseExpiresAt,
|
|
193
|
+
acceptanceDeadline,
|
|
194
|
+
}, replacement.peerId, certificateForReplacement.messageId, {
|
|
195
|
+
sentAt: awardSentAt,
|
|
196
|
+
expiresAt: acceptanceDeadline,
|
|
197
|
+
});
|
|
198
|
+
const awarded = evaluateMeshAllocationCommand(allocationRuntime(ownerState), {
|
|
199
|
+
kind: "allocation.recovery_award",
|
|
200
|
+
certificateId,
|
|
201
|
+
recipient: {
|
|
202
|
+
recipientPeerId: replacement.peerId,
|
|
203
|
+
preparedAt: logicalTimeMs,
|
|
204
|
+
envelope: award,
|
|
205
|
+
},
|
|
206
|
+
}, recoveryVerifiedAt(input.missionIntent, logicalTimeMs), logicalTimeMs);
|
|
207
|
+
states = acceptLocal(states, owner.peerId, awarded, "closed_loop_recovery_award_rejected");
|
|
208
|
+
states = await deliver(context, states, replacement.peerId, award, logicalTimeMs);
|
|
209
|
+
logicalTimeMs += 1;
|
|
210
|
+
const recoveryAcceptanceId = "recovery:acceptance:1";
|
|
211
|
+
const acceptanceSentAt = wallTime(input.missionIntent, logicalTimeMs);
|
|
212
|
+
const acceptance = await signEnvelope(context, replacement.peerId, "work.accept", {
|
|
213
|
+
type: "work.accept",
|
|
214
|
+
acceptanceId: recoveryAcceptanceId,
|
|
215
|
+
awardId: recoveryAwardId,
|
|
216
|
+
objectiveId: input.preEffect.execution.objectiveId,
|
|
217
|
+
objectiveDocumentId: input.preEffect.execution.objectiveDocumentId,
|
|
218
|
+
objectiveRevision: input.preEffect.execution.objectiveRevision,
|
|
219
|
+
workItemId: input.preEffect.execution.workItemId,
|
|
220
|
+
workItemRevision: input.preEffect.execution.workItemRevision,
|
|
221
|
+
ownerPeerId,
|
|
222
|
+
ownerEpoch: input.preEffect.execution.ownerEpoch,
|
|
223
|
+
assigneePeerId: replacement.peerId,
|
|
224
|
+
assignmentEpoch: input.preEffect.execution.assignmentEpoch + 1,
|
|
225
|
+
assignmentAuthorityId: certificateId,
|
|
226
|
+
fencingToken: certificateId,
|
|
227
|
+
acceptanceDeadline: award.payload.acceptanceDeadline,
|
|
228
|
+
}, owner.peerId, award.messageId, {
|
|
229
|
+
sentAt: acceptanceSentAt,
|
|
230
|
+
expiresAt: acceptanceDeadline,
|
|
231
|
+
});
|
|
232
|
+
const accepted = evaluateMeshAllocationCommand(allocationRuntime(states[replacement.peerId]), {
|
|
233
|
+
kind: "allocation.assignment_response",
|
|
234
|
+
awardId: recoveryAwardId,
|
|
235
|
+
preparedAt: logicalTimeMs,
|
|
236
|
+
envelope: acceptance,
|
|
237
|
+
}, recoveryVerifiedAt(input.missionIntent, logicalTimeMs), logicalTimeMs);
|
|
238
|
+
states = acceptLocal(states, replacement.peerId, accepted, "closed_loop_recovery_acceptance_rejected");
|
|
239
|
+
states = await deliver(context, states, owner.peerId, acceptance, logicalTimeMs);
|
|
240
|
+
logicalTimeMs += 1;
|
|
241
|
+
const resumedCheckpointId = "recovery:checkpoint:2";
|
|
242
|
+
const checkpointSentAt = wallTime(input.missionIntent, logicalTimeMs);
|
|
243
|
+
const checkpoint = await signEnvelope(context, replacement.peerId, "work.checkpoint", {
|
|
244
|
+
type: "work.checkpoint",
|
|
245
|
+
checkpointId: resumedCheckpointId,
|
|
246
|
+
checkpointSequence: 2,
|
|
247
|
+
previousCheckpointId: input.preEffect.checkpoint.recordId,
|
|
248
|
+
objectiveId: input.preEffect.execution.objectiveId,
|
|
249
|
+
objectiveDocumentId: input.preEffect.execution.objectiveDocumentId,
|
|
250
|
+
objectiveRevision: input.preEffect.execution.objectiveRevision,
|
|
251
|
+
workItemId: input.preEffect.execution.workItemId,
|
|
252
|
+
workItemRevision: input.preEffect.execution.workItemRevision,
|
|
253
|
+
ownerPeerId,
|
|
254
|
+
ownerEpoch: input.preEffect.execution.ownerEpoch,
|
|
255
|
+
assigneePeerId: replacement.peerId,
|
|
256
|
+
awardId: recoveryAwardId,
|
|
257
|
+
acceptanceId: recoveryAcceptanceId,
|
|
258
|
+
assignmentEpoch: input.preEffect.execution.assignmentEpoch + 1,
|
|
259
|
+
assignmentAuthorityId: certificateId,
|
|
260
|
+
fencingToken: certificateId,
|
|
261
|
+
leaseExpiresAt,
|
|
262
|
+
checkpointDigest: "sha256:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
|
|
263
|
+
checkpointSummary: "Recovered execution checkpoint.",
|
|
264
|
+
}, owner.peerId, acceptance.messageId, {
|
|
265
|
+
sentAt: checkpointSentAt,
|
|
266
|
+
expiresAt: wallTime(input.missionIntent, logicalTimeMs + RECOVERY_ENVELOPE_TTL_MS),
|
|
267
|
+
});
|
|
268
|
+
const checkpointed = evaluateMeshAllocationCommand(allocationRuntime(states[replacement.peerId]), {
|
|
269
|
+
kind: "allocation.execution",
|
|
270
|
+
preparedAt: logicalTimeMs,
|
|
271
|
+
envelope: checkpoint,
|
|
272
|
+
}, recoveryVerifiedAt(input.missionIntent, logicalTimeMs), logicalTimeMs);
|
|
273
|
+
states = acceptLocal(states, replacement.peerId, checkpointed, "closed_loop_recovery_checkpoint_rejected");
|
|
274
|
+
states = await deliver(context, states, owner.peerId, checkpoint, logicalTimeMs);
|
|
275
|
+
const current = currentAuthority(states, owner.peerId, replacement.peerId, resumedCheckpointId);
|
|
276
|
+
const workContract = createWorkContractFromMeshV1({
|
|
277
|
+
workContractId: "recovery:work-contract:2",
|
|
278
|
+
identity: identity(input.missionIntent, replacement),
|
|
279
|
+
objective: current.objective,
|
|
280
|
+
workItem: current.workItem,
|
|
281
|
+
execution: current.execution,
|
|
282
|
+
fenceHead: current.fenceHead,
|
|
283
|
+
mandate: input.mandate,
|
|
284
|
+
roleKey: input.preEffect.workContract.roleKey,
|
|
285
|
+
trustPolicyId: input.preEffect.workContract.trustPolicyId,
|
|
286
|
+
inferencePolicyId: input.preEffect.workContract.inferencePolicyId,
|
|
287
|
+
maximumActionBudgetUnits: input.preEffect.workContract.maximumActionBudgetUnits,
|
|
288
|
+
createdAtLogicalMs: logicalTimeMs,
|
|
289
|
+
});
|
|
290
|
+
const staleRejections = await rejectStaleEpochOneRecords(context, states, owner.peerId, failed, oldLease.currentLeaseExpiresAt, input.preEffect, logicalTimeMs + 1);
|
|
291
|
+
if (!staleRejections.every((item) => item.rejectionCode === "execution_authority_invalid"))
|
|
292
|
+
throw new Error("closed_loop_recovery_stale_fencing_failed");
|
|
293
|
+
let finalization;
|
|
294
|
+
let finalizationBinding;
|
|
295
|
+
const recovery = Object.freeze({
|
|
296
|
+
schemaVersion: 1,
|
|
297
|
+
failedWinnerPeerId: failed,
|
|
298
|
+
replacementPeerId: replacement.peerId,
|
|
299
|
+
recoveryLogicalTimeMs: logicalTimeMs,
|
|
300
|
+
leaseStartsAt: awardSentAt,
|
|
301
|
+
takeoverProposalId: proposalId,
|
|
302
|
+
leaseVoteIds: Object.freeze([...voteIds].sort()),
|
|
303
|
+
certificateId,
|
|
304
|
+
recoveryAwardId,
|
|
305
|
+
recoveryAcceptanceId,
|
|
306
|
+
resumedCheckpointId,
|
|
307
|
+
workContract,
|
|
308
|
+
execution: current.execution,
|
|
309
|
+
fenceHead: current.fenceHead,
|
|
310
|
+
checkpoint: current.checkpoint,
|
|
311
|
+
staleRejectionCodes: Object.freeze(staleRejections.map((item) => item.rejectionCode)),
|
|
312
|
+
staleRejections: Object.freeze(staleRejections),
|
|
313
|
+
meshStates: Object.freeze({ ...states }),
|
|
314
|
+
async finalizeAfterCommittedEffect(finalizeInput) {
|
|
315
|
+
const attempt = validateCollectiveProtectedEffectAttemptV1(finalizeInput.effectAttempt);
|
|
316
|
+
const rawReceipt = finalizeInput.effectReceipt;
|
|
317
|
+
const receipt = validateCollectiveProtectedEffectReceiptV1(rawReceipt);
|
|
318
|
+
assertRecoveredEffectBound(attempt, receipt, workContract);
|
|
319
|
+
// The original evaluator binding is retained by preEffect. Recovery does
|
|
320
|
+
// not create an alternate effect path or a second receipt authority.
|
|
321
|
+
assertCollectiveEffectReceiptProvenanceV1(input.preEffect, attempt, rawReceipt);
|
|
322
|
+
if (receipt.status !== "committed" ||
|
|
323
|
+
receipt.outputDigest !== finalizeInput.resultDigest)
|
|
324
|
+
throw new Error("closed_loop_effect_not_committed");
|
|
325
|
+
const binding = `${attempt.attemptDigest}:${receipt.receiptDigest}:${finalizeInput.resultDigest}:${finalizeInput.resultSummary}`;
|
|
326
|
+
if (finalization !== undefined) {
|
|
327
|
+
if (binding !== finalizationBinding)
|
|
328
|
+
throw new Error("closed_loop_finalization_conflict");
|
|
329
|
+
return finalization;
|
|
330
|
+
}
|
|
331
|
+
finalizationBinding = binding;
|
|
332
|
+
finalization = finalizeRecoveredResult({
|
|
333
|
+
context,
|
|
334
|
+
states,
|
|
335
|
+
ownerPeerId: owner.peerId,
|
|
336
|
+
replacementPeerId: replacement.peerId,
|
|
337
|
+
workContract,
|
|
338
|
+
checkpoint: current.checkpoint,
|
|
339
|
+
recoveryAwardId,
|
|
340
|
+
recoveryAcceptanceId,
|
|
341
|
+
leaseExpiresAt,
|
|
342
|
+
resultDigest: finalizeInput.resultDigest,
|
|
343
|
+
resultSummary: finalizeInput.resultSummary,
|
|
344
|
+
logicalTimeMs,
|
|
345
|
+
});
|
|
346
|
+
return finalization;
|
|
347
|
+
},
|
|
348
|
+
});
|
|
349
|
+
return recovery;
|
|
350
|
+
}
|
|
351
|
+
async function finalizeRecoveredResult(input) {
|
|
352
|
+
const logicalTimeMs = input.logicalTimeMs + 1;
|
|
353
|
+
const resultId = "recovery:result:2";
|
|
354
|
+
const resultEnvelope = await signEnvelope(input.context, input.replacementPeerId, "work.result", {
|
|
355
|
+
type: "work.result",
|
|
356
|
+
resultId,
|
|
357
|
+
resultDigest: meshContentDigest(input.resultDigest),
|
|
358
|
+
objectiveId: input.workContract.objective.objectiveId,
|
|
359
|
+
objectiveDocumentId: input.workContract.objective.objectiveDocumentId,
|
|
360
|
+
objectiveRevision: input.workContract.objective.objectiveRevision,
|
|
361
|
+
workItemId: input.workContract.assignment.workItemId,
|
|
362
|
+
workItemRevision: input.workContract.assignment.workItemRevision,
|
|
363
|
+
ownerPeerId: input.workContract.assignment.ownerPeerId,
|
|
364
|
+
ownerEpoch: 1,
|
|
365
|
+
assigneePeerId: input.replacementPeerId,
|
|
366
|
+
awardId: input.recoveryAwardId,
|
|
367
|
+
acceptanceId: input.recoveryAcceptanceId,
|
|
368
|
+
assignmentEpoch: input.workContract.assignment.assignmentEpoch,
|
|
369
|
+
assignmentAuthorityId: input.workContract.assignment.assignmentAuthorityId,
|
|
370
|
+
fencingToken: input.workContract.assignment.fencingToken,
|
|
371
|
+
leaseExpiresAt: input.leaseExpiresAt,
|
|
372
|
+
checkpointId: input.checkpoint.recordId,
|
|
373
|
+
resultSummary: input.resultSummary,
|
|
374
|
+
}, input.ownerPeerId, input.checkpoint.envelope.messageId, {
|
|
375
|
+
sentAt: wallTime(input.context.input.missionIntent, logicalTimeMs),
|
|
376
|
+
expiresAt: wallTime(input.context.input.missionIntent, logicalTimeMs + RECOVERY_ENVELOPE_TTL_MS),
|
|
377
|
+
});
|
|
378
|
+
const local = evaluateMeshAllocationCommand(allocationRuntime(input.states[input.replacementPeerId]), {
|
|
379
|
+
kind: "allocation.execution",
|
|
380
|
+
preparedAt: logicalTimeMs,
|
|
381
|
+
envelope: resultEnvelope,
|
|
382
|
+
}, recoveryVerifiedAt(input.context.input.missionIntent, logicalTimeMs), logicalTimeMs);
|
|
383
|
+
let states = acceptLocal({ ...input.states }, input.replacementPeerId, local, "closed_loop_recovery_result_rejected");
|
|
384
|
+
states = await deliver(input.context, states, input.ownerPeerId, resultEnvelope, logicalTimeMs);
|
|
385
|
+
const result = states[input.replacementPeerId].allocation.executionRecords[resultId];
|
|
386
|
+
if (!result || result.recordType !== "result")
|
|
387
|
+
throw new Error("closed_loop_recovery_result_evidence_missing");
|
|
388
|
+
return Object.freeze({
|
|
389
|
+
schemaVersion: 1,
|
|
390
|
+
winnerPeerId: input.replacementPeerId,
|
|
391
|
+
workContract: input.workContract,
|
|
392
|
+
result,
|
|
393
|
+
meshStates: Object.freeze(states),
|
|
394
|
+
logicalTimeMs,
|
|
395
|
+
interactionCount: input.context.interactionCount,
|
|
396
|
+
});
|
|
397
|
+
}
|
|
398
|
+
function assertRecoveredEffectBound(attempt, receipt, workContract) {
|
|
399
|
+
if (receipt.attemptDigest !== attempt.attemptDigest ||
|
|
400
|
+
receipt.idempotencyKey !== attempt.idempotencyKey ||
|
|
401
|
+
attempt.tenantId !== workContract.tenantId ||
|
|
402
|
+
attempt.peerId !== workContract.assignment.assignedPeerId ||
|
|
403
|
+
attempt.peerInstanceId !== workContract.assignment.assignedInstanceId ||
|
|
404
|
+
attempt.workItemId !== workContract.assignment.workItemId ||
|
|
405
|
+
attempt.workItemRevision !== workContract.assignment.workItemRevision ||
|
|
406
|
+
attempt.workContractId !== workContract.workContractId ||
|
|
407
|
+
attempt.workContractDigest !== workContract.workContractDigest ||
|
|
408
|
+
attempt.assignmentEpoch !== workContract.assignment.assignmentEpoch ||
|
|
409
|
+
attempt.authorityGeneration !==
|
|
410
|
+
workContract.assignment.authorityGeneration ||
|
|
411
|
+
attempt.fencingToken !== workContract.assignment.fencingToken)
|
|
412
|
+
throw new Error("closed_loop_effect_binding_mismatch");
|
|
413
|
+
}
|
|
414
|
+
function validateInput(value) {
|
|
415
|
+
if (value?.schemaVersion !== 1 ||
|
|
416
|
+
!value.preEffect ||
|
|
417
|
+
!Array.isArray(value.peers) ||
|
|
418
|
+
!value.runner ||
|
|
419
|
+
!value.missionIntent ||
|
|
420
|
+
!value.mandate ||
|
|
421
|
+
typeof value.failedWinnerPeerId !== "string" ||
|
|
422
|
+
typeof value.replacementPeerId !== "string" ||
|
|
423
|
+
!Number.isSafeInteger(value.faultLogicalTimeMs) ||
|
|
424
|
+
value.faultLogicalTimeMs < 0)
|
|
425
|
+
throw new TypeError("closed_loop_recovery_input_invalid");
|
|
426
|
+
if (value.peers.length < 3 || value.peers.length > MAXIMUM_RECOVERY_ROSTER_SIZE)
|
|
427
|
+
throw new TypeError("closed_loop_recovery_roster_size_invalid");
|
|
428
|
+
const peersById = new Map(value.peers.map((peer) => [peer.peerId, peer]));
|
|
429
|
+
if (peersById.size !== value.peers.length ||
|
|
430
|
+
value.peers.some((peer) => peer.schemaVersion !== 1 ||
|
|
431
|
+
typeof peer.peerId !== "string" ||
|
|
432
|
+
peer.peerId.length === 0 ||
|
|
433
|
+
typeof peer.peerInstanceId !== "string" ||
|
|
434
|
+
peer.peerInstanceId.length === 0))
|
|
435
|
+
throw new TypeError("closed_loop_recovery_roster_invalid");
|
|
436
|
+
const contract = value.preEffect.workContract;
|
|
437
|
+
if (value.missionIntent.tenantId !== contract.tenantId ||
|
|
438
|
+
value.missionIntent.policyDomainId !== contract.policyDomainId ||
|
|
439
|
+
value.missionIntent.mandateDigest !== value.mandate.mandateDigest ||
|
|
440
|
+
contract.mandate.mandateDigest !== value.mandate.mandateDigest ||
|
|
441
|
+
value.missionIntent.objective.meshId !== contract.objective.meshId ||
|
|
442
|
+
value.missionIntent.objective.objectiveId !==
|
|
443
|
+
contract.objective.objectiveId ||
|
|
444
|
+
value.missionIntent.objective.objectiveRevision !==
|
|
445
|
+
contract.objective.objectiveRevision ||
|
|
446
|
+
value.missionIntent.missionIntentId !==
|
|
447
|
+
value.preEffect.roleBinding.missionIntentId ||
|
|
448
|
+
value.missionIntent.revision !==
|
|
449
|
+
value.preEffect.roleBinding.intentRevision ||
|
|
450
|
+
value.missionIntent.intentDigest !==
|
|
451
|
+
value.preEffect.roleBinding.intentDigest ||
|
|
452
|
+
value.failedWinnerPeerId !== value.preEffect.winnerPeerId ||
|
|
453
|
+
value.failedWinnerPeerId !== contract.assignment.assignedPeerId ||
|
|
454
|
+
value.replacementPeerId === value.failedWinnerPeerId ||
|
|
455
|
+
value.preEffect.execution.assigneePeerId !==
|
|
456
|
+
contract.assignment.assignedPeerId ||
|
|
457
|
+
value.preEffect.execution.assignmentEpoch !==
|
|
458
|
+
contract.assignment.assignmentEpoch ||
|
|
459
|
+
value.preEffect.execution.fencingToken !==
|
|
460
|
+
contract.assignment.fencingToken ||
|
|
461
|
+
!peersById.has(contract.assignment.ownerPeerId) ||
|
|
462
|
+
!peersById.has(contract.assignment.assignedPeerId) ||
|
|
463
|
+
!peersById.has(value.replacementPeerId) ||
|
|
464
|
+
!peersById.has(value.failedWinnerPeerId) ||
|
|
465
|
+
!Object.hasOwn(value.runner.privateKeys, contract.assignment.ownerPeerId) ||
|
|
466
|
+
!Object.hasOwn(value.runner.privateKeys, contract.assignment.assignedPeerId) ||
|
|
467
|
+
!Object.hasOwn(value.runner.privateKeys, value.replacementPeerId) ||
|
|
468
|
+
value.peers.some((peer) => !Object.hasOwn(value.runner.privateKeys, peer.peerId)) ||
|
|
469
|
+
value.faultLogicalTimeMs < value.preEffect.logicalTimeMs ||
|
|
470
|
+
Object.keys(value.preEffect.meshStates).length !== value.peers.length ||
|
|
471
|
+
value.peers.some((peer) => {
|
|
472
|
+
const state = value.preEffect.meshStates[peer.peerId];
|
|
473
|
+
return (!state ||
|
|
474
|
+
state.coordination.identity.peerId !== peer.peerId ||
|
|
475
|
+
state.coordination.identity.instanceId !== peer.peerInstanceId ||
|
|
476
|
+
state.coordination.identity.tenantId !== value.missionIntent.tenantId ||
|
|
477
|
+
state.coordination.identity.meshId !==
|
|
478
|
+
value.missionIntent.objective.meshId);
|
|
479
|
+
}))
|
|
480
|
+
throw new TypeError("closed_loop_recovery_binding_invalid");
|
|
481
|
+
return value;
|
|
482
|
+
}
|
|
483
|
+
function peer(context, peerId) {
|
|
484
|
+
const value = context.peersById.get(peerId);
|
|
485
|
+
if (!value || !context.input.runner.privateKeys[peerId])
|
|
486
|
+
throw new Error("closed_loop_recovery_peer_missing");
|
|
487
|
+
return value;
|
|
488
|
+
}
|
|
489
|
+
function allocationRuntime(state) {
|
|
490
|
+
return createMeshAllocationRuntimeState(state.coordination, state.discovery, state.objectives, state.allocation);
|
|
491
|
+
}
|
|
492
|
+
function retainInbound(prior, state) {
|
|
493
|
+
return createMeshAllocationInboundRuntimeState(state.coordination, state.discovery, state.objectives, state.allocation, prior.inbound);
|
|
494
|
+
}
|
|
495
|
+
function extendRecoveryAdmission(context, states) {
|
|
496
|
+
const configuredPeers = Object.fromEntries(context.input.peers.map((candidate) => [
|
|
497
|
+
candidate.peerId,
|
|
498
|
+
{
|
|
499
|
+
peerId: candidate.peerId,
|
|
500
|
+
instanceIds: [candidate.peerInstanceId],
|
|
501
|
+
validUntil: context.input.missionIntent.validUntil,
|
|
502
|
+
},
|
|
503
|
+
]));
|
|
504
|
+
return Object.fromEntries(Object.entries(states).map(([peerId, state]) => [
|
|
505
|
+
peerId,
|
|
506
|
+
createMeshAllocationInboundRuntimeState(state.coordination, restoreMeshDiscoveryState({
|
|
507
|
+
...state.discovery,
|
|
508
|
+
admittedPeers: Object.fromEntries([
|
|
509
|
+
...Object.entries(configuredPeers),
|
|
510
|
+
...Object.values(state.objectives.objectivePolicies)
|
|
511
|
+
.flatMap((policy) => policy.recoveryWitnessPeerIds)
|
|
512
|
+
.filter((witnessPeerId) => !Object.hasOwn(configuredPeers, witnessPeerId))
|
|
513
|
+
.map((witnessPeerId) => [
|
|
514
|
+
witnessPeerId,
|
|
515
|
+
{
|
|
516
|
+
peerId: witnessPeerId,
|
|
517
|
+
instanceIds: [`instance:${witnessPeerId}`],
|
|
518
|
+
validUntil: context.input.missionIntent.validUntil,
|
|
519
|
+
},
|
|
520
|
+
]),
|
|
521
|
+
]),
|
|
522
|
+
lastLogicalTime: state.discovery.lastLogicalTime,
|
|
523
|
+
}), state.objectives, state.allocation, state.inbound),
|
|
524
|
+
]));
|
|
525
|
+
}
|
|
526
|
+
function acceptLocal(states, peerId, decision, label) {
|
|
527
|
+
if (!decision.accepted)
|
|
528
|
+
throw new Error(`${label}:${decision.code}`);
|
|
529
|
+
return {
|
|
530
|
+
...states,
|
|
531
|
+
[peerId]: retainInbound(states[peerId], decision.state),
|
|
532
|
+
};
|
|
533
|
+
}
|
|
534
|
+
async function deliver(context, states, peerId, envelope, logicalTimeMs, verifiedAt = recoveryVerifiedAt(context.input.missionIntent, logicalTimeMs)) {
|
|
535
|
+
const state = states[peerId];
|
|
536
|
+
if (!state)
|
|
537
|
+
return states;
|
|
538
|
+
const processor = createMeshAllocationInboundProcessor({
|
|
539
|
+
resolver: context.input.runner.resolver,
|
|
540
|
+
cryptoPolicy: context.input.runner.cryptoPolicy,
|
|
541
|
+
crypto: context.input.runner.crypto,
|
|
542
|
+
});
|
|
543
|
+
const decision = await processor.process(state, {
|
|
544
|
+
envelope,
|
|
545
|
+
verifiedAt,
|
|
546
|
+
receivedAt: logicalTimeMs,
|
|
547
|
+
});
|
|
548
|
+
if (!decision.accepted) {
|
|
549
|
+
const diagnostic = recoveryDeliveryDiagnostic(state, envelope);
|
|
550
|
+
throw new Error(`closed_loop_recovery_delivery_rejected_${envelope.payload.type.replaceAll(".", "_")}_${decision.code}_${diagnostic}`);
|
|
551
|
+
}
|
|
552
|
+
context.interactionCount += 1;
|
|
553
|
+
return { ...states, [peerId]: decision.state };
|
|
554
|
+
}
|
|
555
|
+
function recoveryDeliveryDiagnostic(state, envelope) {
|
|
556
|
+
if (envelope.payload.type !== "work.award")
|
|
557
|
+
return "not_applicable";
|
|
558
|
+
const payload = envelope.payload;
|
|
559
|
+
if (payload.authorityKind !== "recovery_certificate")
|
|
560
|
+
return "authority_kind_invalid";
|
|
561
|
+
const certificate = state.allocation.recoveryCertificates[payload.recoveryCertificateId];
|
|
562
|
+
if (!certificate)
|
|
563
|
+
return "certificate_missing";
|
|
564
|
+
const proposal = state.allocation.takeoverProposals[certificate.takeoverProposalId];
|
|
565
|
+
if (!proposal)
|
|
566
|
+
return "proposal_missing";
|
|
567
|
+
if (!state.allocation.receivedOffers[payload.offerId])
|
|
568
|
+
return "offer_missing";
|
|
569
|
+
if (!state.allocation.localBids[payload.bidId])
|
|
570
|
+
return "bid_missing";
|
|
571
|
+
return "other_precondition_invalid";
|
|
572
|
+
}
|
|
573
|
+
async function copyForWitness(context, source, witnessPeerId, causationId = source.causationId) {
|
|
574
|
+
return signEnvelope(context, source.sender.peerId, source.type, source.payload, witnessPeerId, causationId, { sentAt: source.sentAt, expiresAt: source.expiresAt });
|
|
575
|
+
}
|
|
576
|
+
function expireLeases(initial, peerIds, floor) {
|
|
577
|
+
let states = initial;
|
|
578
|
+
let logicalTimeMs = floor;
|
|
579
|
+
for (const peerId of [...new Set(peerIds)].sort()) {
|
|
580
|
+
const state = states[peerId];
|
|
581
|
+
if (!state)
|
|
582
|
+
continue;
|
|
583
|
+
const timer = Object.values(state.coordination.timers).find((candidate) => candidate.kind === "lease.expiry");
|
|
584
|
+
if (!timer)
|
|
585
|
+
continue;
|
|
586
|
+
logicalTimeMs = Math.max(logicalTimeMs, timer.dueAt);
|
|
587
|
+
const decision = evaluateMeshAllocationTimer(allocationRuntime(state), {
|
|
588
|
+
kind: "timer.fired",
|
|
589
|
+
timerId: timer.timerId,
|
|
590
|
+
generation: timer.generation,
|
|
591
|
+
}, logicalTimeMs);
|
|
592
|
+
if (!decision.accepted)
|
|
593
|
+
throw new Error(`closed_loop_recovery_expiry_rejected:${decision.code}`);
|
|
594
|
+
states = { ...states, [peerId]: retainInbound(state, decision.state) };
|
|
595
|
+
}
|
|
596
|
+
return { states, logicalTimeMs };
|
|
597
|
+
}
|
|
598
|
+
async function recoveryRecipients(context, senderPeerId, payload, causationId, logicalTimeMs) {
|
|
599
|
+
const policy = context.input.preEffect.meshStates[context.input.preEffect.workContract.assignment.ownerPeerId].objectives.objectives[context.input.missionIntent.objective.objectiveId];
|
|
600
|
+
// The expired assignee must observe the takeover proposal so its prior
|
|
601
|
+
// authority is fenced. It is deliberately not a participant in votes or
|
|
602
|
+
// certificates, however. Sending those messages to it works accidentally
|
|
603
|
+
// only when the prior assignee is also a witness, and otherwise causes the
|
|
604
|
+
// receiver to correctly reject them as stale authority at larger scales.
|
|
605
|
+
const recipients = [
|
|
606
|
+
context.input.preEffect.workContract.assignment.ownerPeerId,
|
|
607
|
+
context.input.replacementPeerId,
|
|
608
|
+
...policy.recoveryWitnessPeerIds,
|
|
609
|
+
...(payload.type === "lease.takeover_proposal"
|
|
610
|
+
? [context.input.failedWinnerPeerId]
|
|
611
|
+
: []),
|
|
612
|
+
]
|
|
613
|
+
.filter((peerId, index, all) => peerId !== senderPeerId && all.indexOf(peerId) === index)
|
|
614
|
+
.sort();
|
|
615
|
+
return Promise.all(recipients.map(async (recipientPeerId) => ({
|
|
616
|
+
recipientPeerId,
|
|
617
|
+
preparedAt: logicalTimeMs,
|
|
618
|
+
envelope: await signEnvelope(context, senderPeerId, payload.type, payload, recipientPeerId, causationId),
|
|
619
|
+
})));
|
|
620
|
+
}
|
|
621
|
+
function applyLocalRecovery(context, states, senderPeerId, recipients, logicalTimeMs) {
|
|
622
|
+
const state = states[senderPeerId];
|
|
623
|
+
if (!state)
|
|
624
|
+
throw new Error("closed_loop_recovery_sender_state_missing");
|
|
625
|
+
const decision = evaluateMeshAllocationCommand(allocationRuntime(state), { kind: "allocation.recovery", recipients }, recoveryVerifiedAt(context.input.missionIntent, logicalTimeMs), logicalTimeMs);
|
|
626
|
+
return acceptLocal(states, senderPeerId, decision, "closed_loop_recovery_record_rejected");
|
|
627
|
+
}
|
|
628
|
+
async function deliverRecoveryEffects(context, states, senderPeerId, recipients, logicalTimeMs) {
|
|
629
|
+
let next = states;
|
|
630
|
+
for (const recipient of recipients) {
|
|
631
|
+
if (recipient.recipientPeerId === senderPeerId ||
|
|
632
|
+
!next[recipient.recipientPeerId])
|
|
633
|
+
continue;
|
|
634
|
+
next = await deliver(context, next, recipient.recipientPeerId, recipient.envelope, logicalTimeMs);
|
|
635
|
+
}
|
|
636
|
+
return next;
|
|
637
|
+
}
|
|
638
|
+
async function signEnvelope(context, senderPeerId, type, payload, recipientPeerId, causationId, timing) {
|
|
639
|
+
const sender = peer(context, senderPeerId);
|
|
640
|
+
const sequence = (context.sequences.get(senderPeerId) ?? 10_000) + 1;
|
|
641
|
+
context.sequences.set(senderPeerId, sequence);
|
|
642
|
+
// Reserve every allocator output before the first await. Recipient
|
|
643
|
+
// preparation intentionally runs concurrently, so allocating afterwards
|
|
644
|
+
// would make message IDs depend on crypto promise completion order.
|
|
645
|
+
const envelopeMessageId = messageId(context.nextMessage++);
|
|
646
|
+
const canonical = canonicalizeMeshPayload(payload);
|
|
647
|
+
if (!canonical.ok)
|
|
648
|
+
throw new Error("closed_loop_recovery_payload_invalid");
|
|
649
|
+
const bytes = await context.input.runner.crypto.subtle.digest("SHA-256", canonical.value.slice().buffer);
|
|
650
|
+
const envelope = {
|
|
651
|
+
protocol: "agentplat.mesh",
|
|
652
|
+
wireVersion: WIRE_VERSION,
|
|
653
|
+
messageId: envelopeMessageId,
|
|
654
|
+
tenantId: context.input.missionIntent.tenantId,
|
|
655
|
+
meshId: context.input.missionIntent.objective.meshId,
|
|
656
|
+
objectiveId: context.input.missionIntent.objective.objectiveId,
|
|
657
|
+
type,
|
|
658
|
+
sender: { peerId: sender.peerId, instanceId: sender.peerInstanceId },
|
|
659
|
+
audience: { kind: "peer", peerId: recipientPeerId },
|
|
660
|
+
sequence,
|
|
661
|
+
sentAt: timing?.sentAt ??
|
|
662
|
+
recoverySentAt(context.input.missionIntent, context.input.faultLogicalTimeMs),
|
|
663
|
+
expiresAt: timing?.expiresAt ??
|
|
664
|
+
wallTime(context.input.missionIntent, context.input.faultLogicalTimeMs + RECOVERY_ENVELOPE_TTL_MS),
|
|
665
|
+
...(causationId === undefined ? {} : { causationId }),
|
|
666
|
+
payloadHash: `sha256:${base64Url(new Uint8Array(bytes))}`,
|
|
667
|
+
payload,
|
|
668
|
+
proof: {
|
|
669
|
+
algorithm: "Ed25519",
|
|
670
|
+
keyId: `key:${sender.peerId}`,
|
|
671
|
+
// The signer replaces this placeholder after structural validation.
|
|
672
|
+
value: "A".repeat(86),
|
|
673
|
+
},
|
|
674
|
+
};
|
|
675
|
+
const preflight = validateSignedMeshEnvelope({
|
|
676
|
+
...envelope,
|
|
677
|
+
payloadHash: `sha256:${"A".repeat(43)}`,
|
|
678
|
+
proof: { ...envelope.proof, value: "A".repeat(86) },
|
|
679
|
+
});
|
|
680
|
+
if (!preflight.ok)
|
|
681
|
+
throw new Error(`closed_loop_recovery_envelope_invalid:${preflight.issues
|
|
682
|
+
.map((issue) => `${issue.code}:${issue.path}`)
|
|
683
|
+
.join(",")}`);
|
|
684
|
+
return context.input.runner.signer.sign({
|
|
685
|
+
envelope,
|
|
686
|
+
privateKey: context.input.runner.privateKeys[senderPeerId],
|
|
687
|
+
});
|
|
688
|
+
}
|
|
689
|
+
function currentAuthority(states, ownerPeerId, replacementPeerId, checkpointId) {
|
|
690
|
+
const owner = states[ownerPeerId];
|
|
691
|
+
const replacement = states[replacementPeerId];
|
|
692
|
+
const execution = Object.values(replacement.allocation.executionHeads).find((head) => head.phase === "active");
|
|
693
|
+
const fenceHead = Object.values(replacement.allocation.assignmentFenceHeads).find((head) => head.phase === "active");
|
|
694
|
+
const workItem = Object.values(owner.objectives.workItems).find((work) => work.workItemId === execution?.workItemId);
|
|
695
|
+
const objective = execution === undefined
|
|
696
|
+
? undefined
|
|
697
|
+
: owner.objectives.objectives[execution.objectiveId];
|
|
698
|
+
const checkpoint = replacement.allocation.executionRecords[checkpointId];
|
|
699
|
+
if (!objective || !workItem || !execution || !fenceHead || !checkpoint)
|
|
700
|
+
throw new Error("closed_loop_recovery_current_authority_missing");
|
|
701
|
+
return { objective, workItem, execution, fenceHead, checkpoint };
|
|
702
|
+
}
|
|
703
|
+
async function rejectStaleEpochOneRecords(context, states, ownerPeerId, failedPeerId, oldLeaseExpiresAt, preEffect, logicalTimeMs) {
|
|
704
|
+
const common = {
|
|
705
|
+
objectiveId: preEffect.execution.objectiveId,
|
|
706
|
+
objectiveDocumentId: preEffect.execution.objectiveDocumentId,
|
|
707
|
+
objectiveRevision: preEffect.execution.objectiveRevision,
|
|
708
|
+
workItemId: preEffect.execution.workItemId,
|
|
709
|
+
workItemRevision: preEffect.execution.workItemRevision,
|
|
710
|
+
ownerPeerId,
|
|
711
|
+
ownerEpoch: preEffect.execution.ownerEpoch,
|
|
712
|
+
assigneePeerId: failedPeerId,
|
|
713
|
+
awardId: preEffect.execution.awardId,
|
|
714
|
+
acceptanceId: preEffect.execution.acceptanceId,
|
|
715
|
+
assignmentEpoch: preEffect.execution.assignmentEpoch,
|
|
716
|
+
assignmentAuthorityId: preEffect.execution.assignmentAuthorityId,
|
|
717
|
+
fencingToken: preEffect.execution.fencingToken,
|
|
718
|
+
leaseExpiresAt: oldLeaseExpiresAt,
|
|
719
|
+
};
|
|
720
|
+
const verifiedBeforeOldLease = new Date(Date.parse(oldLeaseExpiresAt) - 2).toISOString();
|
|
721
|
+
const staleTiming = {
|
|
722
|
+
sentAt: new Date(Date.parse(oldLeaseExpiresAt) - 2).toISOString(),
|
|
723
|
+
expiresAt: new Date(Date.parse(oldLeaseExpiresAt) - 1).toISOString(),
|
|
724
|
+
};
|
|
725
|
+
const progress = await signEnvelope(context, failedPeerId, "work.progress", {
|
|
726
|
+
type: "work.progress",
|
|
727
|
+
progressId: "recovery:stale-progress:1",
|
|
728
|
+
progressSequence: 2,
|
|
729
|
+
...common,
|
|
730
|
+
progressSummary: "stale epoch one progress",
|
|
731
|
+
}, ownerPeerId, preEffect.checkpoint.envelope.messageId, staleTiming);
|
|
732
|
+
const result = await signEnvelope(context, failedPeerId, "work.result", {
|
|
733
|
+
type: "work.result",
|
|
734
|
+
resultId: "recovery:stale-result:1",
|
|
735
|
+
resultDigest: "sha256:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
|
|
736
|
+
...common,
|
|
737
|
+
checkpointId: preEffect.checkpoint.recordId,
|
|
738
|
+
resultSummary: "stale epoch one result",
|
|
739
|
+
}, ownerPeerId, preEffect.checkpoint.envelope.messageId, staleTiming);
|
|
740
|
+
const rejections = [];
|
|
741
|
+
for (const [recordType, recordId, envelope] of [
|
|
742
|
+
["work.progress", "recovery:stale-progress:1", progress],
|
|
743
|
+
["work.result", "recovery:stale-result:1", result],
|
|
744
|
+
]) {
|
|
745
|
+
const verification = await verifyMeshEnvelope({
|
|
746
|
+
envelope: envelope,
|
|
747
|
+
resolver: context.input.runner.resolver,
|
|
748
|
+
policy: context.input.runner.cryptoPolicy,
|
|
749
|
+
verifiedAt: verifiedBeforeOldLease,
|
|
750
|
+
});
|
|
751
|
+
if (!verification.verified)
|
|
752
|
+
throw new Error("closed_loop_recovery_stale_envelope_unverified");
|
|
753
|
+
const before = states[ownerPeerId];
|
|
754
|
+
const allocationBefore = allocationRuntime(before);
|
|
755
|
+
const decision = evaluateVerifiedMeshAllocationEnvelope(allocationBefore, {
|
|
756
|
+
envelope: verification.envelope,
|
|
757
|
+
verifiedAt: verifiedBeforeOldLease,
|
|
758
|
+
receivedAt: logicalTimeMs,
|
|
759
|
+
});
|
|
760
|
+
if (decision.accepted || decision.state !== allocationBefore)
|
|
761
|
+
throw new Error("closed_loop_recovery_stale_record_mutated");
|
|
762
|
+
rejections.push(Object.freeze({
|
|
763
|
+
recordType,
|
|
764
|
+
recordId,
|
|
765
|
+
envelope,
|
|
766
|
+
rejectionCode: decision.code,
|
|
767
|
+
logicalTimeMs,
|
|
768
|
+
stateUnchanged: true,
|
|
769
|
+
}));
|
|
770
|
+
}
|
|
771
|
+
return rejections;
|
|
772
|
+
}
|
|
773
|
+
function identity(missionIntent, peerValue) {
|
|
774
|
+
return Object.freeze({
|
|
775
|
+
tenantId: missionIntent.tenantId,
|
|
776
|
+
meshId: missionIntent.objective.meshId,
|
|
777
|
+
peerId: peerValue.peerId,
|
|
778
|
+
instanceId: peerValue.peerInstanceId,
|
|
779
|
+
keyId: `key:${peerValue.peerId}`,
|
|
780
|
+
});
|
|
781
|
+
}
|
|
782
|
+
function recoverySentAt(missionIntent, logicalTimeMs) {
|
|
783
|
+
return wallTime(missionIntent, logicalTimeMs);
|
|
784
|
+
}
|
|
785
|
+
function recoveryVerifiedAt(missionIntent, logicalTimeMs) {
|
|
786
|
+
return wallTime(missionIntent, logicalTimeMs + 1);
|
|
787
|
+
}
|
|
788
|
+
function wallTime(missionIntent, offsetMs) {
|
|
789
|
+
const time = Date.parse(missionIntent.validFrom) + offsetMs;
|
|
790
|
+
if (!Number.isSafeInteger(time) ||
|
|
791
|
+
time >= Date.parse(missionIntent.validUntil))
|
|
792
|
+
throw new RangeError("closed_loop_recovery_wall_time_invalid");
|
|
793
|
+
return new Date(time).toISOString();
|
|
794
|
+
}
|
|
795
|
+
function earlierTimestamp(left, right) {
|
|
796
|
+
return Date.parse(left) <= Date.parse(right) ? left : right;
|
|
797
|
+
}
|
|
798
|
+
function meshContentDigest(value) {
|
|
799
|
+
if (!/^sha256:[0-9a-f]{64}$/u.test(value))
|
|
800
|
+
throw new TypeError("closed_loop_result_digest_invalid");
|
|
801
|
+
const hex = value.slice("sha256:".length);
|
|
802
|
+
const bytes = new Uint8Array(32);
|
|
803
|
+
for (let index = 0; index < bytes.length; index += 1)
|
|
804
|
+
bytes[index] = Number.parseInt(hex.slice(index * 2, index * 2 + 2), 16);
|
|
805
|
+
return `sha256:${base64Url(bytes)}`;
|
|
806
|
+
}
|
|
807
|
+
function messageId(sequence) {
|
|
808
|
+
const bytes = new Uint8Array(16);
|
|
809
|
+
bytes[0] = 0x72;
|
|
810
|
+
bytes[1] = 0x63;
|
|
811
|
+
new DataView(bytes.buffer).setUint32(12, sequence);
|
|
812
|
+
return base64Url(bytes);
|
|
813
|
+
}
|
|
814
|
+
function base64Url(bytes) {
|
|
815
|
+
let binary = "";
|
|
816
|
+
for (const value of bytes)
|
|
817
|
+
binary += String.fromCharCode(value);
|
|
818
|
+
return btoa(binary)
|
|
819
|
+
.replace(/\+/gu, "-")
|
|
820
|
+
.replace(/\//gu, "_")
|
|
821
|
+
.replace(/=+$/gu, "");
|
|
822
|
+
}
|
|
823
|
+
//# sourceMappingURL=collective-closed-loop-recovery.js.map
|