@dharma-ai-labs/agent-fabric-relay-client 0.2.37 → 0.2.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +195 -0
- package/dist/deviceSigningControl.d.ts +18 -0
- package/dist/deviceSigningControl.d.ts.map +1 -1
- package/dist/deviceSigningControl.js +137 -4
- package/dist/deviceSigningControl.js.map +1 -1
- package/dist/deviceSigningControl.test.js +145 -0
- package/dist/deviceSigningControl.test.js.map +1 -1
- package/dist/enrollmentAnchorLock.d.ts.map +1 -1
- package/dist/enrollmentAnchorLock.js +57 -17
- package/dist/enrollmentAnchorLock.js.map +1 -1
- package/dist/enrollmentAnchorLockCorruption.test.js +219 -12
- package/dist/enrollmentAnchorLockCorruption.test.js.map +1 -1
- package/dist/hostScope.test.js +5 -3
- package/dist/hostScope.test.js.map +1 -1
- package/dist/index.d.ts +10 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +771 -88
- package/dist/index.js.map +1 -1
- package/dist/index.test.js +2592 -26
- package/dist/index.test.js.map +1 -1
- package/dist/protocolStateModel.test.d.ts +2 -0
- package/dist/protocolStateModel.test.d.ts.map +1 -0
- package/dist/protocolStateModel.test.js +383 -0
- package/dist/protocolStateModel.test.js.map +1 -0
- package/package.json +4 -1
package/dist/index.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
1
2
|
import { createHash, createPublicKey, generateKeyPairSync, randomBytes, randomUUID, sign } from 'node:crypto';
|
|
2
3
|
import { mkdir, open, readFile, rename, rm } from 'node:fs/promises';
|
|
3
4
|
import { dirname, resolve } from 'node:path';
|
|
@@ -9,13 +10,41 @@ import { HostOperationFence } from './hostOperationScope.js';
|
|
|
9
10
|
import { commitSigningRecoveryJournal, resumeSigningRecoveryJournal } from './signingRecoveryJournal.js';
|
|
10
11
|
import { checkedRecoveryHostContext, retrieveSigningRecoveryDelivery } from './signingRecoveryClient.js';
|
|
11
12
|
import { planSigningRecovery } from './signingRecoveryPlan.js';
|
|
12
|
-
import { withDeviceSigningControl } from './deviceSigningControl.js';
|
|
13
|
-
export { withDeviceSigningControl } from './deviceSigningControl.js';
|
|
13
|
+
import { handBackDeviceSigningControl, insideCallerSigningControlHold, insideDeviceSigningControlHold, outsideDeviceSigningControl, withDeviceSigningControl } from './deviceSigningControl.js';
|
|
14
|
+
export { withDeviceSigningControl, withDeviceSigningLineSignal, withSigningControlPerSend } from './deviceSigningControl.js';
|
|
14
15
|
export { SigningRecoveryClientError } from './signingRecoveryClient.js';
|
|
15
16
|
export { progressPreparedDeviceSigningRecoveryOffer, SigningRecoveryOfferClientError } from './signingRecoveryOfferClient.js';
|
|
16
17
|
export { prepareOwnedSigningRecovery, submitOwnedSigningRecoveryPreparation, progressOwnedSigningRecoveryOffer, completeOwnedSigningRecovery, inspectOwnedSigningRecovery, disposeOwnedSigningRecovery, OwnedSigningRecoveryError, prepareOwnedSigningMaintenance, readOwnedSigningMaintenanceContext } from './signingRecoveryHost.js';
|
|
17
18
|
export { claimSetupReference, setupClaimSourceRegistration, setupClaimFailureMessage } from './setupClaim.js';
|
|
18
19
|
export { planDeviceSigningRecovery, planDeviceSigningMaintenance } from './signingRecoveryPlan.js';
|
|
20
|
+
/** The most entries in one file. Another client's entry stays only while it holds a kept request, kept completions or
|
|
21
|
+
* an answer of the last ORPHAN_RETRY_INTERVAL_MS (see #persist). */
|
|
22
|
+
const OUTBOX_ENTRIES_MAX = 256;
|
|
23
|
+
/** The most recovered task completions in one file. */
|
|
24
|
+
const RECOVERED_TASK_COMPLETIONS_MAX = 1_000;
|
|
25
|
+
/** An entry whose kept request stood unchanged this long is orphaned, and another client sends an orphaned request
|
|
26
|
+
* that got no definite answer again only after this interval. */
|
|
27
|
+
const ORPHAN_RETRY_INTERVAL_MS = 10 * 60_000;
|
|
28
|
+
/** Orphans one hold of signing control sends, so another process that waits for the lock file waits for at most one
|
|
29
|
+
* such send. While more are due, the client's next request sends the next one, in a section of its own. */
|
|
30
|
+
const ORPHAN_SENDS_PER_SECTION = 1;
|
|
31
|
+
/** What one hold's part of a signed request returns after it sent another kept request first, so that the caller's
|
|
32
|
+
* request is signed in a hold of its own. Any answer the server gives, a JSON null among them, is the caller's. */
|
|
33
|
+
const HANDED_BACK = Symbol('handed_back');
|
|
34
|
+
/** Task leases held in this process, by client, until when they run unless extended. While one is held, no client of
|
|
35
|
+
* this process sends an orphan after its own request, so a lease heartbeat never waits behind such a send. */
|
|
36
|
+
const taskLeases = new Map();
|
|
37
|
+
function taskLeaseHeld(now) {
|
|
38
|
+
for (const [client, until] of taskLeases)
|
|
39
|
+
if (until <= now)
|
|
40
|
+
taskLeases.delete(client);
|
|
41
|
+
return taskLeases.size > 0;
|
|
42
|
+
}
|
|
43
|
+
const OWNER_KEY_PATTERN = /^(?:relay|cli|migrated):[A-Za-z0-9_.:-]{1,128}$/;
|
|
44
|
+
/** Part of every default owner key, so a later process that gets the same pid never takes over an earlier entry. */
|
|
45
|
+
const PROCESS_INSTANCE = randomBytes(9).toString('base64url');
|
|
46
|
+
let clientInstances = 0;
|
|
47
|
+
const protocolStateSections = new AsyncLocalStorage();
|
|
19
48
|
const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
|
20
49
|
function isRecoveredTaskCompletion(value) {
|
|
21
50
|
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
@@ -26,29 +55,181 @@ function isRecoveredTaskCompletion(value) {
|
|
|
26
55
|
&& /^sha256:[a-f0-9]{64}$/.test(String(item.receiptHash || ''))
|
|
27
56
|
&& Number.isFinite(Date.parse(String(item.recoveredAt || '')));
|
|
28
57
|
}
|
|
29
|
-
function
|
|
58
|
+
function isPendingRequest(value) {
|
|
30
59
|
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
31
|
-
|
|
32
|
-
const
|
|
33
|
-
|
|
34
|
-
const pending = state.pending;
|
|
35
|
-
const pendingValid = pending === null || (typeof pending === 'object' && !Array.isArray(pending)
|
|
36
|
-
&& (pending.method === 'POST' || pending.method === 'GET')
|
|
60
|
+
return false;
|
|
61
|
+
const pending = value;
|
|
62
|
+
return (pending.method === 'POST' || pending.method === 'GET')
|
|
37
63
|
&& typeof pending.pathname === 'string' && pending.pathname.startsWith('/')
|
|
38
64
|
&& typeof pending.body === 'string'
|
|
39
65
|
&& pending.headers !== null && typeof pending.headers === 'object' && !Array.isArray(pending.headers)
|
|
40
66
|
&& Object.entries(pending.headers)
|
|
41
|
-
.every(([name, header]) => name.length > 0 && typeof header === 'string')
|
|
67
|
+
.every(([name, header]) => name.length > 0 && typeof header === 'string');
|
|
68
|
+
}
|
|
69
|
+
const isInstant = (value) => typeof value === 'string' && Number.isFinite(Date.parse(value));
|
|
70
|
+
function isAdoptedOutcome(value) {
|
|
71
|
+
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
72
|
+
return false;
|
|
73
|
+
const outcome = value;
|
|
74
|
+
const status = Number(outcome.status);
|
|
75
|
+
return typeof outcome.messageId === 'string' && outcome.messageId.length > 0 && Number.isSafeInteger(outcome.status)
|
|
76
|
+
&& ((status >= 200 && status < 300) || (status >= 400 && status < 500 && status !== 408 && status !== 429))
|
|
77
|
+
&& !!outcome.response && typeof outcome.response === 'object' && !Array.isArray(outcome.response);
|
|
78
|
+
}
|
|
79
|
+
function isOutboxEntry(value) {
|
|
80
|
+
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
81
|
+
return false;
|
|
82
|
+
const entry = value;
|
|
83
|
+
return (entry.sessionId === null || typeof entry.sessionId === 'string')
|
|
84
|
+
&& Number.isSafeInteger(entry.nextSequence) && Number(entry.nextSequence) >= 1
|
|
85
|
+
&& (entry.pending === null || isPendingRequest(entry.pending))
|
|
86
|
+
&& Number.isSafeInteger(entry.ownerPid) && Number(entry.ownerPid) >= 0
|
|
87
|
+
&& isInstant(entry.updatedAt) && (entry.adoptedAt === undefined || isInstant(entry.adoptedAt))
|
|
88
|
+
&& (entry.adoptedOutcome === undefined || (entry.adoptedAt !== undefined && isAdoptedOutcome(entry.adoptedOutcome)))
|
|
89
|
+
&& (entry.keptCompletions === undefined || (Array.isArray(entry.keptCompletions)
|
|
90
|
+
&& entry.keptCompletions.length <= RECOVERED_TASK_COMPLETIONS_MAX && entry.keptCompletions.every(isRecoveredTaskCompletion)))
|
|
91
|
+
// A marker an earlier build of this version wrote; it is still read, and has no effect.
|
|
92
|
+
&& (entry.asSignedOnly === undefined || entry.asSignedOnly === true);
|
|
93
|
+
}
|
|
94
|
+
function assertProtocolState(value) {
|
|
95
|
+
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
96
|
+
throw new Error('Relay protocol state is invalid.');
|
|
97
|
+
const state = value;
|
|
98
|
+
const recovered = state.recoveredTaskCompletions;
|
|
99
|
+
const outboxes = state.outboxes;
|
|
42
100
|
if (state.schema !== 'dharma.protocol-state/v1'
|
|
43
101
|
|| !(state.sessionId === null || typeof state.sessionId === 'string')
|
|
44
102
|
|| !Number.isSafeInteger(state.nextSequence) || Number(state.nextSequence) < 1
|
|
45
|
-
|| !
|
|
103
|
+
|| !(state.pending === null || isPendingRequest(state.pending))
|
|
104
|
+
|| (outboxes !== undefined && (!outboxes || typeof outboxes !== 'object' || Array.isArray(outboxes)
|
|
105
|
+
|| Object.keys(outboxes).length > OUTBOX_ENTRIES_MAX
|
|
106
|
+
|| Object.entries(outboxes)
|
|
107
|
+
.some(([owner, entry]) => !OWNER_KEY_PATTERN.test(owner) || !isOutboxEntry(entry))))
|
|
46
108
|
|| (recovered !== undefined && (!Array.isArray(recovered)
|
|
47
|
-
|| recovered.length >
|
|
109
|
+
|| recovered.length > RECOVERED_TASK_COMPLETIONS_MAX
|
|
48
110
|
|| recovered.some((item) => !isRecoveredTaskCompletion(item))))) {
|
|
49
111
|
throw new Error('Relay protocol state is invalid.');
|
|
50
112
|
}
|
|
51
113
|
}
|
|
114
|
+
/** Every outbox entry of a file. The kept request of an earlier single-slot file becomes an orphan keyed by its
|
|
115
|
+
* session; a request that is always rebuilt is dropped as before. Once due, it can be sent only signed again, which
|
|
116
|
+
* only the earlier process could do, so it is settled unsent (see ownerOnlyEntry). Such an entry has
|
|
117
|
+
* no owner process, so it counts as unchanged from when its request was last signed or stamped: an earlier version
|
|
118
|
+
* that still runs keeps 10 minutes to send its own request (see orphanDue). A top-level request with the message id of
|
|
119
|
+
* an entry's kept request is that entry's copy (see #persist), not another orphan. The top-level session and next
|
|
120
|
+
* sequence are never read: only the process that opened a session signs in it, from the sequence it holds in memory. */
|
|
121
|
+
function outboxesOf(state) {
|
|
122
|
+
const outboxes = {};
|
|
123
|
+
for (const [owner, entry] of Object.entries(state.outboxes ?? {}))
|
|
124
|
+
outboxes[owner] = durableOutboxEntry(entry);
|
|
125
|
+
const pending = state.pending;
|
|
126
|
+
const messageId = pending?.headers['x-dharma-message-id'];
|
|
127
|
+
const copied = !!messageId && Object.values(outboxes).some(entry => entry.pending?.headers['x-dharma-message-id'] === messageId);
|
|
128
|
+
if (pending && !copied && !isExplicitlyRebuiltPath(pending.pathname) && !isSessionOpen(pending)) {
|
|
129
|
+
const session = pending.headers['x-dharma-session-id'] || state.sessionId || '';
|
|
130
|
+
const key = `migrated:${/^[A-Za-z0-9_-]{1,128}$/.test(session) ? session : sha256(session).slice(0, 32)}`;
|
|
131
|
+
const signedAt = Date.parse(pending.headers['x-dharma-timestamp'] ?? '');
|
|
132
|
+
const updatedAt = new Date(Number.isFinite(signedAt) ? Math.min(signedAt, Date.now()) : Date.now()).toISOString();
|
|
133
|
+
outboxes[key] ??= { sessionId: session || null, nextSequence: state.nextSequence, pending, ownerPid: 0, updatedAt };
|
|
134
|
+
}
|
|
135
|
+
return outboxes;
|
|
136
|
+
}
|
|
137
|
+
const ACTION_ACKNOWLEDGEMENT_PATH = /\/agent-fabric\/decisions\/[^/]+\/enforcements$/;
|
|
138
|
+
/** Whether an action acknowledgement time is outside the window the server accepts, so a kept acknowledgement is sent
|
|
139
|
+
* again only re-signed with a fresh time, a new message id and a new sequence. */
|
|
140
|
+
function acknowledgementTimeStale(acknowledgedAt, now) {
|
|
141
|
+
const at = Date.parse(acknowledgedAt);
|
|
142
|
+
return !Number.isFinite(at) || now - at > 8 * 60_000 || at > now + 5 * 60_000;
|
|
143
|
+
}
|
|
144
|
+
/** A kept request signed longer ago than this is sent again only with a new time and signature: the server refuses a
|
|
145
|
+
* signature more than 5 minutes old. Only its owner signs it again (see #sendPendingWithSigningControl). */
|
|
146
|
+
const SIGNATURE_SENDABLE_MS = 4 * 60_000;
|
|
147
|
+
/** Whether a kept request's signature is too old to send it again exactly as signed. */
|
|
148
|
+
function signatureTooOld(pending, now) {
|
|
149
|
+
const signedAt = Date.parse(pending.headers['x-dharma-timestamp'] ?? '');
|
|
150
|
+
return !Number.isFinite(signedAt) || now - signedAt > SIGNATURE_SENDABLE_MS;
|
|
151
|
+
}
|
|
152
|
+
/** Whether a kept request can be sent again only with a new signature at a new sequence, or not at all: a
|
|
153
|
+
* content-bearing request, which is never replayed implicitly, or an action acknowledgement that is invalid or outside
|
|
154
|
+
* its time window. Only the process that opened a session signs in it (see #sendPendingWithSigningControl), so another
|
|
155
|
+
* client never sends such a request. */
|
|
156
|
+
function cannotResendAsSigned(pending, now) {
|
|
157
|
+
if (isContentBearingPath(pending.pathname))
|
|
158
|
+
return true;
|
|
159
|
+
if (!ACTION_ACKNOWLEDGEMENT_PATH.test(pending.pathname))
|
|
160
|
+
return false;
|
|
161
|
+
let acknowledgement;
|
|
162
|
+
try {
|
|
163
|
+
acknowledgement = JSON.parse(pending.body);
|
|
164
|
+
}
|
|
165
|
+
catch {
|
|
166
|
+
return true;
|
|
167
|
+
}
|
|
168
|
+
return !validateActionDecisionAcknowledgementContract(acknowledgement).ok
|
|
169
|
+
|| acknowledgementTimeStale(acknowledgement.acknowledgedAt, now);
|
|
170
|
+
}
|
|
171
|
+
/** Whether the process that wrote an outbox entry still runs. A migrated entry has no owner process. */
|
|
172
|
+
function ownerProcessAlive(pid) {
|
|
173
|
+
if (pid === process.pid)
|
|
174
|
+
return true;
|
|
175
|
+
if (!Number.isSafeInteger(pid) || pid < 1 || pid > 0x7fffffff)
|
|
176
|
+
return false;
|
|
177
|
+
try {
|
|
178
|
+
process.kill(pid, 0);
|
|
179
|
+
return true;
|
|
180
|
+
}
|
|
181
|
+
catch (error) {
|
|
182
|
+
return error.code !== 'ESRCH';
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
/** A request that is always rebuilt is never kept in the file, as before. */
|
|
186
|
+
function durableOutboxEntry(entry) {
|
|
187
|
+
return entry.pending && isExplicitlyRebuiltPath(entry.pending.pathname)
|
|
188
|
+
? { ...entry, nextSequence: entry.nextSequence + 1, pending: null } : entry;
|
|
189
|
+
}
|
|
190
|
+
/** A kept session open matters only to its owner, whose next session open replaces it. Another client drops an
|
|
191
|
+
* orphaned one without sending it, and a full file does not keep one. */
|
|
192
|
+
function isSessionOpen(pending) {
|
|
193
|
+
return pending.method === 'POST' && pending.pathname.endsWith('/agent-fabric/sessions');
|
|
194
|
+
}
|
|
195
|
+
/** A kept request only its owner can use: a session open, a poll, whose answer hands work or content to the owner (a
|
|
196
|
+
* task it claimed lapses with its lease), or a read. Another client drops an orphaned one without sending it, so such
|
|
197
|
+
* an answer never reaches the file, and an earlier version is never handed one to replay. */
|
|
198
|
+
function isOwnerOnlyRequest(pending) {
|
|
199
|
+
return pending.method === 'GET' || isSessionOpen(pending) || pending.pathname.endsWith('/poll');
|
|
200
|
+
}
|
|
201
|
+
/** Whether another client may send an entry's kept request now. It is orphaned once its owner process has exited,
|
|
202
|
+
* or once it stood unchanged for ORPHAN_RETRY_INTERVAL_MS, since a pid can be reused or belong to another account. A
|
|
203
|
+
* workspace loop's entry waits that long even after its process exited, so a loop that restarts promptly replays its
|
|
204
|
+
* own request first. So does a migrated entry, whose request an earlier version that may still run left in the file.
|
|
205
|
+
* An orphan sent without a definite answer is due again after the same interval. */
|
|
206
|
+
function orphanDue(owner, entry, now) {
|
|
207
|
+
if (!entry.pending || recentlyAdopted(entry, now))
|
|
208
|
+
return false;
|
|
209
|
+
return now - Date.parse(entry.updatedAt) >= ORPHAN_RETRY_INTERVAL_MS
|
|
210
|
+
|| (owner.startsWith('cli:') && !ownerProcessAlive(entry.ownerPid));
|
|
211
|
+
}
|
|
212
|
+
/** A kept request another client never sends: one only its owner can use, or one that could be sent only signed again,
|
|
213
|
+
* including one whose signature is too old to send exactly as signed. Another client sends a kept request only byte for
|
|
214
|
+
* byte as signed. */
|
|
215
|
+
function ownerOnlyEntry(entry, now) {
|
|
216
|
+
return isOwnerOnlyRequest(entry.pending) || cannotResendAsSigned(entry.pending, now) || signatureTooOld(entry.pending, now);
|
|
217
|
+
}
|
|
218
|
+
/** Whether an entry's owner can never come back: a command that exited, or an entry moved from an earlier version's file. */
|
|
219
|
+
function ownerGone(owner, entry) {
|
|
220
|
+
return owner.startsWith('migrated:') || (owner.startsWith('cli:') && !ownerProcessAlive(entry.ownerPid));
|
|
221
|
+
}
|
|
222
|
+
/** Whether two kept requests are one call: the same message id, or the same session, path and idempotency key (an
|
|
223
|
+
* action acknowledgement signed again keeps those). */
|
|
224
|
+
function sameKeptRequest(a, b) {
|
|
225
|
+
const same = (name) => !!a.headers[name] && a.headers[name] === b.headers[name];
|
|
226
|
+
return same('x-dharma-message-id')
|
|
227
|
+
|| (a.pathname === b.pathname && same('idempotency-key') && same('x-dharma-session-id'));
|
|
228
|
+
}
|
|
229
|
+
/** Whether another client took the entry's kept request less than ORPHAN_RETRY_INTERVAL_MS ago. */
|
|
230
|
+
function recentlyAdopted(entry, now) {
|
|
231
|
+
return entry.adoptedAt !== undefined && now - Date.parse(entry.adoptedAt) < ORPHAN_RETRY_INTERVAL_MS;
|
|
232
|
+
}
|
|
52
233
|
function sha256(value) {
|
|
53
234
|
return createHash('sha256').update(value).digest('hex');
|
|
54
235
|
}
|
|
@@ -545,11 +726,16 @@ function errorMessage(body, status) {
|
|
|
545
726
|
export class AgentFabricRequestError extends Error {
|
|
546
727
|
status;
|
|
547
728
|
definitive;
|
|
729
|
+
/** The server's error code, kept only in the stable form diagnostics may record. The message stays free text. */
|
|
730
|
+
serverCode;
|
|
548
731
|
constructor(body, status, definitive) {
|
|
549
732
|
super(errorMessage(body, status));
|
|
550
733
|
this.name = 'AgentFabricRequestError';
|
|
551
734
|
this.status = status;
|
|
552
735
|
this.definitive = definitive;
|
|
736
|
+
const code = body && typeof body === 'object' ? body.error?.code : undefined;
|
|
737
|
+
if (typeof code === 'string' && /^[a-z0-9_]{1,80}$/.test(code))
|
|
738
|
+
this.serverCode = code;
|
|
553
739
|
}
|
|
554
740
|
}
|
|
555
741
|
export function isDefinitiveAgentFabricRejection(error) {
|
|
@@ -957,6 +1143,32 @@ export class AgentFabricClient {
|
|
|
957
1143
|
#isolatedSession;
|
|
958
1144
|
#readSessionAdmissionUsed = false;
|
|
959
1145
|
#state;
|
|
1146
|
+
#ownerKey;
|
|
1147
|
+
/** The other clients' entries as of this client's last read of the file. */
|
|
1148
|
+
#otherOutboxes;
|
|
1149
|
+
/** The definite answer another client got for this client's kept request, with the request as this client signed
|
|
1150
|
+
* it. Only the next call can take it, and only when it is the same call. */
|
|
1151
|
+
#adoptedOutcome = null;
|
|
1152
|
+
/** Completions this client recorded that no state write has stored yet, such as when the write after the server's
|
|
1153
|
+
* answer failed. A reload of the file keeps them until a write succeeds. */
|
|
1154
|
+
#unwrittenCompletions = new Map();
|
|
1155
|
+
/** The completions in this client's own entry that wait for room in the queue, as of the last read or write. */
|
|
1156
|
+
#keptCompletions = [];
|
|
1157
|
+
/** Whether this client's kept request is in its own entry in the file, as of the last read or write. A kept request
|
|
1158
|
+
* that is not there is written back before it is sent (see #sendPendingWithSigningControl). */
|
|
1159
|
+
#keptOnDisk;
|
|
1160
|
+
/** An action acknowledgement past its time window that this client kept from an earlier session, while openSession
|
|
1161
|
+
* opens a new one: it stays in this client's entry, and is signed again only in the new session once the server
|
|
1162
|
+
* answered its session open (see openSession). */
|
|
1163
|
+
#deferredAcknowledgement = null;
|
|
1164
|
+
/** Opened with an owner key, as a workspace loop is: it also sends due orphans after its own requests. */
|
|
1165
|
+
#longLived;
|
|
1166
|
+
/** Whether the server accepted this client's latest session open. Until it did, a client with a deferred
|
|
1167
|
+
* acknowledgement signs no other call (see #signedRequestWithSigningControl). */
|
|
1168
|
+
#sessionAccepted = false;
|
|
1169
|
+
/** Whether the section that signs the deferred acknowledgement again waits for its caller's hold to end (see
|
|
1170
|
+
* openSession). Until it ran, no other call of this client is signed or sent (see #signedRequestWithSigningControl). */
|
|
1171
|
+
#reissueWaiting = false;
|
|
960
1172
|
#serial = Promise.resolve();
|
|
961
1173
|
constructor(input) {
|
|
962
1174
|
this.config = input.config;
|
|
@@ -971,6 +1183,10 @@ export class AgentFabricClient {
|
|
|
971
1183
|
this.#hostBinding = this.#binding();
|
|
972
1184
|
this.#readOnly = input.readOnly === true;
|
|
973
1185
|
this.#isolatedSession = input.isolatedSession === true;
|
|
1186
|
+
this.#ownerKey = input.ownerKey;
|
|
1187
|
+
this.#longLived = input.longLived;
|
|
1188
|
+
this.#otherOutboxes = input.otherOutboxes;
|
|
1189
|
+
this.#keptOnDisk = input.keptOnDisk;
|
|
974
1190
|
}
|
|
975
1191
|
#binding() {
|
|
976
1192
|
const { schema, organizationId, deviceId, installationId, hqUrl, relayUrl, publicKeyEd25519, serverPublicKeyEd25519, enrolledAt, platform, setupClaimReference, setupClaimRepositoryFingerprint } = this.config;
|
|
@@ -996,6 +1212,11 @@ export class AgentFabricClient {
|
|
|
996
1212
|
throw new Error('relay_read_only_option_invalid');
|
|
997
1213
|
if (isolatedSession !== undefined && typeof isolatedSession !== 'boolean')
|
|
998
1214
|
throw new Error('relay_isolated_session_option_invalid');
|
|
1215
|
+
if (input.ownerKey !== undefined && (typeof input.ownerKey !== 'string'
|
|
1216
|
+
|| !/^(?:relay|cli):/.test(input.ownerKey) || !OWNER_KEY_PATTERN.test(input.ownerKey))) {
|
|
1217
|
+
throw new Error('relay_owner_key_invalid');
|
|
1218
|
+
}
|
|
1219
|
+
const ownerKey = input.ownerKey ?? `cli:${process.pid}:${PROCESS_INSTANCE}:${++clientInstances}`;
|
|
999
1220
|
const fence = hostScope === undefined ? undefined : new HostOperationFence(hostScope);
|
|
1000
1221
|
const rawStore = fence ? await fence.step(async () => suppliedStore ?? await createSystemSecureStore())
|
|
1001
1222
|
: suppliedStore ?? await createSystemSecureStore();
|
|
@@ -1021,12 +1242,23 @@ export class AgentFabricClient {
|
|
|
1021
1242
|
schema: 'dharma.protocol-state/v1', sessionId: null, nextSequence: 1, pending: null,
|
|
1022
1243
|
recoveredTaskCompletions: [],
|
|
1023
1244
|
};
|
|
1245
|
+
let otherOutboxes = {};
|
|
1246
|
+
let keptOnDisk = false;
|
|
1024
1247
|
if (!readOnly && !isolatedSession)
|
|
1025
1248
|
try {
|
|
1026
1249
|
const bytes = fence ? await fence.step(() => readFile(statePath, 'utf8')) : await readFile(statePath, 'utf8');
|
|
1027
1250
|
const parsed = JSON.parse(bytes);
|
|
1028
1251
|
assertProtocolState(parsed);
|
|
1029
|
-
|
|
1252
|
+
const { [ownerKey]: own, ...others } = outboxesOf(parsed);
|
|
1253
|
+
otherOutboxes = others;
|
|
1254
|
+
keptOnDisk = !!own?.pending;
|
|
1255
|
+
// Only the kept request of this owner key is taken over, to be sent again exactly as signed (or, an action
|
|
1256
|
+
// acknowledgement past its time window, signed again in this process's own session; see openSession). The
|
|
1257
|
+
// session that signed it belongs to the process that opened it, so this client opens its own and never signs there.
|
|
1258
|
+
state = {
|
|
1259
|
+
schema: 'dharma.protocol-state/v1', sessionId: null, nextSequence: 1,
|
|
1260
|
+
pending: own?.pending ?? null, recoveredTaskCompletions: parsed.recoveredTaskCompletions ?? [],
|
|
1261
|
+
};
|
|
1030
1262
|
}
|
|
1031
1263
|
catch (error) {
|
|
1032
1264
|
if (error.code !== 'ENOENT') {
|
|
@@ -1039,7 +1271,8 @@ export class AgentFabricClient {
|
|
|
1039
1271
|
await fence?.assert();
|
|
1040
1272
|
return new AgentFabricClient({
|
|
1041
1273
|
config, privateJwk: identity.privateJwk, configPath,
|
|
1042
|
-
statePath, state, store, fetcher, hostFence: fence, readOnly, isolatedSession,
|
|
1274
|
+
statePath, state, store, fetcher, hostFence: fence, readOnly, isolatedSession, ownerKey,
|
|
1275
|
+
longLived: input.ownerKey !== undefined, otherOutboxes, keptOnDisk,
|
|
1043
1276
|
});
|
|
1044
1277
|
}
|
|
1045
1278
|
async openSession(relayVersion = '0.1.0', telemetry = {}) {
|
|
@@ -1048,24 +1281,85 @@ export class AgentFabricClient {
|
|
|
1048
1281
|
if (this.#readOnly && (this.#state.sessionId || this.#readSessionAdmissionUsed)) {
|
|
1049
1282
|
throw new Error('relay_read_only_session_already_started');
|
|
1050
1283
|
}
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1284
|
+
// A durable client sends one due orphan or replays its own kept request, and writes its new session from the
|
|
1285
|
+
// latest file, all in one signing-control section. It calls the sends directly: waiting on #serial while holding
|
|
1286
|
+
// control could deadlock. The section ends before the session POST, which takes control again through #serial.
|
|
1287
|
+
// In a hold of signing control its caller took itself, perSend or not, this client adds no send of its own to that
|
|
1288
|
+
// hold beyond the session open: it sends no orphan there, and after it sends its own earlier kept request there, a
|
|
1289
|
+
// perSend hold is handed back and the session open runs again in a fresh hold, while in any other hold the session
|
|
1290
|
+
// open fails with relay_retry_after_recovery, unsent, and a retry opens it in a hold of its own.
|
|
1291
|
+
const callerHold = insideDeviceSigningControlHold();
|
|
1292
|
+
const replace = async () => {
|
|
1293
|
+
const pending = this.#state.pending;
|
|
1294
|
+
// An action acknowledgement past its time window can be sent again only signed again. Only the process that
|
|
1295
|
+
// opened a session signs in it, and this one may have taken the acknowledgement over from another process with
|
|
1296
|
+
// its owner key, so it is signed again only in the new session, after its session open (see below).
|
|
1297
|
+
const deferred = !!pending && ACTION_ACKNOWLEDGEMENT_PATH.test(pending.pathname) && cannotResendAsSigned(pending, Date.now());
|
|
1298
|
+
// Not while a task lease is held in this process, so the lease heartbeat of a sibling loop never waits on them,
|
|
1299
|
+
// and not when this section replays this client's own kept request, so it holds signing control for one send.
|
|
1300
|
+
const replays = !!pending && !deferred && !isExplicitlyRebuiltPath(pending.pathname);
|
|
1301
|
+
if (this.#durable && !replays && !callerHold && !taskLeaseHeld(Date.now()))
|
|
1302
|
+
await this.#adoptOrphans();
|
|
1303
|
+
if (this.#state.pending && isExplicitlyRebuiltPath(this.#state.pending.pathname)) {
|
|
1304
|
+
// Content-bearing requests require refreshed consent. Repository connect
|
|
1305
|
+
// requests depend on current workspace registration. Both are rebuilt by
|
|
1306
|
+
// their explicit workflows instead of replayed from stale prerequisites.
|
|
1307
|
+
this.#state.pending = null;
|
|
1308
|
+
await this.#persist();
|
|
1309
|
+
}
|
|
1310
|
+
else if (this.#state.pending && !deferred) {
|
|
1311
|
+
await this.#sendPendingWithSigningControl(this.#state);
|
|
1312
|
+
if (callerHold) {
|
|
1313
|
+
handBackDeviceSigningControl();
|
|
1314
|
+
throw new Error('relay_retry_after_recovery');
|
|
1315
|
+
}
|
|
1316
|
+
}
|
|
1317
|
+
if (deferred)
|
|
1318
|
+
this.#deferredAcknowledgement = this.#state.pending;
|
|
1319
|
+
this.#sessionAccepted = false;
|
|
1320
|
+
this.#state.sessionId = randomUUID();
|
|
1321
|
+
this.#state.nextSequence = 1;
|
|
1055
1322
|
this.#state.pending = null;
|
|
1056
1323
|
await this.#persist();
|
|
1057
|
-
}
|
|
1058
|
-
else if (this.#state.pending) {
|
|
1059
|
-
await this.#sendPending();
|
|
1060
|
-
}
|
|
1061
|
-
this.#state = {
|
|
1062
|
-
schema: 'dharma.protocol-state/v1', sessionId: randomUUID(), nextSequence: 1, pending: null,
|
|
1063
|
-
recoveredTaskCompletions: this.#state.recoveredTaskCompletions || [],
|
|
1064
1324
|
};
|
|
1065
|
-
|
|
1066
|
-
|
|
1325
|
+
if (this.#durable || this.#state.pending)
|
|
1326
|
+
await this.#withProtocolState(replace);
|
|
1327
|
+
else
|
|
1328
|
+
await replace();
|
|
1329
|
+
const opened = await this.signedPost('/agent-fabric/sessions', {
|
|
1067
1330
|
connectionId: randomBytes(24).toString('base64url'), durableCursor: null, relayVersion, deviceTelemetry,
|
|
1068
1331
|
});
|
|
1332
|
+
if (this.#deferredAcknowledgement) {
|
|
1333
|
+
// In a section of its own, as this client's next request: the acknowledgement is signed again at the next
|
|
1334
|
+
// sequence of the new session, with its idempotency key, and sent. Its outcome is never this caller's: a
|
|
1335
|
+
// retryable failure keeps it for this client's next request.
|
|
1336
|
+
const send = () => this.#withProtocolState(async () => {
|
|
1337
|
+
if (!this.#deferredAcknowledgement || this.#state.pending)
|
|
1338
|
+
return;
|
|
1339
|
+
this.#state.pending = this.#deferredAcknowledgement;
|
|
1340
|
+
this.#deferredAcknowledgement = null;
|
|
1341
|
+
try {
|
|
1342
|
+
await this.#sendPendingWithSigningControl(this.#state);
|
|
1343
|
+
}
|
|
1344
|
+
catch {
|
|
1345
|
+
await this.#assertHostScope();
|
|
1346
|
+
}
|
|
1347
|
+
});
|
|
1348
|
+
if (callerHold) {
|
|
1349
|
+
// Never in the caller's hold: in a hold of its own once that hold ends. It is not awaited, and not queued in
|
|
1350
|
+
// #serial, since it waits for the caller's hold to end, and the caller's next call in that hold must not wait on it.
|
|
1351
|
+
// Until it ran, another call of this client fails with relay_retry_after_recovery, unsent (owner review 3).
|
|
1352
|
+
this.#reissueWaiting = true;
|
|
1353
|
+
void outsideDeviceSigningControl(() => protocolStateSections.exit(send)).catch(() => undefined)
|
|
1354
|
+
.finally(() => { this.#reissueWaiting = false; });
|
|
1355
|
+
}
|
|
1356
|
+
else {
|
|
1357
|
+
const reissue = this.#serial.then(send);
|
|
1358
|
+
this.#serial = reissue.catch(() => undefined);
|
|
1359
|
+
await reissue;
|
|
1360
|
+
}
|
|
1361
|
+
}
|
|
1362
|
+
return opened;
|
|
1069
1363
|
}
|
|
1070
1364
|
async registerWorkspace(body, telemetry = {}) {
|
|
1071
1365
|
const response = await this.signedPost('/agent-fabric/workspaces', { ...body, deviceTelemetry: deviceKeysetTelemetry(this.config, telemetry) });
|
|
@@ -1107,18 +1401,42 @@ export class AgentFabricClient {
|
|
|
1107
1401
|
return (this.#state.recoveredTaskCompletions || []).map((item) => ({ ...item }));
|
|
1108
1402
|
}
|
|
1109
1403
|
acknowledgeRecoveredTaskCompletion(taskId, receiptHash) {
|
|
1110
|
-
const
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
throw new Error('relay_read_only_mutation_denied');
|
|
1404
|
+
const acknowledge = async () => {
|
|
1405
|
+
// A receipt can wait outside the stored list: in this client's memory or entry, or in another client's entry,
|
|
1406
|
+
// until there is room. It is removed wherever it waits, so no later read moves it back to the list.
|
|
1114
1407
|
const current = this.#state.recoveredTaskCompletions || [];
|
|
1115
|
-
const
|
|
1116
|
-
|
|
1408
|
+
const entries = Object.values(this.#otherOutboxes);
|
|
1409
|
+
const matching = [...current, ...this.#keptCompletions, ...this.#unwrittenCompletions.values(),
|
|
1410
|
+
...entries.flatMap(entry => entry.keptCompletions ?? [])].filter(item => item.taskId === taskId);
|
|
1411
|
+
if (!matching.length)
|
|
1117
1412
|
return;
|
|
1118
|
-
if (matching.receiptHash !== receiptHash)
|
|
1413
|
+
if (matching.some(item => item.receiptHash !== receiptHash)) {
|
|
1119
1414
|
throw new Error('Recovered task completion receipt does not match.');
|
|
1120
|
-
|
|
1415
|
+
}
|
|
1416
|
+
const other = (item) => item.taskId !== taskId;
|
|
1417
|
+
this.#state.recoveredTaskCompletions = current.filter(other);
|
|
1418
|
+
this.#keptCompletions = this.#keptCompletions.filter(other);
|
|
1419
|
+
this.#unwrittenCompletions.delete(taskId);
|
|
1420
|
+
for (const entry of entries) {
|
|
1421
|
+
const waiting = entry.keptCompletions?.filter(other);
|
|
1422
|
+
if (waiting?.length)
|
|
1423
|
+
entry.keptCompletions = waiting;
|
|
1424
|
+
else
|
|
1425
|
+
delete entry.keptCompletions;
|
|
1426
|
+
}
|
|
1121
1427
|
await this.#persist();
|
|
1428
|
+
};
|
|
1429
|
+
if (this.#durable) {
|
|
1430
|
+
// The completion queue belongs to the whole device: acknowledge against the latest file, under signing control.
|
|
1431
|
+
// Never on this client's request queue: a request that already holds signing control can wait there, and this
|
|
1432
|
+
// acknowledgement would then wait in the signing-control line for that request while it waits for us.
|
|
1433
|
+
return (async () => { await this.#assertHostScope(); return this.#withProtocolState(acknowledge); })();
|
|
1434
|
+
}
|
|
1435
|
+
const operation = this.#serial.then(async () => {
|
|
1436
|
+
await this.#assertHostScope();
|
|
1437
|
+
if (this.#readOnly)
|
|
1438
|
+
throw new Error('relay_read_only_mutation_denied');
|
|
1439
|
+
return acknowledge();
|
|
1122
1440
|
});
|
|
1123
1441
|
this.#serial = operation.catch(() => undefined);
|
|
1124
1442
|
return operation;
|
|
@@ -1153,13 +1471,270 @@ export class AgentFabricClient {
|
|
|
1153
1471
|
return operation;
|
|
1154
1472
|
}
|
|
1155
1473
|
async #signedRequestNow(method, route, body) {
|
|
1156
|
-
const
|
|
1157
|
-
|
|
1474
|
+
const event = method === 'POST' && /^\/agent-fabric\/tasks\/[^/]+\/events$/.test(route)
|
|
1475
|
+
? body?.eventType : undefined;
|
|
1476
|
+
if (event === 'completed' || event === 'failed' || event === 'cancelled')
|
|
1477
|
+
taskLeases.delete(this);
|
|
1478
|
+
// Each hold of signing control makes at most one network send. When another kept request is sent first (this
|
|
1479
|
+
// client's own earlier call, or a due orphan that makes room in a full file), that hold ends there, and the caller's
|
|
1480
|
+
// request is signed in a fresh hold, from a fresh read of the file, behind every caller that waited meanwhile. Under
|
|
1481
|
+
// the admission line the line's hold is handed back (see withSigningControlPerSend). A hold its caller took itself
|
|
1482
|
+
// without perSend keeps its work, so this client adds no send to it: it sends no orphan to make room there, and
|
|
1483
|
+
// after it sends its own earlier kept call there, the caller's call fails with relay_retry_after_recovery, a failure
|
|
1484
|
+
// with no outcome that a retry sends in a hold of its own.
|
|
1485
|
+
const callerHold = insideCallerSigningControlHold();
|
|
1486
|
+
let result;
|
|
1487
|
+
for (;;) {
|
|
1488
|
+
const answer = await this.#withProtocolState(() => this.#signedRequestWithSigningControl(method, route, body, callerHold));
|
|
1489
|
+
if (answer !== HANDED_BACK) {
|
|
1490
|
+
result = answer;
|
|
1491
|
+
break;
|
|
1492
|
+
}
|
|
1493
|
+
if (callerHold)
|
|
1494
|
+
throw new Error('relay_retry_after_recovery');
|
|
1495
|
+
handBackDeviceSigningControl();
|
|
1496
|
+
}
|
|
1497
|
+
if (method === 'POST' && route === '/agent-fabric/sessions')
|
|
1498
|
+
this.#sessionAccepted = true;
|
|
1499
|
+
if (!this.#durable)
|
|
1500
|
+
return result;
|
|
1501
|
+
if (route === '/agent-fabric/tasks/poll' && result.task && typeof result.task === 'object') {
|
|
1502
|
+
const seconds = Number(body?.leaseSeconds);
|
|
1503
|
+
this.#leaseMs = (Number.isFinite(seconds) && seconds > 0 ? seconds : 120) * 1000;
|
|
1504
|
+
taskLeases.set(this, Date.now() + this.#leaseMs);
|
|
1505
|
+
}
|
|
1506
|
+
else if (event === 'started' || event === 'lease_extended') {
|
|
1507
|
+
taskLeases.set(this, Date.now() + this.#leaseMs);
|
|
1508
|
+
}
|
|
1509
|
+
// A long-lived client, such as a workspace loop, also sends due orphans after its own requests, so they do not wait
|
|
1510
|
+
// for another session open (which has just sent them before its own POST). A one-shot command does not: a send
|
|
1511
|
+
// after its last call would keep its process running after it has its answer.
|
|
1512
|
+
if (this.#longLived && !(method === 'POST' && route === '/agent-fabric/sessions'))
|
|
1513
|
+
this.#sendDueOrphansLater();
|
|
1514
|
+
return result;
|
|
1515
|
+
}
|
|
1516
|
+
/** The lease length of the task this client last polled. */
|
|
1517
|
+
#leaseMs = 120_000;
|
|
1518
|
+
/** Whether a section that sends due orphans is waiting or running. */
|
|
1519
|
+
#orphanSection = false;
|
|
1520
|
+
/** Sends at most one due orphan in a signing-control section of its own, after the caller already has its answer. It
|
|
1521
|
+
* waits in the signing-control line like any root, so this client's next request may wait for that one send, but
|
|
1522
|
+
* never while a task lease is held in this process. It never runs inside the caller's hold of signing control. */
|
|
1523
|
+
#sendDueOrphansLater() {
|
|
1524
|
+
if (this.#orphanSection || !this.#orphanWorkDue(Date.now()) || taskLeaseHeld(Date.now()))
|
|
1525
|
+
return;
|
|
1526
|
+
this.#orphanSection = true;
|
|
1527
|
+
const send = () => this.#withProtocolState(async () => {
|
|
1528
|
+
if (!taskLeaseHeld(Date.now()))
|
|
1529
|
+
await this.#adoptOrphans();
|
|
1530
|
+
});
|
|
1531
|
+
void outsideDeviceSigningControl(() => protocolStateSections.exit(send))
|
|
1532
|
+
.catch(() => undefined) // A later request tries again; the caller already has its answer.
|
|
1533
|
+
.finally(() => { this.#orphanSection = false; });
|
|
1534
|
+
}
|
|
1535
|
+
get #durable() { return !this.#readOnly && !this.#isolatedSession; }
|
|
1536
|
+
#controlScope() {
|
|
1537
|
+
return this.#hostFence ? { signal: this.#hostFence.signal, current: async () => { await this.#assertHostScope(); return true; } } : undefined;
|
|
1538
|
+
}
|
|
1539
|
+
/** Runs work under device signing control. A root section of a durable client first reloads the device-wide
|
|
1540
|
+
* protocol state from the file; a nested call inside this client's live section reuses it without a reload. */
|
|
1541
|
+
async #withProtocolState(work) {
|
|
1542
|
+
const input = { config: this.config, hostScope: this.#controlScope() };
|
|
1543
|
+
const current = protocolStateSections.getStore();
|
|
1544
|
+
if (current?.client === this && current.active)
|
|
1545
|
+
return withDeviceSigningControl(input, work);
|
|
1546
|
+
return withDeviceSigningControl(input, () => {
|
|
1547
|
+
const section = { client: this, active: true };
|
|
1548
|
+
return protocolStateSections.run(section, async () => {
|
|
1549
|
+
try {
|
|
1550
|
+
if (this.#durable)
|
|
1551
|
+
await this.#reloadSharedState();
|
|
1552
|
+
return await work();
|
|
1553
|
+
}
|
|
1554
|
+
finally {
|
|
1555
|
+
section.active = false;
|
|
1556
|
+
}
|
|
1557
|
+
});
|
|
1558
|
+
});
|
|
1559
|
+
}
|
|
1560
|
+
/** The file wins for every other client's entry and for the recovered completion queue, except for a completion this
|
|
1561
|
+
* client recorded and no write has stored yet, which it keeps. A completion waiting in any entry, or one this client
|
|
1562
|
+
* has not written yet, joins the queue only while the queue stays within what a reader accepts; otherwise it waits
|
|
1563
|
+
* for room, in its entry or, until the next write puts it in this client's entry, in memory. This client's own
|
|
1564
|
+
* session, sequence and kept request are written only by this client, so they are never reloaded. */
|
|
1565
|
+
async #reloadSharedState() {
|
|
1566
|
+
let disk = null;
|
|
1567
|
+
try {
|
|
1568
|
+
const bytes = this.#hostFence ? await this.#hostFence.step(() => readFile(this.#statePath, 'utf8'))
|
|
1569
|
+
: await readFile(this.#statePath, 'utf8');
|
|
1570
|
+
const parsed = JSON.parse(bytes);
|
|
1571
|
+
assertProtocolState(parsed);
|
|
1572
|
+
disk = parsed;
|
|
1573
|
+
}
|
|
1574
|
+
catch (error) {
|
|
1575
|
+
if (error.code !== 'ENOENT') {
|
|
1576
|
+
await this.#hostFence?.assert();
|
|
1577
|
+
throw new Error('Relay protocol state is invalid; preserve the file for recovery and re-enroll if necessary.', { cause: error });
|
|
1578
|
+
}
|
|
1579
|
+
}
|
|
1580
|
+
const others = disk ? outboxesOf(disk) : {};
|
|
1581
|
+
const own = others[this.#ownerKey];
|
|
1582
|
+
delete others[this.#ownerKey];
|
|
1583
|
+
this.#otherOutboxes = others;
|
|
1584
|
+
const merged = [...disk?.recoveredTaskCompletions ?? []];
|
|
1585
|
+
/** Whether the completion is in the queue, adding it while there is room. */
|
|
1586
|
+
const listed = (item) => {
|
|
1587
|
+
if (merged.some(other => other.taskId === item.taskId))
|
|
1588
|
+
return true;
|
|
1589
|
+
if (merged.length >= RECOVERED_TASK_COMPLETIONS_MAX)
|
|
1590
|
+
return false;
|
|
1591
|
+
merged.push(item);
|
|
1592
|
+
return true;
|
|
1593
|
+
};
|
|
1594
|
+
this.#keptCompletions = (own?.keptCompletions ?? []).filter(item => !listed(item));
|
|
1595
|
+
for (const item of this.#unwrittenCompletions.values())
|
|
1596
|
+
listed(item);
|
|
1597
|
+
for (const entry of Object.values(others)) {
|
|
1598
|
+
const waiting = entry.keptCompletions?.filter(item => !listed(item));
|
|
1599
|
+
if (waiting?.length)
|
|
1600
|
+
entry.keptCompletions = waiting;
|
|
1601
|
+
else
|
|
1602
|
+
delete entry.keptCompletions;
|
|
1603
|
+
}
|
|
1604
|
+
this.#state.recoveredTaskCompletions = merged;
|
|
1605
|
+
// This client's session, sequence and kept request are its own, and only it signs in its session, so the file never
|
|
1606
|
+
// changes them, with one exception below: another client's answer to its kept request.
|
|
1607
|
+
const kept = this.#state.pending;
|
|
1608
|
+
if (!kept)
|
|
1609
|
+
return;
|
|
1610
|
+
if (own?.adoptedAt && (own.pending ? sameKeptRequest(own.pending, kept) : true)) {
|
|
1611
|
+
// Another client sent this client's kept request after it stood unchanged (see orphanDue). The file then holds
|
|
1612
|
+
// its outcome: no kept request once it was answered, with the answer, or the request as last sent. When the
|
|
1613
|
+
// answer is not there (never kept, or removed after ORPHAN_RETRY_INTERVAL_MS), this client keeps its signed
|
|
1614
|
+
// request, so a retry sends it again with the same message id and idempotency key instead of signing the call
|
|
1615
|
+
// again. That client sent it only exactly as signed, so no sequence in this session is new to this client.
|
|
1616
|
+
const outcome = own.adoptedOutcome;
|
|
1617
|
+
const answered = !own.pending && !!outcome && outcome.messageId === kept.headers['x-dharma-message-id'];
|
|
1618
|
+
if (answered) {
|
|
1619
|
+
this.#state.pending = null;
|
|
1620
|
+
this.#state.nextSequence = Math.max(this.#state.nextSequence, Number(kept.headers['x-dharma-sequence'] || 0) + 1);
|
|
1621
|
+
}
|
|
1622
|
+
else if (own.pending)
|
|
1623
|
+
this.#state.pending = own.pending;
|
|
1624
|
+
else
|
|
1625
|
+
this.#keptOnDisk = false;
|
|
1626
|
+
this.#adoptedOutcome = answered
|
|
1627
|
+
? { method: kept.method, pathname: kept.pathname, body: kept.body, status: outcome.status, response: outcome.response }
|
|
1628
|
+
: null;
|
|
1629
|
+
return;
|
|
1630
|
+
}
|
|
1631
|
+
if (own?.pending && sameKeptRequest(own.pending, kept))
|
|
1632
|
+
return;
|
|
1633
|
+
// The kept request left this client's entry, although no write by this client drops one: the entry was removed (by
|
|
1634
|
+
// an earlier version that rewrote the file without outboxes, or with the file), or came back in an older state (an
|
|
1635
|
+
// earlier version writes back the whole file it read when it opened) without it. This client keeps it, in its
|
|
1636
|
+
// session, and writes it back to its own entry before it sends it again exactly as signed. Another entry that holds
|
|
1637
|
+
// the same call (the top-level copy, moved to a migrated entry) gives it up, so only this client's entry holds it.
|
|
1638
|
+
this.#keptOnDisk = false;
|
|
1639
|
+
for (const [owner, entry] of Object.entries(others)) {
|
|
1640
|
+
if (entry.pending && sameKeptRequest(entry.pending, kept))
|
|
1641
|
+
others[owner] = { ...entry, pending: null };
|
|
1642
|
+
}
|
|
1158
1643
|
}
|
|
1159
|
-
|
|
1644
|
+
/** Whether the entries as last read, which each request of this client reads again, hold work for #adoptOrphans: a
|
|
1645
|
+
* due orphan it sends, or one it removes unsent because its owner is gone. A request only its owner can use, while
|
|
1646
|
+
* that owner can come back, is no such work. */
|
|
1647
|
+
#orphanWorkDue(now) {
|
|
1648
|
+
return Object.entries(this.#otherOutboxes).some(([owner, entry]) => orphanDue(owner, entry, now)
|
|
1649
|
+
&& (!ownerOnlyEntry(entry, now) || ownerGone(owner, entry)));
|
|
1650
|
+
}
|
|
1651
|
+
/** Sends the kept requests of orphaned entries (see orphanDue): at most ORPHAN_SENDS_PER_SECTION, and none after one
|
|
1652
|
+
* without a definite answer. Their outcome is never this caller's and is never thrown. Each entry is marked in the
|
|
1653
|
+
* file before its send, so when this client stops during the send, the next one waits ORPHAN_RETRY_INTERVAL_MS too.
|
|
1654
|
+
* An answered entry keeps no request; it holds the answer only when its owner may still read it (a workspace loop, or a process
|
|
1655
|
+
* that still runs), so a live owner takes the outcome from the file, and #persist drops that answer after
|
|
1656
|
+
* ORPHAN_RETRY_INTERVAL_MS. A request without a definite answer stays and is due again after
|
|
1657
|
+
* ORPHAN_RETRY_INTERVAL_MS. An orphan is sent only exactly as signed, so this client never signs in another client's
|
|
1658
|
+
* session. While more are due, this client's next request sends the next one. A kept request only its owner can use
|
|
1659
|
+
* (see isOwnerOnlyRequest), or one that could be sent only signed again (see cannotResendAsSigned), is never sent by
|
|
1660
|
+
* another client: it stays in its entry while its owner can come back (a workspace loop that restarts, or a process
|
|
1661
|
+
* that still runs), and is settled unsent once that owner is gone (a command that exited, or a migrated entry).
|
|
1662
|
+
* Returns whether it sent one. */
|
|
1663
|
+
async #adoptOrphans() {
|
|
1664
|
+
const now = Date.now();
|
|
1665
|
+
const due = Object.entries(this.#otherOutboxes).filter(([owner, entry]) => orphanDue(owner, entry, now));
|
|
1666
|
+
const ownerOnly = (entry) => ownerOnlyEntry(entry, now);
|
|
1667
|
+
let unwritten = false;
|
|
1668
|
+
for (const [owner, entry] of due) {
|
|
1669
|
+
if (!ownerOnly(entry) || !ownerGone(owner, entry))
|
|
1670
|
+
continue;
|
|
1671
|
+
entry.pending = null;
|
|
1672
|
+
entry.adoptedAt = entry.updatedAt = new Date().toISOString();
|
|
1673
|
+
delete entry.adoptedOutcome;
|
|
1674
|
+
unwritten = true;
|
|
1675
|
+
}
|
|
1676
|
+
let sent = 0;
|
|
1677
|
+
for (const [owner, entry] of due) {
|
|
1678
|
+
if (!entry.pending || ownerOnly(entry))
|
|
1679
|
+
continue;
|
|
1680
|
+
if (sent === ORPHAN_SENDS_PER_SECTION)
|
|
1681
|
+
break;
|
|
1682
|
+
sent += 1;
|
|
1683
|
+
const messageId = entry.pending.headers['x-dharma-message-id'] ?? '';
|
|
1684
|
+
const ownerReads = owner.startsWith('relay:') || ownerProcessAlive(entry.ownerPid);
|
|
1685
|
+
entry.adoptedAt = new Date().toISOString();
|
|
1686
|
+
delete entry.adoptedOutcome;
|
|
1687
|
+
await this.#persist();
|
|
1688
|
+
unwritten = false;
|
|
1689
|
+
try {
|
|
1690
|
+
await this.#sendPendingWithSigningControl(entry, (status, response) => {
|
|
1691
|
+
// Only an answer every reader accepts is kept; without one, the owner signs its retry again.
|
|
1692
|
+
const outcome = { messageId, status, response };
|
|
1693
|
+
if (ownerReads && isAdoptedOutcome(outcome))
|
|
1694
|
+
entry.adoptedOutcome = outcome;
|
|
1695
|
+
});
|
|
1696
|
+
}
|
|
1697
|
+
catch {
|
|
1698
|
+
await this.#assertHostScope();
|
|
1699
|
+
}
|
|
1700
|
+
entry.adoptedAt = new Date().toISOString();
|
|
1701
|
+
if (!entry.pending)
|
|
1702
|
+
entry.updatedAt = entry.adoptedAt;
|
|
1703
|
+
await this.#persist();
|
|
1704
|
+
if (entry.pending)
|
|
1705
|
+
break;
|
|
1706
|
+
}
|
|
1707
|
+
if (unwritten)
|
|
1708
|
+
await this.#persist();
|
|
1709
|
+
return sent > 0;
|
|
1710
|
+
}
|
|
1711
|
+
/** The call a due orphan was just sent to make room for in a full file. Its next hold signs it without sending
|
|
1712
|
+
* another orphan first, so a file that is still full refuses it, as before. */
|
|
1713
|
+
#madeRoomFor = null;
|
|
1714
|
+
/** One hold's part of a signed request: it makes at most one network send. It returns the caller's answer, whatever
|
|
1715
|
+
* its JSON body, or HANDED_BACK after it sent another kept request first (this client's own earlier call, or a due
|
|
1716
|
+
* orphan that makes room in a full file); the caller's request is then signed in a hold of its own. In a hold its
|
|
1717
|
+
* caller took itself (callerHold), it sends no orphan to make room, so only this client's own earlier call hands back. */
|
|
1718
|
+
async #signedRequestWithSigningControl(method, route, body, callerHold) {
|
|
1160
1719
|
await this.#assertHostScope();
|
|
1161
1720
|
if (!this.#state.sessionId)
|
|
1162
1721
|
throw new Error('Relay session is not open.');
|
|
1722
|
+
// An acknowledgement this client took over waits for its new session (see openSession). Until the server accepted
|
|
1723
|
+
// that session open, no other call is signed or replayed, so its request never takes the acknowledgement's place in
|
|
1724
|
+
// this client's entry: the session is open only once the server accepted it, and the next openSession opens it.
|
|
1725
|
+
if (this.#deferredAcknowledgement && !this.#sessionAccepted && !(method === 'POST' && route === '/agent-fabric/sessions')) {
|
|
1726
|
+
throw new Error('Relay session is not open.');
|
|
1727
|
+
}
|
|
1728
|
+
if (this.#deferredAcknowledgement && !(method === 'POST' && route === '/agent-fabric/sessions')) {
|
|
1729
|
+
// Once the session is accepted, the acknowledgement still keeps its place in this client's entry until it has had a
|
|
1730
|
+
// hold of its own. While its own section waits for the caller's hold to end, another call is neither signed nor
|
|
1731
|
+
// sent, so it never takes that place: it fails with relay_retry_after_recovery, and a retry signs it once that
|
|
1732
|
+
// section ran. Otherwise the acknowledgement is this client's earlier kept call, sent first, as below.
|
|
1733
|
+
if (this.#reissueWaiting)
|
|
1734
|
+
throw new Error('relay_retry_after_recovery');
|
|
1735
|
+
this.#state.pending = this.#deferredAcknowledgement;
|
|
1736
|
+
this.#deferredAcknowledgement = null;
|
|
1737
|
+
}
|
|
1163
1738
|
if (this.#readOnly) {
|
|
1164
1739
|
if (this.#state.pending)
|
|
1165
1740
|
throw new Error('relay_read_only_pending_request');
|
|
@@ -1172,6 +1747,15 @@ export class AgentFabricClient {
|
|
|
1172
1747
|
}
|
|
1173
1748
|
const pathname = `/api/v1/orgs/${encodeURIComponent(this.config.organizationId)}${route}`;
|
|
1174
1749
|
const serialized = method === 'GET' ? '' : canonicalize(body);
|
|
1750
|
+
const adopted = this.#adoptedOutcome;
|
|
1751
|
+
this.#adoptedOutcome = null;
|
|
1752
|
+
if (adopted && !this.#state.pending && adopted.method === method && adopted.pathname === pathname
|
|
1753
|
+
&& adopted.body === serialized) {
|
|
1754
|
+
// Another client delivered this call's kept request: the retry gets that answer, as a recovery of its own would.
|
|
1755
|
+
if (adopted.status >= 200 && adopted.status < 300)
|
|
1756
|
+
return adopted.response;
|
|
1757
|
+
throw new AgentFabricRequestError(adopted.response, adopted.status, true);
|
|
1758
|
+
}
|
|
1175
1759
|
if (this.#state.pending && isContentBearingPath(this.#state.pending.pathname)) {
|
|
1176
1760
|
// An ambiguous content delivery is never replayed implicitly. The next
|
|
1177
1761
|
// explicit caller must rebuild the capsule after rechecking disclosure
|
|
@@ -1185,10 +1769,22 @@ export class AgentFabricClient {
|
|
|
1185
1769
|
const sameRequest = this.#state.pending.method === method
|
|
1186
1770
|
&& this.#state.pending.pathname === pathname
|
|
1187
1771
|
&& this.#state.pending.body === serialized;
|
|
1188
|
-
const recovered = await this.#
|
|
1189
|
-
// Recovery acknowledges only its original caller's request identity.
|
|
1190
|
-
|
|
1191
|
-
|
|
1772
|
+
const recovered = await this.#sendPendingWithSigningControl(this.#state);
|
|
1773
|
+
// Recovery acknowledges only its original caller's request identity. Another call is signed in a hold of its own.
|
|
1774
|
+
return sameRequest ? recovered : HANDED_BACK;
|
|
1775
|
+
}
|
|
1776
|
+
// In a file full of other clients' kept requests, a due orphan is sent first to make room for this one, in a hold
|
|
1777
|
+
// of its own. A completion the queue has no room for is refused before that. In a hold its caller took itself, no
|
|
1778
|
+
// orphan is sent, and the full file refuses this request (relay_protocol_state_outbox_full).
|
|
1779
|
+
const call = `${method} ${pathname} ${serialized}`, unsigned = { method, pathname, body: serialized, headers: {} };
|
|
1780
|
+
const madeRoom = this.#madeRoomFor === call;
|
|
1781
|
+
this.#madeRoomFor = null;
|
|
1782
|
+
this.#assertRecoveredTaskCompletionCapacity(unsigned);
|
|
1783
|
+
if (!madeRoom && !callerHold && this.#durable && !isExplicitlyRebuiltPath(pathname) && !isSessionOpen(unsigned)
|
|
1784
|
+
&& Object.values(this.#otherOutboxes).filter(entry => entry.pending).length >= OUTBOX_ENTRIES_MAX
|
|
1785
|
+
&& await this.#adoptOrphans()) {
|
|
1786
|
+
this.#madeRoomFor = call;
|
|
1787
|
+
return HANDED_BACK;
|
|
1192
1788
|
}
|
|
1193
1789
|
const timestamp = new Date().toISOString();
|
|
1194
1790
|
await this.#assertHostScope();
|
|
@@ -1214,30 +1810,32 @@ export class AgentFabricClient {
|
|
|
1214
1810
|
'x-dharma-sequence': String(sequence), 'x-dharma-signature': signature,
|
|
1215
1811
|
},
|
|
1216
1812
|
};
|
|
1217
|
-
this.#assertRecoveredTaskCompletionCapacity(pending);
|
|
1218
1813
|
this.#state.pending = pending;
|
|
1219
|
-
await this.#persist();
|
|
1220
|
-
return this.#sendPending();
|
|
1221
|
-
}
|
|
1222
|
-
async #sendPending() {
|
|
1223
|
-
const hostScope = this.#hostFence ? { signal: this.#hostFence.signal, current: async () => { await this.#assertHostScope(); return true; } } : undefined;
|
|
1224
|
-
return withDeviceSigningControl({ config: this.config, hostScope }, () => this.#sendPendingWithSigningControl());
|
|
1225
|
-
}
|
|
1226
|
-
async #sendPendingWithSigningControl() {
|
|
1227
|
-
await this.#assertHostScope();
|
|
1228
|
-
const pending = this.#state.pending;
|
|
1229
|
-
if (!pending)
|
|
1230
|
-
throw new Error('No pending protocol request.');
|
|
1231
1814
|
try {
|
|
1232
|
-
this.#
|
|
1815
|
+
await this.#persist();
|
|
1233
1816
|
}
|
|
1234
1817
|
catch (error) {
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1818
|
+
// A request the full file refused is never sent, so it is not kept for a later replay either.
|
|
1819
|
+
if (error.message === 'relay_protocol_state_outbox_full')
|
|
1820
|
+
this.#state.pending = null;
|
|
1238
1821
|
throw error;
|
|
1239
1822
|
}
|
|
1240
|
-
|
|
1823
|
+
return this.#sendPendingWithSigningControl(this.#state);
|
|
1824
|
+
}
|
|
1825
|
+
/** Sends the kept request of one outbox entry: this client's own, or an orphan in #adoptOrphans. answered sees a
|
|
1826
|
+
* definite answer before the entry is written without its request. This client's own kept request is in its entry
|
|
1827
|
+
* in the file before it is sent: when the entry left the file (see #reloadSharedState), it is written back first, so
|
|
1828
|
+
* after a failed send, a crash or a new client with the same owner key, it is still there to send again. */
|
|
1829
|
+
async #sendPendingWithSigningControl(slot, answered) {
|
|
1830
|
+
await this.#assertHostScope();
|
|
1831
|
+
const pending = slot.pending;
|
|
1832
|
+
if (!pending)
|
|
1833
|
+
throw new Error('No pending protocol request.');
|
|
1834
|
+
if (slot === this.#state && !this.#keptOnDisk)
|
|
1835
|
+
await this.#persist();
|
|
1836
|
+
// A kept completion is sent again even when the queue is full: the server may already have accepted it, and its
|
|
1837
|
+
// receipt then waits in this client's entry for room (see #recordRecoveredTaskCompletion).
|
|
1838
|
+
if (ACTION_ACKNOWLEDGEMENT_PATH.test(pending.pathname)) {
|
|
1241
1839
|
let acknowledgement;
|
|
1242
1840
|
try {
|
|
1243
1841
|
acknowledgement = JSON.parse(pending.body);
|
|
@@ -1247,17 +1845,24 @@ export class AgentFabricClient {
|
|
|
1247
1845
|
}
|
|
1248
1846
|
const contract = validateActionDecisionAcknowledgementContract(acknowledgement);
|
|
1249
1847
|
if (!contract.ok) {
|
|
1250
|
-
|
|
1251
|
-
|
|
1848
|
+
slot.nextSequence = Math.max(slot.nextSequence + 1, Number(pending.headers['x-dharma-sequence'] || 0) + 1);
|
|
1849
|
+
slot.pending = null;
|
|
1252
1850
|
await this.#persist();
|
|
1253
1851
|
throw new Error('Stale action acknowledgement is invalid and was discarded.');
|
|
1254
1852
|
}
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1853
|
+
if (acknowledgementTimeStale(acknowledgement.acknowledgedAt, Date.now())) {
|
|
1854
|
+
// Only the process that opened a session signs in it, from the sequences it holds in memory: this client signs
|
|
1855
|
+
// the acknowledgement again only as its own kept request, in its own open session, at its next sequence. It
|
|
1856
|
+
// keeps the idempotency key, so the server can tell it is the same acknowledgement. Every other client sends a
|
|
1857
|
+
// kept request only exactly as signed (see #adoptOrphans), so nothing else signs here.
|
|
1858
|
+
const sessionId = this.#state.sessionId;
|
|
1859
|
+
if (slot !== this.#state || !sessionId) {
|
|
1860
|
+
throw new Error('Stale action acknowledgement can be signed again only in its owner\'s own session.');
|
|
1861
|
+
}
|
|
1259
1862
|
const refreshed = refreshActionDecisionAcknowledgement(acknowledgement);
|
|
1260
|
-
|
|
1863
|
+
// The next sequence of this session, above the acknowledgement's own when it was signed in this session too.
|
|
1864
|
+
const sequence = Math.max(this.#state.nextSequence, pending.headers['x-dharma-session-id'] === sessionId
|
|
1865
|
+
? Number(pending.headers['x-dharma-sequence'] || 0) + 1 : 0);
|
|
1261
1866
|
const timestamp = new Date().toISOString();
|
|
1262
1867
|
const messageId = randomUUID();
|
|
1263
1868
|
const nonce = randomBytes(24).toString('base64url');
|
|
@@ -1266,19 +1871,24 @@ export class AgentFabricClient {
|
|
|
1266
1871
|
bodyHash: `sha256:${sha256(pending.body)}`,
|
|
1267
1872
|
deviceId: this.config.deviceId, messageId, method: pending.method, nonce,
|
|
1268
1873
|
organizationId: this.config.organizationId, pathname: pending.pathname,
|
|
1269
|
-
sequence, sessionId
|
|
1874
|
+
sequence, sessionId, timestamp,
|
|
1270
1875
|
}), 'utf8');
|
|
1876
|
+
pending.headers['x-dharma-session-id'] = sessionId;
|
|
1271
1877
|
pending.headers['x-dharma-message-id'] = messageId;
|
|
1272
1878
|
pending.headers['x-dharma-timestamp'] = timestamp;
|
|
1273
1879
|
pending.headers['x-dharma-nonce'] = nonce;
|
|
1274
1880
|
pending.headers['x-dharma-sequence'] = String(sequence);
|
|
1275
1881
|
pending.headers['x-dharma-signature'] = sign(null, signingPayload, { key: this.#privateJwk, format: 'jwk' }).toString('base64url');
|
|
1276
|
-
|
|
1882
|
+
slot.nextSequence = sequence;
|
|
1277
1883
|
await this.#persist();
|
|
1278
1884
|
}
|
|
1279
1885
|
}
|
|
1280
|
-
|
|
1281
|
-
|
|
1886
|
+
if (signatureTooOld(pending, Date.now())) {
|
|
1887
|
+
// Only this client's own kept request gets a new time and signature, with its message id, session, sequence,
|
|
1888
|
+
// body and idempotency key unchanged. Another client sends a kept request only byte for byte as signed: one too
|
|
1889
|
+
// old for that stays in its entry for its owner (see ownerOnlyEntry), so nothing is signed or sent here.
|
|
1890
|
+
if (slot !== this.#state)
|
|
1891
|
+
throw new Error('A kept request too old to send as signed is signed again only by its owner.');
|
|
1282
1892
|
const timestamp = new Date().toISOString();
|
|
1283
1893
|
const nonce = randomBytes(24).toString('base64url');
|
|
1284
1894
|
const messageId = pending.headers['x-dharma-message-id'];
|
|
@@ -1346,15 +1956,17 @@ export class AgentFabricClient {
|
|
|
1346
1956
|
// signed request, as do upstream failures whose commit state is unknown.
|
|
1347
1957
|
const definitive = status >= 400 && status < 500 && status !== 408 && status !== 429;
|
|
1348
1958
|
if (definitive) {
|
|
1349
|
-
|
|
1350
|
-
|
|
1959
|
+
answered?.(status, body);
|
|
1960
|
+
slot.nextSequence += 1;
|
|
1961
|
+
slot.pending = null;
|
|
1351
1962
|
await this.#persist();
|
|
1352
1963
|
}
|
|
1353
1964
|
throw new AgentFabricRequestError(body, status, definitive);
|
|
1354
1965
|
}
|
|
1355
1966
|
this.#recordRecoveredTaskCompletion(pending, body);
|
|
1356
|
-
|
|
1357
|
-
|
|
1967
|
+
answered?.(status, body);
|
|
1968
|
+
slot.nextSequence += 1;
|
|
1969
|
+
slot.pending = null;
|
|
1358
1970
|
await this.#persist();
|
|
1359
1971
|
return body;
|
|
1360
1972
|
}
|
|
@@ -1392,19 +2004,19 @@ export class AgentFabricClient {
|
|
|
1392
2004
|
throw new Error('Task completion acknowledgement is missing a valid durable receipt.');
|
|
1393
2005
|
}
|
|
1394
2006
|
const current = this.#state.recoveredTaskCompletions || [];
|
|
1395
|
-
const existing = current
|
|
2007
|
+
const existing = [...current, ...this.#keptCompletions, ...this.#unwrittenCompletions.values()]
|
|
2008
|
+
.find((item) => item.taskId === taskId);
|
|
1396
2009
|
if (existing) {
|
|
1397
2010
|
if (existing.receiptHash !== receiptHash || existing.trajectoryCapsuleHash !== trajectoryCapsuleHash) {
|
|
1398
2011
|
throw new Error('Recovered task completion conflicts with its durable receipt.');
|
|
1399
2012
|
}
|
|
1400
2013
|
return;
|
|
1401
2014
|
}
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
...current,
|
|
1406
|
-
|
|
1407
|
-
];
|
|
2015
|
+
const completion = { taskId, trajectoryCapsuleHash, receiptHash, recoveredAt: new Date().toISOString() };
|
|
2016
|
+
// An accepted completion is never dropped: without room in the queue it waits in this client's entry (see #persist).
|
|
2017
|
+
if (current.length < RECOVERED_TASK_COMPLETIONS_MAX)
|
|
2018
|
+
this.#state.recoveredTaskCompletions = [...current, completion];
|
|
2019
|
+
this.#unwrittenCompletions.set(taskId, completion);
|
|
1408
2020
|
}
|
|
1409
2021
|
#assertRecoveredTaskCompletionCapacity(pending) {
|
|
1410
2022
|
const route = pending.pathname.match(/\/agent-fabric\/tasks\/([^/]+)\/events$/);
|
|
@@ -1421,7 +2033,7 @@ export class AgentFabricClient {
|
|
|
1421
2033
|
return;
|
|
1422
2034
|
const taskId = decodeURIComponent(route[1]);
|
|
1423
2035
|
const current = this.#state.recoveredTaskCompletions || [];
|
|
1424
|
-
if (!current.some((item) => item.taskId === taskId) && current.length >=
|
|
2036
|
+
if (!current.some((item) => item.taskId === taskId) && current.length >= RECOVERED_TASK_COMPLETIONS_MAX) {
|
|
1425
2037
|
throw new Error('Recovered task completion queue is full; acknowledge stored completions before sending another completion.');
|
|
1426
2038
|
}
|
|
1427
2039
|
}
|
|
@@ -1437,18 +2049,24 @@ export class AgentFabricClient {
|
|
|
1437
2049
|
if (settled)
|
|
1438
2050
|
return;
|
|
1439
2051
|
settled = true;
|
|
2052
|
+
clearTimeout(connectTimeout);
|
|
1440
2053
|
clearTimeout(timeout);
|
|
1441
2054
|
this.#hostFence?.signal.removeEventListener('abort', abort);
|
|
1442
2055
|
callback();
|
|
1443
2056
|
if (closeSocket && socket.readyState < WebSocket.CLOSING)
|
|
1444
2057
|
socket.close(1000);
|
|
1445
2058
|
};
|
|
1446
|
-
const timeout = setTimeout(() => finish(() => reject(new Error('Relay response timed out.'))), RELAY_ACKNOWLEDGEMENT_TIMEOUT_MS);
|
|
2059
|
+
const timeout = setTimeout(() => finish(() => reject(Object.assign(new Error('Relay response timed out.'), { code: 'relay_response_timeout' }))), RELAY_ACKNOWLEDGEMENT_TIMEOUT_MS);
|
|
2060
|
+
// #512: before the socket opens this attempt has sent nothing, so this limit fails it cleanly. The kept signed
|
|
2061
|
+
// request is sent on the next attempt, exactly as after any other failure that keeps it (an earlier attempt
|
|
2062
|
+
// may already have sent it). It never shortens the window after send.
|
|
2063
|
+
const connectTimeout = setTimeout(() => finish(() => reject(Object.assign(new Error('Relay connection timed out before the request was sent.'), { code: 'relay_connect_timeout' }))), RELAY_CONNECT_TIMEOUT_MS);
|
|
1447
2064
|
const abort = () => finish(() => reject(new Error('relay_host_scope_unavailable')));
|
|
1448
2065
|
this.#hostFence?.signal.addEventListener('abort', abort, { once: true });
|
|
1449
2066
|
if (this.#hostFence?.signal.aborted)
|
|
1450
2067
|
abort();
|
|
1451
2068
|
socket.addEventListener('open', () => {
|
|
2069
|
+
clearTimeout(connectTimeout);
|
|
1452
2070
|
const send = () => {
|
|
1453
2071
|
if (!settled)
|
|
1454
2072
|
socket.send(JSON.stringify({
|
|
@@ -1485,17 +2103,82 @@ export class AgentFabricClient {
|
|
|
1485
2103
|
await this.#assertHostScope();
|
|
1486
2104
|
return;
|
|
1487
2105
|
}
|
|
2106
|
+
const section = protocolStateSections.getStore();
|
|
2107
|
+
if (section?.client !== this || !section.active)
|
|
2108
|
+
throw new Error('relay_protocol_state_unserialized');
|
|
1488
2109
|
await this.#assertHostScope();
|
|
1489
2110
|
// Authorized content and state-dependent repository connects are rebuilt
|
|
1490
2111
|
// from their governed source. Do not retain them as an automatic replay
|
|
1491
2112
|
// that can outlive consent or the workspace registration they depend on.
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
2113
|
+
// Each other client's entry is written back as read while it holds a kept
|
|
2114
|
+
// request, kept completions, or an answer another client got for it in the
|
|
2115
|
+
// last ORPHAN_RETRY_INTERVAL_MS (its owner may still retry the call); every
|
|
2116
|
+
// other entry is dropped, so entries of clients that come and go never fill
|
|
2117
|
+
// the file. A kept request is never dropped.
|
|
2118
|
+
const outboxes = {};
|
|
2119
|
+
const now = Date.now();
|
|
2120
|
+
for (const [owner, read] of Object.entries(this.#otherOutboxes)) {
|
|
2121
|
+
let entry = durableOutboxEntry(read);
|
|
2122
|
+
if (entry.adoptedOutcome && !recentlyAdopted(entry, now)) {
|
|
2123
|
+
const { adoptedOutcome: _answer, ...rest } = entry;
|
|
2124
|
+
entry = rest;
|
|
2125
|
+
}
|
|
2126
|
+
if (entry.pending || entry.keptCompletions || entry.adoptedOutcome)
|
|
2127
|
+
outboxes[owner] = entry;
|
|
2128
|
+
}
|
|
2129
|
+
// A file never holds more entries than a reader accepts. Only a new kept request an earlier version left at the
|
|
2130
|
+
// top level of a full file can exceed them (see outboxesOf); it stays at the top level until it is sent.
|
|
2131
|
+
let leftover;
|
|
2132
|
+
const migrated = Object.keys(outboxes).filter(owner => owner.startsWith('migrated:')).at(-1); // outboxesOf adds it last
|
|
2133
|
+
if (Object.keys(outboxes).length > OUTBOX_ENTRIES_MAX && migrated) {
|
|
2134
|
+
leftover = outboxes[migrated];
|
|
2135
|
+
delete outboxes[migrated];
|
|
2136
|
+
}
|
|
2137
|
+
const { sessionId, nextSequence } = this.#state;
|
|
2138
|
+
// An acknowledgement kept from an earlier session while a new one opens stays in this client's entry, also while
|
|
2139
|
+
// the session open is in flight: a kept session open matters only to this client, whose next open replaces it,
|
|
2140
|
+
// while the acknowledgement must reach the next process with this owner key if this one never re-issues it.
|
|
2141
|
+
const deferred = this.#deferredAcknowledgement;
|
|
2142
|
+
const pending = deferred && (!this.#state.pending || isSessionOpen(this.#state.pending)) ? deferred : this.#state.pending;
|
|
2143
|
+
// A completion this client recorded that the queue has no room for waits in its own entry, which is then kept.
|
|
2144
|
+
const listed = this.#state.recoveredTaskCompletions ?? [];
|
|
2145
|
+
const waiting = [...this.#keptCompletions, ...[...this.#unwrittenCompletions.values()].filter(item => !listed.some(other => other.taskId === item.taskId) && !this.#keptCompletions.some(other => other.taskId === item.taskId))];
|
|
2146
|
+
// The next sequence is written only because the file format has it; no client reads another's.
|
|
2147
|
+
const own = durableOutboxEntry({ sessionId, nextSequence, pending, ownerPid: process.pid,
|
|
2148
|
+
updatedAt: new Date().toISOString(), ...(waiting.length ? { keptCompletions: waiting } : {}) });
|
|
2149
|
+
const room = OUTBOX_ENTRIES_MAX - Object.keys(outboxes).length;
|
|
2150
|
+
if (room < 1 && (waiting.length || (own.pending && !isSessionOpen(own.pending)))) {
|
|
2151
|
+
throw new Error('relay_protocol_state_outbox_full');
|
|
2152
|
+
}
|
|
2153
|
+
if (room > 0)
|
|
2154
|
+
outboxes[this.#ownerKey] = own;
|
|
2155
|
+
// An earlier version reads only the top-level session, sequence and kept request, and its session open replays
|
|
2156
|
+
// that request and then writes a file without outboxes. So the oldest kept request that is already due (see
|
|
2157
|
+
// orphanDue) also goes there, exactly as signed: after a rollback the earlier version still sends that one, with
|
|
2158
|
+
// the same message id. A live client's fresh request is never handed to it, nor any request another client never
|
|
2159
|
+
// sends (see ownerOnlyEntry), nor an action acknowledgement: the earlier version would sign either again in its
|
|
2160
|
+
// owner's session. That version gives a request signed more than 4 minutes ago a new time and signature, so a
|
|
2161
|
+
// request copied here was signed within the last 4 minutes, and it replays that one byte for byte. In practice that
|
|
2162
|
+
// is a command's request after the command exited, since any other entry is due only after 10 minutes unchanged.
|
|
2163
|
+
// outboxesOf never takes this copy for another orphan.
|
|
2164
|
+
const [copy] = leftover ? [leftover] : Object.entries(outboxes)
|
|
2165
|
+
.filter(([owner, entry]) => entry.pending && orphanDue(owner, entry, now) && !ownerOnlyEntry(entry, now)
|
|
2166
|
+
&& !ACTION_ACKNOWLEDGEMENT_PATH.test(entry.pending.pathname))
|
|
2167
|
+
.map(([, entry]) => entry)
|
|
2168
|
+
.sort((a, b) => Date.parse(a.updatedAt) - Date.parse(b.updatedAt));
|
|
2169
|
+
await atomicJson(this.#statePath, {
|
|
2170
|
+
schema: 'dharma.protocol-state/v1', sessionId: copy?.sessionId ?? null, nextSequence: copy?.nextSequence ?? 1,
|
|
2171
|
+
pending: copy?.pending ?? null, recoveredTaskCompletions: this.#state.recoveredTaskCompletions ?? [], outboxes,
|
|
2172
|
+
}, this.#hostFence);
|
|
2173
|
+
// Every completion this client recorded is now in the queue or in its own entry.
|
|
2174
|
+
this.#unwrittenCompletions.clear();
|
|
2175
|
+
this.#keptCompletions = waiting;
|
|
2176
|
+
this.#keptOnDisk = !!outboxes[this.#ownerKey]?.pending;
|
|
1496
2177
|
}
|
|
1497
2178
|
}
|
|
1498
2179
|
export const RELAY_ACKNOWLEDGEMENT_TIMEOUT_MS = 90_000;
|
|
2180
|
+
/** Limit for the relay connection to open. Nothing is sent before it opens; the acknowledgement window is unchanged. */
|
|
2181
|
+
export const RELAY_CONNECT_TIMEOUT_MS = 30_000;
|
|
1499
2182
|
// A date-time the keyset contract accepts (including a lowercase or space separator); sent normalized to ISO.
|
|
1500
2183
|
const TELEMETRY_INSTANT = /^\d{4}-\d{2}-\d{2}[Tt ]\d{2}:\d{2}:\d{2}(?:\.\d{1,9})?(?:[Zz]|[+-]\d{2}:\d{2})$/;
|
|
1501
2184
|
export function deviceKeysetTelemetry(config, options = {}) {
|