@dharma-ai-labs/agent-fabric-relay-client 0.2.38 → 0.2.40

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,3 +1,4 @@
1
+ import { AsyncLocalStorage } from 'node:async_hooks';
1
2
  import { createHash, createPublicKey, generateKeyPairSync, randomBytes, randomUUID, sign } from 'node:crypto';
2
3
  import { mkdir, open, readFile, rename, rm } from 'node:fs/promises';
3
4
  import { dirname, resolve } from 'node:path';
@@ -9,13 +10,41 @@ import { HostOperationFence } from './hostOperationScope.js';
9
10
  import { commitSigningRecoveryJournal, resumeSigningRecoveryJournal } from './signingRecoveryJournal.js';
10
11
  import { checkedRecoveryHostContext, retrieveSigningRecoveryDelivery } from './signingRecoveryClient.js';
11
12
  import { planSigningRecovery } from './signingRecoveryPlan.js';
12
- import { withDeviceSigningControl } from './deviceSigningControl.js';
13
- export { withDeviceSigningControl } from './deviceSigningControl.js';
13
+ import { handBackDeviceSigningControl, insideCallerSigningControlHold, insideDeviceSigningControlHold, outsideDeviceSigningControl, withDeviceSigningControl } from './deviceSigningControl.js';
14
+ export { withDeviceSigningControl, withDeviceSigningLineSignal, withSigningControlPerSend } from './deviceSigningControl.js';
14
15
  export { SigningRecoveryClientError } from './signingRecoveryClient.js';
15
16
  export { progressPreparedDeviceSigningRecoveryOffer, SigningRecoveryOfferClientError } from './signingRecoveryOfferClient.js';
16
17
  export { prepareOwnedSigningRecovery, submitOwnedSigningRecoveryPreparation, progressOwnedSigningRecoveryOffer, completeOwnedSigningRecovery, inspectOwnedSigningRecovery, disposeOwnedSigningRecovery, OwnedSigningRecoveryError, prepareOwnedSigningMaintenance, readOwnedSigningMaintenanceContext } from './signingRecoveryHost.js';
17
18
  export { claimSetupReference, setupClaimSourceRegistration, setupClaimFailureMessage } from './setupClaim.js';
18
19
  export { planDeviceSigningRecovery, planDeviceSigningMaintenance } from './signingRecoveryPlan.js';
20
+ /** The most entries in one file. Another client's entry stays only while it holds a kept request, kept completions or
21
+ * an answer of the last ORPHAN_RETRY_INTERVAL_MS (see #persist). */
22
+ const OUTBOX_ENTRIES_MAX = 256;
23
+ /** The most recovered task completions in one file. */
24
+ const RECOVERED_TASK_COMPLETIONS_MAX = 1_000;
25
+ /** An entry whose kept request stood unchanged this long is orphaned, and another client sends an orphaned request
26
+ * that got no definite answer again only after this interval. */
27
+ const ORPHAN_RETRY_INTERVAL_MS = 10 * 60_000;
28
+ /** Orphans one hold of signing control sends, so another process that waits for the lock file waits for at most one
29
+ * such send. While more are due, the client's next request sends the next one, in a section of its own. */
30
+ const ORPHAN_SENDS_PER_SECTION = 1;
31
+ /** What one hold's part of a signed request returns after it sent another kept request first, so that the caller's
32
+ * request is signed in a hold of its own. Any answer the server gives, a JSON null among them, is the caller's. */
33
+ const HANDED_BACK = Symbol('handed_back');
34
+ /** Task leases held in this process, by client, until when they run unless extended. While one is held, no client of
35
+ * this process sends an orphan after its own request, so a lease heartbeat never waits behind such a send. */
36
+ const taskLeases = new Map();
37
+ function taskLeaseHeld(now) {
38
+ for (const [client, until] of taskLeases)
39
+ if (until <= now)
40
+ taskLeases.delete(client);
41
+ return taskLeases.size > 0;
42
+ }
43
+ const OWNER_KEY_PATTERN = /^(?:relay|cli|migrated):[A-Za-z0-9_.:-]{1,128}$/;
44
+ /** Part of every default owner key, so a later process that gets the same pid never takes over an earlier entry. */
45
+ const PROCESS_INSTANCE = randomBytes(9).toString('base64url');
46
+ let clientInstances = 0;
47
+ const protocolStateSections = new AsyncLocalStorage();
19
48
  const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
20
49
  function isRecoveredTaskCompletion(value) {
21
50
  if (!value || typeof value !== 'object' || Array.isArray(value))
@@ -26,29 +55,181 @@ function isRecoveredTaskCompletion(value) {
26
55
  && /^sha256:[a-f0-9]{64}$/.test(String(item.receiptHash || ''))
27
56
  && Number.isFinite(Date.parse(String(item.recoveredAt || '')));
28
57
  }
29
- function assertProtocolState(value) {
58
+ function isPendingRequest(value) {
30
59
  if (!value || typeof value !== 'object' || Array.isArray(value))
31
- throw new Error('Relay protocol state is invalid.');
32
- const state = value;
33
- const recovered = state.recoveredTaskCompletions;
34
- const pending = state.pending;
35
- const pendingValid = pending === null || (typeof pending === 'object' && !Array.isArray(pending)
36
- && (pending.method === 'POST' || pending.method === 'GET')
60
+ return false;
61
+ const pending = value;
62
+ return (pending.method === 'POST' || pending.method === 'GET')
37
63
  && typeof pending.pathname === 'string' && pending.pathname.startsWith('/')
38
64
  && typeof pending.body === 'string'
39
65
  && pending.headers !== null && typeof pending.headers === 'object' && !Array.isArray(pending.headers)
40
66
  && Object.entries(pending.headers)
41
- .every(([name, header]) => name.length > 0 && typeof header === 'string'));
67
+ .every(([name, header]) => name.length > 0 && typeof header === 'string');
68
+ }
69
+ const isInstant = (value) => typeof value === 'string' && Number.isFinite(Date.parse(value));
70
+ function isAdoptedOutcome(value) {
71
+ if (!value || typeof value !== 'object' || Array.isArray(value))
72
+ return false;
73
+ const outcome = value;
74
+ const status = Number(outcome.status);
75
+ return typeof outcome.messageId === 'string' && outcome.messageId.length > 0 && Number.isSafeInteger(outcome.status)
76
+ && ((status >= 200 && status < 300) || (status >= 400 && status < 500 && status !== 408 && status !== 429))
77
+ && !!outcome.response && typeof outcome.response === 'object' && !Array.isArray(outcome.response);
78
+ }
79
+ function isOutboxEntry(value) {
80
+ if (!value || typeof value !== 'object' || Array.isArray(value))
81
+ return false;
82
+ const entry = value;
83
+ return (entry.sessionId === null || typeof entry.sessionId === 'string')
84
+ && Number.isSafeInteger(entry.nextSequence) && Number(entry.nextSequence) >= 1
85
+ && (entry.pending === null || isPendingRequest(entry.pending))
86
+ && Number.isSafeInteger(entry.ownerPid) && Number(entry.ownerPid) >= 0
87
+ && isInstant(entry.updatedAt) && (entry.adoptedAt === undefined || isInstant(entry.adoptedAt))
88
+ && (entry.adoptedOutcome === undefined || (entry.adoptedAt !== undefined && isAdoptedOutcome(entry.adoptedOutcome)))
89
+ && (entry.keptCompletions === undefined || (Array.isArray(entry.keptCompletions)
90
+ && entry.keptCompletions.length <= RECOVERED_TASK_COMPLETIONS_MAX && entry.keptCompletions.every(isRecoveredTaskCompletion)))
91
+ // A marker an earlier build of this version wrote; it is still read, and has no effect.
92
+ && (entry.asSignedOnly === undefined || entry.asSignedOnly === true);
93
+ }
94
+ function assertProtocolState(value) {
95
+ if (!value || typeof value !== 'object' || Array.isArray(value))
96
+ throw new Error('Relay protocol state is invalid.');
97
+ const state = value;
98
+ const recovered = state.recoveredTaskCompletions;
99
+ const outboxes = state.outboxes;
42
100
  if (state.schema !== 'dharma.protocol-state/v1'
43
101
  || !(state.sessionId === null || typeof state.sessionId === 'string')
44
102
  || !Number.isSafeInteger(state.nextSequence) || Number(state.nextSequence) < 1
45
- || !pendingValid
103
+ || !(state.pending === null || isPendingRequest(state.pending))
104
+ || (outboxes !== undefined && (!outboxes || typeof outboxes !== 'object' || Array.isArray(outboxes)
105
+ || Object.keys(outboxes).length > OUTBOX_ENTRIES_MAX
106
+ || Object.entries(outboxes)
107
+ .some(([owner, entry]) => !OWNER_KEY_PATTERN.test(owner) || !isOutboxEntry(entry))))
46
108
  || (recovered !== undefined && (!Array.isArray(recovered)
47
- || recovered.length > 1_000
109
+ || recovered.length > RECOVERED_TASK_COMPLETIONS_MAX
48
110
  || recovered.some((item) => !isRecoveredTaskCompletion(item))))) {
49
111
  throw new Error('Relay protocol state is invalid.');
50
112
  }
51
113
  }
114
+ /** Every outbox entry of a file. The kept request of an earlier single-slot file becomes an orphan keyed by its
115
+ * session; a request that is always rebuilt is dropped as before. Once due, it can be sent only signed again, which
116
+ * only the earlier process could do, so it is settled unsent (see ownerOnlyEntry). Such an entry has
117
+ * no owner process, so it counts as unchanged from when its request was last signed or stamped: an earlier version
118
+ * that still runs keeps 10 minutes to send its own request (see orphanDue). A top-level request with the message id of
119
+ * an entry's kept request is that entry's copy (see #persist), not another orphan. The top-level session and next
120
+ * sequence are never read: only the process that opened a session signs in it, from the sequence it holds in memory. */
121
+ function outboxesOf(state) {
122
+ const outboxes = {};
123
+ for (const [owner, entry] of Object.entries(state.outboxes ?? {}))
124
+ outboxes[owner] = durableOutboxEntry(entry);
125
+ const pending = state.pending;
126
+ const messageId = pending?.headers['x-dharma-message-id'];
127
+ const copied = !!messageId && Object.values(outboxes).some(entry => entry.pending?.headers['x-dharma-message-id'] === messageId);
128
+ if (pending && !copied && !isExplicitlyRebuiltPath(pending.pathname) && !isSessionOpen(pending)) {
129
+ const session = pending.headers['x-dharma-session-id'] || state.sessionId || '';
130
+ const key = `migrated:${/^[A-Za-z0-9_-]{1,128}$/.test(session) ? session : sha256(session).slice(0, 32)}`;
131
+ const signedAt = Date.parse(pending.headers['x-dharma-timestamp'] ?? '');
132
+ const updatedAt = new Date(Number.isFinite(signedAt) ? Math.min(signedAt, Date.now()) : Date.now()).toISOString();
133
+ outboxes[key] ??= { sessionId: session || null, nextSequence: state.nextSequence, pending, ownerPid: 0, updatedAt };
134
+ }
135
+ return outboxes;
136
+ }
137
+ const ACTION_ACKNOWLEDGEMENT_PATH = /\/agent-fabric\/decisions\/[^/]+\/enforcements$/;
138
+ /** Whether an action acknowledgement time is outside the window the server accepts, so a kept acknowledgement is sent
139
+ * again only re-signed with a fresh time, a new message id and a new sequence. */
140
+ function acknowledgementTimeStale(acknowledgedAt, now) {
141
+ const at = Date.parse(acknowledgedAt);
142
+ return !Number.isFinite(at) || now - at > 8 * 60_000 || at > now + 5 * 60_000;
143
+ }
144
+ /** A kept request signed longer ago than this is sent again only with a new time and signature: the server refuses a
145
+ * signature more than 5 minutes old. Only its owner signs it again (see #sendPendingWithSigningControl). */
146
+ const SIGNATURE_SENDABLE_MS = 4 * 60_000;
147
+ /** Whether a kept request's signature is too old to send it again exactly as signed. */
148
+ function signatureTooOld(pending, now) {
149
+ const signedAt = Date.parse(pending.headers['x-dharma-timestamp'] ?? '');
150
+ return !Number.isFinite(signedAt) || now - signedAt > SIGNATURE_SENDABLE_MS;
151
+ }
152
+ /** Whether a kept request can be sent again only with a new signature at a new sequence, or not at all: a
153
+ * content-bearing request, which is never replayed implicitly, or an action acknowledgement that is invalid or outside
154
+ * its time window. Only the process that opened a session signs in it (see #sendPendingWithSigningControl), so another
155
+ * client never sends such a request. */
156
+ function cannotResendAsSigned(pending, now) {
157
+ if (isContentBearingPath(pending.pathname))
158
+ return true;
159
+ if (!ACTION_ACKNOWLEDGEMENT_PATH.test(pending.pathname))
160
+ return false;
161
+ let acknowledgement;
162
+ try {
163
+ acknowledgement = JSON.parse(pending.body);
164
+ }
165
+ catch {
166
+ return true;
167
+ }
168
+ return !validateActionDecisionAcknowledgementContract(acknowledgement).ok
169
+ || acknowledgementTimeStale(acknowledgement.acknowledgedAt, now);
170
+ }
171
+ /** Whether the process that wrote an outbox entry still runs. A migrated entry has no owner process. */
172
+ function ownerProcessAlive(pid) {
173
+ if (pid === process.pid)
174
+ return true;
175
+ if (!Number.isSafeInteger(pid) || pid < 1 || pid > 0x7fffffff)
176
+ return false;
177
+ try {
178
+ process.kill(pid, 0);
179
+ return true;
180
+ }
181
+ catch (error) {
182
+ return error.code !== 'ESRCH';
183
+ }
184
+ }
185
+ /** A request that is always rebuilt is never kept in the file, as before. */
186
+ function durableOutboxEntry(entry) {
187
+ return entry.pending && isExplicitlyRebuiltPath(entry.pending.pathname)
188
+ ? { ...entry, nextSequence: entry.nextSequence + 1, pending: null } : entry;
189
+ }
190
+ /** A kept session open matters only to its owner, whose next session open replaces it. Another client drops an
191
+ * orphaned one without sending it, and a full file does not keep one. */
192
+ function isSessionOpen(pending) {
193
+ return pending.method === 'POST' && pending.pathname.endsWith('/agent-fabric/sessions');
194
+ }
195
+ /** A kept request only its owner can use: a session open, a poll, whose answer hands work or content to the owner (a
196
+ * task it claimed lapses with its lease), or a read. Another client drops an orphaned one without sending it, so such
197
+ * an answer never reaches the file, and an earlier version is never handed one to replay. */
198
+ function isOwnerOnlyRequest(pending) {
199
+ return pending.method === 'GET' || isSessionOpen(pending) || pending.pathname.endsWith('/poll');
200
+ }
201
+ /** Whether another client may send an entry's kept request now. It is orphaned once its owner process has exited,
202
+ * or once it stood unchanged for ORPHAN_RETRY_INTERVAL_MS, since a pid can be reused or belong to another account. A
203
+ * workspace loop's entry waits that long even after its process exited, so a loop that restarts promptly replays its
204
+ * own request first. So does a migrated entry, whose request an earlier version that may still run left in the file.
205
+ * An orphan sent without a definite answer is due again after the same interval. */
206
+ function orphanDue(owner, entry, now) {
207
+ if (!entry.pending || recentlyAdopted(entry, now))
208
+ return false;
209
+ return now - Date.parse(entry.updatedAt) >= ORPHAN_RETRY_INTERVAL_MS
210
+ || (owner.startsWith('cli:') && !ownerProcessAlive(entry.ownerPid));
211
+ }
212
+ /** A kept request another client never sends: one only its owner can use, or one that could be sent only signed again,
213
+ * including one whose signature is too old to send exactly as signed. Another client sends a kept request only byte for
214
+ * byte as signed. */
215
+ function ownerOnlyEntry(entry, now) {
216
+ return isOwnerOnlyRequest(entry.pending) || cannotResendAsSigned(entry.pending, now) || signatureTooOld(entry.pending, now);
217
+ }
218
+ /** Whether an entry's owner can never come back: a command that exited, or an entry moved from an earlier version's file. */
219
+ function ownerGone(owner, entry) {
220
+ return owner.startsWith('migrated:') || (owner.startsWith('cli:') && !ownerProcessAlive(entry.ownerPid));
221
+ }
222
+ /** Whether two kept requests are one call: the same message id, or the same session, path and idempotency key (an
223
+ * action acknowledgement signed again keeps those). */
224
+ function sameKeptRequest(a, b) {
225
+ const same = (name) => !!a.headers[name] && a.headers[name] === b.headers[name];
226
+ return same('x-dharma-message-id')
227
+ || (a.pathname === b.pathname && same('idempotency-key') && same('x-dharma-session-id'));
228
+ }
229
+ /** Whether another client took the entry's kept request less than ORPHAN_RETRY_INTERVAL_MS ago. */
230
+ function recentlyAdopted(entry, now) {
231
+ return entry.adoptedAt !== undefined && now - Date.parse(entry.adoptedAt) < ORPHAN_RETRY_INTERVAL_MS;
232
+ }
52
233
  function sha256(value) {
53
234
  return createHash('sha256').update(value).digest('hex');
54
235
  }
@@ -545,11 +726,16 @@ function errorMessage(body, status) {
545
726
  export class AgentFabricRequestError extends Error {
546
727
  status;
547
728
  definitive;
729
+ /** The server's error code, kept only in the stable form diagnostics may record. The message stays free text. */
730
+ serverCode;
548
731
  constructor(body, status, definitive) {
549
732
  super(errorMessage(body, status));
550
733
  this.name = 'AgentFabricRequestError';
551
734
  this.status = status;
552
735
  this.definitive = definitive;
736
+ const code = body && typeof body === 'object' ? body.error?.code : undefined;
737
+ if (typeof code === 'string' && /^[a-z0-9_]{1,80}$/.test(code))
738
+ this.serverCode = code;
553
739
  }
554
740
  }
555
741
  export function isDefinitiveAgentFabricRejection(error) {
@@ -957,6 +1143,32 @@ export class AgentFabricClient {
957
1143
  #isolatedSession;
958
1144
  #readSessionAdmissionUsed = false;
959
1145
  #state;
1146
+ #ownerKey;
1147
+ /** The other clients' entries as of this client's last read of the file. */
1148
+ #otherOutboxes;
1149
+ /** The definite answer another client got for this client's kept request, with the request as this client signed
1150
+ * it. Only the next call can take it, and only when it is the same call. */
1151
+ #adoptedOutcome = null;
1152
+ /** Completions this client recorded that no state write has stored yet, such as when the write after the server's
1153
+ * answer failed. A reload of the file keeps them until a write succeeds. */
1154
+ #unwrittenCompletions = new Map();
1155
+ /** The completions in this client's own entry that wait for room in the queue, as of the last read or write. */
1156
+ #keptCompletions = [];
1157
+ /** Whether this client's kept request is in its own entry in the file, as of the last read or write. A kept request
1158
+ * that is not there is written back before it is sent (see #sendPendingWithSigningControl). */
1159
+ #keptOnDisk;
1160
+ /** An action acknowledgement past its time window that this client kept from an earlier session, while openSession
1161
+ * opens a new one: it stays in this client's entry, and is signed again only in the new session once the server
1162
+ * answered its session open (see openSession). */
1163
+ #deferredAcknowledgement = null;
1164
+ /** Opened with an owner key, as a workspace loop is: it also sends due orphans after its own requests. */
1165
+ #longLived;
1166
+ /** Whether the server accepted this client's latest session open. Until it did, a client with a deferred
1167
+ * acknowledgement signs no other call (see #signedRequestWithSigningControl). */
1168
+ #sessionAccepted = false;
1169
+ /** Whether the section that signs the deferred acknowledgement again waits for its caller's hold to end (see
1170
+ * openSession). Until it ran, no other call of this client is signed or sent (see #signedRequestWithSigningControl). */
1171
+ #reissueWaiting = false;
960
1172
  #serial = Promise.resolve();
961
1173
  constructor(input) {
962
1174
  this.config = input.config;
@@ -971,6 +1183,10 @@ export class AgentFabricClient {
971
1183
  this.#hostBinding = this.#binding();
972
1184
  this.#readOnly = input.readOnly === true;
973
1185
  this.#isolatedSession = input.isolatedSession === true;
1186
+ this.#ownerKey = input.ownerKey;
1187
+ this.#longLived = input.longLived;
1188
+ this.#otherOutboxes = input.otherOutboxes;
1189
+ this.#keptOnDisk = input.keptOnDisk;
974
1190
  }
975
1191
  #binding() {
976
1192
  const { schema, organizationId, deviceId, installationId, hqUrl, relayUrl, publicKeyEd25519, serverPublicKeyEd25519, enrolledAt, platform, setupClaimReference, setupClaimRepositoryFingerprint } = this.config;
@@ -996,6 +1212,11 @@ export class AgentFabricClient {
996
1212
  throw new Error('relay_read_only_option_invalid');
997
1213
  if (isolatedSession !== undefined && typeof isolatedSession !== 'boolean')
998
1214
  throw new Error('relay_isolated_session_option_invalid');
1215
+ if (input.ownerKey !== undefined && (typeof input.ownerKey !== 'string'
1216
+ || !/^(?:relay|cli):/.test(input.ownerKey) || !OWNER_KEY_PATTERN.test(input.ownerKey))) {
1217
+ throw new Error('relay_owner_key_invalid');
1218
+ }
1219
+ const ownerKey = input.ownerKey ?? `cli:${process.pid}:${PROCESS_INSTANCE}:${++clientInstances}`;
999
1220
  const fence = hostScope === undefined ? undefined : new HostOperationFence(hostScope);
1000
1221
  const rawStore = fence ? await fence.step(async () => suppliedStore ?? await createSystemSecureStore())
1001
1222
  : suppliedStore ?? await createSystemSecureStore();
@@ -1021,12 +1242,23 @@ export class AgentFabricClient {
1021
1242
  schema: 'dharma.protocol-state/v1', sessionId: null, nextSequence: 1, pending: null,
1022
1243
  recoveredTaskCompletions: [],
1023
1244
  };
1245
+ let otherOutboxes = {};
1246
+ let keptOnDisk = false;
1024
1247
  if (!readOnly && !isolatedSession)
1025
1248
  try {
1026
1249
  const bytes = fence ? await fence.step(() => readFile(statePath, 'utf8')) : await readFile(statePath, 'utf8');
1027
1250
  const parsed = JSON.parse(bytes);
1028
1251
  assertProtocolState(parsed);
1029
- state = parsed;
1252
+ const { [ownerKey]: own, ...others } = outboxesOf(parsed);
1253
+ otherOutboxes = others;
1254
+ keptOnDisk = !!own?.pending;
1255
+ // Only the kept request of this owner key is taken over, to be sent again exactly as signed (or, an action
1256
+ // acknowledgement past its time window, signed again in this process's own session; see openSession). The
1257
+ // session that signed it belongs to the process that opened it, so this client opens its own and never signs there.
1258
+ state = {
1259
+ schema: 'dharma.protocol-state/v1', sessionId: null, nextSequence: 1,
1260
+ pending: own?.pending ?? null, recoveredTaskCompletions: parsed.recoveredTaskCompletions ?? [],
1261
+ };
1030
1262
  }
1031
1263
  catch (error) {
1032
1264
  if (error.code !== 'ENOENT') {
@@ -1039,7 +1271,8 @@ export class AgentFabricClient {
1039
1271
  await fence?.assert();
1040
1272
  return new AgentFabricClient({
1041
1273
  config, privateJwk: identity.privateJwk, configPath,
1042
- statePath, state, store, fetcher, hostFence: fence, readOnly, isolatedSession,
1274
+ statePath, state, store, fetcher, hostFence: fence, readOnly, isolatedSession, ownerKey,
1275
+ longLived: input.ownerKey !== undefined, otherOutboxes, keptOnDisk,
1043
1276
  });
1044
1277
  }
1045
1278
  async openSession(relayVersion = '0.1.0', telemetry = {}) {
@@ -1048,24 +1281,85 @@ export class AgentFabricClient {
1048
1281
  if (this.#readOnly && (this.#state.sessionId || this.#readSessionAdmissionUsed)) {
1049
1282
  throw new Error('relay_read_only_session_already_started');
1050
1283
  }
1051
- if (this.#state.pending && isExplicitlyRebuiltPath(this.#state.pending.pathname)) {
1052
- // Content-bearing requests require refreshed consent. Repository connect
1053
- // requests depend on current workspace registration. Both are rebuilt by
1054
- // their explicit workflows instead of replayed from stale prerequisites.
1284
+ // A durable client sends one due orphan or replays its own kept request, and writes its new session from the
1285
+ // latest file, all in one signing-control section. It calls the sends directly: waiting on #serial while holding
1286
+ // control could deadlock. The section ends before the session POST, which takes control again through #serial.
1287
+ // In a hold of signing control its caller took itself, perSend or not, this client adds no send of its own to that
1288
+ // hold beyond the session open: it sends no orphan there, and after it sends its own earlier kept request there, a
1289
+ // perSend hold is handed back and the session open runs again in a fresh hold, while in any other hold the session
1290
+ // open fails with relay_retry_after_recovery, unsent, and a retry opens it in a hold of its own.
1291
+ const callerHold = insideDeviceSigningControlHold();
1292
+ const replace = async () => {
1293
+ const pending = this.#state.pending;
1294
+ // An action acknowledgement past its time window can be sent again only signed again. Only the process that
1295
+ // opened a session signs in it, and this one may have taken the acknowledgement over from another process with
1296
+ // its owner key, so it is signed again only in the new session, after its session open (see below).
1297
+ const deferred = !!pending && ACTION_ACKNOWLEDGEMENT_PATH.test(pending.pathname) && cannotResendAsSigned(pending, Date.now());
1298
+ // Not while a task lease is held in this process, so the lease heartbeat of a sibling loop never waits on them,
1299
+ // and not when this section replays this client's own kept request, so it holds signing control for one send.
1300
+ const replays = !!pending && !deferred && !isExplicitlyRebuiltPath(pending.pathname);
1301
+ if (this.#durable && !replays && !callerHold && !taskLeaseHeld(Date.now()))
1302
+ await this.#adoptOrphans();
1303
+ if (this.#state.pending && isExplicitlyRebuiltPath(this.#state.pending.pathname)) {
1304
+ // Content-bearing requests require refreshed consent. Repository connect
1305
+ // requests depend on current workspace registration. Both are rebuilt by
1306
+ // their explicit workflows instead of replayed from stale prerequisites.
1307
+ this.#state.pending = null;
1308
+ await this.#persist();
1309
+ }
1310
+ else if (this.#state.pending && !deferred) {
1311
+ await this.#sendPendingWithSigningControl(this.#state);
1312
+ if (callerHold) {
1313
+ handBackDeviceSigningControl();
1314
+ throw new Error('relay_retry_after_recovery');
1315
+ }
1316
+ }
1317
+ if (deferred)
1318
+ this.#deferredAcknowledgement = this.#state.pending;
1319
+ this.#sessionAccepted = false;
1320
+ this.#state.sessionId = randomUUID();
1321
+ this.#state.nextSequence = 1;
1055
1322
  this.#state.pending = null;
1056
1323
  await this.#persist();
1057
- }
1058
- else if (this.#state.pending) {
1059
- await this.#sendPending();
1060
- }
1061
- this.#state = {
1062
- schema: 'dharma.protocol-state/v1', sessionId: randomUUID(), nextSequence: 1, pending: null,
1063
- recoveredTaskCompletions: this.#state.recoveredTaskCompletions || [],
1064
1324
  };
1065
- await this.#persist();
1066
- return this.signedPost('/agent-fabric/sessions', {
1325
+ if (this.#durable || this.#state.pending)
1326
+ await this.#withProtocolState(replace);
1327
+ else
1328
+ await replace();
1329
+ const opened = await this.signedPost('/agent-fabric/sessions', {
1067
1330
  connectionId: randomBytes(24).toString('base64url'), durableCursor: null, relayVersion, deviceTelemetry,
1068
1331
  });
1332
+ if (this.#deferredAcknowledgement) {
1333
+ // In a section of its own, as this client's next request: the acknowledgement is signed again at the next
1334
+ // sequence of the new session, with its idempotency key, and sent. Its outcome is never this caller's: a
1335
+ // retryable failure keeps it for this client's next request.
1336
+ const send = () => this.#withProtocolState(async () => {
1337
+ if (!this.#deferredAcknowledgement || this.#state.pending)
1338
+ return;
1339
+ this.#state.pending = this.#deferredAcknowledgement;
1340
+ this.#deferredAcknowledgement = null;
1341
+ try {
1342
+ await this.#sendPendingWithSigningControl(this.#state);
1343
+ }
1344
+ catch {
1345
+ await this.#assertHostScope();
1346
+ }
1347
+ });
1348
+ if (callerHold) {
1349
+ // Never in the caller's hold: in a hold of its own once that hold ends. It is not awaited, and not queued in
1350
+ // #serial, since it waits for the caller's hold to end, and the caller's next call in that hold must not wait on it.
1351
+ // Until it ran, another call of this client fails with relay_retry_after_recovery, unsent (owner review 3).
1352
+ this.#reissueWaiting = true;
1353
+ void outsideDeviceSigningControl(() => protocolStateSections.exit(send)).catch(() => undefined)
1354
+ .finally(() => { this.#reissueWaiting = false; });
1355
+ }
1356
+ else {
1357
+ const reissue = this.#serial.then(send);
1358
+ this.#serial = reissue.catch(() => undefined);
1359
+ await reissue;
1360
+ }
1361
+ }
1362
+ return opened;
1069
1363
  }
1070
1364
  async registerWorkspace(body, telemetry = {}) {
1071
1365
  const response = await this.signedPost('/agent-fabric/workspaces', { ...body, deviceTelemetry: deviceKeysetTelemetry(this.config, telemetry) });
@@ -1107,18 +1401,42 @@ export class AgentFabricClient {
1107
1401
  return (this.#state.recoveredTaskCompletions || []).map((item) => ({ ...item }));
1108
1402
  }
1109
1403
  acknowledgeRecoveredTaskCompletion(taskId, receiptHash) {
1110
- const operation = this.#serial.then(async () => {
1111
- await this.#assertHostScope();
1112
- if (this.#readOnly)
1113
- throw new Error('relay_read_only_mutation_denied');
1404
+ const acknowledge = async () => {
1405
+ // A receipt can wait outside the stored list: in this client's memory or entry, or in another client's entry,
1406
+ // until there is room. It is removed wherever it waits, so no later read moves it back to the list.
1114
1407
  const current = this.#state.recoveredTaskCompletions || [];
1115
- const matching = current.find((item) => item.taskId === taskId);
1116
- if (!matching)
1408
+ const entries = Object.values(this.#otherOutboxes);
1409
+ const matching = [...current, ...this.#keptCompletions, ...this.#unwrittenCompletions.values(),
1410
+ ...entries.flatMap(entry => entry.keptCompletions ?? [])].filter(item => item.taskId === taskId);
1411
+ if (!matching.length)
1117
1412
  return;
1118
- if (matching.receiptHash !== receiptHash)
1413
+ if (matching.some(item => item.receiptHash !== receiptHash)) {
1119
1414
  throw new Error('Recovered task completion receipt does not match.');
1120
- this.#state.recoveredTaskCompletions = current.filter((item) => item.taskId !== taskId);
1415
+ }
1416
+ const other = (item) => item.taskId !== taskId;
1417
+ this.#state.recoveredTaskCompletions = current.filter(other);
1418
+ this.#keptCompletions = this.#keptCompletions.filter(other);
1419
+ this.#unwrittenCompletions.delete(taskId);
1420
+ for (const entry of entries) {
1421
+ const waiting = entry.keptCompletions?.filter(other);
1422
+ if (waiting?.length)
1423
+ entry.keptCompletions = waiting;
1424
+ else
1425
+ delete entry.keptCompletions;
1426
+ }
1121
1427
  await this.#persist();
1428
+ };
1429
+ if (this.#durable) {
1430
+ // The completion queue belongs to the whole device: acknowledge against the latest file, under signing control.
1431
+ // Never on this client's request queue: a request that already holds signing control can wait there, and this
1432
+ // acknowledgement would then wait in the signing-control line for that request while it waits for us.
1433
+ return (async () => { await this.#assertHostScope(); return this.#withProtocolState(acknowledge); })();
1434
+ }
1435
+ const operation = this.#serial.then(async () => {
1436
+ await this.#assertHostScope();
1437
+ if (this.#readOnly)
1438
+ throw new Error('relay_read_only_mutation_denied');
1439
+ return acknowledge();
1122
1440
  });
1123
1441
  this.#serial = operation.catch(() => undefined);
1124
1442
  return operation;
@@ -1153,13 +1471,270 @@ export class AgentFabricClient {
1153
1471
  return operation;
1154
1472
  }
1155
1473
  async #signedRequestNow(method, route, body) {
1156
- const hostScope = this.#hostFence ? { signal: this.#hostFence.signal, current: async () => { await this.#assertHostScope(); return true; } } : undefined;
1157
- return withDeviceSigningControl({ config: this.config, hostScope }, () => this.#signedRequestWithSigningControl(method, route, body));
1474
+ const event = method === 'POST' && /^\/agent-fabric\/tasks\/[^/]+\/events$/.test(route)
1475
+ ? body?.eventType : undefined;
1476
+ if (event === 'completed' || event === 'failed' || event === 'cancelled')
1477
+ taskLeases.delete(this);
1478
+ // Each hold of signing control makes at most one network send. When another kept request is sent first (this
1479
+ // client's own earlier call, or a due orphan that makes room in a full file), that hold ends there, and the caller's
1480
+ // request is signed in a fresh hold, from a fresh read of the file, behind every caller that waited meanwhile. Under
1481
+ // the admission line the line's hold is handed back (see withSigningControlPerSend). A hold its caller took itself
1482
+ // without perSend keeps its work, so this client adds no send to it: it sends no orphan to make room there, and
1483
+ // after it sends its own earlier kept call there, the caller's call fails with relay_retry_after_recovery, a failure
1484
+ // with no outcome that a retry sends in a hold of its own.
1485
+ const callerHold = insideCallerSigningControlHold();
1486
+ let result;
1487
+ for (;;) {
1488
+ const answer = await this.#withProtocolState(() => this.#signedRequestWithSigningControl(method, route, body, callerHold));
1489
+ if (answer !== HANDED_BACK) {
1490
+ result = answer;
1491
+ break;
1492
+ }
1493
+ if (callerHold)
1494
+ throw new Error('relay_retry_after_recovery');
1495
+ handBackDeviceSigningControl();
1496
+ }
1497
+ if (method === 'POST' && route === '/agent-fabric/sessions')
1498
+ this.#sessionAccepted = true;
1499
+ if (!this.#durable)
1500
+ return result;
1501
+ if (route === '/agent-fabric/tasks/poll' && result.task && typeof result.task === 'object') {
1502
+ const seconds = Number(body?.leaseSeconds);
1503
+ this.#leaseMs = (Number.isFinite(seconds) && seconds > 0 ? seconds : 120) * 1000;
1504
+ taskLeases.set(this, Date.now() + this.#leaseMs);
1505
+ }
1506
+ else if (event === 'started' || event === 'lease_extended') {
1507
+ taskLeases.set(this, Date.now() + this.#leaseMs);
1508
+ }
1509
+ // A long-lived client, such as a workspace loop, also sends due orphans after its own requests, so they do not wait
1510
+ // for another session open (which has just sent them before its own POST). A one-shot command does not: a send
1511
+ // after its last call would keep its process running after it has its answer.
1512
+ if (this.#longLived && !(method === 'POST' && route === '/agent-fabric/sessions'))
1513
+ this.#sendDueOrphansLater();
1514
+ return result;
1515
+ }
1516
+ /** The lease length of the task this client last polled. */
1517
+ #leaseMs = 120_000;
1518
+ /** Whether a section that sends due orphans is waiting or running. */
1519
+ #orphanSection = false;
1520
+ /** Sends at most one due orphan in a signing-control section of its own, after the caller already has its answer. It
1521
+ * waits in the signing-control line like any root, so this client's next request may wait for that one send, but
1522
+ * never while a task lease is held in this process. It never runs inside the caller's hold of signing control. */
1523
+ #sendDueOrphansLater() {
1524
+ if (this.#orphanSection || !this.#orphanWorkDue(Date.now()) || taskLeaseHeld(Date.now()))
1525
+ return;
1526
+ this.#orphanSection = true;
1527
+ const send = () => this.#withProtocolState(async () => {
1528
+ if (!taskLeaseHeld(Date.now()))
1529
+ await this.#adoptOrphans();
1530
+ });
1531
+ void outsideDeviceSigningControl(() => protocolStateSections.exit(send))
1532
+ .catch(() => undefined) // A later request tries again; the caller already has its answer.
1533
+ .finally(() => { this.#orphanSection = false; });
1534
+ }
1535
+ get #durable() { return !this.#readOnly && !this.#isolatedSession; }
1536
+ #controlScope() {
1537
+ return this.#hostFence ? { signal: this.#hostFence.signal, current: async () => { await this.#assertHostScope(); return true; } } : undefined;
1538
+ }
1539
+ /** Runs work under device signing control. A root section of a durable client first reloads the device-wide
1540
+ * protocol state from the file; a nested call inside this client's live section reuses it without a reload. */
1541
+ async #withProtocolState(work) {
1542
+ const input = { config: this.config, hostScope: this.#controlScope() };
1543
+ const current = protocolStateSections.getStore();
1544
+ if (current?.client === this && current.active)
1545
+ return withDeviceSigningControl(input, work);
1546
+ return withDeviceSigningControl(input, () => {
1547
+ const section = { client: this, active: true };
1548
+ return protocolStateSections.run(section, async () => {
1549
+ try {
1550
+ if (this.#durable)
1551
+ await this.#reloadSharedState();
1552
+ return await work();
1553
+ }
1554
+ finally {
1555
+ section.active = false;
1556
+ }
1557
+ });
1558
+ });
1559
+ }
1560
+ /** The file wins for every other client's entry and for the recovered completion queue, except for a completion this
1561
+ * client recorded and no write has stored yet, which it keeps. A completion waiting in any entry, or one this client
1562
+ * has not written yet, joins the queue only while the queue stays within what a reader accepts; otherwise it waits
1563
+ * for room, in its entry or, until the next write puts it in this client's entry, in memory. This client's own
1564
+ * session, sequence and kept request are written only by this client, so they are never reloaded. */
1565
+ async #reloadSharedState() {
1566
+ let disk = null;
1567
+ try {
1568
+ const bytes = this.#hostFence ? await this.#hostFence.step(() => readFile(this.#statePath, 'utf8'))
1569
+ : await readFile(this.#statePath, 'utf8');
1570
+ const parsed = JSON.parse(bytes);
1571
+ assertProtocolState(parsed);
1572
+ disk = parsed;
1573
+ }
1574
+ catch (error) {
1575
+ if (error.code !== 'ENOENT') {
1576
+ await this.#hostFence?.assert();
1577
+ throw new Error('Relay protocol state is invalid; preserve the file for recovery and re-enroll if necessary.', { cause: error });
1578
+ }
1579
+ }
1580
+ const others = disk ? outboxesOf(disk) : {};
1581
+ const own = others[this.#ownerKey];
1582
+ delete others[this.#ownerKey];
1583
+ this.#otherOutboxes = others;
1584
+ const merged = [...disk?.recoveredTaskCompletions ?? []];
1585
+ /** Whether the completion is in the queue, adding it while there is room. */
1586
+ const listed = (item) => {
1587
+ if (merged.some(other => other.taskId === item.taskId))
1588
+ return true;
1589
+ if (merged.length >= RECOVERED_TASK_COMPLETIONS_MAX)
1590
+ return false;
1591
+ merged.push(item);
1592
+ return true;
1593
+ };
1594
+ this.#keptCompletions = (own?.keptCompletions ?? []).filter(item => !listed(item));
1595
+ for (const item of this.#unwrittenCompletions.values())
1596
+ listed(item);
1597
+ for (const entry of Object.values(others)) {
1598
+ const waiting = entry.keptCompletions?.filter(item => !listed(item));
1599
+ if (waiting?.length)
1600
+ entry.keptCompletions = waiting;
1601
+ else
1602
+ delete entry.keptCompletions;
1603
+ }
1604
+ this.#state.recoveredTaskCompletions = merged;
1605
+ // This client's session, sequence and kept request are its own, and only it signs in its session, so the file never
1606
+ // changes them, with one exception below: another client's answer to its kept request.
1607
+ const kept = this.#state.pending;
1608
+ if (!kept)
1609
+ return;
1610
+ if (own?.adoptedAt && (own.pending ? sameKeptRequest(own.pending, kept) : true)) {
1611
+ // Another client sent this client's kept request after it stood unchanged (see orphanDue). The file then holds
1612
+ // its outcome: no kept request once it was answered, with the answer, or the request as last sent. When the
1613
+ // answer is not there (never kept, or removed after ORPHAN_RETRY_INTERVAL_MS), this client keeps its signed
1614
+ // request, so a retry sends it again with the same message id and idempotency key instead of signing the call
1615
+ // again. That client sent it only exactly as signed, so no sequence in this session is new to this client.
1616
+ const outcome = own.adoptedOutcome;
1617
+ const answered = !own.pending && !!outcome && outcome.messageId === kept.headers['x-dharma-message-id'];
1618
+ if (answered) {
1619
+ this.#state.pending = null;
1620
+ this.#state.nextSequence = Math.max(this.#state.nextSequence, Number(kept.headers['x-dharma-sequence'] || 0) + 1);
1621
+ }
1622
+ else if (own.pending)
1623
+ this.#state.pending = own.pending;
1624
+ else
1625
+ this.#keptOnDisk = false;
1626
+ this.#adoptedOutcome = answered
1627
+ ? { method: kept.method, pathname: kept.pathname, body: kept.body, status: outcome.status, response: outcome.response }
1628
+ : null;
1629
+ return;
1630
+ }
1631
+ if (own?.pending && sameKeptRequest(own.pending, kept))
1632
+ return;
1633
+ // The kept request left this client's entry, although no write by this client drops one: the entry was removed (by
1634
+ // an earlier version that rewrote the file without outboxes, or with the file), or came back in an older state (an
1635
+ // earlier version writes back the whole file it read when it opened) without it. This client keeps it, in its
1636
+ // session, and writes it back to its own entry before it sends it again exactly as signed. Another entry that holds
1637
+ // the same call (the top-level copy, moved to a migrated entry) gives it up, so only this client's entry holds it.
1638
+ this.#keptOnDisk = false;
1639
+ for (const [owner, entry] of Object.entries(others)) {
1640
+ if (entry.pending && sameKeptRequest(entry.pending, kept))
1641
+ others[owner] = { ...entry, pending: null };
1642
+ }
1158
1643
  }
1159
- async #signedRequestWithSigningControl(method, route, body) {
1644
+ /** Whether the entries as last read, which each request of this client reads again, hold work for #adoptOrphans: a
1645
+ * due orphan it sends, or one it removes unsent because its owner is gone. A request only its owner can use, while
1646
+ * that owner can come back, is no such work. */
1647
+ #orphanWorkDue(now) {
1648
+ return Object.entries(this.#otherOutboxes).some(([owner, entry]) => orphanDue(owner, entry, now)
1649
+ && (!ownerOnlyEntry(entry, now) || ownerGone(owner, entry)));
1650
+ }
1651
+ /** Sends the kept requests of orphaned entries (see orphanDue): at most ORPHAN_SENDS_PER_SECTION, and none after one
1652
+ * without a definite answer. Their outcome is never this caller's and is never thrown. Each entry is marked in the
1653
+ * file before its send, so when this client stops during the send, the next one waits ORPHAN_RETRY_INTERVAL_MS too.
1654
+ * An answered entry keeps no request; it holds the answer only when its owner may still read it (a workspace loop, or a process
1655
+ * that still runs), so a live owner takes the outcome from the file, and #persist drops that answer after
1656
+ * ORPHAN_RETRY_INTERVAL_MS. A request without a definite answer stays and is due again after
1657
+ * ORPHAN_RETRY_INTERVAL_MS. An orphan is sent only exactly as signed, so this client never signs in another client's
1658
+ * session. While more are due, this client's next request sends the next one. A kept request only its owner can use
1659
+ * (see isOwnerOnlyRequest), or one that could be sent only signed again (see cannotResendAsSigned), is never sent by
1660
+ * another client: it stays in its entry while its owner can come back (a workspace loop that restarts, or a process
1661
+ * that still runs), and is settled unsent once that owner is gone (a command that exited, or a migrated entry).
1662
+ * Returns whether it sent one. */
1663
+ async #adoptOrphans() {
1664
+ const now = Date.now();
1665
+ const due = Object.entries(this.#otherOutboxes).filter(([owner, entry]) => orphanDue(owner, entry, now));
1666
+ const ownerOnly = (entry) => ownerOnlyEntry(entry, now);
1667
+ let unwritten = false;
1668
+ for (const [owner, entry] of due) {
1669
+ if (!ownerOnly(entry) || !ownerGone(owner, entry))
1670
+ continue;
1671
+ entry.pending = null;
1672
+ entry.adoptedAt = entry.updatedAt = new Date().toISOString();
1673
+ delete entry.adoptedOutcome;
1674
+ unwritten = true;
1675
+ }
1676
+ let sent = 0;
1677
+ for (const [owner, entry] of due) {
1678
+ if (!entry.pending || ownerOnly(entry))
1679
+ continue;
1680
+ if (sent === ORPHAN_SENDS_PER_SECTION)
1681
+ break;
1682
+ sent += 1;
1683
+ const messageId = entry.pending.headers['x-dharma-message-id'] ?? '';
1684
+ const ownerReads = owner.startsWith('relay:') || ownerProcessAlive(entry.ownerPid);
1685
+ entry.adoptedAt = new Date().toISOString();
1686
+ delete entry.adoptedOutcome;
1687
+ await this.#persist();
1688
+ unwritten = false;
1689
+ try {
1690
+ await this.#sendPendingWithSigningControl(entry, (status, response) => {
1691
+ // Only an answer every reader accepts is kept; without one, the owner signs its retry again.
1692
+ const outcome = { messageId, status, response };
1693
+ if (ownerReads && isAdoptedOutcome(outcome))
1694
+ entry.adoptedOutcome = outcome;
1695
+ });
1696
+ }
1697
+ catch {
1698
+ await this.#assertHostScope();
1699
+ }
1700
+ entry.adoptedAt = new Date().toISOString();
1701
+ if (!entry.pending)
1702
+ entry.updatedAt = entry.adoptedAt;
1703
+ await this.#persist();
1704
+ if (entry.pending)
1705
+ break;
1706
+ }
1707
+ if (unwritten)
1708
+ await this.#persist();
1709
+ return sent > 0;
1710
+ }
1711
+ /** The call a due orphan was just sent to make room for in a full file. Its next hold signs it without sending
1712
+ * another orphan first, so a file that is still full refuses it, as before. */
1713
+ #madeRoomFor = null;
1714
+ /** One hold's part of a signed request: it makes at most one network send. It returns the caller's answer, whatever
1715
+ * its JSON body, or HANDED_BACK after it sent another kept request first (this client's own earlier call, or a due
1716
+ * orphan that makes room in a full file); the caller's request is then signed in a hold of its own. In a hold its
1717
+ * caller took itself (callerHold), it sends no orphan to make room, so only this client's own earlier call hands back. */
1718
+ async #signedRequestWithSigningControl(method, route, body, callerHold) {
1160
1719
  await this.#assertHostScope();
1161
1720
  if (!this.#state.sessionId)
1162
1721
  throw new Error('Relay session is not open.');
1722
+ // An acknowledgement this client took over waits for its new session (see openSession). Until the server accepted
1723
+ // that session open, no other call is signed or replayed, so its request never takes the acknowledgement's place in
1724
+ // this client's entry: the session is open only once the server accepted it, and the next openSession opens it.
1725
+ if (this.#deferredAcknowledgement && !this.#sessionAccepted && !(method === 'POST' && route === '/agent-fabric/sessions')) {
1726
+ throw new Error('Relay session is not open.');
1727
+ }
1728
+ if (this.#deferredAcknowledgement && !(method === 'POST' && route === '/agent-fabric/sessions')) {
1729
+ // Once the session is accepted, the acknowledgement still keeps its place in this client's entry until it has had a
1730
+ // hold of its own. While its own section waits for the caller's hold to end, another call is neither signed nor
1731
+ // sent, so it never takes that place: it fails with relay_retry_after_recovery, and a retry signs it once that
1732
+ // section ran. Otherwise the acknowledgement is this client's earlier kept call, sent first, as below.
1733
+ if (this.#reissueWaiting)
1734
+ throw new Error('relay_retry_after_recovery');
1735
+ this.#state.pending = this.#deferredAcknowledgement;
1736
+ this.#deferredAcknowledgement = null;
1737
+ }
1163
1738
  if (this.#readOnly) {
1164
1739
  if (this.#state.pending)
1165
1740
  throw new Error('relay_read_only_pending_request');
@@ -1172,6 +1747,15 @@ export class AgentFabricClient {
1172
1747
  }
1173
1748
  const pathname = `/api/v1/orgs/${encodeURIComponent(this.config.organizationId)}${route}`;
1174
1749
  const serialized = method === 'GET' ? '' : canonicalize(body);
1750
+ const adopted = this.#adoptedOutcome;
1751
+ this.#adoptedOutcome = null;
1752
+ if (adopted && !this.#state.pending && adopted.method === method && adopted.pathname === pathname
1753
+ && adopted.body === serialized) {
1754
+ // Another client delivered this call's kept request: the retry gets that answer, as a recovery of its own would.
1755
+ if (adopted.status >= 200 && adopted.status < 300)
1756
+ return adopted.response;
1757
+ throw new AgentFabricRequestError(adopted.response, adopted.status, true);
1758
+ }
1175
1759
  if (this.#state.pending && isContentBearingPath(this.#state.pending.pathname)) {
1176
1760
  // An ambiguous content delivery is never replayed implicitly. The next
1177
1761
  // explicit caller must rebuild the capsule after rechecking disclosure
@@ -1185,10 +1769,22 @@ export class AgentFabricClient {
1185
1769
  const sameRequest = this.#state.pending.method === method
1186
1770
  && this.#state.pending.pathname === pathname
1187
1771
  && this.#state.pending.body === serialized;
1188
- const recovered = await this.#sendPending();
1189
- // Recovery acknowledges only its original caller's request identity.
1190
- if (sameRequest)
1191
- return recovered;
1772
+ const recovered = await this.#sendPendingWithSigningControl(this.#state);
1773
+ // Recovery acknowledges only its original caller's request identity. Another call is signed in a hold of its own.
1774
+ return sameRequest ? recovered : HANDED_BACK;
1775
+ }
1776
+ // In a file full of other clients' kept requests, a due orphan is sent first to make room for this one, in a hold
1777
+ // of its own. A completion the queue has no room for is refused before that. In a hold its caller took itself, no
1778
+ // orphan is sent, and the full file refuses this request (relay_protocol_state_outbox_full).
1779
+ const call = `${method} ${pathname} ${serialized}`, unsigned = { method, pathname, body: serialized, headers: {} };
1780
+ const madeRoom = this.#madeRoomFor === call;
1781
+ this.#madeRoomFor = null;
1782
+ this.#assertRecoveredTaskCompletionCapacity(unsigned);
1783
+ if (!madeRoom && !callerHold && this.#durable && !isExplicitlyRebuiltPath(pathname) && !isSessionOpen(unsigned)
1784
+ && Object.values(this.#otherOutboxes).filter(entry => entry.pending).length >= OUTBOX_ENTRIES_MAX
1785
+ && await this.#adoptOrphans()) {
1786
+ this.#madeRoomFor = call;
1787
+ return HANDED_BACK;
1192
1788
  }
1193
1789
  const timestamp = new Date().toISOString();
1194
1790
  await this.#assertHostScope();
@@ -1214,30 +1810,32 @@ export class AgentFabricClient {
1214
1810
  'x-dharma-sequence': String(sequence), 'x-dharma-signature': signature,
1215
1811
  },
1216
1812
  };
1217
- this.#assertRecoveredTaskCompletionCapacity(pending);
1218
1813
  this.#state.pending = pending;
1219
- await this.#persist();
1220
- return this.#sendPending();
1221
- }
1222
- async #sendPending() {
1223
- const hostScope = this.#hostFence ? { signal: this.#hostFence.signal, current: async () => { await this.#assertHostScope(); return true; } } : undefined;
1224
- return withDeviceSigningControl({ config: this.config, hostScope }, () => this.#sendPendingWithSigningControl());
1225
- }
1226
- async #sendPendingWithSigningControl() {
1227
- await this.#assertHostScope();
1228
- const pending = this.#state.pending;
1229
- if (!pending)
1230
- throw new Error('No pending protocol request.');
1231
1814
  try {
1232
- this.#assertRecoveredTaskCompletionCapacity(pending);
1815
+ await this.#persist();
1233
1816
  }
1234
1817
  catch (error) {
1235
- this.#state.nextSequence += 1;
1236
- this.#state.pending = null;
1237
- await this.#persist();
1818
+ // A request the full file refused is never sent, so it is not kept for a later replay either.
1819
+ if (error.message === 'relay_protocol_state_outbox_full')
1820
+ this.#state.pending = null;
1238
1821
  throw error;
1239
1822
  }
1240
- if (/\/agent-fabric\/decisions\/[^/]+\/enforcements$/.test(pending.pathname)) {
1823
+ return this.#sendPendingWithSigningControl(this.#state);
1824
+ }
1825
+ /** Sends the kept request of one outbox entry: this client's own, or an orphan in #adoptOrphans. answered sees a
1826
+ * definite answer before the entry is written without its request. This client's own kept request is in its entry
1827
+ * in the file before it is sent: when the entry left the file (see #reloadSharedState), it is written back first, so
1828
+ * after a failed send, a crash or a new client with the same owner key, it is still there to send again. */
1829
+ async #sendPendingWithSigningControl(slot, answered) {
1830
+ await this.#assertHostScope();
1831
+ const pending = slot.pending;
1832
+ if (!pending)
1833
+ throw new Error('No pending protocol request.');
1834
+ if (slot === this.#state && !this.#keptOnDisk)
1835
+ await this.#persist();
1836
+ // A kept completion is sent again even when the queue is full: the server may already have accepted it, and its
1837
+ // receipt then waits in this client's entry for room (see #recordRecoveredTaskCompletion).
1838
+ if (ACTION_ACKNOWLEDGEMENT_PATH.test(pending.pathname)) {
1241
1839
  let acknowledgement;
1242
1840
  try {
1243
1841
  acknowledgement = JSON.parse(pending.body);
@@ -1247,17 +1845,24 @@ export class AgentFabricClient {
1247
1845
  }
1248
1846
  const contract = validateActionDecisionAcknowledgementContract(acknowledgement);
1249
1847
  if (!contract.ok) {
1250
- this.#state.nextSequence = Math.max(this.#state.nextSequence + 1, Number(pending.headers['x-dharma-sequence'] || 0) + 1);
1251
- this.#state.pending = null;
1848
+ slot.nextSequence = Math.max(slot.nextSequence + 1, Number(pending.headers['x-dharma-sequence'] || 0) + 1);
1849
+ slot.pending = null;
1252
1850
  await this.#persist();
1253
1851
  throw new Error('Stale action acknowledgement is invalid and was discarded.');
1254
1852
  }
1255
- const acknowledgedAt = Date.parse(acknowledgement.acknowledgedAt);
1256
- if (!Number.isFinite(acknowledgedAt)
1257
- || Date.now() - acknowledgedAt > 8 * 60_000
1258
- || acknowledgedAt > Date.now() + 5 * 60_000) {
1853
+ if (acknowledgementTimeStale(acknowledgement.acknowledgedAt, Date.now())) {
1854
+ // Only the process that opened a session signs in it, from the sequences it holds in memory: this client signs
1855
+ // the acknowledgement again only as its own kept request, in its own open session, at its next sequence. It
1856
+ // keeps the idempotency key, so the server can tell it is the same acknowledgement. Every other client sends a
1857
+ // kept request only exactly as signed (see #adoptOrphans), so nothing else signs here.
1858
+ const sessionId = this.#state.sessionId;
1859
+ if (slot !== this.#state || !sessionId) {
1860
+ throw new Error('Stale action acknowledgement can be signed again only in its owner\'s own session.');
1861
+ }
1259
1862
  const refreshed = refreshActionDecisionAcknowledgement(acknowledgement);
1260
- const sequence = Number(pending.headers['x-dharma-sequence'] || 0) + 1;
1863
+ // The next sequence of this session, above the acknowledgement's own when it was signed in this session too.
1864
+ const sequence = Math.max(this.#state.nextSequence, pending.headers['x-dharma-session-id'] === sessionId
1865
+ ? Number(pending.headers['x-dharma-sequence'] || 0) + 1 : 0);
1261
1866
  const timestamp = new Date().toISOString();
1262
1867
  const messageId = randomUUID();
1263
1868
  const nonce = randomBytes(24).toString('base64url');
@@ -1266,19 +1871,24 @@ export class AgentFabricClient {
1266
1871
  bodyHash: `sha256:${sha256(pending.body)}`,
1267
1872
  deviceId: this.config.deviceId, messageId, method: pending.method, nonce,
1268
1873
  organizationId: this.config.organizationId, pathname: pending.pathname,
1269
- sequence, sessionId: pending.headers['x-dharma-session-id'], timestamp,
1874
+ sequence, sessionId, timestamp,
1270
1875
  }), 'utf8');
1876
+ pending.headers['x-dharma-session-id'] = sessionId;
1271
1877
  pending.headers['x-dharma-message-id'] = messageId;
1272
1878
  pending.headers['x-dharma-timestamp'] = timestamp;
1273
1879
  pending.headers['x-dharma-nonce'] = nonce;
1274
1880
  pending.headers['x-dharma-sequence'] = String(sequence);
1275
1881
  pending.headers['x-dharma-signature'] = sign(null, signingPayload, { key: this.#privateJwk, format: 'jwk' }).toString('base64url');
1276
- this.#state.nextSequence = sequence;
1882
+ slot.nextSequence = sequence;
1277
1883
  await this.#persist();
1278
1884
  }
1279
1885
  }
1280
- const signedAt = Date.parse(pending.headers['x-dharma-timestamp'] || '');
1281
- if (!Number.isFinite(signedAt) || Date.now() - signedAt > 4 * 60_000) {
1886
+ if (signatureTooOld(pending, Date.now())) {
1887
+ // Only this client's own kept request gets a new time and signature, with its message id, session, sequence,
1888
+ // body and idempotency key unchanged. Another client sends a kept request only byte for byte as signed: one too
1889
+ // old for that stays in its entry for its owner (see ownerOnlyEntry), so nothing is signed or sent here.
1890
+ if (slot !== this.#state)
1891
+ throw new Error('A kept request too old to send as signed is signed again only by its owner.');
1282
1892
  const timestamp = new Date().toISOString();
1283
1893
  const nonce = randomBytes(24).toString('base64url');
1284
1894
  const messageId = pending.headers['x-dharma-message-id'];
@@ -1346,15 +1956,17 @@ export class AgentFabricClient {
1346
1956
  // signed request, as do upstream failures whose commit state is unknown.
1347
1957
  const definitive = status >= 400 && status < 500 && status !== 408 && status !== 429;
1348
1958
  if (definitive) {
1349
- this.#state.nextSequence += 1;
1350
- this.#state.pending = null;
1959
+ answered?.(status, body);
1960
+ slot.nextSequence += 1;
1961
+ slot.pending = null;
1351
1962
  await this.#persist();
1352
1963
  }
1353
1964
  throw new AgentFabricRequestError(body, status, definitive);
1354
1965
  }
1355
1966
  this.#recordRecoveredTaskCompletion(pending, body);
1356
- this.#state.nextSequence += 1;
1357
- this.#state.pending = null;
1967
+ answered?.(status, body);
1968
+ slot.nextSequence += 1;
1969
+ slot.pending = null;
1358
1970
  await this.#persist();
1359
1971
  return body;
1360
1972
  }
@@ -1392,19 +2004,19 @@ export class AgentFabricClient {
1392
2004
  throw new Error('Task completion acknowledgement is missing a valid durable receipt.');
1393
2005
  }
1394
2006
  const current = this.#state.recoveredTaskCompletions || [];
1395
- const existing = current.find((item) => item.taskId === taskId);
2007
+ const existing = [...current, ...this.#keptCompletions, ...this.#unwrittenCompletions.values()]
2008
+ .find((item) => item.taskId === taskId);
1396
2009
  if (existing) {
1397
2010
  if (existing.receiptHash !== receiptHash || existing.trajectoryCapsuleHash !== trajectoryCapsuleHash) {
1398
2011
  throw new Error('Recovered task completion conflicts with its durable receipt.');
1399
2012
  }
1400
2013
  return;
1401
2014
  }
1402
- if (current.length >= 1_000)
1403
- throw new Error('Recovered task completion queue is full.');
1404
- this.#state.recoveredTaskCompletions = [
1405
- ...current,
1406
- { taskId, trajectoryCapsuleHash, receiptHash, recoveredAt: new Date().toISOString() },
1407
- ];
2015
+ const completion = { taskId, trajectoryCapsuleHash, receiptHash, recoveredAt: new Date().toISOString() };
2016
+ // An accepted completion is never dropped: without room in the queue it waits in this client's entry (see #persist).
2017
+ if (current.length < RECOVERED_TASK_COMPLETIONS_MAX)
2018
+ this.#state.recoveredTaskCompletions = [...current, completion];
2019
+ this.#unwrittenCompletions.set(taskId, completion);
1408
2020
  }
1409
2021
  #assertRecoveredTaskCompletionCapacity(pending) {
1410
2022
  const route = pending.pathname.match(/\/agent-fabric\/tasks\/([^/]+)\/events$/);
@@ -1421,7 +2033,7 @@ export class AgentFabricClient {
1421
2033
  return;
1422
2034
  const taskId = decodeURIComponent(route[1]);
1423
2035
  const current = this.#state.recoveredTaskCompletions || [];
1424
- if (!current.some((item) => item.taskId === taskId) && current.length >= 1_000) {
2036
+ if (!current.some((item) => item.taskId === taskId) && current.length >= RECOVERED_TASK_COMPLETIONS_MAX) {
1425
2037
  throw new Error('Recovered task completion queue is full; acknowledge stored completions before sending another completion.');
1426
2038
  }
1427
2039
  }
@@ -1437,18 +2049,24 @@ export class AgentFabricClient {
1437
2049
  if (settled)
1438
2050
  return;
1439
2051
  settled = true;
2052
+ clearTimeout(connectTimeout);
1440
2053
  clearTimeout(timeout);
1441
2054
  this.#hostFence?.signal.removeEventListener('abort', abort);
1442
2055
  callback();
1443
2056
  if (closeSocket && socket.readyState < WebSocket.CLOSING)
1444
2057
  socket.close(1000);
1445
2058
  };
1446
- const timeout = setTimeout(() => finish(() => reject(new Error('Relay response timed out.'))), RELAY_ACKNOWLEDGEMENT_TIMEOUT_MS);
2059
+ const timeout = setTimeout(() => finish(() => reject(Object.assign(new Error('Relay response timed out.'), { code: 'relay_response_timeout' }))), RELAY_ACKNOWLEDGEMENT_TIMEOUT_MS);
2060
+ // #512: before the socket opens this attempt has sent nothing, so this limit fails it cleanly. The kept signed
2061
+ // request is sent on the next attempt, exactly as after any other failure that keeps it (an earlier attempt
2062
+ // may already have sent it). It never shortens the window after send.
2063
+ const connectTimeout = setTimeout(() => finish(() => reject(Object.assign(new Error('Relay connection timed out before the request was sent.'), { code: 'relay_connect_timeout' }))), RELAY_CONNECT_TIMEOUT_MS);
1447
2064
  const abort = () => finish(() => reject(new Error('relay_host_scope_unavailable')));
1448
2065
  this.#hostFence?.signal.addEventListener('abort', abort, { once: true });
1449
2066
  if (this.#hostFence?.signal.aborted)
1450
2067
  abort();
1451
2068
  socket.addEventListener('open', () => {
2069
+ clearTimeout(connectTimeout);
1452
2070
  const send = () => {
1453
2071
  if (!settled)
1454
2072
  socket.send(JSON.stringify({
@@ -1485,17 +2103,82 @@ export class AgentFabricClient {
1485
2103
  await this.#assertHostScope();
1486
2104
  return;
1487
2105
  }
2106
+ const section = protocolStateSections.getStore();
2107
+ if (section?.client !== this || !section.active)
2108
+ throw new Error('relay_protocol_state_unserialized');
1488
2109
  await this.#assertHostScope();
1489
2110
  // Authorized content and state-dependent repository connects are rebuilt
1490
2111
  // from their governed source. Do not retain them as an automatic replay
1491
2112
  // that can outlive consent or the workspace registration they depend on.
1492
- const durableState = this.#state.pending && isExplicitlyRebuiltPath(this.#state.pending.pathname)
1493
- ? { ...this.#state, nextSequence: this.#state.nextSequence + 1, pending: null }
1494
- : this.#state;
1495
- return atomicJson(this.#statePath, durableState, this.#hostFence);
2113
+ // Each other client's entry is written back as read while it holds a kept
2114
+ // request, kept completions, or an answer another client got for it in the
2115
+ // last ORPHAN_RETRY_INTERVAL_MS (its owner may still retry the call); every
2116
+ // other entry is dropped, so entries of clients that come and go never fill
2117
+ // the file. A kept request is never dropped.
2118
+ const outboxes = {};
2119
+ const now = Date.now();
2120
+ for (const [owner, read] of Object.entries(this.#otherOutboxes)) {
2121
+ let entry = durableOutboxEntry(read);
2122
+ if (entry.adoptedOutcome && !recentlyAdopted(entry, now)) {
2123
+ const { adoptedOutcome: _answer, ...rest } = entry;
2124
+ entry = rest;
2125
+ }
2126
+ if (entry.pending || entry.keptCompletions || entry.adoptedOutcome)
2127
+ outboxes[owner] = entry;
2128
+ }
2129
+ // A file never holds more entries than a reader accepts. Only a new kept request an earlier version left at the
2130
+ // top level of a full file can exceed them (see outboxesOf); it stays at the top level until it is sent.
2131
+ let leftover;
2132
+ const migrated = Object.keys(outboxes).filter(owner => owner.startsWith('migrated:')).at(-1); // outboxesOf adds it last
2133
+ if (Object.keys(outboxes).length > OUTBOX_ENTRIES_MAX && migrated) {
2134
+ leftover = outboxes[migrated];
2135
+ delete outboxes[migrated];
2136
+ }
2137
+ const { sessionId, nextSequence } = this.#state;
2138
+ // An acknowledgement kept from an earlier session while a new one opens stays in this client's entry, also while
2139
+ // the session open is in flight: a kept session open matters only to this client, whose next open replaces it,
2140
+ // while the acknowledgement must reach the next process with this owner key if this one never re-issues it.
2141
+ const deferred = this.#deferredAcknowledgement;
2142
+ const pending = deferred && (!this.#state.pending || isSessionOpen(this.#state.pending)) ? deferred : this.#state.pending;
2143
+ // A completion this client recorded that the queue has no room for waits in its own entry, which is then kept.
2144
+ const listed = this.#state.recoveredTaskCompletions ?? [];
2145
+ const waiting = [...this.#keptCompletions, ...[...this.#unwrittenCompletions.values()].filter(item => !listed.some(other => other.taskId === item.taskId) && !this.#keptCompletions.some(other => other.taskId === item.taskId))];
2146
+ // The next sequence is written only because the file format has it; no client reads another's.
2147
+ const own = durableOutboxEntry({ sessionId, nextSequence, pending, ownerPid: process.pid,
2148
+ updatedAt: new Date().toISOString(), ...(waiting.length ? { keptCompletions: waiting } : {}) });
2149
+ const room = OUTBOX_ENTRIES_MAX - Object.keys(outboxes).length;
2150
+ if (room < 1 && (waiting.length || (own.pending && !isSessionOpen(own.pending)))) {
2151
+ throw new Error('relay_protocol_state_outbox_full');
2152
+ }
2153
+ if (room > 0)
2154
+ outboxes[this.#ownerKey] = own;
2155
+ // An earlier version reads only the top-level session, sequence and kept request, and its session open replays
2156
+ // that request and then writes a file without outboxes. So the oldest kept request that is already due (see
2157
+ // orphanDue) also goes there, exactly as signed: after a rollback the earlier version still sends that one, with
2158
+ // the same message id. A live client's fresh request is never handed to it, nor any request another client never
2159
+ // sends (see ownerOnlyEntry), nor an action acknowledgement: the earlier version would sign either again in its
2160
+ // owner's session. That version gives a request signed more than 4 minutes ago a new time and signature, so a
2161
+ // request copied here was signed within the last 4 minutes, and it replays that one byte for byte. In practice that
2162
+ // is a command's request after the command exited, since any other entry is due only after 10 minutes unchanged.
2163
+ // outboxesOf never takes this copy for another orphan.
2164
+ const [copy] = leftover ? [leftover] : Object.entries(outboxes)
2165
+ .filter(([owner, entry]) => entry.pending && orphanDue(owner, entry, now) && !ownerOnlyEntry(entry, now)
2166
+ && !ACTION_ACKNOWLEDGEMENT_PATH.test(entry.pending.pathname))
2167
+ .map(([, entry]) => entry)
2168
+ .sort((a, b) => Date.parse(a.updatedAt) - Date.parse(b.updatedAt));
2169
+ await atomicJson(this.#statePath, {
2170
+ schema: 'dharma.protocol-state/v1', sessionId: copy?.sessionId ?? null, nextSequence: copy?.nextSequence ?? 1,
2171
+ pending: copy?.pending ?? null, recoveredTaskCompletions: this.#state.recoveredTaskCompletions ?? [], outboxes,
2172
+ }, this.#hostFence);
2173
+ // Every completion this client recorded is now in the queue or in its own entry.
2174
+ this.#unwrittenCompletions.clear();
2175
+ this.#keptCompletions = waiting;
2176
+ this.#keptOnDisk = !!outboxes[this.#ownerKey]?.pending;
1496
2177
  }
1497
2178
  }
1498
2179
  export const RELAY_ACKNOWLEDGEMENT_TIMEOUT_MS = 90_000;
2180
+ /** Limit for the relay connection to open. Nothing is sent before it opens; the acknowledgement window is unchanged. */
2181
+ export const RELAY_CONNECT_TIMEOUT_MS = 30_000;
1499
2182
  // A date-time the keyset contract accepts (including a lowercase or space separator); sent normalized to ISO.
1500
2183
  const TELEMETRY_INSTANT = /^\d{4}-\d{2}-\d{2}[Tt ]\d{2}:\d{2}:\d{2}(?:\.\d{1,9})?(?:[Zz]|[+-]\d{2}:\d{2})$/;
1501
2184
  export function deviceKeysetTelemetry(config, options = {}) {