@dharma-ai-labs/agent-fabric-relay-client 0.2.37 → 0.2.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +195 -0
- package/dist/deviceSigningControl.d.ts +18 -0
- package/dist/deviceSigningControl.d.ts.map +1 -1
- package/dist/deviceSigningControl.js +137 -4
- package/dist/deviceSigningControl.js.map +1 -1
- package/dist/deviceSigningControl.test.js +145 -0
- package/dist/deviceSigningControl.test.js.map +1 -1
- package/dist/enrollmentAnchorLock.d.ts.map +1 -1
- package/dist/enrollmentAnchorLock.js +57 -17
- package/dist/enrollmentAnchorLock.js.map +1 -1
- package/dist/enrollmentAnchorLockCorruption.test.js +219 -12
- package/dist/enrollmentAnchorLockCorruption.test.js.map +1 -1
- package/dist/hostScope.test.js +5 -3
- package/dist/hostScope.test.js.map +1 -1
- package/dist/index.d.ts +10 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +771 -88
- package/dist/index.js.map +1 -1
- package/dist/index.test.js +2592 -26
- package/dist/index.test.js.map +1 -1
- package/dist/protocolStateModel.test.d.ts +2 -0
- package/dist/protocolStateModel.test.d.ts.map +1 -0
- package/dist/protocolStateModel.test.js +383 -0
- package/dist/protocolStateModel.test.js.map +1 -0
- package/package.json +4 -1
package/dist/index.test.js
CHANGED
|
@@ -1,11 +1,15 @@
|
|
|
1
1
|
import assert from 'node:assert/strict';
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
2
|
+
import { spawn } from 'node:child_process';
|
|
3
|
+
import { createHash, createPublicKey, generateKeyPairSync, randomUUID, verify } from 'node:crypto';
|
|
4
|
+
import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises';
|
|
4
5
|
import { tmpdir } from 'node:os';
|
|
5
6
|
import { resolve } from 'node:path';
|
|
6
7
|
import { test } from 'node:test';
|
|
7
|
-
import { AgentFabricClient, RELAY_ACKNOWLEDGEMENT_TIMEOUT_MS, AgentFabricRequestError, beginEnrollment, deleteActiveSkillAuthorizationAnchor, loadActiveSkillAuthorizationAnchor, loadOrCreateDeviceIdentity, isContentBearingPath, isExplicitlyRebuiltPath, isDefinitiveAgentFabricRejection, installTrustedServerSigningKeyset, loadDeviceEnrollmentAnchor, assertDeviceEnrollmentAnchorAbsent, recoverDeviceEnrollmentConsistency, normalizeHqUrl, normalizeRelayUrl, redeemBootstrapGrant, saveDeviceConfig, saveActiveSkillAuthorizationAnchor, saveDeviceEnrollmentAnchor, loadOrganizationApiToken, saveOrganizationApiToken, deviceKeysetTelemetry, } from './index.js';
|
|
8
|
+
import { AgentFabricClient, RELAY_ACKNOWLEDGEMENT_TIMEOUT_MS, AgentFabricRequestError, beginEnrollment, deleteActiveSkillAuthorizationAnchor, loadActiveSkillAuthorizationAnchor, loadOrCreateDeviceIdentity, isContentBearingPath, isExplicitlyRebuiltPath, isDefinitiveAgentFabricRejection, installTrustedServerSigningKeyset, loadDeviceEnrollmentAnchor, assertDeviceEnrollmentAnchorAbsent, recoverDeviceEnrollmentConsistency, normalizeHqUrl, normalizeRelayUrl, redeemBootstrapGrant, saveDeviceConfig, saveActiveSkillAuthorizationAnchor, saveDeviceEnrollmentAnchor, loadOrganizationApiToken, saveOrganizationApiToken, deviceKeysetTelemetry, withDeviceSigningControl, withDeviceSigningLineSignal, withSigningControlPerSend, } from './index.js';
|
|
9
|
+
import * as relayClientModule from './index.js';
|
|
8
10
|
import { signCanonicalObject, } from '@dharma-ai-labs/agent-fabric-contracts';
|
|
11
|
+
// The relay client that v0.2.225 ships, at the exact version it pins, for the earlier-version tests.
|
|
12
|
+
import { AgentFabricClient as EarlierReleaseClient } from 'agent-fabric-relay-client-v0.2.38';
|
|
9
13
|
function memoryStore() {
|
|
10
14
|
const values = new Map();
|
|
11
15
|
return {
|
|
@@ -1010,19 +1014,22 @@ test('signed outbox retries the exact message after an unknown network outcome',
|
|
|
1010
1014
|
}
|
|
1011
1015
|
return new Response(JSON.stringify({ ok: true, session: { id: headers.get('x-dharma-session-id') } }), { status: 201 });
|
|
1012
1016
|
};
|
|
1013
|
-
const client = await AgentFabricClient.open({ configPath, statePath, store, fetcher });
|
|
1017
|
+
const client = await AgentFabricClient.open(withOwner({ configPath, statePath, store, fetcher }, 'relay:restart'));
|
|
1014
1018
|
await client.openSession();
|
|
1015
1019
|
failAfterAccept = true;
|
|
1016
1020
|
await assert.rejects(client.registerWorkspace({ workspaceId: 'workspace', name: 'Work' }), /socket closed/);
|
|
1017
1021
|
const state = JSON.parse(await readFile(statePath, 'utf8'));
|
|
1018
|
-
state.pending.headers['x-dharma-timestamp'] = '2026-01-01T00:00:00.000Z';
|
|
1022
|
+
state.outboxes['relay:restart'].pending.headers['x-dharma-timestamp'] = '2026-01-01T00:00:00.000Z';
|
|
1019
1023
|
await writeFile(statePath, JSON.stringify(state));
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1024
|
+
// A restarted process with the same owner key signs no new call in the session it inherited: its session open sends
|
|
1025
|
+
// its kept request again with the same message id, session, sequence and idempotency key, with only a new time and
|
|
1026
|
+
// signature since the old one is too old to send, before it opens its own session.
|
|
1027
|
+
const resumed = await AgentFabricClient.open(withOwner({ configPath, statePath, store, fetcher }, 'relay:restart'));
|
|
1028
|
+
await resumed.openSession();
|
|
1029
|
+
assert.equal(messageIds.at(-3), messageIds.at(-2));
|
|
1030
|
+
assert.equal(idempotencyKeys.at(-3), messageIds.at(-3));
|
|
1023
1031
|
assert.equal(idempotencyKeys.at(-2), messageIds.at(-2));
|
|
1024
|
-
assert.
|
|
1025
|
-
assert.notEqual(timestamps.at(-2), timestamps.at(-1));
|
|
1032
|
+
assert.notEqual(timestamps.at(-3), timestamps.at(-2));
|
|
1026
1033
|
});
|
|
1027
1034
|
for (const scenario of [
|
|
1028
1035
|
{ name: 'workspace body', oldMethod: 'POST', oldRoute: '/agent-fabric/skills/poll', oldBody: { workspaceId: 'first', provider: 'codex' }, method: 'POST', route: '/agent-fabric/skills/poll', body: { workspaceId: 'second', provider: 'codex' } },
|
|
@@ -1057,13 +1064,13 @@ for (const scenario of [
|
|
|
1057
1064
|
}
|
|
1058
1065
|
return new Response(JSON.stringify({ ok: true, organizationId: 'org_a', rollout: null, observed: call }), { status: 200 });
|
|
1059
1066
|
};
|
|
1060
|
-
|
|
1067
|
+
const client = await AgentFabricClient.open(withOwner({ configPath, statePath, store, fetcher }, 'relay:restart'));
|
|
1061
1068
|
await client.openSession();
|
|
1062
1069
|
const send = (method, route, body) => method === 'GET' ? client.signedGet(route) : client.signedPost(route, body);
|
|
1063
1070
|
failures = 1;
|
|
1064
1071
|
await assert.rejects(send(scenario.oldMethod, scenario.oldRoute, scenario.oldBody), /unknown delivery/);
|
|
1065
1072
|
const original = calls.at(-1);
|
|
1066
|
-
|
|
1073
|
+
// The same process continues: only the process that opened a session signs in it.
|
|
1067
1074
|
const pendingBytes = await readFile(statePath, 'utf8');
|
|
1068
1075
|
const count = calls.length;
|
|
1069
1076
|
const invalid = {};
|
|
@@ -1074,7 +1081,7 @@ for (const scenario of [
|
|
|
1074
1081
|
failures = 1;
|
|
1075
1082
|
await assert.rejects(send(scenario.method, scenario.route, scenario.body), /unknown delivery/);
|
|
1076
1083
|
assert.equal(calls.at(-1)?.messageId, original.messageId);
|
|
1077
|
-
assert.equal(JSON.parse(await readFile(statePath, 'utf8')).pending.pathname, `/api/v1/orgs/org_a${scenario.oldRoute}`);
|
|
1084
|
+
assert.equal(JSON.parse(await readFile(statePath, 'utf8')).outboxes['relay:restart'].pending.pathname, `/api/v1/orgs/org_a${scenario.oldRoute}`);
|
|
1078
1085
|
const [current, next] = await Promise.all([
|
|
1079
1086
|
send(scenario.method, scenario.route, scenario.body),
|
|
1080
1087
|
client.signedGet('/control-agent/sessions?sessionId=third'),
|
|
@@ -1089,7 +1096,7 @@ for (const scenario of [
|
|
|
1089
1096
|
assert.deepEqual(current.observed, dispatched);
|
|
1090
1097
|
assert.deepEqual(next.observed, calls.at(-1));
|
|
1091
1098
|
assert.equal(calls.at(-1)?.route, '/control-agent/sessions?sessionId=third');
|
|
1092
|
-
assert.equal(JSON.parse(await readFile(statePath, 'utf8')).pending, null);
|
|
1099
|
+
assert.equal(JSON.parse(await readFile(statePath, 'utf8')).outboxes['relay:restart'].pending, null);
|
|
1093
1100
|
});
|
|
1094
1101
|
}
|
|
1095
1102
|
for (const contentRoute of ['trajectories', 'codex-task-exports']) {
|
|
@@ -1196,11 +1203,11 @@ test('accepted task completion replay preserves the server receipt until the CLI
|
|
|
1196
1203
|
}
|
|
1197
1204
|
return new Response(JSON.stringify({ ok: true }), { status: 201 });
|
|
1198
1205
|
};
|
|
1199
|
-
const client = await AgentFabricClient.open({ configPath, statePath, store, fetcher });
|
|
1206
|
+
const client = await AgentFabricClient.open(withOwner({ configPath, statePath, store, fetcher }, 'relay:restart'));
|
|
1200
1207
|
await client.openSession();
|
|
1201
1208
|
await assert.rejects(client.postTaskEvent(taskId, 'completed', { trajectoryCapsuleHash }), /connection lost/);
|
|
1202
1209
|
assert.deepEqual(client.listRecoveredTaskCompletions(), []);
|
|
1203
|
-
const resumed = await AgentFabricClient.open({ configPath, statePath, store, fetcher });
|
|
1210
|
+
const resumed = await AgentFabricClient.open(withOwner({ configPath, statePath, store, fetcher }, 'relay:restart'));
|
|
1204
1211
|
await resumed.openSession();
|
|
1205
1212
|
assert.equal(completionCalls, 2);
|
|
1206
1213
|
assert.deepEqual(resumed.listRecoveredTaskCompletions().map(({ recoveredAt, ...item }) => item), [{
|
|
@@ -1270,13 +1277,13 @@ test('a malformed successful task acknowledgement remains pending until a durabl
|
|
|
1270
1277
|
}
|
|
1271
1278
|
return new Response(JSON.stringify({ ok: true }), { status: 201 });
|
|
1272
1279
|
};
|
|
1273
|
-
const client = await AgentFabricClient.open({ configPath, statePath, store, fetcher });
|
|
1280
|
+
const client = await AgentFabricClient.open(withOwner({ configPath, statePath, store, fetcher }, 'relay:restart'));
|
|
1274
1281
|
await client.openSession();
|
|
1275
1282
|
await assert.rejects(client.postTaskEvent(taskId, 'completed', { trajectoryCapsuleHash }), /missing a valid durable receipt/);
|
|
1276
1283
|
const pending = JSON.parse(await readFile(statePath, 'utf8'));
|
|
1277
|
-
assert.match(pending.pending.pathname, new RegExp(`${taskId}/events$`));
|
|
1284
|
+
assert.match(pending.outboxes['relay:restart'].pending.pathname, new RegExp(`${taskId}/events$`));
|
|
1278
1285
|
validReceipt = true;
|
|
1279
|
-
const resumed = await AgentFabricClient.open({ configPath, statePath, store, fetcher });
|
|
1286
|
+
const resumed = await AgentFabricClient.open(withOwner({ configPath, statePath, store, fetcher }, 'relay:restart'));
|
|
1280
1287
|
await resumed.openSession();
|
|
1281
1288
|
assert.equal(completionCalls, 2);
|
|
1282
1289
|
assert.equal(JSON.parse(await readFile(statePath, 'utf8')).pending, null);
|
|
@@ -1411,13 +1418,14 @@ test('an ambiguous evidence response is neither persisted nor implicitly replaye
|
|
|
1411
1418
|
}
|
|
1412
1419
|
return new Response(JSON.stringify({ ok: true }), { status: 201 });
|
|
1413
1420
|
};
|
|
1414
|
-
const client = await AgentFabricClient.open({ configPath, statePath, store, fetcher });
|
|
1421
|
+
const client = await AgentFabricClient.open(withOwner({ configPath, statePath, store, fetcher }, 'relay:restart'));
|
|
1415
1422
|
await client.openSession();
|
|
1416
1423
|
await assert.rejects(client.postEvidenceResponse('request-1', { contentBase64: 'customer-content' }), /unknown evidence-response delivery/);
|
|
1417
1424
|
const persisted = await readFile(statePath, 'utf8');
|
|
1418
1425
|
assert.doesNotMatch(persisted, /customer-content/);
|
|
1419
|
-
assert.equal(JSON.parse(persisted).nextSequence, 3);
|
|
1420
|
-
const resumed = await AgentFabricClient.open({ configPath, statePath, store, fetcher });
|
|
1426
|
+
assert.equal(JSON.parse(persisted).outboxes['relay:restart'].nextSequence, 3);
|
|
1427
|
+
const resumed = await AgentFabricClient.open(withOwner({ configPath, statePath, store, fetcher }, 'relay:restart'));
|
|
1428
|
+
await resumed.openSession();
|
|
1421
1429
|
await resumed.registerWorkspace({ workspaceId: 'control-operation' });
|
|
1422
1430
|
assert.equal(paths.filter((path) => path.endsWith('/responses')).length, 1);
|
|
1423
1431
|
});
|
|
@@ -1447,16 +1455,17 @@ test('deterministic client rejection advances the outbox instead of blocking the
|
|
|
1447
1455
|
}
|
|
1448
1456
|
return new Response(JSON.stringify({ ok: true }), { status: 201 });
|
|
1449
1457
|
};
|
|
1450
|
-
const client = await AgentFabricClient.open({ configPath, statePath, store, fetcher });
|
|
1458
|
+
const client = await AgentFabricClient.open(withOwner({ configPath, statePath, store, fetcher }, 'relay:restart'));
|
|
1451
1459
|
await client.openSession();
|
|
1452
1460
|
rejectNext = true;
|
|
1453
1461
|
await assert.rejects(client.registerWorkspace({ workspaceId: 'oversized' }), (error) => {
|
|
1454
1462
|
assert.ok(error instanceof AgentFabricRequestError);
|
|
1455
1463
|
assert.equal(error.status, 413);
|
|
1464
|
+
assert.equal(error.serverCode, 'trajectory_capsule_too_large');
|
|
1456
1465
|
assert.equal(isDefinitiveAgentFabricRejection(error), true);
|
|
1457
1466
|
return true;
|
|
1458
1467
|
});
|
|
1459
|
-
const state = JSON.parse(await readFile(statePath, 'utf8'));
|
|
1468
|
+
const state = JSON.parse(await readFile(statePath, 'utf8')).outboxes['relay:restart'];
|
|
1460
1469
|
assert.equal(state.pending, null);
|
|
1461
1470
|
assert.equal(state.nextSequence, 3);
|
|
1462
1471
|
await client.registerWorkspace({ workspaceId: 'bounded' });
|
|
@@ -1479,16 +1488,17 @@ test('retryable server rejection is an ambiguous delivery outcome', async () =>
|
|
|
1479
1488
|
const fetcher = async () => new Response(JSON.stringify({
|
|
1480
1489
|
ok: false, error: { code: 'upstream_unavailable', message: 'Try again.' },
|
|
1481
1490
|
}), { status: retryable ? 503 : 201 });
|
|
1482
|
-
const client = await AgentFabricClient.open({ configPath, statePath, store, fetcher });
|
|
1491
|
+
const client = await AgentFabricClient.open(withOwner({ configPath, statePath, store, fetcher }, 'relay:restart'));
|
|
1483
1492
|
await client.openSession();
|
|
1484
1493
|
retryable = true;
|
|
1485
1494
|
await assert.rejects(client.registerWorkspace({ workspaceId: 'retryable' }), (error) => {
|
|
1486
1495
|
assert.ok(error instanceof AgentFabricRequestError);
|
|
1487
1496
|
assert.equal(error.status, 503);
|
|
1497
|
+
assert.equal(error.serverCode, 'upstream_unavailable');
|
|
1488
1498
|
assert.equal(isDefinitiveAgentFabricRejection(error), false);
|
|
1489
1499
|
return true;
|
|
1490
1500
|
});
|
|
1491
|
-
assert.notEqual(JSON.parse(await readFile(statePath, 'utf8')).pending, null);
|
|
1501
|
+
assert.notEqual(JSON.parse(await readFile(statePath, 'utf8')).outboxes['relay:restart'].pending, null);
|
|
1492
1502
|
});
|
|
1493
1503
|
test('evidence poll and response remain device-signed organization routes', async () => {
|
|
1494
1504
|
const store = memoryStore();
|
|
@@ -1594,6 +1604,281 @@ test('relay acknowledgement settles once when close emits another socket event',
|
|
|
1594
1604
|
test('relay acknowledgement timeout accommodates managed runtime cold starts', () => {
|
|
1595
1605
|
assert.equal(RELAY_ACKNOWLEDGEMENT_TIMEOUT_MS, 90_000);
|
|
1596
1606
|
});
|
|
1607
|
+
test('a relay that never acknowledges fails with the stable relay_response_timeout code', async (t) => {
|
|
1608
|
+
const store = memoryStore();
|
|
1609
|
+
const root = await mkdtemp(resolve(tmpdir(), 'fabric-relay-timeout-'));
|
|
1610
|
+
const identity = await loadOrCreateDeviceIdentity({ hqUrl: 'https://hq.example', organizationId: 'org_a', store });
|
|
1611
|
+
const configPath = resolve(root, 'device.json');
|
|
1612
|
+
await saveDeviceConfig(configPath, {
|
|
1613
|
+
schema: 'dharma.device-config/v1', hqUrl: 'https://hq.example', organizationId: 'org_a',
|
|
1614
|
+
deviceId: 'c72c7f13-e420-49f7-a818-c07f6f9d0915', deviceName: 'Test', platform: 'linux',
|
|
1615
|
+
publicKeyEd25519: identity.publicKeyEd25519, serverPublicKeyEd25519: identity.publicKeyEd25519,
|
|
1616
|
+
relayUrl: 'wss://relay.example', enrolledAt: new Date().toISOString(),
|
|
1617
|
+
});
|
|
1618
|
+
await anchorConfig(configPath, store);
|
|
1619
|
+
const originalWebSocket = globalThis.WebSocket;
|
|
1620
|
+
// Opens and accepts the request, then never answers: only the acknowledgement timer can settle it.
|
|
1621
|
+
class SilentSocket {
|
|
1622
|
+
static CONNECTING = 0;
|
|
1623
|
+
static OPEN = 1;
|
|
1624
|
+
static CLOSING = 2;
|
|
1625
|
+
static CLOSED = 3;
|
|
1626
|
+
readyState = SilentSocket.CONNECTING;
|
|
1627
|
+
#listeners = new Map();
|
|
1628
|
+
constructor(_url) {
|
|
1629
|
+
queueMicrotask(() => {
|
|
1630
|
+
this.readyState = SilentSocket.OPEN;
|
|
1631
|
+
for (const listener of this.#listeners.get('open') || [])
|
|
1632
|
+
listener({});
|
|
1633
|
+
});
|
|
1634
|
+
}
|
|
1635
|
+
addEventListener(type, listener) {
|
|
1636
|
+
this.#listeners.set(type, [...(this.#listeners.get(type) || []), listener]);
|
|
1637
|
+
}
|
|
1638
|
+
send(_serialized) { queueMicrotask(() => t.mock.timers.tick(RELAY_ACKNOWLEDGEMENT_TIMEOUT_MS)); }
|
|
1639
|
+
close(_code) { this.readyState = SilentSocket.CLOSED; }
|
|
1640
|
+
}
|
|
1641
|
+
Object.defineProperty(globalThis, 'WebSocket', { configurable: true, value: SilentSocket, writable: true });
|
|
1642
|
+
try {
|
|
1643
|
+
const client = await AgentFabricClient.open({ configPath, statePath: resolve(root, 'state.json'), store });
|
|
1644
|
+
t.mock.timers.enable({ apis: ['setTimeout'] });
|
|
1645
|
+
await assert.rejects(client.openSession(), (error) => {
|
|
1646
|
+
assert.equal(error.code, 'relay_response_timeout');
|
|
1647
|
+
assert.equal(error.message, 'Relay response timed out.');
|
|
1648
|
+
return true;
|
|
1649
|
+
});
|
|
1650
|
+
}
|
|
1651
|
+
finally {
|
|
1652
|
+
t.mock.timers.reset();
|
|
1653
|
+
Object.defineProperty(globalThis, 'WebSocket', { configurable: true, value: originalWebSocket, writable: true });
|
|
1654
|
+
}
|
|
1655
|
+
});
|
|
1656
|
+
// #512: one signed relay request holds device signing control until it is acknowledged. The connect phase has its
|
|
1657
|
+
// own, shorter limit: before the socket opens this attempt has sent nothing, so failing there is a clean failure and
|
|
1658
|
+
// the kept request is sent on the next attempt, as after any other failure that keeps it. After the request is sent
|
|
1659
|
+
// the 90 s acknowledgement window is unchanged.
|
|
1660
|
+
function relayConnectTimeoutMs() {
|
|
1661
|
+
const value = relayClientModule.RELAY_CONNECT_TIMEOUT_MS;
|
|
1662
|
+
assert.equal(typeof value, 'number', 'the relay connect phase has no time limit of its own');
|
|
1663
|
+
return value;
|
|
1664
|
+
}
|
|
1665
|
+
// A WebSocket double the test drives: onCreate runs after the client has set its timers, onSend after each send.
|
|
1666
|
+
function scriptedRelay(script) {
|
|
1667
|
+
const sockets = [];
|
|
1668
|
+
class Socket {
|
|
1669
|
+
static CONNECTING = 0;
|
|
1670
|
+
static OPEN = 1;
|
|
1671
|
+
static CLOSING = 2;
|
|
1672
|
+
static CLOSED = 3;
|
|
1673
|
+
readyState = Socket.CONNECTING;
|
|
1674
|
+
sent = [];
|
|
1675
|
+
closeCalls = 0;
|
|
1676
|
+
#listeners = new Map();
|
|
1677
|
+
constructor(_url) {
|
|
1678
|
+
const index = sockets.push(this) - 1;
|
|
1679
|
+
queueMicrotask(() => script.onCreate?.(this, index));
|
|
1680
|
+
}
|
|
1681
|
+
addEventListener(type, listener) {
|
|
1682
|
+
this.#listeners.set(type, [...(this.#listeners.get(type) || []), listener]);
|
|
1683
|
+
}
|
|
1684
|
+
#emit(type, event) { for (const listener of this.#listeners.get(type) || [])
|
|
1685
|
+
listener(event); }
|
|
1686
|
+
open() { this.readyState = Socket.OPEN; this.#emit('open', {}); }
|
|
1687
|
+
answer(status, body) {
|
|
1688
|
+
const request = JSON.parse(this.sent.at(-1));
|
|
1689
|
+
this.#emit('message', { data: JSON.stringify({ requestId: request.requestId, status, body: JSON.stringify(body) }) });
|
|
1690
|
+
}
|
|
1691
|
+
send(serialized) { this.sent.push(serialized); queueMicrotask(() => script.onSend?.(this, serialized)); }
|
|
1692
|
+
close(_code) { this.closeCalls += 1; this.readyState = Socket.CLOSED; }
|
|
1693
|
+
}
|
|
1694
|
+
return { Socket, sockets };
|
|
1695
|
+
}
|
|
1696
|
+
async function relayConnectFixture(prefix) {
|
|
1697
|
+
const store = memoryStore();
|
|
1698
|
+
const root = await mkdtemp(resolve(tmpdir(), prefix));
|
|
1699
|
+
const identity = await loadOrCreateDeviceIdentity({ hqUrl: 'https://hq.example', organizationId: 'org_a', store });
|
|
1700
|
+
const configPath = resolve(root, 'device.json'), statePath = resolve(root, 'state.json');
|
|
1701
|
+
const config = {
|
|
1702
|
+
schema: 'dharma.device-config/v1', hqUrl: 'https://hq.example', organizationId: 'org_a',
|
|
1703
|
+
deviceId: 'c72c7f13-e420-49f7-a818-c07f6f9d0915', deviceName: 'Test', platform: 'linux',
|
|
1704
|
+
publicKeyEd25519: identity.publicKeyEd25519, serverPublicKeyEd25519: identity.publicKeyEd25519,
|
|
1705
|
+
relayUrl: 'wss://relay.example', enrolledAt: new Date().toISOString(),
|
|
1706
|
+
};
|
|
1707
|
+
await saveDeviceConfig(configPath, config);
|
|
1708
|
+
await anchorConfig(configPath, store);
|
|
1709
|
+
return { store, configPath, statePath, config };
|
|
1710
|
+
}
|
|
1711
|
+
async function withRelaySocket(Socket, work) {
|
|
1712
|
+
const originalWebSocket = globalThis.WebSocket;
|
|
1713
|
+
Object.defineProperty(globalThis, 'WebSocket', { configurable: true, value: Socket, writable: true });
|
|
1714
|
+
try {
|
|
1715
|
+
return await work();
|
|
1716
|
+
}
|
|
1717
|
+
finally {
|
|
1718
|
+
Object.defineProperty(globalThis, 'WebSocket', { configurable: true, value: originalWebSocket, writable: true });
|
|
1719
|
+
}
|
|
1720
|
+
}
|
|
1721
|
+
async function storedPending(statePath) {
|
|
1722
|
+
const state = JSON.parse(await readFile(statePath, 'utf8'));
|
|
1723
|
+
return Object.values(state.outboxes ?? {}).find(entry => entry.pending)?.pending ?? null;
|
|
1724
|
+
}
|
|
1725
|
+
function isConnectTimeout(error) {
|
|
1726
|
+
assert.equal(error.code, 'relay_connect_timeout');
|
|
1727
|
+
assert.equal(error.message, 'Relay connection timed out before the request was sent.');
|
|
1728
|
+
return true;
|
|
1729
|
+
}
|
|
1730
|
+
test('the relay connect limit is 30 s and shorter than the acknowledgement window', () => {
|
|
1731
|
+
assert.equal(relayConnectTimeoutMs(), 30_000);
|
|
1732
|
+
assert.ok(relayConnectTimeoutMs() < RELAY_ACKNOWLEDGEMENT_TIMEOUT_MS);
|
|
1733
|
+
assert.equal(RELAY_ACKNOWLEDGEMENT_TIMEOUT_MS, 90_000);
|
|
1734
|
+
});
|
|
1735
|
+
test('a relay that never accepts the connection fails at the connect limit, sends nothing and frees signing control', async (t) => {
|
|
1736
|
+
const limit = relayConnectTimeoutMs();
|
|
1737
|
+
const f = await relayConnectFixture('fabric-relay-connect-timeout-');
|
|
1738
|
+
const relay = scriptedRelay({ onCreate: () => t.mock.timers.tick(limit) });
|
|
1739
|
+
await withRelaySocket(relay.Socket, async () => {
|
|
1740
|
+
const client = await AgentFabricClient.open({ configPath: f.configPath, statePath: f.statePath, store: f.store });
|
|
1741
|
+
t.mock.timers.enable({ apis: ['setTimeout'] });
|
|
1742
|
+
try {
|
|
1743
|
+
await assert.rejects(client.openSession(), isConnectTimeout);
|
|
1744
|
+
}
|
|
1745
|
+
finally {
|
|
1746
|
+
t.mock.timers.reset();
|
|
1747
|
+
}
|
|
1748
|
+
});
|
|
1749
|
+
assert.equal(relay.sockets.length, 1);
|
|
1750
|
+
assert.equal(relay.sockets[0].sent.length, 0, 'a connect timeout means the request was never sent');
|
|
1751
|
+
assert.equal(relay.sockets[0].closeCalls, 1);
|
|
1752
|
+
const pending = await storedPending(f.statePath);
|
|
1753
|
+
assert.match(pending?.pathname ?? '', /\/agent-fabric\/sessions$/, 'the signed request is kept for the next attempt');
|
|
1754
|
+
const started = Date.now();
|
|
1755
|
+
assert.equal(await withDeviceSigningControl({ config: f.config, timeoutMs: 1_000 }, async () => 'free'), 'free');
|
|
1756
|
+
assert.ok(Date.now() - started < 1_000, 'signing control is free once the connect limit fails the request');
|
|
1757
|
+
});
|
|
1758
|
+
test('a connection accepted just inside the limit keeps the full acknowledgement window', async (t) => {
|
|
1759
|
+
const limit = relayConnectTimeoutMs();
|
|
1760
|
+
const f = await relayConnectFixture('fabric-relay-connect-late-open-');
|
|
1761
|
+
const relay = scriptedRelay({
|
|
1762
|
+
onCreate: socket => { t.mock.timers.tick(limit - 1); socket.open(); },
|
|
1763
|
+
// Answers at 89 999 ms after the connection started: 1 ms before the unchanged acknowledgement deadline.
|
|
1764
|
+
onSend: socket => { t.mock.timers.tick(RELAY_ACKNOWLEDGEMENT_TIMEOUT_MS - limit); socket.answer(201, { ok: true }); },
|
|
1765
|
+
});
|
|
1766
|
+
await withRelaySocket(relay.Socket, async () => {
|
|
1767
|
+
const client = await AgentFabricClient.open({ configPath: f.configPath, statePath: f.statePath, store: f.store });
|
|
1768
|
+
t.mock.timers.enable({ apis: ['setTimeout'] });
|
|
1769
|
+
try {
|
|
1770
|
+
assert.deepEqual(await client.openSession(), { ok: true });
|
|
1771
|
+
}
|
|
1772
|
+
finally {
|
|
1773
|
+
t.mock.timers.reset();
|
|
1774
|
+
}
|
|
1775
|
+
});
|
|
1776
|
+
assert.equal(relay.sockets[0].sent.length, 1);
|
|
1777
|
+
assert.equal(await storedPending(f.statePath), null);
|
|
1778
|
+
});
|
|
1779
|
+
test('a socket that opens after the connect limit never sends the request', async (t) => {
|
|
1780
|
+
const limit = relayConnectTimeoutMs();
|
|
1781
|
+
const f = await relayConnectFixture('fabric-relay-connect-open-after-');
|
|
1782
|
+
const relay = scriptedRelay({ onCreate: () => t.mock.timers.tick(limit) });
|
|
1783
|
+
await withRelaySocket(relay.Socket, async () => {
|
|
1784
|
+
const client = await AgentFabricClient.open({ configPath: f.configPath, statePath: f.statePath, store: f.store });
|
|
1785
|
+
t.mock.timers.enable({ apis: ['setTimeout'] });
|
|
1786
|
+
try {
|
|
1787
|
+
await assert.rejects(client.openSession(), isConnectTimeout);
|
|
1788
|
+
}
|
|
1789
|
+
finally {
|
|
1790
|
+
t.mock.timers.reset();
|
|
1791
|
+
}
|
|
1792
|
+
});
|
|
1793
|
+
relay.sockets[0].open();
|
|
1794
|
+
await new Promise(accept => setImmediate(accept));
|
|
1795
|
+
assert.equal(relay.sockets[0].sent.length, 0);
|
|
1796
|
+
});
|
|
1797
|
+
test('retrying a request that hit the connect limit sends the original signed request exactly once', async (t) => {
|
|
1798
|
+
const limit = relayConnectTimeoutMs();
|
|
1799
|
+
const f = await relayConnectFixture('fabric-relay-connect-retry-');
|
|
1800
|
+
const relay = scriptedRelay({
|
|
1801
|
+
onCreate: (socket, index) => { if (index === 0)
|
|
1802
|
+
t.mock.timers.tick(limit);
|
|
1803
|
+
else
|
|
1804
|
+
socket.open(); },
|
|
1805
|
+
onSend: socket => socket.answer(201, { ok: true }),
|
|
1806
|
+
});
|
|
1807
|
+
await withRelaySocket(relay.Socket, async () => {
|
|
1808
|
+
const client = await AgentFabricClient.open({ configPath: f.configPath, statePath: f.statePath, store: f.store });
|
|
1809
|
+
t.mock.timers.enable({ apis: ['setTimeout'] });
|
|
1810
|
+
try {
|
|
1811
|
+
await assert.rejects(client.openSession(), isConnectTimeout);
|
|
1812
|
+
}
|
|
1813
|
+
finally {
|
|
1814
|
+
t.mock.timers.reset();
|
|
1815
|
+
}
|
|
1816
|
+
const pending = await storedPending(f.statePath);
|
|
1817
|
+
assert.ok(pending, 'the signed request is kept after a connect timeout');
|
|
1818
|
+
assert.deepEqual(await client.signedPost('/agent-fabric/sessions', JSON.parse(pending.body)), { ok: true });
|
|
1819
|
+
const sends = relay.sockets.flatMap(socket => socket.sent)
|
|
1820
|
+
.map(serialized => JSON.parse(serialized));
|
|
1821
|
+
assert.equal(sends.length, 1, 'the retry is the first and only send');
|
|
1822
|
+
assert.equal(sends[0].requestId, pending.headers['x-dharma-message-id']);
|
|
1823
|
+
assert.equal(sends[0].headers['x-dharma-message-id'], pending.headers['x-dharma-message-id']);
|
|
1824
|
+
assert.equal(sends[0].headers['x-dharma-nonce'], pending.headers['x-dharma-nonce']);
|
|
1825
|
+
assert.equal(sends[0].headers['x-dharma-signature'], pending.headers['x-dharma-signature']);
|
|
1826
|
+
});
|
|
1827
|
+
assert.equal(await storedPending(f.statePath), null);
|
|
1828
|
+
});
|
|
1829
|
+
test('a kept request already sent before a connect timeout is sent again unchanged on retry', async (t) => {
|
|
1830
|
+
const limit = relayConnectTimeoutMs();
|
|
1831
|
+
const f = await relayConnectFixture('fabric-relay-connect-resend-');
|
|
1832
|
+
const relay = scriptedRelay({
|
|
1833
|
+
onCreate: (socket, index) => { if (index === 1)
|
|
1834
|
+
t.mock.timers.tick(limit);
|
|
1835
|
+
else
|
|
1836
|
+
socket.open(); },
|
|
1837
|
+
onSend: socket => {
|
|
1838
|
+
if (socket === relay.sockets[0])
|
|
1839
|
+
t.mock.timers.tick(RELAY_ACKNOWLEDGEMENT_TIMEOUT_MS);
|
|
1840
|
+
else
|
|
1841
|
+
socket.answer(201, { ok: true });
|
|
1842
|
+
},
|
|
1843
|
+
});
|
|
1844
|
+
await withRelaySocket(relay.Socket, async () => {
|
|
1845
|
+
const client = await AgentFabricClient.open({ configPath: f.configPath, statePath: f.statePath, store: f.store });
|
|
1846
|
+
t.mock.timers.enable({ apis: ['setTimeout'] });
|
|
1847
|
+
try {
|
|
1848
|
+
await assert.rejects(client.openSession(), (error) => error.code === 'relay_response_timeout');
|
|
1849
|
+
}
|
|
1850
|
+
finally {
|
|
1851
|
+
t.mock.timers.reset();
|
|
1852
|
+
}
|
|
1853
|
+
const pending = (await storedPending(f.statePath));
|
|
1854
|
+
t.mock.timers.enable({ apis: ['setTimeout'] });
|
|
1855
|
+
try {
|
|
1856
|
+
await assert.rejects(client.signedPost('/agent-fabric/sessions', JSON.parse(pending.body)), isConnectTimeout);
|
|
1857
|
+
}
|
|
1858
|
+
finally {
|
|
1859
|
+
t.mock.timers.reset();
|
|
1860
|
+
}
|
|
1861
|
+
assert.deepEqual(await client.signedPost('/agent-fabric/sessions', JSON.parse(pending.body)), { ok: true });
|
|
1862
|
+
const sends = relay.sockets.flatMap(socket => socket.sent)
|
|
1863
|
+
.map(serialized => JSON.parse(serialized));
|
|
1864
|
+
assert.equal(relay.sockets[1].sent.length, 0, 'the connect-timeout attempt sent nothing');
|
|
1865
|
+
assert.equal(sends.length, 2, 'the retry is the second send of a request first sent before the response timeout');
|
|
1866
|
+
for (const header of ['x-dharma-message-id', 'x-dharma-nonce', 'x-dharma-signature']) {
|
|
1867
|
+
assert.equal(sends[1].headers[header], sends[0].headers[header], header);
|
|
1868
|
+
}
|
|
1869
|
+
});
|
|
1870
|
+
assert.equal(await storedPending(f.statePath), null);
|
|
1871
|
+
});
|
|
1872
|
+
test('the README describes only the signing-control guarantees the client provides (#512 review)', async () => {
|
|
1873
|
+
const readme = (await readFile(new URL('../README.md', import.meta.url), 'utf8')).replace(/\s+/g, ' ');
|
|
1874
|
+
const start = readme.indexOf('## How long one signed relay request holds device signing control');
|
|
1875
|
+
assert.ok(start >= 0, 'the #512 README section exists');
|
|
1876
|
+
const end = readme.indexOf(' ## ', start + 1);
|
|
1877
|
+
const section = readme.slice(start, end === -1 ? undefined : end);
|
|
1878
|
+
assert.doesNotMatch(section, /never in flight twice/, 'a kept request can be sent again after relay_response_timeout');
|
|
1879
|
+
assert.doesNotMatch(section, /first send/, 'a kept request may already have been sent by an earlier attempt');
|
|
1880
|
+
assert.match(section, /may be sent again/, 'the README says a kept signed request may be sent again');
|
|
1881
|
+
});
|
|
1597
1882
|
test('missing-anchor restoration preserves current, legacy, corrupt, cached and unavailable protected state', async () => {
|
|
1598
1883
|
const config = { schema: 'dharma.device-config/v1', hqUrl: 'https://hq.example', organizationId: 'org_fixture',
|
|
1599
1884
|
deviceId: '22222222-2222-4222-8222-222222222222', deviceName: 'Synthetic', platform: 'linux',
|
|
@@ -1813,4 +2098,2285 @@ test('device keyset telemetry reports a keyset-less device as null and rejects m
|
|
|
1813
2098
|
}
|
|
1814
2099
|
assert.throws(() => deviceKeysetTelemetry(legacy, { relayAutostartEnabled: 'yes' }), /device_keyset_telemetry_invalid/);
|
|
1815
2100
|
});
|
|
2101
|
+
/** A stable owner lane, such as the one a relay workspace loop passes, so a restarted client finds its own entry. */
|
|
2102
|
+
const withOwner = (input, ownerKey) => ({ ...input, ownerKey });
|
|
2103
|
+
const keptMessageIds = (state) => Object.values(state.outboxes ?? {})
|
|
2104
|
+
.flatMap(entry => entry.pending ? [entry.pending.headers['x-dharma-message-id']] : []);
|
|
2105
|
+
/** The pid of a process that has exited, standing in for an ad hoc command that left a kept request behind. */
|
|
2106
|
+
async function exitedPid() {
|
|
2107
|
+
const child = spawn(process.execPath, ['-e', ''], { stdio: 'ignore' });
|
|
2108
|
+
await new Promise(accept => child.once('exit', accept));
|
|
2109
|
+
return child.pid;
|
|
2110
|
+
}
|
|
2111
|
+
const sharedOk = (body = {}) => new Response(JSON.stringify({ ok: true, ...body }), { status: 201 });
|
|
2112
|
+
const sharedUnavailable = () => new Response(JSON.stringify({
|
|
2113
|
+
ok: false, error: { code: 'upstream_unavailable', message: 'Try again.' }
|
|
2114
|
+
}), { status: 503 });
|
|
2115
|
+
const sharedGate = () => { let open; const promise = new Promise(accept => { open = accept; }); return { promise, open }; };
|
|
2116
|
+
const sharedDelay = (ms) => new Promise(accept => setTimeout(accept, ms));
|
|
2117
|
+
async function sharedStateFixture(prefix, respond, store = memoryStore()) {
|
|
2118
|
+
const root = await mkdtemp(resolve(tmpdir(), prefix));
|
|
2119
|
+
const identity = await loadOrCreateDeviceIdentity({ hqUrl: 'https://hq.example', organizationId: 'org_a', store });
|
|
2120
|
+
const configPath = resolve(root, 'device.json'), statePath = resolve(root, 'state.json');
|
|
2121
|
+
const config = {
|
|
2122
|
+
schema: 'dharma.device-config/v1', hqUrl: 'https://hq.example', organizationId: 'org_a',
|
|
2123
|
+
deviceId: randomUUID(), deviceName: 'Test', platform: 'linux',
|
|
2124
|
+
publicKeyEd25519: identity.publicKeyEd25519, serverPublicKeyEd25519: identity.publicKeyEd25519,
|
|
2125
|
+
relayUrl: 'wss://relay.example', enrolledAt: new Date().toISOString(),
|
|
2126
|
+
};
|
|
2127
|
+
await saveDeviceConfig(configPath, config);
|
|
2128
|
+
await anchorConfig(configPath, store);
|
|
2129
|
+
const calls = [];
|
|
2130
|
+
const inFlight = new Set();
|
|
2131
|
+
let overlapping = 0;
|
|
2132
|
+
const fetcher = async (url, init) => {
|
|
2133
|
+
const headers = new Headers(init?.headers);
|
|
2134
|
+
const call = {
|
|
2135
|
+
path: new URL(String(url)).pathname.replace('/api/v1/orgs/org_a', ''), body: String(init?.body ?? ''),
|
|
2136
|
+
messageId: headers.get('x-dharma-message-id'), idempotencyKey: headers.get('idempotency-key'),
|
|
2137
|
+
sessionId: headers.get('x-dharma-session-id'), sequence: Number(headers.get('x-dharma-sequence')),
|
|
2138
|
+
headers: Object.fromEntries(headers.entries()),
|
|
2139
|
+
};
|
|
2140
|
+
calls.push(call);
|
|
2141
|
+
if (inFlight.has(call.messageId))
|
|
2142
|
+
overlapping += 1;
|
|
2143
|
+
inFlight.add(call.messageId);
|
|
2144
|
+
try {
|
|
2145
|
+
return await respond(call);
|
|
2146
|
+
}
|
|
2147
|
+
finally {
|
|
2148
|
+
inFlight.delete(call.messageId);
|
|
2149
|
+
}
|
|
2150
|
+
};
|
|
2151
|
+
return {
|
|
2152
|
+
config, configPath, statePath, calls, overlapping: () => overlapping, store, fetcher,
|
|
2153
|
+
open: (ownerKey) => AgentFabricClient.open(ownerKey === undefined
|
|
2154
|
+
? { configPath, statePath, store, fetcher } : withOwner({ configPath, statePath, store, fetcher }, ownerKey)),
|
|
2155
|
+
read: async () => JSON.parse(await readFile(statePath, 'utf8')),
|
|
2156
|
+
};
|
|
2157
|
+
}
|
|
2158
|
+
// The child holds the same device signing control from another process, through the real lock file.
|
|
2159
|
+
function sharedChildHolder(config) {
|
|
2160
|
+
const module = new URL('./index.js', import.meta.url).href;
|
|
2161
|
+
const child = spawn(process.execPath, ['--input-type=module', '-e', `import {withDeviceSigningControl} from ${JSON.stringify(module)};
|
|
2162
|
+
await withDeviceSigningControl({config:JSON.parse(process.argv[1])},async()=>{process.stdout.write('held;');
|
|
2163
|
+
await new Promise(accept=>process.stdin.once('data',accept));});process.stdout.write('released;');`,
|
|
2164
|
+
JSON.stringify({ hqUrl: config.hqUrl, organizationId: config.organizationId, deviceId: config.deviceId })], { stdio: ['pipe', 'pipe', 'inherit'] });
|
|
2165
|
+
let output = '';
|
|
2166
|
+
const waiters = [];
|
|
2167
|
+
child.stdout.on('data', chunk => { output += chunk; for (const wake of waiters.splice(0))
|
|
2168
|
+
wake(); });
|
|
2169
|
+
const saw = async (text) => { while (!output.includes(text))
|
|
2170
|
+
await new Promise(wake => waiters.push(wake)); };
|
|
2171
|
+
const exited = new Promise(accept => child.once('exit', code => accept(code)));
|
|
2172
|
+
return { child, saw, exited };
|
|
2173
|
+
}
|
|
2174
|
+
test('a task completion recorded by one client survives a later write by another client (shared protocol state)', async () => {
|
|
2175
|
+
const taskId = randomUUID(), trajectoryCapsuleHash = `sha256:${'a'.repeat(64)}`, receiptHash = `sha256:${'b'.repeat(64)}`;
|
|
2176
|
+
const f = await sharedStateFixture('fabric-shared-completion-', call => sharedOk(call.path.endsWith('/events') ? { receipt: { hash: receiptHash } } : {}));
|
|
2177
|
+
const a = await f.open(), b = await f.open();
|
|
2178
|
+
await a.openSession();
|
|
2179
|
+
await b.openSession();
|
|
2180
|
+
await a.postTaskEvent(taskId, 'completed', { trajectoryCapsuleHash });
|
|
2181
|
+
await b.pollTask();
|
|
2182
|
+
assert.deepEqual((await f.read()).recoveredTaskCompletions?.map(item => item.taskId), [taskId], 'a write by another client keeps the recorded completion');
|
|
2183
|
+
await b.acknowledgeRecoveredTaskCompletion(taskId, receiptHash);
|
|
2184
|
+
assert.deepEqual((await f.read()).recoveredTaskCompletions, [], 'any client on the device can acknowledge it');
|
|
2185
|
+
});
|
|
2186
|
+
test('opening a session never overwrites a request another client has in flight (shared protocol state)', async () => {
|
|
2187
|
+
const gate = sharedGate(), entered = sharedGate();
|
|
2188
|
+
let block = false;
|
|
2189
|
+
const f = await sharedStateFixture('fabric-shared-inflight-', async (call) => {
|
|
2190
|
+
if (block && call.path === '/agent-fabric/workspaces') {
|
|
2191
|
+
block = false;
|
|
2192
|
+
entered.open();
|
|
2193
|
+
await gate.promise;
|
|
2194
|
+
}
|
|
2195
|
+
return sharedOk();
|
|
2196
|
+
});
|
|
2197
|
+
const a = await f.open(), b = await f.open();
|
|
2198
|
+
await a.openSession();
|
|
2199
|
+
block = true;
|
|
2200
|
+
const registering = a.registerWorkspace({ workspaceId: 'a' });
|
|
2201
|
+
await entered.promise;
|
|
2202
|
+
const inFlight = f.calls.at(-1).messageId;
|
|
2203
|
+
const opening = b.openSession();
|
|
2204
|
+
await sharedDelay(300);
|
|
2205
|
+
assert.deepEqual(keptMessageIds(await f.read()), [inFlight], 'the in-flight request stays in its owner\'s entry');
|
|
2206
|
+
assert.equal(f.calls.filter(call => call.path === '/agent-fabric/sessions').length, 1, 'the second session open waits');
|
|
2207
|
+
gate.open();
|
|
2208
|
+
await registering;
|
|
2209
|
+
await opening;
|
|
2210
|
+
assert.deepEqual(f.calls.map(call => call.path), ['/agent-fabric/sessions', '/agent-fabric/workspaces', '/agent-fabric/sessions']);
|
|
2211
|
+
assert.equal((await f.read()).pending, null);
|
|
2212
|
+
assert.deepEqual(keptMessageIds(await f.read()), []);
|
|
2213
|
+
});
|
|
2214
|
+
test('a request another client already delivered is never sent again from a stale snapshot (shared protocol state)', async () => {
|
|
2215
|
+
const delivered = new Set(), gate = sharedGate(), entered = sharedGate();
|
|
2216
|
+
let mode = 'ok';
|
|
2217
|
+
const f = await sharedStateFixture('fabric-shared-replay-', async (call) => {
|
|
2218
|
+
if (call.path === '/agent-fabric/workspaces' && mode === 'fail') {
|
|
2219
|
+
mode = 'ok';
|
|
2220
|
+
return sharedUnavailable();
|
|
2221
|
+
}
|
|
2222
|
+
if (call.path === '/agent-fabric/workspaces' && mode === 'block') {
|
|
2223
|
+
mode = 'ok';
|
|
2224
|
+
entered.open();
|
|
2225
|
+
await gate.promise;
|
|
2226
|
+
}
|
|
2227
|
+
if (delivered.has(call.messageId)) {
|
|
2228
|
+
return new Response(JSON.stringify({ ok: false, error: { code: 'message_replayed', message: 'Already accepted.' } }), { status: 409 });
|
|
2229
|
+
}
|
|
2230
|
+
delivered.add(call.messageId);
|
|
2231
|
+
return sharedOk();
|
|
2232
|
+
});
|
|
2233
|
+
const a = await f.open();
|
|
2234
|
+
await a.openSession();
|
|
2235
|
+
mode = 'fail';
|
|
2236
|
+
await assert.rejects(a.registerWorkspace({ workspaceId: 'a' }), error => error instanceof AgentFabricRequestError && error.status === 503);
|
|
2237
|
+
const [kept] = keptMessageIds(await f.read());
|
|
2238
|
+
assert.ok(kept);
|
|
2239
|
+
const b = await f.open();
|
|
2240
|
+
mode = 'block';
|
|
2241
|
+
const retry = a.registerWorkspace({ workspaceId: 'a' });
|
|
2242
|
+
await entered.promise;
|
|
2243
|
+
const opening = b.openSession();
|
|
2244
|
+
await sharedDelay(100);
|
|
2245
|
+
gate.open();
|
|
2246
|
+
await retry;
|
|
2247
|
+
await opening;
|
|
2248
|
+
assert.equal(f.calls.filter(call => call.messageId === kept).length, 2, 'the kept request is sent by its first attempt and by its one retry, never again from a stale snapshot');
|
|
2249
|
+
assert.equal(f.overlapping(), 0, 'one message id is never in flight twice at once');
|
|
2250
|
+
});
|
|
2251
|
+
test('another live client never sends, fails on or sets aside a client\'s kept request (shared protocol state)', async () => {
|
|
2252
|
+
const f = await sharedStateFixture('fabric-shared-live-', call => call.path === '/agent-fabric/workspaces' ? sharedUnavailable() : sharedOk({ observed: call }));
|
|
2253
|
+
const a = await f.open(), b = await f.open();
|
|
2254
|
+
await a.openSession();
|
|
2255
|
+
await b.openSession();
|
|
2256
|
+
await assert.rejects(a.registerWorkspace({ workspaceId: 'a' }), error => error instanceof AgentFabricRequestError && error.status === 503);
|
|
2257
|
+
const kept = f.calls.at(-1);
|
|
2258
|
+
for (let call = 0; call < 5; call++) {
|
|
2259
|
+
const polled = await b.pollTask();
|
|
2260
|
+
assert.deepEqual(polled.observed, f.calls.at(-1), 'every call of the other client gets the response to its own request');
|
|
2261
|
+
}
|
|
2262
|
+
const c = await f.open();
|
|
2263
|
+
await c.openSession();
|
|
2264
|
+
await c.pollTask();
|
|
2265
|
+
assert.equal(f.calls.filter(call => call.messageId === kept.messageId).length, 1, 'no other live client sends the kept request');
|
|
2266
|
+
assert.deepEqual(keptMessageIds(await f.read()), [kept.messageId], 'the kept request stays in its owner\'s entry');
|
|
2267
|
+
await assert.rejects(a.registerWorkspace({ workspaceId: 'a' }), error => error instanceof AgentFabricRequestError && error.status === 503);
|
|
2268
|
+
assert.deepEqual([f.calls.at(-1).messageId, f.calls.at(-1).idempotencyKey], [kept.messageId, kept.messageId], 'its owner sends it again with the same message id and idempotency key');
|
|
2269
|
+
});
|
|
2270
|
+
test('a kept task completion survives any number of another client\'s calls during an outage and its owner replays it (shared protocol state)', async () => {
|
|
2271
|
+
const taskId = randomUUID(), trajectoryCapsuleHash = `sha256:${'a'.repeat(64)}`, receiptHash = `sha256:${'b'.repeat(64)}`;
|
|
2272
|
+
let outage = false;
|
|
2273
|
+
const f = await sharedStateFixture('fabric-shared-outage-', call => outage ? sharedUnavailable()
|
|
2274
|
+
: sharedOk(call.path.endsWith('/events') ? { receipt: { hash: receiptHash } } : {}));
|
|
2275
|
+
const a = await f.open(), b = await f.open();
|
|
2276
|
+
await a.openSession();
|
|
2277
|
+
await b.openSession();
|
|
2278
|
+
outage = true;
|
|
2279
|
+
await assert.rejects(a.postTaskEvent(taskId, 'completed', { trajectoryCapsuleHash }), error => error instanceof AgentFabricRequestError && error.status === 503);
|
|
2280
|
+
const kept = f.calls.at(-1);
|
|
2281
|
+
for (let call = 0; call < 6; call++) {
|
|
2282
|
+
await assert.rejects(b.pollTask(), error => error instanceof AgentFabricRequestError && error.status === 503, 'the other client sees only the outcome of its own request');
|
|
2283
|
+
assert.notEqual(f.calls.at(-1).messageId, kept.messageId, 'the other client sends only its own request');
|
|
2284
|
+
}
|
|
2285
|
+
assert.ok(keptMessageIds(await f.read()).includes(kept.messageId), 'the completion is still kept');
|
|
2286
|
+
outage = false;
|
|
2287
|
+
await a.pollTask();
|
|
2288
|
+
const sent = f.calls.filter(call => call.messageId === kept.messageId);
|
|
2289
|
+
assert.equal(sent.length, 2, 'the completion is sent by its first attempt and by its owner\'s replay only');
|
|
2290
|
+
assert.deepEqual(sent.map(call => call.idempotencyKey), [kept.messageId, kept.messageId]);
|
|
2291
|
+
assert.deepEqual((await f.read()).recoveredTaskCompletions?.map(item => item.taskId), [taskId]);
|
|
2292
|
+
assert.equal(keptMessageIds(await f.read()).includes(kept.messageId), false, 'the replayed completion left the file');
|
|
2293
|
+
});
|
|
2294
|
+
test('a workspace loop that restarts replays its kept completion when it opens its session (shared protocol state)', async () => {
|
|
2295
|
+
const taskId = randomUUID(), trajectoryCapsuleHash = `sha256:${'a'.repeat(64)}`, receiptHash = `sha256:${'b'.repeat(64)}`;
|
|
2296
|
+
let eventsDown = false;
|
|
2297
|
+
const f = await sharedStateFixture('fabric-shared-restart-', call => {
|
|
2298
|
+
if (!call.path.endsWith('/events'))
|
|
2299
|
+
return sharedOk();
|
|
2300
|
+
return eventsDown ? sharedUnavailable() : sharedOk({ receipt: { hash: receiptHash } });
|
|
2301
|
+
});
|
|
2302
|
+
const a = await f.open('relay:workspace-a'), b = await f.open();
|
|
2303
|
+
await a.openSession();
|
|
2304
|
+
await b.openSession();
|
|
2305
|
+
eventsDown = true;
|
|
2306
|
+
await assert.rejects(a.postTaskEvent(taskId, 'completed', { trajectoryCapsuleHash }), error => error instanceof AgentFabricRequestError && error.status === 503);
|
|
2307
|
+
const kept = f.calls.at(-1);
|
|
2308
|
+
for (let call = 0; call < 4; call++)
|
|
2309
|
+
await b.pollTask();
|
|
2310
|
+
assert.equal(f.calls.filter(call => call.messageId === kept.messageId).length, 1, 'the other client never sends it');
|
|
2311
|
+
eventsDown = false;
|
|
2312
|
+
const restarted = await f.open('relay:workspace-a');
|
|
2313
|
+
await restarted.openSession();
|
|
2314
|
+
const sent = f.calls.filter(call => call.messageId === kept.messageId);
|
|
2315
|
+
assert.equal(sent.length, 2, 'the restarted loop sends its kept completion once more');
|
|
2316
|
+
assert.deepEqual([sent[1].idempotencyKey, sent[1].sessionId], [kept.messageId, kept.sessionId], 'with the same idempotency key, in the session it was signed in');
|
|
2317
|
+
assert.equal(f.calls.at(-1).path, '/agent-fabric/sessions');
|
|
2318
|
+
assert.notEqual(f.calls.at(-1).sessionId, kept.sessionId, 'then it opens a new session');
|
|
2319
|
+
assert.deepEqual((await f.read()).recoveredTaskCompletions?.map(item => item.taskId), [taskId]);
|
|
2320
|
+
assert.deepEqual(keptMessageIds(await f.read()), []);
|
|
2321
|
+
});
|
|
2322
|
+
// The server accepts a session's sequences only in increasing order; gaps are allowed.
|
|
2323
|
+
async function sharedMonotonicFixture(prefix) {
|
|
2324
|
+
const highest = new Map();
|
|
2325
|
+
/** Paths whose calls fail with a retryable 503 while listed, matched by their ending. */
|
|
2326
|
+
const down = new Set();
|
|
2327
|
+
/** Paths whose calls the server refuses with a definitive 409 while listed, matched by their ending. */
|
|
2328
|
+
const refuse = new Set();
|
|
2329
|
+
let unavailable = 0;
|
|
2330
|
+
const f = await sharedStateFixture(prefix, call => {
|
|
2331
|
+
if (unavailable > 0) {
|
|
2332
|
+
unavailable -= 1;
|
|
2333
|
+
return sharedUnavailable();
|
|
2334
|
+
}
|
|
2335
|
+
if ([...down].some(ending => call.path.endsWith(ending)))
|
|
2336
|
+
return sharedUnavailable();
|
|
2337
|
+
if ([...refuse].some(ending => call.path.endsWith(ending))) {
|
|
2338
|
+
return new Response(JSON.stringify({ ok: false, error: { code: 'already_acknowledged', message: 'Refused.' } }), { status: 409 });
|
|
2339
|
+
}
|
|
2340
|
+
if ((highest.get(call.sessionId) ?? 0) >= call.sequence) {
|
|
2341
|
+
return new Response(JSON.stringify({ ok: false, error: { code: 'sequence_out_of_order', message: 'Out of order.' } }), { status: 409 });
|
|
2342
|
+
}
|
|
2343
|
+
highest.set(call.sessionId, call.sequence);
|
|
2344
|
+
return sharedOk();
|
|
2345
|
+
});
|
|
2346
|
+
return { ...f, highest, down, refuse, failNext: (count = 1) => { unavailable = count; } };
|
|
2347
|
+
}
|
|
2348
|
+
/** Asserts that no sequence was signed twice with different message ids in the session. */
|
|
2349
|
+
function assertNoSequenceSignedTwice(calls, sessionId) {
|
|
2350
|
+
const signed = new Map();
|
|
2351
|
+
for (const call of calls.filter(item => item.sessionId === sessionId)) {
|
|
2352
|
+
assert.ok(!signed.has(call.sequence) || signed.get(call.sequence) === call.messageId, `sequence ${call.sequence} is signed again with another message id in the owner's session`);
|
|
2353
|
+
signed.set(call.sequence, call.messageId);
|
|
2354
|
+
}
|
|
2355
|
+
}
|
|
2356
|
+
const sharedAcknowledgement = () => ({
|
|
2357
|
+
taskId: '11111111-1111-4111-8111-111111111111', endpointId: '22222222-2222-4222-8222-222222222222',
|
|
2358
|
+
actionDigest: `sha256:${'a'.repeat(64)}`, disposition: 'executed',
|
|
2359
|
+
externalIdempotencyKeyHash: 'b'.repeat(64), resultDigest: `sha256:${'c'.repeat(64)}`,
|
|
2360
|
+
acknowledgedAt: new Date().toISOString(),
|
|
2361
|
+
});
|
|
2362
|
+
async function sharedLater(minutes, work) {
|
|
2363
|
+
const now = Date.now;
|
|
2364
|
+
Date.now = () => now() + minutes * 60_000;
|
|
2365
|
+
try {
|
|
2366
|
+
return await work();
|
|
2367
|
+
}
|
|
2368
|
+
finally {
|
|
2369
|
+
Date.now = now;
|
|
2370
|
+
}
|
|
2371
|
+
}
|
|
2372
|
+
/** Resolves once every signing-control section already in this process's line has finished, such as the section in
|
|
2373
|
+
* which a client sends a due orphan after its own request. It writes nothing. */
|
|
2374
|
+
const sharedSettled = (config) => withDeviceSigningControl({ config }, async () => undefined);
|
|
2375
|
+
test('a stale action acknowledgement is re-signed only by its owner, in its own session (shared protocol state)', async () => {
|
|
2376
|
+
const f = await sharedMonotonicFixture('fabric-shared-resign-');
|
|
2377
|
+
const a = await f.open(), b = await f.open();
|
|
2378
|
+
await a.openSession();
|
|
2379
|
+
await b.openSession();
|
|
2380
|
+
const sessionB = f.calls[1].sessionId;
|
|
2381
|
+
f.failNext();
|
|
2382
|
+
await assert.rejects(b.postActionEnforcement('33333333-3333-4333-8333-333333333333', sharedAcknowledgement()), error => error instanceof AgentFabricRequestError && error.status === 503);
|
|
2383
|
+
const kept = f.calls.at(-1);
|
|
2384
|
+
await sharedLater(9, async () => { await a.pollTask(); await a.pollTask(); });
|
|
2385
|
+
assert.equal(f.calls.filter(call => call.sessionId === sessionB).length, 2, 'the other client never re-signs or sends it');
|
|
2386
|
+
await sharedLater(9, () => b.pollTask());
|
|
2387
|
+
const [resigned, next] = f.calls.slice(-2);
|
|
2388
|
+
assert.deepEqual([resigned.sessionId, resigned.path], [sessionB, kept.path], 'its owner re-signs it in its own session');
|
|
2389
|
+
assert.notEqual(resigned.messageId, kept.messageId);
|
|
2390
|
+
assert.ok(resigned.sequence > kept.sequence);
|
|
2391
|
+
assert.deepEqual([next.sessionId, next.path], [sessionB, '/agent-fabric/tasks/poll']);
|
|
2392
|
+
assert.ok(next.sequence > resigned.sequence, 'the owner signs its next request above the re-signed one');
|
|
2393
|
+
});
|
|
2394
|
+
test('a running owner never signs a sequence again after another client took its stale acknowledgement and wrote no answer (shared protocol state)', async () => {
|
|
2395
|
+
const f = await sharedMonotonicFixture('fabric-shared-resign-adopted-');
|
|
2396
|
+
const a = await f.open(), b = await f.open();
|
|
2397
|
+
await a.openSession();
|
|
2398
|
+
await b.openSession();
|
|
2399
|
+
const acknowledgement = sharedAcknowledgement();
|
|
2400
|
+
f.failNext();
|
|
2401
|
+
await assert.rejects(a.postActionEnforcement('33333333-3333-4333-8333-333333333333', acknowledgement), error => error instanceof AgentFabricRequestError && error.status === 503);
|
|
2402
|
+
const kept = f.calls.at(-1);
|
|
2403
|
+
// The entry stood unchanged long enough, but its owner still runs: another client's session open leaves the stale
|
|
2404
|
+
// acknowledgement in the entry, since it cannot be sent as signed and only its owner signs in the owner's session.
|
|
2405
|
+
await sharedLater(11, async () => { const c = await f.open(); await c.openSession(); });
|
|
2406
|
+
assert.equal(f.calls.filter(call => call.path === kept.path).length, 1, 'the other client signs nothing in the owner\'s session');
|
|
2407
|
+
// Later, another client writes; no answer is in the file.
|
|
2408
|
+
await sharedLater(22, () => b.pollTask());
|
|
2409
|
+
assert.ok(!Object.values((await f.read()).outboxes ?? {}).some(entry => entry.adoptedOutcome));
|
|
2410
|
+
const retry = await sharedLater(22, () => a.postActionEnforcement('33333333-3333-4333-8333-333333333333', acknowledgement))
|
|
2411
|
+
.then(() => 'accepted', (error) => error instanceof AgentFabricRequestError ? error.status : error);
|
|
2412
|
+
const signed = new Map();
|
|
2413
|
+
for (const call of f.calls.filter(item => item.sessionId === kept.sessionId)) {
|
|
2414
|
+
assert.ok(!signed.has(call.sequence) || signed.get(call.sequence) === call.messageId, `sequence ${call.sequence} is signed again with another message id in the owner's session`);
|
|
2415
|
+
signed.set(call.sequence, call.messageId);
|
|
2416
|
+
}
|
|
2417
|
+
assert.equal(retry, 'accepted', 'the owner gets no out-of-order refusal');
|
|
2418
|
+
});
|
|
2419
|
+
// Owner review round 1, as simplified: no idle cap evicts an entry whose answer another client kept, and an owner whose
|
|
2420
|
+
// entry did leave the file keeps its session.
|
|
2421
|
+
test('an owner whose entry another client answered keeps its session when newer idle entries fill the file (shared protocol state)', async () => {
|
|
2422
|
+
let down = false;
|
|
2423
|
+
const f = await sharedStateFixture('fabric-shared-idle-eviction-', call => down && call.path === '/agent-fabric/workspaces' ? sharedUnavailable() : sharedOk({ workspace: { id: 'registered' } }));
|
|
2424
|
+
const a = await f.open(), b = await f.open();
|
|
2425
|
+
await a.openSession();
|
|
2426
|
+
await b.openSession();
|
|
2427
|
+
down = true;
|
|
2428
|
+
await assert.rejects(a.registerWorkspace({ workspaceId: 'a' }), error => error instanceof AgentFabricRequestError && error.status === 503);
|
|
2429
|
+
const kept = f.calls.at(-1);
|
|
2430
|
+
down = false;
|
|
2431
|
+
// The owner still runs and its entry stood unchanged for 10 minutes: another client sends the request and keeps the
|
|
2432
|
+
// answer in the owner's entry.
|
|
2433
|
+
const state = await f.read();
|
|
2434
|
+
Object.values(state.outboxes ?? {}).find(entry => entry.pending?.headers['x-dharma-message-id'] === kept.messageId)
|
|
2435
|
+
.updatedAt = new Date(Date.now() - 11 * 60_000).toISOString();
|
|
2436
|
+
await writeFile(f.statePath, JSON.stringify(state));
|
|
2437
|
+
await (await f.open()).openSession();
|
|
2438
|
+
assert.equal(sentTimes(f.calls, kept.messageId), 2);
|
|
2439
|
+
// Newer idle entries of other live clients fill what an idle cap of 32 entries would keep; then another client writes.
|
|
2440
|
+
const filled = await f.read();
|
|
2441
|
+
const newer = new Date(Date.now() + 60 * 60_000).toISOString();
|
|
2442
|
+
for (let index = 0; index < 32; index++) {
|
|
2443
|
+
filled.outboxes[`cli:${process.pid}:idle:${index}`] = { sessionId: randomUUID(), nextSequence: 2, pending: null,
|
|
2444
|
+
ownerPid: process.pid, updatedAt: newer };
|
|
2445
|
+
}
|
|
2446
|
+
await writeFile(f.statePath, JSON.stringify(filled));
|
|
2447
|
+
await b.pollTask();
|
|
2448
|
+
assert.equal(await sharedOutcome(a.pollTask()), 'accepted', 'the owner\'s next call succeeds, with no closed session');
|
|
2449
|
+
const last = f.calls.at(-1);
|
|
2450
|
+
assert.deepEqual([last.path, last.sessionId], ['/agent-fabric/tasks/poll', kept.sessionId], 'in the same session');
|
|
2451
|
+
assert.ok(last.sequence > kept.sequence, 'above the sequence of its answered request');
|
|
2452
|
+
assertNoSequenceSignedTwice(f.calls, kept.sessionId);
|
|
2453
|
+
});
|
|
2454
|
+
const sharedIdleEntry = (ownerPid, minute = 0) => ({ sessionId: randomUUID(), nextSequence: 2, pending: null, ownerPid,
|
|
2455
|
+
updatedAt: new Date(Date.UTC(2026, 0, 1, 0, minute)).toISOString() });
|
|
2456
|
+
test('a protocol-state file with an invalid outbox entry is refused and kept (shared protocol state)', async () => {
|
|
2457
|
+
const f = await sharedStateFixture('fabric-shared-outbox-invalid-', () => sharedOk());
|
|
2458
|
+
const idle = sharedIdleEntry(1);
|
|
2459
|
+
for (const outboxes of [[], { nobody: idle }, { 'cli:1:1': { ...idle, nextSequence: 0 } },
|
|
2460
|
+
{ 'cli:1:1': { ...idle, ownerPid: -1 } }, { 'cli:1:1': { ...idle, updatedAt: 'later' } },
|
|
2461
|
+
{ 'cli:1:1': { ...idle, pending: { method: 'PUT', pathname: '/x', body: '', headers: {} } } },
|
|
2462
|
+
{ 'cli:1:1': { ...idle, adoptedAt: idle.updatedAt, adoptedOutcome: { messageId: 'm', status: 503, response: {} } } },
|
|
2463
|
+
Object.fromEntries(Array.from({ length: 257 }, (_, index) => [`cli:1:${index + 1}`, idle]))]) {
|
|
2464
|
+
const content = JSON.stringify({ schema: 'dharma.protocol-state/v1', sessionId: null, nextSequence: 1, pending: null, outboxes });
|
|
2465
|
+
await writeFile(f.statePath, content);
|
|
2466
|
+
await assert.rejects(f.open(), /protocol state is invalid/);
|
|
2467
|
+
assert.equal(await readFile(f.statePath, 'utf8'), content);
|
|
2468
|
+
}
|
|
2469
|
+
assert.equal(f.calls.length, 0);
|
|
2470
|
+
});
|
|
2471
|
+
for (const outcome of ['accepted', 'refused', 'retryable']) {
|
|
2472
|
+
test(`an orphan left by a command that exited is adopted once and never answers the adopter: ${outcome} (shared protocol state)`, async () => {
|
|
2473
|
+
const taskId = randomUUID(), trajectoryCapsuleHash = `sha256:${'a'.repeat(64)}`, receiptHash = `sha256:${'b'.repeat(64)}`;
|
|
2474
|
+
let mode = 'down';
|
|
2475
|
+
const f = await sharedStateFixture(`fabric-shared-orphan-${outcome}-`, call => {
|
|
2476
|
+
if (!call.path.endsWith('/events'))
|
|
2477
|
+
return sharedOk();
|
|
2478
|
+
if (mode === 'down' || mode === 'retryable')
|
|
2479
|
+
return sharedUnavailable();
|
|
2480
|
+
if (mode === 'refused') {
|
|
2481
|
+
return new Response(JSON.stringify({ ok: false, error: { code: 'lease_expired', message: 'Lease expired.' } }), { status: 409 });
|
|
2482
|
+
}
|
|
2483
|
+
return sharedOk({ receipt: { hash: receiptHash } });
|
|
2484
|
+
});
|
|
2485
|
+
const command = await f.open();
|
|
2486
|
+
await command.openSession();
|
|
2487
|
+
await assert.rejects(command.postTaskEvent(taskId, 'completed', { trajectoryCapsuleHash }), error => error instanceof AgentFabricRequestError && error.status === 503);
|
|
2488
|
+
const kept = f.calls.at(-1);
|
|
2489
|
+
// The command exits: its entry now names a process that is gone.
|
|
2490
|
+
const dead = await exitedPid();
|
|
2491
|
+
const state = await f.read();
|
|
2492
|
+
const [key, entry] = Object.entries(state.outboxes ?? {})
|
|
2493
|
+
.find(([, item]) => item.pending?.headers['x-dharma-message-id'] === kept.messageId) ?? [];
|
|
2494
|
+
assert.ok(key && entry, 'the command kept its request in its own entry');
|
|
2495
|
+
delete state.outboxes[key];
|
|
2496
|
+
state.outboxes[`cli:${dead}:1`] = { ...entry, ownerPid: dead };
|
|
2497
|
+
await writeFile(f.statePath, JSON.stringify(state));
|
|
2498
|
+
mode = outcome;
|
|
2499
|
+
const adopter = await f.open();
|
|
2500
|
+
assert.equal((await adopter.openSession()).ok, true, 'the adopter opens its session whatever the orphan\'s outcome');
|
|
2501
|
+
const later = await f.open();
|
|
2502
|
+
assert.equal((await later.openSession()).ok, true);
|
|
2503
|
+
let sent = f.calls.filter(call => call.messageId === kept.messageId);
|
|
2504
|
+
assert.equal(sent.length, 2, 'the orphan is sent once by the next session open, not again by the one after it');
|
|
2505
|
+
assert.deepEqual([sent[1].idempotencyKey, sent[1].sessionId], [kept.messageId, kept.sessionId]);
|
|
2506
|
+
if (outcome === 'retryable') {
|
|
2507
|
+
assert.deepEqual(keptMessageIds(await f.read()), [kept.messageId], 'an orphan without a definite answer is kept');
|
|
2508
|
+
mode = 'accepted';
|
|
2509
|
+
const next = await f.open();
|
|
2510
|
+
assert.equal((await sharedLater(11, () => next.openSession())).ok, true);
|
|
2511
|
+
sent = f.calls.filter(call => call.messageId === kept.messageId);
|
|
2512
|
+
assert.equal(sent.length, 2, 'once due again it could be sent only signed again, and its owner is gone: it is settled unsent');
|
|
2513
|
+
}
|
|
2514
|
+
assert.deepEqual(keptMessageIds(await f.read()), [], 'an answered or settled orphan leaves the file');
|
|
2515
|
+
assert.deepEqual((await f.read()).recoveredTaskCompletions?.map(item => item.taskId), outcome === 'accepted' ? [taskId] : []);
|
|
2516
|
+
});
|
|
2517
|
+
}
|
|
2518
|
+
test('a kept request in an earlier single-slot file that only its signer could send again is settled unsent by the next session open, which it never fails (shared protocol state)', async () => {
|
|
2519
|
+
const legacy = randomUUID(), legacySession = randomUUID();
|
|
2520
|
+
const f = await sharedStateFixture('fabric-shared-migrate-', call => call.messageId === legacy
|
|
2521
|
+
? new Response(JSON.stringify({ ok: false, error: { code: 'sequence_out_of_order', message: 'Out of order.' } }), { status: 409 })
|
|
2522
|
+
: sharedOk());
|
|
2523
|
+
await writeFile(f.statePath, JSON.stringify({ schema: 'dharma.protocol-state/v1', sessionId: legacySession, nextSequence: 5,
|
|
2524
|
+
pending: { method: 'POST', pathname: '/api/v1/orgs/org_a/agent-fabric/workspaces', body: '{"workspaceId":"legacy"}', headers: {
|
|
2525
|
+
'content-type': 'application/json', 'idempotency-key': legacy, 'x-dharma-device-id': f.config.deviceId,
|
|
2526
|
+
'x-dharma-session-id': legacySession, 'x-dharma-message-id': legacy, 'x-dharma-timestamp': '2026-01-01T00:00:00.000Z',
|
|
2527
|
+
'x-dharma-nonce': 'legacy-nonce', 'x-dharma-sequence': '4', 'x-dharma-signature': 'legacy-signature'
|
|
2528
|
+
} } }));
|
|
2529
|
+
// Signed long ago in the earlier version's session: its signature is older than the server accepts, and no process of
|
|
2530
|
+
// this version opened that session, so none signs it again.
|
|
2531
|
+
const a = await f.open();
|
|
2532
|
+
assert.equal((await a.openSession()).ok, true, 'the migrated request never fails the session open');
|
|
2533
|
+
const b = await f.open();
|
|
2534
|
+
await b.openSession();
|
|
2535
|
+
assert.equal(f.calls.filter(call => call.messageId === legacy).length, 0, 'the migrated request is neither sent nor signed again');
|
|
2536
|
+
const state = await f.read();
|
|
2537
|
+
assert.equal(state.pending, null);
|
|
2538
|
+
assert.deepEqual(keptMessageIds(state), []);
|
|
2539
|
+
});
|
|
2540
|
+
test('entries without a kept request, kept completions or an answer of the last 10 minutes are pruned, whatever their number or owner (shared protocol state)', async () => {
|
|
2541
|
+
const f = await sharedStateFixture('fabric-shared-prune-', () => sharedOk());
|
|
2542
|
+
const dead = await exitedPid();
|
|
2543
|
+
const outboxes = {};
|
|
2544
|
+
for (let index = 1; index <= 3; index++)
|
|
2545
|
+
outboxes[`cli:${dead}:${index}`] = sharedIdleEntry(dead, index);
|
|
2546
|
+
for (let index = 1; index <= 40; index++)
|
|
2547
|
+
outboxes[`cli:${process.pid}:${1000 + index}`] = sharedIdleEntry(process.pid, 10 + index);
|
|
2548
|
+
outboxes['relay:old-answer'] = { ...sharedIdleEntry(process.pid), adoptedAt: new Date(Date.now() - 11 * 60_000).toISOString(),
|
|
2549
|
+
adoptedOutcome: { messageId: randomUUID(), status: 201, response: { ok: true } } };
|
|
2550
|
+
outboxes[`cli:${process.pid}:kept`] = sharedOrphan(f.config.deviceId, process.pid);
|
|
2551
|
+
await writeSharedOutboxes(f.statePath, outboxes);
|
|
2552
|
+
const client = await f.open('relay:workspace-prune');
|
|
2553
|
+
await client.openSession();
|
|
2554
|
+
assert.deepEqual(Object.keys((await f.read()).outboxes ?? {}).sort(), [`cli:${process.pid}:kept`, 'relay:workspace-prune'], 'only the entry with a kept request and the writer\'s own entry stay');
|
|
2555
|
+
});
|
|
2556
|
+
test('protocol state is not written while another process holds device signing control (shared protocol state)', async () => {
|
|
2557
|
+
const taskId = randomUUID(), receiptHash = `sha256:${'b'.repeat(64)}`;
|
|
2558
|
+
const f = await sharedStateFixture('fabric-shared-cross-process-', () => sharedOk());
|
|
2559
|
+
await writeFile(f.statePath, JSON.stringify({ schema: 'dharma.protocol-state/v1', sessionId: null, nextSequence: 1, pending: null,
|
|
2560
|
+
recoveredTaskCompletions: [{ taskId, trajectoryCapsuleHash: `sha256:${'a'.repeat(64)}`, receiptHash,
|
|
2561
|
+
recoveredAt: new Date().toISOString() }] }));
|
|
2562
|
+
const client = await f.open();
|
|
2563
|
+
const before = await readFile(f.statePath, 'utf8');
|
|
2564
|
+
const other = sharedChildHolder(f.config);
|
|
2565
|
+
try {
|
|
2566
|
+
await other.saw('held');
|
|
2567
|
+
const opening = client.openSession();
|
|
2568
|
+
const acknowledging = client.acknowledgeRecoveredTaskCompletion(taskId, receiptHash);
|
|
2569
|
+
await sharedDelay(500);
|
|
2570
|
+
assert.equal(await readFile(f.statePath, 'utf8'), before, 'nothing is written while the other process holds control');
|
|
2571
|
+
assert.equal(f.calls.length, 0);
|
|
2572
|
+
other.child.stdin.end('go');
|
|
2573
|
+
await other.saw('released');
|
|
2574
|
+
await opening;
|
|
2575
|
+
await acknowledging;
|
|
2576
|
+
const after = await f.read();
|
|
2577
|
+
assert.deepEqual(after.recoveredTaskCompletions, []);
|
|
2578
|
+
assert.ok(Object.values(after.outboxes ?? {}).some(entry => entry.sessionId), 'the opened session is in the client\'s entry');
|
|
2579
|
+
assert.equal(await other.exited, 0);
|
|
2580
|
+
}
|
|
2581
|
+
finally {
|
|
2582
|
+
other.child.kill();
|
|
2583
|
+
}
|
|
2584
|
+
});
|
|
2585
|
+
test('opening a session replays this client\'s own kept request inside its signing-control section (shared protocol state)', async () => {
|
|
2586
|
+
let unavailable = 0;
|
|
2587
|
+
const f = await sharedStateFixture('fabric-shared-reentrant-', () => {
|
|
2588
|
+
if (unavailable > 0) {
|
|
2589
|
+
unavailable -= 1;
|
|
2590
|
+
return sharedUnavailable();
|
|
2591
|
+
}
|
|
2592
|
+
return sharedOk();
|
|
2593
|
+
});
|
|
2594
|
+
const client = await f.open();
|
|
2595
|
+
await client.openSession();
|
|
2596
|
+
unavailable = 1;
|
|
2597
|
+
await assert.rejects(client.registerWorkspace({ workspaceId: 'w' }), error => error instanceof AgentFabricRequestError && error.status === 503);
|
|
2598
|
+
const kept = f.calls.at(-1);
|
|
2599
|
+
await client.openSession();
|
|
2600
|
+
assert.deepEqual(f.calls.slice(-2).map(call => call.path), ['/agent-fabric/workspaces', '/agent-fabric/sessions']);
|
|
2601
|
+
assert.equal(f.calls.at(-2).messageId, kept.messageId);
|
|
2602
|
+
assert.equal((await f.read()).pending, null);
|
|
2603
|
+
assert.deepEqual(keptMessageIds(await f.read()), []);
|
|
2604
|
+
});
|
|
2605
|
+
test('a session open queued behind a holder leaves the line when its relay loop stops and writes nothing (shared protocol state)', async () => {
|
|
2606
|
+
const f = await sharedStateFixture('fabric-shared-stop-', () => sharedOk());
|
|
2607
|
+
const client = await f.open();
|
|
2608
|
+
const held = sharedGate(), released = sharedGate(), loop = new AbortController();
|
|
2609
|
+
const holder = withDeviceSigningControl({ config: f.config }, async () => { held.open(); await released.promise; });
|
|
2610
|
+
await held.promise;
|
|
2611
|
+
const stopped = withDeviceSigningLineSignal(loop.signal, () => client.openSession())
|
|
2612
|
+
.then(() => 'opened', (error) => error.name);
|
|
2613
|
+
await sharedDelay(50);
|
|
2614
|
+
loop.abort();
|
|
2615
|
+
assert.equal(await stopped, 'AbortError');
|
|
2616
|
+
released.open();
|
|
2617
|
+
await holder;
|
|
2618
|
+
await assert.rejects(stat(f.statePath), { code: 'ENOENT' }, 'a stopped session open writes no protocol state');
|
|
2619
|
+
assert.equal(f.calls.length, 0);
|
|
2620
|
+
});
|
|
2621
|
+
test('the README says every protocol-state write takes device signing control (shared protocol state)', async () => {
|
|
2622
|
+
const readme = (await readFile(new URL('../README.md', import.meta.url), 'utf8')).replace(/\s+/g, ' ');
|
|
2623
|
+
assert.equal(/without taking signing control/.test(readme), false, 'no protocol-state write skips signing control');
|
|
2624
|
+
assert.equal(/protocol state [^.]*under the device signing control/i.test(readme), true, 'the README says protocol-state writes take device signing control');
|
|
2625
|
+
});
|
|
2626
|
+
test('a protocol-state file corrupted after open is preserved and stops the next write (shared protocol state)', async () => {
|
|
2627
|
+
const f = await sharedStateFixture('fabric-shared-corrupt-', () => sharedOk());
|
|
2628
|
+
const client = await f.open();
|
|
2629
|
+
await client.openSession();
|
|
2630
|
+
await writeFile(f.statePath, '{"schema":"dharma.protocol-state/v1","nextSequence":0}');
|
|
2631
|
+
const corrupt = await readFile(f.statePath, 'utf8');
|
|
2632
|
+
const sent = f.calls.length;
|
|
2633
|
+
await assert.rejects(client.registerWorkspace({ workspaceId: 'w' }), /protocol state is invalid/);
|
|
2634
|
+
await assert.rejects(client.openSession(), /protocol state is invalid/);
|
|
2635
|
+
assert.equal(await readFile(f.statePath, 'utf8'), corrupt, 'the corrupt file is kept for recovery');
|
|
2636
|
+
assert.equal(f.calls.length, sent, 'nothing is signed or sent from a state the client cannot read');
|
|
2637
|
+
});
|
|
2638
|
+
test('an acknowledgement never waits on this client\'s queue behind a request that already holds signing control (shared protocol state)', async () => {
|
|
2639
|
+
const taskId = randomUUID(), receiptHash = `sha256:${'b'.repeat(64)}`;
|
|
2640
|
+
const f = await sharedStateFixture('fabric-shared-ack-line-', () => sharedOk());
|
|
2641
|
+
await writeFile(f.statePath, JSON.stringify({ schema: 'dharma.protocol-state/v1', sessionId: null, nextSequence: 1, pending: null,
|
|
2642
|
+
recoveredTaskCompletions: [{ taskId, trajectoryCapsuleHash: `sha256:${'a'.repeat(64)}`, receiptHash,
|
|
2643
|
+
recoveredAt: new Date().toISOString() }] }));
|
|
2644
|
+
const client = await f.open();
|
|
2645
|
+
await client.openSession();
|
|
2646
|
+
const held = sharedGate(), released = sharedGate();
|
|
2647
|
+
const sibling = withDeviceSigningControl({ config: f.config }, async () => { held.open(); await released.promise; });
|
|
2648
|
+
await held.promise;
|
|
2649
|
+
// An admitted request takes signing control first and then enters the client's queue, as the CLI does.
|
|
2650
|
+
const heartbeat = withDeviceSigningControl({ config: f.config }, () => client.pollTask()).then(() => 'heartbeat');
|
|
2651
|
+
await sharedDelay(50);
|
|
2652
|
+
const acknowledging = client.acknowledgeRecoveredTaskCompletion(taskId, receiptHash).then(() => 'acknowledged');
|
|
2653
|
+
await sharedDelay(50);
|
|
2654
|
+
released.open();
|
|
2655
|
+
await sibling;
|
|
2656
|
+
let timer;
|
|
2657
|
+
const stalled = new Promise(accept => { timer = setTimeout(() => accept('stalled'), 5_000); });
|
|
2658
|
+
try {
|
|
2659
|
+
assert.equal(await Promise.race([Promise.all([heartbeat, acknowledging]).then(done => done.join(',')), stalled]), 'heartbeat,acknowledged', 'both settle once the sibling releases signing control');
|
|
2660
|
+
}
|
|
2661
|
+
finally {
|
|
2662
|
+
clearTimeout(timer);
|
|
2663
|
+
}
|
|
2664
|
+
assert.deepEqual((await f.read()).recoveredTaskCompletions, []);
|
|
2665
|
+
});
|
|
2666
|
+
test('the README limits the shared protocol-state guarantee to one client version on the device (shared protocol state)', async () => {
|
|
2667
|
+
const readme = (await readFile(new URL('../README.md', import.meta.url), 'utf8')).replace(/\s+/g, ' ');
|
|
2668
|
+
assert.equal(/clients on one device never overwrite each other's kept requests or completions\./.test(readme), false, 'the guarantee is not stated without its version condition');
|
|
2669
|
+
assert.equal(/every process on the device runs this version/i.test(readme), true, 'the README says the guarantee needs every process on the device to run this version');
|
|
2670
|
+
assert.equal(/upgrade stops the relay before/i.test(readme), true, 'the README says the relay upgrade stops the relay first');
|
|
2671
|
+
});
|
|
2672
|
+
test('the README describes one outbox entry per client and no draining of another client\'s request (shared protocol state)', async () => {
|
|
2673
|
+
const readme = (await readFile(new URL('../README.md', import.meta.url), 'utf8')).replace(/\s+/g, ' ');
|
|
2674
|
+
assert.equal(/relay_outbox_busy|one kept request for the whole device|set aside/i.test(readme), false, 'no shared outbox slot, busy outbox or set-aside is described');
|
|
2675
|
+
assert.equal(/each client keeps its own outbox entry/i.test(readme), true);
|
|
2676
|
+
assert.equal(/owner process has exited/i.test(readme), true, 'the README says when another client sends a kept request');
|
|
2677
|
+
});
|
|
2678
|
+
// Orphaned kept requests: when another client sends them, how many at once, and what a full file still allows.
|
|
2679
|
+
const sharedPending = (deviceId, route) => {
|
|
2680
|
+
const id = randomUUID();
|
|
2681
|
+
return { method: 'POST', pathname: `/api/v1/orgs/org_a${route}`, body: '{"workspaceId":"orphan"}', headers: {
|
|
2682
|
+
'content-type': 'application/json', 'idempotency-key': id, 'x-dharma-device-id': deviceId,
|
|
2683
|
+
'x-dharma-session-id': randomUUID(), 'x-dharma-message-id': id, 'x-dharma-timestamp': new Date().toISOString(),
|
|
2684
|
+
'x-dharma-nonce': 'orphan-nonce', 'x-dharma-sequence': '1', 'x-dharma-signature': 'orphan-signature'
|
|
2685
|
+
} };
|
|
2686
|
+
};
|
|
2687
|
+
const sharedOrphan = (deviceId, ownerPid, input = {}) => ({
|
|
2688
|
+
sessionId: randomUUID(), nextSequence: 2, pending: sharedPending(deviceId, input.route ?? '/agent-fabric/workspaces'), ownerPid,
|
|
2689
|
+
updatedAt: new Date(Date.now() - (input.minutesAgo ?? 0) * 60_000).toISOString(),
|
|
2690
|
+
...(input.adoptedMinutesAgo === undefined ? {} : { adoptedAt: new Date(Date.now() - input.adoptedMinutesAgo * 60_000).toISOString() }),
|
|
2691
|
+
});
|
|
2692
|
+
const orphanMessageId = (entry) => entry.pending.headers['x-dharma-message-id'];
|
|
2693
|
+
const writeSharedOutboxes = (statePath, outboxes) => writeFile(statePath, JSON.stringify({
|
|
2694
|
+
schema: 'dharma.protocol-state/v1', sessionId: null, nextSequence: 1, pending: null, outboxes
|
|
2695
|
+
}));
|
|
2696
|
+
const sentTimes = (calls, messageId) => calls.filter(call => call.messageId === messageId).length;
|
|
2697
|
+
test('a long-lived client does not send an orphan again within 10 minutes, and after its own request settles one it could then send only signed again (shared protocol state)', async () => {
|
|
2698
|
+
let down = true;
|
|
2699
|
+
const f = await sharedStateFixture('fabric-shared-orphan-poll-', call => down && call.path === '/agent-fabric/workspaces' ? sharedUnavailable() : sharedOk());
|
|
2700
|
+
const orphan = sharedOrphan(f.config.deviceId, await exitedPid());
|
|
2701
|
+
await writeSharedOutboxes(f.statePath, { 'cli:1:orphan:1': orphan });
|
|
2702
|
+
const loop = await f.open('relay:workspace-loop');
|
|
2703
|
+
await loop.openSession();
|
|
2704
|
+
assert.equal(sentTimes(f.calls, orphanMessageId(orphan)), 1, 'the loop\'s session open sends it during the outage');
|
|
2705
|
+
down = false;
|
|
2706
|
+
for (let poll = 0; poll < 20; poll++)
|
|
2707
|
+
await loop.pollTask();
|
|
2708
|
+
assert.equal(sentTimes(f.calls, orphanMessageId(orphan)), 1, 'not again within 10 minutes');
|
|
2709
|
+
assert.deepEqual(keptMessageIds(await f.read()), [orphanMessageId(orphan)]);
|
|
2710
|
+
// Due again, its signature is older than the server accepts and its owner is gone: nobody may sign it again.
|
|
2711
|
+
await sharedLater(11, async () => { await loop.pollTask(); await sharedSettled(f.config); });
|
|
2712
|
+
assert.equal(sentTimes(f.calls, orphanMessageId(orphan)), 1, 'a later request of the same client settles it unsent');
|
|
2713
|
+
assert.deepEqual(keptMessageIds(await f.read()), []);
|
|
2714
|
+
await sharedLater(22, () => loop.pollTask());
|
|
2715
|
+
assert.equal(sentTimes(f.calls, orphanMessageId(orphan)), 1);
|
|
2716
|
+
assert.deepEqual(keptMessageIds(await f.read()), []);
|
|
2717
|
+
});
|
|
2718
|
+
test('a kept request whose entry stood unchanged for 10 minutes is left for its owner, since its signature is then too old to send as signed (shared protocol state)', async () => {
|
|
2719
|
+
const f = await sharedStateFixture('fabric-shared-orphan-stale-', () => sharedOk());
|
|
2720
|
+
const unrelated = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { stdio: 'ignore' });
|
|
2721
|
+
try {
|
|
2722
|
+
const reused = sharedOrphan(f.config.deviceId, unrelated.pid);
|
|
2723
|
+
const removed = sharedOrphan(f.config.deviceId, await exitedPid());
|
|
2724
|
+
await writeSharedOutboxes(f.statePath, { [`cli:${unrelated.pid}:earlier:1`]: reused, 'relay:removed-workspace': removed });
|
|
2725
|
+
const a = await f.open();
|
|
2726
|
+
await a.openSession();
|
|
2727
|
+
await a.pollTask();
|
|
2728
|
+
assert.deepEqual([sentTimes(f.calls, orphanMessageId(reused)), sentTimes(f.calls, orphanMessageId(removed))], [0, 0], 'a fresh entry is left to its owner, and a workspace loop that restarts promptly replays its own');
|
|
2729
|
+
const b = await f.open('relay:workspace-b');
|
|
2730
|
+
// 10 minutes later both are due, but each was signed then too: only its owner may sign it again, and either owner
|
|
2731
|
+
// can still come back (the pid answers, and a workspace loop restarts), so both stay as they are.
|
|
2732
|
+
await sharedLater(11, async () => { await b.openSession(); await b.pollTask(); await sharedSettled(f.config); });
|
|
2733
|
+
assert.deepEqual([sentTimes(f.calls, orphanMessageId(reused)), sentTimes(f.calls, orphanMessageId(removed))], [0, 0]);
|
|
2734
|
+
const outboxes = (await f.read()).outboxes ?? {};
|
|
2735
|
+
assert.deepEqual([outboxes[`cli:${unrelated.pid}:earlier:1`]?.pending, outboxes['relay:removed-workspace']?.pending], [reused.pending, removed.pending], 'each keeps its request exactly as signed');
|
|
2736
|
+
}
|
|
2737
|
+
finally {
|
|
2738
|
+
unrelated.kill();
|
|
2739
|
+
}
|
|
2740
|
+
});
|
|
2741
|
+
test('a new client never takes over an entry an earlier process with the same pid left (shared protocol state)', async () => {
|
|
2742
|
+
const f = await sharedStateFixture('fabric-shared-pid-reuse-', call => call.path === '/agent-fabric/workspaces'
|
|
2743
|
+
? new Response(JSON.stringify({ ok: false, error: { code: 'lease_expired', message: 'Refused.' } }), { status: 409 })
|
|
2744
|
+
: sharedOk());
|
|
2745
|
+
const probe = await f.open();
|
|
2746
|
+
await probe.openSession();
|
|
2747
|
+
const [probeKey] = Object.keys((await f.read()).outboxes ?? {});
|
|
2748
|
+
const counter = Number(probeKey.split(':').at(-1));
|
|
2749
|
+
// The earlier command ran with this pid and numbered its clients from 1, as this process does.
|
|
2750
|
+
const earlier = {};
|
|
2751
|
+
for (let n = counter + 1; n <= counter + 20; n++)
|
|
2752
|
+
earlier[`cli:${process.pid}:${n}`] = sharedOrphan(f.config.deviceId, process.pid);
|
|
2753
|
+
await writeSharedOutboxes(f.statePath, earlier);
|
|
2754
|
+
for (let client = 0; client < 3; client++) {
|
|
2755
|
+
const next = await f.open();
|
|
2756
|
+
assert.equal((await next.openSession()).ok, true, 'another process\'s kept request never fails this session open');
|
|
2757
|
+
}
|
|
2758
|
+
assert.equal(f.calls.filter(call => call.path === '/agent-fabric/workspaces').length, 0, 'nor is it sent as this client\'s own');
|
|
2759
|
+
assert.equal(keptMessageIds(await f.read()).length, 20);
|
|
2760
|
+
});
|
|
2761
|
+
test('one session open sends at most one orphan, none after one without an answer, and drops orphaned session opens (shared protocol state)', async () => {
|
|
2762
|
+
let down = true;
|
|
2763
|
+
const f = await sharedStateFixture('fabric-shared-orphan-batch-', call => down && call.path === '/agent-fabric/workspaces' ? sharedUnavailable() : sharedOk());
|
|
2764
|
+
const dead = await exitedPid();
|
|
2765
|
+
const kept = Array.from({ length: 10 }, () => sharedOrphan(f.config.deviceId, dead));
|
|
2766
|
+
const sessionOpens = Array.from({ length: 3 }, () => sharedOrphan(f.config.deviceId, dead, { route: '/agent-fabric/sessions' }));
|
|
2767
|
+
await writeSharedOutboxes(f.statePath, Object.fromEntries([...sessionOpens, ...kept].map((entry, index) => [`cli:${dead}:x:${index}`, entry])));
|
|
2768
|
+
const orphanSends = () => f.calls.filter(call => [...kept, ...sessionOpens].some(entry => orphanMessageId(entry) === call.messageId)).length;
|
|
2769
|
+
const a = await f.open();
|
|
2770
|
+
await a.openSession();
|
|
2771
|
+
assert.equal(orphanSends(), 1, 'the first orphan without a definite answer ends the batch');
|
|
2772
|
+
assert.equal(sessionOpens.some(entry => sentTimes(f.calls, orphanMessageId(entry)) > 0), false, 'an exited command\'s session open is never sent');
|
|
2773
|
+
assert.deepEqual(keptMessageIds(await f.read()).sort(), kept.map(orphanMessageId).sort(), 'the session opens left the file and the other orphans stay');
|
|
2774
|
+
const unanswered = f.calls.find(call => kept.some(entry => orphanMessageId(entry) === call.messageId)).messageId;
|
|
2775
|
+
down = false;
|
|
2776
|
+
const b = await f.open('relay:workspace-b');
|
|
2777
|
+
await b.openSession();
|
|
2778
|
+
assert.equal(orphanSends(), 2, 'at most one orphan per session open');
|
|
2779
|
+
for (let poll = 1; poll <= 8; poll++) {
|
|
2780
|
+
await b.pollTask();
|
|
2781
|
+
await sharedSettled(f.config);
|
|
2782
|
+
assert.equal(orphanSends(), 2 + poll, 'after each of its requests the loop sends one more while due orphans remain');
|
|
2783
|
+
}
|
|
2784
|
+
assert.equal(orphanSends(), 10, 'each orphan is sent once');
|
|
2785
|
+
assert.deepEqual(keptMessageIds(await f.read()), [unanswered], 'the one without a definite answer waits 10 minutes');
|
|
2786
|
+
// Due again, it could be sent only signed again, and its owner is gone: it is settled unsent.
|
|
2787
|
+
await sharedLater(11, async () => { await b.pollTask(); await sharedSettled(f.config); });
|
|
2788
|
+
assert.equal(orphanSends(), 10);
|
|
2789
|
+
assert.deepEqual(kept.map(entry => sentTimes(f.calls, orphanMessageId(entry))), kept.map(() => 1));
|
|
2790
|
+
assert.deepEqual(keptMessageIds(await f.read()), []);
|
|
2791
|
+
});
|
|
2792
|
+
test('a file full of kept orphans refuses only a new kept request and drains one per hold once they are due (shared protocol state)', async () => {
|
|
2793
|
+
let down = false;
|
|
2794
|
+
const orphanIds = new Set();
|
|
2795
|
+
const f = await sharedStateFixture('fabric-shared-outbox-full-', call => down && call.path === '/agent-fabric/workspaces' && !orphanIds.has(call.messageId) ? sharedUnavailable() : sharedOk());
|
|
2796
|
+
const dead = await exitedPid();
|
|
2797
|
+
const orphans = Array.from({ length: 256 }, () => sharedOrphan(f.config.deviceId, dead, { minutesAgo: 30, adoptedMinutesAgo: 1 }));
|
|
2798
|
+
for (const entry of orphans)
|
|
2799
|
+
orphanIds.add(orphanMessageId(entry));
|
|
2800
|
+
await writeSharedOutboxes(f.statePath, Object.fromEntries(orphans.map((entry, index) => [`cli:${dead}:x:${index}`, entry])));
|
|
2801
|
+
for (let client = 0; client < 3; client++) {
|
|
2802
|
+
assert.equal((await (await f.open()).openSession()).ok, true, 'a client without a kept request opens its session in a full file');
|
|
2803
|
+
}
|
|
2804
|
+
assert.equal(f.calls.filter(call => orphanIds.has(call.messageId)).length, 0, 'no orphan is due yet');
|
|
2805
|
+
const c = await f.open('relay:workspace-c');
|
|
2806
|
+
await c.openSession();
|
|
2807
|
+
down = true;
|
|
2808
|
+
await assert.rejects(c.registerWorkspace({ workspaceId: 'w' }), /relay_protocol_state_outbox_full/);
|
|
2809
|
+
await assert.rejects(c.pollTask(), /relay_protocol_state_outbox_full/, 'while no orphan is due there is no room for any request');
|
|
2810
|
+
assert.equal(f.calls.filter(call => !orphanIds.has(call.messageId) && call.path !== '/agent-fabric/sessions').length, 0, 'a refused request is not sent');
|
|
2811
|
+
down = false;
|
|
2812
|
+
// The same requests as left by commands that exited just now: due at once, and recent enough to send as signed.
|
|
2813
|
+
const due = await f.read();
|
|
2814
|
+
for (const entry of Object.values(due.outboxes ?? {}))
|
|
2815
|
+
delete entry.adoptedAt;
|
|
2816
|
+
await writeFile(f.statePath, JSON.stringify(due));
|
|
2817
|
+
await c.pollTask();
|
|
2818
|
+
await sharedSettled(f.config);
|
|
2819
|
+
assert.equal(f.calls.filter(call => orphanIds.has(call.messageId)).length, 2, 'once orphans are due, one is sent to make room before the request and one more after it');
|
|
2820
|
+
assert.equal(f.calls.filter(call => !orphanIds.has(call.messageId) && call.path === '/agent-fabric/workspaces').length, 0, 'the refused request is not kept for a later replay');
|
|
2821
|
+
for (let open = 1; open <= 5; open++) {
|
|
2822
|
+
const d = await f.open();
|
|
2823
|
+
await d.openSession();
|
|
2824
|
+
assert.equal(keptMessageIds(await f.read()).length, 254 - open, 'each later session open sends one more and makes room');
|
|
2825
|
+
}
|
|
2826
|
+
assert.deepEqual(orphans.filter(entry => sentTimes(f.calls, orphanMessageId(entry)) > 1), [], 'no orphan is sent twice');
|
|
2827
|
+
});
|
|
2828
|
+
test('a live owner whose kept request another client sent after 10 minutes never sends it again (shared protocol state)', async () => {
|
|
2829
|
+
let down = false;
|
|
2830
|
+
const f = await sharedStateFixture('fabric-shared-orphan-live-owner-', call => down && call.path === '/agent-fabric/workspaces' ? sharedUnavailable() : sharedOk());
|
|
2831
|
+
const a = await f.open(), b = await f.open();
|
|
2832
|
+
await a.openSession();
|
|
2833
|
+
await b.openSession();
|
|
2834
|
+
down = true;
|
|
2835
|
+
await assert.rejects(a.registerWorkspace({ workspaceId: 'a' }), error => error instanceof AgentFabricRequestError && error.status === 503);
|
|
2836
|
+
const kept = f.calls.at(-1);
|
|
2837
|
+
down = false;
|
|
2838
|
+
// The entry stood unchanged for 10 minutes. The answer is written in real time, so it is still in the file.
|
|
2839
|
+
const state = await f.read();
|
|
2840
|
+
Object.values(state.outboxes ?? {}).find(entry => entry.pending?.headers['x-dharma-message-id'] === kept.messageId)
|
|
2841
|
+
.updatedAt = new Date(Date.now() - 11 * 60_000).toISOString();
|
|
2842
|
+
await writeFile(f.statePath, JSON.stringify(state));
|
|
2843
|
+
await (await f.open()).openSession();
|
|
2844
|
+
assert.equal(sentTimes(f.calls, kept.messageId), 2, 'another client sends a request whose entry stood unchanged for 10 minutes');
|
|
2845
|
+
await a.pollTask();
|
|
2846
|
+
assert.equal(sentTimes(f.calls, kept.messageId), 2, 'its owner learns the outcome from the file and does not send it again');
|
|
2847
|
+
const last = f.calls.at(-1);
|
|
2848
|
+
assert.deepEqual([last.path, last.sessionId], ['/agent-fabric/tasks/poll', kept.sessionId]);
|
|
2849
|
+
assert.ok(last.sequence > kept.sequence);
|
|
2850
|
+
assert.deepEqual(keptMessageIds(await f.read()), []);
|
|
2851
|
+
});
|
|
2852
|
+
for (const outcome of ['accepted', 'refused']) {
|
|
2853
|
+
test(`a live owner's retry of a call another client sent gets that answer and is not sent again: ${outcome} (shared protocol state)`, async () => {
|
|
2854
|
+
let mode = 'ok';
|
|
2855
|
+
const f = await sharedStateFixture(`fabric-shared-orphan-live-retry-${outcome}-`, call => {
|
|
2856
|
+
if (call.path !== '/agent-fabric/workspaces' || mode === 'ok')
|
|
2857
|
+
return sharedOk();
|
|
2858
|
+
if (mode === 'down')
|
|
2859
|
+
return sharedUnavailable();
|
|
2860
|
+
return outcome === 'accepted' ? sharedOk({ workspace: { id: 'registered' } })
|
|
2861
|
+
: new Response(JSON.stringify({ ok: false, error: { code: 'workspace_conflict', message: 'Refused.' } }), { status: 409 });
|
|
2862
|
+
});
|
|
2863
|
+
const a = await f.open(), b = await f.open();
|
|
2864
|
+
await a.openSession();
|
|
2865
|
+
await b.openSession();
|
|
2866
|
+
mode = 'down';
|
|
2867
|
+
await assert.rejects(a.registerWorkspace({ workspaceId: 'a' }), error => error instanceof AgentFabricRequestError && error.status === 503);
|
|
2868
|
+
const kept = f.calls.at(-1);
|
|
2869
|
+
mode = 'answer';
|
|
2870
|
+
// The entry stood unchanged for 10 minutes. The answer is written in real time, so it is still in the file.
|
|
2871
|
+
const state = await f.read();
|
|
2872
|
+
Object.values(state.outboxes ?? {}).find(entry => entry.pending?.headers['x-dharma-message-id'] === kept.messageId)
|
|
2873
|
+
.updatedAt = new Date(Date.now() - 11 * 60_000).toISOString();
|
|
2874
|
+
await writeFile(f.statePath, JSON.stringify(state));
|
|
2875
|
+
await (await f.open()).openSession();
|
|
2876
|
+
assert.equal(sentTimes(f.calls, kept.messageId), 2, 'another client sends a request whose entry stood unchanged for 10 minutes');
|
|
2877
|
+
const retry = a.registerWorkspace({ workspaceId: 'a' });
|
|
2878
|
+
if (outcome === 'accepted') {
|
|
2879
|
+
assert.deepEqual(await retry, { ok: true, workspace: { id: 'registered' } }, 'the retry returns the answer the other client got');
|
|
2880
|
+
}
|
|
2881
|
+
else {
|
|
2882
|
+
await assert.rejects(retry, error => error instanceof AgentFabricRequestError && error.status === 409 && error.definitive
|
|
2883
|
+
&& error.serverCode === 'workspace_conflict', 'the retry raises the refusal the other client got');
|
|
2884
|
+
}
|
|
2885
|
+
assert.equal(f.calls.filter(call => call.path === '/agent-fabric/workspaces').length, 2, 'the call is not signed or sent again');
|
|
2886
|
+
mode = 'ok';
|
|
2887
|
+
await a.registerWorkspace({ workspaceId: 'a' });
|
|
2888
|
+
const next = f.calls.at(-1);
|
|
2889
|
+
assert.notEqual(next.messageId, kept.messageId, 'a later call is a new request');
|
|
2890
|
+
assert.deepEqual([next.sessionId, next.sequence > kept.sequence], [kept.sessionId, true]);
|
|
2891
|
+
});
|
|
2892
|
+
}
|
|
2893
|
+
test('an owner that read its entry before another client answered it finds the answer after its process id is gone (shared protocol state)', async () => {
|
|
2894
|
+
let down = false;
|
|
2895
|
+
const f = await sharedStateFixture('fabric-shared-orphan-restart-', call => down && call.path === '/agent-fabric/workspaces' ? sharedUnavailable() : sharedOk());
|
|
2896
|
+
const earlier = await f.open('relay:w2');
|
|
2897
|
+
await earlier.openSession();
|
|
2898
|
+
down = true;
|
|
2899
|
+
await assert.rejects(earlier.registerWorkspace({ workspaceId: 'w2' }), error => error instanceof AgentFabricRequestError && error.status === 503);
|
|
2900
|
+
const kept = f.calls.at(-1);
|
|
2901
|
+
down = false;
|
|
2902
|
+
const state = await f.read();
|
|
2903
|
+
state.outboxes['relay:w2'].ownerPid = await exitedPid();
|
|
2904
|
+
state.outboxes['relay:w2'].updatedAt = new Date(Date.now() - 11 * 60_000).toISOString();
|
|
2905
|
+
await writeFile(f.statePath, JSON.stringify(state));
|
|
2906
|
+
// A restarted relay opens both workspace loops' clients, then their sessions in line order.
|
|
2907
|
+
const w2 = await f.open('relay:w2'), w1 = await f.open('relay:w1');
|
|
2908
|
+
await w1.openSession();
|
|
2909
|
+
assert.equal(sentTimes(f.calls, kept.messageId), 2, 'the other loop sends the request that stood unchanged');
|
|
2910
|
+
assert.ok('relay:w2' in ((await f.read()).outboxes ?? {}), 'the answered entry stays in the file for its owner');
|
|
2911
|
+
await w2.openSession();
|
|
2912
|
+
assert.equal(sentTimes(f.calls, kept.messageId), 2, 'its owner takes the answer from the file and does not send it again');
|
|
2913
|
+
assert.deepEqual(keptMessageIds(await f.read()), []);
|
|
2914
|
+
});
|
|
2915
|
+
for (const [label, answer] of [['null', 'null'], ['an array', '[]'], ['a string', '"ok"']]) {
|
|
2916
|
+
test(`an orphan answered with ${label} instead of an object leaves a file every client still opens (shared protocol state)`, async () => {
|
|
2917
|
+
let mode = 'ok';
|
|
2918
|
+
const f = await sharedStateFixture('fabric-shared-orphan-bare-', call => {
|
|
2919
|
+
if (call.path !== '/agent-fabric/workspaces' || mode === 'ok')
|
|
2920
|
+
return sharedOk();
|
|
2921
|
+
return mode === 'down' ? sharedUnavailable() : new Response(answer, { status: 200 });
|
|
2922
|
+
});
|
|
2923
|
+
const earlier = await f.open('relay:w2');
|
|
2924
|
+
await earlier.openSession();
|
|
2925
|
+
mode = 'down';
|
|
2926
|
+
await assert.rejects(earlier.registerWorkspace({ workspaceId: 'w2' }), error => error instanceof AgentFabricRequestError && error.status === 503);
|
|
2927
|
+
const kept = f.calls.at(-1);
|
|
2928
|
+
const state = await f.read();
|
|
2929
|
+
state.outboxes['relay:w2'].ownerPid = await exitedPid();
|
|
2930
|
+
state.outboxes['relay:w2'].updatedAt = new Date(Date.now() - 11 * 60_000).toISOString();
|
|
2931
|
+
await writeFile(f.statePath, JSON.stringify(state));
|
|
2932
|
+
mode = 'bare';
|
|
2933
|
+
const w2 = await f.open('relay:w2'), w1 = await f.open('relay:w1');
|
|
2934
|
+
assert.equal((await w1.openSession()).ok, true, 'the adopter opens its own session after it sends the orphan');
|
|
2935
|
+
assert.equal(sentTimes(f.calls, kept.messageId), 2);
|
|
2936
|
+
assert.equal((await f.read()).outboxes?.['relay:w2']?.adoptedOutcome, undefined, 'an answer that is not an object is not kept');
|
|
2937
|
+
const later = await f.open();
|
|
2938
|
+
assert.equal((await later.openSession()).ok, true, 'a later client still opens the file');
|
|
2939
|
+
await w2.openSession();
|
|
2940
|
+
assert.equal(sentTimes(f.calls, kept.messageId), 3, 'without a kept answer its owner sends its own signed request again, with the same message id, never a new signature');
|
|
2941
|
+
mode = 'ok';
|
|
2942
|
+
await w2.registerWorkspace({ workspaceId: 'w2' });
|
|
2943
|
+
const next = f.calls.at(-1);
|
|
2944
|
+
assert.deepEqual([next.path, next.messageId === kept.messageId], ['/agent-fabric/workspaces', false], 'a later call is a new request');
|
|
2945
|
+
assert.equal(sentTimes(f.calls, kept.messageId), 3);
|
|
2946
|
+
});
|
|
2947
|
+
}
|
|
2948
|
+
function jsonFileStore(path) {
|
|
2949
|
+
const load = async () => JSON.parse(await readFile(path, 'utf8').catch(() => '{}'));
|
|
2950
|
+
return {
|
|
2951
|
+
backend: 'linux-secret-service',
|
|
2952
|
+
async get(account) { return (await load())[account] ?? null; },
|
|
2953
|
+
async put(account, value) { const values = await load(); values[account] = value; await writeFile(path, JSON.stringify(values)); },
|
|
2954
|
+
async delete(account) { const values = await load(); delete values[account]; await writeFile(path, JSON.stringify(values)); },
|
|
2955
|
+
};
|
|
2956
|
+
}
|
|
2957
|
+
test('a client that stops while it sends an orphan leaves the attempt in the file, so the next client waits 10 minutes (shared protocol state)', async () => {
|
|
2958
|
+
const storePath = resolve(await mkdtemp(resolve(tmpdir(), 'fabric-shared-orphan-cut-store-')), 'store.json');
|
|
2959
|
+
const f = await sharedStateFixture('fabric-shared-orphan-cut-', () => sharedOk(), jsonFileStore(storePath));
|
|
2960
|
+
const orphan = sharedOrphan(f.config.deviceId, await exitedPid());
|
|
2961
|
+
await writeSharedOutboxes(f.statePath, { 'cli:1:orphan:1': orphan });
|
|
2962
|
+
// A short command, such as a prompt hook, sends the orphan from its session open and is stopped during that send.
|
|
2963
|
+
const module = new URL('./index.js', import.meta.url).href;
|
|
2964
|
+
const child = spawn(process.execPath, ['--input-type=module', '-e', `import {AgentFabricClient} from ${JSON.stringify(module)};
|
|
2965
|
+
import {readFile,writeFile} from 'node:fs/promises';
|
|
2966
|
+
const [configPath,statePath,storePath,orphanId]=process.argv.slice(1);
|
|
2967
|
+
const load=async()=>JSON.parse(await readFile(storePath,'utf8').catch(()=>'{}'));
|
|
2968
|
+
const store={backend:'linux-secret-service',get:async a=>(await load())[a]??null,
|
|
2969
|
+
put:async(a,v)=>{const s=await load();s[a]=v;await writeFile(storePath,JSON.stringify(s));},
|
|
2970
|
+
delete:async a=>{const s=await load();delete s[a];await writeFile(storePath,JSON.stringify(s));}};
|
|
2971
|
+
const fetcher=async(url,init)=>{if(new Headers(init?.headers).get('x-dharma-message-id')===orphanId){
|
|
2972
|
+
process.stdout.write('sending;');setInterval(()=>{},1000);await new Promise(()=>{});}
|
|
2973
|
+
return new Response(JSON.stringify({ok:true}),{status:201});};
|
|
2974
|
+
const client=await AgentFabricClient.open({configPath,statePath,store,fetcher});
|
|
2975
|
+
await client.openSession();process.stdout.write('done;');`,
|
|
2976
|
+
f.configPath, f.statePath, storePath, orphanMessageId(orphan)], { stdio: ['ignore', 'pipe', 'inherit'] });
|
|
2977
|
+
const exited = new Promise(accept => child.once('exit', accept));
|
|
2978
|
+
let output = '';
|
|
2979
|
+
const sending = new Promise(accept => child.stdout.on('data', data => {
|
|
2980
|
+
output += String(data);
|
|
2981
|
+
if (output.includes('sending;'))
|
|
2982
|
+
accept();
|
|
2983
|
+
}));
|
|
2984
|
+
assert.equal(await Promise.race([sending.then(() => 'sending'), exited.then(() => 'exited')]), 'sending', output);
|
|
2985
|
+
child.kill('SIGKILL');
|
|
2986
|
+
await exited;
|
|
2987
|
+
const a = await f.open();
|
|
2988
|
+
await a.openSession();
|
|
2989
|
+
assert.equal(sentTimes(f.calls, orphanMessageId(orphan)), 0, 'the next client does not send it within 10 minutes');
|
|
2990
|
+
// 10 minutes later it could be sent only signed again, and its owner is gone: it is settled unsent.
|
|
2991
|
+
const b = await f.open();
|
|
2992
|
+
await sharedLater(11, () => b.openSession());
|
|
2993
|
+
assert.equal(sentTimes(f.calls, orphanMessageId(orphan)), 0, 'it is not sent again, as signed or signed again');
|
|
2994
|
+
assert.deepEqual(keptMessageIds(await f.read()), []);
|
|
2995
|
+
});
|
|
2996
|
+
// Convergence review round 1: orphans are sent after the caller has its answer, answers leave the file, and an
|
|
2997
|
+
// earlier version still finds one kept request.
|
|
2998
|
+
const isOrphanCall = (orphans, call) => orphans.some(entry => orphanMessageId(entry) === call.messageId);
|
|
2999
|
+
test('a long-lived client returns its own answer before it sends a due orphan, and sends one per section (shared protocol state)', async () => {
|
|
3000
|
+
const gate = sharedGate(), entered = sharedGate();
|
|
3001
|
+
const slow = new Set();
|
|
3002
|
+
const f = await sharedStateFixture('fabric-shared-orphan-after-', async (call) => {
|
|
3003
|
+
if (slow.delete(call.messageId)) {
|
|
3004
|
+
entered.open();
|
|
3005
|
+
await gate.promise;
|
|
3006
|
+
}
|
|
3007
|
+
return sharedOk();
|
|
3008
|
+
});
|
|
3009
|
+
const dead = await exitedPid();
|
|
3010
|
+
const orphans = [sharedOrphan(f.config.deviceId, dead), sharedOrphan(f.config.deviceId, dead)];
|
|
3011
|
+
for (const entry of orphans)
|
|
3012
|
+
slow.add(orphanMessageId(entry));
|
|
3013
|
+
const loop = await f.open('relay:workspace-loop');
|
|
3014
|
+
await loop.openSession();
|
|
3015
|
+
// Commands that exited left their kept requests after the loop opened its session.
|
|
3016
|
+
await writeSharedOutboxes(f.statePath, { ...(await f.read()).outboxes, [`cli:${dead}:x:1`]: orphans[0], [`cli:${dead}:x:2`]: orphans[1] });
|
|
3017
|
+
assert.equal(f.calls.filter(call => isOrphanCall(orphans, call)).length, 0);
|
|
3018
|
+
try {
|
|
3019
|
+
const first = await Promise.race([loop.pollTask().then(() => 'answered'), sharedDelay(2_000).then(() => 'held')]);
|
|
3020
|
+
assert.equal(first, 'answered', 'the poll returns its own answer while the orphan send is still open');
|
|
3021
|
+
await entered.promise;
|
|
3022
|
+
const next = loop.pollTask();
|
|
3023
|
+
await sharedDelay(100);
|
|
3024
|
+
assert.equal(f.calls.filter(call => call.path === '/agent-fabric/tasks/poll').length, 1, 'the next request waits for that one send');
|
|
3025
|
+
gate.open();
|
|
3026
|
+
await next;
|
|
3027
|
+
assert.deepEqual(f.calls.slice(0, 4).map(call => isOrphanCall(orphans, call) ? 'orphan' : call.path), ['/agent-fabric/sessions', '/agent-fabric/tasks/poll', 'orphan', '/agent-fabric/tasks/poll'], 'one orphan per section, after the request has its answer');
|
|
3028
|
+
await loop.pollTask();
|
|
3029
|
+
await sharedSettled(f.config);
|
|
3030
|
+
}
|
|
3031
|
+
finally {
|
|
3032
|
+
gate.open();
|
|
3033
|
+
}
|
|
3034
|
+
assert.deepEqual(orphans.map(entry => sentTimes(f.calls, orphanMessageId(entry))), [1, 1]);
|
|
3035
|
+
assert.deepEqual(keptMessageIds(await f.read()), []);
|
|
3036
|
+
});
|
|
3037
|
+
test('no client of a process sends orphans after its requests while a task lease is held there (shared protocol state)', async () => {
|
|
3038
|
+
const taskId = randomUUID();
|
|
3039
|
+
let leased = true;
|
|
3040
|
+
const f = await sharedStateFixture('fabric-shared-orphan-lease-', call => call.path === '/agent-fabric/tasks/poll' && leased
|
|
3041
|
+
? sharedOk({ task: { envelope: { taskId } } }) : sharedOk());
|
|
3042
|
+
const dead = await exitedPid();
|
|
3043
|
+
const orphan = sharedOrphan(f.config.deviceId, dead);
|
|
3044
|
+
const a = await f.open('relay:workspace-a'), b = await f.open('relay:workspace-b');
|
|
3045
|
+
await a.openSession();
|
|
3046
|
+
await b.openSession();
|
|
3047
|
+
// A command that exited left its kept request after both loops opened their sessions.
|
|
3048
|
+
await writeSharedOutboxes(f.statePath, { ...(await f.read()).outboxes, [`cli:${dead}:x:1`]: orphan });
|
|
3049
|
+
await a.pollTask();
|
|
3050
|
+
leased = false;
|
|
3051
|
+
await b.pollTask();
|
|
3052
|
+
await a.postTaskEvent(taskId, 'started', {});
|
|
3053
|
+
await a.postTaskEvent(taskId, 'lease_extended', { taskId });
|
|
3054
|
+
await b.pollTask();
|
|
3055
|
+
await b.pollTask();
|
|
3056
|
+
await (await f.open('relay:workspace-c')).openSession();
|
|
3057
|
+
assert.equal(sentTimes(f.calls, orphanMessageId(orphan)), 0, 'no orphan is sent while the task lease is held, not even by a session open');
|
|
3058
|
+
await a.postTaskEvent(taskId, 'completed', {});
|
|
3059
|
+
await b.pollTask();
|
|
3060
|
+
await sharedSettled(f.config);
|
|
3061
|
+
assert.equal(sentTimes(f.calls, orphanMessageId(orphan)), 1, 'once the task ends, it is sent after the next request');
|
|
3062
|
+
assert.deepEqual(keptMessageIds(await f.read()), []);
|
|
3063
|
+
});
|
|
3064
|
+
test('an adopted answer stays in the file only for an owner that can read it, and for at most 10 minutes (shared protocol state)', async () => {
|
|
3065
|
+
let down = false;
|
|
3066
|
+
const f = await sharedStateFixture('fabric-shared-orphan-canary-', call => {
|
|
3067
|
+
if (down && call.path === '/agent-fabric/workspaces')
|
|
3068
|
+
return sharedUnavailable();
|
|
3069
|
+
return sharedOk({ private: `private-${call.path.split('/').at(-1)}-canary` });
|
|
3070
|
+
});
|
|
3071
|
+
const dead = await exitedPid();
|
|
3072
|
+
const poll = sharedOrphan(f.config.deviceId, dead, { route: '/agent-fabric/tasks/poll' });
|
|
3073
|
+
const exited = sharedOrphan(f.config.deviceId, dead);
|
|
3074
|
+
await writeSharedOutboxes(f.statePath, { 'cli:1:exited:1': poll, 'cli:1:exited:2': exited });
|
|
3075
|
+
const a = await f.open(), b = await f.open();
|
|
3076
|
+
await a.openSession();
|
|
3077
|
+
assert.equal(sentTimes(f.calls, orphanMessageId(poll)), 0, 'an orphaned poll is removed without sending, like a session open');
|
|
3078
|
+
assert.equal(sentTimes(f.calls, orphanMessageId(exited)), 1);
|
|
3079
|
+
assert.doesNotMatch(await readFile(f.statePath, 'utf8'), /canary/, 'no answer is kept for an owner that has exited');
|
|
3080
|
+
await b.openSession();
|
|
3081
|
+
down = true;
|
|
3082
|
+
await assert.rejects(a.registerWorkspace({ workspaceId: 'a' }), error => error instanceof AgentFabricRequestError && error.status === 503);
|
|
3083
|
+
const kept = f.calls.at(-1);
|
|
3084
|
+
down = false;
|
|
3085
|
+
// The owner still runs, and its entry stood unchanged for 10 minutes.
|
|
3086
|
+
const state = await f.read();
|
|
3087
|
+
const [ownerKey] = Object.entries(state.outboxes ?? {}).find(([, entry]) => entry.pending) ?? [];
|
|
3088
|
+
state.outboxes[ownerKey].updatedAt = new Date(Date.now() - 11 * 60_000).toISOString();
|
|
3089
|
+
await writeFile(f.statePath, JSON.stringify(state));
|
|
3090
|
+
await (await f.open()).openSession();
|
|
3091
|
+
assert.equal(sentTimes(f.calls, kept.messageId), 2);
|
|
3092
|
+
assert.match(await readFile(f.statePath, 'utf8'), /private-workspaces-canary/, 'the answer stays for its live owner');
|
|
3093
|
+
const c = await f.open();
|
|
3094
|
+
await sharedLater(22, () => c.openSession());
|
|
3095
|
+
assert.doesNotMatch(await readFile(f.statePath, 'utf8'), /canary/, 'and leaves the file once 10 minutes have passed, while the owner still runs');
|
|
3096
|
+
assert.deepEqual(keptMessageIds(await f.read()), []);
|
|
3097
|
+
});
|
|
3098
|
+
test('the copy for an earlier version holds only a due request recent enough to replay exactly as signed, never a live client\'s fresh request, and this version sends it once (shared protocol state)', async () => {
|
|
3099
|
+
let down = false;
|
|
3100
|
+
const f = await sharedStateFixture('fabric-shared-rollback-', call => down && call.path === '/agent-fabric/workspaces' ? sharedUnavailable() : sharedOk());
|
|
3101
|
+
const a = await f.open('relay:w1'), b = await f.open();
|
|
3102
|
+
await a.openSession();
|
|
3103
|
+
await b.openSession();
|
|
3104
|
+
down = true;
|
|
3105
|
+
await assert.rejects(a.registerWorkspace({ workspaceId: 'w1' }), error => error instanceof AgentFabricRequestError && error.status === 503);
|
|
3106
|
+
const kept = f.calls.at(-1);
|
|
3107
|
+
down = false;
|
|
3108
|
+
await b.pollTask();
|
|
3109
|
+
assert.equal((await f.read()).pending, null, 'a live client\'s fresh request is never copied');
|
|
3110
|
+
// 10 minutes later the request is due, but it was signed at least that long ago. An earlier version gives a kept
|
|
3111
|
+
// request signed more than 4 minutes ago a new time, nonce and signature in that request's session at its sequence,
|
|
3112
|
+
// so after a rollback it would sign in the owner's session: the request is never copied there.
|
|
3113
|
+
await sharedLater(11, () => b.pollTask());
|
|
3114
|
+
const state = await f.read();
|
|
3115
|
+
assert.deepEqual([state.sessionId, state.nextSequence, state.pending], [null, 1, null], 'a request signed more than 4 minutes ago is never copied for an earlier version, which would sign it again');
|
|
3116
|
+
assert.deepEqual(keptMessageIds(state), [kept.messageId], 'it stays in its owner\'s entry');
|
|
3117
|
+
// It was signed as long ago: another client leaves it to its owner, which sends it itself.
|
|
3118
|
+
const c = await f.open();
|
|
3119
|
+
await sharedLater(11, () => c.openSession());
|
|
3120
|
+
assert.equal(sentTimes(f.calls, kept.messageId), 1, 'another client does not send it, as signed or signed again');
|
|
3121
|
+
await sharedLater(11, () => a.pollTask());
|
|
3122
|
+
assert.equal(sentTimes(f.calls, kept.messageId), 2, 'its owner sends it once');
|
|
3123
|
+
let after = await f.read();
|
|
3124
|
+
assert.deepEqual([after.pending, keptMessageIds(after)], [null, []]);
|
|
3125
|
+
// A kept request whose command exited is due at once, and is copied exactly as signed while its signature is recent
|
|
3126
|
+
// enough to send unchanged. Another live client's fresh one is not copied, nor an exited command's request signed 5
|
|
3127
|
+
// minutes ago, although it stood unchanged for longer.
|
|
3128
|
+
const dead = await exitedPid();
|
|
3129
|
+
const exited = sharedOrphan(f.config.deviceId, dead), fresh = sharedOrphan(f.config.deviceId, process.pid);
|
|
3130
|
+
const stale = sharedOrphan(f.config.deviceId, dead, { minutesAgo: 5 });
|
|
3131
|
+
stale.pending.headers['x-dharma-timestamp'] = new Date(Date.now() - 5 * 60_000).toISOString();
|
|
3132
|
+
await writeSharedOutboxes(f.statePath, { ...after.outboxes, [`cli:${dead}:x:0`]: stale, [`cli:${dead}:x:1`]: exited,
|
|
3133
|
+
[`cli:${process.pid}:fresh:1`]: fresh });
|
|
3134
|
+
await b.pollTask();
|
|
3135
|
+
after = await f.read();
|
|
3136
|
+
assert.equal(after.pending?.headers['x-dharma-message-id'], orphanMessageId(exited), 'an exited command\'s recent kept request is copied, and one signed 5 minutes ago is not');
|
|
3137
|
+
assert.deepEqual([after.sessionId, after.nextSequence, after.pending], [exited.sessionId, exited.nextSequence, exited.pending], 'every signed header and the body of the copy equal its entry\'s, so an earlier version replays it unchanged');
|
|
3138
|
+
});
|
|
3139
|
+
test('the README says what a rollback to an earlier version keeps (shared protocol state)', async () => {
|
|
3140
|
+
const readme = (await readFile(new URL('../README.md', import.meta.url), 'utf8')).replace(/\s+/g, ' ');
|
|
3141
|
+
assert.equal(/rollback to an earlier version/i.test(readme), true, 'the README describes a rollback');
|
|
3142
|
+
assert.equal(/oldest kept request that is already due/i.test(readme), true, 'the README names the one kept request an earlier version replays');
|
|
3143
|
+
assert.equal(/only the owner of a session — the process that opened it, or a restarted process with the same owner key for its own kept request only — signs in it/i.test(readme), true, 'the README says who signs in a session, with the restarted owner exception in the rule itself');
|
|
3144
|
+
assert.equal(/sends a kept request only exactly as signed/i.test(readme), true, 'the README says every other client resends as signed');
|
|
3145
|
+
assert.equal(/action acknowledgement are never copied/i.test(readme), true, 'the README says no acknowledgement is copied');
|
|
3146
|
+
assert.equal(/a rollback loses at most/i.test(readme), true, 'the README bounds what a rollback loses');
|
|
3147
|
+
assert.equal(/Relay session is not open|idle cap|marked to be sent only as signed/.test(readme), false, 'no closed session, idle cap or as-signed marker is described');
|
|
3148
|
+
});
|
|
3149
|
+
// Owner review: the total send count per hold of signing control is at most 1, including recovery and capacity
|
|
3150
|
+
// pressure, on every path. The README says which holds that covers and does not offer a hold across several calls as
|
|
3151
|
+
// one that keeps it.
|
|
3152
|
+
test('the README says a hold its caller takes makes one send per signed call in it, and the CLI takes none across several (owner review)', async () => {
|
|
3153
|
+
const readme = (await readFile(new URL('../README.md', import.meta.url), 'utf8')).replace(/\s+/g, ' ');
|
|
3154
|
+
const start = readme.indexOf('## How long one signed relay request holds device signing control');
|
|
3155
|
+
const section = readme.slice(start, readme.indexOf(' ## ', start + 1));
|
|
3156
|
+
assert.doesNotMatch(section, /Each hold makes at most one network send/, 'the rule names the holds it covers');
|
|
3157
|
+
assert.doesNotMatch(section, /work turn across several calls/, 'a hold across several calls is not offered as one that keeps the rule');
|
|
3158
|
+
assert.match(section, /each hold that the client or the admission line takes makes at most one network send/i);
|
|
3159
|
+
assert.match(section, /makes one send for each signed call its caller makes in it/);
|
|
3160
|
+
assert.match(section, /named session activation boundary takes only the workspace skill activation lock/);
|
|
3161
|
+
});
|
|
3162
|
+
// Convergence review round 2: an expired answer never becomes a new signature, one orphan send per hold of signing
|
|
3163
|
+
// control, orphan sections of admitted clients and of a lease taken while queued, and a re-signed top-level copy.
|
|
3164
|
+
test('a live owner whose adopted answer has left the file sends its own signed request again, never a new signature of the call (shared protocol state)', async () => {
|
|
3165
|
+
let down = false;
|
|
3166
|
+
const f = await sharedStateFixture('fabric-shared-orphan-expired-', call => down && call.path === '/agent-fabric/workspaces' ? sharedUnavailable() : sharedOk({ workspace: { id: 'registered' } }));
|
|
3167
|
+
const a = await f.open(), b = await f.open();
|
|
3168
|
+
await a.openSession();
|
|
3169
|
+
await b.openSession();
|
|
3170
|
+
down = true;
|
|
3171
|
+
await assert.rejects(a.registerWorkspace({ workspaceId: 'a' }), error => error instanceof AgentFabricRequestError && error.status === 503);
|
|
3172
|
+
const kept = f.calls.at(-1);
|
|
3173
|
+
down = false;
|
|
3174
|
+
// The owner still runs, and its entry stood unchanged for 10 minutes.
|
|
3175
|
+
const state = await f.read();
|
|
3176
|
+
Object.values(state.outboxes ?? {}).find(entry => entry.pending?.headers['x-dharma-message-id'] === kept.messageId)
|
|
3177
|
+
.updatedAt = new Date(Date.now() - 11 * 60_000).toISOString();
|
|
3178
|
+
await writeFile(f.statePath, JSON.stringify(state));
|
|
3179
|
+
await (await f.open()).openSession();
|
|
3180
|
+
assert.equal(sentTimes(f.calls, kept.messageId), 2, 'another client sends it and keeps the answer for its owner');
|
|
3181
|
+
// 10 more minutes pass while the owner stays idle; another client's write removes the answer.
|
|
3182
|
+
await sharedLater(22, () => b.pollTask());
|
|
3183
|
+
assert.equal(Object.values((await f.read()).outboxes ?? {}).some(entry => entry.adoptedOutcome !== undefined), false);
|
|
3184
|
+
assert.deepEqual(await a.registerWorkspace({ workspaceId: 'a' }), { ok: true, workspace: { id: 'registered' } });
|
|
3185
|
+
const sends = f.calls.filter(call => call.path === '/agent-fabric/workspaces');
|
|
3186
|
+
assert.deepEqual(sends.map(call => [call.messageId, call.idempotencyKey, call.sequence]), Array.from({ length: 3 }, () => [kept.messageId, kept.messageId, kept.sequence]), 'the retry sends the request the owner signed, with the same message id, idempotency key and sequence');
|
|
3187
|
+
});
|
|
3188
|
+
test('a session open in another process holds signing control for at most one orphan send, so a lease heartbeat gets in (shared protocol state)', async () => {
|
|
3189
|
+
const taskId = randomUUID();
|
|
3190
|
+
const storePath = resolve(await mkdtemp(resolve(tmpdir(), 'fabric-shared-orphan-heartbeat-store-')), 'store.json');
|
|
3191
|
+
const f = await sharedStateFixture('fabric-shared-orphan-heartbeat-', call => call.path === '/agent-fabric/tasks/poll'
|
|
3192
|
+
? sharedOk({ task: { envelope: { taskId } } }) : sharedOk(), jsonFileStore(storePath));
|
|
3193
|
+
// A relay process: its workspace loop holds a task lease.
|
|
3194
|
+
const loop = await f.open('relay:workspace-a');
|
|
3195
|
+
await loop.openSession();
|
|
3196
|
+
await loop.pollTask();
|
|
3197
|
+
// Four kept requests left by commands that exited.
|
|
3198
|
+
const dead = await exitedPid();
|
|
3199
|
+
const state = await f.read();
|
|
3200
|
+
for (let index = 0; index < 4; index++) {
|
|
3201
|
+
state.outboxes[`cli:${dead}:x:${index}`] = sharedOrphan(f.config.deviceId, dead);
|
|
3202
|
+
}
|
|
3203
|
+
await writeFile(f.statePath, JSON.stringify(state));
|
|
3204
|
+
// An ad hoc command in another process opens its session; each of its sends takes 3 s and is answered.
|
|
3205
|
+
const module = new URL('./index.js', import.meta.url).href;
|
|
3206
|
+
const child = spawn(process.execPath, ['--input-type=module', '-e', `import {AgentFabricClient} from ${JSON.stringify(module)};
|
|
3207
|
+
import {readFile} from 'node:fs/promises';
|
|
3208
|
+
const [configPath,statePath,storePath]=process.argv.slice(1);
|
|
3209
|
+
const load=async()=>JSON.parse(await readFile(storePath,'utf8').catch(()=>'{}'));
|
|
3210
|
+
const store={backend:'linux-secret-service',get:async a=>(await load())[a]??null,put:async()=>{},delete:async()=>{}};
|
|
3211
|
+
const fetcher=async()=>{process.stdout.write('send;');await new Promise(a=>setTimeout(a,3000));
|
|
3212
|
+
return new Response(JSON.stringify({ok:true}),{status:201});};
|
|
3213
|
+
const client=await AgentFabricClient.open({configPath,statePath,store,fetcher});
|
|
3214
|
+
await client.openSession();process.stdout.write('done;');`,
|
|
3215
|
+
f.configPath, f.statePath, storePath], { stdio: ['ignore', 'pipe', 'inherit'] });
|
|
3216
|
+
const exited = new Promise(accept => child.once('exit', accept));
|
|
3217
|
+
let output = '';
|
|
3218
|
+
const sending = new Promise(accept => child.stdout.on('data', data => {
|
|
3219
|
+
output += String(data);
|
|
3220
|
+
if (output.includes('send;'))
|
|
3221
|
+
accept();
|
|
3222
|
+
}));
|
|
3223
|
+
try {
|
|
3224
|
+
assert.equal(await Promise.race([sending.then(() => 'sending'), exited.then(() => 'exited')]), 'sending', output);
|
|
3225
|
+
await loop.postTaskEvent(taskId, 'lease_extended', { taskId });
|
|
3226
|
+
await exited;
|
|
3227
|
+
assert.equal(output.split('send;').length - 1, 2, 'the command sends one orphan and its own session open');
|
|
3228
|
+
}
|
|
3229
|
+
finally {
|
|
3230
|
+
child.kill();
|
|
3231
|
+
await loop.postTaskEvent(taskId, 'failed', {});
|
|
3232
|
+
await sharedSettled(f.config);
|
|
3233
|
+
}
|
|
3234
|
+
});
|
|
3235
|
+
/** The CLI's admission line (admitSignedRequests): each signed request takes signing control before the client's
|
|
3236
|
+
* own queue, so the request runs inside a root hold that is not the client's, one hold per network send. */
|
|
3237
|
+
function sharedAdmitted(client) {
|
|
3238
|
+
const { signedGet, signedPost } = client;
|
|
3239
|
+
let line = Promise.resolve();
|
|
3240
|
+
const admit = (request) => {
|
|
3241
|
+
const turn = line.then(() => withSigningControlPerSend({ config: client.config }, request));
|
|
3242
|
+
line = turn.catch(() => undefined);
|
|
3243
|
+
return turn;
|
|
3244
|
+
};
|
|
3245
|
+
client.signedGet = route => admit(() => signedGet.call(client, route));
|
|
3246
|
+
client.signedPost = (route, body) => admit(() => signedPost.call(client, route, body));
|
|
3247
|
+
return client;
|
|
3248
|
+
}
|
|
3249
|
+
test('a client whose requests go through the admission line still sends a due orphan after its own request (shared protocol state)', async () => {
|
|
3250
|
+
const f = await sharedStateFixture('fabric-shared-orphan-admitted-', () => sharedOk());
|
|
3251
|
+
const dead = await exitedPid();
|
|
3252
|
+
const orphan = sharedOrphan(f.config.deviceId, dead);
|
|
3253
|
+
const loop = sharedAdmitted(await f.open('relay:workspace-a'));
|
|
3254
|
+
await loop.openSession();
|
|
3255
|
+
// A command that exited left its kept request after the loop opened its session.
|
|
3256
|
+
await writeSharedOutboxes(f.statePath, { ...(await f.read()).outboxes, [`cli:${dead}:x:1`]: orphan });
|
|
3257
|
+
await loop.pollTask();
|
|
3258
|
+
await sharedSettled(f.config);
|
|
3259
|
+
assert.deepEqual(f.calls.slice(-2).map(call => call.messageId === orphanMessageId(orphan) ? 'orphan' : call.path), ['/agent-fabric/tasks/poll', 'orphan'], 'the orphan is sent once, after the poll');
|
|
3260
|
+
assert.deepEqual(keptMessageIds(await f.read()), []);
|
|
3261
|
+
});
|
|
3262
|
+
test('an orphan section queued behind a sibling\'s poll sends nothing once that poll took a task lease (shared protocol state)', async () => {
|
|
3263
|
+
const taskId = randomUUID();
|
|
3264
|
+
const f = await sharedStateFixture('fabric-shared-orphan-lease-queued-', call => call.path === '/agent-fabric/tasks/poll' && JSON.parse(call.body).leaseSeconds === 121
|
|
3265
|
+
? sharedOk({ task: { envelope: { taskId } } }) : sharedOk());
|
|
3266
|
+
const dead = await exitedPid();
|
|
3267
|
+
const orphan = sharedOrphan(f.config.deviceId, dead);
|
|
3268
|
+
const a = await f.open('relay:workspace-a'), b = await f.open('relay:workspace-b');
|
|
3269
|
+
await a.openSession();
|
|
3270
|
+
await b.openSession();
|
|
3271
|
+
// A command that exited left its kept request after both loops opened their sessions.
|
|
3272
|
+
await writeSharedOutboxes(f.statePath, { ...(await f.read()).outboxes, [`cli:${dead}:x:1`]: orphan });
|
|
3273
|
+
// A's orphan section waits in line behind B's poll, which returns a task.
|
|
3274
|
+
await Promise.all([a.pollTask(120), b.pollTask(121)]);
|
|
3275
|
+
await sharedSettled(f.config);
|
|
3276
|
+
assert.equal(sentTimes(f.calls, orphanMessageId(orphan)), 0, 'nothing is sent while the lease B took is held');
|
|
3277
|
+
await b.postTaskEvent(taskId, 'failed', {});
|
|
3278
|
+
await sharedSettled(f.config);
|
|
3279
|
+
assert.equal(sentTimes(f.calls, orphanMessageId(orphan)), 1, 'once the task ends, it is sent after the next request');
|
|
3280
|
+
assert.deepEqual(keptMessageIds(await f.read()), []);
|
|
3281
|
+
});
|
|
3282
|
+
/** 256 kept requests of a client that still runs; the first is an action acknowledgement. */
|
|
3283
|
+
const sharedFullOfLiveEntries = (deviceId) => {
|
|
3284
|
+
const outboxes = Object.fromEntries(Array.from({ length: 256 }, (_, index) => [`cli:${process.pid}:live:${index}`, sharedOrphan(deviceId, process.pid, index === 0
|
|
3285
|
+
? { route: '/agent-fabric/decisions/33333333-3333-4333-8333-333333333333/enforcements' } : {})]));
|
|
3286
|
+
outboxes[`cli:${process.pid}:live:0`].pending.body = JSON.stringify(sharedAcknowledgement());
|
|
3287
|
+
return outboxes;
|
|
3288
|
+
};
|
|
3289
|
+
test('a new kept request an earlier version left at the top level of a full file stays there until it is settled (shared protocol state)', async () => {
|
|
3290
|
+
const legacy = sharedPending('', '/agent-fabric/workspaces');
|
|
3291
|
+
const f = await sharedStateFixture('fabric-shared-rollback-overflow-', () => sharedOk());
|
|
3292
|
+
legacy.headers['x-dharma-device-id'] = f.config.deviceId;
|
|
3293
|
+
// The earlier version signed it just now and stopped; the live entries are not due within this test.
|
|
3294
|
+
const outboxes = sharedFullOfLiveEntries(f.config.deviceId);
|
|
3295
|
+
for (const entry of Object.values(outboxes))
|
|
3296
|
+
entry.updatedAt = new Date(Date.now() + 60 * 60_000).toISOString();
|
|
3297
|
+
await writeFile(f.statePath, JSON.stringify({ schema: 'dharma.protocol-state/v1',
|
|
3298
|
+
sessionId: legacy.headers['x-dharma-session-id'], nextSequence: 2, pending: legacy, outboxes }));
|
|
3299
|
+
assert.equal((await (await f.open()).openSession()).ok, true);
|
|
3300
|
+
let after = await f.read();
|
|
3301
|
+
assert.equal(Object.keys(after.outboxes ?? {}).length, 256, 'the file never holds more entries than a reader accepts');
|
|
3302
|
+
assert.equal(after.pending?.headers['x-dharma-message-id'], legacy.headers['x-dharma-message-id'], 'it stays at the top level');
|
|
3303
|
+
assert.equal(sentTimes(f.calls, legacy.headers['x-dharma-message-id']), 0, 'a command of the earlier version may still send it');
|
|
3304
|
+
// 10 minutes later its signature is older than the server accepts, and no process of this version opened its
|
|
3305
|
+
// session: a client settles it unsent.
|
|
3306
|
+
assert.equal((await sharedLater(11, async () => (await f.open()).openSession())).ok, true, 'a client 10 minutes later opens the file');
|
|
3307
|
+
after = await f.read();
|
|
3308
|
+
assert.equal(sentTimes(f.calls, legacy.headers['x-dharma-message-id']), 0, 'it is never sent, as signed or signed again');
|
|
3309
|
+
assert.notEqual(after.pending?.headers['x-dharma-message-id'], legacy.headers['x-dharma-message-id']);
|
|
3310
|
+
assert.equal(Object.keys(after.outboxes ?? {}).length, 256);
|
|
3311
|
+
});
|
|
3312
|
+
// Convergence review round 2 (second loop): a recorded completion whose state write failed, and orphan sends after a one-shot call.
|
|
3313
|
+
test('a task completion the server accepted survives a failed state write and reaches the file on the next request (shared protocol state)', async () => {
|
|
3314
|
+
const taskId = randomUUID(), trajectoryCapsuleHash = `sha256:${'a'.repeat(64)}`, receiptHash = `sha256:${'b'.repeat(64)}`;
|
|
3315
|
+
let blockNextWrite = false, saved = '';
|
|
3316
|
+
const f = await sharedStateFixture('fabric-shared-completion-write-', async (call) => {
|
|
3317
|
+
if (call.path.endsWith('/events') && blockNextWrite) {
|
|
3318
|
+
// The server accepts the completion; the client's state write after it then fails once.
|
|
3319
|
+
blockNextWrite = false;
|
|
3320
|
+
saved = await readFile(f.statePath, 'utf8');
|
|
3321
|
+
await rm(f.statePath, { force: true });
|
|
3322
|
+
await mkdir(resolve(f.statePath, 'blocker'), { recursive: true });
|
|
3323
|
+
}
|
|
3324
|
+
return sharedOk(call.path.endsWith('/events') ? { receipt: { hash: receiptHash } } : {});
|
|
3325
|
+
});
|
|
3326
|
+
const a = await f.open('relay:workspace-a');
|
|
3327
|
+
await a.openSession();
|
|
3328
|
+
blockNextWrite = true;
|
|
3329
|
+
await assert.rejects(a.postTaskEvent(taskId, 'completed', { trajectoryCapsuleHash }));
|
|
3330
|
+
await rm(f.statePath, { recursive: true, force: true });
|
|
3331
|
+
await writeFile(f.statePath, saved);
|
|
3332
|
+
await a.pollTask();
|
|
3333
|
+
assert.deepEqual(a.listRecoveredTaskCompletions().map(item => item.taskId), [taskId], 'the receipt stays in memory');
|
|
3334
|
+
assert.deepEqual((await f.read()).recoveredTaskCompletions?.map(item => item.taskId), [taskId], 'and the next successful write puts it in the file');
|
|
3335
|
+
await a.acknowledgeRecoveredTaskCompletion(taskId, receiptHash);
|
|
3336
|
+
await a.pollTask();
|
|
3337
|
+
assert.deepEqual((await f.read()).recoveredTaskCompletions, [], 'an acknowledged receipt does not come back');
|
|
3338
|
+
});
|
|
3339
|
+
// Convergence review round 1 (third loop): a kept completion never pushes the stored list past what a reader accepts.
|
|
3340
|
+
test('a completion kept after a failed state write waits for room while other clients have filled the stored list (shared protocol state)', async () => {
|
|
3341
|
+
const hex = (seed) => `sha256:${createHash('sha256').update(seed).digest('hex')}`;
|
|
3342
|
+
const taskA = randomUUID(), receiptA = hex('receipt-a');
|
|
3343
|
+
let blockTask = '', saved = '';
|
|
3344
|
+
const f = await sharedStateFixture('fabric-shared-completion-full-', async (call) => {
|
|
3345
|
+
if (blockTask && call.path === `/agent-fabric/tasks/${blockTask}/events`) {
|
|
3346
|
+
// The server accepts the completion; the client's state write after it then fails once.
|
|
3347
|
+
blockTask = '';
|
|
3348
|
+
saved = await readFile(f.statePath, 'utf8');
|
|
3349
|
+
await rm(f.statePath, { force: true });
|
|
3350
|
+
await mkdir(resolve(f.statePath, 'blocker'), { recursive: true });
|
|
3351
|
+
return sharedOk({ receipt: { hash: receiptA } });
|
|
3352
|
+
}
|
|
3353
|
+
return sharedOk(call.path.endsWith('/events') ? { receipt: { hash: hex(call.path) } } : {});
|
|
3354
|
+
});
|
|
3355
|
+
const a = await f.open('relay:workspace-a');
|
|
3356
|
+
await a.openSession();
|
|
3357
|
+
const stored = Array.from({ length: 999 }, (_, index) => ({ taskId: randomUUID(), trajectoryCapsuleHash: hex(`c${index}`),
|
|
3358
|
+
receiptHash: hex(`r${index}`), recoveredAt: new Date().toISOString() }));
|
|
3359
|
+
await writeFile(f.statePath, JSON.stringify({ ...(await f.read()), recoveredTaskCompletions: stored }));
|
|
3360
|
+
blockTask = taskA;
|
|
3361
|
+
await assert.rejects(a.postTaskEvent(taskA, 'completed', { trajectoryCapsuleHash: hex('capsule-a') }));
|
|
3362
|
+
await rm(f.statePath, { recursive: true, force: true });
|
|
3363
|
+
await writeFile(f.statePath, saved);
|
|
3364
|
+
const b = await f.open('relay:workspace-b');
|
|
3365
|
+
await b.openSession();
|
|
3366
|
+
await b.postTaskEvent(randomUUID(), 'completed', { trajectoryCapsuleHash: hex('capsule-b') });
|
|
3367
|
+
assert.equal((await f.read()).recoveredTaskCompletions?.length, 1_000, 'the other client fills the stored list');
|
|
3368
|
+
await a.pollTask();
|
|
3369
|
+
const full = (await f.read()).recoveredTaskCompletions ?? [];
|
|
3370
|
+
assert.equal(full.length, 1_000, 'the kept completion does not push the stored list past what a reader accepts');
|
|
3371
|
+
assert.ok(!full.some(item => item.taskId === taskA));
|
|
3372
|
+
await (await f.open()).openSession();
|
|
3373
|
+
await b.pollTask();
|
|
3374
|
+
await b.acknowledgeRecoveredTaskCompletion(stored[0].taskId, stored[0].receiptHash);
|
|
3375
|
+
await a.pollTask();
|
|
3376
|
+
const after = (await f.read()).recoveredTaskCompletions ?? [];
|
|
3377
|
+
assert.equal(after.length, 1_000);
|
|
3378
|
+
assert.ok(after.some(item => item.taskId === taskA), 'it reaches the file once there is room');
|
|
3379
|
+
});
|
|
3380
|
+
// Owner review round 1: a completion the server accepted stays recoverable by a fresh owner, within the list bound.
|
|
3381
|
+
test('a fresh owner recovers an accepted completion once the stored list has room, after the original client wrote again while it was full (shared protocol state)', async () => {
|
|
3382
|
+
const hex = (seed) => `sha256:${createHash('sha256').update(seed).digest('hex')}`;
|
|
3383
|
+
const taskA = randomUUID(), receiptA = hex('receipt-a');
|
|
3384
|
+
let blockTask = '', saved = '';
|
|
3385
|
+
const f = await sharedStateFixture('fabric-shared-completion-fresh-', async (call) => {
|
|
3386
|
+
if (blockTask && call.path === `/agent-fabric/tasks/${blockTask}/events`) {
|
|
3387
|
+
// The server accepts the completion; the client's state write after it then fails once.
|
|
3388
|
+
blockTask = '';
|
|
3389
|
+
saved = await readFile(f.statePath, 'utf8');
|
|
3390
|
+
await rm(f.statePath, { force: true });
|
|
3391
|
+
await mkdir(resolve(f.statePath, 'blocker'), { recursive: true });
|
|
3392
|
+
}
|
|
3393
|
+
// Every send of one task's completion gets the same receipt.
|
|
3394
|
+
return sharedOk(call.path.endsWith('/events')
|
|
3395
|
+
? { receipt: { hash: call.path === `/agent-fabric/tasks/${taskA}/events` ? receiptA : hex(call.path) } } : {});
|
|
3396
|
+
});
|
|
3397
|
+
const lengths = [];
|
|
3398
|
+
const read = async () => { const state = await f.read(); lengths.push(state.recoveredTaskCompletions?.length ?? 0); return state; };
|
|
3399
|
+
const a = await f.open('relay:workspace-a');
|
|
3400
|
+
await a.openSession();
|
|
3401
|
+
const stored = Array.from({ length: 999 }, (_, index) => ({ taskId: randomUUID(), trajectoryCapsuleHash: hex(`c${index}`),
|
|
3402
|
+
receiptHash: hex(`r${index}`), recoveredAt: new Date().toISOString() }));
|
|
3403
|
+
await writeFile(f.statePath, JSON.stringify({ ...(await f.read()), recoveredTaskCompletions: stored }));
|
|
3404
|
+
blockTask = taskA;
|
|
3405
|
+
await assert.rejects(a.postTaskEvent(taskA, 'completed', { trajectoryCapsuleHash: hex('capsule-a') }));
|
|
3406
|
+
await rm(f.statePath, { recursive: true, force: true });
|
|
3407
|
+
await writeFile(f.statePath, saved);
|
|
3408
|
+
const b = await f.open('relay:workspace-b');
|
|
3409
|
+
await b.openSession();
|
|
3410
|
+
await b.postTaskEvent(randomUUID(), 'completed', { trajectoryCapsuleHash: hex('capsule-b') });
|
|
3411
|
+
assert.equal((await read()).recoveredTaskCompletions?.length, 1_000, 'the other client fills the stored list');
|
|
3412
|
+
// The original client writes again while the list is full. A replacement process never inherits its memory.
|
|
3413
|
+
await a.pollTask().catch(() => undefined);
|
|
3414
|
+
assert.ok(!((await read()).recoveredTaskCompletions ?? []).some(item => item.taskId === taskA));
|
|
3415
|
+
await b.acknowledgeRecoveredTaskCompletion(stored[0].taskId, stored[0].receiptHash);
|
|
3416
|
+
await read();
|
|
3417
|
+
const fresh = await f.open('relay:workspace-a');
|
|
3418
|
+
await fresh.openSession();
|
|
3419
|
+
await fresh.pollTask();
|
|
3420
|
+
const after = (await read()).recoveredTaskCompletions ?? [];
|
|
3421
|
+
assert.ok(after.some(item => item.taskId === taskA && item.receiptHash === receiptA), 'a fresh owner, without the original client, recovers the completion once there is room');
|
|
3422
|
+
assert.ok(lengths.every(length => length <= 1_000), 'the stored list never exceeds what a reader accepts');
|
|
3423
|
+
});
|
|
3424
|
+
// Owner review round 1: replaying a kept completion while the stored list is full never forgets it.
|
|
3425
|
+
test('a kept completion that a fresh owner replays while the stored list is full is not dropped (shared protocol state)', async () => {
|
|
3426
|
+
const hex = (seed) => `sha256:${createHash('sha256').update(seed).digest('hex')}`;
|
|
3427
|
+
const taskA = randomUUID(), receiptA = hex('receipt-a');
|
|
3428
|
+
let down = false;
|
|
3429
|
+
const f = await sharedStateFixture('fabric-shared-completion-replay-full-', call => {
|
|
3430
|
+
const own = call.path === `/agent-fabric/tasks/${taskA}/events`;
|
|
3431
|
+
if (down && own)
|
|
3432
|
+
return sharedUnavailable();
|
|
3433
|
+
return sharedOk(call.path.endsWith('/events') ? { receipt: { hash: own ? receiptA : hex(call.path) } } : {});
|
|
3434
|
+
});
|
|
3435
|
+
const lengths = [];
|
|
3436
|
+
const read = async () => { const state = await f.read(); lengths.push(state.recoveredTaskCompletions?.length ?? 0); return state; };
|
|
3437
|
+
const a = await f.open('relay:workspace-a');
|
|
3438
|
+
await a.openSession();
|
|
3439
|
+
const stored = Array.from({ length: 999 }, (_, index) => ({ taskId: randomUUID(), trajectoryCapsuleHash: hex(`c${index}`),
|
|
3440
|
+
receiptHash: hex(`r${index}`), recoveredAt: new Date().toISOString() }));
|
|
3441
|
+
await writeFile(f.statePath, JSON.stringify({ ...(await f.read()), recoveredTaskCompletions: stored }));
|
|
3442
|
+
down = true;
|
|
3443
|
+
await assert.rejects(a.postTaskEvent(taskA, 'completed', { trajectoryCapsuleHash: hex('capsule-a') }), error => error instanceof AgentFabricRequestError && error.status === 503);
|
|
3444
|
+
const kept = f.calls.at(-1);
|
|
3445
|
+
down = false;
|
|
3446
|
+
const b = await f.open('relay:workspace-b');
|
|
3447
|
+
await b.openSession();
|
|
3448
|
+
await b.postTaskEvent(randomUUID(), 'completed', { trajectoryCapsuleHash: hex('capsule-b') });
|
|
3449
|
+
assert.equal((await read()).recoveredTaskCompletions?.length, 1_000, 'the other client fills the stored list');
|
|
3450
|
+
// A replacement owner replays its kept completion while the list is full; refusing room is not forgetting it.
|
|
3451
|
+
const replacement = await f.open('relay:workspace-a');
|
|
3452
|
+
await replacement.openSession().catch(() => undefined);
|
|
3453
|
+
assert.equal((await read()).recoveredTaskCompletions?.length, 1_000);
|
|
3454
|
+
await b.acknowledgeRecoveredTaskCompletion(stored[0].taskId, stored[0].receiptHash);
|
|
3455
|
+
const fresh = await f.open('relay:workspace-a');
|
|
3456
|
+
await fresh.openSession();
|
|
3457
|
+
await fresh.pollTask();
|
|
3458
|
+
const after = (await read()).recoveredTaskCompletions ?? [];
|
|
3459
|
+
assert.ok(after.some(item => item.taskId === taskA && item.receiptHash === receiptA), 'the kept completion reaches the stored list once there is room');
|
|
3460
|
+
assert.ok(f.calls.filter(call => call.messageId === kept.messageId).length >= 2, 'it is sent again, never signed again');
|
|
3461
|
+
assert.ok(lengths.every(length => length <= 1_000), 'the stored list never exceeds what a reader accepts');
|
|
3462
|
+
});
|
|
3463
|
+
test('a client opened without an owner key sends due orphans only when it opens its session (shared protocol state)', async () => {
|
|
3464
|
+
const f = await sharedStateFixture('fabric-shared-orphan-one-shot-', () => sharedOk());
|
|
3465
|
+
const dead = await exitedPid();
|
|
3466
|
+
const orphans = [sharedOrphan(f.config.deviceId, dead), sharedOrphan(f.config.deviceId, dead)];
|
|
3467
|
+
await writeSharedOutboxes(f.statePath, { [`cli:${dead}:x:1`]: orphans[0], [`cli:${dead}:x:2`]: orphans[1] });
|
|
3468
|
+
const command = await f.open();
|
|
3469
|
+
await command.openSession();
|
|
3470
|
+
await command.pollTask();
|
|
3471
|
+
await sharedSettled(f.config);
|
|
3472
|
+
assert.deepEqual(orphans.map(entry => sentTimes(f.calls, orphanMessageId(entry))).sort(), [0, 1], 'its session open sends one; no section after its call keeps the command running');
|
|
3473
|
+
assert.equal(f.calls.at(-1).path, '/agent-fabric/tasks/poll');
|
|
3474
|
+
const loop = await f.open('relay:workspace-a');
|
|
3475
|
+
await loop.openSession();
|
|
3476
|
+
assert.deepEqual(orphans.map(entry => sentTimes(f.calls, orphanMessageId(entry))), [1, 1], 'the next session open sends the other');
|
|
3477
|
+
});
|
|
3478
|
+
// Owner review round 2: an owner whose kept request left the file without being sent sends it again, and an
|
|
3479
|
+
// acknowledgement removes a receipt wherever it still waits.
|
|
3480
|
+
async function sharedLostEntryFixture(prefix) {
|
|
3481
|
+
const down = new Set();
|
|
3482
|
+
const f = await sharedStateFixture(prefix, call => {
|
|
3483
|
+
const task = call.path.match(/^\/agent-fabric\/tasks\/([^/]+)\/events$/)?.[1];
|
|
3484
|
+
if (task && down.has(task))
|
|
3485
|
+
return sharedUnavailable();
|
|
3486
|
+
return sharedOk(task ? { receipt: { hash: `sha256:${createHash('sha256').update(task).digest('hex')}` } } : {});
|
|
3487
|
+
});
|
|
3488
|
+
return { ...f, down };
|
|
3489
|
+
}
|
|
3490
|
+
test('an owner whose kept request left the file because the file was removed sends it again with its message id (shared protocol state)', async () => {
|
|
3491
|
+
const f = await sharedLostEntryFixture('fabric-shared-lost-entry-removed-');
|
|
3492
|
+
const taskId = randomUUID(), body = { trajectoryCapsuleHash: `sha256:${'a'.repeat(64)}` };
|
|
3493
|
+
const a = await f.open('relay:workspace-a');
|
|
3494
|
+
await a.openSession();
|
|
3495
|
+
f.down.add(taskId);
|
|
3496
|
+
await assert.rejects(a.postTaskEvent(taskId, 'completed', body), error => error instanceof AgentFabricRequestError && error.status === 503);
|
|
3497
|
+
const kept = f.calls.at(-1);
|
|
3498
|
+
f.down.clear();
|
|
3499
|
+
await rm(f.statePath, { force: true });
|
|
3500
|
+
await a.postTaskEvent(taskId, 'completed', body);
|
|
3501
|
+
const sent = f.calls.filter(call => call.messageId === kept.messageId);
|
|
3502
|
+
assert.equal(sent.length, 2, 'the owner sends its kept completion again');
|
|
3503
|
+
assert.deepEqual(sent.map(call => call.idempotencyKey), [kept.messageId, kept.messageId]);
|
|
3504
|
+
assert.deepEqual((await f.read()).recoveredTaskCompletions?.map(item => item.taskId), [taskId]);
|
|
3505
|
+
});
|
|
3506
|
+
test('an owner whose kept request left the file in a rewrite without outboxes sends it again when another request was the top-level copy (shared protocol state)', async () => {
|
|
3507
|
+
const f = await sharedLostEntryFixture('fabric-shared-lost-entry-rewritten-');
|
|
3508
|
+
const older = randomUUID(), taskId = randomUUID(), body = { trajectoryCapsuleHash: `sha256:${'a'.repeat(64)}` };
|
|
3509
|
+
const b = await f.open('relay:workspace-b'), a = await f.open('relay:workspace-a'), c = await f.open();
|
|
3510
|
+
await b.openSession();
|
|
3511
|
+
await a.openSession();
|
|
3512
|
+
await c.openSession();
|
|
3513
|
+
f.down.add(older).add(taskId);
|
|
3514
|
+
await assert.rejects(b.postTaskEvent(older, 'completed', body));
|
|
3515
|
+
const copied = f.calls.at(-1);
|
|
3516
|
+
await sharedDelay(5);
|
|
3517
|
+
await assert.rejects(a.postTaskEvent(taskId, 'completed', body), error => error instanceof AgentFabricRequestError && error.status === 503);
|
|
3518
|
+
const kept = f.calls.at(-1);
|
|
3519
|
+
f.down.clear();
|
|
3520
|
+
// Both entries stood unchanged for 10 minutes, so the next write copies the older one to the top level.
|
|
3521
|
+
const aged = await f.read();
|
|
3522
|
+
aged.outboxes['relay:workspace-b'].updatedAt = new Date(Date.now() - 12 * 60_000).toISOString();
|
|
3523
|
+
aged.outboxes['relay:workspace-a'].updatedAt = new Date(Date.now() - 11 * 60_000).toISOString();
|
|
3524
|
+
await writeFile(f.statePath, JSON.stringify(aged));
|
|
3525
|
+
await c.pollTask();
|
|
3526
|
+
const state = await f.read();
|
|
3527
|
+
assert.equal(state.pending?.headers['x-dharma-message-id'], copied.messageId, 'the older kept request is the top-level copy');
|
|
3528
|
+
// A session open of the earlier version replays only the top-level copy, then writes a file without outboxes.
|
|
3529
|
+
await writeFile(f.statePath, JSON.stringify({ schema: 'dharma.protocol-state/v1', sessionId: randomUUID(), nextSequence: 1, pending: null,
|
|
3530
|
+
recoveredTaskCompletions: state.recoveredTaskCompletions ?? [] }));
|
|
3531
|
+
await a.postTaskEvent(taskId, 'completed', body);
|
|
3532
|
+
const sent = f.calls.filter(call => call.messageId === kept.messageId);
|
|
3533
|
+
assert.equal(sent.length, 2, 'the owner sends its kept completion again');
|
|
3534
|
+
assert.deepEqual(sent.map(call => call.idempotencyKey), [kept.messageId, kept.messageId]);
|
|
3535
|
+
assert.ok((await f.read()).recoveredTaskCompletions?.some(item => item.taskId === taskId));
|
|
3536
|
+
});
|
|
3537
|
+
test('acknowledging a listed receipt that the file still keeps in another client\'s entry removes it there, so it never comes back (shared protocol state)', async () => {
|
|
3538
|
+
const f = await sharedLostEntryFixture('fabric-shared-ack-kept-');
|
|
3539
|
+
const hex = (seed) => `sha256:${createHash('sha256').update(seed).digest('hex')}`;
|
|
3540
|
+
const body = (seed) => ({ trajectoryCapsuleHash: hex(seed) });
|
|
3541
|
+
const a = await f.open('relay:a'), c = await f.open('relay:c'), b = await f.open('relay:b'), d = await f.open('relay:d');
|
|
3542
|
+
for (const client of [a, b, c, d])
|
|
3543
|
+
await client.openSession();
|
|
3544
|
+
const stored = Array.from({ length: 999 }, (_, index) => ({ taskId: randomUUID(), trajectoryCapsuleHash: hex(`c${index}`),
|
|
3545
|
+
receiptHash: hex(`r${index}`), recoveredAt: new Date().toISOString() }));
|
|
3546
|
+
await writeFile(f.statePath, JSON.stringify({ ...(await f.read()), recoveredTaskCompletions: stored }));
|
|
3547
|
+
const x = randomUUID(), w = randomUUID();
|
|
3548
|
+
f.down.add(x).add(w);
|
|
3549
|
+
await assert.rejects(a.postTaskEvent(x, 'completed', body('x')));
|
|
3550
|
+
await assert.rejects(c.postTaskEvent(w, 'completed', body('w')));
|
|
3551
|
+
f.down.clear();
|
|
3552
|
+
await b.postTaskEvent(randomUUID(), 'completed', body('y'));
|
|
3553
|
+
assert.equal((await f.read()).recoveredTaskCompletions?.length, 1_000, 'the stored list is full');
|
|
3554
|
+
// Both are accepted now and wait in their owners' entries.
|
|
3555
|
+
await a.openSession();
|
|
3556
|
+
await c.openSession();
|
|
3557
|
+
const keptIn = async (taskId) => Object.values((await f.read()).outboxes ?? {})
|
|
3558
|
+
.some(entry => entry.keptCompletions?.some(item => item.taskId === taskId));
|
|
3559
|
+
assert.ok(await keptIn(x) && await keptIn(w));
|
|
3560
|
+
// One slot frees; then a section that reads the file and writes nothing lists one of the waiting receipts.
|
|
3561
|
+
await b.acknowledgeRecoveredTaskCompletion(stored[0].taskId, stored[0].receiptHash);
|
|
3562
|
+
await d.acknowledgeRecoveredTaskCompletion(stored[0].taskId, stored[0].receiptHash);
|
|
3563
|
+
const seen = d.listRecoveredTaskCompletions().filter(item => item.taskId === x || item.taskId === w);
|
|
3564
|
+
assert.equal(seen.length, 1);
|
|
3565
|
+
const listed = seen[0];
|
|
3566
|
+
// The other owner writes: its own waiting receipt takes the free slot, so the one d listed waits again.
|
|
3567
|
+
await (listed.taskId === x ? c : a).pollTask();
|
|
3568
|
+
assert.ok(!((await f.read()).recoveredTaskCompletions ?? []).some(item => item.taskId === listed.taskId));
|
|
3569
|
+
await assert.rejects(d.acknowledgeRecoveredTaskCompletion(listed.taskId, hex('another receipt')), /Recovered task completion receipt does not match/);
|
|
3570
|
+
await d.acknowledgeRecoveredTaskCompletion(listed.taskId, listed.receiptHash);
|
|
3571
|
+
assert.equal(await keptIn(listed.taskId), false, 'the acknowledgement removes the receipt from the entry that keeps it');
|
|
3572
|
+
// Room frees again; no reader moves the acknowledged receipt back to the list.
|
|
3573
|
+
await b.acknowledgeRecoveredTaskCompletion(stored[1].taskId, stored[1].receiptHash);
|
|
3574
|
+
await b.pollTask();
|
|
3575
|
+
const fresh = await f.open('relay:e');
|
|
3576
|
+
await fresh.openSession();
|
|
3577
|
+
assert.ok(!((await f.read()).recoveredTaskCompletions ?? []).some(item => item.taskId === listed.taskId), 'the acknowledged receipt never comes back');
|
|
3578
|
+
});
|
|
3579
|
+
// Owner review round 3, as simplified: an owner whose entry left the file, for any reason, keeps its kept request in
|
|
3580
|
+
// memory, writes it back before it sends it, and sends it only exactly as signed. It keeps its session and signs its
|
|
3581
|
+
// next call at the next sequence it holds in memory, since only the process that opened a session signs in it.
|
|
3582
|
+
const sharedOutcome = (promise) => promise.then(() => 'accepted', (error) => error instanceof AgentFabricRequestError ? error.status : error.message);
|
|
3583
|
+
test('an owner whose entry left with the removed file after its stale acknowledgement stood unchanged for 10 minutes never signs a sequence again (shared protocol state)', async () => {
|
|
3584
|
+
const f = await sharedMonotonicFixture('fabric-shared-removed-resigned-');
|
|
3585
|
+
const a = await f.open(), b = await f.open();
|
|
3586
|
+
await a.openSession();
|
|
3587
|
+
await b.openSession();
|
|
3588
|
+
const decisionId = '33333333-3333-4333-8333-333333333333';
|
|
3589
|
+
const acknowledgement = sharedAcknowledgement();
|
|
3590
|
+
f.failNext();
|
|
3591
|
+
await assert.rejects(a.postActionEnforcement(decisionId, acknowledgement), error => error instanceof AgentFabricRequestError && error.status === 503);
|
|
3592
|
+
const kept = f.calls.at(-1);
|
|
3593
|
+
// The entry stood unchanged long enough, but its owner still runs: another client's session open leaves the stale
|
|
3594
|
+
// acknowledgement in the entry, since it cannot be sent as signed and only its owner signs in the owner's session.
|
|
3595
|
+
await sharedLater(11, async () => { const c = await f.open(); await c.openSession(); });
|
|
3596
|
+
assert.equal(f.calls.filter(call => call.path === kept.path).length, 1, 'the other client signs nothing in the owner\'s session');
|
|
3597
|
+
await rm(f.statePath, { force: true });
|
|
3598
|
+
const retry = await sharedLater(12, () => sharedOutcome(a.postActionEnforcement(decisionId, acknowledgement)));
|
|
3599
|
+
assertNoSequenceSignedTwice(f.calls, kept.sessionId);
|
|
3600
|
+
assert.notEqual(retry, 409, 'the owner gets no out-of-order refusal');
|
|
3601
|
+
await sharedLater(12, () => a.openSession());
|
|
3602
|
+
assert.equal(await sharedLater(12, () => sharedOutcome(a.postActionEnforcement(decisionId, acknowledgement))), 'accepted', 'in a new session the owner signs the call again and it is accepted');
|
|
3603
|
+
});
|
|
3604
|
+
test('an owner whose state file was removed sends its kept completion again exactly as signed, keeps its session and signs its next call at the next sequence (shared protocol state)', async () => {
|
|
3605
|
+
const f = await sharedLostEntryFixture('fabric-shared-removed-then-written-');
|
|
3606
|
+
const taskId = randomUUID(), body = { trajectoryCapsuleHash: `sha256:${'a'.repeat(64)}` };
|
|
3607
|
+
const a = await f.open('relay:workspace-a'), b = await f.open('relay:workspace-b');
|
|
3608
|
+
await a.openSession();
|
|
3609
|
+
await b.openSession();
|
|
3610
|
+
f.down.add(taskId);
|
|
3611
|
+
await assert.rejects(a.postTaskEvent(taskId, 'completed', body), error => error instanceof AgentFabricRequestError && error.status === 503);
|
|
3612
|
+
const kept = f.calls.at(-1);
|
|
3613
|
+
f.down.clear();
|
|
3614
|
+
await rm(f.statePath, { force: true });
|
|
3615
|
+
await b.pollTask();
|
|
3616
|
+
assert.ok((await f.read()).outboxes, 'the other client wrote a file with outboxes and no entry for the owner');
|
|
3617
|
+
await a.postTaskEvent(taskId, 'completed', body);
|
|
3618
|
+
const sent = f.calls.filter(call => call.messageId === kept.messageId);
|
|
3619
|
+
assert.equal(sent.length, 2, 'the owner sends its kept completion again');
|
|
3620
|
+
assert.deepEqual(sent.map(call => [call.sequence, call.idempotencyKey]), [[kept.sequence, kept.messageId],
|
|
3621
|
+
[kept.sequence, kept.messageId]], 'with the same sequence and idempotency key');
|
|
3622
|
+
assert.ok((await f.read()).recoveredTaskCompletions?.some(item => item.taskId === taskId), 'its receipt is recorded');
|
|
3623
|
+
assert.equal(await sharedOutcome(a.pollTask()), 'accepted', 'the owner keeps its session');
|
|
3624
|
+
const last = f.calls.at(-1);
|
|
3625
|
+
assert.deepEqual([last.path, last.sessionId, last.sequence], ['/agent-fabric/tasks/poll', kept.sessionId, kept.sequence + 1], 'its next call is signed at the next sequence it holds in memory');
|
|
3626
|
+
assertNoSequenceSignedTwice(f.calls, kept.sessionId);
|
|
3627
|
+
});
|
|
3628
|
+
// Owner review round 4: an owner whose entry left the file writes its kept request back to the file before it sends
|
|
3629
|
+
// it again, so a failed send followed by a new client with the same owner key never loses it.
|
|
3630
|
+
test('an owner whose entry left the file keeps its completion in the file while it sends it again, so a fresh owner replays it after a failed send (shared protocol state)', async () => {
|
|
3631
|
+
const f = await sharedLostEntryFixture('fabric-shared-lost-entry-resend-fails-');
|
|
3632
|
+
const taskId = randomUUID(), body = { trajectoryCapsuleHash: `sha256:${'a'.repeat(64)}` };
|
|
3633
|
+
const a = await f.open('relay:workspace-a');
|
|
3634
|
+
await a.openSession();
|
|
3635
|
+
f.down.add(taskId);
|
|
3636
|
+
await assert.rejects(a.postTaskEvent(taskId, 'completed', body), error => error instanceof AgentFabricRequestError && error.status === 503);
|
|
3637
|
+
const kept = f.calls.at(-1);
|
|
3638
|
+
await rm(f.statePath, { force: true });
|
|
3639
|
+
await assert.rejects(a.postTaskEvent(taskId, 'completed', body), error => error instanceof AgentFabricRequestError && error.status === 503);
|
|
3640
|
+
assert.deepEqual(keptMessageIds(await f.read()), [kept.messageId], 'the kept completion is back in the file');
|
|
3641
|
+
f.down.clear();
|
|
3642
|
+
// The failed stage ends the loop; its supervisor opens a new client with the same owner key.
|
|
3643
|
+
const fresh = await f.open('relay:workspace-a');
|
|
3644
|
+
await fresh.openSession();
|
|
3645
|
+
const sent = f.calls.filter(call => call.messageId === kept.messageId);
|
|
3646
|
+
assert.equal(sent.length, 3, 'the fresh owner sends the kept completion again');
|
|
3647
|
+
assert.deepEqual(sent.map(call => [call.sequence, call.idempotencyKey]), Array.from({ length: 3 }, () => [kept.sequence, kept.messageId]), 'always as signed');
|
|
3648
|
+
assert.ok((await f.read()).recoveredTaskCompletions?.some(item => item.taskId === taskId), 'its receipt is recorded');
|
|
3649
|
+
assert.deepEqual(keptMessageIds(await f.read()), []);
|
|
3650
|
+
assertNoSequenceSignedTwice(f.calls, kept.sessionId);
|
|
3651
|
+
});
|
|
3652
|
+
test('a fresh owner that replays an acknowledgement written back after its entry left the file never signs a sequence again (shared protocol state)', async () => {
|
|
3653
|
+
const f = await sharedMonotonicFixture('fabric-shared-lost-ack-written-back-');
|
|
3654
|
+
const a = await f.open('relay:workspace-a');
|
|
3655
|
+
await a.openSession();
|
|
3656
|
+
const decisionId = '33333333-3333-4333-8333-333333333333';
|
|
3657
|
+
const acknowledgement = sharedAcknowledgement();
|
|
3658
|
+
f.failNext();
|
|
3659
|
+
await assert.rejects(a.postActionEnforcement(decisionId, acknowledgement), error => error instanceof AgentFabricRequestError && error.status === 503);
|
|
3660
|
+
const kept = f.calls.at(-1);
|
|
3661
|
+
await rm(f.statePath, { force: true });
|
|
3662
|
+
f.failNext();
|
|
3663
|
+
await assert.rejects(a.postActionEnforcement(decisionId, acknowledgement), error => error instanceof AgentFabricRequestError && error.status === 503);
|
|
3664
|
+
assert.deepEqual(keptMessageIds(await f.read()), [kept.messageId], 'the kept acknowledgement is back in the file');
|
|
3665
|
+
// The fresh owner opens after the acknowledgement left its time window, and another client after 11 minutes.
|
|
3666
|
+
await sharedLater(9, async () => { const fresh = await f.open('relay:workspace-a'); await fresh.openSession(); });
|
|
3667
|
+
await sharedLater(11, async () => { const other = await f.open('relay:workspace-b'); await other.openSession(); });
|
|
3668
|
+
assertNoSequenceSignedTwice(f.calls, kept.sessionId);
|
|
3669
|
+
assert.deepEqual(keptMessageIds(await f.read()), []);
|
|
3670
|
+
});
|
|
3671
|
+
// Owner review round 5: an earlier version writes back the whole file it read when it opened, outboxes included, so
|
|
3672
|
+
// an owner's entry can come back in an older state without its kept request. The owner writes its kept request back
|
|
3673
|
+
// before it sends it again, exactly as signed. A kept request an earlier version left at the top level waits 10
|
|
3674
|
+
// minutes unchanged, like any other, before another client sends it.
|
|
3675
|
+
test('an owner whose entry an earlier version wrote back in an older state writes its kept request back and sends it again as signed, so a fresh owner replays it (shared protocol state)', async () => {
|
|
3676
|
+
const f = await sharedLostEntryFixture('fabric-shared-older-entry-');
|
|
3677
|
+
const taskId = randomUUID(), body = { trajectoryCapsuleHash: `sha256:${'a'.repeat(64)}` };
|
|
3678
|
+
const a = await f.open('relay:workspace-a'), b = await f.open('relay:workspace-b');
|
|
3679
|
+
await a.openSession();
|
|
3680
|
+
await b.openSession();
|
|
3681
|
+
// An earlier version opens and reads the file, outboxes included.
|
|
3682
|
+
const snapshot = await f.read();
|
|
3683
|
+
f.down.add(taskId);
|
|
3684
|
+
await assert.rejects(a.postTaskEvent(taskId, 'completed', body), error => error instanceof AgentFabricRequestError && error.status === 503);
|
|
3685
|
+
const kept = f.calls.at(-1);
|
|
3686
|
+
assert.ok(keptMessageIds(await f.read()).includes(kept.messageId));
|
|
3687
|
+
// Its session open writes back the file it read, with a kept request of its own at the top level.
|
|
3688
|
+
const earlier = sharedPending(f.config.deviceId, '/agent-fabric/workspaces');
|
|
3689
|
+
await writeFile(f.statePath, JSON.stringify({ ...snapshot, sessionId: earlier.headers['x-dharma-session-id'], nextSequence: 2,
|
|
3690
|
+
pending: earlier }));
|
|
3691
|
+
await b.pollTask();
|
|
3692
|
+
await sharedSettled(f.config);
|
|
3693
|
+
assert.ok(!keptMessageIds(await f.read()).includes(kept.messageId), 'the owner\'s entry is back in its older state');
|
|
3694
|
+
await assert.rejects(a.postTaskEvent(taskId, 'completed', body), error => error instanceof AgentFabricRequestError && error.status === 503);
|
|
3695
|
+
assert.ok(keptMessageIds(await f.read()).includes(kept.messageId), 'the owner writes its kept request back before it sends it');
|
|
3696
|
+
f.down.clear();
|
|
3697
|
+
// The failed stage ends the loop; its supervisor opens a new client with the same owner key.
|
|
3698
|
+
const fresh = await f.open('relay:workspace-a');
|
|
3699
|
+
await fresh.openSession();
|
|
3700
|
+
const sent = f.calls.filter(call => call.messageId === kept.messageId);
|
|
3701
|
+
assert.deepEqual(sent.map(call => [call.sequence, call.idempotencyKey]), Array.from({ length: 3 }, () => [kept.sequence, kept.messageId]), 'the fresh owner sends it again, always as signed');
|
|
3702
|
+
assert.ok((await f.read()).recoveredTaskCompletions?.some(item => item.taskId === taskId), 'its receipt is recorded');
|
|
3703
|
+
assert.ok(!keptMessageIds(await f.read()).includes(kept.messageId));
|
|
3704
|
+
assertNoSequenceSignedTwice(f.calls, kept.sessionId);
|
|
3705
|
+
});
|
|
3706
|
+
/** An action acknowledgement an earlier version signed in its session, stamped minutes from now. */
|
|
3707
|
+
const sharedEarlierAcknowledgement = (deviceId, sessionId, sequence, minutes, idempotencyKey) => {
|
|
3708
|
+
const at = new Date(Date.now() + minutes * 60_000).toISOString();
|
|
3709
|
+
return { method: 'POST', pathname: '/api/v1/orgs/org_a/agent-fabric/decisions/33333333-3333-4333-8333-333333333333/enforcements',
|
|
3710
|
+
body: JSON.stringify({ ...sharedAcknowledgement(), acknowledgedAt: at }), headers: {
|
|
3711
|
+
'content-type': 'application/json', 'idempotency-key': idempotencyKey, 'x-dharma-device-id': deviceId,
|
|
3712
|
+
'x-dharma-session-id': sessionId, 'x-dharma-message-id': randomUUID(), 'x-dharma-timestamp': at,
|
|
3713
|
+
'x-dharma-nonce': 'earlier-nonce', 'x-dharma-sequence': String(sequence), 'x-dharma-signature': 'earlier-signature'
|
|
3714
|
+
} };
|
|
3715
|
+
};
|
|
3716
|
+
test('a kept request of an earlier version that still runs is taken by another client only after it stood unchanged for 10 minutes, and a stale acknowledgement is never signed again (shared protocol state)', async () => {
|
|
3717
|
+
const f = await sharedMonotonicFixture('fabric-shared-earlier-live-');
|
|
3718
|
+
const session = randomUUID(), key = randomUUID();
|
|
3719
|
+
const inSession = () => f.calls.filter(call => call.sessionId === session);
|
|
3720
|
+
// The earlier version kept its acknowledgement after an unknown outcome; it writes no outboxes.
|
|
3721
|
+
const first = sharedEarlierAcknowledgement(f.config.deviceId, session, 2, 0, key);
|
|
3722
|
+
await writeFile(f.statePath, JSON.stringify({ schema: 'dharma.protocol-state/v1', sessionId: session, nextSequence: 3, pending: first }));
|
|
3723
|
+
await sharedLater(3, async () => { const n = await f.open('relay:n'); await n.openSession(); });
|
|
3724
|
+
assert.equal(inSession().length, 0, '3 minutes later another client does not send it');
|
|
3725
|
+
// At 9 minutes the earlier version signs its stale acknowledgement again, one sequence higher, and writes the file.
|
|
3726
|
+
const again = sharedEarlierAcknowledgement(f.config.deviceId, session, 3, 9, key);
|
|
3727
|
+
f.calls.push({ path: again.pathname.replace('/api/v1/orgs/org_a', ''), body: again.body,
|
|
3728
|
+
messageId: again.headers['x-dharma-message-id'], idempotencyKey: key, sessionId: session, sequence: 3, headers: again.headers });
|
|
3729
|
+
f.highest.set(session, 3);
|
|
3730
|
+
await writeFile(f.statePath, JSON.stringify({ schema: 'dharma.protocol-state/v1', sessionId: session, nextSequence: 4, pending: again }));
|
|
3731
|
+
await sharedLater(11, async () => { const c = await f.open('relay:c'); await c.openSession(); });
|
|
3732
|
+
assert.equal(inSession().length, 1, 'its new signature restarts the 10 minutes');
|
|
3733
|
+
await sharedLater(20, async () => { const d = await f.open('relay:d'); await d.openSession(); });
|
|
3734
|
+
assert.equal(inSession().length, 1, 'once the earlier version is gone, another client drops the stale acknowledgement unsent');
|
|
3735
|
+
assert.deepEqual(keptMessageIds(await f.read()), []);
|
|
3736
|
+
assertNoSequenceSignedTwice(f.calls, session);
|
|
3737
|
+
});
|
|
3738
|
+
test('acknowledging a completion whose state write failed removes it before any write stores it, so it never comes back (shared protocol state)', async () => {
|
|
3739
|
+
const taskId = randomUUID(), trajectoryCapsuleHash = `sha256:${'a'.repeat(64)}`, receiptHash = `sha256:${'b'.repeat(64)}`;
|
|
3740
|
+
let blockNextWrite = false, saved = '';
|
|
3741
|
+
const f = await sharedStateFixture('fabric-shared-completion-unwritten-ack-', async (call) => {
|
|
3742
|
+
if (call.path.endsWith('/events') && blockNextWrite) {
|
|
3743
|
+
blockNextWrite = false;
|
|
3744
|
+
saved = await readFile(f.statePath, 'utf8');
|
|
3745
|
+
await rm(f.statePath, { force: true });
|
|
3746
|
+
await mkdir(resolve(f.statePath, 'blocker'), { recursive: true });
|
|
3747
|
+
}
|
|
3748
|
+
return sharedOk(call.path.endsWith('/events') ? { receipt: { hash: receiptHash } } : {});
|
|
3749
|
+
});
|
|
3750
|
+
const a = await f.open('relay:workspace-a');
|
|
3751
|
+
await a.openSession();
|
|
3752
|
+
blockNextWrite = true;
|
|
3753
|
+
await assert.rejects(a.postTaskEvent(taskId, 'completed', { trajectoryCapsuleHash }));
|
|
3754
|
+
await rm(f.statePath, { recursive: true, force: true });
|
|
3755
|
+
await writeFile(f.statePath, saved);
|
|
3756
|
+
assert.ok(!((await f.read()).recoveredTaskCompletions ?? []).some(item => item.taskId === taskId), 'no write stored it yet');
|
|
3757
|
+
await a.acknowledgeRecoveredTaskCompletion(taskId, receiptHash);
|
|
3758
|
+
await a.pollTask();
|
|
3759
|
+
const fresh = await f.open('relay:workspace-b');
|
|
3760
|
+
await fresh.openSession();
|
|
3761
|
+
assert.deepEqual([a.listRecoveredTaskCompletions(), fresh.listRecoveredTaskCompletions(),
|
|
3762
|
+
(await f.read()).recoveredTaskCompletions], [[], [], []], 'the acknowledged receipt never comes back');
|
|
3763
|
+
});
|
|
3764
|
+
// Owner review round 6, as simplified: only the process that opened a session signs in it. Another client sends a
|
|
3765
|
+
// kept request only exactly as signed and leaves a workspace loop's stale action acknowledgement in its entry, and an
|
|
3766
|
+
// owner whose entry an earlier version wrote back holding another call resends its own kept request exactly as signed.
|
|
3767
|
+
for (const adopterOutcome of ['accepted', 'retryable']) {
|
|
3768
|
+
test(`another client never signs an owner's stale acknowledgement, so an owner whose entry an earlier version wrote back never signs a sequence twice: ${adopterOutcome} (shared protocol state)`, async () => {
|
|
3769
|
+
const f = await sharedMonotonicFixture(`fabric-shared-adopter-no-resign-${adopterOutcome}-`);
|
|
3770
|
+
const a = await f.open('relay:a');
|
|
3771
|
+
await a.openSession();
|
|
3772
|
+
const decisionId = '33333333-3333-4333-8333-333333333333', acknowledgement = sharedAcknowledgement();
|
|
3773
|
+
f.failNext();
|
|
3774
|
+
await assert.rejects(a.postActionEnforcement(decisionId, acknowledgement), error => error instanceof AgentFabricRequestError && error.status === 503);
|
|
3775
|
+
const kept = f.calls.at(-1);
|
|
3776
|
+
// An earlier version reads the file before another client takes the entry, which stood unchanged for 11 minutes.
|
|
3777
|
+
const snapshot = await f.read();
|
|
3778
|
+
if (adopterOutcome === 'retryable')
|
|
3779
|
+
f.down.add('/enforcements');
|
|
3780
|
+
await sharedLater(11, async () => { const c = await f.open('relay:c'); await c.openSession(); });
|
|
3781
|
+
f.down.clear();
|
|
3782
|
+
assert.equal(f.calls.filter(call => call.sessionId === kept.sessionId).length, 2, 'the other client signs nothing in the owner\'s session');
|
|
3783
|
+
// The earlier version writes back the file it read, so the owner's entry is back in its older state.
|
|
3784
|
+
await writeFile(f.statePath, JSON.stringify(snapshot));
|
|
3785
|
+
const retry = await sharedLater(12, () => sharedOutcome(a.postActionEnforcement(decisionId, acknowledgement)));
|
|
3786
|
+
assertNoSequenceSignedTwice(f.calls, kept.sessionId);
|
|
3787
|
+
assert.equal(retry, 'accepted', 'the owner re-signs it in its own session at a sequence nobody used');
|
|
3788
|
+
});
|
|
3789
|
+
}
|
|
3790
|
+
test('an owner whose entry an earlier version wrote back holding another call resends its own kept request exactly as signed (shared protocol state)', async () => {
|
|
3791
|
+
const f = await sharedMonotonicFixture('fabric-shared-older-entry-other-call-');
|
|
3792
|
+
const a = await f.open('relay:a');
|
|
3793
|
+
await a.openSession();
|
|
3794
|
+
const first = randomUUID(), second = randomUUID();
|
|
3795
|
+
f.down.add(`/${first}/events`);
|
|
3796
|
+
await assert.rejects(a.postTaskEvent(first, 'started', {}), error => error instanceof AgentFabricRequestError && error.status === 503);
|
|
3797
|
+
// 11 minutes later another client finds the entry due and leaves it to its owner: it could be sent only signed again.
|
|
3798
|
+
await sharedLater(11, async () => { const c = await f.open('relay:c'); await c.openSession(); });
|
|
3799
|
+
f.down.clear();
|
|
3800
|
+
// An earlier version reads the file: the owner's entry holds the first event.
|
|
3801
|
+
const snapshot = await f.read();
|
|
3802
|
+
// The owner's next call sends the first event, then signs the second and keeps it after a 503.
|
|
3803
|
+
f.down.add(`/${second}/events`);
|
|
3804
|
+
await assert.rejects(sharedLater(12, () => a.postTaskEvent(second, 'started', {})), error => error instanceof AgentFabricRequestError && error.status === 503);
|
|
3805
|
+
f.down.clear();
|
|
3806
|
+
const event = f.calls.at(-1);
|
|
3807
|
+
assert.ok(event.path.endsWith(`/${second}/events`));
|
|
3808
|
+
// The earlier version writes back the file it read: the owner's entry holds the first event again.
|
|
3809
|
+
await writeFile(f.statePath, JSON.stringify(snapshot));
|
|
3810
|
+
assert.equal(await sharedLater(12, () => sharedOutcome(a.postTaskEvent(second, 'started', {}))), 'accepted');
|
|
3811
|
+
const sent = f.calls.filter(call => call.path === event.path);
|
|
3812
|
+
assert.deepEqual(sent.map(call => [call.messageId, call.sequence, call.idempotencyKey]), [[event.messageId, event.sequence, event.messageId], [event.messageId, event.sequence, event.messageId]], 'the kept second event is sent again exactly as signed');
|
|
3813
|
+
assertNoSequenceSignedTwice(f.calls, event.sessionId);
|
|
3814
|
+
// A started event holds a task lease in this process; ending the task frees it, so later tests send orphans.
|
|
3815
|
+
await a.postTaskEvent(second, 'failed', {});
|
|
3816
|
+
});
|
|
3817
|
+
// Simplification: only the process that opened a session signs in it. Every other client, a restarted process with the
|
|
3818
|
+
// same owner key included, sends a kept request only exactly as signed. An owner keeps its session and its sequence in
|
|
3819
|
+
// memory whatever the file shows, and the copy for an earlier version holds only a request that is already due and is
|
|
3820
|
+
// not an action acknowledgement.
|
|
3821
|
+
const simplifiedDecision = '33333333-3333-4333-8333-333333333333';
|
|
3822
|
+
const simplifiedAcknowledgementRoute = `/agent-fabric/decisions/${simplifiedDecision}/enforcements`;
|
|
3823
|
+
test('an owner whose entry an older copy wrote back with a higher next sequence keeps its session and signs from memory (shared protocol state)', async () => {
|
|
3824
|
+
const f = await sharedMonotonicFixture('fabric-shared-raised-sequence-');
|
|
3825
|
+
const a = await f.open('relay:workspace-a');
|
|
3826
|
+
await a.openSession();
|
|
3827
|
+
f.failNext();
|
|
3828
|
+
await assert.rejects(a.registerWorkspace({ workspaceId: 'a' }), error => error instanceof AgentFabricRequestError && error.status === 503);
|
|
3829
|
+
const kept = f.calls.at(-1);
|
|
3830
|
+
// An earlier version sent the copy and wrote back the file it read, with the next sequence it reached.
|
|
3831
|
+
const state = await f.read();
|
|
3832
|
+
state.outboxes['relay:workspace-a'].nextSequence = kept.sequence + 1;
|
|
3833
|
+
await writeFile(f.statePath, JSON.stringify(state));
|
|
3834
|
+
assert.equal(await sharedOutcome(a.pollTask()), 'accepted', 'the owner keeps its session');
|
|
3835
|
+
const [resent, next] = f.calls.slice(-2);
|
|
3836
|
+
assert.deepEqual([resent.messageId, resent.sequence], [kept.messageId, kept.sequence], 'its kept request is sent exactly as signed');
|
|
3837
|
+
assert.deepEqual([next.path, next.sessionId, next.sequence], ['/agent-fabric/tasks/poll', kept.sessionId, kept.sequence + 1], 'its next call is signed at the next sequence it holds in memory');
|
|
3838
|
+
assertNoSequenceSignedTwice(f.calls, kept.sessionId);
|
|
3839
|
+
});
|
|
3840
|
+
test('no write puts an action acknowledgement in the copy for an earlier version (shared protocol state)', async () => {
|
|
3841
|
+
const f = await sharedMonotonicFixture('fabric-shared-no-acknowledgement-copy-');
|
|
3842
|
+
const a = await f.open(), b = await f.open();
|
|
3843
|
+
await a.openSession();
|
|
3844
|
+
await b.openSession();
|
|
3845
|
+
f.failNext();
|
|
3846
|
+
await assert.rejects(a.postActionEnforcement(simplifiedDecision, sharedAcknowledgement()), error => error instanceof AgentFabricRequestError && error.status === 503);
|
|
3847
|
+
const kept = f.calls.at(-1);
|
|
3848
|
+
const copies = [];
|
|
3849
|
+
const copy = async () => { copies.push((await f.read()).pending?.headers['x-dharma-message-id']); };
|
|
3850
|
+
await copy();
|
|
3851
|
+
await b.pollTask();
|
|
3852
|
+
await copy();
|
|
3853
|
+
await sharedLater(11, () => b.pollTask());
|
|
3854
|
+
await copy();
|
|
3855
|
+
// A command that exited left a kept acknowledgement too, which is due at once.
|
|
3856
|
+
const dead = await exitedPid();
|
|
3857
|
+
const exited = sharedOrphan(f.config.deviceId, dead, { route: simplifiedAcknowledgementRoute });
|
|
3858
|
+
exited.pending.body = JSON.stringify(sharedAcknowledgement());
|
|
3859
|
+
const state = await f.read();
|
|
3860
|
+
await writeSharedOutboxes(f.statePath, { ...state.outboxes, [`cli:${dead}:x:1`]: exited });
|
|
3861
|
+
await b.pollTask();
|
|
3862
|
+
await copy();
|
|
3863
|
+
assert.deepEqual(copies, [undefined, undefined, undefined, undefined], 'the top level never holds an action acknowledgement');
|
|
3864
|
+
assert.ok(keptMessageIds(await f.read()).includes(kept.messageId), 'the owner\'s entry still holds it');
|
|
3865
|
+
});
|
|
3866
|
+
test('another client leaves a workspace loop\'s stale acknowledgement or poll unsigned and in place, and settles an exited command\'s or a migrated one unsent (shared protocol state)', async () => {
|
|
3867
|
+
const f = await sharedMonotonicFixture('fabric-shared-adopter-owner-only-');
|
|
3868
|
+
const dead = await exitedPid();
|
|
3869
|
+
const stale = (ownerPid) => {
|
|
3870
|
+
const session = randomUUID();
|
|
3871
|
+
return { sessionId: session, nextSequence: 3, ownerPid, updatedAt: new Date(Date.now() - 20 * 60_000).toISOString(),
|
|
3872
|
+
pending: sharedEarlierAcknowledgement(f.config.deviceId, session, 2, -20, randomUUID()) };
|
|
3873
|
+
};
|
|
3874
|
+
const poll = (ownerPid) => sharedOrphan(f.config.deviceId, ownerPid, { route: '/agent-fabric/tasks/poll', minutesAgo: 20 });
|
|
3875
|
+
const left = { 'relay:loop-acknowledgement': stale(process.pid), 'relay:loop-poll': poll(process.pid) };
|
|
3876
|
+
const settled = { [`cli:${dead}:x:1`]: stale(dead), [`cli:${dead}:x:2`]: poll(dead), 'migrated:earlier-acknowledgement': stale(0) };
|
|
3877
|
+
await writeSharedOutboxes(f.statePath, { ...left, ...settled });
|
|
3878
|
+
const ids = [...Object.values(left), ...Object.values(settled)].map(entry => entry.pending.headers['x-dharma-message-id']);
|
|
3879
|
+
const adopter = await f.open('relay:adopter');
|
|
3880
|
+
await adopter.openSession();
|
|
3881
|
+
for (let call = 0; call < 3; call++) {
|
|
3882
|
+
await adopter.pollTask();
|
|
3883
|
+
await sharedSettled(f.config);
|
|
3884
|
+
}
|
|
3885
|
+
assert.equal(f.calls.filter(call => ids.includes(call.messageId)).length, 0, 'none is sent or signed again');
|
|
3886
|
+
const outboxes = (await f.read()).outboxes ?? {};
|
|
3887
|
+
for (const [owner, entry] of Object.entries(left)) {
|
|
3888
|
+
assert.deepEqual(outboxes[owner]?.pending, entry.pending, `${owner} keeps its request untouched for its owner`);
|
|
3889
|
+
assert.equal(outboxes[owner]?.adoptedAt, undefined, `${owner} is not marked`);
|
|
3890
|
+
}
|
|
3891
|
+
for (const owner of Object.keys(settled))
|
|
3892
|
+
assert.equal(outboxes[owner]?.pending ?? null, null, `${owner} is settled unsent`);
|
|
3893
|
+
});
|
|
3894
|
+
for (const answer of ['accepted', 'refused']) {
|
|
3895
|
+
test(`a restarted process with the same owner key re-issues an inherited stale acknowledgement only in its new session: ${answer} (shared protocol state)`, async () => {
|
|
3896
|
+
const f = await sharedMonotonicFixture(`fabric-shared-restart-acknowledgement-${answer}-`);
|
|
3897
|
+
const o = await f.open('relay:o');
|
|
3898
|
+
await o.openSession();
|
|
3899
|
+
const acknowledgement = sharedAcknowledgement();
|
|
3900
|
+
f.failNext();
|
|
3901
|
+
await assert.rejects(o.postActionEnforcement(simplifiedDecision, acknowledgement), error => error instanceof AgentFabricRequestError && error.status === 503);
|
|
3902
|
+
const kept = f.calls.at(-1);
|
|
3903
|
+
// The process exits. 9 minutes later, past the acknowledgement's time window, a new process opens with the same key.
|
|
3904
|
+
if (answer === 'refused')
|
|
3905
|
+
f.refuse.add('/enforcements');
|
|
3906
|
+
const fresh = await f.open('relay:o');
|
|
3907
|
+
assert.equal(await sharedLater(9, () => sharedOutcome(fresh.openSession())), 'accepted', 'its session opens');
|
|
3908
|
+
assert.equal(f.calls.filter(call => call.sessionId === kept.sessionId).length, 2, 'it signs nothing in the inherited session');
|
|
3909
|
+
const [opened, reissued] = f.calls.slice(-2);
|
|
3910
|
+
assert.equal(opened.path, '/agent-fabric/sessions');
|
|
3911
|
+
assert.notEqual(opened.sessionId, kept.sessionId);
|
|
3912
|
+
assert.deepEqual([reissued.path, reissued.sessionId, reissued.sequence, reissued.idempotencyKey], [kept.path, opened.sessionId, opened.sequence + 1, kept.idempotencyKey], 'after its session open it re-issues the acknowledgement there, at the next sequence, with the original idempotency key');
|
|
3913
|
+
assert.notEqual(reissued.messageId, kept.messageId);
|
|
3914
|
+
assert.ok(Date.parse(JSON.parse(reissued.body).acknowledgedAt) > Date.parse(acknowledgement.acknowledgedAt), 'with a refreshed acknowledgement time');
|
|
3915
|
+
assert.deepEqual(keptMessageIds(await f.read()), [], answer === 'accepted' ? 'the answered acknowledgement leaves the file'
|
|
3916
|
+
: 'a refusal settles it');
|
|
3917
|
+
assertNoSequenceSignedTwice(f.calls, kept.sessionId);
|
|
3918
|
+
assertNoSequenceSignedTwice(f.calls, opened.sessionId);
|
|
3919
|
+
});
|
|
3920
|
+
}
|
|
3921
|
+
// Owner review: each hold of signing control makes at most one network send, a hold an SDK caller takes itself around
|
|
3922
|
+
// a session open among them. The inherited acknowledgement is signed again only after that hold, in a hold of its own.
|
|
3923
|
+
for (const [kind, hold] of [
|
|
3924
|
+
['withDeviceSigningControl', withDeviceSigningControl],
|
|
3925
|
+
['withSigningControlPerSend', withSigningControlPerSend],
|
|
3926
|
+
]) {
|
|
3927
|
+
test(`a session open in a hold its caller took itself sends only the session open there, and re-issues an inherited stale acknowledgement after that hold: ${kind} (shared protocol state)`, async () => {
|
|
3928
|
+
const f = await sharedMonotonicFixture(`fabric-shared-restart-acknowledgement-caller-hold-${kind}-`);
|
|
3929
|
+
const o = await f.open('relay:o');
|
|
3930
|
+
await o.openSession();
|
|
3931
|
+
f.failNext();
|
|
3932
|
+
await assert.rejects(o.postActionEnforcement(simplifiedDecision, sharedAcknowledgement()), error => error instanceof AgentFabricRequestError && error.status === 503);
|
|
3933
|
+
const kept = f.calls.at(-1);
|
|
3934
|
+
// The process exits. 9 minutes later a new process with the same key opens its session in a hold it took itself.
|
|
3935
|
+
const fresh = await f.open('relay:o');
|
|
3936
|
+
const inHold = await sharedLater(9, async () => {
|
|
3937
|
+
const sent = await hold({ config: f.config }, async () => {
|
|
3938
|
+
const from = f.calls.length;
|
|
3939
|
+
await fresh.openSession();
|
|
3940
|
+
return f.calls.slice(from).map(call => call.path);
|
|
3941
|
+
});
|
|
3942
|
+
await sharedSettled(f.config);
|
|
3943
|
+
return sent;
|
|
3944
|
+
});
|
|
3945
|
+
assert.deepEqual(inHold, ['/agent-fabric/sessions'], 'only the session open is sent in the caller\'s hold');
|
|
3946
|
+
const [opened, reissued] = f.calls.slice(-2);
|
|
3947
|
+
assert.equal(opened.path, '/agent-fabric/sessions');
|
|
3948
|
+
assert.deepEqual([reissued.path, reissued.sessionId, reissued.sequence, reissued.idempotencyKey], [kept.path, opened.sessionId, opened.sequence + 1, kept.idempotencyKey], 'after that hold it re-issues the acknowledgement in its new session, at the next sequence, with the original idempotency key');
|
|
3949
|
+
assert.deepEqual(keptMessageIds(await f.read()), [], 'the answered acknowledgement leaves the file');
|
|
3950
|
+
for (const session of new Set(f.calls.map(call => call.sessionId)))
|
|
3951
|
+
assertNoSequenceSignedTwice(f.calls, session);
|
|
3952
|
+
});
|
|
3953
|
+
}
|
|
3954
|
+
// Owner review 3: a caller that holds signing control across a session open and another call on the same client. The
|
|
3955
|
+
// inherited acknowledgement keeps its place in the owner's entry until it has had a hold of its own, so an ordinary
|
|
3956
|
+
// call in between, whatever its outcome, never takes that place, and a later process with the owner key still
|
|
3957
|
+
// re-issues it with its original idempotency key. Every send runs in a hold of its own throughout.
|
|
3958
|
+
for (const [kind, hold] of [
|
|
3959
|
+
['withDeviceSigningControl', withDeviceSigningControl],
|
|
3960
|
+
['withSigningControlPerSend', withSigningControlPerSend],
|
|
3961
|
+
]) {
|
|
3962
|
+
test(`an ordinary call after a session open in a hold its caller took itself never takes an inherited stale acknowledgement's place, and a fresh client with the owner key re-issues it with its idempotency key: ${kind} (shared protocol state)`, async () => {
|
|
3963
|
+
const f = await sharedMonotonicFixture(`fabric-shared-acknowledgement-ordinary-call-${kind}-`);
|
|
3964
|
+
const o = await f.open('relay:o');
|
|
3965
|
+
await o.openSession();
|
|
3966
|
+
f.failNext();
|
|
3967
|
+
await assert.rejects(o.postActionEnforcement(simplifiedDecision, sharedAcknowledgement()), error => error instanceof AgentFabricRequestError && error.status === 503);
|
|
3968
|
+
const kept = f.calls.at(-1);
|
|
3969
|
+
// From here every send records the hold of signing control it runs in: the fetcher runs inside that hold, so a
|
|
3970
|
+
// nested withDeviceSigningControl reuses it and returns its lease.
|
|
3971
|
+
const holds = [];
|
|
3972
|
+
const fetcher = async (url, init) => {
|
|
3973
|
+
holds.push(await withDeviceSigningControl({ config: f.config }, async (lease) => lease.controllerInstanceId));
|
|
3974
|
+
return f.fetcher(url, init);
|
|
3975
|
+
};
|
|
3976
|
+
const open = () => AgentFabricClient.open(withOwner({ configPath: f.configPath, statePath: f.statePath, store: f.store,
|
|
3977
|
+
fetcher }, 'relay:o'));
|
|
3978
|
+
// The process exits. 9 minutes later a new process with the same key opens its session in a hold it took itself and
|
|
3979
|
+
// makes another call there. That call's outcome is unknown, and so is the outcome of every acknowledgement send.
|
|
3980
|
+
const fresh = await open();
|
|
3981
|
+
f.down.add('/agent-fabric/tasks/poll');
|
|
3982
|
+
f.down.add('/enforcements');
|
|
3983
|
+
const inHold = await sharedLater(9, async () => {
|
|
3984
|
+
const result = await hold({ config: f.config }, async () => {
|
|
3985
|
+
const from = f.calls.length;
|
|
3986
|
+
await fresh.openSession();
|
|
3987
|
+
const outcome = await fresh.pollTask().then(() => 'answered', (error) => error instanceof AgentFabricRequestError ? `status ${error.status}` : error.message);
|
|
3988
|
+
return { sent: f.calls.slice(from).map(call => call.path), outcome };
|
|
3989
|
+
});
|
|
3990
|
+
await sharedSettled(f.config);
|
|
3991
|
+
return result;
|
|
3992
|
+
});
|
|
3993
|
+
assert.deepEqual(inHold.sent, ['/agent-fabric/sessions'], 'only the session open is sent in the caller\'s hold');
|
|
3994
|
+
assert.equal(inHold.outcome, 'relay_retry_after_recovery', 'the other call is neither signed nor sent, and fails retryably');
|
|
3995
|
+
const entry = (await f.read()).outboxes?.['relay:o'];
|
|
3996
|
+
assert.deepEqual([entry?.pending?.pathname, entry?.pending?.headers['idempotency-key']], [`/api/v1/orgs/org_a${kept.path}`, kept.idempotencyKey], 'the owner\'s entry still holds the acknowledgement, with its original idempotency key');
|
|
3997
|
+
// The process exits again. Another new process with the same key, a fresh client object, recovers it.
|
|
3998
|
+
f.down.clear();
|
|
3999
|
+
const next = await open();
|
|
4000
|
+
const from = f.calls.length;
|
|
4001
|
+
assert.equal(await sharedLater(9, () => sharedOutcome(next.openSession())), 'accepted', 'its session opens');
|
|
4002
|
+
const recovered = f.calls.slice(from).filter(call => call.path === kept.path);
|
|
4003
|
+
assert.ok(recovered.length >= 1, 'the fresh client sends the acknowledgement');
|
|
4004
|
+
assert.deepEqual(recovered.map(call => call.idempotencyKey), recovered.map(() => kept.idempotencyKey), 'with the original idempotency key');
|
|
4005
|
+
assert.deepEqual(keptMessageIds(await f.read()), [], 'the answered acknowledgement leaves the file');
|
|
4006
|
+
assert.equal(new Set(holds).size, holds.length, 'every send runs in a hold of its own, the hand-off included');
|
|
4007
|
+
for (const session of new Set(f.calls.map(call => call.sessionId)))
|
|
4008
|
+
assertNoSequenceSignedTwice(f.calls, session);
|
|
4009
|
+
});
|
|
4010
|
+
}
|
|
4011
|
+
test('an inherited stale acknowledgement stays in the owner\'s entry while its new session opens, so a failed session open leaves it for the next process (shared protocol state)', async () => {
|
|
4012
|
+
const f = await sharedMonotonicFixture('fabric-shared-restart-acknowledgement-open-fails-');
|
|
4013
|
+
const o = await f.open('relay:o');
|
|
4014
|
+
await o.openSession();
|
|
4015
|
+
f.failNext();
|
|
4016
|
+
await assert.rejects(o.postActionEnforcement(simplifiedDecision, sharedAcknowledgement()), error => error instanceof AgentFabricRequestError && error.status === 503);
|
|
4017
|
+
const kept = f.calls.at(-1);
|
|
4018
|
+
// 9 minutes later a restarted process with the same key takes it over, and its session open gets no definite answer.
|
|
4019
|
+
f.down.add('/agent-fabric/sessions');
|
|
4020
|
+
const restarted = await f.open('relay:o');
|
|
4021
|
+
await assert.rejects(sharedLater(9, () => restarted.openSession()), error => error instanceof AgentFabricRequestError && error.status === 503);
|
|
4022
|
+
f.down.clear();
|
|
4023
|
+
const held = Object.values((await f.read()).outboxes ?? {}).flatMap(entry => entry.pending ? [entry.pending.pathname] : []);
|
|
4024
|
+
assert.deepEqual(held, [`/api/v1/orgs/org_a${kept.path}`], 'the owner\'s entry still holds the acknowledgement');
|
|
4025
|
+
// The outage ends and the next process with the same key re-issues it in its own new session.
|
|
4026
|
+
const next = await f.open('relay:o');
|
|
4027
|
+
assert.equal(await sharedLater(10, () => sharedOutcome(next.openSession())), 'accepted');
|
|
4028
|
+
const [opened, reissued] = f.calls.slice(-2);
|
|
4029
|
+
assert.equal(opened.path, '/agent-fabric/sessions');
|
|
4030
|
+
assert.deepEqual([reissued.path, reissued.sessionId, reissued.sequence, reissued.idempotencyKey], [kept.path, opened.sessionId, opened.sequence + 1, kept.idempotencyKey], 'it is sent once more, in the new session, with the original idempotency key');
|
|
4031
|
+
assert.deepEqual(keptMessageIds(await f.read()), [], 'the answered acknowledgement leaves the file');
|
|
4032
|
+
for (const session of new Set(f.calls.map(call => call.sessionId)))
|
|
4033
|
+
assertNoSequenceSignedTwice(f.calls, session);
|
|
4034
|
+
});
|
|
4035
|
+
test('a client whose session open the server refused signs no other call while a taken-over acknowledgement waits for its new session (shared protocol state)', async () => {
|
|
4036
|
+
const f = await sharedMonotonicFixture('fabric-shared-restart-acknowledgement-open-refused-');
|
|
4037
|
+
const o = await f.open('relay:o');
|
|
4038
|
+
await o.openSession();
|
|
4039
|
+
f.failNext();
|
|
4040
|
+
await assert.rejects(o.postActionEnforcement(simplifiedDecision, sharedAcknowledgement()), error => error instanceof AgentFabricRequestError && error.status === 503);
|
|
4041
|
+
const kept = f.calls.at(-1);
|
|
4042
|
+
// 9 minutes later a restarted process with the same key takes it over, and the server refuses its session open.
|
|
4043
|
+
f.refuse.add('/agent-fabric/sessions');
|
|
4044
|
+
const restarted = await f.open('relay:o');
|
|
4045
|
+
await assert.rejects(sharedLater(9, () => restarted.openSession()), error => error instanceof AgentFabricRequestError && error.status === 409);
|
|
4046
|
+
f.refuse.clear();
|
|
4047
|
+
const sent = f.calls.length;
|
|
4048
|
+
f.down.add('/agent-fabric/tasks/poll');
|
|
4049
|
+
assert.equal(await sharedLater(9, () => sharedOutcome(restarted.pollTask())), 'Relay session is not open.');
|
|
4050
|
+
f.down.clear();
|
|
4051
|
+
assert.equal(f.calls.length, sent, 'no call is signed or sent in the session the server refused');
|
|
4052
|
+
const held = Object.values((await f.read()).outboxes ?? {}).flatMap(entry => entry.pending ? [entry.pending.pathname] : []);
|
|
4053
|
+
assert.deepEqual(held, [`/api/v1/orgs/org_a${kept.path}`], 'the owner\'s entry still holds the acknowledgement');
|
|
4054
|
+
// Its next session open re-issues it in the new session.
|
|
4055
|
+
assert.equal(await sharedLater(10, () => sharedOutcome(restarted.openSession())), 'accepted');
|
|
4056
|
+
const [opened, reissued] = f.calls.slice(-2);
|
|
4057
|
+
assert.equal(opened.path, '/agent-fabric/sessions');
|
|
4058
|
+
assert.deepEqual([reissued.path, reissued.sessionId, reissued.sequence, reissued.idempotencyKey], [kept.path, opened.sessionId, opened.sequence + 1, kept.idempotencyKey], 'it is sent once more, in the new session, with the original idempotency key');
|
|
4059
|
+
assert.equal(await sharedLater(10, () => sharedOutcome(restarted.pollTask())), 'accepted', 'the opened session signs calls');
|
|
4060
|
+
assert.deepEqual(keptMessageIds(await f.read()), [], 'the answered acknowledgement leaves the file');
|
|
4061
|
+
for (const session of new Set(f.calls.map(call => call.sessionId)))
|
|
4062
|
+
assertNoSequenceSignedTwice(f.calls, session);
|
|
4063
|
+
});
|
|
4064
|
+
test('a workspace loop sends a kept request that an exited command left after the loop opened its session, after the loop\'s next request (shared protocol state)', async () => {
|
|
4065
|
+
const f = await sharedStateFixture('fabric-shared-orphan-after-open-', () => sharedOk());
|
|
4066
|
+
const loop = await f.open('relay:workspace-loop');
|
|
4067
|
+
await loop.openSession();
|
|
4068
|
+
// A minute later a command that exited during an outage left its kept request in its own entry.
|
|
4069
|
+
const orphan = sharedOrphan(f.config.deviceId, await exitedPid());
|
|
4070
|
+
await writeSharedOutboxes(f.statePath, { 'cli:1:orphan:1': orphan });
|
|
4071
|
+
await sharedLater(1, async () => { await loop.pollTask(); await sharedSettled(f.config); });
|
|
4072
|
+
assert.equal(sentTimes(f.calls, orphanMessageId(orphan)), 1, 'the loop sends it without waiting 10 minutes');
|
|
4073
|
+
assert.equal(f.calls.at(-2).path, '/agent-fabric/tasks/poll', 'after the loop\'s own request');
|
|
4074
|
+
assert.deepEqual(keptMessageIds(await f.read()), []);
|
|
4075
|
+
});
|
|
4076
|
+
test('five answers another client kept survive 35 newer idle entries, and their owners take them from the file instead of sending again (shared protocol state)', async () => {
|
|
4077
|
+
let down = false;
|
|
4078
|
+
const f = await sharedStateFixture('fabric-shared-adopted-survive-', call => down && call.path === '/agent-fabric/workspaces' ? sharedUnavailable() : sharedOk({ workspace: { id: 'registered' } }));
|
|
4079
|
+
const owners = [];
|
|
4080
|
+
for (let index = 0; index < 5; index++) {
|
|
4081
|
+
owners.push(await f.open());
|
|
4082
|
+
await owners.at(-1).openSession();
|
|
4083
|
+
}
|
|
4084
|
+
const b = await f.open();
|
|
4085
|
+
await b.openSession();
|
|
4086
|
+
down = true;
|
|
4087
|
+
const kept = [];
|
|
4088
|
+
for (const owner of owners) {
|
|
4089
|
+
await assert.rejects(owner.registerWorkspace({ workspaceId: 'a' }), error => error instanceof AgentFabricRequestError && error.status === 503);
|
|
4090
|
+
kept.push(f.calls.at(-1));
|
|
4091
|
+
}
|
|
4092
|
+
down = false;
|
|
4093
|
+
// Their entries stood unchanged for 10 minutes; five session opens send one each and keep each answer.
|
|
4094
|
+
const state = await f.read();
|
|
4095
|
+
for (const entry of Object.values(state.outboxes ?? {})) {
|
|
4096
|
+
if (entry.pending)
|
|
4097
|
+
entry.updatedAt = new Date(Date.now() - 11 * 60_000).toISOString();
|
|
4098
|
+
}
|
|
4099
|
+
await writeFile(f.statePath, JSON.stringify(state));
|
|
4100
|
+
for (let index = 0; index < 5; index++)
|
|
4101
|
+
await (await f.open()).openSession();
|
|
4102
|
+
assert.deepEqual(kept.map(call => sentTimes(f.calls, call.messageId)), [2, 2, 2, 2, 2]);
|
|
4103
|
+
const filled = await f.read();
|
|
4104
|
+
const newer = new Date(Date.now() + 60 * 60_000).toISOString();
|
|
4105
|
+
for (let index = 0; index < 35; index++) {
|
|
4106
|
+
filled.outboxes[`cli:${process.pid}:idle:${index}`] = { sessionId: randomUUID(), nextSequence: 2, pending: null,
|
|
4107
|
+
ownerPid: process.pid, updatedAt: newer };
|
|
4108
|
+
}
|
|
4109
|
+
await writeFile(f.statePath, JSON.stringify(filled));
|
|
4110
|
+
await b.pollTask();
|
|
4111
|
+
const answered = Object.values((await f.read()).outboxes ?? {}).filter(entry => entry.adoptedOutcome !== undefined);
|
|
4112
|
+
assert.equal(answered.length, 5, 'all five answers stay in the file');
|
|
4113
|
+
for (const owner of owners) {
|
|
4114
|
+
assert.deepEqual(await owner.registerWorkspace({ workspaceId: 'a' }), { ok: true, workspace: { id: 'registered' } });
|
|
4115
|
+
}
|
|
4116
|
+
assert.deepEqual(kept.map(call => sentTimes(f.calls, call.messageId)), [2, 2, 2, 2, 2], 'no owner sends its request again');
|
|
4117
|
+
assert.equal(f.calls.filter(call => call.path === '/agent-fabric/workspaces').length, 10, 'nor signs the call again');
|
|
4118
|
+
});
|
|
4119
|
+
test('300 one-shot clients in a row never fill the file (shared protocol state)', async () => {
|
|
4120
|
+
const f = await sharedStateFixture('fabric-shared-one-shots-', () => sharedOk());
|
|
4121
|
+
let most = 0;
|
|
4122
|
+
for (let index = 0; index < 300; index++) {
|
|
4123
|
+
await (await f.open()).openSession();
|
|
4124
|
+
most = Math.max(most, Object.keys((await f.read()).outboxes ?? {}).length);
|
|
4125
|
+
}
|
|
4126
|
+
assert.equal(most, 1, 'each write keeps only its writer\'s entry');
|
|
4127
|
+
});
|
|
4128
|
+
test('the relay client that v0.2.225 ships sends nothing in this version\'s sessions except exactly as signed, and opens every file this version writes (shared protocol state)', async () => {
|
|
4129
|
+
const f = await sharedMonotonicFixture('fabric-shared-earlier-release-');
|
|
4130
|
+
const earlierCalls = new Set();
|
|
4131
|
+
const earlierFetcher = async (url, init) => {
|
|
4132
|
+
const answer = f.fetcher(url, init);
|
|
4133
|
+
earlierCalls.add(f.calls.at(-1));
|
|
4134
|
+
return answer;
|
|
4135
|
+
};
|
|
4136
|
+
const openEarlier = () => EarlierReleaseClient.open({ configPath: f.configPath, statePath: f.statePath, store: f.store,
|
|
4137
|
+
fetcher: earlierFetcher });
|
|
4138
|
+
/** The earlier version validates the file when it opens it (I5). */
|
|
4139
|
+
const opens = (step) => assert.doesNotReject(openEarlier(), step);
|
|
4140
|
+
const a = await f.open('relay:a'), b = await f.open('relay:b'), c = await f.open();
|
|
4141
|
+
await a.openSession();
|
|
4142
|
+
await b.openSession();
|
|
4143
|
+
await c.openSession();
|
|
4144
|
+
f.failNext();
|
|
4145
|
+
await assert.rejects(b.postActionEnforcement(simplifiedDecision, sharedAcknowledgement()), error => error instanceof AgentFabricRequestError && error.status === 503);
|
|
4146
|
+
await sharedDelay(5);
|
|
4147
|
+
f.failNext();
|
|
4148
|
+
await assert.rejects(a.registerWorkspace({ workspaceId: 'a' }), error => error instanceof AgentFabricRequestError && error.status === 503);
|
|
4149
|
+
const kept = f.calls.at(-1);
|
|
4150
|
+
assert.equal((await f.read()).pending, null, 'neither a fresh request nor an acknowledgement is copied');
|
|
4151
|
+
await opens('after both owners kept a request');
|
|
4152
|
+
// The workspace owner's entry stood unchanged for 10 minutes; the next write copies that due request. The
|
|
4153
|
+
// acknowledgement owner's entry is not due, so the workspace loop does not send it after its own request.
|
|
4154
|
+
const state = await f.read();
|
|
4155
|
+
state.outboxes['relay:a'].updatedAt = new Date(Date.now() - 11 * 60_000).toISOString();
|
|
4156
|
+
await writeFile(f.statePath, JSON.stringify(state));
|
|
4157
|
+
await c.pollTask();
|
|
4158
|
+
assert.equal((await f.read()).pending?.headers['x-dharma-message-id'], kept.messageId, 'only the due request is copied');
|
|
4159
|
+
await opens('after the copy');
|
|
4160
|
+
// The earlier version opens and reads the file, then the owner and the earlier version both act on it.
|
|
4161
|
+
const earlier = await openEarlier();
|
|
4162
|
+
await a.pollTask();
|
|
4163
|
+
await opens('after the owner sent its own request');
|
|
4164
|
+
// Its first session open replays the copy it read and writes back the file it read; the next one writes a file
|
|
4165
|
+
// without outboxes and opens its own session.
|
|
4166
|
+
await sharedLater(9, () => earlier.openSession()).catch(() => undefined);
|
|
4167
|
+
await sharedLater(9, () => earlier.openSession());
|
|
4168
|
+
await earlier.pollTask();
|
|
4169
|
+
// Both owners keep their sessions: each sends its kept request only exactly as signed and signs from memory.
|
|
4170
|
+
assert.equal(await sharedOutcome(b.pollTask()), 'accepted');
|
|
4171
|
+
await opens('after the acknowledgement owner wrote');
|
|
4172
|
+
assert.equal(await sharedOutcome(a.pollTask()), 'accepted');
|
|
4173
|
+
await opens('after the workspace owner wrote');
|
|
4174
|
+
assert.ok([...earlierCalls].some(call => call.messageId === kept.messageId), 'the earlier version replayed the copy');
|
|
4175
|
+
const earlierSessions = new Set([...earlierCalls].filter(call => call.path === '/agent-fabric/sessions').map(call => call.sessionId));
|
|
4176
|
+
for (const call of earlierCalls) {
|
|
4177
|
+
if (earlierSessions.has(call.sessionId))
|
|
4178
|
+
continue;
|
|
4179
|
+
const first = f.calls.find(other => other.messageId === call.messageId);
|
|
4180
|
+
assert.ok(!earlierCalls.has(first) && first.sequence === call.sequence && first.sessionId === call.sessionId, 'the earlier version sends in this version\'s sessions only exactly as signed');
|
|
4181
|
+
}
|
|
4182
|
+
for (const session of new Set(f.calls.map(call => call.sessionId)))
|
|
4183
|
+
assertNoSequenceSignedTwice(f.calls, session);
|
|
4184
|
+
});
|
|
4185
|
+
test('two processes with the same workspace owner key never sign in each other\'s sessions (shared protocol state)', async () => {
|
|
4186
|
+
const f = await sharedMonotonicFixture('fabric-shared-same-owner-key-');
|
|
4187
|
+
const signer = new Map(), opener = new Map();
|
|
4188
|
+
const as = async (name, work) => {
|
|
4189
|
+
const from = f.calls.length;
|
|
4190
|
+
await sharedOutcome(work());
|
|
4191
|
+
for (const call of f.calls.slice(from)) {
|
|
4192
|
+
if (!signer.has(call.messageId))
|
|
4193
|
+
signer.set(call.messageId, name);
|
|
4194
|
+
if (call.path === '/agent-fabric/sessions' && !opener.has(call.sessionId))
|
|
4195
|
+
opener.set(call.sessionId, name);
|
|
4196
|
+
}
|
|
4197
|
+
};
|
|
4198
|
+
const first = await f.open('relay:w'), second = await f.open('relay:w');
|
|
4199
|
+
await as('first', () => first.openSession());
|
|
4200
|
+
await as('second', () => second.openSession());
|
|
4201
|
+
f.failNext();
|
|
4202
|
+
await as('first', () => first.registerWorkspace({ workspaceId: 'w' }));
|
|
4203
|
+
await as('second', () => second.pollTask());
|
|
4204
|
+
f.failNext();
|
|
4205
|
+
await as('second', () => second.postActionEnforcement(simplifiedDecision, sharedAcknowledgement()));
|
|
4206
|
+
await sharedLater(9, () => as('first', () => first.pollTask()));
|
|
4207
|
+
await sharedLater(9, () => as('second', () => second.pollTask()));
|
|
4208
|
+
const third = await f.open('relay:w');
|
|
4209
|
+
await sharedLater(9, () => as('third', () => third.openSession()));
|
|
4210
|
+
await as('first', () => first.pollTask());
|
|
4211
|
+
for (const call of f.calls) {
|
|
4212
|
+
assert.equal(opener.get(call.sessionId), signer.get(call.messageId), 'each request is signed in a session its process opened');
|
|
4213
|
+
}
|
|
4214
|
+
for (const session of opener.keys())
|
|
4215
|
+
assertNoSequenceSignedTwice(f.calls, session);
|
|
4216
|
+
});
|
|
4217
|
+
// Owner re-review: another client sends a kept request only byte for byte as signed, and leaves one it could send only
|
|
4218
|
+
// signed again for its owner; and each hold of signing control makes at most one network send.
|
|
4219
|
+
/** Every header and the body of a send, in a fixed order, for a byte-for-byte comparison. */
|
|
4220
|
+
const sharedSignedEnvelope = (call) => JSON.stringify([call.path, call.body,
|
|
4221
|
+
Object.entries(call.headers).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))]);
|
|
4222
|
+
test('another client sends an orphan byte for byte as signed, and an orphan without a definite answer keeps its signed request and no outcome (shared protocol state)', async () => {
|
|
4223
|
+
const f = await sharedStateFixture('fabric-shared-orphan-bytes-', call => call.path === '/agent-fabric/workspaces' ? sharedUnavailable() : sharedOk());
|
|
4224
|
+
const command = await f.open();
|
|
4225
|
+
await command.openSession();
|
|
4226
|
+
await assert.rejects(command.registerWorkspace({ workspaceId: 'w' }), error => error instanceof AgentFabricRequestError && error.status === 503);
|
|
4227
|
+
const original = f.calls.at(-1);
|
|
4228
|
+
// The command exits: its kept request is due at once, and its signature is still recent enough to send as signed.
|
|
4229
|
+
const dead = await exitedPid();
|
|
4230
|
+
const state = await f.read();
|
|
4231
|
+
const [key, entry] = Object.entries(state.outboxes ?? {}).find(([, item]) => item.pending) ?? [];
|
|
4232
|
+
assert.ok(key && entry, 'the command kept its request in its own entry');
|
|
4233
|
+
delete state.outboxes[key];
|
|
4234
|
+
state.outboxes[`cli:${dead}:1`] = { ...entry, ownerPid: dead };
|
|
4235
|
+
await writeFile(f.statePath, JSON.stringify(state));
|
|
4236
|
+
const kept = structuredClone(entry.pending);
|
|
4237
|
+
const adopter = await f.open();
|
|
4238
|
+
await adopter.openSession();
|
|
4239
|
+
const sends = f.calls.filter(call => call.messageId === original.messageId);
|
|
4240
|
+
assert.equal(sends.length, 2, 'the adopter sends it once');
|
|
4241
|
+
assert.equal(sharedSignedEnvelope(sends[1]), sharedSignedEnvelope(original), 'every signed header (session, sequence, message id, idempotency key, time, nonce, signature) and the body are byte-identical');
|
|
4242
|
+
const after = (await f.read()).outboxes?.[`cli:${dead}:1`];
|
|
4243
|
+
assert.deepEqual(after?.pending, kept, 'without a definite answer the entry keeps the request exactly as signed');
|
|
4244
|
+
assert.equal(after?.adoptedOutcome, undefined, 'and records no outcome for it');
|
|
4245
|
+
});
|
|
4246
|
+
test('a kept request too old to send as signed is left for its owner, neither sent nor signed again by another client, with its outcome unknown (shared protocol state)', async () => {
|
|
4247
|
+
let down = true;
|
|
4248
|
+
const f = await sharedStateFixture('fabric-shared-orphan-stale-bytes-', call => down && call.path === '/agent-fabric/workspaces' ? sharedUnavailable() : sharedOk());
|
|
4249
|
+
const owner = await f.open('relay:workspace-a');
|
|
4250
|
+
await owner.openSession();
|
|
4251
|
+
await assert.rejects(owner.registerWorkspace({ workspaceId: 'a' }), error => error instanceof AgentFabricRequestError && error.status === 503);
|
|
4252
|
+
const original = f.calls.at(-1);
|
|
4253
|
+
const kept = structuredClone((await f.read()).outboxes['relay:workspace-a'].pending);
|
|
4254
|
+
down = false;
|
|
4255
|
+
// 11 minutes later the owner's entry stood unchanged for 10 minutes, and its signature is older than the server
|
|
4256
|
+
// accepts: it could be sent only signed again, and only the process that opened its session signs there.
|
|
4257
|
+
await sharedLater(11, async () => {
|
|
4258
|
+
const loop = await f.open('relay:workspace-b');
|
|
4259
|
+
await loop.openSession();
|
|
4260
|
+
await loop.pollTask();
|
|
4261
|
+
await sharedSettled(f.config);
|
|
4262
|
+
await (await f.open()).openSession();
|
|
4263
|
+
});
|
|
4264
|
+
assert.equal(sentTimes(f.calls, original.messageId), 1, 'no other client sends it, as signed or signed again');
|
|
4265
|
+
const entry = (await f.read()).outboxes?.['relay:workspace-a'];
|
|
4266
|
+
assert.deepEqual(entry?.pending, kept, 'the owner\'s entry keeps every signed header and the body');
|
|
4267
|
+
assert.deepEqual([entry?.adoptedAt, entry?.adoptedOutcome], [undefined, undefined], 'and no attempt or outcome');
|
|
4268
|
+
// The owner sends it itself, with the same message id, session, sequence, body and idempotency key.
|
|
4269
|
+
await sharedLater(11, () => owner.pollTask());
|
|
4270
|
+
const sends = f.calls.filter(call => call.messageId === original.messageId);
|
|
4271
|
+
assert.equal(sends.length, 2);
|
|
4272
|
+
assert.deepEqual([sends[1].sessionId, sends[1].sequence, sends[1].body, sends[1].idempotencyKey], [original.sessionId, original.sequence, original.body, original.idempotencyKey]);
|
|
4273
|
+
assert.deepEqual(keptMessageIds(await f.read()), []);
|
|
4274
|
+
});
|
|
4275
|
+
/** Another client asks for signing control at each send, from outside any hold, and each send records how many such
|
|
4276
|
+
* askers from earlier sends still wait. A send that finds one waiting shares its hold with an earlier send. */
|
|
4277
|
+
function sharedWaitingClient(config) {
|
|
4278
|
+
const waiting = new Set();
|
|
4279
|
+
const behind = [];
|
|
4280
|
+
let asked = 0, stopped = false, wake = () => undefined;
|
|
4281
|
+
const asking = (async () => {
|
|
4282
|
+
for (;;) {
|
|
4283
|
+
await new Promise(accept => { wake = accept; });
|
|
4284
|
+
if (stopped)
|
|
4285
|
+
return;
|
|
4286
|
+
const id = ++asked;
|
|
4287
|
+
waiting.add(id);
|
|
4288
|
+
void withDeviceSigningControl({ config }, async () => { waiting.delete(id); });
|
|
4289
|
+
}
|
|
4290
|
+
})();
|
|
4291
|
+
return {
|
|
4292
|
+
behind,
|
|
4293
|
+
/** Called by the server at each send, before it answers. */
|
|
4294
|
+
async send() { behind.push(waiting.size); wake(); await sharedDelay(25); },
|
|
4295
|
+
async stop() { stopped = true; wake(); await asking; await sharedSettled(config); },
|
|
4296
|
+
};
|
|
4297
|
+
}
|
|
4298
|
+
test('each hold of signing control makes at most one network send while another client waits: own recovery, orphan adoption and a full file (shared protocol state)', async () => {
|
|
4299
|
+
let down = false;
|
|
4300
|
+
let other;
|
|
4301
|
+
const f = await sharedStateFixture('fabric-shared-one-send-per-hold-', async (call) => {
|
|
4302
|
+
await other?.send();
|
|
4303
|
+
return down && call.path === '/agent-fabric/workspaces' ? sharedUnavailable() : sharedOk();
|
|
4304
|
+
});
|
|
4305
|
+
const waiter = sharedWaitingClient(f.config);
|
|
4306
|
+
other = waiter;
|
|
4307
|
+
const scenario = async (name, sends, work) => {
|
|
4308
|
+
const from = waiter.behind.length;
|
|
4309
|
+
await work();
|
|
4310
|
+
await sharedSettled(f.config);
|
|
4311
|
+
const behind = waiter.behind.slice(from);
|
|
4312
|
+
assert.equal(behind.length, sends, `${name}: ${sends} sends`);
|
|
4313
|
+
assert.deepEqual(behind, behind.map(() => 0), `${name}: no send shares its hold with an earlier one`);
|
|
4314
|
+
};
|
|
4315
|
+
try {
|
|
4316
|
+
// Own recovery: the client's earlier kept call, then the caller's call.
|
|
4317
|
+
const client = await f.open();
|
|
4318
|
+
await client.openSession();
|
|
4319
|
+
down = true;
|
|
4320
|
+
await assert.rejects(client.registerWorkspace({ workspaceId: 'w' }), error => error instanceof AgentFabricRequestError && error.status === 503);
|
|
4321
|
+
down = false;
|
|
4322
|
+
await scenario('own recovery', 2, () => client.pollTask());
|
|
4323
|
+
// Orphan adoption: a session open sends a due orphan, then the session open.
|
|
4324
|
+
const dead = await exitedPid();
|
|
4325
|
+
const orphan = sharedOrphan(f.config.deviceId, dead);
|
|
4326
|
+
await writeSharedOutboxes(f.statePath, { ...(await f.read()).outboxes, [`cli:${dead}:x:0`]: orphan });
|
|
4327
|
+
await scenario('orphan adoption', 2, async () => (await f.open()).openSession());
|
|
4328
|
+
// A full file: a due orphan is sent to make room, then the caller's call.
|
|
4329
|
+
const orphans = Array.from({ length: 256 }, () => sharedOrphan(f.config.deviceId, dead, { adoptedMinutesAgo: 1 }));
|
|
4330
|
+
await writeSharedOutboxes(f.statePath, Object.fromEntries(orphans.map((entry, index) => [`cli:${dead}:y:${index}`, entry])));
|
|
4331
|
+
const full = await f.open();
|
|
4332
|
+
await scenario('session open in a full file', 1, () => full.openSession());
|
|
4333
|
+
// Commands that exited just now left the same requests: due at once and recent enough to send as signed.
|
|
4334
|
+
const due = await f.read();
|
|
4335
|
+
for (const entry of Object.values(due.outboxes ?? {}))
|
|
4336
|
+
delete entry.adoptedAt;
|
|
4337
|
+
await writeFile(f.statePath, JSON.stringify(due));
|
|
4338
|
+
await scenario('full file', 2, () => full.registerWorkspace({ workspaceId: 'w' }));
|
|
4339
|
+
assert.equal(keptMessageIds(await f.read()).length, 255, 'one orphan made room and the call was answered');
|
|
4340
|
+
}
|
|
4341
|
+
finally {
|
|
4342
|
+
await waiter.stop();
|
|
4343
|
+
}
|
|
4344
|
+
});
|
|
4345
|
+
// Owner review: a 2xx answer whose JSON body is null (or false, 0, "") is the caller's answer. It is never read as a
|
|
4346
|
+
// hold that ended after another kept request was sent first, so the call is signed and sent once, directly and in a
|
|
4347
|
+
// hold of the admission line.
|
|
4348
|
+
test('a 2xx answer with a falsy JSON body is the call\'s answer: one signed request and one network send, directly and per send (shared protocol state)', async () => {
|
|
4349
|
+
for (const [answer, value] of [['null', null], ['false', false], ['0', 0], ['""', '']]) {
|
|
4350
|
+
for (const perSend of [false, true]) {
|
|
4351
|
+
const name = `${answer}${perSend ? ' per send' : ''}`;
|
|
4352
|
+
let sends = 0;
|
|
4353
|
+
const f = await sharedStateFixture('fabric-shared-falsy-answer-', call => {
|
|
4354
|
+
if (call.path !== '/agent-fabric/workspaces')
|
|
4355
|
+
return sharedOk();
|
|
4356
|
+
// The cap stands in for a loop that would otherwise sign and send the same call again without end.
|
|
4357
|
+
if (++sends > 3)
|
|
4358
|
+
throw new Error('send cap reached');
|
|
4359
|
+
return new Response(answer, { status: 200 });
|
|
4360
|
+
});
|
|
4361
|
+
const client = await f.open();
|
|
4362
|
+
await client.openSession();
|
|
4363
|
+
let settled = 0;
|
|
4364
|
+
const call = () => client.signedPost('/agent-fabric/workspaces', { workspaceId: 'w' });
|
|
4365
|
+
const outcome = await (perSend ? withSigningControlPerSend({ config: client.config }, call) : call())
|
|
4366
|
+
.then(result => ({ value: result }), error => ({ error: error.message }))
|
|
4367
|
+
.finally(() => { settled += 1; });
|
|
4368
|
+
const workspaceSends = f.calls.filter(item => item.path === '/agent-fabric/workspaces');
|
|
4369
|
+
assert.equal(workspaceSends.length, 1, `${name}: the call is sent once`);
|
|
4370
|
+
assert.equal(new Set(workspaceSends.flatMap(item => [item.messageId, item.idempotencyKey])).size, 1, `${name}: under one message id and idempotency key`);
|
|
4371
|
+
assert.deepEqual(outcome, { value }, `${name}: the caller gets that answer`);
|
|
4372
|
+
assert.equal(settled, 1, `${name}: once`);
|
|
4373
|
+
assert.deepEqual(keptMessageIds(await f.read()), [], `${name}: and nothing stays kept to send again`);
|
|
4374
|
+
// The next call is signed in the same session at the next sequence, and sent once.
|
|
4375
|
+
await client.pollTask();
|
|
4376
|
+
const poll = f.calls.at(-1);
|
|
4377
|
+
assert.deepEqual([poll.path, poll.sessionId, poll.sequence], ['/agent-fabric/tasks/poll', workspaceSends[0].sessionId, workspaceSends[0].sequence + 1], name);
|
|
4378
|
+
await sharedSettled(f.config);
|
|
4379
|
+
}
|
|
4380
|
+
}
|
|
4381
|
+
});
|
|
1816
4382
|
//# sourceMappingURL=index.test.js.map
|