@cydm/happy-elves 0.1.0-beta.487 → 0.1.0-beta.488
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/apps/cli/package.json +1 -1
- package/apps/daemon/dist/orchestration/handler.d.ts +5 -1
- package/apps/daemon/dist/orchestration/handler.js +52 -1
- package/apps/daemon/dist/relay/connection.js +3 -1
- package/apps/daemon/dist/relay/remote-command.js +30 -3
- package/apps/daemon/dist/relay/send.d.ts +1 -0
- package/apps/daemon/dist/relay/send.js +229 -20
- package/apps/daemon/dist/runtime/headless-event-projector.d.ts +6 -1
- package/apps/daemon/dist/runtime/headless-event-projector.js +33 -7
- package/apps/daemon/dist/session/he-official/common.d.ts +1 -0
- package/apps/daemon/dist/session/he-official/orchestration-commands.d.ts +3 -2
- package/apps/daemon/dist/session/he-official/orchestration-commands.js +53 -6
- package/apps/daemon/dist/session/he-official/session-commands.d.ts +2 -2
- package/apps/daemon/dist/session/he-official/session-commands.js +197 -6
- package/apps/daemon/dist/session/he-official/turn-commands.d.ts +12 -3
- package/apps/daemon/dist/session/he-official/turn-commands.js +78 -3
- package/apps/daemon/dist/session/he-official-tools.js +11 -6
- package/apps/daemon/dist/session/lifecycle.js +37 -1
- package/apps/daemon/package.json +1 -1
- package/apps/relay/dist/relay-context.js +47 -3
- package/apps/relay/dist/websocket.js +37 -1
- package/package.json +1 -1
- package/packages/shared/dist/he-official-tools.js +5 -5
- package/packages/shared/dist/protocol-schemas.d.ts +30 -9
- package/packages/shared/dist/protocol-schemas.js +11 -0
- package/packages/shared/dist/protocol-types.d.ts +17 -0
- package/packages/shared/dist/protocol.d.ts +3 -1
- package/packages/shared/dist/protocol.js +3 -1
package/apps/cli/package.json
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type DispatchTaskProjection, type MachineCommand, type RuntimeTurnReadSummary } from "../../../../packages/shared/dist/index.js";
|
|
1
|
+
import { type DispatchTaskProjection, type LoadedSessionInventory, type MachineCommand, type RuntimeTurnReadSummary } from "../../../../packages/shared/dist/index.js";
|
|
2
2
|
import type { DaemonConfig } from "../types.js";
|
|
3
3
|
import { type StoredDispatchAdmissionRecord } from "./store.js";
|
|
4
4
|
type DispatchCommand = Extract<MachineCommand, {
|
|
@@ -18,6 +18,10 @@ export type DispatchListResult = {
|
|
|
18
18
|
export declare function handleDispatchCommand(ws: WebSocket, config: DaemonConfig, command: DispatchCommand): Promise<void>;
|
|
19
19
|
export declare function handleQueueCommand(ws: WebSocket, config: DaemonConfig, command: QueueCommand): Promise<void>;
|
|
20
20
|
export declare function dispatchListResultForCommand(config: DaemonConfig, command: DispatchListCommand): Promise<DispatchListResult>;
|
|
21
|
+
export declare function remoteLoadedSessionInventory(config: DaemonConfig, machineId: string, filters?: {
|
|
22
|
+
cwd?: string;
|
|
23
|
+
agent?: string;
|
|
24
|
+
}): Promise<LoadedSessionInventory[]>;
|
|
21
25
|
export declare function publishRemoteCrewQueueEnqueue(config: DaemonConfig, admission: StoredDispatchAdmissionRecord): Promise<void>;
|
|
22
26
|
export declare function deliverQueuedCrewAdmission(config: DaemonConfig, admission: StoredDispatchAdmissionRecord): Promise<StoredDispatchAdmissionRecord>;
|
|
23
27
|
export type CrewSessionRuntimeState = {
|
|
@@ -500,6 +500,7 @@ async function applyFirstAgentRouteGate(config, consumer, command, crewMachineId
|
|
|
500
500
|
const candidates = filterDispatchRouteCandidates([
|
|
501
501
|
...dispatchRouteCandidates(records),
|
|
502
502
|
...await artifactLineageRouteCandidates(config, command, crewMachineId),
|
|
503
|
+
...await remoteLoadedCrewRouteCandidates(config, crewMachineId, command.cwd),
|
|
503
504
|
], {
|
|
504
505
|
cwd: command.cwd,
|
|
505
506
|
crewMachineId,
|
|
@@ -509,6 +510,41 @@ async function applyFirstAgentRouteGate(config, consumer, command, crewMachineId
|
|
|
509
510
|
return { error: routeError };
|
|
510
511
|
return { command: routed };
|
|
511
512
|
}
|
|
513
|
+
export async function remoteLoadedSessionInventory(config, machineId, filters = {}) {
|
|
514
|
+
if (!machineId || machineId === config.machineId)
|
|
515
|
+
return [];
|
|
516
|
+
const message = await requestRemoteMachineResult(config, {
|
|
517
|
+
targetMachineId: machineId,
|
|
518
|
+
expectedType: "machine:historicalSessions",
|
|
519
|
+
command: {
|
|
520
|
+
type: "controller:listHistoricalSessions",
|
|
521
|
+
requestId: randomId("req_session_inventory"),
|
|
522
|
+
machineId,
|
|
523
|
+
...(filters.cwd ? { cwd: filters.cwd } : {}),
|
|
524
|
+
...(filters.agent ? { agent: filters.agent } : {}),
|
|
525
|
+
limit: 200,
|
|
526
|
+
},
|
|
527
|
+
timeoutMs: 20_000,
|
|
528
|
+
});
|
|
529
|
+
return (message.loadedSessions ?? []).filter((session) => session.status !== "closed");
|
|
530
|
+
}
|
|
531
|
+
async function remoteLoadedCrewRouteCandidates(config, crewMachineId, cwd) {
|
|
532
|
+
const loaded = await remoteLoadedSessionInventory(config, crewMachineId, cwd ? { cwd } : {}).catch(() => []);
|
|
533
|
+
return loaded.map((session) => {
|
|
534
|
+
const busy = session.status === "running" || session.status === "waiting_for_option";
|
|
535
|
+
const checkpointId = session.currentHead ?? session.lastTurnId;
|
|
536
|
+
return {
|
|
537
|
+
crewSessionId: session.sessionId,
|
|
538
|
+
crewMachineId,
|
|
539
|
+
agent: session.agent,
|
|
540
|
+
cwd: session.cwd,
|
|
541
|
+
availability: busy ? "busy" : "idle",
|
|
542
|
+
recentTasks: [],
|
|
543
|
+
...(busy ? { queueArgs: { crewSessionId: session.sessionId, ifBusy: "queue" } } : { reuseArgs: { crewSessionId: session.sessionId } }),
|
|
544
|
+
...(checkpointId ? { forkArgs: { forkFromCrewSessionId: session.sessionId, forkCheckpointId: checkpointId } } : {}),
|
|
545
|
+
};
|
|
546
|
+
});
|
|
547
|
+
}
|
|
512
548
|
async function artifactLineageRouteCandidates(config, command, crewMachineId) {
|
|
513
549
|
if (!command.taskId || !command.routeDecision || command.routeDecision.kind === "new")
|
|
514
550
|
return [];
|
|
@@ -1221,7 +1257,22 @@ async function crewSessionProjection(config, crewMachineId, crewSessionId) {
|
|
|
1221
1257
|
return session ? projectCrewSession(session, crewMachineId) : undefined;
|
|
1222
1258
|
}
|
|
1223
1259
|
const sessionMessage = await readRetainedMachineMessage(config, `node:${crewMachineId}:session:${crewSessionId}`).catch(() => undefined);
|
|
1224
|
-
|
|
1260
|
+
if (sessionMessage)
|
|
1261
|
+
return projectRemoteCrewSession(crewMachineId, sessionMessage);
|
|
1262
|
+
const loaded = await remoteLoadedSessionInventory(config, crewMachineId).catch(() => []);
|
|
1263
|
+
const inventory = loaded.find((session) => session.sessionId === crewSessionId);
|
|
1264
|
+
if (!inventory)
|
|
1265
|
+
return undefined;
|
|
1266
|
+
return {
|
|
1267
|
+
crewMachineId,
|
|
1268
|
+
crewSessionId: inventory.sessionId,
|
|
1269
|
+
agent: inventory.agent,
|
|
1270
|
+
cwd: inventory.cwd,
|
|
1271
|
+
name: inventory.name,
|
|
1272
|
+
status: inventory.status === "running" || inventory.status === "waiting_for_option" ? "running" : "ready",
|
|
1273
|
+
...(inventory.currentHead ? { currentHead: inventory.currentHead } : {}),
|
|
1274
|
+
...(inventory.lastTurnId ? { lastTurnId: inventory.lastTurnId } : {}),
|
|
1275
|
+
};
|
|
1225
1276
|
}
|
|
1226
1277
|
async function dispatchTaskProjection(config, admission) {
|
|
1227
1278
|
const crewMachineId = admission.crewMachineId ?? config.machineId;
|
|
@@ -25,7 +25,7 @@ import { applyRelayMachinePresence, applyRelayMachineSnapshot, reconcileRelayMac
|
|
|
25
25
|
import { handleDevExec, handleDiagnose } from "./devtools.js";
|
|
26
26
|
import { handleInboundPromptCommand, recoverInboundPromptAdmissions } from "./inbound-prompt.js";
|
|
27
27
|
import { registerMachine } from "./register.js";
|
|
28
|
-
import { clearActiveRelaySocket, configureRelaySender, flushRelayOutbox, handleMachineMessageAck, handleRetainedPacketAck, handleRetainedPacketError, sendCommandResponse, setActiveRelaySocket, } from "./send.js";
|
|
28
|
+
import { clearActiveRelaySocket, configureRelaySender, flushRelayOutbox, handleMachineMessageAck, handleMachineMessageError, handleRetainedPacketAck, handleRetainedPacketError, sendCommandResponse, setActiveRelaySocket, } from "./send.js";
|
|
29
29
|
async function applyMachineManifest(config, retained, presence) {
|
|
30
30
|
try {
|
|
31
31
|
const message = parseMachineClientMessage(await decryptJson(config.accountSecret, retained.encryptedPayload));
|
|
@@ -253,6 +253,8 @@ async function handleRelayWireMessage(ws, config, message) {
|
|
|
253
253
|
return;
|
|
254
254
|
}
|
|
255
255
|
if (message.type === "server:error") {
|
|
256
|
+
if (message.packetId)
|
|
257
|
+
await handleMachineMessageError(message.packetId, message.message);
|
|
256
258
|
if (message.retainedKey)
|
|
257
259
|
handleRetainedPacketError(message.retainedKey);
|
|
258
260
|
console.error(`Relay error${message.code ? ` ${message.code}` : ""}: ${message.message}`);
|
|
@@ -25,6 +25,7 @@ export async function publishRemoteControllerCommand(config, input) {
|
|
|
25
25
|
return await new Promise((resolve, reject) => {
|
|
26
26
|
const ws = new WebSocket(relayWsUrl(config.relayUrl));
|
|
27
27
|
let settled = false;
|
|
28
|
+
let commandSent = false;
|
|
28
29
|
const timer = globalThis.setTimeout(() => {
|
|
29
30
|
finish(undefined, new RemoteCommandError(`Timed out waiting for remote command ${input.command.requestId}`, "REMOTE_COMMAND_TIMEOUT"));
|
|
30
31
|
}, timeoutMs);
|
|
@@ -45,6 +46,11 @@ export async function publishRemoteControllerCommand(config, input) {
|
|
|
45
46
|
token: config.machineToken,
|
|
46
47
|
nodeKind: "machine",
|
|
47
48
|
nodeId: config.machineId,
|
|
49
|
+
// This is an ephemeral RPC socket, not the daemon's durable command
|
|
50
|
+
// consumer. Do not replay unrelated toNode packets into it: the
|
|
51
|
+
// durable daemon connection (with its own cursor) must remain the
|
|
52
|
+
// only consumer that can acknowledge those commands.
|
|
53
|
+
cursor: Number.MAX_SAFE_INTEGER,
|
|
48
54
|
}));
|
|
49
55
|
});
|
|
50
56
|
ws.addEventListener("error", () => {
|
|
@@ -57,6 +63,18 @@ export async function publishRemoteControllerCommand(config, input) {
|
|
|
57
63
|
targetMachineId: input.targetMachineId,
|
|
58
64
|
expectedType: input.expectedType,
|
|
59
65
|
send: (message) => ws.send(JSON.stringify(message)),
|
|
66
|
+
sendCommand: async (cursor) => {
|
|
67
|
+
if (commandSent)
|
|
68
|
+
return;
|
|
69
|
+
commandSent = true;
|
|
70
|
+
const subscribe = {
|
|
71
|
+
type: "subscribe",
|
|
72
|
+
channelId: controllerProjectionChannel,
|
|
73
|
+
...(cursor === undefined ? {} : { cursor }),
|
|
74
|
+
};
|
|
75
|
+
ws.send(JSON.stringify(subscribe));
|
|
76
|
+
ws.send(JSON.stringify(await remoteCommandPacket(config, packetId, input.targetMachineId, input.command)));
|
|
77
|
+
},
|
|
60
78
|
finish,
|
|
61
79
|
}).catch((error) => finish(undefined, error));
|
|
62
80
|
});
|
|
@@ -77,8 +95,14 @@ export async function readRetainedMachineMessage(config, retainedKey) {
|
|
|
77
95
|
async function handleRelayMessage(config, raw, context) {
|
|
78
96
|
const wire = parseRelayServerWireMessage(JSON.parse(String(raw)));
|
|
79
97
|
if (wire.type === "server:hello") {
|
|
80
|
-
|
|
81
|
-
|
|
98
|
+
// Wait for the snapshot cursor before subscribing. Short-lived RPC
|
|
99
|
+
// connections must not replay the entire product:controllers history;
|
|
100
|
+
// the snapshot cursor gives the subscription a precise lower bound while
|
|
101
|
+
// still catching packets committed between the snapshot and subscribe.
|
|
102
|
+
return;
|
|
103
|
+
}
|
|
104
|
+
if (wire.type === "server:snapshot") {
|
|
105
|
+
await context.sendCommand(wire.cursor);
|
|
82
106
|
return;
|
|
83
107
|
}
|
|
84
108
|
if (wire.type === "server:error") {
|
|
@@ -92,7 +116,6 @@ async function handleRelayMessage(config, raw, context) {
|
|
|
92
116
|
}
|
|
93
117
|
if (wire.type !== "server:packet")
|
|
94
118
|
return;
|
|
95
|
-
context.send({ type: "ackPacket", packetId: wire.packet.packetId, serverSeq: wire.packet.serverSeq });
|
|
96
119
|
let message;
|
|
97
120
|
try {
|
|
98
121
|
message = parseMachineClientMessage(await decryptJson(config.accountSecret, wire.packet.encryptedPayload));
|
|
@@ -102,6 +125,10 @@ async function handleRelayMessage(config, raw, context) {
|
|
|
102
125
|
}
|
|
103
126
|
if (!("requestId" in message) || message.requestId !== context.command.requestId)
|
|
104
127
|
return;
|
|
128
|
+
// A short-lived RPC socket can share a machine id with the durable daemon
|
|
129
|
+
// connection. Only acknowledge the response that belongs to this request;
|
|
130
|
+
// unrelated packets must remain available for the durable consumer.
|
|
131
|
+
context.send({ type: "ackPacket", packetId: wire.packet.packetId, serverSeq: wire.packet.serverSeq });
|
|
105
132
|
if (message.type === "machine:error") {
|
|
106
133
|
context.finish(undefined, new RemoteCommandError(message.message, message.code ?? "REMOTE_MACHINE_ERROR", false));
|
|
107
134
|
return;
|
|
@@ -9,6 +9,7 @@ export declare function setActiveRelaySocket(ws: WebSocket): void;
|
|
|
9
9
|
export declare function clearActiveRelaySocket(ws: WebSocket): void;
|
|
10
10
|
export declare function flushRelayOutbox(): Promise<void>;
|
|
11
11
|
export declare function handleMachineMessageAck(messageId: string): Promise<void>;
|
|
12
|
+
export declare function handleMachineMessageError(messageId: string, error?: string): Promise<void>;
|
|
12
13
|
export declare function handleRetainedPacketAck(retainedKey: string, version: number | undefined): void;
|
|
13
14
|
export declare function handleRetainedPacketError(retainedKey: string): void;
|
|
14
15
|
export declare function clearRelayOutboxForTests(): Promise<void>;
|
|
@@ -32,7 +32,8 @@ let relayConfig = null;
|
|
|
32
32
|
let outboxLoaded = false;
|
|
33
33
|
let outboxLoadPromise = null;
|
|
34
34
|
let outboxWritePromise = Promise.resolve();
|
|
35
|
-
let
|
|
35
|
+
let outboxDrainPromise = null;
|
|
36
|
+
let outboxFlushTimer;
|
|
36
37
|
const outbox = new Map();
|
|
37
38
|
const directSockets = new WeakSet();
|
|
38
39
|
const retainedVersions = new Map();
|
|
@@ -41,6 +42,14 @@ const transcriptWindows = new Map();
|
|
|
41
42
|
const controllerProjectionChannel = "product:controllers";
|
|
42
43
|
const retainedTranscriptWindowSize = 200;
|
|
43
44
|
const retainedTranscriptMaxPayloadBytes = 650_000;
|
|
45
|
+
const outboxFlushBatchSize = 32;
|
|
46
|
+
const outboxInFlightWindow = 32;
|
|
47
|
+
const outboxFlushBatchBytes = 4 * 1024 * 1024;
|
|
48
|
+
const outboxInFlightBytes = 8 * 1024 * 1024;
|
|
49
|
+
const outboxAckTimeoutMs = 15_000;
|
|
50
|
+
const outboxRetryBaseMs = 1_000;
|
|
51
|
+
const outboxRetryMaxMs = 30_000;
|
|
52
|
+
const relayStoredTtlMs = 24 * 60 * 60 * 1000;
|
|
44
53
|
export function markDirectSocket(ws) {
|
|
45
54
|
directSockets.add(ws);
|
|
46
55
|
}
|
|
@@ -67,24 +76,52 @@ export function configureRelaySender(config) {
|
|
|
67
76
|
}
|
|
68
77
|
export function setActiveRelaySocket(ws) {
|
|
69
78
|
activeRelaySocket = ws;
|
|
79
|
+
scheduleOutboxFlush(0);
|
|
70
80
|
}
|
|
71
81
|
export function clearActiveRelaySocket(ws) {
|
|
72
|
-
if (activeRelaySocket
|
|
73
|
-
activeRelaySocket = null;
|
|
74
|
-
}
|
|
75
|
-
export async function flushRelayOutbox() {
|
|
76
|
-
await ensureOutboxLoaded();
|
|
77
|
-
if (flushing)
|
|
82
|
+
if (activeRelaySocket !== ws)
|
|
78
83
|
return;
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
await attemptSendRecord(record);
|
|
84
|
-
}
|
|
84
|
+
activeRelaySocket = null;
|
|
85
|
+
if (outboxFlushTimer) {
|
|
86
|
+
clearTimeout(outboxFlushTimer);
|
|
87
|
+
outboxFlushTimer = undefined;
|
|
85
88
|
}
|
|
86
|
-
|
|
87
|
-
|
|
89
|
+
}
|
|
90
|
+
export function flushRelayOutbox() {
|
|
91
|
+
outboxDrainPromise ??= flushRelayOutboxOnce().finally(() => {
|
|
92
|
+
outboxDrainPromise = null;
|
|
93
|
+
scheduleOutboxFlush();
|
|
94
|
+
});
|
|
95
|
+
return outboxDrainPromise;
|
|
96
|
+
}
|
|
97
|
+
async function flushRelayOutboxOnce() {
|
|
98
|
+
await ensureOutboxLoaded();
|
|
99
|
+
const records = [...outbox.values()].sort((left, right) => left.createdAt.localeCompare(right.createdAt));
|
|
100
|
+
const now = Date.now();
|
|
101
|
+
const inFlight = records.filter((record) => isInFlight(record, now));
|
|
102
|
+
const inFlightCount = inFlight.length;
|
|
103
|
+
const inFlightBytes = inFlight.reduce((total, record) => total + recordWireBytes(record), 0);
|
|
104
|
+
let attempted = 0;
|
|
105
|
+
let attemptedBytes = 0;
|
|
106
|
+
for (const record of records) {
|
|
107
|
+
if (attempted >= outboxFlushBatchSize)
|
|
108
|
+
break;
|
|
109
|
+
if (inFlightCount + attempted >= outboxInFlightWindow)
|
|
110
|
+
break;
|
|
111
|
+
if (record.nextAttemptAt && Date.parse(record.nextAttemptAt) > now)
|
|
112
|
+
continue;
|
|
113
|
+
const bytes = recordWireBytes(record);
|
|
114
|
+
const exceedsInFlightBudget = inFlightBytes + attemptedBytes + bytes > outboxInFlightBytes;
|
|
115
|
+
const exceedsBatchBudget = attemptedBytes + bytes > outboxFlushBatchBytes;
|
|
116
|
+
// A single record larger than a budget is allowed only when it is the
|
|
117
|
+
// sole packet in this drain. This keeps durable large projections from
|
|
118
|
+
// being stranded forever while ensuring no second packet can join it.
|
|
119
|
+
const allowOversizedSingle = attempted === 0 && inFlightCount === 0;
|
|
120
|
+
if ((exceedsInFlightBudget && !allowOversizedSingle) || (exceedsBatchBudget && !allowOversizedSingle))
|
|
121
|
+
break;
|
|
122
|
+
await attemptSendRecord(record);
|
|
123
|
+
attempted += 1;
|
|
124
|
+
attemptedBytes += bytes;
|
|
88
125
|
}
|
|
89
126
|
}
|
|
90
127
|
export async function handleMachineMessageAck(messageId) {
|
|
@@ -92,6 +129,17 @@ export async function handleMachineMessageAck(messageId) {
|
|
|
92
129
|
if (!outbox.delete(messageId))
|
|
93
130
|
return;
|
|
94
131
|
await persistOutbox();
|
|
132
|
+
scheduleOutboxFlush(0);
|
|
133
|
+
}
|
|
134
|
+
export async function handleMachineMessageError(messageId, error) {
|
|
135
|
+
await ensureOutboxLoaded();
|
|
136
|
+
const record = outbox.get(messageId);
|
|
137
|
+
if (!record)
|
|
138
|
+
return;
|
|
139
|
+
record.nextAttemptAt = new Date(Date.now() + retryDelayMs(record.attemptCount)).toISOString();
|
|
140
|
+
await persistOutbox();
|
|
141
|
+
await appendOutboxAudit(record, "send-failed", error);
|
|
142
|
+
scheduleOutboxFlush();
|
|
95
143
|
}
|
|
96
144
|
export function handleRetainedPacketAck(retainedKey, version) {
|
|
97
145
|
if (version !== undefined)
|
|
@@ -101,6 +149,10 @@ export function handleRetainedPacketError(retainedKey) {
|
|
|
101
149
|
retainedVersions.delete(retainedKey);
|
|
102
150
|
}
|
|
103
151
|
export async function clearRelayOutboxForTests() {
|
|
152
|
+
if (outboxFlushTimer) {
|
|
153
|
+
clearTimeout(outboxFlushTimer);
|
|
154
|
+
outboxFlushTimer = undefined;
|
|
155
|
+
}
|
|
104
156
|
outbox.clear();
|
|
105
157
|
outboxLoaded = true;
|
|
106
158
|
outboxLoadPromise = null;
|
|
@@ -133,10 +185,13 @@ export async function sendReliable(message, ws) {
|
|
|
133
185
|
attemptCount: 0,
|
|
134
186
|
};
|
|
135
187
|
if (!existing) {
|
|
136
|
-
|
|
188
|
+
replaceCoalescedOutboxRecord(record);
|
|
137
189
|
await persistOutbox();
|
|
138
190
|
}
|
|
139
|
-
|
|
191
|
+
// All reliable messages, including live command responses, enter the same
|
|
192
|
+
// durable single-flight drain. Direct sockets remain an explicit local
|
|
193
|
+
// transport above and do not use this relay outbox.
|
|
194
|
+
await flushRelayOutbox();
|
|
140
195
|
return record.message;
|
|
141
196
|
}
|
|
142
197
|
export async function sendCommandResponse(ws, message, options = {}) {
|
|
@@ -227,6 +282,149 @@ function withMessageId(message) {
|
|
|
227
282
|
messageId: existing ?? `msg_${randomUUID()}`,
|
|
228
283
|
};
|
|
229
284
|
}
|
|
285
|
+
function outboxCoalesceKey(message) {
|
|
286
|
+
if (!("sessionId" in message) || typeof message.sessionId !== "string" || !message.sessionId)
|
|
287
|
+
return undefined;
|
|
288
|
+
if (!isReplaceableProjection(message))
|
|
289
|
+
return undefined;
|
|
290
|
+
if (message.type === "machine:sessionReady")
|
|
291
|
+
return `session-ready:${message.sessionId}`;
|
|
292
|
+
if (message.type === "machine:sessionTranscriptReplaced")
|
|
293
|
+
return `session-transcript:${message.sessionId}`;
|
|
294
|
+
return undefined;
|
|
295
|
+
}
|
|
296
|
+
function isReplaceableProjection(message) {
|
|
297
|
+
if (message.type !== "machine:sessionReady" && message.type !== "machine:sessionTranscriptReplaced")
|
|
298
|
+
return false;
|
|
299
|
+
const requestId = "requestId" in message && typeof message.requestId === "string" ? message.requestId : undefined;
|
|
300
|
+
if (!requestId)
|
|
301
|
+
return true;
|
|
302
|
+
// These are daemon-generated retained projection refreshes, not responses
|
|
303
|
+
// to a caller. Correlated create/resume/sync responses must never be
|
|
304
|
+
// coalesced away even when they share a session id.
|
|
305
|
+
return requestId.startsWith("req_projection_refresh_") || requestId.startsWith("req_direct_projection_");
|
|
306
|
+
}
|
|
307
|
+
function replaceCoalescedOutboxRecord(record) {
|
|
308
|
+
const key = outboxCoalesceKey(record.message);
|
|
309
|
+
if (key) {
|
|
310
|
+
record.coalesceKey = key;
|
|
311
|
+
const now = Date.now();
|
|
312
|
+
for (const [messageId, existing] of outbox) {
|
|
313
|
+
if (messageId !== record.messageId
|
|
314
|
+
&& (existing.coalesceKey ?? outboxCoalesceKey(existing.message)) === key
|
|
315
|
+
// A packet that was already sent but has not been ACKed still
|
|
316
|
+
// consumes the in-flight byte window. Keep it durable and counted;
|
|
317
|
+
// only unsent/expired projections may be replaced.
|
|
318
|
+
&& !isInFlight(existing, now)) {
|
|
319
|
+
outbox.delete(messageId);
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
outbox.set(record.messageId, record);
|
|
324
|
+
}
|
|
325
|
+
function retryDelayMs(attemptCount) {
|
|
326
|
+
const exponent = Math.max(0, Math.min(attemptCount - 1, 5));
|
|
327
|
+
return Math.min(outboxRetryMaxMs, outboxRetryBaseMs * (2 ** exponent));
|
|
328
|
+
}
|
|
329
|
+
function scheduleOutboxFlush(delayOverrideMs) {
|
|
330
|
+
if (!openSocket(activeRelaySocket))
|
|
331
|
+
return;
|
|
332
|
+
if (outboxFlushTimer) {
|
|
333
|
+
clearTimeout(outboxFlushTimer);
|
|
334
|
+
outboxFlushTimer = undefined;
|
|
335
|
+
}
|
|
336
|
+
const now = Date.now();
|
|
337
|
+
const records = [...outbox.values()];
|
|
338
|
+
if (records.length === 0)
|
|
339
|
+
return;
|
|
340
|
+
const nextFlushAt = nextOutboxFlushAt(records, now);
|
|
341
|
+
// If a live in-flight window cannot fit the next record, wait for an ACK or
|
|
342
|
+
// retry deadline. Scheduling on the mere fact that some bytes remain would
|
|
343
|
+
// create a zero-delay loop while the head record is still blocked.
|
|
344
|
+
if (nextFlushAt === undefined && delayOverrideMs === undefined)
|
|
345
|
+
return;
|
|
346
|
+
const delay = delayOverrideMs ?? Math.max(0, (nextFlushAt ?? now) - now);
|
|
347
|
+
outboxFlushTimer = setTimeout(() => {
|
|
348
|
+
outboxFlushTimer = undefined;
|
|
349
|
+
void flushRelayOutbox().catch((error) => {
|
|
350
|
+
console.error(`Relay outbox flush failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
351
|
+
});
|
|
352
|
+
}, Math.max(0, delay));
|
|
353
|
+
}
|
|
354
|
+
function nextOutboxFlushAt(records, now) {
|
|
355
|
+
const sorted = [...records].sort((left, right) => left.createdAt.localeCompare(right.createdAt));
|
|
356
|
+
const inFlight = sorted.filter((record) => isInFlight(record, now));
|
|
357
|
+
const inFlightCount = inFlight.length;
|
|
358
|
+
const inFlightBytes = inFlight.reduce((total, record) => total + recordWireBytes(record), 0);
|
|
359
|
+
let attempted = 0;
|
|
360
|
+
let attemptedBytes = 0;
|
|
361
|
+
let nextWakeAt;
|
|
362
|
+
for (const record of sorted) {
|
|
363
|
+
const dueAt = record.nextAttemptAt ? Date.parse(record.nextAttemptAt) : now;
|
|
364
|
+
if (!Number.isFinite(dueAt))
|
|
365
|
+
continue;
|
|
366
|
+
if (isInFlight(record, now)) {
|
|
367
|
+
nextWakeAt = nextWakeAt === undefined ? dueAt : Math.min(nextWakeAt, dueAt);
|
|
368
|
+
continue;
|
|
369
|
+
}
|
|
370
|
+
if (dueAt > now) {
|
|
371
|
+
nextWakeAt = nextWakeAt === undefined ? dueAt : Math.min(nextWakeAt, dueAt);
|
|
372
|
+
continue;
|
|
373
|
+
}
|
|
374
|
+
if (attempted >= outboxFlushBatchSize || inFlightCount + attempted >= outboxInFlightWindow)
|
|
375
|
+
break;
|
|
376
|
+
const bytes = recordWireBytes(record);
|
|
377
|
+
const allowOversizedSingle = attempted === 0 && inFlightCount === 0;
|
|
378
|
+
if ((inFlightBytes + attemptedBytes + bytes > outboxInFlightBytes && !allowOversizedSingle)
|
|
379
|
+
|| (attemptedBytes + bytes > outboxFlushBatchBytes && !allowOversizedSingle))
|
|
380
|
+
break;
|
|
381
|
+
return now;
|
|
382
|
+
}
|
|
383
|
+
return nextWakeAt;
|
|
384
|
+
}
|
|
385
|
+
function isInFlight(record, now) {
|
|
386
|
+
return Boolean(record.lastAttemptAt && record.nextAttemptAt && Date.parse(record.nextAttemptAt) > now);
|
|
387
|
+
}
|
|
388
|
+
function recordWireBytes(record) {
|
|
389
|
+
const payloadBytes = Buffer.byteLength(JSON.stringify(record.message), "utf8");
|
|
390
|
+
const message = record.message;
|
|
391
|
+
const requestId = typeof message.requestId === "string" ? message.requestId : undefined;
|
|
392
|
+
const sessionId = typeof message.sessionId === "string" ? message.sessionId : undefined;
|
|
393
|
+
const scope = `packet:${record.messageId}`;
|
|
394
|
+
// encryptJson has fixed-size AES-GCM overhead: a 32-byte wrapped key plus
|
|
395
|
+
// tag, 12-byte nonces, and one 16-byte data tag. Base64 expands each byte
|
|
396
|
+
// deterministically, so this skeleton is the exact JSON wire size without
|
|
397
|
+
// performing encryption during scheduling.
|
|
398
|
+
const encryptedPayload = {
|
|
399
|
+
v: 1,
|
|
400
|
+
alg: "A256GCM",
|
|
401
|
+
scope,
|
|
402
|
+
key: {
|
|
403
|
+
nonce: "0".repeat(16),
|
|
404
|
+
ciphertext: "0".repeat(64),
|
|
405
|
+
},
|
|
406
|
+
data: {
|
|
407
|
+
nonce: "0".repeat(16),
|
|
408
|
+
ciphertext: "0".repeat(base64EncodedLength(payloadBytes + 16)),
|
|
409
|
+
},
|
|
410
|
+
};
|
|
411
|
+
const packet = {
|
|
412
|
+
type: "publishPacket",
|
|
413
|
+
packet: {
|
|
414
|
+
packetId: record.messageId,
|
|
415
|
+
channelId: controllerProjectionChannel,
|
|
416
|
+
...(sessionId ? { streamId: sessionId } : {}),
|
|
417
|
+
...(requestId ? { correlationId: requestId, idempotencyKey: requestId } : {}),
|
|
418
|
+
ttlMs: relayStoredTtlMs,
|
|
419
|
+
ackMode: "stored",
|
|
420
|
+
encryptedPayload,
|
|
421
|
+
},
|
|
422
|
+
};
|
|
423
|
+
return Buffer.byteLength(JSON.stringify(packet), "utf8");
|
|
424
|
+
}
|
|
425
|
+
function base64EncodedLength(byteLength) {
|
|
426
|
+
return Math.ceil(byteLength / 3) * 4;
|
|
427
|
+
}
|
|
230
428
|
async function ensureOutboxLoaded() {
|
|
231
429
|
if (outboxLoaded)
|
|
232
430
|
return;
|
|
@@ -235,6 +433,7 @@ async function ensureOutboxLoaded() {
|
|
|
235
433
|
}
|
|
236
434
|
async function loadOutbox() {
|
|
237
435
|
outbox.clear();
|
|
436
|
+
let compacted = false;
|
|
238
437
|
try {
|
|
239
438
|
const text = await fs.readFile(relayOutboxPath, "utf8");
|
|
240
439
|
for (const line of text.split(/\n/u)) {
|
|
@@ -242,7 +441,10 @@ async function loadOutbox() {
|
|
|
242
441
|
continue;
|
|
243
442
|
const parsed = JSON.parse(line);
|
|
244
443
|
if (parsed.messageId && parsed.message?.messageId === parsed.messageId) {
|
|
245
|
-
outbox.
|
|
444
|
+
const before = outbox.size;
|
|
445
|
+
replaceCoalescedOutboxRecord(parsed);
|
|
446
|
+
if (outbox.size < before + 1)
|
|
447
|
+
compacted = true;
|
|
246
448
|
}
|
|
247
449
|
}
|
|
248
450
|
}
|
|
@@ -251,6 +453,8 @@ async function loadOutbox() {
|
|
|
251
453
|
throw error;
|
|
252
454
|
}
|
|
253
455
|
outboxLoaded = true;
|
|
456
|
+
if (compacted)
|
|
457
|
+
await persistOutbox();
|
|
254
458
|
}
|
|
255
459
|
async function persistOutbox() {
|
|
256
460
|
outboxWritePromise = outboxWritePromise.then(writeOutbox, writeOutbox);
|
|
@@ -272,7 +476,12 @@ async function attemptSendRecord(record, ws) {
|
|
|
272
476
|
return;
|
|
273
477
|
}
|
|
274
478
|
record.attemptCount += 1;
|
|
275
|
-
|
|
479
|
+
const attemptedAt = Date.now();
|
|
480
|
+
record.lastAttemptAt = new Date(attemptedAt).toISOString();
|
|
481
|
+
// A stored ACK may be lost after the packet is accepted. Keep the record
|
|
482
|
+
// retryable with bounded exponential backoff instead of flooding every
|
|
483
|
+
// reconnect with the entire historical outbox.
|
|
484
|
+
record.nextAttemptAt = new Date(attemptedAt + Math.max(outboxAckTimeoutMs, retryDelayMs(record.attemptCount))).toISOString();
|
|
276
485
|
await persistOutbox();
|
|
277
486
|
try {
|
|
278
487
|
if (directSockets.has(socket)) {
|
|
@@ -301,7 +510,7 @@ async function sendPacket(socket, payload, options) {
|
|
|
301
510
|
channelId: controllerProjectionChannel,
|
|
302
511
|
...(sessionId ? { streamId: sessionId } : {}),
|
|
303
512
|
...(requestId ? { correlationId: requestId, idempotencyKey: requestId } : {}),
|
|
304
|
-
ttlMs: options.ackMode === "stored" ?
|
|
513
|
+
ttlMs: options.ackMode === "stored" ? relayStoredTtlMs : 10 * 60 * 1000,
|
|
305
514
|
ackMode: options.ackMode,
|
|
306
515
|
encryptedPayload: await encryptJson(relayConfig.accountSecret, `packet:${options.packetId}`, payload),
|
|
307
516
|
},
|
|
@@ -5,8 +5,13 @@ export type HeadlessRuntimeEventMappingState = {
|
|
|
5
5
|
emittedCursors: Set<string>;
|
|
6
6
|
itemTypes: Map<string, ThreadItem["type"]>;
|
|
7
7
|
deltaItems: Set<string>;
|
|
8
|
+
textItems: Set<string>;
|
|
9
|
+
authoritativeTextItems: Set<string>;
|
|
8
10
|
};
|
|
9
|
-
export declare function createHeadlessRuntimeEventMappingState(
|
|
11
|
+
export declare function createHeadlessRuntimeEventMappingState(initial?: {
|
|
12
|
+
deltaItems?: Iterable<string>;
|
|
13
|
+
authoritativeTextItems?: Iterable<string>;
|
|
14
|
+
}): HeadlessRuntimeEventMappingState;
|
|
10
15
|
export declare function mapHeadlessRuntimeEvent(event: HeadlessRuntimeEvent, state?: HeadlessRuntimeEventMappingState): RuntimeEvent[];
|
|
11
16
|
export declare function mapHeadlessRuntimeEventsToHistoricalEvents(runtimeEvents: HeadlessRuntimeEvent[]): RuntimeHistoricalEvent[];
|
|
12
17
|
export declare function selectThreadEventsBeforeItem(client: HeadlessRuntimeClient, threadId: string, beforeItemId: string | undefined, limit: number): Promise<{
|
|
@@ -1,8 +1,10 @@
|
|
|
1
|
-
export function createHeadlessRuntimeEventMappingState() {
|
|
1
|
+
export function createHeadlessRuntimeEventMappingState(initial) {
|
|
2
2
|
return {
|
|
3
3
|
emittedCursors: new Set(),
|
|
4
4
|
itemTypes: new Map(),
|
|
5
|
-
deltaItems: new Set(),
|
|
5
|
+
deltaItems: new Set(initial?.deltaItems),
|
|
6
|
+
textItems: new Set(),
|
|
7
|
+
authoritativeTextItems: new Set(initial?.authoritativeTextItems),
|
|
6
8
|
};
|
|
7
9
|
}
|
|
8
10
|
export function mapHeadlessRuntimeEvent(event, state = createHeadlessRuntimeEventMappingState()) {
|
|
@@ -17,10 +19,17 @@ export function mapHeadlessRuntimeEvent(event, state = createHeadlessRuntimeEven
|
|
|
17
19
|
if (event.method === "item/created") {
|
|
18
20
|
if (!item)
|
|
19
21
|
return [];
|
|
20
|
-
|
|
22
|
+
// Some providers include a full text snapshot on item/created and then
|
|
23
|
+
// emit item/delta chunks for the same item. Prefer the deltas; emitting
|
|
24
|
+
// both makes the authoritative turn.result contain the final text twice.
|
|
25
|
+
if (item.type === "agentMessage" && item.text && !state.authoritativeTextItems.has(item.id) && !state.deltaItems.has(item.id) && item.status === "completed") {
|
|
26
|
+
state.textItems.add(item.id);
|
|
21
27
|
return [{ type: "text_delta", text: item.text, stream: "output", ...itemIdentity }];
|
|
22
|
-
|
|
28
|
+
}
|
|
29
|
+
if (item.type === "reasoning" && item.text && !state.authoritativeTextItems.has(item.id) && !state.deltaItems.has(item.id) && item.status === "completed") {
|
|
30
|
+
state.textItems.add(item.id);
|
|
23
31
|
return [{ type: "text_delta", text: item.text, stream: "thought", ...itemIdentity }];
|
|
32
|
+
}
|
|
24
33
|
return toolEventsFromItem(item);
|
|
25
34
|
}
|
|
26
35
|
if (event.method === "item/delta") {
|
|
@@ -28,6 +37,8 @@ export function mapHeadlessRuntimeEvent(event, state = createHeadlessRuntimeEven
|
|
|
28
37
|
if (!delta)
|
|
29
38
|
return [];
|
|
30
39
|
const itemId = typeof event.itemId === "string" ? event.itemId : typeof params.itemId === "string" ? params.itemId : undefined;
|
|
40
|
+
if (itemId && state.authoritativeTextItems.has(itemId))
|
|
41
|
+
return [];
|
|
31
42
|
if (itemId)
|
|
32
43
|
state.deltaItems.add(itemId);
|
|
33
44
|
const type = itemId ? state.itemTypes.get(itemId) : undefined;
|
|
@@ -37,10 +48,16 @@ export function mapHeadlessRuntimeEvent(event, state = createHeadlessRuntimeEven
|
|
|
37
48
|
if (!item)
|
|
38
49
|
return [];
|
|
39
50
|
if (item.type === "agentMessage") {
|
|
40
|
-
|
|
51
|
+
if ((!state.authoritativeTextItems.has(item.id) && (state.deltaItems.has(item.id) || state.textItems.has(item.id))) || !item.text)
|
|
52
|
+
return [];
|
|
53
|
+
state.textItems.add(item.id);
|
|
54
|
+
return [{ type: "text_delta", text: item.text, stream: "output", ...itemIdentity }];
|
|
41
55
|
}
|
|
42
56
|
if (item.type === "reasoning") {
|
|
43
|
-
|
|
57
|
+
if ((!state.authoritativeTextItems.has(item.id) && (state.deltaItems.has(item.id) || state.textItems.has(item.id))) || !item.text)
|
|
58
|
+
return [];
|
|
59
|
+
state.textItems.add(item.id);
|
|
60
|
+
return [{ type: "text_delta", text: item.text, stream: "thought", ...itemIdentity }];
|
|
44
61
|
}
|
|
45
62
|
return toolEventsFromItem(item);
|
|
46
63
|
}
|
|
@@ -303,8 +320,17 @@ export async function selectThreadEventsAfterItem(client, threadId, afterItemId,
|
|
|
303
320
|
};
|
|
304
321
|
}
|
|
305
322
|
export function historicalEventsFromRuntimeEvents(runtimeEvents) {
|
|
306
|
-
const
|
|
323
|
+
const deltaItemIds = runtimeEvents
|
|
324
|
+
.filter((event) => event.method === "item/delta")
|
|
325
|
+
.map((event) => logicalItemIdFromHeadlessEvent(event))
|
|
326
|
+
.filter((itemId) => Boolean(itemId));
|
|
307
327
|
const completedTextItemIds = completedTextItemIdsFromRuntimeEvents(runtimeEvents);
|
|
328
|
+
const state = createHeadlessRuntimeEventMappingState({
|
|
329
|
+
// If a completed item snapshot is present, it is authoritative even when
|
|
330
|
+
// the bounded history page also contains partial deltas for that item.
|
|
331
|
+
deltaItems: deltaItemIds.filter((itemId) => !completedTextItemIds.has(itemId)),
|
|
332
|
+
authoritativeTextItems: completedTextItemIds,
|
|
333
|
+
});
|
|
308
334
|
const events = [];
|
|
309
335
|
const userPromptKeys = new Set();
|
|
310
336
|
const addEvent = (event) => {
|
|
@@ -200,6 +200,7 @@ export declare function messageResult(message: MachineClientMessage | undefined,
|
|
|
200
200
|
requestId: string;
|
|
201
201
|
machineId: string;
|
|
202
202
|
sessions: import("../../../../../packages/shared/dist/index.js").HistoricalSessionSnapshot[];
|
|
203
|
+
loadedSessions?: import("../../../../../packages/shared/dist/index.js").LoadedSessionInventory[];
|
|
203
204
|
hasMore?: boolean;
|
|
204
205
|
nextCursor?: string;
|
|
205
206
|
warnings?: import("../../../../../packages/shared/dist/index.js").HistoricalSessionWarning[];
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type DispatchRouteDecision, type DispatchTaskProjection, type HeOfficialToolResult, type TaskAttentionStatus } from "../../../../../packages/shared/dist/index.js";
|
|
1
|
+
import { type DispatchRouteDecision, type DispatchTaskProjection, type HeOfficialToolResult, type LoadedSessionInventory, type TaskAttentionStatus } from "../../../../../packages/shared/dist/index.js";
|
|
2
2
|
import { type CommandHandlerInput } from "./common.js";
|
|
3
3
|
export declare function dispatchRun(input: CommandHandlerInput): Promise<HeOfficialToolResult>;
|
|
4
4
|
export declare function dispatchStatus(input: CommandHandlerInput): Promise<HeOfficialToolResult>;
|
|
@@ -13,6 +13,7 @@ export type FirstAgentDispatchListProjectionLimits = {
|
|
|
13
13
|
maxRecentTasks?: number;
|
|
14
14
|
recordBriefMax?: number;
|
|
15
15
|
recentTaskBriefMax?: number;
|
|
16
|
+
remoteLoadedSessions?: LoadedSessionInventory[];
|
|
16
17
|
};
|
|
17
18
|
export declare function firstAgentDispatchListProjection(data: {
|
|
18
19
|
action: "list";
|
|
@@ -30,7 +31,7 @@ export declare function firstAgentDispatchListProjection(data: {
|
|
|
30
31
|
crewMachineId: string | undefined;
|
|
31
32
|
agent: string | undefined;
|
|
32
33
|
cwd: string | undefined;
|
|
33
|
-
availability: "unknown" | "
|
|
34
|
+
availability: "unknown" | "busy" | "idle";
|
|
34
35
|
updatedAt: string | undefined;
|
|
35
36
|
reuseArgs: {
|
|
36
37
|
crewSessionId: string;
|