@cydm/happy-elves 0.1.0-beta.494 → 0.1.0-beta.496
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/apps/cli/package.json +1 -1
- package/apps/daemon/dist/direct/server.js +1 -0
- package/apps/daemon/dist/orchestration/remote-work-budget.d.ts +1 -0
- package/apps/daemon/dist/orchestration/remote-work-budget.js +51 -0
- package/apps/daemon/dist/orchestration/runner.js +116 -32
- package/apps/daemon/dist/orchestration/store.d.ts +9 -0
- package/apps/daemon/dist/orchestration/store.js +28 -5
- package/apps/daemon/dist/orchestration/watcher.d.ts +8 -1
- package/apps/daemon/dist/orchestration/watcher.js +138 -5
- package/apps/daemon/dist/relay/register.js +1 -0
- package/apps/daemon/dist/relay/remote-command.js +4 -0
- package/apps/daemon/dist/runtime/headless-event-projector.d.ts +12 -1
- package/apps/daemon/dist/runtime/headless-event-projector.js +55 -5
- package/apps/daemon/dist/runtime/headless-history-reader.d.ts +1 -0
- package/apps/daemon/dist/runtime/headless-history-reader.js +17 -5
- package/apps/daemon/dist/session/he-official/common.d.ts +3 -0
- package/apps/daemon/dist/session/native-history-refresh.js +15 -1
- package/apps/daemon/dist/session/runtime-truth.d.ts +1 -0
- package/apps/daemon/dist/session/runtime-truth.js +13 -1
- package/apps/daemon/package.json +1 -1
- package/package.json +1 -1
- package/packages/runtime/dist/index.d.ts +4 -0
- package/packages/shared/dist/protocol-schemas.d.ts +8 -0
- package/packages/shared/dist/protocol-schemas.js +5 -0
- package/packages/shared/dist/protocol-types.d.ts +5 -0
- package/packages/shared/dist/protocol.d.ts +2 -0
- package/packages/shared/dist/protocol.js +2 -0
package/apps/cli/package.json
CHANGED
|
@@ -742,6 +742,7 @@ class DirectState {
|
|
|
742
742
|
acpxVersion: runtimeCapabilities.version,
|
|
743
743
|
daemonVersion: daemonPackage.version,
|
|
744
744
|
...(runtimeCapabilities.turnPrimitives ? { turnPrimitives: runtimeCapabilities.turnPrimitives } : {}),
|
|
745
|
+
runtimeEvents: { forwardCursor: true },
|
|
745
746
|
sessionPrimitives: runtimePrimitiveSupportForCapabilities(runtimeCapabilities.sessionPrimitives, "native"),
|
|
746
747
|
...(runtimeCapabilities.sidechat ? { sidechat: runtimeCapabilities.sidechat } : {}),
|
|
747
748
|
directoryBrowser: true,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function scheduleRemoteOrchestrationWork<T>(machineId: string, run: () => Promise<T>): Promise<T>;
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* One process-wide budget for every remote orchestration operation.
|
|
3
|
+
*
|
|
4
|
+
* Transcript catch-up, startup reconcile, and queue/handler initiated work
|
|
5
|
+
* all use this scheduler. Keeping the fence here prevents separate entry
|
|
6
|
+
* points from opening more sockets than the peer can service.
|
|
7
|
+
*/
|
|
8
|
+
const maxRemoteWork = 4;
|
|
9
|
+
const maxRemoteWorkPerMachine = 2;
|
|
10
|
+
const pending = [];
|
|
11
|
+
const activeByMachine = new Map();
|
|
12
|
+
let active = 0;
|
|
13
|
+
let draining = false;
|
|
14
|
+
export function scheduleRemoteOrchestrationWork(machineId, run) {
|
|
15
|
+
return new Promise((resolve, reject) => {
|
|
16
|
+
pending.push({ machineId, run: async () => await run(), resolve: (value) => resolve(value), reject });
|
|
17
|
+
void drainRemoteOrchestrationWork();
|
|
18
|
+
});
|
|
19
|
+
}
|
|
20
|
+
async function drainRemoteOrchestrationWork() {
|
|
21
|
+
if (draining)
|
|
22
|
+
return;
|
|
23
|
+
draining = true;
|
|
24
|
+
try {
|
|
25
|
+
for (;;) {
|
|
26
|
+
if (active >= maxRemoteWork)
|
|
27
|
+
return;
|
|
28
|
+
const index = pending.findIndex((work) => (activeByMachine.get(work.machineId) ?? 0) < maxRemoteWorkPerMachine);
|
|
29
|
+
if (index < 0)
|
|
30
|
+
return;
|
|
31
|
+
const work = pending.splice(index, 1)[0];
|
|
32
|
+
if (!work)
|
|
33
|
+
return;
|
|
34
|
+
active += 1;
|
|
35
|
+
activeByMachine.set(work.machineId, (activeByMachine.get(work.machineId) ?? 0) + 1);
|
|
36
|
+
void work.run().then(work.resolve, work.reject).finally(() => {
|
|
37
|
+
active -= 1;
|
|
38
|
+
const machineActive = activeByMachine.get(work.machineId) ?? 1;
|
|
39
|
+
if (machineActive <= 1)
|
|
40
|
+
activeByMachine.delete(work.machineId);
|
|
41
|
+
else
|
|
42
|
+
activeByMachine.set(work.machineId, machineActive - 1);
|
|
43
|
+
void drainRemoteOrchestrationWork();
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
finally {
|
|
48
|
+
draining = false;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
//# sourceMappingURL=remote-work-budget.js.map
|
|
@@ -12,55 +12,129 @@ import { encryptedSessionMetadata } from "../session/metadata.js";
|
|
|
12
12
|
import { loadSessionProjections } from "../session/store.js";
|
|
13
13
|
import { appendAttention, expireInflightLeases, leaseNextAttention, latestAttentionForTask, markAttentionDelivered, markAttentionDeliveryFailed, markSubscriptionLiveness, recordTurnRuntimeBinding, reopenUnresolvedArtifactFeedbackAttentions, readOrchestrationStore, } from "./store.js";
|
|
14
14
|
import { attentionFromReconcile, catchUpSubscriptionEvents } from "./watcher.js";
|
|
15
|
+
import { scheduleRemoteOrchestrationWork } from "./remote-work-budget.js";
|
|
15
16
|
import { resolveWakeIdentity } from "./wake-identity.js";
|
|
16
17
|
const activeDeliveryTicks = new Set();
|
|
17
18
|
const reconciledSubscriptions = new Set();
|
|
19
|
+
const remoteScanStates = new Map();
|
|
20
|
+
const pendingRemoteReconciles = new Map();
|
|
21
|
+
let orchestrationTickInFlight = false;
|
|
18
22
|
const defaultStaleMs = 30 * 60 * 1000;
|
|
23
|
+
const defaultRemoteScanIntervalMs = 10_000;
|
|
24
|
+
const defaultRemoteScanBackoffMs = 5_000;
|
|
25
|
+
const maxRemoteScanBackoffMs = 60_000;
|
|
19
26
|
export async function tickOrchestration(config) {
|
|
20
|
-
|
|
21
|
-
await expireInflightLeases();
|
|
22
|
-
await scanRemoteSubscriptions(config);
|
|
23
|
-
await reconcileSubscriptionsOnce(config);
|
|
24
|
-
await markStaleSubscriptions(config);
|
|
25
|
-
await reopenUnresolvedArtifactFeedbackAttentions();
|
|
26
|
-
const busyConsumerSessionIds = new Set();
|
|
27
|
-
for (const sessionId of sessions.keys()) {
|
|
28
|
-
if (sessionTurns.activeTurnId(sessionId))
|
|
29
|
-
busyConsumerSessionIds.add(sessionId);
|
|
30
|
-
}
|
|
31
|
-
const attention = await leaseNextAttention({
|
|
32
|
-
consumerSessionIds: new Set(sessions.keys()),
|
|
33
|
-
busyConsumerSessionIds,
|
|
34
|
-
leaseMs: deliveryLeaseMs(),
|
|
35
|
-
});
|
|
36
|
-
if (!attention)
|
|
27
|
+
if (orchestrationTickInFlight)
|
|
37
28
|
return;
|
|
38
|
-
|
|
39
|
-
if (activeDeliveryTicks.has(key))
|
|
40
|
-
return;
|
|
41
|
-
activeDeliveryTicks.add(key);
|
|
29
|
+
orchestrationTickInFlight = true;
|
|
42
30
|
try {
|
|
43
|
-
await
|
|
31
|
+
await ensureProjectedSessionsLoaded();
|
|
32
|
+
await expireInflightLeases();
|
|
33
|
+
// This only schedules bounded, per-subscription remote work. It never
|
|
34
|
+
// waits for a remote provider, so a slow/offline peer cannot delay local
|
|
35
|
+
// reconcile, artifact-feedback delivery, or another peer.
|
|
36
|
+
await scanRemoteSubscriptions(config);
|
|
37
|
+
await reconcileSubscriptionsOnce(config);
|
|
38
|
+
await markStaleSubscriptions(config);
|
|
39
|
+
await reopenUnresolvedArtifactFeedbackAttentions();
|
|
40
|
+
const busyConsumerSessionIds = new Set();
|
|
41
|
+
for (const sessionId of sessions.keys()) {
|
|
42
|
+
if (sessionTurns.activeTurnId(sessionId))
|
|
43
|
+
busyConsumerSessionIds.add(sessionId);
|
|
44
|
+
}
|
|
45
|
+
const attention = await leaseNextAttention({
|
|
46
|
+
consumerSessionIds: new Set(sessions.keys()),
|
|
47
|
+
busyConsumerSessionIds,
|
|
48
|
+
leaseMs: deliveryLeaseMs(),
|
|
49
|
+
});
|
|
50
|
+
if (!attention)
|
|
51
|
+
return;
|
|
52
|
+
const key = `${attention.taskId}:${attention.eventCursor}`;
|
|
53
|
+
if (activeDeliveryTicks.has(key))
|
|
54
|
+
return;
|
|
55
|
+
activeDeliveryTicks.add(key);
|
|
56
|
+
try {
|
|
57
|
+
await deliverAttention(config, attention);
|
|
58
|
+
}
|
|
59
|
+
finally {
|
|
60
|
+
activeDeliveryTicks.delete(key);
|
|
61
|
+
}
|
|
44
62
|
}
|
|
45
63
|
finally {
|
|
46
|
-
|
|
64
|
+
orchestrationTickInFlight = false;
|
|
47
65
|
}
|
|
48
66
|
}
|
|
49
67
|
async function scanRemoteSubscriptions(config) {
|
|
50
68
|
const store = await readOrchestrationStore();
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
69
|
+
const now = Date.now();
|
|
70
|
+
const candidates = store.subscriptions
|
|
71
|
+
.filter((subscription) => subscription.status === "active")
|
|
72
|
+
.filter((subscription) => (subscription.crewMachineId ?? config.machineId) !== config.machineId)
|
|
73
|
+
.map((subscription) => {
|
|
74
|
+
const key = remoteSubscriptionKey(subscription);
|
|
75
|
+
const state = remoteScanStates.get(key) ?? { nextAt: 0, backoffMs: 0 };
|
|
76
|
+
remoteScanStates.set(key, state);
|
|
77
|
+
const persistedAt = Date.parse(subscription.lastRemoteScanAt ?? "");
|
|
78
|
+
const persistedNextAt = Number.isFinite(persistedAt) ? persistedAt + remoteScanIntervalMs() : 0;
|
|
79
|
+
return { subscription, key, state, dueAt: Math.max(state.nextAt, persistedNextAt) };
|
|
80
|
+
})
|
|
81
|
+
.filter((candidate) => candidate.dueAt <= now)
|
|
82
|
+
.sort((left, right) => left.dueAt - right.dueAt);
|
|
83
|
+
for (const candidate of candidates) {
|
|
84
|
+
if (candidate.state.inFlight)
|
|
56
85
|
continue;
|
|
57
|
-
|
|
58
|
-
|
|
86
|
+
candidate.state.nextAt = Number.POSITIVE_INFINITY;
|
|
87
|
+
// catchUpSubscriptionEvents owns the budget for the actual remote RPC.
|
|
88
|
+
// Do not acquire it here as well: an outer scan slot waiting for an inner
|
|
89
|
+
// catch-up slot would deadlock a saturated peer backlog.
|
|
90
|
+
const operation = runRemoteSubscriptionScan(config, candidate.subscription, candidate.state)
|
|
91
|
+
.finally(() => {
|
|
92
|
+
candidate.state.inFlight = undefined;
|
|
93
|
+
});
|
|
94
|
+
candidate.state.inFlight = operation;
|
|
95
|
+
}
|
|
96
|
+
void drainRemoteReconciles();
|
|
97
|
+
}
|
|
98
|
+
function enqueueRemoteReconcile(config, subscription) {
|
|
99
|
+
pendingRemoteReconciles.set(remoteSubscriptionKey(subscription), { config, subscription });
|
|
100
|
+
void drainRemoteReconciles();
|
|
101
|
+
}
|
|
102
|
+
async function drainRemoteReconciles() {
|
|
103
|
+
for (const [key, pending] of pendingRemoteReconciles) {
|
|
104
|
+
const crewMachineId = pending.subscription.crewMachineId ?? pending.config.machineId;
|
|
105
|
+
pendingRemoteReconciles.delete(key);
|
|
106
|
+
void scheduleRemoteOrchestrationWork(crewMachineId, () => reconcileRemoteSubscription(pending.config, pending.subscription))
|
|
107
|
+
.catch((error) => {
|
|
108
|
+
console.error(`Remote orchestration reconcile failed for ${pending.subscription.taskId}: ${error instanceof Error ? error.message : String(error)}`);
|
|
59
109
|
});
|
|
60
110
|
}
|
|
61
111
|
}
|
|
112
|
+
async function runRemoteSubscriptionScan(config, subscription, state) {
|
|
113
|
+
let result;
|
|
114
|
+
try {
|
|
115
|
+
result = await scanRemoteTranscript(config, subscription);
|
|
116
|
+
}
|
|
117
|
+
catch (error) {
|
|
118
|
+
console.error(`Remote orchestration transcript scan crashed for ${subscription.taskId}: ${error instanceof Error ? error.message : String(error)}`);
|
|
119
|
+
result = { status: "failed" };
|
|
120
|
+
}
|
|
121
|
+
if (result.status === "ok") {
|
|
122
|
+
state.backoffMs = 0;
|
|
123
|
+
state.nextAt = Date.now() + remoteScanIntervalMs();
|
|
124
|
+
return;
|
|
125
|
+
}
|
|
126
|
+
state.backoffMs = Math.min(maxRemoteScanBackoffMs, state.backoffMs > 0 ? state.backoffMs * 2 : defaultRemoteScanBackoffMs);
|
|
127
|
+
state.nextAt = Date.now() + state.backoffMs;
|
|
128
|
+
}
|
|
129
|
+
function remoteSubscriptionKey(subscription) {
|
|
130
|
+
return `${subscription.crewMachineId ?? "local"}:${subscription.taskId}:${subscription.crewSessionId}:${subscription.turnId}`;
|
|
131
|
+
}
|
|
62
132
|
async function scanRemoteTranscript(config, subscription) {
|
|
63
|
-
await catchUpSubscriptionEvents({ config, subscription });
|
|
133
|
+
const result = await catchUpSubscriptionEvents({ config, subscription });
|
|
134
|
+
if (result.status === "failed") {
|
|
135
|
+
console.error(`Remote orchestration transcript scan failed for ${subscription.taskId}`);
|
|
136
|
+
}
|
|
137
|
+
return result;
|
|
64
138
|
}
|
|
65
139
|
async function reconcileSubscriptionsOnce(config) {
|
|
66
140
|
const store = await readOrchestrationStore();
|
|
@@ -76,7 +150,10 @@ async function reconcileSubscriptionsOnce(config) {
|
|
|
76
150
|
if (latest?.eventTypes.some((type) => type === "terminal" || type === "failure"))
|
|
77
151
|
continue;
|
|
78
152
|
if (crewMachineId !== config.machineId) {
|
|
79
|
-
|
|
153
|
+
// Remote reconcile shares the same bounded/fair scheduler as transcript
|
|
154
|
+
// catch-up. Startup recovery must not create an unbounded second fleet
|
|
155
|
+
// of sockets for one slow peer.
|
|
156
|
+
enqueueRemoteReconcile(config, subscription);
|
|
80
157
|
continue;
|
|
81
158
|
}
|
|
82
159
|
await reconcileLocalSubscription(config, subscription, "startup");
|
|
@@ -428,6 +505,13 @@ function deliveryLeaseMs() {
|
|
|
428
505
|
const parsed = Number(raw);
|
|
429
506
|
return Number.isFinite(parsed) && parsed > 0 ? parsed : 5 * 60 * 1000;
|
|
430
507
|
}
|
|
508
|
+
function remoteScanIntervalMs() {
|
|
509
|
+
const raw = process.env.HAPPY_ELVES_ORCHESTRATION_REMOTE_SCAN_MS;
|
|
510
|
+
if (!raw)
|
|
511
|
+
return defaultRemoteScanIntervalMs;
|
|
512
|
+
const parsed = Number(raw);
|
|
513
|
+
return Number.isFinite(parsed) && parsed >= 1_000 ? parsed : defaultRemoteScanIntervalMs;
|
|
514
|
+
}
|
|
431
515
|
function internalSocket() {
|
|
432
516
|
return {
|
|
433
517
|
readyState: WebSocket.CLOSED,
|
|
@@ -89,6 +89,10 @@ export type OrchestrationSubscriptionRecord = {
|
|
|
89
89
|
lastSeenAt?: string;
|
|
90
90
|
lastRemoteScanAt?: string;
|
|
91
91
|
remoteEventSeq?: number;
|
|
92
|
+
/** Last provider item used as the forward catch-up anchor for remote scans. */
|
|
93
|
+
remoteEventItemId?: string;
|
|
94
|
+
/** Append-only runtime event cursor; bound to the exact Crew session/turn. */
|
|
95
|
+
remoteEventCursor?: string;
|
|
92
96
|
createdAt: string;
|
|
93
97
|
updatedAt: string;
|
|
94
98
|
};
|
|
@@ -236,7 +240,12 @@ export declare function upsertSubscription(input: Omit<OrchestrationSubscription
|
|
|
236
240
|
export declare function activeSubscriptionsForTurn(crewSessionId: string, turnId: string): Promise<OrchestrationSubscriptionRecord[]>;
|
|
237
241
|
export declare function markSubscriptionLiveness(crewSessionId: string, turnId: string, at?: string): Promise<void>;
|
|
238
242
|
export declare function markSubscriptionRemoteScan(taskId: string, input: {
|
|
243
|
+
crewMachineId?: string;
|
|
244
|
+
crewSessionId?: string;
|
|
245
|
+
turnId?: string;
|
|
239
246
|
remoteEventSeq?: number;
|
|
247
|
+
remoteEventItemId?: string;
|
|
248
|
+
remoteEventCursor?: string;
|
|
240
249
|
at?: string;
|
|
241
250
|
}): Promise<void>;
|
|
242
251
|
export declare function appendAttention(input: AttentionInput): Promise<TaskAttentionRecord[]>;
|
|
@@ -712,11 +712,18 @@ export async function upsertSubscription(input) {
|
|
|
712
712
|
function upsertSubscriptionRecord(store, input, now) {
|
|
713
713
|
const cursor = input.cursor ?? cursorFromNumber(store.nextCursor);
|
|
714
714
|
const index = store.subscriptions.findIndex((record) => record.taskId === input.taskId);
|
|
715
|
+
const previous = index >= 0 ? store.subscriptions[index] : undefined;
|
|
716
|
+
const previousRemoteEventSeq = previous?.remoteEventSeq;
|
|
717
|
+
const crewMachineId = input.crewMachineId ?? previous?.crewMachineId;
|
|
718
|
+
const bindingUnchanged = Boolean(previous &&
|
|
719
|
+
previous.crewMachineId === crewMachineId &&
|
|
720
|
+
previous.crewSessionId === input.crewSessionId &&
|
|
721
|
+
previous.turnId === input.turnId);
|
|
715
722
|
const next = {
|
|
716
|
-
...(
|
|
723
|
+
...(previous ? { createdAt: previous.createdAt } : { createdAt: now }),
|
|
717
724
|
taskId: input.taskId,
|
|
718
725
|
...(input.consumerMachineId ? { consumerMachineId: input.consumerMachineId } : {}),
|
|
719
|
-
...(
|
|
726
|
+
...(crewMachineId ? { crewMachineId } : {}),
|
|
720
727
|
consumerSessionId: input.consumerSessionId,
|
|
721
728
|
crewSessionId: input.crewSessionId,
|
|
722
729
|
turnId: input.turnId,
|
|
@@ -725,9 +732,13 @@ function upsertSubscriptionRecord(store, input, now) {
|
|
|
725
732
|
eventTypes: normalizeEventTypes(input.eventTypes),
|
|
726
733
|
cursor,
|
|
727
734
|
status: input.status ?? "active",
|
|
728
|
-
...(input.lastSeenAt ? { lastSeenAt: input.lastSeenAt } : {}),
|
|
729
|
-
...(input.lastRemoteScanAt ? { lastRemoteScanAt: input.lastRemoteScanAt } : {}),
|
|
730
|
-
...(Number.isInteger(
|
|
735
|
+
...(input.lastSeenAt ? { lastSeenAt: input.lastSeenAt } : bindingUnchanged && previous?.lastSeenAt ? { lastSeenAt: previous.lastSeenAt } : {}),
|
|
736
|
+
...(bindingUnchanged && previous?.lastRemoteScanAt ? { lastRemoteScanAt: previous.lastRemoteScanAt } : input.lastRemoteScanAt ? { lastRemoteScanAt: input.lastRemoteScanAt } : {}),
|
|
737
|
+
...(bindingUnchanged && Number.isInteger(previousRemoteEventSeq)
|
|
738
|
+
? { remoteEventSeq: previousRemoteEventSeq }
|
|
739
|
+
: Number.isInteger(input.remoteEventSeq) ? { remoteEventSeq: input.remoteEventSeq } : {}),
|
|
740
|
+
...(bindingUnchanged && previous?.remoteEventItemId ? { remoteEventItemId: previous.remoteEventItemId } : input.remoteEventItemId ? { remoteEventItemId: input.remoteEventItemId } : {}),
|
|
741
|
+
...(bindingUnchanged && previous?.remoteEventCursor ? { remoteEventCursor: previous.remoteEventCursor } : input.remoteEventCursor ? { remoteEventCursor: input.remoteEventCursor } : {}),
|
|
731
742
|
updatedAt: now,
|
|
732
743
|
};
|
|
733
744
|
if (index >= 0)
|
|
@@ -759,11 +770,21 @@ export async function markSubscriptionRemoteScan(taskId, input) {
|
|
|
759
770
|
for (const record of current.subscriptions) {
|
|
760
771
|
if (record.taskId !== taskId || record.status !== "active")
|
|
761
772
|
continue;
|
|
773
|
+
if (input.crewMachineId !== undefined && record.crewMachineId !== input.crewMachineId)
|
|
774
|
+
continue;
|
|
775
|
+
if (input.crewSessionId !== undefined && record.crewSessionId !== input.crewSessionId)
|
|
776
|
+
continue;
|
|
777
|
+
if (input.turnId !== undefined && record.turnId !== input.turnId)
|
|
778
|
+
continue;
|
|
762
779
|
record.lastRemoteScanAt = at;
|
|
763
780
|
const remoteEventSeq = input.remoteEventSeq;
|
|
764
781
|
if (remoteEventSeq !== undefined && Number.isInteger(remoteEventSeq)) {
|
|
765
782
|
record.remoteEventSeq = Math.max(record.remoteEventSeq ?? 0, remoteEventSeq);
|
|
766
783
|
}
|
|
784
|
+
if (input.remoteEventItemId)
|
|
785
|
+
record.remoteEventItemId = input.remoteEventItemId;
|
|
786
|
+
if (input.remoteEventCursor)
|
|
787
|
+
record.remoteEventCursor = input.remoteEventCursor;
|
|
767
788
|
record.updatedAt = at;
|
|
768
789
|
}
|
|
769
790
|
return current;
|
|
@@ -1430,6 +1451,8 @@ function normalizeSubscriptionRecord(value, localMachineId) {
|
|
|
1430
1451
|
...(typeof value.providerTurnId === "string" && value.providerTurnId ? { providerTurnId: value.providerTurnId } : {}),
|
|
1431
1452
|
...(typeof value.lastRemoteScanAt === "string" && value.lastRemoteScanAt ? { lastRemoteScanAt: value.lastRemoteScanAt } : {}),
|
|
1432
1453
|
...(typeof remoteEventSeq === "number" && Number.isInteger(remoteEventSeq) && remoteEventSeq >= 0 ? { remoteEventSeq } : {}),
|
|
1454
|
+
...(typeof value.remoteEventItemId === "string" && value.remoteEventItemId ? { remoteEventItemId: value.remoteEventItemId } : {}),
|
|
1455
|
+
...(typeof value.remoteEventCursor === "string" && value.remoteEventCursor ? { remoteEventCursor: value.remoteEventCursor } : {}),
|
|
1433
1456
|
};
|
|
1434
1457
|
}
|
|
1435
1458
|
function isTurnBindingRecord(value) {
|
|
@@ -12,10 +12,17 @@ export declare function observeTurnReconcile(input: {
|
|
|
12
12
|
config: DaemonConfig;
|
|
13
13
|
result: TurnReconcileResult;
|
|
14
14
|
}): Promise<void>;
|
|
15
|
+
export type SubscriptionCatchUpResult = {
|
|
16
|
+
status: "ok" | "unreachable" | "failed";
|
|
17
|
+
maxSeq?: number;
|
|
18
|
+
nextAfter?: string;
|
|
19
|
+
lastItemId?: string;
|
|
20
|
+
};
|
|
15
21
|
export declare function catchUpSubscriptionEvents(input: {
|
|
16
22
|
config: DaemonConfig;
|
|
17
23
|
subscription: OrchestrationSubscriptionRecord;
|
|
18
|
-
}): Promise<
|
|
24
|
+
}): Promise<SubscriptionCatchUpResult>;
|
|
25
|
+
export declare function remoteEventCursorForSubscription(crewMachineId: string, subscription: Pick<OrchestrationSubscriptionRecord, "remoteEventCursor" | "remoteEventSeq">): string | undefined;
|
|
19
26
|
export declare function attentionFromReconcile(result: TurnReconcileResult): {
|
|
20
27
|
eventType: TaskAttentionEventType;
|
|
21
28
|
reason: string;
|
|
@@ -1,8 +1,16 @@
|
|
|
1
1
|
import { randomId, } from "../../../../packages/shared/dist/index.js";
|
|
2
2
|
import { appendAudit } from "../audit.js";
|
|
3
|
-
import { requestRemoteMachineResult } from "../relay/remote-command.js";
|
|
3
|
+
import { RemoteCommandError, requestRemoteMachineResult } from "../relay/remote-command.js";
|
|
4
4
|
import { runtimeRead } from "../session/runtime-truth.js";
|
|
5
5
|
import { activeSubscriptionsForTurn, appendAttention, markSubscriptionLiveness, markSubscriptionRemoteScan, recordTurnRuntimeBinding, } from "./store.js";
|
|
6
|
+
import { scheduleRemoteOrchestrationWork } from "./remote-work-budget.js";
|
|
7
|
+
import { relayMachineRegistry } from "../state.js";
|
|
8
|
+
const remoteCatchupsInFlight = new Map();
|
|
9
|
+
const legacyRemoteCursorFallbackUntil = new Map();
|
|
10
|
+
const remoteForwardCursorConfirmed = new Set();
|
|
11
|
+
const legacyRemoteCursorFallbackMs = 5 * 60 * 1000;
|
|
12
|
+
const initialRemoteEventCursor = "event_000000000000";
|
|
13
|
+
const forwardEventCursorSinceBeta = 495;
|
|
6
14
|
export async function observeSessionEvent(input) {
|
|
7
15
|
const attention = attentionFromEvent(input.payload);
|
|
8
16
|
if (!attention) {
|
|
@@ -95,6 +103,26 @@ export async function observeTurnReconcile(input) {
|
|
|
95
103
|
}
|
|
96
104
|
export async function catchUpSubscriptionEvents(input) {
|
|
97
105
|
const crewMachineId = input.subscription.crewMachineId ?? input.config.machineId;
|
|
106
|
+
if (crewMachineId !== input.config.machineId) {
|
|
107
|
+
const key = `${crewMachineId}:${input.subscription.taskId}:${input.subscription.crewSessionId}:${input.subscription.turnId}`;
|
|
108
|
+
const existing = remoteCatchupsInFlight.get(key);
|
|
109
|
+
if (existing)
|
|
110
|
+
return await existing;
|
|
111
|
+
const operation = scheduleRemoteOrchestrationWork(crewMachineId, () => catchUpSubscriptionEventsInternal(input)).finally(() => {
|
|
112
|
+
if (remoteCatchupsInFlight.get(key) === operation)
|
|
113
|
+
remoteCatchupsInFlight.delete(key);
|
|
114
|
+
});
|
|
115
|
+
remoteCatchupsInFlight.set(key, operation);
|
|
116
|
+
return await operation;
|
|
117
|
+
}
|
|
118
|
+
return await catchUpSubscriptionEventsInternal(input);
|
|
119
|
+
}
|
|
120
|
+
async function catchUpSubscriptionEventsInternal(input) {
|
|
121
|
+
const crewMachineId = input.subscription.crewMachineId ?? input.config.machineId;
|
|
122
|
+
const remoteCursor = crewMachineId !== input.config.machineId
|
|
123
|
+
? remoteEventCursorForSubscription(crewMachineId, input.subscription)
|
|
124
|
+
: undefined;
|
|
125
|
+
const remoteCursorEnabled = remoteCursor !== undefined;
|
|
98
126
|
try {
|
|
99
127
|
const result = crewMachineId === input.config.machineId
|
|
100
128
|
? await runtimeRead(input.config, {
|
|
@@ -118,6 +146,8 @@ export async function catchUpSubscriptionEvents(input) {
|
|
|
118
146
|
action: "turn.eventsPage",
|
|
119
147
|
sessionId: input.subscription.crewSessionId,
|
|
120
148
|
turnId: input.subscription.turnId,
|
|
149
|
+
...(remoteCursor ? { after: remoteCursor } : {}),
|
|
150
|
+
...(input.subscription.remoteEventItemId && !remoteCursor ? { afterItemId: input.subscription.remoteEventItemId } : {}),
|
|
121
151
|
...(input.subscription.runtimeTurnId ?? input.subscription.providerTurnId
|
|
122
152
|
? { runtimeTurnId: input.subscription.runtimeTurnId ?? input.subscription.providerTurnId }
|
|
123
153
|
: {}),
|
|
@@ -137,7 +167,21 @@ export async function catchUpSubscriptionEvents(input) {
|
|
|
137
167
|
errorCode: "errorCode" in result ? result.errorCode : undefined,
|
|
138
168
|
reason: "reason" in result ? result.reason : undefined,
|
|
139
169
|
});
|
|
140
|
-
return;
|
|
170
|
+
return { status: "unreachable" };
|
|
171
|
+
}
|
|
172
|
+
// A rolling-upgrade peer may accept the old runtimeRead command but omit
|
|
173
|
+
// the forward cursor in its response. Treat that as the legacy path for
|
|
174
|
+
// this machine; in particular, never advance a durable cursor from a
|
|
175
|
+
// page whose peer cannot prove append-only progress.
|
|
176
|
+
if (crewMachineId !== input.config.machineId && result.cursorSupported === true) {
|
|
177
|
+
remoteForwardCursorConfirmed.add(crewMachineId);
|
|
178
|
+
}
|
|
179
|
+
else if (crewMachineId !== input.config.machineId && remoteCursorEnabled && !result.nextAfter) {
|
|
180
|
+
// A peer that accepted the forward request but cannot prove the cursor
|
|
181
|
+
// contract is no longer trusted. Remove the positive handshake so the
|
|
182
|
+
// next safe, legacy-compatible request can renegotiate it.
|
|
183
|
+
remoteForwardCursorConfirmed.delete(crewMachineId);
|
|
184
|
+
legacyRemoteCursorFallbackUntil.set(crewMachineId, Date.now() + legacyRemoteCursorFallbackMs);
|
|
141
185
|
}
|
|
142
186
|
const observed = await observeSubscriptionRuntimeEvents({
|
|
143
187
|
config: input.config,
|
|
@@ -146,11 +190,29 @@ export async function catchUpSubscriptionEvents(input) {
|
|
|
146
190
|
events: result.events,
|
|
147
191
|
skipThroughSeq: crewMachineId === input.config.machineId ? 0 : input.subscription.remoteEventSeq ?? 0,
|
|
148
192
|
});
|
|
149
|
-
if (crewMachineId !== input.config.machineId
|
|
150
|
-
|
|
193
|
+
if (crewMachineId !== input.config.machineId) {
|
|
194
|
+
const canAdvance = Boolean(result.nextAfter);
|
|
195
|
+
await markSubscriptionRemoteScan(input.subscription.taskId, {
|
|
196
|
+
crewMachineId,
|
|
197
|
+
crewSessionId: input.subscription.crewSessionId,
|
|
198
|
+
turnId: input.subscription.turnId,
|
|
199
|
+
...(canAdvance && observed.maxSeq > (input.subscription.remoteEventSeq ?? 0) ? { remoteEventSeq: observed.maxSeq } : {}),
|
|
200
|
+
...(canAdvance && observed.lastItemId ? { remoteEventItemId: observed.lastItemId } : {}),
|
|
201
|
+
...(canAdvance && result.nextAfter ? { remoteEventCursor: result.nextAfter } : {}),
|
|
202
|
+
});
|
|
151
203
|
}
|
|
204
|
+
return {
|
|
205
|
+
status: "ok",
|
|
206
|
+
maxSeq: observed.maxSeq,
|
|
207
|
+
...(result.nextAfter ? { nextAfter: result.nextAfter } : {}),
|
|
208
|
+
...(observed.lastItemId ? { lastItemId: observed.lastItemId } : {}),
|
|
209
|
+
};
|
|
152
210
|
}
|
|
153
211
|
catch (error) {
|
|
212
|
+
if (crewMachineId !== input.config.machineId && remoteCursorEnabled && isRemoteEventCursorCompatibilityFailure(error)) {
|
|
213
|
+
remoteForwardCursorConfirmed.delete(crewMachineId);
|
|
214
|
+
legacyRemoteCursorFallbackUntil.set(crewMachineId, Date.now() + legacyRemoteCursorFallbackMs);
|
|
215
|
+
}
|
|
154
216
|
await appendOrchestrationAudit(input.config, "orchestration.subscription.catchup_failed", `Subscription catch-up failed for ${input.subscription.taskId}`, {
|
|
155
217
|
taskId: input.subscription.taskId,
|
|
156
218
|
crewMachineId,
|
|
@@ -158,17 +220,88 @@ export async function catchUpSubscriptionEvents(input) {
|
|
|
158
220
|
turnId: input.subscription.turnId,
|
|
159
221
|
error: error instanceof Error ? error.message : String(error),
|
|
160
222
|
});
|
|
223
|
+
return { status: "failed" };
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
function canUseRemoteEventCursor(crewMachineId) {
|
|
227
|
+
const until = legacyRemoteCursorFallbackUntil.get(crewMachineId);
|
|
228
|
+
if (until === undefined)
|
|
229
|
+
return remotePeerAdvertisesForwardCursor(crewMachineId);
|
|
230
|
+
if (until <= Date.now()) {
|
|
231
|
+
legacyRemoteCursorFallbackUntil.delete(crewMachineId);
|
|
232
|
+
return remotePeerAdvertisesForwardCursor(crewMachineId);
|
|
233
|
+
}
|
|
234
|
+
return false;
|
|
235
|
+
}
|
|
236
|
+
export function remoteEventCursorForSubscription(crewMachineId, subscription) {
|
|
237
|
+
const fallbackUntil = legacyRemoteCursorFallbackUntil.get(crewMachineId);
|
|
238
|
+
if (fallbackUntil !== undefined && fallbackUntil > Date.now())
|
|
239
|
+
return undefined;
|
|
240
|
+
if (fallbackUntil !== undefined)
|
|
241
|
+
legacyRemoteCursorFallbackUntil.delete(crewMachineId);
|
|
242
|
+
const advertised = remotePeerForwardCursorSupport(crewMachineId);
|
|
243
|
+
// An explicit old version wins over a stale persisted cursor (for example,
|
|
244
|
+
// after a peer downgrade). Unknown manifests are different: a persisted
|
|
245
|
+
// cursor is still positive evidence until the peer identity is refreshed.
|
|
246
|
+
if (advertised === false)
|
|
247
|
+
return undefined;
|
|
248
|
+
// A persisted cursor is positive evidence from a previous successful
|
|
249
|
+
// forward-page response. It remains usable even when the peer's manifest is
|
|
250
|
+
// temporarily unavailable during reconnect.
|
|
251
|
+
if (subscription.remoteEventCursor)
|
|
252
|
+
return subscription.remoteEventCursor;
|
|
253
|
+
if (advertised !== true || !canUseRemoteEventCursor(crewMachineId))
|
|
254
|
+
return undefined;
|
|
255
|
+
return subscription.remoteEventSeq
|
|
256
|
+
? `event_${String(subscription.remoteEventSeq).padStart(12, "0")}`
|
|
257
|
+
: initialRemoteEventCursor;
|
|
258
|
+
}
|
|
259
|
+
function remotePeerAdvertisesForwardCursor(crewMachineId) {
|
|
260
|
+
return remotePeerForwardCursorSupport(crewMachineId) === true;
|
|
261
|
+
}
|
|
262
|
+
function remotePeerForwardCursorSupport(crewMachineId) {
|
|
263
|
+
const machine = relayMachineRegistry.get(crewMachineId);
|
|
264
|
+
const daemonVersion = machine?.capabilities.daemonVersion?.trim();
|
|
265
|
+
const advertisedFeature = machine?.capabilities.runtimeEvents?.forwardCursor;
|
|
266
|
+
if (advertisedFeature === false)
|
|
267
|
+
return false;
|
|
268
|
+
if (!daemonVersion) {
|
|
269
|
+
if (advertisedFeature === true || remoteForwardCursorConfirmed.has(crewMachineId))
|
|
270
|
+
return true;
|
|
271
|
+
return undefined;
|
|
161
272
|
}
|
|
273
|
+
// Published beta495 introduced the append-only eventsPage `after` field.
|
|
274
|
+
// Workspace daemons also publish an explicit feature bit. The historical
|
|
275
|
+
// package version 0.1.0 alone is not evidence: older development daemons
|
|
276
|
+
// used that same version while rejecting `after`.
|
|
277
|
+
const match = /^0\.1\.0-beta\.(\d+)$/.exec(daemonVersion);
|
|
278
|
+
if (match && Number(match[1]) < forwardEventCursorSinceBeta)
|
|
279
|
+
return false;
|
|
280
|
+
if (advertisedFeature === true || remoteForwardCursorConfirmed.has(crewMachineId))
|
|
281
|
+
return true;
|
|
282
|
+
if (match)
|
|
283
|
+
return Number(match[1]) >= forwardEventCursorSinceBeta;
|
|
284
|
+
return undefined;
|
|
285
|
+
}
|
|
286
|
+
function isRemoteEventCursorCompatibilityFailure(error) {
|
|
287
|
+
if (!(error instanceof RemoteCommandError))
|
|
288
|
+
return false;
|
|
289
|
+
return error.code === "REMOTE_COMMAND_TIMEOUT" ||
|
|
290
|
+
error.code === "REMOTE_MACHINE_ERROR" ||
|
|
291
|
+
/INVALID|UNKNOWN|UNRECOGNIZED|SCHEMA|PROTOCOL|CURSOR|AFTER/u.test(error.code);
|
|
162
292
|
}
|
|
163
293
|
async function observeSubscriptionRuntimeEvents(input) {
|
|
164
294
|
const crewMachineId = input.crewMachineId ?? input.subscription.crewMachineId ?? input.config.machineId;
|
|
165
295
|
const skipThroughSeq = input.skipThroughSeq ?? 0;
|
|
166
296
|
let maxSeq = skipThroughSeq;
|
|
167
297
|
let observed = 0;
|
|
298
|
+
let lastItemId;
|
|
168
299
|
for (const event of [...input.events].sort((left, right) => (left.seq ?? 0) - (right.seq ?? 0))) {
|
|
169
300
|
const seq = event.seq ?? 0;
|
|
170
301
|
if (seq > 0)
|
|
171
302
|
maxSeq = Math.max(maxSeq, seq);
|
|
303
|
+
if (event.itemId && seq >= maxSeq)
|
|
304
|
+
lastItemId = event.itemId;
|
|
172
305
|
if (seq > 0 && seq <= skipThroughSeq)
|
|
173
306
|
continue;
|
|
174
307
|
if (event.turnId !== input.subscription.turnId)
|
|
@@ -190,7 +323,7 @@ async function observeSubscriptionRuntimeEvents(input) {
|
|
|
190
323
|
});
|
|
191
324
|
observed += 1;
|
|
192
325
|
}
|
|
193
|
-
return { maxSeq, observed };
|
|
326
|
+
return { maxSeq, observed, ...(lastItemId ? { lastItemId } : {}) };
|
|
194
327
|
}
|
|
195
328
|
function attentionFromEvent(payload) {
|
|
196
329
|
if (payload.type === "option_request") {
|
|
@@ -49,6 +49,7 @@ export async function registerMachine(ws, config) {
|
|
|
49
49
|
acpxVersion: capabilities.version,
|
|
50
50
|
daemonVersion: daemonPackage.version,
|
|
51
51
|
...(capabilities.turnPrimitives ? { turnPrimitives: capabilities.turnPrimitives } : {}),
|
|
52
|
+
runtimeEvents: { forwardCursor: true },
|
|
52
53
|
sessionPrimitives: runtimePrimitiveSupportForCapabilities(capabilities.sessionPrimitives, "native"),
|
|
53
54
|
...(capabilities.sidechat ? { sidechat: capabilities.sidechat } : {}),
|
|
54
55
|
directoryBrowser: true,
|
|
@@ -51,6 +51,10 @@ export async function publishRemoteControllerCommand(config, input) {
|
|
|
51
51
|
// durable daemon connection (with its own cursor) must remain the
|
|
52
52
|
// only consumer that can acknowledge those commands.
|
|
53
53
|
cursor: Number.MAX_SAFE_INTEGER,
|
|
54
|
+
// This ephemeral RPC only needs the session-to-machine routing table
|
|
55
|
+
// to publish one targeted command. Do not make every status/read call
|
|
56
|
+
// download the account's retained transcript and artifact payloads.
|
|
57
|
+
snapshotMode: "routing",
|
|
54
58
|
}));
|
|
55
59
|
});
|
|
56
60
|
ws.addEventListener("error", () => {
|
|
@@ -32,11 +32,22 @@ export declare function selectThreadEventsAfterItem(client: HeadlessRuntimeClien
|
|
|
32
32
|
events: HeadlessRuntimeEvent[];
|
|
33
33
|
anchorFound: boolean;
|
|
34
34
|
}>;
|
|
35
|
+
/**
|
|
36
|
+
* Read a forward page from the append-only runtime event sequence. Unlike an
|
|
37
|
+
* item anchor this cursor advances for every provider event, including
|
|
38
|
+
* repeated updates to one item and terminal events without an item id.
|
|
39
|
+
*/
|
|
40
|
+
export declare function selectThreadEventsAfterCursor(client: HeadlessRuntimeClient, threadId: string, afterCursor: string, limit: number): Promise<{
|
|
41
|
+
events: HeadlessRuntimeEvent[];
|
|
42
|
+
hasMoreBefore: boolean;
|
|
43
|
+
nextAfter?: string;
|
|
44
|
+
}>;
|
|
35
45
|
export declare function historicalEventsFromRuntimeEvents(runtimeEvents: HeadlessRuntimeEvent[]): RuntimeHistoricalEvent[];
|
|
36
|
-
export declare function limitHistoricalEventsByBytes(events: RuntimeHistoricalEvent[], maxBytes: number | undefined): {
|
|
46
|
+
export declare function limitHistoricalEventsByBytes(events: RuntimeHistoricalEvent[], maxBytes: number | undefined, direction?: "before" | "after"): {
|
|
37
47
|
events: RuntimeHistoricalEvent[];
|
|
38
48
|
truncated: boolean;
|
|
39
49
|
nextBefore?: string;
|
|
50
|
+
nextAfter?: string;
|
|
40
51
|
};
|
|
41
52
|
export declare function terminalHistoricalEventFromTurn(turn: Turn & {
|
|
42
53
|
status: "completed" | "interrupted" | "failed";
|
|
@@ -330,6 +330,26 @@ export async function selectThreadEventsAfterItem(client, threadId, afterItemId,
|
|
|
330
330
|
events: sortRuntimeEventsByCursor([...selected, ...terminalEvents]),
|
|
331
331
|
};
|
|
332
332
|
}
|
|
333
|
+
/**
|
|
334
|
+
* Read a forward page from the append-only runtime event sequence. Unlike an
|
|
335
|
+
* item anchor this cursor advances for every provider event, including
|
|
336
|
+
* repeated updates to one item and terminal events without an item id.
|
|
337
|
+
*/
|
|
338
|
+
export async function selectThreadEventsAfterCursor(client, threadId, afterCursor, limit) {
|
|
339
|
+
const page = await client.listThreadEvents({
|
|
340
|
+
threadId,
|
|
341
|
+
order: "asc",
|
|
342
|
+
afterCursor,
|
|
343
|
+
limit,
|
|
344
|
+
});
|
|
345
|
+
const events = sortRuntimeEventsByCursor(page.events);
|
|
346
|
+
const nextAfter = events.at(-1)?.cursor;
|
|
347
|
+
return {
|
|
348
|
+
events,
|
|
349
|
+
hasMoreBefore: false,
|
|
350
|
+
...(nextAfter ? { nextAfter } : {}),
|
|
351
|
+
};
|
|
352
|
+
}
|
|
333
353
|
export function historicalEventsFromRuntimeEvents(runtimeEvents) {
|
|
334
354
|
const deltaItemIds = runtimeEvents
|
|
335
355
|
.filter((event) => event.method === "item/delta")
|
|
@@ -384,27 +404,57 @@ export function historicalEventsFromRuntimeEvents(runtimeEvents) {
|
|
|
384
404
|
.sort((a, b) => (a.occurredAt ?? 0) - (b.occurredAt ?? 0) || (a.seq ?? 0) - (b.seq ?? 0))
|
|
385
405
|
.map((event, index) => ({ ...event, seq: event.seq ?? index + 1 }));
|
|
386
406
|
}
|
|
387
|
-
export function limitHistoricalEventsByBytes(events, maxBytes) {
|
|
407
|
+
export function limitHistoricalEventsByBytes(events, maxBytes, direction = "before") {
|
|
388
408
|
if (maxBytes === undefined || !Number.isFinite(maxBytes) || maxBytes <= 0)
|
|
389
409
|
return { events, truncated: false };
|
|
390
410
|
const groups = groupHistoricalEvents(events);
|
|
391
411
|
const selectedNewestFirst = [];
|
|
392
412
|
let bytes = 0;
|
|
393
|
-
|
|
413
|
+
const orderedGroups = direction === "after" ? groups : [...groups].reverse();
|
|
414
|
+
for (const group of orderedGroups) {
|
|
394
415
|
const groupBytes = Buffer.byteLength(JSON.stringify(group.events), "utf8");
|
|
395
416
|
if (bytes + groupBytes > maxBytes) {
|
|
396
417
|
if (selectedNewestFirst.length === 0) {
|
|
418
|
+
if (direction === "after") {
|
|
419
|
+
// A single item can be larger than the page budget. Emit a marker
|
|
420
|
+
// for that item, then consume as many following complete groups as
|
|
421
|
+
// fit. The cursor advances only through groups represented in the
|
|
422
|
+
// response; it must never jump to the raw page tail.
|
|
423
|
+
const marker = historyTruncatedMarker(group.events, maxBytes, group.events.length);
|
|
424
|
+
const kept = [];
|
|
425
|
+
let markerBytes = Buffer.byteLength(JSON.stringify([marker]), "utf8");
|
|
426
|
+
for (const following of orderedGroups.slice(1)) {
|
|
427
|
+
const followingBytes = Buffer.byteLength(JSON.stringify(following.events), "utf8");
|
|
428
|
+
if (markerBytes + followingBytes > maxBytes)
|
|
429
|
+
break;
|
|
430
|
+
kept.push(following);
|
|
431
|
+
markerBytes += followingBytes;
|
|
432
|
+
}
|
|
433
|
+
const consumed = [group, ...kept];
|
|
434
|
+
const lastConsumedEvent = consumed.at(-1)?.events.at(-1);
|
|
435
|
+
return {
|
|
436
|
+
events: [marker, ...kept.flatMap((item) => item.events)],
|
|
437
|
+
truncated: true,
|
|
438
|
+
...(lastConsumedEvent?.seq ? { nextAfter: headlessCursorFromSeq(lastConsumedEvent.seq) } : {}),
|
|
439
|
+
};
|
|
440
|
+
}
|
|
397
441
|
return {
|
|
398
442
|
events: [historyTruncatedMarker(group.events, maxBytes, group.events.length)],
|
|
399
443
|
truncated: true,
|
|
400
|
-
...(
|
|
444
|
+
...(direction === "before"
|
|
445
|
+
? (group.oldestCursor ? { nextBefore: group.oldestCursor } : {})
|
|
446
|
+
: {}),
|
|
401
447
|
};
|
|
402
448
|
}
|
|
403
|
-
const keptGroups = [...selectedNewestFirst].reverse();
|
|
449
|
+
const keptGroups = direction === "after" ? selectedNewestFirst : [...selectedNewestFirst].reverse();
|
|
450
|
+
const lastKeptEvent = keptGroups.at(-1)?.events.at(-1);
|
|
404
451
|
return {
|
|
405
452
|
events: keptGroups.flatMap((item) => item.events),
|
|
406
453
|
truncated: true,
|
|
407
|
-
...(keptGroups[0]?.oldestCursor ? { nextBefore: keptGroups[0].oldestCursor } : {}),
|
|
454
|
+
...(direction === "before" && keptGroups[0]?.oldestCursor ? { nextBefore: keptGroups[0].oldestCursor } : {}),
|
|
455
|
+
...(direction === "after" && lastKeptEvent?.seq
|
|
456
|
+
? { nextAfter: headlessCursorFromSeq(lastKeptEvent.seq) }
|
|
457
|
+
: {}),
|
|
408
458
|
};
|
|
409
459
|
}
|
|
410
460
|
selectedNewestFirst.push(group);
|
|
@@ -1,20 +1,25 @@
|
|
|
1
|
-
import { limitHistoricalEventsByBytes, historicalEventsFromThread, mapHeadlessRuntimeEventsToHistoricalEvents, selectThreadEventsAfterItem, selectThreadEventsBeforeCursor, selectThreadEventsBeforeItem, selectThreadEventsForTurn, } from "./headless-event-projector.js";
|
|
1
|
+
import { limitHistoricalEventsByBytes, historicalEventsFromThread, mapHeadlessRuntimeEventsToHistoricalEvents, selectThreadEventsAfterCursor, selectThreadEventsAfterItem, selectThreadEventsBeforeCursor, selectThreadEventsBeforeItem, selectThreadEventsForTurn, } from "./headless-event-projector.js";
|
|
2
2
|
const defaultEventLimit = 200;
|
|
3
3
|
export async function readHeadlessHistoricalEventPage(input) {
|
|
4
|
+
assertValidHeadlessEventCursor(input.after);
|
|
4
5
|
assertValidHeadlessEventCursor(input.before);
|
|
5
6
|
const limitItems = clampLimit(input.limitItems);
|
|
6
7
|
const selection = await selectHeadlessRuntimeEvents(input, limitItems);
|
|
7
8
|
const runtimeEvents = selection.events;
|
|
8
9
|
const allEvents = await authoritativeTurnEvents(input, runtimeEvents) ?? mapHeadlessRuntimeEventsToHistoricalEvents(runtimeEvents);
|
|
9
|
-
const
|
|
10
|
+
const visibleEvents = input.turnId ? allEvents.filter((event) => event.turnId === input.turnId) : allEvents;
|
|
11
|
+
const limited = limitHistoricalEventsByBytes(visibleEvents, input.maxBytes, input.after ? "after" : "before");
|
|
10
12
|
const includeSessionHead = !input.turnId && !input.before && !input.beforeItemId;
|
|
11
13
|
const head = includeSessionHead ? latestHeadFromEvents(allEvents) : {};
|
|
12
14
|
const nextBefore = limited.nextBefore ?? selection.nextBefore;
|
|
13
15
|
const hasMoreBefore = Boolean((selection.hasMoreBefore || limited.truncated) && nextBefore);
|
|
16
|
+
const nextAfter = limited.truncated ? limited.nextAfter : selection.nextAfter;
|
|
14
17
|
return {
|
|
15
18
|
events: limited.events,
|
|
16
19
|
hasMoreBefore,
|
|
17
20
|
...(hasMoreBefore && nextBefore ? { nextBefore } : {}),
|
|
21
|
+
...(nextAfter ? { nextAfter } : {}),
|
|
22
|
+
...(input.after ? { cursorSupported: true } : {}),
|
|
18
23
|
...(head.currentHead ? { currentHead: head.currentHead } : {}),
|
|
19
24
|
...(head.lastTurnId ? { lastTurnId: head.lastTurnId } : {}),
|
|
20
25
|
};
|
|
@@ -28,7 +33,7 @@ async function authoritativeTurnEvents(input, runtimeEvents) {
|
|
|
28
33
|
// plus a repeated terminal snapshot for the same item, or a late native
|
|
29
34
|
// reconciliation. Identity is itemId/turnId, never text, so legitimate
|
|
30
35
|
// repeated answers from distinct items remain visible.
|
|
31
|
-
if (input.before || input.beforeItemId || input.afterItemId)
|
|
36
|
+
if (input.after || input.before || input.beforeItemId || input.afterItemId)
|
|
32
37
|
return undefined;
|
|
33
38
|
// A few internal readers (and older provider adapters during rolling
|
|
34
39
|
// upgrades) only expose the event reader. Canonical rebuild is an
|
|
@@ -69,13 +74,20 @@ async function listAllThreadTurns(client, threadId) {
|
|
|
69
74
|
return turns;
|
|
70
75
|
}
|
|
71
76
|
async function selectHeadlessRuntimeEvents(input, limitItems) {
|
|
72
|
-
if (input.
|
|
73
|
-
return await
|
|
77
|
+
if (input.after) {
|
|
78
|
+
return await selectThreadEventsAfterCursor(input.client, input.threadId, input.after, limitItems);
|
|
74
79
|
}
|
|
80
|
+
// An after-item anchor is a thread-level forward cursor. It must take
|
|
81
|
+
// precedence over turn filtering so callers can advance a subscription
|
|
82
|
+
// without re-reading the first page of a long turn on every poll. The
|
|
83
|
+
// runtime-truth layer filters the returned page back to the requested turn.
|
|
75
84
|
if (input.afterItemId) {
|
|
76
85
|
const selection = await selectThreadEventsAfterItem(input.client, input.threadId, input.afterItemId, limitItems);
|
|
77
86
|
return { events: selection.events, hasMoreBefore: selection.anchorFound };
|
|
78
87
|
}
|
|
88
|
+
if (input.turnId) {
|
|
89
|
+
return await selectThreadEventsForTurn(input.client, input.threadId, input.turnId, limitItems);
|
|
90
|
+
}
|
|
79
91
|
if (input.beforeItemId) {
|
|
80
92
|
const selection = await selectThreadEventsBeforeItem(input.client, input.threadId, input.beforeItemId, limitItems);
|
|
81
93
|
return { events: selection.events, hasMoreBefore: selection.hasMoreBefore };
|
|
@@ -45,6 +45,9 @@ export declare function machineList(config: DaemonConfig): {
|
|
|
45
45
|
turnPrimitives?: {
|
|
46
46
|
steer: "native" | "unsupported";
|
|
47
47
|
};
|
|
48
|
+
runtimeEvents?: {
|
|
49
|
+
forwardCursor: boolean;
|
|
50
|
+
};
|
|
48
51
|
sidechat?: import("../../../../../packages/shared/dist/index.js").SidechatCapability;
|
|
49
52
|
directoryBrowser?: boolean;
|
|
50
53
|
filePreview?: boolean;
|
|
@@ -98,7 +98,21 @@ export async function refreshNativeHistoryForLoadedSession(session, config, requ
|
|
|
98
98
|
const entry = { generation, promise: operation, startedAt: Date.now() };
|
|
99
99
|
orchestrationRefreshes.set(refreshKey, entry);
|
|
100
100
|
try {
|
|
101
|
-
const
|
|
101
|
+
const outcome = await Promise.race([
|
|
102
|
+
operation.then((result) => ({ kind: "result", result })),
|
|
103
|
+
new Promise((resolve) => setTimeout(() => resolve({ kind: "timeout" }), orchestrationRefreshMaxWaitMs)),
|
|
104
|
+
]);
|
|
105
|
+
if (outcome.kind === "timeout") {
|
|
106
|
+
entry.abandoned = true;
|
|
107
|
+
if (orchestrationRefreshes.get(refreshKey) === entry)
|
|
108
|
+
orchestrationRefreshes.delete(refreshKey);
|
|
109
|
+
orchestrationRefreshGenerations.set(refreshKey, (orchestrationRefreshGenerations.get(refreshKey) ?? 0) + 1);
|
|
110
|
+
// The first caller must have the same bounded wait as followers. Drop
|
|
111
|
+
// the provider client so the old IPC cannot poison the next refresh.
|
|
112
|
+
void getHeadlessRuntimeManager(session.cwd).invalidateClientForBinding(session.runtimeBinding, session.cwd);
|
|
113
|
+
return sessions.get(session.sessionId) ?? session;
|
|
114
|
+
}
|
|
115
|
+
const result = outcome.result;
|
|
102
116
|
if (!entry.abandoned && generation !== (orchestrationRefreshGenerations.get(refreshKey) ?? 0)) {
|
|
103
117
|
if (orchestrationRefreshes.get(refreshKey) === entry)
|
|
104
118
|
orchestrationRefreshes.delete(refreshKey);
|
|
@@ -17,7 +17,14 @@ export async function runtimeRead(config, input) {
|
|
|
17
17
|
// Reconcile that history before projecting turn.result/status so an
|
|
18
18
|
// upgrade/restart repairs old same-turn duplicates through the official
|
|
19
19
|
// read path, without requiring an internal syncNativeHistory call.
|
|
20
|
-
|
|
20
|
+
// Event catch-up reads the authoritative Headless event log and must stay
|
|
21
|
+
// independent from the provider-native history refresh. The latter can
|
|
22
|
+
// involve a slow/remote provider IPC call; putting it in every polling
|
|
23
|
+
// page made one stale Crew subscription block all other orchestration.
|
|
24
|
+
// Result/status/list retain the refresh for legacy native-history repair.
|
|
25
|
+
if (input.action !== "turn.eventsPage") {
|
|
26
|
+
session = await refreshNativeHistoryForLoadedSession(session, config, `runtime_truth_refresh_${input.requestId}`);
|
|
27
|
+
}
|
|
21
28
|
runtimeSessionId = runtimeSessionIdForSession(session);
|
|
22
29
|
if (!runtimeSessionId)
|
|
23
30
|
return unreachable(input, `Session ${input.sessionId} does not have a runtime session binding`, undefined, "SESSION_BINDING_MISSING");
|
|
@@ -90,6 +97,7 @@ async function readTurnList(input, session, runtimeSessionId) {
|
|
|
90
97
|
before: input.before,
|
|
91
98
|
beforeItemId: input.beforeItemId,
|
|
92
99
|
afterItemId: input.afterItemId,
|
|
100
|
+
after: input.after,
|
|
93
101
|
limitItems: input.limitItems ?? 400,
|
|
94
102
|
maxBytes: input.maxBytes ?? 240_000,
|
|
95
103
|
});
|
|
@@ -207,6 +215,7 @@ async function readTurnEventsPage(input, session, runtimeSessionId) {
|
|
|
207
215
|
before: input.before,
|
|
208
216
|
beforeItemId: input.beforeItemId,
|
|
209
217
|
afterItemId: input.afterItemId,
|
|
218
|
+
after: input.after,
|
|
210
219
|
limitItems: input.limitItems ?? 300,
|
|
211
220
|
maxBytes: input.maxBytes ?? 260_000,
|
|
212
221
|
});
|
|
@@ -223,6 +232,8 @@ async function readTurnEventsPage(input, session, runtimeSessionId) {
|
|
|
223
232
|
events: events.map((event) => runtimeReadEvent(event, input.turnId, runtimeTurnId)),
|
|
224
233
|
hasMoreBefore: page.hasMoreBefore,
|
|
225
234
|
nextBefore: page.nextBefore,
|
|
235
|
+
...(page.nextAfter ? { nextAfter: page.nextAfter } : {}),
|
|
236
|
+
...(page.cursorSupported ? { cursorSupported: true } : {}),
|
|
226
237
|
...(page.currentHead ? { currentHead: page.currentHead } : {}),
|
|
227
238
|
...(page.lastTurnId ? { lastTurnId: page.lastTurnId } : {}),
|
|
228
239
|
};
|
|
@@ -252,6 +263,7 @@ async function readRuntimePage(session, input) {
|
|
|
252
263
|
...(input.turnId ? { turnId: input.turnId } : {}),
|
|
253
264
|
...(input.before ? { before: input.before } : {}),
|
|
254
265
|
...(input.beforeItemId ? { beforeItemId: input.beforeItemId } : {}),
|
|
266
|
+
...(input.after ? { after: input.after } : {}),
|
|
255
267
|
...(input.afterItemId ? { afterItemId: input.afterItemId } : {}),
|
|
256
268
|
limitItems: input.limitItems ?? 200,
|
|
257
269
|
maxBytes: input.maxBytes ?? 200_000,
|
package/apps/daemon/package.json
CHANGED
package/package.json
CHANGED
|
@@ -250,6 +250,10 @@ export type RuntimeHistoricalEventPage = {
|
|
|
250
250
|
events: RuntimeHistoricalEvent[];
|
|
251
251
|
hasMoreBefore?: boolean;
|
|
252
252
|
nextBefore?: string;
|
|
253
|
+
/** Forward event cursor for incremental readers. */
|
|
254
|
+
nextAfter?: string;
|
|
255
|
+
/** True when the reader understood a forward cursor, including an empty page. */
|
|
256
|
+
cursorSupported?: boolean;
|
|
253
257
|
currentHead?: string;
|
|
254
258
|
lastTurnId?: string;
|
|
255
259
|
};
|
|
@@ -136,6 +136,9 @@ export declare const capabilitiesSchema: z.ZodObject<{
|
|
|
136
136
|
unsupported: "unsupported";
|
|
137
137
|
}>;
|
|
138
138
|
}, z.core.$strip>>;
|
|
139
|
+
runtimeEvents: z.ZodOptional<z.ZodObject<{
|
|
140
|
+
forwardCursor: z.ZodBoolean;
|
|
141
|
+
}, z.core.$strip>>;
|
|
139
142
|
sidechat: z.ZodOptional<z.ZodObject<{
|
|
140
143
|
strategy: z.ZodEnum<{
|
|
141
144
|
native: "native";
|
|
@@ -289,6 +292,9 @@ export declare const machineSnapshotSchema: z.ZodObject<{
|
|
|
289
292
|
unsupported: "unsupported";
|
|
290
293
|
}>;
|
|
291
294
|
}, z.core.$strip>>;
|
|
295
|
+
runtimeEvents: z.ZodOptional<z.ZodObject<{
|
|
296
|
+
forwardCursor: z.ZodBoolean;
|
|
297
|
+
}, z.core.$strip>>;
|
|
292
298
|
sidechat: z.ZodOptional<z.ZodObject<{
|
|
293
299
|
strategy: z.ZodEnum<{
|
|
294
300
|
native: "native";
|
|
@@ -1113,6 +1119,8 @@ export declare const runtimeReadCommandResultSchema: z.ZodDiscriminatedUnion<[z.
|
|
|
1113
1119
|
}, z.core.$strict>>;
|
|
1114
1120
|
hasMoreBefore: z.ZodOptional<z.ZodBoolean>;
|
|
1115
1121
|
nextBefore: z.ZodOptional<z.ZodString>;
|
|
1122
|
+
nextAfter: z.ZodOptional<z.ZodString>;
|
|
1123
|
+
cursorSupported: z.ZodOptional<z.ZodBoolean>;
|
|
1116
1124
|
currentHead: z.ZodOptional<z.ZodString>;
|
|
1117
1125
|
lastTurnId: z.ZodOptional<z.ZodString>;
|
|
1118
1126
|
errorCode: z.ZodOptional<z.ZodString>;
|
|
@@ -72,6 +72,9 @@ export const capabilitiesSchema = z.object({
|
|
|
72
72
|
daemonVersion: z.string().optional(),
|
|
73
73
|
sessionPrimitives: sessionPrimitivesSchema.optional(),
|
|
74
74
|
turnPrimitives: turnPrimitivesSchema.optional(),
|
|
75
|
+
runtimeEvents: z.object({
|
|
76
|
+
forwardCursor: z.boolean(),
|
|
77
|
+
}).optional(),
|
|
75
78
|
sidechat: sidechatCapabilitySchema.optional(),
|
|
76
79
|
directoryBrowser: z.boolean().optional(),
|
|
77
80
|
filePreview: z.boolean().optional(),
|
|
@@ -487,6 +490,8 @@ export const runtimeReadCommandResultSchema = z.discriminatedUnion("action", [
|
|
|
487
490
|
events: z.array(runtimeReadEventSchema),
|
|
488
491
|
hasMoreBefore: z.boolean().optional(),
|
|
489
492
|
nextBefore: z.string().min(1).optional(),
|
|
493
|
+
nextAfter: z.string().min(1).optional(),
|
|
494
|
+
cursorSupported: z.boolean().optional(),
|
|
490
495
|
currentHead: z.string().min(1).optional(),
|
|
491
496
|
lastTurnId: z.string().min(1).optional(),
|
|
492
497
|
errorCode: z.string().min(1).optional(),
|
|
@@ -66,6 +66,9 @@ export type MachineSnapshot = {
|
|
|
66
66
|
turnPrimitives?: {
|
|
67
67
|
steer: "native" | "unsupported";
|
|
68
68
|
};
|
|
69
|
+
runtimeEvents?: {
|
|
70
|
+
forwardCursor: boolean;
|
|
71
|
+
};
|
|
69
72
|
sidechat?: SidechatCapability;
|
|
70
73
|
directoryBrowser?: boolean;
|
|
71
74
|
filePreview?: boolean;
|
|
@@ -349,6 +352,8 @@ export type RuntimeTurnEventsPageResult = {
|
|
|
349
352
|
events: RuntimeReadEvent[];
|
|
350
353
|
hasMoreBefore?: boolean;
|
|
351
354
|
nextBefore?: string;
|
|
355
|
+
nextAfter?: string;
|
|
356
|
+
cursorSupported?: boolean;
|
|
352
357
|
currentHead?: string;
|
|
353
358
|
lastTurnId?: string;
|
|
354
359
|
errorCode?: string;
|
|
@@ -138,6 +138,7 @@ export type ControllerClientMessage = {
|
|
|
138
138
|
runtimeTurnId?: string;
|
|
139
139
|
before?: string;
|
|
140
140
|
beforeItemId?: string;
|
|
141
|
+
after?: string;
|
|
141
142
|
afterItemId?: string;
|
|
142
143
|
limitItems?: number;
|
|
143
144
|
maxBytes?: number;
|
|
@@ -767,6 +768,7 @@ export type MachineCommand = {
|
|
|
767
768
|
runtimeTurnId?: string;
|
|
768
769
|
before?: string;
|
|
769
770
|
beforeItemId?: string;
|
|
771
|
+
after?: string;
|
|
770
772
|
afterItemId?: string;
|
|
771
773
|
limitItems?: number;
|
|
772
774
|
maxBytes?: number;
|
|
@@ -155,6 +155,7 @@ const controllerMessageSchema = z.union([
|
|
|
155
155
|
runtimeTurnId: z.string().min(1).optional(),
|
|
156
156
|
before: z.string().min(1).optional(),
|
|
157
157
|
beforeItemId: z.string().min(1).optional(),
|
|
158
|
+
after: z.string().min(1).optional(),
|
|
158
159
|
afterItemId: z.string().min(1).optional(),
|
|
159
160
|
limitItems: z.number().int().positive().max(1000).optional(),
|
|
160
161
|
maxBytes: z.number().int().positive().max(2_000_000).optional(),
|
|
@@ -885,6 +886,7 @@ const machineCommandSchema = z.union([
|
|
|
885
886
|
runtimeTurnId: z.string().min(1).optional(),
|
|
886
887
|
before: z.string().min(1).optional(),
|
|
887
888
|
beforeItemId: z.string().min(1).optional(),
|
|
889
|
+
after: z.string().min(1).optional(),
|
|
888
890
|
afterItemId: z.string().min(1).optional(),
|
|
889
891
|
limitItems: z.number().int().positive().max(1000).optional(),
|
|
890
892
|
maxBytes: z.number().int().positive().max(2_000_000).optional(),
|