@cydm/happy-elves 0.1.0-beta.490 → 0.1.0-beta.494
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/apps/cli/package.json +1 -1
- package/apps/daemon/dist/direct/server.js +21 -19
- package/apps/daemon/dist/relay/connection.js +27 -0
- package/apps/daemon/dist/runtime/headless-event-projector.js +53 -8
- package/apps/daemon/dist/runtime/headless-history-reader.js +11 -9
- package/apps/daemon/dist/session/lifecycle.d.ts +1 -1
- package/apps/daemon/dist/session/lifecycle.js +1 -1
- package/apps/daemon/dist/session/native-history-refresh.d.ts +19 -6
- package/apps/daemon/dist/session/native-history-refresh.js +107 -25
- package/apps/daemon/dist/session/prompt.js +78 -1
- package/apps/daemon/package.json +1 -1
- package/apps/relay/dist/http-routes.js +23 -3
- package/apps/relay/dist/packet-store.js +93 -15
- package/apps/relay/dist/relay-context.js +16 -4
- package/apps/relay/dist/storage-worker.js +4 -1
- package/apps/relay/dist/websocket.d.ts +2 -1
- package/apps/relay/dist/websocket.js +91 -14
- package/package.json +1 -1
- package/packages/client/dist/command-contracts.d.ts +1 -1
- package/packages/client/dist/command-contracts.js +4 -2
- package/packages/client/dist/direct.js +3 -3
- package/packages/client/dist/product-materializer.d.ts +12 -0
- package/packages/client/dist/product-materializer.js +24 -1
- package/packages/client/dist/transport.js +92 -80
- package/packages/headless-runtime/dist/claude-provider.js +20 -2
- package/packages/headless-runtime/dist/codex-provider.js +1 -1
- package/packages/headless-runtime/dist/core.d.ts +6 -0
- package/packages/headless-runtime/dist/core.js +66 -5
- package/packages/shared/dist/packet-relay.d.ts +6 -0
- package/packages/shared/dist/packet-relay.js +2 -0
- package/packages/shared/dist/protocol.d.ts +5 -0
- package/packages/shared/dist/protocol.js +1 -0
package/apps/cli/package.json
CHANGED
|
@@ -168,7 +168,7 @@ class DirectState {
|
|
|
168
168
|
if (!ticketToken || !await this.authorizeTicket(ticketToken)) {
|
|
169
169
|
throw directAuthError("DIRECT_AUTH_REQUIRED", "Direct snapshot requires a valid short-lived ticket");
|
|
170
170
|
}
|
|
171
|
-
return await this.packetSnapshot();
|
|
171
|
+
return await this.packetSnapshot("routing");
|
|
172
172
|
}
|
|
173
173
|
async handleText(connection, raw) {
|
|
174
174
|
let message;
|
|
@@ -216,7 +216,7 @@ class DirectState {
|
|
|
216
216
|
nodeId,
|
|
217
217
|
nodeKind: "controller",
|
|
218
218
|
});
|
|
219
|
-
connection.sendWireMessage(await this.packetSnapshot());
|
|
219
|
+
connection.sendWireMessage(await this.packetSnapshot(message.snapshotMode ?? "full"));
|
|
220
220
|
}
|
|
221
221
|
async handlePacketMessage(connection, message) {
|
|
222
222
|
if (!await this.connectionAuthorized(connection))
|
|
@@ -600,32 +600,34 @@ class DirectState {
|
|
|
600
600
|
}
|
|
601
601
|
}
|
|
602
602
|
}
|
|
603
|
-
async packetSnapshot() {
|
|
604
|
-
const
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
}),
|
|
610
|
-
];
|
|
603
|
+
async packetSnapshot(snapshotMode = "full") {
|
|
604
|
+
const manifestPacket = await this.retainedPacket(`node:${this.config.machineId}:manifest`, {
|
|
605
|
+
type: "machine:register",
|
|
606
|
+
encryptedMetadata: await this.encryptedMachineMetadata(),
|
|
607
|
+
capabilities: await this.capabilities(),
|
|
608
|
+
});
|
|
611
609
|
for (const stored of await loadSessionProjections().catch(() => [])) {
|
|
612
610
|
if (!sessions.has(stored.sessionId))
|
|
613
611
|
sessions.set(stored.sessionId, stored);
|
|
614
612
|
}
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
613
|
+
const sessionRoutes = [...sessions.keys()].map((sessionId) => ({ sessionId, nodeId: this.config.machineId }));
|
|
614
|
+
const retainedPackets = [manifestPacket];
|
|
615
|
+
if (snapshotMode === "full")
|
|
616
|
+
for (const session of sessions.values()) {
|
|
617
|
+
retainedPackets.push(await this.retainedPacket(`node:${this.config.machineId}:session:${session.sessionId}`, {
|
|
618
|
+
type: "machine:sessionReady",
|
|
619
|
+
requestId: `req_direct_projection_${this.config.machineId}_${session.sessionId}_${Date.now()}`,
|
|
620
|
+
sessionId: session.sessionId,
|
|
621
|
+
...sessionReadyProjectionFields(session),
|
|
622
|
+
encryptedMetadata: await encryptedSessionMetadata(this.config, session.sessionId, session, projectionMetadataExtra(session)),
|
|
623
|
+
}));
|
|
624
|
+
}
|
|
624
625
|
return {
|
|
625
626
|
type: "server:snapshot",
|
|
626
627
|
cursor: this.packetSeq,
|
|
627
628
|
presence: [this.presence()],
|
|
628
629
|
retainedPackets,
|
|
630
|
+
...(sessionRoutes.length > 0 ? { sessionRoutes } : {}),
|
|
629
631
|
};
|
|
630
632
|
}
|
|
631
633
|
presence() {
|
|
@@ -84,6 +84,22 @@ export async function handleServerMessage(ws, config, message) {
|
|
|
84
84
|
capability: message.capability,
|
|
85
85
|
},
|
|
86
86
|
});
|
|
87
|
+
// A detached relay operation failure is recoverable only after the
|
|
88
|
+
// authenticated stream is recreated. Keeping this socket open would
|
|
89
|
+
// leave durable packets stranded: the relay has already rejected the
|
|
90
|
+
// replay/live send, while the daemon's normal outbox drain is only
|
|
91
|
+
// kicked on a new socket. Close the stream for storage-generation
|
|
92
|
+
// failures so start() performs its bounded reconnect loop. Do not do
|
|
93
|
+
// this for ordinary command validation errors.
|
|
94
|
+
if (shouldReconnectAfterRelayError(message.code) && ws.readyState === WebSocket.OPEN) {
|
|
95
|
+
try {
|
|
96
|
+
ws.close();
|
|
97
|
+
}
|
|
98
|
+
catch {
|
|
99
|
+
// The close event/error path still performs cleanup if the socket
|
|
100
|
+
// implementation rejects a redundant close.
|
|
101
|
+
}
|
|
102
|
+
}
|
|
87
103
|
return;
|
|
88
104
|
}
|
|
89
105
|
if (message.type === "machine:createSession") {
|
|
@@ -268,6 +284,14 @@ async function handleRelayWireMessage(ws, config, message) {
|
|
|
268
284
|
packetId: message.packetId,
|
|
269
285
|
},
|
|
270
286
|
});
|
|
287
|
+
if (shouldReconnectAfterRelayError(message.code) && ws.readyState === WebSocket.OPEN) {
|
|
288
|
+
try {
|
|
289
|
+
ws.close();
|
|
290
|
+
}
|
|
291
|
+
catch {
|
|
292
|
+
// Cleanup is also driven by the close/error listeners below.
|
|
293
|
+
}
|
|
294
|
+
}
|
|
271
295
|
return;
|
|
272
296
|
}
|
|
273
297
|
if (message.type !== "server:packet")
|
|
@@ -338,6 +362,9 @@ async function handleRelayWireMessage(ws, config, message) {
|
|
|
338
362
|
ackIncomingPacket(ws, message);
|
|
339
363
|
}
|
|
340
364
|
}
|
|
365
|
+
function shouldReconnectAfterRelayError(code) {
|
|
366
|
+
return code === "RELAY_STORAGE_TIMEOUT" || code === "RELAY_STORAGE_UNAVAILABLE" || code === "RELAY_STORAGE_CLOSED";
|
|
367
|
+
}
|
|
341
368
|
function errorCode(error) {
|
|
342
369
|
if (!error || typeof error !== "object")
|
|
343
370
|
return undefined;
|
|
@@ -86,11 +86,14 @@ export function mapHeadlessRuntimeEvent(event, state = createHeadlessRuntimeEven
|
|
|
86
86
|
return text ? [{ type: "text_delta", text, stream: "thought", tag: "reasoning", ...itemIdentity }] : [];
|
|
87
87
|
}
|
|
88
88
|
if (event.method === "item/toolCall" || event.method === "item/toolResult" || event.method === "item/fileChange") {
|
|
89
|
+
const providerItem = recordFromUnknown(params.item);
|
|
90
|
+
const status = stringFromRecord(params, "status") ?? (providerItem ? stringFromRecord(providerItem, "status") : undefined);
|
|
89
91
|
return [{
|
|
90
92
|
type: "tool_call",
|
|
91
93
|
...itemIdentity,
|
|
92
94
|
title: providerToolTitle(params) ?? titleFromMethod(event.method),
|
|
93
95
|
tag: event.method,
|
|
96
|
+
...(status ? { status } : {}),
|
|
94
97
|
toolCallId: providerToolId(params),
|
|
95
98
|
text: readableToolPayload(event.method, params),
|
|
96
99
|
}];
|
|
@@ -797,6 +800,15 @@ function historicalEventFromItem(item) {
|
|
|
797
800
|
export function historicalEventsFromThread(items, turns, limit, runtimeEvents = []) {
|
|
798
801
|
const events = [];
|
|
799
802
|
const userPromptKeys = new Set();
|
|
803
|
+
// Canonical ThreadItems are authoritative for text and for any tool item
|
|
804
|
+
// the provider materialized. Some HE providers intentionally keep
|
|
805
|
+
// orchestration calls only in the event log, though (for example a
|
|
806
|
+
// delegate.batch call is represented by item/toolCall but is omitted from
|
|
807
|
+
// thread/items/list). Keep the canonical identities separate so we can
|
|
808
|
+
// merge those runtime-only calls without re-introducing raw assistant
|
|
809
|
+
// snapshots or duplicate canonical tool rows.
|
|
810
|
+
const canonicalToolItemIds = new Set();
|
|
811
|
+
const canonicalToolCallIds = new Set();
|
|
800
812
|
const addEvent = (event) => {
|
|
801
813
|
if (event.payload.type === "user_prompt") {
|
|
802
814
|
const key = `${event.turnId}\0${event.payload.text}`;
|
|
@@ -808,6 +820,23 @@ export function historicalEventsFromThread(items, turns, limit, runtimeEvents =
|
|
|
808
820
|
};
|
|
809
821
|
for (const item of items) {
|
|
810
822
|
const event = historicalEventFromItem(item);
|
|
823
|
+
const turnId = item.turnId ?? item.threadId;
|
|
824
|
+
if (item.type === "toolCall") {
|
|
825
|
+
canonicalToolItemIds.add(historicalToolIdentityKey(turnId, item.id));
|
|
826
|
+
canonicalToolCallIds.add(historicalToolIdentityKey(turnId, item.toolCallId));
|
|
827
|
+
}
|
|
828
|
+
else if (item.type === "command") {
|
|
829
|
+
canonicalToolItemIds.add(historicalToolIdentityKey(turnId, item.id));
|
|
830
|
+
canonicalToolCallIds.add(historicalToolIdentityKey(turnId, item.commandId));
|
|
831
|
+
}
|
|
832
|
+
else if (item.type === "approval") {
|
|
833
|
+
canonicalToolItemIds.add(historicalToolIdentityKey(turnId, item.id));
|
|
834
|
+
canonicalToolCallIds.add(historicalToolIdentityKey(turnId, item.serverRequestId));
|
|
835
|
+
}
|
|
836
|
+
else if (item.type === "subAgentActivity") {
|
|
837
|
+
canonicalToolItemIds.add(historicalToolIdentityKey(turnId, item.id));
|
|
838
|
+
canonicalToolCallIds.add(historicalToolIdentityKey(turnId, item.agentId));
|
|
839
|
+
}
|
|
811
840
|
if (event)
|
|
812
841
|
addEvent(event);
|
|
813
842
|
}
|
|
@@ -816,15 +845,17 @@ export function historicalEventsFromThread(items, turns, limit, runtimeEvents =
|
|
|
816
845
|
if (event)
|
|
817
846
|
addEvent(event);
|
|
818
847
|
}
|
|
819
|
-
// Server requests
|
|
820
|
-
//
|
|
821
|
-
//
|
|
822
|
-
//
|
|
823
|
-
//
|
|
824
|
-
// reintroducing duplicate assistant snapshots.
|
|
848
|
+
// Server requests and provider/HE tool calls are not guaranteed to be
|
|
849
|
+
// represented by canonical ThreadItems. Preserve only those non-text
|
|
850
|
+
// runtime events which are absent from the canonical identity set. Text
|
|
851
|
+
// snapshots remain canonical-only: merging them would resurrect the raw
|
|
852
|
+
// assistant history that the rebuild deliberately replaces.
|
|
825
853
|
for (const event of historicalEventsFromRuntimeEvents(runtimeEvents)) {
|
|
826
|
-
if (event.payload.type === "
|
|
827
|
-
|
|
854
|
+
if (event.payload.type === "text_delta" || event.payload.type === "done")
|
|
855
|
+
continue;
|
|
856
|
+
if (event.payload.type === "tool_call" && runtimeEventMatchesCanonicalItem(event, canonicalToolItemIds, canonicalToolCallIds))
|
|
857
|
+
continue;
|
|
858
|
+
addEvent(event);
|
|
828
859
|
}
|
|
829
860
|
for (const turn of turns) {
|
|
830
861
|
if (!isTerminalTurn(turn))
|
|
@@ -836,6 +867,20 @@ export function historicalEventsFromThread(items, turns, limit, runtimeEvents =
|
|
|
836
867
|
.slice(limit === undefined ? 0 : -limit)
|
|
837
868
|
.map((event, index) => ({ ...event, seq: event.seq ?? index + 1 }));
|
|
838
869
|
}
|
|
870
|
+
function runtimeEventMatchesCanonicalItem(event, canonicalToolItemIds, canonicalToolCallIds) {
|
|
871
|
+
if (event.payload.type !== "tool_call")
|
|
872
|
+
return false;
|
|
873
|
+
if (event.itemId && canonicalToolItemIds.has(historicalToolIdentityKey(event.turnId, event.itemId)))
|
|
874
|
+
return true;
|
|
875
|
+
if (event.payload.itemId && canonicalToolItemIds.has(historicalToolIdentityKey(event.turnId, event.payload.itemId)))
|
|
876
|
+
return true;
|
|
877
|
+
if (event.payload.toolCallId && canonicalToolCallIds.has(historicalToolIdentityKey(event.turnId, event.payload.toolCallId)))
|
|
878
|
+
return true;
|
|
879
|
+
return false;
|
|
880
|
+
}
|
|
881
|
+
function historicalToolIdentityKey(turnId, identity) {
|
|
882
|
+
return `${turnId}\0${identity}`;
|
|
883
|
+
}
|
|
839
884
|
function toolEventsFromItem(item) {
|
|
840
885
|
const itemIdentity = threadItemIdentity(item);
|
|
841
886
|
if (item.type === "toolCall") {
|
|
@@ -20,13 +20,15 @@ export async function readHeadlessHistoricalEventPage(input) {
|
|
|
20
20
|
};
|
|
21
21
|
}
|
|
22
22
|
async function authoritativeTurnEvents(input, runtimeEvents) {
|
|
23
|
-
// A
|
|
24
|
-
// every provider notification. Read the provider's
|
|
25
|
-
// projections when there is no paging anchor;
|
|
26
|
-
//
|
|
27
|
-
//
|
|
28
|
-
//
|
|
29
|
-
|
|
23
|
+
// A transcript replacement is a canonical rebuild, not a best-effort
|
|
24
|
+
// concatenation of every provider notification. Read the provider's
|
|
25
|
+
// canonical item/turn projections when there is no paging anchor; the
|
|
26
|
+
// projector selectively merges runtime-only non-text tool/option events
|
|
27
|
+
// by identity. This repairs older logs that contain removed items, a delta
|
|
28
|
+
// plus a repeated terminal snapshot for the same item, or a late native
|
|
29
|
+
// reconciliation. Identity is itemId/turnId, never text, so legitimate
|
|
30
|
+
// repeated answers from distinct items remain visible.
|
|
31
|
+
if (input.before || input.beforeItemId || input.afterItemId)
|
|
30
32
|
return undefined;
|
|
31
33
|
// A few internal readers (and older provider adapters during rolling
|
|
32
34
|
// upgrades) only expose the event reader. Canonical rebuild is an
|
|
@@ -39,8 +41,8 @@ async function authoritativeTurnEvents(input, runtimeEvents) {
|
|
|
39
41
|
listAllThreadItems(input.client, input.threadId, input.turnId),
|
|
40
42
|
listAllThreadTurns(input.client, input.threadId),
|
|
41
43
|
]);
|
|
42
|
-
const
|
|
43
|
-
return historicalEventsFromThread(items,
|
|
44
|
+
const selectedTurns = input.turnId ? turns.filter((candidate) => candidate.id === input.turnId) : turns;
|
|
45
|
+
return historicalEventsFromThread(items, selectedTurns, undefined, runtimeEvents);
|
|
44
46
|
}
|
|
45
47
|
async function listAllThreadItems(client, threadId, turnId) {
|
|
46
48
|
const items = [];
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { type MachineCommand } from "../../../../packages/shared/dist/index.js";
|
|
2
2
|
import type { DaemonConfig, SessionFallback, SessionState } from "../types.js";
|
|
3
3
|
import { type HistoricalBackfillMode, type HistoricalBackfillOptions, type HistoricalBackfillResult } from "./history-backfill.js";
|
|
4
|
-
export { refreshNativeHistoryForLoadedSession } from "./native-history-refresh.js";
|
|
4
|
+
export { invalidateNativeHistoryRefreshForLoadedSession, refreshNativeHistoryForLoadedSession, } from "./native-history-refresh.js";
|
|
5
5
|
export declare function handleCreateSession(ws: WebSocket, config: DaemonConfig, command: MachineCommand): Promise<void>;
|
|
6
6
|
export declare function handleImportHistoricalSession(ws: WebSocket, config: DaemonConfig, command: MachineCommand): Promise<void>;
|
|
7
7
|
export declare function handleListHistoricalSessions(ws: WebSocket, config: DaemonConfig, command: MachineCommand): Promise<void>;
|
|
@@ -19,7 +19,7 @@ import { commandSessionMetadata, encryptedSessionMetadata, sessionFallbackFromCo
|
|
|
19
19
|
import { closeSessionProjection, persistSessionProjection, sessionReadyProjectionFields } from "./store.js";
|
|
20
20
|
import { nativeHandoffIdentity } from "./native-identity.js";
|
|
21
21
|
import { refreshNativeHistoryForLoadedSession } from "./native-history-refresh.js";
|
|
22
|
-
export { refreshNativeHistoryForLoadedSession } from "./native-history-refresh.js";
|
|
22
|
+
export { invalidateNativeHistoryRefreshForLoadedSession, refreshNativeHistoryForLoadedSession, } from "./native-history-refresh.js";
|
|
23
23
|
export async function handleCreateSession(ws, config, command) {
|
|
24
24
|
if (command.type !== "machine:createSession")
|
|
25
25
|
return;
|
|
@@ -1,12 +1,25 @@
|
|
|
1
1
|
import type { DaemonConfig, SessionState } from "../types.js";
|
|
2
|
+
/**
|
|
3
|
+
* Mark a managed provider session's history dirty after a new Headless turn.
|
|
4
|
+
*
|
|
5
|
+
* Managed non-native sessions deliberately use a best-effort, one-read refresh cache
|
|
6
|
+
* so an ordinary read does not repeatedly interrogate the provider. That
|
|
7
|
+
* cache must be invalidated at the turn boundary: the provider may have
|
|
8
|
+
* canonicalized the just-finished native turn (for example, collapsing a
|
|
9
|
+
* streamed duplicate assistant item) even though the HE projection observed
|
|
10
|
+
* both live items. The next read then performs the provider-authoritative
|
|
11
|
+
* reconciliation. An in-flight refresh is left alone and remains the
|
|
12
|
+
* singleflight operation for this binding. A terminal generation can abandon
|
|
13
|
+
* that operation so a later read is never satisfied by an older snapshot.
|
|
14
|
+
*/
|
|
15
|
+
export declare function invalidateNativeHistoryRefreshForLoadedSession(session: SessionState): void;
|
|
2
16
|
/**
|
|
3
17
|
* Reconcile provider-native history before a loaded session is projected.
|
|
4
18
|
*
|
|
5
|
-
* Native sessions have always used this path.
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
* caller manually invokes the runtime operation.
|
|
19
|
+
* Native sessions have always used this path. Managed provider sessions also
|
|
20
|
+
* carry a provider session id and native-history metadata, so they must take
|
|
21
|
+
* the same read-only reconciliation path after restart or before runtime
|
|
22
|
+
* truth reads; otherwise an old same-turn duplicate remains visible until an
|
|
23
|
+
* internal caller manually invokes the runtime operation.
|
|
11
24
|
*/
|
|
12
25
|
export declare function refreshNativeHistoryForLoadedSession(session: SessionState, config: DaemonConfig | undefined, requestId: string): Promise<SessionState>;
|
|
@@ -6,26 +6,58 @@ import { nativeHandoffIdentity } from "./native-identity.js";
|
|
|
6
6
|
const orchestrationRefreshes = new Map();
|
|
7
7
|
const orchestrationRefreshCompleted = new Set();
|
|
8
8
|
const orchestrationRefreshRetryAt = new Map();
|
|
9
|
+
const orchestrationRefreshGenerations = new Map();
|
|
9
10
|
const orchestrationRefreshRetryDelayMs = 1_000;
|
|
11
|
+
const orchestrationRefreshMaxWaitMs = 2_000;
|
|
12
|
+
/**
|
|
13
|
+
* Mark a managed provider session's history dirty after a new Headless turn.
|
|
14
|
+
*
|
|
15
|
+
* Managed non-native sessions deliberately use a best-effort, one-read refresh cache
|
|
16
|
+
* so an ordinary read does not repeatedly interrogate the provider. That
|
|
17
|
+
* cache must be invalidated at the turn boundary: the provider may have
|
|
18
|
+
* canonicalized the just-finished native turn (for example, collapsing a
|
|
19
|
+
* streamed duplicate assistant item) even though the HE projection observed
|
|
20
|
+
* both live items. The next read then performs the provider-authoritative
|
|
21
|
+
* reconciliation. An in-flight refresh is left alone and remains the
|
|
22
|
+
* singleflight operation for this binding. A terminal generation can abandon
|
|
23
|
+
* that operation so a later read is never satisfied by an older snapshot.
|
|
24
|
+
*/
|
|
25
|
+
export function invalidateNativeHistoryRefreshForLoadedSession(session) {
|
|
26
|
+
if (session.source === "native" || !session.providerSessionId)
|
|
27
|
+
return;
|
|
28
|
+
const refreshKey = `${session.runtimeBinding.profile}\0${session.runtimeBinding.threadId}`;
|
|
29
|
+
orchestrationRefreshGenerations.set(refreshKey, (orchestrationRefreshGenerations.get(refreshKey) ?? 0) + 1);
|
|
30
|
+
orchestrationRefreshCompleted.delete(refreshKey);
|
|
31
|
+
orchestrationRefreshRetryAt.delete(refreshKey);
|
|
32
|
+
const active = orchestrationRefreshes.get(refreshKey);
|
|
33
|
+
if (active) {
|
|
34
|
+
active.abandoned = true;
|
|
35
|
+
orchestrationRefreshes.delete(refreshKey);
|
|
36
|
+
// A generation change is also the cancellation boundary for an old
|
|
37
|
+
// provider read. Drop its IPC client so the next refresh cannot inherit
|
|
38
|
+
// a request that is already past the terminal boundary.
|
|
39
|
+
void getHeadlessRuntimeManager(session.cwd).invalidateClientForBinding(session.runtimeBinding, session.cwd);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
10
42
|
/**
|
|
11
43
|
* Reconcile provider-native history before a loaded session is projected.
|
|
12
44
|
*
|
|
13
|
-
* Native sessions have always used this path.
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
* caller manually invokes the runtime operation.
|
|
45
|
+
* Native sessions have always used this path. Managed provider sessions also
|
|
46
|
+
* carry a provider session id and native-history metadata, so they must take
|
|
47
|
+
* the same read-only reconciliation path after restart or before runtime
|
|
48
|
+
* truth reads; otherwise an old same-turn duplicate remains visible until an
|
|
49
|
+
* internal caller manually invokes the runtime operation.
|
|
19
50
|
*/
|
|
20
51
|
export async function refreshNativeHistoryForLoadedSession(session, config, requestId) {
|
|
21
52
|
const isNative = session.source === "native";
|
|
22
|
-
const
|
|
23
|
-
if (!isNative && !
|
|
53
|
+
const isManagedProviderSession = session.source !== "native" && Boolean(session.providerSessionId);
|
|
54
|
+
if (!isNative && !isManagedProviderSession)
|
|
24
55
|
return session;
|
|
25
56
|
const refreshKey = `${session.runtimeBinding.profile}\0${session.runtimeBinding.threadId}`;
|
|
26
|
-
if (
|
|
57
|
+
if (isManagedProviderSession) {
|
|
58
|
+
const generation = orchestrationRefreshGenerations.get(refreshKey) ?? 0;
|
|
27
59
|
if (orchestrationRefreshCompleted.has(refreshKey))
|
|
28
|
-
return session;
|
|
60
|
+
return sessions.get(session.sessionId) ?? session;
|
|
29
61
|
const retryAt = orchestrationRefreshRetryAt.get(refreshKey);
|
|
30
62
|
if (retryAt !== undefined) {
|
|
31
63
|
if (retryAt > Date.now())
|
|
@@ -33,21 +65,59 @@ export async function refreshNativeHistoryForLoadedSession(session, config, requ
|
|
|
33
65
|
orchestrationRefreshRetryAt.delete(refreshKey);
|
|
34
66
|
}
|
|
35
67
|
const active = orchestrationRefreshes.get(refreshKey);
|
|
36
|
-
if (active)
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
68
|
+
if (active) {
|
|
69
|
+
const remainingMs = Math.max(0, orchestrationRefreshMaxWaitMs - (Date.now() - active.startedAt));
|
|
70
|
+
const outcome = await Promise.race([
|
|
71
|
+
active.promise.then((result) => ({ kind: "result", result })),
|
|
72
|
+
new Promise((resolve) => setTimeout(() => resolve({ kind: "timeout" }), remainingMs)),
|
|
73
|
+
]);
|
|
74
|
+
if (outcome.kind === "timeout") {
|
|
75
|
+
active.abandoned = true;
|
|
76
|
+
if (orchestrationRefreshes.get(refreshKey) === active)
|
|
77
|
+
orchestrationRefreshes.delete(refreshKey);
|
|
78
|
+
orchestrationRefreshGenerations.set(refreshKey, (orchestrationRefreshGenerations.get(refreshKey) ?? 0) + 1);
|
|
79
|
+
// Disposing the client cancels the bounded IPC request instead of
|
|
80
|
+
// leaving every later official read behind the same hung promise.
|
|
81
|
+
void getHeadlessRuntimeManager(session.cwd).invalidateClientForBinding(session.runtimeBinding, session.cwd);
|
|
82
|
+
return sessions.get(session.sessionId) ?? session;
|
|
83
|
+
}
|
|
84
|
+
const result = outcome.result;
|
|
85
|
+
// A terminal turn may have invalidated the cache while the previous
|
|
86
|
+
// provider read was in flight. Never let that old result satisfy the
|
|
87
|
+
// new generation; run one fresh read after the old singleflight settles.
|
|
88
|
+
if (!active.abandoned && active.generation !== (orchestrationRefreshGenerations.get(refreshKey) ?? 0)) {
|
|
89
|
+
if (orchestrationRefreshes.get(refreshKey) === active)
|
|
90
|
+
orchestrationRefreshes.delete(refreshKey);
|
|
91
|
+
return await refreshNativeHistoryForLoadedSession(session, config, requestId);
|
|
92
|
+
}
|
|
93
|
+
if (active.abandoned)
|
|
94
|
+
return sessions.get(session.sessionId) ?? session;
|
|
95
|
+
return result;
|
|
96
|
+
}
|
|
97
|
+
const operation = refreshNativeHistory(session, config, requestId, false, refreshKey, generation);
|
|
98
|
+
const entry = { generation, promise: operation, startedAt: Date.now() };
|
|
99
|
+
orchestrationRefreshes.set(refreshKey, entry);
|
|
40
100
|
try {
|
|
41
|
-
|
|
101
|
+
const result = await operation;
|
|
102
|
+
if (!entry.abandoned && generation !== (orchestrationRefreshGenerations.get(refreshKey) ?? 0)) {
|
|
103
|
+
if (orchestrationRefreshes.get(refreshKey) === entry)
|
|
104
|
+
orchestrationRefreshes.delete(refreshKey);
|
|
105
|
+
return await refreshNativeHistoryForLoadedSession(session, config, requestId);
|
|
106
|
+
}
|
|
107
|
+
if (entry.abandoned)
|
|
108
|
+
return sessions.get(session.sessionId) ?? session;
|
|
109
|
+
return result;
|
|
42
110
|
}
|
|
43
111
|
finally {
|
|
44
|
-
if (orchestrationRefreshes.get(refreshKey) ===
|
|
112
|
+
if (orchestrationRefreshes.get(refreshKey) === entry)
|
|
45
113
|
orchestrationRefreshes.delete(refreshKey);
|
|
46
114
|
}
|
|
47
115
|
}
|
|
48
116
|
return await refreshNativeHistory(session, config, requestId, true);
|
|
49
117
|
}
|
|
50
|
-
async function refreshNativeHistory(session, config, requestId, strict, orchestrationRefreshKey) {
|
|
118
|
+
async function refreshNativeHistory(session, config, requestId, strict, orchestrationRefreshKey, orchestrationRefreshGeneration) {
|
|
119
|
+
const isNative = session.source === "native";
|
|
120
|
+
const isLegacyCrew = session.source === "orchestration" && Boolean(session.providerSessionId);
|
|
51
121
|
const manager = getHeadlessRuntimeManager(session.cwd);
|
|
52
122
|
let lastError;
|
|
53
123
|
for (let attempt = 0; attempt < 2; attempt += 1) {
|
|
@@ -56,8 +126,9 @@ async function refreshNativeHistory(session, config, requestId, strict, orchestr
|
|
|
56
126
|
await manager.invalidateClientForBinding(session.runtimeBinding, session.cwd);
|
|
57
127
|
const entry = await manager.clientForBinding(session.runtimeBinding, session.cwd);
|
|
58
128
|
if (typeof entry.client.syncNativeHistory !== "function") {
|
|
59
|
-
if (orchestrationRefreshKey)
|
|
129
|
+
if (orchestrationRefreshKey && orchestrationRefreshGeneration === (orchestrationRefreshGenerations.get(orchestrationRefreshKey) ?? 0)) {
|
|
60
130
|
orchestrationRefreshCompleted.add(orchestrationRefreshKey);
|
|
131
|
+
}
|
|
61
132
|
return session;
|
|
62
133
|
}
|
|
63
134
|
const providerSessionId = session.providerSessionId ?? session.runtimeBinding.threadId;
|
|
@@ -69,7 +140,7 @@ async function refreshNativeHistory(session, config, requestId, strict, orchestr
|
|
|
69
140
|
cwd: session.cwd,
|
|
70
141
|
providerSessionId,
|
|
71
142
|
});
|
|
72
|
-
if (
|
|
143
|
+
if (isLegacyCrew && typeof entry.client.readThread === "function" && typeof entry.client.adoptThread === "function") {
|
|
73
144
|
const current = await entry.client.readThread({ threadId: session.runtimeBinding.threadId });
|
|
74
145
|
const metadata = current.thread.providerMetadata ?? {};
|
|
75
146
|
// Legacy Crew threads may have provider history metadata but no
|
|
@@ -92,7 +163,7 @@ async function refreshNativeHistory(session, config, requestId, strict, orchestr
|
|
|
92
163
|
}
|
|
93
164
|
const synced = await entry.client.syncNativeHistory({
|
|
94
165
|
threadId: session.runtimeBinding.threadId,
|
|
95
|
-
nativeIdentity: expectedNativeIdentity,
|
|
166
|
+
...(isNative || isLegacyCrew ? { nativeIdentity: expectedNativeIdentity } : {}),
|
|
96
167
|
});
|
|
97
168
|
const providerMetadata = synced.thread.providerMetadata ?? {};
|
|
98
169
|
const externallyChanged = providerMetadata.nativeHistoryChangedExternally === true;
|
|
@@ -105,8 +176,9 @@ async function refreshNativeHistory(session, config, requestId, strict, orchestr
|
|
|
105
176
|
: session.manualHandoffProof && typeof providerMetadata.nativeHistoryVersion === "string"
|
|
106
177
|
? { ...session.manualHandoffProof, historyVersion: providerMetadata.nativeHistoryVersion }
|
|
107
178
|
: session.manualHandoffProof;
|
|
179
|
+
const currentSession = sessions.get(session.sessionId) ?? session;
|
|
108
180
|
const next = {
|
|
109
|
-
...
|
|
181
|
+
...currentSession,
|
|
110
182
|
...(strict && isUsableString(expectedNativeIdentity) ? { nativeIdentity: expectedNativeIdentity } : {}),
|
|
111
183
|
...(typeof providerMetadata.nativeHistoryVersion === "string" ? { nativeHistoryVersion: providerMetadata.nativeHistoryVersion } : {}),
|
|
112
184
|
...(strict && externallyChanged
|
|
@@ -116,6 +188,13 @@ async function refreshNativeHistory(session, config, requestId, strict, orchestr
|
|
|
116
188
|
: {}),
|
|
117
189
|
...(strict && session.nativeHistorySyncError ? { nativeHistorySyncError: undefined } : {}),
|
|
118
190
|
};
|
|
191
|
+
// If a newer terminal generation was armed while this provider read was
|
|
192
|
+
// in flight, keep the durable provider reconciliation but do not publish
|
|
193
|
+
// this stale read as the completed daemon-side refresh.
|
|
194
|
+
const generationStillCurrent = !orchestrationRefreshKey
|
|
195
|
+
|| orchestrationRefreshGeneration === (orchestrationRefreshGenerations.get(orchestrationRefreshKey) ?? 0);
|
|
196
|
+
if (!generationStillCurrent)
|
|
197
|
+
return sessions.get(session.sessionId) ?? next;
|
|
119
198
|
sessions.set(session.sessionId, next);
|
|
120
199
|
await persistSessionProjection(next);
|
|
121
200
|
if (orchestrationRefreshKey) {
|
|
@@ -136,10 +215,13 @@ async function refreshNativeHistory(session, config, requestId, strict, orchestr
|
|
|
136
215
|
// log remains readable when a provider cannot perform native refresh; do
|
|
137
216
|
// not turn a harmless compatibility probe into a session failure.
|
|
138
217
|
if (orchestrationRefreshKey) {
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
218
|
+
const generationStillCurrent = orchestrationRefreshGeneration === (orchestrationRefreshGenerations.get(orchestrationRefreshKey) ?? 0);
|
|
219
|
+
if (generationStillCurrent) {
|
|
220
|
+
if (isDefinitelyUnsupportedLegacyRefreshError(lastError))
|
|
221
|
+
orchestrationRefreshCompleted.add(orchestrationRefreshKey);
|
|
222
|
+
else
|
|
223
|
+
orchestrationRefreshRetryAt.set(orchestrationRefreshKey, Date.now() + orchestrationRefreshRetryDelayMs);
|
|
224
|
+
}
|
|
143
225
|
}
|
|
144
226
|
return session;
|
|
145
227
|
}
|
|
@@ -15,9 +15,58 @@ import { attachClaimedTurn, claimSessionTurn, releaseSessionTurn, sendCommandRes
|
|
|
15
15
|
import { settleScheduleTurnTerminal } from "../schedule/runner.js";
|
|
16
16
|
import { firstAgentDispatchListProjection } from "./he-official/orchestration-commands.js";
|
|
17
17
|
import { emitEvent, runtimeEventToPayload } from "./events.js";
|
|
18
|
-
import { refreshNativeHistoryForLoadedSession, replaceAuthoritativeTranscript, resumeSession, usesAuthoritativeTranscript } from "./lifecycle.js";
|
|
18
|
+
import { invalidateNativeHistoryRefreshForLoadedSession, refreshNativeHistoryForLoadedSession, replaceAuthoritativeTranscript, resumeSession, usesAuthoritativeTranscript } from "./lifecycle.js";
|
|
19
19
|
import { encryptedSessionMetadata, sessionFallbackFromCommand } from "./metadata.js";
|
|
20
20
|
import { persistSessionProjection, withSessionHead } from "./store.js";
|
|
21
|
+
// A provider can finish its canonical history read after the terminal relay
|
|
22
|
+
// packet has already been emitted. Keep a short, bounded recovery window so
|
|
23
|
+
// the authoritative transcript is actively replaced without requiring a new
|
|
24
|
+
// official read from the user. The key is the HE turn, so a later turn cannot
|
|
25
|
+
// coalesce with (or cancel) the recovery for this completed turn.
|
|
26
|
+
const nativeHistoryRecoveries = new Map();
|
|
27
|
+
const nativeHistoryRecoveryDelaysMs = [500, 1_500, 3_000];
|
|
28
|
+
function scheduleNativeHistoryRecovery(ws, config, session, requestId, turnId) {
|
|
29
|
+
if (session.source === "native" || !session.providerSessionId)
|
|
30
|
+
return;
|
|
31
|
+
const key = `${session.sessionId}\0${turnId}`;
|
|
32
|
+
if (nativeHistoryRecoveries.has(key))
|
|
33
|
+
return;
|
|
34
|
+
const recovery = { sessionId: session.sessionId, turnId, requestId, ws, config, attempt: 0 };
|
|
35
|
+
nativeHistoryRecoveries.set(key, recovery);
|
|
36
|
+
const run = async () => {
|
|
37
|
+
if (nativeHistoryRecoveries.get(key) !== recovery)
|
|
38
|
+
return;
|
|
39
|
+
recovery.attempt += 1;
|
|
40
|
+
const current = sessions.get(session.sessionId);
|
|
41
|
+
if (!current || current.runtimeBinding.threadId !== session.runtimeBinding.threadId || sessionTurns.activeTurnId(session.sessionId)) {
|
|
42
|
+
nativeHistoryRecoveries.delete(key);
|
|
43
|
+
return;
|
|
44
|
+
}
|
|
45
|
+
try {
|
|
46
|
+
// Each attempt gets a fresh provider read. A successful early read can
|
|
47
|
+
// still precede provider canonicalization, so do not let the refresh
|
|
48
|
+
// cache suppress the later attempts.
|
|
49
|
+
invalidateNativeHistoryRefreshForLoadedSession(current);
|
|
50
|
+
const refreshed = await refreshNativeHistoryForLoadedSession(current, config, `${requestId}:history-recovery:${recovery.attempt}`);
|
|
51
|
+
const latest = sessions.get(session.sessionId) ?? refreshed;
|
|
52
|
+
await replaceAuthoritativeTranscript(ws, config, latest, undefined, {
|
|
53
|
+
replacementKind: "sync",
|
|
54
|
+
requestId: `${requestId}:history-recovery:${recovery.attempt}`,
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
catch (error) {
|
|
58
|
+
console.error(`Managed provider history recovery failed for ${session.sessionId}: ${error instanceof Error ? error.message : String(error)}`);
|
|
59
|
+
}
|
|
60
|
+
if (recovery.attempt >= nativeHistoryRecoveryDelaysMs.length) {
|
|
61
|
+
nativeHistoryRecoveries.delete(key);
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
64
|
+
recovery.timer = setTimeout(() => { void run(); }, nativeHistoryRecoveryDelaysMs[recovery.attempt]);
|
|
65
|
+
recovery.timer.unref?.();
|
|
66
|
+
};
|
|
67
|
+
recovery.timer = setTimeout(() => { void run(); }, nativeHistoryRecoveryDelaysMs[0]);
|
|
68
|
+
recovery.timer.unref?.();
|
|
69
|
+
}
|
|
21
70
|
export async function handlePrompt(ws, config, command) {
|
|
22
71
|
if (command.type !== "machine:prompt")
|
|
23
72
|
return;
|
|
@@ -371,6 +420,34 @@ export async function handlePrompt(ws, config, command) {
|
|
|
371
420
|
currentHead: result.currentHead,
|
|
372
421
|
lastTurnId: result.lastTurnId,
|
|
373
422
|
}, { runtimeTurnId: completedRuntimeTurnId, checkpointId: completedRuntimeTurnId });
|
|
423
|
+
if (session.source !== "native" && session.providerSessionId) {
|
|
424
|
+
// A managed provider turn can stream more HE items than the provider
|
|
425
|
+
// later exposes as canonical native history. Invalidate the one-read
|
|
426
|
+
// refresh cache at the terminal boundary and reconcile before replacing
|
|
427
|
+
// the authoritative transcript, so live projections and subsequent
|
|
428
|
+
// turn.result/status reads converge without a second turn execution.
|
|
429
|
+
invalidateNativeHistoryRefreshForLoadedSession(session);
|
|
430
|
+
const nativeRefresh = refreshNativeHistoryForLoadedSession(sessions.get(command.sessionId) ?? session, config, command.requestId).catch((error) => {
|
|
431
|
+
console.error(`Legacy Crew post-turn history refresh failed for ${command.turnId}: ${error instanceof Error ? error.message : String(error)}`);
|
|
432
|
+
});
|
|
433
|
+
// Provider-native history is read-only cleanup. It must not hold the
|
|
434
|
+
// terminal/control path hostage if the provider socket stalls; the
|
|
435
|
+
// coalesced refresh remains in flight and the next official read or
|
|
436
|
+
// recovery pass will observe it (or its bounded retry state).
|
|
437
|
+
const refreshTimeout = Symbol("native-refresh-timeout");
|
|
438
|
+
const refreshOutcome = await Promise.race([
|
|
439
|
+
nativeRefresh.then(() => true),
|
|
440
|
+
new Promise((resolve) => setTimeout(() => resolve(refreshTimeout), 2_000)),
|
|
441
|
+
]);
|
|
442
|
+
if (refreshOutcome === refreshTimeout) {
|
|
443
|
+
// Detach the provider operation at the terminal boundary instead of
|
|
444
|
+
// leaving a stalled IPC request alive until its much longer transport
|
|
445
|
+
// timeout. The next official read/recovery pass starts a new generation.
|
|
446
|
+
invalidateNativeHistoryRefreshForLoadedSession(sessions.get(command.sessionId) ?? session);
|
|
447
|
+
scheduleNativeHistoryRecovery(ws, config, sessions.get(command.sessionId) ?? session, command.requestId, command.turnId);
|
|
448
|
+
}
|
|
449
|
+
session = sessions.get(command.sessionId) ?? session;
|
|
450
|
+
}
|
|
374
451
|
phase = "refresh-authoritative-transcript";
|
|
375
452
|
await refreshAuthoritativeTranscriptAfterTurn(ws, config, session, command.requestId, command.turnId, result.currentHead, result.lastTurnId, result.status);
|
|
376
453
|
phase = "mark-terminal-pending";
|