@cydm/happy-elves 0.1.0-beta.488 → 0.1.0-beta.489

package/apps/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cydm/happy-elves-cli",
-  "version": "0.1.0-beta.488",
+  "version": "0.1.0-beta.489",
   "private": true,
   "type": "module"
 }

package/apps/daemon/dist/paths.d.ts

@@ -2,6 +2,7 @@ export declare const configDir: string;
 export declare const configPath: string;
 export declare const auditLogPath: string;
 export declare const relayOutboxPath: string;
+export declare const relayRetainedOutboxPath: string;
 export declare const inboundPromptAdmissionStorePath: string;
 export declare const sessionProjectionStorePath: string;
 export declare const loopStorePath: string;

package/apps/daemon/dist/paths.js

@@ -7,6 +7,7 @@ export const configDir = process.env.HAPPY_ELVES_HOME
 export const configPath = path.join(configDir, "daemon.json");
 export const auditLogPath = path.join(configDir, "audit.jsonl");
 export const relayOutboxPath = path.join(configDir, "outbox.jsonl");
+export const relayRetainedOutboxPath = path.join(configDir, "retained-outbox.jsonl");
 export const inboundPromptAdmissionStorePath = path.join(configDir, "inbound-prompt-admissions.json");
 export const sessionProjectionStorePath = path.join(configDir, "sessions.json");
 export const loopStorePath = path.join(configDir, "loops.json");

package/apps/daemon/dist/relay/send.js

@@ -3,7 +3,7 @@ import fs from "node:fs/promises";
 import { encryptJson } from "../../../../packages/shared/dist/index.js";
 import { activeCommandRequests, completedCommandResponses, sessions, sessionTurns } from "../state.js";
 import { appendAudit } from "../audit.js";
-import { configDir, relayOutboxPath } from "../paths.js";
+import { configDir, relayOutboxPath, relayRetainedOutboxPath } from "../paths.js";
 const criticalMessageTypes = new Set([
     "machine:sessionReady",
     "machine:event",
@@ -37,7 +37,14 @@ let outboxFlushTimer;
 const outbox = new Map();
 const directSockets = new WeakSet();
 const retainedVersions = new Map();
-const retainedWriteQueues = new Map();
+const retainedOutbox = new Map();
+const retainedInFlightKeys = new Set();
+let retainedOutboxLoaded = false;
+let retainedOutboxLoadPromise = null;
+let retainedOutboxWritePromise = Promise.resolve();
+let retainedOutboxDrainPromise = null;
+let retainedOutboxFlushTimer;
+let retainedOutboxFlushAt;
 const transcriptWindows = new Map();
 const controllerProjectionChannel = "product:controllers";
 const retainedTranscriptWindowSize = 200;
@@ -50,6 +57,13 @@ const outboxAckTimeoutMs = 15_000;
 const outboxRetryBaseMs = 1_000;
 const outboxRetryMaxMs = 30_000;
 const relayStoredTtlMs = 24 * 60 * 60 * 1000;
+const configuredRetainedHttpTimeoutMs = Number(process.env.HAPPY_ELVES_RELAY_RETAINED_HTTP_TIMEOUT_MS);
+const retainedHttpTimeoutMs = Number.isFinite(configuredRetainedHttpTimeoutMs) && configuredRetainedHttpTimeoutMs >= 25
+    ? Math.min(60_000, Math.floor(configuredRetainedHttpTimeoutMs))
+    : 10_000;
+const retainedDrainConcurrency = 4;
+const retainedOutboxRetryBaseMs = 1_000;
+const retainedOutboxRetryMaxMs = 30_000;
 export function markDirectSocket(ws) {
     directSockets.add(ws);
 }
@@ -73,6 +87,9 @@ export function send(ws, message) {
 }
 export function configureRelaySender(config) {
     relayConfig = config;
+    void ensureRetainedOutboxLoaded().then(() => scheduleRetainedOutboxFlush(0)).catch((error) => {
+        console.error(`Relay retained outbox load failed: ${error instanceof Error ? error.message : String(error)}`);
+    });
 }
 export function setActiveRelaySocket(ws) {
     activeRelaySocket = ws;
@@ -157,6 +174,15 @@ export async function clearRelayOutboxForTests() {
     outboxLoaded = true;
     outboxLoadPromise = null;
     await persistOutbox();
+    if (retainedOutboxFlushTimer) {
+        clearTimeout(retainedOutboxFlushTimer);
+        retainedOutboxFlushTimer = undefined;
+    }
+    retainedOutboxFlushAt = undefined;
+    retainedOutbox.clear();
+    retainedOutboxLoaded = true;
+    retainedOutboxLoadPromise = null;
+    await persistRetainedOutbox();
 }
 export async function sendReliable(message, ws) {
     if (ws && directSockets.has(ws)) {
@@ -515,32 +541,195 @@ async function sendPacket(socket, payload, options) {
             encryptedPayload: await encryptJson(relayConfig.accountSecret, `packet:${options.packetId}`, payload),
         },
     };
+    // Retained projections are durable and independently retried. Persisting
+    // the latest projection is local-only work; the WAN/CAS write must never
+    // hold up a control packet or its stored ACK.
+    await sendRetainedProjection(payload);
     socket.send(JSON.stringify(packet));
-    await sendRetainedProjection(socket, payload);
 }
-async function sendRetainedProjection(socket, payload) {
+async function sendRetainedProjection(payload) {
     if (!relayConfig)
         return;
-    void socket;
     const retainedProjection = retainedProjectionForPayload(relayConfig.machineId, payload);
     if (!retainedProjection)
         return;
-    const { retainedKey } = retainedProjection;
-    const previous = retainedWriteQueues.get(retainedKey) ?? Promise.resolve();
-    const next = previous.catch(() => undefined).then(async () => {
-        if (!relayConfig)
+    await enqueueRetainedProjection(retainedProjection.retainedKey, retainedProjection.payload);
+}
+async function enqueueRetainedProjection(retainedKey, payload) {
+    await ensureRetainedOutboxLoaded();
+    const now = new Date().toISOString();
+    const existing = retainedOutbox.get(retainedKey);
+    const record = existing ?? {
+        retainedKey,
+        payload,
+        createdAt: now,
+        updatedAt: now,
+        generation: randomUUID(),
+        attemptCount: 0,
+    };
+    record.payload = payload;
+    record.updatedAt = now;
+    record.generation = randomUUID();
+    // A newer projection supersedes a retry deadline from an older payload.
+    // The per-key in-flight fence still allows the older HTTP request to finish
+    // without letting it delete or overwrite this newer record.
+    record.nextAttemptAt = undefined;
+    if (!existing)
+        retainedOutbox.set(retainedKey, record);
+    await persistRetainedOutbox();
+    scheduleRetainedOutboxFlush(0);
+}
+function scheduleRetainedOutboxFlush(delayOverrideMs) {
+    void ensureRetainedOutboxLoaded().then(() => {
+        if (retainedOutbox.size === 0)
             return;
-        const version = await putRetainedProjection(retainedKey, retainedProjection.payload);
-        retainedVersions.set(retainedKey, version);
+        const now = Date.now();
+        const dueAt = nextRetainedOutboxFlushAt(now);
+        if (dueAt === undefined && delayOverrideMs === undefined)
+            return;
+        const delay = delayOverrideMs ?? Math.max(0, (dueAt ?? now) - now);
+        const targetAt = now + Math.max(0, delay);
+        if (retainedOutboxFlushTimer && retainedOutboxFlushAt !== undefined && retainedOutboxFlushAt <= targetAt)
+            return;
+        if (retainedOutboxFlushTimer)
+            clearTimeout(retainedOutboxFlushTimer);
+        retainedOutboxFlushAt = targetAt;
+        retainedOutboxFlushTimer = setTimeout(() => {
+            retainedOutboxFlushTimer = undefined;
+            retainedOutboxFlushAt = undefined;
+            void flushRetainedOutbox().catch((error) => {
+                console.error(`Relay retained outbox flush failed: ${error instanceof Error ? error.message : String(error)}`);
+            });
+        }, Math.max(0, delay));
+        retainedOutboxFlushTimer.unref?.();
+    }).catch((error) => {
+        console.error(`Relay retained outbox schedule failed: ${error instanceof Error ? error.message : String(error)}`);
+    });
+}
+function flushRetainedOutbox() {
+    retainedOutboxDrainPromise ??= flushRetainedOutboxOnce().finally(() => {
+        retainedOutboxDrainPromise = null;
+        scheduleRetainedOutboxFlush();
     });
-    retainedWriteQueues.set(retainedKey, next);
+    return retainedOutboxDrainPromise;
+}
+async function flushRetainedOutboxOnce() {
+    await ensureRetainedOutboxLoaded();
+    const now = Date.now();
+    const candidates = [...retainedOutbox.values()]
+        .filter((record) => !retainedInFlightKeys.has(record.retainedKey))
+        .filter((record) => !record.nextAttemptAt || Date.parse(record.nextAttemptAt) <= now)
+        .sort((left, right) => left.updatedAt.localeCompare(right.updatedAt))
+        .slice(0, retainedDrainConcurrency);
+    await Promise.all(candidates.map((record) => attemptRetainedOutboxRecord(record)));
+}
+function nextRetainedOutboxFlushAt(now) {
+    let next;
+    for (const record of retainedOutbox.values()) {
+        if (retainedInFlightKeys.has(record.retainedKey))
+            continue;
+        const dueAt = record.nextAttemptAt ? Date.parse(record.nextAttemptAt) : now;
+        if (!Number.isFinite(dueAt))
+            continue;
+        next = next === undefined ? dueAt : Math.min(next, dueAt);
+    }
+    return next;
+}
+async function attemptRetainedOutboxRecord(record) {
+    retainedInFlightKeys.add(record.retainedKey);
+    const generation = record.generation;
+    record.attemptCount += 1;
+    record.nextAttemptAt = new Date(Date.now() + retainedRetryDelayMs(record.attemptCount)).toISOString();
+    await persistRetainedOutbox();
     try {
-        await next;
+        const version = await putRetainedProjection(record.retainedKey, record.payload);
+        retainedVersions.set(record.retainedKey, version);
+        const current = retainedOutbox.get(record.retainedKey);
+        if (current === record && current.generation === generation) {
+            retainedOutbox.delete(record.retainedKey);
+        }
+        await persistRetainedOutbox();
+    }
+    catch (error) {
+        const current = retainedOutbox.get(record.retainedKey);
+        if (current === record && current.generation === generation) {
+            current.nextAttemptAt = new Date(Date.now() + retainedRetryDelayMs(current.attemptCount)).toISOString();
+            await persistRetainedOutbox();
+        }
+        await appendRetainedOutboxAudit(record, "send-failed", error);
     }
     finally {
-        if (retainedWriteQueues.get(retainedKey) === next)
-            retainedWriteQueues.delete(retainedKey);
+        retainedInFlightKeys.delete(record.retainedKey);
+    }
+}
+function retainedRetryDelayMs(attemptCount) {
+    const exponent = Math.max(0, Math.min(attemptCount - 1, 5));
+    return Math.min(retainedOutboxRetryMaxMs, retainedOutboxRetryBaseMs * (2 ** exponent));
+}
+async function ensureRetainedOutboxLoaded() {
+    if (retainedOutboxLoaded)
+        return;
+    retainedOutboxLoadPromise ??= loadRetainedOutbox();
+    await retainedOutboxLoadPromise;
+}
+async function loadRetainedOutbox() {
+    retainedOutbox.clear();
+    try {
+        const text = await fs.readFile(relayRetainedOutboxPath, "utf8");
+        for (const line of text.split(/\n/u)) {
+            if (!line.trim())
+                continue;
+            const parsed = JSON.parse(line);
+            if (typeof parsed.retainedKey === "string" && parsed.retainedKey
+                && parsed.payload && typeof parsed.payload === "object"
+                && typeof parsed.createdAt === "string"
+                && typeof parsed.updatedAt === "string"
+                && typeof parsed.attemptCount === "number") {
+                retainedOutbox.set(parsed.retainedKey, {
+                    retainedKey: parsed.retainedKey,
+                    payload: parsed.payload,
+                    createdAt: parsed.createdAt,
+                    updatedAt: parsed.updatedAt,
+                    generation: typeof parsed.generation === "string" && parsed.generation ? parsed.generation : randomUUID(),
+                    attemptCount: parsed.attemptCount,
+                    ...(typeof parsed.nextAttemptAt === "string" ? { nextAttemptAt: parsed.nextAttemptAt } : {}),
+                });
+            }
+        }
+    }
+    catch (error) {
+        if (!(error instanceof Error) || !("code" in error) || error.code !== "ENOENT")
+            throw error;
     }
+    retainedOutboxLoaded = true;
+}
+async function persistRetainedOutbox() {
+    retainedOutboxWritePromise = retainedOutboxWritePromise.then(writeRetainedOutbox, writeRetainedOutbox);
+    await retainedOutboxWritePromise;
+}
+async function writeRetainedOutbox() {
+    await fs.mkdir(configDir, { recursive: true });
+    const text = [...retainedOutbox.values()].map((record) => JSON.stringify(record)).join("\n");
+    const next = text ? `${text}\n` : "";
+    const tempPath = `${relayRetainedOutboxPath}.tmp`;
+    await fs.writeFile(tempPath, next, { mode: 0o600 });
+    await fs.rename(tempPath, relayRetainedOutboxPath);
+    await fs.chmod(relayRetainedOutboxPath, 0o600).catch(() => undefined);
+}
+async function appendRetainedOutboxAudit(record, status, error) {
+    if (!relayConfig)
+        return;
+    await appendAudit({
+        machineId: relayConfig.machineId,
+        actor: "system",
+        action: `relay.retained-outbox.${status}`,
+        summary: "Retained relay projection retry scheduled",
+        evidence: {
+            retainedKey: record.retainedKey,
+            attemptCount: record.attemptCount,
+            error: error instanceof Error ? error.message : error ? String(error) : undefined,
+        },
+    });
 }
 export async function putEncryptedRetainedPayload(retainedKey, payload) {
     if (!relayConfig)
@@ -549,26 +738,32 @@ export async function putEncryptedRetainedPayload(retainedKey, payload) {
         const baseVersion = retainedVersions.get(retainedKey) ?? await fetchRetainedVersion(retainedKey);
         retainedVersions.set(retainedKey, baseVersion);
         const encryptedPayload = await encryptJson(relayConfig.accountSecret, `retained:${retainedKey}:${baseVersion + 1}`, payload);
-        const response = await fetch(`${relayConfig.relayUrl}/api/retained/${encodeURIComponent(retainedKey)}`, {
+        const timed = await fetchWithTimeout(`${relayConfig.relayUrl}/api/retained/${encodeURIComponent(retainedKey)}`, {
             method: "PUT",
             headers: {
                 authorization: `Bearer ${relayConfig.machineToken}`,
                 "content-type": "application/json",
             },
             body: JSON.stringify({ baseVersion, encryptedPayload }),
-        });
-        if (response.status === 409) {
-            retainedVersions.delete(retainedKey);
-            continue;
+        }, retainedHttpTimeoutMs);
+        const response = timed.response;
+        try {
+            if (response.status === 409) {
+                retainedVersions.delete(retainedKey);
+                continue;
+            }
+            if (!response.ok) {
+                throw new Error(`Retained packet write failed for ${retainedKey}: HTTP ${response.status}`);
+            }
+            const body = await readResponseJsonWithTimeout(response, retainedHttpTimeoutMs, timed.abort);
+            const version = body.retainedPacket?.version;
+            if (typeof version === "number" && Number.isSafeInteger(version) && version > 0)
+                return version;
+            throw new Error(`Retained packet write did not return a version for ${retainedKey}`);
         }
-        if (!response.ok) {
-            throw new Error(`Retained packet write failed for ${retainedKey}: HTTP ${response.status}`);
+        finally {
+            timed.cleanup();
         }
-        const body = await response.json();
-        const version = body.retainedPacket?.version;
-        if (typeof version === "number" && Number.isSafeInteger(version) && version > 0)
-            return version;
-        throw new Error(`Retained packet write did not return a version for ${retainedKey}`);
     }
     throw new Error(`Retained packet CAS conflict for ${retainedKey}`);
 }
@@ -579,19 +774,69 @@ async function fetchRetainedVersion(retainedKey) {
     if (!relayConfig)
         return 0;
     try {
-        const response = await fetch(`${relayConfig.relayUrl}/api/retained/${encodeURIComponent(retainedKey)}`, {
+        const timed = await fetchWithTimeout(`${relayConfig.relayUrl}/api/retained/${encodeURIComponent(retainedKey)}`, {
             headers: { authorization: `Bearer ${relayConfig.machineToken}` },
-        });
-        if (!response.ok)
-            return 0;
-        const body = await response.json();
-        const version = body.retainedPacket?.version;
-        return typeof version === "number" && Number.isSafeInteger(version) && version >= 0 ? version : 0;
+        }, retainedHttpTimeoutMs);
+        const response = timed.response;
+        try {
+            if (!response.ok)
+                return 0;
+            const body = await readResponseJsonWithTimeout(response, retainedHttpTimeoutMs, timed.abort);
+            const version = body.retainedPacket?.version;
+            return typeof version === "number" && Number.isSafeInteger(version) && version >= 0 ? version : 0;
+        }
+        finally {
+            timed.cleanup();
+        }
     }
     catch {
         return 0;
     }
 }
+async function fetchWithTimeout(input, init, timeoutMs) {
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(new Error("retained HTTP request timed out")), timeoutMs);
+    timeout.unref?.();
+    try {
+        const response = await fetch(input, { ...init, signal: controller.signal });
+        clearTimeout(timeout);
+        return {
+            response,
+            abort: (reason) => controller.abort(reason),
+            cleanup: () => {
+                clearTimeout(timeout);
+                // Closing the body also releases the socket when the caller exits
+                // early on a non-2xx response.  A body timeout calls abort() first;
+                // abort is idempotent and fences the underlying fetch stream.
+                if (!controller.signal.aborted)
+                    controller.abort(new Error("retained HTTP request finished"));
+            },
+        };
+    }
+    catch (error) {
+        clearTimeout(timeout);
+        throw error;
+    }
+}
+async function readResponseJsonWithTimeout(response, timeoutMs, abort) {
+    let timer;
+    try {
+        return await Promise.race([
+            response.json(),
+            new Promise((_, reject) => {
+                timer = setTimeout(() => {
+                    abort?.(new Error("retained HTTP response body timed out"));
+                    reject(new Error("retained HTTP response body timed out"));
+                }, timeoutMs);
+                timer.unref?.();
+            }),
+        ]);
+    }
+    finally {
+        if (timer)
+            clearTimeout(timer);
+    }
+}
 function retainedProjectionForPayload(machineId, payload) {
     if (!payload || typeof payload !== "object" || Array.isArray(payload))
         return undefined;

package/apps/daemon/dist/runtime/headless-event-projector.d.ts

@@ -41,3 +41,4 @@ export declare function limitHistoricalEventsByBytes(events: RuntimeHistoricalEv
 export declare function terminalHistoricalEventFromTurn(turn: Turn & {
     status: "completed" | "interrupted" | "failed";
 }): RuntimeHistoricalEvent;
+export declare function historicalEventsFromThread(items: ThreadItem[], turns: Turn[], limit: number | undefined, runtimeEvents?: HeadlessRuntimeEvent[]): RuntimeHistoricalEvent[];

package/apps/daemon/dist/runtime/headless-event-projector.js

@@ -48,12 +48,20 @@ export function mapHeadlessRuntimeEvent(event, state = createHeadlessRuntimeEven
         if (!item)
             return [];
         if (item.type === "agentMessage") {
+            // A repaired/replayed provider log can contain more than one terminal
+            // snapshot for the same item. The item id is the event identity; do not
+            // dedupe by text because two distinct items are allowed to say the same
+            // thing.
+            if (state.textItems.has(item.id))
+                return [];
             if ((!state.authoritativeTextItems.has(item.id) && (state.deltaItems.has(item.id) || state.textItems.has(item.id))) || !item.text)
                 return [];
             state.textItems.add(item.id);
             return [{ type: "text_delta", text: item.text, stream: "output", ...itemIdentity }];
         }
         if (item.type === "reasoning") {
+            if (state.textItems.has(item.id))
+                return [];
             if ((!state.authoritativeTextItems.has(item.id) && (state.deltaItems.has(item.id) || state.textItems.has(item.id))) || !item.text)
                 return [];
             state.textItems.add(item.id);
@@ -786,7 +794,7 @@ function historicalEventFromItem(item) {
     }
     return undefined;
 }
-function historicalEventsFromThread(items, turns, limit, runtimeEvents = []) {
+export function historicalEventsFromThread(items, turns, limit, runtimeEvents = []) {
     const events = [];
     const userPromptKeys = new Set();
     const addEvent = (event) => {
@@ -808,6 +816,16 @@ function historicalEventsFromThread(items, turns, limit, runtimeEvents = []) {
         if (event)
             addEvent(event);
     }
+    // Server requests are not represented by a canonical ThreadItem until
+    // they are resolved (and some providers never materialize them as items at
+    // all). Preserve the raw option lifecycle while the canonical item/turn
+    // projection supplies text, tools, and terminal state. This keeps an
+    // authoritative turn rebuild usable for live needs-user attention without
+    // reintroducing duplicate assistant snapshots.
+    for (const event of historicalEventsFromRuntimeEvents(runtimeEvents)) {
+        if (event.payload.type === "option_request" || event.payload.type === "option_response")
+            addEvent(event);
+    }
     for (const turn of turns) {
         if (!isTerminalTurn(turn))
             continue;

package/apps/daemon/dist/runtime/headless-history-reader.js

@@ -1,11 +1,11 @@
-import { limitHistoricalEventsByBytes, mapHeadlessRuntimeEventsToHistoricalEvents, selectThreadEventsAfterItem, selectThreadEventsBeforeCursor, selectThreadEventsBeforeItem, selectThreadEventsForTurn, } from "./headless-event-projector.js";
+import { limitHistoricalEventsByBytes, historicalEventsFromThread, mapHeadlessRuntimeEventsToHistoricalEvents, selectThreadEventsAfterItem, selectThreadEventsBeforeCursor, selectThreadEventsBeforeItem, selectThreadEventsForTurn, } from "./headless-event-projector.js";
 const defaultEventLimit = 200;
 export async function readHeadlessHistoricalEventPage(input) {
     assertValidHeadlessEventCursor(input.before);
     const limitItems = clampLimit(input.limitItems);
     const selection = await selectHeadlessRuntimeEvents(input, limitItems);
     const runtimeEvents = selection.events;
-    const allEvents = mapHeadlessRuntimeEventsToHistoricalEvents(runtimeEvents);
+    const allEvents = await authoritativeTurnEvents(input, runtimeEvents) ?? mapHeadlessRuntimeEventsToHistoricalEvents(runtimeEvents);
     const limited = limitHistoricalEventsByBytes(allEvents, input.maxBytes);
     const includeSessionHead = !input.turnId && !input.before && !input.beforeItemId;
     const head = includeSessionHead ? latestHeadFromEvents(allEvents) : {};
@@ -19,6 +19,53 @@ export async function readHeadlessHistoricalEventPage(input) {
         ...(head.lastTurnId ? { lastTurnId: head.lastTurnId } : {}),
     };
 }
+async function authoritativeTurnEvents(input, runtimeEvents) {
+    // A turn result is a rebuild operation, not a best-effort concatenation of
+    // every provider notification. Read the provider's canonical item/turn
+    // projections when there is no paging anchor; this repairs older logs that
+    // contain a delta plus a repeated terminal snapshot for the same item.
+    // Identity is itemId/turnId, never text, so legitimate repeated answers
+    // from distinct items remain visible.
+    if (!input.turnId || input.before || input.beforeItemId || input.afterItemId)
+        return undefined;
+    // A few internal readers (and older provider adapters during rolling
+    // upgrades) only expose the event reader.  Canonical rebuild is an
+    // enhancement, not a reason to turn an otherwise reachable runtime into an
+    // unavailable one; fall back to the event projection when the optional
+    // canonical methods are absent.
+    if (typeof input.client.listThreadItems !== "function" || typeof input.client.listThreadTurns !== "function")
+        return undefined;
+    const [items, turns] = await Promise.all([
+        listAllThreadItems(input.client, input.threadId, input.turnId),
+        listAllThreadTurns(input.client, input.threadId),
+    ]);
+    const turn = turns.find((candidate) => candidate.id === input.turnId);
+    return historicalEventsFromThread(items, turn ? [turn] : [], undefined, runtimeEvents);
+}
+async function listAllThreadItems(client, threadId, turnId) {
+    const items = [];
+    let afterCursor;
+    for (let pageIndex = 0; pageIndex < 64; pageIndex += 1) {
+        const page = await client.listThreadItems({ threadId, turnId, order: "asc", limit: 200, ...(afterCursor ? { afterCursor } : {}) });
+        items.push(...page.items);
+        if (!page.nextCursor || page.items.length === 0)
+            break;
+        afterCursor = page.nextCursor;
+    }
+    return items;
+}
+async function listAllThreadTurns(client, threadId) {
+    const turns = [];
+    let afterCursor;
+    for (let pageIndex = 0; pageIndex < 64; pageIndex += 1) {
+        const page = await client.listThreadTurns({ threadId, order: "asc", limit: 200, ...(afterCursor ? { afterCursor } : {}) });
+        turns.push(...page.turns);
+        if (!page.nextCursor || page.turns.length === 0)
+            break;
+        afterCursor = page.nextCursor;
+    }
+    return turns;
+}
 async function selectHeadlessRuntimeEvents(input, limitItems) {
     if (input.turnId) {
         return await selectThreadEventsForTurn(input.client, input.threadId, input.turnId, limitItems);

package/apps/daemon/dist/session/lifecycle.d.ts

@@ -1,11 +1,11 @@
 import { type MachineCommand } from "../../../../packages/shared/dist/index.js";
 import type { DaemonConfig, SessionFallback, SessionState } from "../types.js";
 import { type HistoricalBackfillMode, type HistoricalBackfillOptions, type HistoricalBackfillResult } from "./history-backfill.js";
+export { refreshNativeHistoryForLoadedSession } from "./native-history-refresh.js";
 export declare function handleCreateSession(ws: WebSocket, config: DaemonConfig, command: MachineCommand): Promise<void>;
 export declare function handleImportHistoricalSession(ws: WebSocket, config: DaemonConfig, command: MachineCommand): Promise<void>;
 export declare function handleListHistoricalSessions(ws: WebSocket, config: DaemonConfig, command: MachineCommand): Promise<void>;
 export declare function resumeSession(ws: WebSocket, config: DaemonConfig, sessionId: string, requestId: string, fallback?: SessionFallback, announceSuccess?: boolean, syncDirection?: HistoricalBackfillMode, syncOptions?: Pick<HistoricalBackfillOptions, "afterItemId" | "beforeItemId" | "limitItems" | "maxBytes">): Promise<SessionState | null>;
-export declare function refreshNativeHistoryForLoadedSession(session: SessionState, config: DaemonConfig, requestId: string): Promise<SessionState>;
 export declare function rehydrateRuntimeSession(session: SessionState): Promise<SessionState>;
 export declare function usesAuthoritativeTranscript(session: SessionState): boolean;
 export declare function replaceAuthoritativeTranscript(ws: WebSocket, config: DaemonConfig, session: SessionState, runtimeSessionId?: string, input?: {

package/apps/daemon/dist/session/lifecycle.js

@@ -18,6 +18,8 @@ import { backfillStateWithSessionHead, completedBackfillResult, historicalBackfi
 import { commandSessionMetadata, encryptedSessionMetadata, sessionFallbackFromCommand } from "./metadata.js";
 import { closeSessionProjection, persistSessionProjection, sessionReadyProjectionFields } from "./store.js";
 import { nativeHandoffIdentity } from "./native-identity.js";
+import { refreshNativeHistoryForLoadedSession } from "./native-history-refresh.js";
+export { refreshNativeHistoryForLoadedSession } from "./native-history-refresh.js";
 export async function handleCreateSession(ws, config, command) {
     if (command.type !== "machine:createSession")
         return;
@@ -782,90 +784,6 @@ export async function resumeSession(ws, config, sessionId, requestId, fallback,
         return null;
     }
 }
-export async function refreshNativeHistoryForLoadedSession(session, config, requestId) {
-    if (session.source !== "native")
-        return session;
-    const manager = getHeadlessRuntimeManager(session.cwd);
-    let lastError;
-    for (let attempt = 0; attempt < 2; attempt += 1) {
-        try {
-            if (attempt > 0)
-                await manager.invalidateClientForBinding(session.runtimeBinding, session.cwd);
-            const entry = await manager.clientForBinding(session.runtimeBinding, session.cwd);
-            if (typeof entry.client.syncNativeHistory !== "function")
-                return session;
-            const expectedNativeIdentity = nativeHandoffIdentity({
-                machineId: config.machineId,
-                provider: entry.provider,
-                profile: entry.profile,
-                nativeDataRootFingerprint: entry.nativeDataRootFingerprint,
-                cwd: session.cwd,
-                providerSessionId: session.providerSessionId ?? session.runtimeBinding.threadId,
-            });
-            const synced = await entry.client.syncNativeHistory({
-                threadId: session.runtimeBinding.threadId,
-                nativeIdentity: expectedNativeIdentity,
-            });
-            const providerMetadata = synced.thread.providerMetadata ?? {};
-            const externallyChanged = providerMetadata.nativeHistoryChangedExternally === true;
-            // Native handoff identities are derived from the current daemon/provider
-            // binding.  A session persisted by an earlier host build may carry an
-            // identity computed before the binding included the machine/root
-            // fingerprint.  Read-only sync has already verified the provider thread
-            // and its history, so refresh the derived identity here and require a
-            // fresh explicit handoff confirmation instead of rejecting the session
-            // forever with a stale proof.
-            const nextProof = externallyChanged
-                ? undefined
-                : session.manualHandoffProof && typeof providerMetadata.nativeHistoryVersion === "string"
-                    ? { ...session.manualHandoffProof, historyVersion: providerMetadata.nativeHistoryVersion }
-                    : session.manualHandoffProof;
-            const next = {
-                ...session,
-                nativeIdentity: expectedNativeIdentity,
-                ...(typeof providerMetadata.nativeHistoryVersion === "string" ? { nativeHistoryVersion: providerMetadata.nativeHistoryVersion } : {}),
-                ...(externallyChanged ? { manualHandoffProof: undefined } : nextProof ? { manualHandoffProof: nextProof } : {}),
-                ...(session.nativeHistorySyncError ? { nativeHistorySyncError: undefined } : {}),
-            };
-            if (nextProof && typeof providerMetadata.nativeHistoryVersion === "string" && nextProof.historyVersion !== providerMetadata.nativeHistoryVersion) {
-                next.manualHandoffProof = undefined;
-            }
-            sessions.set(session.sessionId, next);
-            await persistSessionProjection(next);
-            return next;
-        }
-        catch (error) {
-            lastError = error;
-            if (attempt === 0 && isNativeRuntimeTransportFailure(error))
-                continue;
-            break;
-        }
-    }
-    const code = errorCode(lastError) ?? "NATIVE_HISTORY_SYNC_FAILED";
-    const message = lastError instanceof Error ? lastError.message : String(lastError);
-    const failed = {
-        ...session,
-        nativeHistorySyncError: { code, message, checkedAt: new Date().toISOString() },
-    };
-    sessions.set(session.sessionId, failed);
-    await persistSessionProjection(failed);
-    await appendAudit({
-        sessionId: session.sessionId,
-        machineId: config.machineId,
-        actor: "system",
-        action: "session.native_history.refresh_failed",
-        summary: `Read-only native history refresh did not change session ${session.sessionId}`,
-        evidence: { requestId, code, error: message },
-    });
-    return failed;
-}
-function isNativeRuntimeTransportFailure(error) {
-    const code = errorCode(error);
-    if (code && /^(ECONN|EPIPE|RUNTIME_(?:DISPOSED|UNAVAILABLE)|HOST_(?:START|SHUTDOWN)|REQUEST_TIMEOUT)$/u.test(code))
-        return true;
-    const message = error instanceof Error ? error.message : String(error);
-    return /(headless runtime (?:socket|request|exited)|cannot write to headless runtime|socket (?:closed|read|error)|connection (?:reset|lost|refused)|timed out)/iu.test(message);
-}
 function runtimeBindingsMatch(left, right) {
     return left.runtime === right.runtime &&
         left.provider === right.provider &&

package/apps/daemon/dist/session/native-history-refresh.d.ts

@@ -0,0 +1,12 @@
+import type { DaemonConfig, SessionState } from "../types.js";
+/**
+ * Reconcile provider-native history before a loaded session is projected.
+ *
+ * Native sessions have always used this path.  Legacy Crew sessions were
+ * persisted as `source:"orchestration"` but still carry a provider session
+ * id and native-history metadata.  Those sessions must take the same
+ * read-only reconciliation path after restart or before runtime truth reads;
+ * otherwise an old same-turn duplicate remains visible until an internal
+ * caller manually invokes the runtime operation.
+ */
+export declare function refreshNativeHistoryForLoadedSession(session: SessionState, config: DaemonConfig | undefined, requestId: string): Promise<SessionState>;

package/apps/daemon/dist/session/native-history-refresh.js

@@ -0,0 +1,183 @@
+import { appendAudit } from "../audit.js";
+import { getHeadlessRuntimeManager } from "../runtime.js";
+import { sessions } from "../state.js";
+import { persistSessionProjection } from "./store.js";
+import { nativeHandoffIdentity } from "./native-identity.js";
+const orchestrationRefreshes = new Map();
+const orchestrationRefreshCompleted = new Set();
+const orchestrationRefreshRetryAt = new Map();
+const orchestrationRefreshRetryDelayMs = 1_000;
+/**
+ * Reconcile provider-native history before a loaded session is projected.
+ *
+ * Native sessions have always used this path.  Legacy Crew sessions were
+ * persisted as `source:"orchestration"` but still carry a provider session
+ * id and native-history metadata.  Those sessions must take the same
+ * read-only reconciliation path after restart or before runtime truth reads;
+ * otherwise an old same-turn duplicate remains visible until an internal
+ * caller manually invokes the runtime operation.
+ */
+export async function refreshNativeHistoryForLoadedSession(session, config, requestId) {
+    const isNative = session.source === "native";
+    const isLegacyCrew = session.source === "orchestration" && Boolean(session.providerSessionId);
+    if (!isNative && !isLegacyCrew)
+        return session;
+    const refreshKey = `${session.runtimeBinding.profile}\0${session.runtimeBinding.threadId}`;
+    if (isLegacyCrew) {
+        if (orchestrationRefreshCompleted.has(refreshKey))
+            return session;
+        const retryAt = orchestrationRefreshRetryAt.get(refreshKey);
+        if (retryAt !== undefined) {
+            if (retryAt > Date.now())
+                return session;
+            orchestrationRefreshRetryAt.delete(refreshKey);
+        }
+        const active = orchestrationRefreshes.get(refreshKey);
+        if (active)
+            return await active;
+        const operation = refreshNativeHistory(session, config, requestId, false, refreshKey);
+        orchestrationRefreshes.set(refreshKey, operation);
+        try {
+            return await operation;
+        }
+        finally {
+            if (orchestrationRefreshes.get(refreshKey) === operation)
+                orchestrationRefreshes.delete(refreshKey);
+        }
+    }
+    return await refreshNativeHistory(session, config, requestId, true);
+}
+async function refreshNativeHistory(session, config, requestId, strict, orchestrationRefreshKey) {
+    const manager = getHeadlessRuntimeManager(session.cwd);
+    let lastError;
+    for (let attempt = 0; attempt < 2; attempt += 1) {
+        try {
+            if (attempt > 0)
+                await manager.invalidateClientForBinding(session.runtimeBinding, session.cwd);
+            const entry = await manager.clientForBinding(session.runtimeBinding, session.cwd);
+            if (typeof entry.client.syncNativeHistory !== "function") {
+                if (orchestrationRefreshKey)
+                    orchestrationRefreshCompleted.add(orchestrationRefreshKey);
+                return session;
+            }
+            const providerSessionId = session.providerSessionId ?? session.runtimeBinding.threadId;
+            const expectedNativeIdentity = nativeHandoffIdentity({
+                ...(config?.machineId ? { machineId: config.machineId } : {}),
+                provider: entry.provider,
+                profile: entry.profile,
+                nativeDataRootFingerprint: entry.nativeDataRootFingerprint,
+                cwd: session.cwd,
+                providerSessionId,
+            });
+            if (orchestrationRefreshKey && typeof entry.client.readThread === "function" && typeof entry.client.adoptThread === "function") {
+                const current = await entry.client.readThread({ threadId: session.runtimeBinding.threadId });
+                const metadata = current.thread.providerMetadata ?? {};
+                // Legacy Crew threads may have provider history metadata but no
+                // explicit adoption status because they were created before the
+                // native-history state machine existed.  Complete that migration on
+                // the normal load/read path, preserving the existing HE thread id.
+                if (metadata.headlessNativeAdoptionStatus !== "complete" && Array.isArray(metadata.nativeHistoryTurnIds)) {
+                    await entry.client.adoptThread({
+                        threadId: session.runtimeBinding.threadId,
+                        providerSessionId,
+                        cwd: session.cwd,
+                        provider: entry.provider,
+                        nativeIdentity: expectedNativeIdentity,
+                        manualHandoff: true,
+                    });
+                }
+            }
+            const synced = await entry.client.syncNativeHistory({
+                threadId: session.runtimeBinding.threadId,
+                nativeIdentity: expectedNativeIdentity,
+            });
+            const providerMetadata = synced.thread.providerMetadata ?? {};
+            const externallyChanged = providerMetadata.nativeHistoryChangedExternally === true;
+            // A provider thread already verified by read-only sync can refresh a
+            // stale proof produced by an earlier host build. Native sessions retain
+            // the existing safety behavior; legacy Crew sessions only update the
+            // durable observation and never become native-handoff sessions.
+            const nextProof = externallyChanged
+                ? undefined
+                : session.manualHandoffProof && typeof providerMetadata.nativeHistoryVersion === "string"
+                    ? { ...session.manualHandoffProof, historyVersion: providerMetadata.nativeHistoryVersion }
+                    : session.manualHandoffProof;
+            const next = {
+                ...session,
+                ...(strict && isUsableString(expectedNativeIdentity) ? { nativeIdentity: expectedNativeIdentity } : {}),
+                ...(typeof providerMetadata.nativeHistoryVersion === "string" ? { nativeHistoryVersion: providerMetadata.nativeHistoryVersion } : {}),
+                ...(strict && externallyChanged
+                    ? { manualHandoffProof: undefined }
+                    : strict && nextProof
+                        ? { manualHandoffProof: nextProof }
+                        : {}),
+                ...(strict && session.nativeHistorySyncError ? { nativeHistorySyncError: undefined } : {}),
+            };
+            sessions.set(session.sessionId, next);
+            await persistSessionProjection(next);
+            if (orchestrationRefreshKey) {
+                orchestrationRefreshRetryAt.delete(orchestrationRefreshKey);
+                orchestrationRefreshCompleted.add(orchestrationRefreshKey);
+            }
+            return next;
+        }
+        catch (error) {
+            lastError = error;
+            if (attempt === 0 && isNativeRuntimeTransportFailure(error))
+                continue;
+            break;
+        }
+    }
+    if (!strict) {
+        // Legacy Crew reconciliation is best-effort.  The authoritative HE event
+        // log remains readable when a provider cannot perform native refresh; do
+        // not turn a harmless compatibility probe into a session failure.
+        if (orchestrationRefreshKey) {
+            if (isDefinitelyUnsupportedLegacyRefreshError(lastError))
+                orchestrationRefreshCompleted.add(orchestrationRefreshKey);
+            else
+                orchestrationRefreshRetryAt.set(orchestrationRefreshKey, Date.now() + orchestrationRefreshRetryDelayMs);
+        }
+        return session;
+    }
+    const code = errorCode(lastError) ?? "NATIVE_HISTORY_SYNC_FAILED";
+    const message = lastError instanceof Error ? lastError.message : String(lastError);
+    const failed = {
+        ...session,
+        nativeHistorySyncError: { code, message, checkedAt: new Date().toISOString() },
+    };
+    sessions.set(session.sessionId, failed);
+    await persistSessionProjection(failed);
+    if (config) {
+        await appendAudit({
+            sessionId: session.sessionId,
+            machineId: config.machineId,
+            actor: "system",
+            action: "session.native_history.refresh_failed",
+            summary: `Read-only native history refresh did not change session ${session.sessionId}`,
+            evidence: { requestId, code, error: message },
+        });
+    }
+    return failed;
+}
+function isUsableString(value) {
+    return typeof value === "string" && value.length > 0;
+}
+function isDefinitelyUnsupportedLegacyRefreshError(error) {
+    const code = errorCode(error);
+    return code === "CAPABILITY_UNSUPPORTED";
+}
+function errorCode(error) {
+    if (!error || typeof error !== "object")
+        return undefined;
+    const code = error.code;
+    return typeof code === "string" ? code : undefined;
+}
+function isNativeRuntimeTransportFailure(error) {
+    const code = errorCode(error);
+    if (code && /^(ECONN|EPIPE|RUNTIME_(?:DISPOSED|UNAVAILABLE)|HOST_(?:START|SHUTDOWN)|REQUEST_TIMEOUT)$/u.test(code))
+        return true;
+    const message = error instanceof Error ? error.message : String(error);
+    return /(headless runtime (?:socket|request|exited)|cannot write to headless runtime|socket (?:closed|read|error)|connection (?:reset|lost|refused)|timed out)/iu.test(message);
+}
+//# sourceMappingURL=native-history-refresh.js.map

package/apps/daemon/dist/session/runtime-truth.js

@@ -5,13 +5,19 @@ import { runtimeTurnIdForCrewTurn } from "../orchestration/store.js";
 import { runtimeTurnIdForInboundPrompt } from "../relay/prompt-admission-store.js";
 import { loadSessionProjections } from "./store.js";
 import { runtimeSessionIdForSession } from "./he-official/common.js";
+import { refreshNativeHistoryForLoadedSession } from "./native-history-refresh.js";
 export async function runtimeRead(config, input) {
-    void config;
     let runtimeSessionId;
     try {
-        const session = await sessionStateForRuntimeTruth(input.sessionId);
+        let session = await sessionStateForRuntimeTruth(input.sessionId);
         if (!session)
             return unreachable(input, `Session ${input.sessionId} is not loaded on this daemon`, undefined, "SESSION_NOT_FOUND");
+        // Legacy Crew sessions were persisted as source=orchestration even though
+        // their Headless thread already carries provider-native history metadata.
+        // Reconcile that history before projecting turn.result/status so an
+        // upgrade/restart repairs old same-turn duplicates through the official
+        // read path, without requiring an internal syncNativeHistory call.
+        session = await refreshNativeHistoryForLoadedSession(session, config, `runtime_truth_refresh_${input.requestId}`);
         runtimeSessionId = runtimeSessionIdForSession(session);
         if (!runtimeSessionId)
             return unreachable(input, `Session ${input.sessionId} does not have a runtime session binding`, undefined, "SESSION_BINDING_MISSING");

package/apps/daemon/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cydm/happy-elves-daemon",
-  "version": "0.1.0-beta.488",
+  "version": "0.1.0-beta.489",
   "private": true,
   "type": "module"
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cydm/happy-elves",
-  "version": "0.1.0-beta.488",
+  "version": "0.1.0-beta.489",
   "description": "Remote controller for local coding agents with hosted or self-hosted relay support.",
   "type": "module",
   "bin": {

package/packages/headless-runtime/dist/core.js

@@ -514,6 +514,28 @@ export class HeadlessRuntimeCore {
                 changed = true;
             }
             const canonicalMapping = mapNativeItemsToProjection(sourceTurn.items, canonicalItems, sourceItemBindings, sourceTurn.id);
+            // A pre-native-history projection can contain multiple HE text items for
+            // one provider-native item (the old Codex/Claude resume path emitted a
+            // second completion item in the same turn). Once the provider gives us
+            // the authoritative ordered item list, remove only unmapped text items
+            // whose type and exact text are already represented by a mapped native
+            // item. We deliberately do not dedupe by text alone: a native turn may
+            // legitimately contain two equal assistant messages, and tool/status
+            // items outside the native text list must remain visible.
+            if (canonicalMapping.valid && canonicalMapping.itemIds.every((itemId) => typeof itemId === "string")) {
+                const mappedSourceItems = sourceTurn.items.filter((_, index) => canonicalMapping.itemIds[index]);
+                const mappedItemIds = new Set(canonicalMapping.itemIds);
+                const duplicateTextItemIds = canonicalItems
+                    .filter((item) => !mappedItemIds.has(item.id))
+                    .filter((item) => item.type === "agentMessage" || item.type === "userMessage")
+                    .filter((item) => mappedSourceItems.some((sourceItem) => (sourceItem.kind === "assistant" ? item.type === "agentMessage" : item.type === "userMessage")
+                    && item.text === sourceItem.text))
+                    .map((item) => item.id);
+                if (duplicateTextItemIds.length > 0) {
+                    removedItemIds.push(...duplicateTextItemIds);
+                    changed = true;
+                }
+            }
             const duplicateMapping = duplicateTurn
                 ? mapNativeItemsToProjection(sourceTurn.items, duplicateItems, sourceItemBindings, sourceTurn.id)
                 : undefined;