@averyyy/pi-coding-agent 0.82.1-piclient.5 → 0.82.1-piclient.6

package/dist/core/pi-server-client.js

@@ -50,7 +50,7 @@ const PI_SERVER_RESPONSE_NO_PROGRESS_TIMEOUT_MS = 90_000;
 const PI_SERVER_ERROR_BODY_MAX_BYTES = 64 * 1024;
 const PI_SERVER_EVENT_STREAM_MAX_LINE_BYTES = 256 * 1024 * 1024;
 const TRANSIENT_PI_SERVER_RETRY_DELAYS_MS = [250, 750, 1500];
-const TRANSIENT_PI_SERVER_STATUS_CODES = new Set([408, 425, 429, 502, 503, 504, 530]);
+const TRANSIENT_PI_SERVER_STATUS_CODES = new Set([408, 425, 429, 502, 503, 504, 520, 521, 522, 523, 524, 525, 530]);
 export function hashStaticContext(ctx) {
     return hashPiServerStaticContext(ctx);
 }
@@ -946,7 +946,7 @@ function isRecoverableTreeDivergenceError(error) {
 function isRecoverableMissingServerState(response, errorBody) {
     if (response.status !== 400 && response.status !== 404)
         return false;
-    return /Session has no static context|session not found|parent entry .* does not exist|leafId .* does not exist|entry .* already exists/i.test(errorBody);
+    return /Session has no static context|session not found|(?:stream )?run not found|parent entry .* does not exist|leafId .* does not exist|entry .* already exists/i.test(errorBody);
 }
 async function syncFullPiServerTree(sessionId, tree, request) {
     await postTreeJson(request, "/api/session/tree/sync", {
@@ -1374,6 +1374,7 @@ export async function compactPiServer(model, context, options) {
     const recovered = await recoverPendingPiServerCompaction(sessionId, tree, compactStatePath, {
         signal,
         recoveryWindowMs,
+        onHistoryReconciled: options?.onHistoryReconciled,
     });
     if (recovered)
         return recovered;
@@ -1496,6 +1497,14 @@ export async function compactPiServer(model, context, options) {
                     }
                     return await persistAndCompletePiServerCompactTerminal(terminal, compactStatePath, signal);
                 }
+                if (isRecoverableMissingServerState(response, failure.matchText)) {
+                    resetSessionTracking(sessionId);
+                    await ensureSessionInit(sessionId, context, request);
+                    await syncPiServerTreeWithRequest(sessionId, tree, request, options?.onHistoryReconciled);
+                    lastInterruption = new Error(`Server compaction state was rebuilt (${failure.details})`);
+                    recoveryDeadline ??= Date.now() + recoveryWindowMs;
+                    continue;
+                }
                 if (isRecoverablePiServerCompactResponse(response)) {
                     lastInterruption = new Error(`Server compaction failed (${failure.details})`);
                     recoveryDeadline ??= Date.now() + recoveryWindowMs;
@@ -1572,25 +1581,20 @@ export async function compactPiServer(model, context, options) {
         signal?.removeEventListener("abort", onAbort);
     }
 }
-function assertPendingPiServerCompactMatches(pending, sessionId, tree) {
-    if (pending.serverHash !== hashPiServerIdentity(getServerUrl()) ||
-        pending.sessionId !== sessionId ||
-        pending.baseEntryCount > tree.entries.length) {
-        throw new Error(`Cannot safely resume pending pi-server compaction ${pending.operationId}: server, session, or tree identity changed`);
+function getPendingPiServerCompactBase(pending, sessionId, tree) {
+    if (pending.sessionId !== sessionId) {
+        throw new Error(`Cannot recover pending pi-server compaction ${pending.operationId}: durable marker belongs to another session`);
     }
+    if (pending.baseEntryCount > tree.entries.length)
+        return undefined;
     const baseEntries = tree.entries.slice(0, pending.baseEntryCount);
     if (hashEntries(baseEntries) !== pending.baseTreeHash ||
         (pending.baseLeafId !== null && !baseEntries.some((entry) => entry.id === pending.baseLeafId))) {
-        throw new Error(`Cannot safely resume pending pi-server compaction ${pending.operationId}: local base tree no longer matches its durable marker`);
-    }
-    if (!pending.observation || pending.observation.kind === "terminal") {
-        if (tree.entries.length !== pending.baseEntryCount || tree.leafId !== pending.baseLeafId) {
-            throw new Error(`Cannot safely resume pending pi-server compaction ${pending.operationId}: local tree changed before a committed compaction was observed`);
-        }
+        return undefined;
     }
     return { entries: baseEntries, leafId: pending.baseLeafId };
 }
-function recoverLocallyAppliedPiServerCompaction(pending, tree, baseTree, compactStatePath) {
+function tryRecoverLocallyAppliedPiServerCompaction(pending, tree, baseTree, compactStatePath) {
     const applied = pending.observation;
     if (!applied || applied.kind !== "applied") {
         throw new Error("Pending pi-server compaction does not contain a durable local apply observation");
@@ -1604,7 +1608,7 @@ function recoverLocallyAppliedPiServerCompaction(pending, tree, baseTree, compac
         applied.updatedLeafId !== applied.entryId ||
         hashEntries(tree.entries) !== applied.updatedTreeHash ||
         applied.updatedRevision !== pending.baseRevision + 1) {
-        throw new Error(`Cannot safely resume pending pi-server compaction ${pending.operationId}: local applied tree does not match its durable observation`);
+        return undefined;
     }
     return normalizePiServerCompactionResult({
         protocolVersion: PI_SERVER_PROTOCOL_VERSION,
@@ -1635,10 +1639,22 @@ export async function recoverPendingPiServerCompaction(sessionId, tree, compactS
     const pending = readPiServerPendingCompact(compactStatePath);
     if (!pending)
         return undefined;
-    const baseTree = assertPendingPiServerCompactMatches(pending, sessionId, tree);
-    const locallyApplied = pending.observation?.kind === "applied"
-        ? recoverLocallyAppliedPiServerCompaction(pending, tree, baseTree, compactStatePath)
+    if (pending.sessionId !== sessionId) {
+        acknowledgePiServerPendingCompact(compactStatePath, pending.operationId, "server_authoritative_missing");
+        return undefined;
+    }
+    if (pending.serverHash !== hashPiServerIdentity(getServerUrl())) {
+        resetSessionTracking(sessionId);
+    }
+    let baseTree = getPendingPiServerCompactBase(pending, sessionId, tree);
+    const locallyApplied = pending.observation?.kind === "applied" && baseTree
+        ? tryRecoverLocallyAppliedPiServerCompaction(pending, tree, baseTree, compactStatePath)
         : undefined;
+    const localIdentityMatches = pending.serverHash === hashPiServerIdentity(getServerUrl()) &&
+        baseTree !== undefined &&
+        (pending.observation?.kind === "applied"
+            ? locallyApplied !== undefined
+            : tree.entries.length === pending.baseEntryCount && tree.leafId === pending.baseLeafId);
     if (pending.observation?.kind === "terminal" && pending.observation.operationDisposition === "not_started") {
         return await persistAndCompletePiServerCompactTerminal({
             protocolVersion: PI_SERVER_PROTOCOL_VERSION,
@@ -1696,8 +1712,15 @@ export async function recoverPendingPiServerCompaction(sessionId, tree, compactS
                     await acknowledgeObservedPiServerCompactTerminal(pending, compactStatePath, options?.signal);
                     throw new PiServerCompactOperationError(`Pending server compaction failed (${pending.observation.error})`, pending.observation);
                 }
-                throw new Error(`Pending pi-server compaction ${pending.operationId} is missing from the durable server journal; ` +
-                    "the local marker was retained and automatic provider resubmission is disabled");
+                const snapshot = await fetchPiServerHistory(sessionId, request);
+                if (snapshot) {
+                    await applyPiServerHistory(sessionId, snapshot, options?.onHistoryReconciled);
+                }
+                else {
+                    resetSessionTracking(sessionId);
+                }
+                acknowledgePiServerPendingCompact(compactStatePath, pending.operationId, "server_authoritative_missing");
+                return undefined;
             }
             if (!isRecoverablePiServerCompactResponse(response)) {
                 throw new Error(`Pending server compaction recovery failed (${failure.details})`);
@@ -1708,6 +1731,20 @@ export async function recoverPendingPiServerCompaction(sessionId, tree, compactS
         }
         try {
             const responseResult = await readPiServerCompactResponse(response, "Pending server compaction recovery failed");
+            if (!localIdentityMatches) {
+                const snapshot = await fetchPiServerHistory(sessionId, request);
+                if (!snapshot) {
+                    throw new PiServerCompactProtocolError(`Pending server compaction recovery failed (operation journal exists but authoritative session history is missing)`);
+                }
+                baseTree = getPendingPiServerCompactBase(pending, sessionId, snapshot);
+                if (!baseTree) {
+                    throw new PiServerCompactProtocolError(`Pending server compaction recovery failed (authoritative history did not contain the operation base tree)`);
+                }
+                await applyPiServerHistory(sessionId, snapshot, options?.onHistoryReconciled);
+            }
+            if (!baseTree) {
+                throw new PiServerCompactProtocolError(`Pending server compaction recovery failed (local operation base tree was unavailable)`);
+            }
             const result = normalizePiServerCompactionResult(responseResult, sessionId, pending.operationId, pending.requestHash, baseTree, {
                 staticContextHash: pending.baseStaticContextHash,
                 treeHash: pending.baseTreeHash,
@@ -1831,31 +1868,28 @@ function findPendingPiServerRunTerminal(entries, pending) {
         if (entry.type !== "message" || entry.message.role !== "assistant")
             continue;
         const message = entry.message;
-        const diagnostic = getPiServerRunDiagnostic(message);
+        let diagnostic;
+        try {
+            diagnostic = getPiServerRunDiagnostic(message);
+        }
+        catch {
+            continue;
+        }
         if (!diagnostic || diagnostic.runId !== pending.runId)
             continue;
         if (diagnostic.sessionId !== pending.sessionId || diagnostic.requestMac !== pending.requestHash) {
-            throw new Error("Cannot recover pi-server run: persisted terminal diagnostic did not match pending state");
+            continue;
         }
         return message;
     }
     return undefined;
 }
-function assertPendingPiServerRunMatches(pending, serverHash, sessionId, tree) {
-    if (pending.serverHash !== serverHash ||
+function pendingPiServerRunBaseMatches(pending, serverHash, sessionId, tree) {
+    return !(pending.serverHash !== serverHash ||
         pending.sessionId !== sessionId ||
-        pending.baseEntryCount > tree.entries.length ||
+        pending.baseEntryCount !== tree.entries.length ||
         hashLocalTreePrefix(sessionId, tree.entries, pending.baseEntryCount) !== pending.baseTreeHash ||
-        pending.baseLeafId !== tree.leafId) {
-        throw new Error(`Cannot safely resume pending pi-server run ${pending.runId}: server, tree, or provider request identity changed`);
-    }
-}
-function missingDurablePiServerRunError(runId, cause) {
-    const prefix = cause ? `${cause}; ` : "";
-    return new Error(`${prefix}pi-server no longer has durable journal for pending run ${runId}. ` +
-        "The provider outcome is unknown and the pending marker was retained. Restore the server journal, " +
-        "or after independently verifying that the provider did not execute, start an explicit new session or fork; " +
-        "automatic provider resubmission is disabled.");
+        pending.baseLeafId !== tree.leafId);
 }
 export async function streamPiServer(model, context, options) {
     const runMode = options?.runMode;
@@ -2134,12 +2168,11 @@ export async function streamPiServer(model, context, options) {
         };
         try {
             const request = createPiServerRequest(options?.signal);
-            await ensureSessionInit(sessionId, context, request);
             const tree = options?.sessionTree ?? {
                 ...getLinearTreeFromMessages(context.messages),
                 replace: true,
             };
-            const syncTree = tree;
+            let syncTree = tree;
             if (runStatePath) {
                 attachRunStateLease();
             }
@@ -2148,6 +2181,9 @@ export async function streamPiServer(model, context, options) {
                 : sessionPendingRuns.get(sessionId);
             unresolvedRunPending = pending !== undefined;
             let treeSynced = false;
+            if (pending && pending.serverHash !== hashPiServerIdentity(getServerUrl())) {
+                resetSessionTracking(sessionId);
+            }
             if (pending) {
                 const persistedTerminal = findPendingPiServerRunTerminal(syncTree.entries, pending);
                 if (persistedTerminal) {
@@ -2162,6 +2198,9 @@ export async function streamPiServer(model, context, options) {
                     unresolvedRunPending = false;
                 }
             }
+            if (!pending) {
+                await ensureSessionInit(sessionId, context, request);
+            }
             const buildRequestIdentity = (baseTreeHash, baseEntryCount, baseLeafId, baseStaticContextHash, baseRevision) => JSON.parse(JSON.stringify({
                 sessionId,
                 runMode,
@@ -2185,14 +2224,95 @@ export async function streamPiServer(model, context, options) {
             let resumedPendingRun = false;
             let recoveredPendingRun;
             if (pending) {
-                assertPendingPiServerRunMatches(pending, hashPiServerIdentity(getServerUrl()), sessionId, syncTree);
+                if (pending.sessionId !== sessionId) {
+                    if (runStatePath) {
+                        acknowledgePiServerPendingRun(runStatePath, pending.runId, Date.now(), runStateLease, "server_authoritative_rebind");
+                    }
+                    else {
+                        sessionPendingRuns.delete(sessionId);
+                    }
+                    pending = undefined;
+                    unresolvedRunPending = false;
+                    await ensureSessionInit(sessionId, context, request);
+                }
+            }
+            if (pending) {
                 runId = pending.runId;
                 phase = "provider_stream";
                 recoveredPendingRun = await recoverPiServerRun(sessionId, runId, request, options?.signal);
                 if (recoveredPendingRun.status === "not_found") {
-                    throw missingDurablePiServerRunError(pending.runId);
+                    phase = "history_reconcile";
+                    const snapshot = await fetchPiServerHistory(sessionId, request);
+                    if (snapshot) {
+                        syncTree = snapshot;
+                        await applyPiServerHistory(sessionId, snapshot, options?.onHistoryReconciled);
+                        treeSynced = true;
+                    }
+                    else {
+                        resetSessionTracking(sessionId);
+                    }
+                    if (runStatePath) {
+                        acknowledgePiServerPendingRun(runStatePath, pending.runId, Date.now(), runStateLease, "server_authoritative_missing");
+                    }
+                    else {
+                        sessionPendingRuns.delete(sessionId);
+                    }
+                    pending = undefined;
+                    unresolvedRunPending = false;
+                    runId = randomUUID();
+                    await ensureSessionInit(sessionId, context, request);
+                    phase = "provider_stream";
                 }
                 else {
+                    if (recoveredPendingRun.status !== "unavailable" &&
+                        recoveredPendingRun.requestMac !== pending.requestHash) {
+                        phase = "history_reconcile";
+                        const snapshot = await fetchPiServerHistory(sessionId, request);
+                        if (snapshot) {
+                            syncTree = snapshot;
+                            await applyPiServerHistory(sessionId, snapshot, options?.onHistoryReconciled);
+                            treeSynced = true;
+                        }
+                        const authoritativeTree = snapshot ?? syncTree;
+                        const pendingInput = {
+                            serverHash: hashPiServerIdentity(getServerUrl()),
+                            sessionId,
+                            runId: pending.runId,
+                            baseTreeHash: hashLocalTree(sessionId, authoritativeTree.entries),
+                            baseEntryCount: authoritativeTree.entries.length,
+                            baseLeafId: authoritativeTree.leafId,
+                            requestHash: recoveredPendingRun.requestMac,
+                        };
+                        if (runStatePath) {
+                            acknowledgePiServerPendingRun(runStatePath, pending.runId, Date.now(), runStateLease, "server_authoritative_rebind");
+                            pending = writePiServerPendingRun(runStatePath, pendingInput, runStateLease);
+                        }
+                        else {
+                            pending = {
+                                version: 1,
+                                kind: "run",
+                                sequence: 0,
+                                timestamp: Date.now(),
+                                ...pendingInput,
+                            };
+                            sessionPendingRuns.set(sessionId, pending);
+                        }
+                        phase = "provider_stream";
+                    }
+                    if (recoveredPendingRun.status !== "unavailable" &&
+                        !pendingPiServerRunBaseMatches(pending, hashPiServerIdentity(getServerUrl()), sessionId, syncTree)) {
+                        phase = "history_reconcile";
+                        const snapshot = await fetchPiServerHistory(sessionId, request);
+                        if (!snapshot) {
+                            resetSessionTracking(sessionId);
+                        }
+                        else {
+                            syncTree = snapshot;
+                            await applyPiServerHistory(sessionId, snapshot, options?.onHistoryReconciled);
+                            treeSynced = true;
+                        }
+                        phase = "provider_stream";
+                    }
                     resumedPendingRun = true;
                     runSubmissionStarted = true;
                     requestHash = pending.requestHash;
@@ -2231,6 +2351,7 @@ export async function streamPiServer(model, context, options) {
                     };
                 if (!runStatePath)
                     sessionPendingRuns.set(sessionId, newPending);
+                pending = newPending;
                 unresolvedRunPending = true;
             }
             const makeBody = (eventCursor) => ({
@@ -2362,13 +2483,147 @@ export async function streamPiServer(model, context, options) {
                     recoveredRun.status === "completed" ||
                     recoveredRun.status === "failed";
                 if (hasDurableRun && recoveredRun.requestMac !== requestHash) {
-                    await settleStreamFailure(new Error("pi-server recovered run request identity did not match the pending provider request"), false);
-                    return;
+                    phase = "history_reconcile";
+                    const snapshot = await fetchPiServerHistory(sessionId, request);
+                    if (snapshot) {
+                        syncTree = snapshot;
+                        await applyPiServerHistory(sessionId, snapshot, options?.onHistoryReconciled);
+                        treeSynced = true;
+                    }
+                    else {
+                        resetSessionTracking(sessionId);
+                        treeSynced = false;
+                    }
+                    const authoritativeTree = snapshot ?? syncTree;
+                    const reboundPendingInput = {
+                        serverHash: hashPiServerIdentity(getServerUrl()),
+                        sessionId,
+                        runId,
+                        baseTreeHash: hashLocalTree(sessionId, authoritativeTree.entries),
+                        baseEntryCount: authoritativeTree.entries.length,
+                        baseLeafId: authoritativeTree.leafId,
+                        requestHash: recoveredRun.requestMac,
+                    };
+                    if (runStatePath) {
+                        acknowledgePiServerPendingRun(runStatePath, runId, Date.now(), runStateLease, "server_authoritative_rebind");
+                        pending = writePiServerPendingRun(runStatePath, reboundPendingInput, runStateLease);
+                    }
+                    else {
+                        pending = {
+                            version: 1,
+                            kind: "run",
+                            sequence: 0,
+                            timestamp: Date.now(),
+                            ...reboundPendingInput,
+                        };
+                        sessionPendingRuns.set(sessionId, pending);
+                    }
+                    requestHash = recoveredRun.requestMac;
+                    phase = "provider_stream";
                 }
                 if (recoveredRun.status === "not_found") {
-                    const failureMessage = interruption instanceof Error ? interruption.message : String(interruption);
-                    await settleStreamFailure(missingDurablePiServerRunError(runId, failureMessage), false);
-                    return;
+                    phase = "history_reconcile";
+                    const snapshot = await fetchPiServerHistory(sessionId, request);
+                    if (snapshot) {
+                        syncTree = snapshot;
+                        await applyPiServerHistory(sessionId, snapshot, options?.onHistoryReconciled);
+                        treeSynced = true;
+                    }
+                    else {
+                        resetSessionTracking(sessionId);
+                        treeSynced = false;
+                    }
+                    if (runStatePath) {
+                        acknowledgePiServerPendingRun(runStatePath, runId, Date.now(), runStateLease, "server_authoritative_missing");
+                    }
+                    else {
+                        sessionPendingRuns.delete(sessionId);
+                    }
+                    pending = undefined;
+                    unresolvedRunPending = false;
+                    runId = randomUUID();
+                    await ensureSessionInit(sessionId, context, request);
+                    if (!treeSynced) {
+                        phase = "tree_sync";
+                        await syncPiServerTreeWithRequest(sessionId, syncTree, request, options?.onHistoryReconciled);
+                        treeSynced = true;
+                    }
+                    baseTreeHash = hashLocalTree(sessionId, syncTree.entries);
+                    baseEntryCount = syncTree.entries.length;
+                    baseLeafId = syncTree.leafId;
+                    baseStaticContextHash = sessionStaticContextHashes.get(sessionId) ?? hashStaticContext(context);
+                    baseRevision = sessionTreeRevisions.get(sessionId) ?? 0;
+                    requestIdentity = buildRequestIdentity(baseTreeHash, baseEntryCount, baseLeafId, baseStaticContextHash, baseRevision);
+                    requestHash = hashPiServerStreamRequest(requestIdentity);
+                    const replacementPendingInput = {
+                        serverHash: hashPiServerIdentity(getServerUrl()),
+                        sessionId,
+                        runId,
+                        baseTreeHash,
+                        baseEntryCount,
+                        baseLeafId,
+                        requestHash,
+                    };
+                    pending = runStatePath
+                        ? writePiServerPendingRun(runStatePath, replacementPendingInput, runStateLease)
+                        : {
+                            version: 1,
+                            kind: "run",
+                            sequence: 0,
+                            timestamp: Date.now(),
+                            ...replacementPendingInput,
+                        };
+                    if (!runStatePath)
+                        sessionPendingRuns.set(sessionId, pending);
+                    unresolvedRunPending = true;
+                    partial.content = [];
+                    partial.stopReason = "stop";
+                    partial.errorMessage = undefined;
+                    partial.diagnostics = undefined;
+                    partial.usage = {
+                        input: 0,
+                        output: 0,
+                        cacheRead: 0,
+                        cacheWrite: 0,
+                        totalTokens: 0,
+                        cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
+                    };
+                    partial.timestamp = Date.now();
+                    receivedProxyEventCount = 0;
+                    reconnectAttempt = 0;
+                    recoveryDeadline = undefined;
+                    phase = "provider_stream";
+                    try {
+                        const replacementResponse = await postStream(0);
+                        if (replacementResponse.ok) {
+                            activeResponse = replacementResponse;
+                            activeEventCursor = 0;
+                        }
+                        else {
+                            const failure = await readPiServerFailure(replacementResponse);
+                            const retryable = isTransientPiServerResponse(replacementResponse, failure.bodyText) ||
+                                isRecoverableMissingServerState(replacementResponse, failure.matchText);
+                            if (!retryable) {
+                                await settleStreamFailure(new PiServerHttpResponseError(`pi-server replacement run failed: ${failure.details}`, replacementResponse.status, false), false);
+                                return;
+                            }
+                            interruption = new PiServerHttpResponseError(`pi-server replacement run failed: ${failure.details}`, replacementResponse.status, true);
+                            activeResponse = undefined;
+                            recoveryDeadline = Date.now() + recoveryWindowMs;
+                        }
+                    }
+                    catch (error) {
+                        if (isAbortError(error, options?.signal))
+                            throw error;
+                        if (!isRetryablePiServerInterruption(error)) {
+                            await settleStreamFailure(error, false);
+                            return;
+                        }
+                        interruption = error;
+                        activeResponse = undefined;
+                        recoveryDeadline = Date.now() + recoveryWindowMs;
+                    }
+                    continue;
                 }
                 if (recoveredRun.status === "unavailable") {
                     const failureMessage = interruption instanceof Error ? interruption.message : String(interruption);
@@ -2396,7 +2651,8 @@ export async function streamPiServer(model, context, options) {
                     const reconnectResponse = await getRunEvents(eventCursor);
                     if (!reconnectResponse.ok) {
                         const failure = await readPiServerFailure(reconnectResponse);
-                        if (!isTransientPiServerResponse(reconnectResponse, failure.bodyText)) {
+                        if (!isTransientPiServerResponse(reconnectResponse, failure.bodyText) &&
+                            !isRecoverableMissingServerState(reconnectResponse, failure.matchText)) {
                             await settleStreamFailure(new Error(`pi-server reconnect failed: ${failure.details}`), false);
                             return;
                         }