@tangle-network/agent-app 0.45.24 → 0.45.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chat-routes/index.d.ts +57 -6
- package/dist/chat-routes/index.js +225 -17
- package/dist/chat-routes/index.js.map +1 -1
- package/dist/{chunk-AWPK7ZDS.js → chunk-AGCRCOQH.js} +1 -1
- package/dist/chunk-N3G3FSM4.js +509 -0
- package/dist/chunk-N3G3FSM4.js.map +1 -0
- package/dist/chunk-WD2B6HGY.js +1046 -0
- package/dist/chunk-WD2B6HGY.js.map +1 -0
- package/dist/design-canvas-react/index.js +1 -1
- package/dist/design-canvas-react/lazy.js +1 -1
- package/dist/run-tEsZUhAf.d.ts +40 -0
- package/dist/signoff/cli.d.ts +12 -0
- package/dist/signoff/cli.js +166 -0
- package/dist/signoff/cli.js.map +1 -0
- package/dist/signoff/index.d.ts +430 -0
- package/dist/signoff/index.js +63 -0
- package/dist/signoff/index.js.map +1 -0
- package/dist/signoff/proof-cli.d.ts +1 -0
- package/dist/signoff/proof-cli.js +92 -0
- package/dist/signoff/proof-cli.js.map +1 -0
- package/dist/signoff/proof.d.ts +515 -0
- package/dist/signoff/proof.js +148 -0
- package/dist/signoff/proof.js.map +1 -0
- package/dist/types-U7Nz-txa.d.ts +233 -0
- package/package.json +16 -3
- /package/dist/{chunk-AWPK7ZDS.js.map → chunk-AGCRCOQH.js.map} +0 -0
|
@@ -6,7 +6,7 @@ import { C as ChatMessagePart, a as ChatAttachmentPart, e as ChatMentionPart } f
|
|
|
6
6
|
import { InteractionAnswerRoute, InteractionAnswerRouteOptions } from '../interactions/index.js';
|
|
7
7
|
import { PersistedChatMessageForTurn } from '../stream/index.js';
|
|
8
8
|
import { T as TurnEventStore } from '../turn-buffer-COMQUwO2.js';
|
|
9
|
-
import { M as ModelFailoverAttempt } from '../failover-D-3UXXTb.js';
|
|
9
|
+
import { M as ModelFailoverAttempt, a as ModelFailoverExhaustedError } from '../failover-D-3UXXTb.js';
|
|
10
10
|
export { D as DEFAULT_STALE_TURN_LOCK_GRACE_MS, a as DEFAULT_TERMINAL_TURN_LOCK_GRACE_MS, R as ReconcileStaleTurnLockOptions, b as ReconcileStaleTurnLockResult, S as StaleTurnLockSandboxProbeResult, c as StaleTurnLockSessionProbeResult, r as reconcileStaleTurnLock } from '../stale-turn-lock-DucQzvXu.js';
|
|
11
11
|
import { J as JsonRecord } from '../stream-normalizer-DKRxyVlZ.js';
|
|
12
12
|
import { SandboxInstance } from '@tangle-network/sandbox';
|
|
@@ -785,8 +785,8 @@ declare function createChatTurnRoutes<TContext = void>(options: CreateChatTurnRo
|
|
|
785
785
|
* platform flake that a same-model re-run recovers (8 hard cases: 7/8
|
|
786
786
|
* delivered on the first pass, 8/8 with one re-run, at a cost of 1 extra turn
|
|
787
787
|
* in 9). Default `0`, so the behavior is unchanged unless a product asks.
|
|
788
|
-
* - A chain of length 1 costs
|
|
789
|
-
*
|
|
788
|
+
* - A responsive chain of length 1 costs no extra call or latency and remains
|
|
789
|
+
* byte-identical to no failover. A silent chain is now deliberately bounded.
|
|
790
790
|
*/
|
|
791
791
|
|
|
792
792
|
/**
|
|
@@ -834,6 +834,10 @@ type OpenModelStream = (args: {
|
|
|
834
834
|
model: string;
|
|
835
835
|
/** 1 for the preferred model, 2 for the first fallback, and so on. */
|
|
836
836
|
attempt: number;
|
|
837
|
+
/** Aborted when this attempt times out, is abandoned, or finishes. Forward
|
|
838
|
+
* this to `streamSandboxPrompt(..., { signal })` so a blocked transport is
|
|
839
|
+
* cancelled immediately rather than only receiving iterator.return(). */
|
|
840
|
+
signal: AbortSignal;
|
|
837
841
|
}) => AsyncIterable<unknown> | Promise<AsyncIterable<unknown>>;
|
|
838
842
|
/** Fired when a model is abandoned and the next one is about to be tried. */
|
|
839
843
|
interface ModelFallbackInfo {
|
|
@@ -846,6 +850,19 @@ interface ModelFailoverStreamOptions {
|
|
|
846
850
|
/** Preferred model first, then fallbacks in descending preference. */
|
|
847
851
|
models: readonly string[];
|
|
848
852
|
open: OpenModelStream;
|
|
853
|
+
/**
|
|
854
|
+
* Maximum time to open/start one model's event source, including its first
|
|
855
|
+
* iterator pull. Default 120 seconds, matching the sandbox provisioning
|
|
856
|
+
* ceiling so cold infrastructure startup is bounded independently from
|
|
857
|
+
* provider inference.
|
|
858
|
+
*/
|
|
859
|
+
openTimeoutMs?: number;
|
|
860
|
+
/**
|
|
861
|
+
* Hard deadline from the source's first lifecycle event to its first
|
|
862
|
+
* answer-bearing event. Default 60 seconds. Processing/lifecycle events do
|
|
863
|
+
* not commit the attempt and do not extend this deadline.
|
|
864
|
+
*/
|
|
865
|
+
firstResponseTimeoutMs?: number;
|
|
849
866
|
/** Override the commit-point rule. Default {@link isCommittingSandboxEvent}. */
|
|
850
867
|
isCommitting?: (event: unknown) => boolean;
|
|
851
868
|
onFallback?: (info: ModelFallbackInfo) => void;
|
|
@@ -886,6 +903,24 @@ interface ModelFailoverStreamHandle {
|
|
|
886
903
|
/** True when the preferred model did not serve — the attributability signal. */
|
|
887
904
|
usedFallback(): boolean;
|
|
888
905
|
}
|
|
906
|
+
/** Default ceiling for opening/starting one source through its first event. */
|
|
907
|
+
declare const DEFAULT_MODEL_STREAM_OPEN_TIMEOUT_MS = 120000;
|
|
908
|
+
/** Default ceiling for the first answer-bearing event after the source's first event. */
|
|
909
|
+
declare const DEFAULT_MODEL_FIRST_RESPONSE_TIMEOUT_MS = 60000;
|
|
910
|
+
/** Structured timeout codes surfaced by the producer on final exhaustion. */
|
|
911
|
+
type ModelAttemptTimeoutCode = 'model_stream_open_timeout' | 'provider_first_response_timeout';
|
|
912
|
+
/** Every configured model was exhausted and the final one timed out. */
|
|
913
|
+
declare class ModelFailoverTimeoutError extends ModelFailoverExhaustedError {
|
|
914
|
+
readonly code: ModelAttemptTimeoutCode;
|
|
915
|
+
readonly model: string;
|
|
916
|
+
readonly timeoutMs: number;
|
|
917
|
+
constructor(input: {
|
|
918
|
+
attempts: ModelFailoverAttempt[];
|
|
919
|
+
code: ModelAttemptTimeoutCode;
|
|
920
|
+
model: string;
|
|
921
|
+
timeoutMs: number;
|
|
922
|
+
});
|
|
923
|
+
}
|
|
889
924
|
/**
|
|
890
925
|
* Hard ceiling on same-model re-runs. A turn that comes back blank three times
|
|
891
926
|
* running is not a flake this can retry away, and each pass costs a full
|
|
@@ -908,9 +943,11 @@ declare function resolveEmptyTurnRetries(value: number | undefined): number;
|
|
|
908
943
|
* Zero added latency on the happy path: the preferred model is opened first and,
|
|
909
944
|
* the moment it emits anything meaningful, its events flow straight through.
|
|
910
945
|
*
|
|
911
|
-
* @throws
|
|
946
|
+
* @throws ModelFailoverTimeoutError when the final model times out,
|
|
947
|
+
* ModelFailoverExhaustedError when every model's upstream is down, and
|
|
912
948
|
* re-throws a non-outage error from the FIRST model without walking the
|
|
913
|
-
* chain (
|
|
949
|
+
* chain (the latter behaviors are inherited from
|
|
950
|
+
* `runWithModelFailover`).
|
|
914
951
|
*/
|
|
915
952
|
declare function streamWithModelFailover(options: ModelFailoverStreamOptions): ModelFailoverStreamHandle;
|
|
916
953
|
|
|
@@ -993,6 +1030,14 @@ interface SandboxChatProducerOptions {
|
|
|
993
1030
|
/** Opt out of failover while still using {@link openEvents}. `false` collapses
|
|
994
1031
|
* the chain to `model` alone. */
|
|
995
1032
|
modelFailover?: false;
|
|
1033
|
+
/** Maximum time to open/start one model event source through its first event.
|
|
1034
|
+
* Default 120 seconds. This is separate from provider first-output latency
|
|
1035
|
+
* so cold sandbox startup is not mislabeled as an inference failure. */
|
|
1036
|
+
openTimeoutMs?: number;
|
|
1037
|
+
/** Hard deadline after the source's first lifecycle event for the first
|
|
1038
|
+
* answer-bearing event. Default 60 seconds. Processing/lifecycle events do
|
|
1039
|
+
* not extend it. */
|
|
1040
|
+
firstResponseTimeoutMs?: number;
|
|
996
1041
|
/** Fired when a model is abandoned mid-chain (telemetry/alerting). The user
|
|
997
1042
|
* already sees a transcript notice; this is for the operator. */
|
|
998
1043
|
onModelFallback?: (info: ModelFallbackInfo) => void;
|
|
@@ -1121,6 +1166,12 @@ interface DetachedTurnOptions {
|
|
|
1121
1166
|
fallbackModels?: SandboxChatProducerOptions['fallbackModels'];
|
|
1122
1167
|
/** Opt out of failover while still using {@link openEvents}. */
|
|
1123
1168
|
modelFailover?: false;
|
|
1169
|
+
/** Maximum time to open/start one model event source through its first event.
|
|
1170
|
+
* Forwarded to the producer; default 120 seconds. */
|
|
1171
|
+
openTimeoutMs?: SandboxChatProducerOptions['openTimeoutMs'];
|
|
1172
|
+
/** Hard deadline after the source's first lifecycle event for the first
|
|
1173
|
+
* answer-bearing event. Forwarded to the producer; default 60 seconds. */
|
|
1174
|
+
firstResponseTimeoutMs?: SandboxChatProducerOptions['firstResponseTimeoutMs'];
|
|
1124
1175
|
/** Fired when a model is abandoned mid-chain (telemetry/alerting). */
|
|
1125
1176
|
onModelFallback?: SandboxChatProducerOptions['onModelFallback'];
|
|
1126
1177
|
/** The PREFERRED model. Recorded on the persisted assistant message + usage
|
|
@@ -1787,4 +1838,4 @@ interface PromoteAgentFilePartOptions {
|
|
|
1787
1838
|
/** Promote a part of an agent file with optional byte limits and MIME type detection */
|
|
1788
1839
|
declare function promoteAgentFilePart(options: PromoteAgentFilePartOptions): Promise<PromoteFilePartResult>;
|
|
1789
1840
|
|
|
1790
|
-
export { type AssistantDraftSnapshot, type AssistantDraftStore, type AssistantDraftWriter, type AssistantDraftWriterOptions, type AssistantRowValues, type AttachmentPathArgs, type AttachmentPathCheck, type AttachmentReadResult, type AttachmentUploadAuthorization, type AttachmentWriteResult, type BuildDispatchPartsInput, ChatAttachmentKind, type ChatRouteDurableProjection, type ChatRouteDurableProjectionLogger, type ChatRouteEvent, type ChatTurnAuthorization, type ChatTurnAuthorizeArgs, type ChatTurnCompleteInput, ChatTurnFilePartInput, type ChatTurnGateResult, type ChatTurnHeartbeat, type ChatTurnInputPatch, type ChatTurnLifecycle, type ChatTurnLifecycleComplete, type ChatTurnLifecycleError, type ChatTurnLifecycleStart, type ChatTurnLock, type ChatTurnLockResult, type ChatTurnMessageStore, type ChatTurnModelAttribution, type ChatTurnModelFailover, ChatTurnPartInput, type ChatTurnProduceArgs, ChatTurnRequestPayload, type ChatTurnRouteProducer, type ChatTurnRoutes, type ChatTurnUsage, type CompletedSandboxTurnSource, type CreateAttachmentUploadRouteOptions, type CreateChatTurnRoutesOptions, type CreateUploadRouteOptions, type DetachedTurnFinal, type DetachedTurnOptions, type DetachedTurnParts, type DetachedTurnResult, type DispatchPartsOutcome, type DraftPersistenceTuning, type DraftStoredMessage, type EmptyTurnRetryInfo, type FilePartPromotionOutcome, MAX_EMPTY_TURN_RETRIES, type ModelFailoverStreamHandle, type ModelFailoverStreamOptions, type ModelFallbackInfo, type OpenModelStream, PROMOTE_MAX_FILE_BYTES, type PromoteAgentFilePartOptions, type PromoteFilePartResult, PromptInputPart, type RawAgentFilePart, type ReadAttachmentFn, type ReadCompletedSandboxTurnOptions, type ReadSandboxMentionFn, type ResolveChatAttachmentsOptions, type ResolveChatAttachmentsResult, type SandboxChatProducerOptions, type SandboxUploadSink, UPLOAD_INLINE_MAX_BYTES, UPLOAD_MAX_FILE_BYTES, type UploadAuthorization, type UploadedChatFile, type WriteAttachmentFn, assistantRowIdForTurn, buildDispatchParts, bytesToBase64, classifyTerminalFailure, createAssistantDraftWriter, createAttachmentUploadRoute, createChatTurnRoutes, createSandboxChatProducer, createUploadRoute, defaultValidateAttachmentPath, isCommittingSandboxEvent, isDraftContentEvent, normalizeChatPromptForSandbox, promoteAgentFilePart, readCompletedSandboxTurn, resolveChatAttachments, resolveEmptyTurnRetries, rowIdOf, runDetachedTurn, sanitizeUploadFilename, sniffMimeFromName, storeSupportsDraftPersistence, streamWithModelFailover, summarizeFailoverReason, withDurableChatProjection };
|
|
1841
|
+
export { type AssistantDraftSnapshot, type AssistantDraftStore, type AssistantDraftWriter, type AssistantDraftWriterOptions, type AssistantRowValues, type AttachmentPathArgs, type AttachmentPathCheck, type AttachmentReadResult, type AttachmentUploadAuthorization, type AttachmentWriteResult, type BuildDispatchPartsInput, ChatAttachmentKind, type ChatRouteDurableProjection, type ChatRouteDurableProjectionLogger, type ChatRouteEvent, type ChatTurnAuthorization, type ChatTurnAuthorizeArgs, type ChatTurnCompleteInput, ChatTurnFilePartInput, type ChatTurnGateResult, type ChatTurnHeartbeat, type ChatTurnInputPatch, type ChatTurnLifecycle, type ChatTurnLifecycleComplete, type ChatTurnLifecycleError, type ChatTurnLifecycleStart, type ChatTurnLock, type ChatTurnLockResult, type ChatTurnMessageStore, type ChatTurnModelAttribution, type ChatTurnModelFailover, ChatTurnPartInput, type ChatTurnProduceArgs, ChatTurnRequestPayload, type ChatTurnRouteProducer, type ChatTurnRoutes, type ChatTurnUsage, type CompletedSandboxTurnSource, type CreateAttachmentUploadRouteOptions, type CreateChatTurnRoutesOptions, type CreateUploadRouteOptions, DEFAULT_MODEL_FIRST_RESPONSE_TIMEOUT_MS, DEFAULT_MODEL_STREAM_OPEN_TIMEOUT_MS, type DetachedTurnFinal, type DetachedTurnOptions, type DetachedTurnParts, type DetachedTurnResult, type DispatchPartsOutcome, type DraftPersistenceTuning, type DraftStoredMessage, type EmptyTurnRetryInfo, type FilePartPromotionOutcome, MAX_EMPTY_TURN_RETRIES, type ModelAttemptTimeoutCode, type ModelFailoverStreamHandle, type ModelFailoverStreamOptions, ModelFailoverTimeoutError, type ModelFallbackInfo, type OpenModelStream, PROMOTE_MAX_FILE_BYTES, type PromoteAgentFilePartOptions, type PromoteFilePartResult, PromptInputPart, type RawAgentFilePart, type ReadAttachmentFn, type ReadCompletedSandboxTurnOptions, type ReadSandboxMentionFn, type ResolveChatAttachmentsOptions, type ResolveChatAttachmentsResult, type SandboxChatProducerOptions, type SandboxUploadSink, UPLOAD_INLINE_MAX_BYTES, UPLOAD_MAX_FILE_BYTES, type UploadAuthorization, type UploadedChatFile, type WriteAttachmentFn, assistantRowIdForTurn, buildDispatchParts, bytesToBase64, classifyTerminalFailure, createAssistantDraftWriter, createAttachmentUploadRoute, createChatTurnRoutes, createSandboxChatProducer, createUploadRoute, defaultValidateAttachmentPath, isCommittingSandboxEvent, isDraftContentEvent, normalizeChatPromptForSandbox, promoteAgentFilePart, readCompletedSandboxTurn, resolveChatAttachments, resolveEmptyTurnRetries, rowIdOf, runDetachedTurn, sanitizeUploadFilename, sniffMimeFromName, storeSupportsDraftPersistence, streamWithModelFailover, summarizeFailoverReason, withDurableChatProjection };
|
|
@@ -99,6 +99,7 @@ import {
|
|
|
99
99
|
planToPersistedPart
|
|
100
100
|
} from "../chunk-YJMCRXQQ.js";
|
|
101
101
|
import {
|
|
102
|
+
ModelFailoverExhaustedError,
|
|
102
103
|
buildModelChain,
|
|
103
104
|
isUpstreamUnavailable,
|
|
104
105
|
readHttpStatusHint,
|
|
@@ -883,7 +884,7 @@ function isCommittingSandboxEvent(event) {
|
|
|
883
884
|
}
|
|
884
885
|
return true;
|
|
885
886
|
}
|
|
886
|
-
if (type === "start" || type === "execution.started" || type === "status" || type === "model-processing" || type === "session.created" || type === "session.updated" || type === "session.idle" || type === "step-start" || type === "step-finish" || type === "turn" || type === "warning") {
|
|
887
|
+
if (type === "start" || type === "execution.started" || type === "status" || type === "model-processing" || type === "model.processing" || type === "session.created" || type === "session.updated" || type === "session.idle" || type === "step-start" || type === "step-finish" || type === "turn" || type === "warning") {
|
|
887
888
|
return false;
|
|
888
889
|
}
|
|
889
890
|
return true;
|
|
@@ -912,6 +913,44 @@ function classifyTerminalFailure(event) {
|
|
|
912
913
|
const code = asString(data?.errorCode) ?? asString(data?.code);
|
|
913
914
|
return { outage, reason, ...code ? { code } : {} };
|
|
914
915
|
}
|
|
916
|
+
var DEFAULT_MODEL_STREAM_OPEN_TIMEOUT_MS = 12e4;
|
|
917
|
+
var DEFAULT_MODEL_FIRST_RESPONSE_TIMEOUT_MS = 6e4;
|
|
918
|
+
var ModelFailoverTimeoutError = class extends ModelFailoverExhaustedError {
|
|
919
|
+
code;
|
|
920
|
+
model;
|
|
921
|
+
timeoutMs;
|
|
922
|
+
constructor(input) {
|
|
923
|
+
super(input.attempts);
|
|
924
|
+
this.name = "ModelFailoverTimeoutError";
|
|
925
|
+
this.code = input.code;
|
|
926
|
+
this.model = input.model;
|
|
927
|
+
this.timeoutMs = input.timeoutMs;
|
|
928
|
+
this.message = input.code === "model_stream_open_timeout" ? `Opening the model event source for ${input.model} exceeded ${input.timeoutMs}ms.` : `Model ${input.model} produced no first answer-bearing event within ${input.timeoutMs}ms.`;
|
|
929
|
+
}
|
|
930
|
+
};
|
|
931
|
+
function resolveTimeoutMs(value, fallback, label) {
|
|
932
|
+
if (value === void 0) return fallback;
|
|
933
|
+
if (!Number.isFinite(value) || value <= 0) {
|
|
934
|
+
throw new RangeError(`${label} must be a finite number greater than 0`);
|
|
935
|
+
}
|
|
936
|
+
return Math.max(1, Math.trunc(value));
|
|
937
|
+
}
|
|
938
|
+
var TIMED_OUT = /* @__PURE__ */ Symbol("model-attempt-timed-out");
|
|
939
|
+
function deadlineAfter(timeoutMs) {
|
|
940
|
+
let timer;
|
|
941
|
+
const promise = new Promise((resolve) => {
|
|
942
|
+
timer = setTimeout(() => resolve(TIMED_OUT), timeoutMs);
|
|
943
|
+
});
|
|
944
|
+
return {
|
|
945
|
+
promise,
|
|
946
|
+
clear: () => {
|
|
947
|
+
if (timer !== void 0) {
|
|
948
|
+
clearTimeout(timer);
|
|
949
|
+
timer = void 0;
|
|
950
|
+
}
|
|
951
|
+
}
|
|
952
|
+
};
|
|
953
|
+
}
|
|
915
954
|
function isEmptyTerminalReceipt(event) {
|
|
916
955
|
const record = asRecord(event);
|
|
917
956
|
if (!record) return false;
|
|
@@ -940,24 +979,128 @@ async function closeIterator(iterator, log) {
|
|
|
940
979
|
function streamWithModelFailover(options) {
|
|
941
980
|
const committing = options.isCommitting ?? isCommittingSandboxEvent;
|
|
942
981
|
const emptyTurnRetries = resolveEmptyTurnRetries(options.emptyTurnRetries);
|
|
982
|
+
const openTimeoutMs = resolveTimeoutMs(
|
|
983
|
+
options.openTimeoutMs,
|
|
984
|
+
DEFAULT_MODEL_STREAM_OPEN_TIMEOUT_MS,
|
|
985
|
+
"openTimeoutMs"
|
|
986
|
+
);
|
|
987
|
+
const firstResponseTimeoutMs = resolveTimeoutMs(
|
|
988
|
+
options.firstResponseTimeoutMs,
|
|
989
|
+
DEFAULT_MODEL_FIRST_RESPONSE_TIMEOUT_MS,
|
|
990
|
+
"firstResponseTimeoutMs"
|
|
991
|
+
);
|
|
943
992
|
let serving;
|
|
944
993
|
let trail = [];
|
|
945
994
|
let fellBack = false;
|
|
946
995
|
let attemptIndex = 0;
|
|
996
|
+
let finalTimeout;
|
|
947
997
|
const drainOnce = async (model) => {
|
|
948
998
|
attemptIndex += 1;
|
|
949
|
-
const
|
|
950
|
-
const
|
|
999
|
+
const controller = new AbortController();
|
|
1000
|
+
const abort = () => {
|
|
1001
|
+
if (!controller.signal.aborted) controller.abort();
|
|
1002
|
+
};
|
|
1003
|
+
const opening = Promise.resolve().then(() => options.open({
|
|
1004
|
+
model,
|
|
1005
|
+
attempt: attemptIndex,
|
|
1006
|
+
signal: controller.signal
|
|
1007
|
+
}));
|
|
1008
|
+
const openDeadline = deadlineAfter(openTimeoutMs);
|
|
1009
|
+
let opened;
|
|
1010
|
+
try {
|
|
1011
|
+
opened = await Promise.race([opening, openDeadline.promise]);
|
|
1012
|
+
} catch (err) {
|
|
1013
|
+
openDeadline.clear();
|
|
1014
|
+
abort();
|
|
1015
|
+
throw err;
|
|
1016
|
+
}
|
|
1017
|
+
if (opened === TIMED_OUT) {
|
|
1018
|
+
openDeadline.clear();
|
|
1019
|
+
abort();
|
|
1020
|
+
void opening.then(
|
|
1021
|
+
(lateSource) => {
|
|
1022
|
+
try {
|
|
1023
|
+
void closeIterator(lateSource[Symbol.asyncIterator](), options.log);
|
|
1024
|
+
} catch (err) {
|
|
1025
|
+
options.log?.("[chat-routes] closing a late model stream threw", {
|
|
1026
|
+
error: err instanceof Error ? err.message : String(err)
|
|
1027
|
+
});
|
|
1028
|
+
}
|
|
1029
|
+
},
|
|
1030
|
+
(err) => {
|
|
1031
|
+
options.log?.("[chat-routes] abandoned model stream open rejected after timeout", {
|
|
1032
|
+
error: err instanceof Error ? err.message : String(err)
|
|
1033
|
+
});
|
|
1034
|
+
}
|
|
1035
|
+
);
|
|
1036
|
+
return {
|
|
1037
|
+
outcome: {
|
|
1038
|
+
committed: false,
|
|
1039
|
+
error: `Opening the model event source for ${model} exceeded ${openTimeoutMs}ms`,
|
|
1040
|
+
errorCode: "model_stream_open_timeout",
|
|
1041
|
+
timeoutCode: "model_stream_open_timeout"
|
|
1042
|
+
},
|
|
1043
|
+
empty: false
|
|
1044
|
+
};
|
|
1045
|
+
}
|
|
1046
|
+
const source = opened;
|
|
1047
|
+
let iterator;
|
|
1048
|
+
try {
|
|
1049
|
+
iterator = source[Symbol.asyncIterator]();
|
|
1050
|
+
} catch (err) {
|
|
1051
|
+
openDeadline.clear();
|
|
1052
|
+
abort();
|
|
1053
|
+
throw err;
|
|
1054
|
+
}
|
|
951
1055
|
const buffered = [];
|
|
1056
|
+
let next;
|
|
1057
|
+
try {
|
|
1058
|
+
next = await Promise.race([
|
|
1059
|
+
Promise.resolve(iterator.next()),
|
|
1060
|
+
openDeadline.promise
|
|
1061
|
+
]);
|
|
1062
|
+
} catch (err) {
|
|
1063
|
+
openDeadline.clear();
|
|
1064
|
+
abort();
|
|
1065
|
+
void closeIterator(iterator, options.log);
|
|
1066
|
+
throw err;
|
|
1067
|
+
}
|
|
1068
|
+
openDeadline.clear();
|
|
1069
|
+
if (next === TIMED_OUT) {
|
|
1070
|
+
abort();
|
|
1071
|
+
void closeIterator(iterator, options.log);
|
|
1072
|
+
return {
|
|
1073
|
+
outcome: {
|
|
1074
|
+
committed: false,
|
|
1075
|
+
error: `Starting the model event source for ${model} exceeded ${openTimeoutMs}ms`,
|
|
1076
|
+
errorCode: "model_stream_open_timeout",
|
|
1077
|
+
timeoutCode: "model_stream_open_timeout"
|
|
1078
|
+
},
|
|
1079
|
+
empty: false
|
|
1080
|
+
};
|
|
1081
|
+
}
|
|
1082
|
+
if (next.done) {
|
|
1083
|
+
abort();
|
|
1084
|
+
return { outcome: { committed: true, buffered, iterator: null, abort }, empty: true };
|
|
1085
|
+
}
|
|
1086
|
+
const firstResponseDeadline = deadlineAfter(firstResponseTimeoutMs);
|
|
952
1087
|
for (; ; ) {
|
|
953
|
-
const next = await iterator.next();
|
|
954
|
-
if (next.done) {
|
|
955
|
-
return { outcome: { committed: true, buffered, iterator: null }, empty: true };
|
|
956
|
-
}
|
|
957
1088
|
const event = next.value;
|
|
958
|
-
|
|
1089
|
+
let failure;
|
|
1090
|
+
let commits = false;
|
|
1091
|
+
try {
|
|
1092
|
+
failure = classifyTerminalFailure(event);
|
|
1093
|
+
commits = !failure && committing(event);
|
|
1094
|
+
} catch (err) {
|
|
1095
|
+
firstResponseDeadline.clear();
|
|
1096
|
+
abort();
|
|
1097
|
+
void closeIterator(iterator, options.log);
|
|
1098
|
+
throw err;
|
|
1099
|
+
}
|
|
959
1100
|
if (failure?.outage) {
|
|
960
|
-
|
|
1101
|
+
firstResponseDeadline.clear();
|
|
1102
|
+
abort();
|
|
1103
|
+
void closeIterator(iterator, options.log);
|
|
961
1104
|
return {
|
|
962
1105
|
outcome: {
|
|
963
1106
|
committed: false,
|
|
@@ -968,18 +1111,65 @@ function streamWithModelFailover(options) {
|
|
|
968
1111
|
};
|
|
969
1112
|
}
|
|
970
1113
|
buffered.push(event);
|
|
971
|
-
if (failure)
|
|
972
|
-
|
|
973
|
-
return { outcome: { committed: true, buffered, iterator }, empty:
|
|
1114
|
+
if (failure) {
|
|
1115
|
+
firstResponseDeadline.clear();
|
|
1116
|
+
return { outcome: { committed: true, buffered, iterator, abort }, empty: false };
|
|
1117
|
+
}
|
|
1118
|
+
if (commits) {
|
|
1119
|
+
firstResponseDeadline.clear();
|
|
1120
|
+
return { outcome: { committed: true, buffered, iterator, abort }, empty: isEmptyTerminalReceipt(event) };
|
|
1121
|
+
}
|
|
1122
|
+
try {
|
|
1123
|
+
next = await Promise.race([
|
|
1124
|
+
Promise.resolve(iterator.next()),
|
|
1125
|
+
firstResponseDeadline.promise
|
|
1126
|
+
]);
|
|
1127
|
+
} catch (err) {
|
|
1128
|
+
firstResponseDeadline.clear();
|
|
1129
|
+
abort();
|
|
1130
|
+
void closeIterator(iterator, options.log);
|
|
1131
|
+
throw err;
|
|
1132
|
+
}
|
|
1133
|
+
if (next === TIMED_OUT) {
|
|
1134
|
+
firstResponseDeadline.clear();
|
|
1135
|
+
abort();
|
|
1136
|
+
void closeIterator(iterator, options.log);
|
|
1137
|
+
return {
|
|
1138
|
+
outcome: {
|
|
1139
|
+
committed: false,
|
|
1140
|
+
error: `Model ${model} produced no first answer-bearing event within ${firstResponseTimeoutMs}ms`,
|
|
1141
|
+
errorCode: "provider_first_response_timeout",
|
|
1142
|
+
timeoutCode: "provider_first_response_timeout"
|
|
1143
|
+
},
|
|
1144
|
+
empty: false
|
|
1145
|
+
};
|
|
1146
|
+
}
|
|
1147
|
+
if (next.done) {
|
|
1148
|
+
firstResponseDeadline.clear();
|
|
1149
|
+
abort();
|
|
1150
|
+
return { outcome: { committed: true, buffered, iterator: null, abort }, empty: true };
|
|
974
1151
|
}
|
|
975
1152
|
}
|
|
976
1153
|
};
|
|
977
1154
|
const probe = async (model) => {
|
|
1155
|
+
finalTimeout = void 0;
|
|
978
1156
|
for (let retry = 0; ; retry += 1) {
|
|
979
1157
|
const pass = await drainOnce(model);
|
|
980
|
-
if (!pass.empty || retry >= emptyTurnRetries)
|
|
1158
|
+
if (!pass.empty || retry >= emptyTurnRetries) {
|
|
1159
|
+
if (!pass.outcome.committed && pass.outcome.timeoutCode) {
|
|
1160
|
+
finalTimeout = {
|
|
1161
|
+
code: pass.outcome.timeoutCode,
|
|
1162
|
+
model,
|
|
1163
|
+
timeoutMs: pass.outcome.timeoutCode === "model_stream_open_timeout" ? openTimeoutMs : firstResponseTimeoutMs
|
|
1164
|
+
};
|
|
1165
|
+
} else {
|
|
1166
|
+
finalTimeout = void 0;
|
|
1167
|
+
}
|
|
1168
|
+
return pass.outcome;
|
|
1169
|
+
}
|
|
981
1170
|
const iterator = pass.outcome.committed ? pass.outcome.iterator : null;
|
|
982
|
-
if (
|
|
1171
|
+
if (pass.outcome.committed) pass.outcome.abort();
|
|
1172
|
+
if (iterator) void closeIterator(iterator, options.log);
|
|
983
1173
|
const info = { model, retry: retry + 1, remaining: emptyTurnRetries - retry - 1 };
|
|
984
1174
|
options.log?.("[chat-routes] turn completed with no assistant text; re-running the same model", {
|
|
985
1175
|
...info
|
|
@@ -1014,11 +1204,20 @@ function streamWithModelFailover(options) {
|
|
|
1014
1204
|
} catch (err) {
|
|
1015
1205
|
const attempts = err?.attempts;
|
|
1016
1206
|
if (Array.isArray(attempts)) trail = attempts;
|
|
1207
|
+
if (err instanceof ModelFailoverExhaustedError && finalTimeout) {
|
|
1208
|
+
throw new ModelFailoverTimeoutError({
|
|
1209
|
+
attempts: trail,
|
|
1210
|
+
...finalTimeout
|
|
1211
|
+
});
|
|
1212
|
+
}
|
|
1017
1213
|
throw err;
|
|
1018
1214
|
}
|
|
1019
1215
|
for (const event of handle.buffered) yield event;
|
|
1020
1216
|
const live = handle.iterator;
|
|
1021
|
-
if (!live)
|
|
1217
|
+
if (!live) {
|
|
1218
|
+
handle.abort();
|
|
1219
|
+
return;
|
|
1220
|
+
}
|
|
1022
1221
|
try {
|
|
1023
1222
|
for (; ; ) {
|
|
1024
1223
|
const next = await live.next();
|
|
@@ -1026,6 +1225,7 @@ function streamWithModelFailover(options) {
|
|
|
1026
1225
|
yield next.value;
|
|
1027
1226
|
}
|
|
1028
1227
|
} finally {
|
|
1228
|
+
handle.abort();
|
|
1029
1229
|
await closeIterator(live, options.log);
|
|
1030
1230
|
}
|
|
1031
1231
|
})();
|
|
@@ -1154,7 +1354,8 @@ function sandboxStreamFailureDiagnostic(error) {
|
|
|
1154
1354
|
"Please retry. If it repeats, send these support details to the team."
|
|
1155
1355
|
].join("\n\n");
|
|
1156
1356
|
const failureNote = diagnosticText ? `sandbox-stream: ${message}; ${diagnosticText}` : `sandbox-stream: ${message}`;
|
|
1157
|
-
|
|
1357
|
+
const code = error instanceof ModelFailoverTimeoutError ? error.code : "sandbox.stream_failed";
|
|
1358
|
+
return { userMessage, failureNote, code };
|
|
1158
1359
|
}
|
|
1159
1360
|
function createSandboxChatProducer(options) {
|
|
1160
1361
|
const log = options.log ?? ((message, meta) => console.error(message, meta ?? ""));
|
|
@@ -1182,6 +1383,8 @@ function createSandboxChatProducer(options) {
|
|
|
1182
1383
|
models: chain,
|
|
1183
1384
|
open: options.openEvents,
|
|
1184
1385
|
log,
|
|
1386
|
+
...options.openTimeoutMs !== void 0 ? { openTimeoutMs: options.openTimeoutMs } : {},
|
|
1387
|
+
...options.firstResponseTimeoutMs !== void 0 ? { firstResponseTimeoutMs: options.firstResponseTimeoutMs } : {},
|
|
1185
1388
|
...options.emptyTurnRetries !== void 0 ? { emptyTurnRetries: options.emptyTurnRetries } : {},
|
|
1186
1389
|
...options.onEmptyTurnRetry ? { onEmptyTurnRetry: options.onEmptyTurnRetry } : {},
|
|
1187
1390
|
onFallback: (info) => {
|
|
@@ -1519,7 +1722,7 @@ ${diagnostic.userMessage}` : diagnostic.userMessage;
|
|
|
1519
1722
|
type: "error",
|
|
1520
1723
|
data: {
|
|
1521
1724
|
message: diagnostic.userMessage,
|
|
1522
|
-
code:
|
|
1725
|
+
code: diagnostic.code,
|
|
1523
1726
|
details: { failureNote: diagnostic.failureNote }
|
|
1524
1727
|
}
|
|
1525
1728
|
};
|
|
@@ -1712,6 +1915,8 @@ async function runDetachedTurn(opts) {
|
|
|
1712
1915
|
model: opts.model,
|
|
1713
1916
|
...opts.fallbackModels ? { fallbackModels: opts.fallbackModels } : {},
|
|
1714
1917
|
...opts.modelFailover === false ? { modelFailover: false } : {},
|
|
1918
|
+
...opts.openTimeoutMs !== void 0 ? { openTimeoutMs: opts.openTimeoutMs } : {},
|
|
1919
|
+
...opts.firstResponseTimeoutMs !== void 0 ? { firstResponseTimeoutMs: opts.firstResponseTimeoutMs } : {},
|
|
1715
1920
|
...opts.onModelFallback ? { onModelFallback: opts.onModelFallback } : {},
|
|
1716
1921
|
isRenderableInteraction: opts.isRenderableInteraction,
|
|
1717
1922
|
declineInteraction: opts.declineInteraction,
|
|
@@ -2670,6 +2875,8 @@ export {
|
|
|
2670
2875
|
ATTACHMENT_ACCEPT,
|
|
2671
2876
|
ATTACHMENT_MAX_COUNT,
|
|
2672
2877
|
ChatTurnInputError,
|
|
2878
|
+
DEFAULT_MODEL_FIRST_RESPONSE_TIMEOUT_MS,
|
|
2879
|
+
DEFAULT_MODEL_STREAM_OPEN_TIMEOUT_MS,
|
|
2673
2880
|
DEFAULT_STALE_TURN_LOCK_GRACE_MS,
|
|
2674
2881
|
DEFAULT_TERMINAL_TURN_LOCK_GRACE_MS,
|
|
2675
2882
|
DISPATCH_MAX_MEDIA_PARTS,
|
|
@@ -2683,6 +2890,7 @@ export {
|
|
|
2683
2890
|
MAX_EMPTY_TURN_RETRIES,
|
|
2684
2891
|
MAX_TEXT_ATTACHMENT_BYTES,
|
|
2685
2892
|
MENTION_MAX_COUNT,
|
|
2893
|
+
ModelFailoverTimeoutError,
|
|
2686
2894
|
OOXML_PRESENTATION_MACRO_ENABLED_MIME,
|
|
2687
2895
|
OOXML_PRESENTATION_MIME,
|
|
2688
2896
|
OOXML_SNIFFED_MIMES,
|