@ykurilov/agent-chat-core 0.57.0 → 0.58.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/dist/{chunk-BS2HDKZB.js → chunk-KAG3QTZC.js} +165 -24
- package/dist/chunk-KAG3QTZC.js.map +1 -0
- package/dist/index.js +49 -1
- package/dist/index.js.map +1 -1
- package/dist/storage-contract.js +1 -1
- package/dist/types/agent-run/next-attempt.d.ts +12 -1
- package/dist/types/agent-run/run-agent-turn.d.ts +6 -0
- package/dist/types/chat-service.d.ts +19 -0
- package/dist/types/helpers/run-helper.d.ts +3 -0
- package/dist/types/index.d.ts +3 -1
- package/dist/types/provider-errors/classify-provider-error.d.ts +5 -0
- package/dist/types/provider-errors/error-signs.d.ts +2 -2
- package/dist/types/provider-errors/provider-outages.d.ts +41 -0
- package/dist/types/storage/postgres-migrations.d.ts +28 -0
- package/package.json +1 -1
- package/dist/chunk-BS2HDKZB.js.map +0 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,24 @@
|
|
|
1
1
|
# @ykurilov/agent-chat-core
|
|
2
2
|
|
|
3
|
+
## 0.58.0
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- 332d90e: Новый параметр службы `abandonedBursts: { serviceFor }`: при нескольких службах на одном хранилище обход брошенных пачек
|
|
8
|
+
(`answerAfterRestart`) отдаёт пачку службе, которую назвало приложение, — её инструкция и модель отвечают. `serviceFor`
|
|
9
|
+
может создать службу по требованию; вернула `null` или упала — пачка ждёт следующего обхода, чужая служба её не берёт. Тот же параметр — у `createChatServer`.
|
|
10
|
+
- 2ed600d: Пустой счёт (402) или непринятый ключ (401, 403 со словами о ключе или деньгах) у провайдера — без повторов сразу следующий из запасных (`providerFallback`), в любом шаге хода и у помощников. Отказавший провайдер 5 минут на паузе: новые ходы службы сразу идут к запасному, после паузы его пробует один ход. Прочие 403 (регион, доступ к модели) — запасной без паузы, 403 модерации — ошибка без перехода. 429 со словами о квоте — лимит запросов, а не пустой счёт (кроме `insufficient_quota`). Основной отказал в этом ходе, а запасные не ответили — человеку его причина, а если запасной лишь перегружен — перегрузка. Строки журнала `[запасной провайдер] … причина=insufficient_funds|invalid_key` и громкая `[ВНИМАНИЕ провайдер]` не чаще раза в 15 минут на провайдер.
|
|
11
|
+
- 760962e: Хранилища пакетов на PostgreSQL можно открывать одновременно на совсем пустой базе. Раньше пакеты разом создавали
|
|
12
|
+
схему `agent_chat`, проигравший падал на `CREATE SCHEMA IF NOT EXISTS`, и его хранилище не работало до перезапуска.
|
|
13
|
+
Теперь схема создаётся под общей блокировкой всех пакетов, она держится только на создании схемы. Миграции идут под
|
|
14
|
+
своей блокировкой пакета: разные пакеты друг друга не ждут, взаимной блокировки нет. Столкновение «такое уже есть»
|
|
15
|
+
повторяется само. Новое в core: `migratePostgresSchema({ connect, lockKey, migrate })` — для своих пакетов на той же
|
|
16
|
+
схеме.
|
|
17
|
+
|
|
18
|
+
### Как обновить приложение
|
|
19
|
+
|
|
20
|
+
Ничего делать не нужно.
|
|
21
|
+
|
|
3
22
|
## 0.57.0
|
|
4
23
|
|
|
5
24
|
### Minor Changes
|
|
@@ -2484,10 +2484,9 @@ var NOT_RETRYABLE_SIGNS = signs([
|
|
|
2484
2484
|
"available balance",
|
|
2485
2485
|
"insufficient_quota",
|
|
2486
2486
|
"out of budget",
|
|
2487
|
-
"quota exceeded",
|
|
2488
|
-
"billing",
|
|
2489
2487
|
"subscription_sharing_usage_limit_exceeded"
|
|
2490
2488
|
]);
|
|
2489
|
+
var QUOTA_WORDS = signs(["quota exceeded", "billing"]);
|
|
2491
2490
|
var RETRYABLE_SIGNS = signs([
|
|
2492
2491
|
"overloaded",
|
|
2493
2492
|
"currently experiencing high demand",
|
|
@@ -2534,8 +2533,8 @@ function isContextOverflow(text, statusCode) {
|
|
|
2534
2533
|
if (NOT_OVERFLOW_SIGNS.some((sign) => sign.test(text))) return false;
|
|
2535
2534
|
return OVERFLOW_SIGNS.some((sign) => sign.test(text));
|
|
2536
2535
|
}
|
|
2537
|
-
function isNotRetryable(text) {
|
|
2538
|
-
return NOT_RETRYABLE_SIGNS.test(text);
|
|
2536
|
+
function isNotRetryable(text, statusCode) {
|
|
2537
|
+
return NOT_RETRYABLE_SIGNS.test(text) || statusCode !== 429 && QUOTA_WORDS.test(text);
|
|
2539
2538
|
}
|
|
2540
2539
|
function looksTransient(text) {
|
|
2541
2540
|
return RETRYABLE_SIGNS.test(text);
|
|
@@ -2573,7 +2572,7 @@ function isTransient(error) {
|
|
|
2573
2572
|
if (verdict === "false") return false;
|
|
2574
2573
|
const status = errorStatusCode(error);
|
|
2575
2574
|
const text = errorText(error);
|
|
2576
|
-
if (isNotRetryable(text) || isContextOverflow(text, status)) return false;
|
|
2575
|
+
if (isNotRetryable(text, status) || isContextOverflow(text, status)) return false;
|
|
2577
2576
|
if (status !== void 0) return RETRYABLE_STATUS_CODES.has(status) || status >= 500;
|
|
2578
2577
|
if (APICallError2.isInstance(error) && error.isRetryable) return true;
|
|
2579
2578
|
return looksTransient(text);
|
|
@@ -2747,6 +2746,9 @@ function sleep(ms, signal) {
|
|
|
2747
2746
|
}
|
|
2748
2747
|
|
|
2749
2748
|
// src/provider-errors/classify-provider-error.ts
|
|
2749
|
+
var KEY_SIGNS = /api[ _-]?key|authenticat|unauthori[sz]ed|credential|access[ _-]?token|key (?:is )?(?:invalid|revoked|expired|disabled)/i;
|
|
2750
|
+
var FUNDS_SIGNS = /key limit exceeded|insufficient (?:credits|funds|balance)/i;
|
|
2751
|
+
var MODERATION_SIGNS = /flagged|moderation|content[ _-]?policy|safety/i;
|
|
2750
2752
|
var RATE_LIMIT_SIGNS = /rate.?limit|too many requests|throttl|please wait|ResourceExhausted/i;
|
|
2751
2753
|
function classifyProviderError(error, provider) {
|
|
2752
2754
|
if (error instanceof MissingProviderKeyError) return "invalid_key";
|
|
@@ -2756,8 +2758,12 @@ function classifyProviderError(error, provider) {
|
|
|
2756
2758
|
for (const hint of provider?.errorHints ?? []) {
|
|
2757
2759
|
if (hint.match.test(text)) return hint.kind;
|
|
2758
2760
|
}
|
|
2759
|
-
if (status === 401
|
|
2760
|
-
if (status ===
|
|
2761
|
+
if (status === 401) return "invalid_key";
|
|
2762
|
+
if (status === 403) {
|
|
2763
|
+
if (KEY_SIGNS.test(text)) return "invalid_key";
|
|
2764
|
+
return FUNDS_SIGNS.test(text) || isNotRetryable(text) ? "insufficient_funds" : "other";
|
|
2765
|
+
}
|
|
2766
|
+
if (status === 402 || isNotRetryable(text, status)) return "insufficient_funds";
|
|
2761
2767
|
if (isContextOverflow(text, status)) return "context_overflow";
|
|
2762
2768
|
if (status === 429) return "rate_limited";
|
|
2763
2769
|
if (isProviderSilence(cause)) return "overloaded";
|
|
@@ -2765,6 +2771,10 @@ function classifyProviderError(error, provider) {
|
|
|
2765
2771
|
if (RATE_LIMIT_SIGNS.test(text)) return "rate_limited";
|
|
2766
2772
|
return looksTransient(text) ? "overloaded" : "other";
|
|
2767
2773
|
}
|
|
2774
|
+
function isAccessRefusal(error) {
|
|
2775
|
+
const cause = underlyingError(error);
|
|
2776
|
+
return errorStatusCode(cause) === 403 && !MODERATION_SIGNS.test(errorText(cause));
|
|
2777
|
+
}
|
|
2768
2778
|
|
|
2769
2779
|
// src/provider-errors/provider-failure-log.ts
|
|
2770
2780
|
import { APICallError as APICallError3 } from "ai";
|
|
@@ -2790,6 +2800,49 @@ function oneLine(text) {
|
|
|
2790
2800
|
return text.replace(/\s*[\r\n]+\s*/g, " ");
|
|
2791
2801
|
}
|
|
2792
2802
|
|
|
2803
|
+
// src/provider-errors/provider-outages.ts
|
|
2804
|
+
function isProviderOutage(kind) {
|
|
2805
|
+
return kind === "insufficient_funds" || kind === "invalid_key";
|
|
2806
|
+
}
|
|
2807
|
+
var PROVIDER_OUTAGE_PAUSE_MS = 5 * 6e4;
|
|
2808
|
+
var PROVIDER_OUTAGE_ALERT_EVERY_MS = 15 * 6e4;
|
|
2809
|
+
var PROVIDER_OUTAGE_PROBE_MS = 6e4;
|
|
2810
|
+
var SAID = {
|
|
2811
|
+
insufficient_funds: "\u043A\u043E\u043D\u0447\u0438\u043B\u0438\u0441\u044C \u0434\u0435\u043D\u044C\u0433\u0438",
|
|
2812
|
+
invalid_key: "\u043A\u043B\u044E\u0447 \u043D\u0435 \u043F\u0440\u0438\u043D\u044F\u0442"
|
|
2813
|
+
};
|
|
2814
|
+
function createProviderOutages(options2) {
|
|
2815
|
+
const outages = /* @__PURE__ */ new Map();
|
|
2816
|
+
const alertedAt = /* @__PURE__ */ new Map();
|
|
2817
|
+
const pausedKind = (providerId) => {
|
|
2818
|
+
const outage = outages.get(providerId);
|
|
2819
|
+
if (!outage) return void 0;
|
|
2820
|
+
const time = options2.now();
|
|
2821
|
+
return time < outage.pausedUntil || time < (outage.probeUntil ?? 0) ? outage.kind : void 0;
|
|
2822
|
+
};
|
|
2823
|
+
return {
|
|
2824
|
+
report({ providerId, kind, fallbackProviderId, fallbacksPaused }) {
|
|
2825
|
+
const time = options2.now();
|
|
2826
|
+
outages.set(providerId, { kind, pausedUntil: time + PROVIDER_OUTAGE_PAUSE_MS });
|
|
2827
|
+
const last = alertedAt.get(providerId);
|
|
2828
|
+
if (last !== void 0 && time - last < PROVIDER_OUTAGE_ALERT_EVERY_MS) return;
|
|
2829
|
+
alertedAt.set(providerId, time);
|
|
2830
|
+
const route = fallbackProviderId !== void 0 ? `\u0445\u043E\u0434\u044B \u0438\u0434\u0443\u0442 \u0447\u0435\u0440\u0435\u0437 \u0437\u0430\u043F\u0430\u0441\u043D\u043E\u0439 ${fallbackProviderId}` : `${fallbacksPaused ? "\u0437\u0430\u043F\u0430\u0441\u043D\u044B\u0435 \u0442\u043E\u0436\u0435 \u043D\u0430 \u043F\u0430\u0443\u0437\u0435" : "\u0437\u0430\u043F\u0430\u0441\u043D\u043E\u0433\u043E \u043D\u0435\u0442"}, \u0445\u043E\u0434\u044B \u043F\u0430\u0434\u0430\u044E\u0442 \u0441 \u043E\u0448\u0438\u0431\u043A\u043E\u0439`;
|
|
2831
|
+
options2.logError(`[\u0412\u041D\u0418\u041C\u0410\u041D\u0418\u0415 \u043F\u0440\u043E\u0432\u0430\u0439\u0434\u0435\u0440] ${providerId}: ${SAID[kind]} \u2014 ${route}`);
|
|
2832
|
+
},
|
|
2833
|
+
pausedKind,
|
|
2834
|
+
admit(providerId) {
|
|
2835
|
+
const kind = pausedKind(providerId);
|
|
2836
|
+
const outage = outages.get(providerId);
|
|
2837
|
+
if (kind === void 0 && outage) outage.probeUntil = options2.now() + PROVIDER_OUTAGE_PROBE_MS;
|
|
2838
|
+
return kind;
|
|
2839
|
+
},
|
|
2840
|
+
recovered(providerId) {
|
|
2841
|
+
outages.delete(providerId);
|
|
2842
|
+
}
|
|
2843
|
+
};
|
|
2844
|
+
}
|
|
2845
|
+
|
|
2793
2846
|
// src/tool-approval/tool-approval-policy.ts
|
|
2794
2847
|
var needsApprovalOf = (tool4) => tool4.needsApproval;
|
|
2795
2848
|
function prepareToolApproval(tools, alwaysAllowed) {
|
|
@@ -4348,8 +4401,14 @@ function decideNextAttempt(result, state) {
|
|
|
4348
4401
|
if (state.overflowCompaction?.answerSteps) return { action: "compact-and-continue", scope: "answer-steps" };
|
|
4349
4402
|
return { action: "fail" };
|
|
4350
4403
|
}
|
|
4351
|
-
if (
|
|
4404
|
+
if (isProviderOutage(result.kind)) {
|
|
4405
|
+
const next2 = state.fallbackProviderIds[0];
|
|
4406
|
+
return next2 === void 0 ? { action: "fail" } : { action: "fallback", providerId: next2 };
|
|
4407
|
+
}
|
|
4408
|
+
const accessRefused = result.kind === "other" && isAccessRefusal(result.error);
|
|
4409
|
+
if (!TRANSIENT_KINDS.has(result.kind) && !accessRefused) return { action: "fail" };
|
|
4352
4410
|
if (result.contentSeen) {
|
|
4411
|
+
if (accessRefused) return { action: "fail" };
|
|
4353
4412
|
const auto = state.autoContinue;
|
|
4354
4413
|
if (!auto || auto.approvalRequested || auto.done >= AUTO_CONTINUES_PER_TURN) return { action: "fail" };
|
|
4355
4414
|
return { action: "auto-continue", pauseMs: autoContinuePauseMs({ index: auto.done, error: result.error, random: auto.random ?? Math.random }) };
|
|
@@ -4358,6 +4417,10 @@ function decideNextAttempt(result, state) {
|
|
|
4358
4417
|
if (state.fallbackUsed || next === void 0) return { action: "fail" };
|
|
4359
4418
|
return { action: "fallback", providerId: next };
|
|
4360
4419
|
}
|
|
4420
|
+
function errorKindForPerson(lastKind, firstOutage) {
|
|
4421
|
+
if (firstOutage === void 0 || TRANSIENT_KINDS.has(lastKind) || lastKind === "context_overflow") return lastKind;
|
|
4422
|
+
return firstOutage;
|
|
4423
|
+
}
|
|
4361
4424
|
function checkProviderFallback(setting, catalog) {
|
|
4362
4425
|
if (setting === void 0 || setting === false) return;
|
|
4363
4426
|
if (typeof setting !== "object" || setting === null || Array.isArray(setting)) {
|
|
@@ -5995,13 +6058,21 @@ async function runTurn(turnInput, emitRaw) {
|
|
|
5995
6058
|
const seen = latestFrozenContextOf(resume.message.parts);
|
|
5996
6059
|
if (seen) resumeNote = { ...resumeNote ?? { blocks: seen.blocks, sent: { changed: [], removed: [] }, base: messageId }, notices: turnNotices };
|
|
5997
6060
|
}
|
|
5998
|
-
const
|
|
6061
|
+
const outages = input.providerOutages ?? createProviderOutages({ now: () => (input.now ?? (() => /* @__PURE__ */ new Date()))().getTime(), logError });
|
|
6062
|
+
const allFallbackIds = fallbackProviders({
|
|
5999
6063
|
catalog: input.catalog,
|
|
6000
6064
|
modelId,
|
|
6001
6065
|
chosenProviderId: input.providerId,
|
|
6002
6066
|
env: input.env,
|
|
6003
6067
|
...input.providerFallback !== void 0 && { setting: input.providerFallback }
|
|
6004
6068
|
});
|
|
6069
|
+
const fallbackProviderIds = allFallbackIds.filter((candidate) => outages.pausedKind(candidate) === void 0);
|
|
6070
|
+
let firstOutage;
|
|
6071
|
+
const primaryPaused = outages.admit(input.providerId);
|
|
6072
|
+
if (primaryPaused !== void 0 && fallbackProviderIds[0] !== void 0) {
|
|
6073
|
+
providerId = fallbackProviderIds[0];
|
|
6074
|
+
logError(`[\u0437\u0430\u043F\u0430\u0441\u043D\u043E\u0439 \u043F\u0440\u043E\u0432\u0430\u0439\u0434\u0435\u0440] \u0447\u0430\u0442=${input.chatId} \u043C\u043E\u0434\u0435\u043B\u044C=${modelId} \u0441=${input.providerId} \u043D\u0430=${providerId} \u043F\u0440\u0438\u0447\u0438\u043D\u0430=${primaryPaused} \u043E\u0441\u043D\u043E\u0432\u043D\u043E\u0439 \u043D\u0430 \u043F\u0430\u0443\u0437\u0435`);
|
|
6075
|
+
}
|
|
6005
6076
|
emit({ type: "start", messageId, messageMetadata: { modelId, providerId: input.providerId } });
|
|
6006
6077
|
if (input.liveDemos && chatHasExternalContent(await storage.listMessages(input.chatId)) && externalContent.announce()) emitRaw(externalContentChunk());
|
|
6007
6078
|
const silenceSetting = resolveSilenceSwitch(input.providerFallback);
|
|
@@ -6092,13 +6163,14 @@ async function runTurn(turnInput, emitRaw) {
|
|
|
6092
6163
|
} catch (error2) {
|
|
6093
6164
|
if (error2 instanceof MissingProviderKeyError) logError(error2.message);
|
|
6094
6165
|
if (!input.abortSignal.aborted) {
|
|
6095
|
-
errorKind = classify(error2);
|
|
6166
|
+
errorKind = errorKindForPerson(classify(error2), firstOutage);
|
|
6096
6167
|
errors = [{ type: "error", errorText: errorKind }];
|
|
6097
6168
|
}
|
|
6098
6169
|
break;
|
|
6099
6170
|
}
|
|
6100
6171
|
const { result } = attempt;
|
|
6101
6172
|
historyRewritten = false;
|
|
6173
|
+
if (result.outcome === "ok") outages.recovered(providerId);
|
|
6102
6174
|
if (result.outcome !== "failed" || result.contentSeen) answeredProviderId = providerId;
|
|
6103
6175
|
if (result.outcome === "ok" && backToPrimary) {
|
|
6104
6176
|
backToPrimary = false;
|
|
@@ -6133,12 +6205,24 @@ async function runTurn(turnInput, emitRaw) {
|
|
|
6133
6205
|
silenceSwitch: { done: silenceSwitches, max: silenceSwitch.maxSwitches, nextProviderId: silenceTarget() }
|
|
6134
6206
|
});
|
|
6135
6207
|
let next = stepsDone < maxSteps ? decideNextAttempt(result, nextState()) : { action: "fail" };
|
|
6208
|
+
const outage = isProviderOutage(result.kind) ? result.kind : void 0;
|
|
6209
|
+
if (outage) {
|
|
6210
|
+
if (providerId === input.providerId) firstOutage ??= outage;
|
|
6211
|
+
outages.report({
|
|
6212
|
+
providerId,
|
|
6213
|
+
kind: outage,
|
|
6214
|
+
...next.action === "fallback" ? { fallbackProviderId: next.providerId } : { fallbacksPaused: allFallbackIds.some((candidate) => candidate !== providerId && outages.pausedKind(candidate) !== void 0) }
|
|
6215
|
+
});
|
|
6216
|
+
}
|
|
6136
6217
|
if (next.action === "fallback") {
|
|
6137
6218
|
logError(`[\u0437\u0430\u043F\u0430\u0441\u043D\u043E\u0439 \u043F\u0440\u043E\u0432\u0430\u0439\u0434\u0435\u0440] \u0447\u0430\u0442=${input.chatId} \u043C\u043E\u0434\u0435\u043B\u044C=${modelId} \u0441=${providerId} \u043D\u0430=${next.providerId} \u043F\u0440\u0438\u0447\u0438\u043D\u0430=${result.kind} \u043A\u044D\u0448 \u043F\u043E\u0442\u0435\u0440\u044F\u043D`);
|
|
6138
|
-
fallbackUsed = true;
|
|
6219
|
+
if (!outage) fallbackUsed = true;
|
|
6139
6220
|
switchedBySilence = false;
|
|
6140
6221
|
providerId = next.providerId;
|
|
6141
|
-
|
|
6222
|
+
const closing2 = result.contentSeen ? [...openParts.closingChunks(), ...closingChunksForHangingCalls(parts2)] : [];
|
|
6223
|
+
for (const chunk of closing2) emit(chunk);
|
|
6224
|
+
if (closing2.length > 0) answer.continueAfterBreak(parts2, { modelId, providerId: answeredProviderId });
|
|
6225
|
+
else answer.update(parts2, { modelId, providerId: answeredProviderId });
|
|
6142
6226
|
turnUsage.closeAttempt();
|
|
6143
6227
|
if (input.abortSignal.aborted) break;
|
|
6144
6228
|
continue;
|
|
@@ -6210,8 +6294,8 @@ async function runTurn(turnInput, emitRaw) {
|
|
|
6210
6294
|
turnUsage.closeAttempt();
|
|
6211
6295
|
continue;
|
|
6212
6296
|
}
|
|
6213
|
-
errorKind = result.kind;
|
|
6214
|
-
errors = attempt.errors;
|
|
6297
|
+
errorKind = errorKindForPerson(result.kind, firstOutage);
|
|
6298
|
+
errors = errorKind === result.kind ? attempt.errors : [{ type: "error", errorText: errorKind }];
|
|
6215
6299
|
terminal = attempt.terminal;
|
|
6216
6300
|
logError(
|
|
6217
6301
|
describeProviderFailure(result.error, { chatId: input.chatId, modelId, providerId, kind: result.kind, env: input.env, catalog: input.catalog })
|
|
@@ -6823,6 +6907,7 @@ function helperRunContext(input, toolQueue, logError) {
|
|
|
6823
6907
|
...input.toolTimeout && { toolTimeout: input.toolTimeout },
|
|
6824
6908
|
...input.sequentialTools && { sequentialTools: input.sequentialTools },
|
|
6825
6909
|
...input.providerFallback !== void 0 && { providerFallback: input.providerFallback },
|
|
6910
|
+
...input.providerOutages && { providerOutages: input.providerOutages },
|
|
6826
6911
|
...input.modelSilence && { modelSilence: input.modelSilence },
|
|
6827
6912
|
...input.turnObserver && { turnObserver: input.turnObserver },
|
|
6828
6913
|
...input.assertWithinLimit && { assertWithinLimit: input.assertWithinLimit }
|
|
@@ -7053,16 +7138,26 @@ async function* helperSteps(input) {
|
|
|
7053
7138
|
const messages = [...input.priorMessages ?? []];
|
|
7054
7139
|
const maxSteps = helper.maxSteps ?? DEFAULT_HELPER_MAX_STEPS;
|
|
7055
7140
|
const { modelId } = helper.model;
|
|
7056
|
-
const
|
|
7141
|
+
const outages = context.providerOutages ?? createProviderOutages({ now: Date.now, logError: context.logError });
|
|
7142
|
+
const allFallbackIds = fallbackProviders({
|
|
7057
7143
|
catalog: context.catalog,
|
|
7058
7144
|
modelId,
|
|
7059
7145
|
chosenProviderId: helper.model.providerId,
|
|
7060
7146
|
env: context.env,
|
|
7061
7147
|
...context.providerFallback !== void 0 && { setting: context.providerFallback }
|
|
7062
7148
|
});
|
|
7149
|
+
const fallbackProviderIds = allFallbackIds.filter((candidate) => outages.pausedKind(candidate) === void 0);
|
|
7063
7150
|
const toolQueue = context.toolQueue ?? createToolQueue();
|
|
7064
7151
|
let providerId = helper.model.providerId;
|
|
7065
7152
|
let fallbackUsed = false;
|
|
7153
|
+
let firstOutage;
|
|
7154
|
+
const primaryPaused = outages.admit(providerId);
|
|
7155
|
+
if (primaryPaused !== void 0 && fallbackProviderIds[0] !== void 0) {
|
|
7156
|
+
context.logError(
|
|
7157
|
+
`[\u0437\u0430\u043F\u0430\u0441\u043D\u043E\u0439 \u043F\u0440\u043E\u0432\u0430\u0439\u0434\u0435\u0440] \u0447\u0430\u0442=${context.chatId} \u043F\u043E\u043C\u043E\u0449\u043D\u0438\u043A=${input.name} \u043C\u043E\u0434\u0435\u043B\u044C=${modelId} \u0441=${providerId} \u043D\u0430=${fallbackProviderIds[0]} \u043F\u0440\u0438\u0447\u0438\u043D\u0430=${primaryPaused} \u043E\u0441\u043D\u043E\u0432\u043D\u043E\u0439 \u043D\u0430 \u043F\u0430\u0443\u0437\u0435`
|
|
7158
|
+
);
|
|
7159
|
+
providerId = fallbackProviderIds[0];
|
|
7160
|
+
}
|
|
7066
7161
|
let externalContent = false;
|
|
7067
7162
|
const report = (status, extra = {}) => ({
|
|
7068
7163
|
messages: [...messages],
|
|
@@ -7089,17 +7184,28 @@ async function* helperSteps(input) {
|
|
|
7089
7184
|
} catch (error) {
|
|
7090
7185
|
if (abortSignal.aborted) throw error;
|
|
7091
7186
|
const kind = classifyProviderError(error, provider);
|
|
7092
|
-
const
|
|
7187
|
+
const candidates = providerId === helper.model.providerId ? fallbackProviderIds : fallbackProviderIds.slice(fallbackProviderIds.indexOf(providerId) + 1);
|
|
7188
|
+
const next = decideNextAttempt({ outcome: "failed", kind, error, contentSeen: false }, { fallbackUsed, fallbackProviderIds: candidates });
|
|
7189
|
+
const outage = isProviderOutage(kind) ? kind : void 0;
|
|
7190
|
+
if (outage) {
|
|
7191
|
+
if (providerId === helper.model.providerId) firstOutage ??= outage;
|
|
7192
|
+
outages.report({
|
|
7193
|
+
providerId,
|
|
7194
|
+
kind: outage,
|
|
7195
|
+
...next.action === "fallback" ? { fallbackProviderId: next.providerId } : { fallbacksPaused: allFallbackIds.some((candidate) => candidate !== providerId && outages.pausedKind(candidate) !== void 0) }
|
|
7196
|
+
});
|
|
7197
|
+
}
|
|
7093
7198
|
if (next.action === "fallback") {
|
|
7094
7199
|
context.logError(`[\u0437\u0430\u043F\u0430\u0441\u043D\u043E\u0439 \u043F\u0440\u043E\u0432\u0430\u0439\u0434\u0435\u0440] \u0447\u0430\u0442=${context.chatId} \u043F\u043E\u043C\u043E\u0449\u043D\u0438\u043A=${input.name} \u043C\u043E\u0434\u0435\u043B\u044C=${modelId} \u0441=${providerId} \u043D\u0430=${next.providerId} \u043F\u0440\u0438\u0447\u0438\u043D\u0430=${kind}`);
|
|
7095
|
-
fallbackUsed = true;
|
|
7200
|
+
if (!outage) fallbackUsed = true;
|
|
7096
7201
|
providerId = next.providerId;
|
|
7097
7202
|
continue;
|
|
7098
7203
|
}
|
|
7099
7204
|
context.logError(describeProviderFailure(error, { chatId: context.chatId, modelId, providerId, kind, env: context.env, catalog: context.catalog }));
|
|
7100
|
-
yield report("failed", { error: FAILURE_REASONS[kind] });
|
|
7205
|
+
yield report("failed", { error: FAILURE_REASONS[errorKindForPerson(kind, firstOutage)] });
|
|
7101
7206
|
return;
|
|
7102
7207
|
}
|
|
7208
|
+
outages.recovered(providerId);
|
|
7103
7209
|
messages.push(...response);
|
|
7104
7210
|
externalContent ||= callsExternalContentTool(messages, tools);
|
|
7105
7211
|
const pendingApproval = pendingApprovalOf2(messages, tools);
|
|
@@ -11035,6 +11141,7 @@ var InvalidWidgetAnswerError = class extends Error {
|
|
|
11035
11141
|
}
|
|
11036
11142
|
};
|
|
11037
11143
|
var FAR_FUTURE_ISO = "9999-12-31T23:59:59.999Z";
|
|
11144
|
+
var abandonedBurstReceivers = /* @__PURE__ */ new WeakMap();
|
|
11038
11145
|
var TurnEndedBeforeSubscribeError = class extends Error {
|
|
11039
11146
|
constructor(chatId) {
|
|
11040
11147
|
super(`\u0425\u043E\u0434 \u0432 \u0447\u0430\u0442\u0435 ${chatId} \u0437\u0430\u043A\u043E\u043D\u0447\u0438\u043B\u0441\u044F \u0434\u043E \u043F\u043E\u0434\u043F\u0438\u0441\u043A\u0438`);
|
|
@@ -11232,6 +11339,7 @@ function createChatService(input) {
|
|
|
11232
11339
|
const modelSilence = resolveModelSilence(modelSilenceTimeout);
|
|
11233
11340
|
const cacheMissWatch = cacheMissLog === false ? void 0 : createCacheMissWatch({ logError });
|
|
11234
11341
|
checkProviderFallback(providerFallback, options2.catalog);
|
|
11342
|
+
const providerOutages = createProviderOutages({ now: () => now().getTime(), logError });
|
|
11235
11343
|
const placeMaxLength = resolvePlaceMaxLength(placeMaxLengthSetting);
|
|
11236
11344
|
const browserToolTimeoutMs = resolveBrowserToolTimeoutMs(browserToolTimeoutSetting);
|
|
11237
11345
|
const pendingBrowserCalls = createPendingBrowserCalls();
|
|
@@ -11907,6 +12015,7 @@ function createChatService(input) {
|
|
|
11907
12015
|
modelSilence,
|
|
11908
12016
|
...limits.maxOutputTokens !== void 0 && { maxOutputTokens: limits.maxOutputTokens },
|
|
11909
12017
|
...providerFallback !== void 0 && { providerFallback },
|
|
12018
|
+
providerOutages,
|
|
11910
12019
|
...cacheMissWatch && { cacheMissWatch },
|
|
11911
12020
|
...options2.turnObserver && { turnObserver: options2.turnObserver },
|
|
11912
12021
|
...options2.limits && { assertWithinLimit: () => assertWithinSpendingLimit(chat.ownerId, chatId) },
|
|
@@ -12229,10 +12338,10 @@ function createChatService(input) {
|
|
|
12229
12338
|
...page && { page }
|
|
12230
12339
|
});
|
|
12231
12340
|
}
|
|
12232
|
-
function renewingBurstHold(chatId, messageId, dueAt) {
|
|
12341
|
+
function renewingBurstHold(chatId, messageId, dueAt, holder) {
|
|
12233
12342
|
const timer = setInterval(() => {
|
|
12234
12343
|
const heldUntil = new Date(Math.max(now().getTime(), dueAt) + BURST_HOLD_MS).toISOString();
|
|
12235
|
-
storage.renewPendingBurst({ chatId, messageId, holder
|
|
12344
|
+
storage.renewPendingBurst({ chatId, messageId, holder, heldUntil }).then(
|
|
12236
12345
|
(held) => {
|
|
12237
12346
|
if (!held) clearInterval(timer);
|
|
12238
12347
|
},
|
|
@@ -12246,7 +12355,7 @@ function createChatService(input) {
|
|
|
12246
12355
|
function answeringRecordedBurst(ownerId, chatId, questionId, page, burst) {
|
|
12247
12356
|
const unattended = burst.kept && burst.unattended === true;
|
|
12248
12357
|
if (!burst.kept || !keepsPendingBursts) return burstWaits.holding(chatId, () => answeringBurst(ownerId, chatId, questionId, page, unattended));
|
|
12249
|
-
const stopRenewing = renewingBurstHold(chatId, questionId, burst.dueAt);
|
|
12358
|
+
const stopRenewing = renewingBurstHold(chatId, questionId, burst.dueAt, burst.holder ?? copyId);
|
|
12250
12359
|
const answering = burstWaits.holding(chatId, () => answeringBurst(ownerId, chatId, questionId, page, unattended)).finally(stopRenewing);
|
|
12251
12360
|
void answering.then(
|
|
12252
12361
|
() => true,
|
|
@@ -12272,6 +12381,37 @@ function createChatService(input) {
|
|
|
12272
12381
|
const why = error instanceof TurnRefusedError ? `\u043E\u0442\u043A\u0430\u0437 \u043F\u0440\u0438\u043B\u043E\u0436\u0435\u043D\u0438\u044F ${error.code}: ${error.message}` : error instanceof Error ? error.message : String(error);
|
|
12273
12382
|
logError(`\u0425\u043E\u0434 \u043D\u0430 \u0431\u0440\u043E\u0448\u0435\u043D\u043D\u0443\u044E \u043F\u0430\u0447\u043A\u0443 \u0447\u0430\u0442\u0430 ${chatId} \u043D\u0435 \u043D\u0430\u0447\u0430\u043B\u0441\u044F: ${why}`);
|
|
12274
12383
|
}
|
|
12384
|
+
async function receivingAbandonedBurst(burst) {
|
|
12385
|
+
if (burstWaits.has(burst.chatId)) return;
|
|
12386
|
+
await answeringRecordedBurst(burst.ownerId, burst.chatId, burst.messageId, burst.page, { kept: true, dueAt: Date.parse(burst.dueAt), unattended: true, holder: burst.holder }).then(
|
|
12387
|
+
() => {
|
|
12388
|
+
},
|
|
12389
|
+
(error) => reportingAbandonedBurst(burst.chatId, error)
|
|
12390
|
+
);
|
|
12391
|
+
}
|
|
12392
|
+
async function routingAbandonedBurst(burst) {
|
|
12393
|
+
const serviceFor = options2.abandonedBursts?.serviceFor;
|
|
12394
|
+
if (!serviceFor) return receivingAbandonedBurst(burst);
|
|
12395
|
+
let target;
|
|
12396
|
+
try {
|
|
12397
|
+
const chat = await storage.getChat(burst.chatId);
|
|
12398
|
+
if (!chat) return receivingAbandonedBurst(burst);
|
|
12399
|
+
target = await serviceFor({ ownerId: burst.ownerId, chatId: burst.chatId, chat });
|
|
12400
|
+
} catch (error) {
|
|
12401
|
+
logError(`\u0421\u043B\u0443\u0436\u0431\u0430 \u0434\u043B\u044F \u0431\u0440\u043E\u0448\u0435\u043D\u043D\u043E\u0439 \u043F\u0430\u0447\u043A\u0438 \u0447\u0430\u0442\u0430 ${burst.chatId} \u043D\u0435 \u043D\u0430\u0448\u043B\u0430\u0441\u044C (serviceFor): ${error instanceof Error ? error.message : String(error)} \u2014 \u043F\u0430\u0447\u043A\u0430 \u0436\u0434\u0451\u0442 \u0441\u043B\u0435\u0434\u0443\u044E\u0449\u0435\u0433\u043E \u043E\u0431\u0445\u043E\u0434\u0430`);
|
|
12402
|
+
return;
|
|
12403
|
+
}
|
|
12404
|
+
if (!target) {
|
|
12405
|
+
logError(`\u0414\u043B\u044F \u0431\u0440\u043E\u0448\u0435\u043D\u043D\u043E\u0439 \u043F\u0430\u0447\u043A\u0438 \u0447\u0430\u0442\u0430 ${burst.chatId} \u043D\u0435\u0442 \u0441\u043B\u0443\u0436\u0431\u044B (serviceFor \u2192 ${String(target)}): \u043F\u0430\u0447\u043A\u0430 \u0436\u0434\u0451\u0442 \u0441\u043B\u0435\u0434\u0443\u044E\u0449\u0435\u0433\u043E \u043E\u0431\u0445\u043E\u0434\u0430`);
|
|
12406
|
+
return;
|
|
12407
|
+
}
|
|
12408
|
+
const receive = abandonedBurstReceivers.get(target);
|
|
12409
|
+
if (!receive) {
|
|
12410
|
+
logError(`serviceFor \u0434\u043B\u044F \u0431\u0440\u043E\u0448\u0435\u043D\u043D\u043E\u0439 \u043F\u0430\u0447\u043A\u0438 \u0447\u0430\u0442\u0430 ${burst.chatId} \u0432\u0435\u0440\u043D\u0443\u043B \u043D\u0435 \u0441\u043B\u0443\u0436\u0431\u0443 \u0447\u0430\u0442\u0430 SDK: \u043F\u0430\u0447\u043A\u0430 \u0436\u0434\u0451\u0442 \u0441\u043B\u0435\u0434\u0443\u044E\u0449\u0435\u0433\u043E \u043E\u0431\u0445\u043E\u0434\u0430`);
|
|
12411
|
+
return;
|
|
12412
|
+
}
|
|
12413
|
+
await receive(burst);
|
|
12414
|
+
}
|
|
12275
12415
|
function sweepingBursts(take) {
|
|
12276
12416
|
if (!keepsPendingBursts || closing) return Promise.resolve();
|
|
12277
12417
|
const sweep = (async () => {
|
|
@@ -12286,9 +12426,8 @@ function createChatService(input) {
|
|
|
12286
12426
|
const taken = await storage.takeAbandonedBursts({ ...take, holder: copyId, heldUntil, limit: free });
|
|
12287
12427
|
if (taken.length === 0) return;
|
|
12288
12428
|
for (const burst of taken) {
|
|
12289
|
-
if (burstWaits.has(burst.chatId)) continue;
|
|
12290
12429
|
burstDecisions++;
|
|
12291
|
-
|
|
12430
|
+
void routingAbandonedBurst(burst).finally(releasingBurstDecision);
|
|
12292
12431
|
}
|
|
12293
12432
|
}
|
|
12294
12433
|
})().catch((error) => logError(`\u041E\u0431\u0445\u043E\u0434 \u0436\u0434\u0443\u0449\u0438\u0445 \u043F\u0430\u0447\u0435\u043A \u043D\u0435 \u0443\u0434\u0430\u043B\u0441\u044F: ${error instanceof Error ? error.message : String(error)}`)).finally(() => burstSweeps.delete(sweep));
|
|
@@ -13203,6 +13342,7 @@ function createChatService(input) {
|
|
|
13203
13342
|
modelSilence,
|
|
13204
13343
|
logError,
|
|
13205
13344
|
...providerFallback !== void 0 && { providerFallback },
|
|
13345
|
+
providerOutages,
|
|
13206
13346
|
...options2.turnObserver && { turnObserver: options2.turnObserver },
|
|
13207
13347
|
...options2.limits && { assertWithinLimit: () => assertWithinSpendingLimit(ownerId, chatId) }
|
|
13208
13348
|
});
|
|
@@ -13212,6 +13352,7 @@ function createChatService(input) {
|
|
|
13212
13352
|
helpers: backgroundHelpers
|
|
13213
13353
|
};
|
|
13214
13354
|
}
|
|
13355
|
+
abandonedBurstReceivers.set(service, receivingAbandonedBurst);
|
|
13215
13356
|
return service;
|
|
13216
13357
|
function isDetachedAnswer(last, runningId) {
|
|
13217
13358
|
return runningId === null && last?.status === "streaming" && !liveAnswerIds.has(last.id);
|
|
@@ -13473,4 +13614,4 @@ export {
|
|
|
13473
13614
|
InvalidWidgetAnswerError,
|
|
13474
13615
|
createChatService
|
|
13475
13616
|
};
|
|
13476
|
-
//# sourceMappingURL=chunk-
|
|
13617
|
+
//# sourceMappingURL=chunk-KAG3QTZC.js.map
|