@ykurilov/agent-chat-images 0.80.1 → 0.82.0

package/dist/index.js

@@ -1,20 +1,23 @@
 import {
   ImageJobKeyTakenError,
+  hasPendingImageFailure,
   isOpenImageJob,
   memoryImageJobStore
-} from "./chunk-5IORNMTX.js";
+} from "./chunk-JRBKNWG3.js";
 import {
   ImageGenerationError,
   aspectRatios,
+  canRetryImageFailure,
   extendImageCatalog,
   hideSecrets,
   imageQualities,
   imageSizes,
   kieProvider,
+  providerFailureKind,
   runwareProvider,
   standardImageCatalog,
   waitForImageTask
-} from "./chunk-33SNN6BB.js";
+} from "./chunk-W74LUWVQ.js";
 
 // src/create-image-generation.ts
 import { tool } from "ai";
@@ -23,6 +26,7 @@ import { SpendingLimitReachedError as SpendingLimitReachedError2, declareToolTim
 
 // src/download-image.ts
 import { detectFileKind } from "@ykurilov/agent-chat-attachments";
+import { dnsHostResolver, sharedPinnedFetch, WebToolError } from "@ykurilov/agent-chat-web-tools/network";
 
 // src/download-address.ts
 var PRIVATE_IPV4 = [
@@ -109,7 +113,7 @@ function pngSize(bytes) {
 var DOWNLOAD_TIMEOUT_MS = 6e4;
 var MAX_REDIRECTS = 3;
 var imageExtensions = { "image/png": "png", "image/jpeg": "jpg", "image/webp": "webp" };
-var failure = (reason) => new ImageGenerationError(reason, { refusedBeforeWork: false });
+var failure = (reason, kind = "unknown") => new ImageGenerationError(reason, { refusedBeforeWork: false, kind });
 var megabytes = (bytes) => `${Math.round(bytes / (1024 * 1024) * 10) / 10} \u041C\u0411`;
 async function readWithin(response, maxBytes) {
   if (!response.body) return new Uint8Array();
@@ -140,15 +144,20 @@ function checkedAddress(url, base, trustedOrigin) {
   try {
     address = new URL(url, base);
   } catch {
-    throw failure("\u041A\u0430\u0440\u0442\u0438\u043D\u043A\u0430 \u043D\u0435 \u0441\u043A\u0430\u0447\u0430\u043B\u0430\u0441\u044C: \u043D\u0435\u0432\u0435\u0440\u043D\u044B\u0439 \u0430\u0434\u0440\u0435\u0441");
+    throw failure("\u041A\u0430\u0440\u0442\u0438\u043D\u043A\u0430 \u043D\u0435 \u0441\u043A\u0430\u0447\u0430\u043B\u0430\u0441\u044C: \u043D\u0435\u0432\u0435\u0440\u043D\u044B\u0439 \u0430\u0434\u0440\u0435\u0441", "configuration");
   }
-  if (!isAllowedImageAddress(address, trustedOrigin)) throw failure("\u041A\u0430\u0440\u0442\u0438\u043D\u043A\u0430 \u043D\u0435 \u0441\u043A\u0430\u0447\u0430\u043B\u0430\u0441\u044C: \u043D\u0435\u0432\u0435\u0440\u043D\u044B\u0439 \u0430\u0434\u0440\u0435\u0441");
+  if (!isAllowedImageAddress(address, trustedOrigin)) throw failure("\u041A\u0430\u0440\u0442\u0438\u043D\u043A\u0430 \u043D\u0435 \u0441\u043A\u0430\u0447\u0430\u043B\u0430\u0441\u044C: \u043D\u0435\u0432\u0435\u0440\u043D\u044B\u0439 \u0430\u0434\u0440\u0435\u0441", "configuration");
   return address;
 }
+function fetchFor(input, address) {
+  if (input.fetch) return input.fetch;
+  if (input.trustedOrigin !== void 0 && address.origin === input.trustedOrigin) return globalThis.fetch;
+  return sharedPinnedFetch(input.resolveHost ?? dnsHostResolver);
+}
 async function fetchImage(input, first, signal) {
   let address = first;
   for (let hop = 0; ; hop++) {
-    const response = await input.fetch(address.href, { signal, redirect: "manual" });
+    const response = await fetchFor(input, address)(address.href, { signal, redirect: "manual" });
     const location = response.status >= 300 && response.status < 400 ? response.headers.get("location") : null;
     if (location === null) return response;
     await response.body?.cancel().catch(() => {
@@ -166,14 +175,15 @@ async function downloadGeneratedImage(input) {
   let bytes;
   try {
     const response = await fetchImage(input, address, signal);
-    if (!response.ok) throw failure(`\u041A\u0430\u0440\u0442\u0438\u043D\u043A\u0430 \u043D\u0435 \u0441\u043A\u0430\u0447\u0430\u043B\u0430\u0441\u044C: ${response.status}`);
+    if (!response.ok) throw failure(`\u041A\u0430\u0440\u0442\u0438\u043D\u043A\u0430 \u043D\u0435 \u0441\u043A\u0430\u0447\u0430\u043B\u0430\u0441\u044C: ${response.status}`, response.status === 408 || response.status === 429 || response.status >= 500 ? "transient" : "configuration");
     const declared = Number(response.headers.get("content-length"));
     if (Number.isFinite(declared) && declared > input.maxImageBytes) throw failure(`\u041A\u0430\u0440\u0442\u0438\u043D\u043A\u0430 \u0431\u043E\u043B\u044C\u0448\u0435 ${megabytes(input.maxImageBytes)}`);
     bytes = await readWithin(response, input.maxImageBytes);
   } catch (error) {
     if (input.abortSignal?.aborted) throw input.abortSignal.reason ?? error;
     if (error instanceof ImageGenerationError) throw error;
-    throw failure(timeout.signal.aborted ? "\u041A\u0430\u0440\u0442\u0438\u043D\u043A\u0430 \u043D\u0435 \u0441\u043A\u0430\u0447\u0430\u043B\u0430\u0441\u044C: \u043D\u0435 \u0443\u0441\u043F\u0435\u043B\u0430" : "\u041A\u0430\u0440\u0442\u0438\u043D\u043A\u0430 \u043D\u0435 \u0441\u043A\u0430\u0447\u0430\u043B\u0430\u0441\u044C");
+    if (error instanceof WebToolError && error.kind === "bad_address") throw failure("\u041A\u0430\u0440\u0442\u0438\u043D\u043A\u0430 \u043D\u0435 \u0441\u043A\u0430\u0447\u0430\u043B\u0430\u0441\u044C: \u043D\u0435\u0432\u0435\u0440\u043D\u044B\u0439 \u0430\u0434\u0440\u0435\u0441", "configuration");
+    throw failure(timeout.signal.aborted ? "\u041A\u0430\u0440\u0442\u0438\u043D\u043A\u0430 \u043D\u0435 \u0441\u043A\u0430\u0447\u0430\u043B\u0430\u0441\u044C: \u043D\u0435 \u0443\u0441\u043F\u0435\u043B\u0430" : "\u041A\u0430\u0440\u0442\u0438\u043D\u043A\u0430 \u043D\u0435 \u0441\u043A\u0430\u0447\u0430\u043B\u0430\u0441\u044C", "transient");
   } finally {
     clearTimeout(timer);
   }
@@ -302,7 +312,7 @@ function createImageJobs(options) {
     const output = input.setup.output;
     const file = await downloadGeneratedImage({
       url,
-      fetch: last.connection.fetch,
+      ...last.downloadFetch && { fetch: last.downloadFetch },
       maxImageBytes: options.maxImageBytes,
       ...last.trustedOrigin !== void 0 && { trustedOrigin: last.trustedOrigin },
       ...output && { transform: (bytes, mediaType) => postprocessImage(bytes, mediaType, output) },
@@ -534,6 +544,7 @@ function resolveImageSetups(choice) {
       sizes,
       priceUsd: roundUsd(sizePriceUsd * qualityFactor),
       ...provider.baseURL !== void 0 && { trustedOrigin: new URL(provider.baseURL).origin },
+      ...choice.fetch && { downloadFetch: choice.fetch },
       ...providerSettings && { providerSettings },
       ...settings.transparentBackground && { outputFormat: "png" }
     };
@@ -666,9 +677,14 @@ function markRecorded(record, usageId) {
   }));
   if (!found) return null;
   if (isOpen(record)) return { steps };
-  const left = hasUnrecorded(steps);
+  const left = hasUnrecorded(steps) || hasPendingImageFailure(record);
   return { steps, active: left, ...!left && { leaseOwner: void 0, leaseUntil: void 0 } };
 }
+function failureOf(record) {
+  if (record.failure) return record.failure;
+  if (record.status !== "failed" || record.lateUntil !== void 0 || !record.reason) return void 0;
+  return { kind: providerFailureKind(void 0, void 0, record.reason), reason: record.reason };
+}
 function view(record) {
   return {
     id: record.id,
@@ -683,6 +699,7 @@ function view(record) {
     ...record.data !== void 0 && { data: record.data },
     status: record.status,
     ...record.reason !== void 0 && { reason: record.reason },
+    ...failureOf(record) !== void 0 && { failure: structuredClone(failureOf(record)) },
     ...record.result !== void 0 && { result: record.result },
     attempt: record.attempt,
     ...record.lateUntil !== void 0 && { lateUntil: record.lateUntil },
@@ -693,7 +710,7 @@ function view(record) {
 }
 var refused = (reason) => ({ status: "refused", reason });
 function checkOptions(options) {
-  const positive = ["pollIntervalMs", "attemptTimeoutMs", "latePollIntervalMs", "callTimeoutMs", "acceptWaitMs", "leaseMs", "forgetFinishedAfterMs", "maxImageBytes"];
+  const positive = ["failureRetryMs", "pollIntervalMs", "attemptTimeoutMs", "latePollIntervalMs", "callTimeoutMs", "acceptWaitMs", "leaseMs", "forgetFinishedAfterMs", "maxImageBytes"];
   for (const name of positive) {
     const value = options[name];
     if (value !== void 0 && !(Number.isFinite(value) && value > 0)) throw new Error(`\u041A\u0430\u0440\u0442\u0438\u043D\u043A\u0438: ${name} \u2014 \u043D\u0443\u0436\u043D\u043E \u0447\u0438\u0441\u043B\u043E \u0431\u043E\u043B\u044C\u0448\u0435 \u043D\u0443\u043B\u044F`);
@@ -704,6 +721,7 @@ function checkOptions(options) {
   }
   const window = options.lateResultWindowMs;
   if (window !== void 0 && !(Number.isFinite(window) && window >= 0)) throw new Error("\u041A\u0430\u0440\u0442\u0438\u043D\u043A\u0438: lateResultWindowMs \u2014 \u0447\u0438\u0441\u043B\u043E \u043D\u0435 \u043C\u0435\u043D\u044C\u0448\u0435 \u043D\u0443\u043B\u044F");
+  if (options.onFailed !== void 0 && typeof options.onFailed !== "function") throw new TypeError("\u041A\u0430\u0440\u0442\u0438\u043D\u043A\u0438: onFailed \u2014 \u043D\u0443\u0436\u043D\u0430 \u0444\u0443\u043D\u043A\u0446\u0438\u044F");
   if (typeof options.deliver !== "function") throw new Error("\u041A\u0430\u0440\u0442\u0438\u043D\u043A\u0438: deliver \u2014 \u043D\u0443\u0436\u043D\u0430 \u0444\u0443\u043D\u043A\u0446\u0438\u044F-\u043F\u0440\u0438\u0451\u043C\u043D\u0438\u043A");
 }
 function createImageQueue(options) {
@@ -717,6 +735,7 @@ function createImageQueue(options) {
   const nowMs = () => now().getTime();
   const hide = setups.hideSecrets;
   const logError = (message) => options.logError?.(hide(message));
+  const failureRetryMs = options.failureRetryMs ?? 3e4;
   const pollIntervalMs = options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
   const attemptTimeoutMs = options.attemptTimeoutMs ?? DEFAULT_ATTEMPT_TIMEOUT_MS;
   const attempts = options.attempts ?? 1;
@@ -853,7 +872,7 @@ function createImageQueue(options) {
       current = await updateJob(current, (fresh) => markRecorded(fresh, charge.usageId), () => true);
       if (!current) return;
     }
-    await updateJob(current, (fresh) => isOpen(fresh) || !fresh.active && fresh.leaseOwner === void 0 ? null : { active: hasUnrecorded(fresh.steps), leaseOwner: void 0, leaseUntil: void 0 }, () => true);
+    await updateJob(current, (fresh) => isOpen(fresh) || !fresh.active && fresh.leaseOwner === void 0 ? null : { active: hasUnrecorded(fresh.steps) || hasPendingImageFailure(fresh), leaseOwner: void 0, leaseUntil: void 0 }, () => true);
   }
   async function chargeTask(rt, index, costUsd) {
     await write(rt, (record) => {
@@ -878,24 +897,29 @@ function createImageQueue(options) {
       (fresh) => {
         if (!isOpen(fresh)) return null;
         const steps = fresh.steps.map((step) => step.task ? { ...withoutTask(step), charges: [...step.charges, chargeOf(step.task, step.task.priceUsd)] } : step);
-        return { status, reason, run: fresh.run + 1, steps, lateUntil: void 0, active: hasUnrecorded(steps), finishedAt: iso(nowMs()), leaseOwner: void 0, leaseUntil: void 0 };
+        return { status, reason, failure: status === "failed" ? { kind: "configuration", reason: reason ?? "\u041D\u0430\u0441\u0442\u0440\u043E\u0439\u043A\u0430 \u043A\u0430\u0440\u0442\u0438\u043D\u043E\u043A \u0441\u043C\u0435\u043D\u0438\u043B\u0430\u0441\u044C" } : void 0, failureNotification: status === "failed" && options.onFailed ? { id: `image-failed:${fresh.id}:${fresh.run + 1}`, attempts: 0, nextAttemptAt: iso(nowMs()) } : void 0, run: fresh.run + 1, steps, lateUntil: void 0, active: hasUnrecorded(steps) || status === "failed" && options.onFailed !== void 0, finishedAt: iso(nowMs()), leaseOwner: void 0, leaseUntil: void 0 };
       },
       () => true
     );
     if (!closed) return;
     byJob.get(record.id)?.controller.abort(RUN_ENDED);
     await settleClosed(closed);
+    if (status === "failed" && options.onFailed) {
+      const pending = await store.get(closed.id);
+      if (pending && hasPendingImageFailure(pending)) await claim(pending);
+    }
   }
   function openUsd(rt) {
     const record = rt.record;
     if (!record) return rt.reservedUsd;
-    if (!isOpen(record)) return 0;
+    if (!isOpen(record) || !("setup" in rt)) return 0;
+    const setup = rt.setup;
     const late = record.lateUntil !== void 0;
     const last = record.steps.length - 1;
     return record.steps.reduce((total, step, index) => {
-      if (step.urls) return rt.reorderOnDeliveryFailure && index === last ? total + rt.setup.steps[index].priceUsd : total;
+      if (step.urls) return rt.reorderOnDeliveryFailure && index === last ? total + setup.steps[index].priceUsd : total;
       if (step.task) return total + step.task.priceUsd;
-      return late ? total : total + (rt.setup.steps[index]?.priceUsd ?? 0);
+      return late ? total : total + (setup.steps[index]?.priceUsd ?? 0);
     }, 0);
   }
   function pendingUsd(ownerId, chatId, before) {
@@ -941,9 +965,10 @@ function createImageQueue(options) {
   }
   async function failedSend(rt, index, error, wholeAttempt) {
     throwIfEnded(rt);
+    const failure2 = { kind: error instanceof ImageGenerationError ? error.kind : error instanceof CallTimedOut ? "transient" : "unknown", reason: error instanceof ImageGenerationError ? hide(error.reason) : error instanceof CallTimedOut ? error.message : "\u041F\u0440\u043E\u0432\u0430\u0439\u0434\u0435\u0440 \u043D\u0435 \u0441\u043C\u043E\u0433 \u0441\u043E\u0437\u0434\u0430\u0442\u044C \u043A\u0430\u0440\u0442\u0438\u043D\u043A\u0443", ...error instanceof ImageGenerationError && error.providerCode && { providerCode: hide(error.providerCode) } };
     if (error instanceof ImageGenerationError && error.refusedBeforeWork) {
       await write(rt, (record) => ({ steps: withStep(record.steps, index, withoutTask) }));
-      return { kind: "refused", reason: hide(error.reason) };
+      return { kind: "refused", reason: hide(error.reason), failure: failure2 };
     }
     let reason;
     if (error instanceof ImageGenerationError) reason = hide(error.reason);
@@ -953,7 +978,7 @@ function createImageQueue(options) {
       reason = "\u041F\u0440\u043E\u0432\u0430\u0439\u0434\u0435\u0440 \u043D\u0435 \u0441\u043C\u043E\u0433 \u0441\u043E\u0437\u0434\u0430\u0442\u044C \u043A\u0430\u0440\u0442\u0438\u043D\u043A\u0443";
     }
     await chargeTask(rt, index);
-    return { kind: "failed", reason };
+    return { kind: canRetryImageFailure(failure2.kind) ? "failed" : "refused", reason, failure: failure2 };
   }
   async function send(rt, index) {
     const step = rt.setup.steps[index];
@@ -1009,7 +1034,7 @@ function createImageQueue(options) {
         throwIfEnded(rt);
         if (error instanceof ImageGenerationError) {
           await chargeTask(rt, index);
-          return { kind: "failed", reason: hide(error.reason) };
+          return { kind: canRetryImageFailure(error.kind) ? "failed" : "refused", reason: hide(error.reason), failure: { kind: error.kind, reason: hide(error.reason), ...error.providerCode && { providerCode: hide(error.providerCode) } } };
         }
         continue;
       }
@@ -1020,21 +1045,23 @@ function createImageQueue(options) {
       }
       if (state.status === "failed") {
         await chargeTask(rt, index, state.costUsd);
-        return { kind: "failed", reason: hide(state.reason) };
+        return { kind: canRetryImageFailure(state.failureKind) ? "failed" : "refused", reason: hide(state.reason), failure: { kind: state.failureKind ?? "unknown", reason: hide(state.reason), ...state.providerCode && { providerCode: hide(state.providerCode) } } };
       }
     }
   }
-  async function finishFailed(rt, reason, extra = {}) {
+  async function finishFailed(rt, reason, extra = {}, failure2 = { kind: "unknown", reason }) {
     await write(rt, (record) => ({
       status: "failed",
       reason,
       lateUntil: void 0,
       ...extra,
-      active: hasUnrecorded(record.steps),
+      failure: failure2,
+      failureNotification: options.onFailed ? { id: `image-failed:${record.id}:${record.run}`, attempts: 0, nextAttemptAt: iso(nowMs()) } : void 0,
+      active: hasUnrecorded(record.steps) || options.onFailed !== void 0,
       finishedAt: iso(nowMs()),
-      leaseOwner: void 0,
-      leaseUntil: void 0
+      ...options.onFailed ? {} : { leaseOwner: void 0, leaseUntil: void 0 }
     }));
+    rt.sent.resolve();
   }
   const nextAttempt = (rt) => write(rt, (record) => ({ attempt: record.attempt + 1, attemptDeadline: iso(nowMs() + attemptTimeoutMs) }));
   async function runStep(rt, index) {
@@ -1057,7 +1084,7 @@ function createImageQueue(options) {
         return true;
       }
       if (outcome.kind === "refused") {
-        await finishFailed(rt, outcome.reason);
+        await finishFailed(rt, outcome.reason, {}, outcome.failure);
         return false;
       }
       if (outcome.kind === "failed") {
@@ -1065,12 +1092,12 @@ function createImageQueue(options) {
           await nextAttempt(rt);
           continue;
         }
-        await finishFailed(rt, outcome.reason);
+        await finishFailed(rt, outcome.reason, {}, outcome.failure);
         return false;
       }
       if (late) {
         await chargeTask(rt, index);
-        await finishFailed(rt, rt.record.reason ?? timeoutReason);
+        await finishFailed(rt, rt.record.reason ?? timeoutReason, {}, { kind: "transient", reason: rt.record.reason ?? timeoutReason });
         return false;
       }
       if (rt.record.attempt < attempts) {
@@ -1083,7 +1110,7 @@ function createImageQueue(options) {
         continue;
       }
       await chargeTask(rt, index);
-      await finishFailed(rt, timeoutReason);
+      await finishFailed(rt, timeoutReason, {}, { kind: "transient", reason: timeoutReason });
       return false;
     }
   }
@@ -1102,7 +1129,7 @@ function createImageQueue(options) {
       try {
         file = await downloadGeneratedImage({
           url,
-          fetch: last.connection.fetch,
+          ...last.downloadFetch && { fetch: last.downloadFetch },
           maxImageBytes,
           ...last.trustedOrigin !== void 0 && { trustedOrigin: last.trustedOrigin },
           abortSignal: rt.controller.signal,
@@ -1113,7 +1140,7 @@ function createImageQueue(options) {
         throwIfEnded(rt);
         if (attempt >= downloadAttempts) {
           if (rt.reorderOnDeliveryFailure) return "reorder";
-          await finishFailed(rt, error instanceof ImageGenerationError ? hide(error.reason) : "\u041A\u0430\u0440\u0442\u0438\u043D\u043A\u0430 \u043D\u0435 \u0441\u043A\u0430\u0447\u0430\u043B\u0430\u0441\u044C", { deliveryFailed: true });
+          await finishFailed(rt, error instanceof ImageGenerationError ? hide(error.reason) : "\u041A\u0430\u0440\u0442\u0438\u043D\u043A\u0430 \u043D\u0435 \u0441\u043A\u0430\u0447\u0430\u043B\u0430\u0441\u044C", { deliveryFailed: true }, { kind: error instanceof ImageGenerationError ? error.kind : "unknown", reason: error instanceof ImageGenerationError ? hide(error.reason) : "\u041A\u0430\u0440\u0442\u0438\u043D\u043A\u0430 \u043D\u0435 \u0441\u043A\u0430\u0447\u0430\u043B\u0430\u0441\u044C" });
           return "done";
         }
         await sleep(pollIntervalMs, rt.controller.signal);
@@ -1138,7 +1165,7 @@ function createImageQueue(options) {
     } catch (error) {
       throwIfEnded(rt);
       logError(`\u041A\u0430\u0440\u0442\u0438\u043D\u043A\u0438: \u043F\u0440\u0438\u0451\u043C\u043D\u0438\u043A \u043D\u0435 \u043F\u0440\u0438\u043D\u044F\u043B \u043A\u0430\u0440\u0442\u0438\u043D\u043A\u0443 \u0437\u0430\u0434\u0430\u0447\u0438 ${record.id}: ${errorText2(error)}`);
-      await finishFailed(rt, "\u041F\u0440\u0438\u0451\u043C\u043D\u0438\u043A \u043D\u0435 \u043F\u0440\u0438\u043D\u044F\u043B \u043A\u0430\u0440\u0442\u0438\u043D\u043A\u0443", { deliveryFailed: true });
+      await finishFailed(rt, "\u041F\u0440\u0438\u0451\u043C\u043D\u0438\u043A \u043D\u0435 \u043F\u0440\u0438\u043D\u044F\u043B \u043A\u0430\u0440\u0442\u0438\u043D\u043A\u0443", { deliveryFailed: true }, { kind: "delivery", reason: "\u041F\u0440\u0438\u0451\u043C\u043D\u0438\u043A \u043D\u0435 \u043F\u0440\u0438\u043D\u044F\u043B \u043A\u0430\u0440\u0442\u0438\u043D\u043A\u0443" });
       return "done";
     }
     throwIfEnded(rt);
@@ -1174,16 +1201,62 @@ function createImageQueue(options) {
     try {
       await recordCharges(rt);
       if (isOpen(rt.record)) await generate(rt);
+      await notifyFailure(rt);
     } catch (error) {
       if (error !== RUN_ENDED && !rt.controller.signal.aborted) logError(`\u041A\u0430\u0440\u0442\u0438\u043D\u043A\u0438: \u0437\u0430\u0434\u0430\u0447\u0430 ${rt.record?.id ?? ""} \u043E\u0441\u0442\u0430\u043D\u043E\u0432\u0438\u043B\u0430\u0441\u044C: ${errorText2(error)}`);
     } finally {
-      rt.ended = true;
-      clearTimeout(rt.leaseTimer);
-      runtimes.delete(rt);
-      if (rt.record && byJob.get(rt.record.id) === rt) byJob.delete(rt.record.id);
-      rt.sent.resolve();
+      finishRuntime(rt);
+    }
+  }
+  function finishRuntime(rt) {
+    rt.ended = true;
+    clearTimeout(rt.leaseTimer);
+    runtimes.delete(rt);
+    if (rt.record && byJob.get(rt.record.id) === rt) byJob.delete(rt.record.id);
+    rt.sent.resolve();
+  }
+  async function notifyFailure(rt) {
+    const record = rt.record;
+    const notice = record.failureNotification;
+    if (!options.onFailed || !notice || notice.deliveredAt || record.status !== "failed" || !record.failure || record.lateUntil) return;
+    if (Date.parse(notice.nextAttemptAt) > nowMs()) return;
+    const reserved = await write(rt, (fresh) => ({ failureNotification: { ...fresh.failureNotification, attempts: fresh.failureNotification.attempts + 1, nextAttemptAt: iso(nowMs() + failureRetryMs) } }));
+    const id = notice.id;
+    const isCurrent = async () => {
+      if (rt.controller.signal.aborted) return false;
+      const fresh = await store.get(record.id);
+      const current = fresh?.status === "failed" && fresh.run === record.run && fresh.leaseOwner === me && Date.parse(fresh.leaseUntil ?? "") > nowMs() && fresh.failureNotification?.id === id && fresh.failureNotification.deliveredAt === void 0 && !rt.controller.signal.aborted;
+      if (!current) rt.controller.abort(RUN_ENDED);
+      return current;
+    };
+    if (!await isCurrent()) throw RUN_ENDED;
+    try {
+      await options.onFailed({ id, run: reserved.run, job: view(reserved), failure: structuredClone(reserved.failure) }, { signal: rt.controller.signal, isCurrent });
+      if (!await isCurrent()) throw RUN_ENDED;
+      await write(rt, (fresh) => ({ failureNotification: { ...fresh.failureNotification, deliveredAt: iso(nowMs()) }, active: hasUnrecorded(fresh.steps), leaseOwner: void 0, leaseUntil: void 0 }));
+    } catch {
+      throwIfEnded(rt);
+      logError(`\u041A\u0430\u0440\u0442\u0438\u043D\u043A\u0438: \u0441\u043E\u0431\u044B\u0442\u0438\u0435 ${id} \u043D\u0435 \u043F\u0440\u0438\u043D\u044F\u0442\u043E, \u043F\u043E\u0432\u0442\u043E\u0440\u0438\u0442\u0441\u044F \u0431\u0435\u0437 \u0433\u0435\u043D\u0435\u0440\u0430\u0446\u0438\u0438`);
+      await write(rt, () => ({ active: true, leaseOwner: void 0, leaseUntil: void 0 }));
     }
   }
+  function launchFailure(record) {
+    const rt = newJobRuntime(record.ownerId, record.chatId, 0);
+    rt.record = record;
+    runtimes.add(rt);
+    byJob.set(record.id, rt);
+    keepLease(rt);
+    rt.done = (async () => {
+      try {
+        await recordCharges(rt);
+        await notifyFailure(rt);
+      } catch (error) {
+        if (error !== RUN_ENDED && !rt.controller.signal.aborted) logError(`\u041A\u0430\u0440\u0442\u0438\u043D\u043A\u0438: \u0441\u043E\u0431\u044B\u0442\u0438\u0435 ${record.id} \u043E\u0441\u0442\u0430\u043D\u043E\u0432\u0438\u043B\u043E\u0441\u044C, \u0431\u0443\u0434\u0435\u0442 \u043F\u043E\u0434\u043E\u0431\u0440\u0430\u043D\u043E \u043F\u043E\u0441\u043B\u0435 \u0430\u0440\u0435\u043D\u0434\u044B`);
+      } finally {
+        finishRuntime(rt);
+      }
+    })();
+  }
   function keepLease(rt) {
     const renew = () => {
       rt.leaseTimer = setTimeout(async () => {
@@ -1199,10 +1272,13 @@ function createImageQueue(options) {
     };
     renew();
   }
-  function newRuntime(ownerId, chatId, setup, reservedUsd) {
+  function newJobRuntime(ownerId, chatId, reservedUsd) {
     let resolve;
     const promise = new Promise((done) => resolve = done);
-    return { ownerId, chatId, setup, reservedUsd, controller: new AbortController(), chain: Promise.resolve(), sent: { promise, resolve }, ended: false, checkNow: false, reorderOnDeliveryFailure: false };
+    return { ownerId, chatId, reservedUsd, controller: new AbortController(), chain: Promise.resolve(), sent: { promise, resolve }, ended: false, checkNow: false, reorderOnDeliveryFailure: false };
+  }
+  function newRuntime(ownerId, chatId, setup, reservedUsd) {
+    return { ...newJobRuntime(ownerId, chatId, reservedUsd), setup };
   }
   function launch(rt, record) {
     rt.record = record;
@@ -1229,9 +1305,13 @@ function createImageQueue(options) {
   }
   async function claim(record) {
     if (stopped || byJob.has(record.id)) return;
+    if (!isOpen(record) && hasPendingImageFailure(record) && !hasUnrecorded(record.steps) && Date.parse(record.failureNotification.nextAttemptAt) > nowMs()) return;
     const claimed = await store.update(record.id, { leaseOwner: me, leaseUntil: iso(nowMs() + leaseMs), updatedAt: iso(nowMs()) }, record.version);
     if (!claimed) return;
-    if (!isOpen(claimed)) return settleClosed(claimed);
+    if (!isOpen(claimed)) {
+      if (options.onFailed && hasPendingImageFailure(claimed) && Date.parse(claimed.failureNotification.nextAttemptAt) <= nowMs()) return launchFailure(claimed);
+      return settleClosed(claimed);
+    }
     const setup = setupOf(claimed);
     if (!setup) return closeJob(claimed, "failed", `\u041D\u0430\u0441\u0442\u0440\u043E\u0439\u043A\u0430 \u043A\u0430\u0440\u0442\u0438\u043D\u043E\u043A \u0441\u043C\u0435\u043D\u0438\u043B\u0430\u0441\u044C: \u043D\u0430\u0437\u043D\u0430\u0447\u0435\u043D\u0438\u044F \xAB${claimed.purpose}\xBB \u0432 \u043D\u0435\u0439 \u0431\u043E\u043B\u044C\u0448\u0435 \u043D\u0435\u0442`);
     const rt = newRuntime(claimed.ownerId, claimed.chatId, setup, 0);
@@ -1350,11 +1430,13 @@ function createImageQueue(options) {
       throw error;
     }
   }
-  function retryJob(ownerId, key) {
+  const matchesRetry = (record, guard) => guard.expectedJobId === void 0 || record?.id === guard.expectedJobId && record.run === guard.expectedRun;
+  function retryJob(ownerId, key, guard) {
     return underKey(ownerId, key, async () => {
       const latest = (await store.list({ ownerId, key })).at(-1);
-      if (!latest) return null;
+      if (!latest || !matchesRetry(latest, guard)) return null;
       if (latest.status !== "failed") return view(latest);
+      if (failureOf(latest)?.kind === "content") return refused("\u041F\u0440\u043E\u0432\u0430\u0439\u0434\u0435\u0440 \u043E\u0442\u043A\u043B\u043E\u043D\u0438\u043B \u0441\u043E\u0434\u0435\u0440\u0436\u0430\u043D\u0438\u0435: \u0442\u043E\u0442 \u0436\u0435 \u0437\u0430\u043F\u0440\u043E\u0441 \u043D\u0435 \u043F\u043E\u0432\u0442\u043E\u0440\u044F\u0435\u0442\u0441\u044F; \u0432\u044B\u0431\u0435\u0440\u0438\u0442\u0435 \u0440\u0430\u0437\u0440\u0435\u0448\u0451\u043D\u043D\u0443\u044E \u0430\u043B\u044C\u0442\u0435\u0440\u043D\u0430\u0442\u0438\u0432\u0443");
       const setup = setupOf(latest);
       if (!setup) return refused(`\u043D\u0430\u0437\u043D\u0430\u0447\u0435\u043D\u0438\u044F \xAB${latest.purpose}\xBB \u0432 \u043D\u0430\u0441\u0442\u0440\u043E\u0439\u043A\u0435 \u0431\u043E\u043B\u044C\u0448\u0435 \u043D\u0435\u0442`);
       const remaining = latest.steps.reduce((total, step, index) => step.urls ? total : total + (step.task?.priceUsd ?? setup.steps[index].priceUsd), 0);
@@ -1368,6 +1450,7 @@ function createImageQueue(options) {
         let current = latest;
         let revived = null;
         while (current?.status === "failed") {
+          if (!matchesRetry(current, guard)) return null;
           if (stopped) return refused(QUEUE_STOPPED);
           const at = nowMs();
           revived = await reviveOrTaken(
@@ -1378,6 +1461,8 @@ function createImageQueue(options) {
               attempt: 1,
               attemptDeadline: iso(at + attemptTimeoutMs),
               reason: void 0,
+              failure: void 0,
+              failureNotification: void 0,
               lateUntil: void 0,
               deliveryFailed: void 0,
               finishedAt: void 0,
@@ -1389,13 +1474,14 @@ function createImageQueue(options) {
             current.version
           );
           if (revived === "taken") {
+            if (guard.expectedJobId !== void 0) return null;
             const newest = (await store.list({ ownerId, key })).at(-1);
             return newest ? view(newest) : null;
           }
           if (revived) break;
           current = await store.get(latest.id);
         }
-        if (!revived) return current ? view(current) : null;
+        if (!revived) return current && matchesRetry(current, guard) ? view(current) : null;
         if (stopped) {
           await updateJob(revived, (record) => record.leaseOwner === me ? { leaseOwner: void 0, leaseUntil: void 0 } : null, (record) => record.leaseOwner === me);
           return refused(QUEUE_STOPPED);
@@ -1441,12 +1527,14 @@ function createImageQueue(options) {
   void start();
   return {
     order,
-    retry({ ownerId, key }) {
+    retry({ ownerId, key, expectedJobId, expectedRun }) {
+      const guarded = expectedJobId !== void 0 || expectedRun !== void 0;
+      if (guarded && (typeof expectedJobId !== "string" || expectedJobId.length === 0 || !Number.isInteger(expectedRun) || expectedRun < 1)) return Promise.reject(new TypeError("\u041A\u0430\u0440\u0442\u0438\u043D\u043A\u0438: expectedJobId \u0438 expectedRun \u043F\u0435\u0440\u0435\u0434\u0430\u044E\u0442\u0441\u044F \u0432\u043C\u0435\u0441\u0442\u0435; run \u2014 \u0446\u0435\u043B\u043E\u0435 \u0447\u0438\u0441\u043B\u043E \u043E\u0442 1"));
       if (stopped) return Promise.reject(new Error("\u041A\u0430\u0440\u0442\u0438\u043D\u043A\u0438: \u043E\u0447\u0435\u0440\u0435\u0434\u044C \u043E\u0441\u0442\u0430\u043D\u043E\u0432\u043B\u0435\u043D\u0430 (stop) \u2014 \u043F\u043E\u0432\u0442\u043E\u0440 \u043D\u0435 \u043F\u0440\u0438\u043D\u0438\u043C\u0430\u0435\u0442\u0441\u044F"));
-      const flight = JSON.stringify([ownerId, key]);
+      const flight = JSON.stringify([ownerId, key, expectedJobId, expectedRun]);
       const running = retries.get(flight);
       if (running) return running;
-      const promise = retryJob(ownerId, key).finally(() => retries.delete(flight));
+      const promise = retryJob(ownerId, key, { expectedJobId, expectedRun }).finally(() => retries.delete(flight));
       retries.set(flight, promise);
       return promise;
     },
@@ -1533,7 +1621,7 @@ function checkNumbers(options) {
   const limit = options.maxImagesPerTurn;
   if (limit !== void 0 && !(Number.isInteger(limit) && limit >= 1)) throw new Error("\u041A\u0430\u0440\u0442\u0438\u043D\u043A\u0438: maxImagesPerTurn \u2014 \u0446\u0435\u043B\u043E\u0435 \u0447\u0438\u0441\u043B\u043E \u043E\u0442 1");
   if (options.mode !== "background") {
-    const backgroundOnly = ["jobStore", "pollIntervalMs", "attempts", "onQueued"].filter((name) => options[name] !== void 0);
+    const backgroundOnly = ["jobStore", "pollIntervalMs", "attempts", "onQueued", "onFailed", "failureRetryMs"].filter((name) => options[name] !== void 0);
     if (backgroundOnly.length > 0) throw new Error(`\u041A\u0430\u0440\u0442\u0438\u043D\u043A\u0438: ${backgroundOnly.join(", ")} \u2014 \u0442\u043E\u043B\u044C\u043A\u043E \u0434\u043B\u044F mode: "background"`);
   }
 }
@@ -1752,6 +1840,8 @@ function createImageGeneration(options) {
     ...options.purposes && { purposes: options.purposes },
     ...options.fetch && { fetch: options.fetch },
     deliver: receiveQueued,
+    ...options.onFailed !== void 0 && { onFailed: options.onFailed },
+    ...options.failureRetryMs !== void 0 && { failureRetryMs: options.failureRetryMs },
     ...options.jobStore && { store: options.jobStore },
     ...options.spending && { spending: options.spending },
     ...options.pollIntervalMs !== void 0 && { pollIntervalMs: options.pollIntervalMs },