@xmanrui/dsh-im 4.21.2 → 4.23.0

package/src/channels/shared/image-input.mjs

@@ -0,0 +1,124 @@
+import { readFile } from 'node:fs/promises';
+import { basename, join } from 'node:path';
+
+import {
+  detectedImageMediaType, ImagePromptError, imageStorageName, loadedImage, imageDownloadLimitMessage,
+} from './image-prompt.mjs';
+import { t } from './i18n.mjs';
+
+const MAX_INPUT_PIXELS = 64_000_000;
+const COMPRESSION_STEPS = [[2048, 80], [1536, 70], [1024, 60]];
+let sharpPromise;
+async function loadSharp() {
+  sharpPromise ??= import('sharp').then((module) => module.default).catch(() => null);
+  return sharpPromise;
+}
+
+export const IMAGE_ORIGINALS_PROMPT = '图片原文件已保存到工作区，文件清单中的图片序号与本条消息一致。图片内容块可能经过缩放或压缩；需要查看细节时可用工具读取原文件。';
+export const IMAGE_INPUT_FALLBACK_PROMPT = '部分图片未直接提供给模型，图片序号：{indexes}。原文件已保存，请使用可用工具读取或分析；不要假设自己已看到这些图片。';
+export const IMAGE_HOST_LIMIT_FALLBACK_PROMPT = '宿主拒绝了本次图片输入。原图已保存到工作区，请根据文件清单使用可用工具读取或分析，不要假设自己已看到图片。';
+
+/** Keep provider loaders intact; stage and validate each original once. */
+export function inboundImagesAsFiles(images, limits) {
+  if (images.length > limits.maxImages) {
+    throw new ImagePromptError('too-many-images', 'Too many inbound images',
+      t('一次最多只能处理 {maxImages} 张图片。', { maxImages: limits.maxImages }));
+  }
+  const tooLarge = () => new ImagePromptError('image-too-large',
+    `Image exceeds ${limits.maxDownloadBytes} bytes`, imageDownloadLimitMessage(limits.maxDownloadBytes));
+  // Check declared sizes before any download starts, including later images.
+  for (const source of images) {
+    if (Number.isFinite(source?.size) && source.size > limits.maxDownloadBytes) throw tooLarge();
+  }
+  return images.map((source, index) => ({
+    async load({ signal } = {}) {
+      signal?.throwIfAborted();
+      let value;
+      try {
+        value = source.data === undefined
+          ? await source.load?.({ signal, maxBytes: limits.maxDownloadBytes }) : source.data;
+      } catch (error) {
+        if (signal?.aborted) throw signal.reason ?? error;
+        if (error?.code === 'image-too-large') throw tooLarge();
+        if (error instanceof ImagePromptError || ['AbortError', 'TimeoutError'].includes(error?.name)) throw error;
+        throw new ImagePromptError('image-download-failed', 'Unable to download image',
+          t('图片下载失败，请重新发送后再试。'), { cause: error });
+      }
+      signal?.throwIfAborted();
+      const loaded = loadedImage(value);
+      if (!loaded?.data.length) throw new ImagePromptError('invalid-image-data', 'Empty image',
+        t('未能读取图片内容，请重新发送。'));
+      if (loaded.data.length > limits.maxDownloadBytes) throw tooLarge();
+      const mediaType = detectedImageMediaType(loaded.data);
+      if (!mediaType) throw new ImagePromptError('unsupported-image-type', 'Unsupported image',
+        t('暂不支持该图片格式，请发送 JPEG、PNG、WebP 或 GIF 图片。'));
+      return { data: loaded.data, mediaType,
+        name: `${index + 1}-${imageStorageName(loaded.name ?? source.name, mediaType, index)}` };
+    },
+  }));
+}
+
+/** A missing optional codec still permits small native images and file delivery. */
+export async function modelImageFromOriginal(data, mediaType, maxBytes, {
+  signal, sharpLoader = loadSharp,
+} = {}) {
+  signal?.throwIfAborted();
+  if (maxBytes <= 0) return null;
+  const sharp = await sharpLoader();
+  signal?.throwIfAborted();
+  if (!sharp) return data.length <= maxBytes ? { data, mediaType } : null;
+  try {
+    const metadata = await sharp(data, { limitInputPixels: MAX_INPUT_PIXELS }).metadata();
+    signal?.throwIfAborted();
+    if (metadata.width * metadata.height > MAX_INPUT_PIXELS) return null;
+    if ((metadata.pages ?? 1) > 1 || mediaType === 'image/gif') {
+      return data.length <= maxBytes ? { data, mediaType } : null;
+    }
+    if (data.length <= maxBytes && metadata.width <= 2048 && metadata.height <= 2048) {
+      return { data, mediaType };
+    }
+    for (const [side, quality] of COMPRESSION_STEPS) {
+      signal?.throwIfAborted();
+      const operation = sharp(data, { limitInputPixels: MAX_INPUT_PIXELS }).rotate()
+        .resize({ width: side, height: side, fit: 'inside', withoutEnlargement: true });
+      const output = metadata.hasAlpha
+        ? await operation.png({ compressionLevel: 9 }).toBuffer()
+        : await operation.jpeg({ quality }).toBuffer();
+      signal?.throwIfAborted();
+      if (output.length <= maxBytes) return { data: output,
+        mediaType: metadata.hasAlpha ? 'image/png' : 'image/jpeg' };
+    }
+  } catch (error) {
+    signal?.throwIfAborted();
+    // A codec failure cannot discard an original already saved for tools.
+  }
+  return null;
+}
+
+export async function imageContentFromStaged(staged, limits, { signal, sharpLoader } = {}) {
+  const content = [];
+  const fileOnly = [];
+  let remaining = limits.maxTotalImageBytes;
+  for (const [index, file] of staged.files.entries()) {
+    signal?.throwIfAborted();
+    // Staging owns both the directory and basename; never read a user-supplied path.
+    const data = await readFile(join(staged.directory, basename(file.path)), { signal });
+    const result = await modelImageFromOriginal(data, file.mediaType,
+      Math.min(limits.maxImageBytes, remaining), { signal, sharpLoader });
+    if (!result) { fileOnly.push(index + 1); continue; }
+    remaining -= result.data.length;
+    content.push({ type: 'text', text: t('图片 {index}（原图文件：{name}）', { index: index + 1, name: file.name }) },
+      { type: 'image', mediaType: result.mediaType, data: result.data.toString('base64'),
+        name: imageStorageName(file.name.replace(/\.[^.]+$/, ''), result.mediaType, index) });
+  }
+  content.push({ type: 'text', text: t(IMAGE_ORIGINALS_PROMPT) });
+  if (fileOnly.length) content.push({ type: 'text',
+    text: t(IMAGE_INPUT_FALLBACK_PROMPT, { indexes: fileOnly.join(', ') }) });
+  return content;
+}
+
+export function isImageAdmissionRejection(error) {
+  return ['attachment-error', 'session/attachment-invalid'].includes(error?.code)
+    && ['MODEL_DOES_NOT_SUPPORT_IMAGES', 'IMAGE_TOO_LARGE', 'IMAGES_TOO_LARGE',
+      'IMAGE_TOO_MANY_PIXELS', 'TOO_MANY_IMAGES'].includes(error?.details?.reason);
+}

package/src/channels/shared/image-prompt.mjs

@@ -12,6 +12,11 @@ export const DEFAULT_IMAGE_PROMPT = '请分析这张图片。';
  */
 export const IMAGE_FILE_FALLBACK_PROMPT = '当前会话模型不支持直接接收图片输入。用户发送的图片已作为文件保存到工作区（见下方文件清单）。请使用可用工具分析这些图片文件后回答，例如 run_code 或 pwsh 读取字节、解析元数据、调用图像处理或 OCR 库；不要假设自己能直接看到图片内容。';
 
+export function imageDownloadLimitMessage(maxBytes) {
+  return t('图片超过原图接收上限 {maxMb} MB，请压缩后重试，或在通用设置的附件中调整上限。',
+    { maxMb: Math.round(maxBytes / (1024 * 1024) * 100) / 100 });
+}
+
 export class ImagePromptError extends Error {
   constructor(code, message, userMessage, options = {}) {
     super(message, options);
@@ -92,7 +97,7 @@ export async function fetchImageBuffer(url, {
     throw new ImagePromptError(
       'image-too-large',
       `Image response declares ${declaredLength} bytes; the limit is ${maxBytes}`,
-      t('图片超过 5 MB，请压缩后重试。'),
+      imageDownloadLimitMessage(maxBytes),
     );
   }
 
@@ -107,7 +112,7 @@ export async function fetchImageBuffer(url, {
         throw new ImagePromptError(
           'image-too-large',
           `Image response exceeded ${maxBytes} bytes`,
-          t('图片超过 5 MB，请压缩后重试。'),
+          imageDownloadLimitMessage(maxBytes),
         );
       }
       chunks.push(data);
@@ -120,7 +125,7 @@ export async function fetchImageBuffer(url, {
     throw new ImagePromptError(
       'image-too-large',
       `Image response contains ${data.length} bytes; the limit is ${maxBytes}`,
-      t('图片超过 5 MB，请压缩后重试。'),
+      imageDownloadLimitMessage(maxBytes),
     );
   }
   return data;
@@ -146,7 +151,7 @@ function safeName(value) {
   return name || undefined;
 }
 
-function detectedImageMediaType(data) {
+export function detectedImageMediaType(data) {
   if (data.length >= 8
     && data[0] === 0x89 && data[1] === 0x50 && data[2] === 0x4e && data[3] === 0x47
     && data[4] === 0x0d && data[5] === 0x0a && data[6] === 0x1a && data[7] === 0x0a) {
@@ -167,7 +172,7 @@ function detectedImageMediaType(data) {
   return null;
 }
 
-function loadedImage(value) {
+export function loadedImage(value) {
   if (Buffer.isBuffer(value) || value instanceof Uint8Array) {
     return { data: Buffer.from(value) };
   }
@@ -191,12 +196,13 @@ export function hasInboundPrompt(message) {
 
 export async function promptContentForMessage(message, {
   signal,
+  deferImages = false,
   maxImageBytes = DEFAULT_MAX_IMAGE_BYTES,
   maxImages = DEFAULT_MAX_IMAGES,
   maxTotalImageBytes = DEFAULT_MAX_TOTAL_IMAGE_BYTES,
 } = {}) {
   const sources = imageSources(message);
-  if (sources.length > maxImages) {
+  if (!deferImages && sources.length > maxImages) {
     throw new ImagePromptError(
       'too-many-images',
       `Image message contains ${sources.length} images; the limit is ${maxImages}`,
@@ -210,6 +216,8 @@ export async function promptContentForMessage(message, {
   if (text) content.push({ type: 'text', text });
   else if (sources.length > 0) content.push({ type: 'text', text: t(DEFAULT_IMAGE_PROMPT) });
 
+  if (deferImages) return content;
+
   for (const [index, source] of sources.entries()) {
     signal?.throwIfAborted();
     if (Number.isFinite(source?.size) && source.size > maxImageBytes) {
@@ -315,7 +323,7 @@ const IMAGE_FILE_EXTENSIONS = new Map([
 
 const IMAGE_EXTENSION_PATTERN = /\.(?:png|jpe?g|gif|webp)$/i;
 
-function imageStorageName(name, mediaType, index) {
+export function imageStorageName(name, mediaType, index) {
   const extension = IMAGE_FILE_EXTENSIONS.get(mediaType) ?? '.img';
   const cleaned = safeName(name);
   if (cleaned && IMAGE_EXTENSION_PATTERN.test(cleaned)) return cleaned;

package/src/channels/shared/message-failure.mjs

@@ -1,3 +1,5 @@
+import { extractConnectionEvidence } from './connection-error.mjs';
+import { diagnosticFields } from './diagnostic-details.mjs';
 import { randomUUID } from 'node:crypto';
 
 import { t } from './i18n.mjs';
@@ -189,7 +191,9 @@ export function classifyMessageFailure(error, {
     && userMessage.trim()
     ? 'INPUT_INVALID'
     : classifiedCode;
+  const details = extractConnectionEvidence(error).details;
   return Object.freeze({
+    ...(code === 'INTERNAL_UNKNOWN' || details.reason !== 'unknown' || details.httpStatus || details.providerCode ? { details } : {}),
     code,
     reason: safeReason ?? safeFailureReason(error?.code) ?? code,
     message: typeof userMessage === 'string' && userMessage.trim()
@@ -204,9 +208,16 @@ export function messageFailureText(failure) {
   return `${failure.message}\n\n${t('错误码：{code}；参考号：{referenceId}', failure)}`;
 }
 
+export function messageFailureDiagnostic(error, failure) {
+  const evidence = extractConnectionEvidence(error);
+  return { code: failure.code, reason: failure.reason, referenceId: failure.referenceId,
+    details: evidence.details, errors: evidence.errors };
+}
+
 export function setLastMessageFailure(status, error, options) {
   const failure = classifyMessageFailure(error, options);
   status.lastMessageError = failure;
+  status.lastError = failure.message;
   return failure;
 }
 
@@ -241,5 +252,6 @@ export function publicMessageFailure(value) {
     message: value.message.slice(0, 500),
     referenceId: value.referenceId.slice(0, 40),
     at: value.at,
+    ...(value.details ? diagnosticFields(value) : {}),
   };
 }

package/src/channels/shared/semantic/reply-reference.mjs

@@ -138,11 +138,11 @@ export function hasReplyReference(message) {
   return objectReference(message?.replyTo);
 }
 
-export async function promptContentForInboundMessage(message, { signal } = {}) {
+export async function promptContentForInboundMessage(message, { signal, deferImages = false } = {}) {
   if (!hasReplyReference(message)) {
-    return promptContentForMessage(message, { signal });
+    return promptContentForMessage(message, { signal, deferImages });
   }
   const reference = normalizeReference(await resolveReference(message.replyTo, signal));
-  const currentContent = await promptContentForMessage(message, { signal });
+  const currentContent = await promptContentForMessage(message, { signal, deferImages });
   return [{ type: 'text', text: replyBlock(reference) }, ...currentContent];
 }

package/src/channels/shared/text-harness-bridge.mjs

@@ -64,6 +64,7 @@ import {
   providerMessageIdsFor,
 } from './semantic/delivery.mjs';
 import {
+  messageFailureDiagnostic,
   channelDeliveryFailure,
   clearLastMessageFailure,
   messageFailureText,
@@ -74,6 +75,15 @@ import { beginStatusReaction } from './status-reaction.mjs';
 const INTERACTION_RESOLVED_TEXT = '这个问题已在其他客户端处理，无需再次回答。';
 const FILE_ONLY_COMPLETION_TEXT = '任务已完成。';
 
+/**
+ * How many times one inbound message may be re-delivered after a failed turn.
+ *
+ * A failure releases the message's `markSeen` so the next poll retries it, which
+ * would otherwise loop forever on a mail that always fails. After this many
+ * attempts the mark stays and the message is treated as handled.
+ */
+const MAX_DELIVERY_ATTEMPTS = 3;
+
 function cleanText(value) {
   return typeof value === 'string' ? value.trim() : '';
 }
@@ -83,7 +93,7 @@ function canClaimInteractionReply(message, pending, senderId) {
     && (message.kind !== 'group' || message.addressed === true)
     && !hasInboundImages(message)
     && !hasInboundFiles(message)
-    && Boolean(cleanText(message.content));
+    && Boolean(controlTextOf(message));
 }
 
 function artifactFailureText(fileName, error, descriptor) {
@@ -138,6 +148,18 @@ export function createTextBridgeStatus() {
   };
 }
 
+/**
+ * The text a channel's control commands and approval decisions are parsed from.
+ *
+ * A channel may decorate `content` for the model — the email channel prepends
+ * the mail headers so a subject line reaches the model — while `controlText`
+ * keeps the undecorated body. Without this a decorated message made `/help`
+ * and "批准" unrecognisable.
+ */
+function controlTextOf(message) {
+  return cleanText(message?.controlText ?? message?.content);
+}
+
 export class TextHarnessBridge {
   #descriptor;
   #bot;
@@ -161,6 +183,10 @@ export class TextHarnessBridge {
   #approvals;
   #batches = new BatchInputManager();
   #interactionCard;
+  // Delivery attempts per message id. A failed turn releases its `markSeen` so
+  // the next poll can retry, and this counter is what keeps that from becoming
+  // an infinite loop on a mail that always fails.
+  #deliveryAttempts = new Map();
 
   constructor({
     descriptor,
@@ -229,7 +255,11 @@ export class TextHarnessBridge {
       const decision = accessDecision ?? evaluateInboundAccess(this.#accessPolicy, {
         conversationType: kind,
         senderIds: [senderId, cleanText(normalized.senderAlternateId)].filter(Boolean),
-        text: normalized.content,
+        // The same control text the command runner below parses. A channel may
+        // decorate `content` for the model (email prepends the mail headers), and
+        // a decorated body never looks like a command — so reading `content` here
+        // recognized no command at all and skipped the command-permission gate.
+        text: controlTextOf(normalized),
         hasImages,
         hasFiles,
       });
@@ -282,7 +312,7 @@ export class TextHarnessBridge {
 
     const key = `${normalized.kind}:${normalized.conversationId}`;
     const pending = this.#pendingInteractions.get(key);
-    const text = cleanText(normalized.content);
+    const text = controlTextOf(normalized);
     const batchCommand = isBatchInputCommand(text);
     if (batchCommand && normalized.kind === 'group' && normalized.addressed === true) {
       return this.#finishLocalMessage(
@@ -362,7 +392,12 @@ export class TextHarnessBridge {
       key,
       actor: senderId,
       messageId,
-      text: hasInboundImages(normalized) || hasInboundFiles(normalized) ? '' : normalized.content,
+      // Approval decisions are parsed from the control text, exactly like the
+      // commands above: a decorated `content` hid "批准" behind the mail headers,
+      // so a plain-body approval never claimed its pending request.
+      text: hasInboundImages(normalized) || hasInboundFiles(normalized)
+        ? ''
+        : controlTextOf(normalized),
       addressed: normalized.kind !== 'group' || normalized.addressed === true,
       hasPendingQuestion: Boolean(pending),
       questionCompletion: pending?.submitting || pending?.claimedReplyMessageId
@@ -441,7 +476,7 @@ export class TextHarnessBridge {
       const failure = setLastMessageFailure(this.#status, error);
       this.#logger.error?.(
         `[dsh-im:${this.#descriptor.key}] failed to process a batch input message [${failure.referenceId}]:`,
-        error,
+        messageFailureDiagnostic(error, failure),
       );
     }).finally(() => {
       this.#acceptedMessageIds.delete(messageId);
@@ -500,7 +535,7 @@ export class TextHarnessBridge {
     const target = message.replyTarget;
     try {
       const result = await runner(
-        cleanText(message.content),
+        controlTextOf(message),
         this.#harness,
         this.#state,
         key,
@@ -550,7 +585,7 @@ export class TextHarnessBridge {
       const failure = setLastMessageFailure(this.#status, error);
       this.#logger.error?.(
         `[dsh-im:${this.#descriptor.key}] failed to process a command [${failure.referenceId}]:`,
-        error,
+        messageFailureDiagnostic(error, failure),
       );
       await this.#bot.sendText(target, messageFailureText(failure)).catch(() => undefined);
     }
@@ -615,7 +650,7 @@ export class TextHarnessBridge {
     }
 
     const target = message.replyTarget;
-    const text = cleanText(message.content);
+    const text = controlTextOf(message);
     const batchSubmission = message.batchSubmission;
     let stream = null;
     let semanticStream = false;
@@ -710,7 +745,7 @@ export class TextHarnessBridge {
         }
       }
       let content = hasImages || hasReply
-        ? await promptContentForInboundMessage(message, { signal: this.#signal })
+        ? await promptContentForInboundMessage(message, { signal: this.#signal, deferImages: true })
         : undefined;
       const snapshot = this.#acceptedMessageIds.get(messageId);
       let contextEnhanced = false;
@@ -790,6 +825,7 @@ export class TextHarnessBridge {
           },
           onInteractionResolved: (resolution) => this.#handleInteractionResolved(resolution),
           files: message.files,
+          images: message.images,
         },
       });
       stopKeepalive();
@@ -877,6 +913,8 @@ export class TextHarnessBridge {
           clearLastMessageFailure(this.#status);
         }
       }
+      // The turn produced an answer, so its retry budget is spent and reset.
+      this.#deliveryAttempts.delete(messageId);
       return delivery.receipt;
     } catch (error) {
       stopKeepalive();
@@ -934,7 +972,7 @@ export class TextHarnessBridge {
         : messageFailureText(failure);
       this.#logger.error?.(
         `[dsh-im:${this.#descriptor.key}] failed to process a message [${failure.referenceId}]:`,
-        error,
+        messageFailureDiagnostic(error, failure),
       );
       if (await presentStreamFailure(failureText)) {
         return error.deliveryReceipt;
@@ -948,6 +986,10 @@ export class TextHarnessBridge {
           sendError,
         );
       }
+      // The turn produced no answer, so this id is "attempted", not "handled".
+      // Releasing the mark lets the next poll retry it; `alreadyRecorded` means
+      // an outer caller owns the mark, so it is left alone there.
+      if (!alreadyRecorded) await this.#releaseFailedDelivery(messageId);
       return error.deliveryReceipt;
     } finally {
       stopKeepalive();
@@ -958,6 +1000,43 @@ export class TextHarnessBridge {
     }
   }
 
+  /**
+   * Release the `markSeen` of a turn that failed without delivering anything,
+   * so the next poll can pick the message up again.
+   *
+   * Bounded: a message is retried at most `MAX_DELIVERY_ATTEMPTS` times, after
+   * which the mark stays and the id becomes a tombstone. That keeps a mail that
+   * always fails from being re-executed forever, while a transient Harness
+   * outage no longer drops it permanently.
+   *
+   * Returns true when the message was released and will be retried.
+   */
+  async #releaseFailedDelivery(messageId) {
+    const attempts = (this.#deliveryAttempts.get(messageId) ?? 0) + 1;
+    if (attempts >= MAX_DELIVERY_ATTEMPTS) {
+      // Give up: keep the mark so the id is never replayed, and stop tracking
+      // it. The sender has already been told the turn failed.
+      this.#deliveryAttempts.delete(messageId);
+      this.#logger.warn?.(
+        `[dsh-im:${this.#descriptor.key}] giving up on a message after `
+        + `${attempts} failed delivery attempts; it will not be retried.`,
+      );
+      return false;
+    }
+    this.#deliveryAttempts.set(messageId, attempts);
+    if (typeof this.#state?.unmarkSeen !== 'function') return false;
+    try {
+      await this.#state.unmarkSeen(messageId);
+      return true;
+    } catch (error) {
+      this.#logger.warn?.(
+        `[dsh-im:${this.#descriptor.key}] unable to release a failed message for retry:`,
+        error,
+      );
+      return false;
+    }
+  }
+
   /**
    * Advance the pending interaction with one answer.
    *
@@ -1000,7 +1079,7 @@ export class TextHarnessBridge {
     }
 
     const target = message.replyTarget;
-    const text = cleanText(message.content);
+    const text = controlTextOf(message);
     if (!text || hasInboundImages(message) || hasInboundFiles(message)) {
       try {
         await this.#bot.sendText(target, t('请用文字回答当前问题。'));