@qjp88995/provider-client 0.2.0 → 0.3.0

package/src/media/zhipu.adapter.ts

@@ -0,0 +1,510 @@
+import type { VendorFetch } from '../vendors/index.js';
+import {
+  bearer,
+  errorMessage,
+  joinUrl,
+  throwIfCenterConfigError,
+} from './http.js';
+import {
+  type AudioOutcome,
+  type AudioRequest,
+  type CacheKeyOpts,
+  type FileOutcome,
+  type FileRequest,
+  type MediaAdapter,
+  type MediaCapability,
+  type MediaConnection,
+  type MediaInput,
+  type MediaSettings,
+  MediaUpstreamError,
+  type UploadLimitHint,
+  type ZhipuFileToolType,
+} from './types.js';
+import { parseParserPackage } from './zhipu-parser-package.js';
+
+/**
+ * 智谱 BigModel 的文件解析与语音转写（agent-base 里叫 `bigmodel`），借智谱 LLM 的 key。根取自这家的 `media` 端点
+ * （`https://open.bigmodel.cn/api/paas/v4`），下挂 `/audio/transcriptions`、`/files/parser/…`。
+ */
+
+type Settings = MediaSettings<'zhipu'>;
+
+const DEFAULT_AUDIO_MODEL = 'glm-asr-2512';
+const DEFAULT_TOOL_TYPE: ZhipuFileToolType = 'lite';
+const DEFAULT_MAX_WAIT_MS = 60_000;
+
+function resolved(settings: Settings) {
+  return {
+    audioModel: settings.audio?.model ?? DEFAULT_AUDIO_MODEL,
+    toolType: settings.file?.toolType ?? DEFAULT_TOOL_TYPE,
+    maxWaitMs: settings.file?.defaultMaxWaitMs ?? DEFAULT_MAX_WAIT_MS,
+  };
+}
+
+/**
+ * MIME → 上传时给厂商的文件扩展名。别处若另有一张「收得下哪些格式」的表，它必须是本表的子集——两张表漂了的表现是
+ * 「上游收得下的格式，适配器造不出正确的文件名」，是静默的错误。
+ */
+export const ZHIPU_MIME_TO_EXT: Readonly<Record<string, string>> = {
+  'audio/mpeg': 'mp3',
+  'audio/mp3': 'mp3',
+  'audio/wav': 'wav',
+  'audio/wave': 'wav',
+  'audio/x-wav': 'wav',
+  'application/pdf': 'pdf',
+  'application/msword': 'doc',
+  'application/vnd.openxmlformats-officedocument.wordprocessingml.document':
+    'docx',
+  'application/vnd.ms-excel': 'xls',
+  'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': 'xlsx',
+  'application/vnd.ms-powerpoint': 'ppt',
+  'application/vnd.openxmlformats-officedocument.presentationml.presentation':
+    'pptx',
+  'text/plain': 'txt',
+  'text/markdown': 'md',
+  'text/csv': 'csv',
+};
+
+function extOf(mime: string): string | undefined {
+  return ZHIPU_MIME_TO_EXT[mime.toLowerCase()];
+}
+
+/**
+ * 厂商文档写的是「MB」且未注明进制，按十进制（1 MB = 1000²）取下界——若厂商实际按 2 的幂算，我们只是保守了 4.8%，
+ * 反过来就会把一份超限 4.8% 的文件白传一趟。
+ */
+const MB = 1000 * 1000;
+
+/**
+ * 各上限**逐字来自厂商文档**，改前先核对（数字过时的表现是静默的：要么白传被拒、要么把能解析的文件挡在门外）：
+ * - 文件解析 https://docs.bigmodel.cn/cn/guide/tools/file-parser.md ：Lite ≤50M；Expert ≤100M（仅 PDF）；
+ *   Prime 按格式——PDF/DOC/DOCX/PPT ≤100MB、XLS/XLSX/CSV ≤10MB、图片 ≤20MB（图片不该送到这里：看图归 LLM）。
+ *   txt/md 等 Prime 支持列表里有、尺寸表里没有的格式，取 Lite 同款 50MB 当下界。
+ * - 语音转写 https://docs.bigmodel.cn/api-reference/模型-api/语音转文本.md ：单文件 ≤25MB（还有 ≤30 秒时长限制，
+ *   字节层看不出时长，超时长交给厂商响亮报错）。
+ */
+export const ZHIPU_AUDIO_MAX_BYTES = 25 * MB;
+const FILE_PRIME_SPREADSHEET_MAX = 10 * MB;
+const FILE_PRIME_DOCUMENT_MAX = 100 * MB;
+const FILE_LITE_MAX = 50 * MB;
+const FILE_EXPERT_MAX = 100 * MB;
+
+const PRIME_SPREADSHEET_EXTS = new Set(['xls', 'xlsx', 'csv']);
+const PRIME_DOCUMENT_EXTS = new Set(['pdf', 'doc', 'docx', 'ppt', 'pptx']);
+
+async function loadInput(
+  input: MediaInput,
+  download: VendorFetch,
+  signal: AbortSignal | undefined
+): Promise<{ bytes: Uint8Array; mediaType: string }> {
+  if (input.kind === 'bytes') {
+    return { bytes: input.bytes, mediaType: input.mediaType };
+  }
+  const r = await download(input.url, signal ? { signal } : undefined);
+  if (!r.ok) {
+    throw new MediaUpstreamError(`media URL fetch ${r.status}: ${input.url}`);
+  }
+  const mediaType =
+    r.headers.get('content-type')?.split(';')[0]?.trim() ||
+    'application/octet-stream';
+  return { bytes: new Uint8Array(await r.arrayBuffer()), mediaType };
+}
+
+function blobOf(bytes: Uint8Array, mediaType: string): Blob {
+  // 复制一份：调用方给的可能是 Buffer 池里的一段（`byteOffset` 不为 0），直接包会把整块池子都传上去。
+  return new Blob([new Uint8Array(bytes)], { type: mediaType });
+}
+
+interface AudioBody {
+  text?: string;
+  duration?: number;
+  language?: string;
+  segments?: Array<{ start: number; end: number; text: string }>;
+}
+interface ParserCreateBody {
+  task_id?: string;
+}
+interface ParserResultBody {
+  status: 'processing' | 'succeeded' | 'failed';
+  content?: string;
+  message?: string;
+  task_id?: string;
+  parsing_result_url?: string;
+}
+
+export class ZhipuMediaAdapter implements MediaAdapter<'zhipu'> {
+  readonly kind = 'zhipu' as const;
+  readonly endpointProtocol = 'media' as const;
+  readonly capabilities = ['audio', 'file'] as const;
+
+  /**
+   * - `audio`：转写是确定性的，产出由模型与语言提示决定。
+   * - `file`：版面解析同样确定，产出只由档位决定；轮询预算等多久都不改变内容，不进键。
+   *   `images: true` 是 agent-base 2026-09-07 加的**产出版本号**：expert/prime 从此带图片，与之前只取 `/text`
+   *   的缓存行不是同一种产出；lite 的产出没变，键也刻意不动。**照抄，别改**——改了 APP 里的存量缓存全部失效。
+   */
+  cacheKeyParts(
+    capability: MediaCapability,
+    settings: Settings,
+    opts: CacheKeyOpts
+  ): Record<string, unknown> | null {
+    const s = resolved(settings);
+    if (capability === 'audio') {
+      return { model: s.audioModel, languageHint: opts.languageHint ?? null };
+    }
+    if (capability === 'file') {
+      return s.toolType === 'lite'
+        ? { toolType: 'lite' }
+        : { toolType: s.toolType, images: true };
+    }
+    return null;
+  }
+
+  maxUploadBytes(
+    capability: MediaCapability,
+    settings: Settings,
+    hint: UploadLimitHint
+  ): number | null {
+    if (capability === 'audio') return ZHIPU_AUDIO_MAX_BYTES;
+    if (capability !== 'file') return null;
+    switch (resolved(settings).toolType) {
+      case 'lite':
+        return FILE_LITE_MAX;
+      case 'expert':
+        return FILE_EXPERT_MAX;
+      case 'prime': {
+        const ext = hint.mediaType ? extOf(hint.mediaType) : undefined;
+        if (ext && PRIME_SPREADSHEET_EXTS.has(ext)) {
+          return FILE_PRIME_SPREADSHEET_MAX;
+        }
+        if (ext && PRIME_DOCUMENT_EXTS.has(ext)) return FILE_PRIME_DOCUMENT_MAX;
+        // 类型未知或尺寸表没列的格式：取各档最小的文档上限当下界。
+        return FILE_LITE_MAX;
+      }
+    }
+  }
+
+  async transcribeAudio(
+    conn: MediaConnection,
+    settings: Settings,
+    req: AudioRequest
+  ): Promise<AudioOutcome> {
+    const s = resolved(settings);
+    const download = conn.download ?? globalThis.fetch;
+    const { bytes, mediaType } = await loadInput(
+      req.input,
+      download,
+      req.signal
+    );
+
+    const form = new FormData();
+    form.append('model', s.audioModel);
+    form.append(
+      'file',
+      blobOf(bytes, mediaType),
+      `upload.${extOf(mediaType) ?? 'bin'}`
+    );
+    if (req.languageHint && req.languageHint !== 'auto') {
+      form.append('language', req.languageHint);
+    }
+
+    const resp = await sendRetryingNetwork(
+      conn.openTask(),
+      joinUrl(conn.baseUrl, 'audio/transcriptions'),
+      {
+        method: 'POST',
+        headers: bearer(conn.apiKey),
+        body: form,
+      },
+      { signal: req.signal }
+    );
+    if (!resp.ok) {
+      throw new MediaUpstreamError(
+        `bigmodel audio/transcriptions ${resp.status}: ${await resp.text()}`
+      );
+    }
+    const json = (await resp.json()) as AudioBody;
+    return {
+      text: json.text ?? '',
+      rawResponse: json,
+      ...(typeof json.duration === 'number'
+        ? { audioDurationMs: Math.round(json.duration * 1000) }
+        : {}),
+      ...(json.language !== undefined ? { language: json.language } : {}),
+      ...(json.segments !== undefined ? { segments: json.segments } : {}),
+    };
+  }
+
+  async parseFile(
+    conn: MediaConnection,
+    settings: Settings,
+    req: FileRequest
+  ): Promise<FileOutcome> {
+    const s = resolved(settings);
+    const download = conn.download ?? globalThis.fetch;
+    const { bytes, mediaType } = await loadInput(
+      req.input,
+      download,
+      req.signal
+    );
+
+    // 认不出 mime 就当场停下，别送一个 `file_type=bin` 上去：厂商回的是「文件格式暂不支持：支持类型为 pdf, doc,
+    // docx, …」——一句把 docx 列在支持列表里、却正是在拒绝一份 docx 的话，谁看了都会往错的方向查。
+    const ext = extOf(mediaType);
+    if (!ext) {
+      throw new MediaUpstreamError(
+        `bigmodel file parser: 不认识的文件类型 "${mediaType}"，无法告诉厂商这是什么格式。` +
+          `本适配器能标注的是 ${[...new Set(Object.values(ZHIPU_MIME_TO_EXT))].join(' / ')}。`
+      );
+    }
+    // 厂商文档：expert 只吃 PDF。送别的格式上去回的是一句泛泛的「格式不支持」，让人去怀疑文件本身。
+    if (s.toolType === 'expert' && ext !== 'pdf') {
+      throw new MediaUpstreamError(
+        `bigmodel file parser: expert 档只解析 PDF，这是一份 ${ext}。` +
+          '改用 prime 档（全格式）或 lite 档（无图片）。'
+      );
+    }
+
+    // 建任务与轮询必须落在同一把 key 上：任务属于那个账号。
+    const send = conn.openTask();
+    const createForm = new FormData();
+    createForm.append('file', blobOf(bytes, mediaType), `upload.${ext}`);
+    createForm.append('tool_type', s.toolType);
+    createForm.append('file_type', ext);
+
+    const createResp = await sendRetryingNetwork(
+      send,
+      joinUrl(conn.baseUrl, 'files/parser/create'),
+      {
+        method: 'POST',
+        headers: bearer(conn.apiKey),
+        body: createForm,
+      },
+      { timeoutMs: CREATE_TIMEOUT_MS, signal: req.signal }
+    );
+    if (!createResp.ok) {
+      throw new MediaUpstreamError(
+        `bigmodel files/parser/create ${createResp.status}: ${await createResp.text()}`
+      );
+    }
+    const createJson = (await createResp.json()) as ParserCreateBody;
+    if (!createJson.task_id) {
+      throw new MediaUpstreamError(
+        `bigmodel files/parser/create returned no task_id: ${JSON.stringify(createJson)}`
+      );
+    }
+    const taskId = createJson.task_id;
+    const budgetMs = req.maxWaitMs ?? s.maxWaitMs;
+    const vendorExtra = { taskId, toolType: s.toolType };
+    const poll = (ms: number, format: 'text' | 'download_link') =>
+      pollResult(conn, send, taskId, ms, format, req.signal);
+
+    // lite 不产出图片，`/text` 就是全部。
+    if (s.toolType === 'lite') {
+      const result = await poll(budgetMs, 'text');
+      return {
+        text: result.content ?? '',
+        markdown: result.content ?? '',
+        rawResponse: result,
+        vendorExtra,
+      };
+    }
+
+    // expert / prime：图片只在 `download_link` 的结果包里（`/text` 的正文与包里的 md 逐字节相同，但引用的
+    // `images/…` 在我们这边打不开）。包拿不到就退回 `/text` 并标 degraded——正文已经花了钱，不该为一次下载失败作废。
+    const link = await poll(budgetMs, 'download_link');
+    let degraded: string;
+    if (link.parsing_result_url) {
+      try {
+        const zip = await downloadWithRetry(
+          download,
+          link.parsing_result_url,
+          req.signal
+        );
+        const pkg = parseParserPackage(zip);
+        return {
+          text: pkg.markdown,
+          markdown: pkg.markdown,
+          rawResponse: link,
+          ...(pkg.pageCount !== undefined ? { pageCount: pkg.pageCount } : {}),
+          images: pkg.images,
+          vendorExtra: {
+            ...vendorExtra,
+            parsingResultUrl: link.parsing_result_url,
+            ...(pkg.missingRefs.length > 0
+              ? { missingImageRefs: pkg.missingRefs }
+              : {}),
+          },
+        };
+      } catch (err) {
+        if (req.signal?.aborted) throw err;
+        degraded = `结果包没拿到，图片缺失：${errorMessage(err)}`;
+      }
+    } else {
+      degraded = '厂商没有返回结果包链接（parsing_result_url 为空），图片缺失';
+    }
+    // 任务已经 succeeded，`/text` 是即时的；预算只是防御厂商侧抽风。
+    const result = await poll(30_000, 'text');
+    return {
+      text: result.content ?? '',
+      markdown: result.content ?? '',
+      rawResponse: result,
+      vendorExtra,
+      degraded,
+    };
+  }
+}
+
+async function pollResult(
+  conn: MediaConnection,
+  send: VendorFetch,
+  taskId: string,
+  budgetMs: number,
+  format: 'text' | 'download_link',
+  signal: AbortSignal | undefined
+): Promise<ParserResultBody> {
+  const start = Date.now();
+  let delay = 500;
+  let attempts = 0;
+  for (;;) {
+    attempts++;
+    const url = joinUrl(
+      conn.baseUrl,
+      `files/parser/result/${encodeURIComponent(taskId)}/${format}`
+    );
+    const resp = await sendRetryingNetwork(
+      send,
+      url,
+      { method: 'GET', headers: bearer(conn.apiKey) },
+      { timeoutMs: POLL_TIMEOUT_MS, signal }
+    );
+    if (!resp.ok) {
+      throw new MediaUpstreamError(
+        `bigmodel files/parser/result ${resp.status}: ${await resp.text()}`
+      );
+    }
+    const json = (await resp.json()) as ParserResultBody;
+    if (json.status === 'succeeded') return json;
+    if (json.status === 'failed') {
+      throw new MediaUpstreamError(
+        `bigmodel file parser failed: ${json.message ?? 'unknown'} (task_id=${taskId})`
+      );
+    }
+    const elapsed = Date.now() - start;
+    if (elapsed + delay > budgetMs) {
+      throw new MediaUpstreamError(
+        `bigmodel file parser timeout after ${elapsed}ms attempts=${attempts} task_id=${taskId}`
+      );
+    }
+    await sleep(delay, signal);
+    delay = Math.min(delay * 2, 2000);
+  }
+}
+
+/** 连不上厂商（没拿到任何响应）时再试的次数，含第一次。 */
+const NETWORK_ATTEMPTS = 3;
+/**
+ * 建解析任务（带文件上传）单次的超时。几 MB 的文件正常几秒内回；2026-09-25 实测一次上传被拖了 298 秒才回，
+ * 工具先在 300 秒处超时，任务在厂商侧建了、结果没人取。三次 × 60 秒仍落在工具的等待上限之内。
+ */
+const CREATE_TIMEOUT_MS = 60_000;
+/** 查询解析结果单次的超时（只是一个 GET，正常百毫秒级）。 */
+const POLL_TIMEOUT_MS = 15_000;
+
+/**
+ * 发到厂商 API 的请求：**只在没拿到响应时重试**——网络层失败（`TypeError: fetch failed`）、给了 `timeoutMs`
+ * 时的单次超时、中心报 `upstream_error`（中心连不上厂商）；拿到了厂商的响应（哪怕 4xx / 5xx）就原样返回、不重试：
+ * 那说明请求到了厂商那边。中心自己报的其余错误（没有可用 key……）抛 `ProviderClientError`。调用方的中止原样抛。
+ *
+ * 超时重试**可能**让建任务在厂商侧重复一次（请求其实到了、只是回应迟迟没来）；不设超时的话工具整体超时后模型照样会
+ * 重试，而且还要先干等到工具超时。音频转写不给 `timeoutMs`：同步接口，长音频本来就要转一阵。
+ */
+async function sendRetryingNetwork(
+  send: VendorFetch,
+  url: string,
+  init: RequestInit,
+  opts: { timeoutMs?: number; signal?: AbortSignal }
+): Promise<Response> {
+  let lastErr: unknown;
+  for (let attempt = 1; attempt <= NETWORK_ATTEMPTS; attempt++) {
+    const signals = [
+      ...(opts.signal ? [opts.signal] : []),
+      ...(opts.timeoutMs !== undefined
+        ? [AbortSignal.timeout(opts.timeoutMs)]
+        : []),
+    ];
+    try {
+      const res = await send(url, {
+        ...init,
+        ...(signals.length > 0 ? { signal: AbortSignal.any(signals) } : {}),
+      });
+      const center = await throwIfCenterConfigError(res);
+      if (!center) return res;
+      lastErr = new MediaUpstreamError(
+        `bigmodel ${new URL(url).pathname}: ${center.upstream}`
+      );
+    } catch (err) {
+      if (opts.signal?.aborted) throw err;
+      const timedOut = err instanceof Error && err.name === 'TimeoutError';
+      if (!(err instanceof TypeError) && !timedOut) throw err;
+      lastErr = err;
+    }
+    if (attempt < NETWORK_ATTEMPTS) await sleep(1000 * attempt, opts.signal);
+  }
+  throw lastErr;
+}
+
+/** 单次下载的超时与重试次数；数字出处见下方注释。 */
+const PACKAGE_DOWNLOAD_TIMEOUT_MS = 30_000;
+const PACKAGE_DOWNLOAD_ATTEMPTS = 3;
+
+/**
+ * 下载结果包（不经中心：链接是厂商的对象存储，带签名、不要 key）。链接指向 UCloud 对象存储（明文 http，24h 有效）。
+ * **2026-09-07 探针里 Node 原生 fetch 约三分之一的次数在 70s 后 `read ETIMEDOUT`、同一链接换个连接立刻 1s 下完**，
+ * 是对端偶发的半开连接。所以每次给 30s、最多三次，而不是等 undici 自己 5 分钟的 body 超时。
+ */
+async function downloadWithRetry(
+  download: VendorFetch,
+  url: string,
+  signal: AbortSignal | undefined
+): Promise<Uint8Array> {
+  let lastErr: unknown;
+  for (let attempt = 1; attempt <= PACKAGE_DOWNLOAD_ATTEMPTS; attempt++) {
+    const timeout = AbortSignal.timeout(PACKAGE_DOWNLOAD_TIMEOUT_MS);
+    try {
+      const r = await download(url, {
+        signal: signal ? AbortSignal.any([signal, timeout]) : timeout,
+      });
+      if (!r.ok) throw new Error(`HTTP ${r.status}`);
+      return new Uint8Array(await r.arrayBuffer());
+    } catch (err) {
+      if (signal?.aborted) throw err;
+      lastErr = err;
+      if (attempt < PACKAGE_DOWNLOAD_ATTEMPTS) {
+        await sleep(1000 * attempt, signal);
+      }
+    }
+  }
+  throw new Error(
+    `结果包下载 ${PACKAGE_DOWNLOAD_ATTEMPTS} 次均失败：${errorMessage(lastErr)}`
+  );
+}
+
+/** 等一会儿；调用方中止时立刻抛出中止原因。 */
+function sleep(ms: number, signal: AbortSignal | undefined): Promise<void> {
+  return new Promise((resolve, reject) => {
+    if (signal?.aborted) {
+      reject(signal.reason as Error);
+      return;
+    }
+    const timer = setTimeout(() => {
+      signal?.removeEventListener('abort', onAbort);
+      resolve();
+    }, ms);
+    const onAbort = () => {
+      clearTimeout(timer);
+      reject(signal!.reason as Error);
+    };
+    signal?.addEventListener('abort', onAbort, { once: true });
+  });
+}