@imweapp/openclaw-imwe 2026.4.12-alpha.1

package/src/file-transfer/upload.ts

@@ -0,0 +1,656 @@
+/**
+ * upload.ts — 分片加密上传服务
+ *
+ * 与 iOS LargeAttachmentUploadOperation 对齐的三阶段分片上传流程：
+ *   initMultipartUpload → 并发 PUT chunks → batchCompleteChunks
+ *
+ * 双模式策略：
+ *   - Buffer 模式（≤ streamThreshold，默认 10MB）：全量加载 → 分片 → 逐片加密 → 并发上传
+ *   - Stream 模式（> streamThreshold）：顺序读取 chunk → 加密 → 提交并发队列 → 增量 SHA256
+ *
+ * 断点续传：
+ *   - 持久化 UploadState 到 JSON 文件（含 key/iv/sendContext）
+ *   - 恢复时跳过已完成 chunk，刷新过期 URL
+ *   - 恢复时仍顺序遍历所有 chunk 维护 SHA256 digest
+ */
+
+import crypto from 'node:crypto';
+import fs from 'node:fs';
+import fsp from 'node:fs/promises';
+import path from 'node:path';
+
+import {
+  UploadResult,
+  UploadSendContext,
+  UploadState,
+  FileTransferConfig,
+  DEFAULT_CONFIG,
+} from './types.js';
+import { generateFileKey, aesCtrXor } from './file-crypto.js';
+import { runConcurrent, ConcurrentTask } from './concurrency.js';
+import { initMultipartUpload, batchGetChunkPreSignedUrl, batchCompleteChunks } from './api.js';
+import { inferMimeType, computeBlurhash } from '../media-utils.js';
+
+// ─────────────────────────────────────────────────────────────────────────────
+
+// ─────────────────────────────────────────────────────────────────────────────
+// 断点续传状态管理
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * 从持久化状态恢复上传。
+ * 检查 stateDir 中是否存在未完成的 UploadState，若存在则恢复。
+ * 按 mediaUrl 精确匹配。
+ */
+export function loadUploadState(stateDir: string, mediaUrl: string): UploadState | null {
+  if (!stateDir) return null;
+  try {
+    const files = fs.readdirSync(stateDir);
+    for (const f of files) {
+      if (!f.startsWith('upload-') || !f.endsWith('.json')) continue;
+      const filePath = path.join(stateDir, f);
+      const content = fs.readFileSync(filePath, 'utf-8');
+      const state: UploadState = JSON.parse(content);
+      if (state.mediaUrl === mediaUrl) {
+        return state;
+      }
+    }
+  } catch {
+    // stateDir 不存在或读取失败，视为无状态
+  }
+  return null;
+}
+
+/**
+ * 持久化上传状态。
+ * 每个 chunk 完成后调用，原子写入 JSON 文件。
+ */
+export function saveUploadState(stateDir: string, state: UploadState): void {
+  if (!stateDir) return;
+  fs.mkdirSync(stateDir, { recursive: true });
+  const filePath = path.join(stateDir, `upload-${state.fileId}.json`);
+  state.updatedAt = Date.now();
+  // 原子写入：先写临时文件再 rename
+  const tmpPath = filePath + '.tmp';
+  fs.writeFileSync(tmpPath, JSON.stringify(state, null, 2), 'utf-8');
+  fs.renameSync(tmpPath, filePath);
+}
+
+/**
+ * 清理上传状态文件（上传完成或放弃时调用）。
+ */
+export function clearUploadState(stateDir: string, mediaUrl: string): void {
+  if (!stateDir) return;
+  try {
+    const files = fs.readdirSync(stateDir);
+    for (const f of files) {
+      if (!f.startsWith('upload-') || !f.endsWith('.json')) continue;
+      const filePath = path.join(stateDir, f);
+      const content = fs.readFileSync(filePath, 'utf-8');
+      const state: UploadState = JSON.parse(content);
+      if (state.mediaUrl === mediaUrl) {
+        fs.unlinkSync(filePath);
+        return;
+      }
+    }
+  } catch {
+    // 清理失败不阻断
+  }
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// 内部：媒体源解析
+// ─────────────────────────────────────────────────────────────────────────────
+
+/** 解析后的媒体源（Buffer 模式） */
+interface BufferMediaSource {
+  kind: 'buffer';
+  buffer: Buffer;
+  fileName: string;
+  mimeType: string;
+  fileSize: number;
+}
+
+/** 解析后的媒体源（Stream 模式 — 仅保留路径，按需读取 chunk） */
+interface StreamMediaSource {
+  kind: 'stream';
+  filePath: string;
+  fileName: string;
+  mimeType: string;
+  fileSize: number;
+}
+
+type ResolvedMediaSource = BufferMediaSource | StreamMediaSource;
+
+/**
+ * 解析媒体源，根据文件大小选择 Buffer 或 Stream 模式。
+ */
+async function resolveMediaSource(
+  mediaUrl: string,
+  streamThreshold: number,
+  options?: {
+    mediaLocalRoots?: readonly string[];
+    mediaReadFile?: (filePath: string) => Promise<Buffer>;
+  },
+): Promise<ResolvedMediaSource> {
+  const fileName = extractFileName(mediaUrl);
+  const mimeType = inferMimeType(fileName);
+
+  // data URI → Buffer 模式
+  if (mediaUrl.startsWith('data:')) {
+    const match = mediaUrl.match(/^data:([^;,]+)?(?:;base64)?,(.*)$/s);
+    if (!match) throw new Error(`无效的 data URI: ${mediaUrl.slice(0, 40)}...`);
+    const buffer = Buffer.from(match[2], 'base64');
+    const mime = match[1] || 'application/octet-stream';
+    return { kind: 'buffer', buffer, fileName, mimeType: mime, fileSize: buffer.length };
+  }
+
+  // 远程 URL → 始终 Buffer 模式（下载到内存）
+  if (mediaUrl.startsWith('http://') || mediaUrl.startsWith('https://')) {
+    const res = await fetch(mediaUrl);
+    if (!res.ok) throw new Error(`下载远程媒体失败: HTTP ${res.status} ${mediaUrl}`);
+    const buffer = Buffer.from(await res.arrayBuffer());
+    const remoteMime = res.headers.get('content-type')?.split(';')[0]?.trim() || mimeType;
+    return { kind: 'buffer', buffer, fileName, mimeType: remoteMime, fileSize: buffer.length };
+  }
+
+  // 本地路径：优先使用 openclaw plugin-sdk 加载
+  try {
+    const { loadWebMedia } = await import('openclaw/plugin-sdk/web-media');
+    const localRoots = options?.mediaLocalRoots?.length ? options.mediaLocalRoots : undefined;
+    const loaded = await loadWebMedia(mediaUrl, {
+      localRoots,
+      readFile: options?.mediaReadFile,
+    });
+    // loadWebMedia 返回 Buffer，判断大小决定模式
+    if (loaded.buffer.length > streamThreshold) {
+      // 大文件但已加载到内存，仍用 Buffer 模式（loadWebMedia 已全量加载）
+      return {
+        kind: 'buffer',
+        buffer: loaded.buffer,
+        fileName: loaded.fileName ?? fileName,
+        mimeType: loaded.contentType ?? mimeType,
+        fileSize: loaded.buffer.length,
+      };
+    }
+    return {
+      kind: 'buffer',
+      buffer: loaded.buffer,
+      fileName: loaded.fileName ?? fileName,
+      mimeType: loaded.contentType ?? mimeType,
+      fileSize: loaded.buffer.length,
+    };
+  } catch {
+    // loadWebMedia 不可用时回退到直接读取
+  }
+
+  // 自定义 mediaReadFile → Buffer 模式
+  if (options?.mediaReadFile) {
+    const buffer = await options.mediaReadFile(mediaUrl);
+    return { kind: 'buffer', buffer, fileName, mimeType, fileSize: buffer.length };
+  }
+
+  // 直接读取本地文件
+  const fileStat = await fsp.stat(mediaUrl);
+
+  if (fileStat.size > streamThreshold) {
+    // 大文件 → Stream 模式（保留路径，按需读取 chunk）
+    return { kind: 'stream', filePath: mediaUrl, fileName, mimeType, fileSize: fileStat.size };
+  }
+
+  // 小文件 → Buffer 模式
+  const buffer = await fsp.readFile(mediaUrl);
+  return { kind: 'buffer', buffer, fileName, mimeType, fileSize: buffer.length };
+}
+
+/** 从 URL 或路径中提取文件名 */
+function extractFileName(mediaUrl: string): string {
+  try {
+    if (mediaUrl.startsWith('http://') || mediaUrl.startsWith('https://')) {
+      const pathname = new URL(mediaUrl).pathname;
+      return pathname.split('/').pop() || 'file';
+    }
+  } catch {
+    // ignore
+  }
+  return mediaUrl.split('/').pop()?.split('?')[0] || 'file';
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// 内部：分片工具
+// ─────────────────────────────────────────────────────────────────────────────
+
+/** 计算分片计划 */
+function computeChunkPlan(fileSize: number, chunkSize: number) {
+  const totalChunks = Math.ceil(fileSize / chunkSize);
+  return { chunkSize, totalChunks, fileSize };
+}
+
+/** 从 Buffer 中提取指定 chunk 的数据 */
+function getChunkFromBuffer(buffer: Buffer, index: number, chunkSize: number): Buffer {
+  const start = index * chunkSize;
+  const end = Math.min(start + chunkSize, buffer.length);
+  return buffer.subarray(start, end);
+}
+
+/** 从文件中读取指定 chunk 的数据（Stream 模式） */
+async function readChunkFromFile(
+  filePath: string,
+  index: number,
+  chunkSize: number,
+  fileSize: number,
+): Promise<Buffer> {
+  const start = index * chunkSize;
+  const end = Math.min(start + chunkSize, fileSize);
+  const length = end - start;
+  const buf = Buffer.alloc(length);
+  const fd = await fsp.open(filePath, 'r');
+  try {
+    await fd.read(buf, 0, length, start);
+  } finally {
+    await fd.close();
+  }
+  return buf;
+}
+
+/** PUT 加密后的 chunk 到预签名 URL，返回 ETag */
+async function putChunkToS3(
+  url: string,
+  encryptedData: Uint8Array,
+  mimeType: string,
+): Promise<string> {
+  const res = await fetch(url, {
+    method: 'PUT',
+    headers: {
+      'Content-Type': mimeType,
+      'Content-Length': String(encryptedData.length),
+    },
+    body: encryptedData,
+  });
+  if (!res.ok) {
+    const errText = await res.text().catch(() => '');
+    throw new Error(`分片上传失败: HTTP ${res.status} ${errText}`);
+  }
+  // ETag 通常在响应头中
+  const originalEtag = res.headers.get('etag');
+  const etag = originalEtag
+    ? originalEtag.slice(1, -1) // etag 取值去掉头尾一个字符
+    : crypto.randomUUID().replace(/-/g, '');
+  return etag;
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Buffer 模式上传
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Buffer 模式：全量加载 → 分片 → 逐片加密 → 并发上传 → 流式 SHA256
+ */
+async function uploadBufferMode(
+  buffer: Buffer,
+  key: Uint8Array,
+  iv: Uint8Array,
+  chunkUrls: string[],
+  state: UploadState,
+  stateDir: string,
+  concurrency: number,
+  mimeType: string,
+  log?: { info?: (...args: unknown[]) => void; warn?: (...args: unknown[]) => void },
+): Promise<{ digest: string }> {
+  const { chunkSize, totalChunks } = state;
+
+  // 流式 SHA256：顺序遍历所有 chunk 维护 digest
+  const hash = crypto.createHash('sha256');
+
+  // 构建上传任务列表
+  const tasks: ConcurrentTask<string>[] = [];
+
+  for (let i = 0; i < totalChunks; i++) {
+    const chunkData = getChunkFromBuffer(buffer, i, chunkSize);
+    const offset = i * chunkSize;
+
+    // 无论是否已完成，都要 update SHA256 digest
+    hash.update(chunkData);
+
+    if (state.chunks[i].completed) {
+      // 已完成的 chunk 跳过上传
+      continue;
+    }
+
+    // 加密 chunk
+    const encrypted = aesCtrXor(chunkData, key, iv, offset);
+    const url = chunkUrls[i];
+
+    tasks.push({
+      id: i,
+      execute: async () => {
+        const etag = await putChunkToS3(url, encrypted, mimeType);
+        // 更新状态
+        state.chunks[i].completed = true;
+        state.chunks[i].etag = etag;
+        saveUploadState(stateDir, state);
+        log?.info?.(`[upload] chunk ${i + 1}/${totalChunks} 上传完成`);
+        return etag;
+      },
+    });
+  }
+
+  // 并发上传未完成的 chunk
+  if (tasks.length > 0) {
+    const results = await runConcurrent(tasks, concurrency);
+    // 检查是否有失败
+    const failed = results.filter((r) => !r.success);
+    if (failed.length > 0) {
+      throw new Error(
+        `${failed.length} 个分片上传失败: ${failed.map((f) => f.error?.message).join('; ')}`,
+      );
+    }
+  }
+
+  const digest = hash.digest('hex');
+  return { digest };
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Stream 模式上传
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Stream 模式：顺序读取 chunk → 加密 → 提交并发队列 → 增量 SHA256
+ *
+ * 读取是顺序的（不将整个文件加载到内存），加密后的 chunk 并发上传。
+ * 峰值内存占用 ≈ chunkSize × concurrency。
+ */
+async function uploadStreamMode(
+  filePath: string,
+  fileSize: number,
+  key: Uint8Array,
+  iv: Uint8Array,
+  chunkUrls: string[],
+  state: UploadState,
+  stateDir: string,
+  concurrency: number,
+  mimeType: string,
+  log?: { info?: (...args: unknown[]) => void; warn?: (...args: unknown[]) => void },
+): Promise<{ digest: string }> {
+  const { chunkSize, totalChunks } = state;
+
+  // 流式 SHA256
+  const hash = crypto.createHash('sha256');
+
+  // 构建上传任务列表（顺序读取，收集后并发上传）
+  const tasks: ConcurrentTask<string>[] = [];
+
+  for (let i = 0; i < totalChunks; i++) {
+    const chunkData = await readChunkFromFile(filePath, i, chunkSize, fileSize);
+    const offset = i * chunkSize;
+
+    // 无论是否已完成，都要 update SHA256 digest
+    hash.update(chunkData);
+
+    if (state.chunks[i].completed) {
+      // 已完成的 chunk 跳过上传
+      continue;
+    }
+
+    // 加密 chunk
+    const encrypted = aesCtrXor(chunkData, key, iv, offset);
+    const url = chunkUrls[i];
+
+    tasks.push({
+      id: i,
+      execute: async () => {
+        const etag = await putChunkToS3(url, encrypted, mimeType);
+        // 更新状态
+        state.chunks[i].completed = true;
+        state.chunks[i].etag = etag;
+        saveUploadState(stateDir, state);
+        log?.info?.(`[upload] chunk ${i + 1}/${totalChunks} 上传完成`);
+        return etag;
+      },
+    });
+  }
+
+  // 并发上传未完成的 chunk
+  if (tasks.length > 0) {
+    const results = await runConcurrent(tasks, concurrency);
+    const failed = results.filter((r) => !r.success);
+    if (failed.length > 0) {
+      throw new Error(
+        `${failed.length} 个分片上传失败: ${failed.map((f) => f.error?.message).join('; ')}`,
+      );
+    }
+  }
+
+  const digest = hash.digest('hex');
+  return { digest };
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// 公开 API
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * 加密并分片上传文件到平台存储。
+ *
+ * 双模式策略（与 iOS "顺序读取, 并发上传" 对齐）：
+ *   - Buffer 模式（≤ streamThreshold，默认 10MB）：全量加载到内存，分片加密后并发上传
+ *   - Stream 模式（> streamThreshold）：顺序读取 chunk，每个 chunk 加密后提交到并发上传队列
+ *
+ * 完整流程：
+ *   1. 判断文件大小，选择 Buffer 或 Stream 模式
+ *   2. 生成 AES-256 key/iv（或从断点状态恢复）
+ *   3. 计算分片计划（chunkSize 默认 2MB）
+ *   4. initMultipartUpload 获取 fileId + 预签名 URL
+ *   5. 持久化 UploadState（fileId, key, iv, chunkPlan, sendContext）
+ *   6. 加密 + 上传各分片
+ *   7. 流式计算 SHA256(plaintext)
+ *   8. batchCompleteChunks 确认上传完成
+ *   9. 清理 UploadState 文件
+ *
+ * @param mediaUrl    待上传的媒体文件路径或 URL
+ * @param auth        鉴权信息
+ * @param apiBaseUrl  API 基础地址
+ * @param options     可选参数
+ */
+export async function uploadEncryptedFile(
+  mediaUrl: string,
+  auth: { apiKey: string; apiSecret: string },
+  apiBaseUrl: string,
+  options?: {
+    /** 接收方 IM 主账号 ID */
+    imMainAccId?: string;
+    /** 文件传输配置 */
+    config?: FileTransferConfig;
+    /** 消息发送上下文（持久化到 UploadState，断点续传恢复后用于发送消息） */
+    sendContext?: UploadSendContext;
+    /** 允许读取的本地目录列表 */
+    mediaLocalRoots?: readonly string[];
+    /** 自定义文件读取函数 */
+    mediaReadFile?: (filePath: string) => Promise<Buffer>;
+    /** 日志接口 */
+    log?: { info?: (...args: unknown[]) => void; warn?: (...args: unknown[]) => void };
+  },
+): Promise<UploadResult> {
+  const log = options?.log;
+  const config = { ...DEFAULT_CONFIG, ...options?.config };
+  const { chunkSize, uploadConcurrency, stateDir, streamThreshold } = config;
+
+  // 1. 解析媒体源
+  log?.info?.(`[upload] 解析媒体源: ${mediaUrl}`);
+  const media = await resolveMediaSource(mediaUrl, streamThreshold, {
+    mediaLocalRoots: options?.mediaLocalRoots,
+    mediaReadFile: options?.mediaReadFile,
+  });
+  log?.info?.(
+    `[upload] 媒体源解析完成: mode=${media.kind}, fileName=${media.fileName}, size=${media.fileSize}`,
+  );
+
+  // 2. 检查断点续传状态
+  let state = loadUploadState(stateDir, mediaUrl);
+  let key: Uint8Array;
+  let iv: Uint8Array;
+  let chunkUrls: string[];
+
+  if (state) {
+    // 从断点恢复
+    log?.info?.(
+      `[upload] 检测到断点续传状态: fileId=${state.fileId}, 已完成 ${state.chunks.filter((c) => c.completed).length}/${state.totalChunks} 个分片`,
+    );
+    key = Buffer.from(state.keyHex, 'hex');
+    iv = Buffer.from(state.ivHex, 'hex');
+
+    // 刷新未完成 chunk 的预签名 URL
+    const pendingIndices = state.chunks.filter((c) => !c.completed).map((c) => c.index);
+
+    if (pendingIndices.length > 0) {
+      log?.info?.(
+        `[upload] 刷新 ${pendingIndices.length} 个分片的预签名 URL, ${JSON.stringify({
+          imMainAccId: options?.imMainAccId ?? '',
+          fileId: state.fileId,
+          chunkIndices: pendingIndices,
+        })}`,
+      );
+      const refreshed = await batchGetChunkPreSignedUrl(apiBaseUrl, auth, {
+        imMainAccId: options?.imMainAccId ?? '',
+        fileId: state.fileId,
+        chunkIndices: pendingIndices,
+      });
+      // 构建完整的 URL 列表
+      chunkUrls = new Array(state.totalChunks).fill('');
+      for (const item of refreshed.chunkUrlList) {
+        chunkUrls[item.index] = item.url;
+      }
+    } else {
+      chunkUrls = new Array(state.totalChunks).fill('');
+    }
+  } else {
+    // 全新上传
+    const fileKey = generateFileKey();
+    key = fileKey.key;
+    iv = fileKey.iv;
+
+    // 3. 计算分片计划
+    const plan = computeChunkPlan(media.fileSize, chunkSize);
+    log?.info?.(`[upload] 分片计划: chunkSize=${chunkSize}, totalChunks=${plan.totalChunks}`);
+
+    // 4. initMultipartUpload
+    const initResult = await initMultipartUpload(apiBaseUrl, auth, {
+      imMainAccId: options?.imMainAccId ?? '',
+      fileName: media.fileName,
+      fileSize: media.fileSize,
+      chunkSize,
+      totalChunks: plan.totalChunks,
+      mimeType: media.mimeType,
+    });
+    log?.info?.(`[upload] initMultipartUpload 成功: ${JSON.stringify(initResult)}`);
+
+    chunkUrls = (initResult.chunkUrlList || []).filter(Boolean);
+
+    // 5. 创建并持久化 UploadState
+    const sendContext: UploadSendContext = options?.sendContext ?? {
+      to: '',
+      fromId: '',
+      mediaType: 'file',
+    };
+
+    state = {
+      fileId: initResult.fileId,
+      fileName: media.fileName,
+      mediaUrl,
+      imMainAccId: options?.imMainAccId ?? '',
+      plaintextLength: media.fileSize,
+      chunkSize,
+      totalChunks: plan.totalChunks,
+      keyHex: Buffer.from(key).toString('hex'),
+      ivHex: Buffer.from(iv).toString('hex'),
+      chunks: Array.from({ length: plan.totalChunks }, (_, i) => ({
+        index: i,
+        completed: false,
+      })),
+      sendContext,
+      expireTime: initResult.expireTime,
+      createdAt: Date.now(),
+      updatedAt: Date.now(),
+    };
+    saveUploadState(stateDir, state);
+  }
+
+  // 6. 执行加密 + 上传
+  let digest: string;
+
+  if (media.kind === 'buffer') {
+    log?.info?.(`[upload] 使用 Buffer 模式上传`);
+    const result = await uploadBufferMode(
+      media.buffer,
+      key,
+      iv,
+      chunkUrls,
+      state,
+      stateDir,
+      uploadConcurrency,
+      media.mimeType,
+      log,
+    );
+    digest = result.digest;
+  } else {
+    log?.info?.(`[upload] 使用 Stream 模式上传`);
+    const result = await uploadStreamMode(
+      media.filePath,
+      media.fileSize,
+      key,
+      iv,
+      chunkUrls,
+      state,
+      stateDir,
+      uploadConcurrency,
+      media.mimeType,
+      log,
+    );
+    digest = result.digest;
+  }
+
+  // 7. batchCompleteChunks 确认上传完成
+  const completedChunks = state.chunks.map((c) => ({
+    index: c.index,
+    etag: c.etag || '',
+  }));
+
+  log?.info?.(`[upload] 确认上传完成: fileId=${state.fileId}`);
+  const completeResult = await batchCompleteChunks(apiBaseUrl, auth, {
+    imMainAccId: options?.imMainAccId ?? '',
+    fileId: state.fileId,
+    completedChunks,
+  });
+
+  // 8. 清理 UploadState
+  clearUploadState(stateDir, mediaUrl);
+  log?.info?.(`[upload] 上传完成: fileUrl=${completeResult.fileUrl}`);
+
+  // 9. 计算 blurhash（仅 Buffer 模式图片）
+  let blurHash: string | undefined;
+  let width: number | undefined;
+  let height: number | undefined;
+  if (media.kind === 'buffer') {
+    const bh = await computeBlurhash(media.buffer, media.mimeType);
+    if (bh) {
+      blurHash = bh.blurHash;
+      width = bh.width;
+      height = bh.height;
+    }
+  }
+
+  return {
+    url: completeResult.fileUrl,
+    fileKey: key,
+    fileIv: iv,
+    fileDigest: digest,
+    plaintextLength: media.fileSize,
+    opCreds: completeResult.opCreds,
+    contentType: media.mimeType,
+    blurHash,
+    width,
+    height,
+    expireTime: state.expireTime,
+  };
+}