create-bubbles 0.1.25 → 0.1.27

package/template-vp-monorepo-react-nestjs/apps/server/src/modules/upload/storage/minio-storage.adapter.ts

@@ -0,0 +1,276 @@
+import {
+  AbortMultipartUploadCommand,
+  CompleteMultipartUploadCommand,
+  CreateMultipartUploadCommand,
+  HeadBucketCommand,
+  HeadObjectCommand,
+  ListPartsCommand,
+  S3Client,
+  UploadPartCommand,
+} from '@aws-sdk/client-s3'
+import { Injectable, OnModuleDestroy } from '@nestjs/common'
+import { ConfigService } from '@nestjs/config'
+import {
+  CreateMultipartInput,
+  MultipartIdentity,
+  StorageMultipartNotFoundError,
+  StorageObject,
+  StoragePart,
+  StoragePort,
+  UploadPartInput,
+} from './storage.port'
+
+/**
+ * 兼容 AWS SDK 的 name 与 S3 响应的 Code 字段，判断存储错误类别。
+ */
+function hasErrorName(cause: unknown, expected: string): boolean {
+  if (typeof cause !== 'object' || cause === null) {
+    return false
+  }
+  const record = cause as { name?: unknown; Code?: unknown }
+  return record.name === expected || record.Code === expected
+}
+
+/**
+ * 将 NoSuchUpload 转换为存储端口约定的异常，其余错误保留原始原因并抛出。
+ */
+function throwMappedMultipartError(cause: unknown): never {
+  if (hasErrorName(cause, 'NoSuchUpload')) {
+    throw new StorageMultipartNotFoundError(cause)
+  }
+
+  throw cause
+}
+
+@Injectable()
+export class MinioStorageAdapter implements StoragePort, OnModuleDestroy {
+  private readonly client: S3Client
+  private readonly bucket: string
+
+  /**
+   * 根据存储配置创建 S3 客户端，并禁用不可回放上传流的自动重试。
+   */
+  constructor(config: ConfigService) {
+    this.bucket = config.getOrThrow<string>('storage.bucket')
+    this.client = new S3Client({
+      endpoint: config.getOrThrow<string>('storage.endpoint'),
+      region: config.getOrThrow<string>('storage.region'),
+      credentials: {
+        accessKeyId: config.getOrThrow<string>('storage.accessKeyId'),
+        secretAccessKey: config.getOrThrow<string>('storage.secretAccessKey'),
+      },
+      forcePathStyle: config.getOrThrow<boolean>('storage.forcePathStyle'),
+      // UploadPart 的 Body 是不可回放流，禁止 SDK 在后端自动重放。
+      maxAttempts: 1,
+      requestChecksumCalculation: 'WHEN_REQUIRED',
+      responseChecksumValidation: 'WHEN_REQUIRED',
+    })
+  }
+
+  /**
+   * 通过 HeadBucket 验证目标桶存在且可访问，失败交由调用方处理。
+   */
+  private async assertBucketAvailable(): Promise<void> {
+    await this.client.send(
+      new HeadBucketCommand({
+        Bucket: this.bucket,
+      }),
+    )
+  }
+
+  /**
+   * 在模块销毁时释放 S3 客户端的网络资源。
+   */
+  onModuleDestroy() {
+    this.client.destroy()
+  }
+
+  /**
+   * 在模块初始化时探测上传桶，及早暴露配置或存储访问异常。
+   */
+  async onModuleInit() {
+    await this.assertBucketAvailable()
+  }
+
+  /**
+   * 向对象存储创建分片上传，写入内容类型和元数据并返回 UploadId。
+   * @throws 存储未返回 UploadId 时抛出错误。
+   */
+  async createMultipartUpload(input: CreateMultipartInput) {
+    const response = await this.client.send(
+      new CreateMultipartUploadCommand({
+        Bucket: input.bucket,
+        Key: input.objectKey,
+        ContentType: input.contentType,
+        Metadata: input.metadata,
+      }),
+    )
+
+    if (!response.UploadId) {
+      throw new Error('Storage did not return an UploadId')
+    }
+
+    return {
+      storageUploadId: response.UploadId,
+    }
+  }
+
+  /**
+   * 流式上传指定分片并传递取消信号，返回后续合并所需的 ETag。
+   */
+  async uploadPart(input: UploadPartInput) {
+    try {
+      const response = await this.client.send(
+        new UploadPartCommand({
+          Bucket: input.bucket,
+          Key: input.objectKey,
+          UploadId: input.storageUploadId,
+          PartNumber: input.partNumber,
+          Body: input.body,
+          ContentLength: input.contentLength,
+        }),
+        {
+          abortSignal: input.abortSignal,
+        },
+      )
+
+      if (!response.ETag) {
+        throw new Error('Storage did not return an ETag')
+      }
+
+      return {
+        etag: response.ETag,
+      }
+    } catch (cause: unknown) {
+      throwMappedMultipartError(cause)
+    }
+  }
+
+  /**
+   * 逐页读取全部已上传分片，校验记录与游标有效性，并按分片序号升序返回。
+   */
+  async listParts(input: MultipartIdentity): Promise<StoragePart[]> {
+    const parts: StoragePart[] = []
+    let partNumberMarker: string | undefined
+    try {
+      for (;;) {
+        const response = await this.client.send(
+          new ListPartsCommand({
+            Bucket: input.bucket,
+            Key: input.objectKey,
+            UploadId: input.storageUploadId,
+            PartNumberMarker: partNumberMarker,
+          }),
+        )
+
+        for (const part of response.Parts ?? []) {
+          if (part.PartNumber === undefined || part.ETag === undefined || part.Size === undefined) {
+            throw new Error('Storage returned an incomplete part record')
+          }
+          parts.push({
+            partNumber: part.PartNumber,
+            etag: part.ETag,
+            size: part.Size,
+          })
+        }
+
+        if (!response.IsTruncated) {
+          break
+        }
+
+        const nextMarker = response.NextPartNumberMarker
+        if (!nextMarker || nextMarker === partNumberMarker) {
+          throw new Error('Storage returned an invalid ListParts cursor')
+        }
+
+        partNumberMarker = nextMarker
+      }
+    } catch (cause: unknown) {
+      throwMappedMultipartError(cause)
+    }
+
+    return parts.sort((left, right) => left.partNumber - right.partNumber)
+  }
+
+  /**
+   * 提交分片序号和 ETag 列表完成对象合并，返回存储对象的 ETag。
+   */
+  async completeMultipartUpload(input: MultipartIdentity & { parts: readonly StoragePart[] }) {
+    try {
+      const response = await this.client.send(
+        new CompleteMultipartUploadCommand({
+          Bucket: input.bucket,
+          Key: input.objectKey,
+          UploadId: input.storageUploadId,
+          MultipartUpload: {
+            Parts: input.parts.map((part) => ({
+              PartNumber: part.partNumber,
+              ETag: part.etag,
+            })),
+          },
+        }),
+      )
+      return {
+        etag: response.ETag ?? null,
+      }
+    } catch (cause: unknown) {
+      throwMappedMultipartError(cause)
+    }
+  }
+
+  /**
+   * 取消对象存储中的分片上传；UploadId 已不存在时视为成功，保持取消操作幂等。
+   */
+  async abortMultipartUpload(input: MultipartIdentity): Promise<void> {
+    try {
+      await this.client.send(
+        new AbortMultipartUploadCommand({
+          Bucket: input.bucket,
+          Key: input.objectKey,
+          UploadId: input.storageUploadId,
+        }),
+      )
+    } catch (cause: unknown) {
+      if (hasErrorName(cause, 'NoSuchUpload')) {
+        return
+      }
+      throw cause
+    }
+  }
+
+  /**
+   * 读取对象大小、ETag 及元数据；确认仅对象缺失时返回 null，桶不可用时仍抛出错误。
+   */
+  async headObject(input: { bucket: string; objectKey: string }): Promise<StorageObject | null> {
+    try {
+      const response = await this.client.send(
+        new HeadObjectCommand({
+          Bucket: input.bucket,
+          Key: input.objectKey,
+        }),
+      )
+      return {
+        etag: response.ETag ?? null,
+        contentLength: response.ContentLength ?? null,
+        metadata: response.Metadata ?? {},
+      }
+    } catch (cause: unknown) {
+      if (hasErrorName(cause, 'NoSuchBucket')) {
+        throw cause
+      }
+
+      if (hasErrorName(cause, 'NoSuchKey')) {
+        return null
+      }
+
+      if (hasErrorName(cause, 'NotFound')) {
+        // 某些 S3 兼容实现会把“对象不存在”和“Bucket 不存在”都表示成 404。
+        // 额外探测 Bucket；只有 Bucket 确实可用时，才把这次 404 当成对象不存在。
+        await this.assertBucketAvailable()
+        return null
+      }
+
+      throw cause
+    }
+  }
+}

package/template-vp-monorepo-react-nestjs/apps/server/src/modules/upload/storage/storage.port.ts

@@ -0,0 +1,82 @@
+import { Readable } from 'stream'
+
+export const STORAGE_PORT = Symbol('STORAGE_PORT')
+
+export class StorageMultipartNotFoundError extends Error {
+  /**
+   * 封装存储分片上传不存在的错误，并保留原始原因供上层映射和排查。
+   */
+  constructor(cause: unknown) {
+    super('Multipart upload does not exist', { cause })
+    this.name = StorageMultipartNotFoundError.name
+  }
+}
+
+export interface CreateMultipartInput {
+  bucket: string
+  objectKey: string
+  contentType: string
+  metadata: Record<string, string>
+}
+
+export interface UploadPartInput {
+  bucket: string
+  objectKey: string
+  storageUploadId: string
+  partNumber: number
+  body: Readable
+  contentLength: number
+  abortSignal?: AbortSignal
+}
+
+export interface MultipartIdentity {
+  bucket: string
+  objectKey: string
+  storageUploadId: string
+}
+
+export interface StoragePart {
+  partNumber: number
+  etag: string
+  size: number
+}
+
+export interface StorageObject {
+  etag: string | null
+  contentLength: number | null
+  metadata: Record<string, string>
+}
+
+export interface StoragePort {
+  /**
+   * 在指定桶创建分片上传并返回后续请求使用的存储 UploadId。
+   */
+  createMultipartUpload(input: CreateMultipartInput): Promise<{ storageUploadId: string }>
+
+  /**
+   * 写入指定序号的分片流，支持取消信号，并返回合并所需的 ETag。
+   */
+  uploadPart(input: UploadPartInput): Promise<{ etag: string }>
+
+  /**
+   * 返回按序号排列的全部已上传分片，供续传和完整性校验使用。
+   */
+  listParts(input: MultipartIdentity): Promise<StoragePart[]>
+
+  /**
+   * 按已校验的分片列表合并对象，返回对象 ETag。
+   */
+  completeMultipartUpload(
+    input: MultipartIdentity & { parts: readonly StoragePart[] },
+  ): Promise<{ etag: string | null }>
+
+  /**
+   * 清理指定上传的未合并分片，重复取消应保持幂等。
+   */
+  abortMultipartUpload(input: MultipartIdentity): Promise<void>
+
+  /**
+   * 读取对象的长度、ETag 和元数据；对象不存在时返回 null。
+   */
+  headObject(input: { bucket: string; objectKey: string }): Promise<StorageObject | null>
+}

package/template-vp-monorepo-react-nestjs/apps/server/src/modules/upload/upload.constants.ts

@@ -0,0 +1,32 @@
+export const UPLOAD_PART_SIZE = 10 * 1024 * 1024
+export const UPLOAD_MAX_PARTS = 10_000
+export const UPLOAD_MAX_FILE_SIZE = 90 * 1024 * 1024 * 1024
+
+/**
+ * 按固定分片大小向上取整，计算文件上传所需分片数。
+ * @param fileSize 文件字节数。
+ */
+export function calculateTotalParts(fileSize: number): number {
+  return Math.ceil(fileSize / UPLOAD_PART_SIZE)
+}
+
+/**
+ * 计算指定分片应有的字节数，最后一片使用文件剩余长度。
+ * @param partNumber 从 1 开始的分片序号。
+ * @throws 分片序号超出上传范围时抛出 RangeError。
+ */
+export function calculateExpectedPartSize(
+  fileSize: number,
+  totalParts: number,
+  partNumber: number,
+): number {
+  if (partNumber < 1 || partNumber > totalParts) {
+    throw new RangeError('partNumber is outside the upload range')
+  }
+
+  if (partNumber < totalParts) {
+    return UPLOAD_PART_SIZE
+  }
+
+  return fileSize - (totalParts - 1) * UPLOAD_PART_SIZE
+}

package/template-vp-monorepo-react-nestjs/apps/server/src/modules/upload/upload.controller.ts

@@ -0,0 +1,134 @@
+import { CurrentAuth } from '@/common/decorators/current-auth.decorator'
+import { Authenticated } from '@/common/decorators/access-policy.decorator'
+import {
+  Body,
+  Controller,
+  Delete,
+  Get,
+  Headers,
+  HttpCode,
+  HttpStatus,
+  Param,
+  Post,
+  Put,
+  Req,
+  Res,
+} from '@nestjs/common'
+import { ApiBearerAuth, ApiBody, ApiConsumes, ApiOperation, ApiTags } from '@nestjs/swagger'
+import { type FastifyReply, FastifyRequest } from 'fastify'
+import type { Readable } from 'stream'
+import { type CurrentAuthType } from '../auth/session/session.types'
+import { InitiateMultipartUploadDto } from './dto/initiate-mutipart-upload.dto'
+import { UploadPartParamsDto, UploadSessionParamsDto } from './dto/upload-params.dto'
+import { UploadService } from './upload.service'
+
+type OctetStreamRequest = FastifyRequest & {
+  body: Readable
+}
+
+@ApiTags('大文件分片上传')
+@ApiBearerAuth('session')
+@Controller('uploads/multipart')
+@Authenticated()
+export class UploadController {
+  constructor(private readonly uploadService: UploadService) {}
+
+  /**
+   * 为当前认证用户初始化分片上传，客户端上传标识用于幂等复用。
+   */
+  @ApiOperation({ summary: '初始化大文件分片上传' })
+  @Post()
+  initiate(@CurrentAuth() auth: CurrentAuthType, @Body() body: InitiateMultipartUploadDto) {
+    return this.uploadService.initiate(auth.userId, body)
+  }
+
+  /**
+   * 查询当前用户上传会话的状态及已上传分片。
+   */
+  @ApiOperation({ summary: '查询上传状态和已上传分片' })
+  @Get(':uploadSessionId')
+  getStatus(@CurrentAuth() auth: CurrentAuthType, @Param() params: UploadSessionParamsDto) {
+    return this.uploadService.getStatus(auth.userId, params.uploadSessionId)
+  }
+
+  /**
+   * 将原始二进制请求流交给分片服务，并在连接中断时取消存储请求。
+   * @remarks 请求结束后排空未消费字节并移除监听器，避免连接和事件处理器泄漏。
+   */
+  @ApiOperation({ summary: '上传一个原始二进制分片' })
+  @ApiConsumes('application/octet-stream')
+  @ApiBody({
+    schema: {
+      type: 'string',
+      format: 'binary',
+    },
+  })
+  @HttpCode(HttpStatus.OK)
+  @Put(':uploadSessionId/parts/:partNumber')
+  uploadPart(
+    @CurrentAuth() auth: CurrentAuthType,
+    @Param() params: UploadPartParamsDto,
+    @Headers('content-length') contentLength: string | undefined,
+    @Headers('content-encoding') contentEncoding: string | undefined,
+    @Req() request: OctetStreamRequest,
+    @Res({ passthrough: true }) reply: FastifyReply,
+  ) {
+    const abortController = new AbortController()
+    /**
+     * 在客户端中止请求时触发存储上传的取消信号。
+     */
+    const onAborted = () => abortController.abort()
+    /**
+     * 响应尚未正常结束而连接已关闭时，取消正在进行的存储上传。
+     */
+    const onReplyClose = () => {
+      if (!reply.raw.writableEnded) {
+        abortController.abort()
+      }
+    }
+
+    request.raw.once('aborted', onAborted)
+    reply.raw.once('close', onReplyClose)
+
+    return this.uploadService
+      .uploadPart({
+        ownerId: auth.userId,
+        uploadSessionId: params.uploadSessionId,
+        partNumber: params.partNumber,
+        contentLength,
+        contentEncoding,
+        body: request.body,
+        abortSignal: abortController.signal,
+      })
+      /** 无论前置校验还是存储操作成功与否，均排空请求流并释放连接监听器。 */
+      .finally(() => {
+        // 前置校验失败时 Service 可能尚未 pipe 请求体。
+        // 统一排空剩余字节，避免连接一直占用到代理超时；这里不会聚合 Buffer。
+        if (!request.body.readableEnded && !request.body.destroyed) {
+          request.body.resume()
+        }
+
+        request.raw.off('aborted', onAborted)
+        reply.raw.off('close', onReplyClose)
+      })
+  }
+
+  /**
+   * 以当前用户身份请求校验并完成指定分片上传。
+   */
+  @ApiOperation({ summary: '校验并完成分片上传' })
+  @HttpCode(HttpStatus.OK)
+  @Post(':uploadSessionId/complete')
+  complete(@CurrentAuth() auth: CurrentAuthType, @Param() params: UploadSessionParamsDto) {
+    return this.uploadService.complete(auth.userId, params.uploadSessionId)
+  }
+
+  /**
+   * 以当前用户身份取消指定上传会话并清理存储分片。
+   */
+  @ApiOperation({ summary: '取消分片上传' })
+  @Delete(':uploadSessionId')
+  abort(@CurrentAuth() auth: CurrentAuthType, @Param() params: UploadSessionParamsDto) {
+    return this.uploadService.abort(auth.userId, params.uploadSessionId)
+  }
+}

package/template-vp-monorepo-react-nestjs/apps/server/src/modules/upload/upload.errors.ts

@@ -0,0 +1,80 @@
+import type { AppErrorDefinition } from '@/common/exceptions/app.exception'
+import { HttpStatus } from '@nestjs/common'
+
+export const UPLOAD_ERRORS = {
+  FILE_EMPTY: {
+    code: 'UPLOAD.FILE_EMPTY',
+    publicMessage: '空文件不能使用分片上传',
+    status: HttpStatus.UNPROCESSABLE_ENTITY,
+  },
+  FILE_TOO_LARGE: {
+    code: 'UPLOAD.FILE_TOO_LARGE',
+    publicMessage: '文件超过允许的最大大小',
+    status: HttpStatus.PAYLOAD_TOO_LARGE,
+  },
+  CLIENT_UPLOAD_ID_CONFLICT: {
+    code: 'UPLOAD.CLIENT_UPLOAD_ID_CONFLICT',
+    publicMessage: '该客户端上传标识已用于另一个文件',
+    status: HttpStatus.CONFLICT,
+  },
+  SESSION_NOT_FOUND: {
+    code: 'UPLOAD.SESSION_NOT_FOUND',
+    publicMessage: '上传会话不存在',
+    status: HttpStatus.NOT_FOUND,
+  },
+  INVALID_STATE: {
+    code: 'UPLOAD.INVALID_STATE',
+    publicMessage: '当前上传状态不允许执行此操作',
+    status: HttpStatus.CONFLICT,
+  },
+  UPLOAD_EXPIRED: {
+    code: 'UPLOAD.UPLOAD_EXPIRED',
+    publicMessage: '上传会话已过期，请重新开始',
+    status: HttpStatus.GONE,
+  },
+  PART_NUMBER_INVALID: {
+    code: 'UPLOAD.PART_NUMBER_INVALID',
+    publicMessage: '分片编号超出当前文件范围',
+    status: HttpStatus.UNPROCESSABLE_ENTITY,
+  },
+  PART_LENGTH_REQUIRED: {
+    code: 'UPLOAD.PART_LENGTH_REQUIRED',
+    publicMessage: '分片请求缺少有效的 Content-Length',
+    status: HttpStatus.LENGTH_REQUIRED,
+  },
+  PART_SIZE_MISMATCH: {
+    code: 'UPLOAD.PART_SIZE_MISMATCH',
+    publicMessage: '分片实际大小与预期不一致',
+    status: HttpStatus.UNPROCESSABLE_ENTITY,
+  },
+  UNSUPPORTED_CONTENT_ENCODING: {
+    code: 'UPLOAD.UNSUPPORTED_CONTENT_ENCODING',
+    publicMessage: '分片请求不支持压缩编码',
+    status: HttpStatus.UNSUPPORTED_MEDIA_TYPE,
+  },
+  PARTS_INCOMPLETE: {
+    code: 'UPLOAD.PARTS_INCOMPLETE',
+    publicMessage: '分片尚未全部上传或大小不正确',
+    status: HttpStatus.CONFLICT,
+  },
+  PARTS_STILL_ACTIVE: {
+    code: 'UPLOAD.PARTS_STILL_ACTIVE',
+    publicMessage: '仍有分片正在上传，请稍后重试',
+    status: HttpStatus.CONFLICT,
+  },
+  PART_UPLOAD_ABORTED: {
+    code: 'UPLOAD.PART_UPLOAD_ABORTED',
+    publicMessage: '分片上传已取消',
+    status: HttpStatus.REQUEST_TIMEOUT,
+  },
+  STORAGE_UPLOAD_MISSING: {
+    code: 'UPLOAD.STORAGE_UPLOAD_MISSING',
+    publicMessage: '存储中的上传会话已失效，请重新开始',
+    status: HttpStatus.GONE,
+  },
+  STORAGE_UNAVAILABLE: {
+    code: 'UPLOAD.STORAGE_UNAVAILABLE',
+    publicMessage: '文件存储服务暂时不可用，请稍后重试',
+    status: HttpStatus.SERVICE_UNAVAILABLE,
+  },
+} as const satisfies Record<string, AppErrorDefinition>

package/template-vp-monorepo-react-nestjs/apps/server/src/modules/upload/upload.module.ts

@@ -0,0 +1,20 @@
+import { Module } from '@nestjs/common'
+import { MinioStorageAdapter } from './storage/minio-storage.adapter'
+import { STORAGE_PORT } from './storage/storage.port'
+import { UploadController } from './upload.controller'
+import { UploadRepository } from './upload.reponsitory'
+import { UploadService } from './upload.service'
+
+@Module({
+  controllers: [UploadController],
+  providers: [
+    UploadRepository,
+    UploadService,
+    MinioStorageAdapter,
+    {
+      provide: STORAGE_PORT,
+      useExisting: MinioStorageAdapter,
+    },
+  ],
+})
+export class UploadModule {}