@hed-hog/core 0.0.401 → 0.0.404

package/src/file/file.service.ts

@@ -1,4 +1,5 @@
 import {
+  DeleteObjectCommand,
   GetBucketNotificationConfigurationCommand,
   GetObjectCommand,
   HeadBucketCommand,
@@ -25,6 +26,7 @@ import { PaginationDTO, PaginationService } from '@hed-hog/api-pagination';
 import { PrismaService } from '@hed-hog/api-prisma';
 import {
   BadRequestException,
+  ConflictException,
   ForbiddenException,
   forwardRef,
   Inject,
@@ -57,6 +59,8 @@ import {
 } from './image-optimization.service';
 import {
   AbstractProvider,
+  MultipartUploadPart,
+  supportsDirectUpload,
   supportsServerSideCopy,
 } from './provider/abstract.provider';
 import { ProviderFactory } from './provider/provider.factory';
@@ -82,6 +86,49 @@ const PROVIDER_CACHE_TTL_MS = Math.max(
   Number(process.env.FILE_PROVIDER_CACHE_TTL_MS ?? 30_000),
 );
 
+/** Upload direto em partes: mínimo de 16 MiB por parte e até 9000 partes (regra do Studio). */
+export const DIRECT_UPLOAD_MIN_PART_BYTES = 16 * 1024 * 1024;
+export const DIRECT_UPLOAD_MAX_PARTS = 9000;
+/** Validade do token e das URLs de parte: um vídeo grande em conexão lenta leva horas. */
+export const DIRECT_UPLOAD_DEFAULT_TTL_SECONDS = 86_400;
+const DIRECT_UPLOAD_TOKEN_TYPE = 'file-direct-upload';
+/**
+ * `file.size` é BIGINT no banco, e o PrismaService o devolve como number: o teto real é o
+ * maior inteiro exato de um number (~9 PB).
+ */
+const FILE_SIZE_COLUMN_MAX = Number.MAX_SAFE_INTEGER;
+
+export type DirectUploadFileInfo = {
+  originalname: string;
+  mimetype: string;
+  size: number;
+  isPrivate?: boolean;
+  createdByUserId?: number | null;
+  /** Mesmo sentido do `uploadFromPath`: o chamador já conferiu o tipo pelas regras dele. */
+  skipMimeValidation?: boolean;
+  /**
+   * O chamador já conferiu o tamanho pelas regras dele: pula a `storage-max-size`. O teto
+   * da coluna `file.size` continua valendo.
+   */
+  skipSizeValidation?: boolean;
+};
+
+export type CompleteDirectUploadInput = DirectUploadFileInfo & {
+  key: string;
+  uploadId: string;
+  destination: string;
+  parts: MultipartUploadPart[];
+  /**
+   * Confere os primeiros `bytes` do objeto gravado (ex.: sniff do tipo real). Se `check`
+   * lançar, o objeto é apagado e o erro sobe como veio; se devolver um mimetype, é ele
+   * que vai para a linha `file` (o tipo real, no lugar do declarado).
+   */
+  verifyHead?: {
+    bytes: number;
+    check: (head: Buffer) => string | void | Promise<string | void>;
+  };
+};
+
 /** Raw `?w=`/`?f=`/`?q=` query values for the public image endpoint. */
 export type ImageVariantRequest = {
   width?: number | string;
@@ -398,7 +445,7 @@ export class FileService implements OnModuleInit {
 
     const provider = await this.getProvider();
     const providerPath = this.resolveProviderPath(file);
-    const totalSize: number = file.size ?? 0;
+    const totalSize = Number(file.size ?? 0);
 
     const resolvedStart = start ?? 0;
     const resolvedEnd = end !== undefined ? end : totalSize > 0 ? totalSize - 1 : undefined;
@@ -783,6 +830,292 @@ export class FileService implements OnModuleInit {
     return file;
   }
 
+  /**
+   * O storage ativo aceita upload direto em partes assinadas (S3 e Spaces)? Sem provider
+   * configurado, ou com um que não implementa, é `false` — quem oferece o caminho cai no
+   * upload pela API.
+   */
+  async supportsDirectUpload(): Promise<boolean> {
+    try {
+      return supportsDirectUpload(await this.getProvider());
+    } catch {
+      return false;
+    }
+  }
+
+  private async getDirectUploadProvider() {
+    const provider = await this.getProvider();
+    if (!supportsDirectUpload(provider)) {
+      throw new ConflictException('The file storage does not support direct uploads');
+    }
+    return provider;
+  }
+
+  /** `key` gravada pelo start: `${destination}/${uuid.ext}`, sem subpasta extra. */
+  private assertDirectUploadKey(key: string, destination: string): string {
+    const prefix = `${destination}/`;
+    const storedFilename = typeof key === 'string' && key.startsWith(prefix)
+      ? key.slice(prefix.length)
+      : '';
+
+    if (!storedFilename || storedFilename.includes('/')) {
+      throw new BadRequestException('Invalid direct upload key');
+    }
+
+    return storedFilename;
+  }
+
+  /**
+   * Abre um upload multipart direto no storage. A chave segue o mesmo formato que o
+   * `uploadFromPath` grava (`file.location` = destination, `file.path` = uuid.ext do
+   * `getFilename` do provider), então o arquivo final é servido como qualquer outro.
+   *
+   * Nada vai para o banco aqui: o estado entre as chamadas fica com quem chamou (token
+   * assinado, ver `signDirectUploadToken`) e a linha `file` só nasce no complete.
+   * Tamanho da parte com a regra do Studio: no mínimo 16 MiB e no máximo 9000 partes.
+   */
+  async startDirectUpload(destination: string, info: DirectUploadFileInfo) {
+    if (!destination) {
+      throw new BadRequestException(`No destination provided`);
+    }
+
+    const size = Number(info?.size);
+    if (!Number.isInteger(size) || size <= 0) {
+      throw new BadRequestException(`No size provided`);
+    }
+
+    // Acima disso o tamanho não cabe exato num number, e a linha não teria como ser lida.
+    if (size > FILE_SIZE_COLUMN_MAX) {
+      throw new BadRequestException(`File too large: ${size} bytes`);
+    }
+
+    const mimetype = String(info?.mimetype ?? '').trim();
+    if (!info?.skipMimeValidation && !(await this.acceptMimetypes(mimetype))) {
+      throw new BadRequestException(`Invalid file type: ${mimetype}`);
+    }
+
+    if (!info?.skipSizeValidation && !(await this.maxFileSize(size))) {
+      throw new BadRequestException(`File too large: ${size} bytes`);
+    }
+
+    const originalname = this.normalizeUploadedFilename(
+      String(info?.originalname ?? '').trim() || 'file',
+    );
+    const provider = await this.getDirectUploadProvider();
+    const key = `${destination}/${provider.getFilename(originalname)}`;
+    const uploadId = await provider.createMultipartUpload(key, mimetype || 'application/octet-stream');
+
+    const partSize = Math.max(
+      DIRECT_UPLOAD_MIN_PART_BYTES,
+      Math.ceil(size / DIRECT_UPLOAD_MAX_PARTS),
+    );
+
+    return {
+      key,
+      uploadId,
+      partSize,
+      partCount: Math.max(1, Math.ceil(size / partSize)),
+    };
+  }
+
+  /** URLs assinadas de PUT para as partes pedidas (1-based, sem repetir). */
+  async signDirectUploadParts(
+    key: string,
+    uploadId: string,
+    partNumbers: number[],
+    ttlSeconds = DIRECT_UPLOAD_DEFAULT_TTL_SECONDS,
+  ) {
+    if (!key || !uploadId) {
+      throw new BadRequestException('Invalid direct upload');
+    }
+
+    const numbers = Array.isArray(partNumbers) ? partNumbers.map(Number) : [];
+    if (
+      numbers.length === 0 ||
+      numbers.some((n) => !Number.isInteger(n) || n < 1 || n > DIRECT_UPLOAD_MAX_PARTS)
+    ) {
+      throw new BadRequestException('Invalid part numbers');
+    }
+
+    const provider = await this.getDirectUploadProvider();
+    return provider.signUploadParts(
+      key,
+      uploadId,
+      [...new Set(numbers)],
+      Math.max(60, Math.floor(ttlSeconds)),
+    );
+  }
+
+  /** Aborta o multipart (as partes já enviadas somem do bucket). Upload já fechado não é erro. */
+  async abortDirectUpload(key: string, uploadId: string): Promise<void> {
+    if (!key || !uploadId) {
+      throw new BadRequestException('Invalid direct upload');
+    }
+
+    const provider = await this.getDirectUploadProvider();
+    try {
+      await provider.abortMultipartUpload(key, uploadId);
+    } catch (error: any) {
+      const status = error?.$metadata?.httpStatusCode;
+      if (error?.name === 'NoSuchUpload' || error?.Code === 'NoSuchUpload' || status === 404) {
+        return;
+      }
+      throw error;
+    }
+  }
+
+  /**
+   * Fecha o multipart e registra o arquivo. Confere o que o storage gravou antes de criar
+   * a linha — tamanho pelo HeadObject (tem que bater com o declarado no start), teto
+   * `storage-max-size`, allowlist `storage-accept-mimetype` e, quando pedido, os primeiros
+   * bytes (`verifyHead`) — e apaga o objeto quando algo não confere: um upload recusado
+   * não deixa blob órfão no bucket.
+   */
+  async completeDirectUpload(input: CompleteDirectUploadInput) {
+    const { key, uploadId, destination } = input ?? ({} as CompleteDirectUploadInput);
+
+    if (!destination) {
+      throw new BadRequestException(`No destination provided`);
+    }
+    if (!uploadId) {
+      throw new BadRequestException('Invalid direct upload');
+    }
+
+    const storedFilename = this.assertDirectUploadKey(key, destination);
+
+    const parts = Array.isArray(input.parts) ? input.parts : [];
+    if (
+      parts.length === 0 ||
+      parts.some(
+        (part) =>
+          !Number.isInteger(Number(part?.partNumber)) ||
+          Number(part.partNumber) < 1 ||
+          typeof part?.etag !== 'string' ||
+          !part.etag.trim(),
+      )
+    ) {
+      throw new BadRequestException('Invalid upload parts');
+    }
+
+    const provider = await this.getDirectUploadProvider();
+
+    try {
+      await provider.completeMultipartUpload(
+        key,
+        uploadId,
+        parts.map((part) => ({ partNumber: Number(part.partNumber), etag: part.etag.trim() })),
+      );
+    } catch (error: any) {
+      const status = error?.$metadata?.httpStatusCode;
+      if (typeof status === 'number' && status >= 400 && status < 500) {
+        // Parte faltando, ETag errado, parte pequena demais: o multipart não tem conserto.
+        await this.abortDirectUpload(key, uploadId).catch(() => undefined);
+        throw new BadRequestException('The uploaded parts could not be assembled');
+      }
+      throw error;
+    }
+
+    const discard = async () => {
+      await provider.delete(key).catch((error) =>
+        this.logger.warn(`Failed to delete rejected direct upload ${key}: ${error?.message}`),
+      );
+    };
+
+    const declaredSize = Number(input.size);
+    const storedSize = this.getFileSizeFromMetadata(await provider.metaData(key));
+
+    if (storedSize === undefined || storedSize !== declaredSize) {
+      await discard();
+      throw new BadRequestException(
+        `Stored size (${storedSize ?? 'unknown'} bytes) differs from the declared size (${declaredSize} bytes)`,
+      );
+    }
+
+    if (
+      storedSize > FILE_SIZE_COLUMN_MAX ||
+      (!input.skipSizeValidation && !(await this.maxFileSize(storedSize)))
+    ) {
+      await discard();
+      throw new BadRequestException(`File too large: ${storedSize} bytes`);
+    }
+
+    let mimetype = String(input.mimetype ?? '').trim();
+
+    if (input.verifyHead) {
+      try {
+        const verified = await input.verifyHead.check(
+          await provider.readHead(key, input.verifyHead.bytes),
+        );
+        if (typeof verified === 'string' && verified.trim()) {
+          mimetype = verified.trim();
+        }
+      } catch (error) {
+        await discard();
+        throw error;
+      }
+    }
+
+    if (!input.skipMimeValidation && !(await this.acceptMimetypes(mimetype))) {
+      await discard();
+      throw new BadRequestException(`Invalid file type: ${mimetype}`);
+    }
+
+    try {
+      return await this.prismaService.file.create({
+        data: {
+          filename: this.normalizeUploadedFilename(
+            String(input.originalname ?? '').trim() || storedFilename,
+          ),
+          path: storedFilename,
+          provider_id: this.providerId,
+          location: destination,
+          mimetype_id: await this.getMimeType(mimetype),
+          size: storedSize,
+          is_private: input.isPrivate ?? false,
+          created_by_user_id: input.createdByUserId ?? null,
+        },
+      });
+    } catch (error) {
+      await discard();
+      throw error;
+    }
+  }
+
+  /**
+   * Token opaco que carrega o estado de um upload direto entre as chamadas (sem tabela):
+   * JWT HS256 com o JWT_SECRET da API e o tipo `file-direct-upload`, como o token do
+   * `file-upload-ticket`. Quem chama põe nas claims o que precisa amarrar (marca, usuário).
+   */
+  signDirectUploadToken(
+    claims: Record<string, unknown>,
+    ttlSeconds = DIRECT_UPLOAD_DEFAULT_TTL_SECONDS,
+  ): string {
+    return this.jwtService.sign(
+      { ...claims, t: DIRECT_UPLOAD_TOKEN_TYPE },
+      {
+        secret: process.env.JWT_SECRET,
+        expiresIn: `${Math.max(1, Math.floor(ttlSeconds))}s`,
+      },
+    );
+  }
+
+  /** Claims do token de upload direto; assinatura inválida, expirado ou de outro tipo = 400. */
+  readDirectUploadToken<T extends Record<string, unknown>>(token: string): T {
+    let payload: any;
+
+    try {
+      payload = this.jwtService.verify(String(token ?? ''), { secret: process.env.JWT_SECRET });
+    } catch {
+      throw new BadRequestException('Invalid or expired upload token');
+    }
+
+    if (payload?.t !== DIRECT_UPLOAD_TOKEN_TYPE) {
+      throw new BadRequestException('Token is not a direct upload token');
+    }
+
+    return payload as T;
+  }
+
   /**
    * Mimetypes de imagem que `optimizeBuffer` devolve intactos por decisão de projeto
    * (vetor não vira bitmap, GIF animado não vira quadro único). Enfileirá-los para
@@ -881,7 +1214,7 @@ export class FileService implements OnModuleInit {
   }> {
     const { file, buffer } = await this.getBuffer(fileId);
     const currentMimetype = file.file_mimetype.name;
-    const oldSize = file.size ?? buffer.length;
+    const oldSize = Number(file.size ?? buffer.length);
 
     if (!currentMimetype.startsWith('image/')) {
       return { changed: false, oldSize, newSize: oldSize, mimetypeChanged: false };
@@ -2014,6 +2347,34 @@ export class FileService implements OnModuleInit {
     });
   }
 
+  /**
+   * Apaga um objeto de bucket avulso, com as credenciais de quem o configurou.
+   *
+   * Mesmo motivo do `headS3Object` e do `signS3Object`: o bucket e de outra
+   * library (o Studio guarda os brutos no perfil de armazenamento dele), e nao o
+   * destino ativo do core. O S3 responde 204 tambem para key inexistente, entao
+   * apagar duas vezes nao e erro.
+   */
+  async deleteS3Object(params: {
+    accessKeyId: string;
+    secretAccessKey: string;
+    sessionToken?: string;
+    region: string;
+    bucket: string;
+    key: string;
+  }): Promise<void> {
+    const s3 = new S3Client({
+      region: params.region,
+      credentials: {
+        accessKeyId: params.accessKeyId,
+        secretAccessKey: params.secretAccessKey,
+        ...(params.sessionToken ? { sessionToken: params.sessionToken } : {}),
+      },
+    });
+
+    await s3.send(new DeleteObjectCommand({ Bucket: params.bucket, Key: params.key }));
+  }
+
   async headS3Bucket(params: {
     accessKeyId: string;
     secretAccessKey: string;

package/src/file/provider/abstract.provider.ts

@@ -40,6 +40,47 @@ export function supportsServerSideCopy(
   return typeof (provider as Partial<ProviderServerSideCopy>).copy === 'function';
 }
 
+/** Parte ja enviada de um upload multipart: numero (1-based) e o ETag que o storage devolveu. */
+export type MultipartUploadPart = { partNumber: number; etag: string };
+
+/**
+ * Upload direto do cliente para o storage, em partes com URL assinada: os bytes nao
+ * atravessam a API (nem o teto de 100 MB por requisicao do Cloudflare na frente dela).
+ *
+ * Opcional como o `copy`: so S3 e Spaces implementam, e quem chama confere com
+ * `supportsDirectUpload` antes de oferecer o caminho.
+ */
+export interface ProviderDirectUpload {
+  createMultipartUpload(key: string, contentType: string): Promise<string>;
+  signUploadParts(
+    key: string,
+    uploadId: string,
+    partNumbers: number[],
+    ttlSeconds: number,
+  ): Promise<Array<{ partNumber: number; url: string }>>;
+  completeMultipartUpload(
+    key: string,
+    uploadId: string,
+    parts: MultipartUploadPart[],
+  ): Promise<void>;
+  abortMultipartUpload(key: string, uploadId: string): Promise<void>;
+  /** Os primeiros `bytes` do objeto (Range GET), para conferir o tipo real. */
+  readHead(key: string, bytes: number): Promise<Buffer>;
+}
+
+export function supportsDirectUpload(
+  provider: AbstractProvider,
+): provider is AbstractProvider & ProviderDirectUpload {
+  const candidate = provider as Partial<ProviderDirectUpload>;
+  return (
+    typeof candidate.createMultipartUpload === 'function' &&
+    typeof candidate.signUploadParts === 'function' &&
+    typeof candidate.completeMultipartUpload === 'function' &&
+    typeof candidate.abortMultipartUpload === 'function' &&
+    typeof candidate.readHead === 'function'
+  );
+}
+
 export abstract class AbstractProvider implements IProvider {
   abstract uploadFromString(
     destination: string,
@@ -67,6 +108,22 @@ export abstract class AbstractProvider implements IProvider {
    */
   copy?(sourcePath: string, destination: string, filename: string): Promise<string>;
 
+  /** Upload direto em partes; ver `ProviderDirectUpload` e `supportsDirectUpload`. */
+  createMultipartUpload?(key: string, contentType: string): Promise<string>;
+  signUploadParts?(
+    key: string,
+    uploadId: string,
+    partNumbers: number[],
+    ttlSeconds: number,
+  ): Promise<Array<{ partNumber: number; url: string }>>;
+  completeMultipartUpload?(
+    key: string,
+    uploadId: string,
+    parts: MultipartUploadPart[],
+  ): Promise<void>;
+  abortMultipartUpload?(key: string, uploadId: string): Promise<void>;
+  readHead?(key: string, bytes: number): Promise<Buffer>;
+
   getExtension(filename: string) {
     return filename.split('.').pop();
   }

package/src/file/provider/s3-multipart.spec.ts

@@ -0,0 +1,123 @@
+import { afterEach, describe, expect, it, jest } from '@jest/globals';
+import {
+  AbortMultipartUploadCommand,
+  CompleteMultipartUploadCommand,
+  CreateMultipartUploadCommand,
+  GetObjectCommand,
+  S3Client,
+} from '@aws-sdk/client-s3';
+import { Readable } from 'stream';
+import { supportsDirectUpload } from './abstract.provider';
+import { S3Provider } from './s3.provider';
+import { SpacesProvider } from './spaces.provider';
+
+/**
+ * Upload multipart assinado no S3 e no Spaces, com o S3Client mockado: o que vai em cada
+ * comando, a URL de parte assinada de verdade (o presigner roda offline) e o Range GET.
+ */
+const s3Settings = {
+  'storage-s3-key': 'AKIAEXAMPLE',
+  'storage-s3-secret': 'secret',
+  'storage-s3-region': 'us-east-1',
+  'storage-s3-bucket': 'bucket-s3',
+};
+
+const spacesSettings = {
+  'storage-spaces-key': 'DOEXAMPLE',
+  'storage-spaces-secret': 'secret',
+  'storage-spaces-region': 'nyc3',
+  'storage-spaces-bucket': 'spaces-hub',
+};
+
+function mockSend(impl: (command: any) => any) {
+  return jest.spyOn(S3Client.prototype, 'send').mockImplementation(async (command: any) => impl(command));
+}
+
+type ProviderCase = [string, () => any, string, string | undefined, string];
+
+describe.each<ProviderCase>([
+  ['S3', () => new S3Provider(s3Settings), 'bucket-s3', undefined, 'amazonaws.com'],
+  ['Spaces', () => new SpacesProvider(spacesSettings), 'spaces-hub', 'public-read', 'digitaloceanspaces.com'],
+])('%s provider — upload direto', (_name, build, bucket, acl, host) => {
+  afterEach(() => jest.restoreAllMocks());
+
+  it('implementa o contrato de upload direto', () => {
+    expect(supportsDirectUpload(build())).toBe(true);
+  });
+
+  it('createMultipartUpload manda bucket, chave e tipo e devolve o UploadId', async () => {
+    const send = mockSend(() => ({ UploadId: 'up-1' }));
+
+    await expect(build().createMultipartUpload('social/brands/1/a.mp4', 'video/mp4')).resolves.toBe('up-1');
+
+    const command = send.mock.calls[0][0] as any;
+    expect(command).toBeInstanceOf(CreateMultipartUploadCommand);
+    expect(command.input).toEqual({
+      Bucket: bucket,
+      Key: 'social/brands/1/a.mp4',
+      ContentType: 'video/mp4',
+      ...(acl ? { ACL: acl } : {}),
+    });
+  });
+
+  it('createMultipartUpload sem UploadId é erro', async () => {
+    mockSend(() => ({}));
+    await expect(build().createMultipartUpload('k', 'video/mp4')).rejects.toThrow('multipart upload id');
+  });
+
+  it('signUploadParts devolve uma URL assinada de PUT por parte', async () => {
+    const parts = await build().signUploadParts('social/brands/1/a.mp4', 'up-1', [1, 2], 3600);
+
+    expect(parts.map((p) => p.partNumber)).toEqual([1, 2]);
+    for (const part of parts) {
+      const url = new URL(part.url);
+      expect(url.host).toContain(host);
+      expect(url.searchParams.get('uploadId')).toBe('up-1');
+      expect(url.searchParams.get('partNumber')).toBe(String(part.partNumber));
+      expect(url.searchParams.get('X-Amz-Expires')).toBe('3600');
+      expect(url.searchParams.get('X-Amz-Signature')).toBeTruthy();
+    }
+  });
+
+  it('completeMultipartUpload ordena as partes pelo número', async () => {
+    const send = mockSend(() => ({}));
+
+    await build().completeMultipartUpload('k', 'up-1', [
+      { partNumber: 2, etag: '"b"' },
+      { partNumber: 1, etag: '"a"' },
+    ]);
+
+    const command = send.mock.calls[0][0] as any;
+    expect(command).toBeInstanceOf(CompleteMultipartUploadCommand);
+    expect(command.input).toEqual({
+      Bucket: bucket,
+      Key: 'k',
+      UploadId: 'up-1',
+      MultipartUpload: {
+        Parts: [
+          { PartNumber: 1, ETag: '"a"' },
+          { PartNumber: 2, ETag: '"b"' },
+        ],
+      },
+    });
+  });
+
+  it('abortMultipartUpload aborta o upload da chave', async () => {
+    const send = mockSend(() => ({}));
+    await build().abortMultipartUpload('k', 'up-1');
+    const command = send.mock.calls[0][0] as any;
+    expect(command).toBeInstanceOf(AbortMultipartUploadCommand);
+    expect(command.input).toEqual({ Bucket: bucket, Key: 'k', UploadId: 'up-1' });
+  });
+
+  it('readHead pede só os primeiros bytes e corta o que vier a mais', async () => {
+    const send = mockSend(() => ({ Body: Readable.from([Buffer.from('0123456789'), Buffer.from('abc')]) }));
+
+    const head = await build().readHead('k', 4);
+
+    const command = send.mock.calls[0][0] as any;
+    expect(command).toBeInstanceOf(GetObjectCommand);
+    expect(command.input).toEqual({ Bucket: bucket, Key: 'k', Range: 'bytes=0-3' });
+    expect(head.toString()).toBe('0123');
+  });
+});