@hed-hog/lms 0.0.361 → 0.0.365

package/src/course/lms-bulk-upload-automation.service.ts

@@ -0,0 +1,707 @@
+import { GetObjectCommand, S3Client } from '@aws-sdk/client-s3';
+import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
+import { PrismaService } from '@hed-hog/api-prisma';
+import {
+    FileService,
+    SettingService,
+    WebhookCommandRegistry,
+    WebhookIntegrationService,
+} from '@hed-hog/core';
+import { QueueHandlerRegistry, QueueJobService } from '@hed-hog/queue';
+import {
+    BadRequestException,
+    Inject,
+    Injectable,
+    Logger,
+    OnModuleInit,
+    forwardRef,
+} from '@nestjs/common';
+import { createWriteStream, promises as fs } from 'fs';
+import { tmpdir } from 'os';
+import { basename, extname, join } from 'path';
+import { Readable } from 'stream';
+import { pipeline } from 'stream/promises';
+import { CourseVideoConversionService } from './course-video-conversion.service';
+import { CourseVideoHlsService } from './course-video-hls.service';
+import {
+    LMS_BULK_UPLOAD_DOWNLOAD_ORIGINAL_JOB,
+    LMS_BULK_UPLOAD_RECEIVE_VIDEO_COMMAND,
+} from './lms-bulk-upload.constants';
+
+type StorageContext = {
+  region: string;
+  bucket: string;
+  accessKeyId: string;
+  secretAccessKey: string;
+  sessionToken?: string;
+  roleArn?: string;
+  externalId?: string;
+  durationSeconds: number;
+};
+
+@Injectable()
+export class LmsBulkUploadAutomationService implements OnModuleInit {
+  private readonly logger = new Logger(LmsBulkUploadAutomationService.name);
+
+  constructor(
+    @Inject(forwardRef(() => PrismaService))
+    private readonly prisma: PrismaService,
+    @Inject(forwardRef(() => SettingService))
+    private readonly settingService: SettingService,
+    @Inject(forwardRef(() => FileService))
+    private readonly fileService: FileService,
+    @Inject(forwardRef(() => QueueHandlerRegistry))
+    private readonly queueHandlers: QueueHandlerRegistry,
+    @Inject(forwardRef(() => QueueJobService))
+    private readonly queueJobService: QueueJobService,
+    @Inject(forwardRef(() => WebhookCommandRegistry))
+    private readonly commandRegistry: WebhookCommandRegistry,
+    @Inject(forwardRef(() => WebhookIntegrationService))
+    private readonly webhookIntegrationService: WebhookIntegrationService,
+    @Inject(forwardRef(() => CourseVideoConversionService))
+    private readonly courseVideoConversionService: CourseVideoConversionService,
+    @Inject(forwardRef(() => CourseVideoHlsService))
+    private readonly courseVideoHlsService: CourseVideoHlsService,
+  ) {}
+
+  onModuleInit(): void {
+    this.queueHandlers.register(LMS_BULK_UPLOAD_DOWNLOAD_ORIGINAL_JOB, this as any);
+    this.commandRegistry.register({
+      slug: LMS_BULK_UPLOAD_RECEIVE_VIDEO_COMMAND,
+      name: 'Receber vídeo do upload em massa',
+      description:
+        'Reconhece o arquivo recebido no upload em massa, vincula a aula e agenda o download do vídeo original do S3.',
+      inputSchema: {
+        type: 'object',
+        properties: {
+          bucket: { type: 'string' },
+          key: { type: 'string' },
+          fileName: { type: 'string' },
+          uploadId: { type: 'string' },
+        },
+        additionalProperties: true,
+      },
+      handler: async (_params, context) => this.handleWebhookReceive(context ?? {}),
+    });
+
+    this.logger.log(
+      `Registered webhook command and queue handler for LMS bulk upload automation`,
+    );
+  }
+
+  private async handleWebhookReceive(context: Record<string, unknown>) {
+    const payload = this.normalizeWebhookBody(context.webhookBody ?? context);
+    const bucket = payload.bucket || '';
+    const key = payload.key || '';
+    const fileName = payload.fileName || basename(key);
+
+    if (!bucket || !key) {
+      throw new BadRequestException('Webhook payload does not contain bucket and key.');
+    }
+
+    const match = await this.findLessonMatch(fileName);
+    const uploadItem = await this.findUploadItem(payload.uploadId, fileName, key);
+    const now = new Date();
+
+    if (uploadItem) {
+      await this.prisma.$executeRawUnsafe(
+        `UPDATE "lms_bulk_upload_item"
+         SET
+           "status" = 'received',
+           "progress_percent" = 100,
+           "error_message" = NULL,
+           "uploaded_key" = $2,
+           "received_at" = $3,
+           "completed_at" = $3,
+           "matched_course_id" = $4,
+           "matched_session_id" = $5,
+           "matched_lesson_id" = $6,
+           "updated_at" = $3
+         WHERE "id" = $1`,
+        uploadItem.id,
+        key,
+        now,
+        match?.courseId ?? null,
+        match?.sessionId ?? null,
+        match?.lessonId ?? null,
+      );
+    }
+
+    if (!match) {
+      if (uploadItem) {
+        await this.prisma.$executeRawUnsafe(
+          `UPDATE "lms_bulk_upload_item"
+           SET "status" = 'lesson_not_found', "received_at" = $2, "updated_at" = $2
+           WHERE "id" = $1`,
+          uploadItem.id,
+          now,
+        );
+      }
+      return {
+        received: true,
+        matched: false,
+        uploadItemId: uploadItem?.id ?? null,
+      };
+    }
+
+    const job = await this.queueJobService.enqueue({
+      type: LMS_BULK_UPLOAD_DOWNLOAD_ORIGINAL_JOB,
+      queueName: LMS_BULK_UPLOAD_DOWNLOAD_ORIGINAL_JOB,
+      payload: {
+        bucket,
+        key,
+        fileName,
+        uploadItemId: uploadItem?.id ?? null,
+        userId: uploadItem?.userId ?? payload.userId ?? 0,
+        courseId: match.courseId,
+        sessionId: match.sessionId,
+        lessonId: match.lessonId,
+        matchedCourseId: match.courseId,
+        matchedSessionId: match.sessionId,
+        matchedLessonId: match.lessonId,
+      },
+      maxAttempts: 3,
+      sourceModule: 'lms',
+      sourceEntity: 'lms_bulk_upload_item',
+      sourceEntityId: uploadItem?.id ? String(uploadItem.id) : fileName,
+      priority: -6,
+    }, uploadItem?.userId ?? payload.userId ?? undefined);
+
+    return {
+      received: true,
+      matched: true,
+      uploadItemId: uploadItem?.id ?? null,
+      queueJobId: job.id,
+      lesson: {
+        courseId: match.courseId,
+        sessionId: match.sessionId,
+        lessonId: match.lessonId,
+        courseTitle: match.courseTitle,
+        sessionTitle: match.sessionTitle,
+        lessonTitle: match.lessonTitle,
+        courseLogoFileId: match.courseLogoFileId,
+      },
+    };
+  }
+
+  async handle(job: { id: number; payload: Record<string, any> }) {
+    const payload = job.payload as {
+      bucket?: string;
+      key?: string;
+      fileName?: string;
+      uploadItemId?: number | null;
+      userId?: number;
+      courseId?: number;
+      sessionId?: number;
+      lessonId?: number;
+    };
+
+    const bucket = String(payload.bucket ?? '').trim();
+    const key = String(payload.key ?? '').trim();
+    const fileName = String(payload.fileName ?? '').trim() || basename(key);
+    const userId = Number(payload.userId ?? 0);
+    const courseId = Number(payload.courseId ?? 0);
+    const sessionId = Number(payload.sessionId ?? 0);
+    const lessonId = Number(payload.lessonId ?? 0);
+
+    if (!bucket || !key || !userId || !courseId || !sessionId || !lessonId) {
+      throw new BadRequestException('Invalid LMS bulk upload queue payload.');
+    }
+
+    const storage = await this.resolveStorageContext();
+    const tempCredentials = await this.fileService.getTemporaryCredentials({
+      accessKeyId: storage.accessKeyId,
+      secretAccessKey: storage.secretAccessKey,
+      sessionToken: storage.sessionToken,
+      region: storage.region,
+      roleArn: storage.roleArn,
+      externalId: storage.externalId,
+      durationSeconds: storage.durationSeconds,
+      sessionName: `lms-bulk-upload-u${userId}-${Date.now()}`,
+    });
+
+    const workDir = await fs.mkdtemp(join(tmpdir(), 'lms-bulk-upload-'));
+    const tempFilePath = join(
+      workDir,
+      `${Date.now()}-${basename(fileName || key)}${extname(fileName || key) || '.mp4'}`,
+    );
+
+    try {
+      const s3 = new S3Client({
+        region: storage.region,
+        credentials: {
+          accessKeyId: tempCredentials.AccessKeyId,
+          secretAccessKey: tempCredentials.SecretAccessKey,
+          sessionToken: tempCredentials.SessionToken,
+        },
+      });
+
+      const response = await s3.send(
+        new GetObjectCommand({ Bucket: bucket, Key: key }),
+      );
+      const body = response.Body as any;
+      if (!body) {
+        throw new BadRequestException('Could not read the S3 object body.');
+      }
+
+      const readable = typeof body.pipe === 'function' ? body : Readable.fromWeb(body);
+      await pipeline(readable, createWriteStream(tempFilePath));
+
+      const uploaded = await this.fileService.uploadFromPath(
+        'lms/lessons/originals',
+        tempFilePath,
+        {
+          originalname: fileName,
+          filename: fileName,
+          mimetype: this.inferVideoMimeType(fileName || key),
+        },
+      );
+
+      await this.courseVideoHlsService.enqueueHls({
+        userId,
+        courseId,
+        sessionId,
+        lessonId,
+        originalFileId: uploaded.id,
+      });
+
+      if (Number.isInteger(Number(payload.uploadItemId)) && Number(payload.uploadItemId) > 0) {
+        await this.prisma.$executeRawUnsafe(
+          `UPDATE "lms_bulk_upload_item"
+           SET "status" = 'done', "completed_at" = NOW(), "updated_at" = NOW()
+           WHERE "id" = $1`,
+          Number(payload.uploadItemId),
+        ).catch(() => undefined);
+      }
+
+      return {
+        success: true,
+        queueJobId: job.id,
+        originalFileId: uploaded.id,
+      };
+    } catch (error) {
+      if (Number.isInteger(Number(payload.uploadItemId)) && Number(payload.uploadItemId) > 0) {
+        await this.prisma.$executeRawUnsafe(
+          `UPDATE "lms_bulk_upload_item"
+           SET "status" = 'error', "error_message" = $2, "progress_percent" = 100, "updated_at" = NOW()
+           WHERE "id" = $1`,
+          Number(payload.uploadItemId),
+          error instanceof Error ? error.message : 'Unknown error',
+        ).catch(() => undefined);
+      }
+
+      throw error;
+    } finally {
+      await fs.rm(workDir, { recursive: true, force: true }).catch(() => undefined);
+    }
+  }
+
+  private async findUploadItem(
+    uploadId: string | undefined,
+    fileName: string,
+    key: string,
+  ): Promise<{ id: number; userId: number } | null> {
+    const rows = await this.prisma.$queryRawUnsafe<Array<{ id: number; user_id: number }>>(
+      `SELECT i."id", s."user_id"
+       FROM "lms_bulk_upload_item" i
+       INNER JOIN "lms_bulk_upload_session" s ON s."id" = i."session_id"
+       WHERE ($1::text IS NOT NULL AND i."upload_id" = $1)
+          OR i."file_name" = $2
+          OR i."uploaded_key" = $3
+          OR i."uploaded_key" = $2
+       ORDER BY i."id" DESC
+       LIMIT 1`,
+      uploadId || null,
+      fileName,
+      key,
+    );
+
+    const row = rows[0];
+    if (!row) {
+      return null;
+    }
+
+    return { id: Number(row.id), userId: Number(row.user_id) };
+  }
+
+  private async findLessonMatch(fileName: string) {
+    const parsed = this.parseFilename(fileName);
+    if (!parsed) {
+      return null;
+    }
+
+    const courses = await (this.prisma as any).course.findMany({
+      where: {
+        code: { equals: parsed.courseCode, mode: 'insensitive' },
+      },
+      select: {
+        id: true,
+        code: true,
+        title: true,
+        name: true,
+        slug: true,
+        course_image: {
+          select: {
+            file_id: true,
+            image_type: { select: { slug: true } },
+          },
+        },
+        course_module: {
+          orderBy: { order: 'asc' },
+          select: {
+            id: true,
+            title: true,
+            order: true,
+            course_lesson: {
+              orderBy: { order: 'asc' },
+              select: {
+                id: true,
+                title: true,
+                order: true,
+              },
+            },
+          },
+        },
+      },
+    });
+
+    for (const course of courses ?? []) {
+      for (const session of course.course_module ?? []) {
+        const sessionNumeric = parsed.sessionCode.replace(/[^0-9]/g, '');
+        const moduleNumeric = String(Math.max(Number(session.order ?? 0), 0)).padStart(2, '0');
+        if (!sessionNumeric || sessionNumeric !== moduleNumeric) {
+          continue;
+        }
+
+        for (const lesson of session.course_lesson ?? []) {
+          if (this.normalizeComparableText(lesson.title) !== parsed.lessonTitle) {
+            continue;
+          }
+
+          const logo = course.course_image?.find(
+            (image: any) => image.image_type?.slug === 'course-logo',
+          );
+
+          return {
+            courseId: Number(course.id),
+            courseTitle: String(course.title ?? course.name ?? ''),
+            courseSlug: String(course.slug ?? ''),
+            courseLogoFileId: logo?.file_id ? Number(logo.file_id) : null,
+            sessionId: Number(session.id),
+            sessionTitle: String(session.title ?? ''),
+            lessonId: Number(lesson.id),
+            lessonTitle: String(lesson.title ?? ''),
+          };
+        }
+      }
+    }
+
+    return null;
+  }
+
+  private parseFilename(fileName: string) {
+    const withoutTimestamp = fileName.replace(/^\d+[-_]+/, '');
+    const base = this.normalizeComparableText(withoutTimestamp.replace(/\.[^.]+$/, ''));
+    const parts = base.split('_').filter(Boolean);
+    if (parts.length < 3) {
+      return null;
+    }
+
+    const RESOLUTION_SUFFIXES = new Set(['1080', '720', '480', '360', '240', '4k', '2k', 'uhd', 'fhd', 'hd', 'sd']);
+    let titleParts = parts.slice(2);
+    if (titleParts.length > 1 && RESOLUTION_SUFFIXES.has(titleParts[titleParts.length - 1])) {
+      titleParts = titleParts.slice(0, -1);
+    }
+
+    return {
+      sessionCode: this.normalizeComparableText(parts[0]),
+      courseCode: this.normalizeComparableText(parts[1]),
+      lessonTitle: this.normalizeComparableText(titleParts.join('_')),
+    };
+  }
+
+  private normalizeWebhookBody(body: unknown) {
+    const raw = typeof body === 'string' ? this.safeJsonParse(body) : (body ?? {});
+    const records = Array.isArray((raw as any).Records)
+      ? (raw as any).Records
+      : Array.isArray((raw as any).records)
+        ? (raw as any).records
+        : [];
+    const record = records[0] ?? (raw as any).detail ?? raw;
+    const bucket = this.firstString(
+      (raw as any)?.bucketName,
+      typeof (raw as any)?.bucket === 'string' ? (raw as any)?.bucket : undefined,
+      (raw as any)?.bucket?.name,
+      record?.s3?.bucket?.name,
+      record?.bucket?.name,
+      record?.detail?.bucket?.name,
+    );
+    const key = this.decodeKey(
+      this.firstString(
+        (raw as any)?.key,
+        (raw as any)?.object?.key,
+        record?.s3?.object?.key,
+        record?.object?.key,
+        record?.detail?.object?.key,
+      ),
+    );
+
+    return {
+      bucket,
+      key,
+      fileName:
+        this.firstString(
+          (raw as any)?.fileName,
+          (raw as any)?.file_name,
+          record?.fileName,
+          record?.file_name,
+        ) || (key ? basename(key) : ''),
+      uploadId: this.firstString(
+        (raw as any)?.uploadId,
+        (raw as any)?.upload_id,
+        record?.uploadId,
+        record?.upload_id,
+      ),
+      userId:
+        Number(
+          (raw as any)?.userId ?? (raw as any)?.user_id ?? record?.userId ?? 0,
+        ) || 0,
+    };
+  }
+
+  private async resolveStorageContext(): Promise<StorageContext> {
+    const settings = await this.settingService.getSettingValues([
+      'lms-bulk-upload-storage-profile-id',
+      'lms-bulk-upload-sts-duration-seconds',
+    ]);
+
+    const storageProfileId = Number(settings['lms-bulk-upload-storage-profile-id'] ?? 0);
+
+    if (!Number.isFinite(storageProfileId) || storageProfileId <= 0) {
+      throw new BadRequestException('O perfil de armazenamento para upload em massa não está configurado. Acesse as configurações e defina o perfil de integração S3.');
+    }
+
+    const profile = await (this.prisma as any).integration_profile.findUnique({
+      where: { id: storageProfileId },
+      include: {
+        integration_type: { select: { slug: true } },
+        integration_provider: { select: { slug: true } },
+      },
+    });
+
+    if (!profile) {
+      throw new BadRequestException('Bulk upload storage profile not found.');
+    }
+
+    const config = profile.config as Record<string, any>;
+    const bucket = String(config.bucket ?? '').trim();
+    const accessKeyId = String(config.access_key_id ?? '').trim();
+    const secretAccessKey = String(config.secret_access_key ?? '').trim();
+
+    if (!bucket || !accessKeyId || !secretAccessKey) {
+      throw new BadRequestException('Bulk upload storage profile does not expose AWS credentials.');
+    }
+
+    return {
+      region: String(config.region ?? '').trim() || 'us-east-1',
+      bucket,
+      accessKeyId,
+      secretAccessKey,
+      sessionToken: String(config.session_token ?? '').trim() || undefined,
+      roleArn: String(config.role_arn ?? '').trim() || undefined,
+      externalId: String(config.external_id ?? '').trim() || undefined,
+      durationSeconds: Math.max(
+        900,
+        Number(settings['lms-bulk-upload-sts-duration-seconds'] ?? 3600) || 3600,
+      ),
+    };
+  }
+
+  private inferVideoMimeType(filename: string) {
+    switch (extname(filename).toLowerCase()) {
+      case '.mp4':
+      case '.m4v':
+        return 'video/mp4';
+      case '.mov':
+        return 'video/quicktime';
+      case '.mkv':
+        return 'video/x-matroska';
+      case '.webm':
+        return 'video/webm';
+      case '.avi':
+        return 'video/x-msvideo';
+      case '.wmv':
+        return 'video/x-ms-wmv';
+      case '.flv':
+        return 'video/x-flv';
+      case '.ogv':
+        return 'video/ogg';
+      case '.m2ts':
+      case '.ts':
+        return 'video/mp2t';
+      default:
+        return 'video/mp4';
+    }
+  }
+
+  private decodeKey(value: string) {
+    if (!value) {
+      return value;
+    }
+
+    try {
+      return decodeURIComponent(value.replace(/\+/g, ' '));
+    } catch {
+      return value;
+    }
+  }
+
+  private firstString(...values: Array<unknown>) {
+    for (const value of values) {
+      const normalized = String(value ?? '').trim();
+      if (normalized) {
+        return normalized;
+      }
+    }
+
+    return '';
+  }
+
+  private normalizeComparableText(value: string) {
+    return String(value ?? '')
+      .normalize('NFD')
+      .replace(/[\u0300-\u036f]/g, '')
+      .replace(/[^a-zA-Z0-9]+/g, '_')
+      .replace(/_+/g, '_')
+      .replace(/^_+|_+$/g, '')
+      .toLowerCase();
+  }
+
+  private formatCode(prefix: 'S' | 'A', order: number) {
+    return `${prefix}${String(Math.max(order, 0)).padStart(2, '0')}`;
+  }
+
+  async linkLessonManually(
+    itemId: number,
+    params: { courseId: number; sessionId: number; lessonId: number; userId: number },
+  ) {
+    const items = await this.prisma.$queryRawUnsafe<
+      Array<{ id: number; uploaded_key: string | null; file_name: string }>
+    >(
+      `SELECT i."id", i."uploaded_key", i."file_name"
+       FROM "lms_bulk_upload_item" i
+       WHERE i."id" = $1
+       LIMIT 1`,
+      itemId,
+    );
+
+    const item = items[0];
+    if (!item) {
+      throw new BadRequestException('Upload item not found.');
+    }
+
+    const lessonRows = await this.prisma.$queryRawUnsafe<Array<{ id: number }>>(
+      `SELECT cl."id"
+       FROM "course_lesson" cl
+       INNER JOIN "course_module" cm ON cm."id" = cl."course_module_id"
+       WHERE cl."id" = $1 AND cm."id" = $2 AND cm."course_id" = $3
+       LIMIT 1`,
+      params.lessonId,
+      params.sessionId,
+      params.courseId,
+    );
+
+    if (!lessonRows[0]) {
+      throw new BadRequestException('Lesson does not belong to the given course and module.');
+    }
+
+    const now = new Date();
+    await this.prisma.$executeRawUnsafe(
+      `UPDATE "lms_bulk_upload_item"
+       SET "matched_course_id" = $2,
+           "matched_session_id" = $3,
+           "matched_lesson_id" = $4,
+           "status" = 'received',
+           "updated_at" = $5
+       WHERE "id" = $1`,
+      itemId,
+      params.courseId,
+      params.sessionId,
+      params.lessonId,
+      now,
+    );
+
+    const bucket = await this.resolveStorageContext();
+    const key = String(item.uploaded_key ?? '').trim();
+
+    if (!key) {
+      throw new BadRequestException('Upload item does not have a stored S3 key yet.');
+    }
+
+    const job = await this.queueJobService.enqueue(
+      {
+        type: LMS_BULK_UPLOAD_DOWNLOAD_ORIGINAL_JOB,
+        queueName: LMS_BULK_UPLOAD_DOWNLOAD_ORIGINAL_JOB,
+        payload: {
+          bucket: bucket.bucket,
+          key,
+          fileName: item.file_name,
+          uploadItemId: itemId,
+          userId: params.userId,
+          courseId: params.courseId,
+          sessionId: params.sessionId,
+          lessonId: params.lessonId,
+        },
+        maxAttempts: 3,
+        sourceModule: 'lms',
+        sourceEntity: 'lms_bulk_upload_item',
+        sourceEntityId: String(itemId),
+        priority: -6,
+      },
+      params.userId,
+    );
+
+    return { success: true, queueJobId: job.id };
+  }
+
+  async getItemVideoPresignedUrl(itemId: number): Promise<{ url: string; expiresIn: number }> {
+    const items = await this.prisma.$queryRawUnsafe<
+      Array<{ uploaded_key: string | null }>
+    >(
+      `SELECT "uploaded_key" FROM "lms_bulk_upload_item" WHERE "id" = $1 LIMIT 1`,
+      itemId,
+    );
+
+    const key = String(items[0]?.uploaded_key ?? '').trim();
+    if (!key) {
+      throw new BadRequestException('Upload item does not have a stored S3 key.');
+    }
+
+    const storage = await this.resolveStorageContext();
+    const s3 = new S3Client({
+      region: storage.region,
+      credentials: {
+        accessKeyId: storage.accessKeyId,
+        secretAccessKey: storage.secretAccessKey,
+        sessionToken: storage.sessionToken,
+      },
+    });
+
+    const expiresIn = 3600;
+    const url = await getSignedUrl(
+      s3,
+      new GetObjectCommand({ Bucket: storage.bucket, Key: key }),
+      { expiresIn },
+    );
+
+    return { url, expiresIn };
+  }
+
+  private safeJsonParse(value: string) {
+    try {
+      return JSON.parse(value);
+    } catch {
+      return {};
+    }
+  }
+}