@hed-hog/lms 0.0.361 → 0.0.365

package/src/course/lms-bulk-upload-infra.service.ts

@@ -0,0 +1,360 @@
+import { FileService } from '@hed-hog/core';
+import {
+  BadRequestException,
+  Inject,
+  Injectable,
+  Logger,
+  forwardRef,
+} from '@nestjs/common';
+
+type BulkUploadInfraSyncInput = {
+  bucketName: string;
+  region: string;
+  webhookUrl: string;
+  webhookSecret: string;
+  accessKeyId: string;
+  secretAccessKey: string;
+  sessionToken?: string;
+  lambdaRoleArn?: string;
+};
+
+const BULK_UPLOAD_LAMBDA_TEMPLATE_CODE = `'use strict';
+
+const { S3Client, HeadObjectCommand } = require('@aws-sdk/client-s3');
+
+const s3 = new S3Client({ region: process.env.LMS_BULK_UPLOAD_REGION || 'us-east-1' });
+
+exports.handler = async (event) => {
+  const record = event && Array.isArray(event.Records) && event.Records[0];
+  if (!record) {
+    console.log('No S3 record in event. Skipping.', JSON.stringify(event));
+    return { statusCode: 200 };
+  }
+
+  const bucketName = record.s3.bucket.name;
+  const key = decodeURIComponent(record.s3.object.key.replace(/\\+/g, ' '));
+  const size = record.s3.object.size;
+  const etag = record.s3.object.eTag;
+  const eventTime = record.eventTime;
+  const webhookUrl = process.env.WEBHOOK_URL;
+
+  if (!webhookUrl) {
+    console.error('WEBHOOK_URL is not set');
+    return { statusCode: 500 };
+  }
+
+  let contentType = 'application/octet-stream';
+  try {
+    const head = await s3.send(new HeadObjectCommand({ Bucket: bucketName, Key: key }));
+    contentType = head.ContentType || contentType;
+  } catch (err) {
+    console.error('HeadObject error:', err.message);
+  }
+
+  const payload = { bucketName, key, size, etag, eventTime, contentType };
+
+  console.log('Sending webhook:', JSON.stringify(payload));
+
+  const response = await fetch(webhookUrl, {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+      'Authorization': \`Bearer \${process.env.WEBHOOK_SECRET}\`,
+    },
+    body: JSON.stringify(payload),
+  });
+
+  console.log('Webhook response status:', response.status);
+
+  return { statusCode: response.status };
+};
+`;
+
+@Injectable()
+export class LmsBulkUploadInfraService {
+  private readonly logger = new Logger(LmsBulkUploadInfraService.name);
+
+  constructor(
+    @Inject(forwardRef(() => FileService))
+    private readonly fileService: FileService,
+  ) {}
+
+  async syncDesktopUploadInfrastructure(input: BulkUploadInfraSyncInput) {
+    const lambdaNameResolution = this.resolveLambdaFunctionName(input.bucketName);
+    const lambdaFunctionName = lambdaNameResolution.functionName;
+
+    if (!input.webhookUrl || !input.webhookSecret) {
+      throw new BadRequestException(
+        'Bulk upload webhook URL/token is not available for Lambda sync.',
+      );
+    }
+
+    this.logger.log(
+      `syncDesktopUploadInfrastructure: fn=${lambdaFunctionName} region=${input.region} ` +
+      `webhookUrl=${input.webhookUrl} secretLen=${input.webhookSecret.length}`,
+    );
+
+    await this.ensureLambdaExists(lambdaFunctionName, lambdaNameResolution.generatedFromBucket, input);
+
+    const zipFile = this.buildSingleFileZip('index.js', BULK_UPLOAD_LAMBDA_TEMPLATE_CODE);
+    await this.fileService.updateLambdaFunctionCode({
+      functionName: lambdaFunctionName,
+      region: input.region,
+      accessKeyId: input.accessKeyId,
+      secretAccessKey: input.secretAccessKey,
+      sessionToken: input.sessionToken,
+      zipFile,
+    });
+
+    this.logger.log(`syncDesktopUploadInfrastructure: code updated, now updating env vars`);
+
+    const lambdaConfiguration =
+      await this.fileService.updateLambdaEnvironmentVariables({
+        functionName: lambdaFunctionName,
+        region: input.region,
+        accessKeyId: input.accessKeyId,
+        secretAccessKey: input.secretAccessKey,
+        sessionToken: input.sessionToken,
+        environmentVariables: {
+          WEBHOOK_URL: input.webhookUrl,
+          WEBHOOK_SECRET: input.webhookSecret,
+          LMS_BULK_UPLOAD_BUCKET: input.bucketName,
+          LMS_BULK_UPLOAD_REGION: input.region,
+        },
+      });
+
+    this.logger.log(
+      `syncDesktopUploadInfrastructure: env vars update complete, fn=${lambdaConfiguration.functionName} arn=${lambdaConfiguration.functionArn}`,
+    );
+
+    const lambdaArn =
+      String(
+        lambdaConfiguration.functionArn ??
+          process.env.LMS_BULK_UPLOAD_LAMBDA_ARN ??
+          '',
+      ).trim();
+
+    if (!lambdaArn) {
+      throw new BadRequestException(
+        'Lambda ARN could not be resolved. Set LMS_BULK_UPLOAD_LAMBDA_ARN or ensure Lambda function is readable.',
+      );
+    }
+
+    await this.fileService.allowS3InvokeLambda({
+      functionName: lambdaFunctionName,
+      region: input.region,
+      accessKeyId: input.accessKeyId,
+      secretAccessKey: input.secretAccessKey,
+      sessionToken: input.sessionToken,
+      bucketName: input.bucketName,
+      sourceAccount: String(process.env.AWS_ACCOUNT_ID ?? '').trim() || undefined,
+    });
+
+    await this.fileService.configureS3BucketLambdaNotification({
+      bucketName: input.bucketName,
+      lambdaArn,
+      region: input.region,
+      accessKeyId: input.accessKeyId,
+      secretAccessKey: input.secretAccessKey,
+      sessionToken: input.sessionToken,
+      configurationId: 'lms-bulk-upload-object-created',
+    });
+
+    return {
+      outputs: {
+        bucket_name: input.bucketName,
+        bucket_arn: `arn:aws:s3:::${input.bucketName}`,
+        lambda_arn: lambdaArn,
+        lambda_function_name: lambdaFunctionName,
+      },
+    };
+  }
+
+  private resolveLambdaFunctionName(bucketName: string) {
+    const configured =
+      process.env.LMS_BULK_UPLOAD_LAMBDA_FUNCTION_NAME?.trim() ||
+      process.env.LMS_BULK_UPLOAD_LAMBDA_ARN?.trim() ||
+      '';
+
+    if (configured) {
+      return {
+        functionName: configured,
+        generatedFromBucket: false,
+      };
+    }
+
+    return {
+      functionName: this.generateLambdaNameFromBucket(bucketName),
+      generatedFromBucket: true,
+    };
+  }
+
+  private generateLambdaNameFromBucket(bucketName: string) {
+    const base = String(bucketName ?? '')
+      .trim()
+      .replace(/[^a-zA-Z0-9-_]/g, '-')
+      .replace(/-+/g, '-')
+      .replace(/^-|-$/g, '');
+
+    if (!base) {
+      throw new BadRequestException('Could not generate lambda name from bucket.');
+    }
+
+    const suffix = '-lambda';
+    const maxBaseLength = 64 - suffix.length;
+    const normalizedBase = base.slice(0, maxBaseLength);
+    return `${normalizedBase}${suffix}`;
+  }
+
+  private async ensureLambdaExists(
+    lambdaFunctionName: string,
+    generatedFromBucket: boolean,
+    input: BulkUploadInfraSyncInput,
+  ) {
+    try {
+      await this.fileService.getLambdaFunctionConfiguration({
+        functionName: lambdaFunctionName,
+        region: input.region,
+        accessKeyId: input.accessKeyId,
+        secretAccessKey: input.secretAccessKey,
+        sessionToken: input.sessionToken,
+      });
+      return;
+    } catch (error: any) {
+      if (String(error?.name ?? '') !== 'ResourceNotFoundException') {
+        throw error;
+      }
+    }
+
+    const createIfMissing =
+      String(process.env.LMS_BULK_UPLOAD_LAMBDA_CREATE_IF_MISSING ?? 'true')
+        .trim()
+        .toLowerCase() !== 'false';
+
+    if (!createIfMissing) {
+      if (generatedFromBucket) {
+        throw new BadRequestException(
+          `Lambda ${lambdaFunctionName} was not found. Create this function first or set LMS_BULK_UPLOAD_LAMBDA_FUNCTION_NAME/LMS_BULK_UPLOAD_LAMBDA_ARN.`,
+        );
+      }
+
+      throw new BadRequestException(
+        `Configured lambda ${lambdaFunctionName} was not found. Update LMS_BULK_UPLOAD_LAMBDA_FUNCTION_NAME/LMS_BULK_UPLOAD_LAMBDA_ARN.`,
+      );
+    }
+
+    const roleArn =
+      String(input.lambdaRoleArn ?? '').trim() ||
+      String(process.env.LMS_BULK_UPLOAD_LAMBDA_ROLE_ARN ?? '').trim();
+    if (!roleArn) {
+      throw new BadRequestException(
+        'Missing LMS_BULK_UPLOAD_LAMBDA_ROLE_ARN to auto-create Lambda.',
+      );
+    }
+
+    const timeout = Number(process.env.LMS_BULK_UPLOAD_LAMBDA_TIMEOUT ?? 60);
+    const memorySize = Number(process.env.LMS_BULK_UPLOAD_LAMBDA_MEMORY_SIZE ?? 512);
+    const runtime = String(process.env.LMS_BULK_UPLOAD_LAMBDA_RUNTIME ?? 'nodejs20.x').trim();
+    const zipFile = this.buildSingleFileZip('index.js', BULK_UPLOAD_LAMBDA_TEMPLATE_CODE);
+
+    await this.fileService.createLambdaFunctionIfMissing({
+      functionName: lambdaFunctionName,
+      region: input.region,
+      accessKeyId: input.accessKeyId,
+      secretAccessKey: input.secretAccessKey,
+      sessionToken: input.sessionToken,
+      roleArn,
+      packageType: 'Zip',
+      zipFile,
+      handler: 'index.handler',
+      runtime,
+      timeout: Number.isFinite(timeout) ? Math.max(3, timeout) : 60,
+      memorySize: Number.isFinite(memorySize) ? Math.max(128, memorySize) : 512,
+      environmentVariables: {
+        WEBHOOK_URL: input.webhookUrl,
+        WEBHOOK_SECRET: input.webhookSecret,
+        LMS_BULK_UPLOAD_BUCKET: input.bucketName,
+        LMS_BULK_UPLOAD_REGION: input.region,
+      },
+    });
+  }
+
+  private buildSingleFileZip(filename: string, content: string) {
+    const fileNameBuffer = Buffer.from(filename, 'utf8');
+    const fileData = Buffer.from(content, 'utf8');
+    const crc = this.crc32(fileData);
+
+    const localHeader = Buffer.alloc(30 + fileNameBuffer.length);
+    let offset = 0;
+    offset = localHeader.writeUInt32LE(0x04034b50, offset);
+    offset = localHeader.writeUInt16LE(20, offset);
+    offset = localHeader.writeUInt16LE(0, offset);
+    offset = localHeader.writeUInt16LE(0, offset);
+    offset = localHeader.writeUInt16LE(0, offset);
+    offset = localHeader.writeUInt16LE(0, offset);
+    offset = localHeader.writeUInt32LE(crc, offset);
+    offset = localHeader.writeUInt32LE(fileData.length, offset);
+    offset = localHeader.writeUInt32LE(fileData.length, offset);
+    offset = localHeader.writeUInt16LE(fileNameBuffer.length, offset);
+    offset = localHeader.writeUInt16LE(0, offset);
+    fileNameBuffer.copy(localHeader, offset);
+
+    const centralHeader = Buffer.alloc(46 + fileNameBuffer.length);
+    offset = 0;
+    offset = centralHeader.writeUInt32LE(0x02014b50, offset);
+    offset = centralHeader.writeUInt16LE(20, offset);
+    offset = centralHeader.writeUInt16LE(20, offset);
+    offset = centralHeader.writeUInt16LE(0, offset);
+    offset = centralHeader.writeUInt16LE(0, offset);
+    offset = centralHeader.writeUInt16LE(0, offset);
+    offset = centralHeader.writeUInt16LE(0, offset);
+    offset = centralHeader.writeUInt32LE(crc, offset);
+    offset = centralHeader.writeUInt32LE(fileData.length, offset);
+    offset = centralHeader.writeUInt32LE(fileData.length, offset);
+    offset = centralHeader.writeUInt16LE(fileNameBuffer.length, offset);
+    offset = centralHeader.writeUInt16LE(0, offset);
+    offset = centralHeader.writeUInt16LE(0, offset);
+    offset = centralHeader.writeUInt16LE(0, offset);
+    offset = centralHeader.writeUInt16LE(0, offset);
+    offset = centralHeader.writeUInt32LE(0, offset);
+    offset = centralHeader.writeUInt32LE(0, offset);
+    fileNameBuffer.copy(centralHeader, offset);
+
+    const endOfCentralDirectory = Buffer.alloc(22);
+    offset = 0;
+    offset = endOfCentralDirectory.writeUInt32LE(0x06054b50, offset);
+    offset = endOfCentralDirectory.writeUInt16LE(0, offset);
+    offset = endOfCentralDirectory.writeUInt16LE(0, offset);
+    offset = endOfCentralDirectory.writeUInt16LE(1, offset);
+    offset = endOfCentralDirectory.writeUInt16LE(1, offset);
+    offset = endOfCentralDirectory.writeUInt32LE(centralHeader.length, offset);
+    offset = endOfCentralDirectory.writeUInt32LE(localHeader.length + fileData.length, offset);
+    endOfCentralDirectory.writeUInt16LE(0, offset);
+
+    return Buffer.concat([localHeader, fileData, centralHeader, endOfCentralDirectory]);
+  }
+
+  private crc32(buffer: Buffer) {
+    let crc = 0 ^ -1;
+
+    for (let i = 0; i < buffer.length; i++) {
+      crc = (crc >>> 8) ^ this.crc32Table[(crc ^ buffer[i]) & 0xff];
+    }
+
+    return (crc ^ -1) >>> 0;
+  }
+
+  private readonly crc32Table = (() => {
+    const table = new Array<number>(256);
+
+    for (let n = 0; n < 256; n++) {
+      let c = n;
+      for (let k = 0; k < 8; k++) {
+        c = (c & 1) ? (0xedb88320 ^ (c >>> 1)) : (c >>> 1);
+      }
+      table[n] = c >>> 0;
+    }
+
+    return table;
+  })();
+}

package/src/course/lms-bulk-upload.constants.ts

@@ -0,0 +1,5 @@
+export const LMS_BULK_UPLOAD_WEBHOOK_SLUG = 'lms-bulk-upload';
+export const LMS_BULK_UPLOAD_RECEIVE_VIDEO_COMMAND =
+  'lms.bulk-upload.receive-video';
+export const LMS_BULK_UPLOAD_DOWNLOAD_ORIGINAL_JOB =
+  'lms.bulk-upload.download-original';

package/src/course/lms-bulk-upload.controller.ts

@@ -1,11 +1,16 @@
 import { Role, User } from '@hed-hog/api';
-import { Body, Controller, Get, Post } from '@nestjs/common';
+import { Body, Controller, Get, Param, ParseIntPipe, Post, Query } from '@nestjs/common';
+import { CleanupUploadHistoryDto } from './dto/cleanup-upload-history.dto';
+import { LmsBulkUploadAutomationService } from './lms-bulk-upload-automation.service';
 import { LmsBulkUploadService } from './lms-bulk-upload.service';
 
 @Role()
 @Controller('lms/bulk-upload')
 export class LmsBulkUploadController {
-  constructor(private readonly bulkUploadService: LmsBulkUploadService) {}
+  constructor(
+    private readonly bulkUploadService: LmsBulkUploadService,
+    private readonly automationService: LmsBulkUploadAutomationService,
+  ) {}
 
   @Get('settings')
   getSettings() {
@@ -18,10 +23,111 @@ export class LmsBulkUploadController {
   }
 
   @Post('verify')
-  verifyFile(
+  async verifyFile(
     @User('id') userId: number,
-    @Body() payload: { key?: string; fileName?: string },
+    @Body()
+    payload: {
+      key?: string;
+      fileName?: string;
+      configure?: boolean;
+      storageProfileId?: number;
+      lambdaRoleArn?: string;
+    },
   ) {
+    if (payload?.configure) {
+      if (payload.lambdaRoleArn !== undefined) {
+        await this.bulkUploadService.setBulkUploadLambdaRoleArn(payload.lambdaRoleArn);
+      }
+
+      return this.bulkUploadService.configureBulkUpload(userId, {
+        storageProfileId: payload.storageProfileId,
+      });
+    }
+
     return this.bulkUploadService.verifyFileOnS3(userId, payload ?? {});
   }
+
+  @Post('webhook/regenerate-token')
+  regenerateWebhookToken(@User('id') userId: number) {
+    return this.bulkUploadService.regenerateWebhookTokenAndSync(userId);
+  }
+
+  @Post('session/start')
+  startSession(
+    @User('id') userId: number,
+    @Body()
+    payload: {
+      appName?: string;
+      folderPath?: string | null;
+      files?: Array<{ uploadId?: string; fileName?: string; sizeBytes?: number }>;
+    },
+  ) {
+    return this.bulkUploadService.startUploadSession(userId, payload ?? {});
+  }
+
+  @Post('session/heartbeat')
+  heartbeatSession(
+    @User('id') userId: number,
+    @Body()
+    payload: {
+      sessionId?: number;
+      status?: string;
+      progressPercent?: number;
+      items?: Array<{
+        uploadId?: string;
+        status?: string;
+        progressPercent?: number;
+        errorMessage?: string | null;
+        uploadedKey?: string | null;
+      }>;
+    },
+  ) {
+    return this.bulkUploadService.heartbeatUploadSession(userId, payload ?? {});
+  }
+
+  @Get('sessions')
+  listSessions(
+    @Query('page') page?: string,
+    @Query('pageSize') pageSize?: string,
+    @Query('search') search?: string,
+    @Query('status') status?: string,
+  ) {
+    return this.bulkUploadService.listUploadItems({
+      page: page ? Number(page) : 1,
+      pageSize: pageSize ? Number(pageSize) : 20,
+      search,
+      status,
+    });
+  }
+
+  @Post('check-uploaded')
+  checkUploaded(
+    @Body() payload: { files?: Array<{ fileName?: string; sizeBytes?: number }> },
+  ) {
+    return this.bulkUploadService.checkUploadedFiles(payload?.files ?? []);
+  }
+
+  @Post('sessions/cleanup')
+  cleanupSessions(@Body() payload: CleanupUploadHistoryDto) {
+    return this.bulkUploadService.cleanupUploadHistory(payload ?? {});
+  }
+
+  @Get('item/:id/video-url')
+  getItemVideoUrl(@Param('id', ParseIntPipe) itemId: number) {
+    return this.automationService.getItemVideoPresignedUrl(itemId);
+  }
+
+  @Post('item/:id/link-lesson')
+  linkLesson(
+    @User('id') userId: number,
+    @Param('id', ParseIntPipe) itemId: number,
+    @Body() payload: { courseId: number; sessionId: number; lessonId: number },
+  ) {
+    return this.automationService.linkLessonManually(itemId, {
+      courseId: Number(payload.courseId),
+      sessionId: Number(payload.sessionId),
+      lessonId: Number(payload.lessonId),
+      userId,
+    });
+  }
 }