@vue-skuilder/express 0.1.1

package/src/attachment-preprocessing/index.ts

@@ -0,0 +1,204 @@
+import CouchDB from '../couchdb/index.js';
+import nano from 'nano';
+import { normalize } from './normalize.js';
+import AsyncProcessQueue, { Result } from '../utils/processQueue.js';
+import logger from '../logger.js';
+import { CourseLookup } from '@vue-skuilder/db';
+
+// @ts-expect-error [todo]
+const Q = new AsyncProcessQueue<AttachmentProcessingRequest, Result>(
+  processDocAttachments
+);
+
+interface DocForProcessing extends nano.DocumentGetResponse {
+  processed?: boolean | string[];
+  _attachments: {
+    [key: string]: {
+      content_type: string;
+      data?: any; // eslint-disable-line @typescript-eslint/no-explicit-any
+      digest?: string;
+      length?: number;
+      revpos?: number;
+      stub?: boolean;
+    };
+  };
+}
+
+/**
+ * Apply post-processing to a course database. Runs continuously.
+ * @param courseID
+ */
+export function postProcessCourse(courseID: string): void {
+  try {
+    logger.info(`Following course ${courseID}`);
+
+    const crsString = `coursedb-${courseID}`;
+
+    // Get database instance
+    const db = CouchDB.use(crsString);
+
+    const courseFilter = filterFactory(courseID);
+
+    db.changesReader
+      .start({
+        // feed: 'continuous',
+        includeDocs: false,
+      })
+      .on('change', (change: nano.DatabaseChangesResultItem) => {
+        courseFilter(change).catch((e) => {
+          logger.error(`Error in CourseFilter for ${courseID}: ${e}`);
+        });
+      })
+      .on('error', (err: Error) => {
+        logger.error(`Error in changes feed for ${crsString}: ${err}`);
+      });
+  } catch (e) {
+    logger.error(`Error in postProcessCourse: ${e}`);
+  }
+}
+
+/**
+ * Connect to CouchDB, monitor changes to uploaded card data,
+ * perform post-processing on uploaded media
+ */
+export default async function postProcess(): Promise<void> {
+  try {
+    logger.info(`Following all course databases for changes...`);
+
+    const courses = await CourseLookup.allCourses();
+
+    for (const course of courses) {
+      try {
+        postProcessCourse(course._id);
+      } catch (e) {
+        logger.error(`Error processing course ${course._id}: ${e}`);
+        throw e;
+      }
+    }
+  } catch (e) {
+    logger.error(`Error in postProcess: ${e}`);
+  }
+}
+
+function filterFactory(courseID: string) {
+  const courseDatabase = CouchDB.use<DocForProcessing>(`coursedb-${courseID}`);
+
+  return async function filterChanges(
+    changeItem: nano.DatabaseChangesResultItem
+  ) {
+    try {
+      const docNoAttachments = await courseDatabase.get(changeItem.id, {
+        attachments: false,
+      });
+
+      if (
+        docNoAttachments._attachments &&
+        Object.keys(docNoAttachments._attachments).length > 0 &&
+        (docNoAttachments['processed'] === undefined ||
+          docNoAttachments['processed'] === false)
+      ) {
+        const doc = await courseDatabase.get(changeItem.id, {
+          attachments: true,
+        });
+        const processingRequest: AttachmentProcessingRequest = {
+          courseID,
+          docID: doc._id,
+          fields: [],
+        };
+        const atts = doc._attachments;
+        for (const attachment in atts) {
+          const content_type: string = atts[attachment]['content_type'];
+          logger.info(
+            `Course: ${courseID}\n\tAttachment ${attachment} in:\n\t${doc._id}\n should be processed...`
+          );
+
+          if (content_type.includes('audio')) {
+            processingRequest.fields.push({
+              name: attachment,
+              mimetype: content_type,
+            });
+          }
+        }
+        Q.addRequest(processingRequest);
+      }
+    } catch (e) {
+      logger.error(`Error processing doc ${changeItem.id}: ${e}`);
+    }
+  };
+}
+
+async function processDocAttachments(
+  request: AttachmentProcessingRequest
+): Promise<Result> {
+  if (request.fields.length == 0) {
+    logger.info(`No attachments to process for ${request.docID}`);
+    return {
+      error: 'No attachments to process',
+      ok: true,
+      status: 'warning',
+    };
+  }
+  const courseDatabase = CouchDB.use<DocForProcessing>(
+    `coursedb-${request.courseID}`
+  );
+
+  const doc = await courseDatabase.get(request.docID, {
+    attachments: true,
+    att_encoding_info: true,
+  });
+
+  for (const field of request.fields) {
+    logger.info(`Converting ${field.name}`);
+    const attachment = doc._attachments[field.name].data;
+    if (field.mimetype.includes('audio')) {
+      try {
+        const converted = await normalize(attachment);
+        field.returnData = converted;
+      } catch (e) {
+        logger.info(`Exception caught: ${e}`);
+        throw e;
+      }
+    }
+  }
+
+  logger.info('Conversions finished');
+
+  request.fields.forEach((field) => {
+    logger.info(`Replacing doc Data for ${field.name}`);
+    if (doc['processed']) {
+      (doc['processed'] as string[]).push(field.name);
+    } else {
+      doc['processed'] = [field.name];
+    }
+    doc._attachments[field.name].data = field.returnData;
+  });
+
+  // request was a noop.
+  // Mark as processed in order to avoid inifinte loop
+  if (request.fields.length === 0) {
+    doc['processed'] = true;
+  }
+
+  const resp = (await courseDatabase.insert(doc)) as unknown as Result;
+  resp.status = 'ok';
+
+  logger.info(`Processing request reinsert result: ${JSON.stringify(resp)}`);
+  return resp;
+}
+
+// interface _DatabaseChangesResultItemWithDoc
+//   extends nano.DatabaseChangesResultItem {
+//   doc: nano.DocumentGetResponse;
+//   courseID: string;
+// }
+
+interface AttachmentProcessingRequest {
+  courseID: string;
+  docID: string;
+  fields: ProcessingField[];
+}
+interface ProcessingField {
+  name: string;
+  mimetype: string;
+  returnData?: string;
+}

package/src/attachment-preprocessing/normalize.ts

@@ -0,0 +1,123 @@
+import fs from 'fs';
+import logger from '../logger.js';
+import { promisify } from 'util';
+import { exec as execCallback } from 'child_process';
+const exec = promisify(execCallback);
+
+import FFMPEGstatic from 'ffmpeg-static';
+if (!FFMPEGstatic) {
+  const e = 'FFMPEGstatic executable not found';
+  logger.error(e);
+  throw new Error(e);
+}
+
+// string | null here - but we know it's a string from the above check
+const FFMPEG = FFMPEGstatic as unknown as string;
+
+logger.info(`FFMPEG path: ${FFMPEG}`);
+
+checkFFMPEGVersion().catch((e) => {
+  const msg = 'FFMPEG version check failed';
+  logger.error(msg, e);
+  throw new Error(msg + e);
+});
+
+async function checkFFMPEGVersion() {
+  try {
+    if (!fs.existsSync(FFMPEG)) {
+      const e = `FFMPEG executable not found at path: ${FFMPEG}`;
+      logger.error(e);
+      throw new Error(e);
+    }
+
+    const result = await exec(`${FFMPEG} -version`);
+    const version = result.stdout.split('\n')[0];
+    logger.info(`FFMPEG version: ${version}`);
+
+    // Verify loudnorm filter availability
+    const filters = await exec(`${FFMPEG} -filters | grep loudnorm`);
+    if (!filters.stdout.includes('loudnorm')) {
+      throw new Error('loudnorm filter not available');
+    }
+  } catch (error) {
+    logger.error('FFMPEG version check failed:', error);
+    throw error;
+  }
+}
+
+/**
+ * From FFMPEG's loudnorm output - loudness data on a media file
+ */
+interface LoudnessData {
+  // these are numbers, but will be parsed as strings
+  input_i: string; //number;
+  input_tp: string; //number;
+  input_lra: string; //number;
+  input_thresh: string; //number;
+  output_i: string; //number;
+  output_tp: string; //number;
+  output_lra: string; //number;
+  output_thresh: string; //number;
+  normalization_type: string; // this one is actually a string
+  target_offset: string; //number;
+}
+
+/**
+ * Returns normalized, base-64 encoded mp3
+ *
+ * @param fileData the base-64 encoded mp3 data from couchdb
+ */
+export async function normalize(fileData: string): Promise<string> {
+  const encoding = 'base64';
+  const tmpDir = fs.mkdtempSync(`audioNormalize-${encoding}-`);
+  const fileName = tmpDir + '/file.mp3';
+
+  fs.writeFileSync(fileName, fileData, {
+    encoding,
+  });
+
+  const ext = '.' + fileName.split('.')[1];
+
+  const PADDED = tmpDir + '/padded' + ext;
+  const PADDED_NORMALIZED = tmpDir + '/paddedNormalized' + ext;
+  const NORMALIZED = tmpDir + '/normalized' + ext;
+
+  try {
+    // elongate
+    await exec(FFMPEG + ` -i ${fileName} -af "adelay=10000|10000" ${PADDED}`);
+    const info = await exec(
+      FFMPEG +
+        ` -i ${PADDED} -af loudnorm=I=-16:TP=-1.5:LRA=11:print_format=json -f null -`
+    );
+    const data: LoudnessData = JSON.parse(
+      info.stderr.substring(info.stderr.indexOf('{'))
+    );
+    // normalize the elongated file
+    await exec(
+      FFMPEG +
+        ` -i ${PADDED} -af ` +
+        `loudnorm=I=-16:TP=-1.5:LRA=11:measured_I=${data.input_i}:` +
+        `measured_LRA=${data.input_lra}:measured_TP=${data.input_tp}:` +
+        `measured_thresh=${data.input_thresh}:offset=${data.target_offset}:linear=true:` +
+        `print_format=summary -ar 48k ${PADDED_NORMALIZED}`
+    );
+    // cut off the elongated part
+    await exec(
+      FFMPEG +
+        ` -i ${PADDED_NORMALIZED} -ss 00:00:10.000 -acodec copy ${NORMALIZED}`
+    );
+    const ret = fs.readFileSync(NORMALIZED, {
+      encoding,
+    });
+    return ret;
+  } catch (e) {
+    logger.error(e);
+    throw e;
+  } finally {
+    const files = fs.readdirSync(tmpDir);
+    files.forEach((file) => {
+      fs.unlinkSync(tmpDir + '/' + file);
+    });
+    fs.rmdirSync(tmpDir);
+  }
+}

package/src/client-requests/classroom-requests.ts

@@ -0,0 +1,234 @@
+import hashids from 'hashids';
+import Nano from 'nano';
+import {
+  ClassroomConfig,
+  CreateClassroom,
+  JoinClassroom,
+  LeaveClassroom,
+  Status,
+} from '@vue-skuilder/common';
+import { classroomDbDesignDoc } from '../design-docs.js';
+import CouchDB, {
+  SecurityObject,
+  docCount,
+  useOrCreateDB,
+} from '../couchdb/index.js';
+import AsyncProcessQueue, { Result } from '../utils/processQueue.js';
+import logger from '../logger.js';
+
+export const CLASSROOM_DB_LOOKUP = 'classdb-lookup';
+const CLASSROOM_CONFIG = 'ClassroomConfig';
+
+interface lookupData {
+  num: number;
+  uuid: string;
+}
+
+// async function deleteClassroom(_classroom_id: string) {}
+
+async function getClassID(joinCode: string) {
+  try {
+    const doc = await (await useOrCreateDB(CLASSROOM_DB_LOOKUP)).get(joinCode);
+    return (doc as unknown as lookupData).uuid;
+  } catch {
+    return '';
+  }
+}
+
+async function getClassroomConfig(id: string): Promise<ClassroomConfig> {
+  return (await useOrCreateDB(getClassDBNames(id).studentDB)).get(
+    CLASSROOM_CONFIG
+  ) as unknown as ClassroomConfig;
+}
+async function writeClassroomConfig(config: ClassroomConfig, classID: string) {
+  logger.info(`Writing config for class: ${classID}`);
+  const dbNames = getClassDBNames(classID);
+  const studentDB = await useOrCreateDB(dbNames.studentDB);
+  const teacherDB = await useOrCreateDB(dbNames.teacherDB);
+
+  return Promise.all([
+    studentDB
+      .get(CLASSROOM_CONFIG)
+      .then((doc) => {
+        return studentDB.insert({
+          _id: CLASSROOM_CONFIG,
+          _rev: doc._rev,
+          ...config,
+        });
+      })
+      .catch((_err) => {
+        return studentDB.insert({
+          _id: CLASSROOM_CONFIG,
+          ...config,
+        });
+      }),
+    teacherDB
+      .get(CLASSROOM_CONFIG)
+      .then((doc) => {
+        return teacherDB.insert({
+          _id: CLASSROOM_CONFIG,
+          _rev: doc._rev,
+          ...config,
+        });
+      })
+      .catch((_err) => {
+        return teacherDB.insert({
+          _id: CLASSROOM_CONFIG,
+          ...config,
+        });
+      }),
+  ]);
+}
+
+function getClassDBNames(classId: string): {
+  studentDB: string;
+  teacherDB: string;
+} {
+  return {
+    studentDB: `classdb-student-${classId}`,
+    teacherDB: `classdb-teacher-${classId}`,
+  };
+}
+
+async function createClassroom(config: ClassroomConfig) {
+  logger.info(`CreateClass Request:
+    ${JSON.stringify(config)}`);
+
+  const num = (await docCount(CLASSROOM_DB_LOOKUP)) + 1; //
+  const uuid = (await CouchDB.uuids(1)).uuids[0];
+  const hasher = new hashids('', 6, 'abcdefghijklmnopqrstuvwxyz123456789');
+  const studentDbName = `classdb-student-${uuid}`;
+  const teacherDbName = `classdb-teacher-${uuid}`;
+  config.joinCode = hasher.encode(num);
+
+  const security: SecurityObject = {
+    // _id: '_security',
+    admins: {
+      names: [],
+      roles: [],
+    },
+    members: {
+      names: config.teachers,
+      roles: [],
+    },
+  };
+
+  const [studentdb, teacherdb, lookup] = await Promise.all([
+    useOrCreateDB(studentDbName),
+    useOrCreateDB(teacherDbName),
+    useOrCreateDB('classdb-lookup'),
+  ]);
+  await Promise.all([
+    studentdb.insert(
+      {
+        validate_doc_update: classroomDbDesignDoc,
+      } as Nano.MaybeDocument,
+      '_design/_auth'
+    ),
+    // studentdb.insert(security, '_security'),
+    teacherdb.insert(security, '_security'),
+    lookup.insert(
+      {
+        num,
+        uuid,
+      } as Nano.MaybeDocument,
+      config.joinCode
+    ),
+    writeClassroomConfig(config, uuid),
+  ]);
+
+  const res: Result = {
+    ok: true,
+    status: 'ok',
+  };
+  const ret = {
+    joincode: config.joinCode,
+    uuid: uuid,
+    ...res,
+  };
+
+  logger.info(JSON.stringify(ret));
+
+  return ret;
+}
+
+async function leaveClassroom(
+  req: LeaveClassroom['data'] & { username: string }
+) {
+  const cfg: ClassroomConfig = await getClassroomConfig(req.classID);
+  if (cfg) {
+    const index = cfg.students.indexOf(req.username);
+    if (index !== -1) {
+      cfg.students.splice(index, 1);
+    }
+
+    await writeClassroomConfig(cfg, req.classID);
+
+    return {
+      status: Status.ok,
+      ok: true,
+    };
+  } else {
+    return {
+      status: Status.error,
+      ok: false,
+      errorText: 'Course with this ID not found.',
+    };
+  }
+}
+
+async function joinClassroom(req: JoinClassroom['data']) {
+  const classID = await getClassID(req.joinCode);
+  if (classID) {
+    const classDBNames = getClassDBNames(classID);
+
+    void (await useOrCreateDB(classDBNames.studentDB)).get('ClassroomConfig');
+
+    logger.info(`joinClassroom running...
+        \tRequest: ${JSON.stringify(req)}`);
+
+    const cfg: ClassroomConfig = await getClassroomConfig(classID);
+
+    if (req.registerAs === 'student') {
+      if (cfg.students.indexOf(req.user) === -1) {
+        cfg.students.push(req.user);
+      }
+    }
+
+    await writeClassroomConfig(cfg, classID);
+
+    const res: JoinClassroom['response'] = {
+      ok: true,
+      status: Status.ok,
+      id_course: classID,
+      course_name: cfg.name,
+    };
+    return res;
+  } else {
+    return {
+      ok: false,
+      status: Status.error,
+      id_course: '',
+      course_name: '',
+      errorText: 'No course found with this joincode!',
+    };
+  }
+}
+
+export const ClassroomLeaveQueue = new AsyncProcessQueue<
+  // @ts-expect-error Type intersection with username field not properly recognized by AsyncProcessQueue generic
+  LeaveClassroom['data'] & { username: string },
+  LeaveClassroom['response']
+>(leaveClassroom);
+
+export const ClassroomJoinQueue = new AsyncProcessQueue<
+  // @ts-expect-error JoinClassroom data type not fully compatible with AsyncProcessQueue generic constraints
+  JoinClassroom['data'],
+  JoinClassroom['response']
+>(joinClassroom);
+
+export const ClassroomCreationQueue = new AsyncProcessQueue<
+  // @ts-expect-error CreateClassroom data type not fully compatible with AsyncProcessQueue generic constraints
+  CreateClassroom['data'],
+  CreateClassroom['response']
+>(createClassroom);