@rpcbase/server 0.365.0 → 0.366.0-fileupload.1

package/boot/worker.js

@@ -2,11 +2,13 @@
 require("./shared")
 
 require("rb-plugin-worker")
+require("../src/tasks")
 
 const register_queue_listener = require("../queue/register_queue_listener")
 
 const queue = require("../queue")
 
+
 queue.start()
 
 let _queue_listener
@@ -17,7 +19,7 @@ register_queue_listener()
   })
 
 
-const handle_graceful_exit = async () => {
+const handle_graceful_exit = async() => {
   const close_fns = [
     queue.instance().close,
   ]

package/express/index.js

@@ -5,9 +5,10 @@ const body_parser = require("body-parser")
 const request_ip = require("request-ip")
 const Sentry = require("@sentry/node")
 
-// functionality middlewares
+// functional middlewares
 const auth = require("../src/auth")
 const api = require("../src/api")
+const files = require("../src/files")
 const sessions = require("../src/sessions")
 
 const dev_save_coverage = require("./dev_save_coverage")
@@ -84,6 +85,7 @@ module.exports = () => {
 
   auth(app)
   api(app)
+  files(app)
 
   dev_save_coverage(app)
 

package/files.ts

@@ -0,0 +1 @@
+export * from "./src/files/finalize_file_upload"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rpcbase/server",
-  "version": "0.365.0",
+  "version": "0.366.0-fileupload.1",
   "license": "SSPL-1.0",
   "main": "./index.js",
   "scripts": {

package/queue/register_queue_listener.js

@@ -56,10 +56,9 @@ const mongoose_delete_plugin = (schema) => {
 
 
 const assert_doc_id = (change) => {
-  if (process.env.NODE_ENV !== "development") return
-
   const doc_id = change.documentKey?._id?.toString()
 
+  // some changes aren't on documents
   if (!doc_id) return
 
   const sub = doc_id.substring(8, 16)
@@ -73,6 +72,11 @@ const assert_doc_id = (change) => {
 }
 
 const dispatch_change_handler = (change) => {
+  // skip if this is a file upload
+  if (change.ns?.coll?.endsWith(".files") || change.ns?.coll?.endsWith(".chunks")) {
+    return
+  }
+
   // verify we have correct object ids
   assert_doc_id(change)
 
@@ -109,7 +113,10 @@ const register_db_emitter = () => {
   // Set up the change stream with a filter to only listen to the specific database
   const pipeline = [
     { $match: { "ns.db": RB_APP_NAME } },
-    { $match: { operationType: { $in: ["insert", "update" /* "delete"*/] } } },
+    { $match: {
+      "ns.coll": { $nin: ["file-uploads.files", "file-uploads.chunks"] },
+      "operationType": { $in: ["insert", "update" /* "delete"*/] } },
+    },
   ]
 
   // https://www.mongodb.com/docs/manual/reference/method/Mongo.watch/

package/src/files/finalize_file_upload.ts

@@ -0,0 +1,19 @@
+import Promise from "bluebird"
+
+import queue from "../../queue"
+
+
+// after we've uploaded a file, we process it with ocr, llm vectorization, png rendering, etc
+export const finalize_file_upload = async(files: Array<{hash: string}>) => {
+
+  await Promise.map(files,
+  async(file: {hash: string}) => {
+    await queue.add("finalize_file_upload", {hash: file.hash, hello: "world"}, {
+      jobId: `finalize_file_upload-${file.hash}`,
+      removeOnComplete: true,
+      removeOnFail: true,
+    })
+  }
+  )
+
+}

package/src/files/helpers/get_grid_fs_bucket.ts

@@ -0,0 +1,18 @@
+import assert from "assert"
+import mongoose from "../../../mongoose"
+
+
+const CHUNK_SIZE = 1024 * 1024
+
+
+export const get_grid_fs_bucket = (bucket_name: string, chunk_size: number) => {
+  assert(chunk_size === CHUNK_SIZE, "chunk_size must match default CHUNK_SIZE")
+
+  const {db} = mongoose.connection
+  const bucket = new mongoose.mongo.GridFSBucket(db, {
+    bucketName: bucket_name,
+    chunkSizeBytes: chunk_size,
+  })
+
+  return bucket
+}

package/src/files/index.js

@@ -0,0 +1,5 @@
+import {upload_chunk} from "./upload_chunk"
+
+module.exports = (app) => {
+  app.post("/rb-api/v1/files/upload_chunk", upload_chunk)
+}

package/src/files/tasks/finalize_file_upload.ts

@@ -0,0 +1,5 @@
+
+export const finalize_file_upload = async(payload) => {
+  console.log("TASK:FINALIZE FILE UPLOAD")
+  console.log("payload", payload)
+}

package/src/files/tasks/index.ts

@@ -0,0 +1,6 @@
+import queue from "../../../queue"
+
+import {finalize_file_upload} from "./finalize_file_upload"
+
+
+queue.register_task("finalize_file_upload", finalize_file_upload)

package/src/files/upload_chunk.ts

@@ -0,0 +1,82 @@
+import assert from "assert"
+import fs from "fs"
+import {formidable, File} from "formidable"
+
+import {get_grid_fs_bucket} from "./helpers/get_grid_fs_bucket"
+import { sim_test_inject } from "../helpers/sim_test_inject"
+
+
+const upload_file_to_bucket = async(file: File, metadata): Promise<void> => {
+  const bucket = get_grid_fs_bucket("file-uploads", metadata.chunk_size)
+
+  const chunk_filename = `${metadata.hash}.${metadata.chunk_index}`
+
+  const upload_stream = bucket.openUploadStream(chunk_filename, {
+    metadata,
+  })
+
+  const read_stream = fs.createReadStream(file.filepath)
+
+  read_stream.pipe(upload_stream)
+
+  return new Promise((resolve, reject) => {
+    upload_stream.on("finish", () => {
+      // console.log("finished uploading:", upload_stream.id)
+      resolve()
+    })
+    upload_stream.on("error", (error) => {
+      reject(error)
+    })
+  })
+}
+
+
+export const upload_chunk = async(req, res, next) => {
+  const {user_id} = req.session
+  assert(user_id, "upload_chunk: unable to resolve user_id")
+
+  // https://github.com/node-formidable/formidable#options
+  const form = formidable({})
+
+  let fields, files
+  try {
+    [fields, files] = await form.parse(req)
+  } catch (err) {
+    return res.status(500).json({error: "Failed to parse uploaded file"})
+  }
+
+  await sim_test_inject()
+  // await new Promise(resolve => setTimeout(resolve, 3000))
+
+  const file_chunk = files.file_chunk[0] as File
+
+  const metadata = {
+    user_id,
+    original_filename: fields.original_filename[0],
+    is_compressed: fields.is_compressed[0] === "yes",
+    chunk_index: parseInt(fields.chunk_index[0]),
+    total_chunks: parseInt(fields.total_chunks[0]),
+    chunk_size: parseInt(fields.chunk_size[0]),
+    mime_type: fields.mime_type[0],
+    hash: fields.hash[0],
+  }
+
+  await upload_file_to_bucket(file_chunk, metadata)
+
+  const result: {
+    status: string;
+    finalize_token?: string;
+  } = {
+    status: "ok",
+  }
+
+  const is_last_chunk = metadata.chunk_index === metadata.total_chunks - 1
+
+  if (is_last_chunk) {
+    result.finalize_token = metadata.hash
+  }
+
+  res.json({
+    status: "ok",
+  })
+}

package/src/helpers/sim_test_inject.ts

@@ -0,0 +1,21 @@
+const ERROR_PROBABILITY = 0.1
+const THROTTLE_PROBABILITY = 0.2
+const MAX_THROTTLE_MS = 2000
+
+// randomly inject errors or delays
+export const sim_test_inject = async() => {
+  if (!__DEV__) {
+    return
+  }
+
+  const should_error = Math.random() > 1 - ERROR_PROBABILITY
+  if (should_error) {
+    throw new Error("random sim test error")
+  }
+
+  const should_throttle = Math.random() > 1 - THROTTLE_PROBABILITY
+  if (should_throttle) {
+    const throttle_delay = Math.floor(Math.random() * MAX_THROTTLE_MS)
+    await new Promise(resolve => setTimeout(resolve, throttle_delay))
+  }
+}

package/src/tasks/index.js

@@ -1,6 +1,8 @@
 const queue = require("../../queue")
 
+require("../files/tasks")
 
 const index_item = require("./index_item")
 
+
 queue.register_task("rb_index_item", index_item)