@rpcbase/server 0.364.0 → 0.366.0-fileupload.0

package/boot/worker.js

@@ -2,11 +2,13 @@
 require("./shared")
 
 require("rb-plugin-worker")
+require("../src/tasks")
 
 const register_queue_listener = require("../queue/register_queue_listener")
 
 const queue = require("../queue")
 
+
 queue.start()
 
 let _queue_listener
@@ -17,7 +19,7 @@ register_queue_listener()
   })
 
 
-const handle_graceful_exit = async () => {
+const handle_graceful_exit = async() => {
   const close_fns = [
     queue.instance().close,
   ]

package/express/index.js

@@ -5,9 +5,10 @@ const body_parser = require("body-parser")
 const request_ip = require("request-ip")
 const Sentry = require("@sentry/node")
 
-// functionality middlewares
+// functional middlewares
 const auth = require("../src/auth")
 const api = require("../src/api")
+const files = require("../src/files")
 const sessions = require("../src/sessions")
 
 const dev_save_coverage = require("./dev_save_coverage")
@@ -84,6 +85,7 @@ module.exports = () => {
 
   auth(app)
   api(app)
+  files(app)
 
   dev_save_coverage(app)
 

package/files.ts

@@ -0,0 +1 @@
+export * from "./src/files/finalize_file_upload"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rpcbase/server",
-  "version": "0.364.0",
+  "version": "0.366.0-fileupload.0",
   "license": "SSPL-1.0",
   "main": "./index.js",
   "scripts": {

package/queue/register_queue_listener.js

@@ -9,6 +9,7 @@ const queue = require("./index")
 const dispatch_worker_queue = require("./dispatch_worker_queue")
 const dispatch_indexer_queue = require("./dispatch_indexer_queue")
 
+
 const log = debug("rb:queue:listener")
 
 const {RB_APP_NAME, RB_TENANT_ID} = process.env
@@ -30,10 +31,12 @@ const mongoose_delete_plugin = (schema) => {
   })
 
   // TODO: add other delete operations
+  // TODO: this should be deleteOne ?
   // https://mongoosejs.com/docs/queries.html
   schema.post("findOneAndDelete", function(doc) {
     log("queue:findOneAndDelete", "dispatch_doc_change NYI")
     log("del PLUGIN", doc)
+    const change = null // TMP
     const coll_name = change.ns.coll
 
     const model_name = Object.keys(mongoose.models).find((k) => {
@@ -53,10 +56,11 @@ const mongoose_delete_plugin = (schema) => {
 
 
 const assert_doc_id = (change) => {
-  if (process.env.NODE_ENV !== "development") return
+  // if (process.env.NODE_ENV !== "development") return
 
   const doc_id = change.documentKey?._id?.toString()
 
+  // some changes aren't on documents
   if (!doc_id) return
 
   const sub = doc_id.substring(8, 16)
@@ -70,6 +74,11 @@ const assert_doc_id = (change) => {
 }
 
 const dispatch_change_handler = (change) => {
+  // skip if this is a file upload
+  if (change.ns?.coll?.endsWith(".files") || change.ns?.coll?.endsWith(".chunks")) {
+    return
+  }
+
   // verify we have correct object ids
   assert_doc_id(change)
 
@@ -79,8 +88,6 @@ const dispatch_change_handler = (change) => {
     return mongoose.models[k].collection.collectionName === coll_name
   })
 
-  console.log("O3333K", Object.keys(require("../mongoose").models), Object.keys(mongoose.models))
-
   if (!model_name) {
     return
   }
@@ -107,8 +114,11 @@ const register_db_emitter = () => {
 
   // Set up the change stream with a filter to only listen to the specific database
   const pipeline = [
-    { $match: { 'ns.db': RB_APP_NAME } },
-    { $match: { operationType: { $in: ["insert", "update", /*"delete"*/] } } }
+    { $match: { "ns.db": RB_APP_NAME } },
+    { $match: {
+      "ns.coll": { $nin: ["file-uploads.files", "file-uploads.chunks"] },
+      "operationType": { $in: ["insert", "update" /* "delete"*/] } },
+    },
   ]
 
   // https://www.mongodb.com/docs/manual/reference/method/Mongo.watch/
@@ -154,8 +164,8 @@ const register_queue_listener = () => new Promise((resolve, reject) => {
   mongoose.plugin(mongoose_delete_plugin)
 
   mongoose.connection.once("open", () => {
-    const emitter = register_db_emitter()
-    resolve(emitter)
+    const db_emitter = register_db_emitter()
+    resolve(db_emitter)
   })
 
   mongoose.connection.on("error", (err) => {

package/rts/index.js

@@ -37,7 +37,7 @@ const get_query_options = (ctx, options) => {
   // https://mongoosejs.com/docs/api/query.html#Query.prototype.setOptions()
   const query_options = {
     ctx,
-    is_client: true
+    is_client: true,
     //
   }
 
@@ -46,8 +46,7 @@ const get_query_options = (ctx, options) => {
   }
 
   const limit = typeof options.limit === "number" ? Math.min(QUERY_MAX_LIMIT, Math.abs(options.limit)) : QUERY_MAX_LIMIT
-  // TODO: warn if limit greater than default ?
-  query_options.limit = options.limit
+  query_options.limit = limit
 
   return query_options
 }
@@ -99,15 +98,15 @@ const dispatch_doc_change = (model, doc) => {
         return
       }
 
-
       const query_options = get_query_options(ctx, options)
+      // TODO: do not redo query here, use document from change event
       const query_promise = model.find(query, options.projection, query_options)
       if (query_options.sort) {
         query_promise.sort(query_options.sort)
       }
-      if (typeof query_options.limit === "number" && query_options.limit >= 0) {
-        query_promise.limit(query_options.limit)
-      }
+
+      query_promise.limit(query_options.limit)
+
       // WARNING: DANGER
       // TODO: we should not be doing a read here, use doc from event
       const data = await query_promise
@@ -179,8 +178,8 @@ const mongoose_txn_plugin = (schema, options) => {
 
 
 // responds to a change stream "change" event
-const get_dispatch_change_handler = (mongoose, model_name) => async(change) => {
-  const queries = _queries[model_name]
+const get_dispatch_change_handler = (model_name) => async(change) => {
+  // const queries = _queries[model_name]
 
   const model = mongoose.model(model_name)
 
@@ -203,7 +202,7 @@ const get_dispatch_change_handler = (mongoose, model_name) => async(change) => {
 }
 
 
-const add_change_stream = (mongoose, socket_id, {model_name, query, query_key, options}) => {
+const add_change_stream = (socket_id, {model_name, query, query_key, options}) => {
   if (!model_name) throw new Error("empty model name")
 
   if (!_change_streams[model_name]) {
@@ -225,7 +224,7 @@ const add_change_stream = (mongoose, socket_id, {model_name, query, query_key, o
       fullDocumentBeforeChange: "whenAvailable",
     })
 
-    emitter.on("change", get_dispatch_change_handler(mongoose, model_name))
+    emitter.on("change", get_dispatch_change_handler(model_name))
 
     emitter.on("error", (err) => {
       console.log("change listener emitter got error", err)
@@ -261,7 +260,7 @@ const add_change_stream = (mongoose, socket_id, {model_name, query, query_key, o
 }
 
 
-const run_query = async(mongoose, socket_id, {model_name, query, query_key, options}) => {
+const run_query = async(socket_id, {model_name, query, query_key, options}) => {
   const model = mongoose.model(model_name)
 
   log("run_query", {model_name, query, query_key, options})
@@ -399,7 +398,7 @@ const rts_server = async(server) => {
     console.log("socket connected", socket.id)
     //
     socket.on("register_query", (payload) => {
-      add_change_stream(mongoose, socket.id, payload)
+      add_change_stream(socket.id, payload)
     })
 
     socket.on("remove_query", (payload) => {
@@ -408,7 +407,7 @@ const rts_server = async(server) => {
 
     socket.on("run_query", (payload) => {
       try {
-        run_query(mongoose, socket.id, payload)
+        run_query(socket.id, payload)
       } catch (err) {
         console.log("run_query caught error", err)
       }

package/src/files/finalize_file_upload.ts

@@ -0,0 +1,19 @@
+import Promise from "bluebird"
+
+import queue from "../../queue"
+
+
+// after we've uploaded a file, we process it with ocr, llm vectorization, png rendering, etc
+export const finalize_file_upload = async(files: Array<{hash: string}>) => {
+
+  await Promise.map(files,
+  async(file: {hash: string}) => {
+    await queue.add("finalize_file_upload", {hash: file.hash, hello: "world"}, {
+      jobId: `finalize_file_upload-${file.hash}`,
+      removeOnComplete: true,
+      removeOnFail: true,
+    })
+  }
+  )
+
+}

package/src/files/helpers/get_grid_fs_bucket.ts

@@ -0,0 +1,18 @@
+import assert from "assert"
+import mongoose from "../../../mongoose"
+
+
+const CHUNK_SIZE = 1024 * 1024
+
+
+export const get_grid_fs_bucket = (bucket_name: string, chunk_size: number) => {
+  assert(chunk_size === CHUNK_SIZE, "chunk_size must match default CHUNK_SIZE")
+
+  const {db} = mongoose.connection
+  const bucket = new mongoose.mongo.GridFSBucket(db, {
+    bucketName: bucket_name,
+    chunkSizeBytes: chunk_size,
+  })
+
+  return bucket
+}

package/src/files/index.js

@@ -0,0 +1,5 @@
+import {upload_chunk} from "./upload_chunk"
+
+module.exports = (app) => {
+  app.post("/rb-api/v1/files/upload_chunk", upload_chunk)
+}

package/src/files/tasks/finalize_file_upload.ts

@@ -0,0 +1,5 @@
+
+export const finalize_file_upload = async(payload) => {
+  console.log("TASK:FINALIZE FILE UPLOAD")
+  console.log("payload", payload)
+}

package/src/files/tasks/index.ts

@@ -0,0 +1,6 @@
+import queue from "../../../queue"
+
+import {finalize_file_upload} from "./finalize_file_upload"
+
+
+queue.register_task("finalize_file_upload", finalize_file_upload)

package/src/files/upload_chunk.ts

@@ -0,0 +1,82 @@
+import assert from "assert"
+import fs from "fs"
+import {formidable, File} from "formidable"
+
+import {get_grid_fs_bucket} from "./helpers/get_grid_fs_bucket"
+import { sim_test_inject } from "../helpers/sim_test_inject"
+
+
+const upload_file_to_bucket = async(file: File, metadata): Promise<void> => {
+  const bucket = get_grid_fs_bucket("file-uploads", metadata.chunk_size)
+
+  const chunk_filename = `${metadata.hash}.${metadata.chunk_index}`
+
+  const upload_stream = bucket.openUploadStream(chunk_filename, {
+    metadata,
+  })
+
+  const read_stream = fs.createReadStream(file.filepath)
+
+  read_stream.pipe(upload_stream)
+
+  return new Promise((resolve, reject) => {
+    upload_stream.on("finish", () => {
+      // console.log("finished uploading:", upload_stream.id)
+      resolve()
+    })
+    upload_stream.on("error", (error) => {
+      reject(error)
+    })
+  })
+}
+
+
+export const upload_chunk = async(req, res, next) => {
+  const {user_id} = req.session
+  assert(user_id, "upload_chunk: unable to resolve user_id")
+
+  // https://github.com/node-formidable/formidable#options
+  const form = formidable({})
+
+  let fields, files
+  try {
+    [fields, files] = await form.parse(req)
+  } catch (err) {
+    return res.status(500).json({error: "Failed to parse uploaded file"})
+  }
+
+  await sim_test_inject()
+  // await new Promise(resolve => setTimeout(resolve, 3000))
+
+  const file_chunk = files.file_chunk[0] as File
+
+  const metadata = {
+    user_id,
+    original_filename: fields.original_filename[0],
+    is_compressed: fields.is_compressed[0] === "yes",
+    chunk_index: parseInt(fields.chunk_index[0]),
+    total_chunks: parseInt(fields.total_chunks[0]),
+    chunk_size: parseInt(fields.chunk_size[0]),
+    mime_type: fields.mime_type[0],
+    hash: fields.hash[0],
+  }
+
+  await upload_file_to_bucket(file_chunk, metadata)
+
+  const result: {
+    status: string;
+    finalize_token?: string;
+  } = {
+    status: "ok",
+  }
+
+  const is_last_chunk = metadata.chunk_index === metadata.total_chunks - 1
+
+  if (is_last_chunk) {
+    result.finalize_token = metadata.hash
+  }
+
+  res.json({
+    status: "ok",
+  })
+}

package/src/helpers/sim_test_inject.ts

@@ -0,0 +1,21 @@
+const ERROR_PROBABILITY = 0.1
+const THROTTLE_PROBABILITY = 0.2
+const MAX_THROTTLE_MS = 2000
+
+// randomly inject errors or delays
+export const sim_test_inject = async() => {
+  if (!__DEV__) {
+    return
+  }
+
+  const should_error = Math.random() > 1 - ERROR_PROBABILITY
+  if (should_error) {
+    throw new Error("random sim test error")
+  }
+
+  const should_throttle = Math.random() > 1 - THROTTLE_PROBABILITY
+  if (should_throttle) {
+    const throttle_delay = Math.floor(Math.random() * MAX_THROTTLE_MS)
+    await new Promise(resolve => setTimeout(resolve, throttle_delay))
+  }
+}

package/src/tasks/index.js

@@ -1,6 +1,8 @@
 const queue = require("../../queue")
 
+require("../files/tasks")
 
 const index_item = require("./index_item")
 
+
 queue.register_task("rb_index_item", index_item)

package/tsconfig.json

@@ -0,0 +1,8 @@
+{
+  // this tsconfig file is only used by the linter, it is ignored in .npmignore
+  "extends": "../bundler-server/tsconfig.json",
+  "include": [
+    "**/*.ts",
+    "**/*.tsx"
+  ]
+}