@rpcbase/server 0.366.0 → 0.368.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rpcbase/server",
-  "version": "0.366.0",
+  "version": "0.368.0",
   "license": "SSPL-1.0",
   "main": "./index.js",
   "scripts": {

package/src/files/constants.ts

@@ -0,0 +1,2 @@
+export const UPLOAD_BUCKET_NAME = "file-uploads"
+export const TMP_UPLOADS_DIR = "rb-file-uploads"

package/src/files/finalize_file_upload.ts

@@ -3,17 +3,29 @@ import Promise from "bluebird"
 import queue from "../../queue"
 
 
+export type ProcessFilePayload = {
+  hash: string;
+  ocr?: boolean;
+  vectors?: boolean;
+  img_preview?: boolean;
+}
+
+const DEFAULTS = {
+  ocr: false,
+  vectors: false,
+  img_preview: false,
+}
+
 // after we've uploaded a file, we process it with ocr, llm vectorization, png rendering, etc
-export const finalize_file_upload = async(files: Array<{hash: string}>) => {
+export const finalize_file_upload = async(files: Array<ProcessFilePayload>) => {
 
   await Promise.map(files,
-  async(file: {hash: string}) => {
-    await queue.add("finalize_file_upload", {hash: file.hash, hello: "world"}, {
-      jobId: `finalize_file_upload-${file.hash}`,
-      removeOnComplete: true,
-      removeOnFail: true,
+    async(file: ProcessFilePayload) => {
+      await queue.add("finalize_file_upload", {...DEFAULTS, ...file}, {
+        jobId: `finalize_file_upload-${file.hash}`,
+        removeOnComplete: true,
+        removeOnFail: true,
+      })
     })
-  }
-  )
 
 }

package/src/files/helpers/get_grid_fs_bucket.ts

@@ -4,8 +4,10 @@ import mongoose from "../../../mongoose"
 
 const CHUNK_SIZE = 1024 * 1024
 
-
-export const get_grid_fs_bucket = (bucket_name: string, chunk_size: number) => {
+export const get_grid_fs_bucket = (
+  bucket_name: string,
+  chunk_size: number = CHUNK_SIZE,
+) => {
   assert(chunk_size === CHUNK_SIZE, "chunk_size must match default CHUNK_SIZE")
 
   const {db} = mongoose.connection

package/src/files/tasks/finalize_file_upload/apply_img_preview.ts

@@ -0,0 +1,25 @@
+import {exec} from "child_process"
+import {promisify} from "util"
+
+
+const execAsync = promisify(exec)
+
+interface Payload {
+  pdfPath: string;
+  imgOutputPath: string;
+}
+
+export const apply_img_preview = async(payload: Payload): Promise<void> => {
+  console.log("PL", payload, execAsync)
+  // try {
+  //   const {stdout, stderr} = await execAsync(
+  //     `pdftoppm -png ${payload.pdfPath} ${payload.imgOutputPath}`,
+  //   )
+  //   if (stderr) {
+  //     console.error(`Error: ${stderr}`)
+  //   }
+  //   console.log(`Image preview created: ${stdout}`)
+  // } catch (error) {
+  //   console.error(`Execution error: ${error}`)
+  // }
+}

package/src/files/tasks/finalize_file_upload/download_file.ts

@@ -0,0 +1,113 @@
+import fs from "fs/promises"
+import {createReadStream, createWriteStream} from "fs"
+import path from "path"
+import {createBrotliDecompress} from "zlib"
+import Promise from "bluebird"
+
+import mongoose from "../../../../mongoose"
+import {UPLOAD_BUCKET_NAME} from "../../constants"
+import {get_grid_fs_bucket} from "../../helpers/get_grid_fs_bucket"
+
+
+const DL_CHUNK_CONCURRENCY = 8
+
+const get_metadata = async(hash: string) => {
+  const files_coll = mongoose.connection.db.collection(
+    `${UPLOAD_BUCKET_NAME}.files`,
+  )
+
+  const first_chunk_name = `${hash}.0`
+
+  const file_metadata = await files_coll.findOne({
+    filename: first_chunk_name,
+  })
+
+  return file_metadata.metadata
+}
+
+const download_chunk =
+  (bucket: mongoose.mongo.GridFSBucket, tmp_wd: string, metadata: any) =>
+    (chunk_num: number) =>
+      new Promise((resolve, reject) => {
+        const filename = `${metadata.hash}.${chunk_num}`
+
+        const dl_stream = bucket.openDownloadStreamByName(filename)
+
+        const file_path = path.join(tmp_wd, `./${filename}`)
+
+        const write_stream = createWriteStream(file_path)
+
+        if (metadata.is_compressed) {
+          const decompress_stream = createBrotliDecompress()
+          dl_stream.pipe(decompress_stream).pipe(write_stream)
+        } else {
+          dl_stream.pipe(write_stream)
+        }
+
+        dl_stream.on("error", (error) => {
+          console.error("Error downloading file:", error)
+          reject(error)
+        })
+
+        write_stream.on("error", (error) => {
+          console.error("Error writing file:", error)
+          reject(error)
+        })
+
+        write_stream.on("finish", () => {
+          console.log("File downloaded and written successfully", file_path)
+          resolve()
+        })
+      })
+
+
+const rebuild_file_from_chunks = async(
+  tmp_wd: string,
+  metadata: any,
+  chunks: Array<number>,
+) => {
+  const full_file_path = path.join(tmp_wd, `./${metadata.hash}`)
+  const output_stream = createWriteStream(full_file_path)
+
+  const append_chunk = (chunk_num: number) => new Promise<void>((resolve, reject) => {
+    const chunk_path = path.join(tmp_wd, `./${metadata.hash}.${chunk_num}`)
+
+    const chunk_read_stream = createReadStream(chunk_path)
+
+    chunk_read_stream.pipe(output_stream, {end: false})
+
+    chunk_read_stream.on("end", async() => {
+      await fs.rm(chunk_path)
+      resolve()
+    })
+    chunk_read_stream.on("error", reject)
+  })
+
+  for (const chunk_num of chunks) {
+    await append_chunk(chunk_num)
+  }
+
+  output_stream.end()
+
+  return full_file_path
+}
+
+// downloads all chunks, reconstruct if necessary
+export const download_file = async(tmp_wd: string, hash: string): Promise<string> => {
+  const bucket = get_grid_fs_bucket(UPLOAD_BUCKET_NAME)
+
+  const metadata = await get_metadata(hash)
+
+  const chunks = Array.from({length: metadata.total_chunks}, (_, i) => i)
+
+  // download all file chunks
+  await Promise.map(
+    chunks,
+    download_chunk(bucket, tmp_wd, metadata),
+    {concurrency: DL_CHUNK_CONCURRENCY},
+  )
+
+  const full_path = await rebuild_file_from_chunks(tmp_wd, metadata, chunks)
+
+  return full_path
+}

package/src/files/tasks/finalize_file_upload/index.ts

@@ -0,0 +1,32 @@
+import path from "path"
+import os from "os"
+import {mkdirp} from "mkdirp"
+
+import {download_file} from "./download_file"
+
+import {TMP_UPLOADS_DIR} from "../../constants"
+
+
+const {RB_TENANT_ID} = process.env
+
+export const finalize_file_upload = async(payload) => {
+  console.log("TASK:FINALIZE FILE UPLOAD")
+
+  const {hash, img_preview} = payload
+
+  const tmp_wd = path.join(
+    os.tmpdir(),
+    `./${TMP_UPLOADS_DIR}/${RB_TENANT_ID}/${hash}`,
+  )
+
+  await mkdirp(tmp_wd)
+  console.log("created dir", tmp_wd)
+
+  const full_path = await download_file(tmp_wd, hash)
+
+  console.log("got full path", full_path)
+
+  if (img_preview) {
+    console.log("RUN img preview")
+  }
+}

package/src/files/upload_chunk.ts

@@ -3,11 +3,12 @@ import fs from "fs"
 import {formidable, File} from "formidable"
 
 import {get_grid_fs_bucket} from "./helpers/get_grid_fs_bucket"
-import { sim_test_inject } from "../helpers/sim_test_inject"
+import {sim_test_inject} from "../helpers/sim_test_inject"
+import {UPLOAD_BUCKET_NAME} from "./constants"
 
 
 const upload_file_to_bucket = async(file: File, metadata): Promise<void> => {
-  const bucket = get_grid_fs_bucket("file-uploads", metadata.chunk_size)
+  const bucket = get_grid_fs_bucket(UPLOAD_BUCKET_NAME, metadata.chunk_size)
 
   const chunk_filename = `${metadata.hash}.${metadata.chunk_index}`
 

package/src/helpers/sim_test_inject.ts

@@ -1,4 +1,4 @@
-const ERROR_PROBABILITY = 0.1
+const ERROR_PROBABILITY = 0.001
 const THROTTLE_PROBABILITY = 0.2
 const MAX_THROTTLE_MS = 2000
 

package/src/files/tasks/finalize_file_upload.ts

@@ -1,5 +0,0 @@
-
-export const finalize_file_upload = async(payload) => {
-  console.log("TASK:FINALIZE FILE UPLOAD")
-  console.log("payload", payload)
-}