@rpcbase/server 0.367.0 → 0.369.0-finalizeupload.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rpcbase/server",
-  "version": "0.367.0",
+  "version": "0.369.0-finalizeupload.0",
   "license": "SSPL-1.0",
   "main": "./index.js",
   "scripts": {
@@ -72,11 +72,11 @@
     "express": "4.19.2",
     "express-session": "1.18.0",
     "firebase-admin": "12.3.0",
-    "glob": "11.0.0",
     "lodash": "4.17.21",
     "mkdirp": "3.0.1",
     "mongoose": "8.5.2",
     "openai": "4.56.0",
+    "pdf2pic": "3.1.1",
     "picocolors": "1.0.1",
     "postmark": "4.0.4",
     "redis": "4.7.0",

package/src/client/client_router.js

@@ -2,7 +2,7 @@
 const assert = require("assert")
 const fs = require("fs")
 const path = require("path")
-const glob = require("glob")
+const glob = require("glob11")
 const Sentry = require("@sentry/node")
 
 
@@ -21,7 +21,7 @@ const has_sentry = !!SENTRY_DSN
 const is_development = NODE_ENV === "development"
 
 
-const src_path = path.join(process.cwd(), "./src/")
+// const src_path = path.join(process.cwd(), "./src/")
 const build_dir = path.join(process.cwd(), "build/")
 const client_build_dir = path.join(build_dir, "./client")
 
@@ -50,7 +50,7 @@ const serve_file = (req, res, full_path) => {
     headers: {
       "Cache-Control": `public, max-age=1800`, // 30 mins
       "Content-Type": content_type,
-    }
+    },
   })
 }
 
@@ -59,7 +59,6 @@ const client_router = (app) => {
   const client_routes = get_client_routes()
 
 
-
   const index_file_path = path.join(client_build_dir, "./index.html")
   if (fs.existsSync(index_file_path)) {
     const index_file_buffer = fs.readFileSync(index_file_path)
@@ -79,7 +78,7 @@ const client_router = (app) => {
       } else {
         // TODO: should handle 404 here
         res.writeHead(200, {
-          "Content-Type": "text/html"
+          "Content-Type": "text/html",
         })
         res.end(index_file_buffer)
       }

package/src/files/constants.ts

@@ -0,0 +1,9 @@
+export const UPLOAD_BUCKET_NAME = "file-uploads"
+export const TMP_UPLOADS_DIR = "rb-file-uploads"
+
+export type ProcessFilePayload = {
+  hash: string;
+  img_preview?: boolean;
+  ocr?: boolean;
+  text_vectors?: boolean;
+}

package/src/files/finalize_file_upload.ts

@@ -1,18 +1,12 @@
 import Promise from "bluebird"
 
 import queue from "../../queue"
+import {ProcessFilePayload} from "./constants"
 
 
-export type ProcessFilePayload = {
-  hash: string;
-  ocr?: boolean;
-  vectors?: boolean;
-  img_preview?: boolean;
-}
-
 const DEFAULTS = {
   ocr: false,
-  vectors: false,
+  text_vectors: false,
   img_preview: false,
 }
 

package/src/files/helpers/get_grid_fs_bucket.ts

@@ -4,8 +4,10 @@ import mongoose from "../../../mongoose"
 
 const CHUNK_SIZE = 1024 * 1024
 
-
-export const get_grid_fs_bucket = (bucket_name: string, chunk_size: number) => {
+export const get_grid_fs_bucket = (
+  bucket_name: string,
+  chunk_size: number = CHUNK_SIZE,
+) => {
   assert(chunk_size === CHUNK_SIZE, "chunk_size must match default CHUNK_SIZE")
 
   const {db} = mongoose.connection

package/src/files/tasks/finalize_file_upload/apply_img_preview.ts

@@ -0,0 +1,49 @@
+import {DOC_MIME_TYPES} from "./constants"
+import {exec} from "./helpers/exec"
+import {convert_pdf_to_png} from "./helpers/convert_pdf_to_png"
+
+
+const convert_pdf = async({tmp_wd, full_path, metadata}) => {
+  console.log("converting from:", metadata.mime_type, "to: pdf")
+  const cmd = `libreoffice --headless --convert-to pdf:writer_pdf_Export --outdir ${tmp_wd} ${full_path}`
+  const result = await exec(cmd)
+
+  if (result.stderr) {
+    console.warn(result.stderr)
+  }
+
+  if (result.stdout) {
+    console.log(result.stdout)
+  }
+}
+
+
+export const apply_img_preview = async({tmp_wd, full_path, metadata}): Promise<any> => {
+
+  let is_pdf_converted = false
+
+  // DOC
+  if (DOC_MIME_TYPES.includes(metadata.mime_type)) {
+    await convert_pdf({tmp_wd, full_path, metadata})
+    is_pdf_converted = true
+  }
+  // PDF
+  else if (metadata.mime_type === "application/pdf") {
+    // file is already a pdf, do nothing
+    is_pdf_converted = true
+  } else {
+    console.log("apply img, unknown mime type, not proceeding")
+  }
+
+  if (is_pdf_converted) {
+    const pdf_path = `${full_path}.pdf`
+    await convert_pdf_to_png({tmp_wd, pdf_path})
+  }
+
+  // TODO: convert spreadsheets to csv ?
+  // convert powerpoints to pdf also (they should work just the same as docs)
+
+  return {
+    is_pdf_converted,
+  }
+}

package/src/files/tasks/finalize_file_upload/constants.ts

@@ -0,0 +1,23 @@
+export const PDF_IMG_DIR = "img"
+
+export const DOC_MIME_TYPES = [
+  "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+  "application/msword",
+  "application/vnd.oasis.opendocument.text",
+  "application/rtf",
+]
+
+
+export const IMAGES_MIME_TYPES = [
+  "image/jpeg",
+  "image/png",
+  "image/gif",
+  "image/bmp",
+  "image/tiff",
+  "image/webp",
+  "image/svg+xml",
+  "image/vnd.microsoft.icon",
+  "image/heif",
+  "image/heic",
+  "image/vnd.adobe.photoshop",
+]

package/src/files/tasks/finalize_file_upload/download_file.ts

@@ -0,0 +1,98 @@
+import fs from "fs/promises"
+import {createReadStream, createWriteStream} from "fs"
+import path from "path"
+import {createBrotliDecompress} from "zlib"
+import Promise from "bluebird"
+
+import mongoose from "../../../../mongoose"
+import {UPLOAD_BUCKET_NAME} from "../../constants"
+import {get_grid_fs_bucket} from "../../helpers/get_grid_fs_bucket"
+
+
+const DL_CHUNK_CONCURRENCY = 8
+
+const download_chunk =
+  (bucket: mongoose.mongo.GridFSBucket, tmp_wd: string, metadata: any) =>
+    (chunk_num: number) =>
+      new Promise((resolve, reject) => {
+        const filename = `${metadata.hash}.${chunk_num}`
+
+        const dl_stream = bucket.openDownloadStreamByName(filename)
+
+        const file_path = path.join(tmp_wd, `./${filename}`)
+
+        const write_stream = createWriteStream(file_path)
+
+        if (metadata.is_compressed) {
+          const decompress_stream = createBrotliDecompress()
+          dl_stream.pipe(decompress_stream).pipe(write_stream)
+        } else {
+          dl_stream.pipe(write_stream)
+        }
+
+        dl_stream.on("error", (error) => {
+          console.error("Error downloading file:", error)
+          reject(error)
+        })
+
+        write_stream.on("error", (error) => {
+          console.error("Error writing file:", error)
+          reject(error)
+        })
+
+        write_stream.on("finish", () => {
+          console.log("File downloaded and written successfully", file_path)
+          resolve()
+        })
+      })
+
+
+const rebuild_file_from_chunks = async(
+  tmp_wd: string,
+  metadata: any,
+  chunks: Array<number>,
+) => {
+  const full_file_path = path.join(tmp_wd, `./${metadata.hash}`)
+  const output_stream = createWriteStream(full_file_path)
+
+  const append_chunk = (chunk_num: number) => new Promise<void>((resolve, reject) => {
+    const chunk_path = path.join(tmp_wd, `./${metadata.hash}.${chunk_num}`)
+
+    const chunk_read_stream = createReadStream(chunk_path)
+
+    chunk_read_stream.pipe(output_stream, {end: false})
+
+    chunk_read_stream.on("end", async() => {
+      await fs.rm(chunk_path)
+      resolve()
+    })
+    chunk_read_stream.on("error", reject)
+  })
+
+  for (const chunk_num of chunks) {
+    await append_chunk(chunk_num)
+  }
+
+  output_stream.end()
+
+  return full_file_path
+}
+
+// downloads all chunks, reconstruct if necessary
+export const download_file = async({tmp_wd, metadata}:
+{ tmp_wd: string, metadata: any }): Promise<string> => {
+  const bucket = get_grid_fs_bucket(UPLOAD_BUCKET_NAME)
+
+  const chunks = Array.from({length: metadata.total_chunks}, (_, i) => i)
+
+  // download all file chunks
+  await Promise.map(
+    chunks,
+    download_chunk(bucket, tmp_wd, metadata),
+    {concurrency: DL_CHUNK_CONCURRENCY},
+  )
+
+  const full_path = await rebuild_file_from_chunks(tmp_wd, metadata, chunks)
+
+  return full_path
+}

package/src/files/tasks/finalize_file_upload/get_text_vectors.ts

@@ -0,0 +1,13 @@
+
+
+export const get_text_vectors = async(
+  {
+    tmp_wd,
+    metadata,
+  }: {
+    tmp_wd: string,
+    metadata: any
+  }
+) => {
+  console.log("text vectors NYI")
+}

package/src/files/tasks/finalize_file_upload/helpers/convert_pdf_to_png.ts

@@ -0,0 +1,34 @@
+import path from "path"
+import {mkdirp} from "mkdirp"
+import {fromPath} from "pdf2pic"
+
+import {PDF_IMG_DIR} from "../constants"
+
+
+export const convert_pdf_to_png = async(
+  {tmp_wd, pdf_path}:
+  { tmp_wd: string, pdf_path: string }
+) =>
+  new Promise(async(resolve, reject) => {
+    const out_dir = path.join(tmp_wd, `./${PDF_IMG_DIR}/`)
+
+    await mkdirp(out_dir)
+
+    const options = {
+      density: 220,
+      saveFilename: "page",
+      savePath: out_dir,
+      format: "png",
+      // width
+      // height
+      preserveAspectRatio: true,
+    }
+
+    const converter = fromPath(pdf_path, options)
+
+    converter.bulk(-1, {responseType: "image"})
+      .then((pages) => {
+        resolve(pages)
+      })
+
+  })

package/src/files/tasks/finalize_file_upload/helpers/exec.ts

@@ -0,0 +1,5 @@
+import {exec as _exec} from "child_process"
+import {promisify} from "util"
+
+
+export const exec = promisify(_exec)

package/src/files/tasks/finalize_file_upload/helpers/get_metadata.ts

@@ -0,0 +1,18 @@
+import mongoose from "../../../../../mongoose"
+
+import {UPLOAD_BUCKET_NAME} from "../../../constants"
+
+
+export const get_metadata = async(hash: string) => {
+  const files_coll = mongoose.connection.db.collection(
+    `${UPLOAD_BUCKET_NAME}.files`,
+  )
+
+  const first_chunk_name = `${hash}.0`
+
+  const file_metadata = await files_coll.findOne({
+    filename: first_chunk_name,
+  })
+
+  return file_metadata.metadata
+}

package/src/files/tasks/finalize_file_upload/index.ts

@@ -0,0 +1,53 @@
+import path from "path"
+import os from "os"
+import {mkdirp} from "mkdirp"
+
+import {TMP_UPLOADS_DIR} from "../../constants"
+
+import {apply_img_preview} from "./apply_img_preview"
+import {download_file} from "./download_file"
+import {run_ocr} from "./run_ocr"
+import {get_metadata} from "./helpers/get_metadata"
+import {get_text_vectors} from "./get_text_vectors"
+
+import {ProcessFilePayload} from "../../constants"
+
+
+const {RB_TENANT_ID} = process.env
+
+
+export const finalize_file_upload = async(payload: ProcessFilePayload) => {
+  const {hash, img_preview, ocr, text_vectors} = payload
+
+  const tmp_wd = path.join(
+    os.tmpdir(),
+    `./${TMP_UPLOADS_DIR}/${RB_TENANT_ID}/${hash}`,
+  )
+
+  await mkdirp(tmp_wd)
+  console.log("created dir", tmp_wd)
+
+  const metadata = await get_metadata(hash)
+
+  const full_path = await download_file({tmp_wd, metadata})
+
+  if (img_preview) {
+    await apply_img_preview({tmp_wd, full_path, metadata})
+  }
+
+  if (ocr) {
+    if (!img_preview) {
+      throw new Error("img_preview cannot be false when ocr is set to true")
+    }
+
+    await run_ocr({tmp_wd, metadata})
+  }
+
+  if (text_vectors) {
+    if (!ocr) {
+      throw new Error("text_vectors requires ocr to be set to true")
+    }
+
+    await get_text_vectors({tmp_wd, metadata})
+  }
+}

package/src/files/tasks/finalize_file_upload/run_ocr.ts

@@ -0,0 +1,42 @@
+import path from "path"
+import Promise from "bluebird"
+import {glob} from "glob11"
+
+import {exec} from "./helpers/exec"
+
+import {PDF_IMG_DIR} from "./constants"
+
+
+const get_input_files = async(tmp_wd: string): Promise<Array<string>> => {
+  const input_glob = path.join(tmp_wd, `./${PDF_IMG_DIR}/*.png`)
+
+  const input_files = await glob(input_glob)
+
+  const sorted = input_files.sort((a, b) => {
+    // Extract the page numbers from the file names
+    const page_a = parseInt(a.match(/page\.(\d+)\.png/)?.[1] || "0", 10)
+    const page_b = parseInt(b.match(/page\.(\d+)\.png/)?.[1] || "0", 10)
+
+    return page_a - page_b
+  })
+
+  return sorted
+}
+
+
+export const run_ocr = async({tmp_wd, metadata}: { tmp_wd: string, metadata: any }) => {
+
+  const input_files = await get_input_files(tmp_wd)
+
+  const run_ocr_file = async(file_path: string) => {
+    const wd = path.dirname(file_path)
+    const basename = path.basename(file_path, ".png")
+
+    const cmd = `tesseract ${basename}.png ${basename} -l eng --oem 1 --psm 11`
+    const out = await exec(cmd, {cwd: wd})
+    console.log("OUTTT", out)
+  }
+
+  await Promise.map(input_files, run_ocr_file, {concurrency: 4})
+
+}

package/src/files/upload_chunk.ts

@@ -3,11 +3,12 @@ import fs from "fs"
 import {formidable, File} from "formidable"
 
 import {get_grid_fs_bucket} from "./helpers/get_grid_fs_bucket"
-import { sim_test_inject } from "../helpers/sim_test_inject"
+import {sim_test_inject} from "../helpers/sim_test_inject"
+import {UPLOAD_BUCKET_NAME} from "./constants"
 
 
 const upload_file_to_bucket = async(file: File, metadata): Promise<void> => {
-  const bucket = get_grid_fs_bucket("file-uploads", metadata.chunk_size)
+  const bucket = get_grid_fs_bucket(UPLOAD_BUCKET_NAME, metadata.chunk_size)
 
   const chunk_filename = `${metadata.hash}.${metadata.chunk_index}`
 

package/src/helpers/sim_test_inject.ts

@@ -1,4 +1,4 @@
-const ERROR_PROBABILITY = 0.1
+const ERROR_PROBABILITY = 0.001
 const THROTTLE_PROBABILITY = 0.2
 const MAX_THROTTLE_MS = 2000
 

package/src/files/tasks/finalize_file_upload.ts

@@ -1,5 +0,0 @@
-
-export const finalize_file_upload = async(payload) => {
-  console.log("TASK:FINALIZE FILE UPLOAD")
-  console.log("payload", payload)
-}