@rpcbase/server 0.368.0 → 0.369.0-finalizeupload.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rpcbase/server",
-  "version": "0.368.0",
+  "version": "0.369.0-finalizeupload.1",
   "license": "SSPL-1.0",
   "main": "./index.js",
   "scripts": {
@@ -72,11 +72,11 @@
     "express": "4.19.2",
     "express-session": "1.18.0",
     "firebase-admin": "12.3.0",
-    "glob": "11.0.0",
     "lodash": "4.17.21",
     "mkdirp": "3.0.1",
     "mongoose": "8.5.2",
     "openai": "4.56.0",
+    "pdf2pic": "3.1.1",
     "picocolors": "1.0.1",
     "postmark": "4.0.4",
     "redis": "4.7.0",

package/src/client/client_router.js

@@ -2,7 +2,7 @@
 const assert = require("assert")
 const fs = require("fs")
 const path = require("path")
-const glob = require("glob")
+const glob = require("glob11")
 const Sentry = require("@sentry/node")
 
 
@@ -21,7 +21,6 @@ const has_sentry = !!SENTRY_DSN
 const is_development = NODE_ENV === "development"
 
 
-const src_path = path.join(process.cwd(), "./src/")
 const build_dir = path.join(process.cwd(), "build/")
 const client_build_dir = path.join(build_dir, "./client")
 
@@ -50,7 +49,7 @@ const serve_file = (req, res, full_path) => {
     headers: {
       "Cache-Control": `public, max-age=1800`, // 30 mins
       "Content-Type": content_type,
-    }
+    },
   })
 }
 
@@ -59,7 +58,6 @@ const client_router = (app) => {
   const client_routes = get_client_routes()
 
 
-
   const index_file_path = path.join(client_build_dir, "./index.html")
   if (fs.existsSync(index_file_path)) {
     const index_file_buffer = fs.readFileSync(index_file_path)
@@ -79,7 +77,7 @@ const client_router = (app) => {
       } else {
         // TODO: should handle 404 here
         res.writeHead(200, {
-          "Content-Type": "text/html"
+          "Content-Type": "text/html",
         })
         res.end(index_file_buffer)
       }

package/src/files/constants.ts

@@ -1,2 +1,9 @@
 export const UPLOAD_BUCKET_NAME = "file-uploads"
 export const TMP_UPLOADS_DIR = "rb-file-uploads"
+
+export type ProcessFilePayload = {
+  hash: string;
+  img_preview?: boolean;
+  ocr?: boolean;
+  text_vectors?: boolean;
+}

package/src/files/finalize_file_upload.ts

@@ -1,18 +1,12 @@
 import Promise from "bluebird"
 
 import queue from "../../queue"
+import {ProcessFilePayload} from "./constants"
 
 
-export type ProcessFilePayload = {
-  hash: string;
-  ocr?: boolean;
-  vectors?: boolean;
-  img_preview?: boolean;
-}
-
 const DEFAULTS = {
   ocr: false,
-  vectors: false,
+  text_vectors: false,
   img_preview: false,
 }
 

package/src/files/tasks/finalize_file_upload/apply_img_preview.ts

@@ -1,25 +1,49 @@
-import {exec} from "child_process"
-import {promisify} from "util"
+import {DOC_MIME_TYPES} from "./constants"
+import {exec} from "./helpers/exec"
+import {convert_pdf_to_png} from "./helpers/convert_pdf_to_png"
 
 
-const execAsync = promisify(exec)
+const convert_pdf = async({tmp_wd, full_path, metadata}) => {
+  console.log("converting from:", metadata.mime_type, "to: pdf")
+  const cmd = `libreoffice --headless --convert-to pdf:writer_pdf_Export --outdir ${tmp_wd} ${full_path}`
+  const result = await exec(cmd)
 
-interface Payload {
-  pdfPath: string;
-  imgOutputPath: string;
+  if (result.stderr) {
+    console.warn(result.stderr)
+  }
+
+  if (result.stdout) {
+    console.log(result.stdout)
+  }
 }
 
-export const apply_img_preview = async(payload: Payload): Promise<void> => {
-  console.log("PL", payload, execAsync)
-  // try {
-  //   const {stdout, stderr} = await execAsync(
-  //     `pdftoppm -png ${payload.pdfPath} ${payload.imgOutputPath}`,
-  //   )
-  //   if (stderr) {
-  //     console.error(`Error: ${stderr}`)
-  //   }
-  //   console.log(`Image preview created: ${stdout}`)
-  // } catch (error) {
-  //   console.error(`Execution error: ${error}`)
-  // }
+
+export const apply_img_preview = async({tmp_wd, full_path, metadata}): Promise<any> => {
+
+  let is_pdf_converted = false
+
+  // DOC
+  if (DOC_MIME_TYPES.includes(metadata.mime_type)) {
+    await convert_pdf({tmp_wd, full_path, metadata})
+    is_pdf_converted = true
+  }
+  // PDF
+  else if (metadata.mime_type === "application/pdf") {
+    // file is already a pdf, do nothing
+    is_pdf_converted = true
+  } else {
+    console.log("apply img, unknown mime type, not proceeding")
+  }
+
+  if (is_pdf_converted) {
+    const pdf_path = `${full_path}.pdf`
+    await convert_pdf_to_png({tmp_wd, pdf_path})
+  }
+
+  // TODO: convert spreadsheets to csv ?
+  // convert powerpoints to pdf also (they should work just the same as docs)
+
+  return {
+    is_pdf_converted,
+  }
 }

package/src/files/tasks/finalize_file_upload/constants.ts

@@ -0,0 +1,23 @@
+export const PDF_IMG_DIR = "img"
+
+export const DOC_MIME_TYPES = [
+  "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+  "application/msword",
+  "application/vnd.oasis.opendocument.text",
+  "application/rtf",
+]
+
+
+export const IMAGES_MIME_TYPES = [
+  "image/jpeg",
+  "image/png",
+  "image/gif",
+  "image/bmp",
+  "image/tiff",
+  "image/webp",
+  "image/svg+xml",
+  "image/vnd.microsoft.icon",
+  "image/heif",
+  "image/heic",
+  "image/vnd.adobe.photoshop",
+]

package/src/files/tasks/finalize_file_upload/download_file.ts

@@ -11,20 +11,6 @@ import {get_grid_fs_bucket} from "../../helpers/get_grid_fs_bucket"
 
 const DL_CHUNK_CONCURRENCY = 8
 
-const get_metadata = async(hash: string) => {
-  const files_coll = mongoose.connection.db.collection(
-    `${UPLOAD_BUCKET_NAME}.files`,
-  )
-
-  const first_chunk_name = `${hash}.0`
-
-  const file_metadata = await files_coll.findOne({
-    filename: first_chunk_name,
-  })
-
-  return file_metadata.metadata
-}
-
 const download_chunk =
   (bucket: mongoose.mongo.GridFSBucket, tmp_wd: string, metadata: any) =>
     (chunk_num: number) =>
@@ -93,11 +79,10 @@ const rebuild_file_from_chunks = async(
 }
 
 // downloads all chunks, reconstruct if necessary
-export const download_file = async(tmp_wd: string, hash: string): Promise<string> => {
+export const download_file = async({tmp_wd, metadata}:
+{ tmp_wd: string, metadata: any }): Promise<string> => {
   const bucket = get_grid_fs_bucket(UPLOAD_BUCKET_NAME)
 
-  const metadata = await get_metadata(hash)
-
   const chunks = Array.from({length: metadata.total_chunks}, (_, i) => i)
 
   // download all file chunks

package/src/files/tasks/finalize_file_upload/get_text_vectors.ts

@@ -0,0 +1,13 @@
+
+
+export const get_text_vectors = async(
+  {
+    tmp_wd,
+    metadata,
+  }: {
+    tmp_wd: string,
+    metadata: any
+  }
+) => {
+  console.log("text vectors NYI")
+}

package/src/files/tasks/finalize_file_upload/helpers/convert_pdf_to_png.ts

@@ -0,0 +1,34 @@
+import path from "path"
+import {mkdirp} from "mkdirp"
+import {fromPath} from "pdf2pic"
+
+import {PDF_IMG_DIR} from "../constants"
+
+
+export const convert_pdf_to_png = async(
+  {tmp_wd, pdf_path}:
+  { tmp_wd: string, pdf_path: string }
+) =>
+  new Promise(async(resolve, reject) => {
+    const out_dir = path.join(tmp_wd, `./${PDF_IMG_DIR}/`)
+
+    await mkdirp(out_dir)
+
+    const options = {
+      density: 220,
+      saveFilename: "page",
+      savePath: out_dir,
+      format: "png",
+      // width
+      // height
+      preserveAspectRatio: true,
+    }
+
+    const converter = fromPath(pdf_path, options)
+
+    converter.bulk(-1, {responseType: "image"})
+      .then((pages) => {
+        resolve(pages)
+      })
+
+  })

package/src/files/tasks/finalize_file_upload/helpers/exec.ts

@@ -0,0 +1,5 @@
+import {exec as _exec} from "child_process"
+import {promisify} from "util"
+
+
+export const exec = promisify(_exec)

package/src/files/tasks/finalize_file_upload/helpers/get_metadata.ts

@@ -0,0 +1,18 @@
+import mongoose from "../../../../../mongoose"
+
+import {UPLOAD_BUCKET_NAME} from "../../../constants"
+
+
+export const get_metadata = async(hash: string) => {
+  const files_coll = mongoose.connection.db.collection(
+    `${UPLOAD_BUCKET_NAME}.files`,
+  )
+
+  const first_chunk_name = `${hash}.0`
+
+  const file_metadata = await files_coll.findOne({
+    filename: first_chunk_name,
+  })
+
+  return file_metadata.metadata
+}

package/src/files/tasks/finalize_file_upload/index.ts

@@ -2,17 +2,22 @@ import path from "path"
 import os from "os"
 import {mkdirp} from "mkdirp"
 
+import {TMP_UPLOADS_DIR} from "../../constants"
+
+import {apply_img_preview} from "./apply_img_preview"
 import {download_file} from "./download_file"
+import {run_ocr} from "./run_ocr"
+import {get_metadata} from "./helpers/get_metadata"
+import {get_text_vectors} from "./get_text_vectors"
 
-import {TMP_UPLOADS_DIR} from "../../constants"
+import {ProcessFilePayload} from "../../constants"
 
 
 const {RB_TENANT_ID} = process.env
 
-export const finalize_file_upload = async(payload) => {
-  console.log("TASK:FINALIZE FILE UPLOAD")
 
-  const {hash, img_preview} = payload
+export const finalize_file_upload = async(payload: ProcessFilePayload) => {
+  const {hash, img_preview, ocr, text_vectors} = payload
 
   const tmp_wd = path.join(
     os.tmpdir(),
@@ -22,11 +27,27 @@ export const finalize_file_upload = async(payload) => {
   await mkdirp(tmp_wd)
   console.log("created dir", tmp_wd)
 
-  const full_path = await download_file(tmp_wd, hash)
+  const metadata = await get_metadata(hash)
 
-  console.log("got full path", full_path)
+  const full_path = await download_file({tmp_wd, metadata})
 
   if (img_preview) {
-    console.log("RUN img preview")
+    await apply_img_preview({tmp_wd, full_path, metadata})
+  }
+
+  if (ocr) {
+    if (!img_preview) {
+      throw new Error("img_preview cannot be false when ocr is set to true")
+    }
+
+    await run_ocr({tmp_wd, metadata})
+  }
+
+  if (text_vectors) {
+    if (!ocr) {
+      throw new Error("text_vectors requires ocr to be set to true")
+    }
+
+    await get_text_vectors({tmp_wd, metadata})
   }
 }

package/src/files/tasks/finalize_file_upload/run_ocr.ts

@@ -0,0 +1,42 @@
+import path from "path"
+import Promise from "bluebird"
+import {glob} from "glob11"
+
+import {exec} from "./helpers/exec"
+
+import {PDF_IMG_DIR} from "./constants"
+
+
+const get_input_files = async(tmp_wd: string): Promise<Array<string>> => {
+  const input_glob = path.join(tmp_wd, `./${PDF_IMG_DIR}/*.png`)
+
+  const input_files = await glob(input_glob)
+
+  const sorted = input_files.sort((a, b) => {
+    // Extract the page numbers from the file names
+    const page_a = parseInt(a.match(/page\.(\d+)\.png/)?.[1] || "0", 10)
+    const page_b = parseInt(b.match(/page\.(\d+)\.png/)?.[1] || "0", 10)
+
+    return page_a - page_b
+  })
+
+  return sorted
+}
+
+
+export const run_ocr = async({tmp_wd, metadata}: { tmp_wd: string, metadata: any }) => {
+
+  const input_files = await get_input_files(tmp_wd)
+
+  const run_ocr_file = async(file_path: string) => {
+    const wd = path.dirname(file_path)
+    const basename = path.basename(file_path, ".png")
+
+    const cmd = `tesseract ${basename}.png ${basename} -l eng --oem 1 --psm 11`
+    const out = await exec(cmd, {cwd: wd})
+    console.log("OUTTT", out)
+  }
+
+  await Promise.map(input_files, run_ocr_file, {concurrency: 4})
+
+}