@rpcbase/server 0.380.0 → 0.382.0

package/src/auth/set_new_password.js

@@ -1,63 +0,0 @@
-/* @flow */
-const _template = require("lodash/template")
-const fs = require("fs")
-const path = require("path")
-const isEmail = require("validator/lib/isEmail")
-
-const {hash_password, compare_hash} = require("@rpcbase/std/crypto/hash")
-
-const mailer = require("../../mailer")
-const mongoose = require("../../mongoose")
-
-const set_new_password_email = require("./set_new_password_email.html")
-
-const ResetPasswordToken = require("../models/ResetPasswordToken")
-const User = require("../models/User")
-
-const {APP_DOMAIN} = process.env
-
-const email_tpl = _template(set_new_password_email)
-
-const set_new_password = async({user_id, token, password}, ctx) => {
-
-  const reset_tokens = await ResetPasswordToken.find({user_id})
-    .limit(100)
-
-  if (reset_tokens.length === 0) {
-    return {
-      message: "Unable to validate reset token, please try again in a moment"
-    }
-  }
-
-  let is_match = false
-
-  for (let i = 0; i < reset_tokens.length; i++) {
-    const op = await compare_hash(token, reset_tokens[i].token_hash)
-    if (op) {
-      is_match = true
-      break
-    }
-  }
-
-  if (!is_match) {
-    return {
-      message: "Invalid or expired token, please try again in a moment"
-    }
-  }
-
-  // token is valid, update the user password
-  const user = await User.findOne({_id: user_id}, null, {ctx})
-  const hashed_password = await hash_password(password)
-  user.password_hash = hashed_password
-
-  await user.save()
-
-  // WARNING:
-  // TODO: important! notify the user that their password has been reset
-
-  return {
-    status: "ok"
-  }
-}
-
-module.exports = set_new_password

package/src/auth/set_new_password_email.html

@@ -1,3 +0,0 @@
-todo: implement notification email
-
-your password has been reset

package/src/auth/sign_in.js

@@ -1,61 +0,0 @@
-/* @flow */
-const {compare_hash} = require("@rpcbase/std/crypto/hash")
-const debug = require("debug")
-
-const mongoose = require("../../mongoose")
-const User = require("../models/User")
-
-
-const log = debug("rb:auth:signin")
-
-const has_session_store = process.env.RB_SESSION_STORE === "yes"
-
-const USER_ID_HEADER = "rb-user-id"
-
-const fail = () => ({
-  errors: {form: "Invalid email or password"}
-})
-
-
-const sign_in = async({email, password}, ctx) => {
-  const {req, res} = ctx
-
-  // find matching user
-  const user = await User.findOne({email}, null)
-
-  if (!user) {
-    return fail()
-  }
-
-  const hashed_pass = user.password_hash
-
-  const is_match = await compare_hash(password, hashed_pass)
-
-  if (is_match) {
-    log("compare_hash is match")
-    const user_id = user._id.toString()
-
-    if (has_session_store) {
-      req.session.user_id = user_id
-      req.session.signed_in_at_ms = Date.now()
-      await req.session.save()
-    } else {
-      // On sign in success, we set this header to tell the reverse-proxy sign in was OK
-      res.setHeader(USER_ID_HEADER, user_id)
-    }
-
-    log("session saved, user is signed in")
-
-    return {
-      status: "ok",
-      user_id,
-    }
-
-  } else {
-    log("no match, returning error")
-
-    return fail()
-  }
-}
-
-module.exports = sign_in

package/src/auth/sign_out.js

@@ -1,11 +0,0 @@
-/* @flow */
-
-const sign_out = async(payload, ctx) => {
-  const {req} = ctx
-  await req.session.destroy()
-  return {
-    status: "ok"
-  }
-}
-
-module.exports = sign_out

package/src/auth/sign_up.js

@@ -1,56 +0,0 @@
-/* @flow */
-const {hash_password} = require("@rpcbase/std/crypto/hash")
-
-const UserStoredValues = require("../models/UserStoredValues")
-const User = require("../models/User")
-const Invite = require("../models/Invite")
-
-const MIN_PASSWORD_LENGTH = 12
-
-const sign_up = async(payload, ctx, session) => {
-  const {email, password} = payload
-  expect(email).toBeEmail()
-  expect(password.length).toBeGreaterThanOrEqual(MIN_PASSWORD_LENGTH)
-
-  // check if the user already exists
-  const existing_user = await User.findOne({email}, null, {ctx, session})
-
-  if (existing_user) {
-    throw new Error("User already exists")
-  }
-
-  // check if we have an invite for this user
-  const invite = await Invite.findOne({email}, null, {ctx, session})
-
-  // TODO: mark invite as accepted here so it can't be reused
-  if (invite && !invite.is_ready) {
-    console.log("found an invite, but not ready", email)
-    throw new Error("Your invite is still pending approval. Expect an email in the next weeks to activate your account.")
-  } else if (!invite) {
-    console.log("no invite for signup email:", email)
-    throw new Error("No invite was found for this email")
-  }
-
-  const hash = await hash_password(password)
-
-  const user = new User({
-    email,
-    password_hash: hash
-  })
-
-
-  // sign the user in
-  const user_id = user._id.toString()
-
-  const values_store = new UserStoredValues({
-    _owners: [user_id]
-  })
-
-  // WARNING: it is now the responsibility of the app to add user.save with session
-  // await user.save({ctx, session})
-  // await values_store.save({ctx, session})
-
-  return {user, values_store}
-}
-
-module.exports = sign_up

package/src/client/client_router.js

@@ -1,105 +0,0 @@
-/* @flow */
-const assert = require("assert")
-const fs = require("fs")
-const path = require("path")
-const glob = require("glob11")
-const Sentry = require("@sentry/node")
-
-
-const MIME_TYPES = {
-  ".css": "text/css",
-  ".js": "text/javascript",
-  ".json": "application/json",
-  ".svg": "image/svg+xml",
-  ".woff2": "font/woff2",
-  ".png": "image/png",
-}
-
-const {SENTRY_DSN, NODE_ENV} = process.env
-const has_sentry = !!SENTRY_DSN
-
-const is_development = NODE_ENV === "development"
-
-
-const build_dir = path.join(process.cwd(), "build/")
-const client_build_dir = path.join(build_dir, "./client")
-
-
-const get_client_routes = () => {
-  const client_files = glob.sync(path.join(client_build_dir, "./**/*"))
-  const routes = client_files
-    .filter((f) => {
-      const stat = fs.statSync(f)
-      return !stat.isDirectory()
-    })
-    .map((f) => `/${path.relative(client_build_dir, f)}`)
-  return routes
-}
-
-
-const serve_file = (req, res, full_path) => {
-  const file_path = path.join(client_build_dir, full_path)
-
-  const extname = path.extname(file_path)
-
-  const content_type = MIME_TYPES[extname]
-  assert(content_type, "unable to resolve content type")
-
-  res.sendFile(file_path, {
-    headers: {
-      "Cache-Control": `public, max-age=1800`, // 30 mins
-      "Content-Type": content_type,
-    },
-  })
-}
-
-
-const client_router = (app) => {
-  const client_routes = get_client_routes()
-
-
-  const index_file_path = path.join(client_build_dir, "./index.html")
-  if (fs.existsSync(index_file_path)) {
-    const index_file_buffer = fs.readFileSync(index_file_path)
-
-    app.get("*", async(req, res, next) => {
-      const full_path = req.baseUrl + req.path
-
-      // TODO: this shouldn't be here,
-      // it should be handled by the api and rpc routers being declared before this router
-      if (full_path.startsWith("/api/") || full_path.startsWith("/rpc/")) {
-        return next()
-      }
-
-      if (client_routes.indexOf(full_path) > -1) {
-        serve_file(req, res, full_path)
-        return
-      } else {
-        // TODO: should handle 404 here
-        res.writeHead(200, {
-          "Content-Type": "text/html",
-        })
-        res.end(index_file_buffer)
-      }
-    })
-  }
-  // register final error handlers
-
-  if (has_sentry && !is_development) {
-    // The error handler must be registered before any other error middleware and after all controllers
-    app.use(Sentry.Handlers.errorHandler())
-  }
-
-
-  // TODO: this should not be here
-  app.use((err, req, res, next) => {
-    console.error("server got err2", err)
-    console.log("stack:", err.stack)
-    res.status(500).json({
-      status: "error",
-      message: "something went wrong",
-    })
-  })
-}
-
-module.exports = client_router

package/src/files/constants.ts

@@ -1,9 +0,0 @@
-export const UPLOAD_BUCKET_NAME = "file-uploads"
-export const TMP_UPLOADS_DIR = "rb-file-uploads"
-
-export type ProcessFilePayload = {
-  hash: string;
-  img_preview?: boolean;
-  ocr?: boolean;
-  text_vectors?: boolean;
-}

package/src/files/finalize_file_upload.ts

@@ -1,25 +0,0 @@
-import Promise from "bluebird"
-
-import queue from "../../queue"
-import {ProcessFilePayload} from "./constants"
-
-
-const DEFAULTS = {
-  ocr: false,
-  text_vectors: false,
-  img_preview: false,
-}
-
-// after we've uploaded a file, we process it with ocr, llm vectorization, png rendering, etc
-export const finalize_file_upload = async(files: Array<ProcessFilePayload>) => {
-
-  await Promise.map(files,
-    async(file: ProcessFilePayload) => {
-      await queue.add("finalize_file_upload", {...DEFAULTS, ...file}, {
-        jobId: `finalize_file_upload-${file.hash}`,
-        removeOnComplete: true,
-        removeOnFail: true,
-      })
-    })
-
-}

package/src/files/helpers/get_grid_fs_bucket.ts

@@ -1,20 +0,0 @@
-import assert from "assert"
-import mongoose from "../../../mongoose"
-
-
-const CHUNK_SIZE = 1024 * 1024
-
-export const get_grid_fs_bucket = (
-  bucket_name: string,
-  chunk_size: number = CHUNK_SIZE,
-) => {
-  assert(chunk_size === CHUNK_SIZE, "chunk_size must match default CHUNK_SIZE")
-
-  const {db} = mongoose.connection
-  const bucket = new mongoose.mongo.GridFSBucket(db, {
-    bucketName: bucket_name,
-    chunkSizeBytes: chunk_size,
-  })
-
-  return bucket
-}

package/src/files/index.js

@@ -1,5 +0,0 @@
-import {upload_chunk} from "./upload_chunk"
-
-module.exports = (app) => {
-  app.post("/rb-api/v1/files/upload_chunk", upload_chunk)
-}

package/src/files/tasks/finalize_file_upload/apply_img_preview.ts

@@ -1,49 +0,0 @@
-import {DOC_MIME_TYPES} from "./constants"
-import {exec} from "./helpers/exec"
-import {convert_pdf_to_png} from "./helpers/convert_pdf_to_png"
-
-
-const convert_pdf = async({tmp_wd, full_path, metadata}) => {
-  console.log("converting from:", metadata.mime_type, "to: pdf")
-  const cmd = `libreoffice --headless --convert-to pdf:writer_pdf_Export --outdir ${tmp_wd} ${full_path}`
-  const result = await exec(cmd)
-
-  if (result.stderr) {
-    console.warn(result.stderr)
-  }
-
-  if (result.stdout) {
-    console.log(result.stdout)
-  }
-}
-
-
-export const apply_img_preview = async({tmp_wd, full_path, metadata}): Promise<any> => {
-
-  let is_pdf_converted = false
-
-  // DOC
-  if (DOC_MIME_TYPES.includes(metadata.mime_type)) {
-    await convert_pdf({tmp_wd, full_path, metadata})
-    is_pdf_converted = true
-  }
-  // PDF
-  else if (metadata.mime_type === "application/pdf") {
-    // file is already a pdf, do nothing
-    is_pdf_converted = true
-  } else {
-    console.log("apply img, unknown mime type, not proceeding")
-  }
-
-  if (is_pdf_converted) {
-    const pdf_path = `${full_path}.pdf`
-    await convert_pdf_to_png({tmp_wd, pdf_path})
-  }
-
-  // TODO: convert spreadsheets to csv ?
-  // convert powerpoints to pdf also (they should work just the same as docs)
-
-  return {
-    is_pdf_converted,
-  }
-}

package/src/files/tasks/finalize_file_upload/constants.ts

@@ -1,23 +0,0 @@
-export const PDF_IMG_DIR = "img"
-
-export const DOC_MIME_TYPES = [
-  "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
-  "application/msword",
-  "application/vnd.oasis.opendocument.text",
-  "application/rtf",
-]
-
-
-export const IMAGES_MIME_TYPES = [
-  "image/jpeg",
-  "image/png",
-  "image/gif",
-  "image/bmp",
-  "image/tiff",
-  "image/webp",
-  "image/svg+xml",
-  "image/vnd.microsoft.icon",
-  "image/heif",
-  "image/heic",
-  "image/vnd.adobe.photoshop",
-]

package/src/files/tasks/finalize_file_upload/download_file.ts

@@ -1,98 +0,0 @@
-import fs from "fs/promises"
-import {createReadStream, createWriteStream} from "fs"
-import path from "path"
-import {createBrotliDecompress} from "zlib"
-import Promise from "bluebird"
-
-import mongoose from "../../../../mongoose"
-import {UPLOAD_BUCKET_NAME} from "../../constants"
-import {get_grid_fs_bucket} from "../../helpers/get_grid_fs_bucket"
-
-
-const DL_CHUNK_CONCURRENCY = 8
-
-const download_chunk =
-  (bucket: mongoose.mongo.GridFSBucket, tmp_wd: string, metadata: any) =>
-    (chunk_num: number) =>
-      new Promise((resolve, reject) => {
-        const filename = `${metadata.hash}.${chunk_num}`
-
-        const dl_stream = bucket.openDownloadStreamByName(filename)
-
-        const file_path = path.join(tmp_wd, `./${filename}`)
-
-        const write_stream = createWriteStream(file_path)
-
-        if (metadata.is_compressed) {
-          const decompress_stream = createBrotliDecompress()
-          dl_stream.pipe(decompress_stream).pipe(write_stream)
-        } else {
-          dl_stream.pipe(write_stream)
-        }
-
-        dl_stream.on("error", (error) => {
-          console.error("Error downloading file:", error)
-          reject(error)
-        })
-
-        write_stream.on("error", (error) => {
-          console.error("Error writing file:", error)
-          reject(error)
-        })
-
-        write_stream.on("finish", () => {
-          console.log("File downloaded and written successfully", file_path)
-          resolve()
-        })
-      })
-
-
-const rebuild_file_from_chunks = async(
-  tmp_wd: string,
-  metadata: any,
-  chunks: Array<number>,
-) => {
-  const full_file_path = path.join(tmp_wd, `./${metadata.hash}`)
-  const output_stream = createWriteStream(full_file_path)
-
-  const append_chunk = (chunk_num: number) => new Promise<void>((resolve, reject) => {
-    const chunk_path = path.join(tmp_wd, `./${metadata.hash}.${chunk_num}`)
-
-    const chunk_read_stream = createReadStream(chunk_path)
-
-    chunk_read_stream.pipe(output_stream, {end: false})
-
-    chunk_read_stream.on("end", async() => {
-      await fs.rm(chunk_path)
-      resolve()
-    })
-    chunk_read_stream.on("error", reject)
-  })
-
-  for (const chunk_num of chunks) {
-    await append_chunk(chunk_num)
-  }
-
-  output_stream.end()
-
-  return full_file_path
-}
-
-// downloads all chunks, reconstruct if necessary
-export const download_file = async({tmp_wd, metadata}:
-{ tmp_wd: string, metadata: any }): Promise<string> => {
-  const bucket = get_grid_fs_bucket(UPLOAD_BUCKET_NAME)
-
-  const chunks = Array.from({length: metadata.total_chunks}, (_, i) => i)
-
-  // download all file chunks
-  await Promise.map(
-    chunks,
-    download_chunk(bucket, tmp_wd, metadata),
-    {concurrency: DL_CHUNK_CONCURRENCY},
-  )
-
-  const full_path = await rebuild_file_from_chunks(tmp_wd, metadata, chunks)
-
-  return full_path
-}

package/src/files/tasks/finalize_file_upload/get_text_vectors.ts

@@ -1,13 +0,0 @@
-
-
-export const get_text_vectors = async(
-  {
-    tmp_wd,
-    metadata,
-  }: {
-    tmp_wd: string,
-    metadata: any
-  }
-) => {
-  console.log("text vectors NYI")
-}

package/src/files/tasks/finalize_file_upload/helpers/convert_pdf_to_png.ts

@@ -1,34 +0,0 @@
-import path from "path"
-import {mkdirp} from "mkdirp"
-import {fromPath} from "pdf2pic"
-
-import {PDF_IMG_DIR} from "../constants"
-
-
-export const convert_pdf_to_png = async(
-  {tmp_wd, pdf_path}:
-  { tmp_wd: string, pdf_path: string }
-) =>
-  new Promise(async(resolve, reject) => {
-    const out_dir = path.join(tmp_wd, `./${PDF_IMG_DIR}/`)
-
-    await mkdirp(out_dir)
-
-    const options = {
-      density: 220,
-      saveFilename: "page",
-      savePath: out_dir,
-      format: "png",
-      // width
-      // height
-      preserveAspectRatio: true,
-    }
-
-    const converter = fromPath(pdf_path, options)
-
-    converter.bulk(-1, {responseType: "image"})
-      .then((pages) => {
-        resolve(pages)
-      })
-
-  })

package/src/files/tasks/finalize_file_upload/helpers/exec.ts

@@ -1,5 +0,0 @@
-import {exec as _exec} from "child_process"
-import {promisify} from "util"
-
-
-export const exec = promisify(_exec)

package/src/files/tasks/finalize_file_upload/helpers/get_metadata.ts

@@ -1,18 +0,0 @@
-import mongoose from "../../../../../mongoose"
-
-import {UPLOAD_BUCKET_NAME} from "../../../constants"
-
-
-export const get_metadata = async(hash: string) => {
-  const files_coll = mongoose.connection.db.collection(
-    `${UPLOAD_BUCKET_NAME}.files`,
-  )
-
-  const first_chunk_name = `${hash}.0`
-
-  const file_metadata = await files_coll.findOne({
-    filename: first_chunk_name,
-  })
-
-  return file_metadata.metadata
-}

package/src/files/tasks/finalize_file_upload/index.ts

@@ -1,53 +0,0 @@
-import path from "path"
-import os from "os"
-import {mkdirp} from "mkdirp"
-
-import {TMP_UPLOADS_DIR} from "../../constants"
-
-import {apply_img_preview} from "./apply_img_preview"
-import {download_file} from "./download_file"
-import {run_ocr} from "./run_ocr"
-import {get_metadata} from "./helpers/get_metadata"
-import {get_text_vectors} from "./get_text_vectors"
-
-import {ProcessFilePayload} from "../../constants"
-
-
-const {RB_TENANT_ID} = process.env
-
-
-export const finalize_file_upload = async(payload: ProcessFilePayload) => {
-  const {hash, img_preview, ocr, text_vectors} = payload
-
-  const tmp_wd = path.join(
-    os.tmpdir(),
-    `./${TMP_UPLOADS_DIR}/${RB_TENANT_ID}/${hash}`,
-  )
-
-  await mkdirp(tmp_wd)
-  console.log("created dir", tmp_wd)
-
-  const metadata = await get_metadata(hash)
-
-  const full_path = await download_file({tmp_wd, metadata})
-
-  if (img_preview) {
-    await apply_img_preview({tmp_wd, full_path, metadata})
-  }
-
-  if (ocr) {
-    if (!img_preview) {
-      throw new Error("img_preview cannot be false when ocr is set to true")
-    }
-
-    await run_ocr({tmp_wd, metadata})
-  }
-
-  if (text_vectors) {
-    if (!ocr) {
-      throw new Error("text_vectors requires ocr to be set to true")
-    }
-
-    await get_text_vectors({tmp_wd, metadata})
-  }
-}