@rpcbase/server 0.380.0 → 0.382.0

package/src/files/tasks/finalize_file_upload/run_ocr.ts

@@ -1,42 +0,0 @@
-import path from "path"
-import Promise from "bluebird"
-import {glob} from "glob11"
-
-import {exec} from "./helpers/exec"
-
-import {PDF_IMG_DIR} from "./constants"
-
-
-const get_input_files = async(tmp_wd: string): Promise<Array<string>> => {
-  const input_glob = path.join(tmp_wd, `./${PDF_IMG_DIR}/*.png`)
-
-  const input_files = await glob(input_glob)
-
-  const sorted = input_files.sort((a, b) => {
-    // Extract the page numbers from the file names
-    const page_a = parseInt(a.match(/page\.(\d+)\.png/)?.[1] || "0", 10)
-    const page_b = parseInt(b.match(/page\.(\d+)\.png/)?.[1] || "0", 10)
-
-    return page_a - page_b
-  })
-
-  return sorted
-}
-
-
-export const run_ocr = async({tmp_wd, metadata}: { tmp_wd: string, metadata: any }) => {
-
-  const input_files = await get_input_files(tmp_wd)
-
-  const run_ocr_file = async(file_path: string) => {
-    const wd = path.dirname(file_path)
-    const basename = path.basename(file_path, ".png")
-
-    const cmd = `tesseract ${basename}.png ${basename} -l eng --oem 1 --psm 11`
-    const out = await exec(cmd, {cwd: wd})
-    console.log("OUTTT", out)
-  }
-
-  await Promise.map(input_files, run_ocr_file, {concurrency: 4})
-
-}

package/src/files/tasks/index.ts

@@ -1,6 +0,0 @@
-import queue from "../../../queue"
-
-import {finalize_file_upload} from "./finalize_file_upload"
-
-
-queue.register_task("finalize_file_upload", finalize_file_upload)

package/src/files/upload_chunk.ts

@@ -1,83 +0,0 @@
-import assert from "assert"
-import fs from "fs"
-import {formidable, File} from "formidable"
-
-import {get_grid_fs_bucket} from "./helpers/get_grid_fs_bucket"
-import {sim_test_inject} from "../helpers/sim_test_inject"
-import {UPLOAD_BUCKET_NAME} from "./constants"
-
-
-const upload_file_to_bucket = async(file: File, metadata): Promise<void> => {
-  const bucket = get_grid_fs_bucket(UPLOAD_BUCKET_NAME, metadata.chunk_size)
-
-  const chunk_filename = `${metadata.hash}.${metadata.chunk_index}`
-
-  const upload_stream = bucket.openUploadStream(chunk_filename, {
-    metadata,
-  })
-
-  const read_stream = fs.createReadStream(file.filepath)
-
-  read_stream.pipe(upload_stream)
-
-  return new Promise((resolve, reject) => {
-    upload_stream.on("finish", () => {
-      // console.log("finished uploading:", upload_stream.id)
-      resolve()
-    })
-    upload_stream.on("error", (error) => {
-      reject(error)
-    })
-  })
-}
-
-
-export const upload_chunk = async(req, res, next) => {
-  const {user_id} = req.session
-  assert(user_id, "upload_chunk: unable to resolve user_id")
-
-  // https://github.com/node-formidable/formidable#options
-  const form = formidable({})
-
-  let fields, files
-  try {
-    [fields, files] = await form.parse(req)
-  } catch (err) {
-    return res.status(500).json({error: "Failed to parse uploaded file"})
-  }
-
-  await sim_test_inject()
-  // await new Promise(resolve => setTimeout(resolve, 3000))
-
-  const file_chunk = files.file_chunk[0] as File
-
-  const metadata = {
-    user_id,
-    original_filename: fields.original_filename[0],
-    is_compressed: fields.is_compressed[0] === "yes",
-    chunk_index: parseInt(fields.chunk_index[0]),
-    total_chunks: parseInt(fields.total_chunks[0]),
-    chunk_size: parseInt(fields.chunk_size[0]),
-    mime_type: fields.mime_type[0],
-    hash: fields.hash[0],
-  }
-
-  await upload_file_to_bucket(file_chunk, metadata)
-
-  const result: {
-    status: string;
-    finalize_token?: string;
-  } = {
-    status: "ok",
-  }
-
-  const is_last_chunk = metadata.chunk_index === metadata.total_chunks - 1
-
-  if (is_last_chunk) {
-    result.finalize_token = metadata.hash
-  }
-
-  res.json({
-    status: "ok",
-  })
-}

package/src/helpers/sim_test_inject.ts

@@ -1,21 +0,0 @@
-const ERROR_PROBABILITY = 0.001
-const THROTTLE_PROBABILITY = 0.2
-const MAX_THROTTLE_MS = 2000
-
-// randomly inject errors or delays
-export const sim_test_inject = async() => {
-  if (!__DEV__) {
-    return
-  }
-
-  const should_error = Math.random() > 1 - ERROR_PROBABILITY
-  if (should_error) {
-    throw new Error("random sim test error")
-  }
-
-  const should_throttle = Math.random() > 1 - THROTTLE_PROBABILITY
-  if (should_throttle) {
-    const throttle_delay = Math.floor(Math.random() * MAX_THROTTLE_MS)
-    await new Promise(resolve => setTimeout(resolve, throttle_delay))
-  }
-}

package/src/models/Invite.js

@@ -1,23 +0,0 @@
-/* @flow */
-const mongoose = require("../../mongoose")
-
-const Invite = mongoose.model("Invite", {
-  email: String,
-  email_hash: String, // TODO: unused field?
-  token: String,
-  expires: Date,
-  is_ready: {
-    type: Boolean,
-    default: false,
-  },
-  is_sent: {
-    type: Boolean,
-    default: false,
-  },
-  is_accepted: {
-    type: Boolean,
-    default: false,
-  },
-})
-
-module.exports = Invite

package/src/models/Notification.js

@@ -1,44 +0,0 @@
-/* @flow */
-const mongoose = require("../../mongoose")
-
-const NOTIFICATIONS_TYPES = {
-  new_message: "new_message",
-}
-
-const NotificationSchema = new mongoose.Schema(
-  {
-    type: {
-      type: String,
-      enum: Object.keys(NOTIFICATIONS_TYPES),
-    },
-    notification: {
-      title: String,
-      body: String,
-      icon: String,
-    },
-    action_url: String,
-    ack_at_ms: {
-      type: Number,
-      default: null,
-    },
-    server_timestamp_ms: {
-      type: Number,
-      required: true,
-    },
-    context: mongoose.Schema.Types.Mixed,
-    push_sent: {
-      type: Array,
-      default: [],
-    },
-  },
-  {
-    strict: false,
-    versionKey: false,
-  },
-)
-
-const model = mongoose.model("Notification", NotificationSchema)
-
-model.NOTIFICATIONS_TYPES = NOTIFICATIONS_TYPES
-
-module.exports = model

package/src/models/Policy.ts

@@ -1,13 +0,0 @@
-const mongoose = require("../../mongoose")
-
-const Policy = mongoose.model("Policy", {
-  user_id: String,
-  token_hash: String,
-  created_at: {
-    type: Date,
-    expires: 360, // 6min
-    default: Date.now,
-  }
-})
-
-module.exports = Policy

package/src/models/ResetPasswordToken.js

@@ -1,14 +0,0 @@
-/* @flow */
-const mongoose = require("../../mongoose")
-
-const ResetPasswordToken = mongoose.model("ResetPasswordToken", {
-  user_id: String,
-  token_hash: String,
-  created_at: {
-    type: Date,
-    expires: 360, // 6min
-    default: Date.now,
-  }
-})
-
-module.exports = ResetPasswordToken

package/src/models/SearchHistory.ts

@@ -1,22 +0,0 @@
-import mongoose, {Schema, Model, Document} from "@rpcbase/server/mongoose"
-
-
-interface ISearchHistory extends Document {
-  server_timestamp_ms: number;
-  context: any;
-  data: any;
-}
-
-const SearchHistorySchema: Schema<ISearchHistory> = new Schema(
-  {
-    server_timestamp_ms: Number,
-    context: mongoose.Schema.Types.Mixed,
-    data: mongoose.Schema.Types.Mixed,
-  },
-  {timestamps: true},
-)
-
-export const SearchHistory: Model<ISearchHistory> = mongoose.model<ISearchHistory>(
-  "SearchHistory",
-  SearchHistorySchema,
-)

package/src/models/User.js

@@ -1,42 +0,0 @@
-/* @flow */
-const mongoose = require("../../mongoose")
-
-const UserSchema = new mongoose.Schema({
-  email: {
-    type: String,
-    index: true,
-  },
-  is_email_verified: {
-    type: Boolean,
-    default: false,
-  },
-  password_hash: String,
-  first_name: {
-    type: String,
-    default: "",
-  },
-  last_name: {
-    type: String,
-    default: "",
-  },
-  initials: {
-    type: String,
-    default: "",
-  },
-  avatar_color: {
-    type: String,
-  },
-  profile_picture: Buffer,
-  profile_picture_updated_at_ms: Number,
-  devices: {
-    type: Array,
-    default: [],
-  }
-}, {
-  strict: false,
-  versionKey: false,
-})
-
-const model = mongoose.model("User", UserSchema)
-
-module.exports = model

package/src/models/UserStoredValues.js

@@ -1,18 +0,0 @@
-/* @flow */
-const mongoose = require("../../mongoose")
-
-const UserStoredValuesSchema = new mongoose.Schema({
-  _owners: {
-    type: Array,
-    required: true,
-    index: true,
-    default: undefined,
-  },
-}, {
-  strict: false,
-  versionKey: false,
-})
-
-const model = mongoose.model("UserStoredValues", UserStoredValuesSchema, "userstoredvalues")
-
-module.exports = model

package/src/models/index.js

@@ -1,7 +0,0 @@
-require("./Invite")
-require("./Notification")
-require("./Policy")
-require("./ResetPasswordToken")
-require("./User")
-require("./UserStoredValues")
-require("./SearchHistory")

package/src/notitications/ack_notification.js

@@ -1,26 +0,0 @@
-/* @flow */
-const assert = require("assert")
-const debug = require("debug")
-
-const Notification = require("../models/Notification")
-
-
-const log = debug("notifications:ack")
-
-const ack_notification = async(payload, ctx) => {
-  const {notification_id} = payload
-  assert(notification_id, "missing notification id")
-
-  const notif = await Notification.findOne({_id: notification_id}, {ack_at_ms: 1}, {ctx})
-  notif.ack_at_ms = payload.ack_at_ms
-
-  await notif.save({ctx})
-
-  log("acknowledged notification:", notification_id)
-
-  return {
-    status: "ok",
-  }
-}
-
-module.exports = ack_notification

package/src/notitications/get_notifications.js

@@ -1,39 +0,0 @@
-/* @flow */
-// const Group = require("../models/Group")
-
-const get_notifications = async(payload, ctx) => {
-  // const groups = await Group.find({
-  //   // _owners: {$in: []}
-  // }, null, {ctx})
-
-  return {
-    status: "ok",
-    notifications: [],
-  }
-}
-
-module.exports = get_notifications
-
-// const get_notifications = async () => {
-//   const uid = getUid()
-//   const db = firebase.firestore()
-//
-//   const query = await db.collection("notifications").where("owner", "==", uid).get()
-//   const notifications = query.docs.map((doc) => {
-//     return doc.data()
-//   })
-//
-//   // const forms = _.uniqBy(
-//   //   _.flatten(
-//   //     result.map((query) => {
-//   //       return query.docs
-//   //     }),
-//   //   ).map((table) => {
-//   //     const data = table.data()
-//   //     data.id = table.id
-//   //     return data
-//   //   }),
-//   //   "id",
-//   // )
-//   return notifications
-// }

package/src/notitications/llt/README.md

@@ -1,8 +0,0 @@
-## Long Lived Tasks (LLTs)
-
-These are large jobs that typically take a while to run (minutes / hours).
-
-The user and sometimes the system needs to be updated on progress of these tasks.
-
-Any task that takes more than 10 seconds should be a long lived task.
-https://www.nngroup.com/articles/response-times-3-important-limits/

package/src/notitications/llt/get_llts.js

@@ -1,42 +0,0 @@
-/* @flow */
-const assert = require("assert")
-const Promise = require("bluebird")
-
-const queue = require("@rpcbase/server/queue")
-
-
-const get_llts = async(payload, ctx) => {
-  const {env_id} = payload
-  assert(env_id, "unable to find env_id")
-
-  const jobs = await queue.get_jobs(["active"])
-  // console.log("WOW GOT JOBS", jobs)
-
-  const target_jobs = jobs.filter((j) => j.id.toString().startsWith(`llt-${env_id}`))
-
-  const tasks = await Promise.map(
-    target_jobs,
-    async(j) => {
-      const logs = await j.queue.getJobLogs(j.id)
-      console.log("Llogs", logs)
-
-      return {
-        id: j.id,
-      }
-    },
-    {concurrency: 16},
-  )
-
-  tasks.length > 0 && console.log("tasks", tasks)
-
-  target_jobs.forEach((j) => {
-    console.log("LLT TARGET PROGRESS", j._progress)
-  })
-
-  return {
-    status: "ok",
-    llts: [],
-  }
-}
-
-module.exports = get_llts

package/src/notitications/set_seen.js

@@ -1,26 +0,0 @@
-/* @flow */
-
-const WorkflowExecution = require("../models/workflows/WorkflowExecution")
-
-
-const set_seen = async(payload, ctx) => {
-  const {notif_type} = payload
-
-  if (notif_type === "workflow") {
-    const workflow_exec = await WorkflowExecution.findOne({_id: payload.exec_id}, {_id: 1}, {ctx})
-    workflow_exec.seen_at = payload.seen_at
-
-    await workflow_exec.save({ctx})
-  } else {
-    throw new Error("set_seen unknown notif_type")
-  }
-
-  // console.log("Set notif seen", payload)
-
-  return {
-    status: "ok",
-  }
-
-}
-
-module.exports = set_seen

package/src/sessions/index.js

@@ -1,27 +0,0 @@
-/* @flow */
-const pc = require("picocolors")
-
-
-const is_development = process.env.NODE_ENV === "development"
-
-module.exports = (app) => {
-  const has_session_store = process.env.RB_SESSION_STORE === "yes"
-
-  if (has_session_store) {
-    console.log("using", pc.bold(pc.green("SESSION STORE")))
-
-    const session_store_middleware = require("./session_store_middleware")
-    app.use(session_store_middleware)
-  } else {
-    console.log("using", pc.bold(pc.red("REVERSE-PROXY SESSIONS!")), "if you want to use the session store, start with the --sessions argument")
-
-    // warn the user
-    if (is_development) {
-      app.use(require("./warning_proxy_middleware"))
-    }
-
-    const session_proxy_middleware = require("./session_proxy_middleware")
-    app.use(session_proxy_middleware)
-  }
-
-}

package/src/sessions/session_proxy_middleware.js

@@ -1,18 +0,0 @@
-/* @flow */
-
-const USER_ID_HEADER = "rb-user-id"
-
-// Production middleware used to retrieve session from the proxy
-const session_proxy_middleware = (req, res, next) => {
-  req.session = {}
-
-  const user_id = req.headers[USER_ID_HEADER]
-  if (user_id) {
-    req.session.user_id = user_id
-  }
-
-  next()
-}
-
-
-module.exports = session_proxy_middleware

package/src/sessions/session_store_middleware.js

@@ -1,106 +0,0 @@
-/* @flow */
-const assert = require("assert")
-const debug = require("debug")
-const session = require("express-session")
-const validator = require("validator")
-const {createClient} = require("redis")
-const RedisStore = require("connect-redis").default
-
-// const {is_docker} = require("@rpcbase/std")
-const is_docker = () => true
-
-
-const {SESSION_STORE_PORT, APP_DOMAIN} = process.env
-assert(SESSION_STORE_PORT, "SESSION_STORE_PORT is undefined")
-
-
-const log = debug("rb:session")
-
-
-// WARNING:
-// https://stackoverflow.com/questions/70867229/error-connection-timeout-when-connecting-to-redis-docker-instance
-// https://github.com/redis/node-redis/issues/1656/
-
-if (typeof SESSION_STORE_PORT === "string" && !validator.isPort(SESSION_STORE_PORT)) {
-  throw new Error("expected SESSION_STORE_PORT to be a valid port number")
-}
-
-const hostname = is_docker() ? "session-store" : "127.0.0.1"
-
-console.log("session-store hostname", hostname, "port", SESSION_STORE_PORT)
-
-let session_middleware = null
-
-// EXTREME WARNING: docker issue
-// when there is synchronous io redis client fails to connect
-setTimeout(async() => {
-  const reconnectStrategy = (retries) => {
-    log("redis_client::reconnectStrategy::retrying with arg", retries)
-    if (retries < 5) {
-      log("retry count:", retries, "retrying in 1s")
-      return 4000
-    } else {
-      return new Error("max retries expiered")
-    }
-  }
-
-  const redis_client = createClient({
-    socket: {
-      host: hostname,
-      port: SESSION_STORE_PORT,
-      reconnectStrategy,
-      connectTimeout: 10000,
-      keepAlive: 0,
-    },
-  })
-
-  redis_client.on("ready", () => {
-    log("session-storage::redis_client connected")
-  })
-
-  const res = await redis_client.connect()
-
-  const session_config = {
-    name: "rb_sid",
-    store: new RedisStore({client: redis_client}),
-    proxy: true,
-    saveUninitialized: true,
-    // WARNING DO NOT USE IN PROD
-    secret: "NOT_A_SECRET_BECAUSE_THIS_MIDDLEWARE_IS_NOT_MEANT_TO_BE_USED_IN_PROD",
-    resave: true,
-    cookie: {
-      // TODO: set to secure in production
-      // secure: true
-      // TODO: test this
-      maxAge: 1000 * 3600 * 24 * 90 // 90 days
-    }
-  }
-
-  // WARNING: apparently doesn't work on localhost or .local domains
-  // https://stackoverflow.com/questions/1134290/cookies-on-localhost-with-explicit-domain
-  // TODO: WTF IS THIS
-  // if (is_docker() && typeof APP_DOMAIN === "string" && APP_DOMAIN.trim() !== "") {
-  //   log("\n\n")
-  //   log("\n\n")
-  //   session_config.cookie.domain = APP_DOMAIN
-  //   // session_config.cookie.secure = true
-  //   session_config.cookie.sameSite = false
-  // }
-
-  session_middleware = session(session_config)
-}, 200)
-
-
-module.exports = (req, res, next) => {
-  if (typeof session_middleware === "function") {
-    session_middleware(req, res, next)
-  } else {
-    // recheck until it's a function and call it
-    const interval = setInterval(() => {
-      if (typeof session_middleware === "function") {
-        clearInterval(interval)
-        session_middleware(req, res, next)
-      }
-    }, 100)
-  }
-}

package/src/sessions/warning_proxy_middleware.js

@@ -1,17 +0,0 @@
-/* @flow */
-
-const {SUPPRESS_PROXY_SESSION_WARNING} = process.env
-
-const suppress_warning = SUPPRESS_PROXY_SESSION_WARNING === "yes"
-
-const warning_proxy_middleware = (req, res, next) => {
-  const {host} = req.headers
-
-  if (!suppress_warning && host.startsWith("localhost:")) {
-    console.warn("are you running with the proxy address ? you shouldn't be using localhost when you are running the app with the reverse proxyy")
-  }
-
-  next()
-}
-
-module.exports = warning_proxy_middleware

package/src/tasks/index.js

@@ -1,8 +0,0 @@
-const queue = require("../../queue")
-
-require("../files/tasks")
-
-const index_item = require("./index_item")
-
-
-queue.register_task("rb_index_item", index_item)

package/src/tasks/index_item.js

@@ -1,8 +0,0 @@
-/* @flow */
-
-const index_item = async(payload) => {
-  console.log("TASK:INDEX_ITEM")
-  console.log("payload", payload)
-}
-
-module.exports = index_item