@rpcbase/server 0.380.0 → 0.382.0

package/firebase.js

@@ -1,33 +0,0 @@
-/* @flow */
-const fs = require("fs")
-const path = require("path")
-const {promisify} = require("util")
-
-const admin = require("firebase-admin")
-
-const exists = promisify(fs.exists)
-const readFile = promisify(fs.readFile)
-
-const NAME = "./firebase-service-account.json"
-
-let firebase_app
-
-const setup = async() => {
-
-  const fb_service_account_path = path.join(process.cwd(), NAME)
-
-  if (await exists(fb_service_account_path)) {
-    const service_account = JSON.parse(await readFile(fb_service_account_path))
-
-    firebase_app = admin.initializeApp({
-      credential: admin.credential.cert(service_account),
-    })
-  } else {
-    console.log(NAME, "not found, skipping firebase initialization")
-  }
-
-}
-
-module.exports = {setup}
-
-module.exports.firebase_app = firebase_app

package/get_object_id.ts

@@ -1,39 +0,0 @@
-import assert from "assert";
-import crypto from "crypto";
-import mongoose from "./mongoose"
-
-import isHexadecimal from "validator/lib/isHexadecimal"
-
-const {RB_TENANT_ID} = process.env
-
-// TODO: WARNING: DANGER: this code is duplicated in the client
-// TODO: move it to iso
-
-let _is_valid_tenant_id = false
-
-const validate_tenant_id = () => {
-  // Validation: is defined
-  assert(RB_TENANT_ID, "expected RB_TENANT_ID to be defined")
-  // is hexadecimal
-  assert(isHexadecimal(RB_TENANT_ID), "expected RB_TENANT_ID to be a hexadecimal")
-  // is 4 bytes
-  assert(RB_TENANT_ID.length === 8, "RB_TENANT_ID must be exactly 4 bytes (8 hex chars)")
-
-  _is_valid_tenant_id = true
-}
-
-
-// generates a 12 bytes mongodb object id using the org id prefix or custom customer id
-export const get_object_id = (): mongoose.Types.ObjectId => {
-  if (!_is_valid_tenant_id) {
-    validate_tenant_id()
-  }
-
-  const ts_bytes = Math.floor(Date.now() / 1000).toString(16)
-  expect(ts_bytes.length).toBe(8)
-
-  const random_bytes = crypto.randomBytes(4).toString("hex")
-
-  const obj_id = `${ts_bytes}${RB_TENANT_ID}${random_bytes}`
-  return new mongoose.Types.ObjectId(obj_id)
-}

package/index.js

@@ -1,17 +0,0 @@
-/* @flow */
-const database = require("./database")
-const queue = require("./queue")
-const express = require("./express")
-const firebase = require("./firebase")
-const client_router = require("./src/client/client_router")
-const sign_up = require("./src/auth/sign_up")
-
-
-module.exports = {
-  queue,
-  database,
-  express,
-  firebase,
-  client_router,
-  sign_up,
-}

package/mailer/index.js

@@ -1,31 +0,0 @@
-/* @flow */
-const debug = require("debug")
-const postmark = require("postmark")
-
-const log = debug("rb:mailer")
-
-const {POSTMARK_API_KEY, NODE_ENV, RB_FORCE_SEND_EMAILS} = process.env
-
-const is_production = NODE_ENV === "production"
-const force_send = RB_FORCE_SEND_EMAILS === "yes"
-
-
-// TMP use proper mock smtp server in development
-
-log("mailer, is_production:", {is_production})
-
-let client
-
-if (force_send || is_production && typeof POSTMARK_API_KEY === "string" && POSTMARK_API_KEY.trim() !== "") {
-  client = new postmark.ServerClient(POSTMARK_API_KEY)
-} else {
-  client = {
-    sendEmail: async(payload) => {
-      log("sendEmail disabled when not in production")
-      log("From:", payload.From, "To:", payload.To, "Subject:", payload.Subject)
-      return {Message: "OK"}
-    }
-  }
-}
-
-module.exports = client

package/mongoose/index.ts

@@ -1,16 +0,0 @@
-// eslint-disable-next-line @rpcbase/lint/no-rb-restricted-imports
-import mongoose from "mongoose"
-import {object_id_plugin} from "./plugins/object_id_plugin"
-import {disable_default_version_key_plugin} from "./plugins/disable_default_version_key_plugin"
-import {disable_default_timestamps_plugin} from "./plugins/disable_default_timestamps_plugin"
-
-
-mongoose.set("strictQuery", false)
-
-mongoose.plugin(object_id_plugin)
-mongoose.plugin(disable_default_version_key_plugin)
-mongoose.plugin(disable_default_timestamps_plugin)
-
-export default mongoose
-
-export * from "mongoose"

package/mongoose/plugins/disable_default_timestamps_plugin.ts

@@ -1,5 +0,0 @@
-import { Schema } from "../"
-
-export const disable_default_timestamps_plugin = (schema: Schema) => {
-  schema.options.timestamps = false
-}

package/mongoose/plugins/disable_default_version_key_plugin.ts

@@ -1,5 +0,0 @@
-import { Schema } from "../"
-
-export const disable_default_version_key_plugin = (schema: Schema) => {
-  schema.options.versionKey = false
-}

package/mongoose/plugins/object_id_plugin.ts

@@ -1,31 +0,0 @@
-import mongoose, {Schema} from "../"
-
-import {get_object_id} from "../../get_object_id"
-
-
-export const object_id_plugin = (schema: Schema) => {
-  if (schema.options._id === false) {
-    return
-  }
-  // Disable Mongoose's automatic _id field creation
-  if (!schema.options._id) {
-    schema.options._id = false
-  }
-
-  // Add your custom _id field
-  schema.add({
-    _id: {
-      type: mongoose.Schema.Types.ObjectId,
-      default: () => get_object_id(),
-      immutable: true,
-    },
-  })
-
-  // Optional: Ensure the _id field is always set
-  schema.pre("save", function(next) {
-    if (!this._id) {
-      this._id = get_object_id()
-    }
-    next()
-  })
-}

package/openai.js

@@ -1,10 +0,0 @@
-const OpenAI = require("openai")
-
-
-const {OPENAI_API_KEY} = process.env
-
-export const openai = new OpenAI({
-  apiKey: OPENAI_API_KEY,
-})
-
-export const EMBEDDINGS_MODEL_NAME = "text-embedding-3-large"

package/queue/dispatch_indexer_queue.js

@@ -1,22 +0,0 @@
-/* @flow */
-
-const dispatch_indexer_queue = (
-  queue,
-  model_name,
-  coll_name,
-  op,
-  doc,
-  update_description,
-) => {
-  queue.add(
-    "index_item",
-    {op, doc, model_name, coll_name, update_description},
-    {
-      jobId: `rb-index-item-${op}-${doc._id}-${Date.now()}`,
-      removeOnComplete: true,
-      removeOnFail: true,
-    },
-  )
-}
-
-module.exports = dispatch_indexer_queue

package/queue/dispatch_worker_queue.js

@@ -1,38 +0,0 @@
-/* @flow */
-
-const op_names = {
-  // TODO: should we deprecate create, to keep consistency with mongodb?
-  insert: "create",
-  insert: "instert",
-  delete: "delete",
-  update: "update",
-}
-
-const dispatch_worker_queue = (queue, model_name, op, doc, update_description) => {
-  const instance = queue.instance()
-
-  const tasks = queue.get_tasks()
-
-  const handler_name = `on_${op_names[op]}_${model_name.toLowerCase()}`
-  // console.log("dispatch", handler_name)
-  // skip if there's no matching handler
-  if (!Object.keys(tasks).includes(handler_name)) return
-
-  if (op_names[op] === "create") {
-    console.warn("create tasks are deprecated, rename to insert instead")
-  }
-
-  if (update_description?.updatedFields) {
-    update_description.updated_fields = Object.keys(update_description.updatedFields )
-    delete update_description.updatedFields
-  }
-
-  queue.add(handler_name, {doc, update_description}, {
-    jobId: `${op}-${doc._id}`,
-    removeOnComplete: true,
-    removeOnFail: true,
-  })
-}
-
-
-module.exports = dispatch_worker_queue

package/queue/index.js

@@ -1,110 +0,0 @@
-/* @flow */
-const Queue = require("bull")
-
-// const {is_docker} = require("@rpcbase/std")
-const is_docker = () => true
-
-const {WORKER_QUEUE_PORT} = process.env
-
-const hostname = is_docker() ? "worker-queue" : "127.0.0.1"
-
-const worker_queue_url = `redis://${hostname}:${WORKER_QUEUE_PORT}`
-
-// TODO: implement latency queues
-const QUEUE_DEFAULT_NAME = "rb-queue-default"
-// const QUEUE_10S_NAME
-
-const tasks_list = Object.create(null)
-
-let worker_queue
-
-const get_instance = () => {
-  if (!worker_queue) worker_queue = new Queue(QUEUE_DEFAULT_NAME, worker_queue_url)
-  return worker_queue
-}
-
-
-const worker_add = async(task_name, payload, options) => {
-  if (!worker_queue) {
-    worker_queue = new Queue(QUEUE_DEFAULT_NAME, worker_queue_url)
-  }
-  const res = await worker_queue.add({task_name, payload}, options)
-  // console.log("worker_add:created task:", task_name, res.id)
-  return res
-}
-
-const get_job = async(job_id) => {
-  const queue = get_instance()
-  return queue.getJob(job_id)
-}
-
-const get_jobs = async(...args) => {
-  const queue = get_instance()
-  return queue.getJobs(...args)
-}
-
-const register_task = (name, fn) => {
-  // append module to tasks list
-  tasks_list[name] = fn
-}
-
-const get_tasks = () => tasks_list
-
-const start_worker = async() => {
-  console.log("start worker queue", worker_queue_url)
-
-  worker_queue = new Queue(QUEUE_DEFAULT_NAME, worker_queue_url)
-
-  // error handler
-  worker_queue.on("error", (err) => {
-    console.log(`A queue error happened: ${err.message}`)
-  })
-
-  // on stall
-  worker_queue.on("stalled", (job) => {
-    console.log(`Job ${job.id} stalled and will be reprocessed`)
-  })
-
-  worker_queue.on("job failed", (id, err) => {
-    console.log(`Job ${id} failed with error ${err.message}`)
-  })
-
-  // processor function
-  // default concurrency is 2, in practice we will implement this with multiple workers
-  // to take advantage of multi cpu machines
-  worker_queue.process(2, async(job) => {
-    try {
-      const {task_name, payload} = job.data
-
-      if (!job.id.startsWith("rb-index-item-")) {
-        console.log(`starting job ${job.id} ${task_name}`)
-      }
-
-      const task_fn = tasks_list[task_name]
-
-      const res = await task_fn(payload, job)
-      return res
-    } catch (err) {
-      console.log("ERRRR", err)
-    }
-  })
-
-  return worker_queue
-}
-
-
-// TMP used to create additional queues, should be implemented with a better API in the future
-const get_url = () => {
-  return worker_queue_url
-}
-
-module.exports = {
-  start: start_worker,
-  register_task,
-  get_tasks,
-  get_job,
-  get_jobs,
-  add: worker_add,
-  instance: get_instance,
-  get_url,
-}

package/queue/register_queue_listener.js

@@ -1,180 +0,0 @@
-/* @flow */
-const debug = require("debug")
-const _get = require("lodash/get")
-const _set = require("lodash/set")
-
-const mongoose = require("../mongoose")
-const queue = require("./index")
-
-const dispatch_worker_queue = require("./dispatch_worker_queue")
-const dispatch_indexer_queue = require("./dispatch_indexer_queue")
-
-
-const log = debug("rb:queue:listener")
-
-const {RB_APP_NAME, RB_TENANT_ID} = process.env
-
-const RETRY_MAXIMUM_DELAY = 3000
-const RETRY_MINIMUM_DELAY = 50
-const MAX_RETRIES = 20
-
-// TODO: reset retry counters when connection becomes healthy again?
-let retry_counter = 0
-let emitter = null
-
-
-// TODO: use proper mongodb delete event here now
-// mongoose middleware used for delete events
-const mongoose_delete_plugin = (schema) => {
-  schema.pre("deleteOne", function(next) {
-    throw new Error("rts:deleteOne is deprecated, use findOneAndDelete")
-  })
-
-  // TODO: add other delete operations
-  // TODO: this should be deleteOne ?
-  // https://mongoosejs.com/docs/queries.html
-  schema.post("findOneAndDelete", function(doc) {
-    log("queue:findOneAndDelete", "dispatch_doc_change NYI")
-    log("del PLUGIN", doc)
-    const change = null // TMP
-    const coll_name = change.ns.coll
-
-    const model_name = Object.keys(mongoose.models).find((k) => {
-      return mongoose.models[k].collection.collectionName === coll_name
-    })
-
-    // assert(model_name, `unable to resolve model name for coll_name:${coll_name}`)
-    // there are dynamic models so we can't assert model_name is there
-    if (!model_name) {
-      return
-    }
-
-    dispatch_worker_queue(queue, model_name, "delete", doc)
-    dispatch_indexer_queue(queue, model_name, "delete", doc)
-  })
-}
-
-
-const assert_doc_id = (change) => {
-  const doc_id = change.documentKey?._id?.toString()
-
-  // some changes aren't on documents
-  if (!doc_id) return
-
-  const sub = doc_id.substring(8, 16)
-  try {
-    expect(sub).toBe(RB_TENANT_ID)
-  } catch (err) {
-    console.log("in document:", change.ns, change.documentKey)
-    console.log("_id must be instanciated with get_object_id()")
-    console.log(err.message)
-  }
-}
-
-const dispatch_change_handler = (change) => {
-  // skip if this is a file upload
-  if (change.ns?.coll?.endsWith(".files") || change.ns?.coll?.endsWith(".chunks")) {
-    return
-  }
-
-  // verify we have correct object ids
-  assert_doc_id(change)
-
-  const coll_name = change.ns.coll
-
-  const model_name = Object.keys(mongoose.models).find((k) => {
-    return mongoose.models[k].collection.collectionName === coll_name
-  })
-
-  if (!model_name) {
-    return
-  }
-
-  const op = change.operationType
-
-  dispatch_worker_queue(queue, model_name, op, change.fullDocument, change.updateDescription)
-  dispatch_indexer_queue(queue, model_name, coll_name, op, change.fullDocument, change.updateDescription)
-}
-
-const register_db_emitter = () => {
-  // TODO: unclear why it's necessary to have this step here too even though we already removeListeners in the on close event
-  if (emitter) {
-    emitter.removeAllListeners()
-    emitter.close()
-  }
-
-  log("registering db emitter")
-
-  // TODO: implement delete operation fullDocument retrieve,
-  // when this is released https://jira.mongodb.org/browse/SERVER-36941
-  // this is done via a plugin right now
-  // https://mongodb.github.io/node-mongodb-native/4.9/interfaces/ChangeStreamOptions.html
-
-  // Set up the change stream with a filter to only listen to the specific database
-  const pipeline = [
-    { $match: { "ns.db": RB_APP_NAME } },
-    { $match: {
-      "ns.coll": { $nin: ["file-uploads.files", "file-uploads.chunks"] },
-      "operationType": { $in: ["insert", "update" /* "delete"*/] } },
-    },
-  ]
-
-  // https://www.mongodb.com/docs/manual/reference/method/Mongo.watch/
-  emitter = mongoose.connection.watch(pipeline, {fullDocument: "updateLookup"})
-
-  emitter.on("change", dispatch_change_handler)
-
-  emitter.on("error", (err) => {
-    console.log("server:queue:register_queue_listener: change listener emitter got error", err)
-  })
-
-  emitter.on("close", (arg, arg2) => {
-    emitter.removeAllListeners()
-    emitter.close()
-
-    retry_counter++
-
-    if (retry_counter > MAX_RETRIES) {
-      console.log("queue listener reached max retries, exiting with failure")
-      process.exit(1)
-      return
-    }
-
-    const timeout_amount = Math.min(RETRY_MAXIMUM_DELAY, RETRY_MINIMUM_DELAY + 10 * Math.pow(2, retry_counter))
-    console.log("queue_listener: emitter closed, retrying in", timeout_amount)
-
-    setTimeout(() => {
-      register_db_emitter()
-    }, timeout_amount)
-  })
-
-  emitter.on("end", (arg) => {
-    console.log("EMITTER ENDED", arg)
-  })
-
-  return emitter
-}
-
-
-const register_queue_listener = () => new Promise((resolve, reject) => {
-  // register the mongoose delete plugin
-  log("registering mongoose_delete_plugin")
-  mongoose.plugin(mongoose_delete_plugin)
-
-  mongoose.connection.once("open", () => {
-    const db_emitter = register_db_emitter()
-    resolve(db_emitter)
-  })
-
-  mongoose.connection.on("error", (err) => {
-    console.log("register_queue_listener:", "mongoose connection error", err)
-    // reject(err)
-  })
-
-  mongoose.connection.on("disconnected", (err) => {
-    console.log("mongodb:", "mongodb disconnected", err)
-  })
-})
-
-
-module.exports = register_queue_listener

package/redis.js

@@ -1,2 +0,0 @@
-/* @flow */
-module.exports = require("redis")