@rpcbase/server 0.380.0 → 0.381.0

package/rts/index.js

@@ -1,444 +0,0 @@
-/* @flow */
-const Promise = require("bluebird")
-const _remove = require("lodash/remove")
-const _get = require("lodash/get")
-const _set = require("lodash/set")
-const {Server} = require("socket.io")
-const sift = require("sift")
-const debug = require("debug")
-const colors = require("picocolors")
-
-const mongoose = require("../mongoose")
-
-
-const log = debug("rb:rts:server")
-
-const is_production = process.env.NODE_ENV === "production"
-
-const QUERY_MAX_LIMIT = 4096
-
-// TODO: there are edge cases in dev where the client reconnects to the server
-// but the server already has the query results and sends it multiple times
-// the server should ensure it is only registering queries once
-
-// TODO: does this even support multiple clients ?
-// TODO: investigate multiple change streams event triggers per client
-// TODO: add tracing
-const _sockets = {}
-const _handlers = []
-const _change_streams = {}
-// we index both queries and socket queries
-// to avoid going through all queries when a socket disconnects
-const _queries = {}
-const _socket_queries = {}
-
-
-const get_query_options = (ctx, options) => {
-  // https://mongoosejs.com/docs/api/query.html#Query.prototype.setOptions()
-  const query_options = {
-    ctx,
-    is_client: true,
-    //
-  }
-
-  if (options.sort) {
-    query_options.sort = options.sort
-  }
-
-  const limit = typeof options.limit === "number" ? Math.min(QUERY_MAX_LIMIT, Math.abs(options.limit)) : QUERY_MAX_LIMIT
-  query_options.limit = limit
-
-  return query_options
-}
-
-
-// helper to send and update / refresh event to all subscribed queries
-const dispatch_doc_change = (model, doc) => {
-  const model_name = model.modelName
-  const queries = _queries[model_name] || {}
-
-  // WARNING: INVESTIGATE: there are edge cases where you might want to delete an object
-  // that was matched, but now it's not anymore because it's been udpated or removed
-  // rts should still update the query and send a payload to the client
-  // we could use the new mongodb fullDocumentBeforeChange param in watch,
-  // but it doesn't seem to be guaranteed to be there
-
-  // filter queries that match the document
-  const target_queries = Object.keys(queries).filter((query_key) => {
-    const {query} = queries[query_key]
-    // console.log("query", query)
-    const test_fn = sift(query)
-    return test_fn(doc)
-  })
-
-  // we don't need to log every missing txn id, it is expected
-  // if (!doc.__txn_id) {
-  //   console.log(colors.yellow("Warning!"), "change on model:", model.modelName, "matching queries", target_queries, "did not include a valid", colors.yellow("__txn"), "field")
-  // }
-
-  // TODO: we should not re-run each query here!!!!
-  // dispatch change to target query clients
-  target_queries.forEach(async(query_key) => {
-    const {query, options, sockets} = queries[query_key]
-
-    // TODO: remove this
-    await Promise.map(sockets, async(socket_id) => {
-      const s = _sockets[socket_id]
-      // socket was destroyed
-      // TODO: remove socket id from _queries when destroyed
-      if (!s) {
-        log("NO SOCKET, RETURNING")
-        return
-      }
-      // console.log("dipatch change", query_key)
-      const ctx = {req: s.request}
-
-      if (!ctx.req.session.user_id) {
-        console.error("dispatch_doc_change::error", "no user id for client query")
-        return
-      }
-
-      const query_options = get_query_options(ctx, options)
-      // TODO: do not redo query here, use document from change event
-      const query_promise = model.find(query, options.projection, query_options)
-      if (query_options.sort) {
-        query_promise.sort(query_options.sort)
-      }
-
-      query_promise.limit(query_options.limit)
-
-      // WARNING: DANGER
-      // TODO: we should not be doing a read here, use doc from event
-      const data = await query_promise
-
-      const data_buf = JSON.stringify(data)
-
-      log("emit payload")
-
-      s.emit("query_payload", {
-        model_name,
-        query_key,
-        data_buf,
-        txn_id: doc.__txn_id,
-      })
-    }, {concurrency: 1024}) // TODO: tweak concurrency it should be infinite
-
-  })
-}
-
-
-// mongoose middleware used to broadcast delete events to appropriate sockets
-const mongoose_delete_plugin = (schema) => {
-  schema.pre("deleteOne", function(next) {
-    throw new Error("rts::deleteOne is not supported, use findOneAndDelete")
-  })
-
-  // TODO: add other delete operations
-  // https://mongoosejs.com/docs/queries.html
-  schema.post("findOneAndDelete", function(doc) {
-    dispatch_doc_change(this.model, doc)
-  })
-}
-
-
-// Mongoose transaction plugin to save txn on each doc
-// WARNING: query middlewares are not supported yet
-// https://mongoosejs.com/docs/middleware.html#types-of-middleware
-const mongoose_txn_plugin = (schema, options) => {
-  // console.log("register TXN plugin on schema", schema)
-  // WARNING: possible middlewares here:
-  // https://mongoosejs.com/docs/middleware.html
-  // this function isn't async but could be?
-  const apply_txn = /* async */function(next, save_options) {
-    // TODO: keep an eye on this, this seems to happen for nested schemas that aren't models
-    if (!save_options) {
-      next()
-      return
-    }
-
-    const {ctx} = save_options
-    if (!ctx) {
-      log(colors.yellow("Warning!"), "{ctx} not attached to save")
-    }
-
-    const txn_id = _get(ctx, ["req", "headers", "rts-txn-id"])
-
-    if (!txn_id) {
-      log(colors.yellow("Warning!"), "mongoose save request is missing txn_id this will cause issues with rts-client")
-    } else {
-      this.set("__txn_id", txn_id)
-    }
-
-    next()
-  }
-
-
-  schema.pre("save", apply_txn)
-}
-
-
-// responds to a change stream "change" event
-const get_dispatch_change_handler = (model_name) => async(change) => {
-  // const queries = _queries[model_name]
-
-  const model = mongoose.model(model_name)
-
-  let doc
-  if (["insert", "update"].includes(change.operationType)) {
-    // console.log("rts onChange", change)
-    doc = change.fullDocument
-  } else if (change.operationType === "delete") {
-    console.log("op is delete")
-  } else if (!["delete", "drop", "invalidate"].includes(change.operationType)) {
-    console.log("SKIPPING: unknown operation type", change.operationType, change)
-    return
-  }
-
-  if (!doc) {
-    return
-  }
-
-  dispatch_doc_change(model, doc)
-}
-
-
-const add_change_stream = (socket_id, {model_name, query, query_key, options}) => {
-  if (!model_name) throw new Error("empty model name")
-
-  if (!_change_streams[model_name]) {
-    log("rts: initializing change stream for model:", model_name)
-    let model
-    try {
-      model = mongoose.model(model_name)
-    } catch (err) {
-      console.error("add change stream:ERROR registering model:", model_name)
-      console.error(err)
-      return
-    }
-
-    // TODO: fix this, this has been released
-    // TODO: implement delete operation fullDocument retrieve,
-    // when this is released https://jira.mongodb.org/browse/SERVER-36941
-    const emitter = model.watch({
-      fullDocument: "updateLookup",
-      fullDocumentBeforeChange: "whenAvailable",
-    })
-
-    emitter.on("change", get_dispatch_change_handler(model_name))
-
-    emitter.on("error", (err) => {
-      console.log("change listener emitter got error", err)
-    })
-
-    emitter.on("close", () => {
-      delete _change_streams[model_name]
-    })
-
-    _change_streams[model_name] = emitter
-  }
-
-  // save query
-  if (!_queries[model_name]) {
-    _queries[model_name] = {}
-  }
-
-  if (!_queries[model_name][query_key]) {
-    _queries[model_name][query_key] = {query, options, sockets: []}
-  }
-  if (!_queries[model_name][query_key].sockets.includes(socket_id)) {
-    _queries[model_name][query_key].sockets.push(socket_id)
-  }
-
-  // save socket_query
-  if (!_socket_queries[socket_id]) {
-    _socket_queries[socket_id] = {}
-  }
-  if (!_socket_queries[socket_id][model_name]) {
-    _socket_queries[socket_id][model_name] = {}
-  }
-  _socket_queries[socket_id][model_name][query_key] = true
-}
-
-
-const run_query = async(socket_id, {model_name, query, query_key, options}) => {
-  // Check if the model is registered
-  if (!mongoose.models[model_name]) {
-    // throw new Error(`Model "${model_name}" is not registered.`)
-    console.error(colors.red("ERROR: model not registered"), model_name, "will skip query")
-    return
-  }
-
-  const model = mongoose.model(model_name)
-
-  log("run_query", {model_name, query, query_key, options})
-
-  if (!query) throw new Error("run_query empty query")
-
-  const s = _sockets[socket_id]
-  const ctx = {req: s.request}
-
-  // log("initial run_query AUTH", ctx.req.session.user_id)
-  if (!ctx.req.session.user_id) {
-    console.warn("no user ID for client query")
-    s.emit("query_payload", {model_name, query_key, error: "error no user id"})
-    return
-  }
-
-  const query_options = get_query_options(ctx, options)
-  // log("QUERY", query)
-  const query_promise = model.find(query, options.projection, query_options)
-
-  if (query_options.sort) {
-    query_promise.sort(query_options.sort)
-  }
-
-  if (typeof query_options.limit === "number" && query_options.limit >= 0) {
-    query_promise.limit(query_options.limit)
-  }
-
-  const data = await query_promise
-
-  const data_buf = JSON.stringify(data)
-
-  s.emit("query_payload", {model_name, query_key, data_buf})
-}
-
-
-const remove_query = (socket_id, {model_name, query, query_key}) => {
-  const sockets = _queries[model_name]?.[query_key]?.sockets || []
-
-  let new_sockets
-  if (sockets.includes(socket_id)) {
-    new_sockets = _remove(sockets, (s) => s === socket_id)
-  } else new_sockets = sockets
-
-  _set(_queries, `${model_name}.${query_key}.sockets`, new_sockets)
-}
-
-const disconnect_handler = (socket_id, reason) => {
-  if (reason !== "transport close") {
-    console.log("client disconnected, reason:", reason, "socket:", socket_id)
-  }
-
-  const local_queries = _socket_queries[socket_id] || {}
-
-  // remove all queries matching that socket id
-  Object.keys(local_queries)
-    .forEach((model_name) => {
-
-      Object.keys(local_queries[model_name]).forEach((query_key) => {
-        const active_sockets = _queries[model_name]?.[query_key]?.sockets || []
-
-        const socket_index = active_sockets.indexOf(socket_id)
-
-        // remove the socket id from the array
-        if (socket_index > -1) {
-          // _queries[model_name][query_key].sockets.splice(socket_index, 1)
-          active_sockets.splice(socket_index, 1)
-        }
-
-        // remove query if there are no more listeners
-        if (active_sockets.length === 0) {
-          delete _queries[model_name]?.[query_key]
-        }
-
-        // TODO: why the || {} here ?
-        // remove change stream if there are no more queries on this collection
-        if (Object.keys(_queries[model_name] || {}).length === 0) {
-          delete _queries[model_name]
-          _change_streams[model_name]?.close()
-        }
-
-      })
-    })
-
-  // remove socket
-  delete _sockets[socket_id]
-  delete _socket_queries[socket_id]
-}
-
-
-const init_rts = async(server) => {
-  // register the delete plugin
-  mongoose.plugin(mongoose_delete_plugin)
-  // txn plugin
-  mongoose.plugin(mongoose_txn_plugin)
-
-  //
-  const io = new Server(server, {
-    transports: is_production ? ["websocket", "polling"] : ["websocket"],
-    allowUpgrades: true,
-    cors: {
-      // TODO: dynamic client ports
-      // TODO: check if is production
-      origin: `http://localhost:${process.env.CLIENT_PORT}`,
-      methods: ["GET", "POST"],
-      credentials: true,
-    },
-  })
-
-  //
-  const has_session_store = process.env.RB_SESSION_STORE === "yes"
-
-  if (has_session_store) {
-    const session_store_middleware = require("../src/sessions/session_store_middleware")
-    // socket io session middleware
-    io.use((socket, next) => {
-      session_store_middleware(socket.request, {}, next)
-    })
-  } else {
-    const session_proxy_middleware = require("../src/sessions/session_proxy_middleware")
-    // proxy middleware that checks the rb-tenant-id header
-    io.use((socket, next) => {
-      session_proxy_middleware(socket.request, {}, next)
-    })
-  }
-
-  // io.on('connection', (socket) => {
-  //   const session = socket.request.session;
-  //   session.connections++;
-  //   session.save();
-  // });
-
-  io.on("connection", (socket) => {
-    _sockets[socket.id] = socket
-
-    log("socket connected", socket.id)
-
-    // RTS handlers
-    socket.on("registerQuery", (payload) => {
-      add_change_stream(socket.id, payload)
-    })
-
-    socket.on("remove_query", (payload) => {
-      remove_query(socket.id, payload)
-    })
-
-    socket.on("run_query", (payload) => {
-      try {
-        run_query(socket.id, payload)
-      } catch (err) {
-        console.log("run_query caught error", err)
-      }
-    })
-    // !RTS handlers
-
-    // register_rts_handler: custom server generic handlers
-    const cleanup_fns = _handlers.map((handler_fn) => handler_fn(socket))
-
-    socket.on("disconnect", (reason) => {
-      disconnect_handler(socket.id, reason)
-      cleanup_fns
-        .filter((fn) => typeof fn === "function")
-        .forEach((fn) => fn())
-    })
-  })
-}
-
-
-const register_rts_handler = (handler_fn) => {
-  _handlers.push(handler_fn)
-}
-
-
-module.exports = {init_rts, register_rts_handler}

package/search/constants.ts

@@ -1 +0,0 @@
-export const DEFAULT_SEARCH_EMBEDDINGS_DIMENSIONS = 2048

package/search/ensure_index.ts

@@ -1,53 +0,0 @@
-import debug from "debug"
-import {get_client} from "./get_client"
-
-
-const log = debug("rb:search:ensure_index")
-
-export const ensure_index = async({
-  index_id,
-  mappings,
-}: {
-  index_id: string;
-  mappings: any;
-}) => {
-  const client = get_client()
-
-  try {
-    const exists = await client.indices.exists({index: index_id})
-
-    if (exists) {
-      await client.indices.putMapping({
-        index: index_id,
-        body: mappings,
-      })
-      log("updated index mappings", index_id)
-      return
-    }
-  } catch (err) {
-    console.log(
-      "Error checking for index existance",
-      JSON.stringify(err, null, 2),
-    )
-  }
-
-  const res = await client.indices.create({
-    index: index_id,
-    body: {
-      mappings,
-    },
-  })
-
-  expect(res.acknowledged).toBe(true)
-  expect(res.index).toBe(index_id)
-
-  log("created index", index_id)
-}
-
-console.log("ensure_index.ts")
-
-// {
-//   dynamic: true,
-//   // date_detection: false,
-//   properties: mappings_properties,
-// }

package/search/get_client.ts

@@ -1,15 +0,0 @@
-import {Client} from "@elastic/elasticsearch"
-
-// const is_docker = require("@rpcbase/std/is_docker")
-const is_docker = () => true
-
-
-const {SEARCH_INDEX_PORT} = process.env
-// search indexer
-const search_hostname = is_docker() ? "elasticsearch" : "127.0.0.1"
-const indexer_cluster_url = `http://${search_hostname}:${SEARCH_INDEX_PORT}`
-
-export const get_client = () => {
-  const es_client = new Client({node: indexer_cluster_url})
-  return es_client
-}

package/search/index.ts

@@ -1,3 +0,0 @@
-export * from "./get_client"
-export * from "./ensure_index"
-export * from "./constants"

package/src/access-control/apply_policies.js

@@ -1,104 +0,0 @@
-/* @flow */
-const _get = require("lodash/get")
-const colors = require("picocolors")
-
-const get_policies = require("./get_policies")
-
-
-const user_types = ["owner", "user", "any"]
-const perm_operations = ["create", "read", "update", "delete"]
-
-
-// TODO: add LRU cache for policies
-
-// Owner
-// the creator of the document, or an user_id that is in the "owners" field
-
-// User
-// any authenticated user that belongs in this tenant
-
-// Any
-// most relaxed permission, any client can access the resource
-
-
-// given ac config, operation and document / query,
-// check if the user has the permission to run this op
-// if the user doesn't have permission, the system should throw an error
-const apply_policies = async({collection_name, model_name, operation, fields, user_id, doc}) => {
-  // TODO: fix rm policy here
-  // const rule_target = _get(ac_config, `${model_name}.${operation}`)
-  const rule_target = ""
-  // console.log("rule target", rule_target)
-
-  const policies = await get_policies({collection_name, model_name, operation, fields, user_id, doc})
-  console.log("MODEL APPLY POLICIES", {collection_name, model_name, operation, fields, user_id, doc})
-  return
-  // console.log("apply_policies", `'${model_name}:${operation}:${rule_target}'`)
-
-  if (!rule_target) {
-    throw new Error(`undefined rule_target for '${model_name}:${operation}'`)
-  }
-
-  if (!user_types.includes(rule_target)) {
-    throw new Error(`expected rule_target '${model_name}:${operation}' to be in '${JSON.stringify(user_types)}', but got '${rule_target}'`)
-  }
-
-  // Create
-  if (operation === "create") {
-    if (["owner", "user"].includes(rule_target)) {
-      if (!user_id) return new Error("create expected authenticated user_id")
-    } else {
-      return false
-    }
-  }
-  // Read
-  else if (operation === "read") {
-    if (rule_target === "owner") {
-      if (!user_id) throw new Error("read::owner invalid user_id")
-
-      const conditions = doc.getQuery()
-
-      // TMP: warn if user supplied _owners, which is currently overwritten for ACL
-      if (conditions._owners) {
-        if (JSON.stringify(conditions._owners) !== `{"$in":["${user_id}"]}`) {
-          console.log(colors.yellow("Warning!"), model_name, "ACL for read is owner, but client is sending _owners in query, the field will be overwritten", "\nreceived conditions._owners:", JSON.stringify(conditions._owners))
-        }
-      }
-
-      doc.setQuery({
-        ...conditions,
-        _owners: {$in: [user_id]}
-      })
-
-    }
-  }
-  // Update
-  else if (operation === "update") {
-    if (rule_target === "owner") {
-      if (!doc._owners?.includes(user_id)) {
-        console.log("MODEL:", model_name)
-        const err = new Error("expected user_id to be in owners to update document")
-        console.log("failed to update doc:", doc)
-        return err
-      }
-    } else {
-      throw new Error(`unknown rule target '${rule_target}' for operation ${model_name}::update`)
-    }
-  }
-  // Delete
-  else if (operation === "delete") {
-    if (rule_target === "owner") {
-      if (!doc._owners.includes(user_id)) {
-        // TODO: add debug logging
-        return new Error("expected user_id to be in owners to delete document")
-      }
-    } else {
-      throw new Error(`unknown rule target '${rule_target}' for operation ${model_name}::delete`)
-    }
-  }
-
-  return false
-}
-
-
-module.exports = apply_policies

package/src/access-control/get_added_fields.js

@@ -1,23 +0,0 @@
-/* @flow */
-
-const get_added_fields = () => {
-
-  return {
-    _owners: {
-      type: Array,
-      of: String,
-      index: true,
-      required: true,
-    },
-    _viewers: [String],
-    _editors: [String],
-    _created_by: String,
-    _created_at: {
-      type: Date
-    }
-  }
-
-}
-
-
-module.exports = get_added_fields

package/src/access-control/get_policies.js

@@ -1,29 +0,0 @@
-const Policy = require("../models/Policy")
-
-const DEFAULT_POLICY = {
-  "create": "user",
-  "read": "owner",
-  "update": "owner",
-  "delete": "owner",
-}
-
-const get_policies = async({collection_name, model_name, operation, user_id, doc}) => {
-
-  const policies = await Policy.find({
-    $or: [
-      { collection_name, operations: { $in: [operation] }},
-      { doc_id: doc._id, operations: { $in: [operation] }},
-      // TODO: add field level operations
-    ]
-  })
-
-  if (!policies) {
-    return [DEFAULT_POLICY]
-  }
-
-  console.log("POLICIES", policies)
-
-  return policies
-}
-
-module.exports = get_policies

package/src/access-control/hooks/doc_pre_create.js

@@ -1,26 +0,0 @@
-/* @flow */
-//
-// const has_permission = require("../has_permission")
-//
-// const delay = (time) => new Promise((resolve) => setTimeout(resolve, time))
-// // await delay(2000)
-//
-// module.exports = (schema) => async function(next) {
-//   const model_name = this.model.modelName
-//   const operation = "delete"
-//
-//   if (this.op !== "findOneAndDelete") {
-//     throw new Error(`in pre_delete unknown operation: ${this.op}`)
-//   }
-//
-//   const filter = this.getFilter()
-//   const doc = await this.model.findOne(filter)
-//
-//   console.log("DELETE GOT DOC", doc)
-//   // console.log("THIS", this)
-//   // console.log("THIS filter", this.getFilter())
-//   // console.log("SCHEMA DEL", schema)
-//   // TODO: find a way to check if user can delete this document without reading it
-//   // console.log("delete got ac config", ac_config)
-//   next()
-// }