@rpcbase/server 0.363.0 → 0.364.0-mongoosetsindex.1

package/constants/keys.ts

@@ -0,0 +1 @@
+export const SEARCH_INDEX_PREFIX = "rb-indexer"

package/express/index.js

@@ -25,7 +25,7 @@ const has_sentry = !!SENTRY_DSN
 
 // TODO: document this
 // we could have a larger size, but it's good practice to keep it small to prevent body upload ddos attacks
-const BODY_MAX_SIZE_MB = 80
+const BODY_MAX_SIZE_MB = 16
 
 
 log("server is production:", JSON.stringify(is_production))

package/get_object_id.ts

@@ -9,15 +9,26 @@ const {RB_TENANT_ID} = process.env
 // TODO: WARNING: DANGER: this code is duplicated in the client
 // TODO: move it to iso
 
-// Validation: is defined
-assert(RB_TENANT_ID, "expected RB_TENANT_ID to be defined")
-// is hexadecimal
-assert(isHexadecimal(RB_TENANT_ID), "expected RB_TENANT_ID to be a hexadecimal")
-// is 4 bytes
-assert(RB_TENANT_ID.length === 8, "RB_TENANT_ID must be exactly 4 bytes (8 hex chars)")
+let _is_valid_tenant_id = false
+
+const validate_tenant_id = () => {
+  // Validation: is defined
+  assert(RB_TENANT_ID, "expected RB_TENANT_ID to be defined")
+  // is hexadecimal
+  assert(isHexadecimal(RB_TENANT_ID), "expected RB_TENANT_ID to be a hexadecimal")
+  // is 4 bytes
+  assert(RB_TENANT_ID.length === 8, "RB_TENANT_ID must be exactly 4 bytes (8 hex chars)")
+
+  _is_valid_tenant_id = true
+}
+
 
 // generates a 12 bytes mongodb object id using the org id prefix or custom customer id
 export const get_object_id = (): mongoose.Types.ObjectId => {
+  if (!_is_valid_tenant_id) {
+    validate_tenant_id()
+  }
+
   const ts_bytes = Math.floor(Date.now() / 1000).toString(16)
   expect(ts_bytes.length).toBe(8)
 

package/mongoose/index.ts

@@ -1,9 +1,13 @@
 import mongoose from "mongoose"
-import {object_id_plugin} from "./object_id_plugin"
+import {object_id_plugin} from "./plugins/object_id_plugin"
+import {disable_default_version_key_plugin} from "./plugins/disable_default_version_key_plugin"
+import {disable_default_timestamps_plugin} from "./plugins/disable_default_timestamps_plugin"
 
 mongoose.set("strictQuery", false)
 
 mongoose.plugin(object_id_plugin)
+mongoose.plugin(disable_default_version_key_plugin)
+mongoose.plugin(disable_default_timestamps_plugin)
 
 export default mongoose
 

package/mongoose/plugins/disable_default_timestamps_plugin.ts

@@ -0,0 +1,5 @@
+import { Schema } from "../"
+
+export const disable_default_timestamps_plugin = (schema: Schema) => {
+  schema.options.timestamps = false
+}

package/mongoose/plugins/disable_default_version_key_plugin.ts

@@ -0,0 +1,5 @@
+import { Schema } from "../"
+
+export const disable_default_version_key_plugin = (schema: Schema) => {
+  schema.options.versionKey = false
+}

package/mongoose/plugins/object_id_plugin.ts

@@ -0,0 +1,29 @@
+import mongoose, { Schema } from "../"
+
+import { get_object_id } from "../../get_object_id"
+
+export const object_id_plugin = (schema: Schema) => {
+  if (schema.options._id === false) {
+    return
+  }
+  // Disable Mongoose's automatic _id field creation
+  if (!schema.options._id) {
+    schema.options._id = false
+  }
+
+  // Add your custom _id field
+  schema.add({
+    _id: {
+      type: mongoose.Schema.Types.ObjectId,
+      default: () => get_object_id(),
+    },
+  })
+
+  // Optional: Ensure the _id field is always set
+  schema.pre("save", function (next) {
+    if (!this._id) {
+      this._id = get_object_id()
+    }
+    next()
+  })
+}

package/openai.js

@@ -0,0 +1,10 @@
+const OpenAI = require("openai")
+
+const {OPENAI_API_KEY} = process.env
+
+export const openai = new OpenAI({
+  apiKey: OPENAI_API_KEY,
+})
+
+export const EMBEDDINGS_MODEL_NAME = "text-embedding-3-small"
+export const EMBEDDINGS_VECTOR_LENGTH = 1536

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rpcbase/server",
-  "version": "0.363.0",
+  "version": "0.364.0-mongoosetsindex.1",
   "license": "SSPL-1.0",
   "main": "./index.js",
   "scripts": {
@@ -60,6 +60,7 @@
     }
   },
   "dependencies": {
+    "@elastic/elasticsearch": "8.15.0",
     "@sentry/node": "8.24.0",
     "bluebird": "3.7.2",
     "body-parser": "1.20.2",
@@ -75,6 +76,7 @@
     "lodash": "4.17.21",
     "mkdirp": "3.0.1",
     "mongoose": "8.5.2",
+    "openai": "4.56.0",
     "picocolors": "1.0.1",
     "postmark": "4.0.4",
     "redis": "4.7.0",

package/queue/dispatch_indexer_queue.js

@@ -0,0 +1,13 @@
+/* @flow */
+
+const dispatch_indexer_queue = (queue, model_name, coll_name, op, doc, update_description) => {
+
+  queue.add("index_item", {op, doc, model_name, coll_name, update_description}, {
+    jobId: `index-item-${op}-${doc._id}`,
+    removeOnComplete: true,
+    removeOnFail: true,
+  })
+}
+
+
+module.exports = dispatch_indexer_queue

package/queue/{dispatch_queue.js → dispatch_worker_queue.js}

@@ -1,12 +1,14 @@
 /* @flow */
+
 const op_names = {
   // TODO: should we deprecate create, to keep consistency with mongodb?
   insert: "create",
+  insert: "instert",
   delete: "delete",
   update: "update",
 }
 
-const dispatch_queue = (queue, model_name, op, doc, update_description) => {
+const dispatch_worker_queue = (queue, model_name, op, doc, update_description) => {
   const instance = queue.instance()
 
   const tasks = queue.get_tasks()
@@ -16,6 +18,9 @@ const dispatch_queue = (queue, model_name, op, doc, update_description) => {
   // skip if there's no matching handler
   if (!Object.keys(tasks).includes(handler_name)) return
 
+  if (op_names[op] === "create") {
+    console.warn("create tasks are deprecated, rename to insert instead")
+  }
 
   if (update_description?.updatedFields) {
     update_description.updated_fields = Object.keys(update_description.updatedFields )
@@ -30,4 +35,4 @@ const dispatch_queue = (queue, model_name, op, doc, update_description) => {
 }
 
 
-module.exports = dispatch_queue
+module.exports = dispatch_worker_queue

package/queue/register_queue_listener.js

@@ -6,7 +6,8 @@ const _set = require("lodash/set")
 const mongoose = require("../mongoose")
 const queue = require("./index")
 
-const dispatch_queue = require("./dispatch_queue")
+const dispatch_worker_queue = require("./dispatch_worker_queue")
+const dispatch_indexer_queue = require("./dispatch_indexer_queue")
 
 const log = debug("rb:queue:listener")
 
@@ -32,7 +33,7 @@ const mongoose_delete_plugin = (schema) => {
   // https://mongoosejs.com/docs/queries.html
   schema.post("findOneAndDelete", function(doc) {
     log("queue:findOneAndDelete", "dispatch_doc_change NYI")
-    log("DEL PLUGIN", doc)
+    log("del PLUGIN", doc)
     const coll_name = change.ns.coll
 
     const model_name = Object.keys(mongoose.models).find((k) => {
@@ -45,7 +46,8 @@ const mongoose_delete_plugin = (schema) => {
       return
     }
 
-    dispatch_queue(queue, model_name, "delete", doc)
+    dispatch_worker_queue(queue, model_name, "delete", doc)
+    dispatch_indexer_queue(queue, model_name, "delete", doc)
   })
 }
 
@@ -77,13 +79,16 @@ const dispatch_change_handler = (change) => {
     return mongoose.models[k].collection.collectionName === coll_name
   })
 
+  console.log("O3333K", Object.keys(require("../mongoose").models), Object.keys(mongoose.models))
+
   if (!model_name) {
     return
   }
 
   const op = change.operationType
 
-  dispatch_queue(queue, model_name, op, change.fullDocument, change.updateDescription)
+  dispatch_worker_queue(queue, model_name, op, change.fullDocument, change.updateDescription)
+  dispatch_indexer_queue(queue, model_name, coll_name, op, change.fullDocument, change.updateDescription)
 }
 
 const register_db_emitter = () => {

package/search/ensure_index.ts

@@ -0,0 +1,31 @@
+import debug from "debug"
+import { get_client } from "./get_client"
+
+const log = debug("rb:search:ensure_index")
+
+export const ensure_index = async({
+  index_id,
+  mappings_properties,
+}: {
+  index_id: string,
+  mappings_properties: any
+}) => {
+  const client = get_client()
+
+  const exists = await client.indices.exists({index: index_id})
+  if (exists) return
+
+  const res = await client.indices.create({
+    index: index_id,
+    body: {
+      mappings: {
+        properties: mappings_properties,
+      },
+    },
+  })
+
+  expect(res.acknowledged).toBe(true)
+  expect(res.index).toBe(index_id)
+
+  log("created index", index_id)
+}

package/search/get_client.ts

@@ -0,0 +1,15 @@
+const {Client} = require("@elastic/elasticsearch")
+
+// const is_docker = require("@rpcbase/std/is_docker")
+const is_docker = () => true
+
+
+const {SEARCH_INDEX_PORT} = process.env
+// search indexer
+const search_hostname = is_docker() ? "elasticsearch" : "127.0.0.1"
+const indexer_cluster_url = `http://${search_hostname}:${SEARCH_INDEX_PORT}`
+
+export const get_client = () => {
+  const es_client = new Client({node: indexer_cluster_url})
+  return es_client
+}

package/search/index.ts

@@ -0,0 +1,2 @@
+export * from "./get_client"
+export * from "./ensure_index"

package/src/access-control/mongoose_plugin.js

@@ -52,8 +52,8 @@ const get_query_middleware = (op) => (ac_config, schema) => function(next, save_
 
 // https://mongoosejs.com/docs/middleware.html#types-of-middleware
 const mongoose_plugin = (ac_config) => function rb_acl_plugin(schema, options) {
+  // TODO: should strict be true here??
   schema.options.strict = false
-
   // TODO:
   // DANGER: strictQuery to true silently DROPS filter params
   // which can be a critical security risk

package/src/tasks/index.js

@@ -0,0 +1,6 @@
+const queue = require("../../queue")
+
+
+const index_item = require("./index_item")
+
+queue.register_task("rb_index_item", index_item)

package/src/tasks/index_item.js

@@ -0,0 +1,8 @@
+/* @flow */
+
+const index_item = async(payload) => {
+  console.log("TASK:INDEX_ITEM")
+  console.log("payload", payload)
+}
+
+module.exports = index_item

package/mongoose/object_id_plugin.ts

@@ -1,26 +0,0 @@
-import mongoose, { Schema } from "./"
-
-import {get_object_id} from "../get_object_id"
-
-export const object_id_plugin = (schema: Schema) => {
-  // Disable Mongoose's automatic _id field creation
-   if (!schema.options._id) {
-     schema.options._id = false
-   }
-
-   // Add your custom _id field
-   schema.add({
-     _id: {
-       type: mongoose.Schema.Types.ObjectId,
-       default: () => get_object_id(),
-     },
-   })
-
-   // Optional: Ensure the _id field is always set
-   schema.pre("save", function(next) {
-     if (!this._id) {
-       this._id = get_object_id()
-     }
-     next()
-   })
-}