@rpcbase/server 0.375.0 → 0.377.0

package/boot/server.js

@@ -6,19 +6,22 @@ const rb_server_plugin = require("rb-plugin-server")
 
 const http = require("http")
 
-const {rpc_router, client_router, express} = require("../")
+const {client_router, express} = require("../")
+
+const {init_rts} = require("../rts")
 
-const realtime_state = require("../rts")
 
 const {SERVER_PORT} = process.env
 
 const app = express()
 const server = http.createServer(app)
 
-realtime_state(server)
+init_rts(server)
 
 rb_server_plugin.default(app)
 
+client_router(app)
+
 server.listen(SERVER_PORT, "0.0.0.0", () => {
   console.log(`listening on 0.0.0.0:${SERVER_PORT}`)
 })

package/openai.js

@@ -1,10 +1,10 @@
 const OpenAI = require("openai")
 
+
 const {OPENAI_API_KEY} = process.env
 
 export const openai = new OpenAI({
   apiKey: OPENAI_API_KEY,
 })
 
-export const EMBEDDINGS_MODEL_NAME = "text-embedding-3-small"
-export const EMBEDDINGS_VECTOR_LENGTH = 1536
+export const EMBEDDINGS_MODEL_NAME = "text-embedding-3-large"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rpcbase/server",
-  "version": "0.375.0",
+  "version": "0.377.0",
   "license": "SSPL-1.0",
   "main": "./index.js",
   "scripts": {

package/queue/dispatch_indexer_queue.js

@@ -1,13 +1,22 @@
 /* @flow */
 
-const dispatch_indexer_queue = (queue, model_name, coll_name, op, doc, update_description) => {
-
-  queue.add("index_item", {op, doc, model_name, coll_name, update_description}, {
-    jobId: `index-item-${op}-${doc._id}`,
-    removeOnComplete: true,
-    removeOnFail: true,
-  })
+const dispatch_indexer_queue = (
+  queue,
+  model_name,
+  coll_name,
+  op,
+  doc,
+  update_description,
+) => {
+  queue.add(
+    "index_item",
+    {op, doc, model_name, coll_name, update_description},
+    {
+      jobId: `index-item-${op}-${doc._id}-${Date.now()}`,
+      removeOnComplete: true,
+      removeOnFail: true,
+    },
+  )
 }
 
-
 module.exports = dispatch_indexer_queue

package/rts/index.js

@@ -24,8 +24,8 @@ const QUERY_MAX_LIMIT = 4096
 // TODO: does this even support multiple clients ?
 // TODO: investigate multiple change streams event triggers per client
 // TODO: add tracing
-
 const _sockets = {}
+const _handlers = []
 const _change_streams = {}
 // we index both queries and socket queries
 // to avoid going through all queries when a socket disconnects
@@ -87,7 +87,7 @@ const dispatch_doc_change = (model, doc) => {
       // socket was destroyed
       // TODO: remove socket id from _queries when destroyed
       if (!s) {
-        log("NO SOCKET RETURNING")
+        log("NO SOCKET, RETURNING")
         return
       }
       // console.log("dipatch change", query_key)
@@ -211,7 +211,7 @@ const add_change_stream = (socket_id, {model_name, query, query_key, options}) =
     try {
       model = mongoose.model(model_name)
     } catch (err) {
-      console.error("ERROR registering model:", model_name)
+      console.error("add change stream:ERROR registering model:", model_name)
       console.error(err)
       return
     }
@@ -261,6 +261,13 @@ const add_change_stream = (socket_id, {model_name, query, query_key, options}) =
 
 
 const run_query = async(socket_id, {model_name, query, query_key, options}) => {
+  // Check if the model is registered
+  if (!mongoose.models[model_name]) {
+    // throw new Error(`Model "${model_name}" is not registered.`)
+    console.error(colors.red("ERROR: model not registered"), model_name, "will skip query")
+    return
+  }
+
   const model = mongoose.model(model_name)
 
   log("run_query", {model_name, query, query_key, options})
@@ -278,7 +285,7 @@ const run_query = async(socket_id, {model_name, query, query_key, options}) => {
   }
 
   const query_options = get_query_options(ctx, options)
-
+  // log("QUERY", query)
   const query_promise = model.find(query, options.projection, query_options)
 
   if (query_options.sort) {
@@ -351,7 +358,7 @@ const disconnect_handler = (socket_id, reason) => {
 }
 
 
-const rts_server = async(server) => {
+const init_rts = async(server) => {
   // register the delete plugin
   mongoose.plugin(mongoose_delete_plugin)
   // txn plugin
@@ -395,9 +402,11 @@ const rts_server = async(server) => {
 
   io.on("connection", (socket) => {
     _sockets[socket.id] = socket
-    console.log("socket connected", socket.id)
-    //
-    socket.on("register_query", (payload) => {
+
+    log("socket connected", socket.id)
+
+    // RTS handlers
+    socket.on("registerQuery", (payload) => {
       add_change_stream(socket.id, payload)
     })
 
@@ -412,12 +421,25 @@ const rts_server = async(server) => {
         console.log("run_query caught error", err)
       }
     })
+    // !RTS handlers
+
+    // register_rts_handler: custom server generic handlers
+    const cleanup_fns = _handlers.map((handler_fn) => handler_fn(socket))
 
     socket.on("disconnect", (reason) => {
       disconnect_handler(socket.id, reason)
+      cleanup_fns
+        .filter((fn) => typeof fn === "function")
+        .forEach((fn) => fn())
     })
   })
 }
 
 
-module.exports = rts_server
+const register_rts_handler = (handler_fn) => {
+  console.log("REGister handler", handler_fn)
+  _handlers.push(handler_fn)
+}
+
+
+module.exports = {init_rts, register_rts_handler}

package/search/constants.ts

@@ -0,0 +1 @@
+export const DEFAULT_SEARCH_EMBEDDINGS_DIMENSIONS = 2048

package/search/ensure_index.ts

@@ -6,26 +6,35 @@ const log = debug("rb:search:ensure_index")
 
 export const ensure_index = async({
   index_id,
-  mappings_properties,
+  mappings,
 }: {
-  index_id: string,
-  mappings_properties: any
+  index_id: string;
+  mappings: any;
 }) => {
   const client = get_client()
 
   try {
     const exists = await client.indices.exists({index: index_id})
-    if (exists) return
+
+    if (exists) {
+      await client.indices.putMapping({
+        index: index_id,
+        body: mappings,
+      })
+      log("updated index mappings", index_id)
+      return
+    }
   } catch (err) {
-    console.log("Error checking for index existance", JSON.stringify(err, null, 2))
+    console.log(
+      "Error checking for index existance",
+      JSON.stringify(err, null, 2),
+    )
   }
 
   const res = await client.indices.create({
     index: index_id,
     body: {
-      mappings: {
-        properties: mappings_properties,
-      },
+      mappings,
     },
   })
 
@@ -34,4 +43,11 @@ export const ensure_index = async({
 
   log("created index", index_id)
 }
+
 console.log("ensure_index.ts")
+
+// {
+//   dynamic: true,
+//   // date_detection: false,
+//   properties: mappings_properties,
+// }

package/search/get_client.ts

@@ -1,4 +1,4 @@
-const {Client} = require("@elastic/elasticsearch")
+import {Client} from "@elastic/elasticsearch"
 
 // const is_docker = require("@rpcbase/std/is_docker")
 const is_docker = () => true

package/search/index.ts

@@ -1,2 +1,3 @@
 export * from "./get_client"
 export * from "./ensure_index"
+export * from "./constants"

package/src/access-control/apply_policies.js

@@ -11,7 +11,6 @@ const perm_operations = ["create", "read", "update", "delete"]
 
 // TODO: add LRU cache for policies
 
-
 // Owner
 // the creator of the document, or an user_id that is in the "owners" field
 

package/src/models/SearchHistory.ts

@@ -0,0 +1,22 @@
+import mongoose, {Schema, Model, Document} from "@rpcbase/server/mongoose"
+
+
+interface ISearchHistory extends Document {
+  server_timestamp_ms: number;
+  context: any;
+  data: any;
+}
+
+const SearchHistorySchema: Schema<ISearchHistory> = new Schema(
+  {
+    server_timestamp_ms: Number,
+    context: mongoose.Schema.Types.Mixed,
+    data: mongoose.Schema.Types.Mixed,
+  },
+  {timestamps: true},
+)
+
+export const SearchHistory: Model<ISearchHistory> = mongoose.model<ISearchHistory>(
+  "SearchHistory",
+  SearchHistorySchema,
+)

package/src/models/index.js

@@ -4,3 +4,4 @@ require("./Policy")
 require("./ResetPasswordToken")
 require("./User")
 require("./UserStoredValues")
+require("./SearchHistory")