@rpcbase/client 0.45.0 → 0.46.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rpcbase/client",
-  "version": "0.45.0",
+  "version": "0.46.0",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 0"
   },

package/rts/getUseQuery.js

@@ -29,9 +29,11 @@ const getUseQuery = (register_query) => (
   }, [])
 
   const hasFirstReply = useRef(false)
+  const hasNetworkReply = useRef(false)
   const lastDocRef = useRef(null)
   // const [page, setPage] = useState(0)
 
+  const [source, setSource] = useState()
   const [data, setData] = useState()
   const [error, setError] = useState()
   const [loading, setLoading] = useState(true)
@@ -55,6 +57,7 @@ const getUseQuery = (register_query) => (
   useEffect(() => {
     const load = async() => {
       const val = await cacheStorage.get(storageKey)
+      // console.log("got val from storage", val)
       // always initially apply when first load here
       if (val) {
         setData(val)
@@ -65,9 +68,8 @@ const getUseQuery = (register_query) => (
     load()
   }, [storageKey])
 
-  const applyNewData = (newData) => {
+  const applyNewData = (newData, context) => {
     setData(newData)
-
     // write data to storage
     // TODO: is there a better way than doing it on every update ?
     // should this be throttled
@@ -78,6 +80,12 @@ const getUseQuery = (register_query) => (
     }
   }
 
+  const applyContext = (newContext) => {
+    if (newContext.source !== source) {
+      setSource(newContext.source)
+    }
+  }
+
   useEffect(() => {
     const queryKey = key || id
 
@@ -87,8 +95,20 @@ const getUseQuery = (register_query) => (
 
     const start = Date.now()
     const unsubscribe = register_query(model_name, query, {...options, key: queryKey}, (err, queryResult, context) => {
+      log("callback answer with context", context, queryResult?.length)
+
+      // believe it or not, the network can be faster than indexeddb...
+      if (context.source === "cache" && hasNetworkReply.current) {
+        log("skipping cache arriving later than network")
+        return
+      }
 
-      if (options.debug ) {
+      // mark if we received from network
+      if (context.source === "network" && !hasNetworkReply.current) {
+        hasNetworkReply.current = true
+      }
+
+      if (options.debug) {
         console.log("query took", Date.now() - start, model_name, query)
       }
 
@@ -115,19 +135,24 @@ const getUseQuery = (register_query) => (
         hasFirstReply.current = true
 
         // skip if we already have the data
-        if (isEqual(data, newData)) return
+        if (isEqual(data, newData)) {
+          applyContext(context)
+          return
+        }
 
-        applyNewData(newData)
+        applyContext(context)
+        applyNewData(newData, context)
         return
       }
 
       if (context.is_local && options.skip_local && hasFirstReply.current) {
-        // console.log("skipping local update", key)
+        log("skipping local update", key)
         return
       }
 
       if (!isEqual(data, newData)) {
-        applyNewData(newData)
+        applyContext(context)
+        applyNewData(newData, context)
       }
     })
 
@@ -143,11 +168,10 @@ const getUseQuery = (register_query) => (
   }, [])
 
 
-  const result = useMemo(() => ({data, error, loading}), [data, error, loading])
+  const result = useMemo(() => ({data, source, error, loading}), [data, source, error, loading])
 
   return result
 }
 
 
-
 export default getUseQuery

package/rts/index.js

@@ -7,6 +7,7 @@ import debug from "debug"
 
 import {BASE_URL} from "env"
 
+import store from "./store"
 import getUseQuery from "./getUseQuery"
 
 const log = debug("rb:rts:client")
@@ -15,7 +16,7 @@ const MAX_TXN_BUF = 2048
 
 let _socket
 const _callbacks = {}
-const _queries_storage = {}
+const _queries_store = {}
 const _local_txn = []
 
 // TODO: when server disconnects / crashes and loses all server side stored queries
@@ -51,13 +52,13 @@ const dispatch_query_payload = (payload) => {
   if (payload.error) {
     console.warn("rts-client, payload error:", payload.error)
     console.warn("rts-client", "in:", model_name, "query_key:", query_key)
-    callbacks.forEach((cb) => cb(payload.error))
+    callbacks.forEach((cb) => cb(payload.error, undefined, {source: "network"}))
     return
   }
 
   let data
   try {
-    // TODO: zstd decompression here
+    // TODO: zstd / brotli decompression here
     data = JSON.parse(payload.data_buf)
   } catch (err) {
     console.log("Error", err)
@@ -70,11 +71,14 @@ const dispatch_query_payload = (payload) => {
   }
 
   const context = {
+    source: "network",
     is_local: _local_txn.includes(payload.txn_id),
     txn_id: payload.txn_id,
   }
 
   callbacks.forEach((cb) => cb(null, data, context))
+
+  store.update_docs(model_name, data)
 }
 
 
@@ -143,15 +147,17 @@ export const register_query = (model_name, query, _options, _callback) => {
     callback = _options
   }
 
+  log("register_query", {model_name, query, options, callback})
+
 
   if (!_socket) {
-    log("trying to use null socket", {model_name, query})
+    log("register_query: trying to use null socket", {model_name, query})
     return
   }
 
   const key = options.key || ""
-  log("registering query with key", key, model_name, query)
   const query_key = `${key}${JSON.stringify(query)}`
+  log("registering query with key", key, model_name, query, query_key)
 
   // save callback to update hooks
   const cb_key = `${model_name}.${query_key}`
@@ -161,16 +167,20 @@ export const register_query = (model_name, query, _options, _callback) => {
   _callbacks[cb_key].push(callback)
 
   // save query for reconnections and retries
-  if (!_queries_storage[model_name]) {
-    _queries_storage[model_name] = {}
+  if (!_queries_store[model_name]) {
+    _queries_store[model_name] = {}
   }
-  _queries_storage[model_name][query]
+  _queries_store[model_name][query]
 
+  // TODO: why both run and register query here ? the run_query should come straight from register ?
   _socket.emit("run_query", {model_name, query, query_key, options})
   _socket.emit("register_query", {model_name, query, query_key, options})
 
+  // run the query from the cache a first time
+  store.run_query({model_name, query, query_key, options}, callback)
+
   return () => {
-    _socket.emit("remove_query", {model_name, query})
+    _socket.emit("remove_query", {model_name, query, query_key, options})
     // remove callback
     const cb_index = _callbacks[cb_key].indexOf(callback)
     if (cb_index > -1) {

package/rts/store/debug.js

@@ -0,0 +1,24 @@
+/* @flow */
+import debug from "debug"
+import PouchDB from "pouchdb-core"
+
+if (debug.enabled("rb:store")) {
+
+  const log = debug("rb:store:pouch")
+
+  log("store debug is enabled")
+
+  const shouldLogAllPouch = debug.enabled("pouch")
+
+  PouchDB.on("debug", (args) => {
+    if (shouldLogAllPouch) {
+      log(...args)
+    }
+    // log find only when not loggin all pouch
+    else {
+      if (args[0] === "find") {
+        log(...args)
+      }
+    }
+  })
+}

package/rts/store/get_collection.js

@@ -0,0 +1,35 @@
+/* @flow */
+import debug from "debug"
+import PouchDB from "pouchdb-core"
+import IndexedDBAdapter from "pouchdb-adapter-indexeddb"
+import FindPlugin from "pouchdb-find"
+
+import {DATABASE_NAME} from "env"
+
+const log = debug("rb:store")
+
+let prefix = "rb/"
+if (DATABASE_NAME) prefix += `${DATABASE_NAME}/`
+
+PouchDB.prefix = prefix
+
+PouchDB.plugin(IndexedDBAdapter)
+PouchDB.plugin(FindPlugin)
+
+
+const _cols_store = Object.create(null)
+
+
+const get_collection = (col_name) => {
+  if (_cols_store[col_name]) {
+    return _cols_store[col_name]
+  } else {
+    const col = new PouchDB(col_name, { adapter: "indexeddb" });
+    _cols_store[col_name] = col
+
+    return col
+  }
+
+}
+
+export default get_collection

package/rts/store/index.js

@@ -0,0 +1,161 @@
+/* @flow */
+import debug from "debug"
+
+import "./debug"
+
+import get_collection from "./get_collection"
+import update_docs from "./update_docs"
+
+const log = debug("rb:store")
+
+// import updateDocument from "./updateDocument"
+// import {DATABASE_NAME} from "env"
+// let prefix = "rb/"
+// if (DATABASE_NAME) prefix += `${DATABASE_NAME}/`
+//
+// PouchDB.prefix = prefix
+//
+// PouchDB.plugin(IndexedDBAdapter)
+// PouchDB.plugin(FindPlugin)
+//
+//
+// let db = new PouchDB(`db/items`, { adapter: "indexeddb" });
+//
+// // Create a new document
+// let doc = {
+//   _id: "001",
+//   message: "Hello, World!"
+// }
+// //
+//
+// const run = async() => {
+//   // https://github.com/pouchdb/pouchdb/tree/master/packages/node_modules/pouchdb-find#dbcreateindexindex--callback
+//   const res = await db.createIndex({
+//     index: {fields: ["name"]}
+//   })
+//
+//
+//   // Listen for changes on the database
+//   const fn = db.changes({
+//     since: "now",
+//     live: true,
+//     include_docs: true
+//   }).on("change", function(change) {
+//     // handle change
+//     console.log("GOT CHANGE", change)
+//   }).on("error", function (err) {
+//     // handle errors
+//     console.log(err)
+//   })
+//
+//   fn.cancel()
+//
+//   console.log("GOT FN", fn)
+//
+//   console.log("got res", res)
+//
+//   const doc = await db.find({
+//     selector: {name: "Mario"},
+//     sort: ["name"]
+//   })
+//
+//   console.log("GT doc", doc)
+//
+//   const {docs} = await db.find({selector: {}})
+//
+//   console.log("got all docs", docs)
+//
+//   db.put({
+//     _id: "001-" + Date.now(),
+//     message: "Hello, World!"
+//   })
+// }
+//
+//
+// run()
+
+// // Insert the document into the database
+// db.put(doc, function(err, response) {
+//   if (err) {
+//     return console.log(err);
+//   } else {
+//     console.log("Document created successfully");
+//   }
+// })
+//
+//
+// // Insert the document into the database
+// db.put({
+//   _id: "hello world",
+//   fieldVal: 12,
+// }, {force: true}, async function(err, response) {
+//   if (err) {
+//     console.log("errrro", err);
+//     console.log("Stt", JSON.stringify(err))
+//     return
+//   } else {
+//     console.log("Document created successfully", response);
+//   }
+//
+//   console.log("will try to find:")
+//   const doc = await db.find({
+//     selector: {
+//       fieldVal: 10
+//     }
+//   })
+//   console.log("LEDOC", doc)
+//
+// });
+//
+// setInterval(() => {
+//   // Fetch the document
+//   db.get("001", function(err, doc) {
+//     if (err) {
+//       return console.log(err);
+//     } else {
+//       console.log(doc);
+//     }
+//   });
+// }, 2000)
+//
+
+// TODO: should the store be in a worker or the main thread ?
+
+const run_query = async({model_name, query, query_key, options}, callback) => {
+  // console.log("ALAAARM")
+  // console.log("run_query", {model_name, query, query_key, options})
+
+  // console.time("store run_query")
+
+  const col = get_collection(model_name)
+
+
+  // https://github.com/pouchdb/pouchdb/tree/master/packages/node_modules/pouchdb-find#dbcreateindexindex--callback
+  const {docs} = await col.find({
+    // TODO: we should check if the selectors are compatible here
+    selector: query,
+    // DO NOT INCLUDE FIELDS HERE AS IT USES PICK AND WE WOULD GO through the list twice
+    // https://github.com/pouchdb/pouchdb/blob/master/packages/node_modules/pouchdb-find/src/adapters/local/find/index.js
+    // fields: [""]
+  })
+
+
+  const mapped_docs = docs.map(({_rev, ...doc}) => {
+    // TODO: handle projections here
+    const remapped_doc = Object.entries(doc).reduce((new_doc, [key, value]) => {
+      let new_key = key.startsWith('$_') ? key.replace(/^\$_/, "") : key
+      new_doc[new_key] = value
+      return new_doc
+    }, {})
+
+    return remapped_doc
+  })
+
+  // console.log("FOUND INITIAL", mapped_docs)
+  // console.timeEnd("store run_query")
+
+  callback(null, mapped_docs, {source: "cache"})
+}
+
+
+export default {run_query, update_docs}

package/rts/store/update_docs.js

@@ -0,0 +1,44 @@
+/* @flow */
+import get_collection from "./get_collection"
+
+
+const update_docs = async(model_name, data) => {
+  const collection = get_collection(model_name)
+
+  const all_ids = data.map((doc) => doc._id)
+
+  // console.log("will update cache", model_name, data, all_ids)
+
+  // console.time("find")
+
+  // TODO:
+  // there is also a bulk get which could have different performance than find, try both
+  // https://pouchdb.com/api.html#bulk_get
+
+  // https://github.com/pouchdb/pouchdb/tree/master/packages/node_modules/pouchdb-find#dbcreateindexindex--callback
+  const {docs: current_docs} = await collection.find({
+    selector: {_id: {$in: all_ids}},
+    fields: ["_id", "_rev"],
+  })
+
+  // console.log("current_docs", current_docs)
+
+  const revs_map = {}
+  current_docs.forEach((doc) => revs_map[doc._id] = doc._rev)
+
+  const write_docs = data.map((mongo_doc) => {
+    const op = Object.entries(mongo_doc).reduce((new_doc, [key, value]) => {
+      let new_key = key !== "_id" && key.startsWith('_') ? `$_${key}` : key
+      new_doc[new_key] = value
+      return new_doc
+    }, {})
+
+    op._rev = revs_map[mongo_doc._id]
+    return op
+  })
+
+  await collection.bulkDocs(write_docs)
+  // console.timeEnd("find")
+}
+
+export default update_docs