@rpcbase/client 0.44.0 → 0.46.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rpcbase/client",
-  "version": "0.44.0",
+  "version": "0.46.0",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 0"
   },
@@ -10,6 +10,10 @@
     "i18next-chained-backend": "4.4.0",
     "i18next-resources-to-backend": "1.1.4",
     "lodash": "4.17.21",
-    "react-i18next": "13.0.1"
+    "pouchdb-adapter-indexeddb": "8.0.1",
+    "pouchdb-core": "8.0.1",
+    "pouchdb-find": "8.0.1",
+    "react-i18next": "13.0.1",
+    "socket.io-client": "4.7.1"
   }
 }

package/rpc_post.js

@@ -5,7 +5,7 @@ import _get from "lodash/get"
 import _set from "lodash/set"
 
 import get_txn_id from "@rpcbase/std/get_txn_id"
-import {add_local_txn} from "@rpcbase/realtime-state-client"
+import {add_local_txn} from "@rpcbase/client/rts"
 
 import {BASE_URL} from "env"
 

package/rts/boot.js

@@ -0,0 +1,5 @@
+/* @flow */
+
+console.log("RTS BOOT, TODO: connect")
+
+import "./store"

package/rts/cacheStorage/index.js

@@ -0,0 +1,12 @@
+/* @flow */
+import {Platform} from "react-native"
+
+import getWebStorage from "./web"
+import getNativeStorage from "./native"
+
+
+// TODO: add support for storage expiration
+
+const storage = Platform.OS === "web" ? getWebStorage() : getNativeStorage()
+
+export default storage

package/rts/cacheStorage/native.js

@@ -0,0 +1,28 @@
+/* @flow */
+import AsyncStorage from "@react-native-async-storage/async-storage"
+
+const getNativeStorage = () => {
+  const cacheStorage = {
+    get: async(key) => {
+      try {
+        const res = await AsyncStorage.getItem(key)
+        const val = JSON.parse(res)
+        if (val) return val
+      } catch (error) {
+        // There was an error on the native side
+      }
+    },
+    set: async(key, value) => {
+      try {
+        await AsyncStorage.setItem(key, JSON.stringify(value))
+      } catch (error) {
+        // There was an error on the native side
+      }
+    }
+  }
+
+  return cacheStorage
+}
+
+
+export default getNativeStorage

package/rts/cacheStorage/web.js

@@ -0,0 +1,85 @@
+/* @flow */
+const storeName = "store"
+const version = 1
+
+const getWebStorage = (
+  dbName = "rb-query-cache"
+) => {
+  let __db
+
+  const openRequest = indexedDB.open(dbName, version)
+
+  // Handle the creation or upgrade of the database
+  openRequest.onupgradeneeded = (event) => {
+    const db = event.target.result
+    if (!db.objectStoreNames.contains(storeName)) {
+      db.createObjectStore(storeName, { keyPath: "key" })
+    }
+  }
+
+  // Handle errors when opening the database
+  openRequest.onerror = (event) => {
+    console.error("Error opening database:", event.target.errorCode)
+  }
+
+  openRequest.onsuccess = (event) => {
+    const db = event.target.result
+    __db = db
+  }
+
+
+  const cacheStorage = {
+    get: (key) => new Promise((resolve, reject) => {
+      const transaction = __db.transaction([storeName], "readonly")
+      const objectStore = transaction.objectStore(storeName)
+      const request = objectStore.get(key)
+
+      request.onsuccess = (event) => {
+        const obj = event.target.result?.obj
+        resolve(obj)
+      }
+
+      request.onerror = (event) => {
+        console.error("Error reading data:", event.target)
+        reject()
+      }
+
+    }),
+    set: (key, obj) => new Promise((resolve, reject) => {
+
+      const transaction = __db.transaction([storeName], "readwrite")
+      const objectStore = transaction.objectStore(storeName)
+      const request = objectStore.put({obj, key})
+
+      request.onsuccess = (event) => {
+        // console.log("Data written successfully:", event.target.result)
+        resolve()
+      }
+
+      request.onerror = (event) => {
+        // console.error("Error writing data:", event.target.errorCode)
+        console.log("ERRR", event.target)
+        reject()
+      }
+    }),
+    delete: (key) => new Promise((resolve, reject) => {
+      const transaction = __db.transaction([storeName], "readwrite")
+      const objectStore = transaction.objectStore(storeName)
+      const request = objectStore.delete(key)
+
+      request.onsuccess = (event) => {
+        resolve()
+      }
+
+      request.onerror = (event) => {
+        console.error("Error deleting data:", event.target.errorCode)
+        reject()
+      }
+    }),
+  }
+
+  return cacheStorage
+}
+
+
+export default getWebStorage

package/rts/getUseQuery.js

@@ -0,0 +1,177 @@
+/* @flow */
+import assert from "assert"
+import {Platform} from "react-native"
+import {useCallback, useEffect, useState, useMemo, useId, useRef} from "react"
+import debug from "debug"
+import isEqual from "fast-deep-equal/react"
+import _omit from "lodash/omit"
+
+import get_uid from "@rpcbase/client/auth/get_uid"
+
+import cacheStorage from "./cacheStorage"
+
+
+const log = debug("rb:rts:client")
+
+const getUseQuery = (register_query) => (
+  model_name,
+  query = {},
+  options = {},
+) => {
+  const id = useId()
+
+  // TODO: retrieve this from AuthContext in client
+  const uid = useMemo(() => {
+    const _uid = Platform.OS === "web" ? get_uid() : options.userId
+    assert(_uid, "missing uid")
+
+    return _uid
+  }, [])
+
+  const hasFirstReply = useRef(false)
+  const hasNetworkReply = useRef(false)
+  const lastDocRef = useRef(null)
+  // const [page, setPage] = useState(0)
+
+  const [source, setSource] = useState()
+  const [data, setData] = useState()
+  const [error, setError] = useState()
+  const [loading, setLoading] = useState(true)
+
+  const {
+    key = "",
+    projection = {},
+    sort = {},
+  } = options
+
+
+  const storageKey = `${uid}.${key}.${model_name}.${JSON.stringify(query)}.${JSON.stringify(projection)}.${JSON.stringify(sort)}`
+
+
+  useEffect(() => {
+    if (options.debug) {
+      console.log("use query", model_name, query, options)
+    }
+  }, [])
+
+  useEffect(() => {
+    const load = async() => {
+      const val = await cacheStorage.get(storageKey)
+      // console.log("got val from storage", val)
+      // always initially apply when first load here
+      if (val) {
+        setData(val)
+        setLoading(false)
+      }
+    }
+
+    load()
+  }, [storageKey])
+
+  const applyNewData = (newData, context) => {
+    setData(newData)
+    // write data to storage
+    // TODO: is there a better way than doing it on every update ?
+    // should this be throttled
+    cacheStorage.set(storageKey, newData)
+
+    if (newData?.length > 0) {
+      lastDocRef.current = newData[newData.length - 1]
+    }
+  }
+
+  const applyContext = (newContext) => {
+    if (newContext.source !== source) {
+      setSource(newContext.source)
+    }
+  }
+
+  useEffect(() => {
+    const queryKey = key || id
+
+    if (options.debug ) {
+      console.log("register query", model_name, query, options)
+    }
+
+    const start = Date.now()
+    const unsubscribe = register_query(model_name, query, {...options, key: queryKey}, (err, queryResult, context) => {
+      log("callback answer with context", context, queryResult?.length)
+
+      // believe it or not, the network can be faster than indexeddb...
+      if (context.source === "cache" && hasNetworkReply.current) {
+        log("skipping cache arriving later than network")
+        return
+      }
+
+      // mark if we received from network
+      if (context.source === "network" && !hasNetworkReply.current) {
+        hasNetworkReply.current = true
+      }
+
+      if (options.debug) {
+        console.log("query took", Date.now() - start, model_name, query)
+      }
+
+      setLoading(false)
+      if (err) {
+        setError(err)
+        return
+      }
+
+      log("query callback", model_name, queryKey, JSON.stringify(query))
+
+      // return if no data (this should be handled already)
+      if (!queryResult) return
+
+      let newData
+      if (Array.isArray(queryResult)) {
+        newData = queryResult.map((o) => _omit(o, "__txn_id"))
+      } else {
+        newData = _omit(queryResult, "__txn_id")
+      }
+
+      // We return once in any case!
+      if (!hasFirstReply.current) {
+        hasFirstReply.current = true
+
+        // skip if we already have the data
+        if (isEqual(data, newData)) {
+          applyContext(context)
+          return
+        }
+
+        applyContext(context)
+        applyNewData(newData, context)
+        return
+      }
+
+      if (context.is_local && options.skip_local && hasFirstReply.current) {
+        log("skipping local update", key)
+        return
+      }
+
+      if (!isEqual(data, newData)) {
+        applyContext(context)
+        applyNewData(newData, context)
+      }
+    })
+
+    return () => {
+      log && log("useQuery cleanup unsubscribe()")
+      unsubscribe()
+    }
+  }, [JSON.stringify(query), key, storageKey])
+
+
+  const loadNextPage = useCallback(() => {
+    console.log("will load next page after DOC")
+  }, [])
+
+
+  const result = useMemo(() => ({data, source, error, loading}), [data, source, error, loading])
+
+  return result
+}
+
+
+export default getUseQuery

package/rts/index.js

@@ -0,0 +1,194 @@
+/* @flow */
+import assert from "assert"
+import {io} from "socket.io-client"
+import _get from "lodash/get"
+import _set from "lodash/set"
+import debug from "debug"
+
+import {BASE_URL} from "env"
+
+import store from "./store"
+import getUseQuery from "./getUseQuery"
+
+const log = debug("rb:rts:client")
+
+const MAX_TXN_BUF = 2048
+
+let _socket
+const _callbacks = {}
+const _queries_store = {}
+const _local_txn = []
+
+// TODO: when server disconnects / crashes and loses all server side stored queries
+// the clients must reconnect and re-register those queries, or the page will need to be hard refreshed
+
+
+// add_local_txn
+// when a request is made to the server, we generate (or send if provided) the txn_id
+// if the array becomes longer than the default buffer length we shift the array
+export const add_local_txn = (txn_id) => {
+  assert(txn_id, "add_local_txn trying to add an invalid txn_id")
+  _local_txn.push(txn_id)
+  if (_local_txn.length > MAX_TXN_BUF) {
+    _local_txn.shift()
+  }
+}
+
+
+// TODO: add compression / decompression
+const dispatch_query_payload = (payload) => {
+  log("dispatch_query_payload", payload)
+
+  const {model_name, query_key} = payload
+  // const cb = _get(_callbacks, `${model_name}.${query_key}`)
+  const callbacks = _callbacks[`${model_name}.${query_key}`]
+
+  if (!callbacks || !Array.isArray(callbacks)) {
+    log("dispatch_query_payload", "unable to find callback for query payload", payload)
+    console.error("dispatch_query_payload", "unable to find callback for query payload", payload)
+    return
+  }
+
+  if (payload.error) {
+    console.warn("rts-client, payload error:", payload.error)
+    console.warn("rts-client", "in:", model_name, "query_key:", query_key)
+    callbacks.forEach((cb) => cb(payload.error, undefined, {source: "network"}))
+    return
+  }
+
+  let data
+  try {
+    // TODO: zstd / brotli decompression here
+    data = JSON.parse(payload.data_buf)
+  } catch (err) {
+    console.log("Error", err)
+    log("dispatch_query_payload", "unable to parse or send data from payload:", payload.data_buf)
+  }
+
+  if (!data) {
+    // skipping if data parsing failed
+    return
+  }
+
+  const context = {
+    source: "network",
+    is_local: _local_txn.includes(payload.txn_id),
+    txn_id: payload.txn_id,
+  }
+
+  callbacks.forEach((cb) => cb(null, data, context))
+
+  store.update_docs(model_name, data)
+}
+
+
+export const connect = () => new Promise((resolve) => {
+  log("rts client will connect")
+
+  _socket = io(BASE_URL, {
+    forceNew: true,
+    transports: ["websocket", "polling"],
+    withCredentials: true,
+    // extraHeaders: {},
+    // https://socket.io/docs/v4/client-options/#reconnection
+    reconnection: true,
+    reconnectionAttempts: 128,
+    reconnectionDelay: 400, // ms
+    reconnectionDelayMax: 10 * 1000, // 10s
+  })
+
+  _socket.on("connect", () => {
+    log("socket connected")
+    resolve()
+  })
+
+  _socket.io.on("reconnect", (e) => {
+    log("socked reconnected", e)
+  })
+
+  _socket.on("error", (err) => {
+    log("socket error", err)
+  })
+
+  _socket.on("query_payload", (payload) => {
+    // console.log("socket:query_payload", payload)
+    dispatch_query_payload(payload)
+  })
+
+  _socket.on("delete_doc", (payload) => {
+    log("document deleted", payload)
+  })
+
+  _socket.on("disconnect", (arg) => {
+    log("socket disconnected", arg)
+  })
+})
+
+export const reconnect = () => {
+  log("socket will force reconnect")
+  _socket?.disconnect()
+  setTimeout(() => {
+    _socket?.connect()
+  }, 200)
+}
+
+
+// register a query
+export const register_query = (model_name, query, _options, _callback) => {
+  // left shift args if _options is undefined
+  let options
+  let callback
+  if (_callback) {
+    assert(typeof _callback === "function")
+    options = _options
+    callback = _callback
+  } else {
+    options = {}
+    callback = _options
+  }
+
+  log("register_query", {model_name, query, options, callback})
+
+
+  if (!_socket) {
+    log("register_query: trying to use null socket", {model_name, query})
+    return
+  }
+
+  const key = options.key || ""
+  const query_key = `${key}${JSON.stringify(query)}`
+  log("registering query with key", key, model_name, query, query_key)
+
+  // save callback to update hooks
+  const cb_key = `${model_name}.${query_key}`
+  if (!_callbacks[cb_key] || !Array.isArray(_callbacks[cb_key])) {
+    _callbacks[cb_key] = []
+  }
+  _callbacks[cb_key].push(callback)
+
+  // save query for reconnections and retries
+  if (!_queries_store[model_name]) {
+    _queries_store[model_name] = {}
+  }
+  _queries_store[model_name][query]
+
+  // TODO: why both run and register query here ? the run_query should come straight from register ?
+  _socket.emit("run_query", {model_name, query, query_key, options})
+  _socket.emit("register_query", {model_name, query, query_key, options})
+
+  // run the query from the cache a first time
+  store.run_query({model_name, query, query_key, options}, callback)
+
+  return () => {
+    _socket.emit("remove_query", {model_name, query, query_key, options})
+    // remove callback
+    const cb_index = _callbacks[cb_key].indexOf(callback)
+    if (cb_index > -1) {
+      _callbacks[cb_key].splice(cb_index, 1)
+    } else {
+      console.warn("warning, trying to remove a callback that doesn't exist")
+    }
+  }
+}
+
+export const useQuery = getUseQuery(register_query)

package/rts/store/debug.js

@@ -0,0 +1,24 @@
+/* @flow */
+import debug from "debug"
+import PouchDB from "pouchdb-core"
+
+if (debug.enabled("rb:store")) {
+
+  const log = debug("rb:store:pouch")
+
+  log("store debug is enabled")
+
+  const shouldLogAllPouch = debug.enabled("pouch")
+
+  PouchDB.on("debug", (args) => {
+    if (shouldLogAllPouch) {
+      log(...args)
+    }
+    // log find only when not loggin all pouch
+    else {
+      if (args[0] === "find") {
+        log(...args)
+      }
+    }
+  })
+}

package/rts/store/get_collection.js

@@ -0,0 +1,35 @@
+/* @flow */
+import debug from "debug"
+import PouchDB from "pouchdb-core"
+import IndexedDBAdapter from "pouchdb-adapter-indexeddb"
+import FindPlugin from "pouchdb-find"
+
+import {DATABASE_NAME} from "env"
+
+const log = debug("rb:store")
+
+let prefix = "rb/"
+if (DATABASE_NAME) prefix += `${DATABASE_NAME}/`
+
+PouchDB.prefix = prefix
+
+PouchDB.plugin(IndexedDBAdapter)
+PouchDB.plugin(FindPlugin)
+
+
+const _cols_store = Object.create(null)
+
+
+const get_collection = (col_name) => {
+  if (_cols_store[col_name]) {
+    return _cols_store[col_name]
+  } else {
+    const col = new PouchDB(col_name, { adapter: "indexeddb" });
+    _cols_store[col_name] = col
+
+    return col
+  }
+
+}
+
+export default get_collection

package/rts/store/index.js

@@ -0,0 +1,161 @@
+/* @flow */
+import debug from "debug"
+
+import "./debug"
+
+import get_collection from "./get_collection"
+import update_docs from "./update_docs"
+
+const log = debug("rb:store")
+
+// import updateDocument from "./updateDocument"
+// import {DATABASE_NAME} from "env"
+// let prefix = "rb/"
+// if (DATABASE_NAME) prefix += `${DATABASE_NAME}/`
+//
+// PouchDB.prefix = prefix
+//
+// PouchDB.plugin(IndexedDBAdapter)
+// PouchDB.plugin(FindPlugin)
+//
+//
+// let db = new PouchDB(`db/items`, { adapter: "indexeddb" });
+//
+// // Create a new document
+// let doc = {
+//   _id: "001",
+//   message: "Hello, World!"
+// }
+// //
+//
+// const run = async() => {
+//   // https://github.com/pouchdb/pouchdb/tree/master/packages/node_modules/pouchdb-find#dbcreateindexindex--callback
+//   const res = await db.createIndex({
+//     index: {fields: ["name"]}
+//   })
+//
+//
+//   // Listen for changes on the database
+//   const fn = db.changes({
+//     since: "now",
+//     live: true,
+//     include_docs: true
+//   }).on("change", function(change) {
+//     // handle change
+//     console.log("GOT CHANGE", change)
+//   }).on("error", function (err) {
+//     // handle errors
+//     console.log(err)
+//   })
+//
+//   fn.cancel()
+//
+//   console.log("GOT FN", fn)
+//
+//   console.log("got res", res)
+//
+//   const doc = await db.find({
+//     selector: {name: "Mario"},
+//     sort: ["name"]
+//   })
+//
+//   console.log("GT doc", doc)
+//
+//   const {docs} = await db.find({selector: {}})
+//
+//   console.log("got all docs", docs)
+//
+//   db.put({
+//     _id: "001-" + Date.now(),
+//     message: "Hello, World!"
+//   })
+// }
+//
+//
+// run()
+
+// // Insert the document into the database
+// db.put(doc, function(err, response) {
+//   if (err) {
+//     return console.log(err);
+//   } else {
+//     console.log("Document created successfully");
+//   }
+// })
+//
+//
+// // Insert the document into the database
+// db.put({
+//   _id: "hello world",
+//   fieldVal: 12,
+// }, {force: true}, async function(err, response) {
+//   if (err) {
+//     console.log("errrro", err);
+//     console.log("Stt", JSON.stringify(err))
+//     return
+//   } else {
+//     console.log("Document created successfully", response);
+//   }
+//
+//   console.log("will try to find:")
+//   const doc = await db.find({
+//     selector: {
+//       fieldVal: 10
+//     }
+//   })
+//   console.log("LEDOC", doc)
+//
+// });
+//
+// setInterval(() => {
+//   // Fetch the document
+//   db.get("001", function(err, doc) {
+//     if (err) {
+//       return console.log(err);
+//     } else {
+//       console.log(doc);
+//     }
+//   });
+// }, 2000)
+//
+
+// TODO: should the store be in a worker or the main thread ?
+
+const run_query = async({model_name, query, query_key, options}, callback) => {
+  // console.log("ALAAARM")
+  // console.log("run_query", {model_name, query, query_key, options})
+
+  // console.time("store run_query")
+
+  const col = get_collection(model_name)
+
+
+  // https://github.com/pouchdb/pouchdb/tree/master/packages/node_modules/pouchdb-find#dbcreateindexindex--callback
+  const {docs} = await col.find({
+    // TODO: we should check if the selectors are compatible here
+    selector: query,
+    // DO NOT INCLUDE FIELDS HERE AS IT USES PICK AND WE WOULD GO through the list twice
+    // https://github.com/pouchdb/pouchdb/blob/master/packages/node_modules/pouchdb-find/src/adapters/local/find/index.js
+    // fields: [""]
+  })
+
+
+  const mapped_docs = docs.map(({_rev, ...doc}) => {
+    // TODO: handle projections here
+    const remapped_doc = Object.entries(doc).reduce((new_doc, [key, value]) => {
+      let new_key = key.startsWith('$_') ? key.replace(/^\$_/, "") : key
+      new_doc[new_key] = value
+      return new_doc
+    }, {})
+
+    return remapped_doc
+  })
+
+  // console.log("FOUND INITIAL", mapped_docs)
+  // console.timeEnd("store run_query")
+
+  callback(null, mapped_docs, {source: "cache"})
+}
+
+
+export default {run_query, update_docs}

package/rts/store/update_docs.js

@@ -0,0 +1,44 @@
+/* @flow */
+import get_collection from "./get_collection"
+
+
+const update_docs = async(model_name, data) => {
+  const collection = get_collection(model_name)
+
+  const all_ids = data.map((doc) => doc._id)
+
+  // console.log("will update cache", model_name, data, all_ids)
+
+  // console.time("find")
+
+  // TODO:
+  // there is also a bulk get which could have different performance than find, try both
+  // https://pouchdb.com/api.html#bulk_get
+
+  // https://github.com/pouchdb/pouchdb/tree/master/packages/node_modules/pouchdb-find#dbcreateindexindex--callback
+  const {docs: current_docs} = await collection.find({
+    selector: {_id: {$in: all_ids}},
+    fields: ["_id", "_rev"],
+  })
+
+  // console.log("current_docs", current_docs)
+
+  const revs_map = {}
+  current_docs.forEach((doc) => revs_map[doc._id] = doc._rev)
+
+  const write_docs = data.map((mongo_doc) => {
+    const op = Object.entries(mongo_doc).reduce((new_doc, [key, value]) => {
+      let new_key = key !== "_id" && key.startsWith('_') ? `$_${key}` : key
+      new_doc[new_key] = value
+      return new_doc
+    }, {})
+
+    op._rev = revs_map[mongo_doc._id]
+    return op
+  })
+
+  await collection.bulkDocs(write_docs)
+  // console.timeEnd("find")
+}
+
+export default update_docs