@rpcbase/client 0.44.0 → 0.45.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rpcbase/client",
-  "version": "0.44.0",
+  "version": "0.45.0",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 0"
   },
@@ -10,6 +10,10 @@
     "i18next-chained-backend": "4.4.0",
     "i18next-resources-to-backend": "1.1.4",
     "lodash": "4.17.21",
-    "react-i18next": "13.0.1"
+    "pouchdb-adapter-indexeddb": "8.0.1",
+    "pouchdb-core": "8.0.1",
+    "pouchdb-find": "8.0.1",
+    "react-i18next": "13.0.1",
+    "socket.io-client": "4.7.1"
   }
 }

package/rpc_post.js

@@ -5,7 +5,7 @@ import _get from "lodash/get"
 import _set from "lodash/set"
 
 import get_txn_id from "@rpcbase/std/get_txn_id"
-import {add_local_txn} from "@rpcbase/realtime-state-client"
+import {add_local_txn} from "@rpcbase/client/rts"
 
 import {BASE_URL} from "env"
 

package/rts/boot.js

@@ -0,0 +1,5 @@
+/* @flow */
+
+console.log("RTS BOOT, TODO: connect")
+
+import "./store"

package/rts/cacheStorage/index.js

@@ -0,0 +1,12 @@
+/* @flow */
+import {Platform} from "react-native"
+
+import getWebStorage from "./web"
+import getNativeStorage from "./native"
+
+
+// TODO: add support for storage expiration
+
+const storage = Platform.OS === "web" ? getWebStorage() : getNativeStorage()
+
+export default storage

package/rts/cacheStorage/native.js

@@ -0,0 +1,28 @@
+/* @flow */
+import AsyncStorage from "@react-native-async-storage/async-storage"
+
+const getNativeStorage = () => {
+  const cacheStorage = {
+    get: async(key) => {
+      try {
+        const res = await AsyncStorage.getItem(key)
+        const val = JSON.parse(res)
+        if (val) return val
+      } catch (error) {
+        // There was an error on the native side
+      }
+    },
+    set: async(key, value) => {
+      try {
+        await AsyncStorage.setItem(key, JSON.stringify(value))
+      } catch (error) {
+        // There was an error on the native side
+      }
+    }
+  }
+
+  return cacheStorage
+}
+
+
+export default getNativeStorage

package/rts/cacheStorage/web.js

@@ -0,0 +1,85 @@
+/* @flow */
+const storeName = "store"
+const version = 1
+
+const getWebStorage = (
+  dbName = "rb-query-cache"
+) => {
+  let __db
+
+  const openRequest = indexedDB.open(dbName, version)
+
+  // Handle the creation or upgrade of the database
+  openRequest.onupgradeneeded = (event) => {
+    const db = event.target.result
+    if (!db.objectStoreNames.contains(storeName)) {
+      db.createObjectStore(storeName, { keyPath: "key" })
+    }
+  }
+
+  // Handle errors when opening the database
+  openRequest.onerror = (event) => {
+    console.error("Error opening database:", event.target.errorCode)
+  }
+
+  openRequest.onsuccess = (event) => {
+    const db = event.target.result
+    __db = db
+  }
+
+
+  const cacheStorage = {
+    get: (key) => new Promise((resolve, reject) => {
+      const transaction = __db.transaction([storeName], "readonly")
+      const objectStore = transaction.objectStore(storeName)
+      const request = objectStore.get(key)
+
+      request.onsuccess = (event) => {
+        const obj = event.target.result?.obj
+        resolve(obj)
+      }
+
+      request.onerror = (event) => {
+        console.error("Error reading data:", event.target)
+        reject()
+      }
+
+    }),
+    set: (key, obj) => new Promise((resolve, reject) => {
+
+      const transaction = __db.transaction([storeName], "readwrite")
+      const objectStore = transaction.objectStore(storeName)
+      const request = objectStore.put({obj, key})
+
+      request.onsuccess = (event) => {
+        // console.log("Data written successfully:", event.target.result)
+        resolve()
+      }
+
+      request.onerror = (event) => {
+        // console.error("Error writing data:", event.target.errorCode)
+        console.log("ERRR", event.target)
+        reject()
+      }
+    }),
+    delete: (key) => new Promise((resolve, reject) => {
+      const transaction = __db.transaction([storeName], "readwrite")
+      const objectStore = transaction.objectStore(storeName)
+      const request = objectStore.delete(key)
+
+      request.onsuccess = (event) => {
+        resolve()
+      }
+
+      request.onerror = (event) => {
+        console.error("Error deleting data:", event.target.errorCode)
+        reject()
+      }
+    }),
+  }
+
+  return cacheStorage
+}
+
+
+export default getWebStorage

package/rts/getUseQuery.js

@@ -0,0 +1,153 @@
+/* @flow */
+import assert from "assert"
+import {Platform} from "react-native"
+import {useCallback, useEffect, useState, useMemo, useId, useRef} from "react"
+import debug from "debug"
+import isEqual from "fast-deep-equal/react"
+import _omit from "lodash/omit"
+
+import get_uid from "@rpcbase/client/auth/get_uid"
+
+import cacheStorage from "./cacheStorage"
+
+
+const log = debug("rb:rts:client")
+
+const getUseQuery = (register_query) => (
+  model_name,
+  query = {},
+  options = {},
+) => {
+  const id = useId()
+
+  // TODO: retrieve this from AuthContext in client
+  const uid = useMemo(() => {
+    const _uid = Platform.OS === "web" ? get_uid() : options.userId
+    assert(_uid, "missing uid")
+
+    return _uid
+  }, [])
+
+  const hasFirstReply = useRef(false)
+  const lastDocRef = useRef(null)
+  // const [page, setPage] = useState(0)
+
+  const [data, setData] = useState()
+  const [error, setError] = useState()
+  const [loading, setLoading] = useState(true)
+
+  const {
+    key = "",
+    projection = {},
+    sort = {},
+  } = options
+
+
+  const storageKey = `${uid}.${key}.${model_name}.${JSON.stringify(query)}.${JSON.stringify(projection)}.${JSON.stringify(sort)}`
+
+
+  useEffect(() => {
+    if (options.debug) {
+      console.log("use query", model_name, query, options)
+    }
+  }, [])
+
+  useEffect(() => {
+    const load = async() => {
+      const val = await cacheStorage.get(storageKey)
+      // always initially apply when first load here
+      if (val) {
+        setData(val)
+        setLoading(false)
+      }
+    }
+
+    load()
+  }, [storageKey])
+
+  const applyNewData = (newData) => {
+    setData(newData)
+
+    // write data to storage
+    // TODO: is there a better way than doing it on every update ?
+    // should this be throttled
+    cacheStorage.set(storageKey, newData)
+
+    if (newData?.length > 0) {
+      lastDocRef.current = newData[newData.length - 1]
+    }
+  }
+
+  useEffect(() => {
+    const queryKey = key || id
+
+    if (options.debug ) {
+      console.log("register query", model_name, query, options)
+    }
+
+    const start = Date.now()
+    const unsubscribe = register_query(model_name, query, {...options, key: queryKey}, (err, queryResult, context) => {
+
+      if (options.debug ) {
+        console.log("query took", Date.now() - start, model_name, query)
+      }
+
+      setLoading(false)
+      if (err) {
+        setError(err)
+        return
+      }
+
+      log("query callback", model_name, queryKey, JSON.stringify(query))
+
+      // return if no data (this should be handled already)
+      if (!queryResult) return
+
+      let newData
+      if (Array.isArray(queryResult)) {
+        newData = queryResult.map((o) => _omit(o, "__txn_id"))
+      } else {
+        newData = _omit(queryResult, "__txn_id")
+      }
+
+      // We return once in any case!
+      if (!hasFirstReply.current) {
+        hasFirstReply.current = true
+
+        // skip if we already have the data
+        if (isEqual(data, newData)) return
+
+        applyNewData(newData)
+        return
+      }
+
+      if (context.is_local && options.skip_local && hasFirstReply.current) {
+        // console.log("skipping local update", key)
+        return
+      }
+
+      if (!isEqual(data, newData)) {
+        applyNewData(newData)
+      }
+    })
+
+    return () => {
+      log && log("useQuery cleanup unsubscribe()")
+      unsubscribe()
+    }
+  }, [JSON.stringify(query), key, storageKey])
+
+
+  const loadNextPage = useCallback(() => {
+    console.log("will load next page after DOC")
+  }, [])
+
+
+  const result = useMemo(() => ({data, error, loading}), [data, error, loading])
+
+  return result
+}
+
+
+
+export default getUseQuery

package/rts/index.js

@@ -0,0 +1,184 @@
+/* @flow */
+import assert from "assert"
+import {io} from "socket.io-client"
+import _get from "lodash/get"
+import _set from "lodash/set"
+import debug from "debug"
+
+import {BASE_URL} from "env"
+
+import getUseQuery from "./getUseQuery"
+
+const log = debug("rb:rts:client")
+
+const MAX_TXN_BUF = 2048
+
+let _socket
+const _callbacks = {}
+const _queries_storage = {}
+const _local_txn = []
+
+// TODO: when server disconnects / crashes and loses all server side stored queries
+// the clients must reconnect and re-register those queries, or the page will need to be hard refreshed
+
+
+// add_local_txn
+// when a request is made to the server, we generate (or send if provided) the txn_id
+// if the array becomes longer than the default buffer length we shift the array
+export const add_local_txn = (txn_id) => {
+  assert(txn_id, "add_local_txn trying to add an invalid txn_id")
+  _local_txn.push(txn_id)
+  if (_local_txn.length > MAX_TXN_BUF) {
+    _local_txn.shift()
+  }
+}
+
+
+// TODO: add compression / decompression
+const dispatch_query_payload = (payload) => {
+  log("dispatch_query_payload", payload)
+
+  const {model_name, query_key} = payload
+  // const cb = _get(_callbacks, `${model_name}.${query_key}`)
+  const callbacks = _callbacks[`${model_name}.${query_key}`]
+
+  if (!callbacks || !Array.isArray(callbacks)) {
+    log("dispatch_query_payload", "unable to find callback for query payload", payload)
+    console.error("dispatch_query_payload", "unable to find callback for query payload", payload)
+    return
+  }
+
+  if (payload.error) {
+    console.warn("rts-client, payload error:", payload.error)
+    console.warn("rts-client", "in:", model_name, "query_key:", query_key)
+    callbacks.forEach((cb) => cb(payload.error))
+    return
+  }
+
+  let data
+  try {
+    // TODO: zstd decompression here
+    data = JSON.parse(payload.data_buf)
+  } catch (err) {
+    console.log("Error", err)
+    log("dispatch_query_payload", "unable to parse or send data from payload:", payload.data_buf)
+  }
+
+  if (!data) {
+    // skipping if data parsing failed
+    return
+  }
+
+  const context = {
+    is_local: _local_txn.includes(payload.txn_id),
+    txn_id: payload.txn_id,
+  }
+
+  callbacks.forEach((cb) => cb(null, data, context))
+}
+
+
+export const connect = () => new Promise((resolve) => {
+  log("rts client will connect")
+
+  _socket = io(BASE_URL, {
+    forceNew: true,
+    transports: ["websocket", "polling"],
+    withCredentials: true,
+    // extraHeaders: {},
+    // https://socket.io/docs/v4/client-options/#reconnection
+    reconnection: true,
+    reconnectionAttempts: 128,
+    reconnectionDelay: 400, // ms
+    reconnectionDelayMax: 10 * 1000, // 10s
+  })
+
+  _socket.on("connect", () => {
+    log("socket connected")
+    resolve()
+  })
+
+  _socket.io.on("reconnect", (e) => {
+    log("socked reconnected", e)
+  })
+
+  _socket.on("error", (err) => {
+    log("socket error", err)
+  })
+
+  _socket.on("query_payload", (payload) => {
+    // console.log("socket:query_payload", payload)
+    dispatch_query_payload(payload)
+  })
+
+  _socket.on("delete_doc", (payload) => {
+    log("document deleted", payload)
+  })
+
+  _socket.on("disconnect", (arg) => {
+    log("socket disconnected", arg)
+  })
+})
+
+export const reconnect = () => {
+  log("socket will force reconnect")
+  _socket?.disconnect()
+  setTimeout(() => {
+    _socket?.connect()
+  }, 200)
+}
+
+
+// register a query
+export const register_query = (model_name, query, _options, _callback) => {
+  // left shift args if _options is undefined
+  let options
+  let callback
+  if (_callback) {
+    assert(typeof _callback === "function")
+    options = _options
+    callback = _callback
+  } else {
+    options = {}
+    callback = _options
+  }
+
+
+  if (!_socket) {
+    log("trying to use null socket", {model_name, query})
+    return
+  }
+
+  const key = options.key || ""
+  log("registering query with key", key, model_name, query)
+  const query_key = `${key}${JSON.stringify(query)}`
+
+  // save callback to update hooks
+  const cb_key = `${model_name}.${query_key}`
+  if (!_callbacks[cb_key] || !Array.isArray(_callbacks[cb_key])) {
+    _callbacks[cb_key] = []
+  }
+  _callbacks[cb_key].push(callback)
+
+  // save query for reconnections and retries
+  if (!_queries_storage[model_name]) {
+    _queries_storage[model_name] = {}
+  }
+  _queries_storage[model_name][query]
+
+  _socket.emit("run_query", {model_name, query, query_key, options})
+  _socket.emit("register_query", {model_name, query, query_key, options})
+
+  return () => {
+    _socket.emit("remove_query", {model_name, query})
+    // remove callback
+    const cb_index = _callbacks[cb_key].indexOf(callback)
+    if (cb_index > -1) {
+      _callbacks[cb_key].splice(cb_index, 1)
+    } else {
+      console.warn("warning, trying to remove a callback that doesn't exist")
+    }
+  }
+}
+
+export const useQuery = getUseQuery(register_query)