@rpcbase/client 0.95.0 → 0.96.0

package/jest.config.js

@@ -0,0 +1,17 @@
+/* @flow */
+
+module.exports = {
+  testEnvironment: "jsdom",
+  transform: {
+    "^.+\\.js$": ["babel-jest", {
+      presets: ["@babel/preset-react"],
+      plugins: [
+        "@babel/plugin-transform-runtime",
+        "@babel/plugin-transform-modules-commonjs"
+      ],
+    }],
+  },
+  moduleNameMapper: {
+    "^react-native$": "react-native-web",
+ },
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rpcbase/client",
-  "version": "0.95.0",
+  "version": "0.96.0",
   "scripts": {
     "build-firebase": "webpack -c firebase/webpack.config.js",
     "build": "yarn build-firebase",
@@ -20,8 +20,21 @@
     "socket.io-client": "4.7.2"
   },
   "devDependencies": {
+    "@babel/core": "7.23.0",
+    "@babel/plugin-transform-modules-commonjs": "7.23.0",
+    "@babel/plugin-transform-runtime": "7.22.15",
+    "@babel/preset-react": "7.22.15",
+    "@testing-library/react": "14.0.0",
+    "@testing-library/react-hooks": "8.0.1",
+    "babel-jest": "29.7.0",
+    "bluebird": "3.7.2",
     "firebase": "10.2.0",
     "jest": "29.6.3",
+    "jest-environment-jsdom": "29.7.0",
+    "react": "18.2.0",
+    "react-dom": "18.2.0",
+    "react-native-web": "0.19.9",
+    "react-test-renderer": "18.2.0",
     "webpack": "5.88.2",
     "webpack-cli": "5.1.4"
   }

package/rts/getUseQuery.js

@@ -19,31 +19,67 @@ const getUseQuery = (register_query) => (
 ) => {
   const id = useId()
 
-  // TODO: retrieve this from AuthContext in client
+  // TODO: retrieve this from future AuthContext in client
   const uid = useMemo(() => {
+    // TODO: why is there a options.userId here ??
     const _uid = Platform.OS === "web" ? get_uid() : options.userId
     assert(_uid, "missing uid")
-
     return _uid
   }, [])
 
+  const hasInitiallySetFromStorage = useRef(false)
   const hasFirstReply = useRef(false)
   const hasNetworkReply = useRef(false)
   const lastDocRef = useRef(null)
   // const [page, setPage] = useState(0)
 
-  const [source, setSource] = useState()
-  const [data, setData] = useState()
-  const [error, setError] = useState()
-  const [loading, setLoading] = useState(true)
-
   const {
     key = "",
     projection = {},
     sort = {},
+    useStorage = false,
   } = options
 
-  // const storageKey = `${uid}.${key}.${model_name}.${JSON.stringify(query)}.${JSON.stringify(projection)}.${JSON.stringify(sort)}`
+  const storageKey = useMemo(() => {
+    return `${uid}${key ? `.${key}` : ""}.${model_name}.${JSON.stringify(query)}.${JSON.stringify(projection)}.${JSON.stringify(sort)}`
+  }, [uid, key, model_name, query, projection, sort])
+
+  const [source, setSource] = useState()
+
+  const [data, setData] = useState(() => {
+    if (!useStorage) {
+      return
+    }
+
+    if (Platform.OS === "web") {
+      const item = localStorage.getItem(storageKey)
+      if (!item) return
+
+      hasInitiallySetFromStorage.current = true
+
+      let parsedItem
+      try {
+        parsedItem = JSON.parse(item)
+      } catch (err) {
+        //
+      }
+      if (parsedItem) {
+        return parsedItem
+      }
+    } else {
+      // TODO: fast storage not implemented on mobile
+      console.log("fast storage mobile nOT IMPLEMENTED")
+    }
+  })
+
+  const [error, setError] = useState()
+
+  const [loading, setLoading] = useState(() => {
+    if (hasInitiallySetFromStorage.current) {
+      return false
+    }
+    return true
+  })
 
   useEffect(() => {
     if (options.debug) {
@@ -54,6 +90,15 @@ const getUseQuery = (register_query) => (
   const applyNewData = (newData, context) => {
     setData(newData)
 
+    // we only save network queries
+    if (useStorage && context.source === "network") {
+      if (Platform.OS === "web") {
+        localStorage.setItem(storageKey, JSON.stringify(newData))
+      } else {
+        // TODO: rn fast storage async
+      }
+    }
+
     if (newData?.length > 0) {
       lastDocRef.current = newData[newData.length - 1]
     }
@@ -73,7 +118,9 @@ const getUseQuery = (register_query) => (
     }
 
     const start = Date.now()
+
     log("will register query", model_name, query)
+
     const unsubscribe = register_query(model_name, query, {...options, key: queryKey}, (err, queryResult, context) => {
       log("callback answer with context", context, queryResult?.length)
 
@@ -125,7 +172,8 @@ const getUseQuery = (register_query) => (
         return
       }
 
-      if (context.is_local && options.skip_local && hasFirstReply.current) {
+      // TODO: this should be handled by the consumer with the context (cache or network)
+      if (context.is_local && options.skipLocal && hasFirstReply.current) {
         log("skipping local update", key)
         return
       }
@@ -140,6 +188,9 @@ const getUseQuery = (register_query) => (
       log && log("useQuery cleanup unsubscribe()")
       typeof unsubscribe === "function" && unsubscribe()
     }
+
+
+    // TODO: this isnt right we need to update on options change too
   }, [JSON.stringify(query), key])
 
 

package/rts/index.js

@@ -1,222 +1,2 @@
 /* @flow */
-import assert from "assert"
-import {io} from "socket.io-client"
-import _get from "lodash/get"
-import _set from "lodash/set"
-import debug from "debug"
-
-import BASE_URL from "../base_url"
-import {get_tenant_id} from "../auth"
-
-import store from "./store"
-import getUseDocument from "./getUseDocument"
-import getUseQuery from "./getUseQuery"
-
-const log = debug("rb:socket")
-
-const TENANT_ID_HEADER = "rb-tenant-id"
-
-// how many local transaction ids are we keeping, it doesn't really need to be that big
-const MAX_TXN_BUF = 2048
-
-let _socket
-const _callbacks = {}
-const _queries_store = {}
-const _local_txn = []
-
-// TODO: when server disconnects / crashes and loses all server side stored queries
-// the clients must reconnect and re-register those queries, or the page will need to be hard refreshed
-
-// add_local_txn
-// when a request is made to the server, we generate (or send if provided) the txn_id
-// if the array becomes longer than the default buffer length we shift the array
-export const add_local_txn = (txn_id) => {
-  assert(txn_id, "add_local_txn trying to add an invalid txn_id")
-  _local_txn.push(txn_id)
-  if (_local_txn.length > MAX_TXN_BUF) {
-    _local_txn.shift()
-  }
-}
-
-
-// TODO: add compression / decompression
-const dispatch_query_payload = (payload) => {
-  log("dispatch_query_payload", payload)
-
-  const {model_name, query_key} = payload
-  // const cb = _get(_callbacks, `${model_name}.${query_key}`)
-  const callbacks = _callbacks[`${model_name}.${query_key}`]
-
-  if (!callbacks || !Array.isArray(callbacks)) {
-    log("dispatch_query_payload", "unable to find callback for query payload", payload)
-    console.error("dispatch_query_payload", "unable to find callback for query payload", payload)
-    return
-  }
-
-  if (payload.error) {
-    console.warn("rts-client, payload error:", payload.error)
-    console.warn("rts-client", "in:", model_name, "query_key:", query_key)
-    callbacks.forEach((cb) => cb(payload.error, undefined, {source: "network"}))
-    return
-  }
-
-  let data
-  try {
-    // TODO: zstd decompression here
-    data = JSON.parse(payload.data_buf)
-  } catch (err) {
-    console.log("Error", err)
-    log("dispatch_query_payload", "unable to parse or send data from payload:", payload.data_buf)
-  }
-
-  if (!data) {
-    // skipping if data parsing failed
-    return
-  }
-
-  const context = {
-    source: "network",
-    is_local: _local_txn.includes(payload.txn_id),
-    txn_id: payload.txn_id,
-  }
-
-  callbacks.forEach((cb) => cb(null, data, context))
-
-  store.update_docs(model_name, data)
-}
-
-
-export const connect = () => new Promise((resolve) => {
-  const tenant_id = get_tenant_id()
-
-  if (tenant_id) {
-    log("rts client will connect")
-  } else {
-    log("no tenant_id, rts connect will skip")
-    return
-  }
-
-
-  _socket = io(BASE_URL, {
-    forceNew: true,
-    transports: ["websocket", "polling"],
-    withCredentials: true,
-    query: {
-      [TENANT_ID_HEADER]: tenant_id,
-    },
-    // https://socket.io/docs/v4/client-options/#reconnection
-    reconnection: true,
-    reconnectionAttempts: 128,
-    reconnectionDelay: 400, // ms
-    reconnectionDelayMax: 10 * 1000, // 10s
-  })
-
-  _socket.on("connect", () => {
-    log("socket connected")
-    resolve()
-  })
-
-  _socket.io.on("reconnect", (e) => {
-    log("socked reconnected", e)
-  })
-
-  _socket.on("error", (err) => {
-    log("socket error", err)
-  })
-
-  _socket.on("query_payload", (payload) => {
-    // console.log("socket:query_payload", payload)
-    dispatch_query_payload(payload)
-  })
-
-  _socket.on("delete_doc", (payload) => {
-    log("document deleted", payload)
-  })
-
-  _socket.on("disconnect", (arg) => {
-    log("socket disconnected", arg)
-  })
-})
-
-
-export const disconnect = () => {
-  if (_socket) {
-    try {
-      _socket.disconnect()
-    } catch (e) {
-      // TODO: should we be swallowing this error
-    }
-    _socket = null
-  }
-}
-
-
-export const reconnect = () => {
-  log("socket will force reconnect")
-
-  // destroy current socket if exists
-  disconnect()
-
-  connect()
-}
-
-// register a query
-export const register_query = (model_name, query, _options, _callback) => {
-  // left shift args if _options is undefined
-  let options
-  let callback
-  if (_callback) {
-    assert(typeof _callback === "function")
-    options = _options
-    callback = _callback
-  } else {
-    options = {}
-    callback = _options
-  }
-
-  log("register_query", {model_name, query, options, callback})
-
-  if (!_socket) {
-    log("register_query: trying to use null socket", {model_name, query})
-    return
-  }
-
-  const key = options.key || ""
-  const query_key = `${key}${JSON.stringify(query)}${options.projection ? JSON.stringify(options.projection) : ""}`
-  log("registering query with key", key, model_name, query, query_key)
-
-  // save callback to update hooks
-  const cb_key = `${model_name}.${query_key}`
-  if (!_callbacks[cb_key] || !Array.isArray(_callbacks[cb_key])) {
-    _callbacks[cb_key] = []
-  }
-  _callbacks[cb_key].push(callback)
-
-  // save query for reconnections and retries
-  if (!_queries_store[model_name]) {
-    _queries_store[model_name] = {}
-  }
-  _queries_store[model_name][query]
-
-  // TODO: why both run and register query here ? the run_query should come straight from register ?
-  _socket.emit("run_query", {model_name, query, query_key, options})
-  _socket.emit("register_query", {model_name, query, query_key, options})
-
-  // run the query from the cache a first time
-  store.run_query({model_name, query, query_key, options}, callback)
-
-  return () => {
-    _socket.emit("remove_query", {model_name, query, query_key, options})
-    // remove callback
-    const cb_index = _callbacks[cb_key].indexOf(callback)
-    if (cb_index > -1) {
-      _callbacks[cb_key].splice(cb_index, 1)
-    } else {
-      console.warn("warning, trying to remove a callback that doesn't exist")
-    }
-  }
-}
-
-export const useQuery = getUseQuery(register_query)
-
-export const useDocument = getUseDocument(register_query)
+export * from "./rts"

package/rts/rts.js

@@ -0,0 +1,223 @@
+/* @flow */
+import assert from "assert"
+import {io} from "socket.io-client"
+import _get from "lodash/get"
+import _set from "lodash/set"
+import debug from "debug"
+
+import BASE_URL from "../base_url"
+import {get_tenant_id} from "../auth"
+
+import store from "./store"
+import getUseDocument from "./getUseDocument"
+import getUseQuery from "./getUseQuery"
+
+const log = debug("rb:socket")
+
+const TENANT_ID_HEADER = "rb-tenant-id"
+
+// how many local transaction ids are we keeping, it doesn't really need to be that big
+const MAX_TXN_BUF = 2048
+
+let _socket
+const _callbacks = {}
+const _queries_store = {}
+const _local_txn = []
+
+// TODO: when server disconnects / crashes and loses all server side stored queries
+// the clients must reconnect and re-register those queries, or the page will need to be hard refreshed
+
+// add_local_txn
+// when a request is made to the server, we generate (or send if provided) the txn_id
+// if the array becomes longer than the default buffer length we shift the array
+export const add_local_txn = (txn_id) => {
+  assert(txn_id, "add_local_txn trying to add an invalid txn_id")
+  _local_txn.push(txn_id)
+  if (_local_txn.length > MAX_TXN_BUF) {
+    _local_txn.shift()
+  }
+}
+
+
+// TODO: add compression / decompression
+const dispatch_query_payload = (payload) => {
+  log("dispatch_query_payload", payload)
+
+  const {model_name, query_key} = payload
+  // const cb = _get(_callbacks, `${model_name}.${query_key}`)
+  const callbacks = _callbacks[`${model_name}.${query_key}`]
+
+  if (!callbacks || !Array.isArray(callbacks)) {
+    log("dispatch_query_payload", "unable to find callback for query payload", payload)
+    console.error("dispatch_query_payload", "unable to find callback for query payload", payload)
+    return
+  }
+
+  if (payload.error) {
+    console.warn("rts-client, payload error:", payload.error)
+    console.warn("rts-client", "in:", model_name, "query_key:", query_key)
+    callbacks.forEach((cb) => cb(payload.error, undefined, {source: "network"}))
+    return
+  }
+
+  let data
+  try {
+    // TODO: zstd decompression here
+    data = JSON.parse(payload.data_buf)
+  } catch (err) {
+    console.log("Error", err)
+    log("dispatch_query_payload", "unable to parse or send data from payload:", payload.data_buf)
+  }
+
+  if (!data) {
+    // skipping if data parsing failed
+    return
+  }
+
+  const context = {
+    source: "network",
+    is_local: _local_txn.includes(payload.txn_id),
+    txn_id: payload.txn_id,
+  }
+
+  callbacks.forEach((cb) => cb(null, data, context))
+
+  store.update_docs(model_name, data)
+}
+
+
+export const connect = () => new Promise((resolve) => {
+  const tenant_id = get_tenant_id()
+
+  if (tenant_id) {
+    log("rts client will connect")
+  } else {
+    log("no tenant_id, rts connect will skip")
+    return
+  }
+
+
+  _socket = io(BASE_URL, {
+    forceNew: true,
+    transports: ["websocket", "polling"],
+    withCredentials: true,
+    query: {
+      [TENANT_ID_HEADER]: tenant_id,
+    },
+    // https://socket.io/docs/v4/client-options/#reconnection
+    reconnection: true,
+    reconnectionAttempts: 128,
+    reconnectionDelay: 400, // ms
+    reconnectionDelayMax: 10 * 1000, // 10s
+  })
+
+  _socket.on("connect", () => {
+    log("socket connected")
+    resolve()
+  })
+
+  _socket.io.on("reconnect", (e) => {
+    log("socked reconnected", e)
+  })
+
+  _socket.on("error", (err) => {
+    log("socket error", err)
+  })
+
+  _socket.on("query_payload", (payload) => {
+    // console.log("socket:query_payload", payload)
+    dispatch_query_payload(payload)
+  })
+
+  _socket.on("delete_doc", (payload) => {
+    log("document deleted", payload)
+  })
+
+  _socket.on("disconnect", (arg) => {
+    log("socket disconnected", arg)
+  })
+})
+
+
+export const disconnect = () => {
+  if (_socket) {
+    try {
+      _socket.disconnect()
+    } catch (e) {
+      // TODO: should we be swallowing this error
+    }
+    _socket = null
+  }
+}
+
+
+export const reconnect = () => {
+  log("socket will force reconnect")
+
+  // destroy current socket if exists
+  disconnect()
+
+  connect()
+}
+
+// register a query
+export const register_query = (model_name, query, _options, _callback) => {
+  // left shift args if _options is undefined
+  let options
+  let callback
+  if (_callback) {
+    assert(typeof _callback === "function")
+    options = _options
+    callback = _callback
+  } else {
+    options = {}
+    callback = _options
+  }
+
+  log("register_query", {model_name, query, options, callback})
+
+  if (!_socket) {
+    log("register_query: trying to use null socket", {model_name, query})
+    return
+  }
+
+  const key = options.key || ""
+  const query_key = `${key}${JSON.stringify(query)}${options.projection ? JSON.stringify(options.projection) : ""}`
+  log("registering query with key", key, model_name, query, query_key)
+
+  // save callback to update hooks
+  const cb_key = `${model_name}.${query_key}`
+  if (!_callbacks[cb_key] || !Array.isArray(_callbacks[cb_key])) {
+    _callbacks[cb_key] = []
+  }
+  _callbacks[cb_key].push(callback)
+
+  // save query for reconnections and retries
+  if (!_queries_store[model_name]) {
+    _queries_store[model_name] = {}
+  }
+  _queries_store[model_name][query]
+
+  // TODO: why both run and register query here ? the run_query should come straight from register ?
+  _socket.emit("run_query", {model_name, query, query_key, options})
+  _socket.emit("register_query", {model_name, query, query_key, options})
+
+  // run the query from the cache a first time
+  store.run_query({model_name, query, query_key, options}, callback)
+
+  return () => {
+    _socket.emit("remove_query", {model_name, query, query_key, options})
+    // remove callback
+    const cb_index = _callbacks[cb_key].indexOf(callback)
+    if (cb_index > -1) {
+      _callbacks[cb_key].splice(cb_index, 1)
+    } else {
+      console.warn("warning, trying to remove a callback that doesn't exist")
+    }
+  }
+}
+
+
+export const useQuery = getUseQuery(register_query)
+
+export const useDocument = getUseDocument(register_query)

package/rts/store/constants.js

@@ -0,0 +1,3 @@
+/* @flow */
+
+export const UNDERSCORE_PREFIX = "$_"

package/rts/store/index.js

@@ -3,14 +3,18 @@ import debug from "debug"
 
 import "./debug"
 
+import {UNDERSCORE_PREFIX} from "./constants"
+
 import get_collection from "./get_collection"
 import update_docs from "./update_docs"
 import satisfies_projection from "./satisfies_projection"
+import replace_query_keys from "./replace_query_keys"
 
 const log = debug("rb:rts:store")
 
 
-//   // https://github.com/pouchdb/pouchdb/tree/master/packages/node_modules/pouchdb-find#dbcreateindexindex--callback
+// TODO: listening for changes
+// https://github.com/pouchdb/pouchdb/tree/master/packages/node_modules/pouchdb-find#dbcreateindexindex--callback
 //   const res = await db.createIndex({
 //     index: {fields: ["name"]}
 //   })
@@ -26,9 +30,7 @@ const log = debug("rb:rts:store")
 //     // handle errors
 //     console.log(err)
 //   })
-//
 //   fn.cancel()
-//
 //   console.log("got res", res)
 // _cols_store[col_name].getIndexes().then(function (result) {
 //   console.log("got indexes", result)
@@ -42,13 +44,14 @@ const run_query = async({model_name, query, query_key, options}, callback) => {
   // console.time("store run_query")
 
   // TODO: we should prefix model_name with tenant_prefix + env_id
-  const col = get_collection(model_name)
+  const collection = get_collection(model_name)
+
+  const replaced_query = replace_query_keys(query, (k) => (k.startsWith("_") && k !== "_id") ? `${UNDERSCORE_PREFIX}${k}` : k)
 
   // console.time(`query-${model_name}`)
   // https://github.com/pouchdb/pouchdb/tree/master/packages/node_modules/pouchdb-find#dbcreateindexindex--callback
-  const {docs} = await col.find({
-    // TODO: we should check if the selectors are compatible here
-    selector: query,
+  const {docs} = await collection.find({
+    selector: replaced_query,
     // DO NOT INCLUDE FIELDS HERE AS IT USES PICK AND WE WOULD GO through the list twice
     // https://github.com/pouchdb/pouchdb/blob/master/packages/node_modules/pouchdb-find/src/adapters/local/find/index.js
     // fields: [""]

package/rts/store/replace_query_keys.js

@@ -0,0 +1,30 @@
+/* @flow */
+
+
+// WARNING: this is a bit of a toy implementation it's not certain it will work with complex queries where a field isn't referenced by being an object's key
+// TODO: this isn't right it's missing a lot of possible queries
+// https://chat.openai.com/share/da344eed-e3e1-4c8d-8091-cc26b22a8f2e
+
+const replace_query_keys = (obj, fn) => {
+  if (typeof obj !== 'object' || obj === null) {
+    return obj
+  }
+
+  if (Array.isArray(obj)) {
+    return obj.map(item => replace_query_keys(item, fn))
+  }
+
+  // create an empty new obj with the same prototype as obj
+  const new_obj = Object.create(Object.getPrototypeOf(obj))
+
+  Object
+    .keys(obj)
+    .forEach(key => {
+      const new_key = fn(key)
+      new_obj[new_key] = replace_query_keys(obj[key], fn)
+    })
+
+  return new_obj
+}
+
+export default replace_query_keys

package/rts/store/update_docs.js

@@ -1,6 +1,8 @@
 /* @flow */
 import get_collection from "./get_collection"
 
+import {UNDERSCORE_PREFIX} from "./constants"
+
 
 const update_docs = async(model_name, data) => {
   const collection = get_collection(model_name)
@@ -29,7 +31,7 @@ const update_docs = async(model_name, data) => {
 
     const op = Object.entries(mongo_doc)
       .reduce((new_doc, [key, value]) => {
-        let new_key = key !== "_id" && key.startsWith('_') ? `$_${key}` : key
+        let new_key = key !== "_id" && key.startsWith('_') ? `${UNDERSCORE_PREFIX}${key}` : key
         new_doc[new_key] = value
         return new_doc
       }, current_doc)

package/helpers/getInitials.test.js

@@ -1,22 +0,0 @@
-/* @flow */
-const getInitials = require("./getInitials")
-
-
-const inputs = [
-  ["Philippe de Reynal", "PR"],
-  ["Sarah Connor", "SC"],
-  ["Sarah O'Connor", "SC"],
-  ["Cédric O", "CO"],
-  ["Chloë Grace Moretz", "CGM"],
-  ["Ella Marija Lani Yelich-O'Connor", "EML"],
-  ["Hannah John-Kamen", "HJK"],
-  ["npm", "NPM"],
-  ["Apple", "APP"],
-]
-
-inputs.forEach((i) => {
-  test(i[0], () => {
-    const initials = getInitials(i[0])
-    expect(initials).toBe(i[1])
-  })
-})

package/rts/store/satisfies_projection.test.js

@@ -1,58 +0,0 @@
-/* @flow */
-const satisfies_projection = require("./satisfies_projection")
-
-
-let doc = {
-  _id: '1',
-  field1: 'value1',
-  field2: {
-    subField1: 'value2',
-    subField2: 'value3'
-  }
-}
-
-let projection = {
-  field1: 1,
-  'field2.subField1': 1,
-  'field2.subField2': 1
-}
-
-
-
-test("simple", () => {
-  expect(satisfies_projection(doc, projection)).toBe(true)
-})
-
-test("missing field", () => {
-  expect(satisfies_projection(doc, {
-    missing_field: 1,
-    ...projection
-  })).toBe(false)
-})
-
-test("matching and missing field", () => {
-  expect(satisfies_projection(doc, {
-    field1: true,
-    missing_field: 1,
-    ...projection
-  })).toBe(false)
-})
-
-test("missing nested field", () => {
-  expect(satisfies_projection(doc, {
-    "missing_field.missing_nested": 1,
-    ...projection
-  })).toBe(false)
-})
-
-test("empty", () => {
-  expect(satisfies_projection(doc, {field1: 1})).toBe(true)
-})
-
-test("no fields, _id only, empty projection", () => {
-  expect(satisfies_projection({_id: "1012"}, {})).toBe(true)
-})
-
-test("no fields, _id only, mismatching projection", () => {
-  expect(satisfies_projection({_id: "1012"}, {doesntexist: 1})).toBe(false)
-})