@mblaney/holster 0.1.0

package/src/wire.js

@@ -0,0 +1,290 @@
+import Dup from "./dup.js"
+import Get from "./get.js"
+import Ham from "./ham.js"
+import Store from "./store.js"
+import * as utils from "./utils.js"
+
+const isNode = typeof document === "undefined"
+
+const wsModule = isNode ? await import("ws") : undefined
+
+if (typeof globalThis.WebSocket === "undefined") {
+  globalThis.WebSocket = wsModule?.WebSocket
+}
+
+// Wire starts a websocket client or server and returns get and put methods
+// for access to the wire spec and storage.
+const Wire = opt => {
+  if (!utils.obj.is(opt)) opt = {}
+
+  const dup = Dup(opt.maxAge)
+  const store = Store(opt)
+  const graph = {}
+  const queue = {}
+  const listen = {}
+
+  // The check function is required because user data must provide a public key
+  // so that it can be verified. The public key might verify the provided
+  // signature but not actually match the user under which the data is being
+  // stored. To avoid this, the current data on a soul needs to be checked to
+  // make sure the stored public key matches the one provided with the update.
+  const check = async (data, send, cb) => {
+    for (const soul of Object.keys(data)) {
+      const msg = await new Promise(res => {
+        getWithCallback({"#": soul}, res, send)
+      })
+      if (msg.err) {
+        if (cb) cb(msg.err)
+        return false
+      }
+
+      const node = data[soul]
+      const pub = node._ ? node._.p : undefined
+      const key = utils.userPublicKey
+      // If there is no current node then the data is ok to write without
+      // matching public keys, as the provided soul also needs a rel on the
+      // parent node which then also requires checking. Otherwise publc keys
+      // need to match for existing data.
+      if (!msg.put || !msg.put[soul] || msg.put[soul][key] === pub) {
+        continue
+      }
+
+      // If a soul exists but does not have a public key, then one should not be
+      // added because the node is not user data. The above check fails in this
+      // case if a public key is provided. Note that this is only an error case
+      // if called via the API, which is when a callback is provided here.
+      // (The wire spec can fetch and put data on the wire without a signature
+      // or public key and this can be ignored.)
+      if (cb) {
+        cb(`error in wire check public key does not match for soul: ${soul}`)
+      }
+      return false
+    }
+
+    return true
+  }
+
+  const get = (msg, send) => {
+    const ack = Get(msg.get, graph)
+    if (ack) {
+      send(
+        JSON.stringify({
+          "#": dup.track(utils.text.random(9)),
+          "@": msg["#"],
+          put: ack,
+        }),
+      )
+    } else {
+      store.get(msg.get, (err, ack) => {
+        send(
+          JSON.stringify({
+            "#": dup.track(utils.text.random(9)),
+            "@": msg["#"],
+            put: ack,
+            err: err,
+          }),
+        )
+      })
+    }
+  }
+
+  const put = async (msg, send) => {
+    // Store updates returned from Ham.mix and defer updates if required.
+    const update = await Ham.mix(msg.put, graph, opt.secure, listen)
+    if (Object.keys(update.now).length) {
+      if (!(await check(update.now, send))) return
+
+      store.put(update.now, err => {
+        send(
+          JSON.stringify({
+            "#": dup.track(utils.text.random(9)),
+            "@": msg["#"],
+            err: err,
+          }),
+        )
+      })
+    }
+    if (Object.keys(update.defer).length) {
+      setTimeout(() => put({put: update.defer}, send), update.wait)
+    }
+  }
+
+  const getWithCallback = (lex, cb, send, opt) => {
+    if (!cb) return
+
+    if (!utils.obj.is(opt)) opt = {}
+    const ack = Get(lex, graph)
+    if (ack) {
+      cb({put: ack})
+      return
+    }
+
+    store.get(lex, (err, ack) => {
+      if (ack) {
+        cb({put: ack, err: err})
+        return
+      }
+
+      if (err) console.log(err)
+
+      const track = utils.text.random(9)
+      queue[track] = cb
+      send(
+        JSON.stringify({
+          "#": dup.track(track),
+          get: lex,
+        }),
+      )
+      // Respond to callback with null if no response.
+      setTimeout(() => {
+        const cb = queue[track]
+        if (cb) {
+          const id = lex["#"]
+          const ack = {[id]: null}
+          if (typeof lex["."] === "string") ack[id] = {[lex["."]]: null}
+          cb({put: ack})
+          delete queue[track]
+        }
+      }, opt.wait || 100)
+    })
+  }
+
+  const api = send => {
+    return {
+      get: (lex, cb, opt) => {
+        getWithCallback(lex, cb, send, opt)
+      },
+      put: async (data, cb) => {
+        // Deferred updates are only stored using wire spec, they're ignored
+        // here using the api. This is ok because correct timestamps should be
+        // used whereas wire spec needs to handle clock skew for updates
+        // across the network.
+        const update = await Ham.mix(data, graph, opt.secure, listen)
+        const none = Object.keys(update.now).length === 0
+        if (none || !(await check(update.now, send, cb))) return
+
+        store.put(update.now, cb)
+        // Also put data on the wire spec.
+        // TODO: Note that this means all clients now receive all updates, so
+        // need to filter what should be stored, both in graph and on disk.
+        send(
+          JSON.stringify({
+            "#": dup.track(utils.text.random(9)),
+            put: data,
+          }),
+        )
+      },
+      on: (lex, cb) => {
+        const soul = lex && lex["#"]
+        if (!soul || !cb) return
+
+        if (listen[soul]) {
+          listen[soul].push({".": lex["."], cb: cb})
+        } else {
+          listen[soul] = [{".": lex["."], cb: cb}]
+        }
+      },
+      off: (lex, cb) => {
+        const soul = lex && lex["#"]
+        if (!soul || !listen[soul]) return
+
+        if (cb) {
+          const found = listen[soul].find(l => l.cb === cb)
+          if (found) {
+            listen[soul].splice(listen[soul].indexOf(found), 1)
+          }
+        } else {
+          // Remove all callbacks when none provided.
+          delete listen[soul]
+        }
+      },
+    }
+  }
+
+  if (isNode) {
+    let wss = opt.wss
+    // Node's websocket server provides clients as an array, whereas
+    // mock-sockets provides clients as a function that returns an array.
+    let clients = () => wss.clients()
+    if (!wss) {
+      wss = new wsModule.WebSocketServer({port: 8080})
+      clients = () => wss.clients
+    }
+
+    const send = (data, isBinary) => {
+      clients().forEach(client => {
+        if (client.readyState === WebSocket.OPEN) {
+          client.send(data, {binary: isBinary})
+        }
+      })
+    }
+    wss.on("connection", ws => {
+      ws.on("error", console.error)
+
+      ws.on("message", (data, isBinary) => {
+        const msg = JSON.parse(data)
+        if (dup.check(msg["#"])) return
+
+        dup.track(msg["#"])
+        if (msg.get) get(msg, send)
+        if (msg.put) put(msg, send)
+        send(data, isBinary)
+
+        const id = msg["@"]
+        const cb = queue[id]
+        if (cb) {
+          delete msg["#"]
+          delete msg["@"]
+          cb(msg)
+
+          delete queue[id]
+        }
+      })
+    })
+    return api(send)
+  }
+
+  let ws = new WebSocket("ws://localhost:8080")
+  const send = data => {
+    if (!ws || ws.readyState !== WebSocket.OPEN) {
+      console.log("websocket not available")
+      return
+    }
+
+    ws.send(data)
+  }
+  const start = () => {
+    if (!ws) ws = new WebSocket("ws://localhost:8080")
+    ws.onclose = c => {
+      ws = null
+      setTimeout(start, Math.floor(Math.random() * 5000))
+    }
+    ws.onerror = e => {
+      console.error(e)
+    }
+    ws.onmessage = m => {
+      const msg = JSON.parse(m.data)
+      if (dup.check(msg["#"])) return
+
+      dup.track(msg["#"])
+      if (msg.get) get(msg, send)
+      if (msg.put) put(msg, send)
+      send(m.data)
+
+      const id = msg["@"]
+      const cb = queue[id]
+      if (cb) {
+        delete msg["#"]
+        delete msg["@"]
+        cb(msg)
+
+        delete queue[id]
+      }
+    }
+  }
+
+  start()
+  return api(send)
+}
+
+export default Wire

package/test/holster.get.test.js

@@ -0,0 +1,60 @@
+import fs from "fs"
+import {Server} from "mock-socket"
+import {describe, test} from "node:test"
+import assert from "node:assert/strict"
+import Holster from "../src/holster.js"
+
+describe("holster.get", () => {
+  const wss = new Server("ws://localhost:1234")
+  const holster = Holster({file: "test/holster.get", wss: wss, maxAge: 100})
+
+  test("empty key callback null", (t, done) => {
+    holster.get("", data => {
+      assert.equal(data, null)
+      done()
+    })
+  })
+
+  test("null key callback null", (t, done) => {
+    holster.get(null, data => {
+      assert.equal(data, null)
+      done()
+    })
+  })
+
+  test("underscore as key callback null", (t, done) => {
+    holster.get("_", data => {
+      assert.equal(data, null)
+      done()
+    })
+  })
+
+  test("get unknown key callback null", (t, done) => {
+    holster.get("unknown", data => {
+      assert.equal(data, null)
+      done()
+    })
+  })
+
+  // Chaining get creates a new context for each call.
+  test("get chained unknown keys callback null", (t, done) => {
+    holster.get("chained").get("unknown", data => {
+      assert.equal(data, null)
+      done()
+    })
+  })
+
+  test("next chained unknown keys callback null", (t, done) => {
+    holster.get("chained").next("unknown", data => {
+      assert.equal(data, null)
+      done()
+    })
+  })
+
+  test("cleanup", (t, done) => {
+    fs.rm("test/holster.get", {recursive: true, force: true}, err => {
+      assert.equal(err, null)
+      done()
+    })
+  })
+})

package/test/holster.lex.test.js

@@ -0,0 +1,196 @@
+import fs from "fs"
+import {Server} from "mock-socket"
+import {describe, test} from "node:test"
+import assert from "node:assert/strict"
+import Holster from "../src/holster.js"
+
+describe("holster.lex", () => {
+  const wss = new Server("ws://localhost:1234")
+  const holster = Holster({file: "test/holster.lex", wss: wss, maxAge: 100})
+
+  test("object on root in graph format", (t, done) => {
+    const plain = {
+      key: "plain value",
+      true: true,
+      false: false,
+      number: 42,
+    }
+    holster.get("plain").put(plain, err => {
+      assert.equal(err, null)
+
+      // Prefix tests. (Need to be sequential to match return values.)
+      holster.get("plain", {".": {"*": "k"}}, data => {
+        assert.deepEqual(data, {key: "plain value"})
+
+        holster.get("plain", {".": {"*": "t"}}, data => {
+          assert.deepEqual(data, {true: true})
+
+          holster.get("plain", {".": {"*": "f"}}, data => {
+            assert.deepEqual(data, {false: false})
+
+            holster.get("plain", {".": {"*": "n"}}, data => {
+              assert.deepEqual(data, {number: 42})
+            })
+
+            // Both less than and greater than.
+            holster.get("plain", {".": {"<": "n", ">": "falsy"}}, data => {
+              assert.deepEqual(data, {key: "plain value"})
+
+              // Only less than.
+              holster.get("plain", {".": {"<": "k"}}, data => {
+                assert.deepEqual(data, {false: false})
+
+                // Only greater than.
+                holster.get("plain", {".": {">": "numbers"}}, data => {
+                  assert.deepEqual(data, {true: true})
+                  done()
+                })
+              })
+            })
+          })
+        })
+      })
+    })
+  })
+
+  test("put and get nested object", (t, done) => {
+    const nested = {
+      key: "nested value",
+      other: "other value",
+      child: {
+        has: "child value",
+        other: "other child value",
+      },
+    }
+    holster.get("nested").put(nested, err => {
+      assert.equal(err, null)
+
+      // Getting a nested object requires waiting for radisk to write to disk,
+      // as it will batch the writes. (Default wait is 1 millisecond.)
+      setTimeout(() => {
+        holster.get("nested", {".": {"*": "k"}}, data => {
+          assert.deepEqual(data, {key: "nested value"})
+
+          holster.get("nested").next("child", {".": {"*": "h"}}, data => {
+            assert.deepEqual(data, {has: "child value"})
+
+            holster.get("nested").next("child", {".": {"<": "has"}}, data => {
+              // Less than means less than or equal to in lex.
+              assert.deepEqual(data, {has: "child value"})
+              done()
+            })
+          })
+        })
+      }, 2)
+    })
+  })
+
+  test("chained get before put nested object", (t, done) => {
+    const nested = {
+      key: "hello nested value",
+      other: "other value",
+      child: {
+        has: "hello child value",
+        other: "other child value",
+      },
+    }
+    holster
+      .get("hello")
+      .next("nested")
+      .put(nested, err => {
+        assert.equal(err, null)
+
+        setTimeout(() => {
+          holster.get("hello").next("nested", {".": {"*": "k"}}, data => {
+            assert.deepEqual(data, {key: "hello nested value"})
+
+            holster
+              .get("hello")
+              .next("nested")
+              .next("child", {".": {"*": "h"}}, data => {
+                assert.deepEqual(data, {has: "hello child value"})
+
+                holster
+                  .get("hello")
+                  .next("nested")
+                  .next("child", {".": {">": "other"}}, data => {
+                    // Greater than means greater than or equal to in lex.
+                    assert.deepEqual(data, {other: "other child value"})
+                    done()
+                  })
+              })
+          })
+        }, 2)
+      })
+  })
+
+  test("put and get two nested objects", (t, done) => {
+    const two = {
+      key: "two nested values",
+      child1: {
+        has: "child value 1",
+      },
+      child2: {
+        has: "child value 2",
+      },
+    }
+    holster.get("two").put(two, err => {
+      assert.equal(err, null)
+
+      setTimeout(() => {
+        holster.get("two", {".": {"*": "child"}}, data => {
+          assert.deepEqual(data, {
+            child1: two.child1,
+            child2: two.child2,
+          })
+
+          holster
+            .get("two")
+            .next("child1", {".": {">": "g", "<": "i"}}, data => {
+              assert.deepEqual(data, {has: "child value 1"})
+              done()
+            })
+        })
+      }, 2)
+    })
+  })
+
+  test("on and put nested object", (t, done) => {
+    const nested = {
+      key: "nested key",
+      child: {
+        has: "child key",
+      },
+    }
+    // The node needs to exist before it can be listend to for updates.
+    holster.get("on-nested").put(nested, err => {
+      assert.equal(err, null)
+
+      const update = {
+        child: {
+          has: "child update",
+        },
+      }
+      holster
+        .get("on-nested")
+        .next("child")
+        .on({".": {">": "h"}}, data => {
+          assert.deepEqual(data, update.child)
+          done()
+        })
+
+      setTimeout(() => {
+        holster.get("on-nested").put(update, err => {
+          assert.equal(err, null)
+        })
+      }, 10)
+    })
+  })
+
+  test("cleanup", (t, done) => {
+    fs.rm("test/holster.lex", {recursive: true, force: true}, err => {
+      assert.equal(err, null)
+      done()
+    })
+  })
+})

package/test/holster.off.test.js

@@ -0,0 +1,53 @@
+import fs from "fs"
+import {Server} from "mock-socket"
+import {describe, test} from "node:test"
+import assert from "node:assert/strict"
+import Holster from "../src/holster.js"
+
+describe("holster.off", () => {
+  const wss = new Server("ws://localhost:1234")
+  const holster = Holster({file: "test/holster.off", wss: wss, maxAge: 100})
+
+  test("calling off without get callback null", (t, done) => {
+    holster.off(data => {
+      assert.equal(data, null)
+      done()
+    })
+  })
+
+  test("off with cb for property on root then update - no event", (t, done) => {
+    const cb = data => {
+      console.log("should not be called for key:", data)
+      done()
+    }
+    holster.get("key").on(cb)
+    holster.get("key").off(cb)
+
+    holster.get("key").put("value", err => {
+      assert.equal(err, null)
+      done()
+    })
+  })
+
+  test("off no cb for property on root then update - no event", (t, done) => {
+    holster.get("key1").on(data => {
+      console.log("should not be called for key1:", data)
+      done()
+    })
+    setTimeout(() => {
+      holster.get("key1").off()
+
+      holster.get("key1").put("value1", err => {
+        assert.equal(err, null)
+        done()
+      })
+    }, 10)
+  })
+
+  test("cleanup", (t, done) => {
+    fs.rm("test/holster.off", {recursive: true, force: true}, err => {
+      assert.equal(err, null)
+      done()
+    })
+  })
+})