@mblaney/holster 0.1.0

package/examples/user.html

@@ -0,0 +1,31 @@
+<!doctype html>
+<html>
+  <head>
+    <meta charset="utf-8" />
+    <title>Holster.user - test examples</title>
+  </head>
+  <body>
+    <h2>Holster.user - test examples</h2>
+    <p>
+      This file loads <code>user.js</code> which contains examples that test the
+      Holster.user API. See the console log for test output.
+    </p>
+    <p>
+      The tests create two users, Alice and Bob. They both write data to their
+      own user space. Bob writes secret data that can only be read by Alice,
+      then Alice logs in again to read the message.
+    </p>
+    <p>
+      These tests also show how to use the SEA API (Security, Encryption, and
+      Authorization), which can be loaded on it's own, but it made available via
+      holster.SEA. The holster.user API uses the SEA API internally for creating
+      and authenticating users.
+    </p>
+    <p>
+      The <code>user.js</code> file is loaded as a module, which allows it to
+      import the development build of Holster. You can switch to the production
+      build by changing the import path at the top of the file.
+    </p>
+    <script type="module" src="./user.js"></script>
+  </body>
+</html>

package/examples/user.js

@@ -0,0 +1,88 @@
+// Switch src to build for the production version.
+import Holster from "../src/holster.js"
+
+const holster = Holster({indexedDB: true})
+globalThis.holster = holster
+
+const user = holster.user()
+
+let alice, bob
+// Wait for the websocket to connect.
+setTimeout(() => {
+  user.create("alice", "password", err => {
+    if (err !== "Username already exists") {
+      console.log(err)
+      return
+    }
+
+    user.auth("alice", "password", err => {
+      if (err) {
+        console.log(err)
+        return
+      }
+
+      // Store Alice's encryption key so Bob can use it to create shared data.
+      alice = {epub: user.is.epub}
+      console.log("Alice logged in.", user.is)
+
+      const data = "Success! Only Alice can put data on their account"
+      user.get("protected").put(data, err => {
+        if (err) {
+          console.log(err)
+          return
+        }
+
+        user.get("protected", console.log)
+      })
+    })
+  })
+}, 1000)
+
+setTimeout(() => {
+  user.create("bob", "stronger password", err => {
+    if (err !== "Username already exists") {
+      console.log(err)
+      return
+    }
+
+    user.auth("bob", "stronger password", async err => {
+      if (err) {
+        console.log(err)
+        return
+      }
+
+      // Store Bob's public keys so Alice can use them to read shared data.
+      bob = {pub: user.is.pub, epub: user.is.epub}
+      console.log("Bob logged in.", user.is)
+
+      // For this test let's create shared private data for Alice.
+      const secret = await holster.SEA.secret(alice, user.is)
+      const shared = await holster.SEA.encrypt("Hello Alice!", secret)
+      user.get("shared").put(shared, err => {
+        if (err) {
+          console.log(err)
+          return
+        }
+
+        console.log("Success! Bob wrote secret data to his account.")
+      })
+    })
+  })
+}, 2000)
+
+setTimeout(() => {
+  // Use Bob's public key to fetch shared data under his account.
+  holster.user(bob.pub).get("shared", enc => {
+    user.auth("alice", "password", async err => {
+      if (err) {
+        console.log(err)
+        return
+      }
+
+      console.log("Alice logged in again.", user.is)
+      const secret = await holster.SEA.secret(bob, user.is)
+      const shared = await holster.SEA.decrypt(enc, secret)
+      console.log("Alice's shared data from Bob:", shared)
+    })
+  })
+}, 3000)

package/examples/wire.html

@@ -0,0 +1,27 @@
+<!doctype html>
+<html>
+  <head>
+    <meta charset="utf-8" />
+    <title>Holster.wire - test examples</title>
+  </head>
+  <body>
+    <h2>Holster.wire - test examples</h2>
+    <p>
+      This file loads <code>wire.js</code> which contains examples that use the
+      Holster.wire API. These examples show how conflict resolution works, see
+      the console log for test output.
+    </p>
+    <p>
+      These tests also make two updates in random order, but the wire API can
+      set state vectors on updates to control which value should be applied.
+      This means when the "species" field is logged it will always be the same
+      no matter which order the tests run in.
+    </p>
+    <p>
+      The <code>wire.js</code> file is loaded as a module, which allows it to
+      import the development build of Holster. You can switch to the production
+      build by changing the import path at the top of the file.
+    </p>
+    <script type="module" src="./wire.js"></script>
+  </body>
+</html>

package/examples/wire.js

@@ -0,0 +1,55 @@
+// Switch src to build for the production version.
+import Holster from "../src/holster.js"
+
+// Set readFromDisk to true to force reading from disk.
+// (Restart the server so that graph is not in memory.)
+const readFromDisk = false
+
+const holster = Holster({indexedDB: !readFromDisk})
+globalThis.holster = holster
+
+if (readFromDisk) {
+  // Wait for the websocket to connect.
+  setTimeout(() => {
+    holster.wire.get({"#": "FDSA", ".": "species"}, msg => {
+      console.log("read from disk:", msg)
+    })
+  }, 1000)
+} else {
+  // Create two updates that could happen in either order.
+  setTimeout(() => {
+    const update1 = {
+      ASDF: {
+        _: {"#": "ASDF", ">": {name: 2, boss: 2}},
+        name: "Mark Nadal",
+        boss: {"#": "FDSA"},
+      },
+      FDSA: {
+        _: {"#": "FDSA", ">": {name: 2, species: 2, slave: 2}},
+        name: "Fluffy",
+        species: "a kitty",
+        slave: {"#": "ASDF"},
+      },
+    }
+    holster.wire.put(update1, msg => console.log("update1:", msg))
+  }, 1000 * Math.random())
+
+  setTimeout(() => {
+    const update2 = {
+      ASDF: {_: {"#": "ASDF", ">": {name: 1}}, name: "Mark"},
+      FDSA: {
+        _: {"#": "FDSA", ">": {species: 2, color: 3}},
+        species: "felis silvestris",
+        color: "ginger",
+      },
+    }
+    holster.wire.put(update2, msg => console.log("update2:", msg))
+  }, 1000 * Math.random())
+
+  // They should always produce the same result no matter which order.
+  setTimeout(() => {
+    holster.wire.get({"#": "FDSA", ".": "species"}, msg => {
+      console.log("get:", msg)
+    })
+  }, 2000)
+}

package/package.json

@@ -0,0 +1,44 @@
+{
+  "name": "@mblaney/holster",
+  "version": "0.1.0",
+  "description": "Holster is a service that allows synchronising data between devices using Node, Deno, Bun or the browser.",
+  "type": "module",
+  "main": "index.js",
+  "scripts": {
+    "test": "node --test",
+    "build": "esbuild src/holster.js --bundle --format=esm --sourcemap \"--external:node:*\" \"--external:ws\" --minify --outdir=build"
+  },
+  "keywords": [],
+  "author": "Malcolm Blaney",
+  "license": "Zlib OR MIT OR Apache-2.0",
+  "dependencies": {
+    "express": "^5.1.0",
+    "fs": "^0.0.1-security",
+    "ws": "^8.18.0"
+  },
+  "devDependencies": {
+    "esbuild": "0.25.3",
+    "mock-socket": "^9.3.1",
+    "prettier": "^3.4.2"
+  },
+  "prettier": {
+    "semi": false,
+    "bracketSpacing": false,
+    "arrowParens": "avoid"
+  },
+  "browser": {
+    "fs": false
+  },
+  "directories": {
+    "example": "examples",
+    "test": "test"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/mblaney/holster.git"
+  },
+  "bugs": {
+    "url": "https://github.com/mblaney/holster/issues"
+  },
+  "homepage": "https://github.com/mblaney/holster/wiki"
+}

package/src/array.js

@@ -0,0 +1,34 @@
+if (typeof btoa === "undefined") {
+  globalThis.btoa = data => Buffer.from(data, "binary").toString("base64")
+  globalThis.atob = data => Buffer.from(data, "base64").toString("binary")
+}
+
+// This is Array extended to have .toString(["utf8"|"hex"|"base64"])
+function SeaArray() {}
+
+Object.assign(SeaArray, {from: Array.from})
+SeaArray.prototype = Object.create(Array.prototype)
+SeaArray.prototype.toString = function (enc, start, end) {
+  if (!enc) enc = "utf8"
+  if (!start) start = 0
+
+  const length = this.length
+  if (enc === "hex") {
+    const buf = new Uint8Array(this)
+    return [...Array(((end && end + 1) || length) - start).keys()]
+      .map(i => buf[i + start].toString(16).padStart(2, "0"))
+      .join("")
+  }
+
+  if (enc === "utf8") {
+    return Array.from({length: (end || length) - start}, (_, i) =>
+      String.fromCharCode(this[i + start]),
+    ).join("")
+  }
+
+  if (enc === "base64") {
+    return btoa(this)
+  }
+}
+
+export default SeaArray

package/src/buffer.js

@@ -0,0 +1,83 @@
+import SeaArray from "./array.js"
+
+// This is Buffer implementation used in SEA. Functionality is mostly
+// compatible with NodeJS "safe-buffer" and is used for encoding conversions
+// between binary and "hex" | "utf8" | "base64"
+// See documentation and validation for safe implementation in:
+// https://github.com/feross/safe-buffer#update
+function SafeBuffer(...props) {
+  console.warn("new SafeBuffer() is deprecated, please use SafeBuffer.from()")
+  return SafeBuffer.from(...props)
+}
+
+SafeBuffer.prototype = Object.create(Array.prototype)
+Object.assign(SafeBuffer, {
+  // (data, enc) where typeof data === "string" then
+  // enc === "utf8"|"hex"|"base64"
+  from() {
+    if (!Object.keys(arguments).length || arguments[0] == null) {
+      throw new TypeError(
+        "First argument must be a string, Buffer, ArrayBuffer, Array, or array-like object.",
+      )
+    }
+    const input = arguments[0]
+    let buf
+    if (typeof input === "string") {
+      const enc = arguments[1] || "utf8"
+      if (enc === "hex") {
+        const bytes = input
+          .match(/([\da-fA-F]{2})/g)
+          .map(byte => parseInt(byte, 16))
+        if (!bytes || !bytes.length) {
+          throw new TypeError("Invalid first argument for type 'hex'.")
+        }
+        buf = SeaArray.from(bytes)
+      } else if (enc === "utf8" || enc === "binary") {
+        const length = input.length
+        const words = new Uint16Array(length)
+        Array.from({length: length}, (_, i) => (words[i] = input.charCodeAt(i)))
+        buf = SeaArray.from(words)
+      } else if (enc === "base64") {
+        const dec = atob(input)
+        const length = dec.length
+        const bytes = new Uint8Array(length)
+        Array.from({length: length}, (_, i) => (bytes[i] = dec.charCodeAt(i)))
+        buf = SeaArray.from(bytes)
+      } else {
+        console.info("SafeBuffer.from unknown encoding: " + enc)
+      }
+      return buf
+    }
+    const length = input.byteLength ? input.byteLength : input.length
+    if (length) {
+      let buf
+      if (input instanceof ArrayBuffer) {
+        buf = new Uint8Array(input)
+      }
+      const check = SeaArray.from(buf || input)
+      return check
+    }
+  },
+  // This is "safe-buffer.alloc" sans encoding support
+  alloc(length, fill = 0) {
+    return SeaArray.from(
+      new Uint8Array(Array.from({length: length}, () => fill)),
+    )
+  },
+  // This puts together array of array like members
+  concat(arr) {
+    // octet array
+    if (!Array.isArray(arr)) {
+      throw new TypeError(
+        "First argument must be Array containing ArrayBuffer or Uint8Array instances.",
+      )
+    }
+    return SeaArray.from(
+      arr.reduce((ret, item) => ret.concat(Array.from(item)), []),
+    )
+  },
+})
+SafeBuffer.prototype.from = SafeBuffer.from
+SafeBuffer.prototype.toString = SeaArray.prototype.toString
+
+export default SafeBuffer

package/src/dup.js

@@ -0,0 +1,23 @@
+const Dup = maxAge => {
+  // Allow maxAge to be passed in as tests wait on the setTimeout.
+  if (!maxAge) maxAge = 9000
+  const dup = {store: {}}
+  dup.check = id => (dup.store[id] ? dup.track(id) : false)
+  dup.track = id => {
+    // Keep the liveliness of the message up while it is being received.
+    dup.store[id] = Date.now()
+    if (!dup.expiry) {
+      dup.expiry = setTimeout(() => {
+        const now = Date.now()
+        Object.keys(dup.store).forEach(id => {
+          if (now - dup.store[id] > maxAge) delete dup.store[id]
+        })
+        dup.expiry = null
+      }, maxAge)
+    }
+    return id
+  }
+  return dup
+}
+
+export default Dup

package/src/get.js

@@ -0,0 +1,19 @@
+const Get = (lex, graph) => {
+  const soul = lex["#"]
+  const key = typeof lex["."] === "string" ? lex["."] : false
+  var node = graph[soul]
+
+  // Can only return a node if a key is provided, because the graph may not
+  // have all the keys populated for a given soul. This is because Ham.mix
+  // only adds incoming changes to the graph.
+  if (!node || !key) return
+
+  let value = node[key]
+  if (!value) return
+
+  node = {_: node._, [key]: value}
+  node._[">"] = {[key]: node._[">"][key]}
+  return {[soul]: node}
+}
+
+export default Get

package/src/ham.js

@@ -0,0 +1,151 @@
+import * as utils from "./utils.js"
+import SEA from "./sea.js"
+
+// state and value are the incoming changes.
+// currentState and currentValue are the current graph data.
+const Ham = (state, currentState, value, currentValue) => {
+  if (state < currentState) return {historical: true}
+
+  if (state > currentState) return {incoming: true}
+
+  // state is equal to currentState, lexically compare to resolve conflict.
+  if (typeof value !== "string") {
+    value = JSON.stringify(value) || ""
+  }
+  if (typeof currentValue !== "string") {
+    currentValue = JSON.stringify(currentValue) || ""
+  }
+  // No update required.
+  if (value === currentValue) return {state: true}
+
+  // Keep the current value.
+  if (value < currentValue) return {current: true}
+
+  // Otherwise update using the incoming value.
+  return {incoming: true}
+}
+
+Ham.mix = async (change, graph, secure, listen) => {
+  var machine = Date.now()
+  var now = {}
+  var defer = {}
+  let wait = 0
+
+  for (const soul of Object.keys(change)) {
+    const node = change[soul]
+    let updated = false
+    let alias = false
+    let nodeWait = 0
+    let pub = ""
+    let verify = secure
+
+    if (!node || !node._) continue
+
+    // If a signature and public key are provided then always verify.
+    if (node._.s && node._.p) verify = true
+
+    // Special case if soul starts with "~". Node must be system data ie,
+    // ~@alias or ~publickey. For aliases, key and value must be a self
+    // identifying rel. For public keys, data needs to be signed and verified.
+    // (This is also true for any data when the secure flag is used.)
+    if (soul.length > 1 && soul[0] === "~") {
+      if (soul[1] === "@") {
+        alias = true
+        verify = false
+      } else {
+        if (node._.p && soul != "~" + node._.p) {
+          console.log(`error public key does not match for soul: ${soul}`)
+          continue
+        }
+
+        // Need to be able to create public key data before a user is
+        // authenticated, in this case the soul is the public key which
+        // should've been used to sign the data.
+        node._.p = soul.slice(1)
+        verify = true
+      }
+    }
+    if (verify) {
+      if (!node._.s || !node._.p) {
+        console.log("error signature and public key required to verify data")
+        continue
+      }
+
+      if (!(await SEA.verify({m: node, s: node._.s}, {pub: node._.p}))) {
+        console.log(`error could not verify soul: ${soul}`)
+        continue
+      }
+    }
+
+    for (const key of Object.keys(node)) {
+      if (key === "_") continue
+
+      const value = node[key]
+      const state = node._[">"][key]
+      const currentValue = (graph[soul] || {})[key]
+      const currentState = (graph[soul] || {_: {">": {}}})._[">"][key] || 0
+
+      if (alias && key !== utils.rel.is(value)) continue
+
+      // Defer the update if ahead of machine time.
+      const skew = state - machine
+      if (skew > 0) {
+        // Ignore update if ahead by more than 24 hours.
+        if (skew > 86400000) continue
+
+        // Wait the shortest difference before trying the updates again.
+        if (wait === 0 || skew < wait) wait = nodeWait = skew
+        if (!defer[soul]) {
+          defer[soul] = {_: {"#": soul, ">": {}, s: node._.s, p: node._.p}}
+        }
+        defer[soul][key] = value
+        defer[soul]._[">"][key] = state
+      } else {
+        const result = Ham(state, currentState, value, currentValue)
+        if (result.incoming) {
+          if (!now[soul]) {
+            now[soul] = {_: {"#": soul, ">": {}, s: node._.s, p: node._.p}}
+          }
+          // TODO: graph should not just grow indefintitely in memory.
+          // Need to have a max size after which start dropping the oldest state
+          // Do something similar to Dup which can handle deletes?
+          if (!graph[soul]) {
+            graph[soul] = {_: {"#": soul, ">": {}, s: node._.s, p: node._.p}}
+          }
+          graph[soul][key] = now[soul][key] = value
+          graph[soul]._[">"][key] = now[soul]._[">"][key] = state
+          // Call event listeners for update on key, mix is called before
+          // put has finished so wait for what could be multiple nested
+          // updates on a node.
+          if (listen[soul]) {
+            setTimeout(() => {
+              if (listen[soul]) {
+                listen[soul]
+                  .filter(l => utils.match(l["."], key))
+                  .forEach(l => l.cb())
+              }
+            }, 100)
+          }
+          updated = true
+        }
+      }
+    }
+
+    if (verify && nodeWait !== 0 && now[soul]) {
+      // Secure updates can't be split, so move now to deferred as well.
+      Object.assign(defer[soul], now[soul])
+      delete now[soul]
+    }
+    // Call event listeners for update on soul.
+    if (updated && listen[soul]) {
+      setTimeout(() => {
+        if (listen[soul]) {
+          listen[soul].filter(l => utils.match(l["."])).forEach(l => l.cb())
+        }
+      }, 100)
+    }
+  }
+  return {now: now, defer: defer, wait: wait}
+}
+
+export default Ham