@dtudury/streamo 0.1.0

package/public/streamo/CodecRegistry.js

@@ -0,0 +1,195 @@
+import { Addressifier } from './Addressifier.js'
+import { makeCodecs } from './codecs.js'
+
+/**
+ * Extends Addressifier with a codec system.
+ *
+ * Each stored value is a Uint8Array whose last byte (the footer) identifies
+ * its codec. Footers are assigned sequentially as codecs are registered.
+ * Multi-part codecs multiply their option counts to produce a footer range
+ * (mixed-radix offset from baseFooter).
+ *
+ * Negative addresses encode single-byte primitive values without appending:
+ *   address = -(footer + 1)   →   footer = -address - 1
+ * So UNDEFINED, NULL, FALSE, TRUE, and every UINT7 value are addressable
+ * without touching the store.
+ *
+ * Codecs are built by makeCodecs() in codecs.js and wired in here.
+ */
+export class CodecRegistry extends Addressifier {
+  /** @type {Array} footer → codec */
+  footerToCodec = []
+
+  #codecs
+
+  constructor () {
+    super()
+    const self = this
+    this.#codecs = makeCodecs({
+      encode: (v, asRefs) => self.encode(v, asRefs),
+      decode: (code, asRefs) => self.decode(code, asRefs),
+      append: code => self.#appendSubcode(code),
+      resolve: addr => self.resolve(addr),
+      addressOf: code => self.addressOf(code),
+      get byteLength () { return self.byteLength },
+      footerToCodec: this.footerToCodec
+    })
+    this.#registerAll()
+  }
+
+  // Re-expose byteLength so subclasses can override it
+  get byteLength () { return super.byteLength }
+
+  /**
+   * Resolve an address to its Uint8Array.
+   * Negative addresses map to single-byte codes: -(footer+1) → [footer].
+   * @param {number} address
+   * @returns {Uint8Array}
+   */
+  resolve (address) {
+    if (address < 0) return new Uint8Array([-address - 1])
+    return super.resolve(address)
+  }
+
+  /**
+   * Decode any JS value from a code (Uint8Array) or address (number).
+   * @param {Uint8Array|number} codeOrAddress
+   * @param {boolean|boolean[]} [asRefs=false]
+   * @returns {any}
+   */
+  decode (codeOrAddress, asRefs = false) {
+    const code = typeof codeOrAddress === 'number'
+      ? this.resolve(codeOrAddress)
+      : codeOrAddress
+    if (!(code instanceof Uint8Array)) throw new Error('expected Uint8Array')
+    const codec = this.footerToCodec[code.at(-1)]
+    return codec.decode(code, asRefs)
+  }
+
+  /**
+   * Encode a JS value to a Uint8Array.
+   *
+   * When `asRefs` is truthy and `value` is a number it is treated as an
+   * address and resolved directly — this is the inverse of asRefs(), letting
+   * callers round-trip through asRefs → encode without deserialising subtrees.
+   *
+   * @param {any} value
+   * @param {boolean|boolean[]|string} [asRefs]
+   * @returns {Uint8Array}
+   */
+  encode (value, asRefs) {
+    if (asRefs && typeof value === 'number') return this.resolve(value)
+    for (const name in this.#codecs) {
+      const codec = this.#codecs[name]
+      const code = codec.encode?.(value, asRefs)
+      if (code) return code
+    }
+    throw new Error(`no codec for value: ${value}`)
+  }
+
+  /**
+   * Encode a value as a VARIABLE (boxed address) so that changing the
+   * top-level value is representable as a new append.
+   * @param {any} value
+   * @returns {Uint8Array}
+   */
+  encodeVariable (value) {
+    return this.#codecs.VARIABLE._encode(this.encode(value))
+  }
+
+  /**
+   * Return the immediate children of the value at `address` as addresses
+   * rather than decoded values:
+   *   Object → { key: valueAddress }  (names stay as strings)
+   *   Array  → [ addr0, addr1, … ]
+   *   Other  → address itself
+   *
+   * Useful for structural comparison without fully deserialising large trees.
+   *
+   * @param {number} address
+   * @returns {Object|Array|number}
+   */
+  asRefs (address) {
+    const code = this.resolve(address)
+    const { type } = this.footerToCodec[code.at(-1)]
+    if (type === 'VARIABLE' ||
+        type === 'OBJECT' || type === 'EMPTY_OBJECT' ||
+        type === 'ARRAY'  || type === 'EMPTY_ARRAY') {
+      return this.decode(address, true)
+    }
+    return address
+  }
+
+  /**
+   * Copy a value from another CodecRegistry into this one by address.
+   * Uses asRefs to traverse structure level-by-level, avoiding full JS
+   * deserialization of composite values. Negative addresses (single-byte
+   * primitives) are universal and returned as-is.
+   *
+   * @param {CodecRegistry} source
+   * @param {number} address
+   * @returns {number} address of the value in this registry
+   */
+  copyFrom (source, address) {
+    if (address < 0) return address // universal: same footer in any registry
+    const value = source.decode(address)
+    const newCode = this.encode(value)
+    return this.addressOf(newCode) ?? this.append(newCode)
+  }
+
+  /**
+   * Appends a compound code by first splitting it into constituent subcodes
+   * (back-to-front, footer-determined widths) and appending each independently.
+   * Returns the address of the outermost subcode.
+   * @param {Uint8Array} code
+   * @returns {number}
+   */
+  append (code) {
+    const subcodes = []
+    let rest = code
+    while (rest.length) {
+      const codec = this.footerToCodec[rest.at(-1)]
+      const width = codec.getWidth(rest)
+      subcodes.unshift(rest.subarray(-width))
+      rest = rest.subarray(0, -width)
+    }
+    let last = -1
+    for (const sub of subcodes) {
+      const existing = this.addressOf(sub)
+      last = existing !== undefined ? existing : super.append(sub)
+    }
+    return last
+  }
+
+  // Internal append used by codecs (single chunk, no splitting)
+  #appendSubcode (code) {
+    if (this.addressOf(code) !== undefined) return this.addressOf(code)
+    return super.append(code)
+  }
+
+  #registerAll () {
+    for (const name in this.#codecs) {
+      const codec = this.#codecs[name]
+      codec.type = name
+      codec.baseFooter = this.footerToCodec.length
+      if (!codec.getWidth) {
+        codec.getWidth = code => {
+          const footer = code.at(-1)
+          const c = this.footerToCodec[footer]
+          if (!c?.partReaders?.length) return 1
+          let option = footer - c.baseFooter
+          let total = 1
+          for (let i = c.partReaders.length - 1; i >= 0; i--) {
+            const opts = c.partReaders[i]
+            const part = opts[option % opts.length](code.subarray(0, -total))
+            total += part.width
+            option = Math.floor(option / opts.length)
+          }
+          return total
+        }
+      }
+      const options = (codec.partReaders ?? []).reduce((n, opts) => n * opts.length, 1)
+      for (let i = 0; i < options; i++) this.footerToCodec.push(codec)
+    }
+  }
+}

package/public/streamo/ContentMap.js

@@ -0,0 +1,79 @@
+/**
+ * A content-addressable trie mapping Uint8Array → address (number).
+ * Two Uint8Arrays with identical bytes will always resolve to the same entry.
+ * Lookup and insert are O(n) in the number of matching prefix bits, not O(n²).
+ */
+export class ContentMap {
+  #offset
+  #code
+  #address
+  #branches = []
+
+  constructor (offset = 0, code, address = -1) {
+    this.#offset = offset
+    this.#code = code
+    this.#address = address
+  }
+
+  /**
+   * Returns the address stored for this code, or undefined if not found.
+   * @param {Uint8Array} code
+   * @returns {number|undefined}
+   */
+  get (code) {
+    if (this.#code === undefined || this.#address === -1) return undefined
+    const { match, matchingBits } = this.#compare(code)
+    if (match) return this.#address
+    return this.#branches[matchingBits]?.get(code)
+  }
+
+  /**
+   * Store a code → address mapping.
+   * @param {Uint8Array} code
+   * @param {number} address
+   */
+  set (code, address) {
+    if (this.#code === undefined || this.#address === -1) {
+      this.#code = code
+      this.#address = address
+      return
+    }
+    const { match, matchingBits, matchingBytes } = this.#compare(code)
+    if (match) throw new Error('code already exists in ContentMap')
+    if (this.#branches[matchingBits]) return this.#branches[matchingBits].set(code, address)
+    this.#branches[matchingBits] = new ContentMap(matchingBytes, code, address)
+  }
+
+  /**
+   * Clone the map, including only entries with address ≤ maxAddress.
+   * @param {number} maxAddress
+   * @returns {ContentMap}
+   */
+  clone (maxAddress) {
+    if (this.#address > maxAddress) throw new Error('clone address is before branch')
+    const copy = new ContentMap(this.#offset, this.#code, this.#address)
+    for (const i in this.#branches) {
+      const branch = this.#branches[i]
+      if (branch.#address <= maxAddress) copy.#branches[i] = branch.clone(maxAddress)
+    }
+    return copy
+  }
+
+  #compare (code) {
+    const matchingBits = countMatchingBits(code.subarray(this.#offset), this.#code.subarray(this.#offset))
+    const matchingBytes = this.#offset + Math.floor(matchingBits / 8)
+    const match = matchingBytes === code.length && matchingBytes === this.#code.length
+    return { match, matchingBits, matchingBytes }
+  }
+}
+
+function countMatchingBits (a, b) {
+  let bits = 0
+  for (let i = 0; i < a.length && i < b.length; i++) {
+    let j = 0
+    while ((a[i] >> j) !== (b[i] >> j)) j++
+    bits += 8 - j
+    if (j) break
+  }
+  return bits
+}

package/public/streamo/DESIGN.md

@@ -0,0 +1,61 @@
+# Stream — Design Notes (rebuild-1)
+
+## The goal
+
+A personal signed stream of thoughts and information. After signing in, you make
+changes to your data and anyone subscribed to your stream gets live updates. Viewers
+see the same thing you see but with non-interactive controls. The stream is
+cryptographically yours — signed with your key, verifiable by anyone.
+
+## What the previous implementation got right
+
+- **Append-only, content-addressable storage.** Same value always lands at the same
+  address. This makes diffing trivial (compare addresses), deduplication free, and
+  sync simple (just send new bytes).
+
+- **Negative addresses for primitives.** `undefined`, `null`, `false`, `true`, and
+  small integers (UINT7) are fully described by a single footer byte. Using
+  `-(footer + 1)` as their address means every value is addressable without
+  appending to the store. This arrived late in the previous version; it belongs
+  at the foundation.
+
+- **Footer-based self-describing codec.** The last byte of any code identifies its
+  type. Multi-part values pack which storage option was chosen for each part into
+  the footer as a mixed-radix offset. Compact and self-contained.
+
+- **Recaller.** Fine-grained reactive dependency tracking with path-level
+  granularity. Worth keeping essentially as-is.
+
+- **The class hierarchy.** Addressifier → codec layer → reactive layer → signed
+  layer is a clean separation of concerns.
+
+## What I'd do differently
+
+- **Hide Duple.** The balanced binary tree encoding of arrays and objects is an
+  implementation detail. Exposing `Duple` in the public API (users can encode and
+  decode `Duple` instances directly) leaks the internal representation. I'll keep
+  the binary tree structure but make it invisible outside the codec layer.
+
+- **Codecs as separate objects, not one monolithic class.** The current
+  `TurtleCodecRegistry` inlines all codecs as private class fields. I'd rather
+  have each codec be a small, named, independently readable object registered into
+  a registry. The codec for dates shouldn't be physically entangled with the codec
+  for signatures.
+
+- **The address space as a first-class concept.** Rather than `getCode` being a
+  method that happens to handle negative addresses, I'd make the address space
+  explicit: a thin object that knows how to resolve any address (negative or
+  positive) to bytes.
+
+- **Stream as the primary concept, not storage.** The previous version built upward
+  from bytes. This version builds downward from the goal: a Stream is the thing,
+  and the layers beneath it exist to serve it.
+
+## Build order
+
+1. Addressifier — append-only byte store with content addressing
+2. Codecs — encode/decode for all value types, with the address space baked in
+3. Recaller — reactive dependency tracking
+4. Stream — reactive + signed, the primary user-facing class
+5. Sync — WebSocket-based append-only replication
+6. Rendering — hx template engine

package/public/streamo/Repo.js

@@ -0,0 +1,176 @@
+import { Recaller } from './utils/Recaller.js'
+import { Streamo, changedPaths } from './Streamo.js'
+
+/**
+ * A Streamo whose values are commit records.
+ *
+ * Every write goes through a commit: checkout() → set() → commit(). This makes
+ * every connected device an equal author — writes are content-addressed,
+ * signed, and append-only. The server is just another peer; the keypair is the
+ * identity and the commit log is the source of truth.
+ *
+ * get() and set() are overridden to be transparent: callers use the same API
+ * as Streamo. get() reads from the last commit's dataAddress; set() creates a
+ * new commit automatically.
+ *
+ * The raw streamo (commit log) is what gets synced over WebSocket, S3, and
+ * archives. checkout() returns a working Streamo at any commit's dataAddress
+ * for read-only inspection or direct use with the explicit commit() API.
+ */
+export class Repo extends Streamo {
+  /**
+   * The latest commit record, or null if nothing has been committed yet.
+   * Registers a reactive dependency on the commit log length.
+   * @returns {{ message: string, date: Date, dataAddress: number, parent: number|undefined }|null}
+   */
+  get lastCommit () {
+    this.recaller.reportKeyAccess(this, 'length')
+    // Use super.valueAddress (Streamo impl) to bypass our get() override and
+    // avoid a circular dependency: our get() calls lastCommit, lastCommit
+    // must not call our get().
+    const address = super.valueAddress
+    if (address < 0) return null
+    const value = this.decode(address)
+    if (!value || typeof value.message !== 'string' || !(value.date instanceof Date)) return null
+    return value
+  }
+
+  /**
+   * Decode the value at a path, reading from the last commit's dataAddress.
+   * Falls back to Streamo.get() if no commits exist yet.
+   *
+   * Registers reactive dependencies so watchers re-run when new commits land.
+   *
+   * @param {...(number|string)} args
+   * @returns {any}
+   */
+  get (...args) {
+    if (typeof args[0] === 'number') return super.get(...args)
+    const commit = this.lastCommit  // registers 'length' dependency
+    if (!commit) return super.get(...args)
+    this.recaller.reportKeyAccess(this, JSON.stringify(args))
+    if (args.length === 0) return this.decode(commit.dataAddress)
+    let value = this.decode(commit.dataAddress)
+    for (const key of args) {
+      if (value == null) return undefined
+      value = value[key]
+    }
+    return value
+  }
+
+  /**
+   * Write a value by creating a new commit: checkout → set → commit.
+   *
+   * Signature: set([address,] ...path, value)  — same as Streamo.set().
+   * Path-level reactive mutations are fired after commit so watchers only
+   * watching specific paths get precise notifications.
+   *
+   * @param {...(number|string|any)} args
+   * @returns {number} address of the new commit record
+   */
+  set (...args) {
+    if (typeof args[0] === 'number') return super.set(...args)
+    const prevDataAddress = this.lastCommit?.dataAddress
+    const working = this.checkout()
+    working.set(...args)
+    const result = this.commit(working)
+    const newDataAddress = this.lastCommit?.dataAddress
+    for (const changed of changedPaths(this, prevDataAddress, newDataAddress)) {
+      this.recaller.reportKeyMutation(this, JSON.stringify(changed))
+    }
+    return result
+  }
+
+  /**
+   * Like Streamo.getRefs() but reads from the last commit's dataAddress.
+   *
+   * @param {...string} path
+   * @returns {Object|number|undefined}
+   */
+  getRefs (...path) {
+    const commit = this.lastCommit
+    if (!commit) return super.getRefs(...path)
+    let address = commit.dataAddress
+    for (const key of path) {
+      const refs = this.asRefs(address)
+      if (typeof refs === 'number') return undefined
+      address = Array.isArray(refs) ? refs[+key] : refs[key]
+      if (address === undefined) return undefined
+    }
+    return this.asRefs(address)
+  }
+
+  /**
+   * Like Streamo.setRefs() but auto-commits via checkout → setRefs → commit.
+   *
+   * @param {...(string|number)} args  ...path, address
+   * @returns {number} address of the new commit record
+   */
+  setRefs (...args) {
+    const prevDataAddress = this.lastCommit?.dataAddress
+    const working = this.checkout()
+    working.setRefs(...args)
+    const result = this.commit(working)
+    const newDataAddress = this.lastCommit?.dataAddress
+    for (const changed of changedPaths(this, prevDataAddress, newDataAddress)) {
+      this.recaller.reportKeyMutation(this, JSON.stringify(changed))
+    }
+    return result
+  }
+
+  /**
+   * Clone the repository at the last commit's data address.
+   * The returned Streamo's get() immediately returns the last committed value.
+   * Returns an empty Streamo if nothing has been committed yet.
+   * @returns {Streamo}
+   */
+  checkout () {
+    const commit = this.lastCommit
+    if (!commit) return new Streamo()
+    return this.clone(commit.dataAddress, new Recaller('checkout'))
+  }
+
+  /**
+   * The committed data from the last commit, decoded.
+   * Returns undefined if nothing has been committed yet.
+   * @returns {any}
+   */
+  get files () {
+    const commit = this.lastCommit
+    if (!commit) return undefined
+    return this.decode(commit.dataAddress)
+  }
+
+  /**
+   * Iterate commits from newest to oldest.
+   * @yields {{ message: string, date: Date, dataAddress: number, parent: number|undefined }}
+   */
+  * history () {
+    let commit = this.lastCommit
+    while (commit) {
+      yield commit
+      commit = commit.parent !== undefined ? this.decode(commit.parent) : null
+    }
+  }
+
+  /**
+   * Copy the current value of workingStreamo into the repository and append a
+   * commit record referencing it by address.
+   *
+   * Uses super.valueAddress (skipping any trailing signatures) to find the
+   * correct parent commit address rather than byteLength - 1, which could
+   * point to a signature chunk when sign-in auto-signs after each commit.
+   *
+   * @param {Streamo} workingStreamo
+   * @param {string} [message='']
+   * @returns {number} address of the new commit record
+   */
+  commit (workingStreamo, message = '') {
+    if (workingStreamo.byteLength === 0) throw new Error('nothing to commit')
+    const parentAddr = super.valueAddress
+    const parent = parentAddr >= 0 ? parentAddr : undefined
+    const dataAddress = this.copyFrom(workingStreamo, workingStreamo.byteLength - 1)
+    const code = this.encode({ message, date: new Date(), dataAddress, parent })
+    return this.append(code)
+  }
+}

package/public/streamo/Repo.test.js

@@ -0,0 +1,82 @@
+import { describe } from './utils/testing.js'
+import { Repo } from './Repo.js'
+
+describe(import.meta.url, ({ test }) => {
+  test('commit stores message, date, and a reference to the data', ({ assert }) => {
+    const repo = new Repo()
+    const working = repo.checkout()
+    working.set({ a: 1 })
+    const commitAddr = repo.commit(working, 'first commit')
+    const commit = repo.decode(commitAddr)
+    assert.equal(commit.message, 'first commit')
+    assert.ok(commit.date instanceof Date)
+    assert.equal(typeof commit.dataAddress, 'number')
+    assert.deepEqual(repo.decode(commit.dataAddress), { a: 1 })
+  })
+
+  test('checkout starts with last committed value', ({ assert }) => {
+    const repo = new Repo()
+    const working = repo.checkout()
+    working.set({ a: 1 })
+    repo.commit(working, 'first')
+    const working2 = repo.checkout()
+    assert.deepEqual(working2.get(), { a: 1 })
+  })
+
+  test('checkout of empty repo returns empty stream', ({ assert }) => {
+    const repo = new Repo()
+    const working = repo.checkout()
+    assert.equal(working.byteLength, 0)
+  })
+
+  test('working stream modifications do not affect the repository', ({ assert }) => {
+    const repo = new Repo()
+    const working = repo.checkout()
+    working.set({ v: 1 })
+    repo.commit(working, 'first')
+    const working2 = repo.checkout()
+    working2.set({ v: 99 })
+    // repo still has v:1 as last committed value
+    assert.deepEqual(repo.decode(repo.lastCommit.dataAddress), { v: 1 })
+  })
+
+  test('multiple commits produce a linked history via parent', ({ assert }) => {
+    const repo = new Repo()
+    const working = repo.checkout()
+    working.set({ v: 1 })
+    repo.commit(working, 'first')
+    working.set('v', 2)
+    repo.commit(working, 'second')
+    const c2 = repo.lastCommit
+    assert.equal(c2.message, 'second')
+    assert.deepEqual(repo.decode(c2.dataAddress), { v: 2 })
+    const c1 = repo.decode(c2.parent)
+    assert.equal(c1.message, 'first')
+    assert.deepEqual(repo.decode(c1.dataAddress), { v: 1 })
+  })
+
+  test('first commit has no parent', ({ assert }) => {
+    const repo = new Repo()
+    const working = repo.checkout()
+    working.set({ x: 1 })
+    repo.commit(working, 'root')
+    assert.equal(repo.lastCommit.parent, undefined)
+  })
+
+  test('unchanged data reuses the same address across commits', ({ assert }) => {
+    const repo = new Repo()
+    const working = repo.checkout()
+    working.set({ x: 42 })
+    repo.commit(working, 'first')
+    repo.commit(working, 'second')
+    const c2 = repo.lastCommit
+    const c1 = repo.decode(c2.parent)
+    assert.equal(c1.dataAddress, c2.dataAddress, 'same data reuses the same address')
+  })
+
+  test('throws when working stream is empty', ({ assert }) => {
+    const repo = new Repo()
+    const working = repo.checkout()
+    assert.throws(() => repo.commit(working, 'nothing here'))
+  })
+})

package/public/streamo/RepoRegistry.js

@@ -0,0 +1,91 @@
+import { Streamo } from './Streamo.js'
+import { Repo } from './Repo.js'
+
+/**
+ * Manages a collection of Repos keyed by hex-encoded public key.
+ *
+ * Accepts an optional factory function that is called whenever a new repository
+ * is opened. The factory receives the publicKeyHex and should return a
+ * (optionally async) Repo with whatever persistence or sync wired up.
+ *
+ * If no factory is provided, plain in-memory Repos are created.
+ *
+ * Examples:
+ *
+ *   // plain in-memory
+ *   new RepoRegistry()
+ *
+ *   // archive-backed
+ *   new RepoRegistry(async key => {
+ *     const repo = new Repo()
+ *     await archiveSync(repo, dataDir, key)
+ *     return repo
+ *   })
+ *
+ *   // S3-backed
+ *   new RepoRegistry(async key => {
+ *     const repo = new Repo()
+ *     await s3Sync(repo, key, s3Config)
+ *     return repo
+ *   })
+ */
+export class RepoRegistry {
+  #streams = new Map()
+  #factory
+  #openCallbacks = new Set()
+
+  /** @param {(publicKeyHex: string) => Repo | Promise<Repo>} [factory] */
+  constructor (factory = () => new Repo()) {
+    this.#factory = factory
+  }
+
+  /**
+   * Return the Repo for `publicKeyHex`, creating it via the factory if
+   * this is the first call for that key.
+   *
+   * The repository is registered immediately (before the factory resolves) so
+   * concurrent open() calls always return the same instance.
+   *
+   * @param {string} publicKeyHex
+   * @returns {Promise<Repo>}
+   */
+  async open (publicKeyHex) {
+    if (this.#streams.has(publicKeyHex)) return this.#streams.get(publicKeyHex)
+    let resolve
+    const placeholder = new Promise(r => { resolve = r })
+    this.#streams.set(publicKeyHex, placeholder)
+    const stream = await this.#factory(publicKeyHex)
+    this.#streams.set(publicKeyHex, stream)
+    resolve(stream)
+    for (const cb of this.#openCallbacks) cb(publicKeyHex, stream)
+    return stream
+  }
+
+  /** Register a callback invoked whenever a new repo is fully opened. */
+  onOpen (cb) { this.#openCallbacks.add(cb) }
+
+  /** Remove a previously registered onOpen callback. */
+  offOpen (cb) { this.#openCallbacks.delete(cb) }
+
+  /**
+   * Return an already-open Repo, or undefined if not opened yet.
+   * @param {string} publicKeyHex
+   * @returns {Repo|undefined}
+   */
+  get (publicKeyHex) {
+    const entry = this.#streams.get(publicKeyHex)
+    return entry instanceof Streamo ? entry : undefined
+  }
+
+  /** Number of currently open (or opening) repos. */
+  get size () { return this.#streams.size }
+
+  /** Iterate over [publicKeyHex, Repo] pairs (only fully-opened). */
+  [Symbol.iterator] () {
+    return (function * (map) {
+      for (const [k, v] of map) {
+        if (v instanceof Streamo) yield [k, v]
+      }
+    })(this.#streams)
+  }
+}