@dtudury/streamo 0.1.0

package/public/streamo/RepoRegistry.test.js

@@ -0,0 +1,87 @@
+import { describe } from './utils/testing.js'
+import { Streamo } from './Streamo.js'
+import { Repo } from './Repo.js'
+import { RepoRegistry } from './RepoRegistry.js'
+import { archiveSync } from './archiveSync.js'
+
+function archiveRegistry (dir) {
+  return new RepoRegistry(async key => {
+    const repo = new Repo()
+    await archiveSync(repo, dir, key)
+    return repo
+  })
+}
+
+describe(import.meta.url, ({ test }) => {
+  test('plain registry creates in-memory repositories with no factory', async ({ assert }) => {
+    const registry = new RepoRegistry()
+    const s = await registry.open('anykey')
+    assert.ok(s instanceof Repo)
+    s.set({ x: 1 })
+    assert.equal(s.get('x'), 1)
+  })
+
+  test('open creates a repository and returns the same instance on repeat calls', async ({ assert }) => {
+    const registry = new RepoRegistry()
+    const s1 = await registry.open('aabbcc')
+    const s2 = await registry.open('aabbcc')
+    assert.ok(s1 === s2, 'same instance returned')
+    assert.equal(registry.size, 1)
+  })
+
+  test('open creates independent repositories for different keys', async ({ assert }) => {
+    const registry = new RepoRegistry()
+    const s1 = await registry.open('key1')
+    const s2 = await registry.open('key2')
+    assert.ok(s1 !== s2)
+    assert.equal(registry.size, 2)
+    s1.set({ from: 'key1' })
+    s2.set({ from: 'key2' })
+    assert.equal(s1.get('from'), 'key1')
+    assert.equal(s2.get('from'), 'key2')
+  })
+
+  test('concurrent open() calls return the same instance', async ({ assert }) => {
+    let created = 0
+    const registry = new RepoRegistry(async () => {
+      created++
+      await new Promise(r => setTimeout(r, 10))
+      return new Repo()
+    })
+    const [s1, s2, s3] = await Promise.all([
+      registry.open('k'),
+      registry.open('k'),
+      registry.open('k')
+    ])
+    assert.equal(created, 1, 'factory called only once')
+    assert.ok(s1 === s2 && s2 === s3, 'all calls return same instance')
+  })
+
+  test('get returns undefined for unopened or still-opening keys', async ({ assert }) => {
+    const registry = new RepoRegistry()
+    assert.equal(registry.get('nope'), undefined)
+    await registry.open('exists')
+    assert.ok(registry.get('exists') instanceof Streamo)
+  })
+
+  test('iterates over fully-opened repositories only', async ({ assert }) => {
+    const registry = new RepoRegistry()
+    await registry.open('a')
+    await registry.open('b')
+    const entries = [...registry]
+    assert.equal(entries.length, 2)
+    assert.deepEqual(entries.map(([k]) => k).sort(), ['a', 'b'])
+  })
+
+  test('archive factory persists and reloads repository data', async ({ assert }) => {
+    const dir = '/tmp/repository-registry-persist-test-' + Date.now()
+    const r1 = archiveRegistry(dir)
+    const s1 = await r1.open('testkey')
+    s1.set({ saved: true })
+    await new Promise(r => setTimeout(r, 50))
+
+    const r2 = archiveRegistry(dir)
+    const s2 = await r2.open('testkey')
+    assert.equal(s2.get('saved'), true, 'data survived registry reload')
+  })
+})

package/public/streamo/Signature.js

@@ -0,0 +1,15 @@
+/**
+ * A secp256k1 signature over a range of stream bytes.
+ * `address` is the first byte of the signed range.
+ * `compactRawBytes` is the 64-byte compact signature.
+ */
+export class Signature {
+  /**
+   * @param {number} address
+   * @param {Uint8Array} compactRawBytes
+   */
+  constructor (address, compactRawBytes) {
+    this.address = address
+    this.compactRawBytes = compactRawBytes
+  }
+}

package/public/streamo/Signer.js

@@ -0,0 +1,91 @@
+import { getPublicKey, signAsync, verify } from './utils/noble-secp256k1.js'
+
+const cryptoSubtle = typeof crypto !== 'undefined' ? crypto.subtle : (await import('crypto')).webcrypto.subtle
+
+/**
+ * Derive a deterministic private key from (name, password) using PBKDF2.
+ * @param {string} name
+ * @param {string} password
+ * @param {number} [iterations=100000]
+ * @returns {Promise.<string>} hex-encoded key
+ */
+async function deriveKey (name, password, iterations = 100000) {
+  const enc = new TextEncoder()
+  const base = await cryptoSubtle.importKey('raw', enc.encode(password), 'PBKDF2', false, ['deriveBits', 'deriveKey'])
+  const key = await cryptoSubtle.deriveKey(
+    { name: 'PBKDF2', salt: enc.encode(name), iterations, hash: 'SHA-256' },
+    base,
+    { name: 'HMAC', hash: 'SHA-256' },
+    true,
+    ['sign']
+  )
+  const raw = new Uint8Array(await cryptoSubtle.exportKey('raw', key))
+  return Array.from(raw.slice(32)).map(b => b.toString(16).padStart(2, '0')).join('')
+}
+
+async function sha256 (uint8Array) {
+  return new Uint8Array(await cryptoSubtle.digest('SHA-256', uint8Array))
+}
+
+/**
+ * Signs stream content using secp256k1.
+ * Each stream name gets its own deterministic key pair derived from
+ * the user's username and password.
+ */
+export class Signer {
+  #keysByName = {}
+  #hashwordPromise
+  #iterations
+
+  /**
+   * @param {string} username
+   * @param {string} password
+   * @param {number} [iterations=100000]
+   */
+  constructor (username, password, iterations = 100000) {
+    this.username = username
+    this.#iterations = iterations
+    this.#hashwordPromise = deriveKey(username, password, iterations)
+  }
+
+  /**
+   * @param {string} streamName
+   * @returns {Promise.<{privateKey: string, publicKey: Uint8Array}>}
+   */
+  async keysFor (streamName) {
+    if (!this.#keysByName[streamName]) {
+      const hashword = await this.#hashwordPromise
+      const privateKey = await deriveKey(streamName, hashword, this.#iterations)
+      const publicKey = getPublicKey(privateKey)
+      this.#keysByName[streamName] = { privateKey, publicKey }
+    }
+    return this.#keysByName[streamName]
+  }
+
+  /**
+   * @param {string} streamName
+   * @param {Uint8Array} bytes
+   * @returns {Promise.<Uint8Array>} 64-byte compact signature
+   */
+  async sign (streamName, bytes) {
+    const { privateKey } = await this.keysFor(streamName)
+    const hash = await sha256(bytes)
+    const sig = await signAsync(hash, privateKey)
+    return sig.toCompactRawBytes()
+  }
+}
+
+/**
+ * @param {Uint8Array} publicKey
+ * @param {Uint8Array} bytes
+ * @param {Uint8Array} compactRawBytes
+ * @returns {Promise.<boolean>}
+ */
+export async function verifySignature (publicKey, bytes, compactRawBytes) {
+  try {
+    const hash = await sha256(bytes)
+    return verify(compactRawBytes, hash, publicKey)
+  } catch {
+    return false
+  }
+}

package/public/streamo/Streamo.js

@@ -0,0 +1,392 @@
+import { Recaller } from './utils/Recaller.js'
+import { CodecRegistry } from './CodecRegistry.js'
+import { Signature } from './Signature.js'
+import { verifySignature } from './Signer.js'
+
+/**
+ * Thrown by conditionalSet() when the streamo has advanced past the expected tip.
+ * Catch this to detect write conflicts and retry with a fresh read.
+ */
+export class ConflictError extends Error {
+  /**
+   * @param {number} expectedTip  byteLength the caller observed
+   * @param {number} actualTip    byteLength at the moment of the attempted write
+   */
+  constructor (expectedTip, actualTip) {
+    super(`conflict: expected tip ${expectedTip} but streamo is at ${actualTip}`)
+    this.name = 'ConflictError'
+    this.expectedTip = expectedTip
+    this.actualTip = actualTip
+  }
+}
+
+/**
+ * Yield every path where addrA and addrB differ, including the root.
+ * Compares by address so unchanged subtrees are skipped in O(1).
+ */
+export function * changedPaths (streamo, addrA, addrB, path = []) {
+  if (addrA === addrB) return
+  yield path
+  const refsA = addrA !== undefined ? streamo.decode(addrA, true) : undefined
+  const refsB = addrB !== undefined ? streamo.decode(addrB, true) : undefined
+  const isPlain = v => v != null && typeof v === 'object' && (Array.isArray(v) || Object.getPrototypeOf(v) === Object.prototype || Object.getPrototypeOf(v) === null)
+  const objA = isPlain(refsA)
+  const objB = isPlain(refsB)
+  if (objA || objB) {
+    const keys = new Set([...Object.keys(refsA ?? {}), ...Object.keys(refsB ?? {})])
+    for (const key of keys) {
+      const a = objA ? refsA[key] : undefined
+      const b = objB ? refsB[key] : undefined
+      if (a !== b) yield * changedPaths(streamo, a, b, [...path, key])
+    }
+  }
+}
+
+/**
+ * A Streamo is a reactive, signed, append-only data store.
+ *
+ * It combines:
+ *   - CodecRegistry: encode/decode any JS value to/from bytes
+ *   - Recaller: fine-grained reactive dependency tracking (watch/get/set)
+ *   - secp256k1 signing: sign the streamo contents, verify signatures
+ *
+ * This is the primary user-facing class. The layers below it
+ * (Addressifier, CodecRegistry) exist to serve it.
+ */
+export class Streamo extends CodecRegistry {
+  #recaller
+  #signedLength = 0
+
+  /**
+   * @param {Recaller} [recaller]
+   */
+  constructor (recaller = new Recaller('Streamo')) {
+    super()
+    this.#recaller = recaller
+  }
+
+  get recaller () { return this.#recaller }
+
+  get byteLength () {
+    this.#recaller.reportKeyAccess(this, 'length')
+    return super.byteLength
+  }
+
+  /**
+   * Append code and notify reactive watchers.
+   * @param {Uint8Array} code
+   * @returns {number}
+   */
+  append (code) {
+    const address = super.append(code)
+    this.#recaller.reportKeyMutation(this, 'length')
+    return address
+  }
+
+  /**
+   * Decode the value at a path within the most-recently-appended value,
+   * registering reactive dependencies so watchers re-run on changes.
+   *
+   * If the first argument is a number it is treated as an explicit address
+   * (no dependency registered). Otherwise byteLength is accessed (dependency
+   * registered) and all arguments are treated as a path into the decoded value.
+   *
+   * @param {...(number|string)} args
+   * @returns {any}
+   */
+  get (...args) {
+    let address
+    if (typeof args[0] === 'number') {
+      address = args.shift()
+    } else {
+      address = this.valueAddress
+      // 'length': re-run when external bytes arrive (append() fires 'length').
+      // path string: re-run when set() mutates this specific path via changedPaths.
+      this.#recaller.reportKeyAccess(this, 'length')
+      this.#recaller.reportKeyAccess(this, JSON.stringify(args))
+    }
+    if (address < 0) return undefined
+    let value = this.decode(address)
+    for (const key of args) {
+      if (value == null) return undefined
+      value = value[key]
+    }
+    return value
+  }
+
+  /**
+   * Encode and append a new value, optionally updating at a path within the
+   * current top-level decoded value. Notifies reactive watchers of which paths
+   * changed.
+   *
+   * Signature: set([address,] ...path, value)
+   * - If first arg is a number, use it as the base address.
+   * - The last argument is always the value to set.
+   * - Intermediate arguments are the path to update.
+   *
+   * @param {...(number|string|any)} args
+   * @returns {number} address of the newly appended code
+   */
+  set (...args) {
+    const baseAddress = typeof args[0] === 'number' ? args.shift() : this.valueAddress
+    const value = args.pop()
+    const path = args
+
+    const prevAddress = super.byteLength > 0 ? this.valueAddress : undefined
+
+    if (path.length === 0 || baseAddress < 0) {
+      // Whole-value set: encode and store, bypassing Streamo.append so 'length'
+      // is not fired — changedPaths will emit the right path-level mutations.
+      let encodedValue = value
+      if (path.length > 0) {
+        // Empty streamo with a path: build nested object from path
+        let obj = value
+        for (let i = path.length - 1; i >= 0; i--) obj = { [path[i]]: obj }
+        encodedValue = obj
+      }
+      super.append(this.encode(encodedValue))
+    } else {
+      // Path update: navigate via asRefs to avoid decoding untouched subtrees,
+      // then rebuild only the changed path bottom-up, reusing sibling addresses.
+      const levels = []
+      let addr = baseAddress
+      for (let i = 0; i < path.length - 1; i++) {
+        const refs = this.asRefs(addr)
+        levels.push({ refs, key: path[i] })
+        addr = Array.isArray(refs) ? refs[+path[i]] : refs[path[i]]
+      }
+      levels.push({ refs: this.asRefs(addr), key: path[path.length - 1] })
+
+      // Encode the new leaf value
+      const leafCode = this.encode(value)
+      let childAddr = this.addressOf(leafCode) ?? super.append(leafCode)
+
+      // Rebuild from leaf to root, reusing unchanged siblings by address
+      for (let i = levels.length - 1; i >= 0; i--) {
+        const { refs, key } = levels[i]
+        const newRefs = Array.isArray(refs) ? [...refs] : { ...refs }
+        newRefs[Array.isArray(refs) ? +key : key] = childAddr
+        const code = this.encode(newRefs, true)
+        childAddr = this.addressOf(code) ?? super.append(code)
+      }
+    }
+
+    const newAddress = super.byteLength - 1
+    for (const changed of changedPaths(this, prevAddress, newAddress)) {
+      this.#recaller.reportKeyMutation(this, JSON.stringify(changed))
+    }
+    return newAddress
+  }
+
+  /**
+   * Navigate a path and return refs (addresses instead of decoded values).
+   * With no path, returns root refs. Returns a plain number if the target is
+   * a leaf (non-object/array), or undefined if the path doesn't exist.
+   *
+   * @param {...string} path
+   * @returns {Object|number|undefined}
+   */
+  getRefs (...path) {
+    let address = this.valueAddress
+    if (address < 0) return undefined
+    for (const key of path) {
+      const refs = this.asRefs(address)
+      if (typeof refs === 'number') return undefined
+      address = Array.isArray(refs) ? refs[+key] : refs[key]
+      if (address === undefined) return undefined
+    }
+    return this.asRefs(address)
+  }
+
+  /**
+   * Like set(), but the last argument is an address (number) rather than a
+   * decoded value. Rebuilds only the changed path bottom-up, reusing sibling
+   * addresses — same as set() but skips the leaf-encoding step.
+   *
+   * Requires at least one path key and an existing object at that path.
+   *
+   * @param {...(string|number)} args  ...path, address
+   * @returns {number} address of the newly appended code
+   */
+  setRefs (...args) {
+    let childAddr = args.pop()
+    const path = args
+    const baseAddress = this.valueAddress
+    const prevAddress = super.byteLength > 0 ? this.valueAddress : undefined
+
+    const levels = []
+    let addr = baseAddress
+    for (let i = 0; i < path.length - 1; i++) {
+      const refs = this.asRefs(addr)
+      levels.push({ refs, key: path[i] })
+      addr = Array.isArray(refs) ? refs[+path[i]] : refs[path[i]]
+    }
+    levels.push({ refs: this.asRefs(addr), key: path[path.length - 1] })
+
+    for (let i = levels.length - 1; i >= 0; i--) {
+      const { refs, key } = levels[i]
+      const newRefs = Array.isArray(refs) ? [...refs] : { ...refs }
+      newRefs[Array.isArray(refs) ? +key : key] = childAddr
+      const code = this.encode(newRefs, true)
+      childAddr = this.addressOf(code) ?? super.append(code)
+    }
+
+    const newAddress = super.byteLength - 1
+    for (const changed of changedPaths(this, prevAddress, newAddress)) {
+      this.#recaller.reportKeyMutation(this, JSON.stringify(changed))
+    }
+    return newAddress
+  }
+
+  /**
+   * Call f immediately, tracking get() calls. Re-runs f whenever a
+   * subsequent set() touches a path that was accessed.
+   * @param {string} name
+   * @param {function} f
+   */
+  watch (name, f) {
+    this.#recaller.watch(name, f)
+  }
+
+  /**
+   * Stop watching a function that was previously passed to watch().
+   * @param {function} f
+   */
+  unwatch (f) {
+    this.#recaller.unwatch(f)
+  }
+
+  /**
+   * Like set(), but only succeeds if the streamo's current byteLength equals
+   * `expectedTip` — i.e., nothing has been written since the caller last read.
+   *
+   * Throws ConflictError when the precondition fails. Callers should catch it,
+   * re-read the latest state, re-apply their change, and retry.
+   *
+   * @param {number} expectedTip  byteLength observed when the change was prepared
+   * @param {...(string|any)} args  same arguments as set()
+   * @returns {number} address of the newly appended code
+   */
+  conditionalSet (expectedTip, ...args) {
+    const actual = super.byteLength
+    if (actual !== expectedTip) throw new ConflictError(expectedTip, actual)
+    return this.set(...args)
+  }
+
+  /**
+   * Snapshot this streamo up to (and including) `address`.
+   * The returned Streamo shares no mutable state with the original.
+   * @param {number} address
+   * @param {Recaller} [recaller]
+   * @returns {Streamo}
+   */
+  clone (address, recaller = this.#recaller) {
+    return this._applyClone(new Streamo(recaller), address)
+  }
+
+  // ── Signing ──────────────────────────────────────────────────────────────
+
+  /**
+   * Address of the most-recently-appended non-signature chunk.
+   * After streamo.sign() appends a SIGNATURE chunk, byteLength - 1 points to the
+   * signature rather than the user data. This getter skips backward past any
+   * trailing SIGNATURE chunks so get() and set() always operate on real data.
+   */
+  get valueAddress () {
+    let address = super.byteLength - 1
+    while (address >= 0) {
+      const code = this.resolve(address)
+      if (this.footerToCodec[code.at(-1)]?.type !== 'SIGNATURE') break
+      address -= code.length
+    }
+    return address
+  }
+
+  /** Byte length that has been covered by a signature. */
+  get signedLength () { return this.#signedLength }
+
+  /** @override Also resets the signed-length cursor. */
+  _reset () {
+    super._reset()
+    this.#signedLength = 0
+  }
+
+  /**
+   * Sign the bytes appended since the last signature (or from the start).
+   * Appends the signature as a new chunk and advances the signed cursor.
+   *
+   * @param {import('./Signer.js').Signer} signer
+   * @param {string} streamoName
+   * @returns {Promise.<Signature>}
+   */
+  async sign (signer, streamoName) {
+    const before = super.byteLength
+    const bytes = this.slice(this.#signedLength, before - 1)
+    const compactRawBytes = await signer.sign(streamoName, bytes)
+    if (super.byteLength !== before) throw new Error('streamo was modified while signing')
+    const sig = new Signature(this.#signedLength, compactRawBytes)
+    this.append(this.encode(sig))
+    this.#signedLength = before
+    return sig
+  }
+
+  /**
+   * Verify a signature against this streamo's contents.
+   * @param {Signature} sig
+   * @param {Uint8Array} publicKey
+   * @returns {Promise.<boolean>}
+   */
+  async verify (sig, publicKey) {
+    const sigCode = this.encode(sig)
+    const sigAddress = this.addressOf(sigCode)
+    const bytes = this.slice(sig.address, sigAddress - sigCode.length)
+    return verifySignature(publicKey, bytes, sig.compactRawBytes)
+  }
+
+  /**
+   * Like makeWritableStream(), but verifies every SIGNATURE chunk against
+   * `publicKey` before accepting it. Non-signature chunks are appended as
+   * normal; the entire write is rejected (WritableStream errors) if any
+   * signature fails to verify.
+   *
+   * Use this when receiving data from an untrusted source (a peer, a file
+   * written by someone else) to ensure every signed range is authentic.
+   *
+   * @param {Uint8Array} publicKey
+   * @param {number} [maxFrameSize]
+   * @returns {WritableStream}
+   */
+  makeVerifiedWritableStream (publicKey, maxFrameSize = 64 * 1024 * 1024) {
+    const self = this
+    let buf = new Uint8Array(0)
+    return new WritableStream({
+      async write (incoming) {
+        const next = new Uint8Array(buf.length + incoming.length)
+        next.set(buf); next.set(incoming, buf.length)
+        buf = next
+        while (buf.length >= 4) {
+          const len = new Uint32Array(buf.slice(0, 4).buffer)[0]
+          if (len === 0) throw new Error('malformed frame: zero-length chunk')
+          if (len > maxFrameSize) throw new Error(`malformed frame: length ${len} exceeds ${maxFrameSize}`)
+          if (buf.length < 4 + len) break
+          const code = buf.slice(4, 4 + len)
+          buf = buf.slice(4 + len)
+
+          if (self.addressOf(code) !== undefined) continue // already present, skip
+
+          // If this is a SIGNATURE chunk, verify it covers the bytes since its
+          // stated start address before we accept it into the store.
+          const codec = self.footerToCodec[code.at(-1)]
+          if (codec?.type === 'SIGNATURE') {
+            const sig = self.decode(code)
+            const bytes = self.slice(sig.address, self.byteLength - 1)
+            const valid = await verifySignature(publicKey, bytes, sig.compactRawBytes)
+            if (!valid) throw new Error('signature verification failed')
+          }
+
+          self.append(code)
+        }
+      }
+    })
+  }
+}