@dtudury/streamo 3.0.0 → 4.0.1

package/public/streamo/Addressifier.js

@@ -1,3 +1,12 @@
+/**
+ * @file Addressifier — append-only content-addressable byte store.
+ *
+ * The byte-level foundation. Knows nothing about types or values; just
+ * appends Uint8Array chunks and indexes them by content. Higher layers
+ * (CodecRegistry, Streamo, Repo, registrySync) build on this.
+ *
+ * See design.md §1.
+ */
 import { ContentMap } from './ContentMap.js'
 
 /**

package/public/streamo/CodecRegistry.js

@@ -1,3 +1,13 @@
+/**
+ * @file CodecRegistry — codec dispatcher on top of Addressifier.
+ *
+ * Resolves bytes ↔ JS values via a registered codec table; chunks identify
+ * their codec by their last byte (the footer). Public read APIs (asRefs,
+ * directReferences, decode) are mutation-impossible by construction; the
+ * write companion _asRefsForWrite is internal.
+ *
+ * See design.md §3–4.
+ */
 import { Addressifier } from './Addressifier.js'
 import { makeCodecs } from './codecs.js'
 
@@ -21,6 +31,9 @@ export class CodecRegistry extends Addressifier {
   footerToCodec = []
 
   #codecs
+  // Read-only scope depth — only mutated by #runReadOnly below, never
+  // touched directly. The codec interface exposes it as r.readOnly (boolean).
+  #readOnlyDepth = 0
 
   constructor () {
     super()
@@ -32,11 +45,27 @@ export class CodecRegistry extends Addressifier {
       resolve: addr => self.resolve(addr),
       addressOf: code => self.addressOf(code),
       get byteLength () { return self.byteLength },
+      get readOnly () { return self.#readOnlyDepth > 0 },
       footerToCodec: this.footerToCodec
     })
     this.#registerAll()
   }
 
+  /**
+   * Run `fn` in a read-only scope. Codec helpers see `r.readOnly === true`
+   * for the duration and avoid mutation paths (specifically getPartAddress
+   * in codecs.js returns undefined instead of materializing inline
+   * children). Used by asRefs.
+   *
+   * The depth counter handles the case where `fn` itself re-enters this
+   * method — outer scopes stay read-only until they all unwind.
+   */
+  #runReadOnly (fn) {
+    this.#readOnlyDepth++
+    try { return fn() }
+    finally { this.#readOnlyDepth-- }
+  }
+
   // Re-expose byteLength so subclasses can override it
   get byteLength () { return super.byteLength }
 
@@ -110,6 +139,72 @@ export class CodecRegistry extends Addressifier {
    * @returns {Object|Array|number}
    */
   asRefs (address) {
+    const code = this.resolve(address)
+    const { type } = this.footerToCodec[code.at(-1)]
+    if (type === 'VARIABLE' ||
+        type === 'OBJECT' || type === 'EMPTY_OBJECT' ||
+        type === 'ARRAY'  || type === 'EMPTY_ARRAY') {
+      // Decode in a read-only scope so codecs cannot materialize inline
+      // children. Inline addresses come back as `undefined`; callers
+      // handle that (e.g. by rendering the child without a clickable
+      // link). Mutation is unreachable from here regardless of caller —
+      // by control flow, not by caller discipline.
+      return this.#runReadOnly(() => this.decode(address, true))
+    }
+    return address
+  }
+
+  /**
+   * Direct chunk-graph references — the addresses this chunk's bytes point
+   * to (NOT the user-level child values asRefs returns). Walks the codec's
+   * parts: addressed parts contribute their target address; inline parts
+   * are skipped (their bytes are embedded in this chunk, no separate
+   * address). Pure read; never mutates.
+   *
+   * What you see varies by codec:
+   *   - DUPLE   → up to 2 references (left, right)
+   *   - OBJECT/ARRAY/VARIABLE → 1 reference (the embedded Duple-tree or
+   *                              wrapped value, when stored separately)
+   *   - STRING/UINT8ARRAY/DATE/FLOAT64 → 1 reference (the encoded bytes,
+   *                                       when stored separately)
+   *   - WORD, UINT7, EMPTY_*, primitives → none
+   *   - SIGNATURE → none (its parts are data, not chunk references)
+   *
+   * Used by the explorer's storage tab to walk the chunk graph.
+   *
+   * @param {number} address
+   * @returns {number[]}
+   */
+  directReferences (address) {
+    const code = this.resolve(address)
+    const codec = this.footerToCodec[code.at(-1)]
+    if (!codec?.partReaders?.length) return []
+
+    const refs = []
+    const footer = code.at(-1)
+    let option = footer - codec.baseFooter
+    let end = -1
+    for (let i = codec.partReaders.length - 1; i >= 0; i--) {
+      const opts = codec.partReaders[i]
+      const reader = opts[option % opts.length]
+      option = Math.floor(option / opts.length)
+      const part = reader(code.subarray(0, end))
+      end -= part.width
+      if (part.address !== undefined) refs.unshift(part.address)
+    }
+    return refs
+  }
+
+  /**
+   * Internal: like asRefs but materializes inline children when needed
+   * (write context). Used by Streamo.set / setRefs during path traversal,
+   * which DOES need real addresses to navigate composite values; and the
+   * mutation it triggers is part of the same write op anyway. Public callers
+   * should use asRefs (above), which is mutation-impossible.
+   * @param {number} address
+   * @returns {Object|Array|number}
+   */
+  _asRefsForWrite (address) {
     const code = this.resolve(address)
     const { type } = this.footerToCodec[code.at(-1)]
     if (type === 'VARIABLE' ||

package/public/streamo/Repo.js

@@ -1,3 +1,13 @@
+/**
+ * @file Repo — a Streamo whose every set() becomes a signed commit.
+ *
+ * Each commit is a record { message, date, dataAddress, parent }. The
+ * commit log is what flows over the wire during sync. attachSigner
+ * makes commits sign automatically, with concurrent commits batched
+ * into one signature.
+ *
+ * See design.md §8.
+ */
 import { Recaller } from './utils/Recaller.js'
 import { Streamo, changedPaths } from './Streamo.js'
 
@@ -23,6 +33,14 @@ export class Repo extends Streamo {
   #signing     = false
   #signPending = false
 
+  /**
+   * Default commit message attached to every commit made via set() / setRefs().
+   * Empty by default — clients opt in to set this for attribution. The chat web
+   * client sets 'web' so commits are visibly distinguishable from a CLI
+   * client's. Not enforced; explicit commit(working, msg) wins.
+   */
+  defaultMessage = ''
+
   /**
    * Attach a signer so every commit is automatically signed.
    * Concurrent commits are batched: if a sign is in flight when another
@@ -109,7 +127,7 @@ export class Repo extends Streamo {
     const prevDataAddress = this.lastCommit?.dataAddress
     const working = this.checkout()
     working.set(...args)
-    const result = this.commit(working)
+    const result = this.commit(working, this.defaultMessage)
     const newDataAddress = this.lastCommit?.dataAddress
     for (const changed of changedPaths(this, prevDataAddress, newDataAddress)) {
       this.recaller.reportKeyMutation(this, JSON.stringify(changed))
@@ -146,7 +164,7 @@ export class Repo extends Streamo {
     const prevDataAddress = this.lastCommit?.dataAddress
     const working = this.checkout()
     working.setRefs(...args)
-    const result = this.commit(working)
+    const result = this.commit(working, this.defaultMessage)
     const newDataAddress = this.lastCommit?.dataAddress
     for (const changed of changedPaths(this, prevDataAddress, newDataAddress)) {
       this.recaller.reportKeyMutation(this, JSON.stringify(changed))

package/public/streamo/Signer.js

@@ -1,3 +1,13 @@
+/**
+ * @file Signer — deterministic secp256k1 keypairs from credentials.
+ *
+ * PBKDF2-SHA256 (256 bits) is used twice — first to derive a hashword
+ * from password+username, then a private key per stream-name. No key
+ * files; same credentials always produce the same identity. KAT in
+ * Signer.test.js pins the byte output across runtime versions.
+ *
+ * See design.md §7.
+ */
 import { getPublicKey, signAsync, verify } from './utils/noble-secp256k1.js'
 
 const cryptoSubtle = typeof crypto !== 'undefined' ? crypto.subtle : (await import('crypto')).webcrypto.subtle

package/public/streamo/Streamo.js

@@ -1,3 +1,15 @@
+/**
+ * @file Streamo — reactive content-addressed signed byte store.
+ *
+ * Layers Recaller-driven path-level reactivity on top of CodecRegistry,
+ * plus a sign/verify API for secp256k1 attestations over byte ranges.
+ * `valueAddress` skips trailing SIGNATURE chunks so reading the latest
+ * value works whether or not it has been auto-signed yet.
+ *
+ * Exports: Streamo (the class), ConflictError, changedPaths.
+ *
+ * See design.md §5.
+ */
 import { Recaller } from './utils/Recaller.js'
 import { CodecRegistry } from './CodecRegistry.js'
 import { Signature } from './Signature.js'
@@ -23,12 +35,24 @@ export class ConflictError extends Error {
 /**
  * Yield every path where addrA and addrB differ, including the root.
  * Compares by address so unchanged subtrees are skipped in O(1).
+ *
+ * Uses streamo.asRefs (mutation-impossible) rather than decode(_, true)
+ * so the comparison cannot append chunks. Without this, calling
+ * changedPaths during Streamo.set could materialize inline children as
+ * separate chunks AFTER the new commit, moving valueAddress past the
+ * commit and corrupting Repo.lastCommit.
+ *
+ * Tradeoff: asRefs returns `undefined` for inline children's addresses,
+ * so changedPaths can't see differences that happen entirely inside
+ * inline-only subtrees. The parent path still fires, which is enough
+ * for any watcher that doesn't read at a depth past where the structure
+ * goes inline.
  */
 export function * changedPaths (streamo, addrA, addrB, path = []) {
   if (addrA === addrB) return
   yield path
-  const refsA = addrA !== undefined ? streamo.decode(addrA, true) : undefined
-  const refsB = addrB !== undefined ? streamo.decode(addrB, true) : undefined
+  const refsA = addrA !== undefined ? streamo.asRefs(addrA) : undefined
+  const refsB = addrB !== undefined ? streamo.asRefs(addrB) : undefined
   const isPlain = v => v != null && typeof v === 'object' && (Array.isArray(v) || Object.getPrototypeOf(v) === Object.prototype || Object.getPrototypeOf(v) === null)
   const objA = isPlain(refsA)
   const objB = isPlain(refsB)
@@ -86,6 +110,14 @@ export class Streamo extends CodecRegistry {
    */
   append (code) {
     const address = super.append(code)
+    // If the appended chunk is a SIGNATURE, advance the signed cursor — no
+    // matter whether the caller was sign(), an archive replay, or a peer
+    // stream. Otherwise a fresh load (signedLength=0) followed by a new
+    // sign() would re-sign all of history, producing a sig whose signedFrom
+    // collides with every prior sig.
+    if (this.footerToCodec[code.at(-1)]?.type === 'SIGNATURE') {
+      this.#signedLength = super.byteLength - code.length
+    }
     this.#recaller.reportKeyMutation(this, 'length')
     return address
   }
@@ -153,16 +185,19 @@ export class Streamo extends CodecRegistry {
       }
       super.append(this.encode(encodedValue))
     } else {
-      // Path update: navigate via asRefs to avoid decoding untouched subtrees,
-      // then rebuild only the changed path bottom-up, reusing sibling addresses.
+      // Path update: navigate via _asRefsForWrite to avoid decoding untouched
+      // subtrees, then rebuild only the changed path bottom-up, reusing sibling
+      // addresses. (The public asRefs is mutation-impossible; the internal
+      // _asRefsForWrite allows materializing inline children, which is
+      // appropriate here because we're inside a write op.)
       const levels = []
       let addr = baseAddress
       for (let i = 0; i < path.length - 1; i++) {
-        const refs = this.asRefs(addr)
+        const refs = this._asRefsForWrite(addr)
         levels.push({ refs, key: path[i] })
         addr = Array.isArray(refs) ? refs[+path[i]] : refs[path[i]]
       }
-      levels.push({ refs: this.asRefs(addr), key: path[path.length - 1] })
+      levels.push({ refs: this._asRefsForWrite(addr), key: path[path.length - 1] })
 
       // Encode the new leaf value
       const leafCode = this.encode(value)
@@ -224,11 +259,11 @@ export class Streamo extends CodecRegistry {
     const levels = []
     let addr = baseAddress
     for (let i = 0; i < path.length - 1; i++) {
-      const refs = this.asRefs(addr)
+      const refs = this._asRefsForWrite(addr)
       levels.push({ refs, key: path[i] })
       addr = Array.isArray(refs) ? refs[+path[i]] : refs[path[i]]
     }
-    levels.push({ refs: this.asRefs(addr), key: path[path.length - 1] })
+    levels.push({ refs: this._asRefsForWrite(addr), key: path[path.length - 1] })
 
     for (let i = levels.length - 1; i >= 0; i--) {
       const { refs, key } = levels[i]
@@ -337,7 +372,6 @@ export class Streamo extends CodecRegistry {
     if (super.byteLength !== before) throw new Error('streamo was modified while signing')
     const sig = new Signature(this.#signedLength, compactRawBytes)
     this.append(this.encode(sig))
-    this.#signedLength = before
     return sig
   }
 

package/public/streamo/chat-cli.js

@@ -42,21 +42,35 @@ console.log(`root key: ${rootKey.slice(0, 16)}…`)
 console.log('─'.repeat(40))
 
 const registry = new RepoRegistry()
+// Track who we've announced ourselves back to (deduped to prevent
+// ping-pong) — see comment in chat/main.js for the discovery pattern.
+const announcedTo = new Set()
 const session = await registrySync(registry, host, port, {
   filter: k => k === rootKey,
   follow: (keyHex, repo, subscribe) => {
-    // Auto-follow all members listed in the root repo
+    // Auto-follow all members listed in the root repo (only present when
+    // the server has chat-room onAnnounce wiring).
     for (const memberKey of repo.get('members') ?? []) subscribe(memberKey)
   },
   onAnnounce: (key) => {
-    // When someone announces directly, subscribe to their repo immediately
+    // Subscribe to the announcer AND announce ourselves back so they
+    // learn we exist. Makes chat work even when the server has no
+    // member-tracking onAnnounce of its own.
     session.subscribe(key)
+    if (!announcedTo.has(key)) {
+      announcedTo.add(key)
+      session.announce(myKey, rootKey)
+    }
   }
 })
 
-// Open my own repo, set profile if first time
+// Open my own repo and attach our signer so commits go out signed —
+// "every write is provably yours" only holds if we actually sign. Set
+// profile on first run.
 const myRepo = await registry.open(myKey)
+myRepo.attachSigner(signer, 'chat')
 if (!myRepo.get('name')) {
+  myRepo.defaultMessage = `joined as ${username} (cli)`
   myRepo.set({ name: username, messages: [] })
 }
 
@@ -110,7 +124,9 @@ rl.on('line', async line => {
   const text = line.trim()
   if (!text) { rl.prompt(); return }
   const messages = myRepo.get('messages') ?? []
-  myRepo.set({ name: username, messages: [...messages, { text, at: Date.now() }] })
+  const preview = text.length > 50 ? text.slice(0, 50).trim() + '…' : text
+  myRepo.defaultMessage = `"${preview}" (cli)`
+  myRepo.set({ name: username, messages: [...messages, { text, at: new Date() }] })
   rl.prompt()
 })
 

package/public/streamo/codecs.js

@@ -1,3 +1,14 @@
+/**
+ * @file codecs — concrete codecs for every value type Streamo can encode.
+ *
+ * Primitives (UNDEFINED, NULL, FALSE, TRUE, UINT7, FLOAT64, DATE), bytes
+ * (WORD, UINT8ARRAY, EMPTY_UINT8ARRAY), strings, composites (OBJECT,
+ * ARRAY, EMPTY_*), the SIGNATURE chunk, and the internal balanced-tree
+ * node Duple used to scale OBJECT/ARRAY storage. Every codec is a
+ * { encode, decode, partReaders } object.
+ *
+ * See design.md §3.
+ */
 import { numberToVar, varToNumber, range } from './utils.js'
 import { Signature } from './Signature.js'
 
@@ -20,10 +31,16 @@ class Duple {
   }
 
   flat () {
-    return [
-      this.v[0] instanceof Duple ? this.v[0].flat() : this.v[0],
-      this.v[1] instanceof Duple ? this.v[1].flat() : this.v[1]
-    ].flat()
+    // Walk the Duple tree, flattening only Duple nodes — never nested user
+    // values. (Earlier code used Array.prototype.flat() which silently
+    // flattened any nested array a caller had stored, so e.g. [3, [4,5]]
+    // would round-trip as [3, 4, 5].)
+    const out = []
+    for (const child of this.v) {
+      if (child instanceof Duple) out.push(...child.flat())
+      else out.push(child)
+    }
+    return out
   }
 
   flatDuples () {
@@ -158,12 +175,23 @@ export function makeCodecs (r) {
     return parts
   }
 
-  // Stable address of a single-part value (needed for DUPLE decode with asRefs)
+  // Stable address of a single-part value (needed for DUPLE decode with asRefs).
+  //
+  // For inline multi-byte parts that aren't independently stored, the only
+  // way to "give back an address" is to materialize them as a separate chunk —
+  // i.e. mutate. That's appropriate in write contexts (Streamo.set) but a
+  // bug from a read context. When the registry is in read-only mode (set by
+  // CodecRegistry.asRefs around its decode), we return undefined instead of
+  // appending. The caller (asRefs's caller, e.g. the explorer) sees an
+  // undefined child address and renders it as inline.
   function getPartAddress (part) {
     if (part.address !== undefined) return part.address
     const code = part.getCode()
     if (code.length === 1) return -(code[0] + 1) // negative address for single-byte primitives
-    return r.addressOf(code) ?? r.append(code)
+    const existing = r.addressOf(code)
+    if (existing !== undefined) return existing
+    if (r.readOnly) return undefined
+    return r.append(code)
   }
 
   // ── Codec definitions ────────────────────────────────────────────────────
@@ -353,10 +381,14 @@ export function makeCodecs (r) {
   }
 
   const EMPTY_OBJECT = {
+    // Accepts any non-array object with no own enumerable keys, including class
+    // instances. (OBJECT also doesn't check the prototype on encode, so empty
+    // class instances should be encodable too — keeping them symmetric.) Type
+    // information is lost on round-trip in both cases; the decoded value is a
+    // plain {}.
     encode (v) {
       if (!v || typeof v !== 'object' || Array.isArray(v)) return
-      const proto = Object.getPrototypeOf(v)
-      if (proto !== Object.prototype && proto !== null) return
+      if (v instanceof Uint8Array || v instanceof Date) return
       if (Object.keys(v).length === 0) return new Uint8Array([EMPTY_OBJECT.baseFooter])
     },
     decode: () => ({})
@@ -396,5 +428,13 @@ export function makeCodecs (r) {
     }
   }
 
-  return { UNDEFINED, NULL, FALSE, TRUE, WORD, UINT8ARRAY, EMPTY_STRING, STRING, UINT7, FLOAT64, DATE, SIGNATURE, DUPLE, EMPTY_ARRAY, ARRAY, EMPTY_OBJECT, OBJECT, VARIABLE }
+  // Empty-Uint8Array codec is appended at the END of the registration list so
+  // it doesn't shift the footer values of existing codecs — chunks created
+  // before this codec was added still decode correctly.
+  const EMPTY_UINT8ARRAY = {
+    encode: v => v instanceof Uint8Array && v.length === 0 && new Uint8Array([EMPTY_UINT8ARRAY.baseFooter]),
+    decode: () => new Uint8Array(0)
+  }
+
+  return { UNDEFINED, NULL, FALSE, TRUE, WORD, UINT8ARRAY, EMPTY_STRING, STRING, UINT7, FLOAT64, DATE, SIGNATURE, DUPLE, EMPTY_ARRAY, ARRAY, EMPTY_OBJECT, OBJECT, VARIABLE, EMPTY_UINT8ARRAY }
 }

package/public/streamo/registrySync.js

@@ -1,3 +1,14 @@
+/**
+ * @file registrySync — bidirectional multi-repo WebSocket sync.
+ *
+ * After a "registry" handshake, both sides exchange JSON catalog/
+ * subscribe/interest/announce/ping messages and binary
+ * [33-byte-key-prefix][chunk] frames. Discovery happens via filter,
+ * follow (content-driven), or onAnnounce. 20-second keep-alive ping
+ * for PaaS hosts that idle-close.
+ *
+ * See design.md §10.
+ */
 // Use native WebSocket in the browser; fall back to the `ws` package in Node.
 const WS = globalThis.WebSocket ?? (await import('ws')).default
 

package/public/streamo/utils/Recaller.js

@@ -1,3 +1,14 @@
+/**
+ * @file Recaller — fine-grained reactive dependency tracker.
+ *
+ * watch(name, fn) runs fn on a tracking stack; reportKeyAccess(target,
+ * key) inside fn registers (target, key) → fn; reportKeyMutation
+ * elsewhere wakes any matching watcher on the next microtask. The flush
+ * loop is robust against unwatch happening mid-flight, which matters
+ * for slot watchers torn down during DOM reconciliation.
+ *
+ * See design.md §6.
+ */
 import { NestedSet } from './NestedSet.js'
 import { nextTick } from './nextTick.js'