@dtudury/streamo 0.1.0

package/public/streamo/registrySync.test.js

@@ -0,0 +1,373 @@
+import { WebSocketServer } from 'ws'
+import { describe } from './utils/testing.js'
+import { RepoRegistry } from './RepoRegistry.js'
+import { attachStreamSync } from './outletSync.js'
+import { registrySync } from './registrySync.js'
+
+// makeVerifiedWritableStream only rejects invalid SIGNATURE chunks —
+// plain data chunks pass through even with fake keys.  33 bytes = compressed pubkey size.
+const fakeKey = (n = 0) => '02' + n.toString(16).padStart(2, '0').repeat(32)  // 66-char hex
+
+/** Wait up to `ms` ms for `fn()` to return truthy, polling every 10 ms. */
+function waitFor (fn, ms = 500) {
+  return new Promise((resolve, reject) => {
+    const start = Date.now()
+    const poll = () => {
+      const v = fn()
+      if (v) return resolve(v)
+      if (Date.now() - start > ms) return reject(new Error('waitFor timeout'))
+      setTimeout(poll, 10)
+    }
+    poll()
+  })
+}
+
+/** Start a WebSocketServer on a random port backed by a registry. */
+function startServer (registry) {
+  return new Promise((resolve, reject) => {
+    const wss = new WebSocketServer({ port: 0 })
+    wss.on('listening', () => {
+      const { port } = wss.address()
+      attachStreamSync(wss, registry, 'test-outlet')
+      resolve({ wss, port })
+    })
+    wss.on('error', reject)
+  })
+}
+
+describe(import.meta.url, ({ test }) => {
+  test('onOpen fires after registry.open resolves', async ({ assert }) => {
+    const registry = new RepoRegistry()
+    const calls = []
+    registry.onOpen((key, repo) => calls.push({ key, repo }))
+    const repo = await registry.open('abc')
+    assert.equal(calls.length, 1)
+    assert.equal(calls[0].key, 'abc')
+    assert.ok(calls[0].repo === repo)
+  })
+
+  test('offOpen removes the callback', async ({ assert }) => {
+    const registry = new RepoRegistry()
+    let count = 0
+    const cb = () => count++
+    registry.onOpen(cb)
+    await registry.open('x')
+    registry.offOpen(cb)
+    await registry.open('y')
+    assert.equal(count, 1)
+  })
+
+  test('onOpen not called for already-open key (concurrent open)', async ({ assert }) => {
+    const registry = new RepoRegistry()
+    let count = 0
+    registry.onOpen(() => count++)
+    await Promise.all([registry.open('k'), registry.open('k'), registry.open('k')])
+    assert.equal(count, 1)
+  })
+
+  test('two registries sync an existing repo via registrySync', async ({ assert }) => {
+    const serverRegistry = new RepoRegistry()
+    const { wss, port } = await startServer(serverRegistry)
+
+    const keyHex = fakeKey(1)
+    const serverRepo = await serverRegistry.open(keyHex)
+    serverRepo.set({ hello: 'world' })
+
+    const clientRegistry = new RepoRegistry()
+    const { ws } = await registrySync(clientRegistry, 'localhost', port, { filter: k => k === keyHex })
+
+    await waitFor(() => clientRegistry.get(keyHex)?.get('hello') === 'world')
+    assert.equal(clientRegistry.get(keyHex).get('hello'), 'world')
+
+    ws.close()
+    await new Promise(r => wss.close(r))
+  })
+
+  test('changes on server after connect are synced to client', async ({ assert }) => {
+    const serverRegistry = new RepoRegistry()
+    const { wss, port } = await startServer(serverRegistry)
+
+    const keyHex = fakeKey(2)
+    const serverRepo = await serverRegistry.open(keyHex)
+    serverRepo.set({ v: 1 })
+
+    const clientRegistry = new RepoRegistry()
+    const { ws } = await registrySync(clientRegistry, 'localhost', port, { filter: k => k === keyHex })
+
+    await waitFor(() => clientRegistry.get(keyHex)?.get('v') === 1)
+
+    serverRepo.set({ v: 2 })
+    await waitFor(() => clientRegistry.get(keyHex)?.get('v') === 2)
+    assert.equal(clientRegistry.get(keyHex).get('v'), 2)
+
+    ws.close()
+    await new Promise(r => wss.close(r))
+  })
+
+  test('newly opened server repos are announced and synced', async ({ assert }) => {
+    const serverRegistry = new RepoRegistry()
+    const { wss, port } = await startServer(serverRegistry)
+
+    const keyHex = fakeKey(3)
+    const clientRegistry = new RepoRegistry()
+    const { ws } = await registrySync(clientRegistry, 'localhost', port)
+
+    const serverRepo = await serverRegistry.open(keyHex)
+    serverRepo.set({ late: true })
+
+    await waitFor(() => clientRegistry.get(keyHex)?.get('late') === true)
+    assert.equal(clientRegistry.get(keyHex).get('late'), true)
+
+    ws.close()
+    await new Promise(r => wss.close(r))
+  })
+
+  test('filter prevents unwanted repos from syncing', async ({ assert }) => {
+    const serverRegistry = new RepoRegistry()
+    const { wss, port } = await startServer(serverRegistry)
+
+    const keyA = fakeKey(4)
+    const keyB = fakeKey(5)
+
+    const repoA = await serverRegistry.open(keyA)
+    repoA.set({ name: 'a' })
+    const repoB = await serverRegistry.open(keyB)
+    repoB.set({ name: 'b' })
+
+    const clientRegistry = new RepoRegistry()
+    const { ws } = await registrySync(clientRegistry, 'localhost', port, { filter: k => k === keyA })
+
+    await waitFor(() => clientRegistry.get(keyA)?.get('name') === 'a')
+    assert.equal(clientRegistry.get(keyA).get('name'), 'a')
+
+    await new Promise(r => setTimeout(r, 100))
+    assert.equal(clientRegistry.get(keyB), undefined, 'keyB was filtered out')
+
+    ws.close()
+    await new Promise(r => wss.close(r))
+  })
+
+  test('two peers with different repos each sync both after connecting', async ({ assert }) => {
+    const registryA = new RepoRegistry()
+    const registryB = new RepoRegistry()
+
+    const keyA = fakeKey(6)
+    const keyB = fakeKey(7)
+
+    const repoA = await registryA.open(keyA)
+    repoA.set({ owner: 'A' })
+    const repoB = await registryB.open(keyB)
+    repoB.set({ owner: 'B' })
+
+    const { wss, port } = await startServer(registryA)
+    const { ws } = await registrySync(registryB, 'localhost', port)
+
+    await waitFor(() => registryA.get(keyB)?.get('owner') === 'B')
+    await waitFor(() => registryB.get(keyA)?.get('owner') === 'A')
+
+    assert.equal(registryA.get(keyB).get('owner'), 'B')
+    assert.equal(registryB.get(keyA).get('owner'), 'A')
+
+    ws.close()
+    await new Promise(r => wss.close(r))
+  })
+
+  test('follow: auto-subscribes to repos referenced in a synced repo\'s value', async ({ assert }) => {
+    // Simulates a chat app: rootRepo lists participant keys; client follows the
+    // root and should automatically discover and sync all participant repos.
+    const serverRegistry = new RepoRegistry()
+    const { wss, port } = await startServer(serverRegistry)
+
+    const rootKey = fakeKey(10)
+    const aliceKey = fakeKey(11)
+    const bobKey = fakeKey(12)
+
+    const rootRepo = await serverRegistry.open(rootKey)
+    const aliceRepo = await serverRegistry.open(aliceKey)
+    const bobRepo = await serverRegistry.open(bobKey)
+
+    aliceRepo.set({ name: 'alice', message: 'hello' })
+    bobRepo.set({ name: 'bob', message: 'hey' })
+    rootRepo.set({ members: [aliceKey, bobKey] })
+
+    const clientRegistry = new RepoRegistry()
+    const { ws } = await registrySync(clientRegistry, 'localhost', port, {
+      filter: k => k === rootKey,  // only explicitly subscribe to root
+      follow: (keyHex, repo, subscribe) => {
+        // extract participant keys from the chat repo
+        for (const memberKey of repo.get('members') ?? []) subscribe(memberKey)
+      }
+    })
+
+    // Root syncs via filter; participants sync via follow
+    await waitFor(() => clientRegistry.get(aliceKey)?.get('name') === 'alice')
+    await waitFor(() => clientRegistry.get(bobKey)?.get('name') === 'bob')
+
+    assert.equal(clientRegistry.get(aliceKey).get('name'), 'alice')
+    assert.equal(clientRegistry.get(bobKey).get('name'), 'bob')
+
+    ws.close()
+    await new Promise(r => wss.close(r))
+  })
+
+  test('follow: re-runs when a repo changes and discovers newly added refs', async ({ assert }) => {
+    const serverRegistry = new RepoRegistry()
+    const { wss, port } = await startServer(serverRegistry)
+
+    const rootKey = fakeKey(13)
+    const carolKey = fakeKey(14)
+
+    const rootRepo = await serverRegistry.open(rootKey)
+    rootRepo.set({ members: [] })  // starts empty
+
+    const clientRegistry = new RepoRegistry()
+    const { ws } = await registrySync(clientRegistry, 'localhost', port, {
+      filter: k => k === rootKey,
+      follow: (keyHex, repo, subscribe) => {
+        for (const memberKey of repo.get('members') ?? []) subscribe(memberKey)
+      }
+    })
+
+    await waitFor(() => clientRegistry.get(rootKey)?.get('members') !== undefined)
+
+    // Carol joins: her repo is added to the server, root is updated to list her
+    const carolRepo = await serverRegistry.open(carolKey)
+    carolRepo.set({ name: 'carol' })
+    rootRepo.set({ members: [carolKey] })
+
+    await waitFor(() => clientRegistry.get(carolKey)?.get('name') === 'carol')
+    assert.equal(clientRegistry.get(carolKey).get('name'), 'carol')
+
+    ws.close()
+    await new Promise(r => wss.close(r))
+  })
+
+  test('announce is routed to interested peers', async ({ assert }) => {
+    const { wss, port } = await startServer(new RepoRegistry())
+    const topic = fakeKey(20)
+    const announced = fakeKey(21)
+
+    const received = []
+    const sessionA = await registrySync(new RepoRegistry(), 'localhost', port, {
+      onAnnounce: (key, t) => received.push({ key, topic: t })
+    })
+    sessionA.interest(topic)
+
+    const sessionB = await registrySync(new RepoRegistry(), 'localhost', port)
+
+    // Give the interest message time to reach the server
+    await new Promise(r => setTimeout(r, 50))
+    sessionB.announce(announced, topic)
+
+    await waitFor(() => received.length === 1)
+    assert.equal(received[0].key, announced)
+    assert.equal(received[0].topic, topic)
+
+    sessionA.close()
+    sessionB.close()
+    await new Promise(r => wss.close(r))
+  })
+
+  test('announce is not received without interest', async ({ assert }) => {
+    const { wss, port } = await startServer(new RepoRegistry())
+    const topic = fakeKey(22)
+    const announced = fakeKey(23)
+
+    const received = []
+    const sessionA = await registrySync(new RepoRegistry(), 'localhost', port, {
+      onAnnounce: (key) => received.push(key)
+    })
+    // sessionA does NOT call interest(topic)
+
+    const sessionB = await registrySync(new RepoRegistry(), 'localhost', port)
+    await new Promise(r => setTimeout(r, 50))
+    sessionB.announce(announced, topic)
+
+    await new Promise(r => setTimeout(r, 100))
+    assert.equal(received.length, 0, 'no announcements without interest')
+
+    sessionA.close()
+    sessionB.close()
+    await new Promise(r => wss.close(r))
+  })
+
+  test('announce reaches multiple interested peers', async ({ assert }) => {
+    const { wss, port } = await startServer(new RepoRegistry())
+    const topic = fakeKey(24)
+    const announced = fakeKey(25)
+
+    const receivedA = [], receivedB = []
+    const sessionA = await registrySync(new RepoRegistry(), 'localhost', port, {
+      onAnnounce: (key) => receivedA.push(key)
+    })
+    const sessionB = await registrySync(new RepoRegistry(), 'localhost', port, {
+      onAnnounce: (key) => receivedB.push(key)
+    })
+    sessionA.interest(topic)
+    sessionB.interest(topic)
+
+    const sessionC = await registrySync(new RepoRegistry(), 'localhost', port)
+    await new Promise(r => setTimeout(r, 50))
+    sessionC.announce(announced, topic)
+
+    await waitFor(() => receivedA.length === 1 && receivedB.length === 1)
+    assert.equal(receivedA[0], announced)
+    assert.equal(receivedB[0], announced)
+
+    sessionA.close()
+    sessionB.close()
+    sessionC.close()
+    await new Promise(r => wss.close(r))
+  })
+
+  test('announce is not echoed back to the sender', async ({ assert }) => {
+    const { wss, port } = await startServer(new RepoRegistry())
+    const topic = fakeKey(26)
+    const announced = fakeKey(27)
+
+    const received = []
+    const session = await registrySync(new RepoRegistry(), 'localhost', port, {
+      onAnnounce: (key) => received.push(key)
+    })
+    session.interest(topic)
+
+    await new Promise(r => setTimeout(r, 50))
+    session.announce(announced, topic)  // sender declares interest and announces
+
+    await new Promise(r => setTimeout(r, 100))
+    assert.equal(received.length, 0, 'sender should not receive its own announcement')
+
+    session.close()
+    await new Promise(r => wss.close(r))
+  })
+
+  test('after disconnect, interest is cleaned up and announcements stop', async ({ assert }) => {
+    const { wss, port } = await startServer(new RepoRegistry())
+    const topic = fakeKey(28)
+    const announced = fakeKey(29)
+
+    const received = []
+    const sessionA = await registrySync(new RepoRegistry(), 'localhost', port, {
+      onAnnounce: (key) => received.push(key)
+    })
+    sessionA.interest(topic)
+
+    const sessionB = await registrySync(new RepoRegistry(), 'localhost', port)
+
+    // Confirm routing works before disconnect
+    await new Promise(r => setTimeout(r, 50))
+    sessionB.announce(announced, topic)
+    await waitFor(() => received.length === 1)
+
+    // Disconnect A, then announce again
+    sessionA.close()
+    await new Promise(r => setTimeout(r, 50))
+    sessionB.announce(announced, topic)
+
+    await new Promise(r => setTimeout(r, 100))
+    assert.equal(received.length, 1, 'no more announcements after disconnect')
+
+    sessionB.close()
+    await new Promise(r => wss.close(r))
+  })
+})

package/public/streamo/s3Sync.js

@@ -0,0 +1,99 @@
+import { GetObjectCommand, ListObjectsV2Command, PutObjectCommand } from '@aws-sdk/client-s3'
+import { S3Client } from '@aws-sdk/client-s3'
+
+/**
+ * Count objects in S3 with the given prefix (handles pagination).
+ */
+async function listFrameCount (client, bucket, prefix) {
+  let count = 0
+  let token
+  do {
+    const res = await client.send(new ListObjectsV2Command({
+      Bucket: bucket,
+      Prefix: prefix + '/',
+      ContinuationToken: token
+    }))
+    count += res.KeyCount ?? 0
+    token = res.NextContinuationToken
+  } while (token)
+  return count
+}
+
+async function downloadFrame (client, bucket, key) {
+  const res = await client.send(new GetObjectCommand({ Bucket: bucket, Key: key }))
+  return new Uint8Array(await res.Body.transformToByteArray())
+}
+
+async function uploadFrame (client, bucket, key, data) {
+  await client.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: data }))
+}
+
+/**
+ * Sync a Stream to/from S3-compatible object storage.
+ *
+ * Each wire-format frame is stored as a separate numbered object:
+ *   <publicKeyHex>/000000
+ *   <publicKeyHex>/000001
+ *   ...
+ *
+ * On startup, existing frames are downloaded in order and fed into the stream.
+ * New frames are uploaded as they arrive.
+ *
+ * @param {import('./Stream.js').Stream} stream
+ * @param {string} publicKeyHex
+ * @param {object} config
+ * @param {string} config.bucket
+ * @param {string} [config.endpoint]
+ * @param {string} [config.region]
+ * @param {string} [config.accessKeyId]
+ * @param {string} [config.secretAccessKey]
+ */
+export async function s3Sync (stream, publicKeyHex, config) {
+  const { bucket, endpoint, region, accessKeyId, secretAccessKey } = config
+
+  const client = new S3Client({
+    ...(endpoint ? { endpoint, forcePathStyle: true } : {}),
+    ...(region ? { region } : {}),
+    ...(accessKeyId && secretAccessKey
+      ? { credentials: { accessKeyId, secretAccessKey } }
+      : {})
+  })
+
+  const keyFor = i => `${publicKeyHex}/${String(i).padStart(6, '0')}`
+
+  // Load existing frames from S3 into the stream
+  let frameCount = 0
+  try {
+    frameCount = await listFrameCount(client, bucket, publicKeyHex)
+    if (frameCount > 0) {
+      console.log(`[s3] loading ${frameCount} frames for ${publicKeyHex.slice(0, 8)}...`)
+      const writer = stream.makeWritableStream().getWriter()
+      for (let i = 0; i < frameCount; i++) {
+        const frame = await downloadFrame(client, bucket, keyFor(i))
+        await writer.write(frame)
+      }
+    }
+  } catch (e) {
+    console.error('[s3] load error:', e.message)
+  }
+
+  // Upload new frames as they arrive, skipping already-uploaded ones
+  let uploaded = 0
+  const reader = stream.makeReadableStream().getReader()
+  ;(async () => {
+    try {
+      while (true) {
+        const { value, done } = await reader.read()
+        if (done) break
+        if (uploaded < frameCount) {
+          uploaded++ // skip frames we just loaded
+          continue
+        }
+        await uploadFrame(client, bucket, keyFor(uploaded), value)
+        uploaded++
+      }
+    } catch (e) {
+      console.error('[s3] upload error:', e.message)
+    }
+  })()
+}

package/public/streamo/stateFileSync.js

@@ -0,0 +1,17 @@
+import { writeFile } from 'fs/promises'
+
+/**
+ * Watch a stream and write its current state as JSON to filePath on every change.
+ * The file is written whenever a new value is committed (or set on a plain Stream).
+ *
+ * @param {import('./Stream.js').Stream} stream
+ * @param {string} filePath
+ */
+export function stateFileSync (stream, filePath) {
+  stream.watch('state-file-sync', () => {
+    const state = stream.get()
+    if (state != null) {
+      writeFile(filePath, JSON.stringify(state, null, 2) + '\n').catch(console.error)
+    }
+  })
+}

package/public/streamo/sync.test.js

@@ -0,0 +1,98 @@
+import { describe } from './utils/testing.js'
+import { RepoRegistry as StreamRegistry } from './RepoRegistry.js'
+import { outletSync } from './outletSync.js'
+import { originSync } from './originSync.js'
+
+const KEY = 'aabbccddeeff0011'
+
+// Wait until predicate(stream) returns true.
+// Accesses stream.byteLength explicitly so the watcher re-runs on every append,
+// regardless of whether the predicate itself touches a reactive path.
+function waitFor (stream, predicate, timeout = 2000) {
+  return new Promise((resolve, reject) => {
+    const t = setTimeout(() => reject(new Error('timeout')), timeout)
+    let done = false
+    stream.watch('waitFor', () => {
+      if (!done && predicate(stream)) {
+        done = true
+        clearTimeout(t)
+        resolve()
+      }
+    })
+  })
+}
+
+describe(import.meta.url, ({ test }) => {
+  test('outlet syncs existing stream data to a new origin', async ({ assert }) => {
+    const serverRegistry = new StreamRegistry()
+    const serverStream = await serverRegistry.open(KEY)
+    serverStream.set({ hello: 'world' })
+
+    const wss = outletSync(serverRegistry, 0)
+    await new Promise(resolve => wss.on('listening', resolve))
+    const { port } = wss.address()
+
+    const clientRegistry = new StreamRegistry()
+    const clientStream = await clientRegistry.open(KEY)
+    const ws = await originSync(clientStream, KEY, 'localhost', port)
+
+    await waitFor(clientStream, s => s.get('hello') === 'world')
+    assert.equal(clientStream.get('hello'), 'world', 'client received server data')
+
+    ws.close()
+    for (const c of wss.clients) c.terminate()
+    wss.close()
+  })
+
+  test('origin syncs local data up to the outlet', async ({ assert }) => {
+    const serverRegistry = new StreamRegistry()
+    const wss = outletSync(serverRegistry, 0)
+    await new Promise(resolve => wss.on('listening', resolve))
+    const { port } = wss.address()
+
+    const clientRegistry = new StreamRegistry()
+    const clientStream = await clientRegistry.open(KEY)
+    clientStream.set({ from: 'client' })
+
+    const ws = await originSync(clientStream, KEY, 'localhost', port)
+    const serverStream = await serverRegistry.open(KEY)
+
+    await waitFor(serverStream, s => s.get('from') === 'client')
+    assert.equal(serverStream.get('from'), 'client', 'server received client data')
+
+    ws.close()
+    for (const c of wss.clients) c.terminate()
+    wss.close()
+  })
+
+  test('two origins converge on the same state', async ({ assert }) => {
+    const serverRegistry = new StreamRegistry()
+    const wss = outletSync(serverRegistry, 0)
+    await new Promise(resolve => wss.on('listening', resolve))
+    const { port } = wss.address()
+
+    const r1 = new StreamRegistry()
+    const r2 = new StreamRegistry()
+    const s1 = await r1.open(KEY)
+    const s2 = await r2.open(KEY)
+
+    s1.set({ x: 1 })
+    s2.set({ x: 2 })  // will be a conflict at the streamo level, but both chunks land
+
+    const ws1 = await originSync(s1, KEY, 'localhost', port)
+    const ws2 = await originSync(s2, KEY, 'localhost', port)
+
+    // Both clients should end up with the same byteLength once chunks propagate
+    const serverStream = await serverRegistry.open(KEY)
+    await waitFor(serverStream, s => s.byteLength >= s1.byteLength && s.byteLength >= s2.byteLength)
+    await waitFor(s1, s => s.byteLength >= serverStream.byteLength)
+    await waitFor(s2, s => s.byteLength >= serverStream.byteLength)
+
+    assert.equal(s1.byteLength, s2.byteLength, 'both clients converged to same byteLength')
+    assert.equal(s1.byteLength, serverStream.byteLength, 'clients match server')
+
+    ws1.close(); ws2.close()
+    for (const c of wss.clients) c.terminate()
+    wss.close()
+  })
+})

package/public/streamo/utils/NestedSet.js

@@ -0,0 +1,41 @@
+export class NestedSet {
+  #map = new Map()
+  #set = new Set()
+
+  get size () {
+    return this.values().length
+  }
+
+  add (root, ...rest) {
+    if (rest.length) {
+      if (!this.#map.has(root)) this.#map.set(root, new NestedSet())
+      return this.#map.get(root).add(...rest)
+    }
+    return this.#set.add(root)
+  }
+
+  get (root, ...rest) {
+    if (rest.length) return this.#map.get(root)?.get(...rest)
+    return this.#map.get(root)
+  }
+
+  delete (root, ...rest) {
+    if (rest.length) {
+      this.#map.get(root)?.delete(...rest)
+    } else {
+      this.#set.delete(root)
+      this.#map.forEach(nested => nested.delete(root))
+    }
+    if (!this.#map.get(root)?.size) this.#map.delete(root)
+  }
+
+  deleteBranch (root, ...rest) {
+    if (rest.length) return this.#map.get(root)?.deleteBranch(...rest)
+    return this.#map.delete(root)
+  }
+
+  values (...path) {
+    if (path.length) return this.#map.get(path[0])?.values(...path.slice(1)) ?? []
+    return [...new Set([...this.#set, ...[...this.#map.keys()].flatMap(k => this.values(k))])]
+  }
+}