@huynle/hub 0.6.1 → 0.7.2

package/src/central/backend/valkey-events.ts

@@ -0,0 +1,168 @@
+// EventLog on Valkey Streams (design §9): one stream per user, so Last-Event-ID resumes on any pod.
+//
+// Ids are `<epoch>.<n>` like the in-memory log's: n counts per user from 1 and is the entry's stream id (`n-0`,
+// given explicitly rather than by the server clock), so a gap is exact: the entry after the client's id must
+// still be there. The epoch lives in its own key next to the stream and is made the first time a user's log is
+// touched; it goes when the log expires (both keys carry a ttl, renewed on every append and latest), so an id
+// from an earlier log is never mistaken for a current one. Making an epoch deletes any stream left behind, so
+// the stream always outlives its epoch (its ttl is a little longer) and never replays old entries.
+//
+// Bounds: XADD MAXLEN ~ max (approximate: Valkey trims whole nodes, so a log keeps at least `max` events) and,
+// on read, `maxAgeMs` against the `now` clock. since() replays the entries after an id (XRANGE, exclusive) and
+// asks for a resync when it cannot cover the gap: an unknown, another or an expired epoch, an id it never gave
+// out, or the next entry trimmed or older than maxAgeMs.
+import { RedisClient } from "bun"
+import { EVENT_LOG_AGE_MS, EVENT_LOG_MAX } from "../events-log"
+import type { Clock, EventLog, EventMessage, LogEvent, SinceResult } from "./types"
+
+export interface ValkeyEventLogOptions {
+  readonly now?: Clock
+  /** Events kept per user, at least (default EVENT_LOG_MAX). */
+  readonly max?: number
+  /** Events older than this are not replayed (default EVENT_LOG_AGE_MS). */
+  readonly maxAgeMs?: number
+  /** Key prefix (default "hub:"). */
+  readonly prefix?: string
+  /** How long an idle user's log is kept in Valkey, server time (default maxAgeMs + 60 s). */
+  readonly ttlMs?: number
+}
+
+/** Where a user's log lives. Both keys share a hash tag, so a script may use them together in a Valkey cluster. */
+export function valkeyEventKeys(userID: string, prefix = "hub:"): { stream: string; epoch: string } {
+  const base = `${prefix}ev:{${encodeURIComponent(userID)}}`
+  return { stream: base, epoch: `${base}:epoch` }
+}
+
+/** The stream outlives its epoch by this much (see above). */
+const STREAM_TTL_MARGIN_MS = 1000
+
+// KEYS[1] stream, KEYS[2] epoch; ARGV[1] a new epoch (used when there is none), ARGV[2] ttl ms.
+const OPEN = `
+local epoch = redis.call('GET', KEYS[2])
+if not epoch then
+  epoch = ARGV[1]
+  redis.call('DEL', KEYS[1])
+end
+redis.call('SET', KEYS[2], epoch, 'PX', ARGV[2])
+local function renew()
+  redis.call('PEXPIRE', KEYS[1], tonumber(ARGV[2]) + ${STREAM_TTL_MARGIN_MS})
+end
+`
+
+// The n of the user's last entry (0: none yet); trimming keeps the stream's last id.
+const LAST = `
+local function lastN(key)
+  if redis.call('EXISTS', key) == 0 then return 0 end
+  local r = redis.call('XINFO', 'STREAM', key)
+  for i = 1, #r, 2 do
+    if r[i] == 'last-generated-id' then return tonumber(string.match(r[i + 1], '^(%d+)')) end
+  end
+  return 0
+end
+`
+
+// ARGV[3] max, ARGV[4] message JSON, ARGV[5] at, ARGV[6] exceptDevice ('' for none).
+const APPEND = `${OPEN}${LAST}
+local n = string.format('%d', lastN(KEYS[1]) + 1)
+if ARGV[6] ~= '' then
+  redis.call('XADD', KEYS[1], 'MAXLEN', '~', ARGV[3], n .. '-0', 'm', ARGV[4], 'at', ARGV[5], 'x', ARGV[6])
+else
+  redis.call('XADD', KEYS[1], 'MAXLEN', '~', ARGV[3], n .. '-0', 'm', ARGV[4], 'at', ARGV[5])
+end
+renew()
+return {epoch, n}
+`
+
+const LATEST = `${OPEN}${LAST}
+if redis.call('EXISTS', KEYS[1]) == 1 then renew() end
+return {epoch, string.format('%d', lastN(KEYS[1]))}
+`
+
+// ARGV[1] the n the client holds. Returns {} when the user has no log.
+const SINCE = `${LAST}
+local epoch = redis.call('GET', KEYS[2])
+if not epoch then return {} end
+return {epoch, string.format('%d', lastN(KEYS[1])), redis.call('XRANGE', KEYS[1], '(' .. ARGV[1] .. '-0', '+')}
+`
+
+const ID = /^([a-z0-9]+)\.(\d+)$/
+
+const newEpoch = () => `${Date.now().toString(36)}${crypto.randomUUID().replaceAll("-", "").slice(0, 8)}`
+
+type RawEntry = [string, string[]]
+
+export class ValkeyEventLog implements EventLog {
+  private readonly client: RedisClient
+  private readonly ownsClient: boolean
+  private readonly now: Clock
+  private readonly max: number
+  private readonly maxAgeMs: number
+  private readonly prefix: string
+  private readonly ttlMs: number
+
+  /** On a url (its own connection, made on first use) or a client the caller owns. */
+  constructor(valkey: string | RedisClient, options: ValkeyEventLogOptions = {}) {
+    this.ownsClient = typeof valkey === "string"
+    this.client = typeof valkey === "string" ? new RedisClient(valkey) : valkey
+    this.now = options.now ?? Date.now
+    this.max = options.max ?? EVENT_LOG_MAX
+    this.maxAgeMs = options.maxAgeMs ?? EVENT_LOG_AGE_MS
+    this.prefix = options.prefix ?? "hub:"
+    this.ttlMs = options.ttlMs ?? this.maxAgeMs + 60_000
+  }
+
+  async append(userID: string, message: EventMessage, options: { readonly exceptDevice?: string } = {}): Promise<string> {
+    const [epoch, n] = (await this.eval(APPEND, userID, [
+      newEpoch(),
+      String(this.ttlMs),
+      String(this.max),
+      JSON.stringify(message),
+      String(this.now()),
+      options.exceptDevice ?? "",
+    ])) as [string, string]
+    return `${epoch}.${n}`
+  }
+
+  async latest(userID: string): Promise<string> {
+    const [epoch, n] = (await this.eval(LATEST, userID, [newEpoch(), String(this.ttlMs)])) as [string, string]
+    return `${epoch}.${n}`
+  }
+
+  async since(userID: string, lastID: string): Promise<SinceResult> {
+    const resync = { resync: true } as const
+    const match = ID.exec(lastID)
+    const held = Number(match?.[2])
+    if (!match || !Number.isSafeInteger(held)) return resync
+    const reply = (await this.eval(SINCE, userID, [String(held)])) as [] | [string, string, RawEntry[]]
+    if (reply.length === 0) return resync
+    const [epoch, last, entries] = reply
+    if (epoch !== match[1] || held > Number(last)) return resync
+    const events = entries.map(([id, fields]) => this.event(epoch, id, fields))
+    if (held === Number(last)) return { events: [] }
+    // The next event the client needs (held + 1) must still be there, and not too old.
+    const next = events[0]
+    if (!next || next.id !== `${epoch}.${held + 1}` || next.at < this.now() - this.maxAgeMs) return resync
+    return { events }
+  }
+
+  /** Closes the connection when the log made it. */
+  close(): void {
+    if (this.ownsClient) this.client.close()
+  }
+
+  private eval(script: string, userID: string, args: string[]): Promise<unknown> {
+    const keys = valkeyEventKeys(userID, this.prefix)
+    return this.client.send("EVAL", [script, "2", keys.stream, keys.epoch, ...args])
+  }
+
+  private event(epoch: string, id: string, fields: string[]): LogEvent {
+    const values: Record<string, string> = {}
+    for (let i = 0; i + 1 < fields.length; i += 2) values[fields[i]!] = fields[i + 1]!
+    return {
+      id: `${epoch}.${id.slice(0, id.indexOf("-"))}`,
+      message: JSON.parse(values.m ?? "{}") as EventMessage,
+      at: Number(values.at),
+      ...(values.x ? { exceptDevice: values.x } : {}),
+    }
+  }
+}

package/src/central/backend/valkey-kv.ts

@@ -0,0 +1,54 @@
+// Kv in Valkey: SET [PX] [NX], GET, GETDEL and DEL under a key prefix (default "kv:"). Expiry is server-side,
+// so it follows the server's clock, not an injected one.
+import type { Kv, KvSetOptions } from "./types"
+import { ValkeyConnection, type ValkeySource } from "./valkey-client"
+
+export interface ValkeyKvOptions {
+  /** Prepended to every key (default "kv:"). */
+  readonly prefix?: string
+}
+
+export class ValkeyKv implements Kv {
+  private readonly valkey: ValkeyConnection
+  private readonly prefix: string
+
+  constructor(source: ValkeySource, options: ValkeyKvOptions = {}) {
+    this.valkey = new ValkeyConnection(source)
+    this.prefix = options.prefix ?? "kv:"
+  }
+
+  async set(key: string, value: string, options: KvSetOptions = {}): Promise<boolean> {
+    const { px, nx } = options
+    if (px !== undefined && !(Number.isInteger(px) && px > 0)) throw new Error("Kv.set: px must be a positive whole number of milliseconds")
+    const args = [this.prefix + key, value]
+    if (px !== undefined) args.push("PX", String(px))
+    if (nx) args.push("NX")
+    return (await this.valkey.send("SET", args)) === "OK"
+  }
+
+  async get(key: string): Promise<string | undefined> {
+    return text(await this.valkey.send("GET", [this.prefix + key]))
+  }
+
+  async getdel(key: string): Promise<string | undefined> {
+    return text(await this.valkey.send("GETDEL", [this.prefix + key]))
+  }
+
+  async del(key: string): Promise<boolean> {
+    return Number(await this.valkey.send("DEL", [this.prefix + key])) > 0
+  }
+
+  /** Readiness: PING (rejects when the server does not answer). */
+  async ping(): Promise<void> {
+    await this.valkey.send("PING", [])
+  }
+
+  /** Closes the connection when it was opened from a url. */
+  async close(): Promise<void> {
+    this.valkey.close()
+  }
+}
+
+function text(reply: unknown): string | undefined {
+  return reply === null || reply === undefined ? undefined : String(reply)
+}

package/src/central/backend/valkey-locks.ts

@@ -0,0 +1,92 @@
+// Locks in Valkey: a lease is one key "lock:<key>" set with PX (server-side expiry) holding
+// "<fence> <expiresAt> <owner>"; a new holder takes the next value of the counter "fence:<key>", which never
+// expires, so fences keep growing across expiries and releases. Taking (SET PX only when the key is absent or
+// already the caller's), renewing and releasing compare the owner inside Lua scripts, so each is atomic.
+//
+// Expiry follows the server's clock; `expiresAt` in a Lease is stamped by the injected clock (`now`) and is
+// informational. Fence counters stay after their lock goes away (one small key per lock name ever used).
+import type { Clock, Lease, Locks } from "./types"
+import { ValkeyConnection, type ValkeySource } from "./valkey-client"
+
+/** KEYS: lock, fence. ARGV: owner, ttl, expiresAt, mode ("acquire" or "renew"). Returns the lease value or nil. */
+const TAKE = `local cur = redis.call('GET', KEYS[1])
+local fence
+if cur then
+  local f, _, owner = string.match(cur, '^(%d+) (%S+) (.*)$')
+  if owner ~= ARGV[1] then return false end
+  fence = f
+elseif ARGV[4] == 'acquire' then
+  fence = tostring(redis.call('INCR', KEYS[2]))
+else
+  return false
+end
+local value = fence .. ' ' .. ARGV[3] .. ' ' .. ARGV[1]
+redis.call('SET', KEYS[1], value, 'PX', ARGV[2])
+return value`
+
+/** KEYS: lock. ARGV: owner. Deletes the lock when the owner holds it; returns 1 or 0. */
+const RELEASE = `local cur = redis.call('GET', KEYS[1])
+if not cur then return 0 end
+local _, _, owner = string.match(cur, '^(%d+) (%S+) (.*)$')
+if owner ~= ARGV[1] then return 0 end
+redis.call('DEL', KEYS[1])
+return 1`
+
+export interface ValkeyLocksOptions {
+  /** Stamps Lease.expiresAt (default Date.now). Expiry itself is server-side. */
+  readonly now?: Clock
+  /** Prepended to the lock keys (default "lock:"); fence counters use `fencePrefix` (default "fence:"). */
+  readonly prefix?: string
+  readonly fencePrefix?: string
+}
+
+export class ValkeyLocks implements Locks {
+  private readonly valkey: ValkeyConnection
+  private readonly now: Clock
+  private readonly prefix: string
+  private readonly fencePrefix: string
+
+  constructor(source: ValkeySource, options: ValkeyLocksOptions = {}) {
+    this.valkey = new ValkeyConnection(source)
+    this.now = options.now ?? Date.now
+    this.prefix = options.prefix ?? "lock:"
+    this.fencePrefix = options.fencePrefix ?? "fence:"
+  }
+
+  acquire(key: string, owner: string, ttlMs: number): Promise<Lease | undefined> {
+    return this.take("acquire", key, owner, ttlMs)
+  }
+
+  renew(key: string, owner: string, ttlMs: number): Promise<Lease | undefined> {
+    return this.take("renew", key, owner, ttlMs)
+  }
+
+  async release(key: string, owner: string): Promise<boolean> {
+    return Number(await this.valkey.send("EVAL", [RELEASE, "1", this.prefix + key, owner])) === 1
+  }
+
+  async holder(key: string): Promise<Lease | undefined> {
+    return parseLease(key, await this.valkey.send("GET", [this.prefix + key]))
+  }
+
+  /** Closes the connection when it was opened from a url. */
+  async close(): Promise<void> {
+    this.valkey.close()
+  }
+
+  private async take(mode: "acquire" | "renew", key: string, owner: string, ttlMs: number): Promise<Lease | undefined> {
+    if (!(Number.isInteger(ttlMs) && ttlMs > 0)) throw new Error(`Locks.${mode}: ttl must be a positive whole number of milliseconds`)
+    const expiresAt = this.now() + ttlMs
+    const keys = [this.prefix + key, this.fencePrefix + key]
+    return parseLease(key, await this.valkey.send("EVAL", [TAKE, "2", ...keys, owner, String(ttlMs), String(expiresAt), mode]))
+  }
+}
+
+/** A lease from its stored "<fence> <expiresAt> <owner>" (the owner may hold any character). */
+function parseLease(key: string, reply: unknown): Lease | undefined {
+  if (reply === null || reply === undefined) return undefined
+  const value = String(reply)
+  const first = value.indexOf(" ")
+  const second = value.indexOf(" ", first + 1)
+  return { key, owner: value.slice(second + 1), fence: Number(value.slice(0, first)), expiresAt: Number(value.slice(first + 1, second)) }
+}

package/src/central/cluster/directory.ts

@@ -0,0 +1,141 @@
+// The connection directory (design §9, decision vkxkqiuf §5): which pod holds each spoke or node connection, in the
+// shared Kv so any pod can route an op to it (./rpc.ts). Each pod is the single writer of its own connections:
+//   dir:<connID> → {"pod","machineID"}   written when the hello completes, removed when the socket closes
+//   pod:<podID>  → "1"                   the pod's liveness
+// Both expire after DIRECTORY_TTL_MS and are refreshed every DIRECTORY_HEARTBEAT_MS, so a pod that dies without
+// cleaning up drops out within the TTL. A lookup that finds an entry of a pod whose liveness key is gone removes it
+// on the spot (the dead-pod sweep); callers also forget an entry when its pod has no RPC responder. Connection IDs
+// are never reused, so the plain DEL on close cannot remove another pod's entry.
+import { realTimers } from "../backend/memory-bus"
+import type { Kv, Timers } from "../backend/types"
+
+export const DIRECTORY_HEARTBEAT_MS = 15_000
+export const DIRECTORY_TTL_MS = 45_000
+
+const ENTRY = "dir:"
+const POD = "pod:"
+
+/** Where a connection lives. */
+export interface DirectoryEntry {
+  readonly pod: string
+  readonly machineID: string
+}
+
+export interface ConnDirectoryOptions {
+  readonly kv: Kv
+  /** This pod (HUB_POD_ID). */
+  readonly podID: string
+  readonly timers?: Timers
+  readonly heartbeatMs?: number
+  readonly ttlMs?: number
+  /** Called when a Kv write fails (default: logged). The directory keeps going; the next heartbeat retries. */
+  readonly onError?: (error: unknown) => void
+}
+
+export class ConnDirectory {
+  readonly podID: string
+  private readonly kv: Kv
+  private readonly timers: Timers
+  private readonly heartbeatMs: number
+  private readonly ttlMs: number
+  private readonly onError: (error: unknown) => void
+  /** This pod's connections: connID → machineID. */
+  private readonly own = new Map<string, string>()
+  private timer: unknown
+  private running = false
+
+  constructor(options: ConnDirectoryOptions) {
+    this.kv = options.kv
+    this.podID = options.podID
+    this.timers = options.timers ?? realTimers
+    this.heartbeatMs = options.heartbeatMs ?? DIRECTORY_HEARTBEAT_MS
+    this.ttlMs = options.ttlMs ?? DIRECTORY_TTL_MS
+    this.onError = options.onError ?? ((error) => console.error(`[hub] directory: ${error instanceof Error ? error.message : String(error)}`))
+  }
+
+  /** Marks the pod alive and starts the heartbeat. */
+  async start(): Promise<void> {
+    if (this.running) return
+    this.running = true
+    await this.kv.set(POD + this.podID, "1", { px: this.ttlMs })
+    this.arm()
+  }
+
+  /** Stops the heartbeat and removes the pod's entries and liveness key (a clean shutdown). */
+  async stop(): Promise<void> {
+    this.running = false
+    if (this.timer !== undefined) this.timers.clear(this.timer)
+    this.timer = undefined
+    const conns = [...this.own.keys()]
+    this.own.clear()
+    await Promise.all([...conns.map((connID) => this.kv.del(ENTRY + connID)), this.kv.del(POD + this.podID)])
+  }
+
+  /** Records a connection this pod holds. */
+  async register(connID: string, machineID: string): Promise<void> {
+    this.own.set(connID, machineID)
+    await this.write(connID, machineID)
+  }
+
+  /** Removes a connection this pod held (its socket closed). */
+  async unregister(connID: string): Promise<void> {
+    if (!this.own.delete(connID)) return
+    await this.kv.del(ENTRY + connID)
+  }
+
+  /** Where a connection lives; undefined when unknown, expired, or its pod is dead (the entry is then removed). */
+  async lookup(connID: string): Promise<DirectoryEntry | undefined> {
+    const entry = parse(await this.kv.get(ENTRY + connID))
+    if (!entry) return undefined
+    if (entry.pod !== this.podID && !(await this.podAlive(entry.pod))) {
+      await this.forget(connID, entry.pod)
+      return undefined
+    }
+    return entry
+  }
+
+  /** Removes a connection's entry if it still names `pod` (that pod is gone). */
+  async forget(connID: string, pod: string): Promise<void> {
+    const entry = parse(await this.kv.get(ENTRY + connID))
+    if (entry?.pod === pod) await this.kv.del(ENTRY + connID)
+  }
+
+  async podAlive(pod: string): Promise<boolean> {
+    return (await this.kv.get(POD + pod)) !== undefined
+  }
+
+  /** This pod's registered connections. */
+  local(): string[] {
+    return [...this.own.keys()]
+  }
+
+  private write(connID: string, machineID: string): Promise<boolean> {
+    return this.kv.set(ENTRY + connID, JSON.stringify({ pod: this.podID, machineID } satisfies DirectoryEntry), { px: this.ttlMs })
+  }
+
+  private arm(): void {
+    this.timer = this.timers.set(() => void this.beat(), this.heartbeatMs)
+  }
+
+  /** Refreshes the pod's liveness and every entry it holds, then re-arms. */
+  private async beat(): Promise<void> {
+    if (!this.running) return
+    try {
+      await this.kv.set(POD + this.podID, "1", { px: this.ttlMs })
+      await Promise.all([...this.own].map(([connID, machineID]) => this.write(connID, machineID)))
+    } catch (error) {
+      this.onError(error)
+    }
+    if (this.running) this.arm()
+  }
+}
+
+function parse(raw: string | undefined): DirectoryEntry | undefined {
+  if (raw === undefined) return undefined
+  try {
+    const value = JSON.parse(raw) as Partial<DirectoryEntry>
+    return typeof value?.pod === "string" && typeof value.machineID === "string" ? { pod: value.pod, machineID: value.machineID } : undefined
+  } catch {
+    return undefined
+  }
+}

package/src/central/cluster/feed.ts

@@ -0,0 +1,84 @@
+// A per-user feed of items (stored agent messages, launch changes) that a cluster shares over the Bus (design §9,
+// decision vkxkqiuf §6). publish() runs this pod's listeners at once and, once attached, sends the item to the other
+// pods, whose listeners run when it arrives (at most once). A listener subscribed with `origin: true` runs only for
+// items published on its own pod: work that must happen once per item (the console broadcast, which the live
+// fan-out already sends to every pod; webhooks; closing a launch's connections, which pod RPC already reaches).
+// Unattached (one process), every listener runs for every item, as before.
+import type { Bus, Unsubscribe } from "../backend/types"
+
+export type FeedListener<T> = (userID: string, item: T) => void
+
+export interface FeedSubscribeOptions {
+  /** Only items published on this pod. */
+  readonly origin?: boolean
+}
+
+export interface FeedAttachOptions {
+  readonly bus: Bus
+  /** This pod (HUB_POD_ID). */
+  readonly podID: string
+}
+
+interface Envelope<T> {
+  readonly pod: string
+  readonly userID: string
+  readonly item: T
+}
+
+export class Feed<T> {
+  private readonly listeners = new Map<FeedListener<T>, boolean>()
+  private attached?: { readonly bus: Bus; readonly podID: string; readonly unsubscribe: Promise<Unsubscribe> }
+
+  constructor(
+    /** The Bus topic. */
+    private readonly topic: string,
+    /** Logs a throwing listener (never stops the others). */
+    private readonly report: (cause: unknown) => void,
+  ) {}
+
+  subscribe(listener: FeedListener<T>, options: FeedSubscribeOptions = {}): () => void {
+    // Wrapped so subscribing one function twice gives two independent subscriptions.
+    const own: FeedListener<T> = (userID, item) => listener(userID, item)
+    this.listeners.set(own, options.origin === true)
+    return () => void this.listeners.delete(own)
+  }
+
+  publish(userID: string, item: T): void {
+    this.run(userID, item, true)
+    const attached = this.attached
+    if (attached) {
+      attached.bus.publish(this.topic, { pod: attached.podID, userID, item } satisfies Envelope<T>).catch((cause: unknown) => this.report(cause))
+    }
+  }
+
+  /** Shares the feed with the other pods on the Bus. */
+  async attach(options: FeedAttachOptions): Promise<void> {
+    if (this.attached) return
+    const unsubscribe = options.bus.subscribe(this.topic, (raw) => {
+      const envelope = raw as Envelope<T> | null
+      if (typeof envelope !== "object" || envelope === null || typeof envelope.userID !== "string") return
+      if (envelope.pod === options.podID) return
+      this.run(envelope.userID, envelope.item, false)
+    })
+    this.attached = { ...options, unsubscribe }
+    await unsubscribe
+  }
+
+  /** Stops sharing; the Bus subscription is dropped once it is made (a Bus that never connected must not hold up a stop). */
+  async detach(): Promise<void> {
+    const attached = this.attached
+    this.attached = undefined
+    void attached?.unsubscribe.then((off) => off()).catch((cause: unknown) => this.report(cause))
+  }
+
+  private run(userID: string, item: T, local: boolean): void {
+    for (const [listener, origin] of [...this.listeners]) {
+      if (origin && !local) continue
+      try {
+        listener(userID, item)
+      } catch (cause) {
+        this.report(cause)
+      }
+    }
+  }
+}