@statorjs/stator 2.10.0-next.0 → 2.10.0-next.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@statorjs/stator",
-  "version": "2.10.0-next.0",
+  "version": "2.10.0-next.1",
   "description": "Server-canonical web framework: business logic in composable state machines, UI as a thin renderer binding machine outputs to DOM positions. Ships TypeScript source, loaded natively by its own toolchain — no build step.",
   "license": "MIT",
   "type": "module",
@@ -95,7 +95,7 @@
     "pino-pretty": "^13.1.3",
     "vite": "^6.0.0",
     "vitest": "^2.1.0",
-    "@statorjs/stator": "2.10.0-next.0"
+    "@statorjs/stator": "2.10.0-next.1"
   },
   "scripts": {
     "typecheck": "tsc --noEmit",

package/src/build/copy-set.ts

@@ -3,6 +3,7 @@ import { readdir, readFile, stat } from 'node:fs/promises'
 import { builtinModules } from 'node:module'
 import { dirname, isAbsolute, join, relative, resolve } from 'node:path'
 import { build } from 'esbuild'
+import ts from 'typescript'
 import { compile, regionResolverFor } from '../compiler/index.ts'
 
 /**
@@ -55,18 +56,10 @@ const NEVER_SOURCE = new Set(['node_modules', 'dist', 'tests', 'test', '__tests_
 
 const CODE = /\.(ts|tsx|mts|cts|js|mjs|cjs|jsx|stator)$/
 const BUILTIN = new Set([...builtinModules, ...builtinModules.map((m) => `node:${m}`)])
-/** `new URL('./x', import.meta.url)` — invisible to the bundler, so read the
- *  literal directly. Same limit as the client seam: relative string literals.
- *  (Injecting an import instead does NOT work: esbuild's TS loader elides an
- *  import whose binding is unused, so the injected one disappears.) */
-const URL_ASSET_RE = /new\s+URL\(\s*(['"])(\.{1,2}\/[^'"\n]+)\1\s*,\s*import\.meta\.url\s*\)/g
 /** A `node_modules` path segment, either separator. Anything under one is a
  *  dependency, never app source — and a real app's `node_modules` sits INSIDE
  *  its root, so root containment alone cannot tell them apart. */
 const NODE_MODULES = /[\\/]node_modules[\\/]/
-/** `import(` not followed by a string or template literal. */
-const OPAQUE_IMPORT_RE = /(?<![.\w$])import\s*\(\s*([^'"`)\s])/g
-
 const norm = (p: string): string => p.replace(/\\/g, '/')
 
 export interface UntracedImport {
@@ -186,19 +179,67 @@ export async function resolveCopySet(opts: CopySetOptions): Promise<CopySet> {
   const unresolved = new Set<string>()
   const untraced: UntracedImport[] = []
 
-  /** Record the non-import dependencies a source text carries. */
+  /**
+   * The two dependencies a module carries that the bundler's own graph misses,
+   * read from the SYNTAX rather than the text.
+   *
+   * Both used to be regex scans over raw source, which cannot tell code from
+   * prose: a comment explaining why the app avoids `import(name)`, or a string
+   * documenting the pattern, was picked up as a real untraceable import and
+   * failed the build. Found by dogfooding, and the reason this parses. Comments
+   * and string contents are simply not part of the tree.
+   */
   const scan = (text: string, file: string): string => {
-    for (const m of text.matchAll(URL_ASSET_RE)) assets.add(resolve(dirname(file), m[2]!))
-    for (const m of text.matchAll(OPAQUE_IMPORT_RE)) {
-      untraced.push({
-        file: norm(relative(realRoot, file)),
-        line: text.slice(0, m.index).split('\n').length,
-        source: `${text
-          .slice(m.index, m.index + 60)
-          .split('\n')[0]!
-          .trim()}…`,
-      })
+    const sourceFile = ts.createSourceFile(
+      file,
+      text,
+      ts.ScriptTarget.Latest,
+      false,
+      /\.(tsx|jsx)$/.test(file) ? ts.ScriptKind.TSX : ts.ScriptKind.TS,
+    )
+
+    /** `import.meta.url`. */
+    const isImportMetaUrl = (node: ts.Node): boolean =>
+      ts.isPropertyAccessExpression(node) &&
+      node.name.text === 'url' &&
+      ts.isMetaProperty(node.expression) &&
+      node.expression.keywordToken === ts.SyntaxKind.ImportKeyword
+
+    /** A specifier the bundler can follow: a string literal, or a template —
+     *  esbuild glob-expands one with a fixed prefix, pulling in every match. */
+    const analysable = (node: ts.Node): boolean =>
+      ts.isStringLiteralLike(node) || ts.isTemplateExpression(node)
+
+    const visit = (node: ts.Node): void => {
+      // `new URL('./x', import.meta.url)` — an asset reference no import graph
+      // sees. String-literal specifiers only, the same limit as the client seam.
+      if (ts.isNewExpression(node) && ts.isIdentifier(node.expression)) {
+        if (node.expression.text === 'URL') {
+          const [spec, meta] = node.arguments ?? []
+          if (spec && ts.isStringLiteralLike(spec) && spec.text.startsWith('.')) {
+            if (meta && isImportMetaUrl(meta)) assets.add(resolve(dirname(file), spec.text))
+          }
+        }
+      }
+      // `import(expr)` the bundler cannot follow. It says nothing about these,
+      // so this is the only place they are caught.
+      if (ts.isCallExpression(node) && node.expression.kind === ts.SyntaxKind.ImportKeyword) {
+        const arg = node.arguments[0]
+        if (!arg || !analysable(arg)) {
+          const start = node.getStart(sourceFile)
+          untraced.push({
+            file: norm(relative(realRoot, file)),
+            line: sourceFile.getLineAndCharacterOfPosition(start).line + 1,
+            source: `${text
+              .slice(start, start + 60)
+              .split('\n')[0]!
+              .trim()}…`,
+          })
+        }
+      }
+      ts.forEachChild(node, visit)
     }
+    ts.forEachChild(sourceFile, visit)
     return text
   }
 

package/src/server/http.ts

@@ -22,7 +22,7 @@ import { isStatorQueryRoute, type RouteDefinition } from './routing.ts'
 import { CLAIMS_KEY, getOrCreateSessionId, getSessionState } from './session.ts'
 import { withSessionLock } from './session-lock.ts'
 import { SessionRuntime } from './session-runtime.ts'
-import { fanOut, registerConnection, unregisterConnection } from './sse.ts'
+import { enqueue, fanOut, registerConnection, unregisterConnection } from './sse.ts'
 
 const httpLog = scopedLogger('http')
 
@@ -423,7 +423,7 @@ export async function buildHonoApp(config: HttpConfig): Promise<Hono> {
       // otherwise never reach this page.
       const sync = initialSyncPatches(renderState, runtime)
       if (sync.length > 0) {
-        await conn.send(JSON.stringify({ patches: sync }))
+        enqueue(conn, JSON.stringify({ patches: sync }))
       }
 
       // Heartbeat every 25s. This is a real DATA message, not a comment, on
@@ -433,9 +433,11 @@ export async function buildHonoApp(config: HttpConfig): Promise<Hono> {
       // ever arrives, no error fires) unless liveness is OBSERVABLE. The
       // runtime tracks last-message time and reconnects when pings stop.
       const keepAlive = setInterval(() => {
-        void conn.send('{"ping":true}').catch(() => {
-          // Stream closed; the abort handler cleans up.
-        })
+        // Through the same queue fan-out uses, so it is bounded like any other
+        // write: a half-open socket never settles its writes, and the heartbeat
+        // is what reaps it when the app is otherwise idle and no dispatch would
+        // ever notice.
+        enqueue(conn, '{"ping":true}')
       }, config.ssePingMs ?? 25_000)
 
       try {
@@ -547,7 +549,9 @@ export async function buildHonoApp(config: HttpConfig): Promise<Hono> {
         // `touched`) — so it isn't re-created and re-fired next request.
         await runtime.persistTouched(new Set([...touched, ...runtime.entryFiredMachines()]))
 
-        await fanOut(touched, {
+        // Queues patches on every affected channel and returns; no socket is
+        // awaited under the lock.
+        const { originatorQueued } = await fanOut(touched, {
           sessionId,
           originClientId: c.req.header('X-Stator-Client'),
         })
@@ -557,7 +561,15 @@ export async function buildHonoApp(config: HttpConfig): Promise<Hono> {
         // normal event path in server/effects.ts.
         scheduleSessionEffects(runtime, config.store, sessionId)
 
-        const envelope = { patches, directives: [], committed: touched.size > 0 }
+        // This page's patches normally ride the response. When its live
+        // channel already had a backlog, fan-out queued them behind it instead
+        // (two channels to one page must not reorder a positional list op), so
+        // the response carries none.
+        const envelope = {
+          patches: originatorQueued ? [] : patches,
+          directives: [],
+          committed: touched.size > 0,
+        }
         if (body.eventId) record(sessionId, body.eventId, JSON.stringify(envelope))
         return c.json(envelope)
       } finally {

package/src/server/sse.ts

@@ -17,13 +17,21 @@ const sseLog = scopedLogger('sse')
  * recompute will diff against when fan-out fires. Each push updates
  * `lastValue` for the bindings touched by the push, so subsequent pushes
  * only emit deltas.
+ *
+ * Writes never happen inline. Everything outbound — fan-out patches, the
+ * initial sync, the heartbeat, the dev broadcast — goes through `enqueue`,
+ * and one drain loop per connection owns the socket. A request or a lock is
+ * therefore never held across a socket write, and per-connection order is a
+ * property of the queue rather than of who awaited what.
  */
 export interface Connection {
   id: string
   sessionId: string
   /** The browser page-load identity (client-generated). Lets fan-out
-   *  recognize a dispatch's OWN connection: its baseline is advanced but
-   *  nothing is sent — the POST response already delivered those patches. */
+   *  recognize a dispatch's OWN connection: its baseline is advanced and,
+   *  when nothing is queued for it, nothing is sent — the POST response
+   *  delivers those patches. With a backlog they ride the queue instead, so
+   *  the two channels cannot reorder a positional list op. */
   clientId?: string
   routeKey: string
   route: RouteDefinition
@@ -39,14 +47,29 @@ export interface Connection {
    *  transport; absent in tests that register a bare connection. */
   close?: () => void
   closed: boolean
+  /** Outbound writes waiting for the drain loop, oldest first. */
+  queue: string[]
+  /** Bytes waiting in `queue`. The item being written is not counted. */
+  queuedBytes: number
+  /** True while the drain loop owns the socket — an item is in flight. */
+  draining: boolean
 }
 
+type ConnectionInit = Omit<Connection, 'id' | 'closed' | 'queue' | 'queuedBytes' | 'draining'>
+
 const connections = new Map<string, Connection>()
 let nextId = 0
 
-export function registerConnection(init: Omit<Connection, 'id' | 'closed'>): Connection {
+export function registerConnection(init: ConnectionInit): Connection {
   const id = `sse${nextId++}`
-  const conn: Connection = { ...init, id, closed: false }
+  const conn: Connection = {
+    ...init,
+    id,
+    closed: false,
+    queue: [],
+    queuedBytes: 0,
+    draining: false,
+  }
   // A page-load holds at most one channel per route, so an existing
   // connection with the same (clientId, routeKey) is a corpse whose abort we
   // never observed — a half-open socket, or a reconnect that raced its own
@@ -73,21 +96,16 @@ export function registerConnection(init: Omit<Connection, 'id' | 'closed'>): Con
 }
 
 /**
- * Push a pre-serialized envelope to every open connection, bypassing the
+ * Queue a pre-serialized envelope for every open connection, bypassing the
  * recompute path. The dev server's rebuild/error signals ride this so a dev
  * page holds ONE event-stream instead of two — see `dev-native.ts`. Not a
  * fan-out: no diffing, no baseline advance, no per-connection filtering.
  */
-export async function broadcastEnvelope(payload: unknown): Promise<void> {
+export function broadcastEnvelope(payload: unknown): void {
   if (connections.size === 0) return
   const data = JSON.stringify(payload)
   for (const conn of connections.values()) {
-    if (conn.closed) continue
-    try {
-      await conn.send(data)
-    } catch (err) {
-      sseLog.warn({ id: conn.id, err: String(err) }, 'broadcast failed')
-    }
+    if (!conn.closed) enqueue(conn, data)
   }
 }
 
@@ -95,6 +113,10 @@ export function unregisterConnection(id: string): void {
   const conn = connections.get(id)
   if (!conn) return
   conn.closed = true
+  // Whatever was waiting is discarded with the socket: the client reconnects
+  // and resyncs. The drain loop, if mid-write, sees `closed` and exits.
+  conn.queue.length = 0
+  conn.queuedBytes = 0
   conn.runtime.dispose()
   connections.delete(id)
   sseLog.debug(
@@ -107,6 +129,133 @@ export function activeConnectionCount(): number {
   return connections.size
 }
 
+/** How long one write to one connection may take before the connection is
+ *  treated as dead. `STATOR_SSE_WRITE_TIMEOUT_MS` overrides it. */
+const DEFAULT_WRITE_TIMEOUT_MS = 5_000
+
+function writeTimeoutMs(): number {
+  const raw = process.env.STATOR_SSE_WRITE_TIMEOUT_MS
+  if (raw === undefined || raw === '') return DEFAULT_WRITE_TIMEOUT_MS
+  const ms = Number(raw)
+  return Number.isFinite(ms) && ms > 0 ? ms : DEFAULT_WRITE_TIMEOUT_MS
+}
+
+/** Bytes a connection may hold WAITING behind an in-flight write before it is
+ *  treated as dead. The item being written is covered by the write deadline,
+ *  not by this, and a single waiting item never trips it however large — an
+ *  initial sync of a big table is one item. `STATOR_SSE_QUEUE_MAX_BYTES`
+ *  overrides it. */
+const DEFAULT_QUEUE_MAX_BYTES = 1_048_576
+
+function queueMaxBytes(): number {
+  const raw = process.env.STATOR_SSE_QUEUE_MAX_BYTES
+  if (raw === undefined || raw === '') return DEFAULT_QUEUE_MAX_BYTES
+  const bytes = Number(raw)
+  return Number.isFinite(bytes) && bytes > 0 ? bytes : DEFAULT_QUEUE_MAX_BYTES
+}
+
+/** Whether anything is queued or in flight for this connection. */
+export function hasBacklog(conn: Connection): boolean {
+  return conn.draining || conn.queue.length > 0
+}
+
+/**
+ * Queue one envelope for a connection and return at once. The drain loop
+ * writes it when everything ahead of it has been written, so order per
+ * connection is FIFO by construction. Overflow drops the connection, the
+ * same policy as a write that times out: the client reconnects and resyncs.
+ */
+export function enqueue(conn: Connection, data: string): void {
+  if (conn.closed) return
+  conn.queue.push(data)
+  conn.queuedBytes += Buffer.byteLength(data)
+  if (conn.queue.length > 1 && conn.queuedBytes > queueMaxBytes()) {
+    sseLog.warn(
+      {
+        id: conn.id,
+        sid: conn.sessionId,
+        route: conn.routeKey,
+        waiting: conn.queue.length,
+        bytes: conn.queuedBytes,
+        max: queueMaxBytes(),
+      },
+      'sse queue overflowed — dropping the connection',
+    )
+    unregisterConnection(conn.id)
+    conn.close?.()
+    return
+  }
+  if (!conn.draining) void drain(conn)
+}
+
+/** The one writer per connection. Runs until the queue is empty or the
+ *  connection closes under it (a timed-out write, an overflow, an abort). */
+async function drain(conn: Connection): Promise<void> {
+  conn.draining = true
+  try {
+    while (!conn.closed && conn.queue.length > 0) {
+      const data = conn.queue.shift()!
+      conn.queuedBytes -= Buffer.byteLength(data)
+      await pushToConnection(conn, data)
+    }
+  } finally {
+    conn.draining = false
+  }
+}
+
+/**
+ * Write one envelope to one connection, bounded. The drain loop's single
+ * write; nothing else touches the socket.
+ *
+ * A write to a live socket resolves; a write to a closed one rejects. A write
+ * to a HALF-OPEN one does neither once its buffer is full — when a remote
+ * laptop sleeps or a NAT drops the mapping, no FIN ever arrives, the socket
+ * still looks writable, and the promise simply never settles. Fan-out used to
+ * await these serially with no deadline, so one such connection stalled the
+ * loop and every connection registered after it received nothing until the
+ * kernel gave up on the socket (minutes) or the same page reconnected and
+ * evicted it. Under the session lock the same stall ran into the lock's 30s
+ * backstop, failing the dispatching request each time.
+ *
+ * On a timeout the connection is dropped rather than retried: the client
+ * reconnects and gets a full resync, which is cheap by design. Note what the
+ * deadline measures: a write to a socket with buffer space resolves at once
+ * regardless of the peer, so this fires only when the buffer has stayed full
+ * for the whole window — not on a merely slow client. And because the clock
+ * starts when the drain loop issues the write, not when the item was queued,
+ * a burst behind a slow but healthy socket cannot trip it.
+ */
+export async function pushToConnection(conn: Connection, data: string): Promise<Outcome> {
+  let timer: ReturnType<typeof setTimeout> | undefined
+  try {
+    const outcome = await Promise.race<Outcome>([
+      conn.send(data).then((): Outcome => 'sent'),
+      new Promise<Outcome>((settle) => {
+        timer = setTimeout(() => settle('timeout'), writeTimeoutMs())
+        timer.unref?.()
+      }),
+    ])
+    if (outcome === 'timeout') {
+      sseLog.warn(
+        { id: conn.id, sid: conn.sessionId, route: conn.routeKey, ms: writeTimeoutMs() },
+        'sse write timed out — dropping the connection',
+      )
+      unregisterConnection(conn.id)
+      conn.close?.()
+    }
+    return outcome
+  } catch (err) {
+    // Bumped from debug to warn — silent push failures were why "30-50%
+    // delivery" was invisible in production logs.
+    sseLog.warn({ id: conn.id, sid: conn.sessionId, err: String(err) }, 'sse push failed')
+    return 'failed'
+  } finally {
+    clearTimeout(timer)
+  }
+}
+
+type Outcome = 'sent' | 'timeout' | 'failed'
+
 /**
  * Hang up every open connection — the shutdown path. A live response never ends
  * on its own, so `server.close()` cannot complete until these do; see
@@ -125,24 +274,31 @@ export function closeLiveConnections(): number {
 /**
  * After a state-changing dispatch settles, iterate every open connection
  * whose route's `reads:` intersects `touched`, recompute against that
- * connection's slot map, and push any resulting patches over its event
+ * connection's slot map, and queue any resulting patches on its event
  * stream. Recompute mutates the connection's `renderState.lastValue`s so
- * the next push is correctly diffed.
+ * the next push is correctly diffed. Returns as soon as everything is
+ * queued: no socket is awaited here, so a dispatch never waits on another
+ * page's network.
  *
  * Connections from any session receive pushes — that's the point: an
  * admin tab on session C sees updates triggered by session A's POSTs.
+ *
+ * `originatorQueued` tells the dispatch path whether the acting page's own
+ * patches were queued on its channel (because it already had a backlog)
+ * rather than left to the POST response. When true, the response must carry
+ * none, or the same positional list op lands twice.
  */
 export async function fanOut(
   touched: ReadonlySet<string>,
   source?: { sessionId?: string; originClientId?: string },
-): Promise<void> {
-  if (touched.size === 0 || connections.size === 0) return
+): Promise<{ originatorQueued: boolean }> {
+  if (touched.size === 0 || connections.size === 0) return { originatorQueued: false }
 
-  let pushedCount = 0
+  let enqueued = 0
   let skippedNoIntersect = 0
   let skippedNoPatches = 0
   let skippedOriginator = 0
-  let failed = 0
+  let originatorQueued = false
 
   // Expand through the reverse-reads graph once per fan-out: machines whose
   // SELECTORS derive from a touched machine must re-diff even though their
@@ -188,42 +344,43 @@ export async function fanOut(
       continue
     }
 
-    // The dispatching page's own connection: the POST response already
-    // delivered this diff. The recompute above advanced the baseline (so the
-    // NEXT push diffs correctly); sending would double-apply — and keyed
-    // insert/remove/move ops are not idempotent.
-    if (
+    // The dispatching page's own connection. With nothing queued for it, the
+    // POST response delivers this diff: the recompute above advanced the
+    // baseline (so the NEXT push diffs correctly) and sending would
+    // double-apply — keyed insert/remove/move ops are not idempotent. With a
+    // backlog, the response would overtake what is still queued, and a
+    // positional op applied out of order corrupts a list rather than merely
+    // staling it — so the diff rides the queue and the response carries none.
+    const isOriginator =
       source?.originClientId !== undefined &&
       conn.clientId !== undefined &&
       conn.clientId === source.originClientId
-    ) {
-      skippedOriginator++
-      continue
+    if (isOriginator) {
+      if (!hasBacklog(conn)) {
+        skippedOriginator++
+        continue
+      }
+      originatorQueued = true
     }
 
-    try {
-      await conn.send(JSON.stringify({ patches }))
-      pushedCount++
-    } catch (err) {
-      // Bumped from debug to warn — silent push failures were why "30-50%
-      // delivery" was invisible in production logs.
-      failed++
-      sseLog.warn({ id: conn.id, sid: conn.sessionId, err: String(err) }, 'sse push failed')
-    }
+    enqueue(conn, JSON.stringify({ patches }))
+    enqueued++
   }
 
   // Log every fan-out at debug so the user can see touched-machines flow and
-  // correlate against client-side inspector entries — noisy for prod, so debug.
+  // correlate against client-side inspector entries — noisy for prod, so
+  // debug. Write outcomes are logged by the drain loop as they happen.
   sseLog.debug(
     {
       touched: [...touched],
       total: connections.size,
-      pushed: pushedCount,
+      enqueued,
       skippedNoIntersect,
       skippedNoPatches,
       skippedOriginator,
-      failed,
+      originatorQueued,
     },
     'fan-out',
   )
+  return { originatorQueued }
 }