@defrex/autobuild 0.8.0

package/packages/core/src/store/streams/session-writer.ts

@@ -0,0 +1,306 @@
+/**
+ * The session-stream batching writer (SPEC §9): the store-side sink the
+ * build-runner hands to a runtime registration's `openSessionStream`
+ * capability. Buffers appended protocol parts and flushes them to the
+ * store on a short cadence — well inside the one-second latency bound —
+ * instead of one request per delta.
+ *
+ * Failure containment is the contract: `append` NEVER throws, an append
+ * error retries on the next tick, and after three consecutive failed
+ * flushes the writer stops streaming the session, best-effort closes the
+ * stream as `aborted` (first close wins), and reports once through the
+ * diagnostics hook. A failing stream path can therefore never fail a turn,
+ * a deposit, or a phase. `close` is flush-then-close, idempotent, first
+ * outcome wins.
+ *
+ * Per-part payloads above the documented `STREAM_PART_MAX_BYTES` are
+ * truncated on a UTF-8 boundary and followed by a `data-ab-truncation`
+ * part naming the omitted byte count, so no single delta can exceed the
+ * store's batch ceiling.
+ *
+ * Flushes are single-flight: at most one drain runs at a time, so parts are
+ * appended to the store strictly in buffer order (per-stream sequencing can
+ * never reorder), retries re-queue at the front under the same discipline,
+ * and `close` waits out any in-flight drain before closing the stream.
+ */
+import type { SessionStreamSink } from '../../ports/types'
+import { STREAM_PART_MAX_BYTES, truncationPart } from '../../ports/runner/stream-parts'
+import type { JsonRecord } from '../../ports/runner/json-record'
+import {
+  serializedBatchSize,
+  STREAM_BATCH_MAX_BYTES,
+  type StreamOutcome,
+  type StreamPart,
+  type StreamRecord,
+  type StreamScope,
+} from './types'
+
+export interface SessionStreamSinkOptions {
+  /** Any BuildStore handle scoped so `createStream(scope)` succeeds — the
+   * build-runner passes its build-scoped handle. */
+  store: {
+    createStream(scope: StreamScope, label: string): Promise<StreamRecord>
+    appendStreamParts(streamId: string, parts: StreamPart[]): Promise<unknown>
+    closeStream(streamId: string, outcome: StreamOutcome): Promise<StreamRecord>
+  }
+  scope: StreamScope
+  /** Called at most once per sink, when streaming gives up. */
+  onDiagnostic?: (message: string) => void
+  /** Flush cadence in milliseconds. Default 250. */
+  flushMs?: number
+  /** Injectable scheduler for tests. Default: an unref'ed interval. */
+  schedule?: (flush: () => void, ms: number) => () => void
+}
+
+/** Consecutive failed flushes before the writer stops streaming a session. */
+const MAX_FLUSH_FAILURES = 3
+
+/**
+ * Cut `text` to at most `maxBytes` UTF-8 bytes without splitting a
+ * multi-byte sequence. Returns the prefix and the omitted byte count.
+ */
+function truncateUtf8(text: string, maxBytes: number): { text: string; omitted: number } {
+  const bytes = new TextEncoder().encode(text)
+  if (bytes.byteLength <= maxBytes) return { text, omitted: 0 }
+  let cut = maxBytes
+  while (cut > 0 && (bytes[cut]! & 0xc0) === 0x80) cut -= 1
+  return {
+    text: new TextDecoder().decode(bytes.slice(0, cut)),
+    omitted: bytes.byteLength - cut,
+  }
+}
+
+/**
+ * Bound one part's payload against `STREAM_PART_MAX_BYTES`. Text and
+ * reasoning deltas truncate their `delta`; tool inputs/outputs truncate
+ * their JSON serialization, keeping the payload's JSON kind (objects stay
+ * objects, strings stay strings); the prompt part truncates `data.text`.
+ * Anything else passes through untouched. Returns the (possibly replaced)
+ * part plus a `data-ab-truncation` part when bytes were omitted.
+ */
+function boundPart(part: StreamPart, maxBytes: number): StreamPart[] {
+  const record = part as JsonRecord
+  switch (part.type) {
+    case 'text-delta':
+    case 'reasoning-delta': {
+      const delta = record.delta
+      if (typeof delta !== 'string') return [part]
+      const { text, omitted } = truncateUtf8(delta, maxBytes)
+      return omitted === 0
+        ? [part]
+        : [{ ...record, type: part.type, delta: text }, truncationPart(omitted)]
+    }
+    case 'tool-input-available':
+    case 'tool-output-available': {
+      const key = part.type === 'tool-input-available' ? 'input' : 'output'
+      const value = record[key]
+      if (value === undefined) return [part]
+      // Strings truncate as strings; objects stay objects (the shape the
+      // part type carries) with the truncated serialization inside.
+      if (typeof value === 'string') {
+        const { text, omitted } = truncateUtf8(value, maxBytes)
+        return omitted === 0
+          ? [part]
+          : [{ ...record, type: part.type, [key]: text }, truncationPart(omitted)]
+      }
+      const serialized = JSON.stringify(value)
+      if (typeof serialized !== 'string') return [part]
+      const { text, omitted } = truncateUtf8(serialized, maxBytes)
+      return omitted === 0
+        ? [part]
+        : [{ ...record, type: part.type, [key]: { truncated: text } }, truncationPart(omitted)]
+    }
+    default: {
+      // data-ab-prompt carries the turn's prompt in data.text; other data
+      // parts are small bracket metadata and pass through.
+      if (part.type !== 'data-ab-prompt') return [part]
+      const data = record.data
+      if (typeof data !== 'object' || data === null) return [part]
+      const text = (data as JsonRecord).text
+      if (typeof text !== 'string') return [part]
+      const { text: bounded, omitted } = truncateUtf8(text, maxBytes)
+      if (omitted === 0) return [part]
+      return [
+        { ...record, type: part.type, data: { ...(data as JsonRecord), text: bounded } },
+        truncationPart(omitted),
+      ]
+    }
+  }
+}
+
+/**
+ * Create the bracket-scoped sink. `open` creates the store stream and
+ * returns its id; appends buffer until then and flush on the cadence.
+ */
+export function createSessionStreamSink(options: SessionStreamSinkOptions): SessionStreamSink {
+  const { store, scope } = options
+  const flushMs = options.flushMs ?? 250
+  const maxPartBytes = STREAM_PART_MAX_BYTES
+
+  let streamId: string | undefined
+  let openError: unknown
+  const buffer: StreamPart[] = []
+  let flushFailures = 0
+  let defunct = false
+  let diagnosticSent = false
+  let closed = false
+  let firstOutcome: StreamOutcome | undefined
+  let cancelSchedule: (() => void) | undefined
+  /** Single-flight drain discipline: at most one `doFlush` runs at a time;
+   * everything else joins the in-flight promise, so the shared buffer is
+   * never mutated by two concurrent flushes and per-stream part order is
+   * the store-arrival order. */
+  let inFlight: Promise<void> | undefined
+
+  const schedule =
+    options.schedule ??
+    ((flush, ms) => {
+      const timer = setInterval(flush, ms)
+      timer.unref?.()
+      return () => clearInterval(timer)
+    })
+
+  function onDiagnosticOnce(message: string): void {
+    if (diagnosticSent) return
+    diagnosticSent = true
+    options.onDiagnostic?.(message)
+  }
+
+  function ensureTimer(): void {
+    if (cancelSchedule !== undefined || defunct || closed) return
+    cancelSchedule = schedule(() => void requestFlush(), flushMs)
+  }
+
+  /** One drain of the buffer. Never overlapping: it only starts when no
+   * drain is in flight (`requestFlush` gates that), so the shared buffer is
+   * never mutated by two concurrent flushes. */
+  async function doFlush(): Promise<void> {
+    if (streamId === undefined || defunct || closed) return
+    // Split the buffer into store-sized batches (parts were already
+    // per-part bounded at append time, so every batch eventually lands).
+    // Parts appended mid-drain are picked up by the loop's re-check, so a
+    // drain ends only when the buffer is empty or the writer gave up.
+    while (streamId !== undefined && !defunct && !closed && buffer.length > 0) {
+      const batch: StreamPart[] = [buffer[0]!]
+      buffer.shift()
+      while (
+        buffer.length > 0 &&
+        serializedBatchSize([...batch, buffer[0]!]) <= STREAM_BATCH_MAX_BYTES
+      ) {
+        batch.push(buffer.shift()!)
+      }
+      try {
+        await store.appendStreamParts(streamId, batch)
+        flushFailures = 0
+      } catch (error) {
+        // Put the parts back in order and retry on the next tick.
+        buffer.unshift(...batch)
+        flushFailures += 1
+        if (flushFailures >= MAX_FLUSH_FAILURES) {
+          defunct = true
+          cancelSchedule?.()
+          cancelSchedule = undefined
+          onDiagnosticOnce(
+            `session stream "${streamId}" stopped after ${MAX_FLUSH_FAILURES} failed appends: ` +
+              `${error instanceof Error ? error.message : String(error)}`,
+          )
+          // Reentrant close: this drain is the current call stack, so
+          // `closeSink` skips its own drain (`defunct` guards the loop).
+          await closeSink('aborted')
+        }
+        return
+      }
+    }
+  }
+
+  /** The one way to start (or join) a drain. Never overlaps a running one. */
+  function requestFlush(): Promise<void> {
+    if (inFlight !== undefined) return inFlight
+    const run = doFlush()
+    const tracked = run.finally(() => {
+      if (inFlight === tracked) inFlight = undefined
+    })
+    inFlight = tracked
+    return tracked
+  }
+
+  /** First-outcome-wins, flush-then-close, best-effort. An undeliverable
+   * buffer at close time means the stream's content is incomplete: it closes
+   * `aborted` with one diagnostic, whatever outcome was requested. */
+  async function closeSink(outcome: StreamOutcome): Promise<void> {
+    if (closed || firstOutcome !== undefined) return
+    firstOutcome = outcome
+    cancelSchedule?.()
+    cancelSchedule = undefined
+    if (streamId !== undefined) {
+      try {
+        // Flush-then-close under the single-flight discipline: wait out any
+        // in-flight drain, then keep draining until the buffer is empty —
+        // or the writer has given up (defunct, or a failed batch awaiting
+        // its next-tick retry, which closes `aborted` below).
+        while (
+          !defunct &&
+          !closed &&
+          flushFailures === 0 &&
+          (inFlight !== undefined || buffer.length > 0)
+        ) {
+          if (inFlight !== undefined) {
+            await inFlight
+          } else {
+            await requestFlush()
+          }
+        }
+      } catch {
+        // flush is defensive; appends already handle their own failures.
+      }
+      let effective = outcome
+      if (flushFailures > 0) {
+        effective = 'aborted'
+        onDiagnosticOnce(
+          `session stream "${streamId}" closed with ${buffer.length} undelivered parts ` +
+            `after ${flushFailures} failed appends`,
+        )
+      }
+      try {
+        await store.closeStream(streamId, effective)
+      } catch (error) {
+        onDiagnosticOnce(
+          `session stream "${streamId}" failed to close: ` +
+            `${error instanceof Error ? error.message : String(error)}`,
+        )
+      }
+    }
+    closed = true
+  }
+
+  return {
+    async open(label) {
+      try {
+        const record = await store.createStream(scope, label)
+        streamId = record.id
+        ensureTimer()
+        // Parts appended while opening flush immediately after the id lands.
+        await requestFlush()
+        return record.id
+      } catch (error) {
+        openError = error
+        throw error
+      }
+    },
+
+    append(parts) {
+      // Never throws: after defunct/close or before a successful open,
+      // parts are dropped or buffered without surfacing an error.
+      if (defunct || closed) return
+      if (openError !== undefined && streamId === undefined) return
+      for (const part of parts) {
+        buffer.push(...boundPart(part, maxPartBytes))
+      }
+      if (streamId !== undefined) ensureTimer()
+    },
+
+    async close(outcome) {
+      await closeSink(outcome)
+    },
+  }
+}

package/packages/core/src/store/streams/types.ts

@@ -0,0 +1,171 @@
+/**
+ * The stream primitive (SPEC §7.6): the BuildStore's third kind of durable
+ * content, alongside the append-only event log of typed facts and the
+ * content-addressed artifacts of bulk bytes. A stream is an append-only,
+ * per-stream sequenced log of chunks — a chunk being a batch of protocol
+ * parts — with an open-then-closed lifecycle that finalizes into an artifact.
+ *
+ * The chunk vocabulary is the Vercel AI SDK UI Message Stream protocol,
+ * version 1 of the current major (AI SDK 7). Storing the protocol's parts
+ * rather than a private shape means a read route can later re-emit them as
+ * Server-Sent Events that the AI SDK's own client hooks consume unchanged,
+ * and a closed stream finalizes into the protocol's `UIMessage[]` document.
+ *
+ * Streams are presentation, never routing: no kernel, engine, reducer, or
+ * dispatcher decision reads stream content; outcomes travel only the typed
+ * CLI. This module — like everything under `store/` — encodes nothing about
+ * this repository's specifics.
+ */
+
+/** The literal format identifier carried on every stream record. */
+export const STREAM_FORMAT = 'ai-ui-message-stream/v1'
+
+/** The serialized-batch ceiling: the same byte bound artifacts enforce. */
+export const STREAM_BATCH_MAX_BYTES = 1_048_576
+
+/** Upper bound for a read's wait, in whole seconds. Larger requests clamp. */
+export const MAX_STREAM_WAIT_SECONDS = 30
+
+/** One protocol part: a JSON object with a nonempty-string `type`. The store
+ * performs no further protocol validation on append — assembly at close does
+ * that, following the protocol for its defined part types and dropping and
+ * counting parts of undefined types. */
+export type StreamPart = { type: string } & Record<string, unknown>
+
+/** A store-assigned, per-stream sequenced batch of parts. */
+export interface StreamChunk {
+  stream: string
+  /** Per-stream sequence assigned by the store, starting at 1. Producers
+   * cannot fake ordering. */
+  seq: number
+  ts: string
+  parts: StreamPart[]
+}
+
+/** Where a stream lives. The vocabulary is closed: builds, the repository
+ * journal, and operator sessions (SPEC §7.1.1) — every turn's content is one
+ * session-scoped stream. */
+export type StreamScope =
+  | { kind: 'build'; build: string }
+  | { kind: 'repo'; repo: string }
+  | { kind: 'session'; session: string }
+
+export type StreamStatus = 'open' | 'closed'
+export type StreamOutcome = 'completed' | 'aborted'
+
+/** Reference to the artifact the close deposited on the owning scope. */
+export interface StreamArtifactRef {
+  kind: string
+  revision: number
+  blobRef: string
+}
+
+export interface StreamRecord {
+  id: string
+  scope: StreamScope
+  /** Caller-supplied label; presentation metadata only. */
+  label: string
+  format: typeof STREAM_FORMAT
+  status: StreamStatus
+  createdAt: string
+  closedAt?: string
+  outcome?: StreamOutcome
+  /** Present once closed: the finalized `UIMessage[]` document's artifact. */
+  artifact?: StreamArtifactRef
+}
+
+export interface StreamRead {
+  chunks: StreamChunk[]
+  status: StreamStatus
+  outcome?: StreamOutcome
+  artifact?: StreamArtifactRef
+}
+
+/** Appending a batch whose serialized size exceeds the ceiling. Typed so the
+ * remote server maps it to `413 {kind:'validation'}` like the artifact
+ * ceiling and the remote client rehydrates it; the message names the bound. */
+export class StreamBatchTooLargeError extends Error {
+  constructor(
+    readonly bytes: number | undefined,
+    readonly maxBytes: number = STREAM_BATCH_MAX_BYTES,
+    message?: string,
+  ) {
+    super(message ?? `stream batch of ${bytes} bytes exceeds the ${maxBytes}-byte ceiling`)
+    this.name = 'StreamBatchTooLargeError'
+  }
+}
+
+/** Appending to a closed stream. Typed so the remote server maps it to
+ * `409 {kind:'conflict'}` and the remote client rehydrates it. The argument
+ * is the stream id locally, or the server's message over the wire. */
+export class StreamClosedError extends Error {
+  constructor(streamOrMessage: string) {
+    super(
+      streamOrMessage.startsWith('stream "')
+        ? streamOrMessage
+        : `stream "${streamOrMessage}" is closed`,
+    )
+    this.name = 'StreamClosedError'
+  }
+}
+
+/** Append validation: a nonempty array of parts, each a JSON object whose
+ * `type` is a nonempty string. Throws a plain `Error` otherwise. */
+export function validateStreamParts(parts: StreamPart[]): void {
+  if (!Array.isArray(parts) || parts.length === 0) {
+    throw new Error('stream parts must be a nonempty array')
+  }
+  for (const part of parts) {
+    if (typeof part !== 'object' || part === null || Array.isArray(part)) {
+      throw new Error('every stream part must be a JSON object')
+    }
+    if (typeof part.type !== 'string' || part.type.length === 0) {
+      throw new Error('every stream part must carry a nonempty string "type"')
+    }
+  }
+}
+
+/** Serialized JSON size of a batch, in bytes — the value compared against
+ * `STREAM_BATCH_MAX_BYTES`. */
+export function serializedBatchSize(parts: StreamPart[]): number {
+  return new TextEncoder().encode(JSON.stringify(parts)).byteLength
+}
+
+/** The read-wait bound: whole seconds, clamped above 30 and below 0. */
+export function clampWaitSeconds(waitSeconds: number): number {
+  return Math.max(0, Math.min(MAX_STREAM_WAIT_SECONDS, Math.floor(waitSeconds)))
+}
+
+/**
+ * The close-time artifact input every adapter deposits: the assembled
+ * `UIMessage[]` document as JSON bytes, kind `stream:<streamId>`, with
+ * metadata naming the stream id, label, scope, outcome, chunk count, and
+ * dropped-part count. Shared so every adapter's finalized artifact is
+ * byte-identical for the same stream.
+ */
+export function streamArtifactInput(
+  stream: string,
+  scope: StreamScope,
+  label: string,
+  outcome: StreamOutcome,
+  document: unknown,
+  chunkCount: number,
+  droppedPartCount: number,
+): {
+  kind: string
+  content: string
+  metadata: Record<string, unknown>
+} {
+  return {
+    kind: `stream:${stream}`,
+    content: JSON.stringify(document),
+    metadata: {
+      stream,
+      label,
+      scope: { ...scope },
+      outcome,
+      chunkCount,
+      droppedPartCount,
+    },
+  }
+}

package/packages/core/src/store/streams/wait.ts

@@ -0,0 +1,127 @@
+/**
+ * The shared read-wait loop (SPEC §7.6): when a read finds no newer chunk and
+ * the stream is open, the adapter honors the wait bound — returning no later
+ * than the bound, but as soon as a chunk is appended or the stream closes.
+ * The memory, SQLite, and PostgreSQL adapters all delegate here so wait
+ * semantics are adapter-uniform; the remote adapter instead delegates the
+ * wait to its backing store over one held-open request.
+ *
+ * A small poll loop rather than a waiter registry: the uniformity across
+ * synchronous (SQLite) and asynchronous (PostgreSQL) backends is worth more
+ * than wake-on-write precision. The loop serves two cadences: held *stream*
+ * reads are presentation content and poll at the ~25 ms `STREAM_WAIT_POLL_MS`;
+ * held *event* reads (build, repository, session — AUT-334/381/383) take the
+ * one-second `EVENT_WAIT_POLL_MS` budget when the adapter passes it per call
+ * (see `EVENT_WAIT_POLL_MS` for which adapters do). Closed streams never wait.
+ *
+ * Both loops accept an optional `signal` (AUT-380): when it aborts, the hold
+ * ends promptly — the inter-poll sleep resolves early, the loop stops, and the
+ * read resolves with its current result instead of polling to the wait bound.
+ * This is what lets a remote server cancel a disconnected peer's backing read
+ * rather than let it poll an unobserved database. Resolving (never rejecting)
+ * keeps every caller on one code path: callers treat the empty result exactly
+ * as they treat the hold expiring.
+ */
+import { clampWaitSeconds, type StreamRead } from './types'
+
+export const STREAM_WAIT_POLL_MS = 25
+
+/**
+ * The one-second held-event poll budget (the hosted per-query budget,
+ * AUT-334/381/383): held event reads — build, repository, and session —
+ * re-query at most once per second, so an append is observed at the next
+ * poll, typically within about one second. Held *stream* reads are excluded:
+ * they are presentation content and stay at the ~25 ms `STREAM_WAIT_POLL_MS`
+ * cadence. This deliberately diverges from `STREAM_WAIT_POLL_MS`; adapters
+ * pass it as `pollMs` on their event-family calls to `readEventsWithWait` —
+ * the PostgreSQL and memory stores do on all three families (AUT-334/381/383).
+ * The SQLite store has not adopted the budget (out of scope for AUT-383), so
+ * its held event reads fall back to the stream default until it does.
+ *
+ * The PostgreSQL adapter treats the bound as nominal, not hard: a worst-case
+ * ≤1 s guarantee would need Postgres LISTEN/NOTIFY wake-on-append — one
+ * dedicated connection per held request on a pooled provider (Neon) — a cost
+ * considered and declined (AUT-334). Held event reads there re-query the
+ * database at most once per second, so an append by another connection is
+ * observed at the next poll: typically within about one second, worst case
+ * one poll interval plus the query round-trip.
+ */
+export const EVENT_WAIT_POLL_MS = 1000
+
+/** Sleep `ms`, or return early when `signal` aborts. Resolves (never
+ * rejects) on either path; the abort listener is removed on both so a
+ * long-held read cannot leak listeners. No signal (or an already-aborted
+ * one) never waits. */
+function sleepWithAbort(ms: number, signal?: AbortSignal): Promise<void> {
+  if (signal === undefined) return Bun.sleep(ms)
+  if (signal.aborted) return Promise.resolve()
+  return new Promise<void>((resolve) => {
+    let settled = false
+    const finish = (): void => {
+      if (settled) return
+      settled = true
+      clearTimeout(timer)
+      signal.removeEventListener('abort', onAbort)
+      resolve()
+    }
+    const timer = setTimeout(finish, ms)
+    const onAbort = (): void => finish()
+    signal.addEventListener('abort', onAbort, { once: true })
+  })
+}
+
+/**
+ * Read once, then poll until newer chunks arrive, the stream closes, the
+ * clamped wall-clock deadline passes, or `signal` aborts. `read` must
+ * re-query the store on every call; a throw from it propagates immediately.
+ * An abort ends the hold with the current result (a pre-aborted signal still
+ * performs the initial read — reads are cheap and side-effect-free — but
+ * never waits).
+ */
+export async function readStreamWithWait(opts: {
+  read: () => Promise<StreamRead>
+  waitSeconds?: number
+  pollMs?: number
+  signal?: AbortSignal
+}): Promise<StreamRead> {
+  const pollMs = opts.pollMs ?? STREAM_WAIT_POLL_MS
+  const deadline = Date.now() + clampWaitSeconds(opts.waitSeconds ?? 0) * 1000
+  let result = await opts.read()
+  while (
+    !opts.signal?.aborted &&
+    result.status === 'open' &&
+    result.chunks.length === 0 &&
+    Date.now() < deadline
+  ) {
+    await sleepWithAbort(pollMs, opts.signal)
+    if (opts.signal?.aborted) break
+    result = await opts.read()
+  }
+  return result
+}
+
+/**
+ * The event-read variant of the shared wait loop: read once, then poll until
+ * newer events arrive, the clamped deadline passes, or `signal` aborts.
+ * Session event reads (§7.1.1) honor the same clamp/early-return rules as
+ * stream reads so the operator UI's poll loop cannot drift between the two;
+ * there is no "closed" state to short-circuit on — an event log is always
+ * open. An abort ends the hold with the current result (a pre-aborted signal
+ * still performs the initial read but never waits).
+ */
+export async function readEventsWithWait<T>(opts: {
+  read: () => Promise<T[]>
+  waitSeconds?: number
+  pollMs?: number
+  signal?: AbortSignal
+}): Promise<T[]> {
+  const pollMs = opts.pollMs ?? STREAM_WAIT_POLL_MS
+  const deadline = Date.now() + clampWaitSeconds(opts.waitSeconds ?? 0) * 1000
+  let result = await opts.read()
+  while (result.length === 0 && !opts.signal?.aborted && Date.now() < deadline) {
+    await sleepWithAbort(pollMs, opts.signal)
+    if (opts.signal?.aborted) break
+    result = await opts.read()
+  }
+  return result
+}