ag-ui-validate 0.1.0

package/src/rules/checks/context.ts

@@ -0,0 +1,117 @@
+// Internal validator state and the API each check module sees.
+// Everything here is engine-internal; the public surface lives in src/types.ts.
+
+import type { CanonicalFeature } from "../../types.js"
+
+/** Emit a diagnostic for rule `id`; params fill the catalog messageTemplate. */
+export type EmitFn = (
+  ruleId: string,
+  params: Record<string, unknown>,
+  extra?: {
+    eventIndex?: number
+    pointer?: string
+    relatedEventIndex?: number
+    /** Instance-level severity floor (e.g. draft META downgrades AGUI503). */
+    severity?: "error" | "warning" | "info"
+    /** Instance-level spec link override (e.g. draft docs page). */
+    specUrl?: string
+    /** Appended to the formatted message (e.g. casing hints). */
+    messageSuffix?: string
+  },
+) => void
+
+export interface OpenToolCall {
+  startIndex: number
+  args: string
+  sawArgs: boolean
+}
+
+export interface RunState {
+  /** null while the run is implicit (stream opened without RUN_STARTED). */
+  runId: string | null
+  threadId: string | null
+  startIndex: number
+  implicit: boolean
+  terminal: { type: string; index: number } | null
+
+  openMessages: Map<string, { startIndex: number }>
+  /** messageId -> index of the event that closed it. */
+  closedMessages: Map<string, number>
+  /** Every message id observed (starts, chunks, snapshots, tool results). */
+  knownMessageIds: Set<string>
+
+  openToolCalls: Map<string, OpenToolCall>
+  closedToolCalls: Map<string, number>
+  /** Ids also known from MESSAGES_SNAPSHOT history. */
+  knownToolCallIds: Set<string>
+
+  /** stepName -> re-entrant open count (SQ-10) and first-open index. */
+  openSteps: Map<string, { count: number; firstIndex: number }>
+
+  openReasoningBlocks: Map<string, number>
+  openReasoningMessages: Map<string, number>
+
+  state: {
+    /** True once a STATE_SNAPSHOT established an observable base (SQ-1). */
+    known: boolean
+    value: unknown
+    deltasSinceSnapshot: number
+    snapshotSeen: boolean
+    agui301Fired: boolean
+  }
+
+  /** Implicit streams opened by *_CHUNK events; closed by any other event. */
+  textChunk: { messageId: string; startIndex: number } | null
+  toolChunk: { toolCallId: string; startIndex: number; args: string; sawArgs: boolean } | null
+  reasoningChunk: { messageId: string; startIndex: number } | null
+}
+
+export function newRunState(init: {
+  runId: string | null
+  threadId: string | null
+  startIndex: number
+  implicit: boolean
+}): RunState {
+  return {
+    ...init,
+    terminal: null,
+    openMessages: new Map(),
+    closedMessages: new Map(),
+    knownMessageIds: new Set(),
+    openToolCalls: new Map(),
+    closedToolCalls: new Map(),
+    knownToolCallIds: new Set(),
+    openSteps: new Map(),
+    openReasoningBlocks: new Map(),
+    openReasoningMessages: new Map(),
+    state: { known: false, value: undefined, deltasSinceSnapshot: 0, snapshotSeen: false, agui301Fired: false },
+    textChunk: null,
+    toolChunk: null,
+    reasoningChunk: null,
+  }
+}
+
+export interface StreamState {
+  eventCount: number
+  sawTimestamp: boolean
+  anySnapshot: boolean
+  agui001Fired: boolean
+  features: Set<CanonicalFeature>
+}
+
+/** What a per-event check handler receives. */
+export interface CheckApi {
+  index: number
+  type: string
+  event: Record<string, unknown>
+  run: RunState
+  stream: StreamState
+  emit: EmitFn
+  feature: (f: CanonicalFeature) => void
+}
+
+/** Reads a field only if it is a string (schema problems already reported). */
+export function str(event: Record<string, unknown>, field: string): string | undefined {
+  const v = event[field]
+  return typeof v === "string" ? v : undefined
+}

package/src/rules/checks/lifecycle.ts

@@ -0,0 +1,59 @@
+// Lifecycle rules: AGUI001–AGUI008. Run open/terminate logic lives in the
+// engine (src/index.ts) because it owns run-scope resets; this module holds
+// the step pairing and the RUN_FINISHED id-stability check.
+
+import type { CheckApi, RunState, EmitFn } from "./context.js"
+import { str } from "./context.js"
+
+export function handleStepEvent(api: CheckApi): void {
+  const { type, event, run, emit, index } = api
+  const stepName = str(event, "stepName")
+  if (stepName === undefined) return
+
+  if (type === "STEP_STARTED") {
+    const open = run.openSteps.get(stepName)
+    if (open !== undefined) {
+      open.count += 1 // re-entrant same-name steps are tolerated (SQ-10)
+    } else {
+      run.openSteps.set(stepName, { count: 1, firstIndex: index })
+    }
+    return
+  }
+
+  if (type === "STEP_FINISHED") {
+    const open = run.openSteps.get(stepName)
+    if (open === undefined) {
+      emit("AGUI006", { stepName }, { pointer: "/stepName" })
+      return
+    }
+    open.count -= 1
+    if (open.count === 0) run.openSteps.delete(stepName)
+  }
+}
+
+/** AGUI008 — RUN_FINISHED must carry the ids RUN_STARTED established. */
+export function checkRunIdStability(api: CheckApi): void {
+  const { event, run, emit } = api
+  if (run.implicit) return // no RUN_STARTED to compare against
+  for (const field of ["threadId", "runId"] as const) {
+    const actual = str(api.event, field)
+    const expected = field === "threadId" ? run.threadId : run.runId
+    if (actual !== undefined && expected !== null && actual !== expected) {
+      emit("AGUI008", { field, actual, expected }, {
+        pointer: `/${field}`,
+        relatedEventIndex: run.startIndex,
+      })
+    }
+  }
+  void event
+}
+
+/** AGUI007 — open steps when the run reaches a clean end. */
+export function endOfRunSteps(run: RunState, emit: EmitFn, atIndex: number): void {
+  for (const [stepName, open] of run.openSteps) {
+    emit("AGUI007", { stepName }, {
+      eventIndex: atIndex,
+      relatedEventIndex: open.firstIndex,
+    })
+  }
+}

package/src/rules/checks/reasoning.ts

@@ -0,0 +1,97 @@
+// Reasoning rules: AGUI401–AGUI402, plus REASONING_MESSAGE_CHUNK handling.
+//
+// Per SQ-4: block-level nesting (REASONING_MESSAGE_* inside
+// REASONING_START/END) is only described as "a typical flow" by the docs, so
+// it is NOT enforced. AGUI401 enforces the message-level pairing, mirroring
+// the text message rules; AGUI402 reports unterminated starts at warning.
+
+import type { CheckApi, RunState, EmitFn } from "./context.js"
+import { str } from "./context.js"
+
+export function handleReasoningEvent(api: CheckApi): void {
+  const { type, event, run, emit, index } = api
+
+  switch (type) {
+    case "REASONING_START": {
+      const id = str(event, "messageId")
+      if (id !== undefined) run.openReasoningBlocks.set(id, index)
+      return
+    }
+
+    case "REASONING_END": {
+      const id = str(event, "messageId")
+      if (id !== undefined) run.openReasoningBlocks.delete(id)
+      return
+    }
+
+    case "REASONING_MESSAGE_START": {
+      const id = str(event, "messageId")
+      if (id !== undefined) run.openReasoningMessages.set(id, index)
+      return
+    }
+
+    case "REASONING_MESSAGE_CONTENT": {
+      const id = str(event, "messageId")
+      if (id === undefined) return
+      const openChunk = run.reasoningChunk !== null && run.reasoningChunk.messageId === id
+      if (!run.openReasoningMessages.has(id) && !openChunk) {
+        emit("AGUI401", { messageId: id }, { pointer: "/messageId" })
+      }
+      return
+    }
+
+    case "REASONING_MESSAGE_END": {
+      const id = str(event, "messageId")
+      if (id !== undefined) run.openReasoningMessages.delete(id)
+      return
+    }
+
+    case "REASONING_MESSAGE_CHUNK": {
+      const id = str(event, "messageId")
+      if (id === undefined) {
+        if (run.reasoningChunk === null) {
+          emit("AGUI504", { type, detail: "first REASONING_MESSAGE_CHUNK must include messageId" }, {
+            pointer: "/messageId",
+          })
+          return
+        }
+        if (event.delta === "") run.reasoningChunk = null // documented implicit close
+        return
+      }
+      if (run.reasoningChunk !== null && run.reasoningChunk.messageId !== id) {
+        run.reasoningChunk = null
+      }
+      if (event.delta === "") {
+        run.reasoningChunk = null // empty delta implicitly closes the message
+        return
+      }
+      if (run.reasoningChunk === null) {
+        run.reasoningChunk = { messageId: id, startIndex: index }
+      }
+      return
+    }
+
+    // REASONING_ENCRYPTED_VALUE: schema-checked only; no ordering rules cited.
+  }
+}
+
+/** Chunked reasoning also closes on any non-reasoning event (documented). */
+export function closeReasoningChunk(run: RunState): void {
+  run.reasoningChunk = null
+}
+
+/** AGUI402 — unterminated reasoning at a clean run end. */
+export function endOfRunReasoning(run: RunState, emit: EmitFn, atIndex: number): void {
+  for (const [id, startIndex] of run.openReasoningBlocks) {
+    emit("AGUI402", { startType: "REASONING_START", messageId: id }, {
+      eventIndex: atIndex,
+      relatedEventIndex: startIndex,
+    })
+  }
+  for (const [id, startIndex] of run.openReasoningMessages) {
+    emit("AGUI402", { startType: "REASONING_MESSAGE_START", messageId: id }, {
+      eventIndex: atIndex,
+      relatedEventIndex: startIndex,
+    })
+  }
+}

package/src/rules/checks/state.ts

@@ -0,0 +1,72 @@
+// State rules: AGUI301–AGUI305, plus MESSAGES_SNAPSHOT history harvesting.
+
+import { applyPatch, validatePatchShape } from "../../protocol/jsonpatch.js"
+import type { CheckApi } from "./context.js"
+
+export function handleStateEvent(api: CheckApi): void {
+  const { type, event, run, stream, emit } = api
+
+  switch (type) {
+    case "STATE_SNAPSHOT": {
+      api.feature("shared-state")
+      if (run.state.deltasSinceSnapshot > 0) {
+        emit("AGUI304", { deltaCount: run.state.deltasSinceSnapshot }, {})
+      }
+      run.state.known = true
+      run.state.value = event.snapshot
+      run.state.snapshotSeen = true
+      run.state.deltasSinceSnapshot = 0
+      stream.anySnapshot = true
+      return
+    }
+
+    case "STATE_DELTA": {
+      api.feature("shared-state")
+      const delta = event.delta
+      if (!Array.isArray(delta)) return // AGUI504 already reported the kind mismatch
+      const shape = validatePatchShape(delta)
+      if (shape !== null) {
+        emit("AGUI303", { error: shape.error }, { pointer: `/delta${shape.pointer}` })
+        return
+      }
+      if (!run.state.snapshotSeen && !run.state.agui301Fired) {
+        // The base may be seeded out-of-band via RunAgentInput.state (SQ-1),
+        // so this is informational, and patch application is not judged until
+        // a snapshot establishes an observable base.
+        emit("AGUI301", {}, {})
+        run.state.agui301Fired = true
+      }
+      if (run.state.known) {
+        const applied = applyPatch(run.state.value, delta)
+        if (applied.ok) {
+          run.state.value = applied.result
+        } else {
+          emit("AGUI302", { error: applied.error }, { pointer: `/delta/${applied.opIndex}` })
+        }
+      }
+      run.state.deltasSinceSnapshot += 1
+      return
+    }
+
+    case "MESSAGES_SNAPSHOT": {
+      // Snapshots can carry history from before this capture: harvest ids so
+      // reference checks (AGUI207/AGUI208) don't false-positive on them.
+      const messages = event.messages
+      if (!Array.isArray(messages)) return
+      for (const message of messages) {
+        if (typeof message !== "object" || message === null) continue
+        const m = message as Record<string, unknown>
+        if (typeof m.id === "string") run.knownMessageIds.add(m.id)
+        if (Array.isArray(m.toolCalls)) {
+          for (const call of m.toolCalls) {
+            if (typeof call === "object" && call !== null) {
+              const id = (call as Record<string, unknown>).id
+              if (typeof id === "string") run.knownToolCallIds.add(id)
+            }
+          }
+        }
+      }
+      return
+    }
+  }
+}

package/src/rules/checks/text.ts

@@ -0,0 +1,109 @@
+// Text message rules: AGUI101–AGUI106, plus TEXT_MESSAGE_CHUNK stream handling.
+
+import type { CheckApi, RunState, EmitFn } from "./context.js"
+import { str } from "./context.js"
+
+export function handleTextEvent(api: CheckApi): void {
+  const { type, event, run, emit, index } = api
+  api.feature("agentic-chat")
+
+  switch (type) {
+    case "TEXT_MESSAGE_START": {
+      const id = str(event, "messageId")
+      if (id === undefined) return
+      if (run.openMessages.has(id)) {
+        emit("AGUI106", { messageId: id }, {
+          pointer: "/messageId",
+          relatedEventIndex: run.openMessages.get(id)!.startIndex,
+        })
+        return
+      }
+      if (run.closedMessages.has(id)) {
+        emit("AGUI104", { messageId: id }, {
+          pointer: "/messageId",
+          relatedEventIndex: run.closedMessages.get(id)!,
+        })
+        return
+      }
+      run.openMessages.set(id, { startIndex: index })
+      run.knownMessageIds.add(id)
+      return
+    }
+
+    case "TEXT_MESSAGE_CONTENT": {
+      const id = str(event, "messageId")
+      if (id === undefined) return
+      const open = run.openMessages.get(id)
+      if (open === undefined) {
+        const closedAt = run.closedMessages.get(id)
+        emit("AGUI101", { messageId: id }, {
+          pointer: "/messageId",
+          ...(closedAt !== undefined ? { relatedEventIndex: closedAt } : {}),
+        })
+        return
+      }
+      if (event.delta === "") {
+        emit("AGUI105", { messageId: id }, { pointer: "/delta", relatedEventIndex: open.startIndex })
+      }
+      return
+    }
+
+    case "TEXT_MESSAGE_END": {
+      const id = str(event, "messageId")
+      if (id === undefined) return
+      if (!run.openMessages.has(id)) {
+        emit("AGUI102", { messageId: id }, { pointer: "/messageId" })
+        return
+      }
+      run.openMessages.delete(id)
+      run.closedMessages.set(id, index)
+      return
+    }
+
+    case "TEXT_MESSAGE_CHUNK": {
+      const id = str(event, "messageId")
+      if (id === undefined) {
+        // Continuation chunks may omit messageId; the first chunk must not
+        // (docs: "First chunk for a message must include messageId").
+        if (run.textChunk === null) {
+          emit("AGUI504", { type, detail: "first TEXT_MESSAGE_CHUNK for a message must include messageId" }, {
+            pointer: "/messageId",
+          })
+        }
+        return
+      }
+      // Mixing chunk and explicit forms on one id is undefined behaviour
+      // (SQ-9): tolerated, treated as content for the open message.
+      if (run.openMessages.has(id)) return
+      if (run.textChunk !== null && run.textChunk.messageId === id) return
+      closeTextChunk(run, index)
+      if (run.closedMessages.has(id)) {
+        emit("AGUI104", { messageId: id }, {
+          pointer: "/messageId",
+          relatedEventIndex: run.closedMessages.get(id)!,
+        })
+        return
+      }
+      run.textChunk = { messageId: id, startIndex: index }
+      run.knownMessageIds.add(id)
+      return
+    }
+  }
+}
+
+/** Chunk streams close implicitly on the next non-chunk event. */
+export function closeTextChunk(run: RunState, atIndex: number): void {
+  if (run.textChunk === null) return
+  run.closedMessages.set(run.textChunk.messageId, atIndex)
+  run.textChunk = null
+}
+
+/** AGUI103 — open messages when the run reaches a clean end. */
+export function endOfRunText(run: RunState, emit: EmitFn, atIndex: number): void {
+  for (const [id, open] of run.openMessages) {
+    emit("AGUI103", { messageId: id }, {
+      eventIndex: atIndex,
+      relatedEventIndex: open.startIndex,
+    })
+  }
+}

package/src/rules/checks/toolcalls.ts

@@ -0,0 +1,167 @@
+// Tool call rules: AGUI201–AGUI208, plus TOOL_CALL_CHUNK stream handling.
+
+import type { CheckApi, RunState, EmitFn } from "./context.js"
+import { str } from "./context.js"
+
+export function handleToolCallEvent(api: CheckApi): void {
+  const { type, event, run, emit, index } = api
+  api.feature("backend-tool-rendering")
+
+  switch (type) {
+    case "TOOL_CALL_START": {
+      const id = str(event, "toolCallId")
+      if (id === undefined) return
+      if (run.openToolCalls.has(id) || run.closedToolCalls.has(id)) {
+        const related = run.openToolCalls.get(id)?.startIndex ?? run.closedToolCalls.get(id)!
+        emit("AGUI205", { toolCallId: id }, { pointer: "/toolCallId", relatedEventIndex: related })
+        return
+      }
+      run.openToolCalls.set(id, { startIndex: index, args: "", sawArgs: false })
+      const parent = str(event, "parentMessageId")
+      if (parent !== undefined && !run.knownMessageIds.has(parent)) {
+        emit("AGUI208", { parentMessageId: parent }, { pointer: "/parentMessageId" })
+      }
+      return
+    }
+
+    case "TOOL_CALL_ARGS": {
+      const id = str(event, "toolCallId")
+      if (id === undefined) return
+      const open = run.openToolCalls.get(id)
+      if (open === undefined) {
+        emit("AGUI201", { toolCallId: id }, { pointer: "/toolCallId" })
+        return
+      }
+      const delta = str(event, "delta")
+      if (delta !== undefined) {
+        open.args += delta
+        open.sawArgs = true
+      }
+      return
+    }
+
+    case "TOOL_CALL_END": {
+      const id = str(event, "toolCallId")
+      if (id === undefined) return
+      const open = run.openToolCalls.get(id)
+      if (open === undefined) {
+        emit("AGUI202", { toolCallId: id }, { pointer: "/toolCallId" })
+        return
+      }
+      run.openToolCalls.delete(id)
+      run.closedToolCalls.set(id, index)
+      checkArgsJson(id, open, emit, index)
+      return
+    }
+
+    case "TOOL_CALL_RESULT": {
+      const id = str(event, "toolCallId")
+      if (id !== undefined) {
+        const open = run.openToolCalls.get(id)
+        if (open !== undefined) {
+          emit("AGUI206", { toolCallId: id }, {
+            pointer: "/toolCallId",
+            relatedEventIndex: open.startIndex,
+          })
+        } else if (!run.closedToolCalls.has(id) && !run.knownToolCallIds.has(id)) {
+          emit("AGUI207", { toolCallId: id }, { pointer: "/toolCallId" })
+        }
+      }
+      // The result is itself a tool message in the conversation.
+      const messageId = str(event, "messageId")
+      if (messageId !== undefined) run.knownMessageIds.add(messageId)
+      return
+    }
+
+    case "TOOL_CALL_CHUNK": {
+      const id = str(event, "toolCallId")
+      const delta = str(event, "delta")
+      if (id === undefined) {
+        if (run.toolChunk === null) {
+          emit("AGUI504", { type, detail: "first TOOL_CALL_CHUNK for a tool call must include toolCallId and toolCallName" }, {
+            pointer: "/toolCallId",
+          })
+          return
+        }
+        if (delta !== undefined) {
+          run.toolChunk.args += delta
+          run.toolChunk.sawArgs = true
+        }
+        return
+      }
+      // SQ-9: chunk continuing an explicitly-opened call — treat as args.
+      const openExplicit = run.openToolCalls.get(id)
+      if (openExplicit !== undefined) {
+        if (delta !== undefined) {
+          openExplicit.args += delta
+          openExplicit.sawArgs = true
+        }
+        return
+      }
+      if (run.toolChunk !== null && run.toolChunk.toolCallId === id) {
+        if (delta !== undefined) {
+          run.toolChunk.args += delta
+          run.toolChunk.sawArgs = true
+        }
+        return
+      }
+      closeToolChunk(run, emit, index)
+      if (run.closedToolCalls.has(id)) {
+        emit("AGUI205", { toolCallId: id }, {
+          pointer: "/toolCallId",
+          relatedEventIndex: run.closedToolCalls.get(id)!,
+        })
+        return
+      }
+      if (str(event, "toolCallName") === undefined) {
+        emit("AGUI504", { type, detail: "first TOOL_CALL_CHUNK for a tool call must include toolCallName" }, {
+          pointer: "/toolCallName",
+        })
+      }
+      run.toolChunk = {
+        toolCallId: id,
+        startIndex: index,
+        args: delta ?? "",
+        sawArgs: delta !== undefined && delta.length > 0,
+      }
+      return
+    }
+  }
+}
+
+function checkArgsJson(
+  id: string,
+  call: { startIndex: number; args: string; sawArgs: boolean },
+  emit: EmitFn,
+  atIndex: number,
+): void {
+  // A call that streamed no args at all is fine (a no-argument tool).
+  if (!call.sawArgs || call.args.length === 0) return
+  try {
+    JSON.parse(call.args)
+  } catch (e) {
+    emit("AGUI204", { toolCallId: id, error: e instanceof Error ? e.message : String(e) }, {
+      eventIndex: atIndex,
+      relatedEventIndex: call.startIndex,
+    })
+  }
+}
+
+/** Chunk streams close implicitly on the next non-chunk event. */
+export function closeToolChunk(run: RunState, emit: EmitFn, atIndex: number): void {
+  if (run.toolChunk === null) return
+  const { toolCallId, startIndex, args, sawArgs } = run.toolChunk
+  run.toolChunk = null
+  run.closedToolCalls.set(toolCallId, atIndex)
+  checkArgsJson(toolCallId, { startIndex, args, sawArgs }, emit, atIndex)
+}
+
+/** AGUI203 — open tool calls when the run reaches a clean end. */
+export function endOfRunToolCalls(run: RunState, emit: EmitFn, atIndex: number): void {
+  for (const [id, open] of run.openToolCalls) {
+    emit("AGUI203", { toolCallId: id }, {
+      eventIndex: atIndex,
+      relatedEventIndex: open.startIndex,
+    })
+  }
+}

package/src/rules/checks/transport.ts

@@ -0,0 +1,17 @@
+// Transport rules (AGUI501, AGUI505–AGUI508) are only checkable against a
+// live connection: SSE framing, Content-Type, keepalive timing, flush
+// behaviour, abnormal EOF. The core cannot observe any of that from a parsed
+// event sequence, so it reports these rules as skipped — never silently —
+// and the transport layer (ag-ui-validate/transport, M5) evaluates them.
+//
+// AGUI502/503/504 are transport-adjacent but *are* checkable here (the core
+// accepts raw JSON strings), so they live in the engine, not this list.
+
+import { CATALOG } from "../catalog.js"
+
+export const TRANSPORT_SKIP_REASON =
+  "transport-layer rule; only checkable against a live connection (validated by ag-ui-validate/transport)"
+
+export const TRANSPORT_RULE_IDS: readonly string[] = CATALOG.rules
+  .filter((r) => r.checkedIn === "transport")
+  .map((r) => r.id)