ag-ui-validate 0.1.0

package/src/cli.ts

@@ -0,0 +1,147 @@
+#!/usr/bin/env node
+// The ag-ui-validate executable — the one deliberately Node-only file in src/.
+// Everything it does is delegated: parsing to cli-args.ts, validation to the
+// core + transport layers, output to the pure reporters. This file only wires
+// process.argv/stdin/stdout/exit codes to those pieces.
+import { createReadStream, readFileSync, writeFileSync } from "node:fs"
+import process from "node:process"
+import { decideExitCode, parseCliArgs, USAGE } from "./cli-args.js"
+import type { CliConfig } from "./cli-args.js"
+import { formatDiagnosticLine, formatReportSummary, toJsonReport, toJUnit, toSarif } from "./report/index.js"
+import { TransportError, validateBody, validateEndpoint } from "./transport/index.js"
+import type { TransportOptions, TransportResult } from "./transport/index.js"
+import type { Diagnostic, ValidatorOptions } from "./types.js"
+
+const TOOL_NAME = "ag-ui-validate"
+
+function toolVersion(): string {
+  try {
+    const pkg = JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8")) as {
+      version?: string
+    }
+    return pkg.version ?? "0.0.0"
+  } catch {
+    return "0.0.0"
+  }
+}
+
+function transportOptions(
+  config: CliConfig,
+  onDiagnostic: ((d: Diagnostic) => void) | undefined,
+): TransportOptions {
+  const validator: ValidatorOptions = {}
+  if (config.features !== undefined) validator.features = config.features
+  if (Object.keys(config.severityOverrides).length > 0) {
+    validator.severityOverrides = config.severityOverrides
+  }
+  const opts: TransportOptions = { validator }
+  if (onDiagnostic !== undefined) opts.onDiagnostic = onDiagnostic
+  return opts
+}
+
+async function main(): Promise<number> {
+  const parsed = parseCliArgs(process.argv.slice(2))
+  if (!parsed.ok) {
+    process.stderr.write(`error: ${parsed.error}\n\n${USAGE}`)
+    return 2
+  }
+  const config = parsed.config
+  if (config.help) {
+    process.stdout.write(USAGE)
+    return 0
+  }
+  const version = toolVersion()
+  if (config.version) {
+    process.stdout.write(`${TOOL_NAME} ${version}\n`)
+    return 0
+  }
+
+  const color =
+    config.color ?? (process.stdout.isTTY === true && process.env.NO_COLOR === undefined)
+  const pretty = config.format === "pretty"
+
+  // Pretty mode streams findings as they are detected; machine formats emit
+  // one document at the end, so nothing else may touch stdout before it.
+  let printed = 0
+  const onDiagnostic = pretty
+    ? (d: Diagnostic): void => {
+        printed += 1
+        process.stdout.write(`${formatDiagnosticLine(d, { color })}\n`)
+      }
+    : undefined
+
+  const isUrl = /^https?:\/\//i.test(config.target)
+  let result: TransportResult
+  let targetLabel: string
+  if (isUrl) {
+    targetLabel = config.target
+    const opts = transportOptions(config, onDiagnostic)
+    if (Object.keys(config.headers).length > 0) opts.headers = config.headers
+    if (config.timeoutMs !== undefined) opts.timeoutMs = config.timeoutMs
+    result = await validateEndpoint(config.target, opts)
+  } else if (config.target === "-") {
+    targetLabel = "stdin"
+    if (process.stdin.isTTY === true) {
+      process.stderr.write("error: stdin is a terminal — pipe a recording in, or pass a file path\n")
+      return 2
+    }
+    result = await validateBody(process.stdin, null, {
+      ...transportOptions(config, onDiagnostic),
+      recorded: true,
+    })
+  } else {
+    targetLabel = config.target
+    result = await validateBody(createReadStream(config.target), null, {
+      ...transportOptions(config, onDiagnostic),
+      recorded: true,
+    })
+  }
+
+  const report = result.report
+  // Line-based SARIF locations only make sense when events map 1:1 to lines.
+  const lineOriented = !isUrl && config.target !== "-" && /\.(jsonl|ndjson)$/i.test(config.target)
+  const sarifOptions = lineOriented
+    ? { toolVersion: version, artifactUri: config.target }
+    : { toolVersion: version }
+
+  if (config.jsonFile !== undefined) {
+    const doc = toJsonReport(report, { tool: { name: TOOL_NAME, version }, target: targetLabel })
+    writeFileSync(config.jsonFile, `${JSON.stringify(doc, null, 2)}\n`)
+  }
+  if (config.sarifFile !== undefined) {
+    writeFileSync(config.sarifFile, `${JSON.stringify(toSarif(report, sarifOptions), null, 2)}\n`)
+  }
+  if (config.junitFile !== undefined) {
+    writeFileSync(config.junitFile, toJUnit(report, { name: targetLabel }))
+  }
+
+  if (pretty) {
+    if (printed > 0) process.stdout.write("\n")
+    process.stdout.write(`${formatReportSummary(report, { color })}\n`)
+  } else if (config.format === "json") {
+    const doc = toJsonReport(report, { tool: { name: TOOL_NAME, version }, target: targetLabel })
+    process.stdout.write(`${JSON.stringify(doc, null, 2)}\n`)
+  } else if (config.format === "sarif") {
+    process.stdout.write(`${JSON.stringify(toSarif(report, sarifOptions), null, 2)}\n`)
+  } else {
+    process.stdout.write(toJUnit(report, { name: targetLabel }))
+  }
+
+  return decideExitCode(report.summary, config.maxWarnings)
+}
+
+main().then(
+  (code) => {
+    process.exitCode = code
+  },
+  (e: unknown) => {
+    const message =
+      e instanceof TransportError
+        ? e.message
+        : e instanceof Error
+          ? (e.stack ?? e.message)
+          : String(e)
+    process.stderr.write(`error: ${message}\n`)
+    process.exitCode = 2
+  },
+)

package/src/index.ts

@@ -0,0 +1,465 @@
+// ag-ui-validate core: a pure function over an AG-UI event sequence.
+// Zero I/O, zero runtime dependencies, isomorphic. Transports, the CLI, and
+// the Vitest matcher are thin wrappers over this module.
+//
+// Two invariants hold everywhere here:
+//   - the validator NEVER throws on input: broken input is its input;
+//   - every diagnostic cites the spec section that governs it (specUrl).
+
+import { DRAFT_EVENT_TYPES, EVENT_TABLE, EVENT_TYPES, SDK_VERSION } from "./protocol/event-table.js"
+import type { EventSpec } from "./protocol/event-table.js"
+import { RULES, formatMessage } from "./rules/catalog.js"
+import type { CheckApi, EmitFn, RunState, StreamState } from "./rules/checks/context.js"
+import { newRunState, str } from "./rules/checks/context.js"
+import { checkRunIdStability, endOfRunSteps, handleStepEvent } from "./rules/checks/lifecycle.js"
+import { closeReasoningChunk, endOfRunReasoning, handleReasoningEvent } from "./rules/checks/reasoning.js"
+import { handleStateEvent } from "./rules/checks/state.js"
+import { closeTextChunk, endOfRunText, handleTextEvent } from "./rules/checks/text.js"
+import { closeToolChunk, endOfRunToolCalls, handleToolCallEvent } from "./rules/checks/toolcalls.js"
+import { TRANSPORT_RULE_IDS, TRANSPORT_SKIP_REASON } from "./rules/checks/transport.js"
+import type {
+  CanonicalFeature,
+  Diagnostic,
+  FeatureMatrix,
+  Report,
+  Severity,
+  Validator,
+  ValidatorOptions,
+} from "./types.js"
+import { CANONICAL_FEATURES } from "./types.js"
+
+export * from "./types.js"
+export { CATALOG, RULES, formatMessage, validateCatalog } from "./rules/catalog.js"
+export type { Catalog, RuleDefinition, Severity as RuleSeverity, SeverityOrOff } from "./rules/catalog.js"
+export { EVENT_TABLE, EVENT_TYPES, SDK_VERSION } from "./protocol/event-table.js"
+export type { EventCategory, EventSpec, FieldKind, FieldSpec } from "./protocol/event-table.js"
+export { applyPatch, validatePatchShape } from "./protocol/jsonpatch.js"
+
+const DRAFTS_META_URL = "https://docs.ag-ui.com/drafts/meta-events"
+
+// Features whose exercise cannot be distinguished on a passive stream: both
+// generative-UI features need knowledge of the frontend's tool/component
+// registry, which is out-of-band (see SQ-13).
+const NOT_INFERABLE: readonly CanonicalFeature[] = [
+  "agentic-generative-ui",
+  "tool-based-generative-ui",
+]
+
+/** "runStarted" → "RUN_STARTED": case-insensitive match against wire types. */
+const CANONICAL_BY_SQUASHED: ReadonlyMap<string, string> = new Map(
+  EVENT_TYPES.map((t) => [t.replace(/_/g, "").toLowerCase(), t]),
+)
+
+function describeValue(v: unknown): string {
+  if (v === null) return "null"
+  if (Array.isArray(v)) return "array"
+  return typeof v
+}
+
+export function createValidator(opts: ValidatorOptions = {}): Validator {
+  const overrides = opts.severityOverrides ?? {}
+  const declaredFeatures = new Set(opts.features ?? [])
+  const layers = new Set(["core", ...(opts.layers ?? [])])
+
+  const diagnostics: Diagnostic[] = []
+  const internalErrors: string[] = []
+  const explicitSkips = new Map<string, string>()
+  const stream: StreamState = {
+    eventCount: 0,
+    sawTimestamp: false,
+    anySnapshot: false,
+    agui001Fired: false,
+    features: new Set(),
+  }
+  let run: RunState | null = null
+  let finalized = false
+
+  function mkEmit(batch: Diagnostic[], current: { index: number; type: string } | null): EmitFn {
+    return (ruleId, params, extra = {}) => {
+      const rule = RULES.get(ruleId)
+      if (rule === undefined) {
+        internalErrors.push(`emit() for unknown rule ${ruleId}`)
+        return
+      }
+      const override = overrides[ruleId]
+      if (override === "off") return
+      const severity: Severity = override ?? extra.severity ?? rule.severity
+      const aboutCurrentEvent = extra.eventIndex === undefined && current !== null
+      const diag: Diagnostic = {
+        rule: ruleId,
+        severity,
+        message: formatMessage(rule, params) + (extra.messageSuffix ?? ""),
+        eventIndex: extra.eventIndex ?? current?.index ?? -1,
+        specUrl: extra.specUrl ?? rule.specUrl,
+      }
+      if (aboutCurrentEvent) diag.eventType = current.type
+      if (extra.pointer !== undefined) diag.pointer = extra.pointer
+      if (extra.relatedEventIndex !== undefined) diag.relatedEventIndex = extra.relatedEventIndex
+      diagnostics.push(diag)
+      batch.push(diag)
+    }
+  }
+
+  function validateSchema(type: string, spec: EventSpec, ev: Record<string, unknown>, emit: EmitFn): void {
+    for (const [field, fs] of Object.entries(spec.fields)) {
+      const v = ev[field]
+      if (v === undefined) {
+        if (fs.required) {
+          emit("AGUI504", { type, detail: `missing required field '${field}' (${fs.kind})` }, {
+            pointer: `/${field}`,
+          })
+        }
+        continue
+      }
+      let kindOk = true
+      switch (fs.kind) {
+        case "string":
+        case "number":
+        case "boolean":
+          kindOk = typeof v === fs.kind
+          break
+        case "array":
+          kindOk = Array.isArray(v)
+          break
+        case "object":
+          kindOk = typeof v === "object" && v !== null && !Array.isArray(v)
+          break
+        case "any":
+          break
+      }
+      if (!kindOk) {
+        emit("AGUI504", { type, detail: `field '${field}' must be ${fs.kind === "array" ? "an" : "a"} ${fs.kind}, got ${describeValue(v)}` }, {
+          pointer: `/${field}`,
+        })
+        continue
+      }
+      if (fs.enum !== undefined && typeof v === "string" && !fs.enum.includes(v)) {
+        emit("AGUI504", { type, detail: `field '${field}' must be one of ${fs.enum.join("|")}, got '${v}'` }, {
+          pointer: `/${field}`,
+        })
+      }
+    }
+  }
+
+  /** Chunk streams close implicitly on any event of a different type. */
+  function closeChunks(r: RunState, emit: EmitFn, atIndex: number, except?: string): void {
+    if (except !== "TEXT_MESSAGE_CHUNK") closeTextChunk(r, atIndex)
+    if (except !== "TOOL_CALL_CHUNK") closeToolChunk(r, emit, atIndex)
+    if (except !== "REASONING_MESSAGE_CHUNK") closeReasoningChunk(r)
+  }
+
+  /** Unterminated-at-run-end rules. Only on a *clean* end (RUN_FINISHED or a
+   * stream that just stops): after RUN_ERROR, open streams are expected
+   * debris of the failure, and flagging them would manufacture noise. */
+  function endOfRunChecks(r: RunState, emit: EmitFn, atIndex: number): void {
+    endOfRunText(r, emit, atIndex)
+    endOfRunToolCalls(r, emit, atIndex)
+    endOfRunSteps(r, emit, atIndex)
+    endOfRunReasoning(r, emit, atIndex)
+  }
+
+  /** Opens the implicit run scope for streams that never announced one. */
+  function ensureRun(index: number, type: string, emit: EmitFn): RunState {
+    if (run === null) {
+      if (!stream.agui001Fired) {
+        emit("AGUI001", { type }, {})
+        stream.agui001Fired = true
+      }
+      run = newRunState({ runId: null, threadId: null, startIndex: index, implicit: true })
+    }
+    return run
+  }
+
+  function processEvent(input: unknown, batch: Diagnostic[]): void {
+    const index = stream.eventCount
+    stream.eventCount += 1
+
+    // 1. Parse. Malformed input is a diagnostic, never an exception.
+    let parsed: unknown = input
+    if (typeof input === "string") {
+      try {
+        parsed = JSON.parse(input)
+      } catch (e) {
+        mkEmit(batch, { index, type: "" })("AGUI502", {
+          error: e instanceof Error ? e.message : String(e),
+        })
+        return
+      }
+    }
+    if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
+      mkEmit(batch, { index, type: "" })("AGUI502", {
+        error: `payload is ${describeValue(parsed)}, expected a JSON object`,
+      })
+      return
+    }
+    const ev = parsed as Record<string, unknown>
+
+    // 2. The declared type is the key to everything else.
+    if (typeof ev.type !== "string") {
+      mkEmit(batch, { index, type: "" })("AGUI504", {
+        type: "(untyped)",
+        detail: "event has no string 'type' property",
+      }, { pointer: "/type" })
+      return
+    }
+    const type = ev.type
+    const emit = mkEmit(batch, { index, type })
+
+    // 3. Base-event fields.
+    if ("timestamp" in ev && ev.timestamp !== undefined) {
+      if (typeof ev.timestamp === "number") stream.sawTimestamp = true
+      else emit("AGUI504", { type, detail: `field 'timestamp' must be a number, got ${describeValue(ev.timestamp)}` }, { pointer: "/timestamp" })
+    }
+
+    // 4. Unknown types: AGUI503 (documented drafts at info; casing hints for
+    //    near-misses — SQ-7).
+    const spec = EVENT_TABLE[type]
+    if (spec === undefined) {
+      if (DRAFT_EVENT_TYPES.includes(type)) {
+        emit("AGUI503", { type, sdkVersion: SDK_VERSION }, {
+          severity: "info",
+          specUrl: DRAFTS_META_URL,
+          messageSuffix: " — documented draft event type, not yet in @ag-ui/core",
+        })
+      } else {
+        const canonical = CANONICAL_BY_SQUASHED.get(type.replace(/[_\-\s]/g, "").toLowerCase())
+        emit("AGUI503", { type, sdkVersion: SDK_VERSION }, {
+          ...(canonical !== undefined
+            ? { messageSuffix: ` — did you mean '${canonical}'? AG-UI wire types use SCREAMING_SNAKE_CASE` }
+            : {}),
+        })
+      }
+      return
+    }
+
+    // 5. Schema validation for the declared type (AGUI504).
+    validateSchema(type, spec, ev, emit)
+
+    // 6. Lifecycle position.
+    if (type === "RUN_STARTED") {
+      if (run === null) {
+        run = newRunState({
+          runId: str(ev, "runId") ?? null,
+          threadId: str(ev, "threadId") ?? null,
+          startIndex: index,
+          implicit: false,
+        })
+      } else if (run.terminal !== null) {
+        // A new run after a clean terminal: multi-run streams are legal
+        // (serialized logs branch via parentRunId — SQ-6). Fresh scope.
+        run = newRunState({
+          runId: str(ev, "runId") ?? null,
+          threadId: str(ev, "threadId") ?? null,
+          startIndex: index,
+          implicit: false,
+        })
+      } else if (run.implicit) {
+        // The stream opened without RUN_STARTED (AGUI001 already fired); the
+        // late start legitimizes the implicit scope rather than double-firing.
+        closeChunks(run, emit, index)
+        run.implicit = false
+        run.runId = str(ev, "runId") ?? null
+        run.threadId = str(ev, "threadId") ?? null
+      } else {
+        closeChunks(run, emit, index)
+        emit("AGUI002", { runId: str(ev, "runId") ?? "(missing)" }, {
+          relatedEventIndex: run.startIndex,
+        })
+      }
+      return
+    }
+
+    const r = ensureRun(index, type, emit)
+
+    // 7. Nothing may follow a terminal event (AGUI004/AGUI005).
+    if (r.terminal !== null) {
+      const terminalType = r.terminal.type
+      if ((type === "RUN_FINISHED" || type === "RUN_ERROR") && type !== terminalType) {
+        emit("AGUI005", { type, terminalType }, { relatedEventIndex: r.terminal.index })
+      } else {
+        emit("AGUI004", { type, terminalType }, { relatedEventIndex: r.terminal.index })
+      }
+      return
+    }
+
+    closeChunks(r, emit, index, type)
+
+    // 8. Terminal events.
+    if (type === "RUN_FINISHED" || type === "RUN_ERROR") {
+      if (type === "RUN_FINISHED") {
+        checkRunIdStability(api(index, type, ev, r, emit))
+        endOfRunChecks(r, emit, index)
+        const outcome = ev.outcome
+        if (typeof outcome === "object" && outcome !== null &&
+            (outcome as Record<string, unknown>).type === "interrupt") {
+          stream.features.add("human-in-the-loop")
+        }
+      }
+      r.terminal = { type, index }
+      return
+    }
+
+    // 9. Per-category checks.
+    const a = api(index, type, ev, r, emit)
+    switch (spec.category) {
+      case "lifecycle":
+        handleStepEvent(a) // STEP_STARTED / STEP_FINISHED
+        break
+      case "text":
+        handleTextEvent(a)
+        break
+      case "toolcall":
+        handleToolCallEvent(a)
+        break
+      case "state":
+        handleStateEvent(a)
+        break
+      case "reasoning":
+        handleReasoningEvent(a)
+        break
+      case "thinking":
+        // Deprecated but valid (SQ-8): schema-checked above, no ordering rules
+        // in the catalog yet — a "deprecated event used" rule is proposed
+        // upstream rather than invented here.
+        break
+      case "activity":
+        // No activity rules in the catalog yet.
+        break
+      case "special":
+        if (type === "RAW") {
+          const wrapped = ev.event
+          if (typeof wrapped === "object" && wrapped !== null) {
+            const wrappedType = (wrapped as Record<string, unknown>).type
+            if (typeof wrappedType === "string" && EVENT_TABLE[wrappedType] !== undefined) {
+              emit("AGUI901", { wrappedType }, { pointer: "/event/type" })
+            }
+          }
+        } else if (type === "CUSTOM") {
+          const name = str(ev, "name")
+          if (name !== undefined) {
+            if (name === "PredictState") stream.features.add("predictive-state-updates")
+            if (!/[.:/]/.test(name)) emit("AGUI903", { name }, { pointer: "/name" })
+          }
+        }
+        break
+    }
+  }
+
+  function api(
+    index: number,
+    type: string,
+    event: Record<string, unknown>,
+    r: RunState,
+    emit: EmitFn,
+  ): CheckApi {
+    return {
+      index,
+      type,
+      event,
+      run: r,
+      stream,
+      emit,
+      feature: (f) => stream.features.add(f),
+    }
+  }
+
+  return {
+    feed(event: unknown): Diagnostic[] {
+      const batch: Diagnostic[] = []
+      try {
+        processEvent(event, batch)
+      } catch (e) {
+        internalErrors.push(`feed(event ${stream.eventCount - 1}): ${e instanceof Error ? (e.stack ?? e.message) : String(e)}`)
+      }
+      return batch
+    },
+
+    finalize(): Diagnostic[] {
+      if (finalized) return []
+      finalized = true
+      const batch: Diagnostic[] = []
+      const emit = mkEmit(batch, null)
+      try {
+        if (run !== null && run.terminal === null) {
+          closeChunks(run, emit, -1)
+          emit("AGUI003", { runId: run.runId ?? "(unknown)" }, {
+            eventIndex: -1,
+            relatedEventIndex: run.startIndex,
+          })
+          endOfRunChecks(run, emit, -1)
+        }
+        if (stream.eventCount > 0 && !stream.sawTimestamp) {
+          emit("AGUI902", { eventCount: stream.eventCount }, { eventIndex: -1 })
+        }
+        if (declaredFeatures.has("shared-state") && !stream.anySnapshot) {
+          emit("AGUI305", {}, { eventIndex: -1 })
+        }
+      } catch (e) {
+        internalErrors.push(`finalize(): ${e instanceof Error ? (e.stack ?? e.message) : String(e)}`)
+      }
+      return batch
+    },
+
+    emitExternal(rule, params = {}, extra = {}): Diagnostic | null {
+      const batch: Diagnostic[] = []
+      try {
+        mkEmit(batch, null)(rule, params, extra)
+      } catch (e) {
+        internalErrors.push(`emitExternal(${rule}): ${e instanceof Error ? e.message : String(e)}`)
+      }
+      return batch[0] ?? null
+    },
+
+    markSkipped(rule, reason): void {
+      explicitSkips.set(String(rule), String(reason))
+    },
+
+    report(): Report {
+      const summary = { errors: 0, warnings: 0, info: 0 }
+      for (const d of diagnostics) {
+        if (d.severity === "error") summary.errors += 1
+        else if (d.severity === "warning") summary.warnings += 1
+        else summary.info += 1
+      }
+
+      const features = {} as FeatureMatrix
+      for (const f of CANONICAL_FEATURES) {
+        features[f] = NOT_INFERABLE.includes(f)
+          ? "not-inferable"
+          : stream.features.has(f)
+            ? "exercised"
+            : "not-exercised"
+      }
+
+      let skipped = layers.has("transport")
+        ? []
+        : TRANSPORT_RULE_IDS
+            .filter((id) => overrides[id] !== "off")
+            .map((rule) => ({ rule, reason: TRANSPORT_SKIP_REASON }))
+      if (!declaredFeatures.has("shared-state") && overrides.AGUI305 !== "off") {
+        skipped.push({
+          rule: "AGUI305",
+          reason: "only evaluated when the 'shared-state' feature is declared via options.features",
+        })
+      }
+      for (const [rule, severity] of Object.entries(overrides)) {
+        if (severity === "off" && RULES.has(rule)) {
+          skipped.push({ rule, reason: "disabled by severityOverrides" })
+        }
+      }
+      if (explicitSkips.size > 0) {
+        skipped = skipped.filter((s) => !explicitSkips.has(s.rule))
+        for (const [rule, reason] of explicitSkips) skipped.push({ rule, reason })
+      }
+
+      return {
+        diagnostics: [...diagnostics],
+        summary,
+        features,
+        skipped,
+        eventCount: stream.eventCount,
+        internalErrors: [...internalErrors],
+      }
+    },
+  }
+}