@tabnas/transduce 0.1.0 → 0.1.2

package/src/scan.ts

@@ -0,0 +1,83 @@
+/* Copyright (c) 2026 tabnas, MIT License */
+
+// `scan-emit`: declarative state evolution over a stream.
+//
+// A pure step function takes the state and one item and returns the next
+// state with the items to emit; a finish function turns the final state
+// into the closing items. The operator owns the state and the iteration;
+// the functions own nothing. This is what the DSL's `scan-emit` lowers to,
+// and the table transducer is one instance of it.
+
+import { Fail } from './error'
+import { Flow } from './sink'
+
+// The result of one step: the next state and what to emit for it.
+export type Transition<S, O> = { state: S; outputs: O[] }
+
+export const Transition = Object.freeze({
+  of<S, O>(state: S, outputs: O[]): Transition<S, O> {
+    return { state, outputs }
+  },
+  // A step that emits nothing.
+  stay<S, O>(state: S): Transition<S, O> {
+    return { state, outputs: [] }
+  },
+  // A step that emits one item.
+  emit<S, O>(state: S, output: O): Transition<S, O> {
+    return { state, outputs: [output] }
+  },
+})
+
+// The operator. Feed items with `item`, then call `finish` exactly once
+// when the input completed successfully; never call it after a failure or
+// a cancellation. A step or the finish throws a `Fail` to fail the run; a
+// step that failed leaves the operator without a state, so a later item or
+// `finish` is a `PROTOCOL_ORDER_ERROR`.
+export class ScanEmit<S, I, O> {
+  private state: { value: S } | null
+  private step: (state: S, item: I) => Transition<S, O>
+  private finisher: ((state: S) => O[]) | null
+  private out: (output: O) => Flow
+
+  constructor(
+    initial: S,
+    step: (state: S, item: I) => Transition<S, O>,
+    finish: (state: S) => O[],
+    out: (output: O) => Flow,
+  ) {
+    this.state = { value: initial }
+    this.step = step
+    this.finisher = finish
+    this.out = out
+  }
+
+  // One input item. Its outputs go downstream before this returns.
+  item(item: I): Flow {
+    const state = this.state
+    if (null === state) {
+      throw Fail.protocol('scan-emit received an item after it finished')
+    }
+    this.state = null
+    const transition = this.step(state.value, item)
+    this.state = { value: transition.state }
+    for (const o of transition.outputs) {
+      if ('stop' === this.out(o)) return 'stop'
+    }
+    return 'continue'
+  }
+
+  // The input completed: emit the closing items.
+  finish(): Flow {
+    const state = this.state
+    const finish = this.finisher
+    if (null === state || null === finish) {
+      throw Fail.protocol('scan-emit finished twice')
+    }
+    this.state = null
+    this.finisher = null
+    for (const o of finish(state.value)) {
+      if ('stop' === this.out(o)) return 'stop'
+    }
+    return 'continue'
+  }
+}

package/src/selector.ts

@@ -0,0 +1,191 @@
+/* Copyright (c) 2026 tabnas, MIT License */
+
+// Selectors: reusable descriptions of where in a document to look.
+//
+// A selector is data, never code: it is built from constructors or from
+// validated path segments (`Selector.fromSegments`), and a matcher
+// interprets it. A concrete `Path` names one location; a `Selector` may
+// name many (`each_index`, `each_member`). Both print in jq syntax, which
+// is what the rest of the fleet prints and accepts.
+
+import { jsonString } from './json'
+
+// One step of a concrete path: a string is a member's key, a non-negative
+// integer an element's index.
+export type Segment = string | number
+
+// One key as jq writes it: bare when it is an ASCII identifier, a JSON
+// string otherwise.
+export function keyText(key: string): string {
+  return /^[A-Za-z_][A-Za-z0-9_]*$/.test(key) ? '.' + key : '.' + jsonString(key)
+}
+
+// A concrete location in a document.
+export class Path {
+  readonly segments: Segment[]
+
+  constructor(segments: Segment[] = []) {
+    this.segments = segments
+  }
+
+  static root(): Path {
+    return new Path([])
+  }
+
+  push(seg: Segment): void {
+    this.segments.push(seg)
+  }
+
+  pop(): Segment | undefined {
+    return this.segments.pop()
+  }
+
+  depth(): number {
+    return this.segments.length
+  }
+
+  clone(): Path {
+    return new Path(this.segments.slice())
+  }
+
+  // jq syntax: `.a[0]."b-c"`, and `.` for the root.
+  toString(): string {
+    return pathText(this.segments)
+  }
+}
+
+// Segments in jq syntax.
+export function pathText(segments: readonly Segment[]): string {
+  if (0 === segments.length) return '.'
+  let out = ''
+  for (const seg of segments) {
+    out += 'number' === typeof seg ? `[${seg}]` : keyText(seg)
+  }
+  return out
+}
+
+// One step of a selector.
+export type Step =
+  // The member with this name, inside an object.
+  | { readonly type: 'property'; readonly name: string }
+  // The element at this position, inside an array.
+  | { readonly type: 'index'; readonly index: number }
+  // Every element of an array.
+  | { readonly type: 'each_index' }
+  // Every member value of an object.
+  | { readonly type: 'each_member' }
+
+const EACH_INDEX: Step = Object.freeze({ type: 'each_index' })
+const EACH_MEMBER: Step = Object.freeze({ type: 'each_member' })
+
+// A description of locations: the root, narrowed step by step. Immutable:
+// every builder returns a new selector.
+export class Selector {
+  readonly steps: readonly Step[]
+
+  constructor(steps: readonly Step[] = []) {
+    this.steps = Object.freeze(steps.slice())
+  }
+
+  // The document itself.
+  static root(): Selector {
+    return new Selector([])
+  }
+
+  property(name: string): Selector {
+    return new Selector([...this.steps, { type: 'property', name }])
+  }
+
+  index(index: number): Selector {
+    if (!Number.isInteger(index) || index < 0) {
+      throw new TypeError('an index step is a non-negative integer: ' + index)
+    }
+    return new Selector([...this.steps, { type: 'index', index }])
+  }
+
+  eachIndex(): Selector {
+    return new Selector([...this.steps, EACH_INDEX])
+  }
+
+  eachMember(): Selector {
+    return new Selector([...this.steps, EACH_MEMBER])
+  }
+
+  // `this`, then `other` below every location `this` names.
+  compose(other: Selector): Selector {
+    return new Selector([...this.steps, ...other.steps])
+  }
+
+  // A selector naming exactly one location: `as-path` over data.
+  static fromSegments(segments: readonly Segment[]): Selector {
+    let s = Selector.root()
+    for (const seg of segments) {
+      s = 'number' === typeof seg ? s.index(seg) : s.property(seg)
+    }
+    return s
+  }
+
+  isRoot(): boolean {
+    return 0 === this.steps.length
+  }
+
+  // Whether the selector can name more than one location.
+  isMulti(): boolean {
+    return this.steps.some((s) => 'each_index' === s.type || 'each_member' === s.type)
+  }
+
+  // Whether this selector names a location strictly inside a location
+  // `other` names, or the same one: the test a router uses to refuse
+  // overlapping captures.
+  mayOverlap(other: Selector): boolean {
+    const [short, long] =
+      this.steps.length <= other.steps.length ? [this, other] : [other, this]
+    for (let i = 0; i < short.steps.length; i++) {
+      if (!stepMayMatchSame(short.steps[i], long.steps[i])) return false
+    }
+    return true
+  }
+
+  equals(other: Selector): boolean {
+    return this.toString() === other.toString()
+  }
+
+  // jq syntax: `.response.records[*]`, `."odd key"`, `[3]`, `[]` for
+  // every member, `.` for the root.
+  toString(): string {
+    if (0 === this.steps.length) return '.'
+    let out = ''
+    for (const step of this.steps) {
+      switch (step.type) {
+        case 'property':
+          out += keyText(step.name)
+          break
+        case 'index':
+          out += `[${step.index}]`
+          break
+        case 'each_index':
+          out += '[*]'
+          break
+        case 'each_member':
+          out += '[]'
+          break
+      }
+    }
+    return out
+  }
+}
+
+function stepMayMatchSame(a: Step, b: Step): boolean {
+  switch (a.type) {
+    case 'property':
+      return (
+        ('property' === b.type && a.name === b.name) || 'each_member' === b.type
+      )
+    case 'each_member':
+      return 'property' === b.type || 'each_member' === b.type
+    case 'index':
+      return ('index' === b.type && a.index === b.index) || 'each_index' === b.type
+    case 'each_index':
+      return 'index' === b.type || 'each_index' === b.type
+  }
+}

package/src/sink.ts

@@ -0,0 +1,199 @@
+/* Copyright (c) 2026 tabnas, MIT License */
+
+// The push boundary between stages.
+//
+// A pipeline is a chain of sinks. The source calls the first sink once per
+// event, synchronously, inside the parse; each stage does its work and
+// calls the next. Nothing is queued between stages, so a slow writer at the
+// end slows the parser at the start: that is the backpressure, and it
+// costs no buffer. A stage that must stop early (a `take`) answers
+// `'stop'`, which the source turns into a cancelled parse. A stage that
+// fails throws a `Fail`; the source stops the parse and the failure
+// reaches the caller unchanged.
+
+import { Fail } from './error'
+import { JsonEvent } from './event'
+import { Path, Segment } from './selector'
+
+// What a stage wants next: `'continue'`, or `'stop'` (the stage has all it
+// needs; the source stops the parse, releases what it holds and reports
+// nothing further; not an error).
+export type Flow = 'continue' | 'stop'
+
+// A consumer of `JsonEvents/1`. `event` throws a `Fail` to fail the run.
+export interface Sink {
+  event(ev: JsonEvent): Flow
+}
+
+// A sink that records every event, for tests and small results.
+export class EventRecorder implements Sink {
+  readonly events: JsonEvent[] = []
+
+  event(ev: JsonEvent): Flow {
+    this.events.push(ev)
+    return 'continue'
+  }
+}
+
+// A sink made of a function. A function that returns nothing continues.
+export class FnSink implements Sink {
+  private fn: (ev: JsonEvent) => Flow | void
+
+  constructor(fn: (ev: JsonEvent) => Flow | void) {
+    this.fn = fn
+  }
+
+  event(ev: JsonEvent): Flow {
+    return this.fn(ev) || 'continue'
+  }
+}
+
+// A sink that counts events and drops them: the cheapest consumer, for
+// measuring a source on its own.
+export class CountSink implements Sink {
+  events = 0
+
+  event(_ev: JsonEvent): Flow {
+    this.events++
+    return 'continue'
+  }
+}
+
+// Replay a recording into a sink, stopping where the sink stops.
+export function replay(events: Iterable<JsonEvent>, sink: Sink): Flow {
+  for (const ev of events) {
+    if ('stop' === sink.event(ev)) return 'stop'
+  }
+  return 'continue'
+}
+
+// A container `TreeContract` has open: an object with the keys it has had
+// (the last of them the member whose value is due unless `keyDue`), or an
+// array with the index of the element due next.
+type Open =
+  | { array: false; keys: Set<string>; last: string | null; keyDue: boolean }
+  | { array: true; next: number }
+
+// A tree's events, held to their contract in front of a sink that takes
+// them as one (a render that writes a document from them): one root value,
+// and in each object a key and then its value, each key once. A value
+// walked from a parsed tree keeps it by construction. A parse streamed as
+// it proceeds may not: it hands on a member its grammar reads twice
+// (JSON's `{"a":1,"a":2}`, whose value keeps the last) where a tree has
+// one, and the rule-event adapter refuses most shapes it cannot follow but
+// not every one a grammar can produce. A repeated key in one object is
+// refused with `DUPLICATE_MEMBER`, and events no tree has (a value where a
+// key is due, a key outside an object, a close with nothing open, a second
+// root) with `STREAMABILITY_UNKNOWN`, each at the path of the object
+// concerned; the event is not passed on. Each open object keeps the keys
+// it has had, dropped when it closes.
+export class TreeContract<S extends Sink> implements Sink {
+  readonly inner: S
+  private open: Open[] = []
+  private rootDone = false
+
+  constructor(inner: S) {
+    this.inner = inner
+  }
+
+  // The path of the value due next: the open containers, each by the member
+  // or element open in it, and in the innermost object its last key when
+  // that member's value is due.
+  private path(): Path {
+    const segments: Segment[] = []
+    const last = this.open.length - 1
+    this.open.forEach((open, i) => {
+      if (open.array) {
+        segments.push(open.next)
+      } else if ((i !== last || !open.keyDue) && null != open.last) {
+        // A container open inside this object is its last key's value,
+        // whatever `keyDue` says: the member was counted as taken when its
+        // value opened.
+        segments.push(open.last)
+      }
+    })
+    return new Path(segments)
+  }
+
+  private notATree(what: string): Fail {
+    return new Fail(
+      'STREAMABILITY_UNKNOWN',
+      `the stream holds ${what}, which a tree's events never do, so it is not a tree's`,
+    ).atPath(this.path().toString())
+  }
+
+  // A value is complete: the next in its array is due, or the root is.
+  private closed(): void {
+    const top = this.open[this.open.length - 1]
+    if (undefined === top) this.rootDone = true
+    else if (top.array) top.next++
+  }
+
+  event(ev: JsonEvent): Flow {
+    const top = this.open[this.open.length - 1]
+    switch (ev.type) {
+      case 'key': {
+        if (undefined !== top && !top.array && top.keyDue) {
+          if (top.keys.has(ev.key)) {
+            const path = this.path()
+            path.push(ev.key)
+            throw new Fail(
+              'DUPLICATE_MEMBER',
+              `member ${JSON.stringify(ev.key)} appears twice in one object, and a tree's ` +
+                `events hold each key once`,
+            ).atPath(path.toString())
+          }
+          top.keys.add(ev.key)
+          top.last = ev.key
+          top.keyDue = false
+        } else if (undefined !== top && !top.array) {
+          throw this.notATree('a key where a value is due')
+        } else {
+          throw this.notATree('a key outside an object')
+        }
+        break
+      }
+      case 'object_end':
+        if (undefined !== top && !top.array && top.keyDue) {
+          this.open.pop()
+          this.closed()
+        } else if (undefined !== top && !top.array) {
+          throw this.notATree("an object's end where a value is due")
+        } else {
+          throw this.notATree("an object's end where none is due")
+        }
+        break
+      case 'array_end':
+        if (undefined !== top && top.array) {
+          this.open.pop()
+          this.closed()
+        } else {
+          throw this.notATree("an array's end where none is due")
+        }
+        break
+      case 'end':
+        if (0 < this.open.length) {
+          throw this.notATree('its end inside an open container')
+        }
+        break
+      default: {
+        // A value: in an object, only once its key is in; at the root,
+        // only once.
+        if (undefined === top) {
+          if (this.rootDone) throw this.notATree('a second root value')
+        } else if (!top.array) {
+          if (top.keyDue) throw this.notATree('a value where a key is due')
+          top.keyDue = true
+        }
+        if ('object_start' === ev.type) {
+          this.open.push({ array: false, keys: new Set(), last: null, keyDue: true })
+        } else if ('array_start' === ev.type) {
+          this.open.push({ array: true, next: 0 })
+        } else {
+          this.closed()
+        }
+      }
+    }
+    return this.inner.event(ev)
+  }
+}

package/src/source.ts

@@ -0,0 +1,135 @@
+/* Copyright (c) 2026 tabnas, MIT License */
+
+// Sources: where `JsonEvents/1` come from.
+//
+// - `ValueSource` walks a parsed engine value. Always correct, retains the
+//   whole value: the fallback for every grammar.
+// - `ParserSource` drives a tabnas parse of one text and, in incremental
+//   mode, turns its rule events into source events as they happen, for the
+//   grammars the differential suite has verified (`capability`); in
+//   materialize mode it parses and walks.
+// - `LinesSource` reads JSON Lines or CSV a record (or a chunk of records)
+//   at a time, bounding memory whatever the input's size.
+//
+// Every source emits through `Guarded`, which enforces the source limits,
+// polls the abort flag and counts the source metrics.
+
+import { Fail, engineCode, engineDetail } from './error'
+import { Ev } from './event'
+import { AbortFlag } from './limits'
+import { Selector } from './selector'
+import { Flow, Sink } from './sink'
+import { engineKeys, engineScalar } from './value'
+
+// Which arrays the incremental source empties as it streams them: none
+// (`never`), the array whose elements the selector names (`under`: a
+// trailing `[*]` names the elements; without one the selector names the
+// array), or every array (`all_arrays`). Pruning alters the value the
+// engine returns, which the incremental source discards; it is never
+// applied in materialize mode.
+export type Prune =
+  | { type: 'never' }
+  | { type: 'under'; selector: Selector }
+  | { type: 'all_arrays' }
+
+export const Prune = Object.freeze({
+  never(): Prune {
+    return { type: 'never' }
+  },
+  under(selector: Selector): Prune {
+    return { type: 'under', selector }
+  },
+  allArrays(): Prune {
+    return { type: 'all_arrays' }
+  },
+})
+
+// How `ParserSource` produces its events: parse the whole text, then walk
+// the value (`materialize`, sound for every grammar), or emit from the
+// engine's rule events as the parse proceeds (`incremental`, sound for the
+// grammars `capability.incremental` lists).
+export type SourceMode = { type: 'materialize' } | { type: 'incremental'; prune: Prune }
+
+export const SourceMode = Object.freeze({
+  materialize(): SourceMode {
+    return { type: 'materialize' }
+  },
+  incremental(prune: Prune = Prune.never()): SourceMode {
+    return { type: 'incremental', prune }
+  },
+})
+
+// Something that can drive a sink with one document's events. `run`
+// returns `'stop'` when the sink stopped it (the document was not
+// validated past that point) and throws a `Fail` on failure.
+export interface Source {
+  run(sink: Sink): Flow
+}
+
+// The engine's cancel code: what a cancelled parse reports, whether the
+// canceller is this crate's abort or a guard of the grammar's own.
+const CANCEL = 'cancel'
+
+// Map an engine error to a failure. A cancel while the caller's flag is set
+// is `ABORTED`. A cancel otherwise is a guard the GRAMMAR installed, so
+// the message says so instead of "parse cancelled", which would read as
+// the caller's doing; it stays `INPUT_INVALID`, because the document is
+// what the grammar refused. Any other error is the input's, with the
+// engine's code and position. A thrown value that is not an engine error
+// (a grammar's plugin threw) is the input's too, without a position.
+export function engineFailure(error: unknown, abort: AbortFlag): Fail {
+  if (error instanceof Fail) return error
+  if (CANCEL !== engineCode(error)) return Fail.fromTabnas(error)
+  if (abort.isAborted()) return Fail.aborted()
+  const fail = Fail.fromTabnas(error)
+  fail.message =
+    `the grammar stopped the parse with a guard of its own (${engineCode(error)}: ` +
+    `${engineDetail(error)}); a grammar may refuse nesting or size below this crate's Limits`
+  return fail
+}
+
+// Emit one engine value's events (without `end`). `undefined` is `null`,
+// as the engine serializes it; an object's members are in insertion order.
+export function walkValue(value: unknown, sink: Sink): Flow {
+  if (Array.isArray(value)) {
+    if ('stop' === sink.event(Ev.arrayStart)) return 'stop'
+    for (const item of value) {
+      if ('stop' === walkValue(item, sink)) return 'stop'
+    }
+    return sink.event(Ev.arrayEnd)
+  }
+  const scalar = engineScalar(value)
+  if (undefined !== scalar) {
+    switch (scalar.type) {
+      case 'null':
+        return sink.event(Ev.null)
+      case 'bool':
+        return sink.event(Ev.bool(scalar.value))
+      case 'number':
+        return sink.event(Ev.number(scalar.value))
+      case 'string':
+        return sink.event(Ev.string(scalar.value))
+    }
+  }
+  if ('stop' === sink.event(Ev.objectStart)) return 'stop'
+  for (const k of engineKeys(value)) {
+    if ('stop' === sink.event(Ev.key(k))) return 'stop'
+    if ('stop' === walkValue((value as any)[k], sink)) return 'stop'
+  }
+  return sink.event(Ev.objectEnd)
+}
+
+// Emit a parsed engine value as events, ending with `end`. Applies no
+// limits: wrap the sink in `Guarded` for those.
+export class ValueSource implements Source {
+  readonly value: unknown
+
+  constructor(value: unknown) {
+    this.value = value
+  }
+
+  run(sink: Sink): Flow {
+    if ('stop' === walkValue(this.value, sink)) return 'stop'
+    return sink.event(Ev.end)
+  }
+}