@tabnas/transduce 0.1.0 → 0.1.1

package/src/table-from-json.ts

@@ -0,0 +1,296 @@
+/* Copyright (c) 2026 tabnas, MIT License */
+
+// The metadata-first table transducer: `JsonEvents/1` in, `TableRows/1`
+// out.
+//
+// `TableFromJson` is a `Sink` built on a `Router` with at most two
+// captures: the column metadata, when the schema comes from the document,
+// and the rows. The schema must be known before the first row is emitted,
+// and the transducer holds one row at a time. So metadata that has not
+// completed when a row BEGINS is an `INPUT_ORDER_VIOLATION` raised at the
+// row's start, through the router's `began` hook, before a byte of the row
+// is retained; metadata that arrives twice is the same failure; and a
+// document with no rows is a valid empty table, its schema emitted just
+// before `end`. Rows are projected into schema order by path, so the order
+// of members inside a row never matters, and a number keeps the lexeme the
+// source events carried.
+
+import { Datum, Duplicates, getPath } from './datum'
+import { Fail } from './error'
+import { JsonEvent } from './event'
+import { Limits, Metrics, NODE_BYTES, utf8Bytes } from './limits'
+import { CaptureId } from './matcher'
+import { CaptureSpec, RouteSink, Router, Selected } from './route'
+import { Path, Selector } from './selector'
+import { Flow, Sink } from './sink'
+import {
+  BoundColumn,
+  Cell,
+  ColumnMapper,
+  PublicColumn,
+  TableBinding,
+  TableSink,
+  boundColumn,
+} from './table'
+
+// Where the columns are, once known.
+type Columns =
+  | { type: 'static'; bound: BoundColumn[] }
+  | { type: 'metadata'; selector: Selector; column: ColumnMapper; bound: BoundColumn[] | null }
+  | { type: 'infer'; bound: BoundColumn[] | null }
+
+// The route sink behind the transducer: it owns the schema state and the
+// table sink.
+class Core<S extends TableSink> implements RouteSink {
+  readonly sink: S
+  rowId: CaptureId
+  metaId: CaptureId | null
+  columns: Columns
+  schemaSent = false
+  private maxColumns: number
+  // The bound on the inferred columns' names, the table's metadata when
+  // the first row supplies it.
+  private maxMetadataBytes: number
+  private metrics: Metrics
+
+  constructor(
+    sink: S,
+    rowId: CaptureId,
+    metaId: CaptureId | null,
+    columns: Columns,
+    limits: Limits,
+    metrics: Metrics,
+  ) {
+    this.sink = sink
+    this.rowId = rowId
+    this.metaId = metaId
+    this.columns = columns
+    this.maxColumns = limits.max_columns
+    this.maxMetadataBytes = limits.max_metadata_bytes
+    this.metrics = metrics
+  }
+
+  bind(columns: BoundColumn[], from: string): void {
+    if (columns.length > this.maxColumns) {
+      throw Fail.limit(
+        'max_columns',
+        this.maxColumns,
+        `${from} declares ${columns.length} columns, more than ${this.maxColumns}`,
+      )
+    }
+    this.columns.bound = columns
+  }
+
+  private publicColumns(): PublicColumn[] {
+    return (this.columns.bound ?? []).map((c) => ({ label: c.label }))
+  }
+
+  private sendSchema(): Flow {
+    this.schemaSent = true
+    return this.sink.tableEvent({ type: 'schema', columns: this.publicColumns() })
+  }
+
+  private metadata(selected: Selected): void {
+    const columns = this.columns
+    if ('metadata' !== columns.type) {
+      // The router only has a metadata capture when the binding selects one.
+      throw Fail.protocol('metadata was delivered to a static schema')
+    }
+    const at = selected.path.toString()
+    if (null !== columns.bound) {
+      throw new Fail(
+        'INPUT_ORDER_VIOLATION',
+        `the column metadata at ${at} was selected twice; a table has one schema`,
+      ).atPath(at)
+    }
+    const value = selected.value ?? Datum.null
+    if ('array' !== value.type) {
+      throw Fail.input(`the column metadata at ${at} is not an array`).atPath(at)
+    }
+    const descriptors = value.items
+    if (descriptors.length > this.maxColumns) {
+      throw Fail.limit(
+        'max_columns',
+        this.maxColumns,
+        `the metadata at ${at} declares ${descriptors.length} columns, more than ` +
+          `${this.maxColumns}`,
+      ).atPath(at)
+    }
+    const bound = descriptors.map((d, i) => {
+      try {
+        return columns.column(d)
+      } catch (err) {
+        if (err instanceof Fail && null == err.path) {
+          const p = selected.path.clone()
+          p.push(i)
+          err.path = p.toString()
+        }
+        throw err
+      }
+    })
+    this.bind(bound, 'the metadata')
+  }
+
+  private row(selected: Selected): Flow {
+    const row = selected.value ?? Datum.null
+    if (!this.schemaSent) {
+      if ('infer' === this.columns.type && null === this.columns.bound) {
+        const at = selected.path.toString()
+        if ('object' !== row.type) {
+          throw Fail.input(
+            `the first row at ${at} is not an object, so no columns can be inferred from it`,
+          ).atPath(at)
+        }
+        // The names are the table's metadata for as long as it lasts, so
+        // they are held to the bound a metadata capture is: measured as the
+        // array of their strings would be.
+        let bytes = NODE_BYTES
+        for (const k of row.members.keys()) bytes += NODE_BYTES + utf8Bytes(k)
+        if (bytes > this.maxMetadataBytes) {
+          throw Fail.limit(
+            'max_metadata_bytes',
+            this.maxMetadataBytes,
+            `the first row's ${row.members.size} member names take ${bytes} bytes as the ` +
+              `table's columns, more than ${this.maxMetadataBytes}`,
+          ).atPath(at)
+        }
+        this.bind(
+          [...row.members.keys()].map((k) => boundColumn(k, [k])),
+          'the first row',
+        )
+      }
+      if ('stop' === this.sendSchema()) return 'stop'
+    }
+    const columns = this.columns.bound
+    if (null === columns) {
+      throw Fail.protocol('a row was projected before its schema was bound')
+    }
+    const cells: Cell[] = []
+    for (const col of columns) {
+      const found = getPath(row, col.source)
+      if (undefined !== found) {
+        cells.push(Cell.fromDatum(found))
+        continue
+      }
+      switch (col.missing) {
+        case 'missing':
+          cells.push(Cell.missing)
+          break
+        case 'null':
+          cells.push(Cell.null)
+          break
+        case 'error': {
+          const at = new Path([...selected.path.segments, ...col.source]).toString()
+          throw new Fail(
+            'MISSING_VALUE',
+            `column ${JSON.stringify(col.label)} has no value at ${at}`,
+          ).atPath(at)
+        }
+      }
+    }
+    this.metrics.rows++
+    return this.sink.tableEvent({ type: 'row', cells })
+  }
+
+  began(id: CaptureId, _tag: string): void {
+    if (id === this.rowId && 'metadata' === this.columns.type && null === this.columns.bound) {
+      throw new Fail(
+        'INPUT_ORDER_VIOLATION',
+        `a row began before the column metadata at ${this.columns.selector} had completed; ` +
+          `rows must follow their metadata`,
+      )
+    }
+  }
+
+  selected(selected: Selected): Flow {
+    if (selected.id === this.metaId) {
+      this.metadata(selected)
+      return 'continue'
+    }
+    return this.row(selected)
+  }
+
+  end(): Flow {
+    if (!this.schemaSent) {
+      if ('metadata' === this.columns.type && null === this.columns.bound) {
+        const at = this.columns.selector.toString()
+        throw Fail.input(`the document has no column metadata at ${at}`).atPath(at)
+      }
+      // No rows: nothing to infer from, so the table is empty.
+      if ('infer' === this.columns.type && null === this.columns.bound) {
+        this.bind([], 'the binding')
+      }
+      if ('stop' === this.sendSchema()) return 'stop'
+    }
+    return this.sink.tableEvent({ type: 'end' })
+  }
+}
+
+// The table transducer. A `Sink` for one document's events; the table
+// events go to the wrapped `TableSink` as the rows arrive.
+export class TableFromJson<S extends TableSink> implements Sink {
+  private router: Router<Core<S>>
+
+  // Build the transducer. Rows are materialized under `max_record_bytes`,
+  // metadata under `max_metadata_bytes`; a rows selector that may overlap
+  // the metadata selector is refused as the router refuses any overlapping
+  // materializations.
+  constructor(
+    binding: TableBinding,
+    limits: Limits,
+    duplicates: Duplicates,
+    metrics: Metrics,
+    sink: S,
+  ) {
+    const rowSpec = CaptureSpec.materialize('row', binding.rows).withBudget(
+      limits.max_record_bytes,
+      'max_record_bytes',
+    )
+    let columns: Columns
+    let specs: CaptureSpec[]
+    let metaId: CaptureId | null = null
+    const schema = binding.schema
+    switch (schema.type) {
+      case 'static':
+        columns = { type: 'static', bound: schema.columns.slice() }
+        specs = [rowSpec]
+        break
+      case 'metadata':
+        columns = {
+          type: 'metadata',
+          selector: schema.columns,
+          column: schema.column,
+          bound: null,
+        }
+        specs = [
+          CaptureSpec.materialize('metadata', schema.columns).withBudget(
+            limits.max_metadata_bytes,
+            'max_metadata_bytes',
+          ),
+          rowSpec,
+        ]
+        metaId = 0
+        break
+      case 'infer':
+        columns = { type: 'infer', bound: null }
+        specs = [rowSpec]
+        break
+    }
+    const core = new Core(sink, specs.length - 1, metaId, columns, limits, metrics)
+    if ('static' === columns.type) core.bind(columns.bound, 'the binding')
+    this.router = new Router(specs, limits, duplicates, metrics, core)
+  }
+
+  get sink(): S {
+    return this.router.downstream.sink
+  }
+
+  // Whether the table's `end` has been emitted.
+  ended(): boolean {
+    return this.router.ended()
+  }
+
+  event(ev: JsonEvent): Flow {
+    return this.router.event(ev)
+  }
+}

package/src/table.ts

@@ -0,0 +1,213 @@
+/* Copyright (c) 2026 tabnas, MIT License */
+
+// `TableRows/1`: one schema, ordered finite rows, completion.
+//
+// The protocol is flat and budgeted: a row is a finite vector of cells,
+// already projected into schema order by the transducer, and a renderer
+// never sees a source path. Public columns carry a label; the source
+// binding (`BoundColumn`) stays on the transducer's side of the boundary.
+
+import { Datum, toText } from './datum'
+import { Fail } from './error'
+import { jsonNumber, jsonString } from './json'
+import { NODE_BYTES, utf8Bytes } from './limits'
+import { Segment, Selector } from './selector'
+import { Flow } from './sink'
+
+// One projected value. `missing`: the source had no value at the column's
+// path; not null, not the empty string, not zero: a policy maps or rejects
+// it later.
+export type Cell =
+  | { readonly type: 'null' }
+  | { readonly type: 'bool'; readonly value: boolean }
+  | { readonly type: 'number'; readonly value: number; readonly lexeme: string | null }
+  | { readonly type: 'string'; readonly value: string }
+  | { readonly type: 'missing' }
+
+const NULL: Cell = Object.freeze({ type: 'null' })
+const MISSING: Cell = Object.freeze({ type: 'missing' })
+
+export const Cell = Object.freeze({
+  null: NULL,
+  missing: MISSING,
+  bool(value: boolean): Cell {
+    return { type: 'bool', value }
+  },
+  number(value: number, lexeme?: string | null): Cell {
+    return { type: 'number', value, lexeme: lexeme ?? null }
+  },
+  string(value: string): Cell {
+    return { type: 'string', value }
+  },
+
+  // From a retained value. A container is not a cell; it is serialized as
+  // compact JSON text, the lossy but unambiguous choice the standard
+  // binding makes and documents.
+  fromDatum(d: Datum): Cell {
+    switch (d.type) {
+      case 'null':
+        return NULL
+      case 'bool':
+        return { type: 'bool', value: d.value }
+      case 'number':
+        return { type: 'number', value: d.value, lexeme: d.lexeme }
+      case 'string':
+        return { type: 'string', value: d.value }
+      default:
+        return { type: 'string', value: toText(d) }
+    }
+  },
+
+  // The bytes this cell retains, on the same basis as a datum's size.
+  byteSize(c: Cell): number {
+    switch (c.type) {
+      case 'number':
+        return NODE_BYTES + (null != c.lexeme ? utf8Bytes(c.lexeme) : 8)
+      case 'string':
+        return NODE_BYTES + utf8Bytes(c.value)
+      default:
+        return NODE_BYTES
+    }
+  },
+
+  // The cell as JSON text; `missing` prints as `missing`.
+  toText(c: Cell): string {
+    switch (c.type) {
+      case 'null':
+        return 'null'
+      case 'bool':
+        return String(c.value)
+      case 'number':
+        return jsonNumber(c.value, c.lexeme)
+      case 'string':
+        return jsonString(c.value)
+      case 'missing':
+        return 'missing'
+    }
+  },
+})
+
+// What a renderer knows about a column.
+export type PublicColumn = { label: string }
+
+// One event of `TableRows/1`: the `schema` exactly once and first, a `row`
+// per row (each exactly as wide as the schema), the `end` exactly once and
+// last, and only after the source validated to its end.
+export type TableEvent =
+  | { readonly type: 'schema'; readonly columns: readonly PublicColumn[] }
+  | { readonly type: 'row'; readonly cells: readonly Cell[] }
+  | { readonly type: 'end' }
+
+// A consumer of `TableRows/1`. Throws a `Fail` to fail the run.
+export interface TableSink {
+  tableEvent(ev: TableEvent): Flow
+}
+
+// An owned recording of a table, for tests and small results.
+export class Table implements TableSink {
+  columns: PublicColumn[] = []
+  rows: Cell[][] = []
+  ended = false
+
+  tableEvent(ev: TableEvent): Flow {
+    switch (ev.type) {
+      case 'schema':
+        this.columns = ev.columns.slice()
+        break
+      case 'row':
+        this.rows.push(ev.cells.slice())
+        break
+      case 'end':
+        this.ended = true
+        break
+    }
+    return 'continue'
+  }
+}
+
+// What to do when a row has no value at a column's path: deliver a
+// `missing` cell (the default; the renderer's policy decides), deliver
+// `null`, or fail the run with `MISSING_VALUE`.
+export type MissingPolicy = 'missing' | 'null' | 'error'
+
+// A column as the transducer binds it: the public label, and the source
+// path projected from each row. Never crosses into a renderer.
+export type BoundColumn = {
+  label: string
+  source: Segment[]
+  missing: MissingPolicy
+}
+
+export function boundColumn(
+  label: string,
+  source: Segment[],
+  missing: MissingPolicy = 'missing',
+): BoundColumn {
+  return { label, source, missing }
+}
+
+// Maps one metadata descriptor to a bound column; throws a `Fail` for a
+// descriptor it cannot read.
+export type ColumnMapper = (meta: Datum) => BoundColumn
+
+// Where a table's columns come from: declared by the caller (`static`),
+// selected from the source and mapped one descriptor at a time
+// (`metadata`; the metadata must complete before the first row begins),
+// or the first row's member names in its order (`infer`; data-dependent: a
+// later row's extra members are dropped, its absent ones are `missing`).
+export type Schema =
+  | { type: 'static'; columns: BoundColumn[] }
+  | { type: 'metadata'; columns: Selector; column: ColumnMapper }
+  | { type: 'infer' }
+
+export const Schema = Object.freeze({
+  static(columns: BoundColumn[]): Schema {
+    return { type: 'static', columns }
+  },
+  fromMetadata(columns: Selector, column: ColumnMapper = columnFromMeta): Schema {
+    return { type: 'metadata', columns, column }
+  },
+  infer(): Schema {
+    return { type: 'infer' }
+  },
+})
+
+// A table transducer's source binding: the schema, and the rows (each
+// location `rows` names is one row).
+export type TableBinding = {
+  schema: Schema
+  rows: Selector
+}
+
+// The standard mapping from a metadata descriptor to a column: the spec's
+// `column-from-meta`, reading `title` and a `path` of segments.
+export function columnFromMeta(meta: Datum): BoundColumn {
+  if ('object' !== meta.type) {
+    throw Fail.input('a column descriptor is not an object')
+  }
+  const title = meta.members.get('title')
+  if (undefined === title || 'string' !== title.type) {
+    throw Fail.input('a column descriptor has no string "title"')
+  }
+  const label = title.value
+  const path = meta.members.get('path')
+  if (undefined === path || 'array' !== path.type) {
+    throw Fail.input(`column ${JSON.stringify(label)} has no "path" array`)
+  }
+  const source: Segment[] = path.items.map((seg) => {
+    if ('string' === seg.type) return seg.value
+    if (
+      'number' === seg.type &&
+      seg.value >= 0 &&
+      Number.isInteger(seg.value) &&
+      seg.value <= 0xffffffff
+    ) {
+      return seg.value
+    }
+    throw Fail.input(
+      `column ${JSON.stringify(label)} has a path segment that is neither a string nor a ` +
+        `non-negative integer: ${toText(seg)}`,
+    )
+  })
+  return boundColumn(label, source)
+}

package/src/transduce.ts

@@ -0,0 +1,72 @@
+/* Copyright (c) 2026 tabnas, MIT License */
+
+// Streaming transducers over any tabnas parser.
+//
+// A transducer decides what a source contributes to an output model; a
+// renderer decides how that model becomes text. This package is the
+// transducer side:
+//
+// - the source protocol `JsonEvent` (`JsonEvents/1`) and the push boundary
+//   `Sink`;
+// - sources that produce it from a parsed value, from a live tabnas parse
+//   through the engine's rule events, or from line-delimited input;
+// - `Selector`s and the matcher and router that recognize selected scopes
+//   in one pass and materialize them under a byte limit;
+// - the table protocol `TableEvent` (`TableRows/1`) and the standard
+//   metadata-first table transducer;
+// - `Limits`, `Metrics`, `AbortFlag` and the stable failure `Code`s every
+//   stage reports.
+//
+// Everything is synchronous and push-based: the parser calls the first
+// sink from inside its own callback, and a slow writer at the end of the
+// chain slows the parse at the start. That is the backpressure.
+
+// This package's version, as package.json declares it.
+export const VERSION = '0.1.1'
+
+export {
+  Datum,
+  DatumBuilder,
+  byteSize,
+  fromJSON,
+  fromTabnas,
+  getPath,
+  takePath,
+  toJSON,
+  toText,
+  walkDatum,
+} from './datum'
+export type { Duplicates } from './datum'
+export { Code, Fail, isFail } from './error'
+export type { Limit } from './error'
+export { Ev, eventEquals, eventText, isEnd, isScalar, isStart } from './event'
+export type { JsonEvent, JsonEventType } from './event'
+export { isJsonNumber, jsonNumber, jsonString, numberText } from './json'
+export { AbortFlag, LIMIT_NAMES, Limits, Metrics, NODE_BYTES, utf8Bytes } from './limits'
+export { Matcher } from './matcher'
+export type { CaptureId, Hit, HitKind } from './matcher'
+export { CaptureSpec, FnRoute, Router, SelectedRecorder } from './route'
+export type { Budget, CaptureMode, RouteSink, Selected } from './route'
+export { ScanEmit, Transition } from './scan'
+export { Path, Selector, keyText, pathText } from './selector'
+export type { Segment, Step } from './selector'
+export { CountSink, EventRecorder, FnSink, TreeContract, replay } from './sink'
+export type { Flow, Sink } from './sink'
+export { Prune, SourceMode, ValueSource, engineFailure, walkValue } from './source'
+export type { Source } from './source'
+export { Guarded } from './guard'
+export { ParserSource } from './parser-source'
+export { DEFAULT_CHUNK_BYTES, LineFormat, LinesSource } from './lines'
+export type { LinesChunk, LinesInput, LinesWriter } from './lines'
+export { INCREMENTAL, capability, isIncremental } from './capability'
+export { Cell, Schema, Table, boundColumn, columnFromMeta } from './table'
+export type {
+  BoundColumn,
+  ColumnMapper,
+  MissingPolicy,
+  PublicColumn,
+  TableBinding,
+  TableEvent,
+  TableSink,
+} from './table'
+export { TableFromJson } from './table-from-json'

package/src/tsconfig.json

@@ -0,0 +1,16 @@
+{
+  "compilerOptions": {
+    "esModuleInterop": true,
+    "module": "nodenext",
+    "noEmitOnError": true,
+    "outDir": "../dist",
+    "rootDir": ".",
+    "resolveJsonModule": true,
+    "sourceMap": true,
+    "strict": true,
+    "target": "es2021",
+    "declaration": true,
+    "declarationDir": "../dist",
+    "types": ["node"]
+  }
+}

package/src/value.ts

@@ -0,0 +1,93 @@
+/* Copyright (c) 2026 tabnas, MIT License */
+
+// Reading the engine's values: what a tabnas parse returns in TypeScript.
+//
+// The engine builds plain JavaScript values: `null` and `undefined`,
+// booleans, numbers, strings, arrays and objects. A few grammars hand over
+// host types, which read as the scalars they stand for: a boxed `String`
+// (the engine's text annotation), a `BigInt`, and a TOML `Date`, which
+// reads as the source text it carries (`__toml__.src`), as the Rust
+// runtime's date does.
+//
+// An object's member order is its insertion order. A plain object
+// enumerates integer-like keys first whatever order they were written in,
+// so the engine records the order on the node under `map.ordered` (read
+// with `KEY_ORDER`); the sources switch that on, and `engineKeys` reads
+// it, falling back to `Object.keys`.
+
+// The engine's key-order side channel (`keyOrder` in @tabnas/parser).
+const KEY_ORDER = Symbol.for('tabnas.keyOrder')
+
+export type EngineScalar =
+  | { type: 'null' }
+  | { type: 'bool'; value: boolean }
+  | { type: 'number'; value: number }
+  | { type: 'string'; value: string }
+
+const NULL: EngineScalar = { type: 'null' }
+
+// The value as a scalar, or `undefined` when it is a container.
+export function engineScalar(v: unknown): EngineScalar | undefined {
+  switch (typeof v) {
+    case 'undefined':
+      return NULL
+    case 'boolean':
+      return { type: 'bool', value: v }
+    case 'number':
+      return { type: 'number', value: v }
+    case 'bigint':
+      return { type: 'number', value: Number(v) }
+    case 'string':
+      return { type: 'string', value: v }
+    case 'object':
+      if (null === v) return NULL
+      if (Array.isArray(v)) return undefined
+      if (v instanceof String) return { type: 'string', value: v.valueOf() }
+      if (v instanceof Number) return { type: 'number', value: v.valueOf() }
+      if (v instanceof Boolean) return { type: 'bool', value: v.valueOf() }
+      if (v instanceof Date) {
+        const src = (v as any).__toml__?.src
+        return { type: 'string', value: 'string' === typeof src ? src : v.toJSON() ?? 'null' }
+      }
+      return undefined
+    default:
+      // A function or a symbol has no JSON form.
+      return NULL
+  }
+}
+
+// Whether the value is a container: an array, or an object that is no
+// scalar's host type.
+export function isEngineContainer(v: unknown): boolean {
+  return null !== v && 'object' === typeof v && undefined === engineScalar(v)
+}
+
+// Whether the value is a map (a container that is not an array).
+export function isEngineMap(v: unknown): v is Record<string, unknown> {
+  return isEngineContainer(v) && !Array.isArray(v)
+}
+
+// A map's member names, in insertion order: the engine's recorded order
+// where it kept one, each name once and only while the map still holds it,
+// then any member the record missed.
+export function engineKeys(v: unknown): string[] {
+  if (null === v || 'object' !== typeof v) return []
+  const own = Object.keys(v as object)
+  const order: unknown = (v as any)[KEY_ORDER]
+  if (!Array.isArray(order) || 0 === order.length) return own
+  const has = Object.prototype.hasOwnProperty
+  const seen = new Set<string>()
+  const out: string[] = []
+  for (const k of order) {
+    const key = String(k)
+    if (seen.has(key) || !has.call(v, key)) continue
+    seen.add(key)
+    out.push(key)
+  }
+  if (out.length !== own.length) {
+    for (const k of own) {
+      if (!seen.has(k)) out.push(k)
+    }
+  }
+  return out
+}