jsonstat-io 0.1.0

package/dist/index.d.cts

@@ -0,0 +1,445 @@
+import { J as JsonStatDataset, c as JsonStatResponse } from './jsonstat-BX4mBLci.cjs';
+export { C as Coordinates, d as JsonStatCategory, e as JsonStatClass, f as JsonStatCollection, g as JsonStatDimension, h as JsonStatDimensionResponse, i as JsonStatError, j as JsonStatLink, k as JsonStatLinkMap, a as JsonStatRole, l as JsonStatStatus, b as JsonStatUnit, m as JsonStatValue, n as isCollection, o as isDataset, p as isDimensionResponse } from './jsonstat-BX4mBLci.cjs';
+import { D as DatasetMeta, O as Observations } from './ir-BArfNxo-.cjs';
+export { C as CubeModel, a as DimensionColumn, M as MeasureColumn, R as RoleMap, S as StatusColumn, o as observationCount } from './ir-BArfNxo-.cjs';
+export { META_PREFIX, buildFieldMeta, buildSchemaMeta, cubeToArrow, getFieldMeta, getFieldMetaJson, getFieldRole, isMeasureField, isStatusField, readSchemaMeta } from './arrow/index.cjs';
+import { A as ArrowToCubeOptions } from './arrowToCube-BdRWZ5oG.cjs';
+export { a as ArrowConversionError, b as arrowToCube, c as arrowToDataset } from './arrowToCube-BdRWZ5oG.cjs';
+import { Table } from 'apache-arrow';
+
+/**
+ * Row-major index math — the correctness backbone of the importer.
+ *
+ * JSON-stat stores cell values in a flat array laid out in **row-major order**,
+ * described by the spec as "what does not change, first": the *last* dimension
+ * in `id` changes fastest (see wiki/format-specification.md and
+ * wiki/sparse-cubes.md).
+ *
+ * Concretely, for dimensions with sizes `s₀, s₁, …, sₖ₋₁`, the flat position of
+ * a cell at multi-index `(i₀, i₁, …, iₖ₋₁)` is:
+ *
+ *     pos = i₀·(s₁·…·sₖ₋₁) + i₁·(s₂·…·sₖ₋₁) + … + iₖ₋₂·sₖ₋₁ + iₖ₋₁
+ *
+ * i.e. the stride of dimension `d` is the product of the sizes of all
+ * dimensions that come *after* it. This module centralizes that arithmetic so
+ * every converter and the round-trip tests share one source of truth.
+ */
+/** Compute row-major strides from a `size` array. strides[d] = ∏ size[d+1..]. */
+declare function strides(size: number[]): number[];
+/** Total number of cells = product of all sizes (the dense cell count). */
+declare function totalCells(size: number[]): number;
+/**
+ * Convert a multi-index to a flat row-major position.
+ *
+ * `indices` length must equal `size` length; each `indices[d]` must satisfy
+ * `0 ≤ indices[d] < size[d]`.
+ */
+declare function flatPosition(indices: number[], size: number[]): number;
+/**
+ * Inverse of [`flatPosition`](#flatposition): decompose a flat position back
+ * into a multi-index. Used by the cube reader (Phase-2 seam) and by tests.
+ */
+declare function multiIndex(pos: number, size: number[]): number[];
+/**
+ * Enumerate every multi-index in row-major order, yielding flat positions
+ * 0..(total-1). Useful for materializing a dense value array from a sparse map.
+ *
+ * Calls `visit(indices, flatPos)` for each cell. Iterative (no recursion) to
+ * avoid stack overflow on large cubes.
+ */
+declare function enumerateCells(size: number[], visit: (indices: number[], flatPos: number) => void): void;
+
+/**
+ * CubeBuilder — turns an [`Observations`](../model/ir.ts) IR into a JSON-stat
+ * [`Dataset`](../model/jsonstat.ts).
+ *
+ * This is the central transform of the importer: a tidy long table → an
+ * N-dimensional cube laid out in row-major order ("what does not change, first";
+ * see wiki/format-specification.md).
+ *
+ * Responsibilities:
+ *  1. Resolve dimension order and enumerate categories (first-seen order unless
+ *     `categoryOrder` pins it).
+ *  2. Build the `dimension` block with `category.index`/`label`/`unit`/
+ *     `coordinates`/`child` and dimension `label`.
+ *  3. Scatter the measure values into a row-major `value` array (dense) or a
+ *     sparse object, per [`CubeModel.valueForm`](../model/ir.ts) and the null
+ *     ratio threshold.
+ *  4. Emit `status` in the most compact correct form (string/array/object).
+ *  5. Attach `role`, and dataset-level `label`/`source`/`updated`/`extension`.
+ *
+ * Pure: no I/O, no side effects. All index math delegated to
+ * [`strides`](./strides.ts).
+ */
+
+/** Thrown when the IR violates a structural invariant the builder relies on. */
+declare class CubeBuilderError extends Error {
+    constructor(message: string);
+}
+interface BuildOptions {
+    /** Override the emitted value form. Defaults to `model.valueForm ?? "auto"`. */
+    valueForm?: "auto" | "dense" | "sparse";
+    /** Override the sparse threshold (null ratio). Defaults to 0.5. */
+    sparseThreshold?: number;
+    /** Override status emission. Defaults to `model.statusForm ?? "auto"`. */
+    statusForm?: "auto" | "array" | "string" | "object" | "none";
+    /** Override dataset metadata. Defaults to `model.meta`. */
+    meta?: DatasetMeta;
+}
+interface BuildResult {
+    dataset: JsonStatDataset;
+    /** Diagnostics from the build (non-fatal observations). */
+    diagnostics: {
+        valueForm: "dense" | "sparse";
+        nullRatio: number;
+        duplicates: number;
+        cellCount: number;
+    };
+}
+/**
+ * Build a JSON-stat [`Dataset`](../model/jsonstat.ts) from an
+ * [`Observations`](../model/ir.ts) IR.
+ *
+ * @throws [`CubeBuilderError`](#cubebuildererror) on invariant violations.
+ */
+declare function buildDataset(obs: Observations, options?: BuildOptions): BuildResult;
+/**
+ * Convenience: build and return *just* the dataset (no diagnostics).
+ * Useful for the common programmatic case.
+ */
+declare function toDataset(obs: Observations, options?: BuildOptions): JsonStatDataset;
+
+/**
+ * CubeReader — the inverse of [`CubeBuilder`](./cubeBuilder.ts): converts a
+ * JSON-stat [`Dataset`](../model/jsonstat.ts) back into the
+ * [`Observations`](../model/ir.ts) IR (a tidy long table).
+ *
+ * ## Why this exists now (Phase-1 import project)
+ *
+ * Although the primary deliverable is *import* (columnar → JSON-stat), the
+ * reader is shipped in v0.1 for two reasons:
+ *
+ *  1. **Round-trip fidelity tests.** The canonical JSON-stat samples
+ *     (oecd, canada, galicia, order, hierarchy, us-gsp, us-unr — see
+ *     wiki/sample-files.md) are read into the IR, written back through the
+ *     builder, and asserted equal. This is the strongest correctness proof we
+ *     have that the importer faithfully preserves the JSON-stat model.
+ *
+ *  2. **The Phase-2 export seam.** Export (JSON-stat → columnar) is
+ *     `cubeReader` → IR → [`arrowFromCube`](../arrow/arrowFromCube.ts) →
+ *     Arrow/Parquet/DuckDB/Polars/CSV writers. Building the reader now means
+ *     the IR is verified bidirectional, and Phase 2 is "just" the writers.
+ *     See [docs/architecture.md](../../docs/architecture.md) §"The two seams".
+ *
+ * ## Semantics
+ *
+ * The reader *materializes the dense cube* from `value` (handling both the
+ * array form and the sparse object form — see wiki/sparse-cubes.md), then walks
+ * every cell in row-major order ([`multiIndex`](./strides.ts)) to emit one
+ * observation per cell. Missing cells (`null`) are emitted as rows with a
+ * `null` measure, preserving the full Cartesian product of categories — which
+ * is what round-trip equality requires.
+ *
+ * `status` is normalized back to a per-row `StatusColumn` regardless of whether
+ * the source used the string, array, or object form.
+ */
+
+/** Thrown when a dataset cannot be read (malformed or unsupported). */
+declare class CubeReaderError extends Error {
+    constructor(message: string);
+}
+interface ReadOptions {
+    /**
+     * When true (default), cells whose measure is `null` are *omitted* from the
+     * emitted long table (a truly sparse/tidy representation). When false, every
+     * cell — including nulls — is emitted as a row (required for exact round-trip
+     * equality against a dense `value` array).
+     */
+    dropNulls?: boolean;
+}
+/**
+ * Read a JSON-stat [`Dataset`](../model/jsonstat.ts) into the
+ * [`Observations`](../model/ir.ts) IR.
+ *
+ * @throws [`CubeReaderError`](#cubereadererror) on structural problems.
+ */
+declare function readDataset(dataset: JsonStatDataset, options?: ReadOptions): Observations;
+/** Convenience: read a JSON-stat response, rejecting non-dataset classes. */
+declare function readResponse(response: unknown, options?: ReadOptions): Observations;
+
+/**
+ * Serialize — turn a JSON-stat [`Dataset`](../model/jsonstat.ts) into a JSON
+ * string or Buffer for writing to stdout/file.
+ *
+ * Also offers a lightweight "pretty" mode that sorts top-level keys in the
+ * canonical JSON-stat order (version, class, label, href, source, updated,
+ * id, size, dimension, role, value, status, …) so output is stable and
+ * diff-friendly — useful for fixtures and round-trip tests.
+ */
+
+interface SerializeOptions {
+    /** Pretty-print with 2-space indentation (default true). */
+    pretty?: boolean;
+    /** Reorder top-level keys canonically (default true, only with pretty). */
+    canonicalKeys?: boolean;
+}
+/**
+ * Serialize a JSON-stat response to a JSON string.
+ *
+ * @param response The dataset (or any JSON-stat response).
+ */
+declare function serialize(response: JsonStatResponse | JsonStatDataset, options?: SerializeOptions): string;
+/**
+ * Serialize and encode as UTF-8 bytes (Uint8Array). Handy for file writing
+ * without pulling in Node's Buffer in browser contexts.
+ */
+declare function serializeToBytes(response: JsonStatResponse | JsonStatDataset, options?: SerializeOptions): Uint8Array;
+
+/**
+ * Format detection — sniffs the input source to pick the right adapter.
+ *
+ * Two signals are used, in order of reliability:
+ *  1. **Magic bytes** (content sniffing) — authoritative for binary formats
+ *     (Parquet `PAR1`, Arrow IPC `ARROW1`).
+ *  2. **File extension** — used when content is not yet available (e.g. a path
+ *     or URL) or for text formats (`.csv`, `.csvw`).
+ */
+type SourceFormat = "parquet" | "arrow" | "csv" | "csvw" | "jsonstat" | "json" | "unknown";
+/**
+ * Sniff the format from the leading bytes of a binary/text input.
+ *
+ * @returns The detected format, or "unknown" if no magic matches.
+ */
+declare function detectFromBytes(bytes: Uint8Array | ArrayBuffer): SourceFormat;
+/** Map a file extension (without dot) to a source format. */
+declare function detectFromExtension(ext: string): SourceFormat;
+/** Extract the extension from a path/URL (without the dot), or undefined. */
+declare function extensionOf(path: string): string | undefined;
+/**
+ * Detect a format from a filename/URL, falling back to bytes if the extension
+ * is unknown or ambiguous.
+ */
+declare function detectFormat(pathOrUrl: string | undefined, bytes?: Uint8Array | ArrayBuffer): SourceFormat;
+
+/**
+ * Density helper — sparse vs dense `value` decision.
+ *
+ * Centralizes the null-ratio threshold logic so the cube builder and tests
+ * share one definition. See wiki/sparse-cubes.md for the rationale: the object
+ * (sparse) form is preferred when many cells are null.
+ */
+interface DensityResult {
+    form: "dense" | "sparse";
+    nullRatio: number;
+    nullCount: number;
+    total: number;
+}
+interface DensityOptions {
+    /** Null ratio above which "auto" chooses sparse. Default 0.5. */
+    threshold?: number;
+}
+/**
+ * Decide dense vs sparse for a flat value array.
+ *
+ * @param values The dense row-major array (with nulls).
+ * @param requested "auto" | "dense" | "sparse".
+ */
+declare function decideDensity(values: (number | null)[], requested?: "auto" | "dense" | "sparse", options?: DensityOptions): DensityResult;
+
+/**
+ * Isomorphic input loader — works in both Node and the browser.
+ *
+ * Resolves a source specifier (file path, URL, or raw bytes/text) into a
+ * `Uint8Array` of bytes and, where possible, a hint about its origin (path/URL)
+ * so format detection can use the extension.
+ *
+ * - **Node**: reads from `node:fs` for local paths; uses `fetch` for URLs and
+ *   for the browser. Reading from stdin is supported via the special source
+ *   `"-"`.
+ * - **Browser**: uses the global `fetch` for URLs and `File`/`Blob` inputs;
+ *   local paths are not available.
+ *
+ * The Node-specific `fs`/`stdin` imports are guarded by a runtime check so the
+ * same code path works in browsers without bundling `node:fs`.
+ */
+interface LoadedInput {
+    bytes: Uint8Array;
+    /** The original source path/URL, if known (used for extension detection). */
+    source?: string;
+    /** True if the source was stdin ("-"). */
+    fromStdin?: boolean;
+}
+/**
+ * Load input bytes from a path, URL, `"-"` (stdin), `Uint8Array`, `Blob`, or
+ * string.
+ *
+ * @param source A file path, `http(s)://` URL, `"-"` for stdin, or raw data.
+ * @throws If the source cannot be read.
+ */
+declare function loadInput(source: string | Uint8Array | Blob): Promise<LoadedInput>;
+
+/**
+ * jsonstat-io — public API surface (main entry).
+ *
+ * This package is a **bidirectional** bridge between JSON-stat 2.0 and the
+ * columnar stack (Arrow, Parquet, DuckDB, Polars, CSVW, CSV):
+ *
+ * - **Import** (columnar → JSON-stat): [`importToCube`](#importtocube) /
+ *   [`importToDataset`](#importtodataset) — load, auto-detect, convert, build.
+ * - **Export** (JSON-stat → columnar): [`exportDataset`](#exportdataset) — read
+ *   the JSON-stat cube into the IR and route to the requested sink writer.
+ *
+ * This barrel re-exports the **always loaded** layers: the model types, the
+ * core cube engine (strides, builder, reader), the Arrow hub, the sink, and the
+ * isomorphic utilities (format detection, input loading, density decision).
+ *
+ * Source/sink adapters that pull in optional peer dependencies live behind
+ * **subpath exports** so that browser bundles stay lean and tree-shakeable:
+ *
+ * - `jsonstat-io/arrow`  — Arrow hub (re-exported here too, since
+ *   `apache-arrow` is a hard dependency, not an optional peer).
+ * - `jsonstat-io/parquet` — Parquet ↔ Arrow ↔ cube (needs `parquet-wasm`).
+ * - `jsonstat-io/duckdb`  — DuckDB ↔ Arrow ↔ cube (needs duckdb).
+ * - `jsonstat-io/polars`  — Polars ↔ Arrow ↔ cube (Node-only).
+ * - `jsonstat-io/csvw`    — CSVW + CSV metadata-aware path (no deps).
+ * - `jsonstat-io/csv`     — plain CSV inference path (no deps).
+ *
+ * For a one-call convenience, see the dispatchers:
+ * - Import: [`importToCube`](#importtocube) / [`importToDataset`](#importtodataset).
+ * - Export: [`exportDataset`](#exportdataset).
+ *
+ * @packageDocumentation
+ */
+
+/**
+ * Options for the high-level [`importToCube`](#importtocube) /
+ * [`importToDataset`](#importtodataset) dispatchers.
+ *
+ * These combine format detection, source loading, and Arrow-hub conversion
+ * hints into a single call.
+ */
+interface ImportOptions extends ArrowToCubeOptions {
+    /**
+     * Force a source format instead of auto-detecting. One of `"parquet"`,
+     * `"arrow"`, `"csv"`, `"csvw"`, `"jsonstat"`, `"json"`. When omitted the
+     * format is sniffed from magic bytes (Parquet `PAR1`, Arrow IPC `ARROW1`)
+     * and then from the file extension.
+     */
+    from?: SourceFormat | "auto";
+    /** Passed through to [`buildDataset`](./core/cubeBuilder.ts) as `BuildOptions`. */
+    build?: BuildOptions;
+    /**
+     * CSVW metadata, when the source is CSVW. If `from === "csvw"` and this is
+     * omitted, the dispatcher attempts to load a sibling `*-metadata.json` next
+     * to the CSV path (Node only). In the browser the caller must supply it.
+     */
+    csvwMetadata?: unknown;
+    /**
+     * CSV delimiter (default `,`). Only used when the source is plain CSV.
+     */
+    delimiter?: string;
+}
+/**
+ * Error thrown by the high-level dispatchers when a format cannot be detected,
+ * a required peer dependency is missing, or the source produces no data.
+ */
+declare class ImporterError extends Error {
+    constructor(message: string);
+}
+/**
+ * Load a columnar source (file path, URL, stdin `"-"`, `Uint8Array`, or `Blob`)
+ * and convert it to the [`Observations`](./model/ir.ts) IR.
+ *
+ * This is the one-call convenience that ties together
+ * [`loadInput`](./util/fetch.ts), [`detectFormat`](./util/detect.ts), the
+ * appropriate source adapter, and the Arrow hub. It auto-detects the format
+ * unless [`options.from`](#ImportOptions.from) is set.
+ *
+ * Parquet/DuckDB/Polars adapters are imported lazily, so they are only pulled
+ * into the bundle when actually needed.
+ *
+ * @param source A file path, `http(s)://` URL, `"-"` for stdin (Node),
+ *   `Uint8Array`, or `Blob`.
+ * @returns The [`Observations`](./model/ir.ts) IR, ready for
+ *   [`buildDataset`](./core/cubeBuilder.ts).
+ *
+ * @example
+ * ```ts
+ * import { importToDataset } from "jsonstat-io";
+ * const dataset = await importToDataset("./sales.parquet");
+ * console.log(JSON.stringify(dataset, null, 2));
+ * ```
+ */
+declare function importToCube(source: string | Uint8Array | Blob, options?: ImportOptions): Promise<Observations>;
+/**
+ * Like [`importToCube`](#importtocube) but also builds the final
+ * [`JsonStatDataset`](./model/jsonstat.ts) via [`buildDataset`](./core/cubeBuilder.ts).
+ *
+ * @returns The serialized-ready [`JsonStatDataset`](./model/jsonstat.ts).
+ */
+declare function importToDataset(source: string | Uint8Array | Blob, options?: ImportOptions): Promise<JsonStatDataset>;
+/** Supported export sinks for [`exportDataset`](#exportdataset). */
+type ExportTarget = "arrow" | "parquet" | "csv" | "csvw";
+/**
+ * Options for [`exportDataset`](#exportdataset).
+ *
+ * Format-specific knobs are passed through to the underlying writer. For
+ * Parquet, `init` is forwarded to `parquet-wasm`; for CSVW, `url` sets the
+ * metadata `url`. DuckDB and Polars are intentionally not exposed here because
+ * they require a live connection / native module — use the dedicated subpath
+ * exports (`jsonstat-io/duckdb`, `jsonstat-io/polars`) for those.
+ */
+interface ExportOptions {
+    to: ExportTarget;
+    /** Parquet compression codec (forwarded to `parquet-wasm` `writeParquet`). */
+    compression?: string;
+    /** CSV / CSVW delimiter (default ","). */
+    delimiter?: string;
+    /** CSVW metadata `url` field. */
+    url?: string;
+    /** Line terminator for CSV / CSVW output (default "\r\n"). */
+    lineTerminator?: string;
+    /** Async initializer for parquet-wasm (browser base URL, etc.). */
+    init?: () => Promise<void>;
+}
+/**
+ * The polymorphic result of [`exportDataset`](#exportdataset). The concrete
+ * shape depends on [`ExportOptions.to`](#ExportOptions.to):
+ *
+ * - `"arrow"` → an Apache Arrow [`Table`](https://arrow.apache.org/docs/js/).
+ * - `"parquet"` → a `Uint8Array` of the Parquet file bytes.
+ * - `"csv"` → the CSV text (`string`).
+ * - `"csvw"` → `{ csv: string; metadata: CsvwMetadata }`.
+ */
+type ExportResult = Table | Uint8Array | string | CsvwExportShape;
+/** Shape of the `"csvw"` export result (mirrors `csvw.CsvwExport`). */
+interface CsvwExportShape {
+    csv: string;
+    metadata: unknown;
+}
+/**
+ * Export a JSON-stat [`Dataset`](./model/jsonstat.ts) to a columnar format.
+ *
+ * This is the high-level export dispatcher, the mirror of
+ * [`importToDataset`](#importtodataset). It reads the dataset into the
+ * [`Observations`](./model/ir.ts) IR via [`readDataset`](./core/cubeReader.ts),
+ * then routes to the appropriate writer:
+ *
+ * - `"arrow"`  → [`cubeToArrow`](./arrow/arrowFromCube.ts) → Arrow `Table`.
+ * - `"parquet"` → Arrow → `parquet-wasm` `writeParquet` → `Uint8Array`.
+ * - `"csv"`    → [`cubeToCsv`](./sources/csv.ts) → CSV text.
+ * - `"csvw"`   → [`cubeToCsvw`](./sources/csvw.ts) → `{ csv, metadata }`.
+ *
+ * DuckDB and Polars require a live connection / native module and are not
+ * reachable from this dispatcher — use the dedicated subpath writers
+ * (`cubeToDuckdb`, `cubeToPolars`) directly.
+ *
+ * @example
+ * ```ts
+ * import { exportDataset } from "jsonstat-io";
+ * const bytes = await exportDataset(dataset, { to: "parquet" });
+ * ```
+ */
+declare function exportDataset(dataset: JsonStatDataset, options: ExportOptions): Promise<ExportResult>;
+
+export { ArrowToCubeOptions, type BuildOptions, type BuildResult, type CsvwExportShape, CubeBuilderError, CubeReaderError, DatasetMeta, type DensityOptions, type DensityResult, type ExportOptions, type ExportResult, type ExportTarget, type ImportOptions, ImporterError, JsonStatDataset, JsonStatResponse, type LoadedInput, Observations, type ReadOptions, type SerializeOptions, type SourceFormat, buildDataset, decideDensity, detectFormat, detectFromBytes, detectFromExtension, enumerateCells, exportDataset, extensionOf, flatPosition, importToCube, importToDataset, loadInput, multiIndex, readDataset, readResponse, serialize, serializeToBytes, strides, toDataset, totalCells };