@procwire/codec-arrow 0.1.3 → 0.2.0

package/dist/index.d.ts

@@ -1,57 +1,135 @@
 /**
- * Apache Arrow codec for @procwire/transport.
- * Provides columnar data serialization using apache-arrow.
+ * Apache Arrow IPC serialization codec for @procwire/transport.
  *
- * @module Codec Arrow
- */
-import type { Table } from "apache-arrow";
-import type { SerializationCodec } from "@procwire/transport/serialization";
-/**
- * Apache Arrow serialization codec.
- * Implements efficient columnar data serialization ideal for analytical workloads.
+ * Provides high-performance columnar data serialization using Apache Arrow,
+ * optimized for analytics workloads and large datasets. This codec implements
+ * the {@link SerializationCodec} interface for seamless integration with
+ * @procwire/transport channels.
+ *
+ * ## Features
+ *
+ * - **Zero-copy serialization** - Minimizes memory allocations and copies
+ * - **Columnar format** - Optimized for analytics and batch processing
+ * - **Large dataset support** - Efficiently handles millions of rows
+ * - **Cross-language compatibility** - Works with Python (PyArrow), R, Java, etc.
+ * - **Built-in metrics** - Optional monitoring of throughput and errors
+ * - **Configurable formats** - Stream (default) or file format
+ *
+ * ## When to Use Arrow
+ *
+ * Apache Arrow is ideal for:
+ * - Data analytics and processing pipelines
+ * - Transferring tabular data between processes
+ * - Interoperability with data science tools (pandas, R, Spark)
+ * - High-throughput, low-latency data transfer
+ * - Large datasets where columnar access patterns dominate
+ *
+ * For small messages or non-tabular data, consider {@link @procwire/codec-msgpack}
+ * or {@link @procwire/codec-protobuf} instead.
+ *
+ * ## Quick Start
  *
- * @example
  * ```ts
  * import { tableFromArrays } from 'apache-arrow';
  * import { ArrowCodec } from '@procwire/codec-arrow';
- * import { ChannelBuilder } from '@procwire/transport';
  *
  * const codec = new ArrowCodec();
  *
- * // Create a table
+ * // Create an Arrow table
  * const table = tableFromArrays({
+ *   id: [1, 2, 3, 4, 5],
+ *   name: ['Alice', 'Bob', 'Charlie', 'Diana', 'Eve'],
+ *   score: [95.5, 87.3, 92.1, 88.7, 91.2]
+ * });
+ *
+ * // Serialize to IPC format
+ * const buffer = codec.serialize(table);
+ *
+ * // Deserialize back to Table
+ * const decoded = codec.deserialize(buffer);
+ * console.log(decoded.numRows); // 5
+ * ```
+ *
+ * ## IPC Formats
+ *
+ * Arrow supports two IPC formats:
+ *
+ * - **Stream format** (default): Smaller size, no footer, ideal for streaming/IPC
+ * - **File format**: Includes footer for random access, suitable for file storage
+ *
+ * ```ts
+ * // Stream format (default) - for IPC
+ * const streamCodec = new ArrowCodec({ format: 'stream' });
+ *
+ * // File format - for random access
+ * const fileCodec = new ArrowCodec({ format: 'file' });
+ * ```
+ *
+ * ## Integration with @procwire/transport
+ *
+ * ```ts
+ * import { ArrowCodec } from '@procwire/codec-arrow';
+ * import { RequestChannel } from '@procwire/transport/channel';
+ *
+ * const channel = new RequestChannel({
+ *   transport,
+ *   framing,
+ *   serialization: new ArrowCodec(),
+ *   protocol
+ * });
+ * ```
+ *
+ * @packageDocumentation
+ * @module codec-arrow
+ */
+export { ArrowCodec } from "./codec.js";
+export type { ArrowCodecOptions, ArrowCodecMetrics, ArrowIPCFormat } from "./codec.js";
+export { createFastArrowCodec, createMonitoredArrowCodec, createFileArrowCodec } from "./codec.js";
+/**
+ * Re-export of Table from apache-arrow.
+ *
+ * The Table class is the primary data structure for Apache Arrow.
+ * It represents a two-dimensional dataset with named columns,
+ * similar to a DataFrame in pandas or R.
+ *
+ * @example Creating a Table
+ * ```ts
+ * import { tableFromArrays, Table } from 'apache-arrow';
+ *
+ * const table: Table = tableFromArrays({
  *   id: [1, 2, 3],
  *   name: ['Alice', 'Bob', 'Charlie']
  * });
+ * ```
  *
- * // Use with channel
- * const channel = new ChannelBuilder()
- *   .withSerialization(codec)
- *   // ... other configuration
- *   .build();
+ * @see {@link https://arrow.apache.org/docs/js/classes/Arrow_dom.Table.html | Apache Arrow Table documentation}
+ */
+export type { Table } from "apache-arrow";
+/**
+ * Re-export of Schema from apache-arrow.
  *
- * // Send table over channel
- * await channel.request('process', table);
- * ```
+ * The Schema class describes the structure of an Arrow Table,
+ * including column names, types, and metadata.
+ *
+ * @see {@link https://arrow.apache.org/docs/js/classes/Arrow_dom.Schema.html | Apache Arrow Schema documentation}
+ */
+export type { Schema } from "apache-arrow";
+/**
+ * Re-export of Field from apache-arrow.
+ *
+ * The Field class represents a single column definition in a Schema,
+ * including the column name, data type, and nullability.
+ *
+ * @see {@link https://arrow.apache.org/docs/js/classes/Arrow_dom.Field.html | Apache Arrow Field documentation}
+ */
+export type { Field } from "apache-arrow";
+/**
+ * Re-export of RecordBatch from apache-arrow.
+ *
+ * A RecordBatch is a chunk of a Table, containing a fixed number of rows
+ * with the same schema. Tables are composed of one or more RecordBatches.
+ *
+ * @see {@link https://arrow.apache.org/docs/js/classes/Arrow_dom.RecordBatch.html | Apache Arrow RecordBatch documentation}
  */
-export declare class ArrowCodec implements SerializationCodec<Table> {
-    readonly name = "arrow";
-    readonly contentType = "application/vnd.apache.arrow.stream";
-    /**
-     * Serializes an Apache Arrow Table to IPC stream format.
-     *
-     * @param value - Arrow Table to serialize
-     * @returns Buffer containing Arrow IPC stream data
-     * @throws {SerializationError} if encoding fails
-     */
-    serialize(value: Table): Buffer;
-    /**
-     * Deserializes Arrow IPC stream data to an Apache Arrow Table.
-     *
-     * @param buffer - Buffer containing Arrow IPC stream data
-     * @returns Deserialized Arrow Table
-     * @throws {SerializationError} if decoding fails
-     */
-    deserialize(buffer: Buffer): Table;
-}
+export type { RecordBatch } from "apache-arrow";
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,cAAc,CAAC;AAE1C,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,mCAAmC,CAAC;AAG5E;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,qBAAa,UAAW,YAAW,kBAAkB,CAAC,KAAK,CAAC;IAC1D,QAAQ,CAAC,IAAI,WAAW;IACxB,QAAQ,CAAC,WAAW,yCAAyC;IAE7D;;;;;;OAMG;IACH,SAAS,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM;IAY/B;;;;;;OAMG;IACH,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,KAAK;CAUnC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmFG;AAGH,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,YAAY,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAGvF,OAAO,EAAE,oBAAoB,EAAE,yBAAyB,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAEnG;;;;;;;;;;;;;;;;;;GAkBG;AACH,YAAY,EAAE,KAAK,EAAE,MAAM,cAAc,CAAC;AAE1C;;;;;;;GAOG;AACH,YAAY,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAE3C;;;;;;;GAOG;AACH,YAAY,EAAE,KAAK,EAAE,MAAM,cAAc,CAAC;AAE1C;;;;;;;GAOG;AACH,YAAY,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC"}

package/dist/index.js

@@ -1,72 +1,89 @@
 /**
- * Apache Arrow codec for @procwire/transport.
- * Provides columnar data serialization using apache-arrow.
+ * Apache Arrow IPC serialization codec for @procwire/transport.
  *
- * @module Codec Arrow
- */
-import { tableFromIPC, tableToIPC } from "apache-arrow";
-import { SerializationError } from "@procwire/transport";
-/**
- * Apache Arrow serialization codec.
- * Implements efficient columnar data serialization ideal for analytical workloads.
+ * Provides high-performance columnar data serialization using Apache Arrow,
+ * optimized for analytics workloads and large datasets. This codec implements
+ * the {@link SerializationCodec} interface for seamless integration with
+ * @procwire/transport channels.
+ *
+ * ## Features
+ *
+ * - **Zero-copy serialization** - Minimizes memory allocations and copies
+ * - **Columnar format** - Optimized for analytics and batch processing
+ * - **Large dataset support** - Efficiently handles millions of rows
+ * - **Cross-language compatibility** - Works with Python (PyArrow), R, Java, etc.
+ * - **Built-in metrics** - Optional monitoring of throughput and errors
+ * - **Configurable formats** - Stream (default) or file format
+ *
+ * ## When to Use Arrow
+ *
+ * Apache Arrow is ideal for:
+ * - Data analytics and processing pipelines
+ * - Transferring tabular data between processes
+ * - Interoperability with data science tools (pandas, R, Spark)
+ * - High-throughput, low-latency data transfer
+ * - Large datasets where columnar access patterns dominate
+ *
+ * For small messages or non-tabular data, consider {@link @procwire/codec-msgpack}
+ * or {@link @procwire/codec-protobuf} instead.
+ *
+ * ## Quick Start
  *
- * @example
  * ```ts
  * import { tableFromArrays } from 'apache-arrow';
  * import { ArrowCodec } from '@procwire/codec-arrow';
- * import { ChannelBuilder } from '@procwire/transport';
  *
  * const codec = new ArrowCodec();
  *
- * // Create a table
+ * // Create an Arrow table
  * const table = tableFromArrays({
- *   id: [1, 2, 3],
- *   name: ['Alice', 'Bob', 'Charlie']
+ *   id: [1, 2, 3, 4, 5],
+ *   name: ['Alice', 'Bob', 'Charlie', 'Diana', 'Eve'],
+ *   score: [95.5, 87.3, 92.1, 88.7, 91.2]
  * });
  *
- * // Use with channel
- * const channel = new ChannelBuilder()
- *   .withSerialization(codec)
- *   // ... other configuration
- *   .build();
+ * // Serialize to IPC format
+ * const buffer = codec.serialize(table);
  *
- * // Send table over channel
- * await channel.request('process', table);
+ * // Deserialize back to Table
+ * const decoded = codec.deserialize(buffer);
+ * console.log(decoded.numRows); // 5
  * ```
+ *
+ * ## IPC Formats
+ *
+ * Arrow supports two IPC formats:
+ *
+ * - **Stream format** (default): Smaller size, no footer, ideal for streaming/IPC
+ * - **File format**: Includes footer for random access, suitable for file storage
+ *
+ * ```ts
+ * // Stream format (default) - for IPC
+ * const streamCodec = new ArrowCodec({ format: 'stream' });
+ *
+ * // File format - for random access
+ * const fileCodec = new ArrowCodec({ format: 'file' });
+ * ```
+ *
+ * ## Integration with @procwire/transport
+ *
+ * ```ts
+ * import { ArrowCodec } from '@procwire/codec-arrow';
+ * import { RequestChannel } from '@procwire/transport/channel';
+ *
+ * const channel = new RequestChannel({
+ *   transport,
+ *   framing,
+ *   serialization: new ArrowCodec(),
+ *   protocol
+ * });
+ * ```
+ *
+ * @packageDocumentation
+ * @module codec-arrow
  */
-export class ArrowCodec {
-    name = "arrow";
-    contentType = "application/vnd.apache.arrow.stream";
-    /**
-     * Serializes an Apache Arrow Table to IPC stream format.
-     *
-     * @param value - Arrow Table to serialize
-     * @returns Buffer containing Arrow IPC stream data
-     * @throws {SerializationError} if encoding fails
-     */
-    serialize(value) {
-        try {
-            const uint8array = tableToIPC(value);
-            return Buffer.from(uint8array);
-        }
-        catch (error) {
-            throw new SerializationError(`Failed to encode Arrow table: ${error instanceof Error ? error.message : String(error)}`, error);
-        }
-    }
-    /**
-     * Deserializes Arrow IPC stream data to an Apache Arrow Table.
-     *
-     * @param buffer - Buffer containing Arrow IPC stream data
-     * @returns Deserialized Arrow Table
-     * @throws {SerializationError} if decoding fails
-     */
-    deserialize(buffer) {
-        try {
-            return tableFromIPC(buffer);
-        }
-        catch (error) {
-            throw new SerializationError(`Failed to decode Arrow table: ${error instanceof Error ? error.message : String(error)}`, error);
-        }
-    }
-}
+// Main codec class and options
+export { ArrowCodec } from "./codec.js";
+// Helper functions
+export { createFastArrowCodec, createMonitoredArrowCodec, createFileArrowCodec } from "./codec.js";
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAExD,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AAEzD;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,MAAM,OAAO,UAAU;IACZ,IAAI,GAAG,OAAO,CAAC;IACf,WAAW,GAAG,qCAAqC,CAAC;IAE7D;;;;;;OAMG;IACH,SAAS,CAAC,KAAY;QACpB,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;YACrC,OAAO,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACjC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,kBAAkB,CAC1B,iCAAiC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,EACzF,KAAK,CACN,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACH,WAAW,CAAC,MAAc;QACxB,IAAI,CAAC;YACH,OAAO,YAAY,CAAC,MAAM,CAAC,CAAC;QAC9B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,kBAAkB,CAC1B,iCAAiC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,EACzF,KAAK,CACN,CAAC;QACJ,CAAC;IACH,CAAC;CACF"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmFG;AAEH,+BAA+B;AAC/B,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAGxC,mBAAmB;AACnB,OAAO,EAAE,oBAAoB,EAAE,yBAAyB,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@procwire/codec-arrow",
-  "version": "0.1.3",
+  "version": "0.2.0",
   "description": "Apache Arrow IPC codec for @procwire/transport.",
   "keywords": [
     "ipc",