@procwire/codec-arrow 0.1.1

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Sebastian Webdev
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,265 @@
+# @procwire/codec-arrow
+
+Apache Arrow serialization codec for `@procwire/transport`.
+
+Provides efficient columnar data serialization using [apache-arrow](https://github.com/apache/arrow/tree/main/js), ideal for analytical workloads and large datasets.
+
+## Installation
+
+```bash
+npm install @procwire/codec-arrow apache-arrow
+```
+
+Note: `apache-arrow` is a peer dependency and must be installed separately.
+
+## Usage
+
+### Basic Usage
+
+```ts
+import { tableFromArrays } from 'apache-arrow';
+import { ArrowCodec } from '@procwire/codec-arrow';
+import { ChannelBuilder } from '@procwire/transport';
+
+const codec = new ArrowCodec();
+
+// Create a table
+const table = tableFromArrays({
+  id: [1, 2, 3, 4, 5],
+  name: ['Alice', 'Bob', 'Charlie', 'David', 'Eve'],
+  score: [95.5, 87.3, 92.1, 88.7, 94.2]
+});
+
+// Use with ChannelBuilder
+const channel = new ChannelBuilder()
+  .withTransport(transport)
+  .withFraming(framing)
+  .withSerialization(codec)
+  .withProtocol(protocol)
+  .build();
+
+// Send table over channel
+await channel.request('processData', table);
+```
+
+### Standalone Usage
+
+```ts
+import { tableFromArrays } from 'apache-arrow';
+import { ArrowCodec } from '@procwire/codec-arrow';
+
+const codec = new ArrowCodec();
+
+// Serialize
+const table = tableFromArrays({
+  id: [1, 2, 3],
+  value: [10.5, 20.3, 30.1]
+});
+
+const buffer = codec.serialize(table);
+
+// Deserialize
+const decoded = codec.deserialize(buffer);
+console.log(decoded.numRows); // 3
+console.log(decoded.getChild('id')?.toArray()); // [1, 2, 3]
+```
+
+### Working with Large Datasets
+
+```ts
+import { tableFromArrays } from 'apache-arrow';
+import { ArrowCodec } from '@procwire/codec-arrow';
+
+const codec = new ArrowCodec();
+
+// Create large dataset (100K rows)
+const size = 100000;
+const table = tableFromArrays({
+  timestamp: Array.from({ length: size }, (_, i) => Date.now() + i * 1000),
+  sensor_id: Array.from({ length: size }, (_, i) => i % 100),
+  temperature: Array.from({ length: size }, () => 20 + Math.random() * 10),
+  humidity: Array.from({ length: size }, () => 40 + Math.random() * 20)
+});
+
+// Efficient serialization of columnar data
+const buffer = codec.serialize(table);
+console.log(`Serialized ${size} rows in ${buffer.length} bytes`);
+
+// Fast deserialization
+const decoded = codec.deserialize(buffer);
+console.log(`Deserialized table with ${decoded.numRows} rows`);
+```
+
+## Features
+
+- **Columnar Format**: Optimized for analytical queries and large datasets
+- **Type Preservation**: Full type system support (integers, floats, strings, booleans, etc.)
+- **Null Handling**: Native support for null values
+- **Zero-Copy**: Efficient memory usage with zero-copy reads where possible
+- **Error Handling**: Wraps encoding/decoding errors in `SerializationError` from `@procwire/transport`
+- **IPC Stream Format**: Uses Arrow IPC streaming format for efficient transmission
+
+## API
+
+### `ArrowCodec`
+
+Implements `SerializationCodec<Table>` interface.
+
+#### Properties
+
+- `name: "arrow"` - Codec identifier
+- `contentType: "application/vnd.apache.arrow.stream"` - MIME type
+
+#### Methods
+
+##### `serialize(value: Table): Buffer`
+
+Serializes an Apache Arrow Table to IPC stream format.
+
+**Parameters:**
+- `value` - Arrow Table to serialize
+
+**Returns:** `Buffer` containing Arrow IPC stream data
+
+**Throws:** `SerializationError` if encoding fails
+
+##### `deserialize(buffer: Buffer): Table`
+
+Deserializes Arrow IPC stream data to an Apache Arrow Table.
+
+**Parameters:**
+- `buffer` - Buffer containing Arrow IPC stream data
+
+**Returns:** Deserialized Arrow Table
+
+**Throws:** `SerializationError` if decoding fails
+
+## Advanced Usage
+
+### Creating Tables from Arrays
+
+```ts
+import { tableFromArrays } from 'apache-arrow';
+
+const table = tableFromArrays({
+  // Integer column
+  id: [1, 2, 3],
+
+  // String column
+  name: ['Alice', 'Bob', 'Charlie'],
+
+  // Float column
+  score: [95.5, 87.3, 92.1],
+
+  // Boolean column
+  active: [true, false, true],
+
+  // Column with nulls
+  email: ['alice@example.com', null, 'charlie@example.com']
+});
+```
+
+### Typed Arrays for Performance
+
+```ts
+import { tableFromArrays } from 'apache-arrow';
+
+const table = tableFromArrays({
+  int32_col: new Int32Array([1, 2, 3, 4, 5]),
+  float64_col: new Float64Array([1.1, 2.2, 3.3, 4.4, 5.5]),
+  uint8_col: new Uint8Array([255, 128, 64, 32, 0])
+});
+```
+
+### Accessing Column Data
+
+```ts
+const table = tableFromArrays({
+  id: [1, 2, 3],
+  name: ['Alice', 'Bob', 'Charlie']
+});
+
+// Get column
+const idColumn = table.getChild('id');
+const ids = idColumn?.toArray(); // [1, 2, 3]
+
+// Iterate rows
+for (let i = 0; i < table.numRows; i++) {
+  const row = table.get(i);
+  console.log(row);
+}
+```
+
+## Performance
+
+Apache Arrow provides exceptional performance for columnar data:
+
+- **Columnar Storage**: Data stored in columns, not rows - ideal for analytical queries
+- **Zero-Copy Reads**: Direct memory access without deserialization overhead
+- **Compression**: Built-in dictionary encoding for repeated values
+- **Vectorized Operations**: SIMD-friendly data layout for fast processing
+- **Cross-Language**: Same binary format used in Python, R, Java, C++, etc.
+
+### Performance Characteristics
+
+Compared to JSON:
+- **5-50x faster** serialization/deserialization for large datasets
+- **2-10x smaller** binary size for numeric-heavy data
+- **Zero-copy** operations for in-memory analytics
+
+Ideal for:
+- Time-series data
+- Analytics and data science workloads
+- Large datasets (millions of rows)
+- High-throughput data streaming
+- Cross-language data exchange
+- Machine learning pipelines
+
+## Use Cases
+
+### Time-Series Data
+
+```ts
+const timeSeries = tableFromArrays({
+  timestamp: timestamps, // millions of timestamps
+  value: values,        // sensor readings
+  quality: qualities    // quality flags
+});
+```
+
+### Data Analytics
+
+```ts
+const analyticsData = tableFromArrays({
+  user_id: userIds,
+  event_type: eventTypes,
+  timestamp: timestamps,
+  properties: jsonProperties
+});
+```
+
+### Machine Learning
+
+```ts
+const features = tableFromArrays({
+  feature1: feature1Data,
+  feature2: feature2Data,
+  // ... many features
+  label: labels
+});
+```
+
+## Compatibility
+
+Arrow IPC format is cross-platform and cross-language:
+- **Python**: PyArrow
+- **R**: arrow R package
+- **Java**: Arrow Java
+- **C++**: Arrow C++
+- **Rust**: arrow-rs
+
+Tables can be serialized in one language and deserialized in another seamlessly.
+
+## License
+
+MIT

package/dist/index.d.ts

@@ -0,0 +1,57 @@
+/**
+ * Apache Arrow codec for @procwire/transport.
+ * Provides columnar data serialization using apache-arrow.
+ *
+ * @module
+ */
+import type { Table } from "apache-arrow";
+import type { SerializationCodec } from "@procwire/transport/serialization";
+/**
+ * Apache Arrow serialization codec.
+ * Implements efficient columnar data serialization ideal for analytical workloads.
+ *
+ * @example
+ * ```ts
+ * import { tableFromArrays } from 'apache-arrow';
+ * import { ArrowCodec } from '@procwire/codec-arrow';
+ * import { ChannelBuilder } from '@procwire/transport';
+ *
+ * const codec = new ArrowCodec();
+ *
+ * // Create a table
+ * const table = tableFromArrays({
+ *   id: [1, 2, 3],
+ *   name: ['Alice', 'Bob', 'Charlie']
+ * });
+ *
+ * // Use with channel
+ * const channel = new ChannelBuilder()
+ *   .withSerialization(codec)
+ *   // ... other configuration
+ *   .build();
+ *
+ * // Send table over channel
+ * await channel.request('process', table);
+ * ```
+ */
+export declare class ArrowCodec implements SerializationCodec<Table> {
+    readonly name = "arrow";
+    readonly contentType = "application/vnd.apache.arrow.stream";
+    /**
+     * Serializes an Apache Arrow Table to IPC stream format.
+     *
+     * @param value - Arrow Table to serialize
+     * @returns Buffer containing Arrow IPC stream data
+     * @throws {SerializationError} if encoding fails
+     */
+    serialize(value: Table): Buffer;
+    /**
+     * Deserializes Arrow IPC stream data to an Apache Arrow Table.
+     *
+     * @param buffer - Buffer containing Arrow IPC stream data
+     * @returns Deserialized Arrow Table
+     * @throws {SerializationError} if decoding fails
+     */
+    deserialize(buffer: Buffer): Table;
+}
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,cAAc,CAAC;AAE1C,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,mCAAmC,CAAC;AAG5E;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,qBAAa,UAAW,YAAW,kBAAkB,CAAC,KAAK,CAAC;IAC1D,QAAQ,CAAC,IAAI,WAAW;IACxB,QAAQ,CAAC,WAAW,yCAAyC;IAE7D;;;;;;OAMG;IACH,SAAS,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM;IAY/B;;;;;;OAMG;IACH,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,KAAK;CAUnC"}

package/dist/index.js

@@ -0,0 +1,72 @@
+/**
+ * Apache Arrow codec for @procwire/transport.
+ * Provides columnar data serialization using apache-arrow.
+ *
+ * @module
+ */
+import { tableFromIPC, tableToIPC } from "apache-arrow";
+import { SerializationError } from "@procwire/transport";
+/**
+ * Apache Arrow serialization codec.
+ * Implements efficient columnar data serialization ideal for analytical workloads.
+ *
+ * @example
+ * ```ts
+ * import { tableFromArrays } from 'apache-arrow';
+ * import { ArrowCodec } from '@procwire/codec-arrow';
+ * import { ChannelBuilder } from '@procwire/transport';
+ *
+ * const codec = new ArrowCodec();
+ *
+ * // Create a table
+ * const table = tableFromArrays({
+ *   id: [1, 2, 3],
+ *   name: ['Alice', 'Bob', 'Charlie']
+ * });
+ *
+ * // Use with channel
+ * const channel = new ChannelBuilder()
+ *   .withSerialization(codec)
+ *   // ... other configuration
+ *   .build();
+ *
+ * // Send table over channel
+ * await channel.request('process', table);
+ * ```
+ */
+export class ArrowCodec {
+    name = "arrow";
+    contentType = "application/vnd.apache.arrow.stream";
+    /**
+     * Serializes an Apache Arrow Table to IPC stream format.
+     *
+     * @param value - Arrow Table to serialize
+     * @returns Buffer containing Arrow IPC stream data
+     * @throws {SerializationError} if encoding fails
+     */
+    serialize(value) {
+        try {
+            const uint8array = tableToIPC(value);
+            return Buffer.from(uint8array);
+        }
+        catch (error) {
+            throw new SerializationError(`Failed to encode Arrow table: ${error instanceof Error ? error.message : String(error)}`, error);
+        }
+    }
+    /**
+     * Deserializes Arrow IPC stream data to an Apache Arrow Table.
+     *
+     * @param buffer - Buffer containing Arrow IPC stream data
+     * @returns Deserialized Arrow Table
+     * @throws {SerializationError} if decoding fails
+     */
+    deserialize(buffer) {
+        try {
+            return tableFromIPC(buffer);
+        }
+        catch (error) {
+            throw new SerializationError(`Failed to decode Arrow table: ${error instanceof Error ? error.message : String(error)}`, error);
+        }
+    }
+}
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAExD,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AAEzD;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,MAAM,OAAO,UAAU;IACZ,IAAI,GAAG,OAAO,CAAC;IACf,WAAW,GAAG,qCAAqC,CAAC;IAE7D;;;;;;OAMG;IACH,SAAS,CAAC,KAAY;QACpB,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;YACrC,OAAO,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACjC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,kBAAkB,CAC1B,iCAAiC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,EACzF,KAAK,CACN,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACH,WAAW,CAAC,MAAc;QACxB,IAAI,CAAC;YACH,OAAO,YAAY,CAAC,MAAM,CAAC,CAAC;QAC9B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,kBAAkB,CAC1B,iCAAiC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,EACzF,KAAK,CACN,CAAC;QACJ,CAAC;IACH,CAAC;CACF"}

package/package.json

@@ -0,0 +1,67 @@
+{
+  "name": "@procwire/codec-arrow",
+  "version": "0.1.1",
+  "description": "Apache Arrow IPC codec for @procwire/transport.",
+  "keywords": [
+    "ipc",
+    "arrow",
+    "apache-arrow",
+    "serialization",
+    "codec",
+    "columnar",
+    "binary",
+    "node",
+    "typescript"
+  ],
+  "author": "Sebastian Webdev",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/SebastianWebdev/procwire.git",
+    "directory": "codec-arrow"
+  },
+  "bugs": {
+    "url": "https://github.com/SebastianWebdev/procwire/issues"
+  },
+  "homepage": "https://github.com/SebastianWebdev/procwire/tree/main/codec-arrow#readme",
+  "type": "module",
+  "sideEffects": false,
+  "license": "MIT",
+  "engines": {
+    "node": ">=18"
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "files": [
+    "dist",
+    "README.md",
+    "LICENSE"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "dependencies": {
+    "@procwire/transport": "0.1.1"
+  },
+  "peerDependencies": {
+    "apache-arrow": "^21.0.0"
+  },
+  "devDependencies": {
+    "@types/node": "^22.0.0",
+    "apache-arrow": "^21.0.0",
+    "rimraf": "^6.0.1",
+    "typescript": "^5.9.3",
+    "vitest": "^2.1.8"
+  },
+  "scripts": {
+    "clean": "rimraf dist \"*.tsbuildinfo\"",
+    "typecheck": "tsc -p tsconfig.json --noEmit",
+    "build": "tsc -p tsconfig.build.json",
+    "test": "vitest run"
+  }
+}