@databricks/zerobus-ingest-sdk 1.2.0 → 1.3.0

package/Cargo.lock

@@ -409,7 +409,7 @@ dependencies = [
 
 [[package]]
 name = "databricks-zerobus-ingest-sdk"
-version = "2.7.1"
+version = "2.8.0"
 dependencies = [
  "arrow-array",
  "arrow-flight",
@@ -2962,7 +2962,7 @@ dependencies = [
 
 [[package]]
 name = "zerobus-sdk-ts"
-version = "1.2.0"
+version = "1.3.0"
 dependencies = [
  "arrow-array",
  "arrow-ipc",

package/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "zerobus-sdk-ts"
-version = "1.2.0"
+version = "1.3.0"
 authors = ["Databricks"]
 edition = "2021"
 license = "Apache-2.0"
@@ -16,7 +16,7 @@ crate-type = ["cdylib"]
 napi = { version = "2", features = ["async", "tokio_rt", "serde-json", "napi6"] }
 napi-derive = "2"
 
-databricks-zerobus-ingest-sdk = { path = "../rust/sdk", version = "2.7.1" }
+databricks-zerobus-ingest-sdk = { path = "../rust/sdk", version = "2.8.0" }
 
 tokio = { version = "1.52", features = ["macros", "rt-multi-thread"] }
 
@@ -41,7 +41,7 @@ napi-build = "2"
 
 [features]
 default = []
-# Arrow Flight is in Beta - enable with: npm run build:arrow
+# Enable with: npm run build:arrow
 arrow-flight = [
     "databricks-zerobus-ingest-sdk/arrow-flight",
     "dep:arrow-array",

package/README.md

@@ -28,7 +28,7 @@ The Databricks Zerobus Ingest SDK for TypeScript provides a high-performance cli
 - **High-throughput ingestion**: Optimized for high-volume data ingestion with native Rust implementation
 - **Automatic recovery**: Built-in retry and recovery mechanisms for transient failures
 - **Flexible configuration**: Customizable stream behavior and timeouts
-- **Multiple serialization formats**: Support for JSON, Protocol Buffers, and Arrow Flight (Beta) with optional LZ4 / ZSTD compression
+- **Multiple serialization formats**: Support for JSON, Protocol Buffers, and Arrow Flight ingestion with optional LZ4 / ZSTD compression
 - **Type widening**: Accept high-level types (plain objects, protobuf messages) or low-level types (strings, buffers) - automatically handles serialization
 - **Batch ingestion**: Ingest multiple records with a single acknowledgment for higher throughput
 - **OAuth 2.0 authentication**: Secure authentication with client credentials
@@ -90,10 +90,11 @@ npm run build
 
 ### Choose Your Serialization Format
 
-The SDK supports two serialization formats. **Protocol Buffers is the default** and recommended for production use:
+The SDK supports three ingestion formats. Protocol Buffers is the default record format and recommended for production row-oriented workloads:
 
 - **Protocol Buffers (Default)** - Strongly-typed schemas, efficient binary encoding, better performance. This is the default format.
 - **JSON** - Simple, no schema compilation needed. Good for getting started quickly or when schema flexibility is needed.
+- Arrow Flight - High-performance columnar ingestion for applications that already produce Arrow data. See the [Arrow example](examples/arrow/README.md).
 
 > **Note:** If you don't specify `recordType`, the SDK will use Protocol Buffers by default. To use JSON, explicitly set `recordType: RecordType.Json`.
 

package/index.d.ts

@@ -102,6 +102,133 @@ export interface ZerobusSdkOptions {
   /** Identifier appended to the `user-agent` header */
   applicationName?: string
 }
+/**
+ * IPC compression type for Arrow Flight streams.
+ *
+ */
+export const enum IpcCompressionType {
+  /** LZ4 frame compression - fast compression with moderate ratio */
+  Lz4Frame = 0,
+  /** Zstandard compression - better compression ratio, slightly slower */
+  Zstd = 1
+}
+/**
+ * Configuration options for Arrow Flight streams.
+ *
+ */
+export interface ArrowStreamConfigurationOptions {
+  /**
+   * Maximum number of batches that can be in-flight (sent but not acknowledged).
+   * Default: 1,000
+   */
+  maxInflightBatches?: number
+  /**
+   * Whether to enable automatic stream recovery on failure.
+   * Default: true
+   */
+  recovery?: boolean
+  /**
+   * Timeout for recovery operations in milliseconds.
+   * Default: 15,000 (15 seconds)
+   */
+  recoveryTimeoutMs?: number
+  /**
+   * Delay between recovery retry attempts in milliseconds.
+   * Default: 2,000 (2 seconds)
+   */
+  recoveryBackoffMs?: number
+  /**
+   * Maximum number of recovery attempts before giving up.
+   * Default: 4
+   */
+  recoveryRetries?: number
+  /**
+   * Timeout waiting for server acknowledgments in milliseconds.
+   * Default: 60,000 (1 minute)
+   */
+  serverLackOfAckTimeoutMs?: number
+  /**
+   * Timeout for flush operations in milliseconds.
+   * Default: 300,000 (5 minutes)
+   */
+  flushTimeoutMs?: number
+  /**
+   * Timeout for connection establishment in milliseconds.
+   * Default: 30,000 (30 seconds)
+   */
+  connectionTimeoutMs?: number
+  /** Optional IPC compression type (0 = LZ4Frame, 1 = Zstd, undefined = no compression) */
+  ipcCompression?: number
+}
+/**
+ * Arrow data type enum for schema definition.
+ *
+ */
+export const enum ArrowDataType {
+  /** Boolean type */
+  Boolean = 0,
+  /** Signed 8-bit integer */
+  Int8 = 1,
+  /** Signed 16-bit integer */
+  Int16 = 2,
+  /** Signed 32-bit integer */
+  Int32 = 3,
+  /** Signed 64-bit integer */
+  Int64 = 4,
+  /** Unsigned 8-bit integer */
+  UInt8 = 5,
+  /** Unsigned 16-bit integer */
+  UInt16 = 6,
+  /** Unsigned 32-bit integer */
+  UInt32 = 7,
+  /** Unsigned 64-bit integer */
+  UInt64 = 8,
+  /** 32-bit floating point */
+  Float32 = 9,
+  /** 64-bit floating point */
+  Float64 = 10,
+  /** UTF-8 encoded string */
+  Utf8 = 11,
+  /** Large UTF-8 encoded string (64-bit offsets) */
+  LargeUtf8 = 12,
+  /** Binary data */
+  Binary = 13,
+  /** Large binary data (64-bit offsets) */
+  LargeBinary = 14,
+  /** Date (32-bit days since epoch) */
+  Date32 = 15,
+  /** Date (64-bit milliseconds since epoch) */
+  Date64 = 16,
+  /** Timestamp with microsecond precision (UTC) */
+  TimestampMicros = 17,
+  /** Timestamp with nanosecond precision (UTC) */
+  TimestampNanos = 18
+}
+/**
+ * Arrow field definition for schema.
+ *
+ */
+export interface ArrowField {
+  /** Field name */
+  name: string
+  /** Field data type (ArrowDataType enum value) */
+  dataType: number
+  /** Whether the field is nullable */
+  nullable?: boolean
+}
+/**
+ * Properties of the target Delta table for Arrow Flight ingestion.
+ *
+ * Unlike `TableProperties` which uses Protocol Buffers, Arrow Flight streams
+ * require an Arrow schema definition.
+ *
+ */
+export interface ArrowTableProperties {
+  /** Full table name in Unity Catalog (e.g., "catalog.schema.table") */
+  tableName: string
+  /** Arrow schema fields */
+  schemaFields: Array<ArrowField>
+}
 /**
  * Custom error type for Zerobus operations.
  *
@@ -117,8 +244,9 @@ export declare class ZerobusError {
 /**
  * A stream for ingesting data into a Databricks Delta table.
  *
- * The stream manages a bidirectional gRPC connection, handles acknowledgments,
- * and provides automatic recovery on transient failures.
+ * The stream manages JSON or Protocol Buffer ingestion over a bidirectional
+ * gRPC connection, handles acknowledgments, and provides automatic recovery
+ * on transient failures.
  *
  * # Example
  *
@@ -426,7 +554,7 @@ export declare class ZerobusSdk {
   /**
    * Creates a new ingestion stream to a Delta table.
    *
-   * This method establishes a bidirectional gRPC connection to the Zerobus service
+   * This method opens a JSON or Protocol Buffer stream to the Zerobus service
    * and prepares it for data ingestion. By default, it uses OAuth 2.0 Client Credentials
    * authentication. For custom authentication (e.g., Personal Access Tokens), provide
    * a custom headers_provider.
@@ -526,4 +654,146 @@ export declare class ZerobusSdk {
    * ```
    */
   recreateStream(stream: ZerobusStream): Promise<ZerobusStream>
+  /**
+   * Creates a new Arrow Flight stream to a Delta table.
+   *
+   * This method establishes an Arrow Flight connection to the Zerobus service
+   * for high-performance columnar data ingestion.
+   *
+   * # Arguments
+   *
+   * * `table_properties` - Properties of the target table including name and Arrow schema
+   * * `client_id` - OAuth 2.0 client ID
+   * * `client_secret` - OAuth 2.0 client secret
+   * * `options` - Optional stream configuration
+   *
+   * # Returns
+   *
+   * A Promise that resolves to a ZerobusArrowStream ready for data ingestion.
+   *
+   * # Example
+   *
+   * ```typescript
+   * const tableProps = {
+   *   tableName: 'catalog.schema.table',
+   *   schemaFields: [
+   *     { name: 'device_name', dataType: ArrowDataType.Utf8 },
+   *     { name: 'temp', dataType: ArrowDataType.Int32 },
+   *     { name: 'humidity', dataType: ArrowDataType.Int64 }
+   *   ]
+   * };
+   *
+   * const arrowStream = await sdk.createArrowStream(
+   *   tableProps,
+   *   clientId,
+   *   clientSecret,
+   *   { maxInflightBatches: 100 }
+   * );
+   * ```
+   */
+  createArrowStream(tableProperties: ArrowTableProperties, clientId: string, clientSecret: string, options?: ArrowStreamConfigurationOptions | undefined | null): Promise<ZerobusArrowStream>
+  /**
+   * Recreates an Arrow stream with the same configuration and re-ingests unacknowledged batches.
+   *
+   * # Arguments
+   *
+   * * `stream` - The terminally failed Arrow stream to recreate. The TypeScript wrapper
+   *   must not have been closed because `close()` releases its native handle.
+   *
+   * # Returns
+   *
+   * A Promise that resolves to a new ZerobusArrowStream with all unacknowledged batches re-ingested.
+   */
+  recreateArrowStream(stream: ZerobusArrowStream): Promise<ZerobusArrowStream>
+}
+/**
+ * An Arrow Flight stream for ingesting Arrow RecordBatches into a Delta table.
+ *
+ * This stream provides a high-performance interface for streaming Arrow data
+ * to Databricks Delta tables using the Arrow Flight protocol.
+ *
+ * # Lifecycle
+ *
+ * 1. Create a stream via `sdk.createArrowStream()`
+ * 2. Ingest Arrow IPC buffers with `ingestBatch()`
+ * 3. Use `waitForOffset()` to wait for acknowledgments
+ * 4. Call `flush()` to ensure all batches are persisted
+ * 5. Close the stream with `close()`
+ *
+ * # Example
+ *
+ * ```typescript
+ * import { tableToIPC } from 'apache-arrow';
+ *
+ * const arrowStream = await sdk.createArrowStream(
+ *   arrowTableProps,
+ *   clientId,
+ *   clientSecret,
+ *   options
+ * );
+ *
+ * const ipcBuffer = tableToIPC(arrowTable, 'stream');
+ * const offset = await arrowStream.ingestBatch(Buffer.from(ipcBuffer));
+ * await arrowStream.waitForOffset(offset);
+ * await arrowStream.close();
+ * ```
+ */
+export declare class ZerobusArrowStream {
+  /**
+   * Ingests a single Arrow IPC buffer into the stream.
+   *
+   * The buffer should be an Arrow IPC stream format containing one or more RecordBatches.
+   * You can create this using `tableToIPC(table, 'stream')` from the apache-arrow package.
+   *
+   * # Arguments
+   *
+   * * `ipc_buffer` - Arrow IPC stream format buffer
+   *
+   * # Returns
+   *
+   * The offset ID (bigint) assigned to this batch.
+   *
+   * # Example
+   *
+   * ```typescript
+   * const table = tableFromArrays({
+   *   device_name: ['sensor-1'],
+   *   temp: [25],
+   *   humidity: [60]
+   * });
+   * const ipcBuffer = tableToIPC(table, 'stream');
+   * const offset = await stream.ingestBatch(Buffer.from(ipcBuffer));
+   * await stream.waitForOffset(offset);
+   * ```
+   */
+  ingestBatch(ipcBuffer: Buffer): Promise<bigint>
+  /**
+   * Waits for a specific offset to be acknowledged by the server.
+   *
+   * Use this method with `ingestBatch()` to selectively wait for acknowledgments.
+   *
+   * # Arguments
+   *
+   * * `offset_id` - The offset ID to wait for (returned by ingestBatch)
+   */
+  waitForOffset(offsetId: bigint): Promise<void>
+  /** Flushes all pending batches and waits for acknowledgments. */
+  flush(): Promise<void>
+  /** Closes the stream gracefully. */
+  close(): Promise<void>
+  /** Returns whether the stream has been closed. */
+  get isClosed(): boolean
+  /** Returns the table name for this stream. */
+  get tableName(): string
+  /**
+   * Gets unacknowledged batches as Arrow IPC buffers.
+   *
+   * This method should only be called after a stream failure to retrieve batches
+   * that were sent but not acknowledged. These can be re-ingested into a new stream.
+   *
+   * # Returns
+   *
+   * An array of Buffers containing the unacknowledged batches in Arrow IPC format.
+   */
+  getUnackedBatches(): Promise<Array<Buffer>>
 }

package/index.js

@@ -310,9 +310,12 @@ if (!nativeBinding) {
   throw new Error(`Failed to load native binding`)
 }
 
-const { RecordType, ZerobusError, ZerobusStream, ZerobusSdk } = nativeBinding
+const { RecordType, ZerobusError, ZerobusStream, ZerobusSdk, IpcCompressionType, ArrowDataType, ZerobusArrowStream } = nativeBinding
 
 module.exports.RecordType = RecordType
 module.exports.ZerobusError = ZerobusError
 module.exports.ZerobusStream = ZerobusStream
 module.exports.ZerobusSdk = ZerobusSdk
+module.exports.IpcCompressionType = IpcCompressionType
+module.exports.ArrowDataType = ArrowDataType
+module.exports.ZerobusArrowStream = ZerobusArrowStream

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@databricks/zerobus-ingest-sdk",
-  "version": "1.2.0",
+  "version": "1.3.0",
   "description": "TypeScript/Node.js SDK for streaming data ingestion into Databricks Delta tables using Zerobus",
   "main": "index.js",
   "types": "index.d.ts",
@@ -76,11 +76,11 @@
     "example:arrow": "tsx examples/arrow/batch.ts"
   },
   "optionalDependencies": {
-    "@databricks/zerobus-ingest-sdk-linux-x64-gnu": "1.2.0",
-    "@databricks/zerobus-ingest-sdk-linux-arm64-gnu": "1.2.0",
-    "@databricks/zerobus-ingest-sdk-win32-x64-msvc": "1.2.0",
-    "@databricks/zerobus-ingest-sdk-darwin-x64": "1.2.0",
-    "@databricks/zerobus-ingest-sdk-darwin-arm64": "1.2.0"
+    "@databricks/zerobus-ingest-sdk-linux-x64-gnu": "1.3.0",
+    "@databricks/zerobus-ingest-sdk-linux-arm64-gnu": "1.3.0",
+    "@databricks/zerobus-ingest-sdk-win32-x64-msvc": "1.3.0",
+    "@databricks/zerobus-ingest-sdk-darwin-x64": "1.3.0",
+    "@databricks/zerobus-ingest-sdk-darwin-arm64": "1.3.0"
   },
   "dependencies": {
     "protobufjs": "^8.7.1"

package/src/lib.rs

@@ -214,8 +214,9 @@ fn convert_js_to_record_payload(env: &Env, payload: Unknown) -> Result<RustRecor
 
 /// A stream for ingesting data into a Databricks Delta table.
 ///
-/// The stream manages a bidirectional gRPC connection, handles acknowledgments,
-/// and provides automatic recovery on transient failures.
+/// The stream manages JSON or Protocol Buffer ingestion over a bidirectional
+/// gRPC connection, handles acknowledgments, and provides automatic recovery
+/// on transient failures.
 ///
 /// # Example
 ///
@@ -926,7 +927,7 @@ impl ZerobusSdk {
 
     /// Creates a new ingestion stream to a Delta table.
     ///
-    /// This method establishes a bidirectional gRPC connection to the Zerobus service
+    /// This method opens a JSON or Protocol Buffer stream to the Zerobus service
     /// and prepares it for data ingestion. By default, it uses OAuth 2.0 Client Credentials
     /// authentication. For custom authentication (e.g., Personal Access Tokens), provide
     /// a custom headers_provider.
@@ -1181,8 +1182,8 @@ fn base64_decode(input: &str) -> std::result::Result<Vec<u8>, String> {
 }
 
 // =============================================================================
-// Arrow Flight Support (Beta)
-// Enabled with feature flag: cargo build --features arrow-flight
+// Arrow Flight support
+// Behind the arrow-flight feature. Enable with: npm run build:arrow
 // =============================================================================
 
 #[cfg(feature = "arrow-flight")]
@@ -1197,8 +1198,6 @@ use databricks_zerobus_ingest_sdk::{
 
 /// IPC compression type for Arrow Flight streams.
 ///
-/// **Beta**: Arrow Flight support is in Beta. The API is stabilising but
-/// may still change before reaching GA.
 #[cfg(feature = "arrow-flight")]
 #[napi]
 pub enum IpcCompressionType {
@@ -1210,8 +1209,6 @@ pub enum IpcCompressionType {
 
 /// Configuration options for Arrow Flight streams.
 ///
-/// **Beta**: Arrow Flight support is in Beta. The API is stabilising but
-/// may still change before reaching GA.
 #[cfg(feature = "arrow-flight")]
 #[napi(object)]
 #[derive(Debug, Clone)]
@@ -1268,7 +1265,6 @@ fn map_ipc_compression(value: Option<i32>) -> Option<arrow_ipc::CompressionType>
 
 /// Arrow data type enum for schema definition.
 ///
-/// **Beta**: Arrow Flight support is in Beta.
 #[cfg(feature = "arrow-flight")]
 #[napi]
 pub enum ArrowDataType {
@@ -1340,7 +1336,6 @@ fn convert_arrow_data_type(dt: i32) -> RustDataType {
 
 /// Arrow field definition for schema.
 ///
-/// **Beta**: Arrow Flight support is in Beta.
 #[cfg(feature = "arrow-flight")]
 #[napi(object)]
 #[derive(Debug, Clone)]
@@ -1358,8 +1353,6 @@ pub struct ArrowField {
 /// Unlike `TableProperties` which uses Protocol Buffers, Arrow Flight streams
 /// require an Arrow schema definition.
 ///
-/// **Beta**: Arrow Flight support is in Beta. The API is stabilising but
-/// may still change before reaching GA.
 #[cfg(feature = "arrow-flight")]
 #[napi(object)]
 #[derive(Debug, Clone)]
@@ -1393,9 +1386,6 @@ fn build_arrow_schema(fields: &[ArrowField]) -> Arc<RustArrowSchema> {
 /// This stream provides a high-performance interface for streaming Arrow data
 /// to Databricks Delta tables using the Arrow Flight protocol.
 ///
-/// **Beta**: Arrow Flight support is in Beta. The API is stabilising but
-/// may still change before reaching GA.
-///
 /// # Lifecycle
 ///
 /// 1. Create a stream via `sdk.createArrowStream()`
@@ -1617,9 +1607,6 @@ impl ZerobusArrowStream {
 impl ZerobusSdk {
     /// Creates a new Arrow Flight stream to a Delta table.
     ///
-    /// **Beta**: Arrow Flight support is in Beta. The API is stabilising
-    /// but may still change before reaching GA.
-    ///
     /// This method establishes an Arrow Flight connection to the Zerobus service
     /// for high-performance columnar data ingestion.
     ///
@@ -1733,8 +1720,6 @@ impl ZerobusSdk {
 
     /// Recreates an Arrow stream with the same configuration and re-ingests unacknowledged batches.
     ///
-    /// **Beta**: Arrow Flight support is in Beta.
-    ///
     /// # Arguments
     ///
     /// * `stream` - The terminally failed Arrow stream to recreate. The TypeScript wrapper