@databricks/zerobus-ingest-sdk 1.0.2 → 1.1.0

package/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "zerobus-sdk-ts"
-version = "1.0.2"
+version = "1.1.0"
 authors = ["Databricks"]
 edition = "2021"
 license = "Apache-2.0"
@@ -11,44 +11,44 @@ repository = "https://github.com/databricks/zerobus-sdk"
 crate-type = ["cdylib"]
 
 [dependencies]
-# NAPI-RS for Node.js bindings
-napi = { version = "2", features = ["async", "tokio_rt", "serde-json"] }
+# NAPI-RS for Node.js bindings. `napi6` enables `BigInt::get_i64()` for the
+# lossless bigint → i64 path used by `waitForOffset`.
+napi = { version = "2", features = ["async", "tokio_rt", "serde-json", "napi6"] }
 napi-derive = "2"
 
-# The Rust SDK we're wrapping
-databricks-zerobus-ingest-sdk = "1.0.0"
+# Rust SDK 2.0.1 brings Arrow Flight Beta and the `ingest_ipc_batch` IPC
+# entry point used by `ingestBatch`. CI overlays `[patch.crates-io]` to
+# swap in the in-tree Rust SDK when `use_local_sdk` is set (see
+# ci-typescript.yml).
+databricks-zerobus-ingest-sdk = "2.0.1"
 
-# Async runtime (same as the Rust SDK)
-tokio = { version = "1.42", features = ["macros", "rt-multi-thread"] }
+# Match the Rust SDK 2.0 workspace versions.
+tokio = { version = "1.52", features = ["macros", "rt-multi-thread"] }
 
-# Protobuf support
-prost = "0.13.3"
-prost-types = "0.13.3"
+prost = "0.14"
+prost-types = "0.14"
 
-# Serialization
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 
-# Error handling
 thiserror = "1.0"
-
-# Base64 encoding/decoding
 base64 = "0.21"
-
-# Async trait support
 async-trait = "0.1"
+# `Bytes` is the input type for the Rust SDK's `ingest_ipc_batch` entry.
+bytes = "1"
 
-# Arrow dependencies (only used when arrow-flight feature is enabled)
-arrow-array = { version = "56.2.0", optional = true }
-arrow-schema = { version = "56.2.0", optional = true }
-arrow-ipc = { version = "56.2.0", features = ["lz4", "zstd"], optional = true }
+# Arrow dependencies (only used when arrow-flight feature is enabled).
+# 58.2 matches the Rust SDK 2.0 workspace.
+arrow-array = { version = "58.2", optional = true }
+arrow-schema = { version = "58.2", optional = true }
+arrow-ipc = { version = "58.2", features = ["lz4", "zstd"], optional = true }
 
 [build-dependencies]
 napi-build = "2"
 
 [features]
 default = []
-# Arrow Flight is experimental/unsupported - enable with: npm run build:arrow
+# Arrow Flight is in Beta - enable with: npm run build:arrow
 arrow-flight = [
     "databricks-zerobus-ingest-sdk/arrow-flight",
     "dep:arrow-array",

package/LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright [yyyy] [name of copyright owner]
+   Copyright (2025) Databricks, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

package/README.md

@@ -28,7 +28,7 @@ The Databricks Zerobus Ingest SDK for TypeScript provides a high-performance cli
 - **High-throughput ingestion**: Optimized for high-volume data ingestion with native Rust implementation
 - **Automatic recovery**: Built-in retry and recovery mechanisms for transient failures
 - **Flexible configuration**: Customizable stream behavior and timeouts
-- **Multiple serialization formats**: Support for JSON and Protocol Buffers
+- **Multiple serialization formats**: Support for JSON, Protocol Buffers, and Arrow Flight (Beta) with optional LZ4 / ZSTD compression
 - **Type widening**: Accept high-level types (plain objects, protobuf messages) or low-level types (strings, buffers) - automatically handles serialization
 - **Batch ingestion**: Ingest multiple records with a single acknowledgment for higher throughput
 - **OAuth 2.0 authentication**: Secure authentication with client credentials
@@ -137,8 +137,6 @@ source $HOME/.cargo/env
 
 #### Installation Steps
 
-**Note for macOS users**: Pre-built binaries are not available. The package will automatically build from source during `npm install`. Ensure you have Rust toolchain and Xcode Command Line Tools installed (see prerequisites above).
-
 1. Clone the repository:
    ```bash
    git clone https://github.com/databricks/zerobus-sdk.git
@@ -1127,14 +1125,15 @@ The SDK supports all platforms where Node.js and Rust are available.
 Pre-built native binaries are available for:
 
 - **Linux**: x64, ARM64
+- **macOS**: x64, ARM64
 - **Windows**: x64
 
 ### Build from Source
 
-**macOS users**: Pre-built binaries are not available for macOS. The package will automatically build from source during `npm install`, which requires:
+Other platforms (e.g. Linux musl, FreeBSD) need to build from source during `npm install`, which requires:
 
 - **Rust toolchain** (1.70+): Install via `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh`
-- **Xcode Command Line Tools**: Install via `xcode-select --install`
+- Platform C/C++ build tools (Xcode CLT on macOS, build-essential on Debian/Ubuntu, etc.)
 
 The build process happens automatically during installation and typically takes 2-3 minutes.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@databricks/zerobus-ingest-sdk",
-  "version": "1.0.2",
+  "version": "1.1.0",
   "description": "TypeScript/Node.js SDK for streaming data ingestion into Databricks Delta tables using Zerobus",
   "main": "index.js",
   "types": "index.d.ts",
@@ -63,17 +63,18 @@
     "example:json:batch": "tsx examples/json/batch.ts",
     "example:proto:single": "tsx examples/proto/single.ts",
     "example:proto:batch": "tsx examples/proto/batch.ts",
-    "example:arrow:single": "tsx examples/arrow/single.ts",
-    "example:arrow:batch": "tsx examples/arrow/batch.ts"
+    "example:arrow": "tsx examples/arrow/batch.ts"
   },
   "optionalDependencies": {
-    "@databricks/zerobus-ingest-sdk-linux-x64-gnu": "1.0.2",
-    "@databricks/zerobus-ingest-sdk-linux-arm64-gnu": "1.0.2",
-    "@databricks/zerobus-ingest-sdk-win32-x64-msvc": "1.0.2"
+    "@databricks/zerobus-ingest-sdk-linux-x64-gnu": "1.1.0",
+    "@databricks/zerobus-ingest-sdk-linux-arm64-gnu": "1.1.0",
+    "@databricks/zerobus-ingest-sdk-win32-x64-msvc": "1.1.0",
+    "@databricks/zerobus-ingest-sdk-darwin-x64": "1.1.0",
+    "@databricks/zerobus-ingest-sdk-darwin-arm64": "1.1.0"
   },
   "peerDependencies": {
     "protobufjs": "^7.0.0",
-    "apache-arrow": "^56.0.0"
+    "apache-arrow": "^18.0.0"
   },
   "peerDependenciesMeta": {
     "protobufjs": {