@databricks/zerobus-ingest-sdk 0.1.1 → 0.3.0

package/Cargo.toml

@@ -1,11 +1,11 @@
 [package]
 name = "zerobus-sdk-ts"
-version = "0.1.0"
+version = "0.3.0"
 authors = ["Databricks"]
 edition = "2021"
 license-file = "LICENSE"
 description = "TypeScript/Node.js bindings for the Databricks Zerobus SDK"
-repository = "https://github.com/databricks/zerobus-sdk-ts"
+repository = "https://github.com/databricks/zerobus-sdk"
 
 [lib]
 crate-type = ["cdylib"]
@@ -15,8 +15,8 @@ crate-type = ["cdylib"]
 napi = { version = "2", features = ["async", "tokio_rt", "serde-json"] }
 napi-derive = "2"
 
-# The Rust SDK we're wrapping (using local path for development)
-databricks-zerobus-ingest-sdk = { git = "https://github.com/databricks/zerobus-sdk-rs", tag = "v0.2.0" }
+# The Rust SDK we're wrapping
+databricks-zerobus-ingest-sdk = "0.6.0"
 
 # Async runtime (same as the Rust SDK)
 tokio = { version = "1.42", features = ["macros", "rt-multi-thread"] }
@@ -38,9 +38,24 @@ base64 = "0.21"
 # Async trait support
 async-trait = "0.1"
 
+# Arrow dependencies (only used when arrow-flight feature is enabled)
+arrow-array = { version = "56.2.0", optional = true }
+arrow-schema = { version = "56.2.0", optional = true }
+arrow-ipc = { version = "56.2.0", features = ["lz4", "zstd"], optional = true }
+
 [build-dependencies]
 napi-build = "2"
 
+[features]
+default = []
+# Arrow Flight is experimental/unsupported - enable with: npm run build:arrow
+arrow-flight = [
+    "databricks-zerobus-ingest-sdk/arrow-flight",
+    "dep:arrow-array",
+    "dep:arrow-schema",
+    "dep:arrow-ipc"
+]
+
 [profile.release]
 lto = true
 strip = true

package/README.md

@@ -1,17 +1,12 @@
 # Databricks Zerobus Ingest SDK for TypeScript
 
-[Public Preview](https://docs.databricks.com/release-notes/release-types.html): This SDK is supported for production use cases and is available to all customers. Databricks is actively working on stabilizing the Zerobus Ingest SDK for TypeScript. Minor version updates may include backwards-incompatible changes.
-
-We are keen to hear feedback from you on this SDK. Please [file issues](https://github.com/databricks/zerobus-sdk-ts/issues), and we will address them.
-
-The Databricks Zerobus Ingest SDK for TypeScript provides a high-performance client for ingesting data directly into Databricks Delta tables using the Zerobus streaming protocol. This SDK wraps the high-performance [Rust SDK](https://github.com/databricks/zerobus-sdk-rs) using native bindings for optimal performance. | See also the [SDK for Rust](https://github.com/databricks/zerobus-sdk-rs) | See also the [SDK for Python](https://github.com/databricks/zerobus-sdk-py) | See also the [SDK for Java](https://github.com/databricks/zerobus-sdk-java) | See also the [SDK for Go](https://github.com/databricks/zerobus-sdk-go)
+The Databricks Zerobus Ingest SDK for TypeScript provides a high-performance client for ingesting data directly into Databricks Delta tables using the Zerobus streaming protocol. This SDK wraps the high-performance [Rust SDK](https://github.com/databricks/zerobus-sdk/tree/main/rust) using native bindings for optimal performance.
 
 ## Table of Contents
 
 - [Features](#features)
 - [Requirements](#requirements)
 - [Quick Start User Guide](#quick-start-user-guide)
-  - [Prerequisites](#prerequisites)
   - [Installation](#installation)
   - [Choose Your Serialization Format](#choose-your-serialization-format)
   - [Option 1: Using JSON (Quick Start)](#option-1-using-json-quick-start)
@@ -25,8 +20,8 @@ The Databricks Zerobus Ingest SDK for TypeScript provides a high-performance cli
 - [Best Practices](#best-practices)
 - [Platform Support](#platform-support)
 - [Architecture](#architecture)
-- [Contributing](#contributing)
-- [Related Projects](#related-projects)
+- [Community and Contributing](#community-and-contributing)
+- [License](#license)
 
 ## Features
 
@@ -67,65 +62,7 @@ These will be installed automatically:
 
 ### Prerequisites
 
-Before using the SDK, you'll need the following:
-
-#### 1. Workspace URL and Workspace ID
-
-After logging into your Databricks workspace, look at the browser URL:
-
-```
-https://<databricks-instance>.cloud.databricks.com/?o=<workspace-id>
-```
-
-- **Workspace URL**: The part before `/?o=` → `https://<databricks-instance>.cloud.databricks.com`
-- **Workspace ID**: The part after `?o=` → `<workspace-id>`
-- **Zerobus Endpoint**: `https://<workspace-id>.zerobus.<region>.cloud.databricks.com`
-
-> **Note:** The examples above show AWS endpoints (`.cloud.databricks.com`). For Azure deployments, the workspace URL will be `https://<databricks-instance>.azuredatabricks.net` and Zerobus endpoint will use `.azuredatabricks.net`.
-
-Example:
-- Full URL: `https://dbc-a1b2c3d4-e5f6.cloud.databricks.com/?o=1234567890123456`
-- Workspace URL: `https://dbc-a1b2c3d4-e5f6.cloud.databricks.com`
-- Workspace ID: `1234567890123456`
-- Zerobus Endpoint: `https://1234567890123456.zerobus.us-west-2.cloud.databricks.com`
-
-#### 2. Create a Delta Table
-
-Create a table using Databricks SQL:
-
-```sql
-CREATE TABLE <catalog_name>.default.air_quality (
-    device_name STRING,
-    temp INT,
-    humidity BIGINT
-)
-USING DELTA;
-```
-
-Replace `<catalog_name>` with your catalog name (e.g., `main`).
-
-#### 3. Create a Service Principal
-
-1. Navigate to **Settings > Identity and Access** in your Databricks workspace
-2. Click **Service principals** and create a new service principal
-3. Generate a new secret for the service principal and save it securely
-4. Grant the following permissions:
-   - `USE_CATALOG` on the catalog (e.g., `main`)
-   - `USE_SCHEMA` on the schema (e.g., `default`)
-   - `MODIFY` and `SELECT` on the table (e.g., `air_quality`)
-
-Grant permissions using SQL:
-
-```sql
--- Grant catalog permission
-GRANT USE CATALOG ON CATALOG <catalog_name> TO `<service-principal-application-id>`;
-
--- Grant schema permission
-GRANT USE SCHEMA ON SCHEMA <catalog_name>.default TO `<service-principal-application-id>`;
-
--- Grant table permissions
-GRANT SELECT, MODIFY ON TABLE <catalog_name>.default.air_quality TO `<service-principal-application-id>`;
-```
+Before using the SDK, you need a Databricks workspace URL, a Delta table, and a service principal. See the [monorepo prerequisites](https://github.com/databricks/zerobus-sdk/blob/main/README.md#prerequisites) for detailed setup instructions.
 
 ### Installation
 
@@ -202,10 +139,10 @@ source $HOME/.cargo/env
 
 **Note for macOS users**: Pre-built binaries are not available. The package will automatically build from source during `npm install`. Ensure you have Rust toolchain and Xcode Command Line Tools installed (see prerequisites above).
 
-1. Extract the SDK package:
+1. Clone the repository:
    ```bash
-   unzip zerobus-sdk-ts.zip
-   cd zerobus-sdk-ts
+   git clone https://github.com/databricks/zerobus-sdk.git
+   cd zerobus-sdk/ts
    ```
 
 2. Install dependencies:
@@ -256,7 +193,7 @@ import { ZerobusSdk, RecordType } from '@databricks/zerobus-ingest-sdk';
 
 // Configuration
 // For AWS:
-const zerobusEndpoint = '<workspace-id>.zerobus.<region>.cloud.databricks.com';
+const zerobusEndpoint = 'https://<workspace-id>.zerobus.<region>.cloud.databricks.com';
 const workspaceUrl = 'https://<workspace-name>.cloud.databricks.com';
 // For Azure:
 // const zerobusEndpoint = '<workspace-id>.zerobus.<region>.azuredatabricks.net';
@@ -288,35 +225,24 @@ const stream = await sdk.createStream(
 );
 
 try {
-    let lastAckPromise;
+    let lastOffset: bigint;
 
     // Send all records
     for (let i = 0; i < 100; i++) {
-        // Create JSON record
         const record = {
             device_name: `sensor-${i % 10}`,
             temp: 20 + (i % 15),
             humidity: 50 + (i % 40)
         };
 
-        // JSON supports 2 types:
-        // 1. object (high-level) - SDK auto-stringifies
-        lastAckPromise = stream.ingestRecord(record);
-        // 2. string (low-level) - pre-serialized JSON
-        // lastAckPromise = stream.ingestRecord(JSON.stringify(record));
+        // ingestRecordOffset returns immediately after queuing
+        lastOffset = await stream.ingestRecordOffset(record);
     }
 
-    console.log('All records sent. Waiting for last acknowledgment...');
-
-    // Wait for the last record's acknowledgment
-    const lastOffset = await lastAckPromise;
-    console.log(`Last record offset: ${lastOffset}`);
-
-    // Flush to ensure all records are acknowledged
-    await stream.flush();
+    // Wait for all records to be acknowledged
+    await stream.waitForOffset(lastOffset);
     console.log('Successfully ingested 100 records!');
 } finally {
-    // Always close the stream
     await stream.close();
 }
 ```
@@ -430,7 +356,7 @@ import * as airQuality from './examples/generated/air_quality';
 import { loadDescriptorProto } from '@databricks/zerobus-ingest-sdk/utils/descriptor';
 
 // Configuration
-const zerobusEndpoint = '<workspace-id>.zerobus.<region>.cloud.databricks.com';
+const zerobusEndpoint = 'https://<workspace-id>.zerobus.<region>.cloud.databricks.com';
 const workspaceUrl = 'https://<workspace-name>.cloud.databricks.com';
 const tableName = 'main.default.air_quality';
 const clientId = process.env.DATABRICKS_CLIENT_ID!;
@@ -464,7 +390,7 @@ const stream = await sdk.createStream(tableProperties, clientId, clientSecret, o
 
 try {
     const AirQuality = airQuality.examples.AirQuality;
-    let lastAckPromise;
+    let lastOffset: bigint;
 
     // Send all records
     for (let i = 0; i < 100; i++) {
@@ -474,22 +400,12 @@ try {
             humidity: 50 + i
         });
 
-        // Protobuf supports 2 types:
-        // 1. Message object (high-level) - SDK calls .encode().finish()
-        lastAckPromise = stream.ingestRecord(record);
-        // 2. Buffer (low-level) - pre-serialized bytes
-        // const buffer = Buffer.from(AirQuality.encode(record).finish());
-        // lastAckPromise = stream.ingestRecord(buffer);
+        // ingestRecordOffset returns immediately after queuing
+        lastOffset = await stream.ingestRecordOffset(record);
     }
 
-    console.log('All records sent. Waiting for last acknowledgment...');
-
-    // Wait for the last record's acknowledgment
-    const lastOffset = await lastAckPromise;
-    console.log(`Last record offset: ${lastOffset}`);
-
-    // Flush to ensure all records are acknowledged
-    await stream.flush();
+    // Wait for all records to be acknowledged
+    await stream.waitForOffset(lastOffset);
     console.log('Successfully ingested 100 records!');
 } finally {
     await stream.close();
@@ -610,7 +526,7 @@ npm run build:proto
 protoc --descriptor_set_out=schemas/air_quality_descriptor.pb --include_imports schemas/air_quality.proto
 
 # Run example
-npx tsx examples/proto.ts
+npm run example:proto:single
 ```
 
 #### Why Two Steps (TypeScript + Descriptor)?
@@ -635,21 +551,23 @@ See the `examples/` directory for complete, runnable examples. See [examples/REA
 
 ```bash
 # Set environment variables
-export ZEROBUS_SERVER_ENDPOINT="<workspace-id>.zerobus.<region>.cloud.databricks.com"
+export ZEROBUS_SERVER_ENDPOINT="https://<workspace-id>.zerobus.<region>.cloud.databricks.com"
 export DATABRICKS_WORKSPACE_URL="https://<workspace-name>.cloud.databricks.com"
 export DATABRICKS_CLIENT_ID="your-client-id"
 export DATABRICKS_CLIENT_SECRET="your-client-secret"
 export ZEROBUS_TABLE_NAME="main.default.air_quality"
 
-# Run JSON example
-npx tsx examples/json.ts
+# Run JSON examples
+npm run example:json:single
+npm run example:json:batch
 
 # For Protocol Buffers, generate TypeScript code and descriptor
 npm run build:proto
 protoc --descriptor_set_out=schemas/air_quality_descriptor.pb --include_imports schemas/air_quality.proto
 
-# Run Protocol Buffers example
-npx tsx examples/proto.ts
+# Run Protocol Buffers examples
+npm run example:proto:single
+npm run example:proto:batch
 ```
 
 ### Batch Ingestion
@@ -664,13 +582,14 @@ const records = Array.from({ length: 1000 }, (_, i) =>
 );
 
 // Protobuf Type 1: Message objects (high-level) - SDK auto-serializes
-const offsetId = await stream.ingestRecords(records);
+const offsetId = await stream.ingestRecordsOffset(records);
 
 // Protobuf Type 2: Buffers (low-level) - pre-serialized bytes
 // const buffers = records.map(r => Buffer.from(AirQuality.encode(r).finish()));
-// const offsetId = await stream.ingestRecords(buffers);
+// const offsetId = await stream.ingestRecordsOffset(buffers);
 
 if (offsetId !== null) {
+  await stream.waitForOffset(offsetId);
   console.log(`Batch acknowledged at offset ${offsetId}`);
 }
 ```
@@ -685,11 +604,15 @@ const records = Array.from({ length: 1000 }, (_, i) => ({
 }));
 
 // JSON Type 1: objects (high-level) - SDK auto-stringifies
-const offsetId = await stream.ingestRecords(records);
+const offsetId = await stream.ingestRecordsOffset(records);
 
 // JSON Type 2: strings (low-level) - pre-serialized JSON
 // const jsonRecords = records.map(r => JSON.stringify(r));
-// const offsetId = await stream.ingestRecords(jsonRecords);
+// const offsetId = await stream.ingestRecordsOffset(jsonRecords);
+
+if (offsetId !== null) {
+  await stream.waitForOffset(offsetId);
+}
 ```
 
 **Type Widening Support:**
@@ -703,7 +626,7 @@ const offsetId = await stream.ingestRecords(records);
 - Use `recreateStream()` for recovery - it automatically handles unacknowledged batches
 
 **Examples:**
-Both `json.ts` and `proto.ts` examples demonstrate batch ingestion.
+See `examples/json/batch.ts` and `examples/proto/batch.ts` for batch ingestion examples.
 
 ## Authentication
 
@@ -732,29 +655,25 @@ The SDK automatically fetches access tokens and includes these headers:
 Beyond OAuth, you can use custom headers for Personal Access Tokens (PAT) or other auth methods:
 
 ```typescript
-import { ZerobusSdk } from '@databricks/zerobus-ingest-sdk';
-import { HeadersProvider } from '@databricks/zerobus-ingest-sdk/src/headers_provider';
-
-class CustomHeadersProvider implements HeadersProvider {
-  async getHeaders(): Promise<Array<[string, string]>> {
-    return [
-      ["authorization", `Bearer ${myToken}`],
-      ["x-databricks-zerobus-table-name", tableName]
-    ];
-  }
-}
-
-const headersProvider = new CustomHeadersProvider();
 const stream = await sdk.createStream(
   tableProperties,
   '', // client_id (ignored when headers_provider is provided)
   '', // client_secret (ignored when headers_provider is provided)
   options,
-  { getHeadersCallback: headersProvider.getHeaders.bind(headersProvider) }
+  {
+    getHeadersCallback: async () => [
+      ["authorization", `Bearer ${myToken}`],
+      ["x-databricks-zerobus-table-name", tableName]
+    ]
+  }
 );
 ```
 
-**Note:** Custom authentication is integrated into the main `createStream()` method. See the API Reference for details.
+**Required headers:**
+- `authorization` - Bearer token or other auth header
+- `x-databricks-zerobus-table-name` - The fully qualified table name
+
+**Note:** The SDK automatically adds the `user-agent` header if not provided.
 
 ## Configuration
 
@@ -770,6 +689,7 @@ const stream = await sdk.createStream(
 | `recoveryRetries` | 4 | Maximum number of recovery attempts |
 | `flushTimeoutMs` | 300,000 | Timeout for flush operations (ms) |
 | `serverLackOfAckTimeoutMs` | 60,000 | Server acknowledgment timeout (ms) |
+| `streamPausedMaxWaitTimeMs` | undefined | Max wait time during graceful stream close (ms) |
 
 ### Example Configuration
 
@@ -839,7 +759,8 @@ The SDK includes automatic recovery for transient failures (enabled by default w
 
 ```typescript
 try {
-    const offset = await stream.ingestRecord(JSON.stringify(record));
+    const offset = await stream.ingestRecordOffset(record);
+    await stream.waitForOffset(offset);
     console.log(`Success: offset ${offset}`);
 } catch (error) {
     console.error('Ingestion failed:', error);
@@ -889,7 +810,7 @@ new ZerobusSdk(zerobusEndpoint: string, unityCatalogUrl: string)
 ```
 
 **Parameters:**
-- `zerobusEndpoint` (string) - The Zerobus gRPC endpoint (e.g., `<workspace-id>.zerobus.<region>.cloud.databricks.com` for AWS, or `<workspace-id>.zerobus.<region>.azuredatabricks.net` for Azure)
+- `zerobusEndpoint` (string) - The Zerobus gRPC endpoint (e.g., `https://<workspace-id>.zerobus.<region>.cloud.databricks.com` for AWS, or `https://<workspace-id>.zerobus.<region>.azuredatabricks.net` for Azure)
 - `unityCatalogUrl` (string) - The Unity Catalog endpoint (your workspace URL)
 
 **Methods:**
@@ -952,11 +873,44 @@ Represents an active ingestion stream.
 
 **Methods:**
 
+```typescript
+async ingestRecordOffset(payload: Buffer | string | object): Promise<bigint>
+```
+
+**(Recommended)** Ingests a single record. The Promise resolves immediately after the record is queued (before server acknowledgment). Use `waitForOffset()` to wait for acknowledgment when needed.
+
+```typescript
+// High-throughput pattern: send many, wait once
+const offset1 = await stream.ingestRecordOffset(record1);  // Resolves immediately
+const offset2 = await stream.ingestRecordOffset(record2);  // Resolves immediately
+await stream.waitForOffset(offset2);  // Waits for server to acknowledge all records up to offset2
+```
+
+---
+
+```typescript
+async ingestRecordsOffset(payloads: Array<Buffer | string | object>): Promise<bigint | null>
+```
+
+**(Recommended)** Ingests multiple records as a batch. The Promise resolves immediately after the batch is queued (before server acknowledgment). Returns `null` for empty batches.
+
+---
+
+```typescript
+async waitForOffset(offsetId: bigint): Promise<void>
+```
+
+Waits for the server to acknowledge all records up to and including the specified offset ID.
+
+---
+
 ```typescript
 async ingestRecord(payload: Buffer | string | object): Promise<bigint>
 ```
 
-Ingests a single record. This method **blocks** until the record is sent to the SDK's internal landing zone, then returns a Promise for the server acknowledgment. This allows you to send many records without waiting for individual acknowledgments.
+**@deprecated** Use `ingestRecordOffset()` instead.
+
+Ingests a single record. Unlike `ingestRecordOffset()`, the Promise only resolves **after the server acknowledges** the record. This is slower for high-throughput scenarios.
 
 **Parameters:**
 - `payload` - Record data. The SDK supports 4 input types for flexibility:
@@ -994,7 +948,9 @@ await stream.ingestRecord(buffer);
 async ingestRecords(payloads: Array<Buffer | string | object>): Promise<bigint | null>
 ```
 
-Ingests multiple records as a batch. All records in a batch are acknowledged together atomically. This method **blocks** until all records are sent to the SDK's internal landing zone, then returns a Promise for the server acknowledgment.
+**@deprecated** Use `ingestRecordsOffset()` instead.
+
+Ingests multiple records as a batch. Unlike `ingestRecordsOffset()`, the Promise only resolves **after the server acknowledges** the batch. This is slower for high-throughput scenarios.
 
 **Parameters:**
 - `payloads` - Array of record data. Supports the same 4 types as `ingestRecord()`:
@@ -1135,14 +1091,15 @@ Configuration options for stream behavior.
 
 ```typescript
 interface StreamConfigurationOptions {
-    recordType?: RecordType;           // RecordType.Json or RecordType.Proto. Default: RecordType.Proto
-    maxInflightRequests?: number;      // Default: 10,000
-    recovery?: boolean;                // Default: true
-    recoveryTimeoutMs?: number;        // Default: 15,000
-    recoveryBackoffMs?: number;        // Default: 2,000
-    recoveryRetries?: number;          // Default: 4
-    flushTimeoutMs?: number;           // Default: 300,000
-    serverLackOfAckTimeoutMs?: number; // Default: 60,000
+    recordType?: RecordType;              // RecordType.Json or RecordType.Proto. Default: RecordType.Proto
+    maxInflightRequests?: number;         // Default: 10,000
+    recovery?: boolean;                   // Default: true
+    recoveryTimeoutMs?: number;           // Default: 15,000
+    recoveryBackoffMs?: number;           // Default: 2,000
+    recoveryRetries?: number;             // Default: 4
+    flushTimeoutMs?: number;              // Default: 300,000
+    serverLackOfAckTimeoutMs?: number;    // Default: 60,000
+    streamPausedMaxWaitTimeMs?: number;   // Default: undefined (wait for full server duration)
 }
 
 enum RecordType {
@@ -1159,7 +1116,7 @@ enum RecordType {
 4. **Error handling**: The stream handles errors internally with automatic retry. Only use `recreateStream()` for persistent failures after internal retries are exhausted.
 5. **Use Protocol Buffers for production**: Protocol Buffers (the default) provides better performance and schema validation. Use JSON only when you need schema flexibility or for quick prototyping.
 6. **Store credentials securely**: Use environment variables, never hardcode credentials
-7. **Use batch ingestion**: For high-throughput scenarios, use `ingestRecords()` instead of individual `ingestRecord()` calls
+7. **Use batch ingestion**: For high-throughput scenarios, use `ingestRecordsOffset()` instead of individual `ingestRecordOffset()` calls
 
 ## Platform Support
 
@@ -1183,7 +1140,7 @@ The build process happens automatically during installation and typically takes
 
 ## Architecture
 
-This SDK wraps the high-performance [Rust Zerobus SDK](https://github.com/databricks/zerobus-sdk-rs) using [NAPI-RS](https://napi.rs):
+This SDK wraps the high-performance [Rust Zerobus SDK](https://github.com/databricks/zerobus-sdk/tree/main/rust) using [NAPI-RS](https://napi.rs):
 
 ```
 ┌─────────────────────────────┐
@@ -1203,18 +1160,21 @@ This SDK wraps the high-performance [Rust Zerobus SDK](https://github.com/databr
 ```
 
 **Benefits:**
-- **Zero-copy data transfer** between JavaScript and Rust
+- **Native performance** - Rust implementation for high-throughput ingestion
 - **Native async/await support** - Rust futures become JavaScript Promises
 - **Automatic memory management** - No manual cleanup required
 - **Type safety** - Compile-time checks on both sides
 
-## Contributing
+## Community and Contributing
+
+This is an open source project. We welcome contributions, feedback, and bug reports.
 
-We welcome contributions! Please see [CONTRIBUTING.md](CONTRIBUTING.md) for details.
+- **[Contributing Guide](https://github.com/databricks/zerobus-sdk/blob/main/typescript/CONTRIBUTING.md)**: TypeScript-specific development setup and workflow.
+- **[General Contributing Guide](https://github.com/databricks/zerobus-sdk/blob/main/CONTRIBUTING.md)**: Pull request process, commit requirements, and policies.
+- **[Changelog](https://github.com/databricks/zerobus-sdk/blob/main/typescript/CHANGELOG.md)**: See the history of changes in the SDK.
+- **[Security Policy](https://github.com/databricks/zerobus-sdk/blob/main/SECURITY.md)**: Read about our security process and how to report vulnerabilities.
+- **[Developer Certificate of Origin (DCO)](https://github.com/databricks/zerobus-sdk/blob/main/DCO)**: Understand the agreement for contributions.
 
-## Related Projects
+## License
 
-- [Zerobus Rust SDK](https://github.com/databricks/zerobus-sdk-rs) - The underlying Rust implementation
-- [Zerobus Python SDK](https://github.com/databricks/zerobus-sdk-py) - Python SDK for Zerobus
-- [Zerobus Java SDK](https://github.com/databricks/zerobus-sdk-java) - Java SDK for Zerobus
-- [NAPI-RS](https://napi.rs) - Rust/Node.js binding framework
+This SDK is licensed under the Databricks License. See the [LICENSE](https://github.com/databricks/zerobus-sdk/blob/main/LICENSE) file for the full license text. The license is also available online at [https://www.databricks.com/legal/db-license](https://www.databricks.com/legal/db-license).

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@databricks/zerobus-ingest-sdk",
-  "version": "0.1.1",
+  "version": "0.3.0",
   "description": "TypeScript/Node.js SDK for streaming data ingestion into Databricks Delta tables using Zerobus",
   "main": "index.js",
   "types": "index.d.ts",
@@ -15,7 +15,7 @@
   ],
   "repository": {
     "type": "git",
-    "url": "https://github.com/databricks/zerobus-sdk-ts.git"
+    "url": "https://github.com/databricks/zerobus-sdk"
   },
   "license": "Databricks License",
   "files": [
@@ -34,12 +34,13 @@
   "napi": {
     "name": "zerobus-ingest-sdk",
     "triples": {
-      "defaults": true,
+      "defaults": false,
       "additional": [
-        "x86_64-unknown-linux-musl",
+        "x86_64-unknown-linux-gnu",
         "aarch64-unknown-linux-gnu",
-        "aarch64-apple-darwin",
-        "aarch64-unknown-linux-musl"
+        "x86_64-pc-windows-msvc",
+        "x86_64-apple-darwin",
+        "aarch64-apple-darwin"
       ]
     }
   },
@@ -49,7 +50,9 @@
   "scripts": {
     "artifacts": "napi artifacts",
     "build": "napi build --platform --release",
+    "build:arrow": "napi build --platform --release --features arrow-flight",
     "build:debug": "napi build --platform",
+    "build:debug:arrow": "napi build --platform --features arrow-flight",
     "build:proto": "mkdir -p examples/generated && pbjs -t static-module -w commonjs -o examples/generated/air_quality.js schemas/air_quality.proto && pbts -o examples/generated/air_quality.d.ts examples/generated/air_quality.js && protoc --descriptor_set_out=schemas/air_quality_descriptor.pb --include_imports schemas/air_quality.proto",
     "prepublishOnly": "napi prepublish -t npm",
     "test": "tsx --test test/unit.test.ts test/integration.test.ts",
@@ -57,21 +60,29 @@
     "test:integration": "tsx --test test/integration.test.ts",
     "universal": "napi universal",
     "version": "napi version",
-    "example:json": "tsx examples/json.ts",
-    "example:proto": "tsx examples/proto.ts",
-    "example:parallel": "tsx examples/parallel_streams.ts"
+    "example:json:single": "tsx examples/json/single.ts",
+    "example:json:batch": "tsx examples/json/batch.ts",
+    "example:proto:single": "tsx examples/proto/single.ts",
+    "example:proto:batch": "tsx examples/proto/batch.ts",
+    "example:arrow:single": "tsx examples/arrow/single.ts",
+    "example:arrow:batch": "tsx examples/arrow/batch.ts"
   },
   "peerDependencies": {
-    "protobufjs": "^7.0.0"
+    "protobufjs": "^7.0.0",
+    "apache-arrow": "^56.0.0"
   },
   "peerDependenciesMeta": {
     "protobufjs": {
       "optional": true
+    },
+    "apache-arrow": {
+      "optional": true
     }
   },
   "devDependencies": {
     "@napi-rs/cli": "^2.18.0",
     "@types/node": "^20.0.0",
+    "apache-arrow": "^18.1.0",
     "dotenv": "^17.2.3",
     "protobufjs": "^7.5.4",
     "protobufjs-cli": "^2.0.0",
@@ -82,12 +93,10 @@
     "glob": "^10.0.0"
   },
   "optionalDependencies": {
-    "@databricks/zerobus-ingest-sdk-win32-x64-msvc": "0.1.1",
-    "@databricks/zerobus-ingest-sdk-darwin-x64": "0.1.1",
-    "@databricks/zerobus-ingest-sdk-linux-x64-gnu": "0.1.1",
-    "@databricks/zerobus-ingest-sdk-linux-x64-musl": "0.1.1",
-    "@databricks/zerobus-ingest-sdk-linux-arm64-gnu": "0.1.1",
-    "@databricks/zerobus-ingest-sdk-darwin-arm64": "0.1.1",
-    "@databricks/zerobus-ingest-sdk-linux-arm64-musl": "0.1.1"
+    "@databricks/zerobus-ingest-sdk-linux-x64-gnu": "0.3.0",
+    "@databricks/zerobus-ingest-sdk-linux-arm64-gnu": "0.3.0",
+    "@databricks/zerobus-ingest-sdk-win32-x64-msvc": "0.3.0",
+    "@databricks/zerobus-ingest-sdk-darwin-x64": "0.3.0",
+    "@databricks/zerobus-ingest-sdk-darwin-arm64": "0.3.0"
   }
 }