@databricks/zerobus-ingest-sdk 1.1.0 → 1.3.0

package/index.d.ts

@@ -22,7 +22,7 @@ export const enum RecordType {
 export interface StreamConfigurationOptions {
   /**
    * Maximum number of unacknowledged requests that can be in flight.
-   * Default: 10,000
+   * Default: 1,000,000
    */
   maxInflightRequests?: number
   /**
@@ -83,7 +83,8 @@ export interface TableProperties {
   tableName: string
   /**
    * Optional Protocol Buffer descriptor as a base64-encoded string.
-   * If not provided, JSON encoding will be used.
+   * Omitting this does not select JSON. The stream defaults to Protocol Buffers
+   * unless `record_type` is set to JSON.
    */
   descriptorProto?: string
 }
@@ -91,12 +92,143 @@ export interface TableProperties {
  * JavaScript headers provider callback wrapper.
  *
  * Allows TypeScript code to provide custom authentication headers
- * by implementing a getHeaders() function.
+ * by implementing a getHeadersCallback() function.
  */
 export interface JsHeadersProvider {
-  /** JavaScript function: () => Promise<Array<[string, string]>> */
+  /** JavaScript function: () => Array<[string, string]> */
   getHeadersCallback: (...args: any[]) => any
 }
+export interface ZerobusSdkOptions {
+  /** Identifier appended to the `user-agent` header */
+  applicationName?: string
+}
+/**
+ * IPC compression type for Arrow Flight streams.
+ *
+ */
+export const enum IpcCompressionType {
+  /** LZ4 frame compression - fast compression with moderate ratio */
+  Lz4Frame = 0,
+  /** Zstandard compression - better compression ratio, slightly slower */
+  Zstd = 1
+}
+/**
+ * Configuration options for Arrow Flight streams.
+ *
+ */
+export interface ArrowStreamConfigurationOptions {
+  /**
+   * Maximum number of batches that can be in-flight (sent but not acknowledged).
+   * Default: 1,000
+   */
+  maxInflightBatches?: number
+  /**
+   * Whether to enable automatic stream recovery on failure.
+   * Default: true
+   */
+  recovery?: boolean
+  /**
+   * Timeout for recovery operations in milliseconds.
+   * Default: 15,000 (15 seconds)
+   */
+  recoveryTimeoutMs?: number
+  /**
+   * Delay between recovery retry attempts in milliseconds.
+   * Default: 2,000 (2 seconds)
+   */
+  recoveryBackoffMs?: number
+  /**
+   * Maximum number of recovery attempts before giving up.
+   * Default: 4
+   */
+  recoveryRetries?: number
+  /**
+   * Timeout waiting for server acknowledgments in milliseconds.
+   * Default: 60,000 (1 minute)
+   */
+  serverLackOfAckTimeoutMs?: number
+  /**
+   * Timeout for flush operations in milliseconds.
+   * Default: 300,000 (5 minutes)
+   */
+  flushTimeoutMs?: number
+  /**
+   * Timeout for connection establishment in milliseconds.
+   * Default: 30,000 (30 seconds)
+   */
+  connectionTimeoutMs?: number
+  /** Optional IPC compression type (0 = LZ4Frame, 1 = Zstd, undefined = no compression) */
+  ipcCompression?: number
+}
+/**
+ * Arrow data type enum for schema definition.
+ *
+ */
+export const enum ArrowDataType {
+  /** Boolean type */
+  Boolean = 0,
+  /** Signed 8-bit integer */
+  Int8 = 1,
+  /** Signed 16-bit integer */
+  Int16 = 2,
+  /** Signed 32-bit integer */
+  Int32 = 3,
+  /** Signed 64-bit integer */
+  Int64 = 4,
+  /** Unsigned 8-bit integer */
+  UInt8 = 5,
+  /** Unsigned 16-bit integer */
+  UInt16 = 6,
+  /** Unsigned 32-bit integer */
+  UInt32 = 7,
+  /** Unsigned 64-bit integer */
+  UInt64 = 8,
+  /** 32-bit floating point */
+  Float32 = 9,
+  /** 64-bit floating point */
+  Float64 = 10,
+  /** UTF-8 encoded string */
+  Utf8 = 11,
+  /** Large UTF-8 encoded string (64-bit offsets) */
+  LargeUtf8 = 12,
+  /** Binary data */
+  Binary = 13,
+  /** Large binary data (64-bit offsets) */
+  LargeBinary = 14,
+  /** Date (32-bit days since epoch) */
+  Date32 = 15,
+  /** Date (64-bit milliseconds since epoch) */
+  Date64 = 16,
+  /** Timestamp with microsecond precision (UTC) */
+  TimestampMicros = 17,
+  /** Timestamp with nanosecond precision (UTC) */
+  TimestampNanos = 18
+}
+/**
+ * Arrow field definition for schema.
+ *
+ */
+export interface ArrowField {
+  /** Field name */
+  name: string
+  /** Field data type (ArrowDataType enum value) */
+  dataType: number
+  /** Whether the field is nullable */
+  nullable?: boolean
+}
+/**
+ * Properties of the target Delta table for Arrow Flight ingestion.
+ *
+ * Unlike `TableProperties` which uses Protocol Buffers, Arrow Flight streams
+ * require an Arrow schema definition.
+ *
+ */
+export interface ArrowTableProperties {
+  /** Full table name in Unity Catalog (e.g., "catalog.schema.table") */
+  tableName: string
+  /** Arrow schema fields */
+  schemaFields: Array<ArrowField>
+}
 /**
  * Custom error type for Zerobus operations.
  *
@@ -112,15 +244,16 @@ export declare class ZerobusError {
 /**
  * A stream for ingesting data into a Databricks Delta table.
  *
- * The stream manages a bidirectional gRPC connection, handles acknowledgments,
- * and provides automatic recovery on transient failures.
+ * The stream manages JSON or Protocol Buffer ingestion over a bidirectional
+ * gRPC connection, handles acknowledgments, and provides automatic recovery
+ * on transient failures.
  *
  * # Example
  *
  * ```typescript
  * const stream = await sdk.createStream(tableProps, clientId, clientSecret, options);
- * const ackPromise = await stream.ingestRecord(Buffer.from([1, 2, 3]));
- * const offset = await ackPromise;
+ * const offset = await stream.ingestRecordOffset(Buffer.from([1, 2, 3]));
+ * await stream.flush();
  * await stream.close();
  * ```
  */
@@ -210,6 +343,13 @@ export declare class ZerobusStream {
    * This is the recommended API for high-throughput scenarios where you want to
    * decouple record ingestion from acknowledgment tracking.
    *
+   * **Acknowledgments:** the idiomatic flow is to ingest in a loop and then `flush()`
+   * once to confirm everything queued so far. The returned offset, together with
+   * `waitForOffset()`, lets you confirm a specific record when you need it (acks are
+   * ordered, so the last offset confirms the whole run) — prefer `flush()` for bulk.
+   * Avoid calling `waitForOffset()` after every record in a tight loop, since that
+   * limits throughput to one record per round-trip.
+   *
    * # Arguments
    *
    * * `payload` - The record data (Buffer, string, protobuf message, or plain object)
@@ -222,11 +362,14 @@ export declare class ZerobusStream {
    * # Example
    *
    * ```typescript
-   * // Promise resolves immediately with offset (before server ack)
-   * const offset1 = await stream.ingestRecordOffset(record1);
-   * const offset2 = await stream.ingestRecordOffset(record2);
-   * // Wait for both to be acknowledged
-   * await stream.waitForOffset(offset2);
+   * // High-throughput pattern: ingest in a loop, wait once at the end.
+   * let lastOffset: bigint | null = null;
+   * for (const record of records) {
+   *   lastOffset = await stream.ingestRecordOffset(record); // resolves on queue, no round-trip
+   * }
+   * // The ack watermark is monotonic: waiting on the last offset confirms all prior records.
+   * if (lastOffset !== null) await stream.waitForOffset(lastOffset);
+   * // Or simply: await stream.flush();
    * ```
    */
   ingestRecordOffset(payload: unknown): Promise<bigint>
@@ -237,6 +380,12 @@ export declare class ZerobusStream {
    * the batch is queued, without waiting for server acknowledgment. Use
    * `waitForOffset()` to wait for acknowledgment when needed.
    *
+   * **Acknowledgments:** the idiomatic flow is to ingest your batches in a loop and
+   * then `flush()` once to confirm. The returned offset, together with `waitForOffset()`,
+   * confirms a specific batch when you need it (acks are ordered, so the last offset
+   * confirms the whole run) — prefer `flush()` for bulk. Avoid calling `waitForOffset()`
+   * after every batch in a tight loop, since that limits throughput to one round-trip per batch.
+   *
    * # Arguments
    *
    * * `records` - Array of record data
@@ -249,20 +398,26 @@ export declare class ZerobusStream {
    * # Example
    *
    * ```typescript
-   * // Promise resolves immediately with offset (before server ack)
-   * const offset = await stream.ingestRecordsOffset(batch);
-   * if (offset !== null) {
-   *   await stream.waitForOffset(offset);
+   * // Ingest many batches without waiting, then flush once.
+   * let lastOffset = null;
+   * for (const batch of batches) {
+   *   const offset = await stream.ingestRecordsOffset(batch); // resolves on queue
+   *   if (offset !== null) lastOffset = offset;
    * }
+   * if (lastOffset !== null) await stream.waitForOffset(lastOffset);
+   * // Or simply: await stream.flush();
    * ```
    */
   ingestRecordsOffset(records: Array<unknown>): Promise<bigint | null>
   /**
    * Waits for a specific offset to be acknowledged by the server.
    *
-   * Use this method with `ingestRecordOffset()` and `ingestRecordsOffset()` to
-   * selectively wait for acknowledgments. This allows you to ingest many records
-   * quickly and then wait only for specific offsets when needed.
+   * Use this method with `ingestRecordOffset()` and `ingestRecordsOffset()` to confirm
+   * a specific record before continuing. Acks are ordered, so waiting on the LAST offset
+   * confirms every prior record too — you never need to wait on intermediate offsets.
+   * For confirming a bulk run, `flush()` is usually simpler; reach for `waitForOffset()`
+   * when one particular record must be confirmed. Avoid calling it after every record in
+   * a tight loop, since that limits throughput to one record per round-trip.
    *
    * # Arguments
    *
@@ -276,12 +431,12 @@ export declare class ZerobusStream {
    * # Example
    *
    * ```typescript
-   * const offsets = [];
+   * let lastOffset: bigint | null = null;
    * for (const record of records) {
-   *   offsets.push(await stream.ingestRecordOffset(record));
+   *   lastOffset = await stream.ingestRecordOffset(record); // no per-record wait
    * }
-   * // Wait for the last offset (implies all previous are also acknowledged)
-   * await stream.waitForOffset(offsets[offsets.length - 1]);
+   * // Wait for the last offset only (implies all previous are also acknowledged).
+   * if (lastOffset !== null) await stream.waitForOffset(lastOffset);
    * ```
    */
   waitForOffset(offsetId: bigint): Promise<void>
@@ -291,6 +446,11 @@ export declare class ZerobusStream {
    * This method ensures all previously ingested records have been sent to the server
    * and acknowledged. It's useful for checkpointing or ensuring data durability.
    *
+   * This is the idiomatic way to confirm records ingested via `ingestRecordOffset()` /
+   * `ingestRecordsOffset()`: ingest in a loop, then `flush()` once (for a bounded batch,
+   * or periodically for a long-running stream). It resolves once everything queued so
+   * far is acknowledged.
+   *
    * # Errors
    *
    * - Timeout errors if flush takes longer than configured timeout
@@ -340,15 +500,12 @@ export declare class ZerobusStream {
    *
    * ```typescript
    * try {
-   *   await stream.ingestRecords(batch1);
-   *   await stream.ingestRecords(batch2);
+   *   await stream.ingestRecordsOffset(batch1);
+   *   await stream.ingestRecordsOffset(batch2);
+   *   await stream.flush();
    * } catch (error) {
    *   const unackedBatches = await stream.getUnackedBatches();
-   *
-   *   // Re-ingest with new stream
-   *   for (const batch of unackedBatches) {
-   *     await newStream.ingestRecords(batch);
-   *   }
+   *   console.log(`Batches available for recovery: ${unackedBatches.length}`);
    * }
    * ```
    */
@@ -364,7 +521,8 @@ export declare class ZerobusStream {
  * ```typescript
  * const sdk = new ZerobusSdk(
  *   "https://workspace-id.zerobus.region.cloud.databricks.com",
- *   "https://workspace.cloud.databricks.com"
+ *   "https://workspace.cloud.databricks.com",
+ *   { applicationName: "my-app/1.0" }
  * );
  *
  * const stream = await sdk.createStream(
@@ -384,17 +542,19 @@ export declare class ZerobusSdk {
    *   (e.g., "https://workspace-id.zerobus.region.cloud.databricks.com")
    * * `unity_catalog_url` - The Unity Catalog endpoint URL
    *   (e.g., "https://workspace.cloud.databricks.com")
+   * * `options` - Optional SDK configuration (see `ZerobusSdkOptions`),
+   *   including `applicationName` for server-side attribution.
    *
    * # Errors
    *
    * - Invalid endpoint URLs
    * - Failed to extract workspace ID from the endpoint
    */
-  constructor(zerobusEndpoint: string, unityCatalogUrl: string)
+  constructor(zerobusEndpoint: string, unityCatalogUrl: string, options?: ZerobusSdkOptions | undefined | null)
   /**
    * Creates a new ingestion stream to a Delta table.
    *
-   * This method establishes a bidirectional gRPC connection to the Zerobus service
+   * This method opens a JSON or Protocol Buffer stream to the Zerobus service
    * and prepares it for data ingestion. By default, it uses OAuth 2.0 Client Credentials
    * authentication. For custom authentication (e.g., Personal Access Tokens), provide
    * a custom headers_provider.
@@ -436,7 +596,7 @@ export declare class ZerobusSdk {
    *   "", // ignored
    *   undefined,
    *   {
-   *     getHeadersCallback: async () => [
+   *     getHeadersCallback: () => [
    *       ["authorization", `Bearer ${myToken}`],
    *       ["x-databricks-zerobus-table-name", tableName]
    *     ]
@@ -456,7 +616,8 @@ export declare class ZerobusSdk {
    *
    * # Arguments
    *
-   * * `stream` - The failed or closed stream to recreate
+   * * `stream` - The terminally failed stream to recreate. The TypeScript wrapper
+   *   must not have been closed because `close()` releases its native handle.
    *
    * # Returns
    *
@@ -472,14 +633,167 @@ export declare class ZerobusSdk {
    *
    * ```typescript
    * try {
-   *   await stream.ingestRecords(batch);
+   *   await stream.ingestRecordsOffset(batch);
+   *   await stream.flush();
    * } catch (error) {
-   *   await stream.close();
-   *   // Recreate stream with all unacked batches re-ingested
-   *   const newStream = await sdk.recreateStream(stream);
-   *   // Continue ingesting with newStream
+   *   try {
+   *     const newStream = await sdk.recreateStream(stream);
+   *     try {
+   *       await newStream.flush();
+   *     } finally {
+   *       await newStream.close();
+   *     }
+   *   } finally {
+   *     try {
+   *       await stream.close();
+   *     } catch (closeError) {
+   *       console.error("Failed stream released:", closeError);
+   *     }
+   *   }
    * }
    * ```
    */
   recreateStream(stream: ZerobusStream): Promise<ZerobusStream>
+  /**
+   * Creates a new Arrow Flight stream to a Delta table.
+   *
+   * This method establishes an Arrow Flight connection to the Zerobus service
+   * for high-performance columnar data ingestion.
+   *
+   * # Arguments
+   *
+   * * `table_properties` - Properties of the target table including name and Arrow schema
+   * * `client_id` - OAuth 2.0 client ID
+   * * `client_secret` - OAuth 2.0 client secret
+   * * `options` - Optional stream configuration
+   *
+   * # Returns
+   *
+   * A Promise that resolves to a ZerobusArrowStream ready for data ingestion.
+   *
+   * # Example
+   *
+   * ```typescript
+   * const tableProps = {
+   *   tableName: 'catalog.schema.table',
+   *   schemaFields: [
+   *     { name: 'device_name', dataType: ArrowDataType.Utf8 },
+   *     { name: 'temp', dataType: ArrowDataType.Int32 },
+   *     { name: 'humidity', dataType: ArrowDataType.Int64 }
+   *   ]
+   * };
+   *
+   * const arrowStream = await sdk.createArrowStream(
+   *   tableProps,
+   *   clientId,
+   *   clientSecret,
+   *   { maxInflightBatches: 100 }
+   * );
+   * ```
+   */
+  createArrowStream(tableProperties: ArrowTableProperties, clientId: string, clientSecret: string, options?: ArrowStreamConfigurationOptions | undefined | null): Promise<ZerobusArrowStream>
+  /**
+   * Recreates an Arrow stream with the same configuration and re-ingests unacknowledged batches.
+   *
+   * # Arguments
+   *
+   * * `stream` - The terminally failed Arrow stream to recreate. The TypeScript wrapper
+   *   must not have been closed because `close()` releases its native handle.
+   *
+   * # Returns
+   *
+   * A Promise that resolves to a new ZerobusArrowStream with all unacknowledged batches re-ingested.
+   */
+  recreateArrowStream(stream: ZerobusArrowStream): Promise<ZerobusArrowStream>
+}
+/**
+ * An Arrow Flight stream for ingesting Arrow RecordBatches into a Delta table.
+ *
+ * This stream provides a high-performance interface for streaming Arrow data
+ * to Databricks Delta tables using the Arrow Flight protocol.
+ *
+ * # Lifecycle
+ *
+ * 1. Create a stream via `sdk.createArrowStream()`
+ * 2. Ingest Arrow IPC buffers with `ingestBatch()`
+ * 3. Use `waitForOffset()` to wait for acknowledgments
+ * 4. Call `flush()` to ensure all batches are persisted
+ * 5. Close the stream with `close()`
+ *
+ * # Example
+ *
+ * ```typescript
+ * import { tableToIPC } from 'apache-arrow';
+ *
+ * const arrowStream = await sdk.createArrowStream(
+ *   arrowTableProps,
+ *   clientId,
+ *   clientSecret,
+ *   options
+ * );
+ *
+ * const ipcBuffer = tableToIPC(arrowTable, 'stream');
+ * const offset = await arrowStream.ingestBatch(Buffer.from(ipcBuffer));
+ * await arrowStream.waitForOffset(offset);
+ * await arrowStream.close();
+ * ```
+ */
+export declare class ZerobusArrowStream {
+  /**
+   * Ingests a single Arrow IPC buffer into the stream.
+   *
+   * The buffer should be an Arrow IPC stream format containing one or more RecordBatches.
+   * You can create this using `tableToIPC(table, 'stream')` from the apache-arrow package.
+   *
+   * # Arguments
+   *
+   * * `ipc_buffer` - Arrow IPC stream format buffer
+   *
+   * # Returns
+   *
+   * The offset ID (bigint) assigned to this batch.
+   *
+   * # Example
+   *
+   * ```typescript
+   * const table = tableFromArrays({
+   *   device_name: ['sensor-1'],
+   *   temp: [25],
+   *   humidity: [60]
+   * });
+   * const ipcBuffer = tableToIPC(table, 'stream');
+   * const offset = await stream.ingestBatch(Buffer.from(ipcBuffer));
+   * await stream.waitForOffset(offset);
+   * ```
+   */
+  ingestBatch(ipcBuffer: Buffer): Promise<bigint>
+  /**
+   * Waits for a specific offset to be acknowledged by the server.
+   *
+   * Use this method with `ingestBatch()` to selectively wait for acknowledgments.
+   *
+   * # Arguments
+   *
+   * * `offset_id` - The offset ID to wait for (returned by ingestBatch)
+   */
+  waitForOffset(offsetId: bigint): Promise<void>
+  /** Flushes all pending batches and waits for acknowledgments. */
+  flush(): Promise<void>
+  /** Closes the stream gracefully. */
+  close(): Promise<void>
+  /** Returns whether the stream has been closed. */
+  get isClosed(): boolean
+  /** Returns the table name for this stream. */
+  get tableName(): string
+  /**
+   * Gets unacknowledged batches as Arrow IPC buffers.
+   *
+   * This method should only be called after a stream failure to retrieve batches
+   * that were sent but not acknowledged. These can be re-ingested into a new stream.
+   *
+   * # Returns
+   *
+   * An array of Buffers containing the unacknowledged batches in Arrow IPC format.
+   */
+  getUnackedBatches(): Promise<Array<Buffer>>
 }

package/index.js

@@ -310,9 +310,12 @@ if (!nativeBinding) {
   throw new Error(`Failed to load native binding`)
 }
 
-const { RecordType, ZerobusError, ZerobusStream, ZerobusSdk } = nativeBinding
+const { RecordType, ZerobusError, ZerobusStream, ZerobusSdk, IpcCompressionType, ArrowDataType, ZerobusArrowStream } = nativeBinding
 
 module.exports.RecordType = RecordType
 module.exports.ZerobusError = ZerobusError
 module.exports.ZerobusStream = ZerobusStream
 module.exports.ZerobusSdk = ZerobusSdk
+module.exports.IpcCompressionType = IpcCompressionType
+module.exports.ArrowDataType = ArrowDataType
+module.exports.ZerobusArrowStream = ZerobusArrowStream

package/package.json

@@ -1,9 +1,19 @@
 {
   "name": "@databricks/zerobus-ingest-sdk",
-  "version": "1.1.0",
+  "version": "1.3.0",
   "description": "TypeScript/Node.js SDK for streaming data ingestion into Databricks Delta tables using Zerobus",
   "main": "index.js",
   "types": "index.d.ts",
+  "typesVersions": {
+    "*": {
+      "utils/descriptor": [
+        "utils/descriptor.d.ts"
+      ],
+      "utils/descriptor.js": [
+        "utils/descriptor.d.ts"
+      ]
+    }
+  },
   "keywords": [
     "databricks",
     "delta",
@@ -66,20 +76,19 @@
     "example:arrow": "tsx examples/arrow/batch.ts"
   },
   "optionalDependencies": {
-    "@databricks/zerobus-ingest-sdk-linux-x64-gnu": "1.1.0",
-    "@databricks/zerobus-ingest-sdk-linux-arm64-gnu": "1.1.0",
-    "@databricks/zerobus-ingest-sdk-win32-x64-msvc": "1.1.0",
-    "@databricks/zerobus-ingest-sdk-darwin-x64": "1.1.0",
-    "@databricks/zerobus-ingest-sdk-darwin-arm64": "1.1.0"
+    "@databricks/zerobus-ingest-sdk-linux-x64-gnu": "1.3.0",
+    "@databricks/zerobus-ingest-sdk-linux-arm64-gnu": "1.3.0",
+    "@databricks/zerobus-ingest-sdk-win32-x64-msvc": "1.3.0",
+    "@databricks/zerobus-ingest-sdk-darwin-x64": "1.3.0",
+    "@databricks/zerobus-ingest-sdk-darwin-arm64": "1.3.0"
+  },
+  "dependencies": {
+    "protobufjs": "^8.7.1"
   },
   "peerDependencies": {
-    "protobufjs": "^7.0.0",
     "apache-arrow": "^18.0.0"
   },
   "peerDependenciesMeta": {
-    "protobufjs": {
-      "optional": true
-    },
     "apache-arrow": {
       "optional": true
     }
@@ -89,12 +98,18 @@
     "@types/node": "^20.0.0",
     "apache-arrow": "^18.1.0",
     "dotenv": "^17.2.3",
-    "protobufjs": "^7.5.4",
-    "protobufjs-cli": "^2.0.0",
+    "protobufjs-cli": "^2.6.1",
     "tsx": "^4.21.0",
     "typescript": "^5.3.0"
   },
   "overrides": {
-    "glob": "^10.0.0"
+    "@protobufjs/utf8": "^1.1.1",
+    "brace-expansion": "^2.1.2",
+    "glob": "^10.0.0",
+    "lodash": "^4.18.0",
+    "markdown-it": "^14.2.0",
+    "minimatch": "^9.0.7",
+    "tmp": "^0.2.6",
+    "underscore": "^1.13.8"
   }
 }