@databricks/zerobus-ingest-sdk 1.0.2 → 1.2.0

package/index.d.ts

@@ -22,7 +22,7 @@ export const enum RecordType {
 export interface StreamConfigurationOptions {
   /**
    * Maximum number of unacknowledged requests that can be in flight.
-   * Default: 10,000
+   * Default: 1,000,000
    */
   maxInflightRequests?: number
   /**
@@ -83,7 +83,8 @@ export interface TableProperties {
   tableName: string
   /**
    * Optional Protocol Buffer descriptor as a base64-encoded string.
-   * If not provided, JSON encoding will be used.
+   * Omitting this does not select JSON. The stream defaults to Protocol Buffers
+   * unless `record_type` is set to JSON.
    */
   descriptorProto?: string
 }
@@ -91,12 +92,16 @@ export interface TableProperties {
  * JavaScript headers provider callback wrapper.
  *
  * Allows TypeScript code to provide custom authentication headers
- * by implementing a getHeaders() function.
+ * by implementing a getHeadersCallback() function.
  */
 export interface JsHeadersProvider {
-  /** JavaScript function: () => Promise<Array<[string, string]>> */
+  /** JavaScript function: () => Array<[string, string]> */
   getHeadersCallback: (...args: any[]) => any
 }
+export interface ZerobusSdkOptions {
+  /** Identifier appended to the `user-agent` header */
+  applicationName?: string
+}
 /**
  * Custom error type for Zerobus operations.
  *
@@ -119,8 +124,8 @@ export declare class ZerobusError {
  *
  * ```typescript
  * const stream = await sdk.createStream(tableProps, clientId, clientSecret, options);
- * const ackPromise = await stream.ingestRecord(Buffer.from([1, 2, 3]));
- * const offset = await ackPromise;
+ * const offset = await stream.ingestRecordOffset(Buffer.from([1, 2, 3]));
+ * await stream.flush();
  * await stream.close();
  * ```
  */
@@ -210,6 +215,13 @@ export declare class ZerobusStream {
    * This is the recommended API for high-throughput scenarios where you want to
    * decouple record ingestion from acknowledgment tracking.
    *
+   * **Acknowledgments:** the idiomatic flow is to ingest in a loop and then `flush()`
+   * once to confirm everything queued so far. The returned offset, together with
+   * `waitForOffset()`, lets you confirm a specific record when you need it (acks are
+   * ordered, so the last offset confirms the whole run) — prefer `flush()` for bulk.
+   * Avoid calling `waitForOffset()` after every record in a tight loop, since that
+   * limits throughput to one record per round-trip.
+   *
    * # Arguments
    *
    * * `payload` - The record data (Buffer, string, protobuf message, or plain object)
@@ -222,11 +234,14 @@ export declare class ZerobusStream {
    * # Example
    *
    * ```typescript
-   * // Promise resolves immediately with offset (before server ack)
-   * const offset1 = await stream.ingestRecordOffset(record1);
-   * const offset2 = await stream.ingestRecordOffset(record2);
-   * // Wait for both to be acknowledged
-   * await stream.waitForOffset(offset2);
+   * // High-throughput pattern: ingest in a loop, wait once at the end.
+   * let lastOffset: bigint | null = null;
+   * for (const record of records) {
+   *   lastOffset = await stream.ingestRecordOffset(record); // resolves on queue, no round-trip
+   * }
+   * // The ack watermark is monotonic: waiting on the last offset confirms all prior records.
+   * if (lastOffset !== null) await stream.waitForOffset(lastOffset);
+   * // Or simply: await stream.flush();
    * ```
    */
   ingestRecordOffset(payload: unknown): Promise<bigint>
@@ -237,6 +252,12 @@ export declare class ZerobusStream {
    * the batch is queued, without waiting for server acknowledgment. Use
    * `waitForOffset()` to wait for acknowledgment when needed.
    *
+   * **Acknowledgments:** the idiomatic flow is to ingest your batches in a loop and
+   * then `flush()` once to confirm. The returned offset, together with `waitForOffset()`,
+   * confirms a specific batch when you need it (acks are ordered, so the last offset
+   * confirms the whole run) — prefer `flush()` for bulk. Avoid calling `waitForOffset()`
+   * after every batch in a tight loop, since that limits throughput to one round-trip per batch.
+   *
    * # Arguments
    *
    * * `records` - Array of record data
@@ -249,20 +270,26 @@ export declare class ZerobusStream {
    * # Example
    *
    * ```typescript
-   * // Promise resolves immediately with offset (before server ack)
-   * const offset = await stream.ingestRecordsOffset(batch);
-   * if (offset !== null) {
-   *   await stream.waitForOffset(offset);
+   * // Ingest many batches without waiting, then flush once.
+   * let lastOffset = null;
+   * for (const batch of batches) {
+   *   const offset = await stream.ingestRecordsOffset(batch); // resolves on queue
+   *   if (offset !== null) lastOffset = offset;
    * }
+   * if (lastOffset !== null) await stream.waitForOffset(lastOffset);
+   * // Or simply: await stream.flush();
    * ```
    */
   ingestRecordsOffset(records: Array<unknown>): Promise<bigint | null>
   /**
    * Waits for a specific offset to be acknowledged by the server.
    *
-   * Use this method with `ingestRecordOffset()` and `ingestRecordsOffset()` to
-   * selectively wait for acknowledgments. This allows you to ingest many records
-   * quickly and then wait only for specific offsets when needed.
+   * Use this method with `ingestRecordOffset()` and `ingestRecordsOffset()` to confirm
+   * a specific record before continuing. Acks are ordered, so waiting on the LAST offset
+   * confirms every prior record too — you never need to wait on intermediate offsets.
+   * For confirming a bulk run, `flush()` is usually simpler; reach for `waitForOffset()`
+   * when one particular record must be confirmed. Avoid calling it after every record in
+   * a tight loop, since that limits throughput to one record per round-trip.
    *
    * # Arguments
    *
@@ -276,12 +303,12 @@ export declare class ZerobusStream {
    * # Example
    *
    * ```typescript
-   * const offsets = [];
+   * let lastOffset: bigint | null = null;
    * for (const record of records) {
-   *   offsets.push(await stream.ingestRecordOffset(record));
+   *   lastOffset = await stream.ingestRecordOffset(record); // no per-record wait
    * }
-   * // Wait for the last offset (implies all previous are also acknowledged)
-   * await stream.waitForOffset(offsets[offsets.length - 1]);
+   * // Wait for the last offset only (implies all previous are also acknowledged).
+   * if (lastOffset !== null) await stream.waitForOffset(lastOffset);
    * ```
    */
   waitForOffset(offsetId: bigint): Promise<void>
@@ -291,6 +318,11 @@ export declare class ZerobusStream {
    * This method ensures all previously ingested records have been sent to the server
    * and acknowledged. It's useful for checkpointing or ensuring data durability.
    *
+   * This is the idiomatic way to confirm records ingested via `ingestRecordOffset()` /
+   * `ingestRecordsOffset()`: ingest in a loop, then `flush()` once (for a bounded batch,
+   * or periodically for a long-running stream). It resolves once everything queued so
+   * far is acknowledged.
+   *
    * # Errors
    *
    * - Timeout errors if flush takes longer than configured timeout
@@ -340,15 +372,12 @@ export declare class ZerobusStream {
    *
    * ```typescript
    * try {
-   *   await stream.ingestRecords(batch1);
-   *   await stream.ingestRecords(batch2);
+   *   await stream.ingestRecordsOffset(batch1);
+   *   await stream.ingestRecordsOffset(batch2);
+   *   await stream.flush();
    * } catch (error) {
    *   const unackedBatches = await stream.getUnackedBatches();
-   *
-   *   // Re-ingest with new stream
-   *   for (const batch of unackedBatches) {
-   *     await newStream.ingestRecords(batch);
-   *   }
+   *   console.log(`Batches available for recovery: ${unackedBatches.length}`);
    * }
    * ```
    */
@@ -364,7 +393,8 @@ export declare class ZerobusStream {
  * ```typescript
  * const sdk = new ZerobusSdk(
  *   "https://workspace-id.zerobus.region.cloud.databricks.com",
- *   "https://workspace.cloud.databricks.com"
+ *   "https://workspace.cloud.databricks.com",
+ *   { applicationName: "my-app/1.0" }
  * );
  *
  * const stream = await sdk.createStream(
@@ -384,13 +414,15 @@ export declare class ZerobusSdk {
    *   (e.g., "https://workspace-id.zerobus.region.cloud.databricks.com")
    * * `unity_catalog_url` - The Unity Catalog endpoint URL
    *   (e.g., "https://workspace.cloud.databricks.com")
+   * * `options` - Optional SDK configuration (see `ZerobusSdkOptions`),
+   *   including `applicationName` for server-side attribution.
    *
    * # Errors
    *
    * - Invalid endpoint URLs
    * - Failed to extract workspace ID from the endpoint
    */
-  constructor(zerobusEndpoint: string, unityCatalogUrl: string)
+  constructor(zerobusEndpoint: string, unityCatalogUrl: string, options?: ZerobusSdkOptions | undefined | null)
   /**
    * Creates a new ingestion stream to a Delta table.
    *
@@ -436,7 +468,7 @@ export declare class ZerobusSdk {
    *   "", // ignored
    *   undefined,
    *   {
-   *     getHeadersCallback: async () => [
+   *     getHeadersCallback: () => [
    *       ["authorization", `Bearer ${myToken}`],
    *       ["x-databricks-zerobus-table-name", tableName]
    *     ]
@@ -456,7 +488,8 @@ export declare class ZerobusSdk {
    *
    * # Arguments
    *
-   * * `stream` - The failed or closed stream to recreate
+   * * `stream` - The terminally failed stream to recreate. The TypeScript wrapper
+   *   must not have been closed because `close()` releases its native handle.
    *
    * # Returns
    *
@@ -472,12 +505,23 @@ export declare class ZerobusSdk {
    *
    * ```typescript
    * try {
-   *   await stream.ingestRecords(batch);
+   *   await stream.ingestRecordsOffset(batch);
+   *   await stream.flush();
    * } catch (error) {
-   *   await stream.close();
-   *   // Recreate stream with all unacked batches re-ingested
-   *   const newStream = await sdk.recreateStream(stream);
-   *   // Continue ingesting with newStream
+   *   try {
+   *     const newStream = await sdk.recreateStream(stream);
+   *     try {
+   *       await newStream.flush();
+   *     } finally {
+   *       await newStream.close();
+   *     }
+   *   } finally {
+   *     try {
+   *       await stream.close();
+   *     } catch (closeError) {
+   *       console.error("Failed stream released:", closeError);
+   *     }
+   *   }
    * }
    * ```
    */

package/package.json

@@ -1,9 +1,19 @@
 {
   "name": "@databricks/zerobus-ingest-sdk",
-  "version": "1.0.2",
+  "version": "1.2.0",
   "description": "TypeScript/Node.js SDK for streaming data ingestion into Databricks Delta tables using Zerobus",
   "main": "index.js",
   "types": "index.d.ts",
+  "typesVersions": {
+    "*": {
+      "utils/descriptor": [
+        "utils/descriptor.d.ts"
+      ],
+      "utils/descriptor.js": [
+        "utils/descriptor.d.ts"
+      ]
+    }
+  },
   "keywords": [
     "databricks",
     "delta",
@@ -63,22 +73,22 @@
     "example:json:batch": "tsx examples/json/batch.ts",
     "example:proto:single": "tsx examples/proto/single.ts",
     "example:proto:batch": "tsx examples/proto/batch.ts",
-    "example:arrow:single": "tsx examples/arrow/single.ts",
-    "example:arrow:batch": "tsx examples/arrow/batch.ts"
+    "example:arrow": "tsx examples/arrow/batch.ts"
   },
   "optionalDependencies": {
-    "@databricks/zerobus-ingest-sdk-linux-x64-gnu": "1.0.2",
-    "@databricks/zerobus-ingest-sdk-linux-arm64-gnu": "1.0.2",
-    "@databricks/zerobus-ingest-sdk-win32-x64-msvc": "1.0.2"
+    "@databricks/zerobus-ingest-sdk-linux-x64-gnu": "1.2.0",
+    "@databricks/zerobus-ingest-sdk-linux-arm64-gnu": "1.2.0",
+    "@databricks/zerobus-ingest-sdk-win32-x64-msvc": "1.2.0",
+    "@databricks/zerobus-ingest-sdk-darwin-x64": "1.2.0",
+    "@databricks/zerobus-ingest-sdk-darwin-arm64": "1.2.0"
+  },
+  "dependencies": {
+    "protobufjs": "^8.7.1"
   },
   "peerDependencies": {
-    "protobufjs": "^7.0.0",
-    "apache-arrow": "^56.0.0"
+    "apache-arrow": "^18.0.0"
   },
   "peerDependenciesMeta": {
-    "protobufjs": {
-      "optional": true
-    },
     "apache-arrow": {
       "optional": true
     }
@@ -88,12 +98,18 @@
     "@types/node": "^20.0.0",
     "apache-arrow": "^18.1.0",
     "dotenv": "^17.2.3",
-    "protobufjs": "^7.5.4",
-    "protobufjs-cli": "^2.0.0",
+    "protobufjs-cli": "^2.6.1",
     "tsx": "^4.21.0",
     "typescript": "^5.3.0"
   },
   "overrides": {
-    "glob": "^10.0.0"
+    "@protobufjs/utf8": "^1.1.1",
+    "brace-expansion": "^2.1.2",
+    "glob": "^10.0.0",
+    "lodash": "^4.18.0",
+    "markdown-it": "^14.2.0",
+    "minimatch": "^9.0.7",
+    "tmp": "^0.2.6",
+    "underscore": "^1.13.8"
   }
 }

package/src/headers_provider.ts

@@ -1,69 +1,57 @@
 /**
- * Interface for providing custom headers to Zerobus streams.
+ * Custom headers provider accepted by `createStream()`.
  *
- * Implement this interface to use custom authentication beyond OAuth,
- * such as Personal Access Tokens (PAT) or custom auth tokens.
+ * The native adapter invokes `getHeadersCallback` synchronously once during
+ * stream creation and stores the returned tuples. Returning a Promise, or
+ * passing a class with async `getHeaders()`, is not supported and can terminate
+ * the process. Token refresh is not currently wired through this callback.
  */
 export interface HeadersProvider {
     /**
-     * Returns headers as array of [name, value] tuples.
+     * Returns headers as an array of [name, value] tuples.
      *
      * Required headers:
      * - ["authorization", "Bearer <token>"]
      * - ["x-databricks-zerobus-table-name", "<table_name>"]
-     *
-     * @returns Promise resolving to array of header name-value pairs
      */
-    getHeaders(): Promise<Array<[string, string]>>;
+    getHeadersCallback: () => Array<[string, string]>;
 }
 
 /**
  * OAuth 2.0 Client Credentials headers provider.
  *
- * **IMPORTANT: DO NOT instantiate this class directly.**
+ * Do not instantiate this class or pass it to `createStream()`.
  *
  * OAuth authentication is handled automatically by the Rust SDK when you call
- * `createStream()` with clientId and clientSecret parameters (without providing
- * a headers_provider).
- *
- * This class exists for:
- * 1. Documentation purposes - showing the HeadersProvider pattern
- * 2. API consistency with other Zerobus SDKs (Python, Java, Rust)
+ * `createStream()` with clientId and clientSecret and omit the headers provider.
  *
- * **How to use OAuth authentication:**
+ * How to use OAuth authentication:
  * ```typescript
- * // OAuth is the default - just pass clientId and clientSecret
  * const stream = await sdk.createStream(
  *     tableProperties,
- *     clientId,      // OAuth client ID
- *     clientSecret,  // OAuth client secret
+ *     clientId,
+ *     clientSecret,
  *     options
- *     // No headers_provider parameter = OAuth authentication
  * );
  * ```
  *
- * **How to use custom authentication (PAT, etc.):**
+ * How to use custom authentication (PAT or a static token):
  * ```typescript
- * class CustomHeadersProvider implements HeadersProvider {
- *     async getHeaders() {
- *         return [
- *             ["authorization", `Bearer ${myToken}`],
- *             ["x-databricks-zerobus-table-name", tableName]
- *         ];
- *     }
- * }
- *
- * const provider = new CustomHeadersProvider();
  * const stream = await sdk.createStream(
  *     tableProperties,
- *     '', // ignored
- *     '', // ignored
+ *     '',
+ *     '',
  *     options,
- *     { getHeadersCallback: provider.getHeaders.bind(provider) }
+ *     {
+ *         getHeadersCallback: () => [
+ *             ["authorization", `Bearer ${myToken}`],
+ *             ["x-databricks-zerobus-table-name", tableName]
+ *         ]
+ *     }
  * );
  * ```
  */
-export class OAuthHeadersProvider implements HeadersProvider {
+export class OAuthHeadersProvider {
     constructor(
         private clientId: string,
         private clientSecret: string,
@@ -75,8 +63,8 @@ export class OAuthHeadersProvider implements HeadersProvider {
         throw new Error(
             'OAuthHeadersProvider should not be instantiated directly. ' +
             'OAuth authentication is handled internally by the Rust SDK. ' +
-            'To use OAuth: call createStream(tableProperties, clientId, clientSecret, options) without the headers_provider parameter. ' +
-            'To use custom authentication: implement the HeadersProvider interface.'
+            'To use OAuth: call createStream(tableProperties, clientId, clientSecret, options) without a headers provider. ' +
+            'To use custom authentication: pass { getHeadersCallback: () => [...] } as the headers provider.'
         );
     }
 }