@databricks/appkit 0.44.0 → 0.45.0

package/dist/stream/arrow-stream-processor.js

@@ -6,148 +6,124 @@ import "../errors/index.js";
 //#region src/stream/arrow-stream-processor.ts
 const logger = createLogger("stream:arrow");
 const BACKOFF_MULTIPLIER = 1e3;
+/**
+* Streams Arrow IPC bytes for a completed statement's EXTERNAL_LINKS chunks
+* without ever buffering a whole chunk — each chunk's response body is piped
+* through as it arrives, so peak memory is a single network read rather than
+* a full chunk (let alone the full result). No Arrow parsing on the server;
+* the client parses the concatenated IPC bytes.
+*/
 var ArrowStreamProcessor = class ArrowStreamProcessor {
-	static DEFAULT_MAX_CONCURRENT_DOWNLOADS = 5;
 	static DEFAULT_TIMEOUT = 3e4;
 	static DEFAULT_RETRIES = 3;
-	constructor(options = {
-		maxConcurrentDownloads: ArrowStreamProcessor.DEFAULT_MAX_CONCURRENT_DOWNLOADS,
-		timeout: ArrowStreamProcessor.DEFAULT_TIMEOUT,
-		retries: ArrowStreamProcessor.DEFAULT_RETRIES
-	}) {
-		this.options = options;
+	options;
+	constructor(options) {
 		this.options = {
-			maxConcurrentDownloads: options.maxConcurrentDownloads ?? ArrowStreamProcessor.DEFAULT_MAX_CONCURRENT_DOWNLOADS,
-			timeout: options.timeout ?? ArrowStreamProcessor.DEFAULT_TIMEOUT,
-			retries: options.retries ?? ArrowStreamProcessor.DEFAULT_RETRIES
+			timeout: options?.timeout ?? ArrowStreamProcessor.DEFAULT_TIMEOUT,
+			retries: options?.retries ?? ArrowStreamProcessor.DEFAULT_RETRIES
 		};
 	}
 	/**
-	* Process Arrow chunks using zero-copy proxy pattern.
+	* Stream Arrow chunks in array order, piping each chunk's response body.
 	*
-	* Downloads raw IPC bytes from external links and concatenates them
-	* without parsing into Arrow Tables on the server. This reduces:
-	* - Memory usage by ~50% (no parsed Table representation)
-	* - CPU usage (no tableFromIPC/tableToIPC calls)
-	*
-	* The client is responsible for parsing the IPC bytes.
-	*
-	* @param chunks - External links to Arrow IPC data
-	* @param schema - Schema from Databricks manifest
-	* @param signal - Optional abort signal
-	* @returns Raw concatenated IPC bytes with schema
+	* Yields network-sized pieces as they arrive (not whole chunks), so peak
+	* memory is one read buffer. Chunks are fetched sequentially; the
+	* concatenation of everything yielded is byte-identical to the raw Arrow
+	* result, so the client parses it exactly as a buffered response.
 	*/
-	async processChunks(chunks, schema, signal) {
+	async *streamChunks(chunks, signal, refresh) {
 		if (chunks.length === 0) throw ValidationError.missingField("chunks");
-		const buffers = await this.downloadChunksRaw(chunks, signal);
-		return {
-			data: this.concatenateBuffers(buffers),
-			schema
-		};
+		for (const chunk of chunks) yield* this.streamChunkBody(chunk, signal, refresh);
 	}
 	/**
-	* Download all chunks as raw bytes with concurrency control.
-	*/
-	async downloadChunksRaw(chunks, signal) {
-		const semaphore = new Semaphore(this.options.maxConcurrentDownloads);
-		const downloadPromises = chunks.map(async (chunk) => {
-			await semaphore.acquire();
-			try {
-				return await this.downloadChunkRaw(chunk, signal);
-			} finally {
-				semaphore.release();
-			}
-		});
-		return Promise.all(downloadPromises);
-	}
-	/**
-	* Download a single chunk as raw bytes with retry logic.
+	* Pipe one chunk's response body.
+	*
+	* Retry applies only while *establishing* the response (connection failure
+	* or non-2xx) — once bytes have been yielded downstream we cannot re-fetch,
+	* so a mid-body failure propagates and the caller aborts the response. An
+	* idle timeout, reset before every read, aborts a stalled download.
 	*/
-	async downloadChunkRaw(chunk, signal) {
-		let lastError = null;
+	async *streamChunkBody(chunk, signal, refresh) {
+		let externalLink = chunk.external_link;
+		if (!externalLink) throw ExecutionError.statementFailed(`External link missing for chunk ${chunk.chunk_index}`);
+		let response;
+		let controller;
+		let onOuterAbort;
+		let lastError;
 		for (let attempt = 0; attempt < this.options.retries; attempt++) {
-			const timeoutController = new AbortController();
-			const timeoutId = setTimeout(() => {
-				timeoutController.abort();
-			}, this.options.timeout);
-			const combinedSignal = signal ? this.combineAbortSignals(signal, timeoutController.signal) : timeoutController.signal;
+			if (signal?.aborted) throw ExecutionError.canceled();
+			const attemptController = new AbortController();
+			const listener = () => attemptController.abort();
+			signal?.addEventListener("abort", listener, { once: true });
+			const timer = setTimeout(() => attemptController.abort(), this.options.timeout);
 			try {
-				const externalLink = chunk.external_link;
-				if (!externalLink) {
-					logger.error("External link is required for chunk: %O", chunk);
-					continue;
-				}
-				const response = await fetch(externalLink, { signal: combinedSignal });
-				if (!response.ok) throw ExecutionError.statementFailed(`Failed to download chunk ${chunk.chunk_index}: ${response.status} ${response.statusText}`);
-				const arrayBuffer = await response.arrayBuffer();
-				return new Uint8Array(arrayBuffer);
+				const r = await fetch(externalLink, { signal: attemptController.signal });
+				clearTimeout(timer);
+				if (!r.ok) throw ExecutionError.statementFailed(`Failed to download chunk ${chunk.chunk_index}: ${r.status} ${r.statusText}`);
+				response = r;
+				controller = attemptController;
+				onOuterAbort = listener;
+				break;
 			} catch (error) {
+				clearTimeout(timer);
+				signal?.removeEventListener("abort", listener);
 				lastError = error;
-				if (timeoutController.signal.aborted) lastError = /* @__PURE__ */ new Error(`Chunk ${chunk.chunk_index} download timed out after ${this.options.timeout}ms`);
 				if (signal?.aborted) throw ExecutionError.canceled();
-				if (attempt < this.options.retries - 1) await this.delay(2 ** attempt * BACKOFF_MULTIPLIER);
-			} finally {
-				clearTimeout(timeoutId);
+				if (attempt < this.options.retries - 1) {
+					await this.delay(2 ** attempt * BACKOFF_MULTIPLIER, signal);
+					if (refresh && chunk.chunk_index != null) try {
+						const fresh = await refresh(chunk.chunk_index, signal);
+						if (fresh?.external_link) externalLink = fresh.external_link;
+					} catch (refreshError) {
+						logger.warn("Failed to re-resolve link for chunk %s: %O", chunk.chunk_index, refreshError);
+					}
+				}
 			}
 		}
-		throw ExecutionError.statementFailed(`Failed to download chunk ${chunk.chunk_index} after ${this.options.retries} attempts: ${lastError?.message}`);
-	}
-	/**
-	* Concatenate multiple Uint8Array buffers into a single buffer.
-	* Pre-allocates the result array for efficiency.
-	*/
-	concatenateBuffers(buffers) {
-		if (buffers.length === 0) throw ValidationError.missingField("buffers");
-		if (buffers.length === 1) return buffers[0];
-		const totalLength = buffers.reduce((sum, buf) => sum + buf.length, 0);
-		const result = new Uint8Array(totalLength);
-		let offset = 0;
-		for (const buffer of buffers) {
-			result.set(buffer, offset);
-			offset += buffer.length;
-		}
-		return result;
-	}
-	/**
-	* Combines multiple AbortSignals into one.
-	* The combined signal aborts when any of the input signals abort.
-	*/
-	combineAbortSignals(...signals) {
-		const controller = new AbortController();
-		for (const signal of signals) {
-			if (signal.aborted) {
-				controller.abort();
-				return controller.signal;
+		if (!response || !controller) throw ExecutionError.statementFailed(`Failed to download chunk ${chunk.chunk_index} after ${this.options.retries} attempts: ${lastError instanceof Error ? lastError.message : String(lastError)}`);
+		const body = response.body;
+		let idleTimer;
+		try {
+			if (!body) return;
+			const reader = body.getReader();
+			try {
+				while (true) {
+					idleTimer = setTimeout(() => controller.abort(), this.options.timeout);
+					let done;
+					let value;
+					try {
+						({done, value} = await reader.read());
+					} finally {
+						clearTimeout(idleTimer);
+					}
+					if (done) break;
+					if (value && value.byteLength > 0) yield value;
+				}
+			} finally {
+				reader.cancel().catch(() => {});
 			}
-			signal.addEventListener("abort", () => controller.abort(), { once: true });
+		} catch (error) {
+			if (signal?.aborted) throw ExecutionError.canceled();
+			logger.error("Failed streaming chunk %s body: %O", chunk.chunk_index, error);
+			throw error instanceof ExecutionError ? error : ExecutionError.statementFailed(`Failed streaming chunk ${chunk.chunk_index}: ${error instanceof Error ? error.message : String(error)}`);
+		} finally {
+			clearTimeout(idleTimer);
+			if (onOuterAbort) signal?.removeEventListener("abort", onOuterAbort);
 		}
-		return controller.signal;
 	}
-	delay(ms) {
-		return new Promise((resolve) => setTimeout(resolve, ms));
-	}
-};
-var Semaphore = class {
-	permits;
-	waiting = [];
-	constructor(permits) {
-		this.permits = permits;
-	}
-	async acquire() {
-		if (this.permits > 0) {
-			this.permits--;
-			return;
-		}
-		return new Promise((resolve) => {
-			this.waiting.push(resolve);
+	delay(ms, signal) {
+		return new Promise((resolve, reject) => {
+			const timer = setTimeout(() => {
+				signal?.removeEventListener("abort", onAbort);
+				resolve();
+			}, ms);
+			const onAbort = () => {
+				clearTimeout(timer);
+				reject(ExecutionError.canceled());
+			};
+			signal?.addEventListener("abort", onAbort, { once: true });
 		});
 	}
-	release() {
-		if (this.waiting.length > 0) {
-			const next = this.waiting.shift();
-			if (next) next();
-		} else this.permits++;
-	}
 };
 
 //#endregion

package/dist/stream/arrow-stream-processor.js.map

@@ -1 +1 @@
-{"version":3,"file":"arrow-stream-processor.js","names":[],"sources":["../../src/stream/arrow-stream-processor.ts"],"sourcesContent":["import type { sql } from \"@databricks/sdk-experimental\";\nimport { ExecutionError, ValidationError } from \"../errors\";\nimport { createLogger } from \"../logging/logger\";\n\nconst logger = createLogger(\"stream:arrow\");\n\ntype ResultManifest = sql.ResultManifest;\ntype ExternalLink = sql.ExternalLink;\n\ninterface ArrowStreamOptions {\n  maxConcurrentDownloads: number;\n  timeout: number;\n  retries: number;\n}\n\n/**\n * Result from zero-copy Arrow chunk processing.\n * Contains raw IPC bytes without server-side parsing.\n */\ninterface ArrowRawResult {\n  /** Concatenated raw Arrow IPC bytes */\n  data: Uint8Array;\n  /** Schema from Databricks manifest (not parsed from Arrow) */\n  schema: ResultManifest[\"schema\"];\n}\n\nconst BACKOFF_MULTIPLIER = 1000;\n\nexport class ArrowStreamProcessor {\n  static readonly DEFAULT_MAX_CONCURRENT_DOWNLOADS = 5;\n  static readonly DEFAULT_TIMEOUT = 30000;\n  static readonly DEFAULT_RETRIES = 3;\n\n  constructor(\n    private options: ArrowStreamOptions = {\n      maxConcurrentDownloads:\n        ArrowStreamProcessor.DEFAULT_MAX_CONCURRENT_DOWNLOADS,\n      timeout: ArrowStreamProcessor.DEFAULT_TIMEOUT,\n      retries: ArrowStreamProcessor.DEFAULT_RETRIES,\n    },\n  ) {\n    this.options = {\n      maxConcurrentDownloads:\n        options.maxConcurrentDownloads ??\n        ArrowStreamProcessor.DEFAULT_MAX_CONCURRENT_DOWNLOADS,\n      timeout: options.timeout ?? ArrowStreamProcessor.DEFAULT_TIMEOUT,\n      retries: options.retries ?? ArrowStreamProcessor.DEFAULT_RETRIES,\n    };\n  }\n\n  /**\n   * Process Arrow chunks using zero-copy proxy pattern.\n   *\n   * Downloads raw IPC bytes from external links and concatenates them\n   * without parsing into Arrow Tables on the server. This reduces:\n   * - Memory usage by ~50% (no parsed Table representation)\n   * - CPU usage (no tableFromIPC/tableToIPC calls)\n   *\n   * The client is responsible for parsing the IPC bytes.\n   *\n   * @param chunks - External links to Arrow IPC data\n   * @param schema - Schema from Databricks manifest\n   * @param signal - Optional abort signal\n   * @returns Raw concatenated IPC bytes with schema\n   */\n  async processChunks(\n    chunks: ExternalLink[],\n    schema: ResultManifest[\"schema\"],\n    signal?: AbortSignal,\n  ): Promise<ArrowRawResult> {\n    if (chunks.length === 0) {\n      throw ValidationError.missingField(\"chunks\");\n    }\n\n    const buffers = await this.downloadChunksRaw(chunks, signal);\n    const data = this.concatenateBuffers(buffers);\n\n    return { data, schema };\n  }\n\n  /**\n   * Download all chunks as raw bytes with concurrency control.\n   */\n  private async downloadChunksRaw(\n    chunks: ExternalLink[],\n    signal?: AbortSignal,\n  ): Promise<Uint8Array[]> {\n    const semaphore = new Semaphore(this.options.maxConcurrentDownloads);\n\n    const downloadPromises = chunks.map(async (chunk) => {\n      await semaphore.acquire();\n      try {\n        return await this.downloadChunkRaw(chunk, signal);\n      } finally {\n        semaphore.release();\n      }\n    });\n\n    return Promise.all(downloadPromises);\n  }\n\n  /**\n   * Download a single chunk as raw bytes with retry logic.\n   */\n  private async downloadChunkRaw(\n    chunk: ExternalLink,\n    signal?: AbortSignal,\n  ): Promise<Uint8Array> {\n    let lastError: Error | null = null;\n\n    for (let attempt = 0; attempt < this.options.retries; attempt++) {\n      const timeoutController = new AbortController();\n      const timeoutId = setTimeout(() => {\n        timeoutController.abort();\n      }, this.options.timeout);\n\n      const combinedSignal = signal\n        ? this.combineAbortSignals(signal, timeoutController.signal)\n        : timeoutController.signal;\n\n      try {\n        const externalLink = chunk.external_link;\n        if (!externalLink) {\n          logger.error(\"External link is required for chunk: %O\", chunk);\n          continue;\n        }\n\n        const response = await fetch(externalLink, {\n          signal: combinedSignal,\n        });\n\n        if (!response.ok) {\n          throw ExecutionError.statementFailed(\n            `Failed to download chunk ${chunk.chunk_index}: ${response.status} ${response.statusText}`,\n          );\n        }\n\n        const arrayBuffer = await response.arrayBuffer();\n        return new Uint8Array(arrayBuffer);\n      } catch (error) {\n        lastError = error as Error;\n\n        if (timeoutController.signal.aborted) {\n          lastError = new Error(\n            `Chunk ${chunk.chunk_index} download timed out after ${this.options.timeout}ms`,\n          );\n        }\n\n        if (signal?.aborted) {\n          throw ExecutionError.canceled();\n        }\n\n        if (attempt < this.options.retries - 1) {\n          await this.delay(2 ** attempt * BACKOFF_MULTIPLIER);\n        }\n      } finally {\n        clearTimeout(timeoutId);\n      }\n    }\n\n    throw ExecutionError.statementFailed(\n      `Failed to download chunk ${chunk.chunk_index} after ${this.options.retries} attempts: ${lastError?.message}`,\n    );\n  }\n\n  /**\n   * Concatenate multiple Uint8Array buffers into a single buffer.\n   * Pre-allocates the result array for efficiency.\n   */\n  private concatenateBuffers(buffers: Uint8Array[]): Uint8Array {\n    if (buffers.length === 0) {\n      throw ValidationError.missingField(\"buffers\");\n    }\n\n    if (buffers.length === 1) {\n      return buffers[0];\n    }\n\n    const totalLength = buffers.reduce((sum, buf) => sum + buf.length, 0);\n    const result = new Uint8Array(totalLength);\n\n    let offset = 0;\n    for (const buffer of buffers) {\n      result.set(buffer, offset);\n      offset += buffer.length;\n    }\n\n    return result;\n  }\n\n  /**\n   * Combines multiple AbortSignals into one.\n   * The combined signal aborts when any of the input signals abort.\n   */\n  private combineAbortSignals(...signals: AbortSignal[]): AbortSignal {\n    const controller = new AbortController();\n\n    for (const signal of signals) {\n      if (signal.aborted) {\n        controller.abort();\n        return controller.signal;\n      }\n      signal.addEventListener(\"abort\", () => controller.abort(), {\n        once: true,\n      });\n    }\n\n    return controller.signal;\n  }\n\n  private delay(ms: number): Promise<void> {\n    return new Promise((resolve) => setTimeout(resolve, ms));\n  }\n}\n\nclass Semaphore {\n  private permits: number;\n  private waiting: (() => void)[] = [];\n\n  constructor(permits: number) {\n    this.permits = permits;\n  }\n\n  async acquire(): Promise<void> {\n    if (this.permits > 0) {\n      this.permits--;\n      return;\n    }\n\n    return new Promise<void>((resolve) => {\n      this.waiting.push(resolve);\n    });\n  }\n\n  release(): void {\n    if (this.waiting.length > 0) {\n      const next = this.waiting.shift();\n\n      if (next) {\n        next();\n      }\n    } else {\n      this.permits++;\n    }\n  }\n}\n"],"mappings":";;;;;;AAIA,MAAM,SAAS,aAAa,eAAe;AAsB3C,MAAM,qBAAqB;AAE3B,IAAa,uBAAb,MAAa,qBAAqB;CAChC,OAAgB,mCAAmC;CACnD,OAAgB,kBAAkB;CAClC,OAAgB,kBAAkB;CAElC,YACE,AAAQ,UAA8B;EACpC,wBACE,qBAAqB;EACvB,SAAS,qBAAqB;EAC9B,SAAS,qBAAqB;EAC/B,EACD;EANQ;AAOR,OAAK,UAAU;GACb,wBACE,QAAQ,0BACR,qBAAqB;GACvB,SAAS,QAAQ,WAAW,qBAAqB;GACjD,SAAS,QAAQ,WAAW,qBAAqB;GAClD;;;;;;;;;;;;;;;;;CAkBH,MAAM,cACJ,QACA,QACA,QACyB;AACzB,MAAI,OAAO,WAAW,EACpB,OAAM,gBAAgB,aAAa,SAAS;EAG9C,MAAM,UAAU,MAAM,KAAK,kBAAkB,QAAQ,OAAO;AAG5D,SAAO;GAAE,MAFI,KAAK,mBAAmB,QAAQ;GAE9B;GAAQ;;;;;CAMzB,MAAc,kBACZ,QACA,QACuB;EACvB,MAAM,YAAY,IAAI,UAAU,KAAK,QAAQ,uBAAuB;EAEpE,MAAM,mBAAmB,OAAO,IAAI,OAAO,UAAU;AACnD,SAAM,UAAU,SAAS;AACzB,OAAI;AACF,WAAO,MAAM,KAAK,iBAAiB,OAAO,OAAO;aACzC;AACR,cAAU,SAAS;;IAErB;AAEF,SAAO,QAAQ,IAAI,iBAAiB;;;;;CAMtC,MAAc,iBACZ,OACA,QACqB;EACrB,IAAI,YAA0B;AAE9B,OAAK,IAAI,UAAU,GAAG,UAAU,KAAK,QAAQ,SAAS,WAAW;GAC/D,MAAM,oBAAoB,IAAI,iBAAiB;GAC/C,MAAM,YAAY,iBAAiB;AACjC,sBAAkB,OAAO;MACxB,KAAK,QAAQ,QAAQ;GAExB,MAAM,iBAAiB,SACnB,KAAK,oBAAoB,QAAQ,kBAAkB,OAAO,GAC1D,kBAAkB;AAEtB,OAAI;IACF,MAAM,eAAe,MAAM;AAC3B,QAAI,CAAC,cAAc;AACjB,YAAO,MAAM,2CAA2C,MAAM;AAC9D;;IAGF,MAAM,WAAW,MAAM,MAAM,cAAc,EACzC,QAAQ,gBACT,CAAC;AAEF,QAAI,CAAC,SAAS,GACZ,OAAM,eAAe,gBACnB,4BAA4B,MAAM,YAAY,IAAI,SAAS,OAAO,GAAG,SAAS,aAC/E;IAGH,MAAM,cAAc,MAAM,SAAS,aAAa;AAChD,WAAO,IAAI,WAAW,YAAY;YAC3B,OAAO;AACd,gBAAY;AAEZ,QAAI,kBAAkB,OAAO,QAC3B,6BAAY,IAAI,MACd,SAAS,MAAM,YAAY,4BAA4B,KAAK,QAAQ,QAAQ,IAC7E;AAGH,QAAI,QAAQ,QACV,OAAM,eAAe,UAAU;AAGjC,QAAI,UAAU,KAAK,QAAQ,UAAU,EACnC,OAAM,KAAK,MAAM,KAAK,UAAU,mBAAmB;aAE7C;AACR,iBAAa,UAAU;;;AAI3B,QAAM,eAAe,gBACnB,4BAA4B,MAAM,YAAY,SAAS,KAAK,QAAQ,QAAQ,aAAa,WAAW,UACrG;;;;;;CAOH,AAAQ,mBAAmB,SAAmC;AAC5D,MAAI,QAAQ,WAAW,EACrB,OAAM,gBAAgB,aAAa,UAAU;AAG/C,MAAI,QAAQ,WAAW,EACrB,QAAO,QAAQ;EAGjB,MAAM,cAAc,QAAQ,QAAQ,KAAK,QAAQ,MAAM,IAAI,QAAQ,EAAE;EACrE,MAAM,SAAS,IAAI,WAAW,YAAY;EAE1C,IAAI,SAAS;AACb,OAAK,MAAM,UAAU,SAAS;AAC5B,UAAO,IAAI,QAAQ,OAAO;AAC1B,aAAU,OAAO;;AAGnB,SAAO;;;;;;CAOT,AAAQ,oBAAoB,GAAG,SAAqC;EAClE,MAAM,aAAa,IAAI,iBAAiB;AAExC,OAAK,MAAM,UAAU,SAAS;AAC5B,OAAI,OAAO,SAAS;AAClB,eAAW,OAAO;AAClB,WAAO,WAAW;;AAEpB,UAAO,iBAAiB,eAAe,WAAW,OAAO,EAAE,EACzD,MAAM,MACP,CAAC;;AAGJ,SAAO,WAAW;;CAGpB,AAAQ,MAAM,IAA2B;AACvC,SAAO,IAAI,SAAS,YAAY,WAAW,SAAS,GAAG,CAAC;;;AAI5D,IAAM,YAAN,MAAgB;CACd,AAAQ;CACR,AAAQ,UAA0B,EAAE;CAEpC,YAAY,SAAiB;AAC3B,OAAK,UAAU;;CAGjB,MAAM,UAAyB;AAC7B,MAAI,KAAK,UAAU,GAAG;AACpB,QAAK;AACL;;AAGF,SAAO,IAAI,SAAe,YAAY;AACpC,QAAK,QAAQ,KAAK,QAAQ;IAC1B;;CAGJ,UAAgB;AACd,MAAI,KAAK,QAAQ,SAAS,GAAG;GAC3B,MAAM,OAAO,KAAK,QAAQ,OAAO;AAEjC,OAAI,KACF,OAAM;QAGR,MAAK"}
+{"version":3,"file":"arrow-stream-processor.js","names":[],"sources":["../../src/stream/arrow-stream-processor.ts"],"sourcesContent":["import type { sql } from \"@databricks/sdk-experimental\";\nimport { ExecutionError, ValidationError } from \"../errors\";\nimport { createLogger } from \"../logging/logger\";\n\nconst logger = createLogger(\"stream:arrow\");\n\ntype ExternalLink = sql.ExternalLink;\n\n/**\n * Re-mint a chunk's pre-signed URL. DBSQL external links expire in <= 15 min,\n * so a large result whose tail chunks are reached after the earlier chunks\n * finish draining can hit an expired link; this fetches a fresh one for the\n * given chunk index. Created in the request's identity context (it captures the\n * caller's workspace client), so it stays valid even though streaming runs\n * outside that context. Returns `undefined` if the link can't be re-resolved.\n */\nexport type RefreshChunkLink = (\n  chunkIndex: number,\n  signal?: AbortSignal,\n) => Promise<ExternalLink | undefined>;\n\ninterface ArrowStreamOptions {\n  /** Idle timeout (ms) for a single chunk: no progress → abort the download. */\n  timeout: number;\n  /** Attempts to establish a chunk's response before any bytes are yielded. */\n  retries: number;\n}\n\nconst BACKOFF_MULTIPLIER = 1000;\n\n/**\n * Streams Arrow IPC bytes for a completed statement's EXTERNAL_LINKS chunks\n * without ever buffering a whole chunk — each chunk's response body is piped\n * through as it arrives, so peak memory is a single network read rather than\n * a full chunk (let alone the full result). No Arrow parsing on the server;\n * the client parses the concatenated IPC bytes.\n */\nexport class ArrowStreamProcessor {\n  static readonly DEFAULT_TIMEOUT = 30000;\n  static readonly DEFAULT_RETRIES = 3;\n\n  private options: ArrowStreamOptions;\n\n  constructor(options?: Partial<ArrowStreamOptions>) {\n    this.options = {\n      timeout: options?.timeout ?? ArrowStreamProcessor.DEFAULT_TIMEOUT,\n      retries: options?.retries ?? ArrowStreamProcessor.DEFAULT_RETRIES,\n    };\n  }\n\n  /**\n   * Stream Arrow chunks in array order, piping each chunk's response body.\n   *\n   * Yields network-sized pieces as they arrive (not whole chunks), so peak\n   * memory is one read buffer. Chunks are fetched sequentially; the\n   * concatenation of everything yielded is byte-identical to the raw Arrow\n   * result, so the client parses it exactly as a buffered response.\n   */\n  async *streamChunks(\n    chunks: ExternalLink[],\n    signal?: AbortSignal,\n    refresh?: RefreshChunkLink,\n  ): AsyncGenerator<Uint8Array, void, unknown> {\n    if (chunks.length === 0) {\n      throw ValidationError.missingField(\"chunks\");\n    }\n\n    for (const chunk of chunks) {\n      yield* this.streamChunkBody(chunk, signal, refresh);\n    }\n  }\n\n  /**\n   * Pipe one chunk's response body.\n   *\n   * Retry applies only while *establishing* the response (connection failure\n   * or non-2xx) — once bytes have been yielded downstream we cannot re-fetch,\n   * so a mid-body failure propagates and the caller aborts the response. An\n   * idle timeout, reset before every read, aborts a stalled download.\n   */\n  private async *streamChunkBody(\n    chunk: ExternalLink,\n    signal?: AbortSignal,\n    refresh?: RefreshChunkLink,\n  ): AsyncGenerator<Uint8Array, void, unknown> {\n    let externalLink = chunk.external_link;\n    if (!externalLink) {\n      // A missing link cannot be fixed by retrying — fail loudly.\n      throw ExecutionError.statementFailed(\n        `External link missing for chunk ${chunk.chunk_index}`,\n      );\n    }\n\n    let response: Response | undefined;\n    let controller: AbortController | undefined;\n    let onOuterAbort: (() => void) | undefined;\n    let lastError: unknown;\n\n    for (let attempt = 0; attempt < this.options.retries; attempt++) {\n      if (signal?.aborted) throw ExecutionError.canceled();\n\n      const attemptController = new AbortController();\n      const listener = () => attemptController.abort();\n      signal?.addEventListener(\"abort\", listener, { once: true });\n      const timer = setTimeout(\n        () => attemptController.abort(),\n        this.options.timeout,\n      );\n\n      try {\n        const r = await fetch(externalLink, {\n          signal: attemptController.signal,\n        });\n        clearTimeout(timer);\n        if (!r.ok) {\n          throw ExecutionError.statementFailed(\n            `Failed to download chunk ${chunk.chunk_index}: ${r.status} ${r.statusText}`,\n          );\n        }\n        // Keep this attempt's controller alive to drive the body read + idle\n        // timeout below.\n        response = r;\n        controller = attemptController;\n        onOuterAbort = listener;\n        break;\n      } catch (error) {\n        clearTimeout(timer);\n        signal?.removeEventListener(\"abort\", listener);\n        lastError = error;\n        if (signal?.aborted) throw ExecutionError.canceled();\n        if (attempt < this.options.retries - 1) {\n          await this.delay(2 ** attempt * BACKOFF_MULTIPLIER, signal);\n          // Pre-signed URLs expire (<= 15 min). Before retrying, re-mint this\n          // chunk's link — a stale URL would just 403 again on the same address.\n          // Only meaningful before any bytes are yielded (below), which is why\n          // this lives in the establish-response loop.\n          if (refresh && chunk.chunk_index != null) {\n            try {\n              const fresh = await refresh(chunk.chunk_index, signal);\n              if (fresh?.external_link) externalLink = fresh.external_link;\n            } catch (refreshError) {\n              // Keep retrying the current URL; surface the original error if\n              // all attempts fail.\n              logger.warn(\n                \"Failed to re-resolve link for chunk %s: %O\",\n                chunk.chunk_index,\n                refreshError,\n              );\n            }\n          }\n        }\n      }\n    }\n\n    if (!response || !controller) {\n      throw ExecutionError.statementFailed(\n        `Failed to download chunk ${chunk.chunk_index} after ${this.options.retries} attempts: ${\n          lastError instanceof Error ? lastError.message : String(lastError)\n        }`,\n      );\n    }\n\n    const body = response.body;\n    let idleTimer: ReturnType<typeof setTimeout> | undefined;\n    try {\n      if (!body) return; // empty chunk (0 rows)\n      const reader = body.getReader();\n      try {\n        while (true) {\n          // The idle timeout must cover ONLY the upstream read, never the\n          // downstream `yield`. Arm it right before `reader.read()` and clear\n          // it the instant the read resolves — if it stayed armed across the\n          // yield, a slow client backpressuring `writeChunk` would look like an\n          // upstream stall and abort a perfectly healthy download.\n          idleTimer = setTimeout(\n            () => controller.abort(),\n            this.options.timeout,\n          );\n          let done: boolean;\n          let value: Uint8Array | undefined;\n          try {\n            ({ done, value } = await reader.read());\n          } finally {\n            clearTimeout(idleTimer);\n          }\n          if (done) break;\n          if (value && value.byteLength > 0) yield value;\n        }\n      } finally {\n        // Release the stream on early exit (downstream abort / error).\n        reader.cancel().catch(() => {});\n      }\n    } catch (error) {\n      if (signal?.aborted) throw ExecutionError.canceled();\n      logger.error(\n        \"Failed streaming chunk %s body: %O\",\n        chunk.chunk_index,\n        error,\n      );\n      throw error instanceof ExecutionError\n        ? error\n        : ExecutionError.statementFailed(\n            `Failed streaming chunk ${chunk.chunk_index}: ${\n              error instanceof Error ? error.message : String(error)\n            }`,\n          );\n    } finally {\n      clearTimeout(idleTimer);\n      if (onOuterAbort) signal?.removeEventListener(\"abort\", onOuterAbort);\n    }\n  }\n\n  private delay(ms: number, signal?: AbortSignal): Promise<void> {\n    return new Promise((resolve, reject) => {\n      const timer = setTimeout(() => {\n        signal?.removeEventListener(\"abort\", onAbort);\n        resolve();\n      }, ms);\n      const onAbort = () => {\n        clearTimeout(timer);\n        reject(ExecutionError.canceled());\n      };\n      signal?.addEventListener(\"abort\", onAbort, { once: true });\n    });\n  }\n}\n"],"mappings":";;;;;;AAIA,MAAM,SAAS,aAAa,eAAe;AAwB3C,MAAM,qBAAqB;;;;;;;;AAS3B,IAAa,uBAAb,MAAa,qBAAqB;CAChC,OAAgB,kBAAkB;CAClC,OAAgB,kBAAkB;CAElC,AAAQ;CAER,YAAY,SAAuC;AACjD,OAAK,UAAU;GACb,SAAS,SAAS,WAAW,qBAAqB;GAClD,SAAS,SAAS,WAAW,qBAAqB;GACnD;;;;;;;;;;CAWH,OAAO,aACL,QACA,QACA,SAC2C;AAC3C,MAAI,OAAO,WAAW,EACpB,OAAM,gBAAgB,aAAa,SAAS;AAG9C,OAAK,MAAM,SAAS,OAClB,QAAO,KAAK,gBAAgB,OAAO,QAAQ,QAAQ;;;;;;;;;;CAYvD,OAAe,gBACb,OACA,QACA,SAC2C;EAC3C,IAAI,eAAe,MAAM;AACzB,MAAI,CAAC,aAEH,OAAM,eAAe,gBACnB,mCAAmC,MAAM,cAC1C;EAGH,IAAI;EACJ,IAAI;EACJ,IAAI;EACJ,IAAI;AAEJ,OAAK,IAAI,UAAU,GAAG,UAAU,KAAK,QAAQ,SAAS,WAAW;AAC/D,OAAI,QAAQ,QAAS,OAAM,eAAe,UAAU;GAEpD,MAAM,oBAAoB,IAAI,iBAAiB;GAC/C,MAAM,iBAAiB,kBAAkB,OAAO;AAChD,WAAQ,iBAAiB,SAAS,UAAU,EAAE,MAAM,MAAM,CAAC;GAC3D,MAAM,QAAQ,iBACN,kBAAkB,OAAO,EAC/B,KAAK,QAAQ,QACd;AAED,OAAI;IACF,MAAM,IAAI,MAAM,MAAM,cAAc,EAClC,QAAQ,kBAAkB,QAC3B,CAAC;AACF,iBAAa,MAAM;AACnB,QAAI,CAAC,EAAE,GACL,OAAM,eAAe,gBACnB,4BAA4B,MAAM,YAAY,IAAI,EAAE,OAAO,GAAG,EAAE,aACjE;AAIH,eAAW;AACX,iBAAa;AACb,mBAAe;AACf;YACO,OAAO;AACd,iBAAa,MAAM;AACnB,YAAQ,oBAAoB,SAAS,SAAS;AAC9C,gBAAY;AACZ,QAAI,QAAQ,QAAS,OAAM,eAAe,UAAU;AACpD,QAAI,UAAU,KAAK,QAAQ,UAAU,GAAG;AACtC,WAAM,KAAK,MAAM,KAAK,UAAU,oBAAoB,OAAO;AAK3D,SAAI,WAAW,MAAM,eAAe,KAClC,KAAI;MACF,MAAM,QAAQ,MAAM,QAAQ,MAAM,aAAa,OAAO;AACtD,UAAI,OAAO,cAAe,gBAAe,MAAM;cACxC,cAAc;AAGrB,aAAO,KACL,8CACA,MAAM,aACN,aACD;;;;;AAOX,MAAI,CAAC,YAAY,CAAC,WAChB,OAAM,eAAe,gBACnB,4BAA4B,MAAM,YAAY,SAAS,KAAK,QAAQ,QAAQ,aAC1E,qBAAqB,QAAQ,UAAU,UAAU,OAAO,UAAU,GAErE;EAGH,MAAM,OAAO,SAAS;EACtB,IAAI;AACJ,MAAI;AACF,OAAI,CAAC,KAAM;GACX,MAAM,SAAS,KAAK,WAAW;AAC/B,OAAI;AACF,WAAO,MAAM;AAMX,iBAAY,iBACJ,WAAW,OAAO,EACxB,KAAK,QAAQ,QACd;KACD,IAAI;KACJ,IAAI;AACJ,SAAI;AACF,OAAC,CAAE,MAAM,SAAU,MAAM,OAAO,MAAM;eAC9B;AACR,mBAAa,UAAU;;AAEzB,SAAI,KAAM;AACV,SAAI,SAAS,MAAM,aAAa,EAAG,OAAM;;aAEnC;AAER,WAAO,QAAQ,CAAC,YAAY,GAAG;;WAE1B,OAAO;AACd,OAAI,QAAQ,QAAS,OAAM,eAAe,UAAU;AACpD,UAAO,MACL,sCACA,MAAM,aACN,MACD;AACD,SAAM,iBAAiB,iBACnB,QACA,eAAe,gBACb,0BAA0B,MAAM,YAAY,IAC1C,iBAAiB,QAAQ,MAAM,UAAU,OAAO,MAAM,GAEzD;YACG;AACR,gBAAa,UAAU;AACvB,OAAI,aAAc,SAAQ,oBAAoB,SAAS,aAAa;;;CAIxE,AAAQ,MAAM,IAAY,QAAqC;AAC7D,SAAO,IAAI,SAAS,SAAS,WAAW;GACtC,MAAM,QAAQ,iBAAiB;AAC7B,YAAQ,oBAAoB,SAAS,QAAQ;AAC7C,aAAS;MACR,GAAG;GACN,MAAM,gBAAgB;AACpB,iBAAa,MAAM;AACnB,WAAO,eAAe,UAAU,CAAC;;AAEnC,WAAQ,iBAAiB,SAAS,SAAS,EAAE,MAAM,MAAM,CAAC;IAC1D"}

package/dist/stream/defaults.js

@@ -1,7 +1,7 @@
 //#region src/stream/defaults.ts
 const streamDefaults = {
 	bufferSize: 100,
-	maxEventSize: 1024 * 1024,
+	maxEventSize: 1 * 1024 * 1024,
 	bufferTTL: 600 * 1e3,
 	cleanupInterval: 300 * 1e3,
 	maxPersistentBuffers: 1e4,

package/dist/stream/defaults.js.map

@@ -1 +1 @@
-{"version":3,"file":"defaults.js","names":[],"sources":["../../src/stream/defaults.ts"],"sourcesContent":["export const streamDefaults = {\n  bufferSize: 100,\n  maxEventSize: 1024 * 1024, // 1MB\n  bufferTTL: 10 * 60 * 1000, // 10 minutes\n  cleanupInterval: 5 * 60 * 1000, // 5 minutes\n  maxPersistentBuffers: 10000, // 10000 buffers\n  heartbeatInterval: 10 * 1000, // 10 seconds\n  maxActiveStreams: 1000, // 1000 streams\n  disconnectGraceMs: 15_000, // 15 seconds\n} as const;\n"],"mappings":";AAAA,MAAa,iBAAiB;CAC5B,YAAY;CACZ,cAAc,OAAO;CACrB,WAAW,MAAU;CACrB,iBAAiB,MAAS;CAC1B,sBAAsB;CACtB,mBAAmB,KAAK;CACxB,kBAAkB;CAClB,mBAAmB;CACpB"}
+{"version":3,"file":"defaults.js","names":[],"sources":["../../src/stream/defaults.ts"],"sourcesContent":["export const streamDefaults = {\n  bufferSize: 100,\n  // 1 MiB. SSE carries only short JSON control messages — JSON_ARRAY result\n  // rows (already row-size-bounded by the warehouse) plus warehouse-readiness\n  // and error events. ARROW_STREAM never uses SSE: the raw Arrow IPC bytes\n  // stream back on the query response body (`_handleArrowStreamQuery`).\n  maxEventSize: 1 * 1024 * 1024,\n  bufferTTL: 10 * 60 * 1000, // 10 minutes\n  cleanupInterval: 5 * 60 * 1000, // 5 minutes\n  maxPersistentBuffers: 10000, // 10000 buffers\n  heartbeatInterval: 10 * 1000, // 10 seconds\n  maxActiveStreams: 1000, // 1000 streams\n  disconnectGraceMs: 15_000, // 15 seconds\n} as const;\n"],"mappings":";AAAA,MAAa,iBAAiB;CAC5B,YAAY;CAKZ,cAAc,IAAI,OAAO;CACzB,WAAW,MAAU;CACrB,iBAAiB,MAAS;CAC1B,sBAAsB;CACtB,mBAAmB,KAAK;CACxB,kBAAkB;CAClB,mBAAmB;CACpB"}

package/dist/stream/sse-writer.js

@@ -18,11 +18,12 @@ var SSEWriter = class {
 		res.write(`event: ${eventType}\n`);
 		res.write(`data: ${eventData}\n\n`);
 	}
-	writeError(res, eventId, error, code = SSEErrorCode.INTERNAL_ERROR) {
+	writeError(res, eventId, error, code = SSEErrorCode.INTERNAL_ERROR, errorCode) {
 		if (res.writableEnded) return;
 		const errorData = {
 			error,
-			code
+			code,
+			...errorCode ? { errorCode } : {}
 		};
 		res.write(`id: ${eventId}\n`);
 		res.write(`event: error\n`);

package/dist/stream/sse-writer.js.map

@@ -1 +1 @@
-{"version":3,"file":"sse-writer.js","names":[],"sources":["../../src/stream/sse-writer.ts"],"sourcesContent":["import type { IAppResponse } from \"shared\";\nimport { streamDefaults } from \"./defaults\";\nimport {\n  type BufferedEvent,\n  type SSEError,\n  SSEErrorCode,\n  SSEWarningCode,\n} from \"./types\";\nimport { StreamValidator } from \"./validator\";\n\nexport class SSEWriter {\n  // setup SSE headers\n  setupHeaders(res: IAppResponse): void {\n    res.setHeader(\"Content-Type\", \"text/event-stream; charset=utf-8\");\n    res.setHeader(\"Cache-Control\", \"no-cache, no-transform\");\n    // X-Accel-Buffering: no — disables nginx-style proxy response buffering\n    // for SSE (used by Cloudflare, AWS, GCP, and most corporate proxies).\n    res.setHeader(\"X-Accel-Buffering\", \"no\");\n    // Intentionally NOT setting:\n    //   - Connection: keep-alive   (HTTP/2 forbids it; Node manages keep-alive)\n    //   - Content-Encoding: none   (invalid value; can trigger 502/RST in\n    //                                strict intermediaries)\n    res.flushHeaders?.();\n  }\n\n  // write a single event to the response\n  writeEvent(res: IAppResponse, eventId: string, event: any): void {\n    if (res.writableEnded) return;\n\n    const eventType = StreamValidator.sanitizeEventType(event.type);\n    const eventData = JSON.stringify(event);\n\n    res.write(`id: ${eventId}\\n`);\n    res.write(`event: ${eventType}\\n`);\n    res.write(`data: ${eventData}\\n\\n`);\n  }\n  writeError(\n    res: IAppResponse,\n    eventId: string,\n    error: string,\n    code: SSEErrorCode = SSEErrorCode.INTERNAL_ERROR,\n  ): void {\n    if (res.writableEnded) return;\n\n    const errorData: SSEError = {\n      error,\n      code,\n    };\n\n    res.write(`id: ${eventId}\\n`);\n    res.write(`event: error\\n`);\n    res.write(`data: ${JSON.stringify(errorData)}\\n\\n`);\n  }\n\n  // write a buffered event for replay\n  writeBufferedEvent(res: IAppResponse, event: BufferedEvent): void {\n    if (res.writableEnded) return;\n\n    res.write(`id: ${event.id}\\n`);\n    res.write(`event: ${event.type}\\n`);\n    res.write(`data: ${event.data}\\n\\n`);\n  }\n\n  // write a buffer overflow warning\n  writeBufferOverflowWarning(res: IAppResponse, lastEventId: string): void {\n    if (res.writableEnded) return;\n\n    try {\n      res.write(`event: warning\\n`);\n      res.write(\n        `data: ${JSON.stringify({\n          warning: \"Buffer overflow detected - some events were lost\",\n          code: SSEWarningCode.BUFFER_OVERFLOW_RESTART,\n          lastEventId,\n        })}\\n\\n`,\n      );\n    } catch (_error) {\n      // ignore write errors - client will ignore this event\n    }\n  }\n\n  // start the heartbeat interval\n  startHeartbeat(\n    res: IAppResponse,\n    signal: AbortSignal,\n    interval?: number,\n  ): NodeJS.Timeout {\n    const heartbeatInterval = interval ?? streamDefaults.heartbeatInterval;\n\n    return setInterval(() => {\n      if (!signal.aborted && !res.writableEnded) {\n        try {\n          res.write(`: heartbeat\\n\\n`);\n        } catch (_error) {\n          // ignore write errors - client will ignore this event\n        }\n      }\n    }, heartbeatInterval);\n  }\n}\n"],"mappings":";;;;;AAUA,IAAa,YAAb,MAAuB;CAErB,aAAa,KAAyB;AACpC,MAAI,UAAU,gBAAgB,mCAAmC;AACjE,MAAI,UAAU,iBAAiB,yBAAyB;AAGxD,MAAI,UAAU,qBAAqB,KAAK;AAKxC,MAAI,gBAAgB;;CAItB,WAAW,KAAmB,SAAiB,OAAkB;AAC/D,MAAI,IAAI,cAAe;EAEvB,MAAM,YAAY,gBAAgB,kBAAkB,MAAM,KAAK;EAC/D,MAAM,YAAY,KAAK,UAAU,MAAM;AAEvC,MAAI,MAAM,OAAO,QAAQ,IAAI;AAC7B,MAAI,MAAM,UAAU,UAAU,IAAI;AAClC,MAAI,MAAM,SAAS,UAAU,MAAM;;CAErC,WACE,KACA,SACA,OACA,OAAqB,aAAa,gBAC5B;AACN,MAAI,IAAI,cAAe;EAEvB,MAAM,YAAsB;GAC1B;GACA;GACD;AAED,MAAI,MAAM,OAAO,QAAQ,IAAI;AAC7B,MAAI,MAAM,iBAAiB;AAC3B,MAAI,MAAM,SAAS,KAAK,UAAU,UAAU,CAAC,MAAM;;CAIrD,mBAAmB,KAAmB,OAA4B;AAChE,MAAI,IAAI,cAAe;AAEvB,MAAI,MAAM,OAAO,MAAM,GAAG,IAAI;AAC9B,MAAI,MAAM,UAAU,MAAM,KAAK,IAAI;AACnC,MAAI,MAAM,SAAS,MAAM,KAAK,MAAM;;CAItC,2BAA2B,KAAmB,aAA2B;AACvE,MAAI,IAAI,cAAe;AAEvB,MAAI;AACF,OAAI,MAAM,mBAAmB;AAC7B,OAAI,MACF,SAAS,KAAK,UAAU;IACtB,SAAS;IACT,MAAM,eAAe;IACrB;IACD,CAAC,CAAC,MACJ;WACM,QAAQ;;CAMnB,eACE,KACA,QACA,UACgB;EAChB,MAAM,oBAAoB,YAAY,eAAe;AAErD,SAAO,kBAAkB;AACvB,OAAI,CAAC,OAAO,WAAW,CAAC,IAAI,cAC1B,KAAI;AACF,QAAI,MAAM,kBAAkB;YACrB,QAAQ;KAIlB,kBAAkB"}
+{"version":3,"file":"sse-writer.js","names":[],"sources":["../../src/stream/sse-writer.ts"],"sourcesContent":["import type { IAppResponse } from \"shared\";\nimport { streamDefaults } from \"./defaults\";\nimport {\n  type BufferedEvent,\n  type SSEError,\n  SSEErrorCode,\n  SSEWarningCode,\n} from \"./types\";\nimport { StreamValidator } from \"./validator\";\n\nexport class SSEWriter {\n  // setup SSE headers\n  setupHeaders(res: IAppResponse): void {\n    res.setHeader(\"Content-Type\", \"text/event-stream; charset=utf-8\");\n    res.setHeader(\"Cache-Control\", \"no-cache, no-transform\");\n    // X-Accel-Buffering: no — disables nginx-style proxy response buffering\n    // for SSE (used by Cloudflare, AWS, GCP, and most corporate proxies).\n    res.setHeader(\"X-Accel-Buffering\", \"no\");\n    // Intentionally NOT setting:\n    //   - Connection: keep-alive   (HTTP/2 forbids it; Node manages keep-alive)\n    //   - Content-Encoding: none   (invalid value; can trigger 502/RST in\n    //                                strict intermediaries)\n    res.flushHeaders?.();\n  }\n\n  // write a single event to the response\n  writeEvent(res: IAppResponse, eventId: string, event: any): void {\n    if (res.writableEnded) return;\n\n    const eventType = StreamValidator.sanitizeEventType(event.type);\n    const eventData = JSON.stringify(event);\n\n    res.write(`id: ${eventId}\\n`);\n    res.write(`event: ${eventType}\\n`);\n    res.write(`data: ${eventData}\\n\\n`);\n  }\n  writeError(\n    res: IAppResponse,\n    eventId: string,\n    error: string,\n    code: SSEErrorCode = SSEErrorCode.INTERNAL_ERROR,\n    errorCode?: string,\n  ): void {\n    if (res.writableEnded) return;\n\n    const errorData: SSEError = {\n      error,\n      code,\n      ...(errorCode ? { errorCode } : {}),\n    };\n\n    res.write(`id: ${eventId}\\n`);\n    res.write(`event: error\\n`);\n    res.write(`data: ${JSON.stringify(errorData)}\\n\\n`);\n  }\n\n  // write a buffered event for replay\n  writeBufferedEvent(res: IAppResponse, event: BufferedEvent): void {\n    if (res.writableEnded) return;\n\n    res.write(`id: ${event.id}\\n`);\n    res.write(`event: ${event.type}\\n`);\n    res.write(`data: ${event.data}\\n\\n`);\n  }\n\n  // write a buffer overflow warning\n  writeBufferOverflowWarning(res: IAppResponse, lastEventId: string): void {\n    if (res.writableEnded) return;\n\n    try {\n      res.write(`event: warning\\n`);\n      res.write(\n        `data: ${JSON.stringify({\n          warning: \"Buffer overflow detected - some events were lost\",\n          code: SSEWarningCode.BUFFER_OVERFLOW_RESTART,\n          lastEventId,\n        })}\\n\\n`,\n      );\n    } catch (_error) {\n      // ignore write errors - client will ignore this event\n    }\n  }\n\n  // start the heartbeat interval\n  startHeartbeat(\n    res: IAppResponse,\n    signal: AbortSignal,\n    interval?: number,\n  ): NodeJS.Timeout {\n    const heartbeatInterval = interval ?? streamDefaults.heartbeatInterval;\n\n    return setInterval(() => {\n      if (!signal.aborted && !res.writableEnded) {\n        try {\n          res.write(`: heartbeat\\n\\n`);\n        } catch (_error) {\n          // ignore write errors - client will ignore this event\n        }\n      }\n    }, heartbeatInterval);\n  }\n}\n"],"mappings":";;;;;AAUA,IAAa,YAAb,MAAuB;CAErB,aAAa,KAAyB;AACpC,MAAI,UAAU,gBAAgB,mCAAmC;AACjE,MAAI,UAAU,iBAAiB,yBAAyB;AAGxD,MAAI,UAAU,qBAAqB,KAAK;AAKxC,MAAI,gBAAgB;;CAItB,WAAW,KAAmB,SAAiB,OAAkB;AAC/D,MAAI,IAAI,cAAe;EAEvB,MAAM,YAAY,gBAAgB,kBAAkB,MAAM,KAAK;EAC/D,MAAM,YAAY,KAAK,UAAU,MAAM;AAEvC,MAAI,MAAM,OAAO,QAAQ,IAAI;AAC7B,MAAI,MAAM,UAAU,UAAU,IAAI;AAClC,MAAI,MAAM,SAAS,UAAU,MAAM;;CAErC,WACE,KACA,SACA,OACA,OAAqB,aAAa,gBAClC,WACM;AACN,MAAI,IAAI,cAAe;EAEvB,MAAM,YAAsB;GAC1B;GACA;GACA,GAAI,YAAY,EAAE,WAAW,GAAG,EAAE;GACnC;AAED,MAAI,MAAM,OAAO,QAAQ,IAAI;AAC7B,MAAI,MAAM,iBAAiB;AAC3B,MAAI,MAAM,SAAS,KAAK,UAAU,UAAU,CAAC,MAAM;;CAIrD,mBAAmB,KAAmB,OAA4B;AAChE,MAAI,IAAI,cAAe;AAEvB,MAAI,MAAM,OAAO,MAAM,GAAG,IAAI;AAC9B,MAAI,MAAM,UAAU,MAAM,KAAK,IAAI;AACnC,MAAI,MAAM,SAAS,MAAM,KAAK,MAAM;;CAItC,2BAA2B,KAAmB,aAA2B;AACvE,MAAI,IAAI,cAAe;AAEvB,MAAI;AACF,OAAI,MAAM,mBAAmB;AAC7B,OAAI,MACF,SAAS,KAAK,UAAU;IACtB,SAAS;IACT,MAAM,eAAe;IACrB;IACD,CAAC,CAAC,MACJ;WACM,QAAQ;;CAMnB,eACE,KACA,QACA,UACgB;EAChB,MAAM,oBAAoB,YAAY,eAAe;AAErD,SAAO,kBAAkB;AACvB,OAAI,CAAC,OAAO,WAAW,CAAC,IAAI,cAC1B,KAAI;AACF,QAAI,MAAM,kBAAkB;YACrB,QAAQ;KAIlB,kBAAkB"}

package/dist/stream/stream-manager.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"stream-manager.d.ts","names":[],"sources":["../../src/stream/stream-manager.ts"],"mappings":";;;;;cAca,aAAA;EAAA,QACH,gBAAA;EAAA,QACA,cAAA;EAAA,QACA,SAAA;EAAA,QACA,YAAA;EAAA,QACA,SAAA;EAAA,QACA,iBAAA;cAEI,OAAA,GAAU,YAAA;EAahB,MAAA,CACJ,GAAA,EAAK,YAAA,EACL,OAAA,GAAU,MAAA,EAAQ,WAAA,KAAgB,cAAA,sBAClC,OAAA,GAAU,YAAA,EACV,QAAA,YACC,OAAA;EAsCH,QAAA,CAAA;EAaA,cAAA,CAAA;EAAA,QAKc,uBAAA;EAAA,QA4EA,gBAAA;EAAA,QA4EA,6BAAA;EAAA,QA6FN,eAAA;EAAA,QA6BA,yBAAA;EAAA,QAaA,wBAAA;EAAA,QAkBA,gBAAA;EAAA,QASA,mBAAA;EAAA,QAmBA,gBAAA;EAAA,QAQA,cAAA;EAAA,QAUA,gBAAA;AAAA"}
+{"version":3,"file":"stream-manager.d.ts","names":[],"sources":["../../src/stream/stream-manager.ts"],"mappings":";;;;;cAgBa,aAAA;EAAA,QACH,gBAAA;EAAA,QACA,cAAA;EAAA,QACA,SAAA;EAAA,QACA,YAAA;EAAA,QACA,SAAA;EAAA,QACA,iBAAA;cAEI,OAAA,GAAU,YAAA;EAahB,MAAA,CACJ,GAAA,EAAK,YAAA,EACL,OAAA,GAAU,MAAA,EAAQ,WAAA,KAAgB,cAAA,sBAClC,OAAA,GAAU,YAAA,EACV,QAAA,YACC,OAAA;EAsCH,QAAA,CAAA;EAaA,cAAA,CAAA;EAAA,QAKc,uBAAA;EAAA,QA4EA,gBAAA;EAAA,QA4EA,6BAAA;EAAA,QAkHN,eAAA;EAAA,QA6BA,yBAAA;EAAA,QAaA,wBAAA;EAAA,QAyBA,gBAAA;EAAA,QASA,mBAAA;EAAA,QAmBA,gBAAA;EAAA,QAQA,cAAA;EAAA,QAUA,gBAAA;AAAA"}

package/dist/stream/stream-manager.js

@@ -1,4 +1,6 @@
 import { createLogger } from "../logging/logger.js";
+import { AppKitError } from "../errors/base.js";
+import { ExecutionError } from "../errors/execution.js";
 import { EventRingBuffer } from "./buffers.js";
 import { streamDefaults } from "./defaults.js";
 import { SSEErrorCode } from "./types.js";
@@ -158,7 +160,9 @@ var StreamManager = class {
 				this._closeAllClients(streamEntry);
 				this._cleanupStream(streamEntry);
 			} catch (error) {
-				const errorMsg = error instanceof Error ? error.message : "Internal server error";
+				const rawMsg = error instanceof Error ? error.message : "Internal server error";
+				const clientMsg = error instanceof AppKitError ? error.clientMessage : "Internal server error";
+				const upstreamCode = error instanceof ExecutionError ? error.errorCode : void 0;
 				const errorEventId = randomUUID();
 				const errorCode = this._categorizeError(error);
 				if (errorCode === SSEErrorCode.STREAM_ABORTED) {
@@ -169,17 +173,19 @@ var StreamManager = class {
 					this._cleanupStream(streamEntry);
 					return;
 				}
-				logger.error("Stream execution failed: %s (code=%s)", errorMsg, errorCode);
+				logger.error("Stream execution failed: %s (code=%s upstreamCode=%s)", rawMsg, errorCode, upstreamCode ?? "n/a");
+				const payload = {
+					error: clientMsg,
+					code: errorCode
+				};
+				if (upstreamCode) payload.errorCode = upstreamCode;
 				streamEntry.eventBuffer.add({
 					id: errorEventId,
 					type: "error",
-					data: JSON.stringify({
-						error: errorMsg,
-						code: errorCode
-					}),
+					data: JSON.stringify(payload),
 					timestamp: Date.now()
 				});
-				this._broadcastErrorToClients(streamEntry, errorEventId, errorMsg, errorCode, true);
+				this._broadcastErrorToClients(streamEntry, errorEventId, clientMsg, errorCode, true, upstreamCode);
 				streamEntry.isCompleted = true;
 				this._clearGraceTimer(streamEntry);
 			}
@@ -203,9 +209,9 @@ var StreamManager = class {
 	_broadcastEventsToClients(streamEntry, eventId, event) {
 		for (const client of streamEntry.clients) if (!client.writableEnded) this.sseWriter.writeEvent(client, eventId, event);
 	}
-	_broadcastErrorToClients(streamEntry, eventId, errorMessage, errorCode, closeClients = false) {
+	_broadcastErrorToClients(streamEntry, eventId, errorMessage, errorCode, closeClients = false, upstreamCode) {
 		for (const client of streamEntry.clients) if (!client.writableEnded) {
-			this.sseWriter.writeError(client, eventId, errorMessage, errorCode);
+			this.sseWriter.writeError(client, eventId, errorMessage, errorCode, upstreamCode);
 			if (closeClients) client.end();
 		}
 	}

package/dist/stream/stream-manager.js.map

@@ -1 +1 @@
-{"version":3,"file":"stream-manager.js","names":[],"sources":["../../src/stream/stream-manager.ts"],"sourcesContent":["import { randomUUID } from \"node:crypto\";\nimport { context } from \"@opentelemetry/api\";\nimport type { IAppResponse, StreamConfig } from \"shared\";\nimport { createLogger } from \"../logging/logger\";\nimport { EventRingBuffer } from \"./buffers\";\nimport { streamDefaults } from \"./defaults\";\nimport { SSEWriter } from \"./sse-writer\";\nimport { StreamRegistry } from \"./stream-registry\";\nimport { SSEErrorCode, type StreamEntry, type StreamOperation } from \"./types\";\nimport { StreamValidator } from \"./validator\";\n\nconst logger = createLogger(\"stream\");\n\n// main entry point for Server-Sent events streaming\nexport class StreamManager {\n  private activeOperations: Set<StreamOperation>;\n  private streamRegistry: StreamRegistry;\n  private sseWriter: SSEWriter;\n  private maxEventSize: number;\n  private bufferTTL: number;\n  private disconnectGraceMs: number;\n\n  constructor(options?: StreamConfig) {\n    this.streamRegistry = new StreamRegistry(\n      options?.maxActiveStreams ?? streamDefaults.maxActiveStreams,\n    );\n    this.sseWriter = new SSEWriter();\n    this.maxEventSize = options?.maxEventSize ?? streamDefaults.maxEventSize;\n    this.bufferTTL = options?.bufferTTL ?? streamDefaults.bufferTTL;\n    this.disconnectGraceMs =\n      options?.disconnectGraceMs ?? streamDefaults.disconnectGraceMs;\n    this.activeOperations = new Set();\n  }\n\n  // main streaming method - handles new connection and reconnection\n  async stream(\n    res: IAppResponse,\n    handler: (signal: AbortSignal) => AsyncGenerator<any, void, unknown>,\n    options?: StreamConfig,\n    ownerKey?: string,\n  ): Promise<void> {\n    const { streamId } = options || {};\n\n    // check if response is already closed\n    if (res.writableEnded || res.destroyed) {\n      return;\n    }\n\n    // setup SSE headers\n    this.sseWriter.setupHeaders(res);\n\n    // handle reconnection\n    if (streamId && StreamValidator.validateStreamId(streamId)) {\n      const existingStream = this.streamRegistry.get(streamId);\n      if (existingStream) {\n        // Enforce per-user binding: the stream's owner key must match the\n        // requesting caller's owner key. This prevents cross-user stream\n        // takeover via guessed/leaked stream IDs (the SSE registry was\n        // previously a global lookup with no authorization step).\n        if (existingStream.ownerKey !== ownerKey) {\n          this.sseWriter.writeError(\n            res,\n            randomUUID(),\n            \"Stream not found or access denied\",\n            SSEErrorCode.STREAM_FORBIDDEN,\n          );\n          res.end();\n          return;\n        }\n        return this._attachToExistingStream(res, existingStream, options);\n      }\n    }\n\n    // if stream does not exist, create a new one\n    return this._createNewStream(res, handler, options, ownerKey);\n  }\n\n  // abort all active operations\n  abortAll(): void {\n    // pending disconnect-grace timers are cleared by streamRegistry.clear() below\n    this.activeOperations.forEach((operation) => {\n      if (operation.heartbeat) clearInterval(operation.heartbeat);\n      operation.controller.abort(\n        new DOMException(\"Server shutdown\", \"AbortError\"),\n      );\n    });\n    this.activeOperations.clear();\n    this.streamRegistry.clear();\n  }\n\n  // get the number of active operations\n  getActiveCount(): number {\n    return this.activeOperations.size;\n  }\n\n  // attach to existing stream\n  private async _attachToExistingStream(\n    res: IAppResponse,\n    streamEntry: StreamEntry,\n    options?: StreamConfig,\n  ): Promise<void> {\n    // handle reconnection - replay missed events\n    const lastEventId = res.req?.headers[\"last-event-id\"];\n\n    if (StreamValidator.validateEventId(lastEventId)) {\n      // cast to string after validation\n      const validEventId = lastEventId as string;\n      if (streamEntry.eventBuffer.has(validEventId)) {\n        const missedEvents =\n          streamEntry.eventBuffer.getEventsSince(validEventId);\n        // broadcast missed events to client\n        for (const event of missedEvents) {\n          if (options?.userSignal?.aborted) break;\n          this.sseWriter.writeBufferedEvent(res, event);\n        }\n      } else {\n        // buffer overflow - send warning\n        this.sseWriter.writeBufferOverflowWarning(res, validEventId);\n      }\n    }\n\n    // a reconnecting client cancels the pending disconnect-grace abort\n    this._clearGraceTimer(streamEntry);\n\n    // add client to stream entry\n    streamEntry.clients.add(res);\n    streamEntry.lastAccess = Date.now();\n\n    // start heartbeat\n    const combinedSignal = this._combineSignals(\n      streamEntry.abortController.signal,\n      options?.userSignal,\n    );\n    const heartbeat = this.sseWriter.startHeartbeat(res, combinedSignal);\n\n    // track operation\n    const streamOperation: StreamOperation = {\n      controller: streamEntry.abortController,\n      type: \"stream\",\n      heartbeat,\n    };\n    this.activeOperations.add(streamOperation);\n\n    // handle client disconnect\n    res.on(\"close\", () => {\n      clearInterval(heartbeat);\n      streamEntry.clients.delete(res);\n      this.activeOperations.delete(streamOperation);\n\n      // grace-abort instead of aborting now, so a reconnect can resume\n      if (streamEntry.clients.size === 0 && !streamEntry.isCompleted) {\n        this._scheduleGraceAbort(streamEntry);\n      }\n\n      // cleanup if stream is completed and no clients are connected\n      if (streamEntry.isCompleted && streamEntry.clients.size === 0) {\n        setTimeout(() => {\n          if (streamEntry.clients.size === 0) {\n            this.streamRegistry.remove(streamEntry.streamId);\n          }\n        }, this.bufferTTL);\n      }\n    });\n\n    // if stream is completed, close connection\n    if (streamEntry.isCompleted) {\n      res.end();\n      // cleanup operation\n      this.activeOperations.delete(streamOperation);\n      clearInterval(heartbeat);\n    }\n  }\n  private async _createNewStream(\n    res: IAppResponse,\n    handler: (signal: AbortSignal) => AsyncGenerator<any, void, unknown>,\n    options?: StreamConfig,\n    ownerKey?: string,\n  ): Promise<void> {\n    const streamId = options?.streamId ?? randomUUID();\n\n    // abort stream if response is closed\n    if (res.writableEnded || res.destroyed) {\n      return;\n    }\n\n    const abortController = new AbortController();\n\n    // create event buffer\n    const eventBuffer = new EventRingBuffer(\n      options?.bufferSize ?? streamDefaults.bufferSize,\n    );\n\n    // setup signals and heartbeat\n    const combinedSignal = this._combineSignals(\n      abortController.signal,\n      options?.userSignal,\n    );\n    const heartbeat = this.sseWriter.startHeartbeat(res, combinedSignal);\n\n    // capture the current trace context at stream creation time\n    const traceContext = context.active();\n\n    // abort stream if response is closed\n    if (res.writableEnded || res.destroyed) {\n      clearInterval(heartbeat);\n      return;\n    }\n\n    // create stream entry\n    const streamEntry: StreamEntry = {\n      streamId,\n      ownerKey,\n      generator: handler(combinedSignal),\n      eventBuffer,\n      clients: new Set([res]),\n      isCompleted: false,\n      lastAccess: Date.now(),\n      abortController,\n      traceContext,\n    };\n    this.streamRegistry.add(streamEntry);\n\n    // track operation\n    const streamOperation: StreamOperation = {\n      controller: abortController,\n      type: \"stream\",\n      heartbeat,\n    };\n    this.activeOperations.add(streamOperation);\n\n    res.on(\"close\", () => {\n      clearInterval(heartbeat);\n      this.activeOperations.delete(streamOperation);\n      streamEntry.clients.delete(res);\n\n      // grace-abort instead of aborting now, so a reconnect can resume\n      if (streamEntry.clients.size === 0 && !streamEntry.isCompleted) {\n        this._scheduleGraceAbort(streamEntry);\n      }\n    });\n\n    await this._processGeneratorInBackground(streamEntry);\n\n    // cleanup\n    clearInterval(heartbeat);\n    this.activeOperations.delete(streamOperation);\n  }\n\n  private async _processGeneratorInBackground(\n    streamEntry: StreamEntry,\n  ): Promise<void> {\n    // run the entire generator processing within the captured trace context\n    return context.with(streamEntry.traceContext, async () => {\n      try {\n        // retrieve all events from generator\n        for await (const event of streamEntry.generator) {\n          if (streamEntry.abortController.signal.aborted) break;\n          const eventId = randomUUID();\n          const eventData = JSON.stringify(event);\n\n          // validate event size\n          if (eventData.length > this.maxEventSize) {\n            const errorMsg = `Event exceeds max size of ${this.maxEventSize} bytes`;\n            const errorCode = SSEErrorCode.INVALID_REQUEST;\n            // broadcast error to all connected clients\n            this._broadcastErrorToClients(\n              streamEntry,\n              eventId,\n              errorMsg,\n              errorCode,\n            );\n            continue;\n          }\n\n          // buffer event for reconnection\n          streamEntry.eventBuffer.add({\n            id: eventId,\n            type: event.type,\n            data: eventData,\n            timestamp: Date.now(),\n          });\n\n          // broadcast to all connected clients\n          this._broadcastEventsToClients(streamEntry, eventId, event);\n          streamEntry.lastAccess = Date.now();\n        }\n\n        streamEntry.isCompleted = true;\n\n        // no late grace abort should fire on a completed stream\n        this._clearGraceTimer(streamEntry);\n\n        // close all clients\n        this._closeAllClients(streamEntry);\n\n        // cleanup if no clients are connected\n        this._cleanupStream(streamEntry);\n      } catch (error) {\n        const errorMsg =\n          error instanceof Error ? error.message : \"Internal server error\";\n        const errorEventId = randomUUID();\n        const errorCode = this._categorizeError(error);\n\n        // client cancellation is a normal control-flow signal, not a failure\n        if (errorCode === SSEErrorCode.STREAM_ABORTED) {\n          logger.info(\"Stream aborted by client (code=%s)\", errorCode);\n          streamEntry.isCompleted = true;\n          this._clearGraceTimer(streamEntry);\n          this._closeAllClients(streamEntry);\n          this._cleanupStream(streamEntry);\n          return;\n        }\n\n        logger.error(\n          \"Stream execution failed: %s (code=%s)\",\n          errorMsg,\n          errorCode,\n        );\n\n        // buffer error event\n        streamEntry.eventBuffer.add({\n          id: errorEventId,\n          type: \"error\",\n          data: JSON.stringify({ error: errorMsg, code: errorCode }),\n          timestamp: Date.now(),\n        });\n\n        // send error event to all connected clients\n        this._broadcastErrorToClients(\n          streamEntry,\n          errorEventId,\n          errorMsg,\n          errorCode,\n          true,\n        );\n        streamEntry.isCompleted = true;\n        this._clearGraceTimer(streamEntry);\n      }\n    });\n  }\n\n  private _combineSignals(\n    internalSignal?: AbortSignal,\n    userSignal?: AbortSignal,\n  ): AbortSignal {\n    if (!userSignal) return internalSignal || new AbortController().signal;\n\n    const signals = [internalSignal, userSignal].filter(\n      Boolean,\n    ) as AbortSignal[];\n    const controller = new AbortController();\n\n    signals.forEach((signal) => {\n      if (signal?.aborted) {\n        controller.abort(signal.reason);\n        return;\n      }\n\n      signal?.addEventListener(\n        \"abort\",\n        () => {\n          controller.abort(signal.reason);\n        },\n        { once: true },\n      );\n    });\n    return controller.signal;\n  }\n\n  // broadcast events to all connected clients\n  private _broadcastEventsToClients(\n    streamEntry: StreamEntry,\n    eventId: string,\n    event: any,\n  ): void {\n    for (const client of streamEntry.clients) {\n      if (!client.writableEnded) {\n        this.sseWriter.writeEvent(client, eventId, event);\n      }\n    }\n  }\n\n  // broadcast error to all connected clients\n  private _broadcastErrorToClients(\n    streamEntry: StreamEntry,\n    eventId: string,\n    errorMessage: string,\n    errorCode: SSEErrorCode,\n    closeClients: boolean = false,\n  ): void {\n    for (const client of streamEntry.clients) {\n      if (!client.writableEnded) {\n        this.sseWriter.writeError(client, eventId, errorMessage, errorCode);\n        if (closeClients) {\n          client.end();\n        }\n      }\n    }\n  }\n\n  // close all connected clients\n  private _closeAllClients(streamEntry: StreamEntry): void {\n    for (const client of streamEntry.clients) {\n      if (!client.writableEnded) {\n        client.end();\n      }\n    }\n  }\n\n  // abort the generator after the grace window unless a client reconnects first\n  private _scheduleGraceAbort(streamEntry: StreamEntry): void {\n    // clear any existing timer to avoid stacking\n    this._clearGraceTimer(streamEntry);\n\n    const timer = setTimeout(() => {\n      streamEntry.disconnectGraceTimer = undefined;\n      if (streamEntry.clients.size === 0 && !streamEntry.isCompleted) {\n        streamEntry.abortController.abort(\n          new DOMException(\"Client disconnected (grace expired)\", \"AbortError\"),\n        );\n      }\n    }, this.disconnectGraceMs);\n\n    // never keep the process alive solely for a grace timer\n    timer.unref?.();\n    streamEntry.disconnectGraceTimer = timer;\n  }\n\n  // clear a pending disconnect-grace timer, if any\n  private _clearGraceTimer(streamEntry: StreamEntry): void {\n    if (streamEntry.disconnectGraceTimer) {\n      clearTimeout(streamEntry.disconnectGraceTimer);\n      streamEntry.disconnectGraceTimer = undefined;\n    }\n  }\n\n  // cleanup stream if no clients are connected\n  private _cleanupStream(streamEntry: StreamEntry): void {\n    if (streamEntry.clients.size === 0) {\n      setTimeout(() => {\n        if (streamEntry.clients.size === 0) {\n          this.streamRegistry.remove(streamEntry.streamId);\n        }\n      }, this.bufferTTL);\n    }\n  }\n\n  private _categorizeError(error: unknown): SSEErrorCode {\n    if (error instanceof Error) {\n      const message = error.message.toLowerCase();\n      if (message.includes(\"timeout\") || message.includes(\"timed out\")) {\n        return SSEErrorCode.TIMEOUT;\n      }\n\n      if (message.includes(\"unavailable\") || message.includes(\"econnrefused\")) {\n        return SSEErrorCode.TEMPORARY_UNAVAILABLE;\n      }\n\n      if (error.name === \"AbortError\") {\n        return SSEErrorCode.STREAM_ABORTED;\n      }\n\n      // Defense-in-depth: upstream layers (SQL client, cache) may wrap an\n      // AbortError into ExecutionError, losing `name` but keeping the message.\n      if (\n        message.includes(\"operation was aborted\") ||\n        message.includes(\"the request was aborted\") ||\n        message.includes(\"statement was canceled\")\n      ) {\n        return SSEErrorCode.STREAM_ABORTED;\n      }\n\n      // Detect upstream API errors (e.g., from Databricks SDK ApiError)\n      if (\n        \"statusCode\" in error &&\n        typeof (error as any).statusCode === \"number\"\n      ) {\n        return SSEErrorCode.UPSTREAM_ERROR;\n      }\n    }\n\n    return SSEErrorCode.INTERNAL_ERROR;\n  }\n}\n"],"mappings":";;;;;;;;;;;AAWA,MAAM,SAAS,aAAa,SAAS;AAGrC,IAAa,gBAAb,MAA2B;CACzB,AAAQ;CACR,AAAQ;CACR,AAAQ;CACR,AAAQ;CACR,AAAQ;CACR,AAAQ;CAER,YAAY,SAAwB;AAClC,OAAK,iBAAiB,IAAI,eACxB,SAAS,oBAAoB,eAAe,iBAC7C;AACD,OAAK,YAAY,IAAI,WAAW;AAChC,OAAK,eAAe,SAAS,gBAAgB,eAAe;AAC5D,OAAK,YAAY,SAAS,aAAa,eAAe;AACtD,OAAK,oBACH,SAAS,qBAAqB,eAAe;AAC/C,OAAK,mCAAmB,IAAI,KAAK;;CAInC,MAAM,OACJ,KACA,SACA,SACA,UACe;EACf,MAAM,EAAE,aAAa,WAAW,EAAE;AAGlC,MAAI,IAAI,iBAAiB,IAAI,UAC3B;AAIF,OAAK,UAAU,aAAa,IAAI;AAGhC,MAAI,YAAY,gBAAgB,iBAAiB,SAAS,EAAE;GAC1D,MAAM,iBAAiB,KAAK,eAAe,IAAI,SAAS;AACxD,OAAI,gBAAgB;AAKlB,QAAI,eAAe,aAAa,UAAU;AACxC,UAAK,UAAU,WACb,KACA,YAAY,EACZ,qCACA,aAAa,iBACd;AACD,SAAI,KAAK;AACT;;AAEF,WAAO,KAAK,wBAAwB,KAAK,gBAAgB,QAAQ;;;AAKrE,SAAO,KAAK,iBAAiB,KAAK,SAAS,SAAS,SAAS;;CAI/D,WAAiB;AAEf,OAAK,iBAAiB,SAAS,cAAc;AAC3C,OAAI,UAAU,UAAW,eAAc,UAAU,UAAU;AAC3D,aAAU,WAAW,MACnB,IAAI,aAAa,mBAAmB,aAAa,CAClD;IACD;AACF,OAAK,iBAAiB,OAAO;AAC7B,OAAK,eAAe,OAAO;;CAI7B,iBAAyB;AACvB,SAAO,KAAK,iBAAiB;;CAI/B,MAAc,wBACZ,KACA,aACA,SACe;EAEf,MAAM,cAAc,IAAI,KAAK,QAAQ;AAErC,MAAI,gBAAgB,gBAAgB,YAAY,EAAE;GAEhD,MAAM,eAAe;AACrB,OAAI,YAAY,YAAY,IAAI,aAAa,EAAE;IAC7C,MAAM,eACJ,YAAY,YAAY,eAAe,aAAa;AAEtD,SAAK,MAAM,SAAS,cAAc;AAChC,SAAI,SAAS,YAAY,QAAS;AAClC,UAAK,UAAU,mBAAmB,KAAK,MAAM;;SAI/C,MAAK,UAAU,2BAA2B,KAAK,aAAa;;AAKhE,OAAK,iBAAiB,YAAY;AAGlC,cAAY,QAAQ,IAAI,IAAI;AAC5B,cAAY,aAAa,KAAK,KAAK;EAGnC,MAAM,iBAAiB,KAAK,gBAC1B,YAAY,gBAAgB,QAC5B,SAAS,WACV;EACD,MAAM,YAAY,KAAK,UAAU,eAAe,KAAK,eAAe;EAGpE,MAAM,kBAAmC;GACvC,YAAY,YAAY;GACxB,MAAM;GACN;GACD;AACD,OAAK,iBAAiB,IAAI,gBAAgB;AAG1C,MAAI,GAAG,eAAe;AACpB,iBAAc,UAAU;AACxB,eAAY,QAAQ,OAAO,IAAI;AAC/B,QAAK,iBAAiB,OAAO,gBAAgB;AAG7C,OAAI,YAAY,QAAQ,SAAS,KAAK,CAAC,YAAY,YACjD,MAAK,oBAAoB,YAAY;AAIvC,OAAI,YAAY,eAAe,YAAY,QAAQ,SAAS,EAC1D,kBAAiB;AACf,QAAI,YAAY,QAAQ,SAAS,EAC/B,MAAK,eAAe,OAAO,YAAY,SAAS;MAEjD,KAAK,UAAU;IAEpB;AAGF,MAAI,YAAY,aAAa;AAC3B,OAAI,KAAK;AAET,QAAK,iBAAiB,OAAO,gBAAgB;AAC7C,iBAAc,UAAU;;;CAG5B,MAAc,iBACZ,KACA,SACA,SACA,UACe;EACf,MAAM,WAAW,SAAS,YAAY,YAAY;AAGlD,MAAI,IAAI,iBAAiB,IAAI,UAC3B;EAGF,MAAM,kBAAkB,IAAI,iBAAiB;EAG7C,MAAM,cAAc,IAAI,gBACtB,SAAS,cAAc,eAAe,WACvC;EAGD,MAAM,iBAAiB,KAAK,gBAC1B,gBAAgB,QAChB,SAAS,WACV;EACD,MAAM,YAAY,KAAK,UAAU,eAAe,KAAK,eAAe;EAGpE,MAAM,eAAe,QAAQ,QAAQ;AAGrC,MAAI,IAAI,iBAAiB,IAAI,WAAW;AACtC,iBAAc,UAAU;AACxB;;EAIF,MAAM,cAA2B;GAC/B;GACA;GACA,WAAW,QAAQ,eAAe;GAClC;GACA,SAAS,IAAI,IAAI,CAAC,IAAI,CAAC;GACvB,aAAa;GACb,YAAY,KAAK,KAAK;GACtB;GACA;GACD;AACD,OAAK,eAAe,IAAI,YAAY;EAGpC,MAAM,kBAAmC;GACvC,YAAY;GACZ,MAAM;GACN;GACD;AACD,OAAK,iBAAiB,IAAI,gBAAgB;AAE1C,MAAI,GAAG,eAAe;AACpB,iBAAc,UAAU;AACxB,QAAK,iBAAiB,OAAO,gBAAgB;AAC7C,eAAY,QAAQ,OAAO,IAAI;AAG/B,OAAI,YAAY,QAAQ,SAAS,KAAK,CAAC,YAAY,YACjD,MAAK,oBAAoB,YAAY;IAEvC;AAEF,QAAM,KAAK,8BAA8B,YAAY;AAGrD,gBAAc,UAAU;AACxB,OAAK,iBAAiB,OAAO,gBAAgB;;CAG/C,MAAc,8BACZ,aACe;AAEf,SAAO,QAAQ,KAAK,YAAY,cAAc,YAAY;AACxD,OAAI;AAEF,eAAW,MAAM,SAAS,YAAY,WAAW;AAC/C,SAAI,YAAY,gBAAgB,OAAO,QAAS;KAChD,MAAM,UAAU,YAAY;KAC5B,MAAM,YAAY,KAAK,UAAU,MAAM;AAGvC,SAAI,UAAU,SAAS,KAAK,cAAc;MACxC,MAAM,WAAW,6BAA6B,KAAK,aAAa;MAChE,MAAM,YAAY,aAAa;AAE/B,WAAK,yBACH,aACA,SACA,UACA,UACD;AACD;;AAIF,iBAAY,YAAY,IAAI;MAC1B,IAAI;MACJ,MAAM,MAAM;MACZ,MAAM;MACN,WAAW,KAAK,KAAK;MACtB,CAAC;AAGF,UAAK,0BAA0B,aAAa,SAAS,MAAM;AAC3D,iBAAY,aAAa,KAAK,KAAK;;AAGrC,gBAAY,cAAc;AAG1B,SAAK,iBAAiB,YAAY;AAGlC,SAAK,iBAAiB,YAAY;AAGlC,SAAK,eAAe,YAAY;YACzB,OAAO;IACd,MAAM,WACJ,iBAAiB,QAAQ,MAAM,UAAU;IAC3C,MAAM,eAAe,YAAY;IACjC,MAAM,YAAY,KAAK,iBAAiB,MAAM;AAG9C,QAAI,cAAc,aAAa,gBAAgB;AAC7C,YAAO,KAAK,sCAAsC,UAAU;AAC5D,iBAAY,cAAc;AAC1B,UAAK,iBAAiB,YAAY;AAClC,UAAK,iBAAiB,YAAY;AAClC,UAAK,eAAe,YAAY;AAChC;;AAGF,WAAO,MACL,yCACA,UACA,UACD;AAGD,gBAAY,YAAY,IAAI;KAC1B,IAAI;KACJ,MAAM;KACN,MAAM,KAAK,UAAU;MAAE,OAAO;MAAU,MAAM;MAAW,CAAC;KAC1D,WAAW,KAAK,KAAK;KACtB,CAAC;AAGF,SAAK,yBACH,aACA,cACA,UACA,WACA,KACD;AACD,gBAAY,cAAc;AAC1B,SAAK,iBAAiB,YAAY;;IAEpC;;CAGJ,AAAQ,gBACN,gBACA,YACa;AACb,MAAI,CAAC,WAAY,QAAO,kBAAkB,IAAI,iBAAiB,CAAC;EAEhE,MAAM,UAAU,CAAC,gBAAgB,WAAW,CAAC,OAC3C,QACD;EACD,MAAM,aAAa,IAAI,iBAAiB;AAExC,UAAQ,SAAS,WAAW;AAC1B,OAAI,QAAQ,SAAS;AACnB,eAAW,MAAM,OAAO,OAAO;AAC/B;;AAGF,WAAQ,iBACN,eACM;AACJ,eAAW,MAAM,OAAO,OAAO;MAEjC,EAAE,MAAM,MAAM,CACf;IACD;AACF,SAAO,WAAW;;CAIpB,AAAQ,0BACN,aACA,SACA,OACM;AACN,OAAK,MAAM,UAAU,YAAY,QAC/B,KAAI,CAAC,OAAO,cACV,MAAK,UAAU,WAAW,QAAQ,SAAS,MAAM;;CAMvD,AAAQ,yBACN,aACA,SACA,cACA,WACA,eAAwB,OAClB;AACN,OAAK,MAAM,UAAU,YAAY,QAC/B,KAAI,CAAC,OAAO,eAAe;AACzB,QAAK,UAAU,WAAW,QAAQ,SAAS,cAAc,UAAU;AACnE,OAAI,aACF,QAAO,KAAK;;;CAOpB,AAAQ,iBAAiB,aAAgC;AACvD,OAAK,MAAM,UAAU,YAAY,QAC/B,KAAI,CAAC,OAAO,cACV,QAAO,KAAK;;CAMlB,AAAQ,oBAAoB,aAAgC;AAE1D,OAAK,iBAAiB,YAAY;EAElC,MAAM,QAAQ,iBAAiB;AAC7B,eAAY,uBAAuB;AACnC,OAAI,YAAY,QAAQ,SAAS,KAAK,CAAC,YAAY,YACjD,aAAY,gBAAgB,MAC1B,IAAI,aAAa,uCAAuC,aAAa,CACtE;KAEF,KAAK,kBAAkB;AAG1B,QAAM,SAAS;AACf,cAAY,uBAAuB;;CAIrC,AAAQ,iBAAiB,aAAgC;AACvD,MAAI,YAAY,sBAAsB;AACpC,gBAAa,YAAY,qBAAqB;AAC9C,eAAY,uBAAuB;;;CAKvC,AAAQ,eAAe,aAAgC;AACrD,MAAI,YAAY,QAAQ,SAAS,EAC/B,kBAAiB;AACf,OAAI,YAAY,QAAQ,SAAS,EAC/B,MAAK,eAAe,OAAO,YAAY,SAAS;KAEjD,KAAK,UAAU;;CAItB,AAAQ,iBAAiB,OAA8B;AACrD,MAAI,iBAAiB,OAAO;GAC1B,MAAM,UAAU,MAAM,QAAQ,aAAa;AAC3C,OAAI,QAAQ,SAAS,UAAU,IAAI,QAAQ,SAAS,YAAY,CAC9D,QAAO,aAAa;AAGtB,OAAI,QAAQ,SAAS,cAAc,IAAI,QAAQ,SAAS,eAAe,CACrE,QAAO,aAAa;AAGtB,OAAI,MAAM,SAAS,aACjB,QAAO,aAAa;AAKtB,OACE,QAAQ,SAAS,wBAAwB,IACzC,QAAQ,SAAS,0BAA0B,IAC3C,QAAQ,SAAS,yBAAyB,CAE1C,QAAO,aAAa;AAItB,OACE,gBAAgB,SAChB,OAAQ,MAAc,eAAe,SAErC,QAAO,aAAa;;AAIxB,SAAO,aAAa"}
+{"version":3,"file":"stream-manager.js","names":[],"sources":["../../src/stream/stream-manager.ts"],"sourcesContent":["import { randomUUID } from \"node:crypto\";\nimport { context } from \"@opentelemetry/api\";\nimport type { IAppResponse, StreamConfig } from \"shared\";\nimport { AppKitError } from \"../errors/base\";\nimport { ExecutionError } from \"../errors/execution\";\nimport { createLogger } from \"../logging/logger\";\nimport { EventRingBuffer } from \"./buffers\";\nimport { streamDefaults } from \"./defaults\";\nimport { SSEWriter } from \"./sse-writer\";\nimport { StreamRegistry } from \"./stream-registry\";\nimport { SSEErrorCode, type StreamEntry, type StreamOperation } from \"./types\";\nimport { StreamValidator } from \"./validator\";\n\nconst logger = createLogger(\"stream\");\n\n// main entry point for Server-Sent events streaming\nexport class StreamManager {\n  private activeOperations: Set<StreamOperation>;\n  private streamRegistry: StreamRegistry;\n  private sseWriter: SSEWriter;\n  private maxEventSize: number;\n  private bufferTTL: number;\n  private disconnectGraceMs: number;\n\n  constructor(options?: StreamConfig) {\n    this.streamRegistry = new StreamRegistry(\n      options?.maxActiveStreams ?? streamDefaults.maxActiveStreams,\n    );\n    this.sseWriter = new SSEWriter();\n    this.maxEventSize = options?.maxEventSize ?? streamDefaults.maxEventSize;\n    this.bufferTTL = options?.bufferTTL ?? streamDefaults.bufferTTL;\n    this.disconnectGraceMs =\n      options?.disconnectGraceMs ?? streamDefaults.disconnectGraceMs;\n    this.activeOperations = new Set();\n  }\n\n  // main streaming method - handles new connection and reconnection\n  async stream(\n    res: IAppResponse,\n    handler: (signal: AbortSignal) => AsyncGenerator<any, void, unknown>,\n    options?: StreamConfig,\n    ownerKey?: string,\n  ): Promise<void> {\n    const { streamId } = options || {};\n\n    // check if response is already closed\n    if (res.writableEnded || res.destroyed) {\n      return;\n    }\n\n    // setup SSE headers\n    this.sseWriter.setupHeaders(res);\n\n    // handle reconnection\n    if (streamId && StreamValidator.validateStreamId(streamId)) {\n      const existingStream = this.streamRegistry.get(streamId);\n      if (existingStream) {\n        // Enforce per-user binding: the stream's owner key must match the\n        // requesting caller's owner key. This prevents cross-user stream\n        // takeover via guessed/leaked stream IDs (the SSE registry was\n        // previously a global lookup with no authorization step).\n        if (existingStream.ownerKey !== ownerKey) {\n          this.sseWriter.writeError(\n            res,\n            randomUUID(),\n            \"Stream not found or access denied\",\n            SSEErrorCode.STREAM_FORBIDDEN,\n          );\n          res.end();\n          return;\n        }\n        return this._attachToExistingStream(res, existingStream, options);\n      }\n    }\n\n    // if stream does not exist, create a new one\n    return this._createNewStream(res, handler, options, ownerKey);\n  }\n\n  // abort all active operations\n  abortAll(): void {\n    // pending disconnect-grace timers are cleared by streamRegistry.clear() below\n    this.activeOperations.forEach((operation) => {\n      if (operation.heartbeat) clearInterval(operation.heartbeat);\n      operation.controller.abort(\n        new DOMException(\"Server shutdown\", \"AbortError\"),\n      );\n    });\n    this.activeOperations.clear();\n    this.streamRegistry.clear();\n  }\n\n  // get the number of active operations\n  getActiveCount(): number {\n    return this.activeOperations.size;\n  }\n\n  // attach to existing stream\n  private async _attachToExistingStream(\n    res: IAppResponse,\n    streamEntry: StreamEntry,\n    options?: StreamConfig,\n  ): Promise<void> {\n    // handle reconnection - replay missed events\n    const lastEventId = res.req?.headers[\"last-event-id\"];\n\n    if (StreamValidator.validateEventId(lastEventId)) {\n      // cast to string after validation\n      const validEventId = lastEventId as string;\n      if (streamEntry.eventBuffer.has(validEventId)) {\n        const missedEvents =\n          streamEntry.eventBuffer.getEventsSince(validEventId);\n        // broadcast missed events to client\n        for (const event of missedEvents) {\n          if (options?.userSignal?.aborted) break;\n          this.sseWriter.writeBufferedEvent(res, event);\n        }\n      } else {\n        // buffer overflow - send warning\n        this.sseWriter.writeBufferOverflowWarning(res, validEventId);\n      }\n    }\n\n    // a reconnecting client cancels the pending disconnect-grace abort\n    this._clearGraceTimer(streamEntry);\n\n    // add client to stream entry\n    streamEntry.clients.add(res);\n    streamEntry.lastAccess = Date.now();\n\n    // start heartbeat\n    const combinedSignal = this._combineSignals(\n      streamEntry.abortController.signal,\n      options?.userSignal,\n    );\n    const heartbeat = this.sseWriter.startHeartbeat(res, combinedSignal);\n\n    // track operation\n    const streamOperation: StreamOperation = {\n      controller: streamEntry.abortController,\n      type: \"stream\",\n      heartbeat,\n    };\n    this.activeOperations.add(streamOperation);\n\n    // handle client disconnect\n    res.on(\"close\", () => {\n      clearInterval(heartbeat);\n      streamEntry.clients.delete(res);\n      this.activeOperations.delete(streamOperation);\n\n      // grace-abort instead of aborting now, so a reconnect can resume\n      if (streamEntry.clients.size === 0 && !streamEntry.isCompleted) {\n        this._scheduleGraceAbort(streamEntry);\n      }\n\n      // cleanup if stream is completed and no clients are connected\n      if (streamEntry.isCompleted && streamEntry.clients.size === 0) {\n        setTimeout(() => {\n          if (streamEntry.clients.size === 0) {\n            this.streamRegistry.remove(streamEntry.streamId);\n          }\n        }, this.bufferTTL);\n      }\n    });\n\n    // if stream is completed, close connection\n    if (streamEntry.isCompleted) {\n      res.end();\n      // cleanup operation\n      this.activeOperations.delete(streamOperation);\n      clearInterval(heartbeat);\n    }\n  }\n  private async _createNewStream(\n    res: IAppResponse,\n    handler: (signal: AbortSignal) => AsyncGenerator<any, void, unknown>,\n    options?: StreamConfig,\n    ownerKey?: string,\n  ): Promise<void> {\n    const streamId = options?.streamId ?? randomUUID();\n\n    // abort stream if response is closed\n    if (res.writableEnded || res.destroyed) {\n      return;\n    }\n\n    const abortController = new AbortController();\n\n    // create event buffer\n    const eventBuffer = new EventRingBuffer(\n      options?.bufferSize ?? streamDefaults.bufferSize,\n    );\n\n    // setup signals and heartbeat\n    const combinedSignal = this._combineSignals(\n      abortController.signal,\n      options?.userSignal,\n    );\n    const heartbeat = this.sseWriter.startHeartbeat(res, combinedSignal);\n\n    // capture the current trace context at stream creation time\n    const traceContext = context.active();\n\n    // abort stream if response is closed\n    if (res.writableEnded || res.destroyed) {\n      clearInterval(heartbeat);\n      return;\n    }\n\n    // create stream entry\n    const streamEntry: StreamEntry = {\n      streamId,\n      ownerKey,\n      generator: handler(combinedSignal),\n      eventBuffer,\n      clients: new Set([res]),\n      isCompleted: false,\n      lastAccess: Date.now(),\n      abortController,\n      traceContext,\n    };\n    this.streamRegistry.add(streamEntry);\n\n    // track operation\n    const streamOperation: StreamOperation = {\n      controller: abortController,\n      type: \"stream\",\n      heartbeat,\n    };\n    this.activeOperations.add(streamOperation);\n\n    res.on(\"close\", () => {\n      clearInterval(heartbeat);\n      this.activeOperations.delete(streamOperation);\n      streamEntry.clients.delete(res);\n\n      // grace-abort instead of aborting now, so a reconnect can resume\n      if (streamEntry.clients.size === 0 && !streamEntry.isCompleted) {\n        this._scheduleGraceAbort(streamEntry);\n      }\n    });\n\n    await this._processGeneratorInBackground(streamEntry);\n\n    // cleanup\n    clearInterval(heartbeat);\n    this.activeOperations.delete(streamOperation);\n  }\n\n  private async _processGeneratorInBackground(\n    streamEntry: StreamEntry,\n  ): Promise<void> {\n    // run the entire generator processing within the captured trace context\n    return context.with(streamEntry.traceContext, async () => {\n      try {\n        // retrieve all events from generator\n        for await (const event of streamEntry.generator) {\n          if (streamEntry.abortController.signal.aborted) break;\n          const eventId = randomUUID();\n          const eventData = JSON.stringify(event);\n\n          // validate event size\n          if (eventData.length > this.maxEventSize) {\n            const errorMsg = `Event exceeds max size of ${this.maxEventSize} bytes`;\n            const errorCode = SSEErrorCode.INVALID_REQUEST;\n            // broadcast error to all connected clients\n            this._broadcastErrorToClients(\n              streamEntry,\n              eventId,\n              errorMsg,\n              errorCode,\n            );\n            continue;\n          }\n\n          // buffer event for reconnection\n          streamEntry.eventBuffer.add({\n            id: eventId,\n            type: event.type,\n            data: eventData,\n            timestamp: Date.now(),\n          });\n\n          // broadcast to all connected clients\n          this._broadcastEventsToClients(streamEntry, eventId, event);\n          streamEntry.lastAccess = Date.now();\n        }\n\n        streamEntry.isCompleted = true;\n\n        // no late grace abort should fire on a completed stream\n        this._clearGraceTimer(streamEntry);\n\n        // close all clients\n        this._closeAllClients(streamEntry);\n\n        // cleanup if no clients are connected\n        this._cleanupStream(streamEntry);\n      } catch (error) {\n        // Two distinct messages: a *raw* one for server-side logs (full\n        // detail, statement fragments, correlation IDs) and a *client*\n        // one for the SSE payload (sanitized, stable, safe to render in\n        // a UI). Mixing them leaks upstream wording to anyone connected\n        // to the stream — see CWE-209.\n        const rawMsg =\n          error instanceof Error ? error.message : \"Internal server error\";\n        const clientMsg =\n          error instanceof AppKitError\n            ? error.clientMessage\n            : \"Internal server error\";\n        // Upstream structured code (e.g. RESULT_TOO_LARGE_FOR_JSON_FALLBACK,\n        // NOT_IMPLEMENTED). UI should branch on this, not on `error`.\n        const upstreamCode =\n          error instanceof ExecutionError ? error.errorCode : undefined;\n        const errorEventId = randomUUID();\n        const errorCode = this._categorizeError(error);\n\n        // client cancellation is a normal control-flow signal, not a failure\n        if (errorCode === SSEErrorCode.STREAM_ABORTED) {\n          logger.info(\"Stream aborted by client (code=%s)\", errorCode);\n          streamEntry.isCompleted = true;\n          this._clearGraceTimer(streamEntry);\n          this._closeAllClients(streamEntry);\n          this._cleanupStream(streamEntry);\n          return;\n        }\n\n        logger.error(\n          \"Stream execution failed: %s (code=%s upstreamCode=%s)\",\n          rawMsg,\n          errorCode,\n          upstreamCode ?? \"n/a\",\n        );\n\n        const payload: Record<string, unknown> = {\n          error: clientMsg,\n          code: errorCode,\n        };\n        if (upstreamCode) payload.errorCode = upstreamCode;\n\n        // buffer error event\n        streamEntry.eventBuffer.add({\n          id: errorEventId,\n          type: \"error\",\n          data: JSON.stringify(payload),\n          timestamp: Date.now(),\n        });\n\n        // send error event to all connected clients\n        this._broadcastErrorToClients(\n          streamEntry,\n          errorEventId,\n          clientMsg,\n          errorCode,\n          true,\n          upstreamCode,\n        );\n        streamEntry.isCompleted = true;\n        this._clearGraceTimer(streamEntry);\n      }\n    });\n  }\n\n  private _combineSignals(\n    internalSignal?: AbortSignal,\n    userSignal?: AbortSignal,\n  ): AbortSignal {\n    if (!userSignal) return internalSignal || new AbortController().signal;\n\n    const signals = [internalSignal, userSignal].filter(\n      Boolean,\n    ) as AbortSignal[];\n    const controller = new AbortController();\n\n    signals.forEach((signal) => {\n      if (signal?.aborted) {\n        controller.abort(signal.reason);\n        return;\n      }\n\n      signal?.addEventListener(\n        \"abort\",\n        () => {\n          controller.abort(signal.reason);\n        },\n        { once: true },\n      );\n    });\n    return controller.signal;\n  }\n\n  // broadcast events to all connected clients\n  private _broadcastEventsToClients(\n    streamEntry: StreamEntry,\n    eventId: string,\n    event: any,\n  ): void {\n    for (const client of streamEntry.clients) {\n      if (!client.writableEnded) {\n        this.sseWriter.writeEvent(client, eventId, event);\n      }\n    }\n  }\n\n  // broadcast error to all connected clients\n  private _broadcastErrorToClients(\n    streamEntry: StreamEntry,\n    eventId: string,\n    errorMessage: string,\n    errorCode: SSEErrorCode,\n    closeClients: boolean = false,\n    upstreamCode?: string,\n  ): void {\n    for (const client of streamEntry.clients) {\n      if (!client.writableEnded) {\n        this.sseWriter.writeError(\n          client,\n          eventId,\n          errorMessage,\n          errorCode,\n          upstreamCode,\n        );\n        if (closeClients) {\n          client.end();\n        }\n      }\n    }\n  }\n\n  // close all connected clients\n  private _closeAllClients(streamEntry: StreamEntry): void {\n    for (const client of streamEntry.clients) {\n      if (!client.writableEnded) {\n        client.end();\n      }\n    }\n  }\n\n  // abort the generator after the grace window unless a client reconnects first\n  private _scheduleGraceAbort(streamEntry: StreamEntry): void {\n    // clear any existing timer to avoid stacking\n    this._clearGraceTimer(streamEntry);\n\n    const timer = setTimeout(() => {\n      streamEntry.disconnectGraceTimer = undefined;\n      if (streamEntry.clients.size === 0 && !streamEntry.isCompleted) {\n        streamEntry.abortController.abort(\n          new DOMException(\"Client disconnected (grace expired)\", \"AbortError\"),\n        );\n      }\n    }, this.disconnectGraceMs);\n\n    // never keep the process alive solely for a grace timer\n    timer.unref?.();\n    streamEntry.disconnectGraceTimer = timer;\n  }\n\n  // clear a pending disconnect-grace timer, if any\n  private _clearGraceTimer(streamEntry: StreamEntry): void {\n    if (streamEntry.disconnectGraceTimer) {\n      clearTimeout(streamEntry.disconnectGraceTimer);\n      streamEntry.disconnectGraceTimer = undefined;\n    }\n  }\n\n  // cleanup stream if no clients are connected\n  private _cleanupStream(streamEntry: StreamEntry): void {\n    if (streamEntry.clients.size === 0) {\n      setTimeout(() => {\n        if (streamEntry.clients.size === 0) {\n          this.streamRegistry.remove(streamEntry.streamId);\n        }\n      }, this.bufferTTL);\n    }\n  }\n\n  private _categorizeError(error: unknown): SSEErrorCode {\n    if (error instanceof Error) {\n      const message = error.message.toLowerCase();\n      if (message.includes(\"timeout\") || message.includes(\"timed out\")) {\n        return SSEErrorCode.TIMEOUT;\n      }\n\n      if (message.includes(\"unavailable\") || message.includes(\"econnrefused\")) {\n        return SSEErrorCode.TEMPORARY_UNAVAILABLE;\n      }\n\n      if (error.name === \"AbortError\") {\n        return SSEErrorCode.STREAM_ABORTED;\n      }\n\n      // Defense-in-depth: upstream layers (SQL client, cache) may wrap an\n      // AbortError into ExecutionError, losing `name` but keeping the message.\n      if (\n        message.includes(\"operation was aborted\") ||\n        message.includes(\"the request was aborted\") ||\n        message.includes(\"statement was canceled\")\n      ) {\n        return SSEErrorCode.STREAM_ABORTED;\n      }\n\n      // Detect upstream API errors (e.g., from Databricks SDK ApiError)\n      if (\n        \"statusCode\" in error &&\n        typeof (error as any).statusCode === \"number\"\n      ) {\n        return SSEErrorCode.UPSTREAM_ERROR;\n      }\n    }\n\n    return SSEErrorCode.INTERNAL_ERROR;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;AAaA,MAAM,SAAS,aAAa,SAAS;AAGrC,IAAa,gBAAb,MAA2B;CACzB,AAAQ;CACR,AAAQ;CACR,AAAQ;CACR,AAAQ;CACR,AAAQ;CACR,AAAQ;CAER,YAAY,SAAwB;AAClC,OAAK,iBAAiB,IAAI,eACxB,SAAS,oBAAoB,eAAe,iBAC7C;AACD,OAAK,YAAY,IAAI,WAAW;AAChC,OAAK,eAAe,SAAS,gBAAgB,eAAe;AAC5D,OAAK,YAAY,SAAS,aAAa,eAAe;AACtD,OAAK,oBACH,SAAS,qBAAqB,eAAe;AAC/C,OAAK,mCAAmB,IAAI,KAAK;;CAInC,MAAM,OACJ,KACA,SACA,SACA,UACe;EACf,MAAM,EAAE,aAAa,WAAW,EAAE;AAGlC,MAAI,IAAI,iBAAiB,IAAI,UAC3B;AAIF,OAAK,UAAU,aAAa,IAAI;AAGhC,MAAI,YAAY,gBAAgB,iBAAiB,SAAS,EAAE;GAC1D,MAAM,iBAAiB,KAAK,eAAe,IAAI,SAAS;AACxD,OAAI,gBAAgB;AAKlB,QAAI,eAAe,aAAa,UAAU;AACxC,UAAK,UAAU,WACb,KACA,YAAY,EACZ,qCACA,aAAa,iBACd;AACD,SAAI,KAAK;AACT;;AAEF,WAAO,KAAK,wBAAwB,KAAK,gBAAgB,QAAQ;;;AAKrE,SAAO,KAAK,iBAAiB,KAAK,SAAS,SAAS,SAAS;;CAI/D,WAAiB;AAEf,OAAK,iBAAiB,SAAS,cAAc;AAC3C,OAAI,UAAU,UAAW,eAAc,UAAU,UAAU;AAC3D,aAAU,WAAW,MACnB,IAAI,aAAa,mBAAmB,aAAa,CAClD;IACD;AACF,OAAK,iBAAiB,OAAO;AAC7B,OAAK,eAAe,OAAO;;CAI7B,iBAAyB;AACvB,SAAO,KAAK,iBAAiB;;CAI/B,MAAc,wBACZ,KACA,aACA,SACe;EAEf,MAAM,cAAc,IAAI,KAAK,QAAQ;AAErC,MAAI,gBAAgB,gBAAgB,YAAY,EAAE;GAEhD,MAAM,eAAe;AACrB,OAAI,YAAY,YAAY,IAAI,aAAa,EAAE;IAC7C,MAAM,eACJ,YAAY,YAAY,eAAe,aAAa;AAEtD,SAAK,MAAM,SAAS,cAAc;AAChC,SAAI,SAAS,YAAY,QAAS;AAClC,UAAK,UAAU,mBAAmB,KAAK,MAAM;;SAI/C,MAAK,UAAU,2BAA2B,KAAK,aAAa;;AAKhE,OAAK,iBAAiB,YAAY;AAGlC,cAAY,QAAQ,IAAI,IAAI;AAC5B,cAAY,aAAa,KAAK,KAAK;EAGnC,MAAM,iBAAiB,KAAK,gBAC1B,YAAY,gBAAgB,QAC5B,SAAS,WACV;EACD,MAAM,YAAY,KAAK,UAAU,eAAe,KAAK,eAAe;EAGpE,MAAM,kBAAmC;GACvC,YAAY,YAAY;GACxB,MAAM;GACN;GACD;AACD,OAAK,iBAAiB,IAAI,gBAAgB;AAG1C,MAAI,GAAG,eAAe;AACpB,iBAAc,UAAU;AACxB,eAAY,QAAQ,OAAO,IAAI;AAC/B,QAAK,iBAAiB,OAAO,gBAAgB;AAG7C,OAAI,YAAY,QAAQ,SAAS,KAAK,CAAC,YAAY,YACjD,MAAK,oBAAoB,YAAY;AAIvC,OAAI,YAAY,eAAe,YAAY,QAAQ,SAAS,EAC1D,kBAAiB;AACf,QAAI,YAAY,QAAQ,SAAS,EAC/B,MAAK,eAAe,OAAO,YAAY,SAAS;MAEjD,KAAK,UAAU;IAEpB;AAGF,MAAI,YAAY,aAAa;AAC3B,OAAI,KAAK;AAET,QAAK,iBAAiB,OAAO,gBAAgB;AAC7C,iBAAc,UAAU;;;CAG5B,MAAc,iBACZ,KACA,SACA,SACA,UACe;EACf,MAAM,WAAW,SAAS,YAAY,YAAY;AAGlD,MAAI,IAAI,iBAAiB,IAAI,UAC3B;EAGF,MAAM,kBAAkB,IAAI,iBAAiB;EAG7C,MAAM,cAAc,IAAI,gBACtB,SAAS,cAAc,eAAe,WACvC;EAGD,MAAM,iBAAiB,KAAK,gBAC1B,gBAAgB,QAChB,SAAS,WACV;EACD,MAAM,YAAY,KAAK,UAAU,eAAe,KAAK,eAAe;EAGpE,MAAM,eAAe,QAAQ,QAAQ;AAGrC,MAAI,IAAI,iBAAiB,IAAI,WAAW;AACtC,iBAAc,UAAU;AACxB;;EAIF,MAAM,cAA2B;GAC/B;GACA;GACA,WAAW,QAAQ,eAAe;GAClC;GACA,SAAS,IAAI,IAAI,CAAC,IAAI,CAAC;GACvB,aAAa;GACb,YAAY,KAAK,KAAK;GACtB;GACA;GACD;AACD,OAAK,eAAe,IAAI,YAAY;EAGpC,MAAM,kBAAmC;GACvC,YAAY;GACZ,MAAM;GACN;GACD;AACD,OAAK,iBAAiB,IAAI,gBAAgB;AAE1C,MAAI,GAAG,eAAe;AACpB,iBAAc,UAAU;AACxB,QAAK,iBAAiB,OAAO,gBAAgB;AAC7C,eAAY,QAAQ,OAAO,IAAI;AAG/B,OAAI,YAAY,QAAQ,SAAS,KAAK,CAAC,YAAY,YACjD,MAAK,oBAAoB,YAAY;IAEvC;AAEF,QAAM,KAAK,8BAA8B,YAAY;AAGrD,gBAAc,UAAU;AACxB,OAAK,iBAAiB,OAAO,gBAAgB;;CAG/C,MAAc,8BACZ,aACe;AAEf,SAAO,QAAQ,KAAK,YAAY,cAAc,YAAY;AACxD,OAAI;AAEF,eAAW,MAAM,SAAS,YAAY,WAAW;AAC/C,SAAI,YAAY,gBAAgB,OAAO,QAAS;KAChD,MAAM,UAAU,YAAY;KAC5B,MAAM,YAAY,KAAK,UAAU,MAAM;AAGvC,SAAI,UAAU,SAAS,KAAK,cAAc;MACxC,MAAM,WAAW,6BAA6B,KAAK,aAAa;MAChE,MAAM,YAAY,aAAa;AAE/B,WAAK,yBACH,aACA,SACA,UACA,UACD;AACD;;AAIF,iBAAY,YAAY,IAAI;MAC1B,IAAI;MACJ,MAAM,MAAM;MACZ,MAAM;MACN,WAAW,KAAK,KAAK;MACtB,CAAC;AAGF,UAAK,0BAA0B,aAAa,SAAS,MAAM;AAC3D,iBAAY,aAAa,KAAK,KAAK;;AAGrC,gBAAY,cAAc;AAG1B,SAAK,iBAAiB,YAAY;AAGlC,SAAK,iBAAiB,YAAY;AAGlC,SAAK,eAAe,YAAY;YACzB,OAAO;IAMd,MAAM,SACJ,iBAAiB,QAAQ,MAAM,UAAU;IAC3C,MAAM,YACJ,iBAAiB,cACb,MAAM,gBACN;IAGN,MAAM,eACJ,iBAAiB,iBAAiB,MAAM,YAAY;IACtD,MAAM,eAAe,YAAY;IACjC,MAAM,YAAY,KAAK,iBAAiB,MAAM;AAG9C,QAAI,cAAc,aAAa,gBAAgB;AAC7C,YAAO,KAAK,sCAAsC,UAAU;AAC5D,iBAAY,cAAc;AAC1B,UAAK,iBAAiB,YAAY;AAClC,UAAK,iBAAiB,YAAY;AAClC,UAAK,eAAe,YAAY;AAChC;;AAGF,WAAO,MACL,yDACA,QACA,WACA,gBAAgB,MACjB;IAED,MAAM,UAAmC;KACvC,OAAO;KACP,MAAM;KACP;AACD,QAAI,aAAc,SAAQ,YAAY;AAGtC,gBAAY,YAAY,IAAI;KAC1B,IAAI;KACJ,MAAM;KACN,MAAM,KAAK,UAAU,QAAQ;KAC7B,WAAW,KAAK,KAAK;KACtB,CAAC;AAGF,SAAK,yBACH,aACA,cACA,WACA,WACA,MACA,aACD;AACD,gBAAY,cAAc;AAC1B,SAAK,iBAAiB,YAAY;;IAEpC;;CAGJ,AAAQ,gBACN,gBACA,YACa;AACb,MAAI,CAAC,WAAY,QAAO,kBAAkB,IAAI,iBAAiB,CAAC;EAEhE,MAAM,UAAU,CAAC,gBAAgB,WAAW,CAAC,OAC3C,QACD;EACD,MAAM,aAAa,IAAI,iBAAiB;AAExC,UAAQ,SAAS,WAAW;AAC1B,OAAI,QAAQ,SAAS;AACnB,eAAW,MAAM,OAAO,OAAO;AAC/B;;AAGF,WAAQ,iBACN,eACM;AACJ,eAAW,MAAM,OAAO,OAAO;MAEjC,EAAE,MAAM,MAAM,CACf;IACD;AACF,SAAO,WAAW;;CAIpB,AAAQ,0BACN,aACA,SACA,OACM;AACN,OAAK,MAAM,UAAU,YAAY,QAC/B,KAAI,CAAC,OAAO,cACV,MAAK,UAAU,WAAW,QAAQ,SAAS,MAAM;;CAMvD,AAAQ,yBACN,aACA,SACA,cACA,WACA,eAAwB,OACxB,cACM;AACN,OAAK,MAAM,UAAU,YAAY,QAC/B,KAAI,CAAC,OAAO,eAAe;AACzB,QAAK,UAAU,WACb,QACA,SACA,cACA,WACA,aACD;AACD,OAAI,aACF,QAAO,KAAK;;;CAOpB,AAAQ,iBAAiB,aAAgC;AACvD,OAAK,MAAM,UAAU,YAAY,QAC/B,KAAI,CAAC,OAAO,cACV,QAAO,KAAK;;CAMlB,AAAQ,oBAAoB,aAAgC;AAE1D,OAAK,iBAAiB,YAAY;EAElC,MAAM,QAAQ,iBAAiB;AAC7B,eAAY,uBAAuB;AACnC,OAAI,YAAY,QAAQ,SAAS,KAAK,CAAC,YAAY,YACjD,aAAY,gBAAgB,MAC1B,IAAI,aAAa,uCAAuC,aAAa,CACtE;KAEF,KAAK,kBAAkB;AAG1B,QAAM,SAAS;AACf,cAAY,uBAAuB;;CAIrC,AAAQ,iBAAiB,aAAgC;AACvD,MAAI,YAAY,sBAAsB;AACpC,gBAAa,YAAY,qBAAqB;AAC9C,eAAY,uBAAuB;;;CAKvC,AAAQ,eAAe,aAAgC;AACrD,MAAI,YAAY,QAAQ,SAAS,EAC/B,kBAAiB;AACf,OAAI,YAAY,QAAQ,SAAS,EAC/B,MAAK,eAAe,OAAO,YAAY,SAAS;KAEjD,KAAK,UAAU;;CAItB,AAAQ,iBAAiB,OAA8B;AACrD,MAAI,iBAAiB,OAAO;GAC1B,MAAM,UAAU,MAAM,QAAQ,aAAa;AAC3C,OAAI,QAAQ,SAAS,UAAU,IAAI,QAAQ,SAAS,YAAY,CAC9D,QAAO,aAAa;AAGtB,OAAI,QAAQ,SAAS,cAAc,IAAI,QAAQ,SAAS,eAAe,CACrE,QAAO,aAAa;AAGtB,OAAI,MAAM,SAAS,aACjB,QAAO,aAAa;AAKtB,OACE,QAAQ,SAAS,wBAAwB,IACzC,QAAQ,SAAS,0BAA0B,IAC3C,QAAQ,SAAS,yBAAyB,CAE1C,QAAO,aAAa;AAItB,OACE,gBAAgB,SAChB,OAAQ,MAAc,eAAe,SAErC,QAAO,aAAa;;AAIxB,SAAO,aAAa"}

package/dist/stream/types.js.map

@@ -1 +1 @@
-{"version":3,"file":"types.js","names":[],"sources":["../../src/stream/types.ts"],"sourcesContent":["import type { Context } from \"@opentelemetry/api\";\nimport type { IAppResponse } from \"shared\";\nimport type { EventRingBuffer } from \"./buffers\";\n\nexport const SSEWarningCode = {\n  BUFFER_OVERFLOW_RESTART: \"BUFFER_OVERFLOW_RESTART\",\n} as const satisfies Record<string, string>;\n\nexport type SSEWarningCode =\n  (typeof SSEWarningCode)[keyof typeof SSEWarningCode];\n\nexport const SSEErrorCode = {\n  TEMPORARY_UNAVAILABLE: \"TEMPORARY_UNAVAILABLE\",\n  TIMEOUT: \"TIMEOUT\",\n  INTERNAL_ERROR: \"INTERNAL_ERROR\",\n  INVALID_REQUEST: \"INVALID_REQUEST\",\n  STREAM_ABORTED: \"STREAM_ABORTED\",\n  STREAM_EVICTED: \"STREAM_EVICTED\",\n  STREAM_FORBIDDEN: \"STREAM_FORBIDDEN\",\n  UPSTREAM_ERROR: \"UPSTREAM_ERROR\",\n} as const satisfies Record<string, string>;\n\nexport type SSEErrorCode = (typeof SSEErrorCode)[keyof typeof SSEErrorCode];\n\nexport interface SSEError {\n  error: string;\n  code: SSEErrorCode;\n}\n\nexport interface BufferedEvent {\n  id: string;\n  type: string;\n  data: string;\n  timestamp: number;\n}\n\nexport interface StreamEntry {\n  streamId: string;\n  /**\n   * Identifier of the principal that created the stream (e.g. end-user ID\n   * or service principal user ID). When set, only requests sharing the\n   * same owner key may reconnect to the stream.\n   */\n  ownerKey?: string;\n  generator: AsyncGenerator<any, void, unknown>;\n  eventBuffer: EventRingBuffer;\n  clients: Set<IAppResponse>;\n  isCompleted: boolean;\n  lastAccess: number;\n  abortController: AbortController;\n  traceContext: Context;\n  // pending grace-window abort, set while the last client is disconnected\n  disconnectGraceTimer?: NodeJS.Timeout;\n}\n\nexport interface StreamOperation {\n  controller: AbortController;\n  type: \"query\" | \"stream\";\n  heartbeat?: NodeJS.Timeout;\n}\n"],"mappings":";AAIA,MAAa,iBAAiB,EAC5B,yBAAyB,2BAC1B;AAKD,MAAa,eAAe;CAC1B,uBAAuB;CACvB,SAAS;CACT,gBAAgB;CAChB,iBAAiB;CACjB,gBAAgB;CAChB,gBAAgB;CAChB,kBAAkB;CAClB,gBAAgB;CACjB"}
+{"version":3,"file":"types.js","names":[],"sources":["../../src/stream/types.ts"],"sourcesContent":["import type { Context } from \"@opentelemetry/api\";\nimport type { IAppResponse } from \"shared\";\nimport type { EventRingBuffer } from \"./buffers\";\n\nexport const SSEWarningCode = {\n  BUFFER_OVERFLOW_RESTART: \"BUFFER_OVERFLOW_RESTART\",\n} as const satisfies Record<string, string>;\n\nexport type SSEWarningCode =\n  (typeof SSEWarningCode)[keyof typeof SSEWarningCode];\n\nexport const SSEErrorCode = {\n  TEMPORARY_UNAVAILABLE: \"TEMPORARY_UNAVAILABLE\",\n  TIMEOUT: \"TIMEOUT\",\n  INTERNAL_ERROR: \"INTERNAL_ERROR\",\n  INVALID_REQUEST: \"INVALID_REQUEST\",\n  STREAM_ABORTED: \"STREAM_ABORTED\",\n  STREAM_EVICTED: \"STREAM_EVICTED\",\n  STREAM_FORBIDDEN: \"STREAM_FORBIDDEN\",\n  UPSTREAM_ERROR: \"UPSTREAM_ERROR\",\n} as const satisfies Record<string, string>;\n\nexport type SSEErrorCode = (typeof SSEErrorCode)[keyof typeof SSEErrorCode];\n\nexport interface SSEError {\n  error: string;\n  code: SSEErrorCode;\n  /**\n   * Upstream-domain structured code (e.g. `RESULT_TOO_LARGE_FOR_JSON_FALLBACK`,\n   * `NOT_IMPLEMENTED`). UI code should branch on this instead of parsing\n   * the human-readable `error` string.\n   */\n  errorCode?: string;\n}\n\nexport interface BufferedEvent {\n  id: string;\n  type: string;\n  data: string;\n  timestamp: number;\n}\n\nexport interface StreamEntry {\n  streamId: string;\n  /**\n   * Identifier of the principal that created the stream (e.g. end-user ID\n   * or service principal user ID). When set, only requests sharing the\n   * same owner key may reconnect to the stream.\n   */\n  ownerKey?: string;\n  generator: AsyncGenerator<any, void, unknown>;\n  eventBuffer: EventRingBuffer;\n  clients: Set<IAppResponse>;\n  isCompleted: boolean;\n  lastAccess: number;\n  abortController: AbortController;\n  traceContext: Context;\n  // pending grace-window abort, set while the last client is disconnected\n  disconnectGraceTimer?: NodeJS.Timeout;\n}\n\nexport interface StreamOperation {\n  controller: AbortController;\n  type: \"query\" | \"stream\";\n  heartbeat?: NodeJS.Timeout;\n}\n"],"mappings":";AAIA,MAAa,iBAAiB,EAC5B,yBAAyB,2BAC1B;AAKD,MAAa,eAAe;CAC1B,uBAAuB;CACvB,SAAS;CACT,gBAAgB;CAChB,iBAAiB;CACjB,gBAAgB;CAChB,gBAAgB;CAChB,kBAAkB;CAClB,gBAAgB;CACjB"}

package/dist/type-generator/query-registry.js

@@ -10,6 +10,7 @@ import { WorkspaceClient } from "@databricks/sdk-experimental";
 import pc from "picocolors";
 import fs from "node:fs/promises";
 import path from "node:path";
+import { tableFromIPC } from "apache-arrow";
 
 //#region src/type-generator/query-registry.ts
 const logger = createLogger("type-generator:query-registry");
@@ -95,12 +96,41 @@ function formatParametersType(sql) {
 		return `/** ${sqlType || "any"} - use ${helper} */\n      ${p}: ${markerType}`;
 	}).join(";\n      ")};\n    }` : "Record<string, never>";
 }
+/**
+* Decode a base64 Arrow IPC attachment from a DESCRIBE QUERY response and
+* extract column metadata. Returns the same shape as rows parsed from the
+* legacy data_array path.
+*
+* IMPORTANT: a DESCRIBE QUERY response is itself a result *table* with rows
+* shaped like `(col_name, data_type, comment)` describing the user query's
+* output schema. We must read those rows — NOT `table.schema.fields`, which
+* would describe DESCRIBE QUERY's own output (`col_name`, `data_type`,
+* `comment`) and yield bogus types for every query.
+*/
+function columnsFromArrowAttachment(attachment) {
+	return tableFromIPC(Buffer.from(attachment, "base64")).toArray().map((row) => {
+		const obj = row.toJSON();
+		return {
+			name: typeof obj.col_name === "string" ? obj.col_name : "",
+			type_name: typeof obj.data_type === "string" ? obj.data_type.toUpperCase() : "STRING",
+			comment: typeof obj.comment === "string" && obj.comment !== "" ? obj.comment : void 0
+		};
+	});
+}
 function convertToQueryType(result, sql, queryName) {
-	const columns = (result.result?.data_array || []).map((row) => ({
+	let columns = (result.result?.data_array || []).map((row) => ({
 		name: row[0] || "",
 		type_name: row[1]?.toUpperCase() || "STRING",
 		comment: row[2] || void 0
 	}));
+	if (columns.length === 0 && result.result?.attachment) {
+		logger.debug("data_array empty, decoding Arrow IPC attachment for schema");
+		try {
+			columns = columnsFromArrowAttachment(result.result.attachment);
+		} catch (err) {
+			logger.warn("Failed to decode Arrow IPC attachment: %s", err instanceof Error ? err.message : String(err));
+		}
+	}
 	const paramsType = formatParametersType(sql);
 	const resultFields = columns.map((column) => {
 		const mappedType = typeMap[normalizeTypeName(column.type_name)] || "unknown";
@@ -444,7 +474,7 @@ async function generateQueriesFromDescribe(queryFolder, warehouseId, options = {
 				const result = await describeAdaptive(client, `DESCRIBE QUERY ${cleanedSql}`, warehouseId, describeFormat);
 				completed++;
 				spinner.update(`Describing ${total} ${total === 1 ? "query" : "queries"} (${completed}/${total})`);
-				logger.debug("DESCRIBE result for %s: state=%s, rows=%d", queryName, result.status.state, result.result?.data_array?.length ?? 0);
+				logger.debug("DESCRIBE result for %s: state=%s, rows=%d, hasAttachment=%s", queryName, result.status.state, result.result?.data_array?.length ?? 0, !!result.result?.attachment);
 				if (result.status.state === "FAILED") {
 					const sqlError = result.status.error?.message || "Query execution failed";
 					return {