@query-farm/vgi-rpc 0.3.4 → 0.6.0

package/src/client/pipe.ts

@@ -12,6 +12,7 @@ import {
 } from "@query-farm/apache-arrow";
 import { DESCRIBE_METHOD_NAME } from "../constants.js";
 import { RpcError } from "../errors.js";
+import { type ExternalLocationConfig, isExternalLocationBatch, resolveExternalLocation } from "../external.js";
 import { serializeIpcStream } from "../http/common.js";
 import { IpcStreamReader } from "../wire/reader.js";
 import type { RpcClient } from "./connect.js";
@@ -86,6 +87,7 @@ export class PipeStreamSession implements StreamSession {
   private _outputSchema: Schema;
   private _releaseBusy: () => void;
   private _setDrainPromise: (p: Promise<void>) => void;
+  private _externalConfig?: ExternalLocationConfig;
 
   constructor(opts: {
     reader: IpcStreamReader;
@@ -95,6 +97,7 @@ export class PipeStreamSession implements StreamSession {
     outputSchema: Schema;
     releaseBusy: () => void;
     setDrainPromise: (p: Promise<void>) => void;
+    externalConfig?: ExternalLocationConfig;
   }) {
     this._reader = opts.reader;
     this._writeFn = opts.writeFn;
@@ -103,6 +106,7 @@ export class PipeStreamSession implements StreamSession {
     this._outputSchema = opts.outputSchema;
     this._releaseBusy = opts.releaseBusy;
     this._setDrainPromise = opts.setDrainPromise;
+    this._externalConfig = opts.externalConfig;
   }
 
   get header(): Record<string, any> | null {
@@ -120,6 +124,10 @@ export class PipeStreamSession implements StreamSession {
       if (batch === null) return null; // Server closed output stream
 
       if (batch.numRows === 0) {
+        // Check for external location pointer batch
+        if (isExternalLocationBatch(batch)) {
+          return await resolveExternalLocation(batch, this._externalConfig);
+        }
         // Check if it's a log/error batch. If so, dispatch and continue.
         // Otherwise it's a zero-row data batch — return it.
         if (dispatchLogOrError(batch, this._onLog)) {
@@ -375,6 +383,7 @@ export function pipeConnect(
   options?: PipeConnectOptions,
 ): RpcClient {
   const onLog = options?.onLog;
+  const externalConfig = options?.externalLocation;
 
   let reader: IpcStreamReader | null = null;
   let readerPromise: Promise<IpcStreamReader> | null = null;
@@ -483,12 +492,16 @@ export function pipeConnect(
           throw new Error("EOF reading response");
         }
 
-        // Process batches: dispatch logs, find result
+        // Process batches: dispatch logs, resolve external pointers, find result
         let resultBatch: RecordBatch | null = null;
-        for (const batch of response.batches) {
+        for (let batch of response.batches) {
           if (batch.numRows === 0) {
-            dispatchLogOrError(batch, onLog);
-            continue;
+            if (isExternalLocationBatch(batch)) {
+              batch = await resolveExternalLocation(batch, externalConfig);
+            } else {
+              dispatchLogOrError(batch, onLog);
+              continue;
+            }
           }
           resultBatch = batch;
         }
@@ -557,6 +570,7 @@ export function pipeConnect(
           outputSchema,
           releaseBusy,
           setDrainPromise,
+          externalConfig,
         });
       } catch (e) {
         // Init error (e.g., server raised exception during init).
@@ -624,6 +638,7 @@ export function subprocessConnect(cmd: string[], options?: SubprocessConnectOpti
 
   const client = pipeConnect(stdout, writable, {
     onLog: options?.onLog,
+    externalLocation: options?.externalLocation,
   });
 
   // Wrap close to also kill the subprocess

package/src/client/stream.ts

@@ -4,6 +4,7 @@
 import { Field, makeData, RecordBatch, Schema, Struct, vectorFromArray } from "@query-farm/apache-arrow";
 import { STATE_KEY } from "../constants.js";
 import { RpcError } from "../errors.js";
+import { type ExternalLocationConfig, isExternalLocationBatch, resolveExternalLocation } from "../external.js";
 import { ARROW_CONTENT_TYPE, serializeIpcStream } from "../http/common.js";
 import { dispatchLogOrError, extractBatchRows, inferArrowType, readResponseBatches } from "./ipc.js";
 import type { LogMessage, StreamSession } from "./types.js";
@@ -25,6 +26,8 @@ export class HttpStreamSession implements StreamSession {
   private _compressionLevel?: number;
   private _compressFn?: CompressFn;
   private _decompressFn?: DecompressFn;
+  private _authorization?: string;
+  private _externalConfig?: ExternalLocationConfig;
 
   constructor(opts: {
     baseUrl: string;
@@ -40,6 +43,8 @@ export class HttpStreamSession implements StreamSession {
     compressionLevel?: number;
     compressFn?: CompressFn;
     decompressFn?: DecompressFn;
+    authorization?: string;
+    externalConfig?: ExternalLocationConfig;
   }) {
     this._baseUrl = opts.baseUrl;
     this._prefix = opts.prefix;
@@ -54,6 +59,8 @@ export class HttpStreamSession implements StreamSession {
     this._compressionLevel = opts.compressionLevel;
     this._compressFn = opts.compressFn;
     this._decompressFn = opts.decompressFn;
+    this._authorization = opts.authorization;
+    this._externalConfig = opts.externalConfig;
   }
 
   get header(): Record<string, any> | null {
@@ -68,6 +75,9 @@ export class HttpStreamSession implements StreamSession {
       headers["Content-Encoding"] = "zstd";
       headers["Accept-Encoding"] = "zstd";
     }
+    if (this._authorization) {
+      headers.Authorization = this._authorization;
+    }
     return headers;
   }
 
@@ -154,6 +164,9 @@ export class HttpStreamSession implements StreamSession {
       headers: this._buildHeaders(),
       body: this._prepareBody(body) as unknown as BodyInit,
     });
+    if (resp.status === 401) {
+      throw new RpcError("AuthenticationError", "Authentication required", "");
+    }
 
     const responseBody = await this._readResponse(resp);
     const { batches: responseBatches } = await readResponseBatches(responseBody);
@@ -202,10 +215,14 @@ export class HttpStreamSession implements StreamSession {
    */
   async *[Symbol.asyncIterator](): AsyncIterableIterator<Record<string, any>[]> {
     // Yield pre-loaded batches from init
-    for (const batch of this._pendingBatches) {
+    for (let batch of this._pendingBatches) {
       if (batch.numRows === 0) {
-        dispatchLogOrError(batch, this._onLog);
-        continue;
+        if (isExternalLocationBatch(batch)) {
+          batch = await resolveExternalLocation(batch, this._externalConfig);
+        } else {
+          dispatchLogOrError(batch, this._onLog);
+          continue;
+        }
       }
       yield extractBatchRows(batch);
     }
@@ -220,7 +237,7 @@ export class HttpStreamSession implements StreamSession {
       const { batches } = await readResponseBatches(responseBody);
 
       let gotContinuation = false;
-      for (const batch of batches) {
+      for (let batch of batches) {
         if (batch.numRows === 0) {
           // Check for continuation token
           const token = batch.metadata?.get(STATE_KEY);
@@ -229,9 +246,14 @@ export class HttpStreamSession implements StreamSession {
             gotContinuation = true;
             continue;
           }
-          // Log/error batch
-          dispatchLogOrError(batch, this._onLog);
-          continue;
+          // Check for external location pointer
+          if (isExternalLocationBatch(batch)) {
+            batch = await resolveExternalLocation(batch, this._externalConfig);
+          } else {
+            // Log/error batch
+            dispatchLogOrError(batch, this._onLog);
+            continue;
+          }
         }
 
         yield extractBatchRows(batch);
@@ -261,6 +283,9 @@ export class HttpStreamSession implements StreamSession {
       headers: this._buildHeaders(),
       body: this._prepareBody(body) as unknown as BodyInit,
     });
+    if (resp.status === 401) {
+      throw new RpcError("AuthenticationError", "Authentication required", "");
+    }
 
     return this._readResponse(resp);
   }

package/src/client/types.ts

@@ -5,6 +5,10 @@ export interface HttpConnectOptions {
   prefix?: string;
   onLog?: (msg: LogMessage) => void;
   compressionLevel?: number;
+  /** Authorization header value (e.g. "Bearer <token>"). Sent with every request. */
+  authorization?: string;
+  /** External storage config for resolving externalized batches. */
+  externalLocation?: import("../external.js").ExternalLocationConfig;
 }
 
 export interface LogMessage {
@@ -22,6 +26,8 @@ export interface StreamSession {
 
 export interface PipeConnectOptions {
   onLog?: (msg: LogMessage) => void;
+  /** External storage config for resolving externalized batches. */
+  externalLocation?: import("../external.js").ExternalLocationConfig;
 }
 
 export interface SubprocessConnectOptions extends PipeConnectOptions {

package/src/constants.ts

@@ -15,8 +15,11 @@ export const REQUEST_ID_KEY = "vgi_rpc.request_id";
 
 export const PROTOCOL_NAME_KEY = "vgi_rpc.protocol_name";
 export const DESCRIBE_VERSION_KEY = "vgi_rpc.describe_version";
-export const DESCRIBE_VERSION = "2";
+export const DESCRIBE_VERSION = "3";
 
 export const DESCRIBE_METHOD_NAME = "__describe__";
 
 export const STATE_KEY = "vgi_rpc.stream_state#b64";
+
+export const LOCATION_KEY = "vgi_rpc.location";
+export const LOCATION_SHA256_KEY = "vgi_rpc.location.sha256";

package/src/dispatch/describe.ts

@@ -37,6 +37,8 @@ export const DESCRIBE_SCHEMA = new Schema([
   new Field("param_defaults_json", new Utf8(), true),
   new Field("has_header", new Bool(), false),
   new Field("header_schema_ipc", new Binary(), true),
+  new Field("is_exchange", new Bool(), true),
+  new Field("param_docs_json", new Utf8(), true),
 ]);
 
 /**
@@ -60,6 +62,8 @@ export function buildDescribeBatch(
   const paramDefaultsJsons: (string | null)[] = [];
   const hasHeaders: boolean[] = [];
   const headerSchemas: (Uint8Array | null)[] = [];
+  const isExchanges: (boolean | null)[] = [];
+  const paramDocsJsons: (string | null)[] = [];
 
   for (const [name, method] of sortedEntries) {
     names.push(name);
@@ -95,6 +99,18 @@ export function buildDescribeBatch(
 
     hasHeaders.push(!!method.headerSchema);
     headerSchemas.push(method.headerSchema ? serializeSchema(method.headerSchema) : null);
+
+    // is_exchange: true for exchange, false for producer, null for unary
+    if (method.exchangeFn) {
+      isExchanges.push(true);
+    } else if (method.producerFn) {
+      isExchanges.push(false);
+    } else {
+      isExchanges.push(null);
+    }
+
+    // param_docs_json: no docstring source in TypeScript, always null
+    paramDocsJsons.push(null);
   }
 
   // Build the batch using vectorFromArray for each column
@@ -108,6 +124,8 @@ export function buildDescribeBatch(
   const paramDefaultsArr = vectorFromArray(paramDefaultsJsons, new Utf8());
   const hasHeaderArr = vectorFromArray(hasHeaders, new Bool());
   const headerSchemaArr = vectorFromArray(headerSchemas, new Binary());
+  const isExchangeArr = vectorFromArray(isExchanges, new Bool());
+  const paramDocsArr = vectorFromArray(paramDocsJsons, new Utf8());
 
   const children = [
     nameArr.data[0],
@@ -120,6 +138,8 @@ export function buildDescribeBatch(
     paramDefaultsArr.data[0],
     hasHeaderArr.data[0],
     headerSchemaArr.data[0],
+    isExchangeArr.data[0],
+    paramDocsArr.data[0],
   ];
 
   const structType = new Struct(DESCRIBE_SCHEMA.fields);

package/src/dispatch/stream.ts

@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 import { Schema } from "@query-farm/apache-arrow";
+import { type ExternalLocationConfig, maybeExternalizeBatch } from "../external.js";
 import type { MethodDefinition } from "../types.js";
 import { OutputCollector } from "../types.js";
 import { conformBatchToSchema } from "../util/conform.js";
@@ -33,6 +34,7 @@ export async function dispatchStream(
   reader: IpcStreamReader,
   serverId: string,
   requestId: string | null,
+  externalConfig?: ExternalLocationConfig,
 ): Promise<void> {
   const isProducer = !!method.producerFn;
 
@@ -107,12 +109,20 @@ export async function dispatchStream(
       let inputBatch = await reader.readNextBatch();
       if (!inputBatch) break;
 
-      // Cast compatible input types when schema doesn't match exactly
+      // Cast compatible input types when schema doesn't match exactly.
+      // If conformance fails (e.g., completely different schemas like a dummy
+      // registration schema vs actual data), pass the original batch through —
+      // the exchange handler may handle dynamic schemas internally.
       if (expectedInputSchema && !isProducer && inputBatch.schema !== expectedInputSchema) {
         try {
           inputBatch = conformBatchToSchema(inputBatch, expectedInputSchema);
-        } catch {
-          throw new TypeError(`Input schema mismatch: expected ${expectedInputSchema}, got ${inputBatch.schema}`);
+        } catch (e) {
+          if (e instanceof TypeError) {
+            // Field name/count mismatch — propagate as error (matches Python behavior).
+            throw e;
+          }
+          // Other conformance failures: pass through for dynamic schema handlers.
+          console.debug?.(`Schema conformance skipped: ${e instanceof Error ? e.message : e}`);
         }
       }
 
@@ -125,7 +135,11 @@ export async function dispatchStream(
       }
 
       for (const emitted of out.batches) {
-        stream.write(emitted.batch);
+        let batch = emitted.batch;
+        if (externalConfig) {
+          batch = await maybeExternalizeBatch(batch, externalConfig);
+        }
+        stream.write(batch);
       }
 
       if (out.finished) {

package/src/dispatch/unary.ts

@@ -1,6 +1,7 @@
 // © Copyright 2025-2026, Query.Farm LLC - https://query.farm
 // SPDX-License-Identifier: Apache-2.0
 
+import { type ExternalLocationConfig, maybeExternalizeBatch } from "../external.js";
 import type { MethodDefinition } from "../types.js";
 import { OutputCollector } from "../types.js";
 import { buildErrorBatch, buildResultBatch } from "../wire/response.js";
@@ -17,13 +18,17 @@ export async function dispatchUnary(
   writer: IpcStreamWriter,
   serverId: string,
   requestId: string | null,
+  externalConfig?: ExternalLocationConfig,
 ): Promise<void> {
   const schema = method.resultSchema;
   const out = new OutputCollector(schema, true, serverId, requestId);
 
   try {
     const result = await method.handler!(params, out);
-    const resultBatch = buildResultBatch(schema, result, serverId, requestId);
+    let resultBatch = buildResultBatch(schema, result, serverId, requestId);
+    if (externalConfig) {
+      resultBatch = await maybeExternalizeBatch(resultBatch, externalConfig);
+    }
     // Collect log batches (from clientLog) + result batch
     const batches = [...out.batches.map((b) => b.batch), resultBatch];
     writer.writeStream(schema, batches);

package/src/external.ts

@@ -0,0 +1,209 @@
+// © Copyright 2025-2026, Query.Farm LLC - https://query.farm
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * External storage support for large Arrow IPC batches.
+ *
+ * When a batch exceeds a configurable threshold, it is serialized to IPC,
+ * optionally compressed with zstd, and uploaded to pluggable storage.
+ * The batch is replaced with a zero-row "pointer batch" containing the
+ * download URL and SHA-256 checksum in metadata.
+ */
+
+import { type RecordBatch, RecordBatchReader, RecordBatchStreamWriter, type Schema } from "@query-farm/apache-arrow";
+import { LOCATION_KEY, LOCATION_SHA256_KEY, LOG_LEVEL_KEY } from "./constants.js";
+import { zstdCompress, zstdDecompress } from "./util/zstd.js";
+import { buildEmptyBatch } from "./wire/response.js";
+
+// ---------------------------------------------------------------------------
+// Interfaces and configuration
+// ---------------------------------------------------------------------------
+
+/** Pluggable storage backend for uploading large batches. */
+export interface ExternalStorage {
+  /** Upload IPC data and return a URL for retrieval. */
+  upload(data: Uint8Array, contentEncoding: string): Promise<string>;
+}
+
+/** Configuration for external storage of large batches. */
+export interface ExternalLocationConfig {
+  /** Storage backend for uploading. */
+  storage: ExternalStorage;
+  /** Minimum batch byte size to trigger externalization. Default: 1MB. */
+  externalizeThresholdBytes?: number;
+  /** Optional zstd compression for uploaded data. */
+  compression?: { algorithm: "zstd"; level?: number };
+  /** URL validator called before fetching. Throw to reject. Default: HTTPS-only. */
+  urlValidator?: ((url: string) => void) | null;
+}
+
+const DEFAULT_THRESHOLD = 1_048_576; // 1 MB
+
+// ---------------------------------------------------------------------------
+// URL validation
+// ---------------------------------------------------------------------------
+
+/** Default validator that rejects non-HTTPS URLs. */
+export function httpsOnlyValidator(url: string): void {
+  const parsed = new URL(url);
+  if (parsed.protocol !== "https:") {
+    throw new Error(`External location URL must use HTTPS, got "${parsed.protocol}"`);
+  }
+}
+
+// ---------------------------------------------------------------------------
+// SHA-256 helpers
+// ---------------------------------------------------------------------------
+
+async function sha256Hex(data: Uint8Array): Promise<string> {
+  // Copy to a plain ArrayBuffer to satisfy Web Crypto API type requirements
+  const buf = new ArrayBuffer(data.byteLength);
+  new Uint8Array(buf).set(data);
+  const hash = await crypto.subtle.digest("SHA-256", buf);
+  return Array.from(new Uint8Array(hash))
+    .map((b) => b.toString(16).padStart(2, "0"))
+    .join("");
+}
+
+// ---------------------------------------------------------------------------
+// Detection
+// ---------------------------------------------------------------------------
+
+/** Returns true if the batch is a zero-row pointer to external data. */
+export function isExternalLocationBatch(batch: RecordBatch): boolean {
+  if (batch.numRows !== 0) return false;
+  const meta = batch.metadata;
+  if (!meta) return false;
+  return meta.has(LOCATION_KEY) && !meta.has(LOG_LEVEL_KEY);
+}
+
+// ---------------------------------------------------------------------------
+// Pointer batch creation
+// ---------------------------------------------------------------------------
+
+/** Create a zero-row pointer batch with location URL and optional SHA-256. */
+export function makeExternalLocationBatch(schema: Schema, url: string, sha256?: string): RecordBatch {
+  const metadata = new Map<string, string>();
+  metadata.set(LOCATION_KEY, url);
+  if (sha256) {
+    metadata.set(LOCATION_SHA256_KEY, sha256);
+  }
+  return buildEmptyBatch(schema, metadata);
+}
+
+// ---------------------------------------------------------------------------
+// IPC serialization helpers
+// ---------------------------------------------------------------------------
+
+function serializeBatchToIpc(batch: RecordBatch): Uint8Array {
+  const writer = new RecordBatchStreamWriter();
+  writer.reset(undefined, batch.schema);
+  writer.write(batch);
+  writer.close();
+  return writer.toUint8Array(true);
+}
+
+function batchByteSize(batch: RecordBatch): number {
+  // Arrow TS data.byteLength doesn't reflect actual data size.
+  // Estimate from IPC serialization size for threshold check.
+  const writer = new RecordBatchStreamWriter();
+  writer.reset(undefined, batch.schema);
+  writer.write(batch);
+  writer.close();
+  return writer.toUint8Array(true).byteLength;
+}
+
+// ---------------------------------------------------------------------------
+// Write path: externalization
+// ---------------------------------------------------------------------------
+
+/**
+ * Maybe externalize a batch if it exceeds the threshold.
+ * Returns the original batch unchanged if below threshold or no config.
+ */
+export async function maybeExternalizeBatch(
+  batch: RecordBatch,
+  config?: ExternalLocationConfig | null,
+): Promise<RecordBatch> {
+  if (!config?.storage) return batch;
+  if (batch.numRows === 0) return batch;
+
+  const threshold = config.externalizeThresholdBytes ?? DEFAULT_THRESHOLD;
+  if (batchByteSize(batch) < threshold) return batch;
+
+  // Serialize to IPC
+  let ipcData = serializeBatchToIpc(batch);
+
+  // Compute SHA-256 of raw IPC bytes (pre-compression)
+  const checksum = await sha256Hex(ipcData);
+
+  // Optionally compress
+  let contentEncoding = "";
+  if (config.compression?.algorithm === "zstd") {
+    ipcData = zstdCompress(ipcData, config.compression.level ?? 3) as Uint8Array;
+    contentEncoding = "zstd";
+  }
+
+  // Upload
+  const url = await config.storage.upload(ipcData, contentEncoding);
+
+  // Return pointer batch
+  return makeExternalLocationBatch(batch.schema, url, checksum);
+}
+
+// ---------------------------------------------------------------------------
+// Read path: resolution
+// ---------------------------------------------------------------------------
+
+/**
+ * Resolve an external pointer batch by fetching the data from the URL.
+ * Returns the original batch unchanged if not a pointer or no config.
+ */
+export async function resolveExternalLocation(
+  batch: RecordBatch,
+  config?: ExternalLocationConfig | null,
+): Promise<RecordBatch> {
+  if (!config) return batch;
+  if (!isExternalLocationBatch(batch)) return batch;
+
+  const url = batch.metadata?.get(LOCATION_KEY);
+  if (!url) return batch;
+
+  // Validate URL
+  const validator = config.urlValidator === null ? undefined : (config.urlValidator ?? httpsOnlyValidator);
+  if (validator) {
+    validator(url);
+  }
+
+  // Fetch
+  const response = await fetch(url);
+  if (!response.ok) {
+    throw new Error(`External location fetch failed: ${response.status} ${response.statusText} [url: ${url}]`);
+  }
+  let data = new Uint8Array(await response.arrayBuffer());
+
+  // Decompress if needed
+  const contentEncoding = response.headers.get("Content-Encoding");
+  if (contentEncoding === "zstd") {
+    data = new Uint8Array(zstdDecompress(data));
+  }
+
+  // Verify SHA-256 if present
+  const expectedSha256 = batch.metadata?.get(LOCATION_SHA256_KEY);
+  if (expectedSha256) {
+    const actualSha256 = await sha256Hex(data);
+    if (actualSha256 !== expectedSha256) {
+      throw new Error(`SHA-256 checksum mismatch for ${url}: expected ${expectedSha256}, got ${actualSha256}`);
+    }
+  }
+
+  // Parse IPC stream
+  const reader = await RecordBatchReader.from(data);
+  await reader.open();
+  const resolved = reader.next();
+  if (!resolved || resolved.done || !resolved.value) {
+    throw new Error(`No data batch found in external IPC stream from ${url}`);
+  }
+
+  return resolved.value;
+}

package/src/gcs.ts

@@ -0,0 +1,86 @@
+// © Copyright 2025-2026, Query.Farm LLC - https://query.farm
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * Google Cloud Storage backend for external storage of large Arrow IPC batches.
+ *
+ * Requires `@google-cloud/storage` as a peer dependency.
+ *
+ * @example
+ * ```typescript
+ * import { createGCSStorage } from "@query-farm/vgi-rpc/gcs";
+ *
+ * const storage = createGCSStorage({
+ *   bucket: "my-bucket",
+ *   prefix: "vgi-rpc/",
+ * });
+ * const handler = createHttpHandler(protocol, {
+ *   externalLocation: { storage, externalizeThresholdBytes: 1_048_576 },
+ * });
+ * ```
+ */
+
+import type { ExternalStorage } from "./external.js";
+
+/** Configuration for the GCS storage backend. */
+export interface GCSStorageConfig {
+  /** GCS bucket name. */
+  bucket: string;
+  /** Key prefix for uploaded objects. Default: "vgi-rpc/". */
+  prefix?: string;
+  /** Lifetime of signed GET URLs in seconds. Default: 3600 (1 hour). */
+  presignExpirySeconds?: number;
+  /** GCS project ID. If omitted, uses Application Default Credentials. */
+  projectId?: string;
+}
+
+/**
+ * Create a GCS-backed ExternalStorage.
+ *
+ * Lazily imports `@google-cloud/storage` on first upload to avoid
+ * loading the SDK unless needed.
+ */
+export function createGCSStorage(config: GCSStorageConfig): ExternalStorage {
+  const bucket = config.bucket;
+  const prefix = config.prefix ?? "vgi-rpc/";
+  const presignExpiry = config.presignExpirySeconds ?? 3600;
+
+  let storageClient: any = null;
+
+  async function ensureClient(): Promise<any> {
+    if (storageClient) return storageClient;
+    const { Storage } = await import("@google-cloud/storage");
+    const clientConfig: Record<string, any> = {};
+    if (config.projectId) clientConfig.projectId = config.projectId;
+    storageClient = new Storage(clientConfig);
+    return storageClient;
+  }
+
+  return {
+    async upload(data: Uint8Array, contentEncoding: string): Promise<string> {
+      const client = await ensureClient();
+      const bucketRef = client.bucket(bucket);
+      const blobName = `${prefix}${crypto.randomUUID()}${contentEncoding === "zstd" ? ".arrow.zst" : ".arrow"}`;
+      const blob = bucketRef.file(blobName);
+
+      const options: Record<string, any> = {
+        contentType: "application/vnd.apache.arrow.stream",
+        resumable: false,
+      };
+      if (contentEncoding) {
+        options.metadata = { contentEncoding };
+      }
+
+      await blob.save(Buffer.from(data), options);
+
+      // Generate signed GET URL
+      const [url] = await blob.getSignedUrl({
+        version: "v4" as const,
+        action: "read" as const,
+        expires: Date.now() + presignExpiry * 1000,
+      });
+
+      return url;
+    },
+  };
+}