@query-farm/vgi-rpc 0.6.4 → 0.7.0

package/src/arrow/predicates.ts

@@ -0,0 +1,56 @@
+// Backend-agnostic Arrow type predicates. typeId values match the Arrow Type
+// enum and agree across arrow-js and flechette.
+
+import type { VgiDataType } from "./types.js";
+
+export const TypeId = {
+  Null: 1,
+  Int: 2,
+  Float: 3,
+  Binary: 4,
+  Utf8: 5,
+  Bool: 6,
+  Decimal: 7,
+  Date: 8,
+  Time: 9,
+  Timestamp: 10,
+  Interval: 11,
+  List: 12,
+  Struct: 13,
+  Union: 14,
+  FixedSizeBinary: 15,
+  FixedSizeList: 16,
+  Map: 17,
+  Duration: 18,
+  LargeBinary: 19,
+  LargeUtf8: 20,
+  Dictionary: -1,
+} as const;
+
+export const isNull = (t: VgiDataType): boolean => t.typeId === TypeId.Null;
+export const isInt = (t: VgiDataType): boolean => t.typeId === TypeId.Int;
+export const isFloat = (t: VgiDataType): boolean => t.typeId === TypeId.Float;
+export const isBinary = (t: VgiDataType): boolean => t.typeId === TypeId.Binary || t.typeId === TypeId.LargeBinary;
+export const isUtf8 = (t: VgiDataType): boolean => t.typeId === TypeId.Utf8 || t.typeId === TypeId.LargeUtf8;
+export const isLargeUtf8 = (t: VgiDataType): boolean => t.typeId === TypeId.LargeUtf8;
+export const isLargeBinary = (t: VgiDataType): boolean => t.typeId === TypeId.LargeBinary;
+export const isBool = (t: VgiDataType): boolean => t.typeId === TypeId.Bool;
+export const isDecimal = (t: VgiDataType): boolean => t.typeId === TypeId.Decimal;
+export const isDate = (t: VgiDataType): boolean => t.typeId === TypeId.Date;
+export const isTime = (t: VgiDataType): boolean => t.typeId === TypeId.Time;
+export const isTimestamp = (t: VgiDataType): boolean => t.typeId === TypeId.Timestamp;
+export const isDuration = (t: VgiDataType): boolean => t.typeId === TypeId.Duration;
+export const isList = (t: VgiDataType): boolean => t.typeId === TypeId.List;
+export const isStruct = (t: VgiDataType): boolean => t.typeId === TypeId.Struct;
+export const isMap = (t: VgiDataType): boolean => t.typeId === TypeId.Map;
+export const isFixedSizeBinary = (t: VgiDataType): boolean => t.typeId === TypeId.FixedSizeBinary;
+export const isDictionary = (t: VgiDataType): boolean => t.typeId === TypeId.Dictionary;
+
+export function isBatch(x: unknown): x is import("./types.js").VgiBatch {
+  return (
+    x != null &&
+    typeof (x as any).numRows === "number" &&
+    (x as any).schema != null &&
+    Array.isArray((x as any).schema.fields)
+  );
+}

package/src/arrow/types.ts

@@ -0,0 +1,73 @@
+// Backend-agnostic Arrow type surface used inside vgi-rpc-typescript.
+// Mirrors vgi-typescript's facade so structurally compatible values flow
+// freely between the two packages.
+
+export type VgiTypeId = number;
+
+export interface VgiDataType {
+  readonly typeId: VgiTypeId;
+}
+
+export interface VgiField {
+  readonly name: string;
+  readonly type: VgiDataType;
+  readonly nullable: boolean;
+  readonly metadata: Map<string, string>;
+}
+
+export interface VgiSchema {
+  readonly fields: readonly VgiField[];
+  readonly metadata: Map<string, string>;
+}
+
+export interface VgiColumn {
+  readonly type: VgiDataType;
+  readonly length: number;
+  get(index: number): unknown;
+  [Symbol.iterator](): Iterator<unknown>;
+}
+
+export interface VgiBatch {
+  readonly schema: VgiSchema;
+  readonly numRows: number;
+  readonly metadata?: Map<string, string> | null;
+  getChild(name: string): VgiColumn | null;
+  getChildAt(index: number): VgiColumn | null;
+}
+
+export interface VgiBackendInfo {
+  readonly name: "arrow-js" | "flechette";
+  /**
+   * Whether this backend's `.get(0)` round-trip is unreliable for opaque
+   * column types (Map/Date/Time/Timestamp/Duration/Decimal/LargeUtf8/
+   * LargeBinary/FixedSizeBinary/Dictionary), requiring the request parser to
+   * pass the raw column data straight through instead of materializing a
+   * value. True for arrow-js; false for flechette (which extracts cleanly).
+   */
+  readonly opaquePassthrough: boolean;
+}
+
+export type VgiColumnData = unknown;
+
+/**
+ * Incremental IPC stream encoder for the lockstep stdio transport.
+ *
+ * The stdio exchange protocol is lockstep — the client reads each response
+ * batch (and its framing bytes) before sending the next input — so we
+ * cannot buffer-then-emit at close. Each call returns the wire bytes to
+ * flush immediately. Only the stdio server uses this; HTTP serializes
+ * whole responses via {@link serializeBatches}.
+ *
+ * The arrow-js backend implements this over `RecordBatchStreamWriter`.
+ * The flechette backend has no incremental-writer surface and its
+ * factory throws — keeping arrow-js out of the flechette (workerd/
+ * browser) bundle, which is HTTP-only anyway.
+ */
+export interface IncrementalEncoder {
+  /** Bytes for the schema preamble (continuation + schema message). */
+  start(): Uint8Array;
+  /** Bytes for one record batch message. */
+  writeBatch(batch: VgiBatch): Uint8Array;
+  /** Bytes for the end-of-stream marker. */
+  finish(): Uint8Array;
+}

package/src/client/capabilities.ts

@@ -0,0 +1,84 @@
+// © Copyright 2025-2026, Query.Farm LLC - https://query.farm
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * HTTP server capability discovery.
+ *
+ * Mirrors Python's `http_capabilities()`: probes `OPTIONS {prefix}/health`
+ * and reads three response headers:
+ *   - `VGI-Max-Request-Bytes`  — server-enforced inline request cap
+ *   - `VGI-Upload-URL-Support` — "true" when the server vends upload URLs
+ *   - `VGI-Max-Upload-Bytes`   — cap on out-of-band upload size
+ *
+ * Honours `Cache-Control: max-age=N` for refresh scheduling.
+ */
+
+export interface HttpServerCapabilities {
+  /** Server's advertised max inline request body size (bytes). */
+  maxRequestBytes: number | null;
+  /** Whether the server vends upload URLs via `__upload_url__/init`. */
+  uploadUrlSupport: boolean;
+  /** Cap on the size of an externalized upload (bytes). */
+  maxUploadBytes: number | null;
+  /** Monotonic-time-ish epoch (ms) at which this snapshot should be re-probed. */
+  cacheExpiresAt: number | null;
+}
+
+const MAX_REQUEST_BYTES_HEADER = "VGI-Max-Request-Bytes";
+const UPLOAD_URL_HEADER = "VGI-Upload-URL-Support";
+const MAX_UPLOAD_BYTES_HEADER = "VGI-Max-Upload-Bytes";
+
+function parseHeaderInt(headers: Headers, name: string): number | null {
+  const raw = headers.get(name) ?? headers.get(name.toLowerCase());
+  if (raw == null) return null;
+  const parsed = Number.parseInt(raw, 10);
+  return Number.isFinite(parsed) ? parsed : null;
+}
+
+export function parseCapabilitiesFromHeaders(headers: Headers): HttpServerCapabilities {
+  const uploadRaw = headers.get(UPLOAD_URL_HEADER) ?? headers.get(UPLOAD_URL_HEADER.toLowerCase());
+  const uploadUrlSupport = uploadRaw === "true";
+
+  let cacheExpiresAt: number | null = null;
+  const cc = headers.get("Cache-Control") ?? headers.get("cache-control");
+  if (cc) {
+    for (const token of cc.split(",")) {
+      const t = token.trim().toLowerCase();
+      if (t.startsWith("max-age=")) {
+        const seconds = Number.parseFloat(t.slice("max-age=".length));
+        if (Number.isFinite(seconds)) {
+          cacheExpiresAt = Date.now() + seconds * 1000;
+        }
+        break;
+      }
+    }
+  }
+
+  return {
+    maxRequestBytes: parseHeaderInt(headers, MAX_REQUEST_BYTES_HEADER),
+    uploadUrlSupport,
+    maxUploadBytes: parseHeaderInt(headers, MAX_UPLOAD_BYTES_HEADER),
+    cacheExpiresAt,
+  };
+}
+
+export async function discoverHttpCapabilities(
+  baseUrl: string,
+  prefix: string,
+  authorization?: string,
+): Promise<HttpServerCapabilities> {
+  const headers: Record<string, string> = {};
+  if (authorization) headers.Authorization = authorization;
+  const resp = await fetch(`${baseUrl}${prefix}/health`, {
+    method: "OPTIONS",
+    headers,
+  });
+  // Capability headers are advertised on every response; we don't require 200.
+  return parseCapabilitiesFromHeaders(resp.headers);
+}
+
+export function isCapabilitySnapshotFresh(snapshot: HttpServerCapabilities | null): boolean {
+  if (!snapshot) return false;
+  if (snapshot.cacheExpiresAt == null) return true;
+  return Date.now() < snapshot.cacheExpiresAt;
+}

package/src/client/connect.ts

@@ -6,6 +6,11 @@ import { LOG_LEVEL_KEY, STATE_KEY } from "../constants.js";
 import { RpcError } from "../errors.js";
 import { isExternalLocationBatch, resolveExternalLocation } from "../external.js";
 import { ARROW_CONTENT_TYPE } from "../http/common.js";
+import {
+  type HttpServerCapabilities,
+  isCapabilitySnapshotFresh,
+  parseCapabilitiesFromHeaders,
+} from "./capabilities.js";
 import { httpIntrospect, type MethodInfo, type ServiceDescription } from "./introspect.js";
 import {
   buildRequestIpc,
@@ -16,9 +21,10 @@ import {
 } from "./ipc.js";
 import { HttpStreamSession } from "./stream.js";
 import type { HttpConnectOptions, StreamSession } from "./types.js";
+import { externalizeRequestBody } from "./uploadUrl.js";
 
-type CompressFn = (data: Uint8Array, level: number) => Uint8Array;
-type DecompressFn = (data: Uint8Array) => Uint8Array;
+type CompressFn = (data: Uint8Array, level: number) => Promise<Uint8Array>;
+type DecompressFn = (data: Uint8Array) => Promise<Uint8Array>;
 
 export interface RpcClient {
   call(method: string, params?: Record<string, any>): Promise<Record<string, any> | null>;
@@ -35,13 +41,73 @@ export function httpConnect(baseUrl: string, options?: HttpConnectOptions): RpcC
   const externalConfig = options?.externalLocation;
 
   let methodCache: Map<string, MethodInfo> | null = null;
+  /** Application protocol surface version discovered via __describe__. When
+   *  non-empty, the client emits it on every request as
+   *  `vgi_rpc.protocol_version` so a versioned server can validate at the
+   *  dispatch boundary. */
+  let serverProtocolVersion = "";
   let compressFn: CompressFn | undefined;
   let decompressFn: DecompressFn | undefined;
   let compressionLoaded = false;
+  let capabilities: HttpServerCapabilities | null = null;
+
+  function updateCapabilitiesFromResponse(resp: Response): void {
+    const next = parseCapabilitiesFromHeaders(resp.headers);
+    // Only treat the snapshot as authoritative when the server actually
+    // emitted capability hints. Otherwise leave any prior cache in place.
+    if (next.maxRequestBytes != null || next.uploadUrlSupport) {
+      capabilities = next;
+    }
+  }
+
+  async function maybeExternalize(body: Uint8Array): Promise<Uint8Array> {
+    const caps = isCapabilitySnapshotFresh(capabilities) ? capabilities : null;
+    if (!caps) return body;
+    if (!caps.uploadUrlSupport) return body;
+    if (caps.maxRequestBytes == null || body.byteLength <= caps.maxRequestBytes) return body;
+    return externalizeRequestBody(body, {
+      baseUrl,
+      prefix,
+      authorization,
+      urlValidator: externalConfig?.urlValidator ?? null,
+    });
+  }
+
+  /**
+   * Send a POST request, transparently retrying with externalization if
+   * the server returns 413 (Payload Too Large) and advertises upload-URL
+   * support. Mirrors Python's 413 fallback in `_HttpProxy._post_with_externalization`.
+   */
+  async function postWithExternalization(url: string, body: Uint8Array): Promise<Response> {
+    const sendBody = await maybeExternalize(body);
+    let resp = await fetch(url, {
+      method: "POST",
+      headers: buildHeaders(),
+      body: (await prepareBody(sendBody)) as unknown as BodyInit,
+    });
+    updateCapabilitiesFromResponse(resp);
+
+    if (resp.status === 413 && capabilities?.uploadUrlSupport && body.byteLength > 0) {
+      // Refresh-and-retry: caps tell us we can externalize.
+      const externalized = await externalizeRequestBody(body, {
+        baseUrl,
+        prefix,
+        authorization,
+        urlValidator: externalConfig?.urlValidator ?? null,
+      });
+      resp = await fetch(url, {
+        method: "POST",
+        headers: buildHeaders(),
+        body: (await prepareBody(externalized)) as unknown as BodyInit,
+      });
+      updateCapabilitiesFromResponse(resp);
+    }
+
+    return resp;
+  }
 
   async function ensureCompression(): Promise<void> {
     if (compressionLoaded || compressionLevel == null) return;
-    compressionLoaded = true;
     try {
       const mod = await import("../util/zstd.js");
       compressFn = mod.zstdCompress;
@@ -49,14 +115,17 @@ export function httpConnect(baseUrl: string, options?: HttpConnectOptions): RpcC
     } catch {
       // zstd not available in this runtime
     }
+    compressionLoaded = true;
   }
 
   function buildHeaders(): Record<string, string> {
     const headers: Record<string, string> = {
       "Content-Type": ARROW_CONTENT_TYPE,
     };
-    if (compressionLevel != null) {
+    if (compressionLevel != null && compressFn) {
       headers["Content-Encoding"] = "zstd";
+    }
+    if (compressionLevel != null && decompressFn) {
       headers["Accept-Encoding"] = "zstd";
     }
     if (authorization) {
@@ -65,9 +134,9 @@ export function httpConnect(baseUrl: string, options?: HttpConnectOptions): RpcC
     return headers;
   }
 
-  function prepareBody(content: Uint8Array): Uint8Array {
+  async function prepareBody(content: Uint8Array): Promise<Uint8Array> {
     if (compressionLevel != null && compressFn) {
-      return compressFn(content, compressionLevel);
+      return await compressFn(content, compressionLevel);
     }
     return content;
   }
@@ -81,15 +150,23 @@ export function httpConnect(baseUrl: string, options?: HttpConnectOptions): RpcC
   async function readResponse(resp: Response): Promise<Uint8Array<ArrayBuffer>> {
     let body = new Uint8Array(await resp.arrayBuffer());
     if (resp.headers.get("Content-Encoding") === "zstd" && decompressFn) {
-      body = new Uint8Array(decompressFn(body));
+      body = new Uint8Array(await decompressFn(body));
     }
     return body;
   }
 
   async function ensureMethodCache(): Promise<Map<string, MethodInfo>> {
     if (methodCache) return methodCache;
-    const desc = await httpIntrospect(baseUrl, { prefix, authorization });
+    await ensureCompression();
+    const desc = await httpIntrospect(baseUrl, {
+      prefix,
+      authorization,
+      compressionLevel,
+      compressFn,
+      decompressFn,
+    });
     methodCache = new Map(desc.methods.map((m) => [m.name, m]));
+    serverProtocolVersion = desc.protocolVersion;
     return methodCache;
   }
 
@@ -105,12 +182,8 @@ export function httpConnect(baseUrl: string, options?: HttpConnectOptions): RpcC
       // Apply defaults
       const fullParams = { ...(info.defaults ?? {}), ...(params ?? {}) };
 
-      const body = buildRequestIpc(info.paramsSchema, fullParams, method);
-      const resp = await fetch(`${baseUrl}${prefix}/${method}`, {
-        method: "POST",
-        headers: buildHeaders(),
-        body: prepareBody(body) as unknown as BodyInit,
-      });
+      const body = buildRequestIpc(info.paramsSchema, fullParams, method, { protocolVersion: serverProtocolVersion });
+      const resp = await postWithExternalization(`${baseUrl}${prefix}/${method}`, body);
       checkAuth(resp);
 
       const responseBody = await readResponse(resp);
@@ -121,8 +194,8 @@ export function httpConnect(baseUrl: string, options?: HttpConnectOptions): RpcC
       for (let batch of batches) {
         if (batch.numRows === 0) {
           // Check for external location pointer batch
-          if (isExternalLocationBatch(batch)) {
-            batch = await resolveExternalLocation(batch, externalConfig);
+          if (isExternalLocationBatch(batch as any)) {
+            batch = (await resolveExternalLocation(batch as any, externalConfig)) as any;
           } else {
             dispatchLogOrError(batch, onLog);
             continue;
@@ -159,12 +232,8 @@ export function httpConnect(baseUrl: string, options?: HttpConnectOptions): RpcC
       // Apply defaults
       const fullParams = { ...(info.defaults ?? {}), ...(params ?? {}) };
 
-      const body = buildRequestIpc(info.paramsSchema, fullParams, method);
-      const resp = await fetch(`${baseUrl}${prefix}/${method}/init`, {
-        method: "POST",
-        headers: buildHeaders(),
-        body: prepareBody(body) as unknown as BodyInit,
-      });
+      const body = buildRequestIpc(info.paramsSchema, fullParams, method, { protocolVersion: serverProtocolVersion });
+      const resp = await postWithExternalization(`${baseUrl}${prefix}/${method}/init`, body);
       checkAuth(resp);
 
       const responseBody = await readResponse(resp);
@@ -185,7 +254,7 @@ export function httpConnect(baseUrl: string, options?: HttpConnectOptions): RpcC
         // First stream: header
         const headerStream = await reader.readStream();
         if (headerStream) {
-          for (const batch of headerStream.batches) {
+          for (const batch of headerStream.batches as any[]) {
             if (batch.numRows === 0) {
               dispatchLogOrError(batch, onLog);
               continue;
@@ -200,11 +269,11 @@ export function httpConnect(baseUrl: string, options?: HttpConnectOptions): RpcC
         // Second stream: data/state
         const dataStream = await reader.readStream();
         if (dataStream) {
-          streamSchema = dataStream.schema;
+          streamSchema = dataStream.schema as any;
         }
         const headerErrorBatches: RecordBatch[] = [];
         if (dataStream) {
-          for (const batch of dataStream.batches) {
+          for (const batch of dataStream.batches as any[]) {
             if (batch.numRows === 0) {
               // Check for state token
               const token = batch.metadata?.get(STATE_KEY);
@@ -310,11 +379,19 @@ export function httpConnect(baseUrl: string, options?: HttpConnectOptions): RpcC
         decompressFn,
         authorization,
         externalConfig,
+        postFn: postWithExternalization,
       });
     },
 
     async describe(): Promise<ServiceDescription> {
-      return httpIntrospect(baseUrl, { prefix });
+      await ensureCompression();
+      return httpIntrospect(baseUrl, {
+        prefix,
+        authorization,
+        compressionLevel,
+        compressFn,
+        decompressFn,
+      });
     },
 
     close(): void {

package/src/client/introspect.ts

@@ -1,8 +1,9 @@
 // © Copyright 2025-2026, Query.Farm LLC - https://query.farm
 // SPDX-License-Identifier: Apache-2.0
 
-import { Schema as ArrowSchema, type RecordBatch, RecordBatchReader, type Schema } from "@query-farm/apache-arrow";
-import { DESCRIBE_METHOD_NAME, PROTOCOL_NAME_KEY } from "../constants.js";
+import { Schema as ArrowSchema, type RecordBatch, type Schema } from "@query-farm/apache-arrow";
+import { deserializeSchema as deserializeSchemaImpl } from "#vgi-rpc-arrow";
+import { DESCRIBE_METHOD_NAME, PROTOCOL_NAME_KEY, PROTOCOL_VERSION_KEY } from "../constants.js";
 import { RpcError } from "../errors.js";
 import { ARROW_CONTENT_TYPE } from "../http/common.js";
 import { buildRequestIpc, dispatchLogOrError, readResponseBatches } from "./ipc.js";
@@ -23,14 +24,29 @@ export interface MethodInfo {
 
 export interface ServiceDescription {
   protocolName: string;
+  /** Application protocol surface version surfaced by the server's
+   *  __describe__ response. Empty string when the server did not declare
+   *  a `protocolVersion`. */
+  protocolVersion: string;
   methods: MethodInfo[];
 }
 
-/** Deserialize a schema from IPC bytes (schema message + EOS). */
-async function deserializeSchema(bytes: Uint8Array): Promise<Schema> {
-  const reader = await RecordBatchReader.from(bytes);
-  await reader.open();
-  return reader.schema!;
+/**
+ * Deserialize a schema from IPC bytes (schema message + EOS).
+ *
+ * Must dispatch via `#vgi-rpc-arrow` so the resulting type instances are
+ * the same impl (apache-arrow / flechette) as the rest of the active
+ * backend. Using apache-arrow's `RecordBatchReader` directly here used to
+ * silently mix impls: in browser builds the backend is flechette, and a
+ * flechette builder receiving an apache-arrow `Binary` type defaults to
+ * the wrong offsets buffer (Uint8Array instead of Int32Array) and emits
+ * a 0-byte value where a populated binary column was expected. The
+ * downstream symptom is "Tried reading schema message, was null or
+ * length 0" from the server when it tries to open the (empty) binary
+ * column as a nested IPC stream. See test/client/ipc-cross-impl.test.ts.
+ */
+function deserializeSchema(bytes: Uint8Array): Schema {
+  return deserializeSchemaImpl(bytes) as unknown as Schema;
 }
 
 /**
@@ -58,45 +74,30 @@ export async function parseDescribeResponse(
   // Extract metadata from batch
   const meta = dataBatch.metadata;
   const protocolName = meta?.get(PROTOCOL_NAME_KEY) ?? "";
+  const protocolVersion = meta?.get(PROTOCOL_VERSION_KEY) ?? "";
 
+  // Slim DESCRIBE_VERSION 4 wire format (see dispatch/describe.ts):
+  //   0:name 1:method_type 2:has_return 3:params_schema_ipc
+  //   4:result_schema_ipc 5:has_header 6:header_schema_ipc 7:is_exchange
   const methods: MethodInfo[] = [];
   for (let i = 0; i < dataBatch.numRows; i++) {
-    const name = dataBatch.getChildAt(0)!.get(i) as string; // name
-    const methodType = dataBatch.getChildAt(1)!.get(i) as string; // method_type
-    const doc = dataBatch.getChildAt(2)?.get(i) as string | null; // doc
-    const _hasReturn = dataBatch.getChildAt(3)!.get(i) as boolean; // has_return
-    const paramsIpc = dataBatch.getChildAt(4)!.get(i) as Uint8Array; // params_schema_ipc
-    const resultIpc = dataBatch.getChildAt(5)!.get(i) as Uint8Array; // result_schema_ipc
-    const paramTypesJson = dataBatch.getChildAt(6)?.get(i) as string | null; // param_types_json
-    const paramDefaultsJson = dataBatch.getChildAt(7)?.get(i) as string | null; // param_defaults_json
-    const hasHeader = dataBatch.getChildAt(8)!.get(i) as boolean; // has_header
-    const headerIpc = dataBatch.getChildAt(9)?.get(i) as Uint8Array | null; // header_schema_ipc
+    const name = dataBatch.getChildAt(0)!.get(i) as string;
+    const methodType = dataBatch.getChildAt(1)!.get(i) as string;
+    const _hasReturn = dataBatch.getChildAt(2)!.get(i) as boolean;
+    const paramsIpc = dataBatch.getChildAt(3)!.get(i) as Uint8Array;
+    const resultIpc = dataBatch.getChildAt(4)!.get(i) as Uint8Array;
+    const hasHeader = dataBatch.getChildAt(5)!.get(i) as boolean;
+    const headerIpc = dataBatch.getChildAt(6)?.get(i) as Uint8Array | null;
+    // is_exchange (index 7) currently unused on the client side.
 
     const paramsSchema = await deserializeSchema(paramsIpc);
     const resultSchema = await deserializeSchema(resultIpc);
 
-    let paramTypes: Record<string, string> | undefined;
-    if (paramTypesJson) {
-      try {
-        paramTypes = JSON.parse(paramTypesJson);
-      } catch {}
-    }
-
-    let defaults: Record<string, any> | undefined;
-    if (paramDefaultsJson) {
-      try {
-        defaults = JSON.parse(paramDefaultsJson);
-      } catch {}
-    }
-
     const info: MethodInfo = {
       name,
       type: methodType as "unary" | "stream",
       paramsSchema,
       resultSchema,
-      doc: doc ?? undefined,
-      paramTypes,
-      defaults,
     };
 
     // For stream methods, result_schema_ipc actually holds the output schema
@@ -111,7 +112,7 @@ export async function parseDescribeResponse(
     methods.push(info);
   }
 
-  return { protocolName, methods };
+  return { protocolName, protocolVersion, methods };
 }
 
 /**
@@ -119,7 +120,13 @@ export async function parseDescribeResponse(
  */
 export async function httpIntrospect(
   baseUrl: string,
-  options?: { prefix?: string; authorization?: string },
+  options?: {
+    prefix?: string;
+    authorization?: string;
+    compressionLevel?: number;
+    compressFn?: (data: Uint8Array, level: number) => Promise<Uint8Array>;
+    decompressFn?: (data: Uint8Array) => Promise<Uint8Array>;
+  },
 ): Promise<ServiceDescription> {
   const prefix = options?.prefix ?? "";
   const emptySchema = new ArrowSchema([]);
@@ -130,16 +137,31 @@ export async function httpIntrospect(
     headers.Authorization = options.authorization;
   }
 
+  const level = options?.compressionLevel;
+  const compressFn = options?.compressFn;
+  const decompressFn = options?.decompressFn;
+  let sendBody: Uint8Array = body;
+  if (level != null && compressFn) {
+    headers["Content-Encoding"] = "zstd";
+    sendBody = await compressFn(body, level);
+  }
+  if (level != null && decompressFn) {
+    headers["Accept-Encoding"] = "zstd";
+  }
+
   const response = await fetch(`${baseUrl}${prefix}/${DESCRIBE_METHOD_NAME}`, {
     method: "POST",
     headers,
-    body: body as unknown as BodyInit,
+    body: sendBody as unknown as BodyInit,
   });
   if (response.status === 401) {
     throw new RpcError("AuthenticationError", "Authentication required", "");
   }
 
-  const responseBody = new Uint8Array(await response.arrayBuffer());
+  let responseBody = new Uint8Array(await response.arrayBuffer());
+  if (response.headers.get("Content-Encoding") === "zstd" && decompressFn) {
+    responseBody = new Uint8Array(await decompressFn(responseBody));
+  }
   const { batches } = await readResponseBatches(responseBody);
 
   return parseDescribeResponse(batches);