@query-farm/vgi-rpc 0.4.0 → 0.6.0

package/src/dispatch/describe.ts

@@ -37,6 +37,8 @@ export const DESCRIBE_SCHEMA = new Schema([
   new Field("param_defaults_json", new Utf8(), true),
   new Field("has_header", new Bool(), false),
   new Field("header_schema_ipc", new Binary(), true),
+  new Field("is_exchange", new Bool(), true),
+  new Field("param_docs_json", new Utf8(), true),
 ]);
 
 /**
@@ -60,6 +62,8 @@ export function buildDescribeBatch(
   const paramDefaultsJsons: (string | null)[] = [];
   const hasHeaders: boolean[] = [];
   const headerSchemas: (Uint8Array | null)[] = [];
+  const isExchanges: (boolean | null)[] = [];
+  const paramDocsJsons: (string | null)[] = [];
 
   for (const [name, method] of sortedEntries) {
     names.push(name);
@@ -95,6 +99,18 @@ export function buildDescribeBatch(
 
     hasHeaders.push(!!method.headerSchema);
     headerSchemas.push(method.headerSchema ? serializeSchema(method.headerSchema) : null);
+
+    // is_exchange: true for exchange, false for producer, null for unary
+    if (method.exchangeFn) {
+      isExchanges.push(true);
+    } else if (method.producerFn) {
+      isExchanges.push(false);
+    } else {
+      isExchanges.push(null);
+    }
+
+    // param_docs_json: no docstring source in TypeScript, always null
+    paramDocsJsons.push(null);
   }
 
   // Build the batch using vectorFromArray for each column
@@ -108,6 +124,8 @@ export function buildDescribeBatch(
   const paramDefaultsArr = vectorFromArray(paramDefaultsJsons, new Utf8());
   const hasHeaderArr = vectorFromArray(hasHeaders, new Bool());
   const headerSchemaArr = vectorFromArray(headerSchemas, new Binary());
+  const isExchangeArr = vectorFromArray(isExchanges, new Bool());
+  const paramDocsArr = vectorFromArray(paramDocsJsons, new Utf8());
 
   const children = [
     nameArr.data[0],
@@ -120,6 +138,8 @@ export function buildDescribeBatch(
     paramDefaultsArr.data[0],
     hasHeaderArr.data[0],
     headerSchemaArr.data[0],
+    isExchangeArr.data[0],
+    paramDocsArr.data[0],
   ];
 
   const structType = new Struct(DESCRIBE_SCHEMA.fields);

package/src/dispatch/stream.ts

@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 import { Schema } from "@query-farm/apache-arrow";
+import { type ExternalLocationConfig, maybeExternalizeBatch } from "../external.js";
 import type { MethodDefinition } from "../types.js";
 import { OutputCollector } from "../types.js";
 import { conformBatchToSchema } from "../util/conform.js";
@@ -33,6 +34,7 @@ export async function dispatchStream(
   reader: IpcStreamReader,
   serverId: string,
   requestId: string | null,
+  externalConfig?: ExternalLocationConfig,
 ): Promise<void> {
   const isProducer = !!method.producerFn;
 
@@ -133,7 +135,11 @@ export async function dispatchStream(
       }
 
       for (const emitted of out.batches) {
-        stream.write(emitted.batch);
+        let batch = emitted.batch;
+        if (externalConfig) {
+          batch = await maybeExternalizeBatch(batch, externalConfig);
+        }
+        stream.write(batch);
       }
 
       if (out.finished) {

package/src/dispatch/unary.ts

@@ -1,6 +1,7 @@
 // © Copyright 2025-2026, Query.Farm LLC - https://query.farm
 // SPDX-License-Identifier: Apache-2.0
 
+import { type ExternalLocationConfig, maybeExternalizeBatch } from "../external.js";
 import type { MethodDefinition } from "../types.js";
 import { OutputCollector } from "../types.js";
 import { buildErrorBatch, buildResultBatch } from "../wire/response.js";
@@ -17,13 +18,17 @@ export async function dispatchUnary(
   writer: IpcStreamWriter,
   serverId: string,
   requestId: string | null,
+  externalConfig?: ExternalLocationConfig,
 ): Promise<void> {
   const schema = method.resultSchema;
   const out = new OutputCollector(schema, true, serverId, requestId);
 
   try {
     const result = await method.handler!(params, out);
-    const resultBatch = buildResultBatch(schema, result, serverId, requestId);
+    let resultBatch = buildResultBatch(schema, result, serverId, requestId);
+    if (externalConfig) {
+      resultBatch = await maybeExternalizeBatch(resultBatch, externalConfig);
+    }
     // Collect log batches (from clientLog) + result batch
     const batches = [...out.batches.map((b) => b.batch), resultBatch];
     writer.writeStream(schema, batches);

package/src/external.ts

@@ -0,0 +1,209 @@
+// © Copyright 2025-2026, Query.Farm LLC - https://query.farm
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * External storage support for large Arrow IPC batches.
+ *
+ * When a batch exceeds a configurable threshold, it is serialized to IPC,
+ * optionally compressed with zstd, and uploaded to pluggable storage.
+ * The batch is replaced with a zero-row "pointer batch" containing the
+ * download URL and SHA-256 checksum in metadata.
+ */
+
+import { type RecordBatch, RecordBatchReader, RecordBatchStreamWriter, type Schema } from "@query-farm/apache-arrow";
+import { LOCATION_KEY, LOCATION_SHA256_KEY, LOG_LEVEL_KEY } from "./constants.js";
+import { zstdCompress, zstdDecompress } from "./util/zstd.js";
+import { buildEmptyBatch } from "./wire/response.js";
+
+// ---------------------------------------------------------------------------
+// Interfaces and configuration
+// ---------------------------------------------------------------------------
+
+/** Pluggable storage backend for uploading large batches. */
+export interface ExternalStorage {
+  /** Upload IPC data and return a URL for retrieval. */
+  upload(data: Uint8Array, contentEncoding: string): Promise<string>;
+}
+
+/** Configuration for external storage of large batches. */
+export interface ExternalLocationConfig {
+  /** Storage backend for uploading. */
+  storage: ExternalStorage;
+  /** Minimum batch byte size to trigger externalization. Default: 1MB. */
+  externalizeThresholdBytes?: number;
+  /** Optional zstd compression for uploaded data. */
+  compression?: { algorithm: "zstd"; level?: number };
+  /** URL validator called before fetching. Throw to reject. Default: HTTPS-only. */
+  urlValidator?: ((url: string) => void) | null;
+}
+
+const DEFAULT_THRESHOLD = 1_048_576; // 1 MB
+
+// ---------------------------------------------------------------------------
+// URL validation
+// ---------------------------------------------------------------------------
+
+/** Default validator that rejects non-HTTPS URLs. */
+export function httpsOnlyValidator(url: string): void {
+  const parsed = new URL(url);
+  if (parsed.protocol !== "https:") {
+    throw new Error(`External location URL must use HTTPS, got "${parsed.protocol}"`);
+  }
+}
+
+// ---------------------------------------------------------------------------
+// SHA-256 helpers
+// ---------------------------------------------------------------------------
+
+async function sha256Hex(data: Uint8Array): Promise<string> {
+  // Copy to a plain ArrayBuffer to satisfy Web Crypto API type requirements
+  const buf = new ArrayBuffer(data.byteLength);
+  new Uint8Array(buf).set(data);
+  const hash = await crypto.subtle.digest("SHA-256", buf);
+  return Array.from(new Uint8Array(hash))
+    .map((b) => b.toString(16).padStart(2, "0"))
+    .join("");
+}
+
+// ---------------------------------------------------------------------------
+// Detection
+// ---------------------------------------------------------------------------
+
+/** Returns true if the batch is a zero-row pointer to external data. */
+export function isExternalLocationBatch(batch: RecordBatch): boolean {
+  if (batch.numRows !== 0) return false;
+  const meta = batch.metadata;
+  if (!meta) return false;
+  return meta.has(LOCATION_KEY) && !meta.has(LOG_LEVEL_KEY);
+}
+
+// ---------------------------------------------------------------------------
+// Pointer batch creation
+// ---------------------------------------------------------------------------
+
+/** Create a zero-row pointer batch with location URL and optional SHA-256. */
+export function makeExternalLocationBatch(schema: Schema, url: string, sha256?: string): RecordBatch {
+  const metadata = new Map<string, string>();
+  metadata.set(LOCATION_KEY, url);
+  if (sha256) {
+    metadata.set(LOCATION_SHA256_KEY, sha256);
+  }
+  return buildEmptyBatch(schema, metadata);
+}
+
+// ---------------------------------------------------------------------------
+// IPC serialization helpers
+// ---------------------------------------------------------------------------
+
+function serializeBatchToIpc(batch: RecordBatch): Uint8Array {
+  const writer = new RecordBatchStreamWriter();
+  writer.reset(undefined, batch.schema);
+  writer.write(batch);
+  writer.close();
+  return writer.toUint8Array(true);
+}
+
+function batchByteSize(batch: RecordBatch): number {
+  // Arrow TS data.byteLength doesn't reflect actual data size.
+  // Estimate from IPC serialization size for threshold check.
+  const writer = new RecordBatchStreamWriter();
+  writer.reset(undefined, batch.schema);
+  writer.write(batch);
+  writer.close();
+  return writer.toUint8Array(true).byteLength;
+}
+
+// ---------------------------------------------------------------------------
+// Write path: externalization
+// ---------------------------------------------------------------------------
+
+/**
+ * Maybe externalize a batch if it exceeds the threshold.
+ * Returns the original batch unchanged if below threshold or no config.
+ */
+export async function maybeExternalizeBatch(
+  batch: RecordBatch,
+  config?: ExternalLocationConfig | null,
+): Promise<RecordBatch> {
+  if (!config?.storage) return batch;
+  if (batch.numRows === 0) return batch;
+
+  const threshold = config.externalizeThresholdBytes ?? DEFAULT_THRESHOLD;
+  if (batchByteSize(batch) < threshold) return batch;
+
+  // Serialize to IPC
+  let ipcData = serializeBatchToIpc(batch);
+
+  // Compute SHA-256 of raw IPC bytes (pre-compression)
+  const checksum = await sha256Hex(ipcData);
+
+  // Optionally compress
+  let contentEncoding = "";
+  if (config.compression?.algorithm === "zstd") {
+    ipcData = zstdCompress(ipcData, config.compression.level ?? 3) as Uint8Array;
+    contentEncoding = "zstd";
+  }
+
+  // Upload
+  const url = await config.storage.upload(ipcData, contentEncoding);
+
+  // Return pointer batch
+  return makeExternalLocationBatch(batch.schema, url, checksum);
+}
+
+// ---------------------------------------------------------------------------
+// Read path: resolution
+// ---------------------------------------------------------------------------
+
+/**
+ * Resolve an external pointer batch by fetching the data from the URL.
+ * Returns the original batch unchanged if not a pointer or no config.
+ */
+export async function resolveExternalLocation(
+  batch: RecordBatch,
+  config?: ExternalLocationConfig | null,
+): Promise<RecordBatch> {
+  if (!config) return batch;
+  if (!isExternalLocationBatch(batch)) return batch;
+
+  const url = batch.metadata?.get(LOCATION_KEY);
+  if (!url) return batch;
+
+  // Validate URL
+  const validator = config.urlValidator === null ? undefined : (config.urlValidator ?? httpsOnlyValidator);
+  if (validator) {
+    validator(url);
+  }
+
+  // Fetch
+  const response = await fetch(url);
+  if (!response.ok) {
+    throw new Error(`External location fetch failed: ${response.status} ${response.statusText} [url: ${url}]`);
+  }
+  let data = new Uint8Array(await response.arrayBuffer());
+
+  // Decompress if needed
+  const contentEncoding = response.headers.get("Content-Encoding");
+  if (contentEncoding === "zstd") {
+    data = new Uint8Array(zstdDecompress(data));
+  }
+
+  // Verify SHA-256 if present
+  const expectedSha256 = batch.metadata?.get(LOCATION_SHA256_KEY);
+  if (expectedSha256) {
+    const actualSha256 = await sha256Hex(data);
+    if (actualSha256 !== expectedSha256) {
+      throw new Error(`SHA-256 checksum mismatch for ${url}: expected ${expectedSha256}, got ${actualSha256}`);
+    }
+  }
+
+  // Parse IPC stream
+  const reader = await RecordBatchReader.from(data);
+  await reader.open();
+  const resolved = reader.next();
+  if (!resolved || resolved.done || !resolved.value) {
+    throw new Error(`No data batch found in external IPC stream from ${url}`);
+  }
+
+  return resolved.value;
+}

package/src/gcs.ts

@@ -0,0 +1,86 @@
+// © Copyright 2025-2026, Query.Farm LLC - https://query.farm
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * Google Cloud Storage backend for external storage of large Arrow IPC batches.
+ *
+ * Requires `@google-cloud/storage` as a peer dependency.
+ *
+ * @example
+ * ```typescript
+ * import { createGCSStorage } from "@query-farm/vgi-rpc/gcs";
+ *
+ * const storage = createGCSStorage({
+ *   bucket: "my-bucket",
+ *   prefix: "vgi-rpc/",
+ * });
+ * const handler = createHttpHandler(protocol, {
+ *   externalLocation: { storage, externalizeThresholdBytes: 1_048_576 },
+ * });
+ * ```
+ */
+
+import type { ExternalStorage } from "./external.js";
+
+/** Configuration for the GCS storage backend. */
+export interface GCSStorageConfig {
+  /** GCS bucket name. */
+  bucket: string;
+  /** Key prefix for uploaded objects. Default: "vgi-rpc/". */
+  prefix?: string;
+  /** Lifetime of signed GET URLs in seconds. Default: 3600 (1 hour). */
+  presignExpirySeconds?: number;
+  /** GCS project ID. If omitted, uses Application Default Credentials. */
+  projectId?: string;
+}
+
+/**
+ * Create a GCS-backed ExternalStorage.
+ *
+ * Lazily imports `@google-cloud/storage` on first upload to avoid
+ * loading the SDK unless needed.
+ */
+export function createGCSStorage(config: GCSStorageConfig): ExternalStorage {
+  const bucket = config.bucket;
+  const prefix = config.prefix ?? "vgi-rpc/";
+  const presignExpiry = config.presignExpirySeconds ?? 3600;
+
+  let storageClient: any = null;
+
+  async function ensureClient(): Promise<any> {
+    if (storageClient) return storageClient;
+    const { Storage } = await import("@google-cloud/storage");
+    const clientConfig: Record<string, any> = {};
+    if (config.projectId) clientConfig.projectId = config.projectId;
+    storageClient = new Storage(clientConfig);
+    return storageClient;
+  }
+
+  return {
+    async upload(data: Uint8Array, contentEncoding: string): Promise<string> {
+      const client = await ensureClient();
+      const bucketRef = client.bucket(bucket);
+      const blobName = `${prefix}${crypto.randomUUID()}${contentEncoding === "zstd" ? ".arrow.zst" : ".arrow"}`;
+      const blob = bucketRef.file(blobName);
+
+      const options: Record<string, any> = {
+        contentType: "application/vnd.apache.arrow.stream",
+        resumable: false,
+      };
+      if (contentEncoding) {
+        options.metadata = { contentEncoding };
+      }
+
+      await blob.save(Buffer.from(data), options);
+
+      // Generate signed GET URL
+      const [url] = await blob.getSignedUrl({
+        version: "v4" as const,
+        action: "read" as const,
+        expires: Date.now() + presignExpiry * 1000,
+      });
+
+      return url;
+    },
+  };
+}

package/src/http/auth.ts

@@ -12,10 +12,21 @@ export interface OAuthResourceMetadata {
   authorizationServers: string[];
   scopesSupported?: string[];
   bearerMethodsSupported?: string[];
+  resourceSigningAlgValuesSupported?: string[];
   resourceName?: string;
   resourceDocumentation?: string;
   resourcePolicyUri?: string;
   resourceTosUri?: string;
+  /** OAuth client_id that clients should use with the authorization server. */
+  clientId?: string;
+  /** OAuth client_secret that clients should use with the authorization server. */
+  clientSecret?: string;
+  /** OAuth client_id for device code flow. */
+  deviceCodeClientId?: string;
+  /** OAuth client_secret for device code flow. */
+  deviceCodeClientSecret?: string;
+  /** When true, clients should use the OIDC id_token as the Bearer token instead of access_token. */
+  useIdTokenAsBearer?: boolean;
 }
 
 /** Convert OAuthResourceMetadata to RFC 9728 snake_case JSON object. */
@@ -26,10 +37,39 @@ export function oauthResourceMetadataToJson(metadata: OAuthResourceMetadata): Re
   };
   if (metadata.scopesSupported) json.scopes_supported = metadata.scopesSupported;
   if (metadata.bearerMethodsSupported) json.bearer_methods_supported = metadata.bearerMethodsSupported;
+  if (metadata.resourceSigningAlgValuesSupported)
+    json.resource_signing_alg_values_supported = metadata.resourceSigningAlgValuesSupported;
   if (metadata.resourceName) json.resource_name = metadata.resourceName;
   if (metadata.resourceDocumentation) json.resource_documentation = metadata.resourceDocumentation;
   if (metadata.resourcePolicyUri) json.resource_policy_uri = metadata.resourcePolicyUri;
   if (metadata.resourceTosUri) json.resource_tos_uri = metadata.resourceTosUri;
+  if (metadata.clientId) {
+    if (!/^[A-Za-z0-9\-._~]+$/.test(metadata.clientId)) {
+      throw new Error(`Invalid client_id: must contain only URL-safe characters [A-Za-z0-9\\-._~]`);
+    }
+    json.client_id = metadata.clientId;
+  }
+  if (metadata.clientSecret) {
+    if (!/^[A-Za-z0-9\-._~]+$/.test(metadata.clientSecret)) {
+      throw new Error(`Invalid client_secret: must contain only URL-safe characters [A-Za-z0-9\\-._~]`);
+    }
+    json.client_secret = metadata.clientSecret;
+  }
+  if (metadata.deviceCodeClientId) {
+    if (!/^[A-Za-z0-9\-._~]+$/.test(metadata.deviceCodeClientId)) {
+      throw new Error(`Invalid device_code_client_id: must contain only URL-safe characters [A-Za-z0-9\\-._~]`);
+    }
+    json.device_code_client_id = metadata.deviceCodeClientId;
+  }
+  if (metadata.deviceCodeClientSecret) {
+    if (!/^[A-Za-z0-9\-._~]+$/.test(metadata.deviceCodeClientSecret)) {
+      throw new Error(`Invalid device_code_client_secret: must contain only URL-safe characters [A-Za-z0-9\\-._~]`);
+    }
+    json.device_code_client_secret = metadata.deviceCodeClientSecret;
+  }
+  if (metadata.useIdTokenAsBearer) {
+    json.use_id_token_as_bearer = true;
+  }
   return json;
 }
 
@@ -38,10 +78,33 @@ export function wellKnownPath(prefix: string): string {
   return `/.well-known/oauth-protected-resource${prefix}`;
 }
 
-/** Build a WWW-Authenticate header value with optional resource_metadata URL. */
-export function buildWwwAuthenticateHeader(metadataUrl?: string): string {
+/** Build a WWW-Authenticate header value with optional resource_metadata URL, client_id, client_secret, device_code_client_id, device_code_client_secret, and use_id_token_as_bearer. */
+export function buildWwwAuthenticateHeader(
+  metadataUrl?: string,
+  clientId?: string,
+  clientSecret?: string,
+  useIdTokenAsBearer?: boolean,
+  deviceCodeClientId?: string,
+  deviceCodeClientSecret?: string,
+): string {
+  let header = "Bearer";
   if (metadataUrl) {
-    return `Bearer resource_metadata="${metadataUrl}"`;
+    header += ` resource_metadata="${metadataUrl}"`;
+  }
+  if (clientId) {
+    header += `, client_id="${clientId}"`;
+  }
+  if (clientSecret) {
+    header += `, client_secret="${clientSecret}"`;
+  }
+  if (deviceCodeClientId) {
+    header += `, device_code_client_id="${deviceCodeClientId}"`;
+  }
+  if (deviceCodeClientSecret) {
+    header += `, device_code_client_secret="${deviceCodeClientSecret}"`;
+  }
+  if (useIdTokenAsBearer) {
+    header += `, use_id_token_as_bearer="true"`;
   }
-  return "Bearer";
+  return header;
 }

package/src/http/bearer.ts

@@ -0,0 +1,107 @@
+// © Copyright 2025-2026, Query.Farm LLC - https://query.farm
+// SPDX-License-Identifier: Apache-2.0
+
+import { timingSafeEqual } from "node:crypto";
+import type { AuthContext } from "../auth.js";
+import type { AuthenticateFn } from "./auth.js";
+
+/** Receives the raw bearer token string, returns an AuthContext on success. Must throw on failure. */
+export type BearerValidateFn = (token: string) => AuthContext | Promise<AuthContext>;
+
+/**
+ * Create a bearer-token authenticate callback.
+ *
+ * Extracts the `Authorization: Bearer <token>` header and delegates
+ * validation to the user-supplied `validate` callback.
+ */
+export function bearerAuthenticate(options: { validate: BearerValidateFn }): AuthenticateFn {
+  const { validate } = options;
+
+  return async function authenticate(request: Request): Promise<AuthContext> {
+    const authHeader = request.headers.get("Authorization") ?? "";
+    if (!authHeader.startsWith("Bearer ")) {
+      throw new Error("Missing or invalid Authorization header");
+    }
+    const token = authHeader.slice(7);
+    return validate(token);
+  };
+}
+
+/** Constant-time string comparison to prevent timing attacks on token lookup. */
+function safeEqual(a: string, b: string): boolean {
+  const enc = new TextEncoder();
+  const bufA = enc.encode(a);
+  const bufB = enc.encode(b);
+  if (bufA.byteLength !== bufB.byteLength) return false;
+  return timingSafeEqual(bufA, bufB);
+}
+
+/**
+ * Create a bearer-token authenticate callback from a static token map.
+ *
+ * Convenience wrapper around `bearerAuthenticate` that looks up the
+ * token in a pre-built mapping using constant-time comparison.
+ */
+export function bearerAuthenticateStatic(options: {
+  tokens: ReadonlyMap<string, AuthContext> | Record<string, AuthContext>;
+}): AuthenticateFn {
+  const entries: [string, AuthContext][] =
+    options.tokens instanceof Map ? [...options.tokens.entries()] : Object.entries(options.tokens);
+
+  function validate(token: string): AuthContext {
+    for (const [key, ctx] of entries) {
+      if (safeEqual(token, key)) return ctx;
+    }
+    throw new Error("Unknown bearer token");
+  }
+
+  return bearerAuthenticate({ validate });
+}
+
+/**
+ * Check whether an error represents a credential rejection (should be
+ * caught by the chain) vs a bug or authorization failure (should propagate).
+ *
+ * Mirrors Python's semantics where only `ValueError` is caught:
+ * - Plain `Error` (constructor === Error) without `PermissionError` name → credential rejection
+ * - `TypeError`, `RangeError`, etc. (Error subclasses) → bug, propagate
+ * - `PermissionError` name → authorization failure, propagate
+ * - Non-Error throws → propagate
+ */
+function isCredentialError(err: unknown): err is Error {
+  return err instanceof Error && err.constructor === Error && err.name !== "PermissionError";
+}
+
+/**
+ * Chain multiple authenticate callbacks, trying each in order.
+ *
+ * Each authenticator is called in sequence. Plain `Error` (credential
+ * rejection) causes the next authenticator to be tried. Error subclasses
+ * (`TypeError`, `RangeError`, etc.), `PermissionError`-named errors, and
+ * non-Error throws propagate immediately.
+ *
+ * @throws Error if no authenticators are provided.
+ */
+export function chainAuthenticate(...authenticators: AuthenticateFn[]): AuthenticateFn {
+  if (authenticators.length === 0) {
+    throw new Error("chainAuthenticate requires at least one authenticator");
+  }
+
+  return async function authenticate(request: Request): Promise<AuthContext> {
+    let lastError: Error | null = null;
+    for (const authFn of authenticators) {
+      try {
+        return await authFn(request);
+      } catch (err) {
+        if (isCredentialError(err)) {
+          lastError = err;
+          continue;
+        }
+        throw err;
+      }
+    }
+    const error = new Error("No authenticator accepted the request");
+    if (lastError) error.cause = lastError;
+    throw error;
+  };
+}