@query-farm/vgi-rpc 0.3.4 → 0.6.0

package/src/index.ts

@@ -1,6 +1,7 @@
 // © Copyright 2025-2026, Query.Farm LLC - https://query.farm
 // SPDX-License-Identifier: Apache-2.0
 
+export { AuthContext } from "./auth.js";
 export * from "./client/index.js";
 export {
   DESCRIBE_METHOD_NAME,
@@ -18,14 +19,40 @@ export {
   STATE_KEY,
 } from "./constants.js";
 export { RpcError, VersionError } from "./errors.js";
+export {
+  type ExternalLocationConfig,
+  type ExternalStorage,
+  httpsOnlyValidator,
+  isExternalLocationBatch,
+  makeExternalLocationBatch,
+  maybeExternalizeBatch,
+  resolveExternalLocation,
+} from "./external.js";
 export {
   ARROW_CONTENT_TYPE,
+  type AuthenticateFn,
+  type BearerValidateFn,
+  bearerAuthenticate,
+  bearerAuthenticateStatic,
+  type CertValidateFn,
+  chainAuthenticate,
   createHttpHandler,
   type HttpHandlerOptions,
+  type JwtAuthenticateOptions,
   jsonStateSerializer,
+  jwtAuthenticate,
+  mtlsAuthenticate,
+  mtlsAuthenticateFingerprint,
+  mtlsAuthenticateSubject,
+  mtlsAuthenticateXfcc,
+  type OAuthResourceMetadata,
+  oauthResourceMetadataToJson,
+  parseXfcc,
   type StateSerializer,
   type UnpackedToken,
   unpackStateToken,
+  type XfccElement,
+  type XfccValidateFn,
 } from "./http/index.js";
 export { Protocol } from "./protocol.js";
 export {
@@ -42,9 +69,14 @@ export {
 } from "./schema.js";
 export { VgiRpcServer } from "./server.js";
 export {
+  type CallContext,
+  type CallStatistics,
+  type DispatchHook,
+  type DispatchInfo,
   type ExchangeFn,
   type ExchangeInit,
   type HeaderInit,
+  type HookToken,
   type LogContext,
   type MethodDefinition,
   MethodType,

package/src/otel.ts

@@ -0,0 +1,161 @@
+// © Copyright 2025-2026, Query.Farm LLC - https://query.farm
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * OpenTelemetry instrumentation for vgi-rpc TypeScript servers.
+ *
+ * Implements {@link DispatchHook} to add distributed tracing (spans) and
+ * metrics (request counter, duration histogram) to RPC dispatch.
+ *
+ * Requires `@opentelemetry/api` as a peer dependency.
+ *
+ * @example
+ * ```typescript
+ * import { createOtelHook } from "vgi-rpc/otel";
+ * import { createHttpHandler } from "vgi-rpc";
+ *
+ * const handler = createHttpHandler(protocol, {
+ *   dispatchHook: createOtelHook(),
+ * });
+ * ```
+ */
+
+import {
+  type Attributes,
+  type Counter,
+  type Histogram,
+  type Meter,
+  metrics,
+  type Span,
+  SpanKind,
+  SpanStatusCode,
+  type Tracer,
+  trace,
+} from "@opentelemetry/api";
+import type { CallStatistics, DispatchHook, DispatchInfo, HookToken } from "./types.js";
+
+const INSTRUMENTATION_NAME = "vgi_rpc";
+
+/** Configuration for OpenTelemetry instrumentation. */
+export interface OtelConfig {
+  /** Custom TracerProvider; uses the global provider when omitted. */
+  tracerProvider?: { getTracer(name: string): Tracer };
+  /** Custom MeterProvider; uses the global provider when omitted. */
+  meterProvider?: { getMeter(name: string): Meter };
+  /** Enable span creation. Default: true. */
+  enableTracing?: boolean;
+  /** Enable counter/histogram recording. Default: true. */
+  enableMetrics?: boolean;
+  /** Record exceptions on error spans. Default: true. */
+  recordExceptions?: boolean;
+  /** Service name for the rpc.service attribute. Default: "TypeScriptRpcServer". */
+  serviceName?: string;
+}
+
+interface OtelHookToken {
+  span: Span | null;
+  startTime: number;
+}
+
+/**
+ * Create a {@link DispatchHook} that instruments RPC calls with OpenTelemetry.
+ *
+ * Creates a span for each RPC call with method attributes, and records
+ * request count and duration metrics.
+ */
+export function createOtelHook(config?: OtelConfig): DispatchHook {
+  const enableTracing = config?.enableTracing ?? true;
+  const enableMetrics = config?.enableMetrics ?? true;
+  const recordExceptions = config?.recordExceptions ?? true;
+  const serviceName = config?.serviceName ?? "TypeScriptRpcServer";
+
+  const tracer = (config?.tracerProvider ?? trace).getTracer(INSTRUMENTATION_NAME);
+
+  let requestCounter: Counter | null = null;
+  let durationHistogram: Histogram | null = null;
+
+  if (enableMetrics) {
+    const meter = (config?.meterProvider ?? metrics).getMeter(INSTRUMENTATION_NAME);
+    requestCounter = meter.createCounter("rpc.server.requests", {
+      unit: "{request}",
+      description: "Number of RPC requests handled",
+    });
+    durationHistogram = meter.createHistogram("rpc.server.duration", {
+      unit: "s",
+      description: "Duration of RPC requests",
+    });
+  }
+
+  return {
+    onDispatchStart(info: DispatchInfo): HookToken {
+      const startTime = performance.now();
+
+      if (!enableTracing) {
+        return { span: null, startTime } satisfies OtelHookToken;
+      }
+
+      const spanName = `vgi_rpc/${info.method}`;
+      const attrs: Attributes = {
+        "rpc.system": "vgi_rpc",
+        "rpc.service": serviceName,
+        "rpc.method": info.method,
+        "rpc.vgi_rpc.method_type": info.methodType,
+        "rpc.vgi_rpc.server_id": info.serverId,
+      };
+      if (info.requestId) {
+        attrs["rpc.vgi_rpc.request_id"] = info.requestId;
+      }
+
+      const span = tracer.startSpan(spanName, {
+        kind: SpanKind.SERVER,
+        attributes: attrs,
+      });
+
+      return { span, startTime } satisfies OtelHookToken;
+    },
+
+    onDispatchEnd(token: HookToken, info: DispatchInfo, stats: CallStatistics, error?: Error): void {
+      const t = token as OtelHookToken;
+      const durationS = (performance.now() - t.startTime) / 1000;
+      const status = error ? "error" : "ok";
+
+      // Finalize span
+      if (t.span) {
+        if (stats) {
+          t.span.setAttributes({
+            "rpc.vgi_rpc.input_batches": stats.inputBatches,
+            "rpc.vgi_rpc.output_batches": stats.outputBatches,
+            "rpc.vgi_rpc.input_rows": stats.inputRows,
+            "rpc.vgi_rpc.output_rows": stats.outputRows,
+            "rpc.vgi_rpc.input_bytes": stats.inputBytes,
+            "rpc.vgi_rpc.output_bytes": stats.outputBytes,
+          });
+        }
+
+        if (error) {
+          t.span.setStatus({ code: SpanStatusCode.ERROR, message: error.message });
+          t.span.setAttribute("rpc.vgi_rpc.error_type", error.constructor.name);
+          if (recordExceptions) {
+            t.span.recordException(error);
+          }
+        } else {
+          t.span.setStatus({ code: SpanStatusCode.OK });
+        }
+        t.span.end();
+      }
+
+      // Record metrics
+      if (enableMetrics) {
+        const metricAttrs: Attributes = {
+          "rpc.system": "vgi_rpc",
+          "rpc.service": serviceName,
+          "rpc.method": info.method,
+          "rpc.vgi_rpc.method_type": info.methodType,
+          status,
+        };
+        requestCounter?.add(1, metricAttrs);
+        durationHistogram?.record(durationS, metricAttrs);
+      }
+    },
+  };
+}

package/src/s3.ts

@@ -0,0 +1,94 @@
+// © Copyright 2025-2026, Query.Farm LLC - https://query.farm
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * S3 storage backend for external storage of large Arrow IPC batches.
+ *
+ * Requires `@aws-sdk/client-s3` and `@aws-sdk/s3-request-presigner`
+ * as peer dependencies.
+ *
+ * @example
+ * ```typescript
+ * import { createS3Storage } from "@query-farm/vgi-rpc/s3";
+ *
+ * const storage = createS3Storage({
+ *   bucket: "my-bucket",
+ *   prefix: "vgi-rpc/",
+ * });
+ * const handler = createHttpHandler(protocol, {
+ *   externalLocation: { storage, externalizeThresholdBytes: 1_048_576 },
+ * });
+ * ```
+ */
+
+import type { ExternalStorage } from "./external.js";
+
+/** Configuration for the S3 storage backend. */
+export interface S3StorageConfig {
+  /** S3 bucket name. */
+  bucket: string;
+  /** Key prefix for uploaded objects. Default: "vgi-rpc/". */
+  prefix?: string;
+  /** Lifetime of pre-signed GET URLs in seconds. Default: 3600 (1 hour). */
+  presignExpirySeconds?: number;
+  /** AWS region. If omitted, uses default SDK config. */
+  region?: string;
+  /** Custom S3 endpoint URL (for MinIO, LocalStack, etc.). */
+  endpointUrl?: string;
+  /** Force path-style addressing (required for some S3-compatible services). */
+  forcePathStyle?: boolean;
+}
+
+/**
+ * Create an S3-backed ExternalStorage.
+ *
+ * Lazily imports `@aws-sdk/client-s3` and `@aws-sdk/s3-request-presigner`
+ * on first upload to avoid loading the AWS SDK unless needed.
+ */
+export function createS3Storage(config: S3StorageConfig): ExternalStorage {
+  const bucket = config.bucket;
+  const prefix = config.prefix ?? "vgi-rpc/";
+  const presignExpiry = config.presignExpirySeconds ?? 3600;
+
+  // Lazy-loaded AWS SDK clients
+  let s3Client: any = null;
+
+  async function ensureClient(): Promise<any> {
+    if (s3Client) return s3Client;
+    const { S3Client } = await import("@aws-sdk/client-s3");
+    const clientConfig: Record<string, any> = {};
+    if (config.region) clientConfig.region = config.region;
+    if (config.endpointUrl) {
+      clientConfig.endpoint = config.endpointUrl;
+      clientConfig.forcePathStyle = config.forcePathStyle ?? true;
+    }
+    s3Client = new S3Client(clientConfig);
+    return s3Client;
+  }
+
+  return {
+    async upload(data: Uint8Array, contentEncoding: string): Promise<string> {
+      const client = await ensureClient();
+      const { PutObjectCommand } = await import("@aws-sdk/client-s3");
+      const { getSignedUrl } = await import("@aws-sdk/s3-request-presigner");
+      const { GetObjectCommand } = await import("@aws-sdk/client-s3");
+
+      const key = `${prefix}${crypto.randomUUID()}${contentEncoding === "zstd" ? ".arrow.zst" : ".arrow"}`;
+
+      const putCommand = new PutObjectCommand({
+        Bucket: bucket,
+        Key: key,
+        Body: data,
+        ContentType: "application/vnd.apache.arrow.stream",
+        ...(contentEncoding ? { ContentEncoding: contentEncoding } : {}),
+      });
+
+      await client.send(putCommand);
+
+      // Generate pre-signed GET URL
+      const getCommand = new GetObjectCommand({ Bucket: bucket, Key: key });
+      const url = await getSignedUrl(client, getCommand, { expiresIn: presignExpiry });
+      return url;
+    },
+  };
+}

package/src/server.ts

@@ -7,8 +7,9 @@ import { buildDescribeBatch } from "./dispatch/describe.js";
 import { dispatchStream } from "./dispatch/stream.js";
 import { dispatchUnary } from "./dispatch/unary.js";
 import { RpcError, VersionError } from "./errors.js";
+import type { ExternalLocationConfig } from "./external.js";
 import type { Protocol } from "./protocol.js";
-import { MethodType } from "./types.js";
+import { type CallStatistics, type DispatchHook, type DispatchInfo, MethodType } from "./types.js";
 import { IpcStreamReader } from "./wire/reader.js";
 import { parseRequest } from "./wire/request.js";
 import { buildErrorBatch } from "./wire/response.js";
@@ -25,11 +26,23 @@ export class VgiRpcServer {
   private enableDescribe: boolean;
   private serverId: string;
   private describeBatch: import("@query-farm/apache-arrow").RecordBatch | null = null;
-
-  constructor(protocol: Protocol, options?: { enableDescribe?: boolean; serverId?: string }) {
+  private dispatchHook: DispatchHook | null = null;
+  private externalConfig: ExternalLocationConfig | undefined;
+
+  constructor(
+    protocol: Protocol,
+    options?: {
+      enableDescribe?: boolean;
+      serverId?: string;
+      dispatchHook?: DispatchHook;
+      externalLocation?: ExternalLocationConfig;
+    },
+  ) {
     this.protocol = protocol;
     this.enableDescribe = options?.enableDescribe ?? true;
     this.serverId = options?.serverId ?? crypto.randomUUID().replace(/-/g, "").slice(0, 12);
+    this.dispatchHook = options?.dispatchHook ?? null;
+    this.externalConfig = options?.externalLocation;
 
     if (this.enableDescribe) {
       const { batch } = buildDescribeBatch(protocol.name, protocol.getMethods(), this.serverId);
@@ -129,11 +142,32 @@ export class VgiRpcServer {
       return;
     }
 
-    // Dispatch based on method type
-    if (method.type === MethodType.UNARY) {
-      await dispatchUnary(method, params, writer, this.serverId, requestId);
-    } else {
-      await dispatchStream(method, params, writer, reader, this.serverId, requestId);
+    // Dispatch based on method type, with optional hook
+    const methodType = method.type === MethodType.UNARY ? "unary" : "stream";
+    const info: DispatchInfo = { method: methodName, methodType, serverId: this.serverId, requestId };
+    const stats: CallStatistics = {
+      inputBatches: 0,
+      outputBatches: 0,
+      inputRows: 0,
+      outputRows: 0,
+      inputBytes: 0,
+      outputBytes: 0,
+    };
+
+    const token = this.dispatchHook?.onDispatchStart(info);
+    let dispatchError: Error | undefined;
+
+    try {
+      if (method.type === MethodType.UNARY) {
+        await dispatchUnary(method, params, writer, this.serverId, requestId, this.externalConfig);
+      } else {
+        await dispatchStream(method, params, writer, reader, this.serverId, requestId, this.externalConfig);
+      }
+    } catch (e) {
+      dispatchError = e instanceof Error ? e : new Error(String(e));
+      throw e;
+    } finally {
+      this.dispatchHook?.onDispatchEnd(token, info, stats, dispatchError);
     }
   }
 }

package/src/types.ts

@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 import { RecordBatch, recordBatchFromArrays, type Schema } from "@query-farm/apache-arrow";
+import { AuthContext } from "./auth.js";
 import { buildLogBatch, coerceInt64 } from "./wire/response.js";
 
 export enum MethodType {
@@ -14,6 +15,11 @@ export interface LogContext {
   clientLog(level: string, message: string, extra?: Record<string, string>): void;
 }
 
+/** Extended context with authentication info, available to handlers. */
+export interface CallContext extends LogContext {
+  readonly auth: AuthContext;
+}
+
 /** Handler for unary (request-response) RPC methods. */
 export type UnaryHandler = (
   params: Record<string, any>,
@@ -52,6 +58,40 @@ export interface MethodDefinition {
   paramTypes?: Record<string, string>;
 }
 
+/** Metadata passed to dispatch hooks before and after RPC method execution. */
+export interface DispatchInfo {
+  /** RPC method name. */
+  method: string;
+  /** "unary" or "stream". */
+  methodType: string;
+  /** Server identifier. */
+  serverId: string;
+  /** Client-supplied request identifier, or null. */
+  requestId: string | null;
+}
+
+/** Per-call I/O counters, matching Python's CallStatistics. */
+export interface CallStatistics {
+  inputBatches: number;
+  outputBatches: number;
+  inputRows: number;
+  outputRows: number;
+  inputBytes: number;
+  outputBytes: number;
+}
+
+/** Opaque token returned by onDispatchStart, passed back to onDispatchEnd. */
+export type HookToken = unknown;
+
+/**
+ * Observability hook called around RPC dispatch.
+ * Implementations must be safe for concurrent use (HTTP transport is concurrent).
+ */
+export interface DispatchHook {
+  onDispatchStart(info: DispatchInfo): HookToken;
+  onDispatchEnd(token: HookToken, info: DispatchInfo, stats: CallStatistics, error?: Error): void;
+}
+
 export interface EmittedBatch {
   batch: RecordBatch;
   metadata?: Map<string, string>;
@@ -61,7 +101,7 @@ export interface EmittedBatch {
  * Accumulates output batches during a produce/exchange call.
  * Enforces that exactly one data batch is emitted per call (plus any number of log batches).
  */
-export class OutputCollector implements LogContext {
+export class OutputCollector implements CallContext {
   private _batches: EmittedBatch[] = [];
   private _dataBatchIdx: number | null = null;
   private _finished = false;
@@ -69,12 +109,20 @@ export class OutputCollector implements LogContext {
   private _outputSchema: Schema;
   private _serverId: string;
   private _requestId: string | null;
-
-  constructor(outputSchema: Schema, producerMode = true, serverId = "", requestId: string | null = null) {
+  readonly auth: AuthContext;
+
+  constructor(
+    outputSchema: Schema,
+    producerMode = true,
+    serverId = "",
+    requestId: string | null = null,
+    authContext?: AuthContext,
+  ) {
     this._outputSchema = outputSchema;
     this._producerMode = producerMode;
     this._serverId = serverId;
     this._requestId = requestId;
+    this.auth = authContext ?? AuthContext.anonymous();
   }
 
   get outputSchema(): Schema {

package/src/wire/response.ts

@@ -133,25 +133,39 @@ export function buildLogBatch(
   return buildEmptyBatch(schema, metadata);
 }
 
+/**
+ * Recursively create empty (0-row) Data for any Arrow type,
+ * including complex types (Struct, List, FixedSizeList, Map).
+ */
+function makeEmptyData(type: DataType): Data {
+  if (DataType.isStruct(type)) {
+    const children = type.children.map((f: Field) => makeEmptyData(f.type));
+    return makeData({ type, length: 0, children, nullCount: 0 });
+  }
+  if (DataType.isList(type)) {
+    const childData = makeEmptyData(type.children[0].type);
+    return makeData({ type, length: 0, children: [childData], nullCount: 0, valueOffsets: new Int32Array([0]) } as any);
+  }
+  if (DataType.isFixedSizeList(type)) {
+    const childData = makeEmptyData(type.children[0].type);
+    return makeData({ type, length: 0, child: childData, nullCount: 0 } as any);
+  }
+  if (DataType.isMap(type)) {
+    const entryType = type.children[0]?.type;
+    const entryData = entryType
+      ? makeEmptyData(entryType)
+      : makeData({ type: new Struct([]), length: 0, children: [], nullCount: 0 });
+    return makeData({ type, length: 0, children: [entryData], nullCount: 0, valueOffsets: new Int32Array([0]) } as any);
+  }
+  return makeData({ type, length: 0, nullCount: 0 });
+}
+
 /**
  * Build a 0-row batch from a schema with metadata.
  * Used for error/log batches.
  */
 export function buildEmptyBatch(schema: Schema, metadata?: Map<string, string>): RecordBatch {
-  const children = schema.fields.map((f: Field) => {
-    return makeData({ type: f.type, length: 0, nullCount: 0 });
-  });
-
-  if (schema.fields.length === 0) {
-    const structType = new Struct(schema.fields);
-    const data = makeData({
-      type: structType,
-      length: 0,
-      children: [],
-      nullCount: 0,
-    });
-    return new RecordBatch(schema, data, metadata);
-  }
+  const children = schema.fields.map((f: Field) => makeEmptyData(f.type));
 
   const structType = new Struct(schema.fields);
   const data = makeData({