@query-farm/vgi-rpc 0.1.0

package/src/errors.ts

@@ -0,0 +1,22 @@
+// © Copyright 2025-2026, Query.Farm LLC - https://query.farm
+// SPDX-License-Identifier: Apache-2.0
+
+/** Error thrown when the server encounters an RPC protocol error. */
+export class RpcError extends Error {
+  constructor(
+    public readonly errorType: string,
+    public readonly errorMessage: string,
+    public readonly remoteTraceback: string,
+  ) {
+    super(`${errorType}: ${errorMessage}`);
+    this.name = "RpcError";
+  }
+}
+
+/** Error thrown when the client sends an unsupported request version. */
+export class VersionError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "VersionError";
+  }
+}

package/src/http/common.ts

@@ -0,0 +1,89 @@
+// © Copyright 2025-2026, Query.Farm LLC - https://query.farm
+// SPDX-License-Identifier: Apache-2.0
+
+import {
+  RecordBatchStreamWriter,
+  RecordBatchReader,
+  RecordBatch,
+  Schema,
+  Struct,
+  makeData,
+} from "apache-arrow";
+
+export const ARROW_CONTENT_TYPE = "application/vnd.apache.arrow.stream";
+
+export class HttpRpcError extends Error {
+  constructor(
+    message: string,
+    public readonly statusCode: number,
+  ) {
+    super(message);
+    this.name = "HttpRpcError";
+  }
+}
+
+/**
+ * Rebuild a batch's data to match the given schema's field types.
+ *
+ * Batches deserialized from IPC streams (e.g., from PyArrow) may use generic
+ * types (Float) instead of specific ones (Float64).  Arrow-JS's
+ * RecordBatchStreamWriter silently drops batches whose child Data types don't
+ * match the writer's schema.  Cloning each child Data with the schema's field
+ * type fixes the type metadata while preserving the underlying buffers.
+ */
+function conformBatchToSchema(
+  batch: RecordBatch,
+  schema: Schema,
+): RecordBatch {
+  if (batch.numRows === 0) return batch;
+  const children = schema.fields.map((f, i) =>
+    batch.data.children[i].clone(f.type),
+  );
+  const structType = new Struct(schema.fields);
+  const data = makeData({
+    type: structType,
+    length: batch.numRows,
+    children,
+    nullCount: batch.data.nullCount,
+    nullBitmap: batch.data.nullBitmap,
+  });
+  return new RecordBatch(schema, data, batch.metadata);
+}
+
+/** Serialize a schema + batches into a complete IPC stream as Uint8Array. */
+export function serializeIpcStream(
+  schema: Schema,
+  batches: RecordBatch[],
+): Uint8Array {
+  const writer = new RecordBatchStreamWriter();
+  writer.reset(undefined, schema);
+  for (const batch of batches) {
+    writer.write(conformBatchToSchema(batch, schema));
+  }
+  writer.close();
+  return writer.toUint8Array(true);
+}
+
+/** Create a Response with Arrow IPC content type. Casts Uint8Array for TS lib compat. */
+export function arrowResponse(body: Uint8Array, status = 200, extraHeaders?: Headers): Response {
+  const headers = extraHeaders ?? new Headers();
+  headers.set("Content-Type", ARROW_CONTENT_TYPE);
+  return new Response(body as unknown as BodyInit, { status, headers });
+}
+
+/** Read schema + first batch from an IPC stream body. */
+export async function readRequestFromBody(
+  body: Uint8Array,
+): Promise<{ schema: Schema; batch: RecordBatch }> {
+  const reader = await RecordBatchReader.from(body);
+  await reader.open();
+  const schema = reader.schema;
+  if (!schema) {
+    throw new HttpRpcError("Empty IPC stream: no schema", 400);
+  }
+  const batches = reader.readAll();
+  if (batches.length === 0) {
+    throw new HttpRpcError("IPC stream contains no batches", 400);
+  }
+  return { schema, batch: batches[0] };
+}

package/src/http/dispatch.ts

@@ -0,0 +1,340 @@
+// © Copyright 2025-2026, Query.Farm LLC - https://query.farm
+// SPDX-License-Identifier: Apache-2.0
+
+import { Schema, RecordBatch } from "apache-arrow";
+import type { MethodDefinition } from "../types.js";
+import { OutputCollector } from "../types.js";
+import { parseRequest } from "../wire/request.js";
+import {
+  buildResultBatch,
+  buildErrorBatch,
+  buildEmptyBatch,
+} from "../wire/response.js";
+import { buildDescribeBatch, DESCRIBE_SCHEMA } from "../dispatch/describe.js";
+import { STATE_KEY } from "../constants.js";
+import { serializeSchema } from "../util/schema.js";
+import {
+  HttpRpcError,
+  serializeIpcStream,
+  readRequestFromBody,
+  arrowResponse,
+} from "./common.js";
+import { packStateToken, unpackStateToken } from "./token.js";
+import type { StateSerializer } from "./types.js";
+
+const EMPTY_SCHEMA = new Schema([]);
+
+export interface DispatchContext {
+  signingKey: Uint8Array;
+  tokenTtl: number;
+  serverId: string;
+  maxStreamResponseBytes?: number;
+  stateSerializer: StateSerializer;
+}
+
+/** Dispatch a __describe__ request. */
+export function httpDispatchDescribe(
+  protocolName: string,
+  methods: Map<string, MethodDefinition>,
+  serverId: string,
+): Response {
+  const { batch } = buildDescribeBatch(protocolName, methods, serverId);
+  const body = serializeIpcStream(DESCRIBE_SCHEMA, [batch]);
+  return arrowResponse(body);
+}
+
+/** Dispatch a unary HTTP request. */
+export async function httpDispatchUnary(
+  method: MethodDefinition,
+  body: Uint8Array,
+  ctx: DispatchContext,
+): Promise<Response> {
+  const schema = method.resultSchema;
+  const { schema: reqSchema, batch: reqBatch } = await readRequestFromBody(body);
+  const parsed = parseRequest(reqSchema, reqBatch);
+
+  if (parsed.methodName !== method.name) {
+    throw new HttpRpcError(
+      `Method name in request '${parsed.methodName}' does not match URL '${method.name}'`,
+      400,
+    );
+  }
+
+  const out = new OutputCollector(schema, true, ctx.serverId, parsed.requestId);
+
+  try {
+    const result = await method.handler!(parsed.params, out);
+    const resultBatch = buildResultBatch(schema, result, ctx.serverId, parsed.requestId);
+    const batches = [...out.batches.map((b) => b.batch), resultBatch];
+    return arrowResponse(serializeIpcStream(schema, batches));
+  } catch (error: any) {
+    const errBatch = buildErrorBatch(schema, error, ctx.serverId, parsed.requestId);
+    return arrowResponse(serializeIpcStream(schema, [errBatch]), 500);
+  }
+}
+
+/** Dispatch a stream init HTTP request (producer or exchange). */
+export async function httpDispatchStreamInit(
+  method: MethodDefinition,
+  body: Uint8Array,
+  ctx: DispatchContext,
+): Promise<Response> {
+  const isProducer =
+    !method.inputSchema || method.inputSchema.fields.length === 0;
+  const outputSchema = method.outputSchema!;
+  const inputSchema = method.inputSchema ?? EMPTY_SCHEMA;
+
+  const { schema: reqSchema, batch: reqBatch } = await readRequestFromBody(body);
+  const parsed = parseRequest(reqSchema, reqBatch);
+
+  if (parsed.methodName !== method.name) {
+    throw new HttpRpcError(
+      `Method name in request '${parsed.methodName}' does not match URL '${method.name}'`,
+      400,
+    );
+  }
+
+  // Init state
+  let state: any;
+  try {
+    if (isProducer) {
+      state = await method.producerInit!(parsed.params);
+    } else {
+      state = await method.exchangeInit!(parsed.params);
+    }
+  } catch (error: any) {
+    const errSchema = method.headerSchema ?? EMPTY_SCHEMA;
+    const errBatch = buildErrorBatch(errSchema, error, ctx.serverId, parsed.requestId);
+    return arrowResponse(serializeIpcStream(errSchema, [errBatch]), 500);
+  }
+
+  // Support dynamic output schemas (same as pipe transport)
+  const resolvedOutputSchema = state?.__outputSchema ?? outputSchema;
+  const effectiveProducer = state?.__isProducer ?? isProducer;
+
+  // Build header IPC stream if method has a header schema
+  let headerBytes: Uint8Array | null = null;
+  if (method.headerSchema && method.headerInit) {
+    try {
+      const headerOut = new OutputCollector(
+        method.headerSchema,
+        true,
+        ctx.serverId,
+        parsed.requestId,
+      );
+      const headerValues = method.headerInit(parsed.params, state, headerOut);
+      const headerBatch = buildResultBatch(
+        method.headerSchema,
+        headerValues,
+        ctx.serverId,
+        parsed.requestId,
+      );
+      const headerBatches = [
+        ...headerOut.batches.map((b) => b.batch),
+        headerBatch,
+      ];
+      headerBytes = serializeIpcStream(method.headerSchema, headerBatches);
+    } catch (error: any) {
+      const errBatch = buildErrorBatch(
+        method.headerSchema,
+        error,
+        ctx.serverId,
+        parsed.requestId,
+      );
+      return arrowResponse(serializeIpcStream(method.headerSchema, [errBatch]), 500);
+    }
+  }
+
+  if (effectiveProducer) {
+    return produceStreamResponse(
+      method,
+      state,
+      resolvedOutputSchema,
+      inputSchema,
+      ctx,
+      parsed.requestId,
+      headerBytes,
+    );
+  } else {
+    // Exchange: serialize state into signed token, return zero-row batch with token
+    const stateBytes = ctx.stateSerializer.serialize(state);
+    const schemaBytes = serializeSchema(resolvedOutputSchema);
+    const inputSchemaBytes = serializeSchema(inputSchema);
+    const token = packStateToken(
+      stateBytes,
+      schemaBytes,
+      inputSchemaBytes,
+      ctx.signingKey,
+    );
+
+    const tokenMeta = new Map<string, string>();
+    tokenMeta.set(STATE_KEY, token);
+    const tokenBatch = buildEmptyBatch(resolvedOutputSchema, tokenMeta);
+    const tokenStreamBytes = serializeIpcStream(resolvedOutputSchema, [tokenBatch]);
+
+    let responseBody: Uint8Array;
+    if (headerBytes) {
+      responseBody = concatBytes(headerBytes, tokenStreamBytes);
+    } else {
+      responseBody = tokenStreamBytes;
+    }
+
+    return arrowResponse(responseBody);
+  }
+}
+
+/** Dispatch a stream exchange HTTP request (producer continuation or exchange round). */
+export async function httpDispatchStreamExchange(
+  method: MethodDefinition,
+  body: Uint8Array,
+  ctx: DispatchContext,
+): Promise<Response> {
+  const isProducer =
+    !method.inputSchema || method.inputSchema.fields.length === 0;
+
+  const { batch: reqBatch } = await readRequestFromBody(body);
+
+  // Get state token from batch metadata
+  const tokenBase64 = reqBatch.metadata?.get(STATE_KEY);
+  if (!tokenBase64) {
+    throw new HttpRpcError("Missing state token in exchange request", 400);
+  }
+
+  let unpacked;
+  try {
+    unpacked = unpackStateToken(tokenBase64, ctx.signingKey, ctx.tokenTtl);
+  } catch (error: any) {
+    throw new HttpRpcError(`Invalid state token: ${error.message}`, 400);
+  }
+
+  const state = ctx.stateSerializer.deserialize(unpacked.stateBytes);
+
+  // Support dynamic output schemas (same as pipe transport)
+  const outputSchema = state?.__outputSchema ?? method.outputSchema!;
+  const inputSchema = method.inputSchema ?? EMPTY_SCHEMA;
+  const effectiveProducer = state?.__isProducer ?? isProducer;
+
+  if (effectiveProducer) {
+    return produceStreamResponse(
+      method,
+      state,
+      outputSchema,
+      inputSchema,
+      ctx,
+      null,
+      null,
+    );
+  } else {
+    const out = new OutputCollector(outputSchema, false, ctx.serverId, null);
+
+    try {
+      await method.exchangeFn!(state, reqBatch, out);
+    } catch (error: any) {
+      const errBatch = buildErrorBatch(outputSchema, error, ctx.serverId, null);
+      return arrowResponse(serializeIpcStream(outputSchema, [errBatch]), 500);
+    }
+
+    // Repack updated state into new token
+    const stateBytes = ctx.stateSerializer.serialize(state);
+    const schemaBytes = serializeSchema(outputSchema);
+    const inputSchemaBytes = serializeSchema(inputSchema);
+    const token = packStateToken(
+      stateBytes,
+      schemaBytes,
+      inputSchemaBytes,
+      ctx.signingKey,
+    );
+
+    // Merge token into the data batch's metadata (matching Python behavior).
+    // The Python client expects the token on the data batch itself, not a
+    // separate zero-row batch.
+    const batches: RecordBatch[] = [];
+    for (const emitted of out.batches) {
+      const batch = emitted.batch;
+      if (batch.numRows > 0) {
+        // This is the data batch — merge token into its metadata
+        const mergedMeta = new Map<string, string>(batch.metadata ?? []);
+        mergedMeta.set(STATE_KEY, token);
+        batches.push(new RecordBatch(batch.schema, batch.data, mergedMeta));
+      } else {
+        batches.push(batch);
+      }
+    }
+
+    return arrowResponse(serializeIpcStream(outputSchema, batches));
+  }
+}
+
+/** Run the producer loop and build the response. */
+async function produceStreamResponse(
+  method: MethodDefinition,
+  state: any,
+  outputSchema: Schema,
+  inputSchema: Schema,
+  ctx: DispatchContext,
+  requestId: string | null,
+  headerBytes: Uint8Array | null,
+): Promise<Response> {
+  const allBatches: RecordBatch[] = [];
+  const maxBytes = ctx.maxStreamResponseBytes;
+  let estimatedBytes = 0;
+
+  while (true) {
+    const out = new OutputCollector(outputSchema, true, ctx.serverId, requestId);
+
+    try {
+      await method.producerFn!(state, out);
+    } catch (error: any) {
+      allBatches.push(buildErrorBatch(outputSchema, error, ctx.serverId, requestId));
+      break;
+    }
+
+    for (const emitted of out.batches) {
+      allBatches.push(emitted.batch);
+      if (maxBytes != null) {
+        estimatedBytes += emitted.batch.data.byteLength;
+      }
+    }
+
+    if (out.finished) {
+      break;
+    }
+
+    // Check byte budget — if exceeded, emit continuation token
+    if (maxBytes != null && estimatedBytes >= maxBytes) {
+      const stateBytes = ctx.stateSerializer.serialize(state);
+      const schemaBytes = serializeSchema(outputSchema);
+      const inputSchemaBytes = serializeSchema(inputSchema);
+      const token = packStateToken(
+        stateBytes,
+        schemaBytes,
+        inputSchemaBytes,
+        ctx.signingKey,
+      );
+      const tokenMeta = new Map<string, string>();
+      tokenMeta.set(STATE_KEY, token);
+      allBatches.push(buildEmptyBatch(outputSchema, tokenMeta));
+      break;
+    }
+  }
+
+  const dataBytes = serializeIpcStream(outputSchema, allBatches);
+  let responseBody: Uint8Array;
+  if (headerBytes) {
+    responseBody = concatBytes(headerBytes, dataBytes);
+  } else {
+    responseBody = dataBytes;
+  }
+  return arrowResponse(responseBody);
+}
+
+function concatBytes(...arrays: Uint8Array[]): Uint8Array {
+  const totalLen = arrays.reduce((sum, a) => sum + a.length, 0);
+  const result = new Uint8Array(totalLen);
+  let offset = 0;
+  for (const arr of arrays) {
+    result.set(arr, offset);
+    offset += arr.length;
+  }
+  return result;
+}

package/src/http/handler.ts

@@ -0,0 +1,247 @@
+// © Copyright 2025-2026, Query.Farm LLC - https://query.farm
+// SPDX-License-Identifier: Apache-2.0
+
+import { Schema } from "apache-arrow";
+import { randomBytes } from "node:crypto";
+import type { Protocol } from "../protocol.js";
+import { MethodType } from "../types.js";
+import { DESCRIBE_METHOD_NAME } from "../constants.js";
+import { buildErrorBatch } from "../wire/response.js";
+import { jsonStateSerializer, type HttpHandlerOptions } from "./types.js";
+import {
+  ARROW_CONTENT_TYPE,
+  HttpRpcError,
+  serializeIpcStream,
+  arrowResponse,
+} from "./common.js";
+import {
+  httpDispatchDescribe,
+  httpDispatchUnary,
+  httpDispatchStreamInit,
+  httpDispatchStreamExchange,
+} from "./dispatch.js";
+import { zstdCompress, zstdDecompress } from "../util/zstd.js";
+
+const EMPTY_SCHEMA = new Schema([]);
+
+/**
+ * Create a fetch-compatible HTTP handler for a vgi-rpc Protocol.
+ *
+ * Compatible with Bun.serve(), Deno.serve(), Cloudflare Workers, and any
+ * Web API runtime that uses the standard Request/Response types.
+ *
+ * @example
+ * ```typescript
+ * const handler = createHttpHandler(protocol);
+ * Bun.serve({ port: 8080, fetch: handler });
+ * ```
+ */
+export function createHttpHandler(
+  protocol: Protocol,
+  options?: HttpHandlerOptions,
+): (request: Request) => Response | Promise<Response> {
+  const prefix = (options?.prefix ?? "/vgi").replace(/\/+$/, "");
+  const signingKey = options?.signingKey ?? randomBytes(32);
+  const tokenTtl = options?.tokenTtl ?? 3600;
+  const corsOrigins = options?.corsOrigins;
+  const maxRequestBytes = options?.maxRequestBytes;
+  const maxStreamResponseBytes = options?.maxStreamResponseBytes;
+  const serverId =
+    options?.serverId ?? crypto.randomUUID().replace(/-/g, "").slice(0, 12);
+
+  const methods = protocol.getMethods();
+
+  const compressionLevel = options?.compressionLevel;
+  const stateSerializer = options?.stateSerializer ?? jsonStateSerializer;
+
+  const ctx = {
+    signingKey,
+    tokenTtl,
+    serverId,
+    maxStreamResponseBytes,
+    stateSerializer,
+  };
+
+  function addCorsHeaders(headers: Headers): void {
+    if (corsOrigins) {
+      headers.set("Access-Control-Allow-Origin", corsOrigins);
+      headers.set("Access-Control-Allow-Methods", "POST, OPTIONS");
+      headers.set("Access-Control-Allow-Headers", "Content-Type");
+    }
+  }
+
+  async function compressIfAccepted(
+    response: Response,
+    clientAcceptsZstd: boolean,
+  ): Promise<Response> {
+    if (compressionLevel == null || !clientAcceptsZstd) return response;
+    const responseBody = new Uint8Array(await response.arrayBuffer());
+    const compressed = zstdCompress(responseBody, compressionLevel);
+    const headers = new Headers(response.headers);
+    headers.set("Content-Encoding", "zstd");
+    return new Response(compressed as unknown as BodyInit, {
+      status: response.status,
+      headers,
+    });
+  }
+
+  function makeErrorResponse(
+    error: Error,
+    statusCode: number,
+    schema: Schema = EMPTY_SCHEMA,
+  ): Response {
+    const errBatch = buildErrorBatch(schema, error, serverId, null);
+    const body = serializeIpcStream(schema, [errBatch]);
+    const resp = arrowResponse(body, statusCode);
+    addCorsHeaders(resp.headers);
+    return resp;
+  }
+
+  return async function handler(request: Request): Promise<Response> {
+    const url = new URL(request.url);
+    const path = url.pathname;
+
+    // CORS preflight
+    if (request.method === "OPTIONS") {
+      if (path === `${prefix}/__capabilities__`) {
+        const headers = new Headers();
+        addCorsHeaders(headers);
+        if (maxRequestBytes != null) {
+          headers.set("VGI-Max-Request-Bytes", String(maxRequestBytes));
+        }
+        return new Response(null, { status: 204, headers });
+      }
+
+      if (corsOrigins) {
+        const headers = new Headers();
+        addCorsHeaders(headers);
+        return new Response(null, { status: 204, headers });
+      }
+
+      return new Response(null, { status: 405 });
+    }
+
+    if (request.method !== "POST") {
+      return new Response("Method Not Allowed", { status: 405 });
+    }
+
+    // Validate Content-Type
+    const contentType = request.headers.get("Content-Type");
+    if (!contentType || !contentType.includes(ARROW_CONTENT_TYPE)) {
+      return new Response(
+        `Unsupported Media Type: expected ${ARROW_CONTENT_TYPE}`,
+        { status: 415 },
+      );
+    }
+
+    // Check request body size
+    if (maxRequestBytes != null) {
+      const contentLength = request.headers.get("Content-Length");
+      if (contentLength && parseInt(contentLength) > maxRequestBytes) {
+        return new Response("Request body too large", { status: 413 });
+      }
+    }
+
+    const clientAcceptsZstd = (
+      request.headers.get("Accept-Encoding") ?? ""
+    ).includes("zstd");
+
+    // Read body, decompressing if needed
+    let body = new Uint8Array(await request.arrayBuffer());
+    const contentEncoding = request.headers.get("Content-Encoding");
+    if (contentEncoding === "zstd") {
+      body = zstdDecompress(body);
+    }
+
+    // Route: {prefix}/__describe__
+    if (path === `${prefix}/${DESCRIBE_METHOD_NAME}`) {
+      try {
+        const response = httpDispatchDescribe(protocol.name, methods, serverId);
+        addCorsHeaders(response.headers);
+        return compressIfAccepted(response, clientAcceptsZstd);
+      } catch (error: any) {
+        return compressIfAccepted(
+          makeErrorResponse(error, 500),
+          clientAcceptsZstd,
+        );
+      }
+    }
+
+    // Parse method name and sub-path from URL
+    if (!path.startsWith(prefix + "/")) {
+      return new Response("Not Found", { status: 404 });
+    }
+
+    const subPath = path.slice(prefix.length + 1);
+    let methodName: string;
+    let action: "call" | "init" | "exchange";
+
+    if (subPath.endsWith("/init")) {
+      methodName = subPath.slice(0, -5);
+      action = "init";
+    } else if (subPath.endsWith("/exchange")) {
+      methodName = subPath.slice(0, -9);
+      action = "exchange";
+    } else {
+      methodName = subPath;
+      action = "call";
+    }
+
+    // Look up method
+    const method = methods.get(methodName);
+    if (!method) {
+      const available = [...methods.keys()].sort();
+      const err = new Error(
+        `Unknown method: '${methodName}'. Available methods: [${available.join(", ")}]`,
+      );
+      return compressIfAccepted(
+        makeErrorResponse(err, 404),
+        clientAcceptsZstd,
+      );
+    }
+
+    try {
+      let response: Response;
+
+      if (action === "call") {
+        if (method.type !== MethodType.UNARY) {
+          throw new HttpRpcError(
+            `Method '${methodName}' is a stream method. Use /init and /exchange endpoints.`,
+            400,
+          );
+        }
+        response = await httpDispatchUnary(method, body, ctx);
+      } else if (action === "init") {
+        if (method.type !== MethodType.STREAM) {
+          throw new HttpRpcError(
+            `Method '${methodName}' is a unary method. Use POST ${prefix}/${methodName} instead.`,
+            400,
+          );
+        }
+        response = await httpDispatchStreamInit(method, body, ctx);
+      } else {
+        if (method.type !== MethodType.STREAM) {
+          throw new HttpRpcError(
+            `Method '${methodName}' is a unary method. Use POST ${prefix}/${methodName} instead.`,
+            400,
+          );
+        }
+        response = await httpDispatchStreamExchange(method, body, ctx);
+      }
+
+      addCorsHeaders(response.headers);
+      return compressIfAccepted(response, clientAcceptsZstd);
+    } catch (error: any) {
+      if (error instanceof HttpRpcError) {
+        return compressIfAccepted(
+          makeErrorResponse(error, error.statusCode),
+          clientAcceptsZstd,
+        );
+      }
+      return compressIfAccepted(
+        makeErrorResponse(error, 500),
+        clientAcceptsZstd,
+      );
+    }
+  };
+}

package/src/http/index.ts

@@ -0,0 +1,6 @@
+// © Copyright 2025-2026, Query.Farm LLC - https://query.farm
+// SPDX-License-Identifier: Apache-2.0
+
+export { createHttpHandler } from "./handler.js";
+export type { HttpHandlerOptions, StateSerializer } from "./types.js";
+export { ARROW_CONTENT_TYPE } from "./common.js";