@query-farm/vgi-rpc 0.3.1 → 0.3.3

package/src/client/stream.ts

@@ -1,23 +1,11 @@
 // © Copyright 2025-2026, Query.Farm LLC - https://query.farm
 // SPDX-License-Identifier: Apache-2.0
 
-import {
-  RecordBatch,
-  Schema,
-  Field,
-  makeData,
-  Struct,
-  vectorFromArray,
-} from "@query-farm/apache-arrow";
+import { Field, makeData, RecordBatch, Schema, Struct, vectorFromArray } from "@query-farm/apache-arrow";
 import { STATE_KEY } from "../constants.js";
 import { RpcError } from "../errors.js";
 import { ARROW_CONTENT_TYPE, serializeIpcStream } from "../http/common.js";
-import {
-  inferArrowType,
-  dispatchLogOrError,
-  extractBatchRows,
-  readResponseBatches,
-} from "./ipc.js";
+import { dispatchLogOrError, extractBatchRows, inferArrowType, readResponseBatches } from "./ipc.js";
 import type { LogMessage, StreamSession } from "./types.js";
 
 type CompressFn = (data: Uint8Array, level: number) => Uint8Array;
@@ -103,11 +91,7 @@ export class HttpStreamSession implements StreamSession {
    */
   async exchange(input: Record<string, any>[]): Promise<Record<string, any>[]> {
     if (this._stateToken === null) {
-      throw new RpcError(
-        "ProtocolError",
-        "Stream has finished \u2014 no state token available",
-        "",
-      );
+      throw new RpcError("ProtocolError", "Stream has finished \u2014 no state token available", "");
     }
 
     // We need to determine the input schema from the data.
@@ -122,11 +106,7 @@ export class HttpStreamSession implements StreamSession {
       const emptyBatch = this._buildEmptyBatch(zeroSchema);
       const metadata = new Map<string, string>();
       metadata.set(STATE_KEY, this._stateToken);
-      const batchWithMeta = new RecordBatch(
-        zeroSchema,
-        emptyBatch.data,
-        metadata,
-      );
+      const batchWithMeta = new RecordBatch(zeroSchema, emptyBatch.data, metadata);
       return this._doExchange(zeroSchema, [batchWithMeta]);
     }
 
@@ -134,9 +114,12 @@ export class HttpStreamSession implements StreamSession {
     const keys = Object.keys(input[0]);
     const fields = keys.map((key) => {
       // Find first non-null value to infer type
-      let sample: any = undefined;
+      let sample: any;
       for (const row of input) {
-        if (row[key] != null) { sample = row[key]; break; }
+        if (row[key] != null) {
+          sample = row[key];
+          break;
+        }
       }
       const arrowType = inferArrowType(sample);
       const nullable = input.some((row) => row[key] == null);
@@ -164,19 +147,13 @@ export class HttpStreamSession implements StreamSession {
     return this._doExchange(inputSchema, [batch]);
   }
 
-  private async _doExchange(
-    schema: Schema,
-    batches: RecordBatch[],
-  ): Promise<Record<string, any>[]> {
+  private async _doExchange(schema: Schema, batches: RecordBatch[]): Promise<Record<string, any>[]> {
     const body = serializeIpcStream(schema, batches);
-    const resp = await fetch(
-      `${this._baseUrl}${this._prefix}/${this._method}/exchange`,
-      {
-        method: "POST",
-        headers: this._buildHeaders(),
-        body: this._prepareBody(body) as unknown as BodyInit,
-      },
-    );
+    const resp = await fetch(`${this._baseUrl}${this._prefix}/${this._method}/exchange`, {
+      method: "POST",
+      headers: this._buildHeaders(),
+      body: this._prepareBody(body) as unknown as BodyInit,
+    });
 
     const responseBody = await this._readResponse(resp);
     const { batches: responseBatches } = await readResponseBatches(responseBody);
@@ -279,14 +256,11 @@ export class HttpStreamSession implements StreamSession {
     const batch = new RecordBatch(emptySchema, data, metadata);
     const body = serializeIpcStream(emptySchema, [batch]);
 
-    const resp = await fetch(
-      `${this._baseUrl}${this._prefix}/${this._method}/exchange`,
-      {
-        method: "POST",
-        headers: this._buildHeaders(),
-        body: this._prepareBody(body) as unknown as BodyInit,
-      },
-    );
+    const resp = await fetch(`${this._baseUrl}${this._prefix}/${this._method}/exchange`, {
+      method: "POST",
+      headers: this._buildHeaders(),
+      body: this._prepareBody(body) as unknown as BodyInit,
+    });
 
     return this._readResponse(resp);
   }

package/src/dispatch/describe.ts

@@ -2,25 +2,25 @@
 // SPDX-License-Identifier: Apache-2.0
 
 import {
-  Schema,
+  Binary,
+  Bool,
   Field,
+  makeData,
   RecordBatch,
+  Schema,
+  Struct,
   Utf8,
-  Bool,
-  Binary,
   vectorFromArray,
-  makeData,
-  Struct,
 } from "@query-farm/apache-arrow";
-import type { MethodDefinition } from "../types.js";
 import {
+  DESCRIBE_VERSION,
+  DESCRIBE_VERSION_KEY,
   PROTOCOL_NAME_KEY,
-  REQUEST_VERSION_KEY,
   REQUEST_VERSION,
-  DESCRIBE_VERSION_KEY,
-  DESCRIBE_VERSION,
+  REQUEST_VERSION_KEY,
   SERVER_ID_KEY,
 } from "../constants.js";
+import type { MethodDefinition } from "../types.js";
 import { serializeSchema } from "../util/schema.js";
 
 /**
@@ -48,9 +48,7 @@ export function buildDescribeBatch(
   serverId: string,
 ): { batch: RecordBatch; metadata: Map<string, string> } {
   // Sort methods by name for consistent ordering
-  const sortedEntries = [...methods.entries()].sort(([a], [b]) =>
-    a.localeCompare(b),
-  );
+  const sortedEntries = [...methods.entries()].sort(([a], [b]) => a.localeCompare(b));
 
   const names: (string | null)[] = [];
   const methodTypes: (string | null)[] = [];
@@ -69,8 +67,7 @@ export function buildDescribeBatch(
     docs.push(method.doc ?? null);
 
     // Unary methods with non-empty result schema have a return value
-    const hasReturn =
-      method.type === "unary" && method.resultSchema.fields.length > 0;
+    const hasReturn = method.type === "unary" && method.resultSchema.fields.length > 0;
     hasReturns.push(hasReturn);
 
     paramsSchemas.push(serializeSchema(method.paramsSchema));
@@ -87,26 +84,17 @@ export function buildDescribeBatch(
     if (method.defaults && Object.keys(method.defaults).length > 0) {
       const safe: Record<string, any> = {};
       for (const [k, v] of Object.entries(method.defaults)) {
-        if (
-          v === null ||
-          typeof v === "string" ||
-          typeof v === "number" ||
-          typeof v === "boolean"
-        ) {
+        if (v === null || typeof v === "string" || typeof v === "number" || typeof v === "boolean") {
           safe[k] = v;
         }
       }
-      paramDefaultsJsons.push(
-        Object.keys(safe).length > 0 ? JSON.stringify(safe) : null,
-      );
+      paramDefaultsJsons.push(Object.keys(safe).length > 0 ? JSON.stringify(safe) : null);
     } else {
       paramDefaultsJsons.push(null);
     }
 
     hasHeaders.push(!!method.headerSchema);
-    headerSchemas.push(
-      method.headerSchema ? serializeSchema(method.headerSchema) : null,
-    );
+    headerSchemas.push(method.headerSchema ? serializeSchema(method.headerSchema) : null);
   }
 
   // Build the batch using vectorFromArray for each column

package/src/dispatch/stream.ts

@@ -4,9 +4,10 @@
 import { Schema } from "@query-farm/apache-arrow";
 import type { MethodDefinition } from "../types.js";
 import { OutputCollector } from "../types.js";
-import type { IpcStreamWriter } from "../wire/writer.js";
+import { conformBatchToSchema } from "../util/conform.js";
 import type { IpcStreamReader } from "../wire/reader.js";
-import { buildResultBatch, buildErrorBatch } from "../wire/response.js";
+import { buildErrorBatch, buildResultBatch } from "../wire/response.js";
+import type { IpcStreamWriter } from "../wire/writer.js";
 
 const EMPTY_SCHEMA = new Schema([]);
 
@@ -70,16 +71,8 @@ export async function dispatchStream(
     try {
       const headerOut = new OutputCollector(method.headerSchema, true, serverId, requestId);
       const headerValues = method.headerInit(params, state, headerOut);
-      const headerBatch = buildResultBatch(
-        method.headerSchema,
-        headerValues,
-        serverId,
-        requestId,
-      );
-      const headerBatches = [
-        ...headerOut.batches.map((b) => b.batch),
-        headerBatch,
-      ];
+      const headerBatch = buildResultBatch(method.headerSchema, headerValues, serverId, requestId);
+      const headerBatches = [...headerOut.batches.map((b) => b.batch), headerBatch];
       writer.writeStream(method.headerSchema, headerBatches);
     } catch (error: any) {
       const errBatch = buildErrorBatch(method.headerSchema, error, serverId, requestId);
@@ -96,12 +89,7 @@ export async function dispatchStream(
   // Open the input IPC stream (ticks or data from client)
   const inputSchema = await reader.openNextStream();
   if (!inputSchema) {
-    const errBatch = buildErrorBatch(
-      outputSchema,
-      new Error("Expected input stream but got EOF"),
-      serverId,
-      requestId,
-    );
+    const errBatch = buildErrorBatch(outputSchema, new Error("Expected input stream but got EOF"), serverId, requestId);
     writer.writeStream(outputSchema, [errBatch]);
     return;
   }
@@ -111,11 +99,25 @@ export async function dispatchStream(
   // same stream. We use IncrementalStream which writes bytes synchronously.
   const stream = writer.openStream(outputSchema);
 
+  // Expected input schema for casting compatible types (e.g., decimal→double)
+  const expectedInputSchema = method.inputSchema;
+
   try {
     while (true) {
-      const inputBatch = await reader.readNextBatch();
+      let inputBatch = await reader.readNextBatch();
       if (!inputBatch) break;
 
+      // Cast compatible input types when schema doesn't match exactly
+      if (expectedInputSchema && !isProducer && inputBatch.schema !== expectedInputSchema) {
+        try {
+          inputBatch = conformBatchToSchema(inputBatch, expectedInputSchema);
+        } catch {
+          throw new TypeError(
+            `Input schema mismatch: expected ${expectedInputSchema}, got ${inputBatch.schema}`,
+          );
+        }
+      }
+
       const out = new OutputCollector(outputSchema, effectiveProducer, serverId, requestId);
 
       if (isProducer) {

package/src/dispatch/unary.ts

@@ -1,11 +1,10 @@
 // © Copyright 2025-2026, Query.Farm LLC - https://query.farm
 // SPDX-License-Identifier: Apache-2.0
 
-import { Schema } from "@query-farm/apache-arrow";
 import type { MethodDefinition } from "../types.js";
 import { OutputCollector } from "../types.js";
+import { buildErrorBatch, buildResultBatch } from "../wire/response.js";
 import type { IpcStreamWriter } from "../wire/writer.js";
-import { buildResultBatch, buildErrorBatch } from "../wire/response.js";
 
 /**
  * Dispatch a unary RPC call.

package/src/http/common.ts

@@ -2,13 +2,12 @@
 // SPDX-License-Identifier: Apache-2.0
 
 import {
-  RecordBatchStreamWriter,
+  type RecordBatch,
   RecordBatchReader,
-  RecordBatch,
-  Schema,
-  Struct,
-  makeData,
+  RecordBatchStreamWriter,
+  type Schema,
 } from "@query-farm/apache-arrow";
+import { conformBatchToSchema } from "../util/conform.js";
 
 export const ARROW_CONTENT_TYPE = "application/vnd.apache.arrow.stream";
 
@@ -22,39 +21,8 @@ export class HttpRpcError extends Error {
   }
 }
 
-/**
- * Rebuild a batch's data to match the given schema's field types.
- *
- * Batches deserialized from IPC streams (e.g., from PyArrow) may use generic
- * types (Float) instead of specific ones (Float64).  Arrow-JS's
- * RecordBatchStreamWriter silently drops batches whose child Data types don't
- * match the writer's schema.  Cloning each child Data with the schema's field
- * type fixes the type metadata while preserving the underlying buffers.
- */
-function conformBatchToSchema(
-  batch: RecordBatch,
-  schema: Schema,
-): RecordBatch {
-  if (batch.numRows === 0) return batch;
-  const children = schema.fields.map((f, i) =>
-    batch.data.children[i].clone(f.type),
-  );
-  const structType = new Struct(schema.fields);
-  const data = makeData({
-    type: structType,
-    length: batch.numRows,
-    children,
-    nullCount: batch.data.nullCount,
-    nullBitmap: batch.data.nullBitmap,
-  });
-  return new RecordBatch(schema, data, batch.metadata);
-}
-
 /** Serialize a schema + batches into a complete IPC stream as Uint8Array. */
-export function serializeIpcStream(
-  schema: Schema,
-  batches: RecordBatch[],
-): Uint8Array {
+export function serializeIpcStream(schema: Schema, batches: RecordBatch[]): Uint8Array {
   const writer = new RecordBatchStreamWriter();
   writer.reset(undefined, schema);
   for (const batch of batches) {
@@ -72,9 +40,7 @@ export function arrowResponse(body: Uint8Array, status = 200, extraHeaders?: Hea
 }
 
 /** Read schema + first batch from an IPC stream body. */
-export async function readRequestFromBody(
-  body: Uint8Array,
-): Promise<{ schema: Schema; batch: RecordBatch }> {
+export async function readRequestFromBody(body: Uint8Array): Promise<{ schema: Schema; batch: RecordBatch }> {
   const reader = await RecordBatchReader.from(body);
   await reader.open();
   const schema = reader.schema;

package/src/http/dispatch.ts

@@ -1,24 +1,16 @@
 // © Copyright 2025-2026, Query.Farm LLC - https://query.farm
 // SPDX-License-Identifier: Apache-2.0
 
-import { Schema, RecordBatch, RecordBatchReader } from "@query-farm/apache-arrow";
+import { RecordBatch, RecordBatchReader, Schema } from "@query-farm/apache-arrow";
+import { STATE_KEY } from "../constants.js";
+import { buildDescribeBatch, DESCRIBE_SCHEMA } from "../dispatch/describe.js";
 import type { MethodDefinition } from "../types.js";
 import { OutputCollector } from "../types.js";
-import { parseRequest } from "../wire/request.js";
-import {
-  buildResultBatch,
-  buildErrorBatch,
-  buildEmptyBatch,
-} from "../wire/response.js";
-import { buildDescribeBatch, DESCRIBE_SCHEMA } from "../dispatch/describe.js";
-import { STATE_KEY } from "../constants.js";
+import { conformBatchToSchema } from "../util/conform.js";
 import { serializeSchema } from "../util/schema.js";
-import {
-  HttpRpcError,
-  serializeIpcStream,
-  readRequestFromBody,
-  arrowResponse,
-} from "./common.js";
+import { parseRequest } from "../wire/request.js";
+import { buildEmptyBatch, buildErrorBatch, buildResultBatch } from "../wire/response.js";
+import { arrowResponse, HttpRpcError, readRequestFromBody, serializeIpcStream } from "./common.js";
 import { packStateToken, unpackStateToken } from "./token.js";
 import type { StateSerializer } from "./types.js";
 
@@ -60,10 +52,7 @@ export async function httpDispatchUnary(
   const parsed = parseRequest(reqSchema, reqBatch);
 
   if (parsed.methodName !== method.name) {
-    throw new HttpRpcError(
-      `Method name in request '${parsed.methodName}' does not match URL '${method.name}'`,
-      400,
-    );
+    throw new HttpRpcError(`Method name in request '${parsed.methodName}' does not match URL '${method.name}'`, 400);
   }
 
   const out = new OutputCollector(schema, true, ctx.serverId, parsed.requestId);
@@ -93,10 +82,7 @@ export async function httpDispatchStreamInit(
   const parsed = parseRequest(reqSchema, reqBatch);
 
   if (parsed.methodName !== method.name) {
-    throw new HttpRpcError(
-      `Method name in request '${parsed.methodName}' does not match URL '${method.name}'`,
-      400,
-    );
+    throw new HttpRpcError(`Method name in request '${parsed.methodName}' does not match URL '${method.name}'`, 400);
   }
 
   // Init state
@@ -121,31 +107,13 @@ export async function httpDispatchStreamInit(
   let headerBytes: Uint8Array | null = null;
   if (method.headerSchema && method.headerInit) {
     try {
-      const headerOut = new OutputCollector(
-        method.headerSchema,
-        true,
-        ctx.serverId,
-        parsed.requestId,
-      );
+      const headerOut = new OutputCollector(method.headerSchema, true, ctx.serverId, parsed.requestId);
       const headerValues = method.headerInit(parsed.params, state, headerOut);
-      const headerBatch = buildResultBatch(
-        method.headerSchema,
-        headerValues,
-        ctx.serverId,
-        parsed.requestId,
-      );
-      const headerBatches = [
-        ...headerOut.batches.map((b) => b.batch),
-        headerBatch,
-      ];
+      const headerBatch = buildResultBatch(method.headerSchema, headerValues, ctx.serverId, parsed.requestId);
+      const headerBatches = [...headerOut.batches.map((b) => b.batch), headerBatch];
       headerBytes = serializeIpcStream(method.headerSchema, headerBatches);
     } catch (error: any) {
-      const errBatch = buildErrorBatch(
-        method.headerSchema,
-        error,
-        ctx.serverId,
-        parsed.requestId,
-      );
+      const errBatch = buildErrorBatch(method.headerSchema, error, ctx.serverId, parsed.requestId);
       return arrowResponse(serializeIpcStream(method.headerSchema, [errBatch]), 500);
     }
   }
@@ -154,26 +122,13 @@ export async function httpDispatchStreamInit(
     // Producer method — produce data inline in the init response.
     // For exchange-registered methods acting as producers (__isProducer),
     // produceStreamResponse falls back to exchangeFn with tick batches.
-    return produceStreamResponse(
-      method,
-      state,
-      resolvedOutputSchema,
-      inputSchema,
-      ctx,
-      parsed.requestId,
-      headerBytes,
-    );
+    return produceStreamResponse(method, state, resolvedOutputSchema, inputSchema, ctx, parsed.requestId, headerBytes);
   } else {
     // Exchange: serialize state into signed token, return zero-row batch with token
     const stateBytes = ctx.stateSerializer.serialize(state);
     const schemaBytes = serializeSchema(resolvedOutputSchema);
     const inputSchemaBytes = serializeSchema(inputSchema);
-    const token = packStateToken(
-      stateBytes,
-      schemaBytes,
-      inputSchemaBytes,
-      ctx.signingKey,
-    );
+    const token = packStateToken(stateBytes, schemaBytes, inputSchemaBytes, ctx.signingKey);
 
     const tokenMeta = new Map<string, string>();
     tokenMeta.set(STATE_KEY, token);
@@ -207,7 +162,7 @@ export async function httpDispatchStreamExchange(
     throw new HttpRpcError("Missing state token in exchange request", 400);
   }
 
-  let unpacked;
+  let unpacked: import("./token.js").UnpackedToken;
   try {
     unpacked = unpackStateToken(tokenBase64, ctx.signingKey, ctx.tokenTtl);
   } catch (error: any) {
@@ -237,34 +192,37 @@ export async function httpDispatchStreamExchange(
     inputSchema = method.inputSchema ?? EMPTY_SCHEMA;
   }
   const effectiveProducer = state?.__isProducer ?? isProducer;
-  if (process.env.VGI_DISPATCH_DEBUG) console.error(`[httpDispatchStreamExchange] method=${method.name} effectiveProducer=${effectiveProducer} stateKeys=${Object.keys(state || {})}`);
+  if (process.env.VGI_DISPATCH_DEBUG)
+    console.error(
+      `[httpDispatchStreamExchange] method=${method.name} effectiveProducer=${effectiveProducer} stateKeys=${Object.keys(state || {})}`,
+    );
 
   if (effectiveProducer) {
     // Producer continuation — produce more data inline.
     // For exchange-registered methods, falls back to exchangeFn with tick batches.
-    return produceStreamResponse(
-      method,
-      state,
-      outputSchema,
-      inputSchema,
-      ctx,
-      null,
-      null,
-    );
+    return produceStreamResponse(method, state, outputSchema, inputSchema, ctx, null, null);
   } else {
     // Exchange path — also handles exchange-registered methods acting as
     // producers (__isProducer=true). Use producer mode on the OutputCollector
     // when effectiveProducer so finish() is allowed.
     const out = new OutputCollector(outputSchema, effectiveProducer, ctx.serverId, null);
 
+    // Cast compatible input types (e.g., decimal→double, int32→int64)
+    const conformedBatch = conformBatchToSchema(reqBatch, inputSchema);
+
     try {
       if (method.exchangeFn) {
-        await method.exchangeFn(state, reqBatch, out);
+        await method.exchangeFn(state, conformedBatch, out);
       } else {
         await method.producerFn!(state, out);
       }
     } catch (error: any) {
-      if (process.env.VGI_DISPATCH_DEBUG) console.error(`[httpDispatchStreamExchange] exchange handler error:`, error.message, error.stack?.split('\n').slice(0,5).join('\n'));
+      if (process.env.VGI_DISPATCH_DEBUG)
+        console.error(
+          `[httpDispatchStreamExchange] exchange handler error:`,
+          error.message,
+          error.stack?.split("\n").slice(0, 5).join("\n"),
+        );
       const errBatch = buildErrorBatch(outputSchema, error, ctx.serverId, null);
       return arrowResponse(serializeIpcStream(outputSchema, [errBatch]), 500);
     }
@@ -283,12 +241,7 @@ export async function httpDispatchStreamExchange(
       const stateBytes = ctx.stateSerializer.serialize(state);
       const schemaBytes = serializeSchema(outputSchema);
       const inputSchemaBytes = serializeSchema(inputSchema);
-      const token = packStateToken(
-        stateBytes,
-        schemaBytes,
-        inputSchemaBytes,
-        ctx.signingKey,
-      );
+      const token = packStateToken(stateBytes, schemaBytes, inputSchemaBytes, ctx.signingKey);
 
       for (const emitted of out.batches) {
         const batch = emitted.batch;
@@ -304,7 +257,7 @@ export async function httpDispatchStreamExchange(
       // Safety net: if no batch carries a state token (e.g. all rows were
       // filtered out by pushdown filters), emit an empty batch with the
       // token so the client knows to continue exchanging.
-      if (!batches.some(b => b.metadata?.get(STATE_KEY))) {
+      if (!batches.some((b) => b.metadata?.get(STATE_KEY))) {
         const tokenMeta = new Map<string, string>();
         tokenMeta.set(STATE_KEY, token);
         batches.push(buildEmptyBatch(outputSchema, tokenMeta));
@@ -344,7 +297,8 @@ async function produceStreamResponse(
         await method.exchangeFn!(state, tickBatch, out);
       }
     } catch (error: any) {
-      if (process.env.VGI_DISPATCH_DEBUG) console.error(`[produceStreamResponse] error:`, error.message, error.stack?.split('\n').slice(0,3).join('\n'));
+      if (process.env.VGI_DISPATCH_DEBUG)
+        console.error(`[produceStreamResponse] error:`, error.message, error.stack?.split("\n").slice(0, 3).join("\n"));
       allBatches.push(buildErrorBatch(outputSchema, error, ctx.serverId, requestId));
       break;
     }
@@ -365,12 +319,7 @@ async function produceStreamResponse(
       const stateBytes = ctx.stateSerializer.serialize(state);
       const schemaBytes = serializeSchema(outputSchema);
       const inputSchemaBytes = serializeSchema(inputSchema);
-      const token = packStateToken(
-        stateBytes,
-        schemaBytes,
-        inputSchemaBytes,
-        ctx.signingKey,
-      );
+      const token = packStateToken(stateBytes, schemaBytes, inputSchemaBytes, ctx.signingKey);
       const tokenMeta = new Map<string, string>();
       tokenMeta.set(STATE_KEY, token);
       allBatches.push(buildEmptyBatch(outputSchema, tokenMeta));