@query-farm/vgi-rpc 0.1.0

package/dist/server.d.ts

@@ -0,0 +1,19 @@
+import { Protocol } from "./protocol.js";
+/**
+ * RPC server that reads Arrow IPC requests from stdin and writes responses to stdout.
+ * Supports unary and streaming (producer/exchange) methods.
+ */
+export declare class VgiRpcServer {
+    private protocol;
+    private enableDescribe;
+    private serverId;
+    private describeBatch;
+    constructor(protocol: Protocol, options?: {
+        enableDescribe?: boolean;
+        serverId?: string;
+    });
+    /** Start the server loop. Reads requests until stdin closes. */
+    run(): Promise<void>;
+    private serveOne;
+}
+//# sourceMappingURL=server.d.ts.map

package/dist/server.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAczC;;;GAGG;AACH,qBAAa,YAAY;IACvB,OAAO,CAAC,QAAQ,CAAW;IAC3B,OAAO,CAAC,cAAc,CAAU;IAChC,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,aAAa,CAAmD;gBAGtE,QAAQ,EAAE,QAAQ,EAClB,OAAO,CAAC,EAAE;QAAE,cAAc,CAAC,EAAE,OAAO,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE;IAiB3D,gEAAgE;IAC1D,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YAwCZ,QAAQ;CAsEvB"}

package/dist/types.d.ts

@@ -0,0 +1,71 @@
+import { type Schema, RecordBatch } from "apache-arrow";
+export declare enum MethodType {
+    UNARY = "unary",
+    STREAM = "stream"
+}
+/** Logging interface available to handlers. */
+export interface LogContext {
+    clientLog(level: string, message: string, extra?: Record<string, string>): void;
+}
+/** Handler for unary (request-response) RPC methods. */
+export type UnaryHandler = (params: Record<string, any>, ctx: LogContext) => Promise<Record<string, any>> | Record<string, any>;
+/** Initialization function for producer streams. Returns the initial state object. */
+export type ProducerInit<S = any> = (params: Record<string, any>) => Promise<S> | S;
+/** Called repeatedly to produce output batches. Call `out.finish()` to end the stream. */
+export type ProducerFn<S = any> = (state: S, out: OutputCollector) => Promise<void> | void;
+/** Initialization function for exchange streams. Returns the initial state object. */
+export type ExchangeInit<S = any> = (params: Record<string, any>) => Promise<S> | S;
+/** Called once per input batch. Must emit exactly one output batch per call. */
+export type ExchangeFn<S = any> = (state: S, input: RecordBatch, out: OutputCollector) => Promise<void> | void;
+/** Produces a header batch sent before the first output batch in a stream. */
+export type HeaderInit = (params: Record<string, any>, state: any, ctx: LogContext) => Record<string, any>;
+export interface MethodDefinition {
+    name: string;
+    type: MethodType;
+    paramsSchema: Schema;
+    resultSchema: Schema;
+    outputSchema?: Schema;
+    inputSchema?: Schema;
+    handler?: UnaryHandler;
+    producerInit?: ProducerInit;
+    producerFn?: ProducerFn;
+    exchangeInit?: ExchangeInit;
+    exchangeFn?: ExchangeFn;
+    headerSchema?: Schema;
+    headerInit?: HeaderInit;
+    doc?: string;
+    defaults?: Record<string, any>;
+    paramTypes?: Record<string, string>;
+}
+export interface EmittedBatch {
+    batch: RecordBatch;
+    metadata?: Map<string, string>;
+}
+/**
+ * Accumulates output batches during a produce/exchange call.
+ * Enforces that exactly one data batch is emitted per call (plus any number of log batches).
+ */
+export declare class OutputCollector implements LogContext {
+    private _batches;
+    private _dataBatchIdx;
+    private _finished;
+    private _producerMode;
+    private _outputSchema;
+    private _serverId;
+    private _requestId;
+    constructor(outputSchema: Schema, producerMode?: boolean, serverId?: string, requestId?: string | null);
+    get outputSchema(): Schema;
+    get finished(): boolean;
+    get batches(): EmittedBatch[];
+    /** Emit a pre-built RecordBatch as the data batch for this call. */
+    emit(batch: RecordBatch, metadata?: Map<string, string>): void;
+    /** Emit a data batch from column arrays keyed by field name. Int64 Number values are coerced to BigInt. */
+    emit(columns: Record<string, any[]>): void;
+    /** Single-row convenience. Wraps each value in `[value]` then calls `emit()`. */
+    emitRow(values: Record<string, any>): void;
+    /** Signal stream completion for producer streams. Throws if called on exchange streams. */
+    finish(): void;
+    /** Emit a zero-row client-directed log batch. */
+    clientLog(level: string, message: string, extra?: Record<string, string>): void;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,KAAK,MAAM,EAAE,WAAW,EAAyB,MAAM,cAAc,CAAC;AAG/E,oBAAY,UAAU;IACpB,KAAK,UAAU;IACf,MAAM,WAAW;CAClB;AAED,+CAA+C;AAC/C,MAAM,WAAW,UAAU;IACzB,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC;CACjF;AAED,wDAAwD;AACxD,MAAM,MAAM,YAAY,GAAG,CACzB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAC3B,GAAG,EAAE,UAAU,KACZ,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;AAExD,sFAAsF;AACtF,MAAM,MAAM,YAAY,CAAC,CAAC,GAAG,GAAG,IAAI,CAClC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KACxB,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;AACpB,0FAA0F;AAC1F,MAAM,MAAM,UAAU,CAAC,CAAC,GAAG,GAAG,IAAI,CAChC,KAAK,EAAE,CAAC,EACR,GAAG,EAAE,eAAe,KACjB,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;AAE1B,sFAAsF;AACtF,MAAM,MAAM,YAAY,CAAC,CAAC,GAAG,GAAG,IAAI,CAClC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KACxB,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;AACpB,gFAAgF;AAChF,MAAM,MAAM,UAAU,CAAC,CAAC,GAAG,GAAG,IAAI,CAChC,KAAK,EAAE,CAAC,EACR,KAAK,EAAE,WAAW,EAClB,GAAG,EAAE,eAAe,KACjB,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;AAE1B,8EAA8E;AAC9E,MAAM,MAAM,UAAU,GAAG,CACvB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAC3B,KAAK,EAAE,GAAG,EACV,GAAG,EAAE,UAAU,KACZ,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;AAEzB,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,UAAU,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,YAAY,CAAC;IACvB,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC/B,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACrC;AAED,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,WAAW,CAAC;IACnB,QAAQ,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAChC;AAED;;;GAGG;AACH,qBAAa,eAAgB,YAAW,UAAU;IAChD,OAAO,CAAC,QAAQ,CAAsB;IACtC,OAAO,CAAC,aAAa,CAAuB;IAC5C,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,aAAa,CAAU;IAC/B,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,UAAU,CAAgB;gBAEtB,YAAY,EAAE,MAAM,EAAE,YAAY,UAAO,EAAE,QAAQ,SAAK,EAAE,SAAS,GAAE,MAAM,GAAG,IAAW;IAOrG,IAAI,YAAY,IAAI,MAAM,CAEzB;IAED,IAAI,QAAQ,IAAI,OAAO,CAEtB;IAED,IAAI,OAAO,IAAI,YAAY,EAAE,CAE5B;IAED,oEAAoE;IACpE,IAAI,CAAC,KAAK,EAAE,WAAW,EAAE,QAAQ,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI;IAC9D,2GAA2G;IAC3G,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,GAAG,IAAI;IAmB1C,iFAAiF;IACjF,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,IAAI;IAQ1C,2FAA2F;IAC3F,MAAM,IAAI,IAAI;IAUd,iDAAiD;IACjD,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI;CAYhF"}

package/dist/util/schema.d.ts

@@ -0,0 +1,20 @@
+import { type Schema } from "apache-arrow";
+/**
+ * Serialize a Schema to the Arrow IPC Schema message format.
+ * This produces bytes compatible with Python's `pa.ipc.read_schema()`.
+ *
+ * We serialize by writing an empty-batch IPC stream and extracting
+ * the bytes, which includes the schema message. Python's read_schema()
+ * uses `pa.ipc.read_schema(pa.py_buffer(bytes))` which expects
+ * the schema flatbuffer message bytes directly — but the Python side
+ * actually uses `schema.serialize()` which produces Schema message bytes.
+ *
+ * In arrow-js, we can get the equivalent by using Message.from(schema)
+ * and encoding it, or by serializing a zero-batch stream.
+ *
+ * The Python `schema.serialize()` produces the Schema flatbuffer message bytes,
+ * and `pa.ipc.read_schema()` expects an IPC stream containing a schema message.
+ * The actual format is: continuation marker (0xFFFFFFFF) + length + flatbuffer bytes.
+ */
+export declare function serializeSchema(schema: Schema): Uint8Array;
+//# sourceMappingURL=schema.d.ts.map

package/dist/util/schema.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../src/util/schema.ts"],"names":[],"mappings":"AAGA,OAAO,EAA2B,KAAK,MAAM,EAAE,MAAM,cAAc,CAAC;AAEpE;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,UAAU,CAQ1D"}

package/dist/util/zstd.d.ts

@@ -0,0 +1,5 @@
+/** Compress data with zstd at the given level (1-22). */
+export declare function zstdCompress(data: Uint8Array, level: number): Uint8Array<ArrayBuffer>;
+/** Decompress zstd-compressed data. */
+export declare function zstdDecompress(data: Uint8Array): Uint8Array<ArrayBuffer>;
+//# sourceMappingURL=zstd.d.ts.map

package/dist/util/zstd.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"zstd.d.ts","sourceRoot":"","sources":["../../src/util/zstd.ts"],"names":[],"mappings":"AAcA,yDAAyD;AACzD,wBAAgB,YAAY,CAAC,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,GAAG,UAAU,CAAC,WAAW,CAAC,CAkBrF;AAED,uCAAuC;AACvC,wBAAgB,cAAc,CAAC,IAAI,EAAE,UAAU,GAAG,UAAU,CAAC,WAAW,CAAC,CAYxE"}

package/dist/wire/reader.d.ts

@@ -0,0 +1,40 @@
+import { type RecordBatch, type Schema } from "apache-arrow";
+export interface StreamMessage {
+    schema: Schema;
+    batches: RecordBatch[];
+}
+/**
+ * Reads sequential IPC streams from a byte source (e.g., process.stdin).
+ * Uses autoDestroy: false + reset/open pattern to read multiple streams
+ * from the same underlying byte source.
+ */
+export declare class IpcStreamReader {
+    private reader;
+    private initialized;
+    /** True once readNextBatch() returns null (EOS reached for current stream). */
+    private streamEnded;
+    private constructor();
+    static create(input: ReadableStream<Uint8Array> | NodeJS.ReadableStream): Promise<IpcStreamReader>;
+    /**
+     * Read one complete IPC stream (schema + all batches).
+     * Returns null on EOF (no more streams).
+     */
+    readStream(): Promise<StreamMessage | null>;
+    /**
+     * Open the next IPC stream and return its schema.
+     * Use readNextBatch() to read batches one at a time.
+     * Returns null on EOF.
+     */
+    openNextStream(): Promise<Schema | null>;
+    /**
+     * Read the next batch from the currently open IPC stream.
+     * Returns null when the stream ends (EOS).
+     *
+     * Once EOS is reached, subsequent calls return null immediately without
+     * reading from the underlying byte source. This prevents the Arrow-JS
+     * reader from consuming bytes that belong to the next IPC stream.
+     */
+    readNextBatch(): Promise<RecordBatch | null>;
+    cancel(): Promise<void>;
+}
+//# sourceMappingURL=reader.d.ts.map

package/dist/wire/reader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"reader.d.ts","sourceRoot":"","sources":["../../src/wire/reader.ts"],"names":[],"mappings":"AAGA,OAAO,EAAqB,KAAK,WAAW,EAAE,KAAK,MAAM,EAAE,MAAM,cAAc,CAAC;AAEhF,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,WAAW,EAAE,CAAC;CACxB;AAED;;;;GAIG;AACH,qBAAa,eAAe;IAC1B,OAAO,CAAC,MAAM,CAAoB;IAClC,OAAO,CAAC,WAAW,CAAS;IAC5B,+EAA+E;IAC/E,OAAO,CAAC,WAAW,CAAS;IAE5B,OAAO;WAIM,MAAM,CACjB,KAAK,EAAE,cAAc,CAAC,UAAU,CAAC,GAAG,MAAM,CAAC,cAAc,GACxD,OAAO,CAAC,eAAe,CAAC;IAS3B;;;OAGG;IACG,UAAU,IAAI,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IA2BjD;;;;OAIG;IACG,cAAc,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAY9C;;;;;;;OAOG;IACG,aAAa,IAAI,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC;IAiB5C,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;CAG9B"}

package/dist/wire/request.d.ts

@@ -0,0 +1,15 @@
+import { type Schema, type RecordBatch } from "apache-arrow";
+export interface ParsedRequest {
+    methodName: string;
+    requestVersion: string;
+    requestId: string | null;
+    schema: Schema;
+    params: Record<string, any>;
+    rawMetadata: Map<string, string>;
+}
+/**
+ * Parse a request from a RecordBatch with metadata.
+ * Extracts method name, version, and params from the batch.
+ */
+export declare function parseRequest(schema: Schema, batch: RecordBatch): ParsedRequest;
+//# sourceMappingURL=request.d.ts.map

package/dist/wire/request.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"request.d.ts","sourceRoot":"","sources":["../../src/wire/request.ts"],"names":[],"mappings":"AAGA,OAAO,EAAY,KAAK,MAAM,EAAE,KAAK,WAAW,EAAE,MAAM,cAAc,CAAC;AASvE,MAAM,WAAW,aAAa;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC5B,WAAW,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClC;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAC1B,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,WAAW,GACjB,aAAa,CAqEf"}

package/dist/wire/response.d.ts

@@ -0,0 +1,25 @@
+import { RecordBatch, Schema } from "apache-arrow";
+/**
+ * Coerce values for Int64 schema fields from Number to BigInt.
+ * Handles both single values and arrays. Returns a new record with coerced values.
+ */
+export declare function coerceInt64(schema: Schema, values: Record<string, any>): Record<string, any>;
+/**
+ * Build a 1-row result batch with optional metadata.
+ * For unary methods, `values` maps field names to single values.
+ */
+export declare function buildResultBatch(schema: Schema, values: Record<string, any>, serverId: string, requestId: string | null): RecordBatch;
+/**
+ * Build a 0-row error batch with EXCEPTION metadata matching Python's Message.from_exception().
+ */
+export declare function buildErrorBatch(schema: Schema, error: Error, serverId: string, requestId: string | null): RecordBatch;
+/**
+ * Build a 0-row log batch.
+ */
+export declare function buildLogBatch(schema: Schema, level: string, message: string, extra?: Record<string, any>, serverId?: string, requestId?: string | null): RecordBatch;
+/**
+ * Build a 0-row batch from a schema with metadata.
+ * Used for error/log batches.
+ */
+export declare function buildEmptyBatch(schema: Schema, metadata?: Map<string, string>): RecordBatch;
+//# sourceMappingURL=response.d.ts.map

package/dist/wire/response.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"response.d.ts","sourceRoot":"","sources":["../../src/wire/response.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,WAAW,EACX,MAAM,EAOP,MAAM,cAAc,CAAC;AAStB;;;GAGG;AACH,wBAAgB,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAc5F;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAC9B,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAC3B,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,GAAG,IAAI,GACvB,WAAW,CA0Cb;AAED;;GAEG;AACH,wBAAgB,eAAe,CAC7B,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,KAAK,EACZ,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,GAAG,IAAI,GACvB,WAAW,CAiBb;AAED;;GAEG;AACH,wBAAgB,aAAa,CAC3B,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,MAAM,EACf,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAC3B,QAAQ,CAAC,EAAE,MAAM,EACjB,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,GACxB,WAAW,CAeb;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAC7B,MAAM,EAAE,MAAM,EACd,QAAQ,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GAC7B,WAAW,CAyBb"}

package/dist/wire/writer.d.ts

@@ -0,0 +1,59 @@
+import { type RecordBatch, type Schema } from "apache-arrow";
+/**
+ * Writes sequential IPC streams to a file descriptor (e.g., stdout).
+ * Each call to writeStream() writes a complete IPC stream: schema + batches + EOS.
+ *
+ * All writes use synchronous I/O (writeSync) to avoid deadlocks when
+ * interleaving stdout writes with blocking stdin reads.
+ */
+export declare class IpcStreamWriter {
+    private readonly fd;
+    constructor(fd?: number);
+    /**
+     * Write a complete IPC stream with the given schema and batches.
+     * Creates schema message, writes all batches (with their metadata), writes EOS.
+     */
+    writeStream(schema: Schema, batches: RecordBatch[]): void;
+    /**
+     * Open an incremental IPC stream for writing batches one at a time.
+     * Used for streaming methods where output batches are produced incrementally.
+     * Bytes are written synchronously after each batch.
+     */
+    openStream(schema: Schema): IncrementalStream;
+}
+/**
+ * An open IPC stream that supports incremental batch writes.
+ *
+ * Uses RecordBatchStreamWriter with internal buffering (no pipe to stdout).
+ * After each operation, drains the writer's internal AsyncByteQueue buffer
+ * and writes bytes synchronously via writeAll(). This avoids deadlocks
+ * caused by Node.js async stream piping when stdin reads block before
+ * stdout writes flush through the event loop.
+ */
+export declare class IncrementalStream {
+    private writer;
+    private readonly fd;
+    private closed;
+    constructor(fd: number, schema: Schema);
+    /**
+     * Write a single batch to the stream. Bytes are flushed synchronously.
+     *
+     * Uses _writeRecordBatch() directly to bypass the Arrow writer's schema
+     * comparison in write(). The public write() method calls compareSchemas()
+     * and auto-closes the writer if the batch's schema differs (e.g., in
+     * nullability), silently dropping the batch. Since our output schema is
+     * set at stream open time and all batches are structurally compatible,
+     * we skip the comparison.
+     */
+    write(batch: RecordBatch): void;
+    /**
+     * Close the stream (writes EOS marker synchronously).
+     */
+    close(): void;
+    /**
+     * Drain buffered bytes from the Arrow writer's internal queue
+     * and write them synchronously to the output fd.
+     */
+    private drain;
+}
+//# sourceMappingURL=writer.d.ts.map

package/dist/wire/writer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"writer.d.ts","sourceRoot":"","sources":["../../src/wire/writer.ts"],"names":[],"mappings":"AAGA,OAAO,EAEL,KAAK,WAAW,EAChB,KAAK,MAAM,EACZ,MAAM,cAAc,CAAC;AA6BtB;;;;;;GAMG;AACH,qBAAa,eAAe;IAC1B,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAS;gBAEhB,EAAE,GAAE,MAAkB;IAIlC;;;OAGG;IACH,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,GAAG,IAAI;IAYzD;;;;OAIG;IACH,UAAU,CAAC,MAAM,EAAE,MAAM,GAAG,iBAAiB;CAG9C;AAED;;;;;;;;GAQG;AACH,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,MAAM,CAA0B;IACxC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAS;IAC5B,OAAO,CAAC,MAAM,CAAS;gBAEX,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM;IAQtC;;;;;;;;;OASG;IACH,KAAK,CAAC,KAAK,EAAE,WAAW,GAAG,IAAI;IAM/B;;OAEG;IACH,KAAK,IAAI,IAAI;IAQb;;;OAGG;IACH,OAAO,CAAC,KAAK;CAOd"}

package/package.json

@@ -0,0 +1,32 @@
+{
+  "name": "@query-farm/vgi-rpc",
+  "version": "0.1.0",
+  "license": "Apache-2.0",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts",
+      "bun": "./src/index.ts"
+    }
+  },
+  "files": [
+    "dist",
+    "src"
+  ],
+  "dependencies": {
+    "apache-arrow": "github:Query-farm/arrow-js#feat_query_farm_1"
+  },
+  "devDependencies": {
+    "@types/bun": "latest"
+  },
+  "scripts": {
+    "build": "bun run build:types && bun run build:js",
+    "build:types": "bunx tsc -p tsconfig.build.json",
+    "build:js": "bun build ./src/index.ts --outdir dist --target node --format esm --sourcemap=external --external apache-arrow",
+    "postinstall": "cd node_modules/apache-arrow && node -e \"const fs=require('fs');const p=JSON.parse(fs.readFileSync('package.json','utf8'));p.main='index.ts';fs.writeFileSync('package.json',JSON.stringify(p,null,2)+'\\n')\"",
+    "test": "bun test"
+  }
+}

package/src/client/connect.ts

@@ -0,0 +1,310 @@
+// © Copyright 2025-2026, Query.Farm LLC - https://query.farm
+// SPDX-License-Identifier: Apache-2.0
+
+import { RecordBatch, Schema } from "apache-arrow";
+import { STATE_KEY, LOG_LEVEL_KEY, DESCRIBE_METHOD_NAME } from "../constants.js";
+import { ARROW_CONTENT_TYPE } from "../http/common.js";
+import {
+  buildRequestIpc,
+  readResponseBatches,
+  dispatchLogOrError,
+  extractBatchRows,
+  readSequentialStreams,
+} from "./ipc.js";
+import { httpIntrospect, type MethodInfo, type ServiceDescription } from "./introspect.js";
+import { HttpStreamSession } from "./stream.js";
+import type { HttpConnectOptions, LogMessage, StreamSession } from "./types.js";
+
+type CompressFn = (data: Uint8Array, level: number) => Uint8Array;
+type DecompressFn = (data: Uint8Array) => Uint8Array;
+
+export interface RpcClient {
+  call(method: string, params?: Record<string, any>): Promise<Record<string, any> | null>;
+  stream(method: string, params?: Record<string, any>): Promise<StreamSession>;
+  describe(): Promise<ServiceDescription>;
+  close(): void;
+}
+
+export function httpConnect(
+  baseUrl: string,
+  options?: HttpConnectOptions,
+): RpcClient {
+  const prefix = (options?.prefix ?? "/vgi").replace(/\/+$/, "");
+  const onLog = options?.onLog;
+  const compressionLevel = options?.compressionLevel;
+
+  let methodCache: Map<string, MethodInfo> | null = null;
+  let compressFn: CompressFn | undefined;
+  let decompressFn: DecompressFn | undefined;
+  let compressionLoaded = false;
+
+  async function ensureCompression(): Promise<void> {
+    if (compressionLoaded || compressionLevel == null) return;
+    compressionLoaded = true;
+    try {
+      const mod = await import("../util/zstd.js");
+      compressFn = mod.zstdCompress;
+      decompressFn = mod.zstdDecompress;
+    } catch {
+      // zstd not available in this runtime
+    }
+  }
+
+  function buildHeaders(): Record<string, string> {
+    const headers: Record<string, string> = {
+      "Content-Type": ARROW_CONTENT_TYPE,
+    };
+    if (compressionLevel != null) {
+      headers["Content-Encoding"] = "zstd";
+      headers["Accept-Encoding"] = "zstd";
+    }
+    return headers;
+  }
+
+  function prepareBody(content: Uint8Array): Uint8Array {
+    if (compressionLevel != null && compressFn) {
+      return compressFn(content, compressionLevel);
+    }
+    return content;
+  }
+
+  async function readResponse(resp: Response): Promise<Uint8Array<ArrayBuffer>> {
+    let body = new Uint8Array(await resp.arrayBuffer());
+    if (resp.headers.get("Content-Encoding") === "zstd" && decompressFn) {
+      body = new Uint8Array(decompressFn(body));
+    }
+    return body;
+  }
+
+  async function ensureMethodCache(): Promise<Map<string, MethodInfo>> {
+    if (methodCache) return methodCache;
+    const desc = await httpIntrospect(baseUrl, { prefix });
+    methodCache = new Map(desc.methods.map((m) => [m.name, m]));
+    return methodCache;
+  }
+
+  return {
+    async call(
+      method: string,
+      params?: Record<string, any>,
+    ): Promise<Record<string, any> | null> {
+      await ensureCompression();
+      const methods = await ensureMethodCache();
+      const info = methods.get(method);
+      if (!info) {
+        throw new Error(`Unknown method: '${method}'`);
+      }
+
+      // Apply defaults
+      const fullParams = { ...(info.defaults ?? {}), ...(params ?? {}) };
+
+      const body = buildRequestIpc(info.paramsSchema, fullParams, method);
+      const resp = await fetch(`${baseUrl}${prefix}/${method}`, {
+        method: "POST",
+        headers: buildHeaders(),
+        body: prepareBody(body) as unknown as BodyInit,
+      });
+
+      const responseBody = await readResponse(resp);
+      const { batches } = await readResponseBatches(responseBody);
+
+      // Process batches: dispatch logs, find result
+      let resultBatch: RecordBatch | null = null;
+      for (const batch of batches) {
+        if (batch.numRows === 0) {
+          dispatchLogOrError(batch, onLog);
+          continue;
+        }
+        resultBatch = batch;
+      }
+
+      if (!resultBatch) {
+        // Void return (result schema has no fields)
+        return null;
+      }
+
+      // Extract single-row result
+      const rows = extractBatchRows(resultBatch);
+      if (rows.length === 0) return null;
+
+      const result = rows[0];
+      // For void methods (empty result schema), return null
+      if (info.resultSchema.fields.length === 0) return null;
+
+      // For single-field results, return the whole object
+      return result;
+    },
+
+    async stream(
+      method: string,
+      params?: Record<string, any>,
+    ): Promise<HttpStreamSession> {
+      await ensureCompression();
+      const methods = await ensureMethodCache();
+      const info = methods.get(method);
+      if (!info) {
+        throw new Error(`Unknown method: '${method}'`);
+      }
+
+      // Apply defaults
+      const fullParams = { ...(info.defaults ?? {}), ...(params ?? {}) };
+
+      const body = buildRequestIpc(info.paramsSchema, fullParams, method);
+      const resp = await fetch(`${baseUrl}${prefix}/${method}/init`, {
+        method: "POST",
+        headers: buildHeaders(),
+        body: prepareBody(body) as unknown as BodyInit,
+      });
+
+      const responseBody = await readResponse(resp);
+
+      // Parse the response: may contain header stream + data stream
+      let header: Record<string, any> | null = null;
+      let stateToken: string | null = null;
+      const pendingBatches: RecordBatch[] = [];
+      let finished = false;
+      let streamSchema: Schema | null = null;
+
+      if (info.headerSchema) {
+        // Response may contain two concatenated IPC streams:
+        // 1. Header stream
+        // 2. Data stream (with state token and/or data batches)
+        const reader = await readSequentialStreams(responseBody);
+
+        // First stream: header
+        const headerStream = await reader.readStream();
+        if (headerStream) {
+          for (const batch of headerStream.batches) {
+            if (batch.numRows === 0) {
+              dispatchLogOrError(batch, onLog);
+              continue;
+            }
+            const rows = extractBatchRows(batch);
+            if (rows.length > 0) {
+              header = rows[0];
+            }
+          }
+        }
+
+        // Second stream: data/state
+        const dataStream = await reader.readStream();
+        if (dataStream) {
+          streamSchema = dataStream.schema;
+        }
+        const headerErrorBatches: RecordBatch[] = [];
+        if (dataStream) {
+          for (const batch of dataStream.batches) {
+            if (batch.numRows === 0) {
+              // Check for state token
+              const token = batch.metadata?.get(STATE_KEY);
+              if (token) {
+                stateToken = token;
+                continue;
+              }
+              const level = batch.metadata?.get(LOG_LEVEL_KEY);
+              if (level === "EXCEPTION") {
+                headerErrorBatches.push(batch);
+                continue;
+              }
+              dispatchLogOrError(batch, onLog);
+              continue;
+            }
+            pendingBatches.push(batch);
+          }
+        }
+
+        if (headerErrorBatches.length > 0) {
+          if (pendingBatches.length > 0 || stateToken !== null) {
+            pendingBatches.push(...headerErrorBatches);
+          } else {
+            for (const batch of headerErrorBatches) {
+              dispatchLogOrError(batch, onLog);
+            }
+          }
+        }
+
+        if (!dataStream && !stateToken) {
+          finished = true;
+        }
+      } else {
+        // Single IPC stream: data/state (no header)
+        const { schema: responseSchema, batches } = await readResponseBatches(responseBody);
+        streamSchema = responseSchema;
+
+        // Collect error batches separately — only defer them if there are
+        // data batches or state tokens (mid-stream errors). Otherwise throw
+        // immediately (init-only errors like exchange_error_on_init).
+        const errorBatches: RecordBatch[] = [];
+
+        for (const batch of batches) {
+          if (batch.numRows === 0) {
+            // Check for state token
+            const token = batch.metadata?.get(STATE_KEY);
+            if (token) {
+              stateToken = token;
+              continue;
+            }
+            // Collect EXCEPTION batches for deferred dispatch
+            const level = batch.metadata?.get(LOG_LEVEL_KEY);
+            if (level === "EXCEPTION") {
+              errorBatches.push(batch);
+              continue;
+            }
+            dispatchLogOrError(batch, onLog);
+            continue;
+          }
+          pendingBatches.push(batch);
+        }
+
+        // If we have data batches or a state token, defer errors to iteration.
+        // Otherwise throw immediately (error on init).
+        if (errorBatches.length > 0) {
+          if (pendingBatches.length > 0 || stateToken !== null) {
+            pendingBatches.push(...errorBatches);
+          } else {
+            // No data, no state — this is a pure init error. Throw now.
+            for (const batch of errorBatches) {
+              dispatchLogOrError(batch, onLog);
+            }
+          }
+        }
+      }
+
+      if (pendingBatches.length === 0 && stateToken === null) {
+        finished = true;
+      }
+
+      // Determine output schema: prefer the IPC stream schema from the init
+      // response (it carries the server's actual output schema even for
+      // zero-row token batches), then pending batch schemas, then describe info.
+      const outputSchema =
+        (streamSchema && streamSchema.fields.length > 0 ? streamSchema : null)
+          ?? (pendingBatches.length > 0 ? pendingBatches[0].schema : null)
+          ?? info.outputSchema ?? info.resultSchema;
+
+      return new HttpStreamSession({
+        baseUrl,
+        prefix,
+        method,
+        stateToken,
+        outputSchema,
+        inputSchema: info.inputSchema,
+        onLog,
+        pendingBatches,
+        finished,
+        header,
+        compressionLevel,
+        compressFn,
+        decompressFn,
+      });
+    },
+
+    async describe(): Promise<ServiceDescription> {
+      return httpIntrospect(baseUrl, { prefix });
+    },
+
+    close(): void {
+      // No-op (HTTP stateless)
+    },
+  };
+}

package/src/client/index.ts

@@ -0,0 +1,14 @@
+// © Copyright 2025-2026, Query.Farm LLC - https://query.farm
+// SPDX-License-Identifier: Apache-2.0
+
+export { httpConnect, type RpcClient } from "./connect.js";
+export { httpIntrospect, parseDescribeResponse, type ServiceDescription, type MethodInfo } from "./introspect.js";
+export { HttpStreamSession } from "./stream.js";
+export { pipeConnect, subprocessConnect, PipeStreamSession } from "./pipe.js";
+export {
+  type HttpConnectOptions,
+  type LogMessage,
+  type StreamSession,
+  type PipeConnectOptions,
+  type SubprocessConnectOptions,
+} from "./types.js";