@query-farm/vgi-rpc 0.6.4 → 0.7.1

package/dist/wire/writer.d.ts

@@ -1,59 +1,69 @@
-import { type RecordBatch, type Schema } from "@query-farm/apache-arrow";
+import type { Socket } from "node:net";
+import type { VgiBatch, VgiSchema } from "../arrow/index.js";
+type WriterTarget = {
+    kind: "fd";
+    fd: number;
+} | {
+    kind: "socket";
+    socket: Socket;
+};
 /**
- * Writes sequential IPC streams to a file descriptor (e.g., stdout).
- * Each call to writeStream() writes a complete IPC stream: schema + batches + EOS.
+ * Writes sequential IPC streams to either an fd (stdio subprocess transport)
+ * or a Node Socket (AF_UNIX transport). Each call to writeStream() writes a
+ * complete IPC stream: schema + batches + EOS.
  *
- * All writes use synchronous I/O (writeSync) to avoid deadlocks when
- * interleaving stdout writes with blocking stdin reads.
+ * All public methods are async. The fd path resolves immediately after a
+ * synchronous writeSync; the socket path awaits real `'drain'` events on
+ * backpressure so the event loop stays responsive to other connections.
  */
 export declare class IpcStreamWriter {
-    private readonly fd;
-    constructor(fd?: number);
+    private readonly target;
+    /**
+     * Construct from a file descriptor (stdio transport) or a Node net.Socket
+     * (AF_UNIX transport). The default targets stdout for legacy stdio servers
+     * that didn't pass an fd.
+     */
+    constructor(fdOrSocket?: number | Socket);
     /**
      * Write a complete IPC stream with the given schema and batches.
      * Creates schema message, writes all batches (with their metadata), writes EOS.
      */
-    writeStream(schema: Schema, batches: RecordBatch[]): void;
+    writeStream(schema: VgiSchema, batches: VgiBatch[]): Promise<void>;
     /**
      * Open an incremental IPC stream for writing batches one at a time.
-     * Used for streaming methods where output batches are produced incrementally.
-     * Bytes are written synchronously after each batch.
      */
-    openStream(schema: Schema): IncrementalStream;
+    openStream(schema: VgiSchema): IncrementalStream;
 }
 /**
  * An open IPC stream that supports incremental batch writes.
  *
- * Uses RecordBatchStreamWriter with internal buffering (no pipe to stdout).
- * After each operation, drains the writer's internal AsyncByteQueue buffer
- * and writes bytes synchronously via writeAll(). This avoids deadlocks
- * caused by Node.js async stream piping when stdin reads block before
- * stdout writes flush through the event loop.
+ * Drives a backend {@link IncrementalEncoder} and flushes its bytes through
+ * the same target (fd or socket) as the parent IpcStreamWriter. The write()
+ * and close() methods are async so the socket path can yield on backpressure
+ * — critical under AF_UNIX where the kernel send buffer (~8 KB on macOS)
+ * fills quickly and any synchronous busy-wait would starve every other
+ * connection sharing this event loop.
+ *
+ * The encoder is obtained from the Arrow facade, so this file no longer
+ * imports arrow-js directly — keeping arrow-js out of the flechette
+ * (workerd/browser) bundle. The flechette encoder throws on construction;
+ * the stdio exchange protocol is lockstep (the client reads each response
+ * batch before sending the next input) which needs an incremental writer
+ * flechette doesn't provide. workerd/browser deployments use HTTP (no
+ * stdio), so the flechette path is never reached there; `flechette-pipe`
+ * conformance is xfailed for streams.
  */
 export declare class IncrementalStream {
-    private writer;
-    private readonly fd;
+    private readonly encoder;
+    private readonly target;
     private closed;
-    constructor(fd: number, schema: Schema);
-    /**
-     * Write a single batch to the stream. Bytes are flushed synchronously.
-     *
-     * Uses _writeRecordBatch() directly to bypass the Arrow writer's schema
-     * comparison in write(). The public write() method calls compareSchemas()
-     * and auto-closes the writer if the batch's schema differs (e.g., in
-     * nullability), silently dropping the batch. Since our output schema is
-     * set at stream open time and all batches are structurally compatible,
-     * we skip the comparison.
-     */
-    write(batch: RecordBatch): void;
-    /**
-     * Close the stream (writes EOS marker synchronously).
-     */
-    close(): void;
-    /**
-     * Drain buffered bytes from the Arrow writer's internal queue
-     * and write them synchronously to the output fd.
-     */
-    private drain;
+    private writeChain;
+    constructor(target: WriterTarget, schema: VgiSchema);
+    /** Write a single batch. Resolves once the bytes are queued/flushed. */
+    write(batch: VgiBatch): Promise<void>;
+    /** Close the stream (writes EOS marker). */
+    close(): Promise<void>;
+    private enqueue;
 }
+export {};
 //# sourceMappingURL=writer.d.ts.map

package/dist/wire/writer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"writer.d.ts","sourceRoot":"","sources":["../../src/wire/writer.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,KAAK,WAAW,EAA2B,KAAK,MAAM,EAAE,MAAM,0BAA0B,CAAC;AA4BlG;;;;;;GAMG;AACH,qBAAa,eAAe;IAC1B,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAS;gBAEhB,EAAE,GAAE,MAAkB;IAIlC;;;OAGG;IACH,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,GAAG,IAAI;IAYzD;;;;OAIG;IACH,UAAU,CAAC,MAAM,EAAE,MAAM,GAAG,iBAAiB;CAG9C;AAED;;;;;;;;GAQG;AACH,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,MAAM,CAA0B;IACxC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAS;IAC5B,OAAO,CAAC,MAAM,CAAS;gBAEX,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM;IAQtC;;;;;;;;;OASG;IACH,KAAK,CAAC,KAAK,EAAE,WAAW,GAAG,IAAI;IAM/B;;OAEG;IACH,KAAK,IAAI,IAAI;IAQb;;;OAGG;IACH,OAAO,CAAC,KAAK;CAOd"}
+{"version":3,"file":"writer.d.ts","sourceRoot":"","sources":["../../src/wire/writer.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AACvC,OAAO,KAAK,EAAsB,QAAQ,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AA2GjF,KAAK,YAAY,GAAG;IAAE,IAAI,EAAE,IAAI,CAAC;IAAC,EAAE,EAAE,MAAM,CAAA;CAAE,GAAG;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAEpF;;;;;;;;GAQG;AACH,qBAAa,eAAe;IAC1B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAe;IAEtC;;;;OAIG;gBACS,UAAU,GAAE,MAAM,GAAG,MAAkB;IAQnD;;;OAGG;IACG,WAAW,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAexE;;OAEG;IACH,UAAU,CAAC,MAAM,EAAE,SAAS,GAAG,iBAAiB;CAGjD;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAqB;IAC7C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAe;IACtC,OAAO,CAAC,MAAM,CAAS;IAIvB,OAAO,CAAC,UAAU,CAAoC;gBAE1C,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,SAAS;IAQnD,wEAAwE;IAClE,KAAK,CAAC,KAAK,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAK3C,4CAA4C;IACtC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAM5B,OAAO,CAAC,OAAO;CAchB"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@query-farm/vgi-rpc",
-  "version": "0.6.4",
+  "version": "0.7.1",
   "license": "Apache-2.0",
   "homepage": "https://vgi-rpc-typescript.query.farm",
   "repository": {
@@ -8,37 +8,51 @@
     "url": "https://github.com/Query-farm/vgi-rpc-typescript"
   },
   "type": "module",
+  "sideEffects": false,
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "exports": {
     ".": {
-      "import": "./dist/index.js",
+      "workerd": "./src/index.ts",
+      "worker": "./src/index.ts",
+      "browser": "./src/index.ts",
+      "bun": "./src/index.ts",
       "types": "./dist/index.d.ts",
-      "bun": "./src/index.ts"
+      "import": "./dist/index.js"
     },
     "./otel": {
-      "import": "./dist/otel.js",
+      "bun": "./src/otel.ts",
       "types": "./dist/otel.d.ts",
-      "bun": "./src/otel.ts"
+      "import": "./dist/otel.js"
     },
     "./s3": {
-      "import": "./dist/s3.js",
+      "bun": "./src/s3.ts",
       "types": "./dist/s3.d.ts",
-      "bun": "./src/s3.ts"
+      "import": "./dist/s3.js"
     },
     "./gcs": {
-      "import": "./dist/gcs.js",
+      "bun": "./src/gcs.ts",
       "types": "./dist/gcs.d.ts",
-      "bun": "./src/gcs.ts"
+      "import": "./dist/gcs.js"
     }
   },
   "files": [
     "dist",
     "src"
   ],
+  "imports": {
+    "#vgi-rpc-arrow": {
+      "workerd": "./src/arrow/impl-flechette/index.ts",
+      "worker": "./src/arrow/impl-flechette/index.ts",
+      "default": "./src/arrow/impl-arrowjs/index.ts"
+    }
+  },
   "dependencies": {
-    "@query-farm/apache-arrow": "*",
-    "oauth4webapi": "^3.8.5"
+    "@noble/ciphers": "^2.2.0",
+    "@query-farm/apache-arrow": "^21.1.1",
+    "@uwdata/flechette": "github:Query-farm/flechette#fix/timestamp-bigint-encode",
+    "fzstd": "^0.1.1",
+    "oauth4webapi": "^3.8.6"
   },
   "peerDependencies": {
     "@opentelemetry/api": ">=1.4.0",
@@ -61,22 +75,22 @@
     }
   },
   "devDependencies": {
-    "@biomejs/biome": "^2.4.5",
-    "@opentelemetry/api": "^1.9.0",
-    "@opentelemetry/resources": "^2.6.0",
-    "@opentelemetry/sdk-metrics": "^2.6.0",
-    "@opentelemetry/sdk-trace-base": "^2.6.0",
-    "@opentelemetry/semantic-conventions": "^1.40.0",
-    "@aws-sdk/client-s3": "^3.750.0",
-    "@aws-sdk/s3-request-presigner": "^3.750.0",
-    "@google-cloud/storage": "^7.15.0",
+    "@biomejs/biome": "^2.4.16",
+    "@opentelemetry/api": "^1.9.1",
+    "@opentelemetry/resources": "^2.7.1",
+    "@opentelemetry/sdk-metrics": "^2.7.1",
+    "@opentelemetry/sdk-trace-base": "^2.7.1",
+    "@opentelemetry/semantic-conventions": "^1.41.1",
+    "@aws-sdk/client-s3": "^3.1059.0",
+    "@aws-sdk/s3-request-presigner": "^3.1059.0",
+    "@google-cloud/storage": "^7.19.0",
     "@types/bun": "latest"
   },
   "scripts": {
     "build": "bun run build:types && bun run build:js",
     "build:types": "bunx tsc -p tsconfig.build.json",
     "build:js": "bun build ./src/index.ts --outdir dist --target node --format esm --sourcemap=external --external @query-farm/apache-arrow",
-    "postinstall": "cd node_modules/@query-farm/apache-arrow && node -e \"const fs=require('fs');const p=JSON.parse(fs.readFileSync('package.json','utf8'));p.main='src/Arrow.node.ts';fs.writeFileSync('package.json',JSON.stringify(p,null,2)+'\\n')\"",
+    "postinstall": "cd node_modules/@query-farm/apache-arrow && node -e \"const fs=require('fs');const p=JSON.parse(fs.readFileSync('package.json','utf8'));p.main='src/Arrow.node.ts';fs.writeFileSync('package.json',JSON.stringify(p,null,2)+'\\n')\" && cd ../../.. && bun scripts/patch-flechette.mjs",
     "test": "bun test",
     "lint": "biome check .",
     "lint:fix": "biome check --write .",

package/src/access-log.ts

@@ -0,0 +1,200 @@
+// © Copyright 2025-2026, Query.Farm LLC - https://query.farm
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * Cross-language conformance access-log hook.
+ *
+ * Emits one JSON record per RPC dispatch to a {@link Sink} (typically a file
+ * descriptor opened in append mode).  The record shape conforms to the
+ * vgi-rpc access-log specification (`docs/access-log-spec.md` and
+ * `vgi_rpc/access_log.schema.json` in the Python reference repo).
+ *
+ * Use {@link AccessLogHook} to align this implementation with `vgi-rpc-test
+ * --access-log` so worker behaviour is checked across language ports by the
+ * same tool that gates the conformance suite.
+ */
+
+import type { CallStatistics, DispatchHook, DispatchInfo, HookToken } from "./types.js";
+
+/** Where the hook writes formatted JSON lines. */
+export interface AccessLogSink {
+  /** Write one access-log line. The trailing newline is included by the caller. */
+  write(line: string): void;
+}
+
+// Indirect-string require so esbuild can't pull node:fs into the bundle.
+// Workers should use a custom sink (e.g., one backed by `console.log`).
+const _NODE_FS_MOD = "node:fs";
+function _loadWriteSync(): (fd: number, data: Uint8Array, offset?: number, len?: number) => number {
+  const req: any = (import.meta as any).require ?? (globalThis as any).require ?? null;
+  if (!req) {
+    throw new Error(
+      "FdSink requires Node.js or Bun (node:fs.writeSync). For other runtimes, " +
+        "supply a custom AccessLogSink that wraps console.log or your logger.",
+    );
+  }
+  return req(_NODE_FS_MOD).writeSync;
+}
+
+/** A sink backed by a file descriptor; uses synchronous writes for ordering. */
+export class FdSink implements AccessLogSink {
+  private readonly _writeSync = _loadWriteSync();
+  constructor(private readonly fd: number) {}
+  /** Write `line` to the file descriptor, looping until the buffer is fully flushed. */
+  write(line: string): void {
+    const buf = new TextEncoder().encode(line);
+    let offset = 0;
+    while (offset < buf.length) {
+      const n = this._writeSync(this.fd, buf, offset, buf.length - offset);
+      if (n <= 0) throw new Error(`access-log writeSync returned ${n}`);
+      offset += n;
+    }
+  }
+}
+
+interface StartToken {
+  startNs: bigint;
+}
+
+function rfc3339Utc(): string {
+  const d = new Date();
+  const yyyy = d.getUTCFullYear().toString().padStart(4, "0");
+  const mm = (d.getUTCMonth() + 1).toString().padStart(2, "0");
+  const dd = d.getUTCDate().toString().padStart(2, "0");
+  const hh = d.getUTCHours().toString().padStart(2, "0");
+  const mi = d.getUTCMinutes().toString().padStart(2, "0");
+  const ss = d.getUTCSeconds().toString().padStart(2, "0");
+  const ms = d.getUTCMilliseconds().toString().padStart(3, "0");
+  return `${yyyy}-${mm}-${dd}T${hh}:${mi}:${ss}.${ms}Z`;
+}
+
+function base64(bytes: Uint8Array): string {
+  return Buffer.from(bytes).toString("base64");
+}
+
+/** Round to 2 decimal places. */
+function roundTo2(f: number): number {
+  return Math.round(f * 100) / 100;
+}
+
+/**
+ * Options for {@link AccessLogHook}.
+ *
+ * `level` matches Python's logger-level gating in `_emit_access_log`:
+ * at "INFO" the heavy `request_data` field (a base64 of the full
+ * request batch — typically 8+ KiB per init RPC) is replaced with a
+ * `truncated: true` marker plus `original_request_bytes`, so the
+ * access-log schema's "unary requires request_data unless truncated"
+ * invariant still holds. Bump to "DEBUG" to capture full payloads for
+ * replay/audit.
+ */
+export interface AccessLogOptions {
+  /** Server version string (optional). */
+  serverVersion?: string;
+  /** Verbosity for heavy fields. Default: "INFO". */
+  level?: "INFO" | "DEBUG";
+}
+
+export class AccessLogHook implements DispatchHook {
+  private readonly serverVersion: string;
+  private readonly level: "INFO" | "DEBUG";
+
+  constructor(
+    private readonly sink: AccessLogSink,
+    options: AccessLogOptions | string = {},
+  ) {
+    // Backward compatibility: the original signature accepted a bare
+    // serverVersion string as the second arg.
+    if (typeof options === "string") {
+      this.serverVersion = options;
+      this.level = "INFO";
+    } else {
+      this.serverVersion = options.serverVersion ?? "";
+      this.level = options.level ?? "INFO";
+    }
+  }
+
+  /** Capture a high-resolution start timestamp; returned token feeds {@link onDispatchEnd}. */
+  onDispatchStart(_info: DispatchInfo): HookToken {
+    const token: StartToken = { startNs: process.hrtime.bigint() };
+    return token;
+  }
+
+  /** Emit one access-log JSON record for the completed dispatch (best-effort;
+   *  write errors are swallowed so logging never breaks a request). */
+  onDispatchEnd(token: HookToken, info: DispatchInfo, stats: CallStatistics, error?: Error): void {
+    const t = token as StartToken | undefined;
+    const durationMs = t ? roundTo2(Number(process.hrtime.bigint() - t.startNs) / 1_000_000) : 0;
+
+    const status = error ? "error" : "ok";
+    const errType = error ? ((error as Error & { type?: string }).type ?? error.constructor.name) : "";
+    const errMsg = error?.message ?? "";
+
+    const protocol = info.protocol ?? "";
+    const rec: Record<string, unknown> = {
+      timestamp: rfc3339Utc(),
+      level: "INFO",
+      logger: "vgi_rpc.access",
+      message: `${protocol}.${info.method} ${status}`,
+      server_id: info.serverId,
+      protocol,
+      protocol_hash: info.protocolHash ?? "",
+      method: info.method,
+      method_type: info.methodType,
+      principal: info.principal ?? "",
+      auth_domain: info.authDomain ?? "",
+      authenticated: info.authenticated ?? false,
+      remote_addr: info.remoteAddr ?? "",
+      duration_ms: durationMs,
+      status,
+      error_type: errType,
+    };
+
+    if (errMsg) rec.error_message = errMsg;
+    if (this.serverVersion) rec.server_version = this.serverVersion;
+    if (info.protocolVersion) rec.protocol_version = info.protocolVersion;
+    if (info.requestId) rec.request_id = info.requestId;
+    if (info.requestData && info.requestData.length > 0) {
+      // At INFO, the per-request base64 payload dominates record size
+      // (an init RPC commonly logs 8+ KiB of base64 per call) and audit
+      // consumers rarely need the bytes — they care about who/what/when.
+      // Replace with a `truncated: true` marker so the access-log schema's
+      // "unary requires request_data unless truncated" invariant holds.
+      // Bump level to DEBUG to re-enable the full payload.
+      const encoded = base64(info.requestData);
+      if (this.level === "DEBUG") {
+        rec.request_data = encoded;
+      } else {
+        rec.original_request_bytes = encoded.length;
+        rec.truncated = true;
+      }
+    }
+    if (info.methodType === "stream") {
+      rec.stream_id = info.streamId ?? "00000000000000000000000000000000";
+    }
+    if (info.cancelled) rec.cancelled = true;
+
+    if (
+      stats.inputBatches +
+        stats.outputBatches +
+        stats.inputRows +
+        stats.outputRows +
+        stats.inputBytes +
+        stats.outputBytes !==
+      0
+    ) {
+      rec.input_batches = stats.inputBatches;
+      rec.output_batches = stats.outputBatches;
+      rec.input_rows = stats.inputRows;
+      rec.output_rows = stats.outputRows;
+      rec.input_bytes = stats.inputBytes;
+      rec.output_bytes = stats.outputBytes;
+    }
+
+    try {
+      this.sink.write(`${JSON.stringify(rec)}\n`);
+    } catch {
+      // best-effort
+    }
+  }
+}