@query-farm/vgi-rpc 0.4.0 → 0.6.0

package/dist/otel.d.ts

@@ -0,0 +1,47 @@
+/**
+ * OpenTelemetry instrumentation for vgi-rpc TypeScript servers.
+ *
+ * Implements {@link DispatchHook} to add distributed tracing (spans) and
+ * metrics (request counter, duration histogram) to RPC dispatch.
+ *
+ * Requires `@opentelemetry/api` as a peer dependency.
+ *
+ * @example
+ * ```typescript
+ * import { createOtelHook } from "vgi-rpc/otel";
+ * import { createHttpHandler } from "vgi-rpc";
+ *
+ * const handler = createHttpHandler(protocol, {
+ *   dispatchHook: createOtelHook(),
+ * });
+ * ```
+ */
+import { type Meter, type Tracer } from "@opentelemetry/api";
+import type { DispatchHook } from "./types.js";
+/** Configuration for OpenTelemetry instrumentation. */
+export interface OtelConfig {
+    /** Custom TracerProvider; uses the global provider when omitted. */
+    tracerProvider?: {
+        getTracer(name: string): Tracer;
+    };
+    /** Custom MeterProvider; uses the global provider when omitted. */
+    meterProvider?: {
+        getMeter(name: string): Meter;
+    };
+    /** Enable span creation. Default: true. */
+    enableTracing?: boolean;
+    /** Enable counter/histogram recording. Default: true. */
+    enableMetrics?: boolean;
+    /** Record exceptions on error spans. Default: true. */
+    recordExceptions?: boolean;
+    /** Service name for the rpc.service attribute. Default: "TypeScriptRpcServer". */
+    serviceName?: string;
+}
+/**
+ * Create a {@link DispatchHook} that instruments RPC calls with OpenTelemetry.
+ *
+ * Creates a span for each RPC call with method attributes, and records
+ * request count and duration metrics.
+ */
+export declare function createOtelHook(config?: OtelConfig): DispatchHook;
+//# sourceMappingURL=otel.d.ts.map

package/dist/otel.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"otel.d.ts","sourceRoot":"","sources":["../src/otel.ts"],"names":[],"mappings":"AAGA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAIL,KAAK,KAAK,EAKV,KAAK,MAAM,EAEZ,MAAM,oBAAoB,CAAC;AAC5B,OAAO,KAAK,EAAkB,YAAY,EAA2B,MAAM,YAAY,CAAC;AAIxF,uDAAuD;AACvD,MAAM,WAAW,UAAU;IACzB,oEAAoE;IACpE,cAAc,CAAC,EAAE;QAAE,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,CAAC;IACrD,mEAAmE;IACnE,aAAa,CAAC,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,KAAK,CAAA;KAAE,CAAC;IAClD,2CAA2C;IAC3C,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,yDAAyD;IACzD,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,uDAAuD;IACvD,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,kFAAkF;IAClF,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAOD;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,MAAM,CAAC,EAAE,UAAU,GAAG,YAAY,CA+FhE"}

package/dist/s3.d.ts

@@ -0,0 +1,43 @@
+/**
+ * S3 storage backend for external storage of large Arrow IPC batches.
+ *
+ * Requires `@aws-sdk/client-s3` and `@aws-sdk/s3-request-presigner`
+ * as peer dependencies.
+ *
+ * @example
+ * ```typescript
+ * import { createS3Storage } from "@query-farm/vgi-rpc/s3";
+ *
+ * const storage = createS3Storage({
+ *   bucket: "my-bucket",
+ *   prefix: "vgi-rpc/",
+ * });
+ * const handler = createHttpHandler(protocol, {
+ *   externalLocation: { storage, externalizeThresholdBytes: 1_048_576 },
+ * });
+ * ```
+ */
+import type { ExternalStorage } from "./external.js";
+/** Configuration for the S3 storage backend. */
+export interface S3StorageConfig {
+    /** S3 bucket name. */
+    bucket: string;
+    /** Key prefix for uploaded objects. Default: "vgi-rpc/". */
+    prefix?: string;
+    /** Lifetime of pre-signed GET URLs in seconds. Default: 3600 (1 hour). */
+    presignExpirySeconds?: number;
+    /** AWS region. If omitted, uses default SDK config. */
+    region?: string;
+    /** Custom S3 endpoint URL (for MinIO, LocalStack, etc.). */
+    endpointUrl?: string;
+    /** Force path-style addressing (required for some S3-compatible services). */
+    forcePathStyle?: boolean;
+}
+/**
+ * Create an S3-backed ExternalStorage.
+ *
+ * Lazily imports `@aws-sdk/client-s3` and `@aws-sdk/s3-request-presigner`
+ * on first upload to avoid loading the AWS SDK unless needed.
+ */
+export declare function createS3Storage(config: S3StorageConfig): ExternalStorage;
+//# sourceMappingURL=s3.d.ts.map

package/dist/s3.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"s3.d.ts","sourceRoot":"","sources":["../src/s3.ts"],"names":[],"mappings":"AAGA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAErD,gDAAgD;AAChD,MAAM,WAAW,eAAe;IAC9B,sBAAsB;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,4DAA4D;IAC5D,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,0EAA0E;IAC1E,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,uDAAuD;IACvD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,4DAA4D;IAC5D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,8EAA8E;IAC9E,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,eAAe,GAAG,eAAe,CA8CxE"}

package/dist/server.d.ts

@@ -1,4 +1,6 @@
+import type { ExternalLocationConfig } from "./external.js";
 import type { Protocol } from "./protocol.js";
+import { type DispatchHook } from "./types.js";
 /**
  * RPC server that reads Arrow IPC requests from stdin and writes responses to stdout.
  * Supports unary and streaming (producer/exchange) methods.
@@ -8,9 +10,13 @@ export declare class VgiRpcServer {
     private enableDescribe;
     private serverId;
     private describeBatch;
+    private dispatchHook;
+    private externalConfig;
     constructor(protocol: Protocol, options?: {
         enableDescribe?: boolean;
         serverId?: string;
+        dispatchHook?: DispatchHook;
+        externalLocation?: ExternalLocationConfig;
     });
     /** Start the server loop. Reads requests until stdin closes. */
     run(): Promise<void>;

package/dist/server.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAS9C;;;GAGG;AACH,qBAAa,YAAY;IACvB,OAAO,CAAC,QAAQ,CAAW;IAC3B,OAAO,CAAC,cAAc,CAAU;IAChC,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,aAAa,CAA+D;gBAExE,QAAQ,EAAE,QAAQ,EAAE,OAAO,CAAC,EAAE;QAAE,cAAc,CAAC,EAAE,OAAO,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE;IAWzF,gEAAgE;IAC1D,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YAwCZ,QAAQ;CA0DvB"}
+{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,eAAe,CAAC;AAC5D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAuB,KAAK,YAAY,EAAiC,MAAM,YAAY,CAAC;AAQnG;;;GAGG;AACH,qBAAa,YAAY;IACvB,OAAO,CAAC,QAAQ,CAAW;IAC3B,OAAO,CAAC,cAAc,CAAU;IAChC,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,aAAa,CAA+D;IACpF,OAAO,CAAC,YAAY,CAA6B;IACjD,OAAO,CAAC,cAAc,CAAqC;gBAGzD,QAAQ,EAAE,QAAQ,EAClB,OAAO,CAAC,EAAE;QACR,cAAc,CAAC,EAAE,OAAO,CAAC;QACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,YAAY,CAAC,EAAE,YAAY,CAAC;QAC5B,gBAAgB,CAAC,EAAE,sBAAsB,CAAC;KAC3C;IAcH,gEAAgE;IAC1D,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;YAwCZ,QAAQ;CA+EvB"}

package/dist/types.d.ts

@@ -42,6 +42,36 @@ export interface MethodDefinition {
     defaults?: Record<string, any>;
     paramTypes?: Record<string, string>;
 }
+/** Metadata passed to dispatch hooks before and after RPC method execution. */
+export interface DispatchInfo {
+    /** RPC method name. */
+    method: string;
+    /** "unary" or "stream". */
+    methodType: string;
+    /** Server identifier. */
+    serverId: string;
+    /** Client-supplied request identifier, or null. */
+    requestId: string | null;
+}
+/** Per-call I/O counters, matching Python's CallStatistics. */
+export interface CallStatistics {
+    inputBatches: number;
+    outputBatches: number;
+    inputRows: number;
+    outputRows: number;
+    inputBytes: number;
+    outputBytes: number;
+}
+/** Opaque token returned by onDispatchStart, passed back to onDispatchEnd. */
+export type HookToken = unknown;
+/**
+ * Observability hook called around RPC dispatch.
+ * Implementations must be safe for concurrent use (HTTP transport is concurrent).
+ */
+export interface DispatchHook {
+    onDispatchStart(info: DispatchInfo): HookToken;
+    onDispatchEnd(token: HookToken, info: DispatchInfo, stats: CallStatistics, error?: Error): void;
+}
 export interface EmittedBatch {
     batch: RecordBatch;
     metadata?: Map<string, string>;

package/dist/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,WAAW,EAAyB,KAAK,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAC3F,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAGxC,oBAAY,UAAU;IACpB,KAAK,UAAU;IACf,MAAM,WAAW;CAClB;AAED,+CAA+C;AAC/C,MAAM,WAAW,UAAU;IACzB,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC;CACjF;AAED,wEAAwE;AACxE,MAAM,WAAW,WAAY,SAAQ,UAAU;IAC7C,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC;CAC5B;AAED,wDAAwD;AACxD,MAAM,MAAM,YAAY,GAAG,CACzB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAC3B,GAAG,EAAE,UAAU,KACZ,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;AAExD,sFAAsF;AACtF,MAAM,MAAM,YAAY,CAAC,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;AACpF,0FAA0F;AAC1F,MAAM,MAAM,UAAU,CAAC,CAAC,GAAG,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,eAAe,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;AAE3F,sFAAsF;AACtF,MAAM,MAAM,YAAY,CAAC,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;AACpF,gFAAgF;AAChF,MAAM,MAAM,UAAU,CAAC,CAAC,GAAG,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,EAAE,eAAe,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;AAE/G,8EAA8E;AAC9E,MAAM,MAAM,UAAU,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,UAAU,KAAK,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;AAE3G,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,UAAU,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,YAAY,CAAC;IACvB,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC/B,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACrC;AAED,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,WAAW,CAAC;IACnB,QAAQ,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAChC;AAED;;;GAGG;AACH,qBAAa,eAAgB,YAAW,WAAW;IACjD,OAAO,CAAC,QAAQ,CAAsB;IACtC,OAAO,CAAC,aAAa,CAAuB;IAC5C,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,aAAa,CAAU;IAC/B,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,UAAU,CAAgB;IAClC,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC;gBAGzB,YAAY,EAAE,MAAM,EACpB,YAAY,UAAO,EACnB,QAAQ,SAAK,EACb,SAAS,GAAE,MAAM,GAAG,IAAW,EAC/B,WAAW,CAAC,EAAE,WAAW;IAS3B,IAAI,YAAY,IAAI,MAAM,CAEzB;IAED,IAAI,QAAQ,IAAI,OAAO,CAEtB;IAED,IAAI,OAAO,IAAI,YAAY,EAAE,CAE5B;IAED,oEAAoE;IACpE,IAAI,CAAC,KAAK,EAAE,WAAW,EAAE,QAAQ,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI;IAC9D,2GAA2G;IAC3G,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,GAAG,IAAI;IAgB1C,iFAAiF;IACjF,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,IAAI;IAQ1C,2FAA2F;IAC3F,MAAM,IAAI,IAAI;IASd,iDAAiD;IACjD,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI;CAIhF"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,WAAW,EAAyB,KAAK,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAC3F,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAGxC,oBAAY,UAAU;IACpB,KAAK,UAAU;IACf,MAAM,WAAW;CAClB;AAED,+CAA+C;AAC/C,MAAM,WAAW,UAAU;IACzB,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC;CACjF;AAED,wEAAwE;AACxE,MAAM,WAAW,WAAY,SAAQ,UAAU;IAC7C,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC;CAC5B;AAED,wDAAwD;AACxD,MAAM,MAAM,YAAY,GAAG,CACzB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAC3B,GAAG,EAAE,UAAU,KACZ,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;AAExD,sFAAsF;AACtF,MAAM,MAAM,YAAY,CAAC,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;AACpF,0FAA0F;AAC1F,MAAM,MAAM,UAAU,CAAC,CAAC,GAAG,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,eAAe,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;AAE3F,sFAAsF;AACtF,MAAM,MAAM,YAAY,CAAC,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;AACpF,gFAAgF;AAChF,MAAM,MAAM,UAAU,CAAC,CAAC,GAAG,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,EAAE,eAAe,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;AAE/G,8EAA8E;AAC9E,MAAM,MAAM,UAAU,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,UAAU,KAAK,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;AAE3G,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,UAAU,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,YAAY,CAAC;IACvB,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC/B,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACrC;AAED,+EAA+E;AAC/E,MAAM,WAAW,YAAY;IAC3B,uBAAuB;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,2BAA2B;IAC3B,UAAU,EAAE,MAAM,CAAC;IACnB,yBAAyB;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,mDAAmD;IACnD,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B;AAED,+DAA+D;AAC/D,MAAM,WAAW,cAAc;IAC7B,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,8EAA8E;AAC9E,MAAM,MAAM,SAAS,GAAG,OAAO,CAAC;AAEhC;;;GAGG;AACH,MAAM,WAAW,YAAY;IAC3B,eAAe,CAAC,IAAI,EAAE,YAAY,GAAG,SAAS,CAAC;IAC/C,aAAa,CAAC,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,cAAc,EAAE,KAAK,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC;CACjG;AAED,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,WAAW,CAAC;IACnB,QAAQ,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAChC;AAED;;;GAGG;AACH,qBAAa,eAAgB,YAAW,WAAW;IACjD,OAAO,CAAC,QAAQ,CAAsB;IACtC,OAAO,CAAC,aAAa,CAAuB;IAC5C,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,aAAa,CAAU;IAC/B,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,UAAU,CAAgB;IAClC,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC;gBAGzB,YAAY,EAAE,MAAM,EACpB,YAAY,UAAO,EACnB,QAAQ,SAAK,EACb,SAAS,GAAE,MAAM,GAAG,IAAW,EAC/B,WAAW,CAAC,EAAE,WAAW;IAS3B,IAAI,YAAY,IAAI,MAAM,CAEzB;IAED,IAAI,QAAQ,IAAI,OAAO,CAEtB;IAED,IAAI,OAAO,IAAI,YAAY,EAAE,CAE5B;IAED,oEAAoE;IACpE,IAAI,CAAC,KAAK,EAAE,WAAW,EAAE,QAAQ,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI;IAC9D,2GAA2G;IAC3G,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,GAAG,IAAI;IAgB1C,iFAAiF;IACjF,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,IAAI;IAQ1C,2FAA2F;IAC3F,MAAM,IAAI,IAAI;IASd,iDAAiD;IACjD,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI;CAIhF"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@query-farm/vgi-rpc",
-  "version": "0.4.0",
+  "version": "0.6.0",
   "license": "Apache-2.0",
   "homepage": "https://vgi-rpc-typescript.query.farm",
   "repository": {
@@ -15,6 +15,21 @@
       "import": "./dist/index.js",
       "types": "./dist/index.d.ts",
       "bun": "./src/index.ts"
+    },
+    "./otel": {
+      "import": "./dist/otel.js",
+      "types": "./dist/otel.d.ts",
+      "bun": "./src/otel.ts"
+    },
+    "./s3": {
+      "import": "./dist/s3.js",
+      "types": "./dist/s3.d.ts",
+      "bun": "./src/s3.ts"
+    },
+    "./gcs": {
+      "import": "./dist/gcs.js",
+      "types": "./dist/gcs.d.ts",
+      "bun": "./src/gcs.ts"
     }
   },
   "files": [
@@ -25,8 +40,36 @@
     "@query-farm/apache-arrow": "*",
     "oauth4webapi": "^3.8.5"
   },
+  "peerDependencies": {
+    "@opentelemetry/api": ">=1.4.0",
+    "@aws-sdk/client-s3": ">=3.0.0",
+    "@aws-sdk/s3-request-presigner": ">=3.0.0",
+    "@google-cloud/storage": ">=7.0.0"
+  },
+  "peerDependenciesMeta": {
+    "@opentelemetry/api": {
+      "optional": true
+    },
+    "@aws-sdk/client-s3": {
+      "optional": true
+    },
+    "@aws-sdk/s3-request-presigner": {
+      "optional": true
+    },
+    "@google-cloud/storage": {
+      "optional": true
+    }
+  },
   "devDependencies": {
     "@biomejs/biome": "^2.4.5",
+    "@opentelemetry/api": "^1.9.0",
+    "@opentelemetry/resources": "^2.6.0",
+    "@opentelemetry/sdk-metrics": "^2.6.0",
+    "@opentelemetry/sdk-trace-base": "^2.6.0",
+    "@opentelemetry/semantic-conventions": "^1.40.0",
+    "@aws-sdk/client-s3": "^3.750.0",
+    "@aws-sdk/s3-request-presigner": "^3.750.0",
+    "@google-cloud/storage": "^7.15.0",
     "@types/bun": "latest"
   },
   "scripts": {

package/src/client/connect.ts

@@ -4,6 +4,7 @@
 import type { RecordBatch, Schema } from "@query-farm/apache-arrow";
 import { LOG_LEVEL_KEY, STATE_KEY } from "../constants.js";
 import { RpcError } from "../errors.js";
+import { isExternalLocationBatch, resolveExternalLocation } from "../external.js";
 import { ARROW_CONTENT_TYPE } from "../http/common.js";
 import { httpIntrospect, type MethodInfo, type ServiceDescription } from "./introspect.js";
 import {
@@ -27,10 +28,11 @@ export interface RpcClient {
 }
 
 export function httpConnect(baseUrl: string, options?: HttpConnectOptions): RpcClient {
-  const prefix = (options?.prefix ?? "/vgi").replace(/\/+$/, "");
+  const prefix = (options?.prefix ?? "").replace(/\/+$/, "");
   const onLog = options?.onLog;
   const compressionLevel = options?.compressionLevel;
   const authorization = options?.authorization;
+  const externalConfig = options?.externalLocation;
 
   let methodCache: Map<string, MethodInfo> | null = null;
   let compressFn: CompressFn | undefined;
@@ -114,12 +116,17 @@ export function httpConnect(baseUrl: string, options?: HttpConnectOptions): RpcC
       const responseBody = await readResponse(resp);
       const { batches } = await readResponseBatches(responseBody);
 
-      // Process batches: dispatch logs, find result
+      // Process batches: dispatch logs, resolve external pointers, find result
       let resultBatch: RecordBatch | null = null;
-      for (const batch of batches) {
+      for (let batch of batches) {
         if (batch.numRows === 0) {
-          dispatchLogOrError(batch, onLog);
-          continue;
+          // Check for external location pointer batch
+          if (isExternalLocationBatch(batch)) {
+            batch = await resolveExternalLocation(batch, externalConfig);
+          } else {
+            dispatchLogOrError(batch, onLog);
+            continue;
+          }
         }
         resultBatch = batch;
       }
@@ -302,6 +309,7 @@ export function httpConnect(baseUrl: string, options?: HttpConnectOptions): RpcC
         compressFn,
         decompressFn,
         authorization,
+        externalConfig,
       });
     },
 

package/src/client/index.ts

@@ -4,7 +4,16 @@
 export { httpConnect, type RpcClient } from "./connect.js";
 export { httpIntrospect, type MethodInfo, parseDescribeResponse, type ServiceDescription } from "./introspect.js";
 export type { OAuthResourceMetadataResponse } from "./oauth.js";
-export { fetchOAuthMetadata, httpOAuthMetadata, parseResourceMetadataUrl } from "./oauth.js";
+export {
+  fetchOAuthMetadata,
+  httpOAuthMetadata,
+  parseClientId,
+  parseClientSecret,
+  parseDeviceCodeClientId,
+  parseDeviceCodeClientSecret,
+  parseResourceMetadataUrl,
+  parseUseIdTokenAsBearer,
+} from "./oauth.js";
 export { PipeStreamSession, pipeConnect, subprocessConnect } from "./pipe.js";
 export { HttpStreamSession } from "./stream.js";
 export type {

package/src/client/introspect.ts

@@ -121,7 +121,7 @@ export async function httpIntrospect(
   baseUrl: string,
   options?: { prefix?: string; authorization?: string },
 ): Promise<ServiceDescription> {
-  const prefix = options?.prefix ?? "/vgi";
+  const prefix = options?.prefix ?? "";
   const emptySchema = new ArrowSchema([]);
   const body = buildRequestIpc(emptySchema, {}, DESCRIBE_METHOD_NAME);
 

package/src/client/oauth.ts

@@ -7,10 +7,21 @@ export interface OAuthResourceMetadataResponse {
   authorizationServers: string[];
   scopesSupported?: string[];
   bearerMethodsSupported?: string[];
+  resourceSigningAlgValuesSupported?: string[];
   resourceName?: string;
   resourceDocumentation?: string;
   resourcePolicyUri?: string;
   resourceTosUri?: string;
+  /** OAuth client_id advertised by the server. */
+  clientId?: string;
+  /** OAuth client_secret advertised by the server. */
+  clientSecret?: string;
+  /** When true, use the OIDC id_token as the Bearer token instead of access_token. */
+  useIdTokenAsBearer?: boolean;
+  /** OAuth client_id for device code flow. */
+  deviceCodeClientId?: string;
+  /** OAuth client_secret for device code flow. */
+  deviceCodeClientSecret?: string;
 }
 
 function parseMetadataJson(json: Record<string, any>): OAuthResourceMetadataResponse {
@@ -20,10 +31,17 @@ function parseMetadataJson(json: Record<string, any>): OAuthResourceMetadataResp
   };
   if (json.scopes_supported) result.scopesSupported = json.scopes_supported;
   if (json.bearer_methods_supported) result.bearerMethodsSupported = json.bearer_methods_supported;
+  if (json.resource_signing_alg_values_supported)
+    result.resourceSigningAlgValuesSupported = json.resource_signing_alg_values_supported;
   if (json.resource_name) result.resourceName = json.resource_name;
   if (json.resource_documentation) result.resourceDocumentation = json.resource_documentation;
   if (json.resource_policy_uri) result.resourcePolicyUri = json.resource_policy_uri;
   if (json.resource_tos_uri) result.resourceTosUri = json.resource_tos_uri;
+  if (json.client_id) result.clientId = json.client_id;
+  if (json.client_secret) result.clientSecret = json.client_secret;
+  if (json.use_id_token_as_bearer) result.useIdTokenAsBearer = json.use_id_token_as_bearer;
+  if (json.device_code_client_id) result.deviceCodeClientId = json.device_code_client_id;
+  if (json.device_code_client_secret) result.deviceCodeClientSecret = json.device_code_client_secret;
   return result;
 }
 
@@ -35,7 +53,7 @@ export async function httpOAuthMetadata(
   baseUrl: string,
   prefix?: string,
 ): Promise<OAuthResourceMetadataResponse | null> {
-  const effectivePrefix = (prefix ?? "/vgi").replace(/\/+$/, "");
+  const effectivePrefix = (prefix ?? "").replace(/\/+$/, "");
   const metadataUrl = `${baseUrl.replace(/\/+$/, "")}/.well-known/oauth-protected-resource${effectivePrefix}`;
 
   try {
@@ -72,3 +90,78 @@ export function parseResourceMetadataUrl(wwwAuthenticate: string): string | null
 
   return metadataMatch[1];
 }
+
+/**
+ * Extract the `client_id` from a WWW-Authenticate Bearer challenge.
+ * Returns `null` if no client_id parameter is found.
+ */
+export function parseClientId(wwwAuthenticate: string): string | null {
+  const bearerMatch = wwwAuthenticate.match(/^Bearer\s+(.*)/i);
+  if (!bearerMatch) return null;
+
+  const params = bearerMatch[1];
+  const clientIdMatch = params.match(/client_id="([^"]+)"/);
+  if (!clientIdMatch) return null;
+
+  return clientIdMatch[1];
+}
+
+/**
+ * Extract the `client_secret` from a WWW-Authenticate Bearer challenge.
+ * Returns `null` if no client_secret parameter is found.
+ */
+export function parseClientSecret(wwwAuthenticate: string): string | null {
+  const bearerMatch = wwwAuthenticate.match(/^Bearer\s+(.*)/i);
+  if (!bearerMatch) return null;
+
+  const params = bearerMatch[1];
+  const match = params.match(/client_secret="([^"]+)"/);
+  if (!match) return null;
+
+  return match[1];
+}
+
+/**
+ * Extract the `use_id_token_as_bearer` flag from a WWW-Authenticate Bearer challenge.
+ * Returns `true` if the parameter is present and set to "true", `false` otherwise.
+ */
+export function parseUseIdTokenAsBearer(wwwAuthenticate: string): boolean {
+  const bearerMatch = wwwAuthenticate.match(/^Bearer\s+(.*)/i);
+  if (!bearerMatch) return false;
+
+  const params = bearerMatch[1];
+  const match = params.match(/use_id_token_as_bearer="([^"]+)"/);
+  if (!match) return false;
+
+  return match[1] === "true";
+}
+
+/**
+ * Extract the `device_code_client_id` from a WWW-Authenticate Bearer challenge.
+ * Returns `null` if no device_code_client_id parameter is found.
+ */
+export function parseDeviceCodeClientId(wwwAuthenticate: string): string | null {
+  const bearerMatch = wwwAuthenticate.match(/^Bearer\s+(.*)/i);
+  if (!bearerMatch) return null;
+
+  const params = bearerMatch[1];
+  const match = params.match(/device_code_client_id="([^"]+)"/);
+  if (!match) return null;
+
+  return match[1];
+}
+
+/**
+ * Extract the `device_code_client_secret` from a WWW-Authenticate Bearer challenge.
+ * Returns `null` if no device_code_client_secret parameter is found.
+ */
+export function parseDeviceCodeClientSecret(wwwAuthenticate: string): string | null {
+  const bearerMatch = wwwAuthenticate.match(/^Bearer\s+(.*)/i);
+  if (!bearerMatch) return null;
+
+  const params = bearerMatch[1];
+  const match = params.match(/device_code_client_secret="([^"]+)"/);
+  if (!match) return null;
+
+  return match[1];
+}

package/src/client/pipe.ts

@@ -12,6 +12,7 @@ import {
 } from "@query-farm/apache-arrow";
 import { DESCRIBE_METHOD_NAME } from "../constants.js";
 import { RpcError } from "../errors.js";
+import { type ExternalLocationConfig, isExternalLocationBatch, resolveExternalLocation } from "../external.js";
 import { serializeIpcStream } from "../http/common.js";
 import { IpcStreamReader } from "../wire/reader.js";
 import type { RpcClient } from "./connect.js";
@@ -86,6 +87,7 @@ export class PipeStreamSession implements StreamSession {
   private _outputSchema: Schema;
   private _releaseBusy: () => void;
   private _setDrainPromise: (p: Promise<void>) => void;
+  private _externalConfig?: ExternalLocationConfig;
 
   constructor(opts: {
     reader: IpcStreamReader;
@@ -95,6 +97,7 @@ export class PipeStreamSession implements StreamSession {
     outputSchema: Schema;
     releaseBusy: () => void;
     setDrainPromise: (p: Promise<void>) => void;
+    externalConfig?: ExternalLocationConfig;
   }) {
     this._reader = opts.reader;
     this._writeFn = opts.writeFn;
@@ -103,6 +106,7 @@ export class PipeStreamSession implements StreamSession {
     this._outputSchema = opts.outputSchema;
     this._releaseBusy = opts.releaseBusy;
     this._setDrainPromise = opts.setDrainPromise;
+    this._externalConfig = opts.externalConfig;
   }
 
   get header(): Record<string, any> | null {
@@ -120,6 +124,10 @@ export class PipeStreamSession implements StreamSession {
       if (batch === null) return null; // Server closed output stream
 
       if (batch.numRows === 0) {
+        // Check for external location pointer batch
+        if (isExternalLocationBatch(batch)) {
+          return await resolveExternalLocation(batch, this._externalConfig);
+        }
         // Check if it's a log/error batch. If so, dispatch and continue.
         // Otherwise it's a zero-row data batch — return it.
         if (dispatchLogOrError(batch, this._onLog)) {
@@ -375,6 +383,7 @@ export function pipeConnect(
   options?: PipeConnectOptions,
 ): RpcClient {
   const onLog = options?.onLog;
+  const externalConfig = options?.externalLocation;
 
   let reader: IpcStreamReader | null = null;
   let readerPromise: Promise<IpcStreamReader> | null = null;
@@ -483,12 +492,16 @@ export function pipeConnect(
           throw new Error("EOF reading response");
         }
 
-        // Process batches: dispatch logs, find result
+        // Process batches: dispatch logs, resolve external pointers, find result
         let resultBatch: RecordBatch | null = null;
-        for (const batch of response.batches) {
+        for (let batch of response.batches) {
           if (batch.numRows === 0) {
-            dispatchLogOrError(batch, onLog);
-            continue;
+            if (isExternalLocationBatch(batch)) {
+              batch = await resolveExternalLocation(batch, externalConfig);
+            } else {
+              dispatchLogOrError(batch, onLog);
+              continue;
+            }
           }
           resultBatch = batch;
         }
@@ -557,6 +570,7 @@ export function pipeConnect(
           outputSchema,
           releaseBusy,
           setDrainPromise,
+          externalConfig,
         });
       } catch (e) {
         // Init error (e.g., server raised exception during init).
@@ -624,6 +638,7 @@ export function subprocessConnect(cmd: string[], options?: SubprocessConnectOpti
 
   const client = pipeConnect(stdout, writable, {
     onLog: options?.onLog,
+    externalLocation: options?.externalLocation,
   });
 
   // Wrap close to also kill the subprocess

package/src/client/stream.ts

@@ -4,6 +4,7 @@
 import { Field, makeData, RecordBatch, Schema, Struct, vectorFromArray } from "@query-farm/apache-arrow";
 import { STATE_KEY } from "../constants.js";
 import { RpcError } from "../errors.js";
+import { type ExternalLocationConfig, isExternalLocationBatch, resolveExternalLocation } from "../external.js";
 import { ARROW_CONTENT_TYPE, serializeIpcStream } from "../http/common.js";
 import { dispatchLogOrError, extractBatchRows, inferArrowType, readResponseBatches } from "./ipc.js";
 import type { LogMessage, StreamSession } from "./types.js";
@@ -26,6 +27,7 @@ export class HttpStreamSession implements StreamSession {
   private _compressFn?: CompressFn;
   private _decompressFn?: DecompressFn;
   private _authorization?: string;
+  private _externalConfig?: ExternalLocationConfig;
 
   constructor(opts: {
     baseUrl: string;
@@ -42,6 +44,7 @@ export class HttpStreamSession implements StreamSession {
     compressFn?: CompressFn;
     decompressFn?: DecompressFn;
     authorization?: string;
+    externalConfig?: ExternalLocationConfig;
   }) {
     this._baseUrl = opts.baseUrl;
     this._prefix = opts.prefix;
@@ -57,6 +60,7 @@ export class HttpStreamSession implements StreamSession {
     this._compressFn = opts.compressFn;
     this._decompressFn = opts.decompressFn;
     this._authorization = opts.authorization;
+    this._externalConfig = opts.externalConfig;
   }
 
   get header(): Record<string, any> | null {
@@ -211,10 +215,14 @@ export class HttpStreamSession implements StreamSession {
    */
   async *[Symbol.asyncIterator](): AsyncIterableIterator<Record<string, any>[]> {
     // Yield pre-loaded batches from init
-    for (const batch of this._pendingBatches) {
+    for (let batch of this._pendingBatches) {
       if (batch.numRows === 0) {
-        dispatchLogOrError(batch, this._onLog);
-        continue;
+        if (isExternalLocationBatch(batch)) {
+          batch = await resolveExternalLocation(batch, this._externalConfig);
+        } else {
+          dispatchLogOrError(batch, this._onLog);
+          continue;
+        }
       }
       yield extractBatchRows(batch);
     }
@@ -229,7 +237,7 @@ export class HttpStreamSession implements StreamSession {
       const { batches } = await readResponseBatches(responseBody);
 
       let gotContinuation = false;
-      for (const batch of batches) {
+      for (let batch of batches) {
         if (batch.numRows === 0) {
           // Check for continuation token
           const token = batch.metadata?.get(STATE_KEY);
@@ -238,9 +246,14 @@ export class HttpStreamSession implements StreamSession {
             gotContinuation = true;
             continue;
           }
-          // Log/error batch
-          dispatchLogOrError(batch, this._onLog);
-          continue;
+          // Check for external location pointer
+          if (isExternalLocationBatch(batch)) {
+            batch = await resolveExternalLocation(batch, this._externalConfig);
+          } else {
+            // Log/error batch
+            dispatchLogOrError(batch, this._onLog);
+            continue;
+          }
         }
 
         yield extractBatchRows(batch);

package/src/client/types.ts

@@ -7,6 +7,8 @@ export interface HttpConnectOptions {
   compressionLevel?: number;
   /** Authorization header value (e.g. "Bearer <token>"). Sent with every request. */
   authorization?: string;
+  /** External storage config for resolving externalized batches. */
+  externalLocation?: import("../external.js").ExternalLocationConfig;
 }
 
 export interface LogMessage {
@@ -24,6 +26,8 @@ export interface StreamSession {
 
 export interface PipeConnectOptions {
   onLog?: (msg: LogMessage) => void;
+  /** External storage config for resolving externalized batches. */
+  externalLocation?: import("../external.js").ExternalLocationConfig;
 }
 
 export interface SubprocessConnectOptions extends PipeConnectOptions {

package/src/constants.ts

@@ -15,8 +15,11 @@ export const REQUEST_ID_KEY = "vgi_rpc.request_id";
 
 export const PROTOCOL_NAME_KEY = "vgi_rpc.protocol_name";
 export const DESCRIBE_VERSION_KEY = "vgi_rpc.describe_version";
-export const DESCRIBE_VERSION = "2";
+export const DESCRIBE_VERSION = "3";
 
 export const DESCRIBE_METHOD_NAME = "__describe__";
 
 export const STATE_KEY = "vgi_rpc.stream_state#b64";
+
+export const LOCATION_KEY = "vgi_rpc.location";
+export const LOCATION_SHA256_KEY = "vgi_rpc.location.sha256";