@query-farm/vgi-rpc 0.6.4 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (160) hide show
  1. package/dist/access-log.d.ts +50 -0
  2. package/dist/access-log.d.ts.map +1 -0
  3. package/dist/arrow/impl-arrowjs/index.d.ts +96 -0
  4. package/dist/arrow/impl-arrowjs/index.d.ts.map +1 -0
  5. package/dist/arrow/impl-flechette/index.d.ts +102 -0
  6. package/dist/arrow/impl-flechette/index.d.ts.map +1 -0
  7. package/dist/arrow/impl-flechette/message-meta.d.ts +11 -0
  8. package/dist/arrow/impl-flechette/message-meta.d.ts.map +1 -0
  9. package/dist/arrow/index.d.ts +4 -0
  10. package/dist/arrow/index.d.ts.map +1 -0
  11. package/dist/arrow/predicates.d.ts +44 -0
  12. package/dist/arrow/predicates.d.ts.map +1 -0
  13. package/dist/arrow/types.d.ts +62 -0
  14. package/dist/arrow/types.d.ts.map +1 -0
  15. package/dist/client/capabilities.d.ts +25 -0
  16. package/dist/client/capabilities.d.ts.map +1 -0
  17. package/dist/client/connect.d.ts.map +1 -1
  18. package/dist/client/introspect.d.ts +7 -0
  19. package/dist/client/introspect.d.ts.map +1 -1
  20. package/dist/client/ipc.d.ts +8 -2
  21. package/dist/client/ipc.d.ts.map +1 -1
  22. package/dist/client/pipe.d.ts.map +1 -1
  23. package/dist/client/stream.d.ts +11 -2
  24. package/dist/client/stream.d.ts.map +1 -1
  25. package/dist/client/uploadUrl.d.ts +25 -0
  26. package/dist/client/uploadUrl.d.ts.map +1 -0
  27. package/dist/constants.d.ts +15 -1
  28. package/dist/constants.d.ts.map +1 -1
  29. package/dist/crypto.d.ts +22 -0
  30. package/dist/crypto.d.ts.map +1 -0
  31. package/dist/dispatch/describe.d.ts +10 -6
  32. package/dist/dispatch/describe.d.ts.map +1 -1
  33. package/dist/dispatch/stream.d.ts +2 -2
  34. package/dist/dispatch/stream.d.ts.map +1 -1
  35. package/dist/dispatch/unary.d.ts +2 -2
  36. package/dist/dispatch/unary.d.ts.map +1 -1
  37. package/dist/errors.d.ts +46 -0
  38. package/dist/errors.d.ts.map +1 -1
  39. package/dist/external.d.ts +25 -5
  40. package/dist/external.d.ts.map +1 -1
  41. package/dist/http/bearer.d.ts.map +1 -1
  42. package/dist/http/common.d.ts +42 -7
  43. package/dist/http/common.d.ts.map +1 -1
  44. package/dist/http/dispatch.d.ts +20 -2
  45. package/dist/http/dispatch.d.ts.map +1 -1
  46. package/dist/http/handler.d.ts.map +1 -1
  47. package/dist/http/index.d.ts +1 -0
  48. package/dist/http/index.d.ts.map +1 -1
  49. package/dist/http/mtls.d.ts +2 -1
  50. package/dist/http/mtls.d.ts.map +1 -1
  51. package/dist/http/oauth-pkce.d.ts +141 -0
  52. package/dist/http/oauth-pkce.d.ts.map +1 -0
  53. package/dist/http/pages.d.ts +3 -0
  54. package/dist/http/pages.d.ts.map +1 -1
  55. package/dist/http/sticky.d.ts +124 -0
  56. package/dist/http/sticky.d.ts.map +1 -0
  57. package/dist/http/token.d.ts +38 -12
  58. package/dist/http/token.d.ts.map +1 -1
  59. package/dist/http/types.d.ts +66 -5
  60. package/dist/http/types.d.ts.map +1 -1
  61. package/dist/index.d.ts +6 -4
  62. package/dist/index.d.ts.map +1 -1
  63. package/dist/index.js +1275 -3511
  64. package/dist/index.js.map +19 -37
  65. package/dist/launcher/hash.d.ts +22 -0
  66. package/dist/launcher/hash.d.ts.map +1 -0
  67. package/dist/launcher/index.d.ts +23 -0
  68. package/dist/launcher/index.d.ts.map +1 -0
  69. package/dist/launcher/launch.d.ts +27 -0
  70. package/dist/launcher/launch.d.ts.map +1 -0
  71. package/dist/launcher/lock.d.ts +19 -0
  72. package/dist/launcher/lock.d.ts.map +1 -0
  73. package/dist/launcher/serve-unix.d.ts +54 -0
  74. package/dist/launcher/serve-unix.d.ts.map +1 -0
  75. package/dist/launcher/state.d.ts +59 -0
  76. package/dist/launcher/state.d.ts.map +1 -0
  77. package/dist/otel.d.ts.map +1 -1
  78. package/dist/protocol.d.ts +16 -2
  79. package/dist/protocol.d.ts.map +1 -1
  80. package/dist/schema.d.ts +45 -18
  81. package/dist/schema.d.ts.map +1 -1
  82. package/dist/server.d.ts +23 -2
  83. package/dist/server.d.ts.map +1 -1
  84. package/dist/types.d.ts +216 -12
  85. package/dist/types.d.ts.map +1 -1
  86. package/dist/util/gzip.d.ts +10 -0
  87. package/dist/util/gzip.d.ts.map +1 -0
  88. package/dist/util/schema.d.ts +3 -15
  89. package/dist/util/schema.d.ts.map +1 -1
  90. package/dist/util/web-crypto.d.ts +22 -0
  91. package/dist/util/web-crypto.d.ts.map +1 -0
  92. package/dist/util/zstd.d.ts +26 -3
  93. package/dist/util/zstd.d.ts.map +1 -1
  94. package/dist/wire/opaque.d.ts +11 -0
  95. package/dist/wire/opaque.d.ts.map +1 -0
  96. package/dist/wire/reader.d.ts +5 -5
  97. package/dist/wire/reader.d.ts.map +1 -1
  98. package/dist/wire/request.d.ts +11 -3
  99. package/dist/wire/request.d.ts.map +1 -1
  100. package/dist/wire/response.d.ts +6 -6
  101. package/dist/wire/response.d.ts.map +1 -1
  102. package/dist/wire/writer.d.ts +49 -39
  103. package/dist/wire/writer.d.ts.map +1 -1
  104. package/package.json +24 -10
  105. package/src/access-log.ts +195 -0
  106. package/src/arrow/impl-arrowjs/index.ts +433 -0
  107. package/src/arrow/impl-flechette/index.ts +414 -0
  108. package/src/arrow/impl-flechette/message-meta.ts +174 -0
  109. package/src/arrow/index.ts +89 -0
  110. package/src/arrow/predicates.ts +56 -0
  111. package/src/arrow/types.ts +73 -0
  112. package/src/client/capabilities.ts +84 -0
  113. package/src/client/connect.ts +103 -26
  114. package/src/client/introspect.ts +60 -38
  115. package/src/client/ipc.ts +37 -27
  116. package/src/client/pipe.ts +12 -9
  117. package/src/client/stream.ts +34 -19
  118. package/src/client/uploadUrl.ts +169 -0
  119. package/src/constants.ts +18 -1
  120. package/src/crypto.ts +95 -0
  121. package/src/dispatch/describe.ts +146 -107
  122. package/src/dispatch/stream.ts +53 -24
  123. package/src/dispatch/unary.ts +5 -4
  124. package/src/errors.ts +76 -0
  125. package/src/external.ts +43 -29
  126. package/src/http/bearer.ts +2 -5
  127. package/src/http/common.ts +90 -23
  128. package/src/http/dispatch.ts +373 -46
  129. package/src/http/handler.ts +790 -68
  130. package/src/http/index.ts +1 -0
  131. package/src/http/mtls.ts +18 -3
  132. package/src/http/oauth-pkce.ts +1035 -0
  133. package/src/http/pages.ts +30 -15
  134. package/src/http/sticky.ts +429 -0
  135. package/src/http/token.ts +165 -75
  136. package/src/http/types.ts +67 -5
  137. package/src/index.ts +40 -1
  138. package/src/launcher/hash.ts +104 -0
  139. package/src/launcher/index.ts +35 -0
  140. package/src/launcher/launch.ts +284 -0
  141. package/src/launcher/lock.ts +171 -0
  142. package/src/launcher/serve-unix.ts +385 -0
  143. package/src/launcher/state.ts +245 -0
  144. package/src/otel.ts +39 -33
  145. package/src/protocol.ts +27 -3
  146. package/src/schema.ts +107 -56
  147. package/src/server.ts +196 -20
  148. package/src/types.ts +322 -18
  149. package/src/util/gzip.ts +63 -0
  150. package/src/util/schema.ts +4 -22
  151. package/src/util/web-crypto.ts +98 -0
  152. package/src/util/zstd.ts +133 -14
  153. package/src/wire/opaque.ts +37 -0
  154. package/src/wire/reader.ts +5 -4
  155. package/src/wire/request.ts +67 -8
  156. package/src/wire/response.ts +51 -85
  157. package/src/wire/writer.ts +165 -69
  158. package/dist/util/conform.d.ts +0 -18
  159. package/dist/util/conform.d.ts.map +0 -1
  160. package/src/util/conform.ts +0 -94
package/src/client/ipc.ts CHANGED
@@ -7,18 +7,17 @@ import {
7
7
  DataType,
8
8
  Float64,
9
9
  Int64,
10
- makeData,
11
- RecordBatch,
10
+ type RecordBatch,
12
11
  RecordBatchReader,
13
12
  type Schema,
14
- Struct,
15
13
  Utf8,
16
- vectorFromArray,
17
14
  } from "@query-farm/apache-arrow";
15
+ import { emptyBatchWithMetadata, singleRowBatchWithMetadata } from "#vgi-rpc-arrow";
18
16
  import {
19
17
  LOG_EXTRA_KEY,
20
18
  LOG_LEVEL_KEY,
21
19
  LOG_MESSAGE_KEY,
20
+ PROTOCOL_VERSION_KEY,
22
21
  REQUEST_VERSION,
23
22
  REQUEST_VERSION_KEY,
24
23
  RPC_METHOD_KEY,
@@ -79,38 +78,49 @@ function coerceForArrow(type: DataType, value: any): any {
79
78
 
80
79
  /**
81
80
  * Build a 1-row Arrow IPC request batch with method metadata.
81
+ *
82
+ * When `options.protocolVersion` is non-empty, the value is emitted as
83
+ * `vgi_rpc.protocol_version` so servers that declare a Protocol-level
84
+ * version validate the request at the dispatch boundary.
82
85
  */
83
- export function buildRequestIpc(schema: Schema, params: Record<string, any>, method: string): Uint8Array {
86
+ export function buildRequestIpc(
87
+ schema: Schema,
88
+ params: Record<string, any>,
89
+ method: string,
90
+ options?: { protocolVersion?: string },
91
+ ): Uint8Array {
84
92
  const metadata = new Map<string, string>();
85
93
  metadata.set(RPC_METHOD_KEY, method);
86
94
  metadata.set(REQUEST_VERSION_KEY, REQUEST_VERSION);
95
+ if (options?.protocolVersion) {
96
+ metadata.set(PROTOCOL_VERSION_KEY, options.protocolVersion);
97
+ }
87
98
 
99
+ // Build the batch through the impl-agnostic #vgi-rpc-arrow layer so this
100
+ // works under both backends. `buildRequestIpc` previously constructed an
101
+ // apache-arrow RecordBatch directly and handed it to `serializeIpcStream`,
102
+ // which then dispatched to the backend's `serializeBatches`. Under the
103
+ // browser/worker condition that backend is flechette, and flechette's
104
+ // `tablesToIPC` cannot read apache-arrow's RecordBatch shape — the cross-
105
+ // impl mixing was silently broken (no browser tests cover this path; see
106
+ // test/client/ipc-cross-impl.test.ts). The abstract helpers produce a
107
+ // 0-row metadata-bearing batch for the empty-schema case (which servers
108
+ // accept identically to a 1-row × 0-col batch) and a 1-row batch with
109
+ // coerced field values otherwise.
88
110
  if (schema.fields.length === 0) {
89
- const structType = new Struct(schema.fields);
90
- const data = makeData({
91
- type: structType,
92
- length: 1,
93
- children: [],
94
- nullCount: 0,
95
- });
96
- const batch = new RecordBatch(schema, data, metadata);
111
+ const batch = emptyBatchWithMetadata(schema, metadata);
97
112
  return serializeIpcStream(schema, [batch]);
98
113
  }
99
114
 
100
- const children = schema.fields.map((f) => {
101
- const val = coerceForArrow(f.type, params[f.name]);
102
- return vectorFromArray([val], f.type).data[0];
103
- });
104
-
105
- const structType = new Struct(schema.fields);
106
- const data = makeData({
107
- type: structType,
108
- length: 1,
109
- children,
110
- nullCount: 0,
111
- });
112
-
113
- const batch = new RecordBatch(schema, data, metadata);
115
+ const coerced: Record<string, any> = {};
116
+ for (const f of schema.fields) {
117
+ const raw = params[f.name];
118
+ // Missing values must be sent as null. arrow-js's typed-array builders
119
+ // throw "Invalid argument type in ToBigInt" when handed `undefined` for
120
+ // an Int64 column; null builds a proper validity bitmap entry instead.
121
+ coerced[f.name] = raw === undefined ? null : coerceForArrow(f.type, raw);
122
+ }
123
+ const batch = singleRowBatchWithMetadata(schema, coerced, metadata);
114
124
  return serializeIpcStream(schema, [batch]);
115
125
  }
116
126
 
@@ -125,17 +125,17 @@ export class PipeStreamSession implements StreamSession {
125
125
 
126
126
  if (batch.numRows === 0) {
127
127
  // Check for external location pointer batch
128
- if (isExternalLocationBatch(batch)) {
129
- return await resolveExternalLocation(batch, this._externalConfig);
128
+ if (isExternalLocationBatch(batch as any)) {
129
+ return (await resolveExternalLocation(batch as any, this._externalConfig)) as any;
130
130
  }
131
131
  // Check if it's a log/error batch. If so, dispatch and continue.
132
132
  // Otherwise it's a zero-row data batch — return it.
133
- if (dispatchLogOrError(batch, this._onLog)) {
133
+ if (dispatchLogOrError(batch as any, this._onLog)) {
134
134
  continue;
135
135
  }
136
136
  }
137
137
 
138
- return batch;
138
+ return batch as any;
139
139
  }
140
140
  }
141
141
 
@@ -389,6 +389,7 @@ export function pipeConnect(
389
389
  let readerPromise: Promise<IpcStreamReader> | null = null;
390
390
  let methodCache: Map<string, MethodInfo> | null = null;
391
391
  let protocolName = "";
392
+ let serverProtocolVersion = "";
392
393
  let _busy = false;
393
394
  let _drainPromise: Promise<void> | null = null;
394
395
  let closed = false;
@@ -458,8 +459,9 @@ export function pipeConnect(
458
459
  throw new Error("EOF reading __describe__ response");
459
460
  }
460
461
 
461
- const desc = await parseDescribeResponse(response.batches, onLog);
462
+ const desc = await parseDescribeResponse(response.batches as any, onLog);
462
463
  protocolName = desc.protocolName;
464
+ serverProtocolVersion = desc.protocolVersion;
463
465
  methodCache = new Map(desc.methods.map((m) => [m.name, m]));
464
466
  return methodCache;
465
467
  } finally {
@@ -483,7 +485,7 @@ export function pipeConnect(
483
485
  const fullParams = { ...(info.defaults ?? {}), ...(params ?? {}) };
484
486
 
485
487
  // Send request
486
- const body = buildRequestIpc(info.paramsSchema, fullParams, method);
488
+ const body = buildRequestIpc(info.paramsSchema, fullParams, method, { protocolVersion: serverProtocolVersion });
487
489
  writeFn(body);
488
490
 
489
491
  // Read response
@@ -494,7 +496,7 @@ export function pipeConnect(
494
496
 
495
497
  // Process batches: dispatch logs, resolve external pointers, find result
496
498
  let resultBatch: RecordBatch | null = null;
497
- for (let batch of response.batches) {
499
+ for (let batch of response.batches as any[]) {
498
500
  if (batch.numRows === 0) {
499
501
  if (isExternalLocationBatch(batch)) {
500
502
  batch = await resolveExternalLocation(batch, externalConfig);
@@ -537,7 +539,7 @@ export function pipeConnect(
537
539
  const fullParams = { ...(info.defaults ?? {}), ...(params ?? {}) };
538
540
 
539
541
  // Send init request (params as a complete IPC stream)
540
- const body = buildRequestIpc(info.paramsSchema, fullParams, method);
542
+ const body = buildRequestIpc(info.paramsSchema, fullParams, method, { protocolVersion: serverProtocolVersion });
541
543
  writeFn(body);
542
544
 
543
545
  // Read header if method has headerSchema
@@ -545,7 +547,7 @@ export function pipeConnect(
545
547
  if (info.headerSchema) {
546
548
  const headerStream = await r.readStream();
547
549
  if (headerStream) {
548
- for (const batch of headerStream.batches) {
550
+ for (const batch of headerStream.batches as any[]) {
549
551
  if (batch.numRows === 0) {
550
552
  dispatchLogOrError(batch, onLog);
551
553
  continue;
@@ -597,6 +599,7 @@ export function pipeConnect(
597
599
  const methods = await ensureMethodCache();
598
600
  return {
599
601
  protocolName,
602
+ protocolVersion: serverProtocolVersion,
600
603
  methods: [...methods.values()],
601
604
  };
602
605
  },
@@ -9,8 +9,15 @@ import { ARROW_CONTENT_TYPE, serializeIpcStream } from "../http/common.js";
9
9
  import { dispatchLogOrError, extractBatchRows, inferArrowType, readResponseBatches } from "./ipc.js";
10
10
  import type { LogMessage, StreamSession } from "./types.js";
11
11
 
12
- type CompressFn = (data: Uint8Array, level: number) => Uint8Array;
13
- type DecompressFn = (data: Uint8Array) => Uint8Array;
12
+ type CompressFn = (data: Uint8Array, level: number) => Promise<Uint8Array>;
13
+ type DecompressFn = (data: Uint8Array) => Promise<Uint8Array>;
14
+
15
+ /**
16
+ * Posts an Arrow IPC request body to *url*, transparently handling
17
+ * client-vended request externalization. Provided by the parent connection
18
+ * so a single capability cache can drive both unary and stream call paths.
19
+ */
20
+ export type PostFn = (url: string, body: Uint8Array) => Promise<Response>;
14
21
 
15
22
  export class HttpStreamSession implements StreamSession {
16
23
  private _baseUrl: string;
@@ -28,6 +35,7 @@ export class HttpStreamSession implements StreamSession {
28
35
  private _decompressFn?: DecompressFn;
29
36
  private _authorization?: string;
30
37
  private _externalConfig?: ExternalLocationConfig;
38
+ private _postFn?: PostFn;
31
39
 
32
40
  constructor(opts: {
33
41
  baseUrl: string;
@@ -45,6 +53,7 @@ export class HttpStreamSession implements StreamSession {
45
53
  decompressFn?: DecompressFn;
46
54
  authorization?: string;
47
55
  externalConfig?: ExternalLocationConfig;
56
+ postFn?: PostFn;
48
57
  }) {
49
58
  this._baseUrl = opts.baseUrl;
50
59
  this._prefix = opts.prefix;
@@ -61,6 +70,16 @@ export class HttpStreamSession implements StreamSession {
61
70
  this._decompressFn = opts.decompressFn;
62
71
  this._authorization = opts.authorization;
63
72
  this._externalConfig = opts.externalConfig;
73
+ this._postFn = opts.postFn;
74
+ }
75
+
76
+ private async _post(url: string, body: Uint8Array): Promise<Response> {
77
+ if (this._postFn) return this._postFn(url, body);
78
+ return fetch(url, {
79
+ method: "POST",
80
+ headers: this._buildHeaders(),
81
+ body: (await this._prepareBody(body)) as unknown as BodyInit,
82
+ });
64
83
  }
65
84
 
66
85
  get header(): Record<string, any> | null {
@@ -71,8 +90,10 @@ export class HttpStreamSession implements StreamSession {
71
90
  const headers: Record<string, string> = {
72
91
  "Content-Type": ARROW_CONTENT_TYPE,
73
92
  };
74
- if (this._compressionLevel != null) {
93
+ if (this._compressionLevel != null && this._compressFn) {
75
94
  headers["Content-Encoding"] = "zstd";
95
+ }
96
+ if (this._compressionLevel != null && this._decompressFn) {
76
97
  headers["Accept-Encoding"] = "zstd";
77
98
  }
78
99
  if (this._authorization) {
@@ -81,9 +102,9 @@ export class HttpStreamSession implements StreamSession {
81
102
  return headers;
82
103
  }
83
104
 
84
- private _prepareBody(content: Uint8Array): Uint8Array {
105
+ private async _prepareBody(content: Uint8Array): Promise<Uint8Array> {
85
106
  if (this._compressionLevel != null && this._compressFn) {
86
- return this._compressFn(content, this._compressionLevel);
107
+ return await this._compressFn(content, this._compressionLevel);
87
108
  }
88
109
  return content;
89
110
  }
@@ -91,7 +112,7 @@ export class HttpStreamSession implements StreamSession {
91
112
  private async _readResponse(resp: Response): Promise<Uint8Array<ArrayBuffer>> {
92
113
  let body = new Uint8Array(await resp.arrayBuffer());
93
114
  if (resp.headers.get("Content-Encoding") === "zstd" && this._decompressFn) {
94
- body = new Uint8Array(this._decompressFn(body));
115
+ body = new Uint8Array(await this._decompressFn(body));
95
116
  }
96
117
  return body;
97
118
  }
@@ -159,11 +180,7 @@ export class HttpStreamSession implements StreamSession {
159
180
 
160
181
  private async _doExchange(schema: Schema, batches: RecordBatch[]): Promise<Record<string, any>[]> {
161
182
  const body = serializeIpcStream(schema, batches);
162
- const resp = await fetch(`${this._baseUrl}${this._prefix}/${this._method}/exchange`, {
163
- method: "POST",
164
- headers: this._buildHeaders(),
165
- body: this._prepareBody(body) as unknown as BodyInit,
166
- });
183
+ const resp = await this._post(`${this._baseUrl}${this._prefix}/${this._method}/exchange`, body);
167
184
  if (resp.status === 401) {
168
185
  throw new RpcError("AuthenticationError", "Authentication required", "");
169
186
  }
@@ -218,7 +235,7 @@ export class HttpStreamSession implements StreamSession {
218
235
  for (let batch of this._pendingBatches) {
219
236
  if (batch.numRows === 0) {
220
237
  if (isExternalLocationBatch(batch)) {
221
- batch = await resolveExternalLocation(batch, this._externalConfig);
238
+ batch = (await resolveExternalLocation(batch as any, this._externalConfig)) as any;
222
239
  } else {
223
240
  dispatchLogOrError(batch, this._onLog);
224
241
  continue;
@@ -233,7 +250,9 @@ export class HttpStreamSession implements StreamSession {
233
250
 
234
251
  // Follow continuation tokens
235
252
  while (true) {
236
- const responseBody = await this._sendContinuation(this._stateToken);
253
+ const stateToken = this._stateToken;
254
+ if (stateToken === null) return;
255
+ const responseBody = await this._sendContinuation(stateToken);
237
256
  const { batches } = await readResponseBatches(responseBody);
238
257
 
239
258
  let gotContinuation = false;
@@ -248,7 +267,7 @@ export class HttpStreamSession implements StreamSession {
248
267
  }
249
268
  // Check for external location pointer
250
269
  if (isExternalLocationBatch(batch)) {
251
- batch = await resolveExternalLocation(batch, this._externalConfig);
270
+ batch = (await resolveExternalLocation(batch as any, this._externalConfig)) as any;
252
271
  } else {
253
272
  // Log/error batch
254
273
  dispatchLogOrError(batch, this._onLog);
@@ -278,11 +297,7 @@ export class HttpStreamSession implements StreamSession {
278
297
  const batch = new RecordBatch(emptySchema, data, metadata);
279
298
  const body = serializeIpcStream(emptySchema, [batch]);
280
299
 
281
- const resp = await fetch(`${this._baseUrl}${this._prefix}/${this._method}/exchange`, {
282
- method: "POST",
283
- headers: this._buildHeaders(),
284
- body: this._prepareBody(body) as unknown as BodyInit,
285
- });
300
+ const resp = await this._post(`${this._baseUrl}${this._prefix}/${this._method}/exchange`, body);
286
301
  if (resp.status === 401) {
287
302
  throw new RpcError("AuthenticationError", "Authentication required", "");
288
303
  }
@@ -0,0 +1,169 @@
1
+ // © Copyright 2025-2026, Query.Farm LLC - https://query.farm
2
+ // SPDX-License-Identifier: Apache-2.0
3
+
4
+ /**
5
+ * Client-side request-body externalization.
6
+ *
7
+ * When a request body would exceed the server-advertised `maxRequestBytes`,
8
+ * we (1) call `{prefix}/__upload_url__/init` to get a pre-signed upload URL,
9
+ * (2) PUT the body to that URL, and (3) replace the inline body with a
10
+ * zero-row "pointer" IPC stream that carries the original RPC dispatch
11
+ * metadata plus `vgi_rpc.location: <download-url>`. The server then resolves
12
+ * the pointer and dispatches normally.
13
+ *
14
+ * Mirrors Python's `_externalize_via_upload_url()` and `request_upload_urls()`.
15
+ */
16
+
17
+ import { Field, Int64, RecordBatchReader, Schema } from "@query-farm/apache-arrow";
18
+ import { REQUEST_VERSION, REQUEST_VERSION_KEY, RPC_METHOD_KEY } from "../constants.js";
19
+ import { RpcError } from "../errors.js";
20
+ import { makeExternalLocationBatch } from "../external.js";
21
+ import { ARROW_CONTENT_TYPE, serializeIpcStream } from "../http/common.js";
22
+ import { buildRequestIpc } from "./ipc.js";
23
+
24
+ const UPLOAD_URL_METHOD = "__upload_url__";
25
+ const UPLOAD_URL_PARAMS_SCHEMA = new Schema([new Field("count", new Int64(), false)]);
26
+
27
+ export interface UploadUrlPair {
28
+ uploadUrl: string;
29
+ downloadUrl: string;
30
+ expiresAt: Date;
31
+ }
32
+
33
+ /**
34
+ * POST `__upload_url__/init` and return the requested number of pre-signed
35
+ * URL pairs. Server must have an `uploadUrlProvider` configured; otherwise
36
+ * the route returns 404 and we surface that as `RpcError("NotSupported")`.
37
+ */
38
+ export async function requestUploadUrls(
39
+ baseUrl: string,
40
+ prefix: string,
41
+ count: number,
42
+ authorization?: string,
43
+ ): Promise<UploadUrlPair[]> {
44
+ const body = buildRequestIpc(UPLOAD_URL_PARAMS_SCHEMA, { count: BigInt(count) }, UPLOAD_URL_METHOD);
45
+ const headers: Record<string, string> = { "Content-Type": ARROW_CONTENT_TYPE };
46
+ if (authorization) headers.Authorization = authorization;
47
+
48
+ const resp = await fetch(`${baseUrl}${prefix}/${UPLOAD_URL_METHOD}/init`, {
49
+ method: "POST",
50
+ headers,
51
+ body: body as unknown as BodyInit,
52
+ });
53
+ if (resp.status === 404) {
54
+ throw new RpcError("NotSupported", "Server does not support upload URLs", "");
55
+ }
56
+ if (resp.status === 401) {
57
+ throw new RpcError("AuthenticationError", "Authentication required", "");
58
+ }
59
+ if (!resp.ok) {
60
+ throw new RpcError("HttpError", `__upload_url__/init failed: HTTP ${resp.status}`, "");
61
+ }
62
+
63
+ const respBody = new Uint8Array(await resp.arrayBuffer());
64
+ const reader = await RecordBatchReader.from(respBody);
65
+ await reader.open();
66
+
67
+ const pairs: UploadUrlPair[] = [];
68
+ for (const batch of reader.readAll()) {
69
+ if (batch.numRows === 0) continue;
70
+ for (let r = 0; r < batch.numRows; r++) {
71
+ const uploadUrl = batch.getChildAt(0)?.get(r) as string;
72
+ const downloadUrl = batch.getChildAt(1)?.get(r) as string;
73
+ const expiresRaw = batch.getChildAt(2)?.get(r);
74
+ // Timestamp(us) → either a Date, a number (ms), or a bigint (us)
75
+ let expiresAt: Date;
76
+ if (expiresRaw instanceof Date) {
77
+ expiresAt = expiresRaw;
78
+ } else if (typeof expiresRaw === "bigint") {
79
+ expiresAt = new Date(Number(expiresRaw / 1000n));
80
+ } else if (typeof expiresRaw === "number") {
81
+ expiresAt = new Date(expiresRaw);
82
+ } else {
83
+ expiresAt = new Date();
84
+ }
85
+ pairs.push({ uploadUrl, downloadUrl, expiresAt });
86
+ }
87
+ }
88
+
89
+ if (pairs.length === 0) {
90
+ throw new RpcError("ProtocolError", "Server returned no upload URLs", "");
91
+ }
92
+ return pairs;
93
+ }
94
+
95
+ /**
96
+ * Build the externalized pointer body to send in place of *originalBody*.
97
+ *
98
+ * The pointer is a zero-row IPC stream whose first batch carries:
99
+ * - same schema as the original request batch
100
+ * - merged custom_metadata: original RPC dispatch keys + `vgi_rpc.location`
101
+ *
102
+ * The server's request reader honours the dispatch keys for routing, then
103
+ * resolves the pointer to fetch the inline batch for parameter extraction.
104
+ */
105
+ async function buildPointerRequestBody(originalBody: Uint8Array, downloadUrl: string): Promise<Uint8Array> {
106
+ const reader = await RecordBatchReader.from(originalBody);
107
+ await reader.open();
108
+ const schema = reader.schema;
109
+ if (!schema) {
110
+ throw new RpcError("ProtocolError", "Original request body has no schema", "");
111
+ }
112
+ const batches = reader.readAll();
113
+ if (batches.length === 0) {
114
+ throw new RpcError("ProtocolError", "Original request body has no batches", "");
115
+ }
116
+ const original = batches[0];
117
+ const originalMeta = original.metadata ?? new Map<string, string>();
118
+
119
+ const pointer = makeExternalLocationBatch(schema, downloadUrl);
120
+ const merged = new Map<string, string>(pointer.metadata ?? new Map());
121
+ // Preserve the original RPC dispatch metadata so the server can route.
122
+ const method = originalMeta.get(RPC_METHOD_KEY);
123
+ const version = originalMeta.get(REQUEST_VERSION_KEY) ?? REQUEST_VERSION;
124
+ if (method) merged.set(RPC_METHOD_KEY, method);
125
+ merged.set(REQUEST_VERSION_KEY, version);
126
+ // Carry over any other keys (request id, state token for exchange, etc).
127
+ for (const [k, v] of originalMeta) {
128
+ if (!merged.has(k)) merged.set(k, v);
129
+ }
130
+
131
+ // Re-emit the pointer batch with merged metadata.
132
+ const { RecordBatch } = await import("@query-farm/apache-arrow");
133
+ const pointerWithMeta = new RecordBatch(schema as any, (pointer as any).data, merged);
134
+ return serializeIpcStream(schema, [pointerWithMeta]);
135
+ }
136
+
137
+ export interface ExternalizeOptions {
138
+ baseUrl: string;
139
+ prefix: string;
140
+ authorization?: string;
141
+ /** Optional per-URL validator; throw to reject. */
142
+ urlValidator?: ((url: string) => void) | null;
143
+ }
144
+
145
+ /**
146
+ * Upload *body* via a server-vended URL and return the pointer-batch body
147
+ * that should be sent in place of the original. Throws if the server does
148
+ * not advertise upload-URL support or the upload fails.
149
+ */
150
+ export async function externalizeRequestBody(body: Uint8Array, opts: ExternalizeOptions): Promise<Uint8Array> {
151
+ const pairs = await requestUploadUrls(opts.baseUrl, opts.prefix, 1, opts.authorization);
152
+ const pair = pairs[0];
153
+
154
+ if (opts.urlValidator) {
155
+ opts.urlValidator(pair.uploadUrl);
156
+ opts.urlValidator(pair.downloadUrl);
157
+ }
158
+
159
+ const putResp = await fetch(pair.uploadUrl, {
160
+ method: "PUT",
161
+ headers: { "Content-Type": ARROW_CONTENT_TYPE },
162
+ body: body as unknown as BodyInit,
163
+ });
164
+ if (!putResp.ok) {
165
+ throw new RpcError("ExternalUploadFailed", `PUT to upload URL failed: HTTP ${putResp.status}`, "");
166
+ }
167
+
168
+ return buildPointerRequestBody(body, pair.downloadUrl);
169
+ }
package/src/constants.ts CHANGED
@@ -15,11 +15,28 @@ export const REQUEST_ID_KEY = "vgi_rpc.request_id";
15
15
 
16
16
  export const PROTOCOL_NAME_KEY = "vgi_rpc.protocol_name";
17
17
  export const DESCRIBE_VERSION_KEY = "vgi_rpc.describe_version";
18
- export const DESCRIBE_VERSION = "3";
18
+ export const PROTOCOL_HASH_KEY = "vgi_rpc.protocol_hash";
19
+ export const DESCRIBE_VERSION = "4";
20
+
21
+ /** Application protocol surface version. Carried on every request batch from
22
+ * a client bound to a Protocol that declares `protocolVersion`; also emitted
23
+ * in the __describe__ response metadata. Format: canonical semver
24
+ * MAJOR.MINOR.PATCH. Enforced at the dispatch boundary on the server: exact
25
+ * major+minor match required, patch ignored. Distinct from `REQUEST_VERSION`
26
+ * (wire framing). Mirrors Python's `PROTOCOL_VERSION_KEY`. */
27
+ export const PROTOCOL_VERSION_KEY = "vgi_rpc.protocol_version";
19
28
 
20
29
  export const DESCRIBE_METHOD_NAME = "__describe__";
21
30
 
22
31
  export const STATE_KEY = "vgi_rpc.stream_state#b64";
32
+ export const CANCEL_KEY = "vgi_rpc.cancel";
23
33
 
24
34
  export const LOCATION_KEY = "vgi_rpc.location";
25
35
  export const LOCATION_SHA256_KEY = "vgi_rpc.location.sha256";
36
+
37
+ export const RPC_ERROR_HEADER = "X-VGI-RPC-Error";
38
+
39
+ /** Top-level metadata key on an EXCEPTION batch identifying the error category.
40
+ * Hoisted by `buildErrorBatch` when the thrown error has a static or instance
41
+ * `errorKind` property. Mirrors Python's `vgi_rpc.metadata.ERROR_KIND_KEY`. */
42
+ export const ERROR_KIND_KEY = "vgi_rpc.error_kind";
package/src/crypto.ts ADDED
@@ -0,0 +1,95 @@
1
+ // © Copyright 2025-2026, Query.Farm LLC - https://query.farm
2
+ // SPDX-License-Identifier: Apache-2.0
3
+
4
+ /**
5
+ * Generic AEAD seal/open primitives shared by stream-state and sticky-session
6
+ * tokens. Mirrors Python's `vgi_rpc.crypto` module: a tiny envelope around
7
+ * XChaCha20-Poly1305 with a leading version byte so future format bumps stay
8
+ * self-describing.
9
+ *
10
+ * Wire format (returned by {@link sealBytes}, accepted by {@link openBytes}):
11
+ *
12
+ * ```
13
+ * [1B version (1..255)]
14
+ * [24B nonce (XChaCha20-Poly1305, random per envelope)]
15
+ * [.. ciphertext + 16B Poly1305 tag]
16
+ * ```
17
+ *
18
+ * The plaintext frame is fully up to the caller — only the version + nonce +
19
+ * tag overhead is fixed. AAD (`aad`) is bound at the crypto layer so an
20
+ * envelope sealed for one identity fails decryption when presented by another.
21
+ */
22
+
23
+ import { xchacha20poly1305 } from "@noble/ciphers/chacha.js";
24
+ import { randomBytes } from "./util/web-crypto.js";
25
+
26
+ const NONCE_LEN = 24;
27
+ const TAG_LEN = 16;
28
+ const VERSION_LEN = 1;
29
+ const MIN_ENVELOPE_LEN = VERSION_LEN + NONCE_LEN + TAG_LEN;
30
+
31
+ /** Thrown by {@link openBytes} for any envelope it cannot open — malformed,
32
+ * tampered, wrong key, wrong AAD, wrong version, truncated, all surface the
33
+ * same way so callers cannot distinguish failure modes via error content. */
34
+ export class SealError extends Error {
35
+ constructor(message: string) {
36
+ super(message);
37
+ this.name = "SealError";
38
+ }
39
+ }
40
+
41
+ /** Normalise a key to 32 bytes by SHA-256 hashing when it isn't already 32B.
42
+ * Mirrors Python's `normalize_key` so any callers can pass operator-provided
43
+ * keys of arbitrary length without a separate stretching step. */
44
+ export async function normalizeKey(key: Uint8Array): Promise<Uint8Array> {
45
+ if (key.length === 32) return key;
46
+ const digest = await crypto.subtle.digest("SHA-256", key as BufferSource);
47
+ return new Uint8Array(digest);
48
+ }
49
+
50
+ export interface SealOptions {
51
+ /** Associated data bound at the crypto layer — typically a principal or
52
+ * request-scoped identifier. Must match between seal and open. */
53
+ aad: Uint8Array;
54
+ /** Envelope version byte. Defaults to 1; carry through to {@link openBytes}. */
55
+ version?: number;
56
+ }
57
+
58
+ /** Seal `plaintext` under `key` with AEAD, returning the wire envelope. */
59
+ export function sealBytes(plaintext: Uint8Array, key: Uint8Array, opts: SealOptions): Uint8Array {
60
+ if (key.length !== 32) {
61
+ throw new Error("AEAD key must be 32 bytes — call normalizeKey() first");
62
+ }
63
+ const version = opts.version ?? 1;
64
+ if (version < 1 || version > 255) {
65
+ throw new Error(`AEAD envelope version must fit in one byte; got ${version}`);
66
+ }
67
+ const nonce = randomBytes(NONCE_LEN);
68
+ const ciphertext = xchacha20poly1305(key, nonce, opts.aad as Uint8Array).encrypt(plaintext);
69
+ const wire = new Uint8Array(VERSION_LEN + NONCE_LEN + ciphertext.length);
70
+ wire[0] = version;
71
+ wire.set(nonce, VERSION_LEN);
72
+ wire.set(ciphertext, VERSION_LEN + NONCE_LEN);
73
+ return wire;
74
+ }
75
+
76
+ /** Open and verify an envelope produced by {@link sealBytes}. */
77
+ export function openBytes(envelope: Uint8Array, key: Uint8Array, opts: SealOptions): Uint8Array {
78
+ if (key.length !== 32) {
79
+ throw new Error("AEAD key must be 32 bytes — call normalizeKey() first");
80
+ }
81
+ if (envelope.length < MIN_ENVELOPE_LEN) {
82
+ throw new SealError("envelope truncated");
83
+ }
84
+ const expectedVersion = opts.version ?? 1;
85
+ if (envelope[0] !== expectedVersion) {
86
+ throw new SealError(`unsupported envelope version: ${envelope[0]}`);
87
+ }
88
+ const nonce = envelope.subarray(VERSION_LEN, VERSION_LEN + NONCE_LEN);
89
+ const ciphertext = envelope.subarray(VERSION_LEN + NONCE_LEN);
90
+ try {
91
+ return xchacha20poly1305(key, nonce, opts.aad as Uint8Array).decrypt(ciphertext);
92
+ } catch {
93
+ throw new SealError("envelope verification failed");
94
+ }
95
+ }