@query-farm/vgi-rpc 0.6.3 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/access-log.d.ts +50 -0
- package/dist/access-log.d.ts.map +1 -0
- package/dist/arrow/impl-arrowjs/index.d.ts +96 -0
- package/dist/arrow/impl-arrowjs/index.d.ts.map +1 -0
- package/dist/arrow/impl-flechette/index.d.ts +102 -0
- package/dist/arrow/impl-flechette/index.d.ts.map +1 -0
- package/dist/arrow/impl-flechette/message-meta.d.ts +11 -0
- package/dist/arrow/impl-flechette/message-meta.d.ts.map +1 -0
- package/dist/arrow/index.d.ts +4 -0
- package/dist/arrow/index.d.ts.map +1 -0
- package/dist/arrow/predicates.d.ts +44 -0
- package/dist/arrow/predicates.d.ts.map +1 -0
- package/dist/arrow/types.d.ts +62 -0
- package/dist/arrow/types.d.ts.map +1 -0
- package/dist/client/capabilities.d.ts +25 -0
- package/dist/client/capabilities.d.ts.map +1 -0
- package/dist/client/connect.d.ts.map +1 -1
- package/dist/client/introspect.d.ts +7 -0
- package/dist/client/introspect.d.ts.map +1 -1
- package/dist/client/ipc.d.ts +8 -2
- package/dist/client/ipc.d.ts.map +1 -1
- package/dist/client/pipe.d.ts.map +1 -1
- package/dist/client/stream.d.ts +11 -2
- package/dist/client/stream.d.ts.map +1 -1
- package/dist/client/uploadUrl.d.ts +25 -0
- package/dist/client/uploadUrl.d.ts.map +1 -0
- package/dist/constants.d.ts +15 -1
- package/dist/constants.d.ts.map +1 -1
- package/dist/crypto.d.ts +22 -0
- package/dist/crypto.d.ts.map +1 -0
- package/dist/dispatch/describe.d.ts +10 -6
- package/dist/dispatch/describe.d.ts.map +1 -1
- package/dist/dispatch/stream.d.ts +2 -2
- package/dist/dispatch/stream.d.ts.map +1 -1
- package/dist/dispatch/unary.d.ts +2 -2
- package/dist/dispatch/unary.d.ts.map +1 -1
- package/dist/errors.d.ts +46 -0
- package/dist/errors.d.ts.map +1 -1
- package/dist/external.d.ts +25 -5
- package/dist/external.d.ts.map +1 -1
- package/dist/http/bearer.d.ts.map +1 -1
- package/dist/http/common.d.ts +42 -7
- package/dist/http/common.d.ts.map +1 -1
- package/dist/http/dispatch.d.ts +20 -2
- package/dist/http/dispatch.d.ts.map +1 -1
- package/dist/http/handler.d.ts.map +1 -1
- package/dist/http/index.d.ts +1 -0
- package/dist/http/index.d.ts.map +1 -1
- package/dist/http/mtls.d.ts +2 -1
- package/dist/http/mtls.d.ts.map +1 -1
- package/dist/http/oauth-pkce.d.ts +141 -0
- package/dist/http/oauth-pkce.d.ts.map +1 -0
- package/dist/http/pages.d.ts +3 -0
- package/dist/http/pages.d.ts.map +1 -1
- package/dist/http/sticky.d.ts +124 -0
- package/dist/http/sticky.d.ts.map +1 -0
- package/dist/http/token.d.ts +38 -12
- package/dist/http/token.d.ts.map +1 -1
- package/dist/http/types.d.ts +68 -5
- package/dist/http/types.d.ts.map +1 -1
- package/dist/index.d.ts +6 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1275 -3507
- package/dist/index.js.map +19 -37
- package/dist/launcher/hash.d.ts +22 -0
- package/dist/launcher/hash.d.ts.map +1 -0
- package/dist/launcher/index.d.ts +23 -0
- package/dist/launcher/index.d.ts.map +1 -0
- package/dist/launcher/launch.d.ts +27 -0
- package/dist/launcher/launch.d.ts.map +1 -0
- package/dist/launcher/lock.d.ts +19 -0
- package/dist/launcher/lock.d.ts.map +1 -0
- package/dist/launcher/serve-unix.d.ts +54 -0
- package/dist/launcher/serve-unix.d.ts.map +1 -0
- package/dist/launcher/state.d.ts +59 -0
- package/dist/launcher/state.d.ts.map +1 -0
- package/dist/otel.d.ts.map +1 -1
- package/dist/protocol.d.ts +16 -2
- package/dist/protocol.d.ts.map +1 -1
- package/dist/schema.d.ts +45 -18
- package/dist/schema.d.ts.map +1 -1
- package/dist/server.d.ts +23 -2
- package/dist/server.d.ts.map +1 -1
- package/dist/types.d.ts +216 -12
- package/dist/types.d.ts.map +1 -1
- package/dist/util/gzip.d.ts +10 -0
- package/dist/util/gzip.d.ts.map +1 -0
- package/dist/util/schema.d.ts +3 -15
- package/dist/util/schema.d.ts.map +1 -1
- package/dist/util/web-crypto.d.ts +22 -0
- package/dist/util/web-crypto.d.ts.map +1 -0
- package/dist/util/zstd.d.ts +26 -3
- package/dist/util/zstd.d.ts.map +1 -1
- package/dist/wire/opaque.d.ts +11 -0
- package/dist/wire/opaque.d.ts.map +1 -0
- package/dist/wire/reader.d.ts +5 -5
- package/dist/wire/reader.d.ts.map +1 -1
- package/dist/wire/request.d.ts +11 -3
- package/dist/wire/request.d.ts.map +1 -1
- package/dist/wire/response.d.ts +6 -6
- package/dist/wire/response.d.ts.map +1 -1
- package/dist/wire/writer.d.ts +49 -39
- package/dist/wire/writer.d.ts.map +1 -1
- package/package.json +24 -10
- package/src/access-log.ts +195 -0
- package/src/arrow/impl-arrowjs/index.ts +433 -0
- package/src/arrow/impl-flechette/index.ts +414 -0
- package/src/arrow/impl-flechette/message-meta.ts +174 -0
- package/src/arrow/index.ts +89 -0
- package/src/arrow/predicates.ts +56 -0
- package/src/arrow/types.ts +73 -0
- package/src/client/capabilities.ts +84 -0
- package/src/client/connect.ts +103 -26
- package/src/client/introspect.ts +60 -38
- package/src/client/ipc.ts +37 -27
- package/src/client/pipe.ts +12 -9
- package/src/client/stream.ts +34 -19
- package/src/client/uploadUrl.ts +169 -0
- package/src/constants.ts +18 -1
- package/src/crypto.ts +95 -0
- package/src/dispatch/describe.ts +146 -107
- package/src/dispatch/stream.ts +53 -24
- package/src/dispatch/unary.ts +5 -4
- package/src/errors.ts +76 -0
- package/src/external.ts +43 -29
- package/src/http/bearer.ts +2 -5
- package/src/http/common.ts +90 -23
- package/src/http/dispatch.ts +373 -46
- package/src/http/handler.ts +794 -68
- package/src/http/index.ts +1 -0
- package/src/http/mtls.ts +18 -3
- package/src/http/oauth-pkce.ts +1035 -0
- package/src/http/pages.ts +30 -15
- package/src/http/sticky.ts +429 -0
- package/src/http/token.ts +165 -75
- package/src/http/types.ts +69 -5
- package/src/index.ts +40 -1
- package/src/launcher/hash.ts +104 -0
- package/src/launcher/index.ts +35 -0
- package/src/launcher/launch.ts +284 -0
- package/src/launcher/lock.ts +171 -0
- package/src/launcher/serve-unix.ts +385 -0
- package/src/launcher/state.ts +245 -0
- package/src/otel.ts +39 -33
- package/src/protocol.ts +27 -3
- package/src/schema.ts +107 -56
- package/src/server.ts +196 -20
- package/src/types.ts +322 -18
- package/src/util/gzip.ts +63 -0
- package/src/util/schema.ts +4 -22
- package/src/util/web-crypto.ts +98 -0
- package/src/util/zstd.ts +133 -14
- package/src/wire/opaque.ts +37 -0
- package/src/wire/reader.ts +5 -4
- package/src/wire/request.ts +67 -8
- package/src/wire/response.ts +51 -85
- package/src/wire/writer.ts +165 -69
- package/dist/util/conform.d.ts +0 -18
- package/dist/util/conform.d.ts.map +0 -1
- package/src/util/conform.ts +0 -94
package/src/client/ipc.ts
CHANGED
|
@@ -7,18 +7,17 @@ import {
|
|
|
7
7
|
DataType,
|
|
8
8
|
Float64,
|
|
9
9
|
Int64,
|
|
10
|
-
|
|
11
|
-
RecordBatch,
|
|
10
|
+
type RecordBatch,
|
|
12
11
|
RecordBatchReader,
|
|
13
12
|
type Schema,
|
|
14
|
-
Struct,
|
|
15
13
|
Utf8,
|
|
16
|
-
vectorFromArray,
|
|
17
14
|
} from "@query-farm/apache-arrow";
|
|
15
|
+
import { emptyBatchWithMetadata, singleRowBatchWithMetadata } from "#vgi-rpc-arrow";
|
|
18
16
|
import {
|
|
19
17
|
LOG_EXTRA_KEY,
|
|
20
18
|
LOG_LEVEL_KEY,
|
|
21
19
|
LOG_MESSAGE_KEY,
|
|
20
|
+
PROTOCOL_VERSION_KEY,
|
|
22
21
|
REQUEST_VERSION,
|
|
23
22
|
REQUEST_VERSION_KEY,
|
|
24
23
|
RPC_METHOD_KEY,
|
|
@@ -79,38 +78,49 @@ function coerceForArrow(type: DataType, value: any): any {
|
|
|
79
78
|
|
|
80
79
|
/**
|
|
81
80
|
* Build a 1-row Arrow IPC request batch with method metadata.
|
|
81
|
+
*
|
|
82
|
+
* When `options.protocolVersion` is non-empty, the value is emitted as
|
|
83
|
+
* `vgi_rpc.protocol_version` so servers that declare a Protocol-level
|
|
84
|
+
* version validate the request at the dispatch boundary.
|
|
82
85
|
*/
|
|
83
|
-
export function buildRequestIpc(
|
|
86
|
+
export function buildRequestIpc(
|
|
87
|
+
schema: Schema,
|
|
88
|
+
params: Record<string, any>,
|
|
89
|
+
method: string,
|
|
90
|
+
options?: { protocolVersion?: string },
|
|
91
|
+
): Uint8Array {
|
|
84
92
|
const metadata = new Map<string, string>();
|
|
85
93
|
metadata.set(RPC_METHOD_KEY, method);
|
|
86
94
|
metadata.set(REQUEST_VERSION_KEY, REQUEST_VERSION);
|
|
95
|
+
if (options?.protocolVersion) {
|
|
96
|
+
metadata.set(PROTOCOL_VERSION_KEY, options.protocolVersion);
|
|
97
|
+
}
|
|
87
98
|
|
|
99
|
+
// Build the batch through the impl-agnostic #vgi-rpc-arrow layer so this
|
|
100
|
+
// works under both backends. `buildRequestIpc` previously constructed an
|
|
101
|
+
// apache-arrow RecordBatch directly and handed it to `serializeIpcStream`,
|
|
102
|
+
// which then dispatched to the backend's `serializeBatches`. Under the
|
|
103
|
+
// browser/worker condition that backend is flechette, and flechette's
|
|
104
|
+
// `tablesToIPC` cannot read apache-arrow's RecordBatch shape — the cross-
|
|
105
|
+
// impl mixing was silently broken (no browser tests cover this path; see
|
|
106
|
+
// test/client/ipc-cross-impl.test.ts). The abstract helpers produce a
|
|
107
|
+
// 0-row metadata-bearing batch for the empty-schema case (which servers
|
|
108
|
+
// accept identically to a 1-row × 0-col batch) and a 1-row batch with
|
|
109
|
+
// coerced field values otherwise.
|
|
88
110
|
if (schema.fields.length === 0) {
|
|
89
|
-
const
|
|
90
|
-
const data = makeData({
|
|
91
|
-
type: structType,
|
|
92
|
-
length: 1,
|
|
93
|
-
children: [],
|
|
94
|
-
nullCount: 0,
|
|
95
|
-
});
|
|
96
|
-
const batch = new RecordBatch(schema, data, metadata);
|
|
111
|
+
const batch = emptyBatchWithMetadata(schema, metadata);
|
|
97
112
|
return serializeIpcStream(schema, [batch]);
|
|
98
113
|
}
|
|
99
114
|
|
|
100
|
-
const
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
children,
|
|
110
|
-
nullCount: 0,
|
|
111
|
-
});
|
|
112
|
-
|
|
113
|
-
const batch = new RecordBatch(schema, data, metadata);
|
|
115
|
+
const coerced: Record<string, any> = {};
|
|
116
|
+
for (const f of schema.fields) {
|
|
117
|
+
const raw = params[f.name];
|
|
118
|
+
// Missing values must be sent as null. arrow-js's typed-array builders
|
|
119
|
+
// throw "Invalid argument type in ToBigInt" when handed `undefined` for
|
|
120
|
+
// an Int64 column; null builds a proper validity bitmap entry instead.
|
|
121
|
+
coerced[f.name] = raw === undefined ? null : coerceForArrow(f.type, raw);
|
|
122
|
+
}
|
|
123
|
+
const batch = singleRowBatchWithMetadata(schema, coerced, metadata);
|
|
114
124
|
return serializeIpcStream(schema, [batch]);
|
|
115
125
|
}
|
|
116
126
|
|
package/src/client/pipe.ts
CHANGED
|
@@ -125,17 +125,17 @@ export class PipeStreamSession implements StreamSession {
|
|
|
125
125
|
|
|
126
126
|
if (batch.numRows === 0) {
|
|
127
127
|
// Check for external location pointer batch
|
|
128
|
-
if (isExternalLocationBatch(batch)) {
|
|
129
|
-
return await resolveExternalLocation(batch, this._externalConfig);
|
|
128
|
+
if (isExternalLocationBatch(batch as any)) {
|
|
129
|
+
return (await resolveExternalLocation(batch as any, this._externalConfig)) as any;
|
|
130
130
|
}
|
|
131
131
|
// Check if it's a log/error batch. If so, dispatch and continue.
|
|
132
132
|
// Otherwise it's a zero-row data batch — return it.
|
|
133
|
-
if (dispatchLogOrError(batch, this._onLog)) {
|
|
133
|
+
if (dispatchLogOrError(batch as any, this._onLog)) {
|
|
134
134
|
continue;
|
|
135
135
|
}
|
|
136
136
|
}
|
|
137
137
|
|
|
138
|
-
return batch;
|
|
138
|
+
return batch as any;
|
|
139
139
|
}
|
|
140
140
|
}
|
|
141
141
|
|
|
@@ -389,6 +389,7 @@ export function pipeConnect(
|
|
|
389
389
|
let readerPromise: Promise<IpcStreamReader> | null = null;
|
|
390
390
|
let methodCache: Map<string, MethodInfo> | null = null;
|
|
391
391
|
let protocolName = "";
|
|
392
|
+
let serverProtocolVersion = "";
|
|
392
393
|
let _busy = false;
|
|
393
394
|
let _drainPromise: Promise<void> | null = null;
|
|
394
395
|
let closed = false;
|
|
@@ -458,8 +459,9 @@ export function pipeConnect(
|
|
|
458
459
|
throw new Error("EOF reading __describe__ response");
|
|
459
460
|
}
|
|
460
461
|
|
|
461
|
-
const desc = await parseDescribeResponse(response.batches, onLog);
|
|
462
|
+
const desc = await parseDescribeResponse(response.batches as any, onLog);
|
|
462
463
|
protocolName = desc.protocolName;
|
|
464
|
+
serverProtocolVersion = desc.protocolVersion;
|
|
463
465
|
methodCache = new Map(desc.methods.map((m) => [m.name, m]));
|
|
464
466
|
return methodCache;
|
|
465
467
|
} finally {
|
|
@@ -483,7 +485,7 @@ export function pipeConnect(
|
|
|
483
485
|
const fullParams = { ...(info.defaults ?? {}), ...(params ?? {}) };
|
|
484
486
|
|
|
485
487
|
// Send request
|
|
486
|
-
const body = buildRequestIpc(info.paramsSchema, fullParams, method);
|
|
488
|
+
const body = buildRequestIpc(info.paramsSchema, fullParams, method, { protocolVersion: serverProtocolVersion });
|
|
487
489
|
writeFn(body);
|
|
488
490
|
|
|
489
491
|
// Read response
|
|
@@ -494,7 +496,7 @@ export function pipeConnect(
|
|
|
494
496
|
|
|
495
497
|
// Process batches: dispatch logs, resolve external pointers, find result
|
|
496
498
|
let resultBatch: RecordBatch | null = null;
|
|
497
|
-
for (let batch of response.batches) {
|
|
499
|
+
for (let batch of response.batches as any[]) {
|
|
498
500
|
if (batch.numRows === 0) {
|
|
499
501
|
if (isExternalLocationBatch(batch)) {
|
|
500
502
|
batch = await resolveExternalLocation(batch, externalConfig);
|
|
@@ -537,7 +539,7 @@ export function pipeConnect(
|
|
|
537
539
|
const fullParams = { ...(info.defaults ?? {}), ...(params ?? {}) };
|
|
538
540
|
|
|
539
541
|
// Send init request (params as a complete IPC stream)
|
|
540
|
-
const body = buildRequestIpc(info.paramsSchema, fullParams, method);
|
|
542
|
+
const body = buildRequestIpc(info.paramsSchema, fullParams, method, { protocolVersion: serverProtocolVersion });
|
|
541
543
|
writeFn(body);
|
|
542
544
|
|
|
543
545
|
// Read header if method has headerSchema
|
|
@@ -545,7 +547,7 @@ export function pipeConnect(
|
|
|
545
547
|
if (info.headerSchema) {
|
|
546
548
|
const headerStream = await r.readStream();
|
|
547
549
|
if (headerStream) {
|
|
548
|
-
for (const batch of headerStream.batches) {
|
|
550
|
+
for (const batch of headerStream.batches as any[]) {
|
|
549
551
|
if (batch.numRows === 0) {
|
|
550
552
|
dispatchLogOrError(batch, onLog);
|
|
551
553
|
continue;
|
|
@@ -597,6 +599,7 @@ export function pipeConnect(
|
|
|
597
599
|
const methods = await ensureMethodCache();
|
|
598
600
|
return {
|
|
599
601
|
protocolName,
|
|
602
|
+
protocolVersion: serverProtocolVersion,
|
|
600
603
|
methods: [...methods.values()],
|
|
601
604
|
};
|
|
602
605
|
},
|
package/src/client/stream.ts
CHANGED
|
@@ -9,8 +9,15 @@ import { ARROW_CONTENT_TYPE, serializeIpcStream } from "../http/common.js";
|
|
|
9
9
|
import { dispatchLogOrError, extractBatchRows, inferArrowType, readResponseBatches } from "./ipc.js";
|
|
10
10
|
import type { LogMessage, StreamSession } from "./types.js";
|
|
11
11
|
|
|
12
|
-
type CompressFn = (data: Uint8Array, level: number) => Uint8Array
|
|
13
|
-
type DecompressFn = (data: Uint8Array) => Uint8Array
|
|
12
|
+
type CompressFn = (data: Uint8Array, level: number) => Promise<Uint8Array>;
|
|
13
|
+
type DecompressFn = (data: Uint8Array) => Promise<Uint8Array>;
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Posts an Arrow IPC request body to *url*, transparently handling
|
|
17
|
+
* client-vended request externalization. Provided by the parent connection
|
|
18
|
+
* so a single capability cache can drive both unary and stream call paths.
|
|
19
|
+
*/
|
|
20
|
+
export type PostFn = (url: string, body: Uint8Array) => Promise<Response>;
|
|
14
21
|
|
|
15
22
|
export class HttpStreamSession implements StreamSession {
|
|
16
23
|
private _baseUrl: string;
|
|
@@ -28,6 +35,7 @@ export class HttpStreamSession implements StreamSession {
|
|
|
28
35
|
private _decompressFn?: DecompressFn;
|
|
29
36
|
private _authorization?: string;
|
|
30
37
|
private _externalConfig?: ExternalLocationConfig;
|
|
38
|
+
private _postFn?: PostFn;
|
|
31
39
|
|
|
32
40
|
constructor(opts: {
|
|
33
41
|
baseUrl: string;
|
|
@@ -45,6 +53,7 @@ export class HttpStreamSession implements StreamSession {
|
|
|
45
53
|
decompressFn?: DecompressFn;
|
|
46
54
|
authorization?: string;
|
|
47
55
|
externalConfig?: ExternalLocationConfig;
|
|
56
|
+
postFn?: PostFn;
|
|
48
57
|
}) {
|
|
49
58
|
this._baseUrl = opts.baseUrl;
|
|
50
59
|
this._prefix = opts.prefix;
|
|
@@ -61,6 +70,16 @@ export class HttpStreamSession implements StreamSession {
|
|
|
61
70
|
this._decompressFn = opts.decompressFn;
|
|
62
71
|
this._authorization = opts.authorization;
|
|
63
72
|
this._externalConfig = opts.externalConfig;
|
|
73
|
+
this._postFn = opts.postFn;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
private async _post(url: string, body: Uint8Array): Promise<Response> {
|
|
77
|
+
if (this._postFn) return this._postFn(url, body);
|
|
78
|
+
return fetch(url, {
|
|
79
|
+
method: "POST",
|
|
80
|
+
headers: this._buildHeaders(),
|
|
81
|
+
body: (await this._prepareBody(body)) as unknown as BodyInit,
|
|
82
|
+
});
|
|
64
83
|
}
|
|
65
84
|
|
|
66
85
|
get header(): Record<string, any> | null {
|
|
@@ -71,8 +90,10 @@ export class HttpStreamSession implements StreamSession {
|
|
|
71
90
|
const headers: Record<string, string> = {
|
|
72
91
|
"Content-Type": ARROW_CONTENT_TYPE,
|
|
73
92
|
};
|
|
74
|
-
if (this._compressionLevel != null) {
|
|
93
|
+
if (this._compressionLevel != null && this._compressFn) {
|
|
75
94
|
headers["Content-Encoding"] = "zstd";
|
|
95
|
+
}
|
|
96
|
+
if (this._compressionLevel != null && this._decompressFn) {
|
|
76
97
|
headers["Accept-Encoding"] = "zstd";
|
|
77
98
|
}
|
|
78
99
|
if (this._authorization) {
|
|
@@ -81,9 +102,9 @@ export class HttpStreamSession implements StreamSession {
|
|
|
81
102
|
return headers;
|
|
82
103
|
}
|
|
83
104
|
|
|
84
|
-
private _prepareBody(content: Uint8Array): Uint8Array {
|
|
105
|
+
private async _prepareBody(content: Uint8Array): Promise<Uint8Array> {
|
|
85
106
|
if (this._compressionLevel != null && this._compressFn) {
|
|
86
|
-
return this._compressFn(content, this._compressionLevel);
|
|
107
|
+
return await this._compressFn(content, this._compressionLevel);
|
|
87
108
|
}
|
|
88
109
|
return content;
|
|
89
110
|
}
|
|
@@ -91,7 +112,7 @@ export class HttpStreamSession implements StreamSession {
|
|
|
91
112
|
private async _readResponse(resp: Response): Promise<Uint8Array<ArrayBuffer>> {
|
|
92
113
|
let body = new Uint8Array(await resp.arrayBuffer());
|
|
93
114
|
if (resp.headers.get("Content-Encoding") === "zstd" && this._decompressFn) {
|
|
94
|
-
body = new Uint8Array(this._decompressFn(body));
|
|
115
|
+
body = new Uint8Array(await this._decompressFn(body));
|
|
95
116
|
}
|
|
96
117
|
return body;
|
|
97
118
|
}
|
|
@@ -159,11 +180,7 @@ export class HttpStreamSession implements StreamSession {
|
|
|
159
180
|
|
|
160
181
|
private async _doExchange(schema: Schema, batches: RecordBatch[]): Promise<Record<string, any>[]> {
|
|
161
182
|
const body = serializeIpcStream(schema, batches);
|
|
162
|
-
const resp = await
|
|
163
|
-
method: "POST",
|
|
164
|
-
headers: this._buildHeaders(),
|
|
165
|
-
body: this._prepareBody(body) as unknown as BodyInit,
|
|
166
|
-
});
|
|
183
|
+
const resp = await this._post(`${this._baseUrl}${this._prefix}/${this._method}/exchange`, body);
|
|
167
184
|
if (resp.status === 401) {
|
|
168
185
|
throw new RpcError("AuthenticationError", "Authentication required", "");
|
|
169
186
|
}
|
|
@@ -218,7 +235,7 @@ export class HttpStreamSession implements StreamSession {
|
|
|
218
235
|
for (let batch of this._pendingBatches) {
|
|
219
236
|
if (batch.numRows === 0) {
|
|
220
237
|
if (isExternalLocationBatch(batch)) {
|
|
221
|
-
batch = await resolveExternalLocation(batch, this._externalConfig);
|
|
238
|
+
batch = (await resolveExternalLocation(batch as any, this._externalConfig)) as any;
|
|
222
239
|
} else {
|
|
223
240
|
dispatchLogOrError(batch, this._onLog);
|
|
224
241
|
continue;
|
|
@@ -233,7 +250,9 @@ export class HttpStreamSession implements StreamSession {
|
|
|
233
250
|
|
|
234
251
|
// Follow continuation tokens
|
|
235
252
|
while (true) {
|
|
236
|
-
const
|
|
253
|
+
const stateToken = this._stateToken;
|
|
254
|
+
if (stateToken === null) return;
|
|
255
|
+
const responseBody = await this._sendContinuation(stateToken);
|
|
237
256
|
const { batches } = await readResponseBatches(responseBody);
|
|
238
257
|
|
|
239
258
|
let gotContinuation = false;
|
|
@@ -248,7 +267,7 @@ export class HttpStreamSession implements StreamSession {
|
|
|
248
267
|
}
|
|
249
268
|
// Check for external location pointer
|
|
250
269
|
if (isExternalLocationBatch(batch)) {
|
|
251
|
-
batch = await resolveExternalLocation(batch, this._externalConfig);
|
|
270
|
+
batch = (await resolveExternalLocation(batch as any, this._externalConfig)) as any;
|
|
252
271
|
} else {
|
|
253
272
|
// Log/error batch
|
|
254
273
|
dispatchLogOrError(batch, this._onLog);
|
|
@@ -278,11 +297,7 @@ export class HttpStreamSession implements StreamSession {
|
|
|
278
297
|
const batch = new RecordBatch(emptySchema, data, metadata);
|
|
279
298
|
const body = serializeIpcStream(emptySchema, [batch]);
|
|
280
299
|
|
|
281
|
-
const resp = await
|
|
282
|
-
method: "POST",
|
|
283
|
-
headers: this._buildHeaders(),
|
|
284
|
-
body: this._prepareBody(body) as unknown as BodyInit,
|
|
285
|
-
});
|
|
300
|
+
const resp = await this._post(`${this._baseUrl}${this._prefix}/${this._method}/exchange`, body);
|
|
286
301
|
if (resp.status === 401) {
|
|
287
302
|
throw new RpcError("AuthenticationError", "Authentication required", "");
|
|
288
303
|
}
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
// © Copyright 2025-2026, Query.Farm LLC - https://query.farm
|
|
2
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Client-side request-body externalization.
|
|
6
|
+
*
|
|
7
|
+
* When a request body would exceed the server-advertised `maxRequestBytes`,
|
|
8
|
+
* we (1) call `{prefix}/__upload_url__/init` to get a pre-signed upload URL,
|
|
9
|
+
* (2) PUT the body to that URL, and (3) replace the inline body with a
|
|
10
|
+
* zero-row "pointer" IPC stream that carries the original RPC dispatch
|
|
11
|
+
* metadata plus `vgi_rpc.location: <download-url>`. The server then resolves
|
|
12
|
+
* the pointer and dispatches normally.
|
|
13
|
+
*
|
|
14
|
+
* Mirrors Python's `_externalize_via_upload_url()` and `request_upload_urls()`.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import { Field, Int64, RecordBatchReader, Schema } from "@query-farm/apache-arrow";
|
|
18
|
+
import { REQUEST_VERSION, REQUEST_VERSION_KEY, RPC_METHOD_KEY } from "../constants.js";
|
|
19
|
+
import { RpcError } from "../errors.js";
|
|
20
|
+
import { makeExternalLocationBatch } from "../external.js";
|
|
21
|
+
import { ARROW_CONTENT_TYPE, serializeIpcStream } from "../http/common.js";
|
|
22
|
+
import { buildRequestIpc } from "./ipc.js";
|
|
23
|
+
|
|
24
|
+
const UPLOAD_URL_METHOD = "__upload_url__";
|
|
25
|
+
const UPLOAD_URL_PARAMS_SCHEMA = new Schema([new Field("count", new Int64(), false)]);
|
|
26
|
+
|
|
27
|
+
export interface UploadUrlPair {
|
|
28
|
+
uploadUrl: string;
|
|
29
|
+
downloadUrl: string;
|
|
30
|
+
expiresAt: Date;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* POST `__upload_url__/init` and return the requested number of pre-signed
|
|
35
|
+
* URL pairs. Server must have an `uploadUrlProvider` configured; otherwise
|
|
36
|
+
* the route returns 404 and we surface that as `RpcError("NotSupported")`.
|
|
37
|
+
*/
|
|
38
|
+
export async function requestUploadUrls(
|
|
39
|
+
baseUrl: string,
|
|
40
|
+
prefix: string,
|
|
41
|
+
count: number,
|
|
42
|
+
authorization?: string,
|
|
43
|
+
): Promise<UploadUrlPair[]> {
|
|
44
|
+
const body = buildRequestIpc(UPLOAD_URL_PARAMS_SCHEMA, { count: BigInt(count) }, UPLOAD_URL_METHOD);
|
|
45
|
+
const headers: Record<string, string> = { "Content-Type": ARROW_CONTENT_TYPE };
|
|
46
|
+
if (authorization) headers.Authorization = authorization;
|
|
47
|
+
|
|
48
|
+
const resp = await fetch(`${baseUrl}${prefix}/${UPLOAD_URL_METHOD}/init`, {
|
|
49
|
+
method: "POST",
|
|
50
|
+
headers,
|
|
51
|
+
body: body as unknown as BodyInit,
|
|
52
|
+
});
|
|
53
|
+
if (resp.status === 404) {
|
|
54
|
+
throw new RpcError("NotSupported", "Server does not support upload URLs", "");
|
|
55
|
+
}
|
|
56
|
+
if (resp.status === 401) {
|
|
57
|
+
throw new RpcError("AuthenticationError", "Authentication required", "");
|
|
58
|
+
}
|
|
59
|
+
if (!resp.ok) {
|
|
60
|
+
throw new RpcError("HttpError", `__upload_url__/init failed: HTTP ${resp.status}`, "");
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
const respBody = new Uint8Array(await resp.arrayBuffer());
|
|
64
|
+
const reader = await RecordBatchReader.from(respBody);
|
|
65
|
+
await reader.open();
|
|
66
|
+
|
|
67
|
+
const pairs: UploadUrlPair[] = [];
|
|
68
|
+
for (const batch of reader.readAll()) {
|
|
69
|
+
if (batch.numRows === 0) continue;
|
|
70
|
+
for (let r = 0; r < batch.numRows; r++) {
|
|
71
|
+
const uploadUrl = batch.getChildAt(0)?.get(r) as string;
|
|
72
|
+
const downloadUrl = batch.getChildAt(1)?.get(r) as string;
|
|
73
|
+
const expiresRaw = batch.getChildAt(2)?.get(r);
|
|
74
|
+
// Timestamp(us) → either a Date, a number (ms), or a bigint (us)
|
|
75
|
+
let expiresAt: Date;
|
|
76
|
+
if (expiresRaw instanceof Date) {
|
|
77
|
+
expiresAt = expiresRaw;
|
|
78
|
+
} else if (typeof expiresRaw === "bigint") {
|
|
79
|
+
expiresAt = new Date(Number(expiresRaw / 1000n));
|
|
80
|
+
} else if (typeof expiresRaw === "number") {
|
|
81
|
+
expiresAt = new Date(expiresRaw);
|
|
82
|
+
} else {
|
|
83
|
+
expiresAt = new Date();
|
|
84
|
+
}
|
|
85
|
+
pairs.push({ uploadUrl, downloadUrl, expiresAt });
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
if (pairs.length === 0) {
|
|
90
|
+
throw new RpcError("ProtocolError", "Server returned no upload URLs", "");
|
|
91
|
+
}
|
|
92
|
+
return pairs;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Build the externalized pointer body to send in place of *originalBody*.
|
|
97
|
+
*
|
|
98
|
+
* The pointer is a zero-row IPC stream whose first batch carries:
|
|
99
|
+
* - same schema as the original request batch
|
|
100
|
+
* - merged custom_metadata: original RPC dispatch keys + `vgi_rpc.location`
|
|
101
|
+
*
|
|
102
|
+
* The server's request reader honours the dispatch keys for routing, then
|
|
103
|
+
* resolves the pointer to fetch the inline batch for parameter extraction.
|
|
104
|
+
*/
|
|
105
|
+
async function buildPointerRequestBody(originalBody: Uint8Array, downloadUrl: string): Promise<Uint8Array> {
|
|
106
|
+
const reader = await RecordBatchReader.from(originalBody);
|
|
107
|
+
await reader.open();
|
|
108
|
+
const schema = reader.schema;
|
|
109
|
+
if (!schema) {
|
|
110
|
+
throw new RpcError("ProtocolError", "Original request body has no schema", "");
|
|
111
|
+
}
|
|
112
|
+
const batches = reader.readAll();
|
|
113
|
+
if (batches.length === 0) {
|
|
114
|
+
throw new RpcError("ProtocolError", "Original request body has no batches", "");
|
|
115
|
+
}
|
|
116
|
+
const original = batches[0];
|
|
117
|
+
const originalMeta = original.metadata ?? new Map<string, string>();
|
|
118
|
+
|
|
119
|
+
const pointer = makeExternalLocationBatch(schema, downloadUrl);
|
|
120
|
+
const merged = new Map<string, string>(pointer.metadata ?? new Map());
|
|
121
|
+
// Preserve the original RPC dispatch metadata so the server can route.
|
|
122
|
+
const method = originalMeta.get(RPC_METHOD_KEY);
|
|
123
|
+
const version = originalMeta.get(REQUEST_VERSION_KEY) ?? REQUEST_VERSION;
|
|
124
|
+
if (method) merged.set(RPC_METHOD_KEY, method);
|
|
125
|
+
merged.set(REQUEST_VERSION_KEY, version);
|
|
126
|
+
// Carry over any other keys (request id, state token for exchange, etc).
|
|
127
|
+
for (const [k, v] of originalMeta) {
|
|
128
|
+
if (!merged.has(k)) merged.set(k, v);
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
// Re-emit the pointer batch with merged metadata.
|
|
132
|
+
const { RecordBatch } = await import("@query-farm/apache-arrow");
|
|
133
|
+
const pointerWithMeta = new RecordBatch(schema as any, (pointer as any).data, merged);
|
|
134
|
+
return serializeIpcStream(schema, [pointerWithMeta]);
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
export interface ExternalizeOptions {
|
|
138
|
+
baseUrl: string;
|
|
139
|
+
prefix: string;
|
|
140
|
+
authorization?: string;
|
|
141
|
+
/** Optional per-URL validator; throw to reject. */
|
|
142
|
+
urlValidator?: ((url: string) => void) | null;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* Upload *body* via a server-vended URL and return the pointer-batch body
|
|
147
|
+
* that should be sent in place of the original. Throws if the server does
|
|
148
|
+
* not advertise upload-URL support or the upload fails.
|
|
149
|
+
*/
|
|
150
|
+
export async function externalizeRequestBody(body: Uint8Array, opts: ExternalizeOptions): Promise<Uint8Array> {
|
|
151
|
+
const pairs = await requestUploadUrls(opts.baseUrl, opts.prefix, 1, opts.authorization);
|
|
152
|
+
const pair = pairs[0];
|
|
153
|
+
|
|
154
|
+
if (opts.urlValidator) {
|
|
155
|
+
opts.urlValidator(pair.uploadUrl);
|
|
156
|
+
opts.urlValidator(pair.downloadUrl);
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
const putResp = await fetch(pair.uploadUrl, {
|
|
160
|
+
method: "PUT",
|
|
161
|
+
headers: { "Content-Type": ARROW_CONTENT_TYPE },
|
|
162
|
+
body: body as unknown as BodyInit,
|
|
163
|
+
});
|
|
164
|
+
if (!putResp.ok) {
|
|
165
|
+
throw new RpcError("ExternalUploadFailed", `PUT to upload URL failed: HTTP ${putResp.status}`, "");
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
return buildPointerRequestBody(body, pair.downloadUrl);
|
|
169
|
+
}
|
package/src/constants.ts
CHANGED
|
@@ -15,11 +15,28 @@ export const REQUEST_ID_KEY = "vgi_rpc.request_id";
|
|
|
15
15
|
|
|
16
16
|
export const PROTOCOL_NAME_KEY = "vgi_rpc.protocol_name";
|
|
17
17
|
export const DESCRIBE_VERSION_KEY = "vgi_rpc.describe_version";
|
|
18
|
-
export const
|
|
18
|
+
export const PROTOCOL_HASH_KEY = "vgi_rpc.protocol_hash";
|
|
19
|
+
export const DESCRIBE_VERSION = "4";
|
|
20
|
+
|
|
21
|
+
/** Application protocol surface version. Carried on every request batch from
|
|
22
|
+
* a client bound to a Protocol that declares `protocolVersion`; also emitted
|
|
23
|
+
* in the __describe__ response metadata. Format: canonical semver
|
|
24
|
+
* MAJOR.MINOR.PATCH. Enforced at the dispatch boundary on the server: exact
|
|
25
|
+
* major+minor match required, patch ignored. Distinct from `REQUEST_VERSION`
|
|
26
|
+
* (wire framing). Mirrors Python's `PROTOCOL_VERSION_KEY`. */
|
|
27
|
+
export const PROTOCOL_VERSION_KEY = "vgi_rpc.protocol_version";
|
|
19
28
|
|
|
20
29
|
export const DESCRIBE_METHOD_NAME = "__describe__";
|
|
21
30
|
|
|
22
31
|
export const STATE_KEY = "vgi_rpc.stream_state#b64";
|
|
32
|
+
export const CANCEL_KEY = "vgi_rpc.cancel";
|
|
23
33
|
|
|
24
34
|
export const LOCATION_KEY = "vgi_rpc.location";
|
|
25
35
|
export const LOCATION_SHA256_KEY = "vgi_rpc.location.sha256";
|
|
36
|
+
|
|
37
|
+
export const RPC_ERROR_HEADER = "X-VGI-RPC-Error";
|
|
38
|
+
|
|
39
|
+
/** Top-level metadata key on an EXCEPTION batch identifying the error category.
|
|
40
|
+
* Hoisted by `buildErrorBatch` when the thrown error has a static or instance
|
|
41
|
+
* `errorKind` property. Mirrors Python's `vgi_rpc.metadata.ERROR_KIND_KEY`. */
|
|
42
|
+
export const ERROR_KIND_KEY = "vgi_rpc.error_kind";
|
package/src/crypto.ts
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
// © Copyright 2025-2026, Query.Farm LLC - https://query.farm
|
|
2
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Generic AEAD seal/open primitives shared by stream-state and sticky-session
|
|
6
|
+
* tokens. Mirrors Python's `vgi_rpc.crypto` module: a tiny envelope around
|
|
7
|
+
* XChaCha20-Poly1305 with a leading version byte so future format bumps stay
|
|
8
|
+
* self-describing.
|
|
9
|
+
*
|
|
10
|
+
* Wire format (returned by {@link sealBytes}, accepted by {@link openBytes}):
|
|
11
|
+
*
|
|
12
|
+
* ```
|
|
13
|
+
* [1B version (1..255)]
|
|
14
|
+
* [24B nonce (XChaCha20-Poly1305, random per envelope)]
|
|
15
|
+
* [.. ciphertext + 16B Poly1305 tag]
|
|
16
|
+
* ```
|
|
17
|
+
*
|
|
18
|
+
* The plaintext frame is fully up to the caller — only the version + nonce +
|
|
19
|
+
* tag overhead is fixed. AAD (`aad`) is bound at the crypto layer so an
|
|
20
|
+
* envelope sealed for one identity fails decryption when presented by another.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
import { xchacha20poly1305 } from "@noble/ciphers/chacha.js";
|
|
24
|
+
import { randomBytes } from "./util/web-crypto.js";
|
|
25
|
+
|
|
26
|
+
const NONCE_LEN = 24;
|
|
27
|
+
const TAG_LEN = 16;
|
|
28
|
+
const VERSION_LEN = 1;
|
|
29
|
+
const MIN_ENVELOPE_LEN = VERSION_LEN + NONCE_LEN + TAG_LEN;
|
|
30
|
+
|
|
31
|
+
/** Thrown by {@link openBytes} for any envelope it cannot open — malformed,
|
|
32
|
+
* tampered, wrong key, wrong AAD, wrong version, truncated, all surface the
|
|
33
|
+
* same way so callers cannot distinguish failure modes via error content. */
|
|
34
|
+
export class SealError extends Error {
|
|
35
|
+
constructor(message: string) {
|
|
36
|
+
super(message);
|
|
37
|
+
this.name = "SealError";
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** Normalise a key to 32 bytes by SHA-256 hashing when it isn't already 32B.
|
|
42
|
+
* Mirrors Python's `normalize_key` so any callers can pass operator-provided
|
|
43
|
+
* keys of arbitrary length without a separate stretching step. */
|
|
44
|
+
export async function normalizeKey(key: Uint8Array): Promise<Uint8Array> {
|
|
45
|
+
if (key.length === 32) return key;
|
|
46
|
+
const digest = await crypto.subtle.digest("SHA-256", key as BufferSource);
|
|
47
|
+
return new Uint8Array(digest);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export interface SealOptions {
|
|
51
|
+
/** Associated data bound at the crypto layer — typically a principal or
|
|
52
|
+
* request-scoped identifier. Must match between seal and open. */
|
|
53
|
+
aad: Uint8Array;
|
|
54
|
+
/** Envelope version byte. Defaults to 1; carry through to {@link openBytes}. */
|
|
55
|
+
version?: number;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** Seal `plaintext` under `key` with AEAD, returning the wire envelope. */
|
|
59
|
+
export function sealBytes(plaintext: Uint8Array, key: Uint8Array, opts: SealOptions): Uint8Array {
|
|
60
|
+
if (key.length !== 32) {
|
|
61
|
+
throw new Error("AEAD key must be 32 bytes — call normalizeKey() first");
|
|
62
|
+
}
|
|
63
|
+
const version = opts.version ?? 1;
|
|
64
|
+
if (version < 1 || version > 255) {
|
|
65
|
+
throw new Error(`AEAD envelope version must fit in one byte; got ${version}`);
|
|
66
|
+
}
|
|
67
|
+
const nonce = randomBytes(NONCE_LEN);
|
|
68
|
+
const ciphertext = xchacha20poly1305(key, nonce, opts.aad as Uint8Array).encrypt(plaintext);
|
|
69
|
+
const wire = new Uint8Array(VERSION_LEN + NONCE_LEN + ciphertext.length);
|
|
70
|
+
wire[0] = version;
|
|
71
|
+
wire.set(nonce, VERSION_LEN);
|
|
72
|
+
wire.set(ciphertext, VERSION_LEN + NONCE_LEN);
|
|
73
|
+
return wire;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** Open and verify an envelope produced by {@link sealBytes}. */
|
|
77
|
+
export function openBytes(envelope: Uint8Array, key: Uint8Array, opts: SealOptions): Uint8Array {
|
|
78
|
+
if (key.length !== 32) {
|
|
79
|
+
throw new Error("AEAD key must be 32 bytes — call normalizeKey() first");
|
|
80
|
+
}
|
|
81
|
+
if (envelope.length < MIN_ENVELOPE_LEN) {
|
|
82
|
+
throw new SealError("envelope truncated");
|
|
83
|
+
}
|
|
84
|
+
const expectedVersion = opts.version ?? 1;
|
|
85
|
+
if (envelope[0] !== expectedVersion) {
|
|
86
|
+
throw new SealError(`unsupported envelope version: ${envelope[0]}`);
|
|
87
|
+
}
|
|
88
|
+
const nonce = envelope.subarray(VERSION_LEN, VERSION_LEN + NONCE_LEN);
|
|
89
|
+
const ciphertext = envelope.subarray(VERSION_LEN + NONCE_LEN);
|
|
90
|
+
try {
|
|
91
|
+
return xchacha20poly1305(key, nonce, opts.aad as Uint8Array).decrypt(ciphertext);
|
|
92
|
+
} catch {
|
|
93
|
+
throw new SealError("envelope verification failed");
|
|
94
|
+
}
|
|
95
|
+
}
|