@query-farm/vgi-rpc 0.3.1 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client/connect.d.ts.map +1 -1
- package/dist/client/index.d.ts +3 -3
- package/dist/client/index.d.ts.map +1 -1
- package/dist/client/introspect.d.ts +1 -1
- package/dist/client/introspect.d.ts.map +1 -1
- package/dist/client/ipc.d.ts +1 -1
- package/dist/client/ipc.d.ts.map +1 -1
- package/dist/client/pipe.d.ts +1 -1
- package/dist/client/pipe.d.ts.map +1 -1
- package/dist/client/stream.d.ts.map +1 -1
- package/dist/dispatch/describe.d.ts +1 -1
- package/dist/dispatch/describe.d.ts.map +1 -1
- package/dist/dispatch/stream.d.ts +1 -1
- package/dist/dispatch/stream.d.ts.map +1 -1
- package/dist/dispatch/unary.d.ts.map +1 -1
- package/dist/http/common.d.ts +1 -1
- package/dist/http/common.d.ts.map +1 -1
- package/dist/http/dispatch.d.ts.map +1 -1
- package/dist/http/handler.d.ts.map +1 -1
- package/dist/http/index.d.ts +2 -2
- package/dist/http/index.d.ts.map +1 -1
- package/dist/http/token.d.ts.map +1 -1
- package/dist/http/types.d.ts.map +1 -1
- package/dist/index.d.ts +7 -7
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2163 -2173
- package/dist/index.js.map +25 -25
- package/dist/protocol.d.ts +1 -1
- package/dist/protocol.d.ts.map +1 -1
- package/dist/schema.d.ts +1 -1
- package/dist/schema.d.ts.map +1 -1
- package/dist/server.d.ts +1 -1
- package/dist/server.d.ts.map +1 -1
- package/dist/types.d.ts +1 -1
- package/dist/types.d.ts.map +1 -1
- package/dist/util/zstd.d.ts.map +1 -1
- package/dist/wire/reader.d.ts.map +1 -1
- package/dist/wire/request.d.ts +1 -1
- package/dist/wire/request.d.ts.map +1 -1
- package/dist/wire/response.d.ts +1 -1
- package/dist/wire/response.d.ts.map +1 -1
- package/dist/wire/writer.d.ts.map +1 -1
- package/package.json +6 -2
- package/src/client/connect.ts +12 -20
- package/src/client/index.ts +8 -8
- package/src/client/introspect.ts +11 -15
- package/src/client/ipc.ts +17 -31
- package/src/client/pipe.ts +16 -36
- package/src/client/stream.ts +20 -46
- package/src/dispatch/describe.ts +14 -26
- package/src/dispatch/stream.ts +5 -18
- package/src/dispatch/unary.ts +1 -2
- package/src/http/common.ts +8 -18
- package/src/http/dispatch.ts +31 -86
- package/src/http/handler.ts +20 -55
- package/src/http/index.ts +2 -2
- package/src/http/token.ts +2 -7
- package/src/http/types.ts +2 -6
- package/src/index.ts +43 -43
- package/src/protocol.ts +7 -7
- package/src/schema.ts +11 -15
- package/src/server.ts +14 -34
- package/src/types.ts +9 -36
- package/src/util/zstd.ts +2 -8
- package/src/wire/reader.ts +2 -4
- package/src/wire/request.ts +4 -15
- package/src/wire/response.ts +8 -24
- package/src/wire/writer.ts +1 -5
package/src/dispatch/describe.ts
CHANGED
|
@@ -2,25 +2,25 @@
|
|
|
2
2
|
// SPDX-License-Identifier: Apache-2.0
|
|
3
3
|
|
|
4
4
|
import {
|
|
5
|
-
|
|
5
|
+
Binary,
|
|
6
|
+
Bool,
|
|
6
7
|
Field,
|
|
8
|
+
makeData,
|
|
7
9
|
RecordBatch,
|
|
10
|
+
Schema,
|
|
11
|
+
Struct,
|
|
8
12
|
Utf8,
|
|
9
|
-
Bool,
|
|
10
|
-
Binary,
|
|
11
13
|
vectorFromArray,
|
|
12
|
-
makeData,
|
|
13
|
-
Struct,
|
|
14
14
|
} from "@query-farm/apache-arrow";
|
|
15
|
-
import type { MethodDefinition } from "../types.js";
|
|
16
15
|
import {
|
|
16
|
+
DESCRIBE_VERSION,
|
|
17
|
+
DESCRIBE_VERSION_KEY,
|
|
17
18
|
PROTOCOL_NAME_KEY,
|
|
18
|
-
REQUEST_VERSION_KEY,
|
|
19
19
|
REQUEST_VERSION,
|
|
20
|
-
|
|
21
|
-
DESCRIBE_VERSION,
|
|
20
|
+
REQUEST_VERSION_KEY,
|
|
22
21
|
SERVER_ID_KEY,
|
|
23
22
|
} from "../constants.js";
|
|
23
|
+
import type { MethodDefinition } from "../types.js";
|
|
24
24
|
import { serializeSchema } from "../util/schema.js";
|
|
25
25
|
|
|
26
26
|
/**
|
|
@@ -48,9 +48,7 @@ export function buildDescribeBatch(
|
|
|
48
48
|
serverId: string,
|
|
49
49
|
): { batch: RecordBatch; metadata: Map<string, string> } {
|
|
50
50
|
// Sort methods by name for consistent ordering
|
|
51
|
-
const sortedEntries = [...methods.entries()].sort(([a], [b]) =>
|
|
52
|
-
a.localeCompare(b),
|
|
53
|
-
);
|
|
51
|
+
const sortedEntries = [...methods.entries()].sort(([a], [b]) => a.localeCompare(b));
|
|
54
52
|
|
|
55
53
|
const names: (string | null)[] = [];
|
|
56
54
|
const methodTypes: (string | null)[] = [];
|
|
@@ -69,8 +67,7 @@ export function buildDescribeBatch(
|
|
|
69
67
|
docs.push(method.doc ?? null);
|
|
70
68
|
|
|
71
69
|
// Unary methods with non-empty result schema have a return value
|
|
72
|
-
const hasReturn =
|
|
73
|
-
method.type === "unary" && method.resultSchema.fields.length > 0;
|
|
70
|
+
const hasReturn = method.type === "unary" && method.resultSchema.fields.length > 0;
|
|
74
71
|
hasReturns.push(hasReturn);
|
|
75
72
|
|
|
76
73
|
paramsSchemas.push(serializeSchema(method.paramsSchema));
|
|
@@ -87,26 +84,17 @@ export function buildDescribeBatch(
|
|
|
87
84
|
if (method.defaults && Object.keys(method.defaults).length > 0) {
|
|
88
85
|
const safe: Record<string, any> = {};
|
|
89
86
|
for (const [k, v] of Object.entries(method.defaults)) {
|
|
90
|
-
if (
|
|
91
|
-
v === null ||
|
|
92
|
-
typeof v === "string" ||
|
|
93
|
-
typeof v === "number" ||
|
|
94
|
-
typeof v === "boolean"
|
|
95
|
-
) {
|
|
87
|
+
if (v === null || typeof v === "string" || typeof v === "number" || typeof v === "boolean") {
|
|
96
88
|
safe[k] = v;
|
|
97
89
|
}
|
|
98
90
|
}
|
|
99
|
-
paramDefaultsJsons.push(
|
|
100
|
-
Object.keys(safe).length > 0 ? JSON.stringify(safe) : null,
|
|
101
|
-
);
|
|
91
|
+
paramDefaultsJsons.push(Object.keys(safe).length > 0 ? JSON.stringify(safe) : null);
|
|
102
92
|
} else {
|
|
103
93
|
paramDefaultsJsons.push(null);
|
|
104
94
|
}
|
|
105
95
|
|
|
106
96
|
hasHeaders.push(!!method.headerSchema);
|
|
107
|
-
headerSchemas.push(
|
|
108
|
-
method.headerSchema ? serializeSchema(method.headerSchema) : null,
|
|
109
|
-
);
|
|
97
|
+
headerSchemas.push(method.headerSchema ? serializeSchema(method.headerSchema) : null);
|
|
110
98
|
}
|
|
111
99
|
|
|
112
100
|
// Build the batch using vectorFromArray for each column
|
package/src/dispatch/stream.ts
CHANGED
|
@@ -4,9 +4,9 @@
|
|
|
4
4
|
import { Schema } from "@query-farm/apache-arrow";
|
|
5
5
|
import type { MethodDefinition } from "../types.js";
|
|
6
6
|
import { OutputCollector } from "../types.js";
|
|
7
|
-
import type { IpcStreamWriter } from "../wire/writer.js";
|
|
8
7
|
import type { IpcStreamReader } from "../wire/reader.js";
|
|
9
|
-
import {
|
|
8
|
+
import { buildErrorBatch, buildResultBatch } from "../wire/response.js";
|
|
9
|
+
import type { IpcStreamWriter } from "../wire/writer.js";
|
|
10
10
|
|
|
11
11
|
const EMPTY_SCHEMA = new Schema([]);
|
|
12
12
|
|
|
@@ -70,16 +70,8 @@ export async function dispatchStream(
|
|
|
70
70
|
try {
|
|
71
71
|
const headerOut = new OutputCollector(method.headerSchema, true, serverId, requestId);
|
|
72
72
|
const headerValues = method.headerInit(params, state, headerOut);
|
|
73
|
-
const headerBatch = buildResultBatch(
|
|
74
|
-
|
|
75
|
-
headerValues,
|
|
76
|
-
serverId,
|
|
77
|
-
requestId,
|
|
78
|
-
);
|
|
79
|
-
const headerBatches = [
|
|
80
|
-
...headerOut.batches.map((b) => b.batch),
|
|
81
|
-
headerBatch,
|
|
82
|
-
];
|
|
73
|
+
const headerBatch = buildResultBatch(method.headerSchema, headerValues, serverId, requestId);
|
|
74
|
+
const headerBatches = [...headerOut.batches.map((b) => b.batch), headerBatch];
|
|
83
75
|
writer.writeStream(method.headerSchema, headerBatches);
|
|
84
76
|
} catch (error: any) {
|
|
85
77
|
const errBatch = buildErrorBatch(method.headerSchema, error, serverId, requestId);
|
|
@@ -96,12 +88,7 @@ export async function dispatchStream(
|
|
|
96
88
|
// Open the input IPC stream (ticks or data from client)
|
|
97
89
|
const inputSchema = await reader.openNextStream();
|
|
98
90
|
if (!inputSchema) {
|
|
99
|
-
const errBatch = buildErrorBatch(
|
|
100
|
-
outputSchema,
|
|
101
|
-
new Error("Expected input stream but got EOF"),
|
|
102
|
-
serverId,
|
|
103
|
-
requestId,
|
|
104
|
-
);
|
|
91
|
+
const errBatch = buildErrorBatch(outputSchema, new Error("Expected input stream but got EOF"), serverId, requestId);
|
|
105
92
|
writer.writeStream(outputSchema, [errBatch]);
|
|
106
93
|
return;
|
|
107
94
|
}
|
package/src/dispatch/unary.ts
CHANGED
|
@@ -1,11 +1,10 @@
|
|
|
1
1
|
// © Copyright 2025-2026, Query.Farm LLC - https://query.farm
|
|
2
2
|
// SPDX-License-Identifier: Apache-2.0
|
|
3
3
|
|
|
4
|
-
import { Schema } from "@query-farm/apache-arrow";
|
|
5
4
|
import type { MethodDefinition } from "../types.js";
|
|
6
5
|
import { OutputCollector } from "../types.js";
|
|
6
|
+
import { buildErrorBatch, buildResultBatch } from "../wire/response.js";
|
|
7
7
|
import type { IpcStreamWriter } from "../wire/writer.js";
|
|
8
|
-
import { buildResultBatch, buildErrorBatch } from "../wire/response.js";
|
|
9
8
|
|
|
10
9
|
/**
|
|
11
10
|
* Dispatch a unary RPC call.
|
package/src/http/common.ts
CHANGED
|
@@ -2,12 +2,12 @@
|
|
|
2
2
|
// SPDX-License-Identifier: Apache-2.0
|
|
3
3
|
|
|
4
4
|
import {
|
|
5
|
-
|
|
6
|
-
RecordBatchReader,
|
|
5
|
+
makeData,
|
|
7
6
|
RecordBatch,
|
|
8
|
-
|
|
7
|
+
RecordBatchReader,
|
|
8
|
+
RecordBatchStreamWriter,
|
|
9
|
+
type Schema,
|
|
9
10
|
Struct,
|
|
10
|
-
makeData,
|
|
11
11
|
} from "@query-farm/apache-arrow";
|
|
12
12
|
|
|
13
13
|
export const ARROW_CONTENT_TYPE = "application/vnd.apache.arrow.stream";
|
|
@@ -31,14 +31,9 @@ export class HttpRpcError extends Error {
|
|
|
31
31
|
* match the writer's schema. Cloning each child Data with the schema's field
|
|
32
32
|
* type fixes the type metadata while preserving the underlying buffers.
|
|
33
33
|
*/
|
|
34
|
-
function conformBatchToSchema(
|
|
35
|
-
batch: RecordBatch,
|
|
36
|
-
schema: Schema,
|
|
37
|
-
): RecordBatch {
|
|
34
|
+
function conformBatchToSchema(batch: RecordBatch, schema: Schema): RecordBatch {
|
|
38
35
|
if (batch.numRows === 0) return batch;
|
|
39
|
-
const children = schema.fields.map((f, i) =>
|
|
40
|
-
batch.data.children[i].clone(f.type),
|
|
41
|
-
);
|
|
36
|
+
const children = schema.fields.map((f, i) => batch.data.children[i].clone(f.type));
|
|
42
37
|
const structType = new Struct(schema.fields);
|
|
43
38
|
const data = makeData({
|
|
44
39
|
type: structType,
|
|
@@ -51,10 +46,7 @@ function conformBatchToSchema(
|
|
|
51
46
|
}
|
|
52
47
|
|
|
53
48
|
/** Serialize a schema + batches into a complete IPC stream as Uint8Array. */
|
|
54
|
-
export function serializeIpcStream(
|
|
55
|
-
schema: Schema,
|
|
56
|
-
batches: RecordBatch[],
|
|
57
|
-
): Uint8Array {
|
|
49
|
+
export function serializeIpcStream(schema: Schema, batches: RecordBatch[]): Uint8Array {
|
|
58
50
|
const writer = new RecordBatchStreamWriter();
|
|
59
51
|
writer.reset(undefined, schema);
|
|
60
52
|
for (const batch of batches) {
|
|
@@ -72,9 +64,7 @@ export function arrowResponse(body: Uint8Array, status = 200, extraHeaders?: Hea
|
|
|
72
64
|
}
|
|
73
65
|
|
|
74
66
|
/** Read schema + first batch from an IPC stream body. */
|
|
75
|
-
export async function readRequestFromBody(
|
|
76
|
-
body: Uint8Array,
|
|
77
|
-
): Promise<{ schema: Schema; batch: RecordBatch }> {
|
|
67
|
+
export async function readRequestFromBody(body: Uint8Array): Promise<{ schema: Schema; batch: RecordBatch }> {
|
|
78
68
|
const reader = await RecordBatchReader.from(body);
|
|
79
69
|
await reader.open();
|
|
80
70
|
const schema = reader.schema;
|
package/src/http/dispatch.ts
CHANGED
|
@@ -1,24 +1,15 @@
|
|
|
1
1
|
// © Copyright 2025-2026, Query.Farm LLC - https://query.farm
|
|
2
2
|
// SPDX-License-Identifier: Apache-2.0
|
|
3
3
|
|
|
4
|
-
import {
|
|
4
|
+
import { RecordBatch, RecordBatchReader, Schema } from "@query-farm/apache-arrow";
|
|
5
|
+
import { STATE_KEY } from "../constants.js";
|
|
6
|
+
import { buildDescribeBatch, DESCRIBE_SCHEMA } from "../dispatch/describe.js";
|
|
5
7
|
import type { MethodDefinition } from "../types.js";
|
|
6
8
|
import { OutputCollector } from "../types.js";
|
|
7
|
-
import { parseRequest } from "../wire/request.js";
|
|
8
|
-
import {
|
|
9
|
-
buildResultBatch,
|
|
10
|
-
buildErrorBatch,
|
|
11
|
-
buildEmptyBatch,
|
|
12
|
-
} from "../wire/response.js";
|
|
13
|
-
import { buildDescribeBatch, DESCRIBE_SCHEMA } from "../dispatch/describe.js";
|
|
14
|
-
import { STATE_KEY } from "../constants.js";
|
|
15
9
|
import { serializeSchema } from "../util/schema.js";
|
|
16
|
-
import {
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
readRequestFromBody,
|
|
20
|
-
arrowResponse,
|
|
21
|
-
} from "./common.js";
|
|
10
|
+
import { parseRequest } from "../wire/request.js";
|
|
11
|
+
import { buildEmptyBatch, buildErrorBatch, buildResultBatch } from "../wire/response.js";
|
|
12
|
+
import { arrowResponse, HttpRpcError, readRequestFromBody, serializeIpcStream } from "./common.js";
|
|
22
13
|
import { packStateToken, unpackStateToken } from "./token.js";
|
|
23
14
|
import type { StateSerializer } from "./types.js";
|
|
24
15
|
|
|
@@ -60,10 +51,7 @@ export async function httpDispatchUnary(
|
|
|
60
51
|
const parsed = parseRequest(reqSchema, reqBatch);
|
|
61
52
|
|
|
62
53
|
if (parsed.methodName !== method.name) {
|
|
63
|
-
throw new HttpRpcError(
|
|
64
|
-
`Method name in request '${parsed.methodName}' does not match URL '${method.name}'`,
|
|
65
|
-
400,
|
|
66
|
-
);
|
|
54
|
+
throw new HttpRpcError(`Method name in request '${parsed.methodName}' does not match URL '${method.name}'`, 400);
|
|
67
55
|
}
|
|
68
56
|
|
|
69
57
|
const out = new OutputCollector(schema, true, ctx.serverId, parsed.requestId);
|
|
@@ -93,10 +81,7 @@ export async function httpDispatchStreamInit(
|
|
|
93
81
|
const parsed = parseRequest(reqSchema, reqBatch);
|
|
94
82
|
|
|
95
83
|
if (parsed.methodName !== method.name) {
|
|
96
|
-
throw new HttpRpcError(
|
|
97
|
-
`Method name in request '${parsed.methodName}' does not match URL '${method.name}'`,
|
|
98
|
-
400,
|
|
99
|
-
);
|
|
84
|
+
throw new HttpRpcError(`Method name in request '${parsed.methodName}' does not match URL '${method.name}'`, 400);
|
|
100
85
|
}
|
|
101
86
|
|
|
102
87
|
// Init state
|
|
@@ -121,31 +106,13 @@ export async function httpDispatchStreamInit(
|
|
|
121
106
|
let headerBytes: Uint8Array | null = null;
|
|
122
107
|
if (method.headerSchema && method.headerInit) {
|
|
123
108
|
try {
|
|
124
|
-
const headerOut = new OutputCollector(
|
|
125
|
-
method.headerSchema,
|
|
126
|
-
true,
|
|
127
|
-
ctx.serverId,
|
|
128
|
-
parsed.requestId,
|
|
129
|
-
);
|
|
109
|
+
const headerOut = new OutputCollector(method.headerSchema, true, ctx.serverId, parsed.requestId);
|
|
130
110
|
const headerValues = method.headerInit(parsed.params, state, headerOut);
|
|
131
|
-
const headerBatch = buildResultBatch(
|
|
132
|
-
|
|
133
|
-
headerValues,
|
|
134
|
-
ctx.serverId,
|
|
135
|
-
parsed.requestId,
|
|
136
|
-
);
|
|
137
|
-
const headerBatches = [
|
|
138
|
-
...headerOut.batches.map((b) => b.batch),
|
|
139
|
-
headerBatch,
|
|
140
|
-
];
|
|
111
|
+
const headerBatch = buildResultBatch(method.headerSchema, headerValues, ctx.serverId, parsed.requestId);
|
|
112
|
+
const headerBatches = [...headerOut.batches.map((b) => b.batch), headerBatch];
|
|
141
113
|
headerBytes = serializeIpcStream(method.headerSchema, headerBatches);
|
|
142
114
|
} catch (error: any) {
|
|
143
|
-
const errBatch = buildErrorBatch(
|
|
144
|
-
method.headerSchema,
|
|
145
|
-
error,
|
|
146
|
-
ctx.serverId,
|
|
147
|
-
parsed.requestId,
|
|
148
|
-
);
|
|
115
|
+
const errBatch = buildErrorBatch(method.headerSchema, error, ctx.serverId, parsed.requestId);
|
|
149
116
|
return arrowResponse(serializeIpcStream(method.headerSchema, [errBatch]), 500);
|
|
150
117
|
}
|
|
151
118
|
}
|
|
@@ -154,26 +121,13 @@ export async function httpDispatchStreamInit(
|
|
|
154
121
|
// Producer method — produce data inline in the init response.
|
|
155
122
|
// For exchange-registered methods acting as producers (__isProducer),
|
|
156
123
|
// produceStreamResponse falls back to exchangeFn with tick batches.
|
|
157
|
-
return produceStreamResponse(
|
|
158
|
-
method,
|
|
159
|
-
state,
|
|
160
|
-
resolvedOutputSchema,
|
|
161
|
-
inputSchema,
|
|
162
|
-
ctx,
|
|
163
|
-
parsed.requestId,
|
|
164
|
-
headerBytes,
|
|
165
|
-
);
|
|
124
|
+
return produceStreamResponse(method, state, resolvedOutputSchema, inputSchema, ctx, parsed.requestId, headerBytes);
|
|
166
125
|
} else {
|
|
167
126
|
// Exchange: serialize state into signed token, return zero-row batch with token
|
|
168
127
|
const stateBytes = ctx.stateSerializer.serialize(state);
|
|
169
128
|
const schemaBytes = serializeSchema(resolvedOutputSchema);
|
|
170
129
|
const inputSchemaBytes = serializeSchema(inputSchema);
|
|
171
|
-
const token = packStateToken(
|
|
172
|
-
stateBytes,
|
|
173
|
-
schemaBytes,
|
|
174
|
-
inputSchemaBytes,
|
|
175
|
-
ctx.signingKey,
|
|
176
|
-
);
|
|
130
|
+
const token = packStateToken(stateBytes, schemaBytes, inputSchemaBytes, ctx.signingKey);
|
|
177
131
|
|
|
178
132
|
const tokenMeta = new Map<string, string>();
|
|
179
133
|
tokenMeta.set(STATE_KEY, token);
|
|
@@ -207,7 +161,7 @@ export async function httpDispatchStreamExchange(
|
|
|
207
161
|
throw new HttpRpcError("Missing state token in exchange request", 400);
|
|
208
162
|
}
|
|
209
163
|
|
|
210
|
-
let unpacked;
|
|
164
|
+
let unpacked: import("./token.js").UnpackedToken;
|
|
211
165
|
try {
|
|
212
166
|
unpacked = unpackStateToken(tokenBase64, ctx.signingKey, ctx.tokenTtl);
|
|
213
167
|
} catch (error: any) {
|
|
@@ -237,20 +191,15 @@ export async function httpDispatchStreamExchange(
|
|
|
237
191
|
inputSchema = method.inputSchema ?? EMPTY_SCHEMA;
|
|
238
192
|
}
|
|
239
193
|
const effectiveProducer = state?.__isProducer ?? isProducer;
|
|
240
|
-
if (process.env.VGI_DISPATCH_DEBUG)
|
|
194
|
+
if (process.env.VGI_DISPATCH_DEBUG)
|
|
195
|
+
console.error(
|
|
196
|
+
`[httpDispatchStreamExchange] method=${method.name} effectiveProducer=${effectiveProducer} stateKeys=${Object.keys(state || {})}`,
|
|
197
|
+
);
|
|
241
198
|
|
|
242
199
|
if (effectiveProducer) {
|
|
243
200
|
// Producer continuation — produce more data inline.
|
|
244
201
|
// For exchange-registered methods, falls back to exchangeFn with tick batches.
|
|
245
|
-
return produceStreamResponse(
|
|
246
|
-
method,
|
|
247
|
-
state,
|
|
248
|
-
outputSchema,
|
|
249
|
-
inputSchema,
|
|
250
|
-
ctx,
|
|
251
|
-
null,
|
|
252
|
-
null,
|
|
253
|
-
);
|
|
202
|
+
return produceStreamResponse(method, state, outputSchema, inputSchema, ctx, null, null);
|
|
254
203
|
} else {
|
|
255
204
|
// Exchange path — also handles exchange-registered methods acting as
|
|
256
205
|
// producers (__isProducer=true). Use producer mode on the OutputCollector
|
|
@@ -264,7 +213,12 @@ export async function httpDispatchStreamExchange(
|
|
|
264
213
|
await method.producerFn!(state, out);
|
|
265
214
|
}
|
|
266
215
|
} catch (error: any) {
|
|
267
|
-
if (process.env.VGI_DISPATCH_DEBUG)
|
|
216
|
+
if (process.env.VGI_DISPATCH_DEBUG)
|
|
217
|
+
console.error(
|
|
218
|
+
`[httpDispatchStreamExchange] exchange handler error:`,
|
|
219
|
+
error.message,
|
|
220
|
+
error.stack?.split("\n").slice(0, 5).join("\n"),
|
|
221
|
+
);
|
|
268
222
|
const errBatch = buildErrorBatch(outputSchema, error, ctx.serverId, null);
|
|
269
223
|
return arrowResponse(serializeIpcStream(outputSchema, [errBatch]), 500);
|
|
270
224
|
}
|
|
@@ -283,12 +237,7 @@ export async function httpDispatchStreamExchange(
|
|
|
283
237
|
const stateBytes = ctx.stateSerializer.serialize(state);
|
|
284
238
|
const schemaBytes = serializeSchema(outputSchema);
|
|
285
239
|
const inputSchemaBytes = serializeSchema(inputSchema);
|
|
286
|
-
const token = packStateToken(
|
|
287
|
-
stateBytes,
|
|
288
|
-
schemaBytes,
|
|
289
|
-
inputSchemaBytes,
|
|
290
|
-
ctx.signingKey,
|
|
291
|
-
);
|
|
240
|
+
const token = packStateToken(stateBytes, schemaBytes, inputSchemaBytes, ctx.signingKey);
|
|
292
241
|
|
|
293
242
|
for (const emitted of out.batches) {
|
|
294
243
|
const batch = emitted.batch;
|
|
@@ -304,7 +253,7 @@ export async function httpDispatchStreamExchange(
|
|
|
304
253
|
// Safety net: if no batch carries a state token (e.g. all rows were
|
|
305
254
|
// filtered out by pushdown filters), emit an empty batch with the
|
|
306
255
|
// token so the client knows to continue exchanging.
|
|
307
|
-
if (!batches.some(b => b.metadata?.get(STATE_KEY))) {
|
|
256
|
+
if (!batches.some((b) => b.metadata?.get(STATE_KEY))) {
|
|
308
257
|
const tokenMeta = new Map<string, string>();
|
|
309
258
|
tokenMeta.set(STATE_KEY, token);
|
|
310
259
|
batches.push(buildEmptyBatch(outputSchema, tokenMeta));
|
|
@@ -344,7 +293,8 @@ async function produceStreamResponse(
|
|
|
344
293
|
await method.exchangeFn!(state, tickBatch, out);
|
|
345
294
|
}
|
|
346
295
|
} catch (error: any) {
|
|
347
|
-
if (process.env.VGI_DISPATCH_DEBUG)
|
|
296
|
+
if (process.env.VGI_DISPATCH_DEBUG)
|
|
297
|
+
console.error(`[produceStreamResponse] error:`, error.message, error.stack?.split("\n").slice(0, 3).join("\n"));
|
|
348
298
|
allBatches.push(buildErrorBatch(outputSchema, error, ctx.serverId, requestId));
|
|
349
299
|
break;
|
|
350
300
|
}
|
|
@@ -365,12 +315,7 @@ async function produceStreamResponse(
|
|
|
365
315
|
const stateBytes = ctx.stateSerializer.serialize(state);
|
|
366
316
|
const schemaBytes = serializeSchema(outputSchema);
|
|
367
317
|
const inputSchemaBytes = serializeSchema(inputSchema);
|
|
368
|
-
const token = packStateToken(
|
|
369
|
-
stateBytes,
|
|
370
|
-
schemaBytes,
|
|
371
|
-
inputSchemaBytes,
|
|
372
|
-
ctx.signingKey,
|
|
373
|
-
);
|
|
318
|
+
const token = packStateToken(stateBytes, schemaBytes, inputSchemaBytes, ctx.signingKey);
|
|
374
319
|
const tokenMeta = new Map<string, string>();
|
|
375
320
|
tokenMeta.set(STATE_KEY, token);
|
|
376
321
|
allBatches.push(buildEmptyBatch(outputSchema, tokenMeta));
|
package/src/http/handler.ts
CHANGED
|
@@ -1,26 +1,21 @@
|
|
|
1
1
|
// © Copyright 2025-2026, Query.Farm LLC - https://query.farm
|
|
2
2
|
// SPDX-License-Identifier: Apache-2.0
|
|
3
3
|
|
|
4
|
-
import { Schema } from "@query-farm/apache-arrow";
|
|
5
4
|
import { randomBytes } from "node:crypto";
|
|
5
|
+
import { Schema } from "@query-farm/apache-arrow";
|
|
6
|
+
import { DESCRIBE_METHOD_NAME } from "../constants.js";
|
|
6
7
|
import type { Protocol } from "../protocol.js";
|
|
7
8
|
import { MethodType } from "../types.js";
|
|
8
|
-
import {
|
|
9
|
+
import { zstdCompress, zstdDecompress } from "../util/zstd.js";
|
|
9
10
|
import { buildErrorBatch } from "../wire/response.js";
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
12
|
-
ARROW_CONTENT_TYPE,
|
|
13
|
-
HttpRpcError,
|
|
14
|
-
serializeIpcStream,
|
|
15
|
-
arrowResponse,
|
|
16
|
-
} from "./common.js";
|
|
11
|
+
import { ARROW_CONTENT_TYPE, arrowResponse, HttpRpcError, serializeIpcStream } from "./common.js";
|
|
17
12
|
import {
|
|
18
13
|
httpDispatchDescribe,
|
|
19
|
-
httpDispatchUnary,
|
|
20
|
-
httpDispatchStreamInit,
|
|
21
14
|
httpDispatchStreamExchange,
|
|
15
|
+
httpDispatchStreamInit,
|
|
16
|
+
httpDispatchUnary,
|
|
22
17
|
} from "./dispatch.js";
|
|
23
|
-
import {
|
|
18
|
+
import { type HttpHandlerOptions, jsonStateSerializer } from "./types.js";
|
|
24
19
|
|
|
25
20
|
const EMPTY_SCHEMA = new Schema([]);
|
|
26
21
|
|
|
@@ -46,8 +41,7 @@ export function createHttpHandler(
|
|
|
46
41
|
const corsOrigins = options?.corsOrigins;
|
|
47
42
|
const maxRequestBytes = options?.maxRequestBytes;
|
|
48
43
|
const maxStreamResponseBytes = options?.maxStreamResponseBytes;
|
|
49
|
-
const serverId =
|
|
50
|
-
options?.serverId ?? crypto.randomUUID().replace(/-/g, "").slice(0, 12);
|
|
44
|
+
const serverId = options?.serverId ?? crypto.randomUUID().replace(/-/g, "").slice(0, 12);
|
|
51
45
|
|
|
52
46
|
const methods = protocol.getMethods();
|
|
53
47
|
|
|
@@ -70,10 +64,7 @@ export function createHttpHandler(
|
|
|
70
64
|
}
|
|
71
65
|
}
|
|
72
66
|
|
|
73
|
-
async function compressIfAccepted(
|
|
74
|
-
response: Response,
|
|
75
|
-
clientAcceptsZstd: boolean,
|
|
76
|
-
): Promise<Response> {
|
|
67
|
+
async function compressIfAccepted(response: Response, clientAcceptsZstd: boolean): Promise<Response> {
|
|
77
68
|
if (compressionLevel == null || !clientAcceptsZstd) return response;
|
|
78
69
|
const responseBody = new Uint8Array(await response.arrayBuffer());
|
|
79
70
|
const compressed = zstdCompress(responseBody, compressionLevel);
|
|
@@ -85,11 +76,7 @@ export function createHttpHandler(
|
|
|
85
76
|
});
|
|
86
77
|
}
|
|
87
78
|
|
|
88
|
-
function makeErrorResponse(
|
|
89
|
-
error: Error,
|
|
90
|
-
statusCode: number,
|
|
91
|
-
schema: Schema = EMPTY_SCHEMA,
|
|
92
|
-
): Response {
|
|
79
|
+
function makeErrorResponse(error: Error, statusCode: number, schema: Schema = EMPTY_SCHEMA): Response {
|
|
93
80
|
const errBatch = buildErrorBatch(schema, error, serverId, null);
|
|
94
81
|
const body = serializeIpcStream(schema, [errBatch]);
|
|
95
82
|
const resp = arrowResponse(body, statusCode);
|
|
@@ -128,23 +115,18 @@ export function createHttpHandler(
|
|
|
128
115
|
// Validate Content-Type
|
|
129
116
|
const contentType = request.headers.get("Content-Type");
|
|
130
117
|
if (!contentType || !contentType.includes(ARROW_CONTENT_TYPE)) {
|
|
131
|
-
return new Response(
|
|
132
|
-
`Unsupported Media Type: expected ${ARROW_CONTENT_TYPE}`,
|
|
133
|
-
{ status: 415 },
|
|
134
|
-
);
|
|
118
|
+
return new Response(`Unsupported Media Type: expected ${ARROW_CONTENT_TYPE}`, { status: 415 });
|
|
135
119
|
}
|
|
136
120
|
|
|
137
121
|
// Check request body size
|
|
138
122
|
if (maxRequestBytes != null) {
|
|
139
123
|
const contentLength = request.headers.get("Content-Length");
|
|
140
|
-
if (contentLength && parseInt(contentLength) > maxRequestBytes) {
|
|
124
|
+
if (contentLength && parseInt(contentLength, 10) > maxRequestBytes) {
|
|
141
125
|
return new Response("Request body too large", { status: 413 });
|
|
142
126
|
}
|
|
143
127
|
}
|
|
144
128
|
|
|
145
|
-
const clientAcceptsZstd = (
|
|
146
|
-
request.headers.get("Accept-Encoding") ?? ""
|
|
147
|
-
).includes("zstd");
|
|
129
|
+
const clientAcceptsZstd = (request.headers.get("Accept-Encoding") ?? "").includes("zstd");
|
|
148
130
|
|
|
149
131
|
// Read body, decompressing if needed
|
|
150
132
|
let body = new Uint8Array(await request.arrayBuffer());
|
|
@@ -160,15 +142,12 @@ export function createHttpHandler(
|
|
|
160
142
|
addCorsHeaders(response.headers);
|
|
161
143
|
return compressIfAccepted(response, clientAcceptsZstd);
|
|
162
144
|
} catch (error: any) {
|
|
163
|
-
return compressIfAccepted(
|
|
164
|
-
makeErrorResponse(error, 500),
|
|
165
|
-
clientAcceptsZstd,
|
|
166
|
-
);
|
|
145
|
+
return compressIfAccepted(makeErrorResponse(error, 500), clientAcceptsZstd);
|
|
167
146
|
}
|
|
168
147
|
}
|
|
169
148
|
|
|
170
149
|
// Parse method name and sub-path from URL
|
|
171
|
-
if (!path.startsWith(prefix
|
|
150
|
+
if (!path.startsWith(`${prefix}/`)) {
|
|
172
151
|
return new Response("Not Found", { status: 404 });
|
|
173
152
|
}
|
|
174
153
|
|
|
@@ -191,13 +170,8 @@ export function createHttpHandler(
|
|
|
191
170
|
const method = methods.get(methodName);
|
|
192
171
|
if (!method) {
|
|
193
172
|
const available = [...methods.keys()].sort();
|
|
194
|
-
const err = new Error(
|
|
195
|
-
|
|
196
|
-
);
|
|
197
|
-
return compressIfAccepted(
|
|
198
|
-
makeErrorResponse(err, 404),
|
|
199
|
-
clientAcceptsZstd,
|
|
200
|
-
);
|
|
173
|
+
const err = new Error(`Unknown method: '${methodName}'. Available methods: [${available.join(", ")}]`);
|
|
174
|
+
return compressIfAccepted(makeErrorResponse(err, 404), clientAcceptsZstd);
|
|
201
175
|
}
|
|
202
176
|
|
|
203
177
|
try {
|
|
@@ -205,10 +179,7 @@ export function createHttpHandler(
|
|
|
205
179
|
|
|
206
180
|
if (action === "call") {
|
|
207
181
|
if (method.type !== MethodType.UNARY) {
|
|
208
|
-
throw new HttpRpcError(
|
|
209
|
-
`Method '${methodName}' is a stream method. Use /init and /exchange endpoints.`,
|
|
210
|
-
400,
|
|
211
|
-
);
|
|
182
|
+
throw new HttpRpcError(`Method '${methodName}' is a stream method. Use /init and /exchange endpoints.`, 400);
|
|
212
183
|
}
|
|
213
184
|
response = await httpDispatchUnary(method, body, ctx);
|
|
214
185
|
} else if (action === "init") {
|
|
@@ -233,15 +204,9 @@ export function createHttpHandler(
|
|
|
233
204
|
return compressIfAccepted(response, clientAcceptsZstd);
|
|
234
205
|
} catch (error: any) {
|
|
235
206
|
if (error instanceof HttpRpcError) {
|
|
236
|
-
return compressIfAccepted(
|
|
237
|
-
makeErrorResponse(error, error.statusCode),
|
|
238
|
-
clientAcceptsZstd,
|
|
239
|
-
);
|
|
207
|
+
return compressIfAccepted(makeErrorResponse(error, error.statusCode), clientAcceptsZstd);
|
|
240
208
|
}
|
|
241
|
-
return compressIfAccepted(
|
|
242
|
-
makeErrorResponse(error, 500),
|
|
243
|
-
clientAcceptsZstd,
|
|
244
|
-
);
|
|
209
|
+
return compressIfAccepted(makeErrorResponse(error, 500), clientAcceptsZstd);
|
|
245
210
|
}
|
|
246
211
|
};
|
|
247
212
|
}
|
package/src/http/index.ts
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
// © Copyright 2025-2026, Query.Farm LLC - https://query.farm
|
|
2
2
|
// SPDX-License-Identifier: Apache-2.0
|
|
3
3
|
|
|
4
|
+
export { ARROW_CONTENT_TYPE } from "./common.js";
|
|
4
5
|
export { createHttpHandler } from "./handler.js";
|
|
6
|
+
export { type UnpackedToken, unpackStateToken } from "./token.js";
|
|
5
7
|
export type { HttpHandlerOptions, StateSerializer } from "./types.js";
|
|
6
8
|
export { jsonStateSerializer } from "./types.js";
|
|
7
|
-
export { ARROW_CONTENT_TYPE } from "./common.js";
|
|
8
|
-
export { unpackStateToken, type UnpackedToken } from "./token.js";
|
package/src/http/token.ts
CHANGED
|
@@ -28,8 +28,7 @@ export function packStateToken(
|
|
|
28
28
|
): string {
|
|
29
29
|
const now = createdAt ?? Math.floor(Date.now() / 1000);
|
|
30
30
|
|
|
31
|
-
const payloadLen =
|
|
32
|
-
1 + 8 + 4 + stateBytes.length + 4 + schemaBytes.length + 4 + inputSchemaBytes.length;
|
|
31
|
+
const payloadLen = 1 + 8 + 4 + stateBytes.length + 4 + schemaBytes.length + 4 + inputSchemaBytes.length;
|
|
33
32
|
const buf = Buffer.alloc(payloadLen);
|
|
34
33
|
let offset = 0;
|
|
35
34
|
|
|
@@ -77,11 +76,7 @@ export interface UnpackedToken {
|
|
|
77
76
|
* Unpack and verify a state token.
|
|
78
77
|
* Throws on tampered, expired, or malformed tokens.
|
|
79
78
|
*/
|
|
80
|
-
export function unpackStateToken(
|
|
81
|
-
tokenBase64: string,
|
|
82
|
-
signingKey: Uint8Array,
|
|
83
|
-
tokenTtl: number,
|
|
84
|
-
): UnpackedToken {
|
|
79
|
+
export function unpackStateToken(tokenBase64: string, signingKey: Uint8Array, tokenTtl: number): UnpackedToken {
|
|
85
80
|
const token = Buffer.from(tokenBase64, "base64");
|
|
86
81
|
|
|
87
82
|
if (token.length < MIN_TOKEN_LEN) {
|
package/src/http/types.ts
CHANGED
|
@@ -34,16 +34,12 @@ export interface StateSerializer {
|
|
|
34
34
|
export const jsonStateSerializer: StateSerializer = {
|
|
35
35
|
serialize(state: any): Uint8Array {
|
|
36
36
|
return new TextEncoder().encode(
|
|
37
|
-
JSON.stringify(state, (_key, value) =>
|
|
38
|
-
typeof value === "bigint" ? `__bigint__:${value}` : value,
|
|
39
|
-
),
|
|
37
|
+
JSON.stringify(state, (_key, value) => (typeof value === "bigint" ? `__bigint__:${value}` : value)),
|
|
40
38
|
);
|
|
41
39
|
},
|
|
42
40
|
deserialize(bytes: Uint8Array): any {
|
|
43
41
|
return JSON.parse(new TextDecoder().decode(bytes), (_key, value) =>
|
|
44
|
-
typeof value === "string" && value.startsWith("__bigint__:")
|
|
45
|
-
? BigInt(value.slice(11))
|
|
46
|
-
: value,
|
|
42
|
+
typeof value === "string" && value.startsWith("__bigint__:") ? BigInt(value.slice(11)) : value,
|
|
47
43
|
);
|
|
48
44
|
},
|
|
49
45
|
};
|