@query-farm/vgi-rpc 0.2.4 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/dist/client/connect.d.ts.map +1 -1
- package/dist/client/index.d.ts +3 -3
- package/dist/client/index.d.ts.map +1 -1
- package/dist/client/introspect.d.ts +1 -1
- package/dist/client/introspect.d.ts.map +1 -1
- package/dist/client/ipc.d.ts +1 -1
- package/dist/client/ipc.d.ts.map +1 -1
- package/dist/client/pipe.d.ts +2 -2
- package/dist/client/pipe.d.ts.map +1 -1
- package/dist/client/stream.d.ts +1 -1
- package/dist/client/stream.d.ts.map +1 -1
- package/dist/dispatch/describe.d.ts +1 -1
- package/dist/dispatch/describe.d.ts.map +1 -1
- package/dist/dispatch/stream.d.ts +1 -1
- package/dist/dispatch/stream.d.ts.map +1 -1
- package/dist/dispatch/unary.d.ts.map +1 -1
- package/dist/http/common.d.ts +1 -1
- package/dist/http/common.d.ts.map +1 -1
- package/dist/http/dispatch.d.ts.map +1 -1
- package/dist/http/handler.d.ts.map +1 -1
- package/dist/http/index.d.ts +3 -1
- package/dist/http/index.d.ts.map +1 -1
- package/dist/http/token.d.ts.map +1 -1
- package/dist/http/types.d.ts.map +1 -1
- package/dist/index.d.ts +7 -7
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2162 -2119
- package/dist/index.js.map +26 -26
- package/dist/protocol.d.ts +1 -1
- package/dist/protocol.d.ts.map +1 -1
- package/dist/schema.d.ts +1 -1
- package/dist/schema.d.ts.map +1 -1
- package/dist/server.d.ts +1 -1
- package/dist/server.d.ts.map +1 -1
- package/dist/types.d.ts +1 -1
- package/dist/types.d.ts.map +1 -1
- package/dist/util/schema.d.ts +1 -1
- package/dist/util/schema.d.ts.map +1 -1
- package/dist/util/zstd.d.ts.map +1 -1
- package/dist/wire/reader.d.ts +1 -1
- package/dist/wire/reader.d.ts.map +1 -1
- package/dist/wire/request.d.ts +1 -1
- package/dist/wire/request.d.ts.map +1 -1
- package/dist/wire/response.d.ts +1 -1
- package/dist/wire/response.d.ts.map +1 -1
- package/dist/wire/writer.d.ts +1 -1
- package/dist/wire/writer.d.ts.map +1 -1
- package/package.json +9 -5
- package/src/client/connect.ts +12 -20
- package/src/client/index.ts +8 -8
- package/src/client/introspect.ts +11 -15
- package/src/client/ipc.ts +18 -32
- package/src/client/pipe.ts +17 -37
- package/src/client/stream.ts +20 -46
- package/src/dispatch/describe.ts +15 -27
- package/src/dispatch/stream.ts +7 -21
- package/src/dispatch/unary.ts +1 -2
- package/src/http/common.ts +9 -19
- package/src/http/dispatch.ts +115 -110
- package/src/http/handler.ts +20 -55
- package/src/http/index.ts +3 -1
- package/src/http/token.ts +2 -7
- package/src/http/types.ts +2 -6
- package/src/index.ts +44 -41
- package/src/protocol.ts +8 -8
- package/src/schema.ts +12 -16
- package/src/server.ts +16 -36
- package/src/types.ts +9 -36
- package/src/util/schema.ts +1 -1
- package/src/util/zstd.ts +2 -8
- package/src/wire/reader.ts +2 -4
- package/src/wire/request.ts +4 -15
- package/src/wire/response.ts +9 -25
- package/src/wire/writer.ts +1 -5
package/src/http/handler.ts
CHANGED
|
@@ -1,26 +1,21 @@
|
|
|
1
1
|
// © Copyright 2025-2026, Query.Farm LLC - https://query.farm
|
|
2
2
|
// SPDX-License-Identifier: Apache-2.0
|
|
3
3
|
|
|
4
|
-
import { Schema } from "apache-arrow";
|
|
5
4
|
import { randomBytes } from "node:crypto";
|
|
5
|
+
import { Schema } from "@query-farm/apache-arrow";
|
|
6
|
+
import { DESCRIBE_METHOD_NAME } from "../constants.js";
|
|
6
7
|
import type { Protocol } from "../protocol.js";
|
|
7
8
|
import { MethodType } from "../types.js";
|
|
8
|
-
import {
|
|
9
|
+
import { zstdCompress, zstdDecompress } from "../util/zstd.js";
|
|
9
10
|
import { buildErrorBatch } from "../wire/response.js";
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
12
|
-
ARROW_CONTENT_TYPE,
|
|
13
|
-
HttpRpcError,
|
|
14
|
-
serializeIpcStream,
|
|
15
|
-
arrowResponse,
|
|
16
|
-
} from "./common.js";
|
|
11
|
+
import { ARROW_CONTENT_TYPE, arrowResponse, HttpRpcError, serializeIpcStream } from "./common.js";
|
|
17
12
|
import {
|
|
18
13
|
httpDispatchDescribe,
|
|
19
|
-
httpDispatchUnary,
|
|
20
|
-
httpDispatchStreamInit,
|
|
21
14
|
httpDispatchStreamExchange,
|
|
15
|
+
httpDispatchStreamInit,
|
|
16
|
+
httpDispatchUnary,
|
|
22
17
|
} from "./dispatch.js";
|
|
23
|
-
import {
|
|
18
|
+
import { type HttpHandlerOptions, jsonStateSerializer } from "./types.js";
|
|
24
19
|
|
|
25
20
|
const EMPTY_SCHEMA = new Schema([]);
|
|
26
21
|
|
|
@@ -46,8 +41,7 @@ export function createHttpHandler(
|
|
|
46
41
|
const corsOrigins = options?.corsOrigins;
|
|
47
42
|
const maxRequestBytes = options?.maxRequestBytes;
|
|
48
43
|
const maxStreamResponseBytes = options?.maxStreamResponseBytes;
|
|
49
|
-
const serverId =
|
|
50
|
-
options?.serverId ?? crypto.randomUUID().replace(/-/g, "").slice(0, 12);
|
|
44
|
+
const serverId = options?.serverId ?? crypto.randomUUID().replace(/-/g, "").slice(0, 12);
|
|
51
45
|
|
|
52
46
|
const methods = protocol.getMethods();
|
|
53
47
|
|
|
@@ -70,10 +64,7 @@ export function createHttpHandler(
|
|
|
70
64
|
}
|
|
71
65
|
}
|
|
72
66
|
|
|
73
|
-
async function compressIfAccepted(
|
|
74
|
-
response: Response,
|
|
75
|
-
clientAcceptsZstd: boolean,
|
|
76
|
-
): Promise<Response> {
|
|
67
|
+
async function compressIfAccepted(response: Response, clientAcceptsZstd: boolean): Promise<Response> {
|
|
77
68
|
if (compressionLevel == null || !clientAcceptsZstd) return response;
|
|
78
69
|
const responseBody = new Uint8Array(await response.arrayBuffer());
|
|
79
70
|
const compressed = zstdCompress(responseBody, compressionLevel);
|
|
@@ -85,11 +76,7 @@ export function createHttpHandler(
|
|
|
85
76
|
});
|
|
86
77
|
}
|
|
87
78
|
|
|
88
|
-
function makeErrorResponse(
|
|
89
|
-
error: Error,
|
|
90
|
-
statusCode: number,
|
|
91
|
-
schema: Schema = EMPTY_SCHEMA,
|
|
92
|
-
): Response {
|
|
79
|
+
function makeErrorResponse(error: Error, statusCode: number, schema: Schema = EMPTY_SCHEMA): Response {
|
|
93
80
|
const errBatch = buildErrorBatch(schema, error, serverId, null);
|
|
94
81
|
const body = serializeIpcStream(schema, [errBatch]);
|
|
95
82
|
const resp = arrowResponse(body, statusCode);
|
|
@@ -128,23 +115,18 @@ export function createHttpHandler(
|
|
|
128
115
|
// Validate Content-Type
|
|
129
116
|
const contentType = request.headers.get("Content-Type");
|
|
130
117
|
if (!contentType || !contentType.includes(ARROW_CONTENT_TYPE)) {
|
|
131
|
-
return new Response(
|
|
132
|
-
`Unsupported Media Type: expected ${ARROW_CONTENT_TYPE}`,
|
|
133
|
-
{ status: 415 },
|
|
134
|
-
);
|
|
118
|
+
return new Response(`Unsupported Media Type: expected ${ARROW_CONTENT_TYPE}`, { status: 415 });
|
|
135
119
|
}
|
|
136
120
|
|
|
137
121
|
// Check request body size
|
|
138
122
|
if (maxRequestBytes != null) {
|
|
139
123
|
const contentLength = request.headers.get("Content-Length");
|
|
140
|
-
if (contentLength && parseInt(contentLength) > maxRequestBytes) {
|
|
124
|
+
if (contentLength && parseInt(contentLength, 10) > maxRequestBytes) {
|
|
141
125
|
return new Response("Request body too large", { status: 413 });
|
|
142
126
|
}
|
|
143
127
|
}
|
|
144
128
|
|
|
145
|
-
const clientAcceptsZstd = (
|
|
146
|
-
request.headers.get("Accept-Encoding") ?? ""
|
|
147
|
-
).includes("zstd");
|
|
129
|
+
const clientAcceptsZstd = (request.headers.get("Accept-Encoding") ?? "").includes("zstd");
|
|
148
130
|
|
|
149
131
|
// Read body, decompressing if needed
|
|
150
132
|
let body = new Uint8Array(await request.arrayBuffer());
|
|
@@ -160,15 +142,12 @@ export function createHttpHandler(
|
|
|
160
142
|
addCorsHeaders(response.headers);
|
|
161
143
|
return compressIfAccepted(response, clientAcceptsZstd);
|
|
162
144
|
} catch (error: any) {
|
|
163
|
-
return compressIfAccepted(
|
|
164
|
-
makeErrorResponse(error, 500),
|
|
165
|
-
clientAcceptsZstd,
|
|
166
|
-
);
|
|
145
|
+
return compressIfAccepted(makeErrorResponse(error, 500), clientAcceptsZstd);
|
|
167
146
|
}
|
|
168
147
|
}
|
|
169
148
|
|
|
170
149
|
// Parse method name and sub-path from URL
|
|
171
|
-
if (!path.startsWith(prefix
|
|
150
|
+
if (!path.startsWith(`${prefix}/`)) {
|
|
172
151
|
return new Response("Not Found", { status: 404 });
|
|
173
152
|
}
|
|
174
153
|
|
|
@@ -191,13 +170,8 @@ export function createHttpHandler(
|
|
|
191
170
|
const method = methods.get(methodName);
|
|
192
171
|
if (!method) {
|
|
193
172
|
const available = [...methods.keys()].sort();
|
|
194
|
-
const err = new Error(
|
|
195
|
-
|
|
196
|
-
);
|
|
197
|
-
return compressIfAccepted(
|
|
198
|
-
makeErrorResponse(err, 404),
|
|
199
|
-
clientAcceptsZstd,
|
|
200
|
-
);
|
|
173
|
+
const err = new Error(`Unknown method: '${methodName}'. Available methods: [${available.join(", ")}]`);
|
|
174
|
+
return compressIfAccepted(makeErrorResponse(err, 404), clientAcceptsZstd);
|
|
201
175
|
}
|
|
202
176
|
|
|
203
177
|
try {
|
|
@@ -205,10 +179,7 @@ export function createHttpHandler(
|
|
|
205
179
|
|
|
206
180
|
if (action === "call") {
|
|
207
181
|
if (method.type !== MethodType.UNARY) {
|
|
208
|
-
throw new HttpRpcError(
|
|
209
|
-
`Method '${methodName}' is a stream method. Use /init and /exchange endpoints.`,
|
|
210
|
-
400,
|
|
211
|
-
);
|
|
182
|
+
throw new HttpRpcError(`Method '${methodName}' is a stream method. Use /init and /exchange endpoints.`, 400);
|
|
212
183
|
}
|
|
213
184
|
response = await httpDispatchUnary(method, body, ctx);
|
|
214
185
|
} else if (action === "init") {
|
|
@@ -233,15 +204,9 @@ export function createHttpHandler(
|
|
|
233
204
|
return compressIfAccepted(response, clientAcceptsZstd);
|
|
234
205
|
} catch (error: any) {
|
|
235
206
|
if (error instanceof HttpRpcError) {
|
|
236
|
-
return compressIfAccepted(
|
|
237
|
-
makeErrorResponse(error, error.statusCode),
|
|
238
|
-
clientAcceptsZstd,
|
|
239
|
-
);
|
|
207
|
+
return compressIfAccepted(makeErrorResponse(error, error.statusCode), clientAcceptsZstd);
|
|
240
208
|
}
|
|
241
|
-
return compressIfAccepted(
|
|
242
|
-
makeErrorResponse(error, 500),
|
|
243
|
-
clientAcceptsZstd,
|
|
244
|
-
);
|
|
209
|
+
return compressIfAccepted(makeErrorResponse(error, 500), clientAcceptsZstd);
|
|
245
210
|
}
|
|
246
211
|
};
|
|
247
212
|
}
|
package/src/http/index.ts
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
// © Copyright 2025-2026, Query.Farm LLC - https://query.farm
|
|
2
2
|
// SPDX-License-Identifier: Apache-2.0
|
|
3
3
|
|
|
4
|
+
export { ARROW_CONTENT_TYPE } from "./common.js";
|
|
4
5
|
export { createHttpHandler } from "./handler.js";
|
|
6
|
+
export { type UnpackedToken, unpackStateToken } from "./token.js";
|
|
5
7
|
export type { HttpHandlerOptions, StateSerializer } from "./types.js";
|
|
6
|
-
export {
|
|
8
|
+
export { jsonStateSerializer } from "./types.js";
|
package/src/http/token.ts
CHANGED
|
@@ -28,8 +28,7 @@ export function packStateToken(
|
|
|
28
28
|
): string {
|
|
29
29
|
const now = createdAt ?? Math.floor(Date.now() / 1000);
|
|
30
30
|
|
|
31
|
-
const payloadLen =
|
|
32
|
-
1 + 8 + 4 + stateBytes.length + 4 + schemaBytes.length + 4 + inputSchemaBytes.length;
|
|
31
|
+
const payloadLen = 1 + 8 + 4 + stateBytes.length + 4 + schemaBytes.length + 4 + inputSchemaBytes.length;
|
|
33
32
|
const buf = Buffer.alloc(payloadLen);
|
|
34
33
|
let offset = 0;
|
|
35
34
|
|
|
@@ -77,11 +76,7 @@ export interface UnpackedToken {
|
|
|
77
76
|
* Unpack and verify a state token.
|
|
78
77
|
* Throws on tampered, expired, or malformed tokens.
|
|
79
78
|
*/
|
|
80
|
-
export function unpackStateToken(
|
|
81
|
-
tokenBase64: string,
|
|
82
|
-
signingKey: Uint8Array,
|
|
83
|
-
tokenTtl: number,
|
|
84
|
-
): UnpackedToken {
|
|
79
|
+
export function unpackStateToken(tokenBase64: string, signingKey: Uint8Array, tokenTtl: number): UnpackedToken {
|
|
85
80
|
const token = Buffer.from(tokenBase64, "base64");
|
|
86
81
|
|
|
87
82
|
if (token.length < MIN_TOKEN_LEN) {
|
package/src/http/types.ts
CHANGED
|
@@ -34,16 +34,12 @@ export interface StateSerializer {
|
|
|
34
34
|
export const jsonStateSerializer: StateSerializer = {
|
|
35
35
|
serialize(state: any): Uint8Array {
|
|
36
36
|
return new TextEncoder().encode(
|
|
37
|
-
JSON.stringify(state, (_key, value) =>
|
|
38
|
-
typeof value === "bigint" ? `__bigint__:${value}` : value,
|
|
39
|
-
),
|
|
37
|
+
JSON.stringify(state, (_key, value) => (typeof value === "bigint" ? `__bigint__:${value}` : value)),
|
|
40
38
|
);
|
|
41
39
|
},
|
|
42
40
|
deserialize(bytes: Uint8Array): any {
|
|
43
41
|
return JSON.parse(new TextDecoder().decode(bytes), (_key, value) =>
|
|
44
|
-
typeof value === "string" && value.startsWith("__bigint__:")
|
|
45
|
-
? BigInt(value.slice(11))
|
|
46
|
-
: value,
|
|
42
|
+
typeof value === "string" && value.startsWith("__bigint__:") ? BigInt(value.slice(11)) : value,
|
|
47
43
|
);
|
|
48
44
|
},
|
|
49
45
|
};
|
package/src/index.ts
CHANGED
|
@@ -1,52 +1,55 @@
|
|
|
1
1
|
// © Copyright 2025-2026, Query.Farm LLC - https://query.farm
|
|
2
2
|
// SPDX-License-Identifier: Apache-2.0
|
|
3
3
|
|
|
4
|
-
export
|
|
5
|
-
export { Protocol } from "./protocol.js";
|
|
6
|
-
export {
|
|
7
|
-
MethodType,
|
|
8
|
-
OutputCollector,
|
|
9
|
-
type LogContext,
|
|
10
|
-
type MethodDefinition,
|
|
11
|
-
type UnaryHandler,
|
|
12
|
-
type HeaderInit,
|
|
13
|
-
type ProducerInit,
|
|
14
|
-
type ProducerFn,
|
|
15
|
-
type ExchangeInit,
|
|
16
|
-
type ExchangeFn,
|
|
17
|
-
} from "./types.js";
|
|
4
|
+
export * from "./client/index.js";
|
|
18
5
|
export {
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
6
|
+
DESCRIBE_METHOD_NAME,
|
|
7
|
+
DESCRIBE_VERSION,
|
|
8
|
+
DESCRIBE_VERSION_KEY,
|
|
9
|
+
LOG_EXTRA_KEY,
|
|
10
|
+
LOG_LEVEL_KEY,
|
|
11
|
+
LOG_MESSAGE_KEY,
|
|
12
|
+
PROTOCOL_NAME_KEY,
|
|
13
|
+
REQUEST_ID_KEY,
|
|
14
|
+
REQUEST_VERSION,
|
|
15
|
+
REQUEST_VERSION_KEY,
|
|
16
|
+
RPC_METHOD_KEY,
|
|
17
|
+
SERVER_ID_KEY,
|
|
18
|
+
STATE_KEY,
|
|
19
|
+
} from "./constants.js";
|
|
30
20
|
export { RpcError, VersionError } from "./errors.js";
|
|
31
21
|
export {
|
|
32
|
-
createHttpHandler,
|
|
33
22
|
ARROW_CONTENT_TYPE,
|
|
23
|
+
createHttpHandler,
|
|
34
24
|
type HttpHandlerOptions,
|
|
25
|
+
jsonStateSerializer,
|
|
35
26
|
type StateSerializer,
|
|
27
|
+
type UnpackedToken,
|
|
28
|
+
unpackStateToken,
|
|
36
29
|
} from "./http/index.js";
|
|
30
|
+
export { Protocol } from "./protocol.js";
|
|
37
31
|
export {
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
32
|
+
bool,
|
|
33
|
+
bytes,
|
|
34
|
+
float,
|
|
35
|
+
float32,
|
|
36
|
+
inferParamTypes,
|
|
37
|
+
int,
|
|
38
|
+
int32,
|
|
39
|
+
type SchemaLike,
|
|
40
|
+
str,
|
|
41
|
+
toSchema,
|
|
42
|
+
} from "./schema.js";
|
|
43
|
+
export { VgiRpcServer } from "./server.js";
|
|
44
|
+
export {
|
|
45
|
+
type ExchangeFn,
|
|
46
|
+
type ExchangeInit,
|
|
47
|
+
type HeaderInit,
|
|
48
|
+
type LogContext,
|
|
49
|
+
type MethodDefinition,
|
|
50
|
+
MethodType,
|
|
51
|
+
OutputCollector,
|
|
52
|
+
type ProducerFn,
|
|
53
|
+
type ProducerInit,
|
|
54
|
+
type UnaryHandler,
|
|
55
|
+
} from "./types.js";
|
package/src/protocol.ts
CHANGED
|
@@ -1,17 +1,17 @@
|
|
|
1
1
|
// © Copyright 2025-2026, Query.Farm LLC - https://query.farm
|
|
2
2
|
// SPDX-License-Identifier: Apache-2.0
|
|
3
3
|
|
|
4
|
-
import { Schema } from "apache-arrow";
|
|
5
|
-
import { type SchemaLike, toSchema
|
|
4
|
+
import { Schema } from "@query-farm/apache-arrow";
|
|
5
|
+
import { inferParamTypes, type SchemaLike, toSchema } from "./schema.js";
|
|
6
6
|
import {
|
|
7
|
-
|
|
8
|
-
type
|
|
9
|
-
type UnaryHandler,
|
|
7
|
+
type ExchangeFn,
|
|
8
|
+
type ExchangeInit,
|
|
10
9
|
type HeaderInit,
|
|
11
|
-
type
|
|
10
|
+
type MethodDefinition,
|
|
11
|
+
MethodType,
|
|
12
12
|
type ProducerFn,
|
|
13
|
-
type
|
|
14
|
-
type
|
|
13
|
+
type ProducerInit,
|
|
14
|
+
type UnaryHandler,
|
|
15
15
|
} from "./types.js";
|
|
16
16
|
|
|
17
17
|
const EMPTY_SCHEMA = new Schema([]);
|
package/src/schema.ts
CHANGED
|
@@ -2,18 +2,18 @@
|
|
|
2
2
|
// SPDX-License-Identifier: Apache-2.0
|
|
3
3
|
|
|
4
4
|
import {
|
|
5
|
-
Schema,
|
|
6
|
-
Field,
|
|
7
|
-
DataType,
|
|
8
|
-
Utf8,
|
|
9
5
|
Binary,
|
|
10
|
-
Int64,
|
|
11
|
-
Int32,
|
|
12
|
-
Int16,
|
|
13
|
-
Float64,
|
|
14
|
-
Float32,
|
|
15
6
|
Bool,
|
|
16
|
-
|
|
7
|
+
DataType,
|
|
8
|
+
Field,
|
|
9
|
+
Float32,
|
|
10
|
+
Float64,
|
|
11
|
+
Int16,
|
|
12
|
+
Int32,
|
|
13
|
+
Int64,
|
|
14
|
+
Schema,
|
|
15
|
+
Utf8,
|
|
16
|
+
} from "@query-farm/apache-arrow";
|
|
17
17
|
|
|
18
18
|
// ---------------------------------------------------------------------------
|
|
19
19
|
// Convenient DataType singletons — re-export so users avoid arrow imports
|
|
@@ -64,9 +64,7 @@ export function toSchema(spec: SchemaLike): Schema {
|
|
|
64
64
|
} else if (value instanceof DataType) {
|
|
65
65
|
fields.push(new Field(name, value, false));
|
|
66
66
|
} else {
|
|
67
|
-
throw new TypeError(
|
|
68
|
-
`Invalid schema value for "${name}": expected DataType or Field, got ${typeof value}`,
|
|
69
|
-
);
|
|
67
|
+
throw new TypeError(`Invalid schema value for "${name}": expected DataType or Field, got ${typeof value}`);
|
|
70
68
|
}
|
|
71
69
|
}
|
|
72
70
|
return new Schema(fields);
|
|
@@ -92,9 +90,7 @@ const TYPE_MAP: [new (...args: any[]) => DataType, string][] = [
|
|
|
92
90
|
* Maps common Arrow scalar types to Python-style type strings.
|
|
93
91
|
* Returns `undefined` if any field has a complex type (List, Map_, Dictionary, etc.).
|
|
94
92
|
*/
|
|
95
|
-
export function inferParamTypes(
|
|
96
|
-
spec: SchemaLike,
|
|
97
|
-
): Record<string, string> | undefined {
|
|
93
|
+
export function inferParamTypes(spec: SchemaLike): Record<string, string> | undefined {
|
|
98
94
|
const schema = toSchema(spec);
|
|
99
95
|
if (schema.fields.length === 0) return undefined;
|
|
100
96
|
|
package/src/server.ts
CHANGED
|
@@ -1,18 +1,18 @@
|
|
|
1
1
|
// © Copyright 2025-2026, Query.Farm LLC - https://query.farm
|
|
2
2
|
// SPDX-License-Identifier: Apache-2.0
|
|
3
3
|
|
|
4
|
-
import { Schema } from "apache-arrow";
|
|
5
|
-
import {
|
|
6
|
-
import { IpcStreamReader } from "./wire/reader.js";
|
|
7
|
-
import { IpcStreamWriter } from "./wire/writer.js";
|
|
8
|
-
import { parseRequest } from "./wire/request.js";
|
|
9
|
-
import { buildErrorBatch } from "./wire/response.js";
|
|
4
|
+
import { Schema } from "@query-farm/apache-arrow";
|
|
5
|
+
import { DESCRIBE_METHOD_NAME } from "./constants.js";
|
|
10
6
|
import { buildDescribeBatch } from "./dispatch/describe.js";
|
|
11
|
-
import { dispatchUnary } from "./dispatch/unary.js";
|
|
12
7
|
import { dispatchStream } from "./dispatch/stream.js";
|
|
13
|
-
import {
|
|
14
|
-
import { MethodType } from "./types.js";
|
|
8
|
+
import { dispatchUnary } from "./dispatch/unary.js";
|
|
15
9
|
import { RpcError, VersionError } from "./errors.js";
|
|
10
|
+
import type { Protocol } from "./protocol.js";
|
|
11
|
+
import { MethodType } from "./types.js";
|
|
12
|
+
import { IpcStreamReader } from "./wire/reader.js";
|
|
13
|
+
import { parseRequest } from "./wire/request.js";
|
|
14
|
+
import { buildErrorBatch } from "./wire/response.js";
|
|
15
|
+
import { IpcStreamWriter } from "./wire/writer.js";
|
|
16
16
|
|
|
17
17
|
const EMPTY_SCHEMA = new Schema([]);
|
|
18
18
|
|
|
@@ -24,23 +24,15 @@ export class VgiRpcServer {
|
|
|
24
24
|
private protocol: Protocol;
|
|
25
25
|
private enableDescribe: boolean;
|
|
26
26
|
private serverId: string;
|
|
27
|
-
private describeBatch: import("apache-arrow").RecordBatch | null = null;
|
|
27
|
+
private describeBatch: import("@query-farm/apache-arrow").RecordBatch | null = null;
|
|
28
28
|
|
|
29
|
-
constructor(
|
|
30
|
-
protocol: Protocol,
|
|
31
|
-
options?: { enableDescribe?: boolean; serverId?: string },
|
|
32
|
-
) {
|
|
29
|
+
constructor(protocol: Protocol, options?: { enableDescribe?: boolean; serverId?: string }) {
|
|
33
30
|
this.protocol = protocol;
|
|
34
31
|
this.enableDescribe = options?.enableDescribe ?? true;
|
|
35
|
-
this.serverId =
|
|
36
|
-
options?.serverId ?? crypto.randomUUID().replace(/-/g, "").slice(0, 12);
|
|
32
|
+
this.serverId = options?.serverId ?? crypto.randomUUID().replace(/-/g, "").slice(0, 12);
|
|
37
33
|
|
|
38
34
|
if (this.enableDescribe) {
|
|
39
|
-
const { batch } = buildDescribeBatch(
|
|
40
|
-
protocol.name,
|
|
41
|
-
protocol.getMethods(),
|
|
42
|
-
this.serverId,
|
|
43
|
-
);
|
|
35
|
+
const { batch } = buildDescribeBatch(protocol.name, protocol.getMethods(), this.serverId);
|
|
44
36
|
this.describeBatch = batch;
|
|
45
37
|
}
|
|
46
38
|
}
|
|
@@ -86,10 +78,7 @@ export class VgiRpcServer {
|
|
|
86
78
|
}
|
|
87
79
|
}
|
|
88
80
|
|
|
89
|
-
private async serveOne(
|
|
90
|
-
reader: IpcStreamReader,
|
|
91
|
-
writer: IpcStreamWriter,
|
|
92
|
-
): Promise<void> {
|
|
81
|
+
private async serveOne(reader: IpcStreamReader, writer: IpcStreamWriter): Promise<void> {
|
|
93
82
|
const stream = await reader.readStream();
|
|
94
83
|
if (!stream) {
|
|
95
84
|
throw new Error("EOF");
|
|
@@ -134,9 +123,7 @@ export class VgiRpcServer {
|
|
|
134
123
|
const method = methods.get(methodName);
|
|
135
124
|
if (!method) {
|
|
136
125
|
const available = [...methods.keys()].sort();
|
|
137
|
-
const err = new Error(
|
|
138
|
-
`Unknown method: '${methodName}'. Available methods: [${available.join(", ")}]`,
|
|
139
|
-
);
|
|
126
|
+
const err = new Error(`Unknown method: '${methodName}'. Available methods: [${available.join(", ")}]`);
|
|
140
127
|
const errBatch = buildErrorBatch(EMPTY_SCHEMA, err, this.serverId, requestId);
|
|
141
128
|
writer.writeStream(EMPTY_SCHEMA, [errBatch]);
|
|
142
129
|
return;
|
|
@@ -146,14 +133,7 @@ export class VgiRpcServer {
|
|
|
146
133
|
if (method.type === MethodType.UNARY) {
|
|
147
134
|
await dispatchUnary(method, params, writer, this.serverId, requestId);
|
|
148
135
|
} else {
|
|
149
|
-
await dispatchStream(
|
|
150
|
-
method,
|
|
151
|
-
params,
|
|
152
|
-
writer,
|
|
153
|
-
reader,
|
|
154
|
-
this.serverId,
|
|
155
|
-
requestId,
|
|
156
|
-
);
|
|
136
|
+
await dispatchStream(method, params, writer, reader, this.serverId, requestId);
|
|
157
137
|
}
|
|
158
138
|
}
|
|
159
139
|
}
|
package/src/types.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// © Copyright 2025-2026, Query.Farm LLC - https://query.farm
|
|
2
2
|
// SPDX-License-Identifier: Apache-2.0
|
|
3
3
|
|
|
4
|
-
import {
|
|
4
|
+
import { RecordBatch, recordBatchFromArrays, type Schema } from "@query-farm/apache-arrow";
|
|
5
5
|
import { buildLogBatch, coerceInt64 } from "./wire/response.js";
|
|
6
6
|
|
|
7
7
|
export enum MethodType {
|
|
@@ -21,32 +21,17 @@ export type UnaryHandler = (
|
|
|
21
21
|
) => Promise<Record<string, any>> | Record<string, any>;
|
|
22
22
|
|
|
23
23
|
/** Initialization function for producer streams. Returns the initial state object. */
|
|
24
|
-
export type ProducerInit<S = any> = (
|
|
25
|
-
params: Record<string, any>,
|
|
26
|
-
) => Promise<S> | S;
|
|
24
|
+
export type ProducerInit<S = any> = (params: Record<string, any>) => Promise<S> | S;
|
|
27
25
|
/** Called repeatedly to produce output batches. Call `out.finish()` to end the stream. */
|
|
28
|
-
export type ProducerFn<S = any> = (
|
|
29
|
-
state: S,
|
|
30
|
-
out: OutputCollector,
|
|
31
|
-
) => Promise<void> | void;
|
|
26
|
+
export type ProducerFn<S = any> = (state: S, out: OutputCollector) => Promise<void> | void;
|
|
32
27
|
|
|
33
28
|
/** Initialization function for exchange streams. Returns the initial state object. */
|
|
34
|
-
export type ExchangeInit<S = any> = (
|
|
35
|
-
params: Record<string, any>,
|
|
36
|
-
) => Promise<S> | S;
|
|
29
|
+
export type ExchangeInit<S = any> = (params: Record<string, any>) => Promise<S> | S;
|
|
37
30
|
/** Called once per input batch. Must emit exactly one output batch per call. */
|
|
38
|
-
export type ExchangeFn<S = any> = (
|
|
39
|
-
state: S,
|
|
40
|
-
input: RecordBatch,
|
|
41
|
-
out: OutputCollector,
|
|
42
|
-
) => Promise<void> | void;
|
|
31
|
+
export type ExchangeFn<S = any> = (state: S, input: RecordBatch, out: OutputCollector) => Promise<void> | void;
|
|
43
32
|
|
|
44
33
|
/** Produces a header batch sent before the first output batch in a stream. */
|
|
45
|
-
export type HeaderInit = (
|
|
46
|
-
params: Record<string, any>,
|
|
47
|
-
state: any,
|
|
48
|
-
ctx: LogContext,
|
|
49
|
-
) => Record<string, any>;
|
|
34
|
+
export type HeaderInit = (params: Record<string, any>, state: any, ctx: LogContext) => Record<string, any>;
|
|
50
35
|
|
|
51
36
|
export interface MethodDefinition {
|
|
52
37
|
name: string;
|
|
@@ -108,10 +93,7 @@ export class OutputCollector implements LogContext {
|
|
|
108
93
|
emit(batch: RecordBatch, metadata?: Map<string, string>): void;
|
|
109
94
|
/** Emit a data batch from column arrays keyed by field name. Int64 Number values are coerced to BigInt. */
|
|
110
95
|
emit(columns: Record<string, any[]>): void;
|
|
111
|
-
emit(
|
|
112
|
-
batchOrColumns: RecordBatch | Record<string, any[]>,
|
|
113
|
-
metadata?: Map<string, string>,
|
|
114
|
-
): void {
|
|
96
|
+
emit(batchOrColumns: RecordBatch | Record<string, any[]>, metadata?: Map<string, string>): void {
|
|
115
97
|
let batch: RecordBatch;
|
|
116
98
|
if (batchOrColumns instanceof RecordBatch) {
|
|
117
99
|
batch = batchOrColumns;
|
|
@@ -139,8 +121,7 @@ export class OutputCollector implements LogContext {
|
|
|
139
121
|
finish(): void {
|
|
140
122
|
if (!this._producerMode) {
|
|
141
123
|
throw new Error(
|
|
142
|
-
"finish() is not allowed on exchange streams; " +
|
|
143
|
-
"exchange streams must emit exactly one data batch per call",
|
|
124
|
+
"finish() is not allowed on exchange streams; " + "exchange streams must emit exactly one data batch per call",
|
|
144
125
|
);
|
|
145
126
|
}
|
|
146
127
|
this._finished = true;
|
|
@@ -148,15 +129,7 @@ export class OutputCollector implements LogContext {
|
|
|
148
129
|
|
|
149
130
|
/** Emit a zero-row client-directed log batch. */
|
|
150
131
|
clientLog(level: string, message: string, extra?: Record<string, string>): void {
|
|
151
|
-
const batch = buildLogBatch(
|
|
152
|
-
this._outputSchema,
|
|
153
|
-
level,
|
|
154
|
-
message,
|
|
155
|
-
extra,
|
|
156
|
-
this._serverId,
|
|
157
|
-
this._requestId,
|
|
158
|
-
);
|
|
132
|
+
const batch = buildLogBatch(this._outputSchema, level, message, extra, this._serverId, this._requestId);
|
|
159
133
|
this._batches.push({ batch });
|
|
160
134
|
}
|
|
161
|
-
|
|
162
135
|
}
|
package/src/util/schema.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// © Copyright 2025-2026, Query.Farm LLC - https://query.farm
|
|
2
2
|
// SPDX-License-Identifier: Apache-2.0
|
|
3
3
|
|
|
4
|
-
import { RecordBatchStreamWriter, type Schema } from "apache-arrow";
|
|
4
|
+
import { RecordBatchStreamWriter, type Schema } from "@query-farm/apache-arrow";
|
|
5
5
|
|
|
6
6
|
/**
|
|
7
7
|
* Serialize a Schema to the Arrow IPC Schema message format.
|
package/src/util/zstd.ts
CHANGED
|
@@ -19,10 +19,7 @@ export function zstdCompress(data: Uint8Array, level: number): Uint8Array<ArrayB
|
|
|
19
19
|
}
|
|
20
20
|
const fn = (zlib as any).zstdCompressSync;
|
|
21
21
|
if (typeof fn !== "function") {
|
|
22
|
-
throw new Error(
|
|
23
|
-
"zstd is not available in this runtime. " +
|
|
24
|
-
"Requires Bun, Node.js >= 22.15, or Deno >= 2.6.9.",
|
|
25
|
-
);
|
|
22
|
+
throw new Error("zstd is not available in this runtime. " + "Requires Bun, Node.js >= 22.15, or Deno >= 2.6.9.");
|
|
26
23
|
}
|
|
27
24
|
return new Uint8Array(
|
|
28
25
|
fn(data, {
|
|
@@ -40,10 +37,7 @@ export function zstdDecompress(data: Uint8Array): Uint8Array<ArrayBuffer> {
|
|
|
40
37
|
}
|
|
41
38
|
const fn = (zlib as any).zstdDecompressSync;
|
|
42
39
|
if (typeof fn !== "function") {
|
|
43
|
-
throw new Error(
|
|
44
|
-
"zstd is not available in this runtime. " +
|
|
45
|
-
"Requires Bun, Node.js >= 22.15, or Deno >= 2.6.9.",
|
|
46
|
-
);
|
|
40
|
+
throw new Error("zstd is not available in this runtime. " + "Requires Bun, Node.js >= 22.15, or Deno >= 2.6.9.");
|
|
47
41
|
}
|
|
48
42
|
return new Uint8Array(fn(data));
|
|
49
43
|
}
|
package/src/wire/reader.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// © Copyright 2025-2026, Query.Farm LLC - https://query.farm
|
|
2
2
|
// SPDX-License-Identifier: Apache-2.0
|
|
3
3
|
|
|
4
|
-
import {
|
|
4
|
+
import { type RecordBatch, RecordBatchReader, type Schema } from "@query-farm/apache-arrow";
|
|
5
5
|
|
|
6
6
|
export interface StreamMessage {
|
|
7
7
|
schema: Schema;
|
|
@@ -23,9 +23,7 @@ export class IpcStreamReader {
|
|
|
23
23
|
this.reader = reader;
|
|
24
24
|
}
|
|
25
25
|
|
|
26
|
-
static async create(
|
|
27
|
-
input: ReadableStream<Uint8Array> | NodeJS.ReadableStream,
|
|
28
|
-
): Promise<IpcStreamReader> {
|
|
26
|
+
static async create(input: ReadableStream<Uint8Array> | NodeJS.ReadableStream): Promise<IpcStreamReader> {
|
|
29
27
|
const reader = await RecordBatchReader.from(input as any);
|
|
30
28
|
await reader.open({ autoDestroy: false });
|
|
31
29
|
if (reader.closed) {
|
package/src/wire/request.ts
CHANGED
|
@@ -1,13 +1,8 @@
|
|
|
1
1
|
// © Copyright 2025-2026, Query.Farm LLC - https://query.farm
|
|
2
2
|
// SPDX-License-Identifier: Apache-2.0
|
|
3
3
|
|
|
4
|
-
import { DataType, type
|
|
5
|
-
import {
|
|
6
|
-
RPC_METHOD_KEY,
|
|
7
|
-
REQUEST_VERSION_KEY,
|
|
8
|
-
REQUEST_VERSION,
|
|
9
|
-
REQUEST_ID_KEY,
|
|
10
|
-
} from "../constants.js";
|
|
4
|
+
import { DataType, type RecordBatch, type Schema } from "@query-farm/apache-arrow";
|
|
5
|
+
import { REQUEST_ID_KEY, REQUEST_VERSION, REQUEST_VERSION_KEY, RPC_METHOD_KEY } from "../constants.js";
|
|
11
6
|
import { RpcError, VersionError } from "../errors.js";
|
|
12
7
|
|
|
13
8
|
export interface ParsedRequest {
|
|
@@ -23,10 +18,7 @@ export interface ParsedRequest {
|
|
|
23
18
|
* Parse a request from a RecordBatch with metadata.
|
|
24
19
|
* Extracts method name, version, and params from the batch.
|
|
25
20
|
*/
|
|
26
|
-
export function parseRequest(
|
|
27
|
-
schema: Schema,
|
|
28
|
-
batch: RecordBatch,
|
|
29
|
-
): ParsedRequest {
|
|
21
|
+
export function parseRequest(schema: Schema, batch: RecordBatch): ParsedRequest {
|
|
30
22
|
const metadata: Map<string, string> = batch.metadata ?? new Map();
|
|
31
23
|
|
|
32
24
|
const methodName = metadata.get(RPC_METHOD_KEY);
|
|
@@ -77,10 +69,7 @@ export function parseRequest(
|
|
|
77
69
|
let value = batch.getChildAt(i)?.get(0);
|
|
78
70
|
// Convert BigInt to Number when safe
|
|
79
71
|
if (typeof value === "bigint") {
|
|
80
|
-
if (
|
|
81
|
-
value >= BigInt(Number.MIN_SAFE_INTEGER) &&
|
|
82
|
-
value <= BigInt(Number.MAX_SAFE_INTEGER)
|
|
83
|
-
) {
|
|
72
|
+
if (value >= BigInt(Number.MIN_SAFE_INTEGER) && value <= BigInt(Number.MAX_SAFE_INTEGER)) {
|
|
84
73
|
value = Number(value);
|
|
85
74
|
}
|
|
86
75
|
}
|