@query-farm/vgi-rpc 0.2.3 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -11
- package/dist/client/introspect.d.ts +1 -1
- package/dist/client/introspect.d.ts.map +1 -1
- package/dist/client/ipc.d.ts +1 -1
- package/dist/client/ipc.d.ts.map +1 -1
- package/dist/client/pipe.d.ts +1 -1
- package/dist/client/pipe.d.ts.map +1 -1
- package/dist/client/stream.d.ts +1 -1
- package/dist/client/stream.d.ts.map +1 -1
- package/dist/dispatch/describe.d.ts +1 -1
- package/dist/dispatch/describe.d.ts.map +1 -1
- package/dist/dispatch/stream.d.ts.map +1 -1
- package/dist/http/common.d.ts +1 -1
- package/dist/http/common.d.ts.map +1 -1
- package/dist/http/dispatch.d.ts.map +1 -1
- package/dist/http/index.d.ts +2 -0
- package/dist/http/index.d.ts.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +101 -48
- package/dist/index.js.map +22 -22
- package/dist/schema.d.ts +1 -1
- package/dist/schema.d.ts.map +1 -1
- package/dist/server.d.ts.map +1 -1
- package/dist/types.d.ts +1 -1
- package/dist/types.d.ts.map +1 -1
- package/dist/util/schema.d.ts +1 -1
- package/dist/util/schema.d.ts.map +1 -1
- package/dist/wire/reader.d.ts +1 -1
- package/dist/wire/reader.d.ts.map +1 -1
- package/dist/wire/request.d.ts +1 -1
- package/dist/wire/request.d.ts.map +1 -1
- package/dist/wire/response.d.ts +1 -1
- package/dist/wire/response.d.ts.map +1 -1
- package/dist/wire/writer.d.ts +1 -1
- package/dist/wire/writer.d.ts.map +1 -1
- package/package.json +4 -4
- package/src/client/connect.ts +1 -1
- package/src/client/introspect.ts +2 -2
- package/src/client/ipc.ts +1 -1
- package/src/client/pipe.ts +1 -1
- package/src/client/stream.ts +1 -1
- package/src/dispatch/describe.ts +1 -1
- package/src/dispatch/stream.ts +2 -3
- package/src/dispatch/unary.ts +1 -1
- package/src/http/common.ts +1 -1
- package/src/http/dispatch.ts +95 -35
- package/src/http/handler.ts +1 -1
- package/src/http/index.ts +2 -0
- package/src/index.ts +3 -0
- package/src/protocol.ts +1 -1
- package/src/schema.ts +1 -1
- package/src/server.ts +2 -2
- package/src/types.ts +1 -1
- package/src/util/schema.ts +1 -1
- package/src/wire/reader.ts +1 -1
- package/src/wire/request.ts +1 -1
- package/src/wire/response.ts +1 -1
- package/src/wire/writer.ts +1 -1
package/README.md
CHANGED
|
@@ -1,19 +1,15 @@
|
|
|
1
|
-
|
|
2
|
-
<a href="https://vgi-rpc.query.farm">
|
|
3
|
-
<img src="https://vgi-rpc.query.farm/logo-hero.png" alt="VGI — The Vector Gateway Interface" width="200">
|
|
4
|
-
</a>
|
|
5
|
-
</p>
|
|
1
|
+
[](https://vgi-rpc.query.farm)
|
|
6
2
|
|
|
7
3
|
# vgi-rpc
|
|
8
4
|
|
|
9
5
|
TypeScript server library for the [vgi-rpc](https://vgi-rpc.query.farm) framework. Implements RPC servers that communicate over stdin/stdout using [Apache Arrow](https://arrow.apache.org/) IPC serialization.
|
|
10
6
|
|
|
11
|
-
Define RPC methods with Arrow-typed schemas, serve them over stdin/stdout, and interact with them using the Python `vgi-rpc` CLI or any vgi-rpc client. Unlike JSON-over-HTTP, structured data stays in Arrow columnar format for efficient transfer.
|
|
7
|
+
Define RPC methods with Arrow-typed schemas, serve them over stdin/stdout, and interact with them using the Python `vgi-rpc` CLI or any `vgi-rpc` client. Unlike JSON-over-HTTP, structured data stays in Arrow columnar format for efficient transfer.
|
|
12
8
|
|
|
13
9
|
**Key features:**
|
|
14
10
|
|
|
15
11
|
- **Three method types** — unary (request-response), producer (server-streaming), and exchange (bidirectional-streaming)
|
|
16
|
-
- **Apache Arrow IPC wire format** — efficient columnar serialization compatible with the Python vgi-rpc framework
|
|
12
|
+
- **Apache Arrow IPC wire format** — efficient columnar serialization compatible with the Python `vgi-rpc` framework
|
|
17
13
|
- **Schema shorthand** — declare schemas with `{ name: str, count: int }` instead of manual `Schema`/`Field` construction
|
|
18
14
|
- **Fluent protocol builder** — chain `.unary()`, `.producer()`, `.exchange()` calls to define your service
|
|
19
15
|
- **Type-safe streaming state** — generic `<S>` parameter threads state types through init and produce/exchange functions
|
|
@@ -24,10 +20,32 @@ Define RPC methods with Arrow-typed schemas, serve them over stdin/stdout, and i
|
|
|
24
20
|
## Installation
|
|
25
21
|
|
|
26
22
|
```bash
|
|
23
|
+
# Bun
|
|
27
24
|
bun add @query-farm/vgi-rpc
|
|
25
|
+
|
|
26
|
+
# npm
|
|
27
|
+
npm install @query-farm/vgi-rpc
|
|
28
|
+
|
|
29
|
+
# Deno
|
|
30
|
+
deno add npm:@query-farm/vgi-rpc
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
## Runtime Compatibility
|
|
34
|
+
|
|
35
|
+
`vgi-rpc` works with [Bun](https://bun.sh/), [Node.js](https://nodejs.org/) 22+, and [Deno](https://deno.land/) 2+. The HTTP handler and HTTP client are fully runtime-agnostic. The subprocess transport (`subprocessConnect`) requires Bun.
|
|
36
|
+
|
|
37
|
+
### Browser Usage
|
|
38
|
+
|
|
39
|
+
The HTTP client works in browsers. Use `httpConnect` to call a `vgi-rpc` server from any browser application:
|
|
40
|
+
|
|
41
|
+
```typescript
|
|
42
|
+
import { httpConnect } from "@query-farm/vgi-rpc";
|
|
43
|
+
|
|
44
|
+
const client = httpConnect("https://api.example.com");
|
|
45
|
+
const result = await client.call("add", { a: 2, b: 3 });
|
|
28
46
|
```
|
|
29
47
|
|
|
30
|
-
|
|
48
|
+
You will need a bundler (Vite, esbuild, webpack, etc.) that can resolve the `@query-farm/apache-arrow` dependency. Server-only exports (`VgiRpcServer`, `createHttpHandler`, `subprocessConnect`, `pipeConnect`) are not available in browsers.
|
|
31
49
|
|
|
32
50
|
## Quick Start
|
|
33
51
|
|
|
@@ -196,7 +214,7 @@ protocol.unary("echo", {
|
|
|
196
214
|
});
|
|
197
215
|
|
|
198
216
|
// Equivalent verbose form
|
|
199
|
-
import { Schema, Field, Utf8, Int64, Float64 } from "apache-arrow";
|
|
217
|
+
import { Schema, Field, Utf8, Int64, Float64 } from "@query-farm/apache-arrow";
|
|
200
218
|
|
|
201
219
|
protocol.unary("echo", {
|
|
202
220
|
params: new Schema([
|
|
@@ -221,7 +239,7 @@ protocol.unary("echo", {
|
|
|
221
239
|
| `float32` | Float32 | `Annotated[float, ArrowType(pa.float32())]` |
|
|
222
240
|
| `bool` | Bool | `bool` |
|
|
223
241
|
|
|
224
|
-
For complex types (List, Map, Dictionary, nullable fields), use the full `Schema`/`Field` constructors from
|
|
242
|
+
For complex types (List, Map, Dictionary, nullable fields), use the full `Schema`/`Field` constructors from `@query-farm/apache-arrow`.
|
|
225
243
|
|
|
226
244
|
## Emitting Output
|
|
227
245
|
|
|
@@ -301,7 +319,7 @@ vgi-rpc --cmd "bun run examples/streaming.ts" call scale factor=2.0
|
|
|
301
319
|
|
|
302
320
|
## Wire Protocol Compatibility
|
|
303
321
|
|
|
304
|
-
This library implements the same wire protocol as the Python [vgi-rpc](https://github.com/Query-farm/vgi-rpc-python) framework:
|
|
322
|
+
This library implements the same wire protocol as the Python [`vgi-rpc`](https://github.com/Query-farm/vgi-rpc-python) framework:
|
|
305
323
|
|
|
306
324
|
- Multiple sequential Arrow IPC streams on stdin/stdout
|
|
307
325
|
- Request batches carry method name and version in batch metadata
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"introspect.d.ts","sourceRoot":"","sources":["../../src/client/introspect.ts"],"names":[],"mappings":"AAGA,OAAO,EAAqB,WAAW,EAAE,KAAK,MAAM,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"introspect.d.ts","sourceRoot":"","sources":["../../src/client/introspect.ts"],"names":[],"mappings":"AAGA,OAAO,EAAqB,WAAW,EAAE,KAAK,MAAM,EAAE,MAAM,0BAA0B,CAAC;AASvF,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAE7C,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,OAAO,GAAG,QAAQ,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACpC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAChC;AAED,MAAM,WAAW,kBAAkB;IACjC,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,UAAU,EAAE,CAAC;CACvB;AASD;;;GAGG;AACH,wBAAsB,qBAAqB,CACzC,OAAO,EAAE,WAAW,EAAE,EACtB,KAAK,CAAC,EAAE,CAAC,GAAG,EAAE,UAAU,KAAK,IAAI,GAChC,OAAO,CAAC,kBAAkB,CAAC,CAoE7B;AAED;;GAEG;AACH,wBAAsB,cAAc,CAClC,OAAO,EAAE,MAAM,EACf,OAAO,CAAC,EAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,GAC5B,OAAO,CAAC,kBAAkB,CAAC,CAe7B"}
|
package/dist/client/ipc.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { RecordBatch, Schema, DataType } from "apache-arrow";
|
|
1
|
+
import { RecordBatch, Schema, DataType } from "@query-farm/apache-arrow";
|
|
2
2
|
import { IpcStreamReader } from "../wire/reader.js";
|
|
3
3
|
import type { LogMessage } from "./types.js";
|
|
4
4
|
/** Infer an Arrow DataType from a JS value. */
|
package/dist/client/ipc.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ipc.d.ts","sourceRoot":"","sources":["../../src/client/ipc.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,WAAW,EAEX,MAAM,EACN,QAAQ,EAST,MAAM,
|
|
1
|
+
{"version":3,"file":"ipc.d.ts","sourceRoot":"","sources":["../../src/client/ipc.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,WAAW,EAEX,MAAM,EACN,QAAQ,EAST,MAAM,0BAA0B,CAAC;AAWlC,OAAO,EAAE,eAAe,EAAsB,MAAM,mBAAmB,CAAC;AACxE,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAE7C,+CAA+C;AAC/C,wBAAgB,cAAc,CAAC,KAAK,EAAE,GAAG,GAAG,QAAQ,CAOnD;AAyCD;;GAEG;AACH,wBAAgB,eAAe,CAC7B,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAC3B,MAAM,EAAE,MAAM,GACb,UAAU,CAgCZ;AAED;;GAEG;AACH,wBAAsB,mBAAmB,CACvC,IAAI,EAAE,UAAU,GACf,OAAO,CAAC;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,WAAW,EAAE,CAAA;CAAE,CAAC,CASrD;AAED;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAChC,KAAK,EAAE,WAAW,EAClB,KAAK,CAAC,EAAE,CAAC,GAAG,EAAE,UAAU,KAAK,IAAI,GAChC,OAAO,CAqCT;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,WAAW,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,CAoB1E;AAED;;;GAGG;AACH,wBAAsB,qBAAqB,CACzC,IAAI,EAAE,UAAU,GACf,OAAO,CAAC,eAAe,CAAC,CAQ1B"}
|
package/dist/client/pipe.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Schema } from "apache-arrow";
|
|
1
|
+
import { Schema } from "@query-farm/apache-arrow";
|
|
2
2
|
import { IpcStreamReader } from "../wire/reader.js";
|
|
3
3
|
import type { LogMessage, PipeConnectOptions, SubprocessConnectOptions, StreamSession } from "./types.js";
|
|
4
4
|
import type { RpcClient } from "./connect.js";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pipe.d.ts","sourceRoot":"","sources":["../../src/client/pipe.ts"],"names":[],"mappings":"AAGA,OAAO,EAGL,MAAM,EAKP,MAAM,
|
|
1
|
+
{"version":3,"file":"pipe.d.ts","sourceRoot":"","sources":["../../src/client/pipe.ts"],"names":[],"mappings":"AAGA,OAAO,EAGL,MAAM,EAKP,MAAM,0BAA0B,CAAC;AAGlC,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAYpD,OAAO,KAAK,EACV,UAAU,EACV,kBAAkB,EAClB,wBAAwB,EACxB,aAAa,EACd,MAAM,YAAY,CAAC;AACpB,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAO9C,UAAU,YAAY;IACpB,KAAK,CAAC,IAAI,EAAE,UAAU,GAAG,IAAI,CAAC;IAC9B,KAAK,CAAC,IAAI,IAAI,CAAC;IACf,GAAG,IAAI,IAAI,CAAC;CACb;AAED,KAAK,OAAO,GAAG,CAAC,KAAK,EAAE,UAAU,KAAK,IAAI,CAAC;AA6C3C,qBAAa,iBAAkB,YAAW,aAAa;IACrD,OAAO,CAAC,OAAO,CAAkB;IACjC,OAAO,CAAC,QAAQ,CAAU;IAC1B,OAAO,CAAC,MAAM,CAAC,CAA4B;IAC3C,OAAO,CAAC,OAAO,CAA6B;IAC5C,OAAO,CAAC,YAAY,CAAsC;IAC1D,OAAO,CAAC,YAAY,CAAuB;IAC3C,OAAO,CAAC,mBAAmB,CAAS;IACpC,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,YAAY,CAAa;IACjC,OAAO,CAAC,gBAAgB,CAA6B;gBAEzC,IAAI,EAAE;QAChB,MAAM,EAAE,eAAe,CAAC;QACxB,OAAO,EAAE,OAAO,CAAC;QACjB,KAAK,CAAC,EAAE,CAAC,GAAG,EAAE,UAAU,KAAK,IAAI,CAAC;QAClC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,IAAI,CAAC;QACnC,YAAY,EAAE,MAAM,CAAC;QACrB,WAAW,EAAE,MAAM,IAAI,CAAC;QACxB,eAAe,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC;KAC7C;IAUD,IAAI,MAAM,IAAI,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,IAAI,CAEvC;IAED;;;;OAIG;YACW,gBAAgB;IAiB9B;;;;;OAKG;YACW,mBAAmB;IASjC;;OAEG;IACG,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC;IAmG5E;;OAEG;YACW,QAAQ;IAiBtB;;OAEG;IACI,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,qBAAqB,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC;IAqD7E,KAAK,IAAI,IAAI;CAoCd;AAMD,wBAAgB,WAAW,CACzB,QAAQ,EAAE,cAAc,CAAC,UAAU,CAAC,EACpC,QAAQ,EAAE,YAAY,EACtB,OAAO,CAAC,EAAE,kBAAkB,GAC3B,SAAS,CAkOX;AAMD,wBAAgB,iBAAiB,CAC/B,GAAG,EAAE,MAAM,EAAE,EACb,OAAO,CAAC,EAAE,wBAAwB,GACjC,SAAS,CAuCX"}
|
package/dist/client/stream.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { RecordBatch, Schema } from "apache-arrow";
|
|
1
|
+
import { RecordBatch, Schema } from "@query-farm/apache-arrow";
|
|
2
2
|
import type { LogMessage, StreamSession } from "./types.js";
|
|
3
3
|
type CompressFn = (data: Uint8Array, level: number) => Uint8Array;
|
|
4
4
|
type DecompressFn = (data: Uint8Array) => Uint8Array;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"stream.d.ts","sourceRoot":"","sources":["../../src/client/stream.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,WAAW,EACX,MAAM,EAKP,MAAM,
|
|
1
|
+
{"version":3,"file":"stream.d.ts","sourceRoot":"","sources":["../../src/client/stream.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,WAAW,EACX,MAAM,EAKP,MAAM,0BAA0B,CAAC;AAUlC,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAE5D,KAAK,UAAU,GAAG,CAAC,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,KAAK,UAAU,CAAC;AAClE,KAAK,YAAY,GAAG,CAAC,IAAI,EAAE,UAAU,KAAK,UAAU,CAAC;AAErD,qBAAa,iBAAkB,YAAW,aAAa;IACrD,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,WAAW,CAAgB;IACnC,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,YAAY,CAAC,CAAS;IAC9B,OAAO,CAAC,MAAM,CAAC,CAA4B;IAC3C,OAAO,CAAC,eAAe,CAAgB;IACvC,OAAO,CAAC,SAAS,CAAU;IAC3B,OAAO,CAAC,OAAO,CAA6B;IAC5C,OAAO,CAAC,iBAAiB,CAAC,CAAS;IACnC,OAAO,CAAC,WAAW,CAAC,CAAa;IACjC,OAAO,CAAC,aAAa,CAAC,CAAe;gBAEzB,IAAI,EAAE;QAChB,OAAO,EAAE,MAAM,CAAC;QAChB,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;QAC1B,YAAY,EAAE,MAAM,CAAC;QACrB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,KAAK,CAAC,EAAE,CAAC,GAAG,EAAE,UAAU,KAAK,IAAI,CAAC;QAClC,cAAc,EAAE,WAAW,EAAE,CAAC;QAC9B,QAAQ,EAAE,OAAO,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,IAAI,CAAC;QACnC,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAC1B,UAAU,CAAC,EAAE,UAAU,CAAC;QACxB,YAAY,CAAC,EAAE,YAAY,CAAC;KAC7B;IAgBD,IAAI,MAAM,IAAI,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,IAAI,CAEvC;IAED,OAAO,CAAC,aAAa;IAWrB,OAAO,CAAC,YAAY;YAON,aAAa;IAQ3B;;OAEG;IACG,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC;YA+D9D,WAAW;IA0CzB,OAAO,CAAC,gBAAgB;IAcxB;;OAEG;IACI,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,qBAAqB,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC;YAyC/D,iBAAiB;IA2B/B,KAAK,IAAI,IAAI;CAGd"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"describe.d.ts","sourceRoot":"","sources":["../../src/dispatch/describe.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,MAAM,EAEN,WAAW,EAOZ,MAAM,
|
|
1
|
+
{"version":3,"file":"describe.d.ts","sourceRoot":"","sources":["../../src/dispatch/describe.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,MAAM,EAEN,WAAW,EAOZ,MAAM,0BAA0B,CAAC;AAClC,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAWpD;;GAEG;AACH,eAAO,MAAM,eAAe,aAW1B,CAAC;AAEH;;GAEG;AACH,wBAAgB,kBAAkB,CAChC,YAAY,EAAE,MAAM,EACpB,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,gBAAgB,CAAC,EACtC,QAAQ,EAAE,MAAM,GACf;IAAE,KAAK,EAAE,WAAW,CAAC;IAAC,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAAE,CA0GvD"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"stream.d.ts","sourceRoot":"","sources":["../../src/dispatch/stream.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAEpD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACzD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAKzD;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,cAAc,CAClC,MAAM,EAAE,gBAAgB,EACxB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAC3B,MAAM,EAAE,eAAe,EACvB,MAAM,EAAE,eAAe,EACvB,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,GAAG,IAAI,GACvB,OAAO,CAAC,IAAI,CAAC,
|
|
1
|
+
{"version":3,"file":"stream.d.ts","sourceRoot":"","sources":["../../src/dispatch/stream.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAEpD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACzD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAKzD;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,cAAc,CAClC,MAAM,EAAE,gBAAgB,EACxB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAC3B,MAAM,EAAE,eAAe,EACvB,MAAM,EAAE,eAAe,EACvB,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,GAAG,IAAI,GACvB,OAAO,CAAC,IAAI,CAAC,CAmHf"}
|
package/dist/http/common.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { RecordBatch, Schema } from "apache-arrow";
|
|
1
|
+
import { RecordBatch, Schema } from "@query-farm/apache-arrow";
|
|
2
2
|
export declare const ARROW_CONTENT_TYPE = "application/vnd.apache.arrow.stream";
|
|
3
3
|
export declare class HttpRpcError extends Error {
|
|
4
4
|
readonly statusCode: number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"common.d.ts","sourceRoot":"","sources":["../../src/http/common.ts"],"names":[],"mappings":"AAGA,OAAO,EAGL,WAAW,EACX,MAAM,EAGP,MAAM,
|
|
1
|
+
{"version":3,"file":"common.d.ts","sourceRoot":"","sources":["../../src/http/common.ts"],"names":[],"mappings":"AAGA,OAAO,EAGL,WAAW,EACX,MAAM,EAGP,MAAM,0BAA0B,CAAC;AAElC,eAAO,MAAM,kBAAkB,wCAAwC,CAAC;AAExE,qBAAa,YAAa,SAAQ,KAAK;aAGnB,UAAU,EAAE,MAAM;gBADlC,OAAO,EAAE,MAAM,EACC,UAAU,EAAE,MAAM;CAKrC;AA8BD,6EAA6E;AAC7E,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,WAAW,EAAE,GACrB,UAAU,CAQZ;AAED,yFAAyF;AACzF,wBAAgB,aAAa,CAAC,IAAI,EAAE,UAAU,EAAE,MAAM,SAAM,EAAE,YAAY,CAAC,EAAE,OAAO,GAAG,QAAQ,CAI9F;AAED,yDAAyD;AACzD,wBAAsB,mBAAmB,CACvC,IAAI,EAAE,UAAU,GACf,OAAO,CAAC;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,WAAW,CAAA;CAAE,CAAC,CAYjD"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dispatch.d.ts","sourceRoot":"","sources":["../../src/http/dispatch.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAkBpD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"dispatch.d.ts","sourceRoot":"","sources":["../../src/http/dispatch.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAkBpD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAUlD,MAAM,WAAW,eAAe;IAC9B,UAAU,EAAE,UAAU,CAAC;IACvB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,eAAe,EAAE,eAAe,CAAC;CAClC;AAED,uCAAuC;AACvC,wBAAgB,oBAAoB,CAClC,YAAY,EAAE,MAAM,EACpB,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,gBAAgB,CAAC,EACtC,QAAQ,EAAE,MAAM,GACf,QAAQ,CAIV;AAED,qCAAqC;AACrC,wBAAsB,iBAAiB,CACrC,MAAM,EAAE,gBAAgB,EACxB,IAAI,EAAE,UAAU,EAChB,GAAG,EAAE,eAAe,GACnB,OAAO,CAAC,QAAQ,CAAC,CAuBnB;AAED,kEAAkE;AAClE,wBAAsB,sBAAsB,CAC1C,MAAM,EAAE,gBAAgB,EACxB,IAAI,EAAE,UAAU,EAChB,GAAG,EAAE,eAAe,GACnB,OAAO,CAAC,QAAQ,CAAC,CAyGnB;AAED,yFAAyF;AACzF,wBAAsB,0BAA0B,CAC9C,MAAM,EAAE,gBAAgB,EACxB,IAAI,EAAE,UAAU,EAChB,GAAG,EAAE,eAAe,GACnB,OAAO,CAAC,QAAQ,CAAC,CAqHnB"}
|
package/dist/http/index.d.ts
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
export { createHttpHandler } from "./handler.js";
|
|
2
2
|
export type { HttpHandlerOptions, StateSerializer } from "./types.js";
|
|
3
|
+
export { jsonStateSerializer } from "./types.js";
|
|
3
4
|
export { ARROW_CONTENT_TYPE } from "./common.js";
|
|
5
|
+
export { unpackStateToken, type UnpackedToken } from "./token.js";
|
|
4
6
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/http/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/http/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AACjD,YAAY,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AACtE,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/http/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AACjD,YAAY,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AACtE,OAAO,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AACjD,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,EAAE,gBAAgB,EAAE,KAAK,aAAa,EAAE,MAAM,YAAY,CAAC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -3,7 +3,7 @@ export { Protocol } from "./protocol.js";
|
|
|
3
3
|
export { MethodType, OutputCollector, type LogContext, type MethodDefinition, type UnaryHandler, type HeaderInit, type ProducerInit, type ProducerFn, type ExchangeInit, type ExchangeFn, } from "./types.js";
|
|
4
4
|
export { type SchemaLike, toSchema, inferParamTypes, str, bytes, int, int32, float, float32, bool, } from "./schema.js";
|
|
5
5
|
export { RpcError, VersionError } from "./errors.js";
|
|
6
|
-
export { createHttpHandler, ARROW_CONTENT_TYPE, type HttpHandlerOptions, type StateSerializer, } from "./http/index.js";
|
|
6
|
+
export { createHttpHandler, ARROW_CONTENT_TYPE, type HttpHandlerOptions, type StateSerializer, jsonStateSerializer, unpackStateToken, type UnpackedToken, } from "./http/index.js";
|
|
7
7
|
export { RPC_METHOD_KEY, REQUEST_VERSION_KEY, REQUEST_VERSION, LOG_LEVEL_KEY, LOG_MESSAGE_KEY, LOG_EXTRA_KEY, SERVER_ID_KEY, REQUEST_ID_KEY, PROTOCOL_NAME_KEY, DESCRIBE_VERSION_KEY, DESCRIBE_VERSION, DESCRIBE_METHOD_NAME, STATE_KEY, } from "./constants.js";
|
|
8
8
|
export * from "./client/index.js";
|
|
9
9
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EACL,UAAU,EACV,eAAe,EACf,KAAK,UAAU,EACf,KAAK,gBAAgB,EACrB,KAAK,YAAY,EACjB,KAAK,UAAU,EACf,KAAK,YAAY,EACjB,KAAK,UAAU,EACf,KAAK,YAAY,EACjB,KAAK,UAAU,GAChB,MAAM,YAAY,CAAC;AACpB,OAAO,EACL,KAAK,UAAU,EACf,QAAQ,EACR,eAAe,EACf,GAAG,EACH,KAAK,EACL,GAAG,EACH,KAAK,EACL,KAAK,EACL,OAAO,EACP,IAAI,GACL,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,EACL,iBAAiB,EACjB,kBAAkB,EAClB,KAAK,kBAAkB,EACvB,KAAK,eAAe,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EACL,UAAU,EACV,eAAe,EACf,KAAK,UAAU,EACf,KAAK,gBAAgB,EACrB,KAAK,YAAY,EACjB,KAAK,UAAU,EACf,KAAK,YAAY,EACjB,KAAK,UAAU,EACf,KAAK,YAAY,EACjB,KAAK,UAAU,GAChB,MAAM,YAAY,CAAC;AACpB,OAAO,EACL,KAAK,UAAU,EACf,QAAQ,EACR,eAAe,EACf,GAAG,EACH,KAAK,EACL,GAAG,EACH,KAAK,EACL,KAAK,EACL,OAAO,EACP,IAAI,GACL,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,EACL,iBAAiB,EACjB,kBAAkB,EAClB,KAAK,kBAAkB,EACvB,KAAK,eAAe,EACpB,mBAAmB,EACnB,gBAAgB,EAChB,KAAK,aAAa,GACnB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,cAAc,EACd,mBAAmB,EACnB,eAAe,EACf,aAAa,EACb,eAAe,EACf,aAAa,EACb,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,oBAAoB,EACpB,gBAAgB,EAChB,oBAAoB,EACpB,SAAS,GACV,MAAM,gBAAgB,CAAC;AACxB,cAAc,mBAAmB,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -51,10 +51,10 @@ var init_zstd = __esm(() => {
|
|
|
51
51
|
});
|
|
52
52
|
|
|
53
53
|
// src/server.ts
|
|
54
|
-
import { Schema as Schema4 } from "apache-arrow";
|
|
54
|
+
import { Schema as Schema4 } from "@query-farm/apache-arrow";
|
|
55
55
|
|
|
56
56
|
// src/wire/reader.ts
|
|
57
|
-
import { RecordBatchReader } from "apache-arrow";
|
|
57
|
+
import { RecordBatchReader } from "@query-farm/apache-arrow";
|
|
58
58
|
|
|
59
59
|
class IpcStreamReader {
|
|
60
60
|
reader;
|
|
@@ -127,7 +127,7 @@ class IpcStreamReader {
|
|
|
127
127
|
// src/wire/writer.ts
|
|
128
128
|
import {
|
|
129
129
|
RecordBatchStreamWriter
|
|
130
|
-
} from "apache-arrow";
|
|
130
|
+
} from "@query-farm/apache-arrow";
|
|
131
131
|
import { writeSync } from "node:fs";
|
|
132
132
|
var STDOUT_FD = 1;
|
|
133
133
|
function writeAll(fd, data) {
|
|
@@ -201,7 +201,7 @@ class IncrementalStream {
|
|
|
201
201
|
}
|
|
202
202
|
|
|
203
203
|
// src/wire/request.ts
|
|
204
|
-
import { DataType } from "apache-arrow";
|
|
204
|
+
import { DataType } from "@query-farm/apache-arrow";
|
|
205
205
|
|
|
206
206
|
// src/constants.ts
|
|
207
207
|
var RPC_METHOD_KEY = "vgi_rpc.method";
|
|
@@ -290,7 +290,7 @@ import {
|
|
|
290
290
|
makeData,
|
|
291
291
|
Struct,
|
|
292
292
|
vectorFromArray
|
|
293
|
-
} from "apache-arrow";
|
|
293
|
+
} from "@query-farm/apache-arrow";
|
|
294
294
|
function coerceInt64(schema, values) {
|
|
295
295
|
const result = { ...values };
|
|
296
296
|
for (const field of schema.fields) {
|
|
@@ -406,10 +406,10 @@ import {
|
|
|
406
406
|
vectorFromArray as vectorFromArray2,
|
|
407
407
|
makeData as makeData2,
|
|
408
408
|
Struct as Struct2
|
|
409
|
-
} from "apache-arrow";
|
|
409
|
+
} from "@query-farm/apache-arrow";
|
|
410
410
|
|
|
411
411
|
// src/util/schema.ts
|
|
412
|
-
import { RecordBatchStreamWriter as RecordBatchStreamWriter2 } from "apache-arrow";
|
|
412
|
+
import { RecordBatchStreamWriter as RecordBatchStreamWriter2 } from "@query-farm/apache-arrow";
|
|
413
413
|
function serializeSchema(schema) {
|
|
414
414
|
const writer = new RecordBatchStreamWriter2;
|
|
415
415
|
writer.reset(undefined, schema);
|
|
@@ -508,7 +508,7 @@ function buildDescribeBatch(protocolName, methods, serverId) {
|
|
|
508
508
|
}
|
|
509
509
|
|
|
510
510
|
// src/types.ts
|
|
511
|
-
import { RecordBatch as RecordBatch3, recordBatchFromArrays } from "apache-arrow";
|
|
511
|
+
import { RecordBatch as RecordBatch3, recordBatchFromArrays } from "@query-farm/apache-arrow";
|
|
512
512
|
var MethodType;
|
|
513
513
|
((MethodType2) => {
|
|
514
514
|
MethodType2["UNARY"] = "unary";
|
|
@@ -587,10 +587,10 @@ async function dispatchUnary(method, params, writer, serverId, requestId) {
|
|
|
587
587
|
}
|
|
588
588
|
|
|
589
589
|
// src/dispatch/stream.ts
|
|
590
|
-
import { Schema as Schema3 } from "apache-arrow";
|
|
590
|
+
import { Schema as Schema3 } from "@query-farm/apache-arrow";
|
|
591
591
|
var EMPTY_SCHEMA = new Schema3([]);
|
|
592
592
|
async function dispatchStream(method, params, writer, reader, serverId, requestId) {
|
|
593
|
-
const isProducer =
|
|
593
|
+
const isProducer = !!method.producerFn;
|
|
594
594
|
let state;
|
|
595
595
|
try {
|
|
596
596
|
if (isProducer) {
|
|
@@ -753,7 +753,7 @@ class VgiRpcServer {
|
|
|
753
753
|
}
|
|
754
754
|
}
|
|
755
755
|
// src/protocol.ts
|
|
756
|
-
import { Schema as Schema6 } from "apache-arrow";
|
|
756
|
+
import { Schema as Schema6 } from "@query-farm/apache-arrow";
|
|
757
757
|
|
|
758
758
|
// src/schema.ts
|
|
759
759
|
import {
|
|
@@ -768,7 +768,7 @@ import {
|
|
|
768
768
|
Float64,
|
|
769
769
|
Float32,
|
|
770
770
|
Bool as Bool2
|
|
771
|
-
} from "apache-arrow";
|
|
771
|
+
} from "@query-farm/apache-arrow";
|
|
772
772
|
var str = new Utf82;
|
|
773
773
|
var bytes = new Binary2;
|
|
774
774
|
var int = new Int64;
|
|
@@ -887,7 +887,7 @@ class Protocol {
|
|
|
887
887
|
}
|
|
888
888
|
}
|
|
889
889
|
// src/http/handler.ts
|
|
890
|
-
import { Schema as Schema9 } from "apache-arrow";
|
|
890
|
+
import { Schema as Schema9 } from "@query-farm/apache-arrow";
|
|
891
891
|
import { randomBytes } from "node:crypto";
|
|
892
892
|
|
|
893
893
|
// src/http/types.ts
|
|
@@ -907,7 +907,7 @@ import {
|
|
|
907
907
|
RecordBatch as RecordBatch4,
|
|
908
908
|
Struct as Struct3,
|
|
909
909
|
makeData as makeData3
|
|
910
|
-
} from "apache-arrow";
|
|
910
|
+
} from "@query-farm/apache-arrow";
|
|
911
911
|
var ARROW_CONTENT_TYPE = "application/vnd.apache.arrow.stream";
|
|
912
912
|
|
|
913
913
|
class HttpRpcError extends Error {
|
|
@@ -961,7 +961,7 @@ async function readRequestFromBody(body) {
|
|
|
961
961
|
}
|
|
962
962
|
|
|
963
963
|
// src/http/dispatch.ts
|
|
964
|
-
import { Schema as Schema8, RecordBatch as RecordBatch5 } from "apache-arrow";
|
|
964
|
+
import { Schema as Schema8, RecordBatch as RecordBatch5, RecordBatchReader as RecordBatchReader3 } from "@query-farm/apache-arrow";
|
|
965
965
|
|
|
966
966
|
// src/http/token.ts
|
|
967
967
|
import { createHmac, timingSafeEqual } from "node:crypto";
|
|
@@ -1042,6 +1042,11 @@ function unpackStateToken(tokenBase64, signingKey, tokenTtl) {
|
|
|
1042
1042
|
}
|
|
1043
1043
|
|
|
1044
1044
|
// src/http/dispatch.ts
|
|
1045
|
+
async function deserializeSchema(bytes2) {
|
|
1046
|
+
const reader = await RecordBatchReader3.from(bytes2);
|
|
1047
|
+
await reader.open();
|
|
1048
|
+
return reader.schema;
|
|
1049
|
+
}
|
|
1045
1050
|
var EMPTY_SCHEMA4 = new Schema8([]);
|
|
1046
1051
|
function httpDispatchDescribe(protocolName, methods, serverId) {
|
|
1047
1052
|
const { batch } = buildDescribeBatch(protocolName, methods, serverId);
|
|
@@ -1067,7 +1072,7 @@ async function httpDispatchUnary(method, body, ctx) {
|
|
|
1067
1072
|
}
|
|
1068
1073
|
}
|
|
1069
1074
|
async function httpDispatchStreamInit(method, body, ctx) {
|
|
1070
|
-
const isProducer =
|
|
1075
|
+
const isProducer = !!method.producerFn;
|
|
1071
1076
|
const outputSchema = method.outputSchema;
|
|
1072
1077
|
const inputSchema = method.inputSchema ?? EMPTY_SCHEMA4;
|
|
1073
1078
|
const { schema: reqSchema, batch: reqBatch } = await readRequestFromBody(body);
|
|
@@ -1126,7 +1131,7 @@ async function httpDispatchStreamInit(method, body, ctx) {
|
|
|
1126
1131
|
}
|
|
1127
1132
|
}
|
|
1128
1133
|
async function httpDispatchStreamExchange(method, body, ctx) {
|
|
1129
|
-
const isProducer =
|
|
1134
|
+
const isProducer = !!method.producerFn;
|
|
1130
1135
|
const { batch: reqBatch } = await readRequestFromBody(body);
|
|
1131
1136
|
const tokenBase64 = reqBatch.metadata?.get(STATE_KEY);
|
|
1132
1137
|
if (!tokenBase64) {
|
|
@@ -1138,33 +1143,70 @@ async function httpDispatchStreamExchange(method, body, ctx) {
|
|
|
1138
1143
|
} catch (error) {
|
|
1139
1144
|
throw new HttpRpcError(`Invalid state token: ${error.message}`, 400);
|
|
1140
1145
|
}
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1146
|
+
let state;
|
|
1147
|
+
try {
|
|
1148
|
+
state = ctx.stateSerializer.deserialize(unpacked.stateBytes);
|
|
1149
|
+
} catch (error) {
|
|
1150
|
+
console.error(`[httpDispatchStreamExchange] state deserialize error:`, error.message);
|
|
1151
|
+
throw new HttpRpcError(`State deserialization failed: ${error.message}`, 500);
|
|
1152
|
+
}
|
|
1153
|
+
let outputSchema;
|
|
1154
|
+
if (unpacked.schemaBytes.length > 0) {
|
|
1155
|
+
outputSchema = await deserializeSchema(unpacked.schemaBytes);
|
|
1156
|
+
} else {
|
|
1157
|
+
outputSchema = state?.__outputSchema ?? method.outputSchema;
|
|
1158
|
+
}
|
|
1159
|
+
let inputSchema;
|
|
1160
|
+
if (unpacked.inputSchemaBytes.length > 0) {
|
|
1161
|
+
inputSchema = await deserializeSchema(unpacked.inputSchemaBytes);
|
|
1162
|
+
} else {
|
|
1163
|
+
inputSchema = method.inputSchema ?? EMPTY_SCHEMA4;
|
|
1164
|
+
}
|
|
1144
1165
|
const effectiveProducer = state?.__isProducer ?? isProducer;
|
|
1166
|
+
if (process.env.VGI_DISPATCH_DEBUG)
|
|
1167
|
+
console.error(`[httpDispatchStreamExchange] method=${method.name} effectiveProducer=${effectiveProducer} stateKeys=${Object.keys(state || {})}`);
|
|
1145
1168
|
if (effectiveProducer) {
|
|
1146
1169
|
return produceStreamResponse(method, state, outputSchema, inputSchema, ctx, null, null);
|
|
1147
1170
|
} else {
|
|
1148
|
-
const out = new OutputCollector(outputSchema,
|
|
1171
|
+
const out = new OutputCollector(outputSchema, effectiveProducer, ctx.serverId, null);
|
|
1149
1172
|
try {
|
|
1150
|
-
|
|
1173
|
+
if (method.exchangeFn) {
|
|
1174
|
+
await method.exchangeFn(state, reqBatch, out);
|
|
1175
|
+
} else {
|
|
1176
|
+
await method.producerFn(state, out);
|
|
1177
|
+
}
|
|
1151
1178
|
} catch (error) {
|
|
1179
|
+
if (process.env.VGI_DISPATCH_DEBUG)
|
|
1180
|
+
console.error(`[httpDispatchStreamExchange] exchange handler error:`, error.message, error.stack?.split(`
|
|
1181
|
+
`).slice(0, 5).join(`
|
|
1182
|
+
`));
|
|
1152
1183
|
const errBatch = buildErrorBatch(outputSchema, error, ctx.serverId, null);
|
|
1153
1184
|
return arrowResponse(serializeIpcStream(outputSchema, [errBatch]), 500);
|
|
1154
1185
|
}
|
|
1155
|
-
const stateBytes = ctx.stateSerializer.serialize(state);
|
|
1156
|
-
const schemaBytes = serializeSchema(outputSchema);
|
|
1157
|
-
const inputSchemaBytes = serializeSchema(inputSchema);
|
|
1158
|
-
const token = packStateToken(stateBytes, schemaBytes, inputSchemaBytes, ctx.signingKey);
|
|
1159
1186
|
const batches = [];
|
|
1160
|
-
|
|
1161
|
-
const
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1187
|
+
if (out.finished) {
|
|
1188
|
+
for (const emitted of out.batches) {
|
|
1189
|
+
batches.push(emitted.batch);
|
|
1190
|
+
}
|
|
1191
|
+
} else {
|
|
1192
|
+
const stateBytes = ctx.stateSerializer.serialize(state);
|
|
1193
|
+
const schemaBytes = serializeSchema(outputSchema);
|
|
1194
|
+
const inputSchemaBytes = serializeSchema(inputSchema);
|
|
1195
|
+
const token = packStateToken(stateBytes, schemaBytes, inputSchemaBytes, ctx.signingKey);
|
|
1196
|
+
for (const emitted of out.batches) {
|
|
1197
|
+
const batch = emitted.batch;
|
|
1198
|
+
if (batch.numRows > 0) {
|
|
1199
|
+
const mergedMeta = new Map(batch.metadata ?? []);
|
|
1200
|
+
mergedMeta.set(STATE_KEY, token);
|
|
1201
|
+
batches.push(new RecordBatch5(batch.schema, batch.data, mergedMeta));
|
|
1202
|
+
} else {
|
|
1203
|
+
batches.push(batch);
|
|
1204
|
+
}
|
|
1205
|
+
}
|
|
1206
|
+
if (!batches.some((b) => b.metadata?.get(STATE_KEY))) {
|
|
1207
|
+
const tokenMeta = new Map;
|
|
1208
|
+
tokenMeta.set(STATE_KEY, token);
|
|
1209
|
+
batches.push(buildEmptyBatch(outputSchema, tokenMeta));
|
|
1168
1210
|
}
|
|
1169
1211
|
}
|
|
1170
1212
|
return arrowResponse(serializeIpcStream(outputSchema, batches));
|
|
@@ -1177,8 +1219,17 @@ async function produceStreamResponse(method, state, outputSchema, inputSchema, c
|
|
|
1177
1219
|
while (true) {
|
|
1178
1220
|
const out = new OutputCollector(outputSchema, true, ctx.serverId, requestId);
|
|
1179
1221
|
try {
|
|
1180
|
-
|
|
1222
|
+
if (method.producerFn) {
|
|
1223
|
+
await method.producerFn(state, out);
|
|
1224
|
+
} else {
|
|
1225
|
+
const tickBatch = buildEmptyBatch(inputSchema);
|
|
1226
|
+
await method.exchangeFn(state, tickBatch, out);
|
|
1227
|
+
}
|
|
1181
1228
|
} catch (error) {
|
|
1229
|
+
if (process.env.VGI_DISPATCH_DEBUG)
|
|
1230
|
+
console.error(`[produceStreamResponse] error:`, error.message, error.stack?.split(`
|
|
1231
|
+
`).slice(0, 3).join(`
|
|
1232
|
+
`));
|
|
1182
1233
|
allBatches.push(buildErrorBatch(outputSchema, error, ctx.serverId, requestId));
|
|
1183
1234
|
break;
|
|
1184
1235
|
}
|
|
@@ -1369,7 +1420,7 @@ function createHttpHandler(protocol, options) {
|
|
|
1369
1420
|
// src/client/ipc.ts
|
|
1370
1421
|
import {
|
|
1371
1422
|
RecordBatch as RecordBatch6,
|
|
1372
|
-
RecordBatchReader as
|
|
1423
|
+
RecordBatchReader as RecordBatchReader4,
|
|
1373
1424
|
DataType as DataType4,
|
|
1374
1425
|
Float64 as Float642,
|
|
1375
1426
|
Int64 as Int642,
|
|
@@ -1379,7 +1430,7 @@ import {
|
|
|
1379
1430
|
vectorFromArray as vectorFromArray3,
|
|
1380
1431
|
makeData as makeData4,
|
|
1381
1432
|
Struct as Struct4
|
|
1382
|
-
} from "apache-arrow";
|
|
1433
|
+
} from "@query-farm/apache-arrow";
|
|
1383
1434
|
function inferArrowType(value) {
|
|
1384
1435
|
if (typeof value === "string")
|
|
1385
1436
|
return new Utf83;
|
|
@@ -1452,7 +1503,7 @@ function buildRequestIpc(schema, params, method) {
|
|
|
1452
1503
|
return serializeIpcStream(schema, [batch]);
|
|
1453
1504
|
}
|
|
1454
1505
|
async function readResponseBatches(body) {
|
|
1455
|
-
const reader = await
|
|
1506
|
+
const reader = await RecordBatchReader4.from(body);
|
|
1456
1507
|
await reader.open();
|
|
1457
1508
|
const schema = reader.schema;
|
|
1458
1509
|
if (!schema) {
|
|
@@ -1525,10 +1576,10 @@ async function readSequentialStreams(body) {
|
|
|
1525
1576
|
}
|
|
1526
1577
|
|
|
1527
1578
|
// src/client/introspect.ts
|
|
1528
|
-
import { RecordBatchReader as
|
|
1529
|
-
import { Schema as ArrowSchema } from "apache-arrow";
|
|
1530
|
-
async function
|
|
1531
|
-
const reader = await
|
|
1579
|
+
import { RecordBatchReader as RecordBatchReader5 } from "@query-farm/apache-arrow";
|
|
1580
|
+
import { Schema as ArrowSchema } from "@query-farm/apache-arrow";
|
|
1581
|
+
async function deserializeSchema2(bytes2) {
|
|
1582
|
+
const reader = await RecordBatchReader5.from(bytes2);
|
|
1532
1583
|
await reader.open();
|
|
1533
1584
|
return reader.schema;
|
|
1534
1585
|
}
|
|
@@ -1558,8 +1609,8 @@ async function parseDescribeResponse(batches, onLog) {
|
|
|
1558
1609
|
const paramDefaultsJson = dataBatch.getChildAt(7)?.get(i);
|
|
1559
1610
|
const hasHeader = dataBatch.getChildAt(8).get(i);
|
|
1560
1611
|
const headerIpc = dataBatch.getChildAt(9)?.get(i);
|
|
1561
|
-
const paramsSchema = await
|
|
1562
|
-
const resultSchema = await
|
|
1612
|
+
const paramsSchema = await deserializeSchema2(paramsIpc);
|
|
1613
|
+
const resultSchema = await deserializeSchema2(resultIpc);
|
|
1563
1614
|
let paramTypes;
|
|
1564
1615
|
if (paramTypesJson) {
|
|
1565
1616
|
try {
|
|
@@ -1585,7 +1636,7 @@ async function parseDescribeResponse(batches, onLog) {
|
|
|
1585
1636
|
info.outputSchema = resultSchema;
|
|
1586
1637
|
}
|
|
1587
1638
|
if (hasHeader && headerIpc) {
|
|
1588
|
-
info.headerSchema = await
|
|
1639
|
+
info.headerSchema = await deserializeSchema2(headerIpc);
|
|
1589
1640
|
}
|
|
1590
1641
|
methods.push(info);
|
|
1591
1642
|
}
|
|
@@ -1613,7 +1664,7 @@ import {
|
|
|
1613
1664
|
makeData as makeData5,
|
|
1614
1665
|
Struct as Struct5,
|
|
1615
1666
|
vectorFromArray as vectorFromArray4
|
|
1616
|
-
} from "apache-arrow";
|
|
1667
|
+
} from "@query-farm/apache-arrow";
|
|
1617
1668
|
class HttpStreamSession {
|
|
1618
1669
|
_baseUrl;
|
|
1619
1670
|
_prefix;
|
|
@@ -2030,7 +2081,7 @@ import {
|
|
|
2030
2081
|
Struct as Struct6,
|
|
2031
2082
|
makeData as makeData6,
|
|
2032
2083
|
vectorFromArray as vectorFromArray5
|
|
2033
|
-
} from "apache-arrow";
|
|
2084
|
+
} from "@query-farm/apache-arrow";
|
|
2034
2085
|
class PipeIncrementalWriter {
|
|
2035
2086
|
writer;
|
|
2036
2087
|
writeFn;
|
|
@@ -2455,11 +2506,13 @@ function subprocessConnect(cmd, options) {
|
|
|
2455
2506
|
return client;
|
|
2456
2507
|
}
|
|
2457
2508
|
export {
|
|
2509
|
+
unpackStateToken,
|
|
2458
2510
|
toSchema,
|
|
2459
2511
|
subprocessConnect,
|
|
2460
2512
|
str,
|
|
2461
2513
|
pipeConnect,
|
|
2462
2514
|
parseDescribeResponse,
|
|
2515
|
+
jsonStateSerializer,
|
|
2463
2516
|
int32,
|
|
2464
2517
|
int,
|
|
2465
2518
|
inferParamTypes,
|
|
@@ -2494,4 +2547,4 @@ export {
|
|
|
2494
2547
|
ARROW_CONTENT_TYPE
|
|
2495
2548
|
};
|
|
2496
2549
|
|
|
2497
|
-
//# debugId=
|
|
2550
|
+
//# debugId=90D671AA0075054164756E2164756E21
|