@query-farm/vgi 0.28.0 → 0.29.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +52 -8
- package/dist/arguments/argument-spec.d.ts +57 -2
- package/dist/arrow/impl-arrowjs/index.d.ts +1 -1
- package/dist/arrow/impl-arrowjs/iterate.d.ts +10 -1
- package/dist/arrow/impl-arrowjs/schema.d.ts +2 -0
- package/dist/arrow/impl-flechette/index.d.ts +1 -1
- package/dist/arrow/impl-flechette/iterate.d.ts +10 -1
- package/dist/arrow/impl-flechette/schema.d.ts +2 -0
- package/dist/arrow/index.d.ts +1 -1
- package/dist/arrow/schema-types.d.ts +3 -1
- package/dist/arrow/types.d.ts +39 -3
- package/dist/catalog/composite.d.ts +13 -13
- package/dist/catalog/descriptors.d.ts +7 -3
- package/dist/catalog/interface.d.ts +108 -38
- package/dist/catalog/read-only.d.ts +32 -15
- package/dist/client/client.d.ts +37 -30
- package/dist/client/types.d.ts +2 -0
- package/dist/client-entry.d.ts +1 -1
- package/dist/client-entry.js +791 -447
- package/dist/client-entry.js.map +18 -17
- package/dist/filter-pushdown/capabilities.d.ts +3 -0
- package/dist/filter-pushdown/deserialize.d.ts +14 -15
- package/dist/filter-pushdown/evaluate.d.ts +14 -44
- package/dist/filter-pushdown/index.d.ts +2 -2
- package/dist/filter-pushdown/types.d.ts +120 -68
- package/dist/functions/aggregate.d.ts +4 -0
- package/dist/functions/registry.d.ts +5 -5
- package/dist/functions/scalar.d.ts +6 -1
- package/dist/functions/table-buffering.d.ts +9 -0
- package/dist/functions/table-in-out.d.ts +13 -0
- package/dist/functions/table.d.ts +72 -2
- package/dist/functions/types.d.ts +60 -2
- package/dist/generated/vgi-client.d.ts +114 -26
- package/dist/generated/vgi-protocol-schemas.d.ts +34 -0
- package/dist/http/fetch.d.ts +16 -0
- package/dist/index.core.d.ts +4 -3
- package/dist/index.js +3396 -2064
- package/dist/index.js.map +55 -49
- package/dist/metadata/types.d.ts +15 -2
- package/dist/protocol/handlers/catalog/shared.d.ts +0 -8
- package/dist/protocol/handlers/function.d.ts +11 -0
- package/dist/protocol/handlers/shared.d.ts +3 -1
- package/dist/protocol/serializers/bind.d.ts +1 -2
- package/dist/protocol/serializers/cardinality.d.ts +1 -2
- package/dist/protocol/serializers/splits.d.ts +126 -0
- package/dist/protocol/types.d.ts +36 -3
- package/dist/request-auth.d.ts +28 -0
- package/dist/schema-path.d.ts +7 -0
- package/dist/serve-entry.d.ts +25 -0
- package/dist/serve-entry.js +1129 -569
- package/dist/serve-entry.js.map +36 -32
- package/dist/split-token.d.ts +88 -0
- package/dist/types.d.ts +8 -0
- package/dist/worker-cf-entry.js +2928 -1689
- package/dist/worker-cf-entry.js.map +51 -44
- package/dist/worker.d.ts +1 -1
- package/package.json +3 -3
- package/src/arguments/argument-spec.ts +93 -4
- package/src/arguments/arguments.ts +0 -1
- package/src/arrow/impl-arrowjs/index.ts +1 -1
- package/src/arrow/impl-arrowjs/iterate.ts +21 -0
- package/src/arrow/impl-arrowjs/schema.ts +4 -0
- package/src/arrow/impl-flechette/index.ts +1 -1
- package/src/arrow/impl-flechette/iterate.ts +24 -1
- package/src/arrow/impl-flechette/schema.ts +4 -0
- package/src/arrow/index.ts +1 -1
- package/src/arrow/schema-types.ts +4 -0
- package/src/arrow/types.ts +39 -3
- package/src/catalog/composite.ts +26 -26
- package/src/catalog/descriptors.ts +12 -3
- package/src/catalog/interface.ts +217 -72
- package/src/catalog/read-only.ts +141 -84
- package/src/client/client.ts +97 -73
- package/src/client/types.ts +2 -0
- package/src/client-entry.ts +2 -1
- package/src/filter-pushdown/capabilities.ts +13 -0
- package/src/filter-pushdown/collector.ts +22 -35
- package/src/filter-pushdown/deserialize.ts +657 -221
- package/src/filter-pushdown/evaluate.ts +410 -499
- package/src/filter-pushdown/index.ts +16 -2
- package/src/filter-pushdown/types.ts +88 -71
- package/src/functions/aggregate.ts +12 -1
- package/src/functions/copy-from.ts +2 -1
- package/src/functions/copy-to.ts +2 -1
- package/src/functions/registry.ts +24 -22
- package/src/functions/scalar.ts +37 -0
- package/src/functions/table-buffering.ts +48 -6
- package/src/functions/table-in-out.ts +194 -33
- package/src/functions/table.ts +187 -22
- package/src/functions/types.ts +64 -1
- package/src/generated/vgi-client.ts +122 -28
- package/src/generated/vgi-protocol-schemas.ts +455 -58
- package/src/http/fetch.ts +129 -2
- package/src/index.core.ts +4 -2
- package/src/metadata/resolve.ts +43 -1
- package/src/metadata/serialize.ts +43 -5
- package/src/metadata/types.ts +17 -1
- package/src/protocol/dispatch.ts +1 -1
- package/src/protocol/handlers/aggregate.ts +23 -12
- package/src/protocol/handlers/catalog/admin.ts +47 -54
- package/src/protocol/handlers/catalog/index_methods.ts +6 -13
- package/src/protocol/handlers/catalog/macro.ts +12 -20
- package/src/protocol/handlers/catalog/shared.ts +12 -57
- package/src/protocol/handlers/catalog/table.ts +49 -112
- package/src/protocol/handlers/catalog/view.ts +18 -22
- package/src/protocol/handlers/function.ts +311 -18
- package/src/protocol/handlers/shared.ts +12 -3
- package/src/protocol/handlers/table-buffering.ts +11 -8
- package/src/protocol/serializers/bind.ts +69 -67
- package/src/protocol/serializers/cardinality.ts +7 -7
- package/src/protocol/serializers/init.ts +52 -10
- package/src/protocol/serializers/splits.ts +221 -0
- package/src/protocol/state-serializer.ts +55 -1
- package/src/protocol/types.ts +36 -3
- package/src/request-auth.ts +73 -0
- package/src/schema-path.ts +25 -0
- package/src/serve-entry.ts +69 -2
- package/src/split-token.ts +297 -0
- package/src/types.ts +9 -0
- package/src/worker-cf-entry.ts +2 -1
- package/src/worker.ts +79 -14
package/README.md
CHANGED
|
@@ -1,4 +1,20 @@
|
|
|
1
|
-
|
|
1
|
+
<p align="center">
|
|
2
|
+
<img src="https://raw.githubusercontent.com/Query-farm/vgi-typescript/main/docs/vgi-logo.png" alt="Vector Gateway Interface logo" width="320">
|
|
3
|
+
</p>
|
|
4
|
+
|
|
5
|
+
<h1 align="center">VGI for TypeScript</h1>
|
|
6
|
+
|
|
7
|
+
<p align="center">
|
|
8
|
+
Add your own functions and tables to DuckDB with TypeScript and Apache Arrow.<br>
|
|
9
|
+
Built by <a href="https://query.farm">🚜 Query.Farm</a>
|
|
10
|
+
</p>
|
|
11
|
+
|
|
12
|
+
<p align="center">
|
|
13
|
+
<a href="https://github.com/Query-farm/vgi-typescript/actions/workflows/ci.yml"><img src="https://github.com/Query-farm/vgi-typescript/actions/workflows/ci.yml/badge.svg" alt="CI"></a>
|
|
14
|
+
<a href="https://www.npmjs.com/package/@query-farm/vgi"><img src="https://img.shields.io/npm/v/%40query-farm%2Fvgi" alt="npm"></a>
|
|
15
|
+
<a href="https://www.npmjs.com/package/@query-farm/vgi"><img src="https://img.shields.io/npm/dm/%40query-farm%2Fvgi" alt="npm downloads"></a>
|
|
16
|
+
<a href="LICENSE"><img src="https://img.shields.io/badge/license-Query%20Farm%20Source--Available-blue" alt="License"></a>
|
|
17
|
+
</p>
|
|
2
18
|
|
|
3
19
|
TypeScript framework for extending [DuckDB](https://duckdb.org) with user-defined
|
|
4
20
|
functions written in TypeScript, served to DuckDB over the **Vector Gateway
|
|
@@ -95,6 +111,7 @@ const upperCase = defineScalarFunction({
|
|
|
95
111
|
},
|
|
96
112
|
|
|
97
113
|
// Other optional fields: constParams, nullHandling, stability (volatility),
|
|
114
|
+
// argumentMonotonicity,
|
|
98
115
|
// examples, categories, tags, requiredSettings, requiredSecrets, maxWorkers.
|
|
99
116
|
});
|
|
100
117
|
|
|
@@ -361,6 +378,11 @@ const addHour = defineScalarFunction({
|
|
|
361
378
|
});
|
|
362
379
|
```
|
|
363
380
|
|
|
381
|
+
`argumentMonotonicity` is optional and scalar-only. If supplied, it must have
|
|
382
|
+
one entry per ordered parameter declaration. Fixed, defaulted, and constant
|
|
383
|
+
parameters each use one slot; a vararg declaration uses one slot regardless of
|
|
384
|
+
call-time expansion. Named invocation order does not reorder the metadata.
|
|
385
|
+
|
|
364
386
|
```ts
|
|
365
387
|
import {
|
|
366
388
|
Worker, defineScalarFunction, timestampMicros,
|
|
@@ -387,13 +409,32 @@ For manual conversions outside a function, `codecFor(type)` returns the codec wi
|
|
|
387
409
|
|
|
388
410
|
### Factory name note
|
|
389
411
|
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
412
|
+
Four names — `int`, `int32`, `float32`, `bool` — mean different things depending on
|
|
413
|
+
which entry point you import from:
|
|
414
|
+
|
|
415
|
+
| Import from | `int` is | Write |
|
|
416
|
+
| --- | --- | --- |
|
|
417
|
+
| `@query-farm/vgi` | a ready-made `Int64` **type**, re-exported from vgi-rpc | `params: { n: int }` |
|
|
418
|
+
| `@query-farm/vgi/worker-cf` | the typed **factory** `int(bitWidth?, signed?)` | `params: { n: int() }` |
|
|
419
|
+
|
|
420
|
+
The package root also re-exports `float`, `str` and `bytes` as ready-made types;
|
|
421
|
+
the Cloudflare entry does not have them at all (use `float64()`, `utf8()`,
|
|
422
|
+
`binary()`).
|
|
423
|
+
|
|
424
|
+
Everything else in the typed factory set — `int8`/`int16`/`int64`, `uint*`,
|
|
425
|
+
`float16`/`float64`, `utf8`, `decimal*`, `dateDay`, `timestampMicros`, `struct`,
|
|
426
|
+
`list`, `map`, … — is a **factory on both entry points** and is exported from the
|
|
427
|
+
package root as usual.
|
|
428
|
+
|
|
429
|
+
**Prefer the explicit factories in portable code.** `int64()` means the same thing
|
|
430
|
+
everywhere; `int` does not, so moving a worker to Cloudflare Workers can turn a
|
|
431
|
+
correct declaration into a wrong one with no other edit. Since 0.29.0 that mistake
|
|
432
|
+
is caught at definition time rather than silently registering a `Function` as an
|
|
433
|
+
argument type:
|
|
434
|
+
|
|
435
|
+
```
|
|
436
|
+
defineScalarFunction("double"): params.n is a type factory, not an Arrow type — call it: int64().
|
|
437
|
+
```
|
|
397
438
|
|
|
398
439
|
## Migration: the type-handling break
|
|
399
440
|
|
|
@@ -530,6 +571,9 @@ Subpath exports:
|
|
|
530
571
|
- **`vgi-python`** — the reference implementation. This package is a wire-compatible
|
|
531
572
|
TypeScript port; workers from either side interoperate.
|
|
532
573
|
|
|
574
|
+
See [Iroh operations](docs/iroh.md) for native Node/Bun and browser clients,
|
|
575
|
+
and identity-preserving worker bridge setup.
|
|
576
|
+
|
|
533
577
|
## Development
|
|
534
578
|
|
|
535
579
|
```bash
|
|
@@ -55,9 +55,64 @@ export declare function formatRange(ge?: number, le?: number, gt?: number, lt?:
|
|
|
55
55
|
/**
|
|
56
56
|
* Encode raw {@link ArgumentConstraints} into the pre-computed constraint
|
|
57
57
|
* fields of an {@link ArgumentSpec} (presence-only — absent constraints yield
|
|
58
|
-
* no field).
|
|
58
|
+
* no field). `runtimeDefault`, when present, is authoritative for
|
|
59
|
+
* `defaultJson`, keeping runtime binding and discovery metadata in sync.
|
|
60
|
+
* Mirrors Python's `_constraint_kwargs`.
|
|
59
61
|
*/
|
|
60
|
-
export declare function constraintSpecFields(constraints: ArgumentConstraints | undefined): Pick<ArgumentSpec, "defaultJson" | "choicesJson" | "rangeNotation" | "pattern">;
|
|
62
|
+
export declare function constraintSpecFields(constraints: ArgumentConstraints | undefined, runtimeDefault?: unknown): Pick<ArgumentSpec, "defaultJson" | "choicesJson" | "rangeNotation" | "pattern">;
|
|
63
|
+
/**
|
|
64
|
+
* Narrow a bind-time argument value from Arrow's bigint to a plain number.
|
|
65
|
+
*
|
|
66
|
+
* DuckDB types an integer literal as BIGINT, so `series(3)` arrives as `3n`
|
|
67
|
+
* whether or not anyone wanted 64 bits. Nobody writes `BigInt(1000)` for a page
|
|
68
|
+
* size, so the arg paths narrow — and they now all narrow the same way, which
|
|
69
|
+
* they did not before (two sites used safeNumber, two a bare Number).
|
|
70
|
+
*
|
|
71
|
+
* This is lossy above 2^53, and that is a deliberate call rather than an
|
|
72
|
+
* oversight. Every int64 *argument* across the fixtures and every shipped
|
|
73
|
+
* worker is a small control value — count, page_size, batch_size, increment,
|
|
74
|
+
* lag_minutes, restatement_lookback_days. The large int64s in the test suite
|
|
75
|
+
* are all somewhere else entirely:
|
|
76
|
+
*
|
|
77
|
+
* - column data (copy_to/types.test, copy_to/parallel.test) — never touches
|
|
78
|
+
* this path;
|
|
79
|
+
* - filter-pushdown constants (filter_pushdown/integers.test pins 2^53+1 on
|
|
80
|
+
* purpose) — deserialize.ts keeps those bigint, with its own regression
|
|
81
|
+
* test;
|
|
82
|
+
* - varargs values (constant_columns) — read raw off bindCall.arguments,
|
|
83
|
+
* because extractArgs' copy of them is not what the function reads;
|
|
84
|
+
* - time-travel versions — carried as strings (`at_value`).
|
|
85
|
+
*
|
|
86
|
+
* Preserving the bigint on overflow was tried and reverted: it buys nothing for
|
|
87
|
+
* any argument that exists, and costs a JS type that depends on magnitude, so a
|
|
88
|
+
* worker doing `args.n + 1` would work in every test and fail on one caller's
|
|
89
|
+
* input. If an argument ever genuinely needs the full range — a snowflake id, a
|
|
90
|
+
* nanosecond epoch — the fix is to make int64 arguments bigint by declared
|
|
91
|
+
* type, not by magnitude.
|
|
92
|
+
*/
|
|
93
|
+
export declare function narrowArgValue(value: unknown): unknown;
|
|
94
|
+
/**
|
|
95
|
+
* Reject a type *factory* where an Arrow *type* was expected.
|
|
96
|
+
*
|
|
97
|
+
* `params: { n: int64 }` (no parentheses) is accepted by the type system —
|
|
98
|
+
* `VgiDataType` is loose enough — and then silently produces an ArgumentSpec
|
|
99
|
+
* whose `arrowType` is a Function. Nothing complains until the schema is
|
|
100
|
+
* serialized or the bind arrives, and the error at that point names neither the
|
|
101
|
+
* argument nor the cause.
|
|
102
|
+
*
|
|
103
|
+
* The mistake is easy to make on purpose, too: the same identifier is a
|
|
104
|
+
* ready-made type on one entry point and a factory on another.
|
|
105
|
+
*
|
|
106
|
+
* @query-farm/vgi int, int32, float32, bool → type instances
|
|
107
|
+
* @query-farm/vgi/worker-cf int, int32, float32, bool → factories
|
|
108
|
+
*
|
|
109
|
+
* So moving a worker to Cloudflare can turn a correct declaration into a
|
|
110
|
+
* silently wrong one with no other edit. Failing here, at definition time,
|
|
111
|
+
* makes that a startup error naming the field.
|
|
112
|
+
*/
|
|
113
|
+
export declare function assertArrowType(value: unknown, where: string): void;
|
|
114
|
+
/** {@link assertArrowType} over a record of declared types, e.g. `params`. */
|
|
115
|
+
export declare function assertArrowTypes(types: Record<string, unknown> | undefined, where: string): void;
|
|
61
116
|
/**
|
|
62
117
|
* Enforce a bind-time const value against its declared {@link ArgumentConstraints},
|
|
63
118
|
* throwing {@link ArgumentValidationError} on violation. Mirrors the Python SDK's
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { VgiBackendInfo } from "../types.js";
|
|
2
2
|
export declare const backend: VgiBackendInfo;
|
|
3
|
-
export { schema, field, nullType, bool, int, int8, int16, int32, int64, uint8, uint16, uint32, uint64, float16, float32, float64, utf8, binary, fixedSizeBinary, decimal, decimal128, decimal256, date, dateDay, dateMillisecond, time, timeSecond, timeMillisecond, timeMicrosecond, timeNanosecond, timestamp, duration, interval, list, fixedSizeList, struct, map, dictionary, union, sparseUnion, denseUnion, TimeUnit, DateUnit, IntervalUnit, UnionMode, } from "./schema.js";
|
|
3
|
+
export { schema, field, nullType, bool, int, int8, int16, int32, int64, uint8, uint16, uint32, uint64, float16, float32, float64, utf8, largeUtf8, binary, largeBinary, fixedSizeBinary, decimal, decimal128, decimal256, date, dateDay, dateMillisecond, time, timeSecond, timeMillisecond, timeMicrosecond, timeNanosecond, timestamp, duration, interval, list, fixedSizeList, struct, map, dictionary, union, sparseUnion, denseUnion, TimeUnit, DateUnit, IntervalUnit, UnionMode, } from "./schema.js";
|
|
4
4
|
export { emptyBatch } from "./empty.js";
|
|
5
5
|
export { batchFromRows, batchFromColumns, columnFromArray } from "./build.js";
|
|
6
6
|
export { iterRows, batchToScalarDict, batchToSecretDict, safeNumber, decodeDictValue, } from "./iterate.js";
|
|
@@ -34,6 +34,15 @@ export declare function decodeDictValue(value: any, index?: number): any;
|
|
|
34
34
|
*/
|
|
35
35
|
export declare function batchToSecretDict(batch: RecordBatch | VgiBatch | null): Record<string, Record<string, any>>;
|
|
36
36
|
/**
|
|
37
|
-
*
|
|
37
|
+
* Narrow a bigint to a number, refusing to do it lossily.
|
|
38
|
+
*
|
|
39
|
+
* Arrow int64/uint64 values arrive as bigint. Several bind-time argument paths
|
|
40
|
+
* want a plain number, and used to get one from a bare `Number(value)` — which
|
|
41
|
+
* silently rounds above 2^53, so an id or a nanosecond timestamp passed as a
|
|
42
|
+
* function argument came back subtly wrong with nothing logged anywhere.
|
|
43
|
+
*
|
|
44
|
+
* Throwing is the lesser evil: a caller that genuinely wants a lossy narrowing
|
|
45
|
+
* can still write `Number(v)` themselves, but nobody has to discover the loss
|
|
46
|
+
* from mismatched output weeks later.
|
|
38
47
|
*/
|
|
39
48
|
export declare function safeNumber(value: any): number;
|
|
@@ -36,7 +36,9 @@ export declare const float16: () => VgiDataType;
|
|
|
36
36
|
export declare const float32: () => VgiDataType;
|
|
37
37
|
export declare const float64: () => VgiDataType;
|
|
38
38
|
export declare const utf8: () => VgiDataType;
|
|
39
|
+
export declare const largeUtf8: () => VgiDataType;
|
|
39
40
|
export declare const binary: () => VgiDataType;
|
|
41
|
+
export declare const largeBinary: () => VgiDataType;
|
|
40
42
|
export declare const fixedSizeBinary: (byteWidth: number) => VgiDataType;
|
|
41
43
|
export declare function decimal(precision: number, scale: number, bitWidth?: 32 | 64 | 128 | 256): VgiDataType;
|
|
42
44
|
export declare const decimal128: (precision: number, scale: number) => VgiDataType;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { VgiBackendInfo } from "../types.js";
|
|
2
2
|
import "./compat.js";
|
|
3
3
|
export declare const backend: VgiBackendInfo;
|
|
4
|
-
export { schema, field, nullType, bool, int, int8, int16, int32, int64, uint8, uint16, uint32, uint64, float16, float32, float64, utf8, binary, fixedSizeBinary, decimal, decimal128, decimal256, date, dateDay, dateMillisecond, time, timeSecond, timeMillisecond, timeMicrosecond, timeNanosecond, timestamp, duration, interval, list, fixedSizeList, struct, map, dictionary, union, sparseUnion, denseUnion, TimeUnit, DateUnit, IntervalUnit, UnionMode, } from "./schema.js";
|
|
4
|
+
export { schema, field, nullType, bool, int, int8, int16, int32, int64, uint8, uint16, uint32, uint64, float16, float32, float64, utf8, largeUtf8, binary, largeBinary, fixedSizeBinary, decimal, decimal128, decimal256, date, dateDay, dateMillisecond, time, timeSecond, timeMillisecond, timeMicrosecond, timeNanosecond, timestamp, duration, interval, list, fixedSizeList, struct, map, dictionary, union, sparseUnion, denseUnion, TimeUnit, DateUnit, IntervalUnit, UnionMode, } from "./schema.js";
|
|
5
5
|
export { emptyBatch } from "./empty.js";
|
|
6
6
|
export { batchFromRows, batchFromColumns, columnFromArray } from "./build.js";
|
|
7
7
|
export { iterRows, batchToScalarDict, batchToSecretDict, safeNumber, decodeDictValue, } from "./iterate.js";
|
|
@@ -24,6 +24,15 @@ export declare function decodeDictValue(value: any, _index?: number): any;
|
|
|
24
24
|
*/
|
|
25
25
|
export declare function batchToSecretDict(batch: VgiBatch | null): Record<string, Record<string, any>>;
|
|
26
26
|
/**
|
|
27
|
-
*
|
|
27
|
+
* Narrow a bigint to a number, refusing to do it lossily.
|
|
28
|
+
*
|
|
29
|
+
* Arrow int64/uint64 values arrive as bigint. Several bind-time argument paths
|
|
30
|
+
* want a plain number, and used to get one from a bare `Number(value)` — which
|
|
31
|
+
* silently rounds above 2^53, so an id or a nanosecond timestamp passed as a
|
|
32
|
+
* function argument came back subtly wrong with nothing logged anywhere.
|
|
33
|
+
*
|
|
34
|
+
* Throwing is the lesser evil: a caller that genuinely wants a lossy narrowing
|
|
35
|
+
* can still write `Number(v)` themselves, but nobody has to discover the loss
|
|
36
|
+
* from mismatched output weeks later.
|
|
28
37
|
*/
|
|
29
38
|
export declare function safeNumber(value: any): number;
|
|
@@ -21,7 +21,9 @@ export declare const float16: () => VgiDataType;
|
|
|
21
21
|
export declare const float32: () => VgiDataType;
|
|
22
22
|
export declare const float64: () => VgiDataType;
|
|
23
23
|
export declare const utf8: () => VgiDataType;
|
|
24
|
+
export declare const largeUtf8: () => VgiDataType;
|
|
24
25
|
export declare const binary: () => VgiDataType;
|
|
26
|
+
export declare const largeBinary: () => VgiDataType;
|
|
25
27
|
export declare const fixedSizeBinary: (stride: number) => VgiDataType;
|
|
26
28
|
export declare function decimal(precision: number, scale: number, bitWidth?: 32 | 64 | 128 | 256): VgiDataType;
|
|
27
29
|
export declare const decimal128: (precision: number, scale: number) => VgiDataType;
|
package/dist/arrow/index.d.ts
CHANGED
|
@@ -2,7 +2,7 @@ export type { VgiTypeId, VgiDataType, VgiField, VgiSchema, VgiColumn, VgiColumnD
|
|
|
2
2
|
export { TypeId, isNull, isInt, isFloat, isBinary, isUtf8, isBool, isDecimal, isDate, isTime, isTimestamp, isInterval, isList, isStruct, isUnion, isFixedSizeBinary, isFixedSizeList, isMap, isDuration, isDictionary, isBatch, typeSignature, } from "./predicates.js";
|
|
3
3
|
export { backend, emptyBatch, batchFromRows, batchFromColumns, columnFromArray, iterRows, batchToScalarDict, batchToSecretDict, safeNumber, decodeDictValue, filterBatch, projectSchema, projectBatch, withBatchMetadata, adoptArrowJsShape, readCanonicalValue, serializeSchema, deserializeSchema, serializeBatch, deserializeBatch, buildStatisticsBatch, } from "#arrow-impl";
|
|
4
4
|
export type { ColumnStatistics } from "#arrow-impl";
|
|
5
|
-
export { schema, field, nullType, bool, int, int8, int16, int32, int64, uint8, uint16, uint32, uint64, float16, float32, float64, utf8, binary, fixedSizeBinary, decimal, decimal128, decimal256, date, dateDay, dateMillisecond, time, timeSecond, timeMillisecond, timeMicrosecond, timeNanosecond, timestamp, duration, interval, timestampSeconds, timestampMillis, timestampMicros, timestampNanos, durationSeconds, durationMillis, durationMicros, durationNanos, list, fixedSizeList, struct, map, dictionary, union, sparseUnion, denseUnion, TimeUnit, DateUnit, IntervalUnit, UnionMode, } from "./schema-types.js";
|
|
5
|
+
export { schema, field, nullType, bool, int, int8, int16, int32, int64, uint8, uint16, uint32, uint64, float16, float32, float64, utf8, largeUtf8, binary, largeBinary, fixedSizeBinary, decimal, decimal128, decimal256, date, dateDay, dateMillisecond, time, timeSecond, timeMillisecond, timeMicrosecond, timeNanosecond, timestamp, duration, interval, timestampSeconds, timestampMillis, timestampMicros, timestampNanos, durationSeconds, durationMillis, durationMicros, durationNanos, list, fixedSizeList, struct, map, dictionary, union, sparseUnion, denseUnion, TimeUnit, DateUnit, IntervalUnit, UnionMode, } from "./schema-types.js";
|
|
6
6
|
export type { NullDescriptor, BoolType, IntType, Int8Type, Int16Type, Int32Type, Int64Type, Uint8Type, Uint16Type, Uint32Type, Uint64Type, FloatType, Utf8Type, LargeUtf8Type, BinaryType, LargeBinaryType, FixedSizeBinaryType, DecimalType, Date32Type, Date64Type, Time32Type, Time64Type, TimestampType, DurationType, IntervalType, ListType, FixedSizeListType, StructType, MapType, DictionaryType, UnionType, TypedField, TUnit, } from "./codec/type-descriptors.js";
|
|
7
7
|
export type { RichValue, RawValue, ValueFor, Repr } from "./codec/repr.js";
|
|
8
8
|
export { codecFor } from "./codec/registry.js";
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import * as impl from "#arrow-impl";
|
|
2
2
|
import type { VgiField, VgiDataType } from "./types.js";
|
|
3
|
-
import type { NullDescriptor, BoolType, Int8Type, Int16Type, Int32Type, Int64Type, Uint8Type, Uint16Type, Uint32Type, Uint64Type, IntType, FloatType, Utf8Type, BinaryType, FixedSizeBinaryType, DecimalType, Date32Type, Date64Type, Time32Type, Time64Type, TimestampType, DurationType, IntervalType, ListType, FixedSizeListType, StructType, MapType, DictionaryType, UnionType, TypedField, TUnit } from "./codec/type-descriptors.js";
|
|
3
|
+
import type { NullDescriptor, BoolType, Int8Type, Int16Type, Int32Type, Int64Type, Uint8Type, Uint16Type, Uint32Type, Uint64Type, IntType, FloatType, Utf8Type, LargeUtf8Type, BinaryType, LargeBinaryType, FixedSizeBinaryType, DecimalType, Date32Type, Date64Type, Time32Type, Time64Type, TimestampType, DurationType, IntervalType, ListType, FixedSizeListType, StructType, MapType, DictionaryType, UnionType, TypedField, TUnit } from "./codec/type-descriptors.js";
|
|
4
4
|
export declare const TimeUnit: {
|
|
5
5
|
readonly SECOND: import("@query-farm/apache-arrow").TimeUnit.SECOND;
|
|
6
6
|
readonly MILLISECOND: import("@query-farm/apache-arrow").TimeUnit.MILLISECOND;
|
|
@@ -35,7 +35,9 @@ export declare const float16: () => FloatType;
|
|
|
35
35
|
export declare const float32: () => FloatType;
|
|
36
36
|
export declare const float64: () => FloatType;
|
|
37
37
|
export declare const utf8: () => Utf8Type;
|
|
38
|
+
export declare const largeUtf8: () => LargeUtf8Type;
|
|
38
39
|
export declare const binary: () => BinaryType;
|
|
40
|
+
export declare const largeBinary: () => LargeBinaryType;
|
|
39
41
|
export declare const fixedSizeBinary: (byteWidth: number) => FixedSizeBinaryType;
|
|
40
42
|
export declare const decimal: (precision: number, scale: number, bitWidth?: 32 | 64 | 128 | 256) => DecimalType;
|
|
41
43
|
export declare const decimal128: (precision: number, scale: number) => DecimalType;
|
package/dist/arrow/types.d.ts
CHANGED
|
@@ -20,9 +20,43 @@ export interface VgiSchema {
|
|
|
20
20
|
readonly fields: readonly VgiField[];
|
|
21
21
|
readonly metadata: Map<string, string>;
|
|
22
22
|
}
|
|
23
|
-
/**
|
|
24
|
-
*
|
|
25
|
-
*
|
|
23
|
+
/**
|
|
24
|
+
* Column view over one column of a batch.
|
|
25
|
+
*
|
|
26
|
+
* The value type is erased by default: arrow-js parameterizes on DataType and
|
|
27
|
+
* flechette on value type, so the facade cannot name one without picking a
|
|
28
|
+
* backend. Supply `T` at the use site — the column's declared type makes it
|
|
29
|
+
* known there — and the cast disappears:
|
|
30
|
+
*
|
|
31
|
+
* ```ts
|
|
32
|
+
* const ns = batch.getChildAt(0)! as Iterable<bigint | null>;
|
|
33
|
+
* for (const v of ns) { ... } // v: bigint | null
|
|
34
|
+
* ```
|
|
35
|
+
*
|
|
36
|
+
* A type parameter here would read better, but it cannot be had cheaply: both
|
|
37
|
+
* backends' native batches are assigned to VgiBatch structurally, and a
|
|
38
|
+
* concrete `get(): unknown` does not satisfy a generic `get(): T`. Adding one
|
|
39
|
+
* would mean casting at ~25 internal sites to remove one cast in user code.
|
|
40
|
+
*
|
|
41
|
+
* **These are the backend's own values, not codec output.** `get()` and
|
|
42
|
+
* iteration return whatever the Arrow implementation stores, which for some
|
|
43
|
+
* types is not the value the SDK documents:
|
|
44
|
+
*
|
|
45
|
+
* | Arrow type | here | {@link iterRows} |
|
|
46
|
+
* | ---------------- | ----------------------------- | --------------------- |
|
|
47
|
+
* | `int64` | `bigint` | `bigint` (same) |
|
|
48
|
+
* | `utf8`, `float64`| `string`, `number` | same |
|
|
49
|
+
* | `decimal128` | backend limbs (`DecimalBigNum`)| `bigint`, unscaled |
|
|
50
|
+
* | `timestamp[us]` | millisecond `number` | microsecond `bigint` |
|
|
51
|
+
* | `date32` | millisecond `number` | `Date` |
|
|
52
|
+
*
|
|
53
|
+
* `repr: "raw"` on a function selects the *codec's* representation and has no
|
|
54
|
+
* effect on this path.
|
|
55
|
+
*
|
|
56
|
+
* So: reach for `getChildAt` on integer, float, boolean, string and binary
|
|
57
|
+
* columns, where it is the cheapest correct thing. For temporal, decimal and
|
|
58
|
+
* nested types go through {@link iterRows}, which runs the codec.
|
|
59
|
+
*/
|
|
26
60
|
export interface VgiColumn {
|
|
27
61
|
readonly type: VgiDataType;
|
|
28
62
|
readonly length: number;
|
|
@@ -32,7 +66,9 @@ export interface VgiColumn {
|
|
|
32
66
|
export interface VgiBatch {
|
|
33
67
|
readonly schema: VgiSchema;
|
|
34
68
|
readonly numRows: number;
|
|
69
|
+
/** See {@link VgiColumn} — this does NOT run the codec. */
|
|
35
70
|
getChild(name: string): VgiColumn | null;
|
|
71
|
+
/** See {@link VgiColumn} — this does NOT run the codec. */
|
|
36
72
|
getChildAt(index: number): VgiColumn | null;
|
|
37
73
|
}
|
|
38
74
|
export interface VgiBackendInfo {
|
|
@@ -12,22 +12,22 @@ export declare class CompositeCatalogInterface extends CatalogInterface {
|
|
|
12
12
|
detach(attachOpaqueData: AttachOpaqueData): Promise<void>;
|
|
13
13
|
version(attachOpaqueData: AttachOpaqueData, transactionOpaqueData?: TransactionOpaqueData): Promise<number>;
|
|
14
14
|
schemas(attachOpaqueData: AttachOpaqueData, transactionOpaqueData?: TransactionOpaqueData): Promise<SchemaInfo[]>;
|
|
15
|
-
schemaGet(attachOpaqueData: AttachOpaqueData,
|
|
16
|
-
schemaContentsTables(attachOpaqueData: AttachOpaqueData,
|
|
17
|
-
schemaContentsViews(attachOpaqueData: AttachOpaqueData,
|
|
18
|
-
schemaContentsFunctions(attachOpaqueData: AttachOpaqueData,
|
|
19
|
-
schemaContentsMacros(attachOpaqueData: AttachOpaqueData,
|
|
20
|
-
schemaContentsIndexes(attachOpaqueData: AttachOpaqueData,
|
|
21
|
-
indexGet(attachOpaqueData: AttachOpaqueData,
|
|
22
|
-
tableGet(attachOpaqueData: AttachOpaqueData,
|
|
23
|
-
tableScanFunctionGet(attachOpaqueData: AttachOpaqueData,
|
|
24
|
-
tableScanBranchesGet(attachOpaqueData: AttachOpaqueData,
|
|
25
|
-
tableColumnStatisticsGet(attachOpaqueData: AttachOpaqueData,
|
|
15
|
+
schemaGet(attachOpaqueData: AttachOpaqueData, path: string[], transactionOpaqueData?: TransactionOpaqueData): Promise<SchemaInfo | null>;
|
|
16
|
+
schemaContentsTables(attachOpaqueData: AttachOpaqueData, path: string[], transactionOpaqueData?: TransactionOpaqueData): Promise<TableInfo[]>;
|
|
17
|
+
schemaContentsViews(attachOpaqueData: AttachOpaqueData, path: string[], transactionOpaqueData?: TransactionOpaqueData): Promise<ViewInfo[]>;
|
|
18
|
+
schemaContentsFunctions(attachOpaqueData: AttachOpaqueData, path: string[], type: string, transactionOpaqueData?: TransactionOpaqueData): Promise<FunctionInfo[]>;
|
|
19
|
+
schemaContentsMacros(attachOpaqueData: AttachOpaqueData, path: string[], type: string, transactionOpaqueData?: TransactionOpaqueData): Promise<MacroInfo[]>;
|
|
20
|
+
schemaContentsIndexes(attachOpaqueData: AttachOpaqueData, path: string[], transactionOpaqueData?: TransactionOpaqueData): Promise<IndexInfo[]>;
|
|
21
|
+
indexGet(attachOpaqueData: AttachOpaqueData, schemaPath: string[], name: string, transactionOpaqueData?: TransactionOpaqueData): Promise<IndexInfo | null>;
|
|
22
|
+
tableGet(attachOpaqueData: AttachOpaqueData, schemaPath: string[], name: string, atUnit?: string, atValue?: string, transactionOpaqueData?: TransactionOpaqueData): Promise<TableInfo | null>;
|
|
23
|
+
tableScanFunctionGet(attachOpaqueData: AttachOpaqueData, schemaPath: string[], name: string, atUnit?: string, atValue?: string, transactionOpaqueData?: TransactionOpaqueData): Promise<any>;
|
|
24
|
+
tableScanBranchesGet(attachOpaqueData: AttachOpaqueData, schemaPath: string[], name: string, atUnit?: string, atValue?: string, transactionOpaqueData?: TransactionOpaqueData): Promise<any>;
|
|
25
|
+
tableColumnStatisticsGet(attachOpaqueData: AttachOpaqueData, schemaPath: string[], name: string, transactionOpaqueData?: TransactionOpaqueData): Promise<{
|
|
26
26
|
bytes: Uint8Array;
|
|
27
27
|
cacheMaxAgeSeconds: number | null;
|
|
28
28
|
} | null>;
|
|
29
|
-
viewGet(attachOpaqueData: AttachOpaqueData,
|
|
30
|
-
macroGet(attachOpaqueData: AttachOpaqueData,
|
|
29
|
+
viewGet(attachOpaqueData: AttachOpaqueData, schemaPath: string[], name: string, transactionOpaqueData?: TransactionOpaqueData): Promise<ViewInfo | null>;
|
|
30
|
+
macroGet(attachOpaqueData: AttachOpaqueData, schemaPath: string[], name: string, transactionOpaqueData?: TransactionOpaqueData): Promise<MacroInfo | null>;
|
|
31
31
|
copyFromFormats(attachOpaqueData: AttachOpaqueData, transactionOpaqueData?: TransactionOpaqueData): Promise<CopyFromFormatInfo[]>;
|
|
32
32
|
transactionBegin(attachOpaqueData: AttachOpaqueData): Promise<Uint8Array | null>;
|
|
33
33
|
transactionCommit(attachOpaqueData: AttachOpaqueData, transactionOpaqueData: TransactionOpaqueData): Promise<void>;
|
|
@@ -14,7 +14,7 @@ export interface ForeignKeyDef {
|
|
|
14
14
|
columns: string[];
|
|
15
15
|
referencedTable: string;
|
|
16
16
|
referencedColumns: string[];
|
|
17
|
-
|
|
17
|
+
referencedSchemaPath?: string[];
|
|
18
18
|
}
|
|
19
19
|
export type DefaultValue = string | number | boolean | null;
|
|
20
20
|
export interface TableDescriptor {
|
|
@@ -129,7 +129,10 @@ export interface IndexDescriptor {
|
|
|
129
129
|
tags?: Record<string, string>;
|
|
130
130
|
}
|
|
131
131
|
export interface SchemaDescriptor {
|
|
132
|
-
|
|
132
|
+
/** Raw identifier components, from the catalog root to this schema. */
|
|
133
|
+
path?: string[];
|
|
134
|
+
/** @deprecated Protocol 1.x compatibility; prefer `path: [name]`. */
|
|
135
|
+
name?: string;
|
|
133
136
|
tables?: TableDescriptor[];
|
|
134
137
|
views?: ViewDescriptor[];
|
|
135
138
|
macros?: MacroDescriptor[];
|
|
@@ -138,6 +141,7 @@ export interface SchemaDescriptor {
|
|
|
138
141
|
comment?: string;
|
|
139
142
|
tags?: Record<string, string>;
|
|
140
143
|
}
|
|
144
|
+
export declare function schemaDescriptorPath(schema: SchemaDescriptor): string[];
|
|
141
145
|
export interface SecretTypeDescriptor {
|
|
142
146
|
name: string;
|
|
143
147
|
description: string;
|
|
@@ -170,7 +174,7 @@ export interface CatalogDescriptor {
|
|
|
170
174
|
* records on `catalog_attach.global_functions`.
|
|
171
175
|
*
|
|
172
176
|
* Every entry must also be declared in exactly one of this catalog's
|
|
173
|
-
* schemas — the `FunctionInfo.
|
|
177
|
+
* schemas — the `FunctionInfo.schema_path` it carries is the bind-dispatch
|
|
174
178
|
* key, not a sentinel. Mirrors vgi-python's `Catalog.global_functions`.
|
|
175
179
|
*/
|
|
176
180
|
globalFunctions?: VgiFunction[];
|