@query-farm/vgi 0.28.0 → 0.29.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +52 -8
- package/dist/arguments/argument-spec.d.ts +57 -2
- package/dist/arrow/impl-arrowjs/index.d.ts +1 -1
- package/dist/arrow/impl-arrowjs/iterate.d.ts +10 -1
- package/dist/arrow/impl-arrowjs/schema.d.ts +2 -0
- package/dist/arrow/impl-flechette/index.d.ts +1 -1
- package/dist/arrow/impl-flechette/iterate.d.ts +10 -1
- package/dist/arrow/impl-flechette/schema.d.ts +2 -0
- package/dist/arrow/index.d.ts +1 -1
- package/dist/arrow/schema-types.d.ts +3 -1
- package/dist/arrow/types.d.ts +39 -3
- package/dist/catalog/composite.d.ts +13 -13
- package/dist/catalog/descriptors.d.ts +7 -3
- package/dist/catalog/interface.d.ts +108 -38
- package/dist/catalog/read-only.d.ts +32 -15
- package/dist/client/client.d.ts +37 -30
- package/dist/client/types.d.ts +2 -0
- package/dist/client-entry.d.ts +1 -1
- package/dist/client-entry.js +791 -447
- package/dist/client-entry.js.map +18 -17
- package/dist/filter-pushdown/capabilities.d.ts +3 -0
- package/dist/filter-pushdown/deserialize.d.ts +14 -15
- package/dist/filter-pushdown/evaluate.d.ts +14 -44
- package/dist/filter-pushdown/index.d.ts +2 -2
- package/dist/filter-pushdown/types.d.ts +120 -68
- package/dist/functions/aggregate.d.ts +4 -0
- package/dist/functions/registry.d.ts +5 -5
- package/dist/functions/scalar.d.ts +6 -1
- package/dist/functions/table-buffering.d.ts +9 -0
- package/dist/functions/table-in-out.d.ts +13 -0
- package/dist/functions/table.d.ts +72 -2
- package/dist/functions/types.d.ts +60 -2
- package/dist/generated/vgi-client.d.ts +114 -26
- package/dist/generated/vgi-protocol-schemas.d.ts +34 -0
- package/dist/http/fetch.d.ts +16 -0
- package/dist/index.core.d.ts +4 -3
- package/dist/index.js +3396 -2064
- package/dist/index.js.map +55 -49
- package/dist/metadata/types.d.ts +15 -2
- package/dist/protocol/handlers/catalog/shared.d.ts +0 -8
- package/dist/protocol/handlers/function.d.ts +11 -0
- package/dist/protocol/handlers/shared.d.ts +3 -1
- package/dist/protocol/serializers/bind.d.ts +1 -2
- package/dist/protocol/serializers/cardinality.d.ts +1 -2
- package/dist/protocol/serializers/splits.d.ts +126 -0
- package/dist/protocol/types.d.ts +36 -3
- package/dist/request-auth.d.ts +28 -0
- package/dist/schema-path.d.ts +7 -0
- package/dist/serve-entry.d.ts +25 -0
- package/dist/serve-entry.js +1129 -569
- package/dist/serve-entry.js.map +36 -32
- package/dist/split-token.d.ts +88 -0
- package/dist/types.d.ts +8 -0
- package/dist/worker-cf-entry.js +2928 -1689
- package/dist/worker-cf-entry.js.map +51 -44
- package/dist/worker.d.ts +1 -1
- package/package.json +3 -3
- package/src/arguments/argument-spec.ts +93 -4
- package/src/arguments/arguments.ts +0 -1
- package/src/arrow/impl-arrowjs/index.ts +1 -1
- package/src/arrow/impl-arrowjs/iterate.ts +21 -0
- package/src/arrow/impl-arrowjs/schema.ts +4 -0
- package/src/arrow/impl-flechette/index.ts +1 -1
- package/src/arrow/impl-flechette/iterate.ts +24 -1
- package/src/arrow/impl-flechette/schema.ts +4 -0
- package/src/arrow/index.ts +1 -1
- package/src/arrow/schema-types.ts +4 -0
- package/src/arrow/types.ts +39 -3
- package/src/catalog/composite.ts +26 -26
- package/src/catalog/descriptors.ts +12 -3
- package/src/catalog/interface.ts +217 -72
- package/src/catalog/read-only.ts +141 -84
- package/src/client/client.ts +97 -73
- package/src/client/types.ts +2 -0
- package/src/client-entry.ts +2 -1
- package/src/filter-pushdown/capabilities.ts +13 -0
- package/src/filter-pushdown/collector.ts +22 -35
- package/src/filter-pushdown/deserialize.ts +657 -221
- package/src/filter-pushdown/evaluate.ts +410 -499
- package/src/filter-pushdown/index.ts +16 -2
- package/src/filter-pushdown/types.ts +88 -71
- package/src/functions/aggregate.ts +12 -1
- package/src/functions/copy-from.ts +2 -1
- package/src/functions/copy-to.ts +2 -1
- package/src/functions/registry.ts +24 -22
- package/src/functions/scalar.ts +37 -0
- package/src/functions/table-buffering.ts +48 -6
- package/src/functions/table-in-out.ts +194 -33
- package/src/functions/table.ts +187 -22
- package/src/functions/types.ts +64 -1
- package/src/generated/vgi-client.ts +122 -28
- package/src/generated/vgi-protocol-schemas.ts +455 -58
- package/src/http/fetch.ts +129 -2
- package/src/index.core.ts +4 -2
- package/src/metadata/resolve.ts +43 -1
- package/src/metadata/serialize.ts +43 -5
- package/src/metadata/types.ts +17 -1
- package/src/protocol/dispatch.ts +1 -1
- package/src/protocol/handlers/aggregate.ts +23 -12
- package/src/protocol/handlers/catalog/admin.ts +47 -54
- package/src/protocol/handlers/catalog/index_methods.ts +6 -13
- package/src/protocol/handlers/catalog/macro.ts +12 -20
- package/src/protocol/handlers/catalog/shared.ts +12 -57
- package/src/protocol/handlers/catalog/table.ts +49 -112
- package/src/protocol/handlers/catalog/view.ts +18 -22
- package/src/protocol/handlers/function.ts +311 -18
- package/src/protocol/handlers/shared.ts +12 -3
- package/src/protocol/handlers/table-buffering.ts +11 -8
- package/src/protocol/serializers/bind.ts +69 -67
- package/src/protocol/serializers/cardinality.ts +7 -7
- package/src/protocol/serializers/init.ts +52 -10
- package/src/protocol/serializers/splits.ts +221 -0
- package/src/protocol/state-serializer.ts +55 -1
- package/src/protocol/types.ts +36 -3
- package/src/request-auth.ts +73 -0
- package/src/schema-path.ts +25 -0
- package/src/serve-entry.ts +69 -2
- package/src/split-token.ts +297 -0
- package/src/types.ts +9 -0
- package/src/worker-cf-entry.ts +2 -1
- package/src/worker.ts +79 -14
package/dist/worker.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { FunctionRegistry } from "./functions/registry.js";
|
|
2
2
|
import type { VgiFunction } from "./functions/types.js";
|
|
3
|
-
import type
|
|
3
|
+
import { type CatalogDescriptor } from "./catalog/descriptors.js";
|
|
4
4
|
import type { CatalogInterface } from "./catalog/interface.js";
|
|
5
5
|
import { ReadOnlyCatalogInterface } from "./catalog/read-only.js";
|
|
6
6
|
export interface WorkerConfig {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@query-farm/vgi",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.29.1",
|
|
4
4
|
"description": "TypeScript framework for extending DuckDB with user-defined functions via the Vector Gateway Interface (VGI) over Arrow IPC.",
|
|
5
5
|
"license": "SEE LICENSE IN LICENSE",
|
|
6
6
|
"author": "Query Farm LLC (https://query.farm)",
|
|
@@ -90,12 +90,12 @@
|
|
|
90
90
|
},
|
|
91
91
|
"peerDependencies": {
|
|
92
92
|
"@query-farm/apache-arrow": "^21.1.1",
|
|
93
|
-
"@query-farm/vgi-rpc": ">=0.
|
|
93
|
+
"@query-farm/vgi-rpc": ">=0.23.0 <0.24.0"
|
|
94
94
|
},
|
|
95
95
|
"devDependencies": {
|
|
96
96
|
"@duckdb/node-api": "^1.5.5-r.4",
|
|
97
97
|
"@query-farm/apache-arrow": "^21.1.1",
|
|
98
|
-
"@query-farm/vgi-rpc": "^0.
|
|
98
|
+
"@query-farm/vgi-rpc": "^0.23.0",
|
|
99
99
|
"@types/bun": "latest",
|
|
100
100
|
"typescript": "^7.0.2"
|
|
101
101
|
},
|
|
@@ -120,20 +120,28 @@ export function formatRange(
|
|
|
120
120
|
/**
|
|
121
121
|
* Encode raw {@link ArgumentConstraints} into the pre-computed constraint
|
|
122
122
|
* fields of an {@link ArgumentSpec} (presence-only — absent constraints yield
|
|
123
|
-
* no field).
|
|
123
|
+
* no field). `runtimeDefault`, when present, is authoritative for
|
|
124
|
+
* `defaultJson`, keeping runtime binding and discovery metadata in sync.
|
|
125
|
+
* Mirrors Python's `_constraint_kwargs`.
|
|
124
126
|
*/
|
|
125
127
|
export function constraintSpecFields(
|
|
126
128
|
constraints: ArgumentConstraints | undefined,
|
|
129
|
+
runtimeDefault?: unknown,
|
|
127
130
|
): Pick<ArgumentSpec, "defaultJson" | "choicesJson" | "rangeNotation" | "pattern"> {
|
|
128
131
|
const fields: Pick<
|
|
129
132
|
ArgumentSpec,
|
|
130
133
|
"defaultJson" | "choicesJson" | "rangeNotation" | "pattern"
|
|
131
134
|
> = {};
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
+
// argDefaults is the runtime source of truth. Surface that same value to
|
|
136
|
+
// discovery so callers do not have to duplicate it as
|
|
137
|
+
// argConstraints[name].default. Keep the latter as a backwards-compatible
|
|
138
|
+
// discovery-only fallback for APIs that do not have a runtime default.
|
|
139
|
+
if (runtimeDefault !== undefined) {
|
|
140
|
+
fields.defaultJson = safeJson(runtimeDefault);
|
|
141
|
+
} else if (constraints !== undefined && "default" in constraints && constraints.default !== undefined) {
|
|
135
142
|
fields.defaultJson = safeJson(constraints.default);
|
|
136
143
|
}
|
|
144
|
+
if (constraints === undefined) return fields;
|
|
137
145
|
if (constraints.choices !== undefined) {
|
|
138
146
|
fields.choicesJson = safeJson([...constraints.choices]);
|
|
139
147
|
}
|
|
@@ -167,6 +175,87 @@ function choiceEquals(choice: unknown, value: unknown): boolean {
|
|
|
167
175
|
return choice === value;
|
|
168
176
|
}
|
|
169
177
|
|
|
178
|
+
/**
|
|
179
|
+
* Narrow a bind-time argument value from Arrow's bigint to a plain number.
|
|
180
|
+
*
|
|
181
|
+
* DuckDB types an integer literal as BIGINT, so `series(3)` arrives as `3n`
|
|
182
|
+
* whether or not anyone wanted 64 bits. Nobody writes `BigInt(1000)` for a page
|
|
183
|
+
* size, so the arg paths narrow — and they now all narrow the same way, which
|
|
184
|
+
* they did not before (two sites used safeNumber, two a bare Number).
|
|
185
|
+
*
|
|
186
|
+
* This is lossy above 2^53, and that is a deliberate call rather than an
|
|
187
|
+
* oversight. Every int64 *argument* across the fixtures and every shipped
|
|
188
|
+
* worker is a small control value — count, page_size, batch_size, increment,
|
|
189
|
+
* lag_minutes, restatement_lookback_days. The large int64s in the test suite
|
|
190
|
+
* are all somewhere else entirely:
|
|
191
|
+
*
|
|
192
|
+
* - column data (copy_to/types.test, copy_to/parallel.test) — never touches
|
|
193
|
+
* this path;
|
|
194
|
+
* - filter-pushdown constants (filter_pushdown/integers.test pins 2^53+1 on
|
|
195
|
+
* purpose) — deserialize.ts keeps those bigint, with its own regression
|
|
196
|
+
* test;
|
|
197
|
+
* - varargs values (constant_columns) — read raw off bindCall.arguments,
|
|
198
|
+
* because extractArgs' copy of them is not what the function reads;
|
|
199
|
+
* - time-travel versions — carried as strings (`at_value`).
|
|
200
|
+
*
|
|
201
|
+
* Preserving the bigint on overflow was tried and reverted: it buys nothing for
|
|
202
|
+
* any argument that exists, and costs a JS type that depends on magnitude, so a
|
|
203
|
+
* worker doing `args.n + 1` would work in every test and fail on one caller's
|
|
204
|
+
* input. If an argument ever genuinely needs the full range — a snowflake id, a
|
|
205
|
+
* nanosecond epoch — the fix is to make int64 arguments bigint by declared
|
|
206
|
+
* type, not by magnitude.
|
|
207
|
+
*/
|
|
208
|
+
export function narrowArgValue(value: unknown): unknown {
|
|
209
|
+
return typeof value === "bigint" ? Number(value) : value;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
/**
|
|
213
|
+
* Reject a type *factory* where an Arrow *type* was expected.
|
|
214
|
+
*
|
|
215
|
+
* `params: { n: int64 }` (no parentheses) is accepted by the type system —
|
|
216
|
+
* `VgiDataType` is loose enough — and then silently produces an ArgumentSpec
|
|
217
|
+
* whose `arrowType` is a Function. Nothing complains until the schema is
|
|
218
|
+
* serialized or the bind arrives, and the error at that point names neither the
|
|
219
|
+
* argument nor the cause.
|
|
220
|
+
*
|
|
221
|
+
* The mistake is easy to make on purpose, too: the same identifier is a
|
|
222
|
+
* ready-made type on one entry point and a factory on another.
|
|
223
|
+
*
|
|
224
|
+
* @query-farm/vgi int, int32, float32, bool → type instances
|
|
225
|
+
* @query-farm/vgi/worker-cf int, int32, float32, bool → factories
|
|
226
|
+
*
|
|
227
|
+
* So moving a worker to Cloudflare can turn a correct declaration into a
|
|
228
|
+
* silently wrong one with no other edit. Failing here, at definition time,
|
|
229
|
+
* makes that a startup error naming the field.
|
|
230
|
+
*/
|
|
231
|
+
export function assertArrowType(value: unknown, where: string): void {
|
|
232
|
+
if (typeof value === "function") {
|
|
233
|
+
const name = (value as { name?: string }).name;
|
|
234
|
+
const call = name ? `${name}()` : "the factory";
|
|
235
|
+
throw new ArgumentValidationError(
|
|
236
|
+
`${where} is a type factory, not an Arrow type — call it: ${call}. ` +
|
|
237
|
+
`(Note that int, int32, float32 and bool are ready-made types on ` +
|
|
238
|
+
`@query-farm/vgi but factories on @query-farm/vgi/worker-cf, so the ` +
|
|
239
|
+
`same name may need parentheses on one entry point and not the other. ` +
|
|
240
|
+
`int64(), float64(), utf8() and the rest are factories everywhere.)`,
|
|
241
|
+
);
|
|
242
|
+
}
|
|
243
|
+
if (value === null || value === undefined) {
|
|
244
|
+
throw new ArgumentValidationError(`${where} is ${String(value)}, not an Arrow type.`);
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
/** {@link assertArrowType} over a record of declared types, e.g. `params`. */
|
|
249
|
+
export function assertArrowTypes(
|
|
250
|
+
types: Record<string, unknown> | undefined,
|
|
251
|
+
where: string,
|
|
252
|
+
): void {
|
|
253
|
+
if (!types) return;
|
|
254
|
+
for (const [name, type] of Object.entries(types)) {
|
|
255
|
+
assertArrowType(type, `${where}.${name}`);
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
|
|
170
259
|
/**
|
|
171
260
|
* Enforce a bind-time const value against its declared {@link ArgumentConstraints},
|
|
172
261
|
* throwing {@link ArgumentValidationError} on violation. Mirrors the Python SDK's
|
|
@@ -17,7 +17,7 @@ export {
|
|
|
17
17
|
int8, int16, int32, int64,
|
|
18
18
|
uint8, uint16, uint32, uint64,
|
|
19
19
|
float16, float32, float64,
|
|
20
|
-
utf8, binary, fixedSizeBinary,
|
|
20
|
+
utf8, largeUtf8, binary, largeBinary, fixedSizeBinary,
|
|
21
21
|
decimal, decimal128, decimal256,
|
|
22
22
|
date, dateDay, dateMillisecond,
|
|
23
23
|
time, timeSecond, timeMillisecond, timeMicrosecond, timeNanosecond,
|
|
@@ -151,8 +151,29 @@ export function batchToSecretDict(
|
|
|
151
151
|
/**
|
|
152
152
|
* Safe number coercion for BigInt values.
|
|
153
153
|
*/
|
|
154
|
+
const MAX_SAFE = BigInt(Number.MAX_SAFE_INTEGER);
|
|
155
|
+
const MIN_SAFE = -MAX_SAFE;
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* Narrow a bigint to a number, refusing to do it lossily.
|
|
159
|
+
*
|
|
160
|
+
* Arrow int64/uint64 values arrive as bigint. Several bind-time argument paths
|
|
161
|
+
* want a plain number, and used to get one from a bare `Number(value)` — which
|
|
162
|
+
* silently rounds above 2^53, so an id or a nanosecond timestamp passed as a
|
|
163
|
+
* function argument came back subtly wrong with nothing logged anywhere.
|
|
164
|
+
*
|
|
165
|
+
* Throwing is the lesser evil: a caller that genuinely wants a lossy narrowing
|
|
166
|
+
* can still write `Number(v)` themselves, but nobody has to discover the loss
|
|
167
|
+
* from mismatched output weeks later.
|
|
168
|
+
*/
|
|
154
169
|
export function safeNumber(value: any): number {
|
|
155
170
|
if (typeof value === "bigint") {
|
|
171
|
+
if (value > MAX_SAFE || value < MIN_SAFE) {
|
|
172
|
+
throw new RangeError(
|
|
173
|
+
`${value} cannot be represented exactly as a JS number ` +
|
|
174
|
+
`(|value| > Number.MAX_SAFE_INTEGER). Keep it as a bigint.`,
|
|
175
|
+
);
|
|
176
|
+
}
|
|
156
177
|
return Number(value);
|
|
157
178
|
}
|
|
158
179
|
return value as number;
|
|
@@ -20,7 +20,9 @@ import {
|
|
|
20
20
|
Float32 as A_Float32,
|
|
21
21
|
Float64 as A_Float64,
|
|
22
22
|
Utf8 as A_Utf8,
|
|
23
|
+
LargeUtf8 as A_LargeUtf8,
|
|
23
24
|
Binary as A_Binary,
|
|
25
|
+
LargeBinary as A_LargeBinary,
|
|
24
26
|
Decimal as A_Decimal,
|
|
25
27
|
Date_ as A_Date,
|
|
26
28
|
Time as A_Time,
|
|
@@ -134,7 +136,9 @@ export const float32 = (): VgiDataType => new A_Float32() as unknown as VgiDataT
|
|
|
134
136
|
export const float64 = (): VgiDataType => new A_Float64() as unknown as VgiDataType;
|
|
135
137
|
|
|
136
138
|
export const utf8 = (): VgiDataType => new A_Utf8() as unknown as VgiDataType;
|
|
139
|
+
export const largeUtf8 = (): VgiDataType => new A_LargeUtf8() as unknown as VgiDataType;
|
|
137
140
|
export const binary = (): VgiDataType => new A_Binary() as unknown as VgiDataType;
|
|
141
|
+
export const largeBinary = (): VgiDataType => new A_LargeBinary() as unknown as VgiDataType;
|
|
138
142
|
export const fixedSizeBinary = (byteWidth: number): VgiDataType =>
|
|
139
143
|
new A_FixedSizeBinary(byteWidth) as unknown as VgiDataType;
|
|
140
144
|
|
|
@@ -22,7 +22,7 @@ export {
|
|
|
22
22
|
int8, int16, int32, int64,
|
|
23
23
|
uint8, uint16, uint32, uint64,
|
|
24
24
|
float16, float32, float64,
|
|
25
|
-
utf8, binary, fixedSizeBinary,
|
|
25
|
+
utf8, largeUtf8, binary, largeBinary, fixedSizeBinary,
|
|
26
26
|
decimal, decimal128, decimal256,
|
|
27
27
|
date, dateDay, dateMillisecond,
|
|
28
28
|
time, timeSecond, timeMillisecond, timeMicrosecond, timeNanosecond,
|
|
@@ -115,7 +115,30 @@ export function batchToSecretDict(
|
|
|
115
115
|
/**
|
|
116
116
|
* BigInt → number coercion (lossy for values past 2^53).
|
|
117
117
|
*/
|
|
118
|
+
const MAX_SAFE = BigInt(Number.MAX_SAFE_INTEGER);
|
|
119
|
+
const MIN_SAFE = -MAX_SAFE;
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* Narrow a bigint to a number, refusing to do it lossily.
|
|
123
|
+
*
|
|
124
|
+
* Arrow int64/uint64 values arrive as bigint. Several bind-time argument paths
|
|
125
|
+
* want a plain number, and used to get one from a bare `Number(value)` — which
|
|
126
|
+
* silently rounds above 2^53, so an id or a nanosecond timestamp passed as a
|
|
127
|
+
* function argument came back subtly wrong with nothing logged anywhere.
|
|
128
|
+
*
|
|
129
|
+
* Throwing is the lesser evil: a caller that genuinely wants a lossy narrowing
|
|
130
|
+
* can still write `Number(v)` themselves, but nobody has to discover the loss
|
|
131
|
+
* from mismatched output weeks later.
|
|
132
|
+
*/
|
|
118
133
|
export function safeNumber(value: any): number {
|
|
119
|
-
if (typeof value === "bigint")
|
|
134
|
+
if (typeof value === "bigint") {
|
|
135
|
+
if (value > MAX_SAFE || value < MIN_SAFE) {
|
|
136
|
+
throw new RangeError(
|
|
137
|
+
`${value} cannot be represented exactly as a JS number ` +
|
|
138
|
+
`(|value| > Number.MAX_SAFE_INTEGER). Keep it as a bigint.`,
|
|
139
|
+
);
|
|
140
|
+
}
|
|
141
|
+
return Number(value);
|
|
142
|
+
}
|
|
120
143
|
return value as number;
|
|
121
144
|
}
|
|
@@ -23,7 +23,9 @@ import {
|
|
|
23
23
|
float32 as f_float32,
|
|
24
24
|
float64 as f_float64,
|
|
25
25
|
utf8 as f_utf8,
|
|
26
|
+
largeUtf8 as f_largeUtf8,
|
|
26
27
|
binary as f_binary,
|
|
28
|
+
largeBinary as f_largeBinary,
|
|
27
29
|
fixedSizeBinary as f_fixedSizeBinary,
|
|
28
30
|
decimal as f_decimal,
|
|
29
31
|
decimal128 as f_decimal128,
|
|
@@ -111,7 +113,9 @@ export const float32 = (): VgiDataType => f_float32() as unknown as VgiDataType;
|
|
|
111
113
|
export const float64 = (): VgiDataType => f_float64() as unknown as VgiDataType;
|
|
112
114
|
|
|
113
115
|
export const utf8 = (): VgiDataType => f_utf8() as unknown as VgiDataType;
|
|
116
|
+
export const largeUtf8 = (): VgiDataType => f_largeUtf8() as unknown as VgiDataType;
|
|
114
117
|
export const binary = (): VgiDataType => f_binary() as unknown as VgiDataType;
|
|
118
|
+
export const largeBinary = (): VgiDataType => f_largeBinary() as unknown as VgiDataType;
|
|
115
119
|
export const fixedSizeBinary = (stride: number): VgiDataType =>
|
|
116
120
|
f_fixedSizeBinary(stride) as unknown as VgiDataType;
|
|
117
121
|
|
package/src/arrow/index.ts
CHANGED
|
@@ -81,7 +81,7 @@ export {
|
|
|
81
81
|
int, int8, int16, int32, int64,
|
|
82
82
|
uint8, uint16, uint32, uint64,
|
|
83
83
|
float16, float32, float64,
|
|
84
|
-
utf8, binary, fixedSizeBinary,
|
|
84
|
+
utf8, largeUtf8, binary, largeBinary, fixedSizeBinary,
|
|
85
85
|
decimal, decimal128, decimal256,
|
|
86
86
|
date, dateDay, dateMillisecond,
|
|
87
87
|
time, timeSecond, timeMillisecond, timeMicrosecond, timeNanosecond,
|
|
@@ -25,7 +25,9 @@ import type {
|
|
|
25
25
|
IntType,
|
|
26
26
|
FloatType,
|
|
27
27
|
Utf8Type,
|
|
28
|
+
LargeUtf8Type,
|
|
28
29
|
BinaryType,
|
|
30
|
+
LargeBinaryType,
|
|
29
31
|
FixedSizeBinaryType,
|
|
30
32
|
DecimalType,
|
|
31
33
|
Date32Type, Date64Type,
|
|
@@ -83,7 +85,9 @@ export const float32 = (): FloatType => impl.float32() as unknown as FloatType;
|
|
|
83
85
|
export const float64 = (): FloatType => impl.float64() as unknown as FloatType;
|
|
84
86
|
|
|
85
87
|
export const utf8 = (): Utf8Type => impl.utf8() as unknown as Utf8Type;
|
|
88
|
+
export const largeUtf8 = (): LargeUtf8Type => impl.largeUtf8() as unknown as LargeUtf8Type;
|
|
86
89
|
export const binary = (): BinaryType => impl.binary() as unknown as BinaryType;
|
|
90
|
+
export const largeBinary = (): LargeBinaryType => impl.largeBinary() as unknown as LargeBinaryType;
|
|
87
91
|
export const fixedSizeBinary = (byteWidth: number): FixedSizeBinaryType =>
|
|
88
92
|
impl.fixedSizeBinary(byteWidth) as unknown as FixedSizeBinaryType;
|
|
89
93
|
|
package/src/arrow/types.ts
CHANGED
|
@@ -33,9 +33,43 @@ export interface VgiSchema {
|
|
|
33
33
|
readonly metadata: Map<string, string>;
|
|
34
34
|
}
|
|
35
35
|
|
|
36
|
-
/**
|
|
37
|
-
*
|
|
38
|
-
*
|
|
36
|
+
/**
|
|
37
|
+
* Column view over one column of a batch.
|
|
38
|
+
*
|
|
39
|
+
* The value type is erased by default: arrow-js parameterizes on DataType and
|
|
40
|
+
* flechette on value type, so the facade cannot name one without picking a
|
|
41
|
+
* backend. Supply `T` at the use site — the column's declared type makes it
|
|
42
|
+
* known there — and the cast disappears:
|
|
43
|
+
*
|
|
44
|
+
* ```ts
|
|
45
|
+
* const ns = batch.getChildAt(0)! as Iterable<bigint | null>;
|
|
46
|
+
* for (const v of ns) { ... } // v: bigint | null
|
|
47
|
+
* ```
|
|
48
|
+
*
|
|
49
|
+
* A type parameter here would read better, but it cannot be had cheaply: both
|
|
50
|
+
* backends' native batches are assigned to VgiBatch structurally, and a
|
|
51
|
+
* concrete `get(): unknown` does not satisfy a generic `get(): T`. Adding one
|
|
52
|
+
* would mean casting at ~25 internal sites to remove one cast in user code.
|
|
53
|
+
*
|
|
54
|
+
* **These are the backend's own values, not codec output.** `get()` and
|
|
55
|
+
* iteration return whatever the Arrow implementation stores, which for some
|
|
56
|
+
* types is not the value the SDK documents:
|
|
57
|
+
*
|
|
58
|
+
* | Arrow type | here | {@link iterRows} |
|
|
59
|
+
* | ---------------- | ----------------------------- | --------------------- |
|
|
60
|
+
* | `int64` | `bigint` | `bigint` (same) |
|
|
61
|
+
* | `utf8`, `float64`| `string`, `number` | same |
|
|
62
|
+
* | `decimal128` | backend limbs (`DecimalBigNum`)| `bigint`, unscaled |
|
|
63
|
+
* | `timestamp[us]` | millisecond `number` | microsecond `bigint` |
|
|
64
|
+
* | `date32` | millisecond `number` | `Date` |
|
|
65
|
+
*
|
|
66
|
+
* `repr: "raw"` on a function selects the *codec's* representation and has no
|
|
67
|
+
* effect on this path.
|
|
68
|
+
*
|
|
69
|
+
* So: reach for `getChildAt` on integer, float, boolean, string and binary
|
|
70
|
+
* columns, where it is the cheapest correct thing. For temporal, decimal and
|
|
71
|
+
* nested types go through {@link iterRows}, which runs the codec.
|
|
72
|
+
*/
|
|
39
73
|
export interface VgiColumn {
|
|
40
74
|
readonly type: VgiDataType;
|
|
41
75
|
readonly length: number;
|
|
@@ -46,7 +80,9 @@ export interface VgiColumn {
|
|
|
46
80
|
export interface VgiBatch {
|
|
47
81
|
readonly schema: VgiSchema;
|
|
48
82
|
readonly numRows: number;
|
|
83
|
+
/** See {@link VgiColumn} — this does NOT run the codec. */
|
|
49
84
|
getChild(name: string): VgiColumn | null;
|
|
85
|
+
/** See {@link VgiColumn} — this does NOT run the codec. */
|
|
50
86
|
getChildAt(index: number): VgiColumn | null;
|
|
51
87
|
}
|
|
52
88
|
|
package/src/catalog/composite.ts
CHANGED
|
@@ -119,56 +119,56 @@ export class CompositeCatalogInterface extends CatalogInterface {
|
|
|
119
119
|
return await this._route(attachOpaqueData).schemas(attachOpaqueData, transactionOpaqueData);
|
|
120
120
|
}
|
|
121
121
|
|
|
122
|
-
override async schemaGet(attachOpaqueData: AttachOpaqueData,
|
|
123
|
-
return await this._route(attachOpaqueData).schemaGet(attachOpaqueData,
|
|
122
|
+
override async schemaGet(attachOpaqueData: AttachOpaqueData, path: string[], transactionOpaqueData?: TransactionOpaqueData): Promise<SchemaInfo | null> {
|
|
123
|
+
return await this._route(attachOpaqueData).schemaGet(attachOpaqueData, path, transactionOpaqueData);
|
|
124
124
|
}
|
|
125
125
|
|
|
126
|
-
override async schemaContentsTables(attachOpaqueData: AttachOpaqueData,
|
|
127
|
-
return await this._route(attachOpaqueData).schemaContentsTables(attachOpaqueData,
|
|
126
|
+
override async schemaContentsTables(attachOpaqueData: AttachOpaqueData, path: string[], transactionOpaqueData?: TransactionOpaqueData): Promise<TableInfo[]> {
|
|
127
|
+
return await this._route(attachOpaqueData).schemaContentsTables(attachOpaqueData, path, transactionOpaqueData);
|
|
128
128
|
}
|
|
129
129
|
|
|
130
|
-
override async schemaContentsViews(attachOpaqueData: AttachOpaqueData,
|
|
131
|
-
return await this._route(attachOpaqueData).schemaContentsViews(attachOpaqueData,
|
|
130
|
+
override async schemaContentsViews(attachOpaqueData: AttachOpaqueData, path: string[], transactionOpaqueData?: TransactionOpaqueData): Promise<ViewInfo[]> {
|
|
131
|
+
return await this._route(attachOpaqueData).schemaContentsViews(attachOpaqueData, path, transactionOpaqueData);
|
|
132
132
|
}
|
|
133
133
|
|
|
134
|
-
override async schemaContentsFunctions(attachOpaqueData: AttachOpaqueData,
|
|
135
|
-
return await this._route(attachOpaqueData).schemaContentsFunctions(attachOpaqueData,
|
|
134
|
+
override async schemaContentsFunctions(attachOpaqueData: AttachOpaqueData, path: string[], type: string, transactionOpaqueData?: TransactionOpaqueData): Promise<FunctionInfo[]> {
|
|
135
|
+
return await this._route(attachOpaqueData).schemaContentsFunctions(attachOpaqueData, path, type, transactionOpaqueData);
|
|
136
136
|
}
|
|
137
137
|
|
|
138
|
-
override async schemaContentsMacros(attachOpaqueData: AttachOpaqueData,
|
|
139
|
-
return await this._route(attachOpaqueData).schemaContentsMacros(attachOpaqueData,
|
|
138
|
+
override async schemaContentsMacros(attachOpaqueData: AttachOpaqueData, path: string[], type: string, transactionOpaqueData?: TransactionOpaqueData): Promise<MacroInfo[]> {
|
|
139
|
+
return await this._route(attachOpaqueData).schemaContentsMacros(attachOpaqueData, path, type, transactionOpaqueData);
|
|
140
140
|
}
|
|
141
141
|
|
|
142
|
-
override async schemaContentsIndexes(attachOpaqueData: AttachOpaqueData,
|
|
143
|
-
return await this._route(attachOpaqueData).schemaContentsIndexes(attachOpaqueData,
|
|
142
|
+
override async schemaContentsIndexes(attachOpaqueData: AttachOpaqueData, path: string[], transactionOpaqueData?: TransactionOpaqueData): Promise<IndexInfo[]> {
|
|
143
|
+
return await this._route(attachOpaqueData).schemaContentsIndexes(attachOpaqueData, path, transactionOpaqueData);
|
|
144
144
|
}
|
|
145
145
|
|
|
146
|
-
override async indexGet(attachOpaqueData: AttachOpaqueData,
|
|
147
|
-
return await this._route(attachOpaqueData).indexGet(attachOpaqueData,
|
|
146
|
+
override async indexGet(attachOpaqueData: AttachOpaqueData, schemaPath: string[], name: string, transactionOpaqueData?: TransactionOpaqueData): Promise<IndexInfo | null> {
|
|
147
|
+
return await this._route(attachOpaqueData).indexGet(attachOpaqueData, schemaPath, name, transactionOpaqueData);
|
|
148
148
|
}
|
|
149
149
|
|
|
150
|
-
override async tableGet(attachOpaqueData: AttachOpaqueData,
|
|
151
|
-
return await this._route(attachOpaqueData).tableGet(attachOpaqueData,
|
|
150
|
+
override async tableGet(attachOpaqueData: AttachOpaqueData, schemaPath: string[], name: string, atUnit?: string, atValue?: string, transactionOpaqueData?: TransactionOpaqueData): Promise<TableInfo | null> {
|
|
151
|
+
return await this._route(attachOpaqueData).tableGet(attachOpaqueData, schemaPath, name, atUnit, atValue, transactionOpaqueData);
|
|
152
152
|
}
|
|
153
153
|
|
|
154
|
-
override async tableScanFunctionGet(attachOpaqueData: AttachOpaqueData,
|
|
155
|
-
return await this._route(attachOpaqueData).tableScanFunctionGet(attachOpaqueData,
|
|
154
|
+
override async tableScanFunctionGet(attachOpaqueData: AttachOpaqueData, schemaPath: string[], name: string, atUnit?: string, atValue?: string, transactionOpaqueData?: TransactionOpaqueData): Promise<any> {
|
|
155
|
+
return await this._route(attachOpaqueData).tableScanFunctionGet(attachOpaqueData, schemaPath, name, atUnit, atValue, transactionOpaqueData);
|
|
156
156
|
}
|
|
157
157
|
|
|
158
|
-
override async tableScanBranchesGet(attachOpaqueData: AttachOpaqueData,
|
|
159
|
-
return await this._route(attachOpaqueData).tableScanBranchesGet(attachOpaqueData,
|
|
158
|
+
override async tableScanBranchesGet(attachOpaqueData: AttachOpaqueData, schemaPath: string[], name: string, atUnit?: string, atValue?: string, transactionOpaqueData?: TransactionOpaqueData): Promise<any> {
|
|
159
|
+
return await this._route(attachOpaqueData).tableScanBranchesGet(attachOpaqueData, schemaPath, name, atUnit, atValue, transactionOpaqueData);
|
|
160
160
|
}
|
|
161
161
|
|
|
162
|
-
override async tableColumnStatisticsGet(attachOpaqueData: AttachOpaqueData,
|
|
163
|
-
return await this._route(attachOpaqueData).tableColumnStatisticsGet(attachOpaqueData,
|
|
162
|
+
override async tableColumnStatisticsGet(attachOpaqueData: AttachOpaqueData, schemaPath: string[], name: string, transactionOpaqueData?: TransactionOpaqueData): Promise<{ bytes: Uint8Array; cacheMaxAgeSeconds: number | null } | null> {
|
|
163
|
+
return await this._route(attachOpaqueData).tableColumnStatisticsGet(attachOpaqueData, schemaPath, name, transactionOpaqueData);
|
|
164
164
|
}
|
|
165
165
|
|
|
166
|
-
override async viewGet(attachOpaqueData: AttachOpaqueData,
|
|
167
|
-
return await this._route(attachOpaqueData).viewGet(attachOpaqueData,
|
|
166
|
+
override async viewGet(attachOpaqueData: AttachOpaqueData, schemaPath: string[], name: string, transactionOpaqueData?: TransactionOpaqueData): Promise<ViewInfo | null> {
|
|
167
|
+
return await this._route(attachOpaqueData).viewGet(attachOpaqueData, schemaPath, name, transactionOpaqueData);
|
|
168
168
|
}
|
|
169
169
|
|
|
170
|
-
override async macroGet(attachOpaqueData: AttachOpaqueData,
|
|
171
|
-
return await this._route(attachOpaqueData).macroGet(attachOpaqueData,
|
|
170
|
+
override async macroGet(attachOpaqueData: AttachOpaqueData, schemaPath: string[], name: string, transactionOpaqueData?: TransactionOpaqueData): Promise<MacroInfo | null> {
|
|
171
|
+
return await this._route(attachOpaqueData).macroGet(attachOpaqueData, schemaPath, name, transactionOpaqueData);
|
|
172
172
|
}
|
|
173
173
|
|
|
174
174
|
override async copyFromFormats(attachOpaqueData: AttachOpaqueData, transactionOpaqueData?: TransactionOpaqueData): Promise<CopyFromFormatInfo[]> {
|
|
@@ -19,7 +19,7 @@ export interface ForeignKeyDef {
|
|
|
19
19
|
columns: string[];
|
|
20
20
|
referencedTable: string;
|
|
21
21
|
referencedColumns: string[];
|
|
22
|
-
|
|
22
|
+
referencedSchemaPath?: string[];
|
|
23
23
|
}
|
|
24
24
|
|
|
25
25
|
export type DefaultValue = string | number | boolean | null;
|
|
@@ -137,7 +137,10 @@ export interface IndexDescriptor {
|
|
|
137
137
|
}
|
|
138
138
|
|
|
139
139
|
export interface SchemaDescriptor {
|
|
140
|
-
|
|
140
|
+
/** Raw identifier components, from the catalog root to this schema. */
|
|
141
|
+
path?: string[];
|
|
142
|
+
/** @deprecated Protocol 1.x compatibility; prefer `path: [name]`. */
|
|
143
|
+
name?: string;
|
|
141
144
|
tables?: TableDescriptor[];
|
|
142
145
|
views?: ViewDescriptor[];
|
|
143
146
|
macros?: MacroDescriptor[];
|
|
@@ -147,6 +150,12 @@ export interface SchemaDescriptor {
|
|
|
147
150
|
tags?: Record<string, string>;
|
|
148
151
|
}
|
|
149
152
|
|
|
153
|
+
export function schemaDescriptorPath(schema: SchemaDescriptor): string[] {
|
|
154
|
+
if (schema.path) return schema.path;
|
|
155
|
+
if (schema.name) return [schema.name];
|
|
156
|
+
throw new Error("SchemaDescriptor requires a non-empty path");
|
|
157
|
+
}
|
|
158
|
+
|
|
150
159
|
export interface SecretTypeDescriptor {
|
|
151
160
|
name: string;
|
|
152
161
|
description: string;
|
|
@@ -180,7 +189,7 @@ export interface CatalogDescriptor {
|
|
|
180
189
|
* records on `catalog_attach.global_functions`.
|
|
181
190
|
*
|
|
182
191
|
* Every entry must also be declared in exactly one of this catalog's
|
|
183
|
-
* schemas — the `FunctionInfo.
|
|
192
|
+
* schemas — the `FunctionInfo.schema_path` it carries is the bind-dispatch
|
|
184
193
|
* key, not a sentinel. Mirrors vgi-python's `Catalog.global_functions`.
|
|
185
194
|
*/
|
|
186
195
|
globalFunctions?: VgiFunction[];
|