@query-farm/vgi 0.42.0 → 0.44.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -2
- package/dist/attach-ticket.d.ts +18 -7
- package/dist/client-entry.js +215 -91
- package/dist/client-entry.js.map +10 -9
- package/dist/codec/asd.d.ts +3 -1
- package/dist/codec/conform.d.ts +51 -0
- package/dist/errors.d.ts +5 -1
- package/dist/generated/vgi-service.d.ts +1315 -0
- package/dist/index.core.d.ts +1 -1
- package/dist/index.js +2143 -932
- package/dist/index.js.map +29 -28
- package/dist/protocol/dispatch.d.ts +14 -5
- package/dist/protocol/handlers/aggregate.d.ts +3 -2
- package/dist/protocol/handlers/catalog/admin.d.ts +2 -2
- package/dist/protocol/handlers/catalog/index.d.ts +3 -2
- package/dist/protocol/handlers/catalog/index_methods.d.ts +2 -2
- package/dist/protocol/handlers/catalog/macro.d.ts +2 -2
- package/dist/protocol/handlers/catalog/shared.d.ts +7 -14
- package/dist/protocol/handlers/catalog/table.d.ts +2 -2
- package/dist/protocol/handlers/catalog/view.d.ts +2 -2
- package/dist/protocol/handlers/function.d.ts +3 -2
- package/dist/protocol/handlers/shared.d.ts +2 -2
- package/dist/protocol/handlers/table-buffering.d.ts +3 -2
- package/dist/protocol/handlers/unimplemented.d.ts +4 -17
- package/dist/protocol/serializers/shared.d.ts +2 -1
- package/dist/rpc-server.d.ts +2 -2
- package/dist/serve-entry.js +2104 -903
- package/dist/serve-entry.js.map +25 -24
- package/dist/worker-cf-entry.js +2045 -842
- package/dist/worker-cf-entry.js.map +29 -28
- package/package.json +3 -3
- package/src/attach-ticket.ts +41 -13
- package/src/catalog/composite.ts +2 -1
- package/src/catalog/interface.ts +5 -4
- package/src/catalog/read-only.ts +8 -6
- package/src/client/client.ts +13 -5
- package/src/codec/asd.ts +5 -2
- package/src/codec/conform.ts +193 -0
- package/src/errors.ts +27 -7
- package/src/functions/registry.ts +3 -2
- package/src/generated/vgi-service.ts +2678 -0
- package/src/http/fetch.ts +1 -1
- package/src/index.core.ts +3 -1
- package/src/protocol/dispatch.ts +40 -29
- package/src/protocol/handlers/aggregate.ts +205 -232
- package/src/protocol/handlers/catalog/admin.ts +228 -299
- package/src/protocol/handlers/catalog/index.ts +17 -16
- package/src/protocol/handlers/catalog/index_methods.ts +26 -36
- package/src/protocol/handlers/catalog/macro.ts +54 -77
- package/src/protocol/handlers/catalog/shared.ts +30 -47
- package/src/protocol/handlers/catalog/table.ts +205 -270
- package/src/protocol/handlers/catalog/view.ts +59 -85
- package/src/protocol/handlers/function.ts +127 -156
- package/src/protocol/handlers/shared.ts +8 -6
- package/src/protocol/handlers/table-buffering.ts +53 -71
- package/src/protocol/handlers/unimplemented.ts +18 -62
- package/src/protocol/serializers/bind.ts +3 -1
- package/src/protocol/serializers/shared.ts +5 -2
- package/src/rpc-server.ts +7 -7
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@query-farm/vgi",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.44.0",
|
|
4
4
|
"description": "TypeScript framework for extending DuckDB with user-defined functions via the Vector Gateway Interface (VGI) over Arrow IPC.",
|
|
5
5
|
"license": "SEE LICENSE IN LICENSE",
|
|
6
6
|
"author": "Query Farm LLC (https://query.farm)",
|
|
@@ -91,12 +91,12 @@
|
|
|
91
91
|
},
|
|
92
92
|
"peerDependencies": {
|
|
93
93
|
"@query-farm/apache-arrow": "^21.1.1",
|
|
94
|
-
"@query-farm/vgi-rpc": "^0.
|
|
94
|
+
"@query-farm/vgi-rpc": "^0.29.0"
|
|
95
95
|
},
|
|
96
96
|
"devDependencies": {
|
|
97
97
|
"@duckdb/node-api": "^1.5.5-r.4",
|
|
98
98
|
"@query-farm/apache-arrow": "^21.1.1",
|
|
99
|
-
"@query-farm/vgi-rpc": "^0.
|
|
99
|
+
"@query-farm/vgi-rpc": "^0.29.0",
|
|
100
100
|
"@types/bun": "latest",
|
|
101
101
|
"typescript": "^7.0.2"
|
|
102
102
|
},
|
package/src/attach-ticket.ts
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
// that user (`vgi.attach_tickets.v1`).
|
|
6
6
|
//
|
|
7
7
|
// A ticket is the *what* half of an unattended session; a sealed grant
|
|
8
|
-
// (`vgi_rpc.Identity.
|
|
8
|
+
// (`vgi_rpc.Identity.v2` `issue_grant`) is the *who*. While the user is
|
|
9
9
|
// attached and logged in, the client asks the worker to seal the options it
|
|
10
10
|
// attached with -- secret ones included -- into a ticket only this worker can
|
|
11
11
|
// open. Later any runner holding the user's grant attaches with the single
|
|
@@ -35,7 +35,7 @@ import {
|
|
|
35
35
|
errorInfo,
|
|
36
36
|
preconditionFailure,
|
|
37
37
|
} from "@query-farm/vgi-rpc";
|
|
38
|
-
import { field,
|
|
38
|
+
import { field, schema, utf8, binary, int64, timestamp, TimeUnit, batchFromColumns, serializeBatch } from "./arrow/index.js";
|
|
39
39
|
import { deserializeBatch } from "./util/arrow/index.js";
|
|
40
40
|
import { openBytes, OpaqueDataRejectedError, sealBytes } from "./crypto.js";
|
|
41
41
|
import type { CatalogInterface } from "./catalog/interface.js";
|
|
@@ -51,8 +51,9 @@ export const ATTACH_TICKET_PREFIX = "vgia1.";
|
|
|
51
51
|
export const ATTACH_TICKET_OPTION = RESERVED_ATTACH_OPTION;
|
|
52
52
|
/** Wire name of the protocol hosting `seal_attach`. */
|
|
53
53
|
export const ATTACH_TICKETS_PROTOCOL_NAME = "vgi.attach_tickets.v1";
|
|
54
|
-
/** Its declared version.
|
|
55
|
-
|
|
54
|
+
/** Its declared version. 2.0.0 made `AttachTicket.expires_at` a nullable
|
|
55
|
+
* `timestamp[us, UTC]` (1.0.0: float64 unix seconds, `+inf` = no expiry). */
|
|
56
|
+
export const ATTACH_TICKETS_PROTOCOL_VERSION = "2.0.0";
|
|
56
57
|
/** The envelope's version byte, fixed by this format. */
|
|
57
58
|
export const TICKET_ENVELOPE_VERSION = 0x01;
|
|
58
59
|
/** Largest options record (serialized Arrow IPC bytes) a ticket may carry. */
|
|
@@ -455,12 +456,42 @@ export const SEAL_ATTACH_REQUEST_SCHEMA = schema([
|
|
|
455
456
|
field("ttl_seconds", int64(), false),
|
|
456
457
|
]);
|
|
457
458
|
|
|
458
|
-
/**
|
|
459
|
+
/**
|
|
460
|
+
* `AttachTicket`, field for field with vgi-python: `expires_at` is the UTC
|
|
461
|
+
* instant after which the worker refuses the ticket, null when the worker sets
|
|
462
|
+
* no maximum lifetime.
|
|
463
|
+
*/
|
|
459
464
|
export const ATTACH_TICKET_SCHEMA = schema([
|
|
460
465
|
field("ticket", utf8(), false),
|
|
461
|
-
field("expires_at",
|
|
466
|
+
field("expires_at", timestamp(TimeUnit.MICROSECOND, "UTC"), true),
|
|
462
467
|
]);
|
|
463
468
|
|
|
469
|
+
/** A sealed ticket, as `seal_attach` returns it. */
|
|
470
|
+
export interface AttachTicket {
|
|
471
|
+
/** `vgia1.` text. Not a credential, but never logged. */
|
|
472
|
+
ticket: string;
|
|
473
|
+
/** When the worker stops honouring it; `null` = no expiry. */
|
|
474
|
+
expires_at: Date | null;
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
/** `expires_at` as the wire's microsecond count (the codec's raw unit). */
|
|
478
|
+
function expiresAtMicros(value: Date | null): bigint | null {
|
|
479
|
+
if (value === null) return null;
|
|
480
|
+
const ms = value.getTime();
|
|
481
|
+
if (!Number.isFinite(ms)) throw new TypeError("AttachTicket.expires_at must be a valid Date or null");
|
|
482
|
+
return BigInt(ms) * 1000n;
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
/** Encode an {@link AttachTicket} as a single-row Arrow IPC stream. */
|
|
486
|
+
export function encodeAttachTicket(ticket: AttachTicket): Uint8Array {
|
|
487
|
+
return serializeBatch(
|
|
488
|
+
batchFromColumns(
|
|
489
|
+
{ ticket: [ticket.ticket], expires_at: [expiresAtMicros(ticket.expires_at)] },
|
|
490
|
+
ATTACH_TICKET_SCHEMA,
|
|
491
|
+
),
|
|
492
|
+
);
|
|
493
|
+
}
|
|
494
|
+
|
|
464
495
|
/**
|
|
465
496
|
* The ticket lifetime ceiling: the worker's grant maximum, or `null`.
|
|
466
497
|
*
|
|
@@ -517,7 +548,7 @@ export async function sealAttach(
|
|
|
517
548
|
config: AttachTicketsConfig,
|
|
518
549
|
request: Record<string, unknown>,
|
|
519
550
|
auth: AuthContext | undefined,
|
|
520
|
-
): Promise<
|
|
551
|
+
): Promise<AttachTicket> {
|
|
521
552
|
const principal = callerPrincipal(auth);
|
|
522
553
|
if (principal === null) throw new SealAttachDeniedError("an anonymous caller cannot seal an attach ticket");
|
|
523
554
|
|
|
@@ -582,7 +613,8 @@ export async function sealAttach(
|
|
|
582
613
|
} catch (e) {
|
|
583
614
|
throw new AttachTicketRequestError("seal_attach request is invalid", [["request", (e as Error).message]]);
|
|
584
615
|
}
|
|
585
|
-
|
|
616
|
+
// The sealed claim keeps seconds with 0 = none; the reply is an instant or null.
|
|
617
|
+
return { ticket: token, expires_at: expiresAt === 0 ? null : new Date(expiresAt * 1000) };
|
|
586
618
|
}
|
|
587
619
|
|
|
588
620
|
/**
|
|
@@ -600,11 +632,7 @@ export function buildAttachTicketsProtocol(config: AttachTicketsConfig): Protoco
|
|
|
600
632
|
handler: async (params: Record<string, unknown>, ctx: unknown) => {
|
|
601
633
|
const request = unwrapRequest(params.request);
|
|
602
634
|
const sealed = await sealAttach(config, request, (ctx as CallContext | undefined)?.auth);
|
|
603
|
-
|
|
604
|
-
{ ticket: [sealed.ticket], expires_at: [sealed.expires_at] },
|
|
605
|
-
ATTACH_TICKET_SCHEMA,
|
|
606
|
-
);
|
|
607
|
-
return { result: serializeBatch(batch) };
|
|
635
|
+
return { result: encodeAttachTicket(sealed) };
|
|
608
636
|
},
|
|
609
637
|
});
|
|
610
638
|
return p;
|
package/src/catalog/composite.ts
CHANGED
|
@@ -25,6 +25,7 @@ import type {
|
|
|
25
25
|
} from "./interface.js";
|
|
26
26
|
import { CatalogInterface } from "./interface.js";
|
|
27
27
|
import { OpaqueDataRejectedError } from "../crypto.js";
|
|
28
|
+
import { VgiError } from "../errors.js";
|
|
28
29
|
|
|
29
30
|
function bufferEquals(a: AttachOpaqueData, b: AttachOpaqueData): boolean {
|
|
30
31
|
if (a.length !== b.length) return false;
|
|
@@ -109,7 +110,7 @@ export class CompositeCatalogInterface extends CatalogInterface {
|
|
|
109
110
|
return result;
|
|
110
111
|
}
|
|
111
112
|
}
|
|
112
|
-
throw new
|
|
113
|
+
throw new VgiError(`No worker handles catalog '${name}'`, "NOT_FOUND");
|
|
113
114
|
}
|
|
114
115
|
|
|
115
116
|
async detach(attachOpaqueData: AttachOpaqueData): Promise<void> {
|
package/src/catalog/interface.ts
CHANGED
|
@@ -168,6 +168,7 @@ export const encodeAttachCatalogInfo = (v: AttachCatalogInfo): Uint8Array =>
|
|
|
168
168
|
// ============================================================================
|
|
169
169
|
|
|
170
170
|
import { deserializeBatch, serializeBatch, batchFromColumns } from "../util/arrow/index.js";
|
|
171
|
+
import { conformColumns } from "../codec/conform.js";
|
|
171
172
|
import { schema as schema_, field as field_, type VgiDataType, type VgiField } from "../arrow/index.js";
|
|
172
173
|
import { toUint8Array } from "../util/bytes.js";
|
|
173
174
|
import { ScanBranchSchema, ScanFunctionResultSchema } from "../generated/vgi-protocol-schemas.js";
|
|
@@ -221,12 +222,12 @@ export function encodeScanFunctionResult(
|
|
|
221
222
|
): Uint8Array {
|
|
222
223
|
return serializeBatch(
|
|
223
224
|
batchFromColumns(
|
|
224
|
-
{
|
|
225
|
+
conformColumns(ScanFunctionResultSchema as any, {
|
|
225
226
|
function_name: [functionName],
|
|
226
227
|
arguments: [argumentsBytes],
|
|
227
228
|
required_extensions: [requiredExtensions],
|
|
228
229
|
schema_path: [schemaPath],
|
|
229
|
-
},
|
|
230
|
+
}),
|
|
230
231
|
ScanFunctionResultSchema as any,
|
|
231
232
|
),
|
|
232
233
|
);
|
|
@@ -345,7 +346,7 @@ function encodeScanBranch(cols: {
|
|
|
345
346
|
}): Uint8Array {
|
|
346
347
|
return serializeBatch(
|
|
347
348
|
batchFromColumns(
|
|
348
|
-
{
|
|
349
|
+
conformColumns(ScanBranchSchema as any, {
|
|
349
350
|
function_name: [cols.function_name],
|
|
350
351
|
arguments: [cols.arguments],
|
|
351
352
|
branch_filter: [cols.branch_filter ?? null],
|
|
@@ -357,7 +358,7 @@ function encodeScanBranch(cols: {
|
|
|
357
358
|
format_locations: [cols.format_locations ?? null],
|
|
358
359
|
format_options: [cols.format_options ?? null],
|
|
359
360
|
schema_path: [cols.schema_path ?? null],
|
|
360
|
-
},
|
|
361
|
+
}),
|
|
361
362
|
ScanBranchSchema as any,
|
|
362
363
|
),
|
|
363
364
|
);
|
package/src/catalog/read-only.ts
CHANGED
|
@@ -32,6 +32,8 @@ import { FunctionStability, NullHandling, OrderPreservation, DEFAULT_MAX_WORKERS
|
|
|
32
32
|
import { Arguments } from "../arguments/arguments.js";
|
|
33
33
|
import { normalizeSchemaPath, schemaPathDisplay, schemaPathsEqual } from "../schema-path.js";
|
|
34
34
|
import { ForeignKeyInfoSchema } from "../generated/vgi-protocol-schemas.js";
|
|
35
|
+
import { conformColumns } from "../codec/conform.js";
|
|
36
|
+
import { VgiError } from "../errors.js";
|
|
35
37
|
import { encodeFunctionInfoOnce, freezeCatalogItem } from "./item-encoding.js";
|
|
36
38
|
|
|
37
39
|
/**
|
|
@@ -178,7 +180,7 @@ export class ReadOnlyCatalogInterface extends CatalogInterface {
|
|
|
178
180
|
implementationVersion?: string | null,
|
|
179
181
|
): CatalogAttachResult | Promise<CatalogAttachResult> {
|
|
180
182
|
if (!this.catalogs().includes(name)) {
|
|
181
|
-
throw new
|
|
183
|
+
throw new VgiError(`No worker handles catalog '${name}'`, "NOT_FOUND");
|
|
182
184
|
}
|
|
183
185
|
// Refuse an attach that omits a declared-required option, so the caller
|
|
184
186
|
// gets the option name rather than a catalog that looks mysteriously
|
|
@@ -820,11 +822,11 @@ export class ReadOnlyCatalogInterface extends CatalogInterface {
|
|
|
820
822
|
// Find the table descriptor
|
|
821
823
|
const schema = this._descriptor.schemas.find((s) => schemaPathsEqual(schemaDescriptorPath(s), schemaPath));
|
|
822
824
|
if (!schema || !schema.tables) {
|
|
823
|
-
throw new
|
|
825
|
+
throw new VgiError(`Table '${name}' not found in schema ${schemaPathDisplay(schemaPath)}`, "NOT_FOUND");
|
|
824
826
|
}
|
|
825
827
|
const table = schema.tables.find((t) => t.name === name);
|
|
826
828
|
if (!table) {
|
|
827
|
-
throw new
|
|
829
|
+
throw new VgiError(`Table '${name}' not found in schema ${schemaPathDisplay(schemaPath)}`, "NOT_FOUND");
|
|
828
830
|
}
|
|
829
831
|
|
|
830
832
|
// Reject AT clause on tables that don't support time travel
|
|
@@ -958,12 +960,12 @@ function serializeForeignKeys(fks?: ForeignKeyDef[], schemaPath?: string[]): Uin
|
|
|
958
960
|
return fks.map((fk) =>
|
|
959
961
|
serializeBatch(
|
|
960
962
|
batchFromColumns(
|
|
961
|
-
{
|
|
963
|
+
conformColumns(ForeignKeyInfoSchema, {
|
|
962
964
|
fk_columns: [fk.columns],
|
|
963
965
|
pk_columns: [fk.referencedColumns],
|
|
964
966
|
referenced_table: [fk.referencedTable],
|
|
965
967
|
referenced_schema_path: [fk.referencedSchemaPath ?? schemaPath ?? ["main"]],
|
|
966
|
-
},
|
|
968
|
+
}),
|
|
967
969
|
ForeignKeyInfoSchema
|
|
968
970
|
)
|
|
969
971
|
)
|
|
@@ -1145,7 +1147,7 @@ function serializeArgsBatch(args: Arguments, argSpecSchema: VgiSchema): Uint8Arr
|
|
|
1145
1147
|
|
|
1146
1148
|
function validateAtParams(atUnit?: string, atValue?: string): void {
|
|
1147
1149
|
if (Boolean(atUnit) !== Boolean(atValue)) {
|
|
1148
|
-
throw new
|
|
1150
|
+
throw new VgiError("at_unit and at_value must both be provided or both be absent", "INVALID_ARGUMENT");
|
|
1149
1151
|
}
|
|
1150
1152
|
}
|
|
1151
1153
|
|
package/src/client/client.ts
CHANGED
|
@@ -77,6 +77,14 @@ import type {
|
|
|
77
77
|
LoadCatalogOptions,
|
|
78
78
|
} from "./types.js";
|
|
79
79
|
|
|
80
|
+
/**
|
|
81
|
+
* `on_conflict` as it goes on the wire: the vgi-python `OnConflict` member
|
|
82
|
+
* NAME (`ERROR` / `IGNORE` / `REPLACE`), not the lower-case API spelling.
|
|
83
|
+
*/
|
|
84
|
+
function onConflictName(onConflict: OnCreateConflict): string {
|
|
85
|
+
return String(onConflict).toUpperCase();
|
|
86
|
+
}
|
|
87
|
+
|
|
80
88
|
/**
|
|
81
89
|
* High-level client for calling VGI worker functions and catalog API.
|
|
82
90
|
*
|
|
@@ -906,7 +914,7 @@ export class VgiClient {
|
|
|
906
914
|
}
|
|
907
915
|
await this.rpc.call("catalog_create", {
|
|
908
916
|
name,
|
|
909
|
-
on_conflict: onConflict,
|
|
917
|
+
on_conflict: onConflictName(onConflict),
|
|
910
918
|
options: wireOptions,
|
|
911
919
|
});
|
|
912
920
|
}
|
|
@@ -1013,7 +1021,7 @@ export class VgiClient {
|
|
|
1013
1021
|
await this.rpc.call("catalog_schema_create", {
|
|
1014
1022
|
attach_opaque_data: attachOpaqueData,
|
|
1015
1023
|
path: normalizeSchemaPath(path),
|
|
1016
|
-
on_conflict: opts?.onConflict ?? "error",
|
|
1024
|
+
on_conflict: onConflictName(opts?.onConflict ?? "error"),
|
|
1017
1025
|
comment: opts?.comment ?? null,
|
|
1018
1026
|
tags: tagsMap,
|
|
1019
1027
|
transaction_opaque_data: opts?.transactionOpaqueData ?? null,
|
|
@@ -1139,7 +1147,7 @@ export class VgiClient {
|
|
|
1139
1147
|
schema_path: normalizeSchemaPath(schemaPath),
|
|
1140
1148
|
name,
|
|
1141
1149
|
columns,
|
|
1142
|
-
on_conflict: onConflict,
|
|
1150
|
+
on_conflict: onConflictName(onConflict),
|
|
1143
1151
|
not_null_constraints: notNullConstraints ?? null,
|
|
1144
1152
|
unique_constraints: uniqueConstraints ?? null,
|
|
1145
1153
|
check_constraints: checkConstraints ?? null,
|
|
@@ -1430,7 +1438,7 @@ export class VgiClient {
|
|
|
1430
1438
|
schema_path: normalizeSchemaPath(schemaPath),
|
|
1431
1439
|
name,
|
|
1432
1440
|
definition,
|
|
1433
|
-
on_conflict: onConflict,
|
|
1441
|
+
on_conflict: onConflictName(onConflict),
|
|
1434
1442
|
transaction_opaque_data: transactionOpaqueData ?? null,
|
|
1435
1443
|
});
|
|
1436
1444
|
}
|
|
@@ -1554,7 +1562,7 @@ export class VgiClient {
|
|
|
1554
1562
|
macro_type: [macroType],
|
|
1555
1563
|
parameters: [parameters],
|
|
1556
1564
|
definition: [definition],
|
|
1557
|
-
on_conflict: [onConflict],
|
|
1565
|
+
on_conflict: [onConflictName(onConflict)],
|
|
1558
1566
|
parameter_default_values: [parameterDefaultValues ?? null],
|
|
1559
1567
|
arguments_schema: [argumentsSchema ?? null],
|
|
1560
1568
|
transaction_opaque_data: [transactionOpaqueData ?? null],
|
package/src/codec/asd.ts
CHANGED
|
@@ -31,16 +31,19 @@ import {
|
|
|
31
31
|
serializeBatch,
|
|
32
32
|
deserializeBatch,
|
|
33
33
|
} from "../arrow/index.js";
|
|
34
|
+
import { conformRecord } from "./conform.js";
|
|
34
35
|
|
|
35
36
|
/**
|
|
36
37
|
* Encode a typed object as a single-row Arrow IPC stream using `schema`.
|
|
37
|
-
* Matches Python's `ArrowSerializableDataclass.serialize_to_bytes()
|
|
38
|
+
* Matches Python's `ArrowSerializableDataclass.serialize_to_bytes()`, including
|
|
39
|
+
* its container rules: a non-nullable list/map left out or null is written as
|
|
40
|
+
* empty, and list items / map values are never null (see `conformRecord`).
|
|
38
41
|
*/
|
|
39
42
|
export function encodeASD(
|
|
40
43
|
schema: VgiSchema,
|
|
41
44
|
obj: Record<string, any>,
|
|
42
45
|
): Uint8Array {
|
|
43
|
-
return serializeBatch(batchFromRows([sortMapsForSchema(schema, obj)], schema));
|
|
46
|
+
return serializeBatch(batchFromRows([sortMapsForSchema(schema, conformRecord(schema, obj))], schema));
|
|
44
47
|
}
|
|
45
48
|
|
|
46
49
|
// --------------------------------------------------------------------------- //
|
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
// Copyright 2025, 2026 Query Farm LLC - https://query.farm
|
|
2
|
+
// SPDX-License-Identifier: LicenseRef-QueryFarm-Source-Available-1.0
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Wire conformance for protocol records.
|
|
6
|
+
*
|
|
7
|
+
* vgi-python is the reference for the wire shape, and two of its rules are not
|
|
8
|
+
* expressed by a record's Arrow schema nullability flags alone:
|
|
9
|
+
*
|
|
10
|
+
* 1. A **non-nullable list or map column is never null.** Python's dataclass
|
|
11
|
+
* defaults a `list[...]` / `dict[...]` field to an empty container, so an
|
|
12
|
+
* "absent" list goes out as `[]`. A JS caller that leaves the field out (or
|
|
13
|
+
* passes `null`) would otherwise put a null in a column the schema calls
|
|
14
|
+
* non-null — `CatalogInfo.releases`, `CatalogInfo.attach_option_specs`,
|
|
15
|
+
* ... — which a strict client rejects as a protocol error.
|
|
16
|
+
*
|
|
17
|
+
* 2. **List items and map values are never null** unless the Python annotation
|
|
18
|
+
* says so (`list[str | None]`). pyarrow marks every list item field nullable,
|
|
19
|
+
* so the generated schemas carry `item: nullable=true` everywhere and cannot
|
|
20
|
+
* tell the two apart. In the whole protocol exactly one field has optional
|
|
21
|
+
* items — `argument_names` on `BindRequest` / `AggregateBindRequest` (a
|
|
22
|
+
* positional argument has no name). Every other null item is a bug on the
|
|
23
|
+
* sending side (e.g. `BindResponse.lookup_names` built from an unset secret
|
|
24
|
+
* name, where Python sends `""`).
|
|
25
|
+
*
|
|
26
|
+
* `conformRecord` applies both rules to a record about to be encoded: a null or
|
|
27
|
+
* absent non-nullable list/map becomes empty, and a null list item / map value
|
|
28
|
+
* becomes its type's zero value (`""`, `0`, `false`, empty bytes, empty list),
|
|
29
|
+
* which is also what the C++ client reads such an item as. It never mutates its
|
|
30
|
+
* input (catalog items may be frozen and shared) and returns the input itself
|
|
31
|
+
* when nothing needed changing. Nullable columns, and scalars, pass through
|
|
32
|
+
* untouched: a null in a non-nullable scalar column has no safe default and is
|
|
33
|
+
* left for the caller to fix (see `wireViolations`).
|
|
34
|
+
*/
|
|
35
|
+
|
|
36
|
+
import {
|
|
37
|
+
type VgiSchema,
|
|
38
|
+
type VgiField,
|
|
39
|
+
type VgiDataType,
|
|
40
|
+
isBinary,
|
|
41
|
+
isBool,
|
|
42
|
+
isDictionary,
|
|
43
|
+
isFloat,
|
|
44
|
+
isInt,
|
|
45
|
+
isList,
|
|
46
|
+
isMap,
|
|
47
|
+
isStruct,
|
|
48
|
+
isUtf8,
|
|
49
|
+
} from "../arrow/index.js";
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Top-level / struct-child field names whose list items may legitimately be
|
|
53
|
+
* null (the Python annotation is `list[X | None]`). Derived from vgi-python by
|
|
54
|
+
* walking every protocol dataclass's type hints; keep in sync if the protocol
|
|
55
|
+
* adds another optional-item list.
|
|
56
|
+
*/
|
|
57
|
+
export const OPTIONAL_ITEM_FIELDS: ReadonlySet<string> = new Set(["argument_names"]);
|
|
58
|
+
|
|
59
|
+
function children(type: VgiDataType): VgiField[] {
|
|
60
|
+
return ((type as any).children ?? []) as VgiField[];
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function mapValueType(type: VgiDataType): VgiDataType {
|
|
64
|
+
const entries = children(type)[0].type;
|
|
65
|
+
return children(entries)[1].type;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function mapPairs(raw: any): Array<[unknown, unknown]> {
|
|
69
|
+
if (Array.isArray(raw)) return raw.map((e: any) => (Array.isArray(e) ? [e[0], e[1]] : [e?.key, e?.value]));
|
|
70
|
+
if (raw instanceof Map) return Array.from(raw.entries());
|
|
71
|
+
if (typeof raw[Symbol.iterator] === "function") return Array.from(raw as Iterable<[unknown, unknown]>);
|
|
72
|
+
return Object.entries(raw);
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/** The value a non-null slot of `type` takes when the sender left it null. */
|
|
76
|
+
export function zeroValue(type: VgiDataType): unknown {
|
|
77
|
+
if (isUtf8(type) || isDictionary(type)) return "";
|
|
78
|
+
if (isBinary(type)) return new Uint8Array(0);
|
|
79
|
+
if (isBool(type)) return false;
|
|
80
|
+
if (isInt(type)) return (type as any).bitWidth === 64 ? 0n : 0;
|
|
81
|
+
if (isFloat(type)) return 0;
|
|
82
|
+
if (isList(type) || isMap(type)) return [];
|
|
83
|
+
if (isStruct(type)) {
|
|
84
|
+
const out: Record<string, unknown> = {};
|
|
85
|
+
for (const c of children(type)) out[c.name] = c.nullable ? null : conformValue(c.type, undefined, false, c.name);
|
|
86
|
+
return out;
|
|
87
|
+
}
|
|
88
|
+
return null;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/** Whether `type` contains a list or map anywhere (memoized per type object). */
|
|
92
|
+
const HAS_CONTAINER = new WeakMap<object, boolean>();
|
|
93
|
+
function hasContainer(type: VgiDataType): boolean {
|
|
94
|
+
const cached = HAS_CONTAINER.get(type as object);
|
|
95
|
+
if (cached !== undefined) return cached;
|
|
96
|
+
let has = isList(type) || isMap(type);
|
|
97
|
+
if (!has && isStruct(type)) has = children(type).some((c) => hasContainer(c.type));
|
|
98
|
+
HAS_CONTAINER.set(type as object, has);
|
|
99
|
+
return has;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* Conform one value of `type`. `nullable` is the slot's nullability; `name` is
|
|
104
|
+
* the field name that owns the slot (list items inherit their list's name, for
|
|
105
|
+
* the `OPTIONAL_ITEM_FIELDS` lookup).
|
|
106
|
+
*/
|
|
107
|
+
function conformValue(type: VgiDataType, value: any, nullable: boolean, name: string): any {
|
|
108
|
+
if (value == null) {
|
|
109
|
+
if (nullable) return value ?? null;
|
|
110
|
+
if (isList(type) || isMap(type)) return [];
|
|
111
|
+
return value;
|
|
112
|
+
}
|
|
113
|
+
if (!hasContainer(type)) return value;
|
|
114
|
+
|
|
115
|
+
if (isList(type)) {
|
|
116
|
+
const itemType = children(type)[0].type;
|
|
117
|
+
const itemsNullable = OPTIONAL_ITEM_FIELDS.has(name);
|
|
118
|
+
const items = Array.isArray(value) ? value : Array.from(value as Iterable<unknown>);
|
|
119
|
+
let changed = !Array.isArray(value);
|
|
120
|
+
const out = items.map((item) => {
|
|
121
|
+
const next = item == null && !itemsNullable ? zeroValue(itemType) : conformValue(itemType, item, true, name);
|
|
122
|
+
if (next !== item) changed = true;
|
|
123
|
+
return next;
|
|
124
|
+
});
|
|
125
|
+
return changed ? out : value;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
if (isMap(type)) {
|
|
129
|
+
const valueType = mapValueType(type);
|
|
130
|
+
const pairs = mapPairs(value);
|
|
131
|
+
let changed = false;
|
|
132
|
+
const out = pairs.map(([k, v]): [unknown, unknown] => {
|
|
133
|
+
const next = v == null ? zeroValue(valueType) : conformValue(valueType, v, true, name);
|
|
134
|
+
if (next !== v) changed = true;
|
|
135
|
+
return [k, next];
|
|
136
|
+
});
|
|
137
|
+
return changed ? out : value;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
if (isStruct(type)) {
|
|
141
|
+
let out: Record<string, any> | null = null;
|
|
142
|
+
for (const c of children(type)) {
|
|
143
|
+
const v = value[c.name];
|
|
144
|
+
const next = conformValue(c.type, v, c.nullable, c.name);
|
|
145
|
+
if (next !== v) {
|
|
146
|
+
(out ??= { ...value })[c.name] = next;
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
return out ?? value;
|
|
150
|
+
}
|
|
151
|
+
return value;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* `obj` conformed to `schema`'s wire rules (see the module comment). Returns
|
|
156
|
+
* `obj` itself when nothing changed.
|
|
157
|
+
*/
|
|
158
|
+
export function conformRecord<T extends Record<string, any>>(schema: VgiSchema, obj: T): T {
|
|
159
|
+
let out: Record<string, any> | null = null;
|
|
160
|
+
for (const f of schema.fields) {
|
|
161
|
+
const v = obj[f.name];
|
|
162
|
+
const next = conformValue(f.type, v, f.nullable, f.name);
|
|
163
|
+
if (next !== v) {
|
|
164
|
+
(out ??= { ...obj })[f.name] = next;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
return (out ?? obj) as T;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* Column-dict form of `conformRecord` for `batchFromColumns` callers: each
|
|
172
|
+
* value is a one-element array holding the row's value.
|
|
173
|
+
*/
|
|
174
|
+
export function conformColumns(
|
|
175
|
+
schema: VgiSchema,
|
|
176
|
+
columns: Record<string, unknown[]>,
|
|
177
|
+
): Record<string, unknown[]> {
|
|
178
|
+
let out: Record<string, unknown[]> | null = null;
|
|
179
|
+
for (const f of schema.fields) {
|
|
180
|
+
const col = columns[f.name];
|
|
181
|
+
const rows = col ?? [undefined];
|
|
182
|
+
let changed = col === undefined;
|
|
183
|
+
const next = rows.map((v) => {
|
|
184
|
+
const c = conformValue(f.type, v, f.nullable, f.name);
|
|
185
|
+
if (c !== v) changed = true;
|
|
186
|
+
return c;
|
|
187
|
+
});
|
|
188
|
+
if (changed) {
|
|
189
|
+
(out ??= { ...columns })[f.name] = next;
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
return out ?? columns;
|
|
193
|
+
}
|
package/src/errors.ts
CHANGED
|
@@ -1,13 +1,33 @@
|
|
|
1
1
|
// Copyright 2025, 2026 Query Farm LLC - https://query.farm
|
|
2
2
|
// VGI-specific error classes.
|
|
3
|
+
//
|
|
4
|
+
// Each class declares the canonical gRPC-style code vgi-rpc hoists onto the
|
|
5
|
+
// EXCEPTION batch as `vgi_rpc.error_code` (vgi-rpc WIRE_PROTOCOL.md §8), so a
|
|
6
|
+
// client — and DuckDB's `errors_as_json` — can tell "your input was wrong"
|
|
7
|
+
// from a worker bug. The mapping is shared with every other VGI SDK:
|
|
8
|
+
//
|
|
9
|
+
// INVALID_ARGUMENT argument validation / type rejection
|
|
10
|
+
// NOT_FOUND unknown function, table, schema, catalog
|
|
11
|
+
// ALREADY_EXISTS creating something that is already there
|
|
12
|
+
// FAILED_PRECONDITION writes against a read-only catalog
|
|
13
|
+
// UNIMPLEMENTED an operation the SDK does not support
|
|
14
|
+
//
|
|
15
|
+
// Anything else (RowCountMismatchError, internal invariants) stays UNKNOWN.
|
|
16
|
+
|
|
17
|
+
import type { ErrorCode } from "@query-farm/vgi-rpc";
|
|
3
18
|
|
|
4
19
|
export class VgiError extends Error {
|
|
5
|
-
|
|
20
|
+
/** Canonical code hoisted as `vgi_rpc.error_code`; unset means `UNKNOWN`. */
|
|
21
|
+
readonly errorCode?: ErrorCode;
|
|
22
|
+
|
|
23
|
+
constructor(message: string, code?: ErrorCode) {
|
|
6
24
|
super(message);
|
|
7
25
|
this.name = "VgiError";
|
|
26
|
+
if (code !== undefined) this.errorCode = code;
|
|
8
27
|
}
|
|
9
28
|
}
|
|
10
29
|
|
|
30
|
+
/** A worker bug (the function returned the wrong number of rows): UNKNOWN. */
|
|
11
31
|
export class RowCountMismatchError extends VgiError {
|
|
12
32
|
constructor(expected: number, actual: number) {
|
|
13
33
|
super(
|
|
@@ -20,42 +40,42 @@ export class RowCountMismatchError extends VgiError {
|
|
|
20
40
|
|
|
21
41
|
export class FunctionNotFoundError extends VgiError {
|
|
22
42
|
constructor(name: string, available?: string[]) {
|
|
23
|
-
super(`Unknown function '${name}'
|
|
43
|
+
super(`Unknown function '${name}'`, "NOT_FOUND");
|
|
24
44
|
this.name = "FunctionNotFoundError";
|
|
25
45
|
}
|
|
26
46
|
}
|
|
27
47
|
|
|
28
48
|
export class CatalogReadOnlyError extends VgiError {
|
|
29
49
|
constructor(operation: string) {
|
|
30
|
-
super(`catalog is read-only: ${operation} is not supported
|
|
50
|
+
super(`catalog is read-only: ${operation} is not supported`, "FAILED_PRECONDITION");
|
|
31
51
|
this.name = "CatalogReadOnlyError";
|
|
32
52
|
}
|
|
33
53
|
}
|
|
34
54
|
|
|
35
55
|
export class CatalogNotFoundError extends VgiError {
|
|
36
56
|
constructor(entity: string, name: string) {
|
|
37
|
-
super(`${entity} '${name}' not found
|
|
57
|
+
super(`${entity} '${name}' not found`, "NOT_FOUND");
|
|
38
58
|
this.name = "CatalogNotFoundError";
|
|
39
59
|
}
|
|
40
60
|
}
|
|
41
61
|
|
|
42
62
|
export class CatalogAlreadyExistsError extends VgiError {
|
|
43
63
|
constructor(entity: string, name: string) {
|
|
44
|
-
super(`${entity} '${name}' already exists
|
|
64
|
+
super(`${entity} '${name}' already exists`, "ALREADY_EXISTS");
|
|
45
65
|
this.name = "CatalogAlreadyExistsError";
|
|
46
66
|
}
|
|
47
67
|
}
|
|
48
68
|
|
|
49
69
|
export class ArgumentValidationError extends VgiError {
|
|
50
70
|
constructor(message: string) {
|
|
51
|
-
super(message);
|
|
71
|
+
super(message, "INVALID_ARGUMENT");
|
|
52
72
|
this.name = "ArgumentValidationError";
|
|
53
73
|
}
|
|
54
74
|
}
|
|
55
75
|
|
|
56
76
|
export class NoCatalogError extends VgiError {
|
|
57
77
|
constructor() {
|
|
58
|
-
super("No catalog is configured for this worker");
|
|
78
|
+
super("No catalog is configured for this worker", "UNIMPLEMENTED");
|
|
59
79
|
this.name = "NoCatalogError";
|
|
60
80
|
}
|
|
61
81
|
}
|
|
@@ -10,7 +10,7 @@ import type { VgiSchema } from "../arrow/index.js";
|
|
|
10
10
|
import type { VgiFunction } from "./types.js";
|
|
11
11
|
import type { Arguments } from "../arguments/arguments.js";
|
|
12
12
|
import type { ArgumentSpec } from "../arguments/argument-spec.js";
|
|
13
|
-
import { FunctionNotFoundError } from "../errors.js";
|
|
13
|
+
import { FunctionNotFoundError, VgiError } from "../errors.js";
|
|
14
14
|
import { schemaPathDisplay, schemaPathKey } from "../schema-path.js";
|
|
15
15
|
|
|
16
16
|
export interface OverloadContext {
|
|
@@ -289,9 +289,10 @@ export class FunctionRegistry {
|
|
|
289
289
|
} else if (candidates && candidates.length > 0) {
|
|
290
290
|
const schemas = this.schemasFor(name);
|
|
291
291
|
if (schemas.length > 0) {
|
|
292
|
-
throw new
|
|
292
|
+
throw new VgiError(
|
|
293
293
|
`Function '${name}' is not registered in schema ${schemaPathDisplay(schemaPath)}. ` +
|
|
294
294
|
`It is available in: [${schemas.map(schemaPathDisplay).join(", ")}]`,
|
|
295
|
+
"NOT_FOUND",
|
|
295
296
|
);
|
|
296
297
|
}
|
|
297
298
|
}
|