@query-farm/vgi 0.42.0 → 0.44.0

package/src/protocol/handlers/table-buffering.ts

@@ -10,24 +10,16 @@
 // vgi-python's worker._load_table_buffering_params.
 
 import { deserializeBatch } from "../../arrow/index.js";
-import { Protocol } from "@query-farm/vgi-rpc";
+import type { VgiService } from "../../generated/vgi-service.js";
 import type { FunctionRegistry } from "../../functions/registry.js";
 import { toUint8Array } from "../../util/bytes.js";
 import { batchToScalarDict, batchToSecretDict } from "../../util/arrow/index.js";
 import {
-  TableBufferingCombineParamsSchema,
   TableBufferingCombineResultSchema,
-  TableBufferingDestructorParamsSchema,
   TableBufferingDestructorResultSchema,
-  TableBufferingProcessParamsSchema,
   TableBufferingProcessResultSchema,
 } from "../../generated/vgi-protocol-schemas.js";
-import {
-  REQUEST_PARAMS_SCHEMA,
-  RESULT_BINARY_SCHEMA,
-  unwrapRequest,
-  wrapResult,
-} from "./shared.js";
+import { unwrapRequest, wrapResult } from "./shared.js";
 import { openAttach } from "./catalog/shared.js";
 import {
   BoundStorage,
@@ -41,11 +33,11 @@ import {
 } from "../../functions/table-buffering.js";
 import { Arguments } from "../../arguments/arguments.js";
 
-export function registerTableBufferingMethods(
-  protocol: Protocol,
+/** The table_buffering lifecycle's `vgi.v2` handlers. */
+export function tableBufferingHandlers(
   registry: FunctionRegistry,
   signingKey?: Uint8Array,
-): void {
+): Partial<VgiService> {
   // `schemaPath` scopes resolution to the schema the caller named. The
   // buffering process/combine RPCs carry it (protocol 1.2.0), so a name
   // declared in two schemas resolves to the one the request names rather than
@@ -123,72 +115,62 @@ export function registerTableBufferingMethods(
   // ------------------------------------------------------------------------
   // table_buffering_process — sink one batch, return state_id (unary)
   // ------------------------------------------------------------------------
-  protocol.unary("table_buffering_process", {
-    params: TableBufferingProcessParamsSchema,
-    result: RESULT_BINARY_SCHEMA,
-    handler: async (rpcParams, ctx?: any) => {
-      const inner = unwrapRequest(rpcParams.request);
-      const clientLog = makeClientLog(ctx);
-      const { func, params } = await loadParams(inner, clientLog, ctx);
-      if (inner.batch_index != null) {
-        params.batchIndex = Number(inner.batch_index);
-      }
-      const batch = deserializeBatch(toUint8Array(inner.input_batch));
-      const stateId = await func.bufferingConfig.process(batch, params);
-      if (!(stateId instanceof Uint8Array)) {
-        throw new Error(
-          `${func.meta.name}.process() must return Uint8Array (the opaque state_id)`,
-        );
-      }
-      return wrapResult({ state_id: stateId }, TableBufferingProcessResultSchema);
-    },
-  });
+  const tableBufferingProcess: VgiService["tableBufferingProcess"] = async (rpcParams, ctx?: any) => {
+    const inner = unwrapRequest(rpcParams.request);
+    const clientLog = makeClientLog(ctx);
+    const { func, params } = await loadParams(inner, clientLog, ctx);
+    if (inner.batch_index != null) {
+      params.batchIndex = Number(inner.batch_index);
+    }
+    const batch = deserializeBatch(toUint8Array(inner.input_batch));
+    const stateId = await func.bufferingConfig.process(batch, params);
+    if (!(stateId instanceof Uint8Array)) {
+      throw new Error(
+        `${func.meta.name}.process() must return Uint8Array (the opaque state_id)`,
+      );
+    }
+    return wrapResult({ state_id: stateId }, TableBufferingProcessResultSchema);
+  };
 
   // ------------------------------------------------------------------------
   // table_buffering_combine — group/merge state_ids (unary)
   // ------------------------------------------------------------------------
-  protocol.unary("table_buffering_combine", {
-    params: TableBufferingCombineParamsSchema,
-    result: RESULT_BINARY_SCHEMA,
-    handler: async (rpcParams, ctx?: any) => {
-      const inner = unwrapRequest(rpcParams.request);
-      const clientLog = makeClientLog(ctx);
-      const { func, params } = await loadParams(inner, clientLog, ctx);
-      const stateIds = decodeBytesList(inner.state_ids);
-      const finalizeStateIds = await func.bufferingConfig.combine(stateIds, params);
-      const out = finalizeStateIds.map((fid, i) => {
-        if (!(fid instanceof Uint8Array)) {
-          throw new Error(
-            `${func.meta.name}.combine() returned non-Uint8Array finalize_state_id at index ${i}`,
-          );
-        }
-        return fid;
-      });
-      return wrapResult(
-        { finalize_state_ids: out },
-        TableBufferingCombineResultSchema,
-      );
-    },
-  });
+  const tableBufferingCombine: VgiService["tableBufferingCombine"] = async (rpcParams, ctx?: any) => {
+    const inner = unwrapRequest(rpcParams.request);
+    const clientLog = makeClientLog(ctx);
+    const { func, params } = await loadParams(inner, clientLog, ctx);
+    const stateIds = decodeBytesList(inner.state_ids);
+    const finalizeStateIds = await func.bufferingConfig.combine(stateIds, params);
+    const out = finalizeStateIds.map((fid, i) => {
+      if (!(fid instanceof Uint8Array)) {
+        throw new Error(
+          `${func.meta.name}.combine() returned non-Uint8Array finalize_state_id at index ${i}`,
+        );
+      }
+      return fid;
+    });
+    return wrapResult(
+      { finalize_state_ids: out },
+      TableBufferingCombineResultSchema,
+    );
+  };
 
   // ------------------------------------------------------------------------
   // table_buffering_destructor — best-effort end-of-query cleanup (unary)
   // ------------------------------------------------------------------------
-  protocol.unary("table_buffering_destructor", {
-    params: TableBufferingDestructorParamsSchema,
-    result: RESULT_BINARY_SCHEMA,
-    handler: async (rpcParams) => {
-      try {
-        const inner = unwrapRequest(rpcParams.request);
-        const executionId = toUint8Array(inner.execution_id);
-        const bound = new BoundStorage(defaultStorage, executionId);
-        await bound.executionClear();
-      } catch {
-        // Teardown path — swallow; the entry is wiped when the worker exits.
-      }
-      return wrapResult({}, TableBufferingDestructorResultSchema);
-    },
-  });
+  const tableBufferingDestructor: VgiService["tableBufferingDestructor"] = async (rpcParams) => {
+    try {
+      const inner = unwrapRequest(rpcParams.request);
+      const executionId = toUint8Array(inner.execution_id);
+      const bound = new BoundStorage(defaultStorage, executionId);
+      await bound.executionClear();
+    } catch {
+      // Teardown path — swallow; the entry is wiped when the worker exits.
+    }
+    return wrapResult({}, TableBufferingDestructorResultSchema);
+  };
+
+  return { tableBufferingProcess, tableBufferingCombine, tableBufferingDestructor };
 }
 
 function makeClientLog(ctx: any): (level: string, message: string) => void {

package/src/protocol/handlers/unimplemented.ts

@@ -4,71 +4,27 @@
 // "The protocol is the unit of optionality": every SDK hosts EVERY vgi.v2
 // method with exactly the reference's schemas, so vgi_rpc.Reflection.v1
 // reports one vgi.v2 protocol hash everywhere (see VGI_V2_PROTOCOL_HASH in
-// ../dispatch.ts). A method this port has no implementation for is still
-// registered -- with the reference's params/result schemas -- and answers
-// every call with vgi-rpc's MethodNotImplementedError: error_code
-// UNIMPLEMENTED, error_kind method_not_implemented. Never a silent success.
+// ../dispatch.ts). The registration and the UNIMPLEMENTED default for every
+// method are generated (src/generated/vgi-service.ts); this module only names
+// the methods whose generated default this worker leaves in place, so a test
+// can assert each one refuses with UNIMPLEMENTED.
 //
-// When one of these gains a real implementation, move it out of this list
-// into its handler module; the schemas must not change (the hash test fails
-// if they do).
+// To implement one, add its handler to a group in this directory; it drops off
+// this list by itself.
 
-import { MethodNotImplementedError, type Protocol } from "@query-farm/vgi-rpc";
-import type { VgiSchema } from "../../arrow/index.js";
+import { FunctionRegistry } from "../../functions/registry.js";
 import {
-  CatalogIndexCreateParamsSchema,
-  CatalogIndexDropParamsSchema,
-  CatalogTableColumnCommentSetParamsSchema,
-  CatalogTableDeleteFunctionGetParamsSchema,
-  CatalogTableInsertFunctionGetParamsSchema,
-  CatalogTableUpdateFunctionGetParamsSchema,
-} from "../../generated/vgi-protocol-schemas.js";
-import { REQUEST_PARAMS_SCHEMA, RESULT_BINARY_SCHEMA } from "./shared.js";
-import { emptyResultSchema } from "./catalog/shared.js";
+  UNIMPLEMENTED_VGI_SERVICE,
+  VGI_V2_METHODS,
+  type VgiMethodSpec,
+} from "../../generated/vgi-service.js";
+import { buildVgiService } from "../dispatch.js";
 
-/** One unimplemented method: its wire name and the reference's schemas. */
-interface UnimplementedMethod {
-  readonly name: string;
-  readonly params: VgiSchema;
-  readonly result: VgiSchema;
-}
+export { unimplementedMessage } from "../../generated/vgi-service.js";
 
-/**
- * The vgi.v2 methods (of the reference's 72) this SDK does not implement.
- * Exported so a test can assert each one refuses with UNIMPLEMENTED.
- */
-export const UNIMPLEMENTED_METHODS: readonly UnimplementedMethod[] = [
-  // Streaming and window aggregates: request/result are wrapped ASD blobs.
-  { name: "aggregate_streaming_open", params: REQUEST_PARAMS_SCHEMA, result: RESULT_BINARY_SCHEMA },
-  { name: "aggregate_streaming_chunk", params: REQUEST_PARAMS_SCHEMA, result: RESULT_BINARY_SCHEMA },
-  { name: "aggregate_streaming_close", params: REQUEST_PARAMS_SCHEMA, result: RESULT_BINARY_SCHEMA },
-  { name: "aggregate_window_init", params: REQUEST_PARAMS_SCHEMA, result: RESULT_BINARY_SCHEMA },
-  { name: "aggregate_window", params: REQUEST_PARAMS_SCHEMA, result: RESULT_BINARY_SCHEMA },
-  { name: "aggregate_window_batch", params: REQUEST_PARAMS_SCHEMA, result: RESULT_BINARY_SCHEMA },
-  { name: "aggregate_window_destructor", params: REQUEST_PARAMS_SCHEMA, result: RESULT_BINARY_SCHEMA },
-  // Index DDL and per-column comments.
-  { name: "catalog_index_create", params: CatalogIndexCreateParamsSchema, result: emptyResultSchema },
-  { name: "catalog_index_drop", params: CatalogIndexDropParamsSchema, result: emptyResultSchema },
-  { name: "catalog_table_column_comment_set", params: CatalogTableColumnCommentSetParamsSchema, result: emptyResultSchema },
-  // Per-table DML function getters.
-  { name: "catalog_table_insert_function_get", params: CatalogTableInsertFunctionGetParamsSchema, result: RESULT_BINARY_SCHEMA },
-  { name: "catalog_table_update_function_get", params: CatalogTableUpdateFunctionGetParamsSchema, result: RESULT_BINARY_SCHEMA },
-  { name: "catalog_table_delete_function_get", params: CatalogTableDeleteFunctionGetParamsSchema, result: RESULT_BINARY_SCHEMA },
-];
+const service = buildVgiService({ registry: new FunctionRegistry() });
 
-/** The message every unimplemented method answers with. */
-export function unimplementedMessage(method: string): string {
-  return `${method} is not implemented by this worker`;
-}
-
-export function registerUnimplementedMethods(protocol: Protocol): void {
-  for (const m of UNIMPLEMENTED_METHODS) {
-    protocol.unary(m.name, {
-      params: m.params,
-      result: m.result,
-      handler: async () => {
-        throw new MethodNotImplementedError(unimplementedMessage(m.name));
-      },
-    });
-  }
-}
+/** The vgi.v2 methods (of the reference's 72) this SDK leaves at the generated default. */
+export const UNIMPLEMENTED_METHODS: readonly VgiMethodSpec[] = VGI_V2_METHODS.filter(
+  (m) => service[m.key] === UNIMPLEMENTED_VGI_SERVICE[m.key],
+);

package/src/protocol/serializers/bind.ts

@@ -72,7 +72,9 @@ export function serializeBindRequest(req: BindRequest): VgiBatch {
   return buildSingleRowBatch(BIND_REQUEST_SCHEMA, {
     function_name: req.function_name,
     arguments: serializeArguments(req.arguments),
-    function_type: req.function_type,
+    // The wire carries the Python enum member NAME ("TABLE"), not the TS
+    // enum's value ("table"); each FunctionType value upper-cases to its name.
+    function_type: req.function_type == null ? req.function_type : String(req.function_type).toUpperCase(),
     input_schema: req.input_schema ? serializeSchema(req.input_schema) : null,
     settings: req.settings ? serializeBatch(req.settings) : null,
     secrets: req.secrets ? serializeBatch(req.secrets) : null,

package/src/protocol/serializers/shared.ts

@@ -8,6 +8,7 @@ import {
   batchFromColumns,
 } from "../../arrow/index.js";
 import { toUint8Array as toUint8ArrayBase } from "../../util/bytes.js";
+import { conformRecord } from "../../codec/conform.js";
 
 /**
  * Coerce a wire value (string / Uint8Array / Buffer / etc.) to a Uint8Array.
@@ -25,7 +26,8 @@ export function toUint8Array(val: any): Uint8Array {
 
 /**
  * Build a single-row RecordBatch from a {fieldName: value} dict matching the
- * given schema. Auto-coerces JS numbers to BigInt for Int64 fields.
+ * given schema. Auto-coerces JS numbers to BigInt for Int64 fields, and
+ * applies the protocol's container rules (`conformRecord`).
  */
 export function buildSingleRowBatch(
   schema: VgiSchema,
@@ -35,8 +37,9 @@ export function buildSingleRowBatch(
   // declares one — vectorFromArray/columnFromArray rejects plain Number for
   // 64-bit ints under both backends.
   const cols: Record<string, any[]> = {};
+  const row = conformRecord(schema, values);
   for (const f of schema.fields) {
-    let val = values[f.name];
+    let val = row[f.name];
     if (isInt(f.type) && (f.type as any).bitWidth === 64 && typeof val === "number") {
       val = BigInt(val);
     }

package/src/rpc-server.ts

@@ -13,7 +13,7 @@
 //   2. The worker's `hostedProtocols()`, in the order returned -- on EVERY
 //      transport.
 //   3. `vgi_rpc.Reflection.v1` -- on every transport.
-//   4. `vgi_rpc.Identity.v1` -- HTTP only, and only when the worker supplies
+//   4. `vgi_rpc.Identity.v2` -- HTTP only, and only when the worker supplies
 //      `resolveToken` and/or `mintGrant` (or grant keys are configured).
 //   5. `vgi.attach_tickets.v1` -- HTTP only, and only when the signing key was
 //      configured explicitly (never a generated one) *and* the worker can
@@ -44,7 +44,7 @@ export type ServerTransport = "pipe" | "unix" | "tcp" | "iroh" | "stream" | "htt
 const TRANSPORTS: ReadonlySet<string> = new Set<ServerTransport>(["pipe", "unix", "tcp", "iroh", "stream", "http"]);
 
 /** Transports that authenticate their callers, and so may host
- *  `vgi_rpc.Identity.v1`. Identity answers questions about *other* callers'
+ *  `vgi_rpc.Identity.v2`. Identity answers questions about *other* callers'
  *  credentials; its allowlist is a list of principals, which a transport with
  *  no caller identity cannot check. */
 const IDENTITY_TRANSPORTS: ReadonlySet<string> = new Set<ServerTransport>(["http"]);
@@ -78,14 +78,14 @@ export interface HostingOptions {
    *
    * Each needs a distinct wire name, which may not be `vgi.v2` and may not
    * use the reserved `vgi_rpc.` prefix: reflection is hosted automatically and
-   * `vgi_rpc.Identity.v1` is enabled through {@link resolveToken} /
+   * `vgi_rpc.Identity.v2` is enabled through {@link resolveToken} /
    * {@link mintGrant}. Violations are a startup error naming this hook.
    */
   hostedProtocols?: () => Iterable<Protocol>;
 
   /**
    * Resolve an opaque bearer credential to the identity it authenticates as.
-   * Supplying it hosts `vgi_rpc.Identity.v1`'s `introspect_token` -- on HTTP,
+   * Supplying it hosts `vgi_rpc.Identity.v2`'s `introspect_token` -- on HTTP,
    * the transport that authenticates callers. Absent, the method is absent
    * (not hosted-and-refusing), which keeps a dependency upgrade from growing a
    * credential-to-identity oracle on every existing worker.
@@ -232,7 +232,7 @@ function validatedHostedProtocols(hosting: HostingOptions, primary: Protocol): P
       throw new Error(
         `${owner} entry ${index} is named '${name}', which claims the reserved '${RESERVED_PREFIX}' prefix. ` +
           "Framework protocols are not supplied through this hook: reflection is hosted automatically, and " +
-          "vgi_rpc.Identity.v1 is enabled by supplying resolveToken and/or mintGrant.",
+          "vgi_rpc.Identity.v2 is enabled by supplying resolveToken and/or mintGrant.",
       );
     }
     if (!/^[A-Za-z_][A-Za-z0-9_.]*$/.test(name)) {
@@ -259,7 +259,7 @@ function validatedHostedProtocols(hosting: HostingOptions, primary: Protocol): P
   return out;
 }
 
-/** Build the `vgi_rpc.Identity.v1` implementation, or `undefined`.
+/** Build the `vgi_rpc.Identity.v2` implementation, or `undefined`.
  *
  *  `undefined` unless the worker supplies at least one hook -- and that is the
  *  point: the server then does not host the protocol at all, rather than
@@ -329,7 +329,7 @@ function resolveIntrospectPrincipals(
   }
   if (principals.length === 0) {
     throw new Error(
-      "This worker supplies resolveToken, which hosts the vgi_rpc.Identity.v1 protocol, but no " +
+      "This worker supplies resolveToken, which hosts the vgi_rpc.Identity.v2 protocol, but no " +
         `introspector allowlist was configured. Set ${INTROSPECT_PRINCIPALS_ENV} (comma-separated) or pass ` +
         "introspectPrincipals.\n\n" +
         "There is no permissive default on purpose: introspection is a separate capability from " +