@query-farm/vgi 0.6.0

package/src/protocol/handlers/catalog/admin.ts

@@ -0,0 +1,368 @@
+// Copyright 2025, 2026 Query Farm LLC - https://query.farm
+// Catalog admin handlers: attach/detach/create/drop, version, transactions,
+// schemas (list, get, create, drop), schema contents listings.
+
+import { type VgiSchema, schema, type VgiField, field, type VgiDataType, binary, utf8, bool } from "../../../arrow/index.js";
+import { Protocol, type CallContext } from "@query-farm/vgi-rpc";
+import { encodeSchemaInfo, encodeTableInfo, encodeViewInfo, encodeFunctionInfo, encodeCatalogInfo } from "../../../generated/vgi-client.js";
+import { encodeCopyFromFormatInfo } from "../../../catalog/interface.js";
+import {
+  CatalogCatalogsResultSchema,
+  CatalogAttachResultSchema,
+  CatalogVersionResultSchema,
+  CatalogTransactionBeginResultSchema,
+  CatalogSchemasResultSchema,
+  CatalogSchemaGetResultSchema,
+  CatalogSchemaContentsTablesResultSchema,
+  CatalogSchemaContentsViewsResultSchema,
+  CatalogSchemaContentsFunctionsResultSchema,
+  CatalogCopyFromFormatsResultSchema,
+} from "../../../generated/vgi-protocol-schemas.js";
+import { toUint8Array } from "../../../util/bytes.js";
+import { decodeDictValue } from "../../../util/arrow/index.js";
+import {
+  REQUEST_PARAMS_SCHEMA,
+  RESULT_BINARY_SCHEMA,
+  unwrapRequest,
+  wrapResult,
+} from "../shared.js";
+import {
+  type GetCatalog,
+  decodeOptionsBatch,
+  emptyResultSchema,
+  attachOpaqueDataParam,
+  attachOpaqueDataTxnParams,
+  attachOpaqueDataNameTxnParams,
+  catalogUnary,
+  sealAttach,
+  openAttach,
+  sealTransaction,
+} from "./shared.js";
+
+export function registerCatalogAdminMethods(protocol: Protocol, getCatalog: GetCatalog, signingKey?: Uint8Array): void {
+  // catalog_catalogs
+  catalogUnary(protocol, signingKey, "catalog_catalogs", {
+    params: emptyResultSchema,
+    result: RESULT_BINARY_SCHEMA,
+    handler: async () => {
+      const cat = getCatalog();
+      // Each catalog advertised as an IPC-serialized CatalogInfo
+      // {name, implementation_version?, data_version_spec?}. Versioned workers
+      // override catalogsInfo() to supply real values; otherwise both version
+      // fields default to null.
+      const infos = cat.catalogsInfo
+        ? await cat.catalogsInfo()
+        : cat.catalogs().map((name) => ({
+            name,
+            implementation_version: null,
+            data_version_spec: null,
+            attach_option_specs: [],
+            releases: [],
+          }));
+      const items = infos.map((info) => encodeCatalogInfo(info));
+      return wrapResult({ items }, CatalogCatalogsResultSchema);
+    },
+  });
+
+  // catalog_attach (params wrapped in request: Binary like bind/init)
+  protocol.unary("catalog_attach", {
+    params: REQUEST_PARAMS_SCHEMA,
+    result: RESULT_BINARY_SCHEMA,
+    handler: async (params, ctx) => {
+      const innerParams = unwrapRequest(params.request);
+      const cat = getCatalog();
+      // The extension sends user-supplied ATTACH options as an IPC-serialized
+      // RecordBatch of typed columns — one column per option. Deserialize
+      // once here so workers see an ergonomic {name: value} dict instead of
+      // raw bytes. Nullable / absent → {}.
+      const optionsDict = decodeOptionsBatch(innerParams.options);
+      const result = await cat.attach(
+        innerParams.name,
+        optionsDict,
+        innerParams.data_version_spec ?? null,
+        innerParams.implementation_version ?? null,
+      );
+      // Seal the attach value into an AEAD envelope bound to the caller's
+      // identity before it leaves the worker (HTTP transport; pass-through
+      // when there is no signing key).
+      const sealedAttach = await sealAttach(
+        toUint8Array(result.attach_opaque_data),
+        (ctx as CallContext | undefined)?.auth,
+        signingKey,
+      );
+      return wrapResult({
+        attach_opaque_data: sealedAttach,
+        supports_transactions: result.supports_transactions,
+        supports_time_travel: result.supports_time_travel,
+        catalog_version_frozen: result.catalog_version_frozen,
+        catalog_version: result.catalog_version,
+        attach_opaque_data_required: result.attach_opaque_data_required ?? true,
+        default_schema: result.default_schema ?? "main",
+        settings: result.settings ?? [],
+        secret_types: result.secret_types ?? [],
+        comment: result.comment ?? null,
+        tags: result.tags ?? {},
+        // True so DuckDB will route catalog_table_column_statistics_get RPCs
+        // to our handler for tables whose TableInfo.supports_column_statistics
+        // is also true. Catalogs that never serve column stats can override
+        // this in attach() via CatalogAttachResult.supports_column_statistics.
+        supports_column_statistics: result.supports_column_statistics ?? true,
+        resolved_data_version: result.resolved_data_version ?? null,
+        resolved_implementation_version: result.resolved_implementation_version ?? null,
+      }, CatalogAttachResultSchema);
+    },
+  });
+
+  // catalog_detach
+  catalogUnary(protocol, signingKey, "catalog_detach", {
+    params: attachOpaqueDataParam,
+    result: emptyResultSchema,
+    handler: async (params) => {
+      const cat = getCatalog();
+      await cat.detach(toUint8Array(params.attach_opaque_data));
+      return {};
+    },
+  });
+
+  // catalog_create
+  catalogUnary(protocol, signingKey, "catalog_create", {
+    params: schema([
+      field("name", utf8(), false),
+      field("on_conflict", utf8(), false),
+      field("options", binary(), true),
+    ]),
+    result: emptyResultSchema,
+    handler: async (params) => {
+      const cat = getCatalog();
+      await cat.create(params.name, params.on_conflict, params.options);
+      return {};
+    },
+  });
+
+  // catalog_drop
+  catalogUnary(protocol, signingKey, "catalog_drop", {
+    params: schema([field("name", utf8(), false)]),
+    result: emptyResultSchema,
+    handler: async (params) => {
+      const cat = getCatalog();
+      await cat.drop(params.name);
+      return {};
+    },
+  });
+
+  // catalog_version
+  catalogUnary(protocol, signingKey, "catalog_version", {
+    params: attachOpaqueDataTxnParams,
+    result: RESULT_BINARY_SCHEMA,
+    handler: async (params) => {
+      const cat = getCatalog();
+      const version = await cat.version(
+        toUint8Array(params.attach_opaque_data),
+        params.transaction_opaque_data ? toUint8Array(params.transaction_opaque_data) : undefined
+      );
+      return wrapResult({ version }, CatalogVersionResultSchema);
+    },
+  });
+
+  // catalog_transaction_begin — bespoke: it must keep the *sealed* attach
+  // envelope to bind the transaction envelope's AAD, so it cannot route
+  // through catalogUnary (which would replace it with plaintext).
+  protocol.unary("catalog_transaction_begin", {
+    params: attachOpaqueDataParam,
+    result: RESULT_BINARY_SCHEMA,
+    handler: async (params, ctx) => {
+      const auth = (ctx as CallContext | undefined)?.auth;
+      const sealedAttach =
+        params.attach_opaque_data != null ? toUint8Array(params.attach_opaque_data) : new Uint8Array(0);
+      const attachPlain = await openAttach(sealedAttach, auth, signingKey);
+      const cat = getCatalog();
+      const txPlain = await cat.transactionBegin(attachPlain);
+      // Seal the transaction value, binding it to the caller's identity and
+      // the parent attach envelope it was minted under. null → null.
+      const transaction_opaque_data =
+        txPlain != null ? await sealTransaction(toUint8Array(txPlain), sealedAttach, auth, signingKey) : null;
+      return wrapResult({ transaction_opaque_data }, CatalogTransactionBeginResultSchema);
+    },
+  });
+
+  // catalog_transaction_commit
+  catalogUnary(protocol, signingKey, "catalog_transaction_commit", {
+    params: attachOpaqueDataTxnParams,
+    result: emptyResultSchema,
+    handler: async (params) => {
+      const cat = getCatalog();
+      await cat.transactionCommit(
+        toUint8Array(params.attach_opaque_data),
+        toUint8Array(params.transaction_opaque_data)
+      );
+      return {};
+    },
+  });
+
+  // catalog_transaction_rollback
+  catalogUnary(protocol, signingKey, "catalog_transaction_rollback", {
+    params: attachOpaqueDataTxnParams,
+    result: emptyResultSchema,
+    handler: async (params) => {
+      const cat = getCatalog();
+      await cat.transactionRollback(
+        toUint8Array(params.attach_opaque_data),
+        toUint8Array(params.transaction_opaque_data)
+      );
+      return {};
+    },
+  });
+
+  // catalog_schemas
+  catalogUnary(protocol, signingKey, "catalog_schemas", {
+    params: attachOpaqueDataTxnParams,
+    result: RESULT_BINARY_SCHEMA,
+    handler: async (params) => {
+      const cat = getCatalog();
+      const schemas = await cat.schemas(
+        toUint8Array(params.attach_opaque_data),
+        params.transaction_opaque_data ? toUint8Array(params.transaction_opaque_data) : undefined
+      );
+      return wrapResult({
+        items: schemas.map((s) => encodeSchemaInfo(s)),
+      }, CatalogSchemasResultSchema);
+    },
+  });
+
+  // catalog_copy_from_formats — catalog-level (not schema-scoped). Lists the
+  // custom COPY ... FROM formats this catalog advertises; empty list when none.
+  catalogUnary(protocol, signingKey, "catalog_copy_from_formats", {
+    params: attachOpaqueDataTxnParams,
+    result: RESULT_BINARY_SCHEMA,
+    handler: async (params) => {
+      const cat = getCatalog();
+      const formats = await cat.copyFromFormats(
+        toUint8Array(params.attach_opaque_data),
+        params.transaction_opaque_data ? toUint8Array(params.transaction_opaque_data) : undefined,
+      );
+      return wrapResult({
+        items: formats.map((f) => encodeCopyFromFormatInfo(f)),
+      }, CatalogCopyFromFormatsResultSchema);
+    },
+  });
+
+  // catalog_schema_get
+  catalogUnary(protocol, signingKey, "catalog_schema_get", {
+    params: attachOpaqueDataNameTxnParams,
+    result: RESULT_BINARY_SCHEMA,
+    handler: async (params) => {
+      const cat = getCatalog();
+      const info = await cat.schemaGet(
+        toUint8Array(params.attach_opaque_data),
+        params.name,
+        params.transaction_opaque_data ? toUint8Array(params.transaction_opaque_data) : undefined
+      );
+      return wrapResult({
+        items: info ? [encodeSchemaInfo(info)] : [],
+      }, CatalogSchemaGetResultSchema);
+    },
+  });
+
+  // catalog_schema_create
+  catalogUnary(protocol, signingKey, "catalog_schema_create", {
+    params: schema([
+      field("attach_opaque_data", binary(), true),
+      field("name", utf8(), false),
+      field("comment", utf8(), true),
+      field("tags", binary(), true),
+      field("transaction_opaque_data", binary(), true),
+    ]),
+    result: emptyResultSchema,
+    handler: async (params) => {
+      const cat = getCatalog();
+      await cat.schemaCreate(
+        toUint8Array(params.attach_opaque_data),
+        params.name,
+        params.comment,
+        null, // tags
+        params.transaction_opaque_data ? toUint8Array(params.transaction_opaque_data) : undefined
+      );
+      return {};
+    },
+  });
+
+  // catalog_schema_drop
+  catalogUnary(protocol, signingKey, "catalog_schema_drop", {
+    params: schema([
+      field("attach_opaque_data", binary(), true),
+      field("name", utf8(), false),
+      field("ignore_not_found", bool(), true),
+      field("cascade", bool(), true),
+      field("transaction_opaque_data", binary(), true),
+    ]),
+    result: emptyResultSchema,
+    handler: async (params) => {
+      const cat = getCatalog();
+      await cat.schemaDrop(
+        toUint8Array(params.attach_opaque_data),
+        params.name,
+        params.ignore_not_found,
+        params.cascade,
+        params.transaction_opaque_data ? toUint8Array(params.transaction_opaque_data) : undefined
+      );
+      return {};
+    },
+  });
+
+  // catalog_schema_contents_tables
+  catalogUnary(protocol, signingKey, "catalog_schema_contents_tables", {
+    params: attachOpaqueDataNameTxnParams,
+    result: RESULT_BINARY_SCHEMA,
+    handler: async (params) => {
+      const cat = getCatalog();
+      const tables = await cat.schemaContentsTables(
+        toUint8Array(params.attach_opaque_data),
+        params.name,
+        params.transaction_opaque_data ? toUint8Array(params.transaction_opaque_data) : undefined
+      );
+      return wrapResult({
+        items: tables.map((t) => encodeTableInfo(t)),
+      }, CatalogSchemaContentsTablesResultSchema);
+    },
+  });
+
+  // catalog_schema_contents_views
+  catalogUnary(protocol, signingKey, "catalog_schema_contents_views", {
+    params: attachOpaqueDataNameTxnParams,
+    result: RESULT_BINARY_SCHEMA,
+    handler: async (params) => {
+      const cat = getCatalog();
+      const views = await cat.schemaContentsViews(
+        toUint8Array(params.attach_opaque_data),
+        params.name,
+        params.transaction_opaque_data ? toUint8Array(params.transaction_opaque_data) : undefined
+      );
+      return wrapResult({
+        items: views.map((v) => encodeViewInfo(v)),
+      }, CatalogSchemaContentsViewsResultSchema);
+    },
+  });
+
+  // catalog_schema_contents_functions
+  catalogUnary(protocol, signingKey, "catalog_schema_contents_functions", {
+    params: schema([
+      field("attach_opaque_data", binary(), true),
+      field("name", utf8(), false),
+      field("type", utf8(), false),
+      field("transaction_opaque_data", binary(), true),
+    ]),
+    result: RESULT_BINARY_SCHEMA,
+    handler: async (params) => {
+      const cat = getCatalog();
+      const funcs = await cat.schemaContentsFunctions(
+        toUint8Array(params.attach_opaque_data),
+        params.name,
+        decodeDictValue(params.type),
+        params.transaction_opaque_data ? toUint8Array(params.transaction_opaque_data) : undefined
+      );
+      return wrapResult({
+        items: funcs.map((f) => encodeFunctionInfo(f)),
+      }, CatalogSchemaContentsFunctionsResultSchema);
+    },
+  });
+}

package/src/protocol/handlers/catalog/index.ts

@@ -0,0 +1,25 @@
+// Copyright 2025, 2026 Query Farm LLC - https://query.farm
+// Catalog handler orchestrator: registers all catalog_* RPC methods.
+
+import { Protocol } from "@query-farm/vgi-rpc";
+import type { CatalogInterface } from "../../../catalog/interface.js";
+import { makeGetCatalog } from "./shared.js";
+import { registerCatalogAdminMethods } from "./admin.js";
+import { registerCatalogTableMethods } from "./table.js";
+import { registerCatalogViewMethods } from "./view.js";
+import { registerCatalogMacroMethods } from "./macro.js";
+import { registerCatalogIndexMethods } from "./index_methods.js";
+
+export function registerCatalogMethods(
+  protocol: Protocol,
+  catalog: CatalogInterface | undefined,
+  _catalogName: string | undefined,
+  signingKey?: Uint8Array,
+): void {
+  const getCatalog = makeGetCatalog(catalog);
+  registerCatalogAdminMethods(protocol, getCatalog, signingKey);
+  registerCatalogTableMethods(protocol, getCatalog, signingKey);
+  registerCatalogViewMethods(protocol, getCatalog, signingKey);
+  registerCatalogMacroMethods(protocol, getCatalog, signingKey);
+  registerCatalogIndexMethods(protocol, getCatalog, signingKey);
+}

package/src/protocol/handlers/catalog/index_methods.ts

@@ -0,0 +1,65 @@
+// Copyright 2025, 2026 Query Farm LLC - https://query.farm
+// Catalog index handlers: index_get, schema_contents_indexes.
+
+import { type VgiSchema, schema, type VgiField, field, type VgiDataType, binary, utf8 } from "../../../arrow/index.js";
+import { Protocol } from "@query-farm/vgi-rpc";
+import { encodeIndexInfo } from "../../../generated/vgi-client.js";
+import {
+  CatalogIndexGetResultSchema,
+  CatalogSchemaContentsIndexesResultSchema,
+} from "../../../generated/vgi-protocol-schemas.js";
+import { toUint8Array } from "../../../util/bytes.js";
+import {
+  RESULT_BINARY_SCHEMA,
+  wrapResult,
+} from "../shared.js";
+import {
+  type GetCatalog,
+  catalogUnary,
+} from "./shared.js";
+
+export function registerCatalogIndexMethods(protocol: Protocol, getCatalog: GetCatalog, signingKey?: Uint8Array): void {
+  // catalog_schema_contents_indexes
+  catalogUnary(protocol, signingKey, "catalog_schema_contents_indexes", {
+    params: schema([
+      field("attach_opaque_data", binary(), true),
+      field("name", utf8(), false),
+      field("transaction_opaque_data", binary(), true),
+    ]),
+    result: RESULT_BINARY_SCHEMA,
+    handler: async (params) => {
+      const cat = getCatalog();
+      const indexes = await cat.schemaContentsIndexes(
+        toUint8Array(params.attach_opaque_data),
+        params.name,
+        params.transaction_opaque_data ? toUint8Array(params.transaction_opaque_data) : undefined,
+      );
+      return wrapResult({
+        items: indexes.map((i) => encodeIndexInfo(i)),
+      }, CatalogSchemaContentsIndexesResultSchema);
+    },
+  });
+
+  // catalog_index_get
+  catalogUnary(protocol, signingKey, "catalog_index_get", {
+    params: schema([
+      field("attach_opaque_data", binary(), true),
+      field("schema_name", utf8(), false),
+      field("name", utf8(), false),
+      field("transaction_opaque_data", binary(), true),
+    ]),
+    result: RESULT_BINARY_SCHEMA,
+    handler: async (params) => {
+      const cat = getCatalog();
+      const info = await cat.indexGet(
+        toUint8Array(params.attach_opaque_data),
+        params.schema_name,
+        params.name,
+        params.transaction_opaque_data ? toUint8Array(params.transaction_opaque_data) : undefined,
+      );
+      return wrapResult({
+        items: info ? [encodeIndexInfo(info)] : [],
+      }, CatalogIndexGetResultSchema);
+    },
+  });
+}

package/src/protocol/handlers/catalog/macro.ts

@@ -0,0 +1,114 @@
+// Copyright 2025, 2026 Query Farm LLC - https://query.farm
+// Catalog macro handlers: macro_get/create/drop, schema_contents_macros.
+
+import { type VgiSchema, schema, type VgiField, field, type VgiDataType, binary, utf8, bool } from "../../../arrow/index.js";
+import { Protocol } from "@query-farm/vgi-rpc";
+import { encodeMacroInfo } from "../../../generated/vgi-client.js";
+import {
+  CatalogMacroGetResultSchema,
+  CatalogSchemaContentsMacrosResultSchema,
+} from "../../../generated/vgi-protocol-schemas.js";
+import type { MacroType } from "../../../catalog/interface.js";
+import { toUint8Array } from "../../../util/bytes.js";
+import { decodeDictValue } from "../../../util/arrow/index.js";
+import {
+  REQUEST_PARAMS_SCHEMA,
+  RESULT_BINARY_SCHEMA,
+  unwrapRequest,
+  wrapResult,
+} from "../shared.js";
+import {
+  type GetCatalog,
+  emptyResultSchema,
+  attachOpaqueDataSchemaNameTxnParams,
+  catalogUnary,
+} from "./shared.js";
+
+export function registerCatalogMacroMethods(protocol: Protocol, getCatalog: GetCatalog, signingKey?: Uint8Array): void {
+  // catalog_macro_get
+  catalogUnary(protocol, signingKey, "catalog_macro_get", {
+    params: attachOpaqueDataSchemaNameTxnParams,
+    result: RESULT_BINARY_SCHEMA,
+    handler: async (params) => {
+      const cat = getCatalog();
+      const info = await cat.macroGet(
+        toUint8Array(params.attach_opaque_data),
+        params.schema_name,
+        params.name,
+        params.transaction_opaque_data ? toUint8Array(params.transaction_opaque_data) : undefined
+      );
+      return wrapResult({
+        items: info ? [encodeMacroInfo(info)] : [],
+      }, CatalogMacroGetResultSchema);
+    },
+  });
+
+  // catalog_macro_create
+  catalogUnary(protocol, signingKey, "catalog_macro_create", {
+    params: REQUEST_PARAMS_SCHEMA,
+    result: emptyResultSchema,
+    handler: async (params) => {
+      const cat = getCatalog();
+      const innerParams = unwrapRequest(params.request);
+      await cat.macroCreate(
+        toUint8Array(innerParams.attach_opaque_data),
+        innerParams.schema_name,
+        innerParams.name,
+        innerParams.macro_type as MacroType,
+        innerParams.parameters ? (Array.isArray(innerParams.parameters) ? innerParams.parameters : [...innerParams.parameters]) : [],
+        innerParams.definition,
+        innerParams.on_conflict,
+        innerParams.parameter_default_values ? toUint8Array(innerParams.parameter_default_values) : null,
+        innerParams.arguments_schema ? toUint8Array(innerParams.arguments_schema) : null,
+        innerParams.transaction_opaque_data ? toUint8Array(innerParams.transaction_opaque_data) : undefined
+      );
+      return {};
+    },
+  });
+
+  // catalog_macro_drop
+  catalogUnary(protocol, signingKey, "catalog_macro_drop", {
+    params: schema([
+      field("attach_opaque_data", binary(), true),
+      field("schema_name", utf8(), false),
+      field("name", utf8(), false),
+      field("ignore_not_found", bool(), true),
+      field("transaction_opaque_data", binary(), true),
+    ]),
+    result: emptyResultSchema,
+    handler: async (params) => {
+      const cat = getCatalog();
+      await cat.macroDrop(
+        toUint8Array(params.attach_opaque_data),
+        params.schema_name,
+        params.name,
+        params.ignore_not_found,
+        params.transaction_opaque_data ? toUint8Array(params.transaction_opaque_data) : undefined
+      );
+      return {};
+    },
+  });
+
+  // catalog_schema_contents_macros
+  catalogUnary(protocol, signingKey, "catalog_schema_contents_macros", {
+    params: schema([
+      field("attach_opaque_data", binary(), true),
+      field("name", utf8(), false),
+      field("type", utf8(), false),
+      field("transaction_opaque_data", binary(), true),
+    ]),
+    result: RESULT_BINARY_SCHEMA,
+    handler: async (params) => {
+      const cat = getCatalog();
+      const macros = await cat.schemaContentsMacros(
+        toUint8Array(params.attach_opaque_data),
+        params.name,
+        decodeDictValue(params.type),
+        params.transaction_opaque_data ? toUint8Array(params.transaction_opaque_data) : undefined
+      );
+      return wrapResult({
+        items: macros.map((m) => encodeMacroInfo(m)),
+      }, CatalogSchemaContentsMacrosResultSchema);
+    },
+  });
+}