@query-farm/vgi 0.28.0 → 0.29.1

package/src/catalog/interface.ts

@@ -3,6 +3,7 @@
 // Matching Python's vgi/catalog/catalog_interface.py
 
 import { CatalogReadOnlyError } from "../errors.js";
+import { schemaPathsEqual } from "../schema-path.js";
 
 export type AttachOpaqueData = Uint8Array;
 export type TransactionOpaqueData = Uint8Array;
@@ -164,7 +165,7 @@ export const encodeAttachCatalogInfo = (v: AttachCatalogInfo): Uint8Array =>
 import { deserializeBatch, serializeBatch, batchFromColumns } from "../util/arrow/index.js";
 import { schema as schema_, field as field_, type VgiDataType, type VgiField } from "../arrow/index.js";
 import { toUint8Array } from "../util/bytes.js";
-import { ScanBranchSchema } from "../generated/vgi-protocol-schemas.js";
+import { ScanBranchSchema, ScanFunctionResultSchema } from "../generated/vgi-protocol-schemas.js";
 
 /**
  * Result from `tableScanFunctionGet` — tells the VGI DuckDB extension which
@@ -179,15 +180,61 @@ export interface ScanFunctionResult {
   namedArguments: Record<string, unknown>;
   /** DuckDB extensions to load before calling the function. */
   requiredExtensions: string[];
+  /**
+   * Catalog schema path `functionName` is registered in (protocol 2.0.0). A function
+   * name is unique only within a schema, so a client that doesn't know this
+   * cannot tell which implementation a colliding name refers to — set it
+   * whenever the resolving code already knows the schema. `null` for a
+   * pre-1.5.0 peer, or when the resolved function is a native DuckDB function
+   * with no VGI-side schema of its own (e.g. `read_parquet`).
+   */
+  schemaPath: string[] | null;
+}
+
+/**
+ * Encode a ScanFunctionResult wire record — the single-row batch shape
+ * `{ function_name, arguments, required_extensions, schema_path }` that
+ * `catalog_table_scan_function_get` returns and that a function-backed table
+ * inlines into `TableInfo.scan_function`.
+ *
+ * It exists so there is exactly one place that names those columns. The
+ * inline-scan path used to restate the schema locally even though codegen
+ * emits it; a hand-written wire schema is how four SDKs ended up disagreeing
+ * about which columns were binary and which were large_binary.
+ *
+ * `argumentsBytes` is the already-serialized inner arguments batch (one column
+ * per argument: `arg_<index>` positional, bare name for named).
+ *
+ * `schemaPath` (protocol 2.0.0) identifies the schema that registers `functionName`;
+ * leave it null when the caller has no VGI-side schema to report.
+ */
+export function encodeScanFunctionResult(
+  functionName: string,
+  argumentsBytes: Uint8Array,
+  requiredExtensions: string[] = [],
+  schemaPath: string[] | null = null,
+): Uint8Array {
+  return serializeBatch(
+    batchFromColumns(
+      {
+        function_name: [functionName],
+        arguments: [argumentsBytes],
+        required_extensions: [requiredExtensions],
+        schema_path: [schemaPath],
+      },
+      ScanFunctionResultSchema as any,
+    ),
+  );
 }
 
 /**
  * Decode a ScanFunctionResult from a wire-shape inner result dict.
  *
  * The wire shape (per ScanFunctionResultSchema) is
- * `{ function_name: utf8, arguments: binary, required_extensions: list<utf8> }`,
- * where `arguments` is a serialized single-row batch with one column per
- * argument: `arg_<index>` for positional args, the bare name for named args.
+ * `{ function_name: utf8, arguments: binary, required_extensions: list<utf8>,
+ * schema_path: list<utf8>? }`, where `arguments` is a serialized single-row batch
+ * with one column per argument: `arg_<index>` for positional args, the bare
+ * name for named args.
  */
 export function decodeScanFunctionResult(inner: Record<string, unknown>): ScanFunctionResult {
   const argsBytes = inner.arguments;
@@ -221,6 +268,13 @@ export function decodeScanFunctionResult(inner: Record<string, unknown>): ScanFu
     positionalArguments,
     namedArguments,
     requiredExtensions,
+    // Absent for a pre-1.5.0 peer — the column itself won't be on the wire in
+    // that case, which reads the same as "present but null".
+    schemaPath: inner.schema_path == null
+      ? null
+      : (Array.isArray(inner.schema_path) ? inner.schema_path : [...(inner.schema_path as Iterable<unknown>)])
+          .filter((value) => value != null)
+          .map(String),
   };
 }
 
@@ -235,26 +289,75 @@ export function decodeScanFunctionResult(inner: Record<string, unknown>): ScanFu
  * `ScanBranch` expects, so it passes through untouched.
  */
 export function singleBranchResult(
-  legacy: { function_name: string; arguments: unknown; required_extensions?: string[] },
+  legacy: {
+    function_name: string;
+    arguments: unknown;
+    required_extensions?: string[];
+    schema_path?: string[] | null;
+  },
 ): { branches: Uint8Array[]; required_extensions: string[] } {
-  const branchBatch = batchFromColumns(
-    {
-      function_name: [legacy.function_name],
-      arguments: [toUint8Array(legacy.arguments)],
-      branch_filter: [null],
-      writable: [false],
-      source_catalog: [null],
-      source_schema: [null],
-      source_table: [null],
-    },
-    ScanBranchSchema as any,
-  );
   return {
-    branches: [serializeBatch(branchBatch)],
+    branches: [
+      encodeScanBranch({
+        function_name: legacy.function_name,
+        arguments: toUint8Array(legacy.arguments),
+        // This shim has no independent way to know the function's own schema;
+        // propagate whatever `tableScanFunctionGet` already resolved rather
+        // than dropping a value the legacy response carries.
+        schema_path: legacy.schema_path ?? null,
+      }),
+    ],
     required_extensions: legacy.required_extensions ?? [],
   };
 }
 
+/**
+ * The ONE place `ScanBranch`'s columns are named.
+ *
+ * Both branch builders used to carry their own copy of this key list, and both
+ * copies were missing `format_name` / `format_locations` after those fields
+ * were added to the schema. `format_locations` is a LIST, and Arrow's assembler
+ * dereferences a list's `children[0]` while writing, so the omission did not
+ * degrade anything — it killed every scan through those builders with
+ * "Cannot read properties of undefined (reading 'slice')", including tables
+ * that predated format branches entirely. One key list is the fix; the
+ * completeness test in src/__tests__ pins it against ScanBranchSchema.
+ *
+ * Every column the schema declares must appear here, null included.
+ */
+function encodeScanBranch(cols: {
+  function_name: string;
+  arguments: Uint8Array;
+  branch_filter?: string | null;
+  writable?: boolean;
+  source_catalog?: string | null;
+  source_schema_path?: string[] | null;
+  source_table?: string | null;
+  format_name?: string | null;
+  format_locations?: string[] | null;
+  format_options?: Uint8Array | null;
+  schema_path?: string[] | null;
+}): Uint8Array {
+  return serializeBatch(
+    batchFromColumns(
+      {
+        function_name: [cols.function_name],
+        arguments: [cols.arguments],
+        branch_filter: [cols.branch_filter ?? null],
+        writable: [cols.writable ?? false],
+        source_catalog: [cols.source_catalog ?? null],
+        source_schema_path: [cols.source_schema_path ?? null],
+        source_table: [cols.source_table ?? null],
+        format_name: [cols.format_name ?? null],
+        format_locations: [cols.format_locations ?? null],
+        format_options: [cols.format_options ?? null],
+        schema_path: [cols.schema_path ?? null],
+      },
+      ScanBranchSchema as any,
+    ),
+  );
+}
+
 /**
  * One physical source backing a multi-branch scan, in the shape the
  * `buildScanBranchesResult` helper consumes. Mirrors vgi-python's
@@ -275,15 +378,38 @@ export interface ScanBranchInput {
   branchFilter?: string | null;
   /** Declares this branch as the INSERT target (at most one per table). */
   writable?: boolean;
+  /** Custom file FORMAT this branch reads, for a format-backed branch. */
+  formatName?: string | null;
+  /** Locations the format branch reads. */
+  formatLocations?: string[] | null;
+  /** Reader options for a format branch, as a 1-row IPC batch whose COLUMN
+   *  NAMES are the option names — the same shape `arguments` uses, because an
+   *  option value may be any Arrow type. These BECOME the reader's named
+   *  arguments, so without them a format branch cannot pass its reader a
+   *  delimiter, a header flag, or anything else. */
+  formatOptions?: Uint8Array | null;
   /**
    * Catalog-table branch (lakehouse federation): leave `functionName` empty and
    * set these to scan the base table
-   * `sourceCatalog.sourceSchema.sourceTable` in a companion catalog instead of
+   * `sourceCatalog.sourceSchemaPath.sourceTable` in a companion catalog instead of
    * calling a table function.
    */
   sourceCatalog?: string | null;
-  sourceSchema?: string | null;
+  sourceSchemaPath?: string[] | null;
   sourceTable?: string | null;
+  /**
+   * Function branch only — the catalog schema that registers `functionName`
+   * (protocol 2.0.0). Without it a client has to guess (the table's own
+   * schema, then the catalog's default schema) which implementation
+   * `functionName` refers to when the same name is registered in more than
+   * one schema. Leave unset for a catalog-table or FORMAT branch, and for a
+   * branch that delegates straight to a native DuckDB function
+   * (`read_parquet`, `read_csv_auto`, `iceberg_scan`, …) — neither has a
+   * VGI-side schema of its own to report, which is why the field is optional
+   * rather than required. Not to be confused with `sourceSchemaPath` above, which
+   * names a catalog-table branch's *source table's* schema.
+   */
+  schemaPath?: string[] | null;
 }
 
 /**
@@ -292,6 +418,26 @@ export interface ScanBranchInput {
  * positional args, the bare name for named args. Mirrors Python's
  * `ScanBranch.to_row_dict` argument handling.
  */
+/**
+ * Encode a format branch's reader options.
+ *
+ * Same shape as a branch's `arguments` blob — a 1-row batch whose COLUMN NAMES
+ * are the option names — because an option value may be any Arrow type, so no
+ * static schema could describe the map. Exported because a worker building a
+ * format branch needs it and has no other way to produce those bytes.
+ */
+export function encodeFormatOptions(
+  options: Record<string, { value: unknown; type: VgiDataType }>,
+): Uint8Array {
+  const fields: VgiField[] = [];
+  const values: Record<string, unknown[]> = {};
+  for (const [name, arg] of Object.entries(options)) {
+    fields.push(field_(name, arg.type, true));
+    values[name] = [arg.value];
+  }
+  return serializeBatch(batchFromColumns(values, schema_(fields)));
+}
+
 function serializeBranchArguments(branch: ScanBranchInput): Uint8Array {
   const fields: VgiField[] = [];
   const values: Record<string, unknown[]> = {};
@@ -328,21 +474,21 @@ export function buildScanBranchesResult(
   branches: ScanBranchInput[],
   requiredExtensions: string[] = [],
 ): { branches: Uint8Array[]; required_extensions: string[] } {
-  const serializedBranches = branches.map((branch) => {
-    const branchBatch = batchFromColumns(
-      {
-        function_name: [branch.functionName],
-        arguments: [serializeBranchArguments(branch)],
-        branch_filter: [branch.branchFilter ?? null],
-        writable: [branch.writable ?? false],
-        source_catalog: [branch.sourceCatalog ?? null],
-        source_schema: [branch.sourceSchema ?? null],
-        source_table: [branch.sourceTable ?? null],
-      },
-      ScanBranchSchema as any,
-    );
-    return serializeBatch(branchBatch);
-  });
+  const serializedBranches = branches.map((branch) =>
+    encodeScanBranch({
+      function_name: branch.functionName,
+      arguments: serializeBranchArguments(branch),
+      branch_filter: branch.branchFilter,
+      writable: branch.writable,
+      source_catalog: branch.sourceCatalog,
+      source_schema_path: branch.sourceSchemaPath,
+      source_table: branch.sourceTable,
+      format_name: branch.formatName,
+      format_locations: branch.formatLocations,
+      format_options: branch.formatOptions,
+      schema_path: branch.schemaPath,
+    }),
+  );
   return {
     branches: serializedBranches,
     required_extensions: requiredExtensions,
@@ -422,18 +568,18 @@ export abstract class CatalogInterface {
   }
   schemaGet(
     attachOpaqueData: AttachOpaqueData,
-    name: string,
+    path: string[],
     transactionOpaqueData?: TransactionOpaqueData
   ): Awaitable<SchemaInfo | null> {
     const all = this.schemas(attachOpaqueData, transactionOpaqueData);
     if (isPromise(all)) {
-      return all.then((arr) => arr.find((s) => s.name === name) ?? null);
+      return all.then((arr) => arr.find((s) => schemaPathsEqual(s.path, path)) ?? null);
     }
-    return all.find((s) => s.name === name) ?? null;
+    return all.find((s) => schemaPathsEqual(s.path, path)) ?? null;
   }
   schemaCreate(
     attachOpaqueData: AttachOpaqueData,
-    name: string,
+    path: string[],
     comment?: string | null,
     tags?: any,
     transactionOpaqueData?: TransactionOpaqueData
@@ -442,7 +588,7 @@ export abstract class CatalogInterface {
   }
   schemaDrop(
     attachOpaqueData: AttachOpaqueData,
-    name: string,
+    path: string[],
     ignoreNotFound?: boolean,
     cascade?: boolean,
     transactionOpaqueData?: TransactionOpaqueData
@@ -451,21 +597,21 @@ export abstract class CatalogInterface {
   }
   schemaContentsTables(
     attachOpaqueData: AttachOpaqueData,
-    name: string,
+    path: string[],
     transactionOpaqueData?: TransactionOpaqueData
   ): Awaitable<TableInfo[]> {
     return [];
   }
   schemaContentsViews(
     attachOpaqueData: AttachOpaqueData,
-    name: string,
+    path: string[],
     transactionOpaqueData?: TransactionOpaqueData
   ): Awaitable<ViewInfo[]> {
     return [];
   }
   schemaContentsFunctions(
     attachOpaqueData: AttachOpaqueData,
-    name: string,
+    path: string[],
     type: string,
     transactionOpaqueData?: TransactionOpaqueData
   ): Awaitable<FunctionInfo[]> {
@@ -473,7 +619,7 @@ export abstract class CatalogInterface {
   }
   tableGet(
     attachOpaqueData: AttachOpaqueData,
-    schemaName: string,
+    schemaPath: string[],
     name: string,
     atUnit?: string,
     atValue?: string,
@@ -483,7 +629,7 @@ export abstract class CatalogInterface {
   }
   tableCreate(
     attachOpaqueData: AttachOpaqueData,
-    schemaName: string,
+    schemaPath: string[],
     name: string,
     columns: Uint8Array,
     onConflict: string,
@@ -496,7 +642,7 @@ export abstract class CatalogInterface {
   }
   tableDrop(
     attachOpaqueData: AttachOpaqueData,
-    schemaName: string,
+    schemaPath: string[],
     name: string,
     ignoreNotFound?: boolean,
     transactionOpaqueData?: TransactionOpaqueData
@@ -505,7 +651,7 @@ export abstract class CatalogInterface {
   }
   tableScanFunctionGet(
     attachOpaqueData: AttachOpaqueData,
-    schemaName: string,
+    schemaPath: string[],
     name: string,
     atUnit?: string,
     atValue?: string,
@@ -521,14 +667,14 @@ export abstract class CatalogInterface {
    */
   tableScanBranchesGet(
     attachOpaqueData: AttachOpaqueData,
-    schemaName: string,
+    schemaPath: string[],
     name: string,
     atUnit?: string,
     atValue?: string,
     transactionOpaqueData?: TransactionOpaqueData
   ): Awaitable<any> {
     const legacy = this.tableScanFunctionGet(
-      attachOpaqueData, schemaName, name, atUnit, atValue, transactionOpaqueData,
+      attachOpaqueData, schemaPath, name, atUnit, atValue, transactionOpaqueData,
     );
     return isPromise(legacy)
       ? legacy.then((r) => singleBranchResult(r))
@@ -544,7 +690,7 @@ export abstract class CatalogInterface {
    */
   tableColumnStatisticsGet(
     attachOpaqueData: AttachOpaqueData,
-    schemaName: string,
+    schemaPath: string[],
     name: string,
     transactionOpaqueData?: TransactionOpaqueData,
   ): Awaitable<{ bytes: Uint8Array; cacheMaxAgeSeconds: number | null } | null> {
@@ -552,7 +698,7 @@ export abstract class CatalogInterface {
   }
   tableCommentSet(
     attachOpaqueData: AttachOpaqueData,
-    schemaName: string,
+    schemaPath: string[],
     name: string,
     comment?: string | null,
     ignoreNotFound?: boolean,
@@ -562,7 +708,7 @@ export abstract class CatalogInterface {
   }
   tableRename(
     attachOpaqueData: AttachOpaqueData,
-    schemaName: string,
+    schemaPath: string[],
     name: string,
     newName: string,
     ignoreNotFound?: boolean,
@@ -572,7 +718,7 @@ export abstract class CatalogInterface {
   }
   tableColumnAdd(
     attachOpaqueData: AttachOpaqueData,
-    schemaName: string,
+    schemaPath: string[],
     name: string,
     columnName: string,
     columnType: string,
@@ -584,7 +730,7 @@ export abstract class CatalogInterface {
   }
   tableColumnDrop(
     attachOpaqueData: AttachOpaqueData,
-    schemaName: string,
+    schemaPath: string[],
     name: string,
     columnName: string,
     ignoreNotFound?: boolean,
@@ -594,7 +740,7 @@ export abstract class CatalogInterface {
   }
   tableColumnRename(
     attachOpaqueData: AttachOpaqueData,
-    schemaName: string,
+    schemaPath: string[],
     name: string,
     columnName: string,
     newName: string,
@@ -605,7 +751,7 @@ export abstract class CatalogInterface {
   }
   tableColumnDefaultSet(
     attachOpaqueData: AttachOpaqueData,
-    schemaName: string,
+    schemaPath: string[],
     name: string,
     columnName: string,
     defaultValue: string,
@@ -616,7 +762,7 @@ export abstract class CatalogInterface {
   }
   tableColumnDefaultDrop(
     attachOpaqueData: AttachOpaqueData,
-    schemaName: string,
+    schemaPath: string[],
     name: string,
     columnName: string,
     ignoreNotFound?: boolean,
@@ -626,7 +772,7 @@ export abstract class CatalogInterface {
   }
   tableColumnTypeChange(
     attachOpaqueData: AttachOpaqueData,
-    schemaName: string,
+    schemaPath: string[],
     name: string,
     columnName: string,
     newType: string,
@@ -637,7 +783,7 @@ export abstract class CatalogInterface {
   }
   tableNotNullSet(
     attachOpaqueData: AttachOpaqueData,
-    schemaName: string,
+    schemaPath: string[],
     name: string,
     columnName: string,
     ignoreNotFound?: boolean,
@@ -647,7 +793,7 @@ export abstract class CatalogInterface {
   }
   tableNotNullDrop(
     attachOpaqueData: AttachOpaqueData,
-    schemaName: string,
+    schemaPath: string[],
     name: string,
     columnName: string,
     ignoreNotFound?: boolean,
@@ -657,7 +803,7 @@ export abstract class CatalogInterface {
   }
   viewGet(
     attachOpaqueData: AttachOpaqueData,
-    schemaName: string,
+    schemaPath: string[],
     name: string,
     transactionOpaqueData?: TransactionOpaqueData
   ): Awaitable<ViewInfo | null> {
@@ -665,7 +811,7 @@ export abstract class CatalogInterface {
   }
   viewCreate(
     attachOpaqueData: AttachOpaqueData,
-    schemaName: string,
+    schemaPath: string[],
     name: string,
     definition: string,
     onConflict: string,
@@ -675,7 +821,7 @@ export abstract class CatalogInterface {
   }
   viewDrop(
     attachOpaqueData: AttachOpaqueData,
-    schemaName: string,
+    schemaPath: string[],
     name: string,
     ignoreNotFound?: boolean,
     transactionOpaqueData?: TransactionOpaqueData
@@ -684,7 +830,7 @@ export abstract class CatalogInterface {
   }
   viewRename(
     attachOpaqueData: AttachOpaqueData,
-    schemaName: string,
+    schemaPath: string[],
     name: string,
     newName: string,
     ignoreNotFound?: boolean,
@@ -694,7 +840,7 @@ export abstract class CatalogInterface {
   }
   viewCommentSet(
     attachOpaqueData: AttachOpaqueData,
-    schemaName: string,
+    schemaPath: string[],
     name: string,
     comment?: string | null,
     ignoreNotFound?: boolean,
@@ -704,7 +850,7 @@ export abstract class CatalogInterface {
   }
   macroGet(
     attachOpaqueData: AttachOpaqueData,
-    schemaName: string,
+    schemaPath: string[],
     name: string,
     transactionOpaqueData?: TransactionOpaqueData
   ): Awaitable<MacroInfo | null> {
@@ -712,7 +858,7 @@ export abstract class CatalogInterface {
   }
   macroCreate(
     attachOpaqueData: AttachOpaqueData,
-    schemaName: string,
+    schemaPath: string[],
     name: string,
     macroType: MacroType,
     parameters: string[],
@@ -732,7 +878,7 @@ export abstract class CatalogInterface {
   }
   macroDrop(
     attachOpaqueData: AttachOpaqueData,
-    schemaName: string,
+    schemaPath: string[],
     name: string,
     ignoreNotFound?: boolean,
     transactionOpaqueData?: TransactionOpaqueData
@@ -741,7 +887,7 @@ export abstract class CatalogInterface {
   }
   schemaContentsMacros(
     attachOpaqueData: AttachOpaqueData,
-    name: string,
+    path: string[],
     type: string,
     transactionOpaqueData?: TransactionOpaqueData
   ): Awaitable<MacroInfo[]> {
@@ -749,14 +895,14 @@ export abstract class CatalogInterface {
   }
   schemaContentsIndexes(
     attachOpaqueData: AttachOpaqueData,
-    name: string,
+    path: string[],
     transactionOpaqueData?: TransactionOpaqueData
   ): Awaitable<IndexInfo[]> {
     return [];
   }
   indexGet(
     attachOpaqueData: AttachOpaqueData,
-    schemaName: string,
+    schemaPath: string[],
     name: string,
     transactionOpaqueData?: TransactionOpaqueData
   ): Awaitable<IndexInfo | null> {
@@ -787,4 +933,3 @@ export abstract class CatalogInterface {
     transactionOpaqueData: TransactionOpaqueData
   ): Awaitable<void> {}
 }
-