@query-farm/vgi 0.24.1 → 0.25.0

package/dist/worker-cf-entry.d.ts

@@ -3,8 +3,7 @@ export type { VgiFetchOptions } from "./http/fetch.js";
 export { defineScalarFunction, defineTableFunction, defineAggregate, defineTableInOutFunction, FunctionRegistry, ReadOnlyCatalogInterface, CompositeCatalogInterface, CatalogInterface, Arguments, FunctionType, serializeAttachOptionSpec, serializeAttachOptionSpecs, serializeColumnStatistics, BindResultSchema, ScanFunctionResultSchema, } from "./index.core.js";
 export type { AttachOptionSpec, ColumnStatistics, CatalogAttachResult, CatalogDescriptor, CatalogInfo, TableInfo, AttachOpaqueData, TransactionOpaqueData, SchemaInfo, VgiFunction, TableProcessParams, } from "./index.core.js";
 export type { ProtocolConfig } from "./protocol/dispatch.js";
-export { createLandingDescribe } from "./http/describe-json.js";
-export type { LandingDescribeProvider, WorkerDescribeInfo } from "./http/describe-json.js";
+export type { LandingInfo } from "@query-farm/vgi-rpc";
 export { TypeId, isNull, isInt, isFloat, isBinary, isUtf8, isBool, isDecimal, isDate, isTime, isTimestamp, isInterval, isList, isStruct, isUnion, isFixedSizeBinary, isFixedSizeList, isMap, isDuration, isDictionary, isBatch, backend, schema, field, nullType, bool, int, int8, int16, int32, int64, uint8, uint16, uint32, uint64, float16, float32, float64, utf8, binary, fixedSizeBinary, decimal, decimal128, decimal256, date, dateDay, dateMillisecond, time, timeSecond, timeMillisecond, timeMicrosecond, timeNanosecond, timestamp, duration, interval, list, fixedSizeList, struct, map, dictionary, union, sparseUnion, denseUnion, TimeUnit, DateUnit, IntervalUnit, UnionMode, emptyBatch, batchFromRows, batchFromColumns, columnFromArray, iterRows, batchToScalarDict, batchToSecretDict, safeNumber, decodeDictValue, filterBatch, projectSchema, projectBatch, serializeSchema, deserializeSchema, serializeBatch, deserializeBatch, buildStatisticsBatch, } from "./arrow/index.js";
 export type { VgiTypeId, VgiDataType, VgiField, VgiSchema, VgiColumn, VgiColumnData, VgiBatch, VgiBackendInfo, } from "./arrow/index.js";
 export { arrowStateSerializer, EXCHANGE_STATE_SCHEMA } from "./protocol/state-serializer.js";

package/dist/worker-cf-entry.js

@@ -7296,7 +7296,7 @@ function createVgiFetch(opts) {
     stateSerializer: arrowStateSerializer,
     corsOrigins: opts.corsOrigins,
     repositoryUrl: opts.repositoryUrl,
-    landingDescribe: opts.landingDescribe
+    landingInfo: opts.landingInfo
   });
   return async (req) => handler(req);
 }
@@ -8116,6 +8116,55 @@ function resolveMetadata(func) {
   };
 }
 
+// src/catalog/attach-option.ts
+init_arrow();
+var SPEC_SCHEMA = schema2([
+  field2("name", utf82(), false),
+  field2("description", utf82(), false),
+  field2("type", binary2(), false),
+  field2("default_value", binary2(), true),
+  field2("required", bool2(), true)
+]);
+function serializeAttachOptionSpec(spec) {
+  if (spec.required && spec.default !== undefined && spec.default !== null) {
+    throw new Error(`Attach option '${spec.name}' is required but also declares a default ` + `(${JSON.stringify(spec.default)}); an option with a default is always ` + "satisfiable without the caller. Drop one.");
+  }
+  const typeSchema = schema2([field2("value", spec.type, true)]);
+  const typeBytes = serializeSchema(typeSchema);
+  let defaultBytes = null;
+  if (spec.default !== undefined && spec.default !== null) {
+    const defaultBatch = batchFromColumns({ value: [spec.default] }, typeSchema);
+    defaultBytes = serializeBatch(defaultBatch);
+  }
+  const batch = batchFromColumns({
+    name: [spec.name],
+    description: [spec.description],
+    type: [typeBytes],
+    default_value: [defaultBytes],
+    required: [spec.required ?? false]
+  }, SPEC_SCHEMA);
+  return serializeBatch(batch);
+}
+function serializeAttachOptionSpecs(specs) {
+  return Array.from(specs, serializeAttachOptionSpec);
+}
+class MissingAttachOptionsError extends Error {
+  missing;
+  constructor(catalogName, missing) {
+    const joined = missing.map((name) => `'${name}'`).join(", ");
+    super(`Catalog '${catalogName}' cannot be attached without the required ` + `option${missing.length > 1 ? "s" : ""} ${joined}.`);
+    this.name = "MissingAttachOptionsError";
+    this.missing = missing;
+  }
+}
+function validateRequiredAttachOptions(catalogName, specs, options) {
+  const supplied = new Set(Object.keys(options).map((key) => key.toLowerCase()));
+  const missing = Array.from(specs).filter((spec) => spec.required && !supplied.has(spec.name.toLowerCase())).map((spec) => spec.name);
+  if (missing.length > 0) {
+    throw new MissingAttachOptionsError(catalogName, missing);
+  }
+}
+
 // src/catalog/read-only.ts
 class ReadOnlyCatalogInterface extends CatalogInterface {
   _descriptor;
@@ -8141,6 +8190,9 @@ class ReadOnlyCatalogInterface extends CatalogInterface {
   catalogs() {
     return [this._descriptor.name];
   }
+  attachOptionSpecs(_name) {
+    return [];
+  }
   catalogsInfo() {
     return [{
       name: this._descriptor.name,
@@ -8155,6 +8207,7 @@ class ReadOnlyCatalogInterface extends CatalogInterface {
     if (!this.catalogs().includes(name)) {
       throw new Error(`No worker handles catalog '${name}'`);
     }
+    validateRequiredAttachOptions(name, this.attachOptionSpecs(name), options ?? {});
     const attachOpaqueData = new Uint8Array(16);
     crypto.getRandomValues(attachOpaqueData);
     this._attachments.set(name, attachOpaqueData);
@@ -9164,443 +9217,6 @@ class FunctionRegistry {
     return result;
   }
 }
-// src/catalog/attach-option.ts
-init_arrow();
-var SPEC_SCHEMA = schema2([
-  field2("name", utf82(), false),
-  field2("description", utf82(), false),
-  field2("type", binary2(), false),
-  field2("default_value", binary2(), true)
-]);
-function serializeAttachOptionSpec(spec) {
-  const typeSchema = schema2([field2("value", spec.type, true)]);
-  const typeBytes = serializeSchema(typeSchema);
-  let defaultBytes = null;
-  if (spec.default !== undefined && spec.default !== null) {
-    const defaultBatch = batchFromColumns({ value: [spec.default] }, typeSchema);
-    defaultBytes = serializeBatch(defaultBatch);
-  }
-  const batch = batchFromColumns({
-    name: [spec.name],
-    description: [spec.description],
-    type: [typeBytes],
-    default_value: [defaultBytes]
-  }, SPEC_SCHEMA);
-  return serializeBatch(batch);
-}
-function serializeAttachOptionSpecs(specs) {
-  return Array.from(specs, serializeAttachOptionSpec);
-}
-// src/http/describe-json.ts
-init_arrow();
-var LANDING_SCHEMA_VERSION2 = 1;
-var CUPOLA_BASE2 = "https://cupola.query-farm.services";
-var STRING_TAGS = {
-  title: "vgi.title",
-  doc_md: "vgi.doc_md",
-  source_url: "vgi.source_url",
-  license: "vgi.license",
-  author: "vgi.author",
-  copyright: "vgi.copyright",
-  support_contact: "vgi.support_contact",
-  support_policy_url: "vgi.support_policy_url"
-};
-var KEYWORDS_TAG = "vgi.keywords";
-function typeToString(type) {
-  if (isNull(type))
-    return "NULL";
-  if (isUtf8(type))
-    return "VARCHAR";
-  if (isBinary(type))
-    return "BLOB";
-  if (isBool(type))
-    return "BOOLEAN";
-  if (isInt(type)) {
-    const bw = type.bitWidth ?? 32;
-    return bw === 64 ? "BIGINT" : "INTEGER";
-  }
-  if (isFloat(type)) {
-    const precision = type.precision ?? 2;
-    return precision === 1 ? "FLOAT" : "DOUBLE";
-  }
-  const s = type.toString?.();
-  return s ?? String(type.typeId);
-}
-function arrowTypeString(type) {
-  if (isNull(type))
-    return "null";
-  if (isBool(type))
-    return "bool";
-  if (isUtf8(type))
-    return "string";
-  if (isBinary(type))
-    return "binary";
-  if (isInt(type)) {
-    const bw = type.bitWidth ?? 32;
-    const signed = type.isSigned ?? type.signed ?? true;
-    return `${signed ? "" : "u"}int${bw}`;
-  }
-  if (isFloat(type)) {
-    const precision = type.precision ?? 2;
-    return precision === 0 ? "halffloat" : precision === 1 ? "float" : "double";
-  }
-  const s = type.toString?.();
-  return (s ?? String(type.typeId)).toLowerCase();
-}
-function fieldMeta(field4, key) {
-  return field4.metadata.get(key);
-}
-function fieldToColumn(field4) {
-  const col = { name: field4.name, type: typeToString(field4.type) };
-  const comment = fieldMeta(field4, "comment") ?? fieldMeta(field4, VGI_DOC_KEY);
-  if (comment)
-    col.comment = comment;
-  return col;
-}
-function readSchemaSafe(bytes2) {
-  if (!bytes2 || bytes2.length === 0)
-    return null;
-  try {
-    return deserializeSchema(bytes2);
-  } catch {
-    return null;
-  }
-}
-function functionDisplayType(fn) {
-  const t = String(fn.function_type).toUpperCase();
-  if (t === "SCALAR")
-    return "scalar";
-  if (t === "AGGREGATE")
-    return "aggregate";
-  if (fn.has_finalize)
-    return "table_in_out";
-  return "table";
-}
-function functionReturns(fn) {
-  const schema4 = readSchemaSafe(fn.output_schema);
-  if (schema4 === null || schema4.fields.length === 0)
-    return;
-  const t = String(fn.function_type).toUpperCase();
-  if (t === "SCALAR" || t === "AGGREGATE") {
-    return typeToString(schema4.fields[0].type);
-  }
-  const cols = schema4.fields.map((f) => `${f.name} ${typeToString(f.type)}`).join(", ");
-  return `TABLE(${cols})`;
-}
-function functionArgs(fn) {
-  const schema4 = readSchemaSafe(fn.arguments);
-  if (schema4 === null)
-    return [];
-  const args = [];
-  for (const field4 of schema4.fields) {
-    if (fieldMeta(field4, VGI_TYPE_KEY) === VGI_TYPE_TABLE)
-      continue;
-    const arg = { name: field4.name, type: typeToString(field4.type) };
-    if (fieldMeta(field4, VGI_ARG_KEY) === VGI_ARG_NAMED)
-      arg.named = true;
-    const doc = fieldMeta(field4, VGI_DOC_KEY);
-    if (doc)
-      arg.desc = doc;
-    const rawDefault = fieldMeta(field4, VGI_DEFAULT_KEY);
-    if (rawDefault !== undefined) {
-      try {
-        arg.default = JSON.stringify(JSON.parse(rawDefault));
-      } catch {
-        arg.default = rawDefault;
-      }
-    }
-    args.push(arg);
-  }
-  return args;
-}
-function macroDisplayType(m) {
-  return String(m.macro_type).toUpperCase() === "SCALAR" ? "scalar" : "table";
-}
-function macroDefaultJson(value) {
-  if (typeof value === "bigint")
-    return value.toString();
-  try {
-    return JSON.stringify(value);
-  } catch {
-    return String(value);
-  }
-}
-function macroArgs(m) {
-  const defaults = new Map;
-  const defBytes = m.parameter_default_values;
-  if (defBytes && defBytes.length > 0) {
-    try {
-      const batch = deserializeBatch(defBytes);
-      for (const field4 of batch.schema.fields) {
-        try {
-          defaults.set(field4.name, batch.getChild(field4.name)?.get(0));
-        } catch {}
-      }
-    } catch {}
-  }
-  const schema4 = readSchemaSafe(m.arguments_schema);
-  const fields = schema4 ? schema4.fields : null;
-  const fieldByName = new Map;
-  if (fields)
-    for (const f of fields)
-      fieldByName.set(f.name, f);
-  const names = fields ? fields.map((f) => f.name) : m.parameters;
-  const args = [];
-  for (const name of names) {
-    const field4 = fieldByName.get(name);
-    const arg = field4 !== undefined && !isNull(field4.type) ? { name, type: arrowTypeString(field4.type) } : { name, type: "ANY" };
-    const doc = field4 !== undefined ? fieldMeta(field4, VGI_DOC_KEY) : undefined;
-    if (doc)
-      arg.desc = doc;
-    if (defaults.has(name)) {
-      arg.named = true;
-      arg.default = macroDefaultJson(defaults.get(name));
-    }
-    args.push(arg);
-  }
-  return args;
-}
-function macroToObject(m) {
-  return {
-    name: m.name,
-    type: macroDisplayType(m),
-    doc: m.comment ?? "",
-    args: macroArgs(m)
-  };
-}
-async function schemaContents(iface, aod, name, kind) {
-  try {
-    if (kind === "TABLE") {
-      return await iface.schemaContentsTables(aod, name);
-    }
-    return await iface.schemaContentsViews(aod, name);
-  } catch {
-    return [];
-  }
-}
-async function schemaFunctions(iface, aod, name) {
-  const out = [];
-  for (const kind of ["SCALAR_FUNCTION", "TABLE_FUNCTION", "AGGREGATE_FUNCTION"]) {
-    try {
-      out.push(...await iface.schemaContentsFunctions(aod, name, kind));
-    } catch {}
-  }
-  return out;
-}
-async function schemaMacros(iface, aod, name) {
-  const out = [];
-  for (const kind of ["SCALAR_MACRO", "TABLE_MACRO"]) {
-    try {
-      out.push(...await iface.schemaContentsMacros(aod, name, kind));
-    } catch {}
-  }
-  return out;
-}
-async function buildSchemas(iface, aod) {
-  let schemaInfos;
-  try {
-    schemaInfos = await iface.schemas(aod);
-  } catch {
-    return { schemas: [], counts: { schemas: 0, tables: 0, views: 0, functions: 0 } };
-  }
-  const schemas = [];
-  const counts = { schemas: 0, tables: 0, views: 0, functions: 0 };
-  for (const si of schemaInfos) {
-    const tablesRaw = await schemaContents(iface, aod, si.name, "TABLE");
-    const viewsRaw = await schemaContents(iface, aod, si.name, "VIEW");
-    const funcsRaw = await schemaFunctions(iface, aod, si.name);
-    const macrosRaw = await schemaMacros(iface, aod, si.name);
-    const tables = tablesRaw.map((t) => {
-      const schema4 = readSchemaSafe(t.columns);
-      return { name: t.name, cols: schema4 ? schema4.fields.length : 0, comment: t.comment ?? "" };
-    });
-    const views = viewsRaw.map((v) => ({
-      name: v.name,
-      cols: Object.keys(v.column_comments ?? {}).length,
-      comment: v.comment ?? "",
-      def: v.definition
-    }));
-    const functions = funcsRaw.map((fn) => {
-      const obj = {
-        name: fn.name,
-        type: functionDisplayType(fn),
-        doc: fn.description ?? "",
-        args: functionArgs(fn)
-      };
-      const ret = functionReturns(fn);
-      if (ret)
-        obj.returns = ret;
-      return obj;
-    });
-    functions.push(...macrosRaw.map((m) => macroToObject(m)));
-    functions.sort((a, b) => String(a.type).localeCompare(String(b.type)) || String(a.name).localeCompare(String(b.name)));
-    const schemaObj = { name: si.name };
-    if (si.comment)
-      schemaObj.doc = si.comment;
-    schemaObj.tables = tables;
-    schemaObj.views = views;
-    schemaObj.functions = functions;
-    schemas.push(schemaObj);
-    counts.schemas += 1;
-    counts.tables += tables.length;
-    counts.views += views.length;
-    counts.functions += functions.length;
-  }
-  return { schemas, counts };
-}
-function catalogTags(result) {
-  const tags = result?.tags;
-  if (!tags)
-    return {};
-  const out = {};
-  for (const [key, tag] of Object.entries(STRING_TAGS)) {
-    const val = tags[tag];
-    if (val)
-      out[key] = val;
-  }
-  const kw = tags[KEYWORDS_TAG];
-  if (kw) {
-    try {
-      const parsed = JSON.parse(kw);
-      if (Array.isArray(parsed))
-        out.keywords = parsed.map((k) => String(k));
-    } catch {}
-  }
-  return out;
-}
-function decodeAttachOption(bytes2) {
-  try {
-    const batch = deserializeBatch(bytes2);
-    if (batch.numRows === 0)
-      return null;
-    const name = String(batch.getChild("name")?.get(0) ?? "");
-    const description = String(batch.getChild("description")?.get(0) ?? "");
-    const typeBytes = batch.getChild("type")?.get(0);
-    let type = "";
-    const typeSchema = readSchemaSafe(typeBytes ?? null);
-    if (typeSchema && typeSchema.fields.length > 0)
-      type = typeToString(typeSchema.fields[0].type);
-    let defaultStr = "";
-    const defBytes = batch.getChild("default_value")?.get(0);
-    if (defBytes) {
-      try {
-        const defBatch = deserializeBatch(defBytes);
-        if (defBatch.numRows > 0) {
-          const v = defBatch.getChild("value")?.get(0);
-          defaultStr = v == null ? "" : String(v);
-        }
-      } catch {}
-    }
-    return { name, type, default: defaultStr, description };
-  } catch {
-    return null;
-  }
-}
-async function attachCatalog(iface, name) {
-  try {
-    return await iface.attach(name, {}, null, null);
-  } catch {
-    return null;
-  }
-}
-async function buildCatalog(iface, name, info) {
-  const result = await attachCatalog(iface, name);
-  const attachOptions = (info?.attach_option_specs ?? []).map((b) => decodeAttachOption(b)).filter((o) => o !== null);
-  const dataVersions = (info?.releases ?? []).map((r) => r.summary ? { spec: r.version, label: r.summary } : { spec: r.version });
-  let impl = info?.implementation_version ?? null;
-  let dvs = info?.data_version_spec ?? null;
-  if (result) {
-    impl = result.resolved_implementation_version ?? impl;
-    dvs = result.resolved_data_version ?? dvs;
-  }
-  let schemas = [];
-  let counts = { schemas: 0, tables: 0, views: 0, functions: 0 };
-  if (result) {
-    const built = await buildSchemas(iface, result.attach_opaque_data);
-    schemas = built.schemas;
-    counts = built.counts;
-  }
-  return {
-    name,
-    implementation_version: impl,
-    data_version_spec: dvs,
-    data_versions: dataVersions,
-    attach_options: attachOptions,
-    tags: catalogTags(result),
-    counts,
-    schemas
-  };
-}
-async function catalogInfoByName(iface) {
-  const map4 = new Map;
-  if (typeof iface.catalogsInfo === "function") {
-    try {
-      const infos = await iface.catalogsInfo();
-      for (const info of infos)
-        map4.set(info.name, info);
-    } catch {}
-  }
-  return map4;
-}
-async function buildDescribeJson2(iface, worker, opts = {}) {
-  const infos = await catalogInfoByName(iface);
-  let names;
-  try {
-    names = iface.catalogs();
-  } catch {
-    names = [];
-  }
-  const catalogs = [];
-  for (const name of names) {
-    catalogs.push(await buildCatalog(iface, name, infos.get(name)));
-  }
-  return {
-    landing_schema_version: LANDING_SCHEMA_VERSION2,
-    worker: {
-      name: worker.name,
-      doc: worker.doc ?? "",
-      version: worker.version ?? "unknown",
-      lang: "typescript"
-    },
-    server_id: opts.serverId ?? "",
-    oauth: opts.oauth ?? false,
-    cupola_base: opts.cupolaBase ?? CUPOLA_BASE2,
-    catalogs
-  };
-}
-async function buildColumnsJson2(iface, catalog, schema4, table) {
-  const result = await attachCatalog(iface, catalog);
-  if (!result)
-    return null;
-  const aod = result.attach_opaque_data;
-  const tables = await schemaContents(iface, aod, schema4, "TABLE");
-  for (const t of tables) {
-    if (t.name === table) {
-      const arrow = readSchemaSafe(t.columns);
-      return { columns: (arrow?.fields ?? []).map((f) => fieldToColumn(f)) };
-    }
-  }
-  const views = await schemaContents(iface, aod, schema4, "VIEW");
-  for (const v of views) {
-    if (v.name === table) {
-      return {
-        columns: Object.entries(v.column_comments ?? {}).map(([name, comment]) => ({
-          name,
-          type: "",
-          comment
-        }))
-      };
-    }
-  }
-  return null;
-}
-function createLandingDescribe2(iface, worker, opts = {}) {
-  return {
-    describe: (ctx) => buildDescribeJson2(iface, worker, { ...ctx, cupolaBase: opts.cupolaBase }),
-    columns: (catalog, schema4, table) => buildColumnsJson2(iface, catalog, schema4, table)
-  };
-}
-
 // src/worker-cf-entry.ts
 init_arrow();
 import { OutputCollector as OutputCollector2, AuthContext as AuthContext2 } from "@query-farm/vgi-rpc";
@@ -9683,7 +9299,6 @@ export {
   dateDay2 as dateDay,
   date2 as date,
   createVgiFetch,
-  createLandingDescribe2 as createLandingDescribe,
   columnFromArray,
   buildStatisticsBatch,
   bool2 as bool,
@@ -9712,4 +9327,4 @@ export {
   Arguments
 };
 
-//# debugId=27B541E45B6B151B64756E2164756E21
+//# debugId=3225E125CCD893FA64756E2164756E21