@query-farm/vgi 0.24.1 → 0.25.0

package/dist/http/fetch.d.ts

@@ -1,5 +1,5 @@
 import { type ProtocolConfig } from "../protocol/dispatch.js";
-import type { LandingDescribeProvider } from "./describe-json.js";
+import type { LandingInfo } from "@query-farm/vgi-rpc";
 export interface VgiFetchOptions {
     /** Wire-protocol config (registry + catalogInterface). */
     protocol: Omit<ProtocolConfig, "recoverExchangeState">;
@@ -21,10 +21,10 @@ export interface VgiFetchOptions {
     /** Public source-repository URL, surfaced on the landing page. */
     repositoryUrl?: string;
     /** Enables the standardized VGI landing surface: `GET /` serves the shared
-     *  landing.html, `GET /describe.json` the catalog contract, and
-     *  `GET /describe/{catalog}/{schema}/{table}.json` lazy per-object columns.
-     *  Build one with `createLandingDescribe(catalogInterface, workerInfo)`. */
-    landingDescribe?: LandingDescribeProvider;
+     *  landing.html plus a JSON status document carrying this identity, and
+     *  `GET /vgi-client.js` serves the browser client build the page reads the
+     *  catalog with. */
+    landingInfo?: LandingInfo;
 }
 /**
  * Build a fetch handler suitable for `export default { fetch }` in a CF

package/dist/index.core.d.ts

@@ -24,9 +24,9 @@ export { metadatasToArrow, arrowToMetadatas } from "./metadata/serialize.js";
 export { PushdownFilters, FilteringOutputCollector, deserializeFilters, buildJoinKeysLookup, ComparisonOp, type Filter, type ConstantFilter, type IsNullFilter, type IsNotNullFilter, type InFilter, type AndFilter, type OrFilter, type StructFilter, formatPushedFilters, reprPushedFilters, } from "./filter-pushdown/index.js";
 export { arrowStateSerializer, EXCHANGE_STATE_SCHEMA, serializeUserState, deserializeUserState } from "./protocol/state-serializer.js";
 export { buildVgiProtocol, type ProtocolConfig } from "./protocol/dispatch.js";
-export { buildColumnsJson, buildDescribeJson, createLandingDescribe, CUPOLA_BASE, LANDING_SCHEMA_VERSION, type LandingDescribeProvider, type WorkerDescribeInfo, } from "./http/describe-json.js";
+export type { LandingInfo } from "@query-farm/vgi-rpc";
 export { BindResultSchema, ScanFunctionResultSchema } from "./generated/vgi-protocol-schemas.js";
-export { type AttachOptionSpec, serializeAttachOptionSpec, serializeAttachOptionSpecs } from "./catalog/attach-option.js";
+export { type AttachOptionSpec, serializeAttachOptionSpec, serializeAttachOptionSpecs, deserializeAttachOptionSpec, deserializeAttachOptionSpecs } from "./catalog/attach-option.js";
 export { type ColumnStatistics, serializeColumnStatistics } from "./util/statistics.js";
 export { toUint8Array } from "./util/bytes.js";
 export { iterRows, batchFromRows, batchFromColumns, emptyBatch, filterBatch, projectSchema, projectBatch, batchToScalarDict, batchToSecretDict, safeNumber, } from "./util/arrow/index.js";

package/dist/index.js

@@ -7022,6 +7022,83 @@ function resolveMetadata(func) {
   };
 }
 
+// src/catalog/attach-option.ts
+init_arrow();
+var SPEC_SCHEMA = schema2([
+  field2("name", utf82(), false),
+  field2("description", utf82(), false),
+  field2("type", binary2(), false),
+  field2("default_value", binary2(), true),
+  field2("required", bool2(), true)
+]);
+function serializeAttachOptionSpec(spec) {
+  if (spec.required && spec.default !== undefined && spec.default !== null) {
+    throw new Error(`Attach option '${spec.name}' is required but also declares a default ` + `(${JSON.stringify(spec.default)}); an option with a default is always ` + "satisfiable without the caller. Drop one.");
+  }
+  const typeSchema = schema2([field2("value", spec.type, true)]);
+  const typeBytes = serializeSchema(typeSchema);
+  let defaultBytes = null;
+  if (spec.default !== undefined && spec.default !== null) {
+    const defaultBatch = batchFromColumns({ value: [spec.default] }, typeSchema);
+    defaultBytes = serializeBatch(defaultBatch);
+  }
+  const batch = batchFromColumns({
+    name: [spec.name],
+    description: [spec.description],
+    type: [typeBytes],
+    default_value: [defaultBytes],
+    required: [spec.required ?? false]
+  }, SPEC_SCHEMA);
+  return serializeBatch(batch);
+}
+function serializeAttachOptionSpecs(specs) {
+  return Array.from(specs, serializeAttachOptionSpec);
+}
+function deserializeAttachOptionSpec(bytes) {
+  const row = [...iterRows(deserializeBatch(bytes))][0];
+  if (row === undefined) {
+    throw new Error("Cannot deserialize AttachOptionSpec from an empty batch");
+  }
+  const typeSchema = deserializeSchema(toUint8Array(row.type));
+  const type = typeSchema.fields[0].type;
+  let value;
+  const defaultBytes = row.default_value;
+  if (defaultBytes != null) {
+    const asBytes = toUint8Array(defaultBytes);
+    if (asBytes.length > 0) {
+      const defaultRow = [...iterRows(deserializeBatch(asBytes))][0];
+      value = defaultRow?.value;
+    }
+  }
+  return {
+    name: String(row.name),
+    description: String(row.description ?? ""),
+    type,
+    ...value === undefined || value === null ? {} : { default: value },
+    required: row.required === true
+  };
+}
+function deserializeAttachOptionSpecs(specs) {
+  return Array.from(specs, deserializeAttachOptionSpec);
+}
+
+class MissingAttachOptionsError extends Error {
+  missing;
+  constructor(catalogName, missing) {
+    const joined = missing.map((name) => `'${name}'`).join(", ");
+    super(`Catalog '${catalogName}' cannot be attached without the required ` + `option${missing.length > 1 ? "s" : ""} ${joined}.`);
+    this.name = "MissingAttachOptionsError";
+    this.missing = missing;
+  }
+}
+function validateRequiredAttachOptions(catalogName, specs, options) {
+  const supplied = new Set(Object.keys(options).map((key) => key.toLowerCase()));
+  const missing = Array.from(specs).filter((spec) => spec.required && !supplied.has(spec.name.toLowerCase())).map((spec) => spec.name);
+  if (missing.length > 0) {
+    throw new MissingAttachOptionsError(catalogName, missing);
+  }
+}
+
 // src/catalog/read-only.ts
 class ReadOnlyCatalogInterface extends CatalogInterface {
   _descriptor;
@@ -7047,6 +7124,9 @@ class ReadOnlyCatalogInterface extends CatalogInterface {
   catalogs() {
     return [this._descriptor.name];
   }
+  attachOptionSpecs(_name) {
+    return [];
+  }
   catalogsInfo() {
     return [{
       name: this._descriptor.name,
@@ -7061,6 +7141,7 @@ class ReadOnlyCatalogInterface extends CatalogInterface {
     if (!this.catalogs().includes(name)) {
       throw new Error(`No worker handles catalog '${name}'`);
     }
+    validateRequiredAttachOptions(name, this.attachOptionSpecs(name), options ?? {});
     const attachOpaqueData = new Uint8Array(16);
     crypto.getRandomValues(attachOpaqueData);
     this._attachments.set(name, attachOpaqueData);
@@ -10452,442 +10533,6 @@ function buildVgiProtocol(config) {
   registerCatalogMethods(protocol, config.catalogInterface, config.catalogName, config.signingKey);
   return protocol;
 }
-// src/http/describe-json.ts
-init_arrow();
-var LANDING_SCHEMA_VERSION = 1;
-var CUPOLA_BASE = "https://cupola.query-farm.services";
-var STRING_TAGS = {
-  title: "vgi.title",
-  doc_md: "vgi.doc_md",
-  source_url: "vgi.source_url",
-  license: "vgi.license",
-  author: "vgi.author",
-  copyright: "vgi.copyright",
-  support_contact: "vgi.support_contact",
-  support_policy_url: "vgi.support_policy_url"
-};
-var KEYWORDS_TAG = "vgi.keywords";
-function typeToString(type) {
-  if (isNull(type))
-    return "NULL";
-  if (isUtf8(type))
-    return "VARCHAR";
-  if (isBinary(type))
-    return "BLOB";
-  if (isBool(type))
-    return "BOOLEAN";
-  if (isInt(type)) {
-    const bw = type.bitWidth ?? 32;
-    return bw === 64 ? "BIGINT" : "INTEGER";
-  }
-  if (isFloat(type)) {
-    const precision = type.precision ?? 2;
-    return precision === 1 ? "FLOAT" : "DOUBLE";
-  }
-  const s = type.toString?.();
-  return s ?? String(type.typeId);
-}
-function arrowTypeString(type) {
-  if (isNull(type))
-    return "null";
-  if (isBool(type))
-    return "bool";
-  if (isUtf8(type))
-    return "string";
-  if (isBinary(type))
-    return "binary";
-  if (isInt(type)) {
-    const bw = type.bitWidth ?? 32;
-    const signed = type.isSigned ?? type.signed ?? true;
-    return `${signed ? "" : "u"}int${bw}`;
-  }
-  if (isFloat(type)) {
-    const precision = type.precision ?? 2;
-    return precision === 0 ? "halffloat" : precision === 1 ? "float" : "double";
-  }
-  const s = type.toString?.();
-  return (s ?? String(type.typeId)).toLowerCase();
-}
-function fieldMeta(field3, key) {
-  return field3.metadata.get(key);
-}
-function fieldToColumn(field3) {
-  const col = { name: field3.name, type: typeToString(field3.type) };
-  const comment = fieldMeta(field3, "comment") ?? fieldMeta(field3, VGI_DOC_KEY);
-  if (comment)
-    col.comment = comment;
-  return col;
-}
-function readSchemaSafe(bytes) {
-  if (!bytes || bytes.length === 0)
-    return null;
-  try {
-    return deserializeSchema(bytes);
-  } catch {
-    return null;
-  }
-}
-function functionDisplayType(fn) {
-  const t = String(fn.function_type).toUpperCase();
-  if (t === "SCALAR")
-    return "scalar";
-  if (t === "AGGREGATE")
-    return "aggregate";
-  if (fn.has_finalize)
-    return "table_in_out";
-  return "table";
-}
-function functionReturns(fn) {
-  const schema3 = readSchemaSafe(fn.output_schema);
-  if (schema3 === null || schema3.fields.length === 0)
-    return;
-  const t = String(fn.function_type).toUpperCase();
-  if (t === "SCALAR" || t === "AGGREGATE") {
-    return typeToString(schema3.fields[0].type);
-  }
-  const cols = schema3.fields.map((f) => `${f.name} ${typeToString(f.type)}`).join(", ");
-  return `TABLE(${cols})`;
-}
-function functionArgs(fn) {
-  const schema3 = readSchemaSafe(fn.arguments);
-  if (schema3 === null)
-    return [];
-  const args = [];
-  for (const field3 of schema3.fields) {
-    if (fieldMeta(field3, VGI_TYPE_KEY) === VGI_TYPE_TABLE)
-      continue;
-    const arg = { name: field3.name, type: typeToString(field3.type) };
-    if (fieldMeta(field3, VGI_ARG_KEY) === VGI_ARG_NAMED)
-      arg.named = true;
-    const doc = fieldMeta(field3, VGI_DOC_KEY);
-    if (doc)
-      arg.desc = doc;
-    const rawDefault = fieldMeta(field3, VGI_DEFAULT_KEY);
-    if (rawDefault !== undefined) {
-      try {
-        arg.default = JSON.stringify(JSON.parse(rawDefault));
-      } catch {
-        arg.default = rawDefault;
-      }
-    }
-    args.push(arg);
-  }
-  return args;
-}
-function macroDisplayType(m) {
-  return String(m.macro_type).toUpperCase() === "SCALAR" ? "scalar" : "table";
-}
-function macroDefaultJson(value) {
-  if (typeof value === "bigint")
-    return value.toString();
-  try {
-    return JSON.stringify(value);
-  } catch {
-    return String(value);
-  }
-}
-function macroArgs(m) {
-  const defaults = new Map;
-  const defBytes = m.parameter_default_values;
-  if (defBytes && defBytes.length > 0) {
-    try {
-      const batch = deserializeBatch(defBytes);
-      for (const field3 of batch.schema.fields) {
-        try {
-          defaults.set(field3.name, batch.getChild(field3.name)?.get(0));
-        } catch {}
-      }
-    } catch {}
-  }
-  const schema3 = readSchemaSafe(m.arguments_schema);
-  const fields = schema3 ? schema3.fields : null;
-  const fieldByName = new Map;
-  if (fields)
-    for (const f of fields)
-      fieldByName.set(f.name, f);
-  const names = fields ? fields.map((f) => f.name) : m.parameters;
-  const args = [];
-  for (const name of names) {
-    const field3 = fieldByName.get(name);
-    const arg = field3 !== undefined && !isNull(field3.type) ? { name, type: arrowTypeString(field3.type) } : { name, type: "ANY" };
-    const doc = field3 !== undefined ? fieldMeta(field3, VGI_DOC_KEY) : undefined;
-    if (doc)
-      arg.desc = doc;
-    if (defaults.has(name)) {
-      arg.named = true;
-      arg.default = macroDefaultJson(defaults.get(name));
-    }
-    args.push(arg);
-  }
-  return args;
-}
-function macroToObject(m) {
-  return {
-    name: m.name,
-    type: macroDisplayType(m),
-    doc: m.comment ?? "",
-    args: macroArgs(m)
-  };
-}
-async function schemaContents(iface, aod, name, kind) {
-  try {
-    if (kind === "TABLE") {
-      return await iface.schemaContentsTables(aod, name);
-    }
-    return await iface.schemaContentsViews(aod, name);
-  } catch {
-    return [];
-  }
-}
-async function schemaFunctions(iface, aod, name) {
-  const out = [];
-  for (const kind of ["SCALAR_FUNCTION", "TABLE_FUNCTION", "AGGREGATE_FUNCTION"]) {
-    try {
-      out.push(...await iface.schemaContentsFunctions(aod, name, kind));
-    } catch {}
-  }
-  return out;
-}
-async function schemaMacros(iface, aod, name) {
-  const out = [];
-  for (const kind of ["SCALAR_MACRO", "TABLE_MACRO"]) {
-    try {
-      out.push(...await iface.schemaContentsMacros(aod, name, kind));
-    } catch {}
-  }
-  return out;
-}
-async function buildSchemas(iface, aod) {
-  let schemaInfos;
-  try {
-    schemaInfos = await iface.schemas(aod);
-  } catch {
-    return { schemas: [], counts: { schemas: 0, tables: 0, views: 0, functions: 0 } };
-  }
-  const schemas = [];
-  const counts = { schemas: 0, tables: 0, views: 0, functions: 0 };
-  for (const si of schemaInfos) {
-    const tablesRaw = await schemaContents(iface, aod, si.name, "TABLE");
-    const viewsRaw = await schemaContents(iface, aod, si.name, "VIEW");
-    const funcsRaw = await schemaFunctions(iface, aod, si.name);
-    const macrosRaw = await schemaMacros(iface, aod, si.name);
-    const tables = tablesRaw.map((t) => {
-      const schema3 = readSchemaSafe(t.columns);
-      return { name: t.name, cols: schema3 ? schema3.fields.length : 0, comment: t.comment ?? "" };
-    });
-    const views = viewsRaw.map((v) => ({
-      name: v.name,
-      cols: Object.keys(v.column_comments ?? {}).length,
-      comment: v.comment ?? "",
-      def: v.definition
-    }));
-    const functions = funcsRaw.map((fn) => {
-      const obj = {
-        name: fn.name,
-        type: functionDisplayType(fn),
-        doc: fn.description ?? "",
-        args: functionArgs(fn)
-      };
-      const ret = functionReturns(fn);
-      if (ret)
-        obj.returns = ret;
-      return obj;
-    });
-    functions.push(...macrosRaw.map((m) => macroToObject(m)));
-    functions.sort((a, b) => String(a.type).localeCompare(String(b.type)) || String(a.name).localeCompare(String(b.name)));
-    const schemaObj = { name: si.name };
-    if (si.comment)
-      schemaObj.doc = si.comment;
-    schemaObj.tables = tables;
-    schemaObj.views = views;
-    schemaObj.functions = functions;
-    schemas.push(schemaObj);
-    counts.schemas += 1;
-    counts.tables += tables.length;
-    counts.views += views.length;
-    counts.functions += functions.length;
-  }
-  return { schemas, counts };
-}
-function catalogTags(result) {
-  const tags = result?.tags;
-  if (!tags)
-    return {};
-  const out = {};
-  for (const [key, tag] of Object.entries(STRING_TAGS)) {
-    const val = tags[tag];
-    if (val)
-      out[key] = val;
-  }
-  const kw = tags[KEYWORDS_TAG];
-  if (kw) {
-    try {
-      const parsed = JSON.parse(kw);
-      if (Array.isArray(parsed))
-        out.keywords = parsed.map((k) => String(k));
-    } catch {}
-  }
-  return out;
-}
-function decodeAttachOption(bytes) {
-  try {
-    const batch = deserializeBatch(bytes);
-    if (batch.numRows === 0)
-      return null;
-    const name = String(batch.getChild("name")?.get(0) ?? "");
-    const description = String(batch.getChild("description")?.get(0) ?? "");
-    const typeBytes = batch.getChild("type")?.get(0);
-    let type = "";
-    const typeSchema = readSchemaSafe(typeBytes ?? null);
-    if (typeSchema && typeSchema.fields.length > 0)
-      type = typeToString(typeSchema.fields[0].type);
-    let defaultStr = "";
-    const defBytes = batch.getChild("default_value")?.get(0);
-    if (defBytes) {
-      try {
-        const defBatch = deserializeBatch(defBytes);
-        if (defBatch.numRows > 0) {
-          const v = defBatch.getChild("value")?.get(0);
-          defaultStr = v == null ? "" : String(v);
-        }
-      } catch {}
-    }
-    return { name, type, default: defaultStr, description };
-  } catch {
-    return null;
-  }
-}
-async function attachCatalog(iface, name) {
-  try {
-    return await iface.attach(name, {}, null, null);
-  } catch {
-    return null;
-  }
-}
-async function buildCatalog(iface, name, info) {
-  const result = await attachCatalog(iface, name);
-  const attachOptions = (info?.attach_option_specs ?? []).map((b) => decodeAttachOption(b)).filter((o) => o !== null);
-  const dataVersions = (info?.releases ?? []).map((r) => r.summary ? { spec: r.version, label: r.summary } : { spec: r.version });
-  let impl = info?.implementation_version ?? null;
-  let dvs = info?.data_version_spec ?? null;
-  if (result) {
-    impl = result.resolved_implementation_version ?? impl;
-    dvs = result.resolved_data_version ?? dvs;
-  }
-  let schemas = [];
-  let counts = { schemas: 0, tables: 0, views: 0, functions: 0 };
-  if (result) {
-    const built = await buildSchemas(iface, result.attach_opaque_data);
-    schemas = built.schemas;
-    counts = built.counts;
-  }
-  return {
-    name,
-    implementation_version: impl,
-    data_version_spec: dvs,
-    data_versions: dataVersions,
-    attach_options: attachOptions,
-    tags: catalogTags(result),
-    counts,
-    schemas
-  };
-}
-async function catalogInfoByName(iface) {
-  const map3 = new Map;
-  if (typeof iface.catalogsInfo === "function") {
-    try {
-      const infos = await iface.catalogsInfo();
-      for (const info of infos)
-        map3.set(info.name, info);
-    } catch {}
-  }
-  return map3;
-}
-async function buildDescribeJson(iface, worker, opts = {}) {
-  const infos = await catalogInfoByName(iface);
-  let names;
-  try {
-    names = iface.catalogs();
-  } catch {
-    names = [];
-  }
-  const catalogs = [];
-  for (const name of names) {
-    catalogs.push(await buildCatalog(iface, name, infos.get(name)));
-  }
-  return {
-    landing_schema_version: LANDING_SCHEMA_VERSION,
-    worker: {
-      name: worker.name,
-      doc: worker.doc ?? "",
-      version: worker.version ?? "unknown",
-      lang: "typescript"
-    },
-    server_id: opts.serverId ?? "",
-    oauth: opts.oauth ?? false,
-    cupola_base: opts.cupolaBase ?? CUPOLA_BASE,
-    catalogs
-  };
-}
-async function buildColumnsJson(iface, catalog, schema3, table) {
-  const result = await attachCatalog(iface, catalog);
-  if (!result)
-    return null;
-  const aod = result.attach_opaque_data;
-  const tables = await schemaContents(iface, aod, schema3, "TABLE");
-  for (const t of tables) {
-    if (t.name === table) {
-      const arrow = readSchemaSafe(t.columns);
-      return { columns: (arrow?.fields ?? []).map((f) => fieldToColumn(f)) };
-    }
-  }
-  const views = await schemaContents(iface, aod, schema3, "VIEW");
-  for (const v of views) {
-    if (v.name === table) {
-      return {
-        columns: Object.entries(v.column_comments ?? {}).map(([name, comment]) => ({
-          name,
-          type: "",
-          comment
-        }))
-      };
-    }
-  }
-  return null;
-}
-function createLandingDescribe(iface, worker, opts = {}) {
-  return {
-    describe: (ctx) => buildDescribeJson(iface, worker, { ...ctx, cupolaBase: opts.cupolaBase }),
-    columns: (catalog, schema3, table) => buildColumnsJson(iface, catalog, schema3, table)
-  };
-}
-// src/catalog/attach-option.ts
-init_arrow();
-var SPEC_SCHEMA = schema2([
-  field2("name", utf82(), false),
-  field2("description", utf82(), false),
-  field2("type", binary2(), false),
-  field2("default_value", binary2(), true)
-]);
-function serializeAttachOptionSpec(spec) {
-  const typeSchema = schema2([field2("value", spec.type, true)]);
-  const typeBytes = serializeSchema(typeSchema);
-  let defaultBytes = null;
-  if (spec.default !== undefined && spec.default !== null) {
-    const defaultBatch = batchFromColumns({ value: [spec.default] }, typeSchema);
-    defaultBytes = serializeBatch(defaultBatch);
-  }
-  const batch = batchFromColumns({
-    name: [spec.name],
-    description: [spec.description],
-    type: [typeBytes],
-    default_value: [defaultBytes]
-  }, SPEC_SCHEMA);
-  return serializeBatch(batch);
-}
-function serializeAttachOptionSpecs(specs) {
-  return Array.from(specs, serializeAttachOptionSpec);
-}
 // src/secrets/helpers.ts
 function scalarStr(v) {
   if (v === null || v === undefined)
@@ -12045,6 +11690,8 @@ export {
   deserializeSchema,
   deserializeFilters,
   deserializeBatch,
+  deserializeAttachOptionSpecs,
+  deserializeAttachOptionSpec,
   denseUnion2 as denseUnion,
   defineTableInOutFunction,
   defineTableFunction,
@@ -12067,7 +11714,6 @@ export {
   dateMillisecond2 as dateMillisecond,
   dateDay2 as dateDay,
   date2 as date,
-  createLandingDescribe,
   constraintSpecFields,
   columnFromArray,
   codecFor,
@@ -12076,8 +11722,6 @@ export {
   buildVgiProtocol,
   buildScanBranchesResult,
   buildJoinKeysLookup,
-  buildDescribeJson,
-  buildColumnsJson,
   bool3 as bool,
   binary2 as binary,
   batchToSecretDict,
@@ -12125,7 +11769,6 @@ export {
   OrderDependence,
   NullHandling,
   NoCatalogError,
-  LANDING_SCHEMA_VERSION,
   IntervalUnit2 as IntervalUnit,
   GROUP_COLUMN_NAME,
   FunctionType,
@@ -12146,7 +11789,6 @@ export {
   CatalogInterface,
   CatalogFunctionType,
   CatalogAlreadyExistsError,
-  CUPOLA_BASE,
   CACHE_TTL_KEY,
   CACHE_STALE_WHILE_REVALIDATE_KEY,
   CACHE_STALE_IF_ERROR_KEY,
@@ -12170,4 +11812,4 @@ export {
   ArgumentValidationError
 };
 
-//# debugId=F2B3478BD764EF8C64756E2164756E21
+//# debugId=DE208C45586A782664756E2164756E21