@query-farm/vgi 0.12.0 → 0.13.0

package/dist/http/describe-json.d.ts

@@ -0,0 +1,62 @@
+import type { CatalogInterface } from "../catalog/interface.js";
+export declare const LANDING_SCHEMA_VERSION = 1;
+export declare const CUPOLA_BASE = "https://cupola.query-farm.services";
+/**
+ * Producer for the VGI landing surface, passed to `createHttpHandler`'s
+ * `landingDescribe` option. Declared locally (structurally identical to
+ * `@query-farm/vgi-rpc`'s `LandingDescribeProvider`) so this package does not
+ * hard-couple to a specific vgi-rpc version.
+ */
+export interface LandingDescribeProvider {
+    describe(ctx: {
+        serverId: string;
+        oauth: boolean;
+    }): unknown | Promise<unknown>;
+    columns(catalog: string, schema: string, table: string): unknown | null | Promise<unknown | null>;
+}
+/** Identity of the worker software surfaced in `describe.json` → `worker`. */
+export interface WorkerDescribeInfo {
+    /** Worker name (e.g. class or catalog name). */
+    name: string;
+    /** One-line worker description. */
+    doc?: string;
+    /** Worker software version. */
+    version?: string;
+}
+interface ColumnJson {
+    name: string;
+    type: string;
+    comment?: string;
+}
+/**
+ * Build the full `describe.json` document for a worker's catalog interface.
+ *
+ * `oauth` and `serverId` are supplied by the HTTP server (they depend on runtime
+ * config, not the catalog).
+ */
+export declare function buildDescribeJson(iface: CatalogInterface, worker: WorkerDescribeInfo, opts?: {
+    oauth?: boolean;
+    serverId?: string;
+    cupolaBase?: string;
+}): Promise<Record<string, unknown>>;
+/**
+ * Build the lazy per-object column payload for one table or view.
+ *
+ * Returns `{ columns: [{ name, type, comment? }] }`, or `null` when the object
+ * can't be found. Tables deserialize their Arrow column schema; views expose
+ * their declared column comments (types are only known after binding the SQL,
+ * which the worker does not do here).
+ */
+export declare function buildColumnsJson(iface: CatalogInterface, catalog: string, schema: string, table: string): Promise<{
+    columns: ColumnJson[];
+} | null>;
+/**
+ * Build a {@link LandingDescribeProvider} to pass to `createHttpHandler` from
+ * `@query-farm/vgi-rpc`. Wires the worker's catalog interface into the
+ * standardized landing surface (`GET {prefix}/describe.json` and the lazy
+ * per-object column endpoint).
+ */
+export declare function createLandingDescribe(iface: CatalogInterface, worker: WorkerDescribeInfo, opts?: {
+    cupolaBase?: string;
+}): LandingDescribeProvider;
+export {};

package/dist/index.core.d.ts

@@ -23,6 +23,7 @@ export { metadatasToArrow, arrowToMetadatas } from "./metadata/serialize.js";
 export { PushdownFilters, FilteringOutputCollector, deserializeFilters, buildJoinKeysLookup, ComparisonOp, type Filter, type ConstantFilter, type IsNullFilter, type IsNotNullFilter, type InFilter, type AndFilter, type OrFilter, type StructFilter, formatPushedFilters, reprPushedFilters, } from "./filter-pushdown/index.js";
 export { arrowStateSerializer, EXCHANGE_STATE_SCHEMA, serializeUserState, deserializeUserState } from "./protocol/state-serializer.js";
 export { buildVgiProtocol, type ProtocolConfig } from "./protocol/dispatch.js";
+export { buildColumnsJson, buildDescribeJson, createLandingDescribe, CUPOLA_BASE, LANDING_SCHEMA_VERSION, type LandingDescribeProvider, type WorkerDescribeInfo, } from "./http/describe-json.js";
 export { BindResultSchema, ScanFunctionResultSchema } from "./generated/vgi-protocol-schemas.js";
 export { type AttachOptionSpec, serializeAttachOptionSpec, serializeAttachOptionSpecs } from "./catalog/attach-option.js";
 export { type ColumnStatistics, serializeColumnStatistics } from "./util/statistics.js";

package/dist/index.js

@@ -9929,6 +9929,321 @@ function buildVgiProtocol(config) {
   registerCatalogMethods(protocol, config.catalogInterface, config.catalogName, config.signingKey);
   return protocol;
 }
+// src/http/describe-json.ts
+init_arrow();
+var LANDING_SCHEMA_VERSION = 1;
+var CUPOLA_BASE = "https://cupola.query-farm.services";
+var STRING_TAGS = {
+  title: "vgi.title",
+  doc_md: "vgi.doc_md",
+  source_url: "vgi.source_url",
+  license: "vgi.license",
+  author: "vgi.author",
+  copyright: "vgi.copyright",
+  support_contact: "vgi.support_contact",
+  support_policy_url: "vgi.support_policy_url"
+};
+var KEYWORDS_TAG = "vgi.keywords";
+function typeToString(type) {
+  if (isNull(type))
+    return "NULL";
+  if (isUtf8(type))
+    return "VARCHAR";
+  if (isBinary(type))
+    return "BLOB";
+  if (isBool(type))
+    return "BOOLEAN";
+  if (isInt(type)) {
+    const bw = type.bitWidth ?? 32;
+    return bw === 64 ? "BIGINT" : "INTEGER";
+  }
+  if (isFloat(type)) {
+    const precision = type.precision ?? 2;
+    return precision === 1 ? "FLOAT" : "DOUBLE";
+  }
+  const s = type.toString?.();
+  return s ?? String(type.typeId);
+}
+function fieldMeta(field3, key) {
+  return field3.metadata.get(key);
+}
+function fieldToColumn(field3) {
+  const col = { name: field3.name, type: typeToString(field3.type) };
+  const comment = fieldMeta(field3, "comment") ?? fieldMeta(field3, VGI_DOC_KEY);
+  if (comment)
+    col.comment = comment;
+  return col;
+}
+function readSchemaSafe(bytes) {
+  if (!bytes || bytes.length === 0)
+    return null;
+  try {
+    return deserializeSchema(bytes);
+  } catch {
+    return null;
+  }
+}
+function functionDisplayType(fn) {
+  const t = String(fn.function_type).toUpperCase();
+  if (t === "SCALAR")
+    return "scalar";
+  if (t === "AGGREGATE")
+    return "aggregate";
+  if (fn.has_finalize)
+    return "table_in_out";
+  return "table";
+}
+function functionReturns(fn) {
+  const schema3 = readSchemaSafe(fn.output_schema);
+  if (schema3 === null || schema3.fields.length === 0)
+    return;
+  const t = String(fn.function_type).toUpperCase();
+  if (t === "SCALAR" || t === "AGGREGATE") {
+    return typeToString(schema3.fields[0].type);
+  }
+  const cols = schema3.fields.map((f) => `${f.name} ${typeToString(f.type)}`).join(", ");
+  return `TABLE(${cols})`;
+}
+function functionArgs(fn) {
+  const schema3 = readSchemaSafe(fn.arguments);
+  if (schema3 === null)
+    return [];
+  const args = [];
+  for (const field3 of schema3.fields) {
+    if (fieldMeta(field3, VGI_TYPE_KEY) === VGI_TYPE_TABLE)
+      continue;
+    const arg = { name: field3.name, type: typeToString(field3.type) };
+    if (fieldMeta(field3, VGI_ARG_KEY) === VGI_ARG_NAMED)
+      arg.named = true;
+    const doc = fieldMeta(field3, VGI_DOC_KEY);
+    if (doc)
+      arg.desc = doc;
+    const rawDefault = fieldMeta(field3, VGI_DEFAULT_KEY);
+    if (rawDefault !== undefined) {
+      try {
+        arg.default = JSON.stringify(JSON.parse(rawDefault));
+      } catch {
+        arg.default = rawDefault;
+      }
+    }
+    args.push(arg);
+  }
+  return args;
+}
+async function schemaContents(iface, aod, name, kind) {
+  try {
+    if (kind === "TABLE") {
+      return await iface.schemaContentsTables(aod, name);
+    }
+    return await iface.schemaContentsViews(aod, name);
+  } catch {
+    return [];
+  }
+}
+async function schemaFunctions(iface, aod, name) {
+  const out = [];
+  for (const kind of ["SCALAR_FUNCTION", "TABLE_FUNCTION", "AGGREGATE_FUNCTION"]) {
+    try {
+      out.push(...await iface.schemaContentsFunctions(aod, name, kind));
+    } catch {}
+  }
+  return out;
+}
+async function buildSchemas(iface, aod) {
+  let schemaInfos;
+  try {
+    schemaInfos = await iface.schemas(aod);
+  } catch {
+    return { schemas: [], counts: { schemas: 0, tables: 0, views: 0, functions: 0 } };
+  }
+  const schemas = [];
+  const counts = { schemas: 0, tables: 0, views: 0, functions: 0 };
+  for (const si of schemaInfos) {
+    const tablesRaw = await schemaContents(iface, aod, si.name, "TABLE");
+    const viewsRaw = await schemaContents(iface, aod, si.name, "VIEW");
+    const funcsRaw = await schemaFunctions(iface, aod, si.name);
+    const tables = tablesRaw.map((t) => {
+      const schema3 = readSchemaSafe(t.columns);
+      return { name: t.name, cols: schema3 ? schema3.fields.length : 0, comment: t.comment ?? "" };
+    });
+    const views = viewsRaw.map((v) => ({
+      name: v.name,
+      cols: Object.keys(v.column_comments ?? {}).length,
+      comment: v.comment ?? "",
+      def: v.definition
+    }));
+    const functions = funcsRaw.slice().sort((a, b) => String(a.function_type).localeCompare(String(b.function_type)) || a.name.localeCompare(b.name)).map((fn) => {
+      const obj = {
+        name: fn.name,
+        type: functionDisplayType(fn),
+        doc: fn.description ?? "",
+        args: functionArgs(fn)
+      };
+      const ret = functionReturns(fn);
+      if (ret)
+        obj.returns = ret;
+      return obj;
+    });
+    schemas.push({ name: si.name, tables, views, functions });
+    counts.schemas += 1;
+    counts.tables += tables.length;
+    counts.views += views.length;
+    counts.functions += functions.length;
+  }
+  return { schemas, counts };
+}
+function catalogTags(result) {
+  const tags = result?.tags;
+  if (!tags)
+    return {};
+  const out = {};
+  for (const [key, tag] of Object.entries(STRING_TAGS)) {
+    const val = tags[tag];
+    if (val)
+      out[key] = val;
+  }
+  const kw = tags[KEYWORDS_TAG];
+  if (kw) {
+    try {
+      const parsed = JSON.parse(kw);
+      if (Array.isArray(parsed))
+        out.keywords = parsed.map((k) => String(k));
+    } catch {}
+  }
+  return out;
+}
+function decodeAttachOption(bytes) {
+  try {
+    const batch = deserializeBatch(bytes);
+    if (batch.numRows === 0)
+      return null;
+    const name = String(batch.getChild("name")?.get(0) ?? "");
+    const description = String(batch.getChild("description")?.get(0) ?? "");
+    const typeBytes = batch.getChild("type")?.get(0);
+    let type = "";
+    const typeSchema = readSchemaSafe(typeBytes ?? null);
+    if (typeSchema && typeSchema.fields.length > 0)
+      type = typeToString(typeSchema.fields[0].type);
+    let defaultStr = "";
+    const defBytes = batch.getChild("default_value")?.get(0);
+    if (defBytes) {
+      try {
+        const defBatch = deserializeBatch(defBytes);
+        if (defBatch.numRows > 0) {
+          const v = defBatch.getChild("value")?.get(0);
+          defaultStr = v == null ? "" : String(v);
+        }
+      } catch {}
+    }
+    return { name, type, default: defaultStr, description };
+  } catch {
+    return null;
+  }
+}
+async function attachCatalog(iface, name) {
+  try {
+    return await iface.attach(name, {}, null, null);
+  } catch {
+    return null;
+  }
+}
+async function buildCatalog(iface, name, info) {
+  const result = await attachCatalog(iface, name);
+  const attachOptions = (info?.attach_option_specs ?? []).map((b) => decodeAttachOption(b)).filter((o) => o !== null);
+  const dataVersions = (info?.releases ?? []).map((r) => r.summary ? { spec: r.version, label: r.summary } : { spec: r.version });
+  let impl = info?.implementation_version ?? null;
+  let dvs = info?.data_version_spec ?? null;
+  if (result) {
+    impl = result.resolved_implementation_version ?? impl;
+    dvs = result.resolved_data_version ?? dvs;
+  }
+  let schemas = [];
+  let counts = { schemas: 0, tables: 0, views: 0, functions: 0 };
+  if (result) {
+    const built = await buildSchemas(iface, result.attach_opaque_data);
+    schemas = built.schemas;
+    counts = built.counts;
+  }
+  return {
+    name,
+    implementation_version: impl,
+    data_version_spec: dvs,
+    data_versions: dataVersions,
+    attach_options: attachOptions,
+    tags: catalogTags(result),
+    counts,
+    schemas
+  };
+}
+async function catalogInfoByName(iface) {
+  const map3 = new Map;
+  if (typeof iface.catalogsInfo === "function") {
+    try {
+      const infos = await iface.catalogsInfo();
+      for (const info of infos)
+        map3.set(info.name, info);
+    } catch {}
+  }
+  return map3;
+}
+async function buildDescribeJson(iface, worker, opts = {}) {
+  const infos = await catalogInfoByName(iface);
+  let names;
+  try {
+    names = iface.catalogs();
+  } catch {
+    names = [];
+  }
+  const catalogs = [];
+  for (const name of names) {
+    catalogs.push(await buildCatalog(iface, name, infos.get(name)));
+  }
+  return {
+    landing_schema_version: LANDING_SCHEMA_VERSION,
+    worker: {
+      name: worker.name,
+      doc: worker.doc ?? "",
+      version: worker.version ?? "unknown",
+      lang: "typescript"
+    },
+    server_id: opts.serverId ?? "",
+    oauth: opts.oauth ?? false,
+    cupola_base: opts.cupolaBase ?? CUPOLA_BASE,
+    catalogs
+  };
+}
+async function buildColumnsJson(iface, catalog, schema3, table) {
+  const result = await attachCatalog(iface, catalog);
+  if (!result)
+    return null;
+  const aod = result.attach_opaque_data;
+  const tables = await schemaContents(iface, aod, schema3, "TABLE");
+  for (const t of tables) {
+    if (t.name === table) {
+      const arrow = readSchemaSafe(t.columns);
+      return { columns: (arrow?.fields ?? []).map((f) => fieldToColumn(f)) };
+    }
+  }
+  const views = await schemaContents(iface, aod, schema3, "VIEW");
+  for (const v of views) {
+    if (v.name === table) {
+      return {
+        columns: Object.entries(v.column_comments ?? {}).map(([name, comment]) => ({
+          name,
+          type: "",
+          comment
+        }))
+      };
+    }
+  }
+  return null;
+}
+function createLandingDescribe(iface, worker, opts = {}) {
+  return {
+    describe: (ctx) => buildDescribeJson(iface, worker, { ...ctx, cupolaBase: opts.cupolaBase }),
+    columns: (catalog, schema3, table) => buildColumnsJson(iface, catalog, schema3, table)
+  };
+}
 // src/catalog/attach-option.ts
 init_arrow();
 var SPEC_SCHEMA = schema2([
@@ -10989,6 +11304,7 @@ export {
   dateMillisecond2 as dateMillisecond,
   dateDay2 as dateDay,
   date2 as date,
+  createLandingDescribe,
   constraintSpecFields,
   columnFromArray,
   codecFor,
@@ -10996,6 +11312,8 @@ export {
   buildVgiProtocol,
   buildScanBranchesResult,
   buildJoinKeysLookup,
+  buildDescribeJson,
+  buildColumnsJson,
   bool3 as bool,
   binary2 as binary,
   batchToSecretDict,
@@ -11041,6 +11359,7 @@ export {
   OrderDependence,
   NullHandling,
   NoCatalogError,
+  LANDING_SCHEMA_VERSION,
   IntervalUnit2 as IntervalUnit,
   GROUP_COLUMN_NAME,
   FunctionType,
@@ -11061,6 +11380,7 @@ export {
   CatalogInterface,
   CatalogFunctionType,
   CatalogAlreadyExistsError,
+  CUPOLA_BASE,
   BoundStorage,
   BindResultSchema,
   AuthContext,
@@ -11068,4 +11388,4 @@ export {
   ArgumentValidationError
 };
 
-//# debugId=606DBE277F7C8D1D64756E2164756E21
+//# debugId=FB551171F4FDA15864756E2164756E21