@query-farm/vgi 0.14.0 → 0.17.0

package/dist/worker-cf-entry.d.ts

@@ -1,28 +1,10 @@
-import { type ProtocolConfig } from "./protocol/dispatch.js";
-export interface VgiFetchOptions {
-    /** Wire-protocol config (registry + catalogInterface). */
-    protocol: Omit<ProtocolConfig, "recoverExchangeState">;
-    /** HMAC key for state-token signing. Pass a stable, secret 32-byte key
-     *  (e.g. derived from a Wrangler secret). Required because Workers don't
-     *  preserve in-memory state across requests/instances. */
-    signingKey: Uint8Array;
-    /** State-token TTL in seconds (default 3600). */
-    tokenTtl?: number;
-    /** URL path prefix for VGI requests (default "/vgi"). */
-    prefix?: string;
-    /** Server ID for state-token attribution (default "vgi-cf"). */
-    serverId?: string;
-}
-/**
- * Build a fetch handler suitable for `export default { fetch }` in a CF
- * Worker module. The returned handler is async-safe across Workers'
- * isolate-per-request execution model — all state round-trips through the
- * signed state token created here.
- */
-export declare function createVgiFetch(opts: VgiFetchOptions): (req: Request) => Promise<Response>;
+export { createVgiFetch } from "./http/fetch.js";
+export type { VgiFetchOptions } from "./http/fetch.js";
 export { defineScalarFunction, defineTableFunction, defineAggregate, defineTableInOutFunction, FunctionRegistry, ReadOnlyCatalogInterface, CompositeCatalogInterface, CatalogInterface, Arguments, FunctionType, serializeAttachOptionSpec, serializeAttachOptionSpecs, serializeColumnStatistics, BindResultSchema, ScanFunctionResultSchema, } from "./index.core.js";
 export type { AttachOptionSpec, ColumnStatistics, CatalogAttachResult, CatalogDescriptor, CatalogInfo, TableInfo, AttachOpaqueData, TransactionOpaqueData, SchemaInfo, VgiFunction, TableProcessParams, } from "./index.core.js";
 export type { ProtocolConfig } from "./protocol/dispatch.js";
+export { createLandingDescribe } from "./http/describe-json.js";
+export type { LandingDescribeProvider, WorkerDescribeInfo } from "./http/describe-json.js";
 export { TypeId, isNull, isInt, isFloat, isBinary, isUtf8, isBool, isDecimal, isDate, isTime, isTimestamp, isInterval, isList, isStruct, isUnion, isFixedSizeBinary, isFixedSizeList, isMap, isDuration, isDictionary, isBatch, backend, schema, field, nullType, bool, int, int8, int16, int32, int64, uint8, uint16, uint32, uint64, float16, float32, float64, utf8, binary, fixedSizeBinary, decimal, decimal128, decimal256, date, dateDay, dateMillisecond, time, timeSecond, timeMillisecond, timeMicrosecond, timeNanosecond, timestamp, duration, interval, list, fixedSizeList, struct, map, dictionary, union, sparseUnion, denseUnion, TimeUnit, DateUnit, IntervalUnit, UnionMode, emptyBatch, batchFromRows, batchFromColumns, columnFromArray, iterRows, batchToScalarDict, batchToSecretDict, safeNumber, decodeDictValue, filterBatch, projectSchema, projectBatch, serializeSchema, deserializeSchema, serializeBatch, deserializeBatch, buildStatisticsBatch, } from "./arrow/index.js";
 export type { VgiTypeId, VgiDataType, VgiField, VgiSchema, VgiColumn, VgiColumnData, VgiBatch, VgiBackendInfo, } from "./arrow/index.js";
 export { arrowStateSerializer, EXCHANGE_STATE_SCHEMA } from "./protocol/state-serializer.js";

package/dist/worker-cf-entry.js

@@ -2543,7 +2543,7 @@ var init_storage = __esm(() => {
   });
 });
 
-// src/worker-cf-entry.ts
+// src/http/fetch.ts
 import { createHttpHandler, unpackStateToken } from "@query-farm/vgi-rpc";
 
 // src/protocol/state-serializer.ts
@@ -7182,6 +7182,32 @@ function buildVgiProtocol(config) {
   return protocol;
 }
 
+// src/http/fetch.ts
+function createVgiFetch(opts) {
+  const tokenTtl = opts.tokenTtl ?? 3600;
+  const prefix = opts.prefix ?? "/vgi";
+  const serverId = opts.serverId ?? "vgi-cf";
+  const protocol = buildVgiProtocol({
+    signingKey: opts.signingKey,
+    ...opts.protocol,
+    recoverExchangeState: async (opaqueData) => {
+      const tokenString = new TextDecoder().decode(opaqueData);
+      const unpacked = await unpackStateToken(tokenString, opts.signingKey, tokenTtl, undefined);
+      return arrowStateSerializer.deserialize(unpacked.stateBytes);
+    }
+  });
+  const handler = createHttpHandler(protocol, {
+    prefix,
+    serverId,
+    tokenKey: opts.signingKey,
+    tokenTtl,
+    stateSerializer: arrowStateSerializer,
+    corsOrigins: opts.corsOrigins,
+    repositoryUrl: opts.repositoryUrl,
+    landingDescribe: opts.landingDescribe
+  });
+  return async (req) => handler(req);
+}
 // src/functions/scalar.ts
 init_arrow();
 function defineScalarFunction(config) {
@@ -7340,6 +7366,12 @@ function defineScalarFunction(config) {
 init_arrow();
 init_registry();
 init_storage();
+
+// src/cache-control.ts
+var CACHE_IF_NONE_MATCH_KEY = "vgi.cache.if_none_match";
+var CACHE_IF_MODIFIED_SINCE_KEY = "vgi.cache.if_modified_since";
+
+// src/functions/table.ts
 function base64Decode(s) {
   const bin = globalThis.atob ? globalThis.atob(s) : Buffer.from(s, "base64").toString("binary");
   const out = new Uint8Array(bin.length);
@@ -7538,6 +7570,12 @@ function defineTableFunction(config) {
         onTick: (pState, tickMetadata) => {
           if (!tickMetadata)
             return;
+          const ifNoneMatch = tickMetadata.get(CACHE_IF_NONE_MATCH_KEY);
+          const ifModifiedSince = tickMetadata.get(CACHE_IF_MODIFIED_SINCE_KEY);
+          if (ifNoneMatch !== undefined)
+            pState.processParams.ifNoneMatch = ifNoneMatch;
+          if (ifModifiedSince !== undefined)
+            pState.processParams.ifModifiedSince = ifModifiedSince;
           const encoded = tickMetadata.get("vgi_pushdown_filters");
           if (!encoded)
             return;
@@ -8810,31 +8848,419 @@ function serializeAttachOptionSpec(spec) {
 function serializeAttachOptionSpecs(specs) {
   return Array.from(specs, serializeAttachOptionSpec);
 }
-// src/worker-cf-entry.ts
+// src/http/describe-json.ts
 init_arrow();
-import { OutputCollector as OutputCollector2, AuthContext as AuthContext2 } from "@query-farm/vgi-rpc";
-function createVgiFetch(opts) {
-  const tokenTtl = opts.tokenTtl ?? 3600;
-  const prefix = opts.prefix ?? "/vgi";
-  const serverId = opts.serverId ?? "vgi-cf";
-  const protocol = buildVgiProtocol({
-    signingKey: opts.signingKey,
-    ...opts.protocol,
-    recoverExchangeState: async (opaqueData) => {
-      const tokenString = new TextDecoder().decode(opaqueData);
-      const unpacked = await unpackStateToken(tokenString, opts.signingKey, tokenTtl, undefined);
-      return arrowStateSerializer.deserialize(unpacked.stateBytes);
+var LANDING_SCHEMA_VERSION2 = 1;
+var CUPOLA_BASE2 = "https://cupola.query-farm.services";
+var STRING_TAGS = {
+  title: "vgi.title",
+  doc_md: "vgi.doc_md",
+  source_url: "vgi.source_url",
+  license: "vgi.license",
+  author: "vgi.author",
+  copyright: "vgi.copyright",
+  support_contact: "vgi.support_contact",
+  support_policy_url: "vgi.support_policy_url"
+};
+var KEYWORDS_TAG = "vgi.keywords";
+function typeToString(type) {
+  if (isNull(type))
+    return "NULL";
+  if (isUtf8(type))
+    return "VARCHAR";
+  if (isBinary(type))
+    return "BLOB";
+  if (isBool(type))
+    return "BOOLEAN";
+  if (isInt(type)) {
+    const bw = type.bitWidth ?? 32;
+    return bw === 64 ? "BIGINT" : "INTEGER";
+  }
+  if (isFloat(type)) {
+    const precision = type.precision ?? 2;
+    return precision === 1 ? "FLOAT" : "DOUBLE";
+  }
+  const s = type.toString?.();
+  return s ?? String(type.typeId);
+}
+function arrowTypeString(type) {
+  if (isNull(type))
+    return "null";
+  if (isBool(type))
+    return "bool";
+  if (isUtf8(type))
+    return "string";
+  if (isBinary(type))
+    return "binary";
+  if (isInt(type)) {
+    const bw = type.bitWidth ?? 32;
+    const signed = type.isSigned ?? type.signed ?? true;
+    return `${signed ? "" : "u"}int${bw}`;
+  }
+  if (isFloat(type)) {
+    const precision = type.precision ?? 2;
+    return precision === 0 ? "halffloat" : precision === 1 ? "float" : "double";
+  }
+  const s = type.toString?.();
+  return (s ?? String(type.typeId)).toLowerCase();
+}
+function fieldMeta(field4, key) {
+  return field4.metadata.get(key);
+}
+function fieldToColumn(field4) {
+  const col = { name: field4.name, type: typeToString(field4.type) };
+  const comment = fieldMeta(field4, "comment") ?? fieldMeta(field4, VGI_DOC_KEY);
+  if (comment)
+    col.comment = comment;
+  return col;
+}
+function readSchemaSafe(bytes2) {
+  if (!bytes2 || bytes2.length === 0)
+    return null;
+  try {
+    return deserializeSchema(bytes2);
+  } catch {
+    return null;
+  }
+}
+function functionDisplayType(fn) {
+  const t = String(fn.function_type).toUpperCase();
+  if (t === "SCALAR")
+    return "scalar";
+  if (t === "AGGREGATE")
+    return "aggregate";
+  if (fn.has_finalize)
+    return "table_in_out";
+  return "table";
+}
+function functionReturns(fn) {
+  const schema4 = readSchemaSafe(fn.output_schema);
+  if (schema4 === null || schema4.fields.length === 0)
+    return;
+  const t = String(fn.function_type).toUpperCase();
+  if (t === "SCALAR" || t === "AGGREGATE") {
+    return typeToString(schema4.fields[0].type);
+  }
+  const cols = schema4.fields.map((f) => `${f.name} ${typeToString(f.type)}`).join(", ");
+  return `TABLE(${cols})`;
+}
+function functionArgs(fn) {
+  const schema4 = readSchemaSafe(fn.arguments);
+  if (schema4 === null)
+    return [];
+  const args = [];
+  for (const field4 of schema4.fields) {
+    if (fieldMeta(field4, VGI_TYPE_KEY) === VGI_TYPE_TABLE)
+      continue;
+    const arg = { name: field4.name, type: typeToString(field4.type) };
+    if (fieldMeta(field4, VGI_ARG_KEY) === VGI_ARG_NAMED)
+      arg.named = true;
+    const doc = fieldMeta(field4, VGI_DOC_KEY);
+    if (doc)
+      arg.desc = doc;
+    const rawDefault = fieldMeta(field4, VGI_DEFAULT_KEY);
+    if (rawDefault !== undefined) {
+      try {
+        arg.default = JSON.stringify(JSON.parse(rawDefault));
+      } catch {
+        arg.default = rawDefault;
+      }
     }
-  });
-  const handler = createHttpHandler(protocol, {
-    prefix,
-    serverId,
-    tokenKey: opts.signingKey,
-    tokenTtl,
-    stateSerializer: arrowStateSerializer
-  });
-  return async (req) => handler(req);
+    args.push(arg);
+  }
+  return args;
 }
+function macroDisplayType(m) {
+  return String(m.macro_type).toUpperCase() === "SCALAR" ? "scalar" : "table";
+}
+function macroDefaultJson(value) {
+  if (typeof value === "bigint")
+    return value.toString();
+  try {
+    return JSON.stringify(value);
+  } catch {
+    return String(value);
+  }
+}
+function macroArgs(m) {
+  const defaults = new Map;
+  const defBytes = m.parameter_default_values;
+  if (defBytes && defBytes.length > 0) {
+    try {
+      const batch = deserializeBatch(defBytes);
+      for (const field4 of batch.schema.fields) {
+        try {
+          defaults.set(field4.name, batch.getChild(field4.name)?.get(0));
+        } catch {}
+      }
+    } catch {}
+  }
+  const schema4 = readSchemaSafe(m.arguments_schema);
+  const fields = schema4 ? schema4.fields : null;
+  const fieldByName = new Map;
+  if (fields)
+    for (const f of fields)
+      fieldByName.set(f.name, f);
+  const names = fields ? fields.map((f) => f.name) : m.parameters;
+  const args = [];
+  for (const name of names) {
+    const field4 = fieldByName.get(name);
+    const arg = field4 !== undefined && !isNull(field4.type) ? { name, type: arrowTypeString(field4.type) } : { name, type: "ANY" };
+    const doc = field4 !== undefined ? fieldMeta(field4, VGI_DOC_KEY) : undefined;
+    if (doc)
+      arg.desc = doc;
+    if (defaults.has(name)) {
+      arg.named = true;
+      arg.default = macroDefaultJson(defaults.get(name));
+    }
+    args.push(arg);
+  }
+  return args;
+}
+function macroToObject(m) {
+  return {
+    name: m.name,
+    type: macroDisplayType(m),
+    doc: m.comment ?? "",
+    args: macroArgs(m)
+  };
+}
+async function schemaContents(iface, aod, name, kind) {
+  try {
+    if (kind === "TABLE") {
+      return await iface.schemaContentsTables(aod, name);
+    }
+    return await iface.schemaContentsViews(aod, name);
+  } catch {
+    return [];
+  }
+}
+async function schemaFunctions(iface, aod, name) {
+  const out = [];
+  for (const kind of ["SCALAR_FUNCTION", "TABLE_FUNCTION", "AGGREGATE_FUNCTION"]) {
+    try {
+      out.push(...await iface.schemaContentsFunctions(aod, name, kind));
+    } catch {}
+  }
+  return out;
+}
+async function schemaMacros(iface, aod, name) {
+  const out = [];
+  for (const kind of ["SCALAR_MACRO", "TABLE_MACRO"]) {
+    try {
+      out.push(...await iface.schemaContentsMacros(aod, name, kind));
+    } catch {}
+  }
+  return out;
+}
+async function buildSchemas(iface, aod) {
+  let schemaInfos;
+  try {
+    schemaInfos = await iface.schemas(aod);
+  } catch {
+    return { schemas: [], counts: { schemas: 0, tables: 0, views: 0, functions: 0 } };
+  }
+  const schemas = [];
+  const counts = { schemas: 0, tables: 0, views: 0, functions: 0 };
+  for (const si of schemaInfos) {
+    const tablesRaw = await schemaContents(iface, aod, si.name, "TABLE");
+    const viewsRaw = await schemaContents(iface, aod, si.name, "VIEW");
+    const funcsRaw = await schemaFunctions(iface, aod, si.name);
+    const macrosRaw = await schemaMacros(iface, aod, si.name);
+    const tables = tablesRaw.map((t) => {
+      const schema4 = readSchemaSafe(t.columns);
+      return { name: t.name, cols: schema4 ? schema4.fields.length : 0, comment: t.comment ?? "" };
+    });
+    const views = viewsRaw.map((v) => ({
+      name: v.name,
+      cols: Object.keys(v.column_comments ?? {}).length,
+      comment: v.comment ?? "",
+      def: v.definition
+    }));
+    const functions = funcsRaw.map((fn) => {
+      const obj = {
+        name: fn.name,
+        type: functionDisplayType(fn),
+        doc: fn.description ?? "",
+        args: functionArgs(fn)
+      };
+      const ret = functionReturns(fn);
+      if (ret)
+        obj.returns = ret;
+      return obj;
+    });
+    functions.push(...macrosRaw.map((m) => macroToObject(m)));
+    functions.sort((a, b) => String(a.type).localeCompare(String(b.type)) || String(a.name).localeCompare(String(b.name)));
+    const schemaObj = { name: si.name };
+    if (si.comment)
+      schemaObj.doc = si.comment;
+    schemaObj.tables = tables;
+    schemaObj.views = views;
+    schemaObj.functions = functions;
+    schemas.push(schemaObj);
+    counts.schemas += 1;
+    counts.tables += tables.length;
+    counts.views += views.length;
+    counts.functions += functions.length;
+  }
+  return { schemas, counts };
+}
+function catalogTags(result) {
+  const tags = result?.tags;
+  if (!tags)
+    return {};
+  const out = {};
+  for (const [key, tag] of Object.entries(STRING_TAGS)) {
+    const val = tags[tag];
+    if (val)
+      out[key] = val;
+  }
+  const kw = tags[KEYWORDS_TAG];
+  if (kw) {
+    try {
+      const parsed = JSON.parse(kw);
+      if (Array.isArray(parsed))
+        out.keywords = parsed.map((k) => String(k));
+    } catch {}
+  }
+  return out;
+}
+function decodeAttachOption(bytes2) {
+  try {
+    const batch = deserializeBatch(bytes2);
+    if (batch.numRows === 0)
+      return null;
+    const name = String(batch.getChild("name")?.get(0) ?? "");
+    const description = String(batch.getChild("description")?.get(0) ?? "");
+    const typeBytes = batch.getChild("type")?.get(0);
+    let type = "";
+    const typeSchema = readSchemaSafe(typeBytes ?? null);
+    if (typeSchema && typeSchema.fields.length > 0)
+      type = typeToString(typeSchema.fields[0].type);
+    let defaultStr = "";
+    const defBytes = batch.getChild("default_value")?.get(0);
+    if (defBytes) {
+      try {
+        const defBatch = deserializeBatch(defBytes);
+        if (defBatch.numRows > 0) {
+          const v = defBatch.getChild("value")?.get(0);
+          defaultStr = v == null ? "" : String(v);
+        }
+      } catch {}
+    }
+    return { name, type, default: defaultStr, description };
+  } catch {
+    return null;
+  }
+}
+async function attachCatalog(iface, name) {
+  try {
+    return await iface.attach(name, {}, null, null);
+  } catch {
+    return null;
+  }
+}
+async function buildCatalog(iface, name, info) {
+  const result = await attachCatalog(iface, name);
+  const attachOptions = (info?.attach_option_specs ?? []).map((b) => decodeAttachOption(b)).filter((o) => o !== null);
+  const dataVersions = (info?.releases ?? []).map((r) => r.summary ? { spec: r.version, label: r.summary } : { spec: r.version });
+  let impl = info?.implementation_version ?? null;
+  let dvs = info?.data_version_spec ?? null;
+  if (result) {
+    impl = result.resolved_implementation_version ?? impl;
+    dvs = result.resolved_data_version ?? dvs;
+  }
+  let schemas = [];
+  let counts = { schemas: 0, tables: 0, views: 0, functions: 0 };
+  if (result) {
+    const built = await buildSchemas(iface, result.attach_opaque_data);
+    schemas = built.schemas;
+    counts = built.counts;
+  }
+  return {
+    name,
+    implementation_version: impl,
+    data_version_spec: dvs,
+    data_versions: dataVersions,
+    attach_options: attachOptions,
+    tags: catalogTags(result),
+    counts,
+    schemas
+  };
+}
+async function catalogInfoByName(iface) {
+  const map4 = new Map;
+  if (typeof iface.catalogsInfo === "function") {
+    try {
+      const infos = await iface.catalogsInfo();
+      for (const info of infos)
+        map4.set(info.name, info);
+    } catch {}
+  }
+  return map4;
+}
+async function buildDescribeJson2(iface, worker, opts = {}) {
+  const infos = await catalogInfoByName(iface);
+  let names;
+  try {
+    names = iface.catalogs();
+  } catch {
+    names = [];
+  }
+  const catalogs = [];
+  for (const name of names) {
+    catalogs.push(await buildCatalog(iface, name, infos.get(name)));
+  }
+  return {
+    landing_schema_version: LANDING_SCHEMA_VERSION2,
+    worker: {
+      name: worker.name,
+      doc: worker.doc ?? "",
+      version: worker.version ?? "unknown",
+      lang: "typescript"
+    },
+    server_id: opts.serverId ?? "",
+    oauth: opts.oauth ?? false,
+    cupola_base: opts.cupolaBase ?? CUPOLA_BASE2,
+    catalogs
+  };
+}
+async function buildColumnsJson2(iface, catalog, schema4, table) {
+  const result = await attachCatalog(iface, catalog);
+  if (!result)
+    return null;
+  const aod = result.attach_opaque_data;
+  const tables = await schemaContents(iface, aod, schema4, "TABLE");
+  for (const t of tables) {
+    if (t.name === table) {
+      const arrow = readSchemaSafe(t.columns);
+      return { columns: (arrow?.fields ?? []).map((f) => fieldToColumn(f)) };
+    }
+  }
+  const views = await schemaContents(iface, aod, schema4, "VIEW");
+  for (const v of views) {
+    if (v.name === table) {
+      return {
+        columns: Object.entries(v.column_comments ?? {}).map(([name, comment]) => ({
+          name,
+          type: "",
+          comment
+        }))
+      };
+    }
+  }
+  return null;
+}
+function createLandingDescribe2(iface, worker, opts = {}) {
+  return {
+    describe: (ctx) => buildDescribeJson2(iface, worker, { ...ctx, cupolaBase: opts.cupolaBase }),
+    columns: (catalog, schema4, table) => buildColumnsJson2(iface, catalog, schema4, table)
+  };
+}
+
+// src/worker-cf-entry.ts
+init_arrow();
+import { OutputCollector as OutputCollector2, AuthContext as AuthContext2 } from "@query-farm/vgi-rpc";
 export {
   utf82 as utf8,
   union2 as union,
@@ -8914,6 +9340,7 @@ export {
   dateDay2 as dateDay,
   date2 as date,
   createVgiFetch,
+  createLandingDescribe2 as createLandingDescribe,
   columnFromArray,
   buildStatisticsBatch,
   bool2 as bool,
@@ -8942,4 +9369,4 @@ export {
   Arguments
 };
 
-//# debugId=065F8224F5A26C2864756E2164756E21
+//# debugId=C67D053F70399BE264756E2164756E21