@query-farm/vgi 0.15.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +53 -0
- package/dist/cache-control.d.ts +90 -0
- package/dist/catalog/descriptors.d.ts +13 -7
- package/dist/client-entry.js +3862 -1
- package/dist/client-entry.js.map +34 -3
- package/dist/functions/table.d.ts +12 -0
- package/dist/generated/vgi-client.d.ts +1 -1
- package/dist/http/fetch.d.ts +35 -0
- package/dist/index.core.d.ts +1 -0
- package/dist/index.js +225 -11
- package/dist/index.js.map +11 -9
- package/dist/serve-entry.d.ts +84 -0
- package/dist/serve-entry.js +6428 -0
- package/dist/serve-entry.js.map +62 -0
- package/dist/worker-cf-entry.d.ts +4 -22
- package/dist/worker-cf-entry.js +466 -34
- package/dist/worker-cf-entry.js.map +12 -9
- package/package.json +18 -6
- package/src/cache-control.ts +133 -0
- package/src/catalog/descriptors.ts +13 -7
- package/src/catalog/read-only.ts +22 -14
- package/src/functions/table.ts +25 -1
- package/src/generated/vgi-client.ts +2 -2
- package/src/generated/vgi-protocol-schemas.ts +2 -2
- package/src/http/fetch.ts +80 -0
- package/src/index.core.ts +21 -0
- package/src/serve-entry.ts +233 -0
- package/src/worker-cf-entry.ts +7 -51
|
@@ -1,28 +1,10 @@
|
|
|
1
|
-
|
|
2
|
-
export
|
|
3
|
-
/** Wire-protocol config (registry + catalogInterface). */
|
|
4
|
-
protocol: Omit<ProtocolConfig, "recoverExchangeState">;
|
|
5
|
-
/** HMAC key for state-token signing. Pass a stable, secret 32-byte key
|
|
6
|
-
* (e.g. derived from a Wrangler secret). Required because Workers don't
|
|
7
|
-
* preserve in-memory state across requests/instances. */
|
|
8
|
-
signingKey: Uint8Array;
|
|
9
|
-
/** State-token TTL in seconds (default 3600). */
|
|
10
|
-
tokenTtl?: number;
|
|
11
|
-
/** URL path prefix for VGI requests (default "/vgi"). */
|
|
12
|
-
prefix?: string;
|
|
13
|
-
/** Server ID for state-token attribution (default "vgi-cf"). */
|
|
14
|
-
serverId?: string;
|
|
15
|
-
}
|
|
16
|
-
/**
|
|
17
|
-
* Build a fetch handler suitable for `export default { fetch }` in a CF
|
|
18
|
-
* Worker module. The returned handler is async-safe across Workers'
|
|
19
|
-
* isolate-per-request execution model — all state round-trips through the
|
|
20
|
-
* signed state token created here.
|
|
21
|
-
*/
|
|
22
|
-
export declare function createVgiFetch(opts: VgiFetchOptions): (req: Request) => Promise<Response>;
|
|
1
|
+
export { createVgiFetch } from "./http/fetch.js";
|
|
2
|
+
export type { VgiFetchOptions } from "./http/fetch.js";
|
|
23
3
|
export { defineScalarFunction, defineTableFunction, defineAggregate, defineTableInOutFunction, FunctionRegistry, ReadOnlyCatalogInterface, CompositeCatalogInterface, CatalogInterface, Arguments, FunctionType, serializeAttachOptionSpec, serializeAttachOptionSpecs, serializeColumnStatistics, BindResultSchema, ScanFunctionResultSchema, } from "./index.core.js";
|
|
24
4
|
export type { AttachOptionSpec, ColumnStatistics, CatalogAttachResult, CatalogDescriptor, CatalogInfo, TableInfo, AttachOpaqueData, TransactionOpaqueData, SchemaInfo, VgiFunction, TableProcessParams, } from "./index.core.js";
|
|
25
5
|
export type { ProtocolConfig } from "./protocol/dispatch.js";
|
|
6
|
+
export { createLandingDescribe } from "./http/describe-json.js";
|
|
7
|
+
export type { LandingDescribeProvider, WorkerDescribeInfo } from "./http/describe-json.js";
|
|
26
8
|
export { TypeId, isNull, isInt, isFloat, isBinary, isUtf8, isBool, isDecimal, isDate, isTime, isTimestamp, isInterval, isList, isStruct, isUnion, isFixedSizeBinary, isFixedSizeList, isMap, isDuration, isDictionary, isBatch, backend, schema, field, nullType, bool, int, int8, int16, int32, int64, uint8, uint16, uint32, uint64, float16, float32, float64, utf8, binary, fixedSizeBinary, decimal, decimal128, decimal256, date, dateDay, dateMillisecond, time, timeSecond, timeMillisecond, timeMicrosecond, timeNanosecond, timestamp, duration, interval, list, fixedSizeList, struct, map, dictionary, union, sparseUnion, denseUnion, TimeUnit, DateUnit, IntervalUnit, UnionMode, emptyBatch, batchFromRows, batchFromColumns, columnFromArray, iterRows, batchToScalarDict, batchToSecretDict, safeNumber, decodeDictValue, filterBatch, projectSchema, projectBatch, serializeSchema, deserializeSchema, serializeBatch, deserializeBatch, buildStatisticsBatch, } from "./arrow/index.js";
|
|
27
9
|
export type { VgiTypeId, VgiDataType, VgiField, VgiSchema, VgiColumn, VgiColumnData, VgiBatch, VgiBackendInfo, } from "./arrow/index.js";
|
|
28
10
|
export { arrowStateSerializer, EXCHANGE_STATE_SCHEMA } from "./protocol/state-serializer.js";
|
package/dist/worker-cf-entry.js
CHANGED
|
@@ -2543,7 +2543,7 @@ var init_storage = __esm(() => {
|
|
|
2543
2543
|
});
|
|
2544
2544
|
});
|
|
2545
2545
|
|
|
2546
|
-
// src/
|
|
2546
|
+
// src/http/fetch.ts
|
|
2547
2547
|
import { createHttpHandler, unpackStateToken } from "@query-farm/vgi-rpc";
|
|
2548
2548
|
|
|
2549
2549
|
// src/protocol/state-serializer.ts
|
|
@@ -3175,7 +3175,7 @@ var TableInfoSchema = schema2([
|
|
|
3175
3175
|
field2("cardinality_max", int642(), false),
|
|
3176
3176
|
field2("column_statistics", binary2(), false),
|
|
3177
3177
|
field2("bind_result", binary2(), false),
|
|
3178
|
-
field2("
|
|
3178
|
+
field2("required_filters", list2(field2("item", list2(field2("item", utf82(), true)), true)), false)
|
|
3179
3179
|
]);
|
|
3180
3180
|
var ViewInfoSchema = schema2([
|
|
3181
3181
|
field2("comment", utf82(), true),
|
|
@@ -7182,6 +7182,32 @@ function buildVgiProtocol(config) {
|
|
|
7182
7182
|
return protocol;
|
|
7183
7183
|
}
|
|
7184
7184
|
|
|
7185
|
+
// src/http/fetch.ts
|
|
7186
|
+
function createVgiFetch(opts) {
|
|
7187
|
+
const tokenTtl = opts.tokenTtl ?? 3600;
|
|
7188
|
+
const prefix = opts.prefix ?? "/vgi";
|
|
7189
|
+
const serverId = opts.serverId ?? "vgi-cf";
|
|
7190
|
+
const protocol = buildVgiProtocol({
|
|
7191
|
+
signingKey: opts.signingKey,
|
|
7192
|
+
...opts.protocol,
|
|
7193
|
+
recoverExchangeState: async (opaqueData) => {
|
|
7194
|
+
const tokenString = new TextDecoder().decode(opaqueData);
|
|
7195
|
+
const unpacked = await unpackStateToken(tokenString, opts.signingKey, tokenTtl, undefined);
|
|
7196
|
+
return arrowStateSerializer.deserialize(unpacked.stateBytes);
|
|
7197
|
+
}
|
|
7198
|
+
});
|
|
7199
|
+
const handler = createHttpHandler(protocol, {
|
|
7200
|
+
prefix,
|
|
7201
|
+
serverId,
|
|
7202
|
+
tokenKey: opts.signingKey,
|
|
7203
|
+
tokenTtl,
|
|
7204
|
+
stateSerializer: arrowStateSerializer,
|
|
7205
|
+
corsOrigins: opts.corsOrigins,
|
|
7206
|
+
repositoryUrl: opts.repositoryUrl,
|
|
7207
|
+
landingDescribe: opts.landingDescribe
|
|
7208
|
+
});
|
|
7209
|
+
return async (req) => handler(req);
|
|
7210
|
+
}
|
|
7185
7211
|
// src/functions/scalar.ts
|
|
7186
7212
|
init_arrow();
|
|
7187
7213
|
function defineScalarFunction(config) {
|
|
@@ -7340,6 +7366,12 @@ function defineScalarFunction(config) {
|
|
|
7340
7366
|
init_arrow();
|
|
7341
7367
|
init_registry();
|
|
7342
7368
|
init_storage();
|
|
7369
|
+
|
|
7370
|
+
// src/cache-control.ts
|
|
7371
|
+
var CACHE_IF_NONE_MATCH_KEY = "vgi.cache.if_none_match";
|
|
7372
|
+
var CACHE_IF_MODIFIED_SINCE_KEY = "vgi.cache.if_modified_since";
|
|
7373
|
+
|
|
7374
|
+
// src/functions/table.ts
|
|
7343
7375
|
function base64Decode(s) {
|
|
7344
7376
|
const bin = globalThis.atob ? globalThis.atob(s) : Buffer.from(s, "base64").toString("binary");
|
|
7345
7377
|
const out = new Uint8Array(bin.length);
|
|
@@ -7538,6 +7570,12 @@ function defineTableFunction(config) {
|
|
|
7538
7570
|
onTick: (pState, tickMetadata) => {
|
|
7539
7571
|
if (!tickMetadata)
|
|
7540
7572
|
return;
|
|
7573
|
+
const ifNoneMatch = tickMetadata.get(CACHE_IF_NONE_MATCH_KEY);
|
|
7574
|
+
const ifModifiedSince = tickMetadata.get(CACHE_IF_MODIFIED_SINCE_KEY);
|
|
7575
|
+
if (ifNoneMatch !== undefined)
|
|
7576
|
+
pState.processParams.ifNoneMatch = ifNoneMatch;
|
|
7577
|
+
if (ifModifiedSince !== undefined)
|
|
7578
|
+
pState.processParams.ifModifiedSince = ifModifiedSince;
|
|
7541
7579
|
const encoded = tickMetadata.get("vgi_pushdown_filters");
|
|
7542
7580
|
if (!encoded)
|
|
7543
7581
|
return;
|
|
@@ -8020,16 +8058,21 @@ class ReadOnlyCatalogInterface extends CatalogInterface {
|
|
|
8020
8058
|
const uniqueIndices = resolveIndexGroups(colSchema, t.unique);
|
|
8021
8059
|
const pkIndices = resolveIndexGroups(colSchema, t.primaryKey);
|
|
8022
8060
|
const fkBytes = serializeForeignKeys(t.foreignKey, name);
|
|
8023
|
-
const
|
|
8024
|
-
if (
|
|
8061
|
+
const requiredFilters = t.requiredFilters ?? [];
|
|
8062
|
+
if (requiredFilters.length > 0) {
|
|
8025
8063
|
const columnNames = new Set(colSchema.fields.map((f) => f.name));
|
|
8026
|
-
for (const
|
|
8027
|
-
if (!
|
|
8028
|
-
throw new Error(`Table '${t.name}':
|
|
8064
|
+
for (const group of requiredFilters) {
|
|
8065
|
+
if (!group || group.length === 0) {
|
|
8066
|
+
throw new Error(`Table '${t.name}': requiredFilters must not contain empty groups`);
|
|
8029
8067
|
}
|
|
8030
|
-
const
|
|
8031
|
-
|
|
8032
|
-
|
|
8068
|
+
for (const path2 of group) {
|
|
8069
|
+
if (!path2) {
|
|
8070
|
+
throw new Error(`Table '${t.name}': requiredFilters must not contain empty strings`);
|
|
8071
|
+
}
|
|
8072
|
+
const head = path2.split(".", 1)[0];
|
|
8073
|
+
if (!columnNames.has(head)) {
|
|
8074
|
+
throw new Error(`Table '${t.name}': requiredFilters path '${path2}' references ` + `unknown column '${head}'. Available columns: ${[...columnNames].sort().join(", ")}`);
|
|
8075
|
+
}
|
|
8033
8076
|
}
|
|
8034
8077
|
}
|
|
8035
8078
|
}
|
|
@@ -8055,7 +8098,7 @@ class ReadOnlyCatalogInterface extends CatalogInterface {
|
|
|
8055
8098
|
delete_function: null,
|
|
8056
8099
|
cardinality_estimate: t.inlinedCardinality ? Number(t.inlinedCardinality.estimate) : null,
|
|
8057
8100
|
cardinality_max: t.inlinedCardinality ? Number(t.inlinedCardinality.max) : null,
|
|
8058
|
-
|
|
8101
|
+
required_filters: requiredFilters
|
|
8059
8102
|
};
|
|
8060
8103
|
}));
|
|
8061
8104
|
}
|
|
@@ -8810,31 +8853,419 @@ function serializeAttachOptionSpec(spec) {
|
|
|
8810
8853
|
function serializeAttachOptionSpecs(specs) {
|
|
8811
8854
|
return Array.from(specs, serializeAttachOptionSpec);
|
|
8812
8855
|
}
|
|
8813
|
-
// src/
|
|
8856
|
+
// src/http/describe-json.ts
|
|
8814
8857
|
init_arrow();
|
|
8815
|
-
|
|
8816
|
-
|
|
8817
|
-
|
|
8818
|
-
|
|
8819
|
-
|
|
8820
|
-
|
|
8821
|
-
|
|
8822
|
-
|
|
8823
|
-
|
|
8824
|
-
|
|
8825
|
-
|
|
8826
|
-
|
|
8858
|
+
var LANDING_SCHEMA_VERSION2 = 1;
|
|
8859
|
+
var CUPOLA_BASE2 = "https://cupola.query-farm.services";
|
|
8860
|
+
var STRING_TAGS = {
|
|
8861
|
+
title: "vgi.title",
|
|
8862
|
+
doc_md: "vgi.doc_md",
|
|
8863
|
+
source_url: "vgi.source_url",
|
|
8864
|
+
license: "vgi.license",
|
|
8865
|
+
author: "vgi.author",
|
|
8866
|
+
copyright: "vgi.copyright",
|
|
8867
|
+
support_contact: "vgi.support_contact",
|
|
8868
|
+
support_policy_url: "vgi.support_policy_url"
|
|
8869
|
+
};
|
|
8870
|
+
var KEYWORDS_TAG = "vgi.keywords";
|
|
8871
|
+
function typeToString(type) {
|
|
8872
|
+
if (isNull(type))
|
|
8873
|
+
return "NULL";
|
|
8874
|
+
if (isUtf8(type))
|
|
8875
|
+
return "VARCHAR";
|
|
8876
|
+
if (isBinary(type))
|
|
8877
|
+
return "BLOB";
|
|
8878
|
+
if (isBool(type))
|
|
8879
|
+
return "BOOLEAN";
|
|
8880
|
+
if (isInt(type)) {
|
|
8881
|
+
const bw = type.bitWidth ?? 32;
|
|
8882
|
+
return bw === 64 ? "BIGINT" : "INTEGER";
|
|
8883
|
+
}
|
|
8884
|
+
if (isFloat(type)) {
|
|
8885
|
+
const precision = type.precision ?? 2;
|
|
8886
|
+
return precision === 1 ? "FLOAT" : "DOUBLE";
|
|
8887
|
+
}
|
|
8888
|
+
const s = type.toString?.();
|
|
8889
|
+
return s ?? String(type.typeId);
|
|
8890
|
+
}
|
|
8891
|
+
function arrowTypeString(type) {
|
|
8892
|
+
if (isNull(type))
|
|
8893
|
+
return "null";
|
|
8894
|
+
if (isBool(type))
|
|
8895
|
+
return "bool";
|
|
8896
|
+
if (isUtf8(type))
|
|
8897
|
+
return "string";
|
|
8898
|
+
if (isBinary(type))
|
|
8899
|
+
return "binary";
|
|
8900
|
+
if (isInt(type)) {
|
|
8901
|
+
const bw = type.bitWidth ?? 32;
|
|
8902
|
+
const signed = type.isSigned ?? type.signed ?? true;
|
|
8903
|
+
return `${signed ? "" : "u"}int${bw}`;
|
|
8904
|
+
}
|
|
8905
|
+
if (isFloat(type)) {
|
|
8906
|
+
const precision = type.precision ?? 2;
|
|
8907
|
+
return precision === 0 ? "halffloat" : precision === 1 ? "float" : "double";
|
|
8908
|
+
}
|
|
8909
|
+
const s = type.toString?.();
|
|
8910
|
+
return (s ?? String(type.typeId)).toLowerCase();
|
|
8911
|
+
}
|
|
8912
|
+
function fieldMeta(field4, key) {
|
|
8913
|
+
return field4.metadata.get(key);
|
|
8914
|
+
}
|
|
8915
|
+
function fieldToColumn(field4) {
|
|
8916
|
+
const col = { name: field4.name, type: typeToString(field4.type) };
|
|
8917
|
+
const comment = fieldMeta(field4, "comment") ?? fieldMeta(field4, VGI_DOC_KEY);
|
|
8918
|
+
if (comment)
|
|
8919
|
+
col.comment = comment;
|
|
8920
|
+
return col;
|
|
8921
|
+
}
|
|
8922
|
+
function readSchemaSafe(bytes2) {
|
|
8923
|
+
if (!bytes2 || bytes2.length === 0)
|
|
8924
|
+
return null;
|
|
8925
|
+
try {
|
|
8926
|
+
return deserializeSchema(bytes2);
|
|
8927
|
+
} catch {
|
|
8928
|
+
return null;
|
|
8929
|
+
}
|
|
8930
|
+
}
|
|
8931
|
+
function functionDisplayType(fn) {
|
|
8932
|
+
const t = String(fn.function_type).toUpperCase();
|
|
8933
|
+
if (t === "SCALAR")
|
|
8934
|
+
return "scalar";
|
|
8935
|
+
if (t === "AGGREGATE")
|
|
8936
|
+
return "aggregate";
|
|
8937
|
+
if (fn.has_finalize)
|
|
8938
|
+
return "table_in_out";
|
|
8939
|
+
return "table";
|
|
8940
|
+
}
|
|
8941
|
+
function functionReturns(fn) {
|
|
8942
|
+
const schema4 = readSchemaSafe(fn.output_schema);
|
|
8943
|
+
if (schema4 === null || schema4.fields.length === 0)
|
|
8944
|
+
return;
|
|
8945
|
+
const t = String(fn.function_type).toUpperCase();
|
|
8946
|
+
if (t === "SCALAR" || t === "AGGREGATE") {
|
|
8947
|
+
return typeToString(schema4.fields[0].type);
|
|
8948
|
+
}
|
|
8949
|
+
const cols = schema4.fields.map((f) => `${f.name} ${typeToString(f.type)}`).join(", ");
|
|
8950
|
+
return `TABLE(${cols})`;
|
|
8951
|
+
}
|
|
8952
|
+
function functionArgs(fn) {
|
|
8953
|
+
const schema4 = readSchemaSafe(fn.arguments);
|
|
8954
|
+
if (schema4 === null)
|
|
8955
|
+
return [];
|
|
8956
|
+
const args = [];
|
|
8957
|
+
for (const field4 of schema4.fields) {
|
|
8958
|
+
if (fieldMeta(field4, VGI_TYPE_KEY) === VGI_TYPE_TABLE)
|
|
8959
|
+
continue;
|
|
8960
|
+
const arg = { name: field4.name, type: typeToString(field4.type) };
|
|
8961
|
+
if (fieldMeta(field4, VGI_ARG_KEY) === VGI_ARG_NAMED)
|
|
8962
|
+
arg.named = true;
|
|
8963
|
+
const doc = fieldMeta(field4, VGI_DOC_KEY);
|
|
8964
|
+
if (doc)
|
|
8965
|
+
arg.desc = doc;
|
|
8966
|
+
const rawDefault = fieldMeta(field4, VGI_DEFAULT_KEY);
|
|
8967
|
+
if (rawDefault !== undefined) {
|
|
8968
|
+
try {
|
|
8969
|
+
arg.default = JSON.stringify(JSON.parse(rawDefault));
|
|
8970
|
+
} catch {
|
|
8971
|
+
arg.default = rawDefault;
|
|
8972
|
+
}
|
|
8827
8973
|
}
|
|
8828
|
-
|
|
8829
|
-
|
|
8830
|
-
|
|
8831
|
-
|
|
8832
|
-
|
|
8833
|
-
|
|
8834
|
-
|
|
8835
|
-
|
|
8836
|
-
|
|
8974
|
+
args.push(arg);
|
|
8975
|
+
}
|
|
8976
|
+
return args;
|
|
8977
|
+
}
|
|
8978
|
+
function macroDisplayType(m) {
|
|
8979
|
+
return String(m.macro_type).toUpperCase() === "SCALAR" ? "scalar" : "table";
|
|
8980
|
+
}
|
|
8981
|
+
function macroDefaultJson(value) {
|
|
8982
|
+
if (typeof value === "bigint")
|
|
8983
|
+
return value.toString();
|
|
8984
|
+
try {
|
|
8985
|
+
return JSON.stringify(value);
|
|
8986
|
+
} catch {
|
|
8987
|
+
return String(value);
|
|
8988
|
+
}
|
|
8989
|
+
}
|
|
8990
|
+
function macroArgs(m) {
|
|
8991
|
+
const defaults = new Map;
|
|
8992
|
+
const defBytes = m.parameter_default_values;
|
|
8993
|
+
if (defBytes && defBytes.length > 0) {
|
|
8994
|
+
try {
|
|
8995
|
+
const batch = deserializeBatch(defBytes);
|
|
8996
|
+
for (const field4 of batch.schema.fields) {
|
|
8997
|
+
try {
|
|
8998
|
+
defaults.set(field4.name, batch.getChild(field4.name)?.get(0));
|
|
8999
|
+
} catch {}
|
|
9000
|
+
}
|
|
9001
|
+
} catch {}
|
|
9002
|
+
}
|
|
9003
|
+
const schema4 = readSchemaSafe(m.arguments_schema);
|
|
9004
|
+
const fields = schema4 ? schema4.fields : null;
|
|
9005
|
+
const fieldByName = new Map;
|
|
9006
|
+
if (fields)
|
|
9007
|
+
for (const f of fields)
|
|
9008
|
+
fieldByName.set(f.name, f);
|
|
9009
|
+
const names = fields ? fields.map((f) => f.name) : m.parameters;
|
|
9010
|
+
const args = [];
|
|
9011
|
+
for (const name of names) {
|
|
9012
|
+
const field4 = fieldByName.get(name);
|
|
9013
|
+
const arg = field4 !== undefined && !isNull(field4.type) ? { name, type: arrowTypeString(field4.type) } : { name, type: "ANY" };
|
|
9014
|
+
const doc = field4 !== undefined ? fieldMeta(field4, VGI_DOC_KEY) : undefined;
|
|
9015
|
+
if (doc)
|
|
9016
|
+
arg.desc = doc;
|
|
9017
|
+
if (defaults.has(name)) {
|
|
9018
|
+
arg.named = true;
|
|
9019
|
+
arg.default = macroDefaultJson(defaults.get(name));
|
|
9020
|
+
}
|
|
9021
|
+
args.push(arg);
|
|
9022
|
+
}
|
|
9023
|
+
return args;
|
|
9024
|
+
}
|
|
9025
|
+
function macroToObject(m) {
|
|
9026
|
+
return {
|
|
9027
|
+
name: m.name,
|
|
9028
|
+
type: macroDisplayType(m),
|
|
9029
|
+
doc: m.comment ?? "",
|
|
9030
|
+
args: macroArgs(m)
|
|
9031
|
+
};
|
|
9032
|
+
}
|
|
9033
|
+
async function schemaContents(iface, aod, name, kind) {
|
|
9034
|
+
try {
|
|
9035
|
+
if (kind === "TABLE") {
|
|
9036
|
+
return await iface.schemaContentsTables(aod, name);
|
|
9037
|
+
}
|
|
9038
|
+
return await iface.schemaContentsViews(aod, name);
|
|
9039
|
+
} catch {
|
|
9040
|
+
return [];
|
|
9041
|
+
}
|
|
8837
9042
|
}
|
|
9043
|
+
async function schemaFunctions(iface, aod, name) {
|
|
9044
|
+
const out = [];
|
|
9045
|
+
for (const kind of ["SCALAR_FUNCTION", "TABLE_FUNCTION", "AGGREGATE_FUNCTION"]) {
|
|
9046
|
+
try {
|
|
9047
|
+
out.push(...await iface.schemaContentsFunctions(aod, name, kind));
|
|
9048
|
+
} catch {}
|
|
9049
|
+
}
|
|
9050
|
+
return out;
|
|
9051
|
+
}
|
|
9052
|
+
async function schemaMacros(iface, aod, name) {
|
|
9053
|
+
const out = [];
|
|
9054
|
+
for (const kind of ["SCALAR_MACRO", "TABLE_MACRO"]) {
|
|
9055
|
+
try {
|
|
9056
|
+
out.push(...await iface.schemaContentsMacros(aod, name, kind));
|
|
9057
|
+
} catch {}
|
|
9058
|
+
}
|
|
9059
|
+
return out;
|
|
9060
|
+
}
|
|
9061
|
+
async function buildSchemas(iface, aod) {
|
|
9062
|
+
let schemaInfos;
|
|
9063
|
+
try {
|
|
9064
|
+
schemaInfos = await iface.schemas(aod);
|
|
9065
|
+
} catch {
|
|
9066
|
+
return { schemas: [], counts: { schemas: 0, tables: 0, views: 0, functions: 0 } };
|
|
9067
|
+
}
|
|
9068
|
+
const schemas = [];
|
|
9069
|
+
const counts = { schemas: 0, tables: 0, views: 0, functions: 0 };
|
|
9070
|
+
for (const si of schemaInfos) {
|
|
9071
|
+
const tablesRaw = await schemaContents(iface, aod, si.name, "TABLE");
|
|
9072
|
+
const viewsRaw = await schemaContents(iface, aod, si.name, "VIEW");
|
|
9073
|
+
const funcsRaw = await schemaFunctions(iface, aod, si.name);
|
|
9074
|
+
const macrosRaw = await schemaMacros(iface, aod, si.name);
|
|
9075
|
+
const tables = tablesRaw.map((t) => {
|
|
9076
|
+
const schema4 = readSchemaSafe(t.columns);
|
|
9077
|
+
return { name: t.name, cols: schema4 ? schema4.fields.length : 0, comment: t.comment ?? "" };
|
|
9078
|
+
});
|
|
9079
|
+
const views = viewsRaw.map((v) => ({
|
|
9080
|
+
name: v.name,
|
|
9081
|
+
cols: Object.keys(v.column_comments ?? {}).length,
|
|
9082
|
+
comment: v.comment ?? "",
|
|
9083
|
+
def: v.definition
|
|
9084
|
+
}));
|
|
9085
|
+
const functions = funcsRaw.map((fn) => {
|
|
9086
|
+
const obj = {
|
|
9087
|
+
name: fn.name,
|
|
9088
|
+
type: functionDisplayType(fn),
|
|
9089
|
+
doc: fn.description ?? "",
|
|
9090
|
+
args: functionArgs(fn)
|
|
9091
|
+
};
|
|
9092
|
+
const ret = functionReturns(fn);
|
|
9093
|
+
if (ret)
|
|
9094
|
+
obj.returns = ret;
|
|
9095
|
+
return obj;
|
|
9096
|
+
});
|
|
9097
|
+
functions.push(...macrosRaw.map((m) => macroToObject(m)));
|
|
9098
|
+
functions.sort((a, b) => String(a.type).localeCompare(String(b.type)) || String(a.name).localeCompare(String(b.name)));
|
|
9099
|
+
const schemaObj = { name: si.name };
|
|
9100
|
+
if (si.comment)
|
|
9101
|
+
schemaObj.doc = si.comment;
|
|
9102
|
+
schemaObj.tables = tables;
|
|
9103
|
+
schemaObj.views = views;
|
|
9104
|
+
schemaObj.functions = functions;
|
|
9105
|
+
schemas.push(schemaObj);
|
|
9106
|
+
counts.schemas += 1;
|
|
9107
|
+
counts.tables += tables.length;
|
|
9108
|
+
counts.views += views.length;
|
|
9109
|
+
counts.functions += functions.length;
|
|
9110
|
+
}
|
|
9111
|
+
return { schemas, counts };
|
|
9112
|
+
}
|
|
9113
|
+
function catalogTags(result) {
|
|
9114
|
+
const tags = result?.tags;
|
|
9115
|
+
if (!tags)
|
|
9116
|
+
return {};
|
|
9117
|
+
const out = {};
|
|
9118
|
+
for (const [key, tag] of Object.entries(STRING_TAGS)) {
|
|
9119
|
+
const val = tags[tag];
|
|
9120
|
+
if (val)
|
|
9121
|
+
out[key] = val;
|
|
9122
|
+
}
|
|
9123
|
+
const kw = tags[KEYWORDS_TAG];
|
|
9124
|
+
if (kw) {
|
|
9125
|
+
try {
|
|
9126
|
+
const parsed = JSON.parse(kw);
|
|
9127
|
+
if (Array.isArray(parsed))
|
|
9128
|
+
out.keywords = parsed.map((k) => String(k));
|
|
9129
|
+
} catch {}
|
|
9130
|
+
}
|
|
9131
|
+
return out;
|
|
9132
|
+
}
|
|
9133
|
+
function decodeAttachOption(bytes2) {
|
|
9134
|
+
try {
|
|
9135
|
+
const batch = deserializeBatch(bytes2);
|
|
9136
|
+
if (batch.numRows === 0)
|
|
9137
|
+
return null;
|
|
9138
|
+
const name = String(batch.getChild("name")?.get(0) ?? "");
|
|
9139
|
+
const description = String(batch.getChild("description")?.get(0) ?? "");
|
|
9140
|
+
const typeBytes = batch.getChild("type")?.get(0);
|
|
9141
|
+
let type = "";
|
|
9142
|
+
const typeSchema = readSchemaSafe(typeBytes ?? null);
|
|
9143
|
+
if (typeSchema && typeSchema.fields.length > 0)
|
|
9144
|
+
type = typeToString(typeSchema.fields[0].type);
|
|
9145
|
+
let defaultStr = "";
|
|
9146
|
+
const defBytes = batch.getChild("default_value")?.get(0);
|
|
9147
|
+
if (defBytes) {
|
|
9148
|
+
try {
|
|
9149
|
+
const defBatch = deserializeBatch(defBytes);
|
|
9150
|
+
if (defBatch.numRows > 0) {
|
|
9151
|
+
const v = defBatch.getChild("value")?.get(0);
|
|
9152
|
+
defaultStr = v == null ? "" : String(v);
|
|
9153
|
+
}
|
|
9154
|
+
} catch {}
|
|
9155
|
+
}
|
|
9156
|
+
return { name, type, default: defaultStr, description };
|
|
9157
|
+
} catch {
|
|
9158
|
+
return null;
|
|
9159
|
+
}
|
|
9160
|
+
}
|
|
9161
|
+
async function attachCatalog(iface, name) {
|
|
9162
|
+
try {
|
|
9163
|
+
return await iface.attach(name, {}, null, null);
|
|
9164
|
+
} catch {
|
|
9165
|
+
return null;
|
|
9166
|
+
}
|
|
9167
|
+
}
|
|
9168
|
+
async function buildCatalog(iface, name, info) {
|
|
9169
|
+
const result = await attachCatalog(iface, name);
|
|
9170
|
+
const attachOptions = (info?.attach_option_specs ?? []).map((b) => decodeAttachOption(b)).filter((o) => o !== null);
|
|
9171
|
+
const dataVersions = (info?.releases ?? []).map((r) => r.summary ? { spec: r.version, label: r.summary } : { spec: r.version });
|
|
9172
|
+
let impl = info?.implementation_version ?? null;
|
|
9173
|
+
let dvs = info?.data_version_spec ?? null;
|
|
9174
|
+
if (result) {
|
|
9175
|
+
impl = result.resolved_implementation_version ?? impl;
|
|
9176
|
+
dvs = result.resolved_data_version ?? dvs;
|
|
9177
|
+
}
|
|
9178
|
+
let schemas = [];
|
|
9179
|
+
let counts = { schemas: 0, tables: 0, views: 0, functions: 0 };
|
|
9180
|
+
if (result) {
|
|
9181
|
+
const built = await buildSchemas(iface, result.attach_opaque_data);
|
|
9182
|
+
schemas = built.schemas;
|
|
9183
|
+
counts = built.counts;
|
|
9184
|
+
}
|
|
9185
|
+
return {
|
|
9186
|
+
name,
|
|
9187
|
+
implementation_version: impl,
|
|
9188
|
+
data_version_spec: dvs,
|
|
9189
|
+
data_versions: dataVersions,
|
|
9190
|
+
attach_options: attachOptions,
|
|
9191
|
+
tags: catalogTags(result),
|
|
9192
|
+
counts,
|
|
9193
|
+
schemas
|
|
9194
|
+
};
|
|
9195
|
+
}
|
|
9196
|
+
async function catalogInfoByName(iface) {
|
|
9197
|
+
const map4 = new Map;
|
|
9198
|
+
if (typeof iface.catalogsInfo === "function") {
|
|
9199
|
+
try {
|
|
9200
|
+
const infos = await iface.catalogsInfo();
|
|
9201
|
+
for (const info of infos)
|
|
9202
|
+
map4.set(info.name, info);
|
|
9203
|
+
} catch {}
|
|
9204
|
+
}
|
|
9205
|
+
return map4;
|
|
9206
|
+
}
|
|
9207
|
+
async function buildDescribeJson2(iface, worker, opts = {}) {
|
|
9208
|
+
const infos = await catalogInfoByName(iface);
|
|
9209
|
+
let names;
|
|
9210
|
+
try {
|
|
9211
|
+
names = iface.catalogs();
|
|
9212
|
+
} catch {
|
|
9213
|
+
names = [];
|
|
9214
|
+
}
|
|
9215
|
+
const catalogs = [];
|
|
9216
|
+
for (const name of names) {
|
|
9217
|
+
catalogs.push(await buildCatalog(iface, name, infos.get(name)));
|
|
9218
|
+
}
|
|
9219
|
+
return {
|
|
9220
|
+
landing_schema_version: LANDING_SCHEMA_VERSION2,
|
|
9221
|
+
worker: {
|
|
9222
|
+
name: worker.name,
|
|
9223
|
+
doc: worker.doc ?? "",
|
|
9224
|
+
version: worker.version ?? "unknown",
|
|
9225
|
+
lang: "typescript"
|
|
9226
|
+
},
|
|
9227
|
+
server_id: opts.serverId ?? "",
|
|
9228
|
+
oauth: opts.oauth ?? false,
|
|
9229
|
+
cupola_base: opts.cupolaBase ?? CUPOLA_BASE2,
|
|
9230
|
+
catalogs
|
|
9231
|
+
};
|
|
9232
|
+
}
|
|
9233
|
+
async function buildColumnsJson2(iface, catalog, schema4, table) {
|
|
9234
|
+
const result = await attachCatalog(iface, catalog);
|
|
9235
|
+
if (!result)
|
|
9236
|
+
return null;
|
|
9237
|
+
const aod = result.attach_opaque_data;
|
|
9238
|
+
const tables = await schemaContents(iface, aod, schema4, "TABLE");
|
|
9239
|
+
for (const t of tables) {
|
|
9240
|
+
if (t.name === table) {
|
|
9241
|
+
const arrow = readSchemaSafe(t.columns);
|
|
9242
|
+
return { columns: (arrow?.fields ?? []).map((f) => fieldToColumn(f)) };
|
|
9243
|
+
}
|
|
9244
|
+
}
|
|
9245
|
+
const views = await schemaContents(iface, aod, schema4, "VIEW");
|
|
9246
|
+
for (const v of views) {
|
|
9247
|
+
if (v.name === table) {
|
|
9248
|
+
return {
|
|
9249
|
+
columns: Object.entries(v.column_comments ?? {}).map(([name, comment]) => ({
|
|
9250
|
+
name,
|
|
9251
|
+
type: "",
|
|
9252
|
+
comment
|
|
9253
|
+
}))
|
|
9254
|
+
};
|
|
9255
|
+
}
|
|
9256
|
+
}
|
|
9257
|
+
return null;
|
|
9258
|
+
}
|
|
9259
|
+
function createLandingDescribe2(iface, worker, opts = {}) {
|
|
9260
|
+
return {
|
|
9261
|
+
describe: (ctx) => buildDescribeJson2(iface, worker, { ...ctx, cupolaBase: opts.cupolaBase }),
|
|
9262
|
+
columns: (catalog, schema4, table) => buildColumnsJson2(iface, catalog, schema4, table)
|
|
9263
|
+
};
|
|
9264
|
+
}
|
|
9265
|
+
|
|
9266
|
+
// src/worker-cf-entry.ts
|
|
9267
|
+
init_arrow();
|
|
9268
|
+
import { OutputCollector as OutputCollector2, AuthContext as AuthContext2 } from "@query-farm/vgi-rpc";
|
|
8838
9269
|
export {
|
|
8839
9270
|
utf82 as utf8,
|
|
8840
9271
|
union2 as union,
|
|
@@ -8914,6 +9345,7 @@ export {
|
|
|
8914
9345
|
dateDay2 as dateDay,
|
|
8915
9346
|
date2 as date,
|
|
8916
9347
|
createVgiFetch,
|
|
9348
|
+
createLandingDescribe2 as createLandingDescribe,
|
|
8917
9349
|
columnFromArray,
|
|
8918
9350
|
buildStatisticsBatch,
|
|
8919
9351
|
bool2 as bool,
|
|
@@ -8942,4 +9374,4 @@ export {
|
|
|
8942
9374
|
Arguments
|
|
8943
9375
|
};
|
|
8944
9376
|
|
|
8945
|
-
//# debugId=
|
|
9377
|
+
//# debugId=520242AE9AD7B67464756E2164756E21
|