@google-cloud/kcmd 0.1.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -20
- package/build/ts/kcmd/cli.js +6164 -5631
- package/build/ts/kcmd/gcp/biglake.d.ts +2 -0
- package/build/ts/kcmd/gcp/biglake.js +13 -4
- package/build/ts/kcmd/gcp/spanner.d.ts +10 -0
- package/build/ts/kcmd/gcp/spanner.js +6 -3
- package/build/ts/kcmd/layouts/okf.js +49 -7
- package/build/ts/kcmd/layouts/semantic-model.d.ts +13 -0
- package/build/ts/kcmd/layouts/semantic-model.js +129 -50
- package/build/ts/kcmd/okf/concept.d.ts +1 -1
- package/build/ts/kcmd/okf/concept.js +6 -0
- package/build/ts/kcmd/okf/concept_to_entry.d.ts +2 -0
- package/build/ts/kcmd/okf/concept_to_entry.js +29 -2
- package/build/ts/kcmd/okf/init_okf_workspace.js +7 -9
- package/build/ts/kcmd/okf/okf_metadata.js +17 -0
- package/build/ts/kcmd/semantic/bigquery.js +139 -195
- package/build/ts/kcmd/semantic/binding.js +14 -19
- package/build/ts/kcmd/semantic/deploy_bigquery.js +6 -13
- package/build/ts/kcmd/semantic/deploy_knowledge_catalog.d.ts +10 -35
- package/build/ts/kcmd/semantic/deploy_knowledge_catalog.js +105 -162
- package/build/ts/kcmd/semantic/deploy_spanner.js +7 -14
- package/build/ts/kcmd/semantic/deployment_target.d.ts +1 -4
- package/build/ts/kcmd/semantic/deployment_target.js +6 -62
- package/build/ts/kcmd/semantic/graph_ddl.d.ts +10 -0
- package/build/ts/kcmd/semantic/graph_ddl.js +152 -59
- package/build/ts/kcmd/semantic/import_model.d.ts +26 -4
- package/build/ts/kcmd/semantic/import_model.js +531 -111
- package/build/ts/kcmd/semantic/ir.d.ts +2 -4
- package/build/ts/kcmd/semantic/kc_converter.d.ts +7 -1
- package/build/ts/kcmd/semantic/kc_converter.js +32 -27
- package/build/ts/kcmd/semantic/kc_custom_types.d.ts +5 -12
- package/build/ts/kcmd/semantic/kc_custom_types.js +43 -114
- package/build/ts/kcmd/semantic/kc_ids.d.ts +4 -5
- package/build/ts/kcmd/semantic/kc_ids.js +4 -7
- package/build/ts/kcmd/semantic/knowledge_catalog.d.ts +0 -3
- package/build/ts/kcmd/semantic/knowledge_catalog.js +3 -86
- package/build/ts/kcmd/semantic/loader.d.ts +7 -11
- package/build/ts/kcmd/semantic/loader.js +282 -171
- package/build/ts/kcmd/semantic/osi_converter.js +14 -65
- package/build/ts/kcmd/semantic/parameters.d.ts +1 -0
- package/build/ts/kcmd/semantic/parameters.js +22 -3
- package/build/ts/kcmd/semantic/pull_kc.d.ts +1 -5
- package/build/ts/kcmd/semantic/pull_kc.js +38 -112
- package/build/ts/kcmd/semantic/resolve_inheritance.d.ts +7 -2
- package/build/ts/kcmd/semantic/resolve_inheritance.js +37 -21
- package/build/ts/kcmd/semantic/resolve_profiles.d.ts +51 -10
- package/build/ts/kcmd/semantic/resolve_profiles.js +213 -32
- package/build/ts/kcmd/semantic/runtime/runtime.d.ts +4 -12
- package/build/ts/kcmd/semantic/runtime/runtime.js +4 -21
- package/build/ts/kcmd/semantic/runtime/store.d.ts +14 -19
- package/build/ts/kcmd/semantic/runtime/store.js +21 -37
- package/build/ts/kcmd/semantic/source_dialect.d.ts +9 -2
- package/build/ts/kcmd/semantic/source_dialect.js +83 -33
- package/build/ts/kcmd/semantic/source_entries.d.ts +1 -1
- package/build/ts/kcmd/semantic/source_entries.js +95 -36
- package/build/ts/kcmd/semantic/sql_expr_utils.d.ts +13 -7
- package/build/ts/kcmd/semantic/sql_expr_utils.js +240 -67
- package/build/ts/kcmd/semantic/sql_parser.d.ts +5 -2
- package/build/ts/kcmd/semantic/sql_parser.js +22 -0
- package/build/ts/kcmd/semantic/transpile.d.ts +12 -10
- package/build/ts/kcmd/semantic/transpile.js +79 -61
- package/build/ts/kcmd/semantic/validate.d.ts +37 -2
- package/build/ts/kcmd/semantic/validate.js +467 -126
- package/build/ts/kcmd/snapshot.d.ts +4 -0
- package/build/ts/kcmd/snapshot.js +7 -0
- package/build/ts/kcmd/sync.js +44 -7
- package/package.json +1 -1
|
@@ -16,12 +16,19 @@
|
|
|
16
16
|
// table was created, so after a failover an older table's entry can sit in
|
|
17
17
|
// a former primary's region, now a secondary's. The entry is looked up in
|
|
18
18
|
// each replica's region, the primary's first, until one finds it.
|
|
19
|
+
// getHiveCatalog reads a Hive catalog. Its tables are written with
|
|
20
|
+
// `databases/`.
|
|
19
21
|
// - Spanner: the default leader region of the instance's config, which
|
|
20
22
|
// getInstance reads the config's name from. A regional config is named for
|
|
21
23
|
// its region, `regional-us-central1` for us-central1, so it is not read;
|
|
22
24
|
// any other config is, with getInstanceConfig: `nam3` gives us-east4. That
|
|
23
25
|
// read needs spanner.instanceConfigs.get, which the Spanner database roles
|
|
24
|
-
// lack.
|
|
26
|
+
// lack. When it is refused, the entry is looked up in each region the
|
|
27
|
+
// instance has replicas in, which getInstance lists, in the order listed,
|
|
28
|
+
// until one finds it: the default leader's region is one of them, and the
|
|
29
|
+
// list marks no leader (live, 2026-11-04). The instance's
|
|
30
|
+
// `resourceLocation`, such as `us`, is not a Dataplex region. A database's
|
|
31
|
+
// own default leader does not move the entry.
|
|
25
32
|
// - AlloyDB: the location the source names.
|
|
26
33
|
//
|
|
27
34
|
// Dataplex keeps locations in lower case. A lookup in the wrong location
|
|
@@ -34,8 +41,10 @@
|
|
|
34
41
|
// the entry was never looked up, so the message says which read it was, and
|
|
35
42
|
// what the service said: Spanner answers 403 for a project the caller may not
|
|
36
43
|
// read, whether or not the instance exists, so a 403 does not say the table
|
|
37
|
-
// is missing. A
|
|
38
|
-
//
|
|
44
|
+
// is missing. A refused Spanner instance config read is the exception: the
|
|
45
|
+
// entry is then looked up in each replica region the instance lists. A read
|
|
46
|
+
// that fails another way says to try again when a retry may pass (a thrown
|
|
47
|
+
// call, a 429 or a 5xx), and to fix the source otherwise.
|
|
39
48
|
//
|
|
40
49
|
// A Cloud SQL or third-party source is not looked up here: sourceDialects
|
|
41
50
|
// already looks up its entry directly, and fails the push when it is missing.
|
|
@@ -65,10 +74,11 @@ const resolve_profiles_1 = require("./resolve_profiles");
|
|
|
65
74
|
// - A job with no location runs in the primary.
|
|
66
75
|
//
|
|
67
76
|
// Reads each table's location and looks its entry up once per location it
|
|
68
|
-
// may sit in, and reads each
|
|
69
|
-
// parallel, and a table's locations in
|
|
70
|
-
// entity with no source, and a source that
|
|
71
|
-
// (a Cloud SQL or third-party table
|
|
77
|
+
// may sit in, and reads each instance once and each catalog at most once
|
|
78
|
+
// through each API; tables are checked in parallel, and a table's locations in
|
|
79
|
+
// turn. Skips an abstract entity, an entity with no source, and a source that
|
|
80
|
+
// names no table this module knows (a Cloud SQL or third-party table). Never
|
|
81
|
+
// throws.
|
|
72
82
|
async function sourceEntryErrors(model, readers) {
|
|
73
83
|
const uses = [];
|
|
74
84
|
const use = (entity, profile, source) => {
|
|
@@ -149,24 +159,40 @@ async function problemOf(table, located, readers) {
|
|
|
149
159
|
`exist, or you may not be allowed to read its catalog entry.${table.system === 'alloydb' ?
|
|
150
160
|
' A new AlloyDB cluster can take hours to appear in the ' +
|
|
151
161
|
'catalog after Dataplex integration is turned on.' :
|
|
152
|
-
''} Fix the source, or pass
|
|
162
|
+
''}${locations.unfound ?? ''} Fix the source, or pass ` +
|
|
163
|
+
`--skip-source-check.`;
|
|
153
164
|
}
|
|
154
165
|
// `location`, which is kept in lower case, as BigQuery writes it: a
|
|
155
166
|
// multi-region, which has no hyphen, in upper case (`US`, `EU`).
|
|
156
167
|
function shown(location) {
|
|
157
168
|
return location.includes('-') ? location : location.toUpperCase();
|
|
158
169
|
}
|
|
159
|
-
// Said when reading an instance config is refused
|
|
160
|
-
// bigquery-public-data and
|
|
161
|
-
//
|
|
170
|
+
// Said when reading an instance config is refused and the instance lists no
|
|
171
|
+
// replica regions to look in instead (live on bigquery-public-data and
|
|
172
|
+
// dataplex-staging-types, 2026-10-27: Spanner answers 403 naming this
|
|
173
|
+
// permission, even for a Google-managed config such as nam3).
|
|
162
174
|
const CONFIG_REFUSED_HINT = ' Reading it needs the permission ' +
|
|
163
175
|
'spanner.instanceConfigs.get on the project, which roles/spanner.viewer ' +
|
|
164
176
|
'includes and the Spanner database roles do not.';
|
|
177
|
+
// Said when reading an instance is refused: a Spanner database role gives
|
|
178
|
+
// spanner.instances.get only when granted on the instance or its project
|
|
179
|
+
// (from review on #630, 2026-11-04).
|
|
180
|
+
const INSTANCE_REFUSED_HINT = ' The Spanner database roles include ' +
|
|
181
|
+
'spanner.instances.get, but a role granted on one database does not ' +
|
|
182
|
+
'apply to its instance.';
|
|
183
|
+
// Said when a BigLake catalog read fails in a way a retry cannot fix and the
|
|
184
|
+
// other kind of catalog's API finds the catalog. The status alone does not
|
|
185
|
+
// tell: live, the Hive API answers 400 for an Iceberg REST catalog id with a
|
|
186
|
+
// hyphen and for a Hive catalog id in capitals.
|
|
187
|
+
const HIVE_FORM_HINT = ` A table in a Hive catalog is written with 'databases/<database>'.`;
|
|
188
|
+
const ICEBERG_FORM_HINT = ` A table in an Iceberg REST catalog is written ` +
|
|
189
|
+
`with 'namespaces/<namespace>'.`;
|
|
165
190
|
// The locations where `table`'s catalog entry may sit, as Dataplex writes
|
|
166
|
-
// them, in lower case, in the order to look in: one for a BigQuery
|
|
167
|
-
//
|
|
168
|
-
//
|
|
169
|
-
//
|
|
191
|
+
// them, in lower case, in the order to look in: one for a BigQuery or AlloyDB
|
|
192
|
+
// table; one for a Spanner table, or one per replica region of its instance
|
|
193
|
+
// when reading its config is refused; and one per replica for a BigLake table.
|
|
194
|
+
// `cache` keeps each instance's read and each catalog's read through each
|
|
195
|
+
// API, so each is made once; each table is read once already.
|
|
170
196
|
async function locationsOf(table, readers, cache) {
|
|
171
197
|
const once = (key, readIt) => {
|
|
172
198
|
if (!cache.has(key))
|
|
@@ -184,16 +210,23 @@ async function locationsOf(table, readers, cache) {
|
|
|
184
210
|
located(res.result.location, what, 'no location') :
|
|
185
211
|
res;
|
|
186
212
|
}
|
|
187
|
-
case 'biglake':
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
213
|
+
case 'biglake': {
|
|
214
|
+
const readCatalog = (hive) => once(`biglake/${project}/${table.catalog}/${hive}`, () => hive ? readers.biglake.getHiveCatalog(project, table.catalog) :
|
|
215
|
+
readers.biglake.getCatalog(project, table.catalog));
|
|
216
|
+
const kind = table.hive ? 'Hive catalog' : 'catalog';
|
|
217
|
+
const what = `reading the region of ${kind} '${table.catalog}'`;
|
|
218
|
+
// When the read fails and the other API finds the catalog, the source
|
|
219
|
+
// names a table in the other kind of catalog.
|
|
220
|
+
const otherForm = async () => 'result' in await (0, catalog_entry_1.read)(what, () => readCatalog(!table.hive)) ?
|
|
221
|
+
(table.hive ? ICEBERG_FORM_HINT : HIVE_FORM_HINT) :
|
|
222
|
+
'';
|
|
223
|
+
const res = await readLocation(what, `a ${kind}`, () => readCatalog(table.hive), otherForm);
|
|
224
|
+
return 'result' in res ? replicaRegions(res.result, what) : res;
|
|
225
|
+
}
|
|
193
226
|
case 'spanner':
|
|
194
227
|
return once(`spanner/${project}/${table.instance}`, async () => {
|
|
195
228
|
const what = `reading instance '${table.instance}'`;
|
|
196
|
-
const instance = await readLocation(what, 'an instance', () => readers.spanner.getInstance(project, table.instance));
|
|
229
|
+
const instance = await readLocation(what, 'an instance', () => readers.spanner.getInstance(project, table.instance), status => status === 403 ? INSTANCE_REFUSED_HINT : '');
|
|
197
230
|
if ('problem' in instance)
|
|
198
231
|
return instance;
|
|
199
232
|
const configName = instance.result.config;
|
|
@@ -207,7 +240,23 @@ async function locationsOf(table, readers, cache) {
|
|
|
207
240
|
if (region)
|
|
208
241
|
return { result: [region[1]] };
|
|
209
242
|
const configWhat = `reading instance config '${configName}'`;
|
|
210
|
-
const
|
|
243
|
+
const res = await (0, catalog_entry_1.read)(configWhat, () => readers.spanner.getInstanceConfig(configName));
|
|
244
|
+
// The default leader's region is one of the instance's replica
|
|
245
|
+
// regions, so when the config read is refused, each is looked in.
|
|
246
|
+
if ('unreadable' in res && res.status === 403) {
|
|
247
|
+
const regions = (instance.result.replicaComputeCapacity ?? [])
|
|
248
|
+
.map(r => r.replicaSelection?.location)
|
|
249
|
+
.filter((l) => !!l);
|
|
250
|
+
if (regions.length) {
|
|
251
|
+
return {
|
|
252
|
+
result: regions,
|
|
253
|
+
unfound: ` Instance config '${configName}' could not be read ` +
|
|
254
|
+
`(HTTP 403), so the entry was looked up in each of the ` +
|
|
255
|
+
`instance's replica regions: ${regions.join(', ')}.`,
|
|
256
|
+
};
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
const config = await checked(res, configWhat, 'an instance config', status => status === 403 ? CONFIG_REFUSED_HINT : '');
|
|
211
260
|
// A config marks exactly one replica as its default leader.
|
|
212
261
|
return 'result' in config ?
|
|
213
262
|
located(config.result.replicas?.find(r => r.defaultLeaderLocation)
|
|
@@ -216,22 +265,31 @@ async function locationsOf(table, readers, cache) {
|
|
|
216
265
|
});
|
|
217
266
|
}
|
|
218
267
|
}
|
|
219
|
-
// Makes the location read `call`, which `what` names
|
|
220
|
-
//
|
|
221
|
-
|
|
268
|
+
// Makes the location read `call`, which `what` names, and judges its answer
|
|
269
|
+
// with checked.
|
|
270
|
+
async function readLocation(what, kind, call, hintFor) {
|
|
271
|
+
return checked(await (0, catalog_entry_1.read)(what, call), what, kind, hintFor);
|
|
272
|
+
}
|
|
273
|
+
// What the location read `what` gave in `res`. A 404 says the source names
|
|
274
|
+
// `kind`, such as `a table`, that does not exist (the services answer 404 for
|
|
275
|
+
// a missing table, dataset, catalog or instance; live, 2026-10-27 and
|
|
222
276
|
// 2026-11-03). A 403 says the read was refused, followed by the service's
|
|
223
|
-
// message
|
|
224
|
-
|
|
225
|
-
|
|
277
|
+
// message. When a retry cannot pass, `hintFor` gives a sentence to add for the
|
|
278
|
+
// status.
|
|
279
|
+
async function checked(res, what, kind, hintFor = () => '') {
|
|
226
280
|
if ('result' in res)
|
|
227
281
|
return res;
|
|
282
|
+
const { status } = res;
|
|
283
|
+
const hint = status === undefined || (0, api_1.isTransientStatus)(status) ?
|
|
284
|
+
'' :
|
|
285
|
+
await hintFor(status);
|
|
228
286
|
if ('error' in res)
|
|
229
|
-
return failedProblem(res);
|
|
287
|
+
return failedProblem(res, hint);
|
|
230
288
|
return {
|
|
231
289
|
problem: res.status === 404 ?
|
|
232
290
|
`names ${kind} that does not exist: ${what} found nothing (HTTP ` +
|
|
233
|
-
`404)
|
|
234
|
-
`could not be checked: ${what} was refused (HTTP 403).${res.message ? ` ${res.message}` : ''}${
|
|
291
|
+
`404).${hint} Fix the source, or pass --skip-source-check.` :
|
|
292
|
+
`could not be checked: ${what} was refused (HTTP 403).${res.message ? ` ${res.message}` : ''}${hint} The ` +
|
|
235
293
|
`source's catalog entry was not looked up. Fix the access or ` +
|
|
236
294
|
`the source, or pass --skip-source-check.`,
|
|
237
295
|
};
|
|
@@ -265,13 +323,14 @@ function located(location, what, missing) {
|
|
|
265
323
|
}
|
|
266
324
|
// The problem for the failed read `res`. A thrown call, a 429 or a 5xx may
|
|
267
325
|
// pass on retry; any other status, such as BigQuery's 400 for an invalid
|
|
268
|
-
// table ID, never will, so the source must change
|
|
269
|
-
|
|
326
|
+
// table ID, never will, so the source must change, and `hint` follows the
|
|
327
|
+
// service's answer.
|
|
328
|
+
function failedProblem(res, hint = '') {
|
|
270
329
|
return res.status === undefined || (0, api_1.isTransientStatus)(res.status) ?
|
|
271
330
|
transientProblem(res.error) :
|
|
272
331
|
{
|
|
273
|
-
problem: `could not be checked: ${res.error} Fix the source,
|
|
274
|
-
`pass --skip-source-check.`,
|
|
332
|
+
problem: `could not be checked: ${res.error}${hint} Fix the source, ` +
|
|
333
|
+
`or pass --skip-source-check.`,
|
|
275
334
|
};
|
|
276
335
|
}
|
|
277
336
|
// The problem for a read that failed with `detail`, which may pass on retry.
|
|
@@ -1,17 +1,23 @@
|
|
|
1
1
|
import { Field } from './ir';
|
|
2
2
|
import { SqlColumn } from './sql_parser';
|
|
3
|
-
export declare
|
|
4
|
-
export declare function
|
|
5
|
-
export declare function
|
|
6
|
-
export declare function
|
|
7
|
-
export declare function
|
|
8
|
-
export declare function
|
|
3
|
+
export declare function unusedName(text: string, base: string): string;
|
|
4
|
+
export declare function withoutQualifier(expression: string, qualifier: string, dialect?: string): string;
|
|
5
|
+
export declare function withSiblingsSpliced(expression: string, qualifier: string, siblings: ReadonlyMap<string, string>): string;
|
|
6
|
+
export declare function qualifiedColumns(expression: string, qualifier: string, dialect?: string): string[] | undefined;
|
|
7
|
+
export declare function selfReferenceMessage(owner: string, cycle: string[], where?: string): string;
|
|
8
|
+
export declare function requalified(expression: string, from: string, to: string, dialect?: string): string;
|
|
9
|
+
export declare function measureEntities(expression: string, entityNames: ReadonlyArray<string>, entity?: string): string[];
|
|
9
10
|
export declare function sqlWithoutComments(text: string, dialect?: string): string;
|
|
10
11
|
export declare function fieldBindingWithoutComments(field: Pick<Field, 'expression' | 'importedExpression' | 'importedDialect'>, targetDialect?: string): string | undefined;
|
|
11
12
|
export declare function columnReferences(expression: string, dialect: string): ReadonlyArray<SqlColumn> | undefined;
|
|
12
|
-
export declare function columnsForSource(text: string, dialect: string, impliedDialect?: string): ReadonlyArray<SqlColumn> | undefined;
|
|
13
|
+
export declare function columnsForSource(text: string, dialect: string, impliedDialect?: string | null): ReadonlyArray<SqlColumn> | undefined;
|
|
13
14
|
export declare function entitiesReached(columns: ReadonlyArray<SqlColumn>, entityNames: ReadonlyArray<string> | ReadonlySet<string>): string[];
|
|
14
15
|
export declare function fieldsReadOn(columns: ReadonlyArray<SqlColumn>, entityName: string): string[];
|
|
15
16
|
export declare const SUPPORTED_AGGREGATES: string[];
|
|
17
|
+
export declare function singleAggregate(expression: string, dialect: string): {
|
|
18
|
+
fn: string;
|
|
19
|
+
operand: string;
|
|
20
|
+
distinct: boolean;
|
|
21
|
+
} | undefined;
|
|
16
22
|
export declare function isSingleAggregate(expression: string, dialect: string): boolean;
|
|
17
23
|
export declare function isColumnName(value: unknown): boolean;
|
|
@@ -2,87 +2,209 @@
|
|
|
2
2
|
// Helpers for reading and rewriting entity-qualified SQL expressions across the
|
|
3
3
|
// semantic-model layer.
|
|
4
4
|
//
|
|
5
|
-
//
|
|
5
|
+
// An expression names a field of its entity as `<Entity>.<field>`, and a
|
|
6
|
+
// column of its table without a qualifier. The SQL parser
|
|
6
7
|
// (`columnReferences`, `entitiesReached`, `fieldsReadOn`) reads column and
|
|
7
8
|
// entity references across dialects for the loader, validator, profile checks,
|
|
8
|
-
// pull reader, and transpiler. The graph generators
|
|
9
|
-
//
|
|
9
|
+
// pull reader, and transpiler. The graph generators read them with the parser
|
|
10
|
+
// too: `measureEntities` places a measure, `singleAggregate` finds its
|
|
11
|
+
// aggregate, `withoutQualifier` makes an expression table-local,
|
|
12
|
+
// `qualifiedColumns` finds the sibling fields it names and
|
|
13
|
+
// `withSiblingsSpliced` puts their SQL in their place, and `requalified` moves
|
|
14
|
+
// an inherited expression onto the entity inheriting it.
|
|
10
15
|
//
|
|
11
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
-
exports.SUPPORTED_AGGREGATES =
|
|
13
|
-
exports.
|
|
14
|
-
exports.
|
|
15
|
-
exports.
|
|
16
|
-
exports.
|
|
17
|
-
exports.
|
|
17
|
+
exports.SUPPORTED_AGGREGATES = void 0;
|
|
18
|
+
exports.unusedName = unusedName;
|
|
19
|
+
exports.withoutQualifier = withoutQualifier;
|
|
20
|
+
exports.withSiblingsSpliced = withSiblingsSpliced;
|
|
21
|
+
exports.qualifiedColumns = qualifiedColumns;
|
|
22
|
+
exports.selfReferenceMessage = selfReferenceMessage;
|
|
23
|
+
exports.requalified = requalified;
|
|
24
|
+
exports.measureEntities = measureEntities;
|
|
18
25
|
exports.sqlWithoutComments = sqlWithoutComments;
|
|
19
26
|
exports.fieldBindingWithoutComments = fieldBindingWithoutComments;
|
|
20
27
|
exports.columnReferences = columnReferences;
|
|
21
28
|
exports.columnsForSource = columnsForSource;
|
|
22
29
|
exports.entitiesReached = entitiesReached;
|
|
23
30
|
exports.fieldsReadOn = fieldsReadOn;
|
|
31
|
+
exports.singleAggregate = singleAggregate;
|
|
24
32
|
exports.isSingleAggregate = isSingleAggregate;
|
|
25
33
|
exports.isColumnName = isColumnName;
|
|
26
34
|
const ir_1 = require("./ir");
|
|
35
|
+
const sql_identifiers_1 = require("./sql_identifiers");
|
|
27
36
|
const sql_parser_1 = require("./sql_parser");
|
|
28
|
-
//
|
|
29
|
-
//
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
//
|
|
37
|
-
//
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
//
|
|
42
|
-
//
|
|
43
|
-
|
|
37
|
+
// The dialect a graph generator reads `dialect` in: an ANSI_SQL text, or a
|
|
38
|
+
// text with no dialect, is read as BigQuery.
|
|
39
|
+
function graphReading(dialect) {
|
|
40
|
+
return !dialect || dialect === 'ANSI_SQL' ? 'BIGQUERY' : dialect;
|
|
41
|
+
}
|
|
42
|
+
// Each `<name>.` qualifier that `expression`, read in `dialect`, writes before
|
|
43
|
+
// a column, for a name in `names`, as indexes into the text's tokens, in the
|
|
44
|
+
// order written. A name matches case-sensitively, bare or quoted as the
|
|
45
|
+
// dialect quotes an identifier, and not after a dot, so the second `orders` in
|
|
46
|
+
// the struct path `orders.orders.id` is not one. The SQL parser decides which
|
|
47
|
+
// candidates are qualifiers, so a string literal, a comment or a function
|
|
48
|
+
// namespace such as `NET.` in `NET.HOST(orders.url)` holds none. The parser
|
|
49
|
+
// reads each candidate as a placeholder name, so one it reserves as a keyword,
|
|
50
|
+
// such as `Date` in `Date.days`, still reads as a qualifier. When the parser
|
|
51
|
+
// cannot read the text even so, every candidate is taken as a qualifier.
|
|
52
|
+
// Undefined when the engine cannot tokenize the text.
|
|
53
|
+
function qualifiersIn(expression, names, dialect) {
|
|
54
|
+
const tokens = (0, sql_parser_1.sqlTokens)(expression, dialect);
|
|
55
|
+
if (!tokens)
|
|
56
|
+
return undefined;
|
|
57
|
+
const chars = Array.from(expression);
|
|
58
|
+
const asWritten = (t) => chars.slice(t.span.start, t.span.end).join('');
|
|
59
|
+
const candidates = [];
|
|
60
|
+
tokens.forEach((t, i) => {
|
|
61
|
+
if (names.has(t.text) && tokens[i + 1]?.token_type === 'DOT' &&
|
|
62
|
+
tokens[i - 1]?.token_type !== 'DOT' &&
|
|
63
|
+
(t.token_type === 'QUOTED_IDENTIFIER' || asWritten(t) === t.text)) {
|
|
64
|
+
candidates.push(i);
|
|
65
|
+
}
|
|
66
|
+
});
|
|
67
|
+
if (!candidates.length)
|
|
68
|
+
return { tokens, at: [] };
|
|
69
|
+
// A placeholder stem the text does not hold.
|
|
70
|
+
const stem = unusedName(expression, 'kcmd_qualifier_');
|
|
71
|
+
let text = '';
|
|
72
|
+
let end = 0;
|
|
73
|
+
candidates.forEach((i, k) => {
|
|
74
|
+
text += chars.slice(end, tokens[i].span.start).join('') + stem + k;
|
|
75
|
+
end = tokens[i].span.end;
|
|
76
|
+
});
|
|
77
|
+
text += chars.slice(end).join('');
|
|
78
|
+
const columns = columnReferences(text, dialect);
|
|
79
|
+
const at = !columns ? candidates : candidates.filter((_, k) => columns.some(c => c.qualifier?.toLowerCase() === stem + k));
|
|
80
|
+
return { tokens, at };
|
|
81
|
+
}
|
|
82
|
+
// `base`, a lowercase name, with underscores added until `text` does not hold
|
|
83
|
+
// it in any case.
|
|
84
|
+
function unusedName(text, base) {
|
|
85
|
+
let name = base;
|
|
86
|
+
while (text.toLowerCase().includes(name))
|
|
87
|
+
name += '_';
|
|
88
|
+
return name;
|
|
89
|
+
}
|
|
90
|
+
// `expression` without each `<qualifier>.` it writes before a column (see
|
|
91
|
+
// qualifiersIn), so it reads the columns of the qualifier's own table. The
|
|
92
|
+
// rest of the text stays as written. The text is read in `dialect`; an
|
|
93
|
+
// ANSI_SQL text, or one with no dialect, is read as BigQuery, as the graph
|
|
94
|
+
// generators read it. Returns `expression` itself when it writes no such
|
|
95
|
+
// qualifier or the engine cannot tokenize it.
|
|
96
|
+
function withoutQualifier(expression, qualifier, dialect) {
|
|
97
|
+
return spliced(expression, qualifier, graphReading(dialect), new Map());
|
|
98
|
+
}
|
|
99
|
+
// `expression`, BigQuery SQL on `qualifier`'s table, as withoutQualifier gives
|
|
100
|
+
// it, except that each `<qualifier>.<field>` whose field `siblings` holds is
|
|
101
|
+
// replaced, whole, by the SQL it maps to, as given. The graph generators put
|
|
102
|
+
// the SQL of each sibling field an expression names in its place so (see
|
|
103
|
+
// graph_ddl.fieldSql).
|
|
104
|
+
function withSiblingsSpliced(expression, qualifier, siblings) {
|
|
105
|
+
return spliced(expression, qualifier, 'BIGQUERY', siblings);
|
|
106
|
+
}
|
|
107
|
+
// `expression`, read in `dialect`, without each `<qualifier>.` it writes
|
|
108
|
+
// before a column (see qualifiersIn), and with each `<qualifier>.<column>`
|
|
109
|
+
// whose column `replacements` holds replaced, whole, by the SQL it maps to.
|
|
110
|
+
function spliced(expression, qualifier, dialect, replacements) {
|
|
111
|
+
const read = qualifiersIn(expression, new Set([qualifier]), dialect);
|
|
112
|
+
if (!read?.at.length)
|
|
113
|
+
return expression;
|
|
114
|
+
const { tokens, at } = read;
|
|
115
|
+
const chars = Array.from(expression);
|
|
44
116
|
let out = '';
|
|
45
|
-
let
|
|
46
|
-
for (const
|
|
47
|
-
out +=
|
|
48
|
-
|
|
49
|
-
|
|
117
|
+
let end = 0;
|
|
118
|
+
for (const i of at) {
|
|
119
|
+
out += chars.slice(end, tokens[i].span.start).join('');
|
|
120
|
+
const column = tokens[i + 2];
|
|
121
|
+
const replacement = column && replacements.get(nameOf(column, chars));
|
|
122
|
+
if (replacement !== undefined) {
|
|
123
|
+
out += replacement;
|
|
124
|
+
end = column.span.end;
|
|
125
|
+
}
|
|
126
|
+
else {
|
|
127
|
+
// The qualifier, its dot, and what lies between the dot and the column.
|
|
128
|
+
end = column?.span.start ?? tokens[i + 1].span.end;
|
|
129
|
+
}
|
|
50
130
|
}
|
|
51
|
-
out
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
//
|
|
55
|
-
//
|
|
56
|
-
//
|
|
57
|
-
//
|
|
58
|
-
//
|
|
59
|
-
function
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
131
|
+
return out + chars.slice(end).join('');
|
|
132
|
+
}
|
|
133
|
+
// The column written after each `<qualifier>.` that `expression` writes before
|
|
134
|
+
// a column (see qualifiersIn), without its quotes, in the order written, so
|
|
135
|
+
// ``orders.a + `orders`.`b`.c`` gives `a` and `b`. The text is read in
|
|
136
|
+
// `dialect`, as withoutQualifier reads it, so a qualifier the parser reserves,
|
|
137
|
+
// such as `Date` in `Date.days`, gives its column too. Undefined when the
|
|
138
|
+
// engine cannot tokenize the text.
|
|
139
|
+
function qualifiedColumns(expression, qualifier, dialect) {
|
|
140
|
+
const read = qualifiersIn(expression, new Set([qualifier]), graphReading(dialect));
|
|
141
|
+
if (!read)
|
|
142
|
+
return undefined;
|
|
143
|
+
const chars = Array.from(expression);
|
|
144
|
+
return read.at.filter(i => read.tokens[i + 2])
|
|
145
|
+
.map(i => nameOf(read.tokens[i + 2], chars));
|
|
146
|
+
}
|
|
147
|
+
// The name token `t` of the text whose characters are `chars` gives, as
|
|
148
|
+
// written, without its quotes.
|
|
149
|
+
function nameOf(t, chars) {
|
|
150
|
+
return t.token_type === 'QUOTED_IDENTIFIER' ?
|
|
151
|
+
t.text :
|
|
152
|
+
chars.slice(t.span.start, t.span.end).join('');
|
|
153
|
+
}
|
|
154
|
+
// The error for a field of `owner` that names itself. `cycle` starts at that
|
|
155
|
+
// field and runs through the fields it names back to it, and `where` follows
|
|
156
|
+
// the field's name, as ` in model 'm' (doc)` does. The graph renders a field
|
|
157
|
+
// that names a sibling field from that sibling's own SQL (see
|
|
158
|
+
// graph_ddl.fieldSql), so a field that reaches itself has none.
|
|
159
|
+
function selfReferenceMessage(owner, cycle, where = '') {
|
|
160
|
+
const [field, ...through] = cycle;
|
|
161
|
+
return `field '${owner}.${field}'${where} names itself` +
|
|
162
|
+
(through.length ?
|
|
163
|
+
` through ${through.map(n => `'${owner}.${n}'`).join(', ')}` :
|
|
164
|
+
'') +
|
|
165
|
+
`; a field expression may name other fields of its entity, but not ` +
|
|
166
|
+
`itself, directly or through other fields` +
|
|
167
|
+
(through.length ? '' :
|
|
168
|
+
`; to read the column of that name, write it ` +
|
|
169
|
+
`without the qualifier: '${(0, sql_identifiers_1.quoteIfReserved)(field)}'`);
|
|
170
|
+
}
|
|
171
|
+
// `expression` with each `<from>.` qualifier it writes before a column (see
|
|
172
|
+
// qualifiersIn) renamed `<to>.`, quoted as it was written. The rest of the
|
|
173
|
+
// text stays as written. The text is read in `dialect`, as withoutQualifier
|
|
174
|
+
// reads it. Returns `expression` itself when it writes no such qualifier or
|
|
175
|
+
// the engine cannot tokenize it.
|
|
176
|
+
function requalified(expression, from, to, dialect) {
|
|
177
|
+
const read = qualifiersIn(expression, new Set([from]), graphReading(dialect));
|
|
178
|
+
if (!read?.at.length)
|
|
179
|
+
return expression;
|
|
180
|
+
const { tokens, at } = read;
|
|
181
|
+
const chars = Array.from(expression);
|
|
182
|
+
let out = '';
|
|
183
|
+
let end = 0;
|
|
184
|
+
for (const i of at) {
|
|
185
|
+
const { start, end: after } = tokens[i].span;
|
|
186
|
+
out += chars.slice(end, start).join('') +
|
|
187
|
+
chars.slice(start, after).join('').replace(from, () => to);
|
|
188
|
+
end = after;
|
|
72
189
|
}
|
|
73
|
-
return
|
|
190
|
+
return out + chars.slice(end).join('');
|
|
74
191
|
}
|
|
75
|
-
//
|
|
76
|
-
//
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
192
|
+
// The entities from `entityNames` whose qualifier a metric's BigQuery SQL
|
|
193
|
+
// `expression` writes before a column (see qualifiersIn), deduplicated in the
|
|
194
|
+
// order written, or else `entity`, the metric's anchor, when given. The
|
|
195
|
+
// BigQuery generator places the metric's measure on the entity this gives
|
|
196
|
+
// when it gives one, and a BigQuery push rejects a metric it gives several or
|
|
197
|
+
// none for (see validate.ts).
|
|
198
|
+
function measureEntities(expression, entityNames, entity) {
|
|
199
|
+
const read = qualifiersIn(expression, new Set(entityNames), 'BIGQUERY');
|
|
200
|
+
const reached = [...new Set(read?.at.map(i => read.tokens[i].text) ?? [])];
|
|
201
|
+
return reached.length || !entity ? reached : [entity];
|
|
80
202
|
}
|
|
81
203
|
// `text`, written in `dialect`, with its comments removed. An ANSI_SQL text, or
|
|
82
204
|
// a text with no dialect, is read as BigQuery, as the loader and the graph
|
|
83
205
|
// generators read it.
|
|
84
206
|
function sqlWithoutComments(text, dialect) {
|
|
85
|
-
return (0, sql_parser_1.withoutComments)(text,
|
|
207
|
+
return (0, sql_parser_1.withoutComments)(text, graphReading(dialect));
|
|
86
208
|
}
|
|
87
209
|
// `field`'s binding, as fieldBinding gives it, with its comments removed.
|
|
88
210
|
// Undefined when the field is unbound. The target text is read in
|
|
@@ -124,8 +246,11 @@ function columnReferences(expression, dialect) {
|
|
|
124
246
|
// an `ANSI_SQL` entry as its own SQL. When the sources do not imply a single
|
|
125
247
|
// dialect, the generic parser reads the text, and any bare identifier that
|
|
126
248
|
// BigQuery's parser does not also read as a bare column (such as a `MONTH`
|
|
127
|
-
// date part or a double-quoted string literal) is filtered out.
|
|
249
|
+
// date part or a double-quoted string literal) is filtered out. When their
|
|
250
|
+
// dialect is null, which no lookup read, an `ANSI_SQL` text is not read.
|
|
128
251
|
function columnsForSource(text, dialect, impliedDialect) {
|
|
252
|
+
if (dialect === 'ANSI_SQL' && impliedDialect === null)
|
|
253
|
+
return undefined;
|
|
129
254
|
if (dialect === 'ANSI_SQL' && impliedDialect) {
|
|
130
255
|
return columnReferences(text, impliedDialect);
|
|
131
256
|
}
|
|
@@ -165,15 +290,18 @@ function fieldsReadOn(columns, entityName) {
|
|
|
165
290
|
// The aggregates BigQuery accepts in a MEASURE, its complete list. BigQuery
|
|
166
291
|
// rejects any other there, such as COUNTIF or APPROX_COUNT_DISTINCT.
|
|
167
292
|
exports.SUPPORTED_AGGREGATES = ['SUM', 'AVG', 'COUNT', 'MIN', 'MAX'];
|
|
168
|
-
//
|
|
169
|
-
// one operand, or COUNT(*), with at most DISTINCT or ALL
|
|
170
|
-
// other aggregate, window or subquery
|
|
171
|
-
|
|
293
|
+
// When `expression`, read in `dialect`, is one of SUPPORTED_AGGREGATES over
|
|
294
|
+
// one operand, or COUNT(*), with at most DISTINCT or ALL, and the operand holds
|
|
295
|
+
// no other aggregate, window or subquery: the function's name as written, the
|
|
296
|
+
// operand as written without the parentheses around it (so
|
|
297
|
+
// `COUNT(DISTINCT(x))` gives `x`), and whether the call is DISTINCT. COUNT(*)
|
|
298
|
+
// gives the operand `*`. Undefined otherwise.
|
|
299
|
+
function singleAggregate(expression, dialect) {
|
|
172
300
|
const node = unwrapped((0, sql_parser_1.sqlExpressionTree)(expression, dialect));
|
|
173
301
|
const kind = node && Object.keys(node).length === 1 ? Object.keys(node)[0] :
|
|
174
302
|
undefined;
|
|
175
303
|
if (!kind || !exports.SUPPORTED_AGGREGATES.includes(kind.toUpperCase())) {
|
|
176
|
-
return
|
|
304
|
+
return undefined;
|
|
177
305
|
}
|
|
178
306
|
const call = node[kind];
|
|
179
307
|
const star = kind === 'count' && call.star === true && !call.this;
|
|
@@ -181,12 +309,57 @@ function isSingleAggregate(expression, dialect) {
|
|
|
181
309
|
if (!star &&
|
|
182
310
|
(!operand || operand.star || operand.tuple ||
|
|
183
311
|
nestsAggregateOrSubquery(operand))) {
|
|
184
|
-
return
|
|
312
|
+
return undefined;
|
|
185
313
|
}
|
|
186
314
|
// The parser keeps the call's name in `name` or `original_name`.
|
|
187
315
|
const known = new Set(['this', 'star', 'distinct', 'name', 'original_name']);
|
|
188
|
-
|
|
189
|
-
v === false || (Array.isArray(v) && v.length === 0))
|
|
316
|
+
if (!Object.entries(call).every(([key, v]) => known.has(key) || v === null || v === undefined ||
|
|
317
|
+
v === false || (Array.isArray(v) && v.length === 0))) {
|
|
318
|
+
return undefined;
|
|
319
|
+
}
|
|
320
|
+
// The call as written: its name, `(`, DISTINCT or ALL if written, the
|
|
321
|
+
// operand and `)`, inside any parentheses around the whole call.
|
|
322
|
+
const tokens = unparenthesized((0, sql_parser_1.sqlTokens)(expression, dialect) ?? []);
|
|
323
|
+
if (tokens.length < 4 || tokens[1].token_type !== 'L_PAREN' ||
|
|
324
|
+
tokens[tokens.length - 1].token_type !== 'R_PAREN') {
|
|
325
|
+
return undefined;
|
|
326
|
+
}
|
|
327
|
+
let inner = tokens.slice(2, -1);
|
|
328
|
+
if (inner[0]?.token_type === 'DISTINCT' || inner[0]?.token_type === 'ALL') {
|
|
329
|
+
inner = inner.slice(1);
|
|
330
|
+
}
|
|
331
|
+
inner = unparenthesized(inner);
|
|
332
|
+
if (!inner.length)
|
|
333
|
+
return undefined;
|
|
334
|
+
const chars = Array.from(expression);
|
|
335
|
+
return {
|
|
336
|
+
fn: tokens[0].text,
|
|
337
|
+
operand: chars.slice(inner[0].span.start, inner[inner.length - 1].span.end)
|
|
338
|
+
.join(''),
|
|
339
|
+
distinct: call.distinct === true,
|
|
340
|
+
};
|
|
341
|
+
}
|
|
342
|
+
// Whether `expression`, read in `dialect`, is a single aggregate (see
|
|
343
|
+
// singleAggregate), the form a BigQuery measure takes.
|
|
344
|
+
function isSingleAggregate(expression, dialect) {
|
|
345
|
+
return singleAggregate(expression, dialect) !== undefined;
|
|
346
|
+
}
|
|
347
|
+
// `tokens` without the parentheses around all of them.
|
|
348
|
+
function unparenthesized(tokens) {
|
|
349
|
+
while (tokens.length >= 2 && tokens[0].token_type === 'L_PAREN') {
|
|
350
|
+
let depth = 0;
|
|
351
|
+
let closesAt = -1;
|
|
352
|
+
for (let i = 0; i < tokens.length && closesAt < 0; i++) {
|
|
353
|
+
if (tokens[i].token_type === 'L_PAREN')
|
|
354
|
+
depth++;
|
|
355
|
+
if (tokens[i].token_type === 'R_PAREN' && --depth === 0)
|
|
356
|
+
closesAt = i;
|
|
357
|
+
}
|
|
358
|
+
if (closesAt !== tokens.length - 1)
|
|
359
|
+
break;
|
|
360
|
+
tokens = tokens.slice(1, -1);
|
|
361
|
+
}
|
|
362
|
+
return tokens;
|
|
190
363
|
}
|
|
191
364
|
// The parser's tree `node` without the parentheses and comments around it.
|
|
192
365
|
function unwrapped(node) {
|