@google-cloud/kcmd 0.1.0 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/README.md +18 -20
  2. package/build/ts/kcmd/cli.js +6164 -5631
  3. package/build/ts/kcmd/gcp/biglake.d.ts +2 -0
  4. package/build/ts/kcmd/gcp/biglake.js +13 -4
  5. package/build/ts/kcmd/gcp/spanner.d.ts +10 -0
  6. package/build/ts/kcmd/gcp/spanner.js +6 -3
  7. package/build/ts/kcmd/layouts/okf.js +49 -7
  8. package/build/ts/kcmd/layouts/semantic-model.d.ts +13 -0
  9. package/build/ts/kcmd/layouts/semantic-model.js +129 -50
  10. package/build/ts/kcmd/okf/concept.d.ts +1 -1
  11. package/build/ts/kcmd/okf/concept.js +6 -0
  12. package/build/ts/kcmd/okf/concept_to_entry.d.ts +2 -0
  13. package/build/ts/kcmd/okf/concept_to_entry.js +29 -2
  14. package/build/ts/kcmd/okf/init_okf_workspace.js +7 -9
  15. package/build/ts/kcmd/okf/okf_metadata.js +17 -0
  16. package/build/ts/kcmd/semantic/bigquery.js +139 -195
  17. package/build/ts/kcmd/semantic/binding.js +14 -19
  18. package/build/ts/kcmd/semantic/deploy_bigquery.js +6 -13
  19. package/build/ts/kcmd/semantic/deploy_knowledge_catalog.d.ts +10 -35
  20. package/build/ts/kcmd/semantic/deploy_knowledge_catalog.js +105 -162
  21. package/build/ts/kcmd/semantic/deploy_spanner.js +7 -14
  22. package/build/ts/kcmd/semantic/deployment_target.d.ts +1 -4
  23. package/build/ts/kcmd/semantic/deployment_target.js +6 -62
  24. package/build/ts/kcmd/semantic/graph_ddl.d.ts +10 -0
  25. package/build/ts/kcmd/semantic/graph_ddl.js +152 -59
  26. package/build/ts/kcmd/semantic/import_model.d.ts +26 -4
  27. package/build/ts/kcmd/semantic/import_model.js +531 -111
  28. package/build/ts/kcmd/semantic/ir.d.ts +2 -4
  29. package/build/ts/kcmd/semantic/kc_converter.d.ts +7 -1
  30. package/build/ts/kcmd/semantic/kc_converter.js +32 -27
  31. package/build/ts/kcmd/semantic/kc_custom_types.d.ts +5 -12
  32. package/build/ts/kcmd/semantic/kc_custom_types.js +43 -114
  33. package/build/ts/kcmd/semantic/kc_ids.d.ts +4 -5
  34. package/build/ts/kcmd/semantic/kc_ids.js +4 -7
  35. package/build/ts/kcmd/semantic/knowledge_catalog.d.ts +0 -3
  36. package/build/ts/kcmd/semantic/knowledge_catalog.js +3 -86
  37. package/build/ts/kcmd/semantic/loader.d.ts +7 -11
  38. package/build/ts/kcmd/semantic/loader.js +282 -171
  39. package/build/ts/kcmd/semantic/osi_converter.js +14 -65
  40. package/build/ts/kcmd/semantic/parameters.d.ts +1 -0
  41. package/build/ts/kcmd/semantic/parameters.js +22 -3
  42. package/build/ts/kcmd/semantic/pull_kc.d.ts +1 -5
  43. package/build/ts/kcmd/semantic/pull_kc.js +38 -112
  44. package/build/ts/kcmd/semantic/resolve_inheritance.d.ts +7 -2
  45. package/build/ts/kcmd/semantic/resolve_inheritance.js +37 -21
  46. package/build/ts/kcmd/semantic/resolve_profiles.d.ts +51 -10
  47. package/build/ts/kcmd/semantic/resolve_profiles.js +213 -32
  48. package/build/ts/kcmd/semantic/runtime/runtime.d.ts +4 -12
  49. package/build/ts/kcmd/semantic/runtime/runtime.js +4 -21
  50. package/build/ts/kcmd/semantic/runtime/store.d.ts +14 -19
  51. package/build/ts/kcmd/semantic/runtime/store.js +21 -37
  52. package/build/ts/kcmd/semantic/source_dialect.d.ts +9 -2
  53. package/build/ts/kcmd/semantic/source_dialect.js +83 -33
  54. package/build/ts/kcmd/semantic/source_entries.d.ts +1 -1
  55. package/build/ts/kcmd/semantic/source_entries.js +95 -36
  56. package/build/ts/kcmd/semantic/sql_expr_utils.d.ts +13 -7
  57. package/build/ts/kcmd/semantic/sql_expr_utils.js +240 -67
  58. package/build/ts/kcmd/semantic/sql_parser.d.ts +5 -2
  59. package/build/ts/kcmd/semantic/sql_parser.js +22 -0
  60. package/build/ts/kcmd/semantic/transpile.d.ts +12 -10
  61. package/build/ts/kcmd/semantic/transpile.js +79 -61
  62. package/build/ts/kcmd/semantic/validate.d.ts +37 -2
  63. package/build/ts/kcmd/semantic/validate.js +467 -126
  64. package/build/ts/kcmd/snapshot.d.ts +4 -0
  65. package/build/ts/kcmd/snapshot.js +7 -0
  66. package/build/ts/kcmd/sync.js +44 -7
  67. package/package.json +1 -1
@@ -16,12 +16,19 @@
16
16
  // table was created, so after a failover an older table's entry can sit in
17
17
  // a former primary's region, now a secondary's. The entry is looked up in
18
18
  // each replica's region, the primary's first, until one finds it.
19
+ // getHiveCatalog reads a Hive catalog. Its tables are written with
20
+ // `databases/`.
19
21
  // - Spanner: the default leader region of the instance's config, which
20
22
  // getInstance reads the config's name from. A regional config is named for
21
23
  // its region, `regional-us-central1` for us-central1, so it is not read;
22
24
  // any other config is, with getInstanceConfig: `nam3` gives us-east4. That
23
25
  // read needs spanner.instanceConfigs.get, which the Spanner database roles
24
- // lack. A database's own default leader does not move the entry.
26
+ // lack. When it is refused, the entry is looked up in each region the
27
+ // instance has replicas in, which getInstance lists, in the order listed,
28
+ // until one finds it: the default leader's region is one of them, and the
29
+ // list marks no leader (live, 2026-11-04). The instance's
30
+ // `resourceLocation`, such as `us`, is not a Dataplex region. A database's
31
+ // own default leader does not move the entry.
25
32
  // - AlloyDB: the location the source names.
26
33
  //
27
34
  // Dataplex keeps locations in lower case. A lookup in the wrong location
@@ -34,8 +41,10 @@
34
41
  // the entry was never looked up, so the message says which read it was, and
35
42
  // what the service said: Spanner answers 403 for a project the caller may not
36
43
  // read, whether or not the instance exists, so a 403 does not say the table
37
- // is missing. A read that fails another way says to try again when a retry
38
- // may pass (a thrown call, a 429 or a 5xx), and to fix the source otherwise.
44
+ // is missing. A refused Spanner instance config read is the exception: the
45
+ // entry is then looked up in each replica region the instance lists. A read
46
+ // that fails another way says to try again when a retry may pass (a thrown
47
+ // call, a 429 or a 5xx), and to fix the source otherwise.
39
48
  //
40
49
  // A Cloud SQL or third-party source is not looked up here: sourceDialects
41
50
  // already looks up its entry directly, and fails the push when it is missing.
@@ -65,10 +74,11 @@ const resolve_profiles_1 = require("./resolve_profiles");
65
74
  // - A job with no location runs in the primary.
66
75
  //
67
76
  // Reads each table's location and looks its entry up once per location it
68
- // may sit in, and reads each catalog and instance once; tables are checked in
69
- // parallel, and a table's locations in turn. Skips an abstract entity, an
70
- // entity with no source, and a source that names no table this module knows
71
- // (a Cloud SQL or third-party table, or a legacy bare name). Never throws.
77
+ // may sit in, and reads each instance once and each catalog at most once
78
+ // through each API; tables are checked in parallel, and a table's locations in
79
+ // turn. Skips an abstract entity, an entity with no source, and a source that
80
+ // names no table this module knows (a Cloud SQL or third-party table). Never
81
+ // throws.
72
82
  async function sourceEntryErrors(model, readers) {
73
83
  const uses = [];
74
84
  const use = (entity, profile, source) => {
@@ -149,24 +159,40 @@ async function problemOf(table, located, readers) {
149
159
  `exist, or you may not be allowed to read its catalog entry.${table.system === 'alloydb' ?
150
160
  ' A new AlloyDB cluster can take hours to appear in the ' +
151
161
  'catalog after Dataplex integration is turned on.' :
152
- ''} Fix the source, or pass --skip-source-check.`;
162
+ ''}${locations.unfound ?? ''} Fix the source, or pass ` +
163
+ `--skip-source-check.`;
153
164
  }
154
165
  // `location`, which is kept in lower case, as BigQuery writes it: a
155
166
  // multi-region, which has no hyphen, in upper case (`US`, `EU`).
156
167
  function shown(location) {
157
168
  return location.includes('-') ? location : location.toUpperCase();
158
169
  }
159
- // Said when reading an instance config is refused (live on
160
- // bigquery-public-data and dataplex-staging-types, 2026-10-27: Spanner answers
161
- // 403 naming this permission, even for a Google-managed config such as nam3).
170
+ // Said when reading an instance config is refused and the instance lists no
171
+ // replica regions to look in instead (live on bigquery-public-data and
172
+ // dataplex-staging-types, 2026-10-27: Spanner answers 403 naming this
173
+ // permission, even for a Google-managed config such as nam3).
162
174
  const CONFIG_REFUSED_HINT = ' Reading it needs the permission ' +
163
175
  'spanner.instanceConfigs.get on the project, which roles/spanner.viewer ' +
164
176
  'includes and the Spanner database roles do not.';
177
+ // Said when reading an instance is refused: a Spanner database role gives
178
+ // spanner.instances.get only when granted on the instance or its project
179
+ // (from review on #630, 2026-11-04).
180
+ const INSTANCE_REFUSED_HINT = ' The Spanner database roles include ' +
181
+ 'spanner.instances.get, but a role granted on one database does not ' +
182
+ 'apply to its instance.';
183
+ // Said when a BigLake catalog read fails in a way a retry cannot fix and the
184
+ // other kind of catalog's API finds the catalog. The status alone does not
185
+ // tell: live, the Hive API answers 400 for an Iceberg REST catalog id with a
186
+ // hyphen and for a Hive catalog id in capitals.
187
+ const HIVE_FORM_HINT = ` A table in a Hive catalog is written with 'databases/<database>'.`;
188
+ const ICEBERG_FORM_HINT = ` A table in an Iceberg REST catalog is written ` +
189
+ `with 'namespaces/<namespace>'.`;
165
190
  // The locations where `table`'s catalog entry may sit, as Dataplex writes
166
- // them, in lower case, in the order to look in: one for a BigQuery, Spanner
167
- // or AlloyDB table, and one per replica for a BigLake table. `cache` keeps
168
- // each catalog and instance's read, so each is read once; each table is read
169
- // once already.
191
+ // them, in lower case, in the order to look in: one for a BigQuery or AlloyDB
192
+ // table; one for a Spanner table, or one per replica region of its instance
193
+ // when reading its config is refused; and one per replica for a BigLake table.
194
+ // `cache` keeps each instance's read and each catalog's read through each
195
+ // API, so each is made once; each table is read once already.
170
196
  async function locationsOf(table, readers, cache) {
171
197
  const once = (key, readIt) => {
172
198
  if (!cache.has(key))
@@ -184,16 +210,23 @@ async function locationsOf(table, readers, cache) {
184
210
  located(res.result.location, what, 'no location') :
185
211
  res;
186
212
  }
187
- case 'biglake':
188
- return once(`biglake/${project}/${table.catalog}`, async () => {
189
- const what = `reading the region of catalog '${table.catalog}'`;
190
- const res = await readLocation(what, 'a catalog', () => readers.biglake.getCatalog(project, table.catalog));
191
- return 'result' in res ? replicaRegions(res.result, what) : res;
192
- });
213
+ case 'biglake': {
214
+ const readCatalog = (hive) => once(`biglake/${project}/${table.catalog}/${hive}`, () => hive ? readers.biglake.getHiveCatalog(project, table.catalog) :
215
+ readers.biglake.getCatalog(project, table.catalog));
216
+ const kind = table.hive ? 'Hive catalog' : 'catalog';
217
+ const what = `reading the region of ${kind} '${table.catalog}'`;
218
+ // When the read fails and the other API finds the catalog, the source
219
+ // names a table in the other kind of catalog.
220
+ const otherForm = async () => 'result' in await (0, catalog_entry_1.read)(what, () => readCatalog(!table.hive)) ?
221
+ (table.hive ? ICEBERG_FORM_HINT : HIVE_FORM_HINT) :
222
+ '';
223
+ const res = await readLocation(what, `a ${kind}`, () => readCatalog(table.hive), otherForm);
224
+ return 'result' in res ? replicaRegions(res.result, what) : res;
225
+ }
193
226
  case 'spanner':
194
227
  return once(`spanner/${project}/${table.instance}`, async () => {
195
228
  const what = `reading instance '${table.instance}'`;
196
- const instance = await readLocation(what, 'an instance', () => readers.spanner.getInstance(project, table.instance));
229
+ const instance = await readLocation(what, 'an instance', () => readers.spanner.getInstance(project, table.instance), status => status === 403 ? INSTANCE_REFUSED_HINT : '');
197
230
  if ('problem' in instance)
198
231
  return instance;
199
232
  const configName = instance.result.config;
@@ -207,7 +240,23 @@ async function locationsOf(table, readers, cache) {
207
240
  if (region)
208
241
  return { result: [region[1]] };
209
242
  const configWhat = `reading instance config '${configName}'`;
210
- const config = await readLocation(configWhat, 'an instance config', () => readers.spanner.getInstanceConfig(configName), CONFIG_REFUSED_HINT);
243
+ const res = await (0, catalog_entry_1.read)(configWhat, () => readers.spanner.getInstanceConfig(configName));
244
+ // The default leader's region is one of the instance's replica
245
+ // regions, so when the config read is refused, each is looked in.
246
+ if ('unreadable' in res && res.status === 403) {
247
+ const regions = (instance.result.replicaComputeCapacity ?? [])
248
+ .map(r => r.replicaSelection?.location)
249
+ .filter((l) => !!l);
250
+ if (regions.length) {
251
+ return {
252
+ result: regions,
253
+ unfound: ` Instance config '${configName}' could not be read ` +
254
+ `(HTTP 403), so the entry was looked up in each of the ` +
255
+ `instance's replica regions: ${regions.join(', ')}.`,
256
+ };
257
+ }
258
+ }
259
+ const config = await checked(res, configWhat, 'an instance config', status => status === 403 ? CONFIG_REFUSED_HINT : '');
211
260
  // A config marks exactly one replica as its default leader.
212
261
  return 'result' in config ?
213
262
  located(config.result.replicas?.find(r => r.defaultLeaderLocation)
@@ -216,22 +265,31 @@ async function locationsOf(table, readers, cache) {
216
265
  });
217
266
  }
218
267
  }
219
- // Makes the location read `call`, which `what` names. A 404 says the source
220
- // names `kind`, such as `a table`, that does not exist (the services answer
221
- // 404 for a missing table, dataset, catalog or instance; live, 2026-10-27 and
268
+ // Makes the location read `call`, which `what` names, and judges its answer
269
+ // with checked.
270
+ async function readLocation(what, kind, call, hintFor) {
271
+ return checked(await (0, catalog_entry_1.read)(what, call), what, kind, hintFor);
272
+ }
273
+ // What the location read `what` gave in `res`. A 404 says the source names
274
+ // `kind`, such as `a table`, that does not exist (the services answer 404 for
275
+ // a missing table, dataset, catalog or instance; live, 2026-10-27 and
222
276
  // 2026-11-03). A 403 says the read was refused, followed by the service's
223
- // message and `refusedHint`.
224
- async function readLocation(what, kind, call, refusedHint = '') {
225
- const res = await (0, catalog_entry_1.read)(what, call);
277
+ // message. When a retry cannot pass, `hintFor` gives a sentence to add for the
278
+ // status.
279
+ async function checked(res, what, kind, hintFor = () => '') {
226
280
  if ('result' in res)
227
281
  return res;
282
+ const { status } = res;
283
+ const hint = status === undefined || (0, api_1.isTransientStatus)(status) ?
284
+ '' :
285
+ await hintFor(status);
228
286
  if ('error' in res)
229
- return failedProblem(res);
287
+ return failedProblem(res, hint);
230
288
  return {
231
289
  problem: res.status === 404 ?
232
290
  `names ${kind} that does not exist: ${what} found nothing (HTTP ` +
233
- `404). Fix the source, or pass --skip-source-check.` :
234
- `could not be checked: ${what} was refused (HTTP 403).${res.message ? ` ${res.message}` : ''}${refusedHint} The ` +
291
+ `404).${hint} Fix the source, or pass --skip-source-check.` :
292
+ `could not be checked: ${what} was refused (HTTP 403).${res.message ? ` ${res.message}` : ''}${hint} The ` +
235
293
  `source's catalog entry was not looked up. Fix the access or ` +
236
294
  `the source, or pass --skip-source-check.`,
237
295
  };
@@ -265,13 +323,14 @@ function located(location, what, missing) {
265
323
  }
266
324
  // The problem for the failed read `res`. A thrown call, a 429 or a 5xx may
267
325
  // pass on retry; any other status, such as BigQuery's 400 for an invalid
268
- // table ID, never will, so the source must change.
269
- function failedProblem(res) {
326
+ // table ID, never will, so the source must change, and `hint` follows the
327
+ // service's answer.
328
+ function failedProblem(res, hint = '') {
270
329
  return res.status === undefined || (0, api_1.isTransientStatus)(res.status) ?
271
330
  transientProblem(res.error) :
272
331
  {
273
- problem: `could not be checked: ${res.error} Fix the source, or ` +
274
- `pass --skip-source-check.`,
332
+ problem: `could not be checked: ${res.error}${hint} Fix the source, ` +
333
+ `or pass --skip-source-check.`,
275
334
  };
276
335
  }
277
336
  // The problem for a read that failed with `detail`, which may pass on retry.
@@ -1,17 +1,23 @@
1
1
  import { Field } from './ir';
2
2
  import { SqlColumn } from './sql_parser';
3
- export declare const STRING_LITERAL: RegExp;
4
- export declare function escapeRegExp(s: string): string;
5
- export declare function blankStringLiterals(expression: string): string;
6
- export declare function mapOutsideStringLiterals(expression: string, fn: (segment: string) => string): string;
7
- export declare function referencedEntityNames(expression: string, entityNames: string[]): string[];
8
- export declare function stripQualifier(expression: string, entity: string): string;
3
+ export declare function unusedName(text: string, base: string): string;
4
+ export declare function withoutQualifier(expression: string, qualifier: string, dialect?: string): string;
5
+ export declare function withSiblingsSpliced(expression: string, qualifier: string, siblings: ReadonlyMap<string, string>): string;
6
+ export declare function qualifiedColumns(expression: string, qualifier: string, dialect?: string): string[] | undefined;
7
+ export declare function selfReferenceMessage(owner: string, cycle: string[], where?: string): string;
8
+ export declare function requalified(expression: string, from: string, to: string, dialect?: string): string;
9
+ export declare function measureEntities(expression: string, entityNames: ReadonlyArray<string>, entity?: string): string[];
9
10
  export declare function sqlWithoutComments(text: string, dialect?: string): string;
10
11
  export declare function fieldBindingWithoutComments(field: Pick<Field, 'expression' | 'importedExpression' | 'importedDialect'>, targetDialect?: string): string | undefined;
11
12
  export declare function columnReferences(expression: string, dialect: string): ReadonlyArray<SqlColumn> | undefined;
12
- export declare function columnsForSource(text: string, dialect: string, impliedDialect?: string): ReadonlyArray<SqlColumn> | undefined;
13
+ export declare function columnsForSource(text: string, dialect: string, impliedDialect?: string | null): ReadonlyArray<SqlColumn> | undefined;
13
14
  export declare function entitiesReached(columns: ReadonlyArray<SqlColumn>, entityNames: ReadonlyArray<string> | ReadonlySet<string>): string[];
14
15
  export declare function fieldsReadOn(columns: ReadonlyArray<SqlColumn>, entityName: string): string[];
15
16
  export declare const SUPPORTED_AGGREGATES: string[];
17
+ export declare function singleAggregate(expression: string, dialect: string): {
18
+ fn: string;
19
+ operand: string;
20
+ distinct: boolean;
21
+ } | undefined;
16
22
  export declare function isSingleAggregate(expression: string, dialect: string): boolean;
17
23
  export declare function isColumnName(value: unknown): boolean;
@@ -2,87 +2,209 @@
2
2
  // Helpers for reading and rewriting entity-qualified SQL expressions across the
3
3
  // semantic-model layer.
4
4
  //
5
- // Expressions reference columns as `<Entity>.<column>`. The SQL parser
5
+ // An expression names a field of its entity as `<Entity>.<field>`, and a
6
+ // column of its table without a qualifier. The SQL parser
6
7
  // (`columnReferences`, `entitiesReached`, `fieldsReadOn`) reads column and
7
8
  // entity references across dialects for the loader, validator, profile checks,
8
- // pull reader, and transpiler. The graph generators still use the literal-
9
- // aware regex helpers (`referencedEntityNames`, `stripQualifier`).
9
+ // pull reader, and transpiler. The graph generators read them with the parser
10
+ // too: `measureEntities` places a measure, `singleAggregate` finds its
11
+ // aggregate, `withoutQualifier` makes an expression table-local,
12
+ // `qualifiedColumns` finds the sibling fields it names and
13
+ // `withSiblingsSpliced` puts their SQL in their place, and `requalified` moves
14
+ // an inherited expression onto the entity inheriting it.
10
15
  //
11
16
  Object.defineProperty(exports, "__esModule", { value: true });
12
- exports.SUPPORTED_AGGREGATES = exports.STRING_LITERAL = void 0;
13
- exports.escapeRegExp = escapeRegExp;
14
- exports.blankStringLiterals = blankStringLiterals;
15
- exports.mapOutsideStringLiterals = mapOutsideStringLiterals;
16
- exports.referencedEntityNames = referencedEntityNames;
17
- exports.stripQualifier = stripQualifier;
17
+ exports.SUPPORTED_AGGREGATES = void 0;
18
+ exports.unusedName = unusedName;
19
+ exports.withoutQualifier = withoutQualifier;
20
+ exports.withSiblingsSpliced = withSiblingsSpliced;
21
+ exports.qualifiedColumns = qualifiedColumns;
22
+ exports.selfReferenceMessage = selfReferenceMessage;
23
+ exports.requalified = requalified;
24
+ exports.measureEntities = measureEntities;
18
25
  exports.sqlWithoutComments = sqlWithoutComments;
19
26
  exports.fieldBindingWithoutComments = fieldBindingWithoutComments;
20
27
  exports.columnReferences = columnReferences;
21
28
  exports.columnsForSource = columnsForSource;
22
29
  exports.entitiesReached = entitiesReached;
23
30
  exports.fieldsReadOn = fieldsReadOn;
31
+ exports.singleAggregate = singleAggregate;
24
32
  exports.isSingleAggregate = isSingleAggregate;
25
33
  exports.isColumnName = isColumnName;
26
34
  const ir_1 = require("./ir");
35
+ const sql_identifiers_1 = require("./sql_identifiers");
27
36
  const sql_parser_1 = require("./sql_parser");
28
- // Matches a single- or double-quoted SQL string literal, honoring backslash
29
- // escapes. (Triple-quoted / raw literals are uncommon in these expressions and
30
- // are treated as ordinary text.)
31
- exports.STRING_LITERAL = /'(?:\\.|[^'\\])*'|"(?:\\.|[^"\\])*"/g;
32
- // Escapes a string so it can be embedded literally in a RegExp.
33
- function escapeRegExp(s) {
34
- return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
35
- }
36
- // Replaces string-literal contents with blanks of equal length, so scanning sees
37
- // literal-free text without shifting any offsets.
38
- function blankStringLiterals(expression) {
39
- return expression.replace(exports.STRING_LITERAL, m => ' '.repeat(m.length));
40
- }
41
- // Applies `fn` only to the parts of `expression` that lie outside string
42
- // literals, leaving each literal verbatim.
43
- function mapOutsideStringLiterals(expression, fn) {
37
+ // The dialect a graph generator reads `dialect` in: an ANSI_SQL text, or a
38
+ // text with no dialect, is read as BigQuery.
39
+ function graphReading(dialect) {
40
+ return !dialect || dialect === 'ANSI_SQL' ? 'BIGQUERY' : dialect;
41
+ }
42
+ // Each `<name>.` qualifier that `expression`, read in `dialect`, writes before
43
+ // a column, for a name in `names`, as indexes into the text's tokens, in the
44
+ // order written. A name matches case-sensitively, bare or quoted as the
45
+ // dialect quotes an identifier, and not after a dot, so the second `orders` in
46
+ // the struct path `orders.orders.id` is not one. The SQL parser decides which
47
+ // candidates are qualifiers, so a string literal, a comment or a function
48
+ // namespace such as `NET.` in `NET.HOST(orders.url)` holds none. The parser
49
+ // reads each candidate as a placeholder name, so one it reserves as a keyword,
50
+ // such as `Date` in `Date.days`, still reads as a qualifier. When the parser
51
+ // cannot read the text even so, every candidate is taken as a qualifier.
52
+ // Undefined when the engine cannot tokenize the text.
53
+ function qualifiersIn(expression, names, dialect) {
54
+ const tokens = (0, sql_parser_1.sqlTokens)(expression, dialect);
55
+ if (!tokens)
56
+ return undefined;
57
+ const chars = Array.from(expression);
58
+ const asWritten = (t) => chars.slice(t.span.start, t.span.end).join('');
59
+ const candidates = [];
60
+ tokens.forEach((t, i) => {
61
+ if (names.has(t.text) && tokens[i + 1]?.token_type === 'DOT' &&
62
+ tokens[i - 1]?.token_type !== 'DOT' &&
63
+ (t.token_type === 'QUOTED_IDENTIFIER' || asWritten(t) === t.text)) {
64
+ candidates.push(i);
65
+ }
66
+ });
67
+ if (!candidates.length)
68
+ return { tokens, at: [] };
69
+ // A placeholder stem the text does not hold.
70
+ const stem = unusedName(expression, 'kcmd_qualifier_');
71
+ let text = '';
72
+ let end = 0;
73
+ candidates.forEach((i, k) => {
74
+ text += chars.slice(end, tokens[i].span.start).join('') + stem + k;
75
+ end = tokens[i].span.end;
76
+ });
77
+ text += chars.slice(end).join('');
78
+ const columns = columnReferences(text, dialect);
79
+ const at = !columns ? candidates : candidates.filter((_, k) => columns.some(c => c.qualifier?.toLowerCase() === stem + k));
80
+ return { tokens, at };
81
+ }
82
+ // `base`, a lowercase name, with underscores added until `text` does not hold
83
+ // it in any case.
84
+ function unusedName(text, base) {
85
+ let name = base;
86
+ while (text.toLowerCase().includes(name))
87
+ name += '_';
88
+ return name;
89
+ }
90
+ // `expression` without each `<qualifier>.` it writes before a column (see
91
+ // qualifiersIn), so it reads the columns of the qualifier's own table. The
92
+ // rest of the text stays as written. The text is read in `dialect`; an
93
+ // ANSI_SQL text, or one with no dialect, is read as BigQuery, as the graph
94
+ // generators read it. Returns `expression` itself when it writes no such
95
+ // qualifier or the engine cannot tokenize it.
96
+ function withoutQualifier(expression, qualifier, dialect) {
97
+ return spliced(expression, qualifier, graphReading(dialect), new Map());
98
+ }
99
+ // `expression`, BigQuery SQL on `qualifier`'s table, as withoutQualifier gives
100
+ // it, except that each `<qualifier>.<field>` whose field `siblings` holds is
101
+ // replaced, whole, by the SQL it maps to, as given. The graph generators put
102
+ // the SQL of each sibling field an expression names in its place so (see
103
+ // graph_ddl.fieldSql).
104
+ function withSiblingsSpliced(expression, qualifier, siblings) {
105
+ return spliced(expression, qualifier, 'BIGQUERY', siblings);
106
+ }
107
+ // `expression`, read in `dialect`, without each `<qualifier>.` it writes
108
+ // before a column (see qualifiersIn), and with each `<qualifier>.<column>`
109
+ // whose column `replacements` holds replaced, whole, by the SQL it maps to.
110
+ function spliced(expression, qualifier, dialect, replacements) {
111
+ const read = qualifiersIn(expression, new Set([qualifier]), dialect);
112
+ if (!read?.at.length)
113
+ return expression;
114
+ const { tokens, at } = read;
115
+ const chars = Array.from(expression);
44
116
  let out = '';
45
- let last = 0;
46
- for (const m of expression.matchAll(exports.STRING_LITERAL)) {
47
- out += fn(expression.slice(last, m.index));
48
- out += m[0];
49
- last = m.index + m[0].length;
117
+ let end = 0;
118
+ for (const i of at) {
119
+ out += chars.slice(end, tokens[i].span.start).join('');
120
+ const column = tokens[i + 2];
121
+ const replacement = column && replacements.get(nameOf(column, chars));
122
+ if (replacement !== undefined) {
123
+ out += replacement;
124
+ end = column.span.end;
125
+ }
126
+ else {
127
+ // The qualifier, its dot, and what lies between the dot and the column.
128
+ end = column?.span.start ?? tokens[i + 1].span.end;
129
+ }
50
130
  }
51
- out += fn(expression.slice(last));
52
- return out;
53
- }
54
- // Builds a regex matching an `<entity>.` qualifier, including the BigQuery
55
- // backtick-quoted form (`` `entity`. ``). A negative lookbehind keeps the name
56
- // from matching inside a larger identifier (e.g. `customer_orders.`) or after a
57
- // dot in a struct path (e.g. `orders.customer.id`), and the optional backticks
58
- // let it match whether or not the identifier is quoted.
59
- function entityQualifier(name, flags = '') {
60
- return new RegExp(`(?<![\\w\`.])\`?${escapeRegExp(name)}\`?\\.`, flags);
61
- }
62
- // Returns the entity names whose `<name>.` qualifier appears in an expression,
63
- // in the order they first appear, ignoring text inside string literals.
64
- // Matching is case-sensitive (the emitter's behavior).
65
- function referencedEntityNames(expression, entityNames) {
66
- const scannable = blankStringLiterals(expression);
67
- const hits = [];
68
- for (const name of entityNames) {
69
- const m = entityQualifier(name).exec(scannable);
70
- if (m)
71
- hits.push({ name, at: m.index });
131
+ return out + chars.slice(end).join('');
132
+ }
133
+ // The column written after each `<qualifier>.` that `expression` writes before
134
+ // a column (see qualifiersIn), without its quotes, in the order written, so
135
+ // ``orders.a + `orders`.`b`.c`` gives `a` and `b`. The text is read in
136
+ // `dialect`, as withoutQualifier reads it, so a qualifier the parser reserves,
137
+ // such as `Date` in `Date.days`, gives its column too. Undefined when the
138
+ // engine cannot tokenize the text.
139
+ function qualifiedColumns(expression, qualifier, dialect) {
140
+ const read = qualifiersIn(expression, new Set([qualifier]), graphReading(dialect));
141
+ if (!read)
142
+ return undefined;
143
+ const chars = Array.from(expression);
144
+ return read.at.filter(i => read.tokens[i + 2])
145
+ .map(i => nameOf(read.tokens[i + 2], chars));
146
+ }
147
+ // The name token `t` of the text whose characters are `chars` gives, as
148
+ // written, without its quotes.
149
+ function nameOf(t, chars) {
150
+ return t.token_type === 'QUOTED_IDENTIFIER' ?
151
+ t.text :
152
+ chars.slice(t.span.start, t.span.end).join('');
153
+ }
154
+ // The error for a field of `owner` that names itself. `cycle` starts at that
155
+ // field and runs through the fields it names back to it, and `where` follows
156
+ // the field's name, as ` in model 'm' (doc)` does. The graph renders a field
157
+ // that names a sibling field from that sibling's own SQL (see
158
+ // graph_ddl.fieldSql), so a field that reaches itself has none.
159
+ function selfReferenceMessage(owner, cycle, where = '') {
160
+ const [field, ...through] = cycle;
161
+ return `field '${owner}.${field}'${where} names itself` +
162
+ (through.length ?
163
+ ` through ${through.map(n => `'${owner}.${n}'`).join(', ')}` :
164
+ '') +
165
+ `; a field expression may name other fields of its entity, but not ` +
166
+ `itself, directly or through other fields` +
167
+ (through.length ? '' :
168
+ `; to read the column of that name, write it ` +
169
+ `without the qualifier: '${(0, sql_identifiers_1.quoteIfReserved)(field)}'`);
170
+ }
171
+ // `expression` with each `<from>.` qualifier it writes before a column (see
172
+ // qualifiersIn) renamed `<to>.`, quoted as it was written. The rest of the
173
+ // text stays as written. The text is read in `dialect`, as withoutQualifier
174
+ // reads it. Returns `expression` itself when it writes no such qualifier or
175
+ // the engine cannot tokenize it.
176
+ function requalified(expression, from, to, dialect) {
177
+ const read = qualifiersIn(expression, new Set([from]), graphReading(dialect));
178
+ if (!read?.at.length)
179
+ return expression;
180
+ const { tokens, at } = read;
181
+ const chars = Array.from(expression);
182
+ let out = '';
183
+ let end = 0;
184
+ for (const i of at) {
185
+ const { start, end: after } = tokens[i].span;
186
+ out += chars.slice(end, start).join('') +
187
+ chars.slice(start, after).join('').replace(from, () => to);
188
+ end = after;
72
189
  }
73
- return hits.sort((a, b) => a.at - b.at).map(h => h.name);
190
+ return out + chars.slice(end).join('');
74
191
  }
75
- // Removes the `<entity>.` qualifier (bare or backtick-quoted) so an expression
76
- // references table-local columns, without touching text inside string literals.
77
- function stripQualifier(expression, entity) {
78
- const re = entityQualifier(entity, 'g');
79
- return mapOutsideStringLiterals(expression, seg => seg.replace(re, ''));
192
+ // The entities from `entityNames` whose qualifier a metric's BigQuery SQL
193
+ // `expression` writes before a column (see qualifiersIn), deduplicated in the
194
+ // order written, or else `entity`, the metric's anchor, when given. The
195
+ // BigQuery generator places the metric's measure on the entity this gives
196
+ // when it gives one, and a BigQuery push rejects a metric it gives several or
197
+ // none for (see validate.ts).
198
+ function measureEntities(expression, entityNames, entity) {
199
+ const read = qualifiersIn(expression, new Set(entityNames), 'BIGQUERY');
200
+ const reached = [...new Set(read?.at.map(i => read.tokens[i].text) ?? [])];
201
+ return reached.length || !entity ? reached : [entity];
80
202
  }
81
203
  // `text`, written in `dialect`, with its comments removed. An ANSI_SQL text, or
82
204
  // a text with no dialect, is read as BigQuery, as the loader and the graph
83
205
  // generators read it.
84
206
  function sqlWithoutComments(text, dialect) {
85
- return (0, sql_parser_1.withoutComments)(text, !dialect || dialect === 'ANSI_SQL' ? 'BIGQUERY' : dialect);
207
+ return (0, sql_parser_1.withoutComments)(text, graphReading(dialect));
86
208
  }
87
209
  // `field`'s binding, as fieldBinding gives it, with its comments removed.
88
210
  // Undefined when the field is unbound. The target text is read in
@@ -124,8 +246,11 @@ function columnReferences(expression, dialect) {
124
246
  // an `ANSI_SQL` entry as its own SQL. When the sources do not imply a single
125
247
  // dialect, the generic parser reads the text, and any bare identifier that
126
248
  // BigQuery's parser does not also read as a bare column (such as a `MONTH`
127
- // date part or a double-quoted string literal) is filtered out.
249
+ // date part or a double-quoted string literal) is filtered out. When their
250
+ // dialect is null, which no lookup read, an `ANSI_SQL` text is not read.
128
251
  function columnsForSource(text, dialect, impliedDialect) {
252
+ if (dialect === 'ANSI_SQL' && impliedDialect === null)
253
+ return undefined;
129
254
  if (dialect === 'ANSI_SQL' && impliedDialect) {
130
255
  return columnReferences(text, impliedDialect);
131
256
  }
@@ -165,15 +290,18 @@ function fieldsReadOn(columns, entityName) {
165
290
  // The aggregates BigQuery accepts in a MEASURE, its complete list. BigQuery
166
291
  // rejects any other there, such as COUNTIF or APPROX_COUNT_DISTINCT.
167
292
  exports.SUPPORTED_AGGREGATES = ['SUM', 'AVG', 'COUNT', 'MIN', 'MAX'];
168
- // Whether `expression`, read in `dialect`, is one of SUPPORTED_AGGREGATES over
169
- // one operand, or COUNT(*), with at most DISTINCT or ALL. The operand holds no
170
- // other aggregate, window or subquery.
171
- function isSingleAggregate(expression, dialect) {
293
+ // When `expression`, read in `dialect`, is one of SUPPORTED_AGGREGATES over
294
+ // one operand, or COUNT(*), with at most DISTINCT or ALL, and the operand holds
295
+ // no other aggregate, window or subquery: the function's name as written, the
296
+ // operand as written without the parentheses around it (so
297
+ // `COUNT(DISTINCT(x))` gives `x`), and whether the call is DISTINCT. COUNT(*)
298
+ // gives the operand `*`. Undefined otherwise.
299
+ function singleAggregate(expression, dialect) {
172
300
  const node = unwrapped((0, sql_parser_1.sqlExpressionTree)(expression, dialect));
173
301
  const kind = node && Object.keys(node).length === 1 ? Object.keys(node)[0] :
174
302
  undefined;
175
303
  if (!kind || !exports.SUPPORTED_AGGREGATES.includes(kind.toUpperCase())) {
176
- return false;
304
+ return undefined;
177
305
  }
178
306
  const call = node[kind];
179
307
  const star = kind === 'count' && call.star === true && !call.this;
@@ -181,12 +309,57 @@ function isSingleAggregate(expression, dialect) {
181
309
  if (!star &&
182
310
  (!operand || operand.star || operand.tuple ||
183
311
  nestsAggregateOrSubquery(operand))) {
184
- return false;
312
+ return undefined;
185
313
  }
186
314
  // The parser keeps the call's name in `name` or `original_name`.
187
315
  const known = new Set(['this', 'star', 'distinct', 'name', 'original_name']);
188
- return Object.entries(call).every(([key, v]) => known.has(key) || v === null || v === undefined ||
189
- v === false || (Array.isArray(v) && v.length === 0));
316
+ if (!Object.entries(call).every(([key, v]) => known.has(key) || v === null || v === undefined ||
317
+ v === false || (Array.isArray(v) && v.length === 0))) {
318
+ return undefined;
319
+ }
320
+ // The call as written: its name, `(`, DISTINCT or ALL if written, the
321
+ // operand and `)`, inside any parentheses around the whole call.
322
+ const tokens = unparenthesized((0, sql_parser_1.sqlTokens)(expression, dialect) ?? []);
323
+ if (tokens.length < 4 || tokens[1].token_type !== 'L_PAREN' ||
324
+ tokens[tokens.length - 1].token_type !== 'R_PAREN') {
325
+ return undefined;
326
+ }
327
+ let inner = tokens.slice(2, -1);
328
+ if (inner[0]?.token_type === 'DISTINCT' || inner[0]?.token_type === 'ALL') {
329
+ inner = inner.slice(1);
330
+ }
331
+ inner = unparenthesized(inner);
332
+ if (!inner.length)
333
+ return undefined;
334
+ const chars = Array.from(expression);
335
+ return {
336
+ fn: tokens[0].text,
337
+ operand: chars.slice(inner[0].span.start, inner[inner.length - 1].span.end)
338
+ .join(''),
339
+ distinct: call.distinct === true,
340
+ };
341
+ }
342
+ // Whether `expression`, read in `dialect`, is a single aggregate (see
343
+ // singleAggregate), the form a BigQuery measure takes.
344
+ function isSingleAggregate(expression, dialect) {
345
+ return singleAggregate(expression, dialect) !== undefined;
346
+ }
347
+ // `tokens` without the parentheses around all of them.
348
+ function unparenthesized(tokens) {
349
+ while (tokens.length >= 2 && tokens[0].token_type === 'L_PAREN') {
350
+ let depth = 0;
351
+ let closesAt = -1;
352
+ for (let i = 0; i < tokens.length && closesAt < 0; i++) {
353
+ if (tokens[i].token_type === 'L_PAREN')
354
+ depth++;
355
+ if (tokens[i].token_type === 'R_PAREN' && --depth === 0)
356
+ closesAt = i;
357
+ }
358
+ if (closesAt !== tokens.length - 1)
359
+ break;
360
+ tokens = tokens.slice(1, -1);
361
+ }
362
+ return tokens;
190
363
  }
191
364
  // The parser's tree `node` without the parentheses and comments around it.
192
365
  function unwrapped(node) {