@google-cloud/kcmd 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/README.md +18 -20
  2. package/build/ts/kcmd/cli.js +5842 -5168
  3. package/build/ts/kcmd/gcp/biglake.d.ts +2 -0
  4. package/build/ts/kcmd/gcp/biglake.js +13 -4
  5. package/build/ts/kcmd/gcp/spanner.d.ts +10 -0
  6. package/build/ts/kcmd/gcp/spanner.js +6 -3
  7. package/build/ts/kcmd/layouts/okf.js +49 -7
  8. package/build/ts/kcmd/layouts/semantic-model.d.ts +13 -0
  9. package/build/ts/kcmd/layouts/semantic-model.js +129 -50
  10. package/build/ts/kcmd/okf/concept.d.ts +1 -1
  11. package/build/ts/kcmd/okf/concept.js +6 -0
  12. package/build/ts/kcmd/okf/concept_to_entry.d.ts +2 -0
  13. package/build/ts/kcmd/okf/concept_to_entry.js +29 -2
  14. package/build/ts/kcmd/okf/init_okf_workspace.js +8 -10
  15. package/build/ts/kcmd/okf/okf_metadata.js +17 -0
  16. package/build/ts/kcmd/semantic/bigquery.js +139 -195
  17. package/build/ts/kcmd/semantic/binding.js +14 -19
  18. package/build/ts/kcmd/semantic/deploy_knowledge_catalog.d.ts +10 -35
  19. package/build/ts/kcmd/semantic/deploy_knowledge_catalog.js +92 -121
  20. package/build/ts/kcmd/semantic/graph_ddl.d.ts +10 -0
  21. package/build/ts/kcmd/semantic/graph_ddl.js +152 -59
  22. package/build/ts/kcmd/semantic/import_model.d.ts +26 -4
  23. package/build/ts/kcmd/semantic/import_model.js +526 -103
  24. package/build/ts/kcmd/semantic/kc_converter.d.ts +7 -1
  25. package/build/ts/kcmd/semantic/kc_converter.js +20 -5
  26. package/build/ts/kcmd/semantic/kc_custom_types.d.ts +5 -12
  27. package/build/ts/kcmd/semantic/kc_custom_types.js +16 -93
  28. package/build/ts/kcmd/semantic/kc_ids.d.ts +4 -5
  29. package/build/ts/kcmd/semantic/kc_ids.js +4 -7
  30. package/build/ts/kcmd/semantic/knowledge_catalog.d.ts +0 -3
  31. package/build/ts/kcmd/semantic/knowledge_catalog.js +3 -86
  32. package/build/ts/kcmd/semantic/loader.d.ts +7 -11
  33. package/build/ts/kcmd/semantic/loader.js +284 -148
  34. package/build/ts/kcmd/semantic/osi_converter.js +1 -2
  35. package/build/ts/kcmd/semantic/parameters.d.ts +1 -0
  36. package/build/ts/kcmd/semantic/parameters.js +22 -3
  37. package/build/ts/kcmd/semantic/pull_kc.d.ts +1 -5
  38. package/build/ts/kcmd/semantic/pull_kc.js +38 -112
  39. package/build/ts/kcmd/semantic/resolve_inheritance.d.ts +7 -2
  40. package/build/ts/kcmd/semantic/resolve_inheritance.js +37 -21
  41. package/build/ts/kcmd/semantic/resolve_profiles.d.ts +51 -10
  42. package/build/ts/kcmd/semantic/resolve_profiles.js +213 -32
  43. package/build/ts/kcmd/semantic/runtime/runtime.d.ts +0 -6
  44. package/build/ts/kcmd/semantic/runtime/runtime.js +4 -21
  45. package/build/ts/kcmd/semantic/source_dialect.d.ts +9 -2
  46. package/build/ts/kcmd/semantic/source_dialect.js +83 -33
  47. package/build/ts/kcmd/semantic/source_entries.d.ts +1 -1
  48. package/build/ts/kcmd/semantic/source_entries.js +95 -36
  49. package/build/ts/kcmd/semantic/sql_expr_utils.d.ts +13 -7
  50. package/build/ts/kcmd/semantic/sql_expr_utils.js +240 -67
  51. package/build/ts/kcmd/semantic/sql_parser.d.ts +5 -2
  52. package/build/ts/kcmd/semantic/sql_parser.js +22 -0
  53. package/build/ts/kcmd/semantic/transpile.d.ts +12 -10
  54. package/build/ts/kcmd/semantic/transpile.js +79 -61
  55. package/build/ts/kcmd/semantic/validate.d.ts +37 -2
  56. package/build/ts/kcmd/semantic/validate.js +465 -98
  57. package/build/ts/kcmd/snapshot.d.ts +4 -0
  58. package/build/ts/kcmd/snapshot.js +7 -0
  59. package/build/ts/kcmd/sync.js +44 -7
  60. package/package.json +1 -1
@@ -4,10 +4,11 @@
4
4
  // Runs once over the shared, already-parsed models (see loadSemanticModels)
5
5
  // before any destination leg, so a real `kcmd push` AND a `--validate-only` dry
6
6
  // run enforce the same requirements. Returns one message per violation (an
7
- // empty array means valid); the caller (commands.ts) prints them and aborts the
8
- // push. Kept separate from the loader -- which validates a document against the
9
- // schema -- because these are deployment requirements, not schema rules, and
10
- // they read the GOOGLE deployment-target extension the BigQuery leg owns.
7
+ // empty array means valid); push prints them and aborts, and `kcmd import`
8
+ // reports them. Kept separate from the loader -- which validates a document
9
+ // against the schema -- because these are deployment requirements, not schema
10
+ // rules, and they read the GOOGLE deployment-target extension the BigQuery leg
11
+ // owns.
11
12
  Object.defineProperty(exports, "__esModule", { value: true });
12
13
  exports.validatePushRequirements = validatePushRequirements;
13
14
  exports.validateDeployments = validateDeployments;
@@ -18,12 +19,18 @@ exports.validateActions = validateActions;
18
19
  exports.validateInheritance = validateInheritance;
19
20
  exports.validateExpressionReferences = validateExpressionReferences;
20
21
  exports.deploymentTableErrors = deploymentTableErrors;
22
+ exports.spannerMetricsErrors = spannerMetricsErrors;
23
+ exports.kcV2Aspects = kcV2Aspects;
24
+ exports.withProfileFiles = withProfileFiles;
25
+ exports.localChecks = localChecks;
26
+ exports.pruningLines = pruningLines;
21
27
  exports.validateBigQueryDataSources = validateBigQueryDataSources;
22
- exports.storeMessage = storeMessage;
23
28
  exports.validateSpannerActionStatements = validateSpannerActionStatements;
24
29
  exports.validateBigQueryActionStatements = validateBigQueryActionStatements;
30
+ const context_1 = require("../gcp/context");
25
31
  const bigquery_1 = require("./bigquery");
26
32
  const deploy_bigquery_1 = require("./deploy_bigquery");
33
+ const deploy_spanner_1 = require("./deploy_spanner");
27
34
  const deployment_target_1 = require("./deployment_target");
28
35
  const ir_1 = require("./ir");
29
36
  const kc_ids_1 = require("./kc_ids");
@@ -34,7 +41,6 @@ const resolve_inheritance_1 = require("./resolve_inheritance");
34
41
  const resolve_profiles_1 = require("./resolve_profiles");
35
42
  const sql_expr_utils_1 = require("./sql_expr_utils");
36
43
  const sql_identifiers_1 = require("./sql_identifiers");
37
- const sql_parser_1 = require("./sql_parser");
38
44
  // Checks every model against the push requirements and returns the collected
39
45
  // error messages (empty when all models pass), each tagged with the model's
40
46
  // source document so the author can find it.
@@ -49,7 +55,7 @@ const sql_parser_1 = require("./sql_parser");
49
55
  // caller to print.
50
56
  //
51
57
  // `v2Aspects` says push writes the second-generation Knowledge Catalog types
52
- // (selected via KC_V2_ASPECTS=1).
58
+ // (the CLI sets it unless KC_V2_ASPECTS=0).
53
59
  function validatePushRequirements(models, opts = {}) {
54
60
  const errors = [];
55
61
  for (const { document, model } of models) {
@@ -94,7 +100,7 @@ function validatePushRequirements(models, opts = {}) {
94
100
  const flaggedMetrics = new Set();
95
101
  errors.push(...validateExpressionReferences([{ document, model }], !!opts.fieldsPruned, opts.warnings, flaggedMetrics));
96
102
  if (deployInfo.bigQuery.length > 0) {
97
- errors.push(...validateBigQueryMetrics(model, document, flaggedMetrics));
103
+ errors.push(...measurePlacementErrors(model, document, flaggedMetrics));
98
104
  }
99
105
  // An action reaches Knowledge Catalog only, so its checks are
100
106
  // target-independent: each parameter's type must resolve to something in
@@ -171,7 +177,7 @@ function nameAndDescriptionErrors(model, document, v2Aspects) {
171
177
  // Checks each deployment of `model` against the rules of its engine, under the
172
178
  // profile it names in `model.profiles` or the model file's own bindings, and
173
179
  // returns one message per problem, naming every deployment it applies to.
174
- function validateDeployments(model, document) {
180
+ function validateDeployments(model, document, load = {}) {
175
181
  // The deployments each problem applies to, keyed by the text that follows
176
182
  // their names in the message.
177
183
  const problems = new Map();
@@ -193,7 +199,7 @@ function validateDeployments(model, document) {
193
199
  // The model the deployment's bindings give, less what they leave
194
200
  // unavailable. The completeness check reports a named profile that does
195
201
  // not apply, and the fields its metrics read.
196
- const bound = named ? profiled(model, named) : (0, resolve_profiles_1.pruneUnavailable)(model, resolve_profiles_1.DEFAULT_PROFILE, alloyDb ? 'ALLOYDB' : (0, resolve_profiles_1.sourcesDialect)(model));
202
+ const bound = named ? profiled(model, named, load) : (0, resolve_profiles_1.pruneUnavailable)(model, resolve_profiles_1.DEFAULT_PROFILE, alloyDb ? 'ALLOYDB' : (0, resolve_profiles_1.sourcesDialect)(model));
197
203
  if (!bound)
198
204
  continue;
199
205
  const available = bound.model.metrics ?? [];
@@ -251,9 +257,9 @@ function validateDeployments(model, document) {
251
257
  }
252
258
  // `model` with `profile` applied and what it leaves unavailable pruned, or
253
259
  // undefined when the profile does not apply.
254
- function profiled(model, profile) {
260
+ function profiled(model, profile, load) {
255
261
  try {
256
- return (0, resolve_profiles_1.pruneUnavailable)((0, resolve_profiles_1.applyProfile)(model, profile), profile.name);
262
+ return (0, resolve_profiles_1.pruneUnavailable)((0, resolve_profiles_1.applyProfile)(model, profile, { load }), profile.name);
257
263
  }
258
264
  catch (err) {
259
265
  if (!(err instanceof loader_1.SemanticModelError))
@@ -262,7 +268,7 @@ function profiled(model, profile) {
262
268
  }
263
269
  }
264
270
  // Why `metric` cannot be a BigQuery measure, or undefined when it can. Leaves
265
- // out a metric that validateBigQueryMetrics or the inheritance rules report:
271
+ // out a metric that measurePlacementErrors or the inheritance rules report:
266
272
  // one that reads several entities or none, or one on an abstract or extended
267
273
  // entity.
268
274
  function measureProblem(model, metric) {
@@ -273,15 +279,10 @@ function measureProblem(model, metric) {
273
279
  if (!columns)
274
280
  return undefined;
275
281
  const entities = model.entities ?? [];
276
- const names = entities.map(e => e.name);
277
- // Read `text` (from `expressionForDialect(metric, 'BIGQUERY')`), the SQL the
278
- // BigQuery graph runs.
279
- const matched = (0, sql_expr_utils_1.referencedEntityNames)((0, sql_parser_1.withoutComments)(text, 'BIGQUERY'), names);
280
- if (matched.length > 1 || !matched.length && !metric.entity)
282
+ const placed = (0, sql_expr_utils_1.measureEntities)(text, entities.map(e => e.name), metric.entity);
283
+ if (placed.length !== 1)
281
284
  return undefined;
282
- const reached = (0, sql_expr_utils_1.entitiesReached)(columns, names);
283
- const name = reached.length > 1 ? undefined : reached[0] ?? metric.entity;
284
- const entity = entities.find(e => e.name === name);
285
+ const entity = entities.find(e => e.name === placed[0]);
285
286
  if (!entity || entity.abstract ||
286
287
  entities.some(e => e.extends?.includes(entity.name))) {
287
288
  return undefined;
@@ -295,17 +296,73 @@ function measureProblem(model, metric) {
295
296
  }
296
297
  return undefined;
297
298
  }
298
- // Checks that each bound field of the model file has a text for the dialect
299
- // of its entity's source, and returns one message per field.
299
+ // The BigQuery graph's rules for placing each metric of `model` as a measure,
300
+ // one message per metric that breaks them, skipping each one in
301
+ // `alreadyFlagged`, which validateExpressionReferences has reported. The
302
+ // generator (`bigquery.ts`) places a measure on the one entity whose
303
+ // qualifier its BigQuery SQL writes before a column, or on its anchor when the
304
+ // SQL writes none (see measureEntities), and skips a metric that has no
305
+ // BigQuery or ANSI_SQL text. validateExpressionReferences reads every dialect
306
+ // entry together and lets an authored anchor settle SQL that reads two
307
+ // entities, so it does not catch each metric the generator would skip.
308
+ function measurePlacementErrors(model, document, alreadyFlagged) {
309
+ const errors = [];
310
+ const entityNames = (model.entities ?? []).map(e => e.name);
311
+ for (const metric of model.metrics ?? []) {
312
+ if (alreadyFlagged.has(metric.name))
313
+ continue;
314
+ const prefix = `metric '${metric.name}' in model '${model.name}' (${document})`;
315
+ // The text BigQuery runs: its `BIGQUERY` entry, else its `ANSI_SQL` one,
316
+ // else `expression` for a metric held without a dialect list.
317
+ const bqText = (0, ir_1.expressionForDialect)(metric, 'BIGQUERY');
318
+ if (bqText === undefined && metric.importedExpression !== undefined) {
319
+ errors.push(`${prefix} targets a BigQuery graph with only a '${metric.importedDialect ?? 'vendor'}' expression; supply a ` +
320
+ `'BIGQUERY' or 'ANSI_SQL' expression${(0, resolve_profiles_1.importTranspiles)(metric, 'BIGQUERY', model) ?
321
+ `, or add one with '${(0, resolve_profiles_1.importCommand)('--transpile', model.version)}'` :
322
+ ''}.`);
323
+ continue;
324
+ }
325
+ const placed = (0, sql_expr_utils_1.measureEntities)(bqText ?? '', entityNames, metric.entity);
326
+ if (placed.length > 1) {
327
+ errors.push(`${prefix} targets a BigQuery graph and its BigQuery SQL reads ` +
328
+ `several entities (${placed.join(', ')}); a BigQuery measure ` +
329
+ `belongs to one entity's node table, so scope its expression to ` +
330
+ `one entity.`);
331
+ }
332
+ else if (placed.length === 0) {
333
+ const anchor = model.version === loader_1.OSSIE_VERSION ?
334
+ `'entity' in a GOOGLE custom_extensions block on the metric` :
335
+ `top-level 'entity'`;
336
+ errors.push(`${prefix} targets a BigQuery graph but does not resolve to a ` +
337
+ `single entity; the BigQuery generator places a measure on the ` +
338
+ `entity its BigQuery SQL reads, or on ${anchor}.`);
339
+ }
340
+ }
341
+ return errors;
342
+ }
343
+ // Checks that the model file's Spanner tables are all in one database, so they
344
+ // share one dialect, and that each bound field has a text for the dialect of
345
+ // its entity's source. Returns one message per problem.
300
346
  function validateInlineDialects(model, document) {
301
347
  const errors = [];
348
+ const databases = new Map();
349
+ for (const e of model.entities ?? []) {
350
+ const db = (0, loader_1.databaseOf)(e.authoredSource ?? '');
351
+ if (db?.startsWith('spanner/') && !databases.has(db)) {
352
+ databases.set(db, e.name);
353
+ }
354
+ }
355
+ if (databases.size > 1) {
356
+ const named = [...databases.values()].map(n => `'${n}'`).join(', ');
357
+ errors.push(`model '${model.name}' (${document}) binds entities in more than one ` +
358
+ `Spanner database (${named}); bind the model file's entities to ` +
359
+ `tables in one Spanner database.`);
360
+ }
302
361
  for (const { entity: e, field: f, table: { name: dialect } } of (0, resolve_profiles_1.fieldsOnTables)(model)) {
303
362
  if (!(0, resolve_profiles_1.noTextFor)(f, [dialect]))
304
363
  continue;
305
- const hint = (0, resolve_profiles_1.importTranspiles)(f, dialect) ?
306
- `; give it one, or add one with '${resolve_profiles_1.IMPORT_TRANSPILE_COMMAND}'` : '';
307
364
  errors.push(`field '${e.name}.${f.name}' in model '${model.name}' (${document}) has no expression for ${dialect} or ANSI_SQL, and ` +
308
- `the model file binds entity '${e.name}' to ${/^[AEIOU]/.test(dialect) ? 'an' : 'a'} ${dialect} source${hint}.`);
365
+ `the model file binds entity '${e.name}' to ${/^[AEIOU]/.test(dialect) ? 'an' : 'a'} ${dialect} source${(0, resolve_profiles_1.importHint)(f, dialect, model)}.`);
309
366
  }
310
367
  return errors;
311
368
  }
@@ -1292,6 +1349,94 @@ function fieldNoSqlErrors(model, document) {
1292
1349
  }
1293
1350
  return errors;
1294
1351
  }
1352
+ // The errors for the fields of `entity` that name themselves, directly or
1353
+ // through other fields of `entity`. The graph renders a field that names a
1354
+ // sibling as `<entity>.<field>` from that sibling's own SQL, so a field that
1355
+ // reaches itself that way has no SQL to render. A field expression that writes
1356
+ // its own column as `<entity>.<field>` names itself too: write the bare column
1357
+ // instead. Each dialect is read on its own, from the text an engine of that
1358
+ // dialect renders (its entry, else the ANSI_SQL one), so two texts that no one
1359
+ // engine reads together do not form a cycle.
1360
+ //
1361
+ // `resolved` is `entity` with the fields it inherits, which name its fields
1362
+ // as the graph renders them (see resolve_inheritance.localizeInheritedField),
1363
+ // so a field can reach itself through one it inherits. A cycle is reported on
1364
+ // an entity only when a field on it is bound there, so one an ancestor forms on
1365
+ // its own is reported on that ancestor alone.
1366
+ function selfReferenceErrors(entity, resolved, model, document, impliedDialect) {
1367
+ const own = entity.fields ?? [];
1368
+ const inherited = resolved.fields ?? [];
1369
+ // A field a profile excludes is left off `resolved`, but still checked.
1370
+ const fields = [
1371
+ ...inherited,
1372
+ ...own.filter(f => !inherited.some(r => r.name === f.name)),
1373
+ ];
1374
+ const bound = new Set(own.filter(f => (0, ir_1.isFieldBound)(f)).map(f => f.name));
1375
+ const names = new Set(fields.map(f => f.name));
1376
+ const dialects = new Set(['ANSI_SQL']);
1377
+ for (const f of fields) {
1378
+ for (const d of f.dialects ?? [])
1379
+ dialects.add(d.dialect);
1380
+ if (f.importedDialect)
1381
+ dialects.add(f.importedDialect);
1382
+ }
1383
+ const errors = new Set();
1384
+ for (const dialect of dialects) {
1385
+ // The fields of `entity` each field names, in this dialect, read as the
1386
+ // graph generators read them (see graph_ddl.siblingsRead), so a qualifier
1387
+ // the parser reserves, such as `Date` in `Date.days`, names them too. An
1388
+ // ANSI_SQL text is read in the dialect the sources imply, as
1389
+ // columnsForSource reads it, and not at all when their dialect is null.
1390
+ if (dialect === 'ANSI_SQL' && impliedDialect === null)
1391
+ continue;
1392
+ const readIn = dialect === 'ANSI_SQL' && impliedDialect ? impliedDialect : dialect;
1393
+ const reads = new Map();
1394
+ for (const f of fields) {
1395
+ const text = (0, ir_1.expressionForDialect)(f, dialect);
1396
+ if (text === undefined ||
1397
+ (0, loader_1.noSqlReason)(text, dialect, entity.name) !== undefined) {
1398
+ continue;
1399
+ }
1400
+ reads.set(f.name, [...new Set((0, sql_expr_utils_1.qualifiedColumns)(text, entity.name, readIn) ?? [])]
1401
+ .filter(n => names.has(n)));
1402
+ }
1403
+ for (const f of fields) {
1404
+ const through = pathBack(f.name, reads);
1405
+ if (!through || ![f.name, ...through].some(n => bound.has(n)))
1406
+ continue;
1407
+ errors.add((0, sql_expr_utils_1.selfReferenceMessage)(entity.name, [f.name, ...through], ` in model '${model.name}' (${document})`) +
1408
+ '.');
1409
+ }
1410
+ }
1411
+ return [...errors];
1412
+ }
1413
+ // The shortest run of other fields through which `start` reaches itself in
1414
+ // `reads`, empty when it names itself directly, or undefined when it does not
1415
+ // reach itself.
1416
+ function pathBack(start, reads) {
1417
+ const via = new Map();
1418
+ const queue = [];
1419
+ for (const next of reads.get(start) ?? []) {
1420
+ if (next === start)
1421
+ return [];
1422
+ if (!via.has(next)) {
1423
+ via.set(next, [next]);
1424
+ queue.push(next);
1425
+ }
1426
+ }
1427
+ while (queue.length) {
1428
+ const name = queue.shift();
1429
+ for (const next of reads.get(name) ?? []) {
1430
+ if (next === start)
1431
+ return via.get(name);
1432
+ if (!via.has(next)) {
1433
+ via.set(next, [...via.get(name), next]);
1434
+ queue.push(next);
1435
+ }
1436
+ }
1437
+ }
1438
+ return undefined;
1439
+ }
1295
1440
  // The error for a metric with no SQL, or undefined. A metric is rejected when
1296
1441
  // none of its texts holds SQL, with or without an anchor, since the graph
1297
1442
  // would drop it. It is also rejected when any one of its texts holds none, as
@@ -1332,10 +1477,13 @@ function metricNoSqlError(metric, model, document) {
1332
1477
  // differs only in case from a declared entity.
1333
1478
  // 2. For references to E itself, every field named must be declared or
1334
1479
  // inherited on E (gated on `!fieldsPruned`, and run before pruning).
1480
+ // 3. Reject a field that names itself, directly or through other fields of
1481
+ // E, declared or inherited, since the graph renders a field that names
1482
+ // a sibling from that sibling's SQL (see selfReferenceErrors).
1335
1483
  //
1336
- // Exported so `commands.ts` can also run it on the unpruned model before
1337
- // `pruneUnavailable` drops unbound fields and metrics on a graph push. When
1338
- // given, `flaggedMetrics` collects the name of each metric it reports.
1484
+ // deploymentPlan also runs it on the unpruned model before `pruneUnavailable`
1485
+ // drops unbound fields and metrics on a graph push. When given,
1486
+ // `flaggedMetrics` collects the name of each metric it reports.
1339
1487
  function validateExpressionReferences(models, fieldsPruned = false, warnings = [], flaggedMetrics) {
1340
1488
  const errors = [];
1341
1489
  for (const { document, model } of models) {
@@ -1352,8 +1500,18 @@ function validateExpressionReferences(models, fieldsPruned = false, warnings = [
1352
1500
  continue;
1353
1501
  }
1354
1502
  const entityNames = [...fieldsByEntity.keys()];
1503
+ // Each entity's fields as the graph renders them (see selfReferenceErrors).
1504
+ const resolvedByName = new Map((ifResolvable(() => (0, resolve_inheritance_1.resolveInheritance)(model, undefined, { requalify: true })
1505
+ .model.entities) ??
1506
+ model.entities ?? [])
1507
+ .map(e => [e.name, e]));
1355
1508
  const impliedDialect = (0, resolve_profiles_1.sourcesDialect)(model);
1356
- const addUnreadableWarning = (what, total, unreadDialects) => {
1509
+ const addUnreadableWarning = (what, total, unread) => {
1510
+ // A caller that looked up no table's dialect reads no ANSI_SQL
1511
+ // text when every table needs the lookup, so it warns about none.
1512
+ const unreadDialects = impliedDialect === null ?
1513
+ unread.filter(d => d !== 'ANSI_SQL') :
1514
+ unread;
1357
1515
  if (!unreadDialects.length)
1358
1516
  return;
1359
1517
  if (unreadDialects.length === total) {
@@ -1406,6 +1564,7 @@ function validateExpressionReferences(models, fieldsPruned = false, warnings = [
1406
1564
  }
1407
1565
  }
1408
1566
  }
1567
+ errors.push(...selfReferenceErrors(entity, resolvedByName.get(entity.name) ?? entity, model, document, impliedDialect));
1409
1568
  }
1410
1569
  for (const metric of model.metrics ?? []) {
1411
1570
  const before = errors.length;
@@ -1496,53 +1655,6 @@ function validateExpressionReferences(models, fieldsPruned = false, warnings = [
1496
1655
  }
1497
1656
  return errors;
1498
1657
  }
1499
- // Temporary BigQuery-only metric check, until the BigQuery generator places
1500
- // measures with the SQL parser. Rejects a metric that the generator
1501
- // (`bigquery.ts`) would skip: one with only another engine's SQL, which the
1502
- // generator does not emit, or one it drops because its regex
1503
- // (`referencedEntityNames`) disagrees with the parser or because
1504
- // `metric.entity` is unset. Reports only metrics that
1505
- // `validateExpressionReferences` has not already flagged.
1506
- function validateBigQueryMetrics(model, document, alreadyFlagged) {
1507
- const errors = [];
1508
- const entityNames = (model.entities ?? []).map(e => e.name);
1509
- for (const metric of model.metrics ?? []) {
1510
- if (alreadyFlagged.has(metric.name))
1511
- continue;
1512
- const prefix = `metric '${metric.name}' in model '${model.name}' (${document})`;
1513
- // The text BigQuery runs: its `BIGQUERY` entry, else its `ANSI_SQL` one,
1514
- // else `expression` for a metric held without a dialect list.
1515
- const bqText = (0, ir_1.expressionForDialect)(metric, 'BIGQUERY');
1516
- if (bqText === undefined && metric.importedExpression !== undefined) {
1517
- errors.push(`${prefix} targets a BigQuery graph with only a '${metric.importedDialect ?? 'vendor'}' expression; supply a ` +
1518
- `'BIGQUERY' or 'ANSI_SQL' expression, or add one with '${resolve_profiles_1.IMPORT_TRANSPILE_COMMAND}'.`);
1519
- continue;
1520
- }
1521
- // The generator reads the resolved BigQuery expression (`bqText`) with its
1522
- // comments removed.
1523
- const expr = (0, sql_parser_1.withoutComments)(bqText ?? metric.importedExpression ?? '', 'BIGQUERY');
1524
- let referenced = (0, sql_expr_utils_1.referencedEntityNames)(expr, entityNames);
1525
- if (referenced.length === 0 && metric.entity) {
1526
- referenced = [metric.entity];
1527
- }
1528
- if (referenced.length > 1) {
1529
- errors.push(`${prefix} targets a BigQuery graph and references multiple ` +
1530
- `entities (${referenced.join(', ')}); the BigQuery generator ` +
1531
- `reads every '<entity>.' outside comments as a qualifier, ` +
1532
- `including a function namespace such as 'NET.', so remove the ` +
1533
- `extra '<entity>.' match or scope its expression to one entity.`);
1534
- }
1535
- else if (referenced.length === 0) {
1536
- const anchor = model.version === loader_1.OSSIE_VERSION ?
1537
- `'entity' in a GOOGLE custom_extensions block on the metric` :
1538
- `top-level 'entity'`;
1539
- errors.push(`${prefix} targets a BigQuery graph but does not resolve to a ` +
1540
- `single entity; the BigQuery generator reads '<entity>.' ` +
1541
- `qualifiers in the expression or ${anchor}.`);
1542
- }
1543
- }
1544
- return errors;
1545
- }
1546
1658
  // Every field each entity has, inherited and profile-excluded ones included so
1547
1659
  // an unpruned check before `pruneUnavailable` does not report a profile-excluded
1548
1660
  // inherited field as undeclared. Inheritance is resolved through the same pass
@@ -1608,6 +1720,20 @@ function deploymentTableErrors(document, model, deployment) {
1608
1720
  }
1609
1721
  return errors;
1610
1722
  }
1723
+ // The error for a Spanner `deployment` on the model file's own bindings when
1724
+ // `model` declares metrics, which a Spanner graph cannot hold.
1725
+ function spannerMetricsErrors(document, model, deployment) {
1726
+ if (deployment.engine !== 'spanner' ||
1727
+ deployment.profile !== resolve_profiles_1.DEFAULT_PROFILE || !model.metrics?.length) {
1728
+ return [];
1729
+ }
1730
+ return [
1731
+ `[${document}] deployment '${deployment.name}' targets a Spanner graph ` +
1732
+ `and uses the model file's own bindings, but the model declares ` +
1733
+ `metrics, which a Spanner graph cannot hold; name a profile with ` +
1734
+ `metrics_exclude: "*".`,
1735
+ ];
1736
+ }
1611
1737
  // The database the tables of a deployment target must be in, as databaseOf
1612
1738
  // gives it, and how an error names the target and a table outside it.
1613
1739
  function targetDatabase({ engine, coordinates: c }) {
@@ -1621,6 +1747,265 @@ function targetDatabase({ engine, coordinates: c }) {
1621
1747
  return { database: `${engine}/${name}`, target: `${system} database '${name}'`,
1622
1748
  outside: 'which is not a table in that database' };
1623
1749
  }
1750
+ // Whether Knowledge Catalog push and pull use the second-generation built-in
1751
+ // aspect fields and entry-id layout. They do unless `KC_V2_ASPECTS=0`; every
1752
+ // other value, including unset and `''`, is on. `KC_V2_ASPECTS=0` is a
1753
+ // temporary escape hatch to the first-generation path, removed with that path
1754
+ // (b/575695037).
1755
+ function kcV2Aspects() {
1756
+ return process.env.KC_V2_ASPECTS !== '0';
1757
+ }
1758
+ // Adds the profile files of the model in `base` to `model.profiles`, without
1759
+ // their actions. Returns the error of the first profile file that breaks a
1760
+ // file rule. A profile file that does not load is left out, for
1761
+ // namedProfileErrors to report.
1762
+ function withProfileFiles(base, profileDocs) {
1763
+ const { document, model } = base;
1764
+ const profiles = [...(model.profiles ?? [])];
1765
+ let files;
1766
+ try {
1767
+ files = profileDocs(document);
1768
+ }
1769
+ catch (err) {
1770
+ return { errors: [{ error: err.message || String(err), document }] };
1771
+ }
1772
+ const reserved = (0, resolve_profiles_1.reservedProfileNameError)(document, files);
1773
+ if (reserved)
1774
+ return { errors: [{ error: reserved, document }] };
1775
+ for (const p of files) {
1776
+ try {
1777
+ profiles.push((0, resolve_profiles_1.loadProfileFile)(p.text, p.name, { skipActions: true }));
1778
+ }
1779
+ catch (err) {
1780
+ if (!(err instanceof loader_1.SemanticModelError))
1781
+ throw err;
1782
+ }
1783
+ }
1784
+ return { document, model: profiles.length ? { ...model, profiles } : model };
1785
+ }
1786
+ // Runs push's offline checks that follow withProfileFiles and the UTF-8 and
1787
+ // sidecar checks, through the graph generation. Returns the errors of the
1788
+ // first stage that finds any, or else a plan for each deployment `selected`
1789
+ // names, or for every deployment when it is unset.
1790
+ async function localChecks(scope, warn, selected) {
1791
+ const { docs, loaded, models, profileDocs, declaredBy, load } = scope;
1792
+ const stop = (errors) => ({ errors });
1793
+ const ofModel = (document, ...errors) => errors.map(error => ({ error, document }));
1794
+ const ofProfile = (document, profile, ...errors) => errors.map(error => ({ error, document, profile }));
1795
+ const declaredNames = (document) => declaredBy(docs.find(d => d.name === document)).map(p => p.name);
1796
+ // Every deployment's target must be in an accepted form, and its profile one
1797
+ // the model declares, whether or not this push selects the deployment.
1798
+ const selectedByDoc = new Map();
1799
+ const everyByDoc = new Map();
1800
+ for (const { document, model } of models) {
1801
+ const every = (0, deployment_target_1.resolveModelDeployments)(model);
1802
+ if ('error' in every)
1803
+ return stop(ofModel(document, every.error));
1804
+ const resolved = (0, deployment_target_1.resolveModelDeployments)(model, selected);
1805
+ if ('error' in resolved)
1806
+ return stop(ofModel(document, resolved.error));
1807
+ const undeclared = (0, resolve_profiles_1.undeclaredProfileError)(document, model, declaredNames(document));
1808
+ if (undeclared)
1809
+ return stop(ofModel(document, undeclared));
1810
+ everyByDoc.set(document, every.deployments);
1811
+ selectedByDoc.set(document, resolved.deployments);
1812
+ }
1813
+ // Every named profile is checked on its own against the model file, and so
1814
+ // are the model file's own bindings and each deployment. The catalog's model
1815
+ // is checked as a model that deploys nowhere. A selected deployment's own
1816
+ // checks run only after these pass, so these give every error and every
1817
+ // warning about SQL kcmd cannot read.
1818
+ const profileWarnings = [];
1819
+ const profileErrors = (0, resolve_profiles_1.namedProfileErrors)(docs, new Map(loaded.models.map(m => [m.document, m.model])), profileDocs, declaredBy, profileWarnings, load);
1820
+ for (const { document, model } of models) {
1821
+ profileErrors.push(...ofProfile(document, resolve_profiles_1.DEFAULT_PROFILE, ...validateInlineDialects(model, document), ...validateInheritedColumns(model, document)),
1822
+ // One call per profile, so each error carries its deployments' profile.
1823
+ ...[...new Set(everyByDoc.get(document).map(d => d.profile))].flatMap(profile => ofProfile(document, profile, ...validateDeployments({
1824
+ ...model,
1825
+ deployments: model.deployments.filter(d => (d.profile ?? resolve_profiles_1.DEFAULT_PROFILE) === profile),
1826
+ }, document, load))));
1827
+ const doc = docs.find(d => d.name === document);
1828
+ const declared = declaredBy(doc);
1829
+ const named = declared.some(p => p.name.toLowerCase() !== resolve_profiles_1.DEFAULT_PROFILE);
1830
+ profileWarnings.push(...inlineBindingWarnings(model, document, named));
1831
+ // A profile that no selected deployment applies still goes to Knowledge
1832
+ // Catalog, so its load warnings print here, less those the model file's
1833
+ // load gave, unless a check above recorded an error for that profile. A
1834
+ // selected deployment prints its own profile's warnings.
1835
+ const applied = selectedByDoc.get(document).map(d => d.profile);
1836
+ for (const { name } of declared) {
1837
+ if (applied.includes(name))
1838
+ continue;
1839
+ if (profileErrors.some(e => e.document === document && e.profile === name)) {
1840
+ continue;
1841
+ }
1842
+ profileWarnings.push(...(0, resolve_profiles_1.appliedProfileWarnings)(doc, declared, name, loaded.warnings, load));
1843
+ }
1844
+ }
1845
+ const kcWarnings = [];
1846
+ const kcErrors = models.flatMap(m => ofModel(m.document, ...validatePushRequirements([{ ...m, model: { ...m.model, deployments: [] } }], {
1847
+ warnings: kcWarnings,
1848
+ v2Aspects: kcV2Aspects(),
1849
+ })));
1850
+ for (const w of [...profileWarnings, ...kcWarnings])
1851
+ warn(w);
1852
+ if (profileErrors.length || kcErrors.length) {
1853
+ return stop([...profileErrors, ...kcErrors]);
1854
+ }
1855
+ // Each deployment's tables must be in the database it deploys to, whether
1856
+ // or not this push selects the deployment.
1857
+ const tableErrors = models.flatMap(({ document, model }) => (everyByDoc.get(document) ?? []).flatMap(d => ofProfile(document, d.profile, ...deploymentTableErrors(document, model, d))));
1858
+ if (tableErrors.length)
1859
+ return stop(tableErrors);
1860
+ // Spanner creates no property graph in a PostgreSQL-dialect database, so a
1861
+ // selected Spanner deployment there is refused by the dialect already read.
1862
+ const dialectErrors = models.flatMap(({ document }) => (selectedByDoc.get(document) ?? []).flatMap(d => {
1863
+ if (d.engine !== 'spanner')
1864
+ return [];
1865
+ const { project, instance, database } = d.coordinates;
1866
+ const key = `spanner.googleapis.com/projects/${project}/instances/${instance}/databases/${database}`;
1867
+ if (load.sourceDialects?.get(key) !== 'POSTGRES')
1868
+ return [];
1869
+ return ofProfile(document, d.profile, `[${document}] deployment '${d.name}': database ${project}/${instance}/${database} has the PostgreSQL dialect, and ` +
1870
+ `Spanner creates no property graph in a PostgreSQL-dialect ` +
1871
+ `database. Remove the deployment; push still publishes the ` +
1872
+ `model to Knowledge Catalog.`);
1873
+ }));
1874
+ if (dialectErrors.length)
1875
+ return stop(dialectErrors);
1876
+ const plans = [];
1877
+ for (const base of models) {
1878
+ for (const deployment of selectedByDoc.get(base.document) ?? []) {
1879
+ const plan = await deploymentPlan(base, profileDocs(base.document), deployment, load, warn);
1880
+ if ('errors' in plan) {
1881
+ return stop(ofProfile(base.document, deployment.profile, ...plan.errors));
1882
+ }
1883
+ plans.push(plan);
1884
+ }
1885
+ }
1886
+ return { plans };
1887
+ }
1888
+ // Builds and checks the model `deployment` deploys from `base`, reading its
1889
+ // profile file in `files` again with its actions, and generates its DDL
1890
+ // offline. The profile applies with the source dialects in `load`. Returns
1891
+ // the plan, or the errors that stop the push.
1892
+ async function deploymentPlan(base, files, deployment, load, warn) {
1893
+ const { document, model } = base;
1894
+ const at = `[${document}] deployment '${deployment.name}'`;
1895
+ const fail = (...errors) => ({ errors });
1896
+ const spanner = spannerMetricsErrors(document, model, deployment);
1897
+ if (spanner.length)
1898
+ return fail(...spanner);
1899
+ let applied = model;
1900
+ if (deployment.profile !== resolve_profiles_1.DEFAULT_PROFILE) {
1901
+ // localChecks has checked that the model declares the profile and that
1902
+ // its file loads, so `model.profiles` holds it.
1903
+ let spec = model.profiles.find(p => p.name === deployment.profile);
1904
+ const file = files.find(p => p.name === deployment.profile);
1905
+ const warnings = [];
1906
+ try {
1907
+ // A profile in a GOOGLE block has no file and applies as loaded.
1908
+ if (file)
1909
+ spec = (0, resolve_profiles_1.loadProfileFile)(file.text, file.name);
1910
+ }
1911
+ catch (err) {
1912
+ if (!(err instanceof loader_1.SemanticModelError))
1913
+ throw err;
1914
+ return fail(`[${file.fileName}] ${err.message}`);
1915
+ }
1916
+ try {
1917
+ applied = (0, resolve_profiles_1.applyProfile)(model, spec, { load, warnings });
1918
+ }
1919
+ catch (err) {
1920
+ if (!(err instanceof loader_1.SemanticModelError))
1921
+ throw err;
1922
+ return fail(`${at}: ${err.message}`);
1923
+ }
1924
+ for (const w of warnings) {
1925
+ warn(`profile '${deployment.profile}': [${document}] ${w}`);
1926
+ }
1927
+ }
1928
+ // Messages name the deployment.
1929
+ const label = `${document}, deployment '${deployment.name}'`;
1930
+ const checked = { document: label, model: applied };
1931
+ // Pruning can remove the field these rules are about, so they run first.
1932
+ // localChecks has already warned about the SQL they cannot read.
1933
+ const unprunedErrors = [
1934
+ ...validateInheritance([checked]),
1935
+ ...validateExpressionReferences([checked]),
1936
+ ];
1937
+ const { model: pruned, report } = (0, resolve_profiles_1.pruneUnavailable)(checked.model, deployment.profile);
1938
+ // Keeps only this deployment, so every reader sees its target alone.
1939
+ const narrowed = {
1940
+ document: checked.document,
1941
+ model: {
1942
+ ...pruned,
1943
+ deployments: (model.deployments ?? []).filter(d => d.name === deployment.name),
1944
+ },
1945
+ };
1946
+ const errors = [
1947
+ ...unprunedErrors,
1948
+ ...validatePushRequirements([narrowed], {
1949
+ fieldsPruned: true,
1950
+ v2Aspects: kcV2Aspects(),
1951
+ }),
1952
+ ];
1953
+ if (errors.length)
1954
+ return { errors };
1955
+ // An AlloyDB database holds no graph.
1956
+ if (deployment.engine === 'alloydb') {
1957
+ return { deployment, at, model: narrowed, report };
1958
+ }
1959
+ const generated = await generateOffline(deployment.engine === 'bigquery' ? deploy_bigquery_1.deployBigQuery : deploy_spanner_1.deploySpanner, [narrowed]);
1960
+ if (typeof generated === 'string')
1961
+ return fail(`${at}: ${generated}`);
1962
+ return { deployment, at, model: narrowed, report, generated };
1963
+ }
1964
+ // The warnings and the note push prints on what pruning leaves out of the
1965
+ // model `plan` deploys. Each line names the model, and the document and the
1966
+ // deployment.
1967
+ function pruningLines({ deployment, model, report: r }) {
1968
+ const where = `model '${model.model.name}' (${model.document})`;
1969
+ // What becomes of a field left out. An AlloyDB database holds no graph.
1970
+ const outcome = deployment.engine === 'alloydb' ? 'it is unavailable there' :
1971
+ 'the graph omits it';
1972
+ const excluded = new Set(r.excludedFields);
1973
+ const omits = (field, why) => `Warning: field '${field}' in ${where} ${why}, so ${outcome}.`;
1974
+ // Such as "metric 'a'" or "metrics 'a' and 'b'", or '' for no names.
1975
+ const named = (kind, names) => names.length ?
1976
+ `${(0, loader_1.plural)(names.length, kind, `${kind}s`)} ${(0, loader_1.listed)(names.map(n => `'${n}'`))}` :
1977
+ '';
1978
+ const unavailable = [
1979
+ named('metric', r.droppedMetrics.map(m => m.name)),
1980
+ named('action', r.droppedActions.map(a => a.name)),
1981
+ ].filter(n => n).join(r.droppedMetrics.length > 1 ? ', and ' : ' and ');
1982
+ const note = [
1983
+ excluded.size ? `excludes ${named('field', [...excluded])}.` : '',
1984
+ unavailable && `leaves ${unavailable} unavailable.`,
1985
+ ].filter(p => p).join(' It ');
1986
+ return [
1987
+ ...r.unboundFields.filter(f => !excluded.has(f))
1988
+ .map(f => omits(f, 'has no expression')),
1989
+ ...r.droppedFields.map(d => omits(d.name, d.reason)),
1990
+ ...note ? [`Note: ${where} ${note}`] : [],
1991
+ ];
1992
+ }
1993
+ // A validate-only deploy builds the DDL and calls no service, so it needs no
1994
+ // project or token.
1995
+ const NO_SERVICE = new context_1.ApiContext('', '', '');
1996
+ // Generates a graph leg's DDL without calling a service, or gives why it
1997
+ // cannot be generated.
1998
+ async function generateOffline(deployLeg, models) {
1999
+ try {
2000
+ const result = await deployLeg(models, NO_SERVICE, { validateOnly: true });
2001
+ if (result.success)
2002
+ return result;
2003
+ return result.details ?? 'the graph could not be generated';
2004
+ }
2005
+ catch (err) {
2006
+ return err.message || String(err);
2007
+ }
2008
+ }
1624
2009
  // Live pre-flight over the BigQuery-targeting models: confirms every entity's
1625
2010
  // BigQuery source table is reachable BEFORE any destination leg runs, so a push
1626
2011
  // fails fast when the model could not deploy, rather than surfacing a missing
@@ -1781,24 +2166,6 @@ function rejectedStatement(pending, store, detail) {
1781
2166
  return `action '${pending.action.name}' in model '${pending.model}' (${pending.document}): statement ${pending.index + 1} of its sql executor was rejected by ${store}: ${detail} Statements reach the store exactly as written, so every ` +
1782
2167
  `table and column has to be the name the database uses.`;
1783
2168
  }
1784
- // The first line of a multi-line diagnosis. The lines after it echo the
1785
- // statement and point a caret at the offending token, which is worth nothing
1786
- // here: the author has the statement open in the profile, the position in
1787
- // `[at 1:54]` survives, and a caret counted from column 1 lands in the wrong
1788
- // place once the message is embedded in a longer sentence.
1789
- //
1790
- // `\n` is matched in both its forms because Spanner's top-level `error.message`
1791
- // arrives with its newlines ESCAPED -- a literal backslash and an n survive
1792
- // JSON.parse -- while every other message uses real ones.
1793
- function firstLine(message) {
1794
- return message.replace(/\\n/g, '\n').split('\n')[0].trim();
1795
- }
1796
- // An API error's message reduced to its first line as a sentence, such as
1797
- // "Unrecognized name: accountId; Did you mean account_id? [at 1:54]."
1798
- function storeMessage(message, status) {
1799
- const text = (message ?? '').trim();
1800
- return text ? (0, parameters_1.sentence)(firstLine(text)) : `HTTP ${status}.`;
1801
- }
1802
2169
  // The Spanner arm. Each distinct database gets one session, reused across every
1803
2170
  // statement bound to it and deleted at the end -- which also discards the
1804
2171
  // read-write transactions PLAN had to begin and never committed. Returns one
@@ -1838,7 +2205,7 @@ async function validateSpannerActionStatements(models, spanner) {
1838
2205
  // is reported rather than skipped: a push that could not verify its
1839
2206
  // statements must not proceed as though it had.
1840
2207
  errors.push(`could not open a session on ${store} to verify the sql executor ` +
1841
- `statements in model '${statements[0].model}' (${statements[0].document}): ${storeMessage(opened.message, opened.status)}`);
2208
+ `statements in model '${statements[0].model}' (${statements[0].document}): ${(0, parameters_1.storeMessage)(opened.message, opened.status)}`);
1842
2209
  continue;
1843
2210
  }
1844
2211
  try {
@@ -1853,7 +2220,7 @@ async function validateSpannerActionStatements(models, spanner) {
1853
2220
  const res = await spanner.planDml(session, pending.sql, params, paramTypes);
1854
2221
  if (res.status === 200)
1855
2222
  continue;
1856
- errors.push(rejectedStatement(pending, store, storeMessage(res.message, res.status)));
2223
+ errors.push(rejectedStatement(pending, store, (0, parameters_1.storeMessage)(res.message, res.status)));
1857
2224
  }
1858
2225
  }
1859
2226
  finally {
@@ -1892,7 +2259,7 @@ async function validateBigQueryActionStatements(models, bq, defaultProject) {
1892
2259
  const res = await bq.query(project, pending.sql, undefined, true, statementParameterTypes(pending.action, pending.sql));
1893
2260
  if (res.status === 200)
1894
2261
  continue;
1895
- errors.push(rejectedStatement(pending, store, storeMessage(res.message, res.status)));
2262
+ errors.push(rejectedStatement(pending, store, (0, parameters_1.storeMessage)(res.message, res.status)));
1896
2263
  }
1897
2264
  }
1898
2265
  return errors;