@google-cloud/kcmd 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -20
- package/build/ts/kcmd/cli.js +5842 -5168
- package/build/ts/kcmd/gcp/biglake.d.ts +2 -0
- package/build/ts/kcmd/gcp/biglake.js +13 -4
- package/build/ts/kcmd/gcp/spanner.d.ts +10 -0
- package/build/ts/kcmd/gcp/spanner.js +6 -3
- package/build/ts/kcmd/layouts/okf.js +49 -7
- package/build/ts/kcmd/layouts/semantic-model.d.ts +13 -0
- package/build/ts/kcmd/layouts/semantic-model.js +129 -50
- package/build/ts/kcmd/okf/concept.d.ts +1 -1
- package/build/ts/kcmd/okf/concept.js +6 -0
- package/build/ts/kcmd/okf/concept_to_entry.d.ts +2 -0
- package/build/ts/kcmd/okf/concept_to_entry.js +29 -2
- package/build/ts/kcmd/okf/init_okf_workspace.js +8 -10
- package/build/ts/kcmd/okf/okf_metadata.js +17 -0
- package/build/ts/kcmd/semantic/bigquery.js +139 -195
- package/build/ts/kcmd/semantic/binding.js +14 -19
- package/build/ts/kcmd/semantic/deploy_knowledge_catalog.d.ts +10 -35
- package/build/ts/kcmd/semantic/deploy_knowledge_catalog.js +92 -121
- package/build/ts/kcmd/semantic/graph_ddl.d.ts +10 -0
- package/build/ts/kcmd/semantic/graph_ddl.js +152 -59
- package/build/ts/kcmd/semantic/import_model.d.ts +26 -4
- package/build/ts/kcmd/semantic/import_model.js +526 -103
- package/build/ts/kcmd/semantic/kc_converter.d.ts +7 -1
- package/build/ts/kcmd/semantic/kc_converter.js +20 -5
- package/build/ts/kcmd/semantic/kc_custom_types.d.ts +5 -12
- package/build/ts/kcmd/semantic/kc_custom_types.js +16 -93
- package/build/ts/kcmd/semantic/kc_ids.d.ts +4 -5
- package/build/ts/kcmd/semantic/kc_ids.js +4 -7
- package/build/ts/kcmd/semantic/knowledge_catalog.d.ts +0 -3
- package/build/ts/kcmd/semantic/knowledge_catalog.js +3 -86
- package/build/ts/kcmd/semantic/loader.d.ts +7 -11
- package/build/ts/kcmd/semantic/loader.js +284 -148
- package/build/ts/kcmd/semantic/osi_converter.js +1 -2
- package/build/ts/kcmd/semantic/parameters.d.ts +1 -0
- package/build/ts/kcmd/semantic/parameters.js +22 -3
- package/build/ts/kcmd/semantic/pull_kc.d.ts +1 -5
- package/build/ts/kcmd/semantic/pull_kc.js +38 -112
- package/build/ts/kcmd/semantic/resolve_inheritance.d.ts +7 -2
- package/build/ts/kcmd/semantic/resolve_inheritance.js +37 -21
- package/build/ts/kcmd/semantic/resolve_profiles.d.ts +51 -10
- package/build/ts/kcmd/semantic/resolve_profiles.js +213 -32
- package/build/ts/kcmd/semantic/runtime/runtime.d.ts +0 -6
- package/build/ts/kcmd/semantic/runtime/runtime.js +4 -21
- package/build/ts/kcmd/semantic/source_dialect.d.ts +9 -2
- package/build/ts/kcmd/semantic/source_dialect.js +83 -33
- package/build/ts/kcmd/semantic/source_entries.d.ts +1 -1
- package/build/ts/kcmd/semantic/source_entries.js +95 -36
- package/build/ts/kcmd/semantic/sql_expr_utils.d.ts +13 -7
- package/build/ts/kcmd/semantic/sql_expr_utils.js +240 -67
- package/build/ts/kcmd/semantic/sql_parser.d.ts +5 -2
- package/build/ts/kcmd/semantic/sql_parser.js +22 -0
- package/build/ts/kcmd/semantic/transpile.d.ts +12 -10
- package/build/ts/kcmd/semantic/transpile.js +79 -61
- package/build/ts/kcmd/semantic/validate.d.ts +37 -2
- package/build/ts/kcmd/semantic/validate.js +465 -98
- package/build/ts/kcmd/snapshot.d.ts +4 -0
- package/build/ts/kcmd/snapshot.js +7 -0
- package/build/ts/kcmd/sync.js +44 -7
- package/package.json +1 -1
|
@@ -4,10 +4,11 @@
|
|
|
4
4
|
// Runs once over the shared, already-parsed models (see loadSemanticModels)
|
|
5
5
|
// before any destination leg, so a real `kcmd push` AND a `--validate-only` dry
|
|
6
6
|
// run enforce the same requirements. Returns one message per violation (an
|
|
7
|
-
// empty array means valid);
|
|
8
|
-
//
|
|
9
|
-
// schema -- because these are deployment requirements, not schema
|
|
10
|
-
// they read the GOOGLE deployment-target extension the BigQuery leg
|
|
7
|
+
// empty array means valid); push prints them and aborts, and `kcmd import`
|
|
8
|
+
// reports them. Kept separate from the loader -- which validates a document
|
|
9
|
+
// against the schema -- because these are deployment requirements, not schema
|
|
10
|
+
// rules, and they read the GOOGLE deployment-target extension the BigQuery leg
|
|
11
|
+
// owns.
|
|
11
12
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
13
|
exports.validatePushRequirements = validatePushRequirements;
|
|
13
14
|
exports.validateDeployments = validateDeployments;
|
|
@@ -18,12 +19,18 @@ exports.validateActions = validateActions;
|
|
|
18
19
|
exports.validateInheritance = validateInheritance;
|
|
19
20
|
exports.validateExpressionReferences = validateExpressionReferences;
|
|
20
21
|
exports.deploymentTableErrors = deploymentTableErrors;
|
|
22
|
+
exports.spannerMetricsErrors = spannerMetricsErrors;
|
|
23
|
+
exports.kcV2Aspects = kcV2Aspects;
|
|
24
|
+
exports.withProfileFiles = withProfileFiles;
|
|
25
|
+
exports.localChecks = localChecks;
|
|
26
|
+
exports.pruningLines = pruningLines;
|
|
21
27
|
exports.validateBigQueryDataSources = validateBigQueryDataSources;
|
|
22
|
-
exports.storeMessage = storeMessage;
|
|
23
28
|
exports.validateSpannerActionStatements = validateSpannerActionStatements;
|
|
24
29
|
exports.validateBigQueryActionStatements = validateBigQueryActionStatements;
|
|
30
|
+
const context_1 = require("../gcp/context");
|
|
25
31
|
const bigquery_1 = require("./bigquery");
|
|
26
32
|
const deploy_bigquery_1 = require("./deploy_bigquery");
|
|
33
|
+
const deploy_spanner_1 = require("./deploy_spanner");
|
|
27
34
|
const deployment_target_1 = require("./deployment_target");
|
|
28
35
|
const ir_1 = require("./ir");
|
|
29
36
|
const kc_ids_1 = require("./kc_ids");
|
|
@@ -34,7 +41,6 @@ const resolve_inheritance_1 = require("./resolve_inheritance");
|
|
|
34
41
|
const resolve_profiles_1 = require("./resolve_profiles");
|
|
35
42
|
const sql_expr_utils_1 = require("./sql_expr_utils");
|
|
36
43
|
const sql_identifiers_1 = require("./sql_identifiers");
|
|
37
|
-
const sql_parser_1 = require("./sql_parser");
|
|
38
44
|
// Checks every model against the push requirements and returns the collected
|
|
39
45
|
// error messages (empty when all models pass), each tagged with the model's
|
|
40
46
|
// source document so the author can find it.
|
|
@@ -49,7 +55,7 @@ const sql_parser_1 = require("./sql_parser");
|
|
|
49
55
|
// caller to print.
|
|
50
56
|
//
|
|
51
57
|
// `v2Aspects` says push writes the second-generation Knowledge Catalog types
|
|
52
|
-
// (
|
|
58
|
+
// (the CLI sets it unless KC_V2_ASPECTS=0).
|
|
53
59
|
function validatePushRequirements(models, opts = {}) {
|
|
54
60
|
const errors = [];
|
|
55
61
|
for (const { document, model } of models) {
|
|
@@ -94,7 +100,7 @@ function validatePushRequirements(models, opts = {}) {
|
|
|
94
100
|
const flaggedMetrics = new Set();
|
|
95
101
|
errors.push(...validateExpressionReferences([{ document, model }], !!opts.fieldsPruned, opts.warnings, flaggedMetrics));
|
|
96
102
|
if (deployInfo.bigQuery.length > 0) {
|
|
97
|
-
errors.push(...
|
|
103
|
+
errors.push(...measurePlacementErrors(model, document, flaggedMetrics));
|
|
98
104
|
}
|
|
99
105
|
// An action reaches Knowledge Catalog only, so its checks are
|
|
100
106
|
// target-independent: each parameter's type must resolve to something in
|
|
@@ -171,7 +177,7 @@ function nameAndDescriptionErrors(model, document, v2Aspects) {
|
|
|
171
177
|
// Checks each deployment of `model` against the rules of its engine, under the
|
|
172
178
|
// profile it names in `model.profiles` or the model file's own bindings, and
|
|
173
179
|
// returns one message per problem, naming every deployment it applies to.
|
|
174
|
-
function validateDeployments(model, document) {
|
|
180
|
+
function validateDeployments(model, document, load = {}) {
|
|
175
181
|
// The deployments each problem applies to, keyed by the text that follows
|
|
176
182
|
// their names in the message.
|
|
177
183
|
const problems = new Map();
|
|
@@ -193,7 +199,7 @@ function validateDeployments(model, document) {
|
|
|
193
199
|
// The model the deployment's bindings give, less what they leave
|
|
194
200
|
// unavailable. The completeness check reports a named profile that does
|
|
195
201
|
// not apply, and the fields its metrics read.
|
|
196
|
-
const bound = named ? profiled(model, named) : (0, resolve_profiles_1.pruneUnavailable)(model, resolve_profiles_1.DEFAULT_PROFILE, alloyDb ? 'ALLOYDB' : (0, resolve_profiles_1.sourcesDialect)(model));
|
|
202
|
+
const bound = named ? profiled(model, named, load) : (0, resolve_profiles_1.pruneUnavailable)(model, resolve_profiles_1.DEFAULT_PROFILE, alloyDb ? 'ALLOYDB' : (0, resolve_profiles_1.sourcesDialect)(model));
|
|
197
203
|
if (!bound)
|
|
198
204
|
continue;
|
|
199
205
|
const available = bound.model.metrics ?? [];
|
|
@@ -251,9 +257,9 @@ function validateDeployments(model, document) {
|
|
|
251
257
|
}
|
|
252
258
|
// `model` with `profile` applied and what it leaves unavailable pruned, or
|
|
253
259
|
// undefined when the profile does not apply.
|
|
254
|
-
function profiled(model, profile) {
|
|
260
|
+
function profiled(model, profile, load) {
|
|
255
261
|
try {
|
|
256
|
-
return (0, resolve_profiles_1.pruneUnavailable)((0, resolve_profiles_1.applyProfile)(model, profile), profile.name);
|
|
262
|
+
return (0, resolve_profiles_1.pruneUnavailable)((0, resolve_profiles_1.applyProfile)(model, profile, { load }), profile.name);
|
|
257
263
|
}
|
|
258
264
|
catch (err) {
|
|
259
265
|
if (!(err instanceof loader_1.SemanticModelError))
|
|
@@ -262,7 +268,7 @@ function profiled(model, profile) {
|
|
|
262
268
|
}
|
|
263
269
|
}
|
|
264
270
|
// Why `metric` cannot be a BigQuery measure, or undefined when it can. Leaves
|
|
265
|
-
// out a metric that
|
|
271
|
+
// out a metric that measurePlacementErrors or the inheritance rules report:
|
|
266
272
|
// one that reads several entities or none, or one on an abstract or extended
|
|
267
273
|
// entity.
|
|
268
274
|
function measureProblem(model, metric) {
|
|
@@ -273,15 +279,10 @@ function measureProblem(model, metric) {
|
|
|
273
279
|
if (!columns)
|
|
274
280
|
return undefined;
|
|
275
281
|
const entities = model.entities ?? [];
|
|
276
|
-
const
|
|
277
|
-
|
|
278
|
-
// BigQuery graph runs.
|
|
279
|
-
const matched = (0, sql_expr_utils_1.referencedEntityNames)((0, sql_parser_1.withoutComments)(text, 'BIGQUERY'), names);
|
|
280
|
-
if (matched.length > 1 || !matched.length && !metric.entity)
|
|
282
|
+
const placed = (0, sql_expr_utils_1.measureEntities)(text, entities.map(e => e.name), metric.entity);
|
|
283
|
+
if (placed.length !== 1)
|
|
281
284
|
return undefined;
|
|
282
|
-
const
|
|
283
|
-
const name = reached.length > 1 ? undefined : reached[0] ?? metric.entity;
|
|
284
|
-
const entity = entities.find(e => e.name === name);
|
|
285
|
+
const entity = entities.find(e => e.name === placed[0]);
|
|
285
286
|
if (!entity || entity.abstract ||
|
|
286
287
|
entities.some(e => e.extends?.includes(entity.name))) {
|
|
287
288
|
return undefined;
|
|
@@ -295,17 +296,73 @@ function measureProblem(model, metric) {
|
|
|
295
296
|
}
|
|
296
297
|
return undefined;
|
|
297
298
|
}
|
|
298
|
-
//
|
|
299
|
-
//
|
|
299
|
+
// The BigQuery graph's rules for placing each metric of `model` as a measure,
|
|
300
|
+
// one message per metric that breaks them, skipping each one in
|
|
301
|
+
// `alreadyFlagged`, which validateExpressionReferences has reported. The
|
|
302
|
+
// generator (`bigquery.ts`) places a measure on the one entity whose
|
|
303
|
+
// qualifier its BigQuery SQL writes before a column, or on its anchor when the
|
|
304
|
+
// SQL writes none (see measureEntities), and skips a metric that has no
|
|
305
|
+
// BigQuery or ANSI_SQL text. validateExpressionReferences reads every dialect
|
|
306
|
+
// entry together and lets an authored anchor settle SQL that reads two
|
|
307
|
+
// entities, so it does not catch each metric the generator would skip.
|
|
308
|
+
function measurePlacementErrors(model, document, alreadyFlagged) {
|
|
309
|
+
const errors = [];
|
|
310
|
+
const entityNames = (model.entities ?? []).map(e => e.name);
|
|
311
|
+
for (const metric of model.metrics ?? []) {
|
|
312
|
+
if (alreadyFlagged.has(metric.name))
|
|
313
|
+
continue;
|
|
314
|
+
const prefix = `metric '${metric.name}' in model '${model.name}' (${document})`;
|
|
315
|
+
// The text BigQuery runs: its `BIGQUERY` entry, else its `ANSI_SQL` one,
|
|
316
|
+
// else `expression` for a metric held without a dialect list.
|
|
317
|
+
const bqText = (0, ir_1.expressionForDialect)(metric, 'BIGQUERY');
|
|
318
|
+
if (bqText === undefined && metric.importedExpression !== undefined) {
|
|
319
|
+
errors.push(`${prefix} targets a BigQuery graph with only a '${metric.importedDialect ?? 'vendor'}' expression; supply a ` +
|
|
320
|
+
`'BIGQUERY' or 'ANSI_SQL' expression${(0, resolve_profiles_1.importTranspiles)(metric, 'BIGQUERY', model) ?
|
|
321
|
+
`, or add one with '${(0, resolve_profiles_1.importCommand)('--transpile', model.version)}'` :
|
|
322
|
+
''}.`);
|
|
323
|
+
continue;
|
|
324
|
+
}
|
|
325
|
+
const placed = (0, sql_expr_utils_1.measureEntities)(bqText ?? '', entityNames, metric.entity);
|
|
326
|
+
if (placed.length > 1) {
|
|
327
|
+
errors.push(`${prefix} targets a BigQuery graph and its BigQuery SQL reads ` +
|
|
328
|
+
`several entities (${placed.join(', ')}); a BigQuery measure ` +
|
|
329
|
+
`belongs to one entity's node table, so scope its expression to ` +
|
|
330
|
+
`one entity.`);
|
|
331
|
+
}
|
|
332
|
+
else if (placed.length === 0) {
|
|
333
|
+
const anchor = model.version === loader_1.OSSIE_VERSION ?
|
|
334
|
+
`'entity' in a GOOGLE custom_extensions block on the metric` :
|
|
335
|
+
`top-level 'entity'`;
|
|
336
|
+
errors.push(`${prefix} targets a BigQuery graph but does not resolve to a ` +
|
|
337
|
+
`single entity; the BigQuery generator places a measure on the ` +
|
|
338
|
+
`entity its BigQuery SQL reads, or on ${anchor}.`);
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
return errors;
|
|
342
|
+
}
|
|
343
|
+
// Checks that the model file's Spanner tables are all in one database, so they
|
|
344
|
+
// share one dialect, and that each bound field has a text for the dialect of
|
|
345
|
+
// its entity's source. Returns one message per problem.
|
|
300
346
|
function validateInlineDialects(model, document) {
|
|
301
347
|
const errors = [];
|
|
348
|
+
const databases = new Map();
|
|
349
|
+
for (const e of model.entities ?? []) {
|
|
350
|
+
const db = (0, loader_1.databaseOf)(e.authoredSource ?? '');
|
|
351
|
+
if (db?.startsWith('spanner/') && !databases.has(db)) {
|
|
352
|
+
databases.set(db, e.name);
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
if (databases.size > 1) {
|
|
356
|
+
const named = [...databases.values()].map(n => `'${n}'`).join(', ');
|
|
357
|
+
errors.push(`model '${model.name}' (${document}) binds entities in more than one ` +
|
|
358
|
+
`Spanner database (${named}); bind the model file's entities to ` +
|
|
359
|
+
`tables in one Spanner database.`);
|
|
360
|
+
}
|
|
302
361
|
for (const { entity: e, field: f, table: { name: dialect } } of (0, resolve_profiles_1.fieldsOnTables)(model)) {
|
|
303
362
|
if (!(0, resolve_profiles_1.noTextFor)(f, [dialect]))
|
|
304
363
|
continue;
|
|
305
|
-
const hint = (0, resolve_profiles_1.importTranspiles)(f, dialect) ?
|
|
306
|
-
`; give it one, or add one with '${resolve_profiles_1.IMPORT_TRANSPILE_COMMAND}'` : '';
|
|
307
364
|
errors.push(`field '${e.name}.${f.name}' in model '${model.name}' (${document}) has no expression for ${dialect} or ANSI_SQL, and ` +
|
|
308
|
-
`the model file binds entity '${e.name}' to ${/^[AEIOU]/.test(dialect) ? 'an' : 'a'} ${dialect} source${
|
|
365
|
+
`the model file binds entity '${e.name}' to ${/^[AEIOU]/.test(dialect) ? 'an' : 'a'} ${dialect} source${(0, resolve_profiles_1.importHint)(f, dialect, model)}.`);
|
|
309
366
|
}
|
|
310
367
|
return errors;
|
|
311
368
|
}
|
|
@@ -1292,6 +1349,94 @@ function fieldNoSqlErrors(model, document) {
|
|
|
1292
1349
|
}
|
|
1293
1350
|
return errors;
|
|
1294
1351
|
}
|
|
1352
|
+
// The errors for the fields of `entity` that name themselves, directly or
|
|
1353
|
+
// through other fields of `entity`. The graph renders a field that names a
|
|
1354
|
+
// sibling as `<entity>.<field>` from that sibling's own SQL, so a field that
|
|
1355
|
+
// reaches itself that way has no SQL to render. A field expression that writes
|
|
1356
|
+
// its own column as `<entity>.<field>` names itself too: write the bare column
|
|
1357
|
+
// instead. Each dialect is read on its own, from the text an engine of that
|
|
1358
|
+
// dialect renders (its entry, else the ANSI_SQL one), so two texts that no one
|
|
1359
|
+
// engine reads together do not form a cycle.
|
|
1360
|
+
//
|
|
1361
|
+
// `resolved` is `entity` with the fields it inherits, which name its fields
|
|
1362
|
+
// as the graph renders them (see resolve_inheritance.localizeInheritedField),
|
|
1363
|
+
// so a field can reach itself through one it inherits. A cycle is reported on
|
|
1364
|
+
// an entity only when a field on it is bound there, so one an ancestor forms on
|
|
1365
|
+
// its own is reported on that ancestor alone.
|
|
1366
|
+
function selfReferenceErrors(entity, resolved, model, document, impliedDialect) {
|
|
1367
|
+
const own = entity.fields ?? [];
|
|
1368
|
+
const inherited = resolved.fields ?? [];
|
|
1369
|
+
// A field a profile excludes is left off `resolved`, but still checked.
|
|
1370
|
+
const fields = [
|
|
1371
|
+
...inherited,
|
|
1372
|
+
...own.filter(f => !inherited.some(r => r.name === f.name)),
|
|
1373
|
+
];
|
|
1374
|
+
const bound = new Set(own.filter(f => (0, ir_1.isFieldBound)(f)).map(f => f.name));
|
|
1375
|
+
const names = new Set(fields.map(f => f.name));
|
|
1376
|
+
const dialects = new Set(['ANSI_SQL']);
|
|
1377
|
+
for (const f of fields) {
|
|
1378
|
+
for (const d of f.dialects ?? [])
|
|
1379
|
+
dialects.add(d.dialect);
|
|
1380
|
+
if (f.importedDialect)
|
|
1381
|
+
dialects.add(f.importedDialect);
|
|
1382
|
+
}
|
|
1383
|
+
const errors = new Set();
|
|
1384
|
+
for (const dialect of dialects) {
|
|
1385
|
+
// The fields of `entity` each field names, in this dialect, read as the
|
|
1386
|
+
// graph generators read them (see graph_ddl.siblingsRead), so a qualifier
|
|
1387
|
+
// the parser reserves, such as `Date` in `Date.days`, names them too. An
|
|
1388
|
+
// ANSI_SQL text is read in the dialect the sources imply, as
|
|
1389
|
+
// columnsForSource reads it, and not at all when their dialect is null.
|
|
1390
|
+
if (dialect === 'ANSI_SQL' && impliedDialect === null)
|
|
1391
|
+
continue;
|
|
1392
|
+
const readIn = dialect === 'ANSI_SQL' && impliedDialect ? impliedDialect : dialect;
|
|
1393
|
+
const reads = new Map();
|
|
1394
|
+
for (const f of fields) {
|
|
1395
|
+
const text = (0, ir_1.expressionForDialect)(f, dialect);
|
|
1396
|
+
if (text === undefined ||
|
|
1397
|
+
(0, loader_1.noSqlReason)(text, dialect, entity.name) !== undefined) {
|
|
1398
|
+
continue;
|
|
1399
|
+
}
|
|
1400
|
+
reads.set(f.name, [...new Set((0, sql_expr_utils_1.qualifiedColumns)(text, entity.name, readIn) ?? [])]
|
|
1401
|
+
.filter(n => names.has(n)));
|
|
1402
|
+
}
|
|
1403
|
+
for (const f of fields) {
|
|
1404
|
+
const through = pathBack(f.name, reads);
|
|
1405
|
+
if (!through || ![f.name, ...through].some(n => bound.has(n)))
|
|
1406
|
+
continue;
|
|
1407
|
+
errors.add((0, sql_expr_utils_1.selfReferenceMessage)(entity.name, [f.name, ...through], ` in model '${model.name}' (${document})`) +
|
|
1408
|
+
'.');
|
|
1409
|
+
}
|
|
1410
|
+
}
|
|
1411
|
+
return [...errors];
|
|
1412
|
+
}
|
|
1413
|
+
// The shortest run of other fields through which `start` reaches itself in
|
|
1414
|
+
// `reads`, empty when it names itself directly, or undefined when it does not
|
|
1415
|
+
// reach itself.
|
|
1416
|
+
function pathBack(start, reads) {
|
|
1417
|
+
const via = new Map();
|
|
1418
|
+
const queue = [];
|
|
1419
|
+
for (const next of reads.get(start) ?? []) {
|
|
1420
|
+
if (next === start)
|
|
1421
|
+
return [];
|
|
1422
|
+
if (!via.has(next)) {
|
|
1423
|
+
via.set(next, [next]);
|
|
1424
|
+
queue.push(next);
|
|
1425
|
+
}
|
|
1426
|
+
}
|
|
1427
|
+
while (queue.length) {
|
|
1428
|
+
const name = queue.shift();
|
|
1429
|
+
for (const next of reads.get(name) ?? []) {
|
|
1430
|
+
if (next === start)
|
|
1431
|
+
return via.get(name);
|
|
1432
|
+
if (!via.has(next)) {
|
|
1433
|
+
via.set(next, [...via.get(name), next]);
|
|
1434
|
+
queue.push(next);
|
|
1435
|
+
}
|
|
1436
|
+
}
|
|
1437
|
+
}
|
|
1438
|
+
return undefined;
|
|
1439
|
+
}
|
|
1295
1440
|
// The error for a metric with no SQL, or undefined. A metric is rejected when
|
|
1296
1441
|
// none of its texts holds SQL, with or without an anchor, since the graph
|
|
1297
1442
|
// would drop it. It is also rejected when any one of its texts holds none, as
|
|
@@ -1332,10 +1477,13 @@ function metricNoSqlError(metric, model, document) {
|
|
|
1332
1477
|
// differs only in case from a declared entity.
|
|
1333
1478
|
// 2. For references to E itself, every field named must be declared or
|
|
1334
1479
|
// inherited on E (gated on `!fieldsPruned`, and run before pruning).
|
|
1480
|
+
// 3. Reject a field that names itself, directly or through other fields of
|
|
1481
|
+
// E, declared or inherited, since the graph renders a field that names
|
|
1482
|
+
// a sibling from that sibling's SQL (see selfReferenceErrors).
|
|
1335
1483
|
//
|
|
1336
|
-
//
|
|
1337
|
-
//
|
|
1338
|
-
//
|
|
1484
|
+
// deploymentPlan also runs it on the unpruned model before `pruneUnavailable`
|
|
1485
|
+
// drops unbound fields and metrics on a graph push. When given,
|
|
1486
|
+
// `flaggedMetrics` collects the name of each metric it reports.
|
|
1339
1487
|
function validateExpressionReferences(models, fieldsPruned = false, warnings = [], flaggedMetrics) {
|
|
1340
1488
|
const errors = [];
|
|
1341
1489
|
for (const { document, model } of models) {
|
|
@@ -1352,8 +1500,18 @@ function validateExpressionReferences(models, fieldsPruned = false, warnings = [
|
|
|
1352
1500
|
continue;
|
|
1353
1501
|
}
|
|
1354
1502
|
const entityNames = [...fieldsByEntity.keys()];
|
|
1503
|
+
// Each entity's fields as the graph renders them (see selfReferenceErrors).
|
|
1504
|
+
const resolvedByName = new Map((ifResolvable(() => (0, resolve_inheritance_1.resolveInheritance)(model, undefined, { requalify: true })
|
|
1505
|
+
.model.entities) ??
|
|
1506
|
+
model.entities ?? [])
|
|
1507
|
+
.map(e => [e.name, e]));
|
|
1355
1508
|
const impliedDialect = (0, resolve_profiles_1.sourcesDialect)(model);
|
|
1356
|
-
const addUnreadableWarning = (what, total,
|
|
1509
|
+
const addUnreadableWarning = (what, total, unread) => {
|
|
1510
|
+
// A caller that looked up no table's dialect reads no ANSI_SQL
|
|
1511
|
+
// text when every table needs the lookup, so it warns about none.
|
|
1512
|
+
const unreadDialects = impliedDialect === null ?
|
|
1513
|
+
unread.filter(d => d !== 'ANSI_SQL') :
|
|
1514
|
+
unread;
|
|
1357
1515
|
if (!unreadDialects.length)
|
|
1358
1516
|
return;
|
|
1359
1517
|
if (unreadDialects.length === total) {
|
|
@@ -1406,6 +1564,7 @@ function validateExpressionReferences(models, fieldsPruned = false, warnings = [
|
|
|
1406
1564
|
}
|
|
1407
1565
|
}
|
|
1408
1566
|
}
|
|
1567
|
+
errors.push(...selfReferenceErrors(entity, resolvedByName.get(entity.name) ?? entity, model, document, impliedDialect));
|
|
1409
1568
|
}
|
|
1410
1569
|
for (const metric of model.metrics ?? []) {
|
|
1411
1570
|
const before = errors.length;
|
|
@@ -1496,53 +1655,6 @@ function validateExpressionReferences(models, fieldsPruned = false, warnings = [
|
|
|
1496
1655
|
}
|
|
1497
1656
|
return errors;
|
|
1498
1657
|
}
|
|
1499
|
-
// Temporary BigQuery-only metric check, until the BigQuery generator places
|
|
1500
|
-
// measures with the SQL parser. Rejects a metric that the generator
|
|
1501
|
-
// (`bigquery.ts`) would skip: one with only another engine's SQL, which the
|
|
1502
|
-
// generator does not emit, or one it drops because its regex
|
|
1503
|
-
// (`referencedEntityNames`) disagrees with the parser or because
|
|
1504
|
-
// `metric.entity` is unset. Reports only metrics that
|
|
1505
|
-
// `validateExpressionReferences` has not already flagged.
|
|
1506
|
-
function validateBigQueryMetrics(model, document, alreadyFlagged) {
|
|
1507
|
-
const errors = [];
|
|
1508
|
-
const entityNames = (model.entities ?? []).map(e => e.name);
|
|
1509
|
-
for (const metric of model.metrics ?? []) {
|
|
1510
|
-
if (alreadyFlagged.has(metric.name))
|
|
1511
|
-
continue;
|
|
1512
|
-
const prefix = `metric '${metric.name}' in model '${model.name}' (${document})`;
|
|
1513
|
-
// The text BigQuery runs: its `BIGQUERY` entry, else its `ANSI_SQL` one,
|
|
1514
|
-
// else `expression` for a metric held without a dialect list.
|
|
1515
|
-
const bqText = (0, ir_1.expressionForDialect)(metric, 'BIGQUERY');
|
|
1516
|
-
if (bqText === undefined && metric.importedExpression !== undefined) {
|
|
1517
|
-
errors.push(`${prefix} targets a BigQuery graph with only a '${metric.importedDialect ?? 'vendor'}' expression; supply a ` +
|
|
1518
|
-
`'BIGQUERY' or 'ANSI_SQL' expression, or add one with '${resolve_profiles_1.IMPORT_TRANSPILE_COMMAND}'.`);
|
|
1519
|
-
continue;
|
|
1520
|
-
}
|
|
1521
|
-
// The generator reads the resolved BigQuery expression (`bqText`) with its
|
|
1522
|
-
// comments removed.
|
|
1523
|
-
const expr = (0, sql_parser_1.withoutComments)(bqText ?? metric.importedExpression ?? '', 'BIGQUERY');
|
|
1524
|
-
let referenced = (0, sql_expr_utils_1.referencedEntityNames)(expr, entityNames);
|
|
1525
|
-
if (referenced.length === 0 && metric.entity) {
|
|
1526
|
-
referenced = [metric.entity];
|
|
1527
|
-
}
|
|
1528
|
-
if (referenced.length > 1) {
|
|
1529
|
-
errors.push(`${prefix} targets a BigQuery graph and references multiple ` +
|
|
1530
|
-
`entities (${referenced.join(', ')}); the BigQuery generator ` +
|
|
1531
|
-
`reads every '<entity>.' outside comments as a qualifier, ` +
|
|
1532
|
-
`including a function namespace such as 'NET.', so remove the ` +
|
|
1533
|
-
`extra '<entity>.' match or scope its expression to one entity.`);
|
|
1534
|
-
}
|
|
1535
|
-
else if (referenced.length === 0) {
|
|
1536
|
-
const anchor = model.version === loader_1.OSSIE_VERSION ?
|
|
1537
|
-
`'entity' in a GOOGLE custom_extensions block on the metric` :
|
|
1538
|
-
`top-level 'entity'`;
|
|
1539
|
-
errors.push(`${prefix} targets a BigQuery graph but does not resolve to a ` +
|
|
1540
|
-
`single entity; the BigQuery generator reads '<entity>.' ` +
|
|
1541
|
-
`qualifiers in the expression or ${anchor}.`);
|
|
1542
|
-
}
|
|
1543
|
-
}
|
|
1544
|
-
return errors;
|
|
1545
|
-
}
|
|
1546
1658
|
// Every field each entity has, inherited and profile-excluded ones included so
|
|
1547
1659
|
// an unpruned check before `pruneUnavailable` does not report a profile-excluded
|
|
1548
1660
|
// inherited field as undeclared. Inheritance is resolved through the same pass
|
|
@@ -1608,6 +1720,20 @@ function deploymentTableErrors(document, model, deployment) {
|
|
|
1608
1720
|
}
|
|
1609
1721
|
return errors;
|
|
1610
1722
|
}
|
|
1723
|
+
// The error for a Spanner `deployment` on the model file's own bindings when
|
|
1724
|
+
// `model` declares metrics, which a Spanner graph cannot hold.
|
|
1725
|
+
function spannerMetricsErrors(document, model, deployment) {
|
|
1726
|
+
if (deployment.engine !== 'spanner' ||
|
|
1727
|
+
deployment.profile !== resolve_profiles_1.DEFAULT_PROFILE || !model.metrics?.length) {
|
|
1728
|
+
return [];
|
|
1729
|
+
}
|
|
1730
|
+
return [
|
|
1731
|
+
`[${document}] deployment '${deployment.name}' targets a Spanner graph ` +
|
|
1732
|
+
`and uses the model file's own bindings, but the model declares ` +
|
|
1733
|
+
`metrics, which a Spanner graph cannot hold; name a profile with ` +
|
|
1734
|
+
`metrics_exclude: "*".`,
|
|
1735
|
+
];
|
|
1736
|
+
}
|
|
1611
1737
|
// The database the tables of a deployment target must be in, as databaseOf
|
|
1612
1738
|
// gives it, and how an error names the target and a table outside it.
|
|
1613
1739
|
function targetDatabase({ engine, coordinates: c }) {
|
|
@@ -1621,6 +1747,265 @@ function targetDatabase({ engine, coordinates: c }) {
|
|
|
1621
1747
|
return { database: `${engine}/${name}`, target: `${system} database '${name}'`,
|
|
1622
1748
|
outside: 'which is not a table in that database' };
|
|
1623
1749
|
}
|
|
1750
|
+
// Whether Knowledge Catalog push and pull use the second-generation built-in
|
|
1751
|
+
// aspect fields and entry-id layout. They do unless `KC_V2_ASPECTS=0`; every
|
|
1752
|
+
// other value, including unset and `''`, is on. `KC_V2_ASPECTS=0` is a
|
|
1753
|
+
// temporary escape hatch to the first-generation path, removed with that path
|
|
1754
|
+
// (b/575695037).
|
|
1755
|
+
function kcV2Aspects() {
|
|
1756
|
+
return process.env.KC_V2_ASPECTS !== '0';
|
|
1757
|
+
}
|
|
1758
|
+
// Adds the profile files of the model in `base` to `model.profiles`, without
|
|
1759
|
+
// their actions. Returns the error of the first profile file that breaks a
|
|
1760
|
+
// file rule. A profile file that does not load is left out, for
|
|
1761
|
+
// namedProfileErrors to report.
|
|
1762
|
+
function withProfileFiles(base, profileDocs) {
|
|
1763
|
+
const { document, model } = base;
|
|
1764
|
+
const profiles = [...(model.profiles ?? [])];
|
|
1765
|
+
let files;
|
|
1766
|
+
try {
|
|
1767
|
+
files = profileDocs(document);
|
|
1768
|
+
}
|
|
1769
|
+
catch (err) {
|
|
1770
|
+
return { errors: [{ error: err.message || String(err), document }] };
|
|
1771
|
+
}
|
|
1772
|
+
const reserved = (0, resolve_profiles_1.reservedProfileNameError)(document, files);
|
|
1773
|
+
if (reserved)
|
|
1774
|
+
return { errors: [{ error: reserved, document }] };
|
|
1775
|
+
for (const p of files) {
|
|
1776
|
+
try {
|
|
1777
|
+
profiles.push((0, resolve_profiles_1.loadProfileFile)(p.text, p.name, { skipActions: true }));
|
|
1778
|
+
}
|
|
1779
|
+
catch (err) {
|
|
1780
|
+
if (!(err instanceof loader_1.SemanticModelError))
|
|
1781
|
+
throw err;
|
|
1782
|
+
}
|
|
1783
|
+
}
|
|
1784
|
+
return { document, model: profiles.length ? { ...model, profiles } : model };
|
|
1785
|
+
}
|
|
1786
|
+
// Runs push's offline checks that follow withProfileFiles and the UTF-8 and
|
|
1787
|
+
// sidecar checks, through the graph generation. Returns the errors of the
|
|
1788
|
+
// first stage that finds any, or else a plan for each deployment `selected`
|
|
1789
|
+
// names, or for every deployment when it is unset.
|
|
1790
|
+
async function localChecks(scope, warn, selected) {
|
|
1791
|
+
const { docs, loaded, models, profileDocs, declaredBy, load } = scope;
|
|
1792
|
+
const stop = (errors) => ({ errors });
|
|
1793
|
+
const ofModel = (document, ...errors) => errors.map(error => ({ error, document }));
|
|
1794
|
+
const ofProfile = (document, profile, ...errors) => errors.map(error => ({ error, document, profile }));
|
|
1795
|
+
const declaredNames = (document) => declaredBy(docs.find(d => d.name === document)).map(p => p.name);
|
|
1796
|
+
// Every deployment's target must be in an accepted form, and its profile one
|
|
1797
|
+
// the model declares, whether or not this push selects the deployment.
|
|
1798
|
+
const selectedByDoc = new Map();
|
|
1799
|
+
const everyByDoc = new Map();
|
|
1800
|
+
for (const { document, model } of models) {
|
|
1801
|
+
const every = (0, deployment_target_1.resolveModelDeployments)(model);
|
|
1802
|
+
if ('error' in every)
|
|
1803
|
+
return stop(ofModel(document, every.error));
|
|
1804
|
+
const resolved = (0, deployment_target_1.resolveModelDeployments)(model, selected);
|
|
1805
|
+
if ('error' in resolved)
|
|
1806
|
+
return stop(ofModel(document, resolved.error));
|
|
1807
|
+
const undeclared = (0, resolve_profiles_1.undeclaredProfileError)(document, model, declaredNames(document));
|
|
1808
|
+
if (undeclared)
|
|
1809
|
+
return stop(ofModel(document, undeclared));
|
|
1810
|
+
everyByDoc.set(document, every.deployments);
|
|
1811
|
+
selectedByDoc.set(document, resolved.deployments);
|
|
1812
|
+
}
|
|
1813
|
+
// Every named profile is checked on its own against the model file, and so
|
|
1814
|
+
// are the model file's own bindings and each deployment. The catalog's model
|
|
1815
|
+
// is checked as a model that deploys nowhere. A selected deployment's own
|
|
1816
|
+
// checks run only after these pass, so these give every error and every
|
|
1817
|
+
// warning about SQL kcmd cannot read.
|
|
1818
|
+
const profileWarnings = [];
|
|
1819
|
+
const profileErrors = (0, resolve_profiles_1.namedProfileErrors)(docs, new Map(loaded.models.map(m => [m.document, m.model])), profileDocs, declaredBy, profileWarnings, load);
|
|
1820
|
+
for (const { document, model } of models) {
|
|
1821
|
+
profileErrors.push(...ofProfile(document, resolve_profiles_1.DEFAULT_PROFILE, ...validateInlineDialects(model, document), ...validateInheritedColumns(model, document)),
|
|
1822
|
+
// One call per profile, so each error carries its deployments' profile.
|
|
1823
|
+
...[...new Set(everyByDoc.get(document).map(d => d.profile))].flatMap(profile => ofProfile(document, profile, ...validateDeployments({
|
|
1824
|
+
...model,
|
|
1825
|
+
deployments: model.deployments.filter(d => (d.profile ?? resolve_profiles_1.DEFAULT_PROFILE) === profile),
|
|
1826
|
+
}, document, load))));
|
|
1827
|
+
const doc = docs.find(d => d.name === document);
|
|
1828
|
+
const declared = declaredBy(doc);
|
|
1829
|
+
const named = declared.some(p => p.name.toLowerCase() !== resolve_profiles_1.DEFAULT_PROFILE);
|
|
1830
|
+
profileWarnings.push(...inlineBindingWarnings(model, document, named));
|
|
1831
|
+
// A profile that no selected deployment applies still goes to Knowledge
|
|
1832
|
+
// Catalog, so its load warnings print here, less those the model file's
|
|
1833
|
+
// load gave, unless a check above recorded an error for that profile. A
|
|
1834
|
+
// selected deployment prints its own profile's warnings.
|
|
1835
|
+
const applied = selectedByDoc.get(document).map(d => d.profile);
|
|
1836
|
+
for (const { name } of declared) {
|
|
1837
|
+
if (applied.includes(name))
|
|
1838
|
+
continue;
|
|
1839
|
+
if (profileErrors.some(e => e.document === document && e.profile === name)) {
|
|
1840
|
+
continue;
|
|
1841
|
+
}
|
|
1842
|
+
profileWarnings.push(...(0, resolve_profiles_1.appliedProfileWarnings)(doc, declared, name, loaded.warnings, load));
|
|
1843
|
+
}
|
|
1844
|
+
}
|
|
1845
|
+
const kcWarnings = [];
|
|
1846
|
+
const kcErrors = models.flatMap(m => ofModel(m.document, ...validatePushRequirements([{ ...m, model: { ...m.model, deployments: [] } }], {
|
|
1847
|
+
warnings: kcWarnings,
|
|
1848
|
+
v2Aspects: kcV2Aspects(),
|
|
1849
|
+
})));
|
|
1850
|
+
for (const w of [...profileWarnings, ...kcWarnings])
|
|
1851
|
+
warn(w);
|
|
1852
|
+
if (profileErrors.length || kcErrors.length) {
|
|
1853
|
+
return stop([...profileErrors, ...kcErrors]);
|
|
1854
|
+
}
|
|
1855
|
+
// Each deployment's tables must be in the database it deploys to, whether
|
|
1856
|
+
// or not this push selects the deployment.
|
|
1857
|
+
const tableErrors = models.flatMap(({ document, model }) => (everyByDoc.get(document) ?? []).flatMap(d => ofProfile(document, d.profile, ...deploymentTableErrors(document, model, d))));
|
|
1858
|
+
if (tableErrors.length)
|
|
1859
|
+
return stop(tableErrors);
|
|
1860
|
+
// Spanner creates no property graph in a PostgreSQL-dialect database, so a
|
|
1861
|
+
// selected Spanner deployment there is refused by the dialect already read.
|
|
1862
|
+
const dialectErrors = models.flatMap(({ document }) => (selectedByDoc.get(document) ?? []).flatMap(d => {
|
|
1863
|
+
if (d.engine !== 'spanner')
|
|
1864
|
+
return [];
|
|
1865
|
+
const { project, instance, database } = d.coordinates;
|
|
1866
|
+
const key = `spanner.googleapis.com/projects/${project}/instances/${instance}/databases/${database}`;
|
|
1867
|
+
if (load.sourceDialects?.get(key) !== 'POSTGRES')
|
|
1868
|
+
return [];
|
|
1869
|
+
return ofProfile(document, d.profile, `[${document}] deployment '${d.name}': database ${project}/${instance}/${database} has the PostgreSQL dialect, and ` +
|
|
1870
|
+
`Spanner creates no property graph in a PostgreSQL-dialect ` +
|
|
1871
|
+
`database. Remove the deployment; push still publishes the ` +
|
|
1872
|
+
`model to Knowledge Catalog.`);
|
|
1873
|
+
}));
|
|
1874
|
+
if (dialectErrors.length)
|
|
1875
|
+
return stop(dialectErrors);
|
|
1876
|
+
const plans = [];
|
|
1877
|
+
for (const base of models) {
|
|
1878
|
+
for (const deployment of selectedByDoc.get(base.document) ?? []) {
|
|
1879
|
+
const plan = await deploymentPlan(base, profileDocs(base.document), deployment, load, warn);
|
|
1880
|
+
if ('errors' in plan) {
|
|
1881
|
+
return stop(ofProfile(base.document, deployment.profile, ...plan.errors));
|
|
1882
|
+
}
|
|
1883
|
+
plans.push(plan);
|
|
1884
|
+
}
|
|
1885
|
+
}
|
|
1886
|
+
return { plans };
|
|
1887
|
+
}
|
|
1888
|
+
// Builds and checks the model `deployment` deploys from `base`, reading its
|
|
1889
|
+
// profile file in `files` again with its actions, and generates its DDL
|
|
1890
|
+
// offline. The profile applies with the source dialects in `load`. Returns
|
|
1891
|
+
// the plan, or the errors that stop the push.
|
|
1892
|
+
async function deploymentPlan(base, files, deployment, load, warn) {
|
|
1893
|
+
const { document, model } = base;
|
|
1894
|
+
const at = `[${document}] deployment '${deployment.name}'`;
|
|
1895
|
+
const fail = (...errors) => ({ errors });
|
|
1896
|
+
const spanner = spannerMetricsErrors(document, model, deployment);
|
|
1897
|
+
if (spanner.length)
|
|
1898
|
+
return fail(...spanner);
|
|
1899
|
+
let applied = model;
|
|
1900
|
+
if (deployment.profile !== resolve_profiles_1.DEFAULT_PROFILE) {
|
|
1901
|
+
// localChecks has checked that the model declares the profile and that
|
|
1902
|
+
// its file loads, so `model.profiles` holds it.
|
|
1903
|
+
let spec = model.profiles.find(p => p.name === deployment.profile);
|
|
1904
|
+
const file = files.find(p => p.name === deployment.profile);
|
|
1905
|
+
const warnings = [];
|
|
1906
|
+
try {
|
|
1907
|
+
// A profile in a GOOGLE block has no file and applies as loaded.
|
|
1908
|
+
if (file)
|
|
1909
|
+
spec = (0, resolve_profiles_1.loadProfileFile)(file.text, file.name);
|
|
1910
|
+
}
|
|
1911
|
+
catch (err) {
|
|
1912
|
+
if (!(err instanceof loader_1.SemanticModelError))
|
|
1913
|
+
throw err;
|
|
1914
|
+
return fail(`[${file.fileName}] ${err.message}`);
|
|
1915
|
+
}
|
|
1916
|
+
try {
|
|
1917
|
+
applied = (0, resolve_profiles_1.applyProfile)(model, spec, { load, warnings });
|
|
1918
|
+
}
|
|
1919
|
+
catch (err) {
|
|
1920
|
+
if (!(err instanceof loader_1.SemanticModelError))
|
|
1921
|
+
throw err;
|
|
1922
|
+
return fail(`${at}: ${err.message}`);
|
|
1923
|
+
}
|
|
1924
|
+
for (const w of warnings) {
|
|
1925
|
+
warn(`profile '${deployment.profile}': [${document}] ${w}`);
|
|
1926
|
+
}
|
|
1927
|
+
}
|
|
1928
|
+
// Messages name the deployment.
|
|
1929
|
+
const label = `${document}, deployment '${deployment.name}'`;
|
|
1930
|
+
const checked = { document: label, model: applied };
|
|
1931
|
+
// Pruning can remove the field these rules are about, so they run first.
|
|
1932
|
+
// localChecks has already warned about the SQL they cannot read.
|
|
1933
|
+
const unprunedErrors = [
|
|
1934
|
+
...validateInheritance([checked]),
|
|
1935
|
+
...validateExpressionReferences([checked]),
|
|
1936
|
+
];
|
|
1937
|
+
const { model: pruned, report } = (0, resolve_profiles_1.pruneUnavailable)(checked.model, deployment.profile);
|
|
1938
|
+
// Keeps only this deployment, so every reader sees its target alone.
|
|
1939
|
+
const narrowed = {
|
|
1940
|
+
document: checked.document,
|
|
1941
|
+
model: {
|
|
1942
|
+
...pruned,
|
|
1943
|
+
deployments: (model.deployments ?? []).filter(d => d.name === deployment.name),
|
|
1944
|
+
},
|
|
1945
|
+
};
|
|
1946
|
+
const errors = [
|
|
1947
|
+
...unprunedErrors,
|
|
1948
|
+
...validatePushRequirements([narrowed], {
|
|
1949
|
+
fieldsPruned: true,
|
|
1950
|
+
v2Aspects: kcV2Aspects(),
|
|
1951
|
+
}),
|
|
1952
|
+
];
|
|
1953
|
+
if (errors.length)
|
|
1954
|
+
return { errors };
|
|
1955
|
+
// An AlloyDB database holds no graph.
|
|
1956
|
+
if (deployment.engine === 'alloydb') {
|
|
1957
|
+
return { deployment, at, model: narrowed, report };
|
|
1958
|
+
}
|
|
1959
|
+
const generated = await generateOffline(deployment.engine === 'bigquery' ? deploy_bigquery_1.deployBigQuery : deploy_spanner_1.deploySpanner, [narrowed]);
|
|
1960
|
+
if (typeof generated === 'string')
|
|
1961
|
+
return fail(`${at}: ${generated}`);
|
|
1962
|
+
return { deployment, at, model: narrowed, report, generated };
|
|
1963
|
+
}
|
|
1964
|
+
// The warnings and the note push prints on what pruning leaves out of the
|
|
1965
|
+
// model `plan` deploys. Each line names the model, and the document and the
|
|
1966
|
+
// deployment.
|
|
1967
|
+
function pruningLines({ deployment, model, report: r }) {
|
|
1968
|
+
const where = `model '${model.model.name}' (${model.document})`;
|
|
1969
|
+
// What becomes of a field left out. An AlloyDB database holds no graph.
|
|
1970
|
+
const outcome = deployment.engine === 'alloydb' ? 'it is unavailable there' :
|
|
1971
|
+
'the graph omits it';
|
|
1972
|
+
const excluded = new Set(r.excludedFields);
|
|
1973
|
+
const omits = (field, why) => `Warning: field '${field}' in ${where} ${why}, so ${outcome}.`;
|
|
1974
|
+
// Such as "metric 'a'" or "metrics 'a' and 'b'", or '' for no names.
|
|
1975
|
+
const named = (kind, names) => names.length ?
|
|
1976
|
+
`${(0, loader_1.plural)(names.length, kind, `${kind}s`)} ${(0, loader_1.listed)(names.map(n => `'${n}'`))}` :
|
|
1977
|
+
'';
|
|
1978
|
+
const unavailable = [
|
|
1979
|
+
named('metric', r.droppedMetrics.map(m => m.name)),
|
|
1980
|
+
named('action', r.droppedActions.map(a => a.name)),
|
|
1981
|
+
].filter(n => n).join(r.droppedMetrics.length > 1 ? ', and ' : ' and ');
|
|
1982
|
+
const note = [
|
|
1983
|
+
excluded.size ? `excludes ${named('field', [...excluded])}.` : '',
|
|
1984
|
+
unavailable && `leaves ${unavailable} unavailable.`,
|
|
1985
|
+
].filter(p => p).join(' It ');
|
|
1986
|
+
return [
|
|
1987
|
+
...r.unboundFields.filter(f => !excluded.has(f))
|
|
1988
|
+
.map(f => omits(f, 'has no expression')),
|
|
1989
|
+
...r.droppedFields.map(d => omits(d.name, d.reason)),
|
|
1990
|
+
...note ? [`Note: ${where} ${note}`] : [],
|
|
1991
|
+
];
|
|
1992
|
+
}
|
|
1993
|
+
// A validate-only deploy builds the DDL and calls no service, so it needs no
|
|
1994
|
+
// project or token.
|
|
1995
|
+
const NO_SERVICE = new context_1.ApiContext('', '', '');
|
|
1996
|
+
// Generates a graph leg's DDL without calling a service, or gives why it
|
|
1997
|
+
// cannot be generated.
|
|
1998
|
+
async function generateOffline(deployLeg, models) {
|
|
1999
|
+
try {
|
|
2000
|
+
const result = await deployLeg(models, NO_SERVICE, { validateOnly: true });
|
|
2001
|
+
if (result.success)
|
|
2002
|
+
return result;
|
|
2003
|
+
return result.details ?? 'the graph could not be generated';
|
|
2004
|
+
}
|
|
2005
|
+
catch (err) {
|
|
2006
|
+
return err.message || String(err);
|
|
2007
|
+
}
|
|
2008
|
+
}
|
|
1624
2009
|
// Live pre-flight over the BigQuery-targeting models: confirms every entity's
|
|
1625
2010
|
// BigQuery source table is reachable BEFORE any destination leg runs, so a push
|
|
1626
2011
|
// fails fast when the model could not deploy, rather than surfacing a missing
|
|
@@ -1781,24 +2166,6 @@ function rejectedStatement(pending, store, detail) {
|
|
|
1781
2166
|
return `action '${pending.action.name}' in model '${pending.model}' (${pending.document}): statement ${pending.index + 1} of its sql executor was rejected by ${store}: ${detail} Statements reach the store exactly as written, so every ` +
|
|
1782
2167
|
`table and column has to be the name the database uses.`;
|
|
1783
2168
|
}
|
|
1784
|
-
// The first line of a multi-line diagnosis. The lines after it echo the
|
|
1785
|
-
// statement and point a caret at the offending token, which is worth nothing
|
|
1786
|
-
// here: the author has the statement open in the profile, the position in
|
|
1787
|
-
// `[at 1:54]` survives, and a caret counted from column 1 lands in the wrong
|
|
1788
|
-
// place once the message is embedded in a longer sentence.
|
|
1789
|
-
//
|
|
1790
|
-
// `\n` is matched in both its forms because Spanner's top-level `error.message`
|
|
1791
|
-
// arrives with its newlines ESCAPED -- a literal backslash and an n survive
|
|
1792
|
-
// JSON.parse -- while every other message uses real ones.
|
|
1793
|
-
function firstLine(message) {
|
|
1794
|
-
return message.replace(/\\n/g, '\n').split('\n')[0].trim();
|
|
1795
|
-
}
|
|
1796
|
-
// An API error's message reduced to its first line as a sentence, such as
|
|
1797
|
-
// "Unrecognized name: accountId; Did you mean account_id? [at 1:54]."
|
|
1798
|
-
function storeMessage(message, status) {
|
|
1799
|
-
const text = (message ?? '').trim();
|
|
1800
|
-
return text ? (0, parameters_1.sentence)(firstLine(text)) : `HTTP ${status}.`;
|
|
1801
|
-
}
|
|
1802
2169
|
// The Spanner arm. Each distinct database gets one session, reused across every
|
|
1803
2170
|
// statement bound to it and deleted at the end -- which also discards the
|
|
1804
2171
|
// read-write transactions PLAN had to begin and never committed. Returns one
|
|
@@ -1838,7 +2205,7 @@ async function validateSpannerActionStatements(models, spanner) {
|
|
|
1838
2205
|
// is reported rather than skipped: a push that could not verify its
|
|
1839
2206
|
// statements must not proceed as though it had.
|
|
1840
2207
|
errors.push(`could not open a session on ${store} to verify the sql executor ` +
|
|
1841
|
-
`statements in model '${statements[0].model}' (${statements[0].document}): ${storeMessage(opened.message, opened.status)}`);
|
|
2208
|
+
`statements in model '${statements[0].model}' (${statements[0].document}): ${(0, parameters_1.storeMessage)(opened.message, opened.status)}`);
|
|
1842
2209
|
continue;
|
|
1843
2210
|
}
|
|
1844
2211
|
try {
|
|
@@ -1853,7 +2220,7 @@ async function validateSpannerActionStatements(models, spanner) {
|
|
|
1853
2220
|
const res = await spanner.planDml(session, pending.sql, params, paramTypes);
|
|
1854
2221
|
if (res.status === 200)
|
|
1855
2222
|
continue;
|
|
1856
|
-
errors.push(rejectedStatement(pending, store, storeMessage(res.message, res.status)));
|
|
2223
|
+
errors.push(rejectedStatement(pending, store, (0, parameters_1.storeMessage)(res.message, res.status)));
|
|
1857
2224
|
}
|
|
1858
2225
|
}
|
|
1859
2226
|
finally {
|
|
@@ -1892,7 +2259,7 @@ async function validateBigQueryActionStatements(models, bq, defaultProject) {
|
|
|
1892
2259
|
const res = await bq.query(project, pending.sql, undefined, true, statementParameterTypes(pending.action, pending.sql));
|
|
1893
2260
|
if (res.status === 200)
|
|
1894
2261
|
continue;
|
|
1895
|
-
errors.push(rejectedStatement(pending, store, storeMessage(res.message, res.status)));
|
|
2262
|
+
errors.push(rejectedStatement(pending, store, (0, parameters_1.storeMessage)(res.message, res.status)));
|
|
1896
2263
|
}
|
|
1897
2264
|
}
|
|
1898
2265
|
return errors;
|