@google-cloud/kcmd 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/ts/kcmd/cli.js +241 -382
- package/build/ts/kcmd/okf/init_okf_workspace.js +1 -1
- package/build/ts/kcmd/semantic/deploy_bigquery.js +6 -13
- package/build/ts/kcmd/semantic/deploy_knowledge_catalog.d.ts +3 -3
- package/build/ts/kcmd/semantic/deploy_knowledge_catalog.js +16 -44
- package/build/ts/kcmd/semantic/deploy_spanner.js +7 -14
- package/build/ts/kcmd/semantic/deployment_target.d.ts +1 -4
- package/build/ts/kcmd/semantic/deployment_target.js +6 -62
- package/build/ts/kcmd/semantic/import_model.js +5 -8
- package/build/ts/kcmd/semantic/ir.d.ts +2 -4
- package/build/ts/kcmd/semantic/kc_converter.js +12 -22
- package/build/ts/kcmd/semantic/kc_custom_types.js +27 -21
- package/build/ts/kcmd/semantic/loader.js +3 -28
- package/build/ts/kcmd/semantic/osi_converter.js +13 -63
- package/build/ts/kcmd/semantic/runtime/runtime.d.ts +4 -6
- package/build/ts/kcmd/semantic/runtime/store.d.ts +14 -19
- package/build/ts/kcmd/semantic/runtime/store.js +21 -37
- package/build/ts/kcmd/semantic/validate.js +4 -30
- package/package.json +1 -1
|
@@ -107,7 +107,7 @@ async function initOkfWorkspace(ctx, manifest, options = {}) {
|
|
|
107
107
|
.readdirSync(okf_1.BUNDLE_DIR)
|
|
108
108
|
.some((f) => f.endsWith('.md'));
|
|
109
109
|
if (!fs.existsSync(indexPath) && !fs.existsSync(agentsPath) && !hasConceptFiles) {
|
|
110
|
-
fs.writeFileSync(indexPath, `---\nokf_version: "0.
|
|
110
|
+
fs.writeFileSync(indexPath, `---\nokf_version: "0.2"\n---\n# ${groupId}\n`, 'utf8');
|
|
111
111
|
}
|
|
112
112
|
return 0;
|
|
113
113
|
}
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
// shared with the Knowledge Catalog leg so a multi-destination push parses each
|
|
7
7
|
// document once), lowers each to `CREATE OR REPLACE PROPERTY GRAPH` DDL
|
|
8
8
|
// (generator), and executes that DDL against the BigQuery project named by the
|
|
9
|
-
// model's
|
|
9
|
+
// model's deployment target.
|
|
10
10
|
//
|
|
11
11
|
// This is a library module: it emits no console output. The generated DDL and
|
|
12
12
|
// any loader/generator warnings are returned in `DeployResult` for the CLI
|
|
@@ -18,7 +18,7 @@ exports.datasetLocation = datasetLocation;
|
|
|
18
18
|
exports.deployBigQuery = deployBigQuery;
|
|
19
19
|
const bigquery_1 = require("../gcp/bigquery");
|
|
20
20
|
const bigquery_2 = require("./bigquery");
|
|
21
|
-
// The
|
|
21
|
+
// The deployment-target reader lives in its own module so both deploy
|
|
22
22
|
// legs (BigQuery, Spanner) and the push-time validator share one parse. The
|
|
23
23
|
// BigQuery-facing views are re-exported here so existing importers
|
|
24
24
|
// (validate.ts, knowledge_catalog.ts, and the deploy tests) keep their
|
|
@@ -150,25 +150,18 @@ async function deployBigQuery(models, ctx, options = {}) {
|
|
|
150
150
|
};
|
|
151
151
|
for (const { document, model } of models) {
|
|
152
152
|
modelsSeen++;
|
|
153
|
-
|
|
154
|
-
let malformed;
|
|
155
|
-
try {
|
|
156
|
-
({ targets, malformed } = (0, deployment_target_1.bigQueryGraphTargets)(model));
|
|
157
|
-
}
|
|
158
|
-
catch (err) {
|
|
159
|
-
return fail(`Model '${model.name}' (${document}): ${err.message || err}`);
|
|
160
|
-
}
|
|
153
|
+
const { targets, malformed } = (0, deployment_target_1.bigQueryGraphTargets)(model);
|
|
161
154
|
if (!targets.length) {
|
|
162
155
|
// A malformed-but-present target must not be reported as "none
|
|
163
|
-
// declared" -- that sends the author hunting for
|
|
156
|
+
// declared" -- that sends the author hunting for a deployment they
|
|
164
157
|
// already wrote. Name the URIs that failed to parse.
|
|
165
158
|
if (malformed.length) {
|
|
166
159
|
return fail(`Model '${model.name}' (${document}) declares BigQuery Graph ` +
|
|
167
|
-
`
|
|
160
|
+
`deployment target(s) that could not be parsed: ` +
|
|
168
161
|
`${malformed.join(', ')}. Expected ${deployment_target_1.BQ_GRAPH_FORM}.`);
|
|
169
162
|
}
|
|
170
163
|
return fail(`Model '${model.name}' (${document}) declares no BigQuery Graph ` +
|
|
171
|
-
`
|
|
164
|
+
`deployment target; nothing to deploy.`);
|
|
172
165
|
}
|
|
173
166
|
// Malformed targets alongside valid ones: surface them so a typo'd URI is
|
|
174
167
|
// not silently dropped when the deploy otherwise succeeds.
|
|
@@ -94,9 +94,9 @@ export declare function deployEmittedModels(emitted: EmittedModel[], emitWarning
|
|
|
94
94
|
export declare function oneModelPerEntryGroupMessage(entryGroup: string, models: number): string;
|
|
95
95
|
export declare const MAX_ASPECT_DATA_BYTES: number;
|
|
96
96
|
/**
|
|
97
|
-
* The emitter's warnings for `models`, and
|
|
98
|
-
*
|
|
99
|
-
*
|
|
97
|
+
* The emitter's warnings for `models`, and the first aspect too large to
|
|
98
|
+
* store. With v2Aspects on, that error is why push's catalog leg cannot write
|
|
99
|
+
* them. Calls no service.
|
|
100
100
|
*/
|
|
101
101
|
export declare function checkCatalogAspects(models: LoadedModel[], opts: KcDeployOptions): {
|
|
102
102
|
error?: string;
|
|
@@ -121,21 +121,6 @@ function sleep(ms) {
|
|
|
121
121
|
*/
|
|
122
122
|
async function preflightKnowledgeCatalog(models, ctx, opts) {
|
|
123
123
|
const emit = emitModels(models, opts);
|
|
124
|
-
if (emit.error) {
|
|
125
|
-
return {
|
|
126
|
-
result: {
|
|
127
|
-
success: false,
|
|
128
|
-
details: emit.error,
|
|
129
|
-
warnings: emit.warnings,
|
|
130
|
-
created: 0,
|
|
131
|
-
updated: 0,
|
|
132
|
-
deleted: 0,
|
|
133
|
-
linked: 0,
|
|
134
|
-
unlinked: 0,
|
|
135
|
-
plan: [],
|
|
136
|
-
},
|
|
137
|
-
};
|
|
138
|
-
}
|
|
139
124
|
return preflightEmittedModels(emit.emitted, emit.warnings, ctx, opts);
|
|
140
125
|
}
|
|
141
126
|
/**
|
|
@@ -340,13 +325,13 @@ function validateEntryGroup(model, existing, opts, result) {
|
|
|
340
325
|
exports.MAX_ASPECT_DATA_BYTES = 120 * 1024;
|
|
341
326
|
const MAX_SCHEMA_ASPECT_DATA_BYTES = 2 * 1024 * 1024;
|
|
342
327
|
/**
|
|
343
|
-
* The emitter's warnings for `models`, and
|
|
344
|
-
*
|
|
345
|
-
*
|
|
328
|
+
* The emitter's warnings for `models`, and the first aspect too large to
|
|
329
|
+
* store. With v2Aspects on, that error is why push's catalog leg cannot write
|
|
330
|
+
* them. Calls no service.
|
|
346
331
|
*/
|
|
347
332
|
function checkCatalogAspects(models, opts) {
|
|
348
333
|
const emit = emitModels(models, opts);
|
|
349
|
-
const error = emit.
|
|
334
|
+
const error = emit.emitted.map(emittedAspectsError).find(e => e);
|
|
350
335
|
return { error, warnings: emit.warnings };
|
|
351
336
|
}
|
|
352
337
|
// Why push cannot write what `model` emits, or undefined when it can.
|
|
@@ -438,35 +423,22 @@ async function checkCustomTypes(cat, dest, typeIds) {
|
|
|
438
423
|
* Turns every authored model into its catalog resources.
|
|
439
424
|
*
|
|
440
425
|
* Pure (no network I/O), so the dry-run plan and any generation warnings are
|
|
441
|
-
* produced even when a later write fails.
|
|
442
|
-
* `custom_extension` (reached via the `semantic-model` aspect) throws; report
|
|
443
|
-
* it against its document, as the BigQuery leg does, rather than letting it
|
|
444
|
-
* escape as an uncaught stack trace.
|
|
426
|
+
* produced even when a later write fails.
|
|
445
427
|
*/
|
|
446
428
|
function emitModels(models, opts) {
|
|
447
429
|
const emitted = [];
|
|
448
430
|
const warnings = [];
|
|
449
|
-
for (const {
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
overviews: opts.overviews,
|
|
461
|
-
});
|
|
462
|
-
}
|
|
463
|
-
catch (err) {
|
|
464
|
-
return {
|
|
465
|
-
emitted,
|
|
466
|
-
warnings,
|
|
467
|
-
error: `Model '${model.name}' (${document}): ${err.message || err}`,
|
|
468
|
-
};
|
|
469
|
-
}
|
|
431
|
+
for (const { model } of models) {
|
|
432
|
+
const resources = kcEmit.generateCatalogResources(model, {
|
|
433
|
+
project: opts.project,
|
|
434
|
+
location: opts.location,
|
|
435
|
+
entryGroup: opts.entryGroup,
|
|
436
|
+
systemTypeProject: opts.systemTypeProject,
|
|
437
|
+
systemTypeLocation: opts.systemTypeLocation,
|
|
438
|
+
emitExpressions: opts.emitExpressions,
|
|
439
|
+
v2Aspects: opts.v2Aspects,
|
|
440
|
+
overviews: opts.overviews,
|
|
441
|
+
});
|
|
470
442
|
for (const w of resources.warnings)
|
|
471
443
|
warnings.push(`[${model.name}] ${w}`);
|
|
472
444
|
emitted.push({ model: model.name, resources });
|
|
@@ -3,10 +3,10 @@
|
|
|
3
3
|
//
|
|
4
4
|
// This is the Spanner leg of `kcmd push` for the semantic-model scope, a
|
|
5
5
|
// sibling to deploy_bigquery: it consumes models already parsed into the
|
|
6
|
-
// semantic IR (see loadSemanticModels, shared across legs so a
|
|
7
|
-
// push parses each document once), lowers each to `CREATE OR
|
|
8
|
-
// GRAPH` DDL (generator: ./spanner), and applies that DDL to
|
|
9
|
-
// database named by the model's
|
|
6
|
+
// semantic IR (see loadSemanticModels, shared across legs so a
|
|
7
|
+
// multi-destination push parses each document once), lowers each to `CREATE OR
|
|
8
|
+
// REPLACE PROPERTY GRAPH` DDL (generator: ./spanner), and applies that DDL to
|
|
9
|
+
// the Spanner database named by the model's deployment target.
|
|
10
10
|
//
|
|
11
11
|
// Spanner DDL is asynchronous: updateDatabaseDdl returns a long-running
|
|
12
12
|
// operation which this leg polls to completion, unlike the BigQuery leg's
|
|
@@ -86,23 +86,16 @@ async function deploySpanner(models, ctx, options = {}) {
|
|
|
86
86
|
const backoffMs = options.pollBackoffMs ?? POLL_BACKOFF_MS;
|
|
87
87
|
for (const { document, model } of models) {
|
|
88
88
|
modelsSeen++;
|
|
89
|
-
|
|
90
|
-
let malformed;
|
|
91
|
-
try {
|
|
92
|
-
({ targets, malformed } = (0, deployment_target_1.spannerGraphTargets)(model));
|
|
93
|
-
}
|
|
94
|
-
catch (err) {
|
|
95
|
-
return fail(`Model '${model.name}' (${document}): ${err.message || err}`);
|
|
96
|
-
}
|
|
89
|
+
const { targets, malformed } = (0, deployment_target_1.spannerGraphTargets)(model);
|
|
97
90
|
if (!targets.length) {
|
|
98
91
|
// A malformed-but-present target must not be reported as "none declared".
|
|
99
92
|
if (malformed.length) {
|
|
100
93
|
return fail(`Model '${model.name}' (${document}) declares Spanner Graph ` +
|
|
101
|
-
`
|
|
94
|
+
`deployment target(s) that could not be parsed: ` +
|
|
102
95
|
`${malformed.join(', ')}. Expected ${deployment_target_1.SPANNER_GRAPH_FORM}.`);
|
|
103
96
|
}
|
|
104
97
|
return fail(`Model '${model.name}' (${document}) declares no Spanner Graph ` +
|
|
105
|
-
`
|
|
98
|
+
`deployment target; nothing to deploy.`);
|
|
106
99
|
}
|
|
107
100
|
for (const bad of malformed) {
|
|
108
101
|
warnings.push(`[${model.name}] ignoring unparseable Spanner Graph target: ${bad}`);
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { DeploymentSpec, SemanticModel } from './ir';
|
|
2
2
|
export declare const BQ_GRAPH_FORM = "bigquery.googleapis.com/projects/<p>/datasets/<d>/propertyGraphs/<g>";
|
|
3
3
|
export declare const SPANNER_GRAPH_FORM = "spanner.googleapis.com/projects/<p>/instances/<i>/databases/<db>/graphs/<g>";
|
|
4
4
|
export declare const ALLOYDB_FORM = "alloydb.googleapis.com/projects/<p>/locations/<l>/clusters/<c>/databases/<db>";
|
|
@@ -45,8 +45,6 @@ export type ResolvedDeployment = ParsedDeploymentTarget & {
|
|
|
45
45
|
};
|
|
46
46
|
/** Why a deployment's target is not accepted in a `deployments:` list. */
|
|
47
47
|
export declare function deploymentTargetError(model: SemanticModel, d: DeploymentSpec): string;
|
|
48
|
-
/** Why a target in a GOOGLE block's `deploymentTargets` is not accepted. */
|
|
49
|
-
export declare function googleBlockTargetError(model: SemanticModel, target: string): string;
|
|
50
48
|
/**
|
|
51
49
|
* Why a push would reject `target`, or undefined when it accepts it. `at` says
|
|
52
50
|
* where it is written.
|
|
@@ -73,7 +71,6 @@ export declare function resolveModelDeployments(model: SemanticModel, selected?:
|
|
|
73
71
|
} | {
|
|
74
72
|
error: string;
|
|
75
73
|
};
|
|
76
|
-
export declare function googleBlockTargets(exts: readonly CustomExtension[] | undefined, modelName: string): string[];
|
|
77
74
|
export declare function unnamedDeployments(targets: readonly string[]): Array<{
|
|
78
75
|
name: string;
|
|
79
76
|
target: string;
|
|
@@ -2,10 +2,8 @@
|
|
|
2
2
|
// Reads where a model deploys and classifies each declared target.
|
|
3
3
|
//
|
|
4
4
|
// A semantic model names where it deploys with its `deployments:` list. Each
|
|
5
|
-
// entry has a name, a target and optionally the binding profile it uses.
|
|
6
|
-
//
|
|
7
|
-
// `custom_extensions` block, which is also where a vanilla document's GOOGLE
|
|
8
|
-
// block carries them. Three destination types are recognized here, each a
|
|
5
|
+
// entry has a name, a target and optionally the binding profile it uses. Three
|
|
6
|
+
// destination types are recognized here, each a
|
|
9
7
|
// GCP resource name, written with no leading `//`:
|
|
10
8
|
//
|
|
11
9
|
// BigQuery Graph:
|
|
@@ -36,11 +34,9 @@
|
|
|
36
34
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
37
35
|
exports.ALLOYDB_FORM = exports.SPANNER_GRAPH_FORM = exports.BQ_GRAPH_FORM = void 0;
|
|
38
36
|
exports.deploymentTargetError = deploymentTargetError;
|
|
39
|
-
exports.googleBlockTargetError = googleBlockTargetError;
|
|
40
37
|
exports.targetProblem = targetProblem;
|
|
41
38
|
exports.parseDeploymentSelection = parseDeploymentSelection;
|
|
42
39
|
exports.resolveModelDeployments = resolveModelDeployments;
|
|
43
|
-
exports.googleBlockTargets = googleBlockTargets;
|
|
44
40
|
exports.unnamedDeployments = unnamedDeployments;
|
|
45
41
|
exports.googleDeploymentTargets = googleDeploymentTargets;
|
|
46
42
|
exports.bigQueryGraphTargets = bigQueryGraphTargets;
|
|
@@ -48,8 +44,6 @@ exports.spannerGraphTargets = spannerGraphTargets;
|
|
|
48
44
|
exports.alloyDbTargets = alloyDbTargets;
|
|
49
45
|
const loader_1 = require("./loader");
|
|
50
46
|
const resolve_profiles_1 = require("./resolve_profiles");
|
|
51
|
-
// Our own vendor tag in the Ossie `custom_extensions` list.
|
|
52
|
-
const GOOGLE_VENDOR = 'GOOGLE';
|
|
53
47
|
// A BigQuery Graph deployment target.
|
|
54
48
|
const BQ_GRAPH_TARGET = /^\/\/bigquery\.googleapis\.com\/projects\/([A-Za-z0-9_-]+)\/datasets\/([A-Za-z0-9_-]+)\/propertyGraphs\/([A-Za-z0-9_-]+)$/;
|
|
55
49
|
// A Spanner Graph deployment target, named as Dataplex names the graph's
|
|
@@ -140,11 +134,6 @@ function clusterUri(legacy) {
|
|
|
140
134
|
function deploymentTargetError(model, d) {
|
|
141
135
|
return targetError(`Model '${model.name}': deployment '${d.name}' targets '${d.target}'`, d.target);
|
|
142
136
|
}
|
|
143
|
-
/** Why a target in a GOOGLE block's `deploymentTargets` is not accepted. */
|
|
144
|
-
function googleBlockTargetError(model, target) {
|
|
145
|
-
return targetError(`Model '${model.name}': the GOOGLE custom_extension's ` +
|
|
146
|
-
`'deploymentTargets' lists '${target}'`, target);
|
|
147
|
-
}
|
|
148
137
|
// The Dataplex name of a Spanner graph `target` names with `propertyGraphs`,
|
|
149
138
|
// or undefined for any other target.
|
|
150
139
|
function spannerGraphName(target) {
|
|
@@ -251,53 +240,13 @@ function resolveModelDeployments(model, selected) {
|
|
|
251
240
|
}
|
|
252
241
|
// Memoizes the parse per model object so the several callers within one push
|
|
253
242
|
// (validation, source-project lookup, per-leg routing, each deploy leg) share a
|
|
254
|
-
// single parse rather than re-running
|
|
243
|
+
// single parse rather than re-running the regexes once per reader.
|
|
255
244
|
// Keyed by model identity: the IR object is stable through a push (the loader
|
|
256
245
|
// and inheritance passes each hand on a fixed set of objects) and
|
|
257
|
-
//
|
|
246
|
+
// its deployments are not mutated after loading.
|
|
258
247
|
// A caller that narrows `deployments` builds a new model object. The returned
|
|
259
248
|
// value is read-only by convention -- no caller mutates its arrays.
|
|
260
249
|
const targetsCache = new WeakMap();
|
|
261
|
-
// The targets in a model's GOOGLE custom_extension(s). The extension `data` is
|
|
262
|
-
// an opaque, vendor-serialized JSON string (the loader keeps it verbatim); we
|
|
263
|
-
// own its `deploymentTargets` shape. Throws on malformed extension JSON.
|
|
264
|
-
function extensionTargets(model) {
|
|
265
|
-
return googleBlockTargets(model.customExtensions, model.name);
|
|
266
|
-
}
|
|
267
|
-
// The targets listed in the GOOGLE blocks among `exts`, in order. A string
|
|
268
|
-
// target is trimmed, and a blank one is dropped. A padded target is the same
|
|
269
|
-
// target, so trimming here keeps the duplicate check and the stored list in
|
|
270
|
-
// step. An entry that is not a string is kept as its JSON text, which parses
|
|
271
|
-
// as no supported destination. Throws when a GOOGLE block's data is not valid
|
|
272
|
-
// JSON.
|
|
273
|
-
function googleBlockTargets(exts, modelName) {
|
|
274
|
-
const uris = [];
|
|
275
|
-
for (const ext of exts ?? []) {
|
|
276
|
-
if (ext.vendorName !== GOOGLE_VENDOR) {
|
|
277
|
-
continue;
|
|
278
|
-
}
|
|
279
|
-
let data;
|
|
280
|
-
try {
|
|
281
|
-
data = JSON.parse(ext.data);
|
|
282
|
-
}
|
|
283
|
-
catch {
|
|
284
|
-
throw new Error(`Model '${modelName}': GOOGLE custom_extension 'data' is not valid JSON.`);
|
|
285
|
-
}
|
|
286
|
-
const list = data?.deploymentTargets;
|
|
287
|
-
if (!Array.isArray(list)) {
|
|
288
|
-
continue;
|
|
289
|
-
}
|
|
290
|
-
for (const uri of list) {
|
|
291
|
-
if (typeof uri !== 'string') {
|
|
292
|
-
uris.push(JSON.stringify(uri));
|
|
293
|
-
}
|
|
294
|
-
else if (uri.trim() !== '') {
|
|
295
|
-
uris.push(uri.trim());
|
|
296
|
-
}
|
|
297
|
-
}
|
|
298
|
-
}
|
|
299
|
-
return uris;
|
|
300
|
-
}
|
|
301
250
|
// Deployments for targets whose names are not known: a pull reads only the
|
|
302
251
|
// targets back from the catalog. One target is `default`; several are
|
|
303
252
|
// `deployment_1`, `deployment_2`, and so on. Targets are trimmed, and blank
|
|
@@ -317,17 +266,12 @@ function unnamedDeployments(targets) {
|
|
|
317
266
|
}));
|
|
318
267
|
}
|
|
319
268
|
// Reads a model's deployment targets in a single pass and returns the facts
|
|
320
|
-
// every caller derives from them. The targets come from `model.deployments
|
|
321
|
-
// when the model has it, and otherwise from its GOOGLE custom_extension(s).
|
|
322
|
-
// Never both: in the Google flavor, the loader's GOOGLE block holds a copy of
|
|
323
|
-
// the list. Throws on malformed extension JSON.
|
|
269
|
+
// every caller derives from them. The targets come from `model.deployments`.
|
|
324
270
|
function googleDeploymentTargets(model) {
|
|
325
271
|
const cached = targetsCache.get(model);
|
|
326
272
|
if (cached)
|
|
327
273
|
return cached;
|
|
328
|
-
const uris = model.deployments
|
|
329
|
-
model.deployments.map(d => d.target) :
|
|
330
|
-
extensionTargets(model);
|
|
274
|
+
const uris = (model.deployments ?? []).map(d => d.target);
|
|
331
275
|
const bigQuery = [];
|
|
332
276
|
const spanner = [];
|
|
333
277
|
const alloyDb = [];
|
|
@@ -1292,16 +1292,13 @@ function withDialects({ fileName, content }, profile, doc) {
|
|
|
1292
1292
|
return writeDocument({ document, links, text }, root);
|
|
1293
1293
|
}
|
|
1294
1294
|
/**
|
|
1295
|
-
* Validates every deployment target on `model` using `
|
|
1296
|
-
*
|
|
1297
|
-
* message.
|
|
1295
|
+
* Validates every deployment target on `model` using `targetProblem` so
|
|
1296
|
+
* invalid targets fail import with `kcmd push`'s exact message.
|
|
1298
1297
|
*/
|
|
1299
1298
|
function validateDeploymentTargets(model, doc) {
|
|
1300
|
-
for (const
|
|
1301
|
-
if ((0, deployment_target_1.targetProblem)('',
|
|
1302
|
-
|
|
1303
|
-
doc.errors.push(d ? (0, deployment_target_1.deploymentTargetError)(model, d) :
|
|
1304
|
-
(0, deployment_target_1.googleBlockTargetError)(model, t));
|
|
1299
|
+
for (const d of model.deployments ?? []) {
|
|
1300
|
+
if ((0, deployment_target_1.targetProblem)('', d.target)) {
|
|
1301
|
+
doc.errors.push((0, deployment_target_1.deploymentTargetError)(model, d));
|
|
1305
1302
|
}
|
|
1306
1303
|
}
|
|
1307
1304
|
}
|
|
@@ -33,10 +33,8 @@ export interface CustomExtension {
|
|
|
33
33
|
* (edges), with model-level metrics defined over them.
|
|
34
34
|
*
|
|
35
35
|
* The IR is primarily semantics. The loader reads a model's named
|
|
36
|
-
* `deployments:` list onto `deployments
|
|
37
|
-
*
|
|
38
|
-
* consumers that read it. A vanilla model lists its deployments in its GOOGLE
|
|
39
|
-
* block, which the loader reads and does not fold.
|
|
36
|
+
* `deployments:` list onto `deployments` (from `deployments:` in the Google
|
|
37
|
+
* flavor or from the model-level GOOGLE `custom_extensions` block in vanilla).
|
|
40
38
|
*/
|
|
41
39
|
export interface SemanticModel {
|
|
42
40
|
name: string;
|
|
@@ -31,9 +31,8 @@
|
|
|
31
31
|
// `guidelines` aspect, on the model/entity/metric), each metric's attach entity
|
|
32
32
|
// (re-derived from its expression, as the loader does, when the catalog holds
|
|
33
33
|
// one, else the value the emitter persisted), the model's deployment targets
|
|
34
|
-
// (from the semantic-model aspect, back into `deployments` and
|
|
35
|
-
// `
|
|
36
|
-
// `schema-join` entry links a pull fetched -- see
|
|
34
|
+
// (from the semantic-model aspect, back into `deployments`), and 1:1 / 1:N
|
|
35
|
+
// relationships (from the `schema-join` entry links a pull fetched -- see
|
|
37
36
|
// `modelsFromCatalogResources`'s `entryLinks` argument).
|
|
38
37
|
// The per-field `semantics` block -- field/metric expressions and the DIMENSION
|
|
39
38
|
// role
|
|
@@ -176,13 +175,11 @@ function modelsFromCatalogResources(entries, entryLinks = [], opts = {}) {
|
|
|
176
175
|
if (description !== undefined)
|
|
177
176
|
model.description = description;
|
|
178
177
|
readActionsAndConstraints(model, children(actionEntries), children(constraintEntries), warnings);
|
|
179
|
-
// Deployment targets ride back
|
|
180
|
-
//
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
model.customExtensions = [targets.extension];
|
|
185
|
-
model.deployments = targets.deployments;
|
|
178
|
+
// Deployment targets ride back as synthesized `deployments` entries
|
|
179
|
+
// (`default` when one, `deployment_1`, `deployment_2`, ... when several).
|
|
180
|
+
const deployments = readDeploymentTargets(anchor, name, warnings);
|
|
181
|
+
if (deployments) {
|
|
182
|
+
model.deployments = deployments;
|
|
186
183
|
}
|
|
187
184
|
const ai = readAiContext(anchor);
|
|
188
185
|
if (ai)
|
|
@@ -679,10 +676,10 @@ function aspectDataOf(aspects, type, path = '') {
|
|
|
679
676
|
return {};
|
|
680
677
|
}
|
|
681
678
|
// Recovers the model's deployment targets from its semantic-model aspect back
|
|
682
|
-
// into
|
|
683
|
-
//
|
|
684
|
-
//
|
|
685
|
-
//
|
|
679
|
+
// into a `deployments` list (`default` when there is one target,
|
|
680
|
+
// `deployment_1`, `deployment_2`, ... when there are several), warning that the
|
|
681
|
+
// original deployment names were not recovered. Returns undefined when the
|
|
682
|
+
// model has none.
|
|
686
683
|
function readDeploymentTargets(anchor, modelName, warnings) {
|
|
687
684
|
const deployments = (0, deployment_target_1.unnamedDeployments)(asArray(aspectData(anchor, 'semantic-model').deploymentTargets)
|
|
688
685
|
.filter((t) => typeof t === 'string'));
|
|
@@ -695,16 +692,9 @@ function readDeploymentTargets(anchor, modelName, warnings) {
|
|
|
695
692
|
if (problem !== undefined)
|
|
696
693
|
warnings.push(problem);
|
|
697
694
|
}
|
|
698
|
-
const targets = deployments.map(d => d.target);
|
|
699
695
|
warnings.push(`model '${modelName}': deployment names were not recovered (the ` +
|
|
700
696
|
`catalog stores target URIs only); using ${deployments.map(d => `'${d.name}'`).join(', ')}`);
|
|
701
|
-
return
|
|
702
|
-
extension: {
|
|
703
|
-
vendorName: 'GOOGLE',
|
|
704
|
-
data: JSON.stringify({ deploymentTargets: targets }),
|
|
705
|
-
},
|
|
706
|
-
deployments,
|
|
707
|
-
};
|
|
697
|
+
return deployments;
|
|
708
698
|
}
|
|
709
699
|
// Inverts schemaJoinAspectData (knowledge_catalog.ts): each schema-join link
|
|
710
700
|
// whose two endpoints are both this model's entity entries becomes one
|
|
@@ -671,40 +671,46 @@ async function provisionOne(cat, dest, type) {
|
|
|
671
671
|
const aspectLabel = `aspect type '${type.id}'`;
|
|
672
672
|
const aspect = await cat.createAspectType(home.project, home.location, type.id, type.aspectType);
|
|
673
673
|
if (aspect.status === 409) {
|
|
674
|
-
const upd = await cat.updateAspectType(home.project, home.location, type.id, type.aspectType, ['description', 'display_name', 'metadata_template'])
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
//
|
|
678
|
-
//
|
|
679
|
-
// that has already created the entry group.
|
|
680
|
-
//
|
|
681
|
-
// What the warning has to carry is the way OUT, because there is no way
|
|
682
|
-
// forward: `kcmd` creates and patches aspect types and does not delete
|
|
683
|
-
// them, so a project holding a template this build cannot patch stays that
|
|
684
|
-
// way until someone removes it by hand. Naming the resource and the
|
|
685
|
-
// command is the whole of the fix; a reader who has only "left it
|
|
686
|
-
// unchanged" has to work out which of several types in which project, and
|
|
687
|
-
// then that `gcloud dataplex aspect-types` is where it lives.
|
|
674
|
+
const upd = await cat.updateAspectType(home.project, home.location, type.id, type.aspectType, ['description', 'display_name', 'metadata_template'])
|
|
675
|
+
// A thrown error, such as a network failure, gets the retry warning.
|
|
676
|
+
.catch((err) => ({ status: 0, message: String(err?.message || err) }));
|
|
677
|
+
// Patching an existing type is best effort. Published entries depend on
|
|
678
|
+
// the type already there, so a failed patch leaves it and init goes on.
|
|
688
679
|
const reason = upd.status !== 200 ?
|
|
689
680
|
`${upd.message || upd.status}` :
|
|
690
|
-
await cat.awaitOperation(upd.result,
|
|
681
|
+
await cat.awaitOperation(upd.result, 'the update operation')
|
|
682
|
+
.catch((err) => String(err?.message || err));
|
|
691
683
|
if (upd.status === 403) {
|
|
692
684
|
// The type exists, so a refusal to patch it leaves it as it is.
|
|
693
685
|
warnings.push(`no permission to update the existing ${aspectLabel} in project ` +
|
|
694
686
|
`'${home.project}' (${reason}); left it unchanged`);
|
|
695
687
|
}
|
|
696
|
-
else if (reason
|
|
688
|
+
else if (reason &&
|
|
689
|
+
/Backwards incompatible template change/i.test(reason)) {
|
|
690
|
+
// Dataplex refuses a backwards-incompatible template change with this
|
|
691
|
+
// message, and the same template gets the same answer on every run.
|
|
692
|
+
// kcmd never deletes a type, so the warning names the gcloud commands
|
|
693
|
+
// that do.
|
|
697
694
|
const resource = customAspectTypeName(type.id, dest);
|
|
698
695
|
warnings.push(`left the existing ${aspectLabel} unchanged (${reason}). It is ` +
|
|
699
696
|
`${resource}, and Dataplex will not patch it into the template ` +
|
|
700
697
|
`this build states -- a template field that was removed or ` +
|
|
701
698
|
`renumbered is a backwards-incompatible change and is refused. A ` +
|
|
702
699
|
`push will fail wherever the two disagree. To move it forward, ` +
|
|
703
|
-
`delete
|
|
704
|
-
`delete ${type.id}
|
|
705
|
-
`--location=${home.location}
|
|
706
|
-
`
|
|
707
|
-
`
|
|
700
|
+
`delete the entry type and then the aspect type, and re-run this ` +
|
|
701
|
+
`command: gcloud dataplex entry-types delete ${type.id} ` +
|
|
702
|
+
`--project=${home.project} --location=${home.location} && gcloud ` +
|
|
703
|
+
`dataplex aspect-types delete ${type.id} --project=${home.project} ` +
|
|
704
|
+
`--location=${home.location}. Deleting the entry type also deletes ` +
|
|
705
|
+
`every entry of that type in the project, so push the models that ` +
|
|
706
|
+
`had them again afterwards.`);
|
|
707
|
+
}
|
|
708
|
+
else if (reason) {
|
|
709
|
+
// Any other failure, such as a 429 from the write quota, a 5xx, a failed
|
|
710
|
+
// operation or a network error, can pass on a later run.
|
|
711
|
+
warnings.push(`could not update the existing ${aspectLabel} in project ` +
|
|
712
|
+
`'${home.project}' (${reason}). Run this command again in a minute ` +
|
|
713
|
+
`to retry the update.`);
|
|
708
714
|
}
|
|
709
715
|
}
|
|
710
716
|
else if (aspect.status !== 200) {
|
|
@@ -1065,17 +1065,14 @@ function composeDescription(...parts) {
|
|
|
1065
1065
|
.filter((p) => !!p);
|
|
1066
1066
|
return kept.length ? kept.join('\n\n') : undefined;
|
|
1067
1067
|
}
|
|
1068
|
-
// The vendor tag for Google-specific extension blocks
|
|
1069
|
-
// deploy leg's reader). A model-level `deployments:` list folds into one.
|
|
1068
|
+
// The vendor tag for Google-specific extension blocks.
|
|
1070
1069
|
exports.GOOGLE_VENDOR = 'GOOGLE';
|
|
1071
1070
|
// Rewrites the author-friendly sugar forms into the canonical wire shape the
|
|
1072
1071
|
// schema validates, and rejects deprecated or flavor-mismatched model keys up
|
|
1073
1072
|
// front with actionable messages: the removed `deployment_target:` key,
|
|
1074
1073
|
// inline `profiles:` in either flavor, and `deployments:` or
|
|
1075
1074
|
// `entities:` in vanilla. Under the extended profile, `entities:` is rewritten
|
|
1076
|
-
// to `datasets
|
|
1077
|
-
// GOOGLE `custom_extensions` block on the IR after validation (see
|
|
1078
|
-
// convertModel). Operates on the parsed document before validation.
|
|
1075
|
+
// to `datasets:`. Operates on the parsed document before validation.
|
|
1079
1076
|
function normalizeDocumentSugars(doc, extended) {
|
|
1080
1077
|
if (!doc || typeof doc !== 'object')
|
|
1081
1078
|
return doc;
|
|
@@ -1154,17 +1151,6 @@ function normalizeModelSugars(m, extended) {
|
|
|
1154
1151
|
delete m.entities;
|
|
1155
1152
|
}
|
|
1156
1153
|
}
|
|
1157
|
-
// Builds the GOOGLE custom-extension block that carries a model's deployment
|
|
1158
|
-
// target URIs on the IR (the form the deploy leg reads). The native
|
|
1159
|
-
// `deployments:` list folds into this block after validation (see
|
|
1160
|
-
// convertModel). The extended profile rejects any authored GOOGLE block (see
|
|
1161
|
-
// rejectGoogleBlock), so there is never a pre-existing one to reconcile with.
|
|
1162
|
-
function deploymentTargetsExtension(targets) {
|
|
1163
|
-
return {
|
|
1164
|
-
vendorName: exports.GOOGLE_VENDOR,
|
|
1165
|
-
data: JSON.stringify({ deploymentTargets: targets }),
|
|
1166
|
-
};
|
|
1167
|
-
}
|
|
1168
1154
|
// True when `vendorName` names the GOOGLE vendor, in any letter case.
|
|
1169
1155
|
function isGoogleVendor(vendorName) {
|
|
1170
1156
|
return typeof vendorName === 'string' &&
|
|
@@ -2081,20 +2067,9 @@ function convertModel(m, version, opts, warnings) {
|
|
|
2081
2067
|
const ai = aiContextOrUndefined(m.ai_context);
|
|
2082
2068
|
if (ai)
|
|
2083
2069
|
model.aiContext = ai;
|
|
2084
|
-
// The blocks the model keeps are carried verbatim.
|
|
2085
|
-
// profile the native `deployments` list is folded into a GOOGLE block
|
|
2086
|
-
// appended after them; no authored GOOGLE block can collide with it, since
|
|
2087
|
-
// that flavor rejects one (see rejectGoogleBlock).
|
|
2088
|
-
// Vanilla deployments, read from a GOOGLE block, are not folded.
|
|
2070
|
+
// The blocks the model keeps are carried verbatim.
|
|
2089
2071
|
if (block.kept)
|
|
2090
2072
|
model.customExtensions = block.kept;
|
|
2091
|
-
const foldedTargets = version === exports.GOOGLE_VERSION ? deployments.map(d => d.target) : [];
|
|
2092
|
-
if (foldedTargets.length) {
|
|
2093
|
-
model.customExtensions = [
|
|
2094
|
-
...(model.customExtensions ?? []),
|
|
2095
|
-
deploymentTargetsExtension(foldedTargets),
|
|
2096
|
-
];
|
|
2097
|
-
}
|
|
2098
2073
|
return model;
|
|
2099
2074
|
}
|
|
2100
2075
|
// Converts the dataset `ds`, whose GOOGLE blocks convertModel has read into
|