@google-cloud/kcmd 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +203 -0
- package/README.md +264 -2
- package/build/ts/kcmd/cli.js +20252 -0
- package/build/ts/kcmd/gcp/alloydb.d.ts +12 -0
- package/build/ts/kcmd/gcp/alloydb.js +70 -0
- package/build/ts/kcmd/gcp/api.d.ts +29 -0
- package/build/ts/kcmd/gcp/api.js +162 -0
- package/build/ts/kcmd/gcp/biglake.d.ts +18 -0
- package/build/ts/kcmd/gcp/biglake.js +59 -0
- package/build/ts/kcmd/gcp/bigquery.d.ts +57 -0
- package/build/ts/kcmd/gcp/bigquery.js +134 -0
- package/build/ts/kcmd/gcp/context.d.ts +10 -0
- package/build/ts/kcmd/gcp/context.js +74 -0
- package/build/ts/kcmd/gcp/crm.d.ts +13 -0
- package/build/ts/kcmd/gcp/crm.js +91 -0
- package/build/ts/kcmd/gcp/dataplex.d.ts +147 -0
- package/build/ts/kcmd/gcp/dataplex.js +406 -0
- package/build/ts/kcmd/gcp/index.d.ts +3 -0
- package/build/ts/kcmd/gcp/index.js +21 -0
- package/build/ts/kcmd/gcp/spanner.d.ts +73 -0
- package/build/ts/kcmd/gcp/spanner.js +126 -0
- package/build/ts/kcmd/index.d.ts +7 -0
- package/build/ts/kcmd/index.js +46 -0
- package/build/ts/kcmd/layout.d.ts +16 -0
- package/build/ts/kcmd/layout.js +31 -0
- package/build/ts/kcmd/layouts/documents.d.ts +24 -0
- package/build/ts/kcmd/layouts/documents.js +264 -0
- package/build/ts/kcmd/layouts/okf.d.ts +31 -0
- package/build/ts/kcmd/layouts/okf.js +386 -0
- package/build/ts/kcmd/layouts/semantic-model.d.ts +74 -0
- package/build/ts/kcmd/layouts/semantic-model.js +621 -0
- package/build/ts/kcmd/layouts/standard.d.ts +13 -0
- package/build/ts/kcmd/layouts/standard.js +108 -0
- package/build/ts/kcmd/manifest.d.ts +34 -0
- package/build/ts/kcmd/manifest.js +262 -0
- package/build/ts/kcmd/metadata.d.ts +25 -0
- package/build/ts/kcmd/metadata.js +8 -0
- package/build/ts/kcmd/okf/catalog_residue.d.ts +14 -0
- package/build/ts/kcmd/okf/catalog_residue.js +124 -0
- package/build/ts/kcmd/okf/concept.d.ts +14 -0
- package/build/ts/kcmd/okf/concept.js +110 -0
- package/build/ts/kcmd/okf/concept_to_entry.d.ts +10 -0
- package/build/ts/kcmd/okf/concept_to_entry.js +281 -0
- package/build/ts/kcmd/okf/init_okf_workspace.d.ts +15 -0
- package/build/ts/kcmd/okf/init_okf_workspace.js +113 -0
- package/build/ts/kcmd/okf/links.d.ts +23 -0
- package/build/ts/kcmd/okf/links.js +468 -0
- package/build/ts/kcmd/okf/okf_metadata.d.ts +12 -0
- package/build/ts/kcmd/okf/okf_metadata.js +99 -0
- package/build/ts/kcmd/semantic/bigquery.d.ts +22 -0
- package/build/ts/kcmd/semantic/bigquery.js +469 -0
- package/build/ts/kcmd/semantic/binding.d.ts +3 -0
- package/build/ts/kcmd/semantic/binding.js +92 -0
- package/build/ts/kcmd/semantic/catalog_entry.d.ts +24 -0
- package/build/ts/kcmd/semantic/catalog_entry.js +35 -0
- package/build/ts/kcmd/semantic/converters/owl/convert.d.ts +26 -0
- package/build/ts/kcmd/semantic/converters/owl/convert.js +41 -0
- package/build/ts/kcmd/semantic/converters/owl/model.d.ts +98 -0
- package/build/ts/kcmd/semantic/converters/owl/model.js +30 -0
- package/build/ts/kcmd/semantic/converters/owl/parse.d.ts +11 -0
- package/build/ts/kcmd/semantic/converters/owl/parse.js +634 -0
- package/build/ts/kcmd/semantic/converters/owl/to_ir.d.ts +19 -0
- package/build/ts/kcmd/semantic/converters/owl/to_ir.js +390 -0
- package/build/ts/kcmd/semantic/deploy_bigquery.d.ts +20 -0
- package/build/ts/kcmd/semantic/deploy_bigquery.js +226 -0
- package/build/ts/kcmd/semantic/deploy_knowledge_catalog.d.ts +104 -0
- package/build/ts/kcmd/semantic/deploy_knowledge_catalog.js +1367 -0
- package/build/ts/kcmd/semantic/deploy_spanner.d.ts +15 -0
- package/build/ts/kcmd/semantic/deploy_spanner.js +150 -0
- package/build/ts/kcmd/semantic/deployment_target.d.ts +93 -0
- package/build/ts/kcmd/semantic/deployment_target.js +379 -0
- package/build/ts/kcmd/semantic/graph_ddl.d.ts +44 -0
- package/build/ts/kcmd/semantic/graph_ddl.js +656 -0
- package/build/ts/kcmd/semantic/import_model.d.ts +72 -0
- package/build/ts/kcmd/semantic/import_model.js +1698 -0
- package/build/ts/kcmd/semantic/ir.d.ts +589 -0
- package/build/ts/kcmd/semantic/ir.js +247 -0
- package/build/ts/kcmd/semantic/kc_actions.d.ts +45 -0
- package/build/ts/kcmd/semantic/kc_actions.js +531 -0
- package/build/ts/kcmd/semantic/kc_constraints.d.ts +44 -0
- package/build/ts/kcmd/semantic/kc_constraints.js +279 -0
- package/build/ts/kcmd/semantic/kc_converter.d.ts +37 -0
- package/build/ts/kcmd/semantic/kc_converter.js +839 -0
- package/build/ts/kcmd/semantic/kc_custom_types.d.ts +63 -0
- package/build/ts/kcmd/semantic/kc_custom_types.js +751 -0
- package/build/ts/kcmd/semantic/kc_entries.d.ts +155 -0
- package/build/ts/kcmd/semantic/kc_entries.js +284 -0
- package/build/ts/kcmd/semantic/kc_ids.d.ts +102 -0
- package/build/ts/kcmd/semantic/kc_ids.js +188 -0
- package/build/ts/kcmd/semantic/knowledge_catalog.d.ts +29 -0
- package/build/ts/kcmd/semantic/knowledge_catalog.js +960 -0
- package/build/ts/kcmd/semantic/loader.d.ts +258 -0
- package/build/ts/kcmd/semantic/loader.js +3576 -0
- package/build/ts/kcmd/semantic/names.d.ts +10 -0
- package/build/ts/kcmd/semantic/names.js +85 -0
- package/build/ts/kcmd/semantic/osi_converter.d.ts +60 -0
- package/build/ts/kcmd/semantic/osi_converter.js +910 -0
- package/build/ts/kcmd/semantic/parameters.d.ts +19 -0
- package/build/ts/kcmd/semantic/parameters.js +171 -0
- package/build/ts/kcmd/semantic/pull_kc.d.ts +10 -0
- package/build/ts/kcmd/semantic/pull_kc.js +272 -0
- package/build/ts/kcmd/semantic/resolve_inheritance.d.ts +59 -0
- package/build/ts/kcmd/semantic/resolve_inheritance.js +346 -0
- package/build/ts/kcmd/semantic/resolve_profiles.d.ts +291 -0
- package/build/ts/kcmd/semantic/resolve_profiles.js +2003 -0
- package/build/ts/kcmd/semantic/runtime/agent_tools.d.ts +93 -0
- package/build/ts/kcmd/semantic/runtime/agent_tools.js +243 -0
- package/build/ts/kcmd/semantic/runtime/dialect.d.ts +13 -0
- package/build/ts/kcmd/semantic/runtime/dialect.js +39 -0
- package/build/ts/kcmd/semantic/runtime/runtime.d.ts +88 -0
- package/build/ts/kcmd/semantic/runtime/runtime.js +301 -0
- package/build/ts/kcmd/semantic/runtime/store.d.ts +112 -0
- package/build/ts/kcmd/semantic/runtime/store.js +344 -0
- package/build/ts/kcmd/semantic/skills.d.ts +110 -0
- package/build/ts/kcmd/semantic/skills.js +697 -0
- package/build/ts/kcmd/semantic/source_dialect.d.ts +17 -0
- package/build/ts/kcmd/semantic/source_dialect.js +311 -0
- package/build/ts/kcmd/semantic/source_entries.d.ts +12 -0
- package/build/ts/kcmd/semantic/source_entries.js +341 -0
- package/build/ts/kcmd/semantic/spanner.d.ts +18 -0
- package/build/ts/kcmd/semantic/spanner.js +60 -0
- package/build/ts/kcmd/semantic/sql_expr_utils.d.ts +23 -0
- package/build/ts/kcmd/semantic/sql_expr_utils.js +394 -0
- package/build/ts/kcmd/semantic/sql_identifiers.d.ts +6 -0
- package/build/ts/kcmd/semantic/sql_identifiers.js +267 -0
- package/build/ts/kcmd/semantic/sql_parser.d.ts +47 -0
- package/build/ts/kcmd/semantic/sql_parser.js +400 -0
- package/build/ts/kcmd/semantic/transpile.d.ts +76 -0
- package/build/ts/kcmd/semantic/transpile.js +581 -0
- package/build/ts/kcmd/semantic/validate.d.ts +70 -0
- package/build/ts/kcmd/semantic/validate.js +2266 -0
- package/build/ts/kcmd/semantic/yaml_options.d.ts +6 -0
- package/build/ts/kcmd/semantic/yaml_options.js +58 -0
- package/build/ts/kcmd/semantic/yaml_update.d.ts +26 -0
- package/build/ts/kcmd/semantic/yaml_update.js +782 -0
- package/build/ts/kcmd/snapshot.d.ts +37 -0
- package/build/ts/kcmd/snapshot.js +386 -0
- package/build/ts/kcmd/source.d.ts +31 -0
- package/build/ts/kcmd/source.js +119 -0
- package/build/ts/kcmd/sources/bq-dataset.d.ts +17 -0
- package/build/ts/kcmd/sources/bq-dataset.js +131 -0
- package/build/ts/kcmd/sources/entrygroup.d.ts +21 -0
- package/build/ts/kcmd/sources/entrygroup.js +83 -0
- package/build/ts/kcmd/sources/kb.d.ts +21 -0
- package/build/ts/kcmd/sources/kb.js +82 -0
- package/build/ts/kcmd/sources/semantic-model.d.ts +18 -0
- package/build/ts/kcmd/sources/semantic-model.js +48 -0
- package/build/ts/kcmd/sync.d.ts +35 -0
- package/build/ts/kcmd/sync.js +734 -0
- package/package.json +62 -4
|
@@ -0,0 +1,1698 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// Imports a third-party (or first-party) semantic model YAML document into a
|
|
3
|
+
// valid kcmd semantic model document.
|
|
4
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
5
|
+
if (k2 === undefined) k2 = k;
|
|
6
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
7
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
8
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
9
|
+
}
|
|
10
|
+
Object.defineProperty(o, k2, desc);
|
|
11
|
+
}) : (function(o, m, k, k2) {
|
|
12
|
+
if (k2 === undefined) k2 = k;
|
|
13
|
+
o[k2] = m[k];
|
|
14
|
+
}));
|
|
15
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
16
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
17
|
+
}) : function(o, v) {
|
|
18
|
+
o["default"] = v;
|
|
19
|
+
});
|
|
20
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
21
|
+
var ownKeys = function(o) {
|
|
22
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
23
|
+
var ar = [];
|
|
24
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
25
|
+
return ar;
|
|
26
|
+
};
|
|
27
|
+
return ownKeys(o);
|
|
28
|
+
};
|
|
29
|
+
return function (mod) {
|
|
30
|
+
if (mod && mod.__esModule) return mod;
|
|
31
|
+
var result = {};
|
|
32
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
33
|
+
__setModuleDefault(result, mod);
|
|
34
|
+
return result;
|
|
35
|
+
};
|
|
36
|
+
})();
|
|
37
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
38
|
+
exports.importSemanticModelYaml = importSemanticModelYaml;
|
|
39
|
+
const yaml = __importStar(require("yaml"));
|
|
40
|
+
const semantic_model_1 = require("../layouts/semantic-model");
|
|
41
|
+
const deploy_knowledge_catalog_1 = require("./deploy_knowledge_catalog");
|
|
42
|
+
const deployment_target_1 = require("./deployment_target");
|
|
43
|
+
const ir_1 = require("./ir");
|
|
44
|
+
const loader_1 = require("./loader");
|
|
45
|
+
const names_1 = require("./names");
|
|
46
|
+
const parameters_1 = require("./parameters");
|
|
47
|
+
const resolve_profiles_1 = require("./resolve_profiles");
|
|
48
|
+
const source_dialect_1 = require("./source_dialect");
|
|
49
|
+
const sql_expr_utils_1 = require("./sql_expr_utils");
|
|
50
|
+
const sql_parser_1 = require("./sql_parser");
|
|
51
|
+
const transpile_1 = require("./transpile");
|
|
52
|
+
const validate_1 = require("./validate");
|
|
53
|
+
const yaml_options_1 = require("./yaml_options");
|
|
54
|
+
const yaml_update_1 = require("./yaml_update");
|
|
55
|
+
const STANDARD_AI_CONTEXT_KEYS = new Set(['instructions', 'synonyms', 'examples']);
|
|
56
|
+
// The loader's warning for a source it leaves as written.
|
|
57
|
+
const LOADER_SOURCE_WARNING = /^(.*?): source '(.*)' is not a valid GCP resource name\.$/s;
|
|
58
|
+
// Why no name to write is given when the name holds a quote.
|
|
59
|
+
const NO_QUOTE = 'kcmd cannot name a table whose name holds a quote';
|
|
60
|
+
// Each kind of --source-prefix for the service `prefix` names, or every kind
|
|
61
|
+
// when there is no prefix. The prefix's own kind comes first, so advice names
|
|
62
|
+
// its form.
|
|
63
|
+
function prefixKinds(prefix) {
|
|
64
|
+
const host = prefix?.base.split('/')[0];
|
|
65
|
+
const own = prefix &&
|
|
66
|
+
SOURCE_PREFIX_PATTERNS.find(k => k.re.test(prefix.base))?.form;
|
|
67
|
+
return SOURCE_PREFIX_PATTERNS.filter(k => host === undefined || k.form.startsWith(`${host}/`))
|
|
68
|
+
.sort((a, b) => Number(b.form === own) - Number(a.form === own));
|
|
69
|
+
}
|
|
70
|
+
// The warning for `source`, which import leaves as written after a run with
|
|
71
|
+
// `prefix`, if any. It gives the name to write when the loader names one,
|
|
72
|
+
// or another --source-prefix run when a prefix of that kind, or of any kind
|
|
73
|
+
// when none ran, completes the name, or else why the table cannot be named.
|
|
74
|
+
function sourceWarning(ctx, source, prefix) {
|
|
75
|
+
const problem = (0, loader_1.oldSourceForm)(source);
|
|
76
|
+
if (problem !== undefined) {
|
|
77
|
+
return `${ctx}: source '${source}' is ${problem}. 'kcmd push' rejects ` +
|
|
78
|
+
`the file until it is repointed.`;
|
|
79
|
+
}
|
|
80
|
+
const kinds = prefixKinds(prefix);
|
|
81
|
+
const bare = !/[/:]/.test(source);
|
|
82
|
+
const rerun = bare && completes(source, kinds);
|
|
83
|
+
const quoted = kinds.some(k => {
|
|
84
|
+
const parts = prefixSegments(source, k.form);
|
|
85
|
+
return !parts.length || parts.some(p => /[`"]/.test(p));
|
|
86
|
+
});
|
|
87
|
+
const cannot = !bare || rerun ? undefined :
|
|
88
|
+
/\s/.test(source) ? loader_1.NO_SPACE :
|
|
89
|
+
quoted ? NO_QUOTE :
|
|
90
|
+
undefined;
|
|
91
|
+
const dotted = !bare || rerun || cannot ?
|
|
92
|
+
undefined :
|
|
93
|
+
kinds.map(k => ({
|
|
94
|
+
form: k.form,
|
|
95
|
+
why: (0, loader_1.dottedPartProblem)(prefixSegments(source, k.form), k.form),
|
|
96
|
+
}))
|
|
97
|
+
.find(d => d.why);
|
|
98
|
+
return `${ctx}: source '${source}' is not a valid GCP resource name` +
|
|
99
|
+
(cannot ? `, and ${cannot}. ` :
|
|
100
|
+
dotted ? `; ${dotted.why}. Write the table's GCP resource name ` +
|
|
101
|
+
`instead, such as '${dotted.form}'. ` :
|
|
102
|
+
'; ') +
|
|
103
|
+
`'kcmd push' rejects the file until it is repointed` +
|
|
104
|
+
(rerun ? `, or until 'kcmd import' runs again with '--source-prefix'.` :
|
|
105
|
+
'.') +
|
|
106
|
+
((0, loader_1.isGoogleCloudForm)(source) ? '' : ` ${loader_1.THIRD_PARTY_SOURCE}`);
|
|
107
|
+
}
|
|
108
|
+
// Push's error for `source`, read as the loader reads a source in `ctx`, or
|
|
109
|
+
// undefined when push accepts it.
|
|
110
|
+
function sourceError(source, ctx) {
|
|
111
|
+
try {
|
|
112
|
+
(0, loader_1.parseSource)(source, {}, ctx);
|
|
113
|
+
return undefined;
|
|
114
|
+
}
|
|
115
|
+
catch (err) {
|
|
116
|
+
if (!(err instanceof loader_1.SemanticModelError))
|
|
117
|
+
throw err;
|
|
118
|
+
return (0, resolve_profiles_1.readerMessage)(err);
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
const ALLOWED_DIALECT_SET = new Set(ir_1.ALLOWED_DIALECTS);
|
|
122
|
+
const DIALECT_ALIASES = {
|
|
123
|
+
GOOGLE_SQL: 'BIGQUERY',
|
|
124
|
+
GOOGLESQL: 'BIGQUERY',
|
|
125
|
+
POSTGRESQL: 'POSTGRES',
|
|
126
|
+
TSQL: 'SQLSERVER',
|
|
127
|
+
};
|
|
128
|
+
// Each kind of --source-prefix: its pattern, the child container a bare name
|
|
129
|
+
// adds under it, if any, and the resource name of a table under it.
|
|
130
|
+
const SOURCE_PREFIX_PATTERNS = [
|
|
131
|
+
{
|
|
132
|
+
re: /^bigquery\.googleapis\.com\/projects\/([^/]+)$/,
|
|
133
|
+
child: 'datasets',
|
|
134
|
+
form: loader_1.TABLE_FORMS.bigquery,
|
|
135
|
+
},
|
|
136
|
+
{
|
|
137
|
+
re: /^bigquery\.googleapis\.com\/projects\/([^/]+)\/datasets\/([^/.]+)$/,
|
|
138
|
+
form: loader_1.TABLE_FORMS.bigquery,
|
|
139
|
+
},
|
|
140
|
+
{
|
|
141
|
+
re: /^biglake\.googleapis\.com\/projects\/([^/]+)\/catalogs\/([^/.]+)$/,
|
|
142
|
+
child: 'namespaces',
|
|
143
|
+
form: loader_1.TABLE_FORMS.biglake,
|
|
144
|
+
},
|
|
145
|
+
{
|
|
146
|
+
re: /^biglake\.googleapis\.com\/projects\/([^/]+)\/catalogs\/([^/.]+)\/namespaces\/([^/.]+)$/,
|
|
147
|
+
form: loader_1.TABLE_FORMS.biglake,
|
|
148
|
+
},
|
|
149
|
+
{
|
|
150
|
+
re: /^biglake\.googleapis\.com\/projects\/([^/]+)\/catalogs\/([^/.]+)\/databases\/([^/.]+)$/,
|
|
151
|
+
form: loader_1.TABLE_FORMS.biglakeHive,
|
|
152
|
+
},
|
|
153
|
+
{
|
|
154
|
+
re: /^spanner\.googleapis\.com\/projects\/([^/]+)\/instances\/([^/]+)\/databases\/([^/]+)$/,
|
|
155
|
+
form: loader_1.TABLE_FORMS.spanner,
|
|
156
|
+
},
|
|
157
|
+
{
|
|
158
|
+
re: /^alloydb\.googleapis\.com\/projects\/[^/]+\/locations\/[^/]+\/clusters\/[^/]+\/databases\/([^/]+)$/,
|
|
159
|
+
child: 'schemas',
|
|
160
|
+
form: loader_1.TABLE_FORMS.alloydb,
|
|
161
|
+
},
|
|
162
|
+
{
|
|
163
|
+
re: /^alloydb\.googleapis\.com\/projects\/[^/]+\/locations\/[^/]+\/clusters\/[^/]+\/databases\/([^/]+)\/schemas\/([^/]+)$/,
|
|
164
|
+
form: loader_1.TABLE_FORMS.alloydb,
|
|
165
|
+
},
|
|
166
|
+
{
|
|
167
|
+
re: /^cloudsql\.googleapis\.com\/projects\/[^/]+\/locations\/[^/]+\/instances\/[^/]+\/databases\/([^/]+)$/,
|
|
168
|
+
form: loader_1.TABLE_FORMS.cloudsql,
|
|
169
|
+
},
|
|
170
|
+
{
|
|
171
|
+
re: /^cloudsql\.googleapis\.com\/projects\/[^/]+\/locations\/[^/]+\/instances\/[^/]+\/databases\/([^/]+)\/schemas\/([^/]+)$/,
|
|
172
|
+
form: loader_1.TABLE_FORMS.cloudsqlSchema,
|
|
173
|
+
},
|
|
174
|
+
];
|
|
175
|
+
// An entry group a `--source-prefix` names. A system entry group, such as
|
|
176
|
+
// `@bigquery`, is refused, since its entries are Google Cloud tables, which a
|
|
177
|
+
// source names by resource name.
|
|
178
|
+
const ENTRY_GROUP_PREFIX = /^projects\/([^/]+)\/locations\/([^/]+)\/entryGroups\/([^/]+)$/;
|
|
179
|
+
/** Filters an unknown array down to its plain-object elements. */
|
|
180
|
+
function asRecords(v) {
|
|
181
|
+
return Array.isArray(v) ? v.filter(loader_1.isRecord) : [];
|
|
182
|
+
}
|
|
183
|
+
/**
|
|
184
|
+
* Returns true if any node in `doc` carries a `GOOGLE` `custom_extensions`
|
|
185
|
+
* block.
|
|
186
|
+
*/
|
|
187
|
+
function documentHasGoogleBlock(doc) {
|
|
188
|
+
for (const m of asRecords(doc.semantic_model)) {
|
|
189
|
+
const nodes = [
|
|
190
|
+
m,
|
|
191
|
+
...asRecords(m.datasets),
|
|
192
|
+
...asRecords(m.entities),
|
|
193
|
+
...asRecords(m.relationships),
|
|
194
|
+
...asRecords(m.metrics),
|
|
195
|
+
...asRecords(m.datasets).flatMap(ds => asRecords(ds.fields)),
|
|
196
|
+
...asRecords(m.entities).flatMap(ds => asRecords(ds.fields)),
|
|
197
|
+
];
|
|
198
|
+
if (nodes.some(n => (0, loader_1.hasGoogleBlock)(asRecords(n.custom_extensions)))) {
|
|
199
|
+
return true;
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
return false;
|
|
203
|
+
}
|
|
204
|
+
/** Moves non-standard `ai_context` sibling keys into `ai_context.custom`. */
|
|
205
|
+
function adaptAiContext(ai) {
|
|
206
|
+
const customKeys = Object.keys(ai).filter(k => !STANDARD_AI_CONTEXT_KEYS.has(k));
|
|
207
|
+
if (!customKeys.length)
|
|
208
|
+
return;
|
|
209
|
+
const custom = {};
|
|
210
|
+
for (const k of customKeys) {
|
|
211
|
+
custom[k] = ai[k];
|
|
212
|
+
delete ai[k];
|
|
213
|
+
}
|
|
214
|
+
ai.custom = custom;
|
|
215
|
+
}
|
|
216
|
+
/**
|
|
217
|
+
* Phase 1 (Parse): Parses raw YAML, verifies the top-level document envelope,
|
|
218
|
+
* validates `opts.sourcePrefix` when present, refuses a document with a
|
|
219
|
+
* `GOOGLE` block or a Google-flavor model to convert to vanilla unless
|
|
220
|
+
* `strict` is set, and constructs the mutable `ImportDocument` (or returns a
|
|
221
|
+
* fatal error string).
|
|
222
|
+
*/
|
|
223
|
+
function parseImportDocument(rawYaml, opts) {
|
|
224
|
+
// 1. Parse raw YAML with strict YAML 1.2 options (`YAML_OPTIONS`) so syntax
|
|
225
|
+
// errors and duplicate keys are caught immediately.
|
|
226
|
+
const { document, text } = (0, yaml_update_1.parseForUpdate)(rawYaml);
|
|
227
|
+
let parsedDoc;
|
|
228
|
+
try {
|
|
229
|
+
if (document.errors.length > 0)
|
|
230
|
+
throw document.errors[0];
|
|
231
|
+
parsedDoc = document.toJS();
|
|
232
|
+
}
|
|
233
|
+
catch (err) {
|
|
234
|
+
return `Semantic model load error: could not parse input: ${err?.message ?? err}`;
|
|
235
|
+
}
|
|
236
|
+
// 2. Verify the top-level document envelope (`version` is a recognized format
|
|
237
|
+
// version and `semantic_model[0]` is a model object). If malformed, delegate
|
|
238
|
+
// to `loadModels` so the caller receives the loader's canonical schema error.
|
|
239
|
+
if (!(0, loader_1.isRecord)(parsedDoc) || typeof parsedDoc.version !== 'string' ||
|
|
240
|
+
!(0, loader_1.isFormatVersion)(parsedDoc.version) ||
|
|
241
|
+
!Array.isArray(parsedDoc.semantic_model) ||
|
|
242
|
+
!(0, loader_1.isRecord)(parsedDoc.semantic_model[0])) {
|
|
243
|
+
try {
|
|
244
|
+
(0, loader_1.loadModels)(rawYaml, { bindingOptional: true, warnUnacceptedSource: true });
|
|
245
|
+
}
|
|
246
|
+
catch (err) {
|
|
247
|
+
return err?.message ?? String(err);
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
// 3. Validate `opts.sourcePrefix` against the supported first-party GCP
|
|
251
|
+
// container resource name patterns, or as an entry group, before doing any
|
|
252
|
+
// document work.
|
|
253
|
+
let sourcePrefix;
|
|
254
|
+
let entryGroup;
|
|
255
|
+
if (opts.sourcePrefix !== undefined) {
|
|
256
|
+
const base = opts.sourcePrefix.trim();
|
|
257
|
+
const match = SOURCE_PREFIX_PATTERNS.map(p => ({ m: base.match(p.re), p }))
|
|
258
|
+
.find(x => x.m !== null);
|
|
259
|
+
const group = base.match(ENTRY_GROUP_PREFIX);
|
|
260
|
+
if (match?.m) {
|
|
261
|
+
sourcePrefix = { base, known: match.m.slice(1), child: match.p.child };
|
|
262
|
+
}
|
|
263
|
+
else if (group?.[3].startsWith('@')) {
|
|
264
|
+
return `sourcePrefix '${opts.sourcePrefix}' names the system entry ` +
|
|
265
|
+
`group '${group[3]}', whose entries are Google Cloud tables. Pass ` +
|
|
266
|
+
`the GCP resource name of their database instead, such as ` +
|
|
267
|
+
`'bigquery.googleapis.com/projects/<p>/datasets/<d>'.`;
|
|
268
|
+
}
|
|
269
|
+
else if (group) {
|
|
270
|
+
entryGroup = {
|
|
271
|
+
base,
|
|
272
|
+
project: group[1],
|
|
273
|
+
location: group[2],
|
|
274
|
+
group: group[3],
|
|
275
|
+
list: opts.listEntries,
|
|
276
|
+
};
|
|
277
|
+
}
|
|
278
|
+
else {
|
|
279
|
+
return `Invalid sourcePrefix '${opts.sourcePrefix}'; expected the GCP ` +
|
|
280
|
+
`resource name of a BigQuery project or dataset, a BigLake ` +
|
|
281
|
+
`catalog, namespace or Hive database, a Spanner database, an ` +
|
|
282
|
+
`AlloyDB database or schema, or a Cloud SQL database or schema ` +
|
|
283
|
+
`(with no leading '//'), or the name of a Knowledge Catalog entry ` +
|
|
284
|
+
`group ('projects/<p>/locations/<l>/entryGroups/<g>').`;
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
// 4. Reject a document written for kcmd when `strict` is false: one with a
|
|
288
|
+
// `GOOGLE` block, or a Google-flavor model converting to vanilla.
|
|
289
|
+
const strict = !!opts.strict;
|
|
290
|
+
if (!strict && documentHasGoogleBlock(parsedDoc)) {
|
|
291
|
+
return `Document contains a '${loader_1.GOOGLE_VENDOR}' custom_extensions block ` +
|
|
292
|
+
`(written for kcmd); use '--input first-party' to convert a ` +
|
|
293
|
+
`first-party model.`;
|
|
294
|
+
}
|
|
295
|
+
if (!strict && opts.flavor === 'ossie' &&
|
|
296
|
+
parsedDoc.version === loader_1.GOOGLE_VERSION) {
|
|
297
|
+
return `Document is a '${loader_1.GOOGLE_VERSION}' model (written for kcmd); use ` +
|
|
298
|
+
`'--input first-party' to convert it to '${loader_1.OSSIE_VERSION}'.`;
|
|
299
|
+
}
|
|
300
|
+
// 5. Deep-copy `parsedDoc` via `plainCopy` to decouple shared YAML anchor /
|
|
301
|
+
// alias references before in-place mutations, and assemble `ImportDocument`.
|
|
302
|
+
const root = (0, resolve_profiles_1.plainCopy)(parsedDoc);
|
|
303
|
+
const model = root.semantic_model[0];
|
|
304
|
+
const inputVersion = root.version;
|
|
305
|
+
const targetVersion = opts.flavor ?
|
|
306
|
+
(opts.flavor === 'google' ? loader_1.GOOGLE_VERSION : loader_1.OSSIE_VERSION) :
|
|
307
|
+
inputVersion;
|
|
308
|
+
// A model file is YAML, so a JSON file is written as YAML.
|
|
309
|
+
const fileName = opts.fileName ?? `${model.name}.yaml`;
|
|
310
|
+
const warnings = text === rawYaml ? [] : [`Converted '${fileName}' to YAML.`];
|
|
311
|
+
const errors = [];
|
|
312
|
+
return {
|
|
313
|
+
text,
|
|
314
|
+
document,
|
|
315
|
+
links: (0, yaml_update_1.linkNodes)(root, document.contents),
|
|
316
|
+
root,
|
|
317
|
+
model,
|
|
318
|
+
datasets: asRecords(model.entities ?? model.datasets),
|
|
319
|
+
relationships: asRecords(model.relationships),
|
|
320
|
+
metrics: asRecords(model.metrics),
|
|
321
|
+
inputVersion,
|
|
322
|
+
targetVersion,
|
|
323
|
+
sourcePrefix,
|
|
324
|
+
entryGroup,
|
|
325
|
+
strict,
|
|
326
|
+
modified: false,
|
|
327
|
+
warnings,
|
|
328
|
+
errors,
|
|
329
|
+
warnOrError: (msg) => (strict ? errors : warnings).push(msg),
|
|
330
|
+
};
|
|
331
|
+
}
|
|
332
|
+
/**
|
|
333
|
+
* Extracts the profile name from a `<model>.profile.<name>.yaml` filename (or
|
|
334
|
+
* bare `<name>.yaml` / `<name>` identifier).
|
|
335
|
+
*/
|
|
336
|
+
function parseProfileName(fileName) {
|
|
337
|
+
const stem = fileName.replace(/^.*[/\\]/, '').replace(/\.ya?ml$/i, '');
|
|
338
|
+
const match = stem.match(/^.+?\.profile(?:\.(.*))?$/i);
|
|
339
|
+
return match ? (match[1] ?? '') : stem;
|
|
340
|
+
}
|
|
341
|
+
/**
|
|
342
|
+
* Extracts and parses all valid JSON object payloads from `GOOGLE`
|
|
343
|
+
* `custom_extensions` blocks on `exts`, leaving malformed JSON to be reported
|
|
344
|
+
* by `fromDocument`.
|
|
345
|
+
*/
|
|
346
|
+
function parseGoogleBlocks(exts) {
|
|
347
|
+
const blocks = [];
|
|
348
|
+
for (const b of asRecords(exts)) {
|
|
349
|
+
if (!(0, loader_1.isGoogleVendor)(b.vendor_name) || typeof b.data !== 'string')
|
|
350
|
+
continue;
|
|
351
|
+
try {
|
|
352
|
+
const parsed = JSON.parse(b.data);
|
|
353
|
+
if ((0, loader_1.isRecord)(parsed))
|
|
354
|
+
blocks.push(parsed);
|
|
355
|
+
}
|
|
356
|
+
catch {
|
|
357
|
+
// Leave malformed JSON to be reported by `fromDocument`.
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
return blocks;
|
|
361
|
+
}
|
|
362
|
+
/**
|
|
363
|
+
* Validates all model, deployment, profile, entity/dataset, field,
|
|
364
|
+
* relationship, and metric names in `doc` (plus any `googleBlocks`) in a
|
|
365
|
+
* single pass so every invalid or over-length identifier is reported at once.
|
|
366
|
+
*/
|
|
367
|
+
function validateObjectNames(doc, googleBlocks, modelScope) {
|
|
368
|
+
// Validates a single identifier `name` against `kind`'s rules via
|
|
369
|
+
// `nameProblem` and appends any error to `doc.errors`.
|
|
370
|
+
const validateName = (kind, name, scope) => {
|
|
371
|
+
if (typeof name !== 'string')
|
|
372
|
+
return;
|
|
373
|
+
const problem = (0, names_1.nameProblem)(kind, name);
|
|
374
|
+
if (problem)
|
|
375
|
+
doc.errors.push(scope ? `${scope}: ${problem}` : problem);
|
|
376
|
+
};
|
|
377
|
+
// Validates the `name` property of every record in `records`.
|
|
378
|
+
const validateRecords = (kind, records, scope) => {
|
|
379
|
+
for (const record of records)
|
|
380
|
+
validateName(kind, record.name, scope);
|
|
381
|
+
};
|
|
382
|
+
// 1. Validate model, deployment, and profile names (inline and `GOOGLE`
|
|
383
|
+
// block).
|
|
384
|
+
validateName('model', doc.model.name);
|
|
385
|
+
validateRecords('deployment', asRecords(doc.model.deployments), modelScope);
|
|
386
|
+
validateRecords('profile', asRecords(doc.model.profiles), modelScope);
|
|
387
|
+
for (const g of googleBlocks) {
|
|
388
|
+
validateRecords('deployment', asRecords(g.deployments), modelScope);
|
|
389
|
+
validateRecords('profile', asRecords(g.profiles), modelScope);
|
|
390
|
+
}
|
|
391
|
+
// 2. Validate dataset/entity, field, relationship, and metric names.
|
|
392
|
+
for (const ds of doc.datasets) {
|
|
393
|
+
validateName('entity', ds.name);
|
|
394
|
+
const dsScope = typeof ds.name === 'string' ? `dataset '${ds.name}'` : 'dataset';
|
|
395
|
+
validateRecords('field', asRecords(ds.fields), dsScope);
|
|
396
|
+
}
|
|
397
|
+
validateRecords('relationship', doc.relationships);
|
|
398
|
+
validateRecords('metric', doc.metrics);
|
|
399
|
+
}
|
|
400
|
+
/**
|
|
401
|
+
* Phase 2 (Pre-validation): Validates all identifier names in the input
|
|
402
|
+
* document. The loader rejects a model's legacy `deploymentTargets`.
|
|
403
|
+
*/
|
|
404
|
+
function validateInputDocument(doc) {
|
|
405
|
+
const modelScope = typeof doc.model.name === 'string' ?
|
|
406
|
+
`model '${doc.model.name}'` :
|
|
407
|
+
'model';
|
|
408
|
+
const googleBlocks = parseGoogleBlocks(doc.model.custom_extensions);
|
|
409
|
+
validateObjectNames(doc, googleBlocks, modelScope);
|
|
410
|
+
}
|
|
411
|
+
/**
|
|
412
|
+
* The parts of a bare dot-separated table name `source` as `prefix` reads
|
|
413
|
+
* them, each unquoted and marked when it was quoted, or `undefined` when
|
|
414
|
+
* `source` is not a bare name.
|
|
415
|
+
*/
|
|
416
|
+
function sourceParts(source, prefix) {
|
|
417
|
+
// Reject empty strings, already-slashed or colon-prefixed sources, and
|
|
418
|
+
// query-valued sources with unquoted whitespace.
|
|
419
|
+
const trimmed = source.trim();
|
|
420
|
+
if (!trimmed || trimmed.includes('/') || trimmed.includes(':') ||
|
|
421
|
+
(0, loader_1.hasUnquotedWhitespace)(trimmed)) {
|
|
422
|
+
return undefined;
|
|
423
|
+
}
|
|
424
|
+
const segs = prefixParts(trimmed, prefix?.base ?? '');
|
|
425
|
+
return segs.length && segs.every(s => s.text.length) ? segs : undefined;
|
|
426
|
+
}
|
|
427
|
+
/**
|
|
428
|
+
* `text` with its ASCII letters in lower case. PostgreSQL folds only these in a
|
|
429
|
+
* UTF-8 database.
|
|
430
|
+
*/
|
|
431
|
+
function lowerAscii(text) {
|
|
432
|
+
return text.replace(/[A-Z]/g, c => c.toLowerCase());
|
|
433
|
+
}
|
|
434
|
+
/**
|
|
435
|
+
* True when a prefix that starts with `base` names a database that reads
|
|
436
|
+
* unquoted names in lower case.
|
|
437
|
+
*/
|
|
438
|
+
function foldsUnquotedCase(base) {
|
|
439
|
+
return base.startsWith('alloydb.googleapis.com/');
|
|
440
|
+
}
|
|
441
|
+
/** `part` as the database of a prefix that starts with `base` reads it. */
|
|
442
|
+
function readPart(base, part) {
|
|
443
|
+
return foldsUnquotedCase(base) && !part.quoted ? lowerAscii(part.text) :
|
|
444
|
+
part.text;
|
|
445
|
+
}
|
|
446
|
+
/**
|
|
447
|
+
* The first unquoted part of `source` that PostgreSQL reads in lower case and
|
|
448
|
+
* so misses the name with capitals that `prefix` gives it, with the part's
|
|
449
|
+
* kind and both names. Expects a `source` that matches `prefix` when letter
|
|
450
|
+
* case is ignored.
|
|
451
|
+
*/
|
|
452
|
+
function foldedPart(source, prefix) {
|
|
453
|
+
const segs = sourceParts(source, prefix);
|
|
454
|
+
if (!segs || !foldsUnquotedCase(prefix.base))
|
|
455
|
+
return undefined;
|
|
456
|
+
const start = prefix.known.length - segs.length + (prefix.child ? 2 : 1);
|
|
457
|
+
for (let i = start; i < prefix.known.length; i++) {
|
|
458
|
+
const s = segs[i - start];
|
|
459
|
+
const read = lowerAscii(s.text);
|
|
460
|
+
// An AlloyDB prefix names a database, then perhaps a schema.
|
|
461
|
+
if (!s.quoted && read !== prefix.known[i]) {
|
|
462
|
+
return { kind: i ? 'schema' : 'database', name: prefix.known[i], read };
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
return undefined;
|
|
466
|
+
}
|
|
467
|
+
/**
|
|
468
|
+
* Qualifies a bare dot-separated table name `source` against `prefix` into a
|
|
469
|
+
* GCP resource name when `prefix` supplies the missing hierarchy segments and
|
|
470
|
+
* any overlapping segments agree, or returns `undefined` if `source` cannot be
|
|
471
|
+
* qualified. Against an AlloyDB prefix, an unquoted segment is read in lower
|
|
472
|
+
* case, as PostgreSQL reads it. With `ignoreCase`, every segment agrees in any
|
|
473
|
+
* letter case.
|
|
474
|
+
*/
|
|
475
|
+
function qualifyTableSource(source, prefix, ignoreCase = false) {
|
|
476
|
+
// Verify that any leading segments overlapping `prefix.known` match.
|
|
477
|
+
const segs = sourceParts(source, prefix);
|
|
478
|
+
if (!segs)
|
|
479
|
+
return undefined;
|
|
480
|
+
const needed = prefix.child ? 2 : 1;
|
|
481
|
+
const overlap = segs.length - needed;
|
|
482
|
+
if (overlap < 0 || overlap > prefix.known.length)
|
|
483
|
+
return undefined;
|
|
484
|
+
const read = (s) => readPart(prefix.base, s);
|
|
485
|
+
const agrees = (s, k) => read(s) === k || (ignoreCase && lowerAscii(s.text) === lowerAscii(k));
|
|
486
|
+
const tail = prefix.known.slice(prefix.known.length - overlap);
|
|
487
|
+
if (tail.some((k, i) => !agrees(segs[i], k)))
|
|
488
|
+
return undefined;
|
|
489
|
+
// Append the remaining child container (if any) and table segment onto
|
|
490
|
+
// `prefix.base` and verify the resulting resource name is accepted.
|
|
491
|
+
const rest = segs.slice(overlap).map(read);
|
|
492
|
+
const qualified = prefix.child ?
|
|
493
|
+
`${prefix.base}/${prefix.child}/${rest[0]}/tables/${rest[1]}` :
|
|
494
|
+
`${prefix.base}/tables/${rest[0]}`;
|
|
495
|
+
return (0, loader_1.oldSourceForm)(qualified) === undefined && (0, loader_1.isAcceptedSource)(qualified) ?
|
|
496
|
+
qualified :
|
|
497
|
+
undefined;
|
|
498
|
+
}
|
|
499
|
+
// The parts of the bare name `source` as a prefix that begins as `base` does
|
|
500
|
+
// reads them, each with whether quotes wrap it. A BigQuery or BigLake prefix
|
|
501
|
+
// reads a path in one pair of backticks, such as `` `p.d.t` ``, as its parts.
|
|
502
|
+
function prefixParts(source, base) {
|
|
503
|
+
return (0, loader_1.bareNameParts)(source.trim(), /^(bigquery|biglake)\.googleapis\.com\//.test(base));
|
|
504
|
+
}
|
|
505
|
+
// The text of each part prefixParts gives.
|
|
506
|
+
function prefixSegments(source, base) {
|
|
507
|
+
return prefixParts(source, base).map(p => p.text);
|
|
508
|
+
}
|
|
509
|
+
// The prefix of `kind` whose containers are the leading parts of a bare name
|
|
510
|
+
// with parts `parts`, or undefined when `kind` has no such prefix.
|
|
511
|
+
function prefixFor(kind, parts) {
|
|
512
|
+
const base = (0, loader_1.fillForm)(kind.form, parts)?.split('/').slice(0, kind.child ? -4 : -2)
|
|
513
|
+
.join('/');
|
|
514
|
+
const known = base?.match(kind.re)?.slice(1);
|
|
515
|
+
return base && known ? { base, known, child: kind.child } : undefined;
|
|
516
|
+
}
|
|
517
|
+
// True when a --source-prefix of one of `kinds` completes `source`. Each
|
|
518
|
+
// prefix tried is built from the parts as that kind reads them.
|
|
519
|
+
function completes(source, kinds) {
|
|
520
|
+
return kinds.some(kind => {
|
|
521
|
+
const parts = prefixParts(source, kind.form).map(p => readPart(kind.form, p));
|
|
522
|
+
const prefix = prefixFor(kind, parts);
|
|
523
|
+
return prefix !== undefined &&
|
|
524
|
+
qualifyTableSource(source, prefix) !== undefined;
|
|
525
|
+
});
|
|
526
|
+
}
|
|
527
|
+
/**
|
|
528
|
+
* Phase 2b (Catalog read): When `doc.entryGroup` is set and a dataset's
|
|
529
|
+
* source is a bare table name, lists the entry group's entries once into
|
|
530
|
+
* `doc.entryTables`, keeping those whose fullyQualifiedName names a table or
|
|
531
|
+
* view. Returns the error when the list fails.
|
|
532
|
+
*/
|
|
533
|
+
async function listEntryTables(doc) {
|
|
534
|
+
const group = doc.entryGroup;
|
|
535
|
+
if (!group ||
|
|
536
|
+
!doc.datasets.some(ds => typeof ds.source === 'string' && sourceParts(ds.source))) {
|
|
537
|
+
return undefined;
|
|
538
|
+
}
|
|
539
|
+
const tables = [];
|
|
540
|
+
try {
|
|
541
|
+
for await (const e of group.list(group.project, group.location, group.group)) {
|
|
542
|
+
const fqn = e.fullyQualifiedName ?? '';
|
|
543
|
+
const table = (0, source_dialect_1.tableFqnParts)(fqn);
|
|
544
|
+
if (!table)
|
|
545
|
+
continue;
|
|
546
|
+
tables.push({
|
|
547
|
+
name: `${group.base}${e.name.slice(e.name.indexOf('/entries/'))}`,
|
|
548
|
+
fullyQualifiedName: fqn,
|
|
549
|
+
parts: table.parts,
|
|
550
|
+
});
|
|
551
|
+
}
|
|
552
|
+
}
|
|
553
|
+
catch (err) {
|
|
554
|
+
return `Could not list the entries of entry group '${group.base}' to ` +
|
|
555
|
+
`complete bare table names: ${err?.message ?? err}`;
|
|
556
|
+
}
|
|
557
|
+
doc.entryTables = tables;
|
|
558
|
+
return undefined;
|
|
559
|
+
}
|
|
560
|
+
/**
|
|
561
|
+
* Completes the bare table name `source` of `ds` with the one entry in
|
|
562
|
+
* `doc.entryTables` whose fullyQualifiedName ends in its parts, an unquoted
|
|
563
|
+
* part in any letter case and a quoted part in its own case. Leaves a name no
|
|
564
|
+
* entry or several entries end in as written, and reports it with the
|
|
565
|
+
* candidates.
|
|
566
|
+
*/
|
|
567
|
+
function completeFromEntryGroup(ds, source, parts, doc) {
|
|
568
|
+
const dsLabel = `dataset '${ds.name}'`;
|
|
569
|
+
const group = doc.entryGroup.base;
|
|
570
|
+
// Compared from the end, so a name with more parts than an entry's matches
|
|
571
|
+
// none.
|
|
572
|
+
const matches = doc.entryTables.filter(t => parts.every((p, i) => {
|
|
573
|
+
const e = t.parts.at(i - parts.length);
|
|
574
|
+
return p.quoted ? p.text === e : p.text.toLowerCase() === e?.toLowerCase();
|
|
575
|
+
}));
|
|
576
|
+
if (matches.length === 1) {
|
|
577
|
+
// The author asked for this rewrite with --source-prefix, so it is a
|
|
578
|
+
// warning in strict mode too.
|
|
579
|
+
const [m] = matches;
|
|
580
|
+
doc.warnings.push(`${dsLabel}: rewrote bare source '${source}' to '${m.name}', the ` +
|
|
581
|
+
`entry whose fullyQualifiedName is '${m.fullyQualifiedName}'.`);
|
|
582
|
+
ds.source = m.name;
|
|
583
|
+
doc.modified = true;
|
|
584
|
+
return;
|
|
585
|
+
}
|
|
586
|
+
const left = `${dsLabel}: left bare source '${source}' as written, since `;
|
|
587
|
+
if (matches.length) {
|
|
588
|
+
const names = matches.map(m => `'${m.name}' (${m.fullyQualifiedName})`).join(', ');
|
|
589
|
+
doc.warnOrError(`${left}several entries in entry group '${group}' have a ` +
|
|
590
|
+
`fullyQualifiedName that ends in it: ${names}. 'kcmd push' rejects ` +
|
|
591
|
+
`the file until the source names one of them.`);
|
|
592
|
+
}
|
|
593
|
+
else {
|
|
594
|
+
doc.warnOrError(`${left}no entry for a table or view in entry group '${group}' has ` +
|
|
595
|
+
`a fullyQualifiedName that ends in it. 'kcmd push' rejects the file ` +
|
|
596
|
+
`until the source is the table's GCP resource name. ` +
|
|
597
|
+
loader_1.THIRD_PARTY_SOURCE);
|
|
598
|
+
}
|
|
599
|
+
}
|
|
600
|
+
/**
|
|
601
|
+
* When the output's flavor rejects `obj.datatype`, writes it in its canonical
|
|
602
|
+
* casing, such as `Integer` for `integer`, or deletes an `Opaque` in another
|
|
603
|
+
* casing. The change goes through `warnOrError` when the input's flavor
|
|
604
|
+
* rejects the value too.
|
|
605
|
+
*/
|
|
606
|
+
function normalizeDatatype(obj, label, doc) {
|
|
607
|
+
if (typeof obj.datatype !== 'string' ||
|
|
608
|
+
(0, ir_1.normalizeDataType)(obj.datatype, doc.targetVersion).ok) {
|
|
609
|
+
return;
|
|
610
|
+
}
|
|
611
|
+
const res = (0, ir_1.normalizeDataType)(obj.datatype, loader_1.GOOGLE_VERSION);
|
|
612
|
+
if (!res.ok)
|
|
613
|
+
return;
|
|
614
|
+
if (!(0, ir_1.normalizeDataType)(obj.datatype, doc.inputVersion).ok) {
|
|
615
|
+
doc.warnOrError(res.type === undefined ?
|
|
616
|
+
`${label}: removed datatype '${obj.datatype}'.` :
|
|
617
|
+
`${label}: rewrote datatype '${obj.datatype}' to '${res.type}'.`);
|
|
618
|
+
}
|
|
619
|
+
if (res.type === undefined) {
|
|
620
|
+
delete obj.datatype;
|
|
621
|
+
}
|
|
622
|
+
else {
|
|
623
|
+
obj.datatype = res.type;
|
|
624
|
+
}
|
|
625
|
+
doc.modified = true;
|
|
626
|
+
}
|
|
627
|
+
/**
|
|
628
|
+
* Maps dialect aliases onto the 9 allowed SQL dialects and drops unsupported
|
|
629
|
+
* non-SQL dialects on a field or metric, returning `'empty'` if all dialect
|
|
630
|
+
* entries were dropped.
|
|
631
|
+
*/
|
|
632
|
+
function normalizeExpressionDialects(obj, label, doc) {
|
|
633
|
+
if (!(0, loader_1.isRecord)(obj.expression) || !Array.isArray(obj.expression.dialects) ||
|
|
634
|
+
obj.expression.dialects.length === 0) {
|
|
635
|
+
return 'ok';
|
|
636
|
+
}
|
|
637
|
+
const kept = [];
|
|
638
|
+
for (const entry of obj.expression.dialects) {
|
|
639
|
+
if (!(0, loader_1.isRecord)(entry) || typeof entry.dialect !== 'string') {
|
|
640
|
+
kept.push(entry);
|
|
641
|
+
continue;
|
|
642
|
+
}
|
|
643
|
+
const upper = entry.dialect.trim().toUpperCase();
|
|
644
|
+
const mapped = DIALECT_ALIASES[upper] ??
|
|
645
|
+
(ALLOWED_DIALECT_SET.has(upper) ? upper : undefined);
|
|
646
|
+
if (mapped !== undefined) {
|
|
647
|
+
if (entry.dialect !== mapped) {
|
|
648
|
+
doc.warnOrError(`${label}: rewrote dialect '${entry.dialect}' to '${mapped}'.`);
|
|
649
|
+
entry.dialect = mapped;
|
|
650
|
+
doc.modified = true;
|
|
651
|
+
}
|
|
652
|
+
kept.push(entry);
|
|
653
|
+
}
|
|
654
|
+
else {
|
|
655
|
+
doc.warnOrError(`${label}: dropped unsupported dialect '${entry.dialect}'.`);
|
|
656
|
+
doc.modified = true;
|
|
657
|
+
}
|
|
658
|
+
}
|
|
659
|
+
obj.expression.dialects = kept;
|
|
660
|
+
return kept.length === 0 ? 'empty' : 'ok';
|
|
661
|
+
}
|
|
662
|
+
/**
|
|
663
|
+
* Returns why an unanchored metric `mt` is rejected by push's
|
|
664
|
+
* `validateExpressionReferences` (`'reaches no declared entity'` or
|
|
665
|
+
* `'reaches multiple entities (...)'`), or `undefined` when anchored or valid.
|
|
666
|
+
*/
|
|
667
|
+
function unanchoredMetricProblem(mt, entityNames, impliedDialect) {
|
|
668
|
+
// 1. Skip metrics that already declare an `entity` anchor natively or inside
|
|
669
|
+
// a `GOOGLE` `custom_extensions` block.
|
|
670
|
+
if (mt.entity !== undefined ||
|
|
671
|
+
(0, loader_1.hasGoogleBlock)(asRecords(mt.custom_extensions))) {
|
|
672
|
+
return undefined;
|
|
673
|
+
}
|
|
674
|
+
// 2. Collect all SQL dialect expressions on the metric; if none are present
|
|
675
|
+
// or any entry holds no SQL, leave the metric for `loadModels` to diagnose.
|
|
676
|
+
const sources = typeof mt.expression === 'string' ?
|
|
677
|
+
[{ text: mt.expression, dialect: 'ANSI_SQL' }] :
|
|
678
|
+
asRecords(mt.expression?.dialects)
|
|
679
|
+
.filter(d => typeof d.dialect === 'string' &&
|
|
680
|
+
typeof d.expression === 'string')
|
|
681
|
+
.map(d => ({ text: d.expression, dialect: d.dialect }));
|
|
682
|
+
if (!sources.length ||
|
|
683
|
+
sources.some(s => (0, loader_1.noSqlReason)(s.text, s.dialect) !== undefined)) {
|
|
684
|
+
return undefined;
|
|
685
|
+
}
|
|
686
|
+
const parsed = sources.map(s => (0, sql_expr_utils_1.columnsForSource)(s.text, s.dialect, impliedDialect));
|
|
687
|
+
const allColumns = parsed.flatMap(c => c ?? []);
|
|
688
|
+
// 3. Flag the unanchored metric if its parsed expressions reference zero
|
|
689
|
+
// columns (e.g. `COUNT(*)`) or span multiple declared entities.
|
|
690
|
+
const reached = (0, sql_expr_utils_1.entitiesReached)(allColumns, entityNames);
|
|
691
|
+
if (parsed.every(c => c !== undefined) && allColumns.length === 0) {
|
|
692
|
+
return 'reaches no declared entity';
|
|
693
|
+
}
|
|
694
|
+
if (reached.length > 1) {
|
|
695
|
+
return `reaches multiple entities (${reached.join(', ')})`;
|
|
696
|
+
}
|
|
697
|
+
return undefined;
|
|
698
|
+
}
|
|
699
|
+
/**
|
|
700
|
+
* Truncates `obj.description`, without its surrounding whitespace, to at most
|
|
701
|
+
* `MAX_ENTRY_DESCRIPTION` characters when it exceeds the limit, and records a
|
|
702
|
+
* warning (or error in `strict` mode).
|
|
703
|
+
*/
|
|
704
|
+
function truncateDescription(obj, label, doc) {
|
|
705
|
+
// Push counts the description without its surrounding whitespace.
|
|
706
|
+
const text = typeof obj.description === 'string' ? obj.description.trim() : '';
|
|
707
|
+
if (text.length > names_1.MAX_ENTRY_DESCRIPTION) {
|
|
708
|
+
// The cut never ends on the first half of a surrogate pair.
|
|
709
|
+
const cut = text.slice(0, names_1.MAX_ENTRY_DESCRIPTION).replace(/[\uD800-\uDBFF]$/, '');
|
|
710
|
+
doc.warnOrError(`${label}: truncated 'description' from ${text.length} ` +
|
|
711
|
+
`to ${cut.length} characters.`);
|
|
712
|
+
obj.description = cut;
|
|
713
|
+
doc.modified = true;
|
|
714
|
+
}
|
|
715
|
+
}
|
|
716
|
+
/**
|
|
717
|
+
* Normalizes a field's datatype and SQL dialect list, removing `f.expression`
|
|
718
|
+
* with a warning in the Google version (or recording an error in the vanilla
|
|
719
|
+
* version) if no supported SQL dialect remains.
|
|
720
|
+
*/
|
|
721
|
+
function repairField(f, dsName, doc) {
|
|
722
|
+
const fLabel = `field '${dsName}.${f.name}'`;
|
|
723
|
+
normalizeDatatype(f, fLabel, doc);
|
|
724
|
+
if (normalizeExpressionDialects(f, fLabel, doc) === 'empty') {
|
|
725
|
+
if (doc.targetVersion === loader_1.OSSIE_VERSION) {
|
|
726
|
+
doc.errors.push(`${fLabel}: no supported SQL dialect remains in 'expression', ` +
|
|
727
|
+
`and version '${loader_1.OSSIE_VERSION}' requires an 'expression' on ` +
|
|
728
|
+
`every field.`);
|
|
729
|
+
}
|
|
730
|
+
else {
|
|
731
|
+
delete f.expression;
|
|
732
|
+
doc.modified = true;
|
|
733
|
+
doc.warnOrError(`${fLabel}: no supported SQL dialect remains; removed ` +
|
|
734
|
+
`'expression' and left the field unbound.`);
|
|
735
|
+
}
|
|
736
|
+
}
|
|
737
|
+
}
|
|
738
|
+
/**
|
|
739
|
+
* Repairs a dataset's `source` (removing query-valued sources, qualifying bare
|
|
740
|
+
* table names against `doc.sourcePrefix` or completing them from
|
|
741
|
+
* `doc.entryTables`, and warning on unaccepted sources), truncates its
|
|
742
|
+
* `description`, and repairs each of its `fields`.
|
|
743
|
+
*/
|
|
744
|
+
function repairDataset(ds, doc) {
|
|
745
|
+
// 1. Repair or diagnose `ds.source`: remove query-valued sources (in the
|
|
746
|
+
// Google version), qualify bare table names against `doc.sourcePrefix` or
|
|
747
|
+
// complete them from `doc.entryTables`, or warn on unaccepted source
|
|
748
|
+
// formats.
|
|
749
|
+
const dsLabel = `dataset '${ds.name}'`;
|
|
750
|
+
if (typeof ds.source === 'string') {
|
|
751
|
+
const trimmed = ds.source.trim();
|
|
752
|
+
const qualified = doc.sourcePrefix ?
|
|
753
|
+
qualifyTableSource(trimmed, doc.sourcePrefix) :
|
|
754
|
+
undefined;
|
|
755
|
+
const parts = doc.entryTables ? sourceParts(trimmed) : undefined;
|
|
756
|
+
if ((0, loader_1.hasUnquotedWhitespace)(trimmed)) {
|
|
757
|
+
if (doc.targetVersion === loader_1.OSSIE_VERSION) {
|
|
758
|
+
doc.errors.push(`${dsLabel}: query-valued source '${trimmed}' is not supported, ` +
|
|
759
|
+
`and version '${loader_1.OSSIE_VERSION}' requires a table 'source' on ` +
|
|
760
|
+
`every dataset.`);
|
|
761
|
+
}
|
|
762
|
+
else {
|
|
763
|
+
doc.warnOrError(`${dsLabel}: removed query-valued source '${trimmed}'; ` +
|
|
764
|
+
`dataset is now unbound.`);
|
|
765
|
+
delete ds.source;
|
|
766
|
+
doc.modified = true;
|
|
767
|
+
}
|
|
768
|
+
}
|
|
769
|
+
else if (qualified !== undefined) {
|
|
770
|
+
// The author asked for this rewrite with --source-prefix, so it is a
|
|
771
|
+
// warning in strict mode too.
|
|
772
|
+
doc.warnings.push(`${dsLabel}: rewrote bare source '${trimmed}' to '${qualified}'.`);
|
|
773
|
+
ds.source = qualified;
|
|
774
|
+
doc.modified = true;
|
|
775
|
+
}
|
|
776
|
+
else if (parts) {
|
|
777
|
+
completeFromEntryGroup(ds, trimmed, parts, doc);
|
|
778
|
+
}
|
|
779
|
+
else if (sourceError(trimmed, dsLabel) !== undefined) {
|
|
780
|
+
// Why a prefix that nearly fits the name cannot complete it.
|
|
781
|
+
const p = doc.sourcePrefix;
|
|
782
|
+
const caseOnly = !!p && qualifyTableSource(trimmed, p, true) !== undefined;
|
|
783
|
+
const folded = p && caseOnly ? foldedPart(trimmed, p) : undefined;
|
|
784
|
+
let why;
|
|
785
|
+
if (folded) {
|
|
786
|
+
const { kind, name, read } = folded;
|
|
787
|
+
why = `names ${kind} '${read}', since PostgreSQL reads an unquoted ` +
|
|
788
|
+
`name in lower case. '--source-prefix' names ${kind} '${name}', ` +
|
|
789
|
+
`so import cannot complete the source. If the ${kind} is ` +
|
|
790
|
+
`'${name}', write it as '"${name}"' in the source; if it is ` +
|
|
791
|
+
`'${read}', write '${read}' in '--source-prefix'.`;
|
|
792
|
+
}
|
|
793
|
+
else if (caseOnly) {
|
|
794
|
+
why = `differs from '--source-prefix' only in letter case, so ` +
|
|
795
|
+
`import cannot complete it; correct the letter case of the ` +
|
|
796
|
+
`source or of '--source-prefix'.`;
|
|
797
|
+
}
|
|
798
|
+
else if (p?.child && sourceParts(trimmed, p)?.length === 1 &&
|
|
799
|
+
completes(trimmed, prefixKinds(p))) {
|
|
800
|
+
// The container it lacks: a dataset, namespace or schema.
|
|
801
|
+
const lacks = p.child.slice(0, -1);
|
|
802
|
+
why = `names no ${lacks}, and neither does '--source-prefix', so ` +
|
|
803
|
+
`import cannot complete it; add the ${lacks} to the source or to ` +
|
|
804
|
+
`'--source-prefix'.`;
|
|
805
|
+
}
|
|
806
|
+
doc.warnOrError(why ? `${dsLabel}: source '${trimmed}' ${why}` :
|
|
807
|
+
sourceWarning(dsLabel, trimmed, doc.sourcePrefix));
|
|
808
|
+
}
|
|
809
|
+
}
|
|
810
|
+
// 2. Cap the dataset's `description` at 2,000 characters and repair each of
|
|
811
|
+
// its `fields`.
|
|
812
|
+
truncateDescription(ds, dsLabel, doc);
|
|
813
|
+
for (const f of asRecords(ds.fields)) {
|
|
814
|
+
repairField(f, String(ds.name ?? ''), doc);
|
|
815
|
+
}
|
|
816
|
+
}
|
|
817
|
+
/**
|
|
818
|
+
* Repairs a metric's datatype, SQL dialect list, and description, returning
|
|
819
|
+
* `false` when the metric is unanchored and reaches 0 or >1 entities so the
|
|
820
|
+
* caller prunes it from `doc.metrics`.
|
|
821
|
+
*/
|
|
822
|
+
function repairMetric(mt, entityNames, impliedDialect, doc) {
|
|
823
|
+
const mtLabel = `metric '${mt.name}'`;
|
|
824
|
+
normalizeDatatype(mt, mtLabel, doc);
|
|
825
|
+
if (normalizeExpressionDialects(mt, mtLabel, doc) === 'empty') {
|
|
826
|
+
doc.errors.push(`${mtLabel}: no supported SQL dialect remains in 'expression'.`);
|
|
827
|
+
return true;
|
|
828
|
+
}
|
|
829
|
+
const problem = unanchoredMetricProblem(mt, entityNames, impliedDialect);
|
|
830
|
+
if (problem) {
|
|
831
|
+
// In strict mode nothing is written, so the author edits the input.
|
|
832
|
+
const flavor = doc.strict ? doc.inputVersion : doc.targetVersion;
|
|
833
|
+
const how = flavor === loader_1.OSSIE_VERSION ?
|
|
834
|
+
`with 'entity' in a ${loader_1.GOOGLE_VENDOR} custom_extensions block on the ` +
|
|
835
|
+
`metric (or use version '${loader_1.GOOGLE_VERSION}' and set 'entity' on ` +
|
|
836
|
+
`the metric)` :
|
|
837
|
+
`with an 'entity' anchor`;
|
|
838
|
+
doc.warnOrError(`${mtLabel}: removed unanchored metric whose expression ` +
|
|
839
|
+
`${problem}; add the metric back ${how}.`);
|
|
840
|
+
doc.modified = true;
|
|
841
|
+
return false;
|
|
842
|
+
}
|
|
843
|
+
truncateDescription(mt, mtLabel, doc);
|
|
844
|
+
return true;
|
|
845
|
+
}
|
|
846
|
+
/**
|
|
847
|
+
* Reads, with `opts.readDialects`, the dialect of each Cloud SQL, Spanner and
|
|
848
|
+
* third-party table the repaired model and its profile files name, as push
|
|
849
|
+
* reads them. A read that fails is one warning per table, or per database for
|
|
850
|
+
* Spanner, and the dialect of the tables it reads stays unknown. A lookup that
|
|
851
|
+
* cannot start is one warning for the run, and every such table's dialect
|
|
852
|
+
* stays unknown. An entry that was read and gives no dialect is reported as
|
|
853
|
+
* push's other refusals are.
|
|
854
|
+
*/
|
|
855
|
+
async function readSourceDialects(doc, opts) {
|
|
856
|
+
// The repaired sources are in `doc.root`, and JSON is YAML.
|
|
857
|
+
const texts = [
|
|
858
|
+
JSON.stringify(doc.root),
|
|
859
|
+
...(opts.profileFiles ?? []).map(f => f.content),
|
|
860
|
+
];
|
|
861
|
+
if (!opts.readDialects || !(0, source_dialect_1.sourcesToLookUp)(texts).length)
|
|
862
|
+
return {};
|
|
863
|
+
let got;
|
|
864
|
+
try {
|
|
865
|
+
got = await opts.readDialects(texts);
|
|
866
|
+
}
|
|
867
|
+
catch (err) {
|
|
868
|
+
// The lookup could not start, for example with no credentials. gcloud
|
|
869
|
+
// prints the details, so the warning keeps the first line.
|
|
870
|
+
const why = String(err?.message ?? err).split('\n')[0];
|
|
871
|
+
doc.warnings.push(`${(0, parameters_1.sentence)(`kcmd could not look up table dialects: ${why}`)} Import ` +
|
|
872
|
+
"skipped the checks that need a table's dialect.");
|
|
873
|
+
return {};
|
|
874
|
+
}
|
|
875
|
+
const skipped = 'Import skipped the checks that need its dialect.';
|
|
876
|
+
for (const e of got.errors) {
|
|
877
|
+
if (got.unread.includes(e))
|
|
878
|
+
doc.warnings.push(`${(0, parameters_1.sentence)(e)} ${skipped}`);
|
|
879
|
+
else
|
|
880
|
+
doc.warnOrError(e);
|
|
881
|
+
}
|
|
882
|
+
return { sourceDialects: got.dialects, sourceDatabases: got.databases };
|
|
883
|
+
}
|
|
884
|
+
/**
|
|
885
|
+
* Phase 3 (Repair): Repairs third-party constructs in-place on `doc` (pruning
|
|
886
|
+
* extra models, repairing datasets and their fields, pruning unanchored
|
|
887
|
+
* metrics, and truncating descriptions > 2,000 chars). Returns the source
|
|
888
|
+
* dialects it reads after repairing the datasets.
|
|
889
|
+
*/
|
|
890
|
+
async function repairDocument(doc, opts) {
|
|
891
|
+
// 1. Prune extra models after `semantic_model[0]` because `kcmd` allows only
|
|
892
|
+
// a single model per document, and cap the model's `description` at 2,000
|
|
893
|
+
// characters (`MAX_ENTRY_DESCRIPTION`).
|
|
894
|
+
if (doc.root.semantic_model.length > 1) {
|
|
895
|
+
for (const extra of doc.root.semantic_model.slice(1)) {
|
|
896
|
+
const name = (0, loader_1.isRecord)(extra) && typeof extra.name === 'string' ? extra.name : '';
|
|
897
|
+
doc.warnOrError(`Removed extra model${name ? ` '${name}'` : ''} from ` +
|
|
898
|
+
`'semantic_model'; the format allows one model per document.`);
|
|
899
|
+
}
|
|
900
|
+
doc.root.semantic_model = [doc.root.semantic_model[0]];
|
|
901
|
+
doc.modified = true;
|
|
902
|
+
}
|
|
903
|
+
truncateDescription(doc.model, `model '${doc.model.name}'`, doc);
|
|
904
|
+
// 2. Repair each dataset (its table `source`, `description`, and `fields`).
|
|
905
|
+
for (const ds of doc.datasets) {
|
|
906
|
+
repairDataset(ds, doc);
|
|
907
|
+
}
|
|
908
|
+
// 3. Read the source dialects, infer the model's source dialect from its
|
|
909
|
+
// repaired datasets via `sourcesDialect`, and repair or prune each metric
|
|
910
|
+
// (`repairMetric`). An ANSI_SQL text is not read when no lookup gave its
|
|
911
|
+
// tables' dialect, as push's checks read none.
|
|
912
|
+
const load = await readSourceDialects(doc, opts);
|
|
913
|
+
const entityNames = doc.datasets.map(ds => ds.name)
|
|
914
|
+
.filter((n) => typeof n === 'string');
|
|
915
|
+
const impliedDialect = (0, resolve_profiles_1.sourcesDialect)({
|
|
916
|
+
entities: doc.datasets.map(ds => {
|
|
917
|
+
const source = typeof ds.source === 'string' ? ds.source : undefined;
|
|
918
|
+
const key = source && (0, loader_1.dialectLookupKey)(source.trim());
|
|
919
|
+
return {
|
|
920
|
+
abstract: Boolean(ds.abstract),
|
|
921
|
+
authoredSource: source,
|
|
922
|
+
dataSource: '',
|
|
923
|
+
sourceDialect: key ? load.sourceDialects?.get(key) : undefined,
|
|
924
|
+
};
|
|
925
|
+
}),
|
|
926
|
+
});
|
|
927
|
+
doc.metrics = doc.metrics.filter(mt => repairMetric(mt, entityNames, impliedDialect, doc));
|
|
928
|
+
if (Array.isArray(doc.model.metrics) && doc.model.metrics.length) {
|
|
929
|
+
doc.model.metrics = doc.metrics.length ? doc.metrics : undefined;
|
|
930
|
+
}
|
|
931
|
+
// 4. Cap each relationship's `description`. A `GOOGLE` block comes only in
|
|
932
|
+
// strict mode, which writes nothing, so its description is only reported.
|
|
933
|
+
for (const r of doc.relationships) {
|
|
934
|
+
const rLabel = `relationship '${r.name}'`;
|
|
935
|
+
truncateDescription(r, rLabel, doc);
|
|
936
|
+
for (const b of parseGoogleBlocks(r.custom_extensions)) {
|
|
937
|
+
truncateDescription(b, rLabel, doc);
|
|
938
|
+
}
|
|
939
|
+
}
|
|
940
|
+
return load;
|
|
941
|
+
}
|
|
942
|
+
/**
|
|
943
|
+
* Sets `version` to `doc.targetVersion`, and names the model's entity list as
|
|
944
|
+
* that flavor does: `entities:` in the Google flavor, `datasets:` in vanilla.
|
|
945
|
+
* The renamed key keeps its place among the model's keys.
|
|
946
|
+
*/
|
|
947
|
+
function setTargetVersion(doc) {
|
|
948
|
+
const [from, to] = doc.targetVersion === loader_1.GOOGLE_VERSION ?
|
|
949
|
+
['datasets', 'entities'] :
|
|
950
|
+
['entities', 'datasets'];
|
|
951
|
+
doc.root.version = doc.targetVersion;
|
|
952
|
+
doc.modified = true;
|
|
953
|
+
if (doc.model[from] !== undefined && doc.model[to] === undefined) {
|
|
954
|
+
const entries = Object.entries(doc.model);
|
|
955
|
+
for (const k of Object.keys(doc.model))
|
|
956
|
+
delete doc.model[k];
|
|
957
|
+
for (const [k, v] of entries)
|
|
958
|
+
doc.model[k === from ? to : k] = v;
|
|
959
|
+
}
|
|
960
|
+
}
|
|
961
|
+
/**
|
|
962
|
+
* Converts a plain vanilla (`0.2.0.dev0`) document to the Google version
|
|
963
|
+
* (`0.2.0.dev0/google`) by updating `version`, renaming `datasets:` to
|
|
964
|
+
* `entities:`, and moving custom `ai_context` sibling properties under
|
|
965
|
+
* `ai_context.custom`.
|
|
966
|
+
*/
|
|
967
|
+
function adaptToGoogleVersion(doc) {
|
|
968
|
+
setTargetVersion(doc);
|
|
969
|
+
// Move any non-standard sibling keys on `ai_context` objects (at model,
|
|
970
|
+
// entity, field, relationship, or metric scope) under `ai_context.custom`,
|
|
971
|
+
// since `0.2.0.dev0/google` uses strict schema validation on `ai_context`.
|
|
972
|
+
for (const { ai } of (0, loader_1.modelAiContexts)(doc.root)) {
|
|
973
|
+
adaptAiContext(ai);
|
|
974
|
+
}
|
|
975
|
+
}
|
|
976
|
+
/**
|
|
977
|
+
* Adapts `doc` to the vanilla (`0.2.0.dev0`) version: converts a Google file's
|
|
978
|
+
* version, entity list and custom `ai_context` members, wraps short-form
|
|
979
|
+
* string expressions into a single-entry `ANSI_SQL` dialect list, and moves
|
|
980
|
+
* plain relationship `description`s into a `GOOGLE` `custom_extensions` block.
|
|
981
|
+
* The vanilla flavor rejects the last two forms, so in a vanilla file each of
|
|
982
|
+
* those changes is a repair and goes through `warnOrError`.
|
|
983
|
+
*/
|
|
984
|
+
function adaptToOssieVersion(doc) {
|
|
985
|
+
// 0. A Google file takes the vanilla version and entity list, and each
|
|
986
|
+
// member of an `ai_context`'s `custom` mapping goes beside the standard
|
|
987
|
+
// members, where `custom` was. An `ai_context` that held only an empty
|
|
988
|
+
// `custom` is dropped.
|
|
989
|
+
if (doc.inputVersion === loader_1.GOOGLE_VERSION) {
|
|
990
|
+
setTargetVersion(doc);
|
|
991
|
+
for (const owner of [
|
|
992
|
+
doc.model, ...doc.datasets,
|
|
993
|
+
...doc.datasets.flatMap(ds => asRecords(ds.fields)),
|
|
994
|
+
...doc.relationships, ...doc.metrics
|
|
995
|
+
]) {
|
|
996
|
+
const ai = owner.ai_context;
|
|
997
|
+
if (!(0, loader_1.isRecord)(ai) || !(0, loader_1.isRecord)(ai.custom))
|
|
998
|
+
continue;
|
|
999
|
+
const entries = Object.entries(ai);
|
|
1000
|
+
for (const k of Object.keys(ai))
|
|
1001
|
+
delete ai[k];
|
|
1002
|
+
for (const [k, v] of entries) {
|
|
1003
|
+
Object.assign(ai, k === 'custom' ? v : { [k]: v });
|
|
1004
|
+
}
|
|
1005
|
+
if (Object.keys(ai).length === 0)
|
|
1006
|
+
delete owner.ai_context;
|
|
1007
|
+
}
|
|
1008
|
+
}
|
|
1009
|
+
// 1. Wrap any short-form string `expression: "<sql>"` on a field or metric
|
|
1010
|
+
// into `{dialects: [{dialect: 'ANSI_SQL', expression: "<sql>"}]}`, since
|
|
1011
|
+
// `0.2.0.dev0` requires the `expression.dialects` object form.
|
|
1012
|
+
const wrapShortForm = (obj, label) => {
|
|
1013
|
+
if (typeof obj.expression !== 'string')
|
|
1014
|
+
return;
|
|
1015
|
+
obj.expression = {
|
|
1016
|
+
dialects: [{ dialect: 'ANSI_SQL', expression: obj.expression }],
|
|
1017
|
+
};
|
|
1018
|
+
doc.modified = true;
|
|
1019
|
+
if (doc.inputVersion === loader_1.OSSIE_VERSION) {
|
|
1020
|
+
doc.warnOrError(`${label}: wrote the string expression as an 'ANSI_SQL' ` +
|
|
1021
|
+
`dialects entry.`);
|
|
1022
|
+
}
|
|
1023
|
+
};
|
|
1024
|
+
for (const ds of doc.datasets) {
|
|
1025
|
+
for (const f of asRecords(ds.fields)) {
|
|
1026
|
+
wrapShortForm(f, `field '${ds.name}.${f.name}'`);
|
|
1027
|
+
}
|
|
1028
|
+
}
|
|
1029
|
+
for (const mt of doc.metrics)
|
|
1030
|
+
wrapShortForm(mt, `metric '${mt.name}'`);
|
|
1031
|
+
// 2. Relocate any top-level `description` on a relationship into a `GOOGLE`
|
|
1032
|
+
// `custom_extensions` block. The vanilla `0.2.0.dev0` schema does not allow
|
|
1033
|
+
// `description` directly on a relationship, but `kcmd` reads and preserves
|
|
1034
|
+
// relationship descriptions from a `{vendor_name: 'GOOGLE', data: ...}`
|
|
1035
|
+
// extension entry (`FORMAT_VERSION_PLACEMENT['0.2.0.dev0'].relationship`).
|
|
1036
|
+
// A conversion from Google has moved them already (`moveConstructs`).
|
|
1037
|
+
for (const r of doc.relationships) {
|
|
1038
|
+
if (typeof r.description !== 'string')
|
|
1039
|
+
continue;
|
|
1040
|
+
putGoogleBlock(r, { description: r.description });
|
|
1041
|
+
delete r.description;
|
|
1042
|
+
doc.modified = true;
|
|
1043
|
+
doc.warnOrError(`relationship '${r.name}': moved 'description' into a ` +
|
|
1044
|
+
`${loader_1.GOOGLE_VENDOR} custom_extensions block for '${loader_1.OSSIE_VERSION}'.`);
|
|
1045
|
+
}
|
|
1046
|
+
}
|
|
1047
|
+
// Where the Google flavor writes a key that a GOOGLE block carries in vanilla:
|
|
1048
|
+
// right after the last of these keys the object has, as `kcmd pull` does.
|
|
1049
|
+
const NATIVE_KEY_AFTER = {
|
|
1050
|
+
deployments: ['name', 'description', 'ai_context'],
|
|
1051
|
+
extends: ['name', 'source'],
|
|
1052
|
+
description: ['name', 'from', 'to', 'from_columns', 'to_columns'],
|
|
1053
|
+
entity: ['name'],
|
|
1054
|
+
};
|
|
1055
|
+
/**
|
|
1056
|
+
* Sets `obj[key]` to `value` right after the last of `after` that `obj` has,
|
|
1057
|
+
* or first. `obj` stays the same object, so the writer keeps its text.
|
|
1058
|
+
*/
|
|
1059
|
+
function setAfter(obj, after, key, value) {
|
|
1060
|
+
const entries = Object.entries(obj);
|
|
1061
|
+
const at = Math.max(-1, ...after.map(a => entries.findIndex(([k]) => k === a)));
|
|
1062
|
+
entries.splice(at + 1, 0, [key, value]);
|
|
1063
|
+
for (const k of Object.keys(obj))
|
|
1064
|
+
delete obj[k];
|
|
1065
|
+
for (const [k, v] of entries)
|
|
1066
|
+
obj[k] = v;
|
|
1067
|
+
}
|
|
1068
|
+
/**
|
|
1069
|
+
* Removes the `GOOGLE` blocks from `obj.custom_extensions`, and the list when
|
|
1070
|
+
* no block is left in it, and returns the values the blocks carry.
|
|
1071
|
+
*/
|
|
1072
|
+
function takeGoogleBlocks(obj) {
|
|
1073
|
+
const values = Object.assign({}, ...parseGoogleBlocks(obj.custom_extensions));
|
|
1074
|
+
if (!(0, loader_1.hasGoogleBlock)(asRecords(obj.custom_extensions)))
|
|
1075
|
+
return values;
|
|
1076
|
+
const kept = obj.custom_extensions.filter((b) => !(0, loader_1.isRecord)(b) || !(0, loader_1.isGoogleVendor)(b.vendor_name));
|
|
1077
|
+
if (kept.length) {
|
|
1078
|
+
obj.custom_extensions = kept;
|
|
1079
|
+
}
|
|
1080
|
+
else {
|
|
1081
|
+
delete obj.custom_extensions;
|
|
1082
|
+
}
|
|
1083
|
+
return values;
|
|
1084
|
+
}
|
|
1085
|
+
/** True for an empty string, list or mapping. */
|
|
1086
|
+
function isEmpty(value) {
|
|
1087
|
+
return value === '' ||
|
|
1088
|
+
(((0, loader_1.isRecord)(value) || Array.isArray(value)) &&
|
|
1089
|
+
Object.keys(value).length === 0);
|
|
1090
|
+
}
|
|
1091
|
+
/** Appends a `GOOGLE` block that carries `values` to `obj.custom_extensions`. */
|
|
1092
|
+
function putGoogleBlock(obj, values) {
|
|
1093
|
+
if (Object.keys(values).length === 0)
|
|
1094
|
+
return;
|
|
1095
|
+
obj.custom_extensions = [
|
|
1096
|
+
...(Array.isArray(obj.custom_extensions) ? obj.custom_extensions : []),
|
|
1097
|
+
{ vendor_name: loader_1.GOOGLE_VENDOR, data: JSON.stringify(values) },
|
|
1098
|
+
];
|
|
1099
|
+
}
|
|
1100
|
+
/**
|
|
1101
|
+
* Moves each construct that `doc`'s flavor and its target flavor place
|
|
1102
|
+
* differently (`FORMAT_VERSION_PLACEMENT`) between a native key, a `GOOGLE`
|
|
1103
|
+
* block and the profile files. A key the target flavor has no place for is
|
|
1104
|
+
* dropped, and so is an empty value bound for a `GOOGLE` block. Returns the
|
|
1105
|
+
* profile files the Google flavor writes.
|
|
1106
|
+
*/
|
|
1107
|
+
function moveConstructs(doc, profileFiles) {
|
|
1108
|
+
const levels = {
|
|
1109
|
+
model: [doc.model],
|
|
1110
|
+
dataset: doc.datasets,
|
|
1111
|
+
relationship: doc.relationships,
|
|
1112
|
+
metric: doc.metrics,
|
|
1113
|
+
};
|
|
1114
|
+
let files = [];
|
|
1115
|
+
for (const [level, objects] of Object.entries(levels)) {
|
|
1116
|
+
const placement = (version) => loader_1.FORMAT_VERSION_PLACEMENT[version][level];
|
|
1117
|
+
const was = placement(doc.inputVersion);
|
|
1118
|
+
for (const obj of objects) {
|
|
1119
|
+
const block = takeGoogleBlocks(obj);
|
|
1120
|
+
const moved = {};
|
|
1121
|
+
for (const [key, place] of Object.entries(placement(doc.targetVersion))) {
|
|
1122
|
+
// A key the input's flavor has no place for stays as written.
|
|
1123
|
+
if (was[key] === place || was[key] === 'none')
|
|
1124
|
+
continue;
|
|
1125
|
+
const value = was[key] === 'block' ? block[key] :
|
|
1126
|
+
was[key] === 'file' ?
|
|
1127
|
+
profileFiles.map(pf => yaml.parse(pf.content, yaml_options_1.YAML_OPTIONS)) :
|
|
1128
|
+
obj[key];
|
|
1129
|
+
// A third-party file can write natively a key its flavor keeps in a
|
|
1130
|
+
// GOOGLE block. That key stays as written.
|
|
1131
|
+
if (was[key] === 'native')
|
|
1132
|
+
delete obj[key];
|
|
1133
|
+
if (value === undefined)
|
|
1134
|
+
continue;
|
|
1135
|
+
if (place === 'native') {
|
|
1136
|
+
setAfter(obj, NATIVE_KEY_AFTER[key], key, value);
|
|
1137
|
+
}
|
|
1138
|
+
else if (place === 'file') {
|
|
1139
|
+
files = asRecords(value).map(p => ({
|
|
1140
|
+
fileName: `${doc.model.name}.profile.${p.name}.yaml`,
|
|
1141
|
+
content: yaml.stringify(p, {
|
|
1142
|
+
aliasDuplicateObjects: false,
|
|
1143
|
+
compat: 'yaml-1.1',
|
|
1144
|
+
lineWidth: 0,
|
|
1145
|
+
}),
|
|
1146
|
+
}));
|
|
1147
|
+
}
|
|
1148
|
+
else if (place === 'block' && !isEmpty(value)) {
|
|
1149
|
+
moved[key] = value;
|
|
1150
|
+
}
|
|
1151
|
+
}
|
|
1152
|
+
putGoogleBlock(obj, moved);
|
|
1153
|
+
}
|
|
1154
|
+
}
|
|
1155
|
+
return files;
|
|
1156
|
+
}
|
|
1157
|
+
/**
|
|
1158
|
+
* Loads `doc.root` into the `SemanticModel` IR, runs `transpileModel` to
|
|
1159
|
+
* translate vendor or double-quoted `ANSI_SQL` expressions to `BIGQUERY`, and
|
|
1160
|
+
* writes any newly added `BIGQUERY` dialect entries back onto `doc`, including
|
|
1161
|
+
* the field bindings of each BigQuery profile (`isBigQueryProfile`) in the
|
|
1162
|
+
* model's `GOOGLE` blocks. Translates the BigQuery profiles among
|
|
1163
|
+
* `profileFiles` the same way and returns every file. A warning about a JSON
|
|
1164
|
+
* profile file names the matching file of `inputFiles`.
|
|
1165
|
+
*/
|
|
1166
|
+
async function transpileDocument(doc, profileFiles, inputFiles = profileFiles) {
|
|
1167
|
+
let preLoaded;
|
|
1168
|
+
try {
|
|
1169
|
+
preLoaded = (0, loader_1.fromDocument)(doc.root, { bindingOptional: true, warnUnacceptedSource: true });
|
|
1170
|
+
}
|
|
1171
|
+
catch (err) {
|
|
1172
|
+
if (!(err instanceof loader_1.SemanticModelError))
|
|
1173
|
+
throw err;
|
|
1174
|
+
return profileFiles;
|
|
1175
|
+
}
|
|
1176
|
+
const syncNode = (...args) => {
|
|
1177
|
+
if (syncDialects(...args))
|
|
1178
|
+
doc.modified = true;
|
|
1179
|
+
};
|
|
1180
|
+
// Each of `profileFiles` as loaded, or undefined for one that does not
|
|
1181
|
+
// load. That one is written as it is when it parses, since push reports it.
|
|
1182
|
+
const loaded = profileFiles.map(pf => {
|
|
1183
|
+
try {
|
|
1184
|
+
return (0, resolve_profiles_1.loadProfileFile)(pf.content, parseProfileName(pf.fileName), { skipActions: true });
|
|
1185
|
+
}
|
|
1186
|
+
catch (err) {
|
|
1187
|
+
if (!(err instanceof loader_1.SemanticModelError))
|
|
1188
|
+
throw err;
|
|
1189
|
+
return undefined;
|
|
1190
|
+
}
|
|
1191
|
+
});
|
|
1192
|
+
let translatedFiles = profileFiles;
|
|
1193
|
+
for (const m of preLoaded.models) {
|
|
1194
|
+
const transpiled = await (0, transpile_1.transpileModel)({
|
|
1195
|
+
...m,
|
|
1196
|
+
profiles: [...(m.profiles ?? []), ...loaded.filter(p => p !== undefined)],
|
|
1197
|
+
}, { translatesProfile: isBigQueryProfile });
|
|
1198
|
+
doc.warnings.push(...transpiled.warnings);
|
|
1199
|
+
const profile = (name) => transpiled.model.profiles?.find(p => p.name === name);
|
|
1200
|
+
translatedFiles = profileFiles.map((pf, i) => {
|
|
1201
|
+
const p = loaded[i] && profile(loaded[i].name);
|
|
1202
|
+
const named = { ...pf, fileName: inputFiles[i].fileName };
|
|
1203
|
+
return p ? { ...pf, content: withDialects(named, p, doc) } : pf;
|
|
1204
|
+
});
|
|
1205
|
+
for (const ds of doc.datasets) {
|
|
1206
|
+
const irEntity = transpiled.model.entities.find(e => e.name === ds.name);
|
|
1207
|
+
for (const f of asRecords(ds.fields)) {
|
|
1208
|
+
syncNode(f, irEntity?.fields.find(x => x.name === f.name)?.dialects);
|
|
1209
|
+
}
|
|
1210
|
+
}
|
|
1211
|
+
for (const mt of doc.metrics) {
|
|
1212
|
+
syncNode(mt, transpiled.model.metrics.find(x => x.name === mt.name)?.dialects);
|
|
1213
|
+
}
|
|
1214
|
+
// A profile in a GOOGLE block is JSON text, so a block whose profiles
|
|
1215
|
+
// gained an entry is written back as JSON.
|
|
1216
|
+
for (const block of asRecords(doc.model.custom_extensions)) {
|
|
1217
|
+
const data = parseGoogleBlocks([block])[0];
|
|
1218
|
+
let changed = false;
|
|
1219
|
+
for (const p of asRecords(data?.profiles)) {
|
|
1220
|
+
changed = syncProfile(p, profile(p.name)) || changed;
|
|
1221
|
+
}
|
|
1222
|
+
if (changed) {
|
|
1223
|
+
block.data = JSON.stringify(data);
|
|
1224
|
+
doc.modified = true;
|
|
1225
|
+
}
|
|
1226
|
+
}
|
|
1227
|
+
}
|
|
1228
|
+
return translatedFiles;
|
|
1229
|
+
}
|
|
1230
|
+
// Whether `profile` is a BigQuery profile of `model`, by the rule push's import
|
|
1231
|
+
// hint uses: each source push accepts that it states for a concrete entity of
|
|
1232
|
+
// `model` is a BigQuery or BigLake table, and it states at least one.
|
|
1233
|
+
function isBigQueryProfile(profile, model) {
|
|
1234
|
+
const concrete = new Set(model.entities.filter(e => !e.abstract).map(e => e.name));
|
|
1235
|
+
const entities = profile.entities.flatMap(({ name, source }) => concrete.has(name) && source !== undefined &&
|
|
1236
|
+
sourceError(source, '') === undefined ?
|
|
1237
|
+
[{ authoredSource: source, dataSource: source }] :
|
|
1238
|
+
[]);
|
|
1239
|
+
return (0, resolve_profiles_1.sourcesDialect)({ entities }) === 'BIGQUERY';
|
|
1240
|
+
}
|
|
1241
|
+
/**
|
|
1242
|
+
* Adds to `rawNode.expression` each dialect entry, such as `BIGQUERY`, that
|
|
1243
|
+
* `transpileModel` appended to `irDialects`, and returns whether it added any.
|
|
1244
|
+
* The entries already in a dialects list stay as written.
|
|
1245
|
+
*/
|
|
1246
|
+
function syncDialects(rawNode, irDialects) {
|
|
1247
|
+
const dialects = rawNode.expression?.dialects;
|
|
1248
|
+
const prevCount = typeof rawNode.expression === 'string' ?
|
|
1249
|
+
1 :
|
|
1250
|
+
asRecords(dialects).length;
|
|
1251
|
+
if (prevCount === 0 || !irDialects || irDialects.length <= prevCount) {
|
|
1252
|
+
return false;
|
|
1253
|
+
}
|
|
1254
|
+
const entries = irDialects.map(d => ({ dialect: d.dialect, expression: d.expression }));
|
|
1255
|
+
if (typeof rawNode.expression === 'string') {
|
|
1256
|
+
rawNode.expression = { dialects: entries };
|
|
1257
|
+
}
|
|
1258
|
+
else {
|
|
1259
|
+
dialects.push(...entries.slice(prevCount));
|
|
1260
|
+
}
|
|
1261
|
+
return true;
|
|
1262
|
+
}
|
|
1263
|
+
/**
|
|
1264
|
+
* Writes onto the raw profile `p` each field expression that gained a dialect
|
|
1265
|
+
* entry in the translated `irProfile`, and returns whether any did.
|
|
1266
|
+
*/
|
|
1267
|
+
function syncProfile(p, irProfile) {
|
|
1268
|
+
let changed = false;
|
|
1269
|
+
for (const pe of asRecords(p.entities)) {
|
|
1270
|
+
const irEntity = irProfile?.entities.find(x => x.name === pe.name);
|
|
1271
|
+
for (const f of asRecords(pe.fields)) {
|
|
1272
|
+
const irField = irEntity?.fields?.find(x => x.name === f.name);
|
|
1273
|
+
changed = syncDialects(f, irField?.dialects) || changed;
|
|
1274
|
+
}
|
|
1275
|
+
}
|
|
1276
|
+
return changed;
|
|
1277
|
+
}
|
|
1278
|
+
/**
|
|
1279
|
+
* Returns the content of the profile file `file` with each field expression
|
|
1280
|
+
* that gained a dialect entry in the translated `profile`, written as the
|
|
1281
|
+
* model file is. A JSON file that changes is written as YAML, and a warning
|
|
1282
|
+
* on `doc` names it.
|
|
1283
|
+
*/
|
|
1284
|
+
function withDialects({ fileName, content }, profile, doc) {
|
|
1285
|
+
const { document, text } = (0, yaml_update_1.parseForUpdate)(content);
|
|
1286
|
+
const root = (0, resolve_profiles_1.plainCopy)(document.toJS());
|
|
1287
|
+
const links = (0, yaml_update_1.linkNodes)(root, document.contents);
|
|
1288
|
+
if (!syncProfile(root, profile))
|
|
1289
|
+
return content;
|
|
1290
|
+
if (text !== content)
|
|
1291
|
+
doc.warnings.push(`Converted '${fileName}' to YAML.`);
|
|
1292
|
+
return writeDocument({ document, links, text }, root);
|
|
1293
|
+
}
|
|
1294
|
+
/**
|
|
1295
|
+
* Validates every deployment target on `model` using `googleDeploymentTargets`
|
|
1296
|
+
* and `targetProblem` so invalid targets fail import with `kcmd push`'s exact
|
|
1297
|
+
* message.
|
|
1298
|
+
*/
|
|
1299
|
+
function validateDeploymentTargets(model, doc) {
|
|
1300
|
+
for (const t of (0, deployment_target_1.googleDeploymentTargets)(model).uris) {
|
|
1301
|
+
if ((0, deployment_target_1.targetProblem)('', t)) {
|
|
1302
|
+
const d = model.deployments?.find(x => x.target === t);
|
|
1303
|
+
doc.errors.push(d ? (0, deployment_target_1.deploymentTargetError)(model, d) :
|
|
1304
|
+
(0, deployment_target_1.googleBlockTargetError)(model, t));
|
|
1305
|
+
}
|
|
1306
|
+
}
|
|
1307
|
+
}
|
|
1308
|
+
/**
|
|
1309
|
+
* Records push's error for the profile files `profileFiles` beside a vanilla
|
|
1310
|
+
* input in `modelFile`, which keeps its profiles in its `GOOGLE` block. Push
|
|
1311
|
+
* checks the name of each file first. The message is an error in strict mode
|
|
1312
|
+
* and a warning otherwise.
|
|
1313
|
+
*/
|
|
1314
|
+
function checkVanillaProfileFiles(doc, profileFiles, modelFile) {
|
|
1315
|
+
if (doc.inputVersion !== loader_1.OSSIE_VERSION || profileFiles.length === 0)
|
|
1316
|
+
return;
|
|
1317
|
+
const badName = profileFiles
|
|
1318
|
+
.map(pf => (0, semantic_model_1.profileNameError)(pf.fileName, parseProfileName(pf.fileName)))
|
|
1319
|
+
.find(e => e !== undefined);
|
|
1320
|
+
doc.warnOrError(badName ?? (0, semantic_model_1.vanillaSiblingError)(profileFiles[0].fileName, modelFile));
|
|
1321
|
+
}
|
|
1322
|
+
/**
|
|
1323
|
+
* Loads each sibling `<model>.profile.<name>.yaml` entry in `profileFiles` via
|
|
1324
|
+
* `loadProfileFile` and attaches the resulting `ProfileSpec` list to
|
|
1325
|
+
* `model.profiles`. Records in `doc.errors` push's error for the first file
|
|
1326
|
+
* that breaks a file rule beside `modelFile`, or each profile's load error
|
|
1327
|
+
* labeled with its file.
|
|
1328
|
+
*/
|
|
1329
|
+
function attachProfileFiles(model, profileFiles, doc, modelFile) {
|
|
1330
|
+
const files = (profileFiles ?? []).map(pf => ({ ...pf, name: parseProfileName(pf.fileName) }));
|
|
1331
|
+
// Push checks every file's name before it loads any profile, and stops at
|
|
1332
|
+
// the first file that breaks a file rule.
|
|
1333
|
+
const seen = new Map();
|
|
1334
|
+
try {
|
|
1335
|
+
for (const pf of files) {
|
|
1336
|
+
(0, semantic_model_1.checkProfileFileName)(modelFile, pf.fileName, pf.name, pf.content, seen);
|
|
1337
|
+
}
|
|
1338
|
+
}
|
|
1339
|
+
catch (err) {
|
|
1340
|
+
if (!(err instanceof semantic_model_1.ProfileFileError))
|
|
1341
|
+
throw err;
|
|
1342
|
+
doc.errors.push(err.message);
|
|
1343
|
+
return;
|
|
1344
|
+
}
|
|
1345
|
+
const profiles = [];
|
|
1346
|
+
for (const pf of files) {
|
|
1347
|
+
try {
|
|
1348
|
+
profiles.push((0, resolve_profiles_1.loadProfileFile)(pf.content, pf.name));
|
|
1349
|
+
}
|
|
1350
|
+
catch (err) {
|
|
1351
|
+
if (!(err instanceof loader_1.SemanticModelError))
|
|
1352
|
+
throw err;
|
|
1353
|
+
doc.errors.push(`[${pf.fileName}] ${err.message}`);
|
|
1354
|
+
}
|
|
1355
|
+
}
|
|
1356
|
+
if (profiles.length > 0) {
|
|
1357
|
+
model.profiles = profiles;
|
|
1358
|
+
}
|
|
1359
|
+
}
|
|
1360
|
+
/**
|
|
1361
|
+
* Verifies that `model` can be converted to the vanilla (`0.2.0.dev0`) version:
|
|
1362
|
+
* it must declare no `actions`, no `constraints`, and no `abstract` entities,
|
|
1363
|
+
* and every entity must have a `source`, every field an `expression`, and
|
|
1364
|
+
* every relationship its join columns (`from_columns` and `to_columns`).
|
|
1365
|
+
* Appends every gap to `doc.errors`.
|
|
1366
|
+
*/
|
|
1367
|
+
function validateOssieConversion(model, doc) {
|
|
1368
|
+
const prefix = `Cannot convert '${model.name}' to version '${loader_1.OSSIE_VERSION}'`;
|
|
1369
|
+
// 1. Reject actions and constraints, which have no representation in
|
|
1370
|
+
// `0.2.0.dev0`.
|
|
1371
|
+
for (const a of model.actions ?? []) {
|
|
1372
|
+
doc.errors.push(`${prefix}: action '${a.name}' is not supported in version ` +
|
|
1373
|
+
`'${loader_1.OSSIE_VERSION}'.`);
|
|
1374
|
+
}
|
|
1375
|
+
for (const c of model.constraints ?? []) {
|
|
1376
|
+
doc.errors.push(`${prefix}: constraint '${c.name}' is not supported in version ` +
|
|
1377
|
+
`'${loader_1.OSSIE_VERSION}'.`);
|
|
1378
|
+
}
|
|
1379
|
+
// 2. Reject abstract entities and require every concrete entity and field to
|
|
1380
|
+
// be bound.
|
|
1381
|
+
for (const e of model.entities) {
|
|
1382
|
+
if (e.abstract) {
|
|
1383
|
+
doc.errors.push(`${prefix}: entity '${e.name}' is 'abstract', which version ` +
|
|
1384
|
+
`'${loader_1.OSSIE_VERSION}' does not support.`);
|
|
1385
|
+
continue;
|
|
1386
|
+
}
|
|
1387
|
+
if (!e.authoredSource && !e.dataSource) {
|
|
1388
|
+
doc.errors.push(`${prefix}: entity '${e.name}' has no 'source'.`);
|
|
1389
|
+
}
|
|
1390
|
+
for (const f of e.fields) {
|
|
1391
|
+
if (!(0, ir_1.isFieldBound)(f)) {
|
|
1392
|
+
doc.errors.push(`${prefix}: field '${e.name}.${f.name}' has no 'expression'.`);
|
|
1393
|
+
}
|
|
1394
|
+
}
|
|
1395
|
+
}
|
|
1396
|
+
// 3. Require every relationship to declare its join columns.
|
|
1397
|
+
for (const r of model.relationships) {
|
|
1398
|
+
if (!r.source.columns.length || !r.destination.columns.length) {
|
|
1399
|
+
doc.errors.push(`${prefix}: relationship '${r.name}' has no join columns ` +
|
|
1400
|
+
`('from_columns' and 'to_columns').`);
|
|
1401
|
+
}
|
|
1402
|
+
}
|
|
1403
|
+
}
|
|
1404
|
+
/**
|
|
1405
|
+
* Loads `doc.root` into the `SemanticModel` IR for cross-version conversion,
|
|
1406
|
+
* recording any `SemanticModelError` on `doc.errors`.
|
|
1407
|
+
*/
|
|
1408
|
+
function loadSemanticModel(doc) {
|
|
1409
|
+
try {
|
|
1410
|
+
const loaded = (0, loader_1.fromDocument)(doc.root, { bindingOptional: true, warnUnacceptedSource: true });
|
|
1411
|
+
return loaded.models[0];
|
|
1412
|
+
}
|
|
1413
|
+
catch (err) {
|
|
1414
|
+
if (!(err instanceof loader_1.SemanticModelError))
|
|
1415
|
+
throw err;
|
|
1416
|
+
doc.errors.push(err.message);
|
|
1417
|
+
return undefined;
|
|
1418
|
+
}
|
|
1419
|
+
}
|
|
1420
|
+
/**
|
|
1421
|
+
* Phase 4 (Convert): Checks with the `SemanticModel` IR that `doc` converts to
|
|
1422
|
+
* its target version (`0.2.0.dev0` <-> `0.2.0.dev0/google`), attaching profile
|
|
1423
|
+
* files and checking conversion gaps when targeting `0.2.0.dev0`, then edits
|
|
1424
|
+
* `doc` into that version with `generateAdaptedFiles`.
|
|
1425
|
+
*/
|
|
1426
|
+
async function generateConvertedFiles(doc, opts) {
|
|
1427
|
+
if (doc.errors.length > 0)
|
|
1428
|
+
return [];
|
|
1429
|
+
const model = loadSemanticModel(doc);
|
|
1430
|
+
if (!model)
|
|
1431
|
+
return [];
|
|
1432
|
+
if (doc.targetVersion === loader_1.OSSIE_VERSION) {
|
|
1433
|
+
attachProfileFiles(model, opts.profileFiles, doc, opts.fileName ?? `${model.name}.yaml`);
|
|
1434
|
+
validateOssieConversion(model, doc);
|
|
1435
|
+
if (doc.errors.length > 0)
|
|
1436
|
+
return [];
|
|
1437
|
+
}
|
|
1438
|
+
return generateAdaptedFiles(doc, !!opts.transpile, opts.profileFiles);
|
|
1439
|
+
}
|
|
1440
|
+
/**
|
|
1441
|
+
* Phase 4 (Adapt): Adapts `doc` in place to its target version
|
|
1442
|
+
* (`moveConstructs`, then `adaptToGoogleVersion` or `adaptToOssieVersion`),
|
|
1443
|
+
* optionally transpiles expressions via `transpileDocument`, and returns the
|
|
1444
|
+
* output files: the model file, then the profile files. An import that keeps
|
|
1445
|
+
* the flavor writes `profileFiles` under the model's name. Every value these
|
|
1446
|
+
* steps do not change keeps its text.
|
|
1447
|
+
*/
|
|
1448
|
+
async function generateAdaptedFiles(doc, transpile, profileFiles = []) {
|
|
1449
|
+
// 1. Adapt `doc` in place to the target output version (`0.2.0.dev0/google`
|
|
1450
|
+
// or `0.2.0.dev0`). Each of `profileFiles` goes under the model's name. A
|
|
1451
|
+
// nameless `<file>.profile.yaml` stays nameless, so push reports it.
|
|
1452
|
+
let profiles = doc.inputVersion === doc.targetVersion ?
|
|
1453
|
+
profileFiles.map(pf => ({
|
|
1454
|
+
fileName: /\.profile\.ya?ml$/i.test(pf.fileName) &&
|
|
1455
|
+
parseProfileName(pf.fileName) === '' ?
|
|
1456
|
+
`${doc.model.name}.profile.yaml` :
|
|
1457
|
+
`${doc.model.name}.profile.${parseProfileName(pf.fileName)}.yaml`,
|
|
1458
|
+
content: pf.content,
|
|
1459
|
+
})) :
|
|
1460
|
+
moveConstructs(doc, profileFiles);
|
|
1461
|
+
if (doc.inputVersion === loader_1.OSSIE_VERSION &&
|
|
1462
|
+
doc.targetVersion === loader_1.GOOGLE_VERSION) {
|
|
1463
|
+
adaptToGoogleVersion(doc);
|
|
1464
|
+
}
|
|
1465
|
+
else if (doc.targetVersion === loader_1.OSSIE_VERSION) {
|
|
1466
|
+
adaptToOssieVersion(doc);
|
|
1467
|
+
}
|
|
1468
|
+
// 2. When `transpile` is enabled and no prior errors occurred, translate
|
|
1469
|
+
// vendor or double-quoted `ANSI_SQL` expressions to `BIGQUERY`, in the model
|
|
1470
|
+
// and in each BigQuery profile, in its GOOGLE blocks or among `profiles`.
|
|
1471
|
+
if (transpile && doc.errors.length === 0) {
|
|
1472
|
+
profiles = await transpileDocument(doc, profiles, doc.inputVersion === doc.targetVersion ? profileFiles : profiles);
|
|
1473
|
+
}
|
|
1474
|
+
// 3. Write `doc.root` into the input's text when modified or preserve the
|
|
1475
|
+
// text verbatim when unmodified.
|
|
1476
|
+
const content = doc.modified ? writeDocument(doc, doc.root) : doc.text;
|
|
1477
|
+
return [{ fileName: `${doc.model.name}.yaml`, content }, ...profiles];
|
|
1478
|
+
}
|
|
1479
|
+
/**
|
|
1480
|
+
* Returns the input's text changed to hold `root`. What survives from the
|
|
1481
|
+
* input keeps its text, and long SQL stays on one line.
|
|
1482
|
+
*/
|
|
1483
|
+
function writeDocument(doc, root) {
|
|
1484
|
+
// The vanilla flavor lists a model's entities under `datasets:` and the
|
|
1485
|
+
// Google flavor under `entities:`. Renaming the key keeps the comments of
|
|
1486
|
+
// the list and of every entity in it.
|
|
1487
|
+
const node = doc.document.getIn(['semantic_model', 0]);
|
|
1488
|
+
const model = asRecords(root.semantic_model)[0] ?? {};
|
|
1489
|
+
for (const pair of yaml.isMap(node) ? node.items : []) {
|
|
1490
|
+
if (!yaml.isScalar(pair.key))
|
|
1491
|
+
continue;
|
|
1492
|
+
const from = String(pair.key.value);
|
|
1493
|
+
const to = from === 'datasets' ? 'entities' :
|
|
1494
|
+
from === 'entities' ? 'datasets' :
|
|
1495
|
+
undefined;
|
|
1496
|
+
if (to && to in model && !(from in model))
|
|
1497
|
+
pair.key.value = to;
|
|
1498
|
+
}
|
|
1499
|
+
return (0, yaml_update_1.updateText)(doc.text, doc.document, root, doc.links);
|
|
1500
|
+
}
|
|
1501
|
+
/**
|
|
1502
|
+
* Phase 5 (Post-validation): Validates the output model file with `loadModels`
|
|
1503
|
+
* (`warnUnacceptedSource: !doc.strict`), checks deployment targets, and runs
|
|
1504
|
+
* push's local checks (`localChecks`) on the model file and each profile it
|
|
1505
|
+
* declares, with the source dialects in `load`. With KC_V2_ASPECTS on, it also
|
|
1506
|
+
* checks that Knowledge Catalog can store each aspect. It also gives push's
|
|
1507
|
+
* warnings from the Knowledge Catalog entry builder, the pruning and each
|
|
1508
|
+
* graph, and leaves out push's notes.
|
|
1509
|
+
*/
|
|
1510
|
+
async function validateOutputDocument(files, doc, opts, load) {
|
|
1511
|
+
const [{ fileName, content }, ...profileFiles] = files;
|
|
1512
|
+
try {
|
|
1513
|
+
// 1. Load the serialized output YAML with `loadModels`
|
|
1514
|
+
// (`warnUnacceptedSource: !doc.strict`) to run full schema and semantic
|
|
1515
|
+
// validation while turning unaccepted table sources into warnings in
|
|
1516
|
+
// non-strict mode, deduplicating warnings already recorded. Step 2 gives
|
|
1517
|
+
// push's error for a field its table cannot run, so the load does not
|
|
1518
|
+
// warn about it too.
|
|
1519
|
+
const loaded = (0, loader_1.loadModels)(content, {
|
|
1520
|
+
bindingOptional: true,
|
|
1521
|
+
warnUnacceptedSource: !doc.strict,
|
|
1522
|
+
fieldTextChecked: true,
|
|
1523
|
+
sourceDialects: load.sourceDialects,
|
|
1524
|
+
});
|
|
1525
|
+
for (const w of loaded.warnings) {
|
|
1526
|
+
// The repair step has reported each source the loader warns about.
|
|
1527
|
+
if (LOADER_SOURCE_WARNING.test(w))
|
|
1528
|
+
continue;
|
|
1529
|
+
if (!doc.warnings.includes(w) && !doc.errors.includes(w)) {
|
|
1530
|
+
doc.warnings.push(w);
|
|
1531
|
+
}
|
|
1532
|
+
}
|
|
1533
|
+
// 2. Run push's local checks on the model file and each profile it
|
|
1534
|
+
// declares. What push rejects is an error in strict mode and a warning
|
|
1535
|
+
// otherwise.
|
|
1536
|
+
const input = doc.strict ? opts.fileName : undefined;
|
|
1537
|
+
const model = {
|
|
1538
|
+
name: (input ?? fileName).replace(/\.ya?ml$/, ''),
|
|
1539
|
+
text: content,
|
|
1540
|
+
};
|
|
1541
|
+
// Where the input holds profile `name`, which the output writes in
|
|
1542
|
+
// `written`: its profile file, or the model file's GOOGLE block. Strict
|
|
1543
|
+
// mode writes nothing on an error, so its messages name the input.
|
|
1544
|
+
const inputFiles = doc.inputVersion === loader_1.GOOGLE_VERSION ? opts.profileFiles ?? [] : [];
|
|
1545
|
+
const where = (name, written) => !input ? written :
|
|
1546
|
+
inputFiles.find(pf => parseProfileName(pf.fileName) === name)
|
|
1547
|
+
?.fileName ??
|
|
1548
|
+
model.name;
|
|
1549
|
+
// Push stops at a profile file that breaks a file rule, then at a
|
|
1550
|
+
// deployment target in no accepted form. validateDeploymentTargets reports
|
|
1551
|
+
// such a target as an error in either mode.
|
|
1552
|
+
let read;
|
|
1553
|
+
try {
|
|
1554
|
+
const seen = new Map();
|
|
1555
|
+
read = profileFiles.map(f => {
|
|
1556
|
+
const name = parseProfileName(f.fileName);
|
|
1557
|
+
const label = where(name, f.fileName);
|
|
1558
|
+
(0, semantic_model_1.checkProfileFileName)(input ?? fileName, label, name, f.content, seen);
|
|
1559
|
+
return { name, text: f.content, fileName: label };
|
|
1560
|
+
});
|
|
1561
|
+
}
|
|
1562
|
+
catch (err) {
|
|
1563
|
+
if (!(err instanceof semantic_model_1.ProfileFileError))
|
|
1564
|
+
throw err;
|
|
1565
|
+
doc.warnOrError(err.message);
|
|
1566
|
+
}
|
|
1567
|
+
if (!doc.errors.length) {
|
|
1568
|
+
for (const m of loaded.models)
|
|
1569
|
+
validateDeploymentTargets(m, doc);
|
|
1570
|
+
}
|
|
1571
|
+
const profiles = read;
|
|
1572
|
+
if (!profiles)
|
|
1573
|
+
return;
|
|
1574
|
+
const declaredBy = () => (0, resolve_profiles_1.declaredProfiles)(model, profiles).map(p => 'spec' in p ? { ...p, label: where(p.name, model.name) } : p);
|
|
1575
|
+
// Push stops at a source its loader rejects. Import still reports each
|
|
1576
|
+
// profile's problems, its sources among them.
|
|
1577
|
+
if (loaded.warnings.some(w => LOADER_SOURCE_WARNING.test(w))) {
|
|
1578
|
+
for (const { error } of (0, resolve_profiles_1.namedProfileErrors)([model], new Map([[model.name, loaded.models[0]]]), () => profiles, declaredBy, doc.warnings, load)) {
|
|
1579
|
+
doc.warnOrError(error);
|
|
1580
|
+
}
|
|
1581
|
+
return;
|
|
1582
|
+
}
|
|
1583
|
+
if (doc.errors.length)
|
|
1584
|
+
return;
|
|
1585
|
+
const base = { document: model.name, model: loaded.models[0] };
|
|
1586
|
+
const own = loaded.warnings.map(w => `[${model.name}] ${w}`);
|
|
1587
|
+
// Push prints each warning once, after the load's. Each is a warning in
|
|
1588
|
+
// either mode.
|
|
1589
|
+
const warn = (w) => {
|
|
1590
|
+
if (!own.includes(w) && !doc.warnings.includes(w))
|
|
1591
|
+
doc.warnings.push(w);
|
|
1592
|
+
};
|
|
1593
|
+
const withFiles = (0, validate_1.withProfileFiles)(base, () => profiles);
|
|
1594
|
+
const checked = 'errors' in withFiles ? withFiles : await (0, validate_1.localChecks)({
|
|
1595
|
+
docs: [model],
|
|
1596
|
+
loaded: { models: [base], warnings: own },
|
|
1597
|
+
models: [withFiles],
|
|
1598
|
+
profileDocs: () => profiles,
|
|
1599
|
+
declaredBy,
|
|
1600
|
+
load,
|
|
1601
|
+
}, warn);
|
|
1602
|
+
// Push prints each error once.
|
|
1603
|
+
const errors = 'errors' in checked ? checked.errors : [];
|
|
1604
|
+
for (const error of new Set(errors.map(e => e.error))) {
|
|
1605
|
+
doc.warnOrError(error);
|
|
1606
|
+
}
|
|
1607
|
+
if ('errors' in checked || 'errors' in withFiles)
|
|
1608
|
+
return;
|
|
1609
|
+
// Push next builds the Knowledge Catalog entries and prints the builder's
|
|
1610
|
+
// warnings. With KC_V2_ASPECTS on, it also checks that Knowledge Catalog
|
|
1611
|
+
// can store each aspect. The entry group does not change an aspect's size.
|
|
1612
|
+
const v2Aspects = (0, validate_1.kcV2Aspects)();
|
|
1613
|
+
const catalog = (0, deploy_knowledge_catalog_1.checkCatalogAspects)([withFiles], {
|
|
1614
|
+
project: 'p', location: 'l', entryGroup: 'g',
|
|
1615
|
+
systemTypeProject: process.env.KC_TYPE_PROJECT, v2Aspects,
|
|
1616
|
+
});
|
|
1617
|
+
for (const w of catalog.warnings)
|
|
1618
|
+
warn(w);
|
|
1619
|
+
if (v2Aspects && catalog.error) {
|
|
1620
|
+
doc.warnOrError(catalog.error);
|
|
1621
|
+
return;
|
|
1622
|
+
}
|
|
1623
|
+
// Push then warns about what pruning leaves out of each deployment, and
|
|
1624
|
+
// about what each graph leaves out. Import leaves out push's notes.
|
|
1625
|
+
for (const line of checked.plans.flatMap(validate_1.pruningLines)) {
|
|
1626
|
+
if (line.startsWith('Warning: '))
|
|
1627
|
+
warn(line.slice('Warning: '.length));
|
|
1628
|
+
}
|
|
1629
|
+
for (const { generated } of checked.plans) {
|
|
1630
|
+
for (const w of generated?.warnings ?? [])
|
|
1631
|
+
warn(w);
|
|
1632
|
+
}
|
|
1633
|
+
}
|
|
1634
|
+
catch (err) {
|
|
1635
|
+
if (!(err instanceof loader_1.SemanticModelError))
|
|
1636
|
+
throw err;
|
|
1637
|
+
doc.errors.push(err.message);
|
|
1638
|
+
}
|
|
1639
|
+
}
|
|
1640
|
+
/**
|
|
1641
|
+
* Repairs, converts, optionally transpiles, and validates a semantic model YAML
|
|
1642
|
+
* document, returning the normalized output file(s) along with any warnings or
|
|
1643
|
+
* errors.
|
|
1644
|
+
*/
|
|
1645
|
+
async function importSemanticModelYaml(rawYaml, opts = {}) {
|
|
1646
|
+
// 1. Initialize the SQL parser and parse/pre-validate the input document,
|
|
1647
|
+
// and report any profile file beside a vanilla input as push does.
|
|
1648
|
+
await (0, sql_parser_1.loadSqlEngine)();
|
|
1649
|
+
const doc = parseImportDocument(rawYaml, opts);
|
|
1650
|
+
if (typeof doc === 'string') {
|
|
1651
|
+
return { ok: false, files: [], warnings: [], errors: [doc] };
|
|
1652
|
+
}
|
|
1653
|
+
validateInputDocument(doc);
|
|
1654
|
+
checkVanillaProfileFiles(doc, opts.profileFiles ?? [], opts.fileName ?? `${doc.model.name}.yaml`);
|
|
1655
|
+
// 2. Read the `--source-prefix` entry group, if needed, and run third-party
|
|
1656
|
+
// repairs (which record errors in `strict` mode) and read the source
|
|
1657
|
+
// dialects.
|
|
1658
|
+
const listError = await listEntryTables(doc);
|
|
1659
|
+
if (listError) {
|
|
1660
|
+
return {
|
|
1661
|
+
ok: false,
|
|
1662
|
+
files: [],
|
|
1663
|
+
warnings: doc.warnings,
|
|
1664
|
+
errors: [...doc.errors, listError],
|
|
1665
|
+
};
|
|
1666
|
+
}
|
|
1667
|
+
const load = await repairDocument(doc, opts);
|
|
1668
|
+
// 3. Generate the converted or adapted output file(s). A conversion to
|
|
1669
|
+
// `0.2.0.dev0` writes the sibling profile files inside the model file, and
|
|
1670
|
+
// a Google-flavor import of a Google-flavor model writes them beside it.
|
|
1671
|
+
const shouldConvert = doc.inputVersion !== doc.targetVersion && doc.strict;
|
|
1672
|
+
const googleToGoogle = doc.inputVersion === loader_1.GOOGLE_VERSION &&
|
|
1673
|
+
doc.targetVersion === loader_1.GOOGLE_VERSION;
|
|
1674
|
+
const files = shouldConvert ?
|
|
1675
|
+
await generateConvertedFiles(doc, opts) :
|
|
1676
|
+
await generateAdaptedFiles(doc, !!opts.transpile, googleToGoogle ? opts.profileFiles ?? [] : []);
|
|
1677
|
+
// 4. Check that each profile file the import writes parses, validate the
|
|
1678
|
+
// output files, and return the result.
|
|
1679
|
+
for (const [i, f] of files.slice(1).entries()) {
|
|
1680
|
+
const [problem] = yaml.parseDocument(f.content, yaml_options_1.YAML_OPTIONS).errors;
|
|
1681
|
+
if (!problem)
|
|
1682
|
+
continue;
|
|
1683
|
+
// A file import passes through unchanged is the input's own file.
|
|
1684
|
+
const input = opts.profileFiles?.[i];
|
|
1685
|
+
doc.errors.push(input?.content === f.content ?
|
|
1686
|
+
(0, semantic_model_1.profileParseError)(input.fileName, problem.message) :
|
|
1687
|
+
`the import writes '${f.fileName}', which does not parse: ${problem.message}`);
|
|
1688
|
+
}
|
|
1689
|
+
if (files.length > 0 && doc.errors.length === 0) {
|
|
1690
|
+
await validateOutputDocument(files, doc, opts, load);
|
|
1691
|
+
}
|
|
1692
|
+
return {
|
|
1693
|
+
ok: doc.errors.length === 0,
|
|
1694
|
+
files: doc.errors.length === 0 ? files : [],
|
|
1695
|
+
warnings: doc.warnings,
|
|
1696
|
+
errors: doc.errors,
|
|
1697
|
+
};
|
|
1698
|
+
}
|