@google-cloud/kcmd 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +203 -0
- package/README.md +264 -2
- package/build/ts/kcmd/cli.js +20252 -0
- package/build/ts/kcmd/gcp/alloydb.d.ts +12 -0
- package/build/ts/kcmd/gcp/alloydb.js +70 -0
- package/build/ts/kcmd/gcp/api.d.ts +29 -0
- package/build/ts/kcmd/gcp/api.js +162 -0
- package/build/ts/kcmd/gcp/biglake.d.ts +18 -0
- package/build/ts/kcmd/gcp/biglake.js +59 -0
- package/build/ts/kcmd/gcp/bigquery.d.ts +57 -0
- package/build/ts/kcmd/gcp/bigquery.js +134 -0
- package/build/ts/kcmd/gcp/context.d.ts +10 -0
- package/build/ts/kcmd/gcp/context.js +74 -0
- package/build/ts/kcmd/gcp/crm.d.ts +13 -0
- package/build/ts/kcmd/gcp/crm.js +91 -0
- package/build/ts/kcmd/gcp/dataplex.d.ts +147 -0
- package/build/ts/kcmd/gcp/dataplex.js +406 -0
- package/build/ts/kcmd/gcp/index.d.ts +3 -0
- package/build/ts/kcmd/gcp/index.js +21 -0
- package/build/ts/kcmd/gcp/spanner.d.ts +73 -0
- package/build/ts/kcmd/gcp/spanner.js +126 -0
- package/build/ts/kcmd/index.d.ts +7 -0
- package/build/ts/kcmd/index.js +46 -0
- package/build/ts/kcmd/layout.d.ts +16 -0
- package/build/ts/kcmd/layout.js +31 -0
- package/build/ts/kcmd/layouts/documents.d.ts +24 -0
- package/build/ts/kcmd/layouts/documents.js +264 -0
- package/build/ts/kcmd/layouts/okf.d.ts +31 -0
- package/build/ts/kcmd/layouts/okf.js +386 -0
- package/build/ts/kcmd/layouts/semantic-model.d.ts +74 -0
- package/build/ts/kcmd/layouts/semantic-model.js +621 -0
- package/build/ts/kcmd/layouts/standard.d.ts +13 -0
- package/build/ts/kcmd/layouts/standard.js +108 -0
- package/build/ts/kcmd/manifest.d.ts +34 -0
- package/build/ts/kcmd/manifest.js +262 -0
- package/build/ts/kcmd/metadata.d.ts +25 -0
- package/build/ts/kcmd/metadata.js +8 -0
- package/build/ts/kcmd/okf/catalog_residue.d.ts +14 -0
- package/build/ts/kcmd/okf/catalog_residue.js +124 -0
- package/build/ts/kcmd/okf/concept.d.ts +14 -0
- package/build/ts/kcmd/okf/concept.js +110 -0
- package/build/ts/kcmd/okf/concept_to_entry.d.ts +10 -0
- package/build/ts/kcmd/okf/concept_to_entry.js +281 -0
- package/build/ts/kcmd/okf/init_okf_workspace.d.ts +15 -0
- package/build/ts/kcmd/okf/init_okf_workspace.js +113 -0
- package/build/ts/kcmd/okf/links.d.ts +23 -0
- package/build/ts/kcmd/okf/links.js +468 -0
- package/build/ts/kcmd/okf/okf_metadata.d.ts +12 -0
- package/build/ts/kcmd/okf/okf_metadata.js +99 -0
- package/build/ts/kcmd/semantic/bigquery.d.ts +22 -0
- package/build/ts/kcmd/semantic/bigquery.js +469 -0
- package/build/ts/kcmd/semantic/binding.d.ts +3 -0
- package/build/ts/kcmd/semantic/binding.js +92 -0
- package/build/ts/kcmd/semantic/catalog_entry.d.ts +24 -0
- package/build/ts/kcmd/semantic/catalog_entry.js +35 -0
- package/build/ts/kcmd/semantic/converters/owl/convert.d.ts +26 -0
- package/build/ts/kcmd/semantic/converters/owl/convert.js +41 -0
- package/build/ts/kcmd/semantic/converters/owl/model.d.ts +98 -0
- package/build/ts/kcmd/semantic/converters/owl/model.js +30 -0
- package/build/ts/kcmd/semantic/converters/owl/parse.d.ts +11 -0
- package/build/ts/kcmd/semantic/converters/owl/parse.js +634 -0
- package/build/ts/kcmd/semantic/converters/owl/to_ir.d.ts +19 -0
- package/build/ts/kcmd/semantic/converters/owl/to_ir.js +390 -0
- package/build/ts/kcmd/semantic/deploy_bigquery.d.ts +20 -0
- package/build/ts/kcmd/semantic/deploy_bigquery.js +226 -0
- package/build/ts/kcmd/semantic/deploy_knowledge_catalog.d.ts +104 -0
- package/build/ts/kcmd/semantic/deploy_knowledge_catalog.js +1367 -0
- package/build/ts/kcmd/semantic/deploy_spanner.d.ts +15 -0
- package/build/ts/kcmd/semantic/deploy_spanner.js +150 -0
- package/build/ts/kcmd/semantic/deployment_target.d.ts +93 -0
- package/build/ts/kcmd/semantic/deployment_target.js +379 -0
- package/build/ts/kcmd/semantic/graph_ddl.d.ts +44 -0
- package/build/ts/kcmd/semantic/graph_ddl.js +656 -0
- package/build/ts/kcmd/semantic/import_model.d.ts +72 -0
- package/build/ts/kcmd/semantic/import_model.js +1698 -0
- package/build/ts/kcmd/semantic/ir.d.ts +589 -0
- package/build/ts/kcmd/semantic/ir.js +247 -0
- package/build/ts/kcmd/semantic/kc_actions.d.ts +45 -0
- package/build/ts/kcmd/semantic/kc_actions.js +531 -0
- package/build/ts/kcmd/semantic/kc_constraints.d.ts +44 -0
- package/build/ts/kcmd/semantic/kc_constraints.js +279 -0
- package/build/ts/kcmd/semantic/kc_converter.d.ts +37 -0
- package/build/ts/kcmd/semantic/kc_converter.js +839 -0
- package/build/ts/kcmd/semantic/kc_custom_types.d.ts +63 -0
- package/build/ts/kcmd/semantic/kc_custom_types.js +751 -0
- package/build/ts/kcmd/semantic/kc_entries.d.ts +155 -0
- package/build/ts/kcmd/semantic/kc_entries.js +284 -0
- package/build/ts/kcmd/semantic/kc_ids.d.ts +102 -0
- package/build/ts/kcmd/semantic/kc_ids.js +188 -0
- package/build/ts/kcmd/semantic/knowledge_catalog.d.ts +29 -0
- package/build/ts/kcmd/semantic/knowledge_catalog.js +960 -0
- package/build/ts/kcmd/semantic/loader.d.ts +258 -0
- package/build/ts/kcmd/semantic/loader.js +3576 -0
- package/build/ts/kcmd/semantic/names.d.ts +10 -0
- package/build/ts/kcmd/semantic/names.js +85 -0
- package/build/ts/kcmd/semantic/osi_converter.d.ts +60 -0
- package/build/ts/kcmd/semantic/osi_converter.js +910 -0
- package/build/ts/kcmd/semantic/parameters.d.ts +19 -0
- package/build/ts/kcmd/semantic/parameters.js +171 -0
- package/build/ts/kcmd/semantic/pull_kc.d.ts +10 -0
- package/build/ts/kcmd/semantic/pull_kc.js +272 -0
- package/build/ts/kcmd/semantic/resolve_inheritance.d.ts +59 -0
- package/build/ts/kcmd/semantic/resolve_inheritance.js +346 -0
- package/build/ts/kcmd/semantic/resolve_profiles.d.ts +291 -0
- package/build/ts/kcmd/semantic/resolve_profiles.js +2003 -0
- package/build/ts/kcmd/semantic/runtime/agent_tools.d.ts +93 -0
- package/build/ts/kcmd/semantic/runtime/agent_tools.js +243 -0
- package/build/ts/kcmd/semantic/runtime/dialect.d.ts +13 -0
- package/build/ts/kcmd/semantic/runtime/dialect.js +39 -0
- package/build/ts/kcmd/semantic/runtime/runtime.d.ts +88 -0
- package/build/ts/kcmd/semantic/runtime/runtime.js +301 -0
- package/build/ts/kcmd/semantic/runtime/store.d.ts +112 -0
- package/build/ts/kcmd/semantic/runtime/store.js +344 -0
- package/build/ts/kcmd/semantic/skills.d.ts +110 -0
- package/build/ts/kcmd/semantic/skills.js +697 -0
- package/build/ts/kcmd/semantic/source_dialect.d.ts +17 -0
- package/build/ts/kcmd/semantic/source_dialect.js +311 -0
- package/build/ts/kcmd/semantic/source_entries.d.ts +12 -0
- package/build/ts/kcmd/semantic/source_entries.js +341 -0
- package/build/ts/kcmd/semantic/spanner.d.ts +18 -0
- package/build/ts/kcmd/semantic/spanner.js +60 -0
- package/build/ts/kcmd/semantic/sql_expr_utils.d.ts +23 -0
- package/build/ts/kcmd/semantic/sql_expr_utils.js +394 -0
- package/build/ts/kcmd/semantic/sql_identifiers.d.ts +6 -0
- package/build/ts/kcmd/semantic/sql_identifiers.js +267 -0
- package/build/ts/kcmd/semantic/sql_parser.d.ts +47 -0
- package/build/ts/kcmd/semantic/sql_parser.js +400 -0
- package/build/ts/kcmd/semantic/transpile.d.ts +76 -0
- package/build/ts/kcmd/semantic/transpile.js +581 -0
- package/build/ts/kcmd/semantic/validate.d.ts +70 -0
- package/build/ts/kcmd/semantic/validate.js +2266 -0
- package/build/ts/kcmd/semantic/yaml_options.d.ts +6 -0
- package/build/ts/kcmd/semantic/yaml_options.js +58 -0
- package/build/ts/kcmd/semantic/yaml_update.d.ts +26 -0
- package/build/ts/kcmd/semantic/yaml_update.js +782 -0
- package/build/ts/kcmd/snapshot.d.ts +37 -0
- package/build/ts/kcmd/snapshot.js +386 -0
- package/build/ts/kcmd/source.d.ts +31 -0
- package/build/ts/kcmd/source.js +119 -0
- package/build/ts/kcmd/sources/bq-dataset.d.ts +17 -0
- package/build/ts/kcmd/sources/bq-dataset.js +131 -0
- package/build/ts/kcmd/sources/entrygroup.d.ts +21 -0
- package/build/ts/kcmd/sources/entrygroup.js +83 -0
- package/build/ts/kcmd/sources/kb.d.ts +21 -0
- package/build/ts/kcmd/sources/kb.js +82 -0
- package/build/ts/kcmd/sources/semantic-model.d.ts +18 -0
- package/build/ts/kcmd/sources/semantic-model.js +48 -0
- package/build/ts/kcmd/sync.d.ts +35 -0
- package/build/ts/kcmd/sync.js +734 -0
- package/package.json +62 -4
|
@@ -0,0 +1,2266 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// Push-time validation gate for a semantic model.
|
|
3
|
+
//
|
|
4
|
+
// Runs once over the shared, already-parsed models (see loadSemanticModels)
|
|
5
|
+
// before any destination leg, so a real `kcmd push` AND a `--validate-only` dry
|
|
6
|
+
// run enforce the same requirements. Returns one message per violation (an
|
|
7
|
+
// empty array means valid); push prints them and aborts, and `kcmd import`
|
|
8
|
+
// reports them. Kept separate from the loader -- which validates a document
|
|
9
|
+
// against the schema -- because these are deployment requirements, not schema
|
|
10
|
+
// rules, and they read the GOOGLE deployment-target extension the BigQuery leg
|
|
11
|
+
// owns.
|
|
12
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
13
|
+
exports.validatePushRequirements = validatePushRequirements;
|
|
14
|
+
exports.validateDeployments = validateDeployments;
|
|
15
|
+
exports.validateInlineDialects = validateInlineDialects;
|
|
16
|
+
exports.validateInheritedColumns = validateInheritedColumns;
|
|
17
|
+
exports.inlineBindingWarnings = inlineBindingWarnings;
|
|
18
|
+
exports.validateActions = validateActions;
|
|
19
|
+
exports.validateInheritance = validateInheritance;
|
|
20
|
+
exports.validateExpressionReferences = validateExpressionReferences;
|
|
21
|
+
exports.deploymentTableErrors = deploymentTableErrors;
|
|
22
|
+
exports.spannerMetricsErrors = spannerMetricsErrors;
|
|
23
|
+
exports.kcV2Aspects = kcV2Aspects;
|
|
24
|
+
exports.withProfileFiles = withProfileFiles;
|
|
25
|
+
exports.localChecks = localChecks;
|
|
26
|
+
exports.pruningLines = pruningLines;
|
|
27
|
+
exports.validateBigQueryDataSources = validateBigQueryDataSources;
|
|
28
|
+
exports.validateSpannerActionStatements = validateSpannerActionStatements;
|
|
29
|
+
exports.validateBigQueryActionStatements = validateBigQueryActionStatements;
|
|
30
|
+
const context_1 = require("../gcp/context");
|
|
31
|
+
const bigquery_1 = require("./bigquery");
|
|
32
|
+
const deploy_bigquery_1 = require("./deploy_bigquery");
|
|
33
|
+
const deploy_spanner_1 = require("./deploy_spanner");
|
|
34
|
+
const deployment_target_1 = require("./deployment_target");
|
|
35
|
+
const ir_1 = require("./ir");
|
|
36
|
+
const kc_ids_1 = require("./kc_ids");
|
|
37
|
+
const loader_1 = require("./loader");
|
|
38
|
+
const names_1 = require("./names");
|
|
39
|
+
const parameters_1 = require("./parameters");
|
|
40
|
+
const resolve_inheritance_1 = require("./resolve_inheritance");
|
|
41
|
+
const resolve_profiles_1 = require("./resolve_profiles");
|
|
42
|
+
const sql_expr_utils_1 = require("./sql_expr_utils");
|
|
43
|
+
const sql_identifiers_1 = require("./sql_identifiers");
|
|
44
|
+
// Checks every model against the push requirements and returns the collected
|
|
45
|
+
// error messages (empty when all models pass), each tagged with the model's
|
|
46
|
+
// source document so the author can find it.
|
|
47
|
+
//
|
|
48
|
+
// `fieldsPruned` says the caller has already dropped every field the selected
|
|
49
|
+
// binding profile leaves unbound, so `entity.fields` is a subset of what the
|
|
50
|
+
// author declared. Checks that read a field list have to stand down for such a
|
|
51
|
+
// model (see validateConstraints).
|
|
52
|
+
//
|
|
53
|
+
// `warnings` collects non-fatal messages (such as an expression the SQL parser
|
|
54
|
+
// could not read, whose field references were therefore not checked) for the
|
|
55
|
+
// caller to print.
|
|
56
|
+
//
|
|
57
|
+
// `v2Aspects` says push writes the second-generation Knowledge Catalog types
|
|
58
|
+
// (the CLI sets it unless KC_V2_ASPECTS=0).
|
|
59
|
+
function validatePushRequirements(models, opts = {}) {
|
|
60
|
+
const errors = [];
|
|
61
|
+
for (const { document, model } of models) {
|
|
62
|
+
errors.push(...nameAndDescriptionErrors(model, document, !!opts.v2Aspects));
|
|
63
|
+
let deployInfo;
|
|
64
|
+
try {
|
|
65
|
+
// One pass over the model's GOOGLE extension(s): both checks below read
|
|
66
|
+
// the same parse rather than re-parsing the JSON per reader.
|
|
67
|
+
deployInfo = (0, deploy_bigquery_1.googleDeploymentTargets)(model);
|
|
68
|
+
}
|
|
69
|
+
catch (err) {
|
|
70
|
+
// Malformed GOOGLE extension JSON: surface it as a validation error here
|
|
71
|
+
// rather than letting it throw out of a later leg as an uncaught stack.
|
|
72
|
+
errors.push(`${err.message || err} (${document})`);
|
|
73
|
+
continue;
|
|
74
|
+
}
|
|
75
|
+
// A model that targets a graph (BigQuery OR Spanner) must have every
|
|
76
|
+
// relationship's join columns bound. The loader accepts a column-less
|
|
77
|
+
// relationship so a purely logical model (an OWL import) loads and pushes
|
|
78
|
+
// to Knowledge Catalog, but a graph deploy would emit an invalid
|
|
79
|
+
// `DESTINATION KEY () REFERENCES Dest ()` for such an edge -- so reject it
|
|
80
|
+
// here rather than generate broken DDL. A model with no graph target
|
|
81
|
+
// skips it.
|
|
82
|
+
if (deployInfo.bigQuery.length + deployInfo.spanner.length > 0) {
|
|
83
|
+
for (const rel of model.relationships ?? []) {
|
|
84
|
+
// An M:N edge binds through its junction table (association), so its
|
|
85
|
+
// direct source/destination columns are empty by design -- the graph
|
|
86
|
+
// generators render it from `rel.association`. Only a plain FK edge
|
|
87
|
+
// needs direct join columns.
|
|
88
|
+
if (rel.association)
|
|
89
|
+
continue;
|
|
90
|
+
if (!rel.source.columns.length && !rel.destination.columns.length) {
|
|
91
|
+
errors.push(`relationship '${rel.name}' in model '${model.name}' (${document}) targets a graph but has no join columns; add its ` +
|
|
92
|
+
`from_columns and to_columns to the relationship in the model ` +
|
|
93
|
+
`before a BigQuery or Spanner Graph deploy.`);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
errors.push(...validateInheritance([{ document, model }]));
|
|
98
|
+
errors.push(...validateColumnsAndRelationships(model, document));
|
|
99
|
+
errors.push(...relationshipKeyErrors(model, document));
|
|
100
|
+
const flaggedMetrics = new Set();
|
|
101
|
+
errors.push(...validateExpressionReferences([{ document, model }], !!opts.fieldsPruned, opts.warnings, flaggedMetrics));
|
|
102
|
+
if (deployInfo.bigQuery.length > 0) {
|
|
103
|
+
errors.push(...measurePlacementErrors(model, document, flaggedMetrics));
|
|
104
|
+
}
|
|
105
|
+
// An action reaches Knowledge Catalog only, so its checks are
|
|
106
|
+
// target-independent: each parameter's type must resolve to something in
|
|
107
|
+
// the ontology, the executor must carry the coordinates a runtime needs to
|
|
108
|
+
// dispatch it, and each guard must name a constraint the model declares.
|
|
109
|
+
// (The "exactly one executor kind" rule is already guaranteed by the loader
|
|
110
|
+
// schema, so it cannot reach here.)
|
|
111
|
+
errors.push(...validateActions(model, document, !!opts.fieldsPruned, true));
|
|
112
|
+
// Constraints are logical invariants, target-independent like actions.
|
|
113
|
+
errors.push(...validateConstraints(model, document, !!opts.fieldsPruned));
|
|
114
|
+
}
|
|
115
|
+
return errors;
|
|
116
|
+
}
|
|
117
|
+
// Each name in `model` that breaks its kind's pattern or length limit, each
|
|
118
|
+
// pair of relationships whose names give one Knowledge Catalog link ID, and
|
|
119
|
+
// each description too long for the Knowledge Catalog entry that stores it.
|
|
120
|
+
// The link-ID check applies with `v2Aspects` off only.
|
|
121
|
+
// The loader checks names too; this check is for a model built in memory and
|
|
122
|
+
// a profile that reaches `model.profiles` some other way. Link IDs and
|
|
123
|
+
// description limits are checked at push only, since only push writes them.
|
|
124
|
+
function nameAndDescriptionErrors(model, document, v2Aspects) {
|
|
125
|
+
const at = `model '${model.name}' (${document})`;
|
|
126
|
+
const errors = [];
|
|
127
|
+
const check = (kind, name) => {
|
|
128
|
+
const problem = (0, names_1.nameProblem)(kind, name);
|
|
129
|
+
if (problem)
|
|
130
|
+
errors.push(`${at}: ${problem}`);
|
|
131
|
+
};
|
|
132
|
+
const tooLong = (what, description) => {
|
|
133
|
+
if (description !== undefined &&
|
|
134
|
+
description.length > names_1.MAX_ENTRY_DESCRIPTION) {
|
|
135
|
+
errors.push(`${at}: the description of ${what} is ${description.length} characters long; a Knowledge Catalog ` +
|
|
136
|
+
`entry's description holds at most ${names_1.MAX_ENTRY_DESCRIPTION}.`);
|
|
137
|
+
}
|
|
138
|
+
};
|
|
139
|
+
check('model', model.name);
|
|
140
|
+
tooLong(`model '${model.name}'`, model.description);
|
|
141
|
+
for (const e of model.entities ?? []) {
|
|
142
|
+
check('entity', e.name);
|
|
143
|
+
tooLong(`entity '${e.name}'`, e.description);
|
|
144
|
+
for (const f of e.fields ?? [])
|
|
145
|
+
check('field', f.name);
|
|
146
|
+
}
|
|
147
|
+
for (const m of model.metrics ?? []) {
|
|
148
|
+
check('metric', m.name);
|
|
149
|
+
tooLong(`metric '${m.name}'`, m.description);
|
|
150
|
+
}
|
|
151
|
+
// The relationship that first took each link ID.
|
|
152
|
+
const byLinkId = new Map();
|
|
153
|
+
for (const r of model.relationships ?? []) {
|
|
154
|
+
check('relationship', r.name);
|
|
155
|
+
tooLong(`relationship '${r.name}'`, r.description);
|
|
156
|
+
if (v2Aspects)
|
|
157
|
+
continue;
|
|
158
|
+
const linkId = (0, kc_ids_1.relationshipLinkId)(model.name, r.name);
|
|
159
|
+
const first = byLinkId.get(linkId);
|
|
160
|
+
if (first === undefined) {
|
|
161
|
+
byLinkId.set(linkId, r.name);
|
|
162
|
+
}
|
|
163
|
+
else {
|
|
164
|
+
errors.push(`Relationships '${first}' and '${r.name}' in model '${model.name}' (${document}) collide on Knowledge Catalog link ` +
|
|
165
|
+
`ID '${linkId}'; rename one of them.`);
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
for (const d of model.deployments ?? [])
|
|
169
|
+
check('deployment', d.name);
|
|
170
|
+
for (const p of model.profiles ?? [])
|
|
171
|
+
check('profile', p.name);
|
|
172
|
+
const reserved = (0, resolve_profiles_1.reservedProfileNameError)(document, model.profiles ?? []);
|
|
173
|
+
if (reserved)
|
|
174
|
+
errors.push(reserved);
|
|
175
|
+
return errors;
|
|
176
|
+
}
|
|
177
|
+
// Checks each deployment of `model` against the rules of its engine, under the
|
|
178
|
+
// profile it names in `model.profiles` or the model file's own bindings, and
|
|
179
|
+
// returns one message per problem, naming every deployment it applies to.
|
|
180
|
+
function validateDeployments(model, document, load = {}) {
|
|
181
|
+
// The deployments each problem applies to, keyed by the text that follows
|
|
182
|
+
// their names in the message.
|
|
183
|
+
const problems = new Map();
|
|
184
|
+
const add = (deployment, problem) => problems.set(problem, [...(problems.get(problem) ?? []), deployment]);
|
|
185
|
+
const unbound = unboundInline(model);
|
|
186
|
+
const unboundFields = unboundGraphFields(model);
|
|
187
|
+
// The metrics the reference rules report, which the BigQuery rule leaves to
|
|
188
|
+
// them.
|
|
189
|
+
let referenced;
|
|
190
|
+
for (const d of model.deployments ?? []) {
|
|
191
|
+
// A profile file that does not load is not in `model.profiles`, and is
|
|
192
|
+
// left out; push reports the file.
|
|
193
|
+
const named = model.profiles?.find(p => p.name === d.profile);
|
|
194
|
+
if ((d.profile ?? resolve_profiles_1.DEFAULT_PROFILE) !== resolve_profiles_1.DEFAULT_PROFILE && !named)
|
|
195
|
+
continue;
|
|
196
|
+
const resolved = (0, deployment_target_1.resolveModelDeployments)(model, [d.name]);
|
|
197
|
+
const engine = 'error' in resolved ? undefined : resolved.deployments[0].engine;
|
|
198
|
+
const alloyDb = engine === 'alloydb';
|
|
199
|
+
// The model the deployment's bindings give, less what they leave
|
|
200
|
+
// unavailable. The completeness check reports a named profile that does
|
|
201
|
+
// not apply, and the fields its metrics read.
|
|
202
|
+
const bound = named ? profiled(model, named, load) : (0, resolve_profiles_1.pruneUnavailable)(model, resolve_profiles_1.DEFAULT_PROFILE, alloyDb ? 'ALLOYDB' : (0, resolve_profiles_1.sourcesDialect)(model));
|
|
203
|
+
if (!bound)
|
|
204
|
+
continue;
|
|
205
|
+
const available = bound.model.metrics ?? [];
|
|
206
|
+
if (engine === 'bigquery' || engine === 'spanner') {
|
|
207
|
+
for (const p of graphNameProblems(bound.model, engine))
|
|
208
|
+
add(d.name, p);
|
|
209
|
+
}
|
|
210
|
+
if (named) {
|
|
211
|
+
if (engine === 'spanner' && available.length) {
|
|
212
|
+
add(d.name, `targets Spanner, which has no measures, and profile '${named.name}' leaves metric(s) ${(0, loader_1.listed)(available.map(m => `'${m.name}'`))} available; name ` +
|
|
213
|
+
`them in its 'metrics_exclude', or set it to "*".`);
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
else {
|
|
217
|
+
if (unbound) {
|
|
218
|
+
add(d.name, `uses the model file's own bindings, which leave ${unbound}; a deployment needs every entity and relationship ` +
|
|
219
|
+
`bound.`);
|
|
220
|
+
}
|
|
221
|
+
if (unboundFields && (engine === 'bigquery' || engine === 'spanner')) {
|
|
222
|
+
add(d.name, `uses the model file's own bindings, ${unboundFields}`);
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
// A Spanner deployment deploys no metric.
|
|
226
|
+
if (engine === 'spanner')
|
|
227
|
+
continue;
|
|
228
|
+
for (const { name, field } of named ? [] : bound.report.droppedMetrics) {
|
|
229
|
+
if (field === undefined)
|
|
230
|
+
continue;
|
|
231
|
+
add(d.name, `uses the model file's own bindings, under which metric '${name}' reads field '${field}', which ${bound.report.unboundFields.includes(field) ?
|
|
232
|
+
'has no expression' :
|
|
233
|
+
'reads a field with no expression, directly or through ' +
|
|
234
|
+
'other fields'}; a deployment needs every field its ` +
|
|
235
|
+
`metrics read bound.`);
|
|
236
|
+
}
|
|
237
|
+
if (engine !== 'bigquery')
|
|
238
|
+
continue;
|
|
239
|
+
if (!referenced) {
|
|
240
|
+
referenced = new Set();
|
|
241
|
+
validateExpressionReferences([{ document, model }], false, [], referenced);
|
|
242
|
+
}
|
|
243
|
+
for (const m of available) {
|
|
244
|
+
const problem = !referenced.has(m.name) && measureProblem(bound.model, m);
|
|
245
|
+
if (problem) {
|
|
246
|
+
add(d.name, `targets BigQuery, where metric '${m.name}' cannot be a measure: ${problem}.`);
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
for (const p of bigQueryOptionProblems(bound.model))
|
|
250
|
+
add(d.name, p);
|
|
251
|
+
}
|
|
252
|
+
// The verb that starts each problem agrees with the number of deployments.
|
|
253
|
+
return [...problems].map(([problem, names]) => {
|
|
254
|
+
const many = names.length > 1;
|
|
255
|
+
return `model '${model.name}' (${document}) deployment${many ? 's' : ''} ${(0, loader_1.listed)(names.map(n => `'${n}'`))} ${many ? problem.replace(/^(use|target)s /, '$1 ') : problem}`;
|
|
256
|
+
});
|
|
257
|
+
}
|
|
258
|
+
// `model` with `profile` applied and what it leaves unavailable pruned, or
|
|
259
|
+
// undefined when the profile does not apply.
|
|
260
|
+
function profiled(model, profile, load) {
|
|
261
|
+
try {
|
|
262
|
+
return (0, resolve_profiles_1.pruneUnavailable)((0, resolve_profiles_1.applyProfile)(model, profile, { load }), profile.name);
|
|
263
|
+
}
|
|
264
|
+
catch (err) {
|
|
265
|
+
if (!(err instanceof loader_1.SemanticModelError))
|
|
266
|
+
throw err;
|
|
267
|
+
return undefined;
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
// Why `metric` cannot be a BigQuery measure, or undefined when it can. Leaves
|
|
271
|
+
// out a metric that measurePlacementErrors or the inheritance rules report:
|
|
272
|
+
// one that reads several entities or none, or one on an abstract or extended
|
|
273
|
+
// entity.
|
|
274
|
+
function measureProblem(model, metric) {
|
|
275
|
+
const text = (0, ir_1.expressionForDialect)(metric, 'BIGQUERY');
|
|
276
|
+
if (text === undefined)
|
|
277
|
+
return undefined;
|
|
278
|
+
const columns = (0, sql_expr_utils_1.columnsForSource)(text, 'BIGQUERY');
|
|
279
|
+
if (!columns)
|
|
280
|
+
return undefined;
|
|
281
|
+
const entities = model.entities ?? [];
|
|
282
|
+
const placed = (0, sql_expr_utils_1.measureEntities)(text, entities.map(e => e.name), metric.entity);
|
|
283
|
+
if (placed.length !== 1)
|
|
284
|
+
return undefined;
|
|
285
|
+
const entity = entities.find(e => e.name === placed[0]);
|
|
286
|
+
if (!entity || entity.abstract ||
|
|
287
|
+
entities.some(e => e.extends?.includes(entity.name))) {
|
|
288
|
+
return undefined;
|
|
289
|
+
}
|
|
290
|
+
if (!(0, sql_expr_utils_1.isSingleAggregate)(text, 'BIGQUERY')) {
|
|
291
|
+
return 'it is not SUM, AVG, COUNT, MIN or MAX over one operand';
|
|
292
|
+
}
|
|
293
|
+
if (!(0, resolve_profiles_1.statesKey)(entity.keys, entity.uniqueKeys)) {
|
|
294
|
+
return `it belongs to entity '${entity.name}', which has no primary key ` +
|
|
295
|
+
`or unique key, so the graph has no node table for it`;
|
|
296
|
+
}
|
|
297
|
+
return undefined;
|
|
298
|
+
}
|
|
299
|
+
// The BigQuery graph's rules for placing each metric of `model` as a measure,
|
|
300
|
+
// one message per metric that breaks them, skipping each one in
|
|
301
|
+
// `alreadyFlagged`, which validateExpressionReferences has reported. The
|
|
302
|
+
// generator (`bigquery.ts`) places a measure on the one entity whose
|
|
303
|
+
// qualifier its BigQuery SQL writes before a column, or on its anchor when the
|
|
304
|
+
// SQL writes none (see measureEntities), and skips a metric that has no
|
|
305
|
+
// BigQuery or ANSI_SQL text. validateExpressionReferences reads every dialect
|
|
306
|
+
// entry together and lets an authored anchor settle SQL that reads two
|
|
307
|
+
// entities, so it does not catch each metric the generator would skip.
|
|
308
|
+
function measurePlacementErrors(model, document, alreadyFlagged) {
|
|
309
|
+
const errors = [];
|
|
310
|
+
const entityNames = (model.entities ?? []).map(e => e.name);
|
|
311
|
+
for (const metric of model.metrics ?? []) {
|
|
312
|
+
if (alreadyFlagged.has(metric.name))
|
|
313
|
+
continue;
|
|
314
|
+
const prefix = `metric '${metric.name}' in model '${model.name}' (${document})`;
|
|
315
|
+
// The text BigQuery runs: its `BIGQUERY` entry, else its `ANSI_SQL` one,
|
|
316
|
+
// else `expression` for a metric held without a dialect list.
|
|
317
|
+
const bqText = (0, ir_1.expressionForDialect)(metric, 'BIGQUERY');
|
|
318
|
+
if (bqText === undefined && metric.importedExpression !== undefined) {
|
|
319
|
+
errors.push(`${prefix} targets a BigQuery graph with only a '${metric.importedDialect ?? 'vendor'}' expression; supply a ` +
|
|
320
|
+
`'BIGQUERY' or 'ANSI_SQL' expression${(0, resolve_profiles_1.importTranspiles)(metric, 'BIGQUERY', model) ?
|
|
321
|
+
`, or add one with '${(0, resolve_profiles_1.importCommand)('--transpile', model.version)}'` :
|
|
322
|
+
''}.`);
|
|
323
|
+
continue;
|
|
324
|
+
}
|
|
325
|
+
const placed = (0, sql_expr_utils_1.measureEntities)(bqText ?? '', entityNames, metric.entity);
|
|
326
|
+
if (placed.length > 1) {
|
|
327
|
+
errors.push(`${prefix} targets a BigQuery graph and its BigQuery SQL reads ` +
|
|
328
|
+
`several entities (${placed.join(', ')}); a BigQuery measure ` +
|
|
329
|
+
`belongs to one entity's node table, so scope its expression to ` +
|
|
330
|
+
`one entity.`);
|
|
331
|
+
}
|
|
332
|
+
else if (placed.length === 0) {
|
|
333
|
+
const anchor = model.version === loader_1.OSSIE_VERSION ?
|
|
334
|
+
`'entity' in a GOOGLE custom_extensions block on the metric` :
|
|
335
|
+
`top-level 'entity'`;
|
|
336
|
+
errors.push(`${prefix} targets a BigQuery graph but does not resolve to a ` +
|
|
337
|
+
`single entity; the BigQuery generator places a measure on the ` +
|
|
338
|
+
`entity its BigQuery SQL reads, or on ${anchor}.`);
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
return errors;
|
|
342
|
+
}
|
|
343
|
+
// Checks that the model file's Spanner tables are all in one database, so they
|
|
344
|
+
// share one dialect, and that each bound field has a text for the dialect of
|
|
345
|
+
// its entity's source. Returns one message per problem.
|
|
346
|
+
function validateInlineDialects(model, document) {
|
|
347
|
+
const errors = [];
|
|
348
|
+
const databases = new Map();
|
|
349
|
+
for (const e of model.entities ?? []) {
|
|
350
|
+
const db = (0, loader_1.databaseOf)(e.authoredSource ?? '');
|
|
351
|
+
if (db?.startsWith('spanner/') && !databases.has(db)) {
|
|
352
|
+
databases.set(db, e.name);
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
if (databases.size > 1) {
|
|
356
|
+
const named = [...databases.values()].map(n => `'${n}'`).join(', ');
|
|
357
|
+
errors.push(`model '${model.name}' (${document}) binds entities in more than one ` +
|
|
358
|
+
`Spanner database (${named}); bind the model file's entities to ` +
|
|
359
|
+
`tables in one Spanner database.`);
|
|
360
|
+
}
|
|
361
|
+
for (const { entity: e, field: f, table: { name: dialect } } of (0, resolve_profiles_1.fieldsOnTables)(model)) {
|
|
362
|
+
if (!(0, resolve_profiles_1.noTextFor)(f, [dialect]))
|
|
363
|
+
continue;
|
|
364
|
+
errors.push(`field '${e.name}.${f.name}' in model '${model.name}' (${document}) has no expression for ${dialect} or ANSI_SQL, and ` +
|
|
365
|
+
`the model file binds entity '${e.name}' to ${/^[AEIOU]/.test(dialect) ? 'an' : 'a'} ${dialect} source${(0, resolve_profiles_1.importHint)(f, dialect, model)}.`);
|
|
366
|
+
}
|
|
367
|
+
return errors;
|
|
368
|
+
}
|
|
369
|
+
// One message for each field a concrete entity with a source inherits without
|
|
370
|
+
// restating it, when the inherited expression reads a column. Empty when the
|
|
371
|
+
// hierarchy does not resolve, which push reports elsewhere.
|
|
372
|
+
function validateInheritedColumns(model, document) {
|
|
373
|
+
if ((0, resolve_profiles_1.inheritanceErrorOf)(model))
|
|
374
|
+
return [];
|
|
375
|
+
const entities = model.entities ?? [];
|
|
376
|
+
const dialect = (0, resolve_profiles_1.sourcesDialect)(model);
|
|
377
|
+
const errors = [];
|
|
378
|
+
for (const e of entities.filter(e => !e.abstract && e.dataSource)) {
|
|
379
|
+
for (const f of (0, resolve_profiles_1.inheritedNamesOf)(model, e.name)) {
|
|
380
|
+
const read = !e.fields.some(x => x.name === f) &&
|
|
381
|
+
(0, resolve_profiles_1.columnBinding)(model, e.name, f, dialect);
|
|
382
|
+
if (!read)
|
|
383
|
+
continue;
|
|
384
|
+
errors.push(`entity '${e.name}' in model '${model.name}' (${document}): field '${f}' is inherited from '${read.owner}', whose expression reads ` +
|
|
385
|
+
`column '${read.column}' of that entity's table; restate it with ` +
|
|
386
|
+
`an expression for this entity's table.`);
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
return errors;
|
|
390
|
+
}
|
|
391
|
+
// The warning for a model file whose own bindings are incomplete and that no
|
|
392
|
+
// deployment uses. With `namedProfiles`, it says 'default' is unbound, and
|
|
393
|
+
// otherwise that the model cannot be queried at all.
|
|
394
|
+
function inlineBindingWarnings(model, document, namedProfiles) {
|
|
395
|
+
const unbound = unboundInline(model);
|
|
396
|
+
const used = (model.deployments ?? []).some(d => (d.profile ?? resolve_profiles_1.DEFAULT_PROFILE) === resolve_profiles_1.DEFAULT_PROFILE);
|
|
397
|
+
if (!unbound || used)
|
|
398
|
+
return [];
|
|
399
|
+
const where = `model '${model.name}' (${document})`;
|
|
400
|
+
return [
|
|
401
|
+
namedProfiles ?
|
|
402
|
+
`${where} has an unbound 'default' profile: the model file leaves ${unbound}. It cannot be queried through 'default'.` :
|
|
403
|
+
`${where} is non-executable: the model file leaves ${unbound}, and the model declares no named profile. It cannot be ` +
|
|
404
|
+
`queried until it is bound.`,
|
|
405
|
+
];
|
|
406
|
+
}
|
|
407
|
+
// What the model file's own bindings leave unbound, such as "entity 'a'
|
|
408
|
+
// without a source", or '' when nothing is.
|
|
409
|
+
function unboundInline(model) {
|
|
410
|
+
const sourceless = (model.entities ?? [])
|
|
411
|
+
.filter(e => !e.abstract && !e.dataSource)
|
|
412
|
+
.map(e => e.name);
|
|
413
|
+
const unjoined = (model.relationships ?? [])
|
|
414
|
+
.filter(r => !r.association &&
|
|
415
|
+
!(r.source.columns.length && r.destination.columns.length))
|
|
416
|
+
.map(r => r.name);
|
|
417
|
+
const part = (names, one, many, without) => names.length ? `${names.length === 1 ? one : many} ${(0, loader_1.listed)(names.map(n => `'${n}'`))} without ${without}` :
|
|
418
|
+
'';
|
|
419
|
+
return [
|
|
420
|
+
part(sourceless, 'entity', 'entities', 'a source'),
|
|
421
|
+
part(unjoined, 'relationship', 'relationships', 'join columns'),
|
|
422
|
+
].filter(p => p).join(' and ');
|
|
423
|
+
}
|
|
424
|
+
// The fields the model file's own bindings leave without an expression on
|
|
425
|
+
// each concrete leaf entity, counting the fields it inherits, as the end of a
|
|
426
|
+
// deployment's error, or undefined when there are none.
|
|
427
|
+
function unboundGraphFields(model) {
|
|
428
|
+
const extended = new Set((model.entities ?? []).flatMap(e => e.extends ?? []));
|
|
429
|
+
const fields = (0, resolve_profiles_1.resolvedEntities)(model)
|
|
430
|
+
.filter(e => !e.abstract && !extended.has(e.name))
|
|
431
|
+
.flatMap(e => (e.fields ?? [])
|
|
432
|
+
.filter(f => !(0, ir_1.isFieldBound)(f))
|
|
433
|
+
.map(f => `'${e.name}.${f.name}'`));
|
|
434
|
+
if (!fields.length)
|
|
435
|
+
return undefined;
|
|
436
|
+
const many = fields.length > 1;
|
|
437
|
+
return `which leave ${many ? 'fields' : 'field'} ${(0, loader_1.listed)(fields)} without an expression; a graph deployment ` +
|
|
438
|
+
`needs every field of a concrete leaf entity bound. Give ${many ? 'each field' : 'the field'} an expression, or use a ` +
|
|
439
|
+
`named profile that lists ${many ? 'them' : 'it'} in 'fields_exclude'.`;
|
|
440
|
+
}
|
|
441
|
+
// The error for a metric whose name matches, ignoring case, a declared or
|
|
442
|
+
// inherited field of `owner`, or undefined.
|
|
443
|
+
function metricFieldCollisionError(metric, owner, fieldsByEntity, model, document) {
|
|
444
|
+
const fields = owner === undefined ? undefined : fieldsByEntity.get(owner);
|
|
445
|
+
if (!fields)
|
|
446
|
+
return undefined;
|
|
447
|
+
const lower = metric.name.toLowerCase();
|
|
448
|
+
const match = [...fields].find(f => f.toLowerCase() === lower);
|
|
449
|
+
if (match === undefined)
|
|
450
|
+
return undefined;
|
|
451
|
+
return `Metric '${metric.name}' in model '${model.name}' (${document}) collides with field '${match}' on entity '${owner}'; metrics and fields on the same entity share a ` +
|
|
452
|
+
`namespace.`;
|
|
453
|
+
}
|
|
454
|
+
// The names a graph on `engine` cannot hold in `model`, compared ignoring
|
|
455
|
+
// case, each as the end of a sentence about a deployment. A metric named like
|
|
456
|
+
// a field of its own entity is left to metricFieldCollisionError.
|
|
457
|
+
function graphNameProblems(model, engine) {
|
|
458
|
+
const at = `targets ${engine === 'bigquery' ? 'BigQuery' : 'Spanner'}, where`;
|
|
459
|
+
const same = (a, b) => a.toLowerCase() === b.toLowerCase();
|
|
460
|
+
const problems = [];
|
|
461
|
+
for (const r of model.relationships ?? []) {
|
|
462
|
+
const e = model.entities?.find(e => same(e.name, r.name));
|
|
463
|
+
if (!e)
|
|
464
|
+
continue;
|
|
465
|
+
problems.push(`${at} entity '${e.name}' and relationship '${r.name}' cannot both ` +
|
|
466
|
+
`name a table: node and edge tables share one set of names, ` +
|
|
467
|
+
`compared ignoring case; rename one.`);
|
|
468
|
+
}
|
|
469
|
+
if (engine === 'spanner')
|
|
470
|
+
return problems;
|
|
471
|
+
const tables = (0, resolve_profiles_1.resolvedEntities)(model).filter(e => (0, resolve_profiles_1.statesKey)(e.keys, e.uniqueKeys));
|
|
472
|
+
for (const m of model.metrics ?? []) {
|
|
473
|
+
const owner = m.authoredEntity ?? m.entity;
|
|
474
|
+
for (const e of tables.filter(e => e.name !== owner)) {
|
|
475
|
+
const f = e.fields.find(f => same(f.name, m.name));
|
|
476
|
+
if (!f)
|
|
477
|
+
continue;
|
|
478
|
+
problems.push(`${at} metric '${m.name}' cannot share its name with field '${e.name}.${f.name}': a measure's name must differ from every ` +
|
|
479
|
+
`property name in the graph, compared ignoring case; rename one.`);
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
return problems;
|
|
483
|
+
}
|
|
484
|
+
// What a BigQuery graph rejects in the OPTIONS kcmd writes for `model`, each
|
|
485
|
+
// as the end of a sentence about a deployment. It reads only the labels and
|
|
486
|
+
// properties the generator writes OPTIONS for.
|
|
487
|
+
function bigQueryOptionProblems(model) {
|
|
488
|
+
const problems = [];
|
|
489
|
+
const check = (kind, name, description, synonyms = [], entity) => {
|
|
490
|
+
const what = `${kind} '${entity ? `${entity}.` : ''}${name}'`;
|
|
491
|
+
if (description && description.length > names_1.MAX_BIGQUERY_DESCRIPTION) {
|
|
492
|
+
problems.push(`the description kcmd writes for ${what} is ${description.length} characters long; a label or ` +
|
|
493
|
+
`property's description holds at most ${names_1.MAX_BIGQUERY_DESCRIPTION}.`);
|
|
494
|
+
}
|
|
495
|
+
if (synonyms.length > names_1.MAX_BIGQUERY_SYNONYM_COUNT) {
|
|
496
|
+
problems.push(`${what} has ${synonyms.length} synonyms; a label or property ` +
|
|
497
|
+
`holds at most ${names_1.MAX_BIGQUERY_SYNONYM_COUNT}.`);
|
|
498
|
+
}
|
|
499
|
+
const firstByLower = new Map();
|
|
500
|
+
for (const s of synonyms) {
|
|
501
|
+
const lower = s.toLowerCase();
|
|
502
|
+
if (!s)
|
|
503
|
+
problems.push(`${what} has an empty synonym; remove it.`);
|
|
504
|
+
if (s.length > names_1.MAX_BIGQUERY_SYNONYM_LENGTH) {
|
|
505
|
+
problems.push(`a synonym of ${what} is ${s.length} characters long; a ` +
|
|
506
|
+
`synonym holds at most ${names_1.MAX_BIGQUERY_SYNONYM_LENGTH}.`);
|
|
507
|
+
}
|
|
508
|
+
if (lower === name.toLowerCase()) {
|
|
509
|
+
problems.push(`${what} has the synonym '${s}', which BigQuery treats as ` +
|
|
510
|
+
`the ${kind}'s own name; remove it.`);
|
|
511
|
+
}
|
|
512
|
+
const first = firstByLower.get(lower);
|
|
513
|
+
if (first === undefined) {
|
|
514
|
+
firstByLower.set(lower, s);
|
|
515
|
+
}
|
|
516
|
+
else {
|
|
517
|
+
problems.push(`${what} has the synonyms '${first}' and '${s}', which ` +
|
|
518
|
+
`BigQuery treats as one; remove one.`);
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
};
|
|
522
|
+
const element = (kind, x) => check(kind, x.name, (0, bigquery_1.elementDescription)(x.description, x.aiContext), x.aiContext?.synonyms);
|
|
523
|
+
const tables = (0, resolve_profiles_1.resolvedEntities)(model).filter(e => (0, resolve_profiles_1.statesKey)(e.keys, e.uniqueKeys));
|
|
524
|
+
const extended = new Set(tables.flatMap(e => e.extends ?? []));
|
|
525
|
+
for (const e of tables) {
|
|
526
|
+
if (!extended.has(e.name))
|
|
527
|
+
element('entity', e);
|
|
528
|
+
for (const f of e.fields) {
|
|
529
|
+
if ((0, ir_1.expressionForDialect)(f, 'BIGQUERY') === undefined)
|
|
530
|
+
continue;
|
|
531
|
+
check('field', f.name, (0, bigquery_1.fieldDescription)(f), f.aiContext?.synonyms, e.name);
|
|
532
|
+
}
|
|
533
|
+
}
|
|
534
|
+
for (const r of model.relationships ?? [])
|
|
535
|
+
element('relationship', r);
|
|
536
|
+
for (const m of model.metrics ?? [])
|
|
537
|
+
element('metric', m);
|
|
538
|
+
return problems.map(p => `targets BigQuery, where ${p}`);
|
|
539
|
+
}
|
|
540
|
+
// Static, target-independent checks for a model's actions. Returns one message
|
|
541
|
+
// per violation. What can be statically wrong once the model has parsed:
|
|
542
|
+
// - a parameter projects from a concept the model does not declare, or a
|
|
543
|
+
// field that concept does not have (the loader kept it and only warned) --
|
|
544
|
+
// each reported as itself, because "fix the concept" and "fix the field"
|
|
545
|
+
// are two different repairs;
|
|
546
|
+
// - a parameter resolved to no scalar type at all, or to something that is
|
|
547
|
+
// not one -- the commonest case being an entity name written as the
|
|
548
|
+
// parameter's `type`, which is the old object-reference spelling;
|
|
549
|
+
// - an executor is missing a coordinate a runtime needs to dispatch it (an
|
|
550
|
+
// empty server/tool, endpoint/method, or service/method) -- the schema
|
|
551
|
+
// accepts empty strings, so this is caught here rather than at parse;
|
|
552
|
+
// - a guard names a constraint the model does not declare;
|
|
553
|
+
// - an `affects` entry names a concept the model does not declare, names
|
|
554
|
+
// fields on a 'delete' (which takes the whole instance), or names a field
|
|
555
|
+
// the concept does not have.
|
|
556
|
+
//
|
|
557
|
+
// Every `affects` check is a hard error for the reason the guard check is: an
|
|
558
|
+
// entry that names nothing real leaves a reader believing the blast radius is
|
|
559
|
+
// described when it is not, and a consumer routing on it would route on a
|
|
560
|
+
// concept that does not exist.
|
|
561
|
+
function validateActions(model, document, fieldsPruned, checkDescriptions) {
|
|
562
|
+
const errors = [];
|
|
563
|
+
const actions = model.actions ?? [];
|
|
564
|
+
if (!actions.length)
|
|
565
|
+
return errors;
|
|
566
|
+
const constraintNames = new Set((model.constraints ?? []).map(c => c.name));
|
|
567
|
+
// Built when an `affects` entry or a projected parameter will actually read
|
|
568
|
+
// it, and through `ifResolvable` because declaredConcepts resolves
|
|
569
|
+
// inheritance.
|
|
570
|
+
//
|
|
571
|
+
// `fieldsPruned` stands the whole lookup down, which is what a profile push
|
|
572
|
+
// needs: pruning removes unbound and unavailable fields, so the field a
|
|
573
|
+
// parameter projects from may be gone from the model in hand even though the
|
|
574
|
+
// author's document names one that exists. A parameter needs only
|
|
575
|
+
// the logical definition and the loader already copied it down, so nothing
|
|
576
|
+
// about the push depends on resolving the reference a second time here.
|
|
577
|
+
// A `type` naming something that is not a scalar counts too, and it is the
|
|
578
|
+
// case that most needs the lookup: the message worth printing is "that is an
|
|
579
|
+
// entity, project the field you meant", and only the concept table can tell
|
|
580
|
+
// an entity from a typo. Leaving it out meant the one model making exactly
|
|
581
|
+
// this mistake -- the migration from an entity-reference parameter, with no
|
|
582
|
+
// `affects` and nothing yet projected -- got the generic "not a scalar
|
|
583
|
+
// datatype" instead of the instruction.
|
|
584
|
+
const needsConcepts = actions.some(a => a.affects?.length ||
|
|
585
|
+
a.parameters.some(p => p.concept ||
|
|
586
|
+
(p.type &&
|
|
587
|
+
!ir_1.DATA_TYPES.includes(p.type))));
|
|
588
|
+
const concepts = !fieldsPruned && needsConcepts ?
|
|
589
|
+
ifResolvable(() => declaredConcepts(model)) :
|
|
590
|
+
undefined;
|
|
591
|
+
for (const action of actions) {
|
|
592
|
+
const where = `action '${action.name}' in model '${model.name}' (${document})`;
|
|
593
|
+
// Two parameters are confusable when a caller cannot tell from their types
|
|
594
|
+
// which is which, so this keys on the datatype the caller actually sees in
|
|
595
|
+
// the tool schema. Projecting the SAME field is the stronger case of that
|
|
596
|
+
// -- two Account.accountId parameters are not merely both integers, they
|
|
597
|
+
// denote the same kind of thing -- but it needs no key of its own: a
|
|
598
|
+
// projected parameter takes the field's type, so a same-field pair is
|
|
599
|
+
// already a same-type pair. The projection only picks the wording below.
|
|
600
|
+
const byIdentity = new Map();
|
|
601
|
+
for (const param of action.parameters) {
|
|
602
|
+
errors.push(...parameterTypeErrors(param, where, concepts));
|
|
603
|
+
if (param.required === true && param.default !== undefined) {
|
|
604
|
+
errors.push(`${where} has parameter '${param.name}' with both ` +
|
|
605
|
+
`'required: true' and a 'default'; a parameter with a default is ` +
|
|
606
|
+
`optional.`);
|
|
607
|
+
}
|
|
608
|
+
// Every parameter is a scalar now, so every default is checkable --
|
|
609
|
+
// including one on a parameter whose definition came from a field.
|
|
610
|
+
if (param.default !== undefined && param.default !== null &&
|
|
611
|
+
param.type !== undefined) {
|
|
612
|
+
const bound = (0, parameters_1.bindScalar)(param, param.default);
|
|
613
|
+
if ('error' in bound) {
|
|
614
|
+
errors.push(`${where} has parameter '${param.name}' whose default '${param.default}' is invalid: ${bound.error}`);
|
|
615
|
+
}
|
|
616
|
+
}
|
|
617
|
+
const identity = param.type ?? '';
|
|
618
|
+
if (identity === '')
|
|
619
|
+
continue;
|
|
620
|
+
const list = byIdentity.get(identity) ?? [];
|
|
621
|
+
list.push(param);
|
|
622
|
+
byIdentity.set(identity, list);
|
|
623
|
+
}
|
|
624
|
+
if (checkDescriptions) {
|
|
625
|
+
for (const [identity, params] of byIdentity) {
|
|
626
|
+
if (params.length <= 1)
|
|
627
|
+
continue;
|
|
628
|
+
// A description has to be the parameter's OWN to separate it from its
|
|
629
|
+
// twin. Two parameters projected from one field come out of the loader
|
|
630
|
+
// carrying that field's wording verbatim, so both are described and
|
|
631
|
+
// neither is distinguished -- an identical description is worth no
|
|
632
|
+
// more here than a missing one.
|
|
633
|
+
const shares = new Map();
|
|
634
|
+
for (const p of params) {
|
|
635
|
+
const said = p.description?.trim() ?? '';
|
|
636
|
+
shares.set(said, (shares.get(said) ?? 0) + 1);
|
|
637
|
+
}
|
|
638
|
+
const indistinct = params.filter(p => shares.get(p.description?.trim() ?? '') > 1 ||
|
|
639
|
+
!p.description?.trim());
|
|
640
|
+
if (indistinct.length > 0) {
|
|
641
|
+
const names = indistinct.map(p => `'${p.name}'`);
|
|
642
|
+
// Name the projection only when every one of them came from the one
|
|
643
|
+
// field AND that field is what more than one parameter came from.
|
|
644
|
+
// A mixed bucket has nothing in common but the datatype. So does a
|
|
645
|
+
// bucket holding a single projection beside a described parameter of
|
|
646
|
+
// the same type: the projection is indistinct, but blaming
|
|
647
|
+
// "multiple parameters projected from Account.accountId" names a
|
|
648
|
+
// duplication that is not there, and sends the author looking for a
|
|
649
|
+
// second projection to delete.
|
|
650
|
+
const pairOf = (p) => p.concept !== undefined ? `${p.concept}.${p.field}` : '';
|
|
651
|
+
const projections = new Set(indistinct.map(pairOf));
|
|
652
|
+
const only = projections.size === 1 ? [...projections][0] : '';
|
|
653
|
+
const from = only !== '' && params.filter(p => pairOf(p) === only).length > 1 ?
|
|
654
|
+
only :
|
|
655
|
+
'';
|
|
656
|
+
const shared = from !== '' ?
|
|
657
|
+
`multiple parameters projected from '${from}'` :
|
|
658
|
+
`multiple parameters of type '${identity}'`;
|
|
659
|
+
const one = names.length === 1;
|
|
660
|
+
errors.push(`${where} has ${shared}, so ${one ? `parameter ${names[0]}` :
|
|
661
|
+
`parameters ${names.slice(0, -1).join(', ')} and ${names[names.length - 1]}`} must ${one ? 'have' : 'each have'} a 'description' of ${one ? 'its' :
|
|
662
|
+
'their'} own to distinguish ${one ? 'it' : 'them'}.`);
|
|
663
|
+
}
|
|
664
|
+
}
|
|
665
|
+
}
|
|
666
|
+
// No executor is not an error: it is an action no binding performs here,
|
|
667
|
+
// which the availability pass reports rather than the validator.
|
|
668
|
+
const executor = action.executor;
|
|
669
|
+
if (executor !== undefined) {
|
|
670
|
+
for (const missing of missingExecutorFields(executor)) {
|
|
671
|
+
errors.push(`${where} has an ${executor.kind} executor whose '${missing}' is missing or blank.`);
|
|
672
|
+
}
|
|
673
|
+
}
|
|
674
|
+
// An unresolved guard leaves the author believing the write is checked when
|
|
675
|
+
// nothing checks it, so it fails the push rather than warning. Constraint
|
|
676
|
+
// names are model-scoped and the loader rejects duplicates, so a name
|
|
677
|
+
// either resolves here or names nothing at all.
|
|
678
|
+
for (const guard of action.guards ?? []) {
|
|
679
|
+
if (!constraintNames.has(guard)) {
|
|
680
|
+
errors.push(`${where} is guarded by '${guard}', but model '${model.name}' declares no constraint of that name.`);
|
|
681
|
+
}
|
|
682
|
+
}
|
|
683
|
+
errors.push(...affectedConceptErrors(action, where, concepts));
|
|
684
|
+
errors.push(...sqlExecutorErrors(action, where));
|
|
685
|
+
}
|
|
686
|
+
return errors;
|
|
687
|
+
}
|
|
688
|
+
// The errors in a SQL executor's statements. Nothing here for the other three
|
|
689
|
+
// executor kinds: they name a system that performs the write, so the model has
|
|
690
|
+
// no text to check.
|
|
691
|
+
//
|
|
692
|
+
// A SQL executor contains the write, which is what makes its blast radius
|
|
693
|
+
// checkable and its guards enforceable. The same property makes it the one
|
|
694
|
+
// executor kind that could smuggle an unreviewed write into a governed model, so
|
|
695
|
+
// the shape is pinned rather than trusted:
|
|
696
|
+
//
|
|
697
|
+
// - One DML verb per statement. A statement that reads is a query, and a
|
|
698
|
+
// statement that reshapes the schema is not an action.
|
|
699
|
+
// - No statement separator. Each statement is executed on its own, so a
|
|
700
|
+
// semicolon means the author expected a second statement to run and it
|
|
701
|
+
// silently would not.
|
|
702
|
+
// - Every `@parameter` is declared. This is the load-bearing one: it is what
|
|
703
|
+
// lets the runtime BIND every value instead of interpolating it, so no
|
|
704
|
+
// argument can reach the store as SQL.
|
|
705
|
+
function sqlExecutorErrors(action, where) {
|
|
706
|
+
if (action.executor?.kind !== 'sql')
|
|
707
|
+
return [];
|
|
708
|
+
const executor = action.executor;
|
|
709
|
+
const errors = [];
|
|
710
|
+
const bindable = new Set(action.parameters.map(p => p.name));
|
|
711
|
+
executor.sql.statements.forEach((stmt, i) => {
|
|
712
|
+
const at = `${where} has a sql executor whose statement ${i + 1}`;
|
|
713
|
+
const text = stmt.trim();
|
|
714
|
+
if (!text) {
|
|
715
|
+
errors.push(`${at} is blank.`);
|
|
716
|
+
return;
|
|
717
|
+
}
|
|
718
|
+
// The same reader the runtime uses, deliberately: what a model may
|
|
719
|
+
// PUBLISH and what the runtime will EXECUTE have to agree about where a
|
|
720
|
+
// statement's verb is. A naive first-word read disagreed -- the runtime
|
|
721
|
+
// ran `-- why\nUPDATE ...` and a CTE ahead of the verb, both covered by
|
|
722
|
+
// tests, while this rejected them at push time, so the library accepted an
|
|
723
|
+
// authored executor `kcmd` refused. Sharing the scanner does not widen
|
|
724
|
+
// what may be published: it can return MERGE, which SQL_EXECUTOR_VERBS
|
|
725
|
+
// does not list, so a MERGE is still refused here.
|
|
726
|
+
const verb = (0, sql_identifiers_1.leadingDmlVerb)(text);
|
|
727
|
+
if (!ir_1.SQL_EXECUTOR_VERBS.includes(verb)) {
|
|
728
|
+
// Name what the author wrote, not merely that nothing was found. The
|
|
729
|
+
// scanner reads DML verbs only, so a SELECT executor -- the likeliest
|
|
730
|
+
// mistake here -- comes back empty, and 'has no readable DML verb' would
|
|
731
|
+
// describe a query the author can see perfectly well. When the statement
|
|
732
|
+
// does not plainly open with a word, the first token is a comment marker
|
|
733
|
+
// and names nothing, so report the absence instead.
|
|
734
|
+
const first = text.split(/\s/, 1)[0].toUpperCase();
|
|
735
|
+
const wrote = verb || (/^[A-Z_][A-Z0-9_]*$/.test(first) ? first : '');
|
|
736
|
+
errors.push(`${at} ${wrote ? `starts with '${wrote}'` :
|
|
737
|
+
'has no readable DML verb'}, but a statement must be ` +
|
|
738
|
+
`one of ${ir_1.SQL_EXECUTOR_VERBS.join(', ')}.`);
|
|
739
|
+
}
|
|
740
|
+
if (text.slice(0, -1).includes(';')) {
|
|
741
|
+
errors.push(`${at} contains ';'. Each statement runs on its own, so write ` +
|
|
742
|
+
`one statement per list entry.`);
|
|
743
|
+
}
|
|
744
|
+
for (const name of (0, sql_identifiers_1.referencedParameters)(text)) {
|
|
745
|
+
if (!bindable.has(name)) {
|
|
746
|
+
errors.push(`${at} binds '@${name}', but action '${action.name}' ` +
|
|
747
|
+
`declares no parameter of that name.`);
|
|
748
|
+
}
|
|
749
|
+
}
|
|
750
|
+
});
|
|
751
|
+
return errors;
|
|
752
|
+
}
|
|
753
|
+
// The errors in one action's `affects`. Split out because the checks chain: a
|
|
754
|
+
// concept that does not resolve makes every later check about it meaningless,
|
|
755
|
+
// so an entry that fails one of those says nothing more. Undeclared fields do
|
|
756
|
+
// not chain -- each is an independent fact about a concept that did resolve --
|
|
757
|
+
// so an entry reports all of them.
|
|
758
|
+
//
|
|
759
|
+
// An absent `concepts` says the model reaching this point is a profile's view
|
|
760
|
+
// of the author's model, not the author's model. Everything that reads the
|
|
761
|
+
// ontology stands down there, because pruneUnavailable drops the fields a
|
|
762
|
+
// profile leaves unbound, and the metrics that read them. An entry naming a
|
|
763
|
+
// dropped field is the profile's doing rather than the author's, and failing
|
|
764
|
+
// the deploy over it would fail it for no reason. An action reaches no graph in any case. What is left is the one
|
|
765
|
+
// check that reads only the entry itself.
|
|
766
|
+
function affectedConceptErrors(action, where, concepts) {
|
|
767
|
+
const errors = [];
|
|
768
|
+
for (const affected of action.affects ?? []) {
|
|
769
|
+
// A `delete` takes the whole instance, so naming fields alongside one is
|
|
770
|
+
// self-contradictory whatever the ontology says.
|
|
771
|
+
if (affected.fields?.length && affected.operation === 'delete') {
|
|
772
|
+
errors.push(`${where} affects '${affected.concept}' with operation 'delete' ` +
|
|
773
|
+
`and also names fields. A 'delete' takes the whole instance; drop ` +
|
|
774
|
+
`the fields.`);
|
|
775
|
+
continue;
|
|
776
|
+
}
|
|
777
|
+
if (!concepts)
|
|
778
|
+
continue;
|
|
779
|
+
const concept = concepts.get(affected.concept);
|
|
780
|
+
if (!concept) {
|
|
781
|
+
errors.push(`${where} affects '${affected.concept}', which is neither an ` +
|
|
782
|
+
`entity nor a relationship this model declares.`);
|
|
783
|
+
continue;
|
|
784
|
+
}
|
|
785
|
+
for (const field of affected.fields ?? []) {
|
|
786
|
+
if (!concept.fields.has(field)) {
|
|
787
|
+
errors.push(`${where} affects '${affected.concept}.${field}', but ${concept.kind} '${affected.concept}' declares no field '${field}'.`);
|
|
788
|
+
}
|
|
789
|
+
}
|
|
790
|
+
}
|
|
791
|
+
return errors;
|
|
792
|
+
}
|
|
793
|
+
// What a `concept` may name, and the fields it has. Shared with the loader --
|
|
794
|
+
// which resolves a projected parameter against exactly this -- so the two can
|
|
795
|
+
// never disagree about whether `Order` exists or what it declares.
|
|
796
|
+
const declaredConcepts = resolve_inheritance_1.declaredConceptFields;
|
|
797
|
+
// Why a parameter has no usable scalar type, or nothing when it has one.
|
|
798
|
+
//
|
|
799
|
+
// Each shape of the mistake gets its own message, because each has its own
|
|
800
|
+
// repair. Telling an author who misspelled a field that the concept is
|
|
801
|
+
// unknown, or telling one who wrote `type: Account` that `Account` is not a
|
|
802
|
+
// datatype, sends them to the wrong line.
|
|
803
|
+
//
|
|
804
|
+
// The projection is checked only when the ontology is in hand: `concepts` is
|
|
805
|
+
// absent for a profile's pruned view, where a concept the author named may
|
|
806
|
+
// legitimately be gone (see affectedConceptErrors). The type check below
|
|
807
|
+
// survives that, because the loader resolved the type before any pruning ran.
|
|
808
|
+
function parameterTypeErrors(param, where, concepts) {
|
|
809
|
+
const errors = [];
|
|
810
|
+
if (param.concept !== undefined && concepts) {
|
|
811
|
+
const concept = concepts.get(param.concept);
|
|
812
|
+
if (!concept) {
|
|
813
|
+
errors.push(`${where} has parameter '${param.name}' projected from '${param.concept}', which is neither an entity nor a ` +
|
|
814
|
+
`relationship this model declares.`);
|
|
815
|
+
}
|
|
816
|
+
else if (!concept.fields.has(param.field ?? '')) {
|
|
817
|
+
errors.push(`${where} has parameter '${param.name}' projected from field '${param.field}', which ${concept.kind} '${param.concept}' does not declare.`);
|
|
818
|
+
}
|
|
819
|
+
}
|
|
820
|
+
if (param.type === undefined) {
|
|
821
|
+
// A projection this pass could not resolve at all says nothing about the
|
|
822
|
+
// field. `concepts` is undefined when the concept table would not build --
|
|
823
|
+
// a dangling `extends`, which is reported on its own line -- and the
|
|
824
|
+
// loader then drops `extends` model-wide, so a parameter projecting from
|
|
825
|
+
// an INHERITED field arrives here typeless for a reason that has nothing
|
|
826
|
+
// to do with the field. Blaming it would send the author to a field that
|
|
827
|
+
// is correctly typed, and away from the `extends` that is the actual
|
|
828
|
+
// fault.
|
|
829
|
+
if (param.concept !== undefined && !concepts)
|
|
830
|
+
return errors;
|
|
831
|
+
// A projection that failed above already says why there is no type; a
|
|
832
|
+
// second line repeating it would only add noise.
|
|
833
|
+
if (!errors.length) {
|
|
834
|
+
errors.push(param.concept !== undefined ?
|
|
835
|
+
// The projection resolved -- the field is simply untyped. Saying
|
|
836
|
+
// "state a type, or project one from a field" here would advise
|
|
837
|
+
// the author to do what they already did, and send them looking
|
|
838
|
+
// at the action instead of at the field that is missing one.
|
|
839
|
+
`${where} has parameter '${param.name}' projected from field '${param.concept}.${param.field}', which declares no ` +
|
|
840
|
+
`datatype. Give that field a scalar 'type' (${ir_1.DATA_TYPES.join('/')}) and the parameter takes it.` :
|
|
841
|
+
`${where} has parameter '${param.name}' with no type. A ` +
|
|
842
|
+
`parameter states a scalar 'type' (${ir_1.DATA_TYPES.join('/')}), or projects one from a field ` +
|
|
843
|
+
`with 'concept' and 'field'.`);
|
|
844
|
+
}
|
|
845
|
+
return errors;
|
|
846
|
+
}
|
|
847
|
+
if (ir_1.DATA_TYPES.includes(param.type))
|
|
848
|
+
return errors;
|
|
849
|
+
const named = concepts?.get(param.type);
|
|
850
|
+
errors.push(named ? `${where} has parameter '${param.name}' typed '${param.type}', which is ${named.kind === 'entity' ? 'an entity' : 'a relationship'} ` +
|
|
851
|
+
`rather than a scalar datatype. A parameter carries a value, ` +
|
|
852
|
+
`so take one from ${param.type} by projecting it: ` +
|
|
853
|
+
`{concept: ${param.type}, field: <field>}.` :
|
|
854
|
+
`${where} has parameter '${param.name}' typed '${param.type}', which is not a scalar datatype (${ir_1.DATA_TYPES.join('/')}).`);
|
|
855
|
+
return errors;
|
|
856
|
+
}
|
|
857
|
+
// Static, target-independent checks for a model's constraints.
|
|
858
|
+
//
|
|
859
|
+
// A constraint states its rule as a `judgment`, and every constraint must state
|
|
860
|
+
// one: a constraint with no body states no rule at all. (A document that states
|
|
861
|
+
// the removed `expression` body never gets this far -- rejectExpressionBody in
|
|
862
|
+
// loader.ts fails the parse and says where the rule goes instead.)
|
|
863
|
+
//
|
|
864
|
+
// A judgment gets two checks, in judgedConstraintErrors: it must say what a
|
|
865
|
+
// violation does, and every `<Entity>.<field>` token naming a KNOWN entity must
|
|
866
|
+
// name a field that entity declares. The second catches a typo that would
|
|
867
|
+
// otherwise surface only inside an agent's rejected action -- a rule is as easy
|
|
868
|
+
// to misspell in a sentence as anywhere else, and a judge handed `Order.totl`
|
|
869
|
+
// is being asked about a column that does not exist.
|
|
870
|
+
//
|
|
871
|
+
// Everything else is left alone. So a leading qualifier that is not a known
|
|
872
|
+
// entity is not guessed at here: a relationship-qualified name like
|
|
873
|
+
// `OrderedAs.quantity`, a metric reference, an ordinary English word that
|
|
874
|
+
// happens to contain a dot. A valid constraint must never be falsely rejected.
|
|
875
|
+
//
|
|
876
|
+
// Keeping that promise takes care, because the model reaching this function is
|
|
877
|
+
// not the document the author wrote. Its field lists have moved twice:
|
|
878
|
+
// - Inheritance is still AS DECLARED. `extends` is flattened by the graph
|
|
879
|
+
// legs, which run after this gate, so a subtype's `fields` here omit every
|
|
880
|
+
// field it inherits. Looking the field up on the resolved model fixes that.
|
|
881
|
+
// - A profile push has already pruned unbound fields (`fieldsPruned`). The
|
|
882
|
+
// author's field is gone from the model, and resolving does not bring it
|
|
883
|
+
// back, so the field check stands down. A constraint reaches no graph in
|
|
884
|
+
// any case, and failing a push over a field this profile does not bind
|
|
885
|
+
// would refuse a deploy for no reason.
|
|
886
|
+
function validateConstraints(model, document, fieldsPruned) {
|
|
887
|
+
const errors = [];
|
|
888
|
+
const constraints = model.constraints ?? [];
|
|
889
|
+
if (!constraints.length)
|
|
890
|
+
return errors;
|
|
891
|
+
const fieldsByEntity = fieldsPruned ? undefined : ifResolvable(() => declaredFields(model));
|
|
892
|
+
// Names the model declares that are not fields of any one entity, which a
|
|
893
|
+
// token's tail may legitimately carry. See unknownFieldRefs.
|
|
894
|
+
const nonFieldNames = new Set([
|
|
895
|
+
...(model.relationships ?? []).map(r => r.name),
|
|
896
|
+
...(model.metrics ?? []).map(m => m.name),
|
|
897
|
+
]);
|
|
898
|
+
for (const c of constraints) {
|
|
899
|
+
const where = `constraint '${c.name}' in model '${model.name}' (${document})`;
|
|
900
|
+
if (c.judgment === undefined) {
|
|
901
|
+
errors.push(`${where} declares no judgment. A constraint states its rule in ` +
|
|
902
|
+
`words, under 'judgment', and says what a violation does, under ` +
|
|
903
|
+
`'on_violation'.`);
|
|
904
|
+
continue;
|
|
905
|
+
}
|
|
906
|
+
errors.push(...judgedConstraintErrors(c, where, fieldsByEntity, nonFieldNames));
|
|
907
|
+
}
|
|
908
|
+
return errors;
|
|
909
|
+
}
|
|
910
|
+
// What a constraint must satisfy, beyond stating a body at all.
|
|
911
|
+
//
|
|
912
|
+
// It must say what a violation does. Any of the three words is allowed,
|
|
913
|
+
// `reject` included, but silence is not: an unmarked constraint rejects, and
|
|
914
|
+
// inheriting the harshest consequence by omission is the one outcome an author
|
|
915
|
+
// is least likely to have meant. Nothing here reads the prose to check the word
|
|
916
|
+
// against it -- the prose is prose, and a check that asked a model whether a
|
|
917
|
+
// sentence means refusal would be no guardrail at all.
|
|
918
|
+
//
|
|
919
|
+
// The other check resolves the `Entity.field` tokens the prose mentions, which
|
|
920
|
+
// is the whole of the static checking a rule settled by reading can get.
|
|
921
|
+
function judgedConstraintErrors(c, where, fieldsByEntity, nonFieldNames) {
|
|
922
|
+
const errors = [];
|
|
923
|
+
// The two checks are independent, so an empty judgment does not skip the
|
|
924
|
+
// routing one. Reporting only the empty body would send the author back for a
|
|
925
|
+
// second failure over a key they were never told about.
|
|
926
|
+
const empty = !c.judgment.trim();
|
|
927
|
+
if (empty) {
|
|
928
|
+
errors.push(`${where} has an empty judgment.`);
|
|
929
|
+
}
|
|
930
|
+
if (c.onViolation === undefined) {
|
|
931
|
+
errors.push(`${where} must state on_violation: 'reject' to ` +
|
|
932
|
+
`refuse the write, 'escalate' to hold it for a person, or 'warn' to ` +
|
|
933
|
+
`let it through and report it. An unmarked constraint rejects, which ` +
|
|
934
|
+
`is too strong a thing to inherit by leaving the key out.`);
|
|
935
|
+
}
|
|
936
|
+
if (!empty) {
|
|
937
|
+
errors.push(...unknownFieldRefs(c.judgment, where, fieldsByEntity, nonFieldNames));
|
|
938
|
+
}
|
|
939
|
+
return errors;
|
|
940
|
+
}
|
|
941
|
+
// Every `Entity.field` token in a judgment whose entity the model declares and
|
|
942
|
+
// whose field it does not.
|
|
943
|
+
//
|
|
944
|
+
// What makes a token a reference is that the model declares the entity, so no
|
|
945
|
+
// spelling heuristic is needed and none is used: entity names here are as often
|
|
946
|
+
// lowercase (`customer`, `orders`) as capitalized, and a rule keyed on the
|
|
947
|
+
// capital would check some models and quietly skip others. An unrecognized
|
|
948
|
+
// entity name is left alone on the principle that keeps this scan
|
|
949
|
+
// conservative -- a rule may name a concept from another system, and
|
|
950
|
+
// refusing to guess is what stops a valid rule being falsely rejected. A known
|
|
951
|
+
// entity with an unknown field is the case where the author plainly meant this
|
|
952
|
+
// model and got the name wrong, so that one is an error.
|
|
953
|
+
// An entity that declares no fields at all is unknowable in the same way an
|
|
954
|
+
// unrecognized name is, so the scan stands down there too. Fields are optional
|
|
955
|
+
// on an entity, and a logical model bound to nothing but Knowledge Catalog
|
|
956
|
+
// routinely declares none; reading an empty set as "this entity has no such
|
|
957
|
+
// field" would refuse every constraint such a model can write.
|
|
958
|
+
//
|
|
959
|
+
// A tail that names something the model declares is not a misspelled field
|
|
960
|
+
// either. Traversal has no syntax in the model today, so a prose token like
|
|
961
|
+
// `Customer.Order` or `LineItem.BelongsTo` reads as a field of the head and
|
|
962
|
+
// would be rejected for a field the author never claimed existed; the same goes
|
|
963
|
+
// for `Order.total_revenue`, where metrics are model-level and the qualifier is
|
|
964
|
+
// the entity the metric hangs off. The scan therefore settles only the case it
|
|
965
|
+
// can: a tail the model does not declare under any kind is the misspelling.
|
|
966
|
+
//
|
|
967
|
+
// The two segments must be adjacent to the dot, which is what keeps a sentence
|
|
968
|
+
// boundary ("check the memo. Every credit...") out of the scan. A decimal
|
|
969
|
+
// number cannot survive either, since no entity is named `30`.
|
|
970
|
+
function unknownFieldRefs(judgment, where, fieldsByEntity, nonFieldNames) {
|
|
971
|
+
if (!fieldsByEntity)
|
|
972
|
+
return [];
|
|
973
|
+
const errors = [];
|
|
974
|
+
const seen = new Set();
|
|
975
|
+
const token = /\b([A-Za-z_]\w*)\.([A-Za-z_]\w*)/g;
|
|
976
|
+
for (let m = token.exec(judgment); m; m = token.exec(judgment)) {
|
|
977
|
+
const [, entity, field] = m;
|
|
978
|
+
// A token with a third segment (`Order.lineItems.amount`) is a path rather
|
|
979
|
+
// than a field of `Order`, and reading its middle segment as one would
|
|
980
|
+
// reject it for a field the author never claimed existed. Nothing in the
|
|
981
|
+
// model defines path syntax today, so the scan declines to guess in both
|
|
982
|
+
// directions: a token followed by another dotted segment is skipped, and so
|
|
983
|
+
// is one preceded by a dot, which is how the tail of a path presents.
|
|
984
|
+
const after = judgment.slice(m.index + m[0].length);
|
|
985
|
+
if (/^\.\w/.test(after) || (m.index > 0 && judgment[m.index - 1] === '.')) {
|
|
986
|
+
continue;
|
|
987
|
+
}
|
|
988
|
+
const fields = fieldsByEntity.get(entity);
|
|
989
|
+
if (!fields || !fields.size || fields.has(field))
|
|
990
|
+
continue;
|
|
991
|
+
// The tail names something the model declares that is not a field of the
|
|
992
|
+
// head: another entity, a relationship, or a metric. `Customer.Order` and
|
|
993
|
+
// `LineItem.BelongsTo` are traversals and `Order.total_revenue` is a metric
|
|
994
|
+
// reference, none of which this scan can settle, and all of which an author
|
|
995
|
+
// writing prose reaches for. Only a tail the model does not declare at all
|
|
996
|
+
// is read as the misspelling this check exists to catch.
|
|
997
|
+
if (fieldsByEntity.has(field) || nonFieldNames.has(field))
|
|
998
|
+
continue;
|
|
999
|
+
const ref = `${entity}.${field}`;
|
|
1000
|
+
if (seen.has(ref))
|
|
1001
|
+
continue;
|
|
1002
|
+
seen.add(ref);
|
|
1003
|
+
errors.push(`${where} references '${ref}', but entity '${entity}' declares no ` +
|
|
1004
|
+
`field '${field}'.`);
|
|
1005
|
+
}
|
|
1006
|
+
return errors;
|
|
1007
|
+
}
|
|
1008
|
+
// What `build` returns, or nothing when the model's inheritance does not
|
|
1009
|
+
// resolve.
|
|
1010
|
+
//
|
|
1011
|
+
// `declaredFields` and `declaredConcepts` both resolve inheritance, and
|
|
1012
|
+
// resolving THROWS on an `extends` naming an entity the model does not declare,
|
|
1013
|
+
// on a cycle, and on an ambiguously inherited field, rather than reporting it.
|
|
1014
|
+
// The loader accepts such a model, a
|
|
1015
|
+
// Knowledge-Catalog-only push reaches no graph leg to catch it, and a profile
|
|
1016
|
+
// push can create one by pruning a supertype whole. A validation gate reports;
|
|
1017
|
+
// it does not stack-trace, so every caller that resolves inheritance to answer
|
|
1018
|
+
// a question comes through here and stands its own check down when the answer
|
|
1019
|
+
// is unavailable. That is the same thing a pruned profile does. Standing down
|
|
1020
|
+
// is not silence: validatePushRequirements reports the resolution failure once
|
|
1021
|
+
// per model, so the larger problem is named and only the checks that depend on
|
|
1022
|
+
// the resolved model go quiet.
|
|
1023
|
+
function ifResolvable(build) {
|
|
1024
|
+
try {
|
|
1025
|
+
return build();
|
|
1026
|
+
}
|
|
1027
|
+
catch {
|
|
1028
|
+
return undefined;
|
|
1029
|
+
}
|
|
1030
|
+
}
|
|
1031
|
+
// Why resolving the model's inheritance fails, or nothing when it succeeds.
|
|
1032
|
+
//
|
|
1033
|
+
// Reported rather than thrown, and reported once for the model rather than once
|
|
1034
|
+
// per check that needed it. A model declaring no inheritance cannot fail, and
|
|
1035
|
+
// resolving clones, so it is not asked.
|
|
1036
|
+
function inheritanceFailure(model) {
|
|
1037
|
+
if (!(model.entities ?? []).some(e => e.extends?.length))
|
|
1038
|
+
return undefined;
|
|
1039
|
+
try {
|
|
1040
|
+
(0, resolve_inheritance_1.resolveInheritance)(model);
|
|
1041
|
+
return undefined;
|
|
1042
|
+
}
|
|
1043
|
+
catch (e) {
|
|
1044
|
+
const message = e instanceof Error ? e.message : String(e);
|
|
1045
|
+
return {
|
|
1046
|
+
kind: e instanceof resolve_inheritance_1.InheritanceError ? e.kind : 'other',
|
|
1047
|
+
message: message.endsWith('.') ? message : `${message}.`,
|
|
1048
|
+
};
|
|
1049
|
+
}
|
|
1050
|
+
}
|
|
1051
|
+
// The authored name of each property that defines a field rather than binds
|
|
1052
|
+
// it, for messages.
|
|
1053
|
+
const AUTHORED_NAME = {
|
|
1054
|
+
type: 'datatype',
|
|
1055
|
+
aiContext: 'ai_context',
|
|
1056
|
+
customExtensions: 'custom_extensions',
|
|
1057
|
+
};
|
|
1058
|
+
/**
|
|
1059
|
+
* Returns one message per broken inheritance or concrete-leaf rule, or an
|
|
1060
|
+
* empty list: a cycle, an unknown parent, a field inherited from two
|
|
1061
|
+
* unrelated ancestors, or two fields of one entity whose names differ only in
|
|
1062
|
+
* case; a redeclared inherited field; an
|
|
1063
|
+
* abstract entity with a table; and a relationship or metric on an entity that
|
|
1064
|
+
* is not a concrete leaf. Pruning can remove the field a rule is about, so a
|
|
1065
|
+
* push that prunes runs this on the model before pruning as well as after.
|
|
1066
|
+
*/
|
|
1067
|
+
function validateInheritance(models) {
|
|
1068
|
+
const errors = [];
|
|
1069
|
+
for (const { document, model } of models) {
|
|
1070
|
+
// Resolving inheritance throws on an `extends` naming an entity the model
|
|
1071
|
+
// does not declare, on a cycle, on a field inherited from two unrelated
|
|
1072
|
+
// ancestors, and on two fields of one entity whose names differ only in
|
|
1073
|
+
// case, and the checks below stand down rather than stack-trace on
|
|
1074
|
+
// one. Standing down has to mean reporting somewhere or it means
|
|
1075
|
+
// publishing a broken model in silence: the loader accepts such a model,
|
|
1076
|
+
// and a Knowledge-Catalog-only push reaches no graph leg that would resolve
|
|
1077
|
+
// inheritance and catch it. So the failure is reported here, once per
|
|
1078
|
+
// model. Pruning never removes an entity, so an unknown parent is always
|
|
1079
|
+
// the author's.
|
|
1080
|
+
const failure = inheritanceFailure(model);
|
|
1081
|
+
if (failure) {
|
|
1082
|
+
errors.push(`model '${model.name}' (${document}): ${failure.message} Checks ` +
|
|
1083
|
+
`that need the resolved model are skipped until this is fixed.`);
|
|
1084
|
+
}
|
|
1085
|
+
errors.push(...inheritanceRuleErrors(model, document));
|
|
1086
|
+
}
|
|
1087
|
+
return errors;
|
|
1088
|
+
}
|
|
1089
|
+
// The rules inheritance puts on a model beyond resolving it:
|
|
1090
|
+
// - a subtype may redeclare an inherited field only to rebind it: the line
|
|
1091
|
+
// sets `expression` and nothing else;
|
|
1092
|
+
// - an abstract entity has no table, so it declares no source, no key and no
|
|
1093
|
+
// bound field;
|
|
1094
|
+
// - a relationship endpoint, and the entity a metric belongs to, is a
|
|
1095
|
+
// concrete leaf: not abstract, and extended by no other entity. An edge
|
|
1096
|
+
// joins one table to one table and a measure binds to one table, while a
|
|
1097
|
+
// non-leaf entity stands for the union of its descendants' tables.
|
|
1098
|
+
// The first rule needs the resolved model and is skipped when resolution
|
|
1099
|
+
// fails; that failure is reported once elsewhere.
|
|
1100
|
+
function inheritanceRuleErrors(model, document) {
|
|
1101
|
+
const errors = [];
|
|
1102
|
+
const entities = model.entities ?? [];
|
|
1103
|
+
const where = (name) => `entity '${name}' in model '${model.name}' (${document})`;
|
|
1104
|
+
const resolved = entities.some(e => e.extends?.length) ?
|
|
1105
|
+
ifResolvable(() => (0, resolve_inheritance_1.resolveInheritance)(model).model.entities ?? []) :
|
|
1106
|
+
undefined;
|
|
1107
|
+
if (resolved) {
|
|
1108
|
+
const ancestorsOf = new Map(resolved.map(e => [e.name, e.extends ?? []]));
|
|
1109
|
+
const ownNames = new Map(entities.map(e => [e.name, new Set(e.fields.map(f => f.name))]));
|
|
1110
|
+
for (const entity of entities) {
|
|
1111
|
+
const inherited = new Set((ancestorsOf.get(entity.name) ?? [])
|
|
1112
|
+
.flatMap(a => [...(ownNames.get(a) ?? [])]));
|
|
1113
|
+
for (const field of entity.fields) {
|
|
1114
|
+
if (!inherited.has(field.name))
|
|
1115
|
+
continue;
|
|
1116
|
+
const defined = ir_1.FIELD_DEFINITION_KEYS.filter(k => field[k] !== undefined)
|
|
1117
|
+
.map(k => `'${AUTHORED_NAME[k] ?? k}'`);
|
|
1118
|
+
if (defined.length) {
|
|
1119
|
+
errors.push(`${where(entity.name)}: field '${field.name}' is inherited, so ` +
|
|
1120
|
+
`it may only be rebound to a column; remove ${defined.join(', ')}, which an ancestor defines.`);
|
|
1121
|
+
}
|
|
1122
|
+
else if (!(0, ir_1.isFieldBound)(field)) {
|
|
1123
|
+
// A redeclaration sets `expression` and nothing else, so a line with
|
|
1124
|
+
// only the name is not one.
|
|
1125
|
+
const from = (ancestorsOf.get(entity.name) ?? [])
|
|
1126
|
+
.find(a => ownNames.get(a)?.has(field.name));
|
|
1127
|
+
errors.push(`${where(entity.name)}: field '${field.name}' is inherited from ` +
|
|
1128
|
+
`'${from}'; restate it only to rebind it with 'expression', or ` +
|
|
1129
|
+
`remove the line.`);
|
|
1130
|
+
}
|
|
1131
|
+
}
|
|
1132
|
+
}
|
|
1133
|
+
}
|
|
1134
|
+
for (const entity of entities) {
|
|
1135
|
+
if (!entity.abstract)
|
|
1136
|
+
continue;
|
|
1137
|
+
if (entity.dataSource || entity.authoredSource) {
|
|
1138
|
+
errors.push(`${where(entity.name)} is abstract, so it has no table and cannot ` +
|
|
1139
|
+
`declare a source.`);
|
|
1140
|
+
}
|
|
1141
|
+
if (entity.keys.length || entity.uniqueKeys?.length) {
|
|
1142
|
+
errors.push(`${where(entity.name)} is abstract, so it has no table and cannot ` +
|
|
1143
|
+
`declare a primary key or unique keys.`);
|
|
1144
|
+
}
|
|
1145
|
+
for (const field of entity.fields) {
|
|
1146
|
+
if ((0, ir_1.isFieldBound)(field)) {
|
|
1147
|
+
errors.push(`${where(entity.name)} is abstract, so its field '${field.name}' cannot carry an expression; bind it on each ` +
|
|
1148
|
+
`concrete subtype instead.`);
|
|
1149
|
+
}
|
|
1150
|
+
}
|
|
1151
|
+
}
|
|
1152
|
+
const abstractNames = new Set(entities.filter(e => e.abstract).map(e => e.name));
|
|
1153
|
+
const extendedNames = new Set(entities.flatMap(e => e.extends ?? []));
|
|
1154
|
+
const notLeaf = (name) => abstractNames.has(name) ?
|
|
1155
|
+
'abstract' :
|
|
1156
|
+
extendedNames.has(name) ? 'extended by another entity' : undefined;
|
|
1157
|
+
for (const rel of model.relationships ?? []) {
|
|
1158
|
+
// A relationship from an entity to itself is reported once.
|
|
1159
|
+
for (const end of new Set([rel.source.entity, rel.destination.entity])) {
|
|
1160
|
+
const why = notLeaf(end);
|
|
1161
|
+
if (why) {
|
|
1162
|
+
errors.push(`relationship '${rel.name}' in model '${model.name}' (${document}) connects '${end}', which is ${why}; a ` +
|
|
1163
|
+
`relationship may connect only concrete leaf entities.`);
|
|
1164
|
+
}
|
|
1165
|
+
}
|
|
1166
|
+
}
|
|
1167
|
+
for (const metric of model.metrics ?? []) {
|
|
1168
|
+
const owner = metric.authoredEntity ?? metric.entity;
|
|
1169
|
+
const why = owner ? notLeaf(owner) : undefined;
|
|
1170
|
+
if (why) {
|
|
1171
|
+
errors.push(`metric '${metric.name}' in model '${model.name}' (${document}) belongs to '${owner}', which is ${why}; a metric ` +
|
|
1172
|
+
`may belong only to a concrete leaf entity.`);
|
|
1173
|
+
}
|
|
1174
|
+
}
|
|
1175
|
+
return errors;
|
|
1176
|
+
}
|
|
1177
|
+
// True when a rejected column string could be a column name written without
|
|
1178
|
+
// backticks, such as `order date`, `order-date`, `margin%` or `R&D cost`. An
|
|
1179
|
+
// empty value gets no hint. Neither does a value that starts or ends with
|
|
1180
|
+
// whitespace, because the fix there is to delete the whitespace. BigQuery
|
|
1181
|
+
// allows only letters, numeric characters, marks, connector punctuation,
|
|
1182
|
+
// dashes, whitespace and `& % = + : ' < > # |` in a column name, so any other
|
|
1183
|
+
// character rules out the hint. Of the operators isColumnName rejects,
|
|
1184
|
+
// BigQuery allows `- + % = < > | &`. Each of those counts as an operator at
|
|
1185
|
+
// the start of the value, or next to whitespace unless it ends the value. No
|
|
1186
|
+
// operator among them can end an expression, so `growth %` keeps the hint. A
|
|
1187
|
+
// value made only of words, such as `a AND b`, passes this test, so the hint
|
|
1188
|
+
// is worded as a condition.
|
|
1189
|
+
function couldBeUnquotedColumnName(col) {
|
|
1190
|
+
return col === col.trim() &&
|
|
1191
|
+
/^[\p{L}\p{N}\p{Pc}\p{Pd}\p{M}\s&%=+:'<>#|]+$/u.test(col) &&
|
|
1192
|
+
!/^[-+%=<>|&]|\s[-+%=<>|&](?!$)|[-+%=<>|&]\s/.test(col);
|
|
1193
|
+
}
|
|
1194
|
+
function invalidColumnNameError(owner, modelName, document, slot, col) {
|
|
1195
|
+
const hint = couldBeUnquotedColumnName(col) ?
|
|
1196
|
+
' (if it names a column with spaces or special characters, wrap it ' +
|
|
1197
|
+
'in backticks, for example `order date`)' :
|
|
1198
|
+
'';
|
|
1199
|
+
return `${owner} in model '${modelName}' (${document}): ${slot} column '${col}' is not a physical column name; keys and join columns must name ` +
|
|
1200
|
+
`physical table columns, not expressions or field references${hint}.`;
|
|
1201
|
+
}
|
|
1202
|
+
// Checks that every key column and join column is a physical column name and
|
|
1203
|
+
// not a SQL expression or an entity-qualified reference. Key columns are the
|
|
1204
|
+
// entries of `primary_key` and `unique_keys`. Join columns are the entries of
|
|
1205
|
+
// `from_columns` and `to_columns`. Also checks that every relationship's
|
|
1206
|
+
// endpoints name entities the model declares, and that its two join column
|
|
1207
|
+
// lists have equal length. The loader checks endpoints and lengths for a model
|
|
1208
|
+
// it reads, and this repeats them for a model built in memory. A relationship
|
|
1209
|
+
// from an entity to itself is rejected, and so are relationships in both
|
|
1210
|
+
// directions between two entities. Push runs this
|
|
1211
|
+
// on the model with its profile applied, so it also checks the key and join
|
|
1212
|
+
// columns a profile file supplies. An abstract entity's keys and an
|
|
1213
|
+
// association's direct join columns are not checked.
|
|
1214
|
+
function validateColumnsAndRelationships(model, document) {
|
|
1215
|
+
const errors = [];
|
|
1216
|
+
const entityNames = new Set((model.entities ?? []).map(e => e.name));
|
|
1217
|
+
// The first relationship from each entity to each other entity.
|
|
1218
|
+
const firstFrom = new Map();
|
|
1219
|
+
for (const entity of model.entities ?? []) {
|
|
1220
|
+
if (entity.abstract)
|
|
1221
|
+
continue;
|
|
1222
|
+
const owner = `entity '${entity.name}'`;
|
|
1223
|
+
for (const col of entity.keys) {
|
|
1224
|
+
if (!(0, sql_expr_utils_1.isColumnName)(col)) {
|
|
1225
|
+
errors.push(invalidColumnNameError(owner, model.name, document, 'primary_key', col));
|
|
1226
|
+
}
|
|
1227
|
+
}
|
|
1228
|
+
for (const uk of entity.uniqueKeys ?? []) {
|
|
1229
|
+
for (const col of uk) {
|
|
1230
|
+
if (!(0, sql_expr_utils_1.isColumnName)(col)) {
|
|
1231
|
+
errors.push(invalidColumnNameError(owner, model.name, document, 'unique_keys', col));
|
|
1232
|
+
}
|
|
1233
|
+
}
|
|
1234
|
+
}
|
|
1235
|
+
}
|
|
1236
|
+
for (const rel of model.relationships ?? []) {
|
|
1237
|
+
const owner = `relationship '${rel.name}'`;
|
|
1238
|
+
const at = `${owner} in model '${model.name}' (${document})`;
|
|
1239
|
+
if (!entityNames.has(rel.source.entity)) {
|
|
1240
|
+
errors.push(`${at}: ${(0, loader_1.undefinedEndpointProblem)('from', rel.source.entity)}.`);
|
|
1241
|
+
}
|
|
1242
|
+
if (!entityNames.has(rel.destination.entity)) {
|
|
1243
|
+
errors.push(`${at}: ${(0, loader_1.undefinedEndpointProblem)('to', rel.destination.entity)}.`);
|
|
1244
|
+
}
|
|
1245
|
+
const [from, to] = [rel.source.entity, rel.destination.entity];
|
|
1246
|
+
const back = firstFrom.get(`${to}\0${from}`);
|
|
1247
|
+
if (from === to) {
|
|
1248
|
+
errors.push(`${at} runs from '${from}' to itself; a relationship joins two ` +
|
|
1249
|
+
`different entities.`);
|
|
1250
|
+
}
|
|
1251
|
+
else if (back !== undefined) {
|
|
1252
|
+
errors.push(`${at} runs from '${from}' to '${to}', and relationship '${back}' ` +
|
|
1253
|
+
`runs the other way; keep the relationships between two entities ` +
|
|
1254
|
+
`in one direction.`);
|
|
1255
|
+
}
|
|
1256
|
+
if (!firstFrom.has(`${from}\0${to}`)) {
|
|
1257
|
+
firstFrom.set(`${from}\0${to}`, rel.name);
|
|
1258
|
+
}
|
|
1259
|
+
if (rel.association)
|
|
1260
|
+
continue;
|
|
1261
|
+
const fromCount = rel.source.columns.length;
|
|
1262
|
+
const toCount = rel.destination.columns.length;
|
|
1263
|
+
if (!fromCount !== !toCount) {
|
|
1264
|
+
errors.push(`${at}: ${loader_1.HALF_BOUND_JOIN_PROBLEM}`);
|
|
1265
|
+
}
|
|
1266
|
+
else if (fromCount !== toCount) {
|
|
1267
|
+
errors.push(`${at}: ${(0, loader_1.joinLengthProblem)(fromCount, toCount)}.`);
|
|
1268
|
+
}
|
|
1269
|
+
for (const col of rel.source.columns) {
|
|
1270
|
+
if (!(0, sql_expr_utils_1.isColumnName)(col)) {
|
|
1271
|
+
errors.push(invalidColumnNameError(owner, model.name, document, 'from_columns', col));
|
|
1272
|
+
}
|
|
1273
|
+
}
|
|
1274
|
+
for (const col of rel.destination.columns) {
|
|
1275
|
+
if (!(0, sql_expr_utils_1.isColumnName)(col)) {
|
|
1276
|
+
errors.push(invalidColumnNameError(owner, model.name, document, 'to_columns', col));
|
|
1277
|
+
}
|
|
1278
|
+
}
|
|
1279
|
+
}
|
|
1280
|
+
return errors;
|
|
1281
|
+
}
|
|
1282
|
+
// A relationship's `to_columns` must cover a primary or unique key of the `to`
|
|
1283
|
+
// entity. A relationship is many-to-one or one-to-one only when its
|
|
1284
|
+
// `to_columns` cover a key, so a join that covers no key is invalid in
|
|
1285
|
+
// every model, whatever it deploys to. The check runs on every push, a
|
|
1286
|
+
// catalog-only push included. A superset of a key covers it (see
|
|
1287
|
+
// coversKey). Keys and join columns are physical columns, so they are compared
|
|
1288
|
+
// as written, never read through a field of the same name. Skipped here when
|
|
1289
|
+
// the check's own inputs are absent or already invalid (reported by their own
|
|
1290
|
+
// checks): an association (junction-table) relationship, an empty `to_columns`
|
|
1291
|
+
// list, a non-column entry in `to_columns`, an undeclared or abstract `to`
|
|
1292
|
+
// entity, or a non-column entry in the `to` entity's `primary_key` or
|
|
1293
|
+
// `unique_keys`.
|
|
1294
|
+
function relationshipKeyErrors(model, document) {
|
|
1295
|
+
const errors = [];
|
|
1296
|
+
const byName = new Map((model.entities ?? []).map(e => [e.name, e]));
|
|
1297
|
+
for (const rel of model.relationships ?? []) {
|
|
1298
|
+
const to = rel.destination.columns;
|
|
1299
|
+
if (rel.association || !to.length)
|
|
1300
|
+
continue;
|
|
1301
|
+
if (!to.every(sql_expr_utils_1.isColumnName))
|
|
1302
|
+
continue;
|
|
1303
|
+
const target = byName.get(rel.destination.entity);
|
|
1304
|
+
if (!target || target.abstract)
|
|
1305
|
+
continue;
|
|
1306
|
+
if (!target.keys.every(sql_expr_utils_1.isColumnName) ||
|
|
1307
|
+
!(target.uniqueKeys ?? []).flat().every(sql_expr_utils_1.isColumnName)) {
|
|
1308
|
+
continue;
|
|
1309
|
+
}
|
|
1310
|
+
if (!(0, ir_1.coversKey)(to, target.keys, target.uniqueKeys ?? [])) {
|
|
1311
|
+
errors.push(`relationship '${rel.name}' in model '${model.name}' (${document}): its to_columns ${JSON.stringify(to)} cover no primary or unique key of '${target.name}'; join on every column of one of its keys, and ` +
|
|
1312
|
+
`declare that key if '${target.name}' has none.`);
|
|
1313
|
+
}
|
|
1314
|
+
}
|
|
1315
|
+
return errors;
|
|
1316
|
+
}
|
|
1317
|
+
// Why each field of `model` holds no SQL, in the loader's words. Each field
|
|
1318
|
+
// is read on the entity that declares it, and each field an entity inherits
|
|
1319
|
+
// is read on that entity, as the loader does. An association's field is an
|
|
1320
|
+
// edge property, read on its relationship.
|
|
1321
|
+
function fieldNoSqlErrors(model, document) {
|
|
1322
|
+
const where = `in model '${model.name}' (${document})`;
|
|
1323
|
+
const errors = [];
|
|
1324
|
+
for (const e of model.entities ?? []) {
|
|
1325
|
+
for (const f of e.fields ?? []) {
|
|
1326
|
+
const problem = (0, loader_1.noSqlProblemOf)(f, e.name);
|
|
1327
|
+
if (problem) {
|
|
1328
|
+
errors.push(`field '${e.name}.${f.name}' ${where}: ${problem}; write the ` +
|
|
1329
|
+
`field's SQL, or remove 'expression' to leave the field unbound.`);
|
|
1330
|
+
}
|
|
1331
|
+
}
|
|
1332
|
+
}
|
|
1333
|
+
if ((model.entities ?? []).some(e => e.extends?.length)) {
|
|
1334
|
+
const concepts = ifResolvable(() => declaredConcepts(model));
|
|
1335
|
+
const inherited = concepts ?
|
|
1336
|
+
(0, loader_1.inheritedNoSqlProblems)(model.entities ?? [], concepts, loader_1.noSqlProblemOf) :
|
|
1337
|
+
[];
|
|
1338
|
+
for (const problem of inherited) {
|
|
1339
|
+
errors.push(`model '${model.name}' (${document}): ${problem}.`);
|
|
1340
|
+
}
|
|
1341
|
+
}
|
|
1342
|
+
for (const rel of model.relationships ?? []) {
|
|
1343
|
+
for (const f of rel.association?.fields ?? []) {
|
|
1344
|
+
const problem = (0, loader_1.noSqlProblemOf)(f, rel.name);
|
|
1345
|
+
if (problem) {
|
|
1346
|
+
errors.push(`relationship '${rel.name}' field '${f.name}' ${where}: ${problem}; write the field's SQL.`);
|
|
1347
|
+
}
|
|
1348
|
+
}
|
|
1349
|
+
}
|
|
1350
|
+
return errors;
|
|
1351
|
+
}
|
|
1352
|
+
// The errors for the fields of `entity` that name themselves, directly or
|
|
1353
|
+
// through other fields of `entity`. The graph renders a field that names a
|
|
1354
|
+
// sibling as `<entity>.<field>` from that sibling's own SQL, so a field that
|
|
1355
|
+
// reaches itself that way has no SQL to render. A field expression that writes
|
|
1356
|
+
// its own column as `<entity>.<field>` names itself too: write the bare column
|
|
1357
|
+
// instead. Each dialect is read on its own, from the text an engine of that
|
|
1358
|
+
// dialect renders (its entry, else the ANSI_SQL one), so two texts that no one
|
|
1359
|
+
// engine reads together do not form a cycle.
|
|
1360
|
+
//
|
|
1361
|
+
// `resolved` is `entity` with the fields it inherits, which name its fields
|
|
1362
|
+
// as the graph renders them (see resolve_inheritance.localizeInheritedField),
|
|
1363
|
+
// so a field can reach itself through one it inherits. A cycle is reported on
|
|
1364
|
+
// an entity only when a field on it is bound there, so one an ancestor forms on
|
|
1365
|
+
// its own is reported on that ancestor alone.
|
|
1366
|
+
function selfReferenceErrors(entity, resolved, model, document, impliedDialect) {
|
|
1367
|
+
const own = entity.fields ?? [];
|
|
1368
|
+
const inherited = resolved.fields ?? [];
|
|
1369
|
+
// A field a profile excludes is left off `resolved`, but still checked.
|
|
1370
|
+
const fields = [
|
|
1371
|
+
...inherited,
|
|
1372
|
+
...own.filter(f => !inherited.some(r => r.name === f.name)),
|
|
1373
|
+
];
|
|
1374
|
+
const bound = new Set(own.filter(f => (0, ir_1.isFieldBound)(f)).map(f => f.name));
|
|
1375
|
+
const names = new Set(fields.map(f => f.name));
|
|
1376
|
+
const dialects = new Set(['ANSI_SQL']);
|
|
1377
|
+
for (const f of fields) {
|
|
1378
|
+
for (const d of f.dialects ?? [])
|
|
1379
|
+
dialects.add(d.dialect);
|
|
1380
|
+
if (f.importedDialect)
|
|
1381
|
+
dialects.add(f.importedDialect);
|
|
1382
|
+
}
|
|
1383
|
+
const errors = new Set();
|
|
1384
|
+
for (const dialect of dialects) {
|
|
1385
|
+
// The fields of `entity` each field names, in this dialect, read as the
|
|
1386
|
+
// graph generators read them (see graph_ddl.siblingsRead), so a qualifier
|
|
1387
|
+
// the parser reserves, such as `Date` in `Date.days`, names them too. An
|
|
1388
|
+
// ANSI_SQL text is read in the dialect the sources imply, as
|
|
1389
|
+
// columnsForSource reads it, and not at all when their dialect is null.
|
|
1390
|
+
if (dialect === 'ANSI_SQL' && impliedDialect === null)
|
|
1391
|
+
continue;
|
|
1392
|
+
const readIn = dialect === 'ANSI_SQL' && impliedDialect ? impliedDialect : dialect;
|
|
1393
|
+
const reads = new Map();
|
|
1394
|
+
for (const f of fields) {
|
|
1395
|
+
const text = (0, ir_1.expressionForDialect)(f, dialect);
|
|
1396
|
+
if (text === undefined ||
|
|
1397
|
+
(0, loader_1.noSqlReason)(text, dialect, entity.name) !== undefined) {
|
|
1398
|
+
continue;
|
|
1399
|
+
}
|
|
1400
|
+
reads.set(f.name, [...new Set((0, sql_expr_utils_1.qualifiedColumns)(text, entity.name, readIn) ?? [])]
|
|
1401
|
+
.filter(n => names.has(n)));
|
|
1402
|
+
}
|
|
1403
|
+
for (const f of fields) {
|
|
1404
|
+
const through = pathBack(f.name, reads);
|
|
1405
|
+
if (!through || ![f.name, ...through].some(n => bound.has(n)))
|
|
1406
|
+
continue;
|
|
1407
|
+
errors.add((0, sql_expr_utils_1.selfReferenceMessage)(entity.name, [f.name, ...through], ` in model '${model.name}' (${document})`) +
|
|
1408
|
+
'.');
|
|
1409
|
+
}
|
|
1410
|
+
}
|
|
1411
|
+
return [...errors];
|
|
1412
|
+
}
|
|
1413
|
+
// The shortest run of other fields through which `start` reaches itself in
|
|
1414
|
+
// `reads`, empty when it names itself directly, or undefined when it does not
|
|
1415
|
+
// reach itself.
|
|
1416
|
+
function pathBack(start, reads) {
|
|
1417
|
+
const via = new Map();
|
|
1418
|
+
const queue = [];
|
|
1419
|
+
for (const next of reads.get(start) ?? []) {
|
|
1420
|
+
if (next === start)
|
|
1421
|
+
return [];
|
|
1422
|
+
if (!via.has(next)) {
|
|
1423
|
+
via.set(next, [next]);
|
|
1424
|
+
queue.push(next);
|
|
1425
|
+
}
|
|
1426
|
+
}
|
|
1427
|
+
while (queue.length) {
|
|
1428
|
+
const name = queue.shift();
|
|
1429
|
+
for (const next of reads.get(name) ?? []) {
|
|
1430
|
+
if (next === start)
|
|
1431
|
+
return via.get(name);
|
|
1432
|
+
if (!via.has(next)) {
|
|
1433
|
+
via.set(next, [...via.get(name), next]);
|
|
1434
|
+
queue.push(next);
|
|
1435
|
+
}
|
|
1436
|
+
}
|
|
1437
|
+
}
|
|
1438
|
+
return undefined;
|
|
1439
|
+
}
|
|
1440
|
+
// The error for a metric with no SQL, or undefined. A metric is rejected when
|
|
1441
|
+
// none of its texts holds SQL, with or without an anchor, since the graph
|
|
1442
|
+
// would drop it. It is also rejected when any one of its texts holds none, as
|
|
1443
|
+
// the loader rejects it, since a model built in memory skips the loader.
|
|
1444
|
+
function metricNoSqlError(metric, model, document) {
|
|
1445
|
+
const where = `metric '${metric.name}' in model '${model.name}' (${document})`;
|
|
1446
|
+
const readable = (0, resolve_profiles_1.expressionSources)(metric).filter(s => (0, loader_1.noSqlReason)(s.text, s.dialect) === undefined);
|
|
1447
|
+
if (!readable.length) {
|
|
1448
|
+
return `${where} has no SQL in 'expression'; write the metric's SQL.`;
|
|
1449
|
+
}
|
|
1450
|
+
const problem = (0, loader_1.noSqlProblemOf)(metric);
|
|
1451
|
+
return problem && `${where}: ${problem}; write the metric's SQL.`;
|
|
1452
|
+
}
|
|
1453
|
+
// Validates field and metric expressions:
|
|
1454
|
+
// - Reject a field or a metric with no SQL, as the loader does, since a
|
|
1455
|
+
// model built in memory skips the loader (see fieldNoSqlErrors and
|
|
1456
|
+
// metricNoSqlError).
|
|
1457
|
+
// The rules below run when inheritance resolved cleanly:
|
|
1458
|
+
// - For each metric:
|
|
1459
|
+
// 1. Unless it has no SQL, reject a metric with no `authoredEntity` when
|
|
1460
|
+
// its expressions, read together, reach no entity or more than one.
|
|
1461
|
+
// 2. Reject when `authoredEntity ?? entity` does not name a declared
|
|
1462
|
+
// entity (runs on every push).
|
|
1463
|
+
// 3. When `authoredEntity` is set and the expressions reference at least
|
|
1464
|
+
// one entity, at least one of those must be `authoredEntity`.
|
|
1465
|
+
// 4. For every referenced entity, every field it is referenced with must
|
|
1466
|
+
// be declared or inherited on that entity (gated on `!fieldsPruned`,
|
|
1467
|
+
// and run before pruning).
|
|
1468
|
+
// 5. Reject a column reference with no entity qualifier, or a qualified
|
|
1469
|
+
// reference whose qualifier is not a declared entity (including when it
|
|
1470
|
+
// differs in case from a declared entity), in a metric expression.
|
|
1471
|
+
// 6. Reject a metric whose name matches, ignoring case, a declared or
|
|
1472
|
+
// inherited field of `authoredEntity ?? entity`, since metrics and
|
|
1473
|
+
// fields on one entity share a namespace. This runs before pruning, so
|
|
1474
|
+
// a graph push catches it even when pruning drops either side.
|
|
1475
|
+
// - For each field on each entity E:
|
|
1476
|
+
// 1. Reject any reference to an entity other than E, and a qualifier that
|
|
1477
|
+
// differs only in case from a declared entity.
|
|
1478
|
+
// 2. For references to E itself, every field named must be declared or
|
|
1479
|
+
// inherited on E (gated on `!fieldsPruned`, and run before pruning).
|
|
1480
|
+
// 3. Reject a field that names itself, directly or through other fields of
|
|
1481
|
+
// E, declared or inherited, since the graph renders a field that names
|
|
1482
|
+
// a sibling from that sibling's SQL (see selfReferenceErrors).
|
|
1483
|
+
//
|
|
1484
|
+
// deploymentPlan also runs it on the unpruned model before `pruneUnavailable`
|
|
1485
|
+
// drops unbound fields and metrics on a graph push. When given,
|
|
1486
|
+
// `flaggedMetrics` collects the name of each metric it reports.
|
|
1487
|
+
function validateExpressionReferences(models, fieldsPruned = false, warnings = [], flaggedMetrics) {
|
|
1488
|
+
const errors = [];
|
|
1489
|
+
for (const { document, model } of models) {
|
|
1490
|
+
errors.push(...fieldNoSqlErrors(model, document));
|
|
1491
|
+
const fieldsByEntity = ifResolvable(() => declaredFields(model));
|
|
1492
|
+
if (!fieldsByEntity) {
|
|
1493
|
+
for (const metric of model.metrics ?? []) {
|
|
1494
|
+
const noSql = metricNoSqlError(metric, model, document);
|
|
1495
|
+
if (!noSql)
|
|
1496
|
+
continue;
|
|
1497
|
+
errors.push(noSql);
|
|
1498
|
+
flaggedMetrics?.add(metric.name);
|
|
1499
|
+
}
|
|
1500
|
+
continue;
|
|
1501
|
+
}
|
|
1502
|
+
const entityNames = [...fieldsByEntity.keys()];
|
|
1503
|
+
// Each entity's fields as the graph renders them (see selfReferenceErrors).
|
|
1504
|
+
const resolvedByName = new Map((ifResolvable(() => (0, resolve_inheritance_1.resolveInheritance)(model, undefined, { requalify: true })
|
|
1505
|
+
.model.entities) ??
|
|
1506
|
+
model.entities ?? [])
|
|
1507
|
+
.map(e => [e.name, e]));
|
|
1508
|
+
const impliedDialect = (0, resolve_profiles_1.sourcesDialect)(model);
|
|
1509
|
+
const addUnreadableWarning = (what, total, unread) => {
|
|
1510
|
+
// A caller that looked up no table's dialect reads no ANSI_SQL
|
|
1511
|
+
// text when every table needs the lookup, so it warns about none.
|
|
1512
|
+
const unreadDialects = impliedDialect === null ?
|
|
1513
|
+
unread.filter(d => d !== 'ANSI_SQL') :
|
|
1514
|
+
unread;
|
|
1515
|
+
if (!unreadDialects.length)
|
|
1516
|
+
return;
|
|
1517
|
+
if (unreadDialects.length === total) {
|
|
1518
|
+
warnings.push(`${what} in model '${model.name}' (${document}): kcmd could ` +
|
|
1519
|
+
`not read its SQL, so the fields it reads were not checked.`);
|
|
1520
|
+
}
|
|
1521
|
+
else {
|
|
1522
|
+
const listed = unreadDialects.map(d => `'${d}'`).join(', ');
|
|
1523
|
+
const entryText = unreadDialects.length === 1 ?
|
|
1524
|
+
'that entry reads' :
|
|
1525
|
+
'those entries read';
|
|
1526
|
+
warnings.push(`${what} in model '${model.name}' (${document}): kcmd could ` +
|
|
1527
|
+
`not read its ${listed} SQL, so the fields ${entryText} were ` +
|
|
1528
|
+
`not checked.`);
|
|
1529
|
+
}
|
|
1530
|
+
};
|
|
1531
|
+
for (const entity of model.entities ?? []) {
|
|
1532
|
+
const others = entityNames.filter(n => n !== entity.name);
|
|
1533
|
+
const knownFields = fieldsByEntity.get(entity.name);
|
|
1534
|
+
for (const field of entity.fields ?? []) {
|
|
1535
|
+
// A text with no SQL has nothing to read, so it gets no warning.
|
|
1536
|
+
const sources = (0, resolve_profiles_1.expressionSources)(field).filter(s => (0, loader_1.noSqlReason)(s.text, s.dialect, entity.name) === undefined);
|
|
1537
|
+
const allColumns = [];
|
|
1538
|
+
const unreadDialects = [];
|
|
1539
|
+
for (const { text, dialect } of sources) {
|
|
1540
|
+
const columns = (0, sql_expr_utils_1.columnsForSource)(text, dialect, impliedDialect);
|
|
1541
|
+
if (!columns) {
|
|
1542
|
+
unreadDialects.push(dialect);
|
|
1543
|
+
continue;
|
|
1544
|
+
}
|
|
1545
|
+
allColumns.push(...columns);
|
|
1546
|
+
}
|
|
1547
|
+
addUnreadableWarning(`field '${entity.name}.${field.name}'`, sources.length, unreadDialects);
|
|
1548
|
+
for (const other of others) {
|
|
1549
|
+
const refs = (0, sql_expr_utils_1.fieldsReadOn)(allColumns, other);
|
|
1550
|
+
if (refs.length) {
|
|
1551
|
+
errors.push(`field '${entity.name}.${field.name}' in model '${model.name}' (${document}) references entity '${other}' ('${other}.${refs[0]}'); a field expression may only reference its ` +
|
|
1552
|
+
`own entity '${entity.name}' or unqualified columns of its backing ` +
|
|
1553
|
+
`table.`);
|
|
1554
|
+
}
|
|
1555
|
+
}
|
|
1556
|
+
const where = `field '${entity.name}.${field.name}' in model '${model.name}' (${document})`;
|
|
1557
|
+
errors.push(...new Set(allColumns.flatMap(c => (0, names_1.qualifierCaseError)(where, c, entityNames) ?? [])));
|
|
1558
|
+
if (!fieldsPruned) {
|
|
1559
|
+
for (const ref of (0, sql_expr_utils_1.fieldsReadOn)(allColumns, entity.name)) {
|
|
1560
|
+
if (!knownFields.has(ref)) {
|
|
1561
|
+
errors.push(`field '${entity.name}.${field.name}' in model '${model.name}' (${document}) references '${entity.name}.${ref}', which is not a declared or ` +
|
|
1562
|
+
`inherited field of '${entity.name}'.`);
|
|
1563
|
+
}
|
|
1564
|
+
}
|
|
1565
|
+
}
|
|
1566
|
+
}
|
|
1567
|
+
errors.push(...selfReferenceErrors(entity, resolvedByName.get(entity.name) ?? entity, model, document, impliedDialect));
|
|
1568
|
+
}
|
|
1569
|
+
for (const metric of model.metrics ?? []) {
|
|
1570
|
+
const before = errors.length;
|
|
1571
|
+
const sources = (0, resolve_profiles_1.expressionSources)(metric);
|
|
1572
|
+
// A text with no SQL has nothing to read, so it gets no warning.
|
|
1573
|
+
const readable = sources.filter(s => (0, loader_1.noSqlReason)(s.text, s.dialect) === undefined);
|
|
1574
|
+
const allColumns = [];
|
|
1575
|
+
const unreadDialects = [];
|
|
1576
|
+
for (const { text, dialect } of readable) {
|
|
1577
|
+
const columns = (0, sql_expr_utils_1.columnsForSource)(text, dialect, impliedDialect);
|
|
1578
|
+
if (!columns) {
|
|
1579
|
+
unreadDialects.push(dialect);
|
|
1580
|
+
continue;
|
|
1581
|
+
}
|
|
1582
|
+
allColumns.push(...columns);
|
|
1583
|
+
}
|
|
1584
|
+
addUnreadableWarning(`metric '${metric.name}'`, readable.length, unreadDialects);
|
|
1585
|
+
const authored = metric.authoredEntity;
|
|
1586
|
+
const anchor = authored ?? metric.entity;
|
|
1587
|
+
const reached = (0, sql_expr_utils_1.entitiesReached)(allColumns, entityNames);
|
|
1588
|
+
const collision = metricFieldCollisionError(metric, anchor, fieldsByEntity, model, document);
|
|
1589
|
+
if (collision)
|
|
1590
|
+
errors.push(collision);
|
|
1591
|
+
// An anchor does not make up for missing SQL: the graph would drop the
|
|
1592
|
+
// metric.
|
|
1593
|
+
const noSql = metricNoSqlError(metric, model, document);
|
|
1594
|
+
if (noSql) {
|
|
1595
|
+
errors.push(noSql);
|
|
1596
|
+
}
|
|
1597
|
+
else if (!authored && unreadDialects.length === 0 && allColumns.length === 0 ||
|
|
1598
|
+
!authored && reached.length > 1) {
|
|
1599
|
+
const fixHint = model.version === '0.2.0.dev0' ?
|
|
1600
|
+
`set 'entity' in a GOOGLE custom_extensions block on the metric ` +
|
|
1601
|
+
`(or use version '0.2.0.dev0/google' and set 'entity' on the ` +
|
|
1602
|
+
`metric) or scope its expression to one entity.` :
|
|
1603
|
+
`set 'entity' on the metric or scope its expression to one entity.`;
|
|
1604
|
+
errors.push(`metric '${metric.name}' in model '${model.name}' (${document}) has no 'entity' anchor and its expression${reached.length === 0 ?
|
|
1605
|
+
' reaches no declared entity' :
|
|
1606
|
+
` reaches multiple entities (${reached.join(', ')})`}, ` +
|
|
1607
|
+
`so it does not resolve to a single entity; ${fixHint}`);
|
|
1608
|
+
}
|
|
1609
|
+
if (anchor !== undefined && !fieldsByEntity.has(anchor)) {
|
|
1610
|
+
errors.push(`metric '${metric.name}' in model '${model.name}' (${document}): entity '${anchor}' is not declared in the model.`);
|
|
1611
|
+
}
|
|
1612
|
+
if (authored !== undefined && fieldsByEntity.has(authored) &&
|
|
1613
|
+
reached.length > 0 && !reached.includes(authored)) {
|
|
1614
|
+
errors.push(`metric '${metric.name}' in model '${model.name}' (${document}) is anchored to entity '${authored}', but its expression only references ${reached.map(e => `'${e}'`).join(', ')}; the anchor ` +
|
|
1615
|
+
`contradicts the SQL.`);
|
|
1616
|
+
}
|
|
1617
|
+
if (!fieldsPruned) {
|
|
1618
|
+
for (const entityName of reached) {
|
|
1619
|
+
const knownFields = fieldsByEntity.get(entityName);
|
|
1620
|
+
for (const ref of (0, sql_expr_utils_1.fieldsReadOn)(allColumns, entityName)) {
|
|
1621
|
+
if (!knownFields.has(ref)) {
|
|
1622
|
+
errors.push(`metric '${metric.name}' in model '${model.name}' (${document}): expression references '${entityName}.${ref}', but entity '${entityName}' has no field '${ref}'; metric expressions must reference declared ` +
|
|
1623
|
+
`fields, not physical columns.`);
|
|
1624
|
+
}
|
|
1625
|
+
}
|
|
1626
|
+
}
|
|
1627
|
+
}
|
|
1628
|
+
const seenUnqualified = new Set();
|
|
1629
|
+
for (const col of allColumns) {
|
|
1630
|
+
if (col.qualifier !== undefined &&
|
|
1631
|
+
(fieldsByEntity.has(col.qualifier) || col.qualifier === anchor)) {
|
|
1632
|
+
continue;
|
|
1633
|
+
}
|
|
1634
|
+
const token = col.qualifier !== undefined ?
|
|
1635
|
+
`${col.qualifier}.${col.name}` :
|
|
1636
|
+
col.name;
|
|
1637
|
+
if (seenUnqualified.has(token))
|
|
1638
|
+
continue;
|
|
1639
|
+
seenUnqualified.add(token);
|
|
1640
|
+
if (col.qualifier === undefined) {
|
|
1641
|
+
errors.push(`metric '${metric.name}' in model '${model.name}' (${document}): '${token}' must be written as ` +
|
|
1642
|
+
`'entity.field'; a metric reads fields, never columns.`);
|
|
1643
|
+
}
|
|
1644
|
+
else {
|
|
1645
|
+
const where = `metric '${metric.name}' in model '${model.name}' (${document})`;
|
|
1646
|
+
errors.push((0, names_1.qualifierCaseError)(where, col, entityNames) ??
|
|
1647
|
+
`${where}: '${token}' has qualifier '${col.qualifier}', which ` +
|
|
1648
|
+
`is not a declared entity in model '${model.name}'; write ` +
|
|
1649
|
+
`'entity.field' starting with a declared entity.`);
|
|
1650
|
+
}
|
|
1651
|
+
}
|
|
1652
|
+
if (errors.length > before)
|
|
1653
|
+
flaggedMetrics?.add(metric.name);
|
|
1654
|
+
}
|
|
1655
|
+
}
|
|
1656
|
+
return errors;
|
|
1657
|
+
}
|
|
1658
|
+
// Every field each entity has, inherited and profile-excluded ones included so
|
|
1659
|
+
// an unpruned check before `pruneUnavailable` does not report a profile-excluded
|
|
1660
|
+
// inherited field as undeclared. Inheritance is resolved through the same pass
|
|
1661
|
+
// the graph legs use rather than by walking `extends` here, so the two can
|
|
1662
|
+
// never disagree about what a subtype has. The pass clones, so it is skipped
|
|
1663
|
+
// for a model that declares no inheritance.
|
|
1664
|
+
function declaredFields(model) {
|
|
1665
|
+
const inherits = (model.entities ?? []).some(e => e.extends?.length);
|
|
1666
|
+
const entities = (inherits ? (0, resolve_inheritance_1.resolveInheritance)(model).model : model).entities ?? [];
|
|
1667
|
+
return new Map(entities.map(e => [
|
|
1668
|
+
e.name,
|
|
1669
|
+
new Set([
|
|
1670
|
+
...(e.fields ?? []).map(f => f.name),
|
|
1671
|
+
...(e.excludedFields ?? []),
|
|
1672
|
+
]),
|
|
1673
|
+
]));
|
|
1674
|
+
}
|
|
1675
|
+
// The executor coordinate fields that are absent or blank. An executor with no
|
|
1676
|
+
// gaps yields an empty list.
|
|
1677
|
+
function missingExecutorFields(ex) {
|
|
1678
|
+
const blank = (s) => s.trim().length === 0;
|
|
1679
|
+
switch (ex.kind) {
|
|
1680
|
+
case 'mcp':
|
|
1681
|
+
return [['server', ex.mcp.server], ['tool', ex.mcp.tool]]
|
|
1682
|
+
.filter(([, v]) => blank(v))
|
|
1683
|
+
.map(([k]) => k);
|
|
1684
|
+
case 'rest':
|
|
1685
|
+
return [['endpoint', ex.rest.endpoint], ['method', ex.rest.method]]
|
|
1686
|
+
.filter(([, v]) => blank(v))
|
|
1687
|
+
.map(([k]) => k);
|
|
1688
|
+
case 'grpc':
|
|
1689
|
+
return [['service', ex.grpc.service], ['method', ex.grpc.method]]
|
|
1690
|
+
.filter(([, v]) => blank(v))
|
|
1691
|
+
.map(([k]) => k);
|
|
1692
|
+
case 'sql':
|
|
1693
|
+
return ex.sql.statements.every(blank) ? ['statements'] : [];
|
|
1694
|
+
}
|
|
1695
|
+
}
|
|
1696
|
+
// One error for each entity that `deployment` binds to a table outside the
|
|
1697
|
+
// database it deploys to. Sources come from the deployment's profile in
|
|
1698
|
+
// `model.profiles`, or from the model file for `default`.
|
|
1699
|
+
function deploymentTableErrors(document, model, deployment) {
|
|
1700
|
+
let sourceOf = (e) => e.authoredSource;
|
|
1701
|
+
let bindings = 'the model file';
|
|
1702
|
+
if (deployment.profile !== resolve_profiles_1.DEFAULT_PROFILE) {
|
|
1703
|
+
const profile = model.profiles?.find(p => p.name === deployment.profile);
|
|
1704
|
+
// `model.profiles` lacks a profile file that does not load, which push
|
|
1705
|
+
// reports instead.
|
|
1706
|
+
if (!profile)
|
|
1707
|
+
return [];
|
|
1708
|
+
const bound = new Map(profile.entities.map(pe => [pe.name, pe.source]));
|
|
1709
|
+
sourceOf = e => bound.get(e.name);
|
|
1710
|
+
bindings = `profile '${profile.name}'`;
|
|
1711
|
+
}
|
|
1712
|
+
const { database, target, outside } = targetDatabase(deployment);
|
|
1713
|
+
const errors = [];
|
|
1714
|
+
for (const e of model.entities ?? []) {
|
|
1715
|
+
const source = sourceOf(e)?.trim();
|
|
1716
|
+
if (!source || (0, loader_1.databaseOf)(source) === database)
|
|
1717
|
+
continue;
|
|
1718
|
+
errors.push(`[${document}] deployment '${deployment.name}' targets ${target}, ` +
|
|
1719
|
+
`but ${bindings} binds entity '${e.name}' to '${source}', ${outside}.`);
|
|
1720
|
+
}
|
|
1721
|
+
return errors;
|
|
1722
|
+
}
|
|
1723
|
+
// The error for a Spanner `deployment` on the model file's own bindings when
|
|
1724
|
+
// `model` declares metrics, which a Spanner graph cannot hold.
|
|
1725
|
+
function spannerMetricsErrors(document, model, deployment) {
|
|
1726
|
+
if (deployment.engine !== 'spanner' ||
|
|
1727
|
+
deployment.profile !== resolve_profiles_1.DEFAULT_PROFILE || !model.metrics?.length) {
|
|
1728
|
+
return [];
|
|
1729
|
+
}
|
|
1730
|
+
return [
|
|
1731
|
+
`[${document}] deployment '${deployment.name}' targets a Spanner graph ` +
|
|
1732
|
+
`and uses the model file's own bindings, but the model declares ` +
|
|
1733
|
+
`metrics, which a Spanner graph cannot hold; name a profile with ` +
|
|
1734
|
+
`metrics_exclude: "*".`,
|
|
1735
|
+
];
|
|
1736
|
+
}
|
|
1737
|
+
// The database the tables of a deployment target must be in, as databaseOf
|
|
1738
|
+
// gives it, and how an error names the target and a table outside it.
|
|
1739
|
+
function targetDatabase({ engine, coordinates: c }) {
|
|
1740
|
+
if (engine === 'bigquery') {
|
|
1741
|
+
return { database: 'bigquery', target: 'a BigQuery graph',
|
|
1742
|
+
outside: 'which is not a BigQuery or BigLake table' };
|
|
1743
|
+
}
|
|
1744
|
+
const [system, name] = engine === 'spanner' ?
|
|
1745
|
+
['Spanner', `${c.project}/${c.instance}/${c.database}`] :
|
|
1746
|
+
['AlloyDB', `${c.project}/${c.location}/${c.cluster}/${c.database}`];
|
|
1747
|
+
return { database: `${engine}/${name}`, target: `${system} database '${name}'`,
|
|
1748
|
+
outside: 'which is not a table in that database' };
|
|
1749
|
+
}
|
|
1750
|
+
// Whether Knowledge Catalog push and pull use the second-generation built-in
|
|
1751
|
+
// aspect fields and entry-id layout. They do unless `KC_V2_ASPECTS=0`; every
|
|
1752
|
+
// other value, including unset and `''`, is on. `KC_V2_ASPECTS=0` is a
|
|
1753
|
+
// temporary escape hatch to the first-generation path, removed with that path
|
|
1754
|
+
// (b/575695037).
|
|
1755
|
+
function kcV2Aspects() {
|
|
1756
|
+
return process.env.KC_V2_ASPECTS !== '0';
|
|
1757
|
+
}
|
|
1758
|
+
// Adds the profile files of the model in `base` to `model.profiles`, without
|
|
1759
|
+
// their actions. Returns the error of the first profile file that breaks a
|
|
1760
|
+
// file rule. A profile file that does not load is left out, for
|
|
1761
|
+
// namedProfileErrors to report.
|
|
1762
|
+
function withProfileFiles(base, profileDocs) {
|
|
1763
|
+
const { document, model } = base;
|
|
1764
|
+
const profiles = [...(model.profiles ?? [])];
|
|
1765
|
+
let files;
|
|
1766
|
+
try {
|
|
1767
|
+
files = profileDocs(document);
|
|
1768
|
+
}
|
|
1769
|
+
catch (err) {
|
|
1770
|
+
return { errors: [{ error: err.message || String(err), document }] };
|
|
1771
|
+
}
|
|
1772
|
+
const reserved = (0, resolve_profiles_1.reservedProfileNameError)(document, files);
|
|
1773
|
+
if (reserved)
|
|
1774
|
+
return { errors: [{ error: reserved, document }] };
|
|
1775
|
+
for (const p of files) {
|
|
1776
|
+
try {
|
|
1777
|
+
profiles.push((0, resolve_profiles_1.loadProfileFile)(p.text, p.name, { skipActions: true }));
|
|
1778
|
+
}
|
|
1779
|
+
catch (err) {
|
|
1780
|
+
if (!(err instanceof loader_1.SemanticModelError))
|
|
1781
|
+
throw err;
|
|
1782
|
+
}
|
|
1783
|
+
}
|
|
1784
|
+
return { document, model: profiles.length ? { ...model, profiles } : model };
|
|
1785
|
+
}
|
|
1786
|
+
// Runs push's offline checks that follow withProfileFiles and the UTF-8 and
|
|
1787
|
+
// sidecar checks, through the graph generation. Returns the errors of the
|
|
1788
|
+
// first stage that finds any, or else a plan for each deployment `selected`
|
|
1789
|
+
// names, or for every deployment when it is unset.
|
|
1790
|
+
async function localChecks(scope, warn, selected) {
|
|
1791
|
+
const { docs, loaded, models, profileDocs, declaredBy, load } = scope;
|
|
1792
|
+
const stop = (errors) => ({ errors });
|
|
1793
|
+
const ofModel = (document, ...errors) => errors.map(error => ({ error, document }));
|
|
1794
|
+
const ofProfile = (document, profile, ...errors) => errors.map(error => ({ error, document, profile }));
|
|
1795
|
+
const declaredNames = (document) => declaredBy(docs.find(d => d.name === document)).map(p => p.name);
|
|
1796
|
+
// Every deployment's target must be in an accepted form, and its profile one
|
|
1797
|
+
// the model declares, whether or not this push selects the deployment.
|
|
1798
|
+
const selectedByDoc = new Map();
|
|
1799
|
+
const everyByDoc = new Map();
|
|
1800
|
+
for (const { document, model } of models) {
|
|
1801
|
+
const every = (0, deployment_target_1.resolveModelDeployments)(model);
|
|
1802
|
+
if ('error' in every)
|
|
1803
|
+
return stop(ofModel(document, every.error));
|
|
1804
|
+
const resolved = (0, deployment_target_1.resolveModelDeployments)(model, selected);
|
|
1805
|
+
if ('error' in resolved)
|
|
1806
|
+
return stop(ofModel(document, resolved.error));
|
|
1807
|
+
const undeclared = (0, resolve_profiles_1.undeclaredProfileError)(document, model, declaredNames(document));
|
|
1808
|
+
if (undeclared)
|
|
1809
|
+
return stop(ofModel(document, undeclared));
|
|
1810
|
+
everyByDoc.set(document, every.deployments);
|
|
1811
|
+
selectedByDoc.set(document, resolved.deployments);
|
|
1812
|
+
}
|
|
1813
|
+
// Every named profile is checked on its own against the model file, and so
|
|
1814
|
+
// are the model file's own bindings and each deployment. The catalog's model
|
|
1815
|
+
// is checked as a model that deploys nowhere. A selected deployment's own
|
|
1816
|
+
// checks run only after these pass, so these give every error and every
|
|
1817
|
+
// warning about SQL kcmd cannot read.
|
|
1818
|
+
const profileWarnings = [];
|
|
1819
|
+
const profileErrors = (0, resolve_profiles_1.namedProfileErrors)(docs, new Map(loaded.models.map(m => [m.document, m.model])), profileDocs, declaredBy, profileWarnings, load);
|
|
1820
|
+
for (const { document, model } of models) {
|
|
1821
|
+
profileErrors.push(...ofProfile(document, resolve_profiles_1.DEFAULT_PROFILE, ...validateInlineDialects(model, document), ...validateInheritedColumns(model, document)),
|
|
1822
|
+
// One call per profile, so each error carries its deployments' profile.
|
|
1823
|
+
...[...new Set(everyByDoc.get(document).map(d => d.profile))].flatMap(profile => ofProfile(document, profile, ...validateDeployments({
|
|
1824
|
+
...model,
|
|
1825
|
+
deployments: model.deployments.filter(d => (d.profile ?? resolve_profiles_1.DEFAULT_PROFILE) === profile),
|
|
1826
|
+
}, document, load))));
|
|
1827
|
+
const doc = docs.find(d => d.name === document);
|
|
1828
|
+
const declared = declaredBy(doc);
|
|
1829
|
+
const named = declared.some(p => p.name.toLowerCase() !== resolve_profiles_1.DEFAULT_PROFILE);
|
|
1830
|
+
profileWarnings.push(...inlineBindingWarnings(model, document, named));
|
|
1831
|
+
// A profile that no selected deployment applies still goes to Knowledge
|
|
1832
|
+
// Catalog, so its load warnings print here, less those the model file's
|
|
1833
|
+
// load gave, unless a check above recorded an error for that profile. A
|
|
1834
|
+
// selected deployment prints its own profile's warnings.
|
|
1835
|
+
const applied = selectedByDoc.get(document).map(d => d.profile);
|
|
1836
|
+
for (const { name } of declared) {
|
|
1837
|
+
if (applied.includes(name))
|
|
1838
|
+
continue;
|
|
1839
|
+
if (profileErrors.some(e => e.document === document && e.profile === name)) {
|
|
1840
|
+
continue;
|
|
1841
|
+
}
|
|
1842
|
+
profileWarnings.push(...(0, resolve_profiles_1.appliedProfileWarnings)(doc, declared, name, loaded.warnings, load));
|
|
1843
|
+
}
|
|
1844
|
+
}
|
|
1845
|
+
const kcWarnings = [];
|
|
1846
|
+
const kcErrors = models.flatMap(m => ofModel(m.document, ...validatePushRequirements([{ ...m, model: { ...m.model, deployments: [] } }], {
|
|
1847
|
+
warnings: kcWarnings,
|
|
1848
|
+
v2Aspects: kcV2Aspects(),
|
|
1849
|
+
})));
|
|
1850
|
+
for (const w of [...profileWarnings, ...kcWarnings])
|
|
1851
|
+
warn(w);
|
|
1852
|
+
if (profileErrors.length || kcErrors.length) {
|
|
1853
|
+
return stop([...profileErrors, ...kcErrors]);
|
|
1854
|
+
}
|
|
1855
|
+
// Each deployment's tables must be in the database it deploys to, whether
|
|
1856
|
+
// or not this push selects the deployment.
|
|
1857
|
+
const tableErrors = models.flatMap(({ document, model }) => (everyByDoc.get(document) ?? []).flatMap(d => ofProfile(document, d.profile, ...deploymentTableErrors(document, model, d))));
|
|
1858
|
+
if (tableErrors.length)
|
|
1859
|
+
return stop(tableErrors);
|
|
1860
|
+
// Spanner creates no property graph in a PostgreSQL-dialect database, so a
|
|
1861
|
+
// selected Spanner deployment there is refused by the dialect already read.
|
|
1862
|
+
const dialectErrors = models.flatMap(({ document }) => (selectedByDoc.get(document) ?? []).flatMap(d => {
|
|
1863
|
+
if (d.engine !== 'spanner')
|
|
1864
|
+
return [];
|
|
1865
|
+
const { project, instance, database } = d.coordinates;
|
|
1866
|
+
const key = `spanner.googleapis.com/projects/${project}/instances/${instance}/databases/${database}`;
|
|
1867
|
+
if (load.sourceDialects?.get(key) !== 'POSTGRES')
|
|
1868
|
+
return [];
|
|
1869
|
+
return ofProfile(document, d.profile, `[${document}] deployment '${d.name}': database ${project}/${instance}/${database} has the PostgreSQL dialect, and ` +
|
|
1870
|
+
`Spanner creates no property graph in a PostgreSQL-dialect ` +
|
|
1871
|
+
`database. Remove the deployment; push still publishes the ` +
|
|
1872
|
+
`model to Knowledge Catalog.`);
|
|
1873
|
+
}));
|
|
1874
|
+
if (dialectErrors.length)
|
|
1875
|
+
return stop(dialectErrors);
|
|
1876
|
+
const plans = [];
|
|
1877
|
+
for (const base of models) {
|
|
1878
|
+
for (const deployment of selectedByDoc.get(base.document) ?? []) {
|
|
1879
|
+
const plan = await deploymentPlan(base, profileDocs(base.document), deployment, load, warn);
|
|
1880
|
+
if ('errors' in plan) {
|
|
1881
|
+
return stop(ofProfile(base.document, deployment.profile, ...plan.errors));
|
|
1882
|
+
}
|
|
1883
|
+
plans.push(plan);
|
|
1884
|
+
}
|
|
1885
|
+
}
|
|
1886
|
+
return { plans };
|
|
1887
|
+
}
|
|
1888
|
+
// Builds and checks the model `deployment` deploys from `base`, reading its
|
|
1889
|
+
// profile file in `files` again with its actions, and generates its DDL
|
|
1890
|
+
// offline. The profile applies with the source dialects in `load`. Returns
|
|
1891
|
+
// the plan, or the errors that stop the push.
|
|
1892
|
+
async function deploymentPlan(base, files, deployment, load, warn) {
|
|
1893
|
+
const { document, model } = base;
|
|
1894
|
+
const at = `[${document}] deployment '${deployment.name}'`;
|
|
1895
|
+
const fail = (...errors) => ({ errors });
|
|
1896
|
+
const spanner = spannerMetricsErrors(document, model, deployment);
|
|
1897
|
+
if (spanner.length)
|
|
1898
|
+
return fail(...spanner);
|
|
1899
|
+
let applied = model;
|
|
1900
|
+
if (deployment.profile !== resolve_profiles_1.DEFAULT_PROFILE) {
|
|
1901
|
+
// localChecks has checked that the model declares the profile and that
|
|
1902
|
+
// its file loads, so `model.profiles` holds it.
|
|
1903
|
+
let spec = model.profiles.find(p => p.name === deployment.profile);
|
|
1904
|
+
const file = files.find(p => p.name === deployment.profile);
|
|
1905
|
+
const warnings = [];
|
|
1906
|
+
try {
|
|
1907
|
+
// A profile in a GOOGLE block has no file and applies as loaded.
|
|
1908
|
+
if (file)
|
|
1909
|
+
spec = (0, resolve_profiles_1.loadProfileFile)(file.text, file.name);
|
|
1910
|
+
}
|
|
1911
|
+
catch (err) {
|
|
1912
|
+
if (!(err instanceof loader_1.SemanticModelError))
|
|
1913
|
+
throw err;
|
|
1914
|
+
return fail(`[${file.fileName}] ${err.message}`);
|
|
1915
|
+
}
|
|
1916
|
+
try {
|
|
1917
|
+
applied = (0, resolve_profiles_1.applyProfile)(model, spec, { load, warnings });
|
|
1918
|
+
}
|
|
1919
|
+
catch (err) {
|
|
1920
|
+
if (!(err instanceof loader_1.SemanticModelError))
|
|
1921
|
+
throw err;
|
|
1922
|
+
return fail(`${at}: ${err.message}`);
|
|
1923
|
+
}
|
|
1924
|
+
for (const w of warnings) {
|
|
1925
|
+
warn(`profile '${deployment.profile}': [${document}] ${w}`);
|
|
1926
|
+
}
|
|
1927
|
+
}
|
|
1928
|
+
// Messages name the deployment.
|
|
1929
|
+
const label = `${document}, deployment '${deployment.name}'`;
|
|
1930
|
+
const checked = { document: label, model: applied };
|
|
1931
|
+
// Pruning can remove the field these rules are about, so they run first.
|
|
1932
|
+
// localChecks has already warned about the SQL they cannot read.
|
|
1933
|
+
const unprunedErrors = [
|
|
1934
|
+
...validateInheritance([checked]),
|
|
1935
|
+
...validateExpressionReferences([checked]),
|
|
1936
|
+
];
|
|
1937
|
+
const { model: pruned, report } = (0, resolve_profiles_1.pruneUnavailable)(checked.model, deployment.profile);
|
|
1938
|
+
// Keeps only this deployment, so every reader sees its target alone.
|
|
1939
|
+
const narrowed = {
|
|
1940
|
+
document: checked.document,
|
|
1941
|
+
model: {
|
|
1942
|
+
...pruned,
|
|
1943
|
+
deployments: (model.deployments ?? []).filter(d => d.name === deployment.name),
|
|
1944
|
+
},
|
|
1945
|
+
};
|
|
1946
|
+
const errors = [
|
|
1947
|
+
...unprunedErrors,
|
|
1948
|
+
...validatePushRequirements([narrowed], {
|
|
1949
|
+
fieldsPruned: true,
|
|
1950
|
+
v2Aspects: kcV2Aspects(),
|
|
1951
|
+
}),
|
|
1952
|
+
];
|
|
1953
|
+
if (errors.length)
|
|
1954
|
+
return { errors };
|
|
1955
|
+
// An AlloyDB database holds no graph.
|
|
1956
|
+
if (deployment.engine === 'alloydb') {
|
|
1957
|
+
return { deployment, at, model: narrowed, report };
|
|
1958
|
+
}
|
|
1959
|
+
const generated = await generateOffline(deployment.engine === 'bigquery' ? deploy_bigquery_1.deployBigQuery : deploy_spanner_1.deploySpanner, [narrowed]);
|
|
1960
|
+
if (typeof generated === 'string')
|
|
1961
|
+
return fail(`${at}: ${generated}`);
|
|
1962
|
+
return { deployment, at, model: narrowed, report, generated };
|
|
1963
|
+
}
|
|
1964
|
+
// The warnings and the note push prints on what pruning leaves out of the
|
|
1965
|
+
// model `plan` deploys. Each line names the model, and the document and the
|
|
1966
|
+
// deployment.
|
|
1967
|
+
function pruningLines({ deployment, model, report: r }) {
|
|
1968
|
+
const where = `model '${model.model.name}' (${model.document})`;
|
|
1969
|
+
// What becomes of a field left out. An AlloyDB database holds no graph.
|
|
1970
|
+
const outcome = deployment.engine === 'alloydb' ? 'it is unavailable there' :
|
|
1971
|
+
'the graph omits it';
|
|
1972
|
+
const excluded = new Set(r.excludedFields);
|
|
1973
|
+
const omits = (field, why) => `Warning: field '${field}' in ${where} ${why}, so ${outcome}.`;
|
|
1974
|
+
// Such as "metric 'a'" or "metrics 'a' and 'b'", or '' for no names.
|
|
1975
|
+
const named = (kind, names) => names.length ?
|
|
1976
|
+
`${(0, loader_1.plural)(names.length, kind, `${kind}s`)} ${(0, loader_1.listed)(names.map(n => `'${n}'`))}` :
|
|
1977
|
+
'';
|
|
1978
|
+
const unavailable = [
|
|
1979
|
+
named('metric', r.droppedMetrics.map(m => m.name)),
|
|
1980
|
+
named('action', r.droppedActions.map(a => a.name)),
|
|
1981
|
+
].filter(n => n).join(r.droppedMetrics.length > 1 ? ', and ' : ' and ');
|
|
1982
|
+
const note = [
|
|
1983
|
+
excluded.size ? `excludes ${named('field', [...excluded])}.` : '',
|
|
1984
|
+
unavailable && `leaves ${unavailable} unavailable.`,
|
|
1985
|
+
].filter(p => p).join(' It ');
|
|
1986
|
+
return [
|
|
1987
|
+
...r.unboundFields.filter(f => !excluded.has(f))
|
|
1988
|
+
.map(f => omits(f, 'has no expression')),
|
|
1989
|
+
...r.droppedFields.map(d => omits(d.name, d.reason)),
|
|
1990
|
+
...note ? [`Note: ${where} ${note}`] : [],
|
|
1991
|
+
];
|
|
1992
|
+
}
|
|
1993
|
+
// A validate-only deploy builds the DDL and calls no service, so it needs no
|
|
1994
|
+
// project or token.
|
|
1995
|
+
const NO_SERVICE = new context_1.ApiContext('', '', '');
|
|
1996
|
+
// Generates a graph leg's DDL without calling a service, or gives why it
|
|
1997
|
+
// cannot be generated.
|
|
1998
|
+
async function generateOffline(deployLeg, models) {
|
|
1999
|
+
try {
|
|
2000
|
+
const result = await deployLeg(models, NO_SERVICE, { validateOnly: true });
|
|
2001
|
+
if (result.success)
|
|
2002
|
+
return result;
|
|
2003
|
+
return result.details ?? 'the graph could not be generated';
|
|
2004
|
+
}
|
|
2005
|
+
catch (err) {
|
|
2006
|
+
return err.message || String(err);
|
|
2007
|
+
}
|
|
2008
|
+
}
|
|
2009
|
+
// Live pre-flight over the BigQuery-targeting models: confirms every entity's
|
|
2010
|
+
// BigQuery source table is reachable BEFORE any destination leg runs, so a push
|
|
2011
|
+
// fails fast when the model could not deploy, rather than surfacing a missing
|
|
2012
|
+
// table only once the BigQuery leg executes its DDL. The caller passes only the
|
|
2013
|
+
// models whose deployment target is a BigQuery Graph; a Spanner-targeting
|
|
2014
|
+
// model's sources are Spanner tables (probed against a different system) and
|
|
2015
|
+
// are not checked here.
|
|
2016
|
+
//
|
|
2017
|
+
// Each distinct source is probed with a dry-run query (`SELECT 1 FROM <ref>`,
|
|
2018
|
+
// suffixed `WHERE FALSE` so it scans no data),
|
|
2019
|
+
// so BigQuery resolves the reference exactly as the generated DDL will. That
|
|
2020
|
+
// covers every reference form the generator emits -- a three-part
|
|
2021
|
+
// `project.dataset.table`, a four-part federated REST-catalog / Lakehouse name
|
|
2022
|
+
// (e.g. an Apache Iceberg table via BigLake), and quoted identifiers -- rather
|
|
2023
|
+
// than only a three-part name. A source the loader kept verbatim because it is
|
|
2024
|
+
// a query (contains whitespace) is not a table and is skipped. The dry-run is
|
|
2025
|
+
// billed to the model's BigQuery deployment-target project (the same project
|
|
2026
|
+
// the deploy runs against), falling back to `defaultProject`. Each distinct
|
|
2027
|
+
// (billing project, reference) pair is probed once. Returns one message per
|
|
2028
|
+
// unreachable table (empty when all pass).
|
|
2029
|
+
//
|
|
2030
|
+
// With `graph`, each probe runs in `graph.location`, the location of dataset
|
|
2031
|
+
// `graph.dataset`, and a probe that fails there runs again with no location.
|
|
2032
|
+
// When that run finds the table in another location, the error says so,
|
|
2033
|
+
// starting with `graph.at`, which names the deployment.
|
|
2034
|
+
async function validateBigQueryDataSources(models, bq, defaultProject, graph) {
|
|
2035
|
+
// Dedup by billing project + reference so a table shared across
|
|
2036
|
+
// entities/models is probed once; keep the first reference for a locatable
|
|
2037
|
+
// error message.
|
|
2038
|
+
const refs = new Map();
|
|
2039
|
+
for (const { document, model } of models) {
|
|
2040
|
+
const project = billingProject(model, defaultProject);
|
|
2041
|
+
for (const entity of model.entities ?? []) {
|
|
2042
|
+
const ref = probeableRef(entity.dataSource);
|
|
2043
|
+
if (!ref)
|
|
2044
|
+
continue;
|
|
2045
|
+
const key = `${project}\u0000${ref}`;
|
|
2046
|
+
if (!refs.has(key)) {
|
|
2047
|
+
refs.set(key, {
|
|
2048
|
+
project,
|
|
2049
|
+
ref,
|
|
2050
|
+
document,
|
|
2051
|
+
model: model.name,
|
|
2052
|
+
entity: entity.name,
|
|
2053
|
+
});
|
|
2054
|
+
}
|
|
2055
|
+
}
|
|
2056
|
+
}
|
|
2057
|
+
const errors = [];
|
|
2058
|
+
for (const { project, ref, document, model, entity } of refs.values()) {
|
|
2059
|
+
const sql = `SELECT 1 FROM \`${ref}\` WHERE FALSE`;
|
|
2060
|
+
const res = await bq.query(project, sql, graph?.location, true);
|
|
2061
|
+
if (res.status === 200)
|
|
2062
|
+
continue;
|
|
2063
|
+
if (graph) {
|
|
2064
|
+
const anywhere = await bq.query(project, sql, undefined, true);
|
|
2065
|
+
const found = anywhere.result?.jobReference?.location;
|
|
2066
|
+
if (found && found !== graph.location) {
|
|
2067
|
+
errors.push(`${graph.at} targets a BigQuery graph in dataset ` +
|
|
2068
|
+
`'${graph.dataset}', which is in location '${graph.location}', ` +
|
|
2069
|
+
`but entity '${entity}' reads table '${ref}', which is in ` +
|
|
2070
|
+
`location '${found}'. A property graph can use only tables in ` +
|
|
2071
|
+
`its dataset's location.`);
|
|
2072
|
+
continue;
|
|
2073
|
+
}
|
|
2074
|
+
}
|
|
2075
|
+
const msg = res.message?.trim() || `HTTP ${res.status}`;
|
|
2076
|
+
const why = /not found/i.test(msg) ?
|
|
2077
|
+
'does not exist' :
|
|
2078
|
+
/access denied|permission denied|not authorized|does not have permission/i
|
|
2079
|
+
.test(msg) ?
|
|
2080
|
+
'is not accessible (permission denied)' :
|
|
2081
|
+
`could not be verified (${msg})`;
|
|
2082
|
+
errors.push(`entity '${entity}' in model '${model}' (${document}) references ` +
|
|
2083
|
+
`BigQuery table '${ref}', which ${why}; the model cannot be deployed. ` +
|
|
2084
|
+
`Create the table or grant access to it, or fix the entity's source.`);
|
|
2085
|
+
}
|
|
2086
|
+
return errors;
|
|
2087
|
+
}
|
|
2088
|
+
// The BigQuery project a model's deploy -- and thus its dry-run pre-flight --
|
|
2089
|
+
// bills to: the project of the model's first BigQuery Graph deployment target
|
|
2090
|
+
// (where the CREATE PROPERTY GRAPH runs), falling back to the scope's default
|
|
2091
|
+
// project when the model declares no parseable BigQuery Graph target.
|
|
2092
|
+
// googleDeploymentTargets is safe here: validatePushRequirements ran first and
|
|
2093
|
+
// already rejected a malformed GOOGLE extension.
|
|
2094
|
+
function billingProject(model, defaultProject) {
|
|
2095
|
+
try {
|
|
2096
|
+
return (0, deploy_bigquery_1.googleDeploymentTargets)(model).bigQuery[0]?.project ??
|
|
2097
|
+
defaultProject;
|
|
2098
|
+
}
|
|
2099
|
+
catch {
|
|
2100
|
+
return defaultProject;
|
|
2101
|
+
}
|
|
2102
|
+
}
|
|
2103
|
+
// A source that can be probed as a BigQuery table: the canonical `dataSource`,
|
|
2104
|
+
// trimmed, or null when it is empty or a non-BigQuery resource URI. Unlike a
|
|
2105
|
+
// tables.get probe this imposes no part-count limit, so a three-part
|
|
2106
|
+
// `project.dataset.table` and a four-part BigLake / REST-catalog name are both
|
|
2107
|
+
// returned for the dry-run to resolve.
|
|
2108
|
+
function probeableRef(dataSource) {
|
|
2109
|
+
const trimmed = (dataSource ?? '').trim();
|
|
2110
|
+
if (!trimmed)
|
|
2111
|
+
return null;
|
|
2112
|
+
// A non-BigQuery resource URI (Spanner/AlloyDB/iceberg/...) is not a
|
|
2113
|
+
// BigQuery table, so the BigQuery pre-flight does not probe it. (BigQuery
|
|
2114
|
+
// and BigLake source URIs are normalized to dotted names by the loader, so a
|
|
2115
|
+
// URI reaching here is non-BigQuery.)
|
|
2116
|
+
if (trimmed.startsWith('//') || /^[a-z][\w+.-]*:\/\//i.test(trimmed)) {
|
|
2117
|
+
return null;
|
|
2118
|
+
}
|
|
2119
|
+
return trimmed;
|
|
2120
|
+
}
|
|
2121
|
+
// Every `sql` executor statement in a model, in declaration order. A blank one
|
|
2122
|
+
// is skipped: sqlExecutorErrors has already failed the push over it, and
|
|
2123
|
+
// sending it would earn a second, less useful complaint from the store.
|
|
2124
|
+
function pendingStatements({ document, model }) {
|
|
2125
|
+
const pending = [];
|
|
2126
|
+
for (const action of model.actions ?? []) {
|
|
2127
|
+
if (action.executor?.kind !== 'sql')
|
|
2128
|
+
continue;
|
|
2129
|
+
const executor = action.executor;
|
|
2130
|
+
executor.sql.statements.forEach((sql, index) => {
|
|
2131
|
+
if (!sql.trim())
|
|
2132
|
+
return;
|
|
2133
|
+
pending.push({ document, model: model.name, action, index, sql });
|
|
2134
|
+
});
|
|
2135
|
+
}
|
|
2136
|
+
return pending;
|
|
2137
|
+
}
|
|
2138
|
+
// The parameter types to send with one statement's pre-flight request, keyed by
|
|
2139
|
+
// `@name`.
|
|
2140
|
+
//
|
|
2141
|
+
// Only the parameters the statement actually references are sent, and only
|
|
2142
|
+
// those whose declared type maps to a store type we can state. An `Opaque` type
|
|
2143
|
+
// is exactly the case we cannot: it means the model does not know the type, so
|
|
2144
|
+
// asserting one would invent a constraint the author never wrote. Those are
|
|
2145
|
+
// left out and the store resolves them from context, which both backends do.
|
|
2146
|
+
//
|
|
2147
|
+
// Sending the types at all is optional -- a statement with no parameter types
|
|
2148
|
+
// passes either backend's dry run. It is worth doing because it turns the
|
|
2149
|
+
// action's DECLARED parameter types into a claim checked against the real
|
|
2150
|
+
// column types, so a model that says `Float` where the column is a STRING fails
|
|
2151
|
+
// here rather than on the first call.
|
|
2152
|
+
function statementParameterTypes(action, sql) {
|
|
2153
|
+
const declared = new Map(action.parameters.map(p => [p.name, p.type]));
|
|
2154
|
+
const types = {};
|
|
2155
|
+
for (const name of (0, sql_identifiers_1.referencedParameters)(sql)) {
|
|
2156
|
+
const type = declared.get(name);
|
|
2157
|
+
if (!type || type === 'Opaque')
|
|
2158
|
+
continue;
|
|
2159
|
+
types[name] = (0, parameters_1.storeCodeFor)(type);
|
|
2160
|
+
}
|
|
2161
|
+
return types;
|
|
2162
|
+
}
|
|
2163
|
+
// The message for a statement the store refused. `store` names the database so
|
|
2164
|
+
// a multi-profile push says which one answered.
|
|
2165
|
+
function rejectedStatement(pending, store, detail) {
|
|
2166
|
+
return `action '${pending.action.name}' in model '${pending.model}' (${pending.document}): statement ${pending.index + 1} of its sql executor was rejected by ${store}: ${detail} Statements reach the store exactly as written, so every ` +
|
|
2167
|
+
`table and column has to be the name the database uses.`;
|
|
2168
|
+
}
|
|
2169
|
+
// The Spanner arm. Each distinct database gets one session, reused across every
|
|
2170
|
+
// statement bound to it and deleted at the end -- which also discards the
|
|
2171
|
+
// read-write transactions PLAN had to begin and never committed. Returns one
|
|
2172
|
+
// message per statement that failed to plan, empty when all pass.
|
|
2173
|
+
async function validateSpannerActionStatements(models, spanner) {
|
|
2174
|
+
// Group by database so a model set sharing one target opens one session
|
|
2175
|
+
// rather than one per statement.
|
|
2176
|
+
const byDatabase = new Map();
|
|
2177
|
+
for (const loaded of models) {
|
|
2178
|
+
const statements = pendingStatements(loaded);
|
|
2179
|
+
if (!statements.length)
|
|
2180
|
+
continue;
|
|
2181
|
+
// The database an action writes to is the one this profile deploys its
|
|
2182
|
+
// graph to: on Spanner the published graph and the operational rows live in
|
|
2183
|
+
// the same database. A model with no Spanner target contributes nothing.
|
|
2184
|
+
let target;
|
|
2185
|
+
try {
|
|
2186
|
+
target = (0, deployment_target_1.spannerGraphTargets)(loaded.model).targets[0];
|
|
2187
|
+
}
|
|
2188
|
+
catch {
|
|
2189
|
+
continue;
|
|
2190
|
+
}
|
|
2191
|
+
if (!target)
|
|
2192
|
+
continue;
|
|
2193
|
+
const key = `${target.project}/${target.instance}/${target.database}`;
|
|
2194
|
+
const entry = byDatabase.get(key) ?? { target, statements: [] };
|
|
2195
|
+
entry.statements.push(...statements);
|
|
2196
|
+
byDatabase.set(key, entry);
|
|
2197
|
+
}
|
|
2198
|
+
const errors = [];
|
|
2199
|
+
for (const { target, statements } of byDatabase.values()) {
|
|
2200
|
+
const store = `Spanner database '${target.project}/${target.instance}/${target.database}'`;
|
|
2201
|
+
const opened = await spanner.createSession(target.project, target.instance, target.database);
|
|
2202
|
+
const session = opened.result?.name;
|
|
2203
|
+
if (!session) {
|
|
2204
|
+
// Reaching the database is the whole check, so failing to open a session
|
|
2205
|
+
// is reported rather than skipped: a push that could not verify its
|
|
2206
|
+
// statements must not proceed as though it had.
|
|
2207
|
+
errors.push(`could not open a session on ${store} to verify the sql executor ` +
|
|
2208
|
+
`statements in model '${statements[0].model}' (${statements[0].document}): ${(0, parameters_1.storeMessage)(opened.message, opened.status)}`);
|
|
2209
|
+
continue;
|
|
2210
|
+
}
|
|
2211
|
+
try {
|
|
2212
|
+
for (const pending of statements) {
|
|
2213
|
+
const types = statementParameterTypes(pending.action, pending.sql);
|
|
2214
|
+
const params = {};
|
|
2215
|
+
const paramTypes = {};
|
|
2216
|
+
for (const [name, code] of Object.entries(types)) {
|
|
2217
|
+
params[name] = null;
|
|
2218
|
+
paramTypes[name] = { code };
|
|
2219
|
+
}
|
|
2220
|
+
const res = await spanner.planDml(session, pending.sql, params, paramTypes);
|
|
2221
|
+
if (res.status === 200)
|
|
2222
|
+
continue;
|
|
2223
|
+
errors.push(rejectedStatement(pending, store, (0, parameters_1.storeMessage)(res.message, res.status)));
|
|
2224
|
+
}
|
|
2225
|
+
}
|
|
2226
|
+
finally {
|
|
2227
|
+
// Best effort, and the catch is what makes it one. A leaked session
|
|
2228
|
+
// expires on its own, while letting a cleanup failure escape would throw
|
|
2229
|
+
// away every statement error collected above and report a transport
|
|
2230
|
+
// problem as though the statements had never been checked.
|
|
2231
|
+
try {
|
|
2232
|
+
await spanner.deleteSession(session);
|
|
2233
|
+
}
|
|
2234
|
+
catch {
|
|
2235
|
+
// Nothing to say: the session is the store's to reclaim.
|
|
2236
|
+
}
|
|
2237
|
+
}
|
|
2238
|
+
}
|
|
2239
|
+
return errors;
|
|
2240
|
+
}
|
|
2241
|
+
// The BigQuery arm. Each statement is dry-run on jobs.query, billed to the same
|
|
2242
|
+
// project the model's graph deploys to (a dry run scans nothing, but it still
|
|
2243
|
+
// needs a project to run in). There is no session to manage: a dry run is a
|
|
2244
|
+
// single stateless call. Returns one message per statement BigQuery refused.
|
|
2245
|
+
//
|
|
2246
|
+
// Note what this does NOT settle. BigQuery accepts DML against a native or
|
|
2247
|
+
// managed table and refuses it against most external ones, and a dry run
|
|
2248
|
+
// reports that refusal -- which is the point, because a real run would hit the
|
|
2249
|
+
// same wall.
|
|
2250
|
+
async function validateBigQueryActionStatements(models, bq, defaultProject) {
|
|
2251
|
+
const errors = [];
|
|
2252
|
+
for (const loaded of models) {
|
|
2253
|
+
const statements = pendingStatements(loaded);
|
|
2254
|
+
if (!statements.length)
|
|
2255
|
+
continue;
|
|
2256
|
+
const project = billingProject(loaded.model, defaultProject);
|
|
2257
|
+
const store = `BigQuery (project '${project}')`;
|
|
2258
|
+
for (const pending of statements) {
|
|
2259
|
+
const res = await bq.query(project, pending.sql, undefined, true, statementParameterTypes(pending.action, pending.sql));
|
|
2260
|
+
if (res.status === 200)
|
|
2261
|
+
continue;
|
|
2262
|
+
errors.push(rejectedStatement(pending, store, (0, parameters_1.storeMessage)(res.message, res.status)));
|
|
2263
|
+
}
|
|
2264
|
+
}
|
|
2265
|
+
return errors;
|
|
2266
|
+
}
|