@google-cloud/kcmd 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +203 -0
- package/README.md +264 -2
- package/build/ts/kcmd/cli.js +20252 -0
- package/build/ts/kcmd/gcp/alloydb.d.ts +12 -0
- package/build/ts/kcmd/gcp/alloydb.js +70 -0
- package/build/ts/kcmd/gcp/api.d.ts +29 -0
- package/build/ts/kcmd/gcp/api.js +162 -0
- package/build/ts/kcmd/gcp/biglake.d.ts +18 -0
- package/build/ts/kcmd/gcp/biglake.js +59 -0
- package/build/ts/kcmd/gcp/bigquery.d.ts +57 -0
- package/build/ts/kcmd/gcp/bigquery.js +134 -0
- package/build/ts/kcmd/gcp/context.d.ts +10 -0
- package/build/ts/kcmd/gcp/context.js +74 -0
- package/build/ts/kcmd/gcp/crm.d.ts +13 -0
- package/build/ts/kcmd/gcp/crm.js +91 -0
- package/build/ts/kcmd/gcp/dataplex.d.ts +147 -0
- package/build/ts/kcmd/gcp/dataplex.js +406 -0
- package/build/ts/kcmd/gcp/index.d.ts +3 -0
- package/build/ts/kcmd/gcp/index.js +21 -0
- package/build/ts/kcmd/gcp/spanner.d.ts +73 -0
- package/build/ts/kcmd/gcp/spanner.js +126 -0
- package/build/ts/kcmd/index.d.ts +7 -0
- package/build/ts/kcmd/index.js +46 -0
- package/build/ts/kcmd/layout.d.ts +16 -0
- package/build/ts/kcmd/layout.js +31 -0
- package/build/ts/kcmd/layouts/documents.d.ts +24 -0
- package/build/ts/kcmd/layouts/documents.js +264 -0
- package/build/ts/kcmd/layouts/okf.d.ts +31 -0
- package/build/ts/kcmd/layouts/okf.js +386 -0
- package/build/ts/kcmd/layouts/semantic-model.d.ts +74 -0
- package/build/ts/kcmd/layouts/semantic-model.js +621 -0
- package/build/ts/kcmd/layouts/standard.d.ts +13 -0
- package/build/ts/kcmd/layouts/standard.js +108 -0
- package/build/ts/kcmd/manifest.d.ts +34 -0
- package/build/ts/kcmd/manifest.js +262 -0
- package/build/ts/kcmd/metadata.d.ts +25 -0
- package/build/ts/kcmd/metadata.js +8 -0
- package/build/ts/kcmd/okf/catalog_residue.d.ts +14 -0
- package/build/ts/kcmd/okf/catalog_residue.js +124 -0
- package/build/ts/kcmd/okf/concept.d.ts +14 -0
- package/build/ts/kcmd/okf/concept.js +110 -0
- package/build/ts/kcmd/okf/concept_to_entry.d.ts +10 -0
- package/build/ts/kcmd/okf/concept_to_entry.js +281 -0
- package/build/ts/kcmd/okf/init_okf_workspace.d.ts +15 -0
- package/build/ts/kcmd/okf/init_okf_workspace.js +113 -0
- package/build/ts/kcmd/okf/links.d.ts +23 -0
- package/build/ts/kcmd/okf/links.js +468 -0
- package/build/ts/kcmd/okf/okf_metadata.d.ts +12 -0
- package/build/ts/kcmd/okf/okf_metadata.js +99 -0
- package/build/ts/kcmd/semantic/bigquery.d.ts +22 -0
- package/build/ts/kcmd/semantic/bigquery.js +469 -0
- package/build/ts/kcmd/semantic/binding.d.ts +3 -0
- package/build/ts/kcmd/semantic/binding.js +92 -0
- package/build/ts/kcmd/semantic/catalog_entry.d.ts +24 -0
- package/build/ts/kcmd/semantic/catalog_entry.js +35 -0
- package/build/ts/kcmd/semantic/converters/owl/convert.d.ts +26 -0
- package/build/ts/kcmd/semantic/converters/owl/convert.js +41 -0
- package/build/ts/kcmd/semantic/converters/owl/model.d.ts +98 -0
- package/build/ts/kcmd/semantic/converters/owl/model.js +30 -0
- package/build/ts/kcmd/semantic/converters/owl/parse.d.ts +11 -0
- package/build/ts/kcmd/semantic/converters/owl/parse.js +634 -0
- package/build/ts/kcmd/semantic/converters/owl/to_ir.d.ts +19 -0
- package/build/ts/kcmd/semantic/converters/owl/to_ir.js +390 -0
- package/build/ts/kcmd/semantic/deploy_bigquery.d.ts +20 -0
- package/build/ts/kcmd/semantic/deploy_bigquery.js +226 -0
- package/build/ts/kcmd/semantic/deploy_knowledge_catalog.d.ts +104 -0
- package/build/ts/kcmd/semantic/deploy_knowledge_catalog.js +1367 -0
- package/build/ts/kcmd/semantic/deploy_spanner.d.ts +15 -0
- package/build/ts/kcmd/semantic/deploy_spanner.js +150 -0
- package/build/ts/kcmd/semantic/deployment_target.d.ts +93 -0
- package/build/ts/kcmd/semantic/deployment_target.js +379 -0
- package/build/ts/kcmd/semantic/graph_ddl.d.ts +44 -0
- package/build/ts/kcmd/semantic/graph_ddl.js +656 -0
- package/build/ts/kcmd/semantic/import_model.d.ts +72 -0
- package/build/ts/kcmd/semantic/import_model.js +1698 -0
- package/build/ts/kcmd/semantic/ir.d.ts +589 -0
- package/build/ts/kcmd/semantic/ir.js +247 -0
- package/build/ts/kcmd/semantic/kc_actions.d.ts +45 -0
- package/build/ts/kcmd/semantic/kc_actions.js +531 -0
- package/build/ts/kcmd/semantic/kc_constraints.d.ts +44 -0
- package/build/ts/kcmd/semantic/kc_constraints.js +279 -0
- package/build/ts/kcmd/semantic/kc_converter.d.ts +37 -0
- package/build/ts/kcmd/semantic/kc_converter.js +839 -0
- package/build/ts/kcmd/semantic/kc_custom_types.d.ts +63 -0
- package/build/ts/kcmd/semantic/kc_custom_types.js +751 -0
- package/build/ts/kcmd/semantic/kc_entries.d.ts +155 -0
- package/build/ts/kcmd/semantic/kc_entries.js +284 -0
- package/build/ts/kcmd/semantic/kc_ids.d.ts +102 -0
- package/build/ts/kcmd/semantic/kc_ids.js +188 -0
- package/build/ts/kcmd/semantic/knowledge_catalog.d.ts +29 -0
- package/build/ts/kcmd/semantic/knowledge_catalog.js +960 -0
- package/build/ts/kcmd/semantic/loader.d.ts +258 -0
- package/build/ts/kcmd/semantic/loader.js +3576 -0
- package/build/ts/kcmd/semantic/names.d.ts +10 -0
- package/build/ts/kcmd/semantic/names.js +85 -0
- package/build/ts/kcmd/semantic/osi_converter.d.ts +60 -0
- package/build/ts/kcmd/semantic/osi_converter.js +910 -0
- package/build/ts/kcmd/semantic/parameters.d.ts +19 -0
- package/build/ts/kcmd/semantic/parameters.js +171 -0
- package/build/ts/kcmd/semantic/pull_kc.d.ts +10 -0
- package/build/ts/kcmd/semantic/pull_kc.js +272 -0
- package/build/ts/kcmd/semantic/resolve_inheritance.d.ts +59 -0
- package/build/ts/kcmd/semantic/resolve_inheritance.js +346 -0
- package/build/ts/kcmd/semantic/resolve_profiles.d.ts +291 -0
- package/build/ts/kcmd/semantic/resolve_profiles.js +2003 -0
- package/build/ts/kcmd/semantic/runtime/agent_tools.d.ts +93 -0
- package/build/ts/kcmd/semantic/runtime/agent_tools.js +243 -0
- package/build/ts/kcmd/semantic/runtime/dialect.d.ts +13 -0
- package/build/ts/kcmd/semantic/runtime/dialect.js +39 -0
- package/build/ts/kcmd/semantic/runtime/runtime.d.ts +88 -0
- package/build/ts/kcmd/semantic/runtime/runtime.js +301 -0
- package/build/ts/kcmd/semantic/runtime/store.d.ts +112 -0
- package/build/ts/kcmd/semantic/runtime/store.js +344 -0
- package/build/ts/kcmd/semantic/skills.d.ts +110 -0
- package/build/ts/kcmd/semantic/skills.js +697 -0
- package/build/ts/kcmd/semantic/source_dialect.d.ts +17 -0
- package/build/ts/kcmd/semantic/source_dialect.js +311 -0
- package/build/ts/kcmd/semantic/source_entries.d.ts +12 -0
- package/build/ts/kcmd/semantic/source_entries.js +341 -0
- package/build/ts/kcmd/semantic/spanner.d.ts +18 -0
- package/build/ts/kcmd/semantic/spanner.js +60 -0
- package/build/ts/kcmd/semantic/sql_expr_utils.d.ts +23 -0
- package/build/ts/kcmd/semantic/sql_expr_utils.js +394 -0
- package/build/ts/kcmd/semantic/sql_identifiers.d.ts +6 -0
- package/build/ts/kcmd/semantic/sql_identifiers.js +267 -0
- package/build/ts/kcmd/semantic/sql_parser.d.ts +47 -0
- package/build/ts/kcmd/semantic/sql_parser.js +400 -0
- package/build/ts/kcmd/semantic/transpile.d.ts +76 -0
- package/build/ts/kcmd/semantic/transpile.js +581 -0
- package/build/ts/kcmd/semantic/validate.d.ts +70 -0
- package/build/ts/kcmd/semantic/validate.js +2266 -0
- package/build/ts/kcmd/semantic/yaml_options.d.ts +6 -0
- package/build/ts/kcmd/semantic/yaml_options.js +58 -0
- package/build/ts/kcmd/semantic/yaml_update.d.ts +26 -0
- package/build/ts/kcmd/semantic/yaml_update.js +782 -0
- package/build/ts/kcmd/snapshot.d.ts +37 -0
- package/build/ts/kcmd/snapshot.js +386 -0
- package/build/ts/kcmd/source.d.ts +31 -0
- package/build/ts/kcmd/source.js +119 -0
- package/build/ts/kcmd/sources/bq-dataset.d.ts +17 -0
- package/build/ts/kcmd/sources/bq-dataset.js +131 -0
- package/build/ts/kcmd/sources/entrygroup.d.ts +21 -0
- package/build/ts/kcmd/sources/entrygroup.js +83 -0
- package/build/ts/kcmd/sources/kb.d.ts +21 -0
- package/build/ts/kcmd/sources/kb.js +82 -0
- package/build/ts/kcmd/sources/semantic-model.d.ts +18 -0
- package/build/ts/kcmd/sources/semantic-model.js +48 -0
- package/build/ts/kcmd/sync.d.ts +35 -0
- package/build/ts/kcmd/sync.js +734 -0
- package/package.json +62 -4
|
@@ -0,0 +1,3576 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// Loads an open, vendor-neutral AI-first semantics format (YAML/JSON) into the
|
|
3
|
+
// Semantic Model IR (./ir).
|
|
4
|
+
//
|
|
5
|
+
// The format describes a semantic model as datasets (entities), foreign-key
|
|
6
|
+
// relationships, and model-level metrics, with entity-qualified SQL expressions
|
|
7
|
+
// (`Entity.column`) supplied per SQL dialect. This module reads the subset of
|
|
8
|
+
// that logical layer needed to normalize a model into the IR, so the rest of
|
|
9
|
+
// the toolbox (e.g. the BigQuery property-graph generator) can consume models
|
|
10
|
+
// authored in it. Fields outside the supported subset are accepted and ignored.
|
|
11
|
+
//
|
|
12
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
13
|
+
if (k2 === undefined) k2 = k;
|
|
14
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
15
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
16
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
17
|
+
}
|
|
18
|
+
Object.defineProperty(o, k2, desc);
|
|
19
|
+
}) : (function(o, m, k, k2) {
|
|
20
|
+
if (k2 === undefined) k2 = k;
|
|
21
|
+
o[k2] = m[k];
|
|
22
|
+
}));
|
|
23
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
24
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
25
|
+
}) : function(o, v) {
|
|
26
|
+
o["default"] = v;
|
|
27
|
+
});
|
|
28
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
29
|
+
var ownKeys = function(o) {
|
|
30
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
31
|
+
var ar = [];
|
|
32
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
33
|
+
return ar;
|
|
34
|
+
};
|
|
35
|
+
return ownKeys(o);
|
|
36
|
+
};
|
|
37
|
+
return function (mod) {
|
|
38
|
+
if (mod && mod.__esModule) return mod;
|
|
39
|
+
var result = {};
|
|
40
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
41
|
+
__setModuleDefault(result, mod);
|
|
42
|
+
return result;
|
|
43
|
+
};
|
|
44
|
+
})();
|
|
45
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
46
|
+
exports.THIRD_PARTY_SOURCE = exports.ENTRY_NAME_FORM = exports.DIALECT_OF_FQN_PREFIX = exports.DIALECT_OF_ENTRY_TYPE = exports.TABLE_FORMS = exports.NO_SPACE = exports.GOOGLE_VENDOR = exports.FORMAT_VERSION_PLACEMENT = exports.GOOGLE_VERSION = exports.OSSIE_VERSION = exports.DEFAULT_DIALECT = exports.HALF_BOUND_JOIN_PROBLEM = exports.SemanticModelError = void 0;
|
|
47
|
+
exports.joinLengthProblem = joinLengthProblem;
|
|
48
|
+
exports.joinLengthMessage = joinLengthMessage;
|
|
49
|
+
exports.undefinedEndpointProblem = undefinedEndpointProblem;
|
|
50
|
+
exports.isFormatVersion = isFormatVersion;
|
|
51
|
+
exports.noSqlReason = noSqlReason;
|
|
52
|
+
exports.listed = listed;
|
|
53
|
+
exports.plural = plural;
|
|
54
|
+
exports.expressionDocOf = expressionDocOf;
|
|
55
|
+
exports.noSqlProblemOf = noSqlProblemOf;
|
|
56
|
+
exports.inheritedNoSqlProblems = inheritedNoSqlProblems;
|
|
57
|
+
exports.isGoogleVendor = isGoogleVendor;
|
|
58
|
+
exports.hasGoogleBlock = hasGoogleBlock;
|
|
59
|
+
exports.setsAKeyTwice = setsAKeyTwice;
|
|
60
|
+
exports.googleBlockProfiles = googleBlockProfiles;
|
|
61
|
+
exports.modelDocumentVersion = modelDocumentVersion;
|
|
62
|
+
exports.isRecord = isRecord;
|
|
63
|
+
exports.modelAiContexts = modelAiContexts;
|
|
64
|
+
exports.loadModels = loadModels;
|
|
65
|
+
exports.fromDocument = fromDocument;
|
|
66
|
+
exports.inferMetricEntities = inferMetricEntities;
|
|
67
|
+
exports.entityAndMetricWarnings = entityAndMetricWarnings;
|
|
68
|
+
exports.parseFieldExpression = parseFieldExpression;
|
|
69
|
+
exports.readFieldExpression = readFieldExpression;
|
|
70
|
+
exports.parseFieldBinding = parseFieldBinding;
|
|
71
|
+
exports.parseExecutor = parseExecutor;
|
|
72
|
+
exports.cloudSqlTableOf = cloudSqlTableOf;
|
|
73
|
+
exports.sourceTableOf = sourceTableOf;
|
|
74
|
+
exports.asResourceUri = asResourceUri;
|
|
75
|
+
exports.asResourceName = asResourceName;
|
|
76
|
+
exports.resourceUriFix = resourceUriFix;
|
|
77
|
+
exports.oldSourceForm = oldSourceForm;
|
|
78
|
+
exports.fillForm = fillForm;
|
|
79
|
+
exports.dottedPartProblem = dottedPartProblem;
|
|
80
|
+
exports.catalogNameSegments = catalogNameSegments;
|
|
81
|
+
exports.hasUnquotedWhitespace = hasUnquotedWhitespace;
|
|
82
|
+
exports.databaseOf = databaseOf;
|
|
83
|
+
exports.isAcceptedSource = isAcceptedSource;
|
|
84
|
+
exports.writtenSource = writtenSource;
|
|
85
|
+
exports.sourceToDataSource = sourceToDataSource;
|
|
86
|
+
exports.thirdPartyEntryOf = thirdPartyEntryOf;
|
|
87
|
+
exports.bindsThirdPartyTable = bindsThirdPartyTable;
|
|
88
|
+
exports.dialectLookupKey = dialectLookupKey;
|
|
89
|
+
exports.spannerDatabaseOf = spannerDatabaseOf;
|
|
90
|
+
exports.isGoogleCloudForm = isGoogleCloudForm;
|
|
91
|
+
exports.parseSource = parseSource;
|
|
92
|
+
exports.bareNameSegments = bareNameSegments;
|
|
93
|
+
exports.bareNameParts = bareNameParts;
|
|
94
|
+
exports.loadSemanticModels = loadSemanticModels;
|
|
95
|
+
const yaml = __importStar(require("yaml"));
|
|
96
|
+
const z = __importStar(require("zod"));
|
|
97
|
+
const ir_1 = require("./ir");
|
|
98
|
+
const names_1 = require("./names");
|
|
99
|
+
const resolve_inheritance_1 = require("./resolve_inheritance");
|
|
100
|
+
const resolve_profiles_1 = require("./resolve_profiles");
|
|
101
|
+
const sql_expr_utils_1 = require("./sql_expr_utils");
|
|
102
|
+
const sql_parser_1 = require("./sql_parser");
|
|
103
|
+
const transpile_1 = require("./transpile");
|
|
104
|
+
const yaml_options_1 = require("./yaml_options");
|
|
105
|
+
// The error the loader throws for a mistake in a document it reads, and that
|
|
106
|
+
// code reading a profile throws for a mistake in a profile. A caller that
|
|
107
|
+
// reports such an error to the author catches this class only, so an internal
|
|
108
|
+
// bug still surfaces as one.
|
|
109
|
+
class SemanticModelError extends Error {
|
|
110
|
+
}
|
|
111
|
+
exports.SemanticModelError = SemanticModelError;
|
|
112
|
+
// Message texts the loader gives, shared with code that checks the same rules
|
|
113
|
+
// on a model already loaded. The texts that end in Problem leave out the
|
|
114
|
+
// relationship, so a caller can name it in its own way.
|
|
115
|
+
exports.HALF_BOUND_JOIN_PROBLEM = `from_columns and to_columns must be ` +
|
|
116
|
+
`given together (both bind the edge) or both omitted (a logical edge); ` +
|
|
117
|
+
`one without the other is a half-bound join.`;
|
|
118
|
+
function halfBoundJoinMessage(relationship) {
|
|
119
|
+
return `relationship '${relationship}': ${exports.HALF_BOUND_JOIN_PROBLEM}`;
|
|
120
|
+
}
|
|
121
|
+
function joinLengthProblem(from, to) {
|
|
122
|
+
return `from_columns (${from}) and to_columns (${to}) have different ` +
|
|
123
|
+
`lengths; the join keys are mismatched`;
|
|
124
|
+
}
|
|
125
|
+
function joinLengthMessage(relationship, from, to) {
|
|
126
|
+
return `relationship '${relationship}': ${joinLengthProblem(from, to)}`;
|
|
127
|
+
}
|
|
128
|
+
function undefinedEndpointProblem(side, dataset) {
|
|
129
|
+
return `'${side}' dataset '${dataset}' is not defined in the model`;
|
|
130
|
+
}
|
|
131
|
+
function sourceRequiredMessage(dataset) {
|
|
132
|
+
return `dataset '${dataset}': a non-abstract dataset requires a ` +
|
|
133
|
+
`source; set 'source', or mark it 'abstract: true' if it has no table`;
|
|
134
|
+
}
|
|
135
|
+
function ansiFallbackNote(preferred) {
|
|
136
|
+
return `note: no '${preferred}' dialect for one or more expressions; using the portable ` +
|
|
137
|
+
`'${FALLBACK_DIALECT}' dialect verbatim ('${preferred}' accepts the ANSI core subset — ` +
|
|
138
|
+
`supply '${preferred}' variants only for ${preferred}-specific SQL)`;
|
|
139
|
+
}
|
|
140
|
+
exports.DEFAULT_DIALECT = 'BIGQUERY';
|
|
141
|
+
const FALLBACK_DIALECT = 'ANSI_SQL';
|
|
142
|
+
// The two accepted format versions. Every document MUST declare one at the top
|
|
143
|
+
// level (a missing or unrecognized `version` is a load error). The value
|
|
144
|
+
// selects which extension surface is legal:
|
|
145
|
+
// - OSSIE_VERSION: vanilla Apache Ossie. kcmd's extensions ride ONLY in
|
|
146
|
+
// Ossie's `custom_extensions` carrier; the native extension keys
|
|
147
|
+
// (`entities` alias, `extends`, `abstract`, `deployments`, `actions`,
|
|
148
|
+
// `constraints`) are not accepted.
|
|
149
|
+
// - GOOGLE_VERSION: kcmd's extended profile. The native extension keys are
|
|
150
|
+
// first-class. Ossie's `custom_extensions` carrier is accepted for
|
|
151
|
+
// third-party vendors only; a GOOGLE block is rejected, since everything
|
|
152
|
+
// it would carry is a native key here.
|
|
153
|
+
// Both parse into the same IR, so a downstream leg never sees the difference.
|
|
154
|
+
exports.OSSIE_VERSION = '0.2.0.dev0';
|
|
155
|
+
exports.GOOGLE_VERSION = '0.2.0.dev0/google';
|
|
156
|
+
// How a vanilla author declares deployments: in a GOOGLE block on the model,
|
|
157
|
+
// or under `deployments:` after a switch to the Google flavor.
|
|
158
|
+
const DEPLOYMENTS_VERSION_HINT = `list 'deployments' in a GOOGLE custom_extensions block on the model, or ` +
|
|
159
|
+
`set version: '${exports.GOOGLE_VERSION}' and use 'deployments:'`;
|
|
160
|
+
const ACCEPTED_VERSIONS = [
|
|
161
|
+
exports.OSSIE_VERSION,
|
|
162
|
+
exports.GOOGLE_VERSION,
|
|
163
|
+
];
|
|
164
|
+
function isFormatVersion(v) {
|
|
165
|
+
return ACCEPTED_VERSIONS.includes(v);
|
|
166
|
+
}
|
|
167
|
+
/**
|
|
168
|
+
* Where each format version places each Google extension construct. Other
|
|
169
|
+
* differences between the versions, such as required bindings and datatype
|
|
170
|
+
* casing, are checked where they are read.
|
|
171
|
+
*/
|
|
172
|
+
exports.FORMAT_VERSION_PLACEMENT = {
|
|
173
|
+
[exports.GOOGLE_VERSION]: {
|
|
174
|
+
model: {
|
|
175
|
+
profiles: 'file',
|
|
176
|
+
deployments: 'native',
|
|
177
|
+
actions: 'native',
|
|
178
|
+
constraints: 'native',
|
|
179
|
+
},
|
|
180
|
+
dataset: {
|
|
181
|
+
extends: 'native',
|
|
182
|
+
abstract: 'native',
|
|
183
|
+
},
|
|
184
|
+
relationship: {
|
|
185
|
+
description: 'native',
|
|
186
|
+
},
|
|
187
|
+
metric: {
|
|
188
|
+
entity: 'native',
|
|
189
|
+
},
|
|
190
|
+
},
|
|
191
|
+
[exports.OSSIE_VERSION]: {
|
|
192
|
+
model: {
|
|
193
|
+
profiles: 'block',
|
|
194
|
+
deployments: 'block',
|
|
195
|
+
actions: 'none',
|
|
196
|
+
constraints: 'none',
|
|
197
|
+
},
|
|
198
|
+
dataset: {
|
|
199
|
+
extends: 'block',
|
|
200
|
+
abstract: 'none',
|
|
201
|
+
},
|
|
202
|
+
relationship: {
|
|
203
|
+
description: 'block',
|
|
204
|
+
},
|
|
205
|
+
metric: {
|
|
206
|
+
entity: 'block',
|
|
207
|
+
},
|
|
208
|
+
},
|
|
209
|
+
};
|
|
210
|
+
// An expression is supplied as one or more per-dialect variants, or (in the
|
|
211
|
+
// Google flavor only) as a bare SQL string shorthand for a single ANSI_SQL
|
|
212
|
+
// entry. Every dialect is validated against ALLOWED_DIALECTS (exact casing in
|
|
213
|
+
// both flavors), and a repeated dialect inside one list is rejected.
|
|
214
|
+
const expressionDialectEntrySchema = z.object({
|
|
215
|
+
dialect: z.string().superRefine((d, ctx) => {
|
|
216
|
+
if (!ir_1.ALLOWED_DIALECTS.includes(d)) {
|
|
217
|
+
ctx.addIssue({
|
|
218
|
+
code: z.ZodIssueCode.custom,
|
|
219
|
+
message: `Unknown dialect '${d}' in expression dialects. ` +
|
|
220
|
+
`Expected one of ${ir_1.ALLOWED_DIALECTS.join(', ')}.`,
|
|
221
|
+
});
|
|
222
|
+
}
|
|
223
|
+
}),
|
|
224
|
+
expression: z.string(),
|
|
225
|
+
}).strict();
|
|
226
|
+
const expressionObjectSchema = z.object({
|
|
227
|
+
dialects: z.array(expressionDialectEntrySchema)
|
|
228
|
+
.min(1)
|
|
229
|
+
.superRefine((entries, ctx) => {
|
|
230
|
+
const seen = new Set();
|
|
231
|
+
for (let i = 0; i < entries.length; i++) {
|
|
232
|
+
const d = entries[i].dialect;
|
|
233
|
+
if (seen.has(d)) {
|
|
234
|
+
ctx.addIssue({
|
|
235
|
+
code: z.ZodIssueCode.custom,
|
|
236
|
+
path: [i, 'dialect'],
|
|
237
|
+
message: `Duplicate dialect '${d}' in expression dialects.`,
|
|
238
|
+
});
|
|
239
|
+
}
|
|
240
|
+
seen.add(d);
|
|
241
|
+
}
|
|
242
|
+
}),
|
|
243
|
+
}).strict();
|
|
244
|
+
const expressionSchema = z.union([
|
|
245
|
+
z.string(),
|
|
246
|
+
expressionObjectSchema,
|
|
247
|
+
]);
|
|
248
|
+
// Why `text`, an expression written for `dialect`, holds no SQL: it "is
|
|
249
|
+
// blank", "is only a comment", or, for a field of `entity`, is only that
|
|
250
|
+
// entity's qualifier, which the graph generators remove. Undefined when it
|
|
251
|
+
// holds SQL. The SQL parser decides what is a comment. It reads an ANSI_SQL
|
|
252
|
+
// text as generic SQL and as BigQuery SQL, since the graph generators read
|
|
253
|
+
// that text as BigQuery SQL, where `#` also starts a comment.
|
|
254
|
+
function noSqlReason(text, dialect, entity) {
|
|
255
|
+
if (text.trim() === '')
|
|
256
|
+
return 'is blank';
|
|
257
|
+
const readers = dialect === 'ANSI_SQL' ? ['ANSI_SQL', 'BIGQUERY'] : [dialect];
|
|
258
|
+
if (readers.some(d => (0, sql_parser_1.holdsNoSql)(text, d)))
|
|
259
|
+
return 'is only a comment';
|
|
260
|
+
if (entity === undefined)
|
|
261
|
+
return undefined;
|
|
262
|
+
const onlyQualifier = (d) => (0, sql_expr_utils_1.withoutQualifier)((0, sql_parser_1.withoutComments)(text, d), entity, d).trim() === '';
|
|
263
|
+
return readers.some(onlyQualifier) ? `is only the qualifier '${entity}.'` :
|
|
264
|
+
undefined;
|
|
265
|
+
}
|
|
266
|
+
// Why an expression cannot load, or undefined. It cannot load when its string
|
|
267
|
+
// form, or any of its dialect entries, holds no SQL. The problem names each
|
|
268
|
+
// such entry. No expression at all passes. `entity` is the entity a field
|
|
269
|
+
// expression belongs to, and is absent for a metric.
|
|
270
|
+
function noSqlProblem(raw, entity) {
|
|
271
|
+
if (typeof raw === 'string') {
|
|
272
|
+
const reason = noSqlReason(raw, 'ANSI_SQL', entity);
|
|
273
|
+
return reason && `'expression' ${reason}`;
|
|
274
|
+
}
|
|
275
|
+
return listed((raw?.dialects ?? []).flatMap(d => {
|
|
276
|
+
const reason = noSqlReason(d.expression, d.dialect, entity);
|
|
277
|
+
return reason ? [`the ${d.dialect} entry of 'expression' ${reason}`] : [];
|
|
278
|
+
}));
|
|
279
|
+
}
|
|
280
|
+
// `problems` as one list, "a, b and c", or undefined when there are none.
|
|
281
|
+
// `conjunction` replaces the "and".
|
|
282
|
+
function listed(problems, conjunction = 'and') {
|
|
283
|
+
if (problems.length < 2)
|
|
284
|
+
return problems[0];
|
|
285
|
+
return `${problems.slice(0, -1).join(', ')} ${conjunction} ${problems[problems.length - 1]}`;
|
|
286
|
+
}
|
|
287
|
+
// Selects the singular or plural form based on `n` (English count agreement).
|
|
288
|
+
function plural(n, one, many) {
|
|
289
|
+
return n === 1 ? one : many;
|
|
290
|
+
}
|
|
291
|
+
// Whether reading `doc` for a field of `entity` names `text`, or finds every
|
|
292
|
+
// problem that reading `text` in `dialect` finds. The string form and an
|
|
293
|
+
// ANSI_SQL entry are read both as ANSI_SQL and as BigQuery.
|
|
294
|
+
function docReads(doc, text, dialect, entity) {
|
|
295
|
+
const covers = (read) => read === dialect ||
|
|
296
|
+
read === 'ANSI_SQL' && dialect === 'BIGQUERY' ||
|
|
297
|
+
noSqlReason(text, read, entity) !== undefined;
|
|
298
|
+
if (typeof doc === 'string')
|
|
299
|
+
return doc === text && covers('ANSI_SQL');
|
|
300
|
+
return (doc?.dialects ?? [])
|
|
301
|
+
.some(d => d.expression === text && covers(d.dialect));
|
|
302
|
+
}
|
|
303
|
+
// `f`'s expression as a document writes it: its string form or its dialect
|
|
304
|
+
// list. A binding built in memory with no dialect list gives its imported
|
|
305
|
+
// text as an entry of its imported dialect. When it also has a target text,
|
|
306
|
+
// that text is a BIGQUERY entry beside it, as serializeModel writes the two. A
|
|
307
|
+
// target text alone is the string form. So is a target text beside an
|
|
308
|
+
// imported text that a dialect list cannot hold: one with no imported
|
|
309
|
+
// dialect, or one marked BIGQUERY, since a list names each dialect once. An
|
|
310
|
+
// imported text with no imported dialect and no target text is the string
|
|
311
|
+
// form too. Undefined when `f` carries no text.
|
|
312
|
+
function expressionDocOf(f) {
|
|
313
|
+
if (f.dialects?.length) {
|
|
314
|
+
return f.stringForm ? f.dialects[0].expression : { dialects: f.dialects };
|
|
315
|
+
}
|
|
316
|
+
if (f.importedExpression === undefined || !f.importedDialect ||
|
|
317
|
+
f.expression !== undefined && f.importedDialect === 'BIGQUERY') {
|
|
318
|
+
return f.expression ?? f.importedExpression;
|
|
319
|
+
}
|
|
320
|
+
const imported = {
|
|
321
|
+
dialect: f.importedDialect,
|
|
322
|
+
expression: f.importedExpression,
|
|
323
|
+
};
|
|
324
|
+
return {
|
|
325
|
+
dialects: f.expression === undefined ?
|
|
326
|
+
[imported] :
|
|
327
|
+
[{ dialect: 'BIGQUERY', expression: f.expression }, imported],
|
|
328
|
+
};
|
|
329
|
+
}
|
|
330
|
+
// Why a field or metric holds no SQL, or undefined. Its expression is read as
|
|
331
|
+
// a document writes it, as expressionDocOf gives it. Then its target text is
|
|
332
|
+
// read as BigQuery, and its imported text in its imported dialect, or as
|
|
333
|
+
// BigQuery with none, unless that reading already read or named it.
|
|
334
|
+
// serializeModel writes both. Either text can be outside the dialect list
|
|
335
|
+
// when the field or metric was built in memory. The problem names each text
|
|
336
|
+
// that holds no SQL. `entity` is the entity a field belongs to, and is absent
|
|
337
|
+
// for a metric.
|
|
338
|
+
function noSqlProblemOf(x, entity) {
|
|
339
|
+
const doc = expressionDocOf(x);
|
|
340
|
+
const problems = [noSqlProblem(doc, entity)];
|
|
341
|
+
const texts = [
|
|
342
|
+
['expression', x.expression, 'BIGQUERY'],
|
|
343
|
+
['importedExpression', x.importedExpression, x.importedDialect ?? 'BIGQUERY'],
|
|
344
|
+
];
|
|
345
|
+
for (const [key, text, dialect] of texts) {
|
|
346
|
+
if (text === undefined || docReads(doc, text, dialect, entity))
|
|
347
|
+
continue;
|
|
348
|
+
const reason = noSqlReason(text, dialect, entity);
|
|
349
|
+
problems.push(reason && `'${key}' ${reason}`);
|
|
350
|
+
}
|
|
351
|
+
return listed(problems.filter((p) => p !== undefined));
|
|
352
|
+
}
|
|
353
|
+
// Why each field an entity inherits holds no SQL on that entity, one message
|
|
354
|
+
// per field. `concepts` gives each entity's fields with inheritance resolved,
|
|
355
|
+
// so an inherited text has lost its ancestor's qualifier. That can leave only
|
|
356
|
+
// the entity's own qualifier, as `base.orders.` does on `orders`, which the
|
|
357
|
+
// graph generators also remove. A field the entity binds itself keeps its own
|
|
358
|
+
// text, which is checked where it is declared, so it is left out. So is
|
|
359
|
+
// every field of an entity that extends no other. A problem that names no
|
|
360
|
+
// entity, such as a `-- todo` text, is reported where the field is declared,
|
|
361
|
+
// so a field is reported on the entity only when the entity adds to it.
|
|
362
|
+
// `problemOf` says why one field holds no SQL, on one entity when one is
|
|
363
|
+
// named. Unless given, it reads the field as a document writes it, each
|
|
364
|
+
// dialect entry in its own dialect, whatever dialect picked the target text.
|
|
365
|
+
function inheritedNoSqlProblems(entities, concepts, problemOf = (f, entity) => noSqlProblem(expressionDocOf(f), entity)) {
|
|
366
|
+
const problems = [];
|
|
367
|
+
for (const e of entities) {
|
|
368
|
+
if (!e.extends?.length)
|
|
369
|
+
continue;
|
|
370
|
+
const own = new Set((e.fields ?? []).filter(ir_1.isFieldBound).map(f => f.name));
|
|
371
|
+
for (const f of concepts.get(e.name)?.fields.values() ?? []) {
|
|
372
|
+
if (own.has(f.name))
|
|
373
|
+
continue;
|
|
374
|
+
const problem = problemOf(f, e.name);
|
|
375
|
+
if (problem && problem !== problemOf(f)) {
|
|
376
|
+
problems.push(`entity '${e.name}' inherits field '${f.name}': read on '${e.name}', ${problem}; write the field's SQL`);
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
return problems;
|
|
381
|
+
}
|
|
382
|
+
// The format's AI-first annotation. It appears at every level (model, dataset,
|
|
383
|
+
// field, relationship, metric) and is either a bare instructions string or a
|
|
384
|
+
// structured object (Mapping Addendum 4).
|
|
385
|
+
//
|
|
386
|
+
// Three members are CLOSED, as in Ossie's schema: `instructions` is a string,
|
|
387
|
+
// and `synonyms` and `examples` are lists of strings (a non-string example is
|
|
388
|
+
// rejected, not dropped). Any other member is a CUSTOM member: opaque, any
|
|
389
|
+
// value, carried verbatim into AiContext.additionalProperties. Member names are
|
|
390
|
+
// case-sensitive, so `Synonyms` is a custom member, not a misspelled synonym.
|
|
391
|
+
// The flavors write custom members differently:
|
|
392
|
+
// - Google: inside a `custom` mapping. Every other unknown key stays an error,
|
|
393
|
+
// so a misspelled `synonyms` is still caught.
|
|
394
|
+
// - Vanilla: as siblings of the three, as Ossie's open object allows. A
|
|
395
|
+
// sibling named `custom` is a custom member like any other.
|
|
396
|
+
// The bare-string shorthand stands for `instructions` and has no custom
|
|
397
|
+
// members. Both object forms parse to the same shape (AiContextObjectDoc), so
|
|
398
|
+
// nothing downstream needs to know the flavor.
|
|
399
|
+
const AI_CONTEXT_MEMBERS = ['instructions', 'synonyms', 'examples'];
|
|
400
|
+
const aiContextMembers = {
|
|
401
|
+
instructions: z.string().optional(),
|
|
402
|
+
synonyms: z.array(z.string()).optional(),
|
|
403
|
+
examples: z.array(z.string()).optional(),
|
|
404
|
+
};
|
|
405
|
+
// Keeps the closed members and attaches `custom` members only when there are
|
|
406
|
+
// any, so an object with none parses to exactly the closed shape.
|
|
407
|
+
function aiContextObject(closed, custom) {
|
|
408
|
+
const out = {};
|
|
409
|
+
if (closed.instructions !== undefined)
|
|
410
|
+
out.instructions = closed.instructions;
|
|
411
|
+
if (closed.synonyms !== undefined)
|
|
412
|
+
out.synonyms = closed.synonyms;
|
|
413
|
+
if (closed.examples !== undefined)
|
|
414
|
+
out.examples = closed.examples;
|
|
415
|
+
if (Object.keys(custom).length)
|
|
416
|
+
out.additionalProperties = custom;
|
|
417
|
+
return out;
|
|
418
|
+
}
|
|
419
|
+
const googleAiContextSchema = z.union([
|
|
420
|
+
z.string(),
|
|
421
|
+
z.object({
|
|
422
|
+
...aiContextMembers,
|
|
423
|
+
custom: z.record(z.string(), z.unknown()).optional(),
|
|
424
|
+
})
|
|
425
|
+
.strict()
|
|
426
|
+
.superRefine((ai, ctx) => {
|
|
427
|
+
// A closed member written inside `custom` is one vanilla could not
|
|
428
|
+
// express, and almost always a misplaced line; name where it goes.
|
|
429
|
+
for (const member of AI_CONTEXT_MEMBERS) {
|
|
430
|
+
if (ai.custom && Object.hasOwn(ai.custom, member)) {
|
|
431
|
+
ctx.addIssue({
|
|
432
|
+
code: z.ZodIssueCode.custom,
|
|
433
|
+
path: ['custom', member],
|
|
434
|
+
message: `ai_context 'custom' cannot hold '${member}'; it is ` +
|
|
435
|
+
`a standard member, so write it beside 'custom', not ` +
|
|
436
|
+
`inside it.`,
|
|
437
|
+
});
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
})
|
|
441
|
+
.transform(({ custom, ...closed }) => aiContextObject(closed, custom ?? {})),
|
|
442
|
+
]);
|
|
443
|
+
const vanillaAiContextSchema = z.union([
|
|
444
|
+
z.string(),
|
|
445
|
+
z.object(aiContextMembers).passthrough().transform(ai => {
|
|
446
|
+
const custom = {};
|
|
447
|
+
for (const [key, value] of Object.entries(ai)) {
|
|
448
|
+
if (!AI_CONTEXT_MEMBERS.includes(key)) {
|
|
449
|
+
custom[key] = value;
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
return aiContextObject(ai, custom);
|
|
453
|
+
}),
|
|
454
|
+
]);
|
|
455
|
+
// The schema for one flavor. Actions, parameters and constraints exist only in
|
|
456
|
+
// the Google flavor, so they always take the Google form (aiContextSchema).
|
|
457
|
+
function aiContextSchemaFor(extended) {
|
|
458
|
+
return extended ? googleAiContextSchema : vanillaAiContextSchema;
|
|
459
|
+
}
|
|
460
|
+
const aiContextSchema = googleAiContextSchema;
|
|
461
|
+
// A vendor-scoped extension block: opaque `data` (a JSON string) tagged by
|
|
462
|
+
// `vendor_name`. The spec allows these at every level (model, dataset, field,
|
|
463
|
+
// relationship, metric). A vanilla document's GOOGLE blocks on a model,
|
|
464
|
+
// dataset, relationship or metric are read into the IR (see readGoogleBlocks).
|
|
465
|
+
// rejectGoogleBlock says where a GOOGLE block is rejected. Every other block is
|
|
466
|
+
// kept verbatim.
|
|
467
|
+
const customExtensionSchema = z.object({
|
|
468
|
+
vendor_name: z.string(),
|
|
469
|
+
data: z.string(),
|
|
470
|
+
}).strict();
|
|
471
|
+
// A field's dimension metadata; only the time flag is read today.
|
|
472
|
+
const dimensionSchema = z.object({
|
|
473
|
+
is_time: z.boolean().optional(),
|
|
474
|
+
}).strict();
|
|
475
|
+
// The canonical (superset) field shape, for TYPE inference only. The actual
|
|
476
|
+
// validation schemas are rebuilt per-load by buildDocumentSchema, which is
|
|
477
|
+
// strict, gates the native extension keys on the version, and applies the
|
|
478
|
+
// source-completeness refinement. A field is BOUND when it names a physical
|
|
479
|
+
// column via `expression`; a field with no `expression` is UNBOUND (no column
|
|
480
|
+
// under this binding). A field is never required to be bound -- an unbound
|
|
481
|
+
// field loads and the availability pass prunes it before generation.
|
|
482
|
+
const fieldBase = z.object({
|
|
483
|
+
name: z.string(),
|
|
484
|
+
expression: expressionSchema.optional(),
|
|
485
|
+
// A plain string here; whether it is valid, and its canonical spelling,
|
|
486
|
+
// depend on the flavor and are decided in convertField (normalizeDataType).
|
|
487
|
+
datatype: z.string().optional(),
|
|
488
|
+
description: z.string().optional(),
|
|
489
|
+
label: z.string().optional(),
|
|
490
|
+
dimension: dimensionSchema.optional(),
|
|
491
|
+
ai_context: aiContextSchema.optional(),
|
|
492
|
+
custom_extensions: z.array(customExtensionSchema).optional(),
|
|
493
|
+
});
|
|
494
|
+
// The dataset object shape, WITHOUT the source-required refinement (applied
|
|
495
|
+
// per-load by makeDocumentSchema). DatasetDoc is inferred from this base.
|
|
496
|
+
const datasetBase = z.object({
|
|
497
|
+
name: z.string(),
|
|
498
|
+
// A concrete dataset is backed by a physical `source` table; an `abstract`
|
|
499
|
+
// one has no table. Whether a non-abstract dataset must name a `source` is
|
|
500
|
+
// decided per-load by makeDocumentSchema (a KC-only push accepts a logical
|
|
501
|
+
// dataset with none); the abstract+source contradiction is always rejected.
|
|
502
|
+
source: z.string().optional(),
|
|
503
|
+
primary_key: z.array(z.string()).optional(),
|
|
504
|
+
unique_keys: z.array(z.array(z.string())).optional(),
|
|
505
|
+
// Supertype entity names (Ossie `extends`) -- entity-level inheritance. Only
|
|
506
|
+
// datasets carry it; relationships have no `extends`.
|
|
507
|
+
extends: z.array(z.string()).optional(),
|
|
508
|
+
// Marks a conceptual entity with no physical table (see Entity.abstract): it
|
|
509
|
+
// forms no node table and survives only as a label on its concrete
|
|
510
|
+
// descendants. Distinct from an unbound `source` placeholder, which must fail
|
|
511
|
+
// loudly rather than be silently treated as table-less.
|
|
512
|
+
abstract: z.boolean().optional(),
|
|
513
|
+
description: z.string().optional(),
|
|
514
|
+
ai_context: aiContextSchema.optional(),
|
|
515
|
+
fields: z.array(fieldBase).optional(),
|
|
516
|
+
custom_extensions: z.array(customExtensionSchema).optional(),
|
|
517
|
+
});
|
|
518
|
+
const relationshipSchema = z.object({
|
|
519
|
+
name: z.string(),
|
|
520
|
+
from: z.string(),
|
|
521
|
+
to: z.string(),
|
|
522
|
+
// Join columns are the physical binding of the
|
|
523
|
+
// edge and are OPTIONAL, so a purely logical
|
|
524
|
+
// relationship (an ontology edge, direction only)
|
|
525
|
+
// loads. When present they must be non-empty;
|
|
526
|
+
// either both endpoints are bound or neither is
|
|
527
|
+
// (a half-bound edge is a malformed join, caught
|
|
528
|
+
// below). A graph push still requires both (see
|
|
529
|
+
// validatePushRequirements).
|
|
530
|
+
from_columns: z.array(z.string()).min(1).optional(),
|
|
531
|
+
to_columns: z.array(z.string()).min(1).optional(),
|
|
532
|
+
description: z.string().optional(),
|
|
533
|
+
ai_context: aiContextSchema.optional(),
|
|
534
|
+
custom_extensions: z.array(customExtensionSchema).optional(),
|
|
535
|
+
}).superRefine((r, ctx) => {
|
|
536
|
+
if ((r.from_columns === undefined) !== (r.to_columns === undefined)) {
|
|
537
|
+
ctx.addIssue({
|
|
538
|
+
code: z.ZodIssueCode.custom,
|
|
539
|
+
message: halfBoundJoinMessage(r.name),
|
|
540
|
+
});
|
|
541
|
+
}
|
|
542
|
+
});
|
|
543
|
+
const metricSchema = z.object({
|
|
544
|
+
name: z.string(),
|
|
545
|
+
// The entity the metric belongs to, as the author wrote it. Google flavor
|
|
546
|
+
// only (buildDocumentSchema rejects it in vanilla).
|
|
547
|
+
entity: z.string().optional(),
|
|
548
|
+
expression: expressionSchema,
|
|
549
|
+
// A plain string; validated and canonicalized in convertMetric
|
|
550
|
+
// (normalizeDataType).
|
|
551
|
+
datatype: z.string().optional(),
|
|
552
|
+
description: z.string().optional(),
|
|
553
|
+
ai_context: aiContextSchema.optional(),
|
|
554
|
+
custom_extensions: z.array(customExtensionSchema).optional(),
|
|
555
|
+
});
|
|
556
|
+
// An action executor: exactly one kind. The open format expresses it as an
|
|
557
|
+
// object with a single kind key (mcp/rest/grpc/sql); we accept the union and
|
|
558
|
+
// enforce the "exactly one" rule in a refinement so the message names the
|
|
559
|
+
// violation.
|
|
560
|
+
//
|
|
561
|
+
// `sql` carries the write itself rather than a pointer to whoever performs it,
|
|
562
|
+
// so the schema only checks its shape here. What makes it safe -- one DML verb
|
|
563
|
+
// per statement, and every `@parameter` declared by the action -- is checked in
|
|
564
|
+
// validate.ts, where the action's parameter list is in scope. validate.ts also
|
|
565
|
+
// holds the live pre-flight that plans each statement against the bound store,
|
|
566
|
+
// which is where a name the database does not have is caught. See SqlExecutor.
|
|
567
|
+
const EXECUTOR_KINDS = ['mcp', 'rest', 'grpc', 'sql'];
|
|
568
|
+
const executorSchema = z.object({
|
|
569
|
+
mcp: z.object({ server: z.string(), tool: z.string() }).strict().optional(),
|
|
570
|
+
rest: z.object({ endpoint: z.string(), method: z.string() })
|
|
571
|
+
.strict()
|
|
572
|
+
.optional(),
|
|
573
|
+
grpc: z.object({ service: z.string(), method: z.string() })
|
|
574
|
+
.strict()
|
|
575
|
+
.optional(),
|
|
576
|
+
sql: z.object({ statements: z.array(z.string()).min(1) })
|
|
577
|
+
.strict()
|
|
578
|
+
.optional(),
|
|
579
|
+
})
|
|
580
|
+
.strict()
|
|
581
|
+
.superRefine((ex, ctx) => {
|
|
582
|
+
const kinds = EXECUTOR_KINDS.filter(k => ex[k] !== undefined);
|
|
583
|
+
if (kinds.length !== 1) {
|
|
584
|
+
ctx.addIssue({
|
|
585
|
+
code: z.ZodIssueCode.custom,
|
|
586
|
+
message: kinds.length === 0 ?
|
|
587
|
+
`executor requires exactly one kind (${EXECUTOR_KINDS.join(', ')}); none given` :
|
|
588
|
+
`executor requires exactly one kind, but ${kinds.length} given ` +
|
|
589
|
+
`(${kinds.join(', ')})`,
|
|
590
|
+
});
|
|
591
|
+
}
|
|
592
|
+
});
|
|
593
|
+
// An action parameter, in either of its two authored forms (see
|
|
594
|
+
// ir.ActionParameter). DERIVED states `concept` and `field` and no `type`;
|
|
595
|
+
// STANDALONE states a `type` and neither. Every key here is optional because
|
|
596
|
+
// which combination is legal depends on the other keys, and a zod union would
|
|
597
|
+
// report a document that got it wrong as a list of failed alternatives naming a
|
|
598
|
+
// position rather than the mistake. convertParameter decides instead, where the
|
|
599
|
+
// model is in scope and the message can name the fix.
|
|
600
|
+
const parameterSchema = z.object({
|
|
601
|
+
// Optional: a derived parameter takes the field's name unless it renames it.
|
|
602
|
+
name: z.string().optional(),
|
|
603
|
+
// A scalar DataType, and only on a standalone parameter. Resolved against
|
|
604
|
+
// the model in convertParameter, not here, so the schema stays a plain
|
|
605
|
+
// string. `datatype` is accepted as an alias for `type` so parameters can be
|
|
606
|
+
// written with the same key as entity fields and metrics.
|
|
607
|
+
type: z.string().optional(),
|
|
608
|
+
datatype: z.string().optional(),
|
|
609
|
+
concept: z.string().optional(),
|
|
610
|
+
field: z.string().optional(),
|
|
611
|
+
description: z.string().optional(),
|
|
612
|
+
label: z.string().optional(),
|
|
613
|
+
ai_context: aiContextSchema.optional(),
|
|
614
|
+
required: z.boolean().optional(),
|
|
615
|
+
default: z.unknown().optional(),
|
|
616
|
+
});
|
|
617
|
+
// One thing an action changes. Two authored shapes: a bare name, which is the
|
|
618
|
+
// coarse blast radius (`affects: [Order, OrderedAs]`), or a record naming the
|
|
619
|
+
// `concept` plus how it is changed. One key covers an entity and a
|
|
620
|
+
// relationship alike, because the same three operations apply to either. The
|
|
621
|
+
// bare name is not a lesser form -- it is the same entry with the operation
|
|
622
|
+
// left unspecified -- so the two mix freely in one list.
|
|
623
|
+
//
|
|
624
|
+
// The name is resolved against the model in toAffectedConcept, not here, for
|
|
625
|
+
// the same reason a parameter type is: the schema sees one action, and only
|
|
626
|
+
// the model knows what `Order` is.
|
|
627
|
+
const affectedConceptSchema = z.union([
|
|
628
|
+
z.string(),
|
|
629
|
+
z.object({
|
|
630
|
+
concept: z.string(),
|
|
631
|
+
operation: z.enum(ir_1.CONCEPT_OPERATIONS).optional(),
|
|
632
|
+
fields: z.array(z.string()).optional(),
|
|
633
|
+
}).strict(),
|
|
634
|
+
]);
|
|
635
|
+
const actionSchema = z.object({
|
|
636
|
+
name: z.string(),
|
|
637
|
+
description: z.string().optional(),
|
|
638
|
+
// Optional because it is a physical binding: a profile may supply it, and a
|
|
639
|
+
// purely logical model declares actions it cannot perform. See Action.
|
|
640
|
+
// `null` reads the same as absent, so `executor:` with the body commented out
|
|
641
|
+
// means here what `executor: null` means in a profile: performed by nothing.
|
|
642
|
+
executor: executorSchema.nullish(),
|
|
643
|
+
parameters: z.array(parameterSchema).optional(),
|
|
644
|
+
// Names of the constraints that gate this action. Kept as plain strings: they
|
|
645
|
+
// are resolved against the model's own constraints in validate.ts, which sees
|
|
646
|
+
// the whole model, whereas the schema sees one action.
|
|
647
|
+
guards: z.array(z.string()).optional(),
|
|
648
|
+
// What the call changes. See affectedConceptSchema.
|
|
649
|
+
affects: z.array(affectedConceptSchema).optional(),
|
|
650
|
+
ai_context: aiContextSchema.optional(),
|
|
651
|
+
custom_extensions: z.array(customExtensionSchema).optional(),
|
|
652
|
+
});
|
|
653
|
+
// A constraint: a named invariant over the ontology, stated in words.
|
|
654
|
+
//
|
|
655
|
+
// `judgment` is the rule as a sentence. It is a plain string -- validate
|
|
656
|
+
// resolves the `Entity.field` tokens it mentions, and nothing else reads into
|
|
657
|
+
// it. Optional here and required in fact, which `validate` enforces so the
|
|
658
|
+
// author gets one message naming the constraint rather than a schema union
|
|
659
|
+
// error naming a position in the document.
|
|
660
|
+
//
|
|
661
|
+
// `expression` is RESERVED, not supported. It named the removed second body, a
|
|
662
|
+
// boolean in the model's own language, and it is still accepted by the schema
|
|
663
|
+
// for one reason: a model written against the older shape should be told to
|
|
664
|
+
// restate the rule as a judgment, not handed an unrecognized-key error that
|
|
665
|
+
// says only that the word is unknown. `rejectExpressionBody` refuses it. See
|
|
666
|
+
// Constraint in ir.ts.
|
|
667
|
+
const constraintSchema = z.object({
|
|
668
|
+
name: z.string(),
|
|
669
|
+
expression: z.string().optional(),
|
|
670
|
+
judgment: z.string().optional(),
|
|
671
|
+
description: z.string().optional(),
|
|
672
|
+
// What the engine does; absent means `reject`. See VIOLATION_EFFECTS.
|
|
673
|
+
on_violation: z.enum(ir_1.VIOLATION_EFFECTS).optional(),
|
|
674
|
+
// How grave it is; no default, and nothing reads it yet. See
|
|
675
|
+
// CONSTRAINT_SEVERITIES.
|
|
676
|
+
severity: z.enum(ir_1.CONSTRAINT_SEVERITIES).optional(),
|
|
677
|
+
ai_context: aiContextSchema.optional(),
|
|
678
|
+
// No `custom_extensions`: it is a vanilla-Ossie surface, and
|
|
679
|
+
// `constraints` is an extended-profile-only key, so the two never
|
|
680
|
+
// co-occur. See Constraint.
|
|
681
|
+
}).superRefine(rejectExpressionBody);
|
|
682
|
+
// Refuses the removed `expression` body, naming the constraint and saying where
|
|
683
|
+
// the rule goes instead. A hard error rather than a dropped key: a rule the
|
|
684
|
+
// author stated and the loader silently ignored is a guard that stops guarding.
|
|
685
|
+
function rejectExpressionBody(c, ctx) {
|
|
686
|
+
if (c.expression === undefined)
|
|
687
|
+
return;
|
|
688
|
+
ctx.addIssue({
|
|
689
|
+
code: z.ZodIssueCode.custom,
|
|
690
|
+
path: ['expression'],
|
|
691
|
+
message: `constraint '${c.name}' states an expression. A constraint ` +
|
|
692
|
+
`states its rule as a judgment, in words, and nothing evaluates an ` +
|
|
693
|
+
`expression. Restate the rule under 'judgment' and give it an ` +
|
|
694
|
+
`'on_violation'.`,
|
|
695
|
+
});
|
|
696
|
+
}
|
|
697
|
+
const deploymentSchema = z.object({
|
|
698
|
+
name: z.string(),
|
|
699
|
+
target: z.string().trim().min(1),
|
|
700
|
+
profile: z.string().min(1).optional(),
|
|
701
|
+
}).strict();
|
|
702
|
+
const modelBase = z.object({
|
|
703
|
+
name: z.string(),
|
|
704
|
+
description: z.string().optional(),
|
|
705
|
+
ai_context: aiContextSchema.optional(),
|
|
706
|
+
custom_extensions: z.array(customExtensionSchema).optional(),
|
|
707
|
+
datasets: z.array(datasetBase).min(1),
|
|
708
|
+
relationships: z.array(relationshipSchema).optional(),
|
|
709
|
+
metrics: z.array(metricSchema).optional(),
|
|
710
|
+
// Named deployment targets (GOOGLE_VERSION only).
|
|
711
|
+
deployments: z.array(deploymentSchema).optional(),
|
|
712
|
+
// Model-level write operations, also GOOGLE_VERSION only (see actionSchema).
|
|
713
|
+
actions: z.array(actionSchema).optional(),
|
|
714
|
+
constraints: z.array(constraintSchema).optional(),
|
|
715
|
+
});
|
|
716
|
+
// The vendor-scoped `custom_extensions` field, accepted in BOTH flavors at the
|
|
717
|
+
// five levels the format defines it on: model, dataset, field, relationship and
|
|
718
|
+
// metric. Which vendor blocks are legal where is a flavor rule, checked
|
|
719
|
+
// separately. Actions and constraints are extended-profile constructs with no
|
|
720
|
+
// `custom_extensions` encoding, so they do not take it.
|
|
721
|
+
const customExtensionsKey = { custom_extensions: z.array(customExtensionSchema).optional() };
|
|
722
|
+
// The schemas of keys a document schema accepts only so that the loader can
|
|
723
|
+
// reject them with its own message. A message about an unknown key does not
|
|
724
|
+
// list them among the keys an object takes.
|
|
725
|
+
const UNLISTED_KEYS = new WeakSet();
|
|
726
|
+
function unlisted(schema) {
|
|
727
|
+
UNLISTED_KEYS.add(schema);
|
|
728
|
+
return schema;
|
|
729
|
+
}
|
|
730
|
+
// Builds the document schema for one (bindingOptional, extended) combination.
|
|
731
|
+
// Every object is `.strict()`, so an unknown key is a hard error rather than
|
|
732
|
+
// silently dropped. Two axes shape it:
|
|
733
|
+
// - `extended` selects the version's extension surface: under the extended
|
|
734
|
+
// profile the native keys (`extends`, `abstract`, `deployments`,
|
|
735
|
+
// metric `entity`) are accepted; under vanilla Ossie they are not.
|
|
736
|
+
// (`entities` is folded to `datasets` before validation, so it is never a
|
|
737
|
+
// schema key -- see normalizeDocumentSugars.)
|
|
738
|
+
// - `bindingOptional` relaxes the source rule: when false (any push with a
|
|
739
|
+
// graph leg) each concrete dataset must name a `source`; when true (a
|
|
740
|
+
// Knowledge-Catalog-only push) it is optional, so a purely logical model
|
|
741
|
+
// loads. The abstract+source contradiction is rejected either way. Fields
|
|
742
|
+
// are never required to carry an `expression`: a field with no `expression`
|
|
743
|
+
// is simply unbound (there is no separate flag), and the availability pass
|
|
744
|
+
// prunes it before generation -- so it is not a load error under either
|
|
745
|
+
// `bindingOptional`.
|
|
746
|
+
function buildDocumentSchema(bindingOptional, extended) {
|
|
747
|
+
const placement = exports.FORMAT_VERSION_PLACEMENT[extended ? exports.GOOGLE_VERSION : exports.OSSIE_VERSION];
|
|
748
|
+
// The five data levels take this flavor's ai_context; actions and
|
|
749
|
+
// constraints are Google-only and keep the Google form.
|
|
750
|
+
const aiContext = aiContextSchemaFor(extended);
|
|
751
|
+
const field = z.object({
|
|
752
|
+
name: z.string(),
|
|
753
|
+
expression: expressionSchema.optional(),
|
|
754
|
+
// Validated per flavor in convertField (normalizeDataType).
|
|
755
|
+
datatype: z.string().optional(),
|
|
756
|
+
description: z.string().optional(),
|
|
757
|
+
label: z.string().optional(),
|
|
758
|
+
dimension: dimensionSchema.optional(),
|
|
759
|
+
ai_context: aiContext.optional(),
|
|
760
|
+
...customExtensionsKey,
|
|
761
|
+
}).strict();
|
|
762
|
+
const dataset = z.object({
|
|
763
|
+
name: z.string(),
|
|
764
|
+
source: z.string().optional(),
|
|
765
|
+
primary_key: z.array(z.string()).optional(),
|
|
766
|
+
unique_keys: z.array(z.array(z.string())).optional(),
|
|
767
|
+
description: z.string().optional(),
|
|
768
|
+
ai_context: aiContext.optional(),
|
|
769
|
+
fields: z.array(field).optional(),
|
|
770
|
+
...customExtensionsKey,
|
|
771
|
+
// Inheritance (`extends`, `abstract`) is a native extension in the
|
|
772
|
+
// extended profile only. Vanilla still declares both keys so the
|
|
773
|
+
// refinement below can reject each with a message that names the
|
|
774
|
+
// dataset and the fix, not a bare "unrecognized key".
|
|
775
|
+
extends: placement.dataset.extends === 'native' ?
|
|
776
|
+
z.array(z.string()).optional() :
|
|
777
|
+
unlisted(z.unknown().optional()),
|
|
778
|
+
abstract: placement.dataset.abstract === 'native' ?
|
|
779
|
+
z.boolean().optional() :
|
|
780
|
+
unlisted(z.unknown().optional()),
|
|
781
|
+
})
|
|
782
|
+
.strict()
|
|
783
|
+
.superRefine((ds, ctx) => {
|
|
784
|
+
if (placement.dataset.extends !== 'native' &&
|
|
785
|
+
ds.extends !== undefined) {
|
|
786
|
+
ctx.addIssue({
|
|
787
|
+
code: 'custom',
|
|
788
|
+
path: ['extends'],
|
|
789
|
+
message: `dataset '${ds.name}': 'extends' is a ` +
|
|
790
|
+
`'${exports.GOOGLE_VERSION}' extension. Set 'extends' in a ` +
|
|
791
|
+
`${exports.GOOGLE_VENDOR} custom_extensions block on the ` +
|
|
792
|
+
`dataset, or set the document version to ` +
|
|
793
|
+
`'${exports.GOOGLE_VERSION}'.`,
|
|
794
|
+
});
|
|
795
|
+
}
|
|
796
|
+
if (placement.dataset.abstract !== 'native' &&
|
|
797
|
+
ds.abstract !== undefined) {
|
|
798
|
+
ctx.addIssue({
|
|
799
|
+
code: 'custom',
|
|
800
|
+
path: ['abstract'],
|
|
801
|
+
message: `dataset '${ds.name}': 'abstract' is a ` +
|
|
802
|
+
`'${exports.GOOGLE_VERSION}' extension. An abstract dataset has ` +
|
|
803
|
+
`no table, and every '${exports.OSSIE_VERSION}' dataset needs a ` +
|
|
804
|
+
`'source'. A model that needs an abstract entity uses ` +
|
|
805
|
+
`'${exports.GOOGLE_VERSION}'.`,
|
|
806
|
+
});
|
|
807
|
+
}
|
|
808
|
+
const abstract = placement.dataset.abstract === 'native' && ds.abstract === true;
|
|
809
|
+
// In the Google flavor, a concrete (non-abstract) dataset must name
|
|
810
|
+
// its backing table; only an abstract one may omit `source`. Relaxed
|
|
811
|
+
// under bindingOptional. Vanilla has no `abstract` and requires
|
|
812
|
+
// `source` unconditionally (see requireVanillaBindings).
|
|
813
|
+
if (extended && !bindingOptional && !abstract &&
|
|
814
|
+
ds.source === undefined) {
|
|
815
|
+
ctx.addIssue({
|
|
816
|
+
code: z.ZodIssueCode.custom,
|
|
817
|
+
path: ['source'],
|
|
818
|
+
message: sourceRequiredMessage(ds.name),
|
|
819
|
+
});
|
|
820
|
+
}
|
|
821
|
+
// The converse is always contradictory: an abstract dataset has no
|
|
822
|
+
// physical table, so a `source` on it would be silently ignored.
|
|
823
|
+
// Reject it always.
|
|
824
|
+
if (abstract && ds.source !== undefined) {
|
|
825
|
+
ctx.addIssue({
|
|
826
|
+
code: z.ZodIssueCode.custom,
|
|
827
|
+
path: ['source'],
|
|
828
|
+
message: `entity '${ds.name}': an abstract entity has no table; ` +
|
|
829
|
+
`remove 'source', or drop 'abstract: true' to bind it to that table`,
|
|
830
|
+
});
|
|
831
|
+
}
|
|
832
|
+
// A field with no `expression` is unbound -- there is no separate
|
|
833
|
+
// flag. One logical model serves several stores from different
|
|
834
|
+
// profiles, so it is not a load error under either
|
|
835
|
+
// `bindingOptional`. validateDeployments rejects one on a concrete
|
|
836
|
+
// leaf entity when a graph deployment uses the model file's own
|
|
837
|
+
// bindings. An `expression` with no SQL is an
|
|
838
|
+
// error: it binds the field to nothing. A vanilla document's string
|
|
839
|
+
// form is reported by convertExpression instead, since that form is
|
|
840
|
+
// the first thing to fix.
|
|
841
|
+
(ds.fields ?? []).forEach((f, i) => {
|
|
842
|
+
if (!extended && typeof f.expression === 'string')
|
|
843
|
+
return;
|
|
844
|
+
const problem = noSqlProblem(f.expression, ds.name);
|
|
845
|
+
if (problem) {
|
|
846
|
+
ctx.addIssue({
|
|
847
|
+
code: z.ZodIssueCode.custom,
|
|
848
|
+
path: ['fields', i, 'expression'],
|
|
849
|
+
message: `field '${ds.name}.${f.name}': ${problem}; write ` +
|
|
850
|
+
`the field's SQL, or remove 'expression' to leave the ` +
|
|
851
|
+
`field unbound`,
|
|
852
|
+
});
|
|
853
|
+
}
|
|
854
|
+
});
|
|
855
|
+
});
|
|
856
|
+
const relationship = z.object({
|
|
857
|
+
name: z.string(),
|
|
858
|
+
from: z.string(),
|
|
859
|
+
to: z.string(),
|
|
860
|
+
from_columns: z.array(z.string()).min(1).optional(),
|
|
861
|
+
to_columns: z.array(z.string()).min(1).optional(),
|
|
862
|
+
// Vanilla rejects a relationship's description after the schema.
|
|
863
|
+
description: placement.relationship.description === 'native' ?
|
|
864
|
+
z.string().optional() :
|
|
865
|
+
unlisted(z.unknown().optional()),
|
|
866
|
+
ai_context: aiContext.optional(),
|
|
867
|
+
...customExtensionsKey,
|
|
868
|
+
})
|
|
869
|
+
.strict()
|
|
870
|
+
.superRefine((r, ctx) => {
|
|
871
|
+
if ((r.from_columns === undefined) !==
|
|
872
|
+
(r.to_columns === undefined)) {
|
|
873
|
+
ctx.addIssue({
|
|
874
|
+
code: z.ZodIssueCode.custom,
|
|
875
|
+
message: halfBoundJoinMessage(r.name),
|
|
876
|
+
});
|
|
877
|
+
}
|
|
878
|
+
});
|
|
879
|
+
const metric = z.object({
|
|
880
|
+
name: z.string(),
|
|
881
|
+
expression: expressionSchema,
|
|
882
|
+
// Validated per flavor in convertMetric (normalizeDataType).
|
|
883
|
+
datatype: z.string().optional(),
|
|
884
|
+
description: z.string().optional(),
|
|
885
|
+
ai_context: aiContext.optional(),
|
|
886
|
+
...customExtensionsKey,
|
|
887
|
+
// An authored anchor is a native extension: extended profile only.
|
|
888
|
+
// Vanilla still declares the key so the refinement below can reject it
|
|
889
|
+
// with a message that names the fix, not a bare "unrecognized key".
|
|
890
|
+
entity: placement.metric.entity === 'native' ?
|
|
891
|
+
z.string().optional() :
|
|
892
|
+
unlisted(z.unknown().optional()),
|
|
893
|
+
})
|
|
894
|
+
.strict()
|
|
895
|
+
.superRefine((mt, ctx) => {
|
|
896
|
+
// A metric's `expression` is required, so one with no SQL is an
|
|
897
|
+
// error. A vanilla string form is reported as for a field.
|
|
898
|
+
const problem = !extended && typeof mt.expression === 'string' ?
|
|
899
|
+
undefined :
|
|
900
|
+
noSqlProblem(mt.expression);
|
|
901
|
+
if (problem) {
|
|
902
|
+
ctx.addIssue({
|
|
903
|
+
code: z.ZodIssueCode.custom,
|
|
904
|
+
path: ['expression'],
|
|
905
|
+
message: `metric '${mt.name}': ${problem}; write the ` +
|
|
906
|
+
`metric's SQL`,
|
|
907
|
+
});
|
|
908
|
+
}
|
|
909
|
+
if (placement.metric.entity === 'native' ||
|
|
910
|
+
mt.entity === undefined) {
|
|
911
|
+
return;
|
|
912
|
+
}
|
|
913
|
+
ctx.addIssue({
|
|
914
|
+
code: z.ZodIssueCode.custom,
|
|
915
|
+
path: ['entity'],
|
|
916
|
+
message: `metric '${mt.name}': 'entity' is a ` +
|
|
917
|
+
`'${exports.GOOGLE_VERSION}' extension. A '${exports.OSSIE_VERSION}' ` +
|
|
918
|
+
`metric takes its entity from the fields it reads, as in ` +
|
|
919
|
+
`'COUNT(orders.id)'. To anchor it explicitly, set 'entity' ` +
|
|
920
|
+
`in a ${exports.GOOGLE_VENDOR} custom_extensions block on the ` +
|
|
921
|
+
`metric, or set the document version to ` +
|
|
922
|
+
`'${exports.GOOGLE_VERSION}'.`,
|
|
923
|
+
});
|
|
924
|
+
});
|
|
925
|
+
// The same shape the superset uses, closed to unknown keys. Restating it
|
|
926
|
+
// here is what let the two disagree: a key added to one and not the other
|
|
927
|
+
// parses in the loader's own type and is rejected by the document schema,
|
|
928
|
+
// which reports it as an unrecognized key on a line the author just wrote
|
|
929
|
+
// out of the guide.
|
|
930
|
+
const parameter = parameterSchema.strict();
|
|
931
|
+
const action = z.object({
|
|
932
|
+
name: z.string(),
|
|
933
|
+
description: z.string().optional(),
|
|
934
|
+
executor: executorSchema.nullish(),
|
|
935
|
+
parameters: z.array(parameter).optional(),
|
|
936
|
+
guards: z.array(z.string()).optional(),
|
|
937
|
+
affects: z.array(affectedConceptSchema).optional(),
|
|
938
|
+
ai_context: aiContextSchema.optional(),
|
|
939
|
+
}).strict();
|
|
940
|
+
// `judgment` is optional here and required in fact, so validateConstraints
|
|
941
|
+
// can name the constraint rather than a position in the document.
|
|
942
|
+
// `expression` is the reserved name of the removed body; see
|
|
943
|
+
// rejectExpressionBody, which is what turns it into a useful error.
|
|
944
|
+
const constraint = z.object({
|
|
945
|
+
name: z.string(),
|
|
946
|
+
expression: unlisted(z.unknown().optional()),
|
|
947
|
+
judgment: z.string().optional(),
|
|
948
|
+
description: z.string().optional(),
|
|
949
|
+
// What a violation does; required. See
|
|
950
|
+
// VIOLATION_EFFECTS.
|
|
951
|
+
on_violation: z.enum(ir_1.VIOLATION_EFFECTS).optional(),
|
|
952
|
+
// How grave it is; no default. See
|
|
953
|
+
// CONSTRAINT_SEVERITIES.
|
|
954
|
+
severity: z.enum(ir_1.CONSTRAINT_SEVERITIES).optional(),
|
|
955
|
+
ai_context: aiContextSchema.optional(),
|
|
956
|
+
})
|
|
957
|
+
.strict()
|
|
958
|
+
.superRefine(rejectExpressionBody);
|
|
959
|
+
const modelExtensions = {
|
|
960
|
+
...(placement.model.deployments === 'native' ? {
|
|
961
|
+
deployments: z.array(deploymentSchema).optional(),
|
|
962
|
+
} :
|
|
963
|
+
{}),
|
|
964
|
+
...(placement.model.actions === 'native' ?
|
|
965
|
+
{ actions: z.array(action).optional() } :
|
|
966
|
+
{}),
|
|
967
|
+
...(placement.model.constraints === 'native' ?
|
|
968
|
+
{ constraints: z.array(constraint).optional() } :
|
|
969
|
+
{}),
|
|
970
|
+
};
|
|
971
|
+
const model = z.object({
|
|
972
|
+
name: z.string(),
|
|
973
|
+
description: z.string().optional(),
|
|
974
|
+
ai_context: aiContext.optional(),
|
|
975
|
+
datasets: z.array(dataset).min(1),
|
|
976
|
+
relationships: z.array(relationship).optional(),
|
|
977
|
+
metrics: z.array(metric).optional(),
|
|
978
|
+
...customExtensionsKey,
|
|
979
|
+
// Native extension keys: extended profile only. `actions`
|
|
980
|
+
// and `constraints` are among them because vanilla Ossie has
|
|
981
|
+
// neither construct and no `custom_extensions` encoding for
|
|
982
|
+
// either, so under OSSIE_VERSION such a key is rejected as
|
|
983
|
+
// unknown rather than silently dropped.
|
|
984
|
+
...modelExtensions,
|
|
985
|
+
}).strict();
|
|
986
|
+
return z
|
|
987
|
+
.object({
|
|
988
|
+
version: z.string(),
|
|
989
|
+
// The format allows exactly one model per document.
|
|
990
|
+
semantic_model: z.array(model).min(1).superRefine((models, ctx) => {
|
|
991
|
+
if (models.length > 1) {
|
|
992
|
+
ctx.addIssue({
|
|
993
|
+
code: z.ZodIssueCode.custom,
|
|
994
|
+
path: [1],
|
|
995
|
+
message: `a document declares one model; split '${models[1].name}' into its own file`,
|
|
996
|
+
});
|
|
997
|
+
}
|
|
998
|
+
}),
|
|
999
|
+
})
|
|
1000
|
+
.strict();
|
|
1001
|
+
}
|
|
1002
|
+
// Four immutable schema shapes exist -- (bindingOptional) x (extended) -- so
|
|
1003
|
+
// build all four once at module load and select between them rather than
|
|
1004
|
+
// reconstructing the whole field/dataset/model/document graph (with fresh
|
|
1005
|
+
// superRefine closures) on every fromDocument call.
|
|
1006
|
+
const documentSchemas = {
|
|
1007
|
+
'false-false': buildDocumentSchema(false, false),
|
|
1008
|
+
'false-true': buildDocumentSchema(false, true),
|
|
1009
|
+
'true-false': buildDocumentSchema(true, false),
|
|
1010
|
+
'true-true': buildDocumentSchema(true, true),
|
|
1011
|
+
};
|
|
1012
|
+
// Returns z.ZodTypeAny so the four distinct strict object types collapse to one
|
|
1013
|
+
// static type here; the document is re-typed as DocumentDoc after a successful
|
|
1014
|
+
// parse (see fromDocument), which the convert functions consume.
|
|
1015
|
+
function makeDocumentSchema(bindingOptional, extended) {
|
|
1016
|
+
return documentSchemas[`${bindingOptional}-${extended}`];
|
|
1017
|
+
}
|
|
1018
|
+
// The AI-first `ai_context` normalized to the IR's common shape: a bare string
|
|
1019
|
+
// is read as `instructions`; a structured object keeps its parts, custom
|
|
1020
|
+
// members included. `examples` needs no filtering: the schema already rejected
|
|
1021
|
+
// any non-string example.
|
|
1022
|
+
function normalizeAiContext(ai) {
|
|
1023
|
+
if (ai === undefined)
|
|
1024
|
+
return {};
|
|
1025
|
+
if (typeof ai === 'string')
|
|
1026
|
+
return { instructions: ai };
|
|
1027
|
+
const out = {};
|
|
1028
|
+
if (ai.instructions)
|
|
1029
|
+
out.instructions = ai.instructions;
|
|
1030
|
+
if (ai.synonyms && ai.synonyms.length)
|
|
1031
|
+
out.synonyms = [...new Set(ai.synonyms)];
|
|
1032
|
+
if (ai.examples && ai.examples.length)
|
|
1033
|
+
out.examples = ai.examples;
|
|
1034
|
+
if (ai.additionalProperties) {
|
|
1035
|
+
out.additionalProperties = ai.additionalProperties;
|
|
1036
|
+
}
|
|
1037
|
+
return out;
|
|
1038
|
+
}
|
|
1039
|
+
// Normalizes `ai_context` and returns it only when it carries something, so the
|
|
1040
|
+
// IR omits empty aiContext objects. Custom members count: an `ai_context` that
|
|
1041
|
+
// carries only those is not blank.
|
|
1042
|
+
function aiContextOrUndefined(ai) {
|
|
1043
|
+
const ctx = normalizeAiContext(ai);
|
|
1044
|
+
return (ctx.instructions || ctx.synonyms || ctx.examples ||
|
|
1045
|
+
ctx.additionalProperties) ?
|
|
1046
|
+
ctx :
|
|
1047
|
+
undefined;
|
|
1048
|
+
}
|
|
1049
|
+
// Preserves vendor `custom_extensions` verbatim on the IR (`vendor_name` ->
|
|
1050
|
+
// `vendorName`; `data` kept as the opaque, vendor-serialized string) so nothing
|
|
1051
|
+
// is lost and a 1P round-trip stays lossless.
|
|
1052
|
+
function toCustomExtensions(exts) {
|
|
1053
|
+
if (!exts || !exts.length)
|
|
1054
|
+
return undefined;
|
|
1055
|
+
return exts.map(e => ({ vendorName: e.vendor_name, data: e.data }));
|
|
1056
|
+
}
|
|
1057
|
+
// Composes a single description string from ordered parts, dropping empties.
|
|
1058
|
+
// Parts are separated by blank lines so a base description and derived markers
|
|
1059
|
+
// read as distinct paragraphs in the emitted metadata. AI-first annotations
|
|
1060
|
+
// (instructions / synonyms / examples) are NOT folded in here — they are
|
|
1061
|
+
// carried structurally on the IR (aiContext) so an emitter can route them to
|
|
1062
|
+
// their own aspects.
|
|
1063
|
+
function composeDescription(...parts) {
|
|
1064
|
+
const kept = parts.map(p => (p === undefined ? undefined : p.trim()))
|
|
1065
|
+
.filter((p) => !!p);
|
|
1066
|
+
return kept.length ? kept.join('\n\n') : undefined;
|
|
1067
|
+
}
|
|
1068
|
+
// The vendor tag for Google-specific extension blocks (kept in sync with the
|
|
1069
|
+
// deploy leg's reader). A model-level `deployments:` list folds into one.
|
|
1070
|
+
exports.GOOGLE_VENDOR = 'GOOGLE';
|
|
1071
|
+
// Rewrites the author-friendly sugar forms into the canonical wire shape the
|
|
1072
|
+
// schema validates, and rejects deprecated or flavor-mismatched model keys up
|
|
1073
|
+
// front with actionable messages: the removed `deployment_target:` key,
|
|
1074
|
+
// inline `profiles:` in either flavor, and `deployments:` or
|
|
1075
|
+
// `entities:` in vanilla. Under the extended profile, `entities:` is rewritten
|
|
1076
|
+
// to `datasets:` and `deployments:` is left in place here and folded into a
|
|
1077
|
+
// GOOGLE `custom_extensions` block on the IR after validation (see
|
|
1078
|
+
// convertModel). Operates on the parsed document before validation.
|
|
1079
|
+
function normalizeDocumentSugars(doc, extended) {
|
|
1080
|
+
if (!doc || typeof doc !== 'object')
|
|
1081
|
+
return doc;
|
|
1082
|
+
const cloned = structuredClone(doc);
|
|
1083
|
+
const models = cloned.semantic_model;
|
|
1084
|
+
if (!Array.isArray(models))
|
|
1085
|
+
return cloned;
|
|
1086
|
+
for (const m of models) {
|
|
1087
|
+
if (m && typeof m === 'object') {
|
|
1088
|
+
normalizeModelSugars(m, extended);
|
|
1089
|
+
}
|
|
1090
|
+
}
|
|
1091
|
+
return cloned;
|
|
1092
|
+
}
|
|
1093
|
+
function normalizeModelSugars(m, extended) {
|
|
1094
|
+
const label = typeof m.name === 'string' ? `model '${m.name}'` : 'model';
|
|
1095
|
+
const placement = exports.FORMAT_VERSION_PLACEMENT[extended ? exports.GOOGLE_VERSION : exports.OSSIE_VERSION];
|
|
1096
|
+
// `deployment_target` has been replaced by the named `deployments:` list in
|
|
1097
|
+
// the extended profile (and was never in vanilla).
|
|
1098
|
+
if (m.deployment_target !== undefined) {
|
|
1099
|
+
if (!extended) {
|
|
1100
|
+
throw new SemanticModelError(`Semantic model load error: ${label}: 'deployment_target' is not ` +
|
|
1101
|
+
`in the open format ('${exports.OSSIE_VERSION}'); ` +
|
|
1102
|
+
`${DEPLOYMENTS_VERSION_HINT}`);
|
|
1103
|
+
}
|
|
1104
|
+
if (m.deployments !== undefined) {
|
|
1105
|
+
throw new SemanticModelError(`Semantic model load error: ${label}: 'deployment_target' is no ` +
|
|
1106
|
+
`longer supported; remove it, and list its target under ` +
|
|
1107
|
+
`'deployments' if it is not there already.`);
|
|
1108
|
+
}
|
|
1109
|
+
const target = typeof m.deployment_target === 'string' ?
|
|
1110
|
+
asResourceName(m.deployment_target) :
|
|
1111
|
+
'<target>';
|
|
1112
|
+
throw new SemanticModelError(`Semantic model load error: ${label}: 'deployment_target' is no ` +
|
|
1113
|
+
`longer supported; migrate to the named 'deployments' list:\n` +
|
|
1114
|
+
` deployments:\n` +
|
|
1115
|
+
` - name: default\n` +
|
|
1116
|
+
` target: ${target}`);
|
|
1117
|
+
}
|
|
1118
|
+
if (placement.model.deployments !== 'native' && m.deployments !== undefined) {
|
|
1119
|
+
throw new SemanticModelError(`Semantic model load error: ${label}: 'deployments' is not in the ` +
|
|
1120
|
+
`open format ('${exports.OSSIE_VERSION}'); ${DEPLOYMENTS_VERSION_HINT}`);
|
|
1121
|
+
}
|
|
1122
|
+
// In the Google flavor, binding profiles live in sidecar files
|
|
1123
|
+
// (`<model>.profile.<name>.yaml`); in vanilla they live in a GOOGLE
|
|
1124
|
+
// custom_extensions block on the model. Neither flavor takes a native
|
|
1125
|
+
// `profiles:` key on the model.
|
|
1126
|
+
if (m.profiles !== undefined) {
|
|
1127
|
+
const modelFile = typeof m.name === 'string' ? m.name : '<model>';
|
|
1128
|
+
if (placement.model.profiles === 'file') {
|
|
1129
|
+
throw new SemanticModelError(`Semantic model load error: ${label}: inline 'profiles' are not ` +
|
|
1130
|
+
`supported in a '${exports.GOOGLE_VERSION}' document; write each binding ` +
|
|
1131
|
+
`profile in its own sidecar file ` +
|
|
1132
|
+
`'${modelFile}.profile.<name>.yaml'.`);
|
|
1133
|
+
}
|
|
1134
|
+
throw new SemanticModelError(`Semantic model load error: ${label}: 'profiles' is not in the ` +
|
|
1135
|
+
`open format ('${exports.OSSIE_VERSION}'); list 'profiles' in a ` +
|
|
1136
|
+
`${exports.GOOGLE_VENDOR} custom_extensions block on the model, or set ` +
|
|
1137
|
+
`version: '${exports.GOOGLE_VERSION}' and write each binding profile in its ` +
|
|
1138
|
+
`own sidecar file '${modelFile}.profile.<name>.yaml'.`);
|
|
1139
|
+
}
|
|
1140
|
+
// `entities` is a native alias for `datasets`, accepted only under the
|
|
1141
|
+
// extended profile. Under vanilla Ossie the key is unknown; surface a clear
|
|
1142
|
+
// message here rather than the strict schema's generic "unrecognized key".
|
|
1143
|
+
if (m.entities !== undefined) {
|
|
1144
|
+
if (!extended) {
|
|
1145
|
+
throw new SemanticModelError(`Semantic model load error: ${label}: 'entities' is a ` +
|
|
1146
|
+
`'${exports.GOOGLE_VERSION}' extension; use 'datasets', or set the document ` +
|
|
1147
|
+
`version to '${exports.GOOGLE_VERSION}'.`);
|
|
1148
|
+
}
|
|
1149
|
+
if (m.datasets !== undefined) {
|
|
1150
|
+
throw new SemanticModelError(`Semantic model load error: ${label}: set either ` +
|
|
1151
|
+
`'entities' or 'datasets', not both (they are the same key).`);
|
|
1152
|
+
}
|
|
1153
|
+
m.datasets = m.entities;
|
|
1154
|
+
delete m.entities;
|
|
1155
|
+
}
|
|
1156
|
+
}
|
|
1157
|
+
// Builds the GOOGLE custom-extension block that carries a model's deployment
|
|
1158
|
+
// target URIs on the IR (the form the deploy leg reads). The native
|
|
1159
|
+
// `deployments:` list folds into this block after validation (see
|
|
1160
|
+
// convertModel). The extended profile rejects any authored GOOGLE block (see
|
|
1161
|
+
// rejectGoogleBlock), so there is never a pre-existing one to reconcile with.
|
|
1162
|
+
function deploymentTargetsExtension(targets) {
|
|
1163
|
+
return {
|
|
1164
|
+
vendorName: exports.GOOGLE_VENDOR,
|
|
1165
|
+
data: JSON.stringify({ deploymentTargets: targets }),
|
|
1166
|
+
};
|
|
1167
|
+
}
|
|
1168
|
+
// True when `vendorName` names the GOOGLE vendor, in any letter case.
|
|
1169
|
+
function isGoogleVendor(vendorName) {
|
|
1170
|
+
return typeof vendorName === 'string' &&
|
|
1171
|
+
vendorName.toUpperCase() === exports.GOOGLE_VENDOR;
|
|
1172
|
+
}
|
|
1173
|
+
// True when a block list names the GOOGLE vendor.
|
|
1174
|
+
function hasGoogleBlock(exts) {
|
|
1175
|
+
return !!exts?.some(e => isGoogleVendor(e.vendor_name));
|
|
1176
|
+
}
|
|
1177
|
+
// Rejects a GOOGLE `custom_extensions` block where the flavor gives it no
|
|
1178
|
+
// meaning. In the Google flavor that is everywhere: each thing a GOOGLE block
|
|
1179
|
+
// would carry is a native key there. In vanilla it is only a field, the one
|
|
1180
|
+
// level the format defines no GOOGLE payload for. Third-party blocks are
|
|
1181
|
+
// accepted at every level in both flavors.
|
|
1182
|
+
function rejectGoogleBlock(exts, version, level, ctx) {
|
|
1183
|
+
if (!hasGoogleBlock(exts))
|
|
1184
|
+
return;
|
|
1185
|
+
if (version === exports.GOOGLE_VERSION && level === 'field') {
|
|
1186
|
+
throw new SemanticModelError(`Semantic model load error: ${ctx}: a '${exports.GOOGLE_VENDOR}' ` +
|
|
1187
|
+
`custom_extensions block is not accepted on a field; the format ` +
|
|
1188
|
+
`defines no field-level ${exports.GOOGLE_VENDOR} extension.`);
|
|
1189
|
+
}
|
|
1190
|
+
if (version === exports.GOOGLE_VERSION) {
|
|
1191
|
+
throw new SemanticModelError(`Semantic model load error: ${ctx}: a '${exports.GOOGLE_VENDOR}' ` +
|
|
1192
|
+
`custom_extensions block is not accepted in a '${exports.GOOGLE_VERSION}' ` +
|
|
1193
|
+
`document. Everything it would carry is a native key in this ` +
|
|
1194
|
+
`flavor; write it as one.`);
|
|
1195
|
+
}
|
|
1196
|
+
if (level === 'field') {
|
|
1197
|
+
throw new SemanticModelError(`Semantic model load error: ${ctx}: a '${exports.OSSIE_VERSION}' document ` +
|
|
1198
|
+
`cannot carry a '${exports.GOOGLE_VENDOR}' custom_extensions block on a ` +
|
|
1199
|
+
`field; the format defines a ${exports.GOOGLE_VENDOR} payload only on a ` +
|
|
1200
|
+
`model, dataset, relationship or metric.`);
|
|
1201
|
+
}
|
|
1202
|
+
}
|
|
1203
|
+
// How the Google flavor validates each block key. A profile is read
|
|
1204
|
+
// as a profile file is instead (readBlockProfiles).
|
|
1205
|
+
const GOOGLE_BLOCK_SCHEMAS = {
|
|
1206
|
+
deployments: z.array(deploymentSchema),
|
|
1207
|
+
extends: z.array(z.string()),
|
|
1208
|
+
description: z.string(),
|
|
1209
|
+
entity: z.string(),
|
|
1210
|
+
};
|
|
1211
|
+
// Reads the GOOGLE blocks among `exts`, the `custom_extensions` of the object
|
|
1212
|
+
// `ctx` at `level`, and returns the values they carry and the blocks the object
|
|
1213
|
+
// keeps. Only a vanilla document gives a GOOGLE block meaning, and
|
|
1214
|
+
// rejectGoogleBlock rejects one in the Google flavor. Each key is validated as
|
|
1215
|
+
// the Google flavor validates its native key, apart from `profiles`, whose
|
|
1216
|
+
// entries are read as profile files are. A block that is read is not kept, so
|
|
1217
|
+
// an object keeps its third-party blocks. Throws, naming the object, on data
|
|
1218
|
+
// that is not a JSON object, a key one block sets twice, a key the level does
|
|
1219
|
+
// not take, `abstract` included, a key two blocks both set, a value the Google
|
|
1220
|
+
// flavor rejects, and a model's `deploymentTargets`, which `deployments`
|
|
1221
|
+
// replaces.
|
|
1222
|
+
function readGoogleBlocks(exts, version, level, ctx) {
|
|
1223
|
+
rejectGoogleBlock(exts, version, level, ctx);
|
|
1224
|
+
const error = (problem) => new SemanticModelError(`Semantic model load error: ${ctx}: ${problem}`);
|
|
1225
|
+
const block = `${exports.GOOGLE_VENDOR} custom_extensions block`;
|
|
1226
|
+
const levelPlacement = exports.FORMAT_VERSION_PLACEMENT[version][level];
|
|
1227
|
+
const values = {};
|
|
1228
|
+
const kept = [];
|
|
1229
|
+
for (const e of exts ?? []) {
|
|
1230
|
+
if (!isGoogleVendor(e.vendor_name)) {
|
|
1231
|
+
kept.push({ vendorName: e.vendor_name, data: e.data });
|
|
1232
|
+
continue;
|
|
1233
|
+
}
|
|
1234
|
+
let data;
|
|
1235
|
+
try {
|
|
1236
|
+
data = JSON.parse(e.data);
|
|
1237
|
+
}
|
|
1238
|
+
catch {
|
|
1239
|
+
// Reported below, as data that is not an object is.
|
|
1240
|
+
}
|
|
1241
|
+
if (!isRecord(data)) {
|
|
1242
|
+
throw error(`the 'data' of a ${block} must be a JSON object.`);
|
|
1243
|
+
}
|
|
1244
|
+
// JSON.parse keeps the last of a repeated key. The YAML a native key is
|
|
1245
|
+
// read from rejects a repeated key, so a block does too.
|
|
1246
|
+
if (setsAKeyTwice(e.data)) {
|
|
1247
|
+
throw error(`the 'data' of a ${block} sets a key twice; set each key once.`);
|
|
1248
|
+
}
|
|
1249
|
+
for (const [key, value] of Object.entries(data)) {
|
|
1250
|
+
if (level === 'model' && key === 'deploymentTargets') {
|
|
1251
|
+
throw error(`'${key}' is no longer supported; use 'deployments' inside the ` +
|
|
1252
|
+
`${block} instead, for example ` +
|
|
1253
|
+
`{"deployments": [{"name": "default", "target": "<target>"}]}.`);
|
|
1254
|
+
}
|
|
1255
|
+
if (level === 'dataset' && key === 'abstract') {
|
|
1256
|
+
throw error(`a ${block} cannot declare 'abstract' in a '${exports.OSSIE_VERSION}' ` +
|
|
1257
|
+
`document. An abstract dataset has no table, and every ` +
|
|
1258
|
+
`'${exports.OSSIE_VERSION}' dataset needs a 'source'. A model that needs ` +
|
|
1259
|
+
`an abstract entity uses '${exports.GOOGLE_VERSION}'.`);
|
|
1260
|
+
}
|
|
1261
|
+
const allowed = Object.keys(levelPlacement)
|
|
1262
|
+
.filter(k => levelPlacement[k] === 'block');
|
|
1263
|
+
if (!allowed.includes(key)) {
|
|
1264
|
+
throw error(`'${key}' is not a key a ${block} takes on a ${level}; it takes ` +
|
|
1265
|
+
`${allowed.map(k => `'${k}'`).join(' and ')}.`);
|
|
1266
|
+
}
|
|
1267
|
+
if (Object.hasOwn(values, key)) {
|
|
1268
|
+
throw error(`two ${block}s both set '${key}'; set it in one block` +
|
|
1269
|
+
(key === 'extends' ? `, listing every parent there.` : `.`));
|
|
1270
|
+
}
|
|
1271
|
+
if (key === 'profiles') {
|
|
1272
|
+
values.profiles = readBlockProfiles(value, error, ctx);
|
|
1273
|
+
continue;
|
|
1274
|
+
}
|
|
1275
|
+
const parsed = GOOGLE_BLOCK_SCHEMAS[key].safeParse(value);
|
|
1276
|
+
if (!parsed.success) {
|
|
1277
|
+
throw error(keyProblems(parsed.error, GOOGLE_BLOCK_SCHEMAS[key], key, value, `the ${block}`).replaceAll('\n', `\n${ctx}: `));
|
|
1278
|
+
}
|
|
1279
|
+
values[key] = parsed.data;
|
|
1280
|
+
}
|
|
1281
|
+
}
|
|
1282
|
+
return {
|
|
1283
|
+
values: values,
|
|
1284
|
+
kept: kept.length ? kept : undefined,
|
|
1285
|
+
};
|
|
1286
|
+
}
|
|
1287
|
+
// Reads a GOOGLE block's `profiles`: a list of profiles, each read as a profile
|
|
1288
|
+
// file is. A name follows the rules the layout gives a profile file's name: it
|
|
1289
|
+
// passes the profile rule in names.ts, it is not `default` in any letter case,
|
|
1290
|
+
// and no two names are equal ignoring case.
|
|
1291
|
+
function readBlockProfiles(value, error, ctx) {
|
|
1292
|
+
const block = `${exports.GOOGLE_VENDOR} custom_extensions block`;
|
|
1293
|
+
if (!Array.isArray(value)) {
|
|
1294
|
+
throw error(`'profiles' in a ${block} must be a list.`);
|
|
1295
|
+
}
|
|
1296
|
+
const profiles = value.map(p => {
|
|
1297
|
+
if (!isRecord(p) || typeof p.name !== 'string') {
|
|
1298
|
+
throw error(`each profile in a ${block} must be a mapping with a 'name'.`);
|
|
1299
|
+
}
|
|
1300
|
+
if ((0, names_1.nameProblem)('profile', p.name)) {
|
|
1301
|
+
throw error(`profile name '${p.name}' in a ${block} is not valid; a profile ` +
|
|
1302
|
+
`name is a letter followed by up to 499 letters, digits, ` +
|
|
1303
|
+
`underscores and hyphens.`);
|
|
1304
|
+
}
|
|
1305
|
+
if (p.name.toLowerCase() === resolve_profiles_1.DEFAULT_PROFILE) {
|
|
1306
|
+
throw error(`a profile in a ${block} may not be named '${p.name}'; ` +
|
|
1307
|
+
`'${resolve_profiles_1.DEFAULT_PROFILE}' names the model's inline bindings.`);
|
|
1308
|
+
}
|
|
1309
|
+
// profileSpecOf rejects these two keys too, in words about a profile file.
|
|
1310
|
+
for (const key of ['version', 'semantic_model']) {
|
|
1311
|
+
if (p[key] !== undefined) {
|
|
1312
|
+
throw error(`profile '${p.name}' in a ${block} sets '${key}', which a ` +
|
|
1313
|
+
`profile may not set.`);
|
|
1314
|
+
}
|
|
1315
|
+
}
|
|
1316
|
+
try {
|
|
1317
|
+
return (0, resolve_profiles_1.profileSpecOf)(p, p.name);
|
|
1318
|
+
}
|
|
1319
|
+
catch (err) {
|
|
1320
|
+
if (!(err instanceof SemanticModelError))
|
|
1321
|
+
throw err;
|
|
1322
|
+
throw error(err.message);
|
|
1323
|
+
}
|
|
1324
|
+
});
|
|
1325
|
+
rejectDuplicateNames(profiles.map(p => p.name), 'profile name', ctx);
|
|
1326
|
+
const byLowerName = new Map();
|
|
1327
|
+
for (const { name } of profiles) {
|
|
1328
|
+
const twin = byLowerName.get(name.toLowerCase());
|
|
1329
|
+
if (twin !== undefined) {
|
|
1330
|
+
throw error(`profiles '${twin}' and '${name}' in a ${block} name the same ` +
|
|
1331
|
+
`profile; profile names are unique ignoring case.`);
|
|
1332
|
+
}
|
|
1333
|
+
byLowerName.set(name.toLowerCase(), name);
|
|
1334
|
+
}
|
|
1335
|
+
return profiles;
|
|
1336
|
+
}
|
|
1337
|
+
// Whether the JSON text `data` sets a key twice.
|
|
1338
|
+
function setsAKeyTwice(data) {
|
|
1339
|
+
return yaml.parseDocument(data, yaml_options_1.YAML_OPTIONS)
|
|
1340
|
+
.errors.some(err => err.code === 'DUPLICATE_KEY');
|
|
1341
|
+
}
|
|
1342
|
+
// Whether the JSON text `data` sets `profiles` twice, or sets a key twice
|
|
1343
|
+
// inside its profiles.
|
|
1344
|
+
function setsAProfilesKeyTwice(data) {
|
|
1345
|
+
const doc = yaml.parseDocument(data, yaml_options_1.YAML_OPTIONS);
|
|
1346
|
+
if (!yaml.isMap(doc.contents))
|
|
1347
|
+
return false;
|
|
1348
|
+
const pairs = doc.contents.items.filter(p => yaml.isScalar(p.key) && p.key.value === 'profiles');
|
|
1349
|
+
const range = yaml.isNode(pairs[0]?.value) ? pairs[0].value.range : undefined;
|
|
1350
|
+
return pairs.length > 1 ||
|
|
1351
|
+
(!!range && setsAKeyTwice(data.slice(range[0], range[1])));
|
|
1352
|
+
}
|
|
1353
|
+
// `v` as a list: a list as it is, one value as a list of it, and nothing as an
|
|
1354
|
+
// empty list.
|
|
1355
|
+
function asList(v) {
|
|
1356
|
+
return Array.isArray(v) ? v : v === undefined || v === null ? [] : [v];
|
|
1357
|
+
}
|
|
1358
|
+
// The profiles the GOOGLE block `e` lists, whatever its shape: in data written
|
|
1359
|
+
// as a JSON string or as a YAML mapping, or beside `vendor_name` with no data.
|
|
1360
|
+
// `json` is them as the data of a block that lists only them. 'none' when the
|
|
1361
|
+
// block lists no profiles. 'unread' when it has neither data nor profiles
|
|
1362
|
+
// beside `vendor_name`, when its data is not a JSON object, or when its
|
|
1363
|
+
// profiles have no JSON form, as with a recursive alias.
|
|
1364
|
+
function listedProfiles(e) {
|
|
1365
|
+
if (e.data == null && e.profiles === undefined)
|
|
1366
|
+
return 'unread';
|
|
1367
|
+
let data = e.data == null ? { profiles: e.profiles } : e.data;
|
|
1368
|
+
if (typeof data === 'string') {
|
|
1369
|
+
try {
|
|
1370
|
+
data = JSON.parse(data);
|
|
1371
|
+
}
|
|
1372
|
+
catch {
|
|
1373
|
+
return 'unread';
|
|
1374
|
+
}
|
|
1375
|
+
}
|
|
1376
|
+
if (!isRecord(data))
|
|
1377
|
+
return 'unread';
|
|
1378
|
+
if (data.profiles === undefined)
|
|
1379
|
+
return 'none';
|
|
1380
|
+
try {
|
|
1381
|
+
return { json: JSON.stringify({ profiles: data.profiles }) };
|
|
1382
|
+
}
|
|
1383
|
+
catch {
|
|
1384
|
+
return 'unread';
|
|
1385
|
+
}
|
|
1386
|
+
}
|
|
1387
|
+
/**
|
|
1388
|
+
* The profiles the model in a vanilla document carries in its GOOGLE blocks,
|
|
1389
|
+
* read as a load reads them, without loading the rest of the model. So a
|
|
1390
|
+
* caller finds a profile even in a model that does not load, as it finds a
|
|
1391
|
+
* profile file. Only the profiles in the model's GOOGLE blocks are read, and
|
|
1392
|
+
* the load reports a problem with any other key. Empty for a document that is
|
|
1393
|
+
* not a vanilla model.
|
|
1394
|
+
*
|
|
1395
|
+
* A profile's field binding that is not blank needs the SQL parser, so call
|
|
1396
|
+
* `loadSqlEngine()` first. Without it, this throws the parser's plain Error.
|
|
1397
|
+
*
|
|
1398
|
+
* A GOOGLE block whose shape the load rejects is read too, and the load
|
|
1399
|
+
* reports its shape. Its profiles may be in data written as a JSON string or
|
|
1400
|
+
* as a YAML mapping, or listed beside `vendor_name` with no data. Throws
|
|
1401
|
+
* SemanticModelError on profiles a load rejects, on data that sets `profiles`
|
|
1402
|
+
* twice or a key twice inside them, and on data that is not a JSON object in
|
|
1403
|
+
* a block whose shape the load accepts.
|
|
1404
|
+
*
|
|
1405
|
+
* Text that is not valid YAML is read as far as it parses. A document whose
|
|
1406
|
+
* root keys are all strings, and that sets `version` once, to a string other
|
|
1407
|
+
* than the vanilla one, gives none when each YAML error is a key set twice, an
|
|
1408
|
+
* alias whose anchor is not set, or more aliases than a load expands. Otherwise
|
|
1409
|
+
* its profiles are read as in text that parses, and only when that reading is
|
|
1410
|
+
* unambiguous. Each YAML error is a key set twice or an alias whose anchor is
|
|
1411
|
+
* not set, since any other error, such as a bad indent, can move the blocks,
|
|
1412
|
+
* cut them short or change the version. The document sets `version` once, to
|
|
1413
|
+
* the vanilla one, and `semantic_model` at most once. Its model sets
|
|
1414
|
+
* `custom_extensions` at most once, and each block that is a mapping sets
|
|
1415
|
+
* `vendor_name` once, to a string. A GOOGLE block sets `data` and `profiles` at
|
|
1416
|
+
* most once each, and holds no alias. Every key of a mapping on that path is a
|
|
1417
|
+
* string, so not an alias or a merge key, and no value on it is an alias. No
|
|
1418
|
+
* key is set twice inside `custom_extensions`. When any of these fails, when a
|
|
1419
|
+
* GOOGLE block cannot be read, or when a load rejects its profiles, it throws
|
|
1420
|
+
* the error a load reports for the text. So does a vanilla model whose text has
|
|
1421
|
+
* more aliases than a load expands, or a merge key whose value is not a
|
|
1422
|
+
* mapping.
|
|
1423
|
+
*/
|
|
1424
|
+
function googleBlockProfiles(text) {
|
|
1425
|
+
let doc;
|
|
1426
|
+
try {
|
|
1427
|
+
doc = yaml.parse(text, yaml_options_1.YAML_OPTIONS);
|
|
1428
|
+
}
|
|
1429
|
+
catch (err) {
|
|
1430
|
+
return googleBlockProfilesPastYamlError(text, err?.message ?? String(err));
|
|
1431
|
+
}
|
|
1432
|
+
return blockProfilesOf(doc);
|
|
1433
|
+
}
|
|
1434
|
+
// The profiles in the GOOGLE blocks of the model in the parsed document `doc`,
|
|
1435
|
+
// read as googleBlockProfiles reads text that parses. When `unreadable` is
|
|
1436
|
+
// set, a GOOGLE block whose profiles cannot be read throws it.
|
|
1437
|
+
function blockProfilesOf(doc, unreadable) {
|
|
1438
|
+
if (!isRecord(doc) || doc.version !== exports.OSSIE_VERSION)
|
|
1439
|
+
return [];
|
|
1440
|
+
// The load rejects a `semantic_model` written as one mapping, and reports it.
|
|
1441
|
+
const m = asList(doc.semantic_model)[0];
|
|
1442
|
+
if (!isRecord(m))
|
|
1443
|
+
return [];
|
|
1444
|
+
const blocks = [];
|
|
1445
|
+
// The load rejects a `custom_extensions` written as one mapping, and reports
|
|
1446
|
+
// it. The mapping is read as one block.
|
|
1447
|
+
for (const e of asList(m.custom_extensions)) {
|
|
1448
|
+
if (!isRecord(e) || !isGoogleVendor(e.vendor_name))
|
|
1449
|
+
continue;
|
|
1450
|
+
const ext = customExtensionSchema.safeParse(e);
|
|
1451
|
+
const listed = listedProfiles(e);
|
|
1452
|
+
if (listed === 'unread' && unreadable)
|
|
1453
|
+
throw unreadable;
|
|
1454
|
+
if (typeof listed === 'object') {
|
|
1455
|
+
// Data that sets `profiles` twice, or a key twice inside them, is
|
|
1456
|
+
// rejected as for any block.
|
|
1457
|
+
const data = typeof e.data === 'string' && setsAProfilesKeyTwice(e.data) ?
|
|
1458
|
+
e.data :
|
|
1459
|
+
listed.json;
|
|
1460
|
+
blocks.push({ vendor_name: exports.GOOGLE_VENDOR, data });
|
|
1461
|
+
}
|
|
1462
|
+
else if (listed === 'unread' && ext.success) {
|
|
1463
|
+
// Data that is not a JSON object, rejected as for any block.
|
|
1464
|
+
blocks.push(ext.data);
|
|
1465
|
+
}
|
|
1466
|
+
}
|
|
1467
|
+
const label = typeof m.name === 'string' ? `model '${m.name}'` : 'model';
|
|
1468
|
+
return readGoogleBlocks(blocks, exports.OSSIE_VERSION, 'model', label)
|
|
1469
|
+
.values.profiles ??
|
|
1470
|
+
[];
|
|
1471
|
+
}
|
|
1472
|
+
// The profiles the model in a vanilla document carries in its GOOGLE blocks,
|
|
1473
|
+
// when its text is not valid YAML. `parseError` is the error a load reports
|
|
1474
|
+
// for the text. A document whose version is read past the error as another
|
|
1475
|
+
// version gives none (see versionPastYamlError). Otherwise the profiles are
|
|
1476
|
+
// read as in text that parses, when that reading is unambiguous (see
|
|
1477
|
+
// blocksReadOneWay). When it is not, when a GOOGLE block cannot be read, or
|
|
1478
|
+
// when a load rejects its profiles, throws the load's error for the text.
|
|
1479
|
+
function googleBlockProfilesPastYamlError(text, parseError) {
|
|
1480
|
+
const yamlError = new SemanticModelError(`Semantic model load error: could not parse input: ${parseError}`);
|
|
1481
|
+
// An alias whose anchor is not set stays an alias here, so the checks see
|
|
1482
|
+
// it.
|
|
1483
|
+
const parsed = yaml.parseDocument(text, { ...yaml_options_1.YAML_OPTIONS, uniqueKeys: false });
|
|
1484
|
+
const errors = yaml.parseDocument(text, yaml_options_1.YAML_OPTIONS).errors;
|
|
1485
|
+
const version = versionPastYamlError(parsed, errors);
|
|
1486
|
+
if (version !== undefined && version !== exports.OSSIE_VERSION)
|
|
1487
|
+
return [];
|
|
1488
|
+
if (version === undefined || !blocksReadOneWay(parsed, errors)) {
|
|
1489
|
+
throw yamlError;
|
|
1490
|
+
}
|
|
1491
|
+
let doc;
|
|
1492
|
+
try {
|
|
1493
|
+
doc = (0, yaml_options_1.parseLeniently)(text).toJS();
|
|
1494
|
+
}
|
|
1495
|
+
catch {
|
|
1496
|
+
// More aliases than a load expands, or a merge key whose value is not a
|
|
1497
|
+
// mapping.
|
|
1498
|
+
throw yamlError;
|
|
1499
|
+
}
|
|
1500
|
+
try {
|
|
1501
|
+
return blockProfilesOf(doc, yamlError);
|
|
1502
|
+
}
|
|
1503
|
+
catch (err) {
|
|
1504
|
+
if (err instanceof SemanticModelError)
|
|
1505
|
+
throw yamlError;
|
|
1506
|
+
throw err;
|
|
1507
|
+
}
|
|
1508
|
+
}
|
|
1509
|
+
/**
|
|
1510
|
+
* The `version` the model document `text` sets, as googleBlockProfiles reads
|
|
1511
|
+
* it. Valid YAML gives its version. Past a YAML error, the version is read
|
|
1512
|
+
* only when every root key is a string and the root sets `version` once, to a
|
|
1513
|
+
* string. Each error must be a key set twice, an alias whose anchor is not
|
|
1514
|
+
* set, or more aliases than a load expands. Undefined otherwise, and when the
|
|
1515
|
+
* text sets no string version.
|
|
1516
|
+
*/
|
|
1517
|
+
function modelDocumentVersion(text) {
|
|
1518
|
+
let doc;
|
|
1519
|
+
try {
|
|
1520
|
+
doc = yaml.parse(text, yaml_options_1.YAML_OPTIONS);
|
|
1521
|
+
}
|
|
1522
|
+
catch {
|
|
1523
|
+
return versionPastYamlError(yaml.parseDocument(text, { ...yaml_options_1.YAML_OPTIONS, uniqueKeys: false }), yaml.parseDocument(text, yaml_options_1.YAML_OPTIONS).errors);
|
|
1524
|
+
}
|
|
1525
|
+
return isRecord(doc) && typeof doc.version === 'string' ? doc.version :
|
|
1526
|
+
undefined;
|
|
1527
|
+
}
|
|
1528
|
+
// The value of `key` in `node`, read past a YAML error: undefined when `node`
|
|
1529
|
+
// is not a mapping, does not set `key`, or writes `key` with no colon, as in
|
|
1530
|
+
// `{data}`, and null when the mapping has a key that is not a string, such as
|
|
1531
|
+
// an alias or a merge key, sets `key` twice, or gives it an alias.
|
|
1532
|
+
function onlyValue(node, key) {
|
|
1533
|
+
if (!yaml.isMap(node))
|
|
1534
|
+
return undefined;
|
|
1535
|
+
if (!node.items.every(p => yaml.isScalar(p.key) && typeof p.key.value === 'string')) {
|
|
1536
|
+
return null;
|
|
1537
|
+
}
|
|
1538
|
+
const pairs = node.items.filter(p => p.key.value === key);
|
|
1539
|
+
if (pairs.length > 1 || yaml.isAlias(pairs[0]?.value))
|
|
1540
|
+
return null;
|
|
1541
|
+
return pairs[0]?.value ?? undefined;
|
|
1542
|
+
}
|
|
1543
|
+
// The version a model document sets, read past a YAML error. `parsed` is the
|
|
1544
|
+
// text parsed with keys set twice allowed, and `errors` are the errors of the
|
|
1545
|
+
// strict parse. The version is read only when each error is a key set twice,
|
|
1546
|
+
// every root key is a string, and the root sets `version` once, to a string.
|
|
1547
|
+
// Any other YAML error can change the version or hide a second one.
|
|
1548
|
+
function versionPastYamlError(parsed, errors) {
|
|
1549
|
+
const version = onlyValue(parsed.contents, 'version');
|
|
1550
|
+
if (!yaml.isScalar(version) || typeof version.value !== 'string' ||
|
|
1551
|
+
errors.some(err => err.code !== 'DUPLICATE_KEY')) {
|
|
1552
|
+
return undefined;
|
|
1553
|
+
}
|
|
1554
|
+
return version.value;
|
|
1555
|
+
}
|
|
1556
|
+
// Whether a text that is not valid YAML gives one reading of its model's GOOGLE
|
|
1557
|
+
// blocks. `parsed` is the text parsed with keys set twice allowed, and `errors`
|
|
1558
|
+
// are the errors of the strict parse, each a key set twice. The document sets
|
|
1559
|
+
// `semantic_model` at most once, its first model sets `custom_extensions` at
|
|
1560
|
+
// most once, and each block that is a mapping sets `vendor_name` once, to a
|
|
1561
|
+
// string. A GOOGLE block sets `data` and `profiles` at most once each, and
|
|
1562
|
+
// holds no alias, since an alias may name an anchor set on a key set twice.
|
|
1563
|
+
// Every key of a mapping on that path is a string, so not an alias or a merge
|
|
1564
|
+
// key, and no value on it is an alias. No key is set twice inside
|
|
1565
|
+
// `custom_extensions`.
|
|
1566
|
+
function blocksReadOneWay(parsed, errors) {
|
|
1567
|
+
const models = onlyValue(parsed.contents, 'semantic_model');
|
|
1568
|
+
if (models === null)
|
|
1569
|
+
return false;
|
|
1570
|
+
const model = yaml.isSeq(models) ? models.items[0] : models;
|
|
1571
|
+
if (yaml.isAlias(model))
|
|
1572
|
+
return false;
|
|
1573
|
+
const exts = onlyValue(model, 'custom_extensions');
|
|
1574
|
+
if (exts === null)
|
|
1575
|
+
return false;
|
|
1576
|
+
if (!yaml.isNode(exts))
|
|
1577
|
+
return true;
|
|
1578
|
+
const [start, , end] = exts.range ?? [0, 0, 0];
|
|
1579
|
+
if (errors.some(err => err.pos[0] >= start && err.pos[0] < end)) {
|
|
1580
|
+
return false;
|
|
1581
|
+
}
|
|
1582
|
+
for (const block of yaml.isSeq(exts) ? exts.items : [exts]) {
|
|
1583
|
+
if (yaml.isAlias(block))
|
|
1584
|
+
return false;
|
|
1585
|
+
if (!yaml.isMap(block))
|
|
1586
|
+
continue;
|
|
1587
|
+
const vendor = onlyValue(block, 'vendor_name');
|
|
1588
|
+
if (!yaml.isScalar(vendor) || typeof vendor.value !== 'string') {
|
|
1589
|
+
return false;
|
|
1590
|
+
}
|
|
1591
|
+
if (!isGoogleVendor(vendor.value))
|
|
1592
|
+
continue;
|
|
1593
|
+
if (onlyValue(block, 'data') === null ||
|
|
1594
|
+
onlyValue(block, 'profiles') === null || holdsAlias(block)) {
|
|
1595
|
+
return false;
|
|
1596
|
+
}
|
|
1597
|
+
}
|
|
1598
|
+
return true;
|
|
1599
|
+
}
|
|
1600
|
+
// Whether `node` holds an alias, as a key or a value at any depth.
|
|
1601
|
+
function holdsAlias(node) {
|
|
1602
|
+
let found = false;
|
|
1603
|
+
yaml.visit(node, {
|
|
1604
|
+
Alias: () => {
|
|
1605
|
+
found = true;
|
|
1606
|
+
return yaml.visit.BREAK;
|
|
1607
|
+
},
|
|
1608
|
+
});
|
|
1609
|
+
return found;
|
|
1610
|
+
}
|
|
1611
|
+
// Ossie's Relationship has no `description` key and its objects are closed, so
|
|
1612
|
+
// a vanilla document that writes one is no longer valid Ossie. The description
|
|
1613
|
+
// is still expressible there, inside a GOOGLE block; say so, with the notation.
|
|
1614
|
+
function rejectVanillaRelationshipDescription(r, version) {
|
|
1615
|
+
if (exports.FORMAT_VERSION_PLACEMENT[version].relationship.description === 'native' ||
|
|
1616
|
+
r.description === undefined) {
|
|
1617
|
+
return;
|
|
1618
|
+
}
|
|
1619
|
+
throw new SemanticModelError(`Semantic model load error: relationship '${r.name}': a ` +
|
|
1620
|
+
`'${exports.OSSIE_VERSION}' document cannot carry a plain 'description' on a ` +
|
|
1621
|
+
`relationship, since Ossie's Relationship has no such key. The ` +
|
|
1622
|
+
`description is still available; carry it in a ${exports.GOOGLE_VENDOR} ` +
|
|
1623
|
+
`block on the relationship:\n` +
|
|
1624
|
+
` custom_extensions:\n` +
|
|
1625
|
+
` - vendor_name: ${exports.GOOGLE_VENDOR}\n` +
|
|
1626
|
+
` data: '{"description": "..."}'\n` +
|
|
1627
|
+
`or set the document version to '${exports.GOOGLE_VERSION}'.`);
|
|
1628
|
+
}
|
|
1629
|
+
// Ossie's schema requires a dataset's `source`, a field's `expression`, and a
|
|
1630
|
+
// relationship's `from_columns` and `to_columns`. The Model Spec relaxes all
|
|
1631
|
+
// three in the Google flavor only, so a vanilla document must bind all three,
|
|
1632
|
+
// with or without `GOOGLE` blocks. `bindingOptional` does not relax this: it
|
|
1633
|
+
// decides what a push needs, not what the document must say to be valid Ossie.
|
|
1634
|
+
function requireVanillaBindings(m) {
|
|
1635
|
+
const missing = (kind, name, key) => {
|
|
1636
|
+
throw new SemanticModelError(`Semantic model load error: ${kind} '${name}': a ${exports.OSSIE_VERSION} ` +
|
|
1637
|
+
`document requires '${key}'. Only ${exports.GOOGLE_VERSION} lets a ${kind} ` +
|
|
1638
|
+
`leave it to a profile.`);
|
|
1639
|
+
};
|
|
1640
|
+
for (const ds of m.datasets) {
|
|
1641
|
+
if (ds.source === undefined)
|
|
1642
|
+
missing('dataset', ds.name, 'source');
|
|
1643
|
+
for (const f of ds.fields ?? []) {
|
|
1644
|
+
if (f.expression === undefined) {
|
|
1645
|
+
missing('field', `${ds.name}.${f.name}`, 'expression');
|
|
1646
|
+
}
|
|
1647
|
+
}
|
|
1648
|
+
}
|
|
1649
|
+
for (const r of m.relationships ?? []) {
|
|
1650
|
+
if (r.from_columns === undefined) {
|
|
1651
|
+
missing('relationship', r.name, 'from_columns');
|
|
1652
|
+
}
|
|
1653
|
+
if (r.to_columns === undefined) {
|
|
1654
|
+
missing('relationship', r.name, 'to_columns');
|
|
1655
|
+
}
|
|
1656
|
+
}
|
|
1657
|
+
}
|
|
1658
|
+
function isRecord(v) {
|
|
1659
|
+
return v !== null && typeof v === 'object' && !Array.isArray(v);
|
|
1660
|
+
}
|
|
1661
|
+
// Each model, dataset, field, relationship and metric in a parsed document
|
|
1662
|
+
// that carries an `ai_context`, with the label errors use for it. Runs before
|
|
1663
|
+
// schema validation, so it skips anything not shaped as expected.
|
|
1664
|
+
function* modelAiContexts(doc) {
|
|
1665
|
+
const list = (v) => (Array.isArray(v) ? v : []);
|
|
1666
|
+
const named = (kind, name) => typeof name === 'string' ? `${kind} '${name}'` : kind;
|
|
1667
|
+
const models = isRecord(doc) ? list(doc.semantic_model) : [];
|
|
1668
|
+
for (const m of models) {
|
|
1669
|
+
if (!isRecord(m))
|
|
1670
|
+
continue;
|
|
1671
|
+
if (isRecord(m.ai_context)) {
|
|
1672
|
+
yield { ai: m.ai_context, label: named('model', m.name) };
|
|
1673
|
+
}
|
|
1674
|
+
for (const ds of [...list(m.datasets), ...list(m.entities)]) {
|
|
1675
|
+
if (!isRecord(ds))
|
|
1676
|
+
continue;
|
|
1677
|
+
if (isRecord(ds.ai_context)) {
|
|
1678
|
+
yield { ai: ds.ai_context, label: named('dataset', ds.name) };
|
|
1679
|
+
}
|
|
1680
|
+
for (const f of list(ds.fields)) {
|
|
1681
|
+
if (!isRecord(f) || !isRecord(f.ai_context))
|
|
1682
|
+
continue;
|
|
1683
|
+
const label = typeof f.name === 'string' && typeof ds.name === 'string' ?
|
|
1684
|
+
`field '${ds.name}.${f.name}'` :
|
|
1685
|
+
'field';
|
|
1686
|
+
yield { ai: f.ai_context, label };
|
|
1687
|
+
}
|
|
1688
|
+
}
|
|
1689
|
+
for (const [items, kind] of [
|
|
1690
|
+
[m.relationships, 'relationship'], [m.metrics, 'metric'],
|
|
1691
|
+
]) {
|
|
1692
|
+
for (const it of list(items)) {
|
|
1693
|
+
if (isRecord(it) && isRecord(it.ai_context)) {
|
|
1694
|
+
yield { ai: it.ai_context, label: named(kind, it.name) };
|
|
1695
|
+
}
|
|
1696
|
+
}
|
|
1697
|
+
}
|
|
1698
|
+
}
|
|
1699
|
+
}
|
|
1700
|
+
// Why a custom ai_context member named `__proto__` cannot load, or undefined.
|
|
1701
|
+
// Schema validation drops that key as a guard against prototype pollution, so
|
|
1702
|
+
// it is rejected here rather than lost without a word. Only member names are
|
|
1703
|
+
// checked: a member's value is opaque and keeps any key. In the Google flavor
|
|
1704
|
+
// a sibling of the standard members is an unrecognized key rather than a
|
|
1705
|
+
// custom member.
|
|
1706
|
+
function protoMemberError(doc, google) {
|
|
1707
|
+
const custom = (label) => `${label}: custom ai_context member '__proto__' is not supported; ` +
|
|
1708
|
+
`rename it.`;
|
|
1709
|
+
for (const { ai, label } of modelAiContexts(doc)) {
|
|
1710
|
+
if (google && isRecord(ai.custom) && Object.hasOwn(ai.custom, '__proto__')) {
|
|
1711
|
+
return custom(label);
|
|
1712
|
+
}
|
|
1713
|
+
if (Object.hasOwn(ai, '__proto__')) {
|
|
1714
|
+
return google ?
|
|
1715
|
+
`${label}: ai_context has an unrecognized key '__proto__'.` :
|
|
1716
|
+
custom(label);
|
|
1717
|
+
}
|
|
1718
|
+
}
|
|
1719
|
+
return undefined;
|
|
1720
|
+
}
|
|
1721
|
+
/**
|
|
1722
|
+
* Loads YAML or JSON text (a document in the AI-first semantics format) into
|
|
1723
|
+
* the Semantic Model IR. `yaml.parse` accepts JSON too, so both are supported.
|
|
1724
|
+
*
|
|
1725
|
+
* A document with a field or metric expression that is not blank needs the SQL
|
|
1726
|
+
* parser, so call `loadSqlEngine()` first. Without it, this throws a plain
|
|
1727
|
+
* Error instead of a SemanticModelError.
|
|
1728
|
+
*/
|
|
1729
|
+
function loadModels(text, opts = {}) {
|
|
1730
|
+
let doc;
|
|
1731
|
+
try {
|
|
1732
|
+
doc = yaml.parse(text, yaml_options_1.YAML_OPTIONS);
|
|
1733
|
+
}
|
|
1734
|
+
catch (err) {
|
|
1735
|
+
throw new SemanticModelError(`Semantic model load error: could not parse input: ${err?.message ?? err}`);
|
|
1736
|
+
}
|
|
1737
|
+
return fromDocument(doc, opts);
|
|
1738
|
+
}
|
|
1739
|
+
/**
|
|
1740
|
+
* Converts an already-parsed document object into the Semantic Model IR. Throws
|
|
1741
|
+
* on a structurally invalid document; softer, lossy conversions are reported in
|
|
1742
|
+
* `warnings` rather than thrown.
|
|
1743
|
+
*/
|
|
1744
|
+
function fromDocument(doc, opts = {}) {
|
|
1745
|
+
const version = readVersion(doc);
|
|
1746
|
+
const extended = version === exports.GOOGLE_VERSION;
|
|
1747
|
+
const proto = protoMemberError(doc, extended);
|
|
1748
|
+
if (proto) {
|
|
1749
|
+
throw new SemanticModelError(`Semantic model load error: ${proto}`);
|
|
1750
|
+
}
|
|
1751
|
+
const normalized = normalizeDocumentSugars(doc, extended);
|
|
1752
|
+
const schema = makeDocumentSchema(opts.bindingOptional ?? false, extended);
|
|
1753
|
+
const result = schema.safeParse(normalized);
|
|
1754
|
+
if (!result.success) {
|
|
1755
|
+
throw new SemanticModelError(`Semantic model load error: ${schemaProblems(result.error.issues, {
|
|
1756
|
+
schema, doc: normalized, authored: doc, extended,
|
|
1757
|
+
subject: 'the document',
|
|
1758
|
+
})}`);
|
|
1759
|
+
}
|
|
1760
|
+
const warnings = [];
|
|
1761
|
+
const parsed = result.data;
|
|
1762
|
+
const models = parsed.semantic_model.map(m => convertModel(m, version, opts, warnings));
|
|
1763
|
+
return { models, warnings: [...new Set(warnings)] };
|
|
1764
|
+
}
|
|
1765
|
+
// Whether a dialect list takes its ANSI_SQL text for want of one in
|
|
1766
|
+
// `preferred`. pickDialect picks by this and expressionWarning warns by it, so
|
|
1767
|
+
// the two cannot disagree.
|
|
1768
|
+
function fallsBackToAnsi(dialects, preferred) {
|
|
1769
|
+
return !dialects.some(d => d.dialect === preferred.toUpperCase()) &&
|
|
1770
|
+
dialects.some(d => d.dialect === FALLBACK_DIALECT);
|
|
1771
|
+
}
|
|
1772
|
+
// The table dialects whose database reads a double-quoted token as a string,
|
|
1773
|
+
// as hasDoubleQuotedColumn finds it, each with that database's name for a
|
|
1774
|
+
// message. Every other dialect reads it as a name.
|
|
1775
|
+
const DOUBLE_QUOTES_AS_TEXT = {
|
|
1776
|
+
BIGQUERY: 'BigQuery',
|
|
1777
|
+
SPANNER: 'Spanner',
|
|
1778
|
+
MYSQL: 'MySQL',
|
|
1779
|
+
DATABRICKS: 'Databricks',
|
|
1780
|
+
};
|
|
1781
|
+
// The warning `item` gives on a table of `dialect`, in capitals, or undefined
|
|
1782
|
+
// when it has text in that dialect. That is the double-quote warning on a
|
|
1783
|
+
// dialect in DOUBLE_QUOTES_AS_TEXT, and otherwise the transpilation warning or
|
|
1784
|
+
// the ANSI_SQL note, which only the `preferred` dialect gives, and never for
|
|
1785
|
+
// the string form.
|
|
1786
|
+
function expressionWarning(item, subject, dialect, preferred) {
|
|
1787
|
+
const { ctx } = subject;
|
|
1788
|
+
const dialects = item.dialects ?? [];
|
|
1789
|
+
if (!dialects.length)
|
|
1790
|
+
return undefined;
|
|
1791
|
+
if (dialects.some(d => d.dialect === dialect))
|
|
1792
|
+
return undefined;
|
|
1793
|
+
const ansi = dialects.find(d => d.dialect === FALLBACK_DIALECT);
|
|
1794
|
+
const database = DOUBLE_QUOTES_AS_TEXT[dialect];
|
|
1795
|
+
if (ansi && database && (0, transpile_1.hasDoubleQuotedColumn)(ansi.expression)) {
|
|
1796
|
+
const names = (0, transpile_1.doubleQuotedNames)(ansi.expression);
|
|
1797
|
+
const reads = `${ctx}: ${database} reads ${listed(names.map(n => `"${n}"`))} in its '${FALLBACK_DIALECT}' expression as text; `;
|
|
1798
|
+
if (subject.metric) {
|
|
1799
|
+
const several = names.length > 1;
|
|
1800
|
+
return reads + `if you meant ${several ? 'fields' : 'a field'}, give ` +
|
|
1801
|
+
`it a '${dialect}' expression that names ${several ? 'each' : 'it'} as '<entity>.<field>'`;
|
|
1802
|
+
}
|
|
1803
|
+
return reads + `if you meant a column or a field, give it a '${dialect}' expression that names a column alone, in backticks ` +
|
|
1804
|
+
`if it needs quoting, and a field of the entity whose fields list ` +
|
|
1805
|
+
`holds this expression as '<entity>.<field>'`;
|
|
1806
|
+
}
|
|
1807
|
+
if (item.stringForm || dialect !== preferred.toUpperCase())
|
|
1808
|
+
return undefined;
|
|
1809
|
+
if (fallsBackToAnsi(dialects, preferred))
|
|
1810
|
+
return ansiFallbackNote(preferred);
|
|
1811
|
+
const vendor = dialects.find(d => d.dialect !== FALLBACK_DIALECT);
|
|
1812
|
+
return `${ctx}: no '${preferred}' or '${FALLBACK_DIALECT}' dialect; keeping the '${vendor?.dialect}' ` +
|
|
1813
|
+
`expression as imported_expression (needs transpilation to '${preferred}')`;
|
|
1814
|
+
}
|
|
1815
|
+
// Whether `entity`'s table runs SQL in `dialect`. An abstract entity runs
|
|
1816
|
+
// none, and so does one whose table's dialect the load does not know.
|
|
1817
|
+
function runsDialect(entity, dialect) {
|
|
1818
|
+
return (0, resolve_profiles_1.sourcesDialect)({ entities: [entity] }) === dialect.toUpperCase();
|
|
1819
|
+
}
|
|
1820
|
+
// The warning `item` gives on `table`, the table of `entity`:
|
|
1821
|
+
// expressionWarning's when the load knows the table's dialect, or one saying
|
|
1822
|
+
// the table has no text to run. With `profiled`, the second is given only on a
|
|
1823
|
+
// Cloud SQL, Spanner or third-party table no lookup has read. On a table of the
|
|
1824
|
+
// `preferred` dialect, expressionWarning's transpilation warning already says
|
|
1825
|
+
// so.
|
|
1826
|
+
function tableWarning(item, subject, table, entity, preferred, profiled) {
|
|
1827
|
+
const w = table.unread ?
|
|
1828
|
+
undefined :
|
|
1829
|
+
expressionWarning(item, subject, table.name, preferred);
|
|
1830
|
+
if (w || (profiled && !table.unread && !table.guessed) ||
|
|
1831
|
+
!(0, resolve_profiles_1.noTextFor)(item, table.dialects)) {
|
|
1832
|
+
return w;
|
|
1833
|
+
}
|
|
1834
|
+
const own = table.dialects.map(d => `'${d}'`).join(', ');
|
|
1835
|
+
return `${subject.ctx}: no ${own} or '${FALLBACK_DIALECT}' dialect, so the ${table.name} table of entity '${entity}' cannot run it`;
|
|
1836
|
+
}
|
|
1837
|
+
// The warning each field's expression gives on each entity whose table
|
|
1838
|
+
// tableDialects gives. An inherited field is judged on each entity that
|
|
1839
|
+
// inherits it, and named after that entity.
|
|
1840
|
+
function fieldWarnings(model, preferred, profiled) {
|
|
1841
|
+
return (0, resolve_profiles_1.fieldsOnTables)(model, true)
|
|
1842
|
+
.map(({ entity: e, field: f, table }) => tableWarning(f, { ctx: `field '${e.name}.${f.name}'`, metric: false }, table, e.name, preferred, profiled))
|
|
1843
|
+
.filter((w) => w !== undefined);
|
|
1844
|
+
}
|
|
1845
|
+
// The warning `metric`'s expression gives on the tables of the `reached`
|
|
1846
|
+
// entities, in that order, or on every table when it reaches none. On a table
|
|
1847
|
+
// of the `preferred` dialect it is expressionWarning's; otherwise it is the
|
|
1848
|
+
// first tableWarning a table gives.
|
|
1849
|
+
function metricWarning(metric, reached, entities, preferred, profiled) {
|
|
1850
|
+
const subject = { ctx: `metric '${metric.name}'`, metric: true };
|
|
1851
|
+
const named = entities.filter(e => reached.includes(e.name))
|
|
1852
|
+
.sort((a, b) => reached.indexOf(a.name) - reached.indexOf(b.name));
|
|
1853
|
+
const tables = named.length ? named : entities;
|
|
1854
|
+
if (tables.some(e => runsDialect(e, preferred))) {
|
|
1855
|
+
return expressionWarning(metric, subject, preferred.toUpperCase(), preferred);
|
|
1856
|
+
}
|
|
1857
|
+
for (const e of tables) {
|
|
1858
|
+
const table = (0, resolve_profiles_1.tableDialects)(e, true);
|
|
1859
|
+
const w = table &&
|
|
1860
|
+
tableWarning(metric, subject, table, e.name, preferred, profiled);
|
|
1861
|
+
if (w)
|
|
1862
|
+
return w;
|
|
1863
|
+
}
|
|
1864
|
+
return undefined;
|
|
1865
|
+
}
|
|
1866
|
+
// Infers the entities a metric's expressions reach across all dialect entries,
|
|
1867
|
+
// and returns the warning the metric gives when an expression could not be
|
|
1868
|
+
// parsed or when it states no `entity` and its expressions name none. An
|
|
1869
|
+
// authored anchor, which vanilla writes in a GOOGLE block, settles which entity
|
|
1870
|
+
// the metric belongs to, so a `COUNT(*)` with one is not warned about. An
|
|
1871
|
+
// ANSI_SQL text whose `impliedDialect` no lookup read (null) is read as
|
|
1872
|
+
// generic SQL, the closest reading there is.
|
|
1873
|
+
function inferMetricEntities(metric, entityNames, preferred = exports.DEFAULT_DIALECT, impliedDialect, version) {
|
|
1874
|
+
const ctx = `metric '${metric.name}'`;
|
|
1875
|
+
const sources = (metric.dialects ?? [])
|
|
1876
|
+
.map(d => ({ text: d.expression, dialect: d.dialect }));
|
|
1877
|
+
if (!sources.length) {
|
|
1878
|
+
const text = metric.expression ?? metric.importedExpression ?? '';
|
|
1879
|
+
const dialect = metric.expression !== undefined ?
|
|
1880
|
+
preferred :
|
|
1881
|
+
(metric.importedDialect ?? preferred);
|
|
1882
|
+
if (text)
|
|
1883
|
+
sources.push({ text, dialect });
|
|
1884
|
+
}
|
|
1885
|
+
const allCols = [];
|
|
1886
|
+
let unparsed = false;
|
|
1887
|
+
for (const { text, dialect } of sources) {
|
|
1888
|
+
if (!text)
|
|
1889
|
+
continue;
|
|
1890
|
+
const cols = (0, sql_expr_utils_1.columnsForSource)(text, dialect, impliedDialect ?? undefined);
|
|
1891
|
+
if (cols === undefined) {
|
|
1892
|
+
unparsed = true;
|
|
1893
|
+
}
|
|
1894
|
+
else {
|
|
1895
|
+
allCols.push(...cols);
|
|
1896
|
+
}
|
|
1897
|
+
}
|
|
1898
|
+
const referenced = (0, sql_expr_utils_1.entitiesReached)(allCols, entityNames);
|
|
1899
|
+
if (metric.authoredEntity === undefined && !referenced.length) {
|
|
1900
|
+
const entityHint = version === exports.OSSIE_VERSION ?
|
|
1901
|
+
`set 'entity' in a ${exports.GOOGLE_VENDOR} custom_extensions block on the ` +
|
|
1902
|
+
`metric, or use version '${exports.GOOGLE_VERSION}' and set 'entity' on ` +
|
|
1903
|
+
`the metric` :
|
|
1904
|
+
`set 'entity' on the metric`;
|
|
1905
|
+
return {
|
|
1906
|
+
referenced,
|
|
1907
|
+
warning: unparsed ?
|
|
1908
|
+
`${ctx}: expression could not be parsed; entity inference was ` +
|
|
1909
|
+
`skipped (${entityHint}, or backtick-quote a reserved ` +
|
|
1910
|
+
`qualifier, such as \`Date\`.amount)` :
|
|
1911
|
+
`${ctx}: expression references no known entity; ` +
|
|
1912
|
+
`it may not be placeable downstream`,
|
|
1913
|
+
};
|
|
1914
|
+
}
|
|
1915
|
+
return { referenced };
|
|
1916
|
+
}
|
|
1917
|
+
// Every warning a load gives about `model`'s entities and metrics, in the
|
|
1918
|
+
// order a load gives them: each field's expression warning, then each metric's
|
|
1919
|
+
// expression warning and its warning when it names no entity. A metric a
|
|
1920
|
+
// profile excludes gives none. With `profiled`, the profile check reports a
|
|
1921
|
+
// field or metric its tables cannot run, so no warning repeats it. A profile
|
|
1922
|
+
// changes only these; a load's other warnings, about actions, come after them.
|
|
1923
|
+
function entityAndMetricWarnings(model, preferred = exports.DEFAULT_DIALECT, profiled = false) {
|
|
1924
|
+
const entities = model.entities ?? [];
|
|
1925
|
+
const out = fieldWarnings(model, preferred, profiled);
|
|
1926
|
+
const entityNames = entities.map(e => e.name);
|
|
1927
|
+
const impliedDialect = (0, resolve_profiles_1.sourcesDialect)(model);
|
|
1928
|
+
const excluded = new Set(model.excludedMetrics);
|
|
1929
|
+
for (const m of model.metrics ?? []) {
|
|
1930
|
+
if (excluded.has(m.name))
|
|
1931
|
+
continue;
|
|
1932
|
+
const { referenced, warning: mw } = inferMetricEntities(m, entityNames, preferred, impliedDialect, model.version);
|
|
1933
|
+
const w = metricWarning(m, [m.authoredEntity, ...referenced], entities, preferred, profiled);
|
|
1934
|
+
if (w)
|
|
1935
|
+
out.push(w);
|
|
1936
|
+
if (mw)
|
|
1937
|
+
out.push(mw);
|
|
1938
|
+
}
|
|
1939
|
+
return [...new Set(out)];
|
|
1940
|
+
}
|
|
1941
|
+
// Reads and validates the top-level `version`. It is REQUIRED and must be one
|
|
1942
|
+
// of the accepted values: the version selects which extension surface is legal,
|
|
1943
|
+
// so it cannot be guessed, and a missing or unrecognized version is a hard load
|
|
1944
|
+
// error rather than a warning (agreed with Dmitri).
|
|
1945
|
+
function readVersion(doc) {
|
|
1946
|
+
const version = (doc && typeof doc === 'object') ? doc.version : undefined;
|
|
1947
|
+
if (typeof version !== 'string' || version.length === 0) {
|
|
1948
|
+
throw new SemanticModelError(`Semantic model load error: missing 'version'; declare ` +
|
|
1949
|
+
`'${exports.OSSIE_VERSION}' (vanilla Ossie) or '${exports.GOOGLE_VERSION}' (the ` +
|
|
1950
|
+
`extended profile) at the top level.`);
|
|
1951
|
+
}
|
|
1952
|
+
if (!isFormatVersion(version)) {
|
|
1953
|
+
throw new SemanticModelError(`Semantic model load error: unknown version ` +
|
|
1954
|
+
`'${version}'; expected '${exports.OSSIE_VERSION}' or '${exports.GOOGLE_VERSION}'.`);
|
|
1955
|
+
}
|
|
1956
|
+
return version;
|
|
1957
|
+
}
|
|
1958
|
+
// Rejects names that appear more than once within a scope. Uniqueness is
|
|
1959
|
+
// required for a valid graph: a duplicate dataset, field, metric, or
|
|
1960
|
+
// relationship name makes the generated nodes, properties, or edges ambiguous,
|
|
1961
|
+
// so a duplicate is a hard load error (agreed with Dmitri) rather than a
|
|
1962
|
+
// warning. With `ignoreCase`, two names that differ only in case are rejected
|
|
1963
|
+
// too, naming the first-seen spelling.
|
|
1964
|
+
function rejectDuplicateNames(names, kind, scope, ignoreCase = false) {
|
|
1965
|
+
const seen = new Map();
|
|
1966
|
+
for (const n of names) {
|
|
1967
|
+
const key = ignoreCase ? n.toLowerCase() : n;
|
|
1968
|
+
const first = seen.get(key);
|
|
1969
|
+
if (first !== undefined) {
|
|
1970
|
+
const collision = first === n ? '' : ` (case-insensitive collision with '${first}')`;
|
|
1971
|
+
throw new SemanticModelError(`${scope}: duplicate ${kind} '${n}'${collision}; names must be unique.`);
|
|
1972
|
+
}
|
|
1973
|
+
seen.set(key, n);
|
|
1974
|
+
}
|
|
1975
|
+
}
|
|
1976
|
+
// Rejects a name that breaks its kind's pattern or length limit (see
|
|
1977
|
+
// names.ts), naming the scope it is in when one is given.
|
|
1978
|
+
function rejectInvalidName(kind, name, scope) {
|
|
1979
|
+
const problem = (0, names_1.nameProblem)(kind, name);
|
|
1980
|
+
if (problem) {
|
|
1981
|
+
throw new SemanticModelError(scope ? `${scope}: ${problem}` : problem);
|
|
1982
|
+
}
|
|
1983
|
+
}
|
|
1984
|
+
function convertModel(m, version, opts, warnings) {
|
|
1985
|
+
const dialect = opts.dialect ?? exports.DEFAULT_DIALECT;
|
|
1986
|
+
rejectInvalidName('model', m.name);
|
|
1987
|
+
const block = readGoogleBlocks(m.custom_extensions, version, 'model', `model '${m.name}'`);
|
|
1988
|
+
// A dataset's GOOGLE blocks are read before the bindings are checked, so a
|
|
1989
|
+
// dataset that declares `abstract` in one is told that vanilla has no
|
|
1990
|
+
// abstract dataset, even when it names no `source`.
|
|
1991
|
+
const datasetBlocks = m.datasets.map(ds => readGoogleBlocks(ds.custom_extensions, version, 'dataset', `dataset '${ds.name}'`));
|
|
1992
|
+
if (version === exports.OSSIE_VERSION) {
|
|
1993
|
+
requireVanillaBindings(m);
|
|
1994
|
+
}
|
|
1995
|
+
const entities = m.datasets.map((ds, i) => convertDataset(ds, datasetBlocks[i], version, opts, warnings, dialect));
|
|
1996
|
+
rejectDuplicateNames(entities.map(e => e.name), 'dataset name', `model '${m.name}'`, true);
|
|
1997
|
+
const entityNames = entities.map(e => e.name);
|
|
1998
|
+
const entityNameSet = new Set(entityNames);
|
|
1999
|
+
const relationships = (m.relationships ?? []).map(r => convertRelationship(r, version, entityNameSet));
|
|
2000
|
+
rejectDuplicateNames(relationships.map(r => r.name), 'relationship name', `model '${m.name}'`, true);
|
|
2001
|
+
// A field is judged on each entity that inherits it too, so its warning
|
|
2002
|
+
// waits until every entity is read.
|
|
2003
|
+
warnings.push(...fieldWarnings({ name: m.name, entities, relationships, metrics: [] }, dialect, !!opts.fieldTextChecked));
|
|
2004
|
+
const impliedDialect = (0, resolve_profiles_1.sourcesDialect)({ entities });
|
|
2005
|
+
const metrics = (m.metrics ?? []).map(mt => convertMetric(mt, version, entities, warnings, dialect, impliedDialect));
|
|
2006
|
+
rejectDuplicateNames(metrics.map(mt => mt.name), 'metric name', `model '${m.name}'`, true);
|
|
2007
|
+
// Actions name both entities and relationships -- as affected concepts, and
|
|
2008
|
+
// as the concept a parameter projects its definition from -- so they are
|
|
2009
|
+
// converted after each is known.
|
|
2010
|
+
//
|
|
2011
|
+
// The lookup is the one validate uses, so a name resolves the same way in
|
|
2012
|
+
// both places, inheritance included. Resolving THROWS on an `extends` naming
|
|
2013
|
+
// an entity the model does not declare, which this loader is not the place to
|
|
2014
|
+
// report: it accepts such a model deliberately, and validate names the
|
|
2015
|
+
// failure once per model. So an unresolved view stands in -- but it is a view
|
|
2016
|
+
// in which NO entity has an inherited field, because the fallback drops
|
|
2017
|
+
// `extends` model-wide rather than from the one entity that broke. Warning
|
|
2018
|
+
// off it would tell an author that `Savings` does not declare `balance` when
|
|
2019
|
+
// `Savings extends Account` and `Account` declares it, sending them to fix
|
|
2020
|
+
// a parameter that is correct. So the projection warnings are withheld while
|
|
2021
|
+
// the view is degraded, and validate's one accurate line stands alone.
|
|
2022
|
+
const relationshipNameSet = new Set(relationships.map(r => r.name));
|
|
2023
|
+
let concepts;
|
|
2024
|
+
let inheritanceResolved = true;
|
|
2025
|
+
try {
|
|
2026
|
+
concepts =
|
|
2027
|
+
(0, resolve_inheritance_1.declaredConceptFields)({ name: m.name, entities, relationships, metrics });
|
|
2028
|
+
}
|
|
2029
|
+
catch {
|
|
2030
|
+
inheritanceResolved = false;
|
|
2031
|
+
concepts = (0, resolve_inheritance_1.declaredConceptFields)({
|
|
2032
|
+
name: m.name,
|
|
2033
|
+
entities: entities.map(e => ({ ...e, extends: undefined })),
|
|
2034
|
+
relationships,
|
|
2035
|
+
metrics,
|
|
2036
|
+
});
|
|
2037
|
+
}
|
|
2038
|
+
const inheritedProblem = inheritanceResolved ?
|
|
2039
|
+
inheritedNoSqlProblems(entities, concepts)[0] :
|
|
2040
|
+
undefined;
|
|
2041
|
+
if (inheritedProblem) {
|
|
2042
|
+
throw new SemanticModelError(`Semantic model load error: model '${m.name}': ${inheritedProblem}`);
|
|
2043
|
+
}
|
|
2044
|
+
const actions = (m.actions ?? [])
|
|
2045
|
+
.map(a => convertAction(a, concepts, entityNameSet, relationshipNameSet, warnings, inheritanceResolved));
|
|
2046
|
+
rejectDuplicateNames(actions.map(a => a.name), 'action name', `model '${m.name}'`);
|
|
2047
|
+
const constraints = (m.constraints ?? []).map(convertConstraint);
|
|
2048
|
+
rejectDuplicateNames(constraints.map(c => c.name), 'constraint name', `model '${m.name}'`);
|
|
2049
|
+
const deploymentDocs = m.deployments ?? block.values.deployments ?? [];
|
|
2050
|
+
for (const d of deploymentDocs) {
|
|
2051
|
+
rejectInvalidName('deployment', d.name, `model '${m.name}'`);
|
|
2052
|
+
}
|
|
2053
|
+
const deployments = deploymentDocs.map(d => ({
|
|
2054
|
+
name: d.name,
|
|
2055
|
+
target: d.target,
|
|
2056
|
+
...(d.profile !== undefined ? { profile: d.profile } : {}),
|
|
2057
|
+
}));
|
|
2058
|
+
rejectDuplicateNames(deployments.map(d => d.name), 'deployment name', `model '${m.name}'`, true);
|
|
2059
|
+
const seenTargets = new Map();
|
|
2060
|
+
for (const d of deployments) {
|
|
2061
|
+
const first = seenTargets.get(d.target);
|
|
2062
|
+
if (first !== undefined) {
|
|
2063
|
+
throw new SemanticModelError(`Semantic model load error: model '${m.name}': deployments '${first}' and '${d.name}' target the same resource '${d.target}'; each deployment must target a distinct resource`);
|
|
2064
|
+
}
|
|
2065
|
+
seenTargets.set(d.target, d.name);
|
|
2066
|
+
}
|
|
2067
|
+
const description = composeDescription(m.description);
|
|
2068
|
+
// `version` records the flavor the document was written in, so a consumer
|
|
2069
|
+
// (e.g. a pull) can return the model in that same flavor.
|
|
2070
|
+
const model = { name: m.name, version, entities, relationships, metrics };
|
|
2071
|
+
if (actions.length)
|
|
2072
|
+
model.actions = actions;
|
|
2073
|
+
if (constraints.length)
|
|
2074
|
+
model.constraints = constraints;
|
|
2075
|
+
if (deployments.length)
|
|
2076
|
+
model.deployments = deployments;
|
|
2077
|
+
if (block.values.profiles?.length)
|
|
2078
|
+
model.profiles = block.values.profiles;
|
|
2079
|
+
if (description)
|
|
2080
|
+
model.description = description;
|
|
2081
|
+
const ai = aiContextOrUndefined(m.ai_context);
|
|
2082
|
+
if (ai)
|
|
2083
|
+
model.aiContext = ai;
|
|
2084
|
+
// The blocks the model keeps are carried verbatim. Under the extended
|
|
2085
|
+
// profile the native `deployments` list is folded into a GOOGLE block
|
|
2086
|
+
// appended after them; no authored GOOGLE block can collide with it, since
|
|
2087
|
+
// that flavor rejects one (see rejectGoogleBlock).
|
|
2088
|
+
// Vanilla deployments, read from a GOOGLE block, are not folded.
|
|
2089
|
+
if (block.kept)
|
|
2090
|
+
model.customExtensions = block.kept;
|
|
2091
|
+
const foldedTargets = version === exports.GOOGLE_VERSION ? deployments.map(d => d.target) : [];
|
|
2092
|
+
if (foldedTargets.length) {
|
|
2093
|
+
model.customExtensions = [
|
|
2094
|
+
...(model.customExtensions ?? []),
|
|
2095
|
+
deploymentTargetsExtension(foldedTargets),
|
|
2096
|
+
];
|
|
2097
|
+
}
|
|
2098
|
+
return model;
|
|
2099
|
+
}
|
|
2100
|
+
// Converts the dataset `ds`, whose GOOGLE blocks convertModel has read into
|
|
2101
|
+
// `block`.
|
|
2102
|
+
function convertDataset(ds, block, version, opts, warnings, dialect) {
|
|
2103
|
+
rejectInvalidName('entity', ds.name);
|
|
2104
|
+
const ctxLabel = `dataset '${ds.name}'`;
|
|
2105
|
+
// An abstract entity has no physical table, so it carries no source (empty
|
|
2106
|
+
// dataSource) and no key -- both are meaningless for a class never
|
|
2107
|
+
// materialized. Only a concrete entity's source is parsed.
|
|
2108
|
+
const parsedSource = ds.source !== undefined ?
|
|
2109
|
+
parseSource(ds.source, opts, ctxLabel, warnings, version) :
|
|
2110
|
+
undefined;
|
|
2111
|
+
const dataSource = parsedSource ? parsedSource.dataSource : '';
|
|
2112
|
+
const keys = ds.primary_key ?? [];
|
|
2113
|
+
const fields = (ds.fields ?? []).map(f => convertField(f, ds.name, version, dialect));
|
|
2114
|
+
rejectDuplicateNames(fields.map(f => f.name), 'field name', `dataset '${ds.name}'`, true);
|
|
2115
|
+
const entity = { name: ds.name, dataSource, keys, fields };
|
|
2116
|
+
if (parsedSource)
|
|
2117
|
+
entity.authoredSource = parsedSource.authoredSource;
|
|
2118
|
+
if (parsedSource?.sourceDialect) {
|
|
2119
|
+
entity.sourceDialect = parsedSource.sourceDialect;
|
|
2120
|
+
}
|
|
2121
|
+
if (ds.unique_keys && ds.unique_keys.length)
|
|
2122
|
+
entity.uniqueKeys = ds.unique_keys;
|
|
2123
|
+
const parents = ds.extends ?? block.values.extends;
|
|
2124
|
+
if (parents && parents.length)
|
|
2125
|
+
entity.extends = parents;
|
|
2126
|
+
if (ds.abstract)
|
|
2127
|
+
entity.abstract = true;
|
|
2128
|
+
const description = composeDescription(ds.description);
|
|
2129
|
+
if (description)
|
|
2130
|
+
entity.description = description;
|
|
2131
|
+
const ai = aiContextOrUndefined(ds.ai_context);
|
|
2132
|
+
if (ai)
|
|
2133
|
+
entity.aiContext = ai;
|
|
2134
|
+
if (block.kept)
|
|
2135
|
+
entity.customExtensions = block.kept;
|
|
2136
|
+
return entity;
|
|
2137
|
+
}
|
|
2138
|
+
// Resolves an authored `datatype` against the document's flavor: vanilla takes
|
|
2139
|
+
// only the canonical spelling, the Google flavor any casing. An omitted
|
|
2140
|
+
// datatype and `Opaque` both mean "no type" and come back undefined. A value
|
|
2141
|
+
// the flavor does not accept is a load error naming the flavor in force, since
|
|
2142
|
+
// the same string can be valid in one flavor and not the other.
|
|
2143
|
+
function resolveDataType(raw, version, ctx) {
|
|
2144
|
+
const result = (0, ir_1.normalizeDataType)(raw, version);
|
|
2145
|
+
if (!result.ok) {
|
|
2146
|
+
const accepted = version === exports.GOOGLE_VERSION ?
|
|
2147
|
+
'any casing of' :
|
|
2148
|
+
'exactly one of';
|
|
2149
|
+
throw new SemanticModelError(`Semantic model load error: ${ctx}: datatype '${raw}' is not valid ` +
|
|
2150
|
+
`in a '${version}' document; expected ${accepted} ` +
|
|
2151
|
+
`${ir_1.DATA_TYPES.join(', ')}.`);
|
|
2152
|
+
}
|
|
2153
|
+
return result.type;
|
|
2154
|
+
}
|
|
2155
|
+
function rejectVanillaShortFormExpression(expr, ctx) {
|
|
2156
|
+
if (typeof expr !== 'string')
|
|
2157
|
+
return;
|
|
2158
|
+
throw new SemanticModelError(`Semantic model load error: ${ctx}: a '${exports.OSSIE_VERSION}' document ` +
|
|
2159
|
+
`does not support the bare-string 'expression' shorthand. Write the ` +
|
|
2160
|
+
`per-dialect form instead:\n` +
|
|
2161
|
+
` expression:\n` +
|
|
2162
|
+
` dialects:\n` +
|
|
2163
|
+
` - dialect: ANSI_SQL\n` +
|
|
2164
|
+
` expression: ${JSON.stringify(expr)}\n` +
|
|
2165
|
+
`or set the document version to '${exports.GOOGLE_VERSION}'.`);
|
|
2166
|
+
}
|
|
2167
|
+
function convertExpression(raw, version, dialect, ctx) {
|
|
2168
|
+
if (version === exports.OSSIE_VERSION) {
|
|
2169
|
+
rejectVanillaShortFormExpression(raw, ctx);
|
|
2170
|
+
}
|
|
2171
|
+
if (typeof raw === 'string') {
|
|
2172
|
+
return {
|
|
2173
|
+
dialects: [{ dialect: 'ANSI_SQL', expression: raw }],
|
|
2174
|
+
stringForm: true,
|
|
2175
|
+
picked: { expression: raw },
|
|
2176
|
+
};
|
|
2177
|
+
}
|
|
2178
|
+
return {
|
|
2179
|
+
dialects: raw.dialects.map(d => ({ dialect: d.dialect, expression: d.expression })),
|
|
2180
|
+
stringForm: false,
|
|
2181
|
+
picked: pickDialect(raw, dialect),
|
|
2182
|
+
};
|
|
2183
|
+
}
|
|
2184
|
+
// The binding a field takes from its expression: every dialect text, and the
|
|
2185
|
+
// texts picked for `dialect`.
|
|
2186
|
+
function fieldExpression(raw, version, dialect, ctx) {
|
|
2187
|
+
const { dialects, stringForm, picked } = convertExpression(raw, version, dialect, ctx);
|
|
2188
|
+
const out = { dialects, stringForm };
|
|
2189
|
+
if (picked.expression !== undefined)
|
|
2190
|
+
out.expression = picked.expression;
|
|
2191
|
+
if (picked.importedExpression !== undefined) {
|
|
2192
|
+
out.importedExpression = picked.importedExpression;
|
|
2193
|
+
out.importedDialect = picked.importedDialect;
|
|
2194
|
+
}
|
|
2195
|
+
return out;
|
|
2196
|
+
}
|
|
2197
|
+
// A field expression written outside a model document, such as in a profile
|
|
2198
|
+
// file, checked against the loader's schema and read as a load reads a field's
|
|
2199
|
+
// expression, with the texts picked for `dialect`. The string form means
|
|
2200
|
+
// ANSI_SQL in either flavor. Throws naming `where` and every problem the schema
|
|
2201
|
+
// finds, or naming `where` when the expression holds no SQL for a field of
|
|
2202
|
+
// `entity`.
|
|
2203
|
+
function parseFieldExpression(raw, where, dialect, entity) {
|
|
2204
|
+
return readFieldExpression(raw, where, dialect, doc => noSqlProblem(doc, entity));
|
|
2205
|
+
}
|
|
2206
|
+
// A field expression read as parseFieldExpression reads it, with `problemOf`
|
|
2207
|
+
// saying why one the schema accepts holds no SQL.
|
|
2208
|
+
function readFieldExpression(raw, where, dialect, problemOf) {
|
|
2209
|
+
if (typeof raw !== 'string' &&
|
|
2210
|
+
(raw === null || typeof raw !== 'object' || Array.isArray(raw))) {
|
|
2211
|
+
throw new SemanticModelError(`${where}: 'expression' must be a string or a list of dialects`);
|
|
2212
|
+
}
|
|
2213
|
+
const result = typeof raw === 'string' ?
|
|
2214
|
+
z.string().safeParse(raw) :
|
|
2215
|
+
expressionObjectSchema.safeParse(raw);
|
|
2216
|
+
if (!result.success) {
|
|
2217
|
+
throw new SemanticModelError(keyProblems(result.error, expressionObjectSchema, 'expression', raw, where));
|
|
2218
|
+
}
|
|
2219
|
+
const problem = problemOf(result.data);
|
|
2220
|
+
if (problem) {
|
|
2221
|
+
throw new SemanticModelError(`${where}: ${problem}; write the field's SQL`);
|
|
2222
|
+
}
|
|
2223
|
+
return fieldExpression(result.data, exports.GOOGLE_VERSION, dialect, where);
|
|
2224
|
+
}
|
|
2225
|
+
// A profile's binding of a field, read as parseFieldExpression reads a
|
|
2226
|
+
// profile file's expression. A binding built in memory with no dialect list
|
|
2227
|
+
// is read as expressionDocOf gives it, and is checked for SQL as
|
|
2228
|
+
// noSqlProblemOf checks it, since a dialect list cannot hold an imported text
|
|
2229
|
+
// with no imported dialect, or one marked BIGQUERY beside a target text.
|
|
2230
|
+
// Throws as parseFieldExpression does.
|
|
2231
|
+
function parseFieldBinding(binding, where, dialect, entity) {
|
|
2232
|
+
const doc = expressionDocOf(binding);
|
|
2233
|
+
if (binding.dialects?.length) {
|
|
2234
|
+
return parseFieldExpression(doc, where, dialect, entity);
|
|
2235
|
+
}
|
|
2236
|
+
return readFieldExpression(doc, where, dialect, () => noSqlProblemOf(binding, entity));
|
|
2237
|
+
}
|
|
2238
|
+
function convertField(f, entityName, version, dialect) {
|
|
2239
|
+
rejectInvalidName('field', f.name, `dataset '${entityName}'`);
|
|
2240
|
+
// `label`, `dimension`, and AI-first annotations are carried structurally on
|
|
2241
|
+
// the IR (not folded into `description`) so an emitter can route each to its
|
|
2242
|
+
// own destination and a 1P round-trip stays lossless.
|
|
2243
|
+
const description = composeDescription(f.description);
|
|
2244
|
+
const ctx = `field '${entityName}.${f.name}'`;
|
|
2245
|
+
const field = { name: f.name };
|
|
2246
|
+
if (f.expression !== undefined) {
|
|
2247
|
+
Object.assign(field, fieldExpression(f.expression, version, dialect, ctx));
|
|
2248
|
+
}
|
|
2249
|
+
// else: an unbound field. It carries meaning (label, description, AI context)
|
|
2250
|
+
// but names no physical column. A field is unbound exactly when it has no
|
|
2251
|
+
// `expression` -- there is no separate flag. The availability pass
|
|
2252
|
+
// (pruneUnavailable) drops each unbound field, and whatever depends on it,
|
|
2253
|
+
// before generation, so one logical model can serve stores that bind
|
|
2254
|
+
// different subsets of columns. validateDeployments rejects one on a
|
|
2255
|
+
// concrete leaf entity when a graph deployment uses the model file's own
|
|
2256
|
+
// bindings.
|
|
2257
|
+
const type = resolveDataType(f.datatype, version, ctx);
|
|
2258
|
+
if (type)
|
|
2259
|
+
field.type = type;
|
|
2260
|
+
if (f.label)
|
|
2261
|
+
field.label = f.label;
|
|
2262
|
+
if (f.dimension) {
|
|
2263
|
+
field.dimension = {};
|
|
2264
|
+
if (f.dimension.is_time !== undefined)
|
|
2265
|
+
field.dimension.isTime = f.dimension.is_time;
|
|
2266
|
+
}
|
|
2267
|
+
if (description)
|
|
2268
|
+
field.description = description;
|
|
2269
|
+
const ai = aiContextOrUndefined(f.ai_context);
|
|
2270
|
+
if (ai)
|
|
2271
|
+
field.aiContext = ai;
|
|
2272
|
+
rejectGoogleBlock(f.custom_extensions, version, 'field', ctx);
|
|
2273
|
+
const ce = toCustomExtensions(f.custom_extensions);
|
|
2274
|
+
if (ce)
|
|
2275
|
+
field.customExtensions = ce;
|
|
2276
|
+
return field;
|
|
2277
|
+
}
|
|
2278
|
+
// Maps an OSI foreign-key relationship onto the IR edge. `source.columns` are
|
|
2279
|
+
// the FK columns on the `from` table (`from_columns`); `destination.columns`
|
|
2280
|
+
// are the referenced key columns on the `to` table (`to_columns`), paired
|
|
2281
|
+
// positionally. A logical relationship carries no columns (both endpoints
|
|
2282
|
+
// empty); a graph push requires them and rejects a column-less edge (see
|
|
2283
|
+
// validatePushRequirements). The source entity's own primary key is not
|
|
2284
|
+
// duplicated here -- downstream consumers look it up from the entity. A
|
|
2285
|
+
// malformed relationship (an endpoint not declared in the model, or mismatched
|
|
2286
|
+
// column arity) is a hard error, not a warning: the resulting edge would be
|
|
2287
|
+
// structurally invalid.
|
|
2288
|
+
function convertRelationship(r, version, entityNames) {
|
|
2289
|
+
rejectInvalidName('relationship', r.name);
|
|
2290
|
+
const ctx = `relationship '${r.name}'`;
|
|
2291
|
+
rejectVanillaRelationshipDescription(r, version);
|
|
2292
|
+
if (!entityNames.has(r.from)) {
|
|
2293
|
+
throw new SemanticModelError(`${ctx}: ${undefinedEndpointProblem('from', r.from)}`);
|
|
2294
|
+
}
|
|
2295
|
+
if (!entityNames.has(r.to)) {
|
|
2296
|
+
throw new SemanticModelError(`${ctx}: ${undefinedEndpointProblem('to', r.to)}`);
|
|
2297
|
+
}
|
|
2298
|
+
const fromColumns = r.from_columns ?? [];
|
|
2299
|
+
const toColumns = r.to_columns ?? [];
|
|
2300
|
+
if (fromColumns.length !== toColumns.length) {
|
|
2301
|
+
throw new SemanticModelError(joinLengthMessage(r.name, fromColumns.length, toColumns.length));
|
|
2302
|
+
}
|
|
2303
|
+
const relationship = {
|
|
2304
|
+
name: r.name,
|
|
2305
|
+
source: { entity: r.from, columns: fromColumns },
|
|
2306
|
+
destination: { entity: r.to, columns: toColumns },
|
|
2307
|
+
};
|
|
2308
|
+
const block = readGoogleBlocks(r.custom_extensions, version, 'relationship', ctx);
|
|
2309
|
+
const description = composeDescription(r.description ?? block.values.description);
|
|
2310
|
+
if (description)
|
|
2311
|
+
relationship.description = description;
|
|
2312
|
+
const ai = aiContextOrUndefined(r.ai_context);
|
|
2313
|
+
if (ai)
|
|
2314
|
+
relationship.aiContext = ai;
|
|
2315
|
+
if (block.kept)
|
|
2316
|
+
relationship.customExtensions = block.kept;
|
|
2317
|
+
return relationship;
|
|
2318
|
+
}
|
|
2319
|
+
function convertMetric(mt, version, entities, warnings, dialect, impliedDialect) {
|
|
2320
|
+
rejectInvalidName('metric', mt.name);
|
|
2321
|
+
const ctx = `metric '${mt.name}'`;
|
|
2322
|
+
const entityNames = entities.map(e => e.name);
|
|
2323
|
+
const block = readGoogleBlocks(mt.custom_extensions, version, 'metric', ctx);
|
|
2324
|
+
// The anchor the author wrote, natively or in a vanilla GOOGLE block.
|
|
2325
|
+
const authored = mt.entity ?? block.values.entity;
|
|
2326
|
+
const { dialects, stringForm, picked } = convertExpression(mt.expression, version, dialect, ctx);
|
|
2327
|
+
// Infer referenced entities across all dialect entries of the metric.
|
|
2328
|
+
const { referenced, warning: mw } = inferMetricEntities({
|
|
2329
|
+
name: mt.name,
|
|
2330
|
+
authoredEntity: authored,
|
|
2331
|
+
expression: picked.expression,
|
|
2332
|
+
importedExpression: picked.importedExpression,
|
|
2333
|
+
importedDialect: picked.importedDialect,
|
|
2334
|
+
dialects,
|
|
2335
|
+
}, entityNames, dialect, impliedDialect, version);
|
|
2336
|
+
const ew = metricWarning({ name: mt.name, dialects, stringForm }, [authored, ...referenced], entities, dialect, false);
|
|
2337
|
+
if (ew)
|
|
2338
|
+
warnings.push(ew);
|
|
2339
|
+
if (mw)
|
|
2340
|
+
warnings.push(mw);
|
|
2341
|
+
const metric = { name: mt.name, dialects, stringForm };
|
|
2342
|
+
if (authored !== undefined) {
|
|
2343
|
+
// Like a relationship endpoint, an anchor must name a dataset the model
|
|
2344
|
+
// declares; a dangling one would attach the metric to nothing downstream.
|
|
2345
|
+
if (!entityNames.includes(authored)) {
|
|
2346
|
+
throw new SemanticModelError(`Semantic model load error: ${ctx}: 'entity' names '${authored}', ` +
|
|
2347
|
+
`which is not a dataset in this model.`);
|
|
2348
|
+
}
|
|
2349
|
+
// What the author wrote wins over inference. `authoredEntity` records that
|
|
2350
|
+
// it was written, so a pull can write it back rather than re-infer it.
|
|
2351
|
+
metric.authoredEntity = authored;
|
|
2352
|
+
metric.entity = authored;
|
|
2353
|
+
}
|
|
2354
|
+
else if (referenced.length === 1) {
|
|
2355
|
+
// Attach only when the reference is unambiguous; a cross-entity metric is
|
|
2356
|
+
// left unattached (its qualifiers stay inline in the expression for
|
|
2357
|
+
// consumers).
|
|
2358
|
+
metric.entity = referenced[0];
|
|
2359
|
+
}
|
|
2360
|
+
if (picked.expression !== undefined)
|
|
2361
|
+
metric.expression = picked.expression;
|
|
2362
|
+
if (picked.importedExpression !== undefined) {
|
|
2363
|
+
metric.importedExpression = picked.importedExpression;
|
|
2364
|
+
metric.importedDialect = picked.importedDialect;
|
|
2365
|
+
}
|
|
2366
|
+
const type = resolveDataType(mt.datatype, version, ctx);
|
|
2367
|
+
if (type)
|
|
2368
|
+
metric.type = type;
|
|
2369
|
+
const description = composeDescription(mt.description);
|
|
2370
|
+
if (description)
|
|
2371
|
+
metric.description = description;
|
|
2372
|
+
const ai = aiContextOrUndefined(mt.ai_context);
|
|
2373
|
+
if (ai)
|
|
2374
|
+
metric.aiContext = ai;
|
|
2375
|
+
if (block.kept)
|
|
2376
|
+
metric.customExtensions = block.kept;
|
|
2377
|
+
return metric;
|
|
2378
|
+
}
|
|
2379
|
+
// Converts a constraint document to the IR. The judgment is kept verbatim --
|
|
2380
|
+
// it is the text a judge is handed. Description and AI context round-trip like
|
|
2381
|
+
// everywhere else. A document that states `expression` never reaches here;
|
|
2382
|
+
// rejectExpressionBody fails the parse.
|
|
2383
|
+
function convertConstraint(c) {
|
|
2384
|
+
const constraint = { name: c.name };
|
|
2385
|
+
if (c.judgment !== undefined)
|
|
2386
|
+
constraint.judgment = c.judgment;
|
|
2387
|
+
if (c.on_violation)
|
|
2388
|
+
constraint.onViolation = c.on_violation;
|
|
2389
|
+
if (c.severity)
|
|
2390
|
+
constraint.severity = c.severity;
|
|
2391
|
+
const description = composeDescription(c.description);
|
|
2392
|
+
if (description)
|
|
2393
|
+
constraint.description = description;
|
|
2394
|
+
const ai = aiContextOrUndefined(c.ai_context);
|
|
2395
|
+
if (ai)
|
|
2396
|
+
constraint.aiContext = ai;
|
|
2397
|
+
return constraint;
|
|
2398
|
+
}
|
|
2399
|
+
// Maps an authored action onto the IR. Parameter types are resolved against the
|
|
2400
|
+
// model's entities; a type that is neither a known entity nor a scalar datatype
|
|
2401
|
+
// is kept verbatim and warned, so the loader stays lenient (a strict `validate`
|
|
2402
|
+
// gate can promote these later).
|
|
2403
|
+
function convertAction(a, concepts, entityNames, relationshipNames, warnings, inheritanceResolved = true) {
|
|
2404
|
+
const parameters = (a.parameters ?? [])
|
|
2405
|
+
.map(p => convertParameter(p, a.name, concepts, warnings, inheritanceResolved));
|
|
2406
|
+
// Parameter names address the inputs at dispatch, so a collision is as
|
|
2407
|
+
// ambiguous as a duplicate field or metric name -- reject it the same way.
|
|
2408
|
+
rejectDuplicateNames(parameters.map(p => p.name), 'parameter name', `action '${a.name}'`);
|
|
2409
|
+
const action = {
|
|
2410
|
+
name: a.name,
|
|
2411
|
+
parameters,
|
|
2412
|
+
};
|
|
2413
|
+
// Absent when no binding supplies one -- the action is declared but not
|
|
2414
|
+
// performable here.
|
|
2415
|
+
if (a.executor != null) {
|
|
2416
|
+
action.executor = convertExecutor(a.executor);
|
|
2417
|
+
}
|
|
2418
|
+
if (a.guards?.length) {
|
|
2419
|
+
// A repeated guard would check one constraint twice while reading as two
|
|
2420
|
+
// rules, so it is rejected like any other duplicate name.
|
|
2421
|
+
rejectDuplicateNames(a.guards, 'guard', `action '${a.name}'`);
|
|
2422
|
+
action.guards = [...a.guards];
|
|
2423
|
+
}
|
|
2424
|
+
if (a.affects?.length) {
|
|
2425
|
+
const affects = a.affects.map(e => toAffectedConcept(e, a.name, entityNames, relationshipNames, warnings));
|
|
2426
|
+
rejectDuplicateAffectedConcepts(affects, a.name);
|
|
2427
|
+
warnMixedAffectsPrecision(affects, a.name, warnings);
|
|
2428
|
+
action.affects = affects;
|
|
2429
|
+
}
|
|
2430
|
+
const description = composeDescription(a.description);
|
|
2431
|
+
if (description)
|
|
2432
|
+
action.description = description;
|
|
2433
|
+
const ai = aiContextOrUndefined(a.ai_context);
|
|
2434
|
+
if (ai)
|
|
2435
|
+
action.aiContext = ai;
|
|
2436
|
+
const ce = toCustomExtensions(a.custom_extensions);
|
|
2437
|
+
if (ce)
|
|
2438
|
+
action.customExtensions = ce;
|
|
2439
|
+
return action;
|
|
2440
|
+
}
|
|
2441
|
+
// Maps one authored entry onto the IR, normalizing the two shapes to one.
|
|
2442
|
+
//
|
|
2443
|
+
// A bare name and a record with no `operation` mean the same thing, so both
|
|
2444
|
+
// land as an entry whose operation is unset. Whether `Order` is an entity or
|
|
2445
|
+
// an edge is neither authored nor stored -- it is a fact about the model, and
|
|
2446
|
+
// nothing about the entry depends on it -- so the names are consulted only to
|
|
2447
|
+
// warn about one that matches nothing, the way an unresolvable parameter type
|
|
2448
|
+
// does. validate is where it becomes an error.
|
|
2449
|
+
function toAffectedConcept(e, actionName, entityNames, relationshipNames, warnings) {
|
|
2450
|
+
const concept = typeof e === 'string' ? e : e.concept;
|
|
2451
|
+
const affected = { concept };
|
|
2452
|
+
if (!entityNames.has(concept) && !relationshipNames.has(concept)) {
|
|
2453
|
+
warnings.push(`action '${actionName}': affects names '${concept}', which is ` +
|
|
2454
|
+
`neither an entity nor a relationship in this model.`);
|
|
2455
|
+
}
|
|
2456
|
+
if (typeof e !== 'string') {
|
|
2457
|
+
if (e.operation !== undefined)
|
|
2458
|
+
affected.operation = e.operation;
|
|
2459
|
+
if (e.fields?.length) {
|
|
2460
|
+
// Naming one field twice says nothing the single mention does not.
|
|
2461
|
+
rejectDuplicateNames(e.fields, 'affected field', `action '${actionName}' affects '${concept}'`);
|
|
2462
|
+
affected.fields = [...e.fields];
|
|
2463
|
+
}
|
|
2464
|
+
}
|
|
2465
|
+
return affected;
|
|
2466
|
+
}
|
|
2467
|
+
// Two entries on the same concept with the same operation are one entry
|
|
2468
|
+
// written twice. Rejected rather than warned, like a repeated guard: it states
|
|
2469
|
+
// no second fact, and leaving it in would put a duplicate record in the catalog.
|
|
2470
|
+
function rejectDuplicateAffectedConcepts(affects, actionName) {
|
|
2471
|
+
rejectDuplicateNames(affects.map(e => `${e.concept}/${e.operation ?? '(unspecified)'}`), 'affected concept', `action '${actionName}'`);
|
|
2472
|
+
}
|
|
2473
|
+
// An entry with no operation covers the whole concept, so pairing it with a
|
|
2474
|
+
// specific operation on that same concept says both "in some unstated way" and
|
|
2475
|
+
// "in this exact way". That is almost always a half-finished edit -- the author
|
|
2476
|
+
// added precision to one line and left the coarse one behind -- but it is not
|
|
2477
|
+
// contradictory, so it warns rather than fails.
|
|
2478
|
+
function warnMixedAffectsPrecision(affects, actionName, warnings) {
|
|
2479
|
+
const unspecified = new Set(affects.filter(e => !e.operation).map(e => e.concept));
|
|
2480
|
+
if (!unspecified.size)
|
|
2481
|
+
return;
|
|
2482
|
+
for (const concept of new Set(affects.filter(e => e.operation && unspecified.has(e.concept))
|
|
2483
|
+
.map(e => e.concept))) {
|
|
2484
|
+
warnings.push(`action '${actionName}': affects lists '${concept}' both with an ` +
|
|
2485
|
+
`operation and without one. The bare entry already covers every ` +
|
|
2486
|
+
`operation on '${concept}'; drop it, or give it an operation too.`);
|
|
2487
|
+
}
|
|
2488
|
+
}
|
|
2489
|
+
// Maps one authored parameter onto the IR, resolving a derived one against the
|
|
2490
|
+
// concept it projects from. See ir.ActionParameter for the two forms.
|
|
2491
|
+
//
|
|
2492
|
+
// What is a THROW here and what is a warning follows the split the rest of this
|
|
2493
|
+
// file keeps. A contradiction in the parameter's own shape -- a `type` beside a
|
|
2494
|
+
// `concept`, half a reference, no way to work out a name -- makes the document
|
|
2495
|
+
// unloadable, because there is no reading of it to carry forward and the
|
|
2496
|
+
// author's intent is not guessable. A reference that is well formed but does
|
|
2497
|
+
// not resolve is kept verbatim and warned, the way an unresolvable `affects`
|
|
2498
|
+
// concept is, and validate.ts promotes it to the error that stops a push.
|
|
2499
|
+
//
|
|
2500
|
+
// A referenced field that is UNBOUND -- no `expression`, so no column under
|
|
2501
|
+
// this binding -- resolves like any other. That is deliberate. A metric or a
|
|
2502
|
+
// relationship reading an unbound field is unavailable because it needs a
|
|
2503
|
+
// column to read; a parameter needs only the logical definition, which an
|
|
2504
|
+
// unbound field has in full. So a derived parameter never makes its action
|
|
2505
|
+
// unavailable, and a logical-only model with no bindings at all publishes
|
|
2506
|
+
// exactly as it would with them.
|
|
2507
|
+
function convertParameter(p, actionName, concepts, warnings, inheritanceResolved = true) {
|
|
2508
|
+
const where = `action '${actionName}'`;
|
|
2509
|
+
if (p.type !== undefined && p.datatype !== undefined) {
|
|
2510
|
+
throw new SemanticModelError(`${where}: parameter ${p.name ? `'${p.name}' ` : ''}states both ` +
|
|
2511
|
+
`'type' and 'datatype'; set one or the other (they are the same key).`);
|
|
2512
|
+
}
|
|
2513
|
+
const statedType = p.type ?? p.datatype;
|
|
2514
|
+
let conceptName = p.concept;
|
|
2515
|
+
let fieldName = p.field;
|
|
2516
|
+
// Support `field: Account.accountId` as a concise one-key shorthand for
|
|
2517
|
+
// `{concept: Account, field: accountId}`, matching the `Entity.field`
|
|
2518
|
+
// syntax used in metric expressions and constraint judgments.
|
|
2519
|
+
if (conceptName === undefined && fieldName !== undefined &&
|
|
2520
|
+
fieldName.includes('.')) {
|
|
2521
|
+
const parts = fieldName.split('.');
|
|
2522
|
+
if (parts.length === 2 && parts[0].length > 0 && parts[1].length > 0) {
|
|
2523
|
+
conceptName = parts[0];
|
|
2524
|
+
fieldName = parts[1];
|
|
2525
|
+
}
|
|
2526
|
+
else {
|
|
2527
|
+
throw new SemanticModelError(`${where}: parameter ${p.name ? `'${p.name}' ` : ''}states 'field: ${fieldName}', which is not a valid 'Concept.field' reference; ` +
|
|
2528
|
+
`expected two dot-separated parts such as 'field: Order.orderId'.`);
|
|
2529
|
+
}
|
|
2530
|
+
}
|
|
2531
|
+
else if (conceptName !== undefined && fieldName?.includes('.')) {
|
|
2532
|
+
throw new SemanticModelError(`${where}: parameter ${p.name ? `'${p.name}' ` : ''}states 'concept: ${conceptName}' alongside 'field: ${fieldName}', which already contains a concept prefix. ` +
|
|
2533
|
+
`Write either {concept: ${conceptName}, field: ${fieldName.split('.').pop()}} or {field: ${fieldName}}.`);
|
|
2534
|
+
}
|
|
2535
|
+
const hasConcept = conceptName !== undefined;
|
|
2536
|
+
const hasField = fieldName !== undefined;
|
|
2537
|
+
if (hasConcept !== hasField) {
|
|
2538
|
+
const given = hasConcept ? 'concept' : 'field';
|
|
2539
|
+
const missing = hasConcept ? 'field' : 'concept';
|
|
2540
|
+
throw new SemanticModelError(`${where}: parameter ${p.name ? `'${p.name}' ` : ''}states '${given}' without '${missing}'. A parameter projected from a field ` +
|
|
2541
|
+
`states both, as two separate keys: ` +
|
|
2542
|
+
`{concept: Order, field: orderId} (or 'field: Order.orderId').`);
|
|
2543
|
+
}
|
|
2544
|
+
if (hasConcept && statedType !== undefined) {
|
|
2545
|
+
const typeKey = p.datatype !== undefined ? 'datatype' : 'type';
|
|
2546
|
+
const projectionDesc = p.concept !== undefined ?
|
|
2547
|
+
`'concept: ${p.concept}' and 'field: ${p.field}'` :
|
|
2548
|
+
`'field: ${p.field}'`;
|
|
2549
|
+
throw new SemanticModelError(`${where}: parameter ${p.name ? `'${p.name}' ` : ''}states a '${typeKey}' alongside ${projectionDesc}. A ` +
|
|
2550
|
+
`projected parameter takes its type from the field, so stating one ` +
|
|
2551
|
+
`here is a second place for it to be wrong: if the type is wrong, ` +
|
|
2552
|
+
`fix the field; if the call needs a different one, cast in the DML.`);
|
|
2553
|
+
}
|
|
2554
|
+
const name = p.name ?? fieldName;
|
|
2555
|
+
if (name === undefined) {
|
|
2556
|
+
throw new SemanticModelError(`${where}: a parameter states neither a 'name' nor a 'field' to take ` +
|
|
2557
|
+
`one from.`);
|
|
2558
|
+
}
|
|
2559
|
+
const param = { name };
|
|
2560
|
+
if (hasConcept) {
|
|
2561
|
+
// Kept whether or not it resolves: it is what the author wrote, and a
|
|
2562
|
+
// round-trip that dropped it would turn a projected parameter into a
|
|
2563
|
+
// standalone one carrying a type nobody stated.
|
|
2564
|
+
param.concept = conceptName;
|
|
2565
|
+
param.field = fieldName;
|
|
2566
|
+
}
|
|
2567
|
+
const concept = hasConcept ? concepts.get(conceptName) : undefined;
|
|
2568
|
+
const field = concept?.fields.get(fieldName);
|
|
2569
|
+
if (hasConcept && !concept) {
|
|
2570
|
+
warnings.push(`${where}: parameter '${name}' projects from '${conceptName}', which ` +
|
|
2571
|
+
`is neither an entity nor a relationship in this model.`);
|
|
2572
|
+
}
|
|
2573
|
+
else if (hasConcept && !field && inheritanceResolved) {
|
|
2574
|
+
// Gated: with inheritance unresolved every entity looks like it declares
|
|
2575
|
+
// only its own fields, so this would fire on projections that are fine.
|
|
2576
|
+
warnings.push(`${where}: parameter '${name}' projects field '${fieldName}', which ${concept.kind} '${conceptName}' does not declare.`);
|
|
2577
|
+
}
|
|
2578
|
+
// The field's definition flows down; the parameter's own wording wins over
|
|
2579
|
+
// it. The type never does -- a projected parameter states none at all.
|
|
2580
|
+
if (field?.type !== undefined)
|
|
2581
|
+
param.type = field.type;
|
|
2582
|
+
if (statedType !== undefined)
|
|
2583
|
+
param.type = statedType;
|
|
2584
|
+
const description = p.description ?? field?.description;
|
|
2585
|
+
if (description !== undefined)
|
|
2586
|
+
param.description = description;
|
|
2587
|
+
const label = p.label ?? field?.label;
|
|
2588
|
+
if (label !== undefined)
|
|
2589
|
+
param.label = label;
|
|
2590
|
+
const ai = aiContextOrUndefined(p.ai_context) ?? field?.aiContext;
|
|
2591
|
+
if (ai)
|
|
2592
|
+
param.aiContext = ai;
|
|
2593
|
+
// Never inherited. A field says what a thing HAS; whether a CALL must supply
|
|
2594
|
+
// a value for it is the action's own business.
|
|
2595
|
+
if (p.required !== undefined)
|
|
2596
|
+
param.required = p.required;
|
|
2597
|
+
if (p.default !== undefined)
|
|
2598
|
+
param.default = p.default;
|
|
2599
|
+
if (param.type === undefined) {
|
|
2600
|
+
if (!hasConcept) {
|
|
2601
|
+
warnings.push(`${where}: parameter '${name}' states no 'type' and projects no ` +
|
|
2602
|
+
`field. A parameter with no field behind it declares its own ` +
|
|
2603
|
+
`scalar type (${ir_1.DATA_TYPES.join('/')}).`);
|
|
2604
|
+
}
|
|
2605
|
+
else if (field && inheritanceResolved) {
|
|
2606
|
+
// The projection resolved and the field is simply untyped, which is a
|
|
2607
|
+
// third case and not the one above: the author wrote the parameter
|
|
2608
|
+
// correctly, so the fix is on the field. A reference that did NOT
|
|
2609
|
+
// resolve was already warned about above and says why there is no type.
|
|
2610
|
+
warnings.push(`${where}: parameter '${name}' projects field '${conceptName}.${fieldName}', which declares no datatype, so the parameter has ` +
|
|
2611
|
+
`none either. Give that field a scalar type (${ir_1.DATA_TYPES.join('/')}).`);
|
|
2612
|
+
}
|
|
2613
|
+
}
|
|
2614
|
+
else if (!ir_1.DATA_TYPES.includes(param.type)) {
|
|
2615
|
+
// A type naming a concept is the old entity-reference spelling, which has
|
|
2616
|
+
// no meaning now: a parameter carries a value, so the way to take one FROM
|
|
2617
|
+
// an entity is to name the field it comes from.
|
|
2618
|
+
const named = concepts.get(param.type);
|
|
2619
|
+
warnings.push(named ?
|
|
2620
|
+
`${where}: parameter '${name}' is typed '${param.type}', which ` +
|
|
2621
|
+
`is ${named.kind === 'entity' ? 'an entity' :
|
|
2622
|
+
'a relationship'} rather ` +
|
|
2623
|
+
`than a scalar datatype. A parameter carries a value; to ` +
|
|
2624
|
+
`take one from ${param.type}, project it with ` +
|
|
2625
|
+
`{concept: ${param.type}, field: <field>}.` :
|
|
2626
|
+
`${where}: parameter '${name}' type '${param.type}' is not a ` +
|
|
2627
|
+
`scalar datatype (${ir_1.DATA_TYPES.join('/')})`);
|
|
2628
|
+
}
|
|
2629
|
+
return param;
|
|
2630
|
+
}
|
|
2631
|
+
// One line per problem that `schema` finds in the value of `key` on
|
|
2632
|
+
// `subject`.
|
|
2633
|
+
function keyProblems(error, schema, key, value, subject) {
|
|
2634
|
+
const doc = { [key]: value };
|
|
2635
|
+
return schemaProblems(error.issues.map(i => ({ ...i, path: [key, ...i.path] })), { schema: z.object({ [key]: schema }), doc, authored: doc, extended: true,
|
|
2636
|
+
subject });
|
|
2637
|
+
}
|
|
2638
|
+
// The noun a message gives an entry of each list of named objects in a model
|
|
2639
|
+
// document. An entry of `datasets` is an entity in the Google flavor.
|
|
2640
|
+
const ENTRY_NOUNS = new Map([
|
|
2641
|
+
['semantic_model', 'model'], ['datasets', 'dataset'], ['fields', 'field'],
|
|
2642
|
+
['relationships', 'relationship'], ['metrics', 'metric'],
|
|
2643
|
+
['actions', 'action'], ['parameters', 'parameter'],
|
|
2644
|
+
['constraints', 'constraint'], ['deployments', 'deployment'],
|
|
2645
|
+
]);
|
|
2646
|
+
// What a message says a schema expects.
|
|
2647
|
+
const EXPECTED = {
|
|
2648
|
+
string: 'a string',
|
|
2649
|
+
boolean: 'true or false',
|
|
2650
|
+
array: 'a list',
|
|
2651
|
+
object: 'a mapping',
|
|
2652
|
+
record: 'a mapping',
|
|
2653
|
+
};
|
|
2654
|
+
// A value as a message describes it.
|
|
2655
|
+
function described(value) {
|
|
2656
|
+
if (value === undefined)
|
|
2657
|
+
return 'missing';
|
|
2658
|
+
if (value === null)
|
|
2659
|
+
return 'empty';
|
|
2660
|
+
if (Array.isArray(value))
|
|
2661
|
+
return 'a list';
|
|
2662
|
+
if (typeof value === 'object')
|
|
2663
|
+
return 'a mapping';
|
|
2664
|
+
if (typeof value !== 'string')
|
|
2665
|
+
return String(value);
|
|
2666
|
+
return value.includes('\n') ? 'a string' : `'${value}'`;
|
|
2667
|
+
}
|
|
2668
|
+
// The keys the object at `path` in `schema` takes, apart from unlisted keys.
|
|
2669
|
+
function keysAt(schema, path) {
|
|
2670
|
+
// `s` without its optional, nullable and pipe wrappers. Of a union, the
|
|
2671
|
+
// member that is a `kind`.
|
|
2672
|
+
const unwrapped = (s, kind) => {
|
|
2673
|
+
if (s instanceof z.ZodOptional || s instanceof z.ZodNullable) {
|
|
2674
|
+
return unwrapped(s.unwrap(), kind);
|
|
2675
|
+
}
|
|
2676
|
+
if (s instanceof z.ZodPipe)
|
|
2677
|
+
return unwrapped(s.in, kind);
|
|
2678
|
+
if (s instanceof z.ZodUnion) {
|
|
2679
|
+
return s.options.map(o => unwrapped(o, kind))
|
|
2680
|
+
.find(o => o instanceof kind);
|
|
2681
|
+
}
|
|
2682
|
+
return s;
|
|
2683
|
+
};
|
|
2684
|
+
let node = schema;
|
|
2685
|
+
for (const seg of path) {
|
|
2686
|
+
const holder = unwrapped(node, typeof seg === 'number' ? z.ZodArray : z.ZodObject);
|
|
2687
|
+
node = holder instanceof z.ZodArray ? holder.element :
|
|
2688
|
+
holder instanceof z.ZodObject ? holder.shape[String(seg)] :
|
|
2689
|
+
undefined;
|
|
2690
|
+
}
|
|
2691
|
+
const object = unwrapped(node, z.ZodObject);
|
|
2692
|
+
if (!(object instanceof z.ZodObject))
|
|
2693
|
+
return [];
|
|
2694
|
+
return Object.keys(object.shape)
|
|
2695
|
+
.filter(k => !UNLISTED_KEYS.has(object.shape[k]));
|
|
2696
|
+
}
|
|
2697
|
+
// Where `path` points in `input.doc`: the innermost named object, as messages
|
|
2698
|
+
// name it, and the key below it, such as "'data' of entry 1 of
|
|
2699
|
+
// 'custom_extensions'". `spelled` gives a key as the author wrote it: a
|
|
2700
|
+
// Google-flavor model's `datasets` is `entities` unless the author wrote
|
|
2701
|
+
// `datasets`.
|
|
2702
|
+
function schemaLocation(path, input) {
|
|
2703
|
+
let subject = input.subject;
|
|
2704
|
+
let name;
|
|
2705
|
+
let entity;
|
|
2706
|
+
let datasets = 'datasets';
|
|
2707
|
+
let rest = [];
|
|
2708
|
+
let node = input.doc;
|
|
2709
|
+
for (const seg of path) {
|
|
2710
|
+
node = node?.[seg];
|
|
2711
|
+
const list = rest.length === 1 ? String(rest[0]) : '';
|
|
2712
|
+
const noun = list === 'datasets' && input.extended ?
|
|
2713
|
+
'entity' :
|
|
2714
|
+
ENTRY_NOUNS.get(list);
|
|
2715
|
+
if (typeof seg !== 'number' || !noun) {
|
|
2716
|
+
rest.push(seg);
|
|
2717
|
+
continue;
|
|
2718
|
+
}
|
|
2719
|
+
rest = [];
|
|
2720
|
+
name = typeof node?.name === 'string' ? node.name : undefined;
|
|
2721
|
+
const named = name === undefined ? `${noun} number ${seg + 1}` : `${noun} '${name}'`;
|
|
2722
|
+
if (list === 'semantic_model') {
|
|
2723
|
+
const written = input.authored?.semantic_model?.[seg];
|
|
2724
|
+
datasets = input.extended && written?.datasets === undefined ?
|
|
2725
|
+
'entities' :
|
|
2726
|
+
'datasets';
|
|
2727
|
+
}
|
|
2728
|
+
if (list === 'datasets')
|
|
2729
|
+
entity = name;
|
|
2730
|
+
if (list === 'fields' && name !== undefined && entity !== undefined) {
|
|
2731
|
+
name = `${entity}.${name}`;
|
|
2732
|
+
subject = `field '${name}'`;
|
|
2733
|
+
}
|
|
2734
|
+
else {
|
|
2735
|
+
subject = list === 'fields' || list === 'parameters' ?
|
|
2736
|
+
`${named} of ${subject}` :
|
|
2737
|
+
named;
|
|
2738
|
+
}
|
|
2739
|
+
}
|
|
2740
|
+
const spelled = (key) => key === 'datasets' ? datasets : String(key);
|
|
2741
|
+
const place = rest.map(s => typeof s === 'number' ? `entry ${s + 1}` :
|
|
2742
|
+
`'${spelled(s)}'`)
|
|
2743
|
+
.reverse()
|
|
2744
|
+
.join(' of ');
|
|
2745
|
+
return { subject, name, place, value: node, spelled };
|
|
2746
|
+
}
|
|
2747
|
+
// One line per problem of a schema error, naming the object and the key and
|
|
2748
|
+
// saying how to fix it.
|
|
2749
|
+
function schemaProblems(issues, input) {
|
|
2750
|
+
// zod also checks the length of a value of the wrong type, as for
|
|
2751
|
+
// `target: []`. The type is what to fix, so that check gives no line.
|
|
2752
|
+
const mistyped = new Set(issues.filter(i => i.code === 'invalid_type')
|
|
2753
|
+
.map(i => JSON.stringify(i.path)));
|
|
2754
|
+
return issues
|
|
2755
|
+
.filter(i => i.code !== 'too_small' || !mistyped.has(JSON.stringify(i.path)))
|
|
2756
|
+
.map(i => schemaProblem(i, input))
|
|
2757
|
+
.join('\n');
|
|
2758
|
+
}
|
|
2759
|
+
function schemaProblem(issue, input) {
|
|
2760
|
+
if (issue.code === 'invalid_union') {
|
|
2761
|
+
// When only one alternative takes the value's type, report its problems.
|
|
2762
|
+
const typed = issue.errors.filter(b => !b.every(i => i.code === 'invalid_type' && !i.path.length));
|
|
2763
|
+
if (typed.length === 1) {
|
|
2764
|
+
return schemaProblems(typed[0].map(i => ({ ...i, path: [...issue.path, ...i.path] })), input);
|
|
2765
|
+
}
|
|
2766
|
+
}
|
|
2767
|
+
const { subject, name, place, value, spelled } = schemaLocation(issue.path, input);
|
|
2768
|
+
const at = place ? `${subject}: ${place}` : subject;
|
|
2769
|
+
const takes = (what) => `${at} is ${described(value)}; it takes ${what}`;
|
|
2770
|
+
const expected = (type) => EXPECTED[type] ?? type;
|
|
2771
|
+
switch (issue.code) {
|
|
2772
|
+
case 'unrecognized_keys': {
|
|
2773
|
+
const keys = keysAt(input.schema, issue.path).map(k => `'${spelled(k)}'`);
|
|
2774
|
+
return `${subject}: unknown key${issue.keys.length > 1 ? 's' : ''} ${listed(issue.keys.map(k => `'${k}'`))}${place ? ` in ${place}` : ''}; it takes ${listed(keys)}`;
|
|
2775
|
+
}
|
|
2776
|
+
case 'invalid_type':
|
|
2777
|
+
return takes(expected(issue.expected));
|
|
2778
|
+
case 'invalid_union': {
|
|
2779
|
+
// A vanilla document writes an expression only as a mapping.
|
|
2780
|
+
const types = issue.errors.flatMap(b => b.flatMap(i => i.code === 'invalid_type' ? [i.expected] : []));
|
|
2781
|
+
const offered = types.filter(t => input.extended || t !== 'string' ||
|
|
2782
|
+
issue.path.at(-1) !== 'expression');
|
|
2783
|
+
return takes(listed([...new Set(offered.map(expected))], 'or'));
|
|
2784
|
+
}
|
|
2785
|
+
case 'invalid_value':
|
|
2786
|
+
return takes(listed(issue.values.map(v => `'${String(v)}'`), 'or'));
|
|
2787
|
+
case 'too_small':
|
|
2788
|
+
if (issue.origin === 'array') {
|
|
2789
|
+
return `${at} is an empty list; it takes at least one entry`;
|
|
2790
|
+
}
|
|
2791
|
+
if (issue.origin === 'string') {
|
|
2792
|
+
return `${at} is empty; it takes a non-empty string`;
|
|
2793
|
+
}
|
|
2794
|
+
break;
|
|
2795
|
+
case 'custom':
|
|
2796
|
+
// A rule on a named object quotes the object's name in its message.
|
|
2797
|
+
if (name !== undefined && issue.message.includes(`'${name}'`)) {
|
|
2798
|
+
return issue.message;
|
|
2799
|
+
}
|
|
2800
|
+
return `${subject}: ${issue.message}`;
|
|
2801
|
+
}
|
|
2802
|
+
return `${at}: ${issue.message}`;
|
|
2803
|
+
}
|
|
2804
|
+
// An executor written outside a model document, such as in a profile file,
|
|
2805
|
+
// checked against the loader's schema and converted as a load converts one.
|
|
2806
|
+
// Throws naming `where` and every problem the schema finds.
|
|
2807
|
+
function parseExecutor(raw, where) {
|
|
2808
|
+
const result = executorSchema.safeParse(raw);
|
|
2809
|
+
if (!result.success) {
|
|
2810
|
+
throw new SemanticModelError(keyProblems(result.error, executorSchema, 'executor', raw, where));
|
|
2811
|
+
}
|
|
2812
|
+
return convertExecutor(result.data);
|
|
2813
|
+
}
|
|
2814
|
+
// Normalizes the open format's single-key executor object to the IR's tagged
|
|
2815
|
+
// union. The schema already guaranteed exactly one kind is present.
|
|
2816
|
+
function convertExecutor(ex) {
|
|
2817
|
+
if (ex.mcp)
|
|
2818
|
+
return { kind: 'mcp', mcp: { ...ex.mcp } };
|
|
2819
|
+
if (ex.rest)
|
|
2820
|
+
return { kind: 'rest', rest: { ...ex.rest } };
|
|
2821
|
+
if (ex.sql) {
|
|
2822
|
+
return {
|
|
2823
|
+
kind: 'sql',
|
|
2824
|
+
sql: { statements: ex.sql.statements.map(t => t.trim()) },
|
|
2825
|
+
};
|
|
2826
|
+
}
|
|
2827
|
+
// The schema's refinement guarantees one of the four kinds is set.
|
|
2828
|
+
return { kind: 'grpc', grpc: { ...ex.grpc } };
|
|
2829
|
+
}
|
|
2830
|
+
function pickDialect(expr, preferred) {
|
|
2831
|
+
const preferredUpper = preferred.toUpperCase();
|
|
2832
|
+
const byName = (name) => expr.dialects.find(d => d.dialect === name.toUpperCase());
|
|
2833
|
+
// The original vendor variant, if any: the first dialect that is neither the
|
|
2834
|
+
// target nor the portable canonical. Kept as `importedExpression`.
|
|
2835
|
+
const vendor = expr.dialects.find(d => d.dialect !== preferredUpper && d.dialect !== FALLBACK_DIALECT);
|
|
2836
|
+
const out = {};
|
|
2837
|
+
if (vendor) {
|
|
2838
|
+
out.importedExpression = vendor.expression;
|
|
2839
|
+
out.importedDialect = vendor.dialect;
|
|
2840
|
+
}
|
|
2841
|
+
const exact = byName(preferred);
|
|
2842
|
+
if (exact) {
|
|
2843
|
+
out.expression = exact.expression;
|
|
2844
|
+
return out;
|
|
2845
|
+
}
|
|
2846
|
+
if (fallsBackToAnsi(expr.dialects, preferred)) {
|
|
2847
|
+
out.expression = byName(FALLBACK_DIALECT).expression;
|
|
2848
|
+
return out;
|
|
2849
|
+
}
|
|
2850
|
+
// Neither target nor canonical: keep only the imported vendor form; the
|
|
2851
|
+
// target `expression` awaits a transpile pass. expressionWarning says so.
|
|
2852
|
+
return out;
|
|
2853
|
+
}
|
|
2854
|
+
const MAX_CATALOG_NAME_LENGTH = 4000;
|
|
2855
|
+
const BIGQUERY_TABLE_URI = /^\/\/bigquery\.googleapis\.com\/projects\/([^/]+)\/datasets\/([^/.]+)\/tables\/([^/.]+)$/;
|
|
2856
|
+
// A BigLake table sits in a namespace of an Iceberg REST catalog, or in a
|
|
2857
|
+
// database of a Hive catalog.
|
|
2858
|
+
const BIGLAKE_TABLE_URI = /^\/\/biglake\.googleapis\.com\/projects\/([^/]+)\/catalogs\/([^/.]+)\/(namespaces|databases)\/([^/.]+)\/tables\/([^/.]+)$/;
|
|
2859
|
+
const SPANNER_TABLE_URI = /^\/\/spanner\.googleapis\.com\/projects\/([^/]+)\/instances\/([^/]+)\/databases\/([^/]+)\/tables\/[^/]+$/;
|
|
2860
|
+
const ALLOYDB_TABLE_URI = /^\/\/alloydb\.googleapis\.com\/projects\/([^/]+)\/locations\/([^/]+)\/clusters\/([^/]+)\/databases\/([^/]+)\/schemas\/[^/]+\/tables\/[^/]+$/;
|
|
2861
|
+
const LEGACY_ALLOYDB_INSTANCE_URI = /^\/\/alloydb\.googleapis\.com\/projects\/([^/]+)\/locations\/([^/]+)\/clusters\/([^/]+)\/instances\/[^/]+\/databases\/([^/]+)\/tables\/([^/]+)$/;
|
|
2862
|
+
// A Cloud SQL table. A MySQL table sits in a database; a PostgreSQL or SQL
|
|
2863
|
+
// Server table sits in a schema in a database. The shape tells MySQL from the
|
|
2864
|
+
// other two but not PostgreSQL from SQL Server, so the dialect comes from the
|
|
2865
|
+
// table's catalog entry, not from the source (see sourcesDialect).
|
|
2866
|
+
const CLOUDSQL_TABLE_URI = /^\/\/cloudsql\.googleapis\.com\/projects\/([^/]+)\/locations\/([^/]+)\/instances\/([^/]+)\/databases\/[^/]+\/tables\/[^/]+$/;
|
|
2867
|
+
const CLOUDSQL_SCHEMA_TABLE_URI = /^\/\/cloudsql\.googleapis\.com\/projects\/([^/]+)\/locations\/([^/]+)\/instances\/([^/]+)\/databases\/([^/]+)\/schemas\/[^/]+\/tables\/[^/]+$/;
|
|
2868
|
+
// A double quote or a backtick, which no resource name or entry name holds.
|
|
2869
|
+
const QUOTE = /["`]/;
|
|
2870
|
+
// The Cloud SQL table `source` names, or undefined when it names none. A name
|
|
2871
|
+
// with quotes names none, so push reports it without looking it up.
|
|
2872
|
+
function cloudSqlTableOf(source) {
|
|
2873
|
+
const uri = asResourceUri(source.trim());
|
|
2874
|
+
if (QUOTE.test(uri))
|
|
2875
|
+
return undefined;
|
|
2876
|
+
const mysql = uri.match(CLOUDSQL_TABLE_URI);
|
|
2877
|
+
const m = mysql ?? uri.match(CLOUDSQL_SCHEMA_TABLE_URI);
|
|
2878
|
+
if (!m)
|
|
2879
|
+
return undefined;
|
|
2880
|
+
return {
|
|
2881
|
+
project: m[1], location: m[2], instance: m[3], resourceName: uri.slice(2),
|
|
2882
|
+
inSchema: !mysql,
|
|
2883
|
+
};
|
|
2884
|
+
}
|
|
2885
|
+
// The table `source` names, or undefined when it names none of these.
|
|
2886
|
+
function sourceTableOf(source) {
|
|
2887
|
+
const uri = asResourceUri(source.trim());
|
|
2888
|
+
const resourceName = uri.slice(2);
|
|
2889
|
+
let m = uri.match(BIGQUERY_TABLE_URI);
|
|
2890
|
+
if (m) {
|
|
2891
|
+
return {
|
|
2892
|
+
system: 'bigquery', resourceName, project: m[1], dataset: m[2],
|
|
2893
|
+
table: m[3],
|
|
2894
|
+
};
|
|
2895
|
+
}
|
|
2896
|
+
m = uri.match(BIGLAKE_TABLE_URI);
|
|
2897
|
+
if (m) {
|
|
2898
|
+
const hive = m[3] === 'databases';
|
|
2899
|
+
return { system: 'biglake', resourceName, project: m[1], catalog: m[2], hive };
|
|
2900
|
+
}
|
|
2901
|
+
m = uri.match(SPANNER_TABLE_URI);
|
|
2902
|
+
if (m)
|
|
2903
|
+
return { system: 'spanner', resourceName, project: m[1], instance: m[2] };
|
|
2904
|
+
m = uri.match(ALLOYDB_TABLE_URI);
|
|
2905
|
+
if (m)
|
|
2906
|
+
return { system: 'alloydb', resourceName, project: m[1], location: m[2] };
|
|
2907
|
+
return undefined;
|
|
2908
|
+
}
|
|
2909
|
+
// A GCP resource name: a Google Cloud service host and a path, with no leading
|
|
2910
|
+
// `//`, such as `bigquery.googleapis.com/projects/p/datasets/d/tables/t`.
|
|
2911
|
+
const RESOURCE_NAME = /^[a-z][a-z0-9-]*\.googleapis\.com\//;
|
|
2912
|
+
// `value` as a resource URI: a GCP resource name gets the leading `//`, and
|
|
2913
|
+
// anything else is returned as it is. The IR and the checks read a source or a
|
|
2914
|
+
// target as its URI; the resource name the author wrote stays in
|
|
2915
|
+
// `authoredSource` for a source and in `uri` for a target.
|
|
2916
|
+
function asResourceUri(value) {
|
|
2917
|
+
return RESOURCE_NAME.test(value) ? `//${value}` : value;
|
|
2918
|
+
}
|
|
2919
|
+
// `value` as a GCP resource name: a resource URI loses its leading `//`, and
|
|
2920
|
+
// anything else is returned as it is. The inverse of `asResourceUri`, for
|
|
2921
|
+
// messages and output, which show the form an author writes.
|
|
2922
|
+
function asResourceName(value) {
|
|
2923
|
+
const name = value.replace(/^\/\//, '');
|
|
2924
|
+
return RESOURCE_NAME.test(name) ? name : value;
|
|
2925
|
+
}
|
|
2926
|
+
// What to do about a value written as a resource URI: write its GCP resource
|
|
2927
|
+
// name. The source and target messages share it, so they say it alike.
|
|
2928
|
+
function resourceUriFix(uri) {
|
|
2929
|
+
return `a resource URI; write its GCP resource name, without the leading ` +
|
|
2930
|
+
`'//': '${asResourceName(uri)}'`;
|
|
2931
|
+
}
|
|
2932
|
+
// The Knowledge Catalog name prefix of each first-party table, the number of
|
|
2933
|
+
// segments after it, and the GCP resource name those segments give. A
|
|
2934
|
+
// Spanner name's instance config has no place in the resource name.
|
|
2935
|
+
const CATALOG_NAME_FORMS = [
|
|
2936
|
+
['bigquery:', 3,
|
|
2937
|
+
s => `bigquery.googleapis.com/projects/${s[0]}/datasets/${s[1]}/tables/${s[2]}`],
|
|
2938
|
+
['biglake:table:', 4,
|
|
2939
|
+
s => `biglake.googleapis.com/projects/${s[0]}/catalogs/${s[1]}/namespaces/${s[2]}/tables/${s[3]}`],
|
|
2940
|
+
['spanner:', 5,
|
|
2941
|
+
s => `spanner.googleapis.com/projects/${s[0]}/instances/${s[2]}/databases/${s[3]}/tables/${s[4]}`],
|
|
2942
|
+
['alloydb:', 6,
|
|
2943
|
+
s => `alloydb.googleapis.com/projects/${s[0]}/locations/${s[1]}/clusters/${s[2]}/databases/${s[3]}/schemas/${s[4]}/tables/${s[5]}`],
|
|
2944
|
+
['cloudsql_mysql:', 5,
|
|
2945
|
+
s => `cloudsql.googleapis.com/projects/${s[0]}/locations/${s[1]}/instances/${s[2]}/databases/${s[3]}/tables/${s[4]}`],
|
|
2946
|
+
['cloudsql_postgresql:', 6, cloudSqlSchemaTable],
|
|
2947
|
+
['cloudsql_sqlserver:', 6, cloudSqlSchemaTable],
|
|
2948
|
+
];
|
|
2949
|
+
function cloudSqlSchemaTable(s) {
|
|
2950
|
+
return `cloudsql.googleapis.com/projects/${s[0]}/locations/${s[1]}/instances/${s[2]}/databases/${s[3]}/schemas/${s[4]}/tables/${s[5]}`;
|
|
2951
|
+
}
|
|
2952
|
+
// What a source written as a resource URI, as a Knowledge Catalog name of a
|
|
2953
|
+
// table kcmd reads, or with quotes in a resource name or in an entry name
|
|
2954
|
+
// outside a system entry group is, and the name to write instead when there
|
|
2955
|
+
// is one; undefined for any other source. The loader rejects such a source,
|
|
2956
|
+
// and pull warns for one an older push stored.
|
|
2957
|
+
function oldSourceForm(source) {
|
|
2958
|
+
const unquoted = unquotedName(source);
|
|
2959
|
+
if (unquoted !== undefined) {
|
|
2960
|
+
const quotes = 'a name with quotes in it; resource names and entry ' +
|
|
2961
|
+
'names take no quotes';
|
|
2962
|
+
return !unquoted ? quotes :
|
|
2963
|
+
/\s/.test(unquoted) ? `${quotes}, and ${exports.NO_SPACE}` :
|
|
2964
|
+
`${quotes}, so write '${unquoted}'`;
|
|
2965
|
+
}
|
|
2966
|
+
const name = asResourceName(source);
|
|
2967
|
+
if (name !== source && isAcceptedSource(name))
|
|
2968
|
+
return resourceUriFix(source);
|
|
2969
|
+
const built = catalogNameResourceName(source);
|
|
2970
|
+
const plain = built === undefined ? undefined : unquotedName(built) ?? built;
|
|
2971
|
+
const write = "a Knowledge Catalog name; write the table's GCP resource name";
|
|
2972
|
+
if (plain && isAcceptedSource(plain))
|
|
2973
|
+
return `${write}: '${plain}'`;
|
|
2974
|
+
if (!/^[a-z_]+:/.test(source) ||
|
|
2975
|
+
!(isAcceptedSource(source) || isAcceptedSource(built ?? ''))) {
|
|
2976
|
+
return undefined;
|
|
2977
|
+
}
|
|
2978
|
+
return plain && /\s/.test(plain) ?
|
|
2979
|
+
`a Knowledge Catalog name, and ${exports.NO_SPACE}` :
|
|
2980
|
+
write;
|
|
2981
|
+
}
|
|
2982
|
+
// Why no name to write is given when that name would hold a space.
|
|
2983
|
+
exports.NO_SPACE = 'kcmd cannot name a table whose name holds a space';
|
|
2984
|
+
// The resource name of a table in each Google Cloud service a source can
|
|
2985
|
+
// name, with a placeholder for each name in it. A Cloud SQL table has one
|
|
2986
|
+
// form without a schema and one with, and a BigLake table one for each kind
|
|
2987
|
+
// of catalog.
|
|
2988
|
+
exports.TABLE_FORMS = {
|
|
2989
|
+
bigquery: 'bigquery.googleapis.com/projects/<project>/datasets/<dataset>/tables/<table>',
|
|
2990
|
+
spanner: 'spanner.googleapis.com/projects/<project>/instances/<instance>/databases/<database>/tables/<table>',
|
|
2991
|
+
alloydb: 'alloydb.googleapis.com/projects/<project>/locations/<location>/clusters/<cluster>/databases/<database>/schemas/<schema>/tables/<table>',
|
|
2992
|
+
cloudsql: 'cloudsql.googleapis.com/projects/<project>/locations/<location>/instances/<instance>/databases/<database>/tables/<table>',
|
|
2993
|
+
cloudsqlSchema: 'cloudsql.googleapis.com/projects/<project>/locations/<location>/instances/<instance>/databases/<database>/schemas/<schema>/tables/<table>',
|
|
2994
|
+
biglake: 'biglake.googleapis.com/projects/<project>/catalogs/<catalog>/namespaces/<namespace>/tables/<table>',
|
|
2995
|
+
biglakeHive: 'biglake.googleapis.com/projects/<project>/catalogs/<catalog>/databases/<database>/tables/<table>',
|
|
2996
|
+
};
|
|
2997
|
+
// What each form in TABLE_FORMS is for, where its service has two.
|
|
2998
|
+
const FORM_USES = {
|
|
2999
|
+
cloudsql: 'MySQL',
|
|
3000
|
+
cloudsqlSchema: 'PostgreSQL, SQL Server',
|
|
3001
|
+
biglake: 'Iceberg REST catalog',
|
|
3002
|
+
biglakeHive: 'Hive catalog',
|
|
3003
|
+
};
|
|
3004
|
+
// `form` with its last placeholders replaced by `parts`, in order, or
|
|
3005
|
+
// undefined when `form` has fewer placeholders than `parts`.
|
|
3006
|
+
function fillForm(form, parts) {
|
|
3007
|
+
const skip = (form.match(/<\w+>/g) ?? []).length - parts.length;
|
|
3008
|
+
if (skip < 0)
|
|
3009
|
+
return undefined;
|
|
3010
|
+
let n = 0;
|
|
3011
|
+
return form.replace(/<\w+>/g, name => n++ < skip ? name : parts[n - 1 - skip]);
|
|
3012
|
+
}
|
|
3013
|
+
// The name of each service a table form starts with, as messages say it.
|
|
3014
|
+
const SERVICE_NAMES = {
|
|
3015
|
+
bigquery: 'BigQuery',
|
|
3016
|
+
biglake: 'BigLake',
|
|
3017
|
+
spanner: 'Spanner',
|
|
3018
|
+
alloydb: 'AlloyDB',
|
|
3019
|
+
cloudsql: 'Cloud SQL',
|
|
3020
|
+
};
|
|
3021
|
+
// Why `parts`, a bare name's parts read into the last placeholders of `form`,
|
|
3022
|
+
// name no table there only because a part holds a dot. Such a part was
|
|
3023
|
+
// quoted, so it is one name, and `form` takes no dot in its place. Undefined
|
|
3024
|
+
// for any other name. When `service` is false, the reason names neither the
|
|
3025
|
+
// part's role nor the service.
|
|
3026
|
+
function dottedPartProblem(parts, form, service = true) {
|
|
3027
|
+
const takes = (names) => {
|
|
3028
|
+
const name = fillForm(form, names);
|
|
3029
|
+
return name !== undefined && oldSourceForm(name) === undefined &&
|
|
3030
|
+
isAcceptedSource(name);
|
|
3031
|
+
};
|
|
3032
|
+
const plain = parts.map(p => p.replaceAll('.', '_'));
|
|
3033
|
+
if (takes(parts) || !takes(plain))
|
|
3034
|
+
return undefined;
|
|
3035
|
+
const i = parts.findIndex((p, j) => p !== plain[j] && !takes(plain.map((q, k) => k === j ? p : q)));
|
|
3036
|
+
if (i < 0)
|
|
3037
|
+
return undefined;
|
|
3038
|
+
if (!service)
|
|
3039
|
+
return `the quoted name '${parts[i]}' is one name, dot included`;
|
|
3040
|
+
const placeholders = form.match(/<\w+>/g);
|
|
3041
|
+
const role = placeholders[placeholders.length - parts.length + i].slice(1, -1);
|
|
3042
|
+
return `the quoted name '${parts[i]}' is one ${role} name, and a ${SERVICE_NAMES[form.split('.')[0]]} ${role} name cannot hold a dot`;
|
|
3043
|
+
}
|
|
3044
|
+
// `source` with its quotes removed, written as a resource name, when it is a
|
|
3045
|
+
// table's resource name, resource URI or entry name outside a system entry
|
|
3046
|
+
// group, holding them; '' when removing them leaves no such name; undefined
|
|
3047
|
+
// for any other source.
|
|
3048
|
+
function unquotedName(source) {
|
|
3049
|
+
if (!QUOTE.test(source))
|
|
3050
|
+
return undefined;
|
|
3051
|
+
const name = source.replace(/["`]/g, '');
|
|
3052
|
+
const named = (s) => !s.match(ENTRY_NAME)?.[1].startsWith('@') &&
|
|
3053
|
+
(ENTRY_NAME.test(s) ||
|
|
3054
|
+
[
|
|
3055
|
+
BIGQUERY_TABLE_URI, BIGLAKE_TABLE_URI, SPANNER_TABLE_URI,
|
|
3056
|
+
ALLOYDB_TABLE_URI, CLOUDSQL_TABLE_URI, CLOUDSQL_SCHEMA_TABLE_URI,
|
|
3057
|
+
].some(re => re.test(asResourceUri(s))));
|
|
3058
|
+
return named(name) ? asResourceName(name) : named(source) ? '' : undefined;
|
|
3059
|
+
}
|
|
3060
|
+
// The GCP resource name a Knowledge Catalog name of a first-party table
|
|
3061
|
+
// builds, accepted or not, or undefined when `fqn` is in no such shape.
|
|
3062
|
+
function catalogNameResourceName(fqn) {
|
|
3063
|
+
const form = CATALOG_NAME_FORMS.find(([prefix]) => fqn.startsWith(prefix));
|
|
3064
|
+
if (!form || fqn.length > MAX_CATALOG_NAME_LENGTH)
|
|
3065
|
+
return undefined;
|
|
3066
|
+
const [prefix, count, resourceName] = form;
|
|
3067
|
+
const segs = catalogNameSegments(fqn.slice(prefix.length));
|
|
3068
|
+
if (segs.length !== count || segs.some(seg => seg === ''))
|
|
3069
|
+
return undefined;
|
|
3070
|
+
return resourceName(segs);
|
|
3071
|
+
}
|
|
3072
|
+
// A catalog name's dot-separated segments. A segment containing a reserved
|
|
3073
|
+
// character, such as a domain-scoped project, is wrapped in backticks, so the
|
|
3074
|
+
// dots and colons inside backticks do not split or fail. Returns [] when the
|
|
3075
|
+
// quoting is malformed (an unquoted colon, an unclosed backtick, text after a
|
|
3076
|
+
// closing backtick, or a backtick mid-segment outside quotes). With `colons`
|
|
3077
|
+
// set, an unquoted colon after the first backtick stays in its segment.
|
|
3078
|
+
// Dataplex reads a fullyQualifiedName that way: it splits the system and
|
|
3079
|
+
// subtype off at the colons before the first backtick, and keeps the rest as
|
|
3080
|
+
// the path.
|
|
3081
|
+
function catalogNameSegments(path, colons = false) {
|
|
3082
|
+
const out = [];
|
|
3083
|
+
let cur = '';
|
|
3084
|
+
let state = 'start';
|
|
3085
|
+
let colonOk = false;
|
|
3086
|
+
for (let i = 0; i < path.length; i++) {
|
|
3087
|
+
const ch = path[i];
|
|
3088
|
+
if (state === 'start') {
|
|
3089
|
+
if (ch === '`') {
|
|
3090
|
+
state = 'quoted';
|
|
3091
|
+
colonOk = colons;
|
|
3092
|
+
}
|
|
3093
|
+
else if (ch === '.') {
|
|
3094
|
+
out.push('');
|
|
3095
|
+
}
|
|
3096
|
+
else if (ch === ':' && !colonOk) {
|
|
3097
|
+
return [];
|
|
3098
|
+
}
|
|
3099
|
+
else {
|
|
3100
|
+
cur = ch;
|
|
3101
|
+
state = 'unquoted';
|
|
3102
|
+
}
|
|
3103
|
+
}
|
|
3104
|
+
else if (state === 'unquoted') {
|
|
3105
|
+
if (ch === '.') {
|
|
3106
|
+
out.push(cur);
|
|
3107
|
+
cur = '';
|
|
3108
|
+
state = 'start';
|
|
3109
|
+
}
|
|
3110
|
+
else if (ch === '`' || (ch === ':' && !colonOk)) {
|
|
3111
|
+
return [];
|
|
3112
|
+
}
|
|
3113
|
+
else {
|
|
3114
|
+
cur += ch;
|
|
3115
|
+
}
|
|
3116
|
+
}
|
|
3117
|
+
else if (state === 'quoted') {
|
|
3118
|
+
if (ch === '`') {
|
|
3119
|
+
// A doubled backtick inside a quoted segment is a literal backtick.
|
|
3120
|
+
if (path[i + 1] === '`') {
|
|
3121
|
+
cur += '`';
|
|
3122
|
+
i++;
|
|
3123
|
+
}
|
|
3124
|
+
else {
|
|
3125
|
+
state = 'after_quote';
|
|
3126
|
+
}
|
|
3127
|
+
}
|
|
3128
|
+
else {
|
|
3129
|
+
cur += ch;
|
|
3130
|
+
}
|
|
3131
|
+
}
|
|
3132
|
+
else {
|
|
3133
|
+
if (ch === '.') {
|
|
3134
|
+
out.push(cur);
|
|
3135
|
+
cur = '';
|
|
3136
|
+
state = 'start';
|
|
3137
|
+
}
|
|
3138
|
+
else {
|
|
3139
|
+
return [];
|
|
3140
|
+
}
|
|
3141
|
+
}
|
|
3142
|
+
}
|
|
3143
|
+
if (state === 'quoted')
|
|
3144
|
+
return [];
|
|
3145
|
+
out.push(cur);
|
|
3146
|
+
return out;
|
|
3147
|
+
}
|
|
3148
|
+
// The pairs of backticks or double quotes in `text`, as the index of each
|
|
3149
|
+
// pair's opening and closing quote, in order. A quote opens a pair only when
|
|
3150
|
+
// the same character follows later, and the next such character closes it.
|
|
3151
|
+
// Inside a pair, the other quote character is ordinary.
|
|
3152
|
+
function quotePairs(text) {
|
|
3153
|
+
const pairs = [];
|
|
3154
|
+
for (let i = 0; i < text.length; i++) {
|
|
3155
|
+
const close = text[i] === '`' || text[i] === '"' ? text.indexOf(text[i], i + 1) : -1;
|
|
3156
|
+
if (close >= 0) {
|
|
3157
|
+
pairs.push([i, close]);
|
|
3158
|
+
i = close;
|
|
3159
|
+
}
|
|
3160
|
+
}
|
|
3161
|
+
return pairs;
|
|
3162
|
+
}
|
|
3163
|
+
// True when `source` holds whitespace outside a pair of backticks or double
|
|
3164
|
+
// quotes, which marks it as a query.
|
|
3165
|
+
function hasUnquotedWhitespace(source) {
|
|
3166
|
+
let from = 0;
|
|
3167
|
+
for (const [open, close] of quotePairs(source)) {
|
|
3168
|
+
if (/\s/.test(source.slice(from, open)))
|
|
3169
|
+
return true;
|
|
3170
|
+
from = close + 1;
|
|
3171
|
+
}
|
|
3172
|
+
return /\s/.test(source.slice(from));
|
|
3173
|
+
}
|
|
3174
|
+
// The database a source lives in, for the one-database rule and source
|
|
3175
|
+
// validation, or undefined when the source is in no accepted form: a bare name,
|
|
3176
|
+
// a query, a graph rather than a table, a catalog name under a prefix kcmd does
|
|
3177
|
+
// not accept, or one with the wrong number of segments. The result starts with
|
|
3178
|
+
// its system, which DIALECT_OF_SYSTEM maps to the dialect the source infers,
|
|
3179
|
+
// except for a source whose dialect is looked up (see dialectOfSources).
|
|
3180
|
+
//
|
|
3181
|
+
// A database is what a single query can reach: all of BigQuery, BigLake
|
|
3182
|
+
// included because BigQuery reads it; one Spanner, AlloyDB or Cloud SQL for
|
|
3183
|
+
// PostgreSQL database, or one Cloud SQL for MySQL or SQL Server instance. The
|
|
3184
|
+
// segment counts are those of the Knowledge Catalog FQN reference. The shape
|
|
3185
|
+
// of a Cloud SQL resource name cannot tell a SQL Server table from a
|
|
3186
|
+
// PostgreSQL one, so this keys it on its database, and the one-database rule
|
|
3187
|
+
// widens it to its instance once the lookup names SQL Server. Every
|
|
3188
|
+
// third-party table gets the key `thirdparty`, which is no target's database.
|
|
3189
|
+
// The one-database rule puts it in the database its entry's
|
|
3190
|
+
// fullyQualifiedName names, or else with the other tables of its system.
|
|
3191
|
+
function databaseOf(written) {
|
|
3192
|
+
if (hasUnquotedWhitespace(written))
|
|
3193
|
+
return undefined;
|
|
3194
|
+
const source = asResourceUri(written);
|
|
3195
|
+
if (BIGQUERY_TABLE_URI.test(source) || BIGLAKE_TABLE_URI.test(source)) {
|
|
3196
|
+
return 'bigquery';
|
|
3197
|
+
}
|
|
3198
|
+
let m = source.match(SPANNER_TABLE_URI);
|
|
3199
|
+
if (m)
|
|
3200
|
+
return `spanner/${m[1]}/${m[2]}/${m[3]}`;
|
|
3201
|
+
// An AlloyDB database belongs to its cluster, and a table to a schema in it.
|
|
3202
|
+
m = source.match(ALLOYDB_TABLE_URI);
|
|
3203
|
+
if (m)
|
|
3204
|
+
return `alloydb/${m[1]}/${m[2]}/${m[3]}/${m[4]}`;
|
|
3205
|
+
// A Cloud SQL for MySQL table's database is its instance, as for the
|
|
3206
|
+
// `cloudsql_mysql:` form, so both spellings of one instance agree. A table
|
|
3207
|
+
// in a schema is in Cloud SQL for PostgreSQL or SQL Server, which the shape
|
|
3208
|
+
// cannot tell apart, so its key names neither engine.
|
|
3209
|
+
m = source.match(CLOUDSQL_TABLE_URI);
|
|
3210
|
+
if (m)
|
|
3211
|
+
return `cloudsql_mysql/${m[1]}.${m[2]}.${m[3]}`;
|
|
3212
|
+
m = source.match(CLOUDSQL_SCHEMA_TABLE_URI);
|
|
3213
|
+
if (m)
|
|
3214
|
+
return `cloudsql/${m[1]}.${m[2]}.${m[3]}.${m[4]}`;
|
|
3215
|
+
if (thirdPartyEntryOf(source))
|
|
3216
|
+
return 'thirdparty';
|
|
3217
|
+
if (source.length > MAX_CATALOG_NAME_LENGTH)
|
|
3218
|
+
return undefined;
|
|
3219
|
+
m = source.match(/^([a-z_]+):(.+)$/);
|
|
3220
|
+
if (!m)
|
|
3221
|
+
return undefined;
|
|
3222
|
+
const seg = catalogNameSegments(m[2]);
|
|
3223
|
+
if (seg.length === 0 || seg.some(s => s.length === 0))
|
|
3224
|
+
return undefined;
|
|
3225
|
+
const first = (n) => seg.slice(0, n).join('.');
|
|
3226
|
+
switch (m[1]) {
|
|
3227
|
+
case 'bigquery':
|
|
3228
|
+
return seg.length === 3 && !seg[1].includes('.') && !seg[2].includes('.') ?
|
|
3229
|
+
'bigquery' :
|
|
3230
|
+
undefined;
|
|
3231
|
+
case 'spanner':
|
|
3232
|
+
return seg.length === 5 ? `spanner/${seg[0]}/${seg[2]}/${seg[3]}` :
|
|
3233
|
+
undefined;
|
|
3234
|
+
case 'alloydb':
|
|
3235
|
+
return seg.length === 6 ?
|
|
3236
|
+
`alloydb/${seg[0]}/${seg[1]}/${seg[2]}/${seg[3]}` :
|
|
3237
|
+
undefined;
|
|
3238
|
+
case 'cloudsql_mysql':
|
|
3239
|
+
return seg.length === 5 ? `cloudsql_mysql/${first(3)}` : undefined;
|
|
3240
|
+
case 'cloudsql_postgresql':
|
|
3241
|
+
return seg.length === 6 ? `cloudsql_postgresql/${first(4)}` : undefined;
|
|
3242
|
+
default:
|
|
3243
|
+
return undefined;
|
|
3244
|
+
}
|
|
3245
|
+
}
|
|
3246
|
+
// True when `source` names a table in a form `databaseOf` reads, as opposed to
|
|
3247
|
+
// a legacy bare table name. An author writes only its GCP resource name
|
|
3248
|
+
// (`parseSource` rejects the other forms); the resource URI and catalog name
|
|
3249
|
+
// forms are read from an older push. The loader, the Knowledge Catalog emitter
|
|
3250
|
+
// and the serializer share it so they agree on which sources are written back
|
|
3251
|
+
// as authored.
|
|
3252
|
+
function isAcceptedSource(source) {
|
|
3253
|
+
return source !== undefined && databaseOf(source) !== undefined;
|
|
3254
|
+
}
|
|
3255
|
+
// The source an entity is written with: its `authoredSource` when that names a
|
|
3256
|
+
// table in a form `databaseOf` reads, and otherwise its `dataSource`, shown as
|
|
3257
|
+
// a resource name. A model is serialized, shown and judged by it.
|
|
3258
|
+
function writtenSource(entity) {
|
|
3259
|
+
const authored = entity.authoredSource?.trim();
|
|
3260
|
+
return authored !== undefined && isAcceptedSource(authored) ?
|
|
3261
|
+
authored :
|
|
3262
|
+
asResourceName(entity.dataSource ?? '');
|
|
3263
|
+
}
|
|
3264
|
+
// Maps an accepted source string (or catalog resource path) to the IR's
|
|
3265
|
+
// canonical `dataSource`. A GCP resource name is read as its resource URI
|
|
3266
|
+
// first. A BigQuery or BigLake resource URI becomes the dot-separated table
|
|
3267
|
+
// string. A Spanner, AlloyDB or Cloud SQL resource URI is returned as the URI,
|
|
3268
|
+
// with the leading `//`. Anything else is returned trimmed. Shared by the
|
|
3269
|
+
// loader (`parseSource`) and the Knowledge Catalog reader
|
|
3270
|
+
// (`dataSourceFromResource`) so `readRelationships` matches `Entity.dataSource`
|
|
3271
|
+
// against the `schema-join` aspect's table names.
|
|
3272
|
+
function sourceToDataSource(source) {
|
|
3273
|
+
const value = asResourceUri(source.trim());
|
|
3274
|
+
const bqUri = value.match(BIGQUERY_TABLE_URI);
|
|
3275
|
+
if (bqUri)
|
|
3276
|
+
return `${bqUri[1]}.${bqUri[2]}.${bqUri[3]}`;
|
|
3277
|
+
const biglakeUri = value.match(BIGLAKE_TABLE_URI);
|
|
3278
|
+
if (biglakeUri) {
|
|
3279
|
+
const [, project, catalog, , parent, table] = biglakeUri;
|
|
3280
|
+
return `${project}.${catalog}.${parent}.${table}`;
|
|
3281
|
+
}
|
|
3282
|
+
return value;
|
|
3283
|
+
}
|
|
3284
|
+
// The dialect each Cloud SQL table or view entry type gives.
|
|
3285
|
+
exports.DIALECT_OF_ENTRY_TYPE = {
|
|
3286
|
+
'cloudsql-mysql-table': 'MYSQL',
|
|
3287
|
+
'cloudsql-mysql-view': 'MYSQL',
|
|
3288
|
+
'cloudsql-postgresql-table': 'POSTGRES',
|
|
3289
|
+
'cloudsql-postgresql-view': 'POSTGRES',
|
|
3290
|
+
'cloudsql-sqlserver-table': 'SQLSERVER',
|
|
3291
|
+
'cloudsql-sqlserver-view': 'SQLSERVER',
|
|
3292
|
+
};
|
|
3293
|
+
// The dialect each fullyQualifiedName prefix of a third-party table gives.
|
|
3294
|
+
exports.DIALECT_OF_FQN_PREFIX = new Map([
|
|
3295
|
+
['snowflake', 'SNOWFLAKE'], ['databricks', 'DATABRICKS'], ['mysql', 'MYSQL'],
|
|
3296
|
+
['postgresql', 'POSTGRES'], ['sqlserver', 'SQLSERVER'],
|
|
3297
|
+
]);
|
|
3298
|
+
// The shape of a Knowledge Catalog entry name.
|
|
3299
|
+
const ENTRY_NAME = /^projects\/[^/]+\/locations\/[^/]+\/entryGroups\/([^/]+)\/entries\/(.+)$/;
|
|
3300
|
+
// The entry name form, as messages show it.
|
|
3301
|
+
exports.ENTRY_NAME_FORM = 'projects/<p>/locations/<l>/entryGroups/<g>/entries/<id>';
|
|
3302
|
+
// How a source names a table outside Google Cloud, as messages say it.
|
|
3303
|
+
exports.THIRD_PARTY_SOURCE = 'A table outside Google Cloud is written ' +
|
|
3304
|
+
`as the entry name of its Knowledge Catalog entry, '${exports.ENTRY_NAME_FORM}'.`;
|
|
3305
|
+
// The third-party entry `source` names, or undefined when it names none, as
|
|
3306
|
+
// an entry name with quotes does.
|
|
3307
|
+
function thirdPartyEntryOf(source) {
|
|
3308
|
+
const name = source.trim();
|
|
3309
|
+
const m = name.match(ENTRY_NAME);
|
|
3310
|
+
if (!m || m[1].startsWith('@') || QUOTE.test(name))
|
|
3311
|
+
return undefined;
|
|
3312
|
+
const [, project, , location] = name.split('/');
|
|
3313
|
+
return { project, location, name };
|
|
3314
|
+
}
|
|
3315
|
+
// True when `entity` is written with a third-party table's entry name.
|
|
3316
|
+
function bindsThirdPartyTable(entity) {
|
|
3317
|
+
return thirdPartyEntryOf(entity.authoredSource ?? '') !== undefined;
|
|
3318
|
+
}
|
|
3319
|
+
// The key LoadOptions.sourceDialects holds a source's dialect under: a Cloud
|
|
3320
|
+
// SQL table's resource name, a third-party table's entry name, or a Spanner
|
|
3321
|
+
// table's database. Undefined for a source whose own name gives its dialect.
|
|
3322
|
+
function dialectLookupKey(source) {
|
|
3323
|
+
return cloudSqlTableOf(source)?.resourceName ??
|
|
3324
|
+
thirdPartyEntryOf(source)?.name ?? spannerDatabaseOf(source);
|
|
3325
|
+
}
|
|
3326
|
+
// The resource name of the database the Spanner table `source` is in.
|
|
3327
|
+
// Undefined for any other source and for a name with quotes.
|
|
3328
|
+
function spannerDatabaseOf(source) {
|
|
3329
|
+
const uri = asResourceUri(source.trim());
|
|
3330
|
+
return !QUOTE.test(uri) && SPANNER_TABLE_URI.test(uri) ?
|
|
3331
|
+
uri.slice(2, uri.lastIndexOf('/tables/')) :
|
|
3332
|
+
undefined;
|
|
3333
|
+
}
|
|
3334
|
+
// Why `source` is not accepted when it is a Knowledge Catalog entry name in a
|
|
3335
|
+
// system entry group, one whose id starts with `@`, such as `@bigquery`.
|
|
3336
|
+
// Undefined for any other source. The entry id of a Google Cloud table in a
|
|
3337
|
+
// system entry group is its GCP resource name, so the message names it as the
|
|
3338
|
+
// source to write, without quotes. Any other id, such as a dataset's, gets an
|
|
3339
|
+
// example. An id that holds a space gets no name to write.
|
|
3340
|
+
function systemEntrySourceError(source, ctx) {
|
|
3341
|
+
const entry = source.match(ENTRY_NAME);
|
|
3342
|
+
if (!entry?.[1].startsWith('@'))
|
|
3343
|
+
return undefined;
|
|
3344
|
+
const id = entry[2].replace(/^\/\//, '').replace(/["`]/g, '');
|
|
3345
|
+
const where = `Semantic model load error: ${ctx}: source '${source}' is a ` +
|
|
3346
|
+
`Knowledge Catalog entry name in the system entry group '${entry[1]}'`;
|
|
3347
|
+
if (/\s/.test(id))
|
|
3348
|
+
return `${where}, and ${exports.NO_SPACE}.`;
|
|
3349
|
+
const write = RESOURCE_NAME.test(id) && /\/tables\/[^/]+$/.test(id) ?
|
|
3350
|
+
`: '${id}'` :
|
|
3351
|
+
`, such as ` +
|
|
3352
|
+
`'bigquery.googleapis.com/projects/<p>/datasets/<d>/tables/<t>'`;
|
|
3353
|
+
return `${where}; write the table's resource name instead${write}.`;
|
|
3354
|
+
}
|
|
3355
|
+
// Google Cloud services whose tables the preview does not support yet, each
|
|
3356
|
+
// with its resource URI host and its catalog name prefix.
|
|
3357
|
+
const UNSUPPORTED_SERVICES = [
|
|
3358
|
+
{ name: 'Bigtable', host: 'bigtable.googleapis.com', prefix: 'bigtable:' },
|
|
3359
|
+
{
|
|
3360
|
+
name: 'Dataproc Metastore',
|
|
3361
|
+
host: 'metastore.googleapis.com',
|
|
3362
|
+
prefix: 'dataproc_metastore:',
|
|
3363
|
+
},
|
|
3364
|
+
];
|
|
3365
|
+
// Why `source` is not accepted when it names a table of a Google Cloud service
|
|
3366
|
+
// the preview does not support yet, as a resource URI, a resource name, a
|
|
3367
|
+
// catalog name or an entry name whose id is one of those. Undefined for any
|
|
3368
|
+
// other source.
|
|
3369
|
+
function unsupportedServiceSourceError(source, ctx) {
|
|
3370
|
+
const uri = asResourceUri(source.match(ENTRY_NAME)?.[2] ?? source);
|
|
3371
|
+
const service = UNSUPPORTED_SERVICES.find(s => uri.startsWith(`//${s.host}/`) || source.startsWith(s.prefix));
|
|
3372
|
+
return service ?
|
|
3373
|
+
`Semantic model load error: ${ctx}: source '${source}' is a ` +
|
|
3374
|
+
`${service.name} resource; the preview does not support ` +
|
|
3375
|
+
`${service.name} sources yet.` :
|
|
3376
|
+
undefined;
|
|
3377
|
+
}
|
|
3378
|
+
// True when `source` is in a Google Cloud form, accepted or not: a resource
|
|
3379
|
+
// name or resource URI, a catalog name of a Google Cloud system, or an entry
|
|
3380
|
+
// name in a system entry group.
|
|
3381
|
+
function isGoogleCloudForm(source) {
|
|
3382
|
+
const system = source.match(/^([a-z_]+):/)?.[1];
|
|
3383
|
+
return RESOURCE_NAME.test(source.replace(/^\/\//, '')) ||
|
|
3384
|
+
[...CATALOG_NAME_FORMS.map(([p]) => p),
|
|
3385
|
+
...UNSUPPORTED_SERVICES.map(s => s.prefix)]
|
|
3386
|
+
.some(p => p.startsWith(`${system}:`)) ||
|
|
3387
|
+
!!source.match(ENTRY_NAME)?.[1].startsWith('@');
|
|
3388
|
+
}
|
|
3389
|
+
// Why `source` is not accepted when it is a catalog name under a third-party
|
|
3390
|
+
// prefix, such as `snowflake:acme.retail.public.orders`. Undefined for any
|
|
3391
|
+
// other source.
|
|
3392
|
+
function thirdPartyNameError(source, ctx) {
|
|
3393
|
+
return exports.DIALECT_OF_FQN_PREFIX.has(source.match(/^([a-z]+):/)?.[1] ?? '') ?
|
|
3394
|
+
`Semantic model load error: ${ctx}: source '${source}' is a ` +
|
|
3395
|
+
`Knowledge Catalog name; write the entry name of the table's ` +
|
|
3396
|
+
`Knowledge Catalog entry instead, such as '${exports.ENTRY_NAME_FORM}'.` :
|
|
3397
|
+
undefined;
|
|
3398
|
+
}
|
|
3399
|
+
// Reads an entity's `source` into the dotted name the generators use and the
|
|
3400
|
+
// source as written. Throws on a form the loader does not accept. A profile
|
|
3401
|
+
// file's sources are read here too, so they follow the same rules. With
|
|
3402
|
+
// `importVersion`, the version of the model file the source is in, a bare
|
|
3403
|
+
// table name's error also names `kcmd import`, which completes a model file's
|
|
3404
|
+
// bare names but not a profile file's.
|
|
3405
|
+
function parseSource(source, opts, ctx, warnings, importVersion) {
|
|
3406
|
+
const trimmed = source.trim();
|
|
3407
|
+
if (hasUnquotedWhitespace(trimmed)) {
|
|
3408
|
+
throw new SemanticModelError(`Semantic model load error: ${ctx}: source '${trimmed}' looks like a ` +
|
|
3409
|
+
`SQL query; a query-valued source is not supported yet. Bind 'source' ` +
|
|
3410
|
+
`to a physical table using its GCP resource name (e.g. ` +
|
|
3411
|
+
`'bigquery.googleapis.com/projects/<p>/datasets/<d>/tables/<t>').`);
|
|
3412
|
+
}
|
|
3413
|
+
const problem = oldSourceForm(trimmed);
|
|
3414
|
+
// The accepted forms: a GCP resource name, or a third-party entry name.
|
|
3415
|
+
if (problem === undefined && isAcceptedSource(trimmed)) {
|
|
3416
|
+
const key = dialectLookupKey(trimmed);
|
|
3417
|
+
const sourceDialect = key && opts.sourceDialects?.get(key);
|
|
3418
|
+
return {
|
|
3419
|
+
authoredSource: trimmed,
|
|
3420
|
+
dataSource: sourceToDataSource(trimmed),
|
|
3421
|
+
...(sourceDialect ? { sourceDialect } : {}),
|
|
3422
|
+
};
|
|
3423
|
+
}
|
|
3424
|
+
if (opts.warnUnacceptedSource) {
|
|
3425
|
+
warnings?.push(`${ctx}: source '${trimmed}' is not a valid GCP resource name.`);
|
|
3426
|
+
return { authoredSource: trimmed, dataSource: '' };
|
|
3427
|
+
}
|
|
3428
|
+
if (problem !== undefined) {
|
|
3429
|
+
throw new SemanticModelError(`Semantic model load error: ${ctx}: source '${trimmed}' is ${problem}.`);
|
|
3430
|
+
}
|
|
3431
|
+
// Legacy instance-based AlloyDB name: reported in any spelling with the
|
|
3432
|
+
// cluster-based name to write.
|
|
3433
|
+
const legacyAlloy = asResourceUri(trimmed).match(LEGACY_ALLOYDB_INSTANCE_URI);
|
|
3434
|
+
if (legacyAlloy) {
|
|
3435
|
+
throw new SemanticModelError(`Semantic model load error: ${ctx}: source '${trimmed}' uses the ` +
|
|
3436
|
+
`legacy instance-based AlloyDB URI. An AlloyDB database belongs to ` +
|
|
3437
|
+
`its cluster and a table to a schema in it; write ` +
|
|
3438
|
+
`'alloydb.googleapis.com/projects/${legacyAlloy[1]}/locations/${legacyAlloy[2]}/clusters/${legacyAlloy[3]}/databases/${legacyAlloy[4]}/schemas/<schema>/tables/${legacyAlloy[5]}'.`);
|
|
3439
|
+
}
|
|
3440
|
+
const unsupported = unsupportedServiceSourceError(trimmed, ctx) ??
|
|
3441
|
+
systemEntrySourceError(trimmed, ctx) ??
|
|
3442
|
+
thirdPartyNameError(trimmed, ctx);
|
|
3443
|
+
if (unsupported)
|
|
3444
|
+
throw new SemanticModelError(unsupported);
|
|
3445
|
+
if (trimmed.length > MAX_CATALOG_NAME_LENGTH) {
|
|
3446
|
+
throw new SemanticModelError(`Semantic model load error: ${ctx}: source exceeds the ${MAX_CATALOG_NAME_LENGTH}-character limit (${trimmed.length} characters).`);
|
|
3447
|
+
}
|
|
3448
|
+
// Bare table name: no `/`, which every resource URI and resource name has,
|
|
3449
|
+
// and no `:` scheme or prefix.
|
|
3450
|
+
const isBareTableName = trimmed.length > 0 && !trimmed.includes('/') &&
|
|
3451
|
+
!trimmed.includes(':');
|
|
3452
|
+
let dotted;
|
|
3453
|
+
if (isBareTableName) {
|
|
3454
|
+
// With `importVersion`, the example also names the command that completes
|
|
3455
|
+
// a model file's bare names.
|
|
3456
|
+
const viaImport = importVersion ?
|
|
3457
|
+
`\nOr run '${(0, resolve_profiles_1.importCommand)('--source-prefix <prefix>', importVersion)}' to ` +
|
|
3458
|
+
`complete each bare table name from the database or Knowledge ` +
|
|
3459
|
+
`Catalog entry group that <prefix> names.` :
|
|
3460
|
+
'';
|
|
3461
|
+
// A name holding a space gets no name to write. Any other name gets an
|
|
3462
|
+
// example built from its parts when the loader accepts the example, and
|
|
3463
|
+
// the general error below otherwise, which says so when only a quoted
|
|
3464
|
+
// name's dot keeps the example out. A path in one pair of backticks counts
|
|
3465
|
+
// as its parts, as import reads it under a BigQuery or BigLake prefix.
|
|
3466
|
+
const segs = bareNameSegments(trimmed, true);
|
|
3467
|
+
if (segs.length > 0 && segs.length <= 4 && segs.every(s => s.length > 0)) {
|
|
3468
|
+
if (segs.some(s => /\s/.test(s))) {
|
|
3469
|
+
throw new SemanticModelError(`Semantic model load error: ${ctx}: bare table name '${trimmed}' ` +
|
|
3470
|
+
`is not a valid source, and ${exports.NO_SPACE}. ${exports.THIRD_PARTY_SOURCE}`);
|
|
3471
|
+
}
|
|
3472
|
+
const form = segs.length === 4 ? exports.TABLE_FORMS.biglake : exports.TABLE_FORMS.bigquery;
|
|
3473
|
+
const write = segs.length === 4 ?
|
|
3474
|
+
'Write its BigLake resource name instead:' :
|
|
3475
|
+
"Write the table's GCP resource name instead, for example:";
|
|
3476
|
+
const example = fillForm(form, segs);
|
|
3477
|
+
if (oldSourceForm(example) === undefined && isAcceptedSource(example)) {
|
|
3478
|
+
throw new SemanticModelError(`Semantic model load error: ${ctx}: bare table name '${trimmed}' ` +
|
|
3479
|
+
`is not a valid source. ${write}\n source: ${example}${viaImport}`);
|
|
3480
|
+
}
|
|
3481
|
+
dotted = dottedPartProblem(segs, form, false);
|
|
3482
|
+
}
|
|
3483
|
+
}
|
|
3484
|
+
// Each table form with its placeholders shortened, such as <p> for
|
|
3485
|
+
// <project>, and what it is for where its service has two.
|
|
3486
|
+
const forms = Object.entries(exports.TABLE_FORMS).map(([kind, form]) => {
|
|
3487
|
+
const short = form.replace(/<(\w+)>/g, (_, name) => `<${name === 'database' ? 'db' : name[0]}>`);
|
|
3488
|
+
const use = FORM_USES[kind];
|
|
3489
|
+
return `'${short}'${use ? ` (${use})` : ''}`;
|
|
3490
|
+
});
|
|
3491
|
+
throw new SemanticModelError(`Semantic model load error: ${ctx}: source '${trimmed}' is not a valid ` +
|
|
3492
|
+
`table reference${dotted ? `; ${dotted}` : ''}. Expected a GCP ` +
|
|
3493
|
+
`resource name (${forms.slice(0, -1).join(', ')}, or ${forms.at(-1)}), ` +
|
|
3494
|
+
`or, for a table outside Google Cloud, the entry name of its ` +
|
|
3495
|
+
`Knowledge Catalog entry ('${exports.ENTRY_NAME_FORM}').`);
|
|
3496
|
+
}
|
|
3497
|
+
// The parts of a bare table name, without their quotes. A part in backticks
|
|
3498
|
+
// or double quotes is one name, dots included, and a doubled quote character
|
|
3499
|
+
// in it, as in "a""b", is one literal quote. The quotes are those
|
|
3500
|
+
// hasUnquotedWhitespace reads. Returns [] when a quote does not wrap a whole
|
|
3501
|
+
// part. With `bigQuery`, a path in one pair of backticks, as BigQuery writes
|
|
3502
|
+
// `p.d.t`, has the parts of the same path written without them. A path in one
|
|
3503
|
+
// pair of double quotes is one name.
|
|
3504
|
+
function bareNameSegments(name, bigQuery) {
|
|
3505
|
+
return bareNameParts(name, bigQuery).map(p => p.text);
|
|
3506
|
+
}
|
|
3507
|
+
// The parts bareNameSegments gives, each with whether quotes wrap it.
|
|
3508
|
+
function bareNameParts(name, bigQuery) {
|
|
3509
|
+
const pairs = quotePairs(name);
|
|
3510
|
+
if (bigQuery && name[0] === '`' && pairs.length === 1 && pairs[0][0] === 0 &&
|
|
3511
|
+
pairs[0][1] === name.length - 1) {
|
|
3512
|
+
return bareNameParts(name.slice(1, -1), false);
|
|
3513
|
+
}
|
|
3514
|
+
const closeOf = new Map(pairs);
|
|
3515
|
+
const parts = [];
|
|
3516
|
+
let start = 0;
|
|
3517
|
+
while (true) {
|
|
3518
|
+
let part = /^[^.`"]*/.exec(name.slice(start))[0];
|
|
3519
|
+
let end = start + part.length;
|
|
3520
|
+
const firstClose = part ? undefined : closeOf.get(start);
|
|
3521
|
+
if (firstClose !== undefined) {
|
|
3522
|
+
let close = firstClose;
|
|
3523
|
+
part = name.slice(start + 1, close);
|
|
3524
|
+
// Pairs that touch, as in "a""b", are one name.
|
|
3525
|
+
while (name[close + 1] === name[start] && closeOf.has(close + 1)) {
|
|
3526
|
+
const next = closeOf.get(close + 1);
|
|
3527
|
+
part += name.slice(close + 1, next);
|
|
3528
|
+
close = next;
|
|
3529
|
+
}
|
|
3530
|
+
end = close + 1;
|
|
3531
|
+
}
|
|
3532
|
+
parts.push({ text: part, quoted: firstClose !== undefined });
|
|
3533
|
+
if (end === name.length)
|
|
3534
|
+
return parts;
|
|
3535
|
+
if (name[end] !== '.')
|
|
3536
|
+
return [];
|
|
3537
|
+
start = end + 1;
|
|
3538
|
+
}
|
|
3539
|
+
}
|
|
3540
|
+
/**
|
|
3541
|
+
* Loads every authored document into the IR once, so a multi-destination push
|
|
3542
|
+
* parses and validates each model a single time and fans the result out to each
|
|
3543
|
+
* deploy leg (BigQuery, Knowledge Catalog) rather than re-parsing per leg.
|
|
3544
|
+
*
|
|
3545
|
+
* A parse/schema error is returned as `error` (naming the document) rather than
|
|
3546
|
+
* thrown, mirroring how the deploy legs previously reported it; loader warnings
|
|
3547
|
+
* are prefixed with their document name.
|
|
3548
|
+
*
|
|
3549
|
+
* Each document needs the SQL parser as loadModels does, so call
|
|
3550
|
+
* `loadSqlEngine()` first. Without it, this throws the parser's plain Error.
|
|
3551
|
+
*/
|
|
3552
|
+
function loadSemanticModels(docs, opts = {}) {
|
|
3553
|
+
const models = [];
|
|
3554
|
+
const warnings = [];
|
|
3555
|
+
for (const doc of docs) {
|
|
3556
|
+
let loaded;
|
|
3557
|
+
try {
|
|
3558
|
+
loaded = loadModels(doc.text, opts);
|
|
3559
|
+
}
|
|
3560
|
+
catch (err) {
|
|
3561
|
+
if (!(err instanceof SemanticModelError))
|
|
3562
|
+
throw err;
|
|
3563
|
+
return {
|
|
3564
|
+
models,
|
|
3565
|
+
warnings,
|
|
3566
|
+
error: `Model document '${doc.name}': ${err.message}`,
|
|
3567
|
+
};
|
|
3568
|
+
}
|
|
3569
|
+
for (const w of loaded.warnings)
|
|
3570
|
+
warnings.push(`[${doc.name}] ${w}`);
|
|
3571
|
+
for (const model of loaded.models) {
|
|
3572
|
+
models.push({ document: doc.name, model });
|
|
3573
|
+
}
|
|
3574
|
+
}
|
|
3575
|
+
return { models, warnings };
|
|
3576
|
+
}
|