@google-cloud/kcmd 0.1.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -20
- package/build/ts/kcmd/cli.js +6164 -5631
- package/build/ts/kcmd/gcp/biglake.d.ts +2 -0
- package/build/ts/kcmd/gcp/biglake.js +13 -4
- package/build/ts/kcmd/gcp/spanner.d.ts +10 -0
- package/build/ts/kcmd/gcp/spanner.js +6 -3
- package/build/ts/kcmd/layouts/okf.js +49 -7
- package/build/ts/kcmd/layouts/semantic-model.d.ts +13 -0
- package/build/ts/kcmd/layouts/semantic-model.js +129 -50
- package/build/ts/kcmd/okf/concept.d.ts +1 -1
- package/build/ts/kcmd/okf/concept.js +6 -0
- package/build/ts/kcmd/okf/concept_to_entry.d.ts +2 -0
- package/build/ts/kcmd/okf/concept_to_entry.js +29 -2
- package/build/ts/kcmd/okf/init_okf_workspace.js +7 -9
- package/build/ts/kcmd/okf/okf_metadata.js +17 -0
- package/build/ts/kcmd/semantic/bigquery.js +139 -195
- package/build/ts/kcmd/semantic/binding.js +14 -19
- package/build/ts/kcmd/semantic/deploy_bigquery.js +6 -13
- package/build/ts/kcmd/semantic/deploy_knowledge_catalog.d.ts +10 -35
- package/build/ts/kcmd/semantic/deploy_knowledge_catalog.js +105 -162
- package/build/ts/kcmd/semantic/deploy_spanner.js +7 -14
- package/build/ts/kcmd/semantic/deployment_target.d.ts +1 -4
- package/build/ts/kcmd/semantic/deployment_target.js +6 -62
- package/build/ts/kcmd/semantic/graph_ddl.d.ts +10 -0
- package/build/ts/kcmd/semantic/graph_ddl.js +152 -59
- package/build/ts/kcmd/semantic/import_model.d.ts +26 -4
- package/build/ts/kcmd/semantic/import_model.js +531 -111
- package/build/ts/kcmd/semantic/ir.d.ts +2 -4
- package/build/ts/kcmd/semantic/kc_converter.d.ts +7 -1
- package/build/ts/kcmd/semantic/kc_converter.js +32 -27
- package/build/ts/kcmd/semantic/kc_custom_types.d.ts +5 -12
- package/build/ts/kcmd/semantic/kc_custom_types.js +43 -114
- package/build/ts/kcmd/semantic/kc_ids.d.ts +4 -5
- package/build/ts/kcmd/semantic/kc_ids.js +4 -7
- package/build/ts/kcmd/semantic/knowledge_catalog.d.ts +0 -3
- package/build/ts/kcmd/semantic/knowledge_catalog.js +3 -86
- package/build/ts/kcmd/semantic/loader.d.ts +7 -11
- package/build/ts/kcmd/semantic/loader.js +282 -171
- package/build/ts/kcmd/semantic/osi_converter.js +14 -65
- package/build/ts/kcmd/semantic/parameters.d.ts +1 -0
- package/build/ts/kcmd/semantic/parameters.js +22 -3
- package/build/ts/kcmd/semantic/pull_kc.d.ts +1 -5
- package/build/ts/kcmd/semantic/pull_kc.js +38 -112
- package/build/ts/kcmd/semantic/resolve_inheritance.d.ts +7 -2
- package/build/ts/kcmd/semantic/resolve_inheritance.js +37 -21
- package/build/ts/kcmd/semantic/resolve_profiles.d.ts +51 -10
- package/build/ts/kcmd/semantic/resolve_profiles.js +213 -32
- package/build/ts/kcmd/semantic/runtime/runtime.d.ts +4 -12
- package/build/ts/kcmd/semantic/runtime/runtime.js +4 -21
- package/build/ts/kcmd/semantic/runtime/store.d.ts +14 -19
- package/build/ts/kcmd/semantic/runtime/store.js +21 -37
- package/build/ts/kcmd/semantic/source_dialect.d.ts +9 -2
- package/build/ts/kcmd/semantic/source_dialect.js +83 -33
- package/build/ts/kcmd/semantic/source_entries.d.ts +1 -1
- package/build/ts/kcmd/semantic/source_entries.js +95 -36
- package/build/ts/kcmd/semantic/sql_expr_utils.d.ts +13 -7
- package/build/ts/kcmd/semantic/sql_expr_utils.js +240 -67
- package/build/ts/kcmd/semantic/sql_parser.d.ts +5 -2
- package/build/ts/kcmd/semantic/sql_parser.js +22 -0
- package/build/ts/kcmd/semantic/transpile.d.ts +12 -10
- package/build/ts/kcmd/semantic/transpile.js +79 -61
- package/build/ts/kcmd/semantic/validate.d.ts +37 -2
- package/build/ts/kcmd/semantic/validate.js +467 -126
- package/build/ts/kcmd/snapshot.d.ts +4 -0
- package/build/ts/kcmd/snapshot.js +7 -0
- package/build/ts/kcmd/sync.js +44 -7
- package/package.json +1 -1
|
@@ -50,6 +50,7 @@ exports.undefinedEndpointProblem = undefinedEndpointProblem;
|
|
|
50
50
|
exports.isFormatVersion = isFormatVersion;
|
|
51
51
|
exports.noSqlReason = noSqlReason;
|
|
52
52
|
exports.listed = listed;
|
|
53
|
+
exports.plural = plural;
|
|
53
54
|
exports.expressionDocOf = expressionDocOf;
|
|
54
55
|
exports.noSqlProblemOf = noSqlProblemOf;
|
|
55
56
|
exports.inheritedNoSqlProblems = inheritedNoSqlProblems;
|
|
@@ -85,9 +86,9 @@ exports.sourceToDataSource = sourceToDataSource;
|
|
|
85
86
|
exports.thirdPartyEntryOf = thirdPartyEntryOf;
|
|
86
87
|
exports.bindsThirdPartyTable = bindsThirdPartyTable;
|
|
87
88
|
exports.dialectLookupKey = dialectLookupKey;
|
|
89
|
+
exports.spannerDatabaseOf = spannerDatabaseOf;
|
|
88
90
|
exports.isGoogleCloudForm = isGoogleCloudForm;
|
|
89
91
|
exports.parseSource = parseSource;
|
|
90
|
-
exports.unquote = unquote;
|
|
91
92
|
exports.bareNameSegments = bareNameSegments;
|
|
92
93
|
exports.bareNameParts = bareNameParts;
|
|
93
94
|
exports.loadSemanticModels = loadSemanticModels;
|
|
@@ -131,9 +132,6 @@ function sourceRequiredMessage(dataset) {
|
|
|
131
132
|
return `dataset '${dataset}': a non-abstract dataset requires a ` +
|
|
132
133
|
`source; set 'source', or mark it 'abstract: true' if it has no table`;
|
|
133
134
|
}
|
|
134
|
-
function noPrimaryKeyWarning(dataset) {
|
|
135
|
-
return `dataset '${dataset}': no primary_key; the entity's KEY will be empty (invalid for graph generation)`;
|
|
136
|
-
}
|
|
137
135
|
function ansiFallbackNote(preferred) {
|
|
138
136
|
return `note: no '${preferred}' dialect for one or more expressions; using the portable ` +
|
|
139
137
|
`'${FALLBACK_DIALECT}' dialect verbatim ('${preferred}' accepts the ANSI core subset — ` +
|
|
@@ -261,7 +259,7 @@ function noSqlReason(text, dialect, entity) {
|
|
|
261
259
|
return 'is only a comment';
|
|
262
260
|
if (entity === undefined)
|
|
263
261
|
return undefined;
|
|
264
|
-
const onlyQualifier = (d) => (0, sql_expr_utils_1.
|
|
262
|
+
const onlyQualifier = (d) => (0, sql_expr_utils_1.withoutQualifier)((0, sql_parser_1.withoutComments)(text, d), entity, d).trim() === '';
|
|
265
263
|
return readers.some(onlyQualifier) ? `is only the qualifier '${entity}.'` :
|
|
266
264
|
undefined;
|
|
267
265
|
}
|
|
@@ -280,10 +278,15 @@ function noSqlProblem(raw, entity) {
|
|
|
280
278
|
}));
|
|
281
279
|
}
|
|
282
280
|
// `problems` as one list, "a, b and c", or undefined when there are none.
|
|
283
|
-
|
|
281
|
+
// `conjunction` replaces the "and".
|
|
282
|
+
function listed(problems, conjunction = 'and') {
|
|
284
283
|
if (problems.length < 2)
|
|
285
284
|
return problems[0];
|
|
286
|
-
return `${problems.slice(0, -1).join(', ')}
|
|
285
|
+
return `${problems.slice(0, -1).join(', ')} ${conjunction} ${problems[problems.length - 1]}`;
|
|
286
|
+
}
|
|
287
|
+
// Selects the singular or plural form based on `n` (English count agreement).
|
|
288
|
+
function plural(n, one, many) {
|
|
289
|
+
return n === 1 ? one : many;
|
|
287
290
|
}
|
|
288
291
|
// Whether reading `doc` for a field of `entity` names `text`, or finds every
|
|
289
292
|
// problem that reading `text` in `dialect` finds. The string form and an
|
|
@@ -417,10 +420,7 @@ const googleAiContextSchema = z.union([
|
|
|
417
420
|
z.string(),
|
|
418
421
|
z.object({
|
|
419
422
|
...aiContextMembers,
|
|
420
|
-
custom: z.record(z.string(), z.unknown(),
|
|
421
|
-
error: `ai_context 'custom' must be a mapping of custom ` +
|
|
422
|
-
`member names to values`,
|
|
423
|
-
}).optional(),
|
|
423
|
+
custom: z.record(z.string(), z.unknown()).optional(),
|
|
424
424
|
})
|
|
425
425
|
.strict()
|
|
426
426
|
.superRefine((ai, ctx) => {
|
|
@@ -707,10 +707,6 @@ const modelBase = z.object({
|
|
|
707
707
|
datasets: z.array(datasetBase).min(1),
|
|
708
708
|
relationships: z.array(relationshipSchema).optional(),
|
|
709
709
|
metrics: z.array(metricSchema).optional(),
|
|
710
|
-
// A native deployment-target key (GOOGLE_VERSION only, under
|
|
711
|
-
// allowLegacyBareSource). Folded into a GOOGLE `custom_extensions` block on
|
|
712
|
-
// the IR after validation (see convertModel).
|
|
713
|
-
deployment_target: z.string().optional(),
|
|
714
710
|
// Named deployment targets (GOOGLE_VERSION only).
|
|
715
711
|
deployments: z.array(deploymentSchema).optional(),
|
|
716
712
|
// Model-level write operations, also GOOGLE_VERSION only (see actionSchema).
|
|
@@ -723,6 +719,14 @@ const modelBase = z.object({
|
|
|
723
719
|
// separately. Actions and constraints are extended-profile constructs with no
|
|
724
720
|
// `custom_extensions` encoding, so they do not take it.
|
|
725
721
|
const customExtensionsKey = { custom_extensions: z.array(customExtensionSchema).optional() };
|
|
722
|
+
// The schemas of keys a document schema accepts only so that the loader can
|
|
723
|
+
// reject them with its own message. A message about an unknown key does not
|
|
724
|
+
// list them among the keys an object takes.
|
|
725
|
+
const UNLISTED_KEYS = new WeakSet();
|
|
726
|
+
function unlisted(schema) {
|
|
727
|
+
UNLISTED_KEYS.add(schema);
|
|
728
|
+
return schema;
|
|
729
|
+
}
|
|
726
730
|
// Builds the document schema for one (bindingOptional, extended) combination.
|
|
727
731
|
// Every object is `.strict()`, so an unknown key is a hard error rather than
|
|
728
732
|
// silently dropped. Two axes shape it:
|
|
@@ -770,10 +774,10 @@ function buildDocumentSchema(bindingOptional, extended) {
|
|
|
770
774
|
// dataset and the fix, not a bare "unrecognized key".
|
|
771
775
|
extends: placement.dataset.extends === 'native' ?
|
|
772
776
|
z.array(z.string()).optional() :
|
|
773
|
-
z.unknown().optional(),
|
|
777
|
+
unlisted(z.unknown().optional()),
|
|
774
778
|
abstract: placement.dataset.abstract === 'native' ?
|
|
775
779
|
z.boolean().optional() :
|
|
776
|
-
z.unknown().optional(),
|
|
780
|
+
unlisted(z.unknown().optional()),
|
|
777
781
|
})
|
|
778
782
|
.strict()
|
|
779
783
|
.superRefine((ds, ctx) => {
|
|
@@ -821,7 +825,7 @@ function buildDocumentSchema(bindingOptional, extended) {
|
|
|
821
825
|
ctx.addIssue({
|
|
822
826
|
code: z.ZodIssueCode.custom,
|
|
823
827
|
path: ['source'],
|
|
824
|
-
message: `
|
|
828
|
+
message: `entity '${ds.name}': an abstract entity has no table; ` +
|
|
825
829
|
`remove 'source', or drop 'abstract: true' to bind it to that table`,
|
|
826
830
|
});
|
|
827
831
|
}
|
|
@@ -855,7 +859,10 @@ function buildDocumentSchema(bindingOptional, extended) {
|
|
|
855
859
|
to: z.string(),
|
|
856
860
|
from_columns: z.array(z.string()).min(1).optional(),
|
|
857
861
|
to_columns: z.array(z.string()).min(1).optional(),
|
|
858
|
-
description
|
|
862
|
+
// Vanilla rejects a relationship's description after the schema.
|
|
863
|
+
description: placement.relationship.description === 'native' ?
|
|
864
|
+
z.string().optional() :
|
|
865
|
+
unlisted(z.unknown().optional()),
|
|
859
866
|
ai_context: aiContext.optional(),
|
|
860
867
|
...customExtensionsKey,
|
|
861
868
|
})
|
|
@@ -880,8 +887,9 @@ function buildDocumentSchema(bindingOptional, extended) {
|
|
|
880
887
|
// An authored anchor is a native extension: extended profile only.
|
|
881
888
|
// Vanilla still declares the key so the refinement below can reject it
|
|
882
889
|
// with a message that names the fix, not a bare "unrecognized key".
|
|
883
|
-
entity: placement.metric.entity === 'native' ?
|
|
884
|
-
z.
|
|
890
|
+
entity: placement.metric.entity === 'native' ?
|
|
891
|
+
z.string().optional() :
|
|
892
|
+
unlisted(z.unknown().optional()),
|
|
885
893
|
})
|
|
886
894
|
.strict()
|
|
887
895
|
.superRefine((mt, ctx) => {
|
|
@@ -935,7 +943,7 @@ function buildDocumentSchema(bindingOptional, extended) {
|
|
|
935
943
|
// rejectExpressionBody, which is what turns it into a useful error.
|
|
936
944
|
const constraint = z.object({
|
|
937
945
|
name: z.string(),
|
|
938
|
-
expression: z.
|
|
946
|
+
expression: unlisted(z.unknown().optional()),
|
|
939
947
|
judgment: z.string().optional(),
|
|
940
948
|
description: z.string().optional(),
|
|
941
949
|
// What a violation does; required. See
|
|
@@ -950,7 +958,6 @@ function buildDocumentSchema(bindingOptional, extended) {
|
|
|
950
958
|
.superRefine(rejectExpressionBody);
|
|
951
959
|
const modelExtensions = {
|
|
952
960
|
...(placement.model.deployments === 'native' ? {
|
|
953
|
-
deployment_target: z.string().optional(),
|
|
954
961
|
deployments: z.array(deploymentSchema).optional(),
|
|
955
962
|
} :
|
|
956
963
|
{}),
|
|
@@ -1058,18 +1065,15 @@ function composeDescription(...parts) {
|
|
|
1058
1065
|
.filter((p) => !!p);
|
|
1059
1066
|
return kept.length ? kept.join('\n\n') : undefined;
|
|
1060
1067
|
}
|
|
1061
|
-
// The vendor tag for Google-specific extension blocks
|
|
1062
|
-
// deploy leg's reader). A model-level `deployments:` list folds into one.
|
|
1068
|
+
// The vendor tag for Google-specific extension blocks.
|
|
1063
1069
|
exports.GOOGLE_VENDOR = 'GOOGLE';
|
|
1064
1070
|
// Rewrites the author-friendly sugar forms into the canonical wire shape the
|
|
1065
1071
|
// schema validates, and rejects deprecated or flavor-mismatched model keys up
|
|
1066
1072
|
// front with actionable messages: the removed `deployment_target:` key,
|
|
1067
1073
|
// inline `profiles:` in either flavor, and `deployments:` or
|
|
1068
1074
|
// `entities:` in vanilla. Under the extended profile, `entities:` is rewritten
|
|
1069
|
-
// to `datasets
|
|
1070
|
-
|
|
1071
|
-
// convertModel). Operates on the parsed document before validation.
|
|
1072
|
-
function normalizeDocumentSugars(doc, extended, allowLegacyBareSource = false) {
|
|
1075
|
+
// to `datasets:`. Operates on the parsed document before validation.
|
|
1076
|
+
function normalizeDocumentSugars(doc, extended) {
|
|
1073
1077
|
if (!doc || typeof doc !== 'object')
|
|
1074
1078
|
return doc;
|
|
1075
1079
|
const cloned = structuredClone(doc);
|
|
@@ -1078,31 +1082,17 @@ function normalizeDocumentSugars(doc, extended, allowLegacyBareSource = false) {
|
|
|
1078
1082
|
return cloned;
|
|
1079
1083
|
for (const m of models) {
|
|
1080
1084
|
if (m && typeof m === 'object') {
|
|
1081
|
-
normalizeModelSugars(m, extended
|
|
1085
|
+
normalizeModelSugars(m, extended);
|
|
1082
1086
|
}
|
|
1083
1087
|
}
|
|
1084
1088
|
return cloned;
|
|
1085
1089
|
}
|
|
1086
|
-
function normalizeModelSugars(m, extended
|
|
1090
|
+
function normalizeModelSugars(m, extended) {
|
|
1087
1091
|
const label = typeof m.name === 'string' ? `model '${m.name}'` : 'model';
|
|
1088
1092
|
const placement = exports.FORMAT_VERSION_PLACEMENT[extended ? exports.GOOGLE_VERSION : exports.OSSIE_VERSION];
|
|
1089
|
-
// Only a load that still accepts the legacy key can be left holding both.
|
|
1090
|
-
// Elsewhere the `deployment_target:` rejection below speaks to it. The open
|
|
1091
|
-
// format has neither key, so a vanilla document is told to switch flavor.
|
|
1092
|
-
if (allowLegacyBareSource && m.deployment_target !== undefined &&
|
|
1093
|
-
m.deployments !== undefined) {
|
|
1094
|
-
throw new SemanticModelError(extended ?
|
|
1095
|
-
`Semantic model load error: ${label}: set either 'deployments' ` +
|
|
1096
|
-
`or 'deployment_target', not both.` :
|
|
1097
|
-
`Semantic model load error: ${label}: neither 'deployments' nor ` +
|
|
1098
|
-
`'deployment_target' is in the open format ` +
|
|
1099
|
-
`('${exports.OSSIE_VERSION}'); set version: '${exports.GOOGLE_VERSION}', ` +
|
|
1100
|
-
`keep 'deployments' and remove 'deployment_target'.`);
|
|
1101
|
-
}
|
|
1102
1093
|
// `deployment_target` has been replaced by the named `deployments:` list in
|
|
1103
|
-
// the extended profile (and was never in vanilla).
|
|
1104
|
-
|
|
1105
|
-
if (m.deployment_target !== undefined && !allowLegacyBareSource) {
|
|
1094
|
+
// the extended profile (and was never in vanilla).
|
|
1095
|
+
if (m.deployment_target !== undefined) {
|
|
1106
1096
|
if (!extended) {
|
|
1107
1097
|
throw new SemanticModelError(`Semantic model load error: ${label}: 'deployment_target' is not ` +
|
|
1108
1098
|
`in the open format ('${exports.OSSIE_VERSION}'); ` +
|
|
@@ -1161,18 +1151,6 @@ function normalizeModelSugars(m, extended, allowLegacyBareSource = false) {
|
|
|
1161
1151
|
delete m.entities;
|
|
1162
1152
|
}
|
|
1163
1153
|
}
|
|
1164
|
-
// Builds the GOOGLE custom-extension block that carries a model's deployment
|
|
1165
|
-
// target URIs on the IR (the form the deploy leg reads). Both the native
|
|
1166
|
-
// `deployments:` list and (under `allowLegacyBareSource`) the legacy
|
|
1167
|
-
// `deployment_target:` key fold into this block after validation (see
|
|
1168
|
-
// convertModel). The extended profile rejects any authored GOOGLE block (see
|
|
1169
|
-
// rejectGoogleBlock), so there is never a pre-existing one to reconcile with.
|
|
1170
|
-
function deploymentTargetsExtension(targets) {
|
|
1171
|
-
return {
|
|
1172
|
-
vendorName: exports.GOOGLE_VENDOR,
|
|
1173
|
-
data: JSON.stringify({ deploymentTargets: targets }),
|
|
1174
|
-
};
|
|
1175
|
-
}
|
|
1176
1154
|
// True when `vendorName` names the GOOGLE vendor, in any letter case.
|
|
1177
1155
|
function isGoogleVendor(vendorName) {
|
|
1178
1156
|
return typeof vendorName === 'string' &&
|
|
@@ -1282,7 +1260,7 @@ function readGoogleBlocks(exts, version, level, ctx) {
|
|
|
1282
1260
|
}
|
|
1283
1261
|
const parsed = GOOGLE_BLOCK_SCHEMAS[key].safeParse(value);
|
|
1284
1262
|
if (!parsed.success) {
|
|
1285
|
-
throw error(
|
|
1263
|
+
throw error(keyProblems(parsed.error, GOOGLE_BLOCK_SCHEMAS[key], key, value, `the ${block}`).replaceAll('\n', `\n${ctx}: `));
|
|
1286
1264
|
}
|
|
1287
1265
|
values[key] = parsed.data;
|
|
1288
1266
|
}
|
|
@@ -1756,11 +1734,14 @@ function fromDocument(doc, opts = {}) {
|
|
|
1756
1734
|
if (proto) {
|
|
1757
1735
|
throw new SemanticModelError(`Semantic model load error: ${proto}`);
|
|
1758
1736
|
}
|
|
1759
|
-
const normalized = normalizeDocumentSugars(doc, extended
|
|
1760
|
-
const
|
|
1761
|
-
|
|
1737
|
+
const normalized = normalizeDocumentSugars(doc, extended);
|
|
1738
|
+
const schema = makeDocumentSchema(opts.bindingOptional ?? false, extended);
|
|
1739
|
+
const result = schema.safeParse(normalized);
|
|
1762
1740
|
if (!result.success) {
|
|
1763
|
-
throw new SemanticModelError(`Semantic model load error: ${result.error.
|
|
1741
|
+
throw new SemanticModelError(`Semantic model load error: ${schemaProblems(result.error.issues, {
|
|
1742
|
+
schema, doc: normalized, authored: doc, extended,
|
|
1743
|
+
subject: 'the document',
|
|
1744
|
+
})}`);
|
|
1764
1745
|
}
|
|
1765
1746
|
const warnings = [];
|
|
1766
1747
|
const parsed = result.data;
|
|
@@ -1823,16 +1804,19 @@ function runsDialect(entity, dialect) {
|
|
|
1823
1804
|
return (0, resolve_profiles_1.sourcesDialect)({ entities: [entity] }) === dialect.toUpperCase();
|
|
1824
1805
|
}
|
|
1825
1806
|
// The warning `item` gives on `table`, the table of `entity`:
|
|
1826
|
-
// expressionWarning's when the load knows the table's dialect, or
|
|
1827
|
-
//
|
|
1807
|
+
// expressionWarning's when the load knows the table's dialect, or one saying
|
|
1808
|
+
// the table has no text to run. With `profiled`, the second is given only on a
|
|
1809
|
+
// Cloud SQL, Spanner or third-party table no lookup has read. On a table of the
|
|
1828
1810
|
// `preferred` dialect, expressionWarning's transpilation warning already says
|
|
1829
1811
|
// so.
|
|
1830
1812
|
function tableWarning(item, subject, table, entity, preferred, profiled) {
|
|
1831
1813
|
const w = table.unread ?
|
|
1832
1814
|
undefined :
|
|
1833
1815
|
expressionWarning(item, subject, table.name, preferred);
|
|
1834
|
-
if (w || profiled
|
|
1816
|
+
if (w || (profiled && !table.unread && !table.guessed) ||
|
|
1817
|
+
!(0, resolve_profiles_1.noTextFor)(item, table.dialects)) {
|
|
1835
1818
|
return w;
|
|
1819
|
+
}
|
|
1836
1820
|
const own = table.dialects.map(d => `'${d}'`).join(', ');
|
|
1837
1821
|
return `${subject.ctx}: no ${own} or '${FALLBACK_DIALECT}' dialect, so the ${table.name} table of entity '${entity}' cannot run it`;
|
|
1838
1822
|
}
|
|
@@ -1865,19 +1849,13 @@ function metricWarning(metric, reached, entities, preferred, profiled) {
|
|
|
1865
1849
|
}
|
|
1866
1850
|
return undefined;
|
|
1867
1851
|
}
|
|
1868
|
-
// The warning a concrete entity with neither a primary key nor a unique key
|
|
1869
|
-
// gives, or undefined. A unique key is enough, as the graph keys the entity's
|
|
1870
|
-
// node by the first one.
|
|
1871
|
-
function keyWarning(entity) {
|
|
1872
|
-
return !(0, resolve_profiles_1.statesKey)(entity.keys, entity.uniqueKeys) && !entity.abstract ?
|
|
1873
|
-
noPrimaryKeyWarning(entity.name) :
|
|
1874
|
-
undefined;
|
|
1875
|
-
}
|
|
1876
1852
|
// Infers the entities a metric's expressions reach across all dialect entries,
|
|
1877
1853
|
// and returns the warning the metric gives when an expression could not be
|
|
1878
1854
|
// parsed or when it states no `entity` and its expressions name none. An
|
|
1879
1855
|
// authored anchor, which vanilla writes in a GOOGLE block, settles which entity
|
|
1880
|
-
// the metric belongs to, so a `COUNT(*)` with one is not warned about.
|
|
1856
|
+
// the metric belongs to, so a `COUNT(*)` with one is not warned about. An
|
|
1857
|
+
// ANSI_SQL text whose `impliedDialect` no lookup read (null) is read as
|
|
1858
|
+
// generic SQL, the closest reading there is.
|
|
1881
1859
|
function inferMetricEntities(metric, entityNames, preferred = exports.DEFAULT_DIALECT, impliedDialect, version) {
|
|
1882
1860
|
const ctx = `metric '${metric.name}'`;
|
|
1883
1861
|
const sources = (metric.dialects ?? [])
|
|
@@ -1895,7 +1873,7 @@ function inferMetricEntities(metric, entityNames, preferred = exports.DEFAULT_DI
|
|
|
1895
1873
|
for (const { text, dialect } of sources) {
|
|
1896
1874
|
if (!text)
|
|
1897
1875
|
continue;
|
|
1898
|
-
const cols = (0, sql_expr_utils_1.columnsForSource)(text, dialect, impliedDialect);
|
|
1876
|
+
const cols = (0, sql_expr_utils_1.columnsForSource)(text, dialect, impliedDialect ?? undefined);
|
|
1899
1877
|
if (cols === undefined) {
|
|
1900
1878
|
unparsed = true;
|
|
1901
1879
|
}
|
|
@@ -1923,16 +1901,14 @@ function inferMetricEntities(metric, entityNames, preferred = exports.DEFAULT_DI
|
|
|
1923
1901
|
return { referenced };
|
|
1924
1902
|
}
|
|
1925
1903
|
// Every warning a load gives about `model`'s entities and metrics, in the
|
|
1926
|
-
// order a load gives them: each
|
|
1927
|
-
// expression warning
|
|
1928
|
-
//
|
|
1929
|
-
//
|
|
1930
|
-
//
|
|
1931
|
-
// warnings, about actions, come after them.
|
|
1904
|
+
// order a load gives them: each field's expression warning, then each metric's
|
|
1905
|
+
// expression warning and its warning when it names no entity. A metric a
|
|
1906
|
+
// profile excludes gives none. With `profiled`, the profile check reports a
|
|
1907
|
+
// field or metric its tables cannot run, so no warning repeats it. A profile
|
|
1908
|
+
// changes only these; a load's other warnings, about actions, come after them.
|
|
1932
1909
|
function entityAndMetricWarnings(model, preferred = exports.DEFAULT_DIALECT, profiled = false) {
|
|
1933
1910
|
const entities = model.entities ?? [];
|
|
1934
|
-
const out =
|
|
1935
|
-
out.push(...fieldWarnings(model, preferred, profiled));
|
|
1911
|
+
const out = fieldWarnings(model, preferred, profiled);
|
|
1936
1912
|
const entityNames = entities.map(e => e.name);
|
|
1937
1913
|
const impliedDialect = (0, resolve_profiles_1.sourcesDialect)(model);
|
|
1938
1914
|
const excluded = new Set(model.excludedMetrics);
|
|
@@ -2074,10 +2050,6 @@ function convertModel(m, version, opts, warnings) {
|
|
|
2074
2050
|
}
|
|
2075
2051
|
seenTargets.set(d.target, d.name);
|
|
2076
2052
|
}
|
|
2077
|
-
// A blank `deployment_target` names no target, as push reads it.
|
|
2078
|
-
if (!deployments.length && m.deployment_target?.trim()) {
|
|
2079
|
-
deployments.push({ name: 'default', target: m.deployment_target.trim() });
|
|
2080
|
-
}
|
|
2081
2053
|
const description = composeDescription(m.description);
|
|
2082
2054
|
// `version` records the flavor the document was written in, so a consumer
|
|
2083
2055
|
// (e.g. a pull) can return the model in that same flavor.
|
|
@@ -2095,20 +2067,9 @@ function convertModel(m, version, opts, warnings) {
|
|
|
2095
2067
|
const ai = aiContextOrUndefined(m.ai_context);
|
|
2096
2068
|
if (ai)
|
|
2097
2069
|
model.aiContext = ai;
|
|
2098
|
-
// The blocks the model keeps are carried verbatim.
|
|
2099
|
-
// profile the native `deployments` list (or legacy `deployment_target` key)
|
|
2100
|
-
// is folded into a GOOGLE block appended after them; no authored GOOGLE block
|
|
2101
|
-
// can collide with it, since that flavor rejects one (see rejectGoogleBlock).
|
|
2102
|
-
// Vanilla deployments, read from a GOOGLE block, are not folded.
|
|
2070
|
+
// The blocks the model keeps are carried verbatim.
|
|
2103
2071
|
if (block.kept)
|
|
2104
2072
|
model.customExtensions = block.kept;
|
|
2105
|
-
const foldedTargets = version === exports.GOOGLE_VERSION ? deployments.map(d => d.target) : [];
|
|
2106
|
-
if (foldedTargets.length) {
|
|
2107
|
-
model.customExtensions = [
|
|
2108
|
-
...(model.customExtensions ?? []),
|
|
2109
|
-
deploymentTargetsExtension(foldedTargets),
|
|
2110
|
-
];
|
|
2111
|
-
}
|
|
2112
2073
|
return model;
|
|
2113
2074
|
}
|
|
2114
2075
|
// Converts the dataset `ds`, whose GOOGLE blocks convertModel has read into
|
|
@@ -2118,15 +2079,12 @@ function convertDataset(ds, block, version, opts, warnings, dialect) {
|
|
|
2118
2079
|
const ctxLabel = `dataset '${ds.name}'`;
|
|
2119
2080
|
// An abstract entity has no physical table, so it carries no source (empty
|
|
2120
2081
|
// dataSource) and no key -- both are meaningless for a class never
|
|
2121
|
-
// materialized. Only a concrete entity is parsed
|
|
2082
|
+
// materialized. Only a concrete entity's source is parsed.
|
|
2122
2083
|
const parsedSource = ds.source !== undefined ?
|
|
2123
|
-
parseSource(ds.source, opts, ctxLabel, warnings,
|
|
2084
|
+
parseSource(ds.source, opts, ctxLabel, warnings, version) :
|
|
2124
2085
|
undefined;
|
|
2125
2086
|
const dataSource = parsedSource ? parsedSource.dataSource : '';
|
|
2126
2087
|
const keys = ds.primary_key ?? [];
|
|
2127
|
-
const kw = keyWarning({ name: ds.name, keys, uniqueKeys: ds.unique_keys, abstract: ds.abstract });
|
|
2128
|
-
if (kw)
|
|
2129
|
-
warnings.push(kw);
|
|
2130
2088
|
const fields = (ds.fields ?? []).map(f => convertField(f, ds.name, version, dialect));
|
|
2131
2089
|
rejectDuplicateNames(fields.map(f => f.name), 'field name', `dataset '${ds.name}'`, true);
|
|
2132
2090
|
const entity = { name: ds.name, dataSource, keys, fields };
|
|
@@ -2231,7 +2189,7 @@ function readFieldExpression(raw, where, dialect, problemOf) {
|
|
|
2231
2189
|
z.string().safeParse(raw) :
|
|
2232
2190
|
expressionObjectSchema.safeParse(raw);
|
|
2233
2191
|
if (!result.success) {
|
|
2234
|
-
throw new SemanticModelError(
|
|
2192
|
+
throw new SemanticModelError(keyProblems(result.error, expressionObjectSchema, 'expression', raw, where));
|
|
2235
2193
|
}
|
|
2236
2194
|
const problem = problemOf(result.data);
|
|
2237
2195
|
if (problem) {
|
|
@@ -2645,11 +2603,178 @@ function convertParameter(p, actionName, concepts, warnings, inheritanceResolved
|
|
|
2645
2603
|
}
|
|
2646
2604
|
return param;
|
|
2647
2605
|
}
|
|
2648
|
-
//
|
|
2649
|
-
|
|
2650
|
-
|
|
2651
|
-
|
|
2652
|
-
|
|
2606
|
+
// One line per problem that `schema` finds in the value of `key` on
|
|
2607
|
+
// `subject`.
|
|
2608
|
+
function keyProblems(error, schema, key, value, subject) {
|
|
2609
|
+
const doc = { [key]: value };
|
|
2610
|
+
return schemaProblems(error.issues.map(i => ({ ...i, path: [key, ...i.path] })), { schema: z.object({ [key]: schema }), doc, authored: doc, extended: true,
|
|
2611
|
+
subject });
|
|
2612
|
+
}
|
|
2613
|
+
// The noun a message gives an entry of each list of named objects in a model
|
|
2614
|
+
// document. An entry of `datasets` is an entity in the Google flavor.
|
|
2615
|
+
const ENTRY_NOUNS = new Map([
|
|
2616
|
+
['semantic_model', 'model'], ['datasets', 'dataset'], ['fields', 'field'],
|
|
2617
|
+
['relationships', 'relationship'], ['metrics', 'metric'],
|
|
2618
|
+
['actions', 'action'], ['parameters', 'parameter'],
|
|
2619
|
+
['constraints', 'constraint'], ['deployments', 'deployment'],
|
|
2620
|
+
]);
|
|
2621
|
+
// What a message says a schema expects.
|
|
2622
|
+
const EXPECTED = {
|
|
2623
|
+
string: 'a string',
|
|
2624
|
+
boolean: 'true or false',
|
|
2625
|
+
array: 'a list',
|
|
2626
|
+
object: 'a mapping',
|
|
2627
|
+
record: 'a mapping',
|
|
2628
|
+
};
|
|
2629
|
+
// A value as a message describes it.
|
|
2630
|
+
function described(value) {
|
|
2631
|
+
if (value === undefined)
|
|
2632
|
+
return 'missing';
|
|
2633
|
+
if (value === null)
|
|
2634
|
+
return 'empty';
|
|
2635
|
+
if (Array.isArray(value))
|
|
2636
|
+
return 'a list';
|
|
2637
|
+
if (typeof value === 'object')
|
|
2638
|
+
return 'a mapping';
|
|
2639
|
+
if (typeof value !== 'string')
|
|
2640
|
+
return String(value);
|
|
2641
|
+
return value.includes('\n') ? 'a string' : `'${value}'`;
|
|
2642
|
+
}
|
|
2643
|
+
// The keys the object at `path` in `schema` takes, apart from unlisted keys.
|
|
2644
|
+
function keysAt(schema, path) {
|
|
2645
|
+
// `s` without its optional, nullable and pipe wrappers. Of a union, the
|
|
2646
|
+
// member that is a `kind`.
|
|
2647
|
+
const unwrapped = (s, kind) => {
|
|
2648
|
+
if (s instanceof z.ZodOptional || s instanceof z.ZodNullable) {
|
|
2649
|
+
return unwrapped(s.unwrap(), kind);
|
|
2650
|
+
}
|
|
2651
|
+
if (s instanceof z.ZodPipe)
|
|
2652
|
+
return unwrapped(s.in, kind);
|
|
2653
|
+
if (s instanceof z.ZodUnion) {
|
|
2654
|
+
return s.options.map(o => unwrapped(o, kind))
|
|
2655
|
+
.find(o => o instanceof kind);
|
|
2656
|
+
}
|
|
2657
|
+
return s;
|
|
2658
|
+
};
|
|
2659
|
+
let node = schema;
|
|
2660
|
+
for (const seg of path) {
|
|
2661
|
+
const holder = unwrapped(node, typeof seg === 'number' ? z.ZodArray : z.ZodObject);
|
|
2662
|
+
node = holder instanceof z.ZodArray ? holder.element :
|
|
2663
|
+
holder instanceof z.ZodObject ? holder.shape[String(seg)] :
|
|
2664
|
+
undefined;
|
|
2665
|
+
}
|
|
2666
|
+
const object = unwrapped(node, z.ZodObject);
|
|
2667
|
+
if (!(object instanceof z.ZodObject))
|
|
2668
|
+
return [];
|
|
2669
|
+
return Object.keys(object.shape)
|
|
2670
|
+
.filter(k => !UNLISTED_KEYS.has(object.shape[k]));
|
|
2671
|
+
}
|
|
2672
|
+
// Where `path` points in `input.doc`: the innermost named object, as messages
|
|
2673
|
+
// name it, and the key below it, such as "'data' of entry 1 of
|
|
2674
|
+
// 'custom_extensions'". `spelled` gives a key as the author wrote it: a
|
|
2675
|
+
// Google-flavor model's `datasets` is `entities` unless the author wrote
|
|
2676
|
+
// `datasets`.
|
|
2677
|
+
function schemaLocation(path, input) {
|
|
2678
|
+
let subject = input.subject;
|
|
2679
|
+
let name;
|
|
2680
|
+
let entity;
|
|
2681
|
+
let datasets = 'datasets';
|
|
2682
|
+
let rest = [];
|
|
2683
|
+
let node = input.doc;
|
|
2684
|
+
for (const seg of path) {
|
|
2685
|
+
node = node?.[seg];
|
|
2686
|
+
const list = rest.length === 1 ? String(rest[0]) : '';
|
|
2687
|
+
const noun = list === 'datasets' && input.extended ?
|
|
2688
|
+
'entity' :
|
|
2689
|
+
ENTRY_NOUNS.get(list);
|
|
2690
|
+
if (typeof seg !== 'number' || !noun) {
|
|
2691
|
+
rest.push(seg);
|
|
2692
|
+
continue;
|
|
2693
|
+
}
|
|
2694
|
+
rest = [];
|
|
2695
|
+
name = typeof node?.name === 'string' ? node.name : undefined;
|
|
2696
|
+
const named = name === undefined ? `${noun} number ${seg + 1}` : `${noun} '${name}'`;
|
|
2697
|
+
if (list === 'semantic_model') {
|
|
2698
|
+
const written = input.authored?.semantic_model?.[seg];
|
|
2699
|
+
datasets = input.extended && written?.datasets === undefined ?
|
|
2700
|
+
'entities' :
|
|
2701
|
+
'datasets';
|
|
2702
|
+
}
|
|
2703
|
+
if (list === 'datasets')
|
|
2704
|
+
entity = name;
|
|
2705
|
+
if (list === 'fields' && name !== undefined && entity !== undefined) {
|
|
2706
|
+
name = `${entity}.${name}`;
|
|
2707
|
+
subject = `field '${name}'`;
|
|
2708
|
+
}
|
|
2709
|
+
else {
|
|
2710
|
+
subject = list === 'fields' || list === 'parameters' ?
|
|
2711
|
+
`${named} of ${subject}` :
|
|
2712
|
+
named;
|
|
2713
|
+
}
|
|
2714
|
+
}
|
|
2715
|
+
const spelled = (key) => key === 'datasets' ? datasets : String(key);
|
|
2716
|
+
const place = rest.map(s => typeof s === 'number' ? `entry ${s + 1}` :
|
|
2717
|
+
`'${spelled(s)}'`)
|
|
2718
|
+
.reverse()
|
|
2719
|
+
.join(' of ');
|
|
2720
|
+
return { subject, name, place, value: node, spelled };
|
|
2721
|
+
}
|
|
2722
|
+
// One line per problem of a schema error, naming the object and the key and
|
|
2723
|
+
// saying how to fix it.
|
|
2724
|
+
function schemaProblems(issues, input) {
|
|
2725
|
+
// zod also checks the length of a value of the wrong type, as for
|
|
2726
|
+
// `target: []`. The type is what to fix, so that check gives no line.
|
|
2727
|
+
const mistyped = new Set(issues.filter(i => i.code === 'invalid_type')
|
|
2728
|
+
.map(i => JSON.stringify(i.path)));
|
|
2729
|
+
return issues
|
|
2730
|
+
.filter(i => i.code !== 'too_small' || !mistyped.has(JSON.stringify(i.path)))
|
|
2731
|
+
.map(i => schemaProblem(i, input))
|
|
2732
|
+
.join('\n');
|
|
2733
|
+
}
|
|
2734
|
+
function schemaProblem(issue, input) {
|
|
2735
|
+
if (issue.code === 'invalid_union') {
|
|
2736
|
+
// When only one alternative takes the value's type, report its problems.
|
|
2737
|
+
const typed = issue.errors.filter(b => !b.every(i => i.code === 'invalid_type' && !i.path.length));
|
|
2738
|
+
if (typed.length === 1) {
|
|
2739
|
+
return schemaProblems(typed[0].map(i => ({ ...i, path: [...issue.path, ...i.path] })), input);
|
|
2740
|
+
}
|
|
2741
|
+
}
|
|
2742
|
+
const { subject, name, place, value, spelled } = schemaLocation(issue.path, input);
|
|
2743
|
+
const at = place ? `${subject}: ${place}` : subject;
|
|
2744
|
+
const takes = (what) => `${at} is ${described(value)}; it takes ${what}`;
|
|
2745
|
+
const expected = (type) => EXPECTED[type] ?? type;
|
|
2746
|
+
switch (issue.code) {
|
|
2747
|
+
case 'unrecognized_keys': {
|
|
2748
|
+
const keys = keysAt(input.schema, issue.path).map(k => `'${spelled(k)}'`);
|
|
2749
|
+
return `${subject}: unknown key${issue.keys.length > 1 ? 's' : ''} ${listed(issue.keys.map(k => `'${k}'`))}${place ? ` in ${place}` : ''}; it takes ${listed(keys)}`;
|
|
2750
|
+
}
|
|
2751
|
+
case 'invalid_type':
|
|
2752
|
+
return takes(expected(issue.expected));
|
|
2753
|
+
case 'invalid_union': {
|
|
2754
|
+
// A vanilla document writes an expression only as a mapping.
|
|
2755
|
+
const types = issue.errors.flatMap(b => b.flatMap(i => i.code === 'invalid_type' ? [i.expected] : []));
|
|
2756
|
+
const offered = types.filter(t => input.extended || t !== 'string' ||
|
|
2757
|
+
issue.path.at(-1) !== 'expression');
|
|
2758
|
+
return takes(listed([...new Set(offered.map(expected))], 'or'));
|
|
2759
|
+
}
|
|
2760
|
+
case 'invalid_value':
|
|
2761
|
+
return takes(listed(issue.values.map(v => `'${String(v)}'`), 'or'));
|
|
2762
|
+
case 'too_small':
|
|
2763
|
+
if (issue.origin === 'array') {
|
|
2764
|
+
return `${at} is an empty list; it takes at least one entry`;
|
|
2765
|
+
}
|
|
2766
|
+
if (issue.origin === 'string') {
|
|
2767
|
+
return `${at} is empty; it takes a non-empty string`;
|
|
2768
|
+
}
|
|
2769
|
+
break;
|
|
2770
|
+
case 'custom':
|
|
2771
|
+
// A rule on a named object quotes the object's name in its message.
|
|
2772
|
+
if (name !== undefined && issue.message.includes(`'${name}'`)) {
|
|
2773
|
+
return issue.message;
|
|
2774
|
+
}
|
|
2775
|
+
return `${subject}: ${issue.message}`;
|
|
2776
|
+
}
|
|
2777
|
+
return `${at}: ${issue.message}`;
|
|
2653
2778
|
}
|
|
2654
2779
|
// An executor written outside a model document, such as in a profile file,
|
|
2655
2780
|
// checked against the loader's schema and converted as a load converts one.
|
|
@@ -2657,7 +2782,7 @@ function issueList(error) {
|
|
|
2657
2782
|
function parseExecutor(raw, where) {
|
|
2658
2783
|
const result = executorSchema.safeParse(raw);
|
|
2659
2784
|
if (!result.success) {
|
|
2660
|
-
throw new SemanticModelError(
|
|
2785
|
+
throw new SemanticModelError(keyProblems(result.error, executorSchema, 'executor', raw, where));
|
|
2661
2786
|
}
|
|
2662
2787
|
return convertExecutor(result.data);
|
|
2663
2788
|
}
|
|
@@ -2703,7 +2828,9 @@ function pickDialect(expr, preferred) {
|
|
|
2703
2828
|
}
|
|
2704
2829
|
const MAX_CATALOG_NAME_LENGTH = 4000;
|
|
2705
2830
|
const BIGQUERY_TABLE_URI = /^\/\/bigquery\.googleapis\.com\/projects\/([^/]+)\/datasets\/([^/.]+)\/tables\/([^/.]+)$/;
|
|
2706
|
-
|
|
2831
|
+
// A BigLake table sits in a namespace of an Iceberg REST catalog, or in a
|
|
2832
|
+
// database of a Hive catalog.
|
|
2833
|
+
const BIGLAKE_TABLE_URI = /^\/\/biglake\.googleapis\.com\/projects\/([^/]+)\/catalogs\/([^/.]+)\/(namespaces|databases)\/([^/.]+)\/tables\/([^/.]+)$/;
|
|
2707
2834
|
const SPANNER_TABLE_URI = /^\/\/spanner\.googleapis\.com\/projects\/([^/]+)\/instances\/([^/]+)\/databases\/([^/]+)\/tables\/[^/]+$/;
|
|
2708
2835
|
const ALLOYDB_TABLE_URI = /^\/\/alloydb\.googleapis\.com\/projects\/([^/]+)\/locations\/([^/]+)\/clusters\/([^/]+)\/databases\/([^/]+)\/schemas\/[^/]+\/tables\/[^/]+$/;
|
|
2709
2836
|
const LEGACY_ALLOYDB_INSTANCE_URI = /^\/\/alloydb\.googleapis\.com\/projects\/([^/]+)\/locations\/([^/]+)\/clusters\/([^/]+)\/instances\/[^/]+\/databases\/([^/]+)\/tables\/([^/]+)$/;
|
|
@@ -2742,8 +2869,10 @@ function sourceTableOf(source) {
|
|
|
2742
2869
|
};
|
|
2743
2870
|
}
|
|
2744
2871
|
m = uri.match(BIGLAKE_TABLE_URI);
|
|
2745
|
-
if (m)
|
|
2746
|
-
|
|
2872
|
+
if (m) {
|
|
2873
|
+
const hive = m[3] === 'databases';
|
|
2874
|
+
return { system: 'biglake', resourceName, project: m[1], catalog: m[2], hive };
|
|
2875
|
+
}
|
|
2747
2876
|
m = uri.match(SPANNER_TABLE_URI);
|
|
2748
2877
|
if (m)
|
|
2749
2878
|
return { system: 'spanner', resourceName, project: m[1], instance: m[2] };
|
|
@@ -2829,7 +2958,8 @@ function oldSourceForm(source) {
|
|
|
2829
2958
|
exports.NO_SPACE = 'kcmd cannot name a table whose name holds a space';
|
|
2830
2959
|
// The resource name of a table in each Google Cloud service a source can
|
|
2831
2960
|
// name, with a placeholder for each name in it. A Cloud SQL table has one
|
|
2832
|
-
// form without a schema and one with
|
|
2961
|
+
// form without a schema and one with, and a BigLake table one for each kind
|
|
2962
|
+
// of catalog.
|
|
2833
2963
|
exports.TABLE_FORMS = {
|
|
2834
2964
|
bigquery: 'bigquery.googleapis.com/projects/<project>/datasets/<dataset>/tables/<table>',
|
|
2835
2965
|
spanner: 'spanner.googleapis.com/projects/<project>/instances/<instance>/databases/<database>/tables/<table>',
|
|
@@ -2837,11 +2967,14 @@ exports.TABLE_FORMS = {
|
|
|
2837
2967
|
cloudsql: 'cloudsql.googleapis.com/projects/<project>/locations/<location>/instances/<instance>/databases/<database>/tables/<table>',
|
|
2838
2968
|
cloudsqlSchema: 'cloudsql.googleapis.com/projects/<project>/locations/<location>/instances/<instance>/databases/<database>/schemas/<schema>/tables/<table>',
|
|
2839
2969
|
biglake: 'biglake.googleapis.com/projects/<project>/catalogs/<catalog>/namespaces/<namespace>/tables/<table>',
|
|
2970
|
+
biglakeHive: 'biglake.googleapis.com/projects/<project>/catalogs/<catalog>/databases/<database>/tables/<table>',
|
|
2840
2971
|
};
|
|
2841
|
-
//
|
|
2842
|
-
const
|
|
2972
|
+
// What each form in TABLE_FORMS is for, where its service has two.
|
|
2973
|
+
const FORM_USES = {
|
|
2843
2974
|
cloudsql: 'MySQL',
|
|
2844
2975
|
cloudsqlSchema: 'PostgreSQL, SQL Server',
|
|
2976
|
+
biglake: 'Iceberg REST catalog',
|
|
2977
|
+
biglakeHive: 'Hive catalog',
|
|
2845
2978
|
};
|
|
2846
2979
|
// `form` with its last placeholders replaced by `parts`, in order, or
|
|
2847
2980
|
// undefined when `form` has fewer placeholders than `parts`.
|
|
@@ -3095,9 +3228,8 @@ function isAcceptedSource(source) {
|
|
|
3095
3228
|
return source !== undefined && databaseOf(source) !== undefined;
|
|
3096
3229
|
}
|
|
3097
3230
|
// The source an entity is written with: its `authoredSource` when that names a
|
|
3098
|
-
// table in a form `databaseOf` reads, and otherwise its `dataSource`,
|
|
3099
|
-
//
|
|
3100
|
-
// model is serialized, shown and judged by it.
|
|
3231
|
+
// table in a form `databaseOf` reads, and otherwise its `dataSource`, shown as
|
|
3232
|
+
// a resource name. A model is serialized, shown and judged by it.
|
|
3101
3233
|
function writtenSource(entity) {
|
|
3102
3234
|
const authored = entity.authoredSource?.trim();
|
|
3103
3235
|
return authored !== undefined && isAcceptedSource(authored) ?
|
|
@@ -3119,8 +3251,8 @@ function sourceToDataSource(source) {
|
|
|
3119
3251
|
return `${bqUri[1]}.${bqUri[2]}.${bqUri[3]}`;
|
|
3120
3252
|
const biglakeUri = value.match(BIGLAKE_TABLE_URI);
|
|
3121
3253
|
if (biglakeUri) {
|
|
3122
|
-
const [, project, catalog,
|
|
3123
|
-
return `${project}.${catalog}.${
|
|
3254
|
+
const [, project, catalog, , parent, table] = biglakeUri;
|
|
3255
|
+
return `${project}.${catalog}.${parent}.${table}`;
|
|
3124
3256
|
}
|
|
3125
3257
|
return value;
|
|
3126
3258
|
}
|
|
@@ -3160,11 +3292,19 @@ function bindsThirdPartyTable(entity) {
|
|
|
3160
3292
|
return thirdPartyEntryOf(entity.authoredSource ?? '') !== undefined;
|
|
3161
3293
|
}
|
|
3162
3294
|
// The key LoadOptions.sourceDialects holds a source's dialect under: a Cloud
|
|
3163
|
-
// SQL table's resource name,
|
|
3164
|
-
// for a source whose own name gives its dialect.
|
|
3295
|
+
// SQL table's resource name, a third-party table's entry name, or a Spanner
|
|
3296
|
+
// table's database. Undefined for a source whose own name gives its dialect.
|
|
3165
3297
|
function dialectLookupKey(source) {
|
|
3166
3298
|
return cloudSqlTableOf(source)?.resourceName ??
|
|
3167
|
-
thirdPartyEntryOf(source)?.name;
|
|
3299
|
+
thirdPartyEntryOf(source)?.name ?? spannerDatabaseOf(source);
|
|
3300
|
+
}
|
|
3301
|
+
// The resource name of the database the Spanner table `source` is in.
|
|
3302
|
+
// Undefined for any other source and for a name with quotes.
|
|
3303
|
+
function spannerDatabaseOf(source) {
|
|
3304
|
+
const uri = asResourceUri(source.trim());
|
|
3305
|
+
return !QUOTE.test(uri) && SPANNER_TABLE_URI.test(uri) ?
|
|
3306
|
+
uri.slice(2, uri.lastIndexOf('/tables/')) :
|
|
3307
|
+
undefined;
|
|
3168
3308
|
}
|
|
3169
3309
|
// Why `source` is not accepted when it is a Knowledge Catalog entry name in a
|
|
3170
3310
|
// system entry group, one whose id starts with `@`, such as `@bigquery`.
|
|
@@ -3234,14 +3374,12 @@ function thirdPartyNameError(source, ctx) {
|
|
|
3234
3374
|
// Reads an entity's `source` into the dotted name the generators use and the
|
|
3235
3375
|
// source as written. Throws on a form the loader does not accept. A profile
|
|
3236
3376
|
// file's sources are read here too, so they follow the same rules. With
|
|
3237
|
-
// `
|
|
3238
|
-
// names `kcmd import`, which completes a model file's
|
|
3239
|
-
// profile file's.
|
|
3240
|
-
function parseSource(source, opts, ctx, warnings,
|
|
3377
|
+
// `importVersion`, the version of the model file the source is in, a bare
|
|
3378
|
+
// table name's error also names `kcmd import`, which completes a model file's
|
|
3379
|
+
// bare names but not a profile file's.
|
|
3380
|
+
function parseSource(source, opts, ctx, warnings, importVersion) {
|
|
3241
3381
|
const trimmed = source.trim();
|
|
3242
|
-
|
|
3243
|
-
if (hasUnquotedWhitespace(trimmed) || (opts.allowLegacyBareSource &&
|
|
3244
|
-
hasUnquotedWhitespace(trimmed.replaceAll('"', '')))) {
|
|
3382
|
+
if (hasUnquotedWhitespace(trimmed)) {
|
|
3245
3383
|
throw new SemanticModelError(`Semantic model load error: ${ctx}: source '${trimmed}' looks like a ` +
|
|
3246
3384
|
`SQL query; a query-valued source is not supported yet. Bind 'source' ` +
|
|
3247
3385
|
`to a physical table using its GCP resource name (e.g. ` +
|
|
@@ -3288,27 +3426,12 @@ function parseSource(source, opts, ctx, warnings, importHint = false) {
|
|
|
3288
3426
|
!trimmed.includes(':');
|
|
3289
3427
|
let dotted;
|
|
3290
3428
|
if (isBareTableName) {
|
|
3291
|
-
//
|
|
3292
|
-
|
|
3293
|
-
|
|
3294
|
-
|
|
3295
|
-
|
|
3296
|
-
|
|
3297
|
-
}
|
|
3298
|
-
if (parts.length < 3 && opts.defaultProject) {
|
|
3299
|
-
parts.unshift(opts.defaultProject);
|
|
3300
|
-
}
|
|
3301
|
-
return {
|
|
3302
|
-
authoredSource: trimmed,
|
|
3303
|
-
dataSource: parts.join('.'),
|
|
3304
|
-
};
|
|
3305
|
-
}
|
|
3306
|
-
// With `importHint`, the example also names the command that completes a
|
|
3307
|
-
// model file's bare names.
|
|
3308
|
-
const viaImport = importHint ?
|
|
3309
|
-
`\nOr run 'kcmd import <file> --flavor <ossie|google> ` +
|
|
3310
|
-
`--source-prefix <prefix>' to complete each bare table name ` +
|
|
3311
|
-
`from the database that <prefix> names.` :
|
|
3429
|
+
// With `importVersion`, the example also names the command that completes
|
|
3430
|
+
// a model file's bare names.
|
|
3431
|
+
const viaImport = importVersion ?
|
|
3432
|
+
`\nOr run '${(0, resolve_profiles_1.importCommand)('--source-prefix <prefix>', importVersion)}' to ` +
|
|
3433
|
+
`complete each bare table name from the database or Knowledge ` +
|
|
3434
|
+
`Catalog entry group that <prefix> names.` :
|
|
3312
3435
|
'';
|
|
3313
3436
|
// A name holding a space gets no name to write. Any other name gets an
|
|
3314
3437
|
// example built from its parts when the loader accepts the example, and
|
|
@@ -3334,11 +3457,11 @@ function parseSource(source, opts, ctx, warnings, importHint = false) {
|
|
|
3334
3457
|
}
|
|
3335
3458
|
}
|
|
3336
3459
|
// Each table form with its placeholders shortened, such as <p> for
|
|
3337
|
-
// <project>, and
|
|
3460
|
+
// <project>, and what it is for where its service has two.
|
|
3338
3461
|
const forms = Object.entries(exports.TABLE_FORMS).map(([kind, form]) => {
|
|
3339
3462
|
const short = form.replace(/<(\w+)>/g, (_, name) => `<${name === 'database' ? 'db' : name[0]}>`);
|
|
3340
|
-
const
|
|
3341
|
-
return `'${short}'${
|
|
3463
|
+
const use = FORM_USES[kind];
|
|
3464
|
+
return `'${short}'${use ? ` (${use})` : ''}`;
|
|
3342
3465
|
});
|
|
3343
3466
|
throw new SemanticModelError(`Semantic model load error: ${ctx}: source '${trimmed}' is not a valid ` +
|
|
3344
3467
|
`table reference${dotted ? `; ${dotted}` : ''}. Expected a GCP ` +
|
|
@@ -3346,9 +3469,6 @@ function parseSource(source, opts, ctx, warnings, importHint = false) {
|
|
|
3346
3469
|
`or, for a table outside Google Cloud, the entry name of its ` +
|
|
3347
3470
|
`Knowledge Catalog entry ('${exports.ENTRY_NAME_FORM}').`);
|
|
3348
3471
|
}
|
|
3349
|
-
function unquote(part) {
|
|
3350
|
-
return part.replace(/^[`"]/, '').replace(/[`"]$/, '');
|
|
3351
|
-
}
|
|
3352
3472
|
// The parts of a bare table name, without their quotes. A part in backticks
|
|
3353
3473
|
// or double quotes is one name, dots included, and a doubled quote character
|
|
3354
3474
|
// in it, as in "a""b", is one literal quote. The quotes are those
|
|
@@ -3401,9 +3521,6 @@ function bareNameParts(name, bigQuery) {
|
|
|
3401
3521
|
* thrown, mirroring how the deploy legs previously reported it; loader warnings
|
|
3402
3522
|
* are prefixed with their document name.
|
|
3403
3523
|
*
|
|
3404
|
-
* A document's `allowLegacyBareSource` sets that load option for that document
|
|
3405
|
-
* alone.
|
|
3406
|
-
*
|
|
3407
3524
|
* Each document needs the SQL parser as loadModels does, so call
|
|
3408
3525
|
* `loadSqlEngine()` first. Without it, this throws the parser's plain Error.
|
|
3409
3526
|
*/
|
|
@@ -3412,14 +3529,8 @@ function loadSemanticModels(docs, opts = {}) {
|
|
|
3412
3529
|
const warnings = [];
|
|
3413
3530
|
for (const doc of docs) {
|
|
3414
3531
|
let loaded;
|
|
3415
|
-
const docOpts = {
|
|
3416
|
-
...opts,
|
|
3417
|
-
...(doc.allowLegacyBareSource !== undefined ?
|
|
3418
|
-
{ allowLegacyBareSource: doc.allowLegacyBareSource } :
|
|
3419
|
-
{}),
|
|
3420
|
-
};
|
|
3421
3532
|
try {
|
|
3422
|
-
loaded = loadModels(doc.text,
|
|
3533
|
+
loaded = loadModels(doc.text, opts);
|
|
3423
3534
|
}
|
|
3424
3535
|
catch (err) {
|
|
3425
3536
|
if (!(err instanceof SemanticModelError))
|