@google-cloud/kcmd 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -20
- package/build/ts/kcmd/cli.js +5842 -5168
- package/build/ts/kcmd/gcp/biglake.d.ts +2 -0
- package/build/ts/kcmd/gcp/biglake.js +13 -4
- package/build/ts/kcmd/gcp/spanner.d.ts +10 -0
- package/build/ts/kcmd/gcp/spanner.js +6 -3
- package/build/ts/kcmd/layouts/okf.js +49 -7
- package/build/ts/kcmd/layouts/semantic-model.d.ts +13 -0
- package/build/ts/kcmd/layouts/semantic-model.js +129 -50
- package/build/ts/kcmd/okf/concept.d.ts +1 -1
- package/build/ts/kcmd/okf/concept.js +6 -0
- package/build/ts/kcmd/okf/concept_to_entry.d.ts +2 -0
- package/build/ts/kcmd/okf/concept_to_entry.js +29 -2
- package/build/ts/kcmd/okf/init_okf_workspace.js +8 -10
- package/build/ts/kcmd/okf/okf_metadata.js +17 -0
- package/build/ts/kcmd/semantic/bigquery.js +139 -195
- package/build/ts/kcmd/semantic/binding.js +14 -19
- package/build/ts/kcmd/semantic/deploy_knowledge_catalog.d.ts +10 -35
- package/build/ts/kcmd/semantic/deploy_knowledge_catalog.js +92 -121
- package/build/ts/kcmd/semantic/graph_ddl.d.ts +10 -0
- package/build/ts/kcmd/semantic/graph_ddl.js +152 -59
- package/build/ts/kcmd/semantic/import_model.d.ts +26 -4
- package/build/ts/kcmd/semantic/import_model.js +526 -103
- package/build/ts/kcmd/semantic/kc_converter.d.ts +7 -1
- package/build/ts/kcmd/semantic/kc_converter.js +20 -5
- package/build/ts/kcmd/semantic/kc_custom_types.d.ts +5 -12
- package/build/ts/kcmd/semantic/kc_custom_types.js +16 -93
- package/build/ts/kcmd/semantic/kc_ids.d.ts +4 -5
- package/build/ts/kcmd/semantic/kc_ids.js +4 -7
- package/build/ts/kcmd/semantic/knowledge_catalog.d.ts +0 -3
- package/build/ts/kcmd/semantic/knowledge_catalog.js +3 -86
- package/build/ts/kcmd/semantic/loader.d.ts +7 -11
- package/build/ts/kcmd/semantic/loader.js +284 -148
- package/build/ts/kcmd/semantic/osi_converter.js +1 -2
- package/build/ts/kcmd/semantic/parameters.d.ts +1 -0
- package/build/ts/kcmd/semantic/parameters.js +22 -3
- package/build/ts/kcmd/semantic/pull_kc.d.ts +1 -5
- package/build/ts/kcmd/semantic/pull_kc.js +38 -112
- package/build/ts/kcmd/semantic/resolve_inheritance.d.ts +7 -2
- package/build/ts/kcmd/semantic/resolve_inheritance.js +37 -21
- package/build/ts/kcmd/semantic/resolve_profiles.d.ts +51 -10
- package/build/ts/kcmd/semantic/resolve_profiles.js +213 -32
- package/build/ts/kcmd/semantic/runtime/runtime.d.ts +0 -6
- package/build/ts/kcmd/semantic/runtime/runtime.js +4 -21
- package/build/ts/kcmd/semantic/source_dialect.d.ts +9 -2
- package/build/ts/kcmd/semantic/source_dialect.js +83 -33
- package/build/ts/kcmd/semantic/source_entries.d.ts +1 -1
- package/build/ts/kcmd/semantic/source_entries.js +95 -36
- package/build/ts/kcmd/semantic/sql_expr_utils.d.ts +13 -7
- package/build/ts/kcmd/semantic/sql_expr_utils.js +240 -67
- package/build/ts/kcmd/semantic/sql_parser.d.ts +5 -2
- package/build/ts/kcmd/semantic/sql_parser.js +22 -0
- package/build/ts/kcmd/semantic/transpile.d.ts +12 -10
- package/build/ts/kcmd/semantic/transpile.js +79 -61
- package/build/ts/kcmd/semantic/validate.d.ts +37 -2
- package/build/ts/kcmd/semantic/validate.js +465 -98
- package/build/ts/kcmd/snapshot.d.ts +4 -0
- package/build/ts/kcmd/snapshot.js +7 -0
- package/build/ts/kcmd/sync.js +44 -7
- package/package.json +1 -1
|
@@ -6,6 +6,11 @@ export interface ReadResult {
|
|
|
6
6
|
warnings: string[];
|
|
7
7
|
secondGeneration?: boolean;
|
|
8
8
|
overviews?: Map<string, OverviewSidecar>;
|
|
9
|
+
firstGeneration?: FirstGenerationModel[];
|
|
10
|
+
}
|
|
11
|
+
export interface FirstGenerationModel {
|
|
12
|
+
name: string;
|
|
13
|
+
entries: Entry[];
|
|
9
14
|
}
|
|
10
15
|
/**
|
|
11
16
|
* Reconstructs the Semantic Model IR from Knowledge Catalog entries.
|
|
@@ -22,7 +27,8 @@ export interface ReadResult {
|
|
|
22
27
|
*
|
|
23
28
|
* With `opts.v2Aspects` set, a model whose `semantic-model` aspect has
|
|
24
29
|
* `specVersion` is rebuilt from the second-generation types (see
|
|
25
|
-
* readModelV2), which reads no entry link.
|
|
30
|
+
* readModelV2), which reads no entry link. A model without it is returned in
|
|
31
|
+
* `firstGeneration` instead of being rebuilt.
|
|
26
32
|
*/
|
|
27
33
|
export declare function modelsFromCatalogResources(entries: Entry[], entryLinks?: EntryLink[], opts?: {
|
|
28
34
|
v2Aspects?: boolean;
|
|
@@ -102,7 +102,8 @@ function warnDanglingGuards(actions, constraints, modelName, warnings) {
|
|
|
102
102
|
*
|
|
103
103
|
* With `opts.v2Aspects` set, a model whose `semantic-model` aspect has
|
|
104
104
|
* `specVersion` is rebuilt from the second-generation types (see
|
|
105
|
-
* readModelV2), which reads no entry link.
|
|
105
|
+
* readModelV2), which reads no entry link. A model without it is returned in
|
|
106
|
+
* `firstGeneration` instead of being rebuilt.
|
|
106
107
|
*/
|
|
107
108
|
function modelsFromCatalogResources(entries, entryLinks = [], opts = {}) {
|
|
108
109
|
const warnings = [];
|
|
@@ -128,11 +129,24 @@ function modelsFromCatalogResources(entries, entryLinks = [], opts = {}) {
|
|
|
128
129
|
(soleAnchor === anchorName && !anchorNames.has(e.parentEntry ?? '')));
|
|
129
130
|
let secondGeneration = false;
|
|
130
131
|
const overviews = new Map();
|
|
131
|
-
const
|
|
132
|
-
|
|
132
|
+
const nameOf = (anchor) => anchor.entrySource?.displayName ?? idOf(anchor.name);
|
|
133
|
+
// With the second-generation types, a model whose model entry has no
|
|
134
|
+
// specVersion was pushed with the first-generation types. It is left out
|
|
135
|
+
// rather than rebuilt from them.
|
|
136
|
+
const isFirstGeneration = (anchor) => !!opts.v2Aspects &&
|
|
137
|
+
aspectData(anchor, 'semantic-model').specVersion === undefined;
|
|
138
|
+
const firstGeneration = anchors.filter(isFirstGeneration).map(anchor => ({
|
|
139
|
+
name: nameOf(anchor),
|
|
140
|
+
entries: [
|
|
141
|
+
anchor,
|
|
142
|
+
...[entityEntries, metricEntries, actionEntries, constraintEntries]
|
|
143
|
+
.flatMap(pool => childrenOf(anchor.name, pool)),
|
|
144
|
+
],
|
|
145
|
+
}));
|
|
146
|
+
const models = anchors.filter(a => !isFirstGeneration(a)).map(anchor => {
|
|
147
|
+
const name = nameOf(anchor);
|
|
133
148
|
const children = (pool) => childrenOf(anchor.name, pool);
|
|
134
|
-
if (opts.v2Aspects
|
|
135
|
-
aspectData(anchor, 'semantic-model').specVersion !== undefined) {
|
|
149
|
+
if (opts.v2Aspects) {
|
|
136
150
|
secondGeneration = true;
|
|
137
151
|
const [entities, metrics, relationships] = [entityEntries, metricEntries, relationshipEntries].map(children);
|
|
138
152
|
overviewSidecars([anchor, ...entities, ...metrics, ...relationships])
|
|
@@ -189,6 +203,7 @@ function modelsFromCatalogResources(entries, entryLinks = [], opts = {}) {
|
|
|
189
203
|
models,
|
|
190
204
|
warnings: [...new Set(warnings)],
|
|
191
205
|
...(secondGeneration ? { secondGeneration, overviews } : {}),
|
|
206
|
+
...(firstGeneration.length ? { firstGeneration } : {}),
|
|
192
207
|
};
|
|
193
208
|
}
|
|
194
209
|
// Reads a model's action and constraint entries onto `model`.
|
|
@@ -4,21 +4,15 @@ export interface CustomType {
|
|
|
4
4
|
id: string;
|
|
5
5
|
entryType: Omit<EntryType, 'name' | 'requiredAspects'>;
|
|
6
6
|
aspectType: Omit<AspectType, 'name'>;
|
|
7
|
-
required?: boolean;
|
|
8
7
|
}
|
|
9
8
|
export declare function aiContextField(index: number): Record<string, any>;
|
|
10
9
|
export declare function aiContextAspectValue(ai: AiContext | undefined): Record<string, any> | undefined;
|
|
11
10
|
export declare function aiContextFromAspect(value: any): AiContext | undefined;
|
|
12
11
|
export declare const ACTION_TYPE_ID = "semantic-action";
|
|
13
12
|
export declare const CONSTRAINT_TYPE_ID = "semantic-constraint";
|
|
14
|
-
export declare const
|
|
15
|
-
export declare const
|
|
16
|
-
export
|
|
17
|
-
fileName: string;
|
|
18
|
-
content: string;
|
|
19
|
-
pushId?: string;
|
|
20
|
-
pushFileCount?: number;
|
|
21
|
-
}
|
|
13
|
+
export declare const PUSH_STATE_LABEL = "kcmd-push-state";
|
|
14
|
+
export declare const PUSH_WRITING = "writing";
|
|
15
|
+
export declare const PUSH_FINISHED = "finished";
|
|
22
16
|
export declare const CUSTOM_TYPES: readonly CustomType[];
|
|
23
17
|
export declare function isCustomType(id: string): boolean;
|
|
24
18
|
/**
|
|
@@ -56,9 +50,8 @@ export interface ProvisionResult {
|
|
|
56
50
|
* template lacks fails with an opaque parsing error. Dataplex rejects a
|
|
57
51
|
* backwards-incompatible change, so the patch only ever adds appended fields.
|
|
58
52
|
*
|
|
59
|
-
* Stops at the first hard failure
|
|
60
|
-
*
|
|
61
|
-
* a warning. A refusal for an optional type does NOT stop it: the refusals
|
|
53
|
+
* Stops at the first hard failure. A refusal to patch a type that exists is
|
|
54
|
+
* a warning. A refusal to create a type does NOT stop it: the refusals
|
|
62
55
|
* differ per type and per operation -- patching a type that already exists
|
|
63
56
|
* needs `update`, creating one that does not needs `create` -- so a caller
|
|
64
57
|
* refused one type may well be allowed the next, and giving up early would
|
|
@@ -37,7 +37,7 @@
|
|
|
37
37
|
// removing an existing field breaks the update for every project that already
|
|
38
38
|
// provisioned the type.
|
|
39
39
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
40
|
-
exports.CUSTOM_TYPES = exports.
|
|
40
|
+
exports.CUSTOM_TYPES = exports.PUSH_FINISHED = exports.PUSH_WRITING = exports.PUSH_STATE_LABEL = exports.CONSTRAINT_TYPE_ID = exports.ACTION_TYPE_ID = void 0;
|
|
41
41
|
exports.aiContextField = aiContextField;
|
|
42
42
|
exports.aiContextAspectValue = aiContextAspectValue;
|
|
43
43
|
exports.aiContextFromAspect = aiContextFromAspect;
|
|
@@ -549,67 +549,12 @@ const CONSTRAINT_ASPECT_TYPE = {
|
|
|
549
549
|
],
|
|
550
550
|
},
|
|
551
551
|
};
|
|
552
|
-
// The
|
|
553
|
-
//
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
// push that stopped before storing them. kcmd owns this label and no other.
|
|
559
|
-
exports.PUSH_ID_LABEL = 'kcmd-push-id';
|
|
560
|
-
// The aspect that carries a file copy.
|
|
561
|
-
const FILE_COPY_ASPECT_TYPE = {
|
|
562
|
-
displayName: 'Semantic Model File',
|
|
563
|
-
description: 'The text of one semantic model file or profile file, exactly as ' +
|
|
564
|
-
'pushed.',
|
|
565
|
-
metadataTemplate: {
|
|
566
|
-
name: exports.FILE_COPY_TYPE_ID,
|
|
567
|
-
type: 'record',
|
|
568
|
-
recordFields: [
|
|
569
|
-
{
|
|
570
|
-
index: 1,
|
|
571
|
-
name: 'fileName',
|
|
572
|
-
type: 'string',
|
|
573
|
-
constraints: { required: true },
|
|
574
|
-
annotations: {
|
|
575
|
-
displayName: 'File Name',
|
|
576
|
-
description: 'The file\'s path relative to the entry group directory, for ' +
|
|
577
|
-
'example `retail.profile.prod.yaml`.',
|
|
578
|
-
},
|
|
579
|
-
},
|
|
580
|
-
{
|
|
581
|
-
index: 2,
|
|
582
|
-
name: 'content',
|
|
583
|
-
type: 'string',
|
|
584
|
-
constraints: { required: true },
|
|
585
|
-
annotations: {
|
|
586
|
-
displayName: 'Content',
|
|
587
|
-
description: 'The file\'s text, exactly as read.',
|
|
588
|
-
},
|
|
589
|
-
},
|
|
590
|
-
{
|
|
591
|
-
index: 3,
|
|
592
|
-
name: 'pushId',
|
|
593
|
-
type: 'string',
|
|
594
|
-
annotations: {
|
|
595
|
-
displayName: 'Push ID',
|
|
596
|
-
description: 'The id of the push that stored this copy. Every copy one push ' +
|
|
597
|
-
'stores carries the same id.',
|
|
598
|
-
},
|
|
599
|
-
},
|
|
600
|
-
{
|
|
601
|
-
index: 4,
|
|
602
|
-
name: 'pushFileCount',
|
|
603
|
-
type: 'int',
|
|
604
|
-
annotations: {
|
|
605
|
-
displayName: 'Push File Count',
|
|
606
|
-
description: 'How many file copies the push that stored this one ' +
|
|
607
|
-
'stored in all.',
|
|
608
|
-
},
|
|
609
|
-
},
|
|
610
|
-
],
|
|
611
|
-
},
|
|
612
|
-
};
|
|
552
|
+
// The entrySource label every push puts on the model entry: PUSH_WRITING with
|
|
553
|
+
// the model entry's write and PUSH_FINISHED as the push's last write. kcmd
|
|
554
|
+
// owns this label and no other.
|
|
555
|
+
exports.PUSH_STATE_LABEL = 'kcmd-push-state';
|
|
556
|
+
exports.PUSH_WRITING = 'writing';
|
|
557
|
+
exports.PUSH_FINISHED = 'finished';
|
|
613
558
|
// Every type kcmd provisions. This list is the whole of what is custom.
|
|
614
559
|
exports.CUSTOM_TYPES = [
|
|
615
560
|
{
|
|
@@ -628,16 +573,6 @@ exports.CUSTOM_TYPES = [
|
|
|
628
573
|
},
|
|
629
574
|
aspectType: CONSTRAINT_ASPECT_TYPE,
|
|
630
575
|
},
|
|
631
|
-
{
|
|
632
|
-
id: exports.FILE_COPY_TYPE_ID,
|
|
633
|
-
entryType: {
|
|
634
|
-
displayName: 'Semantic Model File',
|
|
635
|
-
description: 'A copy of one semantic model file or profile file.',
|
|
636
|
-
},
|
|
637
|
-
aspectType: FILE_COPY_ASPECT_TYPE,
|
|
638
|
-
// Push stores every model file with it.
|
|
639
|
-
required: true,
|
|
640
|
-
},
|
|
641
576
|
];
|
|
642
577
|
// Custom types are provisioned at `global` so an entry group in any region can
|
|
643
578
|
// reference them, matching where the built-in system types live.
|
|
@@ -692,9 +627,8 @@ function customAspectKey(id, dest) {
|
|
|
692
627
|
* template lacks fails with an opaque parsing error. Dataplex rejects a
|
|
693
628
|
* backwards-incompatible change, so the patch only ever adds appended fields.
|
|
694
629
|
*
|
|
695
|
-
* Stops at the first hard failure
|
|
696
|
-
*
|
|
697
|
-
* a warning. A refusal for an optional type does NOT stop it: the refusals
|
|
630
|
+
* Stops at the first hard failure. A refusal to patch a type that exists is
|
|
631
|
+
* a warning. A refusal to create a type does NOT stop it: the refusals
|
|
698
632
|
* differ per type and per operation -- patching a type that already exists
|
|
699
633
|
* needs `update`, creating one that does not needs `create` -- so a caller
|
|
700
634
|
* refused one type may well be allowed the next, and giving up early would
|
|
@@ -724,28 +658,19 @@ async function provisionOne(cat, dest, type) {
|
|
|
724
658
|
const home = customTypeHome(dest);
|
|
725
659
|
// Creating a type needs permissions that publishing entries does not. A
|
|
726
660
|
// caller who lacks them can still init, push and pull every model that uses
|
|
727
|
-
// none of the
|
|
728
|
-
// A required type is needed by every push, so its refusal stops init.
|
|
661
|
+
// none of the custom types, so that refusal is reported rather than fatal.
|
|
729
662
|
const denial = (what, status, message) => {
|
|
730
663
|
if (status !== 403)
|
|
731
664
|
return undefined;
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
denied: `${refused}; models that need it cannot be pushed until ` +
|
|
737
|
-
`it exists`
|
|
738
|
-
};
|
|
665
|
+
return {
|
|
666
|
+
denied: `no permission to create ${what} in project '${home.project}' (${message || status}); models that need it cannot be pushed ` +
|
|
667
|
+
`until it exists`
|
|
668
|
+
};
|
|
739
669
|
};
|
|
740
|
-
// A caller without create permission may be refused the create of a type
|
|
741
|
-
// that already exists. For a required type a lookup tells the two apart, so
|
|
742
|
-
// init fails only when the type is missing.
|
|
743
|
-
const refusedButExists = async (status, lookup) => status === 403 && !!type.required && (await lookup()).status === 200;
|
|
744
670
|
const warnings = [];
|
|
745
671
|
const aspectLabel = `aspect type '${type.id}'`;
|
|
746
672
|
const aspect = await cat.createAspectType(home.project, home.location, type.id, type.aspectType);
|
|
747
|
-
if (aspect.status === 409
|
|
748
|
-
await refusedButExists(aspect.status, () => cat.getAspectType(home.project, home.location, type.id))) {
|
|
673
|
+
if (aspect.status === 409) {
|
|
749
674
|
const upd = await cat.updateAspectType(home.project, home.location, type.id, type.aspectType, ['description', 'display_name', 'metadata_template']);
|
|
750
675
|
// Patching an existing type forward is best effort. Dataplex refuses a
|
|
751
676
|
// template change it judges backwards-incompatible -- removing a field, or
|
|
@@ -808,10 +733,8 @@ async function provisionOne(cat, dest, type) {
|
|
|
808
733
|
requiredAspects: [{ type: customAspectTypeName(type.id, dest) }],
|
|
809
734
|
});
|
|
810
735
|
const collected = warnings.length ? { warnings } : {};
|
|
811
|
-
if (entryType.status === 409
|
|
812
|
-
await refusedButExists(entryType.status, () => cat.getEntryType(home.project, home.location, type.id))) {
|
|
736
|
+
if (entryType.status === 409)
|
|
813
737
|
return collected;
|
|
814
|
-
}
|
|
815
738
|
const refused = denial(entryLabel, entryType.status, entryType.message);
|
|
816
739
|
if (refused)
|
|
817
740
|
return { ...refused, ...collected };
|
|
@@ -55,11 +55,6 @@ export declare class Namer {
|
|
|
55
55
|
* branch on `v2Aspects`.
|
|
56
56
|
*/
|
|
57
57
|
linkId(model: SemanticModel, rel: Relationship): string;
|
|
58
|
-
/**
|
|
59
|
-
* Bare entry ID for a verbatim model/profile file copy (used while
|
|
60
|
-
* `KC_V2_ASPECTS` is off). Dotted under both layouts.
|
|
61
|
-
*/
|
|
62
|
-
fileEntryId(modelName: string, fileName: string): string;
|
|
63
58
|
}
|
|
64
59
|
/** Entry-link ID for a relationship on a model. */
|
|
65
60
|
export declare function relationshipLinkId(modelName: string, relName: string): string;
|
|
@@ -78,6 +73,10 @@ export declare function semanticLinkId(kind: 'link' | 'relationship' | 'extends'
|
|
|
78
73
|
* relationship entries. Because V2 model names are restricted to letters,
|
|
79
74
|
* digits, underscores and hyphens, all of which `slug()` keeps,
|
|
80
75
|
* `slug(modelName)` and `modelName` are equal for valid V2 models.
|
|
76
|
+
*
|
|
77
|
+
* No push writes under `.files.` any more: an older kcmd stored a copy of
|
|
78
|
+
* each model and profile file there. The prefix stays so the next push
|
|
79
|
+
* deletes those copies.
|
|
81
80
|
*/
|
|
82
81
|
export declare function ownedEntryIdPrefixes(modelName: string): string[];
|
|
83
82
|
/**
|
|
@@ -109,13 +109,6 @@ class Namer {
|
|
|
109
109
|
linkId(model, rel) {
|
|
110
110
|
return relationshipLinkId(model.name, rel.name);
|
|
111
111
|
}
|
|
112
|
-
/**
|
|
113
|
-
* Bare entry ID for a verbatim model/profile file copy (used while
|
|
114
|
-
* `KC_V2_ASPECTS` is off). Dotted under both layouts.
|
|
115
|
-
*/
|
|
116
|
-
fileEntryId(modelName, fileName) {
|
|
117
|
-
return `${slug(modelName)}.files.${slug(fileName)}`;
|
|
118
|
-
}
|
|
119
112
|
}
|
|
120
113
|
exports.Namer = Namer;
|
|
121
114
|
/** Entry-link ID for a relationship on a model. */
|
|
@@ -140,6 +133,10 @@ function semanticLinkId(kind, ...parts) {
|
|
|
140
133
|
* relationship entries. Because V2 model names are restricted to letters,
|
|
141
134
|
* digits, underscores and hyphens, all of which `slug()` keeps,
|
|
142
135
|
* `slug(modelName)` and `modelName` are equal for valid V2 models.
|
|
136
|
+
*
|
|
137
|
+
* No push writes under `.files.` any more: an older kcmd stored a copy of
|
|
138
|
+
* each model and profile file there. The prefix stays so the next push
|
|
139
|
+
* deletes those copies.
|
|
143
140
|
*/
|
|
144
141
|
function ownedEntryIdPrefixes(modelName) {
|
|
145
142
|
const s = slug(modelName);
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import type { Entry, EntryLink } from '../gcp/dataplex';
|
|
2
2
|
import type { OverviewSidecar } from '../layouts/semantic-model';
|
|
3
3
|
import { SemanticModel } from './ir';
|
|
4
|
-
import { FileCopy } from './kc_custom_types';
|
|
5
4
|
export interface KcGenerateOptions {
|
|
6
5
|
project: string;
|
|
7
6
|
location: string;
|
|
@@ -10,7 +9,6 @@ export interface KcGenerateOptions {
|
|
|
10
9
|
systemTypeLocation?: string;
|
|
11
10
|
emitExpressions?: boolean;
|
|
12
11
|
v2Aspects?: boolean;
|
|
13
|
-
sourceFiles?: FileCopy[];
|
|
14
12
|
overviews?: ReadonlyMap<string, OverviewSidecar>;
|
|
15
13
|
}
|
|
16
14
|
export interface KcResources {
|
|
@@ -29,4 +27,3 @@ export interface KcResources {
|
|
|
29
27
|
* The resources reference the built-in system types; they do not create them.
|
|
30
28
|
*/
|
|
31
29
|
export declare function generateCatalogResources(model: SemanticModel, opts: KcGenerateOptions): KcResources;
|
|
32
|
-
export declare function fileCopyData(file: FileCopy): Record<string, any>;
|
|
@@ -28,14 +28,6 @@
|
|
|
28
28
|
// and `kc_actions.ts` and `kc_constraints.ts` encode the aspects they carry.
|
|
29
29
|
// This module only appends the entries those two return.
|
|
30
30
|
//
|
|
31
|
-
// With the first-generation aspects (v2Aspects off), each model and profile
|
|
32
|
-
// file the push read is also stored whole as a file copy, one
|
|
33
|
-
// `semantic-model-file` entry per file, parented to the anchor. That type is
|
|
34
|
-
// custom too. The aspects cannot hold the whole model, so a pull returns
|
|
35
|
-
// these copies instead of rebuilding the file from them. The anchor of a
|
|
36
|
-
// model with copies carries their push id in the `kcmd-push-id` label on its
|
|
37
|
-
// entrySource.
|
|
38
|
-
//
|
|
39
31
|
// Aspect data shapes mirror the aspect types' CLOSED metadataTemplates exactly
|
|
40
32
|
// (a server aspect type rejects an undeclared data field):
|
|
41
33
|
// * semantic-model = { deploymentTargets: string[] }
|
|
@@ -81,11 +73,9 @@
|
|
|
81
73
|
//
|
|
82
74
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
83
75
|
exports.generateCatalogResources = generateCatalogResources;
|
|
84
|
-
exports.fileCopyData = fileCopyData;
|
|
85
76
|
const deployment_target_1 = require("./deployment_target");
|
|
86
77
|
const kc_actions_1 = require("./kc_actions");
|
|
87
78
|
const kc_constraints_1 = require("./kc_constraints");
|
|
88
|
-
const kc_custom_types_1 = require("./kc_custom_types");
|
|
89
79
|
const kc_ids_1 = require("./kc_ids");
|
|
90
80
|
const loader_1 = require("./loader");
|
|
91
81
|
const osi_converter_1 = require("./osi_converter");
|
|
@@ -112,13 +102,6 @@ function generateCatalogResources(model, opts) {
|
|
|
112
102
|
const v2 = opts.v2Aspects ?? false;
|
|
113
103
|
if (v2)
|
|
114
104
|
warnings.push(...unstorableMemberWarnings(model));
|
|
115
|
-
// The first-generation aspects have no field for a vendor's extension
|
|
116
|
-
// blocks, and the stored copy of the model file keeps them.
|
|
117
|
-
if (!v2 && opts.sourceFiles?.length && hasThirdPartyExtensions(model)) {
|
|
118
|
-
warnings.push('custom_extensions from vendors other than GOOGLE are not indexed ' +
|
|
119
|
-
'by the Knowledge Catalog aspects; the stored copy of the model file ' +
|
|
120
|
-
'preserves them');
|
|
121
|
-
}
|
|
122
105
|
// With the first-generation aspects, a subtype field that only rebinds an
|
|
123
106
|
// inherited field is published with the definition it inherits.
|
|
124
107
|
const inherited = v2 ? undefined : inheritedFields(model);
|
|
@@ -161,7 +144,9 @@ function generateCatalogResources(model, opts) {
|
|
|
161
144
|
continue;
|
|
162
145
|
entityEntryName.set(entity.name, names.entry(entityId));
|
|
163
146
|
if (v2 ? lacksKeys(model, entity) : !entity.keys || !entity.keys.length) {
|
|
164
|
-
warnings.push(`entity '${entity.name}': no keys declared in the source model`
|
|
147
|
+
warnings.push(`entity '${entity.name}': no keys declared in the source model; ` +
|
|
148
|
+
`declare primary_key or unique_keys in the model file or in a ` +
|
|
149
|
+
`profile to record a key in Knowledge Catalog`);
|
|
165
150
|
}
|
|
166
151
|
entries.push({
|
|
167
152
|
name: names.entry(entityId),
|
|
@@ -234,38 +219,6 @@ function generateCatalogResources(model, opts) {
|
|
|
234
219
|
anchor: modelEntryName,
|
|
235
220
|
claim: (id, label) => claim(seen, id, 'entry', label, warnings),
|
|
236
221
|
}, warnings));
|
|
237
|
-
// One file copy entry per model and profile file, holding its text as
|
|
238
|
-
// read. Only with the first-generation aspects: a push with v2Aspects on
|
|
239
|
-
// emits none, so delete reconciliation removes the copies under the owned
|
|
240
|
-
// `.files.` prefix. Two file names that slug to one id fail the push, since
|
|
241
|
-
// storing only one would lose the other on the next pull.
|
|
242
|
-
if (!v2) {
|
|
243
|
-
const fileById = new Map();
|
|
244
|
-
let pushId;
|
|
245
|
-
for (const file of opts.sourceFiles ?? []) {
|
|
246
|
-
const id = names.fileEntryId(model.name, file.fileName);
|
|
247
|
-
const earlier = fileById.get(id);
|
|
248
|
-
if (earlier !== undefined) {
|
|
249
|
-
throw new Error(`files '${earlier}' and '${file.fileName}' both map to entry id ` +
|
|
250
|
-
`'${id}'; rename one of them`);
|
|
251
|
-
}
|
|
252
|
-
fileById.set(id, file.fileName);
|
|
253
|
-
if (!claim(seen, id, 'entry', `file '${file.fileName}'`, warnings))
|
|
254
|
-
continue;
|
|
255
|
-
entries.push(fileCopyEntry(names.entry(id), modelEntryName, file, opts));
|
|
256
|
-
pushId ??= file.pushId;
|
|
257
|
-
}
|
|
258
|
-
// The anchor is written before the copies and is labeled with the push id
|
|
259
|
-
// they carry. A pull that finds the label and no copies from that push
|
|
260
|
-
// knows the last push stopped before storing them. A model an older kcmd
|
|
261
|
-
// pushed has no label.
|
|
262
|
-
if (pushId !== undefined) {
|
|
263
|
-
entries[0].entrySource = {
|
|
264
|
-
...entries[0].entrySource,
|
|
265
|
-
labels: { [kc_custom_types_1.PUSH_ID_LABEL]: pushId },
|
|
266
|
-
};
|
|
267
|
-
}
|
|
268
|
-
}
|
|
269
222
|
const entryLinks = v2Relationships ?
|
|
270
223
|
[
|
|
271
224
|
...v2Relationships.links,
|
|
@@ -497,42 +450,6 @@ function schemaJoinAspectData(model, rel) {
|
|
|
497
450
|
function entityByName(model, name) {
|
|
498
451
|
return (model.entities ?? []).find(e => e.name === name);
|
|
499
452
|
}
|
|
500
|
-
// The entry holding one file copy. Its entry and aspect types are custom, so
|
|
501
|
-
// they live in the destination project.
|
|
502
|
-
function fileCopyEntry(name, anchor, file, dest) {
|
|
503
|
-
return {
|
|
504
|
-
name,
|
|
505
|
-
entryType: (0, kc_custom_types_1.customEntryTypeName)(kc_custom_types_1.FILE_COPY_TYPE_ID, dest),
|
|
506
|
-
parentEntry: anchor,
|
|
507
|
-
aspects: {
|
|
508
|
-
[(0, kc_custom_types_1.customAspectKey)(kc_custom_types_1.FILE_COPY_TYPE_ID, dest)]: {
|
|
509
|
-
aspectType: (0, kc_custom_types_1.customAspectTypeName)(kc_custom_types_1.FILE_COPY_TYPE_ID, dest),
|
|
510
|
-
data: fileCopyData(file),
|
|
511
|
-
},
|
|
512
|
-
},
|
|
513
|
-
};
|
|
514
|
-
}
|
|
515
|
-
// The aspect data of a file copy: the file's name and text, and the push id
|
|
516
|
-
// and copy count when the push set them.
|
|
517
|
-
function fileCopyData(file) {
|
|
518
|
-
return compact({
|
|
519
|
-
fileName: file.fileName,
|
|
520
|
-
content: file.content,
|
|
521
|
-
pushId: file.pushId,
|
|
522
|
-
pushFileCount: file.pushFileCount,
|
|
523
|
-
});
|
|
524
|
-
}
|
|
525
|
-
// True when the model, or anything in it, carries a `custom_extensions` block
|
|
526
|
-
// from a vendor other than GOOGLE.
|
|
527
|
-
function hasThirdPartyExtensions(model) {
|
|
528
|
-
const thirdParty = (o) => (o.customExtensions ?? []).some(x => x.vendorName !== 'GOOGLE');
|
|
529
|
-
const entities = model.entities ?? [];
|
|
530
|
-
return thirdParty(model) || entities.some(thirdParty) ||
|
|
531
|
-
entities.some(e => (e.fields ?? []).some(thirdParty)) ||
|
|
532
|
-
(model.relationships ?? []).some(thirdParty) ||
|
|
533
|
-
(model.metrics ?? []).some(thirdParty) ||
|
|
534
|
-
(model.actions ?? []).some(thirdParty);
|
|
535
|
-
}
|
|
536
453
|
// One warning for each custom `ai_context` member that `guidelines` leaves
|
|
537
454
|
// out, on the model, an entity, a field, a metric or a relationship.
|
|
538
455
|
function unstorableMemberWarnings(model) {
|
|
@@ -3,10 +3,7 @@ import { DialectExpression, Entity, Executor, Field, Metric, ProfileSpec, Semant
|
|
|
3
3
|
import { DeclaredConcept } from './resolve_inheritance';
|
|
4
4
|
export interface LoadOptions {
|
|
5
5
|
dialect?: string;
|
|
6
|
-
defaultProject?: string;
|
|
7
|
-
defaultDataset?: string;
|
|
8
6
|
bindingOptional?: boolean;
|
|
9
|
-
allowLegacyBareSource?: boolean;
|
|
10
7
|
sourceDialects?: ReadonlyMap<string, string>;
|
|
11
8
|
warnUnacceptedSource?: boolean;
|
|
12
9
|
fieldTextChecked?: boolean;
|
|
@@ -63,7 +60,8 @@ declare const expressionSchema: z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
|
63
60
|
}, z.core.$strict>>;
|
|
64
61
|
}, z.core.$strict>]>;
|
|
65
62
|
export declare function noSqlReason(text: string, dialect: string, entity?: string): string | undefined;
|
|
66
|
-
export declare function listed(problems: string[]): string | undefined;
|
|
63
|
+
export declare function listed(problems: string[], conjunction?: string): string | undefined;
|
|
64
|
+
export declare function plural(n: number, one: string, many: string): string;
|
|
67
65
|
export declare function expressionDocOf(f: Pick<Field, 'expression' | 'dialects' | 'stringForm' | 'importedExpression' | 'importedDialect'>): string | {
|
|
68
66
|
dialects: DialectExpression[];
|
|
69
67
|
} | undefined;
|
|
@@ -143,7 +141,7 @@ export declare function loadModels(text: string, opts?: LoadOptions): LoadResult
|
|
|
143
141
|
* `warnings` rather than thrown.
|
|
144
142
|
*/
|
|
145
143
|
export declare function fromDocument(doc: unknown, opts?: LoadOptions): LoadResult;
|
|
146
|
-
export declare function inferMetricEntities(metric: Pick<Metric, 'name' | 'authoredEntity' | 'expression' | 'importedExpression' | 'importedDialect' | 'dialects'>, entityNames: string[], preferred?: string, impliedDialect?: string, version?: FormatVersion): {
|
|
144
|
+
export declare function inferMetricEntities(metric: Pick<Metric, 'name' | 'authoredEntity' | 'expression' | 'importedExpression' | 'importedDialect' | 'dialects'>, entityNames: string[], preferred?: string, impliedDialect?: string | null, version?: FormatVersion): {
|
|
147
145
|
referenced: string[];
|
|
148
146
|
warning?: string;
|
|
149
147
|
};
|
|
@@ -176,6 +174,7 @@ export type SourceTable = {
|
|
|
176
174
|
} | {
|
|
177
175
|
system: 'biglake';
|
|
178
176
|
catalog: string;
|
|
177
|
+
hive: boolean;
|
|
179
178
|
} | {
|
|
180
179
|
system: 'spanner';
|
|
181
180
|
instance: string;
|
|
@@ -196,6 +195,7 @@ export declare const TABLE_FORMS: {
|
|
|
196
195
|
cloudsql: string;
|
|
197
196
|
cloudsqlSchema: string;
|
|
198
197
|
biglake: string;
|
|
198
|
+
biglakeHive: string;
|
|
199
199
|
};
|
|
200
200
|
export declare function fillForm(form: string, parts: string[]): string | undefined;
|
|
201
201
|
export declare function dottedPartProblem(parts: string[], form: string, service?: boolean): string | undefined;
|
|
@@ -222,9 +222,9 @@ export declare function bindsThirdPartyTable(entity: {
|
|
|
222
222
|
authoredSource?: string;
|
|
223
223
|
}): boolean;
|
|
224
224
|
export declare function dialectLookupKey(source: string): string | undefined;
|
|
225
|
+
export declare function spannerDatabaseOf(source: string): string | undefined;
|
|
225
226
|
export declare function isGoogleCloudForm(source: string): boolean;
|
|
226
|
-
export declare function parseSource(source: string, opts: LoadOptions, ctx: string, warnings?: string[],
|
|
227
|
-
export declare function unquote(part: string): string;
|
|
227
|
+
export declare function parseSource(source: string, opts: LoadOptions, ctx: string, warnings?: string[], importVersion?: FormatVersion): ParsedSource;
|
|
228
228
|
export declare function bareNameSegments(name: string, bigQuery: boolean): string[];
|
|
229
229
|
export declare function bareNameParts(name: string, bigQuery: boolean): Array<{
|
|
230
230
|
text: string;
|
|
@@ -248,15 +248,11 @@ export interface LoadedModels {
|
|
|
248
248
|
* thrown, mirroring how the deploy legs previously reported it; loader warnings
|
|
249
249
|
* are prefixed with their document name.
|
|
250
250
|
*
|
|
251
|
-
* A document's `allowLegacyBareSource` sets that load option for that document
|
|
252
|
-
* alone.
|
|
253
|
-
*
|
|
254
251
|
* Each document needs the SQL parser as loadModels does, so call
|
|
255
252
|
* `loadSqlEngine()` first. Without it, this throws the parser's plain Error.
|
|
256
253
|
*/
|
|
257
254
|
export declare function loadSemanticModels(docs: {
|
|
258
255
|
name: string;
|
|
259
256
|
text: string;
|
|
260
|
-
allowLegacyBareSource?: boolean;
|
|
261
257
|
}[], opts?: LoadOptions): LoadedModels;
|
|
262
258
|
export {};
|