@google-cloud/kcmd 0.1.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -20
- package/build/ts/kcmd/cli.js +6164 -5631
- package/build/ts/kcmd/gcp/biglake.d.ts +2 -0
- package/build/ts/kcmd/gcp/biglake.js +13 -4
- package/build/ts/kcmd/gcp/spanner.d.ts +10 -0
- package/build/ts/kcmd/gcp/spanner.js +6 -3
- package/build/ts/kcmd/layouts/okf.js +49 -7
- package/build/ts/kcmd/layouts/semantic-model.d.ts +13 -0
- package/build/ts/kcmd/layouts/semantic-model.js +129 -50
- package/build/ts/kcmd/okf/concept.d.ts +1 -1
- package/build/ts/kcmd/okf/concept.js +6 -0
- package/build/ts/kcmd/okf/concept_to_entry.d.ts +2 -0
- package/build/ts/kcmd/okf/concept_to_entry.js +29 -2
- package/build/ts/kcmd/okf/init_okf_workspace.js +7 -9
- package/build/ts/kcmd/okf/okf_metadata.js +17 -0
- package/build/ts/kcmd/semantic/bigquery.js +139 -195
- package/build/ts/kcmd/semantic/binding.js +14 -19
- package/build/ts/kcmd/semantic/deploy_bigquery.js +6 -13
- package/build/ts/kcmd/semantic/deploy_knowledge_catalog.d.ts +10 -35
- package/build/ts/kcmd/semantic/deploy_knowledge_catalog.js +105 -162
- package/build/ts/kcmd/semantic/deploy_spanner.js +7 -14
- package/build/ts/kcmd/semantic/deployment_target.d.ts +1 -4
- package/build/ts/kcmd/semantic/deployment_target.js +6 -62
- package/build/ts/kcmd/semantic/graph_ddl.d.ts +10 -0
- package/build/ts/kcmd/semantic/graph_ddl.js +152 -59
- package/build/ts/kcmd/semantic/import_model.d.ts +26 -4
- package/build/ts/kcmd/semantic/import_model.js +531 -111
- package/build/ts/kcmd/semantic/ir.d.ts +2 -4
- package/build/ts/kcmd/semantic/kc_converter.d.ts +7 -1
- package/build/ts/kcmd/semantic/kc_converter.js +32 -27
- package/build/ts/kcmd/semantic/kc_custom_types.d.ts +5 -12
- package/build/ts/kcmd/semantic/kc_custom_types.js +43 -114
- package/build/ts/kcmd/semantic/kc_ids.d.ts +4 -5
- package/build/ts/kcmd/semantic/kc_ids.js +4 -7
- package/build/ts/kcmd/semantic/knowledge_catalog.d.ts +0 -3
- package/build/ts/kcmd/semantic/knowledge_catalog.js +3 -86
- package/build/ts/kcmd/semantic/loader.d.ts +7 -11
- package/build/ts/kcmd/semantic/loader.js +282 -171
- package/build/ts/kcmd/semantic/osi_converter.js +14 -65
- package/build/ts/kcmd/semantic/parameters.d.ts +1 -0
- package/build/ts/kcmd/semantic/parameters.js +22 -3
- package/build/ts/kcmd/semantic/pull_kc.d.ts +1 -5
- package/build/ts/kcmd/semantic/pull_kc.js +38 -112
- package/build/ts/kcmd/semantic/resolve_inheritance.d.ts +7 -2
- package/build/ts/kcmd/semantic/resolve_inheritance.js +37 -21
- package/build/ts/kcmd/semantic/resolve_profiles.d.ts +51 -10
- package/build/ts/kcmd/semantic/resolve_profiles.js +213 -32
- package/build/ts/kcmd/semantic/runtime/runtime.d.ts +4 -12
- package/build/ts/kcmd/semantic/runtime/runtime.js +4 -21
- package/build/ts/kcmd/semantic/runtime/store.d.ts +14 -19
- package/build/ts/kcmd/semantic/runtime/store.js +21 -37
- package/build/ts/kcmd/semantic/source_dialect.d.ts +9 -2
- package/build/ts/kcmd/semantic/source_dialect.js +83 -33
- package/build/ts/kcmd/semantic/source_entries.d.ts +1 -1
- package/build/ts/kcmd/semantic/source_entries.js +95 -36
- package/build/ts/kcmd/semantic/sql_expr_utils.d.ts +13 -7
- package/build/ts/kcmd/semantic/sql_expr_utils.js +240 -67
- package/build/ts/kcmd/semantic/sql_parser.d.ts +5 -2
- package/build/ts/kcmd/semantic/sql_parser.js +22 -0
- package/build/ts/kcmd/semantic/transpile.d.ts +12 -10
- package/build/ts/kcmd/semantic/transpile.js +79 -61
- package/build/ts/kcmd/semantic/validate.d.ts +37 -2
- package/build/ts/kcmd/semantic/validate.js +467 -126
- package/build/ts/kcmd/snapshot.d.ts +4 -0
- package/build/ts/kcmd/snapshot.js +7 -0
- package/build/ts/kcmd/sync.js +44 -7
- package/package.json +1 -1
|
@@ -83,15 +83,13 @@ exports.modelDocument = modelDocument;
|
|
|
83
83
|
exports.compact = compact;
|
|
84
84
|
exports.nonEmpty = nonEmpty;
|
|
85
85
|
const yaml = __importStar(require("yaml"));
|
|
86
|
-
const deployment_target_1 = require("./deployment_target");
|
|
87
86
|
const ir_1 = require("./ir");
|
|
88
87
|
const loader_1 = require("./loader");
|
|
89
88
|
const names_1 = require("./names");
|
|
90
89
|
const resolve_inheritance_1 = require("./resolve_inheritance");
|
|
91
90
|
const resolve_profiles_1 = require("./resolve_profiles");
|
|
92
91
|
// The vendor tag for Google-specific extension blocks on the IR (kept in sync
|
|
93
|
-
// with loader.GOOGLE_VENDOR).
|
|
94
|
-
// target list rides in one as a folded shim alongside `model.deployments`.
|
|
92
|
+
// with loader.GOOGLE_VENDOR).
|
|
95
93
|
const GOOGLE_VENDOR = 'GOOGLE';
|
|
96
94
|
// The dialect label for the IR's target/canonical `expression` when `dialects`
|
|
97
95
|
// is absent and `stringForm` is not set. BIGQUERY is the loader's default
|
|
@@ -340,17 +338,16 @@ function modelDoc(model, warnings, logical, version) {
|
|
|
340
338
|
`at least one and will not load until an entity is present.`);
|
|
341
339
|
}
|
|
342
340
|
const tableKey = isGoogleVersion ? 'entities' : 'datasets';
|
|
343
|
-
// Under GOOGLE_VERSION, deployments come from `model.deployments
|
|
344
|
-
//
|
|
345
|
-
//
|
|
341
|
+
// Under GOOGLE_VERSION, deployments come from `model.deployments`. Under
|
|
342
|
+
// OSSIE_VERSION, named deployments and profiles go in a GOOGLE block on the
|
|
343
|
+
// model.
|
|
346
344
|
const namedDeployments = (model.deployments ?? []).map(d => compact({
|
|
347
345
|
name: d.name,
|
|
348
346
|
target: d.target,
|
|
349
347
|
profile: d.profile
|
|
350
348
|
}));
|
|
351
349
|
const deployments = placement.model.deployments === 'native' ?
|
|
352
|
-
nonEmpty(namedDeployments
|
|
353
|
-
deploymentsFromGoogleExtensions(model)) :
|
|
350
|
+
nonEmpty(namedDeployments) :
|
|
354
351
|
undefined;
|
|
355
352
|
const blockProfiles = placement.model.profiles === 'block' ?
|
|
356
353
|
nonEmpty(profileDocuments(model, warnings).map(p => p.doc)) :
|
|
@@ -385,8 +382,7 @@ function modelDoc(model, warnings, logical, version) {
|
|
|
385
382
|
constraints: placement.model.constraints === 'native' ?
|
|
386
383
|
nonEmpty((model.constraints ?? []).map(c => constraintDoc(c))) :
|
|
387
384
|
undefined,
|
|
388
|
-
custom_extensions: customExtensionsDoc(model.customExtensions, `model '${model.name}'`, isGoogleVersion,
|
|
389
|
-
/* isModel= */ true, warnings, /* isField= */ false, blockPayload),
|
|
385
|
+
custom_extensions: customExtensionsDoc(model.customExtensions, `model '${model.name}'`, isGoogleVersion, warnings, /* isField= */ false, blockPayload),
|
|
390
386
|
});
|
|
391
387
|
}
|
|
392
388
|
// Warns about each IR construct OSSIE_VERSION has no representation for,
|
|
@@ -436,8 +432,7 @@ function datasetDoc(entity, warnings, logical, version) {
|
|
|
436
432
|
name: entity.name,
|
|
437
433
|
// An abstract entity has no physical table, so its source is empty; omit
|
|
438
434
|
// the key rather than emit `source: ""` (which the loader rejects). The
|
|
439
|
-
// source as written survives serialization
|
|
440
|
-
// as the name the load qualified it to.
|
|
435
|
+
// source as written survives serialization.
|
|
441
436
|
source: (0, loader_1.writtenSource)(entity) || undefined,
|
|
442
437
|
// Supertype entities (entity-level inheritance); native under
|
|
443
438
|
// GOOGLE_VERSION, or in the dataset's GOOGLE block under OSSIE_VERSION.
|
|
@@ -450,8 +445,7 @@ function datasetDoc(entity, warnings, logical, version) {
|
|
|
450
445
|
ai_context: aiContextDoc(entity.aiContext, isGoogleVersion),
|
|
451
446
|
fields: nonEmpty((entity.fields ?? [])
|
|
452
447
|
.map(f => fieldDoc(f, warnings, logical || (isGoogleVersion && !!entity.abstract), isGoogleVersion))),
|
|
453
|
-
custom_extensions: customExtensionsDoc(entity.customExtensions, `entity '${entity.name}'`, isGoogleVersion,
|
|
454
|
-
/* isModel= */ false, warnings, /* isField= */ false, blockPayload),
|
|
448
|
+
custom_extensions: customExtensionsDoc(entity.customExtensions, `entity '${entity.name}'`, isGoogleVersion, warnings, /* isField= */ false, blockPayload),
|
|
455
449
|
});
|
|
456
450
|
}
|
|
457
451
|
// Filters out the IR's `'Opaque'` placeholder so untyped fields and metrics
|
|
@@ -483,8 +477,7 @@ function fieldDoc(field, warnings, logical, isGoogleVersion) {
|
|
|
483
477
|
dimension: dimensionDoc(field),
|
|
484
478
|
description: field.description,
|
|
485
479
|
ai_context: aiContextDoc(field.aiContext, isGoogleVersion),
|
|
486
|
-
custom_extensions: customExtensionsDoc(field.customExtensions, `field '${field.name}'`, isGoogleVersion,
|
|
487
|
-
/* isModel= */ false, warnings, /* isField= */ true),
|
|
480
|
+
custom_extensions: customExtensionsDoc(field.customExtensions, `field '${field.name}'`, isGoogleVersion, warnings, /* isField= */ true),
|
|
488
481
|
});
|
|
489
482
|
}
|
|
490
483
|
function metricDoc(metric, warnings, version) {
|
|
@@ -509,8 +502,7 @@ function metricDoc(metric, warnings, version) {
|
|
|
509
502
|
datatype: datatypeDoc(metric.type),
|
|
510
503
|
description: metric.description,
|
|
511
504
|
ai_context: aiContextDoc(metric.aiContext, isGoogleVersion),
|
|
512
|
-
custom_extensions: customExtensionsDoc(metric.customExtensions, `metric '${metric.name}'`, isGoogleVersion,
|
|
513
|
-
/* isModel= */ false, warnings, /* isField= */ false, blockPayload),
|
|
505
|
+
custom_extensions: customExtensionsDoc(metric.customExtensions, `metric '${metric.name}'`, isGoogleVersion, warnings, /* isField= */ false, blockPayload),
|
|
514
506
|
});
|
|
515
507
|
}
|
|
516
508
|
// The `Concept.field` pairs this document will still resolve on reload. A
|
|
@@ -698,8 +690,7 @@ function relationshipDoc(rel, warnings, version) {
|
|
|
698
690
|
description: placement.relationship.description === 'native' ? description :
|
|
699
691
|
undefined,
|
|
700
692
|
ai_context: aiContextDoc(rel.aiContext, isGoogleVersion),
|
|
701
|
-
custom_extensions: customExtensionsDoc(rel.customExtensions, `relationship '${rel.name}'`, isGoogleVersion,
|
|
702
|
-
/* isModel= */ false, warnings, /* isField= */ false, blockPayload),
|
|
693
|
+
custom_extensions: customExtensionsDoc(rel.customExtensions, `relationship '${rel.name}'`, isGoogleVersion, warnings, /* isField= */ false, blockPayload),
|
|
703
694
|
});
|
|
704
695
|
}
|
|
705
696
|
// Renders a field or metric `expression` for the target version. The IR comes
|
|
@@ -797,24 +788,14 @@ function aiContextDoc(ai, isGoogleVersion) {
|
|
|
797
788
|
}
|
|
798
789
|
return isGoogleVersion ? { ...base, custom } : { ...custom, ...base };
|
|
799
790
|
}
|
|
800
|
-
// The deployments a vanilla model lists only as targets in its GOOGLE blocks,
|
|
801
|
-
// named as a pull names them. A block whose data is not valid JSON gives none,
|
|
802
|
-
// and customExtensionsDoc warns that it was dropped. The other blocks still
|
|
803
|
-
// give theirs.
|
|
804
|
-
function deploymentsFromGoogleExtensions(model) {
|
|
805
|
-
const targets = (model.customExtensions ??
|
|
806
|
-
[]).flatMap(ext => blockTargets(ext, model.name));
|
|
807
|
-
return (0, deployment_target_1.unnamedDeployments)(targets);
|
|
808
|
-
}
|
|
809
791
|
// Serializes `custom_extensions` for one IR element. Under OSSIE_VERSION,
|
|
810
792
|
// third-party extensions are emitted, followed by a synthesized GOOGLE block
|
|
811
|
-
// when `blockPayload` carries any keys. Any
|
|
793
|
+
// when `blockPayload` carries any keys. Any raw GOOGLE block on the IR
|
|
812
794
|
// is dropped with a warning so it cannot collide with the synthesized block or
|
|
813
795
|
// write keys the loader rejects. Under GOOGLE_VERSION,
|
|
814
796
|
// third-party extensions are written out unchanged without warning, while
|
|
815
|
-
// `GOOGLE` blocks are skipped
|
|
816
|
-
|
|
817
|
-
function customExtensionsDoc(exts, where, isGoogleVersion, isModel, warnings, isField = false, blockPayload) {
|
|
797
|
+
// `GOOGLE` blocks are skipped with a warning.
|
|
798
|
+
function customExtensionsDoc(exts, where, isGoogleVersion, warnings, isField = false, blockPayload) {
|
|
818
799
|
const payload = blockPayload ? { ...blockPayload } : {};
|
|
819
800
|
if (!exts?.length && Object.keys(payload).length === 0)
|
|
820
801
|
return undefined;
|
|
@@ -826,12 +807,6 @@ function customExtensionsDoc(exts, where, isGoogleVersion, isModel, warnings, is
|
|
|
826
807
|
kept.push({ vendor_name: e.vendorName, data: e.data });
|
|
827
808
|
continue;
|
|
828
809
|
}
|
|
829
|
-
// The loader's copy of the model's `deployments`, which the GOOGLE
|
|
830
|
-
// block below writes in their own form.
|
|
831
|
-
if (isModel && payload.deployments !== undefined &&
|
|
832
|
-
isDeploymentTargetsBlock(e)) {
|
|
833
|
-
continue;
|
|
834
|
-
}
|
|
835
810
|
droppedGoogle += 1;
|
|
836
811
|
}
|
|
837
812
|
if (droppedGoogle > 0) {
|
|
@@ -860,8 +835,6 @@ function customExtensionsDoc(exts, where, isGoogleVersion, isModel, warnings, is
|
|
|
860
835
|
kept.push({ vendor_name: ext.vendorName, data: ext.data });
|
|
861
836
|
continue;
|
|
862
837
|
}
|
|
863
|
-
if (isModel && blockTargets(ext, '').length > 0)
|
|
864
|
-
continue;
|
|
865
838
|
droppedGoogle += 1;
|
|
866
839
|
}
|
|
867
840
|
if (droppedGoogle > 0) {
|
|
@@ -870,30 +843,6 @@ function customExtensionsDoc(exts, where, isGoogleVersion, isModel, warnings, is
|
|
|
870
843
|
}
|
|
871
844
|
return nonEmpty(kept);
|
|
872
845
|
}
|
|
873
|
-
/**
|
|
874
|
-
* Returns true if `e.data` is a JSON object with a `'deploymentTargets'`
|
|
875
|
-
* property.
|
|
876
|
-
*
|
|
877
|
-
* The loader folds a Google-flavor model's `deployments` into such a block.
|
|
878
|
-
*/
|
|
879
|
-
function isDeploymentTargetsBlock(e) {
|
|
880
|
-
try {
|
|
881
|
-
const d = JSON.parse(e.data);
|
|
882
|
-
return !!d && typeof d === 'object' &&
|
|
883
|
-
Object.hasOwn(d, 'deploymentTargets');
|
|
884
|
-
}
|
|
885
|
-
catch {
|
|
886
|
-
return false;
|
|
887
|
-
}
|
|
888
|
-
}
|
|
889
|
-
function blockTargets(ext, modelName) {
|
|
890
|
-
try {
|
|
891
|
-
return (0, deployment_target_1.googleBlockTargets)([ext], modelName);
|
|
892
|
-
}
|
|
893
|
-
catch {
|
|
894
|
-
return [];
|
|
895
|
-
}
|
|
896
|
-
}
|
|
897
846
|
// Drops undefined-valued keys so the emitted YAML only shows fields the model
|
|
898
847
|
// actually set (matching the emitters' `compact` convention).
|
|
899
848
|
function compact(obj) {
|
|
@@ -2,6 +2,7 @@ import { ActionParameter } from './ir';
|
|
|
2
2
|
export declare function isParameterRequired(param: ActionParameter): boolean;
|
|
3
3
|
export declare function storeCodeFor(dataType: string): string;
|
|
4
4
|
export declare function sentence(text: string): string;
|
|
5
|
+
export declare function storeMessage(message: string | undefined, status: number): string;
|
|
5
6
|
/**
|
|
6
7
|
* One value, parsed to the store type its declared ontology type implies.
|
|
7
8
|
*
|
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
// Leaf helpers for action parameter optionality, scalar type checking,
|
|
3
|
-
//
|
|
4
|
-
// without pulling store clients into the validation layer.
|
|
2
|
+
// Leaf helpers for action parameter optionality, scalar type checking, store
|
|
3
|
+
// type codes and message sentences -- shared by deploy-time validation and
|
|
4
|
+
// run-time execution without pulling store clients into the validation layer.
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.isParameterRequired = isParameterRequired;
|
|
7
7
|
exports.storeCodeFor = storeCodeFor;
|
|
8
8
|
exports.sentence = sentence;
|
|
9
|
+
exports.storeMessage = storeMessage;
|
|
9
10
|
exports.bindScalar = bindScalar;
|
|
10
11
|
function isParameterRequired(param) {
|
|
11
12
|
if (param.required !== undefined)
|
|
@@ -36,6 +37,24 @@ function sentence(text) {
|
|
|
36
37
|
const trimmed = text.trim();
|
|
37
38
|
return /[.!?]$/.test(trimmed) ? trimmed : `${trimmed}.`;
|
|
38
39
|
}
|
|
40
|
+
// The first line of a multi-line diagnosis. The lines after it echo the
|
|
41
|
+
// statement and point a caret at the offending token, which is worth nothing
|
|
42
|
+
// here: the author has the statement open in the profile, the position in
|
|
43
|
+
// `[at 1:54]` survives, and a caret counted from column 1 lands in the wrong
|
|
44
|
+
// place once the message is embedded in a longer sentence.
|
|
45
|
+
//
|
|
46
|
+
// `\n` is matched in both its forms because Spanner's top-level `error.message`
|
|
47
|
+
// arrives with its newlines ESCAPED -- a literal backslash and an n survive
|
|
48
|
+
// JSON.parse -- while every other message uses real ones.
|
|
49
|
+
function firstLine(message) {
|
|
50
|
+
return message.replace(/\\n/g, '\n').split('\n')[0].trim();
|
|
51
|
+
}
|
|
52
|
+
// An API error's message reduced to its first line as a sentence, such as
|
|
53
|
+
// "Unrecognized name: accountId; Did you mean account_id? [at 1:54]."
|
|
54
|
+
function storeMessage(message, status) {
|
|
55
|
+
const text = (message ?? '').trim();
|
|
56
|
+
return text ? sentence(firstLine(text)) : `HTTP ${status}.`;
|
|
57
|
+
}
|
|
39
58
|
// What Spanner accepts for a DATE and a TIMESTAMP parameter. The zone is
|
|
40
59
|
// required rather than defaulted, because a timestamp written without one
|
|
41
60
|
// means a different instant to every reader who supplies the missing part.
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { CatalogClient } from '../gcp/dataplex';
|
|
2
2
|
import { ReadResult } from './kc_converter';
|
|
3
|
-
import { FileCopy } from './kc_custom_types';
|
|
4
3
|
export interface KcPullOptions {
|
|
5
4
|
project: string;
|
|
6
5
|
location: string;
|
|
@@ -8,7 +7,4 @@ export interface KcPullOptions {
|
|
|
8
7
|
v2Aspects?: boolean;
|
|
9
8
|
systemTypeProject?: string;
|
|
10
9
|
}
|
|
11
|
-
export
|
|
12
|
-
rawFiles?: FileCopy[];
|
|
13
|
-
}
|
|
14
|
-
export declare function pullKnowledgeCatalog(cat: CatalogClient, opts: KcPullOptions): Promise<KcPullResult>;
|
|
10
|
+
export declare function pullKnowledgeCatalog(cat: CatalogClient, opts: KcPullOptions): Promise<ReadResult>;
|
|
@@ -8,9 +8,9 @@
|
|
|
8
8
|
// (a BASIC list omits aspect data, so each entry is re-fetched with its aspect
|
|
9
9
|
// types -- an entity needs BOTH its `semantic-entity` aspect and the built-in
|
|
10
10
|
// `schema` aspect), and hands the hydrated entries to the pure reader
|
|
11
|
-
// (`kc_converter.modelsFromCatalogResources`). With the
|
|
12
|
-
//
|
|
13
|
-
//
|
|
11
|
+
// (`kc_converter.modelsFromCatalogResources`). With the second-generation
|
|
12
|
+
// types, a model an older kcmd pushed with the first-generation ones is left
|
|
13
|
+
// out, and a warning names it.
|
|
14
14
|
//
|
|
15
15
|
// TODO(#278): Layer 2 push/pull symmetry. This is the pull half over the
|
|
16
16
|
// pure `kc_converter` codec; the push half (`deployKnowledgeCatalog`) still
|
|
@@ -32,21 +32,17 @@ const HYDRATE_CONCURRENCY = 8;
|
|
|
32
32
|
// Reads the semantic model back from a Knowledge Catalog entry group. Emits no
|
|
33
33
|
// console output; soft warnings (from the reader) are returned for the caller
|
|
34
34
|
// to print. Hard failures -- more than one model in the group, a fetch error
|
|
35
|
-
// on any entry or its links, or
|
|
35
|
+
// on any entry or its links, or a model entry still labeled as being written
|
|
36
36
|
// -- throw, so the pull aborts rather than write a partial model.
|
|
37
37
|
async function pullKnowledgeCatalog(cat, opts) {
|
|
38
38
|
await (0, sql_parser_1.loadSqlEngine)();
|
|
39
39
|
const destination = `${opts.project}.${opts.location}.${opts.entryGroup}`;
|
|
40
|
-
const warnings = [];
|
|
41
40
|
// Enumerate the group (paging is inherently sequential) and pick the semantic
|
|
42
41
|
// entries, then hydrate their aspects concurrently: a BASIC list omits aspect
|
|
43
42
|
// data, so each entry needs its own lookupEntry, and those fetches are
|
|
44
43
|
// independent. The pool preserves input order so warnings stay deterministic.
|
|
45
44
|
const targets = [];
|
|
46
45
|
for await (const entry of cat.listEntries(opts.project, opts.location, opts.entryGroup)) {
|
|
47
|
-
// With v2Aspects on, a pull returns no file copies, so it fetches none.
|
|
48
|
-
if (opts.v2Aspects && isFileCopyEntry(entry))
|
|
49
|
-
continue;
|
|
50
46
|
const aspectTypes = semanticAspectTypes(entry.entryType, opts.v2Aspects);
|
|
51
47
|
if (aspectTypes)
|
|
52
48
|
targets.push({ entry, aspectTypes });
|
|
@@ -96,76 +92,42 @@ async function pullKnowledgeCatalog(cat, opts) {
|
|
|
96
92
|
}
|
|
97
93
|
}
|
|
98
94
|
const read = (0, kc_converter_1.modelsFromCatalogResources)(hydrated, entryLinks, { v2Aspects: opts.v2Aspects });
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
95
|
+
// Every push labels the model entry as writing when it writes the model entry
|
|
96
|
+
// and as finished with its last. The hydrated entries keep the order of
|
|
97
|
+
// `targets`.
|
|
98
|
+
const anchor = anchors.length ? hydrated[targets.indexOf(anchors[0])] : undefined;
|
|
99
|
+
const writingReason = anchor?.entrySource?.labels?.[kc_custom_types_1.PUSH_STATE_LABEL] === kc_custom_types_1.PUSH_WRITING ?
|
|
100
|
+
`its model entry's ${kc_custom_types_1.PUSH_STATE_LABEL} label still says ${kc_custom_types_1.PUSH_WRITING}` :
|
|
101
|
+
undefined;
|
|
102
|
+
if (writingReason) {
|
|
103
|
+
// With v2Aspects on, a model an older kcmd pushed is not in `read.models`,
|
|
104
|
+
// so the model is named from its entry, as the reader names it.
|
|
105
|
+
throw notFinished(anchor.entrySource?.displayName ?? (0, kc_ids_1.entryIdOf)(anchor.name), writingReason);
|
|
102
106
|
}
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
warnings.push(
|
|
106
|
-
return { models: [], rawFiles: [], warnings: [...new Set(warnings)] };
|
|
107
|
+
const warnings = [...read.warnings];
|
|
108
|
+
for (const m of read.firstGeneration ?? []) {
|
|
109
|
+
warnings.push(firstGenerationWarning(opts, m));
|
|
107
110
|
}
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
// the local files. The hydrated entries keep the order of `targets`.
|
|
129
|
-
const count = copies[0]?.pushFileCount;
|
|
130
|
-
const anchor = hydrated[targets.indexOf(anchors[0])];
|
|
131
|
-
const label = anchor.entrySource?.labels?.[kc_custom_types_1.PUSH_ID_LABEL];
|
|
132
|
-
const unfinished = new Set(copies.map(f => f.pushId)).size > 1 ?
|
|
133
|
-
'its file copies in the catalog come from different pushes' :
|
|
134
|
-
count !== undefined && copies.length < count ?
|
|
135
|
-
`the catalog holds ${copies.length} of its ${count} file copies` :
|
|
136
|
-
count !== undefined && copies.length > count ?
|
|
137
|
-
`the catalog holds ${copies.length} file copies, but the push stored ` +
|
|
138
|
-
`${count}` :
|
|
139
|
-
label !== undefined && !copies.length ?
|
|
140
|
-
(skipped.length ? 'the catalog holds no readable file copy' :
|
|
141
|
-
'the catalog holds none of its file copies') :
|
|
142
|
-
label !== undefined && copies[0].pushId !== label ?
|
|
143
|
-
'its model entry and its file copies come from different pushes' :
|
|
144
|
-
undefined;
|
|
145
|
-
if (unfinished) {
|
|
146
|
-
// A skipped copy is often why the count is off, so its warning goes with
|
|
147
|
-
// the error.
|
|
148
|
-
throw new Error(`the last push of model '${model}' did not finish: ${unfinished}, ` +
|
|
149
|
-
`so pull writes nothing. Push again, then pull.` +
|
|
150
|
-
skipped.map(w => ` ${w}.`).join(''));
|
|
151
|
-
}
|
|
152
|
-
if (copies.some(f => f.fileName === `${model}.yaml`)) {
|
|
153
|
-
// The rebuilt model is not written, so its warnings are left out.
|
|
154
|
-
return {
|
|
155
|
-
models: read.models,
|
|
156
|
-
rawFiles: copies,
|
|
157
|
-
warnings: [...new Set(warnings)],
|
|
158
|
-
};
|
|
159
|
-
}
|
|
160
|
-
warnings.push(...read.warnings);
|
|
161
|
-
warnings.push(copies.length ?
|
|
162
|
-
`the stored copies of model '${model}' hold no '${model}.yaml', so ` +
|
|
163
|
-
`the model file is rebuilt from the catalog's aspects and may ` +
|
|
164
|
-
`differ from what was pushed` :
|
|
165
|
-
'no stored copy of the model file was found, so it is rebuilt from ' +
|
|
166
|
-
'the catalog\'s aspects and may differ from what was pushed; ' +
|
|
167
|
-
'push it again to store a copy');
|
|
168
|
-
return { models: read.models, rawFiles: [], warnings: [...new Set(warnings)] };
|
|
111
|
+
return { ...read, warnings: [...new Set(warnings)] };
|
|
112
|
+
}
|
|
113
|
+
// The error for a model whose last push stopped partway, for `reason`.
|
|
114
|
+
function notFinished(model, reason) {
|
|
115
|
+
return new Error(`the last push of model '${model}' did not finish: ${reason}, so pull ` +
|
|
116
|
+
`writes nothing. Push again, then pull.`);
|
|
117
|
+
}
|
|
118
|
+
// The model, entity and metric aspect types a first-generation push writes, in
|
|
119
|
+
// the order the warning names them.
|
|
120
|
+
const FIRST_GEN_ASPECTS = ['semantic-model', 'semantic-entity', 'semantic-metric'];
|
|
121
|
+
// The warning for a model an older kcmd pushed, which a pull with v2Aspects
|
|
122
|
+
// on leaves out. It names the first-generation aspect types found on the
|
|
123
|
+
// model's entries.
|
|
124
|
+
function firstGenerationWarning(opts, model) {
|
|
125
|
+
const found = new Set(model.entries.flatMap(e => Object.values(e.aspects ?? {})
|
|
126
|
+
.map(a => a.aspectType?.split('/aspectTypes/')[1])));
|
|
127
|
+
const types = FIRST_GEN_ASPECTS.filter(t => found.has(t));
|
|
128
|
+
return `model '${model.name}' in entry group '${opts.entryGroup}' was ` +
|
|
129
|
+
`pushed by an older kcmd (first-generation aspects: ${types.join(', ')}). Pull wrote nothing for it; push it again ` +
|
|
130
|
+
`with the current kcmd`;
|
|
169
131
|
}
|
|
170
132
|
// Fetches `entry` with the aspects of `aspectTypes`, at `paths` when given.
|
|
171
133
|
// Throws on a failed fetch.
|
|
@@ -233,39 +195,6 @@ function pathBatches(opts, entry, type, paths) {
|
|
|
233
195
|
}
|
|
234
196
|
return [...batches, batch];
|
|
235
197
|
}
|
|
236
|
-
// True when an entry is a file copy, matched by its entry type's id suffix so
|
|
237
|
-
// the project the custom type lives in need not be known.
|
|
238
|
-
function isFileCopyEntry(entry) {
|
|
239
|
-
return entry.entryType?.endsWith(`/entryTypes/${kc_custom_types_1.FILE_COPY_TYPE_ID}`) ?? false;
|
|
240
|
-
}
|
|
241
|
-
// The file copies among the hydrated entries, sorted by file name, each with
|
|
242
|
-
// the push id and copy count it carries. A copy whose aspect holds no file
|
|
243
|
-
// name or no content is skipped with a warning.
|
|
244
|
-
function readFileCopies(entries, warnings) {
|
|
245
|
-
const files = [];
|
|
246
|
-
for (const entry of entries) {
|
|
247
|
-
if (!isFileCopyEntry(entry))
|
|
248
|
-
continue;
|
|
249
|
-
const aspect = Object.entries(entry.aspects ?? {}).find(([key, a]) => key.endsWith(`.${kc_custom_types_1.FILE_COPY_TYPE_ID}`) ||
|
|
250
|
-
(a.aspectType ?? '').endsWith(`/aspectTypes/${kc_custom_types_1.FILE_COPY_TYPE_ID}`));
|
|
251
|
-
const data = aspect?.[1].data;
|
|
252
|
-
if (typeof data?.fileName !== 'string' || data.fileName === '' ||
|
|
253
|
-
typeof data.content !== 'string') {
|
|
254
|
-
warnings.push(`file copy '${(0, kc_ids_1.entryIdOf)(entry.name)}' holds no file name or no ` +
|
|
255
|
-
`content; it is skipped`);
|
|
256
|
-
continue;
|
|
257
|
-
}
|
|
258
|
-
files.push({
|
|
259
|
-
fileName: data.fileName,
|
|
260
|
-
content: data.content,
|
|
261
|
-
...(typeof data.pushId === 'string' ? { pushId: data.pushId } : {}),
|
|
262
|
-
...(typeof data.pushFileCount === 'number' ?
|
|
263
|
-
{ pushFileCount: data.pushFileCount } :
|
|
264
|
-
{}),
|
|
265
|
-
});
|
|
266
|
-
}
|
|
267
|
-
return files.sort((a, b) => a.fileName < b.fileName ? -1 : a.fileName > b.fileName ? 1 : 0);
|
|
268
|
-
}
|
|
269
198
|
// The aspect types each second-generation entry type is read with.
|
|
270
199
|
const OBJECT_ASPECTS = ['guidelines', 'semantic-extensions', 'overview'];
|
|
271
200
|
const V2_ASPECT_TYPES = new Map([
|
|
@@ -312,9 +241,6 @@ function semanticAspectTypes(entryType, v2Aspects) {
|
|
|
312
241
|
case kc_custom_types_1.CONSTRAINT_TYPE_ID:
|
|
313
242
|
// Custom for the same reason, and named through the same `typeBase`.
|
|
314
243
|
return (0, kc_constraints_1.constraintAspectTypes)(typeBase);
|
|
315
|
-
case kc_custom_types_1.FILE_COPY_TYPE_ID:
|
|
316
|
-
// Custom too: one aspect holding the file's name and text.
|
|
317
|
-
return [aspectType(kc_custom_types_1.FILE_COPY_TYPE_ID)];
|
|
318
244
|
default:
|
|
319
245
|
return undefined;
|
|
320
246
|
}
|
|
@@ -12,6 +12,9 @@ export interface ResolveResult {
|
|
|
12
12
|
model: SemanticModel;
|
|
13
13
|
warnings: string[];
|
|
14
14
|
}
|
|
15
|
+
export interface ResolveOptions {
|
|
16
|
+
requalify?: boolean;
|
|
17
|
+
}
|
|
15
18
|
/**
|
|
16
19
|
* What a `concept` name may refer to, and the fields it declares.
|
|
17
20
|
*
|
|
@@ -46,9 +49,11 @@ export declare function declaredConceptFields(model: SemanticModel): Map<string,
|
|
|
46
49
|
* inheritance and no exclusions resolves to an equivalent model.
|
|
47
50
|
* `targetDialect` is the SQL that an inherited field's target text and its
|
|
48
51
|
* ANSI_SQL entries are read in to find their comments. It is BigQuery unless
|
|
49
|
-
* given, as the graph generators read the target text.
|
|
52
|
+
* given, as the graph generators read the target text. With
|
|
53
|
+
* `options.requalify`, an inherited expression keeps naming the fields it
|
|
54
|
+
* names on its ancestor, under the inheriting entity's qualifier.
|
|
50
55
|
*
|
|
51
56
|
* Reading an inherited field's SQL needs the SQL parser, so call
|
|
52
57
|
* `loadSqlEngine()` first. Without it, this throws the parser's plain Error.
|
|
53
58
|
*/
|
|
54
|
-
export declare function resolveInheritance(model: SemanticModel, targetDialect?: string): ResolveResult;
|
|
59
|
+
export declare function resolveInheritance(model: SemanticModel, targetDialect?: string, options?: ResolveOptions): ResolveResult;
|
|
@@ -83,12 +83,14 @@ function declaredConceptFields(model) {
|
|
|
83
83
|
* inheritance and no exclusions resolves to an equivalent model.
|
|
84
84
|
* `targetDialect` is the SQL that an inherited field's target text and its
|
|
85
85
|
* ANSI_SQL entries are read in to find their comments. It is BigQuery unless
|
|
86
|
-
* given, as the graph generators read the target text.
|
|
86
|
+
* given, as the graph generators read the target text. With
|
|
87
|
+
* `options.requalify`, an inherited expression keeps naming the fields it
|
|
88
|
+
* names on its ancestor, under the inheriting entity's qualifier.
|
|
87
89
|
*
|
|
88
90
|
* Reading an inherited field's SQL needs the SQL parser, so call
|
|
89
91
|
* `loadSqlEngine()` first. Without it, this throws the parser's plain Error.
|
|
90
92
|
*/
|
|
91
|
-
function resolveInheritance(model, targetDialect) {
|
|
93
|
+
function resolveInheritance(model, targetDialect, options = {}) {
|
|
92
94
|
const clone = structuredClone(model);
|
|
93
95
|
const warnings = [];
|
|
94
96
|
const entities = clone.entities ?? [];
|
|
@@ -143,7 +145,9 @@ function resolveInheritance(model, targetDialect) {
|
|
|
143
145
|
}
|
|
144
146
|
// Own fields first, then every inherited field, each merged down its
|
|
145
147
|
// declaring chain. A subtype's redeclaration of an inherited field merges
|
|
146
|
-
// over that chain too, contributing only its binding.
|
|
148
|
+
// over that chain too, contributing only its binding. With `requalify`,
|
|
149
|
+
// each inherited expression names this entity's fields.
|
|
150
|
+
const into = options.requalify ? entity.name : undefined;
|
|
147
151
|
const seenField = new Set();
|
|
148
152
|
const flattened = [];
|
|
149
153
|
for (const f of ownFields.get(entity.name) ?? []) {
|
|
@@ -156,14 +160,14 @@ function resolveInheritance(model, targetDialect) {
|
|
|
156
160
|
continue; // a self-duplicate; keep the first
|
|
157
161
|
seenField.add(f.name);
|
|
158
162
|
const chain = declarers.get(f.name);
|
|
159
|
-
flattened.push(chain ? mergeBinding(mergeChain(f.name, chain, ownFields, ancestorsOf, targetDialect), f) :
|
|
163
|
+
flattened.push(chain ? mergeBinding(mergeChain(f.name, chain, ownFields, ancestorsOf, targetDialect, into), f) :
|
|
160
164
|
f);
|
|
161
165
|
}
|
|
162
166
|
for (const [name, chain] of declarers) {
|
|
163
167
|
if (seenField.has(name))
|
|
164
168
|
continue;
|
|
165
169
|
seenField.add(name);
|
|
166
|
-
flattened.push(mergeChain(name, chain, ownFields, ancestorsOf, targetDialect));
|
|
170
|
+
flattened.push(mergeChain(name, chain, ownFields, ancestorsOf, targetDialect, into));
|
|
167
171
|
}
|
|
168
172
|
// A field the binding profile excludes on this entity is left off it.
|
|
169
173
|
// Descendants read the snapshot of declared fields, so they still inherit
|
|
@@ -186,7 +190,7 @@ function resolveInheritance(model, targetDialect) {
|
|
|
186
190
|
}
|
|
187
191
|
// Clones an ancestor's field for a descendant, rewriting its expression to be
|
|
188
192
|
// TABLE-LOCAL. A field expression written on the ancestor as `<Ancestor>.col`
|
|
189
|
-
//
|
|
193
|
+
// names the ancestor's field `col`; inherited onto a
|
|
190
194
|
// descendant -- whose backing table carries the same column, since inherited
|
|
191
195
|
// fields must physically exist on the child -- it must reference the
|
|
192
196
|
// DESCENDANT's column, so the ancestor's own-name qualifier is stripped.
|
|
@@ -195,23 +199,35 @@ function resolveInheritance(model, targetDialect) {
|
|
|
195
199
|
// tables (e.g. BigQuery's shared labels) would see two different definitions of
|
|
196
200
|
// the same property and reject the graph.
|
|
197
201
|
//
|
|
202
|
+
// Given `into`, the descendant, the ancestor's qualifier is renamed to `into`'s
|
|
203
|
+
// instead. `<Ancestor>.x` names the ancestor's field `x`, and the graph
|
|
204
|
+
// renders such a name from that field's own SQL, so inherited it must name the
|
|
205
|
+
// descendant's `x`, which may bind another column; stripped, it would read a
|
|
206
|
+
// column `x` the descendant's table may not have. The graph
|
|
207
|
+
// generators, and the checks that guard them, ask for this.
|
|
208
|
+
//
|
|
198
209
|
// This composes with (does not duplicate) the emitter's own qualifier
|
|
199
210
|
// stripping: this pass normalizes an inherited expression INTO the child's
|
|
200
211
|
// frame once here, and the emitter's table-local renderer then strips the
|
|
201
|
-
// child's own qualifier uniformly for every field
|
|
202
|
-
// graph_ddl.renderFieldPropertyCore). Both
|
|
203
|
-
//
|
|
212
|
+
// child's own qualifier uniformly for every field, reading each sibling it
|
|
213
|
+
// names (see graph_ddl.renderFieldPropertyCore). Both find the qualifier with
|
|
214
|
+
// the SQL parser (see sql_expr_utils.qualifiersIn).
|
|
204
215
|
//
|
|
205
|
-
//
|
|
206
|
-
//
|
|
207
|
-
//
|
|
208
|
-
//
|
|
209
|
-
//
|
|
210
|
-
//
|
|
211
|
-
//
|
|
212
|
-
|
|
216
|
+
// The text comes out without its comments, and the rest stays as written. The
|
|
217
|
+
// target text is read in `targetDialect`, BigQuery unless given, and so are
|
|
218
|
+
// the ANSI_SQL entries, which a database of that dialect runs as its own SQL.
|
|
219
|
+
// The imported text is read in its imported dialect, and every other dialect
|
|
220
|
+
// entry in its own, so a quoted qualifier (`"Person".name` in PostgreSQL) is
|
|
221
|
+
// found as that dialect quotes it; renamed, it keeps its quotes. A vendor
|
|
222
|
+
// entry that holds the same text as the target does not change how the target
|
|
223
|
+
// is read.
|
|
224
|
+
function localizeInheritedField(field, ancestor, targetDialect, into) {
|
|
213
225
|
const clone = structuredClone(field);
|
|
214
|
-
const local = (text, dialect) =>
|
|
226
|
+
const local = (text, dialect) => {
|
|
227
|
+
const bare = (0, sql_expr_utils_1.sqlWithoutComments)(text, dialect);
|
|
228
|
+
return into === undefined ? (0, sql_expr_utils_1.withoutQualifier)(bare, ancestor, dialect) :
|
|
229
|
+
(0, sql_expr_utils_1.requalified)(bare, ancestor, into, dialect);
|
|
230
|
+
};
|
|
215
231
|
if (clone.expression !== undefined) {
|
|
216
232
|
clone.expression = local(clone.expression, targetDialect);
|
|
217
233
|
}
|
|
@@ -254,13 +270,13 @@ function mergeBinding(base, over) {
|
|
|
254
270
|
// the chain is sorted by how many ancestors each member has; requireOneChain
|
|
255
271
|
// has already checked that every member descends from the next. Every
|
|
256
272
|
// expression is localized to the declaring ancestor's frame first, its target
|
|
257
|
-
// text read in `targetDialect
|
|
258
|
-
function mergeChain(name, chain, ownFields, ancestorsOf, targetDialect) {
|
|
273
|
+
// text read in `targetDialect`, and moved onto `into` when given.
|
|
274
|
+
function mergeChain(name, chain, ownFields, ancestorsOf, targetDialect, into) {
|
|
259
275
|
const depth = (n) => (ancestorsOf.get(n) ?? []).length;
|
|
260
276
|
let merged;
|
|
261
277
|
for (const anc of [...chain].sort((a, b) => depth(a) - depth(b))) {
|
|
262
278
|
const own = (ownFields.get(anc) ?? []).find(f => f.name === name);
|
|
263
|
-
const local = localizeInheritedField(own, anc, targetDialect);
|
|
279
|
+
const local = localizeInheritedField(own, anc, targetDialect, into);
|
|
264
280
|
merged = merged ? mergeBinding(merged, local) : local;
|
|
265
281
|
}
|
|
266
282
|
return merged;
|