@google-cloud/kcmd 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -20
- package/build/ts/kcmd/cli.js +5842 -5168
- package/build/ts/kcmd/gcp/biglake.d.ts +2 -0
- package/build/ts/kcmd/gcp/biglake.js +13 -4
- package/build/ts/kcmd/gcp/spanner.d.ts +10 -0
- package/build/ts/kcmd/gcp/spanner.js +6 -3
- package/build/ts/kcmd/layouts/okf.js +49 -7
- package/build/ts/kcmd/layouts/semantic-model.d.ts +13 -0
- package/build/ts/kcmd/layouts/semantic-model.js +129 -50
- package/build/ts/kcmd/okf/concept.d.ts +1 -1
- package/build/ts/kcmd/okf/concept.js +6 -0
- package/build/ts/kcmd/okf/concept_to_entry.d.ts +2 -0
- package/build/ts/kcmd/okf/concept_to_entry.js +29 -2
- package/build/ts/kcmd/okf/init_okf_workspace.js +8 -10
- package/build/ts/kcmd/okf/okf_metadata.js +17 -0
- package/build/ts/kcmd/semantic/bigquery.js +139 -195
- package/build/ts/kcmd/semantic/binding.js +14 -19
- package/build/ts/kcmd/semantic/deploy_knowledge_catalog.d.ts +10 -35
- package/build/ts/kcmd/semantic/deploy_knowledge_catalog.js +92 -121
- package/build/ts/kcmd/semantic/graph_ddl.d.ts +10 -0
- package/build/ts/kcmd/semantic/graph_ddl.js +152 -59
- package/build/ts/kcmd/semantic/import_model.d.ts +26 -4
- package/build/ts/kcmd/semantic/import_model.js +526 -103
- package/build/ts/kcmd/semantic/kc_converter.d.ts +7 -1
- package/build/ts/kcmd/semantic/kc_converter.js +20 -5
- package/build/ts/kcmd/semantic/kc_custom_types.d.ts +5 -12
- package/build/ts/kcmd/semantic/kc_custom_types.js +16 -93
- package/build/ts/kcmd/semantic/kc_ids.d.ts +4 -5
- package/build/ts/kcmd/semantic/kc_ids.js +4 -7
- package/build/ts/kcmd/semantic/knowledge_catalog.d.ts +0 -3
- package/build/ts/kcmd/semantic/knowledge_catalog.js +3 -86
- package/build/ts/kcmd/semantic/loader.d.ts +7 -11
- package/build/ts/kcmd/semantic/loader.js +284 -148
- package/build/ts/kcmd/semantic/osi_converter.js +1 -2
- package/build/ts/kcmd/semantic/parameters.d.ts +1 -0
- package/build/ts/kcmd/semantic/parameters.js +22 -3
- package/build/ts/kcmd/semantic/pull_kc.d.ts +1 -5
- package/build/ts/kcmd/semantic/pull_kc.js +38 -112
- package/build/ts/kcmd/semantic/resolve_inheritance.d.ts +7 -2
- package/build/ts/kcmd/semantic/resolve_inheritance.js +37 -21
- package/build/ts/kcmd/semantic/resolve_profiles.d.ts +51 -10
- package/build/ts/kcmd/semantic/resolve_profiles.js +213 -32
- package/build/ts/kcmd/semantic/runtime/runtime.d.ts +0 -6
- package/build/ts/kcmd/semantic/runtime/runtime.js +4 -21
- package/build/ts/kcmd/semantic/source_dialect.d.ts +9 -2
- package/build/ts/kcmd/semantic/source_dialect.js +83 -33
- package/build/ts/kcmd/semantic/source_entries.d.ts +1 -1
- package/build/ts/kcmd/semantic/source_entries.js +95 -36
- package/build/ts/kcmd/semantic/sql_expr_utils.d.ts +13 -7
- package/build/ts/kcmd/semantic/sql_expr_utils.js +240 -67
- package/build/ts/kcmd/semantic/sql_parser.d.ts +5 -2
- package/build/ts/kcmd/semantic/sql_parser.js +22 -0
- package/build/ts/kcmd/semantic/transpile.d.ts +12 -10
- package/build/ts/kcmd/semantic/transpile.js +79 -61
- package/build/ts/kcmd/semantic/validate.d.ts +37 -2
- package/build/ts/kcmd/semantic/validate.js +465 -98
- package/build/ts/kcmd/snapshot.d.ts +4 -0
- package/build/ts/kcmd/snapshot.js +7 -0
- package/build/ts/kcmd/sync.js +44 -7
- package/package.json +1 -1
|
@@ -436,8 +436,7 @@ function datasetDoc(entity, warnings, logical, version) {
|
|
|
436
436
|
name: entity.name,
|
|
437
437
|
// An abstract entity has no physical table, so its source is empty; omit
|
|
438
438
|
// the key rather than emit `source: ""` (which the loader rejects). The
|
|
439
|
-
// source as written survives serialization
|
|
440
|
-
// as the name the load qualified it to.
|
|
439
|
+
// source as written survives serialization.
|
|
441
440
|
source: (0, loader_1.writtenSource)(entity) || undefined,
|
|
442
441
|
// Supertype entities (entity-level inheritance); native under
|
|
443
442
|
// GOOGLE_VERSION, or in the dataset's GOOGLE block under OSSIE_VERSION.
|
|
@@ -2,6 +2,7 @@ import { ActionParameter } from './ir';
|
|
|
2
2
|
export declare function isParameterRequired(param: ActionParameter): boolean;
|
|
3
3
|
export declare function storeCodeFor(dataType: string): string;
|
|
4
4
|
export declare function sentence(text: string): string;
|
|
5
|
+
export declare function storeMessage(message: string | undefined, status: number): string;
|
|
5
6
|
/**
|
|
6
7
|
* One value, parsed to the store type its declared ontology type implies.
|
|
7
8
|
*
|
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
// Leaf helpers for action parameter optionality, scalar type checking,
|
|
3
|
-
//
|
|
4
|
-
// without pulling store clients into the validation layer.
|
|
2
|
+
// Leaf helpers for action parameter optionality, scalar type checking, store
|
|
3
|
+
// type codes and message sentences -- shared by deploy-time validation and
|
|
4
|
+
// run-time execution without pulling store clients into the validation layer.
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.isParameterRequired = isParameterRequired;
|
|
7
7
|
exports.storeCodeFor = storeCodeFor;
|
|
8
8
|
exports.sentence = sentence;
|
|
9
|
+
exports.storeMessage = storeMessage;
|
|
9
10
|
exports.bindScalar = bindScalar;
|
|
10
11
|
function isParameterRequired(param) {
|
|
11
12
|
if (param.required !== undefined)
|
|
@@ -36,6 +37,24 @@ function sentence(text) {
|
|
|
36
37
|
const trimmed = text.trim();
|
|
37
38
|
return /[.!?]$/.test(trimmed) ? trimmed : `${trimmed}.`;
|
|
38
39
|
}
|
|
40
|
+
// The first line of a multi-line diagnosis. The lines after it echo the
|
|
41
|
+
// statement and point a caret at the offending token, which is worth nothing
|
|
42
|
+
// here: the author has the statement open in the profile, the position in
|
|
43
|
+
// `[at 1:54]` survives, and a caret counted from column 1 lands in the wrong
|
|
44
|
+
// place once the message is embedded in a longer sentence.
|
|
45
|
+
//
|
|
46
|
+
// `\n` is matched in both its forms because Spanner's top-level `error.message`
|
|
47
|
+
// arrives with its newlines ESCAPED -- a literal backslash and an n survive
|
|
48
|
+
// JSON.parse -- while every other message uses real ones.
|
|
49
|
+
function firstLine(message) {
|
|
50
|
+
return message.replace(/\\n/g, '\n').split('\n')[0].trim();
|
|
51
|
+
}
|
|
52
|
+
// An API error's message reduced to its first line as a sentence, such as
|
|
53
|
+
// "Unrecognized name: accountId; Did you mean account_id? [at 1:54]."
|
|
54
|
+
function storeMessage(message, status) {
|
|
55
|
+
const text = (message ?? '').trim();
|
|
56
|
+
return text ? sentence(firstLine(text)) : `HTTP ${status}.`;
|
|
57
|
+
}
|
|
39
58
|
// What Spanner accepts for a DATE and a TIMESTAMP parameter. The zone is
|
|
40
59
|
// required rather than defaulted, because a timestamp written without one
|
|
41
60
|
// means a different instant to every reader who supplies the missing part.
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { CatalogClient } from '../gcp/dataplex';
|
|
2
2
|
import { ReadResult } from './kc_converter';
|
|
3
|
-
import { FileCopy } from './kc_custom_types';
|
|
4
3
|
export interface KcPullOptions {
|
|
5
4
|
project: string;
|
|
6
5
|
location: string;
|
|
@@ -8,7 +7,4 @@ export interface KcPullOptions {
|
|
|
8
7
|
v2Aspects?: boolean;
|
|
9
8
|
systemTypeProject?: string;
|
|
10
9
|
}
|
|
11
|
-
export
|
|
12
|
-
rawFiles?: FileCopy[];
|
|
13
|
-
}
|
|
14
|
-
export declare function pullKnowledgeCatalog(cat: CatalogClient, opts: KcPullOptions): Promise<KcPullResult>;
|
|
10
|
+
export declare function pullKnowledgeCatalog(cat: CatalogClient, opts: KcPullOptions): Promise<ReadResult>;
|
|
@@ -8,9 +8,9 @@
|
|
|
8
8
|
// (a BASIC list omits aspect data, so each entry is re-fetched with its aspect
|
|
9
9
|
// types -- an entity needs BOTH its `semantic-entity` aspect and the built-in
|
|
10
10
|
// `schema` aspect), and hands the hydrated entries to the pure reader
|
|
11
|
-
// (`kc_converter.modelsFromCatalogResources`). With the
|
|
12
|
-
//
|
|
13
|
-
//
|
|
11
|
+
// (`kc_converter.modelsFromCatalogResources`). With the second-generation
|
|
12
|
+
// types, a model an older kcmd pushed with the first-generation ones is left
|
|
13
|
+
// out, and a warning names it.
|
|
14
14
|
//
|
|
15
15
|
// TODO(#278): Layer 2 push/pull symmetry. This is the pull half over the
|
|
16
16
|
// pure `kc_converter` codec; the push half (`deployKnowledgeCatalog`) still
|
|
@@ -32,21 +32,17 @@ const HYDRATE_CONCURRENCY = 8;
|
|
|
32
32
|
// Reads the semantic model back from a Knowledge Catalog entry group. Emits no
|
|
33
33
|
// console output; soft warnings (from the reader) are returned for the caller
|
|
34
34
|
// to print. Hard failures -- more than one model in the group, a fetch error
|
|
35
|
-
// on any entry or its links, or
|
|
35
|
+
// on any entry or its links, or a model entry still labeled as being written
|
|
36
36
|
// -- throw, so the pull aborts rather than write a partial model.
|
|
37
37
|
async function pullKnowledgeCatalog(cat, opts) {
|
|
38
38
|
await (0, sql_parser_1.loadSqlEngine)();
|
|
39
39
|
const destination = `${opts.project}.${opts.location}.${opts.entryGroup}`;
|
|
40
|
-
const warnings = [];
|
|
41
40
|
// Enumerate the group (paging is inherently sequential) and pick the semantic
|
|
42
41
|
// entries, then hydrate their aspects concurrently: a BASIC list omits aspect
|
|
43
42
|
// data, so each entry needs its own lookupEntry, and those fetches are
|
|
44
43
|
// independent. The pool preserves input order so warnings stay deterministic.
|
|
45
44
|
const targets = [];
|
|
46
45
|
for await (const entry of cat.listEntries(opts.project, opts.location, opts.entryGroup)) {
|
|
47
|
-
// With v2Aspects on, a pull returns no file copies, so it fetches none.
|
|
48
|
-
if (opts.v2Aspects && isFileCopyEntry(entry))
|
|
49
|
-
continue;
|
|
50
46
|
const aspectTypes = semanticAspectTypes(entry.entryType, opts.v2Aspects);
|
|
51
47
|
if (aspectTypes)
|
|
52
48
|
targets.push({ entry, aspectTypes });
|
|
@@ -96,76 +92,42 @@ async function pullKnowledgeCatalog(cat, opts) {
|
|
|
96
92
|
}
|
|
97
93
|
}
|
|
98
94
|
const read = (0, kc_converter_1.modelsFromCatalogResources)(hydrated, entryLinks, { v2Aspects: opts.v2Aspects });
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
95
|
+
// Every push labels the model entry as writing when it writes the model entry
|
|
96
|
+
// and as finished with its last. The hydrated entries keep the order of
|
|
97
|
+
// `targets`.
|
|
98
|
+
const anchor = anchors.length ? hydrated[targets.indexOf(anchors[0])] : undefined;
|
|
99
|
+
const writingReason = anchor?.entrySource?.labels?.[kc_custom_types_1.PUSH_STATE_LABEL] === kc_custom_types_1.PUSH_WRITING ?
|
|
100
|
+
`its model entry's ${kc_custom_types_1.PUSH_STATE_LABEL} label still says ${kc_custom_types_1.PUSH_WRITING}` :
|
|
101
|
+
undefined;
|
|
102
|
+
if (writingReason) {
|
|
103
|
+
// With v2Aspects on, a model an older kcmd pushed is not in `read.models`,
|
|
104
|
+
// so the model is named from its entry, as the reader names it.
|
|
105
|
+
throw notFinished(anchor.entrySource?.displayName ?? (0, kc_ids_1.entryIdOf)(anchor.name), writingReason);
|
|
102
106
|
}
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
warnings.push(
|
|
106
|
-
return { models: [], rawFiles: [], warnings: [...new Set(warnings)] };
|
|
107
|
+
const warnings = [...read.warnings];
|
|
108
|
+
for (const m of read.firstGeneration ?? []) {
|
|
109
|
+
warnings.push(firstGenerationWarning(opts, m));
|
|
107
110
|
}
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
// the local files. The hydrated entries keep the order of `targets`.
|
|
129
|
-
const count = copies[0]?.pushFileCount;
|
|
130
|
-
const anchor = hydrated[targets.indexOf(anchors[0])];
|
|
131
|
-
const label = anchor.entrySource?.labels?.[kc_custom_types_1.PUSH_ID_LABEL];
|
|
132
|
-
const unfinished = new Set(copies.map(f => f.pushId)).size > 1 ?
|
|
133
|
-
'its file copies in the catalog come from different pushes' :
|
|
134
|
-
count !== undefined && copies.length < count ?
|
|
135
|
-
`the catalog holds ${copies.length} of its ${count} file copies` :
|
|
136
|
-
count !== undefined && copies.length > count ?
|
|
137
|
-
`the catalog holds ${copies.length} file copies, but the push stored ` +
|
|
138
|
-
`${count}` :
|
|
139
|
-
label !== undefined && !copies.length ?
|
|
140
|
-
(skipped.length ? 'the catalog holds no readable file copy' :
|
|
141
|
-
'the catalog holds none of its file copies') :
|
|
142
|
-
label !== undefined && copies[0].pushId !== label ?
|
|
143
|
-
'its model entry and its file copies come from different pushes' :
|
|
144
|
-
undefined;
|
|
145
|
-
if (unfinished) {
|
|
146
|
-
// A skipped copy is often why the count is off, so its warning goes with
|
|
147
|
-
// the error.
|
|
148
|
-
throw new Error(`the last push of model '${model}' did not finish: ${unfinished}, ` +
|
|
149
|
-
`so pull writes nothing. Push again, then pull.` +
|
|
150
|
-
skipped.map(w => ` ${w}.`).join(''));
|
|
151
|
-
}
|
|
152
|
-
if (copies.some(f => f.fileName === `${model}.yaml`)) {
|
|
153
|
-
// The rebuilt model is not written, so its warnings are left out.
|
|
154
|
-
return {
|
|
155
|
-
models: read.models,
|
|
156
|
-
rawFiles: copies,
|
|
157
|
-
warnings: [...new Set(warnings)],
|
|
158
|
-
};
|
|
159
|
-
}
|
|
160
|
-
warnings.push(...read.warnings);
|
|
161
|
-
warnings.push(copies.length ?
|
|
162
|
-
`the stored copies of model '${model}' hold no '${model}.yaml', so ` +
|
|
163
|
-
`the model file is rebuilt from the catalog's aspects and may ` +
|
|
164
|
-
`differ from what was pushed` :
|
|
165
|
-
'no stored copy of the model file was found, so it is rebuilt from ' +
|
|
166
|
-
'the catalog\'s aspects and may differ from what was pushed; ' +
|
|
167
|
-
'push it again to store a copy');
|
|
168
|
-
return { models: read.models, rawFiles: [], warnings: [...new Set(warnings)] };
|
|
111
|
+
return { ...read, warnings: [...new Set(warnings)] };
|
|
112
|
+
}
|
|
113
|
+
// The error for a model whose last push stopped partway, for `reason`.
|
|
114
|
+
function notFinished(model, reason) {
|
|
115
|
+
return new Error(`the last push of model '${model}' did not finish: ${reason}, so pull ` +
|
|
116
|
+
`writes nothing. Push again, then pull.`);
|
|
117
|
+
}
|
|
118
|
+
// The model, entity and metric aspect types a first-generation push writes, in
|
|
119
|
+
// the order the warning names them.
|
|
120
|
+
const FIRST_GEN_ASPECTS = ['semantic-model', 'semantic-entity', 'semantic-metric'];
|
|
121
|
+
// The warning for a model an older kcmd pushed, which a pull with v2Aspects
|
|
122
|
+
// on leaves out. It names the first-generation aspect types found on the
|
|
123
|
+
// model's entries.
|
|
124
|
+
function firstGenerationWarning(opts, model) {
|
|
125
|
+
const found = new Set(model.entries.flatMap(e => Object.values(e.aspects ?? {})
|
|
126
|
+
.map(a => a.aspectType?.split('/aspectTypes/')[1])));
|
|
127
|
+
const types = FIRST_GEN_ASPECTS.filter(t => found.has(t));
|
|
128
|
+
return `model '${model.name}' in entry group '${opts.entryGroup}' was ` +
|
|
129
|
+
`pushed by an older kcmd (first-generation aspects: ${types.join(', ')}). Pull wrote nothing for it; push it again ` +
|
|
130
|
+
`with the current kcmd`;
|
|
169
131
|
}
|
|
170
132
|
// Fetches `entry` with the aspects of `aspectTypes`, at `paths` when given.
|
|
171
133
|
// Throws on a failed fetch.
|
|
@@ -233,39 +195,6 @@ function pathBatches(opts, entry, type, paths) {
|
|
|
233
195
|
}
|
|
234
196
|
return [...batches, batch];
|
|
235
197
|
}
|
|
236
|
-
// True when an entry is a file copy, matched by its entry type's id suffix so
|
|
237
|
-
// the project the custom type lives in need not be known.
|
|
238
|
-
function isFileCopyEntry(entry) {
|
|
239
|
-
return entry.entryType?.endsWith(`/entryTypes/${kc_custom_types_1.FILE_COPY_TYPE_ID}`) ?? false;
|
|
240
|
-
}
|
|
241
|
-
// The file copies among the hydrated entries, sorted by file name, each with
|
|
242
|
-
// the push id and copy count it carries. A copy whose aspect holds no file
|
|
243
|
-
// name or no content is skipped with a warning.
|
|
244
|
-
function readFileCopies(entries, warnings) {
|
|
245
|
-
const files = [];
|
|
246
|
-
for (const entry of entries) {
|
|
247
|
-
if (!isFileCopyEntry(entry))
|
|
248
|
-
continue;
|
|
249
|
-
const aspect = Object.entries(entry.aspects ?? {}).find(([key, a]) => key.endsWith(`.${kc_custom_types_1.FILE_COPY_TYPE_ID}`) ||
|
|
250
|
-
(a.aspectType ?? '').endsWith(`/aspectTypes/${kc_custom_types_1.FILE_COPY_TYPE_ID}`));
|
|
251
|
-
const data = aspect?.[1].data;
|
|
252
|
-
if (typeof data?.fileName !== 'string' || data.fileName === '' ||
|
|
253
|
-
typeof data.content !== 'string') {
|
|
254
|
-
warnings.push(`file copy '${(0, kc_ids_1.entryIdOf)(entry.name)}' holds no file name or no ` +
|
|
255
|
-
`content; it is skipped`);
|
|
256
|
-
continue;
|
|
257
|
-
}
|
|
258
|
-
files.push({
|
|
259
|
-
fileName: data.fileName,
|
|
260
|
-
content: data.content,
|
|
261
|
-
...(typeof data.pushId === 'string' ? { pushId: data.pushId } : {}),
|
|
262
|
-
...(typeof data.pushFileCount === 'number' ?
|
|
263
|
-
{ pushFileCount: data.pushFileCount } :
|
|
264
|
-
{}),
|
|
265
|
-
});
|
|
266
|
-
}
|
|
267
|
-
return files.sort((a, b) => a.fileName < b.fileName ? -1 : a.fileName > b.fileName ? 1 : 0);
|
|
268
|
-
}
|
|
269
198
|
// The aspect types each second-generation entry type is read with.
|
|
270
199
|
const OBJECT_ASPECTS = ['guidelines', 'semantic-extensions', 'overview'];
|
|
271
200
|
const V2_ASPECT_TYPES = new Map([
|
|
@@ -312,9 +241,6 @@ function semanticAspectTypes(entryType, v2Aspects) {
|
|
|
312
241
|
case kc_custom_types_1.CONSTRAINT_TYPE_ID:
|
|
313
242
|
// Custom for the same reason, and named through the same `typeBase`.
|
|
314
243
|
return (0, kc_constraints_1.constraintAspectTypes)(typeBase);
|
|
315
|
-
case kc_custom_types_1.FILE_COPY_TYPE_ID:
|
|
316
|
-
// Custom too: one aspect holding the file's name and text.
|
|
317
|
-
return [aspectType(kc_custom_types_1.FILE_COPY_TYPE_ID)];
|
|
318
244
|
default:
|
|
319
245
|
return undefined;
|
|
320
246
|
}
|
|
@@ -12,6 +12,9 @@ export interface ResolveResult {
|
|
|
12
12
|
model: SemanticModel;
|
|
13
13
|
warnings: string[];
|
|
14
14
|
}
|
|
15
|
+
export interface ResolveOptions {
|
|
16
|
+
requalify?: boolean;
|
|
17
|
+
}
|
|
15
18
|
/**
|
|
16
19
|
* What a `concept` name may refer to, and the fields it declares.
|
|
17
20
|
*
|
|
@@ -46,9 +49,11 @@ export declare function declaredConceptFields(model: SemanticModel): Map<string,
|
|
|
46
49
|
* inheritance and no exclusions resolves to an equivalent model.
|
|
47
50
|
* `targetDialect` is the SQL that an inherited field's target text and its
|
|
48
51
|
* ANSI_SQL entries are read in to find their comments. It is BigQuery unless
|
|
49
|
-
* given, as the graph generators read the target text.
|
|
52
|
+
* given, as the graph generators read the target text. With
|
|
53
|
+
* `options.requalify`, an inherited expression keeps naming the fields it
|
|
54
|
+
* names on its ancestor, under the inheriting entity's qualifier.
|
|
50
55
|
*
|
|
51
56
|
* Reading an inherited field's SQL needs the SQL parser, so call
|
|
52
57
|
* `loadSqlEngine()` first. Without it, this throws the parser's plain Error.
|
|
53
58
|
*/
|
|
54
|
-
export declare function resolveInheritance(model: SemanticModel, targetDialect?: string): ResolveResult;
|
|
59
|
+
export declare function resolveInheritance(model: SemanticModel, targetDialect?: string, options?: ResolveOptions): ResolveResult;
|
|
@@ -83,12 +83,14 @@ function declaredConceptFields(model) {
|
|
|
83
83
|
* inheritance and no exclusions resolves to an equivalent model.
|
|
84
84
|
* `targetDialect` is the SQL that an inherited field's target text and its
|
|
85
85
|
* ANSI_SQL entries are read in to find their comments. It is BigQuery unless
|
|
86
|
-
* given, as the graph generators read the target text.
|
|
86
|
+
* given, as the graph generators read the target text. With
|
|
87
|
+
* `options.requalify`, an inherited expression keeps naming the fields it
|
|
88
|
+
* names on its ancestor, under the inheriting entity's qualifier.
|
|
87
89
|
*
|
|
88
90
|
* Reading an inherited field's SQL needs the SQL parser, so call
|
|
89
91
|
* `loadSqlEngine()` first. Without it, this throws the parser's plain Error.
|
|
90
92
|
*/
|
|
91
|
-
function resolveInheritance(model, targetDialect) {
|
|
93
|
+
function resolveInheritance(model, targetDialect, options = {}) {
|
|
92
94
|
const clone = structuredClone(model);
|
|
93
95
|
const warnings = [];
|
|
94
96
|
const entities = clone.entities ?? [];
|
|
@@ -143,7 +145,9 @@ function resolveInheritance(model, targetDialect) {
|
|
|
143
145
|
}
|
|
144
146
|
// Own fields first, then every inherited field, each merged down its
|
|
145
147
|
// declaring chain. A subtype's redeclaration of an inherited field merges
|
|
146
|
-
// over that chain too, contributing only its binding.
|
|
148
|
+
// over that chain too, contributing only its binding. With `requalify`,
|
|
149
|
+
// each inherited expression names this entity's fields.
|
|
150
|
+
const into = options.requalify ? entity.name : undefined;
|
|
147
151
|
const seenField = new Set();
|
|
148
152
|
const flattened = [];
|
|
149
153
|
for (const f of ownFields.get(entity.name) ?? []) {
|
|
@@ -156,14 +160,14 @@ function resolveInheritance(model, targetDialect) {
|
|
|
156
160
|
continue; // a self-duplicate; keep the first
|
|
157
161
|
seenField.add(f.name);
|
|
158
162
|
const chain = declarers.get(f.name);
|
|
159
|
-
flattened.push(chain ? mergeBinding(mergeChain(f.name, chain, ownFields, ancestorsOf, targetDialect), f) :
|
|
163
|
+
flattened.push(chain ? mergeBinding(mergeChain(f.name, chain, ownFields, ancestorsOf, targetDialect, into), f) :
|
|
160
164
|
f);
|
|
161
165
|
}
|
|
162
166
|
for (const [name, chain] of declarers) {
|
|
163
167
|
if (seenField.has(name))
|
|
164
168
|
continue;
|
|
165
169
|
seenField.add(name);
|
|
166
|
-
flattened.push(mergeChain(name, chain, ownFields, ancestorsOf, targetDialect));
|
|
170
|
+
flattened.push(mergeChain(name, chain, ownFields, ancestorsOf, targetDialect, into));
|
|
167
171
|
}
|
|
168
172
|
// A field the binding profile excludes on this entity is left off it.
|
|
169
173
|
// Descendants read the snapshot of declared fields, so they still inherit
|
|
@@ -186,7 +190,7 @@ function resolveInheritance(model, targetDialect) {
|
|
|
186
190
|
}
|
|
187
191
|
// Clones an ancestor's field for a descendant, rewriting its expression to be
|
|
188
192
|
// TABLE-LOCAL. A field expression written on the ancestor as `<Ancestor>.col`
|
|
189
|
-
//
|
|
193
|
+
// names the ancestor's field `col`; inherited onto a
|
|
190
194
|
// descendant -- whose backing table carries the same column, since inherited
|
|
191
195
|
// fields must physically exist on the child -- it must reference the
|
|
192
196
|
// DESCENDANT's column, so the ancestor's own-name qualifier is stripped.
|
|
@@ -195,23 +199,35 @@ function resolveInheritance(model, targetDialect) {
|
|
|
195
199
|
// tables (e.g. BigQuery's shared labels) would see two different definitions of
|
|
196
200
|
// the same property and reject the graph.
|
|
197
201
|
//
|
|
202
|
+
// Given `into`, the descendant, the ancestor's qualifier is renamed to `into`'s
|
|
203
|
+
// instead. `<Ancestor>.x` names the ancestor's field `x`, and the graph
|
|
204
|
+
// renders such a name from that field's own SQL, so inherited it must name the
|
|
205
|
+
// descendant's `x`, which may bind another column; stripped, it would read a
|
|
206
|
+
// column `x` the descendant's table may not have. The graph
|
|
207
|
+
// generators, and the checks that guard them, ask for this.
|
|
208
|
+
//
|
|
198
209
|
// This composes with (does not duplicate) the emitter's own qualifier
|
|
199
210
|
// stripping: this pass normalizes an inherited expression INTO the child's
|
|
200
211
|
// frame once here, and the emitter's table-local renderer then strips the
|
|
201
|
-
// child's own qualifier uniformly for every field
|
|
202
|
-
// graph_ddl.renderFieldPropertyCore). Both
|
|
203
|
-
//
|
|
212
|
+
// child's own qualifier uniformly for every field, reading each sibling it
|
|
213
|
+
// names (see graph_ddl.renderFieldPropertyCore). Both find the qualifier with
|
|
214
|
+
// the SQL parser (see sql_expr_utils.qualifiersIn).
|
|
204
215
|
//
|
|
205
|
-
//
|
|
206
|
-
//
|
|
207
|
-
//
|
|
208
|
-
//
|
|
209
|
-
//
|
|
210
|
-
//
|
|
211
|
-
//
|
|
212
|
-
|
|
216
|
+
// The text comes out without its comments, and the rest stays as written. The
|
|
217
|
+
// target text is read in `targetDialect`, BigQuery unless given, and so are
|
|
218
|
+
// the ANSI_SQL entries, which a database of that dialect runs as its own SQL.
|
|
219
|
+
// The imported text is read in its imported dialect, and every other dialect
|
|
220
|
+
// entry in its own, so a quoted qualifier (`"Person".name` in PostgreSQL) is
|
|
221
|
+
// found as that dialect quotes it; renamed, it keeps its quotes. A vendor
|
|
222
|
+
// entry that holds the same text as the target does not change how the target
|
|
223
|
+
// is read.
|
|
224
|
+
function localizeInheritedField(field, ancestor, targetDialect, into) {
|
|
213
225
|
const clone = structuredClone(field);
|
|
214
|
-
const local = (text, dialect) =>
|
|
226
|
+
const local = (text, dialect) => {
|
|
227
|
+
const bare = (0, sql_expr_utils_1.sqlWithoutComments)(text, dialect);
|
|
228
|
+
return into === undefined ? (0, sql_expr_utils_1.withoutQualifier)(bare, ancestor, dialect) :
|
|
229
|
+
(0, sql_expr_utils_1.requalified)(bare, ancestor, into, dialect);
|
|
230
|
+
};
|
|
215
231
|
if (clone.expression !== undefined) {
|
|
216
232
|
clone.expression = local(clone.expression, targetDialect);
|
|
217
233
|
}
|
|
@@ -254,13 +270,13 @@ function mergeBinding(base, over) {
|
|
|
254
270
|
// the chain is sorted by how many ancestors each member has; requireOneChain
|
|
255
271
|
// has already checked that every member descends from the next. Every
|
|
256
272
|
// expression is localized to the declaring ancestor's frame first, its target
|
|
257
|
-
// text read in `targetDialect
|
|
258
|
-
function mergeChain(name, chain, ownFields, ancestorsOf, targetDialect) {
|
|
273
|
+
// text read in `targetDialect`, and moved onto `into` when given.
|
|
274
|
+
function mergeChain(name, chain, ownFields, ancestorsOf, targetDialect, into) {
|
|
259
275
|
const depth = (n) => (ancestorsOf.get(n) ?? []).length;
|
|
260
276
|
let merged;
|
|
261
277
|
for (const anc of [...chain].sort((a, b) => depth(a) - depth(b))) {
|
|
262
278
|
const own = (ownFields.get(anc) ?? []).find(f => f.name === name);
|
|
263
|
-
const local = localizeInheritedField(own, anc, targetDialect);
|
|
279
|
+
const local = localizeInheritedField(own, anc, targetDialect, into);
|
|
264
280
|
merged = merged ? mergeBinding(merged, local) : local;
|
|
265
281
|
}
|
|
266
282
|
return merged;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Entity, Field, ProfileSpec, SemanticModel } from './ir';
|
|
2
|
-
import { LoadOptions } from './loader';
|
|
2
|
+
import { LoadOptions, SemanticModelError } from './loader';
|
|
3
3
|
export declare const DEFAULT_PROFILE = "default";
|
|
4
4
|
export declare function plainCopy(value: unknown): any;
|
|
5
5
|
export interface AvailabilityReport {
|
|
@@ -38,12 +38,13 @@ export interface AvailabilityReport {
|
|
|
38
38
|
* it. What a field or metric reads comes from the text `dialect` evaluates
|
|
39
39
|
* (see textsFor). `dialect` defaults to the dialect the model's sources
|
|
40
40
|
* imply. When the sources are in systems of different dialects, or one names
|
|
41
|
-
* no system, there is no dialect and every text counts.
|
|
41
|
+
* no system, there is no dialect and every text counts. When no lookup read
|
|
42
|
+
* the dialect, it is null and no text counts.
|
|
42
43
|
* The input is never mutated. The report names each dropped block
|
|
43
44
|
* and the unbound field that stops it, so a caller can state the withheld
|
|
44
45
|
* coverage. A model with nothing unbound resolves unchanged.
|
|
45
46
|
*/
|
|
46
|
-
export declare function pruneUnavailable(model: SemanticModel, profileName: string, dialect?: string | undefined): {
|
|
47
|
+
export declare function pruneUnavailable(model: SemanticModel, profileName: string, dialect?: string | null | undefined): {
|
|
47
48
|
model: SemanticModel;
|
|
48
49
|
report: AvailabilityReport;
|
|
49
50
|
};
|
|
@@ -116,7 +117,9 @@ export interface AppliedProfile {
|
|
|
116
117
|
}
|
|
117
118
|
/**
|
|
118
119
|
* A profile a model declares: the text of a profile file, or a profile already
|
|
119
|
-
* read, such as one a vanilla model carries in its GOOGLE block.
|
|
120
|
+
* read, such as one a vanilla model carries in its GOOGLE block. A profile
|
|
121
|
+
* already read may carry `label`, the file that holds it, which
|
|
122
|
+
* namedProfileErrors names in place of the model document.
|
|
120
123
|
*/
|
|
121
124
|
export type DeclaredProfile = {
|
|
122
125
|
name: string;
|
|
@@ -124,7 +127,17 @@ export type DeclaredProfile = {
|
|
|
124
127
|
} | {
|
|
125
128
|
name: string;
|
|
126
129
|
spec: ProfileSpec;
|
|
130
|
+
label?: string;
|
|
127
131
|
};
|
|
132
|
+
export type DeclaredProfiles = (doc: {
|
|
133
|
+
name: string;
|
|
134
|
+
text: string;
|
|
135
|
+
}) => DeclaredProfile[];
|
|
136
|
+
export type ProfileFiles = (model: string) => Array<{
|
|
137
|
+
name: string;
|
|
138
|
+
text: string;
|
|
139
|
+
fileName: string;
|
|
140
|
+
}>;
|
|
128
141
|
/**
|
|
129
142
|
* The profiles the model document `doc` declares: `files`, its profile files,
|
|
130
143
|
* then the profiles a vanilla model carries in its GOOGLE block, by name, as
|
|
@@ -160,6 +173,16 @@ export declare function profileForDocument(doc: {
|
|
|
160
173
|
name: string;
|
|
161
174
|
text: string;
|
|
162
175
|
}, profiles: DeclaredProfile[], profileName: string, load?: DialectLoad): DocumentProfile;
|
|
176
|
+
/**
|
|
177
|
+
* The warnings that applying the profile `profileName` among `profiles` to the
|
|
178
|
+
* model document `doc` gives, each labeled with the profile. The warnings in
|
|
179
|
+
* `own`, which the document's own load gives, are left out. Empty unless
|
|
180
|
+
* profileForDocument applies the profile as a profile file.
|
|
181
|
+
*/
|
|
182
|
+
export declare function appliedProfileWarnings(doc: {
|
|
183
|
+
name: string;
|
|
184
|
+
text: string;
|
|
185
|
+
}, profiles: DeclaredProfile[], profileName: string, own: string[], load?: DialectLoad): string[];
|
|
163
186
|
/**
|
|
164
187
|
* The error a command gives when a profile cannot be applied to the model
|
|
165
188
|
* document `docName`, whose profile files are `profileFiles`. An error applying
|
|
@@ -173,6 +196,12 @@ export declare function profileErrorMessage(docName: string, res: Extract<Docume
|
|
|
173
196
|
text: string;
|
|
174
197
|
fileName: string;
|
|
175
198
|
}>): string;
|
|
199
|
+
/**
|
|
200
|
+
* Why `model` cannot be used because a deployment names a binding profile
|
|
201
|
+
* `declared` does not hold, or undefined when each names `default` or a
|
|
202
|
+
* declared profile. `document` is the model's document, for the message.
|
|
203
|
+
*/
|
|
204
|
+
export declare function undeclaredProfileError(document: string, model: SemanticModel, declared: string[]): string | undefined;
|
|
176
205
|
/** The error for a profile name a model document does not declare. */
|
|
177
206
|
export declare function unknownProfileMessage(profileName: string, model: string, declared: string[]): string;
|
|
178
207
|
/**
|
|
@@ -184,12 +213,13 @@ export declare function unknownProfileMessage(profileName: string, model: string
|
|
|
184
213
|
export declare function reservedProfileNameError(docName: string, profiles: {
|
|
185
214
|
name: string;
|
|
186
215
|
}[]): string | undefined;
|
|
216
|
+
export declare function readerMessage(err: SemanticModelError): string;
|
|
187
217
|
export declare function profileSpecOf(doc: any, profileName: string): ProfileSpec;
|
|
188
218
|
export declare function expressionSources(f: Pick<Field, 'expression' | 'dialects' | 'importedExpression' | 'importedDialect'>): Array<{
|
|
189
219
|
text: string;
|
|
190
220
|
dialect: string;
|
|
191
221
|
}>;
|
|
192
|
-
declare function textsFor(f: Pick<Field, 'expression' | 'dialects' | 'importedExpression' | 'importedDialect'>, dialect: string | undefined): Array<{
|
|
222
|
+
declare function textsFor(f: Pick<Field, 'expression' | 'dialects' | 'importedExpression' | 'importedDialect'>, dialect: string | null | undefined): Array<{
|
|
193
223
|
text: string;
|
|
194
224
|
dialect: string;
|
|
195
225
|
}>;
|
|
@@ -198,6 +228,7 @@ export interface TableDialects {
|
|
|
198
228
|
name: string;
|
|
199
229
|
dialects: string[];
|
|
200
230
|
unread?: boolean;
|
|
231
|
+
guessed?: boolean;
|
|
201
232
|
}
|
|
202
233
|
export declare function tableDialects(entity: Pick<Entity, 'abstract' | 'authoredSource' | 'dataSource' | 'sourceDialect'>, withUnread: boolean): TableDialects | undefined;
|
|
203
234
|
export declare function fieldsOnTables(model: SemanticModel, withUnread?: boolean): Array<{
|
|
@@ -205,15 +236,16 @@ export declare function fieldsOnTables(model: SemanticModel, withUnread?: boolea
|
|
|
205
236
|
field: Field;
|
|
206
237
|
table: TableDialects;
|
|
207
238
|
}>;
|
|
208
|
-
export declare
|
|
209
|
-
export declare function importTranspiles(f: Parameters<typeof textsFor>[0], dialect: string | undefined): boolean;
|
|
239
|
+
export declare function importCommand(flags: string, version?: string): string;
|
|
240
|
+
export declare function importTranspiles(f: Parameters<typeof textsFor>[0], dialect: string | undefined, model: Pick<SemanticModel, 'entities'>): boolean;
|
|
241
|
+
export declare function importHint(f: Parameters<typeof textsFor>[0], dialect: string, model: Pick<SemanticModel, 'entities' | 'version'>): string;
|
|
210
242
|
export declare function sourcesDialect(model: {
|
|
211
243
|
entities?: ReadonlyArray<Pick<Entity, 'abstract' | 'authoredSource' | 'dataSource' | 'sourceDialect'>>;
|
|
212
|
-
}): string | undefined;
|
|
244
|
+
}): string | null | undefined;
|
|
213
245
|
/** What the caller of validateProfileCompleteness checks itself. */
|
|
214
246
|
export interface CompletenessOptions extends DialectLoad {
|
|
215
247
|
ownSqlOnly?: {
|
|
216
|
-
dialect?: string;
|
|
248
|
+
dialect?: string | null;
|
|
217
249
|
};
|
|
218
250
|
}
|
|
219
251
|
/**
|
|
@@ -241,8 +273,17 @@ export declare function validateProfileCompleteness(baseModel: SemanticModel, pr
|
|
|
241
273
|
* profile's join, or the model file's when the profile states none.
|
|
242
274
|
*/
|
|
243
275
|
export declare function validateProfileConsistency(baseModel: SemanticModel, profiles: ProfileSpec[], applied?: string): string[];
|
|
276
|
+
export interface LocalCheckError {
|
|
277
|
+
error: string;
|
|
278
|
+
document?: string;
|
|
279
|
+
profile?: string;
|
|
280
|
+
}
|
|
281
|
+
export declare function namedProfileErrors(docs: Array<{
|
|
282
|
+
name: string;
|
|
283
|
+
text: string;
|
|
284
|
+
}>, baseByDoc: Map<string, SemanticModel>, profileDocs: ProfileFiles, declared: DeclaredProfiles, warnings: string[], load: DialectLoad): LocalCheckError[];
|
|
244
285
|
export declare function inheritanceErrorOf(model: SemanticModel): string | undefined;
|
|
245
|
-
export declare function columnBinding(model: SemanticModel, entityName: string, field: string, dialect?: string): {
|
|
286
|
+
export declare function columnBinding(model: SemanticModel, entityName: string, field: string, dialect?: string | null): {
|
|
246
287
|
owner: string;
|
|
247
288
|
column: string;
|
|
248
289
|
} | undefined;
|