@google-cloud/kcmd 0.0.0-stage → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (150) hide show
  1. package/LICENSE +203 -0
  2. package/README.md +264 -2
  3. package/build/ts/kcmd/cli.js +20252 -0
  4. package/build/ts/kcmd/gcp/alloydb.d.ts +12 -0
  5. package/build/ts/kcmd/gcp/alloydb.js +70 -0
  6. package/build/ts/kcmd/gcp/api.d.ts +29 -0
  7. package/build/ts/kcmd/gcp/api.js +162 -0
  8. package/build/ts/kcmd/gcp/biglake.d.ts +18 -0
  9. package/build/ts/kcmd/gcp/biglake.js +59 -0
  10. package/build/ts/kcmd/gcp/bigquery.d.ts +57 -0
  11. package/build/ts/kcmd/gcp/bigquery.js +134 -0
  12. package/build/ts/kcmd/gcp/context.d.ts +10 -0
  13. package/build/ts/kcmd/gcp/context.js +74 -0
  14. package/build/ts/kcmd/gcp/crm.d.ts +13 -0
  15. package/build/ts/kcmd/gcp/crm.js +91 -0
  16. package/build/ts/kcmd/gcp/dataplex.d.ts +147 -0
  17. package/build/ts/kcmd/gcp/dataplex.js +406 -0
  18. package/build/ts/kcmd/gcp/index.d.ts +3 -0
  19. package/build/ts/kcmd/gcp/index.js +21 -0
  20. package/build/ts/kcmd/gcp/spanner.d.ts +73 -0
  21. package/build/ts/kcmd/gcp/spanner.js +126 -0
  22. package/build/ts/kcmd/index.d.ts +7 -0
  23. package/build/ts/kcmd/index.js +46 -0
  24. package/build/ts/kcmd/layout.d.ts +16 -0
  25. package/build/ts/kcmd/layout.js +31 -0
  26. package/build/ts/kcmd/layouts/documents.d.ts +24 -0
  27. package/build/ts/kcmd/layouts/documents.js +264 -0
  28. package/build/ts/kcmd/layouts/okf.d.ts +31 -0
  29. package/build/ts/kcmd/layouts/okf.js +386 -0
  30. package/build/ts/kcmd/layouts/semantic-model.d.ts +74 -0
  31. package/build/ts/kcmd/layouts/semantic-model.js +621 -0
  32. package/build/ts/kcmd/layouts/standard.d.ts +13 -0
  33. package/build/ts/kcmd/layouts/standard.js +108 -0
  34. package/build/ts/kcmd/manifest.d.ts +34 -0
  35. package/build/ts/kcmd/manifest.js +262 -0
  36. package/build/ts/kcmd/metadata.d.ts +25 -0
  37. package/build/ts/kcmd/metadata.js +8 -0
  38. package/build/ts/kcmd/okf/catalog_residue.d.ts +14 -0
  39. package/build/ts/kcmd/okf/catalog_residue.js +124 -0
  40. package/build/ts/kcmd/okf/concept.d.ts +14 -0
  41. package/build/ts/kcmd/okf/concept.js +110 -0
  42. package/build/ts/kcmd/okf/concept_to_entry.d.ts +10 -0
  43. package/build/ts/kcmd/okf/concept_to_entry.js +281 -0
  44. package/build/ts/kcmd/okf/init_okf_workspace.d.ts +15 -0
  45. package/build/ts/kcmd/okf/init_okf_workspace.js +113 -0
  46. package/build/ts/kcmd/okf/links.d.ts +23 -0
  47. package/build/ts/kcmd/okf/links.js +468 -0
  48. package/build/ts/kcmd/okf/okf_metadata.d.ts +12 -0
  49. package/build/ts/kcmd/okf/okf_metadata.js +99 -0
  50. package/build/ts/kcmd/semantic/bigquery.d.ts +22 -0
  51. package/build/ts/kcmd/semantic/bigquery.js +469 -0
  52. package/build/ts/kcmd/semantic/binding.d.ts +3 -0
  53. package/build/ts/kcmd/semantic/binding.js +92 -0
  54. package/build/ts/kcmd/semantic/catalog_entry.d.ts +24 -0
  55. package/build/ts/kcmd/semantic/catalog_entry.js +35 -0
  56. package/build/ts/kcmd/semantic/converters/owl/convert.d.ts +26 -0
  57. package/build/ts/kcmd/semantic/converters/owl/convert.js +41 -0
  58. package/build/ts/kcmd/semantic/converters/owl/model.d.ts +98 -0
  59. package/build/ts/kcmd/semantic/converters/owl/model.js +30 -0
  60. package/build/ts/kcmd/semantic/converters/owl/parse.d.ts +11 -0
  61. package/build/ts/kcmd/semantic/converters/owl/parse.js +634 -0
  62. package/build/ts/kcmd/semantic/converters/owl/to_ir.d.ts +19 -0
  63. package/build/ts/kcmd/semantic/converters/owl/to_ir.js +390 -0
  64. package/build/ts/kcmd/semantic/deploy_bigquery.d.ts +20 -0
  65. package/build/ts/kcmd/semantic/deploy_bigquery.js +226 -0
  66. package/build/ts/kcmd/semantic/deploy_knowledge_catalog.d.ts +104 -0
  67. package/build/ts/kcmd/semantic/deploy_knowledge_catalog.js +1367 -0
  68. package/build/ts/kcmd/semantic/deploy_spanner.d.ts +15 -0
  69. package/build/ts/kcmd/semantic/deploy_spanner.js +150 -0
  70. package/build/ts/kcmd/semantic/deployment_target.d.ts +93 -0
  71. package/build/ts/kcmd/semantic/deployment_target.js +379 -0
  72. package/build/ts/kcmd/semantic/graph_ddl.d.ts +44 -0
  73. package/build/ts/kcmd/semantic/graph_ddl.js +656 -0
  74. package/build/ts/kcmd/semantic/import_model.d.ts +72 -0
  75. package/build/ts/kcmd/semantic/import_model.js +1698 -0
  76. package/build/ts/kcmd/semantic/ir.d.ts +589 -0
  77. package/build/ts/kcmd/semantic/ir.js +247 -0
  78. package/build/ts/kcmd/semantic/kc_actions.d.ts +45 -0
  79. package/build/ts/kcmd/semantic/kc_actions.js +531 -0
  80. package/build/ts/kcmd/semantic/kc_constraints.d.ts +44 -0
  81. package/build/ts/kcmd/semantic/kc_constraints.js +279 -0
  82. package/build/ts/kcmd/semantic/kc_converter.d.ts +37 -0
  83. package/build/ts/kcmd/semantic/kc_converter.js +839 -0
  84. package/build/ts/kcmd/semantic/kc_custom_types.d.ts +63 -0
  85. package/build/ts/kcmd/semantic/kc_custom_types.js +751 -0
  86. package/build/ts/kcmd/semantic/kc_entries.d.ts +155 -0
  87. package/build/ts/kcmd/semantic/kc_entries.js +284 -0
  88. package/build/ts/kcmd/semantic/kc_ids.d.ts +102 -0
  89. package/build/ts/kcmd/semantic/kc_ids.js +188 -0
  90. package/build/ts/kcmd/semantic/knowledge_catalog.d.ts +29 -0
  91. package/build/ts/kcmd/semantic/knowledge_catalog.js +960 -0
  92. package/build/ts/kcmd/semantic/loader.d.ts +258 -0
  93. package/build/ts/kcmd/semantic/loader.js +3576 -0
  94. package/build/ts/kcmd/semantic/names.d.ts +10 -0
  95. package/build/ts/kcmd/semantic/names.js +85 -0
  96. package/build/ts/kcmd/semantic/osi_converter.d.ts +60 -0
  97. package/build/ts/kcmd/semantic/osi_converter.js +910 -0
  98. package/build/ts/kcmd/semantic/parameters.d.ts +19 -0
  99. package/build/ts/kcmd/semantic/parameters.js +171 -0
  100. package/build/ts/kcmd/semantic/pull_kc.d.ts +10 -0
  101. package/build/ts/kcmd/semantic/pull_kc.js +272 -0
  102. package/build/ts/kcmd/semantic/resolve_inheritance.d.ts +59 -0
  103. package/build/ts/kcmd/semantic/resolve_inheritance.js +346 -0
  104. package/build/ts/kcmd/semantic/resolve_profiles.d.ts +291 -0
  105. package/build/ts/kcmd/semantic/resolve_profiles.js +2003 -0
  106. package/build/ts/kcmd/semantic/runtime/agent_tools.d.ts +93 -0
  107. package/build/ts/kcmd/semantic/runtime/agent_tools.js +243 -0
  108. package/build/ts/kcmd/semantic/runtime/dialect.d.ts +13 -0
  109. package/build/ts/kcmd/semantic/runtime/dialect.js +39 -0
  110. package/build/ts/kcmd/semantic/runtime/runtime.d.ts +88 -0
  111. package/build/ts/kcmd/semantic/runtime/runtime.js +301 -0
  112. package/build/ts/kcmd/semantic/runtime/store.d.ts +112 -0
  113. package/build/ts/kcmd/semantic/runtime/store.js +344 -0
  114. package/build/ts/kcmd/semantic/skills.d.ts +110 -0
  115. package/build/ts/kcmd/semantic/skills.js +697 -0
  116. package/build/ts/kcmd/semantic/source_dialect.d.ts +17 -0
  117. package/build/ts/kcmd/semantic/source_dialect.js +311 -0
  118. package/build/ts/kcmd/semantic/source_entries.d.ts +12 -0
  119. package/build/ts/kcmd/semantic/source_entries.js +341 -0
  120. package/build/ts/kcmd/semantic/spanner.d.ts +18 -0
  121. package/build/ts/kcmd/semantic/spanner.js +60 -0
  122. package/build/ts/kcmd/semantic/sql_expr_utils.d.ts +23 -0
  123. package/build/ts/kcmd/semantic/sql_expr_utils.js +394 -0
  124. package/build/ts/kcmd/semantic/sql_identifiers.d.ts +6 -0
  125. package/build/ts/kcmd/semantic/sql_identifiers.js +267 -0
  126. package/build/ts/kcmd/semantic/sql_parser.d.ts +47 -0
  127. package/build/ts/kcmd/semantic/sql_parser.js +400 -0
  128. package/build/ts/kcmd/semantic/transpile.d.ts +76 -0
  129. package/build/ts/kcmd/semantic/transpile.js +581 -0
  130. package/build/ts/kcmd/semantic/validate.d.ts +70 -0
  131. package/build/ts/kcmd/semantic/validate.js +2266 -0
  132. package/build/ts/kcmd/semantic/yaml_options.d.ts +6 -0
  133. package/build/ts/kcmd/semantic/yaml_options.js +58 -0
  134. package/build/ts/kcmd/semantic/yaml_update.d.ts +26 -0
  135. package/build/ts/kcmd/semantic/yaml_update.js +782 -0
  136. package/build/ts/kcmd/snapshot.d.ts +37 -0
  137. package/build/ts/kcmd/snapshot.js +386 -0
  138. package/build/ts/kcmd/source.d.ts +31 -0
  139. package/build/ts/kcmd/source.js +119 -0
  140. package/build/ts/kcmd/sources/bq-dataset.d.ts +17 -0
  141. package/build/ts/kcmd/sources/bq-dataset.js +131 -0
  142. package/build/ts/kcmd/sources/entrygroup.d.ts +21 -0
  143. package/build/ts/kcmd/sources/entrygroup.js +83 -0
  144. package/build/ts/kcmd/sources/kb.d.ts +21 -0
  145. package/build/ts/kcmd/sources/kb.js +82 -0
  146. package/build/ts/kcmd/sources/semantic-model.d.ts +18 -0
  147. package/build/ts/kcmd/sources/semantic-model.js +48 -0
  148. package/build/ts/kcmd/sync.d.ts +35 -0
  149. package/build/ts/kcmd/sync.js +734 -0
  150. package/package.json +62 -4
@@ -0,0 +1,19 @@
1
+ import { ActionParameter } from './ir';
2
+ export declare function isParameterRequired(param: ActionParameter): boolean;
3
+ export declare function storeCodeFor(dataType: string): string;
4
+ export declare function sentence(text: string): string;
5
+ export declare function storeMessage(message: string | undefined, status: number): string;
6
+ /**
7
+ * One value, parsed to the store type its declared ontology type implies.
8
+ *
9
+ * Exported because a read has the same problem a write does: a filter on a
10
+ * typed column has to be bound AS that type, or the predicate needs a cast and
11
+ * no index can answer it. Sharing this also means one answer to what counts as
12
+ * an Integer or a Date, rather than one for writes and another for reads.
13
+ */
14
+ export declare function bindScalar(param: ActionParameter, raw: unknown): {
15
+ value: unknown;
16
+ code: string;
17
+ } | {
18
+ error: string;
19
+ };
@@ -0,0 +1,171 @@
1
+ "use strict";
2
+ // Leaf helpers for action parameter optionality, scalar type checking, store
3
+ // type codes and message sentences -- shared by deploy-time validation and
4
+ // run-time execution without pulling store clients into the validation layer.
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ exports.isParameterRequired = isParameterRequired;
7
+ exports.storeCodeFor = storeCodeFor;
8
+ exports.sentence = sentence;
9
+ exports.storeMessage = storeMessage;
10
+ exports.bindScalar = bindScalar;
11
+ function isParameterRequired(param) {
12
+ if (param.required !== undefined)
13
+ return param.required;
14
+ return param.default === undefined;
15
+ }
16
+ function storeCodeFor(dataType) {
17
+ switch (dataType) {
18
+ case 'Integer':
19
+ return 'INT64';
20
+ case 'Float':
21
+ return 'FLOAT64';
22
+ case 'Decimal':
23
+ return 'NUMERIC';
24
+ case 'Boolean':
25
+ return 'BOOL';
26
+ case 'Date':
27
+ return 'DATE';
28
+ case 'DateTime':
29
+ case 'DateTimeTz':
30
+ return 'TIMESTAMP';
31
+ default:
32
+ return 'STRING';
33
+ }
34
+ }
35
+ // Ends a fragment that is about to be followed by another sentence.
36
+ function sentence(text) {
37
+ const trimmed = text.trim();
38
+ return /[.!?]$/.test(trimmed) ? trimmed : `${trimmed}.`;
39
+ }
40
+ // The first line of a multi-line diagnosis. The lines after it echo the
41
+ // statement and point a caret at the offending token, which is worth nothing
42
+ // here: the author has the statement open in the profile, the position in
43
+ // `[at 1:54]` survives, and a caret counted from column 1 lands in the wrong
44
+ // place once the message is embedded in a longer sentence.
45
+ //
46
+ // `\n` is matched in both its forms because Spanner's top-level `error.message`
47
+ // arrives with its newlines ESCAPED -- a literal backslash and an n survive
48
+ // JSON.parse -- while every other message uses real ones.
49
+ function firstLine(message) {
50
+ return message.replace(/\\n/g, '\n').split('\n')[0].trim();
51
+ }
52
+ // An API error's message reduced to its first line as a sentence, such as
53
+ // "Unrecognized name: accountId; Did you mean account_id? [at 1:54]."
54
+ function storeMessage(message, status) {
55
+ const text = (message ?? '').trim();
56
+ return text ? sentence(firstLine(text)) : `HTTP ${status}.`;
57
+ }
58
+ // What Spanner accepts for a DATE and a TIMESTAMP parameter. The zone is
59
+ // required rather than defaulted, because a timestamp written without one
60
+ // means a different instant to every reader who supplies the missing part.
61
+ const ISO_DATE = /^\d{4}-\d{2}-\d{2}$/;
62
+ const RFC3339_TIMESTAMP = /^\d{4}-\d{2}-\d{2}[Tt ]\d{2}:\d{2}:\d{2}(\.\d+)?([Zz]|[-+]\d{2}:?\d{2})$/;
63
+ // Whether `text` is a day that exists, written the one way Spanner reads.
64
+ // `Date.parse` answers neither question: it accepts '03/04/2026', and it reads
65
+ // '2026-02-30' as the second of March rather than rejecting it. A round trip
66
+ // answers both, because a day that rolled over comes back written differently.
67
+ function isCalendarDay(text) {
68
+ if (!ISO_DATE.test(text))
69
+ return false;
70
+ const utc = new Date(`${text}T00:00:00Z`);
71
+ return !Number.isNaN(utc.getTime()) && utc.toISOString().startsWith(text);
72
+ }
73
+ /**
74
+ * One value, parsed to the store type its declared ontology type implies.
75
+ *
76
+ * Exported because a read has the same problem a write does: a filter on a
77
+ * typed column has to be bound AS that type, or the predicate needs a cast and
78
+ * no index can answer it. Sharing this also means one answer to what counts as
79
+ * an Integer or a Date, rather than one for writes and another for reads.
80
+ */
81
+ function bindScalar(param, raw) {
82
+ // A parameter whose type resolved to nothing. The loader warns and validate
83
+ // refuses to push such a model, so this is reached only through the library
84
+ // entry point -- and there is no type to parse the value against, so there
85
+ // is nothing to bind.
86
+ const type = param.type;
87
+ if (type === undefined) {
88
+ return {
89
+ error: `Action parameter '${param.name}' has no type: it states no ` +
90
+ `scalar 'type' and projects no field that supplies one, so a value ` +
91
+ `cannot be bound to it.`,
92
+ };
93
+ }
94
+ // Everything below stringifies the value before matching it against the
95
+ // type, which is what lets a JSON `"12347"` bind as an Integer. An object or
96
+ // an array has a string form too -- `[object Object]` -- and it would sail
97
+ // through as a String and be written to the store verbatim. A parameter
98
+ // carries ONE scalar, so a composite is refused here rather than flattened.
99
+ if (raw !== undefined && raw !== null && typeof raw === 'object') {
100
+ return {
101
+ error: `Action parameter '${param.name}' (${type}) was given ${Array.isArray(raw) ? 'a list' :
102
+ 'an object'}, but a parameter ` +
103
+ `carries a single scalar value.`,
104
+ };
105
+ }
106
+ // An empty String IS a value: `--arg memo=` is the caller saying the memo is
107
+ // blank, which is a different statement from not passing one. For every
108
+ // other type there is no value empty text could be, so it stays an error.
109
+ if (raw === undefined || raw === null ||
110
+ (`${raw}`.trim() === '' && type !== 'String')) {
111
+ return {
112
+ error: `Action parameter '${param.name}' (${type}) was not given ` +
113
+ `a value.`,
114
+ };
115
+ }
116
+ const text = `${raw}`.trim();
117
+ const code = storeCodeFor(type);
118
+ switch (type) {
119
+ case 'Integer':
120
+ if (!/^[-+]?\d+$/.test(text)) {
121
+ return { error: `'${param.name}' is an Integer, but '${text}' is not.` };
122
+ }
123
+ // INT64 travels as a string over the REST surface; a JSON number would
124
+ // lose precision above 2^53.
125
+ return { value: text, code };
126
+ case 'Float':
127
+ if (!Number.isFinite(Number(text))) {
128
+ return { error: `'${param.name}' is a Float, but '${text}' is not.` };
129
+ }
130
+ return { value: Number(text), code };
131
+ case 'Decimal':
132
+ if (!/^[-+]?\d+(\.\d+)?$/.test(text)) {
133
+ return { error: `'${param.name}' is a Decimal, but '${text}' is not.` };
134
+ }
135
+ // NUMERIC travels as a string, for the same reason: an exact decimal
136
+ // routed through a JSON number stops being exact.
137
+ return { value: text, code };
138
+ case 'Boolean':
139
+ if (!/^(true|false)$/i.test(text)) {
140
+ return { error: `'${param.name}' is a Boolean, but '${text}' is not.` };
141
+ }
142
+ return { value: /^true$/i.test(text), code };
143
+ case 'Date':
144
+ // Spanner reads a DATE as YYYY-MM-DD and nothing else. '03/04/2026' is
145
+ // the fourth of March to one reader and the third of April to another,
146
+ // so the shape is checked -- and then the calendar, because a shape is
147
+ // not a day.
148
+ if (!isCalendarDay(text)) {
149
+ return {
150
+ error: `'${param.name}' is a Date, but '${text}' is not one. Dates are written YYYY-MM-DD.`,
151
+ };
152
+ }
153
+ return { value: text, code };
154
+ case 'DateTime':
155
+ case 'DateTimeTz':
156
+ if (!RFC3339_TIMESTAMP.test(text) || !isCalendarDay(text.slice(0, 10))) {
157
+ return {
158
+ error: `'${param.name}' is a ${type}, but '${text}' is not a timestamp. Timestamps are written like ` +
159
+ `2026-03-04T10:00:00Z, with the zone.`,
160
+ };
161
+ }
162
+ return { value: text, code };
163
+ default:
164
+ // `raw`, not `text`. The trim above exists to parse a number or a date
165
+ // off a command line; a String parameter is not parsed, it IS the value.
166
+ // Trimming here would store `see ticket` for `--arg memo=" see ticket "`
167
+ // -- the caller's text altered on the way to the store, by a rule
168
+ // nothing states.
169
+ return { value: `${raw}`, code };
170
+ }
171
+ }
@@ -0,0 +1,10 @@
1
+ import { CatalogClient } from '../gcp/dataplex';
2
+ import { ReadResult } from './kc_converter';
3
+ export interface KcPullOptions {
4
+ project: string;
5
+ location: string;
6
+ entryGroup: string;
7
+ v2Aspects?: boolean;
8
+ systemTypeProject?: string;
9
+ }
10
+ export declare function pullKnowledgeCatalog(cat: CatalogClient, opts: KcPullOptions): Promise<ReadResult>;
@@ -0,0 +1,272 @@
1
+ "use strict";
2
+ // Pull: fetch a semantic model from a live Knowledge Catalog into the IR.
3
+ //
4
+ // The read-direction counterpart of `deployKnowledgeCatalog` (push, in
5
+ // `deploy_knowledge_catalog.ts`). Unlike a write, a pull needs no server-side
6
+ // type provisioning -- only that the `semantic-*` entries exist. It enumerates
7
+ // the entry group, keeps the semantic entries, hydrates each one's aspect data
8
+ // (a BASIC list omits aspect data, so each entry is re-fetched with its aspect
9
+ // types -- an entity needs BOTH its `semantic-entity` aspect and the built-in
10
+ // `schema` aspect), and hands the hydrated entries to the pure reader
11
+ // (`kc_converter.modelsFromCatalogResources`). With the second-generation
12
+ // types, a model an older kcmd pushed with the first-generation ones is left
13
+ // out, and a warning names it.
14
+ //
15
+ // TODO(#278): Layer 2 push/pull symmetry. This is the pull half over the
16
+ // pure `kc_converter` codec; the push half (`deployKnowledgeCatalog`) still
17
+ // lives in `deploy_knowledge_catalog.ts`. Once #278 merges, rename that
18
+ // file to `push_kc.ts` (repointing `src/tool/commands.ts` and its test) so
19
+ // the orchestration layer reads as `push_kc` / `pull_kc`. Rename only -- no
20
+ // logic moves.
21
+ Object.defineProperty(exports, "__esModule", { value: true });
22
+ exports.pullKnowledgeCatalog = pullKnowledgeCatalog;
23
+ const dataplex_1 = require("../gcp/dataplex");
24
+ const kc_actions_1 = require("./kc_actions");
25
+ const kc_constraints_1 = require("./kc_constraints");
26
+ const kc_converter_1 = require("./kc_converter");
27
+ const kc_custom_types_1 = require("./kc_custom_types");
28
+ const kc_ids_1 = require("./kc_ids");
29
+ const sql_parser_1 = require("./sql_parser");
30
+ // Upper bound on in-flight aspect-hydration fetches during a pull.
31
+ const HYDRATE_CONCURRENCY = 8;
32
+ // Reads the semantic model back from a Knowledge Catalog entry group. Emits no
33
+ // console output; soft warnings (from the reader) are returned for the caller
34
+ // to print. Hard failures -- more than one model in the group, a fetch error
35
+ // on any entry or its links, or a model entry still labeled as being written
36
+ // -- throw, so the pull aborts rather than write a partial model.
37
+ async function pullKnowledgeCatalog(cat, opts) {
38
+ await (0, sql_parser_1.loadSqlEngine)();
39
+ const destination = `${opts.project}.${opts.location}.${opts.entryGroup}`;
40
+ // Enumerate the group (paging is inherently sequential) and pick the semantic
41
+ // entries, then hydrate their aspects concurrently: a BASIC list omits aspect
42
+ // data, so each entry needs its own lookupEntry, and those fetches are
43
+ // independent. The pool preserves input order so warnings stay deterministic.
44
+ const targets = [];
45
+ for await (const entry of cat.listEntries(opts.project, opts.location, opts.entryGroup)) {
46
+ const aspectTypes = semanticAspectTypes(entry.entryType, opts.v2Aspects);
47
+ if (aspectTypes)
48
+ targets.push({ entry, aspectTypes });
49
+ // else: not part of a semantic model; ignore it.
50
+ }
51
+ // An entry group holds exactly one semantic model. Zero anchors is a clean
52
+ // "nothing to pull" (the reader returns no models); more than one is an
53
+ // unexpected catalog state we refuse to guess through -- name the anchors and
54
+ // fail so it can be fixed at the source.
55
+ const anchors = targets.filter(t => t.entry.entryType?.endsWith('/entryTypes/semantic-model'));
56
+ if (anchors.length > 1) {
57
+ const ids = anchors.map(a => (0, kc_ids_1.entryIdOf)(a.entry.name)).sort();
58
+ throw new Error(`entry group ${destination} holds ${anchors.length} semantic models (${ids.join(', ')}); expected exactly one. Remove the extra anchor(s) ` +
59
+ `from the catalog and pull again.`);
60
+ }
61
+ // Hydrate every semantic entry. A failed fetch means part of the model would
62
+ // be silently missing, so abort rather than reconstruct an incomplete model.
63
+ const hydrated = await mapConcurrent(targets, HYDRATE_CONCURRENCY, async ({ entry, aspectTypes }) => opts.v2Aspects ?
64
+ fetchEveryAspect(cat, opts, entry, aspectTypes) :
65
+ fetchEntry(cat, opts, entry, aspectTypes));
66
+ // Second fetch pass: with the first-generation types, the schema-join links
67
+ // of the relationships, which listing entries does not return. Links are
68
+ // listed per entity entry and deduplicated, since each comes back from both
69
+ // endpoints, and a failed lookup aborts the pull. With v2Aspects on, pull
70
+ // reads no link.
71
+ const entityEntries = opts.v2Aspects ? [] : hydrated.filter(e => e.entryType?.endsWith('/entryTypes/semantic-entity'));
72
+ const linkLists = await mapConcurrent(entityEntries, HYDRATE_CONCURRENCY, async (entry) => {
73
+ const res = await cat.lookupEntryLinks(opts.project, opts.location, {
74
+ entry: entry.name,
75
+ entryLinkTypes: [new kc_ids_1.Namer(opts).typeName('entryLink', 'schema-join')],
76
+ });
77
+ if (res.status !== 200 || !res.result) {
78
+ throw new Error(`failed to fetch entry links for '${entry.name}' ` +
79
+ `(status ${res.status}); pull aborted`);
80
+ }
81
+ return res.result;
82
+ });
83
+ const seenLinks = new Set();
84
+ const entryLinks = [];
85
+ for (const links of linkLists) {
86
+ for (const link of links) {
87
+ const key = (0, kc_converter_1.linkDedupKey)(link);
88
+ if (seenLinks.has(key))
89
+ continue;
90
+ seenLinks.add(key);
91
+ entryLinks.push(link);
92
+ }
93
+ }
94
+ const read = (0, kc_converter_1.modelsFromCatalogResources)(hydrated, entryLinks, { v2Aspects: opts.v2Aspects });
95
+ // Every push labels the model entry as writing when it writes the model entry
96
+ // and as finished with its last. The hydrated entries keep the order of
97
+ // `targets`.
98
+ const anchor = anchors.length ? hydrated[targets.indexOf(anchors[0])] : undefined;
99
+ const writingReason = anchor?.entrySource?.labels?.[kc_custom_types_1.PUSH_STATE_LABEL] === kc_custom_types_1.PUSH_WRITING ?
100
+ `its model entry's ${kc_custom_types_1.PUSH_STATE_LABEL} label still says ${kc_custom_types_1.PUSH_WRITING}` :
101
+ undefined;
102
+ if (writingReason) {
103
+ // With v2Aspects on, a model an older kcmd pushed is not in `read.models`,
104
+ // so the model is named from its entry, as the reader names it.
105
+ throw notFinished(anchor.entrySource?.displayName ?? (0, kc_ids_1.entryIdOf)(anchor.name), writingReason);
106
+ }
107
+ const warnings = [...read.warnings];
108
+ for (const m of read.firstGeneration ?? []) {
109
+ warnings.push(firstGenerationWarning(opts, m));
110
+ }
111
+ return { ...read, warnings: [...new Set(warnings)] };
112
+ }
113
+ // The error for a model whose last push stopped partway, for `reason`.
114
+ function notFinished(model, reason) {
115
+ return new Error(`the last push of model '${model}' did not finish: ${reason}, so pull ` +
116
+ `writes nothing. Push again, then pull.`);
117
+ }
118
+ // The model, entity and metric aspect types a first-generation push writes, in
119
+ // the order the warning names them.
120
+ const FIRST_GEN_ASPECTS = ['semantic-model', 'semantic-entity', 'semantic-metric'];
121
+ // The warning for a model an older kcmd pushed, which a pull with v2Aspects
122
+ // on leaves out. It names the first-generation aspect types found on the
123
+ // model's entries.
124
+ function firstGenerationWarning(opts, model) {
125
+ const found = new Set(model.entries.flatMap(e => Object.values(e.aspects ?? {})
126
+ .map(a => a.aspectType?.split('/aspectTypes/')[1])));
127
+ const types = FIRST_GEN_ASPECTS.filter(t => found.has(t));
128
+ return `model '${model.name}' in entry group '${opts.entryGroup}' was ` +
129
+ `pushed by an older kcmd (first-generation aspects: ${types.join(', ')}). Pull wrote nothing for it; push it again ` +
130
+ `with the current kcmd`;
131
+ }
132
+ // Fetches `entry` with the aspects of `aspectTypes`, at `paths` when given.
133
+ // Throws on a failed fetch.
134
+ async function fetchEntry(cat, opts, entry, aspectTypes, paths) {
135
+ const res = await cat.lookupEntry(opts.project, opts.location, entry.name, { aspects: aspectTypes, paths });
136
+ if (res.status !== 200 || !res.result) {
137
+ throw new Error(`failed to fetch entry '${entry.name}' (status ${res.status}); pull aborted`);
138
+ }
139
+ return res.result;
140
+ }
141
+ // Most aspects Dataplex returns for one entry in one read.
142
+ const MAX_ASPECTS_PER_READ = 100;
143
+ // Most bytes of path and query Dataplex accepts in a GET.
144
+ const MAX_URL_BYTES = 16_384;
145
+ // Fetches `entry` with every aspect of `aspectTypes`, one read at a time. A
146
+ // read that returns MAX_ASPECTS_PER_READ is repeated once per type, and a type
147
+ // that returns that many is read again at the entry and at each field.
148
+ async function fetchEveryAspect(cat, opts, entry, aspectTypes) {
149
+ const full = (e) => Object.keys(e.aspects ?? {}).length >= MAX_ASPECTS_PER_READ;
150
+ const read = await fetchEntry(cat, opts, entry, aspectTypes);
151
+ if (!full(read))
152
+ return read;
153
+ const byType = [];
154
+ for (const t of aspectTypes) {
155
+ byType.push(await fetchEntry(cat, opts, entry, [t]));
156
+ }
157
+ const schema = byType[aspectTypes.findIndex(t => t.endsWith('/aspectTypes/schema'))];
158
+ const paths = [
159
+ '',
160
+ ...Object.values(schema?.aspects ?? {})
161
+ .flatMap(a => a.data?.fields ?? [])
162
+ .map(f => `Schema.${f.name}`)
163
+ ];
164
+ const byPath = [];
165
+ for (const t of aspectTypes.filter((_, i) => full(byType[i]))) {
166
+ for (const batch of pathBatches(opts, entry, t, paths)) {
167
+ byPath.push(await fetchEntry(cat, opts, entry, [t], batch));
168
+ }
169
+ }
170
+ const reads = [...byType, ...byPath];
171
+ return { ...read, aspects: Object.assign({}, ...reads.map(r => r.aspects)) };
172
+ }
173
+ // Splits `paths` into batches for reads of `type` on `entry`, each within
174
+ // MAX_URL_BYTES. A batch holds fewer than MAX_ASPECTS_PER_READ paths, and a
175
+ // path holds at most one instance of a type, so no read comes back cut.
176
+ function pathBatches(opts, entry, type, paths) {
177
+ // Bytes of `params` in the URL, with the '?' or '&' before them.
178
+ const bytes = (params) => new URLSearchParams(params).toString().length + 1;
179
+ const base = `/v1/${(0, dataplex_1.catalogContainer)(opts.project, opts.location)}:lookupEntry`
180
+ .length +
181
+ bytes([['entry', entry.name], ['view', 'CUSTOM'], ['aspectTypes', type]]);
182
+ const batches = [];
183
+ let batch = [];
184
+ let size = base;
185
+ for (const path of paths) {
186
+ const add = bytes([['paths', path]]);
187
+ if (batch.length === MAX_ASPECTS_PER_READ - 1 ||
188
+ size + add > MAX_URL_BYTES) {
189
+ batches.push(batch);
190
+ batch = [];
191
+ size = base;
192
+ }
193
+ batch.push(path);
194
+ size += add;
195
+ }
196
+ return [...batches, batch];
197
+ }
198
+ // The aspect types each second-generation entry type is read with.
199
+ const OBJECT_ASPECTS = ['guidelines', 'semantic-extensions', 'overview'];
200
+ const V2_ASPECT_TYPES = new Map([
201
+ ['semantic-model', ['semantic-model', ...OBJECT_ASPECTS]],
202
+ [
203
+ 'semantic-entity',
204
+ ['semantic-entity', 'schema', 'sql-expressions', ...OBJECT_ASPECTS]
205
+ ],
206
+ [
207
+ 'semantic-metric', ['semantic-metric', 'sql-expressions', ...OBJECT_ASPECTS]
208
+ ],
209
+ ['semantic-relationship', ['semantic-relationship', ...OBJECT_ASPECTS]],
210
+ ]);
211
+ // The aspect types to read for a semantic entry, in its entry type's project
212
+ // and location: V2_ASPECT_TYPES with `v2Aspects` on, the first-generation set
213
+ // otherwise. Undefined for an entry outside a semantic model.
214
+ function semanticAspectTypes(entryType, v2Aspects) {
215
+ const marker = '/entryTypes/';
216
+ const idx = entryType?.indexOf(marker) ?? -1;
217
+ if (idx < 0)
218
+ return undefined;
219
+ const typeBase = entryType.slice(0, idx);
220
+ const t = entryType.slice(idx + marker.length);
221
+ const aspectType = (name) => `${typeBase}/aspectTypes/${name}`;
222
+ const v2Types = v2Aspects ? V2_ASPECT_TYPES.get(t) : undefined;
223
+ if (v2Types)
224
+ return v2Types.map(aspectType);
225
+ switch (t) {
226
+ case 'semantic-model':
227
+ return [aspectType('semantic-model'), aspectType('guidelines')];
228
+ case 'semantic-entity':
229
+ return [
230
+ aspectType('semantic-entity'), aspectType('schema'),
231
+ aspectType('guidelines')
232
+ ];
233
+ case 'semantic-metric':
234
+ return [aspectType('semantic-metric'), aspectType('guidelines')];
235
+ case kc_custom_types_1.ACTION_TYPE_ID:
236
+ // An action's entry type is custom (see kc_custom_types.ts), so it
237
+ // lives in the destination project rather than under `dataplex-types`;
238
+ // `typeBase` already points there, and kc_actions.ts names the aspects
239
+ // to fetch beneath it.
240
+ return (0, kc_actions_1.actionAspectTypes)(typeBase);
241
+ case kc_custom_types_1.CONSTRAINT_TYPE_ID:
242
+ // Custom for the same reason, and named through the same `typeBase`.
243
+ return (0, kc_constraints_1.constraintAspectTypes)(typeBase);
244
+ default:
245
+ return undefined;
246
+ }
247
+ }
248
+ // Maps `items` through `fn` with at most `limit` calls in flight, returning
249
+ // results in input order (so downstream ordering stays deterministic).
250
+ async function mapConcurrent(items, limit, fn) {
251
+ const results = new Array(items.length);
252
+ let next = 0;
253
+ let failed = false;
254
+ async function worker() {
255
+ // Stop claiming new items once any worker has thrown: Promise.all rejects on
256
+ // the first failure, so fanning out more fetches is wasted work whose own
257
+ // rejections would surface as unhandled-rejection noise.
258
+ while (next < items.length && !failed) {
259
+ const i = next++;
260
+ try {
261
+ results[i] = await fn(items[i]);
262
+ }
263
+ catch (err) {
264
+ failed = true;
265
+ throw err;
266
+ }
267
+ }
268
+ }
269
+ const workers = Array.from({ length: Math.min(limit, items.length) }, () => worker());
270
+ await Promise.all(workers);
271
+ return results;
272
+ }
@@ -0,0 +1,59 @@
1
+ import { Field, SemanticModel } from './ir';
2
+ /**
3
+ * Why inheritance cannot be resolved. `unknown-parent` can be a pruning
4
+ * artefact, since a profile can drop a supertype whole; a cycle or an
5
+ * ambiguously inherited field cannot, because pruning only removes things.
6
+ */
7
+ export declare class InheritanceError extends Error {
8
+ readonly kind: 'unknown-parent' | 'cycle' | 'ambiguous-field';
9
+ constructor(kind: 'unknown-parent' | 'cycle' | 'ambiguous-field', message: string);
10
+ }
11
+ export interface ResolveResult {
12
+ model: SemanticModel;
13
+ warnings: string[];
14
+ }
15
+ export interface ResolveOptions {
16
+ requalify?: boolean;
17
+ }
18
+ /**
19
+ * What a `concept` name may refer to, and the fields it declares.
20
+ *
21
+ * One lookup serves every construct that names a concept -- an action's
22
+ * `affects`, and a derived action parameter's `concept`/`field` pair -- so the
23
+ * two can never disagree about what `Order` is or which fields it has.
24
+ *
25
+ * Entities are indexed FIRST, so a name that is both an entity and a
26
+ * relationship resolves to the entity. `kind` exists to say `entity 'X'` or
27
+ * `relationship 'X'` in a message, and to pick which fields count.
28
+ *
29
+ * Only a many-to-many relationship has fields of its own: they live on the
30
+ * junction table backing it (see Association), and an enrollment's grade is as
31
+ * ordinary a field as an order's total. A plain foreign-key edge declares none.
32
+ *
33
+ * Inheritance is resolved, because a subtype's own `fields` omit everything it
34
+ * inherits. Resolution THROWS on an `extends` naming an entity the model does
35
+ * not declare, on a cycle, and on a field inherited ambiguously; a caller that
36
+ * cannot report that -- the loader, which is lenient by design -- catches it
37
+ * and works from the unresolved model instead.
38
+ */
39
+ export interface DeclaredConcept {
40
+ kind: 'entity' | 'relationship';
41
+ fields: Map<string, Field>;
42
+ }
43
+ export declare function declaredConceptFields(model: SemanticModel): Map<string, DeclaredConcept>;
44
+ /**
45
+ * Returns a clone of `model` with every entity's `extends` expanded to its full
46
+ * transitive ancestor set and its `fields` flattened to include inherited
47
+ * fields, less any a profile excludes on that entity. The input is never
48
+ * mutated. An entity with no `extends` keeps its own fields, so a model with no
49
+ * inheritance and no exclusions resolves to an equivalent model.
50
+ * `targetDialect` is the SQL that an inherited field's target text and its
51
+ * ANSI_SQL entries are read in to find their comments. It is BigQuery unless
52
+ * given, as the graph generators read the target text. With
53
+ * `options.requalify`, an inherited expression keeps naming the fields it
54
+ * names on its ancestor, under the inheriting entity's qualifier.
55
+ *
56
+ * Reading an inherited field's SQL needs the SQL parser, so call
57
+ * `loadSqlEngine()` first. Without it, this throws the parser's plain Error.
58
+ */
59
+ export declare function resolveInheritance(model: SemanticModel, targetDialect?: string, options?: ResolveOptions): ResolveResult;