@google-cloud/kcmd 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -20
- package/build/ts/kcmd/cli.js +5842 -5168
- package/build/ts/kcmd/gcp/biglake.d.ts +2 -0
- package/build/ts/kcmd/gcp/biglake.js +13 -4
- package/build/ts/kcmd/gcp/spanner.d.ts +10 -0
- package/build/ts/kcmd/gcp/spanner.js +6 -3
- package/build/ts/kcmd/layouts/okf.js +49 -7
- package/build/ts/kcmd/layouts/semantic-model.d.ts +13 -0
- package/build/ts/kcmd/layouts/semantic-model.js +129 -50
- package/build/ts/kcmd/okf/concept.d.ts +1 -1
- package/build/ts/kcmd/okf/concept.js +6 -0
- package/build/ts/kcmd/okf/concept_to_entry.d.ts +2 -0
- package/build/ts/kcmd/okf/concept_to_entry.js +29 -2
- package/build/ts/kcmd/okf/init_okf_workspace.js +8 -10
- package/build/ts/kcmd/okf/okf_metadata.js +17 -0
- package/build/ts/kcmd/semantic/bigquery.js +139 -195
- package/build/ts/kcmd/semantic/binding.js +14 -19
- package/build/ts/kcmd/semantic/deploy_knowledge_catalog.d.ts +10 -35
- package/build/ts/kcmd/semantic/deploy_knowledge_catalog.js +92 -121
- package/build/ts/kcmd/semantic/graph_ddl.d.ts +10 -0
- package/build/ts/kcmd/semantic/graph_ddl.js +152 -59
- package/build/ts/kcmd/semantic/import_model.d.ts +26 -4
- package/build/ts/kcmd/semantic/import_model.js +526 -103
- package/build/ts/kcmd/semantic/kc_converter.d.ts +7 -1
- package/build/ts/kcmd/semantic/kc_converter.js +20 -5
- package/build/ts/kcmd/semantic/kc_custom_types.d.ts +5 -12
- package/build/ts/kcmd/semantic/kc_custom_types.js +16 -93
- package/build/ts/kcmd/semantic/kc_ids.d.ts +4 -5
- package/build/ts/kcmd/semantic/kc_ids.js +4 -7
- package/build/ts/kcmd/semantic/knowledge_catalog.d.ts +0 -3
- package/build/ts/kcmd/semantic/knowledge_catalog.js +3 -86
- package/build/ts/kcmd/semantic/loader.d.ts +7 -11
- package/build/ts/kcmd/semantic/loader.js +284 -148
- package/build/ts/kcmd/semantic/osi_converter.js +1 -2
- package/build/ts/kcmd/semantic/parameters.d.ts +1 -0
- package/build/ts/kcmd/semantic/parameters.js +22 -3
- package/build/ts/kcmd/semantic/pull_kc.d.ts +1 -5
- package/build/ts/kcmd/semantic/pull_kc.js +38 -112
- package/build/ts/kcmd/semantic/resolve_inheritance.d.ts +7 -2
- package/build/ts/kcmd/semantic/resolve_inheritance.js +37 -21
- package/build/ts/kcmd/semantic/resolve_profiles.d.ts +51 -10
- package/build/ts/kcmd/semantic/resolve_profiles.js +213 -32
- package/build/ts/kcmd/semantic/runtime/runtime.d.ts +0 -6
- package/build/ts/kcmd/semantic/runtime/runtime.js +4 -21
- package/build/ts/kcmd/semantic/source_dialect.d.ts +9 -2
- package/build/ts/kcmd/semantic/source_dialect.js +83 -33
- package/build/ts/kcmd/semantic/source_entries.d.ts +1 -1
- package/build/ts/kcmd/semantic/source_entries.js +95 -36
- package/build/ts/kcmd/semantic/sql_expr_utils.d.ts +13 -7
- package/build/ts/kcmd/semantic/sql_expr_utils.js +240 -67
- package/build/ts/kcmd/semantic/sql_parser.d.ts +5 -2
- package/build/ts/kcmd/semantic/sql_parser.js +22 -0
- package/build/ts/kcmd/semantic/transpile.d.ts +12 -10
- package/build/ts/kcmd/semantic/transpile.js +79 -61
- package/build/ts/kcmd/semantic/validate.d.ts +37 -2
- package/build/ts/kcmd/semantic/validate.js +465 -98
- package/build/ts/kcmd/snapshot.d.ts +4 -0
- package/build/ts/kcmd/snapshot.js +7 -0
- package/build/ts/kcmd/sync.js +44 -7
- package/package.json +1 -1
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
// The dialect of a Cloud SQL table written as its resource name, or of a
|
|
3
3
|
// third-party table written as its entry name, read from the table's
|
|
4
|
-
// Knowledge Catalog entry.
|
|
4
|
+
// Knowledge Catalog entry, and the dialect of a Spanner table's database.
|
|
5
5
|
//
|
|
6
6
|
// The resource name cannot say the dialect: a MySQL table has no schema, but a
|
|
7
7
|
// PostgreSQL table and a SQL Server table have the same shape. Dataplex keeps an
|
|
@@ -16,9 +16,12 @@
|
|
|
16
16
|
// fullyQualifiedName, so its parts say whether the entry is a table's or, for
|
|
17
17
|
// example, a schema's.
|
|
18
18
|
//
|
|
19
|
-
//
|
|
20
|
-
//
|
|
21
|
-
//
|
|
19
|
+
// Knowledge Catalog's Spanner entries leave their dialect empty, so a Spanner
|
|
20
|
+
// table's dialect is its database's `databaseDialect`, read by databases.get.
|
|
21
|
+
//
|
|
22
|
+
// This is the only place kcmd looks up a source's dialect. A command looks
|
|
23
|
+
// every such source up once, before it loads the model, and passes the
|
|
24
|
+
// dialects to the load (LoadOptions.sourceDialects).
|
|
22
25
|
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
23
26
|
if (k2 === undefined) k2 = k;
|
|
24
27
|
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
@@ -54,10 +57,12 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
54
57
|
})();
|
|
55
58
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
56
59
|
exports.sourceDialects = sourceDialects;
|
|
60
|
+
exports.tableFqnParts = tableFqnParts;
|
|
57
61
|
exports.sourcesToLookUp = sourcesToLookUp;
|
|
58
62
|
const yaml = __importStar(require("yaml"));
|
|
59
63
|
const loader_1 = require("./loader");
|
|
60
64
|
const catalog_entry_1 = require("./catalog_entry");
|
|
65
|
+
const parameters_1 = require("./parameters");
|
|
61
66
|
const yaml_options_1 = require("./yaml_options");
|
|
62
67
|
// The dialect each entrySource.system gives, in lower case.
|
|
63
68
|
const DIALECT_OF_ENTRY_SYSTEM = new Map([...loader_1.DIALECT_OF_FQN_PREFIX, ['postgres', 'POSTGRES']]);
|
|
@@ -70,14 +75,16 @@ const TABLE_FQN_PARTS = new Map([
|
|
|
70
75
|
['postgresql', ['host', 'database', 'schema', 'table']],
|
|
71
76
|
['sqlserver', ['host', 'database', 'schema', 'table']],
|
|
72
77
|
]);
|
|
73
|
-
// Looks up the dialect of every Cloud SQL and
|
|
74
|
-
//
|
|
75
|
-
//
|
|
76
|
-
//
|
|
77
|
-
// `
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
//
|
|
78
|
+
// Looks up, once each and in parallel, the dialect of every Cloud SQL and
|
|
79
|
+
// third-party table the files `texts` write (see sourcesToLookUp), and of every
|
|
80
|
+
// Spanner database they name. Without `spanner`, a Spanner database gives
|
|
81
|
+
// SPANNER, the dialect kcmd's Spanner store speaks, with no call.
|
|
82
|
+
// `thirdParty: false` skips third-party tables. Never throws: a failed lookup
|
|
83
|
+
// goes to `errors`.
|
|
84
|
+
async function sourceDialects(texts, catalog, { thirdParty = true, spanner } = {}) {
|
|
85
|
+
// Keyed as dialectLookupKey gives, so both spellings of one table, and the
|
|
86
|
+
// tables of one Spanner database, are one lookup. A database's lookup names
|
|
87
|
+
// the first of its tables.
|
|
81
88
|
const lookups = new Map();
|
|
82
89
|
for (const source of sourcesToLookUp(texts)) {
|
|
83
90
|
const table = (0, loader_1.cloudSqlTableOf)(source);
|
|
@@ -87,32 +94,42 @@ async function sourceDialects(texts, catalog, { thirdParty = true } = {}) {
|
|
|
87
94
|
const entry = thirdParty ? (0, loader_1.thirdPartyEntryOf)(source) : undefined;
|
|
88
95
|
if (entry)
|
|
89
96
|
lookups.set(entry.name, () => thirdPartyDialect(catalog, entry));
|
|
97
|
+
const database = (0, loader_1.spannerDatabaseOf)(source);
|
|
98
|
+
if (database && !lookups.has(database)) {
|
|
99
|
+
lookups.set(database, spanner ? () => spannerDialect(spanner, database, source) :
|
|
100
|
+
async () => ({ dialect: 'SPANNER' }));
|
|
101
|
+
}
|
|
90
102
|
}
|
|
91
103
|
const dialects = new Map();
|
|
92
104
|
const databases = new Map();
|
|
93
105
|
const errors = [];
|
|
106
|
+
const unread = [];
|
|
94
107
|
await Promise.all([...lookups].map(async ([key, lookUp]) => {
|
|
95
108
|
const got = await lookUp();
|
|
96
109
|
if ('error' in got)
|
|
97
110
|
errors.push(got.error);
|
|
98
111
|
else
|
|
99
112
|
dialects.set(key, got.dialect);
|
|
113
|
+
if ('unread' in got)
|
|
114
|
+
unread.push(got.error);
|
|
100
115
|
if ('database' in got && got.database)
|
|
101
116
|
databases.set(key, got.database);
|
|
102
117
|
}));
|
|
103
118
|
errors.sort();
|
|
104
|
-
|
|
119
|
+
unread.sort();
|
|
120
|
+
return { dialects, databases, errors, unread };
|
|
105
121
|
}
|
|
106
122
|
// The dialect of a Cloud SQL table, by its entry's type.
|
|
107
123
|
async function cloudSqlDialect(catalog, table) {
|
|
108
124
|
const at = `source '${table.resourceName}'`;
|
|
109
125
|
const res = await (0, catalog_entry_1.lookupSystemEntry)(catalog, 'cloudsql', table.project, table.location, table.resourceName);
|
|
110
126
|
if ('error' in res)
|
|
111
|
-
return { error: `${at}: ${res.error}
|
|
127
|
+
return { error: `${at}: ${res.error}`, unread: true };
|
|
112
128
|
// Dataplex answers 403 both when the entry does not exist and when the
|
|
113
129
|
// caller may not read it, so the message names both causes.
|
|
114
130
|
if ('unreadable' in res) {
|
|
115
131
|
return {
|
|
132
|
+
unread: true,
|
|
116
133
|
error: `${at}: kcmd reads a Cloud SQL table's dialect from its ` +
|
|
117
134
|
`Knowledge Catalog entry, and could not read it. Turn on Dataplex ` +
|
|
118
135
|
`integration for instance '${table.instance}' (gcloud sql ` +
|
|
@@ -131,6 +148,27 @@ async function cloudSqlDialect(catalog, table) {
|
|
|
131
148
|
`not a Cloud SQL table or view type kcmd knows (${Object.keys(loader_1.DIALECT_OF_ENTRY_TYPE).join(', ')}).`,
|
|
132
149
|
};
|
|
133
150
|
}
|
|
151
|
+
// The dialect of the Spanner database `database`, which `source` names a
|
|
152
|
+
// table of, by its databaseDialect.
|
|
153
|
+
async function spannerDialect(spanner, database, source) {
|
|
154
|
+
const [, , project, , instance, , name] = database.split('/');
|
|
155
|
+
const failed = (why) => ({
|
|
156
|
+
unread: true,
|
|
157
|
+
error: `source '${source}': kcmd reads its Spanner database's SQL ` +
|
|
158
|
+
`dialect, and could not read database ${project}/${instance}/${name}: ${why}`,
|
|
159
|
+
});
|
|
160
|
+
try {
|
|
161
|
+
const res = await spanner.getDatabase(project, instance, name);
|
|
162
|
+
if (res.status !== 200) {
|
|
163
|
+
return failed((0, parameters_1.storeMessage)(res.message, res.status));
|
|
164
|
+
}
|
|
165
|
+
const postgres = res.result?.databaseDialect === 'POSTGRESQL';
|
|
166
|
+
return { dialect: postgres ? 'POSTGRES' : 'SPANNER' };
|
|
167
|
+
}
|
|
168
|
+
catch (err) {
|
|
169
|
+
return failed(err?.message ?? String(err));
|
|
170
|
+
}
|
|
171
|
+
}
|
|
134
172
|
// The dialect of a third-party table, by its entry's fullyQualifiedName or
|
|
135
173
|
// entrySource.system, and the database the fullyQualifiedName names. An entry
|
|
136
174
|
// whose fullyQualifiedName names no table or view is rejected.
|
|
@@ -138,9 +176,10 @@ async function thirdPartyDialect(catalog, { project, location, name }) {
|
|
|
138
176
|
const at = `source '${name}'`;
|
|
139
177
|
const res = await (0, catalog_entry_1.read)('looking up its Knowledge Catalog entry', () => catalog.lookupEntry(project, location, name));
|
|
140
178
|
if ('error' in res)
|
|
141
|
-
return { error: `${at}: ${res.error}
|
|
179
|
+
return { error: `${at}: ${res.error}`, unread: true };
|
|
142
180
|
if ('unreadable' in res) {
|
|
143
181
|
return {
|
|
182
|
+
unread: true,
|
|
144
183
|
error: `${at} names no table Knowledge Catalog knows about. The entry ` +
|
|
145
184
|
`may not exist, or you may not be allowed to read it. kcmd reads a ` +
|
|
146
185
|
`third-party table's dialect from its entry, so add the table to ` +
|
|
@@ -177,14 +216,12 @@ async function thirdPartyDialect(catalog, { project, location, name }) {
|
|
|
177
216
|
`fullyQualifiedName.`,
|
|
178
217
|
};
|
|
179
218
|
}
|
|
180
|
-
// The
|
|
181
|
-
//
|
|
182
|
-
//
|
|
183
|
-
//
|
|
184
|
-
//
|
|
185
|
-
|
|
186
|
-
// `databricks:view:` name as the same table as the `databricks:` name.
|
|
187
|
-
function databaseOfFqn(fqn) {
|
|
219
|
+
// The prefix of the fullyQualifiedName `fqn` of a third-party table or view,
|
|
220
|
+
// and its unquoted parts, such as `snowflake` and ['acme', 'retail', 'public',
|
|
221
|
+
// 'orders'] for `snowflake:acme.retail.public.orders`. Undefined when `fqn`
|
|
222
|
+
// has another number of parts than TABLE_FQN_PARTS gives, or a subtype other
|
|
223
|
+
// than Databricks' `table:` and `view:`.
|
|
224
|
+
function tableFqnParts(fqn) {
|
|
188
225
|
// The subtype holds no backtick, so a colon inside a quoted host and port
|
|
189
226
|
// stays in the path. A colon in an unquoted database, schema or table
|
|
190
227
|
// name stays in its segment when it comes after the name's first backtick.
|
|
@@ -195,19 +232,31 @@ function databaseOfFqn(fqn) {
|
|
|
195
232
|
(prefix === 'databricks' && (subtype === 'table' || subtype === 'view'));
|
|
196
233
|
if (!parts || !tableSubtype || seg.length !== parts.length)
|
|
197
234
|
return undefined;
|
|
235
|
+
return { prefix, parts: seg };
|
|
236
|
+
}
|
|
237
|
+
// The database the fullyQualifiedName `fqn` of a third-party table or view
|
|
238
|
+
// names: a Snowflake account, a MySQL or SQL Server host, a PostgreSQL host
|
|
239
|
+
// and database, or a Databricks metastore. Each account and host is in lower
|
|
240
|
+
// case. Undefined when `fqn` names no table or view.
|
|
241
|
+
function databaseOfFqn(fqn) {
|
|
242
|
+
const table = tableFqnParts(fqn);
|
|
243
|
+
if (!table)
|
|
244
|
+
return undefined;
|
|
245
|
+
const { prefix, parts } = table;
|
|
198
246
|
const n = prefix === 'postgresql' ? 2 : 1;
|
|
199
247
|
// A Snowflake account and a host name ignore letter case.
|
|
200
248
|
if (prefix !== 'databricks')
|
|
201
|
-
|
|
202
|
-
return [prefix, ...
|
|
249
|
+
parts[0] = parts[0].toLowerCase();
|
|
250
|
+
return [prefix, ...parts.slice(0, n)].join('/');
|
|
203
251
|
}
|
|
204
|
-
// Every Cloud SQL table and third-party entry the model and
|
|
205
|
-
// `texts` write. A Cloud SQL table counts as a resource name or
|
|
206
|
-
// anywhere, JSON strings included.
|
|
207
|
-
// the YAML or in a GOOGLE block's JSON
|
|
208
|
-
// an action parameter's default or
|
|
209
|
-
// from the parsed YAML, so a name in a
|
|
210
|
-
// does not parse collects nothing; the
|
|
252
|
+
// Every Cloud SQL table, Spanner table and third-party entry the model and
|
|
253
|
+
// profile files `texts` write. A Cloud SQL table counts as a resource name or
|
|
254
|
+
// resource URI anywhere, JSON strings included. A Spanner table or an entry
|
|
255
|
+
// name counts as a `source` value in the YAML or in a GOOGLE block's JSON
|
|
256
|
+
// `data`, unless it sits in ai_context, an action parameter's default or
|
|
257
|
+
// another vendor's block. Values are read from the parsed YAML, so a name in a
|
|
258
|
+
// comment is not collected. A file that does not parse collects nothing; the
|
|
259
|
+
// load reports it.
|
|
211
260
|
function sourcesToLookUp(texts) {
|
|
212
261
|
const found = new Set();
|
|
213
262
|
// `key` names the member that holds `value`. `opaque` marks a value whose
|
|
@@ -217,7 +266,8 @@ function sourcesToLookUp(texts) {
|
|
|
217
266
|
const visit = (value, key, opaque = false) => {
|
|
218
267
|
if (typeof value === 'string') {
|
|
219
268
|
if ((0, loader_1.cloudSqlTableOf)(value) ||
|
|
220
|
-
(key === 'source' && !opaque &&
|
|
269
|
+
(key === 'source' && !opaque &&
|
|
270
|
+
((0, loader_1.thirdPartyEntryOf)(value) || (0, loader_1.spannerDatabaseOf)(value)))) {
|
|
221
271
|
found.add(value.trim());
|
|
222
272
|
}
|
|
223
273
|
else if (/^\s*[[{]/.test(value)) {
|
|
@@ -6,7 +6,7 @@ import type { SemanticModel } from './ir';
|
|
|
6
6
|
export interface SourceReaders {
|
|
7
7
|
catalog: EntryLookup;
|
|
8
8
|
bigquery: Pick<BigQueryClient, 'getTable'>;
|
|
9
|
-
biglake: Pick<BigLakeClient, 'getCatalog'>;
|
|
9
|
+
biglake: Pick<BigLakeClient, 'getCatalog' | 'getHiveCatalog'>;
|
|
10
10
|
spanner: Pick<SpannerClient, 'getInstance' | 'getInstanceConfig'>;
|
|
11
11
|
}
|
|
12
12
|
export declare function sourceEntryErrors(model: Pick<SemanticModel, 'name' | 'entities' | 'profiles'>, readers: SourceReaders): Promise<string[]>;
|
|
@@ -16,12 +16,19 @@
|
|
|
16
16
|
// table was created, so after a failover an older table's entry can sit in
|
|
17
17
|
// a former primary's region, now a secondary's. The entry is looked up in
|
|
18
18
|
// each replica's region, the primary's first, until one finds it.
|
|
19
|
+
// getHiveCatalog reads a Hive catalog. Its tables are written with
|
|
20
|
+
// `databases/`.
|
|
19
21
|
// - Spanner: the default leader region of the instance's config, which
|
|
20
22
|
// getInstance reads the config's name from. A regional config is named for
|
|
21
23
|
// its region, `regional-us-central1` for us-central1, so it is not read;
|
|
22
24
|
// any other config is, with getInstanceConfig: `nam3` gives us-east4. That
|
|
23
25
|
// read needs spanner.instanceConfigs.get, which the Spanner database roles
|
|
24
|
-
// lack.
|
|
26
|
+
// lack. When it is refused, the entry is looked up in each region the
|
|
27
|
+
// instance has replicas in, which getInstance lists, in the order listed,
|
|
28
|
+
// until one finds it: the default leader's region is one of them, and the
|
|
29
|
+
// list marks no leader (live, 2026-11-04). The instance's
|
|
30
|
+
// `resourceLocation`, such as `us`, is not a Dataplex region. A database's
|
|
31
|
+
// own default leader does not move the entry.
|
|
25
32
|
// - AlloyDB: the location the source names.
|
|
26
33
|
//
|
|
27
34
|
// Dataplex keeps locations in lower case. A lookup in the wrong location
|
|
@@ -34,8 +41,10 @@
|
|
|
34
41
|
// the entry was never looked up, so the message says which read it was, and
|
|
35
42
|
// what the service said: Spanner answers 403 for a project the caller may not
|
|
36
43
|
// read, whether or not the instance exists, so a 403 does not say the table
|
|
37
|
-
// is missing. A
|
|
38
|
-
//
|
|
44
|
+
// is missing. A refused Spanner instance config read is the exception: the
|
|
45
|
+
// entry is then looked up in each replica region the instance lists. A read
|
|
46
|
+
// that fails another way says to try again when a retry may pass (a thrown
|
|
47
|
+
// call, a 429 or a 5xx), and to fix the source otherwise.
|
|
39
48
|
//
|
|
40
49
|
// A Cloud SQL or third-party source is not looked up here: sourceDialects
|
|
41
50
|
// already looks up its entry directly, and fails the push when it is missing.
|
|
@@ -65,10 +74,11 @@ const resolve_profiles_1 = require("./resolve_profiles");
|
|
|
65
74
|
// - A job with no location runs in the primary.
|
|
66
75
|
//
|
|
67
76
|
// Reads each table's location and looks its entry up once per location it
|
|
68
|
-
// may sit in, and reads each
|
|
69
|
-
// parallel, and a table's locations in
|
|
70
|
-
// entity with no source, and a source that
|
|
71
|
-
// (a Cloud SQL or third-party table
|
|
77
|
+
// may sit in, and reads each instance once and each catalog at most once
|
|
78
|
+
// through each API; tables are checked in parallel, and a table's locations in
|
|
79
|
+
// turn. Skips an abstract entity, an entity with no source, and a source that
|
|
80
|
+
// names no table this module knows (a Cloud SQL or third-party table). Never
|
|
81
|
+
// throws.
|
|
72
82
|
async function sourceEntryErrors(model, readers) {
|
|
73
83
|
const uses = [];
|
|
74
84
|
const use = (entity, profile, source) => {
|
|
@@ -149,24 +159,40 @@ async function problemOf(table, located, readers) {
|
|
|
149
159
|
`exist, or you may not be allowed to read its catalog entry.${table.system === 'alloydb' ?
|
|
150
160
|
' A new AlloyDB cluster can take hours to appear in the ' +
|
|
151
161
|
'catalog after Dataplex integration is turned on.' :
|
|
152
|
-
''} Fix the source, or pass
|
|
162
|
+
''}${locations.unfound ?? ''} Fix the source, or pass ` +
|
|
163
|
+
`--skip-source-check.`;
|
|
153
164
|
}
|
|
154
165
|
// `location`, which is kept in lower case, as BigQuery writes it: a
|
|
155
166
|
// multi-region, which has no hyphen, in upper case (`US`, `EU`).
|
|
156
167
|
function shown(location) {
|
|
157
168
|
return location.includes('-') ? location : location.toUpperCase();
|
|
158
169
|
}
|
|
159
|
-
// Said when reading an instance config is refused
|
|
160
|
-
// bigquery-public-data and
|
|
161
|
-
//
|
|
170
|
+
// Said when reading an instance config is refused and the instance lists no
|
|
171
|
+
// replica regions to look in instead (live on bigquery-public-data and
|
|
172
|
+
// dataplex-staging-types, 2026-10-27: Spanner answers 403 naming this
|
|
173
|
+
// permission, even for a Google-managed config such as nam3).
|
|
162
174
|
const CONFIG_REFUSED_HINT = ' Reading it needs the permission ' +
|
|
163
175
|
'spanner.instanceConfigs.get on the project, which roles/spanner.viewer ' +
|
|
164
176
|
'includes and the Spanner database roles do not.';
|
|
177
|
+
// Said when reading an instance is refused: a Spanner database role gives
|
|
178
|
+
// spanner.instances.get only when granted on the instance or its project
|
|
179
|
+
// (from review on #630, 2026-11-04).
|
|
180
|
+
const INSTANCE_REFUSED_HINT = ' The Spanner database roles include ' +
|
|
181
|
+
'spanner.instances.get, but a role granted on one database does not ' +
|
|
182
|
+
'apply to its instance.';
|
|
183
|
+
// Said when a BigLake catalog read fails in a way a retry cannot fix and the
|
|
184
|
+
// other kind of catalog's API finds the catalog. The status alone does not
|
|
185
|
+
// tell: live, the Hive API answers 400 for an Iceberg REST catalog id with a
|
|
186
|
+
// hyphen and for a Hive catalog id in capitals.
|
|
187
|
+
const HIVE_FORM_HINT = ` A table in a Hive catalog is written with 'databases/<database>'.`;
|
|
188
|
+
const ICEBERG_FORM_HINT = ` A table in an Iceberg REST catalog is written ` +
|
|
189
|
+
`with 'namespaces/<namespace>'.`;
|
|
165
190
|
// The locations where `table`'s catalog entry may sit, as Dataplex writes
|
|
166
|
-
// them, in lower case, in the order to look in: one for a BigQuery
|
|
167
|
-
//
|
|
168
|
-
//
|
|
169
|
-
//
|
|
191
|
+
// them, in lower case, in the order to look in: one for a BigQuery or AlloyDB
|
|
192
|
+
// table; one for a Spanner table, or one per replica region of its instance
|
|
193
|
+
// when reading its config is refused; and one per replica for a BigLake table.
|
|
194
|
+
// `cache` keeps each instance's read and each catalog's read through each
|
|
195
|
+
// API, so each is made once; each table is read once already.
|
|
170
196
|
async function locationsOf(table, readers, cache) {
|
|
171
197
|
const once = (key, readIt) => {
|
|
172
198
|
if (!cache.has(key))
|
|
@@ -184,16 +210,23 @@ async function locationsOf(table, readers, cache) {
|
|
|
184
210
|
located(res.result.location, what, 'no location') :
|
|
185
211
|
res;
|
|
186
212
|
}
|
|
187
|
-
case 'biglake':
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
213
|
+
case 'biglake': {
|
|
214
|
+
const readCatalog = (hive) => once(`biglake/${project}/${table.catalog}/${hive}`, () => hive ? readers.biglake.getHiveCatalog(project, table.catalog) :
|
|
215
|
+
readers.biglake.getCatalog(project, table.catalog));
|
|
216
|
+
const kind = table.hive ? 'Hive catalog' : 'catalog';
|
|
217
|
+
const what = `reading the region of ${kind} '${table.catalog}'`;
|
|
218
|
+
// When the read fails and the other API finds the catalog, the source
|
|
219
|
+
// names a table in the other kind of catalog.
|
|
220
|
+
const otherForm = async () => 'result' in await (0, catalog_entry_1.read)(what, () => readCatalog(!table.hive)) ?
|
|
221
|
+
(table.hive ? ICEBERG_FORM_HINT : HIVE_FORM_HINT) :
|
|
222
|
+
'';
|
|
223
|
+
const res = await readLocation(what, `a ${kind}`, () => readCatalog(table.hive), otherForm);
|
|
224
|
+
return 'result' in res ? replicaRegions(res.result, what) : res;
|
|
225
|
+
}
|
|
193
226
|
case 'spanner':
|
|
194
227
|
return once(`spanner/${project}/${table.instance}`, async () => {
|
|
195
228
|
const what = `reading instance '${table.instance}'`;
|
|
196
|
-
const instance = await readLocation(what, 'an instance', () => readers.spanner.getInstance(project, table.instance));
|
|
229
|
+
const instance = await readLocation(what, 'an instance', () => readers.spanner.getInstance(project, table.instance), status => status === 403 ? INSTANCE_REFUSED_HINT : '');
|
|
197
230
|
if ('problem' in instance)
|
|
198
231
|
return instance;
|
|
199
232
|
const configName = instance.result.config;
|
|
@@ -207,7 +240,23 @@ async function locationsOf(table, readers, cache) {
|
|
|
207
240
|
if (region)
|
|
208
241
|
return { result: [region[1]] };
|
|
209
242
|
const configWhat = `reading instance config '${configName}'`;
|
|
210
|
-
const
|
|
243
|
+
const res = await (0, catalog_entry_1.read)(configWhat, () => readers.spanner.getInstanceConfig(configName));
|
|
244
|
+
// The default leader's region is one of the instance's replica
|
|
245
|
+
// regions, so when the config read is refused, each is looked in.
|
|
246
|
+
if ('unreadable' in res && res.status === 403) {
|
|
247
|
+
const regions = (instance.result.replicaComputeCapacity ?? [])
|
|
248
|
+
.map(r => r.replicaSelection?.location)
|
|
249
|
+
.filter((l) => !!l);
|
|
250
|
+
if (regions.length) {
|
|
251
|
+
return {
|
|
252
|
+
result: regions,
|
|
253
|
+
unfound: ` Instance config '${configName}' could not be read ` +
|
|
254
|
+
`(HTTP 403), so the entry was looked up in each of the ` +
|
|
255
|
+
`instance's replica regions: ${regions.join(', ')}.`,
|
|
256
|
+
};
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
const config = await checked(res, configWhat, 'an instance config', status => status === 403 ? CONFIG_REFUSED_HINT : '');
|
|
211
260
|
// A config marks exactly one replica as its default leader.
|
|
212
261
|
return 'result' in config ?
|
|
213
262
|
located(config.result.replicas?.find(r => r.defaultLeaderLocation)
|
|
@@ -216,22 +265,31 @@ async function locationsOf(table, readers, cache) {
|
|
|
216
265
|
});
|
|
217
266
|
}
|
|
218
267
|
}
|
|
219
|
-
// Makes the location read `call`, which `what` names
|
|
220
|
-
//
|
|
221
|
-
|
|
268
|
+
// Makes the location read `call`, which `what` names, and judges its answer
|
|
269
|
+
// with checked.
|
|
270
|
+
async function readLocation(what, kind, call, hintFor) {
|
|
271
|
+
return checked(await (0, catalog_entry_1.read)(what, call), what, kind, hintFor);
|
|
272
|
+
}
|
|
273
|
+
// What the location read `what` gave in `res`. A 404 says the source names
|
|
274
|
+
// `kind`, such as `a table`, that does not exist (the services answer 404 for
|
|
275
|
+
// a missing table, dataset, catalog or instance; live, 2026-10-27 and
|
|
222
276
|
// 2026-11-03). A 403 says the read was refused, followed by the service's
|
|
223
|
-
// message
|
|
224
|
-
|
|
225
|
-
|
|
277
|
+
// message. When a retry cannot pass, `hintFor` gives a sentence to add for the
|
|
278
|
+
// status.
|
|
279
|
+
async function checked(res, what, kind, hintFor = () => '') {
|
|
226
280
|
if ('result' in res)
|
|
227
281
|
return res;
|
|
282
|
+
const { status } = res;
|
|
283
|
+
const hint = status === undefined || (0, api_1.isTransientStatus)(status) ?
|
|
284
|
+
'' :
|
|
285
|
+
await hintFor(status);
|
|
228
286
|
if ('error' in res)
|
|
229
|
-
return failedProblem(res);
|
|
287
|
+
return failedProblem(res, hint);
|
|
230
288
|
return {
|
|
231
289
|
problem: res.status === 404 ?
|
|
232
290
|
`names ${kind} that does not exist: ${what} found nothing (HTTP ` +
|
|
233
|
-
`404)
|
|
234
|
-
`could not be checked: ${what} was refused (HTTP 403).${res.message ? ` ${res.message}` : ''}${
|
|
291
|
+
`404).${hint} Fix the source, or pass --skip-source-check.` :
|
|
292
|
+
`could not be checked: ${what} was refused (HTTP 403).${res.message ? ` ${res.message}` : ''}${hint} The ` +
|
|
235
293
|
`source's catalog entry was not looked up. Fix the access or ` +
|
|
236
294
|
`the source, or pass --skip-source-check.`,
|
|
237
295
|
};
|
|
@@ -265,13 +323,14 @@ function located(location, what, missing) {
|
|
|
265
323
|
}
|
|
266
324
|
// The problem for the failed read `res`. A thrown call, a 429 or a 5xx may
|
|
267
325
|
// pass on retry; any other status, such as BigQuery's 400 for an invalid
|
|
268
|
-
// table ID, never will, so the source must change
|
|
269
|
-
|
|
326
|
+
// table ID, never will, so the source must change, and `hint` follows the
|
|
327
|
+
// service's answer.
|
|
328
|
+
function failedProblem(res, hint = '') {
|
|
270
329
|
return res.status === undefined || (0, api_1.isTransientStatus)(res.status) ?
|
|
271
330
|
transientProblem(res.error) :
|
|
272
331
|
{
|
|
273
|
-
problem: `could not be checked: ${res.error} Fix the source,
|
|
274
|
-
`pass --skip-source-check.`,
|
|
332
|
+
problem: `could not be checked: ${res.error}${hint} Fix the source, ` +
|
|
333
|
+
`or pass --skip-source-check.`,
|
|
275
334
|
};
|
|
276
335
|
}
|
|
277
336
|
// The problem for a read that failed with `detail`, which may pass on retry.
|
|
@@ -1,17 +1,23 @@
|
|
|
1
1
|
import { Field } from './ir';
|
|
2
2
|
import { SqlColumn } from './sql_parser';
|
|
3
|
-
export declare
|
|
4
|
-
export declare function
|
|
5
|
-
export declare function
|
|
6
|
-
export declare function
|
|
7
|
-
export declare function
|
|
8
|
-
export declare function
|
|
3
|
+
export declare function unusedName(text: string, base: string): string;
|
|
4
|
+
export declare function withoutQualifier(expression: string, qualifier: string, dialect?: string): string;
|
|
5
|
+
export declare function withSiblingsSpliced(expression: string, qualifier: string, siblings: ReadonlyMap<string, string>): string;
|
|
6
|
+
export declare function qualifiedColumns(expression: string, qualifier: string, dialect?: string): string[] | undefined;
|
|
7
|
+
export declare function selfReferenceMessage(owner: string, cycle: string[], where?: string): string;
|
|
8
|
+
export declare function requalified(expression: string, from: string, to: string, dialect?: string): string;
|
|
9
|
+
export declare function measureEntities(expression: string, entityNames: ReadonlyArray<string>, entity?: string): string[];
|
|
9
10
|
export declare function sqlWithoutComments(text: string, dialect?: string): string;
|
|
10
11
|
export declare function fieldBindingWithoutComments(field: Pick<Field, 'expression' | 'importedExpression' | 'importedDialect'>, targetDialect?: string): string | undefined;
|
|
11
12
|
export declare function columnReferences(expression: string, dialect: string): ReadonlyArray<SqlColumn> | undefined;
|
|
12
|
-
export declare function columnsForSource(text: string, dialect: string, impliedDialect?: string): ReadonlyArray<SqlColumn> | undefined;
|
|
13
|
+
export declare function columnsForSource(text: string, dialect: string, impliedDialect?: string | null): ReadonlyArray<SqlColumn> | undefined;
|
|
13
14
|
export declare function entitiesReached(columns: ReadonlyArray<SqlColumn>, entityNames: ReadonlyArray<string> | ReadonlySet<string>): string[];
|
|
14
15
|
export declare function fieldsReadOn(columns: ReadonlyArray<SqlColumn>, entityName: string): string[];
|
|
15
16
|
export declare const SUPPORTED_AGGREGATES: string[];
|
|
17
|
+
export declare function singleAggregate(expression: string, dialect: string): {
|
|
18
|
+
fn: string;
|
|
19
|
+
operand: string;
|
|
20
|
+
distinct: boolean;
|
|
21
|
+
} | undefined;
|
|
16
22
|
export declare function isSingleAggregate(expression: string, dialect: string): boolean;
|
|
17
23
|
export declare function isColumnName(value: unknown): boolean;
|