@google-cloud/kcmd 0.1.0 → 0.2.1

package/build/ts/kcmd/semantic/runtime/store.d.ts

@@ -50,13 +50,13 @@ export type ChosenStore = SpannerStore | AlloyDbDatabase | BigQueryStore;
  * `named` is set when the model's deployments name a target and still no
  * store results, so a caller that runs against the store fails. `rejected` is
  * set with it when the deployments give no store whatever the model is used
- * for: a target in no accepted form in any deployment, a GOOGLE block whose
- * data is not valid JSON, or several targets for the profile. `named` alone
- * means the model binds a table outside its Spanner or AlloyDB store.
+ * for: a target in no accepted form in any deployment, or several targets for
+ * the profile. `named` alone means the model binds a table outside its Spanner
+ * or AlloyDB store.
  *
  * Neither is set when the profile names no store, unless a target is in no
- * accepted form, or a GOOGLE block is not valid JSON. Neither is set either
- * when a BigQuery model binds a table outside the graph's dataset.
+ * accepted form. Neither is set either when a BigQuery model binds a table
+ * outside the graph's dataset.
  */
 export interface StoreError {
     error: string;
@@ -67,21 +67,18 @@ export interface StoreError {
  * The store a model runs against under binding profile `profile`, which
  * defaults to `default`, the model file's own bindings. It is the one the
  * deployments for that profile name: those that name it in `profile`, or that
- * name none or `default` for `default`. A model with no `deployments` list
- * names no profile for any of its targets, so they are all for `default`.
- * With several targets, the store is the single operational one, Spanner or
- * AlloyDB.
+ * name none or `default` for `default`. With several targets, the store is the
+ * single operational one, Spanner or AlloyDB.
  *
- * Fails six ways:
+ * Fails five ways:
  *
- * 1. A GOOGLE block's data is not valid JSON, so its targets cannot be read.
- * 2. A target names no supported destination. Push checks the target of every
+ * 1. A target names no supported destination. Push checks the target of every
  *    deployment, so this fails the store under every profile.
- * 3. No deployment uses the profile.
- * 4. There are several targets and no single operational one, so which store
+ * 2. No deployment uses the profile.
+ * 3. There are several targets and no single operational one, so which store
  *    is meant is ambiguous.
- * 5. There is no target, so there is nothing to run against.
- * 6. The model binds a table outside the store. For a Spanner or AlloyDB
+ * 4. There is no target, so there is nothing to run against.
+ * 5. The model binds a table outside the store. For a Spanner or AlloyDB
  *    store, a write would land in the wrong place, and the error is named. A
  *    BigQuery graph may read tables in any dataset, so
  *    for a BigQuery store the error is not named.
@@ -89,9 +86,7 @@ export interface StoreError {
 export declare function resolveStore(model: SemanticModel, profile?: string): ChosenStore | StoreError;
 /**
  * The targets binding profile `profile` uses, as resolveStore reads them: its
- * deployments' targets, or every target in a GOOGLE block for `default` when
- * the model has no `deployments` list. Empty when no deployment is for the
- * profile. Throws when a GOOGLE block's data is not valid JSON.
+ * deployments' targets. Empty when no deployment is for the profile.
  */
 export declare function profileTargets(model: SemanticModel, profile: string): string[];
 /**

package/build/ts/kcmd/semantic/runtime/store.js

@@ -101,21 +101,18 @@ function strayBindings(model, store) {
  * The store a model runs against under binding profile `profile`, which
  * defaults to `default`, the model file's own bindings. It is the one the
  * deployments for that profile name: those that name it in `profile`, or that
- * name none or `default` for `default`. A model with no `deployments` list
- * names no profile for any of its targets, so they are all for `default`.
- * With several targets, the store is the single operational one, Spanner or
- * AlloyDB.
+ * name none or `default` for `default`. With several targets, the store is the
+ * single operational one, Spanner or AlloyDB.
  *
- * Fails six ways:
+ * Fails five ways:
  *
- * 1. A GOOGLE block's data is not valid JSON, so its targets cannot be read.
- * 2. A target names no supported destination. Push checks the target of every
+ * 1. A target names no supported destination. Push checks the target of every
  *    deployment, so this fails the store under every profile.
- * 3. No deployment uses the profile.
- * 4. There are several targets and no single operational one, so which store
+ * 2. No deployment uses the profile.
+ * 3. There are several targets and no single operational one, so which store
  *    is meant is ambiguous.
- * 5. There is no target, so there is nothing to run against.
- * 6. The model binds a table outside the store. For a Spanner or AlloyDB
+ * 4. There is no target, so there is nothing to run against.
+ * 5. The model binds a table outside the store. For a Spanner or AlloyDB
  *    store, a write would land in the wrong place, and the error is named. A
  *    BigQuery graph may read tables in any dataset, so
  *    for a BigQuery store the error is not named.
@@ -123,15 +120,7 @@ function strayBindings(model, store) {
 function resolveStore(model, profile = resolve_profiles_1.DEFAULT_PROFILE) {
     // Push checks the target of every deployment the model declares, so the
     // targets of every deployment are read before the profile's are picked.
-    let all;
-    try {
-        all = (0, deployment_target_1.googleDeploymentTargets)(model);
-    }
-    catch (err) {
-        // It throws when a GOOGLE block's data is not valid JSON. Push reports
-        // that as an error, so the store fails.
-        return { error: `${err?.message || err}`, named: true, rejected: true };
-    }
+    const all = (0, deployment_target_1.googleDeploymentTargets)(model);
     // The loader reads a vanilla model's `deployments` out of its GOOGLE block,
     // so a message names the block's `deployments`, as push does.
     const vanilla = model.version === loader_1.OSSIE_VERSION;
@@ -141,17 +130,19 @@ function resolveStore(model, profile = resolve_profiles_1.DEFAULT_PROFILE) {
     // A target in no accepted form is reported, alone or beside one that gives
     // a store, whichever profile its deployment is for.
     if (all.malformed.length) {
-        const unparsed = model.deployments?.find(d => d.target === all.malformed[0]);
-        const error = unparsed ? (0, deployment_target_1.deploymentTargetError)(model, unparsed) :
-            (0, deployment_target_1.googleBlockTargetError)(model, all.malformed[0]);
-        return { error, named: true, rejected: true };
+        const unparsed = model.deployments.find(d => d.target === all.malformed[0]);
+        return {
+            error: (0, deployment_target_1.deploymentTargetError)(model, unparsed),
+            named: true,
+            rejected: true,
+        };
     }
     const used = deploymentsFor(model, profile);
     if ('error' in used)
         return used;
     const scoped = used.model;
-    // The profile's targets are among those read above. So this does not
-    // throw, and none of them is in no accepted form.
+    // The profile's targets are among those read above, so none of them is in no
+    // accepted form.
     const targets = (0, deployment_target_1.googleDeploymentTargets)(scoped);
     const { spanner, alloyDb, bigQuery } = targets;
     const declared = spanner.length + alloyDb.length + bigQuery.length;
@@ -202,9 +193,7 @@ function resolveStore(model, profile = resolve_profiles_1.DEFAULT_PROFILE) {
     return store;
 }
 // The model with only the deployments `profile` uses, or why there are none.
-// A model with no `deployments` list keeps every target in its GOOGLE block
-// for `default`. An empty list names no deployment. Either way, any other
-// profile has none.
+// A model with no or empty `deployments` list names no deployment.
 function deploymentsFor(model, profile) {
     const nothingToRun = `so there is no database to run it against`;
     const namedProfile = `binding profile '${profile}' is a named profile, ` +
@@ -235,21 +224,16 @@ function deploymentsFor(model, profile) {
 }
 /**
  * The targets binding profile `profile` uses, as resolveStore reads them: its
- * deployments' targets, or every target in a GOOGLE block for `default` when
- * the model has no `deployments` list. Empty when no deployment is for the
- * profile. Throws when a GOOGLE block's data is not valid JSON.
+ * deployments' targets. Empty when no deployment is for the profile.
  */
 function profileTargets(model, profile) {
     const used = deploymentsFor(model, profile);
     return 'error' in used ? [] : (0, deployment_target_1.googleDeploymentTargets)(used.model).uris;
 }
-// The deployments whose targets are `uris`, for a message, or nothing when
-// the targets come from a GOOGLE block, which names none. `profile` is the
+// The deployments whose targets are `uris`, for a message. `profile` is the
 // profile they are for.
 function deploymentNames(model, uris, profile) {
-    const deployments = model.deployments;
-    if (!deployments)
-        return '';
+    const deployments = model.deployments ?? [];
     const names = uris.map(uri => deployments.find(d => d.target === uri)?.name ?? uri);
     return ` for profile '${profile}' (${names.join(', ')})`;
 }

package/build/ts/kcmd/semantic/source_dialect.d.ts

@@ -1,10 +1,17 @@
+import type { SpannerClient } from '../gcp/spanner';
 import { EntryLookup } from './catalog_entry';
 export interface SourceDialects {
     dialects: Map<string, string>;
     databases: Map<string, string>;
     errors: string[];
+    unread: string[];
 }
-export declare function sourceDialects(texts: Iterable<string>, catalog: EntryLookup, { thirdParty }?: {
-    thirdParty?: boolean | undefined;
+export declare function sourceDialects(texts: Iterable<string>, catalog: EntryLookup, { thirdParty, spanner }?: {
+    thirdParty?: boolean;
+    spanner?: Pick<SpannerClient, 'getDatabase'>;
 }): Promise<SourceDialects>;
+export declare function tableFqnParts(fqn: string): {
+    prefix: string;
+    parts: string[];
+} | undefined;
 export declare function sourcesToLookUp(texts: Iterable<string>): string[];

package/build/ts/kcmd/semantic/source_dialect.js

@@ -1,7 +1,7 @@
 "use strict";
 // The dialect of a Cloud SQL table written as its resource name, or of a
 // third-party table written as its entry name, read from the table's
-// Knowledge Catalog entry.
+// Knowledge Catalog entry, and the dialect of a Spanner table's database.
 //
 // The resource name cannot say the dialect: a MySQL table has no schema, but a
 // PostgreSQL table and a SQL Server table have the same shape. Dataplex keeps an
@@ -16,9 +16,12 @@
 // fullyQualifiedName, so its parts say whether the entry is a table's or, for
 // example, a schema's.
 //
-// This is the only place kcmd makes a network call to learn a dialect. A
-// command looks every such source up once, before it loads the model, and
-// passes the dialects to the load (LoadOptions.sourceDialects).
+// Knowledge Catalog's Spanner entries leave their dialect empty, so a Spanner
+// table's dialect is its database's `databaseDialect`, read by databases.get.
+//
+// This is the only place kcmd looks up a source's dialect. A command looks
+// every such source up once, before it loads the model, and passes the
+// dialects to the load (LoadOptions.sourceDialects).
 var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
     if (k2 === undefined) k2 = k;
     var desc = Object.getOwnPropertyDescriptor(m, k);
@@ -54,10 +57,12 @@ var __importStar = (this && this.__importStar) || (function () {
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.sourceDialects = sourceDialects;
+exports.tableFqnParts = tableFqnParts;
 exports.sourcesToLookUp = sourcesToLookUp;
 const yaml = __importStar(require("yaml"));
 const loader_1 = require("./loader");
 const catalog_entry_1 = require("./catalog_entry");
+const parameters_1 = require("./parameters");
 const yaml_options_1 = require("./yaml_options");
 // The dialect each entrySource.system gives, in lower case.
 const DIALECT_OF_ENTRY_SYSTEM = new Map([...loader_1.DIALECT_OF_FQN_PREFIX, ['postgres', 'POSTGRES']]);
@@ -70,14 +75,16 @@ const TABLE_FQN_PARTS = new Map([
     ['postgresql', ['host', 'database', 'schema', 'table']],
     ['sqlserver', ['host', 'database', 'schema', 'table']],
 ]);
-// Looks up the dialect of every Cloud SQL and third-party table the model and
-// profile files `texts` write (see sourcesToLookUp), once per table, in
-// parallel. With `thirdParty` false, looks up Cloud SQL tables only. Makes no
-// call when they write none. Never throws: a lookup that fails is reported in
-// `errors`.
-async function sourceDialects(texts, catalog, { thirdParty = true } = {}) {
-    // Keyed as dialectLookupKey gives, so both spellings of one table are one
-    // lookup.
+// Looks up, once each and in parallel, the dialect of every Cloud SQL and
+// third-party table the files `texts` write (see sourcesToLookUp), and of every
+// Spanner database they name. Without `spanner`, a Spanner database gives
+// SPANNER, the dialect kcmd's Spanner store speaks, with no call.
+// `thirdParty: false` skips third-party tables. Never throws: a failed lookup
+// goes to `errors`.
+async function sourceDialects(texts, catalog, { thirdParty = true, spanner } = {}) {
+    // Keyed as dialectLookupKey gives, so both spellings of one table, and the
+    // tables of one Spanner database, are one lookup. A database's lookup names
+    // the first of its tables.
     const lookups = new Map();
     for (const source of sourcesToLookUp(texts)) {
         const table = (0, loader_1.cloudSqlTableOf)(source);
@@ -87,32 +94,42 @@ async function sourceDialects(texts, catalog, { thirdParty = true } = {}) {
         const entry = thirdParty ? (0, loader_1.thirdPartyEntryOf)(source) : undefined;
         if (entry)
             lookups.set(entry.name, () => thirdPartyDialect(catalog, entry));
+        const database = (0, loader_1.spannerDatabaseOf)(source);
+        if (database && !lookups.has(database)) {
+            lookups.set(database, spanner ? () => spannerDialect(spanner, database, source) :
+                async () => ({ dialect: 'SPANNER' }));
+        }
     }
     const dialects = new Map();
     const databases = new Map();
     const errors = [];
+    const unread = [];
     await Promise.all([...lookups].map(async ([key, lookUp]) => {
         const got = await lookUp();
         if ('error' in got)
             errors.push(got.error);
         else
             dialects.set(key, got.dialect);
+        if ('unread' in got)
+            unread.push(got.error);
         if ('database' in got && got.database)
             databases.set(key, got.database);
     }));
     errors.sort();
-    return { dialects, databases, errors };
+    unread.sort();
+    return { dialects, databases, errors, unread };
 }
 // The dialect of a Cloud SQL table, by its entry's type.
 async function cloudSqlDialect(catalog, table) {
     const at = `source '${table.resourceName}'`;
     const res = await (0, catalog_entry_1.lookupSystemEntry)(catalog, 'cloudsql', table.project, table.location, table.resourceName);
     if ('error' in res)
-        return { error: `${at}: ${res.error}` };
+        return { error: `${at}: ${res.error}`, unread: true };
     // Dataplex answers 403 both when the entry does not exist and when the
     // caller may not read it, so the message names both causes.
     if ('unreadable' in res) {
         return {
+            unread: true,
             error: `${at}: kcmd reads a Cloud SQL table's dialect from its ` +
                 `Knowledge Catalog entry, and could not read it. Turn on Dataplex ` +
                 `integration for instance '${table.instance}' (gcloud sql ` +
@@ -131,6 +148,27 @@ async function cloudSqlDialect(catalog, table) {
             `not a Cloud SQL table or view type kcmd knows (${Object.keys(loader_1.DIALECT_OF_ENTRY_TYPE).join(', ')}).`,
     };
 }
+// The dialect of the Spanner database `database`, which `source` names a
+// table of, by its databaseDialect.
+async function spannerDialect(spanner, database, source) {
+    const [, , project, , instance, , name] = database.split('/');
+    const failed = (why) => ({
+        unread: true,
+        error: `source '${source}': kcmd reads its Spanner database's SQL ` +
+            `dialect, and could not read database ${project}/${instance}/${name}: ${why}`,
+    });
+    try {
+        const res = await spanner.getDatabase(project, instance, name);
+        if (res.status !== 200) {
+            return failed((0, parameters_1.storeMessage)(res.message, res.status));
+        }
+        const postgres = res.result?.databaseDialect === 'POSTGRESQL';
+        return { dialect: postgres ? 'POSTGRES' : 'SPANNER' };
+    }
+    catch (err) {
+        return failed(err?.message ?? String(err));
+    }
+}
 // The dialect of a third-party table, by its entry's fullyQualifiedName or
 // entrySource.system, and the database the fullyQualifiedName names. An entry
 // whose fullyQualifiedName names no table or view is rejected.
@@ -138,9 +176,10 @@ async function thirdPartyDialect(catalog, { project, location, name }) {
     const at = `source '${name}'`;
     const res = await (0, catalog_entry_1.read)('looking up its Knowledge Catalog entry', () => catalog.lookupEntry(project, location, name));
     if ('error' in res)
-        return { error: `${at}: ${res.error}` };
+        return { error: `${at}: ${res.error}`, unread: true };
     if ('unreadable' in res) {
         return {
+            unread: true,
             error: `${at} names no table Knowledge Catalog knows about. The entry ` +
                 `may not exist, or you may not be allowed to read it. kcmd reads a ` +
                 `third-party table's dialect from its entry, so add the table to ` +
@@ -177,14 +216,12 @@ async function thirdPartyDialect(catalog, { project, location, name }) {
             `fullyQualifiedName.`,
     };
 }
-// The database the fullyQualifiedName `fqn` of a third-party table or view
-// names: a Snowflake account, a MySQL or SQL Server host, a PostgreSQL host
-// and database, or a Databricks metastore. Each account and host is in lower
-// case. Undefined when `fqn` names no table or view. Such a name has another
-// number of parts than TABLE_FQN_PARTS gives, or a subtype, such as the
-// `routine:` of a routine. Dataplex reads a `databricks:table:` or
-// `databricks:view:` name as the same table as the `databricks:` name.
-function databaseOfFqn(fqn) {
+// The prefix of the fullyQualifiedName `fqn` of a third-party table or view,
+// and its unquoted parts, such as `snowflake` and ['acme', 'retail', 'public',
+// 'orders'] for `snowflake:acme.retail.public.orders`. Undefined when `fqn`
+// has another number of parts than TABLE_FQN_PARTS gives, or a subtype other
+// than Databricks' `table:` and `view:`.
+function tableFqnParts(fqn) {
     // The subtype holds no backtick, so a colon inside a quoted host and port
     // stays in the path. A colon in an unquoted database, schema or table
     // name stays in its segment when it comes after the name's first backtick.
@@ -195,19 +232,31 @@ function databaseOfFqn(fqn) {
         (prefix === 'databricks' && (subtype === 'table' || subtype === 'view'));
     if (!parts || !tableSubtype || seg.length !== parts.length)
         return undefined;
+    return { prefix, parts: seg };
+}
+// The database the fullyQualifiedName `fqn` of a third-party table or view
+// names: a Snowflake account, a MySQL or SQL Server host, a PostgreSQL host
+// and database, or a Databricks metastore. Each account and host is in lower
+// case. Undefined when `fqn` names no table or view.
+function databaseOfFqn(fqn) {
+    const table = tableFqnParts(fqn);
+    if (!table)
+        return undefined;
+    const { prefix, parts } = table;
     const n = prefix === 'postgresql' ? 2 : 1;
     // A Snowflake account and a host name ignore letter case.
     if (prefix !== 'databricks')
-        seg[0] = seg[0].toLowerCase();
-    return [prefix, ...seg.slice(0, n)].join('/');
+        parts[0] = parts[0].toLowerCase();
+    return [prefix, ...parts.slice(0, n)].join('/');
 }
-// Every Cloud SQL table and third-party entry the model and profile files
-// `texts` write. A Cloud SQL table counts as a resource name or resource URI
-// anywhere, JSON strings included. An entry name counts as a `source` value in
-// the YAML or in a GOOGLE block's JSON `data`, unless it sits in ai_context,
-// an action parameter's default or another vendor's block. Values are read
-// from the parsed YAML, so a name in a comment is not collected. A file that
-// does not parse collects nothing; the load reports it.
+// Every Cloud SQL table, Spanner table and third-party entry the model and
+// profile files `texts` write. A Cloud SQL table counts as a resource name or
+// resource URI anywhere, JSON strings included. A Spanner table or an entry
+// name counts as a `source` value in the YAML or in a GOOGLE block's JSON
+// `data`, unless it sits in ai_context, an action parameter's default or
+// another vendor's block. Values are read from the parsed YAML, so a name in a
+// comment is not collected. A file that does not parse collects nothing; the
+// load reports it.
 function sourcesToLookUp(texts) {
     const found = new Set();
     // `key` names the member that holds `value`. `opaque` marks a value whose
@@ -217,7 +266,8 @@ function sourcesToLookUp(texts) {
     const visit = (value, key, opaque = false) => {
         if (typeof value === 'string') {
             if ((0, loader_1.cloudSqlTableOf)(value) ||
-                (key === 'source' && !opaque && (0, loader_1.thirdPartyEntryOf)(value))) {
+                (key === 'source' && !opaque &&
+                    ((0, loader_1.thirdPartyEntryOf)(value) || (0, loader_1.spannerDatabaseOf)(value)))) {
                 found.add(value.trim());
             }
             else if (/^\s*[[{]/.test(value)) {

package/build/ts/kcmd/semantic/source_entries.d.ts

@@ -6,7 +6,7 @@ import type { SemanticModel } from './ir';
 export interface SourceReaders {
     catalog: EntryLookup;
     bigquery: Pick<BigQueryClient, 'getTable'>;
-    biglake: Pick<BigLakeClient, 'getCatalog'>;
+    biglake: Pick<BigLakeClient, 'getCatalog' | 'getHiveCatalog'>;
     spanner: Pick<SpannerClient, 'getInstance' | 'getInstanceConfig'>;
 }
 export declare function sourceEntryErrors(model: Pick<SemanticModel, 'name' | 'entities' | 'profiles'>, readers: SourceReaders): Promise<string[]>;