@google-cloud/kcmd 0.0.0-stage → 0.2.0

package/build/ts/kcmd/semantic/source_dialect.d.ts

@@ -0,0 +1,17 @@
+import type { SpannerClient } from '../gcp/spanner';
+import { EntryLookup } from './catalog_entry';
+export interface SourceDialects {
+    dialects: Map<string, string>;
+    databases: Map<string, string>;
+    errors: string[];
+    unread: string[];
+}
+export declare function sourceDialects(texts: Iterable<string>, catalog: EntryLookup, { thirdParty, spanner }?: {
+    thirdParty?: boolean;
+    spanner?: Pick<SpannerClient, 'getDatabase'>;
+}): Promise<SourceDialects>;
+export declare function tableFqnParts(fqn: string): {
+    prefix: string;
+    parts: string[];
+} | undefined;
+export declare function sourcesToLookUp(texts: Iterable<string>): string[];

package/build/ts/kcmd/semantic/source_dialect.js

@@ -0,0 +1,311 @@
+"use strict";
+// The dialect of a Cloud SQL table written as its resource name, or of a
+// third-party table written as its entry name, read from the table's
+// Knowledge Catalog entry, and the dialect of a Spanner table's database.
+//
+// The resource name cannot say the dialect: a MySQL table has no schema, but a
+// PostgreSQL table and a SQL Server table have the same shape. Dataplex keeps an
+// entry for each table of an instance with Dataplex integration on, in the
+// system entry group `@cloudsql`, and the entry's id is the table's resource
+// name. Its entry type names the engine.
+//
+// A third-party table's entry names its system twice: in the prefix of its
+// fullyQualifiedName, which Dataplex checks, and in entrySource.system, which
+// is free text. So the prefix decides, and the system only when the entry has
+// no fullyQualifiedName. Dataplex also checks the rest of the
+// fullyQualifiedName, so its parts say whether the entry is a table's or, for
+// example, a schema's.
+//
+// Knowledge Catalog's Spanner entries leave their dialect empty, so a Spanner
+// table's dialect is its database's `databaseDialect`, read by databases.get.
+//
+// This is the only place kcmd looks up a source's dialect. A command looks
+// every such source up once, before it loads the model, and passes the
+// dialects to the load (LoadOptions.sourceDialects).
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.sourceDialects = sourceDialects;
+exports.tableFqnParts = tableFqnParts;
+exports.sourcesToLookUp = sourcesToLookUp;
+const yaml = __importStar(require("yaml"));
+const loader_1 = require("./loader");
+const catalog_entry_1 = require("./catalog_entry");
+const parameters_1 = require("./parameters");
+const yaml_options_1 = require("./yaml_options");
+// The dialect each entrySource.system gives, in lower case.
+const DIALECT_OF_ENTRY_SYSTEM = new Map([...loader_1.DIALECT_OF_FQN_PREFIX, ['postgres', 'POSTGRES']]);
+// The parts of a table's fullyQualifiedName after each prefix, as Dataplex
+// reads it. A view's name has the same parts as a table's.
+const TABLE_FQN_PARTS = new Map([
+    ['snowflake', ['account', 'database', 'schema', 'table']],
+    ['databricks', ['metastore', 'catalog', 'schema', 'table']],
+    ['mysql', ['host', 'schema', 'table']],
+    ['postgresql', ['host', 'database', 'schema', 'table']],
+    ['sqlserver', ['host', 'database', 'schema', 'table']],
+]);
+// Looks up, once each and in parallel, the dialect of every Cloud SQL and
+// third-party table the files `texts` write (see sourcesToLookUp), and of every
+// Spanner database they name. Without `spanner`, a Spanner database gives
+// SPANNER, the dialect kcmd's Spanner store speaks, with no call.
+// `thirdParty: false` skips third-party tables. Never throws: a failed lookup
+// goes to `errors`.
+async function sourceDialects(texts, catalog, { thirdParty = true, spanner } = {}) {
+    // Keyed as dialectLookupKey gives, so both spellings of one table, and the
+    // tables of one Spanner database, are one lookup. A database's lookup names
+    // the first of its tables.
+    const lookups = new Map();
+    for (const source of sourcesToLookUp(texts)) {
+        const table = (0, loader_1.cloudSqlTableOf)(source);
+        if (table) {
+            lookups.set(table.resourceName, () => cloudSqlDialect(catalog, table));
+        }
+        const entry = thirdParty ? (0, loader_1.thirdPartyEntryOf)(source) : undefined;
+        if (entry)
+            lookups.set(entry.name, () => thirdPartyDialect(catalog, entry));
+        const database = (0, loader_1.spannerDatabaseOf)(source);
+        if (database && !lookups.has(database)) {
+            lookups.set(database, spanner ? () => spannerDialect(spanner, database, source) :
+                async () => ({ dialect: 'SPANNER' }));
+        }
+    }
+    const dialects = new Map();
+    const databases = new Map();
+    const errors = [];
+    const unread = [];
+    await Promise.all([...lookups].map(async ([key, lookUp]) => {
+        const got = await lookUp();
+        if ('error' in got)
+            errors.push(got.error);
+        else
+            dialects.set(key, got.dialect);
+        if ('unread' in got)
+            unread.push(got.error);
+        if ('database' in got && got.database)
+            databases.set(key, got.database);
+    }));
+    errors.sort();
+    unread.sort();
+    return { dialects, databases, errors, unread };
+}
+// The dialect of a Cloud SQL table, by its entry's type.
+async function cloudSqlDialect(catalog, table) {
+    const at = `source '${table.resourceName}'`;
+    const res = await (0, catalog_entry_1.lookupSystemEntry)(catalog, 'cloudsql', table.project, table.location, table.resourceName);
+    if ('error' in res)
+        return { error: `${at}: ${res.error}`, unread: true };
+    // Dataplex answers 403 both when the entry does not exist and when the
+    // caller may not read it, so the message names both causes.
+    if ('unreadable' in res) {
+        return {
+            unread: true,
+            error: `${at}: kcmd reads a Cloud SQL table's dialect from its ` +
+                `Knowledge Catalog entry, and could not read it. Turn on Dataplex ` +
+                `integration for instance '${table.instance}' (gcloud sql ` +
+                `instances patch ${table.instance} --enable-dataplex-integration ` +
+                `--project ${table.project}) and wait for the table to appear in ` +
+                `the catalog, or check that you can read the table. If integration ` +
+                `is on, check the name: the project ID, the instance's region, and ` +
+                `a schema only for PostgreSQL and SQL Server.`,
+        };
+    }
+    // The type comes back as a full name; its last part says the engine.
+    const type = (res.result.entryType ?? '').split('/').pop() ?? '';
+    const dialect = loader_1.DIALECT_OF_ENTRY_TYPE[type];
+    return dialect ? { dialect } : {
+        error: `${at}: its Knowledge Catalog entry has type '${type}', which is ` +
+            `not a Cloud SQL table or view type kcmd knows (${Object.keys(loader_1.DIALECT_OF_ENTRY_TYPE).join(', ')}).`,
+    };
+}
+// The dialect of the Spanner database `database`, which `source` names a
+// table of, by its databaseDialect.
+async function spannerDialect(spanner, database, source) {
+    const [, , project, , instance, , name] = database.split('/');
+    const failed = (why) => ({
+        unread: true,
+        error: `source '${source}': kcmd reads its Spanner database's SQL ` +
+            `dialect, and could not read database ${project}/${instance}/${name}: ${why}`,
+    });
+    try {
+        const res = await spanner.getDatabase(project, instance, name);
+        if (res.status !== 200) {
+            return failed((0, parameters_1.storeMessage)(res.message, res.status));
+        }
+        const postgres = res.result?.databaseDialect === 'POSTGRESQL';
+        return { dialect: postgres ? 'POSTGRES' : 'SPANNER' };
+    }
+    catch (err) {
+        return failed(err?.message ?? String(err));
+    }
+}
+// The dialect of a third-party table, by its entry's fullyQualifiedName or
+// entrySource.system, and the database the fullyQualifiedName names. An entry
+// whose fullyQualifiedName names no table or view is rejected.
+async function thirdPartyDialect(catalog, { project, location, name }) {
+    const at = `source '${name}'`;
+    const res = await (0, catalog_entry_1.read)('looking up its Knowledge Catalog entry', () => catalog.lookupEntry(project, location, name));
+    if ('error' in res)
+        return { error: `${at}: ${res.error}`, unread: true };
+    if ('unreadable' in res) {
+        return {
+            unread: true,
+            error: `${at} names no table Knowledge Catalog knows about. The entry ` +
+                `may not exist, or you may not be allowed to read it. kcmd reads a ` +
+                `third-party table's dialect from its entry, so add the table to ` +
+                `Knowledge Catalog, or fix the source.`,
+        };
+    }
+    const { fullyQualifiedName: fqn, entrySource } = res.result;
+    const system = entrySource?.system;
+    const prefix = fqn?.match(/^([a-z]+):/)?.[1] ?? '';
+    const dialect = fqn ? loader_1.DIALECT_OF_FQN_PREFIX.get(prefix) :
+        DIALECT_OF_ENTRY_SYSTEM.get(system?.toLowerCase() ?? '');
+    if (dialect && fqn) {
+        const database = databaseOfFqn(fqn);
+        if (database)
+            return { dialect, database };
+        const parts = TABLE_FQN_PARTS.get(prefix).map(p => `<${p}>`).join('.');
+        return {
+            error: `${at}: its Knowledge Catalog entry has fullyQualifiedName ` +
+                `'${fqn}', which names no table or view. A table's ` +
+                `fullyQualifiedName has the form ${prefix}:${parts}. Write the ` +
+                `entry name of the table's own entry.`,
+        };
+    }
+    if (dialect)
+        return { dialect };
+    const has = fqn ? `fullyQualifiedName '${fqn}'` :
+        system ? `entrySource.system '${system}' and no fullyQualifiedName` :
+            'neither a fullyQualifiedName nor an entrySource.system';
+    return {
+        error: `${at}: kcmd cannot tell the table's SQL dialect from its ` +
+            `Knowledge Catalog entry, which has ${has}. kcmd reads the prefix of ` +
+            `the entry's fullyQualifiedName (snowflake:, databricks:, mysql:, ` +
+            `postgresql: or sqlserver:), or entrySource.system when there is no ` +
+            `fullyQualifiedName.`,
+    };
+}
+// The prefix of the fullyQualifiedName `fqn` of a third-party table or view,
+// and its unquoted parts, such as `snowflake` and ['acme', 'retail', 'public',
+// 'orders'] for `snowflake:acme.retail.public.orders`. Undefined when `fqn`
+// has another number of parts than TABLE_FQN_PARTS gives, or a subtype other
+// than Databricks' `table:` and `view:`.
+function tableFqnParts(fqn) {
+    // The subtype holds no backtick, so a colon inside a quoted host and port
+    // stays in the path. A colon in an unquoted database, schema or table
+    // name stays in its segment when it comes after the name's first backtick.
+    const [, prefix, subtype, path] = fqn.match(/^([a-z]+):(?:([a-z0-9_-]+):)?(.*)$/s) ?? [];
+    const parts = TABLE_FQN_PARTS.get(prefix ?? '');
+    const seg = (0, loader_1.catalogNameSegments)(path ?? '', true);
+    const tableSubtype = subtype === undefined ||
+        (prefix === 'databricks' && (subtype === 'table' || subtype === 'view'));
+    if (!parts || !tableSubtype || seg.length !== parts.length)
+        return undefined;
+    return { prefix, parts: seg };
+}
+// The database the fullyQualifiedName `fqn` of a third-party table or view
+// names: a Snowflake account, a MySQL or SQL Server host, a PostgreSQL host
+// and database, or a Databricks metastore. Each account and host is in lower
+// case. Undefined when `fqn` names no table or view.
+function databaseOfFqn(fqn) {
+    const table = tableFqnParts(fqn);
+    if (!table)
+        return undefined;
+    const { prefix, parts } = table;
+    const n = prefix === 'postgresql' ? 2 : 1;
+    // A Snowflake account and a host name ignore letter case.
+    if (prefix !== 'databricks')
+        parts[0] = parts[0].toLowerCase();
+    return [prefix, ...parts.slice(0, n)].join('/');
+}
+// Every Cloud SQL table, Spanner table and third-party entry the model and
+// profile files `texts` write. A Cloud SQL table counts as a resource name or
+// resource URI anywhere, JSON strings included. A Spanner table or an entry
+// name counts as a `source` value in the YAML or in a GOOGLE block's JSON
+// `data`, unless it sits in ai_context, an action parameter's default or
+// another vendor's block. Values are read from the parsed YAML, so a name in a
+// comment is not collected. A file that does not parse collects nothing; the
+// load reports it.
+function sourcesToLookUp(texts) {
+    const found = new Set();
+    // `key` names the member that holds `value`. `opaque` marks a value whose
+    // `source` names no table: one inside ai_context, an action parameter's
+    // default or another vendor's block, or in JSON written anywhere but a
+    // `data` member.
+    const visit = (value, key, opaque = false) => {
+        if (typeof value === 'string') {
+            if ((0, loader_1.cloudSqlTableOf)(value) ||
+                (key === 'source' && !opaque &&
+                    ((0, loader_1.thirdPartyEntryOf)(value) || (0, loader_1.spannerDatabaseOf)(value)))) {
+                found.add(value.trim());
+            }
+            else if (/^\s*[[{]/.test(value)) {
+                try {
+                    visit(JSON.parse(value), undefined, opaque || key !== 'data');
+                }
+                catch {
+                    // Not JSON: an ordinary string.
+                }
+            }
+        }
+        else if (Array.isArray(value)) {
+            value.forEach(v => visit(v, undefined, opaque));
+        }
+        else if (value && typeof value === 'object') {
+            const vendor = 'vendor_name' in value && !(0, loader_1.isGoogleVendor)(value.vendor_name);
+            for (const [k, v] of Object.entries(value)) {
+                visit(v, k, opaque || vendor || k === 'ai_context' || k === 'default');
+            }
+        }
+    };
+    for (const text of texts) {
+        // Parsed as the loader parses, so both read the same values.
+        for (const doc of yaml.parseAllDocuments(text, yaml_options_1.YAML_OPTIONS)) {
+            if (doc.errors.length)
+                continue;
+            // Reading the values can still fail: on an alias whose anchor is not
+            // set, on more aliases than a load expands, or, while walking them, on
+            // an alias that refers to itself, which makes the values cyclic. The
+            // load reports that file in its own words, so the rest of it is skipped
+            // here rather than stopping the command.
+            try {
+                visit(doc.toJS());
+            }
+            catch {
+                continue;
+            }
+        }
+    }
+    return [...found];
+}

package/build/ts/kcmd/semantic/source_entries.d.ts

@@ -0,0 +1,12 @@
+import type { BigLakeClient } from '../gcp/biglake';
+import type { BigQueryClient } from '../gcp/bigquery';
+import type { SpannerClient } from '../gcp/spanner';
+import { EntryLookup } from './catalog_entry';
+import type { SemanticModel } from './ir';
+export interface SourceReaders {
+    catalog: EntryLookup;
+    bigquery: Pick<BigQueryClient, 'getTable'>;
+    biglake: Pick<BigLakeClient, 'getCatalog' | 'getHiveCatalog'>;
+    spanner: Pick<SpannerClient, 'getInstance' | 'getInstanceConfig'>;
+}
+export declare function sourceEntryErrors(model: Pick<SemanticModel, 'name' | 'entities' | 'profiles'>, readers: SourceReaders): Promise<string[]>;