@google-cloud/kcmd 0.0.0-stage → 0.2.0

package/build/ts/kcmd/sources/bq-dataset.js

@@ -0,0 +1,131 @@
+"use strict";
+// BigQuery Dataset as Metadata Source
+//
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BigQueryDatasetSource = void 0;
+const gcp = __importStar(require("../gcp"));
+const bq = __importStar(require("../gcp/bigquery"));
+const layout_1 = require("../layout");
+const md = __importStar(require("../metadata"));
+class BigQueryDatasetSource {
+    type;
+    name;
+    ingestedEntries = true;
+    layout = layout_1.Layouts.STANDARD;
+    defaultAspects = [md.OVERVIEW_ASPECT];
+    // BigQuery and Dataplex data scans write these on a table besides the ones
+    // its type requires.
+    systemAspects = [
+        'dataplex-types.global.bigquery-policy',
+        'dataplex-types.global.data-profile',
+        'dataplex-types.global.data-quality-scorecard',
+        'dataplex-types.global.usage',
+    ];
+    _datasets;
+    constructor(type, name, datasets) {
+        this.type = type;
+        this.name = name;
+        this._datasets = datasets;
+    }
+    async *entries(ctx) {
+        const bigQuery = new bq.BigQueryClient(ctx);
+        const catalog = new gcp.CatalogClient(ctx);
+        for (const [_, dsResource] of this._datasets.entries()) {
+            const project = dsResource.datasetReference.projectId;
+            const dataset = dsResource.datasetReference.datasetId;
+            // Fetch the dataset entry
+            const location = dsResource.location.toLowerCase();
+            const dsEntryId = `bigquery.googleapis.com/projects/${project}/datasets/${dataset}`;
+            const dsEntryName = `${gcp.catalogContainer(project, location, '@bigquery')}/entries/${dsEntryId}`;
+            const dsEntryResult = await catalog.lookupEntry(project, location, dsEntryName);
+            if (!dsEntryResult.result) {
+                throw new Error(`Failed to get Entry for dataset ${project}.${dataset}`);
+            }
+            yield dsEntryResult.result;
+            // Fetch the table entries
+            for await (const table of bigQuery.listTables(project, dataset)) {
+                const tableId = table.tableReference.tableId;
+                const tableEntryName = `${dsEntryName}/tables/${tableId}`;
+                const tableEntryResult = await catalog.lookupEntry(project, location, tableEntryName);
+                if (!tableEntryResult.result) {
+                    throw new Error(`Failed to get Entry for table ${project}.${dataset}.${tableId}`);
+                }
+                yield tableEntryResult.result;
+            }
+        }
+        // TODO: Add support for routines, models
+    }
+    localName(entry) {
+        // The local catalog uses simplified path scheme:
+        // dataset -> <project id>.<dataset id>
+        // table -> <project id>.<dataset id>/<table id>
+        // model -> <project id>.<dataset id>/models/<model id>
+        // routine -> <project id>.<dataset id>/routines/<routine id>
+        let match = entry.name.match(/\/projects\/([^/]+)\/datasets\/([^/]+)\/(tables|models|routines)\/(.+)$/);
+        if (match) {
+            const [, project, dataset, type, id] = match;
+            if (type === 'tables') {
+                return `${project}.${dataset}/${id}`;
+            }
+            return `${type}/${project}.${dataset}/${id}`;
+        }
+        match = entry.name.match(/\/projects\/([^/]+)\/datasets\/([^/]+)$/);
+        if (match) {
+            const [, project, dataset] = match;
+            return `${project}.${dataset}`;
+        }
+        throw new Error(`Invalid BigQuery entry: ${entry.name}`);
+    }
+    serviceName(localName) {
+        const nameParts = localName.split('/');
+        const dsResource = this._datasets.get(nameParts[0]);
+        if (!dsResource) {
+            throw new Error(`Failed to find dataset for ${nameParts[0]}`);
+        }
+        const project = dsResource.datasetReference.projectId;
+        const location = dsResource.location.toLowerCase();
+        const dataset = dsResource.datasetReference.datasetId;
+        const entryGroup = `${gcp.catalogContainer(project, location, '@bigquery')}`;
+        const entryName = `${entryGroup}/entries/bigquery.googleapis.com/projects/${project}/datasets/${dataset}`;
+        if (nameParts.length == 1) {
+            return entryName;
+        }
+        const collection = nameParts.length == 2 ? 'tables' : nameParts[1];
+        const resource = nameParts.length == 2 ? nameParts[1] : nameParts[2];
+        return `${entryName}/${collection}/${resource}`;
+    }
+}
+exports.BigQueryDatasetSource = BigQueryDatasetSource;

package/build/ts/kcmd/sources/entrygroup.d.ts

@@ -0,0 +1,21 @@
+import * as gcp from '../gcp';
+import * as dataplex from '../gcp/dataplex';
+import { Layouts } from '../layout';
+import { CatalogSource } from '../source';
+export declare class EntryGroupSource implements CatalogSource {
+    readonly type: string;
+    readonly name: string;
+    readonly ingestedEntries: boolean;
+    readonly layout = Layouts.STANDARD;
+    readonly defaultAspects: string[];
+    readonly systemAspects: never[];
+    readonly project: string;
+    readonly location: string;
+    readonly entryGroup: string;
+    private readonly _name;
+    private readonly _entryGroup;
+    constructor(type: string, name: string, entryGroup: dataplex.EntryGroup);
+    entries(ctx: gcp.ApiContext): AsyncGenerator<gcp.Entry, void, unknown>;
+    localName(entry: gcp.Entry): string;
+    serviceName(localName: string): string;
+}

package/build/ts/kcmd/sources/entrygroup.js

@@ -0,0 +1,83 @@
+"use strict";
+// Dataplex EntryGroup as Metadata Source
+//
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.EntryGroupSource = void 0;
+const gcp = __importStar(require("../gcp"));
+const layout_1 = require("../layout");
+const md = __importStar(require("../metadata"));
+class EntryGroupSource {
+    type;
+    name;
+    ingestedEntries;
+    layout = layout_1.Layouts.STANDARD;
+    defaultAspects = [md.OVERVIEW_ASPECT];
+    systemAspects = [];
+    project;
+    location;
+    entryGroup;
+    _name;
+    _entryGroup;
+    constructor(type, name, entryGroup) {
+        this.type = type;
+        this.name = name;
+        this._name = name.split('.');
+        this.project = this._name[0];
+        this.location = this._name[1];
+        this.entryGroup = this._name[2];
+        this._entryGroup = entryGroup;
+        this.ingestedEntries = this._name[2].startsWith('@');
+    }
+    async *entries(ctx) {
+        // Enumerate all entries in the EntryGroup
+        const catalog = new gcp.CatalogClient(ctx);
+        for await (const entry of catalog.listEntries(this._name[0], this._name[1], this._name[2])) {
+            yield entry;
+        }
+    }
+    localName(entry) {
+        // The local catalog uses the entry id as is
+        const match = entry.name.match(/entryGroups\/([^/]+)\/entries\/(.+)$/);
+        if (!match) {
+            throw new Error(`Invalid entry name for entry: ${entry.name}`);
+        }
+        return match[2];
+    }
+    serviceName(localName) {
+        return `${this._entryGroup.name}/entries/${localName}`;
+    }
+}
+exports.EntryGroupSource = EntryGroupSource;

package/build/ts/kcmd/sources/kb.d.ts

@@ -0,0 +1,21 @@
+import * as gcp from '../gcp';
+import * as dataplex from '../gcp/dataplex';
+import { Layouts } from '../layout';
+import { CatalogSource } from '../source';
+export declare class KnowledgeBaseSource implements CatalogSource {
+    readonly type: string;
+    readonly name: string;
+    readonly ingestedEntries = false;
+    readonly layout = Layouts.DOCUMENTS;
+    readonly defaultAspects: string[];
+    readonly systemAspects: never[];
+    readonly project: string;
+    readonly location: string;
+    readonly entryGroup: string;
+    private readonly _name;
+    private readonly _entryGroup;
+    constructor(type: string, name: string, entryGroup: dataplex.EntryGroup);
+    entries(ctx: gcp.ApiContext): AsyncGenerator<gcp.Entry, void, unknown>;
+    localName(entry: gcp.Entry): string;
+    serviceName(localName: string): string;
+}

package/build/ts/kcmd/sources/kb.js

@@ -0,0 +1,82 @@
+"use strict";
+// Dataplex Wiki managed as an EntryGroup as Metadata Source
+//
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.KnowledgeBaseSource = void 0;
+const gcp = __importStar(require("../gcp"));
+const layout_1 = require("../layout");
+const md = __importStar(require("../metadata"));
+class KnowledgeBaseSource {
+    type;
+    name;
+    ingestedEntries = false;
+    layout = layout_1.Layouts.DOCUMENTS;
+    defaultAspects = [md.OVERVIEW_ASPECT];
+    systemAspects = [];
+    project;
+    location;
+    entryGroup;
+    _name;
+    _entryGroup;
+    constructor(type, name, entryGroup) {
+        this.type = type;
+        this.name = name;
+        this._name = name.split('.');
+        this.project = this._name[0];
+        this.location = this._name[1];
+        this.entryGroup = this._name[2];
+        this._entryGroup = entryGroup;
+    }
+    async *entries(ctx) {
+        // Enumerate all entries in the EntryGroup
+        const catalog = new gcp.CatalogClient(ctx);
+        for await (const entry of catalog.listEntries(this._name[0], this._name[1], this._name[2])) {
+            yield entry;
+        }
+    }
+    localName(entry) {
+        // The local catalog uses the entry id as is
+        const match = entry.name.match(/entryGroups\/([^/]+)\/entries\/(.+)$/);
+        if (!match) {
+            throw new Error(`Invalid entry name for entry: ${entry.name}`);
+        }
+        return match[2];
+    }
+    serviceName(localName) {
+        return `${this._entryGroup.name}/entries/${localName}`;
+    }
+}
+exports.KnowledgeBaseSource = KnowledgeBaseSource;

package/build/ts/kcmd/sources/semantic-model.d.ts

@@ -0,0 +1,18 @@
+import * as gcp from '../gcp';
+import { Layouts } from '../layout';
+import { CatalogSource } from '../source';
+export declare class SemanticModelSource implements CatalogSource {
+    readonly type: string;
+    readonly name: string;
+    readonly ingestedEntries = false;
+    readonly layout = Layouts.SEMANTIC_MODEL;
+    readonly defaultAspects: never[];
+    readonly systemAspects: never[];
+    readonly project: string;
+    readonly location: string;
+    readonly entryGroup: string;
+    constructor(type: string, name: string);
+    entries(_ctx: gcp.ApiContext): AsyncGenerator<gcp.Entry, void, unknown>;
+    localName(_entry: gcp.Entry): string;
+    serviceName(_localName: string): string;
+}

package/build/ts/kcmd/sources/semantic-model.js

@@ -0,0 +1,48 @@
+"use strict";
+// A locally-authored Semantic Model as a Metadata Source
+//
+// The model is authored on the local filesystem as a single Apache Ossie
+// document per model (the SemanticModel layout); it is not ingested from a
+// remote service. `push` deploys the model's BigQuery Graph.
+//
+// Knowledge Catalog resource emit (entries + entryLinks) for the semantic model
+// is a follow-on; the sync-oriented members below are therefore not yet wired.
+//
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SemanticModelSource = void 0;
+const layout_1 = require("../layout");
+class SemanticModelSource {
+    type;
+    name;
+    ingestedEntries = false;
+    layout = layout_1.Layouts.SEMANTIC_MODEL;
+    // A semantic model is pulled by its own path, not by snapshot aspects.
+    defaultAspects = [];
+    systemAspects = [];
+    project;
+    location;
+    entryGroup;
+    constructor(type, name) {
+        const [project, location, entryGroup] = name.split('.');
+        if (!project || !location || !entryGroup) {
+            throw new Error('Semantic model scope must be in format <projectId>.<locationId>.<entryGroupId>');
+        }
+        this.type = type;
+        this.name = name;
+        this.project = project;
+        this.location = location;
+        this.entryGroup = entryGroup;
+    }
+    // The model is authored locally; there is nothing to enumerate from a
+    // service.
+    async *entries(_ctx) {
+        return;
+    }
+    localName(_entry) {
+        throw new Error('Knowledge Catalog resource sync is not supported for the semantic-model scope yet.');
+    }
+    serviceName(_localName) {
+        throw new Error('Knowledge Catalog resource sync is not supported for the semantic-model scope yet.');
+    }
+}
+exports.SemanticModelSource = SemanticModelSource;

package/build/ts/kcmd/sync.d.ts

@@ -0,0 +1,35 @@
+import * as gcp from './gcp';
+import { CatalogSnapshot } from './snapshot';
+export interface SyncResult {
+    success: boolean;
+    details?: string;
+}
+export interface PullSyncOptions {
+    dryRun?: boolean;
+    forceRemove?: boolean;
+}
+export interface ValidationResult {
+    valid: boolean;
+}
+export interface StatusResult {
+    modified: boolean;
+}
+export declare function mapConcurrent<T, R>(items: readonly T[], limit: number, fn: (item: T, index: number) => Promise<R>): Promise<R[]>;
+export declare function isUnimplementedBatch(res: {
+    status: number;
+    body?: string;
+}): boolean;
+export declare class CatalogSync {
+    private _catalog;
+    private _snapshot;
+    constructor(catalog: gcp.CatalogClient, snapshot: CatalogSnapshot);
+    pull(options?: PullSyncOptions): Promise<SyncResult>;
+    push(options?: {
+        force?: boolean;
+        validateOnly?: boolean;
+    }): Promise<SyncResult>;
+    private _pullOkf;
+    private _pushOkf;
+    validate(): Promise<ValidationResult>;
+    status(): Promise<StatusResult>;
+}