@google-cloud/kcmd 0.0.0-stage → 0.2.0

package/build/ts/kcmd/semantic/runtime/runtime.js

@@ -0,0 +1,301 @@
+"use strict";
+// Creating a semantic runtime: the model as authored, merged with one binding
+// profile, paired with the store the deployment that uses that profile names.
+//
+// A semantic model on disk is a declaration. A runtime is that declaration
+// made operational -- bindings applied, inheritance resolved, a store to run
+// against. Every caller that does anything with a model wants the pair, so the
+// pair is what this hands back, rather than a model the caller then has to
+// find a store for.
+//
+// Both halves were the CLI's private business until an agent needed them. An
+// agent skill derives its tools from the same bound model `kcmd skills-generate`
+// reads, against the same store, so it has to reach them the same way -- and a
+// second loader that merges profiles slightly differently is a demo that passes
+// while the product fails. The CLI calls this too, so there is one answer to
+// "what does this scope say" rather than one per caller.
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runtimeStoreError = runtimeStoreError;
+exports.createSemanticRuntimes = createSemanticRuntimes;
+const alloydb_1 = require("../../gcp/alloydb");
+const context = __importStar(require("../../gcp/context"));
+const dataplex_1 = require("../../gcp/dataplex");
+const semantic_model_1 = require("../../layouts/semantic-model");
+const snapshot_1 = require("../../snapshot");
+const source_1 = require("../../source");
+const loader_1 = require("../loader");
+const resolve_inheritance_1 = require("../resolve_inheritance");
+const resolve_profiles_1 = require("../resolve_profiles");
+const source_dialect_1 = require("../source_dialect");
+const sql_parser_1 = require("../sql_parser");
+const store_1 = require("./store");
+/**
+ * Why this runtime cannot execute SQL DML statements, or null when it can.
+ *
+ * Having a store is the whole requirement. Every backend a profile can deploy
+ * to -- Spanner, AlloyDB, BigQuery -- executes DML, so the kind of store is not
+ * a reason to refuse an action.
+ */
+function runtimeStoreError(runtime) {
+    if (!runtime.store) {
+        return runtime.storeError ??
+            `Model '${runtime.model.name}' has no store under profile '${runtime.profile}'.`;
+    }
+    return null;
+}
+/**
+ * One runtime per model document in the scope, in discovery order.
+ *
+ * A lighter path than push's: nothing is pruned or validated for
+ * deployment, because running an action needs the model as authored rather
+ * than the subset a deployed graph can answer. Bindings stay optional, so a
+ * purely logical model yields a runtime with no store -- readable and
+ * inspectable, not runnable -- instead of an error that stops the scope.
+ */
+async function createSemanticRuntimes(options = {}) {
+    const base = options.path ?? '.';
+    const ctx = options.ctx ?? context.ApiContext.default();
+    const warn = options.onWarning ?? ((text) => console.warn(text));
+    let snapshot;
+    try {
+        snapshot = options.snapshot ?? await snapshot_1.CatalogSnapshot.fromPath(base, ctx);
+    }
+    catch (err) {
+        return { error: `${err?.message ?? err}` };
+    }
+    if (snapshot.manifest.source.type !== source_1.Sources.SEMANTIC_MODEL) {
+        return { error: `'${base}' is not a semantic-model scope.` };
+    }
+    const layout = snapshot.layout;
+    const source = snapshot.manifest.source;
+    const profile = options.profile ?? resolve_profiles_1.DEFAULT_PROFILE;
+    const docs = layout.modelDocuments();
+    if (!docs.length)
+        return { error: 'no semantic model documents found.' };
+    // Reading a GOOGLE block profile checks its field bindings for SQL, which
+    // needs the SQL parser.
+    try {
+        await (0, sql_parser_1.loadSqlEngine)();
+    }
+    catch (err) {
+        return { error: `${err?.message ?? err}` };
+    }
+    // The runtime reads every model's profiles first, as `kcmd push` does: its
+    // profile files, then the profiles a vanilla model carries in its GOOGLE
+    // block. It refuses the scope when
+    // profileDocuments rejects a file, such as a `<model>.profile.<name>.yaml`
+    // file whose `name:` does not match its file name, or when a GOOGLE block
+    // profile does not load. It also refuses it when a model declares a profile
+    // named `default` in any letter case.
+    const profilesByDoc = new Map();
+    // Each model's profile files alone, which an error applying one names.
+    const filesByDoc = new Map();
+    for (const doc of docs) {
+        let files;
+        let available;
+        try {
+            files = layout.profileDocuments(doc.name);
+            available = (0, resolve_profiles_1.declaredProfiles)(doc, files);
+        }
+        catch (err) {
+            return { error: err?.message ?? String(err) };
+        }
+        const reserved = (0, resolve_profiles_1.reservedProfileNameError)(doc.name, available);
+        if (reserved)
+            return { error: reserved };
+        profilesByDoc.set(doc.name, available);
+        filesByDoc.set(doc.name, files);
+    }
+    // A Cloud SQL table written as its resource name takes its dialect from its
+    // Knowledge Catalog entry, looked up once for the scope before anything
+    // loads. A table whose entry cannot be read stops the scope, as push does.
+    // A third-party table is not looked up, since no runtime reads one, and a
+    // Spanner table is SPANNER, the dialect the runtime's Spanner store speaks.
+    const lookup = await (0, source_dialect_1.sourceDialects)([...docs.map(d => d.text),
+        ...[...filesByDoc.values()].flat().map(f => f.text)], new dataplex_1.CatalogClient(ctx), { thirdParty: false });
+    if (lookup.errors.length)
+        return { error: lookup.errors.join('\n') };
+    const load = { sourceDialects: lookup.dialects };
+    // The model files the inline bindings are used for, loaded together.
+    const inline = [];
+    // A profile applies to its model file loaded on its own, with its
+    // exclusions already marked.
+    const appliedByDoc = new Map();
+    // The warnings applying each profile gave.
+    const fileWarnings = new Map();
+    for (const doc of docs) {
+        if (profile === resolve_profiles_1.DEFAULT_PROFILE) {
+            inline.push({ name: doc.name, text: doc.text });
+            continue;
+        }
+        const available = profilesByDoc.get(doc.name) ?? [];
+        const res = (0, resolve_profiles_1.profileForDocument)(doc, available, profile, load);
+        if (res.kind === 'missing') {
+            return { error: (0, resolve_profiles_1.unknownProfileMessage)(profile, doc.name, res.declared) };
+        }
+        if (res.kind === 'error') {
+            const files = (filesByDoc.get(doc.name) ?? []).map(p => ({ ...p, fileName: (0, semantic_model_1.fileNameOf)(layout, doc.name, p.path) }));
+            return { error: (0, resolve_profiles_1.profileErrorMessage)(doc.name, res, profile, files) };
+        }
+        fileWarnings.set(doc.name, res.warnings);
+        appliedByDoc.set(doc.name, res.model);
+    }
+    const loaded = (0, loader_1.loadSemanticModels)(inline, { bindingOptional: true, ...load });
+    if (loaded.error)
+        return { error: loaded.error };
+    // In the order of the documents, as one load of them all gives. The load
+    // prefixes each warning with its document's name.
+    for (const doc of docs) {
+        const own = fileWarnings.get(doc.name) ??
+            loaded.warnings.filter(w => w.startsWith(`[${doc.name}] `));
+        for (const w of own)
+            warn(`Warning: ${w}`);
+    }
+    const loadedByDoc = new Map(loaded.models.map(m => [m.document, m.model]));
+    const models = docs.flatMap(doc => {
+        const model = appliedByDoc.get(doc.name) ?? loadedByDoc.get(doc.name);
+        return model ? [{ document: doc.name, model }] : [];
+    });
+    // A deployment that names a profile the model does not declare would be
+    // used by no profile. Push refuses it too.
+    for (const { document, model } of models) {
+        const declared = (profilesByDoc.get(document) ?? []).map(p => p.name);
+        const error = (0, resolve_profiles_1.undeclaredProfileError)(document, model, declared);
+        if (error)
+            return { error };
+    }
+    // Push then loads every profile file, whichever profile it deploys, and
+    // refuses one that does not load.
+    for (const doc of docs) {
+        for (const p of filesByDoc.get(doc.name) ?? []) {
+            try {
+                (0, resolve_profiles_1.loadProfileFile)(p.text, p.name, { skipActions: true });
+            }
+            catch (err) {
+                if (!(err instanceof loader_1.SemanticModelError))
+                    throw err;
+                const file = (0, semantic_model_1.fileNameOf)(layout, doc.name, p.path);
+                return { error: `[${file}] ${err.message}` };
+            }
+        }
+    }
+    // kcmd has no engine that reads a third-party table.
+    for (const { document, model } of models) {
+        const bound = (model.entities ?? []).find(loader_1.bindsThirdPartyTable);
+        if (bound) {
+            return {
+                error: `[${document}] model '${model.name}' binds entity '${bound.name}' to third-party table '${bound.authoredSource}', and kcmd has no engine for a ` +
+                    `third-party table.`,
+            };
+        }
+    }
+    if (options.oneModel && models.length > 1) {
+        return {
+            error: typeof options.oneModel === 'function' ?
+                options.oneModel(models.length) :
+                `this scope holds ${models.length} models, which may name ` +
+                    `different databases, so there is no single store. Narrow ` +
+                    `the scope to one model.`,
+        };
+    }
+    // Inheritance is resolved for the same reason both push legs resolve it: an
+    // inherited field is a field, and every reader downstream reads
+    // `entity.fields`. The runtime is the reader where skipping it is unsafe
+    // rather than merely incomplete. It would not see a subtype's inherited
+    // `name`, so resolving a reference by name would report a row missing that
+    // is there; and it would not see an inherited key's TYPE, so the check that
+    // refuses a generated UUID for an Integer key would read the key as a
+    // String, pass, and let the store take the mismatch instead.
+    const runtimes = [];
+    const alloyDb = new alloydb_1.AlloyDbClient(ctx);
+    for (const { document, model: authored } of models) {
+        // An excluded metric is not available under the profile.
+        const excludedMetrics = new Set(authored.excludedMetrics ?? []);
+        if (excludedMetrics.size) {
+            authored.metrics =
+                (authored.metrics ?? []).filter(m => !excludedMetrics.has(m.name));
+        }
+        // The store is the one the profile's deployments name. Finding it reads
+        // only the deployment targets and the tables the model binds, which
+        // inheritance does not change.
+        const chosen = (0, store_1.resolveStore)(authored, profile);
+        // Resolving throws on an `extends` that names an entity the model does
+        // not declare, on a cycle, on a field inherited from two unrelated
+        // ancestors, and on two fields of one entity whose names differ only in
+        // case. The loader lets each through, and push reports it as an
+        // error. An AlloyDB database runs an inherited text as PostgreSQL, where
+        // `#` starts no comment.
+        let resolved;
+        try {
+            resolved = (0, resolve_inheritance_1.resolveInheritance)(authored, !('error' in chosen) && chosen.kind === 'alloydb' ? 'POSTGRES' :
+                undefined);
+        }
+        catch (err) {
+            const message = `${err?.message ?? err}`;
+            return {
+                error: `[${document}] model '${authored.name}': ${message}${message.endsWith('.') ? '' : '.'}`,
+            };
+        }
+        for (const w of resolved.warnings)
+            warn(`Warning: ${w}`);
+        // An AlloyDB database is reached through its cluster's primary instance.
+        const runtime = {
+            model: resolved.model,
+            document,
+            profile,
+            entryGroup: source.entryGroup,
+        };
+        if ('error' in chosen) {
+            runtime.storeError = chosen.error;
+            if (chosen.named)
+                runtime.storeFailed = true;
+            if (chosen.rejected)
+                runtime.deploymentsRejected = true;
+        }
+        else {
+            const store = await (0, store_1.withPrimaryInstance)(chosen, alloyDb);
+            if ('error' in store) {
+                runtime.storeError = store.error;
+                runtime.storeFailed = true;
+            }
+            else {
+                runtime.store = store;
+            }
+        }
+        runtimes.push(runtime);
+    }
+    return runtimes;
+}

package/build/ts/kcmd/semantic/runtime/store.d.ts

@@ -0,0 +1,112 @@
+import { AlloyDbClient } from '../../gcp/alloydb';
+import { SemanticModel } from '../ir';
+/** A Spanner database. An action's statements can write to it. */
+export interface SpannerStore {
+    kind: 'spanner';
+    /** `projects/<p>/instances/<i>/databases/<d>`. The store's identity. */
+    name: string;
+    project: string;
+    instance: string;
+    database: string;
+}
+/**
+ * An AlloyDB database. An action's statements can write to it. The instance a
+ * connection goes through may not be known yet.
+ */
+export interface AlloyDbDatabase {
+    kind: 'alloydb';
+    /**
+     * `projects/<p>/locations/<l>/clusters/<c>/databases/<d>`. The store's
+     * identity, which on AlloyDB reaches all the way down to the PostgreSQL
+     * database: one cluster holds several, and two of them are two stores.
+     */
+    name: string;
+    project: string;
+    location: string;
+    cluster: string;
+    database: string;
+}
+/** An AlloyDB database, with the instance a connection goes through. */
+export interface AlloyDbStore extends AlloyDbDatabase {
+    instance: string;
+}
+/** A BigQuery dataset. An action's statements can write to it. */
+export interface BigQueryStore {
+    kind: 'bigquery';
+    /** `projects/<p>/datasets/<d>`. The store's identity. */
+    name: string;
+    project: string;
+    dataset: string;
+}
+export type Store = SpannerStore | AlloyDbStore | BigQueryStore;
+/**
+ * The store resolveStore chooses. An AlloyDB database may still need its
+ * instance found with withPrimaryInstance.
+ */
+export type ChosenStore = SpannerStore | AlloyDbDatabase | BigQueryStore;
+/**
+ * Why resolveStore gives no store.
+ *
+ * `named` is set when the model's deployments name a target and still no
+ * store results, so a caller that runs against the store fails. `rejected` is
+ * set with it when the deployments give no store whatever the model is used
+ * for: a target in no accepted form in any deployment, a GOOGLE block whose
+ * data is not valid JSON, or several targets for the profile. `named` alone
+ * means the model binds a table outside its Spanner or AlloyDB store.
+ *
+ * Neither is set when the profile names no store, unless a target is in no
+ * accepted form, or a GOOGLE block is not valid JSON. Neither is set either
+ * when a BigQuery model binds a table outside the graph's dataset.
+ */
+export interface StoreError {
+    error: string;
+    named?: boolean;
+    rejected?: boolean;
+}
+/**
+ * The store a model runs against under binding profile `profile`, which
+ * defaults to `default`, the model file's own bindings. It is the one the
+ * deployments for that profile name: those that name it in `profile`, or that
+ * name none or `default` for `default`. A model with no `deployments` list
+ * names no profile for any of its targets, so they are all for `default`.
+ * With several targets, the store is the single operational one, Spanner or
+ * AlloyDB.
+ *
+ * Fails six ways:
+ *
+ * 1. A GOOGLE block's data is not valid JSON, so its targets cannot be read.
+ * 2. A target names no supported destination. Push checks the target of every
+ *    deployment, so this fails the store under every profile.
+ * 3. No deployment uses the profile.
+ * 4. There are several targets and no single operational one, so which store
+ *    is meant is ambiguous.
+ * 5. There is no target, so there is nothing to run against.
+ * 6. The model binds a table outside the store. For a Spanner or AlloyDB
+ *    store, a write would land in the wrong place, and the error is named. A
+ *    BigQuery graph may read tables in any dataset, so
+ *    for a BigQuery store the error is not named.
+ */
+export declare function resolveStore(model: SemanticModel, profile?: string): ChosenStore | StoreError;
+/**
+ * The targets binding profile `profile` uses, as resolveStore reads them: its
+ * deployments' targets, or every target in a GOOGLE block for `default` when
+ * the model has no `deployments` list. Empty when no deployment is for the
+ * profile. Throws when a GOOGLE block's data is not valid JSON.
+ */
+export declare function profileTargets(model: SemanticModel, profile: string): string[];
+/**
+ * `store` with the instance a connection goes through. An AlloyDB target
+ * names a database in a cluster, so this asks the AlloyDB API for the
+ * cluster's primary instance. Fails, naming the cluster, when it has none,
+ * when the primary is not READY, or when the listing fails. Any other store
+ * comes back unchanged.
+ */
+export declare function withPrimaryInstance(store: ChosenStore, client: AlloyDbClient): Promise<Store | {
+    error: string;
+}>;
+/**
+ * How a store is written down for a reader: the resource it addresses, with
+ * the backend named ahead of it for everything but Spanner, which is the one
+ * an unprefixed line has always meant.
+ */
+export declare function storeLine(store: Store): string;