@google-cloud/kcmd 0.1.0 → 0.2.0

package/build/ts/kcmd/semantic/deploy_knowledge_catalog.js

@@ -19,12 +19,11 @@
 //
 // Publish sequence (split into read-only `preflightKnowledgeCatalog` and write
 // `applyKnowledgeCatalog`):
-//   * Preflight validates that exactly one model is deployed per entry group
-//     and that each file copy fits in one aspect, snapshots the destination
-//     entry group to guard against foreign models (unless `--force-remove` is
-//     set), and checks that any required custom entry/aspect types exist in
-//     the target project. With the second-generation aspects, it also checks
-//     every aspect's size.
+//   * Preflight validates that exactly one model is deployed per entry group,
+//     snapshots the destination entry group to guard against foreign models
+//     (unless `--force-remove` is set), and checks that any required custom
+//     entry/aspect types exist in the target project. With the
+//     second-generation aspects, it also checks every aspect's size.
 //   * Apply creates or updates each model's entries in dependency order: the
 //     `semantic-model` anchor first, then independent child entries
 //     concurrently, and finally dependent child entries (`semantic-metric` and
@@ -44,6 +43,8 @@
 //   * Reconcile deletions: after writing, delete any child entry this push owns
 //     (by `ownedPrefixes`) that is present in the pre-write listing but was not
 //     re-emitted.
+//   * The anchor's write labels it `kcmd-push-state: writing`, and the push's
+//     last write, after the deletions, relabels it `finished`.
 //
 // This is a library module: it emits no console output. Warnings and the
 // dry-run plan are returned in `KcDeployResult` for the CLI (commands.ts) to
@@ -88,9 +89,7 @@ exports.applyKnowledgeCatalog = applyKnowledgeCatalog;
 exports.deployKnowledgeCatalog = deployKnowledgeCatalog;
 exports.deployEmittedModels = deployEmittedModels;
 exports.oneModelPerEntryGroupMessage = oneModelPerEntryGroupMessage;
-exports.checkFileCopies = checkFileCopies;
 exports.checkCatalogAspects = checkCatalogAspects;
-exports.checkCustomTypes = checkCustomTypes;
 const crm = __importStar(require("../gcp/crm"));
 const dataplex_1 = require("../gcp/dataplex");
 const kc_custom_types_1 = require("./kc_custom_types");
@@ -109,14 +108,13 @@ function sleep(ms) {
  *
  * The method performs the following actions:
  *   1. Translates the IR models into catalog resources.
- *   2. Verifies that exactly one model is being deployed and that each file
- *      copy fits in one aspect, builds the dry-run plan, and returns early
- *      when `opts.validateOnly` is set.
- *   3. Lists the destination entry group and fails if an unrecognized model's
- *      anchor entry is present without `--force-remove`.
+ *   2. Verifies that exactly one model is being deployed, builds the dry-run
+ *      plan, and returns early when `opts.validateOnly` is set.
+ *   3. Fails when the destination entry group does not exist. Lists it and
+ *      fails if an unrecognized model's anchor entry is present without
+ *      `--force-remove`.
  *   4. Verifies that every custom entry type and aspect type required by the
- *      model exists in the target project. Skips the file-copy type when
- *      `opts.fileCopyTypeChecked` is set.
+ *      model exists in the target project.
  *   5. Returns the emitted resources, entry-group listing, warnings, and plan
  *      as `prepared` for `applyKnowledgeCatalog`.
  * With `opts.v2Aspects` set, step 2 also checks every aspect's size.
@@ -157,20 +155,10 @@ async function applyKnowledgeCatalog(prepared, ctx, opts) {
     const { emitted, existing } = prepared;
     // Resource Manager is asked for the project number at most once per push.
     const isScopeProject = scopeProjectMatcher(ctx, opts.project);
-    // Some of this model's writes may have landed before a failure in them.
-    // The catalog may then lack some or all of this push's file copies, and
-    // pull refuses that until a push succeeds.
-    const storesCopies = emitted.some(m => m.resources.entries.some(e => (0, kc_entries_1.entryTypeId)(e) === kc_custom_types_1.FILE_COPY_TYPE_ID));
-    const failed = (details) => result(false, storesCopies ?
-        `${details.replace(/\.$/, '')}. The catalog may now lack some or ` +
-            `all of this push's file copies, so push again before you pull.` :
-        details);
     if (opts.forceRemove) {
         const foreignAnchorIds = findForeignAnchorIds(emitted[0], existing);
         if (foreignAnchorIds.length) {
-            // This runs before any of this model's writes. A failure here leaves
-            // none of this push's file copies, so it gets the note too, and so
-            // does a thrown error.
+            // A thrown error is reported like a failed delete.
             const removed = await removeForeignModels(cat, opts, existing, foreignAnchorIds, isScopeProject)
                 .catch((err) => ({
                 deleted: 0,
@@ -180,7 +168,7 @@ async function applyKnowledgeCatalog(prepared, ctx, opts) {
             counts.deleted += removed.deleted;
             counts.unlinked += removed.unlinked;
             if (removed.error)
-                return failed(removed.error);
+                return result(false, removed.error);
         }
     }
     // Write each emitted model's entries and relationship links and delete any
@@ -190,7 +178,7 @@ async function applyKnowledgeCatalog(prepared, ctx, opts) {
     const written = await writeEmittedModels(cat, opts, emitted, existing, counts, isScopeProject)
         .catch((err) => ({ error: String(err?.message || err).trim() }));
     if (written.error)
-        return failed(written.error);
+        return result(false, written.error);
     return result(true);
 }
 /**
@@ -223,15 +211,15 @@ async function deployEmittedModels(emitted, emitWarnings, ctx, opts) {
  * Runs read-only preflight checks on already-emitted models.
  *
  * The method performs the following checks:
- *   1. Validates locally that `emitted` contains a single model, that each
- *      file copy fits in one aspect and, with `opts.v2Aspects` set, that every
- *      aspect size and instance count is within its limit. Builds the
- *      dry-run plan and returns early when `opts.validateOnly` is set.
- *   2. Lists the destination entry group and validates that it contains no
- *      foreign models unless `--force-remove` is set.
+ *   1. Validates locally that `emitted` contains a single model and, with
+ *      `opts.v2Aspects` set, that every aspect size and instance count is
+ *      within its limit. Builds the dry-run plan and returns early when
+ *      `opts.validateOnly` is set.
+ *   2. Validates that the destination entry group exists. Lists it and
+ *      validates that it contains no foreign models unless `--force-remove`
+ *      is set.
  *   3. Validates that every custom type used by the model exists in the
- *      target project. Skips the file-copy type when
- *      `opts.fileCopyTypeChecked` is set.
+ *      target project.
  */
 async function preflightEmittedModels(emitted, emitWarnings, ctx, opts) {
     const warnings = [...emitWarnings];
@@ -246,9 +234,6 @@ async function preflightEmittedModels(emitted, emitWarnings, ctx, opts) {
     // Once `validateEmittedModels` passes and `emitted` is non-empty, `emitted`
     // is guaranteed to contain exactly one model (`emitted[0]`).
     const model = emitted[0];
-    const sizeValidation = validateFileCopySizes(model, result);
-    if (!sizeValidation.success)
-        return { result: sizeValidation };
     const aspectsError = opts.v2Aspects ? emittedAspectsError(model) : undefined;
     if (aspectsError)
         return { result: result(false, aspectsError) };
@@ -260,6 +245,14 @@ async function preflightEmittedModels(emitted, emitWarnings, ctx, opts) {
     //    reconciliation. A re-emitted entry is never a deletion candidate, so a
     //    pre-write snapshot is correct for all three.
     const cat = new dataplex_1.CatalogClient(ctx);
+    // The listing reads a missing group as empty, so read the group itself.
+    const group = await cat.getEntryGroup(opts.project, opts.location, opts.entryGroup)
+        .catch((err) => `reading entry group '${opts.entryGroup}': ${err?.message || String(err)}`);
+    if (typeof group === 'string')
+        return { result: result(false, group) };
+    if (isNotFoundStatus(group)) {
+        return { result: result(false, missingEntryGroupMessage(opts)) };
+    }
     const listing = await listEntryGroup(cat, opts);
     if (listing.error)
         return { result: result(false, listing.error) };
@@ -315,6 +308,14 @@ function oneModelPerEntryGroupMessage(entryGroup, models) {
         `only one model per entry group is supported; split them into ` +
         `separate entry groups.`;
 }
+/** The error for a push to an entry group that does not exist. */
+function missingEntryGroupMessage(opts) {
+    const { project, location, entryGroup } = opts;
+    return `entry group '${entryGroup}' does not exist in location ` +
+        `'${location}' of project '${project}'; run 'kcmd init ` +
+        `--semantic-model ${project}.${location}.${entryGroup}' to create it, ` +
+        `or correct the scope in catalog.yaml.`;
+}
 /**
  * Validates that `existing` (the destination entry-group snapshot) contains no
  * foreign `semantic-model` anchors unless `--force-remove` is set.
@@ -339,62 +340,9 @@ function validateEntryGroup(model, existing, opts, result) {
 exports.MAX_ASPECT_DATA_BYTES = 120 * 1024;
 const MAX_SCHEMA_ASPECT_DATA_BYTES = 2 * 1024 * 1024;
 /**
- * Validates that every file copy in `model` fits in one aspect.
- *
- * A file too large to store then fails before any write, naming the file,
- * rather than partway through the writes. Push runs the same check on its
- * copies, with `checkFileCopies`, before any graph deploys.
- */
-function validateFileCopySizes(model, result) {
-    for (const entry of model.resources.entries) {
-        if ((0, kc_entries_1.entryTypeId)(entry) !== kc_custom_types_1.FILE_COPY_TYPE_ID)
-            continue;
-        for (const aspect of Object.values(entry.aspects ?? {})) {
-            const error = fileCopySizeError(aspect.data ?? {});
-            if (error)
-                return result(false, error);
-        }
-    }
-    return result(true);
-}
-/**
- * Checks a push's file copies before anything is written. Returns the error
- * to report, or undefined when every copy fits in one aspect and no two file
- * names map to one entry id.
- *
- * Push calls it before any graph deploys. Every copy of a model shares the
- * model's part of the entry id, so two file names collide when their slugs
- * do.
- */
-function checkFileCopies(files) {
-    const bySlug = new Map();
-    for (const file of files) {
-        const tooLarge = fileCopySizeError(kcEmit.fileCopyData(file));
-        if (tooLarge)
-            return tooLarge;
-        const earlier = bySlug.get((0, kc_ids_1.slug)(file.fileName));
-        if (earlier !== undefined) {
-            return `files '${earlier}' and '${file.fileName}' both map to one ` +
-                `entry id; rename one of them`;
-        }
-        bySlug.set((0, kc_ids_1.slug)(file.fileName), file.fileName);
-    }
-    return undefined;
-}
-// Why the aspect data of one file copy is too large to store, or undefined
-// when it fits.
-function fileCopySizeError(data) {
-    const bytes = new TextEncoder().encode(JSON.stringify(data)).length;
-    if (bytes <= exports.MAX_ASPECT_DATA_BYTES)
-        return undefined;
-    return `file '${data.fileName}' is too large to store in Knowledge ` +
-        `Catalog: its copy takes ${bytes} bytes, and one aspect holds at most ` +
-        `${exports.MAX_ASPECT_DATA_BYTES}.`;
-}
-/**
- * The emitter's warnings for `models` with v2Aspects on, and why push's
- * catalog leg cannot write them: an emit error or an aspect too large to
- * store. Calls no service.
+ * The emitter's warnings for `models`, and an emit error or the first aspect
+ * too large to store. With v2Aspects on, that error is why push's catalog leg
+ * cannot write them. Calls no service.
  */
 function checkCatalogAspects(models, opts) {
     const emit = emitModels(models, opts);
@@ -446,8 +394,7 @@ function aspectTypeIdOfKey(key) {
 }
 /**
  * Validates that every kcmd-owned custom type (`semantic-action`,
- * `semantic-constraint`, `semantic-model-file`) used by `model` exists in the
- * target project.
+ * `semantic-constraint`) used by `model` exists in the target project.
  *
  * Unlike built-in system types in `dataplex-types`, custom entry and aspect
  * types are provisioned in the destination project by `kcmd init`;
@@ -455,22 +402,15 @@ function aspectTypeIdOfKey(key) {
  * before any catalog entries or entry links are written, rather than failing
  * mid-push when creating an action or constraint entry.
  *
- * Skips the file-copy type when `opts.fileCopyTypeChecked` is set, because
- * the caller has already looked it up. Makes no network calls when no type
- * is left to check.
+ * Makes no network calls when the model uses no custom type.
  */
 async function validateCustomTypes(cat, model, opts, result) {
-    const typeIds = (0, kc_custom_types_1.collectCustomTypes)(model.resources.entries).filter(id => !(opts.fileCopyTypeChecked && id === kc_custom_types_1.FILE_COPY_TYPE_ID));
-    const error = await checkCustomTypes(cat, opts, typeIds);
+    const error = await checkCustomTypes(cat, opts, (0, kc_custom_types_1.collectCustomTypes)(model.resources.entries));
     return error ? result(false, error) : result(true);
 }
 /**
  * Checks that each custom type in `typeIds` exists in the destination
  * project. Returns the error to report, or undefined when every one exists.
- *
- * Push also calls it before any graph deploys, for the file-copy type every
- * push with v2Aspects off needs. Push then sets `fileCopyTypeChecked`, so the
- * catalog preflight looks up only the other custom types.
  */
 async function checkCustomTypes(cat, dest, typeIds) {
     const home = (0, kc_custom_types_1.customTypeHome)(dest);
@@ -517,7 +457,6 @@ function emitModels(models, opts) {
                 systemTypeLocation: opts.systemTypeLocation,
                 emitExpressions: opts.emitExpressions,
                 v2Aspects: opts.v2Aspects,
-                sourceFiles: opts.sourceFiles,
                 overviews: opts.overviews,
             });
         }
@@ -571,7 +510,8 @@ function planSummary(model, resources, opts) {
  * resources in model order.
  *
  * For each model, the method performs the following actions:
- *   1. Creates or updates the model's entries (anchor first).
+ *   1. Creates or updates the model's entries (anchor first, labeled as
+ *      writing).
  *   2. Looks up the model's existing relationship links when the pre-write
  *      entry group contains entities owned by the model.
  *   3. Deletes any orphaned relationship links (no longer emitted) and stale
@@ -581,6 +521,7 @@ function planSummary(model, resources, opts) {
  *   4. Creates or updates the model's relationship links.
  *   5. Deletes any existing entries owned by the model that are no longer
  *      emitted.
+ *   6. Labels the anchor finished.
  *
  * Progress accumulates into `counts`, so a mid-way failure still reports what
  * had been written.
@@ -684,9 +625,40 @@ async function writeEmittedModels(cat, opts, emitted, existing, counts, isScopeP
                 error: `Deleting orphaned entry '${entryId}' from model '${model}' failed: ${message}`,
             };
         }
+        if (resources.entries.length) {
+            const anchor = resources.entries[0];
+            const error = await finishPush(cat, opts, anchor)
+                .catch((err) => String(err?.message || err).trim() || 'unknown error');
+            if (error) {
+                return {
+                    error: `Model '${model}': setting the ${kc_custom_types_1.PUSH_STATE_LABEL} label of ` +
+                        `entry '${(0, kc_entries_1.entryId)(anchor)}' to ${kc_custom_types_1.PUSH_FINISHED} failed: ` +
+                        `${error.replace(/\.$/, '')}. Push again before you pull.`,
+                };
+            }
+        }
     }
     return {};
 }
+/**
+ * Labels the model entry `anchor` finished, and returns the error when that
+ * fails. It reads the entry first and sends back the entrySource the entry
+ * holds then, so what another writer set during the push stays. The update
+ * names only the entrySource.
+ */
+async function finishPush(cat, opts, anchor) {
+    const read = await cat.getEntry(opts.project, opts.location, opts.entryGroup, (0, kc_entries_1.entryId)(anchor), undefined, { keepSourceResource: true });
+    if (!isOkStatus(read))
+        return errText(read);
+    const { name, entryType } = anchor;
+    const entrySource = withPushState(read.result?.entrySource, kc_custom_types_1.PUSH_FINISHED);
+    const res = await cat.updateEntry({ name, entryType, entrySource }, ['entry_source']);
+    return isOkStatus(res) ? undefined : errText(res);
+}
+// `source` with its kcmd-push-state label set to `state`.
+function withPushState(source, state) {
+    return { ...source, labels: { ...source?.labels, [kc_custom_types_1.PUSH_STATE_LABEL]: state } };
+}
 /**
  * Returns the push-again hint for an error, or '' when no link is missing.
  *
@@ -1032,8 +1004,8 @@ async function removeForeignModels(cat, opts, existing, foreignAnchorIds, isScop
  * Creates or updates a model's entries in dependency order.
  *
  * The method writes entries in three stages:
- *   1. Writes the model anchor (`entries[0]`) first, since it is the parent of
- *      every child entry.
+ *   1. Writes the model anchor (`entries[0]`) first, labeled as writing, since
+ *      it is the parent of every child entry.
  *   2. Writes independent child entries (such as entities, relationships,
  *      actions, and constraints) concurrently.
  *   3. Writes dependent child entries (`semantic-metric` via
@@ -1051,7 +1023,8 @@ async function writeEntries(cat, opts, entries, existing = []) {
     const sources = new Map(existing.map(e => [(0, kc_entries_1.entryId)(e), e.entrySource]));
     const write = (e) => writeEntry(cat, opts, e, sources);
     const [anchor, ...children] = entries;
-    const anchorRes = await write(anchor);
+    const entrySource = withPushState(anchor.entrySource, kc_custom_types_1.PUSH_WRITING);
+    const anchorRes = await write({ ...anchor, entrySource });
     if (anchorRes.error)
         return { created: 0, updated: 0, error: anchorRes.error };
     let created = anchorRes.updated ? 0 : 1;
@@ -1314,18 +1287,16 @@ const KEPT_SOURCE_FIELDS = ['resource', 'system', 'platform', 'createTime', 'upd
  *
  * The update mask replaces the whole entrySource, so the update carries over
  * fields from `source`, the entrySource the entry had before this push. Of
- * those fields kcmd owns the display name, the description and the
- * `kcmd-push-id` label. The update keeps another writer's `resource`,
- * `system`, `platform`, `createTime` and `updateTime`, and every label but
- * `kcmd-push-id`. It sets that label when `entry` carries it and drops it
- * when not. Dataplex refuses an update that leaves out an `updateTime` the
- * entry has, or moves it back. A kept field another writer changes after
- * `source` was read is written back with the old value. For `updateTime` the
- * update fails instead.
+ * those fields kcmd owns the display name and the description. The update
+ * keeps another writer's `resource`, `system`, `platform`, `createTime` and
+ * `updateTime`, and every label. A label `entry` carries, such as the model
+ * entry's `kcmd-push-state`, replaces the stored one. Dataplex refuses an
+ * update that leaves out an `updateTime` the entry has, or moves it back. A
+ * kept field another writer changes after `source` was read is written back
+ * with the old value. For `updateTime` the update fails instead.
  */
 async function updateEntry(cat, opts, entry, source) {
-    const { [kc_custom_types_1.PUSH_ID_LABEL]: _own, ...others } = source?.labels ?? {};
-    const labels = { ...others, ...entry.entrySource?.labels };
+    const labels = { ...source?.labels, ...entry.entrySource?.labels };
     const kept = Object.fromEntries(KEPT_SOURCE_FIELDS.filter(k => source?.[k] !== undefined)
         .map(k => [k, source[k]]));
     const merged = {

package/build/ts/kcmd/semantic/graph_ddl.d.ts

@@ -7,6 +7,16 @@ export interface MetricPlacementInput {
     entityByName: ReadonlyMap<string, Entity>;
     skipped: ReadonlySet<string>;
     ancestorsUsed: ReadonlySet<string>;
+    renderedFieldSql(entity: Entity, field: Field): {
+        sql: string;
+    } | {
+        reads: string;
+    };
+    localSql(entity: Entity, sql: string): {
+        sql: string;
+    } | {
+        reads: string;
+    };
 }
 export interface GraphEngine {
     name: string;