@google-cloud/kcmd 0.0.0-stage → 0.2.0

package/build/ts/kcmd/semantic/kc_constraints.js

@@ -0,0 +1,279 @@
+"use strict";
+// How a model's CONSTRAINTS are encoded in Knowledge Catalog.
+//
+// One entry per constraint, parented to the model anchor, carrying one aspect
+// that holds the invariant. An action is published the same way. Dataplex has
+// no built-in constraint type, so the entry and aspect types are the custom
+// `semantic-constraint` pair, declared in kc_custom_types.ts and created by
+// `kcmd init --semantic-model`. That file lists what is custom; this one holds
+// the encoding that fills the aspect.
+//
+// An entry of its own is what makes the invariant governable. A search can list
+// every rule a model states, whichever way the rule is settled. Dropping a
+// constraint from the model deletes its entry, so the catalog never advertises
+// a rule the model stopped requiring.
+//
+// The five authored fields land in two places. `judgment`, `on_violation`,
+// `severity` and `ai_context` go on the aspect, the last whole --
+// aiContextField in kc_custom_types.ts says why. `description` goes on the
+// entry source, where a pull of an action already reads it, and because a
+// violation quotes that sentence back to the caller as the error, which makes
+// it the entry's summary.
+//
+// The aspect type still DECLARES `expression` and `evaluation`, the two fields
+// of the removed deterministic body, and nothing here writes either. They stay
+// declared so an aspect type already created in a project keeps matching the
+// one `kcmd init` would create, and so a catalog that still holds an
+// expression-only constraint from an older push reads back as a skip with a
+// reason rather than as a silent empty rule. See readConstraint.
+//
+// Both routing words are on the aspect because they are machine-readable, and
+// they are two fields because they answer different questions: what the engine
+// does about a breach, and how grave the breach is. A constraint that states
+// neither is published without them and reads back the same way, so a default
+// stays the IR's to define and a pull never invents a value the model never
+// stated.
+//
+// Call sites: knowledge_catalog.ts emits the entries, kc_converter.ts reads
+// them back, pull_kc.ts hydrates the aspect.
+//
+// WHEN A BUILT-IN CONSTRAINT TYPE SHIPS, follow the instructions at the top of
+// kc_custom_types.ts. Nothing in this file changes, because the readers below
+// match a type by its id suffix and ignore the project it lives in.
+//
+// The helpers at the bottom repeat a few lines from the modules above on
+// purpose. This module imports only the IR, the entry shape and the type
+// registry, so it can be swapped or deleted as a unit.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.constraintEntryTypeName = constraintEntryTypeName;
+exports.constraintAspectTypeName = constraintAspectTypeName;
+exports.constraintAspectKey = constraintAspectKey;
+exports.constraintEntryId = constraintEntryId;
+exports.constraintOwnedPrefix = constraintOwnedPrefix;
+exports.constraintEntries = constraintEntries;
+exports.isConstraintEntry = isConstraintEntry;
+exports.constraintAspectTypes = constraintAspectTypes;
+exports.readConstraint = readConstraint;
+const ir_1 = require("./ir");
+const kc_custom_types_1 = require("./kc_custom_types");
+// Full resource name of the constraint entry type for a destination.
+function constraintEntryTypeName(dest) {
+    return (0, kc_custom_types_1.customEntryTypeName)(kc_custom_types_1.CONSTRAINT_TYPE_ID, dest);
+}
+// Full resource name of the constraint aspect type for a destination.
+function constraintAspectTypeName(dest) {
+    return (0, kc_custom_types_1.customAspectTypeName)(kc_custom_types_1.CONSTRAINT_TYPE_ID, dest);
+}
+// Aspect-map key: the `project.location.type` reference form the client keys an
+// entry's aspects by.
+function constraintAspectKey(dest) {
+    return (0, kc_custom_types_1.customAspectKey)(kc_custom_types_1.CONSTRAINT_TYPE_ID, dest);
+}
+// The entry id of one constraint: `<model>.constraints.<name>`, alongside
+// `<model>.metrics.<name>` and `<model>.actions.<name>`.
+function constraintEntryId(modelId, constraintName) {
+    return `${modelId}.constraints.${slug(constraintName)}`;
+}
+// The entry-id prefix a model's constraints occupy, so delete reconciliation
+// removes the entry of a constraint dropped from the model.
+function constraintOwnedPrefix(modelId) {
+    return `${modelId}.constraints.`;
+}
+/**
+ * One entry per constraint, to append to the model's entries.
+ *
+ * Empty when the model declares no constraints, so a model without them is
+ * unchanged from before constraints existed. Warns once when it is non-empty:
+ * constraints reach Knowledge Catalog and nowhere else, which is worth saying
+ * out loud on a push that also deploys a graph.
+ */
+function constraintEntries(model, modelId, ctx, warnings) {
+    const constraints = model.constraints ?? [];
+    if (!constraints.length)
+        return [];
+    const entries = [];
+    for (const constraint of constraints) {
+        const id = constraintEntryId(modelId, constraint.name);
+        if (!ctx.claim(id, `constraint '${constraint.name}'`))
+            continue;
+        entries.push({
+            name: ctx.entry(id),
+            entryType: constraintEntryTypeName(ctx),
+            parentEntry: ctx.anchor,
+            entrySource: compact({
+                displayName: constraint.name,
+                description: constraint.description,
+            }),
+            aspects: {
+                [constraintAspectKey(ctx)]: {
+                    aspectType: constraintAspectTypeName(ctx),
+                    data: constraintAspectData(constraint),
+                },
+            },
+        });
+    }
+    // Counted from what was emitted rather than from the model, so a name
+    // collision that skipped a constraint does not inflate the number.
+    if (entries.length)
+        warnings.push(`model '${model.name}': ${entries.length} constraint(s) published as ` +
+            `${kc_custom_types_1.CONSTRAINT_TYPE_ID} entries (Knowledge Catalog is the only ` +
+            `system a constraint reaches).`);
+    return entries;
+}
+// The aspect payload for one constraint: the invariant, what a violation of it
+// does, how grave it is, and the whole of any `ai_context` declared on it.
+//
+// Every field here is read off the model. Nothing is computed, and in
+// particular the aspect type's `evaluation` field is left unwritten: it existed
+// to say which of two bodies a constraint used, and there is one body.
+function constraintAspectData(constraint) {
+    return compact({
+        judgment: constraint.judgment,
+        aiContext: (0, kc_custom_types_1.aiContextAspectValue)(constraint.aiContext),
+        onViolation: constraint.onViolation,
+        severity: constraint.severity,
+    });
+}
+// ---------------------------------------------------------------------------
+// Read side: a constraint entry -> the IR.
+// ---------------------------------------------------------------------------
+// True when an entry is one of a model's constraints. The match is on the entry
+// type's id suffix, so the project the type lives in need not be known. That is
+// what lets a pull keep working once the custom type is replaced by a built-in
+// one.
+function isConstraintEntry(entry) {
+    return entry.entryType?.endsWith(`/entryTypes/${kc_custom_types_1.CONSTRAINT_TYPE_ID}`) ??
+        false;
+}
+// The aspect type resource names to hydrate for a constraint entry. Named
+// through the entry type's own project so the pull follows the type wherever it
+// lives.
+function constraintAspectTypes(entryTypeBase) {
+    return [`${entryTypeBase}/aspectTypes/${kc_custom_types_1.CONSTRAINT_TYPE_ID}`];
+}
+/**
+ * Recovers one constraint from its entry, the inverse of constraintEntries.
+ *
+ * Returns undefined, with a warning, for an entry that states no judgment: an
+ * invariant that states no rule would be pulled into a model that then fails
+ * its own push-side validation, so one bad entry degrades itself rather than
+ * the pull. An entry left over from a push that wrote the removed `expression`
+ * body lands there too, and the warning says so by name -- that catalog holds a
+ * rule this model can no longer state, and the author has to restate it.
+ *
+ * `evaluation` is not read. There is one body, so the word it carried says
+ * nothing the entry does not.
+ */
+function readConstraint(entry, warnings) {
+    const name = entry.entrySource?.displayName || idOf(entry.name);
+    const data = constraintAspectDataOf(entry);
+    const judgment = aspectText(data.judgment);
+    if (!judgment) {
+        const stale = aspectText(data.expression) ?
+            `, though it states an expression -- the deterministic body was ` +
+                `removed, and the rule has to be restated in words` :
+            '';
+        warnings.push(`constraint '${name}': the ${kc_custom_types_1.CONSTRAINT_TYPE_ID} aspect has no ` +
+            `judgment${stale}; the constraint is skipped, and pushing this model ` +
+            `back will delete the entry`);
+        return undefined;
+    }
+    const constraint = { name, judgment };
+    const description = entry.entrySource?.description;
+    if (description !== undefined && description !== '')
+        constraint.description = description;
+    const onViolation = readEnum(name, 'onViolation', data.onViolation, ir_1.VIOLATION_EFFECTS, warnings);
+    if (onViolation)
+        constraint.onViolation = onViolation;
+    // A constraint must say what a violation does, so a pulled one that states no
+    // routing word is a model that will not push. Saying so here names the
+    // constraint while the pull is in front of the author; the alternative is a
+    // push error about a key they never wrote.
+    if (constraint.onViolation === undefined) {
+        warnings.push(`constraint '${name}': the ${kc_custom_types_1.CONSTRAINT_TYPE_ID} aspect states a ` +
+            `judgment but no onViolation, so the pulled constraint will not push ` +
+            `until one is added; a constraint must say whether a breach rejects, ` +
+            `escalates or warns`);
+    }
+    const severity = readEnum(name, 'severity', data.severity, ir_1.CONSTRAINT_SEVERITIES, warnings);
+    if (severity)
+        constraint.severity = severity;
+    const aiContext = (0, kc_custom_types_1.aiContextFromAspect)(data.aiContext);
+    if (aiContext)
+        constraint.aiContext = aiContext;
+    return constraint;
+}
+// The value an aspect field states, or undefined when it states none.
+//
+// A value outside the ones the IR defines is dropped with a warning rather than
+// kept, because an unrecognized word would fail the pulled model's own
+// push-side validation. What dropping costs differs by field. An unreadable
+// `severity` leaves a rule unranked, which nothing reads yet. An unreadable
+// `onViolation` has nothing to fall back to, because a constraint must state
+// its routing word: the pulled model names the constraint and says the word is
+// missing, which is the outcome to want when the catalog no longer says how a
+// breach routes.
+function readEnum(name, field, value, allowed, warnings) {
+    if (value === undefined || value === null)
+        return undefined;
+    // Only a string can be one of these words, and the check says so rather than
+    // coercing: `String(["escalate"])` is `escalate`, so a coercing reader would
+    // invent a routing word the catalog never stated. A blank or whitespace-only
+    // value is unset rather than wrong, the same reading a blank judgment gets.
+    if (typeof value === 'string') {
+        const text = value.trim();
+        if (text === '')
+            return undefined;
+        if (allowed.includes(text))
+            return text;
+    }
+    warnings.push(`constraint '${name}': the ${kc_custom_types_1.CONSTRAINT_TYPE_ID} aspect states ` +
+        `${field} ${showAspectValue(value)}, which is not one of ${allowed.join(', ')}; the constraint reads back without one`);
+    return undefined;
+}
+// How an unusable aspect value is quoted back in a warning: a string in single
+// quotes, so the word reads plainly, and anything else as JSON, so the reader
+// can see it was never a word at all.
+function showAspectValue(value) {
+    return typeof value === 'string' ? `'${value.trim()}'` :
+        JSON.stringify(value);
+}
+// ---------------------------------------------------------------------------
+// Local helpers (see the file header on why they are not shared).
+// ---------------------------------------------------------------------------
+// The constraint aspect's `data` from an entry, matched by the aspect key's
+// `.semantic-constraint` suffix or the aspectType's
+// `/aspectTypes/semantic-constraint` suffix, so it is found whichever project
+// the type was provisioned in.
+function constraintAspectDataOf(entry) {
+    for (const [key, aspect] of Object.entries(entry.aspects ?? {})) {
+        if (key.endsWith(`.${kc_custom_types_1.CONSTRAINT_TYPE_ID}`) ||
+            aspect.aspectType?.endsWith(`/aspectTypes/${kc_custom_types_1.CONSTRAINT_TYPE_ID}`)) {
+            return aspect.data ?? {};
+        }
+    }
+    return {};
+}
+// The trimmed text an aspect field states, or '' when it states none. A blank
+// or whitespace-only value reads as unset, the same reading readEnum gives.
+function aspectText(value) {
+    return typeof value === 'string' ? value.trim() : '';
+}
+// Entry ids allow letters, numbers, underscores, hyphens, and periods.
+function slug(s) {
+    return s.replace(/[^A-Za-z0-9_.-]/g, '_');
+}
+// The id segment of a full entry resource name.
+function idOf(name) {
+    return name.split('/').pop() ?? name;
+}
+// Drops undefined-valued keys so the emitted aspect (and its golden) only shows
+// fields the model actually set.
+function compact(obj) {
+    const out = {};
+    for (const [k, v] of Object.entries(obj)) {
+        if (v !== undefined)
+            out[k] = v;
+    }
+    return out;
+}

package/build/ts/kcmd/semantic/kc_converter.d.ts

@@ -0,0 +1,37 @@
+import type { Entry, EntryLink } from '../gcp/dataplex';
+import type { OverviewSidecar } from '../layouts/semantic-model';
+import { SemanticModel } from './ir';
+export interface ReadResult {
+    models: SemanticModel[];
+    warnings: string[];
+    secondGeneration?: boolean;
+    overviews?: Map<string, OverviewSidecar>;
+    firstGeneration?: FirstGenerationModel[];
+}
+export interface FirstGenerationModel {
+    name: string;
+    entries: Entry[];
+}
+/**
+ * Reconstructs the Semantic Model IR from Knowledge Catalog entries.
+ *
+ * Returns one model per `semantic-model` anchor plus any warnings (no anchor,
+ * an orphaned child, an entry missing its aspect data). Entries must already be
+ * hydrated with their `semantic-*` (and, for entities, `schema`) aspect data; a
+ * BASIC list omits aspect data, so the puller re-fetches each entry first.
+ *
+ * `entryLinks` are the `schema-join` links a pull fetched for the group (see
+ * `pull_kc`); each link between two of a model's entity entries is
+ * reconstructed as a 1:1 / 1:N relationship. Pass `[]` (the default) to
+ * reconstruct entities and metrics only.
+ *
+ * With `opts.v2Aspects` set, a model whose `semantic-model` aspect has
+ * `specVersion` is rebuilt from the second-generation types (see
+ * readModelV2), which reads no entry link. A model without it is returned in
+ * `firstGeneration` instead of being rebuilt.
+ */
+export declare function modelsFromCatalogResources(entries: Entry[], entryLinks?: EntryLink[], opts?: {
+    v2Aspects?: boolean;
+}): ReadResult;
+export declare function linkDedupKey(link: EntryLink): string;
+export declare function linkNamePrefix(modelName: string): string;