@google-cloud/kcmd 0.0.0-stage → 0.1.0

package/build/ts/kcmd/semantic/knowledge_catalog.js

@@ -0,0 +1,1043 @@
+"use strict";
+// Generates Knowledge Catalog resources from the Semantic Model IR.
+//
+// The IR (./ir) is pure semantics. This module is its Knowledge Catalog
+// emitter, the counterpart to `bigquery.ts`: it maps the model to catalog
+// Entries, each carrying the `semantic-*` Aspect(s) that describe it. Like
+// `bigquery.ts` it is a PURE function of the IR: no GCP calls, no I/O. The
+// orchestration layer (`deploy_knowledge_catalog.ts`, the counterpart to
+// `deploy_bigquery.ts`) drives this emitter and writes the resulting resources
+// via the Knowledge Catalog client.
+//
+// Target schema: the `semantic-model`, `semantic-entity`, and `semantic-metric`
+// entry/aspect types are built-in system types under `dataplex-types/global`,
+// alongside the built-in `schema` and `guidelines` aspects. This emitter
+// references them; it never provisions them. Each entry type declares
+// `required_aspects`; the emitted aspect set per entry is those plus the
+// optional `guidelines` aspect (attached only when the object carries
+// `ai_context.instructions`):
+//   * semantic-model entry -> { semantic-model, guidelines? }
+//   * semantic-entity entry -> { semantic-entity, schema, guidelines? }
+//   * semantic-metric entry -> { semantic-metric, guidelines? }
+//
+// An action (a write operation the model defines) and a constraint (an
+// invariant the model states) are published the same way: one entry each,
+// parented to the anchor. Their entry and aspect types are CUSTOM, because
+// Dataplex has no built-in type for either yet, so `kcmd init` provisions the
+// pairs in the destination project. `kc_custom_types.ts` declares those types
+// and `kc_actions.ts` and `kc_constraints.ts` encode the aspects they carry.
+// This module only appends the entries those two return.
+//
+// With the first-generation aspects (v2Aspects off), each model and profile
+// file the push read is also stored whole as a file copy, one
+// `semantic-model-file` entry per file, parented to the anchor. That type is
+// custom too. The aspects cannot hold the whole model, so a pull returns
+// these copies instead of rebuilding the file from them. The anchor of a
+// model with copies carries their push id in the `kcmd-push-id` label on its
+// entrySource.
+//
+// Aspect data shapes mirror the aspect types' CLOSED metadataTemplates exactly
+// (a server aspect type rejects an undeclared data field):
+//   * semantic-model  = { deploymentTargets: string[] }
+//   * semantic-entity = { source: { resources: string[], importedSystem?,
+//                                    importedResource? } }
+//   * semantic-metric = { entity?, dataType (required) }
+//   * schema          = { fields: [{ name, dataType, metadataType,
+//                                    description?, annotations? }],
+//                         primaryKey?: { fields }, uniqueConstraints?: [...] }
+//   * guidelines      = { instructions, userManaged (required) }
+//
+// The SQL-expression fields -- `semantic-metric.expression` and the per-field
+// `schema.semantics` { expression, role } block -- are NOT in the published
+// system-type templates yet, so they are gated behind
+// KcGenerateOptions.emitExpressions (off by default) and omitted above.
+//
+// With the second-generation types (v2Aspects on), the model entry carries
+// semantic-model, each entity entry carries semantic-entity and schema, and
+// each metric entry carries semantic-metric and sql-expressions. An entity
+// entry has sql-expressions too when one of its fields has an expression.
+// Model, entity, metric and relationship entries also carry guidelines,
+// semantic-extensions and overview when the object has content for them. A
+// field has no entry, so its guidelines, semantic-extensions and overview go
+// on its entity's entry at the field's path. `guidelines` holds the object's
+// instructions and, in `data`, its custom `ai_context` members.
+//
+// Relationships become `schema-join` entry links between the two entity entries.
+// schema-join is a built-in, undirected entry link type in `dataplex-types/global`
+// whose required `schema-join` aspect carries the join detail (the paired join
+// columns, JOIN vs FOREIGN_KEY, USER inference). The join's direction -- which
+// side holds the foreign key -- is preserved inside that aspect, not by the link.
+// Many-to-many (association / junction-table) edges are not emitted yet: a
+// junction is two joins through a third table, which schema-join's single
+// source/target pair does not model; the emitter warns and skips them (the edge
+// still lives in the BigQuery property graph, see graph_ddl.ts).
+//
+// With the second-generation types, each relationship is instead an entry,
+// `<model>/relationships/<name>`, that carries semantic-relationship. A
+// `semantic-link` entry link joins each ordered pair of related entities and
+// lists their relationships in declaration order. A `reference` link runs
+// from each relationship entry to each of its two entities, and from each
+// entity entry to the entry of each parent it extends.
+//
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.generateCatalogResources = generateCatalogResources;
+exports.fileCopyData = fileCopyData;
+const deployment_target_1 = require("./deployment_target");
+const kc_actions_1 = require("./kc_actions");
+const kc_constraints_1 = require("./kc_constraints");
+const kc_custom_types_1 = require("./kc_custom_types");
+const kc_ids_1 = require("./kc_ids");
+const loader_1 = require("./loader");
+const osi_converter_1 = require("./osi_converter");
+const resolve_inheritance_1 = require("./resolve_inheritance");
+const resolve_profiles_1 = require("./resolve_profiles");
+/**
+ * Generates the Knowledge Catalog resources for a semantic model.
+ *
+ * Returns the entries (model anchor first, then entities, metrics and, with
+ * v2Aspects, relationships), the entry links of the relationships and, with
+ * v2Aspects, of each entity's parents, and any warnings collected while
+ * mapping the IR (missing keys, un-typed metrics, skipped M:N relationships).
+ * The resources reference the built-in system types; they do not create them.
+ */
+function generateCatalogResources(model, opts) {
+    const warnings = [];
+    const entities = model.entities ?? [];
+    const metrics = model.metrics ?? [];
+    if (!entities.length) {
+        warnings.push('model has no entities; only the semantic-model entry will be generated');
+    }
+    const names = new kc_ids_1.Namer(opts);
+    const emitExpr = opts.emitExpressions ?? false;
+    const v2 = opts.v2Aspects ?? false;
+    if (v2)
+        warnings.push(...unstorableMemberWarnings(model));
+    // The first-generation aspects have no field for a vendor's extension
+    // blocks, and the stored copy of the model file keeps them.
+    if (!v2 && opts.sourceFiles?.length && hasThirdPartyExtensions(model)) {
+        warnings.push('custom_extensions from vendors other than GOOGLE are not indexed ' +
+            'by the Knowledge Catalog aspects; the stored copy of the model file ' +
+            'preserves them');
+    }
+    // With the first-generation aspects, a subtype field that only rebinds an
+    // inherited field is published with the definition it inherits.
+    const inherited = v2 ? undefined : inheritedFields(model);
+    // Entry ids must be unique within the entry group. Two source names that
+    // normalize to the same id (see slug), or exact duplicates the loader did not
+    // reject, would otherwise emit two entries with the same name and have the
+    // later write silently overwrite the earlier. Track ids and skip a collision.
+    const seen = new Set();
+    // The model entry is the anchor and the parent of every entity/metric entry,
+    // so it is created first: it is entries[0] and the publisher writes in array
+    // order. Reserve its id up front so nothing else can claim it.
+    const modelId = names.modelId(model);
+    const modelEntryName = names.entry(modelId);
+    claim(seen, modelId, 'entry', `model '${model.name}'`, warnings);
+    const entries = [{
+            name: modelEntryName,
+            entryType: names.typeName('entry', 'semantic-model'),
+            entrySource: source(model.name, model.description),
+            aspects: v2 ?
+                aspectMap(names, {
+                    'semantic-model': modelAspectDataV2(model),
+                    ...objectAspectsV2(model),
+                }) :
+                aspectsFor(names, { 'semantic-model': modelAspectData(model) }, model.aiContext),
+        }];
+    // Maps an entity's model name to its published entry name, so a relationship
+    // can resolve its endpoints to the entries the link references. Only entities
+    // actually emitted (not skipped for a duplicate id) are recorded.
+    const entityEntryName = new Map();
+    for (const entity of entities) {
+        // The first-generation types do not model inheritance, so an abstract
+        // entity is skipped with a warning and an edge naming it gets no link.
+        if (entity.abstract && !v2) {
+            warnings.push(`entity '${entity.name}' is abstract (no physical table); skipped ` +
+                `for Knowledge Catalog (KC does not yet model class hierarchies)`);
+            continue;
+        }
+        const entityId = names.entityId(model, entity);
+        if (!claim(seen, entityId, 'entry', `entity '${entity.name}'`, warnings))
+            continue;
+        entityEntryName.set(entity.name, names.entry(entityId));
+        if (v2 ? lacksKeys(model, entity) : !entity.keys || !entity.keys.length) {
+            warnings.push(`entity '${entity.name}': no keys declared in the source model`);
+        }
+        entries.push({
+            name: names.entry(entityId),
+            entryType: names.typeName('entry', 'semantic-entity'),
+            parentEntry: modelEntryName,
+            entrySource: source(entity.name, entity.description),
+            // semantic-entity and schema are required. The file header lists the
+            // optional aspects.
+            aspects: v2 ? aspectMap(names, entityAspectsV2(model, entity)) :
+                aspectsFor(names, {
+                    'semantic-entity': entityAspectData(entity),
+                    'schema': schemaAspectData(entity, emitExpr, inherited?.get(entity.name)),
+                }, entity.aiContext),
+        });
+    }
+    for (const metric of metrics) {
+        const metricId = names.metricId(model, metric);
+        if (!claim(seen, metricId, 'entry', `metric '${metric.name}'`, warnings))
+            continue;
+        entries.push({
+            name: names.entry(metricId),
+            entryType: names.typeName('entry', 'semantic-metric'),
+            parentEntry: modelEntryName,
+            entrySource: source(metric.name, metric.description),
+            aspects: v2 ? aspectMap(names, metricAspectsV2(model, metric)) :
+                aspectsFor(names, { 'semantic-metric': metricAspectData(metric, emitExpr) }, metric.aiContext),
+        });
+    }
+    const v2Relationships = v2 ?
+        relationshipResourcesV2(names, model, modelEntryName, entityEntryName, warnings) :
+        undefined;
+    if (v2Relationships)
+        entries.push(...v2Relationships.entries);
+    // Each entry's sidecar fills the rest of its overview, beside the examples.
+    for (const [id, sidecar] of v2 ? opts.overviews ?? [] : []) {
+        const entry = entries.find(e => e.name === names.entry(id));
+        if (!entry) {
+            warnings.push(`sidecar '${id}.overview.md' names no entry of this model; it is ` +
+                `not pushed`);
+            continue;
+        }
+        const examples = entry.aspects?.[names.aspectRef('overview')]?.data?.examples;
+        const overview = compact({ ...sidecar, examples });
+        if (Object.keys(overview).length) {
+            entry.aspects = { ...entry.aspects, ...aspectMap(names, { overview }) };
+        }
+    }
+    // The second-generation schema keeps a field a profile excludes, so
+    // actionEntries reads the model without exclusions.
+    const actionModel = v2 ? (0, resolve_profiles_1.withoutExclusions)(model) : model;
+    // One entry per action, alongside the entities and metrics. Actions have no
+    // built-in system type, so they reference the custom pair declared in
+    // `kc_custom_types.ts`, and `kc_actions.ts` fills the aspect; it returns
+    // nothing when the model declares no actions.
+    entries.push(...(0, kc_actions_1.actionEntries)(actionModel, modelId, {
+        project: opts.project,
+        entry: (id) => names.entry(id),
+        anchor: modelEntryName,
+        claim: (id, label) => claim(seen, id, 'entry', label, warnings),
+        // The entities the loop above gave an entry. With the first-generation
+        // types an abstract entity has none.
+        publishedEntities: new Set(entityEntryName.keys()),
+    }, warnings));
+    // One entry per constraint, on the same footing as an action: no built-in
+    // system type, so the custom pair declared in `kc_custom_types.ts`, filled by
+    // `kc_constraints.ts`. Nothing when the model declares no constraints.
+    entries.push(...(0, kc_constraints_1.constraintEntries)(model, modelId, {
+        project: opts.project,
+        entry: (id) => names.entry(id),
+        anchor: modelEntryName,
+        claim: (id, label) => claim(seen, id, 'entry', label, warnings),
+    }, warnings));
+    // One file copy entry per model and profile file, holding its text as
+    // read. Only with the first-generation aspects: a push with v2Aspects on
+    // emits none, so delete reconciliation removes the copies under the owned
+    // `.files.` prefix. Two file names that slug to one id fail the push, since
+    // storing only one would lose the other on the next pull.
+    if (!v2) {
+        const fileById = new Map();
+        let pushId;
+        for (const file of opts.sourceFiles ?? []) {
+            const id = names.fileEntryId(model.name, file.fileName);
+            const earlier = fileById.get(id);
+            if (earlier !== undefined) {
+                throw new Error(`files '${earlier}' and '${file.fileName}' both map to entry id ` +
+                    `'${id}'; rename one of them`);
+            }
+            fileById.set(id, file.fileName);
+            if (!claim(seen, id, 'entry', `file '${file.fileName}'`, warnings))
+                continue;
+            entries.push(fileCopyEntry(names.entry(id), modelEntryName, file, opts));
+            pushId ??= file.pushId;
+        }
+        // The anchor is written before the copies and is labeled with the push id
+        // they carry. A pull that finds the label and no copies from that push
+        // knows the last push stopped before storing them. A model an older kcmd
+        // pushed has no label.
+        if (pushId !== undefined) {
+            entries[0].entrySource = {
+                ...entries[0].entrySource,
+                labels: { [kc_custom_types_1.PUSH_ID_LABEL]: pushId },
+            };
+        }
+    }
+    const entryLinks = v2Relationships ?
+        [
+            ...v2Relationships.links,
+            ...extendsLinks(names, model, entityEntryName),
+        ] :
+        schemaJoinLinks(names, model, entityEntryName, warnings);
+    return {
+        entries,
+        entryLinks,
+        warnings: [...new Set(warnings)],
+        ownedPrefixes: (0, kc_ids_1.ownedEntryIdPrefixes)(model.name),
+    };
+}
+// The schema-join links of the first-generation types, one per relationship
+// that `unpublishedReason` accepts and per pair of entities.
+function schemaJoinLinks(names, model, entityEntryName, warnings) {
+    const entryLinks = [];
+    const seenLinks = new Set();
+    const seenPairs = new Map();
+    for (const rel of model.relationships ?? []) {
+        // A relationship that cannot be published gets the warning that says why,
+        // whatever its place in the model, so this check runs before the pair
+        // check below.
+        const reason = unpublishedReason(rel, entityEntryName);
+        if (reason) {
+            warnings.push(`relationship '${rel.name}': ${reason}`);
+            continue;
+        }
+        // Dataplex allows one link of a type between two entries, so the first
+        // relationship on a pair gets the link and the rest get a warning.
+        const pairKey = [rel.source.entity, rel.destination.entity].sort().join('|');
+        const firstRel = seenPairs.get(pairKey);
+        if (firstRel !== undefined) {
+            warnings.push(`relationship '${rel.name}': shares entities '${rel.source.entity}' and '${rel.destination.entity}' with relationship '${firstRel}'; ` +
+                `skipped because the catalog shows one join for those entities.`);
+            continue;
+        }
+        // `unpublishedReason` has checked that both endpoint entities have an
+        // entry.
+        const ends = [
+            entityEntryName.get(rel.source.entity),
+            entityEntryName.get(rel.destination.entity),
+        ];
+        const link = relationshipLink(names, model, rel, ends, seenLinks, warnings);
+        if (link) {
+            seenPairs.set(pairKey, rel.name);
+            entryLinks.push(link);
+        }
+    }
+    return entryLinks;
+}
+// Returns why `rel` cannot be published as a schema-join entry link, or
+// undefined when it can. The reasons, in the order checked: a many-to-many
+// (association) edge, since a junction is two joins, which the single
+// source/target schema-join does not model; an edge whose endpoint entity was
+// not emitted, for example one skipped for a duplicate id; and a column-less
+// (purely logical) edge, whose join columns must be added to the model before
+// it can publish. The BigQuery property graph still carries the association
+// and, once bound, direct edges.
+function unpublishedReason(rel, entityEntryName) {
+    if (rel.association)
+        return ASSOCIATION_REASON;
+    const missing = [rel.source.entity, rel.destination.entity].find(entity => !entityEntryName.get(entity));
+    if (missing !== undefined) {
+        return `endpoint entity '${missing}' is not a published entity; the ` +
+            `relationship link is skipped.`;
+    }
+    // A purely logical edge (an OWL import) carries no join columns. schema-join
+    // is a server-defined CLOSED metadataTemplate whose `fields` requirement we
+    // cannot depend on, so rather than risk a rejected aspect on a KC push we skip
+    // the link until the edge is bound. Add the relationship's from_columns /
+    // to_columns to the model and it publishes; the edge still lives in the model
+    // (and, once bound, the BigQuery/Spanner graph).
+    if (!rel.source.columns.length || !rel.destination.columns.length) {
+        return `no join columns, so it is not published to Knowledge Catalog ` +
+            `yet; add its from_columns and to_columns to the relationship in the ` +
+            `model to publish the link.`;
+    }
+    return undefined;
+}
+// Why an association relationship is not published.
+const ASSOCIATION_REASON = `many-to-many (association) edges are not published to ` +
+    `Knowledge Catalog yet; the edge lives in the BigQuery property graph.`;
+// The relationship entries and their links under the second-generation
+// types: an entry per relationship except an association, a semantic-link per
+// ordered entity pair listing its relationships in order, and a reference link
+// from each relationship entry to each of its two entities.
+function relationshipResourcesV2(names, model, modelEntryName, entityEntryName, warnings) {
+    const entries = [];
+    const references = [];
+    // The relationships of each ordered entity pair, by link ID.
+    const pairs = new Map();
+    // Push never reaches a name with no entry: validation rejects an unknown
+    // entity, and every entity has an entry under these types.
+    for (const rel of model.relationships ?? []) {
+        if (rel.association) {
+            warnings.push(`relationship '${rel.name}': ${ASSOCIATION_REASON}`);
+            continue;
+        }
+        const id = names.relationshipId(model, rel);
+        entries.push({
+            name: names.entry(id),
+            entryType: names.typeName('entry', 'semantic-relationship'),
+            parentEntry: modelEntryName,
+            entrySource: source(rel.name, rel.description),
+            aspects: aspectMap(names, {
+                'semantic-relationship': relationshipAspectData(model, rel),
+                ...objectAspectsV2(rel),
+            }),
+        });
+        const from = rel.source.entity;
+        const to = rel.destination.entity;
+        for (const [end, entity] of [['source', from], ['target', to]]) {
+            references.push(directedLink(names, 'reference', (0, kc_ids_1.semanticLinkId)('relationship', model.name, rel.name, end), names.entry(id), entityEntryName.get(entity)));
+        }
+        const linkId = (0, kc_ids_1.semanticLinkId)('link', model.name, from, to);
+        const pair = pairs.get(linkId) ?? { from, to, relationships: [] };
+        pair.relationships.push(rel.name);
+        pairs.set(linkId, pair);
+    }
+    const links = [...pairs].map(([linkId, { from, to, relationships }]) => ({
+        ...directedLink(names, 'semantic-link', linkId, entityEntryName.get(from), entityEntryName.get(to)),
+        aspects: aspectMap(names, { 'semantic-link': { relationships } }),
+    }));
+    return { entries, links: [...links, ...references] };
+}
+// The reference link from each entity entry to the entry of each parent it
+// extends.
+function extendsLinks(names, model, entityEntryName) {
+    return (model.entities ?? []).flatMap(e => (e.extends ?? []).map(parent => directedLink(names, 'reference', (0, kc_ids_1.semanticLinkId)('extends', model.name, e.name, parent), entityEntryName.get(e.name), entityEntryName.get(parent))));
+}
+// A link of type `type` with ID `id` from the entry `from` to the entry `to`.
+function directedLink(names, type, id, from, to) {
+    return {
+        name: names.entryLink(id),
+        entryLinkType: names.typeName('entryLink', type),
+        entryReferences: [{ name: from, type: 'SOURCE' }, { name: to, type: 'TARGET' }],
+    };
+}
+// The semantic-relationship aspect, with join columns as written.
+function relationshipAspectData(model, rel) {
+    const endpoint = (end) => compact({ name: end.entity, fields: (0, osi_converter_1.nonEmpty)(end.columns) });
+    const profiles = namedProfiles(model).flatMap(p => {
+        const bound = p.relationships?.find(r => r.name === rel.name);
+        if (!bound)
+            return [];
+        return [compact({
+                name: p.name,
+                sourceFields: (0, osi_converter_1.nonEmpty)(bound.fromColumns),
+                targetFields: (0, osi_converter_1.nonEmpty)(bound.toColumns),
+            })];
+    });
+    return compact({
+        source: endpoint(rel.source),
+        target: endpoint(rel.destination),
+        profiles: (0, osi_converter_1.nonEmpty)(profiles),
+        synonyms: (0, osi_converter_1.nonEmpty)(rel.aiContext?.synonyms),
+    });
+}
+// Builds the schema-join entry link for a relationship that
+// `unpublishedReason` accepts, or undefined when another resource already
+// claimed its link id. `ends` holds the entry names of its source and
+// destination entities.
+function relationshipLink(names, model, rel, ends, seenLinks, warnings) {
+    const [src, dst] = ends;
+    const linkId = names.linkId(model, rel);
+    if (!claim(seenLinks, linkId, 'entry link', `relationship '${rel.name}'`, warnings))
+        return undefined;
+    // The name lives only in the link id (schema-join's aspect has no name
+    // field), and link ids are normalized -- lowercase, hyphens only. A pull that
+    // rebuilds the model reads each hyphen back as an underscore, so it returns
+    // the name lowercased; warn when that differs from the authored name.
+    const normalizedName = (0, kc_ids_1.linkSlug)(rel.name).replace(/-/g, '_');
+    if (normalizedName !== rel.name) {
+        warnings.push(`relationship '${rel.name}': Knowledge Catalog stores the name only ` +
+            `in the normalized link id, so a pull that rebuilds the model returns ` +
+            `it as '${normalizedName}'.`);
+    }
+    return {
+        name: names.entryLink(linkId),
+        entryLinkType: names.typeName('entryLink', 'schema-join'),
+        // schema-join is undirected: both endpoints are UNSPECIFIED references. The
+        // join direction lives in the aspect (source is the foreign-key side).
+        entryReferences: [
+            { name: src, type: 'UNSPECIFIED' },
+            { name: dst, type: 'UNSPECIFIED' },
+        ],
+        aspects: aspectMap(names, {
+            'schema-join': schemaJoinAspectData(model, rel),
+        }),
+    };
+}
+// The schema-join aspect for a direct foreign key: a single join pairing the
+// source (FK) columns to the destination (key) columns positionally. Each side's
+// `name` is the entity's SQL table representation (its dataSource). `type` is
+// FOREIGN_KEY (only direct FKs are emitted today); `inferenceSource` is USER
+// because the join is author-declared, and `userManaged` stops Dataplex from
+// overwriting it. Field names/nesting mirror the schema-join aspect type's
+// metadataTemplate.
+//
+// Only bound edges reach here -- relationshipLink skips a column-less (logical)
+// edge -- so `fields` is always present. The `name` still falls back to the
+// entity name when the model is logical-only for KC (columns set but no
+// dataSource binding), matching the entity-not-found fallback.
+function schemaJoinAspectData(model, rel) {
+    const srcEntity = entityByName(model, rel.source.entity);
+    const dstEntity = entityByName(model, rel.destination.entity);
+    return {
+        joins: [compact({
+                source: compact({
+                    name: srcEntity && srcEntity.dataSource ?
+                        srcEntity.dataSource :
+                        rel.source.entity,
+                    fields: rel.source.columns,
+                }),
+                target: compact({
+                    name: dstEntity && dstEntity.dataSource ?
+                        dstEntity.dataSource :
+                        rel.destination.entity,
+                    fields: rel.destination.columns,
+                }),
+                description: rel.description,
+                type: 'FOREIGN_KEY',
+                inferenceSource: 'USER',
+            })],
+        userManaged: true,
+    };
+}
+function entityByName(model, name) {
+    return (model.entities ?? []).find(e => e.name === name);
+}
+// The entry holding one file copy. Its entry and aspect types are custom, so
+// they live in the destination project.
+function fileCopyEntry(name, anchor, file, dest) {
+    return {
+        name,
+        entryType: (0, kc_custom_types_1.customEntryTypeName)(kc_custom_types_1.FILE_COPY_TYPE_ID, dest),
+        parentEntry: anchor,
+        aspects: {
+            [(0, kc_custom_types_1.customAspectKey)(kc_custom_types_1.FILE_COPY_TYPE_ID, dest)]: {
+                aspectType: (0, kc_custom_types_1.customAspectTypeName)(kc_custom_types_1.FILE_COPY_TYPE_ID, dest),
+                data: fileCopyData(file),
+            },
+        },
+    };
+}
+// The aspect data of a file copy: the file's name and text, and the push id
+// and copy count when the push set them.
+function fileCopyData(file) {
+    return compact({
+        fileName: file.fileName,
+        content: file.content,
+        pushId: file.pushId,
+        pushFileCount: file.pushFileCount,
+    });
+}
+// True when the model, or anything in it, carries a `custom_extensions` block
+// from a vendor other than GOOGLE.
+function hasThirdPartyExtensions(model) {
+    const thirdParty = (o) => (o.customExtensions ?? []).some(x => x.vendorName !== 'GOOGLE');
+    const entities = model.entities ?? [];
+    return thirdParty(model) || entities.some(thirdParty) ||
+        entities.some(e => (e.fields ?? []).some(thirdParty)) ||
+        (model.relationships ?? []).some(thirdParty) ||
+        (model.metrics ?? []).some(thirdParty) ||
+        (model.actions ?? []).some(thirdParty);
+}
+// One warning for each custom `ai_context` member that `guidelines` leaves
+// out, on the model, an entity, a field, a metric or a relationship.
+function unstorableMemberWarnings(model) {
+    const warnings = [];
+    const check = (object, ai) => {
+        for (const [member, value] of Object.entries(ai?.additionalProperties ?? {})) {
+            const holds = unstorableValue(value);
+            if (holds) {
+                warnings.push(`${object}: ai_context member '${member}' is not pushed: its ` +
+                    `value holds ${holds}, which Knowledge Catalog cannot store as ` +
+                    `written`);
+            }
+        }
+    };
+    check(`model '${model.name}'`, model.aiContext);
+    for (const entity of model.entities ?? []) {
+        check(`entity '${entity.name}'`, entity.aiContext);
+        for (const field of entity.fields ?? []) {
+            check(`field '${entity.name}.${field.name}'`, field.aiContext);
+        }
+    }
+    for (const metric of model.metrics ?? []) {
+        check(`metric '${metric.name}'`, metric.aiContext);
+    }
+    for (const rel of model.relationships ?? []) {
+        check(`relationship '${rel.name}'`, rel.aiContext);
+    }
+    return warnings;
+}
+// Names the first number in `value` that JSON cannot hold as the document
+// wrote it: .inf, -.inf, .nan, or a number past 2^53, which loses digits when
+// the model loads. Undefined when there is none.
+function unstorableValue(value) {
+    if (typeof value === 'number') {
+        if (Number.isNaN(value))
+            return '.nan';
+        if (!Number.isFinite(value))
+            return value > 0 ? '.inf' : '-.inf';
+        return Number.isInteger(value) && !Number.isSafeInteger(value) ?
+            'a number past 2^53' :
+            undefined;
+    }
+    if (value === null || typeof value !== 'object')
+        return undefined;
+    return Object.values(value).map(unstorableValue).find(Boolean);
+}
+// Each entity's fields with inheritance resolved, by entity and field name, or
+// undefined when no entity extends another or inheritance cannot be resolved.
+// A resolved field keeps what the entity's own declaration sets and takes the
+// rest from the ancestor that defines it.
+function inheritedFields(model) {
+    if (!(model.entities ?? []).some(e => e.extends?.length))
+        return undefined;
+    let resolved;
+    try {
+        resolved = (0, resolve_inheritance_1.resolveInheritance)(model).model;
+    }
+    catch (err) {
+        // Push validation reports it. The schema aspect keeps what each entity
+        // declares.
+        if (err instanceof resolve_inheritance_1.InheritanceError)
+            return undefined;
+        throw err;
+    }
+    return new Map((resolved.entities ?? []).map(e => [e.name, new Map((e.fields ?? []).map(f => [f.name, f]))]));
+}
+// The fields `entity` declares, leaving out each line that only rebinds a
+// field it inherits.
+function declaredFields(model, entity) {
+    const inherited = (0, resolve_profiles_1.inheritedNamesOf)(model, entity.name);
+    return entity.fields.filter(f => !inherited.has(f.name));
+}
+// True when `entity` is concrete and states no primary key and no unique key,
+// in its own bindings or in any named profile.
+function lacksKeys(model, entity) {
+    const bindings = namedProfiles(model).flatMap(p => p.entities.filter(b => b.name === entity.name));
+    return !entity.abstract && !(0, resolve_profiles_1.statesKey)(entity.keys, entity.uniqueKeys) &&
+        !bindings.some(b => (0, resolve_profiles_1.statesKey)(b.primaryKey, b.uniqueKeys));
+}
+// ---------------------------------------------------------------------------
+// Aspect-data mappers. Field names/nesting mirror the server aspect types'
+// CLOSED metadataTemplates exactly (see file header); the golden tests pin
+// these shapes so any schema-driven change is a visible, reviewable diff.
+// ---------------------------------------------------------------------------
+// semantic-model: every deployment target this model declares, in declaration
+// order. Recording all declared targets keeps the entry faithful so a later
+// `kcmd pull` reconstructs `deployments` in the order the author wrote them.
+// Empty data is valid; the aspect is still attached to satisfy
+// required_aspects.
+function modelAspectData(model) {
+    const { uris } = (0, deployment_target_1.googleDeploymentTargets)(model);
+    return compact({
+        deploymentTargets: uris.length ? uris : undefined,
+    });
+}
+// semantic-entity: the base table backing the entity. A source is stored as
+// written, its GCP resource name, in `resources`.
+function entityAspectData(entity) {
+    // A logical-only entity has no physical table (empty dataSource). Still emit
+    // the `source` block with an empty `resources` array rather than a bogus
+    // ['']: the semantic-entity template requires `source` (with its `resources`
+    // list), so omitting it is rejected server-side. An empty list is the honest
+    // "no binding yet"; the reader treats it the same as an absent source
+    // (dataSource becomes '').
+    const authored = entity.authoredSource?.trim();
+    if (authored !== undefined && (0, loader_1.isAcceptedSource)(authored)) {
+        // Stored as written, so pull returns it unchanged.
+        return { source: { resources: [authored] } };
+    }
+    // A legacy instance-based AlloyDB URI is stored as written too.
+    const legacy = authored !== undefined &&
+        (0, loader_1.asResourceUri)(authored) === entity.dataSource &&
+        entity.dataSource.startsWith('//');
+    const path = resourcePath(legacy ? authored : entity.dataSource);
+    return compact({
+        source: { resources: path ? [path] : [] },
+    });
+}
+// The built-in schema aspect, carrying each field's column type and display
+// label, the entity's primary/unique keys, plus the gated per-field `semantics`
+// block (expression / role). name, dataType, and metadataType are required per
+// column. `inherited` holds the entity's fields with inheritance resolved; a
+// field that sets no datatype, description or label of its own takes the one
+// it inherits from there.
+function schemaAspectData(entity, emitExpressions, inherited) {
+    // Key column lists ride the schema aspect verbatim -- they need NOT be
+    // declared fields (the closed template accepts and round-trips columns absent
+    // from fields[], live-verified). Keep only non-empty strings, matching the
+    // reader's stringList, so a degenerate '' member does not round-trip
+    // asymmetrically; a set that is empty after filtering is then dropped.
+    const keyFields = (cols) => (cols ?? []).filter(c => typeof c === 'string' && c !== '');
+    const primaryKey = keyFields(entity.keys);
+    const uniqueConstraints = (entity.uniqueKeys ?? []).map(keyFields).filter(set => set.length);
+    // A field a binding profile excludes on this entity is not published for it
+    // (see Entity.excludedFields).
+    const excluded = new Set(entity.excludedFields ?? []);
+    // A subtype's redeclaration of an inherited field only rebinds it, so its
+    // definition is the inherited one.
+    const withDefinition = (f) => {
+        const from = inherited?.get(f.name);
+        return {
+            ...f,
+            type: f.type ?? from?.type,
+            description: f.description ?? from?.description,
+            label: f.label ?? from?.label,
+        };
+    };
+    return compact({
+        fields: (entity.fields ?? [])
+            .filter(f => !excluded.has(f.name))
+            .map(withDefinition)
+            .map(f => compact({
+            name: f.name,
+            // An untyped field is published as Opaque (STRING +
+            // metadataType OTHER), the explicit "type unknown"
+            // marker, so a pull recovers it as Opaque rather than
+            // dropping the type. Authored `String` maps to STRING.
+            dataType: columnDataType(f.type ?? 'Opaque'),
+            metadataType: columnMetadataType(f.type ?? 'Opaque'),
+            description: f.description,
+            // A field's display label rides in the schema field's
+            // `annotations` map (the template's free-form
+            // string->string map) -- the one slot for metadata the
+            // template has no dedicated column for. Omitted when the
+            // field has no label.
+            annotations: f.label ? { label: f.label } : undefined,
+            // The per-field `semantics` block (expression + role) is
+            // not in the published `schema` aspect template yet, so
+            // it is gated off by default (see
+            // KcGenerateOptions.emitExpressions); compact() then
+            // drops the undefined key.
+            semantics: emitExpressions ? compact({
+                expression: f.expression,
+                // A field with any dimension metadata is a dimension;
+                // otherwise DEFAULT.
+                role: f.dimension ? 'DIMENSION' : 'DEFAULT',
+            }) : undefined,
+        })),
+        // The entity's grain / primary key -> the schema aspect's primaryKey.fields
+        // (ordered, so a composite key's ordinal positions round-trip). Omitted
+        // when the entity declares no (non-empty) keys.
+        primaryKey: primaryKey.length ? { fields: primaryKey } : undefined,
+        // Additional uniqueness constraints beyond the primary key -> one
+        // uniqueConstraints entry per non-empty unique column set. Omitted when
+        // there are none.
+        uniqueConstraints: uniqueConstraints.length ?
+            uniqueConstraints.map(fields => ({ fields })) :
+            undefined,
+    });
+}
+// The built-in guidelines aspect: routes an object's `ai_context.instructions`
+// -- free-form guidance for AI consumers -- to the aspect's `instructions`
+// richText field. `userManaged` is required and always true: these guidelines
+// are author-declared, so Dataplex must not overwrite them. Only `instructions`
+// is routed here; `ai_context.synonyms`/`examples` have no home in this aspect.
+// Returns undefined (attach no aspect) when there are no instructions.
+function guidelinesAspectData(ai) {
+    const instructions = ai?.instructions;
+    if (instructions === undefined || instructions === '')
+        return undefined;
+    return { instructions, userManaged: true };
+}
+// semantic-metric: the model-level aggregate. `dataType` is required by the
+// aspect type; a metric the author left untyped is published as Opaque -- the
+// explicit "type unknown" marker -- rather than guessing a numeric type. (The
+// metric aspect template carries only `dataType`, not a metadataType, so Opaque
+// serializes as STRING and a pull recovers the metric untyped; once the
+// template gains a metadataType it can round-trip as an explicit Opaque.)
+function metricAspectData(metric, emitExpressions) {
+    const dataType = columnDataType(metric.type ?? 'Opaque');
+    return compact({
+        entity: metric.entity,
+        dataType,
+        // `expression` is not in the published semantic-metric aspect template yet;
+        // gated off by default (see KcGenerateOptions.emitExpressions).
+        expression: emitExpressions ? metric.expression : undefined,
+    });
+}
+// The guidelines, semantic-extensions and overview aspects of an object's
+// entry, each left out when the object has nothing for it. For a field's
+// aspects, `at` is `@Schema.<field>`, and each key ends in it.
+function objectAspectsV2(object, at = '') {
+    const examples = object.aiContext?.examples ?? [];
+    return compact({
+        [`guidelines${at}`]: guidelinesAspectDataV2(object.aiContext),
+        [`semantic-extensions${at}`]: extensionsAspectData(object.customExtensions),
+        [`overview${at}`]: examples.length ?
+            { examples: examples.map(content => ({ content })) } :
+            undefined,
+    });
+}
+// The guidelines aspect under the second-generation types: the object's
+// instructions, and in `data` its custom ai_context members by name, leaving
+// out those `unstorableValue` names. Undefined when the object has neither.
+function guidelinesAspectDataV2(ai) {
+    const members = Object.entries(ai?.additionalProperties ?? {})
+        .filter(([, value]) => !unstorableValue(value));
+    const data = members.length ? Object.fromEntries(members) : undefined;
+    const instructions = ai?.instructions || undefined;
+    if (!instructions && !data)
+        return undefined;
+    return compact({ instructions, data, userManaged: true });
+}
+// semantic-extensions: every extension block from a vendor other than GOOGLE,
+// as written and in order. Undefined when there is none.
+function extensionsAspectData(extensions) {
+    const blocks = (extensions ?? []).filter(x => x.vendorName !== 'GOOGLE');
+    return blocks.length ? { customExtensions: blocks } : undefined;
+}
+// The semantic-model aspect of the second-generation types.
+function modelAspectDataV2(model) {
+    const declared = model.deployments ?? [];
+    const deployments = declared.map(d => compact({
+        name: d.name,
+        target: d.target,
+        profile: d.profile === resolve_profiles_1.DEFAULT_PROFILE ? undefined : d.profile,
+    }));
+    return compact({
+        specVersion: model.version ?? loader_1.GOOGLE_VERSION,
+        synonyms: (0, osi_converter_1.nonEmpty)(model.aiContext?.synonyms),
+        profiles: (0, osi_converter_1.nonEmpty)(namedProfiles(model).map(p => ({ name: p.name }))),
+        deployments: (0, osi_converter_1.nonEmpty)(deployments),
+    });
+}
+// The aspects of an entity entry under the second-generation types. Its schema
+// lists the fields it declares, each with its own aspects at its path.
+function entityAspectsV2(model, entity) {
+    const fields = declaredFields(model, entity);
+    return compact({
+        'semantic-entity': entityAspectDataV2(model, entity),
+        'schema': schemaAspectDataV2(fields),
+        ...objectAspectsV2(entity),
+        ...Object.assign({}, ...fields.map(f => objectAspectsV2(f, `@Schema.${f.name}`))),
+        'sql-expressions': fieldExpressionsData(model, entity),
+    });
+}
+// The built-in schema aspect under the second-generation types, one record per
+// field. `is_time` is written only when the document declares it.
+function schemaAspectDataV2(fields) {
+    return {
+        fields: fields.map(f => {
+            const type = f.type ?? 'Opaque';
+            const annotations = compact({
+                label: f.label || undefined,
+                is_time: f.dimension?.isTime?.toString(),
+            });
+            return compact({
+                name: f.name,
+                description: f.description,
+                dataType: type,
+                metadataType: columnMetadataType(type),
+                semantic: f.dimension ? 'DIMENSION' : undefined,
+                synonyms: (0, osi_converter_1.nonEmpty)(f.aiContext?.synonyms),
+                annotations: Object.keys(annotations).length ? annotations : undefined,
+            });
+        }),
+    };
+}
+// semantic-entity under the second-generation types: the entity's source and
+// keys as written, its direct parents, whether it is abstract, its synonyms,
+// and one record per named profile that lists it, holding what that profile
+// states. Anything empty is left out.
+function entityAspectDataV2(model, entity) {
+    const profiles = namedProfiles(model).flatMap(p => {
+        const bound = p.entities.find(b => b.name === entity.name);
+        if (!bound)
+            return [];
+        return [compact({
+                name: p.name,
+                source: bound.source ? { resource: bound.source } : undefined,
+                excludedFields: (0, osi_converter_1.nonEmpty)(bound.fieldsExclude),
+                ...keysData(bound.primaryKey, bound.uniqueKeys),
+            })];
+    });
+    return compact({
+        source: entity.authoredSource ? { resource: entity.authoredSource } :
+            undefined,
+        ...keysData(entity.keys, entity.uniqueKeys),
+        profiles: (0, osi_converter_1.nonEmpty)(profiles),
+        extends: (0, osi_converter_1.nonEmpty)(entity.extends),
+        abstract: entity.abstract || undefined,
+        synonyms: (0, osi_converter_1.nonEmpty)(entity.aiContext?.synonyms),
+    });
+}
+// The `primaryKey` and `uniqueConstraints` of semantic-entity or a profile
+// record, each undefined when empty. A unique key with no column is left out.
+function keysData(primaryKey, uniqueKeys) {
+    return {
+        primaryKey: primaryKey?.length ? { fields: [...primaryKey] } : undefined,
+        uniqueConstraints: (0, osi_converter_1.nonEmpty)(uniqueKeys?.filter(k => k.length).map(k => ({ fields: [...k] }))),
+    };
+}
+// The sql-expressions aspect of an entity entry: the records of the entity's
+// own fields, then those of each named profile that lists it, with the
+// profile's name in `qualifier`. Returns undefined when there is no record.
+function fieldExpressionsData(model, entity) {
+    const records = (fields, qualifier) => fields.flatMap(f => expressionRecords(f).map(r => compact({ name: f.name, qualifier, ...r })));
+    const fieldExpressions = [
+        ...records(entity.fields),
+        ...namedProfiles(model).flatMap(p => records(p.entities.find(b => b.name === entity.name)?.fields ?? [], p.name)),
+    ];
+    return fieldExpressions.length ? { fieldExpressions } : undefined;
+}
+// The aspects of a metric entry under the second-generation types.
+function metricAspectsV2(model, metric) {
+    const expressions = expressionRecords(metric);
+    return compact({
+        'semantic-metric': metricAspectDataV2(model, metric),
+        ...objectAspectsV2(metric),
+        'sql-expressions': expressions.length ? { expressions } : undefined,
+    });
+}
+// The semantic-metric aspect of the second-generation types. `entity` is the
+// metric's entity, declared or derived.
+function metricAspectDataV2(model, metric) {
+    const type = metric.type ?? 'Opaque';
+    const excludedIn = namedProfiles(model)
+        .filter(p => p.metricsExclude === '*' ||
+        p.metricsExclude?.includes(metric.name))
+        .map(p => p.name);
+    return compact({
+        entity: metric.entity,
+        dataType: type,
+        metadataType: columnMetadataType(type),
+        synonyms: (0, osi_converter_1.nonEmpty)(metric.aiContext?.synonyms),
+        excludedInProfiles: (0, osi_converter_1.nonEmpty)(excludedIn),
+    });
+}
+// The records of one expression, in the order written. A plain string gives
+// one record with no `dialect`, so a pull can tell it from a dialect list.
+function expressionRecords({ dialects = [], stringForm }) {
+    return stringForm ?
+        [{ sql: dialects[0].expression }] :
+        dialects.map(d => ({ dialect: d.dialect, sql: d.expression }));
+}
+// The model's named profiles, in declaration order. The default profile has no
+// record of its own.
+function namedProfiles(model) {
+    return (model.profiles ?? []).filter(p => p.name !== resolve_profiles_1.DEFAULT_PROFILE);
+}
+// Maps the IR's logical DataType to a column data-type string for the schema
+// aspect's required `dataType`. A conventional GoogleSQL type name; the field
+// is a free string server-side. Unknown/undefined falls back to STRING.
+function columnDataType(type) {
+    switch (type) {
+        case 'Integer':
+            return 'INT64';
+        case 'Decimal':
+            return 'NUMERIC';
+        case 'Float':
+            return 'FLOAT64';
+        case 'Boolean':
+            return 'BOOL';
+        case 'Date':
+            return 'DATE';
+        case 'Time':
+            return 'TIME';
+        case 'DateTime':
+            return 'DATETIME';
+        case 'DateTimeTz':
+            return 'TIMESTAMP';
+        case 'String':
+        case 'Opaque':
+        default:
+            return 'STRING';
+    }
+}
+// Maps the IR's logical DataType to the schema aspect's required `metadataType`
+// enum (BOOLEAN/NUMBER/STRING/BYTES/DATETIME/TIMESTAMP/GEOSPATIAL/STRUCT/RANGE/
+// OTHER). Numeric types collapse to NUMBER; unknown/undefined falls back to
+// STRING (the loader leaves most fields un-typed).
+function columnMetadataType(type) {
+    switch (type) {
+        case 'Integer':
+        case 'Decimal':
+        case 'Float':
+            return 'NUMBER';
+        case 'Boolean':
+            return 'BOOLEAN';
+        case 'Date':
+        case 'Time':
+        case 'DateTime':
+            return 'DATETIME';
+        case 'DateTimeTz':
+            return 'TIMESTAMP';
+        case 'Opaque':
+            return 'OTHER';
+        case 'String':
+        default:
+            return 'STRING';
+    }
+}
+// Maps the IR's opaque, canonical `dataSource` to a catalog resource string. A
+// clean three-part `project.dataset.table` becomes the BigQuery table's GCP
+// resource name, the form a source is written as; anything else (a query, an
+// under/over-qualified ref) is passed through verbatim so nothing is lost.
+// TODO: settle the linked-resource form for Iceberg / BigLake tables, which may
+// need a different URI shape than the BigQuery managed-table one below.
+function resourcePath(dataSource) {
+    const trimmed = (dataSource ?? '').trim();
+    if (!trimmed || /\s/.test(trimmed) || trimmed.includes('/') ||
+        trimmed.includes(':')) {
+        return trimmed;
+    }
+    const parts = trimmed.split('.').map(unquote);
+    if (parts.length === 3 && parts.every(p => p.length)) {
+        return `bigquery.googleapis.com/projects/${parts[0]}/datasets/${parts[1]}/tables/${parts[2]}`;
+    }
+    return trimmed;
+}
+// ---------------------------------------------------------------------------
+// Naming and small helpers.
+// ---------------------------------------------------------------------------
+// Wraps aspect data (keyed by bare type id, with an optional `@<path>`) as the
+// client's aspect map: each key is the `project.location.type` reference form
+// with the same path, each value the fully-qualified aspectType plus its data.
+function aspectMap(names, byType) {
+    const out = {};
+    for (const [key, data] of Object.entries(byType)) {
+        const type = key.split('@')[0];
+        out[names.aspectRef(type) + key.slice(type.length)] = {
+            aspectType: names.typeName('aspect', type),
+            data
+        };
+    }
+    return out;
+}
+// An entry's aspect map: its required `base` aspects, plus the optional
+// built-in `guidelines` aspect when the object carries
+// `ai_context.instructions`. Kept in one place so every entry kind
+// (model/entity/metric) routes instructions the same way and an object without
+// instructions carries no empty guidelines aspect.
+function aspectsFor(names, base, ai) {
+    const guidelines = guidelinesAspectData(ai);
+    return aspectMap(names, guidelines ? { ...base, guidelines } : base);
+}
+// The native catalog entry source (display name + description), separate from
+// the semantic-* aspect copy that carries full model fidelity.
+function source(displayName, description) {
+    return compact({ displayName, description });
+}
+// Drops undefined-valued keys so the emitted JSON (and its golden) only shows
+// fields the model actually set — matching how the BigQuery emitter omits empty
+// OPTIONS rather than rendering blanks.
+function compact(obj) {
+    const out = {};
+    for (const [k, v] of Object.entries(obj)) {
+        if (v !== undefined)
+            out[k] = v;
+    }
+    return out;
+}
+// Records a generated entry id in `seen`, returning true when it is new. On a
+// repeat it warns and returns false so the caller skips that resource: two
+// entries sharing one name would otherwise have the later write silently
+// overwrite the earlier one on publish.
+function claim(seen, id, kind, label, warnings) {
+    if (seen.has(id)) {
+        warnings.push(`${label}: generated ${kind} id '${id}' duplicates an earlier one; ` +
+            `skipped (rename to avoid overwriting it on publish)`);
+        return false;
+    }
+    seen.add(id);
+    return true;
+}
+function unquote(part) {
+    return part.replace(/^[`"]/, '').replace(/[`"]$/, '');
+}