@google-cloud/kcmd 0.1.0 → 0.2.1

package/build/ts/kcmd/semantic/graph_ddl.js

@@ -16,7 +16,9 @@
 //   3. Element metadata. The per-element `OPTIONS` clause carrying
 //      descriptions and synonyms (BigQuery has one; Spanner has none).
 //   4. SQL text. Which SQL a field binds to: the engine's own dialect entry
-//      (`BIGQUERY` or `SPANNER`), falling back to the `ANSI_SQL` one.
+//      (`BIGQUERY` or `SPANNER`), falling back to the `ANSI_SQL` one. A field
+//      that names a sibling field reads that sibling's SQL in its place (see
+//      fieldSql).
 //
 // ./bigquery supplies the BigQuery engine, and ./spanner the Spanner engine.
 //
@@ -28,6 +30,7 @@ const ir_1 = require("./ir");
 const resolve_inheritance_1 = require("./resolve_inheritance");
 const sql_expr_utils_1 = require("./sql_expr_utils");
 const sql_identifiers_1 = require("./sql_identifiers");
+const sql_parser_1 = require("./sql_parser");
 /**
  * Generates the property-graph DDL for a semantic model on one engine.
  *
@@ -45,8 +48,10 @@ function generateGraphDdl(model, engine) {
     // flattens each subclass's inherited fields down and expands `extends` to the
     // full transitive ancestor set, so labels and property signatures can be read
     // straight off the resolved model. Only the graph legs run this; the KC push
-    // consumes the raw model, so its output is unaffected.
-    const resolved = (0, resolve_inheritance_1.resolveInheritance)(model);
+    // consumes the raw model, so its output is unaffected. An inherited field
+    // that names a field of its ancestor names the subtype's field of that name,
+    // and renders from its SQL (see fieldSql).
+    const resolved = (0, resolve_inheritance_1.resolveInheritance)(model, undefined, { requalify: true });
     warnings.push(...resolved.warnings);
     // The IR requires entities/relationships/metrics, but be defensive against a
     // hand-built or partially-deserialized model rather than throwing on `.map`.
@@ -76,8 +81,7 @@ function generateGraphDdl(model, engine) {
     // A graph node table requires a non-empty KEY: the entity's primary key, else
     // its first unique key (nodeKey). An entity with neither cannot form a valid
     // node, so skip it (and, below, any edge that references it) rather than emit
-    // `KEY()` — invalid DDL. The loader already warns about the missing key; this
-    // records the resulting structural drop.
+    // `KEY()` — invalid DDL.
     const skipped = new Set();
     const validEntities = entities.filter(entity => {
         // An abstract entity is intentionally table-less: skip its node table (it
@@ -142,7 +146,18 @@ function generateGraphDdl(model, engine) {
     // it). The result is keyed by entity name; entries on a skipped entity simply
     // never render.
     const entityByName = new Map(entities.map(e => [e.name, e]));
-    const measurePropertiesByEntity = engine.placeMetrics(metrics, { entityByName, skipped, ancestorsUsed }, warnings);
+    // The SQL `sql` gives, or, when a field on the way has none, what a warning
+    // says it reads: each field on `through`, then those `sql` went through.
+    const readOf = (entity, sql, through) => 'sql' in sql ? { sql: sql.sql } : {
+        reads: missingSibling('through' in sql ? [...through, ...sql.through] : through, entity, engine)
+    };
+    const measurePropertiesByEntity = engine.placeMetrics(metrics, {
+        entityByName,
+        skipped,
+        ancestorsUsed,
+        renderedFieldSql: (entity, field) => readOf(entity, fieldSql(field, entity, engine), [field.name]),
+        localSql: (entity, sql) => readOf(entity, siblingsRead(sql, entity, name => fieldSql(entity.fields.find(f => f.name === name), entity, engine)), []),
+    }, warnings);
     // A subclass's `LABEL <ancestor>` block lists that ancestor's own (flattened)
     // signature, so the label renderer must reach every LABEL CARRIER by name:
     // node-forming entities plus abstract (table-less) supertypes, which have no
@@ -160,7 +175,7 @@ function generateGraphDdl(model, engine) {
     const leftOutUnder = new Map();
     for (const e of validEntities) {
         for (const f of e.fields) {
-            if (!hasOnlyOtherEnginesSql(f, engine))
+            if (!isLeftOut(f, e, engine))
                 continue;
             for (const a of e.extends ?? []) {
                 leftOutUnder.set(a, (leftOutUnder.get(a) ?? new Set()).add(f.name));
@@ -219,11 +234,11 @@ function renderNodeTable(entity, measureProperties, labelByName, leftOutUnder, i
         for (const f of ancestor.fields) {
             // The ancestor's node table leaves out a field this engine has no SQL
             // for, so there is no rendering to defer to.
-            if (engine.fieldExpression(f) === undefined)
+            if (!('sql' in fieldSql(f, ancestor, engine)))
                 continue;
             if (!inheritedRender.has(f.name)) {
-                inheritedRender.set(f.name, renderFieldProperty(f, ancestor.name, engine));
-                inheritedCore.set(f.name, renderFieldPropertyCore(f, ancestor.name, engine));
+                inheritedRender.set(f.name, renderFieldProperty(f, ancestor, engine));
+                inheritedCore.set(f.name, renderFieldPropertyCore(f, ancestor, engine));
             }
         }
     }
@@ -238,13 +253,13 @@ function renderNodeTable(entity, measureProperties, labelByName, leftOutUnder, i
     // subclass loses, rather than mislabeling an added description as a redefined
     // column.
     const renderOwn = (f) => {
-        const own = renderFieldProperty(f, entity.name, engine);
+        const own = renderFieldProperty(f, entity, engine);
         const canonical = inheritedRender.get(f.name);
         if (canonical === undefined)
             return own; // not inherited: render as-is
         if (canonical === own)
             return canonical; // identical: nothing to warn
-        if (renderFieldPropertyCore(f, entity.name, engine) !==
+        if (renderFieldPropertyCore(f, entity, engine) !==
             inheritedCore.get(f.name)) {
             warnings.push(`entity '${entity.name}' remaps inherited property '${f.name}' to a different column/expression than its supertype; ` +
                 `${engine.name} allows only one definition per property under a ` +
@@ -270,19 +285,26 @@ function renderNodeTable(entity, measureProperties, labelByName, leftOutUnder, i
     // pruning normally strips it before generation, but a caller that generates
     // DDL straight from a bindingOptional load without pruning could still reach
     // here with one, such as a purely logical field. Skip it too rather than emit
-    // `<name>` as a phantom bare column the source table does not have.
+    // `<name>` as a phantom bare column the source table does not have. A field
+    // that names a sibling the engine has no SQL for has none either, so it is
+    // skipped with a warning naming that sibling.
     const boundFields = entity.fields.filter(f => {
-        if (engine.fieldExpression(f) !== undefined)
+        const sql = fieldSql(f, entity, engine);
+        if ('sql' in sql)
             return true;
-        warnings.push((0, ir_1.isFieldBound)(f) ?
-            `entity '${entity.name}': field '${f.name}' has no '${engine.dialect}' or 'ANSI_SQL' expression; omitted from the ` +
-                (inheritedRender.has(f.name) ?
-                    `node table's own label, and the supertype's label keeps ` +
-                        `the supertype's definition` :
-                    `node table`) :
-            `entity '${entity.name}': field '${f.name}' has no column under ` +
-                `this binding; omitted from the node table (bind it, or ` +
-                `govern the logical model in Knowledge Catalog instead)`);
+        const omitted = inheritedRender.has(f.name) ?
+            `node table's own label, and the supertype's label keeps the ` +
+                `supertype's definition` :
+            `node table`;
+        warnings.push('through' in sql ?
+            `entity '${entity.name}': field '${f.name}' reads ${missingSibling(sql.through, entity, engine)}; omitted from ` +
+                `the ${omitted}` :
+            (0, ir_1.isFieldBound)(f) ?
+                `entity '${entity.name}': field '${f.name}' has no '${engine.dialect}' or 'ANSI_SQL' expression; omitted from the ` +
+                    omitted :
+                `entity '${entity.name}': field '${f.name}' has no column under ` +
+                    `this binding; omitted from the node table (bind it, or ` +
+                    `govern the logical model in Knowledge Catalog instead)`);
         return false;
     });
     const properties = [...boundFields.map(renderOwn), ...measureProperties];
@@ -320,7 +342,7 @@ function renderNodeTable(entity, measureProperties, labelByName, leftOutUnder, i
         lines.push(line(2, 'DEFAULT LABEL'));
     if (labelOpts)
         lines.push(line(2, labelOpts));
-    lines.push(...propertiesClause(properties, entity.fields.some(f => hasOnlyOtherEnginesSql(f, engine))));
+    lines.push(propertiesClause(properties));
     // Inheritance: declare one LABEL per transitive ancestor (resolveInheritance
     // expanded `extends` to the full ancestor set), each re-listing that
     // ancestor's properties so `MATCH (:Ancestor)` also matches this subclass
@@ -347,7 +369,6 @@ function renderNodeTable(entity, measureProperties, labelByName, leftOutUnder, i
         }
         lines.push(line(2, `LABEL ${(0, sql_identifiers_1.quoteIfReserved)(ancestorName)}`));
         let signature;
-        let anyLeftOut = false;
         if (ancestor.abstract) {
             // Render each inherited field from THIS subtype's own binding, keyed by
             // the abstract ancestor's field names. Skip any the subtype leaves
@@ -356,38 +377,97 @@ function renderNodeTable(entity, measureProperties, labelByName, leftOutUnder, i
             // simply does not expose that property under the shared label.
             signature = [];
             for (const af of ancestor.fields) {
-                if (leftOutUnder.get(ancestorName)?.has(af.name)) {
-                    anyLeftOut = true;
+                if (leftOutUnder.get(ancestorName)?.has(af.name))
                     continue;
-                }
                 const child = boundFields.find(cf => cf.name === af.name);
                 if (child) {
-                    signature.push(renderFieldProperty(child, entity.name, engine));
+                    signature.push(renderFieldProperty(child, entity, engine));
                 }
             }
         }
         else {
             // Only the fields the ancestor's own node table lists.
             signature =
-                ancestor.fields.filter(f => engine.fieldExpression(f) !== undefined)
-                    .map(f => renderFieldProperty(f, ancestor.name, engine));
-            anyLeftOut =
-                ancestor.fields.some(f => hasOnlyOtherEnginesSql(f, engine));
+                ancestor.fields.filter(f => 'sql' in fieldSql(f, ancestor, engine))
+                    .map(f => renderFieldProperty(f, ancestor, engine));
         }
-        lines.push(...propertiesClause(signature, anyLeftOut));
+        lines.push(propertiesClause(signature));
     }
     return lines.join('\n');
 }
-// The properties clause of a table or label. When `anyLeftOut` is set, an
-// empty list gives NO PROPERTIES, since a missing clause exposes every column.
-function propertiesClause(properties, anyLeftOut) {
-    if (properties.length)
-        return [propertiesBlock(properties)];
-    return anyLeftOut ? [line(2, 'NO PROPERTIES')] : [];
+// The properties clause of a node table, label or edge. An empty list gives NO
+// PROPERTIES: BigQuery and Spanner read a missing clause as PROPERTIES ARE ALL
+// COLUMNS, which would expose every column of the table, including one a
+// profile leaves out.
+function propertiesClause(properties) {
+    return properties.length ? propertiesBlock(properties) :
+        line(2, 'NO PROPERTIES');
 }
-// Whether a field has SQL, but none the engine reads.
-function hasOnlyOtherEnginesSql(field, engine) {
-    return (0, ir_1.isFieldBound)(field) && engine.fieldExpression(field) === undefined;
+// Whether a field has SQL, but the engine has none to render it from: none of
+// its SQL is the engine's, or it names a sibling that has none.
+function isLeftOut(field, owner, engine) {
+    return (0, ir_1.isFieldBound)(field) && !('sql' in fieldSql(field, owner, engine));
+}
+// The SQL `field` of `owner` renders from on `engine`. The text is the
+// engine's (fieldExpression), and a field may name a sibling field in it as
+// `<owner>.<field>`. Each sibling it names is replaced with that sibling's
+// own SQL for the engine, in parentheses unless that SQL is a lone column, and
+// every other column loses the owner's qualifier; the rest of the text stays
+// as written. Both engines' SQL is read as BigQuery's.
+// Throws when the field names itself, directly or through its siblings, since
+// no SQL can render it; push validation rejects such a model first
+// (validateExpressionReferences).
+function fieldSql(field, owner, engine, path = []) {
+    const expr = engine.fieldExpression(field);
+    if (expr === undefined)
+        return { noSql: true };
+    const chain = [...path, field.name];
+    return siblingsRead(expr, owner, name => {
+        if (chain.includes(name)) {
+            throw new Error((0, sql_expr_utils_1.selfReferenceMessage)(owner.name, chain.slice(chain.indexOf(name))));
+        }
+        return fieldSql(owner.fields.find(f => f.name === name), owner, engine, chain);
+    });
+}
+// `expr`, SQL on `owner`'s table, local to that table, with each sibling it
+// names as `<owner>.<field>` replaced with what `read` gives for that sibling
+// (see fieldSql). When a sibling has no SQL, `through` runs from it to the
+// sibling with none.
+function siblingsRead(expr, owner, read) {
+    const named = new Set(((0, sql_expr_utils_1.qualifiedColumns)(expr, owner.name) ?? [])
+        .filter(c => owner.fields.some(f => f.name === c)));
+    const replacements = new Map();
+    for (const name of named) {
+        const sibling = read(name);
+        if ('noSql' in sibling)
+            return { through: [name] };
+        if ('through' in sibling)
+            return { through: [name, ...sibling.through] };
+        // A sibling bound to a reserved column, such as `order`, reads it quoted.
+        // One bound to the column of its own name reads the same either way.
+        const sql = (0, sql_identifiers_1.quoteIfReserved)(sibling.sql);
+        if (sql !== name)
+            replacements.set(name, parenthesized(sql));
+    }
+    return { sql: (0, sql_expr_utils_1.withSiblingsSpliced)(expr, owner.name, replacements) };
+}
+// `sql` in parentheses, unless it is a lone column or already in them.
+function parenthesized(sql) {
+    const tree = (0, sql_parser_1.sqlExpressionTree)(sql, 'BIGQUERY');
+    const kind = tree && Object.keys(tree).length === 1 ?
+        Object.keys(tree)[0] :
+        undefined;
+    return kind === 'column' || kind === 'paren' ? sql : `(${sql})`;
+}
+// What a warning says a field reads when it names a sibling the engine has no
+// SQL for: each sibling on `through`, and what the last one lacks.
+function missingSibling(through, owner, engine) {
+    const last = owner.fields.find(f => f.name === through[through.length - 1]);
+    const lacks = last && (0, ir_1.isFieldBound)(last) ?
+        `has no '${engine.dialect}' or 'ANSI_SQL' expression` :
+        `has no column under this binding`;
+    return `${through.map(n => `field '${n}'`).join(', which reads ')}, ` +
+        `which ${lacks}`;
 }
 // Renders a field as a graph property: a bare column when the expression is
 // just the column, else `<expr> AS <name>`. Attaches the engine's metadata
@@ -395,8 +475,8 @@ function hasOnlyOtherEnginesSql(field, engine) {
 // per the `derived_property` rule. A node table leaves out a field with no
 // expression, with a warning. An association edge exposes it as a bare column
 // under its name.
-function renderFieldProperty(field, entity, engine) {
-    const core = renderFieldPropertyCore(field, entity, engine);
+function renderFieldProperty(field, owner, engine) {
+    const core = renderFieldPropertyCore(field, owner, engine);
     const opts = engine.fieldOptions(field);
     return opts ? `${core} ${opts}` : core;
 }
@@ -409,13 +489,17 @@ function renderFieldProperty(field, entity, engine) {
 // renderOwn). This is also the single place a field expression is lowered to
 // its table-local form: the resolve-inheritance pass first rewrites an
 // inherited field's expression into the child's frame (localizeInheritedField),
-// then this strips the entity's own qualifier -- one normalization, in one
+// then fieldSql strips the entity's own qualifier and reads each sibling the
+// expression names through that sibling's SQL -- one normalization, in one
 // place, applied to own and inherited fields alike.
-function renderFieldPropertyCore(field, entity, engine) {
-    const expr = engine.fieldExpression(field);
-    const local = expr !== undefined ? (0, sql_expr_utils_1.stripQualifier)(expr, entity) : field.name;
+function renderFieldPropertyCore(field, owner, engine) {
+    const sql = fieldSql(field, owner, engine);
+    const local = 'sql' in sql ? sql.sql : field.name;
     const alias = (0, sql_identifiers_1.quoteIfReserved)(field.name);
-    return local === field.name ? alias :
+    // A field bound to the column of its name, such as `order` bound to
+    // `` `order` ``, is that column.
+    return local === field.name || local === alias ?
+        alias :
         `${(0, sql_identifiers_1.quoteIfReserved)(local)} AS ${alias}`;
 }
 // The key an entity's node is keyed by: its primary key, else its first
@@ -488,10 +572,14 @@ function renderEdgeTable(rel, entitiesByName, engine, warnings) {
         line(2, `DESTINATION KEY(${destFk}) REFERENCES ${(0, sql_identifiers_1.quoteIfReserved)(rel.destination.entity)}(${destRef})`),
     ];
     // Edge description and synonyms attach to the DEFAULT LABEL: after the
-    // SOURCE/DESTINATION clauses (grammar: element_table_definition).
+    // SOURCE/DESTINATION clauses, before the properties clause (grammar:
+    // element_table_definition).
     const labelOpts = engine.elementOptions(rel.description, rel.aiContext);
     if (labelOpts)
         lines.push(line(2, labelOpts));
+    // A foreign-key relationship declares no fields of its own, so its edge
+    // lists none.
+    lines.push(propertiesClause([]));
     return lines.join('\n');
 }
 // Renders a many-to-many edge backed by an association (junction) table. Unlike
@@ -531,18 +619,23 @@ function renderAssociationEdge(rel, assoc, entitiesByName, engine, warnings) {
     const labelOpts = engine.elementOptions(rel.description, rel.aiContext);
     if (labelOpts)
         lines.push(line(2, labelOpts));
-    // The junction's own non-key fields are the edge's properties.
-    const properties = (assoc.fields ?? [])
+    // The junction's own non-key fields are the edge's properties. They name
+    // each other under the relationship's name.
+    const owner = { name: rel.name, fields: assoc.fields ?? [] };
+    const properties = owner.fields
         .filter(f => {
-        if (engine.fieldExpression(f) !== undefined || !(0, ir_1.isFieldBound)(f)) {
+        const sql = fieldSql(f, owner, engine);
+        if ('sql' in sql || !(0, ir_1.isFieldBound)(f))
             return true;
-        }
-        warnings.push(`relationship '${rel.name}': field '${f.name}' has no '${engine.dialect}' or 'ANSI_SQL' expression; omitted from ` +
-            `the edge table`);
+        warnings.push(`relationship '${rel.name}': field '${f.name}' ` +
+            ('through' in sql ?
+                `reads ${missingSibling(sql.through, owner, engine)}` :
+                `has no '${engine.dialect}' or 'ANSI_SQL' expression`) +
+            `; omitted from the edge table`);
         return false;
     })
-        .map(f => renderFieldProperty(f, rel.name, engine));
-    lines.push(...propertiesClause(properties, (assoc.fields ?? []).some(f => hasOnlyOtherEnginesSql(f, engine))));
+        .map(f => renderFieldProperty(f, owner, engine));
+    lines.push(propertiesClause(properties));
     return lines.join('\n');
 }
 // All generated indentation flows through this single mechanism: one nesting

package/build/ts/kcmd/semantic/import_model.d.ts

@@ -1,4 +1,5 @@
 import { SerializedFile } from './osi_converter';
+import { SourceDialects } from './source_dialect';
 /** Options controlling how a semantic model YAML document is imported. */
 export interface ImportSemanticModelOptions {
     /**
@@ -7,9 +8,22 @@ export interface ImportSemanticModelOptions {
     flavor?: 'ossie' | 'google';
     /**
      * First-party GCP container resource name used to qualify bare table
-     * sources.
+     * sources, or the name of a Knowledge Catalog entry group
+     * (`projects/<p>/locations/<l>/entryGroups/<g>`) whose table entries complete
+     * them by their fullyQualifiedName.
      */
     sourcePrefix?: string;
+    /**
+     * Lists every entry of an entry group, as `CatalogClient.listEntries` does.
+     * Required when `sourcePrefix` names an entry group.
+     */
+    listEntries?: EntryLister;
+    /**
+     * Reads the dialect of each Cloud SQL, Spanner and third-party table the
+     * files `texts` name, as `sourceDialects` does for push. Without it, the
+     * dialect of each such table is unknown.
+     */
+    readDialects?: (texts: string[]) => Promise<SourceDialects>;
     /** When true, translates vendor or double-quoted ANSI SQL to `BIGQUERY`. */
     transpile?: boolean;
     /**
@@ -18,16 +32,24 @@ export interface ImportSemanticModelOptions {
      */
     strict?: boolean;
     /**
-     * Sibling `<model>.profile.<name>.yaml` files located beside the input file,
-     * used when converting a Google-flavor model to vanilla (`'ossie'`).
+     * Sibling `<model>.profile.<name>.yaml` files located beside the input file.
+     * A conversion to vanilla writes them into the model file, and a
+     * Google-flavor import of a Google-flavor model writes them beside it.
      */
     profileFiles?: SerializedFile[];
     /**
      * The input file's name. The warning that a JSON input was written as YAML
-     * names it. Defaults to `<model>.yaml`.
+     * names it, or `<model>.yaml` when it is unset. When set, in strict mode
+     * each message about the model's bindings names the input file or profile
+     * file that holds them.
      */
     fileName?: string;
 }
+/** Lists the entries of an entry group. */
+export type EntryLister = (project: string, location: string, entryGroup: string) => AsyncIterable<{
+    name: string;
+    fullyQualifiedName?: string;
+}>;
 /** Result of importing a semantic model YAML document. */
 export interface ImportSemanticModelResult {
     /** True when `errors` is empty and `files` was produced. */