@google-cloud/kcmd 0.1.0 → 0.2.0

package/build/ts/kcmd/semantic/sql_expr_utils.js

@@ -2,87 +2,209 @@
 // Helpers for reading and rewriting entity-qualified SQL expressions across the
 // semantic-model layer.
 //
-// Expressions reference columns as `<Entity>.<column>`. The SQL parser
+// An expression names a field of its entity as `<Entity>.<field>`, and a
+// column of its table without a qualifier. The SQL parser
 // (`columnReferences`, `entitiesReached`, `fieldsReadOn`) reads column and
 // entity references across dialects for the loader, validator, profile checks,
-// pull reader, and transpiler. The graph generators still use the literal-
-// aware regex helpers (`referencedEntityNames`, `stripQualifier`).
+// pull reader, and transpiler. The graph generators read them with the parser
+// too: `measureEntities` places a measure, `singleAggregate` finds its
+// aggregate, `withoutQualifier` makes an expression table-local,
+// `qualifiedColumns` finds the sibling fields it names and
+// `withSiblingsSpliced` puts their SQL in their place, and `requalified` moves
+// an inherited expression onto the entity inheriting it.
 //
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.SUPPORTED_AGGREGATES = exports.STRING_LITERAL = void 0;
-exports.escapeRegExp = escapeRegExp;
-exports.blankStringLiterals = blankStringLiterals;
-exports.mapOutsideStringLiterals = mapOutsideStringLiterals;
-exports.referencedEntityNames = referencedEntityNames;
-exports.stripQualifier = stripQualifier;
+exports.SUPPORTED_AGGREGATES = void 0;
+exports.unusedName = unusedName;
+exports.withoutQualifier = withoutQualifier;
+exports.withSiblingsSpliced = withSiblingsSpliced;
+exports.qualifiedColumns = qualifiedColumns;
+exports.selfReferenceMessage = selfReferenceMessage;
+exports.requalified = requalified;
+exports.measureEntities = measureEntities;
 exports.sqlWithoutComments = sqlWithoutComments;
 exports.fieldBindingWithoutComments = fieldBindingWithoutComments;
 exports.columnReferences = columnReferences;
 exports.columnsForSource = columnsForSource;
 exports.entitiesReached = entitiesReached;
 exports.fieldsReadOn = fieldsReadOn;
+exports.singleAggregate = singleAggregate;
 exports.isSingleAggregate = isSingleAggregate;
 exports.isColumnName = isColumnName;
 const ir_1 = require("./ir");
+const sql_identifiers_1 = require("./sql_identifiers");
 const sql_parser_1 = require("./sql_parser");
-// Matches a single- or double-quoted SQL string literal, honoring backslash
-// escapes. (Triple-quoted / raw literals are uncommon in these expressions and
-// are treated as ordinary text.)
-exports.STRING_LITERAL = /'(?:\\.|[^'\\])*'|"(?:\\.|[^"\\])*"/g;
-// Escapes a string so it can be embedded literally in a RegExp.
-function escapeRegExp(s) {
-    return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
-}
-// Replaces string-literal contents with blanks of equal length, so scanning sees
-// literal-free text without shifting any offsets.
-function blankStringLiterals(expression) {
-    return expression.replace(exports.STRING_LITERAL, m => ' '.repeat(m.length));
-}
-// Applies `fn` only to the parts of `expression` that lie outside string
-// literals, leaving each literal verbatim.
-function mapOutsideStringLiterals(expression, fn) {
+// The dialect a graph generator reads `dialect` in: an ANSI_SQL text, or a
+// text with no dialect, is read as BigQuery.
+function graphReading(dialect) {
+    return !dialect || dialect === 'ANSI_SQL' ? 'BIGQUERY' : dialect;
+}
+// Each `<name>.` qualifier that `expression`, read in `dialect`, writes before
+// a column, for a name in `names`, as indexes into the text's tokens, in the
+// order written. A name matches case-sensitively, bare or quoted as the
+// dialect quotes an identifier, and not after a dot, so the second `orders` in
+// the struct path `orders.orders.id` is not one. The SQL parser decides which
+// candidates are qualifiers, so a string literal, a comment or a function
+// namespace such as `NET.` in `NET.HOST(orders.url)` holds none. The parser
+// reads each candidate as a placeholder name, so one it reserves as a keyword,
+// such as `Date` in `Date.days`, still reads as a qualifier. When the parser
+// cannot read the text even so, every candidate is taken as a qualifier.
+// Undefined when the engine cannot tokenize the text.
+function qualifiersIn(expression, names, dialect) {
+    const tokens = (0, sql_parser_1.sqlTokens)(expression, dialect);
+    if (!tokens)
+        return undefined;
+    const chars = Array.from(expression);
+    const asWritten = (t) => chars.slice(t.span.start, t.span.end).join('');
+    const candidates = [];
+    tokens.forEach((t, i) => {
+        if (names.has(t.text) && tokens[i + 1]?.token_type === 'DOT' &&
+            tokens[i - 1]?.token_type !== 'DOT' &&
+            (t.token_type === 'QUOTED_IDENTIFIER' || asWritten(t) === t.text)) {
+            candidates.push(i);
+        }
+    });
+    if (!candidates.length)
+        return { tokens, at: [] };
+    // A placeholder stem the text does not hold.
+    const stem = unusedName(expression, 'kcmd_qualifier_');
+    let text = '';
+    let end = 0;
+    candidates.forEach((i, k) => {
+        text += chars.slice(end, tokens[i].span.start).join('') + stem + k;
+        end = tokens[i].span.end;
+    });
+    text += chars.slice(end).join('');
+    const columns = columnReferences(text, dialect);
+    const at = !columns ? candidates : candidates.filter((_, k) => columns.some(c => c.qualifier?.toLowerCase() === stem + k));
+    return { tokens, at };
+}
+// `base`, a lowercase name, with underscores added until `text` does not hold
+// it in any case.
+function unusedName(text, base) {
+    let name = base;
+    while (text.toLowerCase().includes(name))
+        name += '_';
+    return name;
+}
+// `expression` without each `<qualifier>.` it writes before a column (see
+// qualifiersIn), so it reads the columns of the qualifier's own table. The
+// rest of the text stays as written. The text is read in `dialect`; an
+// ANSI_SQL text, or one with no dialect, is read as BigQuery, as the graph
+// generators read it. Returns `expression` itself when it writes no such
+// qualifier or the engine cannot tokenize it.
+function withoutQualifier(expression, qualifier, dialect) {
+    return spliced(expression, qualifier, graphReading(dialect), new Map());
+}
+// `expression`, BigQuery SQL on `qualifier`'s table, as withoutQualifier gives
+// it, except that each `<qualifier>.<field>` whose field `siblings` holds is
+// replaced, whole, by the SQL it maps to, as given. The graph generators put
+// the SQL of each sibling field an expression names in its place so (see
+// graph_ddl.fieldSql).
+function withSiblingsSpliced(expression, qualifier, siblings) {
+    return spliced(expression, qualifier, 'BIGQUERY', siblings);
+}
+// `expression`, read in `dialect`, without each `<qualifier>.` it writes
+// before a column (see qualifiersIn), and with each `<qualifier>.<column>`
+// whose column `replacements` holds replaced, whole, by the SQL it maps to.
+function spliced(expression, qualifier, dialect, replacements) {
+    const read = qualifiersIn(expression, new Set([qualifier]), dialect);
+    if (!read?.at.length)
+        return expression;
+    const { tokens, at } = read;
+    const chars = Array.from(expression);
     let out = '';
-    let last = 0;
-    for (const m of expression.matchAll(exports.STRING_LITERAL)) {
-        out += fn(expression.slice(last, m.index));
-        out += m[0];
-        last = m.index + m[0].length;
+    let end = 0;
+    for (const i of at) {
+        out += chars.slice(end, tokens[i].span.start).join('');
+        const column = tokens[i + 2];
+        const replacement = column && replacements.get(nameOf(column, chars));
+        if (replacement !== undefined) {
+            out += replacement;
+            end = column.span.end;
+        }
+        else {
+            // The qualifier, its dot, and what lies between the dot and the column.
+            end = column?.span.start ?? tokens[i + 1].span.end;
+        }
     }
-    out += fn(expression.slice(last));
-    return out;
-}
-// Builds a regex matching an `<entity>.` qualifier, including the BigQuery
-// backtick-quoted form (`` `entity`. ``). A negative lookbehind keeps the name
-// from matching inside a larger identifier (e.g. `customer_orders.`) or after a
-// dot in a struct path (e.g. `orders.customer.id`), and the optional backticks
-// let it match whether or not the identifier is quoted.
-function entityQualifier(name, flags = '') {
-    return new RegExp(`(?<![\\w\`.])\`?${escapeRegExp(name)}\`?\\.`, flags);
-}
-// Returns the entity names whose `<name>.` qualifier appears in an expression,
-// in the order they first appear, ignoring text inside string literals.
-// Matching is case-sensitive (the emitter's behavior).
-function referencedEntityNames(expression, entityNames) {
-    const scannable = blankStringLiterals(expression);
-    const hits = [];
-    for (const name of entityNames) {
-        const m = entityQualifier(name).exec(scannable);
-        if (m)
-            hits.push({ name, at: m.index });
+    return out + chars.slice(end).join('');
+}
+// The column written after each `<qualifier>.` that `expression` writes before
+// a column (see qualifiersIn), without its quotes, in the order written, so
+// ``orders.a + `orders`.`b`.c`` gives `a` and `b`. The text is read in
+// `dialect`, as withoutQualifier reads it, so a qualifier the parser reserves,
+// such as `Date` in `Date.days`, gives its column too. Undefined when the
+// engine cannot tokenize the text.
+function qualifiedColumns(expression, qualifier, dialect) {
+    const read = qualifiersIn(expression, new Set([qualifier]), graphReading(dialect));
+    if (!read)
+        return undefined;
+    const chars = Array.from(expression);
+    return read.at.filter(i => read.tokens[i + 2])
+        .map(i => nameOf(read.tokens[i + 2], chars));
+}
+// The name token `t` of the text whose characters are `chars` gives, as
+// written, without its quotes.
+function nameOf(t, chars) {
+    return t.token_type === 'QUOTED_IDENTIFIER' ?
+        t.text :
+        chars.slice(t.span.start, t.span.end).join('');
+}
+// The error for a field of `owner` that names itself. `cycle` starts at that
+// field and runs through the fields it names back to it, and `where` follows
+// the field's name, as ` in model 'm' (doc)` does. The graph renders a field
+// that names a sibling field from that sibling's own SQL (see
+// graph_ddl.fieldSql), so a field that reaches itself has none.
+function selfReferenceMessage(owner, cycle, where = '') {
+    const [field, ...through] = cycle;
+    return `field '${owner}.${field}'${where} names itself` +
+        (through.length ?
+            ` through ${through.map(n => `'${owner}.${n}'`).join(', ')}` :
+            '') +
+        `; a field expression may name other fields of its entity, but not ` +
+        `itself, directly or through other fields` +
+        (through.length ? '' :
+            `; to read the column of that name, write it ` +
+                `without the qualifier: '${(0, sql_identifiers_1.quoteIfReserved)(field)}'`);
+}
+// `expression` with each `<from>.` qualifier it writes before a column (see
+// qualifiersIn) renamed `<to>.`, quoted as it was written. The rest of the
+// text stays as written. The text is read in `dialect`, as withoutQualifier
+// reads it. Returns `expression` itself when it writes no such qualifier or
+// the engine cannot tokenize it.
+function requalified(expression, from, to, dialect) {
+    const read = qualifiersIn(expression, new Set([from]), graphReading(dialect));
+    if (!read?.at.length)
+        return expression;
+    const { tokens, at } = read;
+    const chars = Array.from(expression);
+    let out = '';
+    let end = 0;
+    for (const i of at) {
+        const { start, end: after } = tokens[i].span;
+        out += chars.slice(end, start).join('') +
+            chars.slice(start, after).join('').replace(from, () => to);
+        end = after;
     }
-    return hits.sort((a, b) => a.at - b.at).map(h => h.name);
+    return out + chars.slice(end).join('');
 }
-// Removes the `<entity>.` qualifier (bare or backtick-quoted) so an expression
-// references table-local columns, without touching text inside string literals.
-function stripQualifier(expression, entity) {
-    const re = entityQualifier(entity, 'g');
-    return mapOutsideStringLiterals(expression, seg => seg.replace(re, ''));
+// The entities from `entityNames` whose qualifier a metric's BigQuery SQL
+// `expression` writes before a column (see qualifiersIn), deduplicated in the
+// order written, or else `entity`, the metric's anchor, when given. The
+// BigQuery generator places the metric's measure on the entity this gives
+// when it gives one, and a BigQuery push rejects a metric it gives several or
+// none for (see validate.ts).
+function measureEntities(expression, entityNames, entity) {
+    const read = qualifiersIn(expression, new Set(entityNames), 'BIGQUERY');
+    const reached = [...new Set(read?.at.map(i => read.tokens[i].text) ?? [])];
+    return reached.length || !entity ? reached : [entity];
 }
 // `text`, written in `dialect`, with its comments removed. An ANSI_SQL text, or
 // a text with no dialect, is read as BigQuery, as the loader and the graph
 // generators read it.
 function sqlWithoutComments(text, dialect) {
-    return (0, sql_parser_1.withoutComments)(text, !dialect || dialect === 'ANSI_SQL' ? 'BIGQUERY' : dialect);
+    return (0, sql_parser_1.withoutComments)(text, graphReading(dialect));
 }
 // `field`'s binding, as fieldBinding gives it, with its comments removed.
 // Undefined when the field is unbound. The target text is read in
@@ -124,8 +246,11 @@ function columnReferences(expression, dialect) {
 // an `ANSI_SQL` entry as its own SQL. When the sources do not imply a single
 // dialect, the generic parser reads the text, and any bare identifier that
 // BigQuery's parser does not also read as a bare column (such as a `MONTH`
-// date part or a double-quoted string literal) is filtered out.
+// date part or a double-quoted string literal) is filtered out. When their
+// dialect is null, which no lookup read, an `ANSI_SQL` text is not read.
 function columnsForSource(text, dialect, impliedDialect) {
+    if (dialect === 'ANSI_SQL' && impliedDialect === null)
+        return undefined;
     if (dialect === 'ANSI_SQL' && impliedDialect) {
         return columnReferences(text, impliedDialect);
     }
@@ -165,15 +290,18 @@ function fieldsReadOn(columns, entityName) {
 // The aggregates BigQuery accepts in a MEASURE, its complete list. BigQuery
 // rejects any other there, such as COUNTIF or APPROX_COUNT_DISTINCT.
 exports.SUPPORTED_AGGREGATES = ['SUM', 'AVG', 'COUNT', 'MIN', 'MAX'];
-// Whether `expression`, read in `dialect`, is one of SUPPORTED_AGGREGATES over
-// one operand, or COUNT(*), with at most DISTINCT or ALL. The operand holds no
-// other aggregate, window or subquery.
-function isSingleAggregate(expression, dialect) {
+// When `expression`, read in `dialect`, is one of SUPPORTED_AGGREGATES over
+// one operand, or COUNT(*), with at most DISTINCT or ALL, and the operand holds
+// no other aggregate, window or subquery: the function's name as written, the
+// operand as written without the parentheses around it (so
+// `COUNT(DISTINCT(x))` gives `x`), and whether the call is DISTINCT. COUNT(*)
+// gives the operand `*`. Undefined otherwise.
+function singleAggregate(expression, dialect) {
     const node = unwrapped((0, sql_parser_1.sqlExpressionTree)(expression, dialect));
     const kind = node && Object.keys(node).length === 1 ? Object.keys(node)[0] :
         undefined;
     if (!kind || !exports.SUPPORTED_AGGREGATES.includes(kind.toUpperCase())) {
-        return false;
+        return undefined;
     }
     const call = node[kind];
     const star = kind === 'count' && call.star === true && !call.this;
@@ -181,12 +309,57 @@ function isSingleAggregate(expression, dialect) {
     if (!star &&
         (!operand || operand.star || operand.tuple ||
             nestsAggregateOrSubquery(operand))) {
-        return false;
+        return undefined;
     }
     // The parser keeps the call's name in `name` or `original_name`.
     const known = new Set(['this', 'star', 'distinct', 'name', 'original_name']);
-    return Object.entries(call).every(([key, v]) => known.has(key) || v === null || v === undefined ||
-        v === false || (Array.isArray(v) && v.length === 0));
+    if (!Object.entries(call).every(([key, v]) => known.has(key) || v === null || v === undefined ||
+        v === false || (Array.isArray(v) && v.length === 0))) {
+        return undefined;
+    }
+    // The call as written: its name, `(`, DISTINCT or ALL if written, the
+    // operand and `)`, inside any parentheses around the whole call.
+    const tokens = unparenthesized((0, sql_parser_1.sqlTokens)(expression, dialect) ?? []);
+    if (tokens.length < 4 || tokens[1].token_type !== 'L_PAREN' ||
+        tokens[tokens.length - 1].token_type !== 'R_PAREN') {
+        return undefined;
+    }
+    let inner = tokens.slice(2, -1);
+    if (inner[0]?.token_type === 'DISTINCT' || inner[0]?.token_type === 'ALL') {
+        inner = inner.slice(1);
+    }
+    inner = unparenthesized(inner);
+    if (!inner.length)
+        return undefined;
+    const chars = Array.from(expression);
+    return {
+        fn: tokens[0].text,
+        operand: chars.slice(inner[0].span.start, inner[inner.length - 1].span.end)
+            .join(''),
+        distinct: call.distinct === true,
+    };
+}
+// Whether `expression`, read in `dialect`, is a single aggregate (see
+// singleAggregate), the form a BigQuery measure takes.
+function isSingleAggregate(expression, dialect) {
+    return singleAggregate(expression, dialect) !== undefined;
+}
+// `tokens` without the parentheses around all of them.
+function unparenthesized(tokens) {
+    while (tokens.length >= 2 && tokens[0].token_type === 'L_PAREN') {
+        let depth = 0;
+        let closesAt = -1;
+        for (let i = 0; i < tokens.length && closesAt < 0; i++) {
+            if (tokens[i].token_type === 'L_PAREN')
+                depth++;
+            if (tokens[i].token_type === 'R_PAREN' && --depth === 0)
+                closesAt = i;
+        }
+        if (closesAt !== tokens.length - 1)
+            break;
+        tokens = tokens.slice(1, -1);
+    }
+    return tokens;
 }
 // The parser's tree `node` without the parentheses and comments around it.
 function unwrapped(node) {

package/build/ts/kcmd/semantic/sql_parser.d.ts

@@ -23,7 +23,8 @@ export interface PolyglotEngine {
         error?: string;
     };
 }
-interface SqlToken {
+export interface SqlToken {
+    token_type: string;
     text: string;
     span: {
         start: number;
@@ -33,12 +34,14 @@ interface SqlToken {
     trailing_comments: string[];
 }
 export declare function loadSqlEngine(): Promise<PolyglotEngine>;
+export declare function loadedSqlEngine(): PolyglotEngine | undefined;
 export interface SqlColumn {
     qualifier?: string;
     name: string;
 }
 export declare function sqlColumns(expression: string, dialect: string): SqlColumn[] | undefined;
+export declare function soleColumn(expression: string, dialect: string): SqlColumn | undefined;
 export declare function sqlExpressionTree(expression: string, dialect: string): any;
+export declare function sqlTokens(expression: string, dialect: string): SqlToken[] | undefined;
 export declare function holdsNoSql(expression: string, dialect: string): boolean;
 export declare function withoutComments(expression: string, dialect: string): string;
-export {};

package/build/ts/kcmd/semantic/sql_parser.js

@@ -8,8 +8,11 @@
 // first, and so does a test.
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.loadSqlEngine = loadSqlEngine;
+exports.loadedSqlEngine = loadedSqlEngine;
 exports.sqlColumns = sqlColumns;
+exports.soleColumn = soleColumn;
 exports.sqlExpressionTree = sqlExpressionTree;
+exports.sqlTokens = sqlTokens;
 exports.holdsNoSql = holdsNoSql;
 exports.withoutComments = withoutComments;
 const node_fs_1 = require("node:fs");
@@ -42,6 +45,10 @@ function loadSqlEngine() {
     }
     return enginePromise;
 }
+// The engine loadSqlEngine has loaded, or undefined before it has.
+function loadedSqlEngine() {
+    return loadedEngine;
+}
 // The engine's dialect for each kcmd dialect. The engine has no Spanner
 // dialect, so Spanner's GoogleSQL is read as BigQuery's.
 const PARSER_DIALECTS = {
@@ -73,6 +80,21 @@ function sqlColumns(expression, dialect) {
     collectColumns(tree, columns, new Set(), PARSER_DIALECTS[dialect.toUpperCase()] ?? 'generic');
     return columns;
 }
+// The column `expression` is, read in `dialect`, when it is one column and
+// nothing else, perhaps in parentheses: `orders.amount`, `` (`order`) `` or
+// `amount`, but not a struct path such as `orders.address.city` or any other
+// expression. Undefined otherwise, or when the engine cannot read it. Throws
+// when the engine is not loaded.
+function soleColumn(expression, dialect) {
+    let tree = sqlExpressionTree(expression, dialect);
+    const kind = (node) => node && Object.keys(node).length === 1 ? Object.keys(node)[0] : undefined;
+    while (kind(tree) === 'paren')
+        tree = tree.paren.this;
+    if (kind(tree) !== 'column')
+        return undefined;
+    const columns = sqlColumns(expression, dialect);
+    return columns?.length === 1 ? columns[0] : undefined;
+}
 // The engine's tree of `expression` read as one expression in `dialect`, or
 // undefined when it cannot be read so or is empty. A text that starts with a
 // keyword, such as `Order.amount`, is read again inside parentheses on lines

package/build/ts/kcmd/semantic/transpile.d.ts

@@ -1,4 +1,4 @@
-import { SemanticModel } from './ir';
+import { Field, ProfileSpec, SemanticModel } from './ir';
 export interface TranspileRequest {
     id: string;
     dialect: string;
@@ -13,11 +13,13 @@ export type SqlTranspiler = (requests: TranspileRequest[], target: string) => Pr
 export interface TranspileOptions {
     target?: string;
     transpiler?: SqlTranspiler;
+    translatesProfile?: (profile: ProfileSpec, model: SemanticModel) => boolean;
 }
 export interface TranspileResult {
     model: SemanticModel;
     warnings: string[];
 }
+type TranspileNode = Pick<Field, 'expression' | 'dialects' | 'stringForm' | 'importedExpression' | 'importedDialect'>;
 /**
  * Returns true if `expression`, read as generic SQL (`ANSI_SQL`), quotes a
  * column, table qualifier, or struct field with double quotes (`"..."`).
@@ -43,9 +45,10 @@ export declare function hasDoubleQuotedColumn(expression: string): boolean;
  */
 export declare function doubleQuotedNames(expression: string): string[];
 /**
- * Adds a translated `opts.target` dialect entry to every field or metric in
- * `model` whose only texts are in vendor dialects or whose `ANSI_SQL` text
- * double-quotes a column name, returning a cloned model plus diagnostics.
+ * Adds a translated `opts.target` dialect entry to every field, metric and
+ * profile field binding in `model` whose only texts are in vendor dialects or
+ * whose `ANSI_SQL` text double-quotes a column name, returning a cloned model
+ * plus diagnostics.
  *
  * The input model is never mutated. When no node needs transpilation, the clone
  * is returned unchanged and the transpiler is not invoked. The
@@ -56,13 +59,11 @@ export declare function doubleQuotedNames(expression: string): string[];
  */
 export declare function transpileModel(model: SemanticModel, opts?: TranspileOptions): Promise<TranspileResult>;
 /**
- * Returns true if the default translator reads SQL written in `dialect`
- * (treating an empty string as `ANSI_SQL`).
- *
- * `transpileModel` leaves a text in any other dialect as written, with a
- * warning.
+ * Whether `transpileModel`, with the default translator, gives `node` a
+ * BIGQUERY text in a model whose entities are `entityNames`. False until
+ * `loadSqlEngine()` has loaded the engine.
  */
-export declare function transpilesFrom(dialect: string): boolean;
+export declare function translatesToBigQuery(node: TranspileNode, entityNames: string[]): boolean;
 /**
  * The default {@link SqlTranspiler}: transpiles via the embedded
  * @polyglot-sql/sdk Rust/WASM engine. There is no external runtime dependency
@@ -72,3 +73,4 @@ export declare function transpilesFrom(dialect: string): boolean;
  * that request, so the caller degrades to the imported form + warning.
  */
 export declare const polyglotTranspiler: SqlTranspiler;
+export {};