@google-cloud/kcmd 0.0.0-stage → 0.1.0

package/build/ts/kcmd/semantic/sql_parser.js

@@ -0,0 +1,378 @@
+"use strict";
+// The SQL engine kcmd reads and translates expressions with: the
+// @polyglot-sql/sdk Rust/WASM engine, bundled into the tool.
+//
+// Loading the engine is asynchronous and happens once per process, through
+// `loadSqlEngine`. Parsing is synchronous after that, so the validators that
+// read expressions stay synchronous: a command that runs them loads the engine
+// first, and so does a test.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.loadSqlEngine = loadSqlEngine;
+exports.sqlColumns = sqlColumns;
+exports.sqlExpressionTree = sqlExpressionTree;
+exports.holdsNoSql = holdsNoSql;
+exports.withoutComments = withoutComments;
+const node_fs_1 = require("node:fs");
+// The engine, initialized once and shared process-wide. Under Bun, the
+// Rust/WASM blob is embedded into the standalone binary via `import(...,
+// {with: {type: 'file'}})` (statically analyzable, so bundled); under Node.js,
+// it is resolved from `node_modules` via `require.resolve(...)`. In both
+// cases the bytes are handed to `init()` via the `/manual` entry so the same
+// loader works inside a `bun --compile` binary and a standard npm install.
+let enginePromise;
+let loadedEngine;
+function loadSqlEngine() {
+    if (!enginePromise) {
+        enginePromise = (async () => {
+            const wasmPath = 'Bun' in globalThis ?
+                (await import('@polyglot-sql/sdk/polyglot_sql.wasm', { with: { type: 'file' } }))
+                    .default :
+                require.resolve('@polyglot-sql/sdk/polyglot_sql.wasm');
+            const engine = await import('@polyglot-sql/sdk/manual');
+            await engine.init({ wasmUrl: (0, node_fs_1.readFileSync)(wasmPath) });
+            loadedEngine = engine;
+            return engine;
+        })().catch(err => {
+            // Don't cache the failure: a transient init error (e.g. a failed read)
+            // must not permanently disable the engine for a long-lived process.
+            // Clear the memo so the next call retries, then propagate.
+            enginePromise = undefined;
+            throw err;
+        });
+    }
+    return enginePromise;
+}
+// The engine's dialect for each kcmd dialect. The engine has no Spanner
+// dialect, so Spanner's GoogleSQL is read as BigQuery's.
+const PARSER_DIALECTS = {
+    ANSI_SQL: 'generic',
+    BIGQUERY: 'bigquery',
+    SPANNER: 'bigquery',
+    POSTGRES: 'postgresql',
+    ALLOYDB: 'postgresql',
+    MYSQL: 'mysql',
+    SQLSERVER: 'tsql',
+    SNOWFLAKE: 'snowflake',
+    DATABRICKS: 'databricks',
+};
+// Every column `expression` reads, in the order written, or undefined when the
+// engine cannot read it as one expression in `dialect`. An empty expression
+// reads none. A struct path reads as
+// its first two parts, so `orders.address.city` reads the column `address`
+// qualified by `orders`. A function namespace such as `NET` in
+// `NET.HOST(orders.url)` is not a column, and the expression a chained call
+// applies to, such as `(orders.name)` in `(orders.name).UPPER()`, is read.
+// Throws when the engine is not loaded.
+function sqlColumns(expression, dialect) {
+    const tree = sqlExpressionTree(expression, dialect);
+    if (!expression.trim())
+        return [];
+    if (!tree)
+        return undefined;
+    const columns = [];
+    collectColumns(tree, columns, new Set(), PARSER_DIALECTS[dialect.toUpperCase()] ?? 'generic');
+    return columns;
+}
+// The engine's tree of `expression` read as one expression in `dialect`, or
+// undefined when it cannot be read so or is empty. A text that starts with a
+// keyword, such as `Order.amount`, is read again inside parentheses on lines
+// of their own, so a trailing `--` comment stays inside.
+function sqlExpressionTree(expression, dialect) {
+    if (!loadedEngine) {
+        throw new Error('the SQL parser is not loaded; call loadSqlEngine() first');
+    }
+    if (!expression.trim())
+        return undefined;
+    const engineDialect = PARSER_DIALECTS[dialect.toUpperCase()] ?? 'generic';
+    const ast = oneExpression(engineDialect, `SELECT ${expression}`) ??
+        oneExpression(engineDialect, `SELECT (\n${expression}\n)`, 'paren');
+    return ast?.select.expressions[0];
+}
+// The parsed statement when `sql` is a `SELECT` of exactly one expression and
+// nothing else: no alias, no `FROM`, `WHERE`, `DISTINCT` or other clause. The
+// expression is of node kind `kind` when given. Undefined otherwise.
+function oneExpression(dialect, sql, kind) {
+    const res = loadedEngine.parse(sql, dialect);
+    if (!res.success || res.ast?.length !== 1)
+        return undefined;
+    const select = res.ast[0]?.select;
+    const items = select?.expressions;
+    if (!Array.isArray(items) || items.length !== 1)
+        return undefined;
+    const itemKind = Object.keys(items[0] ?? {})[0];
+    if (itemKind === 'alias' || (kind && itemKind !== kind))
+        return undefined;
+    // Inside the parentheses, `x AS y` reads as an alias too.
+    const inner = itemKind === 'paren' ? items[0].paren?.this : undefined;
+    if (inner && Object.keys(inner)[0] === 'alias')
+        return undefined;
+    const isEmpty = (v) => v === null || v === undefined ||
+        v === false || (Array.isArray(v) && v.length === 0) ||
+        (typeof v === 'object' && Object.keys(v).length === 0);
+    for (const [key, value] of Object.entries(select)) {
+        if (key !== 'expressions' && !isEmpty(value))
+            return undefined;
+    }
+    return res.ast[0];
+}
+// Functions whose first argument is a date-part unit (such as `DAY` or `MONTH`)
+// that the parser represents as an unquoted bare `column` node in MySQL,
+// Databricks, T-SQL or generic ANSI SQL.
+const LEADING_DATE_PART_FUNCTIONS = new Set([
+    'DATEDIFF',
+    'DATEADD',
+    'DATE_ADD',
+    'DATE_BUCKET',
+    'DATE_DIFF',
+    'TIMESTAMPDIFF',
+    'TIMESTAMPADD',
+]);
+// In BigQuery (and Spanner), these take a column first, as in
+// `DATE_DIFF(end_col, start_col, DAY)` or `DATE_BUCKET(d, INTERVAL 7 DAY)`, so
+// their first argument is a column even when its name matches a short unit
+// such as `d`.
+const BIGQUERY_TRAILING_UNIT_FUNCTIONS = new Set(['DATE_ADD', 'DATE_BUCKET', 'DATE_DIFF']);
+// BigQuery functions whose last argument is a date part, as in
+// `DATE_TRUNC(d, MONTH)` or `DATE_DIFF(a, b, WEEK(MONDAY))`. Under the `SAFE.`
+// prefix the engine reads that date part as a column.
+const BIGQUERY_DATE_PART_FUNCTIONS = new Set([
+    'DATE_DIFF', 'DATETIME_DIFF', 'TIMESTAMP_DIFF', 'TIME_DIFF', 'DATE_TRUNC',
+    'DATETIME_TRUNC', 'TIMESTAMP_TRUNC', 'TIME_TRUNC', 'LAST_DAY',
+]);
+const DATE_PART_KEYWORDS = new Set([
+    'YEAR', 'YEARS', 'YYYY', 'YY', 'QUARTER',
+    'QUARTERS', 'QQ', 'Q', 'MONTH', 'MONTHS',
+    'MM', 'M', 'WEEK', 'WEEKS', 'WK',
+    'WW', 'DAYOFYEAR', 'DY', 'DOY', 'DAY',
+    'DAYS', 'DD', 'D', 'DAYOFWEEK', 'DW',
+    'WEEKDAY', 'HOUR', 'HOURS', 'HH', 'MINUTE',
+    'MINUTES', 'MI', 'N', 'SECOND', 'SECONDS',
+    'SS', 'S', 'MILLISECOND', 'MILLISECONDS', 'MS',
+    'MICROSECOND', 'MICROSECONDS', 'US', 'NANOSECOND', 'NANOSECONDS',
+    'NS', 'ISOWEEK', 'ISOYEAR',
+]);
+function collectColumns(node, out, bound, dialect) {
+    const bigQuery = dialect === 'bigquery';
+    if (Array.isArray(node)) {
+        for (const item of node) {
+            collectColumns(item, out, bound, dialect);
+        }
+        return;
+    }
+    if (!node || typeof node !== 'object')
+        return;
+    const obj = node;
+    const keys = Object.keys(obj);
+    if (keys.length === 1 && keys[0] === 'column' &&
+        typeof obj.column?.name?.name === 'string') {
+        const qualifier = obj.column.table?.name;
+        const name = obj.column.name.name;
+        if (typeof qualifier === 'string' ? bound.has(qualifier) : bound.has(name)) {
+            return;
+        }
+        out.push({
+            ...(typeof qualifier === 'string' ? { qualifier } : {}),
+            name,
+        });
+        return;
+    }
+    if (keys.length === 1 && keys[0] === 'dot') {
+        // A struct path reads as its first two parts. The engine reads `Time.hour`
+        // as a field of a column named `Time`, because `Time` is also a type name,
+        // so the first part is still the qualifier.
+        let inner = obj.dot;
+        while (inner?.this && Object.keys(inner.this).length === 1 &&
+            inner.this.dot) {
+            inner = inner.this.dot;
+        }
+        const column = inner?.this?.column;
+        if (typeof column?.name?.name === 'string') {
+            const qualifier = column.table?.name;
+            if (typeof qualifier === 'string') {
+                if (!bound.has(qualifier)) {
+                    out.push({ qualifier, name: column.name.name });
+                }
+            }
+            else if (typeof inner.field?.name === 'string') {
+                if (!bound.has(column.name.name)) {
+                    out.push({ qualifier: column.name.name, name: inner.field.name });
+                }
+            }
+            return;
+        }
+        collectColumns(obj.dot?.this, out, bound, dialect);
+        return;
+    }
+    if (keys.length === 1 && keys[0] === 'method_call') {
+        // In `NET.HOST(x)` and `my_project.my_dataset.my_udf(x)` the part before
+        // the dot is a bare name path that names a function namespace, so it is not
+        // read. In T-SQL, `orders.geo.STLength()` calls a method of the column
+        // `orders.geo`, so a receiver written as a qualified column is read. Any
+        // other receiver is the expression a chained call applies to, as in
+        // `(orders.name).UPPER()` or `LOWER(orders.name).TRIM()`, and it is.
+        const receiver = obj.method_call?.this;
+        const columnMethod = dialect === 'tsql' &&
+            typeof receiver?.column?.table?.name === 'string';
+        if (columnMethod || !isNamePath(receiver)) {
+            collectColumns(receiver, out, bound, dialect);
+        }
+        collectColumns(obj.method_call?.args, out, bound, dialect);
+        return;
+    }
+    if (keys.length === 1 && keys[0] === 'function') {
+        const fnName = String(obj.function?.name ?? '').toUpperCase();
+        const hasLeadingUnit = LEADING_DATE_PART_FUNCTIONS.has(fnName) &&
+            !(bigQuery && BIGQUERY_TRAILING_UNIT_FUNCTIONS.has(fnName));
+        const args = Array.isArray(obj.function?.args) ? obj.function.args : [];
+        const col0 = args[0]?.column;
+        if (hasLeadingUnit && args.length >= 3 && col0 &&
+            col0.table == null && col0.name?.quoted === false &&
+            typeof col0.name?.name === 'string' &&
+            DATE_PART_KEYWORDS.has(col0.name.name.toUpperCase())) {
+            for (const arg of args.slice(1)) {
+                collectColumns(arg, out, bound, dialect);
+            }
+            return;
+        }
+        if (bigQuery && fnName.startsWith('SAFE.') &&
+            BIGQUERY_DATE_PART_FUNCTIONS.has(fnName.slice('SAFE.'.length)) &&
+            args.length >= 2 && isDatePart(args[args.length - 1])) {
+            collectColumns(args.slice(0, -1), out, bound, dialect);
+            return;
+        }
+    }
+    if (keys.length === 1 && keys[0] === 'lambda') {
+        const next = new Set(bound);
+        for (const p of obj.lambda?.parameters ?? []) {
+            if (typeof p?.name === 'string')
+                next.add(p.name);
+        }
+        collectColumns(obj.lambda?.body, out, next, dialect);
+        return;
+    }
+    if (keys.length === 1 && keys[0] === 'select') {
+        const select = obj.select;
+        const next = new Set(bound);
+        for (const cte of select?.with?.ctes ?? []) {
+            if (typeof cte?.alias?.name === 'string')
+                next.add(cte.alias.name);
+            collectColumns(cte?.this, out, next, dialect);
+        }
+        for (const expr of select?.from?.expressions ?? []) {
+            collectColumns(expr, out, next, dialect);
+            addSourceAliases(expr, next);
+        }
+        // Inside a subquery with its own `FROM` clause, a bare identifier in the
+        // remaining clauses may come from that `FROM` source (e.g. a derived
+        // table's column or an unaliased `UNNEST` field), so only qualified
+        // references outside `next` are kept.
+        const target = select?.from ? [] : out;
+        for (const join of select?.joins ?? []) {
+            addSourceAliases(join?.this, next);
+            collectColumns(join, target, next, dialect);
+        }
+        for (const item of select?.expressions ?? []) {
+            if (typeof item?.alias?.alias?.name === 'string') {
+                next.add(item.alias.alias.name);
+            }
+        }
+        for (const [k, v] of Object.entries(select ?? {})) {
+            if (k !== 'with' && k !== 'from' && k !== 'joins') {
+                collectColumns(v, target, next, dialect);
+            }
+        }
+        if (select?.from) {
+            out.push(...target.filter(c => c.qualifier !== undefined));
+        }
+        return;
+    }
+    for (const value of Object.values(obj)) {
+        collectColumns(value, out, bound, dialect);
+    }
+}
+// Whether `node` is a date part written as a bare name, such as `DAY`, or as
+// `WEEK(MONDAY)`.
+function isDatePart(node) {
+    const col = node?.column;
+    if (col) {
+        return col.table == null && col.name?.quoted === false &&
+            typeof col.name?.name === 'string' &&
+            DATE_PART_KEYWORDS.has(col.name.name.toUpperCase());
+    }
+    return String(node?.function?.name ?? '').toUpperCase() === 'WEEK';
+}
+function addSourceAliases(source, bound) {
+    if (typeof source?.alias?.alias?.name === 'string') {
+        bound.add(source.alias.alias.name);
+    }
+    const inner = source?.alias?.this ?? source;
+    for (const kind of ['unnest', 'table', 'subquery']) {
+        const node = inner?.[kind];
+        if (typeof node?.alias?.name === 'string')
+            bound.add(node.alias.name);
+        if (typeof node?.offset_alias?.name === 'string') {
+            bound.add(node.offset_alias.name);
+        }
+    }
+}
+// Whether `node` is a name or a dotted run of names, such as `NET` or
+// `my_project.my_dataset`, with no other expression inside it.
+function isNamePath(node) {
+    if (!node || typeof node !== 'object')
+        return false;
+    const obj = node;
+    const keys = Object.keys(obj);
+    if (keys.length !== 1)
+        return false;
+    if (keys[0] === 'column')
+        return true;
+    return keys[0] === 'dot' && isNamePath(obj.dot?.this);
+}
+// The tokens of `expression` in `dialect`, or undefined when the engine cannot
+// tokenize it. Throws when the engine is not loaded.
+function sqlTokens(expression, dialect) {
+    if (!loadedEngine) {
+        throw new Error('the SQL parser is not loaded; call loadSqlEngine() first');
+    }
+    const engineDialect = PARSER_DIALECTS[dialect.toUpperCase()] ?? 'generic';
+    const res = loadedEngine.tokenize(expression, engineDialect);
+    return res.success ? res.tokens ?? [] : undefined;
+}
+// Whether `expression` holds no SQL in `dialect`: it is empty, only whitespace
+// or only comments. False when the engine cannot read it. Throws when the
+// engine is not loaded.
+function holdsNoSql(expression, dialect) {
+    return sqlTokens(expression, dialect)?.length === 0;
+}
+// `expression` with its comments removed, as the engine reads them in
+// `dialect`. Comments between two tokens become one space, or nothing when
+// either token is a `.`, so `orders/* c */.amount` reads `orders.amount`.
+// Comments before the first token or after the last are dropped. The rest of
+// the text stays as written. Returns `expression` itself when it has no
+// comment, holds nothing but comments, or the engine cannot read it. Throws
+// when the engine is not loaded.
+function withoutComments(expression, dialect) {
+    const tokens = sqlTokens(expression, dialect);
+    if (!tokens?.some(t => t.comments.length || t.trailing_comments.length)) {
+        return expression;
+    }
+    const chars = Array.from(expression);
+    let out = '';
+    let end = 0;
+    let previous = '';
+    for (const t of tokens) {
+        // The text between two tokens is whitespace and comments.
+        const between = chars.slice(end, t.span.start).join('');
+        const text = chars.slice(t.span.start, t.span.end).join('');
+        if (out) {
+            if (between.trim() === '')
+                out += between;
+            else if (previous !== '.' && text !== '.')
+                out += ' ';
+        }
+        out += text;
+        previous = text;
+        end = t.span.end;
+    }
+    return out;
+}

package/build/ts/kcmd/semantic/transpile.d.ts

@@ -0,0 +1,74 @@
+import { SemanticModel } from './ir';
+export interface TranspileRequest {
+    id: string;
+    dialect: string;
+    expression: string;
+}
+export interface TranspileResponse {
+    id: string;
+    sql?: string;
+    error?: string;
+}
+export type SqlTranspiler = (requests: TranspileRequest[], target: string) => Promise<TranspileResponse[]>;
+export interface TranspileOptions {
+    target?: string;
+    transpiler?: SqlTranspiler;
+}
+export interface TranspileResult {
+    model: SemanticModel;
+    warnings: string[];
+}
+/**
+ * Returns true if `expression`, read as generic SQL (`ANSI_SQL`), quotes a
+ * column, table qualifier, or struct field with double quotes (`"..."`).
+ *
+ * The `ANSI_SQL` parser marks both `"..."` and `` `...` `` column identifiers
+ * with `quoted: true`, whereas `BIGQUERY` marks only `` `...` `` as `quoted:
+ * true` and treats `"..."` as a string literal (or fails to parse `."..."`).
+ * Comparing how many `quotedNames` the two ASTs hold yields three cases:
+ *   - `ANSI_SQL > BIGQUERY`: at least one column identifier is double-quoted
+ *     (e.g., `"orders"."amount"` or `"amount"`).
+ *   - `ANSI_SQL < BIGQUERY`: `expression` uses BigQuery-only syntax (such as a
+ *     `# "comment"` line comment in `` `orders`.`col` # "comment" ``) that
+ *     `BIGQUERY` parses while `ANSI_SQL` fails to parse (`count = 0`).
+ *   - `ANSI_SQL === BIGQUERY`: any `"` is inside a string literal or comment
+ *     (e.g., `` `orders`.`note` = 'a "quoted" ` string' ``), and any quoted
+ *     column identifiers use backticks in both dialects.
+ */
+export declare function hasDoubleQuotedColumn(expression: string): boolean;
+/**
+ * Returns the names that `expression` double-quotes, each once, for a
+ * message: those the `ANSI_SQL` parser marks quoted that `expression` writes
+ * in double quotes, else the first name it marks quoted.
+ */
+export declare function doubleQuotedNames(expression: string): string[];
+/**
+ * Adds a translated `opts.target` dialect entry to every field or metric in
+ * `model` whose only texts are in vendor dialects or whose `ANSI_SQL` text
+ * double-quotes a column name, returning a cloned model plus diagnostics.
+ *
+ * The input model is never mutated. When no node needs transpilation, the clone
+ * is returned unchanged and the transpiler is not invoked. The
+ * qualifier-preservation guard parses the transpiled output with
+ * `columnReferences`, which requires the SQL parser to be loaded even when a
+ * custom `opts.transpiler` is injected (`loadSqlEngine()` is awaited before
+ * running the guard).
+ */
+export declare function transpileModel(model: SemanticModel, opts?: TranspileOptions): Promise<TranspileResult>;
+/**
+ * Returns true if the default translator reads SQL written in `dialect`
+ * (treating an empty string as `ANSI_SQL`).
+ *
+ * `transpileModel` leaves a text in any other dialect as written, with a
+ * warning.
+ */
+export declare function transpilesFrom(dialect: string): boolean;
+/**
+ * The default {@link SqlTranspiler}: transpiles via the embedded
+ * @polyglot-sql/sdk Rust/WASM engine. There is no external runtime dependency
+ * -- the WASM blob is bundled into the tool. It never throws: an engine that
+ * fails to initialize yields an `error` response for every request, and a
+ * per-request parse failure or unsupported dialect yields an `error` for just
+ * that request, so the caller degrades to the imported form + warning.
+ */
+export declare const polyglotTranspiler: SqlTranspiler;