@malloydata/malloy 0.0.1

package/dist/dialect/duckdb.js

@@ -0,0 +1,349 @@
+"use strict";
+/*
+ * Copyright 2021 Google LLC
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DuckDBDialect = void 0;
+const model_1 = require("../model");
+const utils_1 = require("../model/utils");
+const dialect_1 = require("./dialect");
+// need to refactor runSQL to take a SQLBlock instead of just a sql string.
+const hackSplitComment = "-- hack: split on this";
+const keywords = `
+ALL
+ANALYSE
+ANALYZE
+AND
+ANY
+ARRAY
+AS
+ASC_P
+ASYMMETRIC
+BOTH
+CASE
+CAST
+CHECK_P
+COLLATE
+COLUMN
+CONSTRAINT
+CREATE_P
+CURRENT_CATALOG
+CURRENT_DATE
+CURRENT_ROLE
+CURRENT_TIME
+CURRENT_TIMESTAMP
+CURRENT_USER
+DEFAULT
+DEFERRABLE
+DESC_P
+DISTINCT
+DO
+ELSE
+END_P
+EXCEPT
+FALSE_P
+FETCH
+FOR
+FOREIGN
+FROM
+GRANT
+GROUP_P
+HAVING
+IN_P
+INITIALLY
+INTERSECT
+INTO
+LATERAL_P
+LEADING
+LIMIT
+LOCALTIME
+LOCALTIMESTAMP
+NOT
+NULL_P
+OFFSET
+ON
+ONLY
+OR
+ORDER
+PLACING
+PRIMARY
+REFERENCES
+RETURNING
+SELECT
+SESSION_USER
+SOME
+SYMMETRIC
+TABLE
+THEN
+TO
+TRAILING
+TRUE_P
+UNION
+UNIQUE
+USER
+USING
+VARIADIC
+WHEN
+WHERE
+WINDOW
+WITH
+`.split(/\s/);
+const castMap = {
+    number: "double precision",
+    string: "varchar",
+};
+const pgExtractionMap = {
+    day_of_week: "dow",
+    day_of_year: "doy",
+};
+const inSeconds = {
+    second: 1,
+    minute: 60,
+    hour: 3600,
+    day: 86400,
+    week: 604800,
+};
+class DuckDBDialect extends dialect_1.Dialect {
+    constructor() {
+        super(...arguments);
+        this.name = "duckdb";
+        this.defaultNumberType = "DOUBLE";
+        this.hasFinalStage = false;
+        this.stringTypeName = "VARCHAR";
+        this.divisionIsInteger = true;
+        this.supportsSumDistinctFunction = true;
+        this.unnestWithNumbers = true;
+        this.defaultSampling = { rows: 50000 };
+        this.supportUnnestArrayAgg = true;
+        this.supportsCTEinCoorelatedSubQueries = true;
+        this.functionInfo = {};
+    }
+    // hack until they support temporary macros.
+    get udfPrefix() {
+        return `__udf${Math.floor(Math.random() * 100000)}`;
+    }
+    quoteTablePath(tableName) {
+        return tableName.match(/\//) ? `'${tableName}'` : tableName;
+    }
+    sqlGroupSetTable(groupSetCount) {
+        return `CROSS JOIN (SELECT UNNEST(GENERATE_SERIES(0,${groupSetCount},1)) as group_set  ) as group_set`;
+    }
+    sqlAnyValue(groupSet, fieldName) {
+        return `FIRST(${fieldName}) FILTER (WHERE ${fieldName} IS NOT NULL)`;
+    }
+    mapFields(fieldList) {
+        return fieldList.join(", ");
+    }
+    sqlAggregateTurtle(groupSet, fieldList, orderBy, limit) {
+        let tail = "";
+        if (limit !== undefined) {
+            tail += `[1:${limit}]`;
+        }
+        const fields = fieldList
+            .map((f) => `\n  ${f.sqlOutputName}: ${f.sqlExpression}`)
+            .join(", ");
+        return `COALESCE(LIST({${fields}} ${orderBy}) FILTER (WHERE group_set=${groupSet})${tail},[])`;
+    }
+    sqlAnyValueTurtle(groupSet, fieldList) {
+        const fields = fieldList
+            .map((f) => `${f.sqlExpression} as ${f.sqlOutputName}`)
+            .join(", ");
+        return `ANY_VALUE(CASE WHEN group_set=${groupSet} THEN ROW(${fields}))`;
+    }
+    sqlAnyValueLastTurtle(name, groupSet, sqlName) {
+        return `MAX(CASE WHEN group_set=${groupSet} THEN ${name}__${groupSet} END) as ${sqlName}`;
+    }
+    sqlCoaleseMeasuresInline(groupSet, fieldList) {
+        const fields = fieldList
+            .map((f) => `${f.sqlOutputName}: ${f.sqlExpression} `)
+            .join(", ");
+        const nullValues = fieldList
+            .map((f) => `${f.sqlOutputName}: NULL`)
+            .join(", ");
+        return `COALESCE(FIRST({${fields}}) FILTER(WHERE group_set=${groupSet}), {${nullValues}})`;
+    }
+    sqlUnnestAlias(source, alias, _fieldList, _needDistinctKey) {
+        return `LEFT JOIN (select UNNEST(generate_series(1,
+        100000, --
+        -- (SELECT genres_length FROM movies limit 1),
+        1)) as __row_id) as ${alias} ON  ${alias}.__row_id <= array_length(${source})`;
+    }
+    sqlSumDistinctHashedKey(_sqlDistinctKey) {
+        return "uses sumDistinctFunction, should not be called";
+    }
+    sqlGenerateUUID() {
+        return `GEN_RANDOM_UUID()`;
+    }
+    sqlDateToString(sqlDateExp) {
+        return `(${sqlDateExp})::date::varchar`;
+    }
+    sqlFieldReference(alias, fieldName, _fieldType, _isNested, isArray) {
+        if (isArray) {
+            return alias;
+        }
+        else {
+            return `${alias}.${this.sqlMaybeQuoteIdentifier(fieldName)}`;
+        }
+    }
+    sqlUnnestPipelineHead(isSingleton, sourceSQLExpression) {
+        let p = sourceSQLExpression;
+        if (isSingleton) {
+            p = `[${p}]`;
+        }
+        return `(SELECT UNNEST(${p}) as base)`;
+    }
+    sqlCreateFunction(id, funcText) {
+        return `DROP MACRO IF EXISTS ${id}; \n${hackSplitComment}\n CREATE MACRO ${id}(_param) AS (\n${(0, utils_1.indent)(funcText)}\n);\n${hackSplitComment}\n`;
+    }
+    sqlCreateFunctionCombineLastStage(lastStageName, structDef) {
+        return `SELECT LIST(ROW(${structDef.fields
+            .map((fieldDef) => this.sqlMaybeQuoteIdentifier((0, model_1.getIdentifier)(fieldDef)))
+            .join(",")})) FROM ${lastStageName}\n`;
+    }
+    sqlSelectAliasAsStruct(alias, physicalFieldNames) {
+        return `ROW(${physicalFieldNames
+            .map((name) => `${alias}.${name}`)
+            .join(", ")})`;
+    }
+    // TODO
+    // sqlMaybeQuoteIdentifier(identifier: string): string {
+    //   return keywords.indexOf(identifier.toUpperCase()) > 0 ||
+    //     identifier.match(/[a-zA-Z][a-zA-Z0-9]*/) === null || true
+    //     ? '"' + identifier + '"'
+    //     : identifier;
+    // }
+    sqlMaybeQuoteIdentifier(identifier) {
+        return '"' + identifier + '"';
+    }
+    // The simple way to do this is to add a comment on the table
+    //  with the expiration time. https://www.postgresql.org/docs/current/sql-comment.html
+    //  and have a reaper that read comments.
+    sqlCreateTableAsSelect(_tableName, _sql) {
+        throw new Error("Not implemented Yet");
+    }
+    getFunctionInfo(_functionName) {
+        return undefined;
+    }
+    sqlMeasureTime(from, to, units) {
+        let lVal = from.value;
+        let rVal = to.value;
+        if (inSeconds[units]) {
+            lVal = (0, model_1.mkExpr) `EXTRACT(EPOCH FROM ${lVal})`;
+            rVal = (0, model_1.mkExpr) `EXTRACT(EPOCH FROM ${rVal})`;
+            const duration = (0, model_1.mkExpr) `(${rVal} - ${lVal})`;
+            return units == "second"
+                ? duration
+                : (0, model_1.mkExpr) `TRUNC(${duration}/${inSeconds[units].toString()})`;
+        }
+        const yearDiff = (0, model_1.mkExpr) `TRUNC(EXTRACT(YEAR FROM ${rVal}) - EXTRACT(YEAR FROM ${lVal}))`;
+        if (units == "year") {
+            return yearDiff;
+        }
+        if (units == "month") {
+            const monthDiff = (0, model_1.mkExpr) `TRUNC(EXTRACT(MONTH FROM ${rVal}) - EXTRACT(MONTH FROM ${lVal}))`;
+            return (0, model_1.mkExpr) `${yearDiff} * 12 + ${monthDiff}`;
+        }
+        if (units == "quarter") {
+            const qDiff = (0, model_1.mkExpr) `TRUNC(EXTRACT(QUARTER FROM ${rVal}) - EXTRACT(QUARTER FROM ${lVal}))`;
+            return (0, model_1.mkExpr) `${yearDiff} * 4 + ${qDiff}`;
+        }
+        throw new Error(`Unknown or unhandled postgres time unit: ${units}`);
+    }
+    sqlNow() {
+        return (0, model_1.mkExpr) `CURRENT_TIMESTAMP`;
+    }
+    sqlTrunc(sqlTime, units) {
+        // adjusting for monday/sunday weeks
+        const week = units == "week";
+        const truncThis = week
+            ? (0, model_1.mkExpr) `${sqlTime.value}+interval'1'day`
+            : sqlTime.value;
+        const trunced = (0, model_1.mkExpr) `DATE_TRUNC('${units}', ${truncThis})`;
+        return week ? (0, model_1.mkExpr) `(${trunced}-interval'1'day)` : trunced;
+    }
+    sqlExtract(from, units) {
+        const pgUnits = pgExtractionMap[units] || units;
+        const extracted = (0, model_1.mkExpr) `EXTRACT(${pgUnits} FROM ${from.value})`;
+        return units == "day_of_week" ? (0, model_1.mkExpr) `(${extracted}+1)` : extracted;
+    }
+    sqlAlterTime(op, expr, n, timeframe) {
+        if (timeframe == "quarter") {
+            timeframe = "month";
+            n = (0, model_1.mkExpr) `${n}*3`;
+        }
+        if (timeframe == "week") {
+            timeframe = "day";
+            n = (0, model_1.mkExpr) `${n}*7`;
+        }
+        const interval = (0, model_1.mkExpr) `INTERVAL ${n} ${timeframe}`;
+        return (0, model_1.mkExpr) `((${expr.value})${op}${interval})`;
+    }
+    sqlCast(cast) {
+        if (cast.dstType !== cast.srcType) {
+            const castTo = castMap[cast.dstType] || cast.dstType;
+            return (0, model_1.mkExpr) `cast(${cast.expr} as ${castTo})`;
+        }
+        return cast.expr;
+    }
+    sqlRegexpMatch(expr, regexp) {
+        return (0, model_1.mkExpr) `REGEXP_MATCHES(${expr}, ${regexp})`;
+    }
+    sqlLiteralTime(timeString, type, _timezone) {
+        if (type == "date") {
+            return `DATE('${timeString}')`;
+        }
+        else if (type == "timestamp") {
+            return `TIMESTAMP '${timeString}'`;
+        }
+        else {
+            throw new Error(`Unknown Literal time format ${type}`);
+        }
+    }
+    // sqlSumDistinct(key: string, value: string): string {
+    //   // return `sum_distinct(list({key:${key}, val: ${value}}))`;
+    //   return `(
+    //     fail -- force the query for fail until the bug is fixed.
+    //     SELECT sum(a.val) as value
+    //     FROM (
+    //       SELECT UNNEST(list(distinct {key:${key}, val: ${value}})) a
+    //     )
+    //   )`;
+    // }
+    sqlSumDistinct(key, value) {
+        const _factor = 32;
+        const precision = 0.000001;
+        const keySQL = `md5_number_lower(${key}::varchar)::int128`;
+        return `
+    (SUM(DISTINCT ${keySQL} + FLOOR(IFNULL(${value},0)/${precision})::int128) -  SUM(DISTINCT ${keySQL}))*${precision}
+    `;
+    }
+    // default duckdb to sampling 50K rows.
+    sqlSampleTable(tableSQL, sample) {
+        if (sample !== undefined) {
+            if ((0, model_1.isSamplingEnable)(sample) && sample.enable) {
+                sample = this.defaultSampling;
+            }
+            if ((0, model_1.isSamplingRows)(sample)) {
+                return `(SELECT * FROM ${tableSQL} USING SAMPLE ${sample.rows})`;
+            }
+            else if ((0, model_1.isSamplingPercent)(sample)) {
+                return `(SELECT * FROM ${tableSQL} USING SAMPLE ${sample.percent} PERCENT (bernoulli))`;
+            }
+        }
+        return tableSQL;
+    }
+    sqlOrderBy(orderTerms) {
+        return `ORDER BY ${orderTerms.map((t) => `${t} NULLS LAST`).join(",")}`;
+    }
+}
+exports.DuckDBDialect = DuckDBDialect;
+//# sourceMappingURL=duckdb.js.map

package/dist/dialect/index.d.ts

@@ -0,0 +1,6 @@
+export { Dialect } from "./dialect";
+export type { DialectFieldList } from "./dialect";
+export { StandardSQLDialect } from "./standardsql";
+export { PostgresDialect } from "./postgres";
+export { DuckDBDialect } from "./duckdb";
+export { getDialect, registerDialect } from "./dialect_map";

package/dist/dialect/index.js

@@ -0,0 +1,27 @@
+"use strict";
+/*
+ * Copyright 2021 Google LLC
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.registerDialect = exports.getDialect = exports.DuckDBDialect = exports.PostgresDialect = exports.StandardSQLDialect = exports.Dialect = void 0;
+var dialect_1 = require("./dialect");
+Object.defineProperty(exports, "Dialect", { enumerable: true, get: function () { return dialect_1.Dialect; } });
+var standardsql_1 = require("./standardsql");
+Object.defineProperty(exports, "StandardSQLDialect", { enumerable: true, get: function () { return standardsql_1.StandardSQLDialect; } });
+var postgres_1 = require("./postgres");
+Object.defineProperty(exports, "PostgresDialect", { enumerable: true, get: function () { return postgres_1.PostgresDialect; } });
+var duckdb_1 = require("./duckdb");
+Object.defineProperty(exports, "DuckDBDialect", { enumerable: true, get: function () { return duckdb_1.DuckDBDialect; } });
+var dialect_map_1 = require("./dialect_map");
+Object.defineProperty(exports, "getDialect", { enumerable: true, get: function () { return dialect_map_1.getDialect; } });
+Object.defineProperty(exports, "registerDialect", { enumerable: true, get: function () { return dialect_map_1.registerDialect; } });
+//# sourceMappingURL=index.js.map

package/dist/dialect/postgres.d.ts

@@ -0,0 +1,49 @@
+import { DateUnit, ExtractUnit, TimeFieldType, TimestampUnit, Expr, TimeValue, TypecastFragment, Sampling } from "../model";
+import { Dialect, DialectFieldList, FunctionInfo } from "./dialect";
+export declare class PostgresDialect extends Dialect {
+    name: string;
+    defaultNumberType: string;
+    udfPrefix: string;
+    hasFinalStage: boolean;
+    stringTypeName: string;
+    divisionIsInteger: boolean;
+    supportsSumDistinctFunction: boolean;
+    unnestWithNumbers: boolean;
+    defaultSampling: {
+        rows: number;
+    };
+    supportUnnestArrayAgg: boolean;
+    supportsCTEinCoorelatedSubQueries: boolean;
+    functionInfo: Record<string, FunctionInfo>;
+    quoteTablePath(tablePath: string): string;
+    sqlGroupSetTable(groupSetCount: number): string;
+    sqlAnyValue(groupSet: number, fieldName: string): string;
+    mapFields(fieldList: DialectFieldList): string;
+    sqlAggregateTurtle(groupSet: number, fieldList: DialectFieldList, orderBy: string | undefined, limit: number | undefined): string;
+    sqlAnyValueTurtle(groupSet: number, fieldList: DialectFieldList): string;
+    sqlAnyValueLastTurtle(name: string, groupSet: number, sqlName: string): string;
+    sqlCoaleseMeasuresInline(groupSet: number, fieldList: DialectFieldList): string;
+    sqlUnnestAlias(source: string, alias: string, fieldList: DialectFieldList, needDistinctKey: boolean, isArray: boolean): string;
+    sqlSumDistinctHashedKey(sqlDistinctKey: string): string;
+    sqlGenerateUUID(): string;
+    sqlFieldReference(alias: string, fieldName: string, fieldType: string, isNested: boolean, _isArray: boolean): string;
+    sqlUnnestPipelineHead(isSingleton: boolean, sourceSQLExpression: string): string;
+    sqlCreateFunction(id: string, funcText: string): string;
+    sqlCreateFunctionCombineLastStage(lastStageName: string): string;
+    sqlFinalStage(lastStageName: string, _fields: string[]): string;
+    sqlSelectAliasAsStruct(alias: string): string;
+    sqlMaybeQuoteIdentifier(identifier: string): string;
+    sqlCreateTableAsSelect(_tableName: string, _sql: string): string;
+    sqlNow(): Expr;
+    sqlTrunc(sqlTime: TimeValue, units: TimestampUnit): Expr;
+    sqlExtract(from: TimeValue, units: ExtractUnit): Expr;
+    sqlAlterTime(op: "+" | "-", expr: TimeValue, n: Expr, timeframe: DateUnit): Expr;
+    sqlCast(cast: TypecastFragment): Expr;
+    sqlRegexpMatch(expr: Expr, regexp: string): Expr;
+    sqlLiteralTime(timeString: string, type: TimeFieldType, _timezone: string): string;
+    getFunctionInfo(_functionName: string): FunctionInfo | undefined;
+    sqlMeasureTime(from: TimeValue, to: TimeValue, units: string): Expr;
+    sqlSumDistinct(key: string, value: string): string;
+    sqlSampleTable(tableSQL: string, sample: Sampling | undefined): string;
+    sqlOrderBy(orderTerms: string[]): string;
+}