@malloydata/malloy 0.0.1

package/README.md

@@ -0,0 +1,39 @@
+# Malloy
+
+Malloy is an experimental language for describing data relationships and transformations. It is both a semantic modeling language and a querying language that runs queries against a relational database. Malloy currently connects to BigQuery and Postgres, and natively supports DuckDB. We've built a Visual Studio Code extension to facilitate building Malloy data models, querying and transforming data, and creating simple visualizations and dashboards.
+
+## Building applications or products in javascript with @malloydata/malloy
+
+This package facilitates building the Malloy language - or the usage of data models or queries written in the Malloy language - into your product, data application, or website. The `@malloydata/malloy` library translates complex data operations written into the Malloy language into 1) SQL and 2) metadata. The SQL can be run against a database, and then combined with results and rendered or used in a varitey of ways.
+
+## Show me an example!
+
+```
+import { Runtime } from "@malloydata/malloy";
+import { DuckDBConnection } from "@malloydata/db-duckdb";
+
+const connection = new DuckDBConnection("duckdb-example");
+const runtime = new Runtime(files, connection);
+
+runtime.loadModel("source: flights is table('duckdb:data/flights.parquet')")
+
+const runner = runtime.loadQuery("query: flights->{group_by: flight_num}")
+runner.run() // <- executes query, returns JSON of results
+
+```
+
+Note: These APIs are still in beta and subject to change.
+
+## What's actually going on here?
+
+The `@malloydata/malloy` is basically a compiler that takes a few things as input - 1. Some text that defines a Malloy data model 2. Some text that defines a Malloy query and 3. Some schema information about the table(s) the query might run against. The compiler works by attempting to compile the data model, and stopping compilation when it needs to ask for things it requires to continue compilation. This "loop" is then re-initiated when you provide the compiler with the additional information, until a point when the compiler has all the information it needs to compile a Malloy query based on a data model.
+
+In practice, much of this loop is handled by various database plugins:
+
+- [DuckDB](https://github.com/looker-open-source/malloy/tree/main/packages/malloy-db-duckdb)
+- [BigQuery](https://github.com/looker-open-source/malloy/tree/main/packages/malloy-db-bigquery)
+- [Postgres](https://github.com/looker-open-source/malloy/tree/main/packages/malloy-db-postgres)
+
+## Do you have any examples?
+
+You can find a (very) simple example of writing a CLI for executing Malloy queries [here](https://github.com/looker-open-source/malloy/tree/main/demo/malloy-demo-bq-cli)

package/dist/connection_utils.d.ts

@@ -0,0 +1 @@
+export declare function toAsyncGenerator<T>(startStreaming: (onError: (error: Error) => void, onData: (data: T) => void, onEnd: () => void) => void): AsyncIterableIterator<T>;

package/dist/connection_utils.js

@@ -0,0 +1,56 @@
+"use strict";
+/*
+ * Copyright 2022 Google LLC
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.toAsyncGenerator = void 0;
+async function* toAsyncGenerator(startStreaming) {
+    let done = false;
+    function getResults(startStreaming) {
+        let resolve;
+        const promise = new Promise((res) => {
+            resolve = res;
+        });
+        startStreaming((error) => {
+            resolve({ done: true, isError: true, error });
+        }, (data) => {
+            resolve({
+                done: false,
+                value: data,
+                isError: false,
+                next: new Promise((res) => {
+                    resolve = res;
+                }),
+            });
+        }, () => {
+            resolve({ done: true, isError: false });
+        });
+        return promise;
+    }
+    let next = getResults(startStreaming);
+    while (!done) {
+        const result = await next;
+        if (result.done) {
+            done = true;
+            if (result.isError) {
+                throw result.error;
+            }
+            break;
+        }
+        else {
+            next = result.next;
+            yield result.value;
+        }
+    }
+}
+exports.toAsyncGenerator = toAsyncGenerator;
+//# sourceMappingURL=connection_utils.js.map

package/dist/dialect/dialect-map.d.ts

@@ -0,0 +1,3 @@
+import { Dialect } from "./dialect";
+export declare function getDialect(name: string): Dialect;
+export declare function registerDialect(d: Dialect): void;

package/dist/dialect/dialect-map.js

@@ -0,0 +1,33 @@
+"use strict";
+/*
+ * Copyright 2021 Google LLC
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.registerDialect = exports.getDialect = void 0;
+const postgres_1 = require("./postgres");
+const standardsql_1 = require("./standardsql");
+const dialectMap = new Map();
+function getDialect(name) {
+    const d = dialectMap.get(name);
+    if (d === undefined) {
+        throw new Error(`Unknown Dialect ${name}`);
+    }
+    return d;
+}
+exports.getDialect = getDialect;
+function registerDialect(d) {
+    dialectMap.set(d.name, d);
+}
+exports.registerDialect = registerDialect;
+registerDialect(new postgres_1.PostgresDialect());
+registerDialect(new standardsql_1.StandardSQLDialect());
+//# sourceMappingURL=dialect-map.js.map

package/dist/dialect/dialect.d.ts

@@ -0,0 +1,64 @@
+import { AtomicFieldTypeInner, TimeFieldType, TimestampUnit, ExtractUnit, DialectFragment, TimeValue } from "..";
+import { Expr, Sampling, StructDef, TypecastFragment } from "../model";
+interface DialectField {
+    type: string;
+    sqlExpression: string;
+    sqlOutputName: string;
+}
+/**
+ * Someday this might be used to control how a function call in malloy is
+ * translated into a function call in SQL. Today this is just so that
+ * the expression compiler can know the output type of a function.
+ */
+export interface FunctionInfo {
+    returnType: AtomicFieldTypeInner;
+}
+export declare type DialectFieldList = DialectField[];
+export declare abstract class Dialect {
+    abstract name: string;
+    abstract defaultNumberType: string;
+    abstract udfPrefix: string;
+    abstract hasFinalStage: boolean;
+    abstract stringTypeName: string;
+    abstract divisionIsInteger: boolean;
+    abstract supportsSumDistinctFunction: boolean;
+    abstract unnestWithNumbers: boolean;
+    protected abstract functionInfo: Record<string, FunctionInfo>;
+    abstract defaultSampling: Sampling;
+    abstract supportUnnestArrayAgg: boolean;
+    abstract supportsCTEinCoorelatedSubQueries: boolean;
+    abstract quoteTablePath(tablePath: string): string;
+    abstract sqlGroupSetTable(groupSetCount: number): string;
+    abstract sqlAnyValue(groupSet: number, fieldName: string): string;
+    abstract sqlAggregateTurtle(groupSet: number, fieldList: DialectFieldList, orderBy: string | undefined, limit: number | undefined): string;
+    abstract sqlAnyValueTurtle(groupSet: number, fieldList: DialectFieldList): string;
+    abstract sqlAnyValueLastTurtle(name: string, groupSet: number, sqlName: string): string;
+    abstract sqlCoaleseMeasuresInline(groupSet: number, fieldList: DialectFieldList): string;
+    abstract sqlUnnestAlias(source: string, alias: string, fieldList: DialectFieldList, needDistinctKey: boolean, isArray: boolean): string;
+    abstract sqlSumDistinctHashedKey(sqlDistinctKey: string): string;
+    abstract sqlGenerateUUID(): string;
+    abstract sqlFieldReference(alias: string, fieldName: string, fieldType: string, isNested: boolean, isArray: boolean): string;
+    abstract sqlUnnestPipelineHead(isSingleton: boolean, sourceSQLExpression: string): string;
+    abstract sqlCreateFunction(id: string, funcText: string): string;
+    abstract sqlCreateFunctionCombineLastStage(lastStageName: string, structDef: StructDef): string;
+    abstract sqlCreateTableAsSelect(tableName: string, sql: string): string;
+    abstract sqlSelectAliasAsStruct(alias: string, physicalFieldNames: string[]): string;
+    sqlFinalStage(_lastStageName: string, _fields: string[]): string;
+    sqlDateToString(sqlDateExp: string): string;
+    abstract sqlMaybeQuoteIdentifier(identifier: string): string;
+    abstract sqlNow(): Expr;
+    abstract sqlTrunc(sqlTime: TimeValue, units: TimestampUnit): Expr;
+    abstract sqlExtract(sqlTime: TimeValue, units: ExtractUnit): Expr;
+    abstract sqlMeasureTime(from: TimeValue, to: TimeValue, units: TimestampUnit): Expr;
+    abstract sqlAlterTime(op: "+" | "-", expr: TimeValue, n: Expr, timeframe: TimestampUnit): Expr;
+    ignoreInProject(_fieldName: string): boolean;
+    abstract sqlCast(cast: TypecastFragment): Expr;
+    abstract sqlLiteralTime(timeString: string, type: TimeFieldType, timezone: string): string;
+    abstract sqlRegexpMatch(expr: Expr, regex: string): Expr;
+    getFunctionInfo(functionName: string): FunctionInfo | undefined;
+    dialectExpr(df: DialectFragment): Expr;
+    sqlSumDistinct(_key: string, _value: string): string;
+    sqlSampleTable(tableSQL: string, sample: Sampling | undefined): string;
+    sqlOrderBy(orderTerms: string[]): string;
+}
+export {};

package/dist/dialect/dialect.js

@@ -0,0 +1,64 @@
+"use strict";
+/*
+ * Copyright 2021 Google LLC
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Dialect = void 0;
+class Dialect {
+    sqlFinalStage(_lastStageName, _fields) {
+        throw new Error("Dialect has no final Stage but called Anyway");
+    }
+    // default implementation will probably work most of the time
+    sqlDateToString(sqlDateExp) {
+        return `CAST(DATE(${sqlDateExp}) AS ${this.stringTypeName} )`;
+    }
+    // BigQuery has some fieldNames that are Pseudo Fields and shouldn't be
+    //  included in projections.
+    ignoreInProject(_fieldName) {
+        return false;
+    }
+    getFunctionInfo(functionName) {
+        return this.functionInfo[functionName.toLowerCase()];
+    }
+    dialectExpr(df) {
+        switch (df.function) {
+            case "now":
+                return this.sqlNow();
+            case "timeDiff":
+                return this.sqlMeasureTime(df.left, df.right, df.units);
+            case "delta":
+                return this.sqlAlterTime(df.op, df.base, df.delta, df.units);
+            case "trunc":
+                return this.sqlTrunc(df.expr, df.units);
+            case "extract":
+                return this.sqlExtract(df.expr, df.units);
+            case "cast":
+                return this.sqlCast(df);
+            case "regexpMatch":
+                return this.sqlRegexpMatch(df.expr, df.regexp);
+        }
+    }
+    sqlSumDistinct(_key, _value) {
+        return "sqlSumDistinct called bu not implemented";
+    }
+    sqlSampleTable(tableSQL, sample) {
+        if (sample !== undefined) {
+            throw new Error(`Sampling is not supported on dialect ${this.name}.`);
+        }
+        return tableSQL;
+    }
+    sqlOrderBy(orderTerms) {
+        return `ORDER BY ${orderTerms.join(",")}`;
+    }
+}
+exports.Dialect = Dialect;
+//# sourceMappingURL=dialect.js.map

package/dist/dialect/dialect_map.d.ts

@@ -0,0 +1,3 @@
+import { Dialect } from "./dialect";
+export declare function getDialect(name: string): Dialect;
+export declare function registerDialect(d: Dialect): void;

package/dist/dialect/dialect_map.js

@@ -0,0 +1,35 @@
+"use strict";
+/*
+ * Copyright 2021 Google LLC
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.registerDialect = exports.getDialect = void 0;
+const _1 = require(".");
+const postgres_1 = require("./postgres");
+const standardsql_1 = require("./standardsql");
+const dialectMap = new Map();
+function getDialect(name) {
+    const d = dialectMap.get(name);
+    if (d === undefined) {
+        throw new Error(`Unknown Dialect ${name}`);
+    }
+    return d;
+}
+exports.getDialect = getDialect;
+function registerDialect(d) {
+    dialectMap.set(d.name, d);
+}
+exports.registerDialect = registerDialect;
+registerDialect(new postgres_1.PostgresDialect());
+registerDialect(new standardsql_1.StandardSQLDialect());
+registerDialect(new _1.DuckDBDialect());
+//# sourceMappingURL=dialect_map.js.map

package/dist/dialect/duckdb.d.ts

@@ -0,0 +1,49 @@
+import { DateUnit, Expr, ExtractUnit, Sampling, StructDef, TimeFieldType, TimestampUnit, TimeValue, TypecastFragment } from "../model";
+import { Dialect, DialectFieldList, FunctionInfo } from "./dialect";
+export declare class DuckDBDialect extends Dialect {
+    name: string;
+    defaultNumberType: string;
+    hasFinalStage: boolean;
+    stringTypeName: string;
+    divisionIsInteger: boolean;
+    supportsSumDistinctFunction: boolean;
+    unnestWithNumbers: boolean;
+    defaultSampling: {
+        rows: number;
+    };
+    supportUnnestArrayAgg: boolean;
+    supportsCTEinCoorelatedSubQueries: boolean;
+    functionInfo: Record<string, FunctionInfo>;
+    get udfPrefix(): string;
+    quoteTablePath(tableName: string): string;
+    sqlGroupSetTable(groupSetCount: number): string;
+    sqlAnyValue(groupSet: number, fieldName: string): string;
+    mapFields(fieldList: DialectFieldList): string;
+    sqlAggregateTurtle(groupSet: number, fieldList: DialectFieldList, orderBy: string | undefined, limit: number | undefined): string;
+    sqlAnyValueTurtle(groupSet: number, fieldList: DialectFieldList): string;
+    sqlAnyValueLastTurtle(name: string, groupSet: number, sqlName: string): string;
+    sqlCoaleseMeasuresInline(groupSet: number, fieldList: DialectFieldList): string;
+    sqlUnnestAlias(source: string, alias: string, _fieldList: DialectFieldList, _needDistinctKey: boolean): string;
+    sqlSumDistinctHashedKey(_sqlDistinctKey: string): string;
+    sqlGenerateUUID(): string;
+    sqlDateToString(sqlDateExp: string): string;
+    sqlFieldReference(alias: string, fieldName: string, _fieldType: string, _isNested: boolean, isArray: boolean): string;
+    sqlUnnestPipelineHead(isSingleton: boolean, sourceSQLExpression: string): string;
+    sqlCreateFunction(id: string, funcText: string): string;
+    sqlCreateFunctionCombineLastStage(lastStageName: string, structDef: StructDef): string;
+    sqlSelectAliasAsStruct(alias: string, physicalFieldNames: string[]): string;
+    sqlMaybeQuoteIdentifier(identifier: string): string;
+    sqlCreateTableAsSelect(_tableName: string, _sql: string): string;
+    getFunctionInfo(_functionName: string): FunctionInfo | undefined;
+    sqlMeasureTime(from: TimeValue, to: TimeValue, units: string): Expr;
+    sqlNow(): Expr;
+    sqlTrunc(sqlTime: TimeValue, units: TimestampUnit): Expr;
+    sqlExtract(from: TimeValue, units: ExtractUnit): Expr;
+    sqlAlterTime(op: "+" | "-", expr: TimeValue, n: Expr, timeframe: DateUnit): Expr;
+    sqlCast(cast: TypecastFragment): Expr;
+    sqlRegexpMatch(expr: Expr, regexp: string): Expr;
+    sqlLiteralTime(timeString: string, type: TimeFieldType, _timezone: string): string;
+    sqlSumDistinct(key: string, value: string): string;
+    sqlSampleTable(tableSQL: string, sample: Sampling | undefined): string;
+    sqlOrderBy(orderTerms: string[]): string;
+}