@malloydata/malloy 0.0.32-dev230428002803 → 0.0.32-dev230506172904

package/dist/dialect/dialect.d.ts

@@ -59,7 +59,7 @@ export declare abstract class Dialect {
     abstract sqlRegexpMatch(expr: Expr, regex: string): Expr;
     getFunctionInfo(functionName: string): FunctionInfo | undefined;
     dialectExpr(df: DialectFragment): Expr;
-    sqlSumDistinct(_key: string, _value: string): string;
+    sqlSumDistinct(_key: string, _value: string, _funcName: string): string;
     sqlSampleTable(tableSQL: string, sample: Sampling | undefined): string;
     sqlOrderBy(orderTerms: string[]): string;
 }

package/dist/dialect/dialect.js

@@ -68,7 +68,7 @@ class Dialect {
                 return [this.sqlLiteralString(df.literal)];
         }
     }
-    sqlSumDistinct(_key, _value) {
+    sqlSumDistinct(_key, _value, _funcName) {
         return 'sqlSumDistinct called bu not implemented';
     }
     sqlSampleTable(tableSQL, sample) {

package/dist/dialect/duckdb.d.ts

@@ -25,7 +25,7 @@ export declare class DuckDBDialect extends Dialect {
     sqlAnyValueTurtle(groupSet: number, fieldList: DialectFieldList): string;
     sqlAnyValueLastTurtle(name: string, groupSet: number, sqlName: string): string;
     sqlCoaleseMeasuresInline(groupSet: number, fieldList: DialectFieldList): string;
-    sqlUnnestAlias(source: string, alias: string, _fieldList: DialectFieldList, _needDistinctKey: boolean): string;
+    sqlUnnestAlias(source: string, alias: string, _fieldList: DialectFieldList, needDistinctKey: boolean): string;
     sqlSumDistinctHashedKey(_sqlDistinctKey: string): string;
     sqlGenerateUUID(): string;
     sqlDateToString(sqlDateExp: string): string;
@@ -45,7 +45,7 @@ export declare class DuckDBDialect extends Dialect {
     sqlCast(cast: TypecastFragment): Expr;
     sqlRegexpMatch(expr: Expr, regexp: string): Expr;
     sqlLiteralTime(timeString: string, type: TimeFieldType, _timezone: string): string;
-    sqlSumDistinct(key: string, value: string): string;
+    sqlSumDistinct(key: string, value: string, funcName: string): string;
     sqlSampleTable(tableSQL: string, sample: Sampling | undefined): string;
     sqlOrderBy(orderTerms: string[]): string;
     sqlLiteralString(literal: string): string;

package/dist/dialect/duckdb.js

@@ -129,11 +129,11 @@ class DuckDBDialect extends dialect_1.Dialect {
         this.stringTypeName = 'VARCHAR';
         this.divisionIsInteger = true;
         this.supportsSumDistinctFunction = true;
-        this.unnestWithNumbers = true;
+        this.unnestWithNumbers = false;
         this.defaultSampling = { rows: 50000 };
         this.supportUnnestArrayAgg = true;
         this.supportsCTEinCoorelatedSubQueries = true;
-        this.dontUnionIndex = false;
+        this.dontUnionIndex = true;
         this.supportsQualify = true;
         this.functionInfo = {
             concat: { returnType: 'string' },
@@ -183,13 +183,27 @@ class DuckDBDialect extends dialect_1.Dialect {
             .join(', ');
         return `COALESCE(FIRST({${fields}}) FILTER(WHERE group_set=${groupSet}), {${nullValues}})`;
     }
-    sqlUnnestAlias(source, alias, _fieldList, _needDistinctKey) {
-        return `LEFT JOIN (select UNNEST(generate_series(1,
-        100000, --
-        -- (SELECT genres_length FROM movies limit 1),
-        1)) as __row_id) as ${alias} ON  ${alias}.__row_id <= array_length(${source})`;
-        // When DuckDB supports lateral joins...
-        //return `,(select UNNEST(generate_series(1, length(${source}),1))) as ${alias}(__row_id)`;
+    // sqlUnnestAlias(
+    //   source: string,
+    //   alias: string,
+    //   _fieldList: DialectFieldList,
+    //   _needDistinctKey: boolean
+    // ): string {
+    //   return `LEFT JOIN (select UNNEST(generate_series(1,
+    //       100000, --
+    //       -- (SELECT genres_length FROM movies limit 1),
+    //       1)) as __row_id) as ${alias} ON  ${alias}.__row_id <= array_length(${source})`;
+    //   // When DuckDB supports lateral joins...
+    //   //return `,(select UNNEST(generate_series(1, length(${source}),1))) as ${alias}(__row_id)`;
+    // }
+    sqlUnnestAlias(source, alias, _fieldList, needDistinctKey) {
+        //Simulate left joins by guarenteeing there is at least one row.
+        if (!needDistinctKey) {
+            return `LEFT JOIN LATERAL (SELECT UNNEST(${source}), 1 as ignoreme) as ${alias}_outer(${alias},ignoreme) ON ${alias}_outer.ignoreme=1`;
+        }
+        else {
+            return `LEFT JOIN LATERAL (SELECT UNNEST(GENERATE_SERIES(1, length(${source}),1)) as __row_id, UNNEST(${source}), 1 as ignoreme) as ${alias}_outer(__row_id, ${alias},ignoreme) ON  ${alias}_outer.ignoreme=1`;
+        }
     }
     sqlSumDistinctHashedKey(_sqlDistinctKey) {
         return 'uses sumDistinctFunction, should not be called';
@@ -201,7 +215,11 @@ class DuckDBDialect extends dialect_1.Dialect {
         return `(${sqlDateExp})::date::varchar`;
     }
     sqlFieldReference(alias, fieldName, _fieldType, _isNested, isArray) {
-        if (isArray) {
+        // LTNOTE: hack, in duckdb we can't have structs as tables so we kind of simulate it.
+        if (fieldName === '__row_id') {
+            return `${alias}_outer.__row_id`;
+        }
+        else if (isArray) {
             return alias;
         }
         else {
@@ -321,10 +339,10 @@ class DuckDBDialect extends dialect_1.Dialect {
             throw new Error(`Unknown Literal time format ${type}`);
         }
     }
-    sqlSumDistinct(key, value) {
+    sqlSumDistinct(key, value, funcName) {
         // return `sum_distinct(list({key:${key}, val: ${value}}))`;
         return `(
-      SELECT sum(a.val) as value
+      SELECT ${funcName}(a.val) as value
       FROM (
         SELECT UNNEST(list(distinct {key:${key}, val: ${value}})) a
       )

package/dist/dialect/postgres.d.ts

@@ -45,7 +45,7 @@ export declare class PostgresDialect extends Dialect {
     sqlLiteralTime(timeString: string, type: TimeFieldType, _timezone: string): string;
     getFunctionInfo(functionName: string): FunctionInfo | undefined;
     sqlMeasureTime(from: TimeValue, to: TimeValue, units: string): Expr;
-    sqlSumDistinct(key: string, value: string): string;
+    sqlSumDistinct(key: string, value: string, funcName: string): string;
     sqlSampleTable(tableSQL: string, sample: Sampling | undefined): string;
     sqlOrderBy(orderTerms: string[]): string;
     sqlLiteralString(literal: string): string;

package/dist/dialect/postgres.js

@@ -296,10 +296,10 @@ class PostgresDialect extends dialect_1.Dialect {
         }
         throw new Error(`Unknown or unhandled postgres time unit: ${units}`);
     }
-    sqlSumDistinct(key, value) {
+    sqlSumDistinct(key, value, funcName) {
         // return `sum_distinct(list({key:${key}, val: ${value}}))`;
         return `(
-      SELECT sum((a::json->>'f2')::DOUBLE PRECISION) as value
+      SELECT ${funcName}((a::json->>'f2')::DOUBLE PRECISION) as value
       FROM (
         SELECT UNNEST(array_agg(distinct row_to_json(row(${key},${value}))::text)) a
       ) a

package/dist/model/malloy_query.js

@@ -290,7 +290,7 @@ class QueryField extends QueryNode {
         let ret;
         if (distinctKeySQL) {
             if (this.parent.dialect.supportsSumDistinctFunction) {
-                ret = this.parent.dialect.sqlSumDistinct(distinctKeySQL, dimSQL);
+                ret = this.parent.dialect.sqlSumDistinct(distinctKeySQL, dimSQL, 'SUM');
             }
             else {
                 ret = sqlSumDistinct(this.parent.dialect, dimSQL, distinctKeySQL);
@@ -318,13 +318,15 @@ class QueryField extends QueryNode {
                 countDistinctKeySQL = `CASE WHEN ${state.whereSQL} THEN ${distinctKeySQL} END`;
             }
             let sumDistinctSQL;
+            let avgDistinctSQL;
             if (this.parent.dialect.supportsSumDistinctFunction) {
-                sumDistinctSQL = this.parent.dialect.sqlSumDistinct(distinctKeySQL, dimSQL);
+                avgDistinctSQL = this.parent.dialect.sqlSumDistinct(distinctKeySQL, dimSQL, 'AVG');
             }
             else {
                 sumDistinctSQL = sqlSumDistinct(this.parent.dialect, dimSQL, distinctKeySQL);
+                avgDistinctSQL = `(${sumDistinctSQL})/NULLIF(COUNT(DISTINCT CASE WHEN ${dimSQL} IS NOT NULL THEN ${countDistinctKeySQL} END),0)`;
             }
-            return `(${sumDistinctSQL})/NULLIF(COUNT(DISTINCT ${countDistinctKeySQL}),0)`;
+            return avgDistinctSQL;
         }
         else {
             return `AVG(${dimSQL})`;
@@ -1667,7 +1669,7 @@ class QueryQuery extends QueryField {
         let structSQL = qs.structSourceSQL(stageWriter);
         if ((0, malloy_types_1.isJoinOn)(structRelationship)) {
             if (ji.makeUniqueKey) {
-                structSQL = `(SELECT ${qs.dialect.sqlGenerateUUID()} as __distinct_key, * FROM ${structSQL})`;
+                structSQL = `(SELECT ${qs.dialect.sqlGenerateUUID()} as __distinct_key, * FROM ${structSQL} as x)`;
             }
             let onCondition = '';
             if (qs.parent === undefined) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@malloydata/malloy",
-  "version": "0.0.32-dev230428002803",
+  "version": "0.0.32-dev230506172904",
   "license": "MIT",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",