@malloydata/malloy 0.0.37-dev230525224620 → 0.0.37-dev230526070709

package/dist/dialect/dialect.d.ts

@@ -49,7 +49,7 @@ export declare abstract class Dialect {
     abstract sqlAnyValueTurtle(groupSet: number, fieldList: DialectFieldList): string;
     abstract sqlAnyValueLastTurtle(name: string, groupSet: number, sqlName: string): string;
     abstract sqlCoaleseMeasuresInline(groupSet: number, fieldList: DialectFieldList): string;
-    abstract sqlUnnestAlias(source: string, alias: string, fieldList: DialectFieldList, needDistinctKey: boolean, isArray: boolean): string;
+    abstract sqlUnnestAlias(source: string, alias: string, fieldList: DialectFieldList, needDistinctKey: boolean, isArray: boolean, isInNestedPipeline: boolean): string;
     abstract sqlSumDistinctHashedKey(sqlDistinctKey: string): string;
     abstract sqlGenerateUUID(): string;
     abstract sqlFieldReference(alias: string, fieldName: string, fieldType: string, isNested: boolean, isArray: boolean): string;

package/dist/dialect/duckdb.d.ts

@@ -25,7 +25,7 @@ export declare class DuckDBDialect extends Dialect {
     sqlAnyValueTurtle(groupSet: number, fieldList: DialectFieldList): string;
     sqlAnyValueLastTurtle(name: string, groupSet: number, sqlName: string): string;
     sqlCoaleseMeasuresInline(groupSet: number, fieldList: DialectFieldList): string;
-    sqlUnnestAlias(source: string, alias: string, _fieldList: DialectFieldList, needDistinctKey: boolean): string;
+    sqlUnnestAlias(source: string, alias: string, _fieldList: DialectFieldList, needDistinctKey: boolean, _isArray: boolean, isInNestedPipeline: boolean): string;
     sqlSumDistinctHashedKey(_sqlDistinctKey: string): string;
     sqlGenerateUUID(): string;
     sqlDateToString(sqlDateExp: string): string;

package/dist/dialect/duckdb.js

@@ -45,7 +45,7 @@ class DuckDBDialect extends dialect_1.Dialect {
         this.stringTypeName = 'VARCHAR';
         this.divisionIsInteger = true;
         this.supportsSumDistinctFunction = true;
-        this.unnestWithNumbers = false;
+        this.unnestWithNumbers = true;
         this.defaultSampling = { rows: 50000 };
         this.supportUnnestArrayAgg = true;
         this.supportsCTEinCoorelatedSubQueries = true;
@@ -112,7 +112,16 @@ class DuckDBDialect extends dialect_1.Dialect {
     //   // When DuckDB supports lateral joins...
     //   //return `,(select UNNEST(generate_series(1, length(${source}),1))) as ${alias}(__row_id)`;
     // }
-    sqlUnnestAlias(source, alias, _fieldList, needDistinctKey) {
+    sqlUnnestAlias(source, alias, _fieldList, needDistinctKey, _isArray, isInNestedPipeline) {
+        if (this.unnestWithNumbers) {
+            // Duckdb can't unnest in a coorelated subquery at the moment so we hack it.
+            const arrayLen = isInNestedPipeline
+                ? '100000'
+                : `array_length(${source})`;
+            return `LEFT JOIN (select UNNEST(generate_series(1,
+        ${arrayLen},
+        1)) as __row_id) as ${alias} ON  ${alias}.__row_id <= array_length(${source})`;
+        }
         //Simulate left joins by guarenteeing there is at least one row.
         if (!needDistinctKey) {
             return `LEFT JOIN LATERAL (SELECT UNNEST(${source}), 1 as ignoreme) as ${alias}_outer(${alias},ignoreme) ON ${alias}_outer.ignoreme=1`;
@@ -132,7 +141,7 @@ class DuckDBDialect extends dialect_1.Dialect {
     }
     sqlFieldReference(alias, fieldName, _fieldType, _isNested, isArray) {
         // LTNOTE: hack, in duckdb we can't have structs as tables so we kind of simulate it.
-        if (fieldName === '__row_id') {
+        if (!this.unnestWithNumbers && fieldName === '__row_id') {
             return `${alias}_outer.__row_id`;
         }
         else if (isArray) {

package/dist/dialect/postgres.d.ts

@@ -25,7 +25,7 @@ export declare class PostgresDialect extends Dialect {
     sqlAnyValueTurtle(groupSet: number, fieldList: DialectFieldList): string;
     sqlAnyValueLastTurtle(name: string, groupSet: number, sqlName: string): string;
     sqlCoaleseMeasuresInline(groupSet: number, fieldList: DialectFieldList): string;
-    sqlUnnestAlias(source: string, alias: string, fieldList: DialectFieldList, needDistinctKey: boolean, isArray: boolean): string;
+    sqlUnnestAlias(source: string, alias: string, fieldList: DialectFieldList, needDistinctKey: boolean, isArray: boolean, _isInNestedPipeline: boolean): string;
     sqlSumDistinctHashedKey(sqlDistinctKey: string): string;
     sqlGenerateUUID(): string;
     sqlFieldReference(alias: string, fieldName: string, fieldType: string, isNested: boolean, _isArray: boolean): string;

package/dist/dialect/postgres.js

@@ -144,7 +144,7 @@ class PostgresDialect extends dialect_1.Dialect {
     //     return `UNNEST(${source}) as ${alias}(${fields.join(", ")})`;
     //   }
     // }
-    sqlUnnestAlias(source, alias, fieldList, needDistinctKey, isArray) {
+    sqlUnnestAlias(source, alias, fieldList, needDistinctKey, isArray, _isInNestedPipeline) {
         if (isArray) {
             if (needDistinctKey) {
                 return `LEFT JOIN UNNEST(ARRAY((SELECT jsonb_build_object('__row_id', row_number() over (), 'value', v) FROM UNNEST(${source}) as v))) as ${alias} ON true`;

package/dist/dialect/standardsql.d.ts

@@ -24,7 +24,7 @@ export declare class StandardSQLDialect extends Dialect {
     sqlAnyValueTurtle(groupSet: number, fieldList: DialectFieldList): string;
     sqlAnyValueLastTurtle(name: string, groupSet: number, sqlName: string): string;
     sqlCoaleseMeasuresInline(groupSet: number, fieldList: DialectFieldList): string;
-    sqlUnnestAlias(source: string, alias: string, fieldList: DialectFieldList, needDistinctKey: boolean, isArray: boolean): string;
+    sqlUnnestAlias(source: string, alias: string, fieldList: DialectFieldList, needDistinctKey: boolean, isArray: boolean, _isInNestedPipeline: boolean): string;
     sqlSumDistinctHashedKey(sqlDistinctKey: string): string;
     sqlGenerateUUID(): string;
     sqlFieldReference(alias: string, fieldName: string, _fieldType: string, _isNested: boolean, _isArray: boolean): string;

package/dist/dialect/standardsql.js

@@ -221,7 +221,7 @@ class StandardSQLDialect extends dialect_1.Dialect {
     //
     // BigQuery will allocate more resources if we use a CROSS JOIN so we do that instead.
     //
-    sqlUnnestAlias(source, alias, fieldList, needDistinctKey, isArray) {
+    sqlUnnestAlias(source, alias, fieldList, needDistinctKey, isArray, _isInNestedPipeline) {
         if (isArray) {
             if (needDistinctKey) {
                 return `LEFT JOIN UNNEST(ARRAY(( SELECT AS STRUCT row_number() over() as __row_id, value FROM UNNEST(${source}) value))) as ${alias}`;

package/dist/model/malloy_query.d.ts

@@ -223,6 +223,7 @@ declare class QueryQuery extends QueryField {
     stageWriter: StageWriter | undefined;
     constructor(fieldDef: TurtleDef, parent: QueryStruct, stageWriter: StageWriter | undefined);
     static makeQuery(fieldDef: TurtleDef, parentStruct: QueryStruct, stageWriter?: StageWriter | undefined): QueryQuery;
+    inNestedPipeline(): boolean;
     getFieldList(): QueryFieldDef[];
     expandField(f: QueryFieldDef): {
         as: any;

package/dist/model/malloy_query.js

@@ -1163,6 +1163,10 @@ class QueryQuery extends QueryField {
                 return new QueryQueryIndex(flatTurtleDef, parent, stageWriter);
         }
     }
+    inNestedPipeline() {
+        return (this.parent.fieldDef.structSource.type === 'sql' &&
+            this.parent.fieldDef.structSource.method === 'nested');
+    }
     getFieldList() {
         switch (this.firstSegment.type) {
             // case "index":
@@ -1718,7 +1722,8 @@ class QueryQuery extends QueryField {
             const fieldExpression = this.parent.dialect.sqlFieldReference(qs.parent.getSQLIdentifier(), structRelationship.field, 'struct', qs.parent.fieldDef.structRelationship.type === 'nested', this.parent.fieldDef.structRelationship.type === 'nested' &&
                 this.parent.fieldDef.structRelationship.isArray);
             // we need to generate primary key.  If parent has a primary key combine
-            s += `${this.parent.dialect.sqlUnnestAlias(fieldExpression, ji.alias, ji.getDialectFieldList(), ji.makeUniqueKey, structRelationship.isArray)}\n`;
+            // console.log(ji.alias, fieldExpression, this.inNestedPipeline());
+            s += `${this.parent.dialect.sqlUnnestAlias(fieldExpression, ji.alias, ji.getDialectFieldList(), ji.makeUniqueKey, structRelationship.isArray, this.inNestedPipeline())}\n`;
         }
         else if (structRelationship.type === 'inline') {
             throw new Error('Internal Error: inline structs should never appear in join trees');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@malloydata/malloy",
-  "version": "0.0.37-dev230525224620",
+  "version": "0.0.37-dev230526070709",
   "license": "MIT",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",