@malloydata/malloy-tests 0.0.433 → 0.0.434

package/src/databases/all/expr.spec.ts

@@ -337,11 +337,8 @@ describe.each(runtimes.runtimeList)('%s', (databaseName, runtime) => {
     });
   });
 
-  // TODO not sure why this test needs to be skipped on postgres, feels like an oversight
-  test.when(!['postgres', 'mysql'].includes(databaseName))(
-    'model: dates named',
-    async () => {
-      await expect(`
+  test('model: dates named', async () => {
+    await expect(`
       run: ${databaseName}.table('malloytest.alltypes')->{
         group_by:
           t_date,
@@ -356,19 +353,18 @@ describe.each(runtimes.runtimeList)('%s', (databaseName, runtime) => {
           t_timestamp_year is t_timestamp.year,
       }
     `).toMatchResult(testModel, {
-        t_date: new Date('2020-03-02'),
-        t_date_month: new Date('2020-03-01'),
-        t_date_year: new Date('2020-01-01'),
-        t_timestamp: new Date('2020-03-02T12:35:56.000Z'),
-        t_timestamp_second: new Date('2020-03-02T12:35:56.000Z'),
-        t_timestamp_minute: new Date('2020-03-02T12:35:00.000Z'),
-        t_timestamp_hour: new Date('2020-03-02T12:00:00.000Z'),
-        t_timestamp_date: new Date('2020-03-02'),
-        t_timestamp_month: new Date('2020-03-01'),
-        t_timestamp_year: new Date('2020-01-01'),
-      });
-    }
-  );
+      t_date: new Date('2020-03-02'),
+      t_date_month: new Date('2020-03-01'),
+      t_date_year: new Date('2020-01-01'),
+      t_timestamp: new Date('2020-03-02T12:35:56.000Z'),
+      t_timestamp_second: new Date('2020-03-02T12:35:56.000Z'),
+      t_timestamp_minute: new Date('2020-03-02T12:35:00.000Z'),
+      t_timestamp_hour: new Date('2020-03-02T12:00:00.000Z'),
+      t_timestamp_date: new Date('2020-03-02'),
+      t_timestamp_month: new Date('2020-03-01'),
+      t_timestamp_year: new Date('2020-01-01'),
+    });
+  });
 
   it('named query metadata undefined', async () => {
     const result = await expressionModel
@@ -736,6 +732,142 @@ describe.each(runtimes.runtimeList)('%s', (databaseName, runtime) => {
     }
   );
 
+  // A filtered join is emitted as a subquery, so every join below it lives
+  // inside that subquery and has to be projected back out for the outer query
+  // to name it -- at every depth, not just the direct children.
+  describe('joins below a filtered join', () => {
+    const chain = (filterTwo: string) => `
+      source: sf is ${databaseName}.table('malloytest.state_facts')
+
+      source: three is sf
+      source: two is sf extend {
+        ${filterTwo}
+        join_one: three on three.state = state
+      }
+      source: one is sf extend {
+        where: state ~ 'A%'
+        join_one: two on two.state = state
+      }
+      source: zero is sf extend { join_one: one on one.state = state }
+    `;
+
+    test.when(runtime.dialect.supportsComplexFilteredSources)(
+      'two levels below',
+      async () => {
+        await expect(`
+          ${chain('')}
+          run: zero -> {
+            aggregate:
+              child is one.two.count()
+              grandchild is one.two.three.count()
+          }
+        `).toMatchResult(testModel, {child: 4, grandchild: 4});
+
+        await expect(`
+          ${chain('')}
+          run: zero -> {
+            group_by: grandchild_state is one.two.three.state
+            order_by: grandchild_state
+            limit: 1
+          }
+        `).toMatchResult(testModel, {grandchild_state: 'AK'});
+      }
+    );
+
+    // `having:` and `calculate:` name output fields, so their field usage is
+    // rooted in the query's output and does not resolve against its source.
+    test.when(runtime.dialect.supportsComplexFilteredSources)(
+      'having on an output field, with a filtered join',
+      async () => {
+        await expect(`
+          ${chain('')}
+          run: zero -> {
+            group_by: grandchild_state is one.two.three.state
+            aggregate: rows_here is count()
+            having: rows_here > 0
+            order_by: grandchild_state
+            limit: 1
+          }
+        `).toMatchResult(testModel, {grandchild_state: 'AK', rows_here: 1});
+      }
+    );
+
+    // The packed columns are the source's own column names, so the struct has
+    // to quote them on the value side and in any type it declares.
+    test.when(runtime.dialect.supportsComplexFilteredSources)(
+      'a packed column name which needs quoting',
+      async () => {
+        const oddName = runtime.dialect.sqlQuoteIdentifier('order');
+        await expect(`
+          source: two is ${databaseName}.sql("""SELECT 1 as ${oddName}, 1 as k""")
+          source: one is ${databaseName}.sql("""SELECT 1 as k, 2 as j""") extend {
+            where: j > 0
+            join_one: two on two.k = k
+          }
+          source: zero is ${databaseName}.sql("""SELECT 1 as k""") extend {
+            join_one: one on one.k = k
+          }
+
+          run: zero -> { group_by: g is one.two.\`order\` }
+        `).toMatchResult(testModel, {g: 1});
+      }
+    );
+
+    // A distinct key is the one thing the packed struct must carry which no
+    // expression in the query names, and when the key is computed what the
+    // query reads is the columns its expression reads, not the key's name.
+    test.when(runtime.dialect.supportsComplexFilteredSources)(
+      'two levels below, reached by a computed primary key',
+      async () => {
+        await expect(`
+          source: sf is ${databaseName}.table('malloytest.state_facts')
+
+          source: three is sf extend {
+            dimension: pk is births
+            primary_key: pk
+          }
+          source: two is sf extend { join_one: three on three.state = state }
+          source: one is sf extend {
+            where: state ~ 'A%'
+            join_one: two on two.state = state
+          }
+          source: zero is sf extend { join_one: one on one.state = state }
+
+          run: zero -> {
+            aggregate:
+              child is one.two.count()
+              grandchild is one.two.three.count()
+          }
+        `).toMatchResult(testModel, {child: 4, grandchild: 4});
+      }
+    );
+
+    // `two` is filtered as well, so it becomes a subquery of its own inside
+    // the one for `one`, and `three` reaches the outer query through it.
+    test.when(runtime.dialect.supportsComplexFilteredSources)(
+      'two levels below, both filtered',
+      async () => {
+        await expect(`
+          ${chain("where: state != 'AK'")}
+          run: zero -> {
+            aggregate:
+              child is one.two.count()
+              grandchild is one.two.three.count()
+          }
+        `).toMatchResult(testModel, {child: 3, grandchild: 3});
+
+        await expect(`
+          ${chain("where: state != 'AK'")}
+          run: zero -> {
+            group_by: grandchild_state is one.two.three.state
+            order_by: grandchild_state
+            limit: 1
+          }
+        `).toMatchResult(testModel, {grandchild_state: 'AL'});
+      }
+    );
+  });
+
   test('joined filtered explores with NO dependencies', async () => {
     await expect(`
       source: sf is ${databaseName}.table('malloytest.state_facts') extend {

package/src/databases/all/nest.spec.ts

@@ -0,0 +1,237 @@
+/*
+ * Copyright Contributors to the Malloy project
+ * SPDX-License-Identifier: MIT
+ */
+
+import {RuntimeList, allDatabases} from '../../runtimes';
+import {databasesFromEnvironmentOr} from '../../util';
+import '@malloydata/malloy/test/matchers';
+import {wrapTestModel} from '@malloydata/malloy/test';
+
+const runtimes = new RuntimeList(databasesFromEnvironmentOr(allDatabases));
+
+afterAll(async () => {
+  await runtimes.closeAll();
+});
+
+// A nest is a column whose value is the result of a query. What that column
+// contains, and in what order, is the subject here.
+//
+// The order is the order of the stage which produced it -- the order the user
+// asked for, or the one Malloy picked. Reading the array back is the only way to
+// see that, and the ordering is attached to the SQL which packs the array, which
+// is a different expression for a one-stage nest than for a pipelined one.
+//
+// The ordering tests nest over the four 'N' states whose popular_name is
+// Isabella, because their four candidate orderings are all distinct:
+//   state desc    NY NV NM NJ
+//   state asc     NJ NM NV NY
+//   births desc   NY NJ NM NV
+//   births asc    NV NM NJ NY
+// A pipelined test orders its first stage the opposite way from its last, so an
+// array carrying the wrong stage's order fails instead of coincidentally passing.
+
+describe.each(runtimes.runtimeList)('%s', (databaseName, runtime) => {
+  const model = wrapTestModel(
+    runtime,
+    `
+    query: isabellaN is ${databaseName}.table('malloytest.state_facts') -> {
+      where: popular_name = 'Isabella' and state ~ 'N%'
+      group_by: popular_name
+    }`
+  );
+
+  const stateDesc = [{st: 'NY'}, {st: 'NV'}, {st: 'NM'}, {st: 'NJ'}];
+  const birthsDesc = [{st: 'NY'}, {st: 'NJ'}, {st: 'NM'}, {st: 'NV'}];
+  const keywordStateDesc = [
+    {'select': 'NY'},
+    {'select': 'NV'},
+    {'select': 'NM'},
+    {'select': 'NJ'},
+  ];
+
+  test.when(runtime.supportsNesting)(
+    `one-stage nest is in the order it asked for - ${databaseName}`,
+    async () => {
+      await expect(`
+        run: isabellaN + {
+          nest: n is {
+            group_by: st is state
+            aggregate: b is births.sum()
+            order_by: st desc
+          }
+        }`).toMatchResult(model, {n: stateDesc});
+    }
+  );
+
+  test.when(runtime.supportsNesting)(
+    `one-stage nest orders by a name which needs quoting - ${databaseName}`,
+    async () => {
+      await expect(`
+        run: isabellaN + {
+          nest: n is {
+            group_by: \`select\` is state
+            aggregate: b is births.sum()
+            order_by: \`select\` desc
+          }
+        }`).toMatchResult(model, {n: keywordStateDesc});
+    }
+  );
+
+  test.when(runtime.supportsNesting)(
+    `one-stage nest with no order_by is largest first - ${databaseName}`,
+    async () => {
+      await expect(`
+        run: isabellaN + {
+          nest: n is {
+            group_by: st is state
+            aggregate: b is births.sum()
+          }
+        }`).toMatchResult(model, {n: birthsDesc});
+    }
+  );
+
+  test.when(runtime.dialect.supportsPipelinesInViews)(
+    `pipelined nest is in the order its last stage asked for - ${databaseName}`,
+    async () => {
+      await expect(`
+        run: isabellaN + {
+          nest: n is {
+            group_by: st is state
+            aggregate: b is births.sum()
+            order_by: st asc
+          } -> {
+            group_by: st, b
+            order_by: st desc
+          }
+        }`).toMatchResult(model, {n: stateDesc});
+    }
+  );
+
+  test.when(runtime.dialect.supportsPipelinesInViews)(
+    `pipelined nest orders by a name which needs quoting - ${databaseName}`,
+    async () => {
+      await expect(`
+        run: isabellaN + {
+          nest: n is {
+            group_by: \`select\` is state
+            aggregate: b is births.sum()
+            order_by: \`select\` asc
+          } -> {
+            group_by: \`select\`, b
+            order_by: \`select\` desc
+          }
+        }`).toMatchResult(model, {n: keywordStateDesc});
+    }
+  );
+
+  // An ordinal order_by names an output field by number, counting from one.
+  test.when(runtime.supportsNesting)(
+    `one-stage nest orders by an output field number - ${databaseName}`,
+    async () => {
+      await expect(`
+        run: isabellaN + {
+          nest: n is {
+            group_by: st is state
+            aggregate: b is births.sum()
+            order_by: 1 desc
+          }
+        }`).toMatchResult(model, {n: stateDesc});
+    }
+  );
+
+  test.when(runtime.dialect.supportsPipelinesInViews)(
+    `pipelined nest orders by an output field number - ${databaseName}`,
+    async () => {
+      await expect(`
+        run: isabellaN + {
+          nest: n is {
+            group_by: st is state
+            aggregate: b is births.sum()
+            order_by: st asc
+          } -> {
+            group_by: st, b
+            order_by: 1 desc
+          }
+        }`).toMatchResult(model, {n: stateDesc});
+    }
+  );
+
+  test.when(runtime.dialect.supportsPipelinesInViews)(
+    `pipelined nest ending in select orders by an output field number - ${databaseName}`,
+    async () => {
+      await expect(`
+        run: isabellaN + {
+          nest: n is {
+            group_by: st is state
+            aggregate: b is births.sum()
+            order_by: st asc
+          } -> {
+            select: st, b
+            order_by: 2 desc
+          }
+        }`).toMatchResult(model, {n: birthsDesc});
+    }
+  );
+
+  test.when(runtime.dialect.supportsPipelinesInViews)(
+    `pipelined nest with no order_by is largest first - ${databaseName}`,
+    async () => {
+      await expect(`
+        run: isabellaN + {
+          nest: n is {
+            group_by: st is state
+            aggregate: b is births.sum()
+          } -> {
+            group_by: st
+            aggregate: tot is b.sum()
+          }
+        }`).toMatchResult(model, {n: birthsDesc});
+    }
+  );
+
+  test.when(runtime.supportsNesting)(
+    `an aggregate which is null in a nest reads back as null - ${databaseName}`,
+    async () => {
+      await expect(`
+        run: ${databaseName}.table('malloytest.state_facts') -> {
+          group_by: state
+          nest: ugly is {
+            group_by: popular_name
+            aggregate: foo is NULLIF(sum(airport_count)*0,0)+1
+          }
+        }`).toMatchPaths(model, {'ugly.foo': null});
+    }
+  );
+
+  // A null value in the grouping above a nest must not null out the nest.
+  test.when(runtime.supportsNesting)(
+    `a nest under a null group is still a nest - ${databaseName}`,
+    async () => {
+      const result = await runtime
+        .loadQuery(
+          `
+        run: ${databaseName}.table('malloytest.airports') -> {
+          where: faa_region is null
+          group_by: faa_region
+          aggregate: airport_count is count()
+          nest: by_state is {
+            where: state is not null
+            group_by: state
+            aggregate: airport_count is count()
+          }
+          nest: capped is {
+            where: state is not null
+            group_by: state
+            aggregate: airport_count is count()
+            limit: 1
+          }
+        }`
+        )
+        .run();
+      const row = result.data.toObject()[0] as Record<string, unknown>;
+      expect(row['by_state']).not.toBe(null);
+      expect(row['capped']).not.toBe(null);
+    }
+  );
+});

package/src/databases/all/nomodel.spec.ts

@@ -652,26 +652,6 @@ runtimes.runtimeMap.forEach((runtime, databaseName) => {
     });
   });
 
-  test.when(runtime.supportsNesting)(
-    `number as null 2 - ${databaseName}`,
-    async () => {
-      // a cross join produces a Many to Many result.
-      // symmetric aggregate are needed on both sides of the join
-      // Check the row count and that sums on each side work properly.
-      const testModel = wrapTestModel(runtime, '');
-      await expect(`
-        # test.verbose
-        run: ${databaseName}.table('malloytest.state_facts') -> {
-          group_by: state
-          nest: ugly is {
-            group_by: popular_name
-            aggregate: foo is NULLIF(sum(airport_count)*0,0)+1
-          }
-        }
-      `).toMatchPaths(testModel, {'ugly.foo': null});
-    }
-  );
-
   it(`sql block- ${databaseName}`, async () => {
     const testModel = wrapTestModel(runtime, '');
     await expect(`
@@ -1279,39 +1259,6 @@ SELECT row_to_json(finalStage) as row FROM __stage0 AS finalStage`);
       `).toMatchResult(testModel, {p: 23001594});
   });
 
-  test.when(runtime.supportsNesting)(
-    `nest null - ${databaseName}`,
-    async () => {
-      const result = await runtime
-        .loadQuery(
-          `
-        run: ${databaseName}.table('malloytest.airports') -> {
-          where: faa_region is null
-          group_by: faa_region
-          aggregate: airport_count is count()
-          nest: by_state is {
-            where: state is not null
-            group_by: state
-            aggregate: airport_count is count()
-          }
-          nest: by_state1 is {
-            where: state is not null
-            group_by: state
-            aggregate: airport_count is count()
-            limit: 1
-          }
-        }
-      `
-        )
-        .run();
-
-      // eslint-disable-next-line @typescript-eslint/no-explicit-any
-      const d: any = result.data.toObject();
-      expect(d[0]['by_state']).not.toBe(null);
-      expect(d[0]['by_state1']).not.toBe(null);
-    }
-  );
-
   test.when(
     runtime.supportsNesting && runtime.dialect.supportsPipelinesInViews
   )(`Nested pipelines sort properly - ${databaseName}`, async () => {
@@ -1391,28 +1338,6 @@ SELECT row_to_json(finalStage) as row FROM __stage0 AS finalStage`);
     }
   });
 
-  test.when(runtime.supportsNesting)(
-    `number as null- ${databaseName}`,
-    async () => {
-      const result = await runtime
-        .loadQuery(
-          `
-        source: s is ${databaseName}.table('malloytest.state_facts') extend {
-        }
-        run: s-> {
-          group_by: state
-          nest: ugly is {
-            group_by: popular_name
-            aggregate: foo is NULLIF(sum(airport_count)*0,0)+1
-          }
-        }
-      `
-        )
-        .run();
-      expect(result.data.path(0, 'ugly', 0, 'foo').value).toBe(null);
-    }
-  );
-
   it(`removes surpuflous order_by - solo aggregates - ${databaseName}`, async () => {
     const result = await runtime
       .loadQuery(

package/src/databases/all/problems.spec.ts

@@ -36,11 +36,43 @@ runtimes.runtimeMap.forEach((runtime, databaseName) => {
             aggregate: \`#\` is count(one)
           }
         }
+        // A nest of nothing but aggregates is a single record rather than a
+        // list, and is built by a different dialect method.
+        nest: measures_only is {
+          aggregate: \`#\` is count(one)
+          aggregate: \`order\` is one.sum()
+        }
       }`).toMatchResult(testModel, {
       foo: [{'one': 1, '#': 1, 'deepfoo': [{'one': 1, '#': 1}]}],
+      measures_only: {'#': 1, 'order': 1},
     });
   });
 
+  // A nest's output field names are the user's, so every place a stage
+  // declares or reads them has to quote them -- including the row type a
+  // multi-stage nest builds to carry its first stage into its second.
+  test.when(
+    runtime.supportsNesting && runtime.dialect.supportsPipelinesInViews
+  )(
+    `properly quotes nested field names through a second stage in ${databaseName}`,
+    async () => {
+      await expect(`
+      run: ${databaseName}.table('malloytest.state_facts') -> {
+        where: state = 'CA'
+        group_by: popular_name
+        nest: counted is {
+          group_by: \`order\` is state
+          aggregate: n is count()
+        } -> {
+          group_by: \`order\`
+          aggregate: t is n.sum()
+        }
+      }`).toMatchResult(testModel, {
+        counted: [{'order': 'CA', 't': 1}],
+      });
+    }
+  );
+
   describe('warnings', () => {
     // NOTE: This test generates SQL errors on the console because of
     // a hard-coded console.log() in the duckdb-wasm worker

package/src/runtimes.ts

@@ -207,6 +207,7 @@ export function runtimeFor(dbName: string): SingleConnectionRuntime {
           name: dbName,
           additionalExtensions: ['mssql'],
           setupSQL: [
+            // Must match test/mssql/connection_string.ts
             "ATTACH 'Server=localhost;Port=1433;Database=malloytest;User Id=sa;Password=Malloy_Test_123;TrustServerCertificate=true' AS msdb (TYPE mssql)",
             'USE msdb.malloytest',
           ].join(';\n'),

package/trino/hive.properties

@@ -0,0 +1,11 @@
+# The hive catalog serves test/data/malloytest-parquet, mounted read-only
+# under /data/malloytest, with the tables hive_ddl.ts declares. The file
+# metastore lives inside the container's own data directory, which the trino
+# user can write; nothing but the parquet crosses the container boundary.
+connector.name=hive
+hive.metastore=file
+hive.metastore.catalog.dir=local:///trino/metastore
+fs.native-local.enabled=true
+local.location=/data
+# The parquet timestamps are microseconds; the DDL declares TIMESTAMP(6)
+hive.timestamp-precision=MICROSECONDS