@malloydata/malloy-tests 0.0.149-dev240706223116 → 0.0.150-dev240714131913

package/package.json

@@ -21,13 +21,13 @@
   },
   "dependencies": {
     "@jest/globals": "^29.4.3",
-    "@malloydata/db-bigquery": "^0.0.149-dev240706223116",
-    "@malloydata/db-duckdb": "^0.0.149-dev240706223116",
-    "@malloydata/db-postgres": "^0.0.149-dev240706223116",
-    "@malloydata/db-snowflake": "^0.0.149-dev240706223116",
-    "@malloydata/db-trino": "^0.0.149-dev240706223116",
-    "@malloydata/malloy": "^0.0.149-dev240706223116",
-    "@malloydata/render": "^0.0.149-dev240706223116",
+    "@malloydata/db-bigquery": "^0.0.150-dev240714131913",
+    "@malloydata/db-duckdb": "^0.0.150-dev240714131913",
+    "@malloydata/db-postgres": "^0.0.150-dev240714131913",
+    "@malloydata/db-snowflake": "^0.0.150-dev240714131913",
+    "@malloydata/db-trino": "^0.0.150-dev240714131913",
+    "@malloydata/malloy": "^0.0.150-dev240714131913",
+    "@malloydata/render": "^0.0.150-dev240714131913",
     "jsdom": "^22.1.0",
     "luxon": "^2.4.0",
     "madge": "^6.0.0"
@@ -36,5 +36,5 @@
     "@types/jsdom": "^21.1.1",
     "@types/luxon": "^2.4.0"
   },
-  "version": "0.0.149-dev240706223116"
+  "version": "0.0.150-dev240714131913"
 }

package/src/databases/all/db_index.spec.ts

@@ -97,19 +97,20 @@ runtimes.runtimeMap.forEach((runtime, databaseName) => {
   });
 
   // bigquery doesn't support row count based sampling.
-  test.when(databaseName !== 'bigquery' && databaseName !== 'trino')(
-    `index rows count - ${databaseName}`,
-    async () => {
-      await expect(`
+  test.when(
+    databaseName !== 'bigquery' &&
+      databaseName !== 'trino' &&
+      databaseName !== 'presto'
+  )(`index rows count - ${databaseName}`, async () => {
+    await expect(`
         run: ${databaseName}.table('malloytest.state_facts') extend {
           dimension: one is 'one'
         } -> {index:one, state; sample: 10 }
             -> {select: fieldName, weight, fieldValue; order_by: 2 desc; where: fieldName = 'one'}
       `).malloyResultMatches(runtime, {fieldName: 'one', weight: 10});
-    }
-  );
+  });
 
-  it.when(databaseName !== 'trino')(
+  it.when(databaseName !== 'trino' && databaseName !== 'presto')(
     `index rows count - ${databaseName}`,
     async () => {
       await expect(`

package/src/databases/all/functions.spec.ts

@@ -112,32 +112,32 @@ expressionModels.forEach((expressionModel, databaseName) => {
   };
 
   describe('concat', () => {
-    it.when(!brokenIn('trino', databaseName) /* crswenson */)(
-      `works - ${databaseName}`,
-      async () => {
-        const expected = {
-          'bigquery': 'foo2003-01-01 12:00:00+00',
-          'snowflake': 'foo2003-01-01T12:00:00.000Z',
-        };
-
-        await funcTestMultiple(
-          ["concat('foo', 'bar')", 'foobar'],
-          ["concat(1, 'bar')", '1bar'],
-          [
-            "concat('cons', true)",
-            databaseName === 'postgres' ? 'const' : 'construe',
-          ],
-          ["concat('foo', @2003)", 'foo2003-01-01'],
-          [
-            "concat('foo', @2003-01-01 12:00:00)",
-            expected[databaseName] ?? 'foo2003-01-01 12:00:00',
-          ],
-          // TODO Maybe implement consistent null behavior
-          // ["concat('foo', null)", null],
-          ['concat()', '']
-        );
-      }
-    );
+    it.when(
+      !brokenIn('trino', databaseName) &&
+        !brokenIn('presto', databaseName) /* crswenson */
+    )(`works - ${databaseName}`, async () => {
+      const expected = {
+        'bigquery': 'foo2003-01-01 12:00:00+00',
+        'snowflake': 'foo2003-01-01T12:00:00.000Z',
+      };
+
+      await funcTestMultiple(
+        ["concat('foo', 'bar')", 'foobar'],
+        ["concat(1, 'bar')", '1bar'],
+        [
+          "concat('cons', true)",
+          databaseName === 'postgres' ? 'const' : 'construe',
+        ],
+        ["concat('foo', @2003)", 'foo2003-01-01'],
+        [
+          "concat('foo', @2003-01-01 12:00:00)",
+          expected[databaseName] ?? 'foo2003-01-01 12:00:00',
+        ],
+        // TODO Maybe implement consistent null behavior
+        // ["concat('foo', null)", null],
+        ['concat()', '']
+      );
+    });
   });
 
   describe('round', () => {
@@ -219,13 +219,15 @@ expressionModels.forEach((expressionModel, databaseName) => {
           "replace('axbxc', r'(a).(b).(c)', '\\\\0 - \\\\1 - \\\\2 - \\\\3')",
           databaseName === 'postgres'
             ? '\\0 - a - b - c'
-            : databaseName === 'trino'
+            : databaseName === 'trino' || databaseName === 'presto'
             ? '0 - 1 - 2 - 3'
             : 'axbxc - a - b - c',
         ],
         [
           "replace('aaaa', '', 'c')",
-          databaseName === 'trino' ? 'cacacacac' : 'aaaa',
+          databaseName === 'trino' || databaseName === 'presto'
+            ? 'cacacacac'
+            : 'aaaa',
         ],
         ["replace(null, 'a', 'c')", null],
         ["replace('aaaa', null, 'c')", null],
@@ -262,7 +264,8 @@ expressionModels.forEach((expressionModel, databaseName) => {
 
   describe('stddev', () => {
     // TODO symmetric aggregates don't work with custom aggregate functions in BQ currently
-    if (['bigquery', 'snowflake', 'trino'].includes(databaseName)) return;
+    if (['bigquery', 'snowflake', 'trino', 'presto'].includes(databaseName))
+      return;
     it(`works - ${databaseName}`, async () => {
       await funcTestAgg('round(stddev(aircraft_models.seats))', 29);
     });
@@ -786,7 +789,7 @@ expressionModels.forEach((expressionModel, databaseName) => {
     });
   });
   describe('is_inf', () => {
-    const inf = ['trino'].includes(databaseName)
+    const inf = ['trino', 'presto'].includes(databaseName)
       ? 'infinity!()'
       : "'+inf'::number";
     it(`works - ${databaseName}`, async () => {
@@ -1327,7 +1330,8 @@ describe.each(runtimes.runtimeList)('%s', (databaseName, runtime) => {
 
     it(`works with fanout and order_by - ${databaseName}`, async () => {
       // TODO bigquery cannot handle both fanout and order_by today
-      if (['bigquery', 'snowflake', 'trino'].includes(databaseName)) return;
+      if (['bigquery', 'snowflake', 'trino', 'presto'].includes(databaseName))
+        return;
       await expect(`##! experimental.aggregate_order_by
       run: state_facts extend { join_many:
         state_facts2 is ${databaseName}.table('malloytest.state_facts')

package/src/databases/all/join.spec.ts

@@ -194,10 +194,10 @@ describe('join expression tests', () => {
     `).malloyResultMatches(joinModel, {f_sum2: 60462});
     });
 
-    test.when(runtime.supportsNesting)(
-      `model: unnest is left join - ${database}`,
-      async () => {
-        await expect(`
+    test.when(
+      runtime.supportsNesting && runtime.dialect.supportsLeftJoinUnnest
+    )(`model: unnest is left join - ${database}`, async () => {
+      await expect(`
           // produce a table with 4 rows that has a nested element
           query: a_states is ${database}.table('malloytest.state_facts')-> {
           where: state ? ~ 'A%'
@@ -220,8 +220,7 @@ describe('join expression tests', () => {
             limit: 5
           }
         `).malloyResultMatches(joinModel, [{}, {}, {}, {}, {}]);
-      }
-    );
+    });
 
     // not sure how to solve this one yet, just check for > 4 rows
     it(`All joins at the same level - ${database}`, async () => {

package/src/databases/all/nomodel.spec.ts

@@ -52,6 +52,8 @@ function getSplitFunction(db: string) {
       `split(${column}, '${splitChar}')`,
     'trino': (column: string, splitChar: string) =>
       `split(${column}, '${splitChar}')`,
+    'presto': (column: string, splitChar: string) =>
+      `split(${column}, '${splitChar}')`,
   }[db];
 }
 
@@ -480,11 +482,13 @@ runtimes.runtimeMap.forEach((runtime, databaseName) => {
     expect(q.sql.toLowerCase()).not.toContain('distinct');
   });
 
-  it(`leafy nested count - ${databaseName}`, async () => {
-    // in a joined table when the joined is leafiest
-    //  we need to make sure we don't count rows that
-    //  don't match the join.
-    await expect(`
+  it.when(runtime.dialect.supportsLeftJoinUnnest)(
+    `leafy nested count - ${databaseName}`,
+    async () => {
+      // in a joined table when the joined is leafiest
+      //  we need to make sure we don't count rows that
+      //  don't match the join.
+      await expect(`
       source: am_states is ${databaseName}.table('malloytest.state_facts') -> {
         group_by: state,popular_name
         where: state ~ r'^(A|M)'
@@ -496,7 +500,6 @@ runtimes.runtimeMap.forEach((runtime, databaseName) => {
       source: states is ${databaseName}.table('malloytest.state_facts') extend {
         join_many: am_states on state=am_states.state
       }
-
       run: states -> {
         where: state = 'CA'
         group_by:
@@ -507,12 +510,13 @@ runtimes.runtimeMap.forEach((runtime, databaseName) => {
           root_count is count()
       }
       `).malloyResultMatches(runtime, {
-      leafy_count: 0,
-      root_count: 1,
-      state: 'CA',
-      am_state: null,
-    });
-  });
+        leafy_count: 0,
+        root_count: 1,
+        state: 'CA',
+        am_state: null,
+      });
+    }
+  );
 
   it(`basic index - ${databaseName}`, async () => {
     // Make sure basic indexing works.
@@ -548,6 +552,16 @@ runtimes.runtimeMap.forEach((runtime, databaseName) => {
     }
   );
 
+  it(`sql block- ${databaseName}`, async () => {
+    await expect(`
+      source: one is ${databaseName}.sql("""
+        SELECT 2 as ${q`a`}
+      """)
+      run: one -> {
+        select: a
+      }`).malloyResultMatches(runtime, {a: 2});
+  });
+
   // average should only include non-null values in the denominator
   it(`avg ignore null- ${databaseName}`, async () => {
     await expect(`
@@ -628,6 +642,7 @@ runtimes.runtimeMap.forEach((runtime, databaseName) => {
     'ungrouped nested with no grouping above - ${databaseName}',
     async () => {
       await expect(`
+        // # test.debug
         run: ${databaseName}.table('malloytest.state_facts') extend {
           measure: total_births is births.sum()
           measure: births_per_100k is floor(total_births/ all(total_births) * 100000)
@@ -1041,11 +1056,23 @@ SELECT row_to_json(finalStage) as row FROM __stage0 AS finalStage`);
   );
 
   test.when(runtime.supportsNesting && runtime.dialect.readsNestedData)(
-    `can unnest from file - ${databaseName}`,
+    `can unnest simply from file - ${databaseName}`,
     async () => {
       await expect(`
         source: ga_sample is ${databaseName}.table('malloytest.ga_sample')
+        run: ga_sample -> {
+          aggregate:
+            h is hits.count()
+        }
+      `).malloyResultMatches(runtime, {h: 13233});
+    }
+  );
 
+  test.when(runtime.supportsNesting && runtime.dialect.readsNestedData)(
+    `can unnest from file - ${databaseName}`,
+    async () => {
+      await expect(`
+        source: ga_sample is ${databaseName}.table('malloytest.ga_sample')
         run: ga_sample -> {
           where: hits.product.productBrand != null
           group_by:

package/src/databases/all/time.spec.ts

@@ -462,18 +462,17 @@ describe.each(runtimes.runtimeList)('%s date and time', (dbName, runtime) => {
     });
   });
 
-  test.when(!brokenIn('trino', dbName) /* mtoy */)(
-    'dependant join dialect fragments',
-    async () => {
-      await expect(`
+  test.when(
+    !brokenIn('trino', dbName) && !brokenIn('presto', dbName) /* mtoy */
+  )('dependant join dialect fragments', async () => {
+    await expect(`
       source: timeData is ${dbName}.sql("""${timeSQL}""")
       run: timeData -> {
         extend: {join_one: joined is timeData on t_date = joined.t_date}
         group_by: t_month is joined.t_timestamp.month
       }
     `).malloyResultMatches(runtime, {t_month: new Date('2021-02-01')});
-    }
-  );
+  });
 
   describe('timezone set correctly', () => {
     test('timezone set in source used by query', async () => {
@@ -621,63 +620,60 @@ const utc_2020 = LuxonDateTime.fromObject(
 );
 
 describe.each(runtimes.runtimeList)('%s: tz literals', (dbName, runtime) => {
-  test.when(!brokenIn('trino', dbName) /* mtoy */)(
-    `${dbName} - default timezone is UTC`,
-    async () => {
-      // this makes sure that the tests which use the test timezome are actually
-      // testing something ... file this under "abundance of caution". It
-      // really tests nothing, but I feel calmer with this here.
-      const query = runtime.loadQuery(
-        `
+  test.when(
+    !brokenIn('trino', dbName) && !brokenIn('presto', dbName) /* mtoy */
+  )(`${dbName} - default timezone is UTC`, async () => {
+    // this makes sure that the tests which use the test timezome are actually
+    // testing something ... file this under "abundance of caution". It
+    // really tests nothing, but I feel calmer with this here.
+    const query = runtime.loadQuery(
+      `
         run: ${dbName}.sql("SELECT 1 as one") -> {
           group_by: literal_time is @2020-02-20 00:00:00
         }
 `
-      );
-      const result = await query.run();
-      const literal = result.data.path(0, 'literal_time').value as Date;
-      const have = LuxonDateTime.fromJSDate(literal);
-      expect(have.valueOf()).toEqual(utc_2020.valueOf());
-    }
-  );
-
-  test.when(!brokenIn('trino', dbName) /* mtoy */)(
-    'literal with zone name',
-    async () => {
-      const query = runtime.loadQuery(
-        `
+    );
+    const result = await query.run();
+    const literal = result.data.path(0, 'literal_time').value as Date;
+    const have = LuxonDateTime.fromJSDate(literal);
+    expect(have.valueOf()).toEqual(utc_2020.valueOf());
+  });
+
+  test.when(
+    !brokenIn('trino', dbName) && !brokenIn('presto', dbName) /* mtoy */
+  )('literal with zone name', async () => {
+    const query = runtime.loadQuery(
+      `
         run: ${dbName}.sql("SELECT 1 as one") -> {
           group_by: literal_time is @2020-02-20 00:00:00[America/Mexico_City]
         }
 `
-      );
-      const result = await query.run();
-      const literal = result.data.path(0, 'literal_time').value as Date;
-      const have = LuxonDateTime.fromJSDate(literal);
-      expect(have.valueOf()).toEqual(zone_2020.valueOf());
-    }
-  );
+    );
+    const result = await query.run();
+    const literal = result.data.path(0, 'literal_time').value as Date;
+    const have = LuxonDateTime.fromJSDate(literal);
+    expect(have.valueOf()).toEqual(zone_2020.valueOf());
+  });
 });
 
 describe.each(runtimes.runtimeList)('%s: query tz', (dbName, runtime) => {
   const q = runtime.getQuoter();
-  test.when(!brokenIn('trino', dbName) /* mtoy */)(
-    'literal timestamps',
-    async () => {
-      const query = runtime.loadQuery(
-        `
+  test.when(
+    !brokenIn('trino', dbName) && !brokenIn('presto', dbName) /* mtoy */
+  )('literal timestamps', async () => {
+    const query = runtime.loadQuery(
+      `
         run: ${dbName}.sql("SELECT 1 as one") -> {
           timezone: '${zone}'
           group_by: literal_time is @2020-02-20 00:00:00
         }
       `
-      );
-      const result = await query.run();
-      const literal = result.data.path(0, 'literal_time').value as Date;
-      const have = LuxonDateTime.fromJSDate(literal);
-      expect(have.valueOf()).toEqual(zone_2020.valueOf());
-    }
-  );
+    );
+    const result = await query.run();
+    const literal = result.data.path(0, 'literal_time').value as Date;
+    const have = LuxonDateTime.fromJSDate(literal);
+    expect(have.valueOf()).toEqual(zone_2020.valueOf());
+  });
 
   test('extract', async () => {
     await expect(
@@ -691,7 +687,9 @@ describe.each(runtimes.runtimeList)('%s: query tz', (dbName, runtime) => {
     ).malloyResultMatches(runtime, {mex_midnight: 18, mex_day: 19});
   });
 
-  test.when(!brokenIn('trino', dbName) /* mtoy */)('truncate day', async () => {
+  test.when(
+    !brokenIn('trino', dbName) && !brokenIn('presto', dbName) /* mtoy */
+  )('truncate day', async () => {
     // At midnight in london it the 19th in Mexico, so that truncates to
     // midnight on the 19th
     const mex_19 = LuxonDateTime.fromISO('2020-02-19T00:00:00', {zone});
@@ -704,32 +702,30 @@ describe.each(runtimes.runtimeList)('%s: query tz', (dbName, runtime) => {
     ).malloyResultMatches(runtime, {mex_day: mex_19.toJSDate()});
   });
 
-  test.when(!brokenIn('trino', dbName) /* mtoy */)(
-    'cast timestamp to date',
-    async () => {
-      // At midnight in london it is the 19th in Mexico, so when we cast that
-      // to a date, it should be the 19th.
-      await expect(
-        `run: ${dbName}.sql("SELECT 1 as x") -> {
+  test.when(
+    !brokenIn('trino', dbName) && !brokenIn('presto', dbName) /* mtoy */
+  )('cast timestamp to date', async () => {
+    // At midnight in london it is the 19th in Mexico, so when we cast that
+    // to a date, it should be the 19th.
+    await expect(
+      `run: ${dbName}.sql("SELECT 1 as x") -> {
         timezone: '${zone}'
         extend: { dimension: utc_midnight is @2020-02-20 00:00:00[UTC] }
         select: mex_day is day(utc_midnight::date)
       }`
-      ).malloyResultMatches(runtime, {mex_day: 19});
-    }
-  );
-
-  test.when(!brokenIn('trino', dbName) /* mtoy */)(
-    'cast date to timestamp',
-    async () => {
-      await expect(
-        `run: ${dbName}.sql(""" SELECT DATE '2020-02-20'  AS ${q`mex_20`} """) -> {
+    ).malloyResultMatches(runtime, {mex_day: 19});
+  });
+
+  test.when(
+    !brokenIn('trino', dbName) && !brokenIn('presto', dbName) /* mtoy */
+  )('cast date to timestamp', async () => {
+    await expect(
+      `run: ${dbName}.sql(""" SELECT DATE '2020-02-20'  AS ${q`mex_20`} """) -> {
         timezone: '${zone}'
         select: mex_ts is mex_20::timestamp
       }`
-      ).malloyResultMatches(runtime, {mex_ts: zone_2020.toJSDate()});
-    }
-  );
+    ).malloyResultMatches(runtime, {mex_ts: zone_2020.toJSDate()});
+  });
 });
 
 afterAll(async () => {

package/src/runtimes.ts

@@ -37,6 +37,7 @@ import {SnowflakeConnection} from '@malloydata/db-snowflake';
 import {PooledPostgresConnection} from '@malloydata/db-postgres';
 import {TrinoConnection, TrinoExecutor} from '@malloydata/db-trino';
 import {SnowflakeExecutor} from '@malloydata/db-snowflake/src/snowflake_executor';
+import {PrestoConnection} from '@malloydata/db-trino/src/trino_connection';
 
 export class SnowflakeTestConnection extends SnowflakeConnection {
   public async runSQL(
@@ -166,7 +167,14 @@ export function runtimeFor(dbName: string): SingleConnectionRuntime {
         connection = new TrinoConnection(
           dbName,
           {},
-          TrinoExecutor.getConnectionOptionsFromEnv()
+          TrinoExecutor.getConnectionOptionsFromEnv(dbName)
+        );
+        break;
+      case 'presto':
+        connection = new PrestoConnection(
+          dbName,
+          {},
+          TrinoExecutor.getConnectionOptionsFromEnv(dbName) // they share configs.
         );
         break;
       default:

package/trino/trino_start.sh

@@ -12,7 +12,7 @@ bigquery.arrow-serialization.enabled=false
 EOF
 
 # run docker
-docker run -p 8090:8080 -d -v ./.tmp/bigquery.properties:/etc/trino/catalog/bigquery.properties --name trino-malloy trinodb/trino
+docker run -p 8080:8080 -d -v ./.tmp/bigquery.properties:/etc/trino/catalog/bigquery.properties --name trino-malloy trinodb/trino
 
 # wait for server to start
 counter=0
@@ -21,7 +21,7 @@ do
   sleep 1
   counter=$((counter+1))
   # if doesn't start after 2 minutes, output logs and kill process
-  if [ $counter -eq 120 ]
+  if [ $counter -eq 300 ]
   then
     docker logs trino-malloy >& ./.tmp/trino-malloy.logs
     docker rm -f trino-malloy
@@ -31,4 +31,4 @@ do
   fi
 done
 
-echo "Trino running on port localhost:8090"
+echo "Trino running on port localhost:8080"