@malloydata/malloy-tests 0.0.95-dev231019211822 → 0.0.95

package/src/databases/bigquery-duckdb/nested_source_table.spec.ts

@@ -35,8 +35,9 @@ const [describe, databases] = describeIfDatabaseAvailable([
   'duckdb',
 ]);
 
-const modelText = `
-source:ga_sessions is table('malloytest.ga_sample'){
+function modelText(databaseName: string) {
+  return `
+source: ga_sessions is ${databaseName}.table('malloytest.ga_sample') extend {
 
   measure:
     user_count is count(fullVisitorId)
@@ -49,41 +50,41 @@ source:ga_sessions is table('malloytest.ga_sample'){
     hits_count is hits.count()
     sold_count is hits.count() { where: hits.product.productQuantity > 0 }
 
-  query: by_source is {
+  view: by_source is {
     where: trafficSource.\`source\` != '(direct)'
     group_by: trafficSource.\`source\`
     aggregate: hits_count
     limit: 10
   }
-  query: by_adContent_bar_chart is {
+  view: by_adContent_bar_chart is {
     group_by: device.browser
     aggregate: user_count
     group_by: device.deviceCategory
   }
-  query: by_region is {
+  view: by_region is {
     where: geoNetwork.region !~ '%demo%'
     group_by: geoNetwork.region
     aggregate: user_count
     limit: 10
   }
-  query: by_device is {
+  view: by_device is {
     group_by: device.browser
     aggregate: user_count
     group_by: device.deviceCategory
   }
-  query: by_category is {
+  view: by_category is {
     group_by: category is hits.product.v2ProductCategory
     aggregate: total_productRevenue
     aggregate: sold_count
     limit: 10
   }
-  query: by_hour_of_day is {
+  view: by_hour_of_day is {
     // group_by: gsession_hour is hour(start_time::timestamp)
     aggregate: session_count
     order_by: 1
   }
 
-  query: page_load_times is {
+  view: page_load_times is {
     group_by: hits.page.pageTitle
     aggregate: hit_count is hits.count()
     nest: load_bar_chart is {
@@ -93,20 +94,20 @@ source:ga_sessions is table('malloytest.ga_sample'){
     limit: 10
   }
 
-  query: by_page_title is { where: totals.transactionRevenue > 0
+  view: by_page_title is { where: totals.transactionRevenue > 0
     group_by: hits.page.pageTitle
     aggregate: hits_count
     aggregate: sold_count
   }
 
-  query: by_all is {
+  view: by_all is {
     nest: by_source
     nest: by_adContent_bar_chart
     nest: by_region
     nest: by_category
   }
 
-  query: search_index is {
+  view: search_index is {
     index: *, hits.*, customDimensions.* totals.*, trafficSource.*, hits.product.*
     sample: 1%
   }
@@ -124,26 +125,21 @@ query: sessions_dashboard is ga_sessions -> {
     }
 }
 `;
+}
 
 const runtimes = new RuntimeList(databases);
 describe.each(runtimes.runtimeList)(
   'Nested Source Table - %s',
   (databaseName, runtime) => {
+    const gaModel = runtime.loadModel(modelText(databaseName));
     test(`repeated child of record - ${databaseName}`, async () => {
-      const result = await runtime
-        .loadModel(modelText)
-        .loadQuery(
-          `
-        query: ga_sessions->by_page_title
-        `
-        )
-        .run();
-      // console.log(result.data.toObject());
-      // console.log(result.sql);
-      expect(result.data.path(0, 'pageTitle').value).toBe('Shopping Cart');
+      await expect('run: ga_sessions->by_page_title').malloyResultMatches(
+        gaModel,
+        {pageTitle: 'Shopping Cart'}
+      );
     });
 
-    // SKIPPED because the tests intermittently fail and lloyd said
+    // Tests intermittently fail and lloyd said
     // "I bet it has to do with my sampling test on indexing. Intermittently
     //  getting different results. I'd comment out the test and I'll take a
     //  look at it when I get back." and that seems reasonable.
@@ -155,35 +151,28 @@ describe.each(runtimes.runtimeList)(
     //
     //      Expected: "Organic Search"
     //      Received: "Referral"
-    test.skip(`search_index - ${databaseName}`, async () => {
-      const result = await runtime
-        .loadModel(modelText)
-        .loadQuery(
-          `
-        query: ga_sessions->search_index -> {
+    test(`search_index - ${databaseName}`, async () => {
+      await expect(`
+        run: ga_sessions->search_index -> {
           where: fieldName != null
           select: *
           order_by: fieldName, weight desc
           limit: 10
         }
-        `
-        )
-        .run();
-      // console.log(result.data.toObject());
-      expect(result.data.path(0, 'fieldName').value).toBe('channelGrouping');
-      expect(result.data.path(0, 'fieldValue').value).toBe('Organic Search');
-      // expect(result.data.path(0, "weight").value).toBe(18);
+      `).malloyResultMatches(gaModel, {
+        fieldName: 'channelGrouping',
+        // fieldValue: 'Organic Search',
+        // weight: 10,
+      });
     });
 
-    test.skip(`manual index - ${databaseName}`, async () => {
+    test(`manual index - ${databaseName}`, async () => {
       let sampleSize = '10';
       if (databaseName === 'bigquery') {
         sampleSize = 'false';
       }
-      const _result = await runtime
-        .loadQuery(
-          `
-        query: table('malloytest.ga_sample')-> {
+      await expect(`
+        run: ${databaseName}.table('malloytest.ga_sample')-> {
           index: everything
           sample: ${sampleSize}
         }
@@ -195,70 +184,39 @@ describe.each(runtimes.runtimeList)(
             limit: 100
           }
         }
-        `
-        )
-        .run();
-      // console.log(JSON.stringify(result.data.toObject(), null, 2));
-      // expect(result.data.path(0, "fieldName").value).toBe("channelGrouping");
-      // expect(result.data.path(0, "fieldValue").value).toBe("Organic Search");
-      // expect(result.data.path(0, "weight").value).toBe(18);
+      `).malloyResultMatches(runtime, {
+        // 'top_fields.fieldName': 'channelGrouping',
+        // 'top_fields.fieldValue': 'Organic Search',
+        // 'top_fields.weight': 18,
+      });
     });
 
     test(`autobin - ${databaseName}`, async () => {
-      const _result = await runtime
-        .loadQuery(
-          `
-          source: airports is table('malloytest.airports') + {
-            measure:
-              airport_count is count()
-            query: by_elevation is {
-              aggregate: bin_size is NULLIF((max(elevation) - min(elevation))/30,0)
-              nest: data is {
-                group_by: elevation
-                aggregate: row_count is count()
-              }
-            }
-            -> {
-              group_by: elevation is floor(data.elevation/bin_size)*bin_size + bin_size/2
-              aggregate: airport_count is data.row_count.sum()
-              order_by: elevation
+      await expect(`
+        source: airports is ${databaseName}.table('malloytest.airports') extend {
+          measure: airport_count is count()
+          view: by_elevation is {
+            aggregate: bin_size is NULLIF((max(elevation) - min(elevation))/30,0)
+            nest: data is {
+              group_by: elevation
+              aggregate: row_count is count()
             }
           }
-
-          query: airports -> {
-            group_by: state is state
-            aggregate: airport_count
-            nest: by_elevation_bar_chart is by_elevation
+          -> {
+            group_by: elevation is floor(data.elevation/bin_size)*bin_size + bin_size/2
+            aggregate: airport_count is data.row_count.sum()
+            order_by: elevation
           }
-        `
-        )
-        .run();
-      // console.log(result.sql);
-      // console.log(result.data.toObject());
-      // expect(result.data.path(0, 'fieldName').value).toBe('channelGrouping');
-      // expect(result.data.path(0, 'fieldValue').value).toBe('Organic Search');
-      // expect(result.data.path(0, "weight").value).toBe(18);
-    });
-
-    // this really shouldn't be here, we need to be able to test dialect capiblities
-    //  from the connection.
-    test(`safe_cast - ${databaseName}`, async () => {
-      const result = await runtime
-        .loadQuery(
-          `
-        source: eone is  table('malloytest.airports') {}
+        }
 
-        query: eone -> {
-          select:
-            bad_date is '123':::date
-            bad_number is 'abc':::number
-            limit: 1
+        run: airports -> {
+          group_by: state is state
+          aggregate: airport_count
+          nest: by_elevation_bar_chart is by_elevation
         }
-        `
-        )
-        .run();
-      expect(result.data.value[0]['bad_date']).toBe(null);
-      expect(result.data.value[0]['bad_number']).toBe(null);
+      `).malloyResultMatches(runtime, {
+        // don't know what to expect ...
+      });
     });
   }
 );

package/src/databases/bigquery-postgres/multi_connection.spec.ts

@@ -66,40 +66,22 @@ describe('Multi-connection', () => {
   });
 
   const expressionModelText = `
-source: default_aircraft is table('malloytest.aircraft'){
-  measure: aircraft_count is count(tail_num)
-}
-
-source: bigquery_state_facts is table('malloytest.state_facts'){
+source: bigquery_state_facts is bigquery.table('malloytest.state_facts') extend {
   measure: state_count is count(state)+2
 }
 
-source: postgres_aircraft is table('postgres:malloytest.aircraft'){
+source: postgres_aircraft is postgres.table('malloytest.aircraft') extend {
   measure: aircraft_count is count(tail_num)+4
 }
 `;
 
   const expressionModel = runtime.loadModel(expressionModelText);
 
-  it('default query', async () => {
-    const result = await expressionModel
-      .loadQuery(
-        `
-      query: default_aircraft-> {
-        aggregate: aircraft_count
-      }
-    `
-      )
-      .run();
-    // console.log(result.sql);
-    expect(result.data.path(0, 'aircraft_count').value).toBe(3599);
-  });
-
   it('bigquery query', async () => {
     const result = await expressionModel
       .loadQuery(
         `
-      query: bigquery_state_facts-> {
+      run: bigquery_state_facts-> {
         aggregate: state_count
       }
     `
@@ -113,7 +95,7 @@ source: postgres_aircraft is table('postgres:malloytest.aircraft'){
     const result = await expressionModel
       .loadQuery(
         `
-      query: postgres_aircraft-> {
+      run: postgres_aircraft-> {
         aggregate: aircraft_count
       }
     `
@@ -126,7 +108,7 @@ source: postgres_aircraft is table('postgres:malloytest.aircraft'){
     const result = await runtime
       .loadQuery(
         `
-      query: table('postgres:malloytest.airports')->{
+      run: postgres.table('malloytest.airports')->{
         group_by:
           version is version!()
         aggregate:

package/src/databases/bigquery-postgres/streaming.spec.ts

@@ -57,8 +57,10 @@ describe('Streaming tests', () => {
   runtimes.runtimeMap.forEach((runtime, databaseName) => {
     it(`basic stream test  - ${databaseName}`, async () => {
       const stream = runtime
-        .loadModel("source: airports is table('malloytest.airports') {}")
-        .loadQuery('query: airports -> { select: code }')
+        .loadModel(
+          `source: airports is ${databaseName}.table('malloytest.airports')`
+        )
+        .loadQuery('run: airports -> { select: code }')
         .runStream({rowLimit: 10});
       const rows: DataRecord[] = [];
       for await (const row of stream) {
@@ -70,8 +72,10 @@ describe('Streaming tests', () => {
 
     it(`stream to JSON - ${databaseName}`, async () => {
       const stream = runtime
-        .loadModel("source: airports is table('malloytest.airports') {}")
-        .loadQuery('query: airports -> { select: code }')
+        .loadModel(
+          `source: airports is ${databaseName}.table('malloytest.airports')`
+        )
+        .loadQuery('run: airports -> { select: code }')
         .runStream({rowLimit: 1});
       const accummulator = new StringAccumulator();
       const jsonWriter = new JSONWriter(accummulator);
@@ -88,8 +92,10 @@ describe('Streaming tests', () => {
 
     it(`stream to CSV - ${databaseName}`, async () => {
       const stream = runtime
-        .loadModel("source: airports is table('malloytest.airports') {}")
-        .loadQuery('query: airports -> { select: code }')
+        .loadModel(
+          `source: airports is ${databaseName}.table('malloytest.airports')`
+        )
+        .loadQuery('run: airports -> { select: code }')
         .runStream({rowLimit: 1});
       const accummulator = new StringAccumulator();
       const csvWriter = new CSVWriter(accummulator);

package/src/databases/duckdb/duckdb.spec.ts

@@ -26,6 +26,7 @@ import {RuntimeList} from '../../runtimes';
 import '../../util/db-jest-matchers';
 import {describeIfDatabaseAvailable} from '../../util';
 
+// TODO identify which tests need to run on wasm and move them into their own file
 const runtimes = ['duckdb', 'duckdb_wasm'];
 
 const [_describe, databases] = describeIfDatabaseAvailable(runtimes);
@@ -33,17 +34,12 @@ const allDucks = new RuntimeList(databases);
 
 describe.each(allDucks.runtimeList)('duckdb:%s', (dbName, runtime) => {
   it('can open tables with wildcards', async () => {
-    const result = await runtime
-      .loadQuery(
-        `
-        query: table('duckdb:test/data/duckdb/flights/part.*.parquet') -> {
-          top: 1
-          group_by: carrier;
-        }
-      `
-      )
-      .run();
-    expect(result.data.path(0, 'carrier').value).toEqual('AA');
+    await expect(`
+      run: duckdb.table('test/data/duckdb/flights/part.*.parquet') -> {
+        top: 1
+        group_by: carrier;
+      }
+    `).malloyResultMatches(runtime, {carrier: 'AA'});
   });
 
   it('accepts all schema numbers', async () => {
@@ -60,36 +56,32 @@ describe.each(allDucks.runtimeList)('duckdb:%s', (dbName, runtime) => {
     ];
     const allFields = allInts.map(intType => `a${intType.toLowerCase()}`);
     const query = `
-      sql: allInts is { connection: "${dbName}" select: """
+      run: ${dbName}.sql("""
         SELECT
         ${allInts
           .map(intType => `1::${intType} as a${intType.toLowerCase()}`)
           .join(',\n')}
-      """}
-      query: from_sql(allInts) -> {
+      """) -> {
         aggregate:
         ${allFields
           .map(fieldType => `sum_${fieldType} is sum(${fieldType})`)
           .join('\n')}
       }
     `;
-    const result = await runtime.loadQuery(query).run();
-    for (const fieldType of allFields) {
-      expect(result.data.path(0, `sum_${fieldType}`).value).toEqual(1);
-    }
+    await expect(query).malloyResultMatches(
+      runtime,
+      allInts.reduce<Record<string, number>>((building, ent) => {
+        building[`sum_a${ent.toLowerCase()}`] = 1;
+        return building;
+      }, {})
+    );
   });
 
   it('can open json files', async () => {
-    const result = await runtime
-      .loadQuery(
-        `
-          query: table('duckdb:test/data/duckdb/test.json') -> {
-            select: *
-          }
-        `
-      )
-      .run();
-    expect(result.data.path(0, 'foo').value).toEqual('bar');
+    await expect(`
+      run: duckdb.table('test/data/duckdb/test.json') -> {
+        select: *
+      }`).malloyResultMatches(runtime, {foo: 'bar'});
   });
 
   it('supports timezones', async () => {
@@ -101,10 +93,9 @@ describe.each(allDucks.runtimeList)('duckdb:%s', (dbName, runtime) => {
   });
 
   it('supports varchars with parameters', async () => {
-    await expect(runtime).queryMatches(
-      "run: duckdb.sql(\"SELECT 'a'::VARCHAR as abc, 'a3'::VARCHAR(3) as abc3\")",
-      {abc: 'a', abc3: 'a3'}
-    );
+    await expect(
+      "run: duckdb.sql(\"SELECT 'a'::VARCHAR as abc, 'a3'::VARCHAR(3) as abc3\")"
+    ).malloyResultMatches(runtime, {abc: 'a', abc3: 'a3'});
   });
 
   describe('time', () => {
@@ -118,17 +109,14 @@ describe.each(allDucks.runtimeList)('duckdb:%s', (dbName, runtime) => {
       second: 0,
       zone,
     });
-    // TODO: Investigate why its no working.
-    test.skip('can use unsupported types', async () => {
-      await expect(runtime).queryMatches(
-        `sql: timeData is { connection: "duckdb"  select: """
-          SELECT TIMESTAMPTZ '2020-02-20 00:00:00 ${zone}' as t_tstz
-        """}
-      query: from_sql(timeData) -> {
-        select: mex_220 is t_tstz::timestamp
-      }`,
-        {mex_220: zone_2020.toJSDate()}
-      );
+    test('can cast TIMESTAMPTZ to timestamp', async () => {
+      await expect(
+        `run: duckdb.sql("""
+              SELECT TIMESTAMPTZ '2020-02-20 00:00:00 ${zone}' as t_tstz
+          """) -> {
+            select: mex_220 is t_tstz::timestamp
+          }`
+      ).malloyResultMatches(runtime, {mex_220: zone_2020.toJSDate()});
     });
   });
 });

package/src/databases/postgres/postgres.spec.ts

@@ -23,7 +23,9 @@
 
 /* eslint-disable no-console */
 
+import {ok} from 'assert';
 import {RuntimeList} from '../../runtimes';
+import {Runtime} from '@malloydata/malloy';
 import {describeIfDatabaseAvailable} from '../../util';
 import '../../util/db-jest-matchers';
 import {DateTime} from 'luxon';
@@ -56,96 +58,67 @@ describe('Postgres tests', () => {
     await runtimeList.closeAll();
   });
 
-  it('sql_block', async () => {
-    const result = await runtime
-      .loadQuery(
-        `
-      sql: one is {select:"""
-        SELECT 1 as n
-       """}
-
-      query: from_sql(one) -> { select: n }
-      `
-      )
-      .run();
-    expect(result.data.value[0]['n']).toBe(1);
+  it('run an sql query', async () => {
+    await expect(
+      'run: postgres.sql("SELECT 1 as n") -> { select: n }'
+    ).malloyResultMatches(runtime, {n: 1});
   });
 
-  it('quote field name', async () => {
-    const result = await runtime
-      .loadQuery(
-        `
-      sql: one is {select:"""
-        SELECT 1 as "upperLower"
-       """}
-
-      query: from_sql(one) -> { select: upperLower }
-      `
-      )
-      .run();
-    expect(result.data.value[0]['upperLower']).toBe(1);
+  it('mixed case col names are properly quoted so they retain case in results', async () => {
+    await expect(`
+      run: postgres.sql('SELECT 1 as "upperLower"') -> { select: upperLower }
+    `).malloyResultMatches(runtime, {upperLower: 1});
   });
 
-  it('quote keyword', async () => {
-    const result = await runtime
-      .loadQuery(
-        `
-      sql: one is {select:"""
-        SELECT 1 as "select"
-       """}
-
-      query: from_sql(one) -> {
-        select:
-          select
-          create is select + 1
-      }
-      `
-      )
-      .run();
-    expect(result.data.value[0]['select']).toBe(1);
-    expect(result.data.value[0]['create']).toBe(2);
+  it('fields which are sql keywords are quoted', async () => {
+    await expect(`
+    run: postgres.sql('SELECT 1 as "select"') -> {
+      select:
+        select
+        create is select + 1
+    }
+  `).malloyResultMatches(runtime, {select: 1, create: 2});
   });
 
-  // these started failing and I'm not sure why (lloyd, skipping for now)
-  it.skip('quote table name', async () => {
-    const result = await runtime
-      .loadQuery(
-        `
-      query: table('public.UpperTablePublic') -> { select: one }
-      `
-      )
-      .run();
-    expect(result.data.value[0]['one']).toBe(1);
+  async function oneExists(rt: Runtime, tn: string): Promise<boolean> {
+    try {
+      const lookForOne = await rt
+        .loadQuery(`run: postgres.sql('SELECT one FROM ${tn}')`)
+        .run();
+      const one = lookForOne.data.path(0, 'one').value;
+      return one === 1;
+    } catch (e) {
+      return false;
+    }
+  }
+
+  it('will quote to properly access mixed case table name', async () => {
+    if (await oneExists(runtime, 'public."UpperTablePublic"')) {
+      await expect(`
+        run: postgres.table('public.UpperTablePublic') -> { select: one }
+      `).malloyResultMatches(runtime, {one: 1});
+    }
   });
 
-  it.skip('quote schema name', async () => {
-    const result = await runtime
-      .loadQuery(
-        `
-      query: table('UpperSchema.UpperSchemaUpperTable') -> { select: one }
-      `
-      )
-      .run();
-    expect(result.data.value[0]['one']).toBe(1);
+  it('quote to properly access mixes case schema name', async () => {
+    if (await oneExists(runtime, '"UpperSchema"."UpperSchemaUpperTable"')) {
+      await expect(`
+        run: postgres.table('UpperSchema.UpperSchemaUpperTable') -> { select: one }
+      `).malloyResultMatches(runtime, {one: 1});
+    }
   });
 
   it('passes unsupported data', async () => {
     const result = await runtime
-      .loadQuery(
-        `
-        sql: badType is { select: """SELECT int4range(10, 20) as ranger""" }
-        query: from_sql(badType)->{ select: *}
-      `
-      )
+      .loadQuery('run: postgres.sql("SELECT int4range(10, 20) as ranger")')
       .run();
     expect(result.data.value[0]['ranger']).toBeDefined();
   });
 
   it('supports varchars with parameters', async () => {
-    await expect(runtime).queryMatches(
-      "run: postgres.sql(\"SELECT 'a'::VARCHAR as abc, 'a3'::VARCHAR(3) as abc3\")",
-      {abc: 'a', abc3: 'a3'}
-    );
+    await expect(
+      "run: postgres.sql(\"SELECT 'a'::VARCHAR as abc, 'a3'::VARCHAR(3) as abc3\")"
+    ).malloyResultMatches(runtime, {abc: 'a', abc3: 'a3'});
   });
 
   describe('time', () => {
@@ -159,17 +132,14 @@ describe('Postgres tests', () => {
       second: 0,
       zone,
     });
-    // TODO: Investigate why this test is not working.
-    test.skip('can use unsupported types', async () => {
-      await expect(runtime).queryMatches(
-        `sql: timeData is { connection: "postgres"  select: """
-          SELECT TIMESTAMPTZ '2020-02-20 00:00:00 ${zone}' as t_tstz
-        """}
-      query: from_sql(timeData) -> {
-        select: mex_220 is t_tstz::timestamp
-      }`,
-        {mex_220: zone_2020.toJSDate()}
-      );
+    test('can cast TIMESTAMPTZ to timestamp', async () => {
+      await expect(
+        `run: duckdb.sql("""
+              SELECT TIMESTAMPTZ '2020-02-20 00:00:00 ${zone}' as t_tstz
+          """) -> {
+            select: mex_220 is t_tstz::timestamp
+          }`
+      ).malloyResultMatches(runtime, {mex_220: zone_2020.toJSDate()});
     });
   });
 });