@malloydata/malloy-tests 0.0.66-dev230807232724

package/src/databases/bigquery/performance.skipped.spec.ts

@@ -0,0 +1,73 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files
+ * (the "Software"), to deal in the Software without restriction,
+ * including without limitation the rights to use, copy, modify, merge,
+ * publish, distribute, sublicense, and/or sell copies of the Software,
+ * and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+import * as malloy from '@malloydata/malloy';
+import {RuntimeList} from '../../runtimes';
+
+// Chris, please rework.
+it.skip('accessors are not too expensive', async () => {
+  // If this test fails, consideration should be given to how much time using
+  // the accessors costs.
+  const runtime = new RuntimeList(['bigquery']).runtimeMap.get('bigquery');
+
+  expect(runtime).toBeDefined();
+  if (runtime) {
+    const result = await runtime
+      .loadQuery(
+        "source 'malloy-data.faa.flights' | reduce inner is (reduce top 1000000 distance id2)"
+      )
+      .run();
+    let noAccessorTime;
+    let withAccessorTime;
+    {
+      const start = performance.now();
+      const inner = result.data.value[0]['inner'];
+      let total = 0;
+      let count = 0;
+      for (const row of inner as malloy.QueryData) {
+        total += row['distance'] as number;
+        count += 1;
+      }
+      noAccessorTime = performance.now() - start;
+      expect(total).toBe(88560989);
+      expect(count).toBe(1000000);
+      // Rough bound, not using accessors should be no more 50 milleseconds / million rows
+      expect(noAccessorTime).toBeLessThan(50);
+    }
+    {
+      const start = performance.now();
+      const inner = result.data.row(0).cell('inner').array;
+      let total = 0;
+      let count = 0;
+      for (const row of inner) {
+        total += row.cell('distance').number.value;
+        count += 1;
+      }
+      withAccessorTime = performance.now() - start;
+      expect(total).toBe(88560989);
+      expect(count).toBe(1000000);
+      // Rough bound, using accessors should be no more 70 milleseconds / million rows
+      expect(withAccessorTime).toBeLessThan(150);
+    }
+  }
+});

package/src/databases/bigquery/time.spec.ts

@@ -0,0 +1,59 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files
+ * (the "Software"), to deal in the Software without restriction,
+ * including without limitation the rights to use, copy, modify, merge,
+ * publish, distribute, sublicense, and/or sell copies of the Software,
+ * and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+import {DateTime} from 'luxon';
+import {RuntimeList} from '../../runtimes';
+import '../../util/db-jest-matchers';
+import {describeIfDatabaseAvailable} from '../../util';
+
+const [describe, databases] = describeIfDatabaseAvailable(['bigquery']);
+describe('BigQuery double truncation', () => {
+  const runtimes = new RuntimeList(databases);
+  const runtime = runtimes.runtimeMap.get('bigquery');
+
+  afterAll(async () => {
+    await runtimes.closeAll();
+  });
+
+  const utc_2020 = DateTime.fromObject({
+    year: 2020,
+    month: 2,
+    day: 20,
+    hour: 0,
+    minute: 0,
+    second: 0,
+    zone: 'UTC',
+  });
+  // TODO: this test is not working in this file.
+  test.skip('can use unsupported types', async () => {
+    await expect(runtime).queryMatches(
+      `sql: timeData is { connection: "bigquery" select: """
+      SELECT DATETIME '2020-02-20 00:00:00' as t_datetime
+      """}
+    query: from_sql(timeData) -> {
+      project: mex_220 is t_datetime::timestamp
+    }`,
+      {mex_220: utc_2020.toJSDate()}
+    );
+  });
+});

package/src/databases/bigquery/wildcard_table_names.spec.ts

@@ -0,0 +1,233 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files
+ * (the "Software"), to deal in the Software without restriction,
+ * including without limitation the rights to use, copy, modify, merge,
+ * publish, distribute, sublicense, and/or sell copies of the Software,
+ * and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+import {RuntimeList} from '../../runtimes';
+import {describeIfDatabaseAvailable} from '../../util';
+
+const [describe, databases] = describeIfDatabaseAvailable(['bigquery']);
+describe('Wildcard BigQuery Tables', () => {
+  const runtimes = new RuntimeList(databases);
+
+  afterAll(async () => {
+    await runtimes.closeAll();
+  });
+
+  test('test valid wildcard table name without using _TABLE_SUFFIX', async () => {
+    const runtime = runtimes.runtimeMap.get('bigquery');
+    expect(runtime).toBeDefined();
+    if (runtime) {
+      const result = await runtime
+        .loadQuery(
+          `
+        source: aircraft is table('malloy-data.malloytest.wildcard_aircraft_*') {
+          primary_key: id
+          measure: aircraft_count is count()
+        }
+
+        query: aircraft -> {
+          aggregate: aircraft_count
+        }
+      `
+        )
+        .run();
+      expect(result.data.value[0]['aircraft_count']).toBe(9);
+    }
+  });
+
+  test('test wildcard table name and _TABLE_SUFFIX as a filter', async () => {
+    const runtime = runtimes.runtimeMap.get('bigquery');
+    expect(runtime).toBeDefined();
+    if (runtime) {
+      const result = await runtime
+        .loadQuery(
+          `
+        source: aircraft is table('malloy-data.malloytest.wildcard_aircraft_*') {
+          primary_key: id
+          measure: aircraft_count is count()
+          where: _TABLE_SUFFIX = '01'
+        }
+
+        query: aircraft -> {
+          aggregate: aircraft_count
+        }
+`
+        )
+        .run();
+      expect(result.data.value[0]['aircraft_count']).toBe(5);
+    }
+  });
+
+  test('test join between wildcard tables without using _TABLE_SUFFIX', async () => {
+    const runtime = runtimes.runtimeMap.get('bigquery');
+    expect(runtime).toBeDefined();
+    if (runtime) {
+      const result = await runtime
+        .loadQuery(
+          `
+        source: aircraft is table('malloy-data.malloytest.wildcard_aircraft_*') {
+          primary_key: id
+        }
+
+        source: state_facts is table('malloy-data.malloytest.state_facts') {
+          join_many: aircraft on state = aircraft.state
+        }
+        query: state_facts -> {
+          group_by: aircraft.state
+          aggregate: aircraft_count is aircraft.count()
+          order_by: 1
+        }
+`
+        )
+        .run();
+      expect(result.data.value).toStrictEqual([
+        {state: null, aircraft_count: 43},
+        {state: 'IA', aircraft_count: 1},
+        {state: 'KS', aircraft_count: 1},
+        {state: 'LA', aircraft_count: 1},
+        {state: 'MO', aircraft_count: 1},
+        {state: 'NH', aircraft_count: 1},
+        {state: 'OK', aircraft_count: 1},
+        {state: 'OR', aircraft_count: 2},
+        {state: 'TX', aircraft_count: 1},
+      ]);
+    }
+  });
+
+  test('test join with wildcard tables and _TABLE_SUFFIX as source a filter', async () => {
+    const runtime = runtimes.runtimeMap.get('bigquery');
+    expect(runtime).toBeDefined();
+    if (runtime) {
+      const result = await runtime
+        .loadQuery(
+          `
+        source: aircraft is table('malloy-data.malloytest.wildcard_aircraft_*') {
+          primary_key: id
+          where: _TABLE_SUFFIX = '02'
+        }
+
+        source: state_facts is table('malloy-data.malloytest.state_facts') {
+          join_many: aircraft on state = aircraft.state
+        }
+        query: state_facts -> {
+          group_by: aircraft.state
+          aggregate: aircraft_count is aircraft.count()
+          order_by: 1
+        }
+`
+        )
+        .run();
+      expect(result.data.value).toStrictEqual([
+        {state: null, aircraft_count: 47},
+        {state: 'KS', aircraft_count: 1},
+        {state: 'LA', aircraft_count: 1},
+        {state: 'OK', aircraft_count: 1},
+        {state: 'OR', aircraft_count: 1},
+      ]);
+    }
+  });
+
+  test('test join with wildcard tables and _TABLE_SUFFIX as a filter and generated_uuid', async () => {
+    // issue # 1147
+    const runtime = runtimes.runtimeMap.get('bigquery');
+    expect(runtime).toBeDefined();
+    if (runtime) {
+      const result = await runtime
+        .loadQuery(
+          `
+        source: aircraft is table('malloy-data.malloytest.wildcard_aircraft_*') {
+          join_many: state_facts is table('malloy-data.malloytest.state_facts')
+            on state_facts.state = state
+        }
+
+        query: aircraft -> {
+          group_by: state_facts.state
+          aggregate: aircraft_count is count()
+          where: _TABLE_SUFFIX = '02'
+          order_by: 1 desc
+          limit: 1
+        }
+`
+        )
+        .run();
+      expect(result.data.path(0, 'state').value).toBe('OR');
+    }
+  });
+
+  test('test a join with wildcard tables and _TABLE_SUFFIX column is used as a group-by key', async () => {
+    const runtime = runtimes.runtimeMap.get('bigquery');
+    expect(runtime).toBeDefined();
+    if (runtime) {
+      const result = await runtime
+        .loadQuery(
+          `
+        source: aircraft is table('malloy-data.malloytest.wildcard_aircraft_*') {
+          primary_key: id
+          where: _TABLE_SUFFIX = '02'
+        }
+
+        source: state_facts is table('malloy-data.malloytest.state_facts') {
+          join_many: aircraft on state = aircraft.state
+        }
+        query: state_facts -> {
+          group_by: aircraft._TABLE_SUFFIX
+          aggregate: aircraft_count is aircraft.count()
+          order_by: 1
+        }
+`
+        )
+        .run();
+      expect(result.data.value).toStrictEqual([
+        {_TABLE_SUFFIX: null, aircraft_count: 47},
+        {_TABLE_SUFFIX: '02', aircraft_count: 4},
+      ]);
+    }
+  });
+
+  test('test a join with wildcard tables and _TABLE_SUFFIX column is used as a group-by key and a filter', async () => {
+    const runtime = runtimes.runtimeMap.get('bigquery');
+    expect(runtime).toBeDefined();
+    if (runtime) {
+      const result = await runtime
+        .loadQuery(
+          `
+        source: aircraft is table('malloy-data.malloytest.wildcard_aircraft_*') {
+          primary_key: id
+          where: _TABLE_SUFFIX = '02'
+        }
+
+        source: state_facts is table('malloy-data.malloytest.state_facts') {
+          join_many: aircraft on state = aircraft.state
+        }
+        query: state_facts -> {
+          group_by: aircraft._TABLE_SUFFIX
+          aggregate: aircraft_count is aircraft.count()
+          where: aircraft._TABLE_SUFFIX = '01'
+          order_by: 1
+        }
+`
+        )
+        .run();
+      expect(result.data.value).toStrictEqual([]);
+    }
+  });
+});

package/src/databases/bigquery-duckdb/nested_source_table.spec.ts

@@ -0,0 +1,268 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files
+ * (the "Software"), to deal in the Software without restriction,
+ * including without limitation the rights to use, copy, modify, merge,
+ * publish, distribute, sublicense, and/or sell copies of the Software,
+ * and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+/* eslint-disable no-console */
+
+import '../../util/db-jest-matchers';
+import {RuntimeList} from '../../runtimes';
+import {describeIfDatabaseAvailable} from '../../util';
+
+// No prebuilt shared model, each test is complete.  Makes debugging easier.
+
+const [describe, databases] = describeIfDatabaseAvailable([
+  'bigquery',
+  'duckdb',
+]);
+
+const modelText = `
+source:ga_sessions is table('malloytest.ga_sample'){
+
+  measure:
+    user_count is count(distinct fullVisitorId)
+    session_count is count()
+    total_visits is totals.visits.sum()
+    total_hits is totals.hits.sum()
+    total_page_views is totals.pageviews.sum()
+    t2 is totals.pageviews.sum()
+    total_productRevenue is hits.product.productRevenue.sum()
+    hits_count is hits.count()
+    sold_count is hits.count() { where: hits.product.productQuantity > 0 }
+
+  query: by_source is {
+    where: trafficSource.source != '(direct)'
+    group_by: trafficSource.source
+    aggregate: hits_count
+    limit: 10
+  }
+  query: by_adContent_bar_chart is {
+    group_by: device.browser
+    aggregate: user_count
+    group_by: device.deviceCategory
+  }
+  query: by_region is {
+    where: geoNetwork.region !~ '%demo%'
+    group_by: geoNetwork.region
+    aggregate: user_count
+    limit: 10
+  }
+  query: by_device is {
+    group_by: device.browser
+    aggregate: user_count
+    group_by: device.deviceCategory
+  }
+  query: by_category is {
+    group_by: category is hits.product.v2ProductCategory
+    aggregate: total_productRevenue
+    aggregate: sold_count
+    limit: 10
+  }
+  query: by_hour_of_day is {
+    // group_by: gsession_hour is hour(start_time::timestamp)
+    aggregate: session_count
+    order_by: 1
+  }
+
+  query: page_load_times is {
+    group_by: hits.page.pageTitle
+    aggregate: hit_count is hits.count()
+    nest: load_bar_chart is {
+      group_by: hit_seconds is floor(hits.latencyTracking.pageLoadTime / 2) * 2
+      aggregate: hits_count
+    }
+    limit: 10
+  }
+
+  query: by_page_title is { where: totals.transactionRevenue > 0
+    group_by: hits.page.pageTitle
+    aggregate: hits_count
+    aggregate: sold_count
+  }
+
+  query: by_all is {
+    nest: by_source
+    nest: by_adContent_bar_chart
+    nest: by_region
+    nest: by_category
+  }
+
+  query: search_index is {
+    index: *, hits.*, customDimensions.* totals.*, trafficSource.*, hits.product.*
+    sample: 1%
+  }
+}
+
+query: sessions_dashboard is ga_sessions -> {
+  nest:
+    by_region
+    by_device
+    by_source
+    by_category is {
+      group_by: category is hits.product.v2ProductCategory
+      aggregate: total_productRevenue
+      aggregate: sold_count
+    }
+}
+`;
+
+const runtimes = new RuntimeList(databases);
+describe.each(runtimes.runtimeList)(
+  'Nested Source Table - %s',
+  (databaseName, runtime) => {
+    test(`repeated child of record - ${databaseName}`, async () => {
+      const result = await runtime
+        .loadModel(modelText)
+        .loadQuery(
+          `
+        query: ga_sessions->by_page_title
+        `
+        )
+        .run();
+      // console.log(result.data.toObject());
+      // console.log(result.sql);
+      expect(result.data.path(0, 'pageTitle').value).toBe('Shopping Cart');
+    });
+
+    // SKIPPED because the tests intermittently fail and lloyd said
+    // "I bet it has to do with my sampling test on indexing. Intermittently
+    //  getting different results. I'd comment out the test and I'll take a
+    //  look at it when I get back." and that seems reasonable.
+    // "search_index" is the test that failed, but a skipped both
+    // this one and "manual index". Here's the errror:
+    //   * Nested Source Table › search_index - duckdb
+    //
+    //      expect(received).toBe(expected) // Object.is equality
+    //
+    //      Expected: "Organic Search"
+    //      Received: "Referral"
+    test.skip(`search_index - ${databaseName}`, async () => {
+      const result = await runtime
+        .loadModel(modelText)
+        .loadQuery(
+          `
+        query: ga_sessions->search_index -> {
+          where: fieldName != null
+          project: *
+          order_by: fieldName, weight desc
+          limit: 10
+        }
+        `
+        )
+        .run();
+      // console.log(result.data.toObject());
+      expect(result.data.path(0, 'fieldName').value).toBe('channelGrouping');
+      expect(result.data.path(0, 'fieldValue').value).toBe('Organic Search');
+      // expect(result.data.path(0, "weight").value).toBe(18);
+    });
+
+    test.skip(`manual index - ${databaseName}`, async () => {
+      let sampleSize = '10';
+      if (databaseName === 'bigquery') {
+        sampleSize = 'false';
+      }
+      const _result = await runtime
+        .loadQuery(
+          `
+        query: table('malloytest.ga_sample')-> {
+          index: everything
+          sample: ${sampleSize}
+        }
+        -> {
+          aggregate: field_count is count(DISTINCT fieldName)
+          nest: top_fields is {
+            group_by: fieldName
+            aggregate: row_count is count()
+            limit: 100
+          }
+        }
+        `
+        )
+        .run();
+      // console.log(JSON.stringify(result.data.toObject(), null, 2));
+      // expect(result.data.path(0, "fieldName").value).toBe("channelGrouping");
+      // expect(result.data.path(0, "fieldValue").value).toBe("Organic Search");
+      // expect(result.data.path(0, "weight").value).toBe(18);
+    });
+
+    test(`autobin - ${databaseName}`, async () => {
+      const _result = await runtime
+        .loadQuery(
+          `
+          source: airports is table('malloytest.airports') + {
+            measure:
+              airport_count is count()
+            query: by_elevation is {
+              aggregate: bin_size is NULLIF((max(elevation) - min(elevation))/30,0)
+              nest: data is {
+                group_by: elevation
+                aggregate: row_count is count()
+              }
+            }
+            -> {
+              group_by: elevation is floor(data.elevation/bin_size)*bin_size + bin_size/2
+              aggregate: airport_count is data.row_count.sum()
+              order_by: elevation
+            }
+          }
+
+          query: airports -> {
+            group_by: state is state
+            aggregate: airport_count
+            nest: by_elevation_bar_chart is by_elevation
+          }
+        `
+        )
+        .run();
+      // console.log(result.sql);
+      // console.log(result.data.toObject());
+      // expect(result.data.path(0, 'fieldName').value).toBe('channelGrouping');
+      // expect(result.data.path(0, 'fieldValue').value).toBe('Organic Search');
+      // expect(result.data.path(0, "weight").value).toBe(18);
+    });
+
+    // this really shouldn't be here, we need to be able to test dialect capiblities
+    //  from the connection.
+    test(`safe_cast - ${databaseName}`, async () => {
+      const result = await runtime
+        .loadQuery(
+          `
+        source: eone is  table('malloytest.airports') {}
+
+        query: eone -> {
+          project:
+            bad_date is '123':::date
+            bad_number is 'abc':::number
+            limit: 1
+        }
+        `
+        )
+        .run();
+      expect(result.data.value[0]['bad_date']).toBe(null);
+      expect(result.data.value[0]['bad_number']).toBe(null);
+    });
+  }
+);
+
+afterAll(async () => {
+  await runtimes.closeAll();
+});