@malloydata/malloy-tests 0.0.225-dev250102225314 → 0.0.225-dev250106171109

package/config/core.config.json

@@ -0,0 +1,8 @@
+{
+  "roots": [
+    "../../packages/malloy-malloy-sql/",
+    "../../packages/malloy-syntax-highlight/",
+    "../../packages/malloy/",
+    "../../test/src/core/"
+  ]
+}

package/package.json

@@ -21,13 +21,13 @@
   },
   "dependencies": {
     "@jest/globals": "^29.4.3",
-    "@malloydata/db-bigquery": "^0.0.225-dev250102225314",
-    "@malloydata/db-duckdb": "^0.0.225-dev250102225314",
-    "@malloydata/db-postgres": "^0.0.225-dev250102225314",
-    "@malloydata/db-snowflake": "^0.0.225-dev250102225314",
-    "@malloydata/db-trino": "^0.0.225-dev250102225314",
-    "@malloydata/malloy": "^0.0.225-dev250102225314",
-    "@malloydata/render": "^0.0.225-dev250102225314",
+    "@malloydata/db-bigquery": "^0.0.225-dev250106171109",
+    "@malloydata/db-duckdb": "^0.0.225-dev250106171109",
+    "@malloydata/db-postgres": "^0.0.225-dev250106171109",
+    "@malloydata/db-snowflake": "^0.0.225-dev250106171109",
+    "@malloydata/db-trino": "^0.0.225-dev250106171109",
+    "@malloydata/malloy": "^0.0.225-dev250106171109",
+    "@malloydata/render": "^0.0.225-dev250106171109",
     "events": "^3.3.0",
     "jsdom": "^22.1.0",
     "luxon": "^2.4.0",
@@ -37,5 +37,5 @@
     "@types/jsdom": "^21.1.1",
     "@types/luxon": "^2.4.0"
   },
-  "version": "0.0.225-dev250102225314"
+  "version": "0.0.225-dev250106171109"
 }

package/src/{api.spec.ts → core/api.spec.ts}

@@ -22,8 +22,8 @@
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 
-import {runtimeFor} from './runtimes';
-import './util/db-jest-matchers';
+import {runtimeFor} from '../runtimes';
+import '../util/db-jest-matchers';
 
 const runtime = runtimeFor('duckdb');
 

package/src/{events.spec.ts → core/events.spec.ts}

@@ -5,8 +5,8 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-import {runtimeFor} from './runtimes';
-import './util/db-jest-matchers';
+import {runtimeFor} from '../runtimes';
+import '../util/db-jest-matchers';
 
 const runtime = runtimeFor('duckdb');
 

package/src/{experimental-dialects.spec.ts → core/experimental-dialects.spec.ts}

@@ -28,8 +28,8 @@ import {
   SQLSourceDef,
   registerDialect,
 } from '@malloydata/malloy';
-import {testRuntimeFor} from './runtimes';
-import './util/db-jest-matchers';
+import {testRuntimeFor} from '../runtimes';
+import '../util/db-jest-matchers';
 import {DuckDBConnection} from '@malloydata/db-duckdb';
 
 const envDatabases = (

package/src/{jestMatcher.spec.ts → core/jestMatcher.spec.ts}

@@ -22,8 +22,8 @@
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 
-import {runtimeFor} from './runtimes';
-import './util/db-jest-matchers';
+import {runtimeFor} from '../runtimes';
+import '../util/db-jest-matchers';
 
 const runtime = runtimeFor('duckdb');
 

package/src/{databases/multi-connection → core}/multi_connection.spec.ts

@@ -26,8 +26,8 @@
 
 import * as malloy from '@malloydata/malloy';
 import {EmptyURLReader} from '@malloydata/malloy';
-import {DuckDBTestConnection, PostgresTestConnection} from '../../runtimes';
-import {describeIfDatabaseAvailable} from '../../util';
+import {DuckDBTestConnection, PostgresTestConnection} from '../runtimes';
+import {describeIfDatabaseAvailable} from '../util';
 
 const [, databases] = describeIfDatabaseAvailable(['duckdb', 'postgres']);
 

package/src/{databases/streaming → core}/streaming.spec.ts

@@ -27,8 +27,8 @@ import {
   JSONWriter,
   WriteStream,
 } from '@malloydata/malloy';
-import {RuntimeList} from '../../runtimes';
-import {describeIfDatabaseAvailable} from '../../util';
+import {RuntimeList} from '../runtimes';
+import {describeIfDatabaseAvailable} from '../util';
 
 class StringAccumulator implements WriteStream {
   public accumulatedValue = '';

package/src/{tags.spec.ts → core/tags.spec.ts}

@@ -22,7 +22,7 @@
  */
 
 import {TagDict, Tag} from '@malloydata/malloy';
-import {runtimeFor} from './runtimes';
+import {runtimeFor} from '../runtimes';
 
 declare global {
   // eslint-disable-next-line @typescript-eslint/no-namespace

package/src/databases/bigquery/malloy_query.spec.ts

@@ -32,7 +32,7 @@ import '../../util/db-jest-matchers';
 const runtimeList = new RuntimeList(['bigquery']);
 const runtime = runtimeList.runtimeMap.get('bigquery');
 if (runtime === undefined) {
-  throw new Error("Couldn't build runtime");
+  throw new Error('BigQuery runtime not found');
 }
 const bq = runtime.connection as BigQueryTestConnection;
 
@@ -750,15 +750,6 @@ describe('airport_tests', () => {
     `).malloyResultMatches(model, {'pipe_turtle.total_airports': 1845});
   });
 
-  it(`hyperloglog combine`, async () => {
-    await expect(`run: bigquery.table('malloytest.airports')->{
-        aggregate: code_hll is hll_accumulate(code)
-    } -> {
-        aggregate: code_count is hll_estimate(hll_combine(code_hll))
-    }
-    `).malloyResultMatches(runtime, {code_count: 19799});
-  });
-
   it.skip('crossjoined turtles', async () => {
     // const result = await runQuery(model,`
     //     explore airports

package/src/databases/duckdb/nested_source_table.spec.ts

@@ -0,0 +1,226 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files
+ * (the "Software"), to deal in the Software without restriction,
+ * including without limitation the rights to use, copy, modify, merge,
+ * publish, distribute, sublicense, and/or sell copies of the Software,
+ * and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+/* eslint-disable no-console */
+
+import '../../util/db-jest-matchers';
+import {RuntimeList} from '../../runtimes';
+import {describeIfDatabaseAvailable} from '../../util';
+
+// No prebuilt shared model, each test is complete.  Makes debugging easier.
+
+const [describe, databases] = describeIfDatabaseAvailable([
+  'bigquery',
+  'duckdb',
+]);
+
+function modelText(databaseName: string) {
+  return `
+source: ga_sessions is ${databaseName}.table('malloytest.ga_sample') extend {
+
+  measure:
+    user_count is count(fullVisitorId)
+    session_count is count()
+    total_visits is totals.visits.sum()
+    total_hits is totals.hits.sum()
+    total_page_views is totals.pageviews.sum()
+    t2 is totals.pageviews.sum()
+    total_productRevenue is hits.product.productRevenue.sum()
+    hits_count is hits.count()
+    sold_count is hits.count() { where: hits.product.productQuantity > 0 }
+
+  view: by_source is {
+    where: trafficSource.\`source\` != '(direct)'
+    group_by: trafficSource.\`source\`
+    aggregate: hits_count
+    limit: 10
+  }
+  view: by_adContent_bar_chart is {
+    group_by: device.browser
+    aggregate: user_count
+    group_by: device.deviceCategory
+  }
+  view: by_region is {
+    where: geoNetwork.region !~ '%demo%'
+    group_by: geoNetwork.region
+    aggregate: user_count
+    limit: 10
+  }
+  view: by_device is {
+    group_by: device.browser
+    aggregate: user_count
+    group_by: device.deviceCategory
+  }
+  view: by_category is {
+    group_by: category is hits.product.v2ProductCategory
+    aggregate: total_productRevenue
+    aggregate: sold_count
+    limit: 10
+  }
+  view: by_hour_of_day is {
+    // group_by: gsession_hour is hour(start_time::timestamp)
+    aggregate: session_count
+    order_by: 1
+  }
+
+  view: page_load_times is {
+    group_by: hits.page.pageTitle
+    aggregate: hit_count is hits.count()
+    nest: load_bar_chart is {
+      group_by: hit_seconds is floor(hits.latencyTracking.pageLoadTime / 2) * 2
+      aggregate: hits_count
+    }
+    limit: 10
+  }
+
+  view: by_page_title is { where: totals.transactionRevenue > 0
+    group_by: hits.page.pageTitle
+    aggregate: hits_count
+    aggregate: sold_count
+  }
+
+  view: by_all is {
+    nest: by_source
+    nest: by_adContent_bar_chart
+    nest: by_region
+    nest: by_category
+  }
+
+  view: search_index is {
+    index: *, hits.*, customDimensions.* totals.*, trafficSource.*, hits.product.*
+    sample: 1%
+  }
+}
+
+query: sessions_dashboard is ga_sessions -> {
+  nest:
+    by_region
+    by_device
+    by_source
+    by_category is {
+      group_by: category is hits.product.v2ProductCategory
+      aggregate: total_productRevenue
+      aggregate: sold_count
+    }
+}
+`;
+}
+
+const runtimes = new RuntimeList(databases);
+describe.each(runtimes.runtimeList)(
+  'Nested Source Table - %s',
+  (databaseName, runtime) => {
+    const gaModel = runtime.loadModel(modelText(databaseName));
+    test(`repeated child of record - ${databaseName}`, async () => {
+      await expect('run: ga_sessions->by_page_title').malloyResultMatches(
+        gaModel,
+        {pageTitle: 'Shopping Cart'}
+      );
+    });
+
+    // Tests intermittently fail and lloyd said
+    // "I bet it has to do with my sampling test on indexing. Intermittently
+    //  getting different results. I'd comment out the test and I'll take a
+    //  look at it when I get back." and that seems reasonable.
+    // "search_index" is the test that failed, but a skipped both
+    // this one and "manual index". Here's the errror:
+    //   * Nested Source Table › search_index - duckdb
+    //
+    //      expect(received).toBe(expected) // Object.is equality
+    //
+    //      Expected: "Organic Search"
+    //      Received: "Referral"
+    test(`search_index - ${databaseName}`, async () => {
+      await expect(`
+        run: ga_sessions->search_index -> {
+          where: fieldName != null
+          select: *
+          order_by: fieldName, weight desc
+          limit: 10
+        }
+      `).malloyResultMatches(gaModel, {
+        // fieldName: 'channelGrouping',
+        // fieldValue: 'Organic Search',
+        // weight: 10,
+      });
+    });
+
+    test(`manual index - ${databaseName}`, async () => {
+      let sampleSize = '10';
+      if (databaseName === 'bigquery') {
+        sampleSize = 'false';
+      }
+      await expect(`
+        run: ${databaseName}.table('malloytest.ga_sample')-> {
+          index: *
+          sample: ${sampleSize}
+        }
+        -> {
+          aggregate: field_count is count(fieldName)
+          nest: top_fields is {
+            group_by: fieldName
+            aggregate: row_count is count()
+            limit: 100
+          }
+        }
+      `).malloyResultMatches(runtime, {
+        // 'top_fields.fieldName': 'channelGrouping',
+        // 'top_fields.fieldValue': 'Organic Search',
+        // 'top_fields.weight': 18,
+      });
+    });
+
+    test(`autobin - ${databaseName}`, async () => {
+      await expect(`
+        source: airports is ${databaseName}.table('malloytest.airports') extend {
+          measure: airport_count is count()
+          view: by_elevation is {
+            aggregate: bin_size is NULLIF((max(elevation) - min(elevation))/30,0)
+            nest: data is {
+              group_by: elevation
+              aggregate: row_count is count()
+            }
+          }
+          -> {
+            group_by: elevation is floor(data.elevation/bin_size)*bin_size + bin_size/2
+            aggregate: airport_count is data.row_count.sum()
+            order_by: elevation
+          }
+        }
+
+        run: airports -> {
+          group_by: state is state
+          aggregate: airport_count
+          nest: by_elevation_bar_chart is by_elevation
+        }
+      `).malloyResultMatches(runtime, {
+        // don't know what to expect ...
+      });
+    });
+  }
+);
+
+afterAll(async () => {
+  await runtimes.closeAll();
+});