@malloydata/malloy-tests 0.0.433 → 0.0.435

package/src/core/virtual-sql.spec.ts

@@ -0,0 +1,253 @@
+/*
+ * Copyright Contributors to the Malloy project
+ * SPDX-License-Identifier: MIT
+ */
+
+import {DuckDBConnection} from '@malloydata/db-duckdb';
+import {
+  FixedConnectionMap,
+  InMemoryURLReader,
+  Malloy,
+  MalloyConfig,
+  Runtime,
+} from '@malloydata/malloy';
+import type {VirtualMap} from '@malloydata/malloy';
+
+const virtualSource = `
+  ##! experimental.virtual_source
+  type: row_fields is { x :: number }
+  source: data is duckdb.virtual('data')::row_fields
+`;
+const sqlSource = `
+  source: wrapped is duckdb.sql("""
+    SELECT x + 10 AS y FROM %{ data -> { select: x } }
+  """)
+  run: wrapped -> { select: y }
+`;
+const source = virtualSource + sqlSource;
+
+function virtualMap(table: string): VirtualMap {
+  return new Map([['duckdb', new Map([['data', table]])]]);
+}
+
+// Language cases live in packages/malloy/src/lang/test/virtual-sql.spec.ts.
+// These tests cover API option forwarding and real SQL schemas.
+describe('virtual sources in SQL blocks through the Foundation API', () => {
+  const connection = new DuckDBConnection({
+    name: 'duckdb',
+    databasePath: ':memory:',
+  });
+  const connections = FixedConnectionMap.fromArray([connection]);
+  const compileMap = virtualMap('translation_data');
+  const executionMap = virtualMap('execution_data');
+  const urlReader = new InMemoryURLReader(new Map());
+
+  beforeAll(async () => {
+    await connection.runSQL(
+      'CREATE TEMP TABLE translation_data AS SELECT 1::INTEGER AS x'
+    );
+    await connection.runSQL(
+      'CREATE TEMP TABLE execution_data AS SELECT 2::INTEGER AS x'
+    );
+  });
+
+  afterAll(async () => {
+    await connection.close();
+  });
+
+  afterEach(() => {
+    jest.restoreAllMocks();
+  });
+
+  test('Malloy.compile uses a translation map without binding execution', async () => {
+    const model = await Malloy.compile({
+      source,
+      connections,
+      urlReader,
+      virtualMap: compileMap,
+    });
+    const result = model.preparedQuery.getPreparedResult({
+      virtualMap: executionMap,
+    });
+    expect(result.sql).toContain('execution_data');
+    expect(result.sql).not.toContain('translation_data');
+    expect((await connection.runSQL(result.sql)).rows).toEqual([{y: 12}]);
+    expect(() => model.preparedQuery.getPreparedResult()).toThrow(
+      /No virtual-map entry/
+    );
+  });
+
+  test('loadModel forwards its map and execution can override it', async () => {
+    const runtime = new Runtime({connections});
+    runtime.virtualMap = virtualMap('missing_runtime_table');
+    const result = await runtime
+      .loadModel(source, {virtualMap: compileMap})
+      .loadFinalQuery()
+      .run({virtualMap: executionMap});
+    expect(result.data.value).toEqual([{y: 12}]);
+  });
+
+  test('runtime config supplies the translation map', async () => {
+    const config = new MalloyConfig({
+      virtualMap: {duckdb: {data: 'translation_data'}},
+    });
+    const runtime = new Runtime({connections, config});
+    const result = await runtime.loadQuery(source).run();
+    expect(result.data.value).toEqual([{y: 11}]);
+  });
+
+  test('model.loadQuery uses an explicit translation map', async () => {
+    const runtime = new Runtime({connections});
+    runtime.virtualMap = virtualMap('missing_runtime_table');
+    const result = await runtime
+      .loadModel(virtualSource)
+      .loadQuery(sqlSource, {virtualMap: compileMap})
+      .run({virtualMap: executionMap});
+    expect(result.data.value).toEqual([{y: 12}]);
+  });
+
+  test('model.loadQuery falls back to the runtime translation map', async () => {
+    const runtime = new Runtime({connections});
+    runtime.virtualMap = compileMap;
+    const result = await runtime
+      .loadModel(virtualSource)
+      .loadQuery(sqlSource)
+      .run({virtualMap: executionMap});
+    expect(result.data.value).toEqual([{y: 12}]);
+  });
+
+  test('model.loadQuery inherits the model translation map', async () => {
+    const runtime = new Runtime({connections});
+    runtime.virtualMap = virtualMap('missing_runtime_table');
+    const result = await runtime
+      .loadModel(virtualSource, {virtualMap: compileMap})
+      .loadQuery(sqlSource)
+      .run({virtualMap: executionMap});
+    expect(result.data.value).toEqual([{y: 12}]);
+  });
+
+  test('an empty translation map overrides the runtime map', async () => {
+    const runtime = new Runtime({connections});
+    runtime.virtualMap = compileMap;
+    await expect(
+      runtime.loadModel(source, {virtualMap: new Map()}).getModel()
+    ).rejects.toThrow(/No virtual-map entry/);
+  });
+
+  test('an invalid translation binding fails before SQL schema discovery', async () => {
+    const fetchSchema = jest
+      .spyOn(connection, 'fetchSchemaForSQLStruct')
+      .mockRejectedValue(new Error('Unexpected SQL schema request'));
+    await expect(
+      Malloy.compile({
+        source,
+        connections,
+        urlReader,
+        virtualMap: virtualMap('translation_data; SELECT 1'),
+      })
+    ).rejects.toThrow(/virtualMap entry 'duckdb.data'.*duckdb/);
+    expect(fetchSchema).not.toHaveBeenCalled();
+  });
+
+  test('an invalid execution binding fails before query execution', async () => {
+    const runtime = new Runtime({connections});
+    const query = runtime.loadQuery(source, {virtualMap: compileMap});
+    await query.getPreparedQuery();
+    const runSQL = jest
+      .spyOn(connection, 'runSQL')
+      .mockRejectedValue(new Error('Unexpected query execution'));
+    await expect(
+      query.run({virtualMap: virtualMap('execution_data; SELECT 1')})
+    ).rejects.toThrow(/virtualMap entry 'duckdb.data'.*duckdb/);
+    expect(runSQL).not.toHaveBeenCalled();
+  });
+
+  test.each(['explicit', 'inherited', 'runtime'])(
+    'extendModel uses its %s translation map',
+    async binding => {
+      const runtime = new Runtime({connections});
+      runtime.virtualMap =
+        binding === 'runtime' ? compileMap : virtualMap('missing_table');
+      const model = runtime.loadModel(virtualSource, {
+        virtualMap:
+          binding === 'inherited'
+            ? compileMap
+            : binding === 'explicit'
+              ? virtualMap('missing_inherited_table')
+              : undefined,
+      });
+      const extended = model.extendModel(
+        sqlSource,
+        binding === 'explicit' ? {virtualMap: compileMap} : undefined
+      );
+      const result = await extended.loadFinalQuery().run();
+      expect(result.data.value).toEqual([{y: 11}]);
+    }
+  );
+
+  test('Malloy.compile forwards its map to a pre-parsed translator', async () => {
+    const parse = Malloy.parse({source});
+    const model = await Malloy.compile({
+      parse,
+      connections,
+      urlReader,
+      virtualMap: compileMap,
+    });
+    const result = model.preparedQuery.getPreparedResult({
+      virtualMap: executionMap,
+    });
+    expect((await connection.runSQL(result.sql)).rows).toEqual([{y: 12}]);
+  });
+
+  test.each(['compile', 'run'])(
+    '%s ignores an invalid override binding when it is unused',
+    async operation => {
+      const badMap = virtualMap('translation_data');
+      badMap.get('duckdb')!.set('unused', 'translation_data; SELECT 1');
+      const runtime = new Runtime({connections});
+      const result =
+        operation === 'compile'
+          ? Malloy.compile({source, connections, urlReader, virtualMap: badMap})
+          : runtime
+              .loadQuery(source, {virtualMap: compileMap})
+              .run({virtualMap: badMap});
+      await expect(result).resolves.toBeDefined();
+    }
+  );
+
+  test.each(['config', 'assignment'])(
+    'bindings from %s are preserved and checked only when used',
+    async kind => {
+      const badTable = 'translation_data; SELECT 1';
+      const config =
+        kind === 'config'
+          ? new MalloyConfig({
+              virtualMap: {
+                duckdb: {data: 'translation_data', unused: badTable},
+              },
+            })
+          : undefined;
+      const runtime = new Runtime({connections, config});
+      if (kind === 'assignment') {
+        const map = virtualMap('translation_data');
+        map.get('duckdb')!.set('unused', badTable);
+        runtime.virtualMap = map;
+      }
+      expect(runtime.virtualMap?.get('duckdb')?.get('unused')).toBe(badTable);
+      expect((await runtime.loadQuery(source).run()).data.value).toEqual([
+        {y: 11},
+      ]);
+      await expect(
+        runtime
+          .loadQuery(
+            `
+          ${virtualSource}
+          source: bad is duckdb.virtual('unused')::row_fields
+          run: bad -> { select: x }
+        `
+          )
+          .getSQL()
+      ).rejects.toThrow(/virtualMap entry 'duckdb.unused'.*duckdb/);
+    }
+  );
+});

package/src/databases/all/db_index.spec.ts

@@ -79,11 +79,13 @@ runtimes.runtimeMap.forEach((runtime, databaseName) => {
     `).toMatchResult(testModel, {fieldName: 'one', weight: 51});
   });
 
-  // bigquery doesn't support row count based sampling.
+  // bigquery doesn't support row count based sampling; sqlserver's TABLESAMPLE
+  // returns whole pages, not the count asked for.
   test.when(
     databaseName !== 'bigquery' &&
       databaseName !== 'trino' &&
-      databaseName !== 'presto'
+      databaseName !== 'presto' &&
+      databaseName !== 'sqlserver'
   )(`index rows count - ${databaseName}`, async () => {
     await expect(`
         run: ${databaseName}.table('malloytest.state_facts') extend {

package/src/databases/all/expr.spec.ts

@@ -337,11 +337,8 @@ describe.each(runtimes.runtimeList)('%s', (databaseName, runtime) => {
     });
   });
 
-  // TODO not sure why this test needs to be skipped on postgres, feels like an oversight
-  test.when(!['postgres', 'mysql'].includes(databaseName))(
-    'model: dates named',
-    async () => {
-      await expect(`
+  test('model: dates named', async () => {
+    await expect(`
       run: ${databaseName}.table('malloytest.alltypes')->{
         group_by:
           t_date,
@@ -356,19 +353,18 @@ describe.each(runtimes.runtimeList)('%s', (databaseName, runtime) => {
           t_timestamp_year is t_timestamp.year,
       }
     `).toMatchResult(testModel, {
-        t_date: new Date('2020-03-02'),
-        t_date_month: new Date('2020-03-01'),
-        t_date_year: new Date('2020-01-01'),
-        t_timestamp: new Date('2020-03-02T12:35:56.000Z'),
-        t_timestamp_second: new Date('2020-03-02T12:35:56.000Z'),
-        t_timestamp_minute: new Date('2020-03-02T12:35:00.000Z'),
-        t_timestamp_hour: new Date('2020-03-02T12:00:00.000Z'),
-        t_timestamp_date: new Date('2020-03-02'),
-        t_timestamp_month: new Date('2020-03-01'),
-        t_timestamp_year: new Date('2020-01-01'),
-      });
-    }
-  );
+      t_date: new Date('2020-03-02'),
+      t_date_month: new Date('2020-03-01'),
+      t_date_year: new Date('2020-01-01'),
+      t_timestamp: new Date('2020-03-02T12:35:56.000Z'),
+      t_timestamp_second: new Date('2020-03-02T12:35:56.000Z'),
+      t_timestamp_minute: new Date('2020-03-02T12:35:00.000Z'),
+      t_timestamp_hour: new Date('2020-03-02T12:00:00.000Z'),
+      t_timestamp_date: new Date('2020-03-02'),
+      t_timestamp_month: new Date('2020-03-01'),
+      t_timestamp_year: new Date('2020-01-01'),
+    });
+  });
 
   it('named query metadata undefined', async () => {
     const result = await expressionModel
@@ -736,6 +732,142 @@ describe.each(runtimes.runtimeList)('%s', (databaseName, runtime) => {
     }
   );
 
+  // A filtered join is emitted as a subquery, so every join below it lives
+  // inside that subquery and has to be projected back out for the outer query
+  // to name it -- at every depth, not just the direct children.
+  describe('joins below a filtered join', () => {
+    const chain = (filterTwo: string) => `
+      source: sf is ${databaseName}.table('malloytest.state_facts')
+
+      source: three is sf
+      source: two is sf extend {
+        ${filterTwo}
+        join_one: three on three.state = state
+      }
+      source: one is sf extend {
+        where: state ~ 'A%'
+        join_one: two on two.state = state
+      }
+      source: zero is sf extend { join_one: one on one.state = state }
+    `;
+
+    test.when(runtime.dialect.supportsComplexFilteredSources)(
+      'two levels below',
+      async () => {
+        await expect(`
+          ${chain('')}
+          run: zero -> {
+            aggregate:
+              child is one.two.count()
+              grandchild is one.two.three.count()
+          }
+        `).toMatchResult(testModel, {child: 4, grandchild: 4});
+
+        await expect(`
+          ${chain('')}
+          run: zero -> {
+            group_by: grandchild_state is one.two.three.state
+            order_by: grandchild_state
+            limit: 1
+          }
+        `).toMatchResult(testModel, {grandchild_state: 'AK'});
+      }
+    );
+
+    // `having:` and `calculate:` name output fields, so their field usage is
+    // rooted in the query's output and does not resolve against its source.
+    test.when(runtime.dialect.supportsComplexFilteredSources)(
+      'having on an output field, with a filtered join',
+      async () => {
+        await expect(`
+          ${chain('')}
+          run: zero -> {
+            group_by: grandchild_state is one.two.three.state
+            aggregate: rows_here is count()
+            having: rows_here > 0
+            order_by: grandchild_state
+            limit: 1
+          }
+        `).toMatchResult(testModel, {grandchild_state: 'AK', rows_here: 1});
+      }
+    );
+
+    // The packed columns are the source's own column names, so the struct has
+    // to quote them on the value side and in any type it declares.
+    test.when(runtime.dialect.supportsComplexFilteredSources)(
+      'a packed column name which needs quoting',
+      async () => {
+        const oddName = runtime.dialect.sqlQuoteIdentifier('order');
+        await expect(`
+          source: two is ${databaseName}.sql("""SELECT 1 as ${oddName}, 1 as k""")
+          source: one is ${databaseName}.sql("""SELECT 1 as k, 2 as j""") extend {
+            where: j > 0
+            join_one: two on two.k = k
+          }
+          source: zero is ${databaseName}.sql("""SELECT 1 as k""") extend {
+            join_one: one on one.k = k
+          }
+
+          run: zero -> { group_by: g is one.two.\`order\` }
+        `).toMatchResult(testModel, {g: 1});
+      }
+    );
+
+    // A distinct key is the one thing the packed struct must carry which no
+    // expression in the query names, and when the key is computed what the
+    // query reads is the columns its expression reads, not the key's name.
+    test.when(runtime.dialect.supportsComplexFilteredSources)(
+      'two levels below, reached by a computed primary key',
+      async () => {
+        await expect(`
+          source: sf is ${databaseName}.table('malloytest.state_facts')
+
+          source: three is sf extend {
+            dimension: pk is births
+            primary_key: pk
+          }
+          source: two is sf extend { join_one: three on three.state = state }
+          source: one is sf extend {
+            where: state ~ 'A%'
+            join_one: two on two.state = state
+          }
+          source: zero is sf extend { join_one: one on one.state = state }
+
+          run: zero -> {
+            aggregate:
+              child is one.two.count()
+              grandchild is one.two.three.count()
+          }
+        `).toMatchResult(testModel, {child: 4, grandchild: 4});
+      }
+    );
+
+    // `two` is filtered as well, so it becomes a subquery of its own inside
+    // the one for `one`, and `three` reaches the outer query through it.
+    test.when(runtime.dialect.supportsComplexFilteredSources)(
+      'two levels below, both filtered',
+      async () => {
+        await expect(`
+          ${chain("where: state != 'AK'")}
+          run: zero -> {
+            aggregate:
+              child is one.two.count()
+              grandchild is one.two.three.count()
+          }
+        `).toMatchResult(testModel, {child: 3, grandchild: 3});
+
+        await expect(`
+          ${chain("where: state != 'AK'")}
+          run: zero -> {
+            group_by: grandchild_state is one.two.three.state
+            order_by: grandchild_state
+            limit: 1
+          }
+        `).toMatchResult(testModel, {grandchild_state: 'AL'});
+      }
+    );
+  });
+
   test('joined filtered explores with NO dependencies', async () => {
     await expect(`
       source: sf is ${databaseName}.table('malloytest.state_facts') extend {

package/src/databases/all/nest.spec.ts

@@ -0,0 +1,237 @@
+/*
+ * Copyright Contributors to the Malloy project
+ * SPDX-License-Identifier: MIT
+ */
+
+import {RuntimeList, allDatabases} from '../../runtimes';
+import {databasesFromEnvironmentOr} from '../../util';
+import '@malloydata/malloy/test/matchers';
+import {wrapTestModel} from '@malloydata/malloy/test';
+
+const runtimes = new RuntimeList(databasesFromEnvironmentOr(allDatabases));
+
+afterAll(async () => {
+  await runtimes.closeAll();
+});
+
+// A nest is a column whose value is the result of a query. What that column
+// contains, and in what order, is the subject here.
+//
+// The order is the order of the stage which produced it -- the order the user
+// asked for, or the one Malloy picked. Reading the array back is the only way to
+// see that, and the ordering is attached to the SQL which packs the array, which
+// is a different expression for a one-stage nest than for a pipelined one.
+//
+// The ordering tests nest over the four 'N' states whose popular_name is
+// Isabella, because their four candidate orderings are all distinct:
+//   state desc    NY NV NM NJ
+//   state asc     NJ NM NV NY
+//   births desc   NY NJ NM NV
+//   births asc    NV NM NJ NY
+// A pipelined test orders its first stage the opposite way from its last, so an
+// array carrying the wrong stage's order fails instead of coincidentally passing.
+
+describe.each(runtimes.runtimeList)('%s', (databaseName, runtime) => {
+  const model = wrapTestModel(
+    runtime,
+    `
+    query: isabellaN is ${databaseName}.table('malloytest.state_facts') -> {
+      where: popular_name = 'Isabella' and state ~ 'N%'
+      group_by: popular_name
+    }`
+  );
+
+  const stateDesc = [{st: 'NY'}, {st: 'NV'}, {st: 'NM'}, {st: 'NJ'}];
+  const birthsDesc = [{st: 'NY'}, {st: 'NJ'}, {st: 'NM'}, {st: 'NV'}];
+  const keywordStateDesc = [
+    {'select': 'NY'},
+    {'select': 'NV'},
+    {'select': 'NM'},
+    {'select': 'NJ'},
+  ];
+
+  test.when(runtime.supportsNesting)(
+    `one-stage nest is in the order it asked for - ${databaseName}`,
+    async () => {
+      await expect(`
+        run: isabellaN + {
+          nest: n is {
+            group_by: st is state
+            aggregate: b is births.sum()
+            order_by: st desc
+          }
+        }`).toMatchResult(model, {n: stateDesc});
+    }
+  );
+
+  test.when(runtime.supportsNesting)(
+    `one-stage nest orders by a name which needs quoting - ${databaseName}`,
+    async () => {
+      await expect(`
+        run: isabellaN + {
+          nest: n is {
+            group_by: \`select\` is state
+            aggregate: b is births.sum()
+            order_by: \`select\` desc
+          }
+        }`).toMatchResult(model, {n: keywordStateDesc});
+    }
+  );
+
+  test.when(runtime.supportsNesting)(
+    `one-stage nest with no order_by is largest first - ${databaseName}`,
+    async () => {
+      await expect(`
+        run: isabellaN + {
+          nest: n is {
+            group_by: st is state
+            aggregate: b is births.sum()
+          }
+        }`).toMatchResult(model, {n: birthsDesc});
+    }
+  );
+
+  test.when(runtime.dialect.supportsPipelinesInViews)(
+    `pipelined nest is in the order its last stage asked for - ${databaseName}`,
+    async () => {
+      await expect(`
+        run: isabellaN + {
+          nest: n is {
+            group_by: st is state
+            aggregate: b is births.sum()
+            order_by: st asc
+          } -> {
+            group_by: st, b
+            order_by: st desc
+          }
+        }`).toMatchResult(model, {n: stateDesc});
+    }
+  );
+
+  test.when(runtime.dialect.supportsPipelinesInViews)(
+    `pipelined nest orders by a name which needs quoting - ${databaseName}`,
+    async () => {
+      await expect(`
+        run: isabellaN + {
+          nest: n is {
+            group_by: \`select\` is state
+            aggregate: b is births.sum()
+            order_by: \`select\` asc
+          } -> {
+            group_by: \`select\`, b
+            order_by: \`select\` desc
+          }
+        }`).toMatchResult(model, {n: keywordStateDesc});
+    }
+  );
+
+  // An ordinal order_by names an output field by number, counting from one.
+  test.when(runtime.supportsNesting)(
+    `one-stage nest orders by an output field number - ${databaseName}`,
+    async () => {
+      await expect(`
+        run: isabellaN + {
+          nest: n is {
+            group_by: st is state
+            aggregate: b is births.sum()
+            order_by: 1 desc
+          }
+        }`).toMatchResult(model, {n: stateDesc});
+    }
+  );
+
+  test.when(runtime.dialect.supportsPipelinesInViews)(
+    `pipelined nest orders by an output field number - ${databaseName}`,
+    async () => {
+      await expect(`
+        run: isabellaN + {
+          nest: n is {
+            group_by: st is state
+            aggregate: b is births.sum()
+            order_by: st asc
+          } -> {
+            group_by: st, b
+            order_by: 1 desc
+          }
+        }`).toMatchResult(model, {n: stateDesc});
+    }
+  );
+
+  test.when(runtime.dialect.supportsPipelinesInViews)(
+    `pipelined nest ending in select orders by an output field number - ${databaseName}`,
+    async () => {
+      await expect(`
+        run: isabellaN + {
+          nest: n is {
+            group_by: st is state
+            aggregate: b is births.sum()
+            order_by: st asc
+          } -> {
+            select: st, b
+            order_by: 2 desc
+          }
+        }`).toMatchResult(model, {n: birthsDesc});
+    }
+  );
+
+  test.when(runtime.dialect.supportsPipelinesInViews)(
+    `pipelined nest with no order_by is largest first - ${databaseName}`,
+    async () => {
+      await expect(`
+        run: isabellaN + {
+          nest: n is {
+            group_by: st is state
+            aggregate: b is births.sum()
+          } -> {
+            group_by: st
+            aggregate: tot is b.sum()
+          }
+        }`).toMatchResult(model, {n: birthsDesc});
+    }
+  );
+
+  test.when(runtime.supportsNesting)(
+    `an aggregate which is null in a nest reads back as null - ${databaseName}`,
+    async () => {
+      await expect(`
+        run: ${databaseName}.table('malloytest.state_facts') -> {
+          group_by: state
+          nest: ugly is {
+            group_by: popular_name
+            aggregate: foo is NULLIF(sum(airport_count)*0,0)+1
+          }
+        }`).toMatchPaths(model, {'ugly.foo': null});
+    }
+  );
+
+  // A null value in the grouping above a nest must not null out the nest.
+  test.when(runtime.supportsNesting)(
+    `a nest under a null group is still a nest - ${databaseName}`,
+    async () => {
+      const result = await runtime
+        .loadQuery(
+          `
+        run: ${databaseName}.table('malloytest.airports') -> {
+          where: faa_region is null
+          group_by: faa_region
+          aggregate: airport_count is count()
+          nest: by_state is {
+            where: state is not null
+            group_by: state
+            aggregate: airport_count is count()
+          }
+          nest: capped is {
+            where: state is not null
+            group_by: state
+            aggregate: airport_count is count()
+            limit: 1
+          }
+        }`
+        )
+        .run();
+      const row = result.data.toObject()[0] as Record<string, unknown>;
+      expect(row['by_state']).not.toBe(null);
+      expect(row['capped']).not.toBe(null);
+    }
+  );
+});