@malloydata/malloy-tests 0.0.425 → 0.0.427

package/package.json

@@ -20,15 +20,15 @@
     "malloyc": "ts-node ../scripts/malloy-to-json"
   },
   "dependencies": {
-    "@malloydata/db-bigquery": "0.0.425",
-    "@malloydata/db-duckdb": "0.0.425",
-    "@malloydata/db-postgres": "0.0.425",
-    "@malloydata/db-snowflake": "0.0.425",
-    "@malloydata/db-trino": "0.0.425",
-    "@malloydata/malloy": "0.0.425",
-    "@malloydata/malloy-tag": "0.0.425",
-    "@malloydata/render": "0.0.425",
-    "@malloydata/render-validator": "0.0.425",
+    "@malloydata/db-bigquery": "0.0.427",
+    "@malloydata/db-duckdb": "0.0.427",
+    "@malloydata/db-postgres": "0.0.427",
+    "@malloydata/db-snowflake": "0.0.427",
+    "@malloydata/db-trino": "0.0.427",
+    "@malloydata/malloy": "0.0.427",
+    "@malloydata/malloy-tag": "0.0.427",
+    "@malloydata/render": "0.0.427",
+    "@malloydata/render-validator": "0.0.427",
     "events": "^3.3.0",
     "jsdom": "^22.1.0",
     "luxon": "^3.7.2",
@@ -42,5 +42,5 @@
   "overrides": {
     "typescript": "^6.0.3"
   },
-  "version": "0.0.425"
+  "version": "0.0.427"
 }

package/src/core/persist.spec.ts

@@ -2179,4 +2179,194 @@ describe('source persistence', () => {
       expect(sql).toContain('cached.by_carrier');
     });
   });
+
+  // Reported on #3015. Both describes assert the behavior the documentation
+  // describes, so a failure here is the report reproducing.
+  describe('sql_select substitution', () => {
+    // One model, two persisted sources of the two persistable kinds, plus a
+    // third that reaches the sql_select through interpolation.
+    const TWO_KINDS = `${PERSIST_ANNOTATION}
+      ${FLIGHTS_SOURCE}
+
+      #@ persist
+      source: qs is flights -> {
+        group_by: carrier
+        aggregate: flight_count is count()
+      }
+
+      #@ persist
+      source: ss is ${tstDB}.sql("""
+        SELECT carrier, COUNT(*) as flight_count
+        FROM malloytest.flights
+        GROUP BY carrier
+      """)
+
+      #@ persist
+      source: via_interp is ${tstDB}.sql("""
+        SELECT * FROM %{ ss }
+      """)
+    `;
+
+    // A manifest holding a built table for every persist source in the model.
+    async function manifestForAll(): Promise<BuildManifest> {
+      const model = await wrapTestModel(tstRuntime, TWO_KINDS).model.getModel();
+      const digest = await getDigest();
+      const manifest = createManifest();
+      for (const source of Object.values(model.getBuildPlan().sources)) {
+        addManifestEntry(
+          manifest,
+          source.makeBuildId(digest, source.getSQL()),
+          `cached.${source.name}_table`
+        );
+      }
+      return manifest;
+    }
+
+    async function sqlForQuery(query: string): Promise<string> {
+      const manifest = await manifestForAll();
+      return runtimeWithManifest(manifest)
+        .loadQuery(`${TWO_KINDS}\n${query}`)
+        .getSQL();
+    }
+
+    it('query_source reads its built table', async () => {
+      const sql = await sqlForQuery('run: qs -> { select: * }');
+      expect(sql).toContain('cached.qs_table');
+      expect(sql).not.toContain('COUNT(');
+    });
+
+    // Pending #3016 — a directly-referenced sql_select never reaches the
+    // manifest. The two passing tests around these are the controls: the
+    // other persistable kind substitutes, and the same source substitutes
+    // when interpolation reaches it, so the entry and its key are right.
+    it.skip('sql_select reads its built table', async () => {
+      const sql = await sqlForQuery('run: ss -> { select: * }');
+      expect(sql).toContain('cached.ss_table');
+      expect(sql).not.toContain('COUNT(');
+    });
+
+    it('sql_select reached by interpolation reads its built table', async () => {
+      // The manifest key and the entry are right — this path finds them.
+      const sql = await sqlForQuery('run: via_interp -> { select: * }');
+      expect(sql).toContain('cached.ss_table');
+      expect(sql).not.toContain('COUNT(');
+    });
+
+    it.skip('strict mode catches a missing sql_select entry', async () => {
+      // Strict mode's promise is that a persist source with no manifest entry
+      // throws rather than silently compiling to inline SQL.
+      const strict = createManifest();
+      strict.strict = true;
+      await expect(
+        runtimeWithManifest(strict)
+          .loadQuery(`${TWO_KINDS}\nrun: ss -> { select: * }`)
+          .getSQL()
+      ).rejects.toThrow(/not found in manifest/);
+    });
+
+    it.skip('a join of both kinds substitutes both', async () => {
+      const sql = await sqlForQuery(`
+        run: flights extend {
+          join_one: q is qs on carrier = q.carrier
+          join_one: s is ss on carrier = s.carrier
+        } -> {
+          group_by: carrier
+          aggregate: a is q.flight_count.sum(), b is s.flight_count.sum()
+        }
+      `);
+      expect(sql).toContain('cached.qs_table');
+      expect(sql).toContain('cached.ss_table');
+    });
+  });
+
+  describe('per-call buildManifest on ModelMaterializer.loadQuery', () => {
+    const MODEL = `${PERSIST_ANNOTATION}
+      ${FLIGHTS_SOURCE}
+
+      #@ persist
+      source: by_carrier is flights -> {
+        group_by: carrier
+        aggregate: flight_count is count()
+      }
+    `;
+    const QUERY = 'run: by_carrier -> { select: * }';
+
+    async function manifestForByCarrier(): Promise<BuildManifest> {
+      const model = await wrapTestModel(tstRuntime, MODEL).model.getModel();
+      const {manifest} = await buildManifestFor(
+        model.getBuildPlan(),
+        'by_carrier',
+        'cached.by_carrier'
+      );
+      return manifest;
+    }
+
+    function plainRuntime() {
+      return new SingleConnectionRuntime({
+        connection: tstRuntime.connection,
+        urlReader: testFileSpace,
+      });
+    }
+
+    it('applies a manifest passed to the call', async () => {
+      const manifest = await manifestForByCarrier();
+      const sql = await plainRuntime()
+        .loadModel(MODEL)
+        .loadQuery(QUERY, {buildManifest: manifest})
+        .getSQL();
+      expect(sql).toContain('cached.by_carrier');
+    });
+
+    it('applies a manifest passed to loadModel', async () => {
+      const manifest = await manifestForByCarrier();
+      const sql = await plainRuntime()
+        .loadModel(MODEL, {buildManifest: manifest})
+        .loadQuery(QUERY)
+        .getSQL();
+      expect(sql).toContain('cached.by_carrier');
+    });
+
+    it('EMPTY_BUILD_MANIFEST suppresses the runtime manifest', async () => {
+      // The dangerous direction: asking for unsubstituted SQL and being
+      // handed the substituted form.
+      const manifest = await manifestForByCarrier();
+      const sql = await runtimeWithManifest(manifest)
+        .loadModel(MODEL)
+        .loadQuery(QUERY, {buildManifest: EMPTY_BUILD_MANIFEST})
+        .getSQL();
+      expect(sql).not.toContain('cached.by_carrier');
+      expect(sql).toContain('COUNT(');
+    });
+
+    it('loadQueryByName applies a manifest passed to the call', async () => {
+      // The sibling loaders merge their options; this is the control.
+      const manifest = await manifestForByCarrier();
+      const sql = await plainRuntime()
+        .loadModel(`${MODEL}\nquery: named is by_carrier -> { select: * }`)
+        .loadQueryByName('named', {buildManifest: manifest})
+        .getSQL();
+      expect(sql).toContain('cached.by_carrier');
+    });
+
+    it('Runtime.loadQuery applies a manifest passed to the call', async () => {
+      // The other control: the same options on the Runtime entry point.
+      const manifest = await manifestForByCarrier();
+      const sql = await plainRuntime()
+        .loadQuery(`${MODEL}\n${QUERY}`, {buildManifest: manifest})
+        .getSQL();
+      expect(sql).toContain('cached.by_carrier');
+    });
+
+    it('applies defaultRowLimit passed to the call', async () => {
+      // buildManifest is not special: nothing in CompileQueryOptions
+      // survives this call.
+      const sql = await plainRuntime()
+        .loadModel(MODEL)
+        .loadQuery('run: flights -> { select: carrier }', {
+          defaultRowLimit: 7,
+        })
+        .getSQL();
+      expect(sql).toContain('LIMIT 7');
+    });
+  });
 });

package/src/databases/all/streaming.spec.ts

@@ -0,0 +1,159 @@
+/*
+ * Copyright Contributors to the Malloy project
+ * SPDX-License-Identifier: MIT
+ */
+
+import type {DataRecord, WriteStream} from '@malloydata/malloy';
+import {CSVWriter, JSONWriter} from '@malloydata/malloy';
+import {RuntimeList, allDatabases} from '../../runtimes';
+import {databasesFromEnvironmentOr} from '../../util';
+import {TestSelect} from '../../test-select';
+
+class StringAccumulator implements WriteStream {
+  public accumulatedValue = '';
+
+  write(text: string) {
+    this.accumulatedValue += text;
+  }
+
+  close() {
+    return;
+  }
+}
+
+const runtimes = new RuntimeList(databasesFromEnvironmentOr(allDatabases));
+
+afterAll(async () => {
+  await runtimes.closeAll();
+});
+
+runtimes.runtimeMap.forEach((runtime, databaseName) => {
+  // Streaming is an optional connection capability (StreamingConnection).
+  // Dialects which do not implement it skip rather than fail.
+  const streams = runtime.connection.canStream();
+
+  // TestSelect quotes the column alias per dialect, so the name survives
+  // Snowflake's folding of unquoted identifiers to upper case.
+  const ts = new TestSelect(runtime.dialect);
+  const bigintSQL = ts.generate({big_id: ts.mk_bigint(19999)});
+
+  test.when(streams)(`basic stream test  - ${databaseName}`, async () => {
+    const stream = runtime
+      .loadModel(
+        `source: airports is ${databaseName}.table('malloytest.airports')`
+      )
+      .loadQuery('run: airports -> { select: code; order_by: code }')
+      .runStream({rowLimit: 10});
+    const rows: DataRecord[] = [];
+    for await (const row of stream) {
+      rows.push(row);
+    }
+    expect(rows.length).toBe(10);
+    expect(rows[0].cell('code').string.value).toBe('00A');
+  });
+
+  test.when(streams)(`stream to JSON - ${databaseName}`, async () => {
+    const stream = runtime
+      .loadModel(
+        `source: airports is ${databaseName}.table('malloytest.airports')`
+      )
+      .loadQuery('run: airports -> { select: code; order_by: code }')
+      .runStream({rowLimit: 1});
+    const accummulator = new StringAccumulator();
+    const jsonWriter = new JSONWriter(accummulator);
+    await jsonWriter.process(stream);
+    expect(accummulator.accumulatedValue).toBe(
+      `[
+  {
+    "code": "00A"
+  }
+]
+`
+    );
+  });
+
+  test.when(streams)(`stream to CSV - ${databaseName}`, async () => {
+    const stream = runtime
+      .loadModel(
+        `source: airports is ${databaseName}.table('malloytest.airports')`
+      )
+      .loadQuery('run: airports -> { select: code; order_by: code }')
+      .runStream({rowLimit: 1});
+    const accummulator = new StringAccumulator();
+    const csvWriter = new CSVWriter(accummulator);
+    await csvWriter.process(stream);
+    expect(accummulator.accumulatedValue).toBe('code\n00A\n');
+  });
+
+  test.when(streams)(`JSON with timestamp - ${databaseName}`, async () => {
+    const stream = runtime
+      .loadModel(
+        `source: flights is ${databaseName}.table('malloytest.flights')`
+      )
+      .loadQuery(
+        "run: flights -> { select: dep_time; where: carrier = 'WN' and origin = 'SJC'; order_by: dep_time; limit: 1 }"
+      )
+      .runStream({rowLimit: 1});
+    const accummulator = new StringAccumulator();
+    const jsonWriter = new JSONWriter(accummulator);
+    await jsonWriter.process(stream);
+    const result = JSON.parse(accummulator.accumulatedValue);
+    expect(result.length).toBe(1);
+    // Should be an ISO date string
+    expect(typeof result[0].dep_time).toBe('string');
+    expect(result[0].dep_time).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}/);
+  });
+
+  test.when(streams)(`CSV with timestamp - ${databaseName}`, async () => {
+    const stream = runtime
+      .loadModel(
+        `source: flights is ${databaseName}.table('malloytest.flights')`
+      )
+      .loadQuery(
+        "run: flights -> { select: dep_time; where: carrier = 'WN' and origin = 'SJC'; order_by: dep_time; limit: 1 }"
+      )
+      .runStream({rowLimit: 1});
+    const accummulator = new StringAccumulator();
+    const csvWriter = new CSVWriter(accummulator);
+    await csvWriter.process(stream);
+    const lines = accummulator.accumulatedValue.trim().split('\n');
+    expect(lines.length).toBe(2);
+    expect(lines[0]).toBe('dep_time');
+    // Should be formatted as a date string
+    expect(lines[1]).toMatch(/\d{4}-\d{2}-\d{2}/);
+  });
+
+  test.when(streams)(`JSON with bigint - ${databaseName}`, async () => {
+    const stream = runtime
+      .loadModel(
+        `source: bigint_test is ${databaseName}.sql("""${bigintSQL}""")`
+      )
+      .loadQuery('run: bigint_test -> { select: * }')
+      .runStream({rowLimit: 1});
+    const accummulator = new StringAccumulator();
+    const jsonWriter = new JSONWriter(accummulator);
+    await jsonWriter.process(stream);
+    const result = JSON.parse(accummulator.accumulatedValue);
+    expect(result.length).toBe(1);
+    // Bigint should be serialized as a string to preserve precision
+    expect(typeof result[0].big_id).toBe('string');
+    expect(result[0].big_id).toBe('19999');
+  });
+
+  test.when(streams)(`CSV with bigint - ${databaseName}`, async () => {
+    const stream = runtime
+      .loadModel(
+        `source: bigint_test is ${databaseName}.sql("""${bigintSQL}""")`
+      )
+      .loadQuery('run: bigint_test -> { select: * }')
+      .runStream({rowLimit: 1});
+    const accummulator = new StringAccumulator();
+    const csvWriter = new CSVWriter(accummulator);
+    await csvWriter.process(stream);
+    const lines = accummulator.accumulatedValue.trim().split('\n');
+    expect(lines.length).toBe(2);
+    expect(lines[0]).toBe('big_id');
+    // Should be a number without quotes
+    expect(lines[1]).toBe('19999');
+  });
+});

package/src/core/streaming.spec.ts

@@ -1,163 +0,0 @@
-/*
- * Copyright Contributors to the Malloy project
- * SPDX-License-Identifier: MIT
- */
-
-import type {DataRecord, WriteStream} from '@malloydata/malloy';
-import {CSVWriter, JSONWriter} from '@malloydata/malloy';
-import {RuntimeList} from '../runtimes';
-import {describeIfDatabaseAvailable} from '../util';
-
-class StringAccumulator implements WriteStream {
-  public accumulatedValue = '';
-
-  write(text: string) {
-    this.accumulatedValue += text;
-  }
-
-  close() {
-    return;
-  }
-}
-
-const [describe, databases] = describeIfDatabaseAvailable([
-  'bigquery',
-  'postgres',
-  'duckdb',
-  'duckdb_wasm',
-]);
-
-describe('Streaming tests', () => {
-  if (!databases.length) {
-    it.skip('skipped', () => {});
-  }
-  const runtimes = new RuntimeList(databases);
-
-  afterAll(async () => {
-    await runtimes.closeAll();
-  });
-
-  runtimes.runtimeMap.forEach((runtime, databaseName) => {
-    it(`basic stream test  - ${databaseName}`, async () => {
-      const stream = runtime
-        .loadModel(
-          `source: airports is ${databaseName}.table('malloytest.airports')`
-        )
-        .loadQuery('run: airports -> { select: code; order_by: code }')
-        .runStream({rowLimit: 10});
-      const rows: DataRecord[] = [];
-      for await (const row of stream) {
-        rows.push(row);
-      }
-      expect(rows.length).toBe(10);
-      expect(rows[0].cell('code').string.value).toBe('00A');
-    });
-
-    it(`stream to JSON - ${databaseName}`, async () => {
-      const stream = runtime
-        .loadModel(
-          `source: airports is ${databaseName}.table('malloytest.airports')`
-        )
-        .loadQuery('run: airports -> { select: code; order_by: code }')
-        .runStream({rowLimit: 1});
-      const accummulator = new StringAccumulator();
-      const jsonWriter = new JSONWriter(accummulator);
-      await jsonWriter.process(stream);
-      expect(accummulator.accumulatedValue).toBe(
-        `[
-  {
-    "code": "00A"
-  }
-]
-`
-      );
-    });
-
-    it(`stream to CSV - ${databaseName}`, async () => {
-      const stream = runtime
-        .loadModel(
-          `source: airports is ${databaseName}.table('malloytest.airports')`
-        )
-        .loadQuery('run: airports -> { select: code; order_by: code }')
-        .runStream({rowLimit: 1});
-      const accummulator = new StringAccumulator();
-      const csvWriter = new CSVWriter(accummulator);
-      await csvWriter.process(stream);
-      expect(accummulator.accumulatedValue).toBe('code\n00A\n');
-    });
-
-    it(`JSON with timestamp - ${databaseName}`, async () => {
-      const stream = runtime
-        .loadModel(
-          `source: flights is ${databaseName}.table('malloytest.flights')`
-        )
-        .loadQuery(
-          "run: flights -> { select: dep_time; where: carrier = 'WN' and origin = 'SJC'; order_by: dep_time; limit: 1 }"
-        )
-        .runStream({rowLimit: 1});
-      const accummulator = new StringAccumulator();
-      const jsonWriter = new JSONWriter(accummulator);
-      await jsonWriter.process(stream);
-      const result = JSON.parse(accummulator.accumulatedValue);
-      expect(result.length).toBe(1);
-      // Should be an ISO date string
-      expect(typeof result[0].dep_time).toBe('string');
-      expect(result[0].dep_time).toMatch(
-        /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}/
-      );
-    });
-
-    it(`CSV with timestamp - ${databaseName}`, async () => {
-      const stream = runtime
-        .loadModel(
-          `source: flights is ${databaseName}.table('malloytest.flights')`
-        )
-        .loadQuery(
-          "run: flights -> { select: dep_time; where: carrier = 'WN' and origin = 'SJC'; order_by: dep_time; limit: 1 }"
-        )
-        .runStream({rowLimit: 1});
-      const accummulator = new StringAccumulator();
-      const csvWriter = new CSVWriter(accummulator);
-      await csvWriter.process(stream);
-      const lines = accummulator.accumulatedValue.trim().split('\n');
-      expect(lines.length).toBe(2);
-      expect(lines[0]).toBe('dep_time');
-      // Should be formatted as a date string
-      expect(lines[1]).toMatch(/\d{4}-\d{2}-\d{2}/);
-    });
-
-    it(`JSON with bigint - ${databaseName}`, async () => {
-      const stream = runtime
-        .loadModel(
-          `source: bigint_test is ${databaseName}.sql("SELECT CAST(19999 AS BIGINT) as big_id")`
-        )
-        .loadQuery('run: bigint_test -> { select: * }')
-        .runStream({rowLimit: 1});
-      const accummulator = new StringAccumulator();
-      const jsonWriter = new JSONWriter(accummulator);
-      await jsonWriter.process(stream);
-      const result = JSON.parse(accummulator.accumulatedValue);
-      expect(result.length).toBe(1);
-      // Bigint should be serialized as a string to preserve precision
-      expect(typeof result[0].big_id).toBe('string');
-      expect(result[0].big_id).toBe('19999');
-    });
-
-    it(`CSV with bigint - ${databaseName}`, async () => {
-      const stream = runtime
-        .loadModel(
-          `source: bigint_test is ${databaseName}.sql("SELECT CAST(19999 AS BIGINT) as big_id")`
-        )
-        .loadQuery('run: bigint_test -> { select: * }')
-        .runStream({rowLimit: 1});
-      const accummulator = new StringAccumulator();
-      const csvWriter = new CSVWriter(accummulator);
-      await csvWriter.process(stream);
-      const lines = accummulator.accumulatedValue.trim().split('\n');
-      expect(lines.length).toBe(2);
-      expect(lines[0]).toBe('big_id');
-      // Should be a number without quotes
-      expect(lines[1]).toBe('19999');
-    });
-  });
-});