@malloydata/malloy-tests 0.0.432 → 0.0.434

package/CONTEXT.md

@@ -140,7 +140,7 @@ These commands are optimized for CI and may not work correctly in local developm
 
 ### The test corpus
 
-The shared cross-database parquet files live in `test/data/malloytest-parquet/` (~60 MB total, checked into git). These are the source of truth — every dialect loads from or mirrors this data. DuckDB-only files remain in `test/data/duckdb/`.
+The shared cross-database parquet files live in `test/data/malloytest-parquet/` (~15 MB, checked into git). They are the only description of the test data: every database is loaded from them by a script in the repo (BigQuery excepted, see below). When the parquet changes, the loaders run again; there is no dump to edit. DuckDB-only files remain in `test/data/duckdb/`.
 
 #### Shared parquets (`test/data/malloytest-parquet/`)
 
@@ -171,14 +171,23 @@ All cross-database tests (`test/src/databases/all/`) reference tables as `malloy
 
 | Dialect | Method | Files | Notes |
 |---|---|---|---|
-| DuckDB | TS script: `CREATE TABLE AS SELECT FROM parquet_scan()` | `test/duckdb/load_test_data.sh` (wraps `load_test_data.ts`) | Run via `sh test/duckdb/load_test_data.sh` (or `npm run build-duckdb-db`). Creates `test/data/duckdb/duckdb_test.db` |
-| PostgreSQL | Compressed SQL dump loaded via Docker `psql` | `test/data/postgres/malloytest-postgres.sql.gz` | Docker script: `test/postgres/postgres_start.sh` |
-| MySQL | Compressed SQL dump loaded via Docker | `test/data/mysql/malloytest.mysql.gz` | Docker script: `test/mysql/mysql_start.sh` |
+| DuckDB | DuckDB `CREATE TABLE AS SELECT FROM read_parquet()` | `test/duckdb/load_test_data.sh` (wraps `load_test_data.ts`) | Run via `sh test/duckdb/load_test_data.sh` (or `npm run build-duckdb-db`). Creates `test/data/duckdb/duckdb_test.db` |
+| PostgreSQL | DuckDB's `postgres` extension: `ATTACH` the server, `CREATE TABLE AS SELECT FROM read_parquet()` | `test/postgres/load_test_data.sh` (wraps `load_test_data.ts`) | Run by `test/postgres/postgres_start.sh` and by CI against a running server; connects with the `PG*` variables. No `ga_sample` (no anonymous record type) |
+| MySQL | DuckDB's `mysql` extension, likewise | `test/mysql/load_test_data.sh` (wraps `load_test_data.ts`) | Run by `test/mysql/mysql_start.sh`; connects with the `MYSQL_*` variables. No `ga_sample`, and `alltypes` without its array columns |
+| SQL Server | DuckDB's community `mssql` extension, likewise | `test/mssql/load_test_data.sh` (wraps `load_test_data.ts`) | Run by `test/mssql/mssql_start.sh`; connects as the test runtime does (`sa`, localhost:1433). Waits for the server itself. Same table set as MySQL. Not in CI |
+| Trino/Presto | The hive connector reads the parquet in place: each file is mounted into the container and declared as an external table | `test/trino/hive_ddl.ts`, `test/trino/hive.properties`, `test/presto/hive.properties` | `trino_start.sh` / `presto_start.sh` derive the DDL from the parquet schemas at container start. The hive metastore lowercases names, so a table with nested columns (`ga_sample`) is a `_hive` table plus a view that casts the nested field names back; top-level column names stay lowercase, since Trino lowercases those in every connector |
 | BigQuery | Pre-loaded manually in `malloydata-org` project | (none) | No loader script in repo. Data assumed to exist. |
 | Snowflake | SQL: `PUT` local parquet → stage, `COPY INTO` table | `test/snowflake/load_test_data.sh` (wraps `load_test_data.sql`) | Run via `sh test/snowflake/load_test_data.sh` (needs the `snowsql` CLI) |
-| Trino/Presto | Docker containers with pre-loaded data | `test/trino/trino_start.sh` | Uses Docker volumes |
 | Databricks | TS script: upload to Volume via REST, `CREATE TABLE AS SELECT FROM read_files()` | `test/databricks/load_test_data.sh` (wraps `load_test_data.ts`) | Run manually with the `DATABRICKS_*` env vars set: `sh test/databricks/load_test_data.sh` |
 
+The DuckDB-driven loaders share `test/data/parquet_loader.ts`, which runs everything through Malloy's own `DuckDBConnection`, so a database is built by the same DuckDB the tests read it with. It reads the table list from the parquet directory and takes a `LoadPlan` naming what a target cannot hold.
+
+### The flights timestamps
+
+`flights.dep_time` and `arr_time` are UTC instants whose digits are the local clock at the departure and arrival airport. The source recorded each airport's wall clock and the zone was not kept, so a 6:45 pm departure from LAX is the instant 18:45 UTC. The parquet declares them so (`tz=UTC`), every dialect holds them as instants, and under Malloy's UTC session pin every dialect renders the airport clock. An instant is the one time type every dialect can hold; Malloy has no wall-clock type, and several dialects have none either. Elapsed time between the two columns is not computable from them on a route that crosses zones; `flight_time`, `taxi_out` and `taxi_in` carry it.
+
+A loader must carry the instant through unchanged. Malloy's `DuckDBConnection`, which the loaders run through, pins its session zone to UTC, so a target without a zoned type (MySQL `datetime`) receives the same digits, where a loader in the machine's zone would shift them. Trino and Presto read the parquet's UTC microseconds directly.
+
 ### Cloud warehouse considerations
 
 Cloud SQL warehouses (BigQuery, Snowflake, Databricks) can't read local files via SQL. Each needs a mechanism to get parquet data into the warehouse:

package/data/parquet_loader.ts

@@ -0,0 +1,106 @@
+/*
+ * Copyright Contributors to the Malloy project
+ * SPDX-License-Identifier: MIT
+ */
+
+/* eslint-disable no-console */
+
+/*
+ * The malloytest tables are the parquet files in test/data/malloytest-parquet,
+ * one table per file. Each database's load_test_data script copies them into
+ * that database through DuckDB, so the parquet is the only description of the
+ * test data. See "Test Data" in test/CONTEXT.md.
+ */
+
+import type {QueryRecord} from '@malloydata/malloy';
+import * as fs from 'fs';
+import * as path from 'path';
+import {DuckDBConnection} from '../../packages/malloy-db-duckdb';
+
+export const parquetDir = path.resolve(__dirname, 'malloytest-parquet');
+
+export function parquetTables(): string[] {
+  return fs
+    .readdirSync(parquetDir)
+    .filter(file => file.endsWith('.parquet'))
+    .map(file => file.slice(0, -'.parquet'.length))
+    .sort();
+}
+
+export function parquetPath(table: string): string {
+  return path.join(parquetDir, `${table}.parquet`);
+}
+
+export interface LoadPlan {
+  /** Tables the target cannot hold at all */
+  skip?: string[];
+  /** Columns to leave out of a table, for types the target lacks */
+  omit?: {table: string; columns: string[]}[];
+}
+
+/** The plan for a target with no array or record types */
+export const scalarTypesOnly: LoadPlan = {
+  skip: ['ga_sample'],
+  omit: [{table: 'alltypes', columns: ['t_array_string', 't_array_int64']}],
+};
+
+/** A DuckDB session that runs SQL and returns the rows */
+export interface TestDataDuckDB {
+  run(sql: string): Promise<QueryRecord[]>;
+  close(): Promise<void>;
+}
+
+/**
+ * Malloy's own DuckDB connection, so a database is built by the same DuckDB
+ * the tests read it with. That connection pins the session zone to UTC,
+ * which a loader needs: the parquet timestamps are instants, and a target
+ * without a zoned timestamp type must receive their UTC digits.
+ */
+export async function openDuckDB(
+  databasePath = ':memory:'
+): Promise<TestDataDuckDB> {
+  const connection = new DuckDBConnection('loader', databasePath);
+  return {
+    run: async sql => (await connection.runRawSQL(sql)).rows,
+    close: () => connection.close(),
+  };
+}
+
+export async function loadExtension(
+  db: TestDataDuckDB,
+  extension: string,
+  repository?: 'community'
+): Promise<void> {
+  const from = repository ? ` FROM ${repository}` : '';
+  await db.run(`INSTALL ${extension}${from}`);
+  await db.run(`LOAD ${extension}`);
+}
+
+/**
+ * Replace `schema` with the malloytest tables. `schema` may live in an
+ * attached database (`pg.malloytest`).
+ */
+export async function loadMalloytest(
+  db: TestDataDuckDB,
+  schema: string,
+  plan: LoadPlan = {}
+): Promise<void> {
+  await db.run(`DROP SCHEMA IF EXISTS ${schema} CASCADE`);
+  await db.run(`CREATE SCHEMA ${schema}`);
+  for (const table of parquetTables()) {
+    if (plan.skip?.includes(table)) {
+      console.log(`  ${table}: skipped`);
+      continue;
+    }
+    const omit = plan.omit?.find(o => o.table === table)?.columns ?? [];
+    const columns =
+      omit.length > 0
+        ? `* EXCLUDE (${omit.map(c => `"${c}"`).join(', ')})`
+        : '*';
+    await db.run(
+      `CREATE TABLE ${schema}.${table} AS SELECT ${columns} FROM read_parquet('${parquetPath(table)}')`
+    );
+    const rows = await db.run(`SELECT count(*) AS n FROM ${schema}.${table}`);
+    console.log(`  ${table}: ${rows[0]['n']} rows`);
+  }
+}

package/duckdb/load_test_data.ts

@@ -5,57 +5,24 @@
 
 /* eslint-disable no-console */
 
-import {DuckDBConnection} from '../../packages/malloy-db-duckdb';
-
-import fs from 'fs';
-
-const parquetDir = './test/data/malloytest-parquet/';
-const duckdbDir = './test/data/duckdb/';
-const databasePath = `${duckdbDir}duckdb_test.db`;
-if (fs.existsSync(databasePath)) {
-  console.log(`Database at ${databasePath} already exists, removing`);
-  fs.rmSync(databasePath);
-}
-
-const database = new DuckDBConnection('duckdb', databasePath);
+/*
+ * Build the DuckDB test database, test/data/duckdb/duckdb_test.db, from the
+ * parquet files.
+ */
 
-const run = (sql: string) => {
-  return database.runRawSQL(sql);
-};
+import * as fs from 'fs';
+import * as path from 'path';
+import {loadMalloytest, openDuckDB} from '../data/parquet_loader';
 
-console.log(`Creating database at ${databasePath}`);
+const databasePath = path.resolve(__dirname, '../data/duckdb/duckdb_test.db');
 
 (async () => {
-  try {
-    await run('CREATE SCHEMA malloytest');
-    await run(
-      `CREATE TABLE malloytest.aircraft AS SELECT * FROM parquet_scan('${parquetDir}aircraft.parquet')`
-    );
-    await run(
-      `CREATE TABLE malloytest.aircraft_models AS SELECT * FROM parquet_scan('${parquetDir}aircraft_models.parquet')`
-    );
-    await run(
-      `CREATE TABLE malloytest.airports AS SELECT * FROM parquet_scan('${parquetDir}airports.parquet')`
-    );
-    await run(
-      `CREATE TABLE malloytest.alltypes AS SELECT * FROM parquet_scan('${parquetDir}alltypes.parquet')`
-    );
-    await run(
-      `CREATE TABLE malloytest.carriers AS SELECT * FROM parquet_scan('${parquetDir}carriers.parquet')`
-    );
-    await run(
-      `CREATE TABLE malloytest.flights AS SELECT * FROM parquet_scan('${parquetDir}flights.parquet')`
-    );
-    await run(
-      `CREATE TABLE malloytest.state_facts AS SELECT * FROM parquet_scan('${parquetDir}state_facts.parquet')`
-    );
-    await run(
-      `CREATE TABLE malloytest.ga_sample AS SELECT * FROM parquet_scan('${parquetDir}ga_sample.parquet')`
-    );
-    console.log('Finished populating database with data from parqeut files');
-  } catch (e) {
-    console.log(e);
+  if (fs.existsSync(databasePath)) {
+    console.log(`Database at ${databasePath} already exists, removing`);
+    fs.rmSync(databasePath);
   }
-
-  await database.close();
+  console.log(`Creating database at ${databasePath}`);
+  const loader = await openDuckDB(databasePath);
+  await loadMalloytest(loader, 'malloytest');
+  await loader.close();
 })();

package/mssql/connection_string.ts

@@ -0,0 +1,12 @@
+/*
+ * Copyright Contributors to the Malloy project
+ * SPDX-License-Identifier: MIT
+ */
+
+/**
+ * The SQL Server the mssql tests and loader connect to: the container
+ * mssql_start.sh runs, with the password it sets.
+ */
+export function mssqlConnectionString(database: string): string {
+  return `Server=localhost;Port=1433;Database=${database};User Id=sa;Password=Malloy_Test_123;TrustServerCertificate=true`;
+}

package/mssql/load_test_data.sh

@@ -0,0 +1,12 @@
+#! /bin/bash
+#
+# Load the malloytest tables into a running SQL Server from
+# test/data/malloytest-parquet. Waits for the server itself.
+#
+set -e
+
+SCRIPTDIR=$(cd $(dirname $0); pwd)
+ROOTDIR=$(cd "$SCRIPTDIR/../.."; pwd)
+
+cd "$ROOTDIR"
+npx ts-node test/mssql/load_test_data.ts

package/mssql/load_test_data.ts

@@ -0,0 +1,63 @@
+/*
+ * Copyright Contributors to the Malloy project
+ * SPDX-License-Identifier: MIT
+ */
+
+/* eslint-disable no-console */
+
+/*
+ * Load the malloytest tables into a running SQL Server from the parquet
+ * files, through DuckDB's mssql extension. Connects as the mssql test
+ * connection does. Waits for the server first, since the container image
+ * ships no client to wait with.
+ */
+
+import {
+  loadExtension,
+  loadMalloytest,
+  openDuckDB,
+  scalarTypesOnly,
+} from '../data/parquet_loader';
+import {mssqlConnectionString} from './connection_string';
+
+const database = 'malloytest';
+
+function sleep(ms: number): Promise<void> {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+(async () => {
+  const db = await openDuckDB();
+  await loadExtension(db, 'mssql', 'community');
+
+  // Attach the always-present master database, retrying while the server
+  // comes up, then recreate the test database from scratch.
+  const deadline = Date.now() + 120_000;
+  for (;;) {
+    try {
+      await db.run(
+        `ATTACH '${mssqlConnectionString('master')}' AS ms (TYPE mssql)`
+      );
+      break;
+    } catch (e) {
+      if (Date.now() > deadline) {
+        throw e;
+      }
+      const reason = e instanceof Error ? e.message.split('\n')[0] : `${e}`;
+      console.log(`  waiting for SQL Server: ${reason}`);
+      await sleep(2000);
+    }
+  }
+  await db.run(
+    `SELECT mssql_exec('ms', 'IF DB_ID(''${database}'') IS NOT NULL DROP DATABASE ${database}')`
+  );
+  await db.run(`SELECT mssql_exec('ms', 'CREATE DATABASE ${database}')`);
+  await db.run('DETACH ms');
+  await db.run(
+    `ATTACH '${mssqlConnectionString(database)}' AS ms (TYPE mssql)`
+  );
+
+  console.log('Loading malloytest into mssql');
+  await loadMalloytest(db, `ms.${database}`, scalarTypesOnly);
+  await db.close();
+})();

package/mssql/mssql_start.sh

@@ -1,17 +1,14 @@
 #!/bin/bash
+#
+# Start a SQL Server container and load the malloytest tables into it from
+# test/data/malloytest-parquet.
+#
 set -e
 
 SCRIPTDIR=$(cd $(dirname $0); pwd)
-DATADIR=$(dirname $SCRIPTDIR)/data/malloytest-parquet
 CONTAINER_NAME="mssql-malloy"
+# Must match test/mssql/connection_string.ts
 SA_PASSWORD="Malloy_Test_123"
-DB_NAME="malloytest"
-
-# Require duckdb CLI (used for readiness check and data loading)
-if ! command -v duckdb > /dev/null 2>&1; then
-  echo "Error: duckdb CLI is required but not found"
-  exit 1
-fi
 
 # Check for existing container
 if docker container inspect "$CONTAINER_NAME" > /dev/null 2>&1; then
@@ -21,13 +18,8 @@ if docker container inspect "$CONTAINER_NAME" > /dev/null 2>&1; then
   fi
   echo "Restarting existing $CONTAINER_NAME container..."
   docker start "$CONTAINER_NAME"
-  echo -n "Waiting for MSSQL..."
-  for i in $(seq 1 60); do
-    duckdb -c "LOAD mssql; ATTACH 'Server=localhost;Port=1433;Database=master;User Id=sa;Password=$SA_PASSWORD;TrustServerCertificate=true' AS _ping (TYPE mssql);" > /dev/null 2>&1 && break
-    echo -n "."
-    sleep 2
-  done
-  echo " ready"
+  # The loader waits for the server; reloading is cheap
+  sh "$SCRIPTDIR/load_test_data.sh"
   echo "MSSQL running on port 1433"
   exit 0
 fi
@@ -48,122 +40,11 @@ docker run -d \
   -p 1433:1433 \
   "$IMAGE"
 
-# Wait for server to accept connections (check via DuckDB mssql extension)
-counter=0
-echo -n "Waiting for MSSQL..."
-while true; do
-  duckdb -c "LOAD mssql; ATTACH 'Server=localhost;Port=1433;Database=master;User Id=sa;Password=$SA_PASSWORD;TrustServerCertificate=true' AS _ping (TYPE mssql);" > /dev/null 2>&1 && break
-  counter=$((counter + 1))
-  if [ $counter -ge 60 ]; then
-    echo
-    echo "MSSQL did not start within 2 minutes"
-    docker logs "$CONTAINER_NAME" 2>&1 | tail -20
-    docker rm -f "$CONTAINER_NAME"
-    exit 1
-  fi
-  echo -n "."
-  sleep 2
-done
-echo " ready"
-
-# Load test data using DuckDB + mssql extension
+# The loader waits for the server. A container without its tables must not
+# survive to be "already running".
+trap 'docker rm -f "$CONTAINER_NAME" > /dev/null; echo "MSSQL setup failed"' ERR
 echo "Loading test data..."
-duckdb -c "
-LOAD mssql;
-
--- Create database
-ATTACH 'Server=localhost;Port=1433;Database=master;User Id=sa;Password=$SA_PASSWORD;TrustServerCertificate=true' AS msdb (TYPE mssql);
-SELECT mssql_exec('msdb', 'IF DB_ID(''$DB_NAME'') IS NOT NULL DROP DATABASE $DB_NAME');
-SELECT mssql_exec('msdb', 'CREATE DATABASE $DB_NAME');
-DETACH msdb;
-
--- Connect to test database
-ATTACH 'Server=localhost;Port=1433;Database=$DB_NAME;User Id=sa;Password=$SA_PASSWORD;TrustServerCertificate=true' AS msdb (TYPE mssql);
-
--- Create malloytest schema so table paths (malloytest.flights etc.) match DuckDB tests
-SELECT mssql_exec('msdb', 'CREATE SCHEMA malloytest');
-
--- Create tables: schemas must match the parquet files in test/data/malloytest-parquet/.
--- If a parquet schema changes, update the corresponding CREATE TABLE below.
--- All columns explicitly nullable to match parquet semantics.
-SELECT mssql_exec('msdb', 'CREATE TABLE malloytest.aircraft (
-  id BIGINT NULL, tail_num NVARCHAR(MAX) NULL, aircraft_serial NVARCHAR(MAX) NULL,
-  aircraft_model_code NVARCHAR(MAX) NULL, aircraft_engine_code NVARCHAR(MAX) NULL,
-  year_built INT NULL, aircraft_type_id INT NULL, aircraft_engine_type_id INT NULL,
-  registrant_type_id INT NULL, name NVARCHAR(MAX) NULL, address1 NVARCHAR(MAX) NULL,
-  address2 NVARCHAR(MAX) NULL, city NVARCHAR(MAX) NULL, state NVARCHAR(MAX) NULL,
-  zip NVARCHAR(MAX) NULL, region NVARCHAR(MAX) NULL, county NVARCHAR(MAX) NULL,
-  country NVARCHAR(MAX) NULL, certification NVARCHAR(MAX) NULL, status_code NVARCHAR(MAX) NULL,
-  mode_s_code NVARCHAR(MAX) NULL, fract_owner NVARCHAR(MAX) NULL,
-  last_action_date NVARCHAR(MAX) NULL, cert_issue_date NVARCHAR(MAX) NULL,
-  air_worth_date NVARCHAR(MAX) NULL
-)');
-
-SELECT mssql_exec('msdb', 'CREATE TABLE malloytest.aircraft_models (
-  aircraft_model_code NVARCHAR(MAX) NULL, manufacturer NVARCHAR(MAX) NULL,
-  model NVARCHAR(MAX) NULL, aircraft_type_id INT NULL, aircraft_engine_type_id INT NULL,
-  aircraft_category_id INT NULL, amateur INT NULL, engines INT NULL,
-  seats INT NULL, weight INT NULL, speed INT NULL
-)');
-
-SELECT mssql_exec('msdb', 'CREATE TABLE malloytest.airports (
-  id INT NULL, code NVARCHAR(MAX) NULL, site_number NVARCHAR(MAX) NULL,
-  fac_type NVARCHAR(MAX) NULL, fac_use NVARCHAR(MAX) NULL, faa_region NVARCHAR(MAX) NULL,
-  faa_dist NVARCHAR(MAX) NULL, city NVARCHAR(MAX) NULL, county NVARCHAR(MAX) NULL,
-  state NVARCHAR(MAX) NULL, full_name NVARCHAR(MAX) NULL, own_type NVARCHAR(MAX) NULL,
-  longitude FLOAT NULL, latitude FLOAT NULL, elevation INT NULL,
-  aero_cht NVARCHAR(MAX) NULL, cbd_dist INT NULL, cbd_dir NVARCHAR(MAX) NULL,
-  act_date NVARCHAR(MAX) NULL, cert NVARCHAR(MAX) NULL, fed_agree NVARCHAR(MAX) NULL,
-  cust_intl NVARCHAR(MAX) NULL, c_ldg_rts NVARCHAR(MAX) NULL, joint_use NVARCHAR(MAX) NULL,
-  mil_rts NVARCHAR(MAX) NULL, cntl_twr NVARCHAR(MAX) NULL, major NVARCHAR(MAX) NULL
-)');
-
-SELECT mssql_exec('msdb', 'CREATE TABLE malloytest.carriers (
-  code NVARCHAR(MAX) NULL, name NVARCHAR(MAX) NULL, nickname NVARCHAR(MAX) NULL
-)');
-
-SELECT mssql_exec('msdb', 'CREATE TABLE malloytest.flights (
-  carrier NVARCHAR(MAX) NULL, origin NVARCHAR(MAX) NULL, destination NVARCHAR(MAX) NULL,
-  flight_num INT NULL, flight_time INT NULL, tail_num NVARCHAR(MAX) NULL,
-  dep_time DATETIME NULL, arr_time DATETIME NULL, dep_delay INT NULL,
-  arr_delay INT NULL, taxi_out INT NULL, taxi_in INT NULL, distance INT NULL,
-  cancelled NVARCHAR(MAX) NULL, diverted NVARCHAR(MAX) NULL, id2 INT NULL
-)');
-
-SELECT mssql_exec('msdb', 'CREATE TABLE malloytest.state_facts (
-  state NVARCHAR(MAX) NULL, aircraft_count INT NULL, airport_count INT NULL,
-  births INT NULL, popular_name NVARCHAR(MAX) NULL
-)');
-
-SELECT mssql_exec('msdb', 'CREATE TABLE malloytest.alltypes (
-  t_int64 BIGINT NULL, t_float64 FLOAT NULL,
-  t_numeric DECIMAL(38,9) NULL, t_bignumeric DECIMAL(38,9) NULL,
-  string NVARCHAR(MAX) NULL,
-  t_bool_true BIT NULL, t_bool_false BIT NULL, t_bool_null BIT NULL,
-  t_date DATE NULL, t_datetime DATETIME NULL, t_timestamp DATETIME NULL
-)');
-
-SELECT mssql_refresh_cache('msdb');
-
--- Load data from parquet files
-INSERT INTO msdb.malloytest.aircraft SELECT * FROM read_parquet('$DATADIR/aircraft.parquet');
-INSERT INTO msdb.malloytest.aircraft_models SELECT * FROM read_parquet('$DATADIR/aircraft_models.parquet');
-INSERT INTO msdb.malloytest.airports SELECT * FROM read_parquet('$DATADIR/airports.parquet');
-INSERT INTO msdb.malloytest.carriers SELECT * FROM read_parquet('$DATADIR/carriers.parquet');
-INSERT INTO msdb.malloytest.flights SELECT * FROM read_parquet('$DATADIR/flights.parquet');
-INSERT INTO msdb.malloytest.state_facts SELECT * FROM read_parquet('$DATADIR/state_facts.parquet');
-INSERT INTO msdb.malloytest.alltypes SELECT t_int64, t_float64, t_numeric, t_bignumeric, string, t_bool_true, t_bool_false, t_bool_null, t_date, t_datetime, t_timestamp FROM read_parquet('$DATADIR/alltypes.parquet');
-
--- Verify row counts
-SELECT 'aircraft' as tbl, COUNT(*) as rows FROM msdb.malloytest.aircraft
-UNION ALL SELECT 'aircraft_models', COUNT(*) FROM msdb.malloytest.aircraft_models
-UNION ALL SELECT 'airports', COUNT(*) FROM msdb.malloytest.airports
-UNION ALL SELECT 'carriers', COUNT(*) FROM msdb.malloytest.carriers
-UNION ALL SELECT 'flights', COUNT(*) FROM msdb.malloytest.flights
-UNION ALL SELECT 'state_facts', COUNT(*) FROM msdb.malloytest.state_facts
-UNION ALL SELECT 'alltypes', COUNT(*) FROM msdb.malloytest.alltypes;
-"
+sh "$SCRIPTDIR/load_test_data.sh"
+trap - ERR
 
-echo
-echo "MSSQL running on port 1433, database: $DB_NAME"
-echo "  Connection: Server=localhost;Port=1433;Database=$DB_NAME;User Id=sa;Password=$SA_PASSWORD;TrustServerCertificate=true"
+echo "MSSQL running on port 1433, database: malloytest"

package/mysql/load_test_data.sh

@@ -0,0 +1,12 @@
+#! /bin/bash
+#
+# Load the malloytest tables into a running mysql from
+# test/data/malloytest-parquet. Connects with the MYSQL_* variables.
+#
+set -e
+
+SCRIPTDIR=$(cd $(dirname $0); pwd)
+ROOTDIR=$(cd "$SCRIPTDIR/../.."; pwd)
+
+cd "$ROOTDIR"
+npx ts-node test/mysql/load_test_data.ts

package/mysql/load_test_data.ts

@@ -0,0 +1,39 @@
+/*
+ * Copyright Contributors to the Malloy project
+ * SPDX-License-Identifier: MIT
+ */
+
+/* eslint-disable no-console */
+
+/*
+ * Load the malloytest tables into a running mysql from the parquet files.
+ * Connects with the same MYSQL_* variables, read the same way, as the mysql
+ * test connection.
+ */
+
+import {MySQLExecutor} from '../../packages/malloy-db-mysql';
+import {
+  loadExtension,
+  loadMalloytest,
+  openDuckDB,
+  scalarTypesOnly,
+} from '../data/parquet_loader';
+
+const options = MySQLExecutor.getConnectionOptionsFromEnv();
+const attach = [
+  ...(options.host ? [`host=${options.host}`] : []),
+  ...(options.port ? [`port=${options.port}`] : []),
+  ...(options.user ? [`user=${options.user}`] : []),
+  ...(options.password ? [`password=${options.password}`] : []),
+  // The system database is always there; the load creates malloytest itself
+  'database=mysql',
+].join(' ');
+
+(async () => {
+  const db = await openDuckDB();
+  await loadExtension(db, 'mysql');
+  await db.run(`ATTACH '${attach}' AS my (TYPE mysql)`);
+  console.log('Loading malloytest into mysql');
+  await loadMalloytest(db, 'my.malloytest', scalarTypesOnly);
+  await db.close();
+})();

package/mysql/mysql_start.sh

@@ -4,11 +4,31 @@ set -e
 rm -rf .tmp
 mkdir .tmp
 
-# run docker
 SCRIPTDIR=$(cd $(dirname $0); pwd)
-DATADIR=$(dirname $SCRIPTDIR)/data/mysql
 CONTAINER_NAME="mysql-malloy"
 
+# The image starts mysqld twice: a temporary server to initialize the data
+# directory, listening on the unix socket only, and then the real server on
+# 3306. Both log "mysqld: ready for connections", so the log cannot say which
+# one is up. Wait on a TCP connection, which is what the loader needs.
+wait_for_mysql() {
+  local counter=0
+  echo -n "Waiting for $CONTAINER_NAME ..."
+  until docker exec "$CONTAINER_NAME" mysqladmin ping --protocol=TCP -h127.0.0.1 -uroot --silent > /dev/null 2>&1
+  do
+    counter=$((counter+1))
+    # give up after 2 minutes
+    if [ $counter -eq 60 ]
+    then
+      echo
+      return 1
+    fi
+    echo -n .
+    sleep 2
+  done
+  echo
+}
+
 # Check for existing container
 if docker container inspect "$CONTAINER_NAME" > /dev/null 2>&1; then
   if [ "$(docker inspect -f '{{.State.Running}}' "$CONTAINER_NAME")" = "true" ]; then
@@ -17,39 +37,25 @@ if docker container inspect "$CONTAINER_NAME" > /dev/null 2>&1; then
   fi
   echo "Restarting existing $CONTAINER_NAME container..."
   docker start "$CONTAINER_NAME"
-  while ! docker logs "$CONTAINER_NAME" 2>&1 | grep -q "mysqld: ready for connections"; do
-    sleep 2
-  done
+  if ! wait_for_mysql; then
+    docker logs "$CONTAINER_NAME" >& ./.tmp/mysql-malloy.logs
+    echo "MySQL did not restart successfully, check .tmp/mysql-malloy.logs"
+    exit 1
+  fi
   echo "MySQL running on port 3306"
   exit 0
 fi
 
-docker run -p 3306:3306 -d -v $DATADIR:/init_data --name "$CONTAINER_NAME" -e MYSQL_ALLOW_EMPTY_PASSWORD=yes -d mysql:8.4.2
-
-# wait for server to start
-counter=0
-echo -n Starting Docker ...
-while ! docker logs "$CONTAINER_NAME" 2>&1 | grep -q "mysqld: ready for connections"
-do
-  sleep 10
-  counter=$((counter+1))
-  # if doesn't start after 2 minutes, output logs and kill process
-  if [ $counter -eq 120 ]
-  then
-    docker logs "$CONTAINER_NAME" >& ./.tmp/mysql-malloy.logs
-    docker rm -f "$CONTAINER_NAME"
-    echo "MySQL did not start successfully, check .tmp/mysql-malloy.logs"
-    exit 1
-    break
-  fi
-  echo -n ...
-done
+docker run -p 3306:3306 -d --name "$CONTAINER_NAME" -e MYSQL_ALLOW_EMPTY_PASSWORD=yes mysql:8.4.2
+
+if ! wait_for_mysql; then
+  docker logs "$CONTAINER_NAME" >& ./.tmp/mysql-malloy.logs
+  docker rm -f "$CONTAINER_NAME"
+  echo "MySQL did not start successfully, check .tmp/mysql-malloy.logs"
+  exit 1
+fi
 
-# load the test data.
-echo
 echo Loading Test Data
-docker exec "$CONTAINER_NAME" cp /init_data/malloytest.mysql.gz /tmp
-docker exec "$CONTAINER_NAME" gunzip /tmp/malloytest.mysql.gz
-docker exec "$CONTAINER_NAME" mysql -P3306 -h127.0.0.1 -uroot -e 'drop database if exists malloytest; create database malloytest; use malloytest; source /tmp/malloytest.mysql;'
+MYSQL_HOST=127.0.0.1 MYSQL_PORT=3306 MYSQL_USER=root sh "$SCRIPTDIR/load_test_data.sh"
 
 echo "MySQL running on port 3306"

package/package.json

@@ -20,15 +20,15 @@
     "malloyc": "ts-node ../scripts/malloy-to-json"
   },
   "dependencies": {
-    "@malloydata/db-bigquery": "0.0.432",
-    "@malloydata/db-duckdb": "0.0.432",
-    "@malloydata/db-postgres": "0.0.432",
-    "@malloydata/db-snowflake": "0.0.432",
-    "@malloydata/db-trino": "0.0.432",
-    "@malloydata/malloy": "0.0.432",
-    "@malloydata/malloy-tag": "0.0.432",
-    "@malloydata/render": "0.0.432",
-    "@malloydata/render-validator": "0.0.432",
+    "@malloydata/db-bigquery": "0.0.434",
+    "@malloydata/db-duckdb": "0.0.434",
+    "@malloydata/db-postgres": "0.0.434",
+    "@malloydata/db-snowflake": "0.0.434",
+    "@malloydata/db-trino": "0.0.434",
+    "@malloydata/malloy": "0.0.434",
+    "@malloydata/malloy-tag": "0.0.434",
+    "@malloydata/render": "0.0.434",
+    "@malloydata/render-validator": "0.0.434",
     "events": "^3.3.0",
     "jsdom": "^22.1.0",
     "luxon": "^3.7.2",
@@ -42,5 +42,5 @@
   "overrides": {
     "typescript": "^6.0.3"
   },
-  "version": "0.0.432"
+  "version": "0.0.434"
 }

package/postgres/load_test_data.sh

@@ -0,0 +1,12 @@
+#! /bin/bash
+#
+# Load the malloytest tables into a running postgres from
+# test/data/malloytest-parquet. Connects with the PG* variables.
+#
+set -e
+
+SCRIPTDIR=$(cd $(dirname $0); pwd)
+ROOTDIR=$(cd "$SCRIPTDIR/../.."; pwd)
+
+cd "$ROOTDIR"
+npx ts-node test/postgres/load_test_data.ts