@malloydata/malloy-tests 0.0.433 → 0.0.434

package/trino/hive_ddl.ts

@@ -0,0 +1,148 @@
+/*
+ * Copyright Contributors to the Malloy project
+ * SPDX-License-Identifier: MIT
+ */
+
+/* eslint-disable no-console */
+
+/*
+ * Print the DDL that serves test/data/malloytest-parquet through the trino or
+ * presto hive connector, reading each parquet file in place as an external
+ * table. The hive metastore lowercases every name, so a table with nested
+ * columns is created under a `_hive` suffix and a view casts the row types
+ * back to the parquet's field names.
+ *
+ *   ts-node hive_ddl.ts <catalog> <location> [--presto]
+ *
+ * <location> is the directory holding one directory per table, in the
+ * connector's URL scheme: local:///malloytest for trino, file:///data/malloytest
+ * for presto.
+ */
+
+import type {QueryRecord} from '@malloydata/malloy';
+import {TinyParser} from '../../packages/malloy/src/dialect/tiny_parser';
+import {openDuckDB, parquetPath, parquetTables} from '../data/parquet_loader';
+
+const [catalog, location, ...flags] = process.argv.slice(2);
+const presto = flags.length === 1 && flags[0] === '--presto';
+if (!catalog || !location || (flags.length > 0 && !presto)) {
+  console.error('usage: hive_ddl.ts <catalog> <location> [--presto]');
+  process.exit(1);
+}
+// Presto's timestamp is milliseconds and takes no precision; trino's is
+// declared to match hive.timestamp-precision in the catalog properties.
+const timestampType = presto ? 'TIMESTAMP' : 'TIMESTAMP(6)';
+
+// DuckDB types hive spells the same way
+const sameNameTypes = new Set([
+  'BIGINT',
+  'INTEGER',
+  'DOUBLE',
+  'VARCHAR',
+  'BOOLEAN',
+  'DATE',
+]);
+
+/**
+ * Reads a DuckDB type as DESCRIBE writes it and writes the hive type for the
+ * same parquet column: STRUCT(a T, b T) becomes ROW("a" T, "b" T), T[]
+ * becomes ARRAY(T), DECIMAL(p,s) and the same-name scalars carry over. The
+ * dialect's own DuckDBTypeParser reads the same text into Malloy types, which
+ * drop the decimal precision hive needs; this one keeps the SQL spelling.
+ */
+class HiveTypeParser extends TinyParser {
+  constructor(input: string) {
+    super(input, {
+      space: /^\s+/,
+      qdouble: /^"([^"]|"")*"/,
+      precision: /^\(\d+,\d+\)/,
+      arrayOf: /^\[]/,
+      char: /^[,()]/,
+      id: /^\w+/,
+    });
+  }
+
+  hiveType(): string {
+    const id = this.expect('id').text.toUpperCase();
+    let type: string;
+    if (id === 'STRUCT') {
+      this.expect('(');
+      const fields: string[] = [];
+      for (;;) {
+        const name = this.match('qdouble') ?? this.expect('id');
+        fields.push(`"${name.text}" ${this.hiveType()}`);
+        if (!this.match(',')) {
+          break;
+        }
+      }
+      this.expect(')');
+      type = `ROW(${fields.join(', ')})`;
+    } else if (id === 'DECIMAL') {
+      type = `DECIMAL${this.expect('precision').text}`;
+    } else if (id === 'TIMESTAMP') {
+      this.matchText('WITH', 'TIME', 'ZONE');
+      type = timestampType;
+    } else if (sameNameTypes.has(id)) {
+      type = id;
+    } else {
+      throw this.parseError(`No hive type for DuckDB type ${id}`);
+    }
+    while (this.match('arrayOf')) {
+      type = `ARRAY(${type})`;
+    }
+    return type;
+  }
+}
+
+function hiveType(duckdbType: string): string {
+  const parser = new HiveTypeParser(duckdbType);
+  const type = parser.hiveType();
+  if (!parser.eof()) {
+    throw parser.parseError('Unexpected text after the type');
+  }
+  return type;
+}
+
+function isNested(duckdbType: string): boolean {
+  return duckdbType.includes('STRUCT(');
+}
+
+function text(row: QueryRecord, column: string): string {
+  const value = row[column];
+  if (typeof value !== 'string') {
+    throw new Error(`DESCRIBE gave no ${column}`);
+  }
+  return value;
+}
+
+(async () => {
+  const db = await openDuckDB();
+  const ddl = [`CREATE SCHEMA IF NOT EXISTS ${catalog}.malloytest;`];
+  for (const table of parquetTables()) {
+    const described = await db.run(
+      `DESCRIBE SELECT * FROM read_parquet('${parquetPath(table)}')`
+    );
+    const columns = described.map(row => ({
+      name: text(row, 'column_name'),
+      type: text(row, 'column_type'),
+    }));
+    const nested = columns.some(c => isNested(c.type));
+    const hiveTable = nested ? `${table}_hive` : table;
+    const declarations = columns.map(c => `  "${c.name}" ${hiveType(c.type)}`);
+    ddl.push(
+      `CREATE TABLE IF NOT EXISTS ${catalog}.malloytest.${hiveTable} (\n${declarations.join(',\n')}\n) WITH (format = 'PARQUET', external_location = '${location}/${table}');`
+    );
+    if (nested) {
+      const select = columns.map(c =>
+        isNested(c.type)
+          ? `  CAST("${c.name}" AS ${hiveType(c.type)}) AS "${c.name}"`
+          : `  "${c.name}"`
+      );
+      ddl.push(
+        `CREATE OR REPLACE VIEW ${catalog}.malloytest.${table} AS SELECT\n${select.join(',\n')}\nFROM ${catalog}.malloytest.${hiveTable};`
+      );
+    }
+  }
+  await db.close();
+  console.log(ddl.join('\n'));
+})();

package/trino/trino_start.sh

@@ -1,25 +1,36 @@
 #! /bin/bash
-
+#
+# Start a trino container whose hive catalog serves test/data/malloytest-parquet
+# as the malloytest schema. The parquet is read in place; nothing is loaded.
+#
 set -e
 
+SCRIPTDIR=$(cd $(dirname $0); pwd)
+ROOTDIR=$(cd "$SCRIPTDIR/../.."; pwd)
+DATADIR=$ROOTDIR/test/data/malloytest-parquet
+CONTAINER_NAME="trino-malloy"
+PORT=${TRINO_PORT:-8080}
+
+cd "$ROOTDIR"
 rm -rf .tmp
 mkdir .tmp
 
+# Only the log since this script started: a restarted container's old log
+# already says SERVER STARTED.
+SINCE=$(date -u +%Y-%m-%dT%H:%M:%SZ)
 
-if [ "x${BQ_CREDENTIALS_KEY}" = x ]; then
-  echo "######### BQ_CREDENTIALS_KEY is not set. Cannot start Trino"
-  exit 1
-fi
-# generate config file
-> ./.tmp/bigquery-trino.properties
-cat << EOF > ./.tmp/bigquery-trino.properties
-connector.name=bigquery
-bigquery.project-id=advance-lacing-417917
-bigquery.credentials-key=$BQ_CREDENTIALS_KEY
-bigquery.arrow-serialization.enabled=false
-EOF
-
-CONTAINER_NAME="trino-malloy"
+wait_for_trino() {
+  local counter=0
+  while ! docker logs --since "$SINCE" "$CONTAINER_NAME" 2>&1 | grep -q "SERVER STARTED"; do
+    counter=$((counter+1))
+    # give up after 5 minutes
+    if [ $counter -eq 300 ]; then
+      docker logs "$CONTAINER_NAME" >& ./.tmp/trino-malloy.logs
+      return 1
+    fi
+    sleep 1
+  done
+}
 
 # Check for existing container
 if docker container inspect "$CONTAINER_NAME" > /dev/null 2>&1; then
@@ -29,31 +40,36 @@ if docker container inspect "$CONTAINER_NAME" > /dev/null 2>&1; then
   fi
   echo "Restarting existing $CONTAINER_NAME container..."
   docker start "$CONTAINER_NAME"
-  while ! docker logs "$CONTAINER_NAME" 2>&1 | grep -q "SERVER STARTED"; do
-    sleep 1
-  done
-  echo "Trino running on port localhost:8080"
+  if ! wait_for_trino; then
+    echo "Trino did not restart successfully, check .tmp/trino-malloy.logs"
+    exit 1
+  fi
+  echo "Trino running on port localhost:$PORT"
   exit 0
 fi
 
-# run docker
-docker run -p ${TRINO_PORT:-8080}:8080 -d -e TZ=UTC -v ./.tmp/bigquery-trino.properties:/etc/trino/catalog/bigquery.properties --name "$CONTAINER_NAME" trinodb/trino
-
-# wait for server to start
-counter=0
-while ! docker logs "$CONTAINER_NAME" 2>&1 | grep -q "SERVER STARTED"
-do
-  sleep 1
-  counter=$((counter+1))
-  # if doesn't start after 2 minutes, output logs and kill process
-  if [ $counter -eq 300 ]
-  then
-    docker logs "$CONTAINER_NAME" >& ./.tmp/trino-malloy.logs
-    docker rm -f "$CONTAINER_NAME"
-    echo "Trino did not start successfully, check .tmp/trino-malloy.logs"
-    exit 1
-    break
-  fi
+# A hive external_location is a directory, so each parquet file gets one.
+MOUNTS=()
+for parquet in "$DATADIR"/*.parquet; do
+  table=$(basename "$parquet" .parquet)
+  MOUNTS+=(-v "$parquet:/data/malloytest/$table/$table.parquet:ro")
 done
 
-echo "Trino running on port localhost:8080"
+docker run -p $PORT:8080 -d -e TZ=UTC "${MOUNTS[@]}" \
+  -v "$SCRIPTDIR/hive.properties:/etc/trino/catalog/hive.properties:ro" \
+  --name "$CONTAINER_NAME" trinodb/trino
+
+# A container without its tables must not survive to be "already running"
+trap 'docker rm -f "$CONTAINER_NAME" > /dev/null; echo "Trino setup failed, check .tmp/trino-malloy.logs"' ERR
+
+if ! wait_for_trino; then
+  false
+fi
+
+echo "Creating the malloytest tables"
+npx ts-node test/trino/hive_ddl.ts hive local:///malloytest > .tmp/hive_ddl.sql
+docker cp .tmp/hive_ddl.sql "$CONTAINER_NAME":/tmp/hive_ddl.sql
+docker exec "$CONTAINER_NAME" trino -f /tmp/hive_ddl.sql
+trap - ERR
+
+echo "Trino running on port localhost:$PORT"

package/tsconfig.json

@@ -6,7 +6,18 @@
     "jsx": "react-jsx",
     "jsxImportSource": "solid-js"
   },
-  "include": ["src", "bin", "./jest.setup.ts", "./types/jest.d.ts"],
+  "include": [
+    "src",
+    "bin",
+    "data",
+    "duckdb",
+    "mssql",
+    "mysql",
+    "postgres",
+    "trino",
+    "./jest.setup.ts",
+    "./types/jest.d.ts"
+  ],
   "references": [
     {
       "path": "../packages/malloy-interfaces"

package/bin/dump_malloytest.sh

@@ -1 +0,0 @@
-pg_dump -n malloytest -O -x  | gzip > ../sql/malloytest-postgres.sql.gz

package/bin/postgres_init.sh

@@ -1,10 +0,0 @@
-#! /bin/bash
-export PGHOST=localhost
-pg_ctl -D .tmp/data/malloytestdb -l .tmp/logfile -o "--unix_socket_directories='$PWD/.tmp'" stop
-rm -rf .tmp
-mkdir .tmp
-initdb -d .tmp/data/malloytestdb --no-locale --encoding=UTF8
-pg_ctl -D .tmp/data/malloytestdb -l .tmp/logfile -o "--unix_socket_directories='$PWD/.tmp'" start
-createdb $USER
-gunzip -c ../data/postgres/malloytest-postgres.sql.gz | psql
-echo CREATE EXTENSION tsm_system_rows\; | psql

package/bin/postgres_start.sh

@@ -1,2 +0,0 @@
-#! /bin/bash
-pg_ctl -D .tmp/data/malloytestdb -l logfile -o "--unix_socket_directories='$PWD'" start

package/bin/postgres_stop.sh

@@ -1,2 +0,0 @@
-#!/bin/bash
-pg_ctl -D .tmp/data/malloytestdb stop

package/data/postgres/state_facts.sql

@@ -1,366 +0,0 @@
--- example of how to take a json blob and insert it into a postgres database
---  keeping around because it might be useful as we add new tables.
-
-create type t as (state varchar, aircraft_count int, airport_count int, births int, popular_name varchar);
-create table malloytest.state_facts as (
-select p.*
-from json_populate_recordset(null::t,'[
-      {
-        "state": "MN",
-        "aircraft_count": "83",
-        "airport_count": "507",
-        "births": "5574819",
-        "popular_name": "Ava"
-      },
-      {
-        "state": "LA",
-        "aircraft_count": "57",
-        "airport_count": "500",
-        "births": "5551025",
-        "popular_name": "Ava"
-      },
-      {
-        "state": "IA",
-        "aircraft_count": "28",
-        "airport_count": "319",
-        "births": "4108689",
-        "popular_name": "Ava"
-      },
-      {
-        "state": "IN",
-        "aircraft_count": "62",
-        "airport_count": "643",
-        "births": "7170553",
-        "popular_name": "Emma"
-      },
-      {
-        "state": "NC",
-        "aircraft_count": "72",
-        "airport_count": "400",
-        "births": "8440235",
-        "popular_name": "Emma"
-      },
-      {
-        "state": "AR",
-        "aircraft_count": "31",
-        "airport_count": "299",
-        "births": "3330382",
-        "popular_name": "Emma"
-      },
-      {
-        "state": "AL",
-        "aircraft_count": "58",
-        "airport_count": "260",
-        "births": "5645599",
-        "popular_name": "Emma"
-      },
-      {
-        "state": "MT",
-        "aircraft_count": "50",
-        "airport_count": "259",
-        "births": "942521",
-        "popular_name": "Emma"
-      },
-      {
-        "state": "ME",
-        "aircraft_count": "12",
-        "airport_count": "164",
-        "births": "1383204",
-        "popular_name": "Emma"
-      },
-      {
-        "state": "VA",
-        "aircraft_count": "67",
-        "airport_count": "421",
-        "births": "6693880",
-        "popular_name": "Olivia"
-      },
-      {
-        "state": "NE",
-        "aircraft_count": "24",
-        "airport_count": "308",
-        "births": "2255039",
-        "popular_name": "Olivia"
-      },
-      {
-        "state": "ID",
-        "aircraft_count": "36",
-        "airport_count": "238",
-        "births": "1147445",
-        "popular_name": "Olivia"
-      },
-      {
-        "state": "UT",
-        "aircraft_count": "53",
-        "airport_count": "140",
-        "births": "2306033",
-        "popular_name": "Olivia"
-      },
-      {
-        "state": "AK",
-        "aircraft_count": "85",
-        "airport_count": "608",
-        "births": "407106",
-        "popular_name": "Sophia"
-      },
-      {
-        "state": "WI",
-        "aircraft_count": "66",
-        "airport_count": "543",
-        "births": "6088288",
-        "popular_name": "Sophia"
-      },
-      {
-        "state": "MI",
-        "aircraft_count": "102",
-        "airport_count": "489",
-        "births": "11643455",
-        "popular_name": "Sophia"
-      },
-      {
-        "state": "WA",
-        "aircraft_count": "114",
-        "airport_count": "484",
-        "births": "4631964",
-        "popular_name": "Sophia"
-      },
-      {
-        "state": "OR",
-        "aircraft_count": "94",
-        "airport_count": "441",
-        "births": "2671780",
-        "popular_name": "Sophia"
-      },
-      {
-        "state": "ND",
-        "aircraft_count": "26",
-        "airport_count": "436",
-        "births": "1059330",
-        "popular_name": "Sophia"
-      },
-      {
-        "state": "MD",
-        "aircraft_count": "30",
-        "airport_count": "229",
-        "births": "4468717",
-        "popular_name": "Sophia"
-      },
-      {
-        "state": "SD",
-        "aircraft_count": "23",
-        "airport_count": "180",
-        "births": "1049223",
-        "popular_name": "Sophia"
-      },
-      {
-        "state": "NH",
-        "aircraft_count": "30",
-        "airport_count": "112",
-        "births": "955057",
-        "popular_name": "Sophia"
-      },
-      {
-        "state": "VT",
-        "aircraft_count": "6",
-        "airport_count": "81",
-        "births": "545999",
-        "popular_name": "Sophia"
-      },
-      {
-        "state": "DE",
-        "aircraft_count": "118",
-        "airport_count": "42",
-        "births": "615490",
-        "popular_name": "Sophia"
-      },
-      {
-        "state": "MS",
-        "aircraft_count": "27",
-        "airport_count": "243",
-        "births": "3969549",
-        "popular_name": "Madison"
-      },
-      {
-        "state": "SC",
-        "aircraft_count": "28",
-        "airport_count": "189",
-        "births": "4392042",
-        "popular_name": "Madison"
-      },
-      {
-        "state": "DC",
-        "aircraft_count": "6",
-        "airport_count": "20",
-        "births": "1419047",
-        "popular_name": "Madison"
-      },
-      {
-        "state": "TX",
-        "aircraft_count": "287",
-        "airport_count": "1845",
-        "births": "21467359",
-        "popular_name": "Isabella"
-      },
-      {
-        "state": "CA",
-        "aircraft_count": "367",
-        "airport_count": "984",
-        "births": "28810563",
-        "popular_name": "Isabella"
-      },
-      {
-        "state": "IL",
-        "aircraft_count": "111",
-        "airport_count": "890",
-        "births": "15178876",
-        "popular_name": "Isabella"
-      },
-      {
-        "state": "FL",
-        "aircraft_count": "228",
-        "airport_count": "856",
-        "births": "9277223",
-        "popular_name": "Isabella"
-      },
-      {
-        "state": "PA",
-        "aircraft_count": "83",
-        "airport_count": "804",
-        "births": "16661910",
-        "popular_name": "Isabella"
-      },
-      {
-        "state": "OH",
-        "aircraft_count": "106",
-        "airport_count": "749",
-        "births": "14201526",
-        "popular_name": "Isabella"
-      },
-      {
-        "state": "NY",
-        "aircraft_count": "91",
-        "airport_count": "576",
-        "births": "23694136",
-        "popular_name": "Isabella"
-      },
-      {
-        "state": "MO",
-        "aircraft_count": "62",
-        "airport_count": "537",
-        "births": "6792279",
-        "popular_name": "Isabella"
-      },
-      {
-        "state": "OK",
-        "aircraft_count": "99",
-        "airport_count": "443",
-        "births": "4137117",
-        "popular_name": "Isabella"
-      },
-      {
-        "state": "GA",
-        "aircraft_count": "98",
-        "airport_count": "440",
-        "births": "8183223",
-        "popular_name": "Isabella"
-      },
-      {
-        "state": "CO",
-        "aircraft_count": "98",
-        "airport_count": "425",
-        "births": "3424936",
-        "popular_name": "Isabella"
-      },
-      {
-        "state": "KS",
-        "aircraft_count": "63",
-        "airport_count": "415",
-        "births": "3212573",
-        "popular_name": "Isabella"
-      },
-      {
-        "state": "NJ",
-        "aircraft_count": "47",
-        "airport_count": "378",
-        "births": "8318769",
-        "popular_name": "Isabella"
-      },
-      {
-        "state": "AZ",
-        "aircraft_count": "98",
-        "airport_count": "319",
-        "births": "3328182",
-        "popular_name": "Isabella"
-      },
-      {
-        "state": "TN",
-        "aircraft_count": "62",
-        "airport_count": "285",
-        "births": "6313851",
-        "popular_name": "Isabella"
-      },
-      {
-        "state": "MA",
-        "aircraft_count": "40",
-        "airport_count": "225",
-        "births": "7576933",
-        "popular_name": "Isabella"
-      },
-      {
-        "state": "KY",
-        "aircraft_count": "26",
-        "airport_count": "202",
-        "births": "5352594",
-        "popular_name": "Isabella"
-      },
-      {
-        "state": "NM",
-        "aircraft_count": "46",
-        "airport_count": "181",
-        "births": "1640394",
-        "popular_name": "Isabella"
-      },
-      {
-        "state": "CT",
-        "aircraft_count": "25",
-        "airport_count": "153",
-        "births": "3361014",
-        "popular_name": "Isabella"
-      },
-      {
-        "state": "NV",
-        "aircraft_count": "55",
-        "airport_count": "128",
-        "births": "842854",
-        "popular_name": "Isabella"
-      },
-      {
-        "state": "WV",
-        "aircraft_count": "13",
-        "airport_count": "116",
-        "births": "2962442",
-        "popular_name": "Isabella"
-      },
-      {
-        "state": "WY",
-        "aircraft_count": "15",
-        "airport_count": "115",
-        "births": "423000",
-        "popular_name": "Isabella"
-      },
-      {
-        "state": "HI",
-        "aircraft_count": "10",
-        "airport_count": "52",
-        "births": "950538",
-        "popular_name": "Isabella"
-      },
-      {
-        "state": "RI",
-        "aircraft_count": "5",
-        "airport_count": "28",
-        "births": "1148302",
-        "popular_name": "Isabella"
-      }
-    ]') as p
-);