@malloydata/malloy-tests 0.0.433 → 0.0.435
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTEXT.md +18 -6
- package/data/parquet_loader.ts +106 -0
- package/duckdb/load_test_data.ts +15 -48
- package/mssql/connection_string.ts +28 -0
- package/mssql/load_test_data.sh +12 -0
- package/mssql/load_test_data.ts +63 -0
- package/mssql/mssql_start.sh +30 -143
- package/mssql/mssql_stop.sh +2 -2
- package/mysql/load_test_data.sh +12 -0
- package/mysql/load_test_data.ts +39 -0
- package/mysql/mysql_start.sh +36 -30
- package/package.json +12 -11
- package/postgres/load_test_data.sh +12 -0
- package/postgres/load_test_data.ts +34 -0
- package/postgres/postgres_start.sh +2 -6
- package/presto/Dockerfile.slim +12 -6
- package/presto/hive.properties +8 -0
- package/presto/presto_start.sh +58 -36
- package/src/core/experimental-dialects.spec.ts +125 -41
- package/src/core/restricted-mode.spec.ts +27 -0
- package/src/core/tags.spec.ts +22 -0
- package/src/core/virtual-sql.spec.ts +253 -0
- package/src/databases/all/db_index.spec.ts +4 -2
- package/src/databases/all/expr.spec.ts +150 -18
- package/src/databases/all/nest.spec.ts +237 -0
- package/src/databases/all/nomodel.spec.ts +0 -75
- package/src/databases/all/problems.spec.ts +32 -0
- package/src/databases/sqlserver/sqlserver.spec.ts +395 -0
- package/src/runtimes.ts +49 -9
- package/src/test-select.ts +10 -14
- package/trino/hive.properties +11 -0
- package/trino/hive_ddl.ts +148 -0
- package/trino/trino_start.sh +54 -38
- package/tsconfig.json +15 -1
- package/bin/dump_malloytest.sh +0 -1
- package/bin/postgres_init.sh +0 -10
- package/bin/postgres_start.sh +0 -2
- package/bin/postgres_stop.sh +0 -2
- package/data/mysql/malloytest.mysql.gz +0 -0
- package/data/postgres/malloytest-postgres.sql.gz +0 -0
- package/data/postgres/state_facts.sql +0 -366
package/mysql/mysql_start.sh
CHANGED
|
@@ -4,11 +4,31 @@ set -e
|
|
|
4
4
|
rm -rf .tmp
|
|
5
5
|
mkdir .tmp
|
|
6
6
|
|
|
7
|
-
# run docker
|
|
8
7
|
SCRIPTDIR=$(cd $(dirname $0); pwd)
|
|
9
|
-
DATADIR=$(dirname $SCRIPTDIR)/data/mysql
|
|
10
8
|
CONTAINER_NAME="mysql-malloy"
|
|
11
9
|
|
|
10
|
+
# The image starts mysqld twice: a temporary server to initialize the data
|
|
11
|
+
# directory, listening on the unix socket only, and then the real server on
|
|
12
|
+
# 3306. Both log "mysqld: ready for connections", so the log cannot say which
|
|
13
|
+
# one is up. Wait on a TCP connection, which is what the loader needs.
|
|
14
|
+
wait_for_mysql() {
|
|
15
|
+
local counter=0
|
|
16
|
+
echo -n "Waiting for $CONTAINER_NAME ..."
|
|
17
|
+
until docker exec "$CONTAINER_NAME" mysqladmin ping --protocol=TCP -h127.0.0.1 -uroot --silent > /dev/null 2>&1
|
|
18
|
+
do
|
|
19
|
+
counter=$((counter+1))
|
|
20
|
+
# give up after 2 minutes
|
|
21
|
+
if [ $counter -eq 60 ]
|
|
22
|
+
then
|
|
23
|
+
echo
|
|
24
|
+
return 1
|
|
25
|
+
fi
|
|
26
|
+
echo -n .
|
|
27
|
+
sleep 2
|
|
28
|
+
done
|
|
29
|
+
echo
|
|
30
|
+
}
|
|
31
|
+
|
|
12
32
|
# Check for existing container
|
|
13
33
|
if docker container inspect "$CONTAINER_NAME" > /dev/null 2>&1; then
|
|
14
34
|
if [ "$(docker inspect -f '{{.State.Running}}' "$CONTAINER_NAME")" = "true" ]; then
|
|
@@ -17,39 +37,25 @@ if docker container inspect "$CONTAINER_NAME" > /dev/null 2>&1; then
|
|
|
17
37
|
fi
|
|
18
38
|
echo "Restarting existing $CONTAINER_NAME container..."
|
|
19
39
|
docker start "$CONTAINER_NAME"
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
40
|
+
if ! wait_for_mysql; then
|
|
41
|
+
docker logs "$CONTAINER_NAME" >& ./.tmp/mysql-malloy.logs
|
|
42
|
+
echo "MySQL did not restart successfully, check .tmp/mysql-malloy.logs"
|
|
43
|
+
exit 1
|
|
44
|
+
fi
|
|
23
45
|
echo "MySQL running on port 3306"
|
|
24
46
|
exit 0
|
|
25
47
|
fi
|
|
26
48
|
|
|
27
|
-
docker run -p 3306:3306 -d
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
counter=$((counter+1))
|
|
36
|
-
# if doesn't start after 2 minutes, output logs and kill process
|
|
37
|
-
if [ $counter -eq 120 ]
|
|
38
|
-
then
|
|
39
|
-
docker logs "$CONTAINER_NAME" >& ./.tmp/mysql-malloy.logs
|
|
40
|
-
docker rm -f "$CONTAINER_NAME"
|
|
41
|
-
echo "MySQL did not start successfully, check .tmp/mysql-malloy.logs"
|
|
42
|
-
exit 1
|
|
43
|
-
break
|
|
44
|
-
fi
|
|
45
|
-
echo -n ...
|
|
46
|
-
done
|
|
49
|
+
docker run -p 3306:3306 -d --name "$CONTAINER_NAME" -e MYSQL_ALLOW_EMPTY_PASSWORD=yes mysql:8.4.2
|
|
50
|
+
|
|
51
|
+
if ! wait_for_mysql; then
|
|
52
|
+
docker logs "$CONTAINER_NAME" >& ./.tmp/mysql-malloy.logs
|
|
53
|
+
docker rm -f "$CONTAINER_NAME"
|
|
54
|
+
echo "MySQL did not start successfully, check .tmp/mysql-malloy.logs"
|
|
55
|
+
exit 1
|
|
56
|
+
fi
|
|
47
57
|
|
|
48
|
-
# load the test data.
|
|
49
|
-
echo
|
|
50
58
|
echo Loading Test Data
|
|
51
|
-
|
|
52
|
-
docker exec "$CONTAINER_NAME" gunzip /tmp/malloytest.mysql.gz
|
|
53
|
-
docker exec "$CONTAINER_NAME" mysql -P3306 -h127.0.0.1 -uroot -e 'drop database if exists malloytest; create database malloytest; use malloytest; source /tmp/malloytest.mysql;'
|
|
59
|
+
MYSQL_HOST=127.0.0.1 MYSQL_PORT=3306 MYSQL_USER=root sh "$SCRIPTDIR/load_test_data.sh"
|
|
54
60
|
|
|
55
61
|
echo "MySQL running on port 3306"
|
package/package.json
CHANGED
|
@@ -14,21 +14,22 @@
|
|
|
14
14
|
"scripts": {
|
|
15
15
|
"lint": "eslint '**/*.ts{,x}'",
|
|
16
16
|
"lint-fix": "eslint '**/*.ts{,x}' --fix",
|
|
17
|
-
"test": "
|
|
17
|
+
"test": "echo 'Read the Testing section of the repo-root CONTEXT.md for information on testing in this monorepo'; exit 1",
|
|
18
18
|
"build": "node ../node_modules/typescript/bin/tsc --build",
|
|
19
19
|
"clean": "node ../node_modules/typescript/bin/tsc --build --clean && rm -f tsconfig.tsbuildinfo",
|
|
20
20
|
"malloyc": "ts-node ../scripts/malloy-to-json"
|
|
21
21
|
},
|
|
22
22
|
"dependencies": {
|
|
23
|
-
"@malloydata/db-bigquery": "0.0.
|
|
24
|
-
"@malloydata/db-duckdb": "0.0.
|
|
25
|
-
"@malloydata/db-postgres": "0.0.
|
|
26
|
-
"@malloydata/db-snowflake": "0.0.
|
|
27
|
-
"@malloydata/db-
|
|
28
|
-
"@malloydata/
|
|
29
|
-
"@malloydata/malloy
|
|
30
|
-
"@malloydata/
|
|
31
|
-
"@malloydata/render
|
|
23
|
+
"@malloydata/db-bigquery": "0.0.435",
|
|
24
|
+
"@malloydata/db-duckdb": "0.0.435",
|
|
25
|
+
"@malloydata/db-postgres": "0.0.435",
|
|
26
|
+
"@malloydata/db-snowflake": "0.0.435",
|
|
27
|
+
"@malloydata/db-sqlserver": "0.0.435",
|
|
28
|
+
"@malloydata/db-trino": "0.0.435",
|
|
29
|
+
"@malloydata/malloy": "0.0.435",
|
|
30
|
+
"@malloydata/malloy-tag": "0.0.435",
|
|
31
|
+
"@malloydata/render": "0.0.435",
|
|
32
|
+
"@malloydata/render-validator": "0.0.435",
|
|
32
33
|
"events": "^3.3.0",
|
|
33
34
|
"jsdom": "^22.1.0",
|
|
34
35
|
"luxon": "^3.7.2",
|
|
@@ -42,5 +43,5 @@
|
|
|
42
43
|
"overrides": {
|
|
43
44
|
"typescript": "^6.0.3"
|
|
44
45
|
},
|
|
45
|
-
"version": "0.0.
|
|
46
|
+
"version": "0.0.435"
|
|
46
47
|
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
#! /bin/bash
|
|
2
|
+
#
|
|
3
|
+
# Load the malloytest tables into a running postgres from
|
|
4
|
+
# test/data/malloytest-parquet. Connects with the PG* variables.
|
|
5
|
+
#
|
|
6
|
+
set -e
|
|
7
|
+
|
|
8
|
+
SCRIPTDIR=$(cd $(dirname $0); pwd)
|
|
9
|
+
ROOTDIR=$(cd "$SCRIPTDIR/../.."; pwd)
|
|
10
|
+
|
|
11
|
+
cd "$ROOTDIR"
|
|
12
|
+
npx ts-node test/postgres/load_test_data.ts
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Copyright Contributors to the Malloy project
|
|
3
|
+
* SPDX-License-Identifier: MIT
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
/* eslint-disable no-console */
|
|
7
|
+
|
|
8
|
+
/*
|
|
9
|
+
* Load the malloytest tables into a running postgres from the parquet files.
|
|
10
|
+
* Connects the way psql and the postgres test connection do, through libpq's
|
|
11
|
+
* PG* variables, which the extension reads when the attach string is empty.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import {
|
|
15
|
+
loadExtension,
|
|
16
|
+
loadMalloytest,
|
|
17
|
+
openDuckDB,
|
|
18
|
+
} from '../data/parquet_loader';
|
|
19
|
+
|
|
20
|
+
(async () => {
|
|
21
|
+
const db = await openDuckDB();
|
|
22
|
+
await loadExtension(db, 'postgres');
|
|
23
|
+
await db.run("ATTACH '' AS pg (TYPE postgres)");
|
|
24
|
+
// The sampling tests use TABLESAMPLE SYSTEM_ROWS
|
|
25
|
+
await db.run(
|
|
26
|
+
"CALL postgres_execute('pg', 'CREATE EXTENSION IF NOT EXISTS tsm_system_rows')"
|
|
27
|
+
);
|
|
28
|
+
console.log('Loading malloytest into postgres');
|
|
29
|
+
await loadMalloytest(db, 'pg.malloytest', {
|
|
30
|
+
// Postgres has no anonymous record type for the nested columns
|
|
31
|
+
skip: ['ga_sample'],
|
|
32
|
+
});
|
|
33
|
+
await db.close();
|
|
34
|
+
})();
|
|
@@ -7,9 +7,7 @@ set -e
|
|
|
7
7
|
rm -rf .tmp
|
|
8
8
|
mkdir .tmp
|
|
9
9
|
|
|
10
|
-
# run docker
|
|
11
10
|
SCRIPTDIR=$(cd $(dirname $0); pwd)
|
|
12
|
-
DATADIR=$(dirname $SCRIPTDIR)/data/postgres
|
|
13
11
|
|
|
14
12
|
# Verify required environment variables and show export commands for any that are wrong
|
|
15
13
|
EXPORTS=""
|
|
@@ -55,7 +53,7 @@ if docker container inspect "$CONTAINER_NAME" > /dev/null 2>&1; then
|
|
|
55
53
|
exit 0
|
|
56
54
|
fi
|
|
57
55
|
|
|
58
|
-
docker run -p 5432:5432 -d
|
|
56
|
+
docker run -p 5432:5432 -d \
|
|
59
57
|
--name "$CONTAINER_NAME" \
|
|
60
58
|
-e POSTGRES_USER=root -e POSTGRES_PASSWORD=postgres \
|
|
61
59
|
-e TZ=UTC \
|
|
@@ -74,8 +72,6 @@ done
|
|
|
74
72
|
echo "Container $CONTAINER_NAME is now healthy!"
|
|
75
73
|
|
|
76
74
|
echo "Loading data ..."
|
|
77
|
-
|
|
78
|
-
echo CREATE EXTENSION tsm_system_rows\; | psql
|
|
79
|
-
gunzip -c ${DATADIR}/malloytest-postgres.sql.gz | psql
|
|
75
|
+
sh "$SCRIPTDIR/load_test_data.sh"
|
|
80
76
|
|
|
81
77
|
echo "Ready"
|
package/presto/Dockerfile.slim
CHANGED
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
# Slim Presto image for Malloy CI testing
|
|
2
2
|
#
|
|
3
|
-
# This image is ~
|
|
4
|
-
# It includes only what's needed to run Presto with the
|
|
3
|
+
# This image is ~1.4GB vs ~9GB for the official prestodb/presto image.
|
|
4
|
+
# It includes only what's needed to run Presto with the hive connector, which
|
|
5
|
+
# serves the test parquet (see hive.properties), and the CLI that creates the
|
|
6
|
+
# tables.
|
|
5
7
|
#
|
|
6
8
|
# How this was created:
|
|
7
9
|
# 1. The official prestodb/presto:0.287 image contains 42 connector plugins (~2GB)
|
|
8
|
-
# but we only need
|
|
10
|
+
# but we only need hive-hadoop2 (~306MB)
|
|
9
11
|
# 2. We use eclipse-temurin as the base because:
|
|
10
12
|
# - Presto requires Java 11 (not newer - tested, Java 17 fails with classloading errors)
|
|
11
13
|
# - Presto requires full JDK (not JRE - needs com.sun.tools.attach classes)
|
|
@@ -18,7 +20,7 @@
|
|
|
18
20
|
#
|
|
19
21
|
# Size comparison (as of 0.287):
|
|
20
22
|
# Official image: 9.11 GB
|
|
21
|
-
# This slim image: ~
|
|
23
|
+
# This slim image: ~1.4 GB
|
|
22
24
|
|
|
23
25
|
ARG PRESTO_VERSION=0.287
|
|
24
26
|
|
|
@@ -39,8 +41,12 @@ COPY --from=presto-official /opt/presto-server/lib /opt/presto-server/lib
|
|
|
39
41
|
COPY --from=presto-official /opt/presto-server/bin /opt/presto-server/bin
|
|
40
42
|
COPY --from=presto-official /opt/presto-server/etc /opt/presto-server/etc
|
|
41
43
|
|
|
42
|
-
# Copy only
|
|
43
|
-
COPY --from=presto-official /opt/presto-server/plugin/
|
|
44
|
+
# Copy only the hive plugin (the only connector we use for testing)
|
|
45
|
+
COPY --from=presto-official /opt/presto-server/plugin/hive-hadoop2 /opt/presto-server/plugin/hive-hadoop2
|
|
46
|
+
|
|
47
|
+
# The CLI, which presto_start.sh uses to create the tables
|
|
48
|
+
COPY --from=presto-official /opt/presto-cli /opt/presto-cli
|
|
49
|
+
RUN ln -s /opt/presto-cli /usr/local/bin/presto-cli
|
|
44
50
|
|
|
45
51
|
EXPOSE 8080
|
|
46
52
|
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
# The hive catalog serves test/data/malloytest-parquet, mounted read-only
|
|
2
|
+
# under /data/malloytest, with the tables hive_ddl.ts declares. The file
|
|
3
|
+
# metastore lives inside the container.
|
|
4
|
+
connector.name=hive-hadoop2
|
|
5
|
+
hive.metastore=file
|
|
6
|
+
hive.metastore.catalog.dir=file:///tmp/hive-metastore
|
|
7
|
+
# Match parquet columns by name; presto's default is by position
|
|
8
|
+
hive.parquet.use-column-names=true
|
package/presto/presto_start.sh
CHANGED
|
@@ -1,10 +1,19 @@
|
|
|
1
1
|
#! /bin/bash
|
|
2
|
-
|
|
2
|
+
#
|
|
3
|
+
# Start a presto container whose hive catalog serves test/data/malloytest-parquet
|
|
4
|
+
# as the malloytest schema. The parquet is read in place; nothing is loaded.
|
|
5
|
+
#
|
|
6
|
+
# presto_start.sh [--slim] use the slim image from Dockerfile.slim
|
|
7
|
+
#
|
|
3
8
|
set -e
|
|
4
9
|
|
|
5
|
-
|
|
10
|
+
SCRIPTDIR=$(cd $(dirname $0); pwd)
|
|
11
|
+
ROOTDIR=$(cd "$SCRIPTDIR/../.."; pwd)
|
|
12
|
+
DATADIR=$ROOTDIR/test/data/malloytest-parquet
|
|
6
13
|
PRESTO_VERSION=0.287
|
|
7
14
|
USE_SLIM=false
|
|
15
|
+
CONTAINER_NAME="presto-malloy"
|
|
16
|
+
PORT=${PRESTO_PORT:-8080}
|
|
8
17
|
|
|
9
18
|
# Parse arguments
|
|
10
19
|
for arg in "$@"; do
|
|
@@ -16,32 +25,40 @@ for arg in "$@"; do
|
|
|
16
25
|
esac
|
|
17
26
|
done
|
|
18
27
|
|
|
28
|
+
cd "$ROOTDIR"
|
|
19
29
|
rm -rf .tmp
|
|
20
30
|
mkdir .tmp
|
|
21
31
|
|
|
22
|
-
# generate config file
|
|
23
|
-
> ./.tmp/bigquery-pesto.properties
|
|
24
|
-
cat << EOF > ./.tmp/bigquery-pesto.properties
|
|
25
|
-
connector.name=bigquery
|
|
26
|
-
bigquery.project-id=advance-lacing-417917
|
|
27
|
-
bigquery.credentials-key=$BQ_CREDENTIALS_KEY
|
|
28
|
-
EOF
|
|
29
|
-
|
|
30
32
|
# Select image
|
|
31
33
|
if [ "$USE_SLIM" = true ]; then
|
|
32
34
|
PRESTO_IMAGE="presto-slim:${PRESTO_VERSION}"
|
|
33
35
|
# Build slim image if it doesn't exist
|
|
34
36
|
if ! docker image inspect "$PRESTO_IMAGE" > /dev/null 2>&1; then
|
|
35
37
|
echo "Building slim Presto image..."
|
|
36
|
-
docker build -f "$
|
|
38
|
+
docker build -f "$SCRIPTDIR/Dockerfile.slim" \
|
|
37
39
|
--build-arg PRESTO_VERSION="$PRESTO_VERSION" \
|
|
38
|
-
-t "$PRESTO_IMAGE" "$
|
|
40
|
+
-t "$PRESTO_IMAGE" "$SCRIPTDIR"
|
|
39
41
|
fi
|
|
40
42
|
else
|
|
41
43
|
PRESTO_IMAGE="prestodb/presto:${PRESTO_VERSION}"
|
|
42
44
|
fi
|
|
43
45
|
|
|
44
|
-
|
|
46
|
+
# Only the log since this script started: a restarted container's old log
|
|
47
|
+
# already says SERVER STARTED.
|
|
48
|
+
SINCE=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
|
49
|
+
|
|
50
|
+
wait_for_presto() {
|
|
51
|
+
local counter=0
|
|
52
|
+
while ! docker logs --since "$SINCE" "$CONTAINER_NAME" 2>&1 | grep -q "SERVER STARTED"; do
|
|
53
|
+
counter=$((counter+1))
|
|
54
|
+
# give up after 2 minutes
|
|
55
|
+
if [ $counter -eq 120 ]; then
|
|
56
|
+
docker logs "$CONTAINER_NAME" >& ./.tmp/presto-malloy.logs
|
|
57
|
+
return 1
|
|
58
|
+
fi
|
|
59
|
+
sleep 1
|
|
60
|
+
done
|
|
61
|
+
}
|
|
45
62
|
|
|
46
63
|
# Check for existing container
|
|
47
64
|
if docker container inspect "$CONTAINER_NAME" > /dev/null 2>&1; then
|
|
@@ -51,31 +68,36 @@ if docker container inspect "$CONTAINER_NAME" > /dev/null 2>&1; then
|
|
|
51
68
|
fi
|
|
52
69
|
echo "Restarting existing $CONTAINER_NAME container..."
|
|
53
70
|
docker start "$CONTAINER_NAME"
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
71
|
+
if ! wait_for_presto; then
|
|
72
|
+
echo "Presto did not restart successfully, check .tmp/presto-malloy.logs"
|
|
73
|
+
exit 1
|
|
74
|
+
fi
|
|
75
|
+
echo "Presto running on port $PORT"
|
|
58
76
|
exit 0
|
|
59
77
|
fi
|
|
60
78
|
|
|
61
|
-
#
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
while ! docker logs "$CONTAINER_NAME" 2>&1 | grep -q "SERVER STARTED"
|
|
67
|
-
do
|
|
68
|
-
sleep 1
|
|
69
|
-
counter=$((counter+1))
|
|
70
|
-
# if doesn't start after 2 minutes, output logs and kill process
|
|
71
|
-
if [ $counter -eq 120 ]
|
|
72
|
-
then
|
|
73
|
-
docker logs "$CONTAINER_NAME" >& ./.tmp/presto-malloy.logs
|
|
74
|
-
docker rm -f "$CONTAINER_NAME"
|
|
75
|
-
echo "Presto did not start successfully, check .tmp/presto-malloy.logs"
|
|
76
|
-
exit 1
|
|
77
|
-
break
|
|
78
|
-
fi
|
|
79
|
+
# A hive external_location is a directory, so each parquet file gets one.
|
|
80
|
+
MOUNTS=()
|
|
81
|
+
for parquet in "$DATADIR"/*.parquet; do
|
|
82
|
+
table=$(basename "$parquet" .parquet)
|
|
83
|
+
MOUNTS+=(-v "$parquet:/data/malloytest/$table/$table.parquet:ro")
|
|
79
84
|
done
|
|
80
85
|
|
|
81
|
-
|
|
86
|
+
docker run -p $PORT:8080 -d "${MOUNTS[@]}" \
|
|
87
|
+
-v "$SCRIPTDIR/hive.properties:/opt/presto-server/etc/catalog/hive.properties:ro" \
|
|
88
|
+
--name "$CONTAINER_NAME" "$PRESTO_IMAGE"
|
|
89
|
+
|
|
90
|
+
# A container without its tables must not survive to be "already running"
|
|
91
|
+
trap 'docker rm -f "$CONTAINER_NAME" > /dev/null; echo "Presto setup failed, check .tmp/presto-malloy.logs"' ERR
|
|
92
|
+
|
|
93
|
+
if ! wait_for_presto; then
|
|
94
|
+
false
|
|
95
|
+
fi
|
|
96
|
+
|
|
97
|
+
echo "Creating the malloytest tables"
|
|
98
|
+
npx ts-node test/trino/hive_ddl.ts hive file:///data/malloytest --presto > .tmp/hive_ddl.sql
|
|
99
|
+
docker cp .tmp/hive_ddl.sql "$CONTAINER_NAME":/tmp/hive_ddl.sql
|
|
100
|
+
docker exec "$CONTAINER_NAME" presto-cli -f /tmp/hive_ddl.sql
|
|
101
|
+
trap - ERR
|
|
102
|
+
|
|
103
|
+
echo "Presto running on port $PORT"
|
|
@@ -3,13 +3,14 @@
|
|
|
3
3
|
* SPDX-License-Identifier: MIT
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
|
-
import type {
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
6
|
+
import type {MalloyError} from '@malloydata/malloy';
|
|
7
|
+
import {
|
|
8
|
+
DuckDBDialect,
|
|
9
|
+
getDialect,
|
|
10
|
+
InMemoryURLReader,
|
|
11
|
+
registerDialect,
|
|
12
|
+
SingleConnectionRuntime,
|
|
10
13
|
} from '@malloydata/malloy';
|
|
11
|
-
import {DuckDBDialect, registerDialect} from '@malloydata/malloy';
|
|
12
|
-
import {testRuntimeFor} from '../runtimes';
|
|
13
14
|
import {DuckDBConnection} from '@malloydata/db-duckdb';
|
|
14
15
|
|
|
15
16
|
const envDatabases = (
|
|
@@ -33,30 +34,24 @@ async function getError<T>(promise: Promise<T>): Promise<Error | undefined> {
|
|
|
33
34
|
return undefined;
|
|
34
35
|
}
|
|
35
36
|
|
|
37
|
+
/**
|
|
38
|
+
* A file acknowledges an experimental dialect with
|
|
39
|
+
* `##! experimental.dialect.NAME` wherever it takes one on: by naming a
|
|
40
|
+
* connection on it, or by importing a source or query that uses it. The
|
|
41
|
+
* acknowledgment is a pragma, so an importer gets no credit for the line in
|
|
42
|
+
* the file it imports.
|
|
43
|
+
*/
|
|
36
44
|
describe('experimental dialects', () => {
|
|
37
45
|
const duckdbX = 'duckdb_experimental';
|
|
46
|
+
const flag = `##! experimental.dialect.${duckdbX}`;
|
|
47
|
+
|
|
38
48
|
class DuckdbXConnection extends DuckDBConnection {
|
|
39
49
|
name = duckdbX;
|
|
40
|
-
public async festchSchemaFOrSQLSource(
|
|
41
|
-
sqlRef: SQLSourceDef,
|
|
42
|
-
options: FetchSchemaOptions
|
|
43
|
-
): Promise<
|
|
44
|
-
| {structDef: SQLSourceDef; error?: undefined}
|
|
45
|
-
| {error: string; structDef?: undefined}
|
|
46
|
-
> {
|
|
47
|
-
const result = await super.fetchSchemaForSQLStruct(sqlRef, options);
|
|
48
|
-
if (result.error === undefined) {
|
|
49
|
-
return {structDef: {...result.structDef, dialect: duckdbX}};
|
|
50
|
-
}
|
|
51
|
-
return result;
|
|
52
|
-
}
|
|
53
50
|
get dialectName(): string {
|
|
54
51
|
return duckdbX;
|
|
55
52
|
}
|
|
56
53
|
}
|
|
57
54
|
|
|
58
|
-
const connection = new DuckdbXConnection(duckdbX, ':memory:');
|
|
59
|
-
|
|
60
55
|
class DuckdbXDialect extends DuckDBDialect {
|
|
61
56
|
experimental = true;
|
|
62
57
|
name = duckdbX;
|
|
@@ -65,35 +60,124 @@ describe('experimental dialects', () => {
|
|
|
65
60
|
}
|
|
66
61
|
}
|
|
67
62
|
|
|
63
|
+
const connection = new DuckdbXConnection(duckdbX, ':memory:');
|
|
68
64
|
registerDialect(new DuckdbXDialect());
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
65
|
+
|
|
66
|
+
const libURL = 'file:///lib.malloy';
|
|
67
|
+
const lib = [
|
|
68
|
+
flag,
|
|
69
|
+
`source: s is ${duckdbX}.sql('SELECT 1 as one')`,
|
|
70
|
+
'query: q is s -> { select: one }',
|
|
71
|
+
].join('\n');
|
|
72
|
+
|
|
73
|
+
function runtime(): SingleConnectionRuntime {
|
|
74
|
+
return new SingleConnectionRuntime({
|
|
75
|
+
connection,
|
|
76
|
+
urlReader: new InMemoryURLReader(new Map([[libURL, lib]])),
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** The one problem a refused compile reports, with the line it points at. */
|
|
81
|
+
async function refusal(
|
|
82
|
+
src: string
|
|
83
|
+
): Promise<{message: string; line: number | undefined}> {
|
|
84
|
+
const error = await getError(runtime().getModel(src));
|
|
85
|
+
expect(error).toBeDefined();
|
|
86
|
+
const problems = (error as MalloyError).problems;
|
|
87
|
+
expect(problems).toHaveLength(1);
|
|
88
|
+
return {
|
|
89
|
+
message: problems[0].message,
|
|
90
|
+
line: problems[0].at?.range.start.line,
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
test('naming a connection on the dialect needs the flag', async () => {
|
|
95
|
+
const got = await refusal(
|
|
96
|
+
['-- line zero', `source: s is ${duckdbX}.sql('SELECT 1 as one')`].join(
|
|
97
|
+
'\n'
|
|
98
|
+
)
|
|
77
99
|
);
|
|
78
|
-
expect(
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
100
|
+
expect(got.message).toContain(flag);
|
|
101
|
+
expect(got.line).toBe(1);
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
test.each([['constructor'], ['__proto__'], ['toString']])(
|
|
105
|
+
'a connection named %s is checked like any other',
|
|
106
|
+
async connName => {
|
|
107
|
+
const named = new DuckdbXConnection(duckdbX, ':memory:');
|
|
108
|
+
named.name = connName;
|
|
109
|
+
const error = await getError(
|
|
110
|
+
new SingleConnectionRuntime({connection: named}).getModel(
|
|
111
|
+
`source: s is ${connName}.sql('SELECT 1 as one')`
|
|
112
|
+
)
|
|
84
113
|
);
|
|
114
|
+
expect(error).toBeDefined();
|
|
115
|
+
expect((error as MalloyError).problems[0].message).toContain(flag);
|
|
116
|
+
await named.close();
|
|
85
117
|
}
|
|
118
|
+
);
|
|
119
|
+
|
|
120
|
+
test('the flag admits a connection on the dialect', async () => {
|
|
121
|
+
await runtime().getModel(
|
|
122
|
+
[flag, `source: s is ${duckdbX}.sql('SELECT 1 as one')`].join('\n')
|
|
123
|
+
);
|
|
86
124
|
});
|
|
87
125
|
|
|
88
|
-
test('
|
|
89
|
-
await
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
126
|
+
test('importing a source on the dialect needs the flag, reported at the import', async () => {
|
|
127
|
+
const got = await refusal(
|
|
128
|
+
['-- line zero', `import "${libURL}"`, 'run: s -> { select: one }'].join(
|
|
129
|
+
'\n'
|
|
130
|
+
)
|
|
131
|
+
);
|
|
132
|
+
expect(got.message).toContain(flag);
|
|
133
|
+
expect(got.line).toBe(1);
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
test('importing only a query on the dialect needs the flag', async () => {
|
|
137
|
+
const got = await refusal(`import {q} from "${libURL}"`);
|
|
138
|
+
expect(got.message).toContain(flag);
|
|
139
|
+
});
|
|
140
|
+
|
|
141
|
+
test('the flag admits an import and use of the dialect', async () => {
|
|
142
|
+
const sql = await runtime()
|
|
143
|
+
.loadModel([flag, `import "${libURL}"`].join('\n'))
|
|
144
|
+
.loadQuery('run: s -> { select: one }')
|
|
145
|
+
.getSQL();
|
|
146
|
+
expect(sql).toContain('SELECT');
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
test('a query over a model needs nothing of its own', async () => {
|
|
150
|
+
const sql = await runtime()
|
|
151
|
+
.loadModel(
|
|
152
|
+
[flag, `source: s is ${duckdbX}.sql('SELECT 1 as one')`].join('\n')
|
|
153
|
+
)
|
|
154
|
+
.loadQuery('run: s -> { select: one }')
|
|
155
|
+
.getSQL();
|
|
156
|
+
expect(sql).toContain('SELECT');
|
|
157
|
+
});
|
|
158
|
+
|
|
159
|
+
test('an accepted dialect needs no flag anywhere', async () => {
|
|
160
|
+
// What the shared-suite harness does once per process for the dialect
|
|
161
|
+
// under test. Undone below by registering a fresh dialect object.
|
|
162
|
+
getDialect(duckdbX).acceptExperimental();
|
|
163
|
+
try {
|
|
164
|
+
const sql = await runtime()
|
|
165
|
+
.loadModel(
|
|
166
|
+
[
|
|
167
|
+
`import "${libURL}"`,
|
|
168
|
+
`source: t is ${duckdbX}.sql('SELECT 2 as two')`,
|
|
169
|
+
].join('\n')
|
|
170
|
+
)
|
|
171
|
+
.loadQuery('run: s -> { select: one }')
|
|
172
|
+
.getSQL();
|
|
173
|
+
expect(sql).toContain('SELECT');
|
|
174
|
+
} finally {
|
|
175
|
+
registerDialect(new DuckdbXDialect());
|
|
176
|
+
}
|
|
93
177
|
});
|
|
94
178
|
|
|
95
179
|
afterAll(async () => {
|
|
96
|
-
await
|
|
180
|
+
await connection.close();
|
|
97
181
|
registerDialect(new DuckDBDialect());
|
|
98
182
|
});
|
|
99
183
|
});
|
|
@@ -35,6 +35,33 @@ describe('loadRestrictedQuery — end-to-end through the public API', () => {
|
|
|
35
35
|
expect(result.data.path(0, 'ct').value).toBeGreaterThan(0);
|
|
36
36
|
});
|
|
37
37
|
|
|
38
|
+
test('timezone injection is rejected before SQL generation or execution', async () => {
|
|
39
|
+
const model = runtime.loadModel(`
|
|
40
|
+
source: states is duckdb.table('malloytest.state_facts')
|
|
41
|
+
`);
|
|
42
|
+
const queryText = `run: states -> {
|
|
43
|
+
timezone: "UTC' || (SELECT 'UTC') || '"
|
|
44
|
+
select: d is now.day
|
|
45
|
+
limit: 1
|
|
46
|
+
}`;
|
|
47
|
+
const expectedError = {
|
|
48
|
+
problems: expect.arrayContaining([
|
|
49
|
+
expect.objectContaining({
|
|
50
|
+
code: 'invalid-timezone',
|
|
51
|
+
severity: 'error',
|
|
52
|
+
}),
|
|
53
|
+
]),
|
|
54
|
+
};
|
|
55
|
+
// Separate query instances exercise each entry point without reusing
|
|
56
|
+
// the other entry point's cached compilation failure.
|
|
57
|
+
await expect(
|
|
58
|
+
model.loadRestrictedQuery(queryText).getSQL()
|
|
59
|
+
).rejects.toMatchObject(expectedError);
|
|
60
|
+
await expect(
|
|
61
|
+
model.loadRestrictedQuery(queryText).run()
|
|
62
|
+
).rejects.toMatchObject(expectedError);
|
|
63
|
+
});
|
|
64
|
+
|
|
38
65
|
test('restricted query with `import` throws MalloyError tagged restricted-mode', async () => {
|
|
39
66
|
const model = runtime.loadModel(`
|
|
40
67
|
source: states is duckdb.table('malloytest.state_facts')
|
package/src/core/tags.spec.ts
CHANGED
|
@@ -340,6 +340,28 @@ describe('tags in results', () => {
|
|
|
340
340
|
});
|
|
341
341
|
});
|
|
342
342
|
|
|
343
|
+
describe('property names inherited from Object.prototype', () => {
|
|
344
|
+
// A tag property name comes from source text, so `__proto__`, `constructor`
|
|
345
|
+
// and `toString` are ordinary names with no special meaning.
|
|
346
|
+
test('an annotation carries an inherited name through to the result', async () => {
|
|
347
|
+
const loaded = runtime.loadQuery(
|
|
348
|
+
`run: duckdb.sql("select 1 as num") -> {
|
|
349
|
+
select:
|
|
350
|
+
# __proto__=1 toString.nested=2 constructor
|
|
351
|
+
one is num
|
|
352
|
+
}`
|
|
353
|
+
);
|
|
354
|
+
const result = await loaded.run();
|
|
355
|
+
const one = result.resultExplore.getFieldByName('one');
|
|
356
|
+
const {tag, log} = one.tagParse();
|
|
357
|
+
expect(log).toEqual([]);
|
|
358
|
+
expect(tag.numeric('__proto__')).toEqual(1);
|
|
359
|
+
expect(tag.numeric('toString', 'nested')).toEqual(2);
|
|
360
|
+
expect(tag.has('constructor')).toBe(true);
|
|
361
|
+
expect(tag.has('valueOf')).toBe(false);
|
|
362
|
+
});
|
|
363
|
+
});
|
|
364
|
+
|
|
343
365
|
afterAll(async () => {
|
|
344
366
|
await runtime.connection.close();
|
|
345
367
|
});
|