@malloydata/malloy-tests 0.0.433 → 0.0.434

package/postgres/load_test_data.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright Contributors to the Malloy project
+ * SPDX-License-Identifier: MIT
+ */
+
+/* eslint-disable no-console */
+
+/*
+ * Load the malloytest tables into a running postgres from the parquet files.
+ * Connects the way psql and the postgres test connection do, through libpq's
+ * PG* variables, which the extension reads when the attach string is empty.
+ */
+
+import {
+  loadExtension,
+  loadMalloytest,
+  openDuckDB,
+} from '../data/parquet_loader';
+
+(async () => {
+  const db = await openDuckDB();
+  await loadExtension(db, 'postgres');
+  await db.run("ATTACH '' AS pg (TYPE postgres)");
+  // The sampling tests use TABLESAMPLE SYSTEM_ROWS
+  await db.run(
+    "CALL postgres_execute('pg', 'CREATE EXTENSION IF NOT EXISTS tsm_system_rows')"
+  );
+  console.log('Loading malloytest into postgres');
+  await loadMalloytest(db, 'pg.malloytest', {
+    // Postgres has no anonymous record type for the nested columns
+    skip: ['ga_sample'],
+  });
+  await db.close();
+})();

package/postgres/postgres_start.sh

@@ -7,9 +7,7 @@ set -e
 rm -rf .tmp
 mkdir .tmp
 
-# run docker
 SCRIPTDIR=$(cd $(dirname $0); pwd)
-DATADIR=$(dirname $SCRIPTDIR)/data/postgres
 
 # Verify required environment variables and show export commands for any that are wrong
 EXPORTS=""
@@ -55,7 +53,7 @@ if docker container inspect "$CONTAINER_NAME" > /dev/null 2>&1; then
   exit 0
 fi
 
-docker run -p 5432:5432 -d -v $DATADIR:/init_data \
+docker run -p 5432:5432 -d \
  --name "$CONTAINER_NAME" \
   -e POSTGRES_USER=root -e POSTGRES_PASSWORD=postgres \
   -e TZ=UTC \
@@ -74,8 +72,6 @@ done
 echo "Container $CONTAINER_NAME is now healthy!"
 
 echo "Loading data ..."
-#  configure
-echo CREATE EXTENSION tsm_system_rows\; | psql
-gunzip -c ${DATADIR}/malloytest-postgres.sql.gz | psql
+sh "$SCRIPTDIR/load_test_data.sh"
 
 echo "Ready"

package/presto/Dockerfile.slim

@@ -1,11 +1,13 @@
 # Slim Presto image for Malloy CI testing
 #
-# This image is ~850MB vs ~9GB for the official prestodb/presto image.
-# It includes only what's needed to run Presto with the BigQuery connector.
+# This image is ~1.4GB vs ~9GB for the official prestodb/presto image.
+# It includes only what's needed to run Presto with the hive connector, which
+# serves the test parquet (see hive.properties), and the CLI that creates the
+# tables.
 #
 # How this was created:
 # 1. The official prestodb/presto:0.287 image contains 42 connector plugins (~2GB)
-#    but we only need presto-bigquery (~36MB)
+#    but we only need hive-hadoop2 (~306MB)
 # 2. We use eclipse-temurin as the base because:
 #    - Presto requires Java 11 (not newer - tested, Java 17 fails with classloading errors)
 #    - Presto requires full JDK (not JRE - needs com.sun.tools.attach classes)
@@ -18,7 +20,7 @@
 #
 # Size comparison (as of 0.287):
 #   Official image: 9.11 GB
-#   This slim image: ~850 MB
+#   This slim image: ~1.4 GB
 
 ARG PRESTO_VERSION=0.287
 
@@ -39,8 +41,12 @@ COPY --from=presto-official /opt/presto-server/lib /opt/presto-server/lib
 COPY --from=presto-official /opt/presto-server/bin /opt/presto-server/bin
 COPY --from=presto-official /opt/presto-server/etc /opt/presto-server/etc
 
-# Copy only BigQuery plugin (the only connector we use for testing)
-COPY --from=presto-official /opt/presto-server/plugin/presto-bigquery /opt/presto-server/plugin/presto-bigquery
+# Copy only the hive plugin (the only connector we use for testing)
+COPY --from=presto-official /opt/presto-server/plugin/hive-hadoop2 /opt/presto-server/plugin/hive-hadoop2
+
+# The CLI, which presto_start.sh uses to create the tables
+COPY --from=presto-official /opt/presto-cli /opt/presto-cli
+RUN ln -s /opt/presto-cli /usr/local/bin/presto-cli
 
 EXPOSE 8080
 

package/presto/hive.properties

@@ -0,0 +1,8 @@
+# The hive catalog serves test/data/malloytest-parquet, mounted read-only
+# under /data/malloytest, with the tables hive_ddl.ts declares. The file
+# metastore lives inside the container.
+connector.name=hive-hadoop2
+hive.metastore=file
+hive.metastore.catalog.dir=file:///tmp/hive-metastore
+# Match parquet columns by name; presto's default is by position
+hive.parquet.use-column-names=true

package/presto/presto_start.sh

@@ -1,10 +1,19 @@
 #! /bin/bash
-
+#
+# Start a presto container whose hive catalog serves test/data/malloytest-parquet
+# as the malloytest schema. The parquet is read in place; nothing is loaded.
+#
+#   presto_start.sh [--slim]   use the slim image from Dockerfile.slim
+#
 set -e
 
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SCRIPTDIR=$(cd $(dirname $0); pwd)
+ROOTDIR=$(cd "$SCRIPTDIR/../.."; pwd)
+DATADIR=$ROOTDIR/test/data/malloytest-parquet
 PRESTO_VERSION=0.287
 USE_SLIM=false
+CONTAINER_NAME="presto-malloy"
+PORT=${PRESTO_PORT:-8080}
 
 # Parse arguments
 for arg in "$@"; do
@@ -16,32 +25,40 @@ for arg in "$@"; do
   esac
 done
 
+cd "$ROOTDIR"
 rm -rf .tmp
 mkdir .tmp
 
-# generate config file
-> ./.tmp/bigquery-pesto.properties
-cat << EOF > ./.tmp/bigquery-pesto.properties
-connector.name=bigquery
-bigquery.project-id=advance-lacing-417917
-bigquery.credentials-key=$BQ_CREDENTIALS_KEY
-EOF
-
 # Select image
 if [ "$USE_SLIM" = true ]; then
   PRESTO_IMAGE="presto-slim:${PRESTO_VERSION}"
   # Build slim image if it doesn't exist
   if ! docker image inspect "$PRESTO_IMAGE" > /dev/null 2>&1; then
     echo "Building slim Presto image..."
-    docker build -f "$SCRIPT_DIR/Dockerfile.slim" \
+    docker build -f "$SCRIPTDIR/Dockerfile.slim" \
       --build-arg PRESTO_VERSION="$PRESTO_VERSION" \
-      -t "$PRESTO_IMAGE" "$SCRIPT_DIR"
+      -t "$PRESTO_IMAGE" "$SCRIPTDIR"
   fi
 else
   PRESTO_IMAGE="prestodb/presto:${PRESTO_VERSION}"
 fi
 
-CONTAINER_NAME="presto-malloy"
+# Only the log since this script started: a restarted container's old log
+# already says SERVER STARTED.
+SINCE=$(date -u +%Y-%m-%dT%H:%M:%SZ)
+
+wait_for_presto() {
+  local counter=0
+  while ! docker logs --since "$SINCE" "$CONTAINER_NAME" 2>&1 | grep -q "SERVER STARTED"; do
+    counter=$((counter+1))
+    # give up after 2 minutes
+    if [ $counter -eq 120 ]; then
+      docker logs "$CONTAINER_NAME" >& ./.tmp/presto-malloy.logs
+      return 1
+    fi
+    sleep 1
+  done
+}
 
 # Check for existing container
 if docker container inspect "$CONTAINER_NAME" > /dev/null 2>&1; then
@@ -51,31 +68,36 @@ if docker container inspect "$CONTAINER_NAME" > /dev/null 2>&1; then
   fi
   echo "Restarting existing $CONTAINER_NAME container..."
   docker start "$CONTAINER_NAME"
-  while ! docker logs "$CONTAINER_NAME" 2>&1 | grep -q "SERVER STARTED"; do
-    sleep 1
-  done
-  echo "Presto running on port 8080"
+  if ! wait_for_presto; then
+    echo "Presto did not restart successfully, check .tmp/presto-malloy.logs"
+    exit 1
+  fi
+  echo "Presto running on port $PORT"
   exit 0
 fi
 
-# run docker
-docker run -p ${PRESTO_PORT:-8080}:8080 -d -v ./.tmp/bigquery-pesto.properties:/opt/presto-server/etc/catalog/bigquery.properties --name "$CONTAINER_NAME" "$PRESTO_IMAGE"
-
-# wait for server to start
-counter=0
-while ! docker logs "$CONTAINER_NAME" 2>&1 | grep -q "SERVER STARTED"
-do
-  sleep 1
-  counter=$((counter+1))
-  # if doesn't start after 2 minutes, output logs and kill process
-  if [ $counter -eq 120 ]
-  then
-    docker logs "$CONTAINER_NAME" >& ./.tmp/presto-malloy.logs
-    docker rm -f "$CONTAINER_NAME"
-    echo "Presto did not start successfully, check .tmp/presto-malloy.logs"
-    exit 1
-    break
-  fi
+# A hive external_location is a directory, so each parquet file gets one.
+MOUNTS=()
+for parquet in "$DATADIR"/*.parquet; do
+  table=$(basename "$parquet" .parquet)
+  MOUNTS+=(-v "$parquet:/data/malloytest/$table/$table.parquet:ro")
 done
 
-echo "Presto running on port 8080"
+docker run -p $PORT:8080 -d "${MOUNTS[@]}" \
+  -v "$SCRIPTDIR/hive.properties:/opt/presto-server/etc/catalog/hive.properties:ro" \
+  --name "$CONTAINER_NAME" "$PRESTO_IMAGE"
+
+# A container without its tables must not survive to be "already running"
+trap 'docker rm -f "$CONTAINER_NAME" > /dev/null; echo "Presto setup failed, check .tmp/presto-malloy.logs"' ERR
+
+if ! wait_for_presto; then
+  false
+fi
+
+echo "Creating the malloytest tables"
+npx ts-node test/trino/hive_ddl.ts hive file:///data/malloytest --presto > .tmp/hive_ddl.sql
+docker cp .tmp/hive_ddl.sql "$CONTAINER_NAME":/tmp/hive_ddl.sql
+docker exec "$CONTAINER_NAME" presto-cli -f /tmp/hive_ddl.sql
+trap - ERR
+
+echo "Presto running on port $PORT"

package/src/core/restricted-mode.spec.ts

@@ -35,6 +35,33 @@ describe('loadRestrictedQuery — end-to-end through the public API', () => {
     expect(result.data.path(0, 'ct').value).toBeGreaterThan(0);
   });
 
+  test('timezone injection is rejected before SQL generation or execution', async () => {
+    const model = runtime.loadModel(`
+      source: states is duckdb.table('malloytest.state_facts')
+    `);
+    const queryText = `run: states -> {
+      timezone: "UTC' || (SELECT 'UTC') || '"
+      select: d is now.day
+      limit: 1
+    }`;
+    const expectedError = {
+      problems: expect.arrayContaining([
+        expect.objectContaining({
+          code: 'invalid-timezone',
+          severity: 'error',
+        }),
+      ]),
+    };
+    // Separate query instances exercise each entry point without reusing
+    // the other entry point's cached compilation failure.
+    await expect(
+      model.loadRestrictedQuery(queryText).getSQL()
+    ).rejects.toMatchObject(expectedError);
+    await expect(
+      model.loadRestrictedQuery(queryText).run()
+    ).rejects.toMatchObject(expectedError);
+  });
+
   test('restricted query with `import` throws MalloyError tagged restricted-mode', async () => {
     const model = runtime.loadModel(`
       source: states is duckdb.table('malloytest.state_facts')

package/src/core/tags.spec.ts

@@ -340,6 +340,28 @@ describe('tags in results', () => {
   });
 });
 
+describe('property names inherited from Object.prototype', () => {
+  // A tag property name comes from source text, so `__proto__`, `constructor`
+  // and `toString` are ordinary names with no special meaning.
+  test('an annotation carries an inherited name through to the result', async () => {
+    const loaded = runtime.loadQuery(
+      `run: duckdb.sql("select 1 as num") -> {
+        select:
+          # __proto__=1 toString.nested=2 constructor
+          one is num
+      }`
+    );
+    const result = await loaded.run();
+    const one = result.resultExplore.getFieldByName('one');
+    const {tag, log} = one.tagParse();
+    expect(log).toEqual([]);
+    expect(tag.numeric('__proto__')).toEqual(1);
+    expect(tag.numeric('toString', 'nested')).toEqual(2);
+    expect(tag.has('constructor')).toBe(true);
+    expect(tag.has('valueOf')).toBe(false);
+  });
+});
+
 afterAll(async () => {
   await runtime.connection.close();
 });

package/src/core/virtual-sql.spec.ts

@@ -0,0 +1,253 @@
+/*
+ * Copyright Contributors to the Malloy project
+ * SPDX-License-Identifier: MIT
+ */
+
+import {DuckDBConnection} from '@malloydata/db-duckdb';
+import {
+  FixedConnectionMap,
+  InMemoryURLReader,
+  Malloy,
+  MalloyConfig,
+  Runtime,
+} from '@malloydata/malloy';
+import type {VirtualMap} from '@malloydata/malloy';
+
+const virtualSource = `
+  ##! experimental.virtual_source
+  type: row_fields is { x :: number }
+  source: data is duckdb.virtual('data')::row_fields
+`;
+const sqlSource = `
+  source: wrapped is duckdb.sql("""
+    SELECT x + 10 AS y FROM %{ data -> { select: x } }
+  """)
+  run: wrapped -> { select: y }
+`;
+const source = virtualSource + sqlSource;
+
+function virtualMap(table: string): VirtualMap {
+  return new Map([['duckdb', new Map([['data', table]])]]);
+}
+
+// Language cases live in packages/malloy/src/lang/test/virtual-sql.spec.ts.
+// These tests cover API option forwarding and real SQL schemas.
+describe('virtual sources in SQL blocks through the Foundation API', () => {
+  const connection = new DuckDBConnection({
+    name: 'duckdb',
+    databasePath: ':memory:',
+  });
+  const connections = FixedConnectionMap.fromArray([connection]);
+  const compileMap = virtualMap('translation_data');
+  const executionMap = virtualMap('execution_data');
+  const urlReader = new InMemoryURLReader(new Map());
+
+  beforeAll(async () => {
+    await connection.runSQL(
+      'CREATE TEMP TABLE translation_data AS SELECT 1::INTEGER AS x'
+    );
+    await connection.runSQL(
+      'CREATE TEMP TABLE execution_data AS SELECT 2::INTEGER AS x'
+    );
+  });
+
+  afterAll(async () => {
+    await connection.close();
+  });
+
+  afterEach(() => {
+    jest.restoreAllMocks();
+  });
+
+  test('Malloy.compile uses a translation map without binding execution', async () => {
+    const model = await Malloy.compile({
+      source,
+      connections,
+      urlReader,
+      virtualMap: compileMap,
+    });
+    const result = model.preparedQuery.getPreparedResult({
+      virtualMap: executionMap,
+    });
+    expect(result.sql).toContain('execution_data');
+    expect(result.sql).not.toContain('translation_data');
+    expect((await connection.runSQL(result.sql)).rows).toEqual([{y: 12}]);
+    expect(() => model.preparedQuery.getPreparedResult()).toThrow(
+      /No virtual-map entry/
+    );
+  });
+
+  test('loadModel forwards its map and execution can override it', async () => {
+    const runtime = new Runtime({connections});
+    runtime.virtualMap = virtualMap('missing_runtime_table');
+    const result = await runtime
+      .loadModel(source, {virtualMap: compileMap})
+      .loadFinalQuery()
+      .run({virtualMap: executionMap});
+    expect(result.data.value).toEqual([{y: 12}]);
+  });
+
+  test('runtime config supplies the translation map', async () => {
+    const config = new MalloyConfig({
+      virtualMap: {duckdb: {data: 'translation_data'}},
+    });
+    const runtime = new Runtime({connections, config});
+    const result = await runtime.loadQuery(source).run();
+    expect(result.data.value).toEqual([{y: 11}]);
+  });
+
+  test('model.loadQuery uses an explicit translation map', async () => {
+    const runtime = new Runtime({connections});
+    runtime.virtualMap = virtualMap('missing_runtime_table');
+    const result = await runtime
+      .loadModel(virtualSource)
+      .loadQuery(sqlSource, {virtualMap: compileMap})
+      .run({virtualMap: executionMap});
+    expect(result.data.value).toEqual([{y: 12}]);
+  });
+
+  test('model.loadQuery falls back to the runtime translation map', async () => {
+    const runtime = new Runtime({connections});
+    runtime.virtualMap = compileMap;
+    const result = await runtime
+      .loadModel(virtualSource)
+      .loadQuery(sqlSource)
+      .run({virtualMap: executionMap});
+    expect(result.data.value).toEqual([{y: 12}]);
+  });
+
+  test('model.loadQuery inherits the model translation map', async () => {
+    const runtime = new Runtime({connections});
+    runtime.virtualMap = virtualMap('missing_runtime_table');
+    const result = await runtime
+      .loadModel(virtualSource, {virtualMap: compileMap})
+      .loadQuery(sqlSource)
+      .run({virtualMap: executionMap});
+    expect(result.data.value).toEqual([{y: 12}]);
+  });
+
+  test('an empty translation map overrides the runtime map', async () => {
+    const runtime = new Runtime({connections});
+    runtime.virtualMap = compileMap;
+    await expect(
+      runtime.loadModel(source, {virtualMap: new Map()}).getModel()
+    ).rejects.toThrow(/No virtual-map entry/);
+  });
+
+  test('an invalid translation binding fails before SQL schema discovery', async () => {
+    const fetchSchema = jest
+      .spyOn(connection, 'fetchSchemaForSQLStruct')
+      .mockRejectedValue(new Error('Unexpected SQL schema request'));
+    await expect(
+      Malloy.compile({
+        source,
+        connections,
+        urlReader,
+        virtualMap: virtualMap('translation_data; SELECT 1'),
+      })
+    ).rejects.toThrow(/virtualMap entry 'duckdb.data'.*duckdb/);
+    expect(fetchSchema).not.toHaveBeenCalled();
+  });
+
+  test('an invalid execution binding fails before query execution', async () => {
+    const runtime = new Runtime({connections});
+    const query = runtime.loadQuery(source, {virtualMap: compileMap});
+    await query.getPreparedQuery();
+    const runSQL = jest
+      .spyOn(connection, 'runSQL')
+      .mockRejectedValue(new Error('Unexpected query execution'));
+    await expect(
+      query.run({virtualMap: virtualMap('execution_data; SELECT 1')})
+    ).rejects.toThrow(/virtualMap entry 'duckdb.data'.*duckdb/);
+    expect(runSQL).not.toHaveBeenCalled();
+  });
+
+  test.each(['explicit', 'inherited', 'runtime'])(
+    'extendModel uses its %s translation map',
+    async binding => {
+      const runtime = new Runtime({connections});
+      runtime.virtualMap =
+        binding === 'runtime' ? compileMap : virtualMap('missing_table');
+      const model = runtime.loadModel(virtualSource, {
+        virtualMap:
+          binding === 'inherited'
+            ? compileMap
+            : binding === 'explicit'
+              ? virtualMap('missing_inherited_table')
+              : undefined,
+      });
+      const extended = model.extendModel(
+        sqlSource,
+        binding === 'explicit' ? {virtualMap: compileMap} : undefined
+      );
+      const result = await extended.loadFinalQuery().run();
+      expect(result.data.value).toEqual([{y: 11}]);
+    }
+  );
+
+  test('Malloy.compile forwards its map to a pre-parsed translator', async () => {
+    const parse = Malloy.parse({source});
+    const model = await Malloy.compile({
+      parse,
+      connections,
+      urlReader,
+      virtualMap: compileMap,
+    });
+    const result = model.preparedQuery.getPreparedResult({
+      virtualMap: executionMap,
+    });
+    expect((await connection.runSQL(result.sql)).rows).toEqual([{y: 12}]);
+  });
+
+  test.each(['compile', 'run'])(
+    '%s ignores an invalid override binding when it is unused',
+    async operation => {
+      const badMap = virtualMap('translation_data');
+      badMap.get('duckdb')!.set('unused', 'translation_data; SELECT 1');
+      const runtime = new Runtime({connections});
+      const result =
+        operation === 'compile'
+          ? Malloy.compile({source, connections, urlReader, virtualMap: badMap})
+          : runtime
+              .loadQuery(source, {virtualMap: compileMap})
+              .run({virtualMap: badMap});
+      await expect(result).resolves.toBeDefined();
+    }
+  );
+
+  test.each(['config', 'assignment'])(
+    'bindings from %s are preserved and checked only when used',
+    async kind => {
+      const badTable = 'translation_data; SELECT 1';
+      const config =
+        kind === 'config'
+          ? new MalloyConfig({
+              virtualMap: {
+                duckdb: {data: 'translation_data', unused: badTable},
+              },
+            })
+          : undefined;
+      const runtime = new Runtime({connections, config});
+      if (kind === 'assignment') {
+        const map = virtualMap('translation_data');
+        map.get('duckdb')!.set('unused', badTable);
+        runtime.virtualMap = map;
+      }
+      expect(runtime.virtualMap?.get('duckdb')?.get('unused')).toBe(badTable);
+      expect((await runtime.loadQuery(source).run()).data.value).toEqual([
+        {y: 11},
+      ]);
+      await expect(
+        runtime
+          .loadQuery(
+            `
+          ${virtualSource}
+          source: bad is duckdb.virtual('unused')::row_fields
+          run: bad -> { select: x }
+        `
+          )
+          .getSQL()
+      ).rejects.toThrow(/virtualMap entry 'duckdb.unused'.*duckdb/);
+    }
+  );
+});