@malloydata/db-snowflake 0.0.375 → 0.0.377

package/src/snowflake_connection.spec.ts

@@ -22,11 +22,19 @@
  */
 
 import * as malloy from '@malloydata/malloy';
-import {createTestRuntime, mkTestModel} from '@malloydata/malloy/test';
+import type {QueryData, RunSQLOptions} from '@malloydata/malloy';
+import {
+  createTestRuntime,
+  describeIfDatabaseAvailable,
+  mkTestModel,
+} from '@malloydata/malloy/test';
 import '@malloydata/malloy/test/matchers';
+import crypto from 'crypto';
 import {SnowflakeConnection} from './snowflake_connection';
 import {SnowflakeExecutor} from './snowflake_executor';
 
+const [describeSnowflakeExecutor] = describeIfDatabaseAvailable(['snowflake']);
+
 describe('db:Snowflake', () => {
   const connOptions =
     SnowflakeExecutor.getConnectionOptionsFromEnv() ||
@@ -129,7 +137,7 @@ describe('db:Snowflake', () => {
   it('discovers variant schema through a view', async () => {
     // Create a view with a variant column, then fetch its schema.
     // This exercises the TABLESAMPLE fallback path — TABLESAMPLE fails
-    // on views, so the code should fall back to LIMIT 100.
+    // on views, so the code should fall back to a plain LIMIT sample.
     const salt = Math.random().toString(36).slice(2, 10);
     const viewName = `malloytest.test_variant_view_${salt}`;
     await conn.runSQL(
@@ -147,6 +155,38 @@ describe('db:Snowflake', () => {
     }
   });
 
+  it('preserves top-level array shape when sample rows have no descendants', async () => {
+    // ARRAY comes from DESCRIBE TABLE, so even if recursive flatten sees no
+    // element paths we should still return an array<variant> field.
+    const salt = Math.random().toString(36).slice(2, 10);
+    const viewName = `malloytest.test_array_seed_${salt}`;
+    await conn.runSQL(
+      `CREATE OR REPLACE VIEW ${viewName} AS
+       SELECT ARRAY_CONSTRUCT() AS data`
+    );
+    try {
+      const schema = await conn.fetchTableSchema(viewName, viewName);
+      const dataField = schema.fields.find(f => f.name === 'DATA');
+      expect(dataField).toEqual({
+        type: 'array',
+        name: 'DATA',
+        join: 'many',
+        elementTypeDef: {type: 'sql native', rawType: 'variant'},
+        fields: [
+          {name: 'value', type: 'sql native', rawType: 'variant'},
+          {
+            name: 'each',
+            type: 'sql native',
+            rawType: 'variant',
+            e: {node: 'field', path: ['value']},
+          },
+        ],
+      });
+    } finally {
+      await conn.runSQL(`DROP VIEW IF EXISTS ${viewName}`);
+    }
+  });
+
   it('maps integer types to bigint', async () => {
     const x: malloy.SQLSourceDef = {
       type: 'sql_select',
@@ -170,16 +210,18 @@ describe('db:Snowflake', () => {
     ]);
   });
 
-  it('degrades variant field to sql native when types conflict across rows', async () => {
-    // data.foo is a scalar in one row and an object in another.
-    // Schema discovery should not throw — foo should degrade to sql native.
+  it('degrades scalar-vs-object field to sql native without losing siblings', async () => {
+    // data.foo is an object in one row and a scalar in another. Honest
+    // policy: foo becomes sql native variant (caller must cast with
+    // `foo :: {bar :: number}` to query bar). The enclosing record is
+    // unaffected — sibling fields keep their inferred types.
     const salt = Math.random().toString(36).slice(2, 10);
     const viewName = `malloytest.test_variant_conflict_${salt}`;
     await conn.runSQL(
       `CREATE OR REPLACE VIEW ${viewName} AS
-       SELECT parse_json('{"foo": {"bar": 1}}') AS data
+       SELECT parse_json('{"foo": {"bar": 1}, "sib": "hello"}') AS data
        UNION ALL
-       SELECT parse_json('{"foo": "oops"}') AS data`
+       SELECT parse_json('{"foo": "oops", "sib": "world"}') AS data`
     );
     try {
       const schema = await conn.fetchTableSchema(viewName, viewName);
@@ -193,22 +235,24 @@ describe('db:Snowflake', () => {
           rawType: 'variant',
           name: 'foo',
         });
+        const sibField = dataField!.fields.find(f => f.name === 'sib');
+        expect(sibField).toEqual({type: 'string', name: 'sib'});
       }
     } finally {
       await conn.runSQL(`DROP VIEW IF EXISTS ${viewName}`);
     }
   });
 
-  it('degrades nested object inside array when types conflict', async () => {
-    // Array analogue of the customer bug: items[*].foo is an object in
-    // one row and a scalar in another. foo should degrade to sql native.
+  it('degrades scalar-vs-object inside an array element without losing the array', async () => {
+    // Array analogue: items[*].foo is an object in one row and a scalar
+    // in another. foo degrades to variant; items stays array<record>.
     const salt = Math.random().toString(36).slice(2, 10);
     const viewName = `malloytest.test_variant_array_obj_conflict_${salt}`;
     await conn.runSQL(
       `CREATE OR REPLACE VIEW ${viewName} AS
-       SELECT parse_json('{"items": [{"foo": {"bar": 1}}]}') AS data
+       SELECT parse_json('{"items": [{"foo": {"bar": 1}, "sib": "a"}]}') AS data
        UNION ALL
-       SELECT parse_json('{"items": [{"foo": "oops"}]}') AS data`
+       SELECT parse_json('{"items": [{"foo": "oops", "sib": "b"}]}') AS data`
     );
     try {
       const schema = await conn.fetchTableSchema(viewName, viewName);
@@ -229,6 +273,8 @@ describe('db:Snowflake', () => {
             rawType: 'variant',
             name: 'foo',
           });
+          const sibField = itemsField!.fields.find(f => f.name === 'sib');
+          expect(sibField).toEqual({type: 'string', name: 'sib'});
         }
       }
     } finally {
@@ -236,6 +282,41 @@ describe('db:Snowflake', () => {
     }
   });
 
+  it('full-scans a small base table with variant columns under the byte threshold', async () => {
+    // Base table (not view) small enough that BYTES lands under the
+    // default 100 MB schemaSampleFullScanMaxBytes. The probe sees the
+    // size and the code takes the full-scan branch — no TABLESAMPLE,
+    // no LIMIT. Every row contributes to the (path, type) histogram,
+    // so rare fields are caught.
+    const salt = Math.random().toString(36).slice(2, 10);
+    const tableName = `malloytest.test_variant_fullscan_${salt}`;
+    await conn.runSQL(
+      `CREATE OR REPLACE TABLE ${tableName} AS
+       SELECT parse_json('{"foo": 1, "bar": "hi"}') AS data
+       UNION ALL
+       SELECT parse_json('{"foo": 2, "bar": "bye"}') AS data`
+    );
+    try {
+      const schema = await conn.fetchTableSchema(tableName, tableName);
+      const dataField = schema.fields.find(f => f.name === 'DATA');
+      expect(dataField).toBeDefined();
+      expect(dataField!.type).toBe('record');
+      if (dataField!.type === 'record') {
+        expect(dataField!.fields.find(f => f.name === 'foo')).toEqual({
+          name: 'foo',
+          type: 'number',
+          numberType: 'bigint',
+        });
+        expect(dataField!.fields.find(f => f.name === 'bar')).toEqual({
+          name: 'bar',
+          type: 'string',
+        });
+      }
+    } finally {
+      await conn.runSQL(`DROP TABLE IF EXISTS ${tableName}`);
+    }
+  });
+
   it('degrades when same path is object in one row and array in another', async () => {
     // foo is an object in one row and an array in another.
     // foo should degrade to sql native.
@@ -265,9 +346,10 @@ describe('db:Snowflake', () => {
     }
   });
 
-  it('preserves sibling fields when one field degrades', async () => {
-    // foo has conflicting types but stable is consistent.
-    // stable should come through normally.
+  it('preserves sibling fields when one field degrades to variant', async () => {
+    // foo is scalar in one row and object in another, while stable is
+    // always consistent. The degradation should stay local to foo and
+    // keep stable untouched.
     const salt = Math.random().toString(36).slice(2, 10);
     const viewName = `malloytest.test_variant_sibling_${salt}`;
     await conn.runSQL(
@@ -348,3 +430,125 @@ describe('numeric value reading', () => {
     );
   });
 });
+
+class SnowflakeExecutorTestSetup {
+  private executor_: SnowflakeExecutor;
+  constructor(executor: SnowflakeExecutor) {
+    this.executor_ = executor;
+  }
+
+  async runBatch(sqlText: string): Promise<QueryData> {
+    let ret: QueryData = [];
+    await (async () => {
+      const rows = await this.executor_.batch(sqlText);
+      return rows;
+    })().then((rows: QueryData) => {
+      ret = rows;
+    });
+    return ret;
+  }
+
+  async runStreaming(sqlText: string, queryOptions?: RunSQLOptions) {
+    const rows: QueryData = [];
+    await (async () => {
+      for await (const row of await this.executor_.stream(
+        sqlText,
+        queryOptions
+      )) {
+        rows.push(row);
+      }
+    })();
+    return rows;
+  }
+
+  async done() {
+    await this.executor_.done();
+  }
+}
+
+describeSnowflakeExecutor('db:SnowflakeExecutor', () => {
+  let db: SnowflakeExecutorTestSetup;
+  let query: string;
+
+  beforeAll(() => {
+    const connOptions =
+      SnowflakeExecutor.getConnectionOptionsFromEnv() ||
+      SnowflakeExecutor.getConnectionOptionsFromToml();
+    const executor = new SnowflakeExecutor(connOptions);
+    db = new SnowflakeExecutorTestSetup(executor);
+    query = `
+    select
+    *
+  from
+    (
+      values
+        (1, 'one'),
+        (2, 'two'),
+        (3, 'three'),
+        (4, 'four'),
+        (5, 'five')
+    );
+    `;
+  });
+
+  afterAll(async () => {
+    await db.done();
+  });
+
+  it('verifies batch execute', async () => {
+    const rows = await db.runBatch(query);
+    expect(rows.length).toBe(5);
+  });
+
+  it('verifies stream iterable', async () => {
+    const rows = await db.runStreaming(query, {rowLimit: 2});
+    expect(rows.length).toBe(2);
+  });
+});
+
+describe('setupSQL', () => {
+  const connOptions =
+    SnowflakeExecutor.getConnectionOptionsFromEnv() ||
+    SnowflakeExecutor.getConnectionOptionsFromToml();
+  const uid = crypto.randomBytes(4).toString('hex');
+  const connections: SnowflakeConnection[] = [];
+
+  function makeConn(name: string, setupSQL: string): SnowflakeConnection {
+    const conn = new SnowflakeConnection(name, {connOptions, setupSQL});
+    connections.push(conn);
+    return conn;
+  }
+
+  afterAll(async () => {
+    await Promise.all(connections.map(c => c.close()));
+  });
+
+  it('runs a single setup statement', async () => {
+    const conn = makeConn(
+      'snowflake_setup_single',
+      `SET setup_test_${uid} = 42`
+    );
+    const result = await conn.runSQL(`SELECT $setup_test_${uid} AS V`);
+    expect(malloy.API.rowDataToNumber(result.rows[0]['V'])).toBe(42);
+  });
+
+  it('runs multiple semicolon-newline-separated statements', async () => {
+    const conn = makeConn(
+      'snowflake_setup_multi',
+      [`SET setup_a_${uid} = 10`, `SET setup_b_${uid} = 20`].join(';\n')
+    );
+    const result = await conn.runSQL(
+      `SELECT $setup_a_${uid} + $setup_b_${uid} AS V`
+    );
+    expect(malloy.API.rowDataToNumber(result.rows[0]['V'])).toBe(30);
+  });
+
+  it('handles multi-line statements', async () => {
+    const conn = makeConn(
+      'snowflake_setup_multiline',
+      `SET\n  setup_ml_${uid} = 99`
+    );
+    const result = await conn.runSQL(`SELECT $setup_ml_${uid} AS V`);
+    expect(malloy.API.rowDataToNumber(result.rows[0]['V'])).toBe(99);
+  });
+});