@malloydata/db-snowflake 0.0.375 → 0.0.377

package/dist/snowflake_variant_schema.d.ts

@@ -0,0 +1,43 @@
+import type { Dialect, FieldDef } from '@malloydata/malloy';
+import { TinyParser } from '@malloydata/malloy/internal';
+export type NestedColumnKind = 'variant' | 'array' | 'object';
+export interface NestedColumn {
+    kind: NestedColumnKind;
+    name: string;
+}
+export type Segment = {
+    kind: 'name';
+    name: string;
+} | {
+    kind: 'array';
+};
+export type Shape = {
+    kind: 'object';
+} | {
+    kind: 'array';
+} | {
+    kind: 'leaf';
+    type: string;
+} | {
+    kind: 'variant';
+};
+export type PrefixKey = string;
+export interface Children {
+    elem?: PrefixKey;
+    named: Map<string, PrefixKey>;
+}
+export interface VariantSchemaState {
+    children: Map<PrefixKey, Children>;
+    seededTopLevels: Set<PrefixKey>;
+    shapes: Map<PrefixKey, Shape>;
+}
+export declare class PathParser extends TinyParser {
+    constructor(pathName: string);
+    getName(): string;
+    segments(): Segment[];
+}
+export declare function createVariantSchemaState(): VariantSchemaState;
+export declare function mergeShape(existing: Shape | undefined, incoming: Shape): Shape;
+export declare function seedTopLevelShape(state: VariantSchemaState, nestedColumn: NestedColumn): void;
+export declare function accumulateVariantPath(state: VariantSchemaState, segments: Segment[], fieldType: string): void;
+export declare function buildTopLevelField(nestedColumn: NestedColumn, state: VariantSchemaState, dialect: Dialect): FieldDef;

package/dist/snowflake_variant_schema.js

@@ -0,0 +1,203 @@
+"use strict";
+/*
+ * Copyright Contributors to the Malloy project
+ * SPDX-License-Identifier: MIT
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PathParser = void 0;
+exports.createVariantSchemaState = createVariantSchemaState;
+exports.mergeShape = mergeShape;
+exports.seedTopLevelShape = seedTopLevelShape;
+exports.accumulateVariantPath = accumulateVariantPath;
+exports.buildTopLevelField = buildTopLevelField;
+const internal_1 = require("@malloydata/malloy/internal");
+class PathParser extends internal_1.TinyParser {
+    constructor(pathName) {
+        super(pathName, {
+            quoted: /^'(\\'|[^'])*'/,
+            array_of: /^\[\*]/,
+            char: /^[[.\]]/,
+            number: /^\d+/,
+            word: /^\w+/,
+        });
+    }
+    getName() {
+        const word = this.match('word');
+        if (word)
+            return word.text;
+        if (this.match('[')) {
+            const quotedName = this.expect('quoted');
+            this.expect(']');
+            return quotedName.text;
+        }
+        throw this.parseError('Expected column name');
+    }
+    segments() {
+        const segments = [{ kind: 'name', name: this.getName() }];
+        while (!this.eof()) {
+            if (this.match('.')) {
+                segments.push({ kind: 'name', name: this.expect('word').text });
+            }
+            else if (this.match('array_of')) {
+                segments.push({ kind: 'array' });
+            }
+            else if (this.match('[')) {
+                const quoted = this.expect('quoted');
+                this.expect(']');
+                segments.push({ kind: 'name', name: quoted.text });
+            }
+            else {
+                throw this.parseError(`Unexpected ${this.peek().type}`);
+            }
+        }
+        return segments;
+    }
+}
+exports.PathParser = PathParser;
+function createVariantSchemaState() {
+    return {
+        children: new Map(),
+        seededTopLevels: new Set(),
+        shapes: new Map(),
+    };
+}
+// The single consistency-policy point: any shape conflict across samples
+// collapses to `variant`, and `variant` is absorbing (monotonic).
+function mergeShape(existing, incoming) {
+    if (existing === undefined) {
+        return incoming;
+    }
+    if (existing.kind === 'variant' || incoming.kind === 'variant') {
+        return { kind: 'variant' };
+    }
+    if (existing.kind !== incoming.kind) {
+        return { kind: 'variant' };
+    }
+    if (existing.kind === 'leaf' &&
+        incoming.kind === 'leaf' &&
+        existing.type !== incoming.type) {
+        return { kind: 'variant' };
+    }
+    return existing;
+}
+function seedTopLevelShape(state, nestedColumn) {
+    const key = prefixKey([{ kind: 'name', name: nestedColumn.name }]);
+    if (nestedColumn.kind === 'array' || nestedColumn.kind === 'object') {
+        state.seededTopLevels.add(key);
+        state.shapes.set(key, { kind: nestedColumn.kind });
+    }
+}
+function accumulateVariantPath(state, segments, fieldType) {
+    if (segments.length === 0) {
+        return;
+    }
+    const topLevelKey = prefixKey(segments.slice(0, 1));
+    const topLevelShape = state.shapes.get(topLevelKey);
+    const topLevelIncoming = segments.length === 1
+        ? observedTypeToShape(fieldType)
+        : nextSegmentToShape(segments[1]);
+    if (state.seededTopLevels.has(topLevelKey) &&
+        topLevelShape &&
+        (topLevelShape.kind === 'array' || topLevelShape.kind === 'object') &&
+        topLevelShape.kind !== topLevelIncoming.kind) {
+        // Rule 1: top-level ARRAY/OBJECT from DESCRIBE are authoritative.
+        // Ignore impossible sample rows rather than letting them override the seed.
+        return;
+    }
+    let parentKey;
+    for (let i = 0; i < segments.length; i++) {
+        const prefixSegments = segments.slice(0, i + 1);
+        const key = prefixKey(prefixSegments);
+        const impliedShape = i === segments.length - 1
+            ? observedTypeToShape(fieldType)
+            : nextSegmentToShape(segments[i + 1]);
+        state.shapes.set(key, mergeShape(state.shapes.get(key), impliedShape));
+        if (parentKey !== undefined) {
+            recordChild(state, parentKey, segments[i], key);
+        }
+        parentKey = key;
+    }
+}
+function buildTopLevelField(nestedColumn, state, dialect) {
+    // Snowflake nested-schema inference follows these rules:
+    // - top-level ARRAY/OBJECT from DESCRIBE are authoritative
+    // - descendant paths imply ancestor shape
+    // - conflicting shapes degrade only that prefix to variant
+    // - every top-level nested column still produces a field
+    //
+    // Snowflake-specific semantic note: reconstructing object shape from
+    // descendant paths is safe because path access on an incompatible VARIANT
+    // value yields NULL rather than raising an error.
+    const key = prefixKey([{ kind: 'name', name: nestedColumn.name }]);
+    const shape = state.shapes.get(key);
+    if (shape === undefined) {
+        // Top-level ARRAY with no usable descendants still stays queryable as
+        // array<variant>; top-level OBJECT/VARIANT degrades to opaque variant.
+        return (0, internal_1.mkFieldDef)(nestedColumn.kind === 'array'
+            ? (0, internal_1.mkArrayTypeDef)(opaqueVariantType())
+            : opaqueVariantType(), nestedColumn.name);
+    }
+    return (0, internal_1.mkFieldDef)(buildTypeForKey(key, state, dialect), nestedColumn.name);
+}
+function buildTypeForKey(key, state, dialect) {
+    var _a, _b;
+    const shape = state.shapes.get(key);
+    if (shape === undefined || shape.kind === 'variant') {
+        return opaqueVariantType();
+    }
+    if (shape.kind === 'leaf') {
+        return dialect.sqlTypeToMalloyType(shape.type);
+    }
+    if (shape.kind === 'object') {
+        const namedChildren = (_a = state.children.get(key)) === null || _a === void 0 ? void 0 : _a.named;
+        if (!namedChildren || namedChildren.size === 0) {
+            return opaqueVariantType();
+        }
+        return {
+            type: 'record',
+            fields: [...namedChildren.entries()].map(([childName, childKey]) => (0, internal_1.mkFieldDef)(buildTypeForKey(childKey, state, dialect), childName)),
+        };
+    }
+    const elemKey = (_b = state.children.get(key)) === null || _b === void 0 ? void 0 : _b.elem;
+    const elementType = elemKey
+        ? buildTypeForKey(elemKey, state, dialect)
+        : opaqueVariantType();
+    return (0, internal_1.mkArrayTypeDef)(elementType);
+}
+function nextSegmentToShape(segment) {
+    return segment.kind === 'array' ? { kind: 'array' } : { kind: 'object' };
+}
+function observedTypeToShape(fieldType) {
+    // Defensive: production callers only pass Snowflake TYPEOF() results, which
+    // are expected to be array/object or scalar leaf types, not "variant".
+    if (fieldType === 'array' ||
+        fieldType === 'object' ||
+        fieldType === 'variant') {
+        return { kind: fieldType };
+    }
+    return { kind: 'leaf', type: fieldType };
+}
+function recordChild(state, parentKey, segment, childKey) {
+    let children = state.children.get(parentKey);
+    if (children === undefined) {
+        children = { named: new Map() };
+        state.children.set(parentKey, children);
+    }
+    if (segment.kind === 'array') {
+        children.elem = childKey;
+    }
+    else {
+        children.named.set(segment.name, childKey);
+    }
+}
+function opaqueVariantType() {
+    return { type: 'sql native', rawType: 'variant' };
+}
+function prefixKey(segments) {
+    const [first, ...rest] = segments;
+    if ((first === null || first === void 0 ? void 0 : first.kind) !== 'name') {
+        throw new Error('Snowflake schema path must start with a named segment');
+    }
+    return (0, internal_1.pathToKey)(first.name, rest.map(segment => (segment.kind === 'array' ? '[*]' : segment.name)));
+}
+//# sourceMappingURL=snowflake_variant_schema.js.map

package/dist/snowflake_variant_schema.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"snowflake_variant_schema.js","sourceRoot":"","sources":["../src/snowflake_variant_schema.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAyGH,4DAMC;AAID,gCAqBC;AAED,8CASC;AAED,sDAsCC;AAED,gDA2BC;AA3LD,0DAKqC;AA8BrC,MAAa,UAAW,SAAQ,qBAAU;IACxC,YAAY,QAAgB;QAC1B,KAAK,CAAC,QAAQ,EAAE;YACd,MAAM,EAAE,gBAAgB;YACxB,QAAQ,EAAE,QAAQ;YAClB,IAAI,EAAE,SAAS;YACf,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,MAAM;SACb,CAAC,CAAC;IACL,CAAC;IAED,OAAO;QACL,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAChC,IAAI,IAAI;YAAE,OAAO,IAAI,CAAC,IAAI,CAAC;QAC3B,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;YACpB,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACzC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACjB,OAAO,UAAU,CAAC,IAAI,CAAC;QACzB,CAAC;QACD,MAAM,IAAI,CAAC,UAAU,CAAC,sBAAsB,CAAC,CAAC;IAChD,CAAC;IAED,QAAQ;QACN,MAAM,QAAQ,GAAc,CAAC,EAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,OAAO,EAAE,EAAC,CAAC,CAAC;QACnE,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;YACnB,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;gBACpB,QAAQ,CAAC,IAAI,CAAC,EAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAC,CAAC,CAAC;YAChE,CAAC;iBAAM,IAAI,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC;gBAClC,QAAQ,CAAC,IAAI,CAAC,EAAC,IAAI,EAAE,OAAO,EAAC,CAAC,CAAC;YACjC,CAAC;iBAAM,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC3B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;gBACrC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBACjB,QAAQ,CAAC,IAAI,CAAC,EAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAC,CAAC,CAAC;YACnD,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,CAAC,UAAU,CAAC,cAAc,IAAI,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;YAC1D,CAAC;QACH,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;CACF;AAvCD,gCAuCC;AAED,SAAgB,wBAAwB;IACtC,OAAO;QACL,QAAQ,EAAE,IAAI,GAAG,EAAE;QACnB,eAAe,EAAE,IAAI,GAAG,EAAE;QAC1B,MAAM,EAAE,IAAI,GAAG,EAAE;KAClB,CAAC;AACJ,CAAC;AAED,yEAAyE;AACzE,kEAAkE;AAClE,SAAgB,UAAU,CACxB,QAA2B,EAC3B,QAAe;IAEf,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3B,OAAO,QAAQ,CAAC;IAClB,CAAC;IACD,IAAI,QAAQ,CAAC,IAAI,KAAK,SAAS,IAAI,QAAQ,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC/D,OAAO,EAAC,IAAI,EAAE,SAAS,EAAC,CAAC;IAC3B,CAAC;IACD,IAAI,QAAQ,CAAC,IAAI,KAAK,QAAQ,CAAC,IAAI,EAAE,CAAC;QACpC,OAAO,EAAC,IAAI,EAAE,SAAS,EAAC,CAAC;IAC3B,CAAC;IACD,IACE,QAAQ,CAAC,IAAI,KAAK,MAAM;QACxB,QAAQ,CAAC,IAAI,KAAK,MAAM;QACxB,QAAQ,CAAC,IAAI,KAAK,QAAQ,CAAC,IAAI,EAC/B,CAAC;QACD,OAAO,EAAC,IAAI,EAAE,SAAS,EAAC,CAAC;IAC3B,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAgB,iBAAiB,CAC/B,KAAyB,EACzB,YAA0B;IAE1B,MAAM,GAAG,GAAG,SAAS,CAAC,CAAC,EAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,CAAC,IAAI,EAAC,CAAC,CAAC,CAAC;IACjE,IAAI,YAAY,CAAC,IAAI,KAAK,OAAO,IAAI,YAAY,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QACpE,KAAK,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC/B,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,EAAC,IAAI,EAAE,YAAY,CAAC,IAAI,EAAC,CAAC,CAAC;IACnD,CAAC;AACH,CAAC;AAED,SAAgB,qBAAqB,CACnC,KAAyB,EACzB,QAAmB,EACnB,SAAiB;IAEjB,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO;IACT,CAAC;IACD,MAAM,WAAW,GAAG,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACpD,MAAM,aAAa,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;IACpD,MAAM,gBAAgB,GACpB,QAAQ,CAAC,MAAM,KAAK,CAAC;QACnB,CAAC,CAAC,mBAAmB,CAAC,SAAS,CAAC;QAChC,CAAC,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;IACtC,IACE,KAAK,CAAC,eAAe,CAAC,GAAG,CAAC,WAAW,CAAC;QACtC,aAAa;QACb,CAAC,aAAa,CAAC,IAAI,KAAK,OAAO,IAAI,aAAa,CAAC,IAAI,KAAK,QAAQ,CAAC;QACnE,aAAa,CAAC,IAAI,KAAK,gBAAgB,CAAC,IAAI,EAC5C,CAAC;QACD,kEAAkE;QAClE,4EAA4E;QAC5E,OAAO;IACT,CAAC;IACD,IAAI,SAAgC,CAAC;IACrC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,MAAM,cAAc,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QAChD,MAAM,GAAG,GAAG,SAAS,CAAC,cAAc,CAAC,CAAC;QACtC,MAAM,YAAY,GAChB,CAAC,KAAK,QAAQ,CAAC,MAAM,GAAG,CAAC;YACvB,CAAC,CAAC,mBAAmB,CAAC,SAAS,CAAC;YAChC,CAAC,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC1C,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,YAAY,CAAC,CAAC,CAAC;QACvE,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;YAC5B,WAAW,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAClD,CAAC;QACD,SAAS,GAAG,GAAG,CAAC;IAClB,CAAC;AACH,CAAC;AAED,SAAgB,kBAAkB,CAChC,YAA0B,EAC1B,KAAyB,EACzB,OAAgB;IAEhB,yDAAyD;IACzD,2DAA2D;IAC3D,0CAA0C;IAC1C,2DAA2D;IAC3D,yDAAyD;IACzD,EAAE;IACF,qEAAqE;IACrE,0EAA0E;IAC1E,kDAAkD;IAClD,MAAM,GAAG,GAAG,SAAS,CAAC,CAAC,EAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,CAAC,IAAI,EAAC,CAAC,CAAC,CAAC;IACjE,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACpC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,sEAAsE;QACtE,uEAAuE;QACvE,OAAO,IAAA,qBAAU,EACf,YAAY,CAAC,IAAI,KAAK,OAAO;YAC3B,CAAC,CAAC,IAAA,yBAAc,EAAC,iBAAiB,EAAE,CAAC;YACrC,CAAC,CAAC,iBAAiB,EAAE,EACvB,YAAY,CAAC,IAAI,CAClB,CAAC;IACJ,CAAC;IACD,OAAO,IAAA,qBAAU,EAAC,eAAe,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,EAAE,YAAY,CAAC,IAAI,CAAC,CAAC;AAC7E,CAAC;AAED,SAAS,eAAe,CACtB,GAAc,EACd,KAAyB,EACzB,OAAgB;;IAEhB,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACpC,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QACpD,OAAO,iBAAiB,EAAE,CAAC;IAC7B,CAAC;IACD,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QAC1B,OAAO,OAAO,CAAC,mBAAmB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACjD,CAAC;IACD,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC5B,MAAM,aAAa,GAAG,MAAA,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,0CAAE,KAAK,CAAC;QACrD,IAAI,CAAC,aAAa,IAAI,aAAa,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YAC/C,OAAO,iBAAiB,EAAE,CAAC;QAC7B,CAAC;QACD,OAAO;YACL,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,CAAC,GAAG,aAAa,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,EAAE,QAAQ,CAAC,EAAE,EAAE,CACjE,IAAA,qBAAU,EAAC,eAAe,CAAC,QAAQ,EAAE,KAAK,EAAE,OAAO,CAAC,EAAE,SAAS,CAAC,CACjE;SACF,CAAC;IACJ,CAAC;IACD,MAAM,OAAO,GAAG,MAAA,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,0CAAE,IAAI,CAAC;IAC9C,MAAM,WAAW,GAAG,OAAO;QACzB,CAAC,CAAC,eAAe,CAAC,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC;QAC1C,CAAC,CAAC,iBAAiB,EAAE,CAAC;IACxB,OAAO,IAAA,yBAAc,EAAC,WAAW,CAAC,CAAC;AACrC,CAAC;AAED,SAAS,kBAAkB,CAAC,OAAgB;IAC1C,OAAO,OAAO,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,EAAC,IAAI,EAAE,OAAO,EAAC,CAAC,CAAC,CAAC,EAAC,IAAI,EAAE,QAAQ,EAAC,CAAC;AACvE,CAAC;AAED,SAAS,mBAAmB,CAAC,SAAiB;IAC5C,4EAA4E;IAC5E,uEAAuE;IACvE,IACE,SAAS,KAAK,OAAO;QACrB,SAAS,KAAK,QAAQ;QACtB,SAAS,KAAK,SAAS,EACvB,CAAC;QACD,OAAO,EAAC,IAAI,EAAE,SAAS,EAAC,CAAC;IAC3B,CAAC;IACD,OAAO,EAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAC,CAAC;AACzC,CAAC;AAED,SAAS,WAAW,CAClB,KAAyB,EACzB,SAAoB,EACpB,OAAgB,EAChB,QAAmB;IAEnB,IAAI,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IAC7C,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3B,QAAQ,GAAG,EAAC,KAAK,EAAE,IAAI,GAAG,EAAE,EAAC,CAAC;QAC9B,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IAC1C,CAAC;IACD,IAAI,OAAO,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;QAC7B,QAAQ,CAAC,IAAI,GAAG,QAAQ,CAAC;IAC3B,CAAC;SAAM,CAAC;QACN,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAC7C,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB;IACxB,OAAO,EAAC,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,SAAS,EAAC,CAAC;AAClD,CAAC;AAED,SAAS,SAAS,CAAC,QAAmB;IACpC,MAAM,CAAC,KAAK,EAAE,GAAG,IAAI,CAAC,GAAG,QAAQ,CAAC;IAClC,IAAI,CAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,IAAI,MAAK,MAAM,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAC;IAC3E,CAAC;IACD,OAAO,IAAA,oBAAS,EACd,KAAK,CAAC,IAAI,EACV,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CACvE,CAAC;AACJ,CAAC"}

package/dist/snowflake_variant_schema.spec.js

@@ -0,0 +1,150 @@
+"use strict";
+/*
+ * Copyright Contributors to the Malloy project
+ * SPDX-License-Identifier: MIT
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+const malloy_1 = require("@malloydata/malloy");
+const snowflake_variant_schema_1 = require("./snowflake_variant_schema");
+describe('snowflake variant schema helper', () => {
+    const dialect = new malloy_1.SnowflakeDialect();
+    function inferField(nestedColumn, rows) {
+        const state = (0, snowflake_variant_schema_1.createVariantSchemaState)();
+        (0, snowflake_variant_schema_1.seedTopLevelShape)(state, nestedColumn);
+        for (const row of rows) {
+            (0, snowflake_variant_schema_1.accumulateVariantPath)(state, new snowflake_variant_schema_1.PathParser(row.path).segments(), row.type);
+        }
+        return (0, snowflake_variant_schema_1.buildTopLevelField)(nestedColumn, state, dialect);
+    }
+    test('reconstructs object shape from descendant-only evidence', () => {
+        expect(inferField({ kind: 'variant', name: 'BASE_TOUCHPOINT' }, [
+            { path: 'BASE_TOUCHPOINT.NETWORK', type: 'varchar' },
+            { path: 'BASE_TOUCHPOINT.PLATFORM', type: 'varchar' },
+        ])).toEqual({
+            type: 'record',
+            name: 'BASE_TOUCHPOINT',
+            join: 'one',
+            fields: [
+                { name: 'NETWORK', type: 'string' },
+                { name: 'PLATFORM', type: 'string' },
+            ],
+        });
+    });
+    test('degrades object-array conflict at a shared prefix', () => {
+        expect(inferField({ kind: 'variant', name: 'X' }, [
+            { path: 'X.Y', type: 'varchar' },
+            { path: 'X[*].Z', type: 'decimal' },
+        ])).toEqual({
+            type: 'sql native',
+            rawType: 'variant',
+            name: 'X',
+        });
+    });
+    test('builds array of records from stable descendants', () => {
+        expect(inferField({ kind: 'variant', name: 'ITEMS' }, [
+            { path: 'ITEMS[*].FOO', type: 'varchar' },
+            { path: 'ITEMS[*].BAR', type: 'boolean' },
+        ])).toEqual({
+            type: 'array',
+            name: 'ITEMS',
+            join: 'many',
+            elementTypeDef: { type: 'record_element' },
+            fields: [
+                { name: 'FOO', type: 'string' },
+                { name: 'BAR', type: 'boolean' },
+            ],
+        });
+    });
+    test('top-level array with no descendants becomes array of variant', () => {
+        expect(inferField({ kind: 'array', name: 'DIMENSION_SET_IDS' }, [])).toEqual({
+            type: 'array',
+            name: 'DIMENSION_SET_IDS',
+            join: 'many',
+            elementTypeDef: { type: 'sql native', rawType: 'variant' },
+            fields: [
+                { name: 'value', type: 'sql native', rawType: 'variant' },
+                {
+                    name: 'each',
+                    type: 'sql native',
+                    rawType: 'variant',
+                    e: { node: 'field', path: ['value'] },
+                },
+            ],
+        });
+    });
+    test('top-level DESCRIBE seed stays authoritative over conflicting sample', () => {
+        expect(inferField({ kind: 'array', name: 'DIMENSION_SET_IDS' }, [
+            { path: 'DIMENSION_SET_IDS.foo', type: 'varchar' },
+        ])).toEqual({
+            type: 'array',
+            name: 'DIMENSION_SET_IDS',
+            join: 'many',
+            elementTypeDef: { type: 'sql native', rawType: 'variant' },
+            fields: [
+                { name: 'value', type: 'sql native', rawType: 'variant' },
+                {
+                    name: 'each',
+                    type: 'sql native',
+                    rawType: 'variant',
+                    e: { node: 'field', path: ['value'] },
+                },
+            ],
+        });
+    });
+    test('top-level object with no descendants becomes opaque variant', () => {
+        expect(inferField({ kind: 'object', name: 'PAYLOAD' }, [])).toEqual({
+            type: 'sql native',
+            rawType: 'variant',
+            name: 'PAYLOAD',
+        });
+    });
+    test('quoted path names with punctuation are preserved', () => {
+        expect(inferField({ kind: 'variant', name: 'DATA' }, [
+            { path: "DATA['a.b'][*]['c[d]']", type: 'varchar' },
+        ])).toEqual({
+            type: 'record',
+            name: 'DATA',
+            join: 'one',
+            fields: [
+                {
+                    type: 'array',
+                    name: 'a.b',
+                    join: 'many',
+                    elementTypeDef: { type: 'record_element' },
+                    fields: [{ name: 'c[d]', type: 'string' }],
+                },
+            ],
+        });
+    });
+    test('leaf-type conflicts degrade to variant', () => {
+        const first = { kind: 'leaf', type: 'varchar' };
+        const second = { kind: 'leaf', type: 'decimal' };
+        expect((0, snowflake_variant_schema_1.mergeShape)(first, second)).toEqual({ kind: 'variant' });
+    });
+    test('scalar-vs-object at same path degrades that field only', () => {
+        // The sample emits both the scalar observation and the object
+        // observation (same path, two rows from the distinct (path, type)
+        // query). mergeShape collapses DATA.foo to variant; the parent
+        // DATA stays a record and siblings keep their types.
+        expect(inferField({ kind: 'variant', name: 'DATA' }, [
+            { path: 'DATA.foo', type: 'object' },
+            { path: 'DATA.foo', type: 'varchar' },
+            { path: 'DATA.foo.bar', type: 'decimal' },
+            { path: 'DATA.sib', type: 'varchar' },
+        ])).toEqual({
+            type: 'record',
+            name: 'DATA',
+            join: 'one',
+            fields: [
+                { type: 'sql native', rawType: 'variant', name: 'foo' },
+                { type: 'string', name: 'sib' },
+            ],
+        });
+    });
+    test('variant shape is monotonic', () => {
+        expect((0, snowflake_variant_schema_1.mergeShape)({ kind: 'variant' }, { kind: 'object' })).toEqual({
+            kind: 'variant',
+        });
+    });
+});
+//# sourceMappingURL=snowflake_variant_schema.spec.js.map

package/dist/snowflake_variant_schema.spec.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"snowflake_variant_schema.spec.js","sourceRoot":"","sources":["../src/snowflake_variant_schema.spec.ts"],"names":[],"mappings":";AAAA;;;GAGG;;AAEH,+CAAoD;AACpD,yEASoC;AAEpC,QAAQ,CAAC,iCAAiC,EAAE,GAAG,EAAE;IAC/C,MAAM,OAAO,GAAG,IAAI,yBAAgB,EAAE,CAAC;IAEvC,SAAS,UAAU,CACjB,YAA0B,EAC1B,IAAyC;QAEzC,MAAM,KAAK,GAAG,IAAA,mDAAwB,GAAE,CAAC;QACzC,IAAA,4CAAiB,EAAC,KAAK,EAAE,YAAY,CAAC,CAAC;QACvC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,IAAA,gDAAqB,EACnB,KAAK,EACL,IAAI,qCAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EACnC,GAAG,CAAC,IAAI,CACT,CAAC;QACJ,CAAC;QACD,OAAO,IAAA,6CAAkB,EAAC,YAAY,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;IAC1D,CAAC;IAED,IAAI,CAAC,yDAAyD,EAAE,GAAG,EAAE;QACnE,MAAM,CACJ,UAAU,CAAC,EAAC,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,iBAAiB,EAAC,EAAE;YACrD,EAAC,IAAI,EAAE,yBAAyB,EAAE,IAAI,EAAE,SAAS,EAAC;YAClD,EAAC,IAAI,EAAE,0BAA0B,EAAE,IAAI,EAAE,SAAS,EAAC;SACpD,CAAC,CACH,CAAC,OAAO,CAAC;YACR,IAAI,EAAE,QAAQ;YACd,IAAI,EAAE,iBAAiB;YACvB,IAAI,EAAE,KAAK;YACX,MAAM,EAAE;gBACN,EAAC,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,QAAQ,EAAC;gBACjC,EAAC,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAC;aACnC;SACF,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,mDAAmD,EAAE,GAAG,EAAE;QAC7D,MAAM,CACJ,UAAU,CAAC,EAAC,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,GAAG,EAAC,EAAE;YACvC,EAAC,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAC;YAC9B,EAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAC;SAClC,CAAC,CACH,CAAC,OAAO,CAAC;YACR,IAAI,EAAE,YAAY;YAClB,OAAO,EAAE,SAAS;YAClB,IAAI,EAAE,GAAG;SACV,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,iDAAiD,EAAE,GAAG,EAAE;QAC3D,MAAM,CACJ,UAAU,CAAC,EAAC,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,OAAO,EAAC,EAAE;YAC3C,EAAC,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE,SAAS,EAAC;YACvC,EAAC,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE,SAAS,EAAC;SACxC,CAAC,CACH,CAAC,OAAO,CAAC;YACR,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,MAAM;YACZ,cAAc,EAAE,EAAC,IAAI,EAAE,gBAAgB,EAAC;YACxC,MAAM,EAAE;gBACN,EAAC,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAC;gBAC7B,EAAC,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAC;aAC/B;SACF,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,8DAA8D,EAAE,GAAG,EAAE;QACxE,MAAM,CAAC,UAAU,CAAC,EAAC,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,mBAAmB,EAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;YACzE,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,mBAAmB;YACzB,IAAI,EAAE,MAAM;YACZ,cAAc,EAAE,EAAC,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,SAAS,EAAC;YACxD,MAAM,EAAE;gBACN,EAAC,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,SAAS,EAAC;gBACvD;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,YAAY;oBAClB,OAAO,EAAE,SAAS;oBAClB,CAAC,EAAE,EAAC,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,OAAO,CAAC,EAAC;iBACpC;aACF;SACF,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,qEAAqE,EAAE,GAAG,EAAE;QAC/E,MAAM,CACJ,UAAU,CAAC,EAAC,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,mBAAmB,EAAC,EAAE;YACrD,EAAC,IAAI,EAAE,uBAAuB,EAAE,IAAI,EAAE,SAAS,EAAC;SACjD,CAAC,CACH,CAAC,OAAO,CAAC;YACR,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,mBAAmB;YACzB,IAAI,EAAE,MAAM;YACZ,cAAc,EAAE,EAAC,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,SAAS,EAAC;YACxD,MAAM,EAAE;gBACN,EAAC,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,SAAS,EAAC;gBACvD;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,YAAY;oBAClB,OAAO,EAAE,SAAS;oBAClB,CAAC,EAAE,EAAC,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,OAAO,CAAC,EAAC;iBACpC;aACF;SACF,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,6DAA6D,EAAE,GAAG,EAAE;QACvE,MAAM,CAAC,UAAU,CAAC,EAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;YAChE,IAAI,EAAE,YAAY;YAClB,OAAO,EAAE,SAAS;YAClB,IAAI,EAAE,SAAS;SAChB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,kDAAkD,EAAE,GAAG,EAAE;QAC5D,MAAM,CACJ,UAAU,CAAC,EAAC,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAC,EAAE;YAC1C,EAAC,IAAI,EAAE,wBAAwB,EAAE,IAAI,EAAE,SAAS,EAAC;SAClD,CAAC,CACH,CAAC,OAAO,CAAC;YACR,IAAI,EAAE,QAAQ;YACd,IAAI,EAAE,MAAM;YACZ,IAAI,EAAE,KAAK;YACX,MAAM,EAAE;gBACN;oBACE,IAAI,EAAE,OAAO;oBACb,IAAI,EAAE,KAAK;oBACX,IAAI,EAAE,MAAM;oBACZ,cAAc,EAAE,EAAC,IAAI,EAAE,gBAAgB,EAAC;oBACxC,MAAM,EAAE,CAAC,EAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAC,CAAC;iBACzC;aACF;SACF,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,wCAAwC,EAAE,GAAG,EAAE;QAClD,MAAM,KAAK,GAAU,EAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAC,CAAC;QACrD,MAAM,MAAM,GAAU,EAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAC,CAAC;QACtD,MAAM,CAAC,IAAA,qCAAU,EAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,EAAC,IAAI,EAAE,SAAS,EAAC,CAAC,CAAC;IAC/D,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,wDAAwD,EAAE,GAAG,EAAE;QAClE,8DAA8D;QAC9D,kEAAkE;QAClE,+DAA+D;QAC/D,qDAAqD;QACrD,MAAM,CACJ,UAAU,CAAC,EAAC,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAC,EAAE;YAC1C,EAAC,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAC;YAClC,EAAC,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,SAAS,EAAC;YACnC,EAAC,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE,SAAS,EAAC;YACvC,EAAC,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,SAAS,EAAC;SACpC,CAAC,CACH,CAAC,OAAO,CAAC;YACR,IAAI,EAAE,QAAQ;YACd,IAAI,EAAE,MAAM;YACZ,IAAI,EAAE,KAAK;YACX,MAAM,EAAE;gBACN,EAAC,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAC;gBACrD,EAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAC;aAC9B;SACF,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,4BAA4B,EAAE,GAAG,EAAE;QACtC,MAAM,CAAC,IAAA,qCAAU,EAAC,EAAC,IAAI,EAAE,SAAS,EAAC,EAAE,EAAC,IAAI,EAAE,QAAQ,EAAC,CAAC,CAAC,CAAC,OAAO,CAAC;YAC9D,IAAI,EAAE,SAAS;SAChB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@malloydata/db-snowflake",
-  "version": "0.0.375",
+  "version": "0.0.377",
   "license": "MIT",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -22,7 +22,7 @@
     "prepublishOnly": "npm run build"
   },
   "dependencies": {
-    "@malloydata/malloy": "0.0.375",
+    "@malloydata/malloy": "0.0.377",
     "generic-pool": "^3.9.0",
     "snowflake-sdk": "2.3.1",
     "toml": "^3.0.0"

package/src/index.ts

@@ -37,6 +37,8 @@ registerConnectionType('snowflake', {
       setupSQL,
       timeoutMs,
       schemaSampleTimeoutMs,
+      schemaSampleRowLimit,
+      schemaSampleFullScanMaxBytes,
       ...props
     } = config;
     // ConnectionConfig values are trusted to match ConnectionOptions fields
@@ -60,6 +62,18 @@ registerConnectionType('snowflake', {
           : typeof schemaSampleTimeoutMs === 'string'
             ? parseInt(schemaSampleTimeoutMs, 10)
             : undefined,
+      schemaSampleRowLimit:
+        typeof schemaSampleRowLimit === 'number'
+          ? schemaSampleRowLimit
+          : typeof schemaSampleRowLimit === 'string'
+            ? parseInt(schemaSampleRowLimit, 10)
+            : undefined,
+      schemaSampleFullScanMaxBytes:
+        typeof schemaSampleFullScanMaxBytes === 'number'
+          ? schemaSampleFullScanMaxBytes
+          : typeof schemaSampleFullScanMaxBytes === 'string'
+            ? parseInt(schemaSampleFullScanMaxBytes, 10)
+            : undefined,
     });
   },
   properties: [
@@ -101,14 +115,33 @@ registerConnectionType('snowflake', {
       displayName: 'Timeout (ms)',
       type: 'number',
       optional: true,
+      default: 600000,
     },
     {
       name: 'schemaSampleTimeoutMs',
       displayName: 'Schema Sample Timeout (ms)',
       type: 'number',
       optional: true,
+      default: 15000,
+      description:
+        'Timeout for the query that samples variant columns to detect their schema.',
+    },
+    {
+      name: 'schemaSampleRowLimit',
+      displayName: 'Schema Sample Row Limit',
+      type: 'number',
+      optional: true,
+      default: 1000,
+      description:
+        'Row limit for the variant schema sample. Ignored for tables small enough to full-scan.',
+    },
+    {
+      name: 'schemaSampleFullScanMaxBytes',
+      displayName: 'Schema Full-Scan Max Bytes',
+      type: 'number',
+      optional: true,
       description:
-        'Timeout for the query that samples variant columns to detect their schema (default 15000)',
+        'Tables with BYTES at or below this value are full-scanned during variant schema inference instead of sampled. When unset, the connection uses an internal threshold; picking a value here is a policy choice tied to the size-probe behavior.',
     },
     {
       name: 'setupSQL',