@malloydata/malloy 0.0.15 → 0.0.16-dev221214152428

package/dist/model/malloy_query.js

@@ -1658,6 +1658,9 @@ class QueryQuery extends QueryField {
         let structSQL = qs.structSourceSQL(stageWriter);
         if ((0, malloy_types_1.isIndexSegment)(this.firstSegment)) {
             structSQL = this.parent.dialect.sqlSampleTable(structSQL, this.firstSegment.sample);
+            if (this.firstSegment.sample) {
+                structSQL = stageWriter.addStage(`SELECT * from ${structSQL} as x limit 100000 `);
+            }
         }
         const structRelationship = qs.fieldDef.structRelationship;
         if (structRelationship.type === "basetable") {
@@ -2211,23 +2214,13 @@ class QueryQueryReduce extends QueryQuery {
 }
 class QueryQueryProject extends QueryQuery {
 }
-class QueryQueryIndex extends QueryQuery {
+// generates a single stage query for the index.
+//  wildcards have been expanded
+//  nested repeated fields are safe to use.
+class QueryQueryIndexStage extends QueryQuery {
     constructor(fieldDef, parent, stageWriter) {
         super(fieldDef, parent, stageWriter);
-        this.fanPrefixes = [];
         this.fieldDef = fieldDef;
-        this.findFanPrefexes(parent);
-    }
-    // we want to generate a different query for each
-    //  nested structure so we don't do a crazy cross product.
-    findFanPrefexes(qs) {
-        for (const [_name, f] of qs.nameMap) {
-            if (f instanceof QueryStruct &&
-                (f.fieldDef.structRelationship.type === "many" ||
-                    f.fieldDef.structRelationship.type === "nested")) {
-                this.fanPrefixes.push(f.getFullOutputName());
-            }
-        }
     }
     // get a field ref and expand it.
     expandField(f) {
@@ -2238,13 +2231,7 @@ class QueryQueryIndex extends QueryQuery {
         let resultIndex = 1;
         const groupIndex = resultStruct.groupSet;
         this.maxGroupSet = groupIndex;
-        // if no fields were specified, look in the parent struct for strings.
-        let fieldNames = this.firstSegment.fields || [];
-        if (fieldNames.length === 0) {
-            fieldNames.push("**");
-        }
-        fieldNames = this.expandWildCards(fieldNames, (qf) => ["string", "number", "timestamp", "date"].indexOf(qf.fieldDef.type) !==
-            -1);
+        const fieldNames = this.firstSegment.fields || [];
         for (const f of fieldNames) {
             const { as, field } = this.expandField(f);
             resultStruct.addField(as, field, {
@@ -2343,6 +2330,110 @@ class QueryQueryIndex extends QueryQuery {
 FROM ${resultStage}\n`);
         return this.resultStage;
     }
+}
+class QueryQueryIndex extends QueryQuery {
+    constructor(fieldDef, parent, stageWriter) {
+        super(fieldDef, parent, stageWriter);
+        this.rootFields = [];
+        this.fanPrefixMap = {};
+        this.fieldDef = fieldDef;
+        this.findFanPrefexes(parent);
+    }
+    // we want to generate a different query for each
+    //  nested structure so we don't do a crazy cross product.
+    findFanPrefexes(qs) {
+        for (const [_name, f] of qs.nameMap) {
+            if (f instanceof QueryStruct &&
+                (f.fieldDef.structRelationship.type === "many" ||
+                    f.fieldDef.structRelationship.type === "nested") &&
+                f.fieldDef.fields.length > 1 && // leave arrays in parent.
+                this.parent.dialect.dontUnionIndex === false) {
+                this.fanPrefixMap[f.getFullOutputName()] = [];
+                this.findFanPrefexes(f);
+            }
+        }
+    }
+    expandIndexWildCards() {
+        // if no fields were specified, look in the parent struct for strings.
+        let fieldNames = this.firstSegment.fields || [];
+        if (fieldNames.length === 0) {
+            fieldNames.push("**");
+        }
+        fieldNames = this.expandWildCards(fieldNames, (qf) => ["string", "number", "timestamp", "date"].indexOf(qf.fieldDef.type) !==
+            -1);
+        return fieldNames;
+    }
+    // return the number of stages it is going to take to generate this index.
+    getStageFields() {
+        const s = [];
+        if (this.rootFields.length > 0) {
+            s.push(this.rootFields);
+        }
+        for (const fieldList of Object.values(this.fanPrefixMap)) {
+            if (fieldList.length > 0) {
+                s.push(fieldList);
+            }
+        }
+        return s;
+    }
+    // Map fields into stages based on their level of repeated nesting
+    //
+    mapFieldsIntoStages(fieldNames) {
+        // find all the fanned prefixes, longest ones first.
+        const fannedPrefixes = Object.keys(this.fanPrefixMap).sort((k1, k2) => {
+            if (k1.length < k2.length) {
+                return 1;
+            }
+            if (k1.length > k2.length) {
+                return -1;
+            }
+            return 0;
+        });
+        // Find the deepest fanned prefix
+        for (const fn of fieldNames) {
+            let found = false;
+            for (const prefix of fannedPrefixes) {
+                if (fn.startsWith(prefix)) {
+                    this.fanPrefixMap[prefix].push(fn);
+                    found = true;
+                }
+            }
+            if (!found) {
+                this.rootFields.push(fn);
+            }
+        }
+    }
+    expandFields(_resultStruct) {
+        const fieldNames = this.expandIndexWildCards();
+        this.mapFieldsIntoStages(fieldNames);
+    }
+    generateSQL(stageWriter) {
+        const stages = this.getStageFields();
+        const outputStageNames = [];
+        for (const fields of stages) {
+            const q = new QueryQueryIndexStage({
+                ...this.fieldDef,
+                pipeline: [
+                    {
+                        ...this.fieldDef.pipeline[0],
+                        fields: fields,
+                    },
+                ],
+            }, this.parent, stageWriter);
+            q.prepare(stageWriter);
+            const lastStageName = q.generateSQL(stageWriter);
+            outputStageNames.push(lastStageName);
+        }
+        if (outputStageNames.length === 1) {
+            this.resultStage = outputStageNames[0];
+        }
+        else {
+            this.resultStage = stageWriter.addStage(outputStageNames
+                .map((n) => `SELECT * FROM ${n}\n`)
+                .join(" UNION ALL \n"));
+        }
+        return this.resultStage;
+    }
     /**  All Indexes have the same output schema */
     getResultStructDef() {
         return {
@@ -2499,7 +2590,7 @@ class QueryStruct extends QueryNode {
         }
         // if this is an inline object, include the parents alias.
         if (this.fieldDef.structRelationship.type === "inline" && this.parent) {
-            return this.parent.getIdentifier() + "." + super.getIdentifier();
+            return this.parent.getSQLIdentifier() + "." + super.getIdentifier();
         }
         // we are somewhere in the join tree.  Make sure the alias is unique.
         return this.getAliasIdentifier();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@malloydata/malloy",
-  "version": "0.0.15",
+  "version": "0.0.16-dev221214152428",
   "license": "GPL-2.0",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",