@malloydata/malloy 0.0.105-dev231122194042 → 0.0.105-dev231127164722

package/dist/malloy.d.ts

@@ -1234,6 +1234,7 @@ declare class DataNull extends Data<null> {
 export declare class DataArray extends Data<QueryData> implements Iterable<DataRecord> {
     private queryData;
     protected _field: Explore;
+    private rowCache;
     constructor(queryData: QueryData, field: Explore, parent: DataArrayOrRecord | undefined, parentRecord: DataRecord | undefined);
     /**
      * @return The `Explore` that describes the structure of this data.
@@ -1256,6 +1257,7 @@ export declare class DataRecord extends Data<{
     private queryDataRow;
     protected _field: Explore;
     readonly index: number | undefined;
+    private cellCache;
     constructor(queryDataRow: QueryDataRow, index: number | undefined, field: Explore, parent: DataArrayOrRecord | undefined, parentRecord: DataRecord | undefined);
     toObject(): QueryDataRow;
     path(...path: (number | string)[]): DataColumn;

package/dist/malloy.js

@@ -2611,6 +2611,7 @@ class DataNull extends Data {
 class DataArray extends Data {
     constructor(queryData, field, parent, parentRecord) {
         super(field, parent, parentRecord);
+        this.rowCache = new Map();
         this.queryData = queryData;
         this._field = field;
     }
@@ -2630,6 +2631,12 @@ class DataArray extends Data {
         return getPath(this, path);
     }
     row(index) {
+        let record = this.rowCache.get(index);
+        if (!record) {
+            record = new DataRecord(this.queryData[index], index, this.field, this, this.parentRecord);
+            this.rowCache.set(index, record);
+        }
+        return record;
         return new DataRecord(this.queryData[index], index, this.field, this, this.parentRecord);
     }
     get rowCount() {
@@ -2674,6 +2681,7 @@ function getPath(data, path) {
 class DataRecord extends Data {
     constructor(queryDataRow, index, field, parent, parentRecord) {
         super(field, parent, parentRecord);
+        this.cellCache = new Map();
         this.queryDataRow = queryDataRow;
         this._field = field;
         this.index = index;
@@ -2687,41 +2695,48 @@ class DataRecord extends Data {
     cell(fieldOrName) {
         const fieldName = typeof fieldOrName === 'string' ? fieldOrName : fieldOrName.name;
         const field = this._field.getFieldByName(fieldName);
-        const value = this.queryDataRow[fieldName];
-        if (value === null) {
-            return new DataNull(field, this, this);
-        }
-        if (field.isAtomicField()) {
-            if (field.isBoolean()) {
-                return new DataBoolean(value, field, this, this);
-            }
-            else if (field.isDate()) {
-                return new DataDate(value, field, this, this);
-            }
-            else if (field.isJSON()) {
-                return new DataJSON(value, field, this, this);
-            }
-            else if (field.isTimestamp()) {
-                return new DataTimestamp(value, field, this, this);
+        let column = this.cellCache.get(fieldName);
+        if (!column) {
+            const value = this.queryDataRow[fieldName];
+            if (value === null) {
+                column = new DataNull(field, this, this);
             }
-            else if (field.isNumber()) {
-                return new DataNumber(value, field, this, this);
-            }
-            else if (field.isString()) {
-                return new DataString(value, field, this, this);
-            }
-            else if (field.isUnsupported()) {
-                return new DataUnsupported(value, field, this, this);
-            }
-        }
-        else if (field.isExploreField()) {
-            if (Array.isArray(value)) {
-                return new DataArray(value, field, this, this);
+            else if (field.isAtomicField()) {
+                if (field.isBoolean()) {
+                    column = new DataBoolean(value, field, this, this);
+                }
+                else if (field.isDate()) {
+                    column = new DataDate(value, field, this, this);
+                }
+                else if (field.isJSON()) {
+                    column = new DataJSON(value, field, this, this);
+                }
+                else if (field.isTimestamp()) {
+                    column = new DataTimestamp(value, field, this, this);
+                }
+                else if (field.isNumber()) {
+                    column = new DataNumber(value, field, this, this);
+                }
+                else if (field.isString()) {
+                    column = new DataString(value, field, this, this);
+                }
+                else if (field.isUnsupported()) {
+                    column = new DataUnsupported(value, field, this, this);
+                }
             }
-            else {
-                return new DataRecord(value, undefined, field, this, this);
+            else if (field.isExploreField()) {
+                if (Array.isArray(value)) {
+                    column = new DataArray(value, field, this, this);
+                }
+                else {
+                    column = new DataRecord(value, undefined, field, this, this);
+                }
             }
+            if (column)
+                this.cellCache.set(fieldName, column);
         }
+        if (column)
+            return column;
         throw new Error(`Internal Error: could not construct data column for field '${fieldName}'.`);
     }
     get value() {

package/dist/model/malloy_query.d.ts

@@ -230,8 +230,9 @@ declare class QueryQuery extends QueryField {
     rootResult: FieldInstanceResultRoot;
     resultStage: string | undefined;
     stageWriter: StageWriter | undefined;
-    constructor(fieldDef: TurtleDef, parent: QueryStruct, stageWriter: StageWriter | undefined);
-    static makeQuery(fieldDef: TurtleDef, parentStruct: QueryStruct, stageWriter?: StageWriter | undefined): QueryQuery;
+    isJoinedSubquery: boolean;
+    constructor(fieldDef: TurtleDef, parent: QueryStruct, stageWriter: StageWriter | undefined, isJoinedSubquery: boolean);
+    static makeQuery(fieldDef: TurtleDef, parentStruct: QueryStruct, stageWriter: StageWriter | undefined, isJoinedSubquery: boolean): QueryQuery;
     inNestedPipeline(): boolean;
     getFieldList(): QueryFieldDef[];
     expandField(f: QueryFieldDef): {
@@ -355,7 +356,7 @@ export declare class QueryModel {
     };
     getStructByName(name: string): QueryStruct;
     getStructFromRef(structRef: StructRef): QueryStruct;
-    loadQuery(query: Query, stageWriter: StageWriter | undefined, emitFinalStage?: boolean): QueryResults;
+    loadQuery(query: Query, stageWriter: StageWriter | undefined, emitFinalStage?: boolean, isJoinedSubquery?: boolean): QueryResults;
     compileQuery(query: Query, finalize?: boolean): CompiledQuery;
     exploreSearchSQLMap: Map<any, any>;
     searchIndex(connection: Connection, explore: string, searchValue: string, limit?: number, searchField?: string | undefined): Promise<SearchIndexResult[] | undefined>;

package/dist/model/malloy_query.js

@@ -1262,15 +1262,15 @@ class Segment {
             name: 'ignoreme',
             pipeline: [segment],
         };
-        const queryQueryQuery = QueryQuery.makeQuery(turtleDef, qs, new StageWriter(true, undefined) // stage write indicates we want to get a result.
-        );
+        const queryQueryQuery = QueryQuery.makeQuery(turtleDef, qs, new StageWriter(true, undefined), // stage write indicates we want to get a result.
+        false);
         return queryQueryQuery.getResultStructDef();
     }
 }
 exports.Segment = Segment;
 /** Query builder object. */
 class QueryQuery extends QueryField {
-    constructor(fieldDef, parent, stageWriter) {
+    constructor(fieldDef, parent, stageWriter, isJoinedSubquery) {
         super(fieldDef, parent);
         this.prepared = false;
         this.maxDepth = 0;
@@ -1280,8 +1280,9 @@ class QueryQuery extends QueryField {
         this.stageWriter = stageWriter;
         // do some magic here to get the first segment.
         this.firstSegment = fieldDef.pipeline[0];
+        this.isJoinedSubquery = isJoinedSubquery;
     }
-    static makeQuery(fieldDef, parentStruct, stageWriter = undefined) {
+    static makeQuery(fieldDef, parentStruct, stageWriter = undefined, isJoinedSubquery) {
         let flatTurtleDef = parentStruct.flattenTurtleDef(fieldDef);
         let parent = parentStruct;
         const firstStage = flatTurtleDef.pipeline[0];
@@ -1315,11 +1316,11 @@ class QueryQuery extends QueryField {
         }
         switch (firstStage.type) {
             case 'reduce':
-                return new QueryQueryReduce(flatTurtleDef, parent, stageWriter);
+                return new QueryQueryReduce(flatTurtleDef, parent, stageWriter, isJoinedSubquery);
             case 'project':
-                return new QueryQueryProject(flatTurtleDef, parent, stageWriter);
+                return new QueryQueryProject(flatTurtleDef, parent, stageWriter, isJoinedSubquery);
             case 'index':
-                return new QueryQueryIndex(flatTurtleDef, parent, stageWriter);
+                return new QueryQueryIndex(flatTurtleDef, parent, stageWriter, isJoinedSubquery);
             case 'partial':
                 throw new Error('Attempt to make query out of partial stage');
         }
@@ -1370,7 +1371,7 @@ class QueryQuery extends QueryField {
                 const newFieldDef = (0, cloneDeep_1.default)(field.fieldDef);
                 newFieldDef.as = f.name;
                 newFieldDef.filterList = f.filterList;
-                field = QueryQuery.makeQuery(newFieldDef, this.parent);
+                field = QueryQuery.makeQuery(newFieldDef, this.parent, undefined, this.isJoinedSubquery);
             }
             else if (!(field instanceof QueryFieldTimestamp ||
                 field instanceof QueryFieldDate)) {
@@ -1984,6 +1985,22 @@ class QueryQuery extends QueryField {
         if (this.firstSegment.type === 'project' && !queryDef.orderBy) {
             return ''; // No default ordering for project.
         }
+        // Intermediate results (in a pipeline or join) that have no limit, don't need an orderby
+        //  Some database don't have this optimization.
+        if (this.fieldDef.pipeline.length > 1 && queryDef.limit === undefined) {
+            return '';
+        }
+        // ignore orderby if all aggregates.
+        if (resultStruct.getRepeatedResultType() === 'inline_all_numbers') {
+            return '';
+        }
+        // if we are in the last stage of a query and the query is a subquery
+        //  and has no limit, ORDER BY is superfluous
+        if (this.isJoinedSubquery &&
+            this.fieldDef.pipeline.length === 1 &&
+            queryDef.limit === undefined) {
+            return '';
+        }
         const orderBy = queryDef.orderBy || resultStruct.calculateDefaultOrderBy();
         const o = [];
         for (const f of orderBy) {
@@ -2439,7 +2456,7 @@ class QueryQuery extends QueryField {
             const qs = new QueryStruct(structDef, {
                 model: this.parent.getModel(),
             });
-            const q = QueryQuery.makeQuery(newTurtle, qs, stageWriter);
+            const q = QueryQuery.makeQuery(newTurtle, qs, stageWriter, this.isJoinedSubquery);
             pipeOut = q.generateSQLFromPipeline(stageWriter);
             // console.log(stageWriter.generateSQLStages());
             structDef = pipeOut.outputStruct;
@@ -2497,7 +2514,7 @@ class QueryQuery extends QueryField {
                 const s = new QueryStruct(structDef, {
                     model: this.parent.getModel(),
                 });
-                const q = QueryQuery.makeQuery({ type: 'turtle', name: 'ignoreme', pipeline: [transform] }, s, stageWriter);
+                const q = QueryQuery.makeQuery({ type: 'turtle', name: 'ignoreme', pipeline: [transform] }, s, stageWriter, this.isJoinedSubquery);
                 q.prepare(stageWriter);
                 lastStageName = q.generateSQL(stageWriter);
                 outputStruct = q.getResultStructDef();
@@ -2518,8 +2535,8 @@ class QueryQueryProject extends QueryQuery {
 //  wildcards have been expanded
 //  nested repeated fields are safe to use.
 class QueryQueryIndexStage extends QueryQuery {
-    constructor(fieldDef, parent, stageWriter) {
-        super(fieldDef, parent, stageWriter);
+    constructor(fieldDef, parent, stageWriter, isJoinedSubquery) {
+        super(fieldDef, parent, stageWriter, isJoinedSubquery);
         this.fieldDef = fieldDef;
     }
     // get a field ref and expand it.
@@ -2632,8 +2649,8 @@ FROM ${resultStage}\n`);
     }
 }
 class QueryQueryIndex extends QueryQuery {
-    constructor(fieldDef, parent, stageWriter) {
-        super(fieldDef, parent, stageWriter);
+    constructor(fieldDef, parent, stageWriter, isJoinedSubquery) {
+        super(fieldDef, parent, stageWriter, isJoinedSubquery);
         this.rootFields = [];
         this.fanPrefixMap = {};
         this.fieldDef = fieldDef;
@@ -2720,7 +2737,7 @@ class QueryQueryIndex extends QueryQuery {
                         fields: fields,
                     },
                 ],
-            }, this.parent, stageWriter);
+            }, this.parent, stageWriter, this.isJoinedSubquery);
             q.prepare(stageWriter);
             const lastStageName = q.generateSQL(stageWriter);
             outputStageNames.push(lastStageName);
@@ -2799,7 +2816,7 @@ class QueryStruct extends QueryNode {
                 // case "reduce" || "project" || "index": {
                 case 'turtle': {
                     // not sure why we need to cast here...
-                    this.addFieldToNameMap(as, QueryQuery.makeQuery(field, this));
+                    this.addFieldToNameMap(as, QueryQuery.makeQuery(field, this, undefined, false));
                     break;
                 }
                 default: {
@@ -3077,12 +3094,13 @@ class QueryStruct extends QueryNode {
                 //  should be cached.
                 if (name.includes('cache')) {
                     const dtStageWriter = new StageWriter(true, stageWriter);
-                    this.model.loadQuery(this.fieldDef.structSource.query, dtStageWriter);
+                    this.model.loadQuery(this.fieldDef.structSource.query, dtStageWriter, false, false);
                     return dtStageWriter.addPDT(name, this.dialect);
                 }
                 else {
                     // returns the stage name.
-                    return this.model.loadQuery(this.fieldDef.structSource.query, stageWriter).lastStageName;
+                    return this.model.loadQuery(this.fieldDef.structSource.query, stageWriter, false, true // this is an intermediate stage.
+                    ).lastStageName;
                 }
             }
             default:
@@ -3269,7 +3287,7 @@ class QueryModel {
         }
         return new QueryStruct(structDef, { model: this });
     }
-    loadQuery(query, stageWriter, emitFinalStage = false) {
+    loadQuery(query, stageWriter, emitFinalStage = false, isJoinedSubquery = false) {
         const malloy = '';
         if (!stageWriter) {
             stageWriter = new StageWriter(true, undefined);
@@ -3281,7 +3299,7 @@ class QueryModel {
             pipeline: query.pipeline,
             filterList: query.filterList,
         };
-        const q = QueryQuery.makeQuery(turtleDef, this.getStructFromRef(query.structRef), stageWriter);
+        const q = QueryQuery.makeQuery(turtleDef, this.getStructFromRef(query.structRef), stageWriter, isJoinedSubquery);
         const ret = q.generateSQLFromPipeline(stageWriter);
         if (emitFinalStage && q.parent.dialect.hasFinalStage) {
             // const fieldNames: string[] = [];
@@ -3302,7 +3320,7 @@ class QueryModel {
     compileQuery(query, finalize = true) {
         let newModel;
         const m = newModel || this;
-        const ret = m.loadQuery(query, undefined, finalize);
+        const ret = m.loadQuery(query, undefined, finalize, false);
         const sourceExplore = typeof query.structRef === 'string'
             ? query.structRef
             : // LTNOTE: the parser needs to capture the query before the |.  This will work

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@malloydata/malloy",
-  "version": "0.0.105-dev231122194042",
+  "version": "0.0.105-dev231127164722",
   "license": "MIT",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",