@malloydata/malloy 0.0.307 → 0.0.309

package/dist/dialect/postgres/postgres.d.ts

@@ -1,6 +1,6 @@
-import type { Sampling, AtomicTypeDef, TimeDeltaExpr, TypecastExpr, MeasureTimeExpr, BasicAtomicTypeDef, RecordLiteralNode, ArrayLiteralNode } from '../../model/malloy_types';
+import type { Sampling, AtomicTypeDef, TimeDeltaExpr, TypecastExpr, MeasureTimeExpr, BasicAtomicTypeDef, RecordLiteralNode, ArrayLiteralNode, TimeExtractExpr } from '../../model/malloy_types';
 import type { DialectFunctionOverloadDef } from '../functions';
-import type { DialectFieldList, FieldReferenceType, QueryInfo } from '../dialect';
+import { type DialectFieldList, type FieldReferenceType, type QueryInfo } from '../dialect';
 import { PostgresBase } from '../pg_impl';
 export declare class PostgresDialect extends PostgresBase {
     name: string;
@@ -66,4 +66,5 @@ export declare class PostgresDialect extends PostgresBase {
     validateTypeName(sqlType: string): boolean;
     sqlLiteralRecord(lit: RecordLiteralNode): string;
     sqlLiteralArray(lit: ArrayLiteralNode): string;
+    sqlTimeExtractExpr(qi: QueryInfo, from: TimeExtractExpr): string;
 }

package/dist/dialect/postgres/postgres.js

@@ -26,6 +26,7 @@ exports.PostgresDialect = void 0;
 const utils_1 = require("../../model/utils");
 const malloy_types_1 = require("../../model/malloy_types");
 const functions_1 = require("../functions");
+const dialect_1 = require("../dialect");
 const pg_impl_1 = require("../pg_impl");
 const dialect_functions_1 = require("./dialect_functions");
 const function_overrides_1 = require("./function_overrides");
@@ -80,7 +81,7 @@ class PostgresDialect extends pg_impl_1.PostgresBase {
         this.udfPrefix = 'pg_temp.__udf';
         this.hasFinalStage = true;
         this.divisionIsInteger = true;
-        this.supportsSumDistinctFunction = false;
+        this.supportsSumDistinctFunction = true;
         this.unnestWithNumbers = false;
         this.defaultSampling = { rows: 50000 };
         this.supportUnnestArrayAgg = true;
@@ -242,14 +243,33 @@ class PostgresDialect extends pg_impl_1.PostgresBase {
         }
         throw new Error(`Unknown or unhandled postgres time unit: ${df.units}`);
     }
+    // This looks like a partial implementation of a method similar to how DuckDB
+    // does symmetric aggregates, which was abandoned. Leaving it in here for now
+    // in case the original author wants to pick it up again.
+    // sqlSumDistinct(key: string, value: string, funcName: string): string {
+    //   // return `sum_distinct(list({key:${key}, val: ${value}}))`;
+    //   return `(
+    //     SELECT ${funcName}((a::json->>'f2')::DOUBLE PRECISION) as value
+    //     FROM (
+    //       SELECT UNNEST(array_agg(distinct row_to_json(row(${key},${value}))::text)) a
+    //     ) a
+    //   )`;
+    // }
     sqlSumDistinct(key, value, funcName) {
-        // return `sum_distinct(list({key:${key}, val: ${value}}))`;
-        return `(
-      SELECT ${funcName}((a::json->>'f2')::DOUBLE PRECISION) as value
-      FROM (
-        SELECT UNNEST(array_agg(distinct row_to_json(row(${key},${value}))::text)) a
-      ) a
-    )`;
+        const hashKey = this.sqlSumDistinctHashedKey(key);
+        // PostgreSQL requires CAST to NUMERIC before ROUND, which is different
+        // than the generic implementation of sqlSumDistinct, but is OK in
+        // PostgreSQL because NUMERIC has arbitrary precision.
+        const roundedValue = `ROUND(CAST(COALESCE(${value}, 0) AS NUMERIC), 9)`;
+        const sumSQL = `SUM(DISTINCT ${roundedValue} + ${hashKey}) - SUM(DISTINCT ${hashKey})`;
+        const ret = `CAST(${sumSQL} AS DOUBLE PRECISION)`;
+        if (funcName === 'SUM') {
+            return ret;
+        }
+        else if (funcName === 'AVG') {
+            return `(${ret})/NULLIF(COUNT(DISTINCT CASE WHEN ${value} IS NOT NULL THEN ${key} END), 0)`;
+        }
+        throw new Error(`Unknown Symmetric Aggregate function ${funcName}`);
     }
     // TODO this does not preserve the types of the arguments, meaning we have to hack
     // around this in the definitions of functions that use this to cast back to the correct
@@ -339,6 +359,20 @@ class PostgresDialect extends pg_impl_1.PostgresBase {
         const array = lit.kids.values.map(val => val.sql);
         return 'JSONB_BUILD_ARRAY(' + array.join(',') + ')';
     }
+    sqlTimeExtractExpr(qi, from) {
+        const units = pg_impl_1.timeExtractMap[from.units] || from.units;
+        let extractFrom = from.e.sql;
+        if (malloy_types_1.TD.isTimestamp(from.e.typeDef)) {
+            const tz = (0, dialect_1.qtz)(qi);
+            if (tz) {
+                extractFrom = `(${extractFrom}::TIMESTAMPTZ AT TIME ZONE '${tz}')`;
+            }
+        }
+        // PostgreSQL before 14 returns a double precision for EXTRACT, cast to integer
+        // since it is common to pass an extraction to mod ( like in fitler expressions )
+        const extracted = `EXTRACT(${units} FROM ${extractFrom})::integer`;
+        return from.units === 'day_of_week' ? `(${extracted}+1)` : `(${extracted})`;
+    }
 }
 exports.PostgresDialect = PostgresDialect;
 //# sourceMappingURL=postgres.js.map

package/dist/model/expression_compiler.js

@@ -33,16 +33,15 @@ function sqlSumDistinct(dialect, sqlExp, sqlDistintKey) {
     const precision = 9;
     const uniqueInt = dialect.sqlSumDistinctHashedKey(sqlDistintKey);
     const multiplier = 10 ** (precision - NUMERIC_DECIMAL_PRECISION);
-    const sumSQL = `
-  (
-    SUM(DISTINCT
-      (CAST(ROUND(COALESCE(${sqlExp},0)*(${multiplier}*1.0), ${NUMERIC_DECIMAL_PRECISION}) AS ${dialect.defaultDecimalType}) +
-      ${uniqueInt}
-    ))
-    -
-     SUM(DISTINCT ${uniqueInt})
+    // Ensure value is numeric and handle nulls
+    const safeValue = `CAST(COALESCE(${sqlExp}, 0) AS ${dialect.defaultDecimalType})`;
+    // Scale and round to eliminate floating point differences
+    const roundedValue = `ROUND(${safeValue}*${multiplier}, ${NUMERIC_DECIMAL_PRECISION})`;
+    const sumSQL = `(
+    SUM(DISTINCT ${roundedValue} + ${uniqueInt})
+    - SUM(DISTINCT ${uniqueInt})
   )`;
-    let ret = `(${sumSQL}/(${multiplier}*1.0))`;
+    let ret = `(${sumSQL}/${multiplier})`;
     ret = `CAST(${ret} AS ${dialect.defaultNumberType})`;
     return ret;
 }

package/dist/model/field_instance.js

@@ -47,18 +47,6 @@ class FieldInstanceField {
         if ((0, malloy_types_1.hasExpression)(this.f.fieldDef)) {
             return FieldInstanceField.exprCompiler(this.parent, this.f.parent, this.f.fieldDef.e);
         }
-        // Walk the tree and compute record aliases if needed
-        for (let ancestor = this.f.parent; ancestor !== undefined; ancestor = ancestor.parent) {
-            if (ancestor.structDef.type === 'record' &&
-                (0, malloy_types_1.hasExpression)(ancestor.structDef) &&
-                ancestor.recordAlias === undefined) {
-                if (!ancestor.parent) {
-                    throw new Error('Inconceivable record ancestor with expression but no parent');
-                }
-                const aliasValue = FieldInstanceField.exprCompiler(this.parent, ancestor.parent, ancestor.structDef.e);
-                ancestor.informOfAliasValue(aliasValue);
-            }
-        }
         return sqlFullChildReference(this.f.parent, this.f.fieldDef.name, this.f.parent.structDef.type === 'record'
             ? {
                 result: this.parent,

package/dist/model/filter_compilers.d.ts

@@ -33,4 +33,5 @@ export declare class TemporalFilterCompiler {
     mod7(n: string): string;
     private moment;
     private isIn;
+    private weekdayMoment;
 }

package/dist/model/filter_compilers.js

@@ -521,51 +521,20 @@ class TemporalFilterCompiler {
                 return this.lastUnit(m.units);
             case 'next':
                 return this.nextUnit(m.units);
+            case 'sunday':
+                return this.weekdayMoment(1, m.which);
             case 'monday':
+                return this.weekdayMoment(2, m.which);
             case 'tuesday':
+                return this.weekdayMoment(3, m.which);
             case 'wednesday':
+                return this.weekdayMoment(4, m.which);
             case 'thursday':
+                return this.weekdayMoment(5, m.which);
             case 'friday':
+                return this.weekdayMoment(6, m.which);
             case 'saturday':
-            case 'sunday': {
-                const destDay = [
-                    'sunday',
-                    'monday',
-                    'tuesday',
-                    'wednesday',
-                    'thursday',
-                    'friday',
-                    'saturday',
-                ].indexOf(m.moment);
-                const dow = this.dayofWeek(this.nowExpr()).sql;
-                if (m.which === 'next') {
-                    const nForwards = `${this.mod7(`${destDay}-(${dow}-1)+6`)}+1`;
-                    const begin = this.delta(this.thisUnit('day').begin, '+', nForwards, 'day');
-                    const end = this.delta(this.thisUnit('day').begin, '+', `${nForwards}+1`, 'day');
-                    // console.log(
-                    //   `SELECT ${
-                    //     this.nowExpr().sql
-                    //   } as now,\n  ${destDay} as destDay,\n  ${dow} as dow,\n  ${nForwards} as nForwards,\n  ${
-                    //     begin.sql
-                    //   } as begin,\n   ${end.sql} as end`
-                    // );
-                    return { begin, end: end.sql };
-                }
-                // dacks back = mod((daw0 - dst) + 6, 7) + 1;
-                // dacks back = mod(((daw - 1) - dst) + 6, 7) + 1;
-                // dacks back = mod(((daw) - dst) + 7, 7) + 1;
-                const nBack = `${this.mod7(`(${dow}-1)-${destDay}+6`)}+1`;
-                const begin = this.delta(this.thisUnit('day').begin, '-', nBack, 'day');
-                const end = this.delta(this.thisUnit('day').begin, '-', `(${nBack})-1`, 'day');
-                // console.log(
-                //   `SELECT ${
-                //     this.nowExpr().sql
-                //   } as now,\n  ${destDay} as destDay,\n  ${dow} as dow,\n  ${nBack} as nBack,\n  ${
-                //     begin.sql
-                //   } as begin,\n   ${end.sql} as end`
-                // );
-                return { begin, end: end.sql };
-            }
+                return this.weekdayMoment(7, m.which);
         }
     }
     isIn(notIn, begin, end) {
@@ -581,6 +550,27 @@ class TemporalFilterCompiler {
         end = this.time(end);
         return `${this.expr} ${begOp} ${begin} ${joinOp} ${this.expr} ${endOp} ${end}`;
     }
+    weekdayMoment(destDay, which) {
+        const direction = which || 'last';
+        const dow = this.dayofWeek(this.nowExpr());
+        const todayBegin = this.thisUnit('day').begin;
+        // destDay comes in as 1-7 (Malloy format), convert to 0-6
+        const destDayZeroBased = destDay - 1;
+        // dow is 1-7, convert to 0-6 for the arithmetic
+        const dowZeroBased = `(${dow.sql}-1)`;
+        let daysOffset;
+        if (direction === 'next') {
+            // Days forward: ((destDay - dow + 6) % 7) + 1
+            daysOffset = `${this.mod7(`${destDayZeroBased}-${dowZeroBased}+6`)}+1`;
+        }
+        else {
+            // Days back: ((dow - destDay + 6) % 7) + 1
+            daysOffset = `${this.mod7(`${dowZeroBased}-${destDayZeroBased}+6`)}+1`;
+        }
+        const begin = this.delta(todayBegin, direction === 'next' ? '+' : '-', daysOffset, 'day');
+        const end = this.delta(begin, '+', '1', 'day');
+        return { begin, end: end.sql };
+    }
 }
 exports.TemporalFilterCompiler = TemporalFilterCompiler;
 //# sourceMappingURL=filter_compilers.js.map

package/dist/model/query_node.d.ts

@@ -77,14 +77,22 @@ export declare class QueryStruct {
     pathAliasMap: Map<string, string>;
     dialect: Dialect;
     connectionName: string;
-    recordAlias?: string;
+    /**
+     * For fields which are a record, but the value is an expression
+     * we capture the context needed to generate the expression in
+     * QueryQuery.expandFields. Later in the compilation if a
+     * reference passes through this struct, this will call
+     * the expression compiler with the correct context
+     * to compute the record value.
+     */
+    computeRecordExpression?: () => string;
+    recordValue?: string;
     constructor(structDef: StructDef, sourceArguments: Record<string, Argument> | undefined, parent: ParentQueryStruct | ParentQueryModel, prepareResultOptions: PrepareResultOptions);
     private static turtleFieldMaker;
     static registerTurtleFieldMaker(maker: (field: TurtleDef, parent: QueryStruct) => QueryField): void;
     private _modelTag;
     modelCompilerFlags(): Tag;
     protected findFirstDialect(): string;
-    informOfAliasValue(av: string): void;
     maybeEmitParameterizedSourceUsage(): void;
     private resolveParentParameterReferences;
     private _arguments;

package/dist/model/query_node.js

@@ -219,9 +219,6 @@ class QueryStruct {
         }
         throw new Error('Cannot create QueryStruct from record with model parent');
     }
-    informOfAliasValue(av) {
-        this.recordAlias = av;
-    }
     maybeEmitParameterizedSourceUsage() {
         var _a;
         if ((0, malloy_types_1.isSourceDef)(this.structDef)) {
@@ -373,8 +370,11 @@ class QueryStruct {
         // will have joins and thus be in the namespace. We can't compute it here
         // because we don't have access to the Query to call exprToSQL.
         if (this.structDef.type === 'record' && (0, malloy_types_1.hasExpression)(this.structDef)) {
-            if (this.recordAlias) {
-                return this.recordAlias;
+            if (this.computeRecordExpression) {
+                if (!this.recordValue) {
+                    this.recordValue = this.computeRecordExpression();
+                }
+                return this.recordValue;
             }
             throw new Error('INTERNAL ERROR, record field alias not pre-computed');
         }

package/dist/model/query_query.d.ts

@@ -43,10 +43,18 @@ export declare class QueryQuery extends QueryField {
     };
     private addDependantPath;
     private dependenciesFromFieldUsage;
-    findRecordAliases(context: QueryStruct, path: string[]): void;
     getSegmentFields(resultStruct: FieldInstanceResult): SegmentFieldDef[];
     private getDrillExpression;
     expandFields(resultStruct: FieldInstanceResult): void;
+    /**
+     * Recursively walks the input QueryStruct tree and sets up lazy expression
+     * compilation for all records with computed expressions, so that records with
+     * expression values have the correct context for evaluating them if needed.
+     *
+     * @param resultStruct - The FieldInstanceResult containing compilation context
+     * @param source - The QueryStruct to traverse (initially the query's parent/input)
+     */
+    expandSource(resultStruct: FieldInstanceResult, source: QueryStruct): void;
     generateSQLFilters(resultStruct: FieldInstanceResult, which: 'where' | 'having'): AndChain;
     prepare(_stageWriter: StageWriter | undefined): void;
     private findJoins;

package/dist/model/query_query.js

@@ -157,14 +157,10 @@ class QueryQuery extends query_node_1.QueryField {
                     resultStruct.addStructToJoin(this.parent, usage.uniqueKeyRequirement);
                 }
                 else {
-                    this.findRecordAliases(this.parent, usage.path);
                     this.addDependantPath(resultStruct, this.parent, usage.path, usage.uniqueKeyRequirement);
                 }
                 continue;
             }
-            if (usage.path.length > 1) {
-                this.findRecordAliases(this.parent, usage.path);
-            }
         }
         const expandedUngroupings = 'expandedUngroupings' in this.firstSegment
             ? this.firstSegment.expandedUngroupings || []
@@ -207,30 +203,6 @@ class QueryQuery extends query_node_1.QueryField {
             }
         }
     }
-    /*
-     ** Later on, when a record is referenced, the context needed to translate the
-     ** reference won't exist, so we translate them all in prepare. The better fix
-     ** involves understanding more about what a "translation state" is and how
-     ** to create it at the moment when a field is referenced, but I couldn't do
-     ** that at the time I did this work. TODO come back and do that.
-     */
-    findRecordAliases(context, path) {
-        for (const seg of path) {
-            const field = context.getChildByName(seg);
-            if (!field) {
-                throw new Error('findRecordAliases: field not found: ' + path.join('.'));
-            }
-            if (field instanceof query_node_1.QueryFieldStruct) {
-                const qs = field.queryStruct;
-                if (qs.structDef.type === 'record' &&
-                    (0, malloy_types_1.hasExpression)(qs.structDef) &&
-                    qs.parent) {
-                    qs.informOfAliasValue((0, expression_compiler_1.exprToSQL)(this.rootResult, qs.parent, qs.structDef.e));
-                }
-                context = qs;
-            }
-        }
-    }
     getSegmentFields(resultStruct) {
         const fs = resultStruct.firstSegment;
         return fs.type === 'index'
@@ -284,6 +256,31 @@ class QueryQuery extends query_node_1.QueryField {
             resultIndex++;
         }
     }
+    /**
+     * Recursively walks the input QueryStruct tree and sets up lazy expression
+     * compilation for all records with computed expressions, so that records with
+     * expression values have the correct context for evaluating them if needed.
+     *
+     * @param resultStruct - The FieldInstanceResult containing compilation context
+     * @param source - The QueryStruct to traverse (initially the query's parent/input)
+     */
+    expandSource(resultStruct, source) {
+        for (const field of source.nameMap.values()) {
+            if (field instanceof query_node_1.QueryFieldStruct) {
+                const qs = field.queryStruct;
+                // Set up closure if this is a record with expression
+                if (qs.structDef.type === 'record' &&
+                    (0, malloy_types_1.hasExpression)(qs.structDef) &&
+                    qs.parent) {
+                    const parent = qs.parent;
+                    const e = qs.structDef.e;
+                    qs.computeRecordExpression = () => (0, expression_compiler_1.exprToSQL)(resultStruct, parent, e);
+                }
+                // Recurse into this structure
+                this.expandSource(resultStruct, qs);
+            }
+        }
+    }
     generateSQLFilters(resultStruct, which
     // filterList: FilterCondition[] | undefined = undefined
     ) {
@@ -307,6 +304,7 @@ class QueryQuery extends query_node_1.QueryField {
     }
     prepare(_stageWriter) {
         if (!this.prepared) {
+            this.expandSource(this.rootResult, this.parent);
             // Add the root base join to the joins map
             this.rootResult.addStructToJoin(this.parent, undefined);
             // Expand fields (just adds them to result, no dependency tracking)

package/dist/version.d.ts

@@ -1 +1 @@
-export declare const MALLOY_VERSION = "0.0.307";
+export declare const MALLOY_VERSION = "0.0.309";

package/dist/version.js

@@ -2,5 +2,5 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.MALLOY_VERSION = void 0;
 // generated with 'generate-version-file' script; do not edit manually
-exports.MALLOY_VERSION = '0.0.307';
+exports.MALLOY_VERSION = '0.0.309';
 //# sourceMappingURL=version.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@malloydata/malloy",
-  "version": "0.0.307",
+  "version": "0.0.309",
   "license": "MIT",
   "exports": {
     ".": "./dist/index.js",
@@ -41,9 +41,9 @@
     "generate-version-file": "VERSION=$(npm pkg get version --workspaces=false | tr -d \\\")\necho \"// generated with 'generate-version-file' script; do not edit manually\\nexport const MALLOY_VERSION = '$VERSION';\" > src/version.ts"
   },
   "dependencies": {
-    "@malloydata/malloy-filter": "0.0.307",
-    "@malloydata/malloy-interfaces": "0.0.307",
-    "@malloydata/malloy-tag": "0.0.307",
+    "@malloydata/malloy-filter": "0.0.309",
+    "@malloydata/malloy-interfaces": "0.0.309",
+    "@malloydata/malloy-tag": "0.0.309",
     "antlr4ts": "^0.5.0-alpha.4",
     "assert": "^2.0.0",
     "jaro-winkler": "^0.2.8",