@malloydata/malloy 0.0.122 → 0.0.123-dev240205152128

package/dist/lang/malloy-to-ast.js

@@ -695,7 +695,8 @@ class MalloyToAST extends AbstractParseTreeVisitor_1.AbstractParseTreeVisitor {
     }
     visitAggregateOrderBySpec(pcx) {
         const dir = pcx.ASC() ? 'asc' : pcx.DESC() ? 'desc' : undefined;
-        const f = this.getFieldExpr(pcx.fieldExpr());
+        const fCx = pcx.fieldExpr();
+        const f = fCx ? this.getFieldExpr(fCx) : undefined;
         return this.astAt(new ast.FunctionOrderBy(f, dir), pcx);
     }
     visitAggregateOrderByStatement(pcx) {

package/dist/lang/test/expressions.spec.js

@@ -202,6 +202,42 @@ describe('expressions', () => {
           group_by: ai
           calculate: x is lag(ai) { partition_by: ai; order_by: ai }
         }
+      `).toTranslate();
+        });
+        test('analytics order_by requires expression', () => {
+            expect((0, test_translator_1.markSource) `
+        ##! experimental { function_order_by }
+        run: a -> {
+          group_by: ai
+          calculate: x is lag(ai) { order_by: asc }
+        }
+      `).translationToFailWith('analytic `order_by` must specify an aggregate expression or output field reference');
+        });
+        test('string_agg_distinct order by cannot specify expression', () => {
+            expect((0, test_translator_1.markSource) `
+        ##! experimental { function_order_by }
+        run: a -> {
+          group_by: ai
+          aggregate: x is string_agg_distinct(astr) { order_by: ai }
+        }
+      `).translationToFailWith('`order_by` must be only `asc` or `desc` with no expression');
+        });
+        test('string_agg_distinct order by can be just direction', () => {
+            expect((0, test_translator_1.markSource) `
+        ##! experimental { function_order_by }
+        run: a -> {
+          group_by: ai
+          aggregate: x is string_agg_distinct(astr) { order_by: asc }
+        }
+      `).toTranslate();
+        });
+        test('string_agg order by can be just direction', () => {
+            expect((0, test_translator_1.markSource) `
+        ##! experimental { function_order_by }
+        run: a -> {
+          group_by: ai
+          aggregate: x is string_agg(astr) { order_by: asc }
+        }
       `).toTranslate();
         });
         test('can specify multiple partition_bys', () => {

package/dist/model/malloy_query.d.ts

@@ -1,5 +1,5 @@
 import { Dialect, DialectFieldList } from '../dialect';
-import { AggregateFragment, AggregateFunctionType, Annotation, CompiledQuery, DialectFragment, Expr, FieldDef, FieldFragment, Filtered, FilterExpression, FilterFragment, FunctionCallFragment, FunctionOverloadDef, JoinRelationship, ModelDef, OrderBy, OutputFieldFragment, Parameter, ParameterFragment, PipeSegment, Query, QueryFieldDef, QuerySegment, ResultMetadataDef, ResultStructMetadataDef, SearchIndexResult, SourceReferenceFragment, SegmentFieldDef, SpreadFragment, SQLExpressionFragment, SqlStringFragment, StructDef, StructRef, TurtleDef, UngroupFragment } from './malloy_types';
+import { AggregateFragment, AggregateFunctionType, Annotation, CompiledQuery, DialectFragment, Expr, FieldDef, FieldFragment, Filtered, FilterExpression, FilterFragment, FunctionCallFragment, FunctionOverloadDef, FunctionParameterDef, JoinRelationship, ModelDef, OrderBy, OutputFieldFragment, Parameter, ParameterFragment, PipeSegment, Query, QueryFieldDef, QuerySegment, ResultMetadataDef, ResultStructMetadataDef, SearchIndexResult, SourceReferenceFragment, SegmentFieldDef, SpreadFragment, SQLExpressionFragment, SqlStringFragment, StructDef, StructRef, TurtleDef, UngroupFragment, FunctionOrderBy } from './malloy_types';
 import { Connection } from '../runtime_types';
 import { AndChain } from './utils';
 import { QueryInfo } from '../dialect/dialect';
@@ -17,7 +17,7 @@ interface OutputPipelinedSQL {
     sqlFieldName: string;
     pipelineSQL: string;
 }
-type UniqueKeyPossibleUse = AggregateFunctionType | 'generic_aggregate';
+type UniqueKeyPossibleUse = AggregateFunctionType | 'generic_asymmetric_aggregate';
 declare class UniqueKeyUse extends Set<UniqueKeyPossibleUse> {
     add_use(k: UniqueKeyPossibleUse | undefined): this | undefined;
     hasAsymetricFunctions(): boolean;
@@ -71,6 +71,9 @@ declare class QueryField extends QueryNode {
     generateSQLExpression(resultSet: FieldInstanceResult, context: QueryStruct, frag: SQLExpressionFragment, state: GenerateState): string;
     private getParameterMap;
     private expandFunctionCall;
+    getFunctionOrderBy(resultSet: FieldInstanceResult, context: QueryStruct, state: GenerateState, orderBy: FunctionOrderBy[], args: Expr[], overload: FunctionOverloadDef): string | undefined;
+    generateAsymmetricStringAggExpression(resultSet: FieldInstanceResult, context: QueryStruct, value: Expr, separator: Expr | undefined, distinctKey: string, orderBy: FunctionOrderBy[] | undefined, dialectName: string, state: GenerateState): string;
+    getParamForArgIndex(params: FunctionParameterDef[], argIndex: number): FunctionParameterDef;
     generateFunctionCallExpression(resultSet: FieldInstanceResult, context: QueryStruct, frag: FunctionCallFragment, state: GenerateState): string;
     generateSpread(_resultSet: FieldInstanceResult, _context: QueryStruct, _frag: SpreadFragment, _state: GenerateState): string;
     generateParameterFragment(resultSet: FieldInstanceResult, context: QueryStruct, expr: ParameterFragment, _state: GenerateState): string;

package/dist/model/malloy_query.js

@@ -44,7 +44,10 @@ class UniqueKeyUse extends Set {
         }
     }
     hasAsymetricFunctions() {
-        return this.has('sum') || this.has('avg') || this.has('count');
+        return (this.has('sum') ||
+            this.has('avg') ||
+            this.has('count') ||
+            this.has('generic_asymmetric_aggregate'));
     }
 }
 class StageWriter {
@@ -295,34 +298,91 @@ class QueryField extends QueryNode {
             return [fragment];
         });
     }
+    getFunctionOrderBy(resultSet, context, state, orderBy, args, overload) {
+        if (orderBy.length === 0)
+            return undefined;
+        return ('ORDER BY ' +
+            orderBy
+                .map(ob => {
+                var _a, _b;
+                const defaultOrderByArgIndex = (_a = overload.dialect[context.dialect.name].defaultOrderByArgIndex) !== null && _a !== void 0 ? _a : 0;
+                const expr = (_b = ob.e) !== null && _b !== void 0 ? _b : args[defaultOrderByArgIndex];
+                const osql = this.generateDimFragment(resultSet, context, expr, state);
+                const dirsql = ob.dir === 'asc' ? ' ASC' : ob.dir === 'desc' ? ' DESC' : '';
+                return `${osql}${dirsql}`;
+            })
+                .join(', '));
+    }
+    generateAsymmetricStringAggExpression(resultSet, context, value, separator, distinctKey, orderBy, dialectName, state) {
+        if (orderBy) {
+            throw new Error(`Function \`string_agg\` does not support fanning out with an order by in ${dialectName}`);
+        }
+        const valueSQL = this.generateDimFragment(resultSet, context, value, state);
+        const separatorSQL = separator
+            ? ' ,' + this.generateDimFragment(resultSet, context, separator, state)
+            : '';
+        const keyStart = '__STRING_AGG_KS__';
+        const keyEnd = '__STRING_AGG_KE__';
+        const distinctValueSQL = `concat('${keyStart}', ${distinctKey}, '${keyEnd}', ${valueSQL})`;
+        return `REGEXP_REPLACE(
+      STRING_AGG(DISTINCT ${distinctValueSQL}${separatorSQL}),
+      '${keyStart}.*?${keyEnd}',
+      ''
+    )`;
+    }
+    getParamForArgIndex(params, argIndex) {
+        const prevVariadic = params.slice(0, argIndex).find(p => p.isVariadic);
+        return prevVariadic !== null && prevVariadic !== void 0 ? prevVariadic : params[argIndex];
+    }
     generateFunctionCallExpression(resultSet, context, frag, state) {
-        var _a, _b;
+        var _a, _b, _c;
         const overload = frag.overload;
         const args = frag.args;
         const isSymmetric = (_a = frag.overload.isSymmetric) !== null && _a !== void 0 ? _a : false;
         const distinctKey = (0, malloy_types_1.expressionIsAggregate)(overload.returnType.expressionType) &&
             !isSymmetric &&
             this.generateDistinctKeyIfNecessary(resultSet, context, frag.structPath);
-        const aggregateOrderBy = frag.orderBy
-            ? 'ORDER BY ' +
-                frag.orderBy
-                    .map(ob => {
-                    const osql = this.generateDimFragment(resultSet, context, ob.e, state);
-                    const dirsql = ob.dir === 'asc' ? ' ASC' : ob.dir === 'desc' ? ' DESC' : '';
-                    return `${osql}${dirsql}`;
-                })
-                    .join(', ')
-            : undefined;
         const aggregateLimit = frag.limit ? `LIMIT ${frag.limit}` : undefined;
+        if (frag.name === 'string_agg' &&
+            distinctKey &&
+            !context.dialect.supportsAggDistinct) {
+            return this.generateAsymmetricStringAggExpression(resultSet, context, args[0], args[1], distinctKey, frag.orderBy, context.dialect.name, state);
+        }
         if (distinctKey) {
             if (!context.dialect.supportsAggDistinct) {
-                throw new Error(`Asymmetric aggregates are not supported for custom functions in ${context.dialect.name}.`);
+                throw new Error(`Function \`${frag.name}\` does not support fanning out in ${context.dialect.name}`);
             }
             const argsExpressions = args.map(arg => {
                 return this.generateDimFragment(resultSet, context, arg, state);
             });
-            return context.dialect.sqlAggDistinct(distinctKey, argsExpressions, valNames => {
-                const funcCall = this.expandFunctionCall(context.dialect.name, overload, valNames.map(v => [v]), aggregateOrderBy, aggregateLimit);
+            const orderBys = (_b = frag.orderBy) !== null && _b !== void 0 ? _b : [];
+            const orderByExpressions = orderBys.map(ob => {
+                var _a, _b;
+                const defaultOrderByArgIndex = (_a = overload.dialect[context.dialect.name].defaultOrderByArgIndex) !== null && _a !== void 0 ? _a : 0;
+                const expr = (_b = ob.e) !== null && _b !== void 0 ? _b : args[defaultOrderByArgIndex];
+                return this.generateDimFragment(resultSet, context, expr, state);
+            });
+            return context.dialect.sqlAggDistinct(distinctKey, [...argsExpressions, ...orderByExpressions], valNames => {
+                const vals = valNames.map((v, i) => {
+                    // Special case: the argument is required to be literal, so we use the actual argument
+                    // rather than the packed value
+                    // TODO don't even pack the value in the first place
+                    if (i < args.length) {
+                        const param = this.getParamForArgIndex(overload.params, i);
+                        if (param.allowedTypes.every(t => (0, malloy_types_1.isLiteral)(t.evalSpace))) {
+                            return args[i];
+                        }
+                    }
+                    return [v];
+                });
+                const newArgs = vals.slice(0, argsExpressions.length);
+                const orderBy = vals
+                    .slice(argsExpressions.length)
+                    .map((e, i) => {
+                    return { e, dir: orderBys[i].dir };
+                });
+                const orderBySQL = this.getFunctionOrderBy(resultSet, context, state, orderBy, newArgs, overload);
+                const funcCall = this.expandFunctionCall(context.dialect.name, overload, newArgs, orderBySQL, aggregateLimit);
                 return this.generateExpressionFromExpr(resultSet, context, funcCall, state);
             });
         }
@@ -337,7 +397,7 @@ class QueryField extends QueryNode {
                     // Update: Now we apply this only to arguments whose parameter is not constant-requiring.
                     // So in `string_agg(val, sep)`, `sep` does not get filters applied to it because
                     // it must be constant
-                    const param = overload.params[index];
+                    const param = this.getParamForArgIndex(overload.params, index);
                     // TODO technically this should probably look at _which_ allowed param type was matched
                     // for this argument and see if that type is at most constant... but we lose type information
                     // by this point in the compilation, so that info would have to be passed into the func call
@@ -347,14 +407,15 @@ class QueryField extends QueryNode {
                         : [this.generateDimFragment(resultSet, context, arg, state)];
                 })
                 : args;
-            const funcCall = this.expandFunctionCall(context.dialect.name, overload, mappedArgs, aggregateOrderBy, aggregateLimit);
+            const orderBySql = frag.orderBy
+                ? this.getFunctionOrderBy(resultSet, context, state, frag.orderBy, args, overload)
+                : '';
+            const funcCall = this.expandFunctionCall(context.dialect.name, overload, mappedArgs, orderBySql, aggregateLimit);
             if ((0, malloy_types_1.expressionIsAnalytic)(overload.returnType.expressionType)) {
-                const extraPartitions = ((_b = frag.partitionBy) !== null && _b !== void 0 ? _b : []).map(outputName => {
+                const extraPartitions = ((_c = frag.partitionBy) !== null && _c !== void 0 ? _c : []).map(outputName => {
                     return `(${resultSet.getField(outputName).getAnalyticalSQL(false)})`;
                 });
-                // TODO probably need to pass in the function and arguments separately
-                // in order to generate parameter SQL correctly in BQ re: partition
-                return this.generateAnalyticFragment(resultSet, context, funcCall, overload, state, args, extraPartitions, aggregateOrderBy);
+                return this.generateAnalyticFragment(resultSet, context, funcCall, overload, state, args, extraPartitions, orderBySql);
             }
             return this.generateExpressionFromExpr(resultSet, context, funcCall, state);
         }
@@ -1473,6 +1534,7 @@ class QueryQuery extends QueryField {
             .addStructToJoin(struct.getJoinableParent(), this, uniqueKeyPossibleUse, joinStack);
     }
     addDependantExpr(resultStruct, context, e, joinStack) {
+        var _a;
         for (const expr of e) {
             if ((0, malloy_types_1.isFunctionCallFragment)(expr) &&
                 (0, malloy_types_1.expressionIsAnalytic)(expr.overload.returnType.expressionType) &&
@@ -1564,11 +1626,18 @@ class QueryQuery extends QueryField {
                 this.addDependantExpr(resultStruct, context, expr.e, joinStack);
             }
             else if ((0, malloy_types_1.isFunctionCallFragment)(expr)) {
+                const isSymmetric = (_a = expr.overload.isSymmetric) !== null && _a !== void 0 ? _a : false;
+                const isAggregate = (0, malloy_types_1.expressionIsAggregate)(expr.overload.returnType.expressionType);
+                const isAsymmetricAggregate = isAggregate && !isSymmetric;
+                const uniqueKeyPossibleUse = isAsymmetricAggregate
+                    ? 'generic_asymmetric_aggregate'
+                    : undefined;
                 if (expr.structPath) {
-                    this.addDependantPath(resultStruct, context, expr.structPath, 'generic_aggregate', joinStack);
+                    this.addDependantPath(resultStruct, context, expr.structPath, uniqueKeyPossibleUse, joinStack);
+                }
+                else if (isAsymmetricAggregate) {
+                    resultStruct.addStructToJoin(context, this, uniqueKeyPossibleUse, joinStack);
                 }
-                // TODO Do we need to call `addStructToJoin` here in the case when there is no `structPath`
-                // and the function is an aggregate function?
                 for (const e of expr.args) {
                     this.addDependantExpr(resultStruct, context, e, joinStack);
                 }
@@ -1577,7 +1646,9 @@ class QueryQuery extends QueryField {
                 }
                 if (expr.orderBy) {
                     for (const ob of expr.orderBy) {
-                        this.addDependantExpr(resultStruct, context, ob.e, joinStack);
+                        if (ob.e) {
+                            this.addDependantExpr(resultStruct, context, ob.e, joinStack);
+                        }
                     }
                 }
             }

package/dist/model/malloy_types.d.ts

@@ -132,6 +132,7 @@ export interface AggregateLimitFragment {
 }
 export interface FunctionCallFragment {
     type: 'function_call';
+    name: string;
     overload: FunctionOverloadDef;
     expressionType: ExpressionType;
     args: Expr[];
@@ -386,10 +387,15 @@ export interface OrderBy {
     field: string | number;
     dir?: 'asc' | 'desc';
 }
-export interface FunctionOrderBy {
+export interface FunctionOrderByExpression {
     e: Expr;
     dir?: 'asc' | 'desc';
 }
+export interface FunctionOrderByDefaultExpression {
+    e: undefined;
+    dir: 'asc' | 'desc';
+}
+export type FunctionOrderBy = FunctionOrderByExpression | FunctionOrderByDefaultExpression;
 export interface ByName {
     by: 'name';
     name: string;
@@ -599,7 +605,8 @@ export interface FunctionOverloadDef {
     dialect: {
         [dialect: string]: {
             e: Expr;
-            supportsOrderBy?: boolean;
+            supportsOrderBy?: boolean | 'only_default';
+            defaultOrderByArgIndex?: number;
             supportsLimit?: boolean;
         };
     };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@malloydata/malloy",
-  "version": "0.0.122",
+  "version": "0.0.123-dev240205152128",
   "license": "MIT",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",