@malloydata/malloy 0.0.295 → 0.0.296

package/dist/dialect/trino/dialect_functions.js

@@ -211,6 +211,106 @@ const string_reverse = {
     returns: 'string',
     impl: { sql: 'REVERSE(CAST(${str} AS VARCHAR))' },
 };
+const set_agg = {
+    generic: { 'T': ['any'] },
+    takes: { 'value': { dimension: T } },
+    returns: { measure: { array: T } },
+    impl: { function: 'SET_AGG' },
+    isSymmetric: true,
+};
+const set_union = {
+    generic: { 'T': ['any'] },
+    takes: { x: { array: T } },
+    returns: { measure: { array: T } },
+    impl: { function: 'SET_UNION' },
+};
+const hll_accumulate_moving = {
+    preceding: {
+        takes: {
+            'value': { dimension: T },
+            'preceding': { literal: 'number' },
+        },
+        returns: { calculation: { sql_native: 'hyperloglog' } },
+        generic: {
+            'T': ['string', 'number', 'date', 'timestamp', 'boolean', 'json'],
+        },
+        impl: {
+            function: 'APPROX_SET',
+            needsWindowOrderBy: true,
+            between: { preceding: 'preceding', following: 0 },
+        },
+    },
+    following: {
+        takes: {
+            'value': { dimension: T },
+            'preceding': { literal: 'number' },
+            'following': { literal: 'number' },
+        },
+        returns: { calculation: { sql_native: 'hyperloglog' } },
+        generic: {
+            'T': ['string', 'number', 'date', 'timestamp', 'boolean', 'json'],
+        },
+        impl: {
+            function: 'APPROX_SET',
+            needsWindowOrderBy: true,
+            between: { preceding: 'preceding', following: 'following' },
+        },
+    },
+};
+const hll_combine_moving = {
+    preceding: {
+        takes: {
+            'value': { sql_native: 'hyperloglog' },
+            'preceding': { literal: 'number' },
+        },
+        returns: { calculation: { sql_native: 'hyperloglog' } },
+        impl: {
+            function: 'MERGE',
+            needsWindowOrderBy: true,
+            between: { preceding: 'preceding', following: 0 },
+        },
+    },
+    following: {
+        takes: {
+            'value': { sql_native: 'hyperloglog' },
+            'preceding': { literal: 'number' },
+            'following': { literal: 'number' },
+        },
+        returns: { calculation: { sql_native: 'hyperloglog' } },
+        impl: {
+            function: 'MERGE',
+            needsWindowOrderBy: true,
+            between: { preceding: 'preceding', following: 'following' },
+        },
+    },
+};
+const hll_estimate_moving = {
+    preceding: {
+        takes: {
+            'value': { sql_native: 'hyperloglog' },
+            'preceding': { literal: 'number' },
+        },
+        returns: { calculation: 'number' },
+        impl: {
+            function: 'CARDINALITY',
+            needsWindowOrderBy: true,
+            between: { preceding: 'preceding', following: 0 },
+        },
+    },
+    following: {
+        takes: {
+            'value': { sql_native: 'hyperloglog' },
+            'preceding': { literal: 'number' },
+            'following': { literal: 'number' },
+        },
+        returns: { calculation: 'number' },
+        impl: {
+            function: 'CARDINALITY',
+            needsWindowOrderBy: true,
+            between: { preceding: 'preceding', following: 'following' },
+        },
+    },
+};
 /**
  * This map is for functions which exist in both Presto and Trino.
  * If you are adding functions which only exist in Presto, put them in
@@ -223,6 +323,10 @@ exports.TRINO_DIALECT_FUNCTIONS = {
     // string functions
     reverse: string_reverse,
     // aggregate functions
+    max_by,
+    min_by,
+    string_agg,
+    string_agg_distinct,
     // TODO: Approx percentile can be called with a third argument; we probably
     // want to implement that at some point
     // In Presto, this is an "error" parameter between 0 and 1
@@ -284,10 +388,9 @@ exports.TRINO_DIALECT_FUNCTIONS = {
         returns: { dimension: { sql_native: 'hyperloglog' } },
         impl: { sql: 'CAST(${value} AS HyperLogLog)' },
     },
-    max_by,
-    min_by,
-    string_agg,
-    string_agg_distinct,
+    hll_accumulate_moving,
+    hll_combine_moving,
+    hll_estimate_moving,
     ...(0, util_1.def)('variance', { 'n': 'number' }, { measure: 'number' }),
     // scalar functions
     ...(0, util_1.def)('bitwise_and', { 'val1': 'number', 'val2': 'number' }, 'number'),
@@ -315,6 +418,8 @@ exports.TRINO_DIALECT_FUNCTIONS = {
     array_agg_distinct,
     array_join,
     sequence,
+    set_agg,
+    set_union,
     ...(0, util_1.def)('array_distinct', { 'x': { array: T } }, { array: T }),
     ...(0, util_1.def)('array_except', { 'x': { array: T }, 'y': { array: T } }, { array: T }),
     ...(0, util_1.def)('array_intersect', { 'x': { array: T }, 'y': { array: T } }, { array: T }),
@@ -402,5 +507,38 @@ exports.PRESTO_DIALECT_FUNCTIONS = {
             impl: { sql: 'APPROX_SET(${value}, 0.0040625)' },
         },
     },
+    hll_accumulate_moving: {
+        preceding: {
+            takes: {
+                'value': { dimension: T },
+                'preceding': { literal: 'number' },
+            },
+            returns: { calculation: { sql_native: 'hyperloglog' } },
+            generic: {
+                'T': ['string', 'number', 'date', 'timestamp', 'boolean', 'json'],
+            },
+            impl: {
+                sql: 'APPROX_SET(${value}, 0.0040625)',
+                needsWindowOrderBy: true,
+                between: { preceding: 'preceding', following: 0 },
+            },
+        },
+        following: {
+            takes: {
+                'value': { dimension: T },
+                'preceding': { literal: 'number' },
+                'following': { literal: 'number' },
+            },
+            returns: { calculation: { sql_native: 'hyperloglog' } },
+            generic: {
+                'T': ['string', 'number', 'date', 'timestamp', 'boolean', 'json'],
+            },
+            impl: {
+                sql: 'APPROX_SET(${value}, 0.0040625)',
+                needsWindowOrderBy: true,
+                between: { preceding: 'preceding', following: 'following' },
+            },
+        },
+    },
 };
 //# sourceMappingURL=dialect_functions.js.map

package/dist/lang/ast/field-space/reference-field.js

@@ -133,17 +133,17 @@ class ReferenceField extends space_field_1.SpaceField {
         if (refTo) {
             const joinPath = this.fieldRef.list.slice(0, -1).map(x => x.refString);
             const typeDesc = refTo.typeDesc();
+            const usage = {
+                path: this.fieldRef.path,
+                at: this.fieldRef.location,
+            };
             this.memoTypeDesc = {
                 ...typeDesc,
-                fieldUsage: [
-                    {
-                        path: this.fieldRef.path,
-                        at: this.fieldRef.location,
-                    },
-                ],
+                fieldUsage: [usage],
                 requiresGroupBy: (_a = typeDesc.requiresGroupBy) === null || _a === void 0 ? void 0 : _a.map(gb => ({
                     ...gb,
                     path: [...joinPath, ...gb.path],
+                    fieldUsage: usage,
                 })),
             };
             return this.memoTypeDesc;

package/dist/lang/parse-log.d.ts

@@ -1,4 +1,4 @@
-import type { FieldUsage, DocumentLocation, ExpressionValueType } from '../model/malloy_types';
+import type { DocumentLocation, ExpressionValueType } from '../model/malloy_types';
 import type { EventStream } from '../runtime_types';
 export type LogSeverity = 'error' | 'warn' | 'debug';
 /**
@@ -159,11 +159,6 @@ type MessageParameterTypes = {
     'field-not-found': string;
     'composite-field-type-mismatch': string;
     'invalid-composite-source-input': string;
-    'invalid-composite-field-usage': {
-        newUsage: FieldUsage[];
-        allUsage: FieldUsage[];
-        conflictingUsage: FieldUsage[];
-    };
     'could-not-resolve-composite-source': string;
     'empty-composite-source': string;
     'unnecessary-composite-source': string;

package/dist/lang/parse-log.js

@@ -24,7 +24,6 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.MESSAGE_FORMATTERS = exports.BaseMessageLogger = void 0;
 exports.makeLogMessage = makeLogMessage;
-const composite_source_utils_1 = require("../model/composite_source_utils");
 class BaseMessageLogger {
     constructor(eventStream) {
         this.eventStream = eventStream;
@@ -96,13 +95,6 @@ exports.MESSAGE_FORMATTERS = {
     'case-else-type-does-not-match': e => `Case else type ${e.elseType} does not match return type ${e.returnType}`,
     'case-when-must-be-boolean': e => `Case when expression must be boolean, not ${e.whenType}`,
     'case-when-type-does-not-match': e => `Case when type ${e.whenType} does not match value type ${e.valueType}`,
-    'invalid-composite-field-usage': e => {
-        const formattedNewCompositeUsage = (0, composite_source_utils_1.formatFieldUsages)(e.newUsage);
-        const formattedConflictingCompositeUsage = (0, composite_source_utils_1.formatFieldUsages)(e.conflictingUsage);
-        const formattedAllCompositeUsage = (0, composite_source_utils_1.formatFieldUsages)(e.allUsage);
-        const pluralUse = (0, composite_source_utils_1.fieldUsageIsPlural)(e.newUsage) ? 's' : '';
-        return `This operation uses field${pluralUse} ${formattedNewCompositeUsage}, resulting in invalid usage of the composite source, as there is no composite input source which defines all of ${formattedConflictingCompositeUsage}\nFields required in source: ${formattedAllCompositeUsage}`;
-    },
 };
 function makeLogMessage(code, parameters, options) {
     var _a, _b, _c, _d, _e;

package/dist/model/composite_source_utils.d.ts

@@ -84,7 +84,6 @@ export declare function fieldUsageJoinPaths(fieldUsage: FieldUsage[]): string[][
 export declare function checkRequiredGroupBys(compositeResolvedSourceDef: SourceDef, segment: PipeSegment): RequiredGroupBy[];
 export declare function pathEq(a: string[], b: string[]): boolean;
 export declare function pathBegins(path: string[], prefix: string[]): boolean;
-export declare function sortFieldUsageByReferenceLocation(usage: FieldUsage[]): FieldUsage[];
 export declare function hasCompositesAnywhere(source: StructDef): boolean;
 export declare function logCompositeError(error: CompositeError, logTo: MalloyElement): void;
 export declare function compileFilterExpression(ft: string, fexpr: Expr): {

package/dist/model/composite_source_utils.js

@@ -21,13 +21,13 @@ exports.fieldUsageJoinPaths = fieldUsageJoinPaths;
 exports.checkRequiredGroupBys = checkRequiredGroupBys;
 exports.pathEq = pathEq;
 exports.pathBegins = pathBegins;
-exports.sortFieldUsageByReferenceLocation = sortFieldUsageByReferenceLocation;
 exports.hasCompositesAnywhere = hasCompositesAnywhere;
 exports.logCompositeError = logCompositeError;
 exports.compileFilterExpression = compileFilterExpression;
 const malloy_filter_1 = require("@malloydata/malloy-filter");
 const malloy_types_1 = require("./malloy_types");
 const utils_1 = require("./utils");
+const utils_2 = require("../lang/utils");
 function _resolveCompositeSources(path, source, rootFields, nests, fieldUsage, 
 // for resolving nested composites; the list of sources to try
 sources) {
@@ -403,27 +403,40 @@ function resolveCompositeSources(source, segment, fieldUsage) {
 function fieldUsagePaths(fieldUsage) {
     return fieldUsage.map(u => u.path);
 }
-function formatFieldUsages(fieldUsage) {
+function dedupPaths(paths) {
     const deduped = [];
-    for (const usage of fieldUsage) {
-        if (!deduped.some(p => pathEq(p, usage.path))) {
-            deduped.push(usage.path);
+    for (const path of paths) {
+        if (!deduped.some(p => pathEq(p, path))) {
+            deduped.push(path);
         }
     }
+    return deduped;
+}
+function formatPaths(paths, combinator = 'and') {
+    const deduped = dedupPaths(paths);
     const formattedUsages = deduped.map(fieldUsage => formatFieldUsage(fieldUsage));
-    if (formattedUsages.length === 0) {
+    return commaAndList(formattedUsages, combinator);
+}
+function formatRequiredGroupings(requiredGroupings) {
+    return formatPaths(requiredGroupings.map(g => g.path), 'and/or');
+}
+function commaAndList(strs, combinator = 'and') {
+    if (strs.length === 0) {
         return '';
     }
-    else if (formattedUsages.length === 1) {
-        return formattedUsages[0];
+    else if (strs.length === 1) {
+        return strs[0];
     }
-    else if (formattedUsages.length === 2) {
-        return `${formattedUsages[0]} and ${formattedUsages[1]}`;
+    else if (strs.length === 2) {
+        return `${strs[0]} ${combinator} ${strs[1]}`;
     }
     else {
-        return `${formattedUsages.slice(0, -1).join(', ')}, and ${formattedUsages[formattedUsages.length - 1]}`;
+        return `${strs.slice(0, -1).join(', ')}, ${combinator} ${strs[strs.length - 1]}`;
     }
 }
+function formatFieldUsages(fieldUsage) {
+    return formatPaths(fieldUsage.map(u => u.path));
+}
 function countFieldUsage(fieldUsage) {
     const paths = [];
     for (const usage of fieldUsage) {
@@ -529,6 +542,7 @@ function requiredGroupBysAt(requiredGroupBys, at) {
         return requiredGroupBys;
     return requiredGroupBys === null || requiredGroupBys === void 0 ? void 0 : requiredGroupBys.map(r => ({
         ...r,
+        fieldUsage: r.fieldUsage ? fieldUsageAt([r.fieldUsage], at)[0] : undefined,
         at,
     }));
 }
@@ -698,7 +712,7 @@ function expandRefs(nests, fields) {
                         missingFields.push(field);
                         continue;
                     }
-                    requiredGroupBys.push({ path, at: field.at });
+                    requiredGroupBys.push({ path, at: field.at, fieldUsage: field });
                 }
             }
             if (def.ungroupings) {
@@ -827,8 +841,21 @@ function compareLocations(a, b) {
         return 1;
     return 0;
 }
-function sortFieldUsageByReferenceLocation(usage) {
-    return usage.sort((a, b) => compareLocations(a.at, b.at));
+function issueLocation(issue) {
+    if (issue.type === 'missing-field') {
+        return issue.field.at;
+    }
+    else if (issue.type === 'missing-required-group-by') {
+        return issue.requiredGroupBy.at;
+    }
+    else {
+        return issue.firstUsage.at;
+    }
+}
+function sortIssuesByReferenceLocation(issues) {
+    return issues.sort((a, b) => {
+        return compareLocations(issueLocation(a), issueLocation(b));
+    });
 }
 function hasCompositesAnywhere(source) {
     if (source.type === 'composite')
@@ -840,55 +867,60 @@ function hasCompositesAnywhere(source) {
     }
     return false;
 }
+function issueFieldUsage(issue) {
+    if (issue.type === 'missing-field') {
+        return issue.field;
+    }
+    else if (issue.type === 'missing-required-group-by') {
+        return issue.requiredGroupBy.fieldUsage;
+    }
+    else {
+        return undefined;
+    }
+}
 function logCompositeError(error, logTo) {
     if (error.code === 'no_suitable_composite_source_input') {
-        if (error.data.failures.length > 0 &&
-            error.data.failures.every(failure => {
-                return (failure.issues.length > 0 &&
-                    failure.issues.every(issue => issue.type === 'missing-field'));
-            })) {
-            const firstFails = error.data.failures.map(failure => {
-                if (failure.issues[0].type !== 'missing-field')
-                    throw new Error('Programmer error');
-                return failure.issues[0].field;
-            });
-            const sorted = sortFieldUsageByReferenceLocation(firstFails);
-            const lastUsage = sorted[sorted.length - 1];
-            logTo.logError('invalid-composite-field-usage', {
-                newUsage: [lastUsage],
-                conflictingUsage: sorted,
-                allUsage: error.data.usage,
-            }, {
-                at: lastUsage.at,
-            });
-        }
-        else {
-            for (let i = 0; i < error.data.failures.length; i++) {
-                const failure = error.data.failures[i];
-                const sourceName = failure.source.as
-                    ? ` (\`${failure.source.as}\`)`
-                    : '';
-                const join = error.data.path.length === 0
-                    ? ''
-                    : `join ${error.data.path.join('.')} of `;
-                const source = `${join}composed source #${i + 1}${sourceName}`;
-                const requiredFields = `\nFields required in source: ${formatFieldUsages(error.data.usage)}`;
-                for (const issue of failure.issues) {
-                    if (issue.type === 'missing-field') {
-                        const fieldRef = `\`${issue.field.path.join('.')}\``;
-                        logTo.logError('could-not-resolve-composite-source', `Could not resolve composite source: missing field ${fieldRef} in ${source}${requiredFields}`, { at: issue.field.at });
-                    }
-                    else if (issue.type === 'missing-required-group-by') {
-                        const fieldRef = `\`${issue.requiredGroupBy.path.join('.')}\``;
-                        logTo.logError('could-not-resolve-composite-source', `Could not resolve composite source: missing group by or single value filter of ${fieldRef} as required in ${source}${requiredFields}`, { at: issue.requiredGroupBy.at });
-                    }
-                    else {
-                        const joinRef = `\`${issue.path.join('.')}\``;
-                        logTo.logError('could-not-resolve-composite-source', `Could not resolve composite source: join ${joinRef} could not be resolved in ${source}${requiredFields}`, { at: issue.firstUsage.at });
-                    }
-                }
-            }
-        }
+        const firstFails = error.data.failures.map(failure => failure.issues[0]);
+        const sorted = sortIssuesByReferenceLocation(firstFails);
+        const usages = sorted.map(issueFieldUsage);
+        const lastIssue = sorted[sorted.length - 1];
+        const lastUsage = usages[usages.length - 1];
+        const conflictingUsage = firstFails
+            .filter(i => i.type === 'missing-field')
+            .map(i => i.field);
+        const fConflictingUsage = formatFieldUsages(conflictingUsage);
+        const dConflictingUsage = conflictingUsage.length > 0
+            ? `there is no composite input source which defines all of ${fConflictingUsage}`
+            : undefined;
+        const missingGroupBys = firstFails
+            .filter(i => i.type === 'missing-required-group-by')
+            .map(i => i.requiredGroupBy);
+        const fMissingGroupBys = formatRequiredGroupings(missingGroupBys);
+        const dGrouping = 'required group by or single value filter';
+        const dMissingGroupBys = missingGroupBys.length > 0
+            ? `there is a missing ${dGrouping} of ${fMissingGroupBys}`
+            : undefined;
+        const dConflictingUsageAndMissingGroupBys = conflictingUsage.length > 0 && missingGroupBys.length > 0
+            ? `there is no composite input source which defines ${fConflictingUsage} without having an unsatisfied ${dGrouping} on ${fMissingGroupBys}`
+            : undefined;
+        const failedJoins = firstFails
+            .filter(i => i.type === 'join-failed')
+            .map(i => i.path);
+        const uniqueFailedJoins = dedupPaths(failedJoins);
+        const joinPlural = uniqueFailedJoins.length > 1 ? 'joins' : 'join';
+        const dFailedJoins = failedJoins.length > 0
+            ? `${joinPlural} ${formatPaths(uniqueFailedJoins)} could not be resolved`
+            : undefined;
+        const dLastIssue = lastUsage
+            ? `uses field ${formatFieldUsages([lastUsage])}, resulting in`
+            : 'results in';
+        const dIssues = dConflictingUsageAndMissingGroupBys
+            ? commaAndList([dConflictingUsageAndMissingGroupBys, dFailedJoins].filter(utils_2.isNotUndefined))
+            : commaAndList([dConflictingUsage, dMissingGroupBys, dFailedJoins].filter(utils_2.isNotUndefined));
+        const message = `This operation ${dLastIssue} invalid usage of the composite source, as ${dIssues} (fields required in source: ${formatFieldUsages(error.data.usage)})`;
+        logTo.logError('could-not-resolve-composite-source', message, {
+            at: issueLocation(lastIssue),
+        });
     }
     else {
         logTo.logError('could-not-resolve-composite-source', 'Could not resolve composite source');

package/dist/model/malloy_types.d.ts

@@ -685,6 +685,7 @@ export type ExpressionValueType = AtomicFieldType | NonAtomicType;
 export type ExpressionValueTypeDef = AtomicTypeDef | NonAtomicTypeDef;
 export type BasicExpressionType = Exclude<ExpressionValueType, JoinElementType | 'turtle'>;
 export interface RequiredGroupBy {
+    fieldUsage?: FieldUsage;
     at?: DocumentLocation;
     path: string[];
 }

package/dist/version.d.ts

@@ -1 +1 @@
-export declare const MALLOY_VERSION = "0.0.295";
+export declare const MALLOY_VERSION = "0.0.296";

package/dist/version.js

@@ -2,5 +2,5 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.MALLOY_VERSION = void 0;
 // generated with 'generate-version-file' script; do not edit manually
-exports.MALLOY_VERSION = '0.0.295';
+exports.MALLOY_VERSION = '0.0.296';
 //# sourceMappingURL=version.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@malloydata/malloy",
-  "version": "0.0.295",
+  "version": "0.0.296",
   "license": "MIT",
   "exports": {
     ".": "./dist/index.js",
@@ -41,9 +41,9 @@
     "generate-version-file": "VERSION=$(npm pkg get version --workspaces=false | tr -d \\\")\necho \"// generated with 'generate-version-file' script; do not edit manually\\nexport const MALLOY_VERSION = '$VERSION';\" > src/version.ts"
   },
   "dependencies": {
-    "@malloydata/malloy-filter": "0.0.295",
-    "@malloydata/malloy-interfaces": "0.0.295",
-    "@malloydata/malloy-tag": "0.0.295",
+    "@malloydata/malloy-filter": "0.0.296",
+    "@malloydata/malloy-interfaces": "0.0.296",
+    "@malloydata/malloy-tag": "0.0.296",
     "antlr4ts": "^0.5.0-alpha.4",
     "assert": "^2.0.0",
     "jaro-winkler": "^0.2.8",