@malloydata/malloy 0.0.433 → 0.0.435

package/dist/lang/parse-malloy.d.ts

@@ -1,6 +1,7 @@
-import type { DocumentLocation, DocumentPosition, DocumentReference, ImportLocation, ModelDef, SourceDef, SQLSourceDef, DependencyTree, DocumentRange } from '../model/malloy_types';
+import type { DocumentLocation, DocumentPosition, DocumentReference, ImportLocation, ModelDef, SourceDef, SQLSourceDef, DependencyTree, DocumentRange, VirtualMap } from '../model/malloy_types';
 import type { LogMessage, LogMessageOptions, MessageCode, MessageParameterType } from './parse-log';
 import { BaseMessageLogger } from './parse-log';
+import type { FindReferencesData } from './parse-tree-walkers/find-external-references';
 import type { ZoneData } from './zone';
 import { Zone } from './zone';
 import { ReferenceList } from './reference-list';
@@ -52,7 +53,7 @@ declare class ParseStep implements TranslationStep {
 }
 declare class ImportsAndTablesStep implements TranslationStep {
     readonly parseStep: ParseStep;
-    private parseReferences;
+    parseReferences: FindReferencesData | undefined;
     constructor(parseStep: ParseStep);
     step(that: MalloyTranslation): DataRequestResponse | ParseResponse;
 }
@@ -61,6 +62,16 @@ declare class ASTStep implements TranslationStep {
     response?: ASTResponse;
     private walked;
     constructor(importStep: ImportsAndTablesStep);
+    /**
+     * A file acknowledges an experimental dialect with
+     * `##! experimental.dialect.NAME` wherever it names a connection on that
+     * dialect. Every connection name in this file is already resolved to a
+     * dialect by the time the AST exists, and the file's compiler flags are
+     * known only now, so this is the first moment the check can run. The
+     * other place a file takes on a dialect is `import`, checked by
+     * `ImportStatement`.
+     */
+    private checkExperimentalDialects;
     step(that: MalloyTranslation): ASTResponse;
 }
 declare class MetadataStep implements TranslationStep {
@@ -166,8 +177,12 @@ export declare abstract class MalloyTranslation {
     rangeFromContext(pcx: ParserRuleContext): DocumentRange;
     private dialectAlreadyChecked;
     firstReferenceToDialect(dialect: string): boolean;
-    allDialectsEnabled: boolean;
     experimentalDialectEnabled(dialect: string): boolean;
+    /**
+     * Log, once per dialect per file, that this file uses an experimental
+     * dialect without acknowledging it.
+     */
+    checkExperimentalDialect(dialect: string, at: DocumentLocation): void;
 }
 export declare class MalloyChildTranslator extends MalloyTranslation {
     readonly root: MalloyTranslator;
@@ -186,6 +201,7 @@ export declare class MalloyChildTranslator extends MalloyTranslation {
 export declare class MalloyTranslator extends MalloyTranslation {
     private readonly eventStream;
     readonly restrictedMode: boolean;
+    virtualMap?: VirtualMap | undefined;
     schemaZone: Zone<SourceDef>;
     importZone: Zone<string>;
     pretranslatedModels: Map<string, ModelDef>;
@@ -193,7 +209,7 @@ export declare class MalloyTranslator extends MalloyTranslation {
     connectionDialectZone: Zone<string>;
     logger: BaseMessageLogger;
     readonly root: MalloyTranslator;
-    constructor(rootURL: string, importURL?: string | null, preload?: ParseUpdate | null, eventStream?: EventStream | null, restrictedMode?: boolean);
+    constructor(rootURL: string, importURL?: string | null, preload?: ParseUpdate | null, eventStream?: EventStream | null, restrictedMode?: boolean, virtualMap?: VirtualMap | undefined);
     update(dd: ParseUpdate): void;
     private lockZonesIfRestricted;
     translate(extendingModel?: ModelDef): TranslateResponse;

package/dist/lang/parse-malloy.js

@@ -38,6 +38,7 @@ var __importStar = (this && this.__importStar) || (function () {
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.MalloyTranslator = exports.MalloyChildTranslator = exports.MalloyTranslation = void 0;
+const malloy_types_1 = require("../model/malloy_types");
 const utils_1 = require("../model/utils");
 const annotation_utils_1 = require("../model/annotation_utils");
 const ast = __importStar(require("./ast"));
@@ -132,14 +133,13 @@ class ImportsAndTablesStep {
             // Register connection dialects and imports immediately. Table
             // references are deferred until dialects are resolved, because
             // validating a table path requires knowing its dialect's grammar.
-            for (const connName in this.parseReferences.connectionDialects) {
+            for (const [connName, need] of this.parseReferences.connectionDialects) {
                 that.root.connectionDialectZone.reference(connName, {
                     url: that.sourceURL,
-                    range: this.parseReferences.connectionDialects[connName].firstReference,
+                    range: need.firstReference,
                 });
             }
-            for (const relativeRef in this.parseReferences.urls) {
-                const firstRef = this.parseReferences.urls[relativeRef];
+            for (const [relativeRef, firstRef] of this.parseReferences.urls) {
                 try {
                     const ref = decodeURI(new URL(relativeRef, that.importBaseURL || that.sourceURL).toString());
                     that.addChild(ref);
@@ -171,8 +171,7 @@ class ImportsAndTablesStep {
         // Invalid paths are dropped here and re-reported by the AST step.
         // tableRequests feeds the missing-table loop below.
         const tableRequests = {};
-        for (const rawKey in this.parseReferences.tables) {
-            const info = this.parseReferences.tables[rawKey];
+        for (const info of this.parseReferences.tables.values()) {
             const dialectName = that.root.connectionDialectZone.get(info.connectionName);
             if (dialectName === undefined)
                 continue;
@@ -204,7 +203,9 @@ class ImportsAndTablesStep {
         }
         const missingDialects = that.root.connectionDialectZone.getUndefined();
         if (missingDialects) {
-            const connectionDialects = {};
+            // Keyed by connection name, user text: a plain `{}` would turn a
+            // connection named `__proto__` into a prototype and drop the request.
+            const connectionDialects = (0, malloy_types_1.mkSafeRecord)();
             for (const connName of missingDialects) {
                 connectionDialects[connName] = { connectionName: connName };
             }
@@ -237,6 +238,31 @@ class ASTStep {
         this.importStep = importStep;
         this.walked = false;
     }
+    /**
+     * A file acknowledges an experimental dialect with
+     * `##! experimental.dialect.NAME` wherever it names a connection on that
+     * dialect. Every connection name in this file is already resolved to a
+     * dialect by the time the AST exists, and the file's compiler flags are
+     * known only now, so this is the first moment the check can run. The
+     * other place a file takes on a dialect is `import`, checked by
+     * `ImportStatement`.
+     */
+    checkExperimentalDialects(that) {
+        var _a;
+        const connections = (_a = this.importStep.parseReferences) === null || _a === void 0 ? void 0 : _a.connectionDialects;
+        if (!connections) {
+            return;
+        }
+        for (const [connName, need] of connections) {
+            const dialect = that.root.connectionDialectZone.get(connName);
+            if (dialect !== undefined) {
+                that.checkExperimentalDialect(dialect, {
+                    url: that.sourceURL,
+                    range: need.firstReference,
+                });
+            }
+        }
+    }
     step(that) {
         const stepTimer = new timing_1.Timer('ast_step');
         if (this.response) {
@@ -262,6 +288,7 @@ class ASTStep {
         const { ast: newAST, compilerFlagSrc, timingInfo } = secondPass.run();
         stepTimer.contribute([timingInfo]);
         that.compilerFlagSrc = compilerFlagSrc;
+        this.checkExperimentalDialects(that);
         if (newAST.elementType === 'unimplemented') {
             newAST.logError('untranslated-parse-node', 'INTERNAL COMPILER ERROR: Untranslated parse node');
         }
@@ -429,9 +456,9 @@ class TranslateStep {
         // seeding (e.g. TestTranslator's compilerFlags option) survive.
         if (extendingModel && !this.importedAnnotations) {
             const parseCompilerFlagsTimer = new timing_1.Timer('parse_compiler_flags');
-            // Compiler flags from the extending base's `##` annotations. NOTE: `##!`
-            // flag semantics are still to be settled; this keeps the existing
-            // behavior (flags from the base model) green and is not the final design.
+            // `##!` is a pragma: it governs the file it sits in and does not cross
+            // `import`. An extension is a continuation of the base's text, so the
+            // base's flags are in force here.
             that.compilerFlagSrc.push(...new annotation_1.Annotations((0, annotation_utils_1.getModelAnnotations)(extendingModel)).texts('!'));
             stepTimer.contribute([parseCompilerFlagsTimer.stop()]);
             this.importedAnnotations = true;
@@ -515,7 +542,6 @@ class MalloyTranslation {
          Not sure where the logic for this needs to go, this is just my first guess
          */
         this.dialectAlreadyChecked = {};
-        this.allDialectsEnabled = false;
         this.childTranslators = new Map();
         this.modelDef = (0, utils_1.mkModelDef)(sourceURL, (0, utils_1.mkModelID)(sourceURL));
         /**
@@ -753,13 +779,21 @@ class MalloyTranslation {
         return true;
     }
     experimentalDialectEnabled(dialect) {
-        if (this.allDialectsEnabled) {
-            return true;
-        }
         const experimental = this.getCompilerFlags().tag('experimental');
         return (experimental !== undefined &&
             (experimental.bare() || experimental.has('dialect', dialect)));
     }
+    /**
+     * Log, once per dialect per file, that this file uses an experimental
+     * dialect without acknowledging it.
+     */
+    checkExperimentalDialect(dialect, at) {
+        if (this.firstReferenceToDialect(dialect) &&
+            (0, dialect_1.getDialect)(dialect).isExperimental() &&
+            !this.experimentalDialectEnabled(dialect)) {
+            this.root.logError('experimental-dialect-not-enabled', { dialect }, { at });
+        }
+    }
 }
 exports.MalloyTranslation = MalloyTranslation;
 class MalloyChildTranslator extends MalloyTranslation {
@@ -780,10 +814,11 @@ exports.MalloyChildTranslator = MalloyChildTranslator;
  * no need to call again, the translation is finished or error'd.
  */
 class MalloyTranslator extends MalloyTranslation {
-    constructor(rootURL, importURL = null, preload = null, eventStream = null, restrictedMode = false) {
+    constructor(rootURL, importURL = null, preload = null, eventStream = null, restrictedMode = false, virtualMap) {
         super(rootURL, importURL);
         this.eventStream = eventStream;
         this.restrictedMode = restrictedMode;
+        this.virtualMap = virtualMap;
         this.schemaZone = new zone_1.Zone();
         this.importZone = new zone_1.Zone();
         this.pretranslatedModels = new Map();

package/dist/lang/parse-tree-walkers/find-external-references.d.ts

@@ -2,13 +2,13 @@ import type { CommonTokenStream } from 'antlr4ts';
 import type { ParseTree } from 'antlr4ts/tree';
 import type { DocumentRange } from '../../model/malloy_types';
 import type { MalloyTranslation } from '../parse-malloy';
-type NeedImports = Record<string, DocumentRange>;
-type NeedTables = Record<string, {
+type NeedImports = Map<string, DocumentRange>;
+type NeedTables = Map<string, {
     connectionName: string;
     tablePath: string;
     firstReference: DocumentRange;
 }>;
-type NeedConnectionDialects = Record<string, {
+type NeedConnectionDialects = Map<string, {
     firstReference: DocumentRange;
 }>;
 export declare function constructTableKey(connectionName: string, tablePath: string): string;

package/dist/lang/parse-tree-walkers/find-external-references.js

@@ -32,18 +32,18 @@ class FindExternalReferences {
     constructor(trans, tokens) {
         this.trans = trans;
         this.tokens = tokens;
-        this.needTables = {};
-        this.needImports = {};
-        this.needConnectionDialects = {};
+        this.needTables = new Map();
+        this.needImports = new Map();
+        this.needConnectionDialects = new Map();
     }
     registerTableReference(connectionName, tablePath, reference) {
         const key = constructTableKey(connectionName, tablePath);
-        if (!this.needTables[key]) {
-            this.needTables[key] = {
+        if (!this.needTables.has(key)) {
+            this.needTables.set(key, {
                 connectionName,
                 tablePath,
                 firstReference: reference,
-            };
+            });
         }
     }
     enterExploreTable(pcx) {
@@ -53,22 +53,23 @@ class FindExternalReferences {
         this.registerTableReference(connId, tablePath, reference);
         // Register a need for the connection's dialect so the validator in
         // ImportsAndTablesStep can run against it.
-        if (!this.needConnectionDialects[connId]) {
-            this.needConnectionDialects[connId] = { firstReference: reference };
+        this.registerConnection(connId, reference);
+    }
+    registerConnection(connId, reference) {
+        if (connId && !this.needConnectionDialects.has(connId)) {
+            this.needConnectionDialects.set(connId, { firstReference: reference });
         }
     }
+    enterSqlSource(pcx) {
+        this.registerConnection((0, parse_utils_1.getId)(pcx.connectionId()), this.trans.rangeFromContext(pcx));
+    }
     enterVirtualSource(pcx) {
-        const connId = (0, parse_utils_1.getId)(pcx.connectionId());
-        if (connId && !this.needConnectionDialects[connId]) {
-            this.needConnectionDialects[connId] = {
-                firstReference: this.trans.rangeFromContext(pcx),
-            };
-        }
+        this.registerConnection((0, parse_utils_1.getId)(pcx.connectionId()), this.trans.rangeFromContext(pcx));
     }
     enterImportURL(pcx) {
         const url = getPlainString(pcx);
-        if (!this.needImports[url]) {
-            this.needImports[url] = this.trans.rangeFromContext(pcx);
+        if (!this.needImports.has(url)) {
+            this.needImports.set(url, this.trans.rangeFromContext(pcx));
         }
     }
 }

package/dist/lang/test/test-translator.d.ts

@@ -1,8 +1,9 @@
-import type { DocumentLocation, FieldDef, ModelDef, NamedModelObject, PipeSegment, Query, QueryFieldDef, StructDef, TurtleDef, SourceDef, TableSourceDef, SQLSourceDef, UserTypeDef } from '../../model/malloy_types';
+import type { DocumentLocation, FieldDef, ModelDef, NamedModelObject, PipeSegment, Query, QueryFieldDef, StructDef, TurtleDef, SourceDef, TableSourceDef, SQLSourceDef, UserTypeDef, VirtualMap } from '../../model/malloy_types';
 import { MalloyElement } from '../ast';
 import type { NameSpace } from '../ast/types/name-space';
 import type { ModelEntry } from '../ast/types/model-entry';
 import { MalloyChildTranslator, MalloyTranslator } from '../parse-malloy';
+import { Zone } from '../zone';
 import type { DataRequestResponse, SQLSourceRequest, TranslateResponse } from '../translate-response';
 import type { ExprValue } from '../ast/types/expr-value';
 import type { LogSeverity, MessageCode, MessageParameterType } from '../parse-log';
@@ -56,6 +57,7 @@ export interface TestTranslatorOptions {
     eventStream?: EventStream | null;
     internalModel?: ModelDef;
     restrictedMode?: boolean;
+    virtualMap?: VirtualMap;
     /**
      * Each entry is a compiler-flag tag fragment — the content that
      * would follow `##! ` in a Malloy source annotation. Rendered to
@@ -65,10 +67,18 @@ export interface TestTranslatorOptions {
      */
     compilerFlags?: string[];
 }
+/**
+ * Every connection name a test mentions resolves to a dialect, as it would
+ * through a host's connection lookup: the four listed below to theirs, and
+ * any other name to `TEST_DIALECT`.
+ */
+declare class TestConnectionDialectZone extends Zone<string> {
+    reference(str: string, loc: DocumentLocation): void;
+}
 export declare class TestTranslator extends MalloyTranslator {
     readonly testSrc: string;
-    allDialectsEnabled: boolean;
     testRoot?: TestRoot;
+    connectionDialectZone: TestConnectionDialectZone;
     internalModel: ModelDef;
     constructor(testSrc: string, options?: TestTranslatorOptions);
     translate(): TranslateResponse;

package/dist/lang/test/test-translator.js

@@ -29,6 +29,7 @@ const util_1 = require("util");
 const malloy_types_1 = require("../../model/malloy_types");
 const ast_1 = require("../ast");
 const parse_malloy_1 = require("../parse-malloy");
+const zone_1 = require("../zone");
 const static_space_1 = require("../ast/field-space/static-space");
 const global_name_space_1 = require("../ast/types/global-name-space");
 const sql_block_1 = require("../../model/sql_block");
@@ -336,12 +337,25 @@ class TestChildTranslator extends parse_malloy_1.MalloyChildTranslator {
 }
 exports.TestChildTranslator = TestChildTranslator;
 const testURI = 'internal://test/langtests/root.malloy';
+/**
+ * Every connection name a test mentions resolves to a dialect, as it would
+ * through a host's connection lookup: the four listed below to theirs, and
+ * any other name to `TEST_DIALECT`.
+ */
+class TestConnectionDialectZone extends zone_1.Zone {
+    reference(str, loc) {
+        super.reference(str, loc);
+        if (this.getEntry(str).status === 'reference') {
+            this.define(str, exports.TEST_DIALECT);
+        }
+    }
+}
 class TestTranslator extends parse_malloy_1.MalloyTranslator {
     constructor(testSrc, options = {}) {
         var _a, _b, _c, _d, _e;
-        super(testURI, (_a = options.importBaseURL) !== null && _a !== void 0 ? _a : null, null, (_b = options.eventStream) !== null && _b !== void 0 ? _b : null, (_c = options.restrictedMode) !== null && _c !== void 0 ? _c : false);
+        super(testURI, (_a = options.importBaseURL) !== null && _a !== void 0 ? _a : null, null, (_b = options.eventStream) !== null && _b !== void 0 ? _b : null, (_c = options.restrictedMode) !== null && _c !== void 0 ? _c : false, options.virtualMap);
         this.testSrc = testSrc;
-        this.allDialectsEnabled = true;
+        this.connectionDialectZone = new TestConnectionDialectZone();
         /*
          * There are four connections:
          *   _db_  - duckdb dialect, with the following tables ...

package/dist/model/expression_compiler.js

@@ -225,7 +225,7 @@ function compileExpr(resultSet, context, expr, state = new utils_1.GenerateState
                 return `${expr.e.sql} IS NOT NULL`;
             case 'true':
             case 'false':
-                return expr.node;
+                return context.dialect.sqlBoolean(expr.node === 'true');
             case 'null':
                 return 'NULL';
             case 'case':

package/dist/model/join_instance.d.ts

@@ -12,6 +12,7 @@ export declare class JoinInstance {
     joinFilterConditions?: QueryFieldBoolean[];
     children: JoinInstance[];
     constructor(queryStruct: QueryStruct, alias: string, parent: JoinInstance | undefined);
+    isSubquery(): boolean;
     parentRelationship(): 'root' | JoinRelationship;
     forceAllSymmetricCalculations(): boolean;
     getDialectFieldList(): DialectFieldList;

package/dist/model/join_instance.js

@@ -30,6 +30,14 @@ class JoinInstance {
             }, this.queryStruct));
         }
     }
+    // A filtered join whose own joins go inside its subquery, so that the
+    // subquery has to project them back out for anything outside to name them.
+    isSubquery() {
+        return (this.children.length > 0 &&
+            this.joinFilterConditions !== undefined &&
+            (0, malloy_types_1.isJoinedSource)(this.queryStruct.structDef) &&
+            this.queryStruct.dialect.supportsComplexFilteredSources);
+    }
     parentRelationship() {
         if (this.queryStruct.parent === undefined) {
             return 'root';
@@ -64,7 +72,7 @@ class JoinInstance {
     }
     // postgres unnest needs to know the names of the physical fields.
     getDialectFieldList() {
-        return (0, utils_1.getDialectFieldList)(this.queryStruct.structDef);
+        return (0, utils_1.getDialectFieldList)(this.queryStruct.structDef, this.queryStruct.dialect);
     }
 }
 exports.JoinInstance = JoinInstance;

package/dist/model/query_query.d.ts

@@ -17,8 +17,10 @@ type StageGroupMaping = {
 };
 interface StageOutputColumn {
     sql: string;
+    expr: string;
     name: string;
     isDimension: boolean;
+    constant?: boolean;
 }
 type StageOutputContext = {
     columns: StageOutputColumn[];
@@ -34,6 +36,7 @@ export declare class QueryQuery extends QueryField {
     maxDepth: number;
     maxGroupSet: number;
     rootResult: FieldInstanceResultRoot;
+    private packedColumns;
     resultStage: string | undefined;
     stageWriter: StageWriter | undefined;
     isJoinedSubquery: boolean;
@@ -90,6 +93,30 @@ export declare class QueryQuery extends QueryField {
         sql?: string;
     };
     generateSQLJoinBlock(stageWriter: StageWriter, ji: JoinInstance, depth: number): string;
+    /**
+     * Which column of which join a reference needs, or undefined when it needs
+     * none.  Walking `path` from `from`, the answer is the member of the
+     * innermost join along it -- a record is kept whole, since a reference into
+     * one is a reference to the column holding it.
+     *
+     * Two kinds of path need no column.  One ending at a computed field
+     * contributes that field's name, which is no column of the source; the
+     * columns the computation reads arrive as references of their own.  And a
+     * `having:` or `calculate:` names an *output* field, so its path is rooted
+     * in the query's output rather than its source and need not resolve here at
+     * all.  An output name which happens to match a source column resolves as
+     * though it were one, which costs nothing: a query's own source is never
+     * packed, only the joins beneath a filtered one.
+     */
+    private packedColumnFor;
+    /**
+     * Which columns of each join have to survive being packed into a struct:
+     * every column the segment's field usage names, plus each join's primary
+     * key, which the compiler reaches for as a distinct key without any
+     * expression in the query naming it.
+     */
+    private packedColumnsByJoin;
+    private joinProjectionFieldList;
     generateSQLPassthroughKeys(qs: QueryStruct): string;
     generateSQLJoins(stageWriter: StageWriter): string;
     /**
@@ -97,7 +124,7 @@ export declare class QueryQuery extends QueryField {
      * This ordering ensures parent arrays are ordered before child arrays.
      */
     collectArrayJoins(ji: JoinInstance): JoinInstance[];
-    genereateSQLOrderBy(queryDef: QuerySegment, resultStruct: FieldInstanceResult): string;
+    genereateSQLOrderBy(queryDef: QuerySegment, resultStruct: FieldInstanceResult): string[];
     /**
      * The single-scan form, with no `group_set` fan-out: one `SELECT` over the
      * scan, grouped by the dimensions. Used both for a query with no nests and --