@malloydata/malloy 0.0.428 → 0.0.430

package/dist/lang/ast/sql-elements/sql-string.js

@@ -35,7 +35,7 @@ class SQLString extends malloy_element_1.MalloyElement {
             el.logError('invalid-sql-source-interpolation', 'This element is not legal inside an SQL string');
         }
     }
-    sqlPhrases() {
+    sqlPhrases(forConnection) {
         const ret = [];
         let valid = true;
         for (const el of this.elements) {
@@ -47,19 +47,39 @@ class SQLString extends malloy_element_1.MalloyElement {
                 const source = el.getSource();
                 if (source) {
                     const sourceDef = source.getSourceDef(undefined);
-                    if ((0, malloy_types_1.isPersistableSourceDef)(sourceDef)) {
+                    const connectionMismatch = sourceDef.connection !== forConnection;
+                    const persistable = (0, malloy_types_1.isPersistableSourceDef)(sourceDef);
+                    if (!connectionMismatch && persistable) {
                         ret.push(sourceDef);
                         continue;
                     }
+                    valid = false;
+                    // A source can be wrong in both ways at once, and both are worth
+                    // saying, so claim the report and then say everything.
+                    if (el.sqClaimError()) {
+                        if (connectionMismatch) {
+                            el.logError('sql-source-connection-mismatch', `Source is on connection ${sourceDef.connection}, but this SQL is for ${forConnection}`);
+                        }
+                        if (!persistable) {
+                            el.logError('invalid-sql-source-interpolation', 'Source is not persistable, cannot be used in SQL');
+                        }
+                    }
+                }
+                else {
+                    el.sqLog('failed-to-expand-sql-source', 'Cannot expand into a query');
+                    valid = false;
                 }
-                el.sqLog('failed-to-expand-sql-source', 'Cannot expand into a query');
-                valid = false;
             }
             else {
                 // Not a source - try as a query
                 const queryObject = el.getQuery();
                 if (queryObject) {
-                    ret.push(queryObject.query(false));
+                    const queryComp = queryObject.queryComp(false, false);
+                    if (queryComp.inputStruct.connection !== forConnection) {
+                        el.sqLog('sql-source-connection-mismatch', `Query is on connection ${queryComp.inputStruct.connection}, but this SQL is for ${forConnection}`);
+                        valid = false;
+                    }
+                    ret.push(queryComp.query);
                 }
                 else {
                     el.sqLog('failed-to-expand-sql-source', 'Cannot expand into a query');

package/dist/lang/ast/statements/import-statement.js

@@ -10,13 +10,6 @@ const malloy_types_1 = require("../../../model/malloy_types");
 const source_def_utils_1 = require("../../../model/source_def_utils");
 const persist_utils_1 = require("../../../model/persist_utils");
 const utils_1 = require("../../../model/utils");
-/** Walk BuildNode tree and collect all sourceIDs */
-function collectSourceIDs(nodes, into) {
-    for (const node of nodes) {
-        into.add(node.sourceID);
-        collectSourceIDs(node.dependsOn, into);
-    }
-}
 class ImportSourceName extends malloy_element_1.MalloyElement {
     constructor(text) {
         super();
@@ -160,10 +153,14 @@ class ImportStatement extends malloy_element_1.ListOf {
                             const importMe = { ...sourceEntry };
                             importMe.as = dstName;
                             doc.setEntry(dstName, { entry: importMe, exported: false });
-                            // Collect dependencies for persistable sources
+                            // Every source the walk touched, routes included — not just the
+                            // persistent ones. A `#@ -persist` wrapper that adds a join is
+                            // not a table, but it can be the only way to reach the tables
+                            // under it, and this model has to be able to take the same route.
                             if ((0, malloy_types_1.isSourceDef)(importMe) && (0, malloy_types_1.isPersistableSourceDef)(importMe)) {
-                                const deps = (0, persist_utils_1.findPersistentDependencies)(importMe, importedModel);
-                                collectSourceIDs(deps, neededSourceIDs);
+                                for (const found of (0, persist_utils_1.walkPersistentDependencies)([importMe], importedModel)) {
+                                    neededSourceIDs.add(found.sourceID);
+                                }
                             }
                         }
                     }

package/dist/lang/ast/types/query-element.d.ts

@@ -2,7 +2,21 @@ import type { MalloyElement } from './malloy-element';
 import type { Query } from '../../../model/malloy_types';
 import type { QueryComp } from './query-comp';
 export interface QueryElement extends MalloyElement {
-    queryComp(isRefOk: boolean): QueryComp;
+    /**
+     * Both arguments say what looseness the caller will accept.
+     *
+     * @param isRefOk `structRef` may be left as the name of a source. False
+     *   means expand it, which is what a caller writing this query into
+     *   something that must stand on its own needs.
+     * @param isPartialOk The pipeline may contain `partial` segments, whose
+     *   query class is not decided yet. Only the base of a refinement can
+     *   accept those, because the refinement is what decides the class.
+     *   False means the pipeline is finished and the compiler can read it —
+     *   any partial found is made a `reduce` and reported as
+     *   `ambiguous-view-type`, so passing false decides where that error
+     *   lands and when it is logged relative to the rest of the pipeline.
+     */
+    queryComp(isRefOk: boolean, isPartialOk: boolean): QueryComp;
     query(isRefOk?: boolean): Query;
 }
 export declare function isQueryElement(e: MalloyElement): e is QueryElement;

package/dist/lang/parse-log.d.ts

@@ -248,11 +248,13 @@ type MessageParameterTypes = {
     'unexpected-source-property': string;
     'aggregate-in-source-filter': string;
     'invalid-connection-for-sql-source': string;
+    'sql-source-connection-mismatch': string;
     'failed-to-fetch-sql-source-schema': string;
     'invalid-sql-source': string;
     'non-top-level-sql-source': string;
     'failed-to-fetch-table-schema': string;
     'invalid-connection-for-table-source': string;
+    'join-connection-mismatch': string;
     'join-on-primary-key-type-mismatch': string;
     'join-primary-key-not-found': string;
     'join-with-without-primary-key': string;

package/dist/lang/test/test-translator.d.ts

@@ -19,6 +19,13 @@ export declare function pretty(thing: unknown): string;
 export declare function humanify(value: unknown): string;
 export declare const TEST_DIALECT = "duckdb";
 export declare const aTableDef: TableSourceDef;
+/**
+ * A second connection speaking the same dialect as `_db_`. Two sources can
+ * then compile to identical SQL on different connections, which isolates the
+ * connection check from the dialect check, and is exactly the case where
+ * persistence could forget that a build target is a table on a connection.
+ */
+export declare const db2TableDef: TableSourceDef;
 export declare const bqTableDef: SourceDef;
 export declare const pgTableDef: SourceDef;
 /**
@@ -128,4 +135,14 @@ export declare function warningMessage(message: string | RegExp): {
     message: string | RegExp;
     severity: LogSeverity;
 };
+/**
+ * Drain a translation's SQL schema requests, answering each with the schema of
+ * `aTable` on the connection which asked, until it stops asking. A `sql()`
+ * source needs a schema before the translation can finish, and nothing in a
+ * TestTranslator provides one.
+ *
+ * @param selectStr What to record as the block's SELECT. Defaults to the
+ *   statement which was requested; pass a value to hand back something else.
+ */
+export declare function answerSQLSchemaRequests(translator: TestTranslator, selectStr?: string): void;
 export {};

package/dist/lang/test/test-translator.js

@@ -5,7 +5,7 @@
  * SPDX-License-Identifier: MIT
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.BetaExpression = exports.TestTranslator = exports.TestChildTranslator = exports.mockSchema = exports.pgTableDef = exports.bqTableDef = exports.aTableDef = exports.TEST_DIALECT = void 0;
+exports.BetaExpression = exports.TestTranslator = exports.TestChildTranslator = exports.mockSchema = exports.pgTableDef = exports.bqTableDef = exports.db2TableDef = exports.aTableDef = exports.TEST_DIALECT = void 0;
 exports.pretty = pretty;
 exports.humanify = humanify;
 exports.getExplore = getExplore;
@@ -24,6 +24,7 @@ exports.error = error;
 exports.warning = warning;
 exports.errorMessage = errorMessage;
 exports.warningMessage = warningMessage;
+exports.answerSQLSchemaRequests = answerSQLSchemaRequests;
 const util_1 = require("util");
 const malloy_types_1 = require("../../model/malloy_types");
 const ast_1 = require("../ast");
@@ -162,6 +163,16 @@ exports.aTableDef = {
         { type: 'timestamptz', name: 'atstz' }, // duckdb supports timestamptz
     ],
 };
+/**
+ * A second connection speaking the same dialect as `_db_`. Two sources can
+ * then compile to identical SQL on different connections, which isolates the
+ * connection check from the dialect check, and is exactly the case where
+ * persistence could forget that a build target is a table on a connection.
+ */
+exports.db2TableDef = {
+    ...exports.aTableDef,
+    connection: '_db2_',
+};
 // BigQuery-compatible table definition (no timestamptz support)
 exports.bqTableDef = {
     type: 'table',
@@ -188,6 +199,7 @@ exports.pgTableDef = {
  */
 exports.mockSchema = [
     exports.aTableDef,
+    exports.db2TableDef,
     exports.bqTableDef,
     exports.pgTableDef,
     {
@@ -331,11 +343,15 @@ class TestTranslator extends parse_malloy_1.MalloyTranslator {
         this.testSrc = testSrc;
         this.allDialectsEnabled = true;
         /*
-         * There are two connections:
+         * There are four connections:
          *   _db_  - duckdb dialect, with the following tables ...
          *      aTable, malloytest.carriers, malloytest.flights, malloytest.airports
+         *   _db2_ - duckdb dialect, with one table, aTable. A second connection
+         *      on the same dialect as _db_, for cross-connection tests.
          *   _bq_  - bigquery/standardsql dialect, with one table
          *      aTable
+         *   _pg_  - postgres dialect, with one table
+         *      aTable
          *
          * The "aTable" table is a mocked table with one column of each type.
          * The _bq_ version does not have the timestamptz column, and when
@@ -361,6 +377,7 @@ class TestTranslator extends parse_malloy_1.MalloyTranslator {
             ...(0, utils_1.mkModelDef)(testURI, testURI),
             contents: {
                 _db_: { type: 'connection', name: '_db_' },
+                _db2_: { type: 'connection', name: '_db2_' },
                 _bq_: { type: 'connection', name: '_bq_' },
                 _pg_: { type: 'connection', name: '_pg_' },
                 a: { ...exports.aTableDef, primaryKey: 'astr', name: 'a' },
@@ -417,6 +434,7 @@ class TestTranslator extends parse_malloy_1.MalloyTranslator {
             this.schemaZone.define(`${actualSchema.connection}:${actualSchema.tablePath}`, actualSchema);
         }
         this.connectionDialectZone.define('_db_', exports.TEST_DIALECT);
+        this.connectionDialectZone.define('_db2_', exports.TEST_DIALECT);
         this.connectionDialectZone.define('_bq_', 'standardsql');
         this.connectionDialectZone.define('_pg_', 'postgres');
         for (const flag of (_e = options.compilerFlags) !== null && _e !== void 0 ? _e : []) {
@@ -694,4 +712,30 @@ function errorMessage(message) {
 function warningMessage(message) {
     return { message, severity: 'warn' };
 }
+/**
+ * Drain a translation's SQL schema requests, answering each with the schema of
+ * `aTable` on the connection which asked, until it stops asking. A `sql()`
+ * source needs a schema before the translation can finish, and nothing in a
+ * TestTranslator provides one.
+ *
+ * @param selectStr What to record as the block's SELECT. Defaults to the
+ *   statement which was requested; pass a value to hand back something else.
+ */
+function answerSQLSchemaRequests(translator, selectStr) {
+    for (;;) {
+        const compileSQL = translator.translate().compileSQL;
+        if (compileSQL === undefined)
+            return;
+        const key = (0, sql_block_1.sqlKey)(compileSQL.connection, compileSQL.selectStr);
+        const schema = {
+            type: 'sql_select',
+            name: key,
+            dialect: exports.TEST_DIALECT,
+            connection: compileSQL.connection,
+            selectStr: selectStr !== null && selectStr !== void 0 ? selectStr : compileSQL.selectStr,
+            fields: exports.aTableDef.fields,
+        };
+        translator.update({ compileSQL: { [key]: schema } });
+    }
+}
 //# sourceMappingURL=test-translator.js.map

package/dist/model/persist_utils.d.ts

@@ -1,4 +1,4 @@
-import type { ModelDef, SourceDef, Query } from './malloy_types';
+import type { ModelDef, SourceDef, Query, SourceID } from './malloy_types';
 import type { LogMessage } from '../lang';
 import type { BuildNode } from '../api/foundation/types';
 /**
@@ -10,15 +10,50 @@ export declare function checkPersistAnnotation(source: SourceDef): {
     log: LogMessage[];
 };
 /**
- * Find persistent dependencies for a source or query, returning a nested DAG.
+ * One persistable source the walk found, and the sources it directly
+ * references.
  *
- * Walks the full dependency tree but only includes persistent sources in the
- * result. Non-persistent sources are "flattened out" - their persistent
- * dependencies bubble up to become direct dependencies of the caller.
+ * Edges are sourceIDs rather than nested nodes, so there is no shared object
+ * to be careful about and nothing to recurse through — a consumer folds the
+ * stream in one pass.
+ */
+export interface PersistNode {
+    sourceID: SourceID;
+    /**
+     * Whether this source is itself something to build. `false` means it is a
+     * *route*: it materializes nothing, but it is how the walk reached something
+     * that does.
+     */
+    persistent: boolean;
+    /** The sources it reaches directly, each of which was yielded already */
+    dependsOn: SourceID[];
+}
+/**
+ * A walk in progress: yields a {@link PersistNode} per source kept, and
+ * finally returns the sources the roots themselves reached.
+ */
+export type PersistWalk = Generator<PersistNode, SourceID[]>;
+/**
+ * Walk everything a source or query references, and emit the sources that
+ * matter to persistence — in dependency order, each one after everything it
+ * depends on.
+ *
+ * A source is emitted if it is persistent — something to build — or if it
+ * is the *route* to something persistent. Everything else is dropped; a
+ * source with nothing persistent beneath it is of no interest to anybody.
+ *
+ * One rule decides both halves, applied on the way back up:
+ *
+ *     keep me if I am persistent, or if any child survived
  *
- * Example: source_c (persist) -> source_b (NOT persist) -> source_a (persist)
- * Returns: [{sourceID: source_a, dependsOn: []}]
- * (source_b is flattened out, source_a becomes direct dependency)
+ * The second clause needs no lookahead. A child only survived under this same
+ * rule, so a surviving child *is* the proof that something persistent lies
+ * below. For `a(persist) → b → c(persist) → d`, d has nothing beneath it
+ * and is dropped, c is kept for being persistent, b is kept for leading to c.
+ *
+ * Because a node arrives only after everything it depends on, a consumer can
+ * fold the stream in a single pass with no second lookup — whether it wants
+ * every source touched, or only the ones that are tables.
  *
  * ## The 6 Dependency Paths in the IR
  *
@@ -35,9 +70,25 @@ export declare function checkPersistAnnotation(source: SourceDef): {
  * Note: CompositeSourceDef.sources[] is ignored - composite sources and
  * persistence may be incompatible features.
  *
- * @param root The source or query to find dependencies for
+ * Joins are followed whether or not a query activates them, so what comes back
+ * is what a source *could* depend on. An unused join is a dependency the SQL
+ * will not contain — an over-approximation, which costs ordering freedom and
+ * never correctness.
+ *
+ * @param roots The sources and queries to walk, sharing one memo
  * @param modelDef The model definition containing the source registry
- * @returns Array of BuildNode representing the persistent dependency DAG
+ * @param tagParseLog Collects errors from parsing `#@` annotations
+ */
+export declare function walkPersistentDependencies(roots: (SourceDef | Query)[], modelDef: ModelDef, tagParseLog?: LogMessage[]): PersistWalk;
+/**
+ * The persistent sources a source or query depends on, as a nested DAG.
+ *
+ * A source that is not itself persistent never appears; its persistent
+ * dependencies become direct dependencies of whoever referenced it. So
+ * `c (persist) → b → a (persist)` yields `[{sourceID: a, dependsOn: []}]`.
+ *
+ * @deprecated The nested shape exists for `Model.getBuildPlan()`, which is on
+ * its way out. Consume {@link walkPersistentDependencies} directly.
  */
 export declare function findPersistentDependencies(root: SourceDef | Query, modelDef: ModelDef, tagParseLog?: LogMessage[]): BuildNode[];
 /**