@malloydata/malloy 0.0.430 → 0.0.432

package/dist/api/foundation/build_targets.d.ts

@@ -1,3 +1,4 @@
+import type { BuildIdOptions } from '../../model';
 import type { LogMessage } from '../../lang';
 import type { PersistNode } from '../../model/persist_utils';
 import type { Model, PersistSource } from './core';
@@ -46,5 +47,9 @@ export declare function resolvePersistWalk(model: Model, tagParseLog: LogMessage
  *
  * @param walk A resolved walk from {@link resolvePersistWalk}, in order
  * @param connectionDigests One digest per connection named by a persist source
+ * @param buildIdOpts What the BuildID's SQL is compiled under. The compiler
+ *   recomputes the same key at serve time through `buildIdOptions()`; supply a
+ *   different `virtualMap` here and the table is built under a key nothing
+ *   looks up.
  */
-export declare function mkBuildTargets(walk: ResolvedNode[], connectionDigests: Record<string, string>): ConnectionBuild[];
+export declare function mkBuildTargets(walk: ResolvedNode[], connectionDigests: Record<string, string>, buildIdOpts: BuildIdOptions): ConnectionBuild[];

package/dist/api/foundation/build_targets.js

@@ -75,8 +75,12 @@ function addUnique(into, seen, keys) {
  *
  * @param walk A resolved walk from {@link resolvePersistWalk}, in order
  * @param connectionDigests One digest per connection named by a persist source
+ * @param buildIdOpts What the BuildID's SQL is compiled under. The compiler
+ *   recomputes the same key at serve time through `buildIdOptions()`; supply a
+ *   different `virtualMap` here and the table is built under a key nothing
+ *   looks up.
  */
-function mkBuildTargets(walk, connectionDigests) {
+function mkBuildTargets(walk, connectionDigests, buildIdOpts) {
     var _a;
     const targets = new Map();
     // What each source hands to whoever referenced it: its own target if it is
@@ -101,7 +105,7 @@ function mkBuildTargets(walk, connectionDigests) {
                 `BuildID of '${node.sourceID}'. Supply a digest for every connection ` +
                 'named by a persist source.');
         }
-        const sql = source.getSQL();
+        const sql = source.getSQL(buildIdOpts);
         const buildId = source.makeBuildId(digest, sql);
         const key = targetKey(connectionName, buildId);
         let target = targets.get(key);

package/dist/api/foundation/config.js

@@ -6,6 +6,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.MalloyConfig = exports.Manifest = void 0;
 exports.isBuildManifestEntry = isBuildManifestEntry;
+const registry_1 = require("../../connection/registry");
 const config_compile_1 = require("./config_compile");
 const config_resolve_1 = require("./config_resolve");
 const config_lookup_1 = require("./config_lookup");
@@ -165,7 +166,7 @@ class MalloyConfig {
         // it happens async at `lookupConnection` time, so overlays that touch
         // IO (secret stores, session reads) have a natural async seam.
         const prepared = (0, config_resolve_1.prepareConfig)(compiled.compiled, mergedOverlays, log);
-        this._managedLookup = (0, config_lookup_1.buildManagedLookup)(prepared.compiledConnections, mergedOverlays, log);
+        this._managedLookup = (0, config_lookup_1.buildManagedLookup)(prepared.compiledConnections, rawConnectionEntries(pojo), mergedOverlays, log);
         this._connections = this._managedLookup;
         this._overlays = mergedOverlays;
         this.virtualMap = toVirtualMap(prepared.virtualMap, log);
@@ -476,6 +477,27 @@ function toVirtualMap(raw, log) {
 function isRecord(value) {
     return typeof value === 'object' && value !== null && !Array.isArray(value);
 }
+/**
+ * The connection entries exactly as authored, kept alongside the compiled tree
+ * and handed to each factory as `rawConfigData`. Compilation replaces an
+ * overlay reference with a `ConfigReference` node and resolution replaces that
+ * with the value, so this is the only place a connector can still see *which*
+ * overlay a value came from — which is what BigQuery's `getDigest` needs to
+ * tell two impersonated identities apart.
+ *
+ * No validation here beyond the shape: entries the compiler rejected are
+ * absent from `compiledConnections` and so are never looked up.
+ */
+function rawConnectionEntries(pojo) {
+    const entries = {};
+    if (!isRecord(pojo) || !isRecord(pojo['connections']))
+        return entries;
+    for (const [name, entry] of Object.entries(pojo['connections'])) {
+        if ((0, registry_1.isConnectionConfigEntry)(entry))
+            entries[name] = entry;
+    }
+    return entries;
+}
 function isBuildManifestEntry(value) {
     return isRecord(value) && typeof value['tableName'] === 'string';
 }

package/dist/api/foundation/config_compile.d.ts

@@ -3,7 +3,7 @@ import type { LogMessage } from '../../lang/parse-log';
  * A compiled config node. The compiler produces a tree of these; the resolver
  * walks the tree against a ConfigOverlays dict and produces a plain POJO.
  */
-export type ConfigNode = ConfigDict | ConfigLiteral | ConfigReference;
+export type ConfigNode = ConfigDict | ConfigLiteral | ConfigReference | ConfigInvalid;
 export interface ConfigDict {
     kind: 'dict';
     entries: Record<string, ConfigNode>;
@@ -23,6 +23,17 @@ export interface ConfigReference {
     /** Path into the overlay. */
     path: string[];
 }
+/**
+ * A property the user wrote that can never produce a value — it failed its
+ * type check. Compilation records it rather than dropping it so that a
+ * `mustHaveValue` property can tell "the user wrote something unusable" apart
+ * from "the user wrote nothing." It resolves to `undefined` like any other
+ * failure; the difference is only that the key is still visible in the
+ * compiled tree.
+ */
+export interface ConfigInvalid {
+    kind: 'invalid';
+}
 export type SectionCompiler = (value: unknown, log: LogMessage[]) => ConfigNode | undefined;
 /**
  * Compile a POJO into a typed dictionary tree, collecting validation warnings.

package/dist/api/foundation/config_compile.js

@@ -110,34 +110,49 @@ function compileConnectionEntry(prefix, typeName, rawEntry, log) {
     return entry;
 }
 /**
- * Compile a single connection property value. At non-`json` property slots,
- * a single-key object whose value is a string or string[] is recognized as
- * an overlay reference. `json`-typed slots always pass through as literal
- * data — this is the security invariant that prevents reference injection
- * into structured config.
+ * Compile a single connection property value.
+ *
+ * At non-`json` property slots, a single-key object whose value is a string or
+ * string[] is recognized as an overlay reference. `json`-typed slots always
+ * pass through as literal data — this is the security invariant that prevents
+ * reference injection into structured config.
+ *
+ * A property's `source` overrides that reading in both directions. `'literal'`
+ * refuses a reference; `'overlay'` refuses a literal, which is also what lets
+ * a reference be recognized in a slot that would otherwise hold a dictionary.
  */
 function compileConnectionProperty(path, propDef, value, log) {
-    if (value === undefined || value === null)
-        return undefined;
-    if (propDef.type === 'json') {
-        return { kind: 'value', value };
+    if (value === undefined || value === null) {
+        return propDef.mustHaveValue ? { kind: 'invalid' } : undefined;
     }
     const ref = asReferenceShape(value);
-    if (ref !== undefined) {
-        if (propDef.requireLiteralString) {
-            log.push(makeWarning(path, 'must be a literal string and cannot use an overlay reference'));
-            return { kind: 'value', value };
-        }
-        return ref;
+    switch (propDef.source) {
+        case 'overlay':
+            if (ref === undefined) {
+                log.push(makeWarning(path, 'must name an overlay, as in {env: "NAME"}; a value written here ' +
+                    'directly is not allowed'));
+                return { kind: 'invalid' };
+            }
+            return ref;
+        case 'literal':
+            if (ref !== undefined) {
+                log.push(makeWarning(path, 'must be written here directly and cannot name an overlay'));
+                return { kind: 'invalid' };
+            }
+            break;
+        default:
+            if (propDef.type === 'json') {
+                return { kind: 'value', value };
+            }
+            if (ref !== undefined) {
+                return ref;
+            }
+            break;
     }
     const typeError = checkValueType(value, propDef.type);
     if (typeError) {
-        if (propDef.requireLiteralString) {
-            log.push(makeWarning(path, `must be a literal string, got ${describeConfigValue(value)}`));
-            return { kind: 'value', value };
-        }
         log.push(makeWarning(path, `${typeError} (expected ${propDef.type})`));
-        return undefined;
+        return propDef.mustHaveValue ? { kind: 'invalid' } : undefined;
     }
     return { kind: 'value', value };
 }
@@ -221,11 +236,6 @@ function makeWarning(path, message) {
         code: 'config-validation',
     };
 }
-function describeConfigValue(value) {
-    if (Array.isArray(value))
-        return 'array';
-    return typeof value;
-}
 function checkValueType(value, expectedType) {
     switch (expectedType) {
         case 'number':
@@ -245,6 +255,7 @@ function checkValueType(value, expectedType) {
                 return `should be a string, got ${typeof value}`;
             break;
         case 'json':
+        case 'opaque':
             break;
     }
     return undefined;

package/dist/api/foundation/config_lookup.d.ts

@@ -1,5 +1,5 @@
 import type { LogMessage } from '../../lang/parse-log';
-import type { ManagedConnectionLookup } from '../../connection/registry';
+import type { ConnectionConfigEntry, ManagedConnectionLookup } from '../../connection/registry';
 import type { ConfigDict } from './config_compile';
 import type { ConfigOverlays } from './config_overlays';
 /**
@@ -28,4 +28,4 @@ import type { ConfigOverlays } from './config_overlays';
  * intentional consequence of deferred resolution: we don't pay for
  * warnings on connections nobody asks about.
  */
-export declare function buildManagedLookup(compiledConnections: Record<string, ConfigDict>, overlays: ConfigOverlays, log: LogMessage[]): ManagedConnectionLookup;
+export declare function buildManagedLookup(compiledConnections: Record<string, ConfigDict>, rawConnections: Record<string, ConnectionConfigEntry>, overlays: ConfigOverlays, log: LogMessage[]): ManagedConnectionLookup;

package/dist/api/foundation/config_lookup.js

@@ -32,7 +32,7 @@ const registry_1 = require("../../connection/registry");
  * intentional consequence of deferred resolution: we don't pay for
  * warnings on connections nobody asks about.
  */
-function buildManagedLookup(compiledConnections, overlays, log) {
+function buildManagedLookup(compiledConnections, rawConnections, overlays, log) {
     const entries = Object.entries(compiledConnections);
     const firstConnectionName = entries.length > 0 ? entries[0][0] : undefined;
     const cache = new Map();
@@ -51,7 +51,7 @@ function buildManagedLookup(compiledConnections, overlays, log) {
             if (!compiledEntry) {
                 throw new Error(`No connection named "${connectionName}" found in config`);
             }
-            const resolved = await resolveCompiledEntry(compiledEntry, overlays, log);
+            const resolved = await resolveCompiledEntry(connectionName, compiledEntry, overlays, log);
             // compileConnections guarantees `is` is present and a string-valued
             // literal node — resolveCompiledEntry preserves it. Defensive check
             // in case a compiler bug sneaks through.
@@ -71,8 +71,7 @@ function buildManagedLookup(compiledConnections, overlays, log) {
                     connConfig[key] = value;
                 }
             }
-            (0, registry_1.validateConnectionConfigProperties)(connectionName, resolved.is, connConfig);
-            const connection = await typeDef.factory(connConfig);
+            const connection = await typeDef.factory(connConfig, rawConnections[connectionName]);
             cache.set(connectionName, connection);
             return connection;
         },
@@ -97,7 +96,7 @@ function buildManagedLookup(compiledConnections, overlays, log) {
  * followed by property-default application. Returns the plain POJO that
  * gets handed to the factory.
  */
-async function resolveCompiledEntry(entry, overlays, log) {
+async function resolveCompiledEntry(connectionName, entry, overlays, log) {
     const resolved = await resolveNode(entry, overlays, log);
     // resolveNode returns `unknown`. The compileConnections pipeline
     // guarantees every connection entry is an object dict with `is` set to
@@ -106,9 +105,35 @@ async function resolveCompiledEntry(entry, overlays, log) {
     if (!(0, registry_1.isConnectionConfigEntry)(resolved)) {
         throw new Error('Connection entry did not resolve to a valid {is: string, ...} dict');
     }
+    // Before defaults, so that a declared default can't quietly satisfy a
+    // promise the user's own value failed to keep.
+    requireAuthoredValues(connectionName, entry, resolved);
     await applyPropertyDefaults(resolved, overlays);
     return resolved;
 }
+/**
+ * Enforce `mustHaveValue`: a property the user wrote must survive resolution.
+ *
+ * The compiled entry is the record of what was authored — a reference that
+ * resolved to nothing, or a value that failed its type check, is gone from
+ * `resolved` but still present here. Absence from both is fine; that's the
+ * user declining to set the property, which is what `optional` already means.
+ */
+function requireAuthoredValues(connectionName, entry, resolved) {
+    var _a;
+    for (const prop of (_a = (0, registry_1.getConnectionProperties)(resolved.is)) !== null && _a !== void 0 ? _a : []) {
+        if (!prop.mustHaveValue)
+            continue;
+        if (entry.entries[prop.name] === undefined)
+            continue;
+        if (resolved[prop.name] !== undefined)
+            continue;
+        throw new Error(`Connection "${connectionName}" sets "${prop.name}", but no value ` +
+            'arrived — an overlay reference that did not resolve, or a value that ' +
+            'was rejected. Omitting the property is allowed; this one cannot fall ' +
+            'back silently.');
+    }
+}
 /**
  * Walk a node, awaiting any overlay calls. References that fail to resolve
  * return `undefined`; the parent dict walker drops the corresponding
@@ -119,6 +144,8 @@ async function resolveNode(node, overlays, log) {
     switch (node.kind) {
         case 'value':
             return node.value;
+        case 'invalid':
+            return undefined;
         case 'reference':
             return resolveReference(node, overlays, log);
         case 'dict': {

package/dist/api/foundation/core.d.ts

@@ -611,7 +611,11 @@ export declare class PersistSource implements Taggable {
      * For sql_select sources, returns the SQL string (with segment expansion).
      * For query_source sources, compiles the inner query to SQL.
      *
-     * @param options - Compile options including buildManifest for persistence.
+     * @param options - Compile options. `buildManifest` and `connectionDigests`
+     *   substitute already-built dependencies, giving the SQL to execute.
+     *   Anything that changes the SQL without them — `virtualMap` — must match
+     *   what the compiler will use, or the table is built under a key nothing
+     *   looks up.
      * @return The SQL string for this source.
      */
     getSQL(options?: CompileQueryOptions): string;

package/dist/api/foundation/core.js

@@ -1357,7 +1357,11 @@ class PersistSource {
      * For sql_select sources, returns the SQL string (with segment expansion).
      * For query_source sources, compiles the inner query to SQL.
      *
-     * @param options - Compile options including buildManifest for persistence.
+     * @param options - Compile options. `buildManifest` and `connectionDigests`
+     *   substitute already-built dependencies, giving the SQL to execute.
+     *   Anything that changes the SQL without them — `virtualMap` — must match
+     *   what the compiler will use, or the table is built under a key nothing
+     *   looks up.
      * @return The SQL string for this source.
      */
     getSQL(options) {
@@ -1365,7 +1369,7 @@ class PersistSource {
         const queryModel = this.model.queryModel;
         // Compile with finalize=false so this SQL is the bare source SELECT.
         // The build-time key (makeBuildId over this SQL) must equal the serve-time
-        // manifest lookup key, which query_query.ts recomputes from the same
+        // manifest lookup key, which persistedTableFor recomputes from the same
         // unfinalized SELECT; finalizing would diverge the two on dialects with a
         // final stage (Postgres) and mis-materialize the table.
         if (sd.type === 'sql_select') {

package/dist/api/foundation/runtime.js

@@ -511,7 +511,9 @@ class Runtime {
             }
         }
         return {
-            connections: (0, build_targets_1.mkBuildTargets)(walk, connectionDigests),
+            connections: (0, build_targets_1.mkBuildTargets)(walk, connectionDigests, {
+                virtualMap: this.virtualMap,
+            }),
             tagParseLog,
         };
     }

package/dist/connection/registry.d.ts

@@ -1,12 +1,29 @@
 import type { Connection, ConnectionConfig, LookupConnection } from './types';
 /**
  * A factory function that creates a Connection from a config object.
+ *
+ * `rawConfigData` is the connection's entry exactly as the user wrote it,
+ * before overlay references were resolved — so an overlay-supplied property
+ * still reads as `{tenantAuth: "acme"}` rather than as the value it produced.
+ * A connector needs it when a resolved value has no stable identity of its own
+ * but the connection's digest depends on which one it is; BigQuery's
+ * `authClient` is the case that exists today. It is undefined for hosts that
+ * build connections from an already-resolved config.
+ *
+ * Being unresolved, it is the *non-secret* twin of `config`: it holds
+ * `{env: "PGPASSWORD"}` where `config` holds the password.
  */
-export type ConnectionTypeFactory = (config: ConnectionConfig) => Promise<Connection>;
+export type ConnectionTypeFactory = (config: ConnectionConfig, rawConfigData?: ConnectionConfigEntry) => Promise<Connection>;
 /**
  * The type of a connection property value.
  */
-export type ConnectionPropertyType = 'string' | 'number' | 'boolean' | 'password' | 'secret' | 'file' | 'json' | 'text';
+export type ConnectionPropertyType = 'string' | 'number' | 'boolean' | 'password' | 'secret' | 'file' | 'json' | 'text'
+/**
+ * A value with no declared shape that Malloy never inspects. Only reachable
+ * through an overlay — pair it with `source: 'overlay'`, since a config file
+ * has no way to write a live object down.
+ */
+ | 'opaque';
 /**
  * Describes a single configuration property for a connection type.
  *
@@ -35,12 +52,31 @@ export interface ConnectionPropertyDefinition {
     /** For type 'file': extension filters for picker dialogs. */
     fileFilters?: Record<string, string[]>;
     /**
-     * For security-sensitive string slots, preserve malformed/reference-shaped
-     * raw values so registry lookup can fail closed instead of silently dropping
-     * the property during generic compilation. Factories must not rely on this
-     * metadata as their only validation layer.
+     * Where this property's value is allowed to come from. Unset means either —
+     * a literal in the config file, or an overlay reference.
+     *
+     * `'literal'` refuses overlay references. For a property whose value is only
+     * meaningful if it was written down deliberately.
+     *
+     * `'overlay'` refuses literals: the config file can only *name* an overlay
+     * the host registered, never supply the value itself. This is also what
+     * makes a reference unambiguous in a slot that would otherwise hold a
+     * dictionary — if a literal isn't legal there, a single-key object can only
+     * be a reference.
      */
-    requireLiteralString?: true;
+    source?: 'literal' | 'overlay';
+    /**
+     * Writing this property is a promise that it will produce a value. Omitting
+     * it is fine; setting it and having nothing survive resolution is an error
+     * rather than a silent drop.
+     *
+     * For properties whose absence triggers a fallback that isn't safe — DuckDB's
+     * `securityPolicy`, where absent means the `none` policy, or a credential
+     * whose absence means the ambient one. Without this, a misspelled overlay
+     * source or an unset environment variable removes the property and the
+     * connection is built as if the user had never asked for it.
+     */
+    mustHaveValue?: true;
 }
 /**
  * A connection type definition: factory plus property metadata.
@@ -106,11 +142,6 @@ export declare function getRegisteredConnectionTypes(): string[];
  * lookup to hand fully-resolved configs to the right factory.
  */
 export declare function getConnectionTypeDef(typeName: string): ConnectionTypeDef | undefined;
-/**
- * Enforce registry-level literal-string requirements after overlay resolution
- * and before a connection factory sees the config.
- */
-export declare function validateConnectionConfigProperties(connectionName: string, typeName: string, config: ConnectionConfig): void;
 /**
  * Parse a JSON config string into a ConnectionsConfig.
  * Entries without a valid `is` field are silently dropped.

package/dist/connection/registry.js

@@ -10,7 +10,6 @@ exports.getConnectionProperties = getConnectionProperties;
 exports.getConnectionTypeDisplayName = getConnectionTypeDisplayName;
 exports.getRegisteredConnectionTypes = getRegisteredConnectionTypes;
 exports.getConnectionTypeDef = getConnectionTypeDef;
-exports.validateConnectionConfigProperties = validateConnectionConfigProperties;
 exports.readConnectionsConfig = readConnectionsConfig;
 exports.writeConnectionsConfig = writeConnectionsConfig;
 exports.createConnectionsFromConfig = createConnectionsFromConfig;
@@ -67,22 +66,6 @@ function getRegisteredConnectionTypes() {
 function getConnectionTypeDef(typeName) {
     return registry.get(typeName);
 }
-/**
- * Enforce registry-level literal-string requirements after overlay resolution
- * and before a connection factory sees the config.
- */
-function validateConnectionConfigProperties(connectionName, typeName, config) {
-    var _a, _b;
-    const props = (_b = (_a = registry.get(typeName)) === null || _a === void 0 ? void 0 : _a.properties) !== null && _b !== void 0 ? _b : [];
-    for (const prop of props) {
-        if (!prop.requireLiteralString)
-            continue;
-        const value = config[prop.name];
-        if (value !== undefined && typeof value !== 'string') {
-            throw new Error(`Connection "${connectionName}" property "${prop.name}" must be a literal string`);
-        }
-    }
-}
 /**
  * Parse a JSON config string into a ConnectionsConfig.
  * Entries without a valid `is` field are silently dropped.
@@ -144,7 +127,6 @@ function createConnectionsFromConfig(config, onConnectionCreated) {
                     connConfig[key] = value;
                 }
             }
-            validateConnectionConfigProperties(connectionName, entry.is, connConfig);
             const connection = await typeDef.factory(connConfig);
             if (onConnectionCreated) {
                 onConnectionCreated(connectionName, connection);

package/dist/connection/types.d.ts

@@ -49,9 +49,15 @@ export interface InfoConnection {
 export type ConnectionParameterValue = string | number | boolean | null | Array<ConnectionParameterValue> | {
     [key: string]: ConnectionParameterValue;
 };
+/**
+ * A value with no declared shape, supplied by a host overlay rather than
+ * written in a config file — an auth client, a session handle, anything live.
+ * Malloy never inspects one; it carries it from the overlay to the factory.
+ */
+export type OpaqueConnectionValue = object;
 export interface ConnectionConfig {
     name: string;
-    [key: string]: ConnectionParameterValue | undefined;
+    [key: string]: ConnectionParameterValue | OpaqueConnectionValue | undefined;
 }
 export interface ConnectionMetadata {
     url?: string;

package/dist/model/index.d.ts

@@ -7,7 +7,8 @@ export { getResultStructDefForQuery, getResultStructDefForView, } from './query_
 export { indent, composeSQLExpr, makeDigest, mkModelDef, mkModelID, pathToKey, typeDefToString, } from './utils';
 export { getModelAnnotations } from './annotation_utils';
 export { constantExprToSQL } from './constant_expression_compiler';
-export { getCompiledSQL } from './sql_compiled';
+export { getCompiledSQL, buildIdOptions } from './sql_compiled';
+export type { BuildIdOptions } from './sql_compiled';
 export { MalloyCompileError } from './malloy_compile_error';
 export { mkSourceID, mkBuildID, mkQuerySourceDef, mkSQLSourceDef, mkTableSourceDef, resolveSourceID, resolveSourceRef, sourceNamespaceReference, registerSource, hasSourceRegistryEntry, } from './source_def_utils';
 export type { NamespaceReference } from './source_def_utils';

package/dist/model/index.js

@@ -18,7 +18,7 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.hasSourceRegistryEntry = exports.registerSource = exports.sourceNamespaceReference = exports.resolveSourceRef = exports.resolveSourceID = exports.mkTableSourceDef = exports.mkSQLSourceDef = exports.mkQuerySourceDef = exports.mkBuildID = exports.mkSourceID = exports.MalloyCompileError = exports.getCompiledSQL = exports.constantExprToSQL = exports.getModelAnnotations = exports.typeDefToString = exports.pathToKey = exports.mkModelID = exports.mkModelDef = exports.makeDigest = exports.composeSQLExpr = exports.indent = exports.getResultStructDefForView = exports.getResultStructDefForQuery = exports.QueryModel = exports.QueryQuery = exports.QueryStruct = exports.QueryField = void 0;
+exports.hasSourceRegistryEntry = exports.registerSource = exports.sourceNamespaceReference = exports.resolveSourceRef = exports.resolveSourceID = exports.mkTableSourceDef = exports.mkSQLSourceDef = exports.mkQuerySourceDef = exports.mkBuildID = exports.mkSourceID = exports.MalloyCompileError = exports.buildIdOptions = exports.getCompiledSQL = exports.constantExprToSQL = exports.getModelAnnotations = exports.typeDefToString = exports.pathToKey = exports.mkModelID = exports.mkModelDef = exports.makeDigest = exports.composeSQLExpr = exports.indent = exports.getResultStructDefForView = exports.getResultStructDefForQuery = exports.QueryModel = exports.QueryQuery = exports.QueryStruct = exports.QueryField = void 0;
 __exportStar(require("./malloy_types"), exports);
 const query_node_1 = require("./query_node");
 Object.defineProperty(exports, "QueryField", { enumerable: true, get: function () { return query_node_1.QueryField; } });
@@ -51,6 +51,7 @@ var constant_expression_compiler_1 = require("./constant_expression_compiler");
 Object.defineProperty(exports, "constantExprToSQL", { enumerable: true, get: function () { return constant_expression_compiler_1.constantExprToSQL; } });
 var sql_compiled_1 = require("./sql_compiled");
 Object.defineProperty(exports, "getCompiledSQL", { enumerable: true, get: function () { return sql_compiled_1.getCompiledSQL; } });
+Object.defineProperty(exports, "buildIdOptions", { enumerable: true, get: function () { return sql_compiled_1.buildIdOptions; } });
 var malloy_compile_error_1 = require("./malloy_compile_error");
 Object.defineProperty(exports, "MalloyCompileError", { enumerable: true, get: function () { return malloy_compile_error_1.MalloyCompileError; } });
 var source_def_utils_1 = require("./source_def_utils");

package/dist/model/query_query.d.ts

@@ -6,6 +6,7 @@ import { QueryStruct, QueryField } from './query_node';
 import { StageWriter } from './stage_writer';
 import type { FieldInstance } from './field_instance';
 import { FieldInstanceResult, FieldInstanceResultRoot } from './field_instance';
+import type { CompileQueryCallback } from './sql_compiled';
 interface OutputPipelinedSQL {
     sqlFieldName: string;
     pipelineSQL: string;
@@ -66,6 +67,12 @@ export declare class QueryQuery extends QueryField {
     /**  returns a fields and primary key of a struct for this query */
     getResultStructDef(resultStruct?: FieldInstanceResult, isRoot?: boolean): QueryResultDef;
     getStructSourceSQL(qs: QueryStruct, stageWriter: StageWriter): string;
+    /**
+     * A compile callback which writes to its own StageWriter rather than the
+     * one this query is building, so the SQL comes back as a string. Both the
+     * manifest lookup and inline expansion need a source's SQL in that form.
+     */
+    protected isolatedQueryCompiler(): CompileQueryCallback;
     /**
      * Compile a Query into SQL stages. Used by both query_source compilation
      * and getCompiledSQL for interpolated sources.

package/dist/model/query_query.js

@@ -12,7 +12,6 @@ const query_node_1 = require("./query_node");
 const stage_writer_1 = require("./stage_writer");
 const field_instance_1 = require("./field_instance");
 const sql_compiled_1 = require("./sql_compiled");
-const source_def_utils_1 = require("./source_def_utils");
 const malloy_compile_error_1 = require("./malloy_compile_error");
 const nest_capability_1 = require("./nest-capability");
 function pathToCol(path) {
@@ -532,7 +531,7 @@ class QueryQuery extends query_node_1.QueryField {
         return outputStruct;
     }
     getStructSourceSQL(qs, stageWriter) {
-        var _a, _b, _c, _d;
+        var _a, _b, _c, _d, _e;
         switch (qs.structDef.type) {
             case 'table':
                 // tablePath is canonical SQL — translator pre-validated.
@@ -555,38 +554,21 @@ class QueryQuery extends query_node_1.QueryField {
                 return '{COMPOSITE SOURCE}';
             case 'finalize':
                 return qs.structDef.name;
-            case 'sql_select':
-                return `(${(0, sql_compiled_1.getCompiledSQL)(qs.structDef, (_c = qs.prepareResultOptions) !== null && _c !== void 0 ? _c : {}, (query, opts) => {
-                    // Compile query to isolated SQL (not into parent's stageWriter)
-                    const ret = this.compileQueryToStages(query, opts !== null && opts !== void 0 ? opts : {}, undefined, false);
-                    return ret.sql;
-                })})`;
+            case 'sql_select': {
+                // A hit is a table name, which is what a FROM clause wants; the
+                // inline form is a SELECT and has to be parenthesized to sit there.
+                const tableName = (0, sql_compiled_1.persistedTableFor)(qs.structDef, (_c = qs.prepareResultOptions) !== null && _c !== void 0 ? _c : {}, this.isolatedQueryCompiler());
+                if (tableName !== undefined) {
+                    return tableName;
+                }
+                return `(${(0, sql_compiled_1.getCompiledSQL)(qs.structDef, (_d = qs.prepareResultOptions) !== null && _d !== void 0 ? _d : {}, this.isolatedQueryCompiler())})`;
+            }
             case 'nest_source':
                 return qs.structDef.pipeSQL;
             case 'query_source': {
-                const { buildManifest, connectionDigests } = (_d = qs.prepareResultOptions) !== null && _d !== void 0 ? _d : {};
-                // Check manifest for this source (only if it was marked persistent at definition time)
-                if (buildManifest && connectionDigests && qs.structDef.persistent) {
-                    const connDigest = (0, malloy_types_1.safeRecordGet)(connectionDigests, qs.structDef.connection);
-                    if (connDigest) {
-                        // Compile with empty opts to get manifest-ignorant SQL for BuildID
-                        const fullRet = this.compileQueryToStages(qs.structDef.query, {}, undefined, false);
-                        const buildId = (0, source_def_utils_1.mkBuildID)(connDigest, fullRet.sql);
-                        const entry = buildManifest.entries[buildId];
-                        if (entry) {
-                            // Found in manifest - use persisted table.
-                            // entry.tableName comes from the manifest, assumed canonical.
-                            return entry.tableName;
-                        }
-                        if (buildManifest.strict) {
-                            const base = `Persist source '${qs.structDef.sourceID}' not found ` +
-                                `in manifest (buildId: ${buildId}); strict manifest mode ` +
-                                'forbids fallback to live compilation.';
-                            throw new malloy_compile_error_1.MalloyCompileError(buildManifest.loadError
-                                ? `${base}\n  ${buildManifest.loadError}`
-                                : base, 'runtime-manifest-strict-miss', qs.structDef.location);
-                        }
-                    }
+                const tableName = (0, sql_compiled_1.persistedTableFor)(qs.structDef, (_e = qs.prepareResultOptions) !== null && _e !== void 0 ? _e : {}, this.isolatedQueryCompiler());
+                if (tableName !== undefined) {
+                    return tableName;
                 }
                 // Not in manifest - compile normally
                 const ret = this.compileQueryToStages(qs.structDef.query, qs.prepareResultOptions, stageWriter, qs.parent !== undefined);
@@ -596,6 +578,14 @@ class QueryQuery extends query_node_1.QueryField {
                 throw new Error(`Cannot create SQL StageWriter from '${(0, malloy_types_1.activeName)(qs.structDef)}' type '${qs.structDef.type}`);
         }
     }
+    /**
+     * A compile callback which writes to its own StageWriter rather than the
+     * one this query is building, so the SQL comes back as a string. Both the
+     * manifest lookup and inline expansion need a source's SQL in that form.
+     */
+    isolatedQueryCompiler() {
+        return (query, opts) => this.compileQueryToStages(query, opts !== null && opts !== void 0 ? opts : {}, undefined, false).sql;
+    }
     /**
      * Compile a Query into SQL stages. Used by both query_source compilation
      * and getCompiledSQL for interpolated sources.
@@ -1993,11 +1983,11 @@ class QueryQueryRaw extends QueryQuery {
         if (this.parent.structDef.type !== 'sql_select') {
             throw new Error('Invalid struct for QueryQueryRaw, currently only supports SQL');
         }
-        return stageWriter.addStage((0, sql_compiled_1.getCompiledSQL)(this.parent.structDef, (_a = this.parent.prepareResultOptions) !== null && _a !== void 0 ? _a : {}, (query, opts) => {
-            // Compile query to isolated SQL (not into parent's stageWriter)
-            const ret = this.compileQueryToStages(query, opts !== null && opts !== void 0 ? opts : {}, undefined, false);
-            return ret.sql;
-        }));
+        // No manifest lookup for the source itself: a raw pipeline's source is
+        // always the `conn.sql(...)` written at the run site, which is unnamed and
+        // so never persistent. Its `%{ }` dependencies do substitute, through
+        // getCompiledSQL.
+        return stageWriter.addStage((0, sql_compiled_1.getCompiledSQL)(this.parent.structDef, (_a = this.parent.prepareResultOptions) !== null && _a !== void 0 ? _a : {}, this.isolatedQueryCompiler()));
     }
     prepare() {
         // Do nothing!

package/dist/model/sql_compiled.d.ts

@@ -1,4 +1,4 @@
-import type { PrepareResultOptions, SQLSourceDef, PersistableSourceDef, Query } from './malloy_types';
+import type { PrepareResultOptions, SQLSourceDef, PersistableSourceDef, Query, VirtualMap } from './malloy_types';
 /**
  * Callback type for compiling a Query to SQL.
  * opts is optional - omit for "full SQL" (BuildID computation),
@@ -16,6 +16,45 @@ export type CompileQueryCallback = (query: Query, opts?: PrepareResultOptions) =
  * @param compileQuery Callback to compile a Query to SQL
  */
 export declare function getCompiledSQL(src: SQLSourceDef, opts: PrepareResultOptions, compileQuery: CompileQueryCallback): string;
+/**
+ * The options a BuildID's SQL is compiled under — everything the builder and
+ * the compiler must both know, and nothing else.
+ *
+ * Narrow on purpose. A BuildID must not depend on what has already been built
+ * or build order would change the key, so this type cannot express a manifest;
+ * `mkBuildTargets` takes it for the same reason.
+ */
+export interface BuildIdOptions {
+    virtualMap?: VirtualMap;
+}
+/**
+ * Project compile options down to what a BuildID is computed under.
+ *
+ * `virtualMap` decides which table a virtual source reads, so it changes the
+ * SQL and has to be in the key; the builder supplies its own.
+ *
+ * `resolvedGivens` is deliberately absent even though it also changes the SQL.
+ * The builder has no way to produce one — `Runtime.getBuildTargets` takes no
+ * givens and `PersistSource.getSQL` never resolves them — so including it makes
+ * the two sides disagree: the compiler folds `$IS_ADMIN` to a literal while the
+ * builder re-derives it from the declaration and emits `'admin'='admin'`. Both
+ * sides therefore compile givens the same way, by not resolving them. The
+ * consequence — a source whose SQL depends on a supplied given cannot be
+ * persisted, and says so badly — is #3041.
+ */
+export declare function buildIdOptions(opts: PrepareResultOptions): BuildIdOptions;
+/**
+ * Ask the manifest what table backs a persistable source.
+ *
+ * Every place the compiler needs SQL for a persistable source goes through
+ * here, so the rule is stated once: a source marked persistent is looked up
+ * by BuildID, a hit yields the table, a miss under `strict` throws, and
+ * anything else falls through to the source's own SQL.
+ *
+ * @return The table name, canonical SQL as the manifest supplied it, or
+ *         undefined when the caller should emit the source's own SQL.
+ */
+export declare function persistedTableFor(source: PersistableSourceDef, opts: PrepareResultOptions, compileQuery: CompileQueryCallback): string | undefined;
 /**
  * Get the SQL for a PersistableSourceDef.
  *

package/dist/model/sql_compiled.js

@@ -5,6 +5,8 @@
  */
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.getCompiledSQL = getCompiledSQL;
+exports.buildIdOptions = buildIdOptions;
+exports.persistedTableFor = persistedTableFor;
 exports.getSourceSQL = getSourceSQL;
 const malloy_types_1 = require("./malloy_types");
 const source_def_utils_1 = require("./source_def_utils");
@@ -47,34 +49,70 @@ function expandSegment(segment, opts, compileQuery) {
     return expandQuery(segment, opts, compileQuery);
 }
 /**
- * Expand a PersistableSourceDef, checking manifest for pre-built table.
+ * Project compile options down to what a BuildID is computed under.
+ *
+ * `virtualMap` decides which table a virtual source reads, so it changes the
+ * SQL and has to be in the key; the builder supplies its own.
+ *
+ * `resolvedGivens` is deliberately absent even though it also changes the SQL.
+ * The builder has no way to produce one — `Runtime.getBuildTargets` takes no
+ * givens and `PersistSource.getSQL` never resolves them — so including it makes
+ * the two sides disagree: the compiler folds `$IS_ADMIN` to a literal while the
+ * builder re-derives it from the declaration and emits `'admin'='admin'`. Both
+ * sides therefore compile givens the same way, by not resolving them. The
+ * consequence — a source whose SQL depends on a supplied given cannot be
+ * persisted, and says so badly — is #3041.
+ */
+function buildIdOptions(opts) {
+    return { virtualMap: opts.virtualMap };
+}
+/**
+ * Ask the manifest what table backs a persistable source.
+ *
+ * Every place the compiler needs SQL for a persistable source goes through
+ * here, so the rule is stated once: a source marked persistent is looked up
+ * by BuildID, a hit yields the table, a miss under `strict` throws, and
+ * anything else falls through to the source's own SQL.
+ *
+ * @return The table name, canonical SQL as the manifest supplied it, or
+ *         undefined when the caller should emit the source's own SQL.
+ */
+function persistedTableFor(source, opts, compileQuery) {
+    const { buildManifest, connectionDigests } = opts;
+    if (!buildManifest || !connectionDigests || !source.persistent) {
+        return undefined;
+    }
+    const connDigest = (0, malloy_types_1.safeRecordGet)(connectionDigests, source.connection);
+    if (connDigest === undefined) {
+        return undefined;
+    }
+    const buildId = (0, source_def_utils_1.mkBuildID)(connDigest, getSourceSQL(source, compileQuery, buildIdOptions(opts)));
+    const entry = buildManifest.entries[buildId];
+    if (entry) {
+        return entry.tableName;
+    }
+    if (buildManifest.strict) {
+        // `sourceID` is deleted, not inherited, when a source is modified, so it
+        // either names this source or is absent. `as` and `extends` survive a
+        // modification and would name the base, so neither stands in for it.
+        const named = source.sourceID
+            ? `Persisted source '${source.sourceID}'`
+            : 'Persisted source';
+        const base = `${named} not found in manifest (buildId: ${buildId}); ` +
+            'strict manifest mode forbids fallback to live compilation.';
+        throw new malloy_compile_error_1.MalloyCompileError(buildManifest.loadError ? `${base}\n  ${buildManifest.loadError}` : base, 'runtime-manifest-strict-miss', source.location);
+    }
+    return undefined;
+}
+/**
+ * Expand a PersistableSourceDef as a `%{ }` segment.
  * Always returns a subquery form: (SELECT * FROM table) or (inline SQL)
  */
 function expandPersistableSource(source, opts, compileQuery) {
-    const { buildManifest, connectionDigests } = opts;
-    // Try manifest lookup if we have the required info (only for persistent sources)
-    if (buildManifest && connectionDigests && source.persistent) {
-        const connDigest = (0, malloy_types_1.safeRecordGet)(connectionDigests, source.connection);
-        if (connDigest) {
-            // Get the SQL for this source to compute BuildID (no opts = full SQL)
-            const sql = getSourceSQL(source, compileQuery);
-            const buildId = (0, source_def_utils_1.mkBuildID)(connDigest, sql);
-            const entry = buildManifest.entries[buildId];
-            if (entry) {
-                // Found in manifest - substitute with subquery from persisted table.
-                // entry.tableName is canonical SQL, supplied by the manifest builder.
-                return `(SELECT * FROM ${entry.tableName})`;
-            }
-            // Not in manifest
-            if (buildManifest.strict) {
-                const base = `Persist source '${source.sourceID}' not found in manifest ` +
-                    `(buildId: ${buildId}); strict manifest mode forbids fallback ` +
-                    'to live compilation.';
-                throw new malloy_compile_error_1.MalloyCompileError(buildManifest.loadError
-                    ? `${base}\n  ${buildManifest.loadError}`
-                    : base, 'runtime-manifest-strict-miss', source.location);
-            }
-        }
+    // A segment has to be usable as a subquery, so a hit is wrapped.
+    const tableName = persistedTableFor(source, opts, compileQuery);
+    if (tableName !== undefined) {
+        return `(SELECT * FROM ${tableName})`;
     }
     // No manifest or not found - expand inline as subquery
     const sql = getSourceSQL(source, compileQuery, opts);

package/dist/version.d.ts

@@ -1 +1 @@
-export declare const MALLOY_VERSION = "0.0.430";
+export declare const MALLOY_VERSION = "0.0.432";

package/dist/version.js

@@ -2,5 +2,5 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.MALLOY_VERSION = void 0;
 // generated with 'generate-version-file' script; do not edit manually
-exports.MALLOY_VERSION = '0.0.430';
+exports.MALLOY_VERSION = '0.0.432';
 //# sourceMappingURL=version.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@malloydata/malloy",
-  "version": "0.0.430",
+  "version": "0.0.432",
   "license": "MIT",
   "exports": {
     ".": "./dist/index.js",
@@ -51,9 +51,9 @@
     "generate-version-file": "VERSION=$(npm pkg get version --workspaces=false | tr -d \\\")\necho \"// generated with 'generate-version-file' script; do not edit manually\\nexport const MALLOY_VERSION = '$VERSION';\" > src/version.ts"
   },
   "dependencies": {
-    "@malloydata/malloy-filter": "0.0.430",
-    "@malloydata/malloy-interfaces": "0.0.430",
-    "@malloydata/malloy-tag": "0.0.430",
+    "@malloydata/malloy-filter": "0.0.432",
+    "@malloydata/malloy-interfaces": "0.0.432",
+    "@malloydata/malloy-tag": "0.0.432",
     "@noble/hashes": "^1.8.0",
     "antlr4ts": "^0.5.0-alpha.4",
     "assert": "^2.0.0",