@malloydata/malloy 0.0.335 → 0.0.337

package/dist/model/persist_utils.js

@@ -0,0 +1,257 @@
+"use strict";
+/*
+ * Copyright Contributors to the Malloy project
+ * SPDX-License-Identifier: MIT
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.findPersistentDependencies = findPersistentDependencies;
+exports.minimalBuildGraph = minimalBuildGraph;
+const malloy_types_1 = require("./malloy_types");
+const source_def_utils_1 = require("./source_def_utils");
+const annotation_1 = require("../annotation");
+/**
+ * Resolve a source name to its definition from model contents.
+ */
+function resolveSource(modelDef, name) {
+    const obj = modelDef.contents[name];
+    return obj && (0, malloy_types_1.isSourceDef)(obj) ? obj : undefined;
+}
+/**
+ * Check if a source has the #@ persist annotation.
+ */
+function checkPersistAnnotation(source) {
+    if (!source.annotation)
+        return false;
+    const { tag } = (0, annotation_1.annotationToTag)(source.annotation, { prefix: /^#@ / });
+    return tag.has('persist');
+}
+/**
+ * Check if a sourceID is persistent, using lazy evaluation and caching.
+ * Sets the persist flag on the registry entry as a side effect.
+ */
+function isPersistent(sourceID, modelDef) {
+    const value = modelDef.sourceRegistry[sourceID];
+    if (!value)
+        return false;
+    if (value.persist === undefined) {
+        const sourceDef = (0, source_def_utils_1.resolveSourceID)(modelDef, sourceID);
+        value.persist = sourceDef ? checkPersistAnnotation(sourceDef) : false;
+    }
+    return value.persist;
+}
+/**
+ * Find persistent dependencies for a source or query, returning a nested DAG.
+ *
+ * Walks the full dependency tree but only includes persistent sources in the
+ * result. Non-persistent sources are "flattened out" - their persistent
+ * dependencies bubble up to become direct dependencies of the caller.
+ *
+ * Example: source_c (persist) -> source_b (NOT persist) -> source_a (persist)
+ * Returns: [{sourceID: source_a, dependsOn: []}]
+ * (source_b is flattened out, source_a becomes direct dependency)
+ *
+ * ## The 6 Dependency Paths in the IR
+ *
+ * Starting from a Query or SourceDef, these are ALL the ways a SourceDef
+ * can be referenced (and thus must be walked for dependency tracking):
+ *
+ * 1. **Query.structRef** → SourceDef (the FROM clause)
+ * 2. **Query.pipeline[].extendSource[]** → JoinFieldDef (joins in extend blocks)
+ * 3. **SourceDef.fields[]** → JoinFieldDef (joins defined on a source)
+ * 4. **PersistableSourceDef.extends** → SourceID (extend chain reference)
+ * 5. **SQLSourceDef.selectSegments[]** → Query | PersistableSourceDef (SQL interpolation)
+ * 6. **QuerySourceDef.query** → Query (nested query in query_source)
+ *
+ * Note: CompositeSourceDef.sources[] is ignored - composite sources and
+ * persistence may be incompatible features.
+ *
+ * @param root The source or query to find dependencies for
+ * @param modelDef The model definition containing the source registry
+ * @returns Array of BuildNode representing the persistent dependency DAG
+ */
+function findPersistentDependencies(root, modelDef) {
+    const visited = new Set();
+    function processSourceID(sourceID) {
+        if (visited.has(sourceID)) {
+            return [];
+        }
+        visited.add(sourceID);
+        const sourceDef = (0, source_def_utils_1.resolveSourceID)(modelDef, sourceID);
+        if (!sourceDef) {
+            return [];
+        }
+        const childDeps = processSourceDef(sourceDef);
+        const persistent = isPersistent(sourceID, modelDef);
+        if (persistent) {
+            return [{ sourceID, dependsOn: childDeps }];
+        }
+        else {
+            return childDeps;
+        }
+    }
+    function processSourceDef(source) {
+        const results = [];
+        // Path 4: PersistableSourceDef.extends
+        if ((0, malloy_types_1.isPersistableSourceDef)(source) && source.extends) {
+            results.push(...processSourceID(source.extends));
+        }
+        // Path 6: QuerySourceDef.query
+        if (source.type === 'query_source') {
+            results.push(...processQuery(source.query));
+        }
+        // Path 5: SQLSourceDef.selectSegments[]
+        if (source.type === 'sql_select' && source.selectSegments) {
+            for (const segment of source.selectSegments) {
+                results.push(...processSQLSegment(segment));
+            }
+        }
+        // Path 3: SourceDef.fields[] - joins defined on the source
+        for (const field of source.fields) {
+            if ((0, malloy_types_1.isJoined)(field) && (0, malloy_types_1.isSourceDef)(field)) {
+                results.push(...processJoinedSource(field));
+            }
+        }
+        return results;
+    }
+    function processQuery(query) {
+        const results = [];
+        // Path 1: Query.structRef
+        results.push(...processStructRef(query.structRef));
+        // Path 2: Query.pipeline[].extendSource[]
+        for (const segment of query.pipeline) {
+            if (segment.type === 'reduce' ||
+                segment.type === 'project' ||
+                segment.type === 'partial') {
+                const querySegment = segment;
+                if (querySegment.extendSource) {
+                    for (const field of querySegment.extendSource) {
+                        if ((0, malloy_types_1.isJoined)(field) && (0, malloy_types_1.isSourceDef)(field)) {
+                            results.push(...processJoinedSource(field));
+                        }
+                    }
+                }
+            }
+        }
+        return results;
+    }
+    function processJoinedSource(source) {
+        // If it has a sourceID, go through the registry
+        if ((0, malloy_types_1.isPersistableSourceDef)(source) && source.sourceID) {
+            return processSourceID(source.sourceID);
+        }
+        // Otherwise walk through it transparently
+        return processSourceDef(source);
+    }
+    function processStructRef(ref) {
+        if (typeof ref === 'string') {
+            const source = resolveSource(modelDef, ref);
+            if (!source)
+                return [];
+            if ((0, malloy_types_1.isPersistableSourceDef)(source) && source.sourceID) {
+                return processSourceID(source.sourceID);
+            }
+            return processSourceDef(source);
+        }
+        else if ((0, malloy_types_1.isSourceDef)(ref)) {
+            if ((0, malloy_types_1.isPersistableSourceDef)(ref) && ref.sourceID) {
+                return processSourceID(ref.sourceID);
+            }
+            return processSourceDef(ref);
+        }
+        return [];
+    }
+    function processSQLSegment(segment) {
+        if ((0, malloy_types_1.isSegmentSQL)(segment)) {
+            return [];
+        }
+        else if ((0, malloy_types_1.isSegmentSource)(segment)) {
+            if ((0, malloy_types_1.isPersistableSourceDef)(segment) && segment.sourceID) {
+                return processSourceID(segment.sourceID);
+            }
+            return processSourceDef(segment);
+        }
+        else {
+            // It's a Query
+            return processQuery(segment);
+        }
+    }
+    // Entry point: handle both SourceDef and Query
+    // Query has required 'structRef', SourceDef does not
+    if ('structRef' in root) {
+        return processQuery(root);
+    }
+    else {
+        // If the root source itself is persistable and has a sourceID, process it through
+        // processSourceID so it gets included in the result if persistent
+        if ((0, malloy_types_1.isPersistableSourceDef)(root) && root.sourceID) {
+            return processSourceID(root.sourceID);
+        }
+        return processSourceDef(root);
+    }
+}
+/**
+ * Collect all sourceIDs from a BuildNode forest (for analysis only).
+ */
+function collectAllSourceIDs(nodes) {
+    const result = new Set();
+    for (const node of nodes) {
+        result.add(node.sourceID);
+        for (const id of collectAllSourceIDs(node.dependsOn)) {
+            result.add(id);
+        }
+    }
+    return result;
+}
+/**
+ * Collect all sourceIDs that appear in any dependsOn (for analysis only).
+ */
+function collectAllDependedOn(nodes) {
+    const result = new Set();
+    for (const node of nodes) {
+        for (const dep of node.dependsOn) {
+            result.add(dep.sourceID);
+        }
+        for (const id of collectAllDependedOn(node.dependsOn)) {
+            result.add(id);
+        }
+    }
+    return result;
+}
+/**
+ * Find the minimal set of root build graphs from a forest of BuildNodes.
+ *
+ * Uses flattening for ANALYSIS ONLY to identify unique nodes and find roots.
+ * Returns original graph structures (NOT flattened) - preserves branching
+ * for parallel builds.
+ *
+ * Roots are sourceIDs that exist but nothing depends on them - these are
+ * the entry points for building.
+ *
+ * @param deps Array of BuildNode trees (potentially overlapping)
+ * @returns Array of root BuildNode trees (deduplicated)
+ */
+function minimalBuildGraph(deps) {
+    if (deps.length === 0)
+        return [];
+    // Use flattening for analysis only
+    const allSourceIDs = collectAllSourceIDs(deps);
+    const dependedOn = collectAllDependedOn(deps);
+    // Roots are sourceIDs that exist but nothing depends on them
+    const rootIDs = new Set();
+    for (const id of allSourceIDs) {
+        if (!dependedOn.has(id)) {
+            rootIDs.add(id);
+        }
+    }
+    // Return original graph structures for roots (deduplicated by sourceID)
+    const seen = new Set();
+    const roots = [];
+    for (const node of deps) {
+        if (rootIDs.has(node.sourceID) && !seen.has(node.sourceID)) {
+            seen.add(node.sourceID);
+            roots.push(node);
+        }
+    }
+    return roots;
+}
+//# sourceMappingURL=persist_utils.js.map

package/dist/model/query_model_impl.d.ts

@@ -1,18 +1,16 @@
 import type { ModelDef, StructRef, Argument, PrepareResultOptions, Query, SourceDef, SearchIndexResult, CompiledQuery, TurtleDef } from './malloy_types';
 import { StageWriter } from './stage_writer';
 import { type Dialect } from '../dialect';
-import type { EventStream } from '../runtime_types';
 import type { Connection } from '../connection/types';
 import type { ModelRootInterface } from './query_node';
 import { QueryStruct } from './query_node';
 import type { QueryModel, QueryResults } from './query_model_contract';
-export declare function makeQueryModel(modelDef: ModelDef | undefined, eventStream?: EventStream): QueryModel;
+export declare function makeQueryModel(modelDef: ModelDef | undefined): QueryModel;
 export declare class QueryModelImpl implements QueryModel, ModelRootInterface {
-    readonly eventStream?: EventStream | undefined;
     dialect: Dialect;
     modelDef: ModelDef | undefined;
     structs: Map<string, QueryStruct>;
-    constructor(modelDef: ModelDef | undefined, eventStream?: EventStream | undefined);
+    constructor(modelDef: ModelDef | undefined);
     getFinalOutputStruct(query: Query, options: PrepareResultOptions | undefined): SourceDef | undefined;
     loadModelFromDef(modelDef: ModelDef): void;
     getStructByName(name: string): QueryStruct;

package/dist/model/query_model_impl.js

@@ -12,15 +12,13 @@ const query_query_1 = require("./query_query");
 const malloy_types_1 = require("./malloy_types");
 const stage_writer_1 = require("./stage_writer");
 const dialect_1 = require("../dialect");
-const utils_1 = require("./materialization/utils");
 const query_node_1 = require("./query_node");
 const row_data_utils_1 = require("../api/row_data_utils");
-function makeQueryModel(modelDef, eventStream) {
-    return new QueryModelImpl(modelDef, eventStream);
+function makeQueryModel(modelDef) {
+    return new QueryModelImpl(modelDef);
 }
 class QueryModelImpl {
-    constructor(modelDef, eventStream) {
-        this.eventStream = eventStream;
+    constructor(modelDef) {
         this.dialect = new dialect_1.StandardSQLDialect();
         // dialect: Dialect = new PostgresDialect();
         this.modelDef = undefined;
@@ -141,13 +139,11 @@ class QueryModelImpl {
         };
     }
     compileQuery(query, prepareResultOptions, finalize = true) {
-        var _a, _b, _c;
-        let newModel;
+        var _a, _b;
         const addDefaultRowLimit = this.addDefaultRowLimit(query, prepareResultOptions === null || prepareResultOptions === void 0 ? void 0 : prepareResultOptions.defaultRowLimit);
         query = addDefaultRowLimit.query;
         const addedDefaultRowLimit = addDefaultRowLimit.addedDefaultRowLimit;
-        const m = newModel || this;
-        const ret = m.loadQuery(query, undefined, prepareResultOptions, finalize, false);
+        const ret = this.loadQuery(query, undefined, prepareResultOptions, finalize, false);
         const structRef = (_a = query.compositeResolvedSourceDef) !== null && _a !== void 0 ? _a : query.structRef;
         const sourceExplore = typeof structRef === 'string'
             ? structRef
@@ -163,10 +159,6 @@ class QueryModelImpl {
             lastStageName: ret.lastStageName,
             malloy: ret.malloy,
             sql: ret.stageWriter.generateSQLStages(),
-            dependenciesToMaterialize: ret.stageWriter.dependenciesToMaterialize,
-            materialization: (0, utils_1.shouldMaterialize)(query.annotation)
-                ? (0, utils_1.buildQueryMaterializationSpec)((_c = query.location) === null || _c === void 0 ? void 0 : _c.url, query.name, prepareResultOptions === null || prepareResultOptions === void 0 ? void 0 : prepareResultOptions.materializedTablePrefix)
-                : undefined,
             structs: ret.structs,
             sourceExplore,
             sourceFilters: query.filterList,

package/dist/model/query_node.d.ts

@@ -61,7 +61,7 @@ export interface ParentQueryStruct {
     struct: QueryStruct;
 }
 export interface ModelRootInterface {
-    eventStream?: EventStream;
+    structs: Map<string, QueryStruct>;
 }
 export interface ParentQueryModel {
     model: ModelRootInterface;
@@ -119,7 +119,6 @@ export declare class QueryStruct {
     resolveQueryFields(finalOutputStruct: (query: Query, options: PrepareResultOptions | undefined) => SourceDef | undefined): void;
     getModel(): ModelRootInterface;
     get eventStream(): EventStream | undefined;
-    setParent(parent: ParentQueryStruct | ParentQueryModel): void;
     /** makes a new queryable field object from a fieldDef */
     makeQueryField(field: FieldDef, referenceId?: string): QueryField;
     root(): QueryStruct;

package/dist/model/query_node.js

@@ -180,7 +180,6 @@ class QueryStruct {
         this.nameMap = new Map();
         this._modelTag = undefined;
         this._arguments = undefined;
-        this.setParent(parent);
         if ('model' in parent) {
             this.model = parent.model;
             this.pathAliasMap = new Map();
@@ -192,6 +191,7 @@ class QueryStruct {
             }
         }
         else {
+            this.parent = parent.struct;
             this.model = this.getModel();
             this.pathAliasMap = this.root().pathAliasMap;
             this.connectionName = this.root().connectionName;
@@ -471,18 +471,8 @@ class QueryStruct {
         }
     }
     get eventStream() {
-        return this.getModel().eventStream;
-    }
-    setParent(parent) {
-        if ('struct' in parent) {
-            this.parent = parent.struct;
-        }
-        if ('model' in parent) {
-            this.model = parent.model;
-        }
-        else {
-            this.model = this.getModel();
-        }
+        var _a;
+        return (_a = this.prepareResultOptions) === null || _a === void 0 ? void 0 : _a.eventStream;
     }
     /** makes a new queryable field object from a fieldDef */
     makeQueryField(field, referenceId) {

package/dist/model/query_query.d.ts

@@ -1,5 +1,5 @@
 import type { DialectFieldList } from '../dialect';
-import type { TurtleDef, QueryResultDef, ResultStructMetadataDef, ResultMetadataDef, PipeSegment, QuerySegment, QueryFieldDef, SegmentFieldDef } from './malloy_types';
+import type { TurtleDef, QueryResultDef, ResultStructMetadataDef, ResultMetadataDef, PipeSegment, QuerySegment, QueryFieldDef, SegmentFieldDef, Query, PrepareResultOptions } from './malloy_types';
 import { AndChain } from './utils';
 import type { JoinInstance } from './join_instance';
 import { QueryStruct, QueryField } from './query_node';
@@ -63,6 +63,22 @@ export declare class QueryQuery extends QueryField {
     /**  returns a fields and primary key of a struct for this query */
     getResultStructDef(resultStruct?: FieldInstanceResult, isRoot?: boolean): QueryResultDef;
     getStructSourceSQL(qs: QueryStruct, stageWriter: StageWriter): string;
+    /**
+     * Compile a Query into SQL stages. Used by both query_source compilation
+     * and getCompiledSQL for interpolated sources.
+     *
+     * @param query The query to compile
+     * @param prepareResultOptions Options including manifest for substitution
+     * @param stageWriter If provided, stages are added to this writer and lastStageName is returned.
+     *                    If undefined, a new isolated writer is created and full SQL is returned.
+     * @param isJoinedSubquery Whether this is a joined subquery
+     * @returns { lastStageName, stageWriter, sql } - sql is only set if stageWriter was undefined
+     */
+    compileQueryToStages(query: Query, prepareResultOptions: PrepareResultOptions, stageWriter: StageWriter | undefined, isJoinedSubquery: boolean): {
+        lastStageName: string;
+        stageWriter: StageWriter;
+        sql?: string;
+    };
     generateSQLJoinBlock(stageWriter: StageWriter, ji: JoinInstance, depth: number): string;
     generateSQLPassthroughKeys(qs: QueryStruct): string;
     generateSQLJoins(stageWriter: StageWriter): string;

package/dist/model/query_query.js

@@ -11,7 +11,8 @@ const utils_1 = require("./utils");
 const query_node_1 = require("./query_node");
 const stage_writer_1 = require("./stage_writer");
 const field_instance_1 = require("./field_instance");
-const utils_2 = require("./materialization/utils");
+const sql_compiled_1 = require("./sql_compiled");
+const source_def_utils_1 = require("./source_def_utils");
 function pathToCol(path) {
     return path.map(el => encodeURIComponent(el)).join('/');
 }
@@ -51,7 +52,7 @@ class QueryQuery extends query_node_1.QueryField {
             parent = new query_node_1.QueryStruct({
                 ...sourceDef,
                 fields: [...sourceDef.fields, ...firstStage.extendSource],
-            }, parentStruct.sourceArguments, parent.parent ? { struct: parent } : { model: parent.model }, parent.prepareResultOptions);
+            }, parentStruct.sourceArguments, parent.parent ? { struct: parent } : { model: parent.getModel() }, parent.prepareResultOptions);
             turtleWithFilters = {
                 ...turtleWithFilters,
                 pipeline: [
@@ -514,7 +515,7 @@ class QueryQuery extends query_node_1.QueryField {
         return outputStruct;
     }
     getStructSourceSQL(qs, stageWriter) {
-        var _a, _b, _c;
+        var _a, _b;
         switch (qs.structDef.type) {
             case 'table':
                 return this.parent.dialect.quoteTablePath(qs.structDef.tablePath);
@@ -525,47 +526,86 @@ class QueryQuery extends query_node_1.QueryField {
             case 'finalize':
                 return qs.structDef.name;
             case 'sql_select':
-                return `(${qs.structDef.selectStr})`;
+                return `(${(0, sql_compiled_1.getCompiledSQL)(qs.structDef, (_a = qs.prepareResultOptions) !== null && _a !== void 0 ? _a : {}, path => this.parent.dialect.quoteTablePath(path), (query, opts) => {
+                    // Compile query to isolated SQL (not into parent's stageWriter)
+                    const ret = this.compileQueryToStages(query, opts !== null && opts !== void 0 ? opts : {}, undefined, false);
+                    return ret.sql;
+                })})`;
             case 'nest_source':
                 return qs.structDef.pipeSQL;
             case 'query_source': {
-                // cache derived table.
-                if (((_a = qs.prepareResultOptions) === null || _a === void 0 ? void 0 : _a.replaceMaterializedReferences) &&
-                    (0, utils_2.shouldMaterialize)(qs.structDef.query.annotation)) {
-                    return stageWriter.addMaterializedQuery((0, malloy_types_1.getIdentifier)(qs.structDef), qs.structDef.query, (_b = qs.prepareResultOptions) === null || _b === void 0 ? void 0 : _b.materializedTablePrefix);
-                }
-                else {
-                    // Inline what loadQuery does, circularity workaround, finds the
-                    // the name of the last stage
-                    const query = qs.structDef.query;
-                    const turtleDef = {
-                        type: 'turtle',
-                        name: 'ignoreme',
-                        pipeline: query.pipeline,
-                        filterList: query.filterList,
-                    };
-                    const structRef = (_c = query.compositeResolvedSourceDef) !== null && _c !== void 0 ? _c : query.structRef;
-                    let sourceStruct;
-                    if (typeof structRef === 'string') {
-                        const struct = this.structRefToQueryStruct(structRef);
-                        if (!struct) {
-                            throw new Error(`Unexpected reference to an undefined source '${structRef}'`);
+                const { buildManifest, connectionDigests, strictPersist } = (_b = qs.prepareResultOptions) !== null && _b !== void 0 ? _b : {};
+                // Check manifest for this source
+                if (buildManifest && connectionDigests) {
+                    const connDigest = connectionDigests[qs.structDef.connection];
+                    if (connDigest) {
+                        // Compile with empty opts to get manifest-ignorant SQL for BuildID
+                        const fullRet = this.compileQueryToStages(qs.structDef.query, {}, undefined, false);
+                        const buildId = (0, source_def_utils_1.mkBuildID)(connDigest, fullRet.sql);
+                        const entry = buildManifest.buildEntries[buildId];
+                        if (entry) {
+                            // Found in manifest - use persisted table
+                            return this.parent.dialect.quoteTablePath(entry.tableName);
+                        }
+                        if (strictPersist) {
+                            throw new Error(`Persist source '${qs.structDef.sourceID}' not found in manifest (buildId: ${buildId})`);
                         }
-                        sourceStruct = struct;
-                    }
-                    else {
-                        sourceStruct = new query_node_1.QueryStruct(structRef, query.sourceArguments, { model: this.parent.getModel() }, qs.prepareResultOptions);
                     }
-                    const q = QueryQuery.makeQuery(turtleDef, sourceStruct, stageWriter, qs.parent !== undefined, // isJoinedSubquery
-                    this.structRefToQueryStruct);
-                    const ret = q.generateSQLFromPipeline(stageWriter);
-                    return ret.lastStageName;
                 }
+                // Not in manifest - compile normally
+                const ret = this.compileQueryToStages(qs.structDef.query, qs.prepareResultOptions, stageWriter, qs.parent !== undefined);
+                return ret.lastStageName;
             }
             default:
                 throw new Error(`Cannot create SQL StageWriter from '${(0, malloy_types_1.getIdentifier)(qs.structDef)}' type '${qs.structDef.type}`);
         }
     }
+    /**
+     * Compile a Query into SQL stages. Used by both query_source compilation
+     * and getCompiledSQL for interpolated sources.
+     *
+     * @param query The query to compile
+     * @param prepareResultOptions Options including manifest for substitution
+     * @param stageWriter If provided, stages are added to this writer and lastStageName is returned.
+     *                    If undefined, a new isolated writer is created and full SQL is returned.
+     * @param isJoinedSubquery Whether this is a joined subquery
+     * @returns { lastStageName, stageWriter, sql } - sql is only set if stageWriter was undefined
+     */
+    compileQueryToStages(query, prepareResultOptions, stageWriter, isJoinedSubquery) {
+        var _a;
+        const turtleDef = {
+            type: 'turtle',
+            name: 'ignoreme',
+            pipeline: query.pipeline,
+            filterList: query.filterList,
+        };
+        const structRef = (_a = query.compositeResolvedSourceDef) !== null && _a !== void 0 ? _a : query.structRef;
+        let sourceStruct;
+        if (typeof structRef === 'string') {
+            const struct = this.structRefToQueryStruct(structRef);
+            if (!struct) {
+                throw new Error(`Unexpected reference to an undefined source '${structRef}'`);
+            }
+            sourceStruct = struct;
+        }
+        else {
+            sourceStruct = new query_node_1.QueryStruct(structRef, query.sourceArguments, { model: this.parent.getModel() }, prepareResultOptions);
+        }
+        // Create isolated stageWriter if none provided
+        const isolated = stageWriter === undefined;
+        // When isPartialQuery is set (e.g., SQL interpolation), don't use CTEs
+        // for dialects that don't support them in subqueries
+        const noCTE = prepareResultOptions.isPartialQuery &&
+            !sourceStruct.dialect.supportsCTEinCoorelatedSubQueries;
+        const writer = stageWriter !== null && stageWriter !== void 0 ? stageWriter : new stage_writer_1.StageWriter(!noCTE, undefined);
+        const q = QueryQuery.makeQuery(turtleDef, sourceStruct, writer, isJoinedSubquery, this.structRefToQueryStruct);
+        const ret = q.generateSQLFromPipeline(writer);
+        return {
+            lastStageName: ret.lastStageName,
+            stageWriter: writer,
+            sql: isolated ? writer.generateSQLStages() : undefined,
+        };
+    }
     generateSQLJoinBlock(stageWriter, ji, depth) {
         var _a;
         let s = '';
@@ -1490,10 +1530,10 @@ class QueryQuery extends query_node_1.QueryField {
             };
             pipeline.shift();
             for (const transform of pipeline) {
-                const parent = this.parent.parent
+                const parentArg = this.parent.parent
                     ? { struct: this.parent.parent }
                     : { model: this.parent.getModel() };
-                const s = new query_node_1.QueryStruct(structDef, undefined, parent, this.parent.prepareResultOptions);
+                const s = new query_node_1.QueryStruct(structDef, undefined, parentArg, this.parent.prepareResultOptions);
                 const q = QueryQuery.makeQuery({ type: 'turtle', name: '~computeLastStage~', pipeline: [transform] }, s, stageWriter, this.isJoinedSubquery, this.structRefToQueryStruct);
                 q.prepare(stageWriter);
                 lastStageName = q.generateSQL(stageWriter);
@@ -1727,10 +1767,15 @@ class QueryQueryProject extends QueryQuery {
 }
 class QueryQueryRaw extends QueryQuery {
     generateSQL(stageWriter) {
+        var _a;
         if (this.parent.structDef.type !== 'sql_select') {
             throw new Error('Invalid struct for QueryQueryRaw, currently only supports SQL');
         }
-        return stageWriter.addStage(this.parent.structDef.selectStr);
+        return stageWriter.addStage((0, sql_compiled_1.getCompiledSQL)(this.parent.structDef, (_a = this.parent.prepareResultOptions) !== null && _a !== void 0 ? _a : {}, path => this.parent.dialect.quoteTablePath(path), (query, opts) => {
+            // Compile query to isolated SQL (not into parent's stageWriter)
+            const ret = this.compileQueryToStages(query, opts !== null && opts !== void 0 ? opts : {}, undefined, false);
+            return ret.sql;
+        }));
     }
     prepare() {
         // Do nothing!

package/dist/model/source_def_utils.d.ts

@@ -0,0 +1,50 @@
+/**
+ * SourceDef Utilities for Persistence
+ *
+ * Key invariant: sourceID is ONLY assigned in DefineSource when a source
+ * gets a name. Factory functions explicitly copy only the fields they need,
+ * never using spread, to prevent accidental propagation of sourceID/extends.
+ *
+ * The `extends` property is set by callers when processing extend blocks.
+ */
+import type { BuildID, FieldDef, ModelDef, PersistableSourceDef, Query, QuerySourceDef, SourceDef, SourceID, SourceRegistryEntry, SourceRegistryValue, SQLPhraseSegment, SQLSourceDef, TableSourceDef } from './malloy_types';
+export declare function mkSourceID(name: string, url: string | undefined): SourceID;
+/**
+ * Create a BuildID from connection digest and SQL.
+ * BuildID is a hash that uniquely identifies a build artifact.
+ */
+export declare function mkBuildID(connectionDigest: string, sql: string): BuildID;
+/**
+ * Create a QuerySourceDef from query compilation output.
+ * Explicitly copies SourceDefBase fields - no spread.
+ */
+export declare function mkQuerySourceDef(base: SourceDef, query: Query, name: string): QuerySourceDef;
+/**
+ * Create an SQLSourceDef from schema lookup result.
+ * Explicitly copies SourceDefBase fields - no spread.
+ */
+export declare function mkSQLSourceDef(base: SourceDef, selectStr: string, selectSegments?: SQLPhraseSegment[]): SQLSourceDef;
+/**
+ * Create a TableSourceDef. All fields specified, no base to copy from.
+ */
+export declare function mkTableSourceDef(name: string, connection: string, tablePath: string, dialect: string, fields: FieldDef[]): TableSourceDef;
+/**
+ * Resolve a sourceID to a SourceDef using the sourceRegistry.
+ *
+ * @param modelDef The model definition containing the registry
+ * @param sourceID The sourceID to resolve
+ * @returns The SourceDef if found, undefined otherwise
+ */
+export declare function resolveSourceID(modelDef: ModelDef, sourceID: SourceID): PersistableSourceDef | undefined;
+/**
+ * Add an entry to the sourceRegistry.
+ *
+ * @param registry The sourceRegistry to modify (from ModelDef or Document)
+ * @param sourceID The sourceID to register
+ * @param entry Either a SourceRegistryReference (for namespace sources) or a PersistableSourceDef (for hidden deps)
+ */
+export declare function registerSource(registry: Record<SourceID, SourceRegistryValue>, sourceID: SourceID, entry: SourceRegistryEntry): void;
+/**
+ * Check if a sourceID is already in the registry.
+ */
+export declare function hasSourceRegistryEntry(modelDef: ModelDef, sourceID: SourceID): boolean;