@malloydata/malloy 0.0.428 → 0.0.429

package/dist/api/foundation/build_targets.d.ts

@@ -0,0 +1,50 @@
+import type { LogMessage } from '../../lang';
+import type { PersistNode } from '../../model/persist_utils';
+import type { Model, PersistSource } from './core';
+import type { ConnectionBuild } from './types';
+/**
+ * A walk node with its source already resolved.
+ *
+ * "Is this node a table, and which one" is decided here, once. Deciding it
+ * twice — once to collect connection digests and once to fold — means two
+ * predicates that have to agree, and a disagreement shows up as a digest
+ * missing for a connection that was right there.
+ */
+export interface ResolvedNode {
+    node: PersistNode;
+    /** The table this node builds, absent if it is a route or cannot resolve */
+    source: PersistSource | undefined;
+}
+/**
+ * Walk the model and resolve each node's source in one pass.
+ *
+ * Materializing is not laziness lost: the fold needs a connection digest per
+ * source and fetching those is async, so the walk has to be crossed twice.
+ */
+export declare function resolvePersistWalk(model: Model, tagParseLog: LogMessage[]): ResolvedNode[];
+/**
+ * Fold a walk into the artifacts it produces, grouped by connection and in
+ * dependency order.
+ *
+ * The walk emits persistable *sources*; a builder needs *tables*, and several
+ * sources routinely land on one (see `BuildTarget`). This merges them by
+ * BuildID, keeping every source that mapped onto a target in `target.sources`.
+ *
+ * One pass suffices because the walk is in dependency order: by the time a
+ * source arrives, everything it references has already been placed, so its
+ * edges can be written immediately. A source that is not itself a table
+ * contributes its children's targets to whoever referenced it, which is how an
+ * edge survives a route.
+ *
+ * Targets come back in dependency order, but no coarser schedule than that —
+ * a builder wanting concurrency reads `dependsOn` and starts each target when
+ * its own dependencies finish, which waits on strictly less than batching by
+ * depth would.
+ *
+ * Separate from `Runtime.getBuildTargets` because given the digests it is
+ * pure, and can be tested without a connection.
+ *
+ * @param walk A resolved walk from {@link resolvePersistWalk}, in order
+ * @param connectionDigests One digest per connection named by a persist source
+ */
+export declare function mkBuildTargets(walk: ResolvedNode[], connectionDigests: Record<string, string>): ConnectionBuild[];

package/dist/api/foundation/build_targets.js

@@ -0,0 +1,221 @@
+"use strict";
+/*
+ * Copyright Contributors to the Malloy project
+ * SPDX-License-Identifier: MIT
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.resolvePersistWalk = resolvePersistWalk;
+exports.mkBuildTargets = mkBuildTargets;
+/**
+ * Read a key this function's own construction guarantees is present.
+ *
+ * Every key here came out of `targets`, which is fully built before anything
+ * reads it back. Saying that once beats five bare `!`s that each look like a
+ * claim needing checking.
+ */
+function mustGet(map, key) {
+    const value = map.get(key);
+    if (value === undefined) {
+        throw new Error('build target ' + String(key) + ' vanished between passes');
+    }
+    return value;
+}
+/**
+ * Walk the model and resolve each node's source in one pass.
+ *
+ * Materializing is not laziness lost: the fold needs a connection digest per
+ * source and fetching those is async, so the walk has to be crossed twice.
+ */
+function resolvePersistWalk(model, tagParseLog) {
+    const resolved = [];
+    for (const node of model._walkPersistSources(tagParseLog)) {
+        resolved.push({
+            node,
+            source: node.persistent
+                ? model._persistSourceFor(node.sourceID)
+                : undefined,
+        });
+    }
+    return resolved;
+}
+/** A BuildID is a hex digest, so it cannot contain the separator. */
+function targetKey(connectionName, buildId) {
+    return `${connectionName}:${buildId}`;
+}
+/** Append without repeating. */
+function addUnique(into, seen, keys) {
+    for (const key of keys) {
+        if (!seen.has(key)) {
+            seen.add(key);
+            into.push(key);
+        }
+    }
+}
+/**
+ * Fold a walk into the artifacts it produces, grouped by connection and in
+ * dependency order.
+ *
+ * The walk emits persistable *sources*; a builder needs *tables*, and several
+ * sources routinely land on one (see `BuildTarget`). This merges them by
+ * BuildID, keeping every source that mapped onto a target in `target.sources`.
+ *
+ * One pass suffices because the walk is in dependency order: by the time a
+ * source arrives, everything it references has already been placed, so its
+ * edges can be written immediately. A source that is not itself a table
+ * contributes its children's targets to whoever referenced it, which is how an
+ * edge survives a route.
+ *
+ * Targets come back in dependency order, but no coarser schedule than that —
+ * a builder wanting concurrency reads `dependsOn` and starts each target when
+ * its own dependencies finish, which waits on strictly less than batching by
+ * depth would.
+ *
+ * Separate from `Runtime.getBuildTargets` because given the digests it is
+ * pure, and can be tested without a connection.
+ *
+ * @param walk A resolved walk from {@link resolvePersistWalk}, in order
+ * @param connectionDigests One digest per connection named by a persist source
+ */
+function mkBuildTargets(walk, connectionDigests) {
+    var _a;
+    const targets = new Map();
+    // What each source hands to whoever referenced it: its own target if it is
+    // one, otherwise whatever it was a route to.
+    const contributes = new Map();
+    for (const { node, source } of walk) {
+        const childKeys = [];
+        const seen = new Set();
+        for (const dep of node.dependsOn) {
+            addUnique(childKeys, seen, (_a = contributes.get(dep)) !== null && _a !== void 0 ? _a : []);
+        }
+        if (source === undefined) {
+            // A route, or a source whose definition could not be resolved. Either
+            // way it is not a table; pass its dependencies up.
+            contributes.set(node.sourceID, childKeys);
+            continue;
+        }
+        const connectionName = source.connectionName;
+        const digest = connectionDigests[connectionName];
+        if (digest === undefined) {
+            throw new Error(`No connection digest for '${connectionName}', needed to compute the ` +
+                `BuildID of '${node.sourceID}'. Supply a digest for every connection ` +
+                'named by a persist source.');
+        }
+        const sql = source.getSQL();
+        const buildId = source.makeBuildId(digest, sql);
+        const key = targetKey(connectionName, buildId);
+        let target = targets.get(key);
+        if (target === undefined) {
+            target = {
+                buildId,
+                connectionName,
+                sql,
+                sources: [],
+                dependsOn: new Set(),
+            };
+            targets.set(key, target);
+        }
+        const mine = new Set();
+        for (const childKey of childKeys) {
+            // A child on my own key is an extension of me — the same table twice,
+            // not a dependency.
+            if (childKey === key)
+                continue;
+            mine.add(childKey);
+        }
+        // Intersect, do not union. Union makes cycles: `source: alias is base
+        // extend { join_one: mid }` where `mid` reads `base` merges alias onto
+        // base's target carrying an edge to `mid`, while `mid` depends on `base`.
+        // Only one of those two edges is real.
+        //
+        // Intersecting keeps the real ones because of the six-paths list at
+        // `walkPersistentDependencies`: those are every way one source's SQL can
+        // incorporate another's — except `CompositeSourceDef.sources[]`, which the
+        // walk skips, and which is why composites and persistence are documented
+        // as incompatible. So a source that compiles to this target's SQL
+        // necessarily reaches everything that SQL inlines: every set here is a
+        // superset of the real dependencies, and what differs between them is only
+        // the join over-approximation, which is what the intersection removes.
+        //
+        // The safety therefore rests on that list staying complete, composites
+        // being its one known and deliberate hole. A path added to the IR and not
+        // to the walk — or composites being made to work with persistence without
+        // revisiting this — would break it, and the symptom here is quieter than a
+        // wrong plan: a table built from inlined SQL instead of reading the one
+        // below it. No error, just the expensive query persistence exists to
+        // avoid.
+        if (target.sources.length === 0) {
+            target.dependsOn = mine;
+        }
+        else {
+            for (const had of [...target.dependsOn]) {
+                if (!mine.has(had))
+                    target.dependsOn.delete(had);
+            }
+        }
+        target.sources.push(source);
+        contributes.set(node.sourceID, [key]);
+    }
+    // Every target exists before any `dependsOn` is filled in, so references can
+    // point both ways round a diamond.
+    const built = new Map();
+    for (const [key, partial] of targets) {
+        built.set(key, {
+            buildId: partial.buildId,
+            connectionName: partial.connectionName,
+            sql: partial.sql,
+            sources: partial.sources,
+            dependsOn: [],
+        });
+    }
+    for (const [key, partial] of targets) {
+        const target = mustGet(built, key);
+        for (const depKey of partial.dependsOn) {
+            target.dependsOn.push(mustGet(built, depKey));
+        }
+    }
+    // Emit dependencies before dependents. Depth-first with a memo: a target is
+    // appended once, after everything it reads.
+    const byConnection = new Map();
+    const placed = new Set();
+    const openKeys = new Set();
+    const place = (key) => {
+        if (placed.has(key))
+            return;
+        if (openKeys.has(key)) {
+            // A real edge cannot make a cycle: a target's SQL contains its
+            // dependencies' SQL inline, so two targets cannot each contain the
+            // other. But an edge here is not always a real one — a source merging
+            // onto a target can bring a join the target's SQL never mentions, and
+            // that is what intersecting the merged sources' edges above removes. If
+            // one gets through anyway, say so rather than hang.
+            const names = mustGet(targets, key)
+                .sources.map(s => s.sourceID)
+                .join(', ');
+            throw new Error(`Cycle in the persistence build graph at: ${names}`);
+        }
+        openKeys.add(key);
+        for (const depKey of mustGet(targets, key).dependsOn) {
+            place(depKey);
+        }
+        openKeys.delete(key);
+        placed.add(key);
+        const target = mustGet(built, key);
+        const forConnection = byConnection.get(target.connectionName);
+        if (forConnection === undefined) {
+            byConnection.set(target.connectionName, [target]);
+        }
+        else {
+            forConnection.push(target);
+        }
+    };
+    for (const key of targets.keys()) {
+        place(key);
+    }
+    const connections = [];
+    for (const [connectionName, connectionTargets] of byConnection) {
+        connections.push({ connectionName, targets: connectionTargets });
+    }
+    return connections;
+}
+//# sourceMappingURL=build_targets.js.map

package/dist/api/foundation/core.d.ts

@@ -3,6 +3,7 @@ import type { BuildID, CompiledQuery, ConstantExpr, DocumentLocation, BooleanFie
 import { QueryModel } from '../../model';
 import type { Dialect } from '../../dialect';
 import type { BuildGraph, CompileQueryOptions } from './types';
+import type { PersistWalk } from '../../model/persist_utils';
 import { Tag } from '@malloydata/malloy-tag';
 import type { MalloyTagParse, TagParseSpec } from './annotation';
 import { Annotations } from './annotation';
@@ -464,19 +465,44 @@ export declare class Model implements Taggable {
     get exportedExplores(): Explore[];
     get _modelDef(): ModelDef;
     /**
-     * Get the build plan for all #@ persist sources.
+     * Require the `experimental.persistence` compiler flag.
      *
-     * Walks through ALL queries and sources in the model, finding any persistent
-     * dependencies they reference (including hidden dependencies from imports).
+     * Read off the resolved model annotations (`.modelAnnotations`, the
+     * import/extend fold) rather than this model's own `##`, so the flag carries
+     * across extend.
+     */
+    private requirePersistence;
+    /**
+     * Walk every persistable source this model reaches, in dependency order.
+     *
+     * The roots are every source and query the model names, plus its unnamed
+     * queries. They share one walk, so a source several of them reach is visited
+     * once.
+     *
+     * This is the raw material {@link Runtime.getBuildTargets} folds into
+     * tables. Nothing here is keyed by artifact: that needs a BuildID, and a
+     * model cannot reach a connection to compute one.
+     */
+    _walkPersistSources(tagParseLog: LogMessage[]): PersistWalk;
+    /**
+     * The {@link PersistSource} for a sourceID, or undefined if this model
+     * cannot resolve it.
+     */
+    _persistSourceFor(sourceID: string): PersistSource | undefined;
+    /**
+     * Every `#@ persist` source in the model, as a graph keyed by sourceID.
      *
-     * Returns a BuildPlan containing:
-     * - `graphs`: Build graphs for root sources only (minimal build set)
-     * - `sources`: Map from sourceId to PersistSource (all persist sources)
+     * Walks all queries and sources, including dependencies that arrived through
+     * an import and are in no namespace here, and returns the *roots* — the
+     * sources nothing else depends on — each carrying its dependencies in
+     * `dependsOn`.
      *
-     * The minimal build set contains only "root" sources - those not depended
-     * on by any other persist source. Each root includes its transitive
-     * dependencies in the dependsOn field, preserving the tree structure
-     * for parallel building.
+     * @deprecated A builder wants tables, and this reports sources. Those are not
+     * one to one: several sources routinely map onto one table, so keying on
+     * sourceID leaves every builder to discover the mapping by hashing. Its roots
+     * are computed by sourceID too, so for a persisted source with an extension
+     * the root is the extension and the declaring source is never named. Use
+     * {@link Runtime.getBuildTargets}, which answers in tables.
      *
      * @return BuildPlan with graphs and sources map
      */
@@ -538,6 +564,24 @@ export declare class PersistSource implements Taggable {
     get annotations(): Annotations;
     /** The model annotations resolved for this source. */
     get modelAnnotations(): Annotations;
+    /**
+     * Where this source was declared: the URL of the model that declared it, and
+     * the range of the `source:` statement.
+     *
+     * This is the handle to report a build failure against. A name is ambiguous
+     * across models, a sourceID is a name and a URL glued together, and a BuildID
+     * is a hash — none of them answer "where do I go to fix this," and a location
+     * does.
+     *
+     * The URL is whatever the model was loaded from, which may be a scheme only
+     * the caller understands. Rendering it for a human is the builder's job for
+     * the same reason `name=` is: the core supplied no URLReader and has no idea
+     * what these URLs mean.
+     *
+     * Undefined for a source with no recorded position — one synthesized rather
+     * than written down.
+     */
+    get location(): DocumentLocation | undefined;
     /**
      * The connection name for this source.
      */

package/dist/api/foundation/core.js

@@ -1107,30 +1107,69 @@ class Model {
         return this.modelDef;
     }
     /**
-     * Get the build plan for all #@ persist sources.
+     * Require the `experimental.persistence` compiler flag.
      *
-     * Walks through ALL queries and sources in the model, finding any persistent
-     * dependencies they reference (including hidden dependencies from imports).
+     * Read off the resolved model annotations (`.modelAnnotations`, the
+     * import/extend fold) rather than this model's own `##`, so the flag carries
+     * across extend.
+     */
+    requirePersistence(api) {
+        const modelTag = this.modelAnnotations.parseAsTag('!').tag;
+        if (!modelTag.has('experimental', 'persistence')) {
+            throw new Error(`Model must have ##! experimental.persistence to use ${api}`);
+        }
+    }
+    /**
+     * Walk every persistable source this model reaches, in dependency order.
      *
-     * Returns a BuildPlan containing:
-     * - `graphs`: Build graphs for root sources only (minimal build set)
-     * - `sources`: Map from sourceId to PersistSource (all persist sources)
+     * The roots are every source and query the model names, plus its unnamed
+     * queries. They share one walk, so a source several of them reach is visited
+     * once.
      *
-     * The minimal build set contains only "root" sources - those not depended
-     * on by any other persist source. Each root includes its transitive
-     * dependencies in the dependsOn field, preserving the tree structure
-     * for parallel building.
+     * This is the raw material {@link Runtime.getBuildTargets} folds into
+     * tables. Nothing here is keyed by artifact: that needs a BuildID, and a
+     * model cannot reach a connection to compute one.
+     */
+    _walkPersistSources(tagParseLog) {
+        this.requirePersistence('_walkPersistSources()');
+        const roots = [];
+        for (const obj of Object.values(this.modelDef.contents)) {
+            if (obj.type === 'query' || (0, model_1.isSourceDef)(obj)) {
+                roots.push(obj);
+            }
+        }
+        roots.push(...this.modelDef.queryList);
+        return (0, persist_utils_1.walkPersistentDependencies)(roots, this.modelDef, tagParseLog);
+    }
+    /**
+     * The {@link PersistSource} for a sourceID, or undefined if this model
+     * cannot resolve it.
+     */
+    _persistSourceFor(sourceID) {
+        const sourceDef = (0, source_def_utils_1.resolveSourceID)(this.modelDef, sourceID);
+        return sourceDef
+            ? new PersistSource(new Explore(this.modelDef, sourceDef), this)
+            : undefined;
+    }
+    /**
+     * Every `#@ persist` source in the model, as a graph keyed by sourceID.
+     *
+     * Walks all queries and sources, including dependencies that arrived through
+     * an import and are in no namespace here, and returns the *roots* — the
+     * sources nothing else depends on — each carrying its dependencies in
+     * `dependsOn`.
+     *
+     * @deprecated A builder wants tables, and this reports sources. Those are not
+     * one to one: several sources routinely map onto one table, so keying on
+     * sourceID leaves every builder to discover the mapping by hashing. Its roots
+     * are computed by sourceID too, so for a persisted source with an extension
+     * the root is the extension and the declaring source is never named. Use
+     * {@link Runtime.getBuildTargets}, which answers in tables.
      *
      * @return BuildPlan with graphs and sources map
      */
     getBuildPlan() {
-        // Require experimental.persistence compiler flag. Read the resolved model
-        // annotations (`.modelAnnotations`, the import/extend fold) rather than this
-        // model's own `##` so the flag carries across extend.
-        const modelTag = this.modelAnnotations.parseAsTag('!').tag;
-        if (!modelTag.has('experimental', 'persistence')) {
-            throw new Error('Model must have ##! experimental.persistence to use getBuildPlan()');
-        }
+        this.requirePersistence('getBuildPlan()');
         const allDeps = [];
         const tagParseLog = [];
         // Walk all objects in the model to find persistent dependencies
@@ -1150,8 +1189,12 @@ class Model {
         const rootNodes = (0, persist_utils_1.minimalBuildGraph)(allDeps);
         // Build the sources map from all persistent sourceIDs encountered
         const sourcesMap = {};
+        const seen = new Set();
         const collectSources = (nodes) => {
             for (const node of nodes) {
+                if (seen.has(node))
+                    continue;
+                seen.add(node);
                 if (!(node.sourceID in sourcesMap)) {
                     const sourceDef = (0, source_def_utils_1.resolveSourceID)(this.modelDef, node.sourceID);
                     if (sourceDef) {
@@ -1257,6 +1300,26 @@ class PersistSource {
     get modelAnnotations() {
         return this.explore.modelAnnotations;
     }
+    /**
+     * Where this source was declared: the URL of the model that declared it, and
+     * the range of the `source:` statement.
+     *
+     * This is the handle to report a build failure against. A name is ambiguous
+     * across models, a sourceID is a name and a URL glued together, and a BuildID
+     * is a hash — none of them answer "where do I go to fix this," and a location
+     * does.
+     *
+     * The URL is whatever the model was loaded from, which may be a scheme only
+     * the caller understands. Rendering it for a human is the builder's job for
+     * the same reason `name=` is: the core supplied no URLReader and has no idea
+     * what these URLs mean.
+     *
+     * Undefined for a source with no recorded position — one synthesized rather
+     * than written down.
+     */
+    get location() {
+        return this.persistableDef.location;
+    }
     /**
      * The connection name for this source.
      */

package/dist/api/foundation/index.d.ts

@@ -1,4 +1,4 @@
-export type { Taggable, Loggable, ParseOptions, CompileOptions, CompileQueryOptions, BuildNode, BuildGraph, } from './types';
+export type { Taggable, Loggable, ParseOptions, CompileOptions, CompileQueryOptions, BuildNode, BuildGraph, BuildTarget, BuildTargets, ConnectionBuild, } from './types';
 export { EMPTY_BUILD_MANIFEST } from './types';
 export { EmptyURLReader, InMemoryURLReader, FixedConnectionMap, hashForInvalidationKey, isInternalURL, readURL, getInvalidationKey, } from './readers';
 export type { ModelCache, CachedModel } from './cache';

package/dist/api/foundation/runtime.d.ts

@@ -6,7 +6,7 @@ import type { Dialect } from '../../dialect';
 import type { RunSQLOptions } from '../../run_sql_options';
 import type { CacheManager } from './cache';
 import type { MalloyConfig } from './config';
-import type { ParseOptions, CompileOptions, CompileQueryOptions } from './types';
+import type { ParseOptions, CompileOptions, CompileQueryOptions, BuildTargets } from './types';
 import type { PreparedResult, Explore } from './core';
 import { Model, PreparedQuery } from './core';
 import type { DataRecord, Result } from './result';
@@ -285,6 +285,23 @@ export declare class Runtime {
      * @return A promise of a compiled `PreparedQuery`.
      */
     getQueryByName(model: ModelURL | ModelString, name: string, options?: ParseOptions & CompileOptions): Promise<PreparedQuery>;
+    /**
+     * What a builder has to build for this model, and in what order.
+     *
+     * This is the builder entry point. A model can only enumerate persistable
+     * *sources*, which is not the same set as the tables they produce — several
+     * sources routinely map onto one table, and only a connection digest can
+     * tell you which. That is why this lives on Runtime: it holds the
+     * connections, so it can finish the answer a model can only start.
+     *
+     * Connections are independent of one another; within one, targets come back
+     * in dependency order and each carries its own `dependsOn`. See
+     * {@link BuildTargets}.
+     *
+     * @param model A compiled model with `##! experimental.persistence`
+     * @return The targets to build, with any annotation parse messages
+     */
+    getBuildTargets(model: Model): Promise<BuildTargets>;
 }
 export declare class ConnectionRuntime extends Runtime {
     readonly rawConnections: Connection[];

package/dist/api/foundation/runtime.js

@@ -11,6 +11,7 @@ const validate_table_path_1 = require("../../connection/validate_table_path");
 const config_1 = require("./config");
 const row_data_utils_1 = require("../../api/row_data_utils");
 const readers_1 = require("./readers");
+const build_targets_1 = require("./build_targets");
 const core_1 = require("./core");
 const compile_1 = require("./compile");
 // =============================================================================
@@ -480,6 +481,40 @@ class Runtime {
     getQueryByName(model, name, options) {
         return this.loadQueryByName(model, name, options).getPreparedQuery();
     }
+    /**
+     * What a builder has to build for this model, and in what order.
+     *
+     * This is the builder entry point. A model can only enumerate persistable
+     * *sources*, which is not the same set as the tables they produce — several
+     * sources routinely map onto one table, and only a connection digest can
+     * tell you which. That is why this lives on Runtime: it holds the
+     * connections, so it can finish the answer a model can only start.
+     *
+     * Connections are independent of one another; within one, targets come back
+     * in dependency order and each carries its own `dependsOn`. See
+     * {@link BuildTargets}.
+     *
+     * @param model A compiled model with `##! experimental.persistence`
+     * @return The targets to build, with any annotation parse messages
+     */
+    async getBuildTargets(model) {
+        const tagParseLog = [];
+        const walk = (0, build_targets_1.resolvePersistWalk)(model, tagParseLog);
+        const connectionDigests = (0, model_1.mkSafeRecord)();
+        for (const { source } of walk) {
+            if (source === undefined)
+                continue;
+            const connectionName = source.connectionName;
+            if (!(connectionName in connectionDigests)) {
+                const connection = await this.connections.lookupConnection(connectionName);
+                connectionDigests[connectionName] = connection.getDigest();
+            }
+        }
+        return {
+            connections: (0, build_targets_1.mkBuildTargets)(walk, connectionDigests),
+            tagParseLog,
+        };
+    }
 }
 exports.Runtime = Runtime;
 // =============================================================================