@malloydata/malloy 0.0.428 → 0.0.430

package/dist/model/persist_utils.js

@@ -5,6 +5,7 @@
  */
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.checkPersistAnnotation = checkPersistAnnotation;
+exports.walkPersistentDependencies = walkPersistentDependencies;
 exports.findPersistentDependencies = findPersistentDependencies;
 exports.minimalBuildGraph = minimalBuildGraph;
 const malloy_types_1 = require("./malloy_types");
@@ -50,15 +51,26 @@ function isPersistent(sourceID, modelDef, tagParseLog) {
     return value.persist;
 }
 /**
- * Find persistent dependencies for a source or query, returning a nested DAG.
+ * Walk everything a source or query references, and emit the sources that
+ * matter to persistence — in dependency order, each one after everything it
+ * depends on.
  *
- * Walks the full dependency tree but only includes persistent sources in the
- * result. Non-persistent sources are "flattened out" - their persistent
- * dependencies bubble up to become direct dependencies of the caller.
+ * A source is emitted if it is persistent — something to build — or if it
+ * is the *route* to something persistent. Everything else is dropped; a
+ * source with nothing persistent beneath it is of no interest to anybody.
  *
- * Example: source_c (persist) -> source_b (NOT persist) -> source_a (persist)
- * Returns: [{sourceID: source_a, dependsOn: []}]
- * (source_b is flattened out, source_a becomes direct dependency)
+ * One rule decides both halves, applied on the way back up:
+ *
+ *     keep me if I am persistent, or if any child survived
+ *
+ * The second clause needs no lookahead. A child only survived under this same
+ * rule, so a surviving child *is* the proof that something persistent lies
+ * below. For `a(persist) → b → c(persist) → d`, d has nothing beneath it
+ * and is dropped, c is kept for being persistent, b is kept for leading to c.
+ *
+ * Because a node arrives only after everything it depends on, a consumer can
+ * fold the stream in a single pass with no second lookup — whether it wants
+ * every source touched, or only the ones that are tables.
  *
  * ## The 6 Dependency Paths in the IR
  *
@@ -75,58 +87,91 @@ function isPersistent(sourceID, modelDef, tagParseLog) {
  * Note: CompositeSourceDef.sources[] is ignored - composite sources and
  * persistence may be incompatible features.
  *
- * @param root The source or query to find dependencies for
+ * Joins are followed whether or not a query activates them, so what comes back
+ * is what a source *could* depend on. An unused join is a dependency the SQL
+ * will not contain — an over-approximation, which costs ordering freedom and
+ * never correctness.
+ *
+ * @param roots The sources and queries to walk, sharing one memo
  * @param modelDef The model definition containing the source registry
- * @returns Array of BuildNode representing the persistent dependency DAG
+ * @param tagParseLog Collects errors from parsing `#@` annotations
  */
-function findPersistentDependencies(root, modelDef, tagParseLog = []) {
-    const visited = new Set();
-    function processSourceID(sourceID) {
-        if (visited.has(sourceID)) {
-            return [];
+function* walkPersistentDependencies(roots, modelDef, tagParseLog = []) {
+    // Memoized, not merely visited: a source reached a second time reports the
+    // same answer it gave the first time, so every dependent records the edge
+    // rather than only the first one to arrive. The memo spans every root, so a
+    // subtree two model objects both reach is walked once and yielded once.
+    const done = new Map();
+    const openIDs = new Set();
+    function* processSourceID(sourceID) {
+        const memo = done.get(sourceID);
+        if (memo !== undefined) {
+            return memo;
         }
-        visited.add(sourceID);
-        const sourceDef = (0, source_def_utils_1.resolveSourceID)(modelDef, sourceID);
-        if (!sourceDef) {
+        // A source cannot reach itself, but a malformed registry could; stop rather
+        // than recur forever.
+        if (openIDs.has(sourceID)) {
             return [];
         }
-        const childDeps = processSourceDef(sourceDef);
-        const persistent = isPersistent(sourceID, modelDef, tagParseLog);
-        if (persistent) {
-            return [{ sourceID, dependsOn: childDeps }];
+        openIDs.add(sourceID);
+        const sourceDef = (0, source_def_utils_1.resolveSourceID)(modelDef, sourceID);
+        let result = [];
+        if (sourceDef) {
+            const dependsOn = yield* processSourceDef(sourceDef);
+            const persistent = isPersistent(sourceID, modelDef, tagParseLog);
+            // Kept for being a table, or for being the road to one. Yielded after
+            // its dependencies, which is what lets a consumer fold in one pass.
+            if (persistent || dependsOn.length > 0) {
+                yield { sourceID, persistent, dependsOn };
+                result = [sourceID];
+            }
         }
-        else {
-            return childDeps;
+        openIDs.delete(sourceID);
+        done.set(sourceID, result);
+        return result;
+    }
+    /** Concatenate what several paths found, without repeating a source. */
+    function* gather(parts) {
+        const found = [];
+        const seen = new Set();
+        for (const part of parts) {
+            for (const id of yield* part) {
+                if (!seen.has(id)) {
+                    seen.add(id);
+                    found.push(id);
+                }
+            }
         }
+        return found;
     }
-    function processSourceDef(source) {
-        const results = [];
+    function* processSourceDef(source) {
+        const parts = [];
         // Path 4: PersistableSourceDef.extends
         if ((0, malloy_types_1.isPersistableSourceDef)(source) && source.extends) {
-            results.push(...processSourceID(source.extends));
+            parts.push(processSourceID(source.extends));
         }
         // Path 6: QuerySourceDef.query
         if (source.type === 'query_source') {
-            results.push(...processQuery(source.query));
+            parts.push(processQuery(source.query));
         }
         // Path 5: SQLSourceDef.selectSegments[]
         if (source.type === 'sql_select' && source.selectSegments) {
             for (const segment of source.selectSegments) {
-                results.push(...processSQLSegment(segment));
+                parts.push(processSQLSegment(segment));
             }
         }
         // Path 3: SourceDef.fields[] - joins defined on the source
         for (const field of source.fields) {
             if ((0, malloy_types_1.isJoined)(field) && (0, malloy_types_1.isSourceDef)(field)) {
-                results.push(...processJoinedSource(field));
+                parts.push(processJoinedSource(field));
             }
         }
-        return results;
+        return yield* gather(parts);
     }
-    function processQuery(query) {
-        const results = [];
+    function* processQuery(query) {
+        const parts = [];
         // Path 1: Query.structRef
-        results.push(...processStructRef(query.structRef));
+        parts.push(processStructRef(query.structRef));
         // Path 2: Query.pipeline[].extendSource[]
         for (const segment of query.pipeline) {
             if (segment.type === 'reduce' ||
@@ -136,80 +181,140 @@ function findPersistentDependencies(root, modelDef, tagParseLog = []) {
                 if (querySegment.extendSource) {
                     for (const field of querySegment.extendSource) {
                         if ((0, malloy_types_1.isJoined)(field) && (0, malloy_types_1.isSourceDef)(field)) {
-                            results.push(...processJoinedSource(field));
+                            parts.push(processJoinedSource(field));
                         }
                     }
                 }
             }
         }
-        return results;
+        return yield* gather(parts);
     }
-    function processJoinedSource(source) {
+    function* processJoinedSource(source) {
         // If it has an sourceID, go through the registry
         if ((0, malloy_types_1.isPersistableSourceDef)(source) && source.sourceID) {
-            return processSourceID(source.sourceID);
+            return yield* processSourceID(source.sourceID);
         }
         // Otherwise walk through it transparently
-        return processSourceDef(source);
+        return yield* processSourceDef(source);
     }
-    function processStructRef(ref) {
+    function* processStructRef(ref) {
         if (typeof ref === 'string') {
             const source = resolveSource(modelDef, ref);
             if (!source)
                 return [];
             if ((0, malloy_types_1.isPersistableSourceDef)(source) && source.sourceID) {
-                return processSourceID(source.sourceID);
+                return yield* processSourceID(source.sourceID);
             }
-            return processSourceDef(source);
+            return yield* processSourceDef(source);
         }
         else if ((0, malloy_types_1.isSourceDef)(ref)) {
             if ((0, malloy_types_1.isPersistableSourceDef)(ref) && ref.sourceID) {
-                return processSourceID(ref.sourceID);
+                return yield* processSourceID(ref.sourceID);
             }
-            return processSourceDef(ref);
+            return yield* processSourceDef(ref);
         }
         return [];
     }
-    function processSQLSegment(segment) {
+    function* processSQLSegment(segment) {
         if ((0, malloy_types_1.isSegmentSQL)(segment)) {
             return [];
         }
         else if ((0, malloy_types_1.isSegmentSource)(segment)) {
             if ((0, malloy_types_1.isPersistableSourceDef)(segment) && segment.sourceID) {
-                return processSourceID(segment.sourceID);
+                return yield* processSourceID(segment.sourceID);
             }
-            return processSourceDef(segment);
+            return yield* processSourceDef(segment);
         }
         else {
             // It's a Query
-            return processQuery(segment);
+            return yield* processQuery(segment);
         }
     }
-    // Entry point: handle both SourceDef and Query
-    // Query has required 'structRef', SourceDef does not
-    if ('structRef' in root) {
-        return processQuery(root);
-    }
-    else {
-        // If the root source itself is persistable and has a sourceID, process it through
-        // processSourceID so it gets included in the result if persistent
+    // Entry point: a Query has a required 'structRef', a SourceDef does not.
+    return yield* gather(roots.map(root => {
+        if ('structRef' in root) {
+            return processQuery(root);
+        }
+        // A root that is persistable and named goes through processSourceID so
+        // it appears in its own right if it is persistent.
         if ((0, malloy_types_1.isPersistableSourceDef)(root) && root.sourceID) {
             return processSourceID(root.sourceID);
         }
         return processSourceDef(root);
+    }));
+}
+/**
+ * The persistent sources a source or query depends on, as a nested DAG.
+ *
+ * A source that is not itself persistent never appears; its persistent
+ * dependencies become direct dependencies of whoever referenced it. So
+ * `c (persist) → b → a (persist)` yields `[{sourceID: a, dependsOn: []}]`.
+ *
+ * @deprecated The nested shape exists for `Model.getBuildPlan()`, which is on
+ * its way out. Consume {@link walkPersistentDependencies} directly.
+ */
+function findPersistentDependencies(root, modelDef, tagParseLog = []) {
+    var _a, _b;
+    // What each source contributes to whoever referenced it: itself if it is a
+    // table, otherwise whatever it was a route to. Post-order means every
+    // dependency is already in the map by the time its dependent arrives.
+    const contributes = new Map();
+    const walk = walkPersistentDependencies([root], modelDef, tagParseLog);
+    let step = walk.next();
+    while (!step.done) {
+        const node = step.value;
+        // Two of a source's references can lead to one table — a join and an
+        // extend of the same thing, say. That is one dependency, listed once.
+        const dependsOn = [];
+        const seen = new Set();
+        for (const id of node.dependsOn) {
+            for (const dep of (_a = contributes.get(id)) !== null && _a !== void 0 ? _a : []) {
+                if (!seen.has(dep)) {
+                    seen.add(dep);
+                    dependsOn.push(dep);
+                }
+            }
+        }
+        contributes.set(node.sourceID, node.persistent ? [{ sourceID: node.sourceID, dependsOn }] : dependsOn);
+        step = walk.next();
     }
+    // The generator's return value is what the root itself reached — the only
+    // thing a `for...of` would throw away.
+    //
+    // Deduplicated because two routes can land on one table: a source reached
+    // through two different `#@ -persist` wrappers contributes the same node
+    // twice, and this function has always returned each node once.
+    const roots = [];
+    const rootSeen = new Set();
+    for (const id of step.value) {
+        for (const node of (_b = contributes.get(id)) !== null && _b !== void 0 ? _b : []) {
+            if (!rootSeen.has(node)) {
+                rootSeen.add(node);
+                roots.push(node);
+            }
+        }
+    }
+    return roots;
 }
 /**
  * Collect all sourceIDs from a BuildNode forest (for analysis only).
+ *
+ * A forest shares nodes — one memoized node is reachable from every dependent
+ * that reads it — so the walk tracks which nodes it has already descended into.
  */
 function collectAllSourceIDs(nodes) {
     const result = new Set();
-    for (const node of nodes) {
+    const seen = new Set();
+    function visit(node) {
+        if (seen.has(node))
+            return;
+        seen.add(node);
         result.add(node.sourceID);
-        for (const id of collectAllSourceIDs(node.dependsOn)) {
-            result.add(id);
-        }
+        for (const dep of node.dependsOn)
+            visit(dep);
     }
+    for (const node of nodes)
+        visit(node);
     return result;
 }
 /**
@@ -217,14 +322,18 @@ function collectAllSourceIDs(nodes) {
  */
 function collectAllDependedOn(nodes) {
     const result = new Set();
-    for (const node of nodes) {
+    const seen = new Set();
+    function visit(node) {
+        if (seen.has(node))
+            return;
+        seen.add(node);
         for (const dep of node.dependsOn) {
             result.add(dep.sourceID);
-        }
-        for (const id of collectAllDependedOn(node.dependsOn)) {
-            result.add(id);
+            visit(dep);
         }
     }
+    for (const node of nodes)
+        visit(node);
     return result;
 }
 /**

package/dist/version.d.ts

@@ -1 +1 @@
-export declare const MALLOY_VERSION = "0.0.428";
+export declare const MALLOY_VERSION = "0.0.430";

package/dist/version.js

@@ -2,5 +2,5 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.MALLOY_VERSION = void 0;
 // generated with 'generate-version-file' script; do not edit manually
-exports.MALLOY_VERSION = '0.0.428';
+exports.MALLOY_VERSION = '0.0.430';
 //# sourceMappingURL=version.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@malloydata/malloy",
-  "version": "0.0.428",
+  "version": "0.0.430",
   "license": "MIT",
   "exports": {
     ".": "./dist/index.js",
@@ -51,9 +51,9 @@
     "generate-version-file": "VERSION=$(npm pkg get version --workspaces=false | tr -d \\\")\necho \"// generated with 'generate-version-file' script; do not edit manually\\nexport const MALLOY_VERSION = '$VERSION';\" > src/version.ts"
   },
   "dependencies": {
-    "@malloydata/malloy-filter": "0.0.428",
-    "@malloydata/malloy-interfaces": "0.0.428",
-    "@malloydata/malloy-tag": "0.0.428",
+    "@malloydata/malloy-filter": "0.0.430",
+    "@malloydata/malloy-interfaces": "0.0.430",
+    "@malloydata/malloy-tag": "0.0.430",
     "@noble/hashes": "^1.8.0",
     "antlr4ts": "^0.5.0-alpha.4",
     "assert": "^2.0.0",