@dudousxd/nestjs-catalog 0.13.0 → 0.15.0

package/dist/catalog.pipeline.js

@@ -9,12 +9,13 @@
  * systems each believing they decide when a load runs.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.CATALOG_PIPELINE_STORE = exports.WORKFLOW_ISSUE_CODES = exports.WORKFLOW_EXECUTION_MODES = exports.WORKFLOW_STATUSES = exports.WORKFLOW_NODE_ID_PATTERN = exports.WORKFLOW_NODE_KINDS = exports.TRANSFORM_RUNNER = exports.TRANSFORM_LANGUAGES = exports.CONNECTOR_KINDS = void 0;
+exports.CATALOG_PIPELINE_STORE = exports.WORKFLOW_ISSUE_CODES = exports.WORKFLOW_CALL_CONTRACT = exports.WORKFLOW_EXECUTION_MODES = exports.WORKFLOW_STATUSES = exports.WORKFLOW_NODE_ID_PATTERN = exports.WORKFLOW_NODE_KINDS = exports.TRANSFORM_RUNNER = exports.TRANSFORM_LANGUAGES = exports.CONNECTOR_KINDS = void 0;
 exports.isConnectorKind = isConnectorKind;
 exports.isTransformLanguage = isTransformLanguage;
 exports.isWorkflowNodeKind = isWorkflowNodeKind;
 exports.isWorkflowStatus = isWorkflowStatus;
 exports.isWorkflowExecutionMode = isWorkflowExecutionMode;
+exports.readWorkflowCallOutput = readWorkflowCallOutput;
 exports.validateWorkflow = validateWorkflow;
 exports.workflowRunOrder = workflowRunOrder;
 exports.workflowGraphHash = workflowGraphHash;
@@ -92,12 +93,13 @@ exports.TRANSFORM_RUNNER = Symbol('TRANSFORM_RUNNER');
 /**
  * What a node can be.
  *
- * Three kinds, and they are exactly the three verbs the existing connector
- * runner already performs in sequence: fetch, transform, publish. Nothing here
- * is a kind this service cannot execute, which is the same rule
- * {@link CONNECTOR_KINDS} follows — a kind that exists in the type and throws
- * at run time is worse than one that is absent, because the first looks
- * supported in a palette.
+ * The first three are exactly the three verbs the existing connector runner
+ * already performs in sequence: fetch, transform, publish. The fourth hands a
+ * position in the graph to a durable workflow that already exists in the
+ * deployment. Nothing here is a kind this service cannot execute, which is the
+ * same rule {@link CONNECTOR_KINDS} follows — a kind that exists in the type
+ * and throws at run time is worse than one that is absent, because the first
+ * looks supported in a palette.
  *
  * The kinds that were considered and rejected, since a small vocabulary is only
  * defensible if the omissions are:
@@ -115,6 +117,15 @@ exports.TRANSFORM_RUNNER = Symbol('TRANSFORM_RUNNER');
  *   A `merge` kind would have had to carry a strategy field whose values the
  *   runner would have to implement one by one, and an unimplemented strategy in
  *   a dropdown is the failure this list exists to avoid.
+ * - **call a durable *step*** — the sibling of {@link WorkflowCallNode} that
+ *   somebody will eventually come looking for, and it cannot be built. A
+ *   durable step has no global identity: it is dispatched by a routing name
+ *   that a worker subscribes to, and within a run it is addressed by its `seq`
+ *   — a position in one workflow's history. There is no "run step X" entry
+ *   point on the engine to call, no lifecycle of its own to await, and nothing
+ *   to cancel. A workflow is the smallest thing that is addressable from
+ *   outside a run, which is why `call` names one and not a step. If a step is
+ *   what you want, the thing to call is a one-step workflow wrapping it.
  */
 exports.WORKFLOW_NODE_KINDS = [
     /** Reads records out of a system. The roots of the graph. */
@@ -123,6 +134,8 @@ exports.WORKFLOW_NODE_KINDS = [
     'transform',
     /** Writes into an object type and commits. Exactly one per workflow. */
     'sink',
+    /** Hands this position to an existing durable workflow. See {@link WorkflowCallNode}. */
+    'call',
 ];
 /** Same reason as {@link isConnectorKind}: one list, no second copy to drift. */
 function isWorkflowNodeKind(value) {
@@ -236,6 +249,56 @@ exports.WORKFLOW_EXECUTION_MODES = [
 function isWorkflowExecutionMode(value) {
     return exports.WORKFLOW_EXECUTION_MODES.some((mode) => mode === value);
 }
+/**
+ * The number in {@link WorkflowCallEnvelope.contract}.
+ *
+ * A version on the *shape the catalog sends*, separate from the version of the
+ * workflow being called, because the two change for different reasons and a
+ * callee written against one has to be able to say which. A callee that reads
+ * this and does not recognise it should refuse rather than guess — a failed
+ * child is a failed node with a name attached, and a guess is a load nobody can
+ * account for.
+ */
+exports.WORKFLOW_CALL_CONTRACT = 1;
+/**
+ * Read a child's return value as staged rows — or as nothing, or refuse it.
+ *
+ * Three answers rather than two, because a call has two legitimate purposes and
+ * they must not be confused with a bug:
+ *
+ * - `undefined` — the child returned nothing this graph can read rows from. A
+ *   perfectly ordinary outcome for a workflow called for its effect, and the
+ *   node reports zero rows, out loud, in its logs. It is not silently treated
+ *   as success-with-data: a full sink that then receives nothing refuses to
+ *   commit an empty snapshot, which is the loud end of this path.
+ * - a {@link WorkflowCallOutput} — the child staged rows for this node.
+ * - a **throw** — the child answered with `batches`/`rowCount` that are not
+ *   usable counts. Half a contract is a bug in the callee, and reading it as
+ *   "no rows" would turn that bug into a load that quietly came out short.
+ *
+ * The catalog cannot check any of this before the graph runs; there is no
+ * schema for a workflow's output anywhere in the durable contract, and no way
+ * to reach one if there were. So the check is here, at the one moment the
+ * answer exists, and it names what it saw.
+ */
+function readWorkflowCallOutput(value) {
+    if (typeof value !== 'object' || value === null)
+        return undefined;
+    const batches = Reflect.get(value, 'batches');
+    const rowCount = Reflect.get(value, 'rowCount');
+    if (batches === undefined && rowCount === undefined)
+        return undefined;
+    if (!isCount(batches) || !isCount(rowCount)) {
+        throw new Error(`It answered with batches=${describeCount(batches)} and rowCount=${describeCount(rowCount)}. A workflow that stages rows for a call node returns both as whole numbers of at least zero; returning one of them, or a value that is not a count, would leave this node to guess how much of the stage to read.`);
+    }
+    return { batches, rowCount };
+}
+function isCount(value) {
+    return typeof value === 'number' && Number.isInteger(value) && value >= 0;
+}
+function describeCount(value) {
+    return value === undefined ? 'nothing' : JSON.stringify(value);
+}
 /** Every way a graph can be refused. Exported so a canvas can key off the code. */
 exports.WORKFLOW_ISSUE_CODES = [
     'empty',
@@ -253,6 +316,7 @@ exports.WORKFLOW_ISSUE_CODES = [
     'unreachable',
     'dead-end',
     'transform-not-named',
+    'call-not-named',
 ];
 /**
  * Everything that makes a graph unrunnable, in one pure function.
@@ -291,10 +355,10 @@ function validateWorkflow(graph) {
     if (issues.length > 0)
         return issues;
     const { outgoing, incoming } = buildAdjacency(nodes, edges);
-    const sources = nodes.filter((node) => node.kind === 'source');
+    const originators = nodes.filter(originatesRows);
     const sinks = nodes.filter((node) => node.kind === 'sink');
     checkNodeWiring(nodes, incoming, outgoing, issues);
-    checkEndpoints(sources, sinks, issues);
+    checkEndpoints(originators, sinks, issues);
     const looped = findCycle(nodes, incoming, outgoing);
     if (looped) {
         issues.push({
@@ -306,9 +370,27 @@ function validateWorkflow(graph) {
         // only unreachable *because* of the cycle, which points at the wrong boxes.
         return issues;
     }
-    checkReachability(nodes, sources, sinks, incoming, outgoing, issues);
+    checkReachability(nodes, originators, sinks, incoming, outgoing, issues);
     return issues;
 }
+/**
+ * Whether a node can produce rows without anything wired into it.
+ *
+ * A source obviously can. A **call** node can too, and this is the one rule the
+ * `call` kind changes rather than extends: the workflow it hands off to may
+ * itself read from a system, so a graph of `call → sink` is a real pipeline and
+ * refusing it for having "no source" would be false. What is not weakened is
+ * that a graph still needs *something* that originates rows and *something*
+ * that commits them — a graph of transforms alone is still refused.
+ *
+ * Every call node counts, not only the ones with no inbound edge, and that is
+ * the conservative direction: it makes this the root set for reachability too,
+ * so a mid-graph call node cannot make everything downstream of it look
+ * unreachable when its own upstream is fine.
+ */
+function originatesRows(node) {
+    return node.kind === 'source' || node.kind === 'call';
+}
 /**
  * Index the nodes by id, reporting the ids that cannot be used as one.
  *
@@ -403,15 +485,48 @@ function checkNodeWiring(nodes, incoming, outgoing, issues) {
                 message: `Sink "${node.name}" (${node.id}) has an outbound edge. The sink commits the snapshot, so nothing can run after it.`,
             });
         }
-        if (node.kind === 'transform' && node.transformId.length === 0) {
-            issues.push({
-                code: 'transform-not-named',
-                nodeIds: [node.id],
-                message: `Transform node "${node.name}" (${node.id}) names no transform, so there is no code for it to run.`,
-            });
-        }
+        const unconfigured = nodeIsUnconfigured(node);
+        if (unconfigured)
+            issues.push(unconfigured);
     }
 }
+/**
+ * A node that names none of the thing it exists to run.
+ *
+ * The two kinds that point at something outside themselves — a transform at
+ * stored code, a call at a registered workflow — and both are reported the same
+ * way because they are the same mistake: a box on the canvas with nothing
+ * behind it, which looks finished and fails at run time.
+ */
+function nodeIsUnconfigured(node) {
+    if (node.kind === 'transform' && node.transformId.length === 0) {
+        return {
+            code: 'transform-not-named',
+            nodeIds: [node.id],
+            message: `Transform node "${node.name}" (${node.id}) names no transform, so there is no code for it to run.`,
+        };
+    }
+    if (node.kind === 'call')
+        return callIsUnnamed(node);
+    return undefined;
+}
+/**
+ * A call that names half of what it needs, or nothing at all.
+ *
+ * Both halves, checked separately, because the version is the one people leave
+ * blank: a call that named only a workflow would run whichever version happens
+ * to be registered on the day the load runs, which is the single thing this
+ * node is built not to do.
+ */
+function callIsUnnamed(node) {
+    if (node.callName.length > 0 && node.callVersion.length > 0)
+        return undefined;
+    return {
+        code: 'call-not-named',
+        nodeIds: [node.id],
+        message: `Call node "${node.name}" (${node.id}) does not name ${node.callName.length === 0 ? 'a workflow to call' : 'a version of the workflow it calls'}. A call pins a name and a version together — without the version it would run whichever one is registered when the load happens, and somebody else's deploy would change what this graph does.`,
+    };
+}
 /**
  * That the graph has both ends, and that no two sinks claim the same type.
  *
@@ -424,12 +539,12 @@ function checkNodeWiring(nodes, incoming, outgoing, issues) {
  * model does not pretend otherwise — but two snapshots of one type in one run
  * leaves nothing to say which of them the readers should get.
  */
-function checkEndpoints(sources, sinks, issues) {
-    if (sources.length === 0) {
+function checkEndpoints(originators, sinks, issues) {
+    if (originators.length === 0) {
         issues.push({
             code: 'no-source',
             nodeIds: [],
-            message: 'This workflow has no source node, so nothing would ever be read and the sink would commit an empty snapshot.',
+            message: 'This workflow has nothing that reads: no source node, and no call node handing off to a workflow that reads. Nothing would ever be fetched and the sink would commit an empty snapshot.',
         });
     }
     if (sinks.length === 0) {
@@ -512,12 +627,12 @@ function peelTails(leftover, outgoing) {
     }
     return leftover;
 }
-/** Nodes that no source reaches, and nodes that reach no sink. */
-function checkReachability(nodes, sources, sinks, incoming, outgoing, issues) {
-    const reachableFromSources = walk(sources.map((node) => node.id), outgoing);
+/** Nodes that nothing reading reaches, and nodes that reach no sink. */
+function checkReachability(nodes, originators, sinks, incoming, outgoing, issues) {
+    const reachableFromSources = walk(originators.map((node) => node.id), outgoing);
     const reachesASink = walk(sinks.map((sink) => sink.id), incoming);
     for (const node of nodes) {
-        if (sources.length > 0 && !reachableFromSources.has(node.id)) {
+        if (originators.length > 0 && !reachableFromSources.has(node.id)) {
             issues.push({
                 code: 'unreachable',
                 nodeIds: [node.id],
@@ -648,6 +763,22 @@ function canonicalNode(node) {
         // changed when it did not.
         return JSON.stringify([node.id, node.kind, node.transformId]);
     }
+    if (node.kind === 'call') {
+        // The called version IS in here, and that is the opposite choice from a
+        // transform above — for the reason the two differ. A transform's version is
+        // this catalog's own record of an edit somebody made here; a call's version
+        // is a different piece of code entirely. Repointing a node from `foo@1` to
+        // `foo@2` changes what the load does as surely as rewiring it does, so it
+        // is a new version of the graph and the run that used the old one stays
+        // identifiable.
+        return JSON.stringify([
+            node.id,
+            node.kind,
+            node.callName,
+            node.callVersion,
+            sortedEntries(node.config),
+        ]);
+    }
     return JSON.stringify([node.id, node.kind, node.targetType, node.mode ?? 'full']);
 }
 function sortedEntries(config) {
@@ -692,6 +823,18 @@ function isWorkflowNode(value) {
     if (kind === 'sink') {
         return typeof Reflect.get(value, 'targetType') === 'string';
     }
+    if (kind === 'call') {
+        // Both strings, and the config object, exactly as strictly as a source's:
+        // a stored call node missing its version is a node that would run whatever
+        // is registered today, which is the failure the pin exists to remove — and
+        // a graph that half-narrows is a load that runs nine nodes of ten.
+        const config = Reflect.get(value, 'config');
+        return (typeof Reflect.get(value, 'callName') === 'string' &&
+            typeof Reflect.get(value, 'callVersion') === 'string' &&
+            typeof config === 'object' &&
+            config !== null &&
+            !Array.isArray(config));
+    }
     const sourceKind = Reflect.get(value, 'sourceKind');
     const config = Reflect.get(value, 'config');
     return isConnectorKind(sourceKind) && typeof config === 'object' && config !== null;
@@ -716,7 +859,14 @@ function supportsWorkflows(store) {
         // save and not the transition would narrow cleanly here and then fail one
         // call later, in the middle of an apply that has already written types and
         // transforms into the target.
-        typeof store.publishWorkflow === 'function');
+        typeof store.publishWorkflow === 'function' &&
+        // Asked for by name for the same reason `publishWorkflow` is, and it earned
+        // the place the hard way: a schedule authored on a graph is worthless if the
+        // store cannot hold one, and a predicate that narrowed without checking
+        // would let the schedule route resolve, accept a cron, and throw
+        // "saveWorkflowSchedule is not a function" at the person who typed it. This
+        // is the surface a scheduling incident already came through once.
+        typeof store.saveWorkflowSchedule === 'function');
 }
 /**
  * Whether this store keeps a transform's history.

package/dist/catalog.query-cache.d.ts

@@ -21,5 +21,3 @@ export declare class QueryCache {
     clear(): void;
     get size(): number;
 }
-/** CSV, for the export button. */
-export declare function toCsv(result: CatalogQueryResult): string;

package/dist/catalog.query-cache.js

@@ -1,7 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.QueryCache = void 0;
-exports.toCsv = toCsv;
 const node_crypto_1 = require("node:crypto");
 /**
  * A small in-process cache for query results.
@@ -59,20 +58,3 @@ class QueryCache {
     }
 }
 exports.QueryCache = QueryCache;
-/** CSV, for the export button. */
-function toCsv(result) {
-    const escapeCell = (value) => {
-        if (value === null || value === undefined)
-            return '';
-        const text = typeof value === 'object' ? JSON.stringify(value) : String(value);
-        // Quote when the value could otherwise break the row apart. Doubling the
-        // quote is the CSV escape, not a backslash.
-        return /[",\n\r]/.test(text) ? `"${text.replace(/"/g, '""')}"` : text;
-    };
-    const lines = [result.columns.map(escapeCell).join(',')];
-    for (const row of result.rows) {
-        lines.push(result.columns.map((column) => escapeCell(row[column])).join(','));
-    }
-    // CRLF: Excel still treats a bare LF file as one long row in some locales.
-    return `${lines.join('\r\n')}\r\n`;
-}

package/dist/catalog.query.d.ts

@@ -52,6 +52,27 @@ export interface CatalogQueryRelation {
         type: string;
     }>;
 }
+/**
+ * The same statement, asked for a row at a time.
+ *
+ * Two fields differ from {@link CatalogQueryRequest} and both differ because the
+ * caller is an export rather than a screen.
+ *
+ * `maxRows` is optional here and **absent means every row**. A table is a page
+ * of a result and a cap is what makes the page; an export is the whole thing by
+ * definition, and a capped export is a prefix presented as a file.
+ *
+ * `timeoutMs` is optional for the matching reason. A statement backing a screen
+ * has to answer while somebody waits; an export of a large table legitimately
+ * runs for minutes, and the bound that matters for it is that no stage holds
+ * more than a row — which is what the stream is for. An implementation given no
+ * timeout should impose none of its own.
+ */
+export interface CatalogQueryStreamRequest {
+    sql: string;
+    maxRows?: number;
+    timeoutMs?: number;
+}
 export interface CatalogQueryStore {
     /**
      * Run a read-only statement.
@@ -63,8 +84,30 @@ export interface CatalogQueryStore {
     runQuery(request: CatalogQueryRequest): Promise<CatalogQueryResult>;
     /** What a query may select from. */
     queryRelations(): Promise<CatalogQueryRelation[]>;
+    /**
+     * The same read, handing rows over as the engine produces them.
+     *
+     * **Optional, and the option is the store's to take rather than the caller's.**
+     * A store fronting an API, or one on a driver that buffers its result set
+     * before resolving, cannot offer this honestly, and a shim that collected the
+     * rows and yielded them back would satisfy the type while doing the exact
+     * thing the type exists to avoid. So an absent `streamQuery` is a real answer,
+     * and {@link CatalogService.streamSavedQuery} falls back to the capped
+     * buffered read for it — see the note there about what that costs.
+     *
+     * The contract on an implementation is one sentence: **do not read ahead of
+     * the consumer.** Whatever the driver offers must pause when the consumer
+     * stops pulling, all the way to the socket, or the memory has only moved.
+     *
+     * Returned synchronously — an async generator, not a promise for one — so that
+     * a consumer's `for await` owns the resource from the first pull, and an
+     * abandoned iteration runs the generator's `finally`.
+     */
+    streamQuery?(request: CatalogQueryStreamRequest): AsyncIterable<Record<string, unknown>>;
 }
 export declare function isQueryStore(store: unknown): store is CatalogQueryStore;
+/** A query store that can hand rows over without materialising the result set. */
+export declare function isStreamingQueryStore(store: unknown): store is CatalogQueryStore & Required<Pick<CatalogQueryStore, 'streamQuery'>>;
 /**
  * A cheap sanity check on the shape of a statement.
  *

package/dist/catalog.query.js

@@ -8,12 +8,17 @@
  */
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.isQueryStore = isQueryStore;
+exports.isStreamingQueryStore = isStreamingQueryStore;
 exports.assertReadOnlyShape = assertReadOnlyShape;
 function isQueryStore(store) {
     return (typeof store === 'object' &&
         store !== null &&
         typeof Reflect.get(store, 'runQuery') === 'function');
 }
+/** A query store that can hand rows over without materialising the result set. */
+function isStreamingQueryStore(store) {
+    return isQueryStore(store) && typeof Reflect.get(store, 'streamQuery') === 'function';
+}
 /**
  * A cheap sanity check on the shape of a statement.
  *

package/dist/catalog.service.d.ts

@@ -22,6 +22,7 @@ export declare class CatalogService {
     private readonly workspace?;
     constructor(registry: CatalogRegistry, store: CatalogReadStore, options: CatalogModuleOptions, workspace?: CatalogWorkspaceStore | undefined);
     private readonly cache;
+    private readonly logger;
     /** The whole model, as data. */
     getSnapshot(): CatalogSnapshot;
     getType(name: string): CatalogObjectTypeDef | undefined;
@@ -63,6 +64,21 @@ export declare class CatalogService {
     readObjects(typeName: string, query: CatalogObjectQuery & {
         snapshot?: string;
     }): Promise<CatalogObjectPage>;
+    /** What the mounted store can push into a read predicate. Empty when it cannot. */
+    private filterOperators;
+    /**
+     * Every filter, or a refusal naming all of them at once.
+     *
+     * One message listing every problem rather than the first: somebody who built
+     * four filters and got two of them wrong should learn that in one round trip.
+     *
+     * The store is asked whether it can honour the operators before the read runs,
+     * which is what stops a store that does not filter from answering with an
+     * unfiltered page. That refusal is worth more than it costs — a screen only
+     * offers what `filterOperators` reported, so a caller reaching this branch is
+     * one that built the request itself.
+     */
+    private resolveFilters;
     /** Empty when the store keeps no history. */
     listSnapshots(typeName: string): Promise<SnapshotRef[]>;
     /** What the mounted store can do — the screens branch on this. */
@@ -117,6 +133,41 @@ export declare class CatalogService {
         savedQuery: SavedQuery;
         result: CatalogQueryResult;
     }>;
+    /**
+     * The same saved query, as rows arriving rather than a result.
+     *
+     * For the export route, and it differs from {@link runSavedQuery} in three
+     * ways that are all the same decision seen from different sides.
+     *
+     * **No cap when the store streams.** An export is the whole result by
+     * definition — that is what distinguishes it from the table it was exported
+     * from — so `maxQueryRows` is not applied. It cannot be: a capped export is a
+     * prefix handed over as a complete file, with nothing in the file to say so.
+     *
+     * **No cache, in either direction.** Nothing is read from it, because what it
+     * holds is a *capped* page and serving that would silently truncate; and
+     * nothing is written to it, because the thing being produced is the object the
+     * cache exists to avoid holding.
+     *
+     * **No timeout.** {@link CatalogModuleOptions.queryTimeoutMs} bounds a
+     * statement somebody is waiting on behind a screen. An export of a large table
+     * runs for as long as the table takes, and the bound that matters is that
+     * neither this process nor the driver holds more than a row — which the stream
+     * is what provides. A client that gives up closes the connection, and the
+     * consumer stopping its pull is what releases the read.
+     *
+     * **A store that cannot stream falls back to the capped buffered read**, and
+     * that is a real difference in what the same route returns depending on what
+     * is mounted underneath. It is the honest option: lifting the cap on a store
+     * that materialises its result set would not make the export complete, it
+     * would move the failure into the driver, where there is no cap to report. The
+     * truncation is logged, since a CSV has nowhere to carry the fact.
+     */
+    streamSavedQuery(id: string): Promise<{
+        savedQuery: SavedQuery;
+        columns?: string[];
+        rows: AsyncIterable<Record<string, unknown>>;
+    }>;
     listDashboards(): Promise<Dashboard[]>;
     getDashboard(id: string): Promise<Dashboard>;
     /**