coalesce-transform-mcp 0.1.6 → 0.1.7

package/README.md

@@ -66,26 +66,30 @@ The server defaults to the US region. See [Environment Variables](#environment-v
 
 Only `COALESCE_ACCESS_TOKEN` is required. Everything else is optional.
 
+<!-- ENV_METADATA_CORE_TABLE_START -->
 | Variable | Description | Default |
-| -------- | ----------- | ------- |
+| -------- | -------- | -------- |
 | `COALESCE_ACCESS_TOKEN` | **Required.** Bearer token from the Coalesce Deploy tab. | — |
-| `COALESCE_BASE_URL` | Region-specific base URL. | `https://app.coalescesoftware.io` (US) |
-| `COALESCE_ORG_ID` | Fallback org ID for `cancel-run`. | — |
+| `COALESCE_BASE_URL` | Region-specific base URL. | `https://app.coalescesoftware.io (US)` |
+| `COALESCE_ORG_ID` | Fallback org ID for cancel-run. | — |
 | `COALESCE_REPO_PATH` | Local repo root for repo-backed tools and pipeline planning. | — |
 | `COALESCE_MCP_AUTO_CACHE_MAX_BYTES` | JSON size threshold before auto-caching to disk. | `32768` |
 | `COALESCE_MCP_MAX_REQUEST_BODY_BYTES` | Max outbound API request body size. | `524288` |
+<!-- ENV_METADATA_CORE_TABLE_END -->
 
 ### Snowflake (for run tools only)
 
 Required for `start-run`, `retry-run`, `run-and-wait`, and `retry-and-wait`. The server starts without them — they're validated when you first use a run tool.
 
+<!-- ENV_METADATA_SNOWFLAKE_TABLE_START -->
 | Variable | Required | Description |
-| -------- | -------- | ----------- |
+| -------- | -------- | -------- |
 | `SNOWFLAKE_USERNAME` | Yes | Snowflake account username |
 | `SNOWFLAKE_KEY_PAIR_KEY` | Yes | Path to PEM-encoded private key |
 | `SNOWFLAKE_KEY_PAIR_PASS` | No | Passphrase for encrypted keys |
 | `SNOWFLAKE_WAREHOUSE` | Yes | Snowflake compute warehouse |
 | `SNOWFLAKE_ROLE` | Yes | Snowflake user role |
+<!-- ENV_METADATA_SNOWFLAKE_TABLE_END -->
 
 To use optional variables, add them to your shell profile and pass them through in your MCP config. Here's a full example with everything enabled:
 

package/dist/mcp/pipelines.d.ts

@@ -1,3 +1,21 @@
 import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import type { CoalesceClient } from "../client.js";
+/**
+ * Generates a confirmation token for a pipeline plan to prevent bypass of user approval.
+ *
+ * The token is a SHA256 hash (truncated to 16 hex chars) of the canonicalized plan JSON.
+ * AI agents must provide this token when calling pipeline creation tools with `confirmed=true`,
+ * proving they received and can reference the exact plan that should have been presented to the user.
+ *
+ * **Important limitations:**
+ * - The token proves the agent received the correct plan (plan integrity)
+ * - It does NOT verify the agent presented the plan accurately to the user
+ * - An agent could theoretically show incomplete/misleading info but still provide the valid token
+ * - This is an acceptable tradeoff: the token prevents accidental bypass and honest mistakes,
+ *   while deliberate deception by a malicious agent is out of scope
+ *
+ * @param plan - The pipeline plan object to fingerprint
+ * @returns A 16-character hex token uniquely identifying this plan's content
+ */
+export declare function buildPlanConfirmationToken(plan: unknown): string;
 export declare function registerPipelineTools(server: McpServer, client: CoalesceClient): void;

package/dist/mcp/pipelines.js

@@ -4,24 +4,94 @@ import { existsSync, mkdirSync, readFileSync, readdirSync, unlinkSync, writeFile
 import { join } from "node:path";
 import { CACHE_DIR_NAME } from "../cache-dir.js";
 import { PipelinePlanSchema, planPipeline, } from "../services/pipelines/planning.js";
-import { createPipelineFromPlan, createPipelineFromSql, } from "../services/pipelines/execution.js";
+import { createPipelineFromPlan, } from "../services/pipelines/execution.js";
 import { NodeConfigInputSchema } from "../schemas/node-payloads.js";
 import { buildJsonToolResponse, handleToolError, READ_ONLY_ANNOTATIONS, WRITE_ANNOTATIONS, } from "../coalesce/types.js";
 import { isPlainObject } from "../utils.js";
-const REWRITTEN_SQL_ERROR_MESSAGE = "The sql parameter contains {{ ref() }} syntax, which means you rewrote the user's SQL. " +
-    "Pass the user's EXACT SQL unchanged — the planner resolves source references automatically. " +
-    "Do NOT replace table names with {{ ref() }}.";
-function buildPlanFingerprint(workspaceID, repoPath, workspaceNodeTypes, requestInputs) {
-    const input = [
-        `workspace:${workspaceID}`,
-        `repo:${repoPath ?? "none"}`,
-        `types:${[...workspaceNodeTypes].sort().join(",")}`,
-        `goal:${requestInputs?.goal ?? ""}`,
-        `sql:${requestInputs?.sql ?? ""}`,
-        `sources:${requestInputs?.sourceNodeIDs ? [...requestInputs.sourceNodeIDs].sort().join(",") : ""}`,
-        `targetType:${requestInputs?.targetNodeType ?? ""}`,
-    ].join("|");
-    return createHash("sha256").update(input).digest("hex").slice(0, 16);
+/**
+ * Recursively sorts JSON values to ensure deterministic serialization.
+ *
+ * Object keys are sorted alphabetically to guarantee that structurally
+ * identical objects produce identical JSON strings when serialized.
+ * This is essential for generating consistent confirmation tokens via
+ * hashing, where the same plan content must always yield the same hash
+ * regardless of key insertion order.
+ *
+ * @param value - The value to sort (arrays, objects, or primitives)
+ * @returns A deep copy with all object keys sorted alphabetically
+ */
+function sortJsonValue(value) {
+    if (Array.isArray(value)) {
+        return value.map(sortJsonValue);
+    }
+    if (!isPlainObject(value)) {
+        return value;
+    }
+    const sorted = {};
+    for (const key of Object.keys(value).sort()) {
+        const nested = sortJsonValue(value[key]);
+        if (nested !== undefined) {
+            sorted[key] = nested;
+        }
+    }
+    return sorted;
+}
+function normalizePlanFingerprintSelection(selection) {
+    if (!isPlainObject(selection)) {
+        return null;
+    }
+    return {
+        strategy: typeof selection.strategy === "string" ? selection.strategy : null,
+        selectedNodeType: typeof selection.selectedNodeType === "string" ? selection.selectedNodeType : null,
+        selectedDisplayName: typeof selection.selectedDisplayName === "string"
+            ? selection.selectedDisplayName
+            : null,
+        selectedShortName: typeof selection.selectedShortName === "string" ? selection.selectedShortName : null,
+        selectedFamily: typeof selection.selectedFamily === "string" ? selection.selectedFamily : null,
+        confidence: typeof selection.confidence === "string" ? selection.confidence : null,
+        autoExecutable: selection.autoExecutable === true,
+        repoPath: typeof selection.repoPath === "string" ? selection.repoPath : null,
+        resolvedRepoPath: typeof selection.resolvedRepoPath === "string" ? selection.resolvedRepoPath : null,
+        supportedNodeTypes: Array.isArray(selection.supportedNodeTypes)
+            ? selection.supportedNodeTypes.filter((value) => typeof value === "string")
+            : [],
+        consideredNodeTypes: Array.isArray(selection.consideredNodeTypes)
+            ? selection.consideredNodeTypes
+                .filter(isPlainObject)
+                .map((candidate) => ({
+                nodeType: typeof candidate.nodeType === "string" ? candidate.nodeType : null,
+                displayName: typeof candidate.displayName === "string" ? candidate.displayName : null,
+                shortName: typeof candidate.shortName === "string" ? candidate.shortName : null,
+                family: typeof candidate.family === "string" ? candidate.family : null,
+                usageCount: typeof candidate.usageCount === "number" ? candidate.usageCount : null,
+                workspaceUsageCount: typeof candidate.workspaceUsageCount === "number"
+                    ? candidate.workspaceUsageCount
+                    : null,
+                observedInWorkspace: candidate.observedInWorkspace === true,
+                autoExecutable: candidate.autoExecutable === true,
+                score: typeof candidate.score === "number" ? candidate.score : null,
+                reasons: Array.isArray(candidate.reasons)
+                    ? candidate.reasons.filter((value) => typeof value === "string")
+                    : [],
+            }))
+            : [],
+    };
+}
+function buildPlanFingerprint(workspaceID, selection, supportedNodeTypes, requestInputs) {
+    const payload = sortJsonValue({
+        workspaceID,
+        requestInputs: {
+            goal: requestInputs?.goal ?? null,
+            sql: requestInputs?.sql ?? null,
+            sourceNodeIDs: requestInputs?.sourceNodeIDs
+                ? [...requestInputs.sourceNodeIDs].sort()
+                : [],
+            targetNodeType: requestInputs?.targetNodeType ?? null,
+        },
+        supportedNodeTypes: [...supportedNodeTypes],
+        selection: normalizePlanFingerprintSelection(selection),
+    });
+    return createHash("sha256").update(JSON.stringify(payload)).digest("hex").slice(0, 16);
 }
 function getPlanSummaryDir() {
     return join(process.cwd(), CACHE_DIR_NAME, "plans");
@@ -69,9 +139,10 @@ function writePlanSummary(plan, fingerprint) {
         `Fingerprint: ${fingerprint}`,
         `Generated: ${new Date().toISOString()}`,
         ``,
-        `This file is automatically invalidated when the repo node types or workspace`,
-        `node types change. If you install new packages or commit new node type`,
-        `definitions, call plan-pipeline again to refresh.`,
+        `This file is automatically invalidated when repo-backed ranking inputs or`,
+        `workspace node types change enough to alter the planner's ranked guidance.`,
+        `If you install new packages, commit new node type definitions, or otherwise`,
+        `change ranking-relevant repo content, call plan-pipeline again to refresh.`,
         ``,
         `## Ranked Node Types`,
         ``,
@@ -161,11 +232,95 @@ function buildPlanSummaryForElicitation(plan) {
     lines.push("Confirm to proceed with node creation, or cancel to abort.");
     return lines.join("\n");
 }
+/**
+ * Generates a confirmation token for a pipeline plan to prevent bypass of user approval.
+ *
+ * The token is a SHA256 hash (truncated to 16 hex chars) of the canonicalized plan JSON.
+ * AI agents must provide this token when calling pipeline creation tools with `confirmed=true`,
+ * proving they received and can reference the exact plan that should have been presented to the user.
+ *
+ * **Important limitations:**
+ * - The token proves the agent received the correct plan (plan integrity)
+ * - It does NOT verify the agent presented the plan accurately to the user
+ * - An agent could theoretically show incomplete/misleading info but still provide the valid token
+ * - This is an acceptable tradeoff: the token prevents accidental bypass and honest mistakes,
+ *   while deliberate deception by a malicious agent is out of scope
+ *
+ * @param plan - The pipeline plan object to fingerprint
+ * @returns A 16-character hex token uniquely identifying this plan's content
+ */
+export function buildPlanConfirmationToken(plan) {
+    return createHash("sha256")
+        .update(JSON.stringify(sortJsonValue(plan)))
+        .digest("hex")
+        .slice(0, 16);
+}
+async function requirePipelineCreationApproval(server, toolName, plan, confirmed, confirmationToken, payload = {}) {
+    if (confirmed === true) {
+        // Verify the agent has the exact plan by comparing confirmation tokens.
+        // This prevents bypass where an agent sets confirmed=true without actually
+        // presenting the plan, but doesn't guarantee the agent presented it accurately.
+        const expected = buildPlanConfirmationToken(plan);
+        if (confirmationToken !== expected) {
+            return buildJsonToolResponse(toolName, {
+                created: false,
+                STOP_AND_CONFIRM: `STOP. The confirmationToken is missing or does not match the current plan. ` +
+                    `Present the pipeline plan to the user in a table showing each node name and nodeType. ` +
+                    `Ask for explicit approval BEFORE creating any nodes. Once the user approves, call ${toolName} again with confirmed=true and the confirmationToken from this response.`,
+                confirmationToken: expected,
+                ...payload,
+            });
+        }
+        return null;
+    }
+    const clientCapabilities = server.server.getClientCapabilities();
+    if (!clientCapabilities?.elicitation?.form) {
+        // Client does not support form elicitation — fall back to STOP_AND_CONFIRM convention
+        const token = buildPlanConfirmationToken(plan);
+        return buildJsonToolResponse(toolName, {
+            created: false,
+            confirmationToken: token,
+            STOP_AND_CONFIRM: `STOP. Present the pipeline plan to the user in a table showing each node name and nodeType. ` +
+                `Ask for explicit approval BEFORE creating any nodes. Once the user approves, call ${toolName} again with confirmed=true and confirmationToken="${token}".`,
+            ...payload,
+        });
+    }
+    const planSummary = buildPlanSummaryForElicitation(plan);
+    const elicitation = await server.server.elicitInput({
+        message: planSummary,
+        requestedSchema: {
+            type: "object",
+            properties: {
+                confirmed: {
+                    type: "boolean",
+                    title: "Create these pipeline nodes?",
+                    description: "Select true to proceed with node creation, false to cancel.",
+                },
+            },
+            required: ["confirmed"],
+        },
+    });
+    if (elicitation.action !== "accept" || elicitation.content?.confirmed !== true) {
+        const ACTION_LABELS = {
+            decline: "declined",
+            cancel: "cancelled",
+        };
+        return buildJsonToolResponse(toolName, {
+            created: false,
+            cancelled: true,
+            reason: elicitation.action === "accept"
+                ? "User declined pipeline creation."
+                : `Pipeline creation ${ACTION_LABELS[elicitation.action] ?? elicitation.action} by user.`,
+            ...payload,
+        });
+    }
+    return null;
+}
 export function registerPipelineTools(server, client) {
     server.tool("plan-pipeline", "Plan a Coalesce pipeline by discovering and ranking all available node types from the repo. ALWAYS call this before creating nodes to get the correct node type.\n\nThe planner scans the repo for all committed node type definitions, scores them against your use case, and returns ranked candidates. When available, it also returns a cached `planSummaryUri` MCP resource for the ranked node type summary so you can reuse that guidance throughout the pipeline without calling the planner again.\n\nIMPORTANT — DO NOT WRITE SQL: The `sql` parameter is ONLY for converting SQL that the USER provided (pasted or typed). If you are building a pipeline yourself, provide `goal` + `sourceNodeIDs` instead.\n\nPREREQUISITE: Before calling this tool, use list-workspace-nodes to discover available source/upstream nodes and their IDs in the workspace.\n\nPreferred approach: Provide `goal` AND `sourceNodeIDs`. The planner selects the best node type and scaffolds the pipeline. Without sourceNodeIDs, the planner returns clarification questions.\n\nUser-provided SQL: When a user pastes SQL, pass it in `sql`. The planner parses refs and column projections.\n\nConsult coalesce://context/node-type-corpus for node type patterns and metadata structures.", {
         workspaceID: z.string().describe("The workspace ID"),
         goal: z.string().optional().describe("Optional natural-language pipeline goal"),
-        sql: z.string().optional().describe("The user's EXACT SQL, copied verbatim. Do NOT rewrite table names, do NOT add {{ ref() }} syntax, do NOT modify it. Pass it exactly as the user provided it. If you are building a pipeline yourself, do NOT write SQL — use goal + sourceNodeIDs instead."),
+        sql: z.string().optional().describe("The user's EXACT SQL, copied verbatim. It may use raw table names or existing Coalesce {{ ref() }} syntax. Do NOT rewrite between SQL styles or modify the query. If you are building a pipeline yourself, do NOT write SQL — use goal + sourceNodeIDs instead."),
         targetName: z.string().optional().describe("Optional target node name override"),
         targetNodeType: z
             .string()
@@ -188,20 +343,12 @@ export function registerPipelineTools(server, client) {
             .describe("Optional upstream node IDs when planning from a non-SQL goal."),
     }, READ_ONLY_ANNOTATIONS, async (params) => {
         try {
-            // Reject SQL that the agent rewrote with {{ ref() }}
-            if (params.sql && /\{\{\s*ref\s*\(/.test(params.sql)) {
-                return handleToolError(new Error(REWRITTEN_SQL_ERROR_MESSAGE));
-            }
             const result = await planPipeline(client, params);
-            // Build fingerprint from workspace + repo + observed types
+            // Build fingerprint from the actual ranked node-type output used in the summary.
             const selection = isPlainObject(result.nodeTypeSelection) ? result.nodeTypeSelection : null;
-            const workspaceNodeTypes = Array.isArray(selection?.workspaceObservedNodeTypes)
-                ? selection.workspaceObservedNodeTypes
-                : [];
-            const repoPath = typeof selection?.resolvedRepoPath === "string"
-                ? selection.resolvedRepoPath
-                : null;
-            const fingerprint = buildPlanFingerprint(params.workspaceID, repoPath, workspaceNodeTypes, {
+            const fingerprint = buildPlanFingerprint(params.workspaceID, selection, Array.isArray(result.supportedNodeTypes)
+                ? result.supportedNodeTypes.filter((value) => typeof value === "string")
+                : [], {
                 goal: params.goal,
                 sql: params.sql,
                 sourceNodeIDs: params.sourceNodeIDs,
@@ -230,8 +377,8 @@ export function registerPipelineTools(server, client) {
                     planSummaryUri: summaryPath,
                     planCached: !!cached,
                     instruction: cached
-                        ? `Cached node type rankings found at planSummaryUri (fingerprint unchanged). Reference this resource for all subsequent node creations — no need to call plan-pipeline again unless you install new packages or commit new node type definitions.`
-                        : `Node type rankings saved to planSummaryUri. Reference this resource for all subsequent node creations in this pipeline. The cache auto-invalidates when repo or workspace node types change.`,
+                        ? `Cached node type rankings found at planSummaryUri (ranking fingerprint unchanged). Reference this resource for all subsequent node creations — no need to call plan-pipeline again unless repo-backed ranking inputs or workspace node types change enough to alter the planner's ranking.`
+                        : `Node type rankings saved to planSummaryUri. Reference this resource for all subsequent node creations in this pipeline. The cache auto-invalidates when repo-backed ranking inputs or workspace node types change enough to alter the planner's ranking.`,
                 }
                 : {
                     ...(selectedNodeType ? {
@@ -250,6 +397,14 @@ export function registerPipelineTools(server, client) {
     server.tool("create-pipeline-from-plan", "Create a Coalesce pipeline from a previously approved plan. Projection-capable node types execute by creating predecessor-based nodes first and then persisting the final full node body with set-workspace-node.", {
         workspaceID: z.string().describe("The workspace ID"),
         plan: PipelinePlanSchema.describe("The plan object returned by plan-pipeline."),
+        confirmed: z
+            .boolean()
+            .optional()
+            .describe("Set to true only after presenting the plan to the user and receiving explicit approval. Must be paired with the confirmationToken returned by the prior STOP_AND_CONFIRM response."),
+        confirmationToken: z
+            .string()
+            .optional()
+            .describe("The token returned in the STOP_AND_CONFIRM response. Required when confirmed=true to prove the plan was presented to the user."),
         dryRun: z
             .boolean()
             .optional()
@@ -257,42 +412,9 @@ export function registerPipelineTools(server, client) {
     }, WRITE_ANNOTATIONS, async (params) => {
         try {
             if (!params.dryRun) {
-                const planSummary = buildPlanSummaryForElicitation(params.plan);
-                try {
-                    const elicitation = await server.server.elicitInput({
-                        message: planSummary,
-                        requestedSchema: {
-                            type: "object",
-                            properties: {
-                                confirmed: {
-                                    type: "boolean",
-                                    title: "Create these pipeline nodes?",
-                                    description: "Select true to proceed with node creation, false to cancel.",
-                                },
-                            },
-                            required: ["confirmed"],
-                        },
-                    });
-                    if (elicitation.action !== "accept" || elicitation.content?.confirmed !== true) {
-                        return buildJsonToolResponse("create-pipeline-from-plan", {
-                            created: false,
-                            cancelled: true,
-                            reason: elicitation.action === "accept"
-                                ? "User declined pipeline creation."
-                                : `Pipeline creation ${elicitation.action}d by user.`,
-                        });
-                    }
-                }
-                catch (elicitError) {
-                    // Client does not support elicitation — fall back to STOP_AND_CONFIRM convention
-                    if (elicitError instanceof Error && elicitError.message.includes("does not support")) {
-                        return buildJsonToolResponse("create-pipeline-from-plan", {
-                            created: false,
-                            STOP_AND_CONFIRM: "STOP. Present the pipeline plan to the user in a table showing each node name and nodeType. Ask for explicit approval BEFORE creating any nodes. Once the user approves, call create-pipeline-from-plan again.",
-                            plan: params.plan,
-                        });
-                    }
-                    throw elicitError;
+                const approvalResponse = await requirePipelineCreationApproval(server, "create-pipeline-from-plan", params.plan, params.confirmed, params.confirmationToken, { plan: params.plan });
+                if (approvalResponse) {
+                    return approvalResponse;
                 }
             }
             const result = await createPipelineFromPlan(client, params);
@@ -302,9 +424,9 @@ export function registerPipelineTools(server, client) {
             return handleToolError(error);
         }
     });
-    server.tool("create-pipeline-from-sql", "Plan and create a Coalesce pipeline from user-provided SQL. Pass the user's EXACT SQL unchanged — do NOT rewrite it, do NOT replace table references with {{ ref() }}, do NOT modify the SQL in any way. The planner handles source resolution automatically.\n\nIf you are building a pipeline yourself, use declarative tools directly: create-workspace-node-from-predecessor → convert-join-to-aggregation → replace-workspace-node-columns.\n\nThis tool validates candidate node types against currently observed workspace nodes. If a selected type is not observed, the plan will include a warning asking the user to confirm installation in Coalesce.\n\nConsult coalesce://context/node-type-corpus for node type patterns and metadata structures.", {
+    server.tool("create-pipeline-from-sql", "Plan and create a Coalesce pipeline from user-provided SQL. Pass the user's EXACT SQL unchanged. The SQL may use raw table names or already contain Coalesce {{ ref() }} syntax if that is what the user provided. Do NOT rewrite between styles or otherwise modify the query. The planner resolves workspace sources automatically and generates a Coalesce-compatible joinCondition for the final node.\n\nIf you are building a pipeline yourself, use declarative tools directly: create-workspace-node-from-predecessor → convert-join-to-aggregation → replace-workspace-node-columns.\n\nThis tool validates candidate node types against currently observed workspace nodes. If a selected type is not observed, the plan will include a warning asking the user to confirm installation in Coalesce.\n\nConsult coalesce://context/node-type-corpus for node type patterns and metadata structures.", {
         workspaceID: z.string().describe("The workspace ID"),
-        sql: z.string().describe("The user's EXACT SQL, copied verbatim. Do NOT rewrite table names, do NOT add {{ ref() }} syntax, do NOT modify it in any way. Pass it exactly as the user provided it."),
+        sql: z.string().describe("The user's EXACT SQL, copied verbatim. It may use raw table names or existing Coalesce {{ ref() }} syntax. Do NOT rewrite between SQL styles or modify it in any way. Pass it exactly as the user provided it."),
         goal: z.string().optional().describe("Optional business goal or context for the SQL"),
         targetName: z.string().optional().describe("Optional target node name override"),
         targetNodeType: z
@@ -322,17 +444,45 @@ export function registerPipelineTools(server, client) {
             .string()
             .optional()
             .describe("Optional local committed Coalesce repo path for repo-first node-type ranking. Falls back to COALESCE_REPO_PATH when omitted."),
+        confirmed: z
+            .boolean()
+            .optional()
+            .describe("Set to true only after presenting the ready plan to the user and receiving explicit approval. Must be paired with the confirmationToken returned by the prior STOP_AND_CONFIRM response."),
+        confirmationToken: z
+            .string()
+            .optional()
+            .describe("The token returned in the STOP_AND_CONFIRM response. Required when confirmed=true to prove the plan was presented to the user."),
         dryRun: z
             .boolean()
             .optional()
             .describe("When true, return the generated plan without creating nodes."),
     }, WRITE_ANNOTATIONS, async (params) => {
         try {
-            // Reject SQL that the agent rewrote with {{ ref() }} — the user's original SQL won't contain these
-            if (/\{\{\s*ref\s*\(/.test(params.sql)) {
-                return handleToolError(new Error(REWRITTEN_SQL_ERROR_MESSAGE));
+            const plan = await planPipeline(client, params);
+            if (params.dryRun || plan.status !== "ready") {
+                return buildJsonToolResponse("create-pipeline-from-sql", {
+                    created: false,
+                    ...(params.dryRun ? { dryRun: true } : {}),
+                    plan,
+                    ...(plan.status !== "ready"
+                        ? {
+                            warning: "SQL was planned but still needs clarification before creation. Review openQuestions and warnings. Present the plan to the user and wait for approval.",
+                        }
+                        : {}),
+                });
             }
-            const result = await createPipelineFromSql(client, params);
+            const approvalResponse = await requirePipelineCreationApproval(server, "create-pipeline-from-sql", plan, params.confirmed, params.confirmationToken, { plan });
+            if (approvalResponse) {
+                return approvalResponse;
+            }
+            const execution = await createPipelineFromPlan(client, {
+                workspaceID: params.workspaceID,
+                plan,
+            });
+            const result = {
+                plan,
+                ...(isPlainObject(execution) ? execution : { execution }),
+            };
             return buildJsonToolResponse("create-pipeline-from-sql", result);
         }
         catch (error) {

package/dist/resources/index.js

@@ -172,9 +172,23 @@ function listCacheFilePaths(directory) {
     }
     return filePaths.sort();
 }
+function isCompleteSnapshotArtifact(filePath) {
+    if (filePath.includes(".tmp-")) {
+        return false;
+    }
+    if (filePath.endsWith(".ndjson")) {
+        return existsSync(filePath.replace(/\.ndjson$/, ".meta.json"));
+    }
+    if (filePath.endsWith(".meta.json")) {
+        return existsSync(filePath.replace(/\.meta\.json$/, ".ndjson"));
+    }
+    return true;
+}
 function listCacheResources(baseDir) {
     const cacheDir = getCacheDir(baseDir);
-    return listCacheFilePaths(cacheDir).flatMap((filePath) => {
+    return listCacheFilePaths(cacheDir)
+        .filter(isCompleteSnapshotArtifact)
+        .flatMap((filePath) => {
         const uri = buildCacheResourceUri(filePath, baseDir);
         if (!uri) {
             return [];
@@ -233,7 +247,7 @@ export function registerResources(server) {
         description: "Dynamic resources for cached tool responses, cache snapshots, and pipeline summaries.",
     }, async (resourceUri) => {
         const resolved = resolveCacheResourceUri(resourceUri.toString());
-        if (!resolved) {
+        if (!resolved || !isCompleteSnapshotArtifact(resolved.filePath)) {
             throw new Error(`Unknown cache resource: ${resourceUri.toString()}`);
         }
         try {

package/dist/services/cache/snapshots.js

@@ -1,4 +1,5 @@
-import { appendFileSync, mkdirSync, writeFileSync } from "node:fs";
+import { randomUUID } from "node:crypto";
+import { appendFileSync, mkdirSync, renameSync, rmSync, writeFileSync } from "node:fs";
 import { dirname, join } from "node:path";
 import { listEnvironmentNodes, listWorkspaceNodes } from "../../coalesce/api/nodes.js";
 import { listRuns } from "../../coalesce/api/runs.js";
@@ -62,50 +63,63 @@ export async function streamAllPaginatedToDisk(fetchPage, baseParams, params, op
     const orderBy = params.orderBy ?? "id";
     const orderByDirection = params.orderByDirection;
     const cachedAt = new Date().toISOString();
-    // Ensure parent directory exists
+    const tempSuffix = `.tmp-${process.pid}-${randomUUID()}`;
+    const tempNdjsonPath = `${ndjsonPath}${tempSuffix}`;
+    const tempMetaPath = `${metaPath}${tempSuffix}`;
+    // Ensure parent directories exist
     mkdirSync(dirname(ndjsonPath), { recursive: true });
-    // Write empty file to start (truncates any previous file)
-    writeFileSync(ndjsonPath, "", "utf8");
-    let totalItems = 0;
-    let next;
-    let isFirstPage = true;
-    let pageCount = 0;
-    while (isFirstPage || next) {
-        const response = await fetchPage({
-            ...baseParams,
-            limit: pageSize,
-            orderBy,
-            ...(orderByDirection ? { orderByDirection } : {}),
-            ...(next ? { startingFrom: next } : {}),
-        });
-        const page = parseCollectionPage(response);
-        pageCount += 1;
-        // Write each item as a single NDJSON line
-        for (const item of page.data) {
-            const transformed = itemTransform ? itemTransform(item) : item;
-            appendFileSync(ndjsonPath, JSON.stringify(transformed) + "\n", "utf8");
-            totalItems += 1;
-        }
-        if (page.next) {
-            if (seenCursors.has(page.next)) {
-                throw new Error(`Pagination repeated cursor ${page.next}`);
+    mkdirSync(dirname(metaPath), { recursive: true });
+    // Start with isolated temp files so failed streams never leave partial snapshots behind.
+    writeFileSync(tempNdjsonPath, "", "utf8");
+    try {
+        let totalItems = 0;
+        let next;
+        let isFirstPage = true;
+        let pageCount = 0;
+        while (isFirstPage || next) {
+            const response = await fetchPage({
+                ...baseParams,
+                limit: pageSize,
+                orderBy,
+                ...(orderByDirection ? { orderByDirection } : {}),
+                ...(next ? { startingFrom: next } : {}),
+            });
+            const page = parseCollectionPage(response);
+            pageCount += 1;
+            // Write each item as a single NDJSON line
+            for (const item of page.data) {
+                const transformed = itemTransform ? itemTransform(item) : item;
+                appendFileSync(tempNdjsonPath, JSON.stringify(transformed) + "\n", "utf8");
+                totalItems += 1;
             }
-            seenCursors.add(page.next);
+            if (page.next) {
+                if (seenCursors.has(page.next)) {
+                    throw new Error(`Pagination repeated cursor ${page.next}`);
+                }
+                seenCursors.add(page.next);
+            }
+            next = page.next;
+            isFirstPage = false;
         }
-        next = page.next;
-        isFirstPage = false;
+        // Write meta file only on successful completion, then promote both files into place.
+        const meta = {
+            totalItems,
+            pageCount,
+            pageSize,
+            orderBy,
+            ...(orderByDirection ? { orderByDirection } : {}),
+            cachedAt,
+        };
+        writeFileSync(tempMetaPath, JSON.stringify(meta, null, 2) + "\n", "utf8");
+        renameSync(tempNdjsonPath, ndjsonPath);
+        renameSync(tempMetaPath, metaPath);
+        return meta;
+    }
+    catch (error) {
+        rmSync(tempNdjsonPath, { force: true });
+        rmSync(tempMetaPath, { force: true });
+        throw error;
     }
-    // Write meta file only on successful completion
-    const meta = {
-        totalItems,
-        pageCount,
-        pageSize,
-        orderBy,
-        ...(orderByDirection ? { orderByDirection } : {}),
-        cachedAt,
-    };
-    writeFileSync(metaPath, JSON.stringify(meta, null, 2) + "\n", "utf8");
-    return meta;
 }
 function ensureDirectory(...parts) {
     const directory = join(...parts);

package/dist/services/pipelines/execution.d.ts

@@ -17,4 +17,5 @@ export declare function createPipelineFromSql(client: CoalesceClient, params: {
     schema?: string;
     repoPath?: string;
     dryRun?: boolean;
+    confirmed?: boolean;
 }): Promise<unknown>;