coalesce-transform-mcp 0.1.0

package/dist/prompts/index.js

@@ -0,0 +1,58 @@
+export function registerPrompts(server) {
+    server.registerPrompt("coalesce-start-here", {
+        title: "Coalesce Start Here",
+        description: "Discover projects, workspaces, environments, jobs, and node IDs before calling mutating tools.",
+    }, async () => ({
+        messages: [
+            {
+                role: "user",
+                content: {
+                    type: "text",
+                    text: "Start with discovery before mutation. Use list-projects(includeWorkspaces=true) or get-project(includeWorkspaces=true) to resolve workspace IDs, list-environments for environment IDs, list-jobs for job IDs, and list-workspace-nodes or get-workspace-node before editing node bodies. Read coalesce://context/id-discovery and coalesce://context/tool-usage for the detailed lookup patterns.",
+                },
+            },
+        ],
+    }));
+    server.registerPrompt("safe-pipeline-planning", {
+        title: "Safe Pipeline Planning",
+        description: "Planner-first pipeline workflow, including review and approval before any workspace mutation.",
+    }, async () => ({
+        messages: [
+            {
+                role: "user",
+                content: {
+                    type: "text",
+                    text: "Always call plan-pipeline before create-pipeline-from-plan or create-pipeline-from-sql. If the planner returns status needs_clarification, stop and address openQuestions and warnings first. If it returns status ready, present the planned nodes, exact nodeType values, transforms, and filters to the user and wait for explicit approval before creating anything. Review coalesce://context/pipeline-workflows and coalesce://context/tool-usage for the mandatory planner-first sequence.",
+                },
+            },
+        ],
+    }));
+    server.registerPrompt("run-operations-guide", {
+        title: "Run Operations Guide",
+        description: "Choose the right run helper and interpret run statuses, results, warnings, and timeouts correctly.",
+    }, async () => ({
+        messages: [
+            {
+                role: "user",
+                content: {
+                    type: "text",
+                    text: "Use run-and-wait when the user wants a final outcome in one call, retry-and-wait for immediate reruns of failed runs, run-status for live scheduler polling, and get-run-details when you need metadata plus results together. Treat waitingToRun and running as non-terminal, and completed, failed, and canceled as terminal. Inspect validation, warning, resultsError, incomplete, and timedOut fields before reporting success. See coalesce://context/run-operations for the full lifecycle.",
+                },
+            },
+        ],
+    }));
+    server.registerPrompt("large-result-handling", {
+        title: "Large Result Handling",
+        description: "Use cache tools and coalesce://cache resource URIs when payloads are too large to return inline.",
+    }, async () => ({
+        messages: [
+            {
+                role: "user",
+                content: {
+                    type: "text",
+                    text: "Large JSON responses may be returned as cache metadata with a coalesce://cache/... resource URI instead of full inline payloads. Read the referenced resource rather than assuming the JSON is embedded in the tool result. When you know a large snapshot is needed, prefer explicit cache tools like cache-workspace-nodes, cache-environment-nodes, cache-runs, or cache-org-users so the artifact can be reused. See coalesce://context/tool-usage for paging and cache-handling guidance.",
+                },
+            },
+        ],
+    }));
+}

package/dist/resources/context/aggregation-patterns.md

@@ -0,0 +1,145 @@
+# Aggregation and GROUP BY Patterns
+
+## Automatic JOIN ON Generation
+
+When creating or converting multi-predecessor nodes, the system analyzes common columns between predecessors and generates JOIN ON clauses.
+
+1. **Column Analysis**: Compares columns from each predecessor pair
+2. **Normalization**: Case-insensitive column name matching
+3. **SQL Generation**: Produces FROM/JOIN/ON clauses
+
+Example: predecessors ORDERS (ORDER_ID, CUSTOMER_ID) and CUSTOMERS (CUSTOMER_ID, CUSTOMER_NAME) produce:
+
+```sql
+FROM "ORDERS"
+INNER JOIN "CUSTOMERS"
+  ON "ORDERS"."CUSTOMER_ID" = "CUSTOMERS"."CUSTOMER_ID"
+```
+
+## Automatic Datatype Inference
+
+The system infers datatypes from transform expressions:
+
+| Transform Pattern | Inferred Datatype |
+|-------------------|-------------------|
+| `COUNT(...)` | `NUMBER` |
+| `SUM(...)` | `NUMBER(38,4)` |
+| `AVG(...)` | `NUMBER(38,4)` |
+| `MIN/MAX(..._TS)` | `TIMESTAMP_NTZ(9)` |
+| `MIN/MAX(..._DATE)` | `DATE` |
+| `DATEDIFF(...)` | `NUMBER` |
+| `CURRENT_DATE` | `DATE` |
+| `CURRENT_TIMESTAMP` | `TIMESTAMP_NTZ(9)` |
+| `ROW_NUMBER()` | `NUMBER` |
+| `CONCAT(...)` | `VARCHAR` |
+
+## GROUP BY Analysis and Validation
+
+The system automatically detects aggregate functions, identifies non-aggregate columns needing GROUP BY, validates coverage, and generates the GROUP BY clause.
+
+### Detection Rules
+
+**Aggregate functions** (column goes into aggregate list):
+`COUNT`, `SUM`, `AVG`, `MIN`, `MAX`, `STDDEV`, `VARIANCE`, `LISTAGG`, `ARRAY_AGG`
+
+**Window functions** (column goes into aggregate list):
+`ROW_NUMBER`, `RANK`, `DENSE_RANK`, `LEAD`, `LAG`, `FIRST_VALUE`, `LAST_VALUE`
+
+**Non-aggregate columns** (must be in GROUP BY):
+Simple references (`"TABLE"."COLUMN"`), expressions without aggregation (`UPPER("TABLE"."NAME")`)
+
+### Validation
+
+The `validation.valid` flag indicates whether GROUP BY is correct. When `valid: false`, the query has aggregate functions but missing GROUP BY columns — all non-aggregate columns must be included.
+
+## Common Patterns
+
+### Customer Lifetime Metrics
+
+```javascript
+{
+  groupByColumns: ['"ORDERS"."CUSTOMER_ID"'],
+  aggregates: [
+    { name: "TOTAL_ORDERS", function: "COUNT", expression: 'DISTINCT "ORDERS"."ORDER_ID"' },
+    { name: "LIFETIME_VALUE", function: "SUM", expression: '"ORDERS"."ORDER_TOTAL"' },
+    { name: "AVG_ORDER_VALUE", function: "AVG", expression: '"ORDERS"."ORDER_TOTAL"' },
+    { name: "FIRST_ORDER_DATE", function: "MIN", expression: '"ORDERS"."ORDER_TS"' },
+    { name: "LAST_ORDER_DATE", function: "MAX", expression: '"ORDERS"."ORDER_TS"' },
+    { name: "DAYS_SINCE_LAST", function: "DATEDIFF", expression: 'day, MAX("ORDERS"."ORDER_TS"), CURRENT_DATE()' }
+  ]
+}
+```
+
+### Daily Sales Summary
+
+```javascript
+{
+  groupByColumns: [
+    'DATE_TRUNC(\'day\', "ORDERS"."ORDER_TS")',
+    '"ORDERS"."LOCATION_ID"'
+  ],
+  aggregates: [
+    { name: "DAILY_ORDERS", function: "COUNT", expression: 'DISTINCT "ORDERS"."ORDER_ID"' },
+    { name: "DAILY_REVENUE", function: "SUM", expression: '"ORDER_DETAIL"."LINE_TOTAL"' },
+    { name: "AVG_ORDER_SIZE", function: "AVG", expression: '"ORDER_DETAIL"."QUANTITY"' }
+  ]
+}
+```
+
+### Product Category Performance
+
+```javascript
+{
+  groupByColumns: ['"PRODUCTS"."CATEGORY"', '"PRODUCTS"."SUBCATEGORY"'],
+  aggregates: [
+    { name: "TOTAL_SALES", function: "SUM", expression: '"ORDERS"."AMOUNT"' },
+    { name: "UNITS_SOLD", function: "SUM", expression: '"ORDERS"."QUANTITY"' },
+    { name: "UNIQUE_CUSTOMERS", function: "COUNT", expression: 'DISTINCT "ORDERS"."CUSTOMER_ID"' },
+    { name: "AVG_PRICE", function: "AVG", expression: '"ORDERS"."UNIT_PRICE"' }
+  ]
+}
+```
+
+## groupByColumns is Analysis Data Only
+
+**CRITICAL**: The `groupByColumns` field returned by `convert-join-to-aggregation` is for analysis only. It must NEVER be included in node metadata sent to the Coalesce API.
+
+The Coalesce API rejects it with: `"request/body must NOT have additional properties"`
+
+Our tools (`convert-join-to-aggregation`, `replace-workspace-node-columns`, `update-workspace-node`) automatically strip `groupByColumns` from metadata. But if you call `set-workspace-node` with a body containing `groupByColumns` in metadata, you'll get an error.
+
+## Automatic Config Completion
+
+`convert-join-to-aggregation` automatically completes config fields after transformation:
+
+### Column-Level Attributes
+
+- `isBusinessKey: true` on GROUP BY columns (dimensions)
+- `isChangeTracking: true` on aggregate columns (measures)
+
+These are column-level attributes set directly on each column object. See `coalesce://context/node-operations` for details on columnSelector attributes.
+
+### Node-Type-Aware Config
+
+Automatically set based on context:
+
+- `selectDistinct: false` (incompatible with aggregates)
+- `truncateBefore: false` (table materialization default)
+- `insertStrategy`: based on multi-source detection
+
+See `coalesce://context/intelligent-node-configuration` for complete details.
+
+## Tips
+
+1. Always use fully-qualified column names: `"TABLE"."COLUMN"`
+2. Check `validation.valid` for GROUP BY correctness
+3. Use `maintainJoins: true` for automatic JOIN ON generation
+4. Let datatype inference work — don't manually specify unless needed
+5. Review `joinSQL.fullSQL` for the generated SQL
+6. Never include groupByColumns in metadata sent to the API
+
+## Related Resources
+
+- `coalesce://context/pipeline-workflows` — using aggregation in pipelines
+- `coalesce://context/node-operations` — column-level attributes, config fields
+- `coalesce://context/intelligent-node-configuration` — config completion details

package/dist/resources/context/data-engineering-principles.md

@@ -0,0 +1,183 @@
+# Data Engineering Principles for Coalesce
+
+## How to Use This Guide
+
+When to consult:
+- Before creating workspace nodes that need architecture decisions
+- When evaluating existing workspace structure or methodology
+- When choosing materialization strategies
+
+Application pattern:
+- If the user already specified the exact node type, use it and skip analysis.
+- Use `analyze-workspace-patterns` for a compact inline profile of workspace conventions.
+- Use `cache-workspace-nodes` when the full node list should bypass chat context or be reused.
+- If workspace differs from recommendations, inform user with rationale.
+- If workspace aligns, proceed with existing pattern.
+
+For node type selection by pipeline layer, see `coalesce://context/pipeline-workflows`.
+
+## Platform Awareness
+
+Coalesce supports multiple data platforms. Materialization strategies, cost models, and features differ significantly. Determine the platform before recommending materialization.
+
+To detect: check `coalesce://context/sql-platform-selection`, inspect existing node configurations, or ask.
+
+### Snowflake
+
+- Compute model: Per-second compute billing
+- Key features: Transient tables (no Fail-safe, lower cost for staging), Dynamic Tables (declarative, auto-refreshing), Streams/Tasks (CDC), materialized views
+- Staging best practice: Transient tables for bronze/silver layers
+- Incremental: MERGE with high-water mark, Streams for CDC
+- Config indicators: `insertStrategy`, `truncateBefore`, `materializationType`
+
+### Databricks
+
+- Compute model: DBU-based billing, Photon engine
+- Key features: Delta format (time travel, OPTIMIZE, ZORDER), Delta Live Tables (DLT), streaming tables, Unity Catalog
+- Staging best practice: Delta tables with OPTIMIZE for large staging
+- Incremental: Delta MERGE, APPLY CHANGES (DLT), streaming tables
+- Note: Views are logical views over Delta tables; materialized views less common than Snowflake/BigQuery
+
+### BigQuery
+
+- Compute model: Per-bytes-scanned (on-demand) or slot-based (reservations). Views are expensive because each query rescans underlying data.
+- Key features: Partitioned tables, clustered tables, materialized views (auto-refreshing), table snapshots, expiration policies
+- Staging best practice: Partitioned tables with expiration; clustering on high-cardinality filter columns
+- Incremental: MERGE with partition pruning, streaming inserts
+- IMPORTANT: Avoid views for repeatedly queried or large-scan nodes. Prefer materialized views or tables.
+
+## Workspace Pattern Analysis
+
+### Package Detection
+
+Scan node types for package prefixes:
+- `base-nodes:::*` -> base-nodes package observed
+- `custom-package:::*` -> custom package observed
+- No prefix -> built-in type
+
+Presence indicates observed usage, not exhaustive inventory. Absence doesn't prove a package is unavailable.
+
+Package categories:
+- **base-nodes**: Enhanced Stage, View, Dimension, Fact, Work
+- **Specialized**: Data Vault, Kimball extensions, semantic layers
+- **Platform-specific**: Databricks DLT, BigQuery advanced, Dynamic Tables, Streams/Tasks
+- **Data quality**: Incremental loading, test filtering, validation
+- **Built-in**: Stage, View, Dimension, Fact, persistentStage (no prefix)
+
+### DAG Topology Analysis
+
+| Layer | DAG Signature | Typical Names |
+|-------|--------------|---------------|
+| Bronze/Landing | 0 predecessors or source predecessors | RAW_*, SRC_*, LANDING_* |
+| Silver/Staging | 1-2 predecessors from bronze | STG_*, STAGE_*, CLEAN_* |
+| Intermediate | Mid-pipeline (has predecessors AND consumers) | INT_*, WORK_*, TRANSFORM_* |
+| Gold/Mart | Multiple predecessors, few/no consumers | DIM_*, FACT_*, FCT_*, MART_* |
+
+### Methodology Detection
+
+**Kimball**: DIM_*/FACT_* separation, facts with 3+ dimension predecessors, star/snowflake topology. SCD indicators: EFFECTIVE_FROM, EFFECTIVE_TO, IS_CURRENT columns.
+
+**Data Vault 2.0**: Hub (few columns, business key + metadata, many downstream satellites), Satellite (many columns, single hub predecessor, HASH_DIFF), Link (multiple hub predecessors). May use specialized packages from github.com/coalesceio.
+
+**dbt-Style**: stg_ -> int_ -> fct_/dim_ naming, heavy View usage in intermediate layer, selective materialization.
+
+**Mixed/Unclear**: Default to staging -> mart pattern. Don't force methodology on established workspaces.
+
+## Materialization Strategies
+
+### Table (Full Refresh)
+
+When: Data changes significantly each run, staging/bronze layer, dimensions with low update frequency.
+Trade-offs: Simple, predictable, consistent. Higher compute for large datasets.
+
+- Snowflake: `truncateBefore: true`, `materializationType: "table"`. Transient tables for staging.
+- BigQuery: Cost-effective even for large datasets (bytes written, not row count).
+- Databricks: Delta tables with ACID guarantees.
+
+### View
+
+When: Intermediate transforms, low query frequency, always-fresh data, simple transforms without aggregation.
+
+IMPORTANT: `View` node types can ONLY materialize as views. Cannot convert to tables. For aggregations or frequently queried nodes, use `Dimension`, `Fact`, `Stage`, or `Work`.
+
+- Snowflake: Each query consumes compute. Acceptable for low-frequency.
+- BigQuery: Expensive — every query rescans and bills per bytes. Avoid for frequent queries or large tables.
+- Databricks: Benefits from Delta caching but recomputes on each query.
+
+### Incremental (Merge/Append)
+
+When: Large datasets, time-series/event data, fact tables with high volume, staging with clear update patterns.
+Trade-offs: Efficient, faster runs. More complex, risk of drift.
+
+- Snowflake: MERGE with high-water mark, `insertStrategy: "MERGE"`. Streams/Tasks for CDC.
+- BigQuery: MERGE with partition pruning (always partition incremental tables).
+- Databricks: Delta MERGE with schema evolution. APPLY CHANGES in DLT.
+
+### Dynamic / Auto-Refreshing
+
+- Snowflake Dynamic Tables: Declarative SQL, auto-refresh based on lag target
+- BigQuery Materialized Views: Auto-refresh with smart tuning (single-table aggregations)
+- Databricks DLT: Declarative pipeline definitions with automatic refresh
+
+Check for platform-specific packages (Dynamic-Table-Nodes, databricks-DLT).
+
+### Layer-Specific Defaults
+
+- Bronze -> Tables (preserve raw; Snowflake: transient; BigQuery: partitioned with expiration)
+- Silver -> Tables for small, incremental for large (BigQuery: always partition)
+- Intermediate -> Views (BigQuery: materialize if queried by multiple downstream nodes)
+- Gold Dimensions -> Tables (small, need persistence)
+- Gold Facts -> Incremental tables (large, time-series)
+- Metrics -> Tables via `Dimension` or `Fact`; consider materialized views for single-table aggregations
+
+## Dependency Management
+
+### Healthy DAG Patterns
+
+- **Fan-out** (1 -> many): One staging feeds multiple downstream. Promotes reusability.
+- **Fan-in** (many -> 1): Multiple sources join into one. Sweet spot: 2-4 predecessors.
+- **Linear chains** (1 -> 1 -> 1): Acceptable at 3-5 steps if each adds clear value.
+
+### Problematic Patterns
+
+- **Excessive fan-in** (>5 predecessors): Break into intermediate nodes.
+- **Deep chains** (>6 steps): Consolidate or use views.
+- **Circular dependencies**: Break by extracting shared logic upstream.
+- **Cross-layer skips**: Gold reading Bronze directly. Route through proper layers.
+
+### Warnings to Surface
+
+- predecessorNodeIDs.length > 5 -> warn about excessive fan-in
+- 7th node in linear chain -> warn about deep chain
+- Cross-layer skip -> warn about layer violation
+
+## Package Recommendations
+
+### Base Nodes
+
+If workspace has NO base-nodes types: "The base-nodes package offers enhanced Stage, View, Dimension, and Fact. Install via Build Settings > Packages."
+If workspace HAS base-nodes types: use base-nodes versions by default.
+
+### Common Packages
+
+| Package | When to Recommend | Node Types |
+|---------|------------------|------------|
+| Incremental-Nodes | Large fact tables, "incremental"/"delta" mentions | Incremental Load, Test Passed/Failed Records, Looped Load |
+| Dynamic Tables | Auto-refreshing aggregations (Snowflake) | Dynamic Table Work, Dimension |
+| Materialized Views | Simple single-table aggregations (Snowflake) | Materialized View |
+| Streams/Tasks | CDC, event-driven (Snowflake) | Stream, Task |
+| DLT | Declarative pipelines (Databricks) | DLT nodes |
+
+### Recommendation Logic
+
+1. Match existing workspace patterns first
+2. Fresh workspace with built-in types -> soft recommend base-nodes
+3. Specialized need -> point to specific package, remind "Build Settings > Packages to install"
+4. Never assume installed: "If you have the package..." not "Use the node type"
+5. Don't push packages when built-in types work fine
+
+## Related Resources
+
+- `coalesce://context/pipeline-workflows` — node type selection by layer, pipeline building
+- `coalesce://context/node-type-corpus` — node type discovery and corpus search
+- `coalesce://context/sql-platform-selection` — platform detection

package/dist/resources/context/hydrated-metadata.md

@@ -0,0 +1,92 @@
+# Hydrated Metadata
+
+Use this resource when the user wants to provide or edit raw node `metadata`, `config`, or `storageLocations`.
+
+## What This Covers
+
+Coalesce hydrated node bodies commonly include:
+- `metadata.columns`
+- `sources`
+- `config`
+- `storageLocations`
+
+The exact structure varies by node type and configuration.
+
+## Practical Summary
+
+### `metadata.columns`
+
+Columns map to the Mapping Grid in Coalesce.
+
+Common fields on a column include:
+- `id`
+- `name`
+- `dataType`
+- `description`
+- `nullable`
+- `defaultValue`
+- `tests`
+- `transform`
+
+Columns can also contain lineage-related source information.
+
+### `sources`
+
+Hydrated source metadata can include:
+- source `name`
+- source `columns`
+- source `join`
+- `dependencies`
+
+This is useful for multisource and join-oriented nodes.
+
+### `config`
+
+Hydrated config can contain both simple scalar fields and nested structures.
+
+Examples include:
+- `preSQL`
+- `postSQL`
+- `insertStrategy`
+- booleans
+- dropdown values
+- nested tabular config items
+- config entries that themselves reference column-like objects
+
+Treat config as node-type-specific. Preserve unknown keys unless the user intends to replace them.
+
+### `storageLocations`
+
+Hydrated storage locations are arrays of objects with fields such as:
+- `database`
+- `schema`
+- `name`
+
+Do not assume storage location names can be normalized for SQL style. They must match the actual Coalesce objects.
+
+## Editing Rules
+
+- Prefer `update-workspace-node` for partial changes.
+- Treat arrays as full-replacement fields (see `coalesce://context/node-payloads` for array safety details).
+- Preserve existing hydrated structures you are not intentionally changing.
+- When working from scratch, provide `metadata.columns` explicitly if the user expects a configured node.
+
+## When To Use Raw Hydrated Input
+
+Use raw hydrated structures when:
+- the user already knows the exact payload shape they want
+- the node type has custom nested config
+- you need to preserve advanced lineage or source structures
+
+If the user only wants a normal create/update flow, prefer the higher-level helpers and simpler fields first.
+
+## Official Reference
+
+See the Coalesce documentation for the fuller field inventory:
+- [Hydrated Metadata](https://docs.coalesce.io/docs/build-your-pipeline/user-defined-nodes/hydrated-metadata)
+- [Hydrated Metadata Reference](https://docs.coalesce.io/docs/build-your-pipeline/user-defined-nodes/hydrated-metadata-reference)
+
+## Related Resources
+
+- `coalesce://context/node-payloads`
+- `coalesce://context/storage-mappings`

package/dist/resources/context/id-discovery.md

@@ -0,0 +1,64 @@
+# ID Discovery
+
+Use this resource when the user knows names but not Coalesce IDs.
+
+## Core Rule
+
+Prefer list/get discovery tools over guessing IDs from URLs or names.
+
+## Common ID Lookups
+
+### Project IDs
+
+- Use `list-projects` to browse projects.
+- Use `get-project` when you already know the `projectID`.
+
+### Workspace IDs
+
+- Workspace IDs are nested under projects.
+- Use:
+  - `list-projects({ includeWorkspaces: true })`
+  - `get-project({ projectID, includeWorkspaces: true })`
+
+Do not assume workspace IDs are visible unless `includeWorkspaces` was requested.
+
+### Job IDs
+
+- Jobs are nested under projects and workspaces.
+- Use:
+  - `list-projects({ includeJobs: true, includeWorkspaces: true })`
+  - `get-project({ projectID, includeJobs: true, includeWorkspaces: true })`
+
+If the user gives a job name, resolve it to a job ID before calling `start-run`.
+
+### Environment IDs
+
+- Use `list-environments` to discover environments by name.
+- Use `get-environment` only after you already know the `environmentID`.
+
+### Node IDs
+
+- Use `list-workspace-nodes` when working in a workspace.
+- Use `list-environment-nodes` when working against an environment.
+- Use `get-workspace-node` or `get-environment-node` only after you know the node ID.
+
+### Run IDs and Run Counters
+
+- Use `list-runs` to discover recent runs when needed.
+- For run ID format details (runCounter vs UUID) and operational usage, see `coalesce://context/run-operations`.
+
+### Org IDs
+
+- For org ID requirements in run operations (cancel-run), see `coalesce://context/run-operations`.
+
+## Good Defaults
+
+1. Discover by name with a list tool.
+2. Resolve the exact ID.
+3. Use the matching get/mutate tool with that ID.
+
+## Related Resources
+
+- `coalesce://context/tool-usage`
+- `coalesce://context/run-operations`
+- `coalesce://context/sql-platform-selection`