coalesce-transform-mcp 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (134) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +304 -0
  3. package/dist/cache-dir.d.ts +26 -0
  4. package/dist/cache-dir.js +106 -0
  5. package/dist/client.d.ts +25 -0
  6. package/dist/client.js +212 -0
  7. package/dist/coalesce/api/environments.d.ts +20 -0
  8. package/dist/coalesce/api/environments.js +15 -0
  9. package/dist/coalesce/api/git-accounts.d.ts +21 -0
  10. package/dist/coalesce/api/git-accounts.js +21 -0
  11. package/dist/coalesce/api/jobs.d.ts +25 -0
  12. package/dist/coalesce/api/jobs.js +21 -0
  13. package/dist/coalesce/api/nodes.d.ts +29 -0
  14. package/dist/coalesce/api/nodes.js +33 -0
  15. package/dist/coalesce/api/projects.d.ts +22 -0
  16. package/dist/coalesce/api/projects.js +25 -0
  17. package/dist/coalesce/api/runs.d.ts +19 -0
  18. package/dist/coalesce/api/runs.js +34 -0
  19. package/dist/coalesce/api/subgraphs.d.ts +20 -0
  20. package/dist/coalesce/api/subgraphs.js +17 -0
  21. package/dist/coalesce/api/users.d.ts +30 -0
  22. package/dist/coalesce/api/users.js +31 -0
  23. package/dist/coalesce/types.d.ts +298 -0
  24. package/dist/coalesce/types.js +746 -0
  25. package/dist/generated/.gitkeep +0 -0
  26. package/dist/generated/node-type-corpus.json +42656 -0
  27. package/dist/index.d.ts +2 -0
  28. package/dist/index.js +10 -0
  29. package/dist/mcp/cache.d.ts +3 -0
  30. package/dist/mcp/cache.js +137 -0
  31. package/dist/mcp/environments.d.ts +3 -0
  32. package/dist/mcp/environments.js +61 -0
  33. package/dist/mcp/git-accounts.d.ts +3 -0
  34. package/dist/mcp/git-accounts.js +70 -0
  35. package/dist/mcp/jobs.d.ts +3 -0
  36. package/dist/mcp/jobs.js +77 -0
  37. package/dist/mcp/node-type-corpus.d.ts +3 -0
  38. package/dist/mcp/node-type-corpus.js +173 -0
  39. package/dist/mcp/nodes.d.ts +3 -0
  40. package/dist/mcp/nodes.js +341 -0
  41. package/dist/mcp/pipelines.d.ts +3 -0
  42. package/dist/mcp/pipelines.js +342 -0
  43. package/dist/mcp/projects.d.ts +3 -0
  44. package/dist/mcp/projects.js +70 -0
  45. package/dist/mcp/repo-node-types.d.ts +135 -0
  46. package/dist/mcp/repo-node-types.js +387 -0
  47. package/dist/mcp/runs.d.ts +3 -0
  48. package/dist/mcp/runs.js +92 -0
  49. package/dist/mcp/subgraphs.d.ts +3 -0
  50. package/dist/mcp/subgraphs.js +60 -0
  51. package/dist/mcp/users.d.ts +3 -0
  52. package/dist/mcp/users.js +107 -0
  53. package/dist/prompts/index.d.ts +2 -0
  54. package/dist/prompts/index.js +58 -0
  55. package/dist/resources/context/aggregation-patterns.md +145 -0
  56. package/dist/resources/context/data-engineering-principles.md +183 -0
  57. package/dist/resources/context/hydrated-metadata.md +92 -0
  58. package/dist/resources/context/id-discovery.md +64 -0
  59. package/dist/resources/context/intelligent-node-configuration.md +162 -0
  60. package/dist/resources/context/node-creation-decision-tree.md +156 -0
  61. package/dist/resources/context/node-operations.md +316 -0
  62. package/dist/resources/context/node-payloads.md +114 -0
  63. package/dist/resources/context/node-type-corpus.md +166 -0
  64. package/dist/resources/context/node-type-selection-guide.md +96 -0
  65. package/dist/resources/context/overview.md +135 -0
  66. package/dist/resources/context/pipeline-workflows.md +355 -0
  67. package/dist/resources/context/run-operations.md +55 -0
  68. package/dist/resources/context/sql-bigquery.md +41 -0
  69. package/dist/resources/context/sql-databricks.md +40 -0
  70. package/dist/resources/context/sql-platform-selection.md +70 -0
  71. package/dist/resources/context/sql-snowflake.md +43 -0
  72. package/dist/resources/context/storage-mappings.md +49 -0
  73. package/dist/resources/context/tool-usage.md +98 -0
  74. package/dist/resources/index.d.ts +5 -0
  75. package/dist/resources/index.js +254 -0
  76. package/dist/schemas/node-payloads.d.ts +5019 -0
  77. package/dist/schemas/node-payloads.js +147 -0
  78. package/dist/server.d.ts +7 -0
  79. package/dist/server.js +63 -0
  80. package/dist/services/cache/snapshots.d.ts +108 -0
  81. package/dist/services/cache/snapshots.js +275 -0
  82. package/dist/services/config/context-analyzer.d.ts +14 -0
  83. package/dist/services/config/context-analyzer.js +76 -0
  84. package/dist/services/config/field-classifier.d.ts +23 -0
  85. package/dist/services/config/field-classifier.js +47 -0
  86. package/dist/services/config/intelligent.d.ts +55 -0
  87. package/dist/services/config/intelligent.js +306 -0
  88. package/dist/services/config/rules.d.ts +6 -0
  89. package/dist/services/config/rules.js +44 -0
  90. package/dist/services/config/schema-resolver.d.ts +18 -0
  91. package/dist/services/config/schema-resolver.js +80 -0
  92. package/dist/services/corpus/loader.d.ts +56 -0
  93. package/dist/services/corpus/loader.js +25 -0
  94. package/dist/services/corpus/search.d.ts +49 -0
  95. package/dist/services/corpus/search.js +69 -0
  96. package/dist/services/corpus/templates.d.ts +4 -0
  97. package/dist/services/corpus/templates.js +11 -0
  98. package/dist/services/pipelines/execution.d.ts +20 -0
  99. package/dist/services/pipelines/execution.js +290 -0
  100. package/dist/services/pipelines/node-type-intent.d.ts +96 -0
  101. package/dist/services/pipelines/node-type-intent.js +356 -0
  102. package/dist/services/pipelines/node-type-selection.d.ts +66 -0
  103. package/dist/services/pipelines/node-type-selection.js +758 -0
  104. package/dist/services/pipelines/planning.d.ts +543 -0
  105. package/dist/services/pipelines/planning.js +1839 -0
  106. package/dist/services/policies/sql-override.d.ts +7 -0
  107. package/dist/services/policies/sql-override.js +109 -0
  108. package/dist/services/repo/operations.d.ts +6 -0
  109. package/dist/services/repo/operations.js +10 -0
  110. package/dist/services/repo/parser.d.ts +70 -0
  111. package/dist/services/repo/parser.js +365 -0
  112. package/dist/services/repo/path.d.ts +2 -0
  113. package/dist/services/repo/path.js +58 -0
  114. package/dist/services/templates/nodes.d.ts +50 -0
  115. package/dist/services/templates/nodes.js +336 -0
  116. package/dist/services/workspace/analysis.d.ts +56 -0
  117. package/dist/services/workspace/analysis.js +151 -0
  118. package/dist/services/workspace/mutations.d.ts +150 -0
  119. package/dist/services/workspace/mutations.js +1718 -0
  120. package/dist/utils.d.ts +5 -0
  121. package/dist/utils.js +7 -0
  122. package/dist/workflows/get-environment-overview.d.ts +9 -0
  123. package/dist/workflows/get-environment-overview.js +23 -0
  124. package/dist/workflows/get-run-details.d.ts +10 -0
  125. package/dist/workflows/get-run-details.js +28 -0
  126. package/dist/workflows/progress.d.ts +20 -0
  127. package/dist/workflows/progress.js +54 -0
  128. package/dist/workflows/retry-and-wait.d.ts +13 -0
  129. package/dist/workflows/retry-and-wait.js +139 -0
  130. package/dist/workflows/run-and-wait.d.ts +13 -0
  131. package/dist/workflows/run-and-wait.js +141 -0
  132. package/dist/workflows/run-status.d.ts +10 -0
  133. package/dist/workflows/run-status.js +27 -0
  134. package/package.json +34 -0
@@ -0,0 +1,56 @@
1
+ export type NodeTypeCorpusSupportStatus = "supported" | "partial" | "parse_error";
2
+ export type NodeTypeCorpusOccurrence = {
3
+ packageName: string;
4
+ nodeTypeDirName: string;
5
+ nodeTypeDirPath: string;
6
+ definitionPath: string;
7
+ createPath: string;
8
+ runPath: string;
9
+ };
10
+ export type NodeTypeCorpusDefinitionSummary = {
11
+ capitalized: string | null;
12
+ short: string | null;
13
+ plural: string | null;
14
+ tagColor: string | null;
15
+ deployStrategy: string | null;
16
+ configGroupCount: number;
17
+ configItemCount: number;
18
+ };
19
+ export type NodeTypeCorpusVariant = {
20
+ variantKey: string;
21
+ normalizedFamily: string;
22
+ packageNames: string[];
23
+ occurrenceCount: number;
24
+ occurrences: NodeTypeCorpusOccurrence[];
25
+ definitionHash: string;
26
+ createHash: string;
27
+ runHash: string;
28
+ primitiveSignature: string[];
29
+ controlSignature: string[];
30
+ unsupportedPrimitives: string[];
31
+ supportStatus: NodeTypeCorpusSupportStatus;
32
+ definitionSummary: NodeTypeCorpusDefinitionSummary;
33
+ outerDefinition: {
34
+ fileVersion: unknown;
35
+ id: string | null;
36
+ isDisabled: boolean | null;
37
+ name: string | null;
38
+ type: string | null;
39
+ };
40
+ nodeMetadataSpec: string;
41
+ nodeDefinition: Record<string, unknown> | null;
42
+ parseError: string | null;
43
+ };
44
+ export type NodeTypeCorpusSnapshot = {
45
+ generatedAt: string;
46
+ sourceRoot: string;
47
+ packageCount: number;
48
+ definitionCount: number;
49
+ uniqueVariantCount: number;
50
+ uniqueNormalizedFamilyCount: number;
51
+ supportedVariantCount: number;
52
+ partialVariantCount: number;
53
+ parseErrorVariantCount: number;
54
+ variants: NodeTypeCorpusVariant[];
55
+ };
56
+ export declare function loadNodeTypeCorpusSnapshot(): NodeTypeCorpusSnapshot;
@@ -0,0 +1,25 @@
1
+ import { readFileSync } from "node:fs";
2
+ import { fileURLToPath } from "node:url";
3
+ import { isPlainObject } from "../../utils.js";
4
+ let cachedSnapshot = null;
5
+ function getSnapshotPath() {
6
+ return fileURLToPath(new URL("../../generated/node-type-corpus.json", import.meta.url));
7
+ }
8
+ export function loadNodeTypeCorpusSnapshot() {
9
+ if (cachedSnapshot) {
10
+ return cachedSnapshot;
11
+ }
12
+ const filePath = getSnapshotPath();
13
+ let parsed;
14
+ try {
15
+ parsed = JSON.parse(readFileSync(filePath, "utf8"));
16
+ }
17
+ catch {
18
+ throw new Error(`Node type corpus snapshot not found or unreadable at ${filePath}. Run npm run import:node-type-corpus before building.`);
19
+ }
20
+ if (!isPlainObject(parsed) || !Array.isArray(parsed.variants)) {
21
+ throw new Error(`Node type corpus snapshot is invalid at ${filePath}.`);
22
+ }
23
+ cachedSnapshot = parsed;
24
+ return cachedSnapshot;
25
+ }
@@ -0,0 +1,49 @@
1
+ import type { NodeTypeCorpusSnapshot, NodeTypeCorpusVariant, NodeTypeCorpusSupportStatus } from "./loader.js";
2
+ export declare function summarizeNodeTypeCorpus(snapshot: NodeTypeCorpusSnapshot): {
3
+ generatedAt: string;
4
+ sourceRoot: string;
5
+ packageCount: number;
6
+ definitionCount: number;
7
+ uniqueVariantCount: number;
8
+ uniqueNormalizedFamilyCount: number;
9
+ supportedVariantCount: number;
10
+ partialVariantCount: number;
11
+ parseErrorVariantCount: number;
12
+ topFamilies: {
13
+ normalizedFamily: string;
14
+ variantCount: number;
15
+ }[];
16
+ };
17
+ export declare function buildVariantSummary(variant: NodeTypeCorpusVariant): {
18
+ variantKey: string;
19
+ normalizedFamily: string;
20
+ packageNames: string[];
21
+ occurrenceCount: number;
22
+ supportStatus: NodeTypeCorpusSupportStatus;
23
+ unsupportedPrimitives: string[];
24
+ primitiveSignature: string[];
25
+ controlSignature: string[];
26
+ definitionSummary: import("./loader.js").NodeTypeCorpusDefinitionSummary;
27
+ };
28
+ export declare function searchNodeTypeCorpusVariants(snapshot: NodeTypeCorpusSnapshot, params: {
29
+ normalizedFamily?: string;
30
+ packageName?: string;
31
+ primitive?: string;
32
+ supportStatus?: NodeTypeCorpusSupportStatus;
33
+ limit?: number;
34
+ }): {
35
+ matchedCount: number;
36
+ returnedCount: number;
37
+ matches: {
38
+ variantKey: string;
39
+ normalizedFamily: string;
40
+ packageNames: string[];
41
+ occurrenceCount: number;
42
+ supportStatus: NodeTypeCorpusSupportStatus;
43
+ unsupportedPrimitives: string[];
44
+ primitiveSignature: string[];
45
+ controlSignature: string[];
46
+ definitionSummary: import("./loader.js").NodeTypeCorpusDefinitionSummary;
47
+ }[];
48
+ };
49
+ export declare function getNodeTypeCorpusVariant(snapshot: NodeTypeCorpusSnapshot, variantKey: string): NodeTypeCorpusVariant;
@@ -0,0 +1,69 @@
1
+ import { filterSqlOverrideControls } from "../policies/sql-override.js";
2
+ export function summarizeNodeTypeCorpus(snapshot) {
3
+ const familyCounts = new Map();
4
+ for (const variant of snapshot.variants) {
5
+ familyCounts.set(variant.normalizedFamily, (familyCounts.get(variant.normalizedFamily) ?? 0) + 1);
6
+ }
7
+ const topFamilies = Array.from(familyCounts.entries())
8
+ .sort((left, right) => right[1] === left[1]
9
+ ? left[0].localeCompare(right[0])
10
+ : right[1] - left[1])
11
+ .slice(0, 20)
12
+ .map(([normalizedFamily, variantCount]) => ({
13
+ normalizedFamily,
14
+ variantCount,
15
+ }));
16
+ return {
17
+ generatedAt: snapshot.generatedAt,
18
+ sourceRoot: snapshot.sourceRoot,
19
+ packageCount: snapshot.packageCount,
20
+ definitionCount: snapshot.definitionCount,
21
+ uniqueVariantCount: snapshot.uniqueVariantCount,
22
+ uniqueNormalizedFamilyCount: snapshot.uniqueNormalizedFamilyCount,
23
+ supportedVariantCount: snapshot.supportedVariantCount,
24
+ partialVariantCount: snapshot.partialVariantCount,
25
+ parseErrorVariantCount: snapshot.parseErrorVariantCount,
26
+ topFamilies,
27
+ };
28
+ }
29
+ export function buildVariantSummary(variant) {
30
+ return {
31
+ variantKey: variant.variantKey,
32
+ normalizedFamily: variant.normalizedFamily,
33
+ packageNames: variant.packageNames,
34
+ occurrenceCount: variant.occurrenceCount,
35
+ supportStatus: variant.supportStatus,
36
+ unsupportedPrimitives: filterSqlOverrideControls(variant.unsupportedPrimitives),
37
+ primitiveSignature: filterSqlOverrideControls(variant.primitiveSignature),
38
+ controlSignature: filterSqlOverrideControls(variant.controlSignature),
39
+ definitionSummary: variant.definitionSummary,
40
+ };
41
+ }
42
+ export function searchNodeTypeCorpusVariants(snapshot, params) {
43
+ const limit = Math.max(1, Math.min(params.limit ?? 25, 200));
44
+ const normalizedFamilyFilter = params.normalizedFamily?.trim().toLowerCase();
45
+ const packageNameFilter = params.packageName?.trim().toLowerCase();
46
+ const primitiveFilter = params.primitive?.trim().toLowerCase();
47
+ const matches = snapshot.variants.filter((variant) => {
48
+ const familyMatches = !normalizedFamilyFilter ||
49
+ variant.normalizedFamily.toLowerCase() === normalizedFamilyFilter;
50
+ const packageMatches = !packageNameFilter ||
51
+ variant.packageNames.some((packageName) => packageName.toLowerCase() === packageNameFilter);
52
+ const primitiveMatches = !primitiveFilter ||
53
+ variant.primitiveSignature.some((primitive) => primitive.toLowerCase() === primitiveFilter);
54
+ const supportMatches = !params.supportStatus || variant.supportStatus === params.supportStatus;
55
+ return familyMatches && packageMatches && primitiveMatches && supportMatches;
56
+ });
57
+ return {
58
+ matchedCount: matches.length,
59
+ returnedCount: Math.min(matches.length, limit),
60
+ matches: matches.slice(0, limit).map(buildVariantSummary),
61
+ };
62
+ }
63
+ export function getNodeTypeCorpusVariant(snapshot, variantKey) {
64
+ const variant = snapshot.variants.find((entry) => entry.variantKey === variantKey);
65
+ if (!variant) {
66
+ throw new Error(`No node type corpus variant found for variantKey ${variantKey}.`);
67
+ }
68
+ return variant;
69
+ }
@@ -0,0 +1,4 @@
1
+ import type { NodeTypeCorpusVariant } from "./loader.js";
2
+ export declare function validateVariantTemplateGeneration(variant: NodeTypeCorpusVariant, options?: {
3
+ allowPartial?: boolean;
4
+ }): void;
@@ -0,0 +1,11 @@
1
+ export function validateVariantTemplateGeneration(variant, options = {}) {
2
+ if (!variant.nodeDefinition) {
3
+ throw new Error(`Variant ${variant.variantKey} cannot generate a workspace template because its embedded nodeMetadataSpec could not be parsed. Parse error: ${variant.parseError ?? "unknown"}`);
4
+ }
5
+ if (variant.supportStatus === "partial" && !options.allowPartial) {
6
+ const unsupported = variant.unsupportedPrimitives.length > 0
7
+ ? variant.unsupportedPrimitives.join(", ")
8
+ : "unknown unsupported controls";
9
+ throw new Error(`Variant ${variant.variantKey} is only partially supported because it uses unsupported primitives: ${unsupported}. Pass allowPartial=true to generate a best-effort template.`);
10
+ }
11
+ }
@@ -0,0 +1,20 @@
1
+ import { type CoalesceClient } from "../../client.js";
2
+ export declare function createPipelineFromPlan(client: CoalesceClient, params: {
3
+ workspaceID: string;
4
+ plan: Record<string, unknown>;
5
+ dryRun?: boolean;
6
+ }): Promise<unknown>;
7
+ export declare function createPipelineFromSql(client: CoalesceClient, params: {
8
+ workspaceID: string;
9
+ sql: string;
10
+ goal?: string;
11
+ targetName?: string;
12
+ targetNodeType?: string;
13
+ description?: string;
14
+ configOverrides?: Record<string, unknown>;
15
+ locationName?: string;
16
+ database?: string;
17
+ schema?: string;
18
+ repoPath?: string;
19
+ dryRun?: boolean;
20
+ }): Promise<unknown>;
@@ -0,0 +1,290 @@
1
+ import { randomUUID } from "node:crypto";
2
+ import { CoalesceApiError } from "../../client.js";
3
+ import { validatePathSegment } from "../../coalesce/types.js";
4
+ import { PipelinePlanSchema, planPipeline, DEFAULT_STAGE_CONFIG, deepClone, findMatchingBaseColumn, buildStageSourceMappingFromPlan, renameSourceMappingEntries, getColumnNamesFromNode, normalizeSqlIdentifier, normalizeWhitespace, getNodeColumnArray, getColumnSourceNodeIDs, } from "./planning.js";
5
+ import { getWorkspaceNode, setWorkspaceNode, } from "../../coalesce/api/nodes.js";
6
+ import { createWorkspaceNodeFromPredecessor, buildUpdatedWorkspaceNodeBody } from "../workspace/mutations.js";
7
+ import { isPlainObject } from "../../utils.js";
8
+ function isStageLikeNode(nodePlan) {
9
+ return (nodePlan.nodeTypeFamily === "stage" ||
10
+ nodePlan.nodeTypeFamily === "persistent-stage" ||
11
+ /(?:^|:::)(?:persistent)?Stage$/u.test(nodePlan.nodeType));
12
+ }
13
+ function buildNodeBodyFromPlan(currentNode, nodePlan) {
14
+ const updatedNode = deepClone(currentNode);
15
+ updatedNode.name = nodePlan.name;
16
+ if (nodePlan.description !== null) {
17
+ updatedNode.description = nodePlan.description;
18
+ }
19
+ if (Object.keys(nodePlan.location).length > 0) {
20
+ Object.assign(updatedNode, nodePlan.location);
21
+ }
22
+ const templateDefaults = isPlainObject(nodePlan.templateDefaults)
23
+ ? nodePlan.templateDefaults
24
+ : undefined;
25
+ const inferredTopLevelFields = templateDefaults && isPlainObject(templateDefaults.inferredTopLevelFields)
26
+ ? templateDefaults.inferredTopLevelFields
27
+ : {};
28
+ for (const [key, value] of Object.entries(inferredTopLevelFields)) {
29
+ if (updatedNode[key] === undefined) {
30
+ updatedNode[key] = deepClone(value);
31
+ }
32
+ }
33
+ updatedNode.config = {
34
+ ...(isStageLikeNode(nodePlan) ? DEFAULT_STAGE_CONFIG : {}),
35
+ ...(templateDefaults && isPlainObject(templateDefaults.inferredConfig)
36
+ ? deepClone(templateDefaults.inferredConfig)
37
+ : {}),
38
+ ...(isPlainObject(updatedNode.config) ? updatedNode.config : {}),
39
+ ...nodePlan.configOverrides,
40
+ };
41
+ const plannedColumns = [];
42
+ for (const selectItem of nodePlan.selectItems) {
43
+ const baseColumn = findMatchingBaseColumn(updatedNode, selectItem);
44
+ if (!baseColumn) {
45
+ throw new Error(`Could not map planned output column ${selectItem.outputName ?? selectItem.expression} onto the created predecessor-based node body.`);
46
+ }
47
+ baseColumn.name = selectItem.outputName ?? baseColumn.name;
48
+ if (isPlainObject(baseColumn.columnReference)) {
49
+ baseColumn.columnReference = {
50
+ ...baseColumn.columnReference,
51
+ columnCounter: randomUUID(),
52
+ };
53
+ }
54
+ if (typeof baseColumn.columnID === "string") {
55
+ baseColumn.columnID = randomUUID();
56
+ }
57
+ plannedColumns.push(baseColumn);
58
+ }
59
+ const currentMetadata = isPlainObject(updatedNode.metadata)
60
+ ? updatedNode.metadata
61
+ : {};
62
+ updatedNode.metadata = {
63
+ ...currentMetadata,
64
+ columns: plannedColumns,
65
+ sourceMapping: buildStageSourceMappingFromPlan(updatedNode, nodePlan),
66
+ };
67
+ return renameSourceMappingEntries(updatedNode, nodePlan.name);
68
+ }
69
+ function getSavedNodeColumnNames(node) {
70
+ return getColumnNamesFromNode(node);
71
+ }
72
+ function validateSavedNode(node, nodePlan) {
73
+ const savedColumnNames = getSavedNodeColumnNames(node);
74
+ const expectedColumnNames = nodePlan.outputColumnNames;
75
+ const normalizedSaved = savedColumnNames.map(normalizeSqlIdentifier);
76
+ const normalizedExpected = expectedColumnNames.map(normalizeSqlIdentifier);
77
+ const referencedPredecessorNodeIDs = new Set();
78
+ const metadata = isPlainObject(node.metadata) ? node.metadata : undefined;
79
+ const sourceMappingEntry = metadata && Array.isArray(metadata.sourceMapping)
80
+ ? metadata.sourceMapping.find(isPlainObject)
81
+ : undefined;
82
+ const savedDependencies = isPlainObject(sourceMappingEntry) && Array.isArray(sourceMappingEntry.dependencies)
83
+ ? sourceMappingEntry.dependencies
84
+ .filter(isPlainObject)
85
+ .flatMap((dependency) => typeof dependency.nodeName === "string" ? [dependency.nodeName] : [])
86
+ : [];
87
+ const savedJoinCondition = isPlainObject(sourceMappingEntry) &&
88
+ isPlainObject(sourceMappingEntry.join) &&
89
+ typeof sourceMappingEntry.join.joinCondition === "string"
90
+ ? normalizeWhitespace(sourceMappingEntry.join.joinCondition)
91
+ : "";
92
+ for (const column of getNodeColumnArray(node)) {
93
+ for (const nodeID of getColumnSourceNodeIDs(column)) {
94
+ referencedPredecessorNodeIDs.add(nodeID);
95
+ }
96
+ }
97
+ return {
98
+ nodeNameSatisfied: node.name === nodePlan.name,
99
+ expectedColumnCount: expectedColumnNames.length,
100
+ actualColumnCount: savedColumnNames.length,
101
+ outputColumnsSatisfied: normalizedExpected.length === normalizedSaved.length &&
102
+ normalizedExpected.every((name, index) => normalizedSaved[index] === name),
103
+ expectedColumnNames,
104
+ actualColumnNames: savedColumnNames,
105
+ sourceMappingDependenciesSatisfied: savedDependencies.length === nodePlan.sourceRefs.length &&
106
+ nodePlan.sourceRefs.every((ref) => savedDependencies.includes(ref.nodeName)),
107
+ expectedDependencyNodeNames: nodePlan.sourceRefs.map((ref) => ref.nodeName),
108
+ actualDependencyNodeNames: savedDependencies,
109
+ joinConditionSatisfied: (nodePlan.joinCondition === null && savedJoinCondition.length === 0) ||
110
+ savedJoinCondition === normalizeWhitespace(nodePlan.joinCondition ?? ""),
111
+ expectedJoinCondition: nodePlan.joinCondition,
112
+ actualJoinCondition: savedJoinCondition.length > 0 ? savedJoinCondition : null,
113
+ predecessorCoverageSatisfied: nodePlan.predecessorNodeIDs.every((nodeID) => referencedPredecessorNodeIDs.has(nodeID)),
114
+ predecessorNodeIDs: nodePlan.predecessorNodeIDs,
115
+ referencedPredecessorNodeIDs: Array.from(referencedPredecessorNodeIDs),
116
+ };
117
+ }
118
+ async function deleteWorkspaceNode(client, workspaceID, nodeID) {
119
+ await client.delete(`/api/v1/workspaces/${validatePathSegment(workspaceID, "workspaceID")}/nodes/${validatePathSegment(nodeID, "nodeID")}`);
120
+ }
121
+ function serializePipelineExecutionError(error) {
122
+ if (error instanceof CoalesceApiError) {
123
+ return {
124
+ message: error.message,
125
+ status: error.status,
126
+ ...(error.detail !== undefined ? { detail: error.detail } : {}),
127
+ };
128
+ }
129
+ if (error instanceof Error) {
130
+ return { message: error.message };
131
+ }
132
+ return { message: "Pipeline creation failed", detail: error };
133
+ }
134
+ async function rollbackCreatedPipelineNodes(client, workspaceID, nodeIDs) {
135
+ const rollbackFailures = [];
136
+ const uniqueNodeIDs = Array.from(new Set(nodeIDs));
137
+ for (const nodeID of uniqueNodeIDs.reverse()) {
138
+ try {
139
+ await deleteWorkspaceNode(client, workspaceID, nodeID);
140
+ }
141
+ catch (error) {
142
+ rollbackFailures.push({
143
+ nodeID,
144
+ ...serializePipelineExecutionError(error),
145
+ });
146
+ }
147
+ }
148
+ return rollbackFailures;
149
+ }
150
+ export async function createPipelineFromPlan(client, params) {
151
+ const plan = PipelinePlanSchema.parse(params.plan);
152
+ if (plan.workspaceID !== params.workspaceID) {
153
+ throw new Error(`Pipeline plan workspaceID ${plan.workspaceID} does not match requested workspaceID ${params.workspaceID}.`);
154
+ }
155
+ if (plan.status !== "ready") {
156
+ return {
157
+ created: false,
158
+ warning: "Pipeline plan still needs clarification. Review openQuestions and warnings before creation.",
159
+ plan,
160
+ };
161
+ }
162
+ if (params.dryRun) {
163
+ return {
164
+ created: false,
165
+ dryRun: true,
166
+ plan,
167
+ };
168
+ }
169
+ const createdNodes = [];
170
+ const createdNodeIDsByPlanNodeID = new Map();
171
+ const createdNodeIDsForRollback = [];
172
+ for (const nodePlan of plan.nodes) {
173
+ const predecessorNodeIDs = [
174
+ ...nodePlan.predecessorNodeIDs,
175
+ ...nodePlan.predecessorPlanNodeIDs.flatMap((planNodeID) => {
176
+ const createdNodeID = createdNodeIDsByPlanNodeID.get(planNodeID);
177
+ return createdNodeID ? [createdNodeID] : [];
178
+ }),
179
+ ];
180
+ if (predecessorNodeIDs.length === 0) {
181
+ throw new Error(`Pipeline node ${nodePlan.planNodeID} has no resolved predecessor node IDs.`);
182
+ }
183
+ let createdNodeID = null;
184
+ try {
185
+ const created = await createWorkspaceNodeFromPredecessor(client, {
186
+ workspaceID: params.workspaceID,
187
+ nodeType: nodePlan.nodeType,
188
+ predecessorNodeIDs,
189
+ });
190
+ if (!isPlainObject(created) || !isPlainObject(created.node)) {
191
+ throw new Error(`Pipeline node ${nodePlan.planNodeID} did not return a created node body.`);
192
+ }
193
+ if (typeof created.node.id === "string") {
194
+ createdNodeID = created.node.id;
195
+ createdNodeIDsForRollback.push(createdNodeID);
196
+ }
197
+ if ("warning" in created && typeof created.warning === "string") {
198
+ throw new Error(`Predecessor-based creation for ${nodePlan.name} did not confirm full auto-population: ${created.warning}`);
199
+ }
200
+ if (!createdNodeID) {
201
+ throw new Error(`Created pipeline node ${nodePlan.planNodeID} did not return a node ID.`);
202
+ }
203
+ const plannedBody = buildNodeBodyFromPlan(created.node, {
204
+ ...nodePlan,
205
+ predecessorNodeIDs,
206
+ });
207
+ // Route through the shared validation path to ensure all API-required
208
+ // fields (dataType, columnID, enabledColumnTestIDs, etc.) are present.
209
+ const finalBody = buildUpdatedWorkspaceNodeBody(created.node, plannedBody);
210
+ await setWorkspaceNode(client, {
211
+ workspaceID: params.workspaceID,
212
+ nodeID: createdNodeID,
213
+ body: finalBody,
214
+ });
215
+ const savedNode = await getWorkspaceNode(client, {
216
+ workspaceID: params.workspaceID,
217
+ nodeID: createdNodeID,
218
+ });
219
+ if (!isPlainObject(savedNode)) {
220
+ throw new Error(`Saved pipeline node ${nodePlan.name} did not return an object body.`);
221
+ }
222
+ const validation = validateSavedNode(savedNode, {
223
+ ...nodePlan,
224
+ predecessorNodeIDs,
225
+ });
226
+ if (!validation.nodeNameSatisfied ||
227
+ !validation.outputColumnsSatisfied ||
228
+ !validation.sourceMappingDependenciesSatisfied ||
229
+ !validation.joinConditionSatisfied ||
230
+ !validation.predecessorCoverageSatisfied) {
231
+ throw new Error(`Saved pipeline node ${nodePlan.name} did not match the planned body after set-workspace-node.`);
232
+ }
233
+ createdNodeIDsByPlanNodeID.set(nodePlan.planNodeID, createdNodeID);
234
+ createdNodes.push({
235
+ planNodeID: nodePlan.planNodeID,
236
+ nodeID: createdNodeID,
237
+ name: nodePlan.name,
238
+ nodeType: nodePlan.nodeType,
239
+ validation,
240
+ });
241
+ }
242
+ catch (error) {
243
+ const rollbackFailures = await rollbackCreatedPipelineNodes(client, params.workspaceID, createdNodeIDsForRollback);
244
+ if (rollbackFailures.length > 0) {
245
+ return {
246
+ created: false,
247
+ incomplete: true,
248
+ failedPlanNodeID: nodePlan.planNodeID,
249
+ createdNodes,
250
+ cleanupFailedNodeIDs: rollbackFailures.map((failure) => failure.nodeID),
251
+ cleanupFailures: rollbackFailures,
252
+ warning: "Pipeline creation failed after nodes were created, and automatic cleanup did not fully succeed. Review the workspace manually before continuing.",
253
+ error: serializePipelineExecutionError(error),
254
+ };
255
+ }
256
+ throw error;
257
+ }
258
+ }
259
+ return {
260
+ created: true,
261
+ workspaceID: params.workspaceID,
262
+ nodeCount: createdNodes.length,
263
+ createdNodes,
264
+ };
265
+ }
266
+ export async function createPipelineFromSql(client, params) {
267
+ const plan = await planPipeline(client, params);
268
+ if (plan.status !== "ready" || params.dryRun) {
269
+ return {
270
+ created: false,
271
+ STOP_AND_CONFIRM: "STOP. Present the pipeline summary to the user in a table format and ask for confirmation BEFORE creating any nodes. For EACH node, display: name, the EXACT nodeType string (e.g. 'Coalesce-Base-Node-Types:::Stage'), transforms, and filters. Use the cteNodeSummary or nodes array — do NOT paraphrase or simplify the nodeType values. Do NOT proceed until the user explicitly approves.",
272
+ ...(params.dryRun ? { dryRun: true } : {}),
273
+ plan,
274
+ ...(plan.status !== "ready"
275
+ ? {
276
+ warning: "SQL was planned but still needs clarification before creation. Review openQuestions and warnings. Present the plan to the user and wait for approval.",
277
+ }
278
+ : {}),
279
+ };
280
+ }
281
+ const execution = await createPipelineFromPlan(client, {
282
+ workspaceID: params.workspaceID,
283
+ plan,
284
+ dryRun: params.dryRun,
285
+ });
286
+ return {
287
+ plan,
288
+ ...(isPlainObject(execution) ? execution : { execution }),
289
+ };
290
+ }
@@ -0,0 +1,96 @@
1
+ /**
2
+ * Node Type Intent Corpus
3
+ *
4
+ * Authoritative reference for what each Coalesce node type family is designed for,
5
+ * derived from official Coalesce node type package READMEs.
6
+ *
7
+ * Used by the scoring logic to make informed decisions about which node type
8
+ * fits a given use case — instead of relying on name heuristics alone.
9
+ */
10
+ import type { PipelineNodeTypeFamily } from "./node-type-selection.js";
11
+ export type NodeTypeIntent = {
12
+ family: PipelineNodeTypeFamily;
13
+ /** What this node type is designed for */
14
+ purpose: string;
15
+ /** What materialization it creates */
16
+ materialization: string[];
17
+ /** When to use this type — positive signals */
18
+ useWhen: string[];
19
+ /** When NOT to use this type — anti-patterns */
20
+ doNotUseWhen: string[];
21
+ /** Whether it requires semantic config (business keys, SCD, etc.) */
22
+ requiresSemanticConfig: boolean;
23
+ /** Whether it handles multi-source (joins, unions) */
24
+ supportsMultiSource: boolean;
25
+ /** Keywords in goal/name that strongly indicate this type */
26
+ strongSignals: RegExp;
27
+ /** Keywords that should NOT trigger this type */
28
+ antiSignals: RegExp | null;
29
+ };
30
+ /**
31
+ * Specialized materialization patterns that exist across families.
32
+ * These are NOT families — they modify how a family-level node is materialized.
33
+ * Scoring penalizes these when the context doesn't explicitly request them.
34
+ */
35
+ export type SpecializedPattern = {
36
+ name: string;
37
+ /** Regex to detect this pattern in candidate name/displayName */
38
+ detect: RegExp;
39
+ /** Regex the context must match to avoid penalty */
40
+ contextRequired: RegExp;
41
+ /** Scoring penalty when context doesn't match */
42
+ penalty: number;
43
+ /** Why this pattern exists and when to use it */
44
+ purpose: string;
45
+ /** When NOT to use this pattern */
46
+ doNotUseWhen: string[];
47
+ };
48
+ /**
49
+ * Intent corpus indexed by family.
50
+ *
51
+ * Source: Coalesce node type package READMEs
52
+ * - Coalesce-Base-Node-Types
53
+ * - Coalesce-Base-Node-Types---Advanced-Deploy
54
+ * - Dynamic-Table-Nodes
55
+ * - Incremental-Nodes
56
+ * - Materialized-View-Node
57
+ * - functional-node-types
58
+ * - create-alter-node-types
59
+ */
60
+ export declare const NODE_TYPE_INTENT: Record<PipelineNodeTypeFamily, NodeTypeIntent>;
61
+ /**
62
+ * Specialized materialization patterns that cross-cut families.
63
+ *
64
+ * These detect node types with specialized materialization behavior
65
+ * (Dynamic Tables, Incremental Loads, Materialized Views, etc.)
66
+ * and penalize them when the context doesn't explicitly call for that pattern.
67
+ *
68
+ * Source: Coalesce node type package READMEs
69
+ */
70
+ export declare const SPECIALIZED_PATTERNS: SpecializedPattern[];
71
+ /**
72
+ * Check whether a use case matches a family's anti-signals.
73
+ * Anti-signals indicate the family should NOT be used for this context.
74
+ */
75
+ export declare function hasAntiSignal(family: PipelineNodeTypeFamily, text: string): boolean;
76
+ /**
77
+ * Check whether a use case matches a family's strong signals.
78
+ */
79
+ export declare function hasStrongSignal(family: PipelineNodeTypeFamily, text: string): boolean;
80
+ /**
81
+ * Detect specialized materialization patterns in a candidate and return
82
+ * the penalty to apply if the context doesn't explicitly request it.
83
+ */
84
+ /**
85
+ * Detect if a candidate matches a specialized pattern where the context
86
+ * does NOT request it. Returns the pattern info if the type should not be used.
87
+ */
88
+ export declare function detectSpecializedPatternPenalty(candidateSignals: string, contextText: string): {
89
+ penalty: number;
90
+ reason: string;
91
+ } | null;
92
+ /**
93
+ * Detect if a candidate matches a specialized pattern AND the context
94
+ * explicitly requests it. Returns the pattern name if it's a positive match.
95
+ */
96
+ export declare function detectSpecializedPatternMatch(candidateSignals: string, contextText: string): string | null;