coalesce-transform-mcp 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +304 -0
- package/dist/cache-dir.d.ts +26 -0
- package/dist/cache-dir.js +106 -0
- package/dist/client.d.ts +25 -0
- package/dist/client.js +212 -0
- package/dist/coalesce/api/environments.d.ts +20 -0
- package/dist/coalesce/api/environments.js +15 -0
- package/dist/coalesce/api/git-accounts.d.ts +21 -0
- package/dist/coalesce/api/git-accounts.js +21 -0
- package/dist/coalesce/api/jobs.d.ts +25 -0
- package/dist/coalesce/api/jobs.js +21 -0
- package/dist/coalesce/api/nodes.d.ts +29 -0
- package/dist/coalesce/api/nodes.js +33 -0
- package/dist/coalesce/api/projects.d.ts +22 -0
- package/dist/coalesce/api/projects.js +25 -0
- package/dist/coalesce/api/runs.d.ts +19 -0
- package/dist/coalesce/api/runs.js +34 -0
- package/dist/coalesce/api/subgraphs.d.ts +20 -0
- package/dist/coalesce/api/subgraphs.js +17 -0
- package/dist/coalesce/api/users.d.ts +30 -0
- package/dist/coalesce/api/users.js +31 -0
- package/dist/coalesce/types.d.ts +298 -0
- package/dist/coalesce/types.js +746 -0
- package/dist/generated/.gitkeep +0 -0
- package/dist/generated/node-type-corpus.json +42656 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +10 -0
- package/dist/mcp/cache.d.ts +3 -0
- package/dist/mcp/cache.js +137 -0
- package/dist/mcp/environments.d.ts +3 -0
- package/dist/mcp/environments.js +61 -0
- package/dist/mcp/git-accounts.d.ts +3 -0
- package/dist/mcp/git-accounts.js +70 -0
- package/dist/mcp/jobs.d.ts +3 -0
- package/dist/mcp/jobs.js +77 -0
- package/dist/mcp/node-type-corpus.d.ts +3 -0
- package/dist/mcp/node-type-corpus.js +173 -0
- package/dist/mcp/nodes.d.ts +3 -0
- package/dist/mcp/nodes.js +341 -0
- package/dist/mcp/pipelines.d.ts +3 -0
- package/dist/mcp/pipelines.js +342 -0
- package/dist/mcp/projects.d.ts +3 -0
- package/dist/mcp/projects.js +70 -0
- package/dist/mcp/repo-node-types.d.ts +135 -0
- package/dist/mcp/repo-node-types.js +387 -0
- package/dist/mcp/runs.d.ts +3 -0
- package/dist/mcp/runs.js +92 -0
- package/dist/mcp/subgraphs.d.ts +3 -0
- package/dist/mcp/subgraphs.js +60 -0
- package/dist/mcp/users.d.ts +3 -0
- package/dist/mcp/users.js +107 -0
- package/dist/prompts/index.d.ts +2 -0
- package/dist/prompts/index.js +58 -0
- package/dist/resources/context/aggregation-patterns.md +145 -0
- package/dist/resources/context/data-engineering-principles.md +183 -0
- package/dist/resources/context/hydrated-metadata.md +92 -0
- package/dist/resources/context/id-discovery.md +64 -0
- package/dist/resources/context/intelligent-node-configuration.md +162 -0
- package/dist/resources/context/node-creation-decision-tree.md +156 -0
- package/dist/resources/context/node-operations.md +316 -0
- package/dist/resources/context/node-payloads.md +114 -0
- package/dist/resources/context/node-type-corpus.md +166 -0
- package/dist/resources/context/node-type-selection-guide.md +96 -0
- package/dist/resources/context/overview.md +135 -0
- package/dist/resources/context/pipeline-workflows.md +355 -0
- package/dist/resources/context/run-operations.md +55 -0
- package/dist/resources/context/sql-bigquery.md +41 -0
- package/dist/resources/context/sql-databricks.md +40 -0
- package/dist/resources/context/sql-platform-selection.md +70 -0
- package/dist/resources/context/sql-snowflake.md +43 -0
- package/dist/resources/context/storage-mappings.md +49 -0
- package/dist/resources/context/tool-usage.md +98 -0
- package/dist/resources/index.d.ts +5 -0
- package/dist/resources/index.js +254 -0
- package/dist/schemas/node-payloads.d.ts +5019 -0
- package/dist/schemas/node-payloads.js +147 -0
- package/dist/server.d.ts +7 -0
- package/dist/server.js +63 -0
- package/dist/services/cache/snapshots.d.ts +108 -0
- package/dist/services/cache/snapshots.js +275 -0
- package/dist/services/config/context-analyzer.d.ts +14 -0
- package/dist/services/config/context-analyzer.js +76 -0
- package/dist/services/config/field-classifier.d.ts +23 -0
- package/dist/services/config/field-classifier.js +47 -0
- package/dist/services/config/intelligent.d.ts +55 -0
- package/dist/services/config/intelligent.js +306 -0
- package/dist/services/config/rules.d.ts +6 -0
- package/dist/services/config/rules.js +44 -0
- package/dist/services/config/schema-resolver.d.ts +18 -0
- package/dist/services/config/schema-resolver.js +80 -0
- package/dist/services/corpus/loader.d.ts +56 -0
- package/dist/services/corpus/loader.js +25 -0
- package/dist/services/corpus/search.d.ts +49 -0
- package/dist/services/corpus/search.js +69 -0
- package/dist/services/corpus/templates.d.ts +4 -0
- package/dist/services/corpus/templates.js +11 -0
- package/dist/services/pipelines/execution.d.ts +20 -0
- package/dist/services/pipelines/execution.js +290 -0
- package/dist/services/pipelines/node-type-intent.d.ts +96 -0
- package/dist/services/pipelines/node-type-intent.js +356 -0
- package/dist/services/pipelines/node-type-selection.d.ts +66 -0
- package/dist/services/pipelines/node-type-selection.js +758 -0
- package/dist/services/pipelines/planning.d.ts +543 -0
- package/dist/services/pipelines/planning.js +1839 -0
- package/dist/services/policies/sql-override.d.ts +7 -0
- package/dist/services/policies/sql-override.js +109 -0
- package/dist/services/repo/operations.d.ts +6 -0
- package/dist/services/repo/operations.js +10 -0
- package/dist/services/repo/parser.d.ts +70 -0
- package/dist/services/repo/parser.js +365 -0
- package/dist/services/repo/path.d.ts +2 -0
- package/dist/services/repo/path.js +58 -0
- package/dist/services/templates/nodes.d.ts +50 -0
- package/dist/services/templates/nodes.js +336 -0
- package/dist/services/workspace/analysis.d.ts +56 -0
- package/dist/services/workspace/analysis.js +151 -0
- package/dist/services/workspace/mutations.d.ts +150 -0
- package/dist/services/workspace/mutations.js +1718 -0
- package/dist/utils.d.ts +5 -0
- package/dist/utils.js +7 -0
- package/dist/workflows/get-environment-overview.d.ts +9 -0
- package/dist/workflows/get-environment-overview.js +23 -0
- package/dist/workflows/get-run-details.d.ts +10 -0
- package/dist/workflows/get-run-details.js +28 -0
- package/dist/workflows/progress.d.ts +20 -0
- package/dist/workflows/progress.js +54 -0
- package/dist/workflows/retry-and-wait.d.ts +13 -0
- package/dist/workflows/retry-and-wait.js +139 -0
- package/dist/workflows/run-and-wait.d.ts +13 -0
- package/dist/workflows/run-and-wait.js +141 -0
- package/dist/workflows/run-status.d.ts +10 -0
- package/dist/workflows/run-status.js +27 -0
- package/package.json +34 -0
|
@@ -0,0 +1,336 @@
|
|
|
1
|
+
import { sanitizeNodeDefinitionSqlOverridePolicy } from "../policies/sql-override.js";
|
|
2
|
+
import { isPlainObject } from "../../utils.js";
|
|
3
|
+
function getString(value) {
|
|
4
|
+
return typeof value === "string" ? value : null;
|
|
5
|
+
}
|
|
6
|
+
function getFirstOptionValue(value) {
|
|
7
|
+
if (!Array.isArray(value) || value.length === 0) {
|
|
8
|
+
return undefined;
|
|
9
|
+
}
|
|
10
|
+
const [first] = value;
|
|
11
|
+
if (isPlainObject(first) &&
|
|
12
|
+
Object.prototype.hasOwnProperty.call(first, "value")) {
|
|
13
|
+
return first.value;
|
|
14
|
+
}
|
|
15
|
+
return first;
|
|
16
|
+
}
|
|
17
|
+
function cloneValue(value) {
|
|
18
|
+
return JSON.parse(JSON.stringify(value));
|
|
19
|
+
}
|
|
20
|
+
function inferDefaultValue(item) {
|
|
21
|
+
if (Object.prototype.hasOwnProperty.call(item, "default")) {
|
|
22
|
+
return cloneValue(item.default);
|
|
23
|
+
}
|
|
24
|
+
const type = getString(item.type);
|
|
25
|
+
switch (type) {
|
|
26
|
+
case "materializationSelector":
|
|
27
|
+
return getFirstOptionValue(item.options) ?? "table";
|
|
28
|
+
case "multisourceToggle":
|
|
29
|
+
return false;
|
|
30
|
+
case "overrideSQLToggle":
|
|
31
|
+
return false;
|
|
32
|
+
case "toggleButton":
|
|
33
|
+
return false;
|
|
34
|
+
case "textBox":
|
|
35
|
+
return "";
|
|
36
|
+
case "dropdownSelector":
|
|
37
|
+
return getFirstOptionValue(item.options);
|
|
38
|
+
default:
|
|
39
|
+
return undefined;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
function inferTargetMappings(item) {
|
|
43
|
+
const type = getString(item.type);
|
|
44
|
+
const attributeName = getString(item.attributeName);
|
|
45
|
+
switch (type) {
|
|
46
|
+
case "materializationSelector": {
|
|
47
|
+
const mappings = [
|
|
48
|
+
{
|
|
49
|
+
targetPath: "materializationType",
|
|
50
|
+
note: "Built-in selector maps to the top-level node materialization field.",
|
|
51
|
+
},
|
|
52
|
+
];
|
|
53
|
+
if (attributeName) {
|
|
54
|
+
mappings.push({
|
|
55
|
+
targetPath: `config.${attributeName}`,
|
|
56
|
+
note: "Built-in selector also persists under config.<attributeName> when the definition supplies a custom attributeName.",
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
return mappings;
|
|
60
|
+
}
|
|
61
|
+
case "multisourceToggle": {
|
|
62
|
+
const mappings = [
|
|
63
|
+
{
|
|
64
|
+
targetPath: "isMultisource",
|
|
65
|
+
note: "Built-in toggle maps to the top-level multisource flag.",
|
|
66
|
+
},
|
|
67
|
+
];
|
|
68
|
+
if (attributeName) {
|
|
69
|
+
mappings.push({
|
|
70
|
+
targetPath: `config.${attributeName}`,
|
|
71
|
+
note: "Built-in toggle also persists under config.<attributeName> when the definition supplies a custom attributeName.",
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
return mappings;
|
|
75
|
+
}
|
|
76
|
+
case "overrideSQLToggle": {
|
|
77
|
+
const mappings = [
|
|
78
|
+
{
|
|
79
|
+
targetPath: "overrideSQL",
|
|
80
|
+
note: "Built-in toggle maps to the top-level override SQL flag.",
|
|
81
|
+
},
|
|
82
|
+
];
|
|
83
|
+
if (attributeName) {
|
|
84
|
+
mappings.push({
|
|
85
|
+
targetPath: `config.${attributeName}`,
|
|
86
|
+
note: "Built-in toggle also persists under config.<attributeName> when the definition supplies a custom attributeName.",
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
return mappings;
|
|
90
|
+
}
|
|
91
|
+
case "columnSelector":
|
|
92
|
+
if (attributeName) {
|
|
93
|
+
return [
|
|
94
|
+
{
|
|
95
|
+
targetPath: `columns[].${attributeName}`,
|
|
96
|
+
note: `Column-level attribute. Set "${attributeName}: true" on each column object in metadata.columns that should be selected. Look up the attributeName in the node type definition file under nodeTypes/ in the local repo.`,
|
|
97
|
+
},
|
|
98
|
+
];
|
|
99
|
+
}
|
|
100
|
+
return [
|
|
101
|
+
{
|
|
102
|
+
targetPath: null,
|
|
103
|
+
note: "columnSelector without attributeName — cannot determine column-level target.",
|
|
104
|
+
},
|
|
105
|
+
];
|
|
106
|
+
default:
|
|
107
|
+
if (attributeName) {
|
|
108
|
+
return [
|
|
109
|
+
{
|
|
110
|
+
targetPath: `config.${attributeName}`,
|
|
111
|
+
note: "Generic node-definition input maps into the hydrated config object.",
|
|
112
|
+
},
|
|
113
|
+
];
|
|
114
|
+
}
|
|
115
|
+
return [
|
|
116
|
+
{
|
|
117
|
+
targetPath: null,
|
|
118
|
+
note: "No attributeName or built-in target mapping was found for this item.",
|
|
119
|
+
},
|
|
120
|
+
];
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
function setByPath(target, path, value) {
|
|
124
|
+
const parts = path.split(".");
|
|
125
|
+
let cursor = target;
|
|
126
|
+
for (let index = 0; index < parts.length - 1; index += 1) {
|
|
127
|
+
const part = parts[index];
|
|
128
|
+
const current = cursor[part];
|
|
129
|
+
if (!isPlainObject(current)) {
|
|
130
|
+
cursor[part] = {};
|
|
131
|
+
}
|
|
132
|
+
cursor = cursor[part];
|
|
133
|
+
}
|
|
134
|
+
cursor[parts[parts.length - 1]] = value;
|
|
135
|
+
}
|
|
136
|
+
function getByPath(source, path) {
|
|
137
|
+
const parts = path.split(".");
|
|
138
|
+
let current = source;
|
|
139
|
+
for (const part of parts) {
|
|
140
|
+
if (!isPlainObject(current) || !(part in current)) {
|
|
141
|
+
return undefined;
|
|
142
|
+
}
|
|
143
|
+
current = current[part];
|
|
144
|
+
}
|
|
145
|
+
return current;
|
|
146
|
+
}
|
|
147
|
+
export function buildSetWorkspaceNodeTemplateFromDefinition(nodeDefinition, options = {}) {
|
|
148
|
+
const sanitizedDefinition = sanitizeNodeDefinitionSqlOverridePolicy(nodeDefinition);
|
|
149
|
+
const configGroups = Array.isArray(sanitizedDefinition.nodeDefinition.config)
|
|
150
|
+
? sanitizedDefinition.nodeDefinition.config.filter(isPlainObject)
|
|
151
|
+
: [];
|
|
152
|
+
const configItemCount = configGroups.reduce((count, group) => {
|
|
153
|
+
const items = Array.isArray(group.items) ? group.items.filter(isPlainObject) : [];
|
|
154
|
+
return count + items.length;
|
|
155
|
+
}, 0);
|
|
156
|
+
const fieldMappings = [];
|
|
157
|
+
const inferredTopLevelFields = {};
|
|
158
|
+
const inferredConfig = {};
|
|
159
|
+
const warnings = [...sanitizedDefinition.warnings];
|
|
160
|
+
configGroups.forEach((group, groupIndex) => {
|
|
161
|
+
const groupName = getString(group.groupName);
|
|
162
|
+
const items = Array.isArray(group.items) ? group.items.filter(isPlainObject) : [];
|
|
163
|
+
items.forEach((item, itemIndex) => {
|
|
164
|
+
const defaultValue = inferDefaultValue(item);
|
|
165
|
+
const targetMappings = inferTargetMappings(item);
|
|
166
|
+
targetMappings.forEach(({ targetPath, note }) => {
|
|
167
|
+
const mapping = {
|
|
168
|
+
groupIndex,
|
|
169
|
+
itemIndex,
|
|
170
|
+
groupName,
|
|
171
|
+
itemType: getString(item.type),
|
|
172
|
+
displayName: getString(item.displayName),
|
|
173
|
+
attributeName: getString(item.attributeName),
|
|
174
|
+
targetPath,
|
|
175
|
+
defaultValue,
|
|
176
|
+
enableIf: getString(item.enableIf),
|
|
177
|
+
note,
|
|
178
|
+
};
|
|
179
|
+
fieldMappings.push(mapping);
|
|
180
|
+
if (!targetPath) {
|
|
181
|
+
warnings.push(`Config item ${groupIndex}.${itemIndex} (${getString(item.type) ?? "unknown"}) does not map cleanly to set-workspace-node body fields.`);
|
|
182
|
+
return;
|
|
183
|
+
}
|
|
184
|
+
if (defaultValue === undefined) {
|
|
185
|
+
warnings.push(`Config item ${targetPath} has no inferred default. Fill it before calling set-workspace-node if the node type requires it.`);
|
|
186
|
+
return;
|
|
187
|
+
}
|
|
188
|
+
if (targetPath.startsWith("config.")) {
|
|
189
|
+
setByPath(inferredConfig, targetPath.replace(/^config\./u, ""), defaultValue);
|
|
190
|
+
return;
|
|
191
|
+
}
|
|
192
|
+
setByPath(inferredTopLevelFields, targetPath, defaultValue);
|
|
193
|
+
});
|
|
194
|
+
});
|
|
195
|
+
});
|
|
196
|
+
const capitalized = getString(sanitizedDefinition.nodeDefinition.capitalized);
|
|
197
|
+
const short = getString(sanitizedDefinition.nodeDefinition.short);
|
|
198
|
+
const plural = getString(sanitizedDefinition.nodeDefinition.plural);
|
|
199
|
+
const tagColor = getString(sanitizedDefinition.nodeDefinition.tagColor);
|
|
200
|
+
const defaultNodeName = options.nodeName ??
|
|
201
|
+
(short ? `${short}_NODE` : capitalized ? `${capitalized.toUpperCase()}_NODE` : "NEW_NODE");
|
|
202
|
+
const nodeType = options.nodeType ?? capitalized ?? "Stage";
|
|
203
|
+
const setWorkspaceNodeBodyTemplate = {
|
|
204
|
+
name: defaultNodeName,
|
|
205
|
+
description: "",
|
|
206
|
+
nodeType,
|
|
207
|
+
...(options.database !== undefined ? { database: options.database } : {}),
|
|
208
|
+
...(options.schema !== undefined ? { schema: options.schema } : {}),
|
|
209
|
+
...(options.locationName !== undefined
|
|
210
|
+
? { locationName: options.locationName }
|
|
211
|
+
: {}),
|
|
212
|
+
...inferredTopLevelFields,
|
|
213
|
+
config: inferredConfig,
|
|
214
|
+
metadata: {
|
|
215
|
+
columns: [],
|
|
216
|
+
sourceMapping: [],
|
|
217
|
+
cteString: "",
|
|
218
|
+
appliedNodeTests: [],
|
|
219
|
+
enabledColumnTestIDs: [],
|
|
220
|
+
},
|
|
221
|
+
};
|
|
222
|
+
return {
|
|
223
|
+
definitionSummary: {
|
|
224
|
+
capitalized,
|
|
225
|
+
short,
|
|
226
|
+
plural,
|
|
227
|
+
tagColor,
|
|
228
|
+
configGroupCount: configGroups.length,
|
|
229
|
+
configItemCount,
|
|
230
|
+
},
|
|
231
|
+
fieldMappings,
|
|
232
|
+
inferredTopLevelFields,
|
|
233
|
+
inferredConfig,
|
|
234
|
+
setWorkspaceNodeBodyTemplate,
|
|
235
|
+
usageGuidance: [
|
|
236
|
+
"Use create-workspace-node-from-predecessor or create-workspace-node-from-scratch first to get a real workspace node ID.",
|
|
237
|
+
"Fill metadata.columns and metadata.sourceMapping before calling set-workspace-node; those arrays are replace-on-write.",
|
|
238
|
+
"Keep materializationType and isMultisource at the top level when the definition uses the built-in selector/toggle items.",
|
|
239
|
+
"Keep generic definition attributes under config.<attributeName>.",
|
|
240
|
+
"If a built-in selector or toggle also defines attributeName, mirror the same value under both the top-level field and config.<attributeName>.",
|
|
241
|
+
"For columnSelector items (e.g., isBusinessKey, isChangeTracking), set the attributeName as a boolean directly on each column in metadata.columns — e.g., { name: 'CUSTOMER_ID', isBusinessKey: true, ... }. Look up attribute names in the node type definition file under nodeTypes/ in the local repo.",
|
|
242
|
+
"Do not add overrideSQL or override.* fields; SQL override is intentionally disallowed in this project.",
|
|
243
|
+
],
|
|
244
|
+
warnings,
|
|
245
|
+
};
|
|
246
|
+
}
|
|
247
|
+
export function compareGeneratedTemplateToWorkspaceNode(generated, workspaceNode) {
|
|
248
|
+
const checks = generated.fieldMappings
|
|
249
|
+
.filter((mapping) => mapping.targetPath)
|
|
250
|
+
.map((mapping) => {
|
|
251
|
+
const targetPath = mapping.targetPath;
|
|
252
|
+
const actualValue = getByPath(workspaceNode, targetPath);
|
|
253
|
+
const inferredDefault = mapping.defaultValue;
|
|
254
|
+
if (actualValue === undefined) {
|
|
255
|
+
return {
|
|
256
|
+
targetPath,
|
|
257
|
+
inferredDefault,
|
|
258
|
+
actualValue: undefined,
|
|
259
|
+
status: "missing",
|
|
260
|
+
};
|
|
261
|
+
}
|
|
262
|
+
return {
|
|
263
|
+
targetPath,
|
|
264
|
+
inferredDefault,
|
|
265
|
+
actualValue,
|
|
266
|
+
status: JSON.stringify(actualValue) === JSON.stringify(inferredDefault)
|
|
267
|
+
? "matched"
|
|
268
|
+
: "mismatched",
|
|
269
|
+
};
|
|
270
|
+
});
|
|
271
|
+
return {
|
|
272
|
+
checkedFieldCount: checks.length,
|
|
273
|
+
matchedFieldCount: checks.filter((check) => check.status === "matched").length,
|
|
274
|
+
mismatchedFieldCount: checks.filter((check) => check.status === "mismatched").length,
|
|
275
|
+
missingFieldCount: checks.filter((check) => check.status === "missing").length,
|
|
276
|
+
fields: checks,
|
|
277
|
+
};
|
|
278
|
+
}
|
|
279
|
+
function stringifyYamlScalar(value) {
|
|
280
|
+
if (typeof value === "string") {
|
|
281
|
+
if (value.length === 0) {
|
|
282
|
+
return '""';
|
|
283
|
+
}
|
|
284
|
+
if (/^[A-Za-z0-9_.-]+$/u.test(value)) {
|
|
285
|
+
return value;
|
|
286
|
+
}
|
|
287
|
+
return JSON.stringify(value);
|
|
288
|
+
}
|
|
289
|
+
if (typeof value === "number" || typeof value === "boolean") {
|
|
290
|
+
return String(value);
|
|
291
|
+
}
|
|
292
|
+
if (value === null) {
|
|
293
|
+
return "null";
|
|
294
|
+
}
|
|
295
|
+
return JSON.stringify(value);
|
|
296
|
+
}
|
|
297
|
+
function renderYamlValue(value, indentLevel) {
|
|
298
|
+
const indent = " ".repeat(indentLevel);
|
|
299
|
+
if (Array.isArray(value)) {
|
|
300
|
+
if (value.length === 0) {
|
|
301
|
+
return [`${indent}[]`];
|
|
302
|
+
}
|
|
303
|
+
const lines = [];
|
|
304
|
+
value.forEach((entry) => {
|
|
305
|
+
if (Array.isArray(entry) || isPlainObject(entry)) {
|
|
306
|
+
lines.push(`${indent}-`);
|
|
307
|
+
lines.push(...renderYamlValue(entry, indentLevel + 1));
|
|
308
|
+
}
|
|
309
|
+
else {
|
|
310
|
+
lines.push(`${indent}- ${stringifyYamlScalar(entry)}`);
|
|
311
|
+
}
|
|
312
|
+
});
|
|
313
|
+
return lines;
|
|
314
|
+
}
|
|
315
|
+
if (isPlainObject(value)) {
|
|
316
|
+
const entries = Object.entries(value);
|
|
317
|
+
if (entries.length === 0) {
|
|
318
|
+
return [`${indent}{}`];
|
|
319
|
+
}
|
|
320
|
+
const lines = [];
|
|
321
|
+
for (const [key, entryValue] of entries) {
|
|
322
|
+
if (Array.isArray(entryValue) || isPlainObject(entryValue)) {
|
|
323
|
+
lines.push(`${indent}${key}:`);
|
|
324
|
+
lines.push(...renderYamlValue(entryValue, indentLevel + 1));
|
|
325
|
+
}
|
|
326
|
+
else {
|
|
327
|
+
lines.push(`${indent}${key}: ${stringifyYamlScalar(entryValue)}`);
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
return lines;
|
|
331
|
+
}
|
|
332
|
+
return [`${indent}${stringifyYamlScalar(value)}`];
|
|
333
|
+
}
|
|
334
|
+
export function renderYaml(value) {
|
|
335
|
+
return `${renderYamlValue(value, 0).join("\n")}\n`;
|
|
336
|
+
}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Workspace analysis functions for detecting patterns in Coalesce workspace nodes.
|
|
3
|
+
* All functions are pure (no I/O) and operate on node data arrays.
|
|
4
|
+
*/
|
|
5
|
+
export interface NodeSummary {
|
|
6
|
+
nodeType: string;
|
|
7
|
+
name: string;
|
|
8
|
+
predecessors?: string[];
|
|
9
|
+
}
|
|
10
|
+
export interface PackageDetectionResult {
|
|
11
|
+
packages: string[];
|
|
12
|
+
packageAdoption: Record<string, boolean>;
|
|
13
|
+
builtInTypes: string[];
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Detect package usage patterns in a workspace by scanning observed node type prefixes.
|
|
17
|
+
* Presence of any node with a package prefix is strong evidence the package is installed and in use,
|
|
18
|
+
* but absence is not a reliable negative signal because this is not an installed-type registry.
|
|
19
|
+
*/
|
|
20
|
+
export declare function detectPackages(nodes: NodeSummary[]): PackageDetectionResult;
|
|
21
|
+
export type NodeLayer = "bronze" | "staging" | "intermediate" | "mart" | "unknown";
|
|
22
|
+
export interface LayerSummary {
|
|
23
|
+
nodeTypes: string[];
|
|
24
|
+
count: number;
|
|
25
|
+
}
|
|
26
|
+
export interface LayerAnalysis {
|
|
27
|
+
bronze: LayerSummary;
|
|
28
|
+
staging: LayerSummary;
|
|
29
|
+
intermediate: LayerSummary;
|
|
30
|
+
mart: LayerSummary;
|
|
31
|
+
unknown: LayerSummary;
|
|
32
|
+
}
|
|
33
|
+
export declare function inferNodeLayer(node: NodeSummary): NodeLayer;
|
|
34
|
+
export declare function inferLayers(nodes: NodeSummary[]): LayerAnalysis;
|
|
35
|
+
export type Methodology = "kimball" | "data-vault" | "dbt-style" | "mixed";
|
|
36
|
+
export declare function detectMethodology(nodes: NodeSummary[]): Methodology;
|
|
37
|
+
export interface WorkspaceProfile {
|
|
38
|
+
workspaceID: string;
|
|
39
|
+
analyzedAt: string;
|
|
40
|
+
nodeCount: number;
|
|
41
|
+
packageAdoption: PackageDetectionResult;
|
|
42
|
+
layerPatterns: LayerAnalysis;
|
|
43
|
+
methodology: Methodology;
|
|
44
|
+
recommendations: {
|
|
45
|
+
defaultPackage: string | null;
|
|
46
|
+
stagingType: string;
|
|
47
|
+
transformType: string;
|
|
48
|
+
dimensionType: string;
|
|
49
|
+
factType: string;
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Build a complete workspace profile from a list of nodes.
|
|
54
|
+
* This is the main entry point for workspace analysis.
|
|
55
|
+
*/
|
|
56
|
+
export declare function buildWorkspaceProfile(workspaceID: string, nodes: NodeSummary[]): WorkspaceProfile;
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Workspace analysis functions for detecting patterns in Coalesce workspace nodes.
|
|
3
|
+
* All functions are pure (no I/O) and operate on node data arrays.
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* Detect package usage patterns in a workspace by scanning observed node type prefixes.
|
|
7
|
+
* Presence of any node with a package prefix is strong evidence the package is installed and in use,
|
|
8
|
+
* but absence is not a reliable negative signal because this is not an installed-type registry.
|
|
9
|
+
*/
|
|
10
|
+
export function detectPackages(nodes) {
|
|
11
|
+
const packageSet = new Set();
|
|
12
|
+
const builtInSet = new Set();
|
|
13
|
+
for (const node of nodes) {
|
|
14
|
+
const separatorIndex = node.nodeType.indexOf(":::");
|
|
15
|
+
if (separatorIndex > 0) {
|
|
16
|
+
packageSet.add(node.nodeType.substring(0, separatorIndex));
|
|
17
|
+
}
|
|
18
|
+
else {
|
|
19
|
+
builtInSet.add(node.nodeType);
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
const packages = Array.from(packageSet).sort();
|
|
23
|
+
const packageAdoption = {};
|
|
24
|
+
for (const pkg of packages) {
|
|
25
|
+
packageAdoption[pkg] = true;
|
|
26
|
+
}
|
|
27
|
+
return {
|
|
28
|
+
packages,
|
|
29
|
+
packageAdoption,
|
|
30
|
+
builtInTypes: Array.from(builtInSet).sort(),
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
const LAYER_NAME_PATTERNS = [
|
|
34
|
+
[/^(RAW_|SRC_|LANDING_|L0_)/i, "bronze"],
|
|
35
|
+
[/^(STG_|STAGE_|CLEAN_|L1_)/i, "staging"],
|
|
36
|
+
[/^(INT_|TMP_|WORK_|TRANSFORM_)/i, "intermediate"],
|
|
37
|
+
[/^(DIM_|DIMENSION_|FACT_|FCT_|MART_|RPT_)/i, "mart"],
|
|
38
|
+
];
|
|
39
|
+
const MART_NODE_TYPES = new Set(["Dimension", "Fact"]);
|
|
40
|
+
export function inferNodeLayer(node) {
|
|
41
|
+
const upperName = node.name.toUpperCase();
|
|
42
|
+
for (const [pattern, layer] of LAYER_NAME_PATTERNS) {
|
|
43
|
+
if (pattern.test(upperName)) {
|
|
44
|
+
return layer;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
const baseType = node.nodeType.includes(":::")
|
|
48
|
+
? node.nodeType.split(":::")[1]
|
|
49
|
+
: node.nodeType;
|
|
50
|
+
if (MART_NODE_TYPES.has(baseType)) {
|
|
51
|
+
return "mart";
|
|
52
|
+
}
|
|
53
|
+
return "unknown";
|
|
54
|
+
}
|
|
55
|
+
export function inferLayers(nodes) {
|
|
56
|
+
const layers = {
|
|
57
|
+
bronze: { types: new Set(), count: 0 },
|
|
58
|
+
staging: { types: new Set(), count: 0 },
|
|
59
|
+
intermediate: { types: new Set(), count: 0 },
|
|
60
|
+
mart: { types: new Set(), count: 0 },
|
|
61
|
+
unknown: { types: new Set(), count: 0 },
|
|
62
|
+
};
|
|
63
|
+
for (const node of nodes) {
|
|
64
|
+
const layer = inferNodeLayer(node);
|
|
65
|
+
layers[layer].types.add(node.nodeType);
|
|
66
|
+
layers[layer].count += 1;
|
|
67
|
+
}
|
|
68
|
+
const toSummary = (entry) => ({
|
|
69
|
+
nodeTypes: Array.from(entry.types).sort(),
|
|
70
|
+
count: entry.count,
|
|
71
|
+
});
|
|
72
|
+
return {
|
|
73
|
+
bronze: toSummary(layers.bronze),
|
|
74
|
+
staging: toSummary(layers.staging),
|
|
75
|
+
intermediate: toSummary(layers.intermediate),
|
|
76
|
+
mart: toSummary(layers.mart),
|
|
77
|
+
unknown: toSummary(layers.unknown),
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
export function detectMethodology(nodes) {
|
|
81
|
+
if (nodes.length === 0) {
|
|
82
|
+
return "mixed";
|
|
83
|
+
}
|
|
84
|
+
const upperNames = nodes.map((n) => n.name.toUpperCase());
|
|
85
|
+
// Data Vault signals: HUB_, SAT_, LINK_ naming
|
|
86
|
+
const hubCount = upperNames.filter((n) => /^HUB_|_HUB$/.test(n)).length;
|
|
87
|
+
const satCount = upperNames.filter((n) => /^SAT_|_SAT$/.test(n)).length;
|
|
88
|
+
const linkCount = upperNames.filter((n) => /^LINK_|_LINK$/.test(n)).length;
|
|
89
|
+
if (hubCount >= 1 && satCount >= 1) {
|
|
90
|
+
return "data-vault";
|
|
91
|
+
}
|
|
92
|
+
// Kimball signals: DIM_/FACT_ naming or Dimension/Fact node types
|
|
93
|
+
const dimCount = nodes.filter((n) => /^DIM_|^DIMENSION_/i.test(n.name) ||
|
|
94
|
+
n.nodeType === "Dimension" ||
|
|
95
|
+
n.nodeType.endsWith(":::Dimension")).length;
|
|
96
|
+
const factCount = nodes.filter((n) => /^FACT_|^FCT_/i.test(n.name) ||
|
|
97
|
+
n.nodeType === "Fact" ||
|
|
98
|
+
n.nodeType.endsWith(":::Fact")).length;
|
|
99
|
+
if (dimCount >= 1 && factCount >= 1) {
|
|
100
|
+
return "kimball";
|
|
101
|
+
}
|
|
102
|
+
// dbt-style signals: stg_/int_/fct_ lowercase naming with view intermediates
|
|
103
|
+
const stgCount = nodes.filter((n) => /^stg_/i.test(n.name)).length;
|
|
104
|
+
const intCount = nodes.filter((n) => /^int_/i.test(n.name)).length;
|
|
105
|
+
if (stgCount >= 1 && intCount >= 1) {
|
|
106
|
+
return "dbt-style";
|
|
107
|
+
}
|
|
108
|
+
return "mixed";
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* Build a complete workspace profile from a list of nodes.
|
|
112
|
+
* This is the main entry point for workspace analysis.
|
|
113
|
+
*/
|
|
114
|
+
export function buildWorkspaceProfile(workspaceID, nodes) {
|
|
115
|
+
const packageAdoption = detectPackages(nodes);
|
|
116
|
+
const layerPatterns = inferLayers(nodes);
|
|
117
|
+
const methodology = detectMethodology(nodes);
|
|
118
|
+
const preferredPackage = packageAdoption.packages.includes("base-nodes")
|
|
119
|
+
? "base-nodes"
|
|
120
|
+
: packageAdoption.packages[0] ?? null;
|
|
121
|
+
const prefix = preferredPackage ? `${preferredPackage}:::` : "";
|
|
122
|
+
const findDominantType = (layer, fallback) => {
|
|
123
|
+
if (layer.nodeTypes.length === 0) {
|
|
124
|
+
return prefix ? `${prefix}${fallback}` : fallback;
|
|
125
|
+
}
|
|
126
|
+
// Prefer the packaged version matching fallback, then any packaged, then matching built-in, then first
|
|
127
|
+
const packagedMatch = layer.nodeTypes.find((t) => t.includes(":::") && t.endsWith(`:::${fallback}`));
|
|
128
|
+
if (packagedMatch)
|
|
129
|
+
return packagedMatch;
|
|
130
|
+
const builtInMatch = layer.nodeTypes.find((t) => t === fallback);
|
|
131
|
+
if (builtInMatch)
|
|
132
|
+
return builtInMatch;
|
|
133
|
+
const packaged = layer.nodeTypes.find((t) => t.includes(":::"));
|
|
134
|
+
return packaged ?? layer.nodeTypes[0];
|
|
135
|
+
};
|
|
136
|
+
return {
|
|
137
|
+
workspaceID,
|
|
138
|
+
analyzedAt: new Date().toISOString(),
|
|
139
|
+
nodeCount: nodes.length,
|
|
140
|
+
packageAdoption,
|
|
141
|
+
layerPatterns,
|
|
142
|
+
methodology,
|
|
143
|
+
recommendations: {
|
|
144
|
+
defaultPackage: preferredPackage,
|
|
145
|
+
stagingType: findDominantType(layerPatterns.staging, "Stage"),
|
|
146
|
+
transformType: findDominantType(layerPatterns.intermediate, "View"),
|
|
147
|
+
dimensionType: findDominantType(layerPatterns.mart, "Dimension"),
|
|
148
|
+
factType: findDominantType(layerPatterns.mart, "Fact"),
|
|
149
|
+
},
|
|
150
|
+
};
|
|
151
|
+
}
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
import type { CoalesceClient } from "../../client.js";
|
|
2
|
+
import { type ConfigCompletionResult } from "../config/intelligent.js";
|
|
3
|
+
type ScratchNodeCompletionLevel = "created" | "named" | "configured";
|
|
4
|
+
export type WorkspaceNodeChanges = Record<string, unknown>;
|
|
5
|
+
export declare function buildUpdatedWorkspaceNodeBody(current: unknown, changes: WorkspaceNodeChanges): Record<string, unknown>;
|
|
6
|
+
type JoinColumnSuggestion = {
|
|
7
|
+
normalizedName: string;
|
|
8
|
+
leftColumnName: string;
|
|
9
|
+
rightColumnName: string;
|
|
10
|
+
};
|
|
11
|
+
type JoinSuggestion = {
|
|
12
|
+
leftPredecessorNodeID: string;
|
|
13
|
+
leftPredecessorName: string | null;
|
|
14
|
+
rightPredecessorNodeID: string;
|
|
15
|
+
rightPredecessorName: string | null;
|
|
16
|
+
commonColumns: JoinColumnSuggestion[];
|
|
17
|
+
};
|
|
18
|
+
type GroupByAnalysis = {
|
|
19
|
+
groupByColumns: string[];
|
|
20
|
+
aggregateColumns: {
|
|
21
|
+
name: string;
|
|
22
|
+
transform: string;
|
|
23
|
+
}[];
|
|
24
|
+
hasAggregates: boolean;
|
|
25
|
+
groupByClause: string;
|
|
26
|
+
validation: {
|
|
27
|
+
valid: boolean;
|
|
28
|
+
errors: string[];
|
|
29
|
+
};
|
|
30
|
+
};
|
|
31
|
+
export declare function updateWorkspaceNode(client: CoalesceClient, params: {
|
|
32
|
+
workspaceID: string;
|
|
33
|
+
nodeID: string;
|
|
34
|
+
changes: Record<string, unknown>;
|
|
35
|
+
}): Promise<unknown>;
|
|
36
|
+
export declare function replaceWorkspaceNodeColumns(client: CoalesceClient, params: {
|
|
37
|
+
workspaceID: string;
|
|
38
|
+
nodeID: string;
|
|
39
|
+
columns: unknown[];
|
|
40
|
+
whereCondition?: string;
|
|
41
|
+
additionalChanges?: Record<string, unknown>;
|
|
42
|
+
}): Promise<unknown>;
|
|
43
|
+
export declare function createWorkspaceNodeFromScratch(client: CoalesceClient, params: {
|
|
44
|
+
workspaceID: string;
|
|
45
|
+
nodeType: string;
|
|
46
|
+
completionLevel?: ScratchNodeCompletionLevel;
|
|
47
|
+
name?: string;
|
|
48
|
+
description?: string;
|
|
49
|
+
storageLocations?: unknown[];
|
|
50
|
+
config?: Record<string, unknown>;
|
|
51
|
+
metadata?: Record<string, unknown>;
|
|
52
|
+
changes?: Record<string, unknown>;
|
|
53
|
+
repoPath?: string;
|
|
54
|
+
goal?: string;
|
|
55
|
+
}): Promise<unknown>;
|
|
56
|
+
export declare function createWorkspaceNodeFromPredecessor(client: CoalesceClient, params: {
|
|
57
|
+
workspaceID: string;
|
|
58
|
+
nodeType: string;
|
|
59
|
+
predecessorNodeIDs: string[];
|
|
60
|
+
changes?: Record<string, unknown>;
|
|
61
|
+
repoPath?: string;
|
|
62
|
+
goal?: string;
|
|
63
|
+
/** Replace auto-populated columns with these specific columns+transforms in a single call. */
|
|
64
|
+
columns?: Array<{
|
|
65
|
+
name: string;
|
|
66
|
+
transform?: string;
|
|
67
|
+
dataType?: string;
|
|
68
|
+
description?: string;
|
|
69
|
+
}>;
|
|
70
|
+
/** WHERE filter to append to the joinCondition. Only valid with `columns`, not with aggregation. */
|
|
71
|
+
whereCondition?: string;
|
|
72
|
+
/** GROUP BY columns for aggregation. Must be provided with `aggregates`. */
|
|
73
|
+
groupByColumns?: string[];
|
|
74
|
+
/** Aggregate columns. Must be provided with `groupByColumns`. */
|
|
75
|
+
aggregates?: Array<{
|
|
76
|
+
name: string;
|
|
77
|
+
function: string;
|
|
78
|
+
expression: string;
|
|
79
|
+
description?: string;
|
|
80
|
+
}>;
|
|
81
|
+
/** JOIN type for aggregation nodes. Only used with groupByColumns/aggregates. */
|
|
82
|
+
joinType?: "INNER JOIN" | "LEFT JOIN" | "RIGHT JOIN" | "FULL OUTER JOIN";
|
|
83
|
+
}): Promise<unknown>;
|
|
84
|
+
export declare function convertJoinToAggregation(client: CoalesceClient, params: {
|
|
85
|
+
workspaceID: string;
|
|
86
|
+
nodeID: string;
|
|
87
|
+
groupByColumns: string[];
|
|
88
|
+
aggregates: Array<{
|
|
89
|
+
name: string;
|
|
90
|
+
function: string;
|
|
91
|
+
expression: string;
|
|
92
|
+
description?: string;
|
|
93
|
+
}>;
|
|
94
|
+
joinType?: "INNER JOIN" | "LEFT JOIN" | "RIGHT JOIN" | "FULL OUTER JOIN";
|
|
95
|
+
maintainJoins?: boolean;
|
|
96
|
+
repoPath?: string;
|
|
97
|
+
}): Promise<{
|
|
98
|
+
node: unknown;
|
|
99
|
+
joinSQL: {
|
|
100
|
+
fromClause: string;
|
|
101
|
+
joinClauses: string[];
|
|
102
|
+
fullSQL: string;
|
|
103
|
+
warnings: string[];
|
|
104
|
+
};
|
|
105
|
+
groupByAnalysis: GroupByAnalysis;
|
|
106
|
+
validation: {
|
|
107
|
+
valid: boolean;
|
|
108
|
+
warnings: string[];
|
|
109
|
+
};
|
|
110
|
+
configCompletion?: ConfigCompletionResult;
|
|
111
|
+
configCompletionSkipped?: string;
|
|
112
|
+
}>;
|
|
113
|
+
export declare function applyJoinCondition(client: CoalesceClient, params: {
|
|
114
|
+
workspaceID: string;
|
|
115
|
+
nodeID: string;
|
|
116
|
+
joinType?: "INNER JOIN" | "LEFT JOIN" | "RIGHT JOIN" | "FULL OUTER JOIN";
|
|
117
|
+
whereClause?: string;
|
|
118
|
+
qualifyClause?: string;
|
|
119
|
+
joinColumnOverrides?: Array<{
|
|
120
|
+
leftPredecessor: string;
|
|
121
|
+
rightPredecessor: string;
|
|
122
|
+
leftColumn: string;
|
|
123
|
+
rightColumn: string;
|
|
124
|
+
}>;
|
|
125
|
+
}): Promise<{
|
|
126
|
+
joinCondition: string;
|
|
127
|
+
joinSuggestions: JoinSuggestion[];
|
|
128
|
+
predecessors: Array<{
|
|
129
|
+
nodeID: string;
|
|
130
|
+
nodeName: string;
|
|
131
|
+
locationName: string;
|
|
132
|
+
columnCount: number;
|
|
133
|
+
}>;
|
|
134
|
+
warnings: string[];
|
|
135
|
+
}>;
|
|
136
|
+
/**
|
|
137
|
+
* Returns the distinct node types observed in existing workspace nodes.
|
|
138
|
+
* This is intentionally observation-based and should not be treated as a true
|
|
139
|
+
* installed-type registry for the workspace.
|
|
140
|
+
*/
|
|
141
|
+
export declare function listWorkspaceNodeTypes(client: CoalesceClient, params: {
|
|
142
|
+
workspaceID: string;
|
|
143
|
+
}): Promise<{
|
|
144
|
+
workspaceID: string;
|
|
145
|
+
basis: "observed_nodes";
|
|
146
|
+
nodeTypes: string[];
|
|
147
|
+
counts: Record<string, number>;
|
|
148
|
+
total: number;
|
|
149
|
+
}>;
|
|
150
|
+
export {};
|