coalesce-transform-mcp 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +304 -0
- package/dist/cache-dir.d.ts +26 -0
- package/dist/cache-dir.js +106 -0
- package/dist/client.d.ts +25 -0
- package/dist/client.js +212 -0
- package/dist/coalesce/api/environments.d.ts +20 -0
- package/dist/coalesce/api/environments.js +15 -0
- package/dist/coalesce/api/git-accounts.d.ts +21 -0
- package/dist/coalesce/api/git-accounts.js +21 -0
- package/dist/coalesce/api/jobs.d.ts +25 -0
- package/dist/coalesce/api/jobs.js +21 -0
- package/dist/coalesce/api/nodes.d.ts +29 -0
- package/dist/coalesce/api/nodes.js +33 -0
- package/dist/coalesce/api/projects.d.ts +22 -0
- package/dist/coalesce/api/projects.js +25 -0
- package/dist/coalesce/api/runs.d.ts +19 -0
- package/dist/coalesce/api/runs.js +34 -0
- package/dist/coalesce/api/subgraphs.d.ts +20 -0
- package/dist/coalesce/api/subgraphs.js +17 -0
- package/dist/coalesce/api/users.d.ts +30 -0
- package/dist/coalesce/api/users.js +31 -0
- package/dist/coalesce/types.d.ts +298 -0
- package/dist/coalesce/types.js +746 -0
- package/dist/generated/.gitkeep +0 -0
- package/dist/generated/node-type-corpus.json +42656 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +10 -0
- package/dist/mcp/cache.d.ts +3 -0
- package/dist/mcp/cache.js +137 -0
- package/dist/mcp/environments.d.ts +3 -0
- package/dist/mcp/environments.js +61 -0
- package/dist/mcp/git-accounts.d.ts +3 -0
- package/dist/mcp/git-accounts.js +70 -0
- package/dist/mcp/jobs.d.ts +3 -0
- package/dist/mcp/jobs.js +77 -0
- package/dist/mcp/node-type-corpus.d.ts +3 -0
- package/dist/mcp/node-type-corpus.js +173 -0
- package/dist/mcp/nodes.d.ts +3 -0
- package/dist/mcp/nodes.js +341 -0
- package/dist/mcp/pipelines.d.ts +3 -0
- package/dist/mcp/pipelines.js +342 -0
- package/dist/mcp/projects.d.ts +3 -0
- package/dist/mcp/projects.js +70 -0
- package/dist/mcp/repo-node-types.d.ts +135 -0
- package/dist/mcp/repo-node-types.js +387 -0
- package/dist/mcp/runs.d.ts +3 -0
- package/dist/mcp/runs.js +92 -0
- package/dist/mcp/subgraphs.d.ts +3 -0
- package/dist/mcp/subgraphs.js +60 -0
- package/dist/mcp/users.d.ts +3 -0
- package/dist/mcp/users.js +107 -0
- package/dist/prompts/index.d.ts +2 -0
- package/dist/prompts/index.js +58 -0
- package/dist/resources/context/aggregation-patterns.md +145 -0
- package/dist/resources/context/data-engineering-principles.md +183 -0
- package/dist/resources/context/hydrated-metadata.md +92 -0
- package/dist/resources/context/id-discovery.md +64 -0
- package/dist/resources/context/intelligent-node-configuration.md +162 -0
- package/dist/resources/context/node-creation-decision-tree.md +156 -0
- package/dist/resources/context/node-operations.md +316 -0
- package/dist/resources/context/node-payloads.md +114 -0
- package/dist/resources/context/node-type-corpus.md +166 -0
- package/dist/resources/context/node-type-selection-guide.md +96 -0
- package/dist/resources/context/overview.md +135 -0
- package/dist/resources/context/pipeline-workflows.md +355 -0
- package/dist/resources/context/run-operations.md +55 -0
- package/dist/resources/context/sql-bigquery.md +41 -0
- package/dist/resources/context/sql-databricks.md +40 -0
- package/dist/resources/context/sql-platform-selection.md +70 -0
- package/dist/resources/context/sql-snowflake.md +43 -0
- package/dist/resources/context/storage-mappings.md +49 -0
- package/dist/resources/context/tool-usage.md +98 -0
- package/dist/resources/index.d.ts +5 -0
- package/dist/resources/index.js +254 -0
- package/dist/schemas/node-payloads.d.ts +5019 -0
- package/dist/schemas/node-payloads.js +147 -0
- package/dist/server.d.ts +7 -0
- package/dist/server.js +63 -0
- package/dist/services/cache/snapshots.d.ts +108 -0
- package/dist/services/cache/snapshots.js +275 -0
- package/dist/services/config/context-analyzer.d.ts +14 -0
- package/dist/services/config/context-analyzer.js +76 -0
- package/dist/services/config/field-classifier.d.ts +23 -0
- package/dist/services/config/field-classifier.js +47 -0
- package/dist/services/config/intelligent.d.ts +55 -0
- package/dist/services/config/intelligent.js +306 -0
- package/dist/services/config/rules.d.ts +6 -0
- package/dist/services/config/rules.js +44 -0
- package/dist/services/config/schema-resolver.d.ts +18 -0
- package/dist/services/config/schema-resolver.js +80 -0
- package/dist/services/corpus/loader.d.ts +56 -0
- package/dist/services/corpus/loader.js +25 -0
- package/dist/services/corpus/search.d.ts +49 -0
- package/dist/services/corpus/search.js +69 -0
- package/dist/services/corpus/templates.d.ts +4 -0
- package/dist/services/corpus/templates.js +11 -0
- package/dist/services/pipelines/execution.d.ts +20 -0
- package/dist/services/pipelines/execution.js +290 -0
- package/dist/services/pipelines/node-type-intent.d.ts +96 -0
- package/dist/services/pipelines/node-type-intent.js +356 -0
- package/dist/services/pipelines/node-type-selection.d.ts +66 -0
- package/dist/services/pipelines/node-type-selection.js +758 -0
- package/dist/services/pipelines/planning.d.ts +543 -0
- package/dist/services/pipelines/planning.js +1839 -0
- package/dist/services/policies/sql-override.d.ts +7 -0
- package/dist/services/policies/sql-override.js +109 -0
- package/dist/services/repo/operations.d.ts +6 -0
- package/dist/services/repo/operations.js +10 -0
- package/dist/services/repo/parser.d.ts +70 -0
- package/dist/services/repo/parser.js +365 -0
- package/dist/services/repo/path.d.ts +2 -0
- package/dist/services/repo/path.js +58 -0
- package/dist/services/templates/nodes.d.ts +50 -0
- package/dist/services/templates/nodes.js +336 -0
- package/dist/services/workspace/analysis.d.ts +56 -0
- package/dist/services/workspace/analysis.js +151 -0
- package/dist/services/workspace/mutations.d.ts +150 -0
- package/dist/services/workspace/mutations.js +1718 -0
- package/dist/utils.d.ts +5 -0
- package/dist/utils.js +7 -0
- package/dist/workflows/get-environment-overview.d.ts +9 -0
- package/dist/workflows/get-environment-overview.js +23 -0
- package/dist/workflows/get-run-details.d.ts +10 -0
- package/dist/workflows/get-run-details.js +28 -0
- package/dist/workflows/progress.d.ts +20 -0
- package/dist/workflows/progress.js +54 -0
- package/dist/workflows/retry-and-wait.d.ts +13 -0
- package/dist/workflows/retry-and-wait.js +139 -0
- package/dist/workflows/run-and-wait.d.ts +13 -0
- package/dist/workflows/run-and-wait.js +141 -0
- package/dist/workflows/run-status.d.ts +10 -0
- package/dist/workflows/run-status.js +27 -0
- package/package.json +34 -0
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
export const StorageLocationInputSchema = z
|
|
3
|
+
.object({
|
|
4
|
+
locationName: z
|
|
5
|
+
.string()
|
|
6
|
+
.optional()
|
|
7
|
+
.describe("Common storage location field used in hydrated node bodies."),
|
|
8
|
+
name: z
|
|
9
|
+
.string()
|
|
10
|
+
.optional()
|
|
11
|
+
.describe("Alternative storage location name field used by some node types."),
|
|
12
|
+
database: z.string().optional().describe("Optional database for the storage location."),
|
|
13
|
+
schema: z.string().optional().describe("Optional schema for the storage location."),
|
|
14
|
+
})
|
|
15
|
+
.passthrough();
|
|
16
|
+
const ColumnReferenceInputSchema = z
|
|
17
|
+
.object({
|
|
18
|
+
nodeID: z.string().optional(),
|
|
19
|
+
columnID: z.string().optional(),
|
|
20
|
+
columnName: z.string().optional(),
|
|
21
|
+
stepCounter: z.string().optional(),
|
|
22
|
+
columnCounter: z.string().optional(),
|
|
23
|
+
})
|
|
24
|
+
.passthrough();
|
|
25
|
+
const ColumnSourceInputSchema = z
|
|
26
|
+
.object({
|
|
27
|
+
name: z.string().optional(),
|
|
28
|
+
transform: z.string().optional(),
|
|
29
|
+
columnReferences: z.array(ColumnReferenceInputSchema).optional(),
|
|
30
|
+
})
|
|
31
|
+
.passthrough();
|
|
32
|
+
export const WorkspaceNodeColumnInputSchema = z
|
|
33
|
+
.object({
|
|
34
|
+
name: z.string().describe("Column name."),
|
|
35
|
+
transform: z
|
|
36
|
+
.string()
|
|
37
|
+
.optional()
|
|
38
|
+
.describe("Optional SQL expression or source reference for the column."),
|
|
39
|
+
dataType: z.string().optional().describe("Optional Coalesce data type."),
|
|
40
|
+
description: z.string().optional().describe("Optional column description."),
|
|
41
|
+
nullable: z.boolean().optional().describe("Optional nullability flag."),
|
|
42
|
+
columnID: z
|
|
43
|
+
.string()
|
|
44
|
+
.optional()
|
|
45
|
+
.describe("Existing Coalesce column ID when preserving hydrated metadata."),
|
|
46
|
+
columnReference: ColumnReferenceInputSchema
|
|
47
|
+
.optional()
|
|
48
|
+
.describe("Optional lineage reference preserved from hydrated node bodies."),
|
|
49
|
+
sources: z
|
|
50
|
+
.array(ColumnSourceInputSchema)
|
|
51
|
+
.optional()
|
|
52
|
+
.describe("Optional lineage/source entries used by hydrated metadata."),
|
|
53
|
+
placement: z
|
|
54
|
+
.union([z.string(), z.number()])
|
|
55
|
+
.optional()
|
|
56
|
+
.describe("Optional placement/order metadata."),
|
|
57
|
+
})
|
|
58
|
+
.passthrough();
|
|
59
|
+
const SourceMappingDependencyInputSchema = z
|
|
60
|
+
.object({
|
|
61
|
+
locationName: z.string().optional(),
|
|
62
|
+
nodeName: z.string().optional(),
|
|
63
|
+
nodeID: z.string().optional(),
|
|
64
|
+
})
|
|
65
|
+
.passthrough();
|
|
66
|
+
const SourceMappingJoinInputSchema = z
|
|
67
|
+
.object({
|
|
68
|
+
joinCondition: z
|
|
69
|
+
.string()
|
|
70
|
+
.optional()
|
|
71
|
+
.describe("Optional FROM/JOIN/WHERE clause stored on the mapping entry."),
|
|
72
|
+
})
|
|
73
|
+
.passthrough();
|
|
74
|
+
const SourceMappingCustomSqlInputSchema = z
|
|
75
|
+
.object({
|
|
76
|
+
customSQL: z.string().optional(),
|
|
77
|
+
})
|
|
78
|
+
.passthrough();
|
|
79
|
+
const SourceMappingNoLinkRefInputSchema = z
|
|
80
|
+
.union([z.string(), z.object({}).passthrough()]);
|
|
81
|
+
const SourceMappingInputSchema = z
|
|
82
|
+
.object({
|
|
83
|
+
name: z.string().optional(),
|
|
84
|
+
aliases: z
|
|
85
|
+
.record(z.string())
|
|
86
|
+
.optional()
|
|
87
|
+
.describe("Optional alias-to-node lookup map used by hydrated source mappings."),
|
|
88
|
+
dependencies: z
|
|
89
|
+
.array(SourceMappingDependencyInputSchema)
|
|
90
|
+
.optional()
|
|
91
|
+
.describe("Optional predecessor dependency records."),
|
|
92
|
+
join: SourceMappingJoinInputSchema.optional(),
|
|
93
|
+
customSQL: SourceMappingCustomSqlInputSchema.optional(),
|
|
94
|
+
noLinkRefs: z.array(SourceMappingNoLinkRefInputSchema).optional(),
|
|
95
|
+
})
|
|
96
|
+
.passthrough();
|
|
97
|
+
export const WorkspaceNodeMetadataInputSchema = z
|
|
98
|
+
.object({
|
|
99
|
+
columns: z
|
|
100
|
+
.array(WorkspaceNodeColumnInputSchema)
|
|
101
|
+
.optional()
|
|
102
|
+
.describe("Hydrated mapping-grid columns for the node."),
|
|
103
|
+
sourceMapping: z
|
|
104
|
+
.array(SourceMappingInputSchema)
|
|
105
|
+
.optional()
|
|
106
|
+
.describe("Hydrated source/join mapping entries for the node."),
|
|
107
|
+
enabledColumnTestIDs: z
|
|
108
|
+
.array(z.string())
|
|
109
|
+
.optional()
|
|
110
|
+
.describe("Enabled column test IDs preserved by the Coalesce PUT API."),
|
|
111
|
+
})
|
|
112
|
+
.passthrough();
|
|
113
|
+
export const NodeConfigInputSchema = z
|
|
114
|
+
.object({
|
|
115
|
+
preSQL: z.string().optional().describe("Optional pre-SQL hook."),
|
|
116
|
+
postSQL: z.string().optional().describe("Optional post-SQL hook."),
|
|
117
|
+
testsEnabled: z.boolean().optional().describe("Enable or disable node tests."),
|
|
118
|
+
materializationType: z
|
|
119
|
+
.string()
|
|
120
|
+
.optional()
|
|
121
|
+
.describe("Optional materialization type such as table or view."),
|
|
122
|
+
insertStrategy: z
|
|
123
|
+
.string()
|
|
124
|
+
.optional()
|
|
125
|
+
.describe("Optional insert strategy for nodes that support it."),
|
|
126
|
+
truncateBefore: z
|
|
127
|
+
.boolean()
|
|
128
|
+
.optional()
|
|
129
|
+
.describe("Optional truncate-before-load toggle."),
|
|
130
|
+
})
|
|
131
|
+
.passthrough();
|
|
132
|
+
export const WorkspaceNodeWriteInputSchema = z
|
|
133
|
+
.object({
|
|
134
|
+
id: z.string().optional(),
|
|
135
|
+
name: z.string().optional(),
|
|
136
|
+
description: z.string().optional(),
|
|
137
|
+
nodeType: z.string().optional(),
|
|
138
|
+
table: z.string().optional(),
|
|
139
|
+
database: z.string().optional(),
|
|
140
|
+
schema: z.string().optional(),
|
|
141
|
+
locationName: z.string().optional(),
|
|
142
|
+
materializationType: z.string().optional(),
|
|
143
|
+
storageLocations: z.array(StorageLocationInputSchema).optional(),
|
|
144
|
+
config: NodeConfigInputSchema.optional(),
|
|
145
|
+
metadata: WorkspaceNodeMetadataInputSchema.optional(),
|
|
146
|
+
})
|
|
147
|
+
.passthrough();
|
package/dist/server.d.ts
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
2
|
+
import type { CoalesceClient } from "./client.js";
|
|
3
|
+
export declare const SERVER_NAME = "coalesce-transform-mcp";
|
|
4
|
+
export declare const SERVER_VERSION: string;
|
|
5
|
+
export declare const SERVER_INSTRUCTIONS: string;
|
|
6
|
+
export declare function registerServerSurface(server: McpServer, client: CoalesceClient): void;
|
|
7
|
+
export declare function createCoalesceMcpServer(client: CoalesceClient): McpServer;
|
package/dist/server.js
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import { createRequire } from "node:module";
|
|
2
|
+
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
3
|
+
import { registerEnvironmentTools } from "./mcp/environments.js";
|
|
4
|
+
import { registerNodeTools } from "./mcp/nodes.js";
|
|
5
|
+
import { registerPipelineTools } from "./mcp/pipelines.js";
|
|
6
|
+
import { registerRunTools } from "./mcp/runs.js";
|
|
7
|
+
import { registerProjectTools } from "./mcp/projects.js";
|
|
8
|
+
import { registerGitAccountTools } from "./mcp/git-accounts.js";
|
|
9
|
+
import { registerUserTools } from "./mcp/users.js";
|
|
10
|
+
import { registerNodeTypeCorpusTools } from "./mcp/node-type-corpus.js";
|
|
11
|
+
import { registerRepoNodeTypeTools } from "./mcp/repo-node-types.js";
|
|
12
|
+
import { registerJobTools } from "./mcp/jobs.js";
|
|
13
|
+
import { registerSubgraphTools } from "./mcp/subgraphs.js";
|
|
14
|
+
import { registerCacheTools } from "./mcp/cache.js";
|
|
15
|
+
import { registerRunAndWait } from "./workflows/run-and-wait.js";
|
|
16
|
+
import { registerRetryAndWait } from "./workflows/retry-and-wait.js";
|
|
17
|
+
import { registerGetRunDetails } from "./workflows/get-run-details.js";
|
|
18
|
+
import { registerGetEnvironmentOverview } from "./workflows/get-environment-overview.js";
|
|
19
|
+
import { registerResources } from "./resources/index.js";
|
|
20
|
+
import { registerPrompts } from "./prompts/index.js";
|
|
21
|
+
import { ensureJsonToolOutputSchemas } from "./coalesce/types.js";
|
|
22
|
+
const require = createRequire(import.meta.url);
|
|
23
|
+
const { version } = require("../package.json");
|
|
24
|
+
export const SERVER_NAME = "coalesce-transform-mcp";
|
|
25
|
+
export const SERVER_VERSION = version;
|
|
26
|
+
export const SERVER_INSTRUCTIONS = [
|
|
27
|
+
"Resolve IDs before mutating projects, workspaces, environments, jobs, runs, or nodes.",
|
|
28
|
+
"Always use plan-pipeline before creating pipeline nodes, and wait for explicit user approval before calling creation tools.",
|
|
29
|
+
"Inspect warning, validation, resultsError, incomplete, timedOut, and cleanupFailures fields before continuing.",
|
|
30
|
+
"Prefer run-and-wait or retry-and-wait when the user wants an end-to-end run outcome in one call.",
|
|
31
|
+
"Large payloads may be exposed through coalesce://cache resource URIs; read the resource rather than assuming inline JSON.",
|
|
32
|
+
].join("\n");
|
|
33
|
+
export function registerServerSurface(server, client) {
|
|
34
|
+
ensureJsonToolOutputSchemas(server);
|
|
35
|
+
registerEnvironmentTools(server, client);
|
|
36
|
+
registerNodeTools(server, client);
|
|
37
|
+
registerPipelineTools(server, client);
|
|
38
|
+
registerRunTools(server, client);
|
|
39
|
+
registerProjectTools(server, client);
|
|
40
|
+
registerGitAccountTools(server, client);
|
|
41
|
+
registerUserTools(server, client);
|
|
42
|
+
registerNodeTypeCorpusTools(server, client);
|
|
43
|
+
registerRepoNodeTypeTools(server, client);
|
|
44
|
+
registerJobTools(server, client);
|
|
45
|
+
registerSubgraphTools(server, client);
|
|
46
|
+
registerCacheTools(server, client);
|
|
47
|
+
registerRunAndWait(server, client);
|
|
48
|
+
registerRetryAndWait(server, client);
|
|
49
|
+
registerGetRunDetails(server, client);
|
|
50
|
+
registerGetEnvironmentOverview(server, client);
|
|
51
|
+
registerResources(server);
|
|
52
|
+
registerPrompts(server);
|
|
53
|
+
}
|
|
54
|
+
export function createCoalesceMcpServer(client) {
|
|
55
|
+
const server = new McpServer({
|
|
56
|
+
name: SERVER_NAME,
|
|
57
|
+
version: SERVER_VERSION,
|
|
58
|
+
}, {
|
|
59
|
+
instructions: SERVER_INSTRUCTIONS,
|
|
60
|
+
});
|
|
61
|
+
registerServerSurface(server, client);
|
|
62
|
+
return server;
|
|
63
|
+
}
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import type { CoalesceClient, QueryParams } from "../../client.js";
|
|
2
|
+
import type { NodeSummary } from "../workspace/analysis.js";
|
|
3
|
+
type PaginatedParams = {
|
|
4
|
+
pageSize?: number;
|
|
5
|
+
orderBy?: string;
|
|
6
|
+
orderByDirection?: "asc" | "desc";
|
|
7
|
+
};
|
|
8
|
+
type FetchPage = (params: QueryParams) => Promise<unknown>;
|
|
9
|
+
type PaginatedCollectionResult = {
|
|
10
|
+
items: unknown[];
|
|
11
|
+
pageCount: number;
|
|
12
|
+
pageSize: number;
|
|
13
|
+
orderBy: string;
|
|
14
|
+
orderByDirection?: "asc" | "desc";
|
|
15
|
+
};
|
|
16
|
+
type CacheWriteOptions = {
|
|
17
|
+
baseDir?: string;
|
|
18
|
+
};
|
|
19
|
+
type StreamToDiskOptions = {
|
|
20
|
+
ndjsonPath: string;
|
|
21
|
+
metaPath: string;
|
|
22
|
+
itemTransform?: (item: unknown) => unknown;
|
|
23
|
+
};
|
|
24
|
+
type StreamToDiskResult = {
|
|
25
|
+
totalItems: number;
|
|
26
|
+
pageCount: number;
|
|
27
|
+
pageSize: number;
|
|
28
|
+
orderBy: string;
|
|
29
|
+
orderByDirection?: "asc" | "desc";
|
|
30
|
+
cachedAt: string;
|
|
31
|
+
};
|
|
32
|
+
export declare function streamAllPaginatedToDisk(fetchPage: FetchPage, baseParams: QueryParams, params: PaginatedParams, options: StreamToDiskOptions): Promise<StreamToDiskResult>;
|
|
33
|
+
export declare function fetchAllWorkspaceNodes(client: CoalesceClient, params: {
|
|
34
|
+
workspaceID: string;
|
|
35
|
+
detail?: boolean;
|
|
36
|
+
} & PaginatedParams): Promise<PaginatedCollectionResult>;
|
|
37
|
+
export declare function fetchAllEnvironmentNodes(client: CoalesceClient, params: {
|
|
38
|
+
environmentID: string;
|
|
39
|
+
detail?: boolean;
|
|
40
|
+
} & PaginatedParams): Promise<PaginatedCollectionResult>;
|
|
41
|
+
export declare function fetchAllRuns(client: CoalesceClient, params: {
|
|
42
|
+
runType?: "deploy" | "refresh";
|
|
43
|
+
runStatus?: "completed" | "failed" | "canceled" | "running" | "waitingToRun";
|
|
44
|
+
environmentID?: string;
|
|
45
|
+
detail?: boolean;
|
|
46
|
+
} & PaginatedParams): Promise<PaginatedCollectionResult>;
|
|
47
|
+
export declare function fetchAllOrgUsers(client: CoalesceClient, params: PaginatedParams): Promise<PaginatedCollectionResult>;
|
|
48
|
+
export declare function toNodeSummaries(nodes: unknown[]): NodeSummary[];
|
|
49
|
+
export declare function cacheWorkspaceNodes(client: CoalesceClient, params: {
|
|
50
|
+
workspaceID: string;
|
|
51
|
+
detail?: boolean;
|
|
52
|
+
} & PaginatedParams, options?: CacheWriteOptions): Promise<{
|
|
53
|
+
workspaceID: string;
|
|
54
|
+
detail: boolean;
|
|
55
|
+
totalNodes: number;
|
|
56
|
+
pageCount: number;
|
|
57
|
+
pageSize: number;
|
|
58
|
+
orderBy: string;
|
|
59
|
+
orderByDirection?: "asc" | "desc";
|
|
60
|
+
filePath: string;
|
|
61
|
+
metaPath: string;
|
|
62
|
+
cachedAt: string;
|
|
63
|
+
}>;
|
|
64
|
+
export declare function cacheEnvironmentNodes(client: CoalesceClient, params: {
|
|
65
|
+
environmentID: string;
|
|
66
|
+
detail?: boolean;
|
|
67
|
+
} & PaginatedParams, options?: CacheWriteOptions): Promise<{
|
|
68
|
+
environmentID: string;
|
|
69
|
+
detail: boolean;
|
|
70
|
+
totalNodes: number;
|
|
71
|
+
pageCount: number;
|
|
72
|
+
pageSize: number;
|
|
73
|
+
orderBy: string;
|
|
74
|
+
orderByDirection?: "asc" | "desc";
|
|
75
|
+
filePath: string;
|
|
76
|
+
metaPath: string;
|
|
77
|
+
cachedAt: string;
|
|
78
|
+
}>;
|
|
79
|
+
export declare function cacheRuns(client: CoalesceClient, params: {
|
|
80
|
+
runType?: "deploy" | "refresh";
|
|
81
|
+
runStatus?: "completed" | "failed" | "canceled" | "running" | "waitingToRun";
|
|
82
|
+
environmentID?: string;
|
|
83
|
+
detail?: boolean;
|
|
84
|
+
} & PaginatedParams, options?: CacheWriteOptions): Promise<{
|
|
85
|
+
detail: boolean;
|
|
86
|
+
totalRuns: number;
|
|
87
|
+
pageCount: number;
|
|
88
|
+
pageSize: number;
|
|
89
|
+
orderBy: string;
|
|
90
|
+
orderByDirection?: "asc" | "desc";
|
|
91
|
+
filePath: string;
|
|
92
|
+
metaPath: string;
|
|
93
|
+
cachedAt: string;
|
|
94
|
+
runType?: "deploy" | "refresh";
|
|
95
|
+
runStatus?: "completed" | "failed" | "canceled" | "running" | "waitingToRun";
|
|
96
|
+
environmentID?: string;
|
|
97
|
+
}>;
|
|
98
|
+
export declare function cacheOrgUsers(client: CoalesceClient, params: PaginatedParams, options?: CacheWriteOptions): Promise<{
|
|
99
|
+
totalUsers: number;
|
|
100
|
+
pageCount: number;
|
|
101
|
+
pageSize: number;
|
|
102
|
+
orderBy: string;
|
|
103
|
+
orderByDirection?: "asc" | "desc";
|
|
104
|
+
filePath: string;
|
|
105
|
+
metaPath: string;
|
|
106
|
+
cachedAt: string;
|
|
107
|
+
}>;
|
|
108
|
+
export {};
|
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
import { appendFileSync, mkdirSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { dirname, join } from "node:path";
|
|
3
|
+
import { listEnvironmentNodes, listWorkspaceNodes } from "../../coalesce/api/nodes.js";
|
|
4
|
+
import { listRuns } from "../../coalesce/api/runs.js";
|
|
5
|
+
import { listOrgUsers } from "../../coalesce/api/users.js";
|
|
6
|
+
import { sanitizeResponse, validatePathSegment } from "../../coalesce/types.js";
|
|
7
|
+
import { isPlainObject } from "../../utils.js";
|
|
8
|
+
import { CACHE_DIR_NAME } from "../../cache-dir.js";
|
|
9
|
+
const DEFAULT_PAGE_SIZE = 250;
|
|
10
|
+
function parseCollectionPage(response) {
|
|
11
|
+
if (!isPlainObject(response)) {
|
|
12
|
+
throw new Error("Paginated collection response was not an object");
|
|
13
|
+
}
|
|
14
|
+
return {
|
|
15
|
+
data: Array.isArray(response.data) ? response.data : [],
|
|
16
|
+
next: typeof response.next === "string" && response.next.trim().length > 0
|
|
17
|
+
? response.next
|
|
18
|
+
: undefined,
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
async function fetchAllPaginatedToMemory(fetchPage, baseParams, params) {
|
|
22
|
+
const items = [];
|
|
23
|
+
const seenCursors = new Set();
|
|
24
|
+
const pageSize = Math.max(1, Math.floor(params.pageSize ?? DEFAULT_PAGE_SIZE));
|
|
25
|
+
const orderBy = params.orderBy ?? "id";
|
|
26
|
+
const orderByDirection = params.orderByDirection;
|
|
27
|
+
let next;
|
|
28
|
+
let isFirstPage = true;
|
|
29
|
+
let pageCount = 0;
|
|
30
|
+
while (isFirstPage || next) {
|
|
31
|
+
const response = await fetchPage({
|
|
32
|
+
...baseParams,
|
|
33
|
+
limit: pageSize,
|
|
34
|
+
orderBy,
|
|
35
|
+
...(orderByDirection ? { orderByDirection } : {}),
|
|
36
|
+
...(next ? { startingFrom: next } : {}),
|
|
37
|
+
});
|
|
38
|
+
const page = parseCollectionPage(response);
|
|
39
|
+
items.push(...page.data);
|
|
40
|
+
pageCount += 1;
|
|
41
|
+
if (page.next) {
|
|
42
|
+
if (seenCursors.has(page.next)) {
|
|
43
|
+
throw new Error(`Pagination repeated cursor ${page.next}`);
|
|
44
|
+
}
|
|
45
|
+
seenCursors.add(page.next);
|
|
46
|
+
}
|
|
47
|
+
next = page.next;
|
|
48
|
+
isFirstPage = false;
|
|
49
|
+
}
|
|
50
|
+
return {
|
|
51
|
+
items,
|
|
52
|
+
pageCount,
|
|
53
|
+
pageSize,
|
|
54
|
+
orderBy,
|
|
55
|
+
...(orderByDirection ? { orderByDirection } : {}),
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
export async function streamAllPaginatedToDisk(fetchPage, baseParams, params, options) {
|
|
59
|
+
const { ndjsonPath, metaPath, itemTransform } = options;
|
|
60
|
+
const seenCursors = new Set();
|
|
61
|
+
const pageSize = Math.max(1, Math.floor(params.pageSize ?? DEFAULT_PAGE_SIZE));
|
|
62
|
+
const orderBy = params.orderBy ?? "id";
|
|
63
|
+
const orderByDirection = params.orderByDirection;
|
|
64
|
+
const cachedAt = new Date().toISOString();
|
|
65
|
+
// Ensure parent directory exists
|
|
66
|
+
mkdirSync(dirname(ndjsonPath), { recursive: true });
|
|
67
|
+
// Write empty file to start (truncates any previous file)
|
|
68
|
+
writeFileSync(ndjsonPath, "", "utf8");
|
|
69
|
+
let totalItems = 0;
|
|
70
|
+
let next;
|
|
71
|
+
let isFirstPage = true;
|
|
72
|
+
let pageCount = 0;
|
|
73
|
+
while (isFirstPage || next) {
|
|
74
|
+
const response = await fetchPage({
|
|
75
|
+
...baseParams,
|
|
76
|
+
limit: pageSize,
|
|
77
|
+
orderBy,
|
|
78
|
+
...(orderByDirection ? { orderByDirection } : {}),
|
|
79
|
+
...(next ? { startingFrom: next } : {}),
|
|
80
|
+
});
|
|
81
|
+
const page = parseCollectionPage(response);
|
|
82
|
+
pageCount += 1;
|
|
83
|
+
// Write each item as a single NDJSON line
|
|
84
|
+
for (const item of page.data) {
|
|
85
|
+
const transformed = itemTransform ? itemTransform(item) : item;
|
|
86
|
+
appendFileSync(ndjsonPath, JSON.stringify(transformed) + "\n", "utf8");
|
|
87
|
+
totalItems += 1;
|
|
88
|
+
}
|
|
89
|
+
if (page.next) {
|
|
90
|
+
if (seenCursors.has(page.next)) {
|
|
91
|
+
throw new Error(`Pagination repeated cursor ${page.next}`);
|
|
92
|
+
}
|
|
93
|
+
seenCursors.add(page.next);
|
|
94
|
+
}
|
|
95
|
+
next = page.next;
|
|
96
|
+
isFirstPage = false;
|
|
97
|
+
}
|
|
98
|
+
// Write meta file only on successful completion
|
|
99
|
+
const meta = {
|
|
100
|
+
totalItems,
|
|
101
|
+
pageCount,
|
|
102
|
+
pageSize,
|
|
103
|
+
orderBy,
|
|
104
|
+
...(orderByDirection ? { orderByDirection } : {}),
|
|
105
|
+
cachedAt,
|
|
106
|
+
};
|
|
107
|
+
writeFileSync(metaPath, JSON.stringify(meta, null, 2) + "\n", "utf8");
|
|
108
|
+
return meta;
|
|
109
|
+
}
|
|
110
|
+
function ensureDirectory(...parts) {
|
|
111
|
+
const directory = join(...parts);
|
|
112
|
+
mkdirSync(directory, { recursive: true });
|
|
113
|
+
return directory;
|
|
114
|
+
}
|
|
115
|
+
function buildNodeCacheFileName(scope, id, detail) {
|
|
116
|
+
const safeID = validatePathSegment(id, `${scope}ID`);
|
|
117
|
+
if (detail) {
|
|
118
|
+
return `${scope}-${safeID}-nodes.ndjson`;
|
|
119
|
+
}
|
|
120
|
+
return `${scope}-${safeID}-nodes-summary.ndjson`;
|
|
121
|
+
}
|
|
122
|
+
function buildRunsCacheFileName(params) {
|
|
123
|
+
const parts = ["runs"];
|
|
124
|
+
if (params.environmentID) {
|
|
125
|
+
parts.push(`env-${validatePathSegment(params.environmentID, "environmentID")}`);
|
|
126
|
+
}
|
|
127
|
+
if (params.runType) {
|
|
128
|
+
parts.push(params.runType);
|
|
129
|
+
}
|
|
130
|
+
if (params.runStatus) {
|
|
131
|
+
parts.push(params.runStatus);
|
|
132
|
+
}
|
|
133
|
+
parts.push(params.detail === true ? "detail" : "summary");
|
|
134
|
+
return `${parts.join("-")}.ndjson`;
|
|
135
|
+
}
|
|
136
|
+
export async function fetchAllWorkspaceNodes(client, params) {
|
|
137
|
+
return fetchAllPaginatedToMemory((queryParams) => listWorkspaceNodes(client, queryParams), {
|
|
138
|
+
workspaceID: validatePathSegment(params.workspaceID, "workspaceID"),
|
|
139
|
+
...(params.detail !== undefined ? { detail: params.detail } : {}),
|
|
140
|
+
}, params);
|
|
141
|
+
}
|
|
142
|
+
export async function fetchAllEnvironmentNodes(client, params) {
|
|
143
|
+
return fetchAllPaginatedToMemory((queryParams) => listEnvironmentNodes(client, queryParams), {
|
|
144
|
+
environmentID: validatePathSegment(params.environmentID, "environmentID"),
|
|
145
|
+
...(params.detail !== undefined ? { detail: params.detail } : {}),
|
|
146
|
+
}, params);
|
|
147
|
+
}
|
|
148
|
+
export async function fetchAllRuns(client, params) {
|
|
149
|
+
return fetchAllPaginatedToMemory((queryParams) => listRuns(client, queryParams), {
|
|
150
|
+
...(params.runType ? { runType: params.runType } : {}),
|
|
151
|
+
...(params.runStatus ? { runStatus: params.runStatus } : {}),
|
|
152
|
+
...(params.environmentID
|
|
153
|
+
? { environmentID: validatePathSegment(params.environmentID, "environmentID") }
|
|
154
|
+
: {}),
|
|
155
|
+
...(params.detail !== undefined ? { detail: params.detail } : {}),
|
|
156
|
+
}, params);
|
|
157
|
+
}
|
|
158
|
+
export async function fetchAllOrgUsers(client, params) {
|
|
159
|
+
return fetchAllPaginatedToMemory((queryParams) => listOrgUsers(client, queryParams), {}, params);
|
|
160
|
+
}
|
|
161
|
+
export function toNodeSummaries(nodes) {
|
|
162
|
+
return nodes.flatMap((node) => {
|
|
163
|
+
if (!isPlainObject(node)) {
|
|
164
|
+
return [];
|
|
165
|
+
}
|
|
166
|
+
if (typeof node.nodeType !== "string" || typeof node.name !== "string") {
|
|
167
|
+
return [];
|
|
168
|
+
}
|
|
169
|
+
return [
|
|
170
|
+
{
|
|
171
|
+
nodeType: node.nodeType,
|
|
172
|
+
name: node.name,
|
|
173
|
+
},
|
|
174
|
+
];
|
|
175
|
+
});
|
|
176
|
+
}
|
|
177
|
+
export async function cacheWorkspaceNodes(client, params, options) {
|
|
178
|
+
const detail = params.detail ?? true;
|
|
179
|
+
const baseDir = options?.baseDir ?? process.cwd();
|
|
180
|
+
const directory = ensureDirectory(baseDir, CACHE_DIR_NAME, "nodes");
|
|
181
|
+
const baseName = buildNodeCacheFileName("workspace", params.workspaceID, detail);
|
|
182
|
+
const ndjsonPath = join(directory, baseName);
|
|
183
|
+
const metaPath = join(directory, baseName.replace(/\.ndjson$/, ".meta.json"));
|
|
184
|
+
const result = await streamAllPaginatedToDisk((queryParams) => listWorkspaceNodes(client, queryParams), {
|
|
185
|
+
workspaceID: validatePathSegment(params.workspaceID, "workspaceID"),
|
|
186
|
+
...(detail !== undefined ? { detail } : {}),
|
|
187
|
+
}, params, { ndjsonPath, metaPath });
|
|
188
|
+
return {
|
|
189
|
+
workspaceID: params.workspaceID,
|
|
190
|
+
detail,
|
|
191
|
+
totalNodes: result.totalItems,
|
|
192
|
+
pageCount: result.pageCount,
|
|
193
|
+
pageSize: result.pageSize,
|
|
194
|
+
orderBy: result.orderBy,
|
|
195
|
+
...(result.orderByDirection ? { orderByDirection: result.orderByDirection } : {}),
|
|
196
|
+
filePath: ndjsonPath,
|
|
197
|
+
metaPath,
|
|
198
|
+
cachedAt: result.cachedAt,
|
|
199
|
+
};
|
|
200
|
+
}
|
|
201
|
+
export async function cacheEnvironmentNodes(client, params, options) {
|
|
202
|
+
const detail = params.detail ?? true;
|
|
203
|
+
const baseDir = options?.baseDir ?? process.cwd();
|
|
204
|
+
const directory = ensureDirectory(baseDir, CACHE_DIR_NAME, "nodes");
|
|
205
|
+
const baseName = buildNodeCacheFileName("environment", params.environmentID, detail);
|
|
206
|
+
const ndjsonPath = join(directory, baseName);
|
|
207
|
+
const metaPath = join(directory, baseName.replace(/\.ndjson$/, ".meta.json"));
|
|
208
|
+
const result = await streamAllPaginatedToDisk((queryParams) => listEnvironmentNodes(client, queryParams), {
|
|
209
|
+
environmentID: validatePathSegment(params.environmentID, "environmentID"),
|
|
210
|
+
...(detail !== undefined ? { detail } : {}),
|
|
211
|
+
}, params, { ndjsonPath, metaPath });
|
|
212
|
+
return {
|
|
213
|
+
environmentID: params.environmentID,
|
|
214
|
+
detail,
|
|
215
|
+
totalNodes: result.totalItems,
|
|
216
|
+
pageCount: result.pageCount,
|
|
217
|
+
pageSize: result.pageSize,
|
|
218
|
+
orderBy: result.orderBy,
|
|
219
|
+
...(result.orderByDirection ? { orderByDirection: result.orderByDirection } : {}),
|
|
220
|
+
filePath: ndjsonPath,
|
|
221
|
+
metaPath,
|
|
222
|
+
cachedAt: result.cachedAt,
|
|
223
|
+
};
|
|
224
|
+
}
|
|
225
|
+
export async function cacheRuns(client, params, options) {
|
|
226
|
+
const detail = params.detail ?? false;
|
|
227
|
+
const baseDir = options?.baseDir ?? process.cwd();
|
|
228
|
+
const directory = ensureDirectory(baseDir, CACHE_DIR_NAME, "runs");
|
|
229
|
+
const baseName = buildRunsCacheFileName({ ...params, detail });
|
|
230
|
+
const ndjsonPath = join(directory, baseName);
|
|
231
|
+
const metaPath = join(directory, baseName.replace(/\.ndjson$/, ".meta.json"));
|
|
232
|
+
const result = await streamAllPaginatedToDisk((queryParams) => listRuns(client, queryParams), {
|
|
233
|
+
...(params.runType ? { runType: params.runType } : {}),
|
|
234
|
+
...(params.runStatus ? { runStatus: params.runStatus } : {}),
|
|
235
|
+
...(params.environmentID
|
|
236
|
+
? { environmentID: validatePathSegment(params.environmentID, "environmentID") }
|
|
237
|
+
: {}),
|
|
238
|
+
...(detail !== undefined ? { detail } : {}),
|
|
239
|
+
}, params, {
|
|
240
|
+
ndjsonPath,
|
|
241
|
+
metaPath,
|
|
242
|
+
itemTransform: (item) => sanitizeResponse(item),
|
|
243
|
+
});
|
|
244
|
+
return {
|
|
245
|
+
detail,
|
|
246
|
+
totalRuns: result.totalItems,
|
|
247
|
+
pageCount: result.pageCount,
|
|
248
|
+
pageSize: result.pageSize,
|
|
249
|
+
orderBy: result.orderBy,
|
|
250
|
+
...(result.orderByDirection ? { orderByDirection: result.orderByDirection } : {}),
|
|
251
|
+
filePath: ndjsonPath,
|
|
252
|
+
metaPath,
|
|
253
|
+
cachedAt: result.cachedAt,
|
|
254
|
+
...(params.runType ? { runType: params.runType } : {}),
|
|
255
|
+
...(params.runStatus ? { runStatus: params.runStatus } : {}),
|
|
256
|
+
...(params.environmentID ? { environmentID: params.environmentID } : {}),
|
|
257
|
+
};
|
|
258
|
+
}
|
|
259
|
+
export async function cacheOrgUsers(client, params, options) {
|
|
260
|
+
const baseDir = options?.baseDir ?? process.cwd();
|
|
261
|
+
const directory = ensureDirectory(baseDir, CACHE_DIR_NAME, "users");
|
|
262
|
+
const ndjsonPath = join(directory, "org-users.ndjson");
|
|
263
|
+
const metaPath = join(directory, "org-users.meta.json");
|
|
264
|
+
const result = await streamAllPaginatedToDisk((queryParams) => listOrgUsers(client, queryParams), {}, params, { ndjsonPath, metaPath });
|
|
265
|
+
return {
|
|
266
|
+
totalUsers: result.totalItems,
|
|
267
|
+
pageCount: result.pageCount,
|
|
268
|
+
pageSize: result.pageSize,
|
|
269
|
+
orderBy: result.orderBy,
|
|
270
|
+
...(result.orderByDirection ? { orderByDirection: result.orderByDirection } : {}),
|
|
271
|
+
filePath: ndjsonPath,
|
|
272
|
+
metaPath,
|
|
273
|
+
cachedAt: result.cachedAt,
|
|
274
|
+
};
|
|
275
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export interface NodeContext {
|
|
2
|
+
hasMultipleSources: boolean;
|
|
3
|
+
hasAggregates: boolean;
|
|
4
|
+
hasTimestampColumns: boolean;
|
|
5
|
+
hasType2Pattern: boolean;
|
|
6
|
+
materializationType: "table" | "view";
|
|
7
|
+
columnPatterns: {
|
|
8
|
+
timestamps: string[];
|
|
9
|
+
dates: string[];
|
|
10
|
+
businessKeys: string[];
|
|
11
|
+
changeTrackingCandidates: string[];
|
|
12
|
+
};
|
|
13
|
+
}
|
|
14
|
+
export declare function analyzeNodeContext(node: Record<string, unknown>): NodeContext;
|