@tinybirdco/sdk 0.0.41 → 0.0.43
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +7 -0
- package/README.md +29 -3
- package/dist/api/resources.d.ts +72 -1
- package/dist/api/resources.d.ts.map +1 -1
- package/dist/api/resources.js +197 -1
- package/dist/api/resources.js.map +1 -1
- package/dist/api/resources.test.js +82 -1
- package/dist/api/resources.test.js.map +1 -1
- package/dist/cli/commands/migrate.d.ts +11 -0
- package/dist/cli/commands/migrate.d.ts.map +1 -0
- package/dist/cli/commands/migrate.js +196 -0
- package/dist/cli/commands/migrate.js.map +1 -0
- package/dist/cli/commands/migrate.test.d.ts +2 -0
- package/dist/cli/commands/migrate.test.d.ts.map +1 -0
- package/dist/cli/commands/migrate.test.js +473 -0
- package/dist/cli/commands/migrate.test.js.map +1 -0
- package/dist/cli/commands/pull.d.ts +59 -0
- package/dist/cli/commands/pull.d.ts.map +1 -0
- package/dist/cli/commands/pull.js +104 -0
- package/dist/cli/commands/pull.js.map +1 -0
- package/dist/cli/commands/pull.test.d.ts +2 -0
- package/dist/cli/commands/pull.test.d.ts.map +1 -0
- package/dist/cli/commands/pull.test.js +140 -0
- package/dist/cli/commands/pull.test.js.map +1 -0
- package/dist/cli/config.d.ts +10 -0
- package/dist/cli/config.d.ts.map +1 -1
- package/dist/cli/config.js +22 -0
- package/dist/cli/config.js.map +1 -1
- package/dist/cli/index.js +77 -0
- package/dist/cli/index.js.map +1 -1
- package/dist/generator/client.js +2 -2
- package/dist/generator/client.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/migrate/discovery.d.ts +7 -0
- package/dist/migrate/discovery.d.ts.map +1 -0
- package/dist/migrate/discovery.js +125 -0
- package/dist/migrate/discovery.js.map +1 -0
- package/dist/migrate/emit-ts.d.ts +4 -0
- package/dist/migrate/emit-ts.d.ts.map +1 -0
- package/dist/migrate/emit-ts.js +387 -0
- package/dist/migrate/emit-ts.js.map +1 -0
- package/dist/migrate/parse-connection.d.ts +3 -0
- package/dist/migrate/parse-connection.d.ts.map +1 -0
- package/dist/migrate/parse-connection.js +74 -0
- package/dist/migrate/parse-connection.js.map +1 -0
- package/dist/migrate/parse-datasource.d.ts +3 -0
- package/dist/migrate/parse-datasource.d.ts.map +1 -0
- package/dist/migrate/parse-datasource.js +324 -0
- package/dist/migrate/parse-datasource.js.map +1 -0
- package/dist/migrate/parse-pipe.d.ts +3 -0
- package/dist/migrate/parse-pipe.d.ts.map +1 -0
- package/dist/migrate/parse-pipe.js +332 -0
- package/dist/migrate/parse-pipe.js.map +1 -0
- package/dist/migrate/parse.d.ts +3 -0
- package/dist/migrate/parse.d.ts.map +1 -0
- package/dist/migrate/parse.js +18 -0
- package/dist/migrate/parse.js.map +1 -0
- package/dist/migrate/parser-utils.d.ts +20 -0
- package/dist/migrate/parser-utils.d.ts.map +1 -0
- package/dist/migrate/parser-utils.js +130 -0
- package/dist/migrate/parser-utils.js.map +1 -0
- package/dist/migrate/types.d.ts +110 -0
- package/dist/migrate/types.d.ts.map +1 -0
- package/dist/migrate/types.js +2 -0
- package/dist/migrate/types.js.map +1 -0
- package/dist/schema/project.d.ts +20 -9
- package/dist/schema/project.d.ts.map +1 -1
- package/dist/schema/project.js +127 -136
- package/dist/schema/project.js.map +1 -1
- package/dist/schema/project.test.js +22 -0
- package/dist/schema/project.test.js.map +1 -1
- package/package.json +2 -1
- package/src/api/resources.test.ts +121 -0
- package/src/api/resources.ts +292 -1
- package/src/cli/commands/migrate.test.ts +564 -0
- package/src/cli/commands/migrate.ts +240 -0
- package/src/cli/commands/pull.test.ts +173 -0
- package/src/cli/commands/pull.ts +177 -0
- package/src/cli/config.ts +26 -0
- package/src/cli/index.ts +112 -0
- package/src/generator/client.ts +2 -2
- package/src/index.ts +1 -1
- package/src/migrate/discovery.ts +151 -0
- package/src/migrate/emit-ts.ts +469 -0
- package/src/migrate/parse-connection.ts +128 -0
- package/src/migrate/parse-datasource.ts +453 -0
- package/src/migrate/parse-pipe.ts +518 -0
- package/src/migrate/parse.ts +20 -0
- package/src/migrate/parser-utils.ts +160 -0
- package/src/migrate/types.ts +125 -0
- package/src/schema/project.test.ts +28 -0
- package/src/schema/project.ts +173 -181
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
import * as fs from "node:fs";
|
|
2
|
+
import * as path from "node:path";
|
|
3
|
+
import { discoverResourceFiles } from "../../migrate/discovery.js";
|
|
4
|
+
import { emitMigrationFileContent, validateResourceForEmission } from "../../migrate/emit-ts.js";
|
|
5
|
+
import { parseResourceFile } from "../../migrate/parse.js";
|
|
6
|
+
import { MigrationParseError } from "../../migrate/parser-utils.js";
|
|
7
|
+
function toMigrationError(resource, error) {
|
|
8
|
+
const message = error.message || String(error);
|
|
9
|
+
return {
|
|
10
|
+
filePath: resource.filePath,
|
|
11
|
+
resourceName: resource.name,
|
|
12
|
+
resourceKind: resource.kind,
|
|
13
|
+
message,
|
|
14
|
+
};
|
|
15
|
+
}
|
|
16
|
+
function sortResourcesForOutput(resources) {
|
|
17
|
+
const order = {
|
|
18
|
+
connection: 0,
|
|
19
|
+
datasource: 1,
|
|
20
|
+
pipe: 2,
|
|
21
|
+
};
|
|
22
|
+
return [...resources].sort((a, b) => {
|
|
23
|
+
const byType = order[a.kind] - order[b.kind];
|
|
24
|
+
if (byType !== 0) {
|
|
25
|
+
return byType;
|
|
26
|
+
}
|
|
27
|
+
return a.name.localeCompare(b.name);
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
export async function runMigrate(options) {
|
|
31
|
+
const cwd = options.cwd ?? process.cwd();
|
|
32
|
+
const strict = options.strict ?? true;
|
|
33
|
+
const dryRun = options.dryRun ?? false;
|
|
34
|
+
const force = options.force ?? false;
|
|
35
|
+
const outputPath = path.isAbsolute(options.out ?? "")
|
|
36
|
+
? options.out
|
|
37
|
+
: path.resolve(cwd, options.out ?? "tinybird.migration.ts");
|
|
38
|
+
const errors = [];
|
|
39
|
+
if (!options.patterns || options.patterns.length === 0) {
|
|
40
|
+
return {
|
|
41
|
+
success: false,
|
|
42
|
+
outputPath,
|
|
43
|
+
migrated: [],
|
|
44
|
+
errors: [
|
|
45
|
+
{
|
|
46
|
+
filePath: ".",
|
|
47
|
+
resourceName: "patterns",
|
|
48
|
+
resourceKind: "datasource",
|
|
49
|
+
message: "At least one file, directory, or glob pattern is required.",
|
|
50
|
+
},
|
|
51
|
+
],
|
|
52
|
+
dryRun,
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
const discovered = discoverResourceFiles(options.patterns, cwd);
|
|
56
|
+
errors.push(...discovered.errors);
|
|
57
|
+
const parsedResources = [];
|
|
58
|
+
for (const resource of discovered.resources) {
|
|
59
|
+
try {
|
|
60
|
+
parsedResources.push(parseResourceFile(resource));
|
|
61
|
+
}
|
|
62
|
+
catch (error) {
|
|
63
|
+
if (error instanceof MigrationParseError) {
|
|
64
|
+
errors.push({
|
|
65
|
+
filePath: error.filePath,
|
|
66
|
+
resourceName: error.resourceName,
|
|
67
|
+
resourceKind: error.resourceKind,
|
|
68
|
+
message: error.message,
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
else {
|
|
72
|
+
errors.push(toMigrationError(resource, error));
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
const parsedConnections = parsedResources.filter((resource) => resource.kind === "connection");
|
|
77
|
+
const parsedDatasources = parsedResources.filter((resource) => resource.kind === "datasource");
|
|
78
|
+
const parsedPipes = parsedResources.filter((resource) => resource.kind === "pipe");
|
|
79
|
+
const migrated = [];
|
|
80
|
+
const migratedConnectionNames = new Set();
|
|
81
|
+
const migratedDatasourceNames = new Set();
|
|
82
|
+
for (const connection of parsedConnections) {
|
|
83
|
+
try {
|
|
84
|
+
validateResourceForEmission(connection);
|
|
85
|
+
migrated.push(connection);
|
|
86
|
+
migratedConnectionNames.add(connection.name);
|
|
87
|
+
}
|
|
88
|
+
catch (error) {
|
|
89
|
+
errors.push({
|
|
90
|
+
filePath: connection.filePath,
|
|
91
|
+
resourceName: connection.name,
|
|
92
|
+
resourceKind: connection.kind,
|
|
93
|
+
message: error.message,
|
|
94
|
+
});
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
for (const datasource of parsedDatasources) {
|
|
98
|
+
if (datasource.kafka &&
|
|
99
|
+
!migratedConnectionNames.has(datasource.kafka.connectionName)) {
|
|
100
|
+
errors.push({
|
|
101
|
+
filePath: datasource.filePath,
|
|
102
|
+
resourceName: datasource.name,
|
|
103
|
+
resourceKind: datasource.kind,
|
|
104
|
+
message: `Datasource references missing/unmigrated connection "${datasource.kafka.connectionName}".`,
|
|
105
|
+
});
|
|
106
|
+
continue;
|
|
107
|
+
}
|
|
108
|
+
try {
|
|
109
|
+
validateResourceForEmission(datasource);
|
|
110
|
+
migrated.push(datasource);
|
|
111
|
+
migratedDatasourceNames.add(datasource.name);
|
|
112
|
+
}
|
|
113
|
+
catch (error) {
|
|
114
|
+
errors.push({
|
|
115
|
+
filePath: datasource.filePath,
|
|
116
|
+
resourceName: datasource.name,
|
|
117
|
+
resourceKind: datasource.kind,
|
|
118
|
+
message: error.message,
|
|
119
|
+
});
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
for (const pipe of parsedPipes) {
|
|
123
|
+
if (pipe.type === "materialized" &&
|
|
124
|
+
(!pipe.materializedDatasource ||
|
|
125
|
+
!migratedDatasourceNames.has(pipe.materializedDatasource))) {
|
|
126
|
+
errors.push({
|
|
127
|
+
filePath: pipe.filePath,
|
|
128
|
+
resourceName: pipe.name,
|
|
129
|
+
resourceKind: pipe.kind,
|
|
130
|
+
message: `Materialized pipe references missing/unmigrated datasource "${pipe.materializedDatasource ?? "(none)"}".`,
|
|
131
|
+
});
|
|
132
|
+
continue;
|
|
133
|
+
}
|
|
134
|
+
if (pipe.type === "copy" &&
|
|
135
|
+
(!pipe.copyTargetDatasource ||
|
|
136
|
+
!migratedDatasourceNames.has(pipe.copyTargetDatasource))) {
|
|
137
|
+
errors.push({
|
|
138
|
+
filePath: pipe.filePath,
|
|
139
|
+
resourceName: pipe.name,
|
|
140
|
+
resourceKind: pipe.kind,
|
|
141
|
+
message: `Copy pipe references missing/unmigrated datasource "${pipe.copyTargetDatasource ?? "(none)"}".`,
|
|
142
|
+
});
|
|
143
|
+
continue;
|
|
144
|
+
}
|
|
145
|
+
try {
|
|
146
|
+
validateResourceForEmission(pipe);
|
|
147
|
+
migrated.push(pipe);
|
|
148
|
+
}
|
|
149
|
+
catch (error) {
|
|
150
|
+
errors.push({
|
|
151
|
+
filePath: pipe.filePath,
|
|
152
|
+
resourceName: pipe.name,
|
|
153
|
+
resourceKind: pipe.kind,
|
|
154
|
+
message: error.message,
|
|
155
|
+
});
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
const sortedMigrated = sortResourcesForOutput(migrated);
|
|
159
|
+
let outputContent;
|
|
160
|
+
if (sortedMigrated.length > 0) {
|
|
161
|
+
try {
|
|
162
|
+
outputContent = emitMigrationFileContent(sortedMigrated);
|
|
163
|
+
}
|
|
164
|
+
catch (error) {
|
|
165
|
+
errors.push({
|
|
166
|
+
filePath: ".",
|
|
167
|
+
resourceName: "output",
|
|
168
|
+
resourceKind: "datasource",
|
|
169
|
+
message: `Failed to emit migration output: ${error.message}`,
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
if (!dryRun && outputContent) {
|
|
174
|
+
if (fs.existsSync(outputPath) && !force) {
|
|
175
|
+
errors.push({
|
|
176
|
+
filePath: path.relative(cwd, outputPath),
|
|
177
|
+
resourceName: path.basename(outputPath),
|
|
178
|
+
resourceKind: "datasource",
|
|
179
|
+
message: `Output file already exists: ${outputPath}. Use --force to overwrite.`,
|
|
180
|
+
});
|
|
181
|
+
}
|
|
182
|
+
else {
|
|
183
|
+
fs.writeFileSync(outputPath, outputContent);
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
const success = strict ? errors.length === 0 : true;
|
|
187
|
+
return {
|
|
188
|
+
success,
|
|
189
|
+
outputPath,
|
|
190
|
+
migrated: sortedMigrated,
|
|
191
|
+
errors,
|
|
192
|
+
dryRun,
|
|
193
|
+
outputContent,
|
|
194
|
+
};
|
|
195
|
+
}
|
|
196
|
+
//# sourceMappingURL=migrate.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"migrate.js","sourceRoot":"","sources":["../../../src/cli/commands/migrate.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,qBAAqB,EAAE,MAAM,4BAA4B,CAAC;AACnE,OAAO,EAAE,wBAAwB,EAAE,2BAA2B,EAAE,MAAM,0BAA0B,CAAC;AACjG,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EAAE,mBAAmB,EAAE,MAAM,+BAA+B,CAAC;AAiBpE,SAAS,gBAAgB,CAAC,QAAsB,EAAE,KAAc;IAC9D,MAAM,OAAO,GAAI,KAAe,CAAC,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC;IAC1D,OAAO;QACL,QAAQ,EAAE,QAAQ,CAAC,QAAQ;QAC3B,YAAY,EAAE,QAAQ,CAAC,IAAI;QAC3B,YAAY,EAAE,QAAQ,CAAC,IAAI;QAC3B,OAAO;KACR,CAAC;AACJ,CAAC;AAED,SAAS,sBAAsB,CAAC,SAA2B;IACzD,MAAM,KAAK,GAA2C;QACpD,UAAU,EAAE,CAAC;QACb,UAAU,EAAE,CAAC;QACb,IAAI,EAAE,CAAC;KACR,CAAC;IACF,OAAO,CAAC,GAAG,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QAClC,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC7C,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;YACjB,OAAO,MAAM,CAAC;QAChB,CAAC;QACD,OAAO,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACtC,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,OAA8B;IAE9B,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;IACzC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC;IACtC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,KAAK,CAAC;IACvC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,KAAK,CAAC;IACrC,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,IAAI,EAAE,CAAC;QACnD,CAAC,CAAE,OAAO,CAAC,GAAc;QACzB,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,GAAG,IAAI,uBAAuB,CAAC,CAAC;IAE9D,MAAM,MAAM,GAAqB,EAAE,CAAC;IAEpC,IAAI,CAAC,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvD,OAAO;YACL,OAAO,EAAE,KAAK;YACd,UAAU;YACV,QAAQ,EAAE,EAAE;YACZ,MAAM,EAAE;gBACN;oBACE,QAAQ,EAAE,GAAG;oBACb,YAAY,EAAE,UAAU;oBACxB,YAAY,EAAE,YAAY;oBAC1B,OAAO,EAAE,4DAA4D;iBACtE;aACF;YACD,MAAM;SACP,CAAC;IACJ,CAAC;IAED,MAAM,UAAU,GAAG,qBAAqB,CAAC,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;IAChE,MAAM,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;IAElC,MAAM,eAAe,GAAqB,EAAE,CAAC;IAC7C,KAAK,MAAM,QAAQ,IAAI,UAAU,CAAC,SAAS,EAAE,CAAC;QAC5C,IAAI,CAAC;YACH,eAAe,CAAC,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC,CAAC;QACpD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,mBAAmB,EAAE,CAAC;gBACzC,MAAM,CAAC,IAAI,CAAC;oBACV,QAAQ,EAAE,KAAK,CAAC,QAAQ;oBACxB,YAAY,EAAE,KAAK,CAAC,YAAY;oBAChC,YAAY,EAAE,KAAK,CAAC,YAAY;oBAChC,OAAO,EAAE,KAAK,CAAC,OAAO;iBACvB,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC;YACjD,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,iBAAiB,GAAG,eAAe,CAAC,MAAM,CAC9C,CAAC,QAAQ,EAA+D,EAAE,CACxE,QAAQ,CAAC,IAAI,KAAK,YAAY,CACjC,CAAC;IACF,MAAM,iBAAiB,GAAG,eAAe,CAAC,MAAM,CAC9C,CAAC,QAAQ,EAA+D,EAAE,CACxE,QAAQ,CAAC,IAAI,KAAK,YAAY,CACjC,CAAC;IACF,MAAM,WAAW,GAAG,eAAe,CAAC,MAAM,CACxC,CAAC,QAAQ,EAAyD,EAAE,CAClE,QAAQ,CAAC,IAAI,KAAK,MAAM,CAC3B,CAAC;IAEF,MAAM,QAAQ,GAAqB,EAAE,CAAC;IACtC,MAAM,uBAAuB,GAAG,IAAI,GAAG,EAAU,CAAC;IAClD,MAAM,uBAAuB,GAAG,IAAI,GAAG,EAAU,CAAC;IAElD,KAAK,MAAM,UAAU,IAAI,iBAAiB,EAAE,CAAC;QAC3C,IAAI,CAAC;YACH,2BAA2B,CAAC,UAAU,CAAC,CAAC;YACxC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC1B,uBAAuB,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QAC/C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,IAAI,CAAC;gBACV,QAAQ,EAAE,UAAU,CAAC,QAAQ;gBAC7B,YAAY,EAAE,UAAU,CAAC,IAAI;gBAC7B,YAAY,EAAE,UAAU,CAAC,IAAI;gBAC7B,OAAO,EAAG,KAAe,CAAC,OAAO;aAClC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,KAAK,MAAM,UAAU,IAAI,iBAAiB,EAAE,CAAC;QAC3C,IACE,UAAU,CAAC,KAAK;YAChB,CAAC,uBAAuB,CAAC,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,cAAc,CAAC,EAC7D,CAAC;YACD,MAAM,CAAC,IAAI,CAAC;gBACV,QAAQ,EAAE,UAAU,CAAC,QAAQ;gBAC7B,YAAY,EAAE,UAAU,CAAC,IAAI;gBAC7B,YAAY,EAAE,UAAU,CAAC,IAAI;gBAC7B,OAAO,EAAE,wDAAwD,UAAU,CAAC,KAAK,CAAC,cAAc,IAAI;aACrG,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QAED,IAAI,CAAC;YACH,2BAA2B,CAAC,UAAU,CAAC,CAAC;YACxC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC1B,uBAAuB,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QAC/C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,IAAI,CAAC;gBACV,QAAQ,EAAE,UAAU,CAAC,QAAQ;gBAC7B,YAAY,EAAE,UAAU,CAAC,IAAI;gBAC7B,YAAY,EAAE,UAAU,CAAC,IAAI;gBAC7B,OAAO,EAAG,KAAe,CAAC,OAAO;aAClC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;QAC/B,IACE,IAAI,CAAC,IAAI,KAAK,cAAc;YAC5B,CAAC,CAAC,IAAI,CAAC,sBAAsB;gBAC3B,CAAC,uBAAuB,CAAC,GAAG,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC,EAC5D,CAAC;YACD,MAAM,CAAC,IAAI,CAAC;gBACV,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,YAAY,EAAE,IAAI,CAAC,IAAI;gBACvB,YAAY,EAAE,IAAI,CAAC,IAAI;gBACvB,OAAO,EAAE,+DAA+D,IAAI,CAAC,sBAAsB,IAAI,QAAQ,IAAI;aACpH,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QAED,IACE,IAAI,CAAC,IAAI,KAAK,MAAM;YACpB,CAAC,CAAC,IAAI,CAAC,oBAAoB;gBACzB,CAAC,uBAAuB,CAAC,GAAG,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,EAC1D,CAAC;YACD,MAAM,CAAC,IAAI,CAAC;gBACV,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,YAAY,EAAE,IAAI,CAAC,IAAI;gBACvB,YAAY,EAAE,IAAI,CAAC,IAAI;gBACvB,OAAO,EAAE,uDAAuD,IAAI,CAAC,oBAAoB,IAAI,QAAQ,IAAI;aAC1G,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QAED,IAAI,CAAC;YACH,2BAA2B,CAAC,IAAI,CAAC,CAAC;YAClC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,IAAI,CAAC;gBACV,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,YAAY,EAAE,IAAI,CAAC,IAAI;gBACvB,YAAY,EAAE,IAAI,CAAC,IAAI;gBACvB,OAAO,EAAG,KAAe,CAAC,OAAO;aAClC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,MAAM,cAAc,GAAG,sBAAsB,CAAC,QAAQ,CAAC,CAAC;IACxD,IAAI,aAAiC,CAAC;IAEtC,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9B,IAAI,CAAC;YACH,aAAa,GAAG,wBAAwB,CAAC,cAAc,CAAC,CAAC;QAC3D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,IAAI,CAAC;gBACV,QAAQ,EAAE,GAAG;gBACb,YAAY,EAAE,QAAQ;gBACtB,YAAY,EAAE,YAAY;gBAC1B,OAAO,EAAE,oCAAqC,KAAe,CAAC,OAAO,EAAE;aACxE,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,IAAI,CAAC,MAAM,IAAI,aAAa,EAAE,CAAC;QAC7B,IAAI,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YACxC,MAAM,CAAC,IAAI,CAAC;gBACV,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,UAAU,CAAC;gBACxC,YAAY,EAAE,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC;gBACvC,YAAY,EAAE,YAAY;gBAC1B,OAAO,EAAE,+BAA+B,UAAU,6BAA6B;aAChF,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,aAAa,CAAC,CAAC;QAC9C,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACpD,OAAO;QACL,OAAO;QACP,UAAU;QACV,QAAQ,EAAE,cAAc;QACxB,MAAM;QACN,MAAM;QACN,aAAa;KACd,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"migrate.test.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/migrate.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,473 @@
|
|
|
1
|
+
import * as fs from "node:fs";
|
|
2
|
+
import * as os from "node:os";
|
|
3
|
+
import * as path from "node:path";
|
|
4
|
+
import { afterEach, describe, expect, it } from "vitest";
|
|
5
|
+
import { runMigrate } from "./migrate.js";
|
|
6
|
+
function writeFile(dir, relativePath, content) {
|
|
7
|
+
const fullPath = path.join(dir, relativePath);
|
|
8
|
+
fs.mkdirSync(path.dirname(fullPath), { recursive: true });
|
|
9
|
+
fs.writeFileSync(fullPath, content);
|
|
10
|
+
}
|
|
11
|
+
const EXPECTED_COMPLEX_OUTPUT = `/**
|
|
12
|
+
* Generated by tinybird migrate.
|
|
13
|
+
* Review endpoint output schemas and any defaults before production use.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import { createKafkaConnection, defineDatasource, definePipe, defineMaterializedView, defineCopyPipe, node, t, engine, column, p } from "@tinybirdco/sdk";
|
|
17
|
+
|
|
18
|
+
// Connections
|
|
19
|
+
|
|
20
|
+
export const stream = createKafkaConnection("stream", {
|
|
21
|
+
bootstrapServers: "localhost:9092",
|
|
22
|
+
securityProtocol: "SASL_SSL",
|
|
23
|
+
saslMechanism: "PLAIN",
|
|
24
|
+
key: "api-key",
|
|
25
|
+
secret: "api-secret",
|
|
26
|
+
sslCaPem: "ca-pem-content",
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
// Datasources
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Events from Kafka stream
|
|
33
|
+
*/
|
|
34
|
+
export const events = defineDatasource("events", {
|
|
35
|
+
description: "Events from Kafka stream",
|
|
36
|
+
schema: {
|
|
37
|
+
event_id: column(t.string(), { jsonPath: "$.event_id" }),
|
|
38
|
+
user_id: column(t.uint64(), { jsonPath: "$.user.id" }),
|
|
39
|
+
env: column(t.string().default("prod"), { jsonPath: "$.env" }),
|
|
40
|
+
is_test: column(t.bool().default(false), { jsonPath: "$.meta.is_test" }),
|
|
41
|
+
updated_at: column(t.dateTime(), { jsonPath: "$.updated_at" }),
|
|
42
|
+
payload: column(t.string().default("{}").codec("ZSTD(1)"), { jsonPath: "$.payload" }),
|
|
43
|
+
},
|
|
44
|
+
engine: engine.replacingMergeTree({ sortingKey: ["event_id", "user_id"], partitionKey: "toYYYYMM(updated_at)", primaryKey: "event_id", ttl: "updated_at + toIntervalDay(30)", ver: "updated_at", settings: { "index_granularity": 8192, "enable_mixed_granularity_parts": true } }),
|
|
45
|
+
kafka: {
|
|
46
|
+
connection: stream,
|
|
47
|
+
topic: "events_topic",
|
|
48
|
+
groupId: "events-consumer",
|
|
49
|
+
autoOffsetReset: "earliest",
|
|
50
|
+
},
|
|
51
|
+
forwardQuery: \`
|
|
52
|
+
SELECT *
|
|
53
|
+
FROM events_mv
|
|
54
|
+
\`,
|
|
55
|
+
tokens: [
|
|
56
|
+
{ name: "events_read", permissions: ["READ"] },
|
|
57
|
+
{ name: "events_append", permissions: ["APPEND"] },
|
|
58
|
+
],
|
|
59
|
+
sharedWith: ["workspace_a", "workspace_b"],
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
export const eventsRollup = defineDatasource("events_rollup", {
|
|
63
|
+
jsonPaths: false,
|
|
64
|
+
schema: {
|
|
65
|
+
user_id: t.uint64(),
|
|
66
|
+
total: t.uint64(),
|
|
67
|
+
},
|
|
68
|
+
engine: engine.summingMergeTree({ sortingKey: "user_id", columns: ["total"] }),
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
// Pipes
|
|
72
|
+
|
|
73
|
+
export const copyEvents = defineCopyPipe("copy_events", {
|
|
74
|
+
datasource: eventsRollup,
|
|
75
|
+
copy_mode: "replace",
|
|
76
|
+
copy_schedule: "@on-demand",
|
|
77
|
+
nodes: [
|
|
78
|
+
node({
|
|
79
|
+
name: "copy_node",
|
|
80
|
+
sql: \`
|
|
81
|
+
SELECT event_id, user_id
|
|
82
|
+
FROM events
|
|
83
|
+
\`,
|
|
84
|
+
}),
|
|
85
|
+
],
|
|
86
|
+
tokens: [
|
|
87
|
+
{ name: "copy_token" },
|
|
88
|
+
],
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Endpoint for filtered events
|
|
93
|
+
*/
|
|
94
|
+
export const eventsEndpoint = definePipe("events_endpoint", {
|
|
95
|
+
description: "Endpoint for filtered events",
|
|
96
|
+
params: {
|
|
97
|
+
env: p.string().optional("prod"),
|
|
98
|
+
user_id: p.uint64(),
|
|
99
|
+
},
|
|
100
|
+
nodes: [
|
|
101
|
+
node({
|
|
102
|
+
name: "base",
|
|
103
|
+
description: "Base filter",
|
|
104
|
+
sql: \`
|
|
105
|
+
SELECT event_id, user_id, payload
|
|
106
|
+
FROM events
|
|
107
|
+
WHERE user_id = {{UInt64(user_id)}}
|
|
108
|
+
AND env = {{String(env, 'prod')}}
|
|
109
|
+
\`,
|
|
110
|
+
}),
|
|
111
|
+
node({
|
|
112
|
+
name: "endpoint",
|
|
113
|
+
sql: \`
|
|
114
|
+
SELECT event_id AS event_id, user_id AS user_id
|
|
115
|
+
FROM base
|
|
116
|
+
\`,
|
|
117
|
+
}),
|
|
118
|
+
],
|
|
119
|
+
endpoint: { enabled: true, cache: { enabled: true, ttl: 120 } },
|
|
120
|
+
output: {
|
|
121
|
+
event_id: t.string(),
|
|
122
|
+
user_id: t.string(),
|
|
123
|
+
},
|
|
124
|
+
tokens: [
|
|
125
|
+
{ name: "endpoint_token" },
|
|
126
|
+
],
|
|
127
|
+
});
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* Materialized rollup
|
|
131
|
+
*/
|
|
132
|
+
export const eventsMv = defineMaterializedView("events_mv", {
|
|
133
|
+
description: "Materialized rollup",
|
|
134
|
+
datasource: eventsRollup,
|
|
135
|
+
deploymentMethod: "alter",
|
|
136
|
+
nodes: [
|
|
137
|
+
node({
|
|
138
|
+
name: "rollup",
|
|
139
|
+
sql: \`
|
|
140
|
+
SELECT user_id, count() AS total
|
|
141
|
+
FROM events
|
|
142
|
+
GROUP BY user_id
|
|
143
|
+
\`,
|
|
144
|
+
}),
|
|
145
|
+
],
|
|
146
|
+
tokens: [
|
|
147
|
+
{ name: "mv_token" },
|
|
148
|
+
],
|
|
149
|
+
});
|
|
150
|
+
|
|
151
|
+
export const statsPipe = definePipe("stats_pipe", {
|
|
152
|
+
params: {
|
|
153
|
+
min_total: p.uint32().optional(10),
|
|
154
|
+
},
|
|
155
|
+
nodes: [
|
|
156
|
+
node({
|
|
157
|
+
name: "agg",
|
|
158
|
+
sql: \`
|
|
159
|
+
SELECT user_id, count() AS total
|
|
160
|
+
FROM events
|
|
161
|
+
GROUP BY user_id
|
|
162
|
+
\`,
|
|
163
|
+
}),
|
|
164
|
+
node({
|
|
165
|
+
name: "final",
|
|
166
|
+
sql: \`
|
|
167
|
+
SELECT user_id, total
|
|
168
|
+
FROM agg
|
|
169
|
+
WHERE total > {{UInt32(min_total, 10)}}
|
|
170
|
+
\`,
|
|
171
|
+
}),
|
|
172
|
+
],
|
|
173
|
+
tokens: [
|
|
174
|
+
{ name: "stats_token" },
|
|
175
|
+
],
|
|
176
|
+
});
|
|
177
|
+
`;
|
|
178
|
+
const EXPECTED_PARTIAL_OUTPUT = `/**
|
|
179
|
+
* Generated by tinybird migrate.
|
|
180
|
+
* Review endpoint output schemas and any defaults before production use.
|
|
181
|
+
*/
|
|
182
|
+
|
|
183
|
+
import { createKafkaConnection, defineDatasource, definePipe, defineMaterializedView, defineCopyPipe, node, t, engine, p } from "@tinybirdco/sdk";
|
|
184
|
+
|
|
185
|
+
// Connections
|
|
186
|
+
|
|
187
|
+
export const stream = createKafkaConnection("stream", {
|
|
188
|
+
bootstrapServers: "localhost:9092",
|
|
189
|
+
});
|
|
190
|
+
|
|
191
|
+
// Datasources
|
|
192
|
+
|
|
193
|
+
export const events = defineDatasource("events", {
|
|
194
|
+
jsonPaths: false,
|
|
195
|
+
schema: {
|
|
196
|
+
event_id: t.string(),
|
|
197
|
+
user_id: t.uint64(),
|
|
198
|
+
created_at: t.dateTime(),
|
|
199
|
+
},
|
|
200
|
+
engine: engine.mergeTree({ sortingKey: "event_id" }),
|
|
201
|
+
kafka: {
|
|
202
|
+
connection: stream,
|
|
203
|
+
topic: "events_topic",
|
|
204
|
+
},
|
|
205
|
+
});
|
|
206
|
+
|
|
207
|
+
// Pipes
|
|
208
|
+
|
|
209
|
+
export const eventsEndpoint = definePipe("events_endpoint", {
|
|
210
|
+
params: {
|
|
211
|
+
user_id: p.uint64(),
|
|
212
|
+
},
|
|
213
|
+
nodes: [
|
|
214
|
+
node({
|
|
215
|
+
name: "source",
|
|
216
|
+
sql: \`
|
|
217
|
+
SELECT event_id, user_id
|
|
218
|
+
FROM events
|
|
219
|
+
\`,
|
|
220
|
+
}),
|
|
221
|
+
node({
|
|
222
|
+
name: "endpoint",
|
|
223
|
+
sql: \`
|
|
224
|
+
SELECT event_id AS event_id, user_id AS user_id
|
|
225
|
+
FROM source
|
|
226
|
+
WHERE user_id = {{UInt64(user_id)}}
|
|
227
|
+
\`,
|
|
228
|
+
}),
|
|
229
|
+
],
|
|
230
|
+
endpoint: true,
|
|
231
|
+
output: {
|
|
232
|
+
event_id: t.string(),
|
|
233
|
+
user_id: t.string(),
|
|
234
|
+
},
|
|
235
|
+
tokens: [
|
|
236
|
+
{ name: "endpoint_token" },
|
|
237
|
+
],
|
|
238
|
+
});
|
|
239
|
+
`;
|
|
240
|
+
describe("runMigrate", () => {
|
|
241
|
+
const tempDirs = [];
|
|
242
|
+
afterEach(() => {
|
|
243
|
+
for (const dir of tempDirs) {
|
|
244
|
+
try {
|
|
245
|
+
fs.rmSync(dir, { recursive: true });
|
|
246
|
+
}
|
|
247
|
+
catch {
|
|
248
|
+
// Ignore cleanup failures
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
tempDirs.length = 0;
|
|
252
|
+
});
|
|
253
|
+
it("migrates complex resources including endpoint, materialized, and copy pipes", async () => {
|
|
254
|
+
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-migrate-"));
|
|
255
|
+
tempDirs.push(tempDir);
|
|
256
|
+
writeFile(tempDir, "stream.connection", `TYPE kafka
|
|
257
|
+
KAFKA_BOOTSTRAP_SERVERS localhost:9092
|
|
258
|
+
KAFKA_SECURITY_PROTOCOL SASL_SSL
|
|
259
|
+
KAFKA_SASL_MECHANISM PLAIN
|
|
260
|
+
KAFKA_KEY api-key
|
|
261
|
+
KAFKA_SECRET api-secret
|
|
262
|
+
KAFKA_SSL_CA_PEM ca-pem-content
|
|
263
|
+
`);
|
|
264
|
+
writeFile(tempDir, "events.datasource", `DESCRIPTION >
|
|
265
|
+
Events from Kafka stream
|
|
266
|
+
SCHEMA >
|
|
267
|
+
event_id String \`json:$.event_id\`,
|
|
268
|
+
user_id UInt64 \`json:$.user.id\`,
|
|
269
|
+
env String \`json:$.env\` DEFAULT 'prod',
|
|
270
|
+
is_test Bool \`json:$.meta.is_test\` DEFAULT 0,
|
|
271
|
+
updated_at DateTime \`json:$.updated_at\`,
|
|
272
|
+
payload String \`json:$.payload\` DEFAULT '{}' CODEC(ZSTD(1))
|
|
273
|
+
|
|
274
|
+
ENGINE "ReplacingMergeTree"
|
|
275
|
+
ENGINE_SORTING_KEY "event_id, user_id"
|
|
276
|
+
ENGINE_PARTITION_KEY "toYYYYMM(updated_at)"
|
|
277
|
+
ENGINE_PRIMARY_KEY "event_id"
|
|
278
|
+
ENGINE_TTL "updated_at + toIntervalDay(30)"
|
|
279
|
+
ENGINE_VER "updated_at"
|
|
280
|
+
ENGINE_SETTINGS "index_granularity=8192, enable_mixed_granularity_parts=true"
|
|
281
|
+
KAFKA_CONNECTION_NAME stream
|
|
282
|
+
KAFKA_TOPIC events_topic
|
|
283
|
+
KAFKA_GROUP_ID events-consumer
|
|
284
|
+
KAFKA_AUTO_OFFSET_RESET earliest
|
|
285
|
+
TOKEN events_read READ
|
|
286
|
+
TOKEN events_append APPEND
|
|
287
|
+
SHARED_WITH >
|
|
288
|
+
workspace_a,
|
|
289
|
+
workspace_b
|
|
290
|
+
FORWARD_QUERY >
|
|
291
|
+
SELECT *
|
|
292
|
+
FROM events_mv
|
|
293
|
+
`);
|
|
294
|
+
writeFile(tempDir, "events_rollup.datasource", `SCHEMA >
|
|
295
|
+
user_id UInt64,
|
|
296
|
+
total UInt64
|
|
297
|
+
|
|
298
|
+
ENGINE "SummingMergeTree"
|
|
299
|
+
ENGINE_SORTING_KEY "user_id"
|
|
300
|
+
ENGINE_SUMMING_COLUMNS "total"
|
|
301
|
+
`);
|
|
302
|
+
writeFile(tempDir, "events_endpoint.pipe", `DESCRIPTION >
|
|
303
|
+
Endpoint for filtered events
|
|
304
|
+
NODE base
|
|
305
|
+
DESCRIPTION >
|
|
306
|
+
Base filter
|
|
307
|
+
SQL >
|
|
308
|
+
%
|
|
309
|
+
SELECT event_id, user_id, payload
|
|
310
|
+
FROM events
|
|
311
|
+
WHERE user_id = {{UInt64(user_id)}}
|
|
312
|
+
AND env = {{String(env, 'prod')}}
|
|
313
|
+
NODE endpoint
|
|
314
|
+
SQL >
|
|
315
|
+
SELECT event_id AS event_id, user_id AS user_id
|
|
316
|
+
FROM base
|
|
317
|
+
TYPE endpoint
|
|
318
|
+
CACHE 120
|
|
319
|
+
TOKEN endpoint_token READ
|
|
320
|
+
`);
|
|
321
|
+
writeFile(tempDir, "events_mv.pipe", `DESCRIPTION >
|
|
322
|
+
Materialized rollup
|
|
323
|
+
NODE rollup
|
|
324
|
+
SQL >
|
|
325
|
+
SELECT user_id, count() AS total
|
|
326
|
+
FROM events
|
|
327
|
+
GROUP BY user_id
|
|
328
|
+
TYPE MATERIALIZED
|
|
329
|
+
DATASOURCE events_rollup
|
|
330
|
+
DEPLOYMENT_METHOD alter
|
|
331
|
+
TOKEN mv_token READ
|
|
332
|
+
`);
|
|
333
|
+
writeFile(tempDir, "copy_events.pipe", `NODE copy_node
|
|
334
|
+
SQL >
|
|
335
|
+
SELECT event_id, user_id
|
|
336
|
+
FROM events
|
|
337
|
+
TYPE COPY
|
|
338
|
+
TARGET_DATASOURCE events_rollup
|
|
339
|
+
COPY_SCHEDULE @on-demand
|
|
340
|
+
COPY_MODE replace
|
|
341
|
+
TOKEN copy_token READ
|
|
342
|
+
`);
|
|
343
|
+
writeFile(tempDir, "stats_pipe.pipe", `NODE agg
|
|
344
|
+
SQL >
|
|
345
|
+
SELECT user_id, count() AS total
|
|
346
|
+
FROM events
|
|
347
|
+
GROUP BY user_id
|
|
348
|
+
NODE final
|
|
349
|
+
SQL >
|
|
350
|
+
SELECT user_id, total
|
|
351
|
+
FROM agg
|
|
352
|
+
WHERE total > {{UInt32(min_total, 10)}}
|
|
353
|
+
TOKEN stats_token READ
|
|
354
|
+
`);
|
|
355
|
+
const result = await runMigrate({
|
|
356
|
+
cwd: tempDir,
|
|
357
|
+
patterns: ["."],
|
|
358
|
+
strict: true,
|
|
359
|
+
});
|
|
360
|
+
expect(result.success).toBe(true);
|
|
361
|
+
expect(result.errors).toHaveLength(0);
|
|
362
|
+
expect(result.migrated).toHaveLength(7);
|
|
363
|
+
expect(result.migrated.filter((resource) => resource.kind === "connection")).toHaveLength(1);
|
|
364
|
+
expect(result.migrated.filter((resource) => resource.kind === "datasource")).toHaveLength(2);
|
|
365
|
+
expect(result.migrated.filter((resource) => resource.kind === "pipe")).toHaveLength(4);
|
|
366
|
+
expect(path.basename(result.outputPath)).toBe("tinybird.migration.ts");
|
|
367
|
+
expect(fs.existsSync(result.outputPath)).toBe(true);
|
|
368
|
+
const output = fs.readFileSync(result.outputPath, "utf-8");
|
|
369
|
+
expect(output).toBe(EXPECTED_COMPLEX_OUTPUT);
|
|
370
|
+
});
|
|
371
|
+
it("continues processing and reports all errors while writing complex migratable resources", async () => {
|
|
372
|
+
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-migrate-"));
|
|
373
|
+
tempDirs.push(tempDir);
|
|
374
|
+
writeFile(tempDir, "stream.connection", `TYPE kafka
|
|
375
|
+
KAFKA_BOOTSTRAP_SERVERS localhost:9092
|
|
376
|
+
`);
|
|
377
|
+
writeFile(tempDir, "events.datasource", `SCHEMA >
|
|
378
|
+
event_id String,
|
|
379
|
+
user_id UInt64,
|
|
380
|
+
created_at DateTime
|
|
381
|
+
|
|
382
|
+
ENGINE "MergeTree"
|
|
383
|
+
ENGINE_SORTING_KEY "event_id"
|
|
384
|
+
KAFKA_CONNECTION_NAME stream
|
|
385
|
+
KAFKA_TOPIC events_topic
|
|
386
|
+
`);
|
|
387
|
+
writeFile(tempDir, "events_endpoint.pipe", `NODE source
|
|
388
|
+
SQL >
|
|
389
|
+
SELECT event_id, user_id
|
|
390
|
+
FROM events
|
|
391
|
+
NODE endpoint
|
|
392
|
+
SQL >
|
|
393
|
+
SELECT event_id AS event_id, user_id AS user_id
|
|
394
|
+
FROM source
|
|
395
|
+
WHERE user_id = {{UInt64(user_id)}}
|
|
396
|
+
TYPE endpoint
|
|
397
|
+
TOKEN endpoint_token READ
|
|
398
|
+
`);
|
|
399
|
+
writeFile(tempDir, "events_mv.pipe", `NODE rollup
|
|
400
|
+
SQL >
|
|
401
|
+
SELECT user_id, count() AS total
|
|
402
|
+
FROM events
|
|
403
|
+
GROUP BY user_id
|
|
404
|
+
TYPE MATERIALIZED
|
|
405
|
+
DATASOURCE missing_ds
|
|
406
|
+
`);
|
|
407
|
+
writeFile(tempDir, "broken.pipe", `NODE broken
|
|
408
|
+
SQL >
|
|
409
|
+
SELECT *
|
|
410
|
+
FROM events
|
|
411
|
+
TYPE endpoint
|
|
412
|
+
UNSUPPORTED_DIRECTIVE true
|
|
413
|
+
`);
|
|
414
|
+
const result = await runMigrate({
|
|
415
|
+
cwd: tempDir,
|
|
416
|
+
patterns: ["."],
|
|
417
|
+
strict: true,
|
|
418
|
+
});
|
|
419
|
+
expect(result.success).toBe(false);
|
|
420
|
+
expect(result.errors).toHaveLength(2);
|
|
421
|
+
expect(result.errors.map((error) => error.message)).toEqual(expect.arrayContaining([
|
|
422
|
+
'Unsupported pipe directive in strict mode: "UNSUPPORTED_DIRECTIVE true"',
|
|
423
|
+
'Materialized pipe references missing/unmigrated datasource "missing_ds".',
|
|
424
|
+
]));
|
|
425
|
+
expect(result.migrated.filter((resource) => resource.kind === "connection")).toHaveLength(1);
|
|
426
|
+
expect(result.migrated.filter((resource) => resource.kind === "datasource")).toHaveLength(1);
|
|
427
|
+
expect(result.migrated.filter((resource) => resource.kind === "pipe")).toHaveLength(1);
|
|
428
|
+
expect(fs.existsSync(result.outputPath)).toBe(true);
|
|
429
|
+
const output = fs.readFileSync(result.outputPath, "utf-8");
|
|
430
|
+
expect(output).toBe(EXPECTED_PARTIAL_OUTPUT);
|
|
431
|
+
});
|
|
432
|
+
it("returns exact output content in dry-run mode for complex migratable resources", async () => {
|
|
433
|
+
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-migrate-"));
|
|
434
|
+
tempDirs.push(tempDir);
|
|
435
|
+
writeFile(tempDir, "stream.connection", `TYPE kafka
|
|
436
|
+
KAFKA_BOOTSTRAP_SERVERS localhost:9092
|
|
437
|
+
`);
|
|
438
|
+
writeFile(tempDir, "events.datasource", `SCHEMA >
|
|
439
|
+
event_id String,
|
|
440
|
+
user_id UInt64,
|
|
441
|
+
created_at DateTime
|
|
442
|
+
|
|
443
|
+
ENGINE "MergeTree"
|
|
444
|
+
ENGINE_SORTING_KEY "event_id"
|
|
445
|
+
KAFKA_CONNECTION_NAME stream
|
|
446
|
+
KAFKA_TOPIC events_topic
|
|
447
|
+
`);
|
|
448
|
+
writeFile(tempDir, "events_endpoint.pipe", `NODE source
|
|
449
|
+
SQL >
|
|
450
|
+
SELECT event_id, user_id
|
|
451
|
+
FROM events
|
|
452
|
+
NODE endpoint
|
|
453
|
+
SQL >
|
|
454
|
+
SELECT event_id AS event_id, user_id AS user_id
|
|
455
|
+
FROM source
|
|
456
|
+
WHERE user_id = {{UInt64(user_id)}}
|
|
457
|
+
TYPE endpoint
|
|
458
|
+
TOKEN endpoint_token READ
|
|
459
|
+
`);
|
|
460
|
+
const result = await runMigrate({
|
|
461
|
+
cwd: tempDir,
|
|
462
|
+
patterns: ["."],
|
|
463
|
+
strict: true,
|
|
464
|
+
dryRun: true,
|
|
465
|
+
});
|
|
466
|
+
expect(result.success).toBe(true);
|
|
467
|
+
expect(result.errors).toHaveLength(0);
|
|
468
|
+
expect(result.migrated).toHaveLength(3);
|
|
469
|
+
expect(result.outputContent).toBe(EXPECTED_PARTIAL_OUTPUT);
|
|
470
|
+
expect(fs.existsSync(result.outputPath)).toBe(false);
|
|
471
|
+
});
|
|
472
|
+
});
|
|
473
|
+
//# sourceMappingURL=migrate.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"migrate.test.js","sourceRoot":"","sources":["../../../src/cli/commands/migrate.test.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AACzD,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAE1C,SAAS,SAAS,CAAC,GAAW,EAAE,YAAoB,EAAE,OAAe;IACnE,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;IAC9C,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1D,EAAE,CAAC,aAAa,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;AACtC,CAAC;AAED,MAAM,uBAAuB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAsK/B,CAAC;AAEF,MAAM,uBAAuB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA6D/B,CAAC;AAEF,QAAQ,CAAC,YAAY,EAAE,GAAG,EAAE;IAC1B,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,SAAS,CAAC,GAAG,EAAE;QACb,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;YAC3B,IAAI,CAAC;gBACH,EAAE,CAAC,MAAM,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACtC,CAAC;YAAC,MAAM,CAAC;gBACP,0BAA0B;YAC5B,CAAC;QACH,CAAC;QACD,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;IACtB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6EAA6E,EAAE,KAAK,IAAI,EAAE;QAC3F,MAAM,OAAO,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,mBAAmB,CAAC,CAAC,CAAC;QAC5E,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAEvB,SAAS,CACP,OAAO,EACP,mBAAmB,EACnB;;;;;;;CAOL,CACI,CAAC;QAEF,SAAS,CACP,OAAO,EACP,mBAAmB,EACnB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA6BL,CACI,CAAC;QAEF,SAAS,CACP,OAAO,EACP,0BAA0B,EAC1B;;;;;;;CAOL,CACI,CAAC;QAEF,SAAS,CACP,OAAO,EACP,sBAAsB,EACtB;;;;;;;;;;;;;;;;;;CAkBL,CACI,CAAC;QAEF,SAAS,CACP,OAAO,EACP,gBAAgB,EAChB;;;;;;;;;;;CAWL,CACI,CAAC;QAEF,SAAS,CACP,OAAO,EACP,kBAAkB,EAClB;;;;;;;;;CASL,CACI,CAAC;QAEF,SAAS,CACP,OAAO,EACP,iBAAiB,EACjB;;;;;;;;;;;CAWL,CACI,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC;YAC9B,GAAG,EAAE,OAAO;YACZ,QAAQ,EAAE,CAAC,GAAG,CAAC;YACf,MAAM,EAAE,IAAI;SACb,CAAC,CAAC;QAEH,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACtC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACxC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,KAAK,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC7F,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,KAAK,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC7F,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACvF,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;QACvE,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEpD,MAAM,MAAM,GAAG,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QAC3D,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wFAAwF,EAAE,KAAK,IAAI,EAAE;QACtG,MAAM,OAAO,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,mBAAmB,CAAC,CAAC,CAAC;QAC5E,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAEvB,SAAS,CACP,OAAO,EACP,mBAAmB,EACnB;;CAEL,CACI,CAAC;QAEF,SAAS,CACP,OAAO,EACP,mBAAmB,EACnB;;;;;;;;;CASL,CACI,CAAC;QAEF,SAAS,CACP,OAAO,EACP,sBAAsB,EACtB;;;;;;;;;;;CAWL,CACI,CAAC;QAEF,SAAS,CACP,OAAO,EACP,gBAAgB,EAChB;;;;;;;CAOL,CACI,CAAC;QAEF,SAAS,CACP,OAAO,EACP,aAAa,EACb;;;;;;CAML,CACI,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC;YAC9B,GAAG,EAAE,OAAO;YACZ,QAAQ,EAAE,CAAC,GAAG,CAAC;YACf,MAAM,EAAE,IAAI;SACb,CAAC,CAAC;QAEH,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACtC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CACzD,MAAM,CAAC,eAAe,CAAC;YACrB,yEAAyE;YACzE,0EAA0E;SAC3E,CAAC,CACH,CAAC;QACF,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,KAAK,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC7F,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,KAAK,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC7F,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACvF,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEpD,MAAM,MAAM,GAAG,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QAC3D,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+EAA+E,EAAE,KAAK,IAAI,EAAE;QAC7F,MAAM,OAAO,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,mBAAmB,CAAC,CAAC,CAAC;QAC5E,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAEvB,SAAS,CACP,OAAO,EACP,mBAAmB,EACnB;;CAEL,CACI,CAAC;QAEF,SAAS,CACP,OAAO,EACP,mBAAmB,EACnB;;;;;;;;;CASL,CACI,CAAC;QAEF,SAAS,CACP,OAAO,EACP,sBAAsB,EACtB;;;;;;;;;;;CAWL,CACI,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC;YAC9B,GAAG,EAAE,OAAO;YACZ,QAAQ,EAAE,CAAC,GAAG,CAAC;YACf,MAAM,EAAE,IAAI;YACZ,MAAM,EAAE,IAAI;SACb,CAAC,CAAC;QAEH,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACtC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACxC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;QAC3D,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACvD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|