@tinybirdco/sdk 0.0.77 → 0.0.79
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -2
- package/dist/api/branches.d.ts +4 -3
- package/dist/api/branches.d.ts.map +1 -1
- package/dist/api/branches.js +4 -4
- package/dist/api/branches.js.map +1 -1
- package/dist/api/branches.test.js +34 -1
- package/dist/api/branches.test.js.map +1 -1
- package/dist/cli/commands/build.d.ts.map +1 -1
- package/dist/cli/commands/build.js +7 -1
- package/dist/cli/commands/build.js.map +1 -1
- package/dist/cli/commands/build.test.js +159 -0
- package/dist/cli/commands/build.test.js.map +1 -1
- package/dist/cli/commands/clear.d.ts.map +1 -1
- package/dist/cli/commands/clear.js +4 -1
- package/dist/cli/commands/clear.js.map +1 -1
- package/dist/cli/commands/deploy.test.js +1 -0
- package/dist/cli/commands/deploy.test.js.map +1 -1
- package/dist/cli/commands/dev.d.ts.map +1 -1
- package/dist/cli/commands/dev.js +7 -1
- package/dist/cli/commands/dev.js.map +1 -1
- package/dist/cli/commands/generate.test.js +3 -0
- package/dist/cli/commands/generate.test.js.map +1 -1
- package/dist/cli/commands/migrate.d.ts.map +1 -1
- package/dist/cli/commands/migrate.js +14 -1
- package/dist/cli/commands/migrate.js.map +1 -1
- package/dist/cli/commands/migrate.test.js +60 -0
- package/dist/cli/commands/migrate.test.js.map +1 -1
- package/dist/cli/commands/preview.d.ts.map +1 -1
- package/dist/cli/commands/preview.js +4 -1
- package/dist/cli/commands/preview.js.map +1 -1
- package/dist/cli/commands/preview.test.js +116 -2
- package/dist/cli/commands/preview.test.js.map +1 -1
- package/dist/cli/config-types.d.ts +4 -0
- package/dist/cli/config-types.d.ts.map +1 -1
- package/dist/cli/config-types.js +1 -1
- package/dist/cli/config-types.js.map +1 -1
- package/dist/cli/config.d.ts +4 -2
- package/dist/cli/config.d.ts.map +1 -1
- package/dist/cli/config.js +27 -0
- package/dist/cli/config.js.map +1 -1
- package/dist/cli/config.test.js +79 -0
- package/dist/cli/config.test.js.map +1 -1
- package/dist/cli/index.js +2 -2
- package/dist/cli/index.js.map +1 -1
- package/dist/client/base.d.ts.map +1 -1
- package/dist/client/base.js +4 -1
- package/dist/client/base.js.map +1 -1
- package/dist/client/base.test.js +2 -1
- package/dist/client/base.test.js.map +1 -1
- package/dist/generator/connection.d.ts.map +1 -1
- package/dist/generator/connection.js +15 -1
- package/dist/generator/connection.js.map +1 -1
- package/dist/generator/connection.test.js +10 -1
- package/dist/generator/connection.test.js.map +1 -1
- package/dist/generator/datasource.d.ts.map +1 -1
- package/dist/generator/datasource.js +17 -1
- package/dist/generator/datasource.js.map +1 -1
- package/dist/generator/datasource.test.js +44 -1
- package/dist/generator/datasource.test.js.map +1 -1
- package/dist/index.d.ts +4 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/migrate/emit-ts.d.ts.map +1 -1
- package/dist/migrate/emit-ts.js +28 -0
- package/dist/migrate/emit-ts.js.map +1 -1
- package/dist/migrate/parse-connection.d.ts +2 -2
- package/dist/migrate/parse-connection.d.ts.map +1 -1
- package/dist/migrate/parse-connection.js +60 -6
- package/dist/migrate/parse-connection.js.map +1 -1
- package/dist/migrate/parse-connection.test.js +18 -0
- package/dist/migrate/parse-connection.test.js.map +1 -1
- package/dist/migrate/parse-datasource.d.ts.map +1 -1
- package/dist/migrate/parse-datasource.js +27 -2
- package/dist/migrate/parse-datasource.js.map +1 -1
- package/dist/migrate/types.d.ts +15 -1
- package/dist/migrate/types.d.ts.map +1 -1
- package/dist/schema/connection.d.ts +46 -1
- package/dist/schema/connection.d.ts.map +1 -1
- package/dist/schema/connection.js +39 -0
- package/dist/schema/connection.js.map +1 -1
- package/dist/schema/connection.test.js +46 -1
- package/dist/schema/connection.test.js.map +1 -1
- package/dist/schema/datasource.d.ts +14 -1
- package/dist/schema/datasource.d.ts.map +1 -1
- package/dist/schema/datasource.js +2 -2
- package/dist/schema/datasource.js.map +1 -1
- package/dist/schema/datasource.test.js +2 -2
- package/dist/schema/datasource.test.js.map +1 -1
- package/package.json +1 -1
- package/src/api/branches.test.ts +38 -1
- package/src/api/branches.ts +8 -6
- package/src/cli/commands/build.test.ts +176 -0
- package/src/cli/commands/build.ts +9 -2
- package/src/cli/commands/clear.ts +8 -1
- package/src/cli/commands/deploy.test.ts +1 -0
- package/src/cli/commands/dev.ts +9 -1
- package/src/cli/commands/generate.test.ts +3 -0
- package/src/cli/commands/migrate.test.ts +95 -0
- package/src/cli/commands/migrate.ts +17 -1
- package/src/cli/commands/preview.test.ts +133 -2
- package/src/cli/commands/preview.ts +6 -2
- package/src/cli/config-types.ts +4 -0
- package/src/cli/config.test.ts +123 -0
- package/src/cli/config.ts +41 -3
- package/src/cli/index.ts +8 -2
- package/src/client/base.test.ts +3 -1
- package/src/client/base.ts +7 -1
- package/src/generator/connection.test.ts +17 -0
- package/src/generator/connection.ts +18 -0
- package/src/generator/datasource.test.ts +54 -1
- package/src/generator/datasource.ts +24 -1
- package/src/index.ts +6 -1
- package/src/migrate/emit-ts.ts +44 -3
- package/src/migrate/parse-connection.test.ts +41 -0
- package/src/migrate/parse-connection.ts +86 -7
- package/src/migrate/parse-datasource.ts +35 -2
- package/src/migrate/types.ts +18 -1
- package/src/schema/connection.test.ts +65 -0
- package/src/schema/connection.ts +76 -1
- package/src/schema/datasource.test.ts +2 -2
- package/src/schema/datasource.ts +21 -2
|
@@ -1,7 +1,45 @@
|
|
|
1
|
-
import { describe, it, expect } from "vitest";
|
|
2
|
-
import { generatePreviewBranchName } from "./preview.js";
|
|
1
|
+
import { describe, it, expect, vi, beforeEach } from "vitest";
|
|
2
|
+
import { generatePreviewBranchName, runPreview } from "./preview.js";
|
|
3
|
+
|
|
4
|
+
vi.mock("../config.js", () => ({
|
|
5
|
+
loadConfigAsync: vi.fn(),
|
|
6
|
+
LOCAL_BASE_URL: "http://localhost:7181",
|
|
7
|
+
}));
|
|
8
|
+
|
|
9
|
+
vi.mock("../../generator/index.js", () => ({
|
|
10
|
+
buildFromInclude: vi.fn(),
|
|
11
|
+
}));
|
|
12
|
+
|
|
13
|
+
vi.mock("../../api/branches.js", () => ({
|
|
14
|
+
createBranch: vi.fn(),
|
|
15
|
+
deleteBranch: vi.fn(),
|
|
16
|
+
getBranch: vi.fn(),
|
|
17
|
+
}));
|
|
18
|
+
|
|
19
|
+
vi.mock("../../api/deploy.js", () => ({
|
|
20
|
+
deployToMain: vi.fn(),
|
|
21
|
+
}));
|
|
22
|
+
|
|
23
|
+
vi.mock("../../api/build.js", () => ({
|
|
24
|
+
buildToTinybird: vi.fn(),
|
|
25
|
+
}));
|
|
26
|
+
|
|
27
|
+
vi.mock("../../api/local.js", () => ({
|
|
28
|
+
getLocalTokens: vi.fn(),
|
|
29
|
+
getOrCreateLocalWorkspace: vi.fn(),
|
|
30
|
+
LocalNotRunningError: class LocalNotRunningError extends Error {},
|
|
31
|
+
}));
|
|
32
|
+
|
|
33
|
+
vi.mock("../git.js", () => ({
|
|
34
|
+
sanitizeBranchName: (value: string) => value.replace(/[^a-zA-Z0-9]/g, "_"),
|
|
35
|
+
getCurrentGitBranch: vi.fn(() => "feature/test"),
|
|
36
|
+
}));
|
|
3
37
|
|
|
4
38
|
describe("Preview command", () => {
|
|
39
|
+
beforeEach(() => {
|
|
40
|
+
vi.clearAllMocks();
|
|
41
|
+
});
|
|
42
|
+
|
|
5
43
|
describe("generatePreviewBranchName", () => {
|
|
6
44
|
it("generates name with tmp_ci prefix", () => {
|
|
7
45
|
const result = generatePreviewBranchName("feature-branch");
|
|
@@ -34,4 +72,97 @@ describe("Preview command", () => {
|
|
|
34
72
|
expect(result1).toBe(result2);
|
|
35
73
|
});
|
|
36
74
|
});
|
|
75
|
+
|
|
76
|
+
describe("branch_data_mode wiring", () => {
|
|
77
|
+
it("uses config-only last_partition when creating cloud preview branch", async () => {
|
|
78
|
+
const { loadConfigAsync } = await import("../config.js");
|
|
79
|
+
const { buildFromInclude } = await import("../../generator/index.js");
|
|
80
|
+
const { getBranch, createBranch } = await import("../../api/branches.js");
|
|
81
|
+
const { deployToMain } = await import("../../api/deploy.js");
|
|
82
|
+
|
|
83
|
+
vi.mocked(loadConfigAsync).mockResolvedValue({
|
|
84
|
+
include: ["test.ts"],
|
|
85
|
+
token: "p.test-token",
|
|
86
|
+
baseUrl: "https://api.tinybird.co",
|
|
87
|
+
configPath: "/test/tinybird.config.json",
|
|
88
|
+
devMode: "branch",
|
|
89
|
+
cwd: "/test",
|
|
90
|
+
gitBranch: "feature-test",
|
|
91
|
+
tinybirdBranch: "feature_test",
|
|
92
|
+
isMainBranch: false,
|
|
93
|
+
branchDataMode: "last_partition",
|
|
94
|
+
});
|
|
95
|
+
vi.mocked(buildFromInclude).mockResolvedValue({
|
|
96
|
+
resources: { datasources: [], pipes: [], connections: [] },
|
|
97
|
+
entities: { datasources: {}, pipes: {}, connections: {}, rawDatasources: [], rawPipes: [], sourceFiles: [] },
|
|
98
|
+
stats: { datasourceCount: 0, pipeCount: 0, connectionCount: 0 },
|
|
99
|
+
});
|
|
100
|
+
vi.mocked(getBranch).mockRejectedValue(new Error("not found"));
|
|
101
|
+
vi.mocked(createBranch).mockResolvedValue({
|
|
102
|
+
id: "b1",
|
|
103
|
+
name: "tmp_ci_feature_test",
|
|
104
|
+
token: "p.branch",
|
|
105
|
+
created_at: "2024-01-01T00:00:00Z",
|
|
106
|
+
});
|
|
107
|
+
vi.mocked(deployToMain).mockResolvedValue({
|
|
108
|
+
success: true,
|
|
109
|
+
result: "success",
|
|
110
|
+
datasourceCount: 0,
|
|
111
|
+
pipeCount: 0,
|
|
112
|
+
connectionCount: 0,
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
await runPreview();
|
|
116
|
+
expect(createBranch).toHaveBeenCalledWith(
|
|
117
|
+
expect.any(Object),
|
|
118
|
+
"tmp_ci_feature_test",
|
|
119
|
+
{ branch_data_mode: "last_partition" }
|
|
120
|
+
);
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
it("ignores config branch_data_mode in local mode", async () => {
|
|
124
|
+
const { loadConfigAsync } = await import("../config.js");
|
|
125
|
+
const { buildFromInclude } = await import("../../generator/index.js");
|
|
126
|
+
const { createBranch } = await import("../../api/branches.js");
|
|
127
|
+
const { getLocalTokens, getOrCreateLocalWorkspace } = await import("../../api/local.js");
|
|
128
|
+
const { buildToTinybird } = await import("../../api/build.js");
|
|
129
|
+
|
|
130
|
+
vi.mocked(loadConfigAsync).mockResolvedValue({
|
|
131
|
+
include: ["test.ts"],
|
|
132
|
+
token: "p.test-token",
|
|
133
|
+
baseUrl: "https://api.tinybird.co",
|
|
134
|
+
configPath: "/test/tinybird.config.json",
|
|
135
|
+
devMode: "local",
|
|
136
|
+
cwd: "/test",
|
|
137
|
+
gitBranch: "feature-test",
|
|
138
|
+
tinybirdBranch: "feature_test",
|
|
139
|
+
isMainBranch: false,
|
|
140
|
+
branchDataMode: "last_partition",
|
|
141
|
+
});
|
|
142
|
+
vi.mocked(buildFromInclude).mockResolvedValue({
|
|
143
|
+
resources: { datasources: [], pipes: [], connections: [] },
|
|
144
|
+
entities: { datasources: {}, pipes: {}, connections: {}, rawDatasources: [], rawPipes: [], sourceFiles: [] },
|
|
145
|
+
stats: { datasourceCount: 0, pipeCount: 0, connectionCount: 0 },
|
|
146
|
+
});
|
|
147
|
+
vi.mocked(getLocalTokens).mockResolvedValue({
|
|
148
|
+
admin_token: "admin-token",
|
|
149
|
+
user_token: "user-token",
|
|
150
|
+
workspace_admin_token: "workspace-admin-token",
|
|
151
|
+
});
|
|
152
|
+
vi.mocked(getOrCreateLocalWorkspace).mockResolvedValue({
|
|
153
|
+
workspace: { id: "lw1", name: "tmp_ci_feature_test", token: "local-token" },
|
|
154
|
+
wasCreated: true,
|
|
155
|
+
});
|
|
156
|
+
vi.mocked(buildToTinybird).mockResolvedValue({
|
|
157
|
+
success: true,
|
|
158
|
+
result: "success",
|
|
159
|
+
datasourceCount: 0,
|
|
160
|
+
pipeCount: 0,
|
|
161
|
+
connectionCount: 0,
|
|
162
|
+
});
|
|
163
|
+
|
|
164
|
+
await runPreview();
|
|
165
|
+
expect(createBranch).not.toHaveBeenCalled();
|
|
166
|
+
});
|
|
167
|
+
});
|
|
37
168
|
});
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
import { loadConfigAsync, LOCAL_BASE_URL, type ResolvedConfig, type DevMode } from "../config.js";
|
|
6
6
|
import { buildFromInclude, type BuildFromIncludeResult } from "../../generator/index.js";
|
|
7
|
-
import { createBranch, deleteBranch, getBranch, type TinybirdBranch } from "../../api/branches.js";
|
|
7
|
+
import { createBranch, deleteBranch, getBranch, type CreateBranchOptions, type TinybirdBranch } from "../../api/branches.js";
|
|
8
8
|
import { deployToMain } from "../../api/deploy.js";
|
|
9
9
|
import { buildToTinybird } from "../../api/build.js";
|
|
10
10
|
import {
|
|
@@ -226,6 +226,10 @@ export async function runPreview(options: PreviewCommandOptions = {}): Promise<P
|
|
|
226
226
|
let branch: TinybirdBranch;
|
|
227
227
|
try {
|
|
228
228
|
const apiConfig = { baseUrl: config.baseUrl, token: config.token };
|
|
229
|
+
const branchOptions: CreateBranchOptions | undefined =
|
|
230
|
+
config.branchDataMode === "last_partition"
|
|
231
|
+
? { branch_data_mode: "last_partition" }
|
|
232
|
+
: undefined;
|
|
229
233
|
|
|
230
234
|
// Check if branch already exists and delete it for a fresh start
|
|
231
235
|
try {
|
|
@@ -250,7 +254,7 @@ export async function runPreview(options: PreviewCommandOptions = {}): Promise<P
|
|
|
250
254
|
console.log(`[debug] Creating preview branch: ${previewBranchName}`);
|
|
251
255
|
}
|
|
252
256
|
|
|
253
|
-
branch = await createBranch(apiConfig, previewBranchName);
|
|
257
|
+
branch = await createBranch(apiConfig, previewBranchName, branchOptions);
|
|
254
258
|
|
|
255
259
|
if (debug) {
|
|
256
260
|
console.log(`[debug] Branch created: ${branch.name} (${branch.id})`);
|
package/src/cli/config-types.ts
CHANGED
|
@@ -11,6 +11,8 @@
|
|
|
11
11
|
* - "local": Use local Tinybird container at localhost:7181
|
|
12
12
|
*/
|
|
13
13
|
export type DevMode = "branch" | "local";
|
|
14
|
+
export type BranchDataMode = "last_partition";
|
|
15
|
+
export const BRANCH_DATA_MODE_VALUES = ["last_partition"] as const satisfies readonly BranchDataMode[];
|
|
14
16
|
|
|
15
17
|
/**
|
|
16
18
|
* Tinybird configuration file structure
|
|
@@ -26,4 +28,6 @@ export interface TinybirdConfig {
|
|
|
26
28
|
baseUrl?: string;
|
|
27
29
|
/** Development mode: "branch" (default) or "local" */
|
|
28
30
|
devMode?: DevMode;
|
|
31
|
+
/** Branch data mode applied on cloud branch creation (shared snake_case key) */
|
|
32
|
+
branch_data_mode?: BranchDataMode;
|
|
29
33
|
}
|
package/src/cli/config.test.ts
CHANGED
|
@@ -327,6 +327,129 @@ describe("Config", () => {
|
|
|
327
327
|
|
|
328
328
|
expect(result.devMode).toBe("local");
|
|
329
329
|
});
|
|
330
|
+
|
|
331
|
+
it("resolves branch_data_mode as last_partition", () => {
|
|
332
|
+
const config = {
|
|
333
|
+
include: ["lib/datasources.ts"],
|
|
334
|
+
token: "test-token",
|
|
335
|
+
branch_data_mode: "last_partition",
|
|
336
|
+
};
|
|
337
|
+
fs.writeFileSync(
|
|
338
|
+
path.join(tempDir, "tinybird.json"),
|
|
339
|
+
JSON.stringify(config)
|
|
340
|
+
);
|
|
341
|
+
|
|
342
|
+
const result = loadConfig(tempDir);
|
|
343
|
+
expect(result.branchDataMode).toBe("last_partition");
|
|
344
|
+
});
|
|
345
|
+
|
|
346
|
+
it("defaults branch_data_mode to last_partition when missing", () => {
|
|
347
|
+
const config = {
|
|
348
|
+
include: ["lib/datasources.ts"],
|
|
349
|
+
token: "test-token",
|
|
350
|
+
};
|
|
351
|
+
fs.writeFileSync(
|
|
352
|
+
path.join(tempDir, "tinybird.json"),
|
|
353
|
+
JSON.stringify(config)
|
|
354
|
+
);
|
|
355
|
+
|
|
356
|
+
const result = loadConfig(tempDir);
|
|
357
|
+
expect(result.branchDataMode).toBe("last_partition");
|
|
358
|
+
});
|
|
359
|
+
|
|
360
|
+
it("defaults empty branch_data_mode to last_partition", () => {
|
|
361
|
+
const config = {
|
|
362
|
+
include: ["lib/datasources.ts"],
|
|
363
|
+
token: "test-token",
|
|
364
|
+
branch_data_mode: " ",
|
|
365
|
+
};
|
|
366
|
+
fs.writeFileSync(
|
|
367
|
+
path.join(tempDir, "tinybird.json"),
|
|
368
|
+
JSON.stringify(config)
|
|
369
|
+
);
|
|
370
|
+
|
|
371
|
+
const result = loadConfig(tempDir);
|
|
372
|
+
expect(result.branchDataMode).toBe("last_partition");
|
|
373
|
+
});
|
|
374
|
+
|
|
375
|
+
it("throws when branch_data_mode is all_partitions", () => {
|
|
376
|
+
const config = {
|
|
377
|
+
include: ["lib/datasources.ts"],
|
|
378
|
+
token: "test-token",
|
|
379
|
+
branch_data_mode: "all_partitions",
|
|
380
|
+
};
|
|
381
|
+
fs.writeFileSync(
|
|
382
|
+
path.join(tempDir, "tinybird.json"),
|
|
383
|
+
JSON.stringify(config)
|
|
384
|
+
);
|
|
385
|
+
|
|
386
|
+
expect(() => loadConfig(tempDir)).toThrow("Invalid branch_data_mode");
|
|
387
|
+
});
|
|
388
|
+
|
|
389
|
+
it("throws when branch_data_mode is invalid", () => {
|
|
390
|
+
const config = {
|
|
391
|
+
include: ["lib/datasources.ts"],
|
|
392
|
+
token: "test-token",
|
|
393
|
+
branch_data_mode: "invalid",
|
|
394
|
+
};
|
|
395
|
+
fs.writeFileSync(
|
|
396
|
+
path.join(tempDir, "tinybird.json"),
|
|
397
|
+
JSON.stringify(config)
|
|
398
|
+
);
|
|
399
|
+
|
|
400
|
+
expect(() => loadConfig(tempDir)).toThrow("Invalid branch_data_mode");
|
|
401
|
+
});
|
|
402
|
+
|
|
403
|
+
it("warns when branch_data_mode is set with devMode local", () => {
|
|
404
|
+
const config = {
|
|
405
|
+
include: ["lib/datasources.ts"],
|
|
406
|
+
token: "test-token",
|
|
407
|
+
devMode: "local",
|
|
408
|
+
branch_data_mode: "last_partition",
|
|
409
|
+
};
|
|
410
|
+
fs.writeFileSync(
|
|
411
|
+
path.join(tempDir, "tinybird.json"),
|
|
412
|
+
JSON.stringify(config)
|
|
413
|
+
);
|
|
414
|
+
const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
|
|
415
|
+
|
|
416
|
+
loadConfig(tempDir);
|
|
417
|
+
|
|
418
|
+
expect(warnSpy).toHaveBeenCalledWith(
|
|
419
|
+
expect.stringContaining("branch_data_mode is set")
|
|
420
|
+
);
|
|
421
|
+
});
|
|
422
|
+
|
|
423
|
+
it("does not warn when branch_data_mode is implicit in local mode", () => {
|
|
424
|
+
const config = {
|
|
425
|
+
include: ["lib/datasources.ts"],
|
|
426
|
+
token: "test-token",
|
|
427
|
+
devMode: "local",
|
|
428
|
+
};
|
|
429
|
+
fs.writeFileSync(
|
|
430
|
+
path.join(tempDir, "tinybird.json"),
|
|
431
|
+
JSON.stringify(config)
|
|
432
|
+
);
|
|
433
|
+
const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
|
|
434
|
+
|
|
435
|
+
loadConfig(tempDir);
|
|
436
|
+
|
|
437
|
+
expect(warnSpy).not.toHaveBeenCalled();
|
|
438
|
+
});
|
|
439
|
+
|
|
440
|
+
it("throws when legacy branch_data_on_create key is used", () => {
|
|
441
|
+
const config = {
|
|
442
|
+
include: ["lib/datasources.ts"],
|
|
443
|
+
token: "test-token",
|
|
444
|
+
branch_data_on_create: "last_partition",
|
|
445
|
+
};
|
|
446
|
+
fs.writeFileSync(
|
|
447
|
+
path.join(tempDir, "tinybird.json"),
|
|
448
|
+
JSON.stringify(config)
|
|
449
|
+
);
|
|
450
|
+
|
|
451
|
+
expect(() => loadConfig(tempDir)).toThrow("renamed to `branch_data_mode`");
|
|
452
|
+
});
|
|
330
453
|
});
|
|
331
454
|
|
|
332
455
|
describe("loadConfigAsync", () => {
|
package/src/cli/config.ts
CHANGED
|
@@ -7,9 +7,17 @@ import * as path from "path";
|
|
|
7
7
|
import { config as loadDotenv } from "dotenv";
|
|
8
8
|
import { getCurrentGitBranch, isMainBranch, getTinybirdBranchName } from "./git.js";
|
|
9
9
|
|
|
10
|
-
// Re-export types from config-types.ts (separate file to avoid bundling esbuild)
|
|
11
|
-
export
|
|
12
|
-
|
|
10
|
+
// Re-export config types/constants from config-types.ts (separate file to avoid bundling esbuild)
|
|
11
|
+
export {
|
|
12
|
+
BRANCH_DATA_MODE_VALUES,
|
|
13
|
+
type BranchDataMode,
|
|
14
|
+
type DevMode,
|
|
15
|
+
type TinybirdConfig,
|
|
16
|
+
} from "./config-types.js";
|
|
17
|
+
import { BRANCH_DATA_MODE_VALUES } from "./config-types.js";
|
|
18
|
+
import type { BranchDataMode, DevMode, TinybirdConfig } from "./config-types.js";
|
|
19
|
+
|
|
20
|
+
const DEFAULT_BRANCH_DATA_MODE: BranchDataMode = "last_partition";
|
|
13
21
|
|
|
14
22
|
/**
|
|
15
23
|
* Resolved configuration with all values expanded
|
|
@@ -33,6 +41,8 @@ export interface ResolvedConfig {
|
|
|
33
41
|
isMainBranch: boolean;
|
|
34
42
|
/** Development mode: "branch" or "local" */
|
|
35
43
|
devMode: DevMode;
|
|
44
|
+
/** Branch data mode configured in tinybird.config.json */
|
|
45
|
+
branchDataMode?: BranchDataMode | null;
|
|
36
46
|
}
|
|
37
47
|
|
|
38
48
|
/**
|
|
@@ -196,6 +206,24 @@ export function findConfigFile(startDir: string): ConfigFileResult | null {
|
|
|
196
206
|
// Import the universal config loader
|
|
197
207
|
import { loadConfigFile } from "./config-loader.js";
|
|
198
208
|
|
|
209
|
+
function resolveBranchDataMode(raw: Record<string, unknown>): { mode: BranchDataMode | null; explicit: boolean } {
|
|
210
|
+
if (raw["branch_data_on_create"] !== undefined) {
|
|
211
|
+
throw new Error("`branch_data_on_create` has been renamed to `branch_data_mode`.");
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
const value = raw["branch_data_mode"];
|
|
215
|
+
if (value === undefined || value === null) return { mode: DEFAULT_BRANCH_DATA_MODE, explicit: false };
|
|
216
|
+
if (typeof value !== "string") throw new Error("branch_data_mode must be a string.");
|
|
217
|
+
const mode = value.trim().toLowerCase();
|
|
218
|
+
if (!mode) return { mode: DEFAULT_BRANCH_DATA_MODE, explicit: false };
|
|
219
|
+
if (!BRANCH_DATA_MODE_VALUES.includes(mode as BranchDataMode)) {
|
|
220
|
+
throw new Error(
|
|
221
|
+
`Invalid branch_data_mode '${value}'. Allowed values are: ${BRANCH_DATA_MODE_VALUES.join(", ")}.`
|
|
222
|
+
);
|
|
223
|
+
}
|
|
224
|
+
return { mode: mode as BranchDataMode, explicit: true };
|
|
225
|
+
}
|
|
226
|
+
|
|
199
227
|
/**
|
|
200
228
|
* Resolve a TinybirdConfig to a ResolvedConfig
|
|
201
229
|
*/
|
|
@@ -255,6 +283,15 @@ function resolveConfig(config: TinybirdConfig, configPath: string): ResolvedConf
|
|
|
255
283
|
|
|
256
284
|
// Resolve devMode (default to "branch")
|
|
257
285
|
const devMode: DevMode = config.devMode ?? "branch";
|
|
286
|
+
const { mode: branchDataMode, explicit: branchDataModeExplicit } = resolveBranchDataMode(
|
|
287
|
+
config as unknown as Record<string, unknown>
|
|
288
|
+
);
|
|
289
|
+
if (branchDataModeExplicit && branchDataMode && devMode === "local") {
|
|
290
|
+
console.warn(
|
|
291
|
+
"branch_data_mode is set in tinybird.config.json but dev_mode='local'. " +
|
|
292
|
+
"Branch data settings only apply to cloud branches."
|
|
293
|
+
);
|
|
294
|
+
}
|
|
258
295
|
|
|
259
296
|
return {
|
|
260
297
|
include,
|
|
@@ -266,6 +303,7 @@ function resolveConfig(config: TinybirdConfig, configPath: string): ResolvedConf
|
|
|
266
303
|
tinybirdBranch,
|
|
267
304
|
isMainBranch: isMainBranch(),
|
|
268
305
|
devMode,
|
|
306
|
+
branchDataMode,
|
|
269
307
|
};
|
|
270
308
|
}
|
|
271
309
|
|
package/src/cli/index.ts
CHANGED
|
@@ -270,7 +270,10 @@ function createCli(): Command {
|
|
|
270
270
|
.option("--debug", "Show debug output including API requests/responses")
|
|
271
271
|
.option("--local", "Use local Tinybird container")
|
|
272
272
|
.option("--branch", "Use Tinybird cloud with branches")
|
|
273
|
-
.option(
|
|
273
|
+
.option(
|
|
274
|
+
"--last-partition",
|
|
275
|
+
'[DEPRECATED] Use `branch_data_mode: "last_partition"` in tinybird.config.json. Copy the last partition of production data when creating a cloud branch.'
|
|
276
|
+
)
|
|
274
277
|
.action(async (options) => {
|
|
275
278
|
if (options.debug) {
|
|
276
279
|
process.env.TINYBIRD_DEBUG = "1";
|
|
@@ -679,7 +682,10 @@ function createCli(): Command {
|
|
|
679
682
|
.description("Watch for changes and sync with Tinybird")
|
|
680
683
|
.option("--local", "Use local Tinybird container")
|
|
681
684
|
.option("--branch", "Use Tinybird cloud with branches")
|
|
682
|
-
.option(
|
|
685
|
+
.option(
|
|
686
|
+
"--last-partition",
|
|
687
|
+
'[DEPRECATED] Use `branch_data_mode: "last_partition"` in tinybird.config.json. Copy the last partition of production data when creating a cloud branch.'
|
|
688
|
+
)
|
|
683
689
|
.action(async (options) => {
|
|
684
690
|
// Determine devMode override
|
|
685
691
|
let devModeOverride: DevMode | undefined;
|
package/src/client/base.test.ts
CHANGED
|
@@ -167,6 +167,7 @@ describe("TinybirdClient", () => {
|
|
|
167
167
|
tinybirdBranch: "feature_add_fetch",
|
|
168
168
|
isMainBranch: false,
|
|
169
169
|
devMode: "branch",
|
|
170
|
+
branchDataMode: null,
|
|
170
171
|
});
|
|
171
172
|
mockedGetOrCreateBranch.mockResolvedValue({
|
|
172
173
|
id: "branch-123",
|
|
@@ -192,7 +193,8 @@ describe("TinybirdClient", () => {
|
|
|
192
193
|
token: "workspace-token",
|
|
193
194
|
fetch: customFetch,
|
|
194
195
|
},
|
|
195
|
-
"feature_add_fetch"
|
|
196
|
+
"feature_add_fetch",
|
|
197
|
+
undefined
|
|
196
198
|
);
|
|
197
199
|
expect(context.token).toBe("branch-token");
|
|
198
200
|
expect(context.isBranchToken).toBe(true);
|
package/src/client/base.ts
CHANGED
|
@@ -19,6 +19,7 @@ import type {
|
|
|
19
19
|
} from "./types.js";
|
|
20
20
|
import { TinybirdError } from "./types.js";
|
|
21
21
|
import { TinybirdApi, TinybirdApiError } from "../api/api.js";
|
|
22
|
+
import type { CreateBranchOptions } from "../api/branches.js";
|
|
22
23
|
import { TokensNamespace } from "./tokens.js";
|
|
23
24
|
|
|
24
25
|
/**
|
|
@@ -340,6 +341,10 @@ export class TinybirdClient {
|
|
|
340
341
|
}
|
|
341
342
|
|
|
342
343
|
const branchName = config.tinybirdBranch;
|
|
344
|
+
const branchOptions: CreateBranchOptions | undefined =
|
|
345
|
+
config.devMode !== "local" && config.branchDataMode === "last_partition"
|
|
346
|
+
? { branch_data_mode: "last_partition" }
|
|
347
|
+
: undefined;
|
|
343
348
|
|
|
344
349
|
// Get or create branch (always fetch fresh to avoid stale cache issues)
|
|
345
350
|
const branch = await getOrCreateBranch(
|
|
@@ -348,7 +353,8 @@ export class TinybirdClient {
|
|
|
348
353
|
token: this.config.token,
|
|
349
354
|
fetch: this.config.fetch,
|
|
350
355
|
},
|
|
351
|
-
branchName
|
|
356
|
+
branchName,
|
|
357
|
+
branchOptions
|
|
352
358
|
);
|
|
353
359
|
|
|
354
360
|
if (!branch.token) {
|
|
@@ -4,6 +4,7 @@ import {
|
|
|
4
4
|
defineKafkaConnection,
|
|
5
5
|
defineS3Connection,
|
|
6
6
|
defineGCSConnection,
|
|
7
|
+
defineDynamoDBConnection,
|
|
7
8
|
} from "../schema/connection.js";
|
|
8
9
|
|
|
9
10
|
describe("Connection Generator", () => {
|
|
@@ -218,6 +219,22 @@ describe("Connection Generator", () => {
|
|
|
218
219
|
'GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON {{ tb_secret("GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON") }}'
|
|
219
220
|
);
|
|
220
221
|
});
|
|
222
|
+
|
|
223
|
+
it("generates DynamoDB connection with arn and region", () => {
|
|
224
|
+
const conn = defineDynamoDBConnection("my_dynamo", {
|
|
225
|
+
region: "us-east-1",
|
|
226
|
+
arn: '{{ tb_secret("DYNAMODB_ROLE_ARN") }}',
|
|
227
|
+
});
|
|
228
|
+
|
|
229
|
+
const result = generateConnection(conn);
|
|
230
|
+
|
|
231
|
+
expect(result.name).toBe("my_dynamo");
|
|
232
|
+
expect(result.content).toBe(
|
|
233
|
+
['TYPE dynamodb', 'DYNAMODB_ARN {{ tb_secret("DYNAMODB_ROLE_ARN") }}', "DYNAMODB_REGION us-east-1"].join(
|
|
234
|
+
"\n"
|
|
235
|
+
)
|
|
236
|
+
);
|
|
237
|
+
});
|
|
221
238
|
});
|
|
222
239
|
|
|
223
240
|
describe("generateAllConnections", () => {
|
|
@@ -7,10 +7,12 @@ import type {
|
|
|
7
7
|
ConnectionDefinition,
|
|
8
8
|
KafkaConnectionDefinition,
|
|
9
9
|
GCSConnectionDefinition,
|
|
10
|
+
DynamoDBConnectionDefinition,
|
|
10
11
|
} from "../schema/connection.js";
|
|
11
12
|
import {
|
|
12
13
|
isS3ConnectionDefinition,
|
|
13
14
|
isGCSConnectionDefinition,
|
|
15
|
+
isDynamoDBConnectionDefinition,
|
|
14
16
|
type S3ConnectionDefinition,
|
|
15
17
|
} from "../schema/connection.js";
|
|
16
18
|
|
|
@@ -123,6 +125,20 @@ function generateGCSConnection(connection: GCSConnectionDefinition): string {
|
|
|
123
125
|
return parts.join("\n");
|
|
124
126
|
}
|
|
125
127
|
|
|
128
|
+
/**
|
|
129
|
+
* Generate a DynamoDB connection content
|
|
130
|
+
*/
|
|
131
|
+
function generateDynamoDBConnection(connection: DynamoDBConnectionDefinition): string {
|
|
132
|
+
const parts: string[] = [];
|
|
133
|
+
const options = connection.options;
|
|
134
|
+
|
|
135
|
+
parts.push("TYPE dynamodb");
|
|
136
|
+
parts.push(`DYNAMODB_ARN ${options.arn}`);
|
|
137
|
+
parts.push(`DYNAMODB_REGION ${options.region}`);
|
|
138
|
+
|
|
139
|
+
return parts.join("\n");
|
|
140
|
+
}
|
|
141
|
+
|
|
126
142
|
/**
|
|
127
143
|
* Generate a .connection file content from a ConnectionDefinition
|
|
128
144
|
*
|
|
@@ -160,6 +176,8 @@ export function generateConnection(
|
|
|
160
176
|
content = generateS3Connection(connection);
|
|
161
177
|
} else if (isGCSConnectionDefinition(connection)) {
|
|
162
178
|
content = generateGCSConnection(connection);
|
|
179
|
+
} else if (isDynamoDBConnectionDefinition(connection)) {
|
|
180
|
+
content = generateDynamoDBConnection(connection);
|
|
163
181
|
} else {
|
|
164
182
|
throw new Error("Unsupported connection type.");
|
|
165
183
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { describe, it, expect } from 'vitest';
|
|
2
2
|
import { generateDatasource, generateAllDatasources } from './datasource.js';
|
|
3
3
|
import { defineDatasource } from '../schema/datasource.js';
|
|
4
|
-
import { defineKafkaConnection, defineS3Connection, defineGCSConnection } from '../schema/connection.js';
|
|
4
|
+
import { defineKafkaConnection, defineS3Connection, defineGCSConnection, defineDynamoDBConnection } from '../schema/connection.js';
|
|
5
5
|
import { defineToken } from '../schema/token.js';
|
|
6
6
|
import { t } from '../schema/types.js';
|
|
7
7
|
import { engine } from '../schema/engines.js';
|
|
@@ -629,6 +629,59 @@ describe('Datasource Generator', () => {
|
|
|
629
629
|
});
|
|
630
630
|
});
|
|
631
631
|
|
|
632
|
+
describe('DynamoDB configuration', () => {
|
|
633
|
+
it('includes DynamoDB import directives', () => {
|
|
634
|
+
const dynamoConn = defineDynamoDBConnection('my_dynamo', {
|
|
635
|
+
region: 'us-east-1',
|
|
636
|
+
arn: '{{ tb_secret("DYNAMODB_ROLE_ARN") }}',
|
|
637
|
+
});
|
|
638
|
+
|
|
639
|
+
const ds = defineDatasource('dynamo_events', {
|
|
640
|
+
schema: {
|
|
641
|
+
id: t.string(),
|
|
642
|
+
_record: t.string(),
|
|
643
|
+
},
|
|
644
|
+
engine: engine.replacingMergeTree({ sortingKey: ['id'] }),
|
|
645
|
+
dynamodb: {
|
|
646
|
+
connection: dynamoConn,
|
|
647
|
+
tableArn: 'arn:aws:dynamodb:us-east-1:123456789012:table/events',
|
|
648
|
+
exportBucket: 's3://my-export-bucket',
|
|
649
|
+
},
|
|
650
|
+
});
|
|
651
|
+
|
|
652
|
+
const result = generateDatasource(ds);
|
|
653
|
+
|
|
654
|
+
expect(result.content).toContain('IMPORT_CONNECTION_NAME my_dynamo');
|
|
655
|
+
expect(result.content).toContain(
|
|
656
|
+
'IMPORT_TABLE_ARN arn:aws:dynamodb:us-east-1:123456789012:table/events'
|
|
657
|
+
);
|
|
658
|
+
expect(result.content).toContain('IMPORT_EXPORT_BUCKET s3://my-export-bucket');
|
|
659
|
+
});
|
|
660
|
+
|
|
661
|
+
it('rejects mixing DynamoDB with other ingestion options', () => {
|
|
662
|
+
const dynamoConn = defineDynamoDBConnection('my_dynamo', {
|
|
663
|
+
region: 'us-east-1',
|
|
664
|
+
arn: '{{ tb_secret("DYNAMODB_ROLE_ARN") }}',
|
|
665
|
+
});
|
|
666
|
+
const s3Conn = defineS3Connection('my_s3', {
|
|
667
|
+
region: 'us-east-1',
|
|
668
|
+
arn: 'arn:aws:iam::123456789012:role/x',
|
|
669
|
+
});
|
|
670
|
+
|
|
671
|
+
expect(() =>
|
|
672
|
+
defineDatasource('mixed', {
|
|
673
|
+
schema: { id: t.string() },
|
|
674
|
+
dynamodb: {
|
|
675
|
+
connection: dynamoConn,
|
|
676
|
+
tableArn: 'arn:aws:dynamodb:us-east-1:123456789012:table/events',
|
|
677
|
+
exportBucket: 's3://my-export-bucket',
|
|
678
|
+
},
|
|
679
|
+
s3: { connection: s3Conn, bucketUri: 's3://b/*.csv' },
|
|
680
|
+
})
|
|
681
|
+
).toThrow('only define one ingestion option');
|
|
682
|
+
});
|
|
683
|
+
});
|
|
684
|
+
|
|
632
685
|
describe('Token generation', () => {
|
|
633
686
|
it('generates TOKEN lines with inline config', () => {
|
|
634
687
|
const ds = defineDatasource('test_ds', {
|
|
@@ -10,6 +10,7 @@ import type {
|
|
|
10
10
|
KafkaConfig,
|
|
11
11
|
S3Config,
|
|
12
12
|
GCSConfig,
|
|
13
|
+
DynamoDBConfig,
|
|
13
14
|
TokenConfig,
|
|
14
15
|
DatasourceIndex,
|
|
15
16
|
} from "../schema/datasource.js";
|
|
@@ -199,6 +200,19 @@ function generateImportConfig(importConfig: S3Config | GCSConfig): string {
|
|
|
199
200
|
return parts.join("\n");
|
|
200
201
|
}
|
|
201
202
|
|
|
203
|
+
/**
|
|
204
|
+
* Generate DynamoDB import configuration lines
|
|
205
|
+
*/
|
|
206
|
+
function generateDynamoDBConfig(dynamodb: DynamoDBConfig): string {
|
|
207
|
+
const parts: string[] = [];
|
|
208
|
+
|
|
209
|
+
parts.push(`IMPORT_CONNECTION_NAME ${dynamodb.connection._name}`);
|
|
210
|
+
parts.push(`IMPORT_TABLE_ARN ${dynamodb.tableArn}`);
|
|
211
|
+
parts.push(`IMPORT_EXPORT_BUCKET ${dynamodb.exportBucket}`);
|
|
212
|
+
|
|
213
|
+
return parts.join("\n");
|
|
214
|
+
}
|
|
215
|
+
|
|
202
216
|
/**
|
|
203
217
|
* Generate forward query section
|
|
204
218
|
*/
|
|
@@ -318,9 +332,12 @@ export function generateDatasource(
|
|
|
318
332
|
datasource.options.kafka,
|
|
319
333
|
datasource.options.s3,
|
|
320
334
|
datasource.options.gcs,
|
|
335
|
+
datasource.options.dynamodb,
|
|
321
336
|
].filter(Boolean).length;
|
|
322
337
|
if (ingestionConfigCount > 1) {
|
|
323
|
-
throw new Error(
|
|
338
|
+
throw new Error(
|
|
339
|
+
"Datasource can only define one ingestion option: `kafka`, `s3`, `gcs`, or `dynamodb`."
|
|
340
|
+
);
|
|
324
341
|
}
|
|
325
342
|
|
|
326
343
|
// Add description if present
|
|
@@ -369,6 +386,12 @@ export function generateDatasource(
|
|
|
369
386
|
parts.push(generateImportConfig(datasource.options.gcs));
|
|
370
387
|
}
|
|
371
388
|
|
|
389
|
+
// Add DynamoDB configuration if present
|
|
390
|
+
if (datasource.options.dynamodb) {
|
|
391
|
+
parts.push("");
|
|
392
|
+
parts.push(generateDynamoDBConfig(datasource.options.dynamodb));
|
|
393
|
+
}
|
|
394
|
+
|
|
372
395
|
// Add forward query if present
|
|
373
396
|
const forwardQuery = generateForwardQuery(datasource.options.forwardQuery);
|
|
374
397
|
if (forwardQuery) {
|