@tinybirdco/sdk 0.0.77 → 0.0.79

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. package/README.md +27 -2
  2. package/dist/api/branches.d.ts +4 -3
  3. package/dist/api/branches.d.ts.map +1 -1
  4. package/dist/api/branches.js +4 -4
  5. package/dist/api/branches.js.map +1 -1
  6. package/dist/api/branches.test.js +34 -1
  7. package/dist/api/branches.test.js.map +1 -1
  8. package/dist/cli/commands/build.d.ts.map +1 -1
  9. package/dist/cli/commands/build.js +7 -1
  10. package/dist/cli/commands/build.js.map +1 -1
  11. package/dist/cli/commands/build.test.js +159 -0
  12. package/dist/cli/commands/build.test.js.map +1 -1
  13. package/dist/cli/commands/clear.d.ts.map +1 -1
  14. package/dist/cli/commands/clear.js +4 -1
  15. package/dist/cli/commands/clear.js.map +1 -1
  16. package/dist/cli/commands/deploy.test.js +1 -0
  17. package/dist/cli/commands/deploy.test.js.map +1 -1
  18. package/dist/cli/commands/dev.d.ts.map +1 -1
  19. package/dist/cli/commands/dev.js +7 -1
  20. package/dist/cli/commands/dev.js.map +1 -1
  21. package/dist/cli/commands/generate.test.js +3 -0
  22. package/dist/cli/commands/generate.test.js.map +1 -1
  23. package/dist/cli/commands/migrate.d.ts.map +1 -1
  24. package/dist/cli/commands/migrate.js +14 -1
  25. package/dist/cli/commands/migrate.js.map +1 -1
  26. package/dist/cli/commands/migrate.test.js +60 -0
  27. package/dist/cli/commands/migrate.test.js.map +1 -1
  28. package/dist/cli/commands/preview.d.ts.map +1 -1
  29. package/dist/cli/commands/preview.js +4 -1
  30. package/dist/cli/commands/preview.js.map +1 -1
  31. package/dist/cli/commands/preview.test.js +116 -2
  32. package/dist/cli/commands/preview.test.js.map +1 -1
  33. package/dist/cli/config-types.d.ts +4 -0
  34. package/dist/cli/config-types.d.ts.map +1 -1
  35. package/dist/cli/config-types.js +1 -1
  36. package/dist/cli/config-types.js.map +1 -1
  37. package/dist/cli/config.d.ts +4 -2
  38. package/dist/cli/config.d.ts.map +1 -1
  39. package/dist/cli/config.js +27 -0
  40. package/dist/cli/config.js.map +1 -1
  41. package/dist/cli/config.test.js +79 -0
  42. package/dist/cli/config.test.js.map +1 -1
  43. package/dist/cli/index.js +2 -2
  44. package/dist/cli/index.js.map +1 -1
  45. package/dist/client/base.d.ts.map +1 -1
  46. package/dist/client/base.js +4 -1
  47. package/dist/client/base.js.map +1 -1
  48. package/dist/client/base.test.js +2 -1
  49. package/dist/client/base.test.js.map +1 -1
  50. package/dist/generator/connection.d.ts.map +1 -1
  51. package/dist/generator/connection.js +15 -1
  52. package/dist/generator/connection.js.map +1 -1
  53. package/dist/generator/connection.test.js +10 -1
  54. package/dist/generator/connection.test.js.map +1 -1
  55. package/dist/generator/datasource.d.ts.map +1 -1
  56. package/dist/generator/datasource.js +17 -1
  57. package/dist/generator/datasource.js.map +1 -1
  58. package/dist/generator/datasource.test.js +44 -1
  59. package/dist/generator/datasource.test.js.map +1 -1
  60. package/dist/index.d.ts +4 -4
  61. package/dist/index.d.ts.map +1 -1
  62. package/dist/index.js +1 -1
  63. package/dist/index.js.map +1 -1
  64. package/dist/migrate/emit-ts.d.ts.map +1 -1
  65. package/dist/migrate/emit-ts.js +28 -0
  66. package/dist/migrate/emit-ts.js.map +1 -1
  67. package/dist/migrate/parse-connection.d.ts +2 -2
  68. package/dist/migrate/parse-connection.d.ts.map +1 -1
  69. package/dist/migrate/parse-connection.js +60 -6
  70. package/dist/migrate/parse-connection.js.map +1 -1
  71. package/dist/migrate/parse-connection.test.js +18 -0
  72. package/dist/migrate/parse-connection.test.js.map +1 -1
  73. package/dist/migrate/parse-datasource.d.ts.map +1 -1
  74. package/dist/migrate/parse-datasource.js +27 -2
  75. package/dist/migrate/parse-datasource.js.map +1 -1
  76. package/dist/migrate/types.d.ts +15 -1
  77. package/dist/migrate/types.d.ts.map +1 -1
  78. package/dist/schema/connection.d.ts +46 -1
  79. package/dist/schema/connection.d.ts.map +1 -1
  80. package/dist/schema/connection.js +39 -0
  81. package/dist/schema/connection.js.map +1 -1
  82. package/dist/schema/connection.test.js +46 -1
  83. package/dist/schema/connection.test.js.map +1 -1
  84. package/dist/schema/datasource.d.ts +14 -1
  85. package/dist/schema/datasource.d.ts.map +1 -1
  86. package/dist/schema/datasource.js +2 -2
  87. package/dist/schema/datasource.js.map +1 -1
  88. package/dist/schema/datasource.test.js +2 -2
  89. package/dist/schema/datasource.test.js.map +1 -1
  90. package/package.json +1 -1
  91. package/src/api/branches.test.ts +38 -1
  92. package/src/api/branches.ts +8 -6
  93. package/src/cli/commands/build.test.ts +176 -0
  94. package/src/cli/commands/build.ts +9 -2
  95. package/src/cli/commands/clear.ts +8 -1
  96. package/src/cli/commands/deploy.test.ts +1 -0
  97. package/src/cli/commands/dev.ts +9 -1
  98. package/src/cli/commands/generate.test.ts +3 -0
  99. package/src/cli/commands/migrate.test.ts +95 -0
  100. package/src/cli/commands/migrate.ts +17 -1
  101. package/src/cli/commands/preview.test.ts +133 -2
  102. package/src/cli/commands/preview.ts +6 -2
  103. package/src/cli/config-types.ts +4 -0
  104. package/src/cli/config.test.ts +123 -0
  105. package/src/cli/config.ts +41 -3
  106. package/src/cli/index.ts +8 -2
  107. package/src/client/base.test.ts +3 -1
  108. package/src/client/base.ts +7 -1
  109. package/src/generator/connection.test.ts +17 -0
  110. package/src/generator/connection.ts +18 -0
  111. package/src/generator/datasource.test.ts +54 -1
  112. package/src/generator/datasource.ts +24 -1
  113. package/src/index.ts +6 -1
  114. package/src/migrate/emit-ts.ts +44 -3
  115. package/src/migrate/parse-connection.test.ts +41 -0
  116. package/src/migrate/parse-connection.ts +86 -7
  117. package/src/migrate/parse-datasource.ts +35 -2
  118. package/src/migrate/types.ts +18 -1
  119. package/src/schema/connection.test.ts +65 -0
  120. package/src/schema/connection.ts +76 -1
  121. package/src/schema/datasource.test.ts +2 -2
  122. package/src/schema/datasource.ts +21 -2
@@ -1,7 +1,45 @@
1
- import { describe, it, expect } from "vitest";
2
- import { generatePreviewBranchName } from "./preview.js";
1
+ import { describe, it, expect, vi, beforeEach } from "vitest";
2
+ import { generatePreviewBranchName, runPreview } from "./preview.js";
3
+
4
+ vi.mock("../config.js", () => ({
5
+ loadConfigAsync: vi.fn(),
6
+ LOCAL_BASE_URL: "http://localhost:7181",
7
+ }));
8
+
9
+ vi.mock("../../generator/index.js", () => ({
10
+ buildFromInclude: vi.fn(),
11
+ }));
12
+
13
+ vi.mock("../../api/branches.js", () => ({
14
+ createBranch: vi.fn(),
15
+ deleteBranch: vi.fn(),
16
+ getBranch: vi.fn(),
17
+ }));
18
+
19
+ vi.mock("../../api/deploy.js", () => ({
20
+ deployToMain: vi.fn(),
21
+ }));
22
+
23
+ vi.mock("../../api/build.js", () => ({
24
+ buildToTinybird: vi.fn(),
25
+ }));
26
+
27
+ vi.mock("../../api/local.js", () => ({
28
+ getLocalTokens: vi.fn(),
29
+ getOrCreateLocalWorkspace: vi.fn(),
30
+ LocalNotRunningError: class LocalNotRunningError extends Error {},
31
+ }));
32
+
33
+ vi.mock("../git.js", () => ({
34
+ sanitizeBranchName: (value: string) => value.replace(/[^a-zA-Z0-9]/g, "_"),
35
+ getCurrentGitBranch: vi.fn(() => "feature/test"),
36
+ }));
3
37
 
4
38
  describe("Preview command", () => {
39
+ beforeEach(() => {
40
+ vi.clearAllMocks();
41
+ });
42
+
5
43
  describe("generatePreviewBranchName", () => {
6
44
  it("generates name with tmp_ci prefix", () => {
7
45
  const result = generatePreviewBranchName("feature-branch");
@@ -34,4 +72,97 @@ describe("Preview command", () => {
34
72
  expect(result1).toBe(result2);
35
73
  });
36
74
  });
75
+
76
+ describe("branch_data_mode wiring", () => {
77
+ it("uses config-only last_partition when creating cloud preview branch", async () => {
78
+ const { loadConfigAsync } = await import("../config.js");
79
+ const { buildFromInclude } = await import("../../generator/index.js");
80
+ const { getBranch, createBranch } = await import("../../api/branches.js");
81
+ const { deployToMain } = await import("../../api/deploy.js");
82
+
83
+ vi.mocked(loadConfigAsync).mockResolvedValue({
84
+ include: ["test.ts"],
85
+ token: "p.test-token",
86
+ baseUrl: "https://api.tinybird.co",
87
+ configPath: "/test/tinybird.config.json",
88
+ devMode: "branch",
89
+ cwd: "/test",
90
+ gitBranch: "feature-test",
91
+ tinybirdBranch: "feature_test",
92
+ isMainBranch: false,
93
+ branchDataMode: "last_partition",
94
+ });
95
+ vi.mocked(buildFromInclude).mockResolvedValue({
96
+ resources: { datasources: [], pipes: [], connections: [] },
97
+ entities: { datasources: {}, pipes: {}, connections: {}, rawDatasources: [], rawPipes: [], sourceFiles: [] },
98
+ stats: { datasourceCount: 0, pipeCount: 0, connectionCount: 0 },
99
+ });
100
+ vi.mocked(getBranch).mockRejectedValue(new Error("not found"));
101
+ vi.mocked(createBranch).mockResolvedValue({
102
+ id: "b1",
103
+ name: "tmp_ci_feature_test",
104
+ token: "p.branch",
105
+ created_at: "2024-01-01T00:00:00Z",
106
+ });
107
+ vi.mocked(deployToMain).mockResolvedValue({
108
+ success: true,
109
+ result: "success",
110
+ datasourceCount: 0,
111
+ pipeCount: 0,
112
+ connectionCount: 0,
113
+ });
114
+
115
+ await runPreview();
116
+ expect(createBranch).toHaveBeenCalledWith(
117
+ expect.any(Object),
118
+ "tmp_ci_feature_test",
119
+ { branch_data_mode: "last_partition" }
120
+ );
121
+ });
122
+
123
+ it("ignores config branch_data_mode in local mode", async () => {
124
+ const { loadConfigAsync } = await import("../config.js");
125
+ const { buildFromInclude } = await import("../../generator/index.js");
126
+ const { createBranch } = await import("../../api/branches.js");
127
+ const { getLocalTokens, getOrCreateLocalWorkspace } = await import("../../api/local.js");
128
+ const { buildToTinybird } = await import("../../api/build.js");
129
+
130
+ vi.mocked(loadConfigAsync).mockResolvedValue({
131
+ include: ["test.ts"],
132
+ token: "p.test-token",
133
+ baseUrl: "https://api.tinybird.co",
134
+ configPath: "/test/tinybird.config.json",
135
+ devMode: "local",
136
+ cwd: "/test",
137
+ gitBranch: "feature-test",
138
+ tinybirdBranch: "feature_test",
139
+ isMainBranch: false,
140
+ branchDataMode: "last_partition",
141
+ });
142
+ vi.mocked(buildFromInclude).mockResolvedValue({
143
+ resources: { datasources: [], pipes: [], connections: [] },
144
+ entities: { datasources: {}, pipes: {}, connections: {}, rawDatasources: [], rawPipes: [], sourceFiles: [] },
145
+ stats: { datasourceCount: 0, pipeCount: 0, connectionCount: 0 },
146
+ });
147
+ vi.mocked(getLocalTokens).mockResolvedValue({
148
+ admin_token: "admin-token",
149
+ user_token: "user-token",
150
+ workspace_admin_token: "workspace-admin-token",
151
+ });
152
+ vi.mocked(getOrCreateLocalWorkspace).mockResolvedValue({
153
+ workspace: { id: "lw1", name: "tmp_ci_feature_test", token: "local-token" },
154
+ wasCreated: true,
155
+ });
156
+ vi.mocked(buildToTinybird).mockResolvedValue({
157
+ success: true,
158
+ result: "success",
159
+ datasourceCount: 0,
160
+ pipeCount: 0,
161
+ connectionCount: 0,
162
+ });
163
+
164
+ await runPreview();
165
+ expect(createBranch).not.toHaveBeenCalled();
166
+ });
167
+ });
37
168
  });
@@ -4,7 +4,7 @@
4
4
 
5
5
  import { loadConfigAsync, LOCAL_BASE_URL, type ResolvedConfig, type DevMode } from "../config.js";
6
6
  import { buildFromInclude, type BuildFromIncludeResult } from "../../generator/index.js";
7
- import { createBranch, deleteBranch, getBranch, type TinybirdBranch } from "../../api/branches.js";
7
+ import { createBranch, deleteBranch, getBranch, type CreateBranchOptions, type TinybirdBranch } from "../../api/branches.js";
8
8
  import { deployToMain } from "../../api/deploy.js";
9
9
  import { buildToTinybird } from "../../api/build.js";
10
10
  import {
@@ -226,6 +226,10 @@ export async function runPreview(options: PreviewCommandOptions = {}): Promise<P
226
226
  let branch: TinybirdBranch;
227
227
  try {
228
228
  const apiConfig = { baseUrl: config.baseUrl, token: config.token };
229
+ const branchOptions: CreateBranchOptions | undefined =
230
+ config.branchDataMode === "last_partition"
231
+ ? { branch_data_mode: "last_partition" }
232
+ : undefined;
229
233
 
230
234
  // Check if branch already exists and delete it for a fresh start
231
235
  try {
@@ -250,7 +254,7 @@ export async function runPreview(options: PreviewCommandOptions = {}): Promise<P
250
254
  console.log(`[debug] Creating preview branch: ${previewBranchName}`);
251
255
  }
252
256
 
253
- branch = await createBranch(apiConfig, previewBranchName);
257
+ branch = await createBranch(apiConfig, previewBranchName, branchOptions);
254
258
 
255
259
  if (debug) {
256
260
  console.log(`[debug] Branch created: ${branch.name} (${branch.id})`);
@@ -11,6 +11,8 @@
11
11
  * - "local": Use local Tinybird container at localhost:7181
12
12
  */
13
13
  export type DevMode = "branch" | "local";
14
+ export type BranchDataMode = "last_partition";
15
+ export const BRANCH_DATA_MODE_VALUES = ["last_partition"] as const satisfies readonly BranchDataMode[];
14
16
 
15
17
  /**
16
18
  * Tinybird configuration file structure
@@ -26,4 +28,6 @@ export interface TinybirdConfig {
26
28
  baseUrl?: string;
27
29
  /** Development mode: "branch" (default) or "local" */
28
30
  devMode?: DevMode;
31
+ /** Branch data mode applied on cloud branch creation (shared snake_case key) */
32
+ branch_data_mode?: BranchDataMode;
29
33
  }
@@ -327,6 +327,129 @@ describe("Config", () => {
327
327
 
328
328
  expect(result.devMode).toBe("local");
329
329
  });
330
+
331
+ it("resolves branch_data_mode as last_partition", () => {
332
+ const config = {
333
+ include: ["lib/datasources.ts"],
334
+ token: "test-token",
335
+ branch_data_mode: "last_partition",
336
+ };
337
+ fs.writeFileSync(
338
+ path.join(tempDir, "tinybird.json"),
339
+ JSON.stringify(config)
340
+ );
341
+
342
+ const result = loadConfig(tempDir);
343
+ expect(result.branchDataMode).toBe("last_partition");
344
+ });
345
+
346
+ it("defaults branch_data_mode to last_partition when missing", () => {
347
+ const config = {
348
+ include: ["lib/datasources.ts"],
349
+ token: "test-token",
350
+ };
351
+ fs.writeFileSync(
352
+ path.join(tempDir, "tinybird.json"),
353
+ JSON.stringify(config)
354
+ );
355
+
356
+ const result = loadConfig(tempDir);
357
+ expect(result.branchDataMode).toBe("last_partition");
358
+ });
359
+
360
+ it("defaults empty branch_data_mode to last_partition", () => {
361
+ const config = {
362
+ include: ["lib/datasources.ts"],
363
+ token: "test-token",
364
+ branch_data_mode: " ",
365
+ };
366
+ fs.writeFileSync(
367
+ path.join(tempDir, "tinybird.json"),
368
+ JSON.stringify(config)
369
+ );
370
+
371
+ const result = loadConfig(tempDir);
372
+ expect(result.branchDataMode).toBe("last_partition");
373
+ });
374
+
375
+ it("throws when branch_data_mode is all_partitions", () => {
376
+ const config = {
377
+ include: ["lib/datasources.ts"],
378
+ token: "test-token",
379
+ branch_data_mode: "all_partitions",
380
+ };
381
+ fs.writeFileSync(
382
+ path.join(tempDir, "tinybird.json"),
383
+ JSON.stringify(config)
384
+ );
385
+
386
+ expect(() => loadConfig(tempDir)).toThrow("Invalid branch_data_mode");
387
+ });
388
+
389
+ it("throws when branch_data_mode is invalid", () => {
390
+ const config = {
391
+ include: ["lib/datasources.ts"],
392
+ token: "test-token",
393
+ branch_data_mode: "invalid",
394
+ };
395
+ fs.writeFileSync(
396
+ path.join(tempDir, "tinybird.json"),
397
+ JSON.stringify(config)
398
+ );
399
+
400
+ expect(() => loadConfig(tempDir)).toThrow("Invalid branch_data_mode");
401
+ });
402
+
403
+ it("warns when branch_data_mode is set with devMode local", () => {
404
+ const config = {
405
+ include: ["lib/datasources.ts"],
406
+ token: "test-token",
407
+ devMode: "local",
408
+ branch_data_mode: "last_partition",
409
+ };
410
+ fs.writeFileSync(
411
+ path.join(tempDir, "tinybird.json"),
412
+ JSON.stringify(config)
413
+ );
414
+ const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
415
+
416
+ loadConfig(tempDir);
417
+
418
+ expect(warnSpy).toHaveBeenCalledWith(
419
+ expect.stringContaining("branch_data_mode is set")
420
+ );
421
+ });
422
+
423
+ it("does not warn when branch_data_mode is implicit in local mode", () => {
424
+ const config = {
425
+ include: ["lib/datasources.ts"],
426
+ token: "test-token",
427
+ devMode: "local",
428
+ };
429
+ fs.writeFileSync(
430
+ path.join(tempDir, "tinybird.json"),
431
+ JSON.stringify(config)
432
+ );
433
+ const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
434
+
435
+ loadConfig(tempDir);
436
+
437
+ expect(warnSpy).not.toHaveBeenCalled();
438
+ });
439
+
440
+ it("throws when legacy branch_data_on_create key is used", () => {
441
+ const config = {
442
+ include: ["lib/datasources.ts"],
443
+ token: "test-token",
444
+ branch_data_on_create: "last_partition",
445
+ };
446
+ fs.writeFileSync(
447
+ path.join(tempDir, "tinybird.json"),
448
+ JSON.stringify(config)
449
+ );
450
+
451
+ expect(() => loadConfig(tempDir)).toThrow("renamed to `branch_data_mode`");
452
+ });
330
453
  });
331
454
 
332
455
  describe("loadConfigAsync", () => {
package/src/cli/config.ts CHANGED
@@ -7,9 +7,17 @@ import * as path from "path";
7
7
  import { config as loadDotenv } from "dotenv";
8
8
  import { getCurrentGitBranch, isMainBranch, getTinybirdBranchName } from "./git.js";
9
9
 
10
- // Re-export types from config-types.ts (separate file to avoid bundling esbuild)
11
- export type { DevMode, TinybirdConfig } from "./config-types.js";
12
- import type { DevMode, TinybirdConfig } from "./config-types.js";
10
+ // Re-export config types/constants from config-types.ts (separate file to avoid bundling esbuild)
11
+ export {
12
+ BRANCH_DATA_MODE_VALUES,
13
+ type BranchDataMode,
14
+ type DevMode,
15
+ type TinybirdConfig,
16
+ } from "./config-types.js";
17
+ import { BRANCH_DATA_MODE_VALUES } from "./config-types.js";
18
+ import type { BranchDataMode, DevMode, TinybirdConfig } from "./config-types.js";
19
+
20
+ const DEFAULT_BRANCH_DATA_MODE: BranchDataMode = "last_partition";
13
21
 
14
22
  /**
15
23
  * Resolved configuration with all values expanded
@@ -33,6 +41,8 @@ export interface ResolvedConfig {
33
41
  isMainBranch: boolean;
34
42
  /** Development mode: "branch" or "local" */
35
43
  devMode: DevMode;
44
+ /** Branch data mode configured in tinybird.config.json */
45
+ branchDataMode?: BranchDataMode | null;
36
46
  }
37
47
 
38
48
  /**
@@ -196,6 +206,24 @@ export function findConfigFile(startDir: string): ConfigFileResult | null {
196
206
  // Import the universal config loader
197
207
  import { loadConfigFile } from "./config-loader.js";
198
208
 
209
+ function resolveBranchDataMode(raw: Record<string, unknown>): { mode: BranchDataMode | null; explicit: boolean } {
210
+ if (raw["branch_data_on_create"] !== undefined) {
211
+ throw new Error("`branch_data_on_create` has been renamed to `branch_data_mode`.");
212
+ }
213
+
214
+ const value = raw["branch_data_mode"];
215
+ if (value === undefined || value === null) return { mode: DEFAULT_BRANCH_DATA_MODE, explicit: false };
216
+ if (typeof value !== "string") throw new Error("branch_data_mode must be a string.");
217
+ const mode = value.trim().toLowerCase();
218
+ if (!mode) return { mode: DEFAULT_BRANCH_DATA_MODE, explicit: false };
219
+ if (!BRANCH_DATA_MODE_VALUES.includes(mode as BranchDataMode)) {
220
+ throw new Error(
221
+ `Invalid branch_data_mode '${value}'. Allowed values are: ${BRANCH_DATA_MODE_VALUES.join(", ")}.`
222
+ );
223
+ }
224
+ return { mode: mode as BranchDataMode, explicit: true };
225
+ }
226
+
199
227
  /**
200
228
  * Resolve a TinybirdConfig to a ResolvedConfig
201
229
  */
@@ -255,6 +283,15 @@ function resolveConfig(config: TinybirdConfig, configPath: string): ResolvedConf
255
283
 
256
284
  // Resolve devMode (default to "branch")
257
285
  const devMode: DevMode = config.devMode ?? "branch";
286
+ const { mode: branchDataMode, explicit: branchDataModeExplicit } = resolveBranchDataMode(
287
+ config as unknown as Record<string, unknown>
288
+ );
289
+ if (branchDataModeExplicit && branchDataMode && devMode === "local") {
290
+ console.warn(
291
+ "branch_data_mode is set in tinybird.config.json but dev_mode='local'. " +
292
+ "Branch data settings only apply to cloud branches."
293
+ );
294
+ }
258
295
 
259
296
  return {
260
297
  include,
@@ -266,6 +303,7 @@ function resolveConfig(config: TinybirdConfig, configPath: string): ResolvedConf
266
303
  tinybirdBranch,
267
304
  isMainBranch: isMainBranch(),
268
305
  devMode,
306
+ branchDataMode,
269
307
  };
270
308
  }
271
309
 
package/src/cli/index.ts CHANGED
@@ -270,7 +270,10 @@ function createCli(): Command {
270
270
  .option("--debug", "Show debug output including API requests/responses")
271
271
  .option("--local", "Use local Tinybird container")
272
272
  .option("--branch", "Use Tinybird cloud with branches")
273
- .option("--last-partition", "Copy the last partition of production data when creating a branch")
273
+ .option(
274
+ "--last-partition",
275
+ '[DEPRECATED] Use `branch_data_mode: "last_partition"` in tinybird.config.json. Copy the last partition of production data when creating a cloud branch.'
276
+ )
274
277
  .action(async (options) => {
275
278
  if (options.debug) {
276
279
  process.env.TINYBIRD_DEBUG = "1";
@@ -679,7 +682,10 @@ function createCli(): Command {
679
682
  .description("Watch for changes and sync with Tinybird")
680
683
  .option("--local", "Use local Tinybird container")
681
684
  .option("--branch", "Use Tinybird cloud with branches")
682
- .option("--last-partition", "Copy the last partition of production data when creating a branch")
685
+ .option(
686
+ "--last-partition",
687
+ '[DEPRECATED] Use `branch_data_mode: "last_partition"` in tinybird.config.json. Copy the last partition of production data when creating a cloud branch.'
688
+ )
683
689
  .action(async (options) => {
684
690
  // Determine devMode override
685
691
  let devModeOverride: DevMode | undefined;
@@ -167,6 +167,7 @@ describe("TinybirdClient", () => {
167
167
  tinybirdBranch: "feature_add_fetch",
168
168
  isMainBranch: false,
169
169
  devMode: "branch",
170
+ branchDataMode: null,
170
171
  });
171
172
  mockedGetOrCreateBranch.mockResolvedValue({
172
173
  id: "branch-123",
@@ -192,7 +193,8 @@ describe("TinybirdClient", () => {
192
193
  token: "workspace-token",
193
194
  fetch: customFetch,
194
195
  },
195
- "feature_add_fetch"
196
+ "feature_add_fetch",
197
+ undefined
196
198
  );
197
199
  expect(context.token).toBe("branch-token");
198
200
  expect(context.isBranchToken).toBe(true);
@@ -19,6 +19,7 @@ import type {
19
19
  } from "./types.js";
20
20
  import { TinybirdError } from "./types.js";
21
21
  import { TinybirdApi, TinybirdApiError } from "../api/api.js";
22
+ import type { CreateBranchOptions } from "../api/branches.js";
22
23
  import { TokensNamespace } from "./tokens.js";
23
24
 
24
25
  /**
@@ -340,6 +341,10 @@ export class TinybirdClient {
340
341
  }
341
342
 
342
343
  const branchName = config.tinybirdBranch;
344
+ const branchOptions: CreateBranchOptions | undefined =
345
+ config.devMode !== "local" && config.branchDataMode === "last_partition"
346
+ ? { branch_data_mode: "last_partition" }
347
+ : undefined;
343
348
 
344
349
  // Get or create branch (always fetch fresh to avoid stale cache issues)
345
350
  const branch = await getOrCreateBranch(
@@ -348,7 +353,8 @@ export class TinybirdClient {
348
353
  token: this.config.token,
349
354
  fetch: this.config.fetch,
350
355
  },
351
- branchName
356
+ branchName,
357
+ branchOptions
352
358
  );
353
359
 
354
360
  if (!branch.token) {
@@ -4,6 +4,7 @@ import {
4
4
  defineKafkaConnection,
5
5
  defineS3Connection,
6
6
  defineGCSConnection,
7
+ defineDynamoDBConnection,
7
8
  } from "../schema/connection.js";
8
9
 
9
10
  describe("Connection Generator", () => {
@@ -218,6 +219,22 @@ describe("Connection Generator", () => {
218
219
  'GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON {{ tb_secret("GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON") }}'
219
220
  );
220
221
  });
222
+
223
+ it("generates DynamoDB connection with arn and region", () => {
224
+ const conn = defineDynamoDBConnection("my_dynamo", {
225
+ region: "us-east-1",
226
+ arn: '{{ tb_secret("DYNAMODB_ROLE_ARN") }}',
227
+ });
228
+
229
+ const result = generateConnection(conn);
230
+
231
+ expect(result.name).toBe("my_dynamo");
232
+ expect(result.content).toBe(
233
+ ['TYPE dynamodb', 'DYNAMODB_ARN {{ tb_secret("DYNAMODB_ROLE_ARN") }}', "DYNAMODB_REGION us-east-1"].join(
234
+ "\n"
235
+ )
236
+ );
237
+ });
221
238
  });
222
239
 
223
240
  describe("generateAllConnections", () => {
@@ -7,10 +7,12 @@ import type {
7
7
  ConnectionDefinition,
8
8
  KafkaConnectionDefinition,
9
9
  GCSConnectionDefinition,
10
+ DynamoDBConnectionDefinition,
10
11
  } from "../schema/connection.js";
11
12
  import {
12
13
  isS3ConnectionDefinition,
13
14
  isGCSConnectionDefinition,
15
+ isDynamoDBConnectionDefinition,
14
16
  type S3ConnectionDefinition,
15
17
  } from "../schema/connection.js";
16
18
 
@@ -123,6 +125,20 @@ function generateGCSConnection(connection: GCSConnectionDefinition): string {
123
125
  return parts.join("\n");
124
126
  }
125
127
 
128
+ /**
129
+ * Generate a DynamoDB connection content
130
+ */
131
+ function generateDynamoDBConnection(connection: DynamoDBConnectionDefinition): string {
132
+ const parts: string[] = [];
133
+ const options = connection.options;
134
+
135
+ parts.push("TYPE dynamodb");
136
+ parts.push(`DYNAMODB_ARN ${options.arn}`);
137
+ parts.push(`DYNAMODB_REGION ${options.region}`);
138
+
139
+ return parts.join("\n");
140
+ }
141
+
126
142
  /**
127
143
  * Generate a .connection file content from a ConnectionDefinition
128
144
  *
@@ -160,6 +176,8 @@ export function generateConnection(
160
176
  content = generateS3Connection(connection);
161
177
  } else if (isGCSConnectionDefinition(connection)) {
162
178
  content = generateGCSConnection(connection);
179
+ } else if (isDynamoDBConnectionDefinition(connection)) {
180
+ content = generateDynamoDBConnection(connection);
163
181
  } else {
164
182
  throw new Error("Unsupported connection type.");
165
183
  }
@@ -1,7 +1,7 @@
1
1
  import { describe, it, expect } from 'vitest';
2
2
  import { generateDatasource, generateAllDatasources } from './datasource.js';
3
3
  import { defineDatasource } from '../schema/datasource.js';
4
- import { defineKafkaConnection, defineS3Connection, defineGCSConnection } from '../schema/connection.js';
4
+ import { defineKafkaConnection, defineS3Connection, defineGCSConnection, defineDynamoDBConnection } from '../schema/connection.js';
5
5
  import { defineToken } from '../schema/token.js';
6
6
  import { t } from '../schema/types.js';
7
7
  import { engine } from '../schema/engines.js';
@@ -629,6 +629,59 @@ describe('Datasource Generator', () => {
629
629
  });
630
630
  });
631
631
 
632
+ describe('DynamoDB configuration', () => {
633
+ it('includes DynamoDB import directives', () => {
634
+ const dynamoConn = defineDynamoDBConnection('my_dynamo', {
635
+ region: 'us-east-1',
636
+ arn: '{{ tb_secret("DYNAMODB_ROLE_ARN") }}',
637
+ });
638
+
639
+ const ds = defineDatasource('dynamo_events', {
640
+ schema: {
641
+ id: t.string(),
642
+ _record: t.string(),
643
+ },
644
+ engine: engine.replacingMergeTree({ sortingKey: ['id'] }),
645
+ dynamodb: {
646
+ connection: dynamoConn,
647
+ tableArn: 'arn:aws:dynamodb:us-east-1:123456789012:table/events',
648
+ exportBucket: 's3://my-export-bucket',
649
+ },
650
+ });
651
+
652
+ const result = generateDatasource(ds);
653
+
654
+ expect(result.content).toContain('IMPORT_CONNECTION_NAME my_dynamo');
655
+ expect(result.content).toContain(
656
+ 'IMPORT_TABLE_ARN arn:aws:dynamodb:us-east-1:123456789012:table/events'
657
+ );
658
+ expect(result.content).toContain('IMPORT_EXPORT_BUCKET s3://my-export-bucket');
659
+ });
660
+
661
+ it('rejects mixing DynamoDB with other ingestion options', () => {
662
+ const dynamoConn = defineDynamoDBConnection('my_dynamo', {
663
+ region: 'us-east-1',
664
+ arn: '{{ tb_secret("DYNAMODB_ROLE_ARN") }}',
665
+ });
666
+ const s3Conn = defineS3Connection('my_s3', {
667
+ region: 'us-east-1',
668
+ arn: 'arn:aws:iam::123456789012:role/x',
669
+ });
670
+
671
+ expect(() =>
672
+ defineDatasource('mixed', {
673
+ schema: { id: t.string() },
674
+ dynamodb: {
675
+ connection: dynamoConn,
676
+ tableArn: 'arn:aws:dynamodb:us-east-1:123456789012:table/events',
677
+ exportBucket: 's3://my-export-bucket',
678
+ },
679
+ s3: { connection: s3Conn, bucketUri: 's3://b/*.csv' },
680
+ })
681
+ ).toThrow('only define one ingestion option');
682
+ });
683
+ });
684
+
632
685
  describe('Token generation', () => {
633
686
  it('generates TOKEN lines with inline config', () => {
634
687
  const ds = defineDatasource('test_ds', {
@@ -10,6 +10,7 @@ import type {
10
10
  KafkaConfig,
11
11
  S3Config,
12
12
  GCSConfig,
13
+ DynamoDBConfig,
13
14
  TokenConfig,
14
15
  DatasourceIndex,
15
16
  } from "../schema/datasource.js";
@@ -199,6 +200,19 @@ function generateImportConfig(importConfig: S3Config | GCSConfig): string {
199
200
  return parts.join("\n");
200
201
  }
201
202
 
203
+ /**
204
+ * Generate DynamoDB import configuration lines
205
+ */
206
+ function generateDynamoDBConfig(dynamodb: DynamoDBConfig): string {
207
+ const parts: string[] = [];
208
+
209
+ parts.push(`IMPORT_CONNECTION_NAME ${dynamodb.connection._name}`);
210
+ parts.push(`IMPORT_TABLE_ARN ${dynamodb.tableArn}`);
211
+ parts.push(`IMPORT_EXPORT_BUCKET ${dynamodb.exportBucket}`);
212
+
213
+ return parts.join("\n");
214
+ }
215
+
202
216
  /**
203
217
  * Generate forward query section
204
218
  */
@@ -318,9 +332,12 @@ export function generateDatasource(
318
332
  datasource.options.kafka,
319
333
  datasource.options.s3,
320
334
  datasource.options.gcs,
335
+ datasource.options.dynamodb,
321
336
  ].filter(Boolean).length;
322
337
  if (ingestionConfigCount > 1) {
323
- throw new Error("Datasource can only define one ingestion option: `kafka`, `s3`, or `gcs`.");
338
+ throw new Error(
339
+ "Datasource can only define one ingestion option: `kafka`, `s3`, `gcs`, or `dynamodb`."
340
+ );
324
341
  }
325
342
 
326
343
  // Add description if present
@@ -369,6 +386,12 @@ export function generateDatasource(
369
386
  parts.push(generateImportConfig(datasource.options.gcs));
370
387
  }
371
388
 
389
+ // Add DynamoDB configuration if present
390
+ if (datasource.options.dynamodb) {
391
+ parts.push("");
392
+ parts.push(generateDynamoDBConfig(datasource.options.dynamodb));
393
+ }
394
+
372
395
  // Add forward query if present
373
396
  const forwardQuery = generateForwardQuery(datasource.options.forwardQuery);
374
397
  if (forwardQuery) {