@tinybirdco/sdk 0.0.49 → 0.0.51

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/README.md +19 -2
  2. package/dist/cli/commands/migrate.d.ts.map +1 -1
  3. package/dist/cli/commands/migrate.js +36 -1
  4. package/dist/cli/commands/migrate.js.map +1 -1
  5. package/dist/cli/commands/migrate.test.js +307 -2
  6. package/dist/cli/commands/migrate.test.js.map +1 -1
  7. package/dist/codegen/type-mapper.d.ts.map +1 -1
  8. package/dist/codegen/type-mapper.js +70 -7
  9. package/dist/codegen/type-mapper.js.map +1 -1
  10. package/dist/codegen/type-mapper.test.js +9 -0
  11. package/dist/codegen/type-mapper.test.js.map +1 -1
  12. package/dist/generator/connection.d.ts.map +1 -1
  13. package/dist/generator/connection.js +14 -1
  14. package/dist/generator/connection.js.map +1 -1
  15. package/dist/generator/connection.test.js +20 -4
  16. package/dist/generator/connection.test.js.map +1 -1
  17. package/dist/generator/datasource.d.ts.map +1 -1
  18. package/dist/generator/datasource.js +39 -10
  19. package/dist/generator/datasource.js.map +1 -1
  20. package/dist/generator/datasource.test.js +42 -1
  21. package/dist/generator/datasource.test.js.map +1 -1
  22. package/dist/generator/pipe.d.ts.map +1 -1
  23. package/dist/generator/pipe.js +92 -3
  24. package/dist/generator/pipe.js.map +1 -1
  25. package/dist/generator/pipe.test.js +19 -0
  26. package/dist/generator/pipe.test.js.map +1 -1
  27. package/dist/index.d.ts +3 -3
  28. package/dist/index.d.ts.map +1 -1
  29. package/dist/index.js +1 -1
  30. package/dist/index.js.map +1 -1
  31. package/dist/migrate/emit-ts.d.ts.map +1 -1
  32. package/dist/migrate/emit-ts.js +56 -11
  33. package/dist/migrate/emit-ts.js.map +1 -1
  34. package/dist/migrate/parse-connection.d.ts +2 -2
  35. package/dist/migrate/parse-connection.d.ts.map +1 -1
  36. package/dist/migrate/parse-connection.js +34 -4
  37. package/dist/migrate/parse-connection.js.map +1 -1
  38. package/dist/migrate/parse-datasource.d.ts.map +1 -1
  39. package/dist/migrate/parse-datasource.js +39 -2
  40. package/dist/migrate/parse-datasource.js.map +1 -1
  41. package/dist/migrate/parse-pipe.d.ts.map +1 -1
  42. package/dist/migrate/parse-pipe.js +212 -93
  43. package/dist/migrate/parse-pipe.js.map +1 -1
  44. package/dist/migrate/parser-utils.d.ts.map +1 -1
  45. package/dist/migrate/parser-utils.js +3 -1
  46. package/dist/migrate/parser-utils.js.map +1 -1
  47. package/dist/migrate/types.d.ts +22 -1
  48. package/dist/migrate/types.d.ts.map +1 -1
  49. package/dist/schema/connection.d.ts +34 -1
  50. package/dist/schema/connection.d.ts.map +1 -1
  51. package/dist/schema/connection.js +26 -0
  52. package/dist/schema/connection.js.map +1 -1
  53. package/dist/schema/connection.test.js +35 -1
  54. package/dist/schema/connection.test.js.map +1 -1
  55. package/dist/schema/datasource.d.ts +32 -1
  56. package/dist/schema/datasource.d.ts.map +1 -1
  57. package/dist/schema/datasource.js +19 -2
  58. package/dist/schema/datasource.js.map +1 -1
  59. package/dist/schema/datasource.test.js +71 -3
  60. package/dist/schema/datasource.test.js.map +1 -1
  61. package/package.json +1 -1
  62. package/src/cli/commands/migrate.test.ts +448 -2
  63. package/src/cli/commands/migrate.ts +39 -1
  64. package/src/codegen/type-mapper.test.ts +18 -0
  65. package/src/codegen/type-mapper.ts +79 -7
  66. package/src/generator/connection.test.ts +29 -4
  67. package/src/generator/connection.ts +25 -2
  68. package/src/generator/datasource.test.ts +52 -1
  69. package/src/generator/datasource.ts +47 -10
  70. package/src/generator/pipe.test.ts +21 -0
  71. package/src/generator/pipe.ts +119 -3
  72. package/src/index.ts +6 -0
  73. package/src/migrate/emit-ts.ts +67 -14
  74. package/src/migrate/parse-connection.ts +56 -6
  75. package/src/migrate/parse-datasource.ts +74 -3
  76. package/src/migrate/parse-pipe.ts +250 -111
  77. package/src/migrate/parser-utils.ts +5 -1
  78. package/src/migrate/types.ts +26 -1
  79. package/src/schema/connection.test.ts +48 -0
  80. package/src/schema/connection.ts +60 -1
  81. package/src/schema/datasource.test.ts +91 -3
  82. package/src/schema/datasource.ts +62 -3
@@ -1,6 +1,10 @@
1
1
  import { describe, it, expect } from "vitest";
2
2
  import { generateConnection, generateAllConnections } from "./connection.js";
3
- import { defineKafkaConnection, defineS3Connection } from "../schema/connection.js";
3
+ import {
4
+ defineKafkaConnection,
5
+ defineS3Connection,
6
+ defineGCSConnection,
7
+ } from "../schema/connection.js";
4
8
 
5
9
  describe("Connection Generator", () => {
6
10
  describe("generateConnection", () => {
@@ -152,6 +156,20 @@ describe("Connection Generator", () => {
152
156
  expect(result.content).toContain('S3_ACCESS_KEY {{ tb_secret("S3_ACCESS_KEY") }}');
153
157
  expect(result.content).toContain('S3_SECRET {{ tb_secret("S3_SECRET") }}');
154
158
  });
159
+
160
+ it("generates GCS connection with service account credentials", () => {
161
+ const conn = defineGCSConnection("my_gcs", {
162
+ serviceAccountCredentialsJson: '{{ tb_secret("GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON") }}',
163
+ });
164
+
165
+ const result = generateConnection(conn);
166
+
167
+ expect(result.name).toBe("my_gcs");
168
+ expect(result.content).toContain("TYPE gcs");
169
+ expect(result.content).toContain(
170
+ 'GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON {{ tb_secret("GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON") }}'
171
+ );
172
+ });
155
173
  });
156
174
 
157
175
  describe("generateAllConnections", () => {
@@ -163,11 +181,18 @@ describe("Connection Generator", () => {
163
181
  region: "us-east-1",
164
182
  arn: "arn:aws:iam::123456789012:role/tinybird-s3-access",
165
183
  });
184
+ const conn3 = defineGCSConnection("gcs_landing", {
185
+ serviceAccountCredentialsJson: '{{ tb_secret("GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON") }}',
186
+ });
166
187
 
167
- const results = generateAllConnections({ kafka1: conn1, s3_logs: conn2 });
188
+ const results = generateAllConnections({
189
+ kafka1: conn1,
190
+ s3_logs: conn2,
191
+ gcs_landing: conn3,
192
+ });
168
193
 
169
- expect(results).toHaveLength(2);
170
- expect(results.map((r) => r.name).sort()).toEqual(["kafka1", "s3_logs"]);
194
+ expect(results).toHaveLength(3);
195
+ expect(results.map((r) => r.name).sort()).toEqual(["gcs_landing", "kafka1", "s3_logs"]);
171
196
  });
172
197
 
173
198
  it("returns empty array for empty connections", () => {
@@ -3,8 +3,16 @@
3
3
  * Converts ConnectionDefinition to native .connection file format
4
4
  */
5
5
 
6
- import type { ConnectionDefinition, KafkaConnectionDefinition } from "../schema/connection.js";
7
- import { isS3ConnectionDefinition, type S3ConnectionDefinition } from "../schema/connection.js";
6
+ import type {
7
+ ConnectionDefinition,
8
+ KafkaConnectionDefinition,
9
+ GCSConnectionDefinition,
10
+ } from "../schema/connection.js";
11
+ import {
12
+ isS3ConnectionDefinition,
13
+ isGCSConnectionDefinition,
14
+ type S3ConnectionDefinition,
15
+ } from "../schema/connection.js";
8
16
 
9
17
  /**
10
18
  * Generated connection content
@@ -78,6 +86,19 @@ function generateS3Connection(connection: S3ConnectionDefinition): string {
78
86
  return parts.join("\n");
79
87
  }
80
88
 
89
+ /**
90
+ * Generate a GCS connection content
91
+ */
92
+ function generateGCSConnection(connection: GCSConnectionDefinition): string {
93
+ const parts: string[] = [];
94
+ const options = connection.options;
95
+
96
+ parts.push("TYPE gcs");
97
+ parts.push(`GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON ${options.serviceAccountCredentialsJson}`);
98
+
99
+ return parts.join("\n");
100
+ }
101
+
81
102
  /**
82
103
  * Generate a .connection file content from a ConnectionDefinition
83
104
  *
@@ -113,6 +134,8 @@ export function generateConnection(
113
134
  content = generateKafkaConnection(connection as KafkaConnectionDefinition);
114
135
  } else if (isS3ConnectionDefinition(connection)) {
115
136
  content = generateS3Connection(connection);
137
+ } else if (isGCSConnectionDefinition(connection)) {
138
+ content = generateGCSConnection(connection);
116
139
  } else {
117
140
  throw new Error("Unsupported connection type.");
118
141
  }
@@ -1,7 +1,7 @@
1
1
  import { describe, it, expect } from 'vitest';
2
2
  import { generateDatasource, generateAllDatasources } from './datasource.js';
3
3
  import { defineDatasource } from '../schema/datasource.js';
4
- import { defineKafkaConnection, defineS3Connection } from '../schema/connection.js';
4
+ import { defineKafkaConnection, defineS3Connection, defineGCSConnection } from '../schema/connection.js';
5
5
  import { defineToken } from '../schema/token.js';
6
6
  import { t } from '../schema/types.js';
7
7
  import { engine } from '../schema/engines.js';
@@ -95,6 +95,28 @@ describe('Datasource Generator', () => {
95
95
  expect(result.content).toContain('FORWARD_QUERY >');
96
96
  expect(result.content).toContain(' SELECT id');
97
97
  });
98
+
99
+ it("includes indexes block when provided", () => {
100
+ const ds = defineDatasource("test_ds", {
101
+ schema: {
102
+ id: t.string(),
103
+ pipe_name: t.string(),
104
+ },
105
+ indexes: [
106
+ { name: "pipe_name_set", expr: "pipe_name", type: "set(100)", granularity: 1 },
107
+ { name: "id_bf", expr: "lower(id)", type: "bloom_filter(0.001)", granularity: 4 },
108
+ ],
109
+ });
110
+
111
+ const result = generateDatasource(ds);
112
+ expect(result.content).toContain("INDEXES >");
113
+ expect(result.content).toContain(
114
+ "pipe_name_set pipe_name TYPE set(100) GRANULARITY 1"
115
+ );
116
+ expect(result.content).toContain(
117
+ "id_bf lower(id) TYPE bloom_filter(0.001) GRANULARITY 4"
118
+ );
119
+ });
98
120
  });
99
121
 
100
122
  describe('Column formatting', () => {
@@ -529,6 +551,35 @@ describe('Datasource Generator', () => {
529
551
  });
530
552
  });
531
553
 
554
+ describe('GCS configuration', () => {
555
+ it('includes GCS import directives', () => {
556
+ const gcsConn = defineGCSConnection('my_gcs', {
557
+ serviceAccountCredentialsJson: '{{ tb_secret("GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON") }}',
558
+ });
559
+
560
+ const ds = defineDatasource('gcs_events', {
561
+ schema: {
562
+ timestamp: t.dateTime(),
563
+ event: t.string(),
564
+ },
565
+ engine: engine.mergeTree({ sortingKey: ['timestamp'] }),
566
+ gcs: {
567
+ connection: gcsConn,
568
+ bucketUri: 'gs://my-bucket/events/*.csv',
569
+ schedule: '@auto',
570
+ fromTimestamp: '2024-01-01T00:00:00Z',
571
+ },
572
+ });
573
+
574
+ const result = generateDatasource(ds);
575
+
576
+ expect(result.content).toContain('IMPORT_CONNECTION_NAME my_gcs');
577
+ expect(result.content).toContain('IMPORT_BUCKET_URI gs://my-bucket/events/*.csv');
578
+ expect(result.content).toContain('IMPORT_SCHEDULE @auto');
579
+ expect(result.content).toContain('IMPORT_FROM_TIMESTAMP 2024-01-01T00:00:00Z');
580
+ });
581
+ });
582
+
532
583
  describe('Token generation', () => {
533
584
  it('generates TOKEN lines with inline config', () => {
534
585
  const ds = defineDatasource('test_ds', {
@@ -9,7 +9,9 @@ import type {
9
9
  ColumnDefinition,
10
10
  KafkaConfig,
11
11
  S3Config,
12
+ GCSConfig,
12
13
  TokenConfig,
14
+ DatasourceIndex,
13
15
  } from "../schema/datasource.js";
14
16
  import type { AnyTypeValidator, TypeModifiers } from "../schema/types.js";
15
17
  import { getColumnType, getColumnJsonPath } from "../schema/datasource.js";
@@ -176,18 +178,18 @@ function generateKafkaConfig(kafka: KafkaConfig): string {
176
178
  /**
177
179
  * Generate S3 import configuration lines
178
180
  */
179
- function generateS3Config(s3: S3Config): string {
181
+ function generateImportConfig(importConfig: S3Config | GCSConfig): string {
180
182
  const parts: string[] = [];
181
183
 
182
- parts.push(`IMPORT_CONNECTION_NAME ${s3.connection._name}`);
183
- parts.push(`IMPORT_BUCKET_URI ${s3.bucketUri}`);
184
+ parts.push(`IMPORT_CONNECTION_NAME ${importConfig.connection._name}`);
185
+ parts.push(`IMPORT_BUCKET_URI ${importConfig.bucketUri}`);
184
186
 
185
- if (s3.schedule) {
186
- parts.push(`IMPORT_SCHEDULE ${s3.schedule}`);
187
+ if (importConfig.schedule) {
188
+ parts.push(`IMPORT_SCHEDULE ${importConfig.schedule}`);
187
189
  }
188
190
 
189
- if (s3.fromTimestamp) {
190
- parts.push(`IMPORT_FROM_TIMESTAMP ${s3.fromTimestamp}`);
191
+ if (importConfig.fromTimestamp) {
192
+ parts.push(`IMPORT_FROM_TIMESTAMP ${importConfig.fromTimestamp}`);
191
193
  }
192
194
 
193
195
  return parts.join("\n");
@@ -210,6 +212,23 @@ function generateForwardQuery(forwardQuery?: string): string | null {
210
212
  return ["FORWARD_QUERY >", ...lines.map((line) => ` ${line}`)].join("\n");
211
213
  }
212
214
 
215
+ /**
216
+ * Generate INDEXES section
217
+ */
218
+ function generateIndexes(indexes?: readonly DatasourceIndex[]): string | null {
219
+ if (!indexes || indexes.length === 0) {
220
+ return null;
221
+ }
222
+
223
+ const lines = ["INDEXES >"];
224
+ for (const index of indexes) {
225
+ lines.push(
226
+ ` ${index.name} ${index.expr} TYPE ${index.type} GRANULARITY ${index.granularity}`
227
+ );
228
+ }
229
+ return lines.join("\n");
230
+ }
231
+
213
232
  /**
214
233
  * Generate SHARED_WITH section for sharing datasource with other workspaces
215
234
  */
@@ -291,8 +310,13 @@ export function generateDatasource(
291
310
  ): GeneratedDatasource {
292
311
  const parts: string[] = [];
293
312
 
294
- if (datasource.options.kafka && datasource.options.s3) {
295
- throw new Error("Datasource cannot define both `kafka` and `s3` ingestion options.");
313
+ const ingestionConfigCount = [
314
+ datasource.options.kafka,
315
+ datasource.options.s3,
316
+ datasource.options.gcs,
317
+ ].filter(Boolean).length;
318
+ if (ingestionConfigCount > 1) {
319
+ throw new Error("Datasource can only define one ingestion option: `kafka`, `s3`, or `gcs`.");
296
320
  }
297
321
 
298
322
  // Add description if present
@@ -311,6 +335,13 @@ export function generateDatasource(
311
335
  // Add engine configuration
312
336
  parts.push(generateEngineConfig(datasource.options.engine));
313
337
 
338
+ // Add indexes if present
339
+ const indexes = generateIndexes(datasource.options.indexes);
340
+ if (indexes) {
341
+ parts.push("");
342
+ parts.push(indexes);
343
+ }
344
+
314
345
  // Add Kafka configuration if present
315
346
  if (datasource.options.kafka) {
316
347
  parts.push("");
@@ -320,7 +351,13 @@ export function generateDatasource(
320
351
  // Add S3 configuration if present
321
352
  if (datasource.options.s3) {
322
353
  parts.push("");
323
- parts.push(generateS3Config(datasource.options.s3));
354
+ parts.push(generateImportConfig(datasource.options.s3));
355
+ }
356
+
357
+ // Add GCS configuration if present
358
+ if (datasource.options.gcs) {
359
+ parts.push("");
360
+ parts.push(generateImportConfig(datasource.options.gcs));
324
361
  }
325
362
 
326
363
  // Add forward query if present
@@ -164,6 +164,27 @@ describe('Pipe Generator', () => {
164
164
  const result = generatePipe(pipe);
165
165
  expect(result.content).toContain(' %\n');
166
166
  });
167
+
168
+ it('injects param defaults into placeholders when SQL omits them', () => {
169
+ const pipe = definePipe('defaults_pipe', {
170
+ params: {
171
+ start_date: p.date().optional('2025-03-01'),
172
+ page: p.int32().optional(0),
173
+ },
174
+ nodes: [
175
+ node({
176
+ name: 'endpoint',
177
+ sql: 'SELECT * FROM events WHERE d >= {{Date(start_date)}} LIMIT 10 OFFSET {{Int32(page)}}',
178
+ }),
179
+ ],
180
+ output: simpleOutput,
181
+ endpoint: true,
182
+ });
183
+
184
+ const result = generatePipe(pipe);
185
+ expect(result.content).toContain("{{ Date(start_date, '2025-03-01') }}");
186
+ expect(result.content).toContain('{{ Int32(page, 0) }}');
187
+ });
167
188
  });
168
189
 
169
190
  describe('Multiple nodes', () => {
@@ -18,6 +18,8 @@ import {
18
18
  getCopyConfig,
19
19
  getSinkConfig,
20
20
  } from "../schema/pipe.js";
21
+ import type { AnyParamValidator } from "../schema/params.js";
22
+ import { getParamDefault } from "../schema/params.js";
21
23
 
22
24
  /**
23
25
  * Generated pipe content
@@ -36,10 +38,123 @@ function hasDynamicParameters(sql: string): boolean {
36
38
  return /\{\{[^}]+\}\}/.test(sql) || /\{%[^%]+%\}/.test(sql);
37
39
  }
38
40
 
41
+ function splitTopLevelComma(input: string): string[] {
42
+ const parts: string[] = [];
43
+ let current = "";
44
+ let depth = 0;
45
+ let inSingleQuote = false;
46
+ let inDoubleQuote = false;
47
+
48
+ for (let i = 0; i < input.length; i += 1) {
49
+ const char = input[i];
50
+ const prev = i > 0 ? input[i - 1] : "";
51
+
52
+ if (char === "'" && !inDoubleQuote && prev !== "\\") {
53
+ inSingleQuote = !inSingleQuote;
54
+ current += char;
55
+ continue;
56
+ }
57
+
58
+ if (char === '"' && !inSingleQuote && prev !== "\\") {
59
+ inDoubleQuote = !inDoubleQuote;
60
+ current += char;
61
+ continue;
62
+ }
63
+
64
+ if (!inSingleQuote && !inDoubleQuote) {
65
+ if (char === "(") {
66
+ depth += 1;
67
+ current += char;
68
+ continue;
69
+ }
70
+ if (char === ")" && depth > 0) {
71
+ depth -= 1;
72
+ current += char;
73
+ continue;
74
+ }
75
+ if (char === "," && depth === 0) {
76
+ parts.push(current.trim());
77
+ current = "";
78
+ continue;
79
+ }
80
+ }
81
+
82
+ current += char;
83
+ }
84
+
85
+ if (current.trim()) {
86
+ parts.push(current.trim());
87
+ }
88
+
89
+ return parts;
90
+ }
91
+
92
+ function toTemplateDefaultLiteral(value: string | number | boolean): string {
93
+ if (typeof value === "string") {
94
+ return `'${value.replace(/\\/g, "\\\\").replace(/'/g, "\\'")}'`;
95
+ }
96
+ return String(value);
97
+ }
98
+
99
+ function applyParamDefaultsToSql(
100
+ sql: string,
101
+ params?: Record<string, AnyParamValidator>
102
+ ): string {
103
+ if (!params) {
104
+ return sql;
105
+ }
106
+
107
+ const defaults = new Map<string, string>();
108
+ for (const [name, validator] of Object.entries(params)) {
109
+ const defaultValue = getParamDefault(validator);
110
+ if (defaultValue !== undefined) {
111
+ defaults.set(name, toTemplateDefaultLiteral(defaultValue as string | number | boolean));
112
+ }
113
+ }
114
+
115
+ if (defaults.size === 0) {
116
+ return sql;
117
+ }
118
+
119
+ const placeholderRegex = /\{\{\s*([^{}]+?)\s*\}\}/g;
120
+ return sql.replace(placeholderRegex, (fullMatch, rawExpression) => {
121
+ const expression = String(rawExpression);
122
+ const rewritten = expression.replace(
123
+ /([a-zA-Z_][a-zA-Z0-9_]*)\s*\(([^()]*)\)/g,
124
+ (call, _functionName, rawArgs) => {
125
+ const args = splitTopLevelComma(String(rawArgs));
126
+ if (args.length !== 1) {
127
+ return call;
128
+ }
129
+
130
+ const paramName = args[0]?.trim() ?? "";
131
+ if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(paramName)) {
132
+ return call;
133
+ }
134
+
135
+ const defaultLiteral = defaults.get(paramName);
136
+ if (!defaultLiteral) {
137
+ return call;
138
+ }
139
+
140
+ return call.replace(/\)\s*$/, `, ${defaultLiteral})`);
141
+ }
142
+ );
143
+
144
+ if (rewritten === expression) {
145
+ return fullMatch;
146
+ }
147
+ return `{{ ${rewritten.trim()} }}`;
148
+ });
149
+ }
150
+
39
151
  /**
40
152
  * Generate a NODE section for the pipe
41
153
  */
42
- function generateNode(node: NodeDefinition): string {
154
+ function generateNode(
155
+ node: NodeDefinition,
156
+ params?: Record<string, AnyParamValidator>
157
+ ): string {
43
158
  const parts: string[] = [];
44
159
 
45
160
  parts.push(`NODE ${node._name}`);
@@ -57,7 +172,8 @@ function generateNode(node: NodeDefinition): string {
57
172
  parts.push(` %`);
58
173
  }
59
174
 
60
- const sqlLines = node.sql.trim().split("\n");
175
+ const sqlWithDefaults = applyParamDefaultsToSql(node.sql, params);
176
+ const sqlLines = sqlWithDefaults.trim().split("\n");
61
177
  sqlLines.forEach((line) => {
62
178
  parts.push(` ${line}`);
63
179
  });
@@ -225,7 +341,7 @@ export function generatePipe(pipe: PipeDefinition): GeneratedPipe {
225
341
 
226
342
  // Add all nodes
227
343
  pipe.options.nodes.forEach((node, index) => {
228
- parts.push(generateNode(node));
344
+ parts.push(generateNode(node, pipe.options.params as Record<string, AnyParamValidator> | undefined));
229
345
  // Add empty line between nodes
230
346
  if (index < pipe.options.nodes.length - 1) {
231
347
  parts.push("");
package/src/index.ts CHANGED
@@ -113,6 +113,8 @@ export type {
113
113
  ExtractSchema,
114
114
  KafkaConfig,
115
115
  S3Config,
116
+ GCSConfig,
117
+ DatasourceIndex,
116
118
  } from "./schema/datasource.js";
117
119
 
118
120
  // ============ Connection ============
@@ -120,9 +122,11 @@ export {
120
122
  defineKafkaConnection,
121
123
  createKafkaConnection,
122
124
  defineS3Connection,
125
+ defineGCSConnection,
123
126
  isConnectionDefinition,
124
127
  isKafkaConnectionDefinition,
125
128
  isS3ConnectionDefinition,
129
+ isGCSConnectionDefinition,
126
130
  getConnectionType,
127
131
  } from "./schema/connection.js";
128
132
  export type {
@@ -133,6 +137,8 @@ export type {
133
137
  KafkaSaslMechanism,
134
138
  S3ConnectionDefinition,
135
139
  S3ConnectionOptions,
140
+ GCSConnectionDefinition,
141
+ GCSConnectionOptions,
136
142
  } from "./schema/connection.js";
137
143
 
138
144
  // ============ Token ============
@@ -4,6 +4,7 @@ import { parseLiteralFromDatafile, toTsLiteral } from "./parser-utils.js";
4
4
  import type {
5
5
  DatasourceModel,
6
6
  DatasourceEngineModel,
7
+ GCSConnectionModel,
7
8
  KafkaConnectionModel,
8
9
  ParsedResource,
9
10
  PipeModel,
@@ -59,11 +60,13 @@ function hasSecretTemplate(resources: ParsedResource[]): boolean {
59
60
  if (resource.secret) values.push(resource.secret);
60
61
  if (resource.sslCaPem) values.push(resource.sslCaPem);
61
62
  if (resource.schemaRegistryUrl) values.push(resource.schemaRegistryUrl);
62
- } else {
63
+ } else if (resource.connectionType === "s3") {
63
64
  values.push(resource.region);
64
65
  if (resource.arn) values.push(resource.arn);
65
66
  if (resource.accessKey) values.push(resource.accessKey);
66
67
  if (resource.secret) values.push(resource.secret);
68
+ } else {
69
+ values.push(resource.serviceAccountCredentialsJson);
67
70
  }
68
71
  continue;
69
72
  }
@@ -80,6 +83,11 @@ function hasSecretTemplate(resources: ParsedResource[]): boolean {
80
83
  if (resource.s3.schedule) values.push(resource.s3.schedule);
81
84
  if (resource.s3.fromTimestamp) values.push(resource.s3.fromTimestamp);
82
85
  }
86
+ if (resource.gcs) {
87
+ values.push(resource.gcs.bucketUri);
88
+ if (resource.gcs.schedule) values.push(resource.gcs.schedule);
89
+ if (resource.gcs.fromTimestamp) values.push(resource.gcs.fromTimestamp);
90
+ }
83
91
  continue;
84
92
  }
85
93
  }
@@ -125,6 +133,8 @@ function strictParamBaseValidator(type: string): string {
125
133
  const map: Record<string, string> = {
126
134
  String: "p.string()",
127
135
  UUID: "p.uuid()",
136
+ Int: "p.int32()",
137
+ Integer: "p.int32()",
128
138
  Int8: "p.int8()",
129
139
  Int16: "p.int16()",
130
140
  Int32: "p.int32()",
@@ -141,6 +151,8 @@ function strictParamBaseValidator(type: string): string {
141
151
  DateTime: "p.dateTime()",
142
152
  DateTime64: "p.dateTime64()",
143
153
  Array: "p.array(p.string())",
154
+ column: "p.column()",
155
+ JSON: "p.json()",
144
156
  };
145
157
  const validator = map[type];
146
158
  if (!validator) {
@@ -305,6 +317,15 @@ function emitDatasource(ds: DatasourceModel): string {
305
317
  if (ds.engine) {
306
318
  lines.push(` engine: ${emitEngineOptions(ds.engine)},`);
307
319
  }
320
+ if (ds.indexes.length > 0) {
321
+ lines.push(" indexes: [");
322
+ for (const index of ds.indexes) {
323
+ lines.push(
324
+ ` { name: ${escapeString(index.name)}, expr: ${escapeString(index.expr)}, type: ${escapeString(index.type)}, granularity: ${index.granularity} },`
325
+ );
326
+ }
327
+ lines.push(" ],");
328
+ }
308
329
 
309
330
  if (ds.kafka) {
310
331
  const connectionVar = toCamelCase(ds.kafka.connectionName);
@@ -337,6 +358,20 @@ function emitDatasource(ds: DatasourceModel): string {
337
358
  lines.push(" },");
338
359
  }
339
360
 
361
+ if (ds.gcs) {
362
+ const connectionVar = toCamelCase(ds.gcs.connectionName);
363
+ lines.push(" gcs: {");
364
+ lines.push(` connection: ${connectionVar},`);
365
+ lines.push(` bucketUri: ${emitStringOrSecret(ds.gcs.bucketUri)},`);
366
+ if (ds.gcs.schedule) {
367
+ lines.push(` schedule: ${emitStringOrSecret(ds.gcs.schedule)},`);
368
+ }
369
+ if (ds.gcs.fromTimestamp) {
370
+ lines.push(` fromTimestamp: ${emitStringOrSecret(ds.gcs.fromTimestamp)},`);
371
+ }
372
+ lines.push(" },");
373
+ }
374
+
340
375
  if (ds.forwardQuery) {
341
376
  lines.push(" forwardQuery: `");
342
377
  lines.push(ds.forwardQuery.replace(/`/g, "\\`").replace(/\${/g, "\\${"));
@@ -364,7 +399,9 @@ function emitDatasource(ds: DatasourceModel): string {
364
399
  return lines.join("\n");
365
400
  }
366
401
 
367
- function emitConnection(connection: KafkaConnectionModel | S3ConnectionModel): string {
402
+ function emitConnection(
403
+ connection: KafkaConnectionModel | S3ConnectionModel | GCSConnectionModel
404
+ ): string {
368
405
  const variableName = toCamelCase(connection.name);
369
406
  const lines: string[] = [];
370
407
 
@@ -396,19 +433,31 @@ function emitConnection(connection: KafkaConnectionModel | S3ConnectionModel): s
396
433
  return lines.join("\n");
397
434
  }
398
435
 
436
+ if (connection.connectionType === "s3") {
437
+ lines.push(
438
+ `export const ${variableName} = defineS3Connection(${escapeString(connection.name)}, {`
439
+ );
440
+ lines.push(` region: ${emitStringOrSecret(connection.region)},`);
441
+ if (connection.arn) {
442
+ lines.push(` arn: ${emitStringOrSecret(connection.arn)},`);
443
+ }
444
+ if (connection.accessKey) {
445
+ lines.push(` accessKey: ${emitStringOrSecret(connection.accessKey)},`);
446
+ }
447
+ if (connection.secret) {
448
+ lines.push(` secret: ${emitStringOrSecret(connection.secret)},`);
449
+ }
450
+ lines.push("});");
451
+ lines.push("");
452
+ return lines.join("\n");
453
+ }
454
+
399
455
  lines.push(
400
- `export const ${variableName} = defineS3Connection(${escapeString(connection.name)}, {`
456
+ `export const ${variableName} = defineGCSConnection(${escapeString(connection.name)}, {`
457
+ );
458
+ lines.push(
459
+ ` serviceAccountCredentialsJson: ${emitStringOrSecret(connection.serviceAccountCredentialsJson)},`
401
460
  );
402
- lines.push(` region: ${emitStringOrSecret(connection.region)},`);
403
- if (connection.arn) {
404
- lines.push(` arn: ${emitStringOrSecret(connection.arn)},`);
405
- }
406
- if (connection.accessKey) {
407
- lines.push(` accessKey: ${emitStringOrSecret(connection.accessKey)},`);
408
- }
409
- if (connection.secret) {
410
- lines.push(` secret: ${emitStringOrSecret(connection.secret)},`);
411
- }
412
461
  lines.push("});");
413
462
  lines.push("");
414
463
  return lines.join("\n");
@@ -542,7 +591,7 @@ function emitPipe(pipe: PipeModel): string {
542
591
 
543
592
  export function emitMigrationFileContent(resources: ParsedResource[]): string {
544
593
  const connections = resources.filter(
545
- (resource): resource is KafkaConnectionModel | S3ConnectionModel =>
594
+ (resource): resource is KafkaConnectionModel | S3ConnectionModel | GCSConnectionModel =>
546
595
  resource.kind === "connection"
547
596
  );
548
597
  const datasources = resources.filter(
@@ -569,6 +618,9 @@ export function emitMigrationFileContent(resources: ParsedResource[]): string {
569
618
  if (connections.some((connection) => connection.connectionType === "s3")) {
570
619
  imports.add("defineS3Connection");
571
620
  }
621
+ if (connections.some((connection) => connection.connectionType === "gcs")) {
622
+ imports.add("defineGCSConnection");
623
+ }
572
624
  if (needsParams) {
573
625
  imports.add("p");
574
626
  }
@@ -585,6 +637,7 @@ export function emitMigrationFileContent(resources: ParsedResource[]): string {
585
637
  const orderedImports = [
586
638
  "defineKafkaConnection",
587
639
  "defineS3Connection",
640
+ "defineGCSConnection",
588
641
  "defineDatasource",
589
642
  "definePipe",
590
643
  "defineMaterializedView",