@tinybirdco/sdk 0.0.49 → 0.0.51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -2
- package/dist/cli/commands/migrate.d.ts.map +1 -1
- package/dist/cli/commands/migrate.js +36 -1
- package/dist/cli/commands/migrate.js.map +1 -1
- package/dist/cli/commands/migrate.test.js +307 -2
- package/dist/cli/commands/migrate.test.js.map +1 -1
- package/dist/codegen/type-mapper.d.ts.map +1 -1
- package/dist/codegen/type-mapper.js +70 -7
- package/dist/codegen/type-mapper.js.map +1 -1
- package/dist/codegen/type-mapper.test.js +9 -0
- package/dist/codegen/type-mapper.test.js.map +1 -1
- package/dist/generator/connection.d.ts.map +1 -1
- package/dist/generator/connection.js +14 -1
- package/dist/generator/connection.js.map +1 -1
- package/dist/generator/connection.test.js +20 -4
- package/dist/generator/connection.test.js.map +1 -1
- package/dist/generator/datasource.d.ts.map +1 -1
- package/dist/generator/datasource.js +39 -10
- package/dist/generator/datasource.js.map +1 -1
- package/dist/generator/datasource.test.js +42 -1
- package/dist/generator/datasource.test.js.map +1 -1
- package/dist/generator/pipe.d.ts.map +1 -1
- package/dist/generator/pipe.js +92 -3
- package/dist/generator/pipe.js.map +1 -1
- package/dist/generator/pipe.test.js +19 -0
- package/dist/generator/pipe.test.js.map +1 -1
- package/dist/index.d.ts +3 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/migrate/emit-ts.d.ts.map +1 -1
- package/dist/migrate/emit-ts.js +56 -11
- package/dist/migrate/emit-ts.js.map +1 -1
- package/dist/migrate/parse-connection.d.ts +2 -2
- package/dist/migrate/parse-connection.d.ts.map +1 -1
- package/dist/migrate/parse-connection.js +34 -4
- package/dist/migrate/parse-connection.js.map +1 -1
- package/dist/migrate/parse-datasource.d.ts.map +1 -1
- package/dist/migrate/parse-datasource.js +39 -2
- package/dist/migrate/parse-datasource.js.map +1 -1
- package/dist/migrate/parse-pipe.d.ts.map +1 -1
- package/dist/migrate/parse-pipe.js +212 -93
- package/dist/migrate/parse-pipe.js.map +1 -1
- package/dist/migrate/parser-utils.d.ts.map +1 -1
- package/dist/migrate/parser-utils.js +3 -1
- package/dist/migrate/parser-utils.js.map +1 -1
- package/dist/migrate/types.d.ts +22 -1
- package/dist/migrate/types.d.ts.map +1 -1
- package/dist/schema/connection.d.ts +34 -1
- package/dist/schema/connection.d.ts.map +1 -1
- package/dist/schema/connection.js +26 -0
- package/dist/schema/connection.js.map +1 -1
- package/dist/schema/connection.test.js +35 -1
- package/dist/schema/connection.test.js.map +1 -1
- package/dist/schema/datasource.d.ts +32 -1
- package/dist/schema/datasource.d.ts.map +1 -1
- package/dist/schema/datasource.js +19 -2
- package/dist/schema/datasource.js.map +1 -1
- package/dist/schema/datasource.test.js +71 -3
- package/dist/schema/datasource.test.js.map +1 -1
- package/package.json +1 -1
- package/src/cli/commands/migrate.test.ts +448 -2
- package/src/cli/commands/migrate.ts +39 -1
- package/src/codegen/type-mapper.test.ts +18 -0
- package/src/codegen/type-mapper.ts +79 -7
- package/src/generator/connection.test.ts +29 -4
- package/src/generator/connection.ts +25 -2
- package/src/generator/datasource.test.ts +52 -1
- package/src/generator/datasource.ts +47 -10
- package/src/generator/pipe.test.ts +21 -0
- package/src/generator/pipe.ts +119 -3
- package/src/index.ts +6 -0
- package/src/migrate/emit-ts.ts +67 -14
- package/src/migrate/parse-connection.ts +56 -6
- package/src/migrate/parse-datasource.ts +74 -3
- package/src/migrate/parse-pipe.ts +250 -111
- package/src/migrate/parser-utils.ts +5 -1
- package/src/migrate/types.ts +26 -1
- package/src/schema/connection.test.ts +48 -0
- package/src/schema/connection.ts +60 -1
- package/src/schema/datasource.test.ts +91 -3
- package/src/schema/datasource.ts +62 -3
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
import { describe, it, expect } from "vitest";
|
|
2
2
|
import { generateConnection, generateAllConnections } from "./connection.js";
|
|
3
|
-
import {
|
|
3
|
+
import {
|
|
4
|
+
defineKafkaConnection,
|
|
5
|
+
defineS3Connection,
|
|
6
|
+
defineGCSConnection,
|
|
7
|
+
} from "../schema/connection.js";
|
|
4
8
|
|
|
5
9
|
describe("Connection Generator", () => {
|
|
6
10
|
describe("generateConnection", () => {
|
|
@@ -152,6 +156,20 @@ describe("Connection Generator", () => {
|
|
|
152
156
|
expect(result.content).toContain('S3_ACCESS_KEY {{ tb_secret("S3_ACCESS_KEY") }}');
|
|
153
157
|
expect(result.content).toContain('S3_SECRET {{ tb_secret("S3_SECRET") }}');
|
|
154
158
|
});
|
|
159
|
+
|
|
160
|
+
it("generates GCS connection with service account credentials", () => {
|
|
161
|
+
const conn = defineGCSConnection("my_gcs", {
|
|
162
|
+
serviceAccountCredentialsJson: '{{ tb_secret("GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON") }}',
|
|
163
|
+
});
|
|
164
|
+
|
|
165
|
+
const result = generateConnection(conn);
|
|
166
|
+
|
|
167
|
+
expect(result.name).toBe("my_gcs");
|
|
168
|
+
expect(result.content).toContain("TYPE gcs");
|
|
169
|
+
expect(result.content).toContain(
|
|
170
|
+
'GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON {{ tb_secret("GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON") }}'
|
|
171
|
+
);
|
|
172
|
+
});
|
|
155
173
|
});
|
|
156
174
|
|
|
157
175
|
describe("generateAllConnections", () => {
|
|
@@ -163,11 +181,18 @@ describe("Connection Generator", () => {
|
|
|
163
181
|
region: "us-east-1",
|
|
164
182
|
arn: "arn:aws:iam::123456789012:role/tinybird-s3-access",
|
|
165
183
|
});
|
|
184
|
+
const conn3 = defineGCSConnection("gcs_landing", {
|
|
185
|
+
serviceAccountCredentialsJson: '{{ tb_secret("GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON") }}',
|
|
186
|
+
});
|
|
166
187
|
|
|
167
|
-
const results = generateAllConnections({
|
|
188
|
+
const results = generateAllConnections({
|
|
189
|
+
kafka1: conn1,
|
|
190
|
+
s3_logs: conn2,
|
|
191
|
+
gcs_landing: conn3,
|
|
192
|
+
});
|
|
168
193
|
|
|
169
|
-
expect(results).toHaveLength(
|
|
170
|
-
expect(results.map((r) => r.name).sort()).toEqual(["kafka1", "s3_logs"]);
|
|
194
|
+
expect(results).toHaveLength(3);
|
|
195
|
+
expect(results.map((r) => r.name).sort()).toEqual(["gcs_landing", "kafka1", "s3_logs"]);
|
|
171
196
|
});
|
|
172
197
|
|
|
173
198
|
it("returns empty array for empty connections", () => {
|
|
@@ -3,8 +3,16 @@
|
|
|
3
3
|
* Converts ConnectionDefinition to native .connection file format
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
|
-
import type {
|
|
7
|
-
|
|
6
|
+
import type {
|
|
7
|
+
ConnectionDefinition,
|
|
8
|
+
KafkaConnectionDefinition,
|
|
9
|
+
GCSConnectionDefinition,
|
|
10
|
+
} from "../schema/connection.js";
|
|
11
|
+
import {
|
|
12
|
+
isS3ConnectionDefinition,
|
|
13
|
+
isGCSConnectionDefinition,
|
|
14
|
+
type S3ConnectionDefinition,
|
|
15
|
+
} from "../schema/connection.js";
|
|
8
16
|
|
|
9
17
|
/**
|
|
10
18
|
* Generated connection content
|
|
@@ -78,6 +86,19 @@ function generateS3Connection(connection: S3ConnectionDefinition): string {
|
|
|
78
86
|
return parts.join("\n");
|
|
79
87
|
}
|
|
80
88
|
|
|
89
|
+
/**
|
|
90
|
+
* Generate a GCS connection content
|
|
91
|
+
*/
|
|
92
|
+
function generateGCSConnection(connection: GCSConnectionDefinition): string {
|
|
93
|
+
const parts: string[] = [];
|
|
94
|
+
const options = connection.options;
|
|
95
|
+
|
|
96
|
+
parts.push("TYPE gcs");
|
|
97
|
+
parts.push(`GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON ${options.serviceAccountCredentialsJson}`);
|
|
98
|
+
|
|
99
|
+
return parts.join("\n");
|
|
100
|
+
}
|
|
101
|
+
|
|
81
102
|
/**
|
|
82
103
|
* Generate a .connection file content from a ConnectionDefinition
|
|
83
104
|
*
|
|
@@ -113,6 +134,8 @@ export function generateConnection(
|
|
|
113
134
|
content = generateKafkaConnection(connection as KafkaConnectionDefinition);
|
|
114
135
|
} else if (isS3ConnectionDefinition(connection)) {
|
|
115
136
|
content = generateS3Connection(connection);
|
|
137
|
+
} else if (isGCSConnectionDefinition(connection)) {
|
|
138
|
+
content = generateGCSConnection(connection);
|
|
116
139
|
} else {
|
|
117
140
|
throw new Error("Unsupported connection type.");
|
|
118
141
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { describe, it, expect } from 'vitest';
|
|
2
2
|
import { generateDatasource, generateAllDatasources } from './datasource.js';
|
|
3
3
|
import { defineDatasource } from '../schema/datasource.js';
|
|
4
|
-
import { defineKafkaConnection, defineS3Connection } from '../schema/connection.js';
|
|
4
|
+
import { defineKafkaConnection, defineS3Connection, defineGCSConnection } from '../schema/connection.js';
|
|
5
5
|
import { defineToken } from '../schema/token.js';
|
|
6
6
|
import { t } from '../schema/types.js';
|
|
7
7
|
import { engine } from '../schema/engines.js';
|
|
@@ -95,6 +95,28 @@ describe('Datasource Generator', () => {
|
|
|
95
95
|
expect(result.content).toContain('FORWARD_QUERY >');
|
|
96
96
|
expect(result.content).toContain(' SELECT id');
|
|
97
97
|
});
|
|
98
|
+
|
|
99
|
+
it("includes indexes block when provided", () => {
|
|
100
|
+
const ds = defineDatasource("test_ds", {
|
|
101
|
+
schema: {
|
|
102
|
+
id: t.string(),
|
|
103
|
+
pipe_name: t.string(),
|
|
104
|
+
},
|
|
105
|
+
indexes: [
|
|
106
|
+
{ name: "pipe_name_set", expr: "pipe_name", type: "set(100)", granularity: 1 },
|
|
107
|
+
{ name: "id_bf", expr: "lower(id)", type: "bloom_filter(0.001)", granularity: 4 },
|
|
108
|
+
],
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
const result = generateDatasource(ds);
|
|
112
|
+
expect(result.content).toContain("INDEXES >");
|
|
113
|
+
expect(result.content).toContain(
|
|
114
|
+
"pipe_name_set pipe_name TYPE set(100) GRANULARITY 1"
|
|
115
|
+
);
|
|
116
|
+
expect(result.content).toContain(
|
|
117
|
+
"id_bf lower(id) TYPE bloom_filter(0.001) GRANULARITY 4"
|
|
118
|
+
);
|
|
119
|
+
});
|
|
98
120
|
});
|
|
99
121
|
|
|
100
122
|
describe('Column formatting', () => {
|
|
@@ -529,6 +551,35 @@ describe('Datasource Generator', () => {
|
|
|
529
551
|
});
|
|
530
552
|
});
|
|
531
553
|
|
|
554
|
+
describe('GCS configuration', () => {
|
|
555
|
+
it('includes GCS import directives', () => {
|
|
556
|
+
const gcsConn = defineGCSConnection('my_gcs', {
|
|
557
|
+
serviceAccountCredentialsJson: '{{ tb_secret("GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON") }}',
|
|
558
|
+
});
|
|
559
|
+
|
|
560
|
+
const ds = defineDatasource('gcs_events', {
|
|
561
|
+
schema: {
|
|
562
|
+
timestamp: t.dateTime(),
|
|
563
|
+
event: t.string(),
|
|
564
|
+
},
|
|
565
|
+
engine: engine.mergeTree({ sortingKey: ['timestamp'] }),
|
|
566
|
+
gcs: {
|
|
567
|
+
connection: gcsConn,
|
|
568
|
+
bucketUri: 'gs://my-bucket/events/*.csv',
|
|
569
|
+
schedule: '@auto',
|
|
570
|
+
fromTimestamp: '2024-01-01T00:00:00Z',
|
|
571
|
+
},
|
|
572
|
+
});
|
|
573
|
+
|
|
574
|
+
const result = generateDatasource(ds);
|
|
575
|
+
|
|
576
|
+
expect(result.content).toContain('IMPORT_CONNECTION_NAME my_gcs');
|
|
577
|
+
expect(result.content).toContain('IMPORT_BUCKET_URI gs://my-bucket/events/*.csv');
|
|
578
|
+
expect(result.content).toContain('IMPORT_SCHEDULE @auto');
|
|
579
|
+
expect(result.content).toContain('IMPORT_FROM_TIMESTAMP 2024-01-01T00:00:00Z');
|
|
580
|
+
});
|
|
581
|
+
});
|
|
582
|
+
|
|
532
583
|
describe('Token generation', () => {
|
|
533
584
|
it('generates TOKEN lines with inline config', () => {
|
|
534
585
|
const ds = defineDatasource('test_ds', {
|
|
@@ -9,7 +9,9 @@ import type {
|
|
|
9
9
|
ColumnDefinition,
|
|
10
10
|
KafkaConfig,
|
|
11
11
|
S3Config,
|
|
12
|
+
GCSConfig,
|
|
12
13
|
TokenConfig,
|
|
14
|
+
DatasourceIndex,
|
|
13
15
|
} from "../schema/datasource.js";
|
|
14
16
|
import type { AnyTypeValidator, TypeModifiers } from "../schema/types.js";
|
|
15
17
|
import { getColumnType, getColumnJsonPath } from "../schema/datasource.js";
|
|
@@ -176,18 +178,18 @@ function generateKafkaConfig(kafka: KafkaConfig): string {
|
|
|
176
178
|
/**
|
|
177
179
|
* Generate S3 import configuration lines
|
|
178
180
|
*/
|
|
179
|
-
function
|
|
181
|
+
function generateImportConfig(importConfig: S3Config | GCSConfig): string {
|
|
180
182
|
const parts: string[] = [];
|
|
181
183
|
|
|
182
|
-
parts.push(`IMPORT_CONNECTION_NAME ${
|
|
183
|
-
parts.push(`IMPORT_BUCKET_URI ${
|
|
184
|
+
parts.push(`IMPORT_CONNECTION_NAME ${importConfig.connection._name}`);
|
|
185
|
+
parts.push(`IMPORT_BUCKET_URI ${importConfig.bucketUri}`);
|
|
184
186
|
|
|
185
|
-
if (
|
|
186
|
-
parts.push(`IMPORT_SCHEDULE ${
|
|
187
|
+
if (importConfig.schedule) {
|
|
188
|
+
parts.push(`IMPORT_SCHEDULE ${importConfig.schedule}`);
|
|
187
189
|
}
|
|
188
190
|
|
|
189
|
-
if (
|
|
190
|
-
parts.push(`IMPORT_FROM_TIMESTAMP ${
|
|
191
|
+
if (importConfig.fromTimestamp) {
|
|
192
|
+
parts.push(`IMPORT_FROM_TIMESTAMP ${importConfig.fromTimestamp}`);
|
|
191
193
|
}
|
|
192
194
|
|
|
193
195
|
return parts.join("\n");
|
|
@@ -210,6 +212,23 @@ function generateForwardQuery(forwardQuery?: string): string | null {
|
|
|
210
212
|
return ["FORWARD_QUERY >", ...lines.map((line) => ` ${line}`)].join("\n");
|
|
211
213
|
}
|
|
212
214
|
|
|
215
|
+
/**
|
|
216
|
+
* Generate INDEXES section
|
|
217
|
+
*/
|
|
218
|
+
function generateIndexes(indexes?: readonly DatasourceIndex[]): string | null {
|
|
219
|
+
if (!indexes || indexes.length === 0) {
|
|
220
|
+
return null;
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
const lines = ["INDEXES >"];
|
|
224
|
+
for (const index of indexes) {
|
|
225
|
+
lines.push(
|
|
226
|
+
` ${index.name} ${index.expr} TYPE ${index.type} GRANULARITY ${index.granularity}`
|
|
227
|
+
);
|
|
228
|
+
}
|
|
229
|
+
return lines.join("\n");
|
|
230
|
+
}
|
|
231
|
+
|
|
213
232
|
/**
|
|
214
233
|
* Generate SHARED_WITH section for sharing datasource with other workspaces
|
|
215
234
|
*/
|
|
@@ -291,8 +310,13 @@ export function generateDatasource(
|
|
|
291
310
|
): GeneratedDatasource {
|
|
292
311
|
const parts: string[] = [];
|
|
293
312
|
|
|
294
|
-
|
|
295
|
-
|
|
313
|
+
const ingestionConfigCount = [
|
|
314
|
+
datasource.options.kafka,
|
|
315
|
+
datasource.options.s3,
|
|
316
|
+
datasource.options.gcs,
|
|
317
|
+
].filter(Boolean).length;
|
|
318
|
+
if (ingestionConfigCount > 1) {
|
|
319
|
+
throw new Error("Datasource can only define one ingestion option: `kafka`, `s3`, or `gcs`.");
|
|
296
320
|
}
|
|
297
321
|
|
|
298
322
|
// Add description if present
|
|
@@ -311,6 +335,13 @@ export function generateDatasource(
|
|
|
311
335
|
// Add engine configuration
|
|
312
336
|
parts.push(generateEngineConfig(datasource.options.engine));
|
|
313
337
|
|
|
338
|
+
// Add indexes if present
|
|
339
|
+
const indexes = generateIndexes(datasource.options.indexes);
|
|
340
|
+
if (indexes) {
|
|
341
|
+
parts.push("");
|
|
342
|
+
parts.push(indexes);
|
|
343
|
+
}
|
|
344
|
+
|
|
314
345
|
// Add Kafka configuration if present
|
|
315
346
|
if (datasource.options.kafka) {
|
|
316
347
|
parts.push("");
|
|
@@ -320,7 +351,13 @@ export function generateDatasource(
|
|
|
320
351
|
// Add S3 configuration if present
|
|
321
352
|
if (datasource.options.s3) {
|
|
322
353
|
parts.push("");
|
|
323
|
-
parts.push(
|
|
354
|
+
parts.push(generateImportConfig(datasource.options.s3));
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
// Add GCS configuration if present
|
|
358
|
+
if (datasource.options.gcs) {
|
|
359
|
+
parts.push("");
|
|
360
|
+
parts.push(generateImportConfig(datasource.options.gcs));
|
|
324
361
|
}
|
|
325
362
|
|
|
326
363
|
// Add forward query if present
|
|
@@ -164,6 +164,27 @@ describe('Pipe Generator', () => {
|
|
|
164
164
|
const result = generatePipe(pipe);
|
|
165
165
|
expect(result.content).toContain(' %\n');
|
|
166
166
|
});
|
|
167
|
+
|
|
168
|
+
it('injects param defaults into placeholders when SQL omits them', () => {
|
|
169
|
+
const pipe = definePipe('defaults_pipe', {
|
|
170
|
+
params: {
|
|
171
|
+
start_date: p.date().optional('2025-03-01'),
|
|
172
|
+
page: p.int32().optional(0),
|
|
173
|
+
},
|
|
174
|
+
nodes: [
|
|
175
|
+
node({
|
|
176
|
+
name: 'endpoint',
|
|
177
|
+
sql: 'SELECT * FROM events WHERE d >= {{Date(start_date)}} LIMIT 10 OFFSET {{Int32(page)}}',
|
|
178
|
+
}),
|
|
179
|
+
],
|
|
180
|
+
output: simpleOutput,
|
|
181
|
+
endpoint: true,
|
|
182
|
+
});
|
|
183
|
+
|
|
184
|
+
const result = generatePipe(pipe);
|
|
185
|
+
expect(result.content).toContain("{{ Date(start_date, '2025-03-01') }}");
|
|
186
|
+
expect(result.content).toContain('{{ Int32(page, 0) }}');
|
|
187
|
+
});
|
|
167
188
|
});
|
|
168
189
|
|
|
169
190
|
describe('Multiple nodes', () => {
|
package/src/generator/pipe.ts
CHANGED
|
@@ -18,6 +18,8 @@ import {
|
|
|
18
18
|
getCopyConfig,
|
|
19
19
|
getSinkConfig,
|
|
20
20
|
} from "../schema/pipe.js";
|
|
21
|
+
import type { AnyParamValidator } from "../schema/params.js";
|
|
22
|
+
import { getParamDefault } from "../schema/params.js";
|
|
21
23
|
|
|
22
24
|
/**
|
|
23
25
|
* Generated pipe content
|
|
@@ -36,10 +38,123 @@ function hasDynamicParameters(sql: string): boolean {
|
|
|
36
38
|
return /\{\{[^}]+\}\}/.test(sql) || /\{%[^%]+%\}/.test(sql);
|
|
37
39
|
}
|
|
38
40
|
|
|
41
|
+
function splitTopLevelComma(input: string): string[] {
|
|
42
|
+
const parts: string[] = [];
|
|
43
|
+
let current = "";
|
|
44
|
+
let depth = 0;
|
|
45
|
+
let inSingleQuote = false;
|
|
46
|
+
let inDoubleQuote = false;
|
|
47
|
+
|
|
48
|
+
for (let i = 0; i < input.length; i += 1) {
|
|
49
|
+
const char = input[i];
|
|
50
|
+
const prev = i > 0 ? input[i - 1] : "";
|
|
51
|
+
|
|
52
|
+
if (char === "'" && !inDoubleQuote && prev !== "\\") {
|
|
53
|
+
inSingleQuote = !inSingleQuote;
|
|
54
|
+
current += char;
|
|
55
|
+
continue;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
if (char === '"' && !inSingleQuote && prev !== "\\") {
|
|
59
|
+
inDoubleQuote = !inDoubleQuote;
|
|
60
|
+
current += char;
|
|
61
|
+
continue;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
if (!inSingleQuote && !inDoubleQuote) {
|
|
65
|
+
if (char === "(") {
|
|
66
|
+
depth += 1;
|
|
67
|
+
current += char;
|
|
68
|
+
continue;
|
|
69
|
+
}
|
|
70
|
+
if (char === ")" && depth > 0) {
|
|
71
|
+
depth -= 1;
|
|
72
|
+
current += char;
|
|
73
|
+
continue;
|
|
74
|
+
}
|
|
75
|
+
if (char === "," && depth === 0) {
|
|
76
|
+
parts.push(current.trim());
|
|
77
|
+
current = "";
|
|
78
|
+
continue;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
current += char;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
if (current.trim()) {
|
|
86
|
+
parts.push(current.trim());
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
return parts;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function toTemplateDefaultLiteral(value: string | number | boolean): string {
|
|
93
|
+
if (typeof value === "string") {
|
|
94
|
+
return `'${value.replace(/\\/g, "\\\\").replace(/'/g, "\\'")}'`;
|
|
95
|
+
}
|
|
96
|
+
return String(value);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
function applyParamDefaultsToSql(
|
|
100
|
+
sql: string,
|
|
101
|
+
params?: Record<string, AnyParamValidator>
|
|
102
|
+
): string {
|
|
103
|
+
if (!params) {
|
|
104
|
+
return sql;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
const defaults = new Map<string, string>();
|
|
108
|
+
for (const [name, validator] of Object.entries(params)) {
|
|
109
|
+
const defaultValue = getParamDefault(validator);
|
|
110
|
+
if (defaultValue !== undefined) {
|
|
111
|
+
defaults.set(name, toTemplateDefaultLiteral(defaultValue as string | number | boolean));
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
if (defaults.size === 0) {
|
|
116
|
+
return sql;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
const placeholderRegex = /\{\{\s*([^{}]+?)\s*\}\}/g;
|
|
120
|
+
return sql.replace(placeholderRegex, (fullMatch, rawExpression) => {
|
|
121
|
+
const expression = String(rawExpression);
|
|
122
|
+
const rewritten = expression.replace(
|
|
123
|
+
/([a-zA-Z_][a-zA-Z0-9_]*)\s*\(([^()]*)\)/g,
|
|
124
|
+
(call, _functionName, rawArgs) => {
|
|
125
|
+
const args = splitTopLevelComma(String(rawArgs));
|
|
126
|
+
if (args.length !== 1) {
|
|
127
|
+
return call;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
const paramName = args[0]?.trim() ?? "";
|
|
131
|
+
if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(paramName)) {
|
|
132
|
+
return call;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
const defaultLiteral = defaults.get(paramName);
|
|
136
|
+
if (!defaultLiteral) {
|
|
137
|
+
return call;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
return call.replace(/\)\s*$/, `, ${defaultLiteral})`);
|
|
141
|
+
}
|
|
142
|
+
);
|
|
143
|
+
|
|
144
|
+
if (rewritten === expression) {
|
|
145
|
+
return fullMatch;
|
|
146
|
+
}
|
|
147
|
+
return `{{ ${rewritten.trim()} }}`;
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
|
|
39
151
|
/**
|
|
40
152
|
* Generate a NODE section for the pipe
|
|
41
153
|
*/
|
|
42
|
-
function generateNode(
|
|
154
|
+
function generateNode(
|
|
155
|
+
node: NodeDefinition,
|
|
156
|
+
params?: Record<string, AnyParamValidator>
|
|
157
|
+
): string {
|
|
43
158
|
const parts: string[] = [];
|
|
44
159
|
|
|
45
160
|
parts.push(`NODE ${node._name}`);
|
|
@@ -57,7 +172,8 @@ function generateNode(node: NodeDefinition): string {
|
|
|
57
172
|
parts.push(` %`);
|
|
58
173
|
}
|
|
59
174
|
|
|
60
|
-
const
|
|
175
|
+
const sqlWithDefaults = applyParamDefaultsToSql(node.sql, params);
|
|
176
|
+
const sqlLines = sqlWithDefaults.trim().split("\n");
|
|
61
177
|
sqlLines.forEach((line) => {
|
|
62
178
|
parts.push(` ${line}`);
|
|
63
179
|
});
|
|
@@ -225,7 +341,7 @@ export function generatePipe(pipe: PipeDefinition): GeneratedPipe {
|
|
|
225
341
|
|
|
226
342
|
// Add all nodes
|
|
227
343
|
pipe.options.nodes.forEach((node, index) => {
|
|
228
|
-
parts.push(generateNode(node));
|
|
344
|
+
parts.push(generateNode(node, pipe.options.params as Record<string, AnyParamValidator> | undefined));
|
|
229
345
|
// Add empty line between nodes
|
|
230
346
|
if (index < pipe.options.nodes.length - 1) {
|
|
231
347
|
parts.push("");
|
package/src/index.ts
CHANGED
|
@@ -113,6 +113,8 @@ export type {
|
|
|
113
113
|
ExtractSchema,
|
|
114
114
|
KafkaConfig,
|
|
115
115
|
S3Config,
|
|
116
|
+
GCSConfig,
|
|
117
|
+
DatasourceIndex,
|
|
116
118
|
} from "./schema/datasource.js";
|
|
117
119
|
|
|
118
120
|
// ============ Connection ============
|
|
@@ -120,9 +122,11 @@ export {
|
|
|
120
122
|
defineKafkaConnection,
|
|
121
123
|
createKafkaConnection,
|
|
122
124
|
defineS3Connection,
|
|
125
|
+
defineGCSConnection,
|
|
123
126
|
isConnectionDefinition,
|
|
124
127
|
isKafkaConnectionDefinition,
|
|
125
128
|
isS3ConnectionDefinition,
|
|
129
|
+
isGCSConnectionDefinition,
|
|
126
130
|
getConnectionType,
|
|
127
131
|
} from "./schema/connection.js";
|
|
128
132
|
export type {
|
|
@@ -133,6 +137,8 @@ export type {
|
|
|
133
137
|
KafkaSaslMechanism,
|
|
134
138
|
S3ConnectionDefinition,
|
|
135
139
|
S3ConnectionOptions,
|
|
140
|
+
GCSConnectionDefinition,
|
|
141
|
+
GCSConnectionOptions,
|
|
136
142
|
} from "./schema/connection.js";
|
|
137
143
|
|
|
138
144
|
// ============ Token ============
|
package/src/migrate/emit-ts.ts
CHANGED
|
@@ -4,6 +4,7 @@ import { parseLiteralFromDatafile, toTsLiteral } from "./parser-utils.js";
|
|
|
4
4
|
import type {
|
|
5
5
|
DatasourceModel,
|
|
6
6
|
DatasourceEngineModel,
|
|
7
|
+
GCSConnectionModel,
|
|
7
8
|
KafkaConnectionModel,
|
|
8
9
|
ParsedResource,
|
|
9
10
|
PipeModel,
|
|
@@ -59,11 +60,13 @@ function hasSecretTemplate(resources: ParsedResource[]): boolean {
|
|
|
59
60
|
if (resource.secret) values.push(resource.secret);
|
|
60
61
|
if (resource.sslCaPem) values.push(resource.sslCaPem);
|
|
61
62
|
if (resource.schemaRegistryUrl) values.push(resource.schemaRegistryUrl);
|
|
62
|
-
} else {
|
|
63
|
+
} else if (resource.connectionType === "s3") {
|
|
63
64
|
values.push(resource.region);
|
|
64
65
|
if (resource.arn) values.push(resource.arn);
|
|
65
66
|
if (resource.accessKey) values.push(resource.accessKey);
|
|
66
67
|
if (resource.secret) values.push(resource.secret);
|
|
68
|
+
} else {
|
|
69
|
+
values.push(resource.serviceAccountCredentialsJson);
|
|
67
70
|
}
|
|
68
71
|
continue;
|
|
69
72
|
}
|
|
@@ -80,6 +83,11 @@ function hasSecretTemplate(resources: ParsedResource[]): boolean {
|
|
|
80
83
|
if (resource.s3.schedule) values.push(resource.s3.schedule);
|
|
81
84
|
if (resource.s3.fromTimestamp) values.push(resource.s3.fromTimestamp);
|
|
82
85
|
}
|
|
86
|
+
if (resource.gcs) {
|
|
87
|
+
values.push(resource.gcs.bucketUri);
|
|
88
|
+
if (resource.gcs.schedule) values.push(resource.gcs.schedule);
|
|
89
|
+
if (resource.gcs.fromTimestamp) values.push(resource.gcs.fromTimestamp);
|
|
90
|
+
}
|
|
83
91
|
continue;
|
|
84
92
|
}
|
|
85
93
|
}
|
|
@@ -125,6 +133,8 @@ function strictParamBaseValidator(type: string): string {
|
|
|
125
133
|
const map: Record<string, string> = {
|
|
126
134
|
String: "p.string()",
|
|
127
135
|
UUID: "p.uuid()",
|
|
136
|
+
Int: "p.int32()",
|
|
137
|
+
Integer: "p.int32()",
|
|
128
138
|
Int8: "p.int8()",
|
|
129
139
|
Int16: "p.int16()",
|
|
130
140
|
Int32: "p.int32()",
|
|
@@ -141,6 +151,8 @@ function strictParamBaseValidator(type: string): string {
|
|
|
141
151
|
DateTime: "p.dateTime()",
|
|
142
152
|
DateTime64: "p.dateTime64()",
|
|
143
153
|
Array: "p.array(p.string())",
|
|
154
|
+
column: "p.column()",
|
|
155
|
+
JSON: "p.json()",
|
|
144
156
|
};
|
|
145
157
|
const validator = map[type];
|
|
146
158
|
if (!validator) {
|
|
@@ -305,6 +317,15 @@ function emitDatasource(ds: DatasourceModel): string {
|
|
|
305
317
|
if (ds.engine) {
|
|
306
318
|
lines.push(` engine: ${emitEngineOptions(ds.engine)},`);
|
|
307
319
|
}
|
|
320
|
+
if (ds.indexes.length > 0) {
|
|
321
|
+
lines.push(" indexes: [");
|
|
322
|
+
for (const index of ds.indexes) {
|
|
323
|
+
lines.push(
|
|
324
|
+
` { name: ${escapeString(index.name)}, expr: ${escapeString(index.expr)}, type: ${escapeString(index.type)}, granularity: ${index.granularity} },`
|
|
325
|
+
);
|
|
326
|
+
}
|
|
327
|
+
lines.push(" ],");
|
|
328
|
+
}
|
|
308
329
|
|
|
309
330
|
if (ds.kafka) {
|
|
310
331
|
const connectionVar = toCamelCase(ds.kafka.connectionName);
|
|
@@ -337,6 +358,20 @@ function emitDatasource(ds: DatasourceModel): string {
|
|
|
337
358
|
lines.push(" },");
|
|
338
359
|
}
|
|
339
360
|
|
|
361
|
+
if (ds.gcs) {
|
|
362
|
+
const connectionVar = toCamelCase(ds.gcs.connectionName);
|
|
363
|
+
lines.push(" gcs: {");
|
|
364
|
+
lines.push(` connection: ${connectionVar},`);
|
|
365
|
+
lines.push(` bucketUri: ${emitStringOrSecret(ds.gcs.bucketUri)},`);
|
|
366
|
+
if (ds.gcs.schedule) {
|
|
367
|
+
lines.push(` schedule: ${emitStringOrSecret(ds.gcs.schedule)},`);
|
|
368
|
+
}
|
|
369
|
+
if (ds.gcs.fromTimestamp) {
|
|
370
|
+
lines.push(` fromTimestamp: ${emitStringOrSecret(ds.gcs.fromTimestamp)},`);
|
|
371
|
+
}
|
|
372
|
+
lines.push(" },");
|
|
373
|
+
}
|
|
374
|
+
|
|
340
375
|
if (ds.forwardQuery) {
|
|
341
376
|
lines.push(" forwardQuery: `");
|
|
342
377
|
lines.push(ds.forwardQuery.replace(/`/g, "\\`").replace(/\${/g, "\\${"));
|
|
@@ -364,7 +399,9 @@ function emitDatasource(ds: DatasourceModel): string {
|
|
|
364
399
|
return lines.join("\n");
|
|
365
400
|
}
|
|
366
401
|
|
|
367
|
-
function emitConnection(
|
|
402
|
+
function emitConnection(
|
|
403
|
+
connection: KafkaConnectionModel | S3ConnectionModel | GCSConnectionModel
|
|
404
|
+
): string {
|
|
368
405
|
const variableName = toCamelCase(connection.name);
|
|
369
406
|
const lines: string[] = [];
|
|
370
407
|
|
|
@@ -396,19 +433,31 @@ function emitConnection(connection: KafkaConnectionModel | S3ConnectionModel): s
|
|
|
396
433
|
return lines.join("\n");
|
|
397
434
|
}
|
|
398
435
|
|
|
436
|
+
if (connection.connectionType === "s3") {
|
|
437
|
+
lines.push(
|
|
438
|
+
`export const ${variableName} = defineS3Connection(${escapeString(connection.name)}, {`
|
|
439
|
+
);
|
|
440
|
+
lines.push(` region: ${emitStringOrSecret(connection.region)},`);
|
|
441
|
+
if (connection.arn) {
|
|
442
|
+
lines.push(` arn: ${emitStringOrSecret(connection.arn)},`);
|
|
443
|
+
}
|
|
444
|
+
if (connection.accessKey) {
|
|
445
|
+
lines.push(` accessKey: ${emitStringOrSecret(connection.accessKey)},`);
|
|
446
|
+
}
|
|
447
|
+
if (connection.secret) {
|
|
448
|
+
lines.push(` secret: ${emitStringOrSecret(connection.secret)},`);
|
|
449
|
+
}
|
|
450
|
+
lines.push("});");
|
|
451
|
+
lines.push("");
|
|
452
|
+
return lines.join("\n");
|
|
453
|
+
}
|
|
454
|
+
|
|
399
455
|
lines.push(
|
|
400
|
-
`export const ${variableName} =
|
|
456
|
+
`export const ${variableName} = defineGCSConnection(${escapeString(connection.name)}, {`
|
|
457
|
+
);
|
|
458
|
+
lines.push(
|
|
459
|
+
` serviceAccountCredentialsJson: ${emitStringOrSecret(connection.serviceAccountCredentialsJson)},`
|
|
401
460
|
);
|
|
402
|
-
lines.push(` region: ${emitStringOrSecret(connection.region)},`);
|
|
403
|
-
if (connection.arn) {
|
|
404
|
-
lines.push(` arn: ${emitStringOrSecret(connection.arn)},`);
|
|
405
|
-
}
|
|
406
|
-
if (connection.accessKey) {
|
|
407
|
-
lines.push(` accessKey: ${emitStringOrSecret(connection.accessKey)},`);
|
|
408
|
-
}
|
|
409
|
-
if (connection.secret) {
|
|
410
|
-
lines.push(` secret: ${emitStringOrSecret(connection.secret)},`);
|
|
411
|
-
}
|
|
412
461
|
lines.push("});");
|
|
413
462
|
lines.push("");
|
|
414
463
|
return lines.join("\n");
|
|
@@ -542,7 +591,7 @@ function emitPipe(pipe: PipeModel): string {
|
|
|
542
591
|
|
|
543
592
|
export function emitMigrationFileContent(resources: ParsedResource[]): string {
|
|
544
593
|
const connections = resources.filter(
|
|
545
|
-
(resource): resource is KafkaConnectionModel | S3ConnectionModel =>
|
|
594
|
+
(resource): resource is KafkaConnectionModel | S3ConnectionModel | GCSConnectionModel =>
|
|
546
595
|
resource.kind === "connection"
|
|
547
596
|
);
|
|
548
597
|
const datasources = resources.filter(
|
|
@@ -569,6 +618,9 @@ export function emitMigrationFileContent(resources: ParsedResource[]): string {
|
|
|
569
618
|
if (connections.some((connection) => connection.connectionType === "s3")) {
|
|
570
619
|
imports.add("defineS3Connection");
|
|
571
620
|
}
|
|
621
|
+
if (connections.some((connection) => connection.connectionType === "gcs")) {
|
|
622
|
+
imports.add("defineGCSConnection");
|
|
623
|
+
}
|
|
572
624
|
if (needsParams) {
|
|
573
625
|
imports.add("p");
|
|
574
626
|
}
|
|
@@ -585,6 +637,7 @@ export function emitMigrationFileContent(resources: ParsedResource[]): string {
|
|
|
585
637
|
const orderedImports = [
|
|
586
638
|
"defineKafkaConnection",
|
|
587
639
|
"defineS3Connection",
|
|
640
|
+
"defineGCSConnection",
|
|
588
641
|
"defineDatasource",
|
|
589
642
|
"definePipe",
|
|
590
643
|
"defineMaterializedView",
|