@tinybirdco/sdk 0.0.44 → 0.0.45
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +53 -0
- package/dist/cli/commands/migrate.d.ts.map +1 -1
- package/dist/cli/commands/migrate.js +4 -3
- package/dist/cli/commands/migrate.js.map +1 -1
- package/dist/cli/commands/migrate.test.js +42 -4
- package/dist/cli/commands/migrate.test.js.map +1 -1
- package/dist/generator/connection.d.ts +1 -1
- package/dist/generator/connection.d.ts.map +1 -1
- package/dist/generator/connection.js +25 -2
- package/dist/generator/connection.js.map +1 -1
- package/dist/generator/connection.test.js +37 -14
- package/dist/generator/connection.test.js.map +1 -1
- package/dist/generator/datasource.d.ts.map +1 -1
- package/dist/generator/datasource.js +23 -0
- package/dist/generator/datasource.js.map +1 -1
- package/dist/generator/datasource.test.js +49 -5
- package/dist/generator/datasource.test.js.map +1 -1
- package/dist/index.d.ts +3 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/migrate/emit-ts.d.ts.map +1 -1
- package/dist/migrate/emit-ts.js +69 -13
- package/dist/migrate/emit-ts.js.map +1 -1
- package/dist/migrate/parse-connection.d.ts +2 -2
- package/dist/migrate/parse-connection.d.ts.map +1 -1
- package/dist/migrate/parse-connection.js +61 -18
- package/dist/migrate/parse-connection.js.map +1 -1
- package/dist/migrate/parse-datasource.d.ts.map +1 -1
- package/dist/migrate/parse-datasource.js +31 -0
- package/dist/migrate/parse-datasource.js.map +1 -1
- package/dist/migrate/types.d.ts +18 -1
- package/dist/migrate/types.d.ts.map +1 -1
- package/dist/schema/connection.d.ts +49 -6
- package/dist/schema/connection.d.ts.map +1 -1
- package/dist/schema/connection.js +44 -9
- package/dist/schema/connection.js.map +1 -1
- package/dist/schema/connection.test.js +72 -17
- package/dist/schema/connection.test.js.map +1 -1
- package/dist/schema/datasource.d.ts +16 -1
- package/dist/schema/datasource.d.ts.map +1 -1
- package/dist/schema/datasource.js +3 -0
- package/dist/schema/datasource.js.map +1 -1
- package/dist/schema/datasource.test.js +21 -0
- package/dist/schema/datasource.test.js.map +1 -1
- package/package.json +1 -1
- package/src/cli/commands/migrate.test.ts +58 -4
- package/src/cli/commands/migrate.ts +6 -4
- package/src/generator/connection.test.ts +45 -14
- package/src/generator/connection.ts +30 -2
- package/src/generator/datasource.test.ts +57 -5
- package/src/generator/datasource.ts +38 -1
- package/src/index.ts +12 -1
- package/src/migrate/emit-ts.ts +80 -16
- package/src/migrate/parse-connection.ts +108 -30
- package/src/migrate/parse-datasource.ts +46 -1
- package/src/migrate/types.ts +24 -2
- package/src/schema/connection.test.ts +92 -17
- package/src/schema/connection.ts +86 -10
- package/src/schema/datasource.test.ts +25 -0
- package/src/schema/datasource.ts +21 -1
package/README.md
CHANGED
|
@@ -526,6 +526,59 @@ In local mode:
|
|
|
526
526
|
|
|
527
527
|
## Defining Resources
|
|
528
528
|
|
|
529
|
+
### Connections
|
|
530
|
+
|
|
531
|
+
```typescript
|
|
532
|
+
import { defineKafkaConnection, defineS3Connection } from "@tinybirdco/sdk";
|
|
533
|
+
|
|
534
|
+
export const eventsKafka = defineKafkaConnection("events_kafka", {
|
|
535
|
+
bootstrapServers: "kafka.example.com:9092",
|
|
536
|
+
securityProtocol: "SASL_SSL",
|
|
537
|
+
saslMechanism: "PLAIN",
|
|
538
|
+
key: '{{ tb_secret("KAFKA_KEY") }}',
|
|
539
|
+
secret: '{{ tb_secret("KAFKA_SECRET") }}',
|
|
540
|
+
});
|
|
541
|
+
|
|
542
|
+
export const landingS3 = defineS3Connection("landing_s3", {
|
|
543
|
+
region: "us-east-1",
|
|
544
|
+
arn: "arn:aws:iam::123456789012:role/tinybird-s3-access",
|
|
545
|
+
});
|
|
546
|
+
```
|
|
547
|
+
|
|
548
|
+
Use connections from datasources:
|
|
549
|
+
|
|
550
|
+
```typescript
|
|
551
|
+
import { defineDatasource, t, engine } from "@tinybirdco/sdk";
|
|
552
|
+
import { eventsKafka, landingS3 } from "./connections";
|
|
553
|
+
|
|
554
|
+
export const kafkaEvents = defineDatasource("kafka_events", {
|
|
555
|
+
schema: {
|
|
556
|
+
timestamp: t.dateTime(),
|
|
557
|
+
payload: t.string(),
|
|
558
|
+
},
|
|
559
|
+
engine: engine.mergeTree({ sortingKey: ["timestamp"] }),
|
|
560
|
+
kafka: {
|
|
561
|
+
connection: eventsKafka,
|
|
562
|
+
topic: "events",
|
|
563
|
+
groupId: "events-consumer",
|
|
564
|
+
autoOffsetReset: "earliest",
|
|
565
|
+
},
|
|
566
|
+
});
|
|
567
|
+
|
|
568
|
+
export const s3Landing = defineDatasource("s3_landing", {
|
|
569
|
+
schema: {
|
|
570
|
+
timestamp: t.dateTime(),
|
|
571
|
+
session_id: t.string(),
|
|
572
|
+
},
|
|
573
|
+
engine: engine.mergeTree({ sortingKey: ["timestamp"] }),
|
|
574
|
+
s3: {
|
|
575
|
+
connection: landingS3,
|
|
576
|
+
bucketUri: "s3://my-bucket/events/*.csv",
|
|
577
|
+
schedule: "@auto",
|
|
578
|
+
},
|
|
579
|
+
});
|
|
580
|
+
```
|
|
581
|
+
|
|
529
582
|
### Datasources
|
|
530
583
|
|
|
531
584
|
```typescript
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"migrate.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/migrate.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAEV,eAAe,EAGhB,MAAM,wBAAwB,CAAC;AAEhC,MAAM,WAAW,qBAAqB;IACpC,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AA2BD,wBAAsB,UAAU,CAC9B,OAAO,EAAE,qBAAqB,GAC7B,OAAO,CAAC,eAAe,CAAC,
|
|
1
|
+
{"version":3,"file":"migrate.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/migrate.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAEV,eAAe,EAGhB,MAAM,wBAAwB,CAAC;AAEhC,MAAM,WAAW,qBAAqB;IACpC,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AA2BD,wBAAsB,UAAU,CAC9B,OAAO,EAAE,qBAAqB,GAC7B,OAAO,CAAC,eAAe,CAAC,CAgM1B"}
|
|
@@ -95,13 +95,14 @@ export async function runMigrate(options) {
|
|
|
95
95
|
}
|
|
96
96
|
}
|
|
97
97
|
for (const datasource of parsedDatasources) {
|
|
98
|
-
|
|
99
|
-
|
|
98
|
+
const referencedConnectionName = datasource.kafka?.connectionName ?? datasource.s3?.connectionName;
|
|
99
|
+
if (referencedConnectionName &&
|
|
100
|
+
!migratedConnectionNames.has(referencedConnectionName)) {
|
|
100
101
|
errors.push({
|
|
101
102
|
filePath: datasource.filePath,
|
|
102
103
|
resourceName: datasource.name,
|
|
103
104
|
resourceKind: datasource.kind,
|
|
104
|
-
message: `Datasource references missing/unmigrated connection "${
|
|
105
|
+
message: `Datasource references missing/unmigrated connection "${referencedConnectionName}".`,
|
|
105
106
|
});
|
|
106
107
|
continue;
|
|
107
108
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"migrate.js","sourceRoot":"","sources":["../../../src/cli/commands/migrate.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,qBAAqB,EAAE,MAAM,4BAA4B,CAAC;AACnE,OAAO,EAAE,wBAAwB,EAAE,2BAA2B,EAAE,MAAM,0BAA0B,CAAC;AACjG,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EAAE,mBAAmB,EAAE,MAAM,+BAA+B,CAAC;AAiBpE,SAAS,gBAAgB,CAAC,QAAsB,EAAE,KAAc;IAC9D,MAAM,OAAO,GAAI,KAAe,CAAC,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC;IAC1D,OAAO;QACL,QAAQ,EAAE,QAAQ,CAAC,QAAQ;QAC3B,YAAY,EAAE,QAAQ,CAAC,IAAI;QAC3B,YAAY,EAAE,QAAQ,CAAC,IAAI;QAC3B,OAAO;KACR,CAAC;AACJ,CAAC;AAED,SAAS,sBAAsB,CAAC,SAA2B;IACzD,MAAM,KAAK,GAA2C;QACpD,UAAU,EAAE,CAAC;QACb,UAAU,EAAE,CAAC;QACb,IAAI,EAAE,CAAC;KACR,CAAC;IACF,OAAO,CAAC,GAAG,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QAClC,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC7C,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;YACjB,OAAO,MAAM,CAAC;QAChB,CAAC;QACD,OAAO,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACtC,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,OAA8B;IAE9B,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;IACzC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC;IACtC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,KAAK,CAAC;IACvC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,KAAK,CAAC;IACrC,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,IAAI,EAAE,CAAC;QACnD,CAAC,CAAE,OAAO,CAAC,GAAc;QACzB,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,GAAG,IAAI,uBAAuB,CAAC,CAAC;IAE9D,MAAM,MAAM,GAAqB,EAAE,CAAC;IAEpC,IAAI,CAAC,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvD,OAAO;YACL,OAAO,EAAE,KAAK;YACd,UAAU;YACV,QAAQ,EAAE,EAAE;YACZ,MAAM,EAAE;gBACN;oBACE,QAAQ,EAAE,GAAG;oBACb,YAAY,EAAE,UAAU;oBACxB,YAAY,EAAE,YAAY;oBAC1B,OAAO,EAAE,4DAA4D;iBACtE;aACF;YACD,MAAM;SACP,CAAC;IACJ,CAAC;IAED,MAAM,UAAU,GAAG,qBAAqB,CAAC,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;IAChE,MAAM,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;IAElC,MAAM,eAAe,GAAqB,EAAE,CAAC;IAC7C,KAAK,MAAM,QAAQ,IAAI,UAAU,CAAC,SAAS,EAAE,CAAC;QAC5C,IAAI,CAAC;YACH,eAAe,CAAC,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC,CAAC;QACpD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,mBAAmB,EAAE,CAAC;gBACzC,MAAM,CAAC,IAAI,CAAC;oBACV,QAAQ,EAAE,KAAK,CAAC,QAAQ;oBACxB,YAAY,EAAE,KAAK,CAAC,YAAY;oBAChC,YAAY,EAAE,KAAK,CAAC,YAAY;oBAChC,OAAO,EAAE,KAAK,CAAC,OAAO;iBACvB,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC;YACjD,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,iBAAiB,GAAG,eAAe,CAAC,MAAM,CAC9C,CAAC,QAAQ,EAA+D,EAAE,CACxE,QAAQ,CAAC,IAAI,KAAK,YAAY,CACjC,CAAC;IACF,MAAM,iBAAiB,GAAG,eAAe,CAAC,MAAM,CAC9C,CAAC,QAAQ,EAA+D,EAAE,CACxE,QAAQ,CAAC,IAAI,KAAK,YAAY,CACjC,CAAC;IACF,MAAM,WAAW,GAAG,eAAe,CAAC,MAAM,CACxC,CAAC,QAAQ,EAAyD,EAAE,CAClE,QAAQ,CAAC,IAAI,KAAK,MAAM,CAC3B,CAAC;IAEF,MAAM,QAAQ,GAAqB,EAAE,CAAC;IACtC,MAAM,uBAAuB,GAAG,IAAI,GAAG,EAAU,CAAC;IAClD,MAAM,uBAAuB,GAAG,IAAI,GAAG,EAAU,CAAC;IAElD,KAAK,MAAM,UAAU,IAAI,iBAAiB,EAAE,CAAC;QAC3C,IAAI,CAAC;YACH,2BAA2B,CAAC,UAAU,CAAC,CAAC;YACxC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC1B,uBAAuB,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QAC/C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,IAAI,CAAC;gBACV,QAAQ,EAAE,UAAU,CAAC,QAAQ;gBAC7B,YAAY,EAAE,UAAU,CAAC,IAAI;gBAC7B,YAAY,EAAE,UAAU,CAAC,IAAI;gBAC7B,OAAO,EAAG,KAAe,CAAC,OAAO;aAClC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,KAAK,MAAM,UAAU,IAAI,iBAAiB,EAAE,CAAC;QAC3C,
|
|
1
|
+
{"version":3,"file":"migrate.js","sourceRoot":"","sources":["../../../src/cli/commands/migrate.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,qBAAqB,EAAE,MAAM,4BAA4B,CAAC;AACnE,OAAO,EAAE,wBAAwB,EAAE,2BAA2B,EAAE,MAAM,0BAA0B,CAAC;AACjG,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EAAE,mBAAmB,EAAE,MAAM,+BAA+B,CAAC;AAiBpE,SAAS,gBAAgB,CAAC,QAAsB,EAAE,KAAc;IAC9D,MAAM,OAAO,GAAI,KAAe,CAAC,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC;IAC1D,OAAO;QACL,QAAQ,EAAE,QAAQ,CAAC,QAAQ;QAC3B,YAAY,EAAE,QAAQ,CAAC,IAAI;QAC3B,YAAY,EAAE,QAAQ,CAAC,IAAI;QAC3B,OAAO;KACR,CAAC;AACJ,CAAC;AAED,SAAS,sBAAsB,CAAC,SAA2B;IACzD,MAAM,KAAK,GAA2C;QACpD,UAAU,EAAE,CAAC;QACb,UAAU,EAAE,CAAC;QACb,IAAI,EAAE,CAAC;KACR,CAAC;IACF,OAAO,CAAC,GAAG,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QAClC,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC7C,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;YACjB,OAAO,MAAM,CAAC;QAChB,CAAC;QACD,OAAO,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACtC,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,OAA8B;IAE9B,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;IACzC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC;IACtC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,KAAK,CAAC;IACvC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,KAAK,CAAC;IACrC,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,IAAI,EAAE,CAAC;QACnD,CAAC,CAAE,OAAO,CAAC,GAAc;QACzB,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,GAAG,IAAI,uBAAuB,CAAC,CAAC;IAE9D,MAAM,MAAM,GAAqB,EAAE,CAAC;IAEpC,IAAI,CAAC,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvD,OAAO;YACL,OAAO,EAAE,KAAK;YACd,UAAU;YACV,QAAQ,EAAE,EAAE;YACZ,MAAM,EAAE;gBACN;oBACE,QAAQ,EAAE,GAAG;oBACb,YAAY,EAAE,UAAU;oBACxB,YAAY,EAAE,YAAY;oBAC1B,OAAO,EAAE,4DAA4D;iBACtE;aACF;YACD,MAAM;SACP,CAAC;IACJ,CAAC;IAED,MAAM,UAAU,GAAG,qBAAqB,CAAC,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;IAChE,MAAM,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;IAElC,MAAM,eAAe,GAAqB,EAAE,CAAC;IAC7C,KAAK,MAAM,QAAQ,IAAI,UAAU,CAAC,SAAS,EAAE,CAAC;QAC5C,IAAI,CAAC;YACH,eAAe,CAAC,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC,CAAC;QACpD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,mBAAmB,EAAE,CAAC;gBACzC,MAAM,CAAC,IAAI,CAAC;oBACV,QAAQ,EAAE,KAAK,CAAC,QAAQ;oBACxB,YAAY,EAAE,KAAK,CAAC,YAAY;oBAChC,YAAY,EAAE,KAAK,CAAC,YAAY;oBAChC,OAAO,EAAE,KAAK,CAAC,OAAO;iBACvB,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC;YACjD,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,iBAAiB,GAAG,eAAe,CAAC,MAAM,CAC9C,CAAC,QAAQ,EAA+D,EAAE,CACxE,QAAQ,CAAC,IAAI,KAAK,YAAY,CACjC,CAAC;IACF,MAAM,iBAAiB,GAAG,eAAe,CAAC,MAAM,CAC9C,CAAC,QAAQ,EAA+D,EAAE,CACxE,QAAQ,CAAC,IAAI,KAAK,YAAY,CACjC,CAAC;IACF,MAAM,WAAW,GAAG,eAAe,CAAC,MAAM,CACxC,CAAC,QAAQ,EAAyD,EAAE,CAClE,QAAQ,CAAC,IAAI,KAAK,MAAM,CAC3B,CAAC;IAEF,MAAM,QAAQ,GAAqB,EAAE,CAAC;IACtC,MAAM,uBAAuB,GAAG,IAAI,GAAG,EAAU,CAAC;IAClD,MAAM,uBAAuB,GAAG,IAAI,GAAG,EAAU,CAAC;IAElD,KAAK,MAAM,UAAU,IAAI,iBAAiB,EAAE,CAAC;QAC3C,IAAI,CAAC;YACH,2BAA2B,CAAC,UAAU,CAAC,CAAC;YACxC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC1B,uBAAuB,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QAC/C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,IAAI,CAAC;gBACV,QAAQ,EAAE,UAAU,CAAC,QAAQ;gBAC7B,YAAY,EAAE,UAAU,CAAC,IAAI;gBAC7B,YAAY,EAAE,UAAU,CAAC,IAAI;gBAC7B,OAAO,EAAG,KAAe,CAAC,OAAO;aAClC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,KAAK,MAAM,UAAU,IAAI,iBAAiB,EAAE,CAAC;QAC3C,MAAM,wBAAwB,GAC5B,UAAU,CAAC,KAAK,EAAE,cAAc,IAAI,UAAU,CAAC,EAAE,EAAE,cAAc,CAAC;QAEpE,IACE,wBAAwB;YACxB,CAAC,uBAAuB,CAAC,GAAG,CAAC,wBAAwB,CAAC,EACtD,CAAC;YACD,MAAM,CAAC,IAAI,CAAC;gBACV,QAAQ,EAAE,UAAU,CAAC,QAAQ;gBAC7B,YAAY,EAAE,UAAU,CAAC,IAAI;gBAC7B,YAAY,EAAE,UAAU,CAAC,IAAI;gBAC7B,OAAO,EAAE,wDAAwD,wBAAwB,IAAI;aAC9F,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QAED,IAAI,CAAC;YACH,2BAA2B,CAAC,UAAU,CAAC,CAAC;YACxC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC1B,uBAAuB,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QAC/C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,IAAI,CAAC;gBACV,QAAQ,EAAE,UAAU,CAAC,QAAQ;gBAC7B,YAAY,EAAE,UAAU,CAAC,IAAI;gBAC7B,YAAY,EAAE,UAAU,CAAC,IAAI;gBAC7B,OAAO,EAAG,KAAe,CAAC,OAAO;aAClC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;QAC/B,IACE,IAAI,CAAC,IAAI,KAAK,cAAc;YAC5B,CAAC,CAAC,IAAI,CAAC,sBAAsB;gBAC3B,CAAC,uBAAuB,CAAC,GAAG,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC,EAC5D,CAAC;YACD,MAAM,CAAC,IAAI,CAAC;gBACV,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,YAAY,EAAE,IAAI,CAAC,IAAI;gBACvB,YAAY,EAAE,IAAI,CAAC,IAAI;gBACvB,OAAO,EAAE,+DAA+D,IAAI,CAAC,sBAAsB,IAAI,QAAQ,IAAI;aACpH,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QAED,IACE,IAAI,CAAC,IAAI,KAAK,MAAM;YACpB,CAAC,CAAC,IAAI,CAAC,oBAAoB;gBACzB,CAAC,uBAAuB,CAAC,GAAG,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,EAC1D,CAAC;YACD,MAAM,CAAC,IAAI,CAAC;gBACV,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,YAAY,EAAE,IAAI,CAAC,IAAI;gBACvB,YAAY,EAAE,IAAI,CAAC,IAAI;gBACvB,OAAO,EAAE,uDAAuD,IAAI,CAAC,oBAAoB,IAAI,QAAQ,IAAI;aAC1G,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QAED,IAAI,CAAC;YACH,2BAA2B,CAAC,IAAI,CAAC,CAAC;YAClC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,IAAI,CAAC;gBACV,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,YAAY,EAAE,IAAI,CAAC,IAAI;gBACvB,YAAY,EAAE,IAAI,CAAC,IAAI;gBACvB,OAAO,EAAG,KAAe,CAAC,OAAO;aAClC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,MAAM,cAAc,GAAG,sBAAsB,CAAC,QAAQ,CAAC,CAAC;IACxD,IAAI,aAAiC,CAAC;IAEtC,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9B,IAAI,CAAC;YACH,aAAa,GAAG,wBAAwB,CAAC,cAAc,CAAC,CAAC;QAC3D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,IAAI,CAAC;gBACV,QAAQ,EAAE,GAAG;gBACb,YAAY,EAAE,QAAQ;gBACtB,YAAY,EAAE,YAAY;gBAC1B,OAAO,EAAE,oCAAqC,KAAe,CAAC,OAAO,EAAE;aACxE,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,IAAI,CAAC,MAAM,IAAI,aAAa,EAAE,CAAC;QAC7B,IAAI,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YACxC,MAAM,CAAC,IAAI,CAAC;gBACV,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,UAAU,CAAC;gBACxC,YAAY,EAAE,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC;gBACvC,YAAY,EAAE,YAAY;gBAC1B,OAAO,EAAE,+BAA+B,UAAU,6BAA6B;aAChF,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,aAAa,CAAC,CAAC;QAC9C,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACpD,OAAO;QACL,OAAO;QACP,UAAU;QACV,QAAQ,EAAE,cAAc;QACxB,MAAM;QACN,MAAM;QACN,aAAa;KACd,CAAC;AACJ,CAAC"}
|
|
@@ -13,11 +13,11 @@ const EXPECTED_COMPLEX_OUTPUT = `/**
|
|
|
13
13
|
* Review endpoint output schemas and any defaults before production use.
|
|
14
14
|
*/
|
|
15
15
|
|
|
16
|
-
import {
|
|
16
|
+
import { defineKafkaConnection, defineDatasource, definePipe, defineMaterializedView, defineCopyPipe, node, t, engine, column, p } from "@tinybirdco/sdk";
|
|
17
17
|
|
|
18
18
|
// Connections
|
|
19
19
|
|
|
20
|
-
export const stream =
|
|
20
|
+
export const stream = defineKafkaConnection("stream", {
|
|
21
21
|
bootstrapServers: "localhost:9092",
|
|
22
22
|
securityProtocol: "SASL_SSL",
|
|
23
23
|
saslMechanism: "PLAIN",
|
|
@@ -180,11 +180,11 @@ const EXPECTED_PARTIAL_OUTPUT = `/**
|
|
|
180
180
|
* Review endpoint output schemas and any defaults before production use.
|
|
181
181
|
*/
|
|
182
182
|
|
|
183
|
-
import {
|
|
183
|
+
import { defineKafkaConnection, defineDatasource, definePipe, defineMaterializedView, defineCopyPipe, node, t, engine, p } from "@tinybirdco/sdk";
|
|
184
184
|
|
|
185
185
|
// Connections
|
|
186
186
|
|
|
187
|
-
export const stream =
|
|
187
|
+
export const stream = defineKafkaConnection("stream", {
|
|
188
188
|
bootstrapServers: "localhost:9092",
|
|
189
189
|
});
|
|
190
190
|
|
|
@@ -469,5 +469,43 @@ TOKEN endpoint_token READ
|
|
|
469
469
|
expect(result.outputContent).toBe(EXPECTED_PARTIAL_OUTPUT);
|
|
470
470
|
expect(fs.existsSync(result.outputPath)).toBe(false);
|
|
471
471
|
});
|
|
472
|
+
it("migrates s3 connection and import datasource directives", async () => {
|
|
473
|
+
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-migrate-"));
|
|
474
|
+
tempDirs.push(tempDir);
|
|
475
|
+
writeFile(tempDir, "s3sample.connection", `TYPE s3
|
|
476
|
+
S3_REGION "us-east-1"
|
|
477
|
+
S3_ARN "arn:aws:iam::123456789012:role/tinybird-s3-access"
|
|
478
|
+
`);
|
|
479
|
+
writeFile(tempDir, "events_landing.datasource", `SCHEMA >
|
|
480
|
+
timestamp DateTime,
|
|
481
|
+
session_id String
|
|
482
|
+
|
|
483
|
+
ENGINE "MergeTree"
|
|
484
|
+
ENGINE_SORTING_KEY "timestamp"
|
|
485
|
+
IMPORT_CONNECTION_NAME s3sample
|
|
486
|
+
IMPORT_BUCKET_URI s3://my-bucket/events/*.csv
|
|
487
|
+
IMPORT_SCHEDULE @auto
|
|
488
|
+
IMPORT_FROM_TIMESTAMP 2024-01-01T00:00:00Z
|
|
489
|
+
`);
|
|
490
|
+
const result = await runMigrate({
|
|
491
|
+
cwd: tempDir,
|
|
492
|
+
patterns: ["."],
|
|
493
|
+
strict: true,
|
|
494
|
+
});
|
|
495
|
+
expect(result.success).toBe(true);
|
|
496
|
+
expect(result.errors).toHaveLength(0);
|
|
497
|
+
expect(result.migrated.filter((resource) => resource.kind === "connection")).toHaveLength(1);
|
|
498
|
+
expect(result.migrated.filter((resource) => resource.kind === "datasource")).toHaveLength(1);
|
|
499
|
+
const output = fs.readFileSync(result.outputPath, "utf-8");
|
|
500
|
+
expect(output).toContain("defineS3Connection");
|
|
501
|
+
expect(output).toContain('export const s3sample = defineS3Connection("s3sample", {');
|
|
502
|
+
expect(output).toContain('region: "us-east-1"');
|
|
503
|
+
expect(output).toContain('arn: "arn:aws:iam::123456789012:role/tinybird-s3-access"');
|
|
504
|
+
expect(output).toContain("s3: {");
|
|
505
|
+
expect(output).toContain("connection: s3sample");
|
|
506
|
+
expect(output).toContain('bucketUri: "s3://my-bucket/events/*.csv"');
|
|
507
|
+
expect(output).toContain('schedule: "@auto"');
|
|
508
|
+
expect(output).toContain('fromTimestamp: "2024-01-01T00:00:00Z"');
|
|
509
|
+
});
|
|
472
510
|
});
|
|
473
511
|
//# sourceMappingURL=migrate.test.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"migrate.test.js","sourceRoot":"","sources":["../../../src/cli/commands/migrate.test.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AACzD,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAE1C,SAAS,SAAS,CAAC,GAAW,EAAE,YAAoB,EAAE,OAAe;IACnE,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;IAC9C,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1D,EAAE,CAAC,aAAa,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;AACtC,CAAC;AAED,MAAM,uBAAuB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAsK/B,CAAC;AAEF,MAAM,uBAAuB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA6D/B,CAAC;AAEF,QAAQ,CAAC,YAAY,EAAE,GAAG,EAAE;IAC1B,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,SAAS,CAAC,GAAG,EAAE;QACb,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;YAC3B,IAAI,CAAC;gBACH,EAAE,CAAC,MAAM,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACtC,CAAC;YAAC,MAAM,CAAC;gBACP,0BAA0B;YAC5B,CAAC;QACH,CAAC;QACD,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;IACtB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6EAA6E,EAAE,KAAK,IAAI,EAAE;QAC3F,MAAM,OAAO,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,mBAAmB,CAAC,CAAC,CAAC;QAC5E,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAEvB,SAAS,CACP,OAAO,EACP,mBAAmB,EACnB;;;;;;;CAOL,CACI,CAAC;QAEF,SAAS,CACP,OAAO,EACP,mBAAmB,EACnB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA6BL,CACI,CAAC;QAEF,SAAS,CACP,OAAO,EACP,0BAA0B,EAC1B;;;;;;;CAOL,CACI,CAAC;QAEF,SAAS,CACP,OAAO,EACP,sBAAsB,EACtB;;;;;;;;;;;;;;;;;;CAkBL,CACI,CAAC;QAEF,SAAS,CACP,OAAO,EACP,gBAAgB,EAChB;;;;;;;;;;;CAWL,CACI,CAAC;QAEF,SAAS,CACP,OAAO,EACP,kBAAkB,EAClB;;;;;;;;;CASL,CACI,CAAC;QAEF,SAAS,CACP,OAAO,EACP,iBAAiB,EACjB;;;;;;;;;;;CAWL,CACI,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC;YAC9B,GAAG,EAAE,OAAO;YACZ,QAAQ,EAAE,CAAC,GAAG,CAAC;YACf,MAAM,EAAE,IAAI;SACb,CAAC,CAAC;QAEH,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACtC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACxC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,KAAK,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC7F,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,KAAK,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC7F,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACvF,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;QACvE,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEpD,MAAM,MAAM,GAAG,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QAC3D,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wFAAwF,EAAE,KAAK,IAAI,EAAE;QACtG,MAAM,OAAO,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,mBAAmB,CAAC,CAAC,CAAC;QAC5E,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAEvB,SAAS,CACP,OAAO,EACP,mBAAmB,EACnB;;CAEL,CACI,CAAC;QAEF,SAAS,CACP,OAAO,EACP,mBAAmB,EACnB;;;;;;;;;CASL,CACI,CAAC;QAEF,SAAS,CACP,OAAO,EACP,sBAAsB,EACtB;;;;;;;;;;;CAWL,CACI,CAAC;QAEF,SAAS,CACP,OAAO,EACP,gBAAgB,EAChB;;;;;;;CAOL,CACI,CAAC;QAEF,SAAS,CACP,OAAO,EACP,aAAa,EACb;;;;;;CAML,CACI,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC;YAC9B,GAAG,EAAE,OAAO;YACZ,QAAQ,EAAE,CAAC,GAAG,CAAC;YACf,MAAM,EAAE,IAAI;SACb,CAAC,CAAC;QAEH,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACtC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CACzD,MAAM,CAAC,eAAe,CAAC;YACrB,yEAAyE;YACzE,0EAA0E;SAC3E,CAAC,CACH,CAAC;QACF,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,KAAK,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC7F,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,KAAK,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC7F,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACvF,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEpD,MAAM,MAAM,GAAG,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QAC3D,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+EAA+E,EAAE,KAAK,IAAI,EAAE;QAC7F,MAAM,OAAO,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,mBAAmB,CAAC,CAAC,CAAC;QAC5E,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAEvB,SAAS,CACP,OAAO,EACP,mBAAmB,EACnB;;CAEL,CACI,CAAC;QAEF,SAAS,CACP,OAAO,EACP,mBAAmB,EACnB;;;;;;;;;CASL,CACI,CAAC;QAEF,SAAS,CACP,OAAO,EACP,sBAAsB,EACtB;;;;;;;;;;;CAWL,CACI,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC;YAC9B,GAAG,EAAE,OAAO;YACZ,QAAQ,EAAE,CAAC,GAAG,CAAC;YACf,MAAM,EAAE,IAAI;YACZ,MAAM,EAAE,IAAI;SACb,CAAC,CAAC;QAEH,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACtC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACxC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;QAC3D,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACvD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
1
|
+
{"version":3,"file":"migrate.test.js","sourceRoot":"","sources":["../../../src/cli/commands/migrate.test.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AACzD,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAE1C,SAAS,SAAS,CAAC,GAAW,EAAE,YAAoB,EAAE,OAAe;IACnE,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;IAC9C,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1D,EAAE,CAAC,aAAa,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;AACtC,CAAC;AAED,MAAM,uBAAuB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAsK/B,CAAC;AAEF,MAAM,uBAAuB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA6D/B,CAAC;AAEF,QAAQ,CAAC,YAAY,EAAE,GAAG,EAAE;IAC1B,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,SAAS,CAAC,GAAG,EAAE;QACb,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;YAC3B,IAAI,CAAC;gBACH,EAAE,CAAC,MAAM,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACtC,CAAC;YAAC,MAAM,CAAC;gBACP,0BAA0B;YAC5B,CAAC;QACH,CAAC;QACD,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;IACtB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6EAA6E,EAAE,KAAK,IAAI,EAAE;QAC3F,MAAM,OAAO,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,mBAAmB,CAAC,CAAC,CAAC;QAC5E,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAEvB,SAAS,CACP,OAAO,EACP,mBAAmB,EACnB;;;;;;;CAOL,CACI,CAAC;QAEF,SAAS,CACP,OAAO,EACP,mBAAmB,EACnB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA6BL,CACI,CAAC;QAEF,SAAS,CACP,OAAO,EACP,0BAA0B,EAC1B;;;;;;;CAOL,CACI,CAAC;QAEF,SAAS,CACP,OAAO,EACP,sBAAsB,EACtB;;;;;;;;;;;;;;;;;;CAkBL,CACI,CAAC;QAEF,SAAS,CACP,OAAO,EACP,gBAAgB,EAChB;;;;;;;;;;;CAWL,CACI,CAAC;QAEF,SAAS,CACP,OAAO,EACP,kBAAkB,EAClB;;;;;;;;;CASL,CACI,CAAC;QAEF,SAAS,CACP,OAAO,EACP,iBAAiB,EACjB;;;;;;;;;;;CAWL,CACI,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC;YAC9B,GAAG,EAAE,OAAO;YACZ,QAAQ,EAAE,CAAC,GAAG,CAAC;YACf,MAAM,EAAE,IAAI;SACb,CAAC,CAAC;QAEH,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACtC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACxC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,KAAK,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC7F,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,KAAK,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC7F,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACvF,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;QACvE,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEpD,MAAM,MAAM,GAAG,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QAC3D,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wFAAwF,EAAE,KAAK,IAAI,EAAE;QACtG,MAAM,OAAO,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,mBAAmB,CAAC,CAAC,CAAC;QAC5E,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAEvB,SAAS,CACP,OAAO,EACP,mBAAmB,EACnB;;CAEL,CACI,CAAC;QAEF,SAAS,CACP,OAAO,EACP,mBAAmB,EACnB;;;;;;;;;CASL,CACI,CAAC;QAEF,SAAS,CACP,OAAO,EACP,sBAAsB,EACtB;;;;;;;;;;;CAWL,CACI,CAAC;QAEF,SAAS,CACP,OAAO,EACP,gBAAgB,EAChB;;;;;;;CAOL,CACI,CAAC;QAEF,SAAS,CACP,OAAO,EACP,aAAa,EACb;;;;;;CAML,CACI,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC;YAC9B,GAAG,EAAE,OAAO;YACZ,QAAQ,EAAE,CAAC,GAAG,CAAC;YACf,MAAM,EAAE,IAAI;SACb,CAAC,CAAC;QAEH,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACtC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CACzD,MAAM,CAAC,eAAe,CAAC;YACrB,yEAAyE;YACzE,0EAA0E;SAC3E,CAAC,CACH,CAAC;QACF,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,KAAK,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC7F,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,KAAK,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC7F,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACvF,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEpD,MAAM,MAAM,GAAG,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QAC3D,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+EAA+E,EAAE,KAAK,IAAI,EAAE;QAC7F,MAAM,OAAO,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,mBAAmB,CAAC,CAAC,CAAC;QAC5E,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAEvB,SAAS,CACP,OAAO,EACP,mBAAmB,EACnB;;CAEL,CACI,CAAC;QAEF,SAAS,CACP,OAAO,EACP,mBAAmB,EACnB;;;;;;;;;CASL,CACI,CAAC;QAEF,SAAS,CACP,OAAO,EACP,sBAAsB,EACtB;;;;;;;;;;;CAWL,CACI,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC;YAC9B,GAAG,EAAE,OAAO;YACZ,QAAQ,EAAE,CAAC,GAAG,CAAC;YACf,MAAM,EAAE,IAAI;YACZ,MAAM,EAAE,IAAI;SACb,CAAC,CAAC;QAEH,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACtC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACxC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;QAC3D,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACvD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yDAAyD,EAAE,KAAK,IAAI,EAAE;QACvE,MAAM,OAAO,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,mBAAmB,CAAC,CAAC,CAAC;QAC5E,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAEvB,SAAS,CACP,OAAO,EACP,qBAAqB,EACrB;;;CAGL,CACI,CAAC;QAEF,SAAS,CACP,OAAO,EACP,2BAA2B,EAC3B;;;;;;;;;;CAUL,CACI,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC;YAC9B,GAAG,EAAE,OAAO;YACZ,QAAQ,EAAE,CAAC,GAAG,CAAC;YACf,MAAM,EAAE,IAAI;SACb,CAAC,CAAC;QAEH,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACtC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,KAAK,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC7F,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,KAAK,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAE7F,MAAM,MAAM,GAAG,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QAC3D,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,oBAAoB,CAAC,CAAC;QAC/C,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,0DAA0D,CAAC,CAAC;QACrF,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,qBAAqB,CAAC,CAAC;QAChD,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CACtB,0DAA0D,CAC3D,CAAC;QACF,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAClC,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,sBAAsB,CAAC,CAAC;QACjD,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,0CAA0C,CAAC,CAAC;QACrE,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,mBAAmB,CAAC,CAAC;QAC9C,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,uCAAuC,CAAC,CAAC;IACpE,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -20,7 +20,7 @@ export interface GeneratedConnection {
|
|
|
20
20
|
*
|
|
21
21
|
* @example
|
|
22
22
|
* ```ts
|
|
23
|
-
* const myKafka =
|
|
23
|
+
* const myKafka = defineKafkaConnection('my_kafka', {
|
|
24
24
|
* bootstrapServers: 'kafka.example.com:9092',
|
|
25
25
|
* securityProtocol: 'SASL_SSL',
|
|
26
26
|
* saslMechanism: 'PLAIN',
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"connection.d.ts","sourceRoot":"","sources":["../../src/generator/connection.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,oBAAoB,EAA6B,MAAM,yBAAyB,CAAC;
|
|
1
|
+
{"version":3,"file":"connection.d.ts","sourceRoot":"","sources":["../../src/generator/connection.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,oBAAoB,EAA6B,MAAM,yBAAyB,CAAC;AAG/F;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,sBAAsB;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,6CAA6C;IAC7C,OAAO,EAAE,MAAM,CAAC;CACjB;AA4DD;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAgB,kBAAkB,CAChC,UAAU,EAAE,oBAAoB,GAC/B,mBAAmB,CAerB;AAED;;;;;GAKG;AACH,wBAAgB,sBAAsB,CACpC,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,oBAAoB,CAAC,GAChD,mBAAmB,EAAE,CAEvB"}
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
* Connection content generator
|
|
3
3
|
* Converts ConnectionDefinition to native .connection file format
|
|
4
4
|
*/
|
|
5
|
+
import { isS3ConnectionDefinition } from "../schema/connection.js";
|
|
5
6
|
/**
|
|
6
7
|
* Generate a Kafka connection content
|
|
7
8
|
*/
|
|
@@ -27,6 +28,25 @@ function generateKafkaConnection(connection) {
|
|
|
27
28
|
}
|
|
28
29
|
return parts.join("\n");
|
|
29
30
|
}
|
|
31
|
+
/**
|
|
32
|
+
* Generate an S3 connection content
|
|
33
|
+
*/
|
|
34
|
+
function generateS3Connection(connection) {
|
|
35
|
+
const parts = [];
|
|
36
|
+
const options = connection.options;
|
|
37
|
+
parts.push("TYPE s3");
|
|
38
|
+
parts.push(`S3_REGION ${options.region}`);
|
|
39
|
+
if (options.arn) {
|
|
40
|
+
parts.push(`S3_ARN ${options.arn}`);
|
|
41
|
+
}
|
|
42
|
+
if (options.accessKey) {
|
|
43
|
+
parts.push(`S3_ACCESS_KEY ${options.accessKey}`);
|
|
44
|
+
}
|
|
45
|
+
if (options.secret) {
|
|
46
|
+
parts.push(`S3_SECRET ${options.secret}`);
|
|
47
|
+
}
|
|
48
|
+
return parts.join("\n");
|
|
49
|
+
}
|
|
30
50
|
/**
|
|
31
51
|
* Generate a .connection file content from a ConnectionDefinition
|
|
32
52
|
*
|
|
@@ -35,7 +55,7 @@ function generateKafkaConnection(connection) {
|
|
|
35
55
|
*
|
|
36
56
|
* @example
|
|
37
57
|
* ```ts
|
|
38
|
-
* const myKafka =
|
|
58
|
+
* const myKafka = defineKafkaConnection('my_kafka', {
|
|
39
59
|
* bootstrapServers: 'kafka.example.com:9092',
|
|
40
60
|
* securityProtocol: 'SASL_SSL',
|
|
41
61
|
* saslMechanism: 'PLAIN',
|
|
@@ -58,8 +78,11 @@ export function generateConnection(connection) {
|
|
|
58
78
|
if (connection._connectionType === "kafka") {
|
|
59
79
|
content = generateKafkaConnection(connection);
|
|
60
80
|
}
|
|
81
|
+
else if (isS3ConnectionDefinition(connection)) {
|
|
82
|
+
content = generateS3Connection(connection);
|
|
83
|
+
}
|
|
61
84
|
else {
|
|
62
|
-
throw new Error(
|
|
85
|
+
throw new Error("Unsupported connection type.");
|
|
63
86
|
}
|
|
64
87
|
return {
|
|
65
88
|
name: connection._name,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"connection.js","sourceRoot":"","sources":["../../src/generator/connection.ts"],"names":[],"mappings":"AAAA;;;GAGG;
|
|
1
|
+
{"version":3,"file":"connection.js","sourceRoot":"","sources":["../../src/generator/connection.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,wBAAwB,EAA+B,MAAM,yBAAyB,CAAC;AAYhG;;GAEG;AACH,SAAS,uBAAuB,CAAC,UAAqC;IACpE,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,OAAO,GAAG,UAAU,CAAC,OAAO,CAAC;IAEnC,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IACzB,KAAK,CAAC,IAAI,CAAC,2BAA2B,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC;IAElE,IAAI,OAAO,CAAC,gBAAgB,EAAE,CAAC;QAC7B,KAAK,CAAC,IAAI,CAAC,2BAA2B,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC;IACpE,CAAC;IAED,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;QAC1B,KAAK,CAAC,IAAI,CAAC,wBAAwB,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC;IAC9D,CAAC;IAED,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;QAChB,KAAK,CAAC,IAAI,CAAC,aAAa,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IACzC,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,KAAK,CAAC,IAAI,CAAC,gBAAgB,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAC/C,CAAC;IAED,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACrB,KAAK,CAAC,IAAI,CAAC,oBAAoB,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;IACrD,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,SAAS,oBAAoB,CAAC,UAAkC;IAC9D,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,OAAO,GAAG,UAAU,CAAC,OAAO,CAAC;IAEnC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACtB,KAAK,CAAC,IAAI,CAAC,aAAa,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAE1C,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;QAChB,KAAK,CAAC,IAAI,CAAC,UAAU,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IACtC,CAAC;IAED,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;QACtB,KAAK,CAAC,IAAI,CAAC,iBAAiB,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;IACnD,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,KAAK,CAAC,IAAI,CAAC,aAAa,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAC5C,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,UAAU,kBAAkB,CAChC,UAAgC;IAEhC,IAAI,OAAe,CAAC;IAEpB,IAAI,UAAU,CAAC,eAAe,KAAK,OAAO,EAAE,CAAC;QAC3C,OAAO,GAAG,uBAAuB,CAAC,UAAuC,CAAC,CAAC;IAC7E,CAAC;SAAM,IAAI,wBAAwB,CAAC,UAAU,CAAC,EAAE,CAAC;QAChD,OAAO,GAAG,oBAAoB,CAAC,UAAU,CAAC,CAAC;IAC7C,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;IAClD,CAAC;IAED,OAAO;QACL,IAAI,EAAE,UAAU,CAAC,KAAK;QACtB,OAAO;KACR,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,sBAAsB,CACpC,WAAiD;IAEjD,OAAO,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;AAC5D,CAAC"}
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { describe, it, expect } from "vitest";
|
|
2
2
|
import { generateConnection, generateAllConnections } from "./connection.js";
|
|
3
|
-
import {
|
|
3
|
+
import { defineKafkaConnection, defineS3Connection } from "../schema/connection.js";
|
|
4
4
|
describe("Connection Generator", () => {
|
|
5
5
|
describe("generateConnection", () => {
|
|
6
6
|
it("generates basic Kafka connection with required fields", () => {
|
|
7
|
-
const conn =
|
|
7
|
+
const conn = defineKafkaConnection("my_kafka", {
|
|
8
8
|
bootstrapServers: "kafka.example.com:9092",
|
|
9
9
|
});
|
|
10
10
|
const result = generateConnection(conn);
|
|
@@ -13,7 +13,7 @@ describe("Connection Generator", () => {
|
|
|
13
13
|
expect(result.content).toContain("KAFKA_BOOTSTRAP_SERVERS kafka.example.com:9092");
|
|
14
14
|
});
|
|
15
15
|
it("includes security protocol when provided", () => {
|
|
16
|
-
const conn =
|
|
16
|
+
const conn = defineKafkaConnection("my_kafka", {
|
|
17
17
|
bootstrapServers: "kafka.example.com:9092",
|
|
18
18
|
securityProtocol: "SASL_SSL",
|
|
19
19
|
});
|
|
@@ -21,7 +21,7 @@ describe("Connection Generator", () => {
|
|
|
21
21
|
expect(result.content).toContain("KAFKA_SECURITY_PROTOCOL SASL_SSL");
|
|
22
22
|
});
|
|
23
23
|
it("includes SASL mechanism when provided", () => {
|
|
24
|
-
const conn =
|
|
24
|
+
const conn = defineKafkaConnection("my_kafka", {
|
|
25
25
|
bootstrapServers: "kafka.example.com:9092",
|
|
26
26
|
saslMechanism: "PLAIN",
|
|
27
27
|
});
|
|
@@ -29,7 +29,7 @@ describe("Connection Generator", () => {
|
|
|
29
29
|
expect(result.content).toContain("KAFKA_SASL_MECHANISM PLAIN");
|
|
30
30
|
});
|
|
31
31
|
it("includes key and secret when provided", () => {
|
|
32
|
-
const conn =
|
|
32
|
+
const conn = defineKafkaConnection("my_kafka", {
|
|
33
33
|
bootstrapServers: "kafka.example.com:9092",
|
|
34
34
|
key: '{{ tb_secret("KAFKA_KEY") }}',
|
|
35
35
|
secret: '{{ tb_secret("KAFKA_SECRET") }}',
|
|
@@ -39,7 +39,7 @@ describe("Connection Generator", () => {
|
|
|
39
39
|
expect(result.content).toContain('KAFKA_SECRET {{ tb_secret("KAFKA_SECRET") }}');
|
|
40
40
|
});
|
|
41
41
|
it("includes SSL CA PEM when provided", () => {
|
|
42
|
-
const conn =
|
|
42
|
+
const conn = defineKafkaConnection("my_kafka", {
|
|
43
43
|
bootstrapServers: "kafka.example.com:9092",
|
|
44
44
|
sslCaPem: '{{ tb_secret("KAFKA_CA_CERT") }}',
|
|
45
45
|
});
|
|
@@ -47,7 +47,7 @@ describe("Connection Generator", () => {
|
|
|
47
47
|
expect(result.content).toContain('KAFKA_SSL_CA_PEM {{ tb_secret("KAFKA_CA_CERT") }}');
|
|
48
48
|
});
|
|
49
49
|
it("generates full Kafka connection with all options", () => {
|
|
50
|
-
const conn =
|
|
50
|
+
const conn = defineKafkaConnection("my_kafka", {
|
|
51
51
|
bootstrapServers: "kafka.example.com:9092",
|
|
52
52
|
securityProtocol: "SASL_SSL",
|
|
53
53
|
saslMechanism: "SCRAM-SHA-256",
|
|
@@ -66,7 +66,7 @@ describe("Connection Generator", () => {
|
|
|
66
66
|
expect(result.content).toContain('KAFKA_SSL_CA_PEM {{ tb_secret("KAFKA_CA_CERT") }}');
|
|
67
67
|
});
|
|
68
68
|
it("supports PLAINTEXT security protocol", () => {
|
|
69
|
-
const conn =
|
|
69
|
+
const conn = defineKafkaConnection("local_kafka", {
|
|
70
70
|
bootstrapServers: "localhost:9092",
|
|
71
71
|
securityProtocol: "PLAINTEXT",
|
|
72
72
|
});
|
|
@@ -76,7 +76,7 @@ describe("Connection Generator", () => {
|
|
|
76
76
|
it("supports different SASL mechanisms", () => {
|
|
77
77
|
const mechanisms = ["PLAIN", "SCRAM-SHA-256", "SCRAM-SHA-512", "OAUTHBEARER"];
|
|
78
78
|
mechanisms.forEach((mechanism) => {
|
|
79
|
-
const conn =
|
|
79
|
+
const conn = defineKafkaConnection("my_kafka", {
|
|
80
80
|
bootstrapServers: "kafka.example.com:9092",
|
|
81
81
|
saslMechanism: mechanism,
|
|
82
82
|
});
|
|
@@ -84,18 +84,41 @@ describe("Connection Generator", () => {
|
|
|
84
84
|
expect(result.content).toContain(`KAFKA_SASL_MECHANISM ${mechanism}`);
|
|
85
85
|
});
|
|
86
86
|
});
|
|
87
|
+
it("generates basic S3 connection with IAM role auth", () => {
|
|
88
|
+
const conn = defineS3Connection("my_s3", {
|
|
89
|
+
region: "us-east-1",
|
|
90
|
+
arn: "arn:aws:iam::123456789012:role/tinybird-s3-access",
|
|
91
|
+
});
|
|
92
|
+
const result = generateConnection(conn);
|
|
93
|
+
expect(result.name).toBe("my_s3");
|
|
94
|
+
expect(result.content).toContain("TYPE s3");
|
|
95
|
+
expect(result.content).toContain("S3_REGION us-east-1");
|
|
96
|
+
expect(result.content).toContain("S3_ARN arn:aws:iam::123456789012:role/tinybird-s3-access");
|
|
97
|
+
});
|
|
98
|
+
it("generates S3 connection with access key auth", () => {
|
|
99
|
+
const conn = defineS3Connection("my_s3", {
|
|
100
|
+
region: "us-east-1",
|
|
101
|
+
accessKey: '{{ tb_secret("S3_ACCESS_KEY") }}',
|
|
102
|
+
secret: '{{ tb_secret("S3_SECRET") }}',
|
|
103
|
+
});
|
|
104
|
+
const result = generateConnection(conn);
|
|
105
|
+
expect(result.content).toContain("TYPE s3");
|
|
106
|
+
expect(result.content).toContain('S3_ACCESS_KEY {{ tb_secret("S3_ACCESS_KEY") }}');
|
|
107
|
+
expect(result.content).toContain('S3_SECRET {{ tb_secret("S3_SECRET") }}');
|
|
108
|
+
});
|
|
87
109
|
});
|
|
88
110
|
describe("generateAllConnections", () => {
|
|
89
111
|
it("generates all connections", () => {
|
|
90
|
-
const conn1 =
|
|
112
|
+
const conn1 = defineKafkaConnection("kafka1", {
|
|
91
113
|
bootstrapServers: "kafka1.example.com:9092",
|
|
92
114
|
});
|
|
93
|
-
const conn2 =
|
|
94
|
-
|
|
115
|
+
const conn2 = defineS3Connection("s3_logs", {
|
|
116
|
+
region: "us-east-1",
|
|
117
|
+
arn: "arn:aws:iam::123456789012:role/tinybird-s3-access",
|
|
95
118
|
});
|
|
96
|
-
const results = generateAllConnections({ kafka1: conn1,
|
|
119
|
+
const results = generateAllConnections({ kafka1: conn1, s3_logs: conn2 });
|
|
97
120
|
expect(results).toHaveLength(2);
|
|
98
|
-
expect(results.map((r) => r.name).sort()).toEqual(["kafka1", "
|
|
121
|
+
expect(results.map((r) => r.name).sort()).toEqual(["kafka1", "s3_logs"]);
|
|
99
122
|
});
|
|
100
123
|
it("returns empty array for empty connections", () => {
|
|
101
124
|
const results = generateAllConnections({});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"connection.test.js","sourceRoot":"","sources":["../../src/generator/connection.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,kBAAkB,EAAE,sBAAsB,EAAE,MAAM,iBAAiB,CAAC;AAC7E,OAAO,EAAE,qBAAqB,EAAE,MAAM,yBAAyB,CAAC;
|
|
1
|
+
{"version":3,"file":"connection.test.js","sourceRoot":"","sources":["../../src/generator/connection.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,kBAAkB,EAAE,sBAAsB,EAAE,MAAM,iBAAiB,CAAC;AAC7E,OAAO,EAAE,qBAAqB,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAEpF,QAAQ,CAAC,sBAAsB,EAAE,GAAG,EAAE;IACpC,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;QAClC,EAAE,CAAC,uDAAuD,EAAE,GAAG,EAAE;YAC/D,MAAM,IAAI,GAAG,qBAAqB,CAAC,UAAU,EAAE;gBAC7C,gBAAgB,EAAE,wBAAwB;aAC3C,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;YAExC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACrC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;YAC/C,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,gDAAgD,CAAC,CAAC;QACrF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;YAClD,MAAM,IAAI,GAAG,qBAAqB,CAAC,UAAU,EAAE;gBAC7C,gBAAgB,EAAE,wBAAwB;gBAC1C,gBAAgB,EAAE,UAAU;aAC7B,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;YAExC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,kCAAkC,CAAC,CAAC;QACvE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,MAAM,IAAI,GAAG,qBAAqB,CAAC,UAAU,EAAE;gBAC7C,gBAAgB,EAAE,wBAAwB;gBAC1C,aAAa,EAAE,OAAO;aACvB,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;YAExC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,4BAA4B,CAAC,CAAC;QACjE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,MAAM,IAAI,GAAG,qBAAqB,CAAC,UAAU,EAAE;gBAC7C,gBAAgB,EAAE,wBAAwB;gBAC1C,GAAG,EAAE,8BAA8B;gBACnC,MAAM,EAAE,iCAAiC;aAC1C,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;YAExC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,wCAAwC,CAAC,CAAC;YAC3E,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,8CAA8C,CAAC,CAAC;QACnF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;YAC3C,MAAM,IAAI,GAAG,qBAAqB,CAAC,UAAU,EAAE;gBAC7C,gBAAgB,EAAE,wBAAwB;gBAC1C,QAAQ,EAAE,kCAAkC;aAC7C,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;YAExC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,mDAAmD,CAAC,CAAC;QACxF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kDAAkD,EAAE,GAAG,EAAE;YAC1D,MAAM,IAAI,GAAG,qBAAqB,CAAC,UAAU,EAAE;gBAC7C,gBAAgB,EAAE,wBAAwB;gBAC1C,gBAAgB,EAAE,UAAU;gBAC5B,aAAa,EAAE,eAAe;gBAC9B,GAAG,EAAE,8BAA8B;gBACnC,MAAM,EAAE,iCAAiC;gBACzC,QAAQ,EAAE,kCAAkC;aAC7C,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;YAExC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACrC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;YAC/C,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,gDAAgD,CAAC,CAAC;YACnF,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,kCAAkC,CAAC,CAAC;YACrE,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,oCAAoC,CAAC,CAAC;YACvE,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,wCAAwC,CAAC,CAAC;YAC3E,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,8CAA8C,CAAC,CAAC;YACjF,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,mDAAmD,CAAC,CAAC;QACxF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sCAAsC,EAAE,GAAG,EAAE;YAC9C,MAAM,IAAI,GAAG,qBAAqB,CAAC,aAAa,EAAE;gBAChD,gBAAgB,EAAE,gBAAgB;gBAClC,gBAAgB,EAAE,WAAW;aAC9B,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;YAExC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,mCAAmC,CAAC,CAAC;QACxE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;YAC5C,MAAM,UAAU,GAAG,CAAC,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,aAAa,CAAU,CAAC;YAEvF,UAAU,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,EAAE;gBAC/B,MAAM,IAAI,GAAG,qBAAqB,CAAC,UAAU,EAAE;oBAC7C,gBAAgB,EAAE,wBAAwB;oBAC1C,aAAa,EAAE,SAAS;iBACzB,CAAC,CAAC;gBAEH,MAAM,MAAM,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;gBAExC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,wBAAwB,SAAS,EAAE,CAAC,CAAC;YACxE,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kDAAkD,EAAE,GAAG,EAAE;YAC1D,MAAM,IAAI,GAAG,kBAAkB,CAAC,OAAO,EAAE;gBACvC,MAAM,EAAE,WAAW;gBACnB,GAAG,EAAE,mDAAmD;aACzD,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;YAExC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAClC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;YAC5C,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,qBAAqB,CAAC,CAAC;YACxD,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAC9B,0DAA0D,CAC3D,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;YACtD,MAAM,IAAI,GAAG,kBAAkB,CAAC,OAAO,EAAE;gBACvC,MAAM,EAAE,WAAW;gBACnB,SAAS,EAAE,kCAAkC;gBAC7C,MAAM,EAAE,8BAA8B;aACvC,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;YAExC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;YAC5C,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,gDAAgD,CAAC,CAAC;YACnF,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,wCAAwC,CAAC,CAAC;QAC7E,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,wBAAwB,EAAE,GAAG,EAAE;QACtC,EAAE,CAAC,2BAA2B,EAAE,GAAG,EAAE;YACnC,MAAM,KAAK,GAAG,qBAAqB,CAAC,QAAQ,EAAE;gBAC5C,gBAAgB,EAAE,yBAAyB;aAC5C,CAAC,CAAC;YACH,MAAM,KAAK,GAAG,kBAAkB,CAAC,SAAS,EAAE;gBAC1C,MAAM,EAAE,WAAW;gBACnB,GAAG,EAAE,mDAAmD;aACzD,CAAC,CAAC;YAEH,MAAM,OAAO,GAAG,sBAAsB,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;YAE1E,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAChC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC;QAC3E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;YACnD,MAAM,OAAO,GAAG,sBAAsB,CAAC,EAAE,CAAC,CAAC;YAE3C,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"datasource.d.ts","sourceRoot":"","sources":["../../src/generator/datasource.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"datasource.d.ts","sourceRoot":"","sources":["../../src/generator/datasource.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EACV,oBAAoB,EAMrB,MAAM,yBAAyB,CAAC;AAKjC;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,sBAAsB;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,6CAA6C;IAC7C,OAAO,EAAE,MAAM,CAAC;CACjB;AAkOD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,wBAAgB,kBAAkB,CAChC,UAAU,EAAE,oBAAoB,GAC/B,mBAAmB,CA4DrB;AAED;;;;;GAKG;AACH,wBAAgB,sBAAsB,CACpC,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,oBAAoB,CAAC,GAChD,mBAAmB,EAAE,CAEvB"}
|
|
@@ -119,6 +119,21 @@ function generateKafkaConfig(kafka) {
|
|
|
119
119
|
}
|
|
120
120
|
return parts.join("\n");
|
|
121
121
|
}
|
|
122
|
+
/**
|
|
123
|
+
* Generate S3 import configuration lines
|
|
124
|
+
*/
|
|
125
|
+
function generateS3Config(s3) {
|
|
126
|
+
const parts = [];
|
|
127
|
+
parts.push(`IMPORT_CONNECTION_NAME ${s3.connection._name}`);
|
|
128
|
+
parts.push(`IMPORT_BUCKET_URI ${s3.bucketUri}`);
|
|
129
|
+
if (s3.schedule) {
|
|
130
|
+
parts.push(`IMPORT_SCHEDULE ${s3.schedule}`);
|
|
131
|
+
}
|
|
132
|
+
if (s3.fromTimestamp) {
|
|
133
|
+
parts.push(`IMPORT_FROM_TIMESTAMP ${s3.fromTimestamp}`);
|
|
134
|
+
}
|
|
135
|
+
return parts.join("\n");
|
|
136
|
+
}
|
|
122
137
|
/**
|
|
123
138
|
* Generate forward query section
|
|
124
139
|
*/
|
|
@@ -205,6 +220,9 @@ function generateTokens(tokens) {
|
|
|
205
220
|
*/
|
|
206
221
|
export function generateDatasource(datasource) {
|
|
207
222
|
const parts = [];
|
|
223
|
+
if (datasource.options.kafka && datasource.options.s3) {
|
|
224
|
+
throw new Error("Datasource cannot define both `kafka` and `s3` ingestion options.");
|
|
225
|
+
}
|
|
208
226
|
// Add description if present
|
|
209
227
|
if (datasource.options.description) {
|
|
210
228
|
parts.push(`DESCRIPTION >\n ${datasource.options.description}`);
|
|
@@ -222,6 +240,11 @@ export function generateDatasource(datasource) {
|
|
|
222
240
|
parts.push("");
|
|
223
241
|
parts.push(generateKafkaConfig(datasource.options.kafka));
|
|
224
242
|
}
|
|
243
|
+
// Add S3 configuration if present
|
|
244
|
+
if (datasource.options.s3) {
|
|
245
|
+
parts.push("");
|
|
246
|
+
parts.push(generateS3Config(datasource.options.s3));
|
|
247
|
+
}
|
|
225
248
|
// Add forward query if present
|
|
226
249
|
const forwardQuery = generateForwardQuery(datasource.options.forwardQuery);
|
|
227
250
|
if (forwardQuery) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"datasource.js","sourceRoot":"","sources":["../../src/generator/datasource.ts"],"names":[],"mappings":"AAAA;;;GAGG;
|
|
1
|
+
{"version":3,"file":"datasource.js","sourceRoot":"","sources":["../../src/generator/datasource.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAWH,OAAO,EAAE,aAAa,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAC3E,OAAO,EAAE,eAAe,EAAqB,MAAM,sBAAsB,CAAC;AAY1E;;;GAGG;AACH,SAAS,4BAA4B,CAAC,SAA2B;IAC/D,qDAAqD;IACrD,OAAO,SAAS,CAAC,aAAa,CAAC;AACjC,CAAC;AAED;;GAEG;AACH,SAAS,yBAAyB,CAAC,SAA2B;IAC5D,OAAO,SAAS,CAAC,UAAU,CAAC;AAC9B,CAAC;AAED;;GAEG;AACH,SAAS,kBAAkB,CAAC,KAAc,EAAE,YAAoB;IAC9D,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;QACnB,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,uBAAuB;QACvB,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,CAAC;IAC3C,CAAC;IAED,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC3D,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;IACvB,CAAC;IAED,IAAI,OAAO,KAAK,KAAK,SAAS,EAAE,CAAC;QAC/B,OAAO,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IAC3B,CAAC;IAED,IAAI,KAAK,YAAY,IAAI,EAAE,CAAC;QAC1B,uBAAuB;QACvB,IAAI,YAAY,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YACtE,OAAO,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;QAClD,CAAC;QACD,OAAO,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC;IACnE,CAAC;IAED,mDAAmD;IACnD,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC;IAED,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;QAChD,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC;IAED,yDAAyD;IACzD,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,CAAC;AACnD,CAAC;AAED;;GAEG;AACH,SAAS,kBAAkB,CACzB,UAAkB,EAClB,MAA2C,EAC3C,gBAAyB;IAEzB,MAAM,SAAS,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;IACxC,MAAM,QAAQ,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC;IAC3C,MAAM,YAAY,GAAG,4BAA4B,CAAC,SAAS,CAAC,CAAC;IAC7D,MAAM,SAAS,GAAG,yBAAyB,CAAC,SAAS,CAAC,CAAC;IAEvD,MAAM,KAAK,GAAa,CAAC,OAAO,UAAU,IAAI,YAAY,EAAE,CAAC,CAAC;IAE9D,4DAA4D;IAC5D,sEAAsE;IACtE,IAAI,gBAAgB,EAAE,CAAC;QACrB,MAAM,iBAAiB,GAAG,QAAQ,IAAI,KAAK,UAAU,EAAE,CAAC;QACxD,KAAK,CAAC,IAAI,CAAC,UAAU,iBAAiB,IAAI,CAAC,CAAC;IAC9C,CAAC;IAED,+BAA+B;IAC/B,IAAI,SAAS,CAAC,UAAU,IAAI,SAAS,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;QACjE,MAAM,UAAU,GAAG,kBAAkB,CAAC,SAAS,CAAC,YAAY,EAAE,YAAY,CAAC,CAAC;QAC5E,KAAK,CAAC,IAAI,CAAC,WAAW,UAAU,EAAE,CAAC,CAAC;IACtC,CAAC;IAED,uBAAuB;IACvB,IAAI,SAAS,CAAC,KAAK,EAAE,CAAC;QACpB,KAAK,CAAC,IAAI,CAAC,SAAS,SAAS,CAAC,KAAK,GAAG,CAAC,CAAC;IAC1C,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACzB,CAAC;AAED;;GAEG;AACH,SAAS,cAAc,CAAC,MAAwB,EAAE,gBAAyB;IACzE,MAAM,KAAK,GAAG,CAAC,UAAU,CAAC,CAAC;IAE3B,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACxC,WAAW,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QAClC,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QAC5B,MAAM,IAAI,GAAG,kBAAkB,CAAC,IAAI,EAAE,MAAM,EAAE,gBAAgB,CAAC,CAAC;QAChE,mCAAmC;QACnC,MAAM,MAAM,GAAG,KAAK,GAAG,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QACzD,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,MAAM,CAAC,CAAC;IAC5B,CAAC,CAAC,CAAC;IAEH,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;;GAGG;AACH,SAAS,oBAAoB,CAAC,MAAqB;IACjD,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,wDAAwD;QACxD,OAAO,oBAAoB,CAAC;IAC9B,CAAC;IAED,OAAO,eAAe,CAAC,MAAM,CAAC,CAAC;AACjC,CAAC;AAED;;GAEG;AACH,SAAS,mBAAmB,CAAC,KAAkB;IAC7C,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,KAAK,CAAC,IAAI,CAAC,yBAAyB,KAAK,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC,CAAC;IAC9D,KAAK,CAAC,IAAI,CAAC,eAAe,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;IAEzC,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;QAClB,KAAK,CAAC,IAAI,CAAC,kBAAkB,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;IAChD,CAAC;IAED,IAAI,KAAK,CAAC,eAAe,EAAE,CAAC;QAC1B,KAAK,CAAC,IAAI,CAAC,2BAA2B,KAAK,CAAC,eAAe,EAAE,CAAC,CAAC;IACjE,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,SAAS,gBAAgB,CAAC,EAAY;IACpC,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,KAAK,CAAC,IAAI,CAAC,0BAA0B,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC,CAAC;IAC5D,KAAK,CAAC,IAAI,CAAC,qBAAqB,EAAE,CAAC,SAAS,EAAE,CAAC,CAAC;IAEhD,IAAI,EAAE,CAAC,QAAQ,EAAE,CAAC;QAChB,KAAK,CAAC,IAAI,CAAC,mBAAmB,EAAE,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC/C,CAAC;IAED,IAAI,EAAE,CAAC,aAAa,EAAE,CAAC;QACrB,KAAK,CAAC,IAAI,CAAC,yBAAyB,EAAE,CAAC,aAAa,EAAE,CAAC,CAAC;IAC1D,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,SAAS,oBAAoB,CAAC,YAAqB;IACjD,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,EAAE,CAAC;IACpC,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACrC,OAAO,CAAC,iBAAiB,EAAE,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC/E,CAAC;AAED;;GAEG;AACH,SAAS,kBAAkB,CAAC,UAA8B;IACxD,IAAI,CAAC,UAAU,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3C,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,KAAK,GAAG,CAAC,eAAe,CAAC,CAAC;IAChC,UAAU,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,EAAE;QACtC,MAAM,MAAM,GAAG,KAAK,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QACxD,KAAK,CAAC,IAAI,CAAC,OAAO,SAAS,GAAG,MAAM,EAAE,CAAC,CAAC;IAC1C,CAAC,CAAC,CAAC;IAEH,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,SAAS,cAAc,CAAC,MAA+B;IACrD,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnC,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,OAAO,IAAI,KAAK,EAAE,CAAC;YACrB,gCAAgC;YAChC,KAAK,CAAC,IAAI,CAAC,SAAS,KAAK,CAAC,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;QAC1D,CAAC;aAAM,CAAC;YACN,uCAAuC;YACvC,KAAK,MAAM,UAAU,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC;gBAC3C,KAAK,CAAC,IAAI,CAAC,SAAS,KAAK,CAAC,IAAI,IAAI,UAAU,EAAE,CAAC,CAAC;YAClD,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,MAAM,UAAU,kBAAkB,CAChC,UAAgC;IAEhC,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,IAAI,UAAU,CAAC,OAAO,CAAC,KAAK,IAAI,UAAU,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC;QACtD,MAAM,IAAI,KAAK,CAAC,mEAAmE,CAAC,CAAC;IACvF,CAAC;IAED,6BAA6B;IAC7B,IAAI,UAAU,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;QACnC,KAAK,CAAC,IAAI,CAAC,sBAAsB,UAAU,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;QACnE,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,4DAA4D;IAC5D,MAAM,gBAAgB,GAAG,UAAU,CAAC,OAAO,CAAC,SAAS,KAAK,KAAK,CAAC;IAEhE,aAAa;IACb,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,OAAO,EAAE,gBAAgB,CAAC,CAAC,CAAC;IACjE,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,2BAA2B;IAC3B,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,UAAU,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;IAE5D,qCAAqC;IACrC,IAAI,UAAU,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QAC7B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;IAC5D,CAAC;IAED,kCAAkC;IAClC,IAAI,UAAU,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC;QAC1B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC;IACtD,CAAC;IAED,+BAA+B;IAC/B,MAAM,YAAY,GAAG,oBAAoB,CAAC,UAAU,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;IAC3E,IAAI,YAAY,EAAE,CAAC;QACjB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAC3B,CAAC;IAED,wBAAwB;IACxB,MAAM,UAAU,GAAG,cAAc,CAAC,UAAU,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAC7D,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IACpC,CAAC;IAED,6BAA6B;IAC7B,MAAM,eAAe,GAAG,kBAAkB,CAAC,UAAU,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IAC1E,IAAI,eAAe,EAAE,CAAC;QACpB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IAC9B,CAAC;IAED,OAAO;QACL,IAAI,EAAE,UAAU,CAAC,KAAK;QACtB,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;KAC1B,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,sBAAsB,CACpC,WAAiD;IAEjD,OAAO,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;AAC5D,CAAC"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { describe, it, expect } from 'vitest';
|
|
2
2
|
import { generateDatasource, generateAllDatasources } from './datasource.js';
|
|
3
3
|
import { defineDatasource } from '../schema/datasource.js';
|
|
4
|
-
import {
|
|
4
|
+
import { defineKafkaConnection, defineS3Connection } from '../schema/connection.js';
|
|
5
5
|
import { defineToken } from '../schema/token.js';
|
|
6
6
|
import { t } from '../schema/types.js';
|
|
7
7
|
import { engine } from '../schema/engines.js';
|
|
@@ -265,7 +265,7 @@ describe('Datasource Generator', () => {
|
|
|
265
265
|
});
|
|
266
266
|
describe('Kafka configuration', () => {
|
|
267
267
|
it('includes Kafka connection name and topic', () => {
|
|
268
|
-
const kafkaConn =
|
|
268
|
+
const kafkaConn = defineKafkaConnection('my_kafka', {
|
|
269
269
|
bootstrapServers: 'kafka.example.com:9092',
|
|
270
270
|
});
|
|
271
271
|
const ds = defineDatasource('kafka_events', {
|
|
@@ -284,7 +284,7 @@ describe('Datasource Generator', () => {
|
|
|
284
284
|
expect(result.content).toContain('KAFKA_TOPIC events');
|
|
285
285
|
});
|
|
286
286
|
it('includes Kafka group ID when provided', () => {
|
|
287
|
-
const kafkaConn =
|
|
287
|
+
const kafkaConn = defineKafkaConnection('my_kafka', {
|
|
288
288
|
bootstrapServers: 'kafka.example.com:9092',
|
|
289
289
|
});
|
|
290
290
|
const ds = defineDatasource('kafka_events', {
|
|
@@ -303,7 +303,7 @@ describe('Datasource Generator', () => {
|
|
|
303
303
|
expect(result.content).toContain('KAFKA_GROUP_ID my-consumer-group');
|
|
304
304
|
});
|
|
305
305
|
it('includes auto offset reset when provided', () => {
|
|
306
|
-
const kafkaConn =
|
|
306
|
+
const kafkaConn = defineKafkaConnection('my_kafka', {
|
|
307
307
|
bootstrapServers: 'kafka.example.com:9092',
|
|
308
308
|
});
|
|
309
309
|
const ds = defineDatasource('kafka_events', {
|
|
@@ -322,7 +322,7 @@ describe('Datasource Generator', () => {
|
|
|
322
322
|
expect(result.content).toContain('KAFKA_AUTO_OFFSET_RESET earliest');
|
|
323
323
|
});
|
|
324
324
|
it('generates complete Kafka datasource with all options', () => {
|
|
325
|
-
const kafkaConn =
|
|
325
|
+
const kafkaConn = defineKafkaConnection('my_kafka', {
|
|
326
326
|
bootstrapServers: 'kafka.example.com:9092',
|
|
327
327
|
securityProtocol: 'SASL_SSL',
|
|
328
328
|
saslMechanism: 'PLAIN',
|
|
@@ -354,6 +354,50 @@ describe('Datasource Generator', () => {
|
|
|
354
354
|
expect(result.content).toContain('KAFKA_AUTO_OFFSET_RESET earliest');
|
|
355
355
|
});
|
|
356
356
|
});
|
|
357
|
+
describe('S3 configuration', () => {
|
|
358
|
+
it('includes S3 connection name and bucket uri', () => {
|
|
359
|
+
const s3Conn = defineS3Connection('my_s3', {
|
|
360
|
+
region: 'us-east-1',
|
|
361
|
+
arn: 'arn:aws:iam::123456789012:role/tinybird-s3-access',
|
|
362
|
+
});
|
|
363
|
+
const ds = defineDatasource('s3_events', {
|
|
364
|
+
schema: {
|
|
365
|
+
timestamp: t.dateTime(),
|
|
366
|
+
event: t.string(),
|
|
367
|
+
},
|
|
368
|
+
engine: engine.mergeTree({ sortingKey: ['timestamp'] }),
|
|
369
|
+
s3: {
|
|
370
|
+
connection: s3Conn,
|
|
371
|
+
bucketUri: 's3://my-bucket/events/*.csv',
|
|
372
|
+
},
|
|
373
|
+
});
|
|
374
|
+
const result = generateDatasource(ds);
|
|
375
|
+
expect(result.content).toContain('IMPORT_CONNECTION_NAME my_s3');
|
|
376
|
+
expect(result.content).toContain('IMPORT_BUCKET_URI s3://my-bucket/events/*.csv');
|
|
377
|
+
});
|
|
378
|
+
it('includes optional S3 schedule and from timestamp', () => {
|
|
379
|
+
const s3Conn = defineS3Connection('my_s3', {
|
|
380
|
+
region: 'us-east-1',
|
|
381
|
+
arn: 'arn:aws:iam::123456789012:role/tinybird-s3-access',
|
|
382
|
+
});
|
|
383
|
+
const ds = defineDatasource('s3_events', {
|
|
384
|
+
schema: {
|
|
385
|
+
timestamp: t.dateTime(),
|
|
386
|
+
event: t.string(),
|
|
387
|
+
},
|
|
388
|
+
engine: engine.mergeTree({ sortingKey: ['timestamp'] }),
|
|
389
|
+
s3: {
|
|
390
|
+
connection: s3Conn,
|
|
391
|
+
bucketUri: 's3://my-bucket/events/*.csv',
|
|
392
|
+
schedule: '@auto',
|
|
393
|
+
fromTimestamp: '2024-01-01T00:00:00Z',
|
|
394
|
+
},
|
|
395
|
+
});
|
|
396
|
+
const result = generateDatasource(ds);
|
|
397
|
+
expect(result.content).toContain('IMPORT_SCHEDULE @auto');
|
|
398
|
+
expect(result.content).toContain('IMPORT_FROM_TIMESTAMP 2024-01-01T00:00:00Z');
|
|
399
|
+
});
|
|
400
|
+
});
|
|
357
401
|
describe('Token generation', () => {
|
|
358
402
|
it('generates TOKEN lines with inline config', () => {
|
|
359
403
|
const ds = defineDatasource('test_ds', {
|