@tinybirdco/sdk 0.0.43 → 0.0.45

package/src/schema/datasource.test.ts

@@ -9,6 +9,7 @@ import {
 } from "./datasource.js";
 import { t } from "./types.js";
 import { engine } from "./engines.js";
+import { defineKafkaConnection, defineS3Connection } from "./connection.js";
 
 describe("Datasource Schema", () => {
   describe("defineDatasource", () => {
@@ -85,6 +86,30 @@ describe("Datasource Schema", () => {
       });
       expect(ds2._name).toBe("events_v2");
     });
+
+    it("throws when both kafka and s3 ingestion are configured", () => {
+      const kafkaConn = defineKafkaConnection("my_kafka", {
+        bootstrapServers: "kafka.example.com:9092",
+      });
+      const s3Conn = defineS3Connection("my_s3", {
+        region: "us-east-1",
+        arn: "arn:aws:iam::123456789012:role/tinybird-s3-access",
+      });
+
+      expect(() =>
+        defineDatasource("events", {
+          schema: { id: t.string() },
+          kafka: {
+            connection: kafkaConn,
+            topic: "events",
+          },
+          s3: {
+            connection: s3Conn,
+            bucketUri: "s3://my-bucket/events/*.csv",
+          },
+        })
+      ).toThrow("Datasource cannot define both `kafka` and `s3` ingestion options.");
+    });
   });
 
   describe("isDatasourceDefinition", () => {

package/src/schema/datasource.ts

@@ -5,7 +5,7 @@
 
 import type { AnyTypeValidator } from "./types.js";
 import type { EngineConfig } from "./engines.js";
-import type { KafkaConnectionDefinition } from "./connection.js";
+import type { KafkaConnectionDefinition, S3ConnectionDefinition } from "./connection.js";
 import type { TokenDefinition, DatasourceTokenScope } from "./token.js";
 
 // Symbol for brand typing - use Symbol.for() for global registry
@@ -68,6 +68,20 @@ export interface KafkaConfig {
   autoOffsetReset?: "earliest" | "latest";
 }
 
+/**
+ * S3 import configuration for a datasource
+ */
+export interface S3Config {
+  /** S3 connection to use */
+  connection: S3ConnectionDefinition;
+  /** S3 bucket URI, for example: s3://my-bucket/path/*.csv */
+  bucketUri: string;
+  /** Import schedule, for example: @auto or @once */
+  schedule?: string;
+  /** Incremental import lower bound timestamp expression */
+  fromTimestamp?: string;
+}
+
 /**
  * Options for defining a datasource
  */
@@ -95,6 +109,8 @@ export interface DatasourceOptions<TSchema extends SchemaDefinition> {
   forwardQuery?: string;
   /** Kafka ingestion configuration */
   kafka?: KafkaConfig;
+  /** S3 ingestion configuration */
+  s3?: S3Config;
 }
 
 /**
@@ -152,6 +168,10 @@ export function defineDatasource<TSchema extends SchemaDefinition>(
     );
   }
 
+  if (options.kafka && options.s3) {
+    throw new Error("Datasource cannot define both `kafka` and `s3` ingestion options.");
+  }
+
   return {
     [DATASOURCE_BRAND]: true,
     _name: name,