@tinybirdco/sdk 0.0.44 → 0.0.45

package/src/migrate/parse-connection.ts

@@ -1,14 +1,18 @@
-import type { KafkaConnectionModel, ResourceFile } from "./types.js";
+import type { KafkaConnectionModel, ResourceFile, S3ConnectionModel } from "./types.js";
 import {
   MigrationParseError,
   isBlank,
   parseDirectiveLine,
+  parseQuotedValue,
   splitLines,
 } from "./parser-utils.js";
 
-export function parseConnectionFile(resource: ResourceFile): KafkaConnectionModel {
+export function parseConnectionFile(
+  resource: ResourceFile
+): KafkaConnectionModel | S3ConnectionModel {
   const lines = splitLines(resource.content);
   let connectionType: string | undefined;
+
   let bootstrapServers: string | undefined;
   let securityProtocol:
     | "SASL_SSL"
@@ -25,6 +29,11 @@ export function parseConnectionFile(resource: ResourceFile): KafkaConnectionMode
   let secret: string | undefined;
   let sslCaPem: string | undefined;
 
+  let region: string | undefined;
+  let arn: string | undefined;
+  let accessKey: string | undefined;
+  let accessSecret: string | undefined;
+
   for (const rawLine of lines) {
     const line = rawLine.trim();
     if (isBlank(line)) {
@@ -34,7 +43,7 @@ export function parseConnectionFile(resource: ResourceFile): KafkaConnectionMode
     const { key: directive, value } = parseDirectiveLine(line);
     switch (directive) {
       case "TYPE":
-        connectionType = value;
+        connectionType = parseQuotedValue(value);
         break;
       case "KAFKA_BOOTSTRAP_SERVERS":
         bootstrapServers = value;
@@ -75,6 +84,18 @@ export function parseConnectionFile(resource: ResourceFile): KafkaConnectionMode
       case "KAFKA_SSL_CA_PEM":
         sslCaPem = value;
         break;
+      case "S3_REGION":
+        region = parseQuotedValue(value);
+        break;
+      case "S3_ARN":
+        arn = parseQuotedValue(value);
+        break;
+      case "S3_ACCESS_KEY":
+        accessKey = parseQuotedValue(value);
+        break;
+      case "S3_SECRET":
+        accessSecret = parseQuotedValue(value);
+        break;
       default:
         throw new MigrationParseError(
           resource.filePath,
@@ -94,35 +115,92 @@ export function parseConnectionFile(resource: ResourceFile): KafkaConnectionMode
     );
   }
 
-  if (connectionType !== "kafka") {
-    throw new MigrationParseError(
-      resource.filePath,
-      "connection",
-      resource.name,
-      `Unsupported connection type in strict mode: "${connectionType}"`
-    );
+  if (connectionType === "kafka") {
+    if (region || arn || accessKey || accessSecret) {
+      throw new MigrationParseError(
+        resource.filePath,
+        "connection",
+        resource.name,
+        "S3 directives are not valid for kafka connections."
+      );
+    }
+
+    if (!bootstrapServers) {
+      throw new MigrationParseError(
+        resource.filePath,
+        "connection",
+        resource.name,
+        "KAFKA_BOOTSTRAP_SERVERS is required for kafka connections."
+      );
+    }
+
+    return {
+      kind: "connection",
+      name: resource.name,
+      filePath: resource.filePath,
+      connectionType: "kafka",
+      bootstrapServers,
+      securityProtocol,
+      saslMechanism,
+      key,
+      secret,
+      sslCaPem,
+    };
   }
 
-  if (!bootstrapServers) {
-    throw new MigrationParseError(
-      resource.filePath,
-      "connection",
-      resource.name,
-      "KAFKA_BOOTSTRAP_SERVERS is required for kafka connections."
-    );
+  if (connectionType === "s3") {
+    if (bootstrapServers || securityProtocol || saslMechanism || key || secret || sslCaPem) {
+      throw new MigrationParseError(
+        resource.filePath,
+        "connection",
+        resource.name,
+        "Kafka directives are not valid for s3 connections."
+      );
+    }
+
+    if (!region) {
+      throw new MigrationParseError(
+        resource.filePath,
+        "connection",
+        resource.name,
+        "S3_REGION is required for s3 connections."
+      );
+    }
+
+    if (!arn && !(accessKey && accessSecret)) {
+      throw new MigrationParseError(
+        resource.filePath,
+        "connection",
+        resource.name,
+        "S3 connections require S3_ARN or both S3_ACCESS_KEY and S3_SECRET."
+      );
+    }
+
+    if ((accessKey && !accessSecret) || (!accessKey && accessSecret)) {
+      throw new MigrationParseError(
+        resource.filePath,
+        "connection",
+        resource.name,
+        "S3_ACCESS_KEY and S3_SECRET must be provided together."
+      );
+    }
+
+    return {
+      kind: "connection",
+      name: resource.name,
+      filePath: resource.filePath,
+      connectionType: "s3",
+      region,
+      arn,
+      accessKey,
+      secret: accessSecret,
+    };
   }
 
-  return {
-    kind: "connection",
-    name: resource.name,
-    filePath: resource.filePath,
-    connectionType: "kafka",
-    bootstrapServers,
-    securityProtocol,
-    saslMechanism,
-    key,
-    secret,
-    sslCaPem,
-  };
+  throw new MigrationParseError(
+    resource.filePath,
+    "connection",
+    resource.name,
+    `Unsupported connection type in strict mode: "${connectionType}"`
+  );
 }
-

package/src/migrate/parse-datasource.ts

@@ -233,6 +233,11 @@ export function parseDatasourceFile(resource: ResourceFile): DatasourceModel {
   let kafkaGroupId: string | undefined;
   let kafkaAutoOffsetReset: "earliest" | "latest" | undefined;
 
+  let importConnectionName: string | undefined;
+  let importBucketUri: string | undefined;
+  let importSchedule: string | undefined;
+  let importFromTimestamp: string | undefined;
+
   let i = 0;
   while (i < lines.length) {
     const rawLine = lines[i] ?? "";
@@ -365,6 +370,18 @@ export function parseDatasourceFile(resource: ResourceFile): DatasourceModel {
         }
         kafkaAutoOffsetReset = value;
         break;
+      case "IMPORT_CONNECTION_NAME":
+        importConnectionName = parseQuotedValue(value);
+        break;
+      case "IMPORT_BUCKET_URI":
+        importBucketUri = parseQuotedValue(value);
+        break;
+      case "IMPORT_SCHEDULE":
+        importSchedule = parseQuotedValue(value);
+        break;
+      case "IMPORT_FROM_TIMESTAMP":
+        importFromTimestamp = parseQuotedValue(value);
+        break;
       case "TOKEN":
         tokens.push(parseToken(resource.filePath, resource.name, value));
         break;
@@ -426,6 +443,34 @@ export function parseDatasourceFile(resource: ResourceFile): DatasourceModel {
     );
   }
 
+  const s3 =
+    importConnectionName || importBucketUri || importSchedule || importFromTimestamp
+      ? {
+          connectionName: importConnectionName ?? "",
+          bucketUri: importBucketUri ?? "",
+          schedule: importSchedule,
+          fromTimestamp: importFromTimestamp,
+        }
+      : undefined;
+
+  if (s3 && (!s3.connectionName || !s3.bucketUri)) {
+    throw new MigrationParseError(
+      resource.filePath,
+      "datasource",
+      resource.name,
+      "IMPORT_CONNECTION_NAME and IMPORT_BUCKET_URI are required when S3 import directives are used."
+    );
+  }
+
+  if (kafka && s3) {
+    throw new MigrationParseError(
+      resource.filePath,
+      "datasource",
+      resource.name,
+      "Datasource cannot mix Kafka directives with S3 import directives."
+    );
+  }
+
   return {
     kind: "datasource",
     name: resource.name,
@@ -445,9 +490,9 @@ export function parseDatasourceFile(resource: ResourceFile): DatasourceModel {
       settings,
     },
     kafka,
+    s3,
     forwardQuery,
     tokens,
     sharedWith,
   };
 }
-

package/src/migrate/types.ts

@@ -43,6 +43,13 @@ export interface DatasourceKafkaModel {
   autoOffsetReset?: "earliest" | "latest";
 }
 
+export interface DatasourceS3Model {
+  connectionName: string;
+  bucketUri: string;
+  schedule?: string;
+  fromTimestamp?: string;
+}
+
 export interface DatasourceTokenModel {
   name: string;
   scope: "READ" | "APPEND";
@@ -56,6 +63,7 @@ export interface DatasourceModel {
   columns: DatasourceColumnModel[];
   engine: DatasourceEngineModel;
   kafka?: DatasourceKafkaModel;
+  s3?: DatasourceS3Model;
   forwardQuery?: string;
   tokens: DatasourceTokenModel[];
   sharedWith: string[];
@@ -112,7 +120,22 @@ export interface KafkaConnectionModel {
   sslCaPem?: string;
 }
 
-export type ParsedResource = DatasourceModel | PipeModel | KafkaConnectionModel;
+export interface S3ConnectionModel {
+  kind: "connection";
+  name: string;
+  filePath: string;
+  connectionType: "s3";
+  region: string;
+  arn?: string;
+  accessKey?: string;
+  secret?: string;
+}
+
+export type ParsedResource =
+  | DatasourceModel
+  | PipeModel
+  | KafkaConnectionModel
+  | S3ConnectionModel;
 
 export interface MigrationResult {
   success: boolean;
@@ -122,4 +145,3 @@ export interface MigrationResult {
   dryRun: boolean;
   outputContent?: string;
 }
-

package/src/schema/connection.test.ts

@@ -1,15 +1,17 @@
 import { describe, it, expect } from "vitest";
 import {
-  createKafkaConnection,
+  defineKafkaConnection,
+  defineS3Connection,
   isConnectionDefinition,
   isKafkaConnectionDefinition,
+  isS3ConnectionDefinition,
   getConnectionType,
 } from "./connection.js";
 
 describe("Connection Schema", () => {
-  describe("createKafkaConnection", () => {
+  describe("defineKafkaConnection", () => {
     it("creates a Kafka connection with required fields", () => {
-      const conn = createKafkaConnection("my_kafka", {
+      const conn = defineKafkaConnection("my_kafka", {
         bootstrapServers: "kafka.example.com:9092",
       });
 
@@ -20,7 +22,7 @@ describe("Connection Schema", () => {
     });
 
     it("creates a Kafka connection with all options", () => {
-      const conn = createKafkaConnection("my_kafka", {
+      const conn = defineKafkaConnection("my_kafka", {
         bootstrapServers: "kafka.example.com:9092",
         securityProtocol: "SASL_SSL",
         saslMechanism: "PLAIN",
@@ -37,19 +39,19 @@ describe("Connection Schema", () => {
     });
 
     it("supports different SASL mechanisms", () => {
-      const scramConn = createKafkaConnection("scram_kafka", {
+      const scramConn = defineKafkaConnection("scram_kafka", {
         bootstrapServers: "kafka.example.com:9092",
         saslMechanism: "SCRAM-SHA-256",
       });
       expect(scramConn.options.saslMechanism).toBe("SCRAM-SHA-256");
 
-      const scram512Conn = createKafkaConnection("scram512_kafka", {
+      const scram512Conn = defineKafkaConnection("scram512_kafka", {
         bootstrapServers: "kafka.example.com:9092",
         saslMechanism: "SCRAM-SHA-512",
       });
       expect(scram512Conn.options.saslMechanism).toBe("SCRAM-SHA-512");
 
-      const oauthConn = createKafkaConnection("oauth_kafka", {
+      const oauthConn = defineKafkaConnection("oauth_kafka", {
         bootstrapServers: "kafka.example.com:9092",
         saslMechanism: "OAUTHBEARER",
       });
@@ -57,13 +59,13 @@ describe("Connection Schema", () => {
     });
 
     it("supports different security protocols", () => {
-      const plaintext = createKafkaConnection("plaintext_kafka", {
+      const plaintext = defineKafkaConnection("plaintext_kafka", {
         bootstrapServers: "localhost:9092",
         securityProtocol: "PLAINTEXT",
       });
       expect(plaintext.options.securityProtocol).toBe("PLAINTEXT");
 
-      const saslPlaintext = createKafkaConnection("sasl_plaintext_kafka", {
+      const saslPlaintext = defineKafkaConnection("sasl_plaintext_kafka", {
         bootstrapServers: "localhost:9092",
         securityProtocol: "SASL_PLAINTEXT",
       });
@@ -72,40 +74,88 @@ describe("Connection Schema", () => {
 
     it("throws error for invalid connection name", () => {
       expect(() =>
-        createKafkaConnection("123invalid", {
+        defineKafkaConnection("123invalid", {
           bootstrapServers: "kafka.example.com:9092",
         })
       ).toThrow("Invalid connection name");
 
       expect(() =>
-        createKafkaConnection("my-connection", {
+        defineKafkaConnection("my-connection", {
           bootstrapServers: "kafka.example.com:9092",
         })
       ).toThrow("Invalid connection name");
 
       expect(() =>
-        createKafkaConnection("", {
+        defineKafkaConnection("", {
           bootstrapServers: "kafka.example.com:9092",
         })
       ).toThrow("Invalid connection name");
     });
 
     it("allows valid naming patterns", () => {
-      const conn1 = createKafkaConnection("_private_kafka", {
+      const conn1 = defineKafkaConnection("_private_kafka", {
         bootstrapServers: "kafka.example.com:9092",
       });
       expect(conn1._name).toBe("_private_kafka");
 
-      const conn2 = createKafkaConnection("kafka_v2", {
+      const conn2 = defineKafkaConnection("kafka_v2", {
         bootstrapServers: "kafka.example.com:9092",
       });
       expect(conn2._name).toBe("kafka_v2");
     });
   });
 
+  describe("defineS3Connection", () => {
+    it("creates an S3 connection with IAM role auth", () => {
+      const conn = defineS3Connection("my_s3", {
+        region: "us-east-1",
+        arn: "arn:aws:iam::123456789012:role/tinybird-s3-access",
+      });
+
+      expect(conn._name).toBe("my_s3");
+      expect(conn._type).toBe("connection");
+      expect(conn._connectionType).toBe("s3");
+      expect(conn.options.region).toBe("us-east-1");
+      expect(conn.options.arn).toBe("arn:aws:iam::123456789012:role/tinybird-s3-access");
+    });
+
+    it("creates an S3 connection with access key auth", () => {
+      const conn = defineS3Connection("my_s3", {
+        region: "us-east-1",
+        accessKey: '{{ tb_secret("S3_ACCESS_KEY") }}',
+        secret: '{{ tb_secret("S3_SECRET") }}',
+      });
+
+      expect(conn.options.accessKey).toBe('{{ tb_secret("S3_ACCESS_KEY") }}');
+      expect(conn.options.secret).toBe('{{ tb_secret("S3_SECRET") }}');
+    });
+
+    it("throws when auth config is incomplete", () => {
+      expect(() =>
+        defineS3Connection("my_s3", {
+          region: "us-east-1",
+        })
+      ).toThrow("S3 connection requires either `arn` or both `accessKey` and `secret`.");
+
+      expect(() =>
+        defineS3Connection("my_s3", {
+          region: "us-east-1",
+          accessKey: "key-only",
+        })
+      ).toThrow("S3 connection requires either `arn` or both `accessKey` and `secret`.");
+
+      expect(() =>
+        defineS3Connection("my_s3", {
+          region: "us-east-1",
+          secret: "secret-only",
+        })
+      ).toThrow("S3 connection requires either `arn` or both `accessKey` and `secret`.");
+    });
+  });
+
   describe("isConnectionDefinition", () => {
     it("returns true for valid connection", () => {
-      const conn = createKafkaConnection("my_kafka", {
+      const conn = defineKafkaConnection("my_kafka", {
         bootstrapServers: "kafka.example.com:9092",
       });
 
@@ -124,7 +174,7 @@ describe("Connection Schema", () => {
 
   describe("isKafkaConnectionDefinition", () => {
     it("returns true for Kafka connection", () => {
-      const conn = createKafkaConnection("my_kafka", {
+      const conn = defineKafkaConnection("my_kafka", {
         bootstrapServers: "kafka.example.com:9092",
       });
 
@@ -137,13 +187,38 @@ describe("Connection Schema", () => {
     });
   });
 
+  describe("isS3ConnectionDefinition", () => {
+    it("returns true for S3 connection", () => {
+      const conn = defineS3Connection("my_s3", {
+        region: "us-east-1",
+        arn: "arn:aws:iam::123456789012:role/tinybird-s3-access",
+      });
+
+      expect(isS3ConnectionDefinition(conn)).toBe(true);
+    });
+
+    it("returns false for non-S3 objects", () => {
+      expect(isS3ConnectionDefinition({})).toBe(false);
+      expect(isS3ConnectionDefinition(null)).toBe(false);
+    });
+  });
+
   describe("getConnectionType", () => {
     it("returns the connection type", () => {
-      const conn = createKafkaConnection("my_kafka", {
+      const conn = defineKafkaConnection("my_kafka", {
         bootstrapServers: "kafka.example.com:9092",
       });
 
       expect(getConnectionType(conn)).toBe("kafka");
     });
+
+    it("returns the s3 connection type", () => {
+      const conn = defineS3Connection("my_s3", {
+        region: "us-east-1",
+        arn: "arn:aws:iam::123456789012:role/tinybird-s3-access",
+      });
+
+      expect(getConnectionType(conn)).toBe("s3");
+    });
   });
 });

package/src/schema/connection.ts

@@ -1,6 +1,6 @@
 /**
  * Connection definition for Tinybird
- * Define external connections (Kafka, etc.) as TypeScript with full type safety
+ * Define external connections (Kafka, S3, etc.) as TypeScript with full type safety
  */
 
 // Symbol for brand typing - use Symbol.for() for global registry
@@ -50,13 +50,50 @@ export interface KafkaConnectionDefinition {
   readonly options: KafkaConnectionOptions;
 }
 
+/**
+ * Options for defining an S3 connection
+ */
+export interface S3ConnectionOptions {
+  /** S3 bucket region (for example: us-east-1) */
+  region: string;
+  /** IAM role ARN used by Tinybird to access the bucket */
+  arn?: string;
+  /** S3 access key for key/secret auth */
+  accessKey?: string;
+  /** S3 secret key for key/secret auth */
+  secret?: string;
+}
+
+/**
+ * S3-specific connection definition
+ */
+export interface S3ConnectionDefinition {
+  readonly [CONNECTION_BRAND]: true;
+  /** Connection name */
+  readonly _name: string;
+  /** Type marker for inference */
+  readonly _type: "connection";
+  /** Connection type */
+  readonly _connectionType: "s3";
+  /** S3 options */
+  readonly options: S3ConnectionOptions;
+}
+
 /**
  * A connection definition - union of all connection types
  */
-export type ConnectionDefinition = KafkaConnectionDefinition;
+export type ConnectionDefinition = KafkaConnectionDefinition | S3ConnectionDefinition;
+
+function validateConnectionName(name: string): void {
+  if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) {
+    throw new Error(
+      `Invalid connection name: "${name}". Must start with a letter or underscore and contain only alphanumeric characters and underscores.`
+    );
+  }
+}
 
 /**
- * Create a Kafka connection
+ * Define a Kafka connection
  *
  * @param name - The connection name (must be valid identifier)
  * @param options - Kafka connection configuration
@@ -64,9 +101,9 @@ export type ConnectionDefinition = KafkaConnectionDefinition;
  *
  * @example
  * ```ts
- * import { createKafkaConnection } from '@tinybirdco/sdk';
+ * import { defineKafkaConnection } from '@tinybirdco/sdk';
  *
- * export const myKafka = createKafkaConnection('my_kafka', {
+ * export const myKafka = defineKafkaConnection('my_kafka', {
  *   bootstrapServers: 'kafka.example.com:9092',
  *   securityProtocol: 'SASL_SSL',
  *   saslMechanism: 'PLAIN',
@@ -75,22 +112,54 @@ export type ConnectionDefinition = KafkaConnectionDefinition;
  * });
  * ```
  */
-export function createKafkaConnection(
+export function defineKafkaConnection(
   name: string,
   options: KafkaConnectionOptions
 ): KafkaConnectionDefinition {
-  // Validate name is a valid identifier
-  if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) {
+  validateConnectionName(name);
+
+  return {
+    [CONNECTION_BRAND]: true,
+    _name: name,
+    _type: "connection",
+    _connectionType: "kafka",
+    options,
+  };
+}
+
+/**
+ * @deprecated Use defineKafkaConnection instead.
+ */
+export const createKafkaConnection = defineKafkaConnection;
+
+/**
+ * Define an S3 connection
+ *
+ * @param name - The connection name (must be valid identifier)
+ * @param options - S3 connection configuration
+ * @returns A connection definition that can be used in a project
+ */
+export function defineS3Connection(
+  name: string,
+  options: S3ConnectionOptions
+): S3ConnectionDefinition {
+  validateConnectionName(name);
+
+  if (!options.arn && !(options.accessKey && options.secret)) {
     throw new Error(
-      `Invalid connection name: "${name}". Must start with a letter or underscore and contain only alphanumeric characters and underscores.`
+      "S3 connection requires either `arn` or both `accessKey` and `secret`."
     );
   }
 
+  if ((options.accessKey && !options.secret) || (!options.accessKey && options.secret)) {
+    throw new Error("S3 connection `accessKey` and `secret` must be provided together.");
+  }
+
   return {
     [CONNECTION_BRAND]: true,
     _name: name,
     _type: "connection",
-    _connectionType: "kafka",
+    _connectionType: "s3",
     options,
   };
 }
@@ -114,6 +183,13 @@ export function isKafkaConnectionDefinition(value: unknown): value is KafkaConne
   return isConnectionDefinition(value) && value._connectionType === "kafka";
 }
 
+/**
+ * Check if a value is an S3 connection definition
+ */
+export function isS3ConnectionDefinition(value: unknown): value is S3ConnectionDefinition {
+  return isConnectionDefinition(value) && value._connectionType === "s3";
+}
+
 /**
  * Get the connection type from a connection definition
  */