@tinybirdco/sdk 0.0.77 → 0.0.78

package/src/migrate/parse-connection.ts

@@ -1,4 +1,5 @@
 import type {
+  DynamoDBConnectionModel,
   GCSConnectionModel,
   KafkaConnectionModel,
   ResourceFile,
@@ -31,6 +32,8 @@ const CONNECTION_DIRECTIVES = new Set([
   "S3_ACCESS_KEY",
   "S3_SECRET",
   "GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON",
+  "DYNAMODB_ARN",
+  "DYNAMODB_REGION",
 ]);
 
 function isConnectionDirectiveLine(line: string): boolean {
@@ -43,7 +46,7 @@ function isConnectionDirectiveLine(line: string): boolean {
 
 export function parseConnectionFile(
   resource: ResourceFile
-): KafkaConnectionModel | S3ConnectionModel | GCSConnectionModel {
+): KafkaConnectionModel | S3ConnectionModel | GCSConnectionModel | DynamoDBConnectionModel {
   const lines = splitLines(resource.content);
   let connectionType: string | undefined;
 
@@ -74,6 +77,9 @@ export function parseConnectionFile(
   let accessSecret: string | undefined;
   let serviceAccountCredentialsJson: string | undefined;
 
+  let dynamodbArn: string | undefined;
+  let dynamodbRegion: string | undefined;
+
   let i = 0;
   while (i < lines.length) {
     const rawLine = lines[i] ?? "";
@@ -177,6 +183,12 @@ export function parseConnectionFile(
         }
         serviceAccountCredentialsJson = parseQuotedValue(value);
         break;
+      case "DYNAMODB_ARN":
+        dynamodbArn = parseQuotedValue(value);
+        break;
+      case "DYNAMODB_REGION":
+        dynamodbRegion = parseQuotedValue(value);
+        break;
       default:
         throw new MigrationParseError(
           resource.filePath,
@@ -198,12 +210,20 @@ export function parseConnectionFile(
   }
 
   if (connectionType === "kafka") {
-    if (region || arn || accessKey || accessSecret || serviceAccountCredentialsJson) {
+    if (
+      region ||
+      arn ||
+      accessKey ||
+      accessSecret ||
+      serviceAccountCredentialsJson ||
+      dynamodbArn ||
+      dynamodbRegion
+    ) {
       throw new MigrationParseError(
         resource.filePath,
         "connection",
         resource.name,
-        "S3/GCS directives are not valid for kafka connections."
+        "S3/GCS/DynamoDB directives are not valid for kafka connections."
       );
     }
 
@@ -248,13 +268,15 @@ export function parseConnectionFile(
       secret ||
       schemaRegistryUrl ||
       sslCaPem ||
-      serviceAccountCredentialsJson
+      serviceAccountCredentialsJson ||
+      dynamodbArn ||
+      dynamodbRegion
     ) {
       throw new MigrationParseError(
         resource.filePath,
         "connection",
         resource.name,
-        "Kafka/GCS directives are not valid for s3 connections."
+        "Kafka/GCS/DynamoDB directives are not valid for s3 connections."
       );
     }
 
@@ -313,13 +335,15 @@ export function parseConnectionFile(
       region ||
       arn ||
       accessKey ||
-      accessSecret
+      accessSecret ||
+      dynamodbArn ||
+      dynamodbRegion
     ) {
       throw new MigrationParseError(
         resource.filePath,
         "connection",
         resource.name,
-        "Kafka/S3 directives are not valid for gcs connections."
+        "Kafka/S3/DynamoDB directives are not valid for gcs connections."
       );
     }
 
@@ -341,6 +365,61 @@ export function parseConnectionFile(
     };
   }
 
+  if (connectionType === "dynamodb") {
+    if (
+      bootstrapServers ||
+      securityProtocol ||
+      saslMechanism ||
+      saslOauthbearerMethod ||
+      saslOauthbearerAwsRegion ||
+      saslOauthbearerAwsRoleArn ||
+      saslOauthbearerAwsExternalId ||
+      key ||
+      secret ||
+      schemaRegistryUrl ||
+      sslCaPem ||
+      region ||
+      arn ||
+      accessKey ||
+      accessSecret ||
+      serviceAccountCredentialsJson
+    ) {
+      throw new MigrationParseError(
+        resource.filePath,
+        "connection",
+        resource.name,
+        "Kafka/S3/GCS directives are not valid for dynamodb connections."
+      );
+    }
+
+    if (!dynamodbArn) {
+      throw new MigrationParseError(
+        resource.filePath,
+        "connection",
+        resource.name,
+        "DYNAMODB_ARN is required for dynamodb connections."
+      );
+    }
+
+    if (!dynamodbRegion) {
+      throw new MigrationParseError(
+        resource.filePath,
+        "connection",
+        resource.name,
+        "DYNAMODB_REGION is required for dynamodb connections."
+      );
+    }
+
+    return {
+      kind: "connection",
+      name: resource.name,
+      filePath: resource.filePath,
+      connectionType: "dynamodb",
+      arn: dynamodbArn,
+      region: dynamodbRegion,
+    };
+  }
+
   throw new MigrationParseError(
     resource.filePath,
     "connection",

package/src/migrate/parse-datasource.ts

@@ -39,6 +39,8 @@ const DATASOURCE_DIRECTIVES = new Set([
   "IMPORT_BUCKET_URI",
   "IMPORT_SCHEDULE",
   "IMPORT_FROM_TIMESTAMP",
+  "IMPORT_TABLE_ARN",
+  "IMPORT_EXPORT_BUCKET",
   "TOKEN",
 ]);
 
@@ -321,6 +323,8 @@ export function parseDatasourceFile(resource: ResourceFile): DatasourceModel {
   let importBucketUri: string | undefined;
   let importSchedule: string | undefined;
   let importFromTimestamp: string | undefined;
+  let importTableArn: string | undefined;
+  let importExportBucket: string | undefined;
 
   let i = 0;
   while (i < lines.length) {
@@ -498,6 +502,12 @@ export function parseDatasourceFile(resource: ResourceFile): DatasourceModel {
       case "IMPORT_FROM_TIMESTAMP":
         importFromTimestamp = parseQuotedValue(value);
         break;
+      case "IMPORT_TABLE_ARN":
+        importTableArn = parseQuotedValue(value);
+        break;
+      case "IMPORT_EXPORT_BUCKET":
+        importExportBucket = parseQuotedValue(value);
+        break;
       case "TOKEN":
         tokens.push(parseToken(resource.filePath, resource.name, value));
         break;
@@ -573,8 +583,30 @@ export function parseDatasourceFile(resource: ResourceFile): DatasourceModel {
     );
   }
 
+  // DynamoDB import directives share IMPORT_CONNECTION_NAME with blob storage,
+  // so treat the import block as DynamoDB whenever a DynamoDB-only directive is present.
+  const isDynamoDB = importTableArn !== undefined || importExportBucket !== undefined;
+
+  const dynamodb = isDynamoDB
+    ? {
+        connectionName: importConnectionName ?? "",
+        tableArn: importTableArn ?? "",
+        exportBucket: importExportBucket ?? "",
+      }
+    : undefined;
+
+  if (dynamodb && (!dynamodb.connectionName || !dynamodb.tableArn || !dynamodb.exportBucket)) {
+    throw new MigrationParseError(
+      resource.filePath,
+      "datasource",
+      resource.name,
+      "IMPORT_CONNECTION_NAME, IMPORT_TABLE_ARN and IMPORT_EXPORT_BUCKET are required when DynamoDB directives are used."
+    );
+  }
+
   const s3 =
-    importConnectionName || importBucketUri || importSchedule || importFromTimestamp
+    !isDynamoDB &&
+    (importConnectionName || importBucketUri || importSchedule || importFromTimestamp)
       ? {
           connectionName: importConnectionName ?? "",
           bucketUri: importBucketUri ?? "",
@@ -592,7 +624,7 @@ export function parseDatasourceFile(resource: ResourceFile): DatasourceModel {
     );
   }
 
-  if (kafka && s3) {
+  if (kafka && (s3 || dynamodb)) {
     throw new MigrationParseError(
       resource.filePath,
       "datasource",
@@ -625,6 +657,7 @@ export function parseDatasourceFile(resource: ResourceFile): DatasourceModel {
     indexes,
     kafka,
     s3,
+    dynamodb,
     forwardQuery,
     tokens,
     sharedWith,

package/src/migrate/types.ts

@@ -59,6 +59,12 @@ export interface DatasourceGCSModel {
   fromTimestamp?: string;
 }
 
+export interface DatasourceDynamoDBModel {
+  connectionName: string;
+  tableArn: string;
+  exportBucket: string;
+}
+
 export interface DatasourceTokenModel {
   name: string;
   scope: "READ" | "APPEND";
@@ -82,6 +88,7 @@ export interface DatasourceModel {
   kafka?: DatasourceKafkaModel;
   s3?: DatasourceS3Model;
   gcs?: DatasourceGCSModel;
+  dynamodb?: DatasourceDynamoDBModel;
   forwardQuery?: string;
   tokens: DatasourceTokenModel[];
   sharedWith: string[];
@@ -184,12 +191,22 @@ export interface GCSConnectionModel {
   serviceAccountCredentialsJson: string;
 }
 
+export interface DynamoDBConnectionModel {
+  kind: "connection";
+  name: string;
+  filePath: string;
+  connectionType: "dynamodb";
+  region: string;
+  arn: string;
+}
+
 export type ParsedResource =
   | DatasourceModel
   | PipeModel
   | KafkaConnectionModel
   | S3ConnectionModel
-  | GCSConnectionModel;
+  | GCSConnectionModel
+  | DynamoDBConnectionModel;
 
 export interface MigrationResult {
   success: boolean;

package/src/schema/connection.test.ts

@@ -3,10 +3,12 @@ import {
   defineKafkaConnection,
   defineS3Connection,
   defineGCSConnection,
+  defineDynamoDBConnection,
   isConnectionDefinition,
   isKafkaConnectionDefinition,
   isS3ConnectionDefinition,
   isGCSConnectionDefinition,
+  isDynamoDBConnectionDefinition,
   getConnectionType,
 } from "./connection.js";
 
@@ -199,6 +201,48 @@ describe("Connection Schema", () => {
     });
   });
 
+  describe("defineDynamoDBConnection", () => {
+    it("creates a DynamoDB connection with required fields", () => {
+      const conn = defineDynamoDBConnection("my_dynamo", {
+        region: "us-east-1",
+        arn: '{{ tb_secret("DYNAMODB_ROLE_ARN") }}',
+      });
+
+      expect(conn._name).toBe("my_dynamo");
+      expect(conn._type).toBe("connection");
+      expect(conn._connectionType).toBe("dynamodb");
+      expect(conn.options.region).toBe("us-east-1");
+      expect(conn.options.arn).toBe('{{ tb_secret("DYNAMODB_ROLE_ARN") }}');
+    });
+
+    it("throws when region is missing", () => {
+      expect(() =>
+        defineDynamoDBConnection("my_dynamo", {
+          region: "   ",
+          arn: '{{ tb_secret("DYNAMODB_ROLE_ARN") }}',
+        })
+      ).toThrow("DynamoDB connection `region` is required.");
+    });
+
+    it("throws when arn is missing", () => {
+      expect(() =>
+        defineDynamoDBConnection("my_dynamo", {
+          region: "us-east-1",
+          arn: "",
+        })
+      ).toThrow("DynamoDB connection `arn` is required.");
+    });
+
+    it("throws for invalid connection name", () => {
+      expect(() =>
+        defineDynamoDBConnection("123-invalid", {
+          region: "us-east-1",
+          arn: '{{ tb_secret("DYNAMODB_ROLE_ARN") }}',
+        })
+      ).toThrow("Invalid connection name");
+    });
+  });
+
   describe("isConnectionDefinition", () => {
     it("returns true for valid connection", () => {
       const conn = defineKafkaConnection("my_kafka", {
@@ -264,6 +308,27 @@ describe("Connection Schema", () => {
     });
   });
 
+  describe("isDynamoDBConnectionDefinition", () => {
+    it("returns true for DynamoDB connection", () => {
+      const conn = defineDynamoDBConnection("my_dynamo", {
+        region: "us-east-1",
+        arn: '{{ tb_secret("DYNAMODB_ROLE_ARN") }}',
+      });
+
+      expect(isDynamoDBConnectionDefinition(conn)).toBe(true);
+    });
+
+    it("returns false for non-DynamoDB objects", () => {
+      expect(isDynamoDBConnectionDefinition({})).toBe(false);
+      expect(isDynamoDBConnectionDefinition(null)).toBe(false);
+      expect(
+        isDynamoDBConnectionDefinition(
+          defineS3Connection("my_s3", { region: "us-east-1", arn: "arn:aws:iam::1:role/x" })
+        )
+      ).toBe(false);
+    });
+  });
+
   describe("getConnectionType", () => {
     it("returns the connection type", () => {
       const conn = defineKafkaConnection("my_kafka", {

package/src/schema/connection.ts

@@ -117,13 +117,39 @@ export interface GCSConnectionDefinition {
   readonly options: GCSConnectionOptions;
 }
 
+/**
+ * Options for defining a DynamoDB connection
+ */
+export interface DynamoDBConnectionOptions {
+  /** DynamoDB table region (for example: us-east-1) */
+  region: string;
+  /** IAM role ARN used by Tinybird to access the table - can use {{ tb_secret(...) }} */
+  arn: string;
+}
+
+/**
+ * DynamoDB-specific connection definition
+ */
+export interface DynamoDBConnectionDefinition {
+  readonly [CONNECTION_BRAND]: true;
+  /** Connection name */
+  readonly _name: string;
+  /** Type marker for inference */
+  readonly _type: "connection";
+  /** Connection type */
+  readonly _connectionType: "dynamodb";
+  /** DynamoDB options */
+  readonly options: DynamoDBConnectionOptions;
+}
+
 /**
  * A connection definition - union of all connection types
  */
 export type ConnectionDefinition =
   | KafkaConnectionDefinition
   | S3ConnectionDefinition
-  | GCSConnectionDefinition;
+  | GCSConnectionDefinition
+  | DynamoDBConnectionDefinition;
 
 function validateConnectionName(name: string): void {
   if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) {
@@ -231,6 +257,46 @@ export function defineGCSConnection(
   };
 }
 
+/**
+ * Define a DynamoDB connection
+ *
+ * @param name - The connection name (must be valid identifier)
+ * @param options - DynamoDB connection configuration
+ * @returns A connection definition that can be used in a project
+ *
+ * @example
+ * ```ts
+ * import { defineDynamoDBConnection } from '@tinybirdco/sdk';
+ *
+ * export const myDynamo = defineDynamoDBConnection('my_dynamo', {
+ *   region: 'us-east-1',
+ *   arn: '{{ tb_secret("DYNAMODB_ROLE_ARN") }}',
+ * });
+ * ```
+ */
+export function defineDynamoDBConnection(
+  name: string,
+  options: DynamoDBConnectionOptions
+): DynamoDBConnectionDefinition {
+  validateConnectionName(name);
+
+  if (!options.region?.trim()) {
+    throw new Error("DynamoDB connection `region` is required.");
+  }
+
+  if (!options.arn?.trim()) {
+    throw new Error("DynamoDB connection `arn` is required.");
+  }
+
+  return {
+    [CONNECTION_BRAND]: true,
+    _name: name,
+    _type: "connection",
+    _connectionType: "dynamodb",
+    options,
+  };
+}
+
 /**
  * Check if a value is a connection definition
  */
@@ -264,6 +330,15 @@ export function isGCSConnectionDefinition(value: unknown): value is GCSConnectio
   return isConnectionDefinition(value) && value._connectionType === "gcs";
 }
 
+/**
+ * Check if a value is a DynamoDB connection definition
+ */
+export function isDynamoDBConnectionDefinition(
+  value: unknown
+): value is DynamoDBConnectionDefinition {
+  return isConnectionDefinition(value) && value._connectionType === "dynamodb";
+}
+
 /**
  * Get the connection type from a connection definition
  */

package/src/schema/datasource.test.ts

@@ -112,7 +112,7 @@ describe("Datasource Schema", () => {
             bucketUri: "s3://my-bucket/events/*.csv",
           },
         })
-      ).toThrow("Datasource can only define one ingestion option: `kafka`, `s3`, or `gcs`.");
+      ).toThrow("Datasource can only define one ingestion option: `kafka`, `s3`, `gcs`, or `dynamodb`.");
 
       expect(() =>
         defineDatasource("events_gcs", {
@@ -126,7 +126,7 @@ describe("Datasource Schema", () => {
             bucketUri: "gs://my-bucket/events/*.csv",
           },
         })
-      ).toThrow("Datasource can only define one ingestion option: `kafka`, `s3`, or `gcs`.");
+      ).toThrow("Datasource can only define one ingestion option: `kafka`, `s3`, `gcs`, or `dynamodb`.");
     });
 
     it("accepts gcs ingestion configuration", () => {

package/src/schema/datasource.ts

@@ -9,6 +9,7 @@ import type {
   KafkaConnectionDefinition,
   S3ConnectionDefinition,
   GCSConnectionDefinition,
+  DynamoDBConnectionDefinition,
 } from "./connection.js";
 import type { TokenDefinition, DatasourceTokenScope } from "./token.js";
 
@@ -102,6 +103,18 @@ export interface GCSConfig {
   fromTimestamp?: string;
 }
 
+/**
+ * DynamoDB import configuration for a datasource
+ */
+export interface DynamoDBConfig {
+  /** DynamoDB connection to use */
+  connection: DynamoDBConnectionDefinition;
+  /** Source DynamoDB table ARN */
+  tableArn: string;
+  /** S3 bucket Tinybird uses to stage the initial table export */
+  exportBucket: string;
+}
+
 /**
  * Datasource index configuration.
  * Emits as: `<name> <expr> TYPE <type> GRANULARITY <n>`
@@ -152,6 +165,8 @@ export interface DatasourceOptions<TSchema extends SchemaDefinition> {
   s3?: S3Config;
   /** GCS ingestion configuration */
   gcs?: GCSConfig;
+  /** DynamoDB ingestion configuration */
+  dynamodb?: DynamoDBConfig;
 }
 
 /**
@@ -209,9 +224,13 @@ export function defineDatasource<TSchema extends SchemaDefinition>(
     );
   }
 
-  const ingestionConfigCount = [options.kafka, options.s3, options.gcs].filter(Boolean).length;
+  const ingestionConfigCount = [options.kafka, options.s3, options.gcs, options.dynamodb].filter(
+    Boolean
+  ).length;
   if (ingestionConfigCount > 1) {
-    throw new Error("Datasource can only define one ingestion option: `kafka`, `s3`, or `gcs`.");
+    throw new Error(
+      "Datasource can only define one ingestion option: `kafka`, `s3`, `gcs`, or `dynamodb`."
+    );
   }
 
   if (options.indexes) {