@tinybirdco/sdk 0.0.77 → 0.0.79

package/src/index.ts

@@ -116,6 +116,7 @@ export type {
   KafkaConfig,
   S3Config,
   GCSConfig,
+  DynamoDBConfig,
   DatasourceIndex,
 } from "./schema/datasource.js";
 
@@ -125,10 +126,12 @@ export {
   createKafkaConnection,
   defineS3Connection,
   defineGCSConnection,
+  defineDynamoDBConnection,
   isConnectionDefinition,
   isKafkaConnectionDefinition,
   isS3ConnectionDefinition,
   isGCSConnectionDefinition,
+  isDynamoDBConnectionDefinition,
   getConnectionType,
 } from "./schema/connection.js";
 export type {
@@ -142,6 +145,8 @@ export type {
   S3ConnectionOptions,
   GCSConnectionDefinition,
   GCSConnectionOptions,
+  DynamoDBConnectionDefinition,
+  DynamoDBConnectionOptions,
 } from "./schema/connection.js";
 
 // ============ Token ============
@@ -312,4 +317,4 @@ export type {
 
 // ============ Config Types ============
 // Import from config-types.ts to avoid bundling esbuild in client code
-export type { TinybirdConfig, DevMode } from "./cli/config-types.js";
+export type { TinybirdConfig, DevMode, BranchDataMode } from "./cli/config-types.js";

package/src/migrate/emit-ts.ts

@@ -4,6 +4,7 @@ import { parseLiteralFromDatafile, toTsLiteral } from "./parser-utils.js";
 import type {
   DatasourceModel,
   DatasourceEngineModel,
+  DynamoDBConnectionModel,
   GCSConnectionModel,
   KafkaConnectionModel,
   ParsedResource,
@@ -72,6 +73,9 @@ function hasSecretTemplate(resources: ParsedResource[]): boolean {
         if (resource.arn) values.push(resource.arn);
         if (resource.accessKey) values.push(resource.accessKey);
         if (resource.secret) values.push(resource.secret);
+      } else if (resource.connectionType === "dynamodb") {
+        values.push(resource.region);
+        values.push(resource.arn);
       } else {
         values.push(resource.serviceAccountCredentialsJson);
       }
@@ -95,6 +99,10 @@ function hasSecretTemplate(resources: ParsedResource[]): boolean {
         if (resource.gcs.schedule) values.push(resource.gcs.schedule);
         if (resource.gcs.fromTimestamp) values.push(resource.gcs.fromTimestamp);
       }
+      if (resource.dynamodb) {
+        values.push(resource.dynamodb.tableArn);
+        values.push(resource.dynamodb.exportBucket);
+      }
       continue;
     }
   }
@@ -388,6 +396,15 @@ function emitDatasource(ds: DatasourceModel): string {
     lines.push("  },");
   }
 
+  if (ds.dynamodb) {
+    const connectionVar = toCamelCase(ds.dynamodb.connectionName);
+    lines.push("  dynamodb: {");
+    lines.push(`    connection: ${connectionVar},`);
+    lines.push(`    tableArn: ${emitStringOrSecret(ds.dynamodb.tableArn)},`);
+    lines.push(`    exportBucket: ${emitStringOrSecret(ds.dynamodb.exportBucket)},`);
+    lines.push("  },");
+  }
+
   if (ds.forwardQuery) {
     lines.push("  forwardQuery: `");
     lines.push(escapeTemplateLiteral(ds.forwardQuery));
@@ -416,7 +433,11 @@ function emitDatasource(ds: DatasourceModel): string {
 }
 
 function emitConnection(
-  connection: KafkaConnectionModel | S3ConnectionModel | GCSConnectionModel
+  connection:
+    | KafkaConnectionModel
+    | S3ConnectionModel
+    | GCSConnectionModel
+    | DynamoDBConnectionModel
 ): string {
   const variableName = toCamelCase(connection.name);
   const lines: string[] = [];
@@ -486,6 +507,17 @@ function emitConnection(
     return lines.join("\n");
   }
 
+  if (connection.connectionType === "dynamodb") {
+    lines.push(
+      `export const ${variableName} = defineDynamoDBConnection(${escapeString(connection.name)}, {`
+    );
+    lines.push(`  region: ${emitStringOrSecret(connection.region)},`);
+    lines.push(`  arn: ${emitStringOrSecret(connection.arn)},`);
+    lines.push("});");
+    lines.push("");
+    return lines.join("\n");
+  }
+
   lines.push(
     `export const ${variableName} = defineGCSConnection(${escapeString(connection.name)}, {`
   );
@@ -625,8 +657,13 @@ function emitPipe(pipe: PipeModel): string {
 
 export function emitMigrationFileContent(resources: ParsedResource[]): string {
   const connections = resources.filter(
-    (resource): resource is KafkaConnectionModel | S3ConnectionModel | GCSConnectionModel =>
-      resource.kind === "connection"
+    (
+      resource
+    ): resource is
+      | KafkaConnectionModel
+      | S3ConnectionModel
+      | GCSConnectionModel
+      | DynamoDBConnectionModel => resource.kind === "connection"
   );
   const datasources = resources.filter(
     (resource): resource is DatasourceModel => resource.kind === "datasource"
@@ -655,6 +692,9 @@ export function emitMigrationFileContent(resources: ParsedResource[]): string {
   if (connections.some((connection) => connection.connectionType === "gcs")) {
     imports.add("defineGCSConnection");
   }
+  if (connections.some((connection) => connection.connectionType === "dynamodb")) {
+    imports.add("defineDynamoDBConnection");
+  }
   if (needsParams) {
     imports.add("p");
   }
@@ -672,6 +712,7 @@ export function emitMigrationFileContent(resources: ParsedResource[]): string {
     "defineKafkaConnection",
     "defineS3Connection",
     "defineGCSConnection",
+    "defineDynamoDBConnection",
     "defineDatasource",
     "definePipe",
     "defineMaterializedView",

package/src/migrate/parse-connection.test.ts

@@ -122,4 +122,45 @@ KAFKA_SASL_OAUTHBEARER_AWS_EXTERNAL_ID {{ tb_secret("KAFKA_AWS_EXTERNAL_ID") }}`
       '{{ tb_secret("KAFKA_AWS_EXTERNAL_ID") }}'
     );
   });
+
+  it("parses basic DynamoDB connection", () => {
+    const result = parseConnectionFile(
+      resource(
+        "my_dynamo",
+        `TYPE dynamodb
+DYNAMODB_ARN {{ tb_secret("DYNAMODB_ROLE_ARN") }}
+DYNAMODB_REGION us-east-1`
+      )
+    );
+
+    expect(result.connectionType).toBe("dynamodb");
+    expect(result).toHaveProperty("arn", '{{ tb_secret("DYNAMODB_ROLE_ARN") }}');
+    expect(result).toHaveProperty("region", "us-east-1");
+  });
+
+  it("throws when DynamoDB connection is missing DYNAMODB_ARN", () => {
+    expect(() =>
+      parseConnectionFile(
+        resource(
+          "my_dynamo",
+          `TYPE dynamodb
+DYNAMODB_REGION us-east-1`
+        )
+      )
+    ).toThrow("DYNAMODB_ARN is required");
+  });
+
+  it("throws when DynamoDB connection mixes S3 directives", () => {
+    expect(() =>
+      parseConnectionFile(
+        resource(
+          "my_dynamo",
+          `TYPE dynamodb
+DYNAMODB_ARN {{ tb_secret("DYNAMODB_ROLE_ARN") }}
+DYNAMODB_REGION us-east-1
+S3_ARN arn:aws:iam::1:role/x`
+        )
+      )
+    ).toThrow("Kafka/S3/GCS directives are not valid for dynamodb connections.");
+  });
 });

package/src/migrate/parse-connection.ts

@@ -1,4 +1,5 @@
 import type {
+  DynamoDBConnectionModel,
   GCSConnectionModel,
   KafkaConnectionModel,
   ResourceFile,
@@ -31,6 +32,8 @@ const CONNECTION_DIRECTIVES = new Set([
   "S3_ACCESS_KEY",
   "S3_SECRET",
   "GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON",
+  "DYNAMODB_ARN",
+  "DYNAMODB_REGION",
 ]);
 
 function isConnectionDirectiveLine(line: string): boolean {
@@ -43,7 +46,7 @@ function isConnectionDirectiveLine(line: string): boolean {
 
 export function parseConnectionFile(
   resource: ResourceFile
-): KafkaConnectionModel | S3ConnectionModel | GCSConnectionModel {
+): KafkaConnectionModel | S3ConnectionModel | GCSConnectionModel | DynamoDBConnectionModel {
   const lines = splitLines(resource.content);
   let connectionType: string | undefined;
 
@@ -74,6 +77,9 @@ export function parseConnectionFile(
   let accessSecret: string | undefined;
   let serviceAccountCredentialsJson: string | undefined;
 
+  let dynamodbArn: string | undefined;
+  let dynamodbRegion: string | undefined;
+
   let i = 0;
   while (i < lines.length) {
     const rawLine = lines[i] ?? "";
@@ -177,6 +183,12 @@ export function parseConnectionFile(
         }
         serviceAccountCredentialsJson = parseQuotedValue(value);
         break;
+      case "DYNAMODB_ARN":
+        dynamodbArn = parseQuotedValue(value);
+        break;
+      case "DYNAMODB_REGION":
+        dynamodbRegion = parseQuotedValue(value);
+        break;
       default:
         throw new MigrationParseError(
           resource.filePath,
@@ -198,12 +210,20 @@ export function parseConnectionFile(
   }
 
   if (connectionType === "kafka") {
-    if (region || arn || accessKey || accessSecret || serviceAccountCredentialsJson) {
+    if (
+      region ||
+      arn ||
+      accessKey ||
+      accessSecret ||
+      serviceAccountCredentialsJson ||
+      dynamodbArn ||
+      dynamodbRegion
+    ) {
       throw new MigrationParseError(
         resource.filePath,
         "connection",
         resource.name,
-        "S3/GCS directives are not valid for kafka connections."
+        "S3/GCS/DynamoDB directives are not valid for kafka connections."
       );
     }
 
@@ -248,13 +268,15 @@ export function parseConnectionFile(
       secret ||
       schemaRegistryUrl ||
       sslCaPem ||
-      serviceAccountCredentialsJson
+      serviceAccountCredentialsJson ||
+      dynamodbArn ||
+      dynamodbRegion
     ) {
       throw new MigrationParseError(
         resource.filePath,
         "connection",
         resource.name,
-        "Kafka/GCS directives are not valid for s3 connections."
+        "Kafka/GCS/DynamoDB directives are not valid for s3 connections."
       );
     }
 
@@ -313,13 +335,15 @@ export function parseConnectionFile(
       region ||
       arn ||
       accessKey ||
-      accessSecret
+      accessSecret ||
+      dynamodbArn ||
+      dynamodbRegion
     ) {
       throw new MigrationParseError(
         resource.filePath,
         "connection",
         resource.name,
-        "Kafka/S3 directives are not valid for gcs connections."
+        "Kafka/S3/DynamoDB directives are not valid for gcs connections."
       );
     }
 
@@ -341,6 +365,61 @@ export function parseConnectionFile(
     };
   }
 
+  if (connectionType === "dynamodb") {
+    if (
+      bootstrapServers ||
+      securityProtocol ||
+      saslMechanism ||
+      saslOauthbearerMethod ||
+      saslOauthbearerAwsRegion ||
+      saslOauthbearerAwsRoleArn ||
+      saslOauthbearerAwsExternalId ||
+      key ||
+      secret ||
+      schemaRegistryUrl ||
+      sslCaPem ||
+      region ||
+      arn ||
+      accessKey ||
+      accessSecret ||
+      serviceAccountCredentialsJson
+    ) {
+      throw new MigrationParseError(
+        resource.filePath,
+        "connection",
+        resource.name,
+        "Kafka/S3/GCS directives are not valid for dynamodb connections."
+      );
+    }
+
+    if (!dynamodbArn) {
+      throw new MigrationParseError(
+        resource.filePath,
+        "connection",
+        resource.name,
+        "DYNAMODB_ARN is required for dynamodb connections."
+      );
+    }
+
+    if (!dynamodbRegion) {
+      throw new MigrationParseError(
+        resource.filePath,
+        "connection",
+        resource.name,
+        "DYNAMODB_REGION is required for dynamodb connections."
+      );
+    }
+
+    return {
+      kind: "connection",
+      name: resource.name,
+      filePath: resource.filePath,
+      connectionType: "dynamodb",
+      arn: dynamodbArn,
+      region: dynamodbRegion,
+    };
+  }
+
   throw new MigrationParseError(
     resource.filePath,
     "connection",

package/src/migrate/parse-datasource.ts

@@ -39,6 +39,8 @@ const DATASOURCE_DIRECTIVES = new Set([
   "IMPORT_BUCKET_URI",
   "IMPORT_SCHEDULE",
   "IMPORT_FROM_TIMESTAMP",
+  "IMPORT_TABLE_ARN",
+  "IMPORT_EXPORT_BUCKET",
   "TOKEN",
 ]);
 
@@ -321,6 +323,8 @@ export function parseDatasourceFile(resource: ResourceFile): DatasourceModel {
   let importBucketUri: string | undefined;
   let importSchedule: string | undefined;
   let importFromTimestamp: string | undefined;
+  let importTableArn: string | undefined;
+  let importExportBucket: string | undefined;
 
   let i = 0;
   while (i < lines.length) {
@@ -498,6 +502,12 @@ export function parseDatasourceFile(resource: ResourceFile): DatasourceModel {
       case "IMPORT_FROM_TIMESTAMP":
         importFromTimestamp = parseQuotedValue(value);
         break;
+      case "IMPORT_TABLE_ARN":
+        importTableArn = parseQuotedValue(value);
+        break;
+      case "IMPORT_EXPORT_BUCKET":
+        importExportBucket = parseQuotedValue(value);
+        break;
       case "TOKEN":
         tokens.push(parseToken(resource.filePath, resource.name, value));
         break;
@@ -573,8 +583,30 @@ export function parseDatasourceFile(resource: ResourceFile): DatasourceModel {
     );
   }
 
+  // DynamoDB import directives share IMPORT_CONNECTION_NAME with blob storage,
+  // so treat the import block as DynamoDB whenever a DynamoDB-only directive is present.
+  const isDynamoDB = importTableArn !== undefined || importExportBucket !== undefined;
+
+  const dynamodb = isDynamoDB
+    ? {
+        connectionName: importConnectionName ?? "",
+        tableArn: importTableArn ?? "",
+        exportBucket: importExportBucket ?? "",
+      }
+    : undefined;
+
+  if (dynamodb && (!dynamodb.connectionName || !dynamodb.tableArn || !dynamodb.exportBucket)) {
+    throw new MigrationParseError(
+      resource.filePath,
+      "datasource",
+      resource.name,
+      "IMPORT_CONNECTION_NAME, IMPORT_TABLE_ARN and IMPORT_EXPORT_BUCKET are required when DynamoDB directives are used."
+    );
+  }
+
   const s3 =
-    importConnectionName || importBucketUri || importSchedule || importFromTimestamp
+    !isDynamoDB &&
+    (importConnectionName || importBucketUri || importSchedule || importFromTimestamp)
       ? {
           connectionName: importConnectionName ?? "",
           bucketUri: importBucketUri ?? "",
@@ -592,7 +624,7 @@ export function parseDatasourceFile(resource: ResourceFile): DatasourceModel {
     );
   }
 
-  if (kafka && s3) {
+  if (kafka && (s3 || dynamodb)) {
     throw new MigrationParseError(
       resource.filePath,
       "datasource",
@@ -625,6 +657,7 @@ export function parseDatasourceFile(resource: ResourceFile): DatasourceModel {
     indexes,
     kafka,
     s3,
+    dynamodb,
     forwardQuery,
     tokens,
     sharedWith,

package/src/migrate/types.ts

@@ -59,6 +59,12 @@ export interface DatasourceGCSModel {
   fromTimestamp?: string;
 }
 
+export interface DatasourceDynamoDBModel {
+  connectionName: string;
+  tableArn: string;
+  exportBucket: string;
+}
+
 export interface DatasourceTokenModel {
   name: string;
   scope: "READ" | "APPEND";
@@ -82,6 +88,7 @@ export interface DatasourceModel {
   kafka?: DatasourceKafkaModel;
   s3?: DatasourceS3Model;
   gcs?: DatasourceGCSModel;
+  dynamodb?: DatasourceDynamoDBModel;
   forwardQuery?: string;
   tokens: DatasourceTokenModel[];
   sharedWith: string[];
@@ -184,12 +191,22 @@ export interface GCSConnectionModel {
   serviceAccountCredentialsJson: string;
 }
 
+export interface DynamoDBConnectionModel {
+  kind: "connection";
+  name: string;
+  filePath: string;
+  connectionType: "dynamodb";
+  region: string;
+  arn: string;
+}
+
 export type ParsedResource =
   | DatasourceModel
   | PipeModel
   | KafkaConnectionModel
   | S3ConnectionModel
-  | GCSConnectionModel;
+  | GCSConnectionModel
+  | DynamoDBConnectionModel;
 
 export interface MigrationResult {
   success: boolean;

package/src/schema/connection.test.ts

@@ -3,10 +3,12 @@ import {
   defineKafkaConnection,
   defineS3Connection,
   defineGCSConnection,
+  defineDynamoDBConnection,
   isConnectionDefinition,
   isKafkaConnectionDefinition,
   isS3ConnectionDefinition,
   isGCSConnectionDefinition,
+  isDynamoDBConnectionDefinition,
   getConnectionType,
 } from "./connection.js";
 
@@ -199,6 +201,48 @@ describe("Connection Schema", () => {
     });
   });
 
+  describe("defineDynamoDBConnection", () => {
+    it("creates a DynamoDB connection with required fields", () => {
+      const conn = defineDynamoDBConnection("my_dynamo", {
+        region: "us-east-1",
+        arn: '{{ tb_secret("DYNAMODB_ROLE_ARN") }}',
+      });
+
+      expect(conn._name).toBe("my_dynamo");
+      expect(conn._type).toBe("connection");
+      expect(conn._connectionType).toBe("dynamodb");
+      expect(conn.options.region).toBe("us-east-1");
+      expect(conn.options.arn).toBe('{{ tb_secret("DYNAMODB_ROLE_ARN") }}');
+    });
+
+    it("throws when region is missing", () => {
+      expect(() =>
+        defineDynamoDBConnection("my_dynamo", {
+          region: "   ",
+          arn: '{{ tb_secret("DYNAMODB_ROLE_ARN") }}',
+        })
+      ).toThrow("DynamoDB connection `region` is required.");
+    });
+
+    it("throws when arn is missing", () => {
+      expect(() =>
+        defineDynamoDBConnection("my_dynamo", {
+          region: "us-east-1",
+          arn: "",
+        })
+      ).toThrow("DynamoDB connection `arn` is required.");
+    });
+
+    it("throws for invalid connection name", () => {
+      expect(() =>
+        defineDynamoDBConnection("123-invalid", {
+          region: "us-east-1",
+          arn: '{{ tb_secret("DYNAMODB_ROLE_ARN") }}',
+        })
+      ).toThrow("Invalid connection name");
+    });
+  });
+
   describe("isConnectionDefinition", () => {
     it("returns true for valid connection", () => {
       const conn = defineKafkaConnection("my_kafka", {
@@ -264,6 +308,27 @@ describe("Connection Schema", () => {
     });
   });
 
+  describe("isDynamoDBConnectionDefinition", () => {
+    it("returns true for DynamoDB connection", () => {
+      const conn = defineDynamoDBConnection("my_dynamo", {
+        region: "us-east-1",
+        arn: '{{ tb_secret("DYNAMODB_ROLE_ARN") }}',
+      });
+
+      expect(isDynamoDBConnectionDefinition(conn)).toBe(true);
+    });
+
+    it("returns false for non-DynamoDB objects", () => {
+      expect(isDynamoDBConnectionDefinition({})).toBe(false);
+      expect(isDynamoDBConnectionDefinition(null)).toBe(false);
+      expect(
+        isDynamoDBConnectionDefinition(
+          defineS3Connection("my_s3", { region: "us-east-1", arn: "arn:aws:iam::1:role/x" })
+        )
+      ).toBe(false);
+    });
+  });
+
   describe("getConnectionType", () => {
     it("returns the connection type", () => {
       const conn = defineKafkaConnection("my_kafka", {

package/src/schema/connection.ts

@@ -117,13 +117,39 @@ export interface GCSConnectionDefinition {
   readonly options: GCSConnectionOptions;
 }
 
+/**
+ * Options for defining a DynamoDB connection
+ */
+export interface DynamoDBConnectionOptions {
+  /** DynamoDB table region (for example: us-east-1) */
+  region: string;
+  /** IAM role ARN used by Tinybird to access the table - can use {{ tb_secret(...) }} */
+  arn: string;
+}
+
+/**
+ * DynamoDB-specific connection definition
+ */
+export interface DynamoDBConnectionDefinition {
+  readonly [CONNECTION_BRAND]: true;
+  /** Connection name */
+  readonly _name: string;
+  /** Type marker for inference */
+  readonly _type: "connection";
+  /** Connection type */
+  readonly _connectionType: "dynamodb";
+  /** DynamoDB options */
+  readonly options: DynamoDBConnectionOptions;
+}
+
 /**
  * A connection definition - union of all connection types
  */
 export type ConnectionDefinition =
   | KafkaConnectionDefinition
   | S3ConnectionDefinition
-  | GCSConnectionDefinition;
+  | GCSConnectionDefinition
+  | DynamoDBConnectionDefinition;
 
 function validateConnectionName(name: string): void {
   if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) {
@@ -231,6 +257,46 @@ export function defineGCSConnection(
   };
 }
 
+/**
+ * Define a DynamoDB connection
+ *
+ * @param name - The connection name (must be valid identifier)
+ * @param options - DynamoDB connection configuration
+ * @returns A connection definition that can be used in a project
+ *
+ * @example
+ * ```ts
+ * import { defineDynamoDBConnection } from '@tinybirdco/sdk';
+ *
+ * export const myDynamo = defineDynamoDBConnection('my_dynamo', {
+ *   region: 'us-east-1',
+ *   arn: '{{ tb_secret("DYNAMODB_ROLE_ARN") }}',
+ * });
+ * ```
+ */
+export function defineDynamoDBConnection(
+  name: string,
+  options: DynamoDBConnectionOptions
+): DynamoDBConnectionDefinition {
+  validateConnectionName(name);
+
+  if (!options.region?.trim()) {
+    throw new Error("DynamoDB connection `region` is required.");
+  }
+
+  if (!options.arn?.trim()) {
+    throw new Error("DynamoDB connection `arn` is required.");
+  }
+
+  return {
+    [CONNECTION_BRAND]: true,
+    _name: name,
+    _type: "connection",
+    _connectionType: "dynamodb",
+    options,
+  };
+}
+
 /**
  * Check if a value is a connection definition
  */
@@ -264,6 +330,15 @@ export function isGCSConnectionDefinition(value: unknown): value is GCSConnectio
   return isConnectionDefinition(value) && value._connectionType === "gcs";
 }
 
+/**
+ * Check if a value is a DynamoDB connection definition
+ */
+export function isDynamoDBConnectionDefinition(
+  value: unknown
+): value is DynamoDBConnectionDefinition {
+  return isConnectionDefinition(value) && value._connectionType === "dynamodb";
+}
+
 /**
  * Get the connection type from a connection definition
  */

package/src/schema/datasource.test.ts

@@ -112,7 +112,7 @@ describe("Datasource Schema", () => {
             bucketUri: "s3://my-bucket/events/*.csv",
           },
         })
-      ).toThrow("Datasource can only define one ingestion option: `kafka`, `s3`, or `gcs`.");
+      ).toThrow("Datasource can only define one ingestion option: `kafka`, `s3`, `gcs`, or `dynamodb`.");
 
       expect(() =>
         defineDatasource("events_gcs", {
@@ -126,7 +126,7 @@ describe("Datasource Schema", () => {
             bucketUri: "gs://my-bucket/events/*.csv",
           },
         })
-      ).toThrow("Datasource can only define one ingestion option: `kafka`, `s3`, or `gcs`.");
+      ).toThrow("Datasource can only define one ingestion option: `kafka`, `s3`, `gcs`, or `dynamodb`.");
     });
 
     it("accepts gcs ingestion configuration", () => {