@tinybirdco/sdk 0.0.76 → 0.0.78

package/src/cli/commands/migrate.test.ts

@@ -716,6 +716,101 @@ IMPORT_SCHEDULE '@on-demand'
     expect(output).toContain('schedule: "@on-demand"');
   });
 
+  it("migrates dynamodb connection and import datasource directives", async () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-migrate-"));
+    tempDirs.push(tempDir);
+
+    writeFile(
+      tempDir,
+      "dynamo_sample.connection",
+      `TYPE dynamodb
+DYNAMODB_ARN {{ tb_secret("DYNAMODB_ROLE_ARN") }}
+DYNAMODB_REGION us-east-1
+`
+    );
+
+    writeFile(
+      tempDir,
+      "events_dynamo.datasource",
+      `SCHEMA >
+    id String \`json:$.Id\`,
+    _record String \`json:$.NewImage\`
+
+ENGINE "ReplacingMergeTree"
+ENGINE_SORTING_KEY "id"
+IMPORT_CONNECTION_NAME dynamo_sample
+IMPORT_TABLE_ARN arn:aws:dynamodb:us-east-1:123456789012:table/events
+IMPORT_EXPORT_BUCKET s3://my-export-bucket
+`
+    );
+
+    const result = await runMigrate({
+      cwd: tempDir,
+      patterns: ["."],
+      strict: true,
+    });
+
+    expect(result.success).toBe(true);
+    expect(result.errors).toHaveLength(0);
+    expect(result.migrated.filter((resource) => resource.kind === "connection")).toHaveLength(1);
+    expect(result.migrated.filter((resource) => resource.kind === "datasource")).toHaveLength(1);
+
+    const output = fs.readFileSync(result.outputPath, "utf-8");
+    expect(output).toContain("defineDynamoDBConnection");
+    expect(output).toContain('export const dynamoSample = defineDynamoDBConnection("dynamo_sample", {');
+    expect(output).toContain('region: "us-east-1"');
+    expect(output).toContain('arn: secret("DYNAMODB_ROLE_ARN")');
+    expect(output).toContain("dynamodb: {");
+    expect(output).toContain("connection: dynamoSample");
+    expect(output).toContain(
+      'tableArn: "arn:aws:dynamodb:us-east-1:123456789012:table/events"'
+    );
+    expect(output).toContain('exportBucket: "s3://my-export-bucket"');
+  });
+
+  it("reports an error when DynamoDB directives use a non-dynamodb connection type", async () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-migrate-"));
+    tempDirs.push(tempDir);
+
+    writeFile(
+      tempDir,
+      "s3sample.connection",
+      `TYPE s3
+S3_REGION us-east-1
+S3_ARN "arn:aws:iam::123456789012:role/tinybird-s3-access"
+`
+    );
+
+    writeFile(
+      tempDir,
+      "events_dynamo.datasource",
+      `SCHEMA >
+    id String
+
+ENGINE "ReplacingMergeTree"
+ENGINE_SORTING_KEY "id"
+IMPORT_CONNECTION_NAME s3sample
+IMPORT_TABLE_ARN arn:aws:dynamodb:us-east-1:123456789012:table/events
+IMPORT_EXPORT_BUCKET s3://my-export-bucket
+`
+    );
+
+    const result = await runMigrate({
+      cwd: tempDir,
+      patterns: ["."],
+      strict: true,
+    });
+
+    expect(result.success).toBe(false);
+    expect(
+      result.errors.some((error) =>
+        error.message.includes(
+          'Datasource DynamoDB ingestion requires a dynamodb connection, found "s3".'
+        )
+      )
+    ).toBe(true);
+  });
+
   it("reports an error when import directives use a non-bucket connection type", async () => {
     const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-migrate-"));
     tempDirs.push(tempDir);

package/src/cli/commands/migrate.ts

@@ -135,7 +135,8 @@ export async function runMigrate(
     const referencedConnectionName =
       datasource.kafka?.connectionName ??
       datasource.s3?.connectionName ??
-      datasource.gcs?.connectionName;
+      datasource.gcs?.connectionName ??
+      datasource.dynamodb?.connectionName;
 
     if (
       referencedConnectionName &&
@@ -186,6 +187,21 @@ export async function runMigrate(
       }
     }
 
+    if (datasource.dynamodb) {
+      const dynamodbConnectionType = parsedConnectionTypeByName.get(
+        datasource.dynamodb.connectionName
+      );
+      if (dynamodbConnectionType !== "dynamodb") {
+        errors.push({
+          filePath: datasource.filePath,
+          resourceName: datasource.name,
+          resourceKind: datasource.kind,
+          message: `Datasource DynamoDB ingestion requires a dynamodb connection, found "${dynamodbConnectionType ?? "(none)"}".`,
+        });
+        continue;
+      }
+    }
+
     try {
       validateResourceForEmission(datasource);
       migrated.push(datasource);

package/src/generator/connection.test.ts

@@ -4,6 +4,7 @@ import {
   defineKafkaConnection,
   defineS3Connection,
   defineGCSConnection,
+  defineDynamoDBConnection,
 } from "../schema/connection.js";
 
 describe("Connection Generator", () => {
@@ -218,6 +219,22 @@ describe("Connection Generator", () => {
         'GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON {{ tb_secret("GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON") }}'
       );
     });
+
+    it("generates DynamoDB connection with arn and region", () => {
+      const conn = defineDynamoDBConnection("my_dynamo", {
+        region: "us-east-1",
+        arn: '{{ tb_secret("DYNAMODB_ROLE_ARN") }}',
+      });
+
+      const result = generateConnection(conn);
+
+      expect(result.name).toBe("my_dynamo");
+      expect(result.content).toBe(
+        ['TYPE dynamodb', 'DYNAMODB_ARN {{ tb_secret("DYNAMODB_ROLE_ARN") }}', "DYNAMODB_REGION us-east-1"].join(
+          "\n"
+        )
+      );
+    });
   });
 
   describe("generateAllConnections", () => {

package/src/generator/connection.ts

@@ -7,10 +7,12 @@ import type {
   ConnectionDefinition,
   KafkaConnectionDefinition,
   GCSConnectionDefinition,
+  DynamoDBConnectionDefinition,
 } from "../schema/connection.js";
 import {
   isS3ConnectionDefinition,
   isGCSConnectionDefinition,
+  isDynamoDBConnectionDefinition,
   type S3ConnectionDefinition,
 } from "../schema/connection.js";
 
@@ -123,6 +125,20 @@ function generateGCSConnection(connection: GCSConnectionDefinition): string {
   return parts.join("\n");
 }
 
+/**
+ * Generate a DynamoDB connection content
+ */
+function generateDynamoDBConnection(connection: DynamoDBConnectionDefinition): string {
+  const parts: string[] = [];
+  const options = connection.options;
+
+  parts.push("TYPE dynamodb");
+  parts.push(`DYNAMODB_ARN ${options.arn}`);
+  parts.push(`DYNAMODB_REGION ${options.region}`);
+
+  return parts.join("\n");
+}
+
 /**
  * Generate a .connection file content from a ConnectionDefinition
  *
@@ -160,6 +176,8 @@ export function generateConnection(
     content = generateS3Connection(connection);
   } else if (isGCSConnectionDefinition(connection)) {
     content = generateGCSConnection(connection);
+  } else if (isDynamoDBConnectionDefinition(connection)) {
+    content = generateDynamoDBConnection(connection);
   } else {
     throw new Error("Unsupported connection type.");
   }

package/src/generator/datasource.test.ts

@@ -1,7 +1,7 @@
 import { describe, it, expect } from 'vitest';
 import { generateDatasource, generateAllDatasources } from './datasource.js';
 import { defineDatasource } from '../schema/datasource.js';
-import { defineKafkaConnection, defineS3Connection, defineGCSConnection } from '../schema/connection.js';
+import { defineKafkaConnection, defineS3Connection, defineGCSConnection, defineDynamoDBConnection } from '../schema/connection.js';
 import { defineToken } from '../schema/token.js';
 import { t } from '../schema/types.js';
 import { engine } from '../schema/engines.js';
@@ -629,6 +629,59 @@ describe('Datasource Generator', () => {
     });
   });
 
+  describe('DynamoDB configuration', () => {
+    it('includes DynamoDB import directives', () => {
+      const dynamoConn = defineDynamoDBConnection('my_dynamo', {
+        region: 'us-east-1',
+        arn: '{{ tb_secret("DYNAMODB_ROLE_ARN") }}',
+      });
+
+      const ds = defineDatasource('dynamo_events', {
+        schema: {
+          id: t.string(),
+          _record: t.string(),
+        },
+        engine: engine.replacingMergeTree({ sortingKey: ['id'] }),
+        dynamodb: {
+          connection: dynamoConn,
+          tableArn: 'arn:aws:dynamodb:us-east-1:123456789012:table/events',
+          exportBucket: 's3://my-export-bucket',
+        },
+      });
+
+      const result = generateDatasource(ds);
+
+      expect(result.content).toContain('IMPORT_CONNECTION_NAME my_dynamo');
+      expect(result.content).toContain(
+        'IMPORT_TABLE_ARN arn:aws:dynamodb:us-east-1:123456789012:table/events'
+      );
+      expect(result.content).toContain('IMPORT_EXPORT_BUCKET s3://my-export-bucket');
+    });
+
+    it('rejects mixing DynamoDB with other ingestion options', () => {
+      const dynamoConn = defineDynamoDBConnection('my_dynamo', {
+        region: 'us-east-1',
+        arn: '{{ tb_secret("DYNAMODB_ROLE_ARN") }}',
+      });
+      const s3Conn = defineS3Connection('my_s3', {
+        region: 'us-east-1',
+        arn: 'arn:aws:iam::123456789012:role/x',
+      });
+
+      expect(() =>
+        defineDatasource('mixed', {
+          schema: { id: t.string() },
+          dynamodb: {
+            connection: dynamoConn,
+            tableArn: 'arn:aws:dynamodb:us-east-1:123456789012:table/events',
+            exportBucket: 's3://my-export-bucket',
+          },
+          s3: { connection: s3Conn, bucketUri: 's3://b/*.csv' },
+        })
+      ).toThrow('only define one ingestion option');
+    });
+  });
+
   describe('Token generation', () => {
     it('generates TOKEN lines with inline config', () => {
       const ds = defineDatasource('test_ds', {

package/src/generator/datasource.ts

@@ -10,6 +10,7 @@ import type {
   KafkaConfig,
   S3Config,
   GCSConfig,
+  DynamoDBConfig,
   TokenConfig,
   DatasourceIndex,
 } from "../schema/datasource.js";
@@ -199,6 +200,19 @@ function generateImportConfig(importConfig: S3Config | GCSConfig): string {
   return parts.join("\n");
 }
 
+/**
+ * Generate DynamoDB import configuration lines
+ */
+function generateDynamoDBConfig(dynamodb: DynamoDBConfig): string {
+  const parts: string[] = [];
+
+  parts.push(`IMPORT_CONNECTION_NAME ${dynamodb.connection._name}`);
+  parts.push(`IMPORT_TABLE_ARN ${dynamodb.tableArn}`);
+  parts.push(`IMPORT_EXPORT_BUCKET ${dynamodb.exportBucket}`);
+
+  return parts.join("\n");
+}
+
 /**
  * Generate forward query section
  */
@@ -318,9 +332,12 @@ export function generateDatasource(
     datasource.options.kafka,
     datasource.options.s3,
     datasource.options.gcs,
+    datasource.options.dynamodb,
   ].filter(Boolean).length;
   if (ingestionConfigCount > 1) {
-    throw new Error("Datasource can only define one ingestion option: `kafka`, `s3`, or `gcs`.");
+    throw new Error(
+      "Datasource can only define one ingestion option: `kafka`, `s3`, `gcs`, or `dynamodb`."
+    );
   }
 
   // Add description if present
@@ -369,6 +386,12 @@ export function generateDatasource(
     parts.push(generateImportConfig(datasource.options.gcs));
   }
 
+  // Add DynamoDB configuration if present
+  if (datasource.options.dynamodb) {
+    parts.push("");
+    parts.push(generateDynamoDBConfig(datasource.options.dynamodb));
+  }
+
   // Add forward query if present
   const forwardQuery = generateForwardQuery(datasource.options.forwardQuery);
   if (forwardQuery) {

package/src/index.ts

@@ -116,6 +116,7 @@ export type {
   KafkaConfig,
   S3Config,
   GCSConfig,
+  DynamoDBConfig,
   DatasourceIndex,
 } from "./schema/datasource.js";
 
@@ -125,10 +126,12 @@ export {
   createKafkaConnection,
   defineS3Connection,
   defineGCSConnection,
+  defineDynamoDBConnection,
   isConnectionDefinition,
   isKafkaConnectionDefinition,
   isS3ConnectionDefinition,
   isGCSConnectionDefinition,
+  isDynamoDBConnectionDefinition,
   getConnectionType,
 } from "./schema/connection.js";
 export type {
@@ -142,6 +145,8 @@ export type {
   S3ConnectionOptions,
   GCSConnectionDefinition,
   GCSConnectionOptions,
+  DynamoDBConnectionDefinition,
+  DynamoDBConnectionOptions,
 } from "./schema/connection.js";
 
 // ============ Token ============

package/src/migrate/emit-ts.ts

@@ -4,6 +4,7 @@ import { parseLiteralFromDatafile, toTsLiteral } from "./parser-utils.js";
 import type {
   DatasourceModel,
   DatasourceEngineModel,
+  DynamoDBConnectionModel,
   GCSConnectionModel,
   KafkaConnectionModel,
   ParsedResource,
@@ -72,6 +73,9 @@ function hasSecretTemplate(resources: ParsedResource[]): boolean {
         if (resource.arn) values.push(resource.arn);
         if (resource.accessKey) values.push(resource.accessKey);
         if (resource.secret) values.push(resource.secret);
+      } else if (resource.connectionType === "dynamodb") {
+        values.push(resource.region);
+        values.push(resource.arn);
       } else {
         values.push(resource.serviceAccountCredentialsJson);
       }
@@ -95,6 +99,10 @@ function hasSecretTemplate(resources: ParsedResource[]): boolean {
         if (resource.gcs.schedule) values.push(resource.gcs.schedule);
         if (resource.gcs.fromTimestamp) values.push(resource.gcs.fromTimestamp);
       }
+      if (resource.dynamodb) {
+        values.push(resource.dynamodb.tableArn);
+        values.push(resource.dynamodb.exportBucket);
+      }
       continue;
     }
   }
@@ -388,6 +396,15 @@ function emitDatasource(ds: DatasourceModel): string {
     lines.push("  },");
   }
 
+  if (ds.dynamodb) {
+    const connectionVar = toCamelCase(ds.dynamodb.connectionName);
+    lines.push("  dynamodb: {");
+    lines.push(`    connection: ${connectionVar},`);
+    lines.push(`    tableArn: ${emitStringOrSecret(ds.dynamodb.tableArn)},`);
+    lines.push(`    exportBucket: ${emitStringOrSecret(ds.dynamodb.exportBucket)},`);
+    lines.push("  },");
+  }
+
   if (ds.forwardQuery) {
     lines.push("  forwardQuery: `");
     lines.push(escapeTemplateLiteral(ds.forwardQuery));
@@ -416,7 +433,11 @@ function emitDatasource(ds: DatasourceModel): string {
 }
 
 function emitConnection(
-  connection: KafkaConnectionModel | S3ConnectionModel | GCSConnectionModel
+  connection:
+    | KafkaConnectionModel
+    | S3ConnectionModel
+    | GCSConnectionModel
+    | DynamoDBConnectionModel
 ): string {
   const variableName = toCamelCase(connection.name);
   const lines: string[] = [];
@@ -486,6 +507,17 @@ function emitConnection(
     return lines.join("\n");
   }
 
+  if (connection.connectionType === "dynamodb") {
+    lines.push(
+      `export const ${variableName} = defineDynamoDBConnection(${escapeString(connection.name)}, {`
+    );
+    lines.push(`  region: ${emitStringOrSecret(connection.region)},`);
+    lines.push(`  arn: ${emitStringOrSecret(connection.arn)},`);
+    lines.push("});");
+    lines.push("");
+    return lines.join("\n");
+  }
+
   lines.push(
     `export const ${variableName} = defineGCSConnection(${escapeString(connection.name)}, {`
   );
@@ -625,8 +657,13 @@ function emitPipe(pipe: PipeModel): string {
 
 export function emitMigrationFileContent(resources: ParsedResource[]): string {
   const connections = resources.filter(
-    (resource): resource is KafkaConnectionModel | S3ConnectionModel | GCSConnectionModel =>
-      resource.kind === "connection"
+    (
+      resource
+    ): resource is
+      | KafkaConnectionModel
+      | S3ConnectionModel
+      | GCSConnectionModel
+      | DynamoDBConnectionModel => resource.kind === "connection"
   );
   const datasources = resources.filter(
     (resource): resource is DatasourceModel => resource.kind === "datasource"
@@ -655,6 +692,9 @@ export function emitMigrationFileContent(resources: ParsedResource[]): string {
   if (connections.some((connection) => connection.connectionType === "gcs")) {
     imports.add("defineGCSConnection");
   }
+  if (connections.some((connection) => connection.connectionType === "dynamodb")) {
+    imports.add("defineDynamoDBConnection");
+  }
   if (needsParams) {
     imports.add("p");
   }
@@ -672,6 +712,7 @@ export function emitMigrationFileContent(resources: ParsedResource[]): string {
     "defineKafkaConnection",
     "defineS3Connection",
     "defineGCSConnection",
+    "defineDynamoDBConnection",
     "defineDatasource",
     "definePipe",
     "defineMaterializedView",

package/src/migrate/parse-connection.test.ts

@@ -122,4 +122,45 @@ KAFKA_SASL_OAUTHBEARER_AWS_EXTERNAL_ID {{ tb_secret("KAFKA_AWS_EXTERNAL_ID") }}`
       '{{ tb_secret("KAFKA_AWS_EXTERNAL_ID") }}'
     );
   });
+
+  it("parses basic DynamoDB connection", () => {
+    const result = parseConnectionFile(
+      resource(
+        "my_dynamo",
+        `TYPE dynamodb
+DYNAMODB_ARN {{ tb_secret("DYNAMODB_ROLE_ARN") }}
+DYNAMODB_REGION us-east-1`
+      )
+    );
+
+    expect(result.connectionType).toBe("dynamodb");
+    expect(result).toHaveProperty("arn", '{{ tb_secret("DYNAMODB_ROLE_ARN") }}');
+    expect(result).toHaveProperty("region", "us-east-1");
+  });
+
+  it("throws when DynamoDB connection is missing DYNAMODB_ARN", () => {
+    expect(() =>
+      parseConnectionFile(
+        resource(
+          "my_dynamo",
+          `TYPE dynamodb
+DYNAMODB_REGION us-east-1`
+        )
+      )
+    ).toThrow("DYNAMODB_ARN is required");
+  });
+
+  it("throws when DynamoDB connection mixes S3 directives", () => {
+    expect(() =>
+      parseConnectionFile(
+        resource(
+          "my_dynamo",
+          `TYPE dynamodb
+DYNAMODB_ARN {{ tb_secret("DYNAMODB_ROLE_ARN") }}
+DYNAMODB_REGION us-east-1
+S3_ARN arn:aws:iam::1:role/x`
+        )
+      )
+    ).toThrow("Kafka/S3/GCS directives are not valid for dynamodb connections.");
+  });
 });

package/src/migrate/parse-connection.ts

@@ -1,4 +1,5 @@
 import type {
+  DynamoDBConnectionModel,
   GCSConnectionModel,
   KafkaConnectionModel,
   ResourceFile,
@@ -31,6 +32,8 @@ const CONNECTION_DIRECTIVES = new Set([
   "S3_ACCESS_KEY",
   "S3_SECRET",
   "GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON",
+  "DYNAMODB_ARN",
+  "DYNAMODB_REGION",
 ]);
 
 function isConnectionDirectiveLine(line: string): boolean {
@@ -43,7 +46,7 @@ function isConnectionDirectiveLine(line: string): boolean {
 
 export function parseConnectionFile(
   resource: ResourceFile
-): KafkaConnectionModel | S3ConnectionModel | GCSConnectionModel {
+): KafkaConnectionModel | S3ConnectionModel | GCSConnectionModel | DynamoDBConnectionModel {
   const lines = splitLines(resource.content);
   let connectionType: string | undefined;
 
@@ -74,6 +77,9 @@ export function parseConnectionFile(
   let accessSecret: string | undefined;
   let serviceAccountCredentialsJson: string | undefined;
 
+  let dynamodbArn: string | undefined;
+  let dynamodbRegion: string | undefined;
+
   let i = 0;
   while (i < lines.length) {
     const rawLine = lines[i] ?? "";
@@ -177,6 +183,12 @@ export function parseConnectionFile(
         }
         serviceAccountCredentialsJson = parseQuotedValue(value);
         break;
+      case "DYNAMODB_ARN":
+        dynamodbArn = parseQuotedValue(value);
+        break;
+      case "DYNAMODB_REGION":
+        dynamodbRegion = parseQuotedValue(value);
+        break;
       default:
         throw new MigrationParseError(
           resource.filePath,
@@ -198,12 +210,20 @@ export function parseConnectionFile(
   }
 
   if (connectionType === "kafka") {
-    if (region || arn || accessKey || accessSecret || serviceAccountCredentialsJson) {
+    if (
+      region ||
+      arn ||
+      accessKey ||
+      accessSecret ||
+      serviceAccountCredentialsJson ||
+      dynamodbArn ||
+      dynamodbRegion
+    ) {
       throw new MigrationParseError(
         resource.filePath,
         "connection",
         resource.name,
-        "S3/GCS directives are not valid for kafka connections."
+        "S3/GCS/DynamoDB directives are not valid for kafka connections."
       );
     }
 
@@ -248,13 +268,15 @@ export function parseConnectionFile(
       secret ||
       schemaRegistryUrl ||
       sslCaPem ||
-      serviceAccountCredentialsJson
+      serviceAccountCredentialsJson ||
+      dynamodbArn ||
+      dynamodbRegion
     ) {
       throw new MigrationParseError(
         resource.filePath,
         "connection",
         resource.name,
-        "Kafka/GCS directives are not valid for s3 connections."
+        "Kafka/GCS/DynamoDB directives are not valid for s3 connections."
       );
     }
 
@@ -313,13 +335,15 @@ export function parseConnectionFile(
       region ||
       arn ||
       accessKey ||
-      accessSecret
+      accessSecret ||
+      dynamodbArn ||
+      dynamodbRegion
     ) {
       throw new MigrationParseError(
         resource.filePath,
         "connection",
         resource.name,
-        "Kafka/S3 directives are not valid for gcs connections."
+        "Kafka/S3/DynamoDB directives are not valid for gcs connections."
       );
     }
 
@@ -341,6 +365,61 @@ export function parseConnectionFile(
     };
   }
 
+  if (connectionType === "dynamodb") {
+    if (
+      bootstrapServers ||
+      securityProtocol ||
+      saslMechanism ||
+      saslOauthbearerMethod ||
+      saslOauthbearerAwsRegion ||
+      saslOauthbearerAwsRoleArn ||
+      saslOauthbearerAwsExternalId ||
+      key ||
+      secret ||
+      schemaRegistryUrl ||
+      sslCaPem ||
+      region ||
+      arn ||
+      accessKey ||
+      accessSecret ||
+      serviceAccountCredentialsJson
+    ) {
+      throw new MigrationParseError(
+        resource.filePath,
+        "connection",
+        resource.name,
+        "Kafka/S3/GCS directives are not valid for dynamodb connections."
+      );
+    }
+
+    if (!dynamodbArn) {
+      throw new MigrationParseError(
+        resource.filePath,
+        "connection",
+        resource.name,
+        "DYNAMODB_ARN is required for dynamodb connections."
+      );
+    }
+
+    if (!dynamodbRegion) {
+      throw new MigrationParseError(
+        resource.filePath,
+        "connection",
+        resource.name,
+        "DYNAMODB_REGION is required for dynamodb connections."
+      );
+    }
+
+    return {
+      kind: "connection",
+      name: resource.name,
+      filePath: resource.filePath,
+      connectionType: "dynamodb",
+      arn: dynamodbArn,
+      region: dynamodbRegion,
+    };
+  }
+
   throw new MigrationParseError(
     resource.filePath,
     "connection",