@tinybirdco/sdk 0.0.43 → 0.0.45

package/src/codegen/index.ts

@@ -267,7 +267,7 @@ export function generateClientFile(
     " * Generated from existing workspace resources.",
     " */",
     "",
-    'import { createTinybirdClient } from "@tinybirdco/sdk";',
+    'import { Tinybird } from "@tinybirdco/sdk";',
     // Node imports for deriving configDir from import.meta.url (monorepo support)
     'import { fileURLToPath } from "url";',
     'import { dirname } from "path";',
@@ -313,7 +313,7 @@ export function generateClientFile(
   lines.push("const __configDir = dirname(fileURLToPath(import.meta.url));");
   lines.push("");
   lines.push("// Create the typed Tinybird client");
-  lines.push("export const tinybird = createTinybirdClient({");
+  lines.push("export const tinybird = new Tinybird({");
 
   if (datasources.length > 0) {
     const dsNames = datasources.map((ds) => toCamelCase(ds.name)).join(", ");
@@ -417,7 +417,7 @@ export function generateCombinedFile(
   ];
 
   // Build imports
-  const sdkImports: string[] = ["createTinybirdClient", "t"];
+  const sdkImports: string[] = ["Tinybird", "t"];
 
   if (datasources.length > 0) {
     sdkImports.push("defineDatasource", "engine", "type InferRow");
@@ -505,7 +505,7 @@ export function generateCombinedFile(
   lines.push("// This ensures tinybird.json is found regardless of where the app runs from");
   lines.push("const __configDir = dirname(fileURLToPath(import.meta.url));");
   lines.push("");
-  lines.push("export const tinybird = createTinybirdClient({");
+  lines.push("export const tinybird = new Tinybird({");
   lines.push(`  datasources: { ${dsNames} },`);
   lines.push(`  pipes: { ${pipeNames} },`);
   lines.push("  configDir: __configDir,");

package/src/generator/client.ts

@@ -127,7 +127,7 @@ export function generateClientFile(options: GenerateClientOptions): GeneratedCli
     reexportLines.push(`export { ${allExports.join(", ")} } from "${relativePath}";`);
   }
 
-  // Build createTinybirdClient call
+  // Build Tinybird constructor call
   const datasourceNames = Object.keys(entities.datasources);
   const pipeNames = Object.keys(entities.pipes);
 

package/src/generator/connection.test.ts

@@ -1,11 +1,11 @@
 import { describe, it, expect } from "vitest";
 import { generateConnection, generateAllConnections } from "./connection.js";
-import { createKafkaConnection } from "../schema/connection.js";
+import { defineKafkaConnection, defineS3Connection } from "../schema/connection.js";
 
 describe("Connection Generator", () => {
   describe("generateConnection", () => {
     it("generates basic Kafka connection with required fields", () => {
-      const conn = createKafkaConnection("my_kafka", {
+      const conn = defineKafkaConnection("my_kafka", {
         bootstrapServers: "kafka.example.com:9092",
       });
 
@@ -17,7 +17,7 @@ describe("Connection Generator", () => {
     });
 
     it("includes security protocol when provided", () => {
-      const conn = createKafkaConnection("my_kafka", {
+      const conn = defineKafkaConnection("my_kafka", {
         bootstrapServers: "kafka.example.com:9092",
         securityProtocol: "SASL_SSL",
       });
@@ -28,7 +28,7 @@ describe("Connection Generator", () => {
     });
 
     it("includes SASL mechanism when provided", () => {
-      const conn = createKafkaConnection("my_kafka", {
+      const conn = defineKafkaConnection("my_kafka", {
         bootstrapServers: "kafka.example.com:9092",
         saslMechanism: "PLAIN",
       });
@@ -39,7 +39,7 @@ describe("Connection Generator", () => {
     });
 
     it("includes key and secret when provided", () => {
-      const conn = createKafkaConnection("my_kafka", {
+      const conn = defineKafkaConnection("my_kafka", {
         bootstrapServers: "kafka.example.com:9092",
         key: '{{ tb_secret("KAFKA_KEY") }}',
         secret: '{{ tb_secret("KAFKA_SECRET") }}',
@@ -52,7 +52,7 @@ describe("Connection Generator", () => {
     });
 
     it("includes SSL CA PEM when provided", () => {
-      const conn = createKafkaConnection("my_kafka", {
+      const conn = defineKafkaConnection("my_kafka", {
         bootstrapServers: "kafka.example.com:9092",
         sslCaPem: '{{ tb_secret("KAFKA_CA_CERT") }}',
       });
@@ -63,7 +63,7 @@ describe("Connection Generator", () => {
     });
 
     it("generates full Kafka connection with all options", () => {
-      const conn = createKafkaConnection("my_kafka", {
+      const conn = defineKafkaConnection("my_kafka", {
         bootstrapServers: "kafka.example.com:9092",
         securityProtocol: "SASL_SSL",
         saslMechanism: "SCRAM-SHA-256",
@@ -85,7 +85,7 @@ describe("Connection Generator", () => {
     });
 
     it("supports PLAINTEXT security protocol", () => {
-      const conn = createKafkaConnection("local_kafka", {
+      const conn = defineKafkaConnection("local_kafka", {
         bootstrapServers: "localhost:9092",
         securityProtocol: "PLAINTEXT",
       });
@@ -99,7 +99,7 @@ describe("Connection Generator", () => {
       const mechanisms = ["PLAIN", "SCRAM-SHA-256", "SCRAM-SHA-512", "OAUTHBEARER"] as const;
 
       mechanisms.forEach((mechanism) => {
-        const conn = createKafkaConnection("my_kafka", {
+        const conn = defineKafkaConnection("my_kafka", {
           bootstrapServers: "kafka.example.com:9092",
           saslMechanism: mechanism,
         });
@@ -109,21 +109,52 @@ describe("Connection Generator", () => {
         expect(result.content).toContain(`KAFKA_SASL_MECHANISM ${mechanism}`);
       });
     });
+
+    it("generates basic S3 connection with IAM role auth", () => {
+      const conn = defineS3Connection("my_s3", {
+        region: "us-east-1",
+        arn: "arn:aws:iam::123456789012:role/tinybird-s3-access",
+      });
+
+      const result = generateConnection(conn);
+
+      expect(result.name).toBe("my_s3");
+      expect(result.content).toContain("TYPE s3");
+      expect(result.content).toContain("S3_REGION us-east-1");
+      expect(result.content).toContain(
+        "S3_ARN arn:aws:iam::123456789012:role/tinybird-s3-access"
+      );
+    });
+
+    it("generates S3 connection with access key auth", () => {
+      const conn = defineS3Connection("my_s3", {
+        region: "us-east-1",
+        accessKey: '{{ tb_secret("S3_ACCESS_KEY") }}',
+        secret: '{{ tb_secret("S3_SECRET") }}',
+      });
+
+      const result = generateConnection(conn);
+
+      expect(result.content).toContain("TYPE s3");
+      expect(result.content).toContain('S3_ACCESS_KEY {{ tb_secret("S3_ACCESS_KEY") }}');
+      expect(result.content).toContain('S3_SECRET {{ tb_secret("S3_SECRET") }}');
+    });
   });
 
   describe("generateAllConnections", () => {
     it("generates all connections", () => {
-      const conn1 = createKafkaConnection("kafka1", {
+      const conn1 = defineKafkaConnection("kafka1", {
         bootstrapServers: "kafka1.example.com:9092",
       });
-      const conn2 = createKafkaConnection("kafka2", {
-        bootstrapServers: "kafka2.example.com:9092",
+      const conn2 = defineS3Connection("s3_logs", {
+        region: "us-east-1",
+        arn: "arn:aws:iam::123456789012:role/tinybird-s3-access",
       });
 
-      const results = generateAllConnections({ kafka1: conn1, kafka2: conn2 });
+      const results = generateAllConnections({ kafka1: conn1, s3_logs: conn2 });
 
       expect(results).toHaveLength(2);
-      expect(results.map((r) => r.name).sort()).toEqual(["kafka1", "kafka2"]);
+      expect(results.map((r) => r.name).sort()).toEqual(["kafka1", "s3_logs"]);
     });
 
     it("returns empty array for empty connections", () => {

package/src/generator/connection.ts

@@ -4,6 +4,7 @@
  */
 
 import type { ConnectionDefinition, KafkaConnectionDefinition } from "../schema/connection.js";
+import { isS3ConnectionDefinition, type S3ConnectionDefinition } from "../schema/connection.js";
 
 /**
  * Generated connection content
@@ -48,6 +49,31 @@ function generateKafkaConnection(connection: KafkaConnectionDefinition): string
   return parts.join("\n");
 }
 
+/**
+ * Generate an S3 connection content
+ */
+function generateS3Connection(connection: S3ConnectionDefinition): string {
+  const parts: string[] = [];
+  const options = connection.options;
+
+  parts.push("TYPE s3");
+  parts.push(`S3_REGION ${options.region}`);
+
+  if (options.arn) {
+    parts.push(`S3_ARN ${options.arn}`);
+  }
+
+  if (options.accessKey) {
+    parts.push(`S3_ACCESS_KEY ${options.accessKey}`);
+  }
+
+  if (options.secret) {
+    parts.push(`S3_SECRET ${options.secret}`);
+  }
+
+  return parts.join("\n");
+}
+
 /**
  * Generate a .connection file content from a ConnectionDefinition
  *
@@ -56,7 +82,7 @@ function generateKafkaConnection(connection: KafkaConnectionDefinition): string
  *
  * @example
  * ```ts
- * const myKafka = createKafkaConnection('my_kafka', {
+ * const myKafka = defineKafkaConnection('my_kafka', {
  *   bootstrapServers: 'kafka.example.com:9092',
  *   securityProtocol: 'SASL_SSL',
  *   saslMechanism: 'PLAIN',
@@ -81,8 +107,10 @@ export function generateConnection(
 
   if (connection._connectionType === "kafka") {
     content = generateKafkaConnection(connection as KafkaConnectionDefinition);
+  } else if (isS3ConnectionDefinition(connection)) {
+    content = generateS3Connection(connection);
   } else {
-    throw new Error(`Unsupported connection type: ${connection._connectionType}`);
+    throw new Error("Unsupported connection type.");
   }
 
   return {

package/src/generator/datasource.test.ts

@@ -1,7 +1,7 @@
 import { describe, it, expect } from 'vitest';
 import { generateDatasource, generateAllDatasources } from './datasource.js';
 import { defineDatasource } from '../schema/datasource.js';
-import { createKafkaConnection } from '../schema/connection.js';
+import { defineKafkaConnection, defineS3Connection } from '../schema/connection.js';
 import { defineToken } from '../schema/token.js';
 import { t } from '../schema/types.js';
 import { engine } from '../schema/engines.js';
@@ -312,7 +312,7 @@ describe('Datasource Generator', () => {
 
   describe('Kafka configuration', () => {
     it('includes Kafka connection name and topic', () => {
-      const kafkaConn = createKafkaConnection('my_kafka', {
+      const kafkaConn = defineKafkaConnection('my_kafka', {
         bootstrapServers: 'kafka.example.com:9092',
       });
 
@@ -335,7 +335,7 @@ describe('Datasource Generator', () => {
     });
 
     it('includes Kafka group ID when provided', () => {
-      const kafkaConn = createKafkaConnection('my_kafka', {
+      const kafkaConn = defineKafkaConnection('my_kafka', {
         bootstrapServers: 'kafka.example.com:9092',
       });
 
@@ -358,7 +358,7 @@ describe('Datasource Generator', () => {
     });
 
     it('includes auto offset reset when provided', () => {
-      const kafkaConn = createKafkaConnection('my_kafka', {
+      const kafkaConn = defineKafkaConnection('my_kafka', {
         bootstrapServers: 'kafka.example.com:9092',
       });
 
@@ -381,7 +381,7 @@ describe('Datasource Generator', () => {
     });
 
     it('generates complete Kafka datasource with all options', () => {
-      const kafkaConn = createKafkaConnection('my_kafka', {
+      const kafkaConn = defineKafkaConnection('my_kafka', {
         bootstrapServers: 'kafka.example.com:9092',
         securityProtocol: 'SASL_SSL',
         saslMechanism: 'PLAIN',
@@ -417,6 +417,58 @@ describe('Datasource Generator', () => {
     });
   });
 
+  describe('S3 configuration', () => {
+    it('includes S3 connection name and bucket uri', () => {
+      const s3Conn = defineS3Connection('my_s3', {
+        region: 'us-east-1',
+        arn: 'arn:aws:iam::123456789012:role/tinybird-s3-access',
+      });
+
+      const ds = defineDatasource('s3_events', {
+        schema: {
+          timestamp: t.dateTime(),
+          event: t.string(),
+        },
+        engine: engine.mergeTree({ sortingKey: ['timestamp'] }),
+        s3: {
+          connection: s3Conn,
+          bucketUri: 's3://my-bucket/events/*.csv',
+        },
+      });
+
+      const result = generateDatasource(ds);
+
+      expect(result.content).toContain('IMPORT_CONNECTION_NAME my_s3');
+      expect(result.content).toContain('IMPORT_BUCKET_URI s3://my-bucket/events/*.csv');
+    });
+
+    it('includes optional S3 schedule and from timestamp', () => {
+      const s3Conn = defineS3Connection('my_s3', {
+        region: 'us-east-1',
+        arn: 'arn:aws:iam::123456789012:role/tinybird-s3-access',
+      });
+
+      const ds = defineDatasource('s3_events', {
+        schema: {
+          timestamp: t.dateTime(),
+          event: t.string(),
+        },
+        engine: engine.mergeTree({ sortingKey: ['timestamp'] }),
+        s3: {
+          connection: s3Conn,
+          bucketUri: 's3://my-bucket/events/*.csv',
+          schedule: '@auto',
+          fromTimestamp: '2024-01-01T00:00:00Z',
+        },
+      });
+
+      const result = generateDatasource(ds);
+
+      expect(result.content).toContain('IMPORT_SCHEDULE @auto');
+      expect(result.content).toContain('IMPORT_FROM_TIMESTAMP 2024-01-01T00:00:00Z');
+    });
+  });
+
   describe('Token generation', () => {
     it('generates TOKEN lines with inline config', () => {
       const ds = defineDatasource('test_ds', {

package/src/generator/datasource.ts

@@ -3,7 +3,14 @@
  * Converts DatasourceDefinition to native .datasource file format
  */
 
-import type { DatasourceDefinition, SchemaDefinition, ColumnDefinition, KafkaConfig, TokenConfig } from "../schema/datasource.js";
+import type {
+  DatasourceDefinition,
+  SchemaDefinition,
+  ColumnDefinition,
+  KafkaConfig,
+  S3Config,
+  TokenConfig,
+} from "../schema/datasource.js";
 import type { AnyTypeValidator, TypeModifiers } from "../schema/types.js";
 import { getColumnType, getColumnJsonPath } from "../schema/datasource.js";
 import { getEngineClause, type EngineConfig } from "../schema/engines.js";
@@ -163,6 +170,26 @@ function generateKafkaConfig(kafka: KafkaConfig): string {
   return parts.join("\n");
 }
 
+/**
+ * Generate S3 import configuration lines
+ */
+function generateS3Config(s3: S3Config): string {
+  const parts: string[] = [];
+
+  parts.push(`IMPORT_CONNECTION_NAME ${s3.connection._name}`);
+  parts.push(`IMPORT_BUCKET_URI ${s3.bucketUri}`);
+
+  if (s3.schedule) {
+    parts.push(`IMPORT_SCHEDULE ${s3.schedule}`);
+  }
+
+  if (s3.fromTimestamp) {
+    parts.push(`IMPORT_FROM_TIMESTAMP ${s3.fromTimestamp}`);
+  }
+
+  return parts.join("\n");
+}
+
 /**
  * Generate forward query section
  */
@@ -261,6 +288,10 @@ export function generateDatasource(
 ): GeneratedDatasource {
   const parts: string[] = [];
 
+  if (datasource.options.kafka && datasource.options.s3) {
+    throw new Error("Datasource cannot define both `kafka` and `s3` ingestion options.");
+  }
+
   // Add description if present
   if (datasource.options.description) {
     parts.push(`DESCRIPTION >\n    ${datasource.options.description}`);
@@ -283,6 +314,12 @@ export function generateDatasource(
     parts.push(generateKafkaConfig(datasource.options.kafka));
   }
 
+  // Add S3 configuration if present
+  if (datasource.options.s3) {
+    parts.push("");
+    parts.push(generateS3Config(datasource.options.s3));
+  }
+
   // Add forward query if present
   const forwardQuery = generateForwardQuery(datasource.options.forwardQuery);
   if (forwardQuery) {

package/src/index.test.ts

@@ -0,0 +1,13 @@
+import { describe, expect, it } from "vitest";
+import * as sdk from "./index.js";
+
+describe("root public exports", () => {
+  it("does not expose legacy project-client aliases", () => {
+    expect("createTinybirdClient" in sdk).toBe(false);
+  });
+
+  it("keeps documented project constructors", () => {
+    expect(typeof sdk.defineProject).toBe("function");
+    expect(typeof sdk.Tinybird).toBe("function");
+  });
+});

package/src/index.ts

@@ -109,16 +109,27 @@ export type {
   DatasourceTokenReference,
   ExtractSchema,
   KafkaConfig,
+  S3Config,
 } from "./schema/datasource.js";
 
 // ============ Connection ============
-export { createKafkaConnection, isConnectionDefinition, isKafkaConnectionDefinition, getConnectionType } from "./schema/connection.js";
+export {
+  defineKafkaConnection,
+  createKafkaConnection,
+  defineS3Connection,
+  isConnectionDefinition,
+  isKafkaConnectionDefinition,
+  isS3ConnectionDefinition,
+  getConnectionType,
+} from "./schema/connection.js";
 export type {
   ConnectionDefinition,
   KafkaConnectionDefinition,
   KafkaConnectionOptions,
   KafkaSecurityProtocol,
   KafkaSaslMechanism,
+  S3ConnectionDefinition,
+  S3ConnectionOptions,
 } from "./schema/connection.js";
 
 // ============ Token ============
@@ -168,12 +179,18 @@ export type {
 } from "./schema/pipe.js";
 
 // ============ Project ============
-export { defineProject, isProjectDefinition, getDatasourceNames, getPipeNames, getDatasource, getPipe, Tinybird, createTinybirdClient } from "./schema/project.js";
+export {
+  defineProject,
+  isProjectDefinition,
+  getDatasourceNames,
+  getPipeNames,
+  getDatasource,
+  getPipe,
+  Tinybird,
+} from "./schema/project.js";
 export type {
   ProjectDefinition,
   ProjectConfig,
-  ProjectClient,
-  TinybirdClientConfig,
   DatasourcesDefinition,
   PipesDefinition,
   ConnectionsDefinition,

package/src/migrate/emit-ts.ts

@@ -6,6 +6,7 @@ import type {
   KafkaConnectionModel,
   ParsedResource,
   PipeModel,
+  S3ConnectionModel,
 } from "./types.js";
 
 function escapeString(value: string): string {
@@ -243,6 +244,20 @@ function emitDatasource(ds: DatasourceModel): string {
     lines.push("  },");
   }
 
+  if (ds.s3) {
+    const connectionVar = toCamelCase(ds.s3.connectionName);
+    lines.push("  s3: {");
+    lines.push(`    connection: ${connectionVar},`);
+    lines.push(`    bucketUri: ${escapeString(ds.s3.bucketUri)},`);
+    if (ds.s3.schedule) {
+      lines.push(`    schedule: ${escapeString(ds.s3.schedule)},`);
+    }
+    if (ds.s3.fromTimestamp) {
+      lines.push(`    fromTimestamp: ${escapeString(ds.s3.fromTimestamp)},`);
+    }
+    lines.push("  },");
+  }
+
   if (ds.forwardQuery) {
     lines.push("  forwardQuery: `");
     lines.push(ds.forwardQuery.replace(/`/g, "\\`").replace(/\${/g, "\\${"));
@@ -270,28 +285,48 @@ function emitDatasource(ds: DatasourceModel): string {
   return lines.join("\n");
 }
 
-function emitConnection(connection: KafkaConnectionModel): string {
+function emitConnection(connection: KafkaConnectionModel | S3ConnectionModel): string {
   const variableName = toCamelCase(connection.name);
   const lines: string[] = [];
+
+  if (connection.connectionType === "kafka") {
+    lines.push(
+      `export const ${variableName} = defineKafkaConnection(${escapeString(connection.name)}, {`
+    );
+    lines.push(`  bootstrapServers: ${escapeString(connection.bootstrapServers)},`);
+    if (connection.securityProtocol) {
+      lines.push(`  securityProtocol: ${escapeString(connection.securityProtocol)},`);
+    }
+    if (connection.saslMechanism) {
+      lines.push(`  saslMechanism: ${escapeString(connection.saslMechanism)},`);
+    }
+    if (connection.key) {
+      lines.push(`  key: ${escapeString(connection.key)},`);
+    }
+    if (connection.secret) {
+      lines.push(`  secret: ${escapeString(connection.secret)},`);
+    }
+    if (connection.sslCaPem) {
+      lines.push(`  sslCaPem: ${escapeString(connection.sslCaPem)},`);
+    }
+    lines.push("});");
+    lines.push("");
+    return lines.join("\n");
+  }
+
   lines.push(
-    `export const ${variableName} = createKafkaConnection(${escapeString(connection.name)}, {`
+    `export const ${variableName} = defineS3Connection(${escapeString(connection.name)}, {`
   );
-  lines.push(`  bootstrapServers: ${escapeString(connection.bootstrapServers)},`);
-  if (connection.securityProtocol) {
-    lines.push(`  securityProtocol: ${escapeString(connection.securityProtocol)},`);
-  }
-  if (connection.saslMechanism) {
-    lines.push(`  saslMechanism: ${escapeString(connection.saslMechanism)},`);
+  lines.push(`  region: ${escapeString(connection.region)},`);
+  if (connection.arn) {
+    lines.push(`  arn: ${escapeString(connection.arn)},`);
   }
-  if (connection.key) {
-    lines.push(`  key: ${escapeString(connection.key)},`);
+  if (connection.accessKey) {
+    lines.push(`  accessKey: ${escapeString(connection.accessKey)},`);
   }
   if (connection.secret) {
     lines.push(`  secret: ${escapeString(connection.secret)},`);
   }
-  if (connection.sslCaPem) {
-    lines.push(`  sslCaPem: ${escapeString(connection.sslCaPem)},`);
-  }
   lines.push("});");
   lines.push("");
   return lines.join("\n");
@@ -398,7 +433,8 @@ function emitPipe(pipe: PipeModel): string {
 
 export function emitMigrationFileContent(resources: ParsedResource[]): string {
   const connections = resources.filter(
-    (resource): resource is KafkaConnectionModel => resource.kind === "connection"
+    (resource): resource is KafkaConnectionModel | S3ConnectionModel =>
+      resource.kind === "connection"
   );
   const datasources = resources.filter(
     (resource): resource is DatasourceModel => resource.kind === "datasource"
@@ -412,7 +448,21 @@ export function emitMigrationFileContent(resources: ParsedResource[]): string {
   );
   const needsParams = pipes.some((pipe) => pipe.params.length > 0);
 
-  const imports = new Set<string>(["createKafkaConnection", "defineDatasource", "definePipe", "defineMaterializedView", "defineCopyPipe", "node", "t", "engine"]);
+  const imports = new Set<string>([
+    "defineDatasource",
+    "definePipe",
+    "defineMaterializedView",
+    "defineCopyPipe",
+    "node",
+    "t",
+    "engine",
+  ]);
+  if (connections.some((connection) => connection.connectionType === "kafka")) {
+    imports.add("defineKafkaConnection");
+  }
+  if (connections.some((connection) => connection.connectionType === "s3")) {
+    imports.add("defineS3Connection");
+  }
   if (needsColumn) {
     imports.add("column");
   }
@@ -420,13 +470,27 @@ export function emitMigrationFileContent(resources: ParsedResource[]): string {
     imports.add("p");
   }
 
+  const orderedImports = [
+    "defineKafkaConnection",
+    "defineS3Connection",
+    "defineDatasource",
+    "definePipe",
+    "defineMaterializedView",
+    "defineCopyPipe",
+    "node",
+    "t",
+    "engine",
+    "column",
+    "p",
+  ].filter((name) => imports.has(name));
+
   const lines: string[] = [];
   lines.push("/**");
   lines.push(" * Generated by tinybird migrate.");
   lines.push(" * Review endpoint output schemas and any defaults before production use.");
   lines.push(" */");
   lines.push("");
-  lines.push(`import { ${Array.from(imports).join(", ")} } from "@tinybirdco/sdk";`);
+  lines.push(`import { ${orderedImports.join(", ")} } from "@tinybirdco/sdk";`);
   lines.push("");
 
   if (connections.length > 0) {