@tinybirdco/sdk 0.0.44 → 0.0.46

package/src/schema/connection.test.ts

@@ -1,15 +1,17 @@
 import { describe, it, expect } from "vitest";
 import {
-  createKafkaConnection,
+  defineKafkaConnection,
+  defineS3Connection,
   isConnectionDefinition,
   isKafkaConnectionDefinition,
+  isS3ConnectionDefinition,
   getConnectionType,
 } from "./connection.js";
 
 describe("Connection Schema", () => {
-  describe("createKafkaConnection", () => {
+  describe("defineKafkaConnection", () => {
     it("creates a Kafka connection with required fields", () => {
-      const conn = createKafkaConnection("my_kafka", {
+      const conn = defineKafkaConnection("my_kafka", {
         bootstrapServers: "kafka.example.com:9092",
       });
 
@@ -20,7 +22,7 @@ describe("Connection Schema", () => {
     });
 
     it("creates a Kafka connection with all options", () => {
-      const conn = createKafkaConnection("my_kafka", {
+      const conn = defineKafkaConnection("my_kafka", {
         bootstrapServers: "kafka.example.com:9092",
         securityProtocol: "SASL_SSL",
         saslMechanism: "PLAIN",
@@ -37,19 +39,19 @@ describe("Connection Schema", () => {
     });
 
     it("supports different SASL mechanisms", () => {
-      const scramConn = createKafkaConnection("scram_kafka", {
+      const scramConn = defineKafkaConnection("scram_kafka", {
         bootstrapServers: "kafka.example.com:9092",
         saslMechanism: "SCRAM-SHA-256",
       });
       expect(scramConn.options.saslMechanism).toBe("SCRAM-SHA-256");
 
-      const scram512Conn = createKafkaConnection("scram512_kafka", {
+      const scram512Conn = defineKafkaConnection("scram512_kafka", {
         bootstrapServers: "kafka.example.com:9092",
         saslMechanism: "SCRAM-SHA-512",
       });
       expect(scram512Conn.options.saslMechanism).toBe("SCRAM-SHA-512");
 
-      const oauthConn = createKafkaConnection("oauth_kafka", {
+      const oauthConn = defineKafkaConnection("oauth_kafka", {
         bootstrapServers: "kafka.example.com:9092",
         saslMechanism: "OAUTHBEARER",
       });
@@ -57,13 +59,13 @@ describe("Connection Schema", () => {
     });
 
     it("supports different security protocols", () => {
-      const plaintext = createKafkaConnection("plaintext_kafka", {
+      const plaintext = defineKafkaConnection("plaintext_kafka", {
         bootstrapServers: "localhost:9092",
         securityProtocol: "PLAINTEXT",
       });
       expect(plaintext.options.securityProtocol).toBe("PLAINTEXT");
 
-      const saslPlaintext = createKafkaConnection("sasl_plaintext_kafka", {
+      const saslPlaintext = defineKafkaConnection("sasl_plaintext_kafka", {
         bootstrapServers: "localhost:9092",
         securityProtocol: "SASL_PLAINTEXT",
       });
@@ -72,40 +74,88 @@ describe("Connection Schema", () => {
 
     it("throws error for invalid connection name", () => {
       expect(() =>
-        createKafkaConnection("123invalid", {
+        defineKafkaConnection("123invalid", {
           bootstrapServers: "kafka.example.com:9092",
         })
       ).toThrow("Invalid connection name");
 
       expect(() =>
-        createKafkaConnection("my-connection", {
+        defineKafkaConnection("my-connection", {
           bootstrapServers: "kafka.example.com:9092",
         })
       ).toThrow("Invalid connection name");
 
       expect(() =>
-        createKafkaConnection("", {
+        defineKafkaConnection("", {
           bootstrapServers: "kafka.example.com:9092",
         })
       ).toThrow("Invalid connection name");
     });
 
     it("allows valid naming patterns", () => {
-      const conn1 = createKafkaConnection("_private_kafka", {
+      const conn1 = defineKafkaConnection("_private_kafka", {
         bootstrapServers: "kafka.example.com:9092",
       });
       expect(conn1._name).toBe("_private_kafka");
 
-      const conn2 = createKafkaConnection("kafka_v2", {
+      const conn2 = defineKafkaConnection("kafka_v2", {
         bootstrapServers: "kafka.example.com:9092",
       });
       expect(conn2._name).toBe("kafka_v2");
     });
   });
 
+  describe("defineS3Connection", () => {
+    it("creates an S3 connection with IAM role auth", () => {
+      const conn = defineS3Connection("my_s3", {
+        region: "us-east-1",
+        arn: "arn:aws:iam::123456789012:role/tinybird-s3-access",
+      });
+
+      expect(conn._name).toBe("my_s3");
+      expect(conn._type).toBe("connection");
+      expect(conn._connectionType).toBe("s3");
+      expect(conn.options.region).toBe("us-east-1");
+      expect(conn.options.arn).toBe("arn:aws:iam::123456789012:role/tinybird-s3-access");
+    });
+
+    it("creates an S3 connection with access key auth", () => {
+      const conn = defineS3Connection("my_s3", {
+        region: "us-east-1",
+        accessKey: '{{ tb_secret("S3_ACCESS_KEY") }}',
+        secret: '{{ tb_secret("S3_SECRET") }}',
+      });
+
+      expect(conn.options.accessKey).toBe('{{ tb_secret("S3_ACCESS_KEY") }}');
+      expect(conn.options.secret).toBe('{{ tb_secret("S3_SECRET") }}');
+    });
+
+    it("throws when auth config is incomplete", () => {
+      expect(() =>
+        defineS3Connection("my_s3", {
+          region: "us-east-1",
+        })
+      ).toThrow("S3 connection requires either `arn` or both `accessKey` and `secret`.");
+
+      expect(() =>
+        defineS3Connection("my_s3", {
+          region: "us-east-1",
+          accessKey: "key-only",
+        })
+      ).toThrow("S3 connection requires either `arn` or both `accessKey` and `secret`.");
+
+      expect(() =>
+        defineS3Connection("my_s3", {
+          region: "us-east-1",
+          secret: "secret-only",
+        })
+      ).toThrow("S3 connection requires either `arn` or both `accessKey` and `secret`.");
+    });
+  });
+
   describe("isConnectionDefinition", () => {
     it("returns true for valid connection", () => {
-      const conn = createKafkaConnection("my_kafka", {
+      const conn = defineKafkaConnection("my_kafka", {
         bootstrapServers: "kafka.example.com:9092",
       });
 
@@ -124,7 +174,7 @@ describe("Connection Schema", () => {
 
   describe("isKafkaConnectionDefinition", () => {
     it("returns true for Kafka connection", () => {
-      const conn = createKafkaConnection("my_kafka", {
+      const conn = defineKafkaConnection("my_kafka", {
         bootstrapServers: "kafka.example.com:9092",
       });
 
@@ -137,13 +187,38 @@ describe("Connection Schema", () => {
     });
   });
 
+  describe("isS3ConnectionDefinition", () => {
+    it("returns true for S3 connection", () => {
+      const conn = defineS3Connection("my_s3", {
+        region: "us-east-1",
+        arn: "arn:aws:iam::123456789012:role/tinybird-s3-access",
+      });
+
+      expect(isS3ConnectionDefinition(conn)).toBe(true);
+    });
+
+    it("returns false for non-S3 objects", () => {
+      expect(isS3ConnectionDefinition({})).toBe(false);
+      expect(isS3ConnectionDefinition(null)).toBe(false);
+    });
+  });
+
   describe("getConnectionType", () => {
     it("returns the connection type", () => {
-      const conn = createKafkaConnection("my_kafka", {
+      const conn = defineKafkaConnection("my_kafka", {
         bootstrapServers: "kafka.example.com:9092",
       });
 
       expect(getConnectionType(conn)).toBe("kafka");
     });
+
+    it("returns the s3 connection type", () => {
+      const conn = defineS3Connection("my_s3", {
+        region: "us-east-1",
+        arn: "arn:aws:iam::123456789012:role/tinybird-s3-access",
+      });
+
+      expect(getConnectionType(conn)).toBe("s3");
+    });
   });
 });

package/src/schema/connection.ts

@@ -1,6 +1,6 @@
 /**
  * Connection definition for Tinybird
- * Define external connections (Kafka, etc.) as TypeScript with full type safety
+ * Define external connections (Kafka, S3, etc.) as TypeScript with full type safety
  */
 
 // Symbol for brand typing - use Symbol.for() for global registry
@@ -50,13 +50,50 @@ export interface KafkaConnectionDefinition {
   readonly options: KafkaConnectionOptions;
 }
 
+/**
+ * Options for defining an S3 connection
+ */
+export interface S3ConnectionOptions {
+  /** S3 bucket region (for example: us-east-1) */
+  region: string;
+  /** IAM role ARN used by Tinybird to access the bucket */
+  arn?: string;
+  /** S3 access key for key/secret auth */
+  accessKey?: string;
+  /** S3 secret key for key/secret auth */
+  secret?: string;
+}
+
+/**
+ * S3-specific connection definition
+ */
+export interface S3ConnectionDefinition {
+  readonly [CONNECTION_BRAND]: true;
+  /** Connection name */
+  readonly _name: string;
+  /** Type marker for inference */
+  readonly _type: "connection";
+  /** Connection type */
+  readonly _connectionType: "s3";
+  /** S3 options */
+  readonly options: S3ConnectionOptions;
+}
+
 /**
  * A connection definition - union of all connection types
  */
-export type ConnectionDefinition = KafkaConnectionDefinition;
+export type ConnectionDefinition = KafkaConnectionDefinition | S3ConnectionDefinition;
+
+function validateConnectionName(name: string): void {
+  if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) {
+    throw new Error(
+      `Invalid connection name: "${name}". Must start with a letter or underscore and contain only alphanumeric characters and underscores.`
+    );
+  }
+}
 
 /**
- * Create a Kafka connection
+ * Define a Kafka connection
  *
  * @param name - The connection name (must be valid identifier)
  * @param options - Kafka connection configuration
@@ -64,9 +101,9 @@ export type ConnectionDefinition = KafkaConnectionDefinition;
  *
  * @example
  * ```ts
- * import { createKafkaConnection } from '@tinybirdco/sdk';
+ * import { defineKafkaConnection } from '@tinybirdco/sdk';
  *
- * export const myKafka = createKafkaConnection('my_kafka', {
+ * export const myKafka = defineKafkaConnection('my_kafka', {
  *   bootstrapServers: 'kafka.example.com:9092',
  *   securityProtocol: 'SASL_SSL',
  *   saslMechanism: 'PLAIN',
@@ -75,22 +112,54 @@ export type ConnectionDefinition = KafkaConnectionDefinition;
  * });
  * ```
  */
-export function createKafkaConnection(
+export function defineKafkaConnection(
   name: string,
   options: KafkaConnectionOptions
 ): KafkaConnectionDefinition {
-  // Validate name is a valid identifier
-  if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) {
+  validateConnectionName(name);
+
+  return {
+    [CONNECTION_BRAND]: true,
+    _name: name,
+    _type: "connection",
+    _connectionType: "kafka",
+    options,
+  };
+}
+
+/**
+ * @deprecated Use defineKafkaConnection instead.
+ */
+export const createKafkaConnection = defineKafkaConnection;
+
+/**
+ * Define an S3 connection
+ *
+ * @param name - The connection name (must be valid identifier)
+ * @param options - S3 connection configuration
+ * @returns A connection definition that can be used in a project
+ */
+export function defineS3Connection(
+  name: string,
+  options: S3ConnectionOptions
+): S3ConnectionDefinition {
+  validateConnectionName(name);
+
+  if (!options.arn && !(options.accessKey && options.secret)) {
     throw new Error(
-      `Invalid connection name: "${name}". Must start with a letter or underscore and contain only alphanumeric characters and underscores.`
+      "S3 connection requires either `arn` or both `accessKey` and `secret`."
     );
   }
 
+  if ((options.accessKey && !options.secret) || (!options.accessKey && options.secret)) {
+    throw new Error("S3 connection `accessKey` and `secret` must be provided together.");
+  }
+
   return {
     [CONNECTION_BRAND]: true,
     _name: name,
     _type: "connection",
-    _connectionType: "kafka",
+    _connectionType: "s3",
     options,
   };
 }
@@ -114,6 +183,13 @@ export function isKafkaConnectionDefinition(value: unknown): value is KafkaConne
   return isConnectionDefinition(value) && value._connectionType === "kafka";
 }
 
+/**
+ * Check if a value is an S3 connection definition
+ */
+export function isS3ConnectionDefinition(value: unknown): value is S3ConnectionDefinition {
+  return isConnectionDefinition(value) && value._connectionType === "s3";
+}
+
 /**
  * Get the connection type from a connection definition
  */

package/src/schema/datasource.test.ts

@@ -9,6 +9,7 @@ import {
 } from "./datasource.js";
 import { t } from "./types.js";
 import { engine } from "./engines.js";
+import { defineKafkaConnection, defineS3Connection } from "./connection.js";
 
 describe("Datasource Schema", () => {
   describe("defineDatasource", () => {
@@ -85,6 +86,30 @@ describe("Datasource Schema", () => {
       });
       expect(ds2._name).toBe("events_v2");
     });
+
+    it("throws when both kafka and s3 ingestion are configured", () => {
+      const kafkaConn = defineKafkaConnection("my_kafka", {
+        bootstrapServers: "kafka.example.com:9092",
+      });
+      const s3Conn = defineS3Connection("my_s3", {
+        region: "us-east-1",
+        arn: "arn:aws:iam::123456789012:role/tinybird-s3-access",
+      });
+
+      expect(() =>
+        defineDatasource("events", {
+          schema: { id: t.string() },
+          kafka: {
+            connection: kafkaConn,
+            topic: "events",
+          },
+          s3: {
+            connection: s3Conn,
+            bucketUri: "s3://my-bucket/events/*.csv",
+          },
+        })
+      ).toThrow("Datasource cannot define both `kafka` and `s3` ingestion options.");
+    });
   });
 
   describe("isDatasourceDefinition", () => {

package/src/schema/datasource.ts

@@ -5,7 +5,7 @@
 
 import type { AnyTypeValidator } from "./types.js";
 import type { EngineConfig } from "./engines.js";
-import type { KafkaConnectionDefinition } from "./connection.js";
+import type { KafkaConnectionDefinition, S3ConnectionDefinition } from "./connection.js";
 import type { TokenDefinition, DatasourceTokenScope } from "./token.js";
 
 // Symbol for brand typing - use Symbol.for() for global registry
@@ -68,6 +68,20 @@ export interface KafkaConfig {
   autoOffsetReset?: "earliest" | "latest";
 }
 
+/**
+ * S3 import configuration for a datasource
+ */
+export interface S3Config {
+  /** S3 connection to use */
+  connection: S3ConnectionDefinition;
+  /** S3 bucket URI, for example: s3://my-bucket/path/*.csv */
+  bucketUri: string;
+  /** Import schedule, for example: @auto or @once */
+  schedule?: string;
+  /** Incremental import lower bound timestamp expression */
+  fromTimestamp?: string;
+}
+
 /**
  * Options for defining a datasource
  */
@@ -95,6 +109,8 @@ export interface DatasourceOptions<TSchema extends SchemaDefinition> {
   forwardQuery?: string;
   /** Kafka ingestion configuration */
   kafka?: KafkaConfig;
+  /** S3 ingestion configuration */
+  s3?: S3Config;
 }
 
 /**
@@ -152,6 +168,10 @@ export function defineDatasource<TSchema extends SchemaDefinition>(
     );
   }
 
+  if (options.kafka && options.s3) {
+    throw new Error("Datasource cannot define both `kafka` and `s3` ingestion options.");
+  }
+
   return {
     [DATASOURCE_BRAND]: true,
     _name: name,

package/src/schema/params.ts

@@ -173,13 +173,13 @@ export const p = {
 
   // ============ Date/Time Types ============
 
-  /** Date parameter (YYYY-MM-DD format) */
+  /** Date parameter (YYYY-MM-DD format, e.g. 2024-01-15) */
   date: () => createParamValidator<string, "Date">("Date"),
 
-  /** DateTime parameter (YYYY-MM-DD HH:MM:SS format) */
+  /** DateTime parameter (YYYY-MM-DD HH:MM:SS format, e.g. 2024-01-15 10:30:00) */
   dateTime: () => createParamValidator<string, "DateTime">("DateTime"),
 
-  /** DateTime64 parameter with sub-second precision */
+  /** DateTime64 parameter (YYYY-MM-DD HH:MM:SS[.fraction] format, e.g. 2024-01-15 10:30:00.123) */
   dateTime64: () => createParamValidator<string, "DateTime64">("DateTime64"),
 
   // ============ Array Types ============

package/src/schema/project.ts

@@ -272,13 +272,13 @@ interface TinybirdConstructor {
  *
  * // Query a pipe (fully typed)
  * const result = await tinybird.topPages.query({
- *   start_date: new Date('2024-01-01'),
- *   end_date: new Date('2024-01-31'),
+ *   start_date: '2024-01-01 00:00:00',
+ *   end_date: '2024-01-31 23:59:59',
  * });
  *
  * // Ingest an event (fully typed)
  * await tinybird.pageViews.ingest({
- *   timestamp: new Date(),
+ *   timestamp: '2024-01-15 10:30:00',
  *   pathname: '/home',
  *   session_id: 'abc123',
  * });

package/src/schema/types.ts

@@ -203,25 +203,25 @@ export const t = {
 
   // ============ Date/Time Types ============
 
-  /** Date - date without time (YYYY-MM-DD) */
-  date: () => createValidator<Date, "Date">("Date"),
+  /** Date - string in YYYY-MM-DD format (e.g. 2024-01-15) */
+  date: () => createValidator<string, "Date">("Date"),
 
-  /** Date32 - extended date range */
-  date32: () => createValidator<Date, "Date32">("Date32"),
+  /** Date32 - string in YYYY-MM-DD format (e.g. 2024-01-15, extended date range) */
+  date32: () => createValidator<string, "Date32">("Date32"),
 
-  /** DateTime - date and time with second precision */
+  /** DateTime - string in YYYY-MM-DD HH:MM:SS format (e.g. 2024-01-15 10:30:00) */
   dateTime: (timezone?: string) =>
     timezone
-      ? createValidator<Date, `DateTime('${string}')`>(`DateTime('${timezone}')`)
-      : createValidator<Date, "DateTime">("DateTime"),
+      ? createValidator<string, `DateTime('${string}')`>(`DateTime('${timezone}')`)
+      : createValidator<string, "DateTime">("DateTime"),
 
-  /** DateTime64 - date and time with sub-second precision */
+  /** DateTime64 - string in YYYY-MM-DD HH:MM:SS[.fraction] format (e.g. 2024-01-15 10:30:00.123) */
   dateTime64: (precision: 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 = 3, timezone?: string) =>
     timezone
-      ? createValidator<Date, `DateTime64(${number}, '${string}')`>(
+      ? createValidator<string, `DateTime64(${number}, '${string}')`>(
           `DateTime64(${precision}, '${timezone}')`
         )
-      : createValidator<Date, `DateTime64(${number})`>(`DateTime64(${precision})`),
+      : createValidator<string, `DateTime64(${number})`>(`DateTime64(${precision})`),
 
   // ============ Complex Types ============