@tinybirdco/sdk 0.0.44 → 0.0.46

package/src/index.ts

@@ -109,16 +109,27 @@ export type {
   DatasourceTokenReference,
   ExtractSchema,
   KafkaConfig,
+  S3Config,
 } from "./schema/datasource.js";
 
 // ============ Connection ============
-export { createKafkaConnection, isConnectionDefinition, isKafkaConnectionDefinition, getConnectionType } from "./schema/connection.js";
+export {
+  defineKafkaConnection,
+  createKafkaConnection,
+  defineS3Connection,
+  isConnectionDefinition,
+  isKafkaConnectionDefinition,
+  isS3ConnectionDefinition,
+  getConnectionType,
+} from "./schema/connection.js";
 export type {
   ConnectionDefinition,
   KafkaConnectionDefinition,
   KafkaConnectionOptions,
   KafkaSecurityProtocol,
   KafkaSaslMechanism,
+  S3ConnectionDefinition,
+  S3ConnectionOptions,
 } from "./schema/connection.js";
 
 // ============ Token ============

package/src/infer/index.ts

@@ -52,7 +52,7 @@ type InferColumn<T> = T extends TypeValidator<infer U, string, TypeModifiers>
  * });
  *
  * type EventRow = InferRow<typeof events>;
- * // { id: string; count: number; timestamp: Date }
+ * // { id: string; count: number; timestamp: string }
  * ```
  */
 export type InferRow<T> = T extends DatasourceDefinition<infer S>
@@ -156,10 +156,10 @@ export type InferOutputRow<T> = T extends PipeDefinition<ParamsDefinition, infer
  * });
  *
  * type Event = InferEvent<typeof events>;
- * // { id: string; timestamp: Date }
+ * // { id: string; timestamp: string }
  *
  * // Use for type-safe event ingestion
- * const event: Event = { id: '123', timestamp: new Date() };
+ * const event: Event = { id: '123', timestamp: '2024-01-01 00:00:00' };
  * ```
  */
 export type InferEvent<T> = InferRow<T>;

package/src/migrate/emit-ts.ts

@@ -6,6 +6,7 @@ import type {
   KafkaConnectionModel,
   ParsedResource,
   PipeModel,
+  S3ConnectionModel,
 } from "./types.js";
 
 function escapeString(value: string): string {
@@ -243,6 +244,20 @@ function emitDatasource(ds: DatasourceModel): string {
     lines.push("  },");
   }
 
+  if (ds.s3) {
+    const connectionVar = toCamelCase(ds.s3.connectionName);
+    lines.push("  s3: {");
+    lines.push(`    connection: ${connectionVar},`);
+    lines.push(`    bucketUri: ${escapeString(ds.s3.bucketUri)},`);
+    if (ds.s3.schedule) {
+      lines.push(`    schedule: ${escapeString(ds.s3.schedule)},`);
+    }
+    if (ds.s3.fromTimestamp) {
+      lines.push(`    fromTimestamp: ${escapeString(ds.s3.fromTimestamp)},`);
+    }
+    lines.push("  },");
+  }
+
   if (ds.forwardQuery) {
     lines.push("  forwardQuery: `");
     lines.push(ds.forwardQuery.replace(/`/g, "\\`").replace(/\${/g, "\\${"));
@@ -270,28 +285,48 @@ function emitDatasource(ds: DatasourceModel): string {
   return lines.join("\n");
 }
 
-function emitConnection(connection: KafkaConnectionModel): string {
+function emitConnection(connection: KafkaConnectionModel | S3ConnectionModel): string {
   const variableName = toCamelCase(connection.name);
   const lines: string[] = [];
+
+  if (connection.connectionType === "kafka") {
+    lines.push(
+      `export const ${variableName} = defineKafkaConnection(${escapeString(connection.name)}, {`
+    );
+    lines.push(`  bootstrapServers: ${escapeString(connection.bootstrapServers)},`);
+    if (connection.securityProtocol) {
+      lines.push(`  securityProtocol: ${escapeString(connection.securityProtocol)},`);
+    }
+    if (connection.saslMechanism) {
+      lines.push(`  saslMechanism: ${escapeString(connection.saslMechanism)},`);
+    }
+    if (connection.key) {
+      lines.push(`  key: ${escapeString(connection.key)},`);
+    }
+    if (connection.secret) {
+      lines.push(`  secret: ${escapeString(connection.secret)},`);
+    }
+    if (connection.sslCaPem) {
+      lines.push(`  sslCaPem: ${escapeString(connection.sslCaPem)},`);
+    }
+    lines.push("});");
+    lines.push("");
+    return lines.join("\n");
+  }
+
   lines.push(
-    `export const ${variableName} = createKafkaConnection(${escapeString(connection.name)}, {`
+    `export const ${variableName} = defineS3Connection(${escapeString(connection.name)}, {`
   );
-  lines.push(`  bootstrapServers: ${escapeString(connection.bootstrapServers)},`);
-  if (connection.securityProtocol) {
-    lines.push(`  securityProtocol: ${escapeString(connection.securityProtocol)},`);
-  }
-  if (connection.saslMechanism) {
-    lines.push(`  saslMechanism: ${escapeString(connection.saslMechanism)},`);
+  lines.push(`  region: ${escapeString(connection.region)},`);
+  if (connection.arn) {
+    lines.push(`  arn: ${escapeString(connection.arn)},`);
   }
-  if (connection.key) {
-    lines.push(`  key: ${escapeString(connection.key)},`);
+  if (connection.accessKey) {
+    lines.push(`  accessKey: ${escapeString(connection.accessKey)},`);
   }
   if (connection.secret) {
     lines.push(`  secret: ${escapeString(connection.secret)},`);
   }
-  if (connection.sslCaPem) {
-    lines.push(`  sslCaPem: ${escapeString(connection.sslCaPem)},`);
-  }
   lines.push("});");
   lines.push("");
   return lines.join("\n");
@@ -398,7 +433,8 @@ function emitPipe(pipe: PipeModel): string {
 
 export function emitMigrationFileContent(resources: ParsedResource[]): string {
   const connections = resources.filter(
-    (resource): resource is KafkaConnectionModel => resource.kind === "connection"
+    (resource): resource is KafkaConnectionModel | S3ConnectionModel =>
+      resource.kind === "connection"
   );
   const datasources = resources.filter(
     (resource): resource is DatasourceModel => resource.kind === "datasource"
@@ -412,7 +448,21 @@ export function emitMigrationFileContent(resources: ParsedResource[]): string {
   );
   const needsParams = pipes.some((pipe) => pipe.params.length > 0);
 
-  const imports = new Set<string>(["createKafkaConnection", "defineDatasource", "definePipe", "defineMaterializedView", "defineCopyPipe", "node", "t", "engine"]);
+  const imports = new Set<string>([
+    "defineDatasource",
+    "definePipe",
+    "defineMaterializedView",
+    "defineCopyPipe",
+    "node",
+    "t",
+    "engine",
+  ]);
+  if (connections.some((connection) => connection.connectionType === "kafka")) {
+    imports.add("defineKafkaConnection");
+  }
+  if (connections.some((connection) => connection.connectionType === "s3")) {
+    imports.add("defineS3Connection");
+  }
   if (needsColumn) {
     imports.add("column");
   }
@@ -420,13 +470,27 @@ export function emitMigrationFileContent(resources: ParsedResource[]): string {
     imports.add("p");
   }
 
+  const orderedImports = [
+    "defineKafkaConnection",
+    "defineS3Connection",
+    "defineDatasource",
+    "definePipe",
+    "defineMaterializedView",
+    "defineCopyPipe",
+    "node",
+    "t",
+    "engine",
+    "column",
+    "p",
+  ].filter((name) => imports.has(name));
+
   const lines: string[] = [];
   lines.push("/**");
   lines.push(" * Generated by tinybird migrate.");
   lines.push(" * Review endpoint output schemas and any defaults before production use.");
   lines.push(" */");
   lines.push("");
-  lines.push(`import { ${Array.from(imports).join(", ")} } from "@tinybirdco/sdk";`);
+  lines.push(`import { ${orderedImports.join(", ")} } from "@tinybirdco/sdk";`);
   lines.push("");
 
   if (connections.length > 0) {

package/src/migrate/parse-connection.ts

@@ -1,14 +1,18 @@
-import type { KafkaConnectionModel, ResourceFile } from "./types.js";
+import type { KafkaConnectionModel, ResourceFile, S3ConnectionModel } from "./types.js";
 import {
   MigrationParseError,
   isBlank,
   parseDirectiveLine,
+  parseQuotedValue,
   splitLines,
 } from "./parser-utils.js";
 
-export function parseConnectionFile(resource: ResourceFile): KafkaConnectionModel {
+export function parseConnectionFile(
+  resource: ResourceFile
+): KafkaConnectionModel | S3ConnectionModel {
   const lines = splitLines(resource.content);
   let connectionType: string | undefined;
+
   let bootstrapServers: string | undefined;
   let securityProtocol:
     | "SASL_SSL"
@@ -25,6 +29,11 @@ export function parseConnectionFile(resource: ResourceFile): KafkaConnectionMode
   let secret: string | undefined;
   let sslCaPem: string | undefined;
 
+  let region: string | undefined;
+  let arn: string | undefined;
+  let accessKey: string | undefined;
+  let accessSecret: string | undefined;
+
   for (const rawLine of lines) {
     const line = rawLine.trim();
     if (isBlank(line)) {
@@ -34,7 +43,7 @@ export function parseConnectionFile(resource: ResourceFile): KafkaConnectionMode
     const { key: directive, value } = parseDirectiveLine(line);
     switch (directive) {
       case "TYPE":
-        connectionType = value;
+        connectionType = parseQuotedValue(value);
         break;
       case "KAFKA_BOOTSTRAP_SERVERS":
         bootstrapServers = value;
@@ -75,6 +84,18 @@ export function parseConnectionFile(resource: ResourceFile): KafkaConnectionMode
       case "KAFKA_SSL_CA_PEM":
         sslCaPem = value;
         break;
+      case "S3_REGION":
+        region = parseQuotedValue(value);
+        break;
+      case "S3_ARN":
+        arn = parseQuotedValue(value);
+        break;
+      case "S3_ACCESS_KEY":
+        accessKey = parseQuotedValue(value);
+        break;
+      case "S3_SECRET":
+        accessSecret = parseQuotedValue(value);
+        break;
       default:
         throw new MigrationParseError(
           resource.filePath,
@@ -94,35 +115,92 @@ export function parseConnectionFile(resource: ResourceFile): KafkaConnectionMode
     );
   }
 
-  if (connectionType !== "kafka") {
-    throw new MigrationParseError(
-      resource.filePath,
-      "connection",
-      resource.name,
-      `Unsupported connection type in strict mode: "${connectionType}"`
-    );
+  if (connectionType === "kafka") {
+    if (region || arn || accessKey || accessSecret) {
+      throw new MigrationParseError(
+        resource.filePath,
+        "connection",
+        resource.name,
+        "S3 directives are not valid for kafka connections."
+      );
+    }
+
+    if (!bootstrapServers) {
+      throw new MigrationParseError(
+        resource.filePath,
+        "connection",
+        resource.name,
+        "KAFKA_BOOTSTRAP_SERVERS is required for kafka connections."
+      );
+    }
+
+    return {
+      kind: "connection",
+      name: resource.name,
+      filePath: resource.filePath,
+      connectionType: "kafka",
+      bootstrapServers,
+      securityProtocol,
+      saslMechanism,
+      key,
+      secret,
+      sslCaPem,
+    };
   }
 
-  if (!bootstrapServers) {
-    throw new MigrationParseError(
-      resource.filePath,
-      "connection",
-      resource.name,
-      "KAFKA_BOOTSTRAP_SERVERS is required for kafka connections."
-    );
+  if (connectionType === "s3") {
+    if (bootstrapServers || securityProtocol || saslMechanism || key || secret || sslCaPem) {
+      throw new MigrationParseError(
+        resource.filePath,
+        "connection",
+        resource.name,
+        "Kafka directives are not valid for s3 connections."
+      );
+    }
+
+    if (!region) {
+      throw new MigrationParseError(
+        resource.filePath,
+        "connection",
+        resource.name,
+        "S3_REGION is required for s3 connections."
+      );
+    }
+
+    if (!arn && !(accessKey && accessSecret)) {
+      throw new MigrationParseError(
+        resource.filePath,
+        "connection",
+        resource.name,
+        "S3 connections require S3_ARN or both S3_ACCESS_KEY and S3_SECRET."
+      );
+    }
+
+    if ((accessKey && !accessSecret) || (!accessKey && accessSecret)) {
+      throw new MigrationParseError(
+        resource.filePath,
+        "connection",
+        resource.name,
+        "S3_ACCESS_KEY and S3_SECRET must be provided together."
+      );
+    }
+
+    return {
+      kind: "connection",
+      name: resource.name,
+      filePath: resource.filePath,
+      connectionType: "s3",
+      region,
+      arn,
+      accessKey,
+      secret: accessSecret,
+    };
   }
 
-  return {
-    kind: "connection",
-    name: resource.name,
-    filePath: resource.filePath,
-    connectionType: "kafka",
-    bootstrapServers,
-    securityProtocol,
-    saslMechanism,
-    key,
-    secret,
-    sslCaPem,
-  };
+  throw new MigrationParseError(
+    resource.filePath,
+    "connection",
+    resource.name,
+    `Unsupported connection type in strict mode: "${connectionType}"`
+  );
 }
-

package/src/migrate/parse-datasource.ts

@@ -233,6 +233,11 @@ export function parseDatasourceFile(resource: ResourceFile): DatasourceModel {
   let kafkaGroupId: string | undefined;
   let kafkaAutoOffsetReset: "earliest" | "latest" | undefined;
 
+  let importConnectionName: string | undefined;
+  let importBucketUri: string | undefined;
+  let importSchedule: string | undefined;
+  let importFromTimestamp: string | undefined;
+
   let i = 0;
   while (i < lines.length) {
     const rawLine = lines[i] ?? "";
@@ -365,6 +370,18 @@ export function parseDatasourceFile(resource: ResourceFile): DatasourceModel {
         }
         kafkaAutoOffsetReset = value;
         break;
+      case "IMPORT_CONNECTION_NAME":
+        importConnectionName = parseQuotedValue(value);
+        break;
+      case "IMPORT_BUCKET_URI":
+        importBucketUri = parseQuotedValue(value);
+        break;
+      case "IMPORT_SCHEDULE":
+        importSchedule = parseQuotedValue(value);
+        break;
+      case "IMPORT_FROM_TIMESTAMP":
+        importFromTimestamp = parseQuotedValue(value);
+        break;
       case "TOKEN":
         tokens.push(parseToken(resource.filePath, resource.name, value));
         break;
@@ -426,6 +443,34 @@ export function parseDatasourceFile(resource: ResourceFile): DatasourceModel {
     );
   }
 
+  const s3 =
+    importConnectionName || importBucketUri || importSchedule || importFromTimestamp
+      ? {
+          connectionName: importConnectionName ?? "",
+          bucketUri: importBucketUri ?? "",
+          schedule: importSchedule,
+          fromTimestamp: importFromTimestamp,
+        }
+      : undefined;
+
+  if (s3 && (!s3.connectionName || !s3.bucketUri)) {
+    throw new MigrationParseError(
+      resource.filePath,
+      "datasource",
+      resource.name,
+      "IMPORT_CONNECTION_NAME and IMPORT_BUCKET_URI are required when S3 import directives are used."
+    );
+  }
+
+  if (kafka && s3) {
+    throw new MigrationParseError(
+      resource.filePath,
+      "datasource",
+      resource.name,
+      "Datasource cannot mix Kafka directives with S3 import directives."
+    );
+  }
+
   return {
     kind: "datasource",
     name: resource.name,
@@ -445,9 +490,9 @@ export function parseDatasourceFile(resource: ResourceFile): DatasourceModel {
       settings,
     },
     kafka,
+    s3,
     forwardQuery,
     tokens,
     sharedWith,
   };
 }
-

package/src/migrate/types.ts

@@ -43,6 +43,13 @@ export interface DatasourceKafkaModel {
   autoOffsetReset?: "earliest" | "latest";
 }
 
+export interface DatasourceS3Model {
+  connectionName: string;
+  bucketUri: string;
+  schedule?: string;
+  fromTimestamp?: string;
+}
+
 export interface DatasourceTokenModel {
   name: string;
   scope: "READ" | "APPEND";
@@ -56,6 +63,7 @@ export interface DatasourceModel {
   columns: DatasourceColumnModel[];
   engine: DatasourceEngineModel;
   kafka?: DatasourceKafkaModel;
+  s3?: DatasourceS3Model;
   forwardQuery?: string;
   tokens: DatasourceTokenModel[];
   sharedWith: string[];
@@ -112,7 +120,22 @@ export interface KafkaConnectionModel {
   sslCaPem?: string;
 }
 
-export type ParsedResource = DatasourceModel | PipeModel | KafkaConnectionModel;
+export interface S3ConnectionModel {
+  kind: "connection";
+  name: string;
+  filePath: string;
+  connectionType: "s3";
+  region: string;
+  arn?: string;
+  accessKey?: string;
+  secret?: string;
+}
+
+export type ParsedResource =
+  | DatasourceModel
+  | PipeModel
+  | KafkaConnectionModel
+  | S3ConnectionModel;
 
 export interface MigrationResult {
   success: boolean;
@@ -122,4 +145,3 @@ export interface MigrationResult {
   dryRun: boolean;
   outputContent?: string;
 }
-