@tinybirdco/sdk 0.0.48 → 0.0.50

package/src/cli/commands/migrate.ts

@@ -112,6 +112,9 @@ export async function runMigrate(
   const migrated: ParsedResource[] = [];
   const migratedConnectionNames = new Set<string>();
   const migratedDatasourceNames = new Set<string>();
+  const parsedConnectionTypeByName = new Map(
+    parsedConnections.map((connection) => [connection.name, connection.connectionType] as const)
+  );
 
   for (const connection of parsedConnections) {
     try {
@@ -130,7 +133,9 @@ export async function runMigrate(
 
   for (const datasource of parsedDatasources) {
     const referencedConnectionName =
-      datasource.kafka?.connectionName ?? datasource.s3?.connectionName;
+      datasource.kafka?.connectionName ??
+      datasource.s3?.connectionName ??
+      datasource.gcs?.connectionName;
 
     if (
       referencedConnectionName &&
@@ -145,6 +150,42 @@ export async function runMigrate(
       continue;
     }
 
+    if (datasource.kafka) {
+      const kafkaConnectionType = parsedConnectionTypeByName.get(datasource.kafka.connectionName);
+      if (kafkaConnectionType !== "kafka") {
+        errors.push({
+          filePath: datasource.filePath,
+          resourceName: datasource.name,
+          resourceKind: datasource.kind,
+          message: `Datasource kafka ingestion requires a kafka connection, found "${kafkaConnectionType ?? "(none)"}".`,
+        });
+        continue;
+      }
+    }
+
+    const importConfig = datasource.s3 ?? datasource.gcs;
+    if (importConfig) {
+      const importConnectionType = parsedConnectionTypeByName.get(importConfig.connectionName);
+      if (importConnectionType !== "s3" && importConnectionType !== "gcs") {
+        errors.push({
+          filePath: datasource.filePath,
+          resourceName: datasource.name,
+          resourceKind: datasource.kind,
+          message:
+            `Datasource import directives require an s3 or gcs connection, found "${importConnectionType ?? "(none)"}".`,
+        });
+        continue;
+      }
+
+      if (importConnectionType === "gcs") {
+        datasource.gcs = { ...importConfig };
+        datasource.s3 = undefined;
+      } else {
+        datasource.s3 = { ...importConfig };
+        datasource.gcs = undefined;
+      }
+    }
+
     try {
       validateResourceForEmission(datasource);
       migrated.push(datasource);
@@ -160,6 +201,38 @@ export async function runMigrate(
   }
 
   for (const pipe of parsedPipes) {
+    if (pipe.type === "sink") {
+      const sinkConnectionName = pipe.sink?.connectionName;
+      if (!sinkConnectionName || !migratedConnectionNames.has(sinkConnectionName)) {
+        errors.push({
+          filePath: pipe.filePath,
+          resourceName: pipe.name,
+          resourceKind: pipe.kind,
+          message: `Sink pipe references missing/unmigrated connection "${sinkConnectionName ?? "(none)"}".`,
+        });
+        continue;
+      }
+      const sinkConnectionType = parsedConnectionTypeByName.get(sinkConnectionName);
+      if (!sinkConnectionType) {
+        errors.push({
+          filePath: pipe.filePath,
+          resourceName: pipe.name,
+          resourceKind: pipe.kind,
+          message: `Sink pipe connection "${sinkConnectionName}" could not be resolved.`,
+        });
+        continue;
+      }
+      if (sinkConnectionType !== pipe.sink?.service) {
+        errors.push({
+          filePath: pipe.filePath,
+          resourceName: pipe.name,
+          resourceKind: pipe.kind,
+          message: `Sink pipe service "${pipe.sink?.service}" is incompatible with connection "${sinkConnectionName}" type "${sinkConnectionType}".`,
+        });
+        continue;
+      }
+    }
+
     if (
       pipe.type === "materialized" &&
       (!pipe.materializedDatasource ||

package/src/generator/connection.test.ts

@@ -1,6 +1,10 @@
 import { describe, it, expect } from "vitest";
 import { generateConnection, generateAllConnections } from "./connection.js";
-import { defineKafkaConnection, defineS3Connection } from "../schema/connection.js";
+import {
+  defineKafkaConnection,
+  defineS3Connection,
+  defineGCSConnection,
+} from "../schema/connection.js";
 
 describe("Connection Generator", () => {
   describe("generateConnection", () => {
@@ -152,6 +156,20 @@ describe("Connection Generator", () => {
       expect(result.content).toContain('S3_ACCESS_KEY {{ tb_secret("S3_ACCESS_KEY") }}');
       expect(result.content).toContain('S3_SECRET {{ tb_secret("S3_SECRET") }}');
     });
+
+    it("generates GCS connection with service account credentials", () => {
+      const conn = defineGCSConnection("my_gcs", {
+        serviceAccountCredentialsJson: '{{ tb_secret("GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON") }}',
+      });
+
+      const result = generateConnection(conn);
+
+      expect(result.name).toBe("my_gcs");
+      expect(result.content).toContain("TYPE gcs");
+      expect(result.content).toContain(
+        'GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON {{ tb_secret("GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON") }}'
+      );
+    });
   });
 
   describe("generateAllConnections", () => {
@@ -163,11 +181,18 @@ describe("Connection Generator", () => {
         region: "us-east-1",
         arn: "arn:aws:iam::123456789012:role/tinybird-s3-access",
       });
+      const conn3 = defineGCSConnection("gcs_landing", {
+        serviceAccountCredentialsJson: '{{ tb_secret("GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON") }}',
+      });
 
-      const results = generateAllConnections({ kafka1: conn1, s3_logs: conn2 });
+      const results = generateAllConnections({
+        kafka1: conn1,
+        s3_logs: conn2,
+        gcs_landing: conn3,
+      });
 
-      expect(results).toHaveLength(2);
-      expect(results.map((r) => r.name).sort()).toEqual(["kafka1", "s3_logs"]);
+      expect(results).toHaveLength(3);
+      expect(results.map((r) => r.name).sort()).toEqual(["gcs_landing", "kafka1", "s3_logs"]);
     });
 
     it("returns empty array for empty connections", () => {

package/src/generator/connection.ts

@@ -3,8 +3,16 @@
  * Converts ConnectionDefinition to native .connection file format
  */
 
-import type { ConnectionDefinition, KafkaConnectionDefinition } from "../schema/connection.js";
-import { isS3ConnectionDefinition, type S3ConnectionDefinition } from "../schema/connection.js";
+import type {
+  ConnectionDefinition,
+  KafkaConnectionDefinition,
+  GCSConnectionDefinition,
+} from "../schema/connection.js";
+import {
+  isS3ConnectionDefinition,
+  isGCSConnectionDefinition,
+  type S3ConnectionDefinition,
+} from "../schema/connection.js";
 
 /**
  * Generated connection content
@@ -78,6 +86,19 @@ function generateS3Connection(connection: S3ConnectionDefinition): string {
   return parts.join("\n");
 }
 
+/**
+ * Generate a GCS connection content
+ */
+function generateGCSConnection(connection: GCSConnectionDefinition): string {
+  const parts: string[] = [];
+  const options = connection.options;
+
+  parts.push("TYPE gcs");
+  parts.push(`GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON ${options.serviceAccountCredentialsJson}`);
+
+  return parts.join("\n");
+}
+
 /**
  * Generate a .connection file content from a ConnectionDefinition
  *
@@ -113,6 +134,8 @@ export function generateConnection(
     content = generateKafkaConnection(connection as KafkaConnectionDefinition);
   } else if (isS3ConnectionDefinition(connection)) {
     content = generateS3Connection(connection);
+  } else if (isGCSConnectionDefinition(connection)) {
+    content = generateGCSConnection(connection);
   } else {
     throw new Error("Unsupported connection type.");
   }

package/src/generator/datasource.test.ts

@@ -1,7 +1,7 @@
 import { describe, it, expect } from 'vitest';
 import { generateDatasource, generateAllDatasources } from './datasource.js';
 import { defineDatasource } from '../schema/datasource.js';
-import { defineKafkaConnection, defineS3Connection } from '../schema/connection.js';
+import { defineKafkaConnection, defineS3Connection, defineGCSConnection } from '../schema/connection.js';
 import { defineToken } from '../schema/token.js';
 import { t } from '../schema/types.js';
 import { engine } from '../schema/engines.js';
@@ -529,6 +529,35 @@ describe('Datasource Generator', () => {
     });
   });
 
+  describe('GCS configuration', () => {
+    it('includes GCS import directives', () => {
+      const gcsConn = defineGCSConnection('my_gcs', {
+        serviceAccountCredentialsJson: '{{ tb_secret("GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON") }}',
+      });
+
+      const ds = defineDatasource('gcs_events', {
+        schema: {
+          timestamp: t.dateTime(),
+          event: t.string(),
+        },
+        engine: engine.mergeTree({ sortingKey: ['timestamp'] }),
+        gcs: {
+          connection: gcsConn,
+          bucketUri: 'gs://my-bucket/events/*.csv',
+          schedule: '@auto',
+          fromTimestamp: '2024-01-01T00:00:00Z',
+        },
+      });
+
+      const result = generateDatasource(ds);
+
+      expect(result.content).toContain('IMPORT_CONNECTION_NAME my_gcs');
+      expect(result.content).toContain('IMPORT_BUCKET_URI gs://my-bucket/events/*.csv');
+      expect(result.content).toContain('IMPORT_SCHEDULE @auto');
+      expect(result.content).toContain('IMPORT_FROM_TIMESTAMP 2024-01-01T00:00:00Z');
+    });
+  });
+
   describe('Token generation', () => {
     it('generates TOKEN lines with inline config', () => {
       const ds = defineDatasource('test_ds', {

package/src/generator/datasource.ts

@@ -9,6 +9,7 @@ import type {
   ColumnDefinition,
   KafkaConfig,
   S3Config,
+  GCSConfig,
   TokenConfig,
 } from "../schema/datasource.js";
 import type { AnyTypeValidator, TypeModifiers } from "../schema/types.js";
@@ -176,18 +177,18 @@ function generateKafkaConfig(kafka: KafkaConfig): string {
 /**
  * Generate S3 import configuration lines
  */
-function generateS3Config(s3: S3Config): string {
+function generateImportConfig(importConfig: S3Config | GCSConfig): string {
   const parts: string[] = [];
 
-  parts.push(`IMPORT_CONNECTION_NAME ${s3.connection._name}`);
-  parts.push(`IMPORT_BUCKET_URI ${s3.bucketUri}`);
+  parts.push(`IMPORT_CONNECTION_NAME ${importConfig.connection._name}`);
+  parts.push(`IMPORT_BUCKET_URI ${importConfig.bucketUri}`);
 
-  if (s3.schedule) {
-    parts.push(`IMPORT_SCHEDULE ${s3.schedule}`);
+  if (importConfig.schedule) {
+    parts.push(`IMPORT_SCHEDULE ${importConfig.schedule}`);
   }
 
-  if (s3.fromTimestamp) {
-    parts.push(`IMPORT_FROM_TIMESTAMP ${s3.fromTimestamp}`);
+  if (importConfig.fromTimestamp) {
+    parts.push(`IMPORT_FROM_TIMESTAMP ${importConfig.fromTimestamp}`);
   }
 
   return parts.join("\n");
@@ -291,8 +292,13 @@ export function generateDatasource(
 ): GeneratedDatasource {
   const parts: string[] = [];
 
-  if (datasource.options.kafka && datasource.options.s3) {
-    throw new Error("Datasource cannot define both `kafka` and `s3` ingestion options.");
+  const ingestionConfigCount = [
+    datasource.options.kafka,
+    datasource.options.s3,
+    datasource.options.gcs,
+  ].filter(Boolean).length;
+  if (ingestionConfigCount > 1) {
+    throw new Error("Datasource can only define one ingestion option: `kafka`, `s3`, or `gcs`.");
   }
 
   // Add description if present
@@ -320,7 +326,13 @@ export function generateDatasource(
   // Add S3 configuration if present
   if (datasource.options.s3) {
     parts.push("");
-    parts.push(generateS3Config(datasource.options.s3));
+    parts.push(generateImportConfig(datasource.options.s3));
+  }
+
+  // Add GCS configuration if present
+  if (datasource.options.gcs) {
+    parts.push("");
+    parts.push(generateImportConfig(datasource.options.gcs));
   }
 
   // Add forward query if present

package/src/generator/pipe.test.ts

@@ -1,7 +1,8 @@
 import { describe, it, expect } from 'vitest';
 import { generatePipe, generateAllPipes } from './pipe.js';
-import { definePipe, defineMaterializedView, node } from '../schema/pipe.js';
+import { definePipe, defineMaterializedView, defineSinkPipe, node } from '../schema/pipe.js';
 import { defineDatasource } from '../schema/datasource.js';
+import { defineKafkaConnection, defineS3Connection } from '../schema/connection.js';
 import { defineToken } from '../schema/token.js';
 import { t } from '../schema/types.js';
 import { p } from '../schema/params.js';
@@ -472,6 +473,60 @@ GROUP BY day, country
     });
   });
 
+  describe('Sink configuration', () => {
+    it('generates Kafka sink directives', () => {
+      const kafka = defineKafkaConnection('events_kafka', {
+        bootstrapServers: 'localhost:9092',
+      });
+
+      const pipe = defineSinkPipe('events_sink', {
+        nodes: [node({ name: 'publish', sql: 'SELECT * FROM events' })],
+        sink: {
+          connection: kafka,
+          topic: 'events_out',
+          schedule: '@on-demand',
+        },
+      });
+
+      const result = generatePipe(pipe);
+      expect(result.content).toContain('TYPE sink');
+      expect(result.content).toContain('EXPORT_CONNECTION_NAME events_kafka');
+      expect(result.content).toContain('EXPORT_KAFKA_TOPIC events_out');
+      expect(result.content).toContain('EXPORT_SCHEDULE @on-demand');
+      expect(result.content).not.toContain('EXPORT_STRATEGY');
+    });
+
+    it('generates S3 sink directives', () => {
+      const s3 = defineS3Connection('exports_s3', {
+        region: 'us-east-1',
+        arn: 'arn:aws:iam::123456789012:role/tinybird-s3-access',
+      });
+
+      const pipe = defineSinkPipe('events_s3_sink', {
+        nodes: [node({ name: 'export', sql: 'SELECT * FROM events' })],
+        sink: {
+          connection: s3,
+          bucketUri: 's3://bucket/events/',
+          fileTemplate: 'events_{date}',
+          format: 'csv',
+          schedule: '@once',
+          compression: 'gzip',
+          strategy: 'replace',
+        },
+      });
+
+      const result = generatePipe(pipe);
+      expect(result.content).toContain('TYPE sink');
+      expect(result.content).toContain('EXPORT_CONNECTION_NAME exports_s3');
+      expect(result.content).toContain('EXPORT_BUCKET_URI s3://bucket/events/');
+      expect(result.content).toContain('EXPORT_FILE_TEMPLATE events_{date}');
+      expect(result.content).toContain('EXPORT_FORMAT csv');
+      expect(result.content).toContain('EXPORT_SCHEDULE @once');
+      expect(result.content).toContain('EXPORT_STRATEGY replace');
+      expect(result.content).toContain('EXPORT_COMPRESSION gzip');
+    });
+  });
+
   describe('Token generation', () => {
     it('generates TOKEN lines with inline config', () => {
       const pipe = definePipe('test_pipe', {

package/src/generator/pipe.ts

@@ -9,9 +9,15 @@ import type {
   EndpointConfig,
   MaterializedConfig,
   CopyConfig,
+  SinkConfig,
   PipeTokenConfig,
 } from "../schema/pipe.js";
-import { getEndpointConfig, getMaterializedConfig, getCopyConfig } from "../schema/pipe.js";
+import {
+  getEndpointConfig,
+  getMaterializedConfig,
+  getCopyConfig,
+  getSinkConfig,
+} from "../schema/pipe.js";
 
 /**
  * Generated pipe content
@@ -114,6 +120,33 @@ function generateCopy(config: CopyConfig): string {
   return parts.join("\n");
 }
 
+/**
+ * Generate the TYPE sink section
+ */
+function generateSink(config: SinkConfig): string {
+  const parts: string[] = ["TYPE sink"];
+
+  parts.push(`EXPORT_CONNECTION_NAME ${config.connection._name}`);
+
+  if ("topic" in config) {
+    parts.push(`EXPORT_KAFKA_TOPIC ${config.topic}`);
+    parts.push(`EXPORT_SCHEDULE ${config.schedule}`);
+  } else {
+    parts.push(`EXPORT_BUCKET_URI ${config.bucketUri}`);
+    parts.push(`EXPORT_FILE_TEMPLATE ${config.fileTemplate}`);
+    parts.push(`EXPORT_SCHEDULE ${config.schedule}`);
+    parts.push(`EXPORT_FORMAT ${config.format}`);
+    if (config.strategy) {
+      parts.push(`EXPORT_STRATEGY ${config.strategy}`);
+    }
+    if (config.compression) {
+      parts.push(`EXPORT_COMPRESSION ${config.compression}`);
+    }
+  }
+
+  return parts.join("\n");
+}
+
 /**
  * Generate TOKEN lines for a pipe
  */
@@ -220,6 +253,13 @@ export function generatePipe(pipe: PipeDefinition): GeneratedPipe {
     parts.push(generateCopy(copyConfig));
   }
 
+  // Add sink configuration if this is a sink pipe
+  const sinkConfig = getSinkConfig(pipe);
+  if (sinkConfig) {
+    parts.push("");
+    parts.push(generateSink(sinkConfig));
+  }
+
   // Add tokens if present
   const tokenLines = generateTokens(pipe.options.tokens);
   if (tokenLines.length > 0) {

package/src/index.ts

@@ -113,6 +113,7 @@ export type {
   ExtractSchema,
   KafkaConfig,
   S3Config,
+  GCSConfig,
 } from "./schema/datasource.js";
 
 // ============ Connection ============
@@ -120,9 +121,11 @@ export {
   defineKafkaConnection,
   createKafkaConnection,
   defineS3Connection,
+  defineGCSConnection,
   isConnectionDefinition,
   isKafkaConnectionDefinition,
   isS3ConnectionDefinition,
+  isGCSConnectionDefinition,
   getConnectionType,
 } from "./schema/connection.js";
 export type {
@@ -133,6 +136,8 @@ export type {
   KafkaSaslMechanism,
   S3ConnectionDefinition,
   S3ConnectionOptions,
+  GCSConnectionDefinition,
+  GCSConnectionOptions,
 } from "./schema/connection.js";
 
 // ============ Token ============
@@ -149,14 +154,17 @@ export {
   defineEndpoint,
   defineMaterializedView,
   defineCopyPipe,
+  defineSinkPipe,
   node,
   isPipeDefinition,
   isNodeDefinition,
   getEndpointConfig,
   getMaterializedConfig,
   getCopyConfig,
+  getSinkConfig,
   isMaterializedView,
   isCopyPipe,
+  isSinkPipe,
   getNodeNames,
   getNode,
   sql,
@@ -166,7 +174,13 @@ export type {
   PipeOptions,
   EndpointOptions,
   CopyPipeOptions,
+  SinkPipeOptions,
   CopyConfig,
+  SinkConfig,
+  SinkStrategy,
+  SinkCompression,
+  KafkaSinkConfig,
+  S3SinkConfig,
   NodeDefinition,
   NodeOptions,
   ParamsDefinition,