@tinybirdco/sdk 0.0.49 → 0.0.50

package/src/generator/connection.test.ts

@@ -1,6 +1,10 @@
 import { describe, it, expect } from "vitest";
 import { generateConnection, generateAllConnections } from "./connection.js";
-import { defineKafkaConnection, defineS3Connection } from "../schema/connection.js";
+import {
+  defineKafkaConnection,
+  defineS3Connection,
+  defineGCSConnection,
+} from "../schema/connection.js";
 
 describe("Connection Generator", () => {
   describe("generateConnection", () => {
@@ -152,6 +156,20 @@ describe("Connection Generator", () => {
       expect(result.content).toContain('S3_ACCESS_KEY {{ tb_secret("S3_ACCESS_KEY") }}');
       expect(result.content).toContain('S3_SECRET {{ tb_secret("S3_SECRET") }}');
     });
+
+    it("generates GCS connection with service account credentials", () => {
+      const conn = defineGCSConnection("my_gcs", {
+        serviceAccountCredentialsJson: '{{ tb_secret("GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON") }}',
+      });
+
+      const result = generateConnection(conn);
+
+      expect(result.name).toBe("my_gcs");
+      expect(result.content).toContain("TYPE gcs");
+      expect(result.content).toContain(
+        'GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON {{ tb_secret("GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON") }}'
+      );
+    });
   });
 
   describe("generateAllConnections", () => {
@@ -163,11 +181,18 @@ describe("Connection Generator", () => {
         region: "us-east-1",
         arn: "arn:aws:iam::123456789012:role/tinybird-s3-access",
       });
+      const conn3 = defineGCSConnection("gcs_landing", {
+        serviceAccountCredentialsJson: '{{ tb_secret("GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON") }}',
+      });
 
-      const results = generateAllConnections({ kafka1: conn1, s3_logs: conn2 });
+      const results = generateAllConnections({
+        kafka1: conn1,
+        s3_logs: conn2,
+        gcs_landing: conn3,
+      });
 
-      expect(results).toHaveLength(2);
-      expect(results.map((r) => r.name).sort()).toEqual(["kafka1", "s3_logs"]);
+      expect(results).toHaveLength(3);
+      expect(results.map((r) => r.name).sort()).toEqual(["gcs_landing", "kafka1", "s3_logs"]);
     });
 
     it("returns empty array for empty connections", () => {

package/src/generator/connection.ts

@@ -3,8 +3,16 @@
  * Converts ConnectionDefinition to native .connection file format
  */
 
-import type { ConnectionDefinition, KafkaConnectionDefinition } from "../schema/connection.js";
-import { isS3ConnectionDefinition, type S3ConnectionDefinition } from "../schema/connection.js";
+import type {
+  ConnectionDefinition,
+  KafkaConnectionDefinition,
+  GCSConnectionDefinition,
+} from "../schema/connection.js";
+import {
+  isS3ConnectionDefinition,
+  isGCSConnectionDefinition,
+  type S3ConnectionDefinition,
+} from "../schema/connection.js";
 
 /**
  * Generated connection content
@@ -78,6 +86,19 @@ function generateS3Connection(connection: S3ConnectionDefinition): string {
   return parts.join("\n");
 }
 
+/**
+ * Generate a GCS connection content
+ */
+function generateGCSConnection(connection: GCSConnectionDefinition): string {
+  const parts: string[] = [];
+  const options = connection.options;
+
+  parts.push("TYPE gcs");
+  parts.push(`GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON ${options.serviceAccountCredentialsJson}`);
+
+  return parts.join("\n");
+}
+
 /**
  * Generate a .connection file content from a ConnectionDefinition
  *
@@ -113,6 +134,8 @@ export function generateConnection(
     content = generateKafkaConnection(connection as KafkaConnectionDefinition);
   } else if (isS3ConnectionDefinition(connection)) {
     content = generateS3Connection(connection);
+  } else if (isGCSConnectionDefinition(connection)) {
+    content = generateGCSConnection(connection);
   } else {
     throw new Error("Unsupported connection type.");
   }

package/src/generator/datasource.test.ts

@@ -1,7 +1,7 @@
 import { describe, it, expect } from 'vitest';
 import { generateDatasource, generateAllDatasources } from './datasource.js';
 import { defineDatasource } from '../schema/datasource.js';
-import { defineKafkaConnection, defineS3Connection } from '../schema/connection.js';
+import { defineKafkaConnection, defineS3Connection, defineGCSConnection } from '../schema/connection.js';
 import { defineToken } from '../schema/token.js';
 import { t } from '../schema/types.js';
 import { engine } from '../schema/engines.js';
@@ -529,6 +529,35 @@ describe('Datasource Generator', () => {
     });
   });
 
+  describe('GCS configuration', () => {
+    it('includes GCS import directives', () => {
+      const gcsConn = defineGCSConnection('my_gcs', {
+        serviceAccountCredentialsJson: '{{ tb_secret("GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON") }}',
+      });
+
+      const ds = defineDatasource('gcs_events', {
+        schema: {
+          timestamp: t.dateTime(),
+          event: t.string(),
+        },
+        engine: engine.mergeTree({ sortingKey: ['timestamp'] }),
+        gcs: {
+          connection: gcsConn,
+          bucketUri: 'gs://my-bucket/events/*.csv',
+          schedule: '@auto',
+          fromTimestamp: '2024-01-01T00:00:00Z',
+        },
+      });
+
+      const result = generateDatasource(ds);
+
+      expect(result.content).toContain('IMPORT_CONNECTION_NAME my_gcs');
+      expect(result.content).toContain('IMPORT_BUCKET_URI gs://my-bucket/events/*.csv');
+      expect(result.content).toContain('IMPORT_SCHEDULE @auto');
+      expect(result.content).toContain('IMPORT_FROM_TIMESTAMP 2024-01-01T00:00:00Z');
+    });
+  });
+
   describe('Token generation', () => {
     it('generates TOKEN lines with inline config', () => {
       const ds = defineDatasource('test_ds', {

package/src/generator/datasource.ts

@@ -9,6 +9,7 @@ import type {
   ColumnDefinition,
   KafkaConfig,
   S3Config,
+  GCSConfig,
   TokenConfig,
 } from "../schema/datasource.js";
 import type { AnyTypeValidator, TypeModifiers } from "../schema/types.js";
@@ -176,18 +177,18 @@ function generateKafkaConfig(kafka: KafkaConfig): string {
 /**
  * Generate S3 import configuration lines
  */
-function generateS3Config(s3: S3Config): string {
+function generateImportConfig(importConfig: S3Config | GCSConfig): string {
   const parts: string[] = [];
 
-  parts.push(`IMPORT_CONNECTION_NAME ${s3.connection._name}`);
-  parts.push(`IMPORT_BUCKET_URI ${s3.bucketUri}`);
+  parts.push(`IMPORT_CONNECTION_NAME ${importConfig.connection._name}`);
+  parts.push(`IMPORT_BUCKET_URI ${importConfig.bucketUri}`);
 
-  if (s3.schedule) {
-    parts.push(`IMPORT_SCHEDULE ${s3.schedule}`);
+  if (importConfig.schedule) {
+    parts.push(`IMPORT_SCHEDULE ${importConfig.schedule}`);
   }
 
-  if (s3.fromTimestamp) {
-    parts.push(`IMPORT_FROM_TIMESTAMP ${s3.fromTimestamp}`);
+  if (importConfig.fromTimestamp) {
+    parts.push(`IMPORT_FROM_TIMESTAMP ${importConfig.fromTimestamp}`);
   }
 
   return parts.join("\n");
@@ -291,8 +292,13 @@ export function generateDatasource(
 ): GeneratedDatasource {
   const parts: string[] = [];
 
-  if (datasource.options.kafka && datasource.options.s3) {
-    throw new Error("Datasource cannot define both `kafka` and `s3` ingestion options.");
+  const ingestionConfigCount = [
+    datasource.options.kafka,
+    datasource.options.s3,
+    datasource.options.gcs,
+  ].filter(Boolean).length;
+  if (ingestionConfigCount > 1) {
+    throw new Error("Datasource can only define one ingestion option: `kafka`, `s3`, or `gcs`.");
   }
 
   // Add description if present
@@ -320,7 +326,13 @@ export function generateDatasource(
   // Add S3 configuration if present
   if (datasource.options.s3) {
     parts.push("");
-    parts.push(generateS3Config(datasource.options.s3));
+    parts.push(generateImportConfig(datasource.options.s3));
+  }
+
+  // Add GCS configuration if present
+  if (datasource.options.gcs) {
+    parts.push("");
+    parts.push(generateImportConfig(datasource.options.gcs));
   }
 
   // Add forward query if present

package/src/index.ts

@@ -113,6 +113,7 @@ export type {
   ExtractSchema,
   KafkaConfig,
   S3Config,
+  GCSConfig,
 } from "./schema/datasource.js";
 
 // ============ Connection ============
@@ -120,9 +121,11 @@ export {
   defineKafkaConnection,
   createKafkaConnection,
   defineS3Connection,
+  defineGCSConnection,
   isConnectionDefinition,
   isKafkaConnectionDefinition,
   isS3ConnectionDefinition,
+  isGCSConnectionDefinition,
   getConnectionType,
 } from "./schema/connection.js";
 export type {
@@ -133,6 +136,8 @@ export type {
   KafkaSaslMechanism,
   S3ConnectionDefinition,
   S3ConnectionOptions,
+  GCSConnectionDefinition,
+  GCSConnectionOptions,
 } from "./schema/connection.js";
 
 // ============ Token ============

package/src/migrate/emit-ts.ts

@@ -4,6 +4,7 @@ import { parseLiteralFromDatafile, toTsLiteral } from "./parser-utils.js";
 import type {
   DatasourceModel,
   DatasourceEngineModel,
+  GCSConnectionModel,
   KafkaConnectionModel,
   ParsedResource,
   PipeModel,
@@ -59,11 +60,13 @@ function hasSecretTemplate(resources: ParsedResource[]): boolean {
         if (resource.secret) values.push(resource.secret);
         if (resource.sslCaPem) values.push(resource.sslCaPem);
         if (resource.schemaRegistryUrl) values.push(resource.schemaRegistryUrl);
-      } else {
+      } else if (resource.connectionType === "s3") {
         values.push(resource.region);
         if (resource.arn) values.push(resource.arn);
         if (resource.accessKey) values.push(resource.accessKey);
         if (resource.secret) values.push(resource.secret);
+      } else {
+        values.push(resource.serviceAccountCredentialsJson);
       }
       continue;
     }
@@ -80,6 +83,11 @@ function hasSecretTemplate(resources: ParsedResource[]): boolean {
         if (resource.s3.schedule) values.push(resource.s3.schedule);
         if (resource.s3.fromTimestamp) values.push(resource.s3.fromTimestamp);
       }
+      if (resource.gcs) {
+        values.push(resource.gcs.bucketUri);
+        if (resource.gcs.schedule) values.push(resource.gcs.schedule);
+        if (resource.gcs.fromTimestamp) values.push(resource.gcs.fromTimestamp);
+      }
       continue;
     }
   }
@@ -337,6 +345,20 @@ function emitDatasource(ds: DatasourceModel): string {
     lines.push("  },");
   }
 
+  if (ds.gcs) {
+    const connectionVar = toCamelCase(ds.gcs.connectionName);
+    lines.push("  gcs: {");
+    lines.push(`    connection: ${connectionVar},`);
+    lines.push(`    bucketUri: ${emitStringOrSecret(ds.gcs.bucketUri)},`);
+    if (ds.gcs.schedule) {
+      lines.push(`    schedule: ${emitStringOrSecret(ds.gcs.schedule)},`);
+    }
+    if (ds.gcs.fromTimestamp) {
+      lines.push(`    fromTimestamp: ${emitStringOrSecret(ds.gcs.fromTimestamp)},`);
+    }
+    lines.push("  },");
+  }
+
   if (ds.forwardQuery) {
     lines.push("  forwardQuery: `");
     lines.push(ds.forwardQuery.replace(/`/g, "\\`").replace(/\${/g, "\\${"));
@@ -364,7 +386,9 @@ function emitDatasource(ds: DatasourceModel): string {
   return lines.join("\n");
 }
 
-function emitConnection(connection: KafkaConnectionModel | S3ConnectionModel): string {
+function emitConnection(
+  connection: KafkaConnectionModel | S3ConnectionModel | GCSConnectionModel
+): string {
   const variableName = toCamelCase(connection.name);
   const lines: string[] = [];
 
@@ -396,19 +420,31 @@ function emitConnection(connection: KafkaConnectionModel | S3ConnectionModel): s
     return lines.join("\n");
   }
 
+  if (connection.connectionType === "s3") {
+    lines.push(
+      `export const ${variableName} = defineS3Connection(${escapeString(connection.name)}, {`
+    );
+    lines.push(`  region: ${emitStringOrSecret(connection.region)},`);
+    if (connection.arn) {
+      lines.push(`  arn: ${emitStringOrSecret(connection.arn)},`);
+    }
+    if (connection.accessKey) {
+      lines.push(`  accessKey: ${emitStringOrSecret(connection.accessKey)},`);
+    }
+    if (connection.secret) {
+      lines.push(`  secret: ${emitStringOrSecret(connection.secret)},`);
+    }
+    lines.push("});");
+    lines.push("");
+    return lines.join("\n");
+  }
+
   lines.push(
-    `export const ${variableName} = defineS3Connection(${escapeString(connection.name)}, {`
+    `export const ${variableName} = defineGCSConnection(${escapeString(connection.name)}, {`
+  );
+  lines.push(
+    `  serviceAccountCredentialsJson: ${emitStringOrSecret(connection.serviceAccountCredentialsJson)},`
   );
-  lines.push(`  region: ${emitStringOrSecret(connection.region)},`);
-  if (connection.arn) {
-    lines.push(`  arn: ${emitStringOrSecret(connection.arn)},`);
-  }
-  if (connection.accessKey) {
-    lines.push(`  accessKey: ${emitStringOrSecret(connection.accessKey)},`);
-  }
-  if (connection.secret) {
-    lines.push(`  secret: ${emitStringOrSecret(connection.secret)},`);
-  }
   lines.push("});");
   lines.push("");
   return lines.join("\n");
@@ -542,7 +578,7 @@ function emitPipe(pipe: PipeModel): string {
 
 export function emitMigrationFileContent(resources: ParsedResource[]): string {
   const connections = resources.filter(
-    (resource): resource is KafkaConnectionModel | S3ConnectionModel =>
+    (resource): resource is KafkaConnectionModel | S3ConnectionModel | GCSConnectionModel =>
       resource.kind === "connection"
   );
   const datasources = resources.filter(
@@ -569,6 +605,9 @@ export function emitMigrationFileContent(resources: ParsedResource[]): string {
   if (connections.some((connection) => connection.connectionType === "s3")) {
     imports.add("defineS3Connection");
   }
+  if (connections.some((connection) => connection.connectionType === "gcs")) {
+    imports.add("defineGCSConnection");
+  }
   if (needsParams) {
     imports.add("p");
   }
@@ -585,6 +624,7 @@ export function emitMigrationFileContent(resources: ParsedResource[]): string {
   const orderedImports = [
     "defineKafkaConnection",
     "defineS3Connection",
+    "defineGCSConnection",
     "defineDatasource",
     "definePipe",
     "defineMaterializedView",

package/src/migrate/parse-connection.ts

@@ -1,4 +1,9 @@
-import type { KafkaConnectionModel, ResourceFile, S3ConnectionModel } from "./types.js";
+import type {
+  GCSConnectionModel,
+  KafkaConnectionModel,
+  ResourceFile,
+  S3ConnectionModel,
+} from "./types.js";
 import {
   MigrationParseError,
   isBlank,
@@ -9,7 +14,7 @@ import {
 
 export function parseConnectionFile(
   resource: ResourceFile
-): KafkaConnectionModel | S3ConnectionModel {
+): KafkaConnectionModel | S3ConnectionModel | GCSConnectionModel {
   const lines = splitLines(resource.content);
   let connectionType: string | undefined;
 
@@ -34,6 +39,7 @@ export function parseConnectionFile(
   let arn: string | undefined;
   let accessKey: string | undefined;
   let accessSecret: string | undefined;
+  let serviceAccountCredentialsJson: string | undefined;
 
   for (const rawLine of lines) {
     const line = rawLine.trim();
@@ -100,6 +106,9 @@ export function parseConnectionFile(
       case "S3_SECRET":
         accessSecret = parseQuotedValue(value);
         break;
+      case "GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON":
+        serviceAccountCredentialsJson = parseQuotedValue(value);
+        break;
       default:
         throw new MigrationParseError(
           resource.filePath,
@@ -120,12 +129,12 @@ export function parseConnectionFile(
   }
 
   if (connectionType === "kafka") {
-    if (region || arn || accessKey || accessSecret) {
+    if (region || arn || accessKey || accessSecret || serviceAccountCredentialsJson) {
       throw new MigrationParseError(
         resource.filePath,
         "connection",
         resource.name,
-        "S3 directives are not valid for kafka connections."
+        "S3/GCS directives are not valid for kafka connections."
       );
     }
 
@@ -161,13 +170,14 @@ export function parseConnectionFile(
       key ||
       secret ||
       schemaRegistryUrl ||
-      sslCaPem
+      sslCaPem ||
+      serviceAccountCredentialsJson
     ) {
       throw new MigrationParseError(
         resource.filePath,
         "connection",
         resource.name,
-        "Kafka directives are not valid for s3 connections."
+        "Kafka/GCS directives are not valid for s3 connections."
       );
     }
 
@@ -210,6 +220,46 @@ export function parseConnectionFile(
     };
   }
 
+  if (connectionType === "gcs") {
+    if (
+      bootstrapServers ||
+      securityProtocol ||
+      saslMechanism ||
+      key ||
+      secret ||
+      schemaRegistryUrl ||
+      sslCaPem ||
+      region ||
+      arn ||
+      accessKey ||
+      accessSecret
+    ) {
+      throw new MigrationParseError(
+        resource.filePath,
+        "connection",
+        resource.name,
+        "Kafka/S3 directives are not valid for gcs connections."
+      );
+    }
+
+    if (!serviceAccountCredentialsJson) {
+      throw new MigrationParseError(
+        resource.filePath,
+        "connection",
+        resource.name,
+        "GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON is required for gcs connections."
+      );
+    }
+
+    return {
+      kind: "connection",
+      name: resource.name,
+      filePath: resource.filePath,
+      connectionType: "gcs",
+      serviceAccountCredentialsJson,
+    };
+  }
+
   throw new MigrationParseError(
     resource.filePath,
     "connection",

package/src/migrate/parse-datasource.ts

@@ -518,7 +518,7 @@ export function parseDatasourceFile(resource: ResourceFile): DatasourceModel {
       resource.filePath,
       "datasource",
       resource.name,
-      "IMPORT_CONNECTION_NAME and IMPORT_BUCKET_URI are required when S3 import directives are used."
+      "IMPORT_CONNECTION_NAME and IMPORT_BUCKET_URI are required when import directives are used."
     );
   }
 
@@ -527,7 +527,7 @@ export function parseDatasourceFile(resource: ResourceFile): DatasourceModel {
       resource.filePath,
       "datasource",
       resource.name,
-      "Datasource cannot mix Kafka directives with S3 import directives."
+      "Datasource cannot mix Kafka directives with import directives."
     );
   }
 

package/src/migrate/types.ts

@@ -52,6 +52,13 @@ export interface DatasourceS3Model {
   fromTimestamp?: string;
 }
 
+export interface DatasourceGCSModel {
+  connectionName: string;
+  bucketUri: string;
+  schedule?: string;
+  fromTimestamp?: string;
+}
+
 export interface DatasourceTokenModel {
   name: string;
   scope: "READ" | "APPEND";
@@ -66,6 +73,7 @@ export interface DatasourceModel {
   engine?: DatasourceEngineModel;
   kafka?: DatasourceKafkaModel;
   s3?: DatasourceS3Model;
+  gcs?: DatasourceGCSModel;
   forwardQuery?: string;
   tokens: DatasourceTokenModel[];
   sharedWith: string[];
@@ -156,11 +164,20 @@ export interface S3ConnectionModel {
   secret?: string;
 }
 
+export interface GCSConnectionModel {
+  kind: "connection";
+  name: string;
+  filePath: string;
+  connectionType: "gcs";
+  serviceAccountCredentialsJson: string;
+}
+
 export type ParsedResource =
   | DatasourceModel
   | PipeModel
   | KafkaConnectionModel
-  | S3ConnectionModel;
+  | S3ConnectionModel
+  | GCSConnectionModel;
 
 export interface MigrationResult {
   success: boolean;

package/src/schema/connection.test.ts

@@ -2,9 +2,11 @@ import { describe, it, expect } from "vitest";
 import {
   defineKafkaConnection,
   defineS3Connection,
+  defineGCSConnection,
   isConnectionDefinition,
   isKafkaConnectionDefinition,
   isS3ConnectionDefinition,
+  isGCSConnectionDefinition,
   getConnectionType,
 } from "./connection.js";
 
@@ -153,6 +155,29 @@ describe("Connection Schema", () => {
     });
   });
 
+  describe("defineGCSConnection", () => {
+    it("creates a GCS connection with required fields", () => {
+      const conn = defineGCSConnection("my_gcs", {
+        serviceAccountCredentialsJson: '{{ tb_secret("GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON") }}',
+      });
+
+      expect(conn._name).toBe("my_gcs");
+      expect(conn._type).toBe("connection");
+      expect(conn._connectionType).toBe("gcs");
+      expect(conn.options.serviceAccountCredentialsJson).toBe(
+        '{{ tb_secret("GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON") }}'
+      );
+    });
+
+    it("throws when credentials json is empty", () => {
+      expect(() =>
+        defineGCSConnection("my_gcs", {
+          serviceAccountCredentialsJson: "   ",
+        })
+      ).toThrow("GCS connection `serviceAccountCredentialsJson` is required.");
+    });
+  });
+
   describe("isConnectionDefinition", () => {
     it("returns true for valid connection", () => {
       const conn = defineKafkaConnection("my_kafka", {
@@ -203,6 +228,21 @@ describe("Connection Schema", () => {
     });
   });
 
+  describe("isGCSConnectionDefinition", () => {
+    it("returns true for GCS connection", () => {
+      const conn = defineGCSConnection("my_gcs", {
+        serviceAccountCredentialsJson: '{{ tb_secret("GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON") }}',
+      });
+
+      expect(isGCSConnectionDefinition(conn)).toBe(true);
+    });
+
+    it("returns false for non-GCS objects", () => {
+      expect(isGCSConnectionDefinition({})).toBe(false);
+      expect(isGCSConnectionDefinition(null)).toBe(false);
+    });
+  });
+
   describe("getConnectionType", () => {
     it("returns the connection type", () => {
       const conn = defineKafkaConnection("my_kafka", {
@@ -220,5 +260,13 @@ describe("Connection Schema", () => {
 
       expect(getConnectionType(conn)).toBe("s3");
     });
+
+    it("returns the gcs connection type", () => {
+      const conn = defineGCSConnection("my_gcs", {
+        serviceAccountCredentialsJson: '{{ tb_secret("GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON") }}',
+      });
+
+      expect(getConnectionType(conn)).toBe("gcs");
+    });
   });
 });