@tinybirdco/sdk 0.0.48 → 0.0.50

package/src/schema/connection.test.ts

@@ -2,9 +2,11 @@ import { describe, it, expect } from "vitest";
 import {
   defineKafkaConnection,
   defineS3Connection,
+  defineGCSConnection,
   isConnectionDefinition,
   isKafkaConnectionDefinition,
   isS3ConnectionDefinition,
+  isGCSConnectionDefinition,
   getConnectionType,
 } from "./connection.js";
 
@@ -153,6 +155,29 @@ describe("Connection Schema", () => {
     });
   });
 
+  describe("defineGCSConnection", () => {
+    it("creates a GCS connection with required fields", () => {
+      const conn = defineGCSConnection("my_gcs", {
+        serviceAccountCredentialsJson: '{{ tb_secret("GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON") }}',
+      });
+
+      expect(conn._name).toBe("my_gcs");
+      expect(conn._type).toBe("connection");
+      expect(conn._connectionType).toBe("gcs");
+      expect(conn.options.serviceAccountCredentialsJson).toBe(
+        '{{ tb_secret("GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON") }}'
+      );
+    });
+
+    it("throws when credentials json is empty", () => {
+      expect(() =>
+        defineGCSConnection("my_gcs", {
+          serviceAccountCredentialsJson: "   ",
+        })
+      ).toThrow("GCS connection `serviceAccountCredentialsJson` is required.");
+    });
+  });
+
   describe("isConnectionDefinition", () => {
     it("returns true for valid connection", () => {
       const conn = defineKafkaConnection("my_kafka", {
@@ -203,6 +228,21 @@ describe("Connection Schema", () => {
     });
   });
 
+  describe("isGCSConnectionDefinition", () => {
+    it("returns true for GCS connection", () => {
+      const conn = defineGCSConnection("my_gcs", {
+        serviceAccountCredentialsJson: '{{ tb_secret("GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON") }}',
+      });
+
+      expect(isGCSConnectionDefinition(conn)).toBe(true);
+    });
+
+    it("returns false for non-GCS objects", () => {
+      expect(isGCSConnectionDefinition({})).toBe(false);
+      expect(isGCSConnectionDefinition(null)).toBe(false);
+    });
+  });
+
   describe("getConnectionType", () => {
     it("returns the connection type", () => {
       const conn = defineKafkaConnection("my_kafka", {
@@ -220,5 +260,13 @@ describe("Connection Schema", () => {
 
       expect(getConnectionType(conn)).toBe("s3");
     });
+
+    it("returns the gcs connection type", () => {
+      const conn = defineGCSConnection("my_gcs", {
+        serviceAccountCredentialsJson: '{{ tb_secret("GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON") }}',
+      });
+
+      expect(getConnectionType(conn)).toBe("gcs");
+    });
   });
 });

package/src/schema/connection.ts

@@ -81,10 +81,36 @@ export interface S3ConnectionDefinition {
   readonly options: S3ConnectionOptions;
 }
 
+/**
+ * Options for defining a GCS connection
+ */
+export interface GCSConnectionOptions {
+  /** Service account credentials JSON */
+  serviceAccountCredentialsJson: string;
+}
+
+/**
+ * GCS-specific connection definition
+ */
+export interface GCSConnectionDefinition {
+  readonly [CONNECTION_BRAND]: true;
+  /** Connection name */
+  readonly _name: string;
+  /** Type marker for inference */
+  readonly _type: "connection";
+  /** Connection type */
+  readonly _connectionType: "gcs";
+  /** GCS options */
+  readonly options: GCSConnectionOptions;
+}
+
 /**
  * A connection definition - union of all connection types
  */
-export type ConnectionDefinition = KafkaConnectionDefinition | S3ConnectionDefinition;
+export type ConnectionDefinition =
+  | KafkaConnectionDefinition
+  | S3ConnectionDefinition
+  | GCSConnectionDefinition;
 
 function validateConnectionName(name: string): void {
   if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) {
@@ -166,6 +192,32 @@ export function defineS3Connection(
   };
 }
 
+/**
+ * Define a GCS connection
+ *
+ * @param name - The connection name (must be valid identifier)
+ * @param options - GCS connection configuration
+ * @returns A connection definition that can be used in a project
+ */
+export function defineGCSConnection(
+  name: string,
+  options: GCSConnectionOptions
+): GCSConnectionDefinition {
+  validateConnectionName(name);
+
+  if (!options.serviceAccountCredentialsJson.trim()) {
+    throw new Error("GCS connection `serviceAccountCredentialsJson` is required.");
+  }
+
+  return {
+    [CONNECTION_BRAND]: true,
+    _name: name,
+    _type: "connection",
+    _connectionType: "gcs",
+    options,
+  };
+}
+
 /**
  * Check if a value is a connection definition
  */
@@ -192,6 +244,13 @@ export function isS3ConnectionDefinition(value: unknown): value is S3ConnectionD
   return isConnectionDefinition(value) && value._connectionType === "s3";
 }
 
+/**
+ * Check if a value is a GCS connection definition
+ */
+export function isGCSConnectionDefinition(value: unknown): value is GCSConnectionDefinition {
+  return isConnectionDefinition(value) && value._connectionType === "gcs";
+}
+
 /**
  * Get the connection type from a connection definition
  */

package/src/schema/datasource.test.ts

@@ -9,7 +9,7 @@ import {
 } from "./datasource.js";
 import { t } from "./types.js";
 import { engine } from "./engines.js";
-import { defineKafkaConnection, defineS3Connection } from "./connection.js";
+import { defineKafkaConnection, defineS3Connection, defineGCSConnection } from "./connection.js";
 
 describe("Datasource Schema", () => {
   describe("defineDatasource", () => {
@@ -24,6 +24,7 @@ describe("Datasource Schema", () => {
       expect(ds._name).toBe("events");
       expect(ds._type).toBe("datasource");
       expect(ds.options.schema).toBeDefined();
+      expect(ds.options.engine).toBeUndefined();
     });
 
     it("creates a datasource with description", () => {
@@ -87,7 +88,7 @@ describe("Datasource Schema", () => {
       expect(ds2._name).toBe("events_v2");
     });
 
-    it("throws when both kafka and s3 ingestion are configured", () => {
+    it("throws when multiple ingestion configs are configured", () => {
       const kafkaConn = defineKafkaConnection("my_kafka", {
         bootstrapServers: "kafka.example.com:9092",
       });
@@ -95,6 +96,9 @@ describe("Datasource Schema", () => {
         region: "us-east-1",
         arn: "arn:aws:iam::123456789012:role/tinybird-s3-access",
       });
+      const gcsConn = defineGCSConnection("my_gcs", {
+        serviceAccountCredentialsJson: '{{ tb_secret("GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON") }}',
+      });
 
       expect(() =>
         defineDatasource("events", {
@@ -108,7 +112,39 @@ describe("Datasource Schema", () => {
             bucketUri: "s3://my-bucket/events/*.csv",
           },
         })
-      ).toThrow("Datasource cannot define both `kafka` and `s3` ingestion options.");
+      ).toThrow("Datasource can only define one ingestion option: `kafka`, `s3`, or `gcs`.");
+
+      expect(() =>
+        defineDatasource("events_gcs", {
+          schema: { id: t.string() },
+          kafka: {
+            connection: kafkaConn,
+            topic: "events",
+          },
+          gcs: {
+            connection: gcsConn,
+            bucketUri: "gs://my-bucket/events/*.csv",
+          },
+        })
+      ).toThrow("Datasource can only define one ingestion option: `kafka`, `s3`, or `gcs`.");
+    });
+
+    it("accepts gcs ingestion configuration", () => {
+      const gcsConn = defineGCSConnection("my_gcs", {
+        serviceAccountCredentialsJson: '{{ tb_secret("GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON") }}',
+      });
+
+      const ds = defineDatasource("events_gcs", {
+        schema: { id: t.string() },
+        gcs: {
+          connection: gcsConn,
+          bucketUri: "gs://my-bucket/events/*.csv",
+          schedule: "@auto",
+        },
+      });
+
+      expect(ds.options.gcs?.connection._name).toBe("my_gcs");
+      expect(ds.options.gcs?.bucketUri).toBe("gs://my-bucket/events/*.csv");
     });
   });
 

package/src/schema/datasource.ts

@@ -5,7 +5,11 @@
 
 import { getModifiers, isTypeValidator, type AnyTypeValidator } from "./types.js";
 import type { EngineConfig } from "./engines.js";
-import type { KafkaConnectionDefinition, S3ConnectionDefinition } from "./connection.js";
+import type {
+  KafkaConnectionDefinition,
+  S3ConnectionDefinition,
+  GCSConnectionDefinition,
+} from "./connection.js";
 import type { TokenDefinition, DatasourceTokenScope } from "./token.js";
 
 // Symbol for brand typing - use Symbol.for() for global registry
@@ -84,6 +88,20 @@ export interface S3Config {
   fromTimestamp?: string;
 }
 
+/**
+ * GCS import configuration for a datasource
+ */
+export interface GCSConfig {
+  /** GCS connection to use */
+  connection: GCSConnectionDefinition;
+  /** GCS bucket URI, for example: gs://my-bucket/path/*.csv */
+  bucketUri: string;
+  /** Import schedule, for example: @auto or @once */
+  schedule?: string;
+  /** Incremental import lower bound timestamp expression */
+  fromTimestamp?: string;
+}
+
 /**
  * Options for defining a datasource
  */
@@ -113,6 +131,8 @@ export interface DatasourceOptions<TSchema extends SchemaDefinition> {
   kafka?: KafkaConfig;
   /** S3 ingestion configuration */
   s3?: S3Config;
+  /** GCS ingestion configuration */
+  gcs?: GCSConfig;
 }
 
 /**
@@ -170,8 +190,9 @@ export function defineDatasource<TSchema extends SchemaDefinition>(
     );
   }
 
-  if (options.kafka && options.s3) {
-    throw new Error("Datasource cannot define both `kafka` and `s3` ingestion options.");
+  const ingestionConfigCount = [options.kafka, options.s3, options.gcs].filter(Boolean).length;
+  if (ingestionConfigCount > 1) {
+    throw new Error("Datasource can only define one ingestion option: `kafka`, `s3`, or `gcs`.");
   }
 
   return {

package/src/schema/pipe.test.ts

@@ -1,17 +1,21 @@
 import { describe, it, expect } from "vitest";
 import {
   definePipe,
+  defineSinkPipe,
   defineMaterializedView,
   node,
   isPipeDefinition,
   getEndpointConfig,
   getMaterializedConfig,
+  getSinkConfig,
   isMaterializedView,
+  isSinkPipe,
   getNodeNames,
   getNode,
   sql,
 } from "./pipe.js";
 import { defineDatasource } from "./datasource.js";
+import { defineKafkaConnection, defineS3Connection } from "./connection.js";
 import { t } from "./types.js";
 import { p } from "./params.js";
 import { engine } from "./engines.js";
@@ -154,6 +158,91 @@ describe("Pipe Schema", () => {
     });
   });
 
+  describe("Sink pipes", () => {
+    it("creates a Kafka sink pipe", () => {
+      const kafka = defineKafkaConnection("events_kafka", {
+        bootstrapServers: "localhost:9092",
+      });
+
+      const pipe = defineSinkPipe("events_sink", {
+        nodes: [node({ name: "publish", sql: "SELECT * FROM events" })],
+        sink: {
+          connection: kafka,
+          topic: "events_out",
+          schedule: "@on-demand",
+        },
+      });
+
+      const sink = getSinkConfig(pipe);
+      expect(sink).toBeTruthy();
+      expect(sink && "topic" in sink ? sink.topic : undefined).toBe("events_out");
+      expect(isSinkPipe(pipe)).toBe(true);
+    });
+
+    it("creates an S3 sink pipe", () => {
+      const s3 = defineS3Connection("exports_s3", {
+        region: "us-east-1",
+        arn: "arn:aws:iam::123456789012:role/tinybird-s3-access",
+      });
+
+      const pipe = defineSinkPipe("exports_sink", {
+        nodes: [node({ name: "export", sql: "SELECT * FROM events" })],
+        sink: {
+          connection: s3,
+          bucketUri: "s3://exports/events/",
+          fileTemplate: "events_{date}",
+          schedule: "@once",
+          format: "csv",
+          strategy: "create_new",
+          compression: "gzip",
+        },
+      });
+
+      const sink = getSinkConfig(pipe);
+      expect(sink).toBeTruthy();
+      expect(sink && "bucketUri" in sink ? sink.bucketUri : undefined).toBe("s3://exports/events/");
+      expect(isSinkPipe(pipe)).toBe(true);
+    });
+
+    it("throws when Kafka sink connection type is invalid", () => {
+      const s3 = defineS3Connection("exports_s3", {
+        region: "us-east-1",
+        arn: "arn:aws:iam::123456789012:role/tinybird-s3-access",
+      });
+
+      expect(() =>
+        defineSinkPipe("bad_sink", {
+          nodes: [node({ name: "export", sql: "SELECT * FROM events" })],
+          sink: {
+            // Runtime validation rejects mismatched connection/type
+            connection: s3 as unknown as ReturnType<typeof defineKafkaConnection>,
+            topic: "events_out",
+            schedule: "@on-demand",
+          },
+        })
+      ).toThrow("requires a Kafka connection");
+    });
+
+    it("throws when sink configuration is passed to definePipe", () => {
+      const kafka = defineKafkaConnection("events_kafka", {
+        bootstrapServers: "localhost:9092",
+      });
+
+      expect(() =>
+        definePipe(
+          "bad_via_define_pipe",
+          {
+            nodes: [node({ name: "export", sql: "SELECT * FROM events" })],
+            sink: {
+              connection: kafka,
+              topic: "events_out",
+            },
+          } as unknown as Parameters<typeof definePipe>[1]
+        )
+      ).toThrow("must be created with defineSinkPipe");
+    });
+  });
+
   describe("getEndpointConfig", () => {
     it("returns null when endpoint is false", () => {
       const pipe = definePipe("my_pipe", {

package/src/schema/pipe.ts

@@ -9,6 +9,8 @@ import type { DatasourceDefinition, SchemaDefinition, ColumnDefinition } from ".
 import { getColumnType } from "./datasource.js";
 import { getTinybirdType } from "./types.js";
 import type { TokenDefinition, PipeTokenScope } from "./token.js";
+import type { KafkaConnectionDefinition, S3ConnectionDefinition } from "./connection.js";
+import { isKafkaConnectionDefinition, isS3ConnectionDefinition } from "./connection.js";
 
 /** Symbol for brand typing pipes - use Symbol.for() for global registry */
 export const PIPE_BRAND = Symbol.for("tinybird.pipe");
@@ -155,6 +157,55 @@ export interface CopyConfig<
   copy_schedule?: string;
 }
 
+/**
+ * Sink export strategy.
+ * - 'create_new': write new files on each run
+ * - 'replace': replace destination data on each run
+ */
+export type SinkStrategy = "create_new" | "replace";
+
+/**
+ * S3 sink compression codec.
+ */
+export type SinkCompression = "none" | "gzip" | "snappy";
+
+/**
+ * Kafka sink configuration
+ */
+export interface KafkaSinkConfig {
+  /** Kafka connection used to publish records */
+  connection: KafkaConnectionDefinition;
+  /** Destination Kafka topic */
+  topic: string;
+  /** Sink schedule (for example: @on-demand, @once, cron expression) */
+  schedule: string;
+}
+
+/**
+ * S3 sink configuration
+ */
+export interface S3SinkConfig {
+  /** S3 connection used to write exported files */
+  connection: S3ConnectionDefinition;
+  /** Destination bucket URI (for example: s3://bucket/prefix/) */
+  bucketUri: string;
+  /** Output filename template (supports Tinybird placeholders) */
+  fileTemplate: string;
+  /** Output format (for example: csv, ndjson) */
+  format: string;
+  /** Sink schedule (for example: @on-demand, @once, cron expression) */
+  schedule: string;
+  /** Export strategy */
+  strategy?: SinkStrategy;
+  /** Compression codec */
+  compression?: SinkCompression;
+}
+
+/**
+ * Sink pipe configuration (Kafka or S3 only)
+ */
+export type SinkConfig = KafkaSinkConfig | S3SinkConfig;
+
 /**
  * Inline token configuration for pipe access
  */
@@ -194,9 +245,9 @@ export interface PipeOptions<
   nodes: readonly NodeDefinition[];
   /** Output schema (optional for reusable pipes, required for endpoints) */
   output?: TOutput;
-  /** Whether this pipe is an API endpoint (shorthand for { enabled: true }). Mutually exclusive with materialized and copy. */
+  /** Whether this pipe is an API endpoint (shorthand for { enabled: true }). Mutually exclusive with materialized, copy, and sink. */
   endpoint?: boolean | EndpointConfig;
-  /** Materialized view configuration. Mutually exclusive with endpoint and copy. */
+  /** Materialized view configuration. Mutually exclusive with endpoint, copy, and sink. */
   materialized?: MaterializedConfig;
   /** Copy pipe configuration. Mutually exclusive with endpoint and materialized. */
   copy?: CopyConfig;
@@ -204,6 +255,33 @@ export interface PipeOptions<
   tokens?: readonly PipeTokenConfig[];
 }
 
+/**
+ * Options for defining a sink pipe
+ */
+export interface SinkPipeOptions<TParams extends ParamsDefinition> {
+  /** Human-readable description of the sink pipe */
+  description?: string;
+  /** Parameter definitions for query inputs */
+  params?: TParams;
+  /** Nodes in the transformation pipeline */
+  nodes: readonly NodeDefinition[];
+  /** Sink export configuration */
+  sink: SinkConfig;
+  /** Sink pipes are not endpoints */
+  endpoint?: never;
+  /** Sink pipes are not materialized views */
+  materialized?: never;
+  /** Sink pipes are not copy pipes */
+  copy?: never;
+  /** Access tokens for this sink pipe */
+  tokens?: readonly PipeTokenConfig[];
+}
+
+type PipeRuntimeOptions<
+  TParams extends ParamsDefinition,
+  TOutput extends OutputDefinition
+> = (PipeOptions<TParams, TOutput> & { sink?: never }) | SinkPipeOptions<TParams>;
+
 /**
  * Options for defining an endpoint (API-exposed pipe)
  */
@@ -277,7 +355,7 @@ export interface PipeDefinition<
   /** Output schema (optional for reusable pipes) */
   readonly _output?: TOutput;
   /** Full options */
-  readonly options: PipeOptions<TParams, TOutput>;
+  readonly options: PipeRuntimeOptions<TParams, TOutput>;
 }
 
 /**
@@ -428,6 +506,52 @@ function validateMaterializedSchema(
   }
 }
 
+function validateSinkConfig(pipeName: string, sink: SinkConfig): void {
+  if ("topic" in sink) {
+    if (!isKafkaConnectionDefinition(sink.connection)) {
+      throw new Error(
+        `Pipe "${pipeName}" sink with topic requires a Kafka connection.`
+      );
+    }
+    if (typeof sink.topic !== "string" || !sink.topic.trim()) {
+      throw new Error(`Pipe "${pipeName}" sink topic cannot be empty.`);
+    }
+    if (typeof sink.schedule !== "string" || !sink.schedule.trim()) {
+      throw new Error(`Pipe "${pipeName}" sink schedule cannot be empty.`);
+    }
+    return;
+  }
+
+  if (!isS3ConnectionDefinition(sink.connection)) {
+    throw new Error(
+      `Pipe "${pipeName}" S3 sink requires an S3 connection.`
+    );
+  }
+  if (typeof sink.bucketUri !== "string" || !sink.bucketUri.trim()) {
+    throw new Error(`Pipe "${pipeName}" sink bucketUri cannot be empty.`);
+  }
+  if (typeof sink.fileTemplate !== "string" || !sink.fileTemplate.trim()) {
+    throw new Error(`Pipe "${pipeName}" sink fileTemplate cannot be empty.`);
+  }
+  if (typeof sink.format !== "string" || !sink.format.trim()) {
+    throw new Error(`Pipe "${pipeName}" sink format cannot be empty.`);
+  }
+  if (typeof sink.schedule !== "string" || !sink.schedule.trim()) {
+    throw new Error(`Pipe "${pipeName}" sink schedule cannot be empty.`);
+  }
+  if (sink.strategy && sink.strategy !== "create_new" && sink.strategy !== "replace") {
+    throw new Error(`Pipe "${pipeName}" sink strategy must be "create_new" or "replace".`);
+  }
+  if (
+    sink.compression &&
+    sink.compression !== "none" &&
+    sink.compression !== "gzip" &&
+    sink.compression !== "snappy"
+  ) {
+    throw new Error(`Pipe "${pipeName}" sink compression must be "none", "gzip", or "snappy".`);
+  }
+}
+
 export function definePipe<
   TParams extends ParamsDefinition,
   TOutput extends OutputDefinition
@@ -447,6 +571,12 @@ export function definePipe<
     throw new Error(`Pipe "${name}" must have at least one node.`);
   }
 
+  if ("sink" in (options as unknown as object)) {
+    throw new Error(
+      `Pipe "${name}" sink configuration must be created with defineSinkPipe().`
+    );
+  }
+
   // Validate output is provided for endpoints and materialized views
   if ((options.endpoint || options.materialized) && (!options.output || Object.keys(options.output).length === 0)) {
     throw new Error(
@@ -480,7 +610,47 @@ export function definePipe<
     options: {
       ...options,
       params,
-    },
+    } as PipeRuntimeOptions<TParams, TOutput>,
+  };
+}
+
+/**
+ * Define a Tinybird sink pipe
+ *
+ * Sink pipes export query results to external systems via Kafka or S3.
+ *
+ * @param name - The sink pipe name (must be valid identifier)
+ * @param options - Sink pipe configuration
+ * @returns A pipe definition configured as a sink pipe
+ */
+export function defineSinkPipe<TParams extends ParamsDefinition>(
+  name: string,
+  options: SinkPipeOptions<TParams>
+): PipeDefinition<TParams, Record<string, never>> {
+  if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) {
+    throw new Error(
+      `Invalid pipe name: "${name}". Must start with a letter or underscore and contain only alphanumeric characters and underscores.`
+    );
+  }
+
+  if (!options.nodes || options.nodes.length === 0) {
+    throw new Error(`Pipe "${name}" must have at least one node.`);
+  }
+
+  validateSinkConfig(name, options.sink);
+
+  const params = (options.params ?? {}) as TParams;
+
+  return {
+    [PIPE_BRAND]: true,
+    _name: name,
+    _type: "pipe",
+    _params: params,
+    _output: undefined,
+    options: {
+      ...options,
+      params,
+    } as PipeRuntimeOptions<TParams, Record<string, never>>,
   };
 }
 
@@ -790,6 +960,20 @@ export function isCopyPipe(pipe: PipeDefinition): boolean {
   return pipe.options.copy !== undefined;
 }
 
+/**
+ * Get the sink configuration from a pipe
+ */
+export function getSinkConfig(pipe: PipeDefinition): SinkConfig | null {
+  return "sink" in pipe.options ? (pipe.options.sink ?? null) : null;
+}
+
+/**
+ * Check if a pipe is a sink pipe
+ */
+export function isSinkPipe(pipe: PipeDefinition): boolean {
+  return pipe.options.sink !== undefined;
+}
+
 /**
  * Get all node names from a pipe
  */