@tinybirdco/sdk 0.0.49 → 0.0.51

package/src/schema/connection.ts

@@ -81,10 +81,36 @@ export interface S3ConnectionDefinition {
   readonly options: S3ConnectionOptions;
 }
 
+/**
+ * Options for defining a GCS connection
+ */
+export interface GCSConnectionOptions {
+  /** Service account credentials JSON */
+  serviceAccountCredentialsJson: string;
+}
+
+/**
+ * GCS-specific connection definition
+ */
+export interface GCSConnectionDefinition {
+  readonly [CONNECTION_BRAND]: true;
+  /** Connection name */
+  readonly _name: string;
+  /** Type marker for inference */
+  readonly _type: "connection";
+  /** Connection type */
+  readonly _connectionType: "gcs";
+  /** GCS options */
+  readonly options: GCSConnectionOptions;
+}
+
 /**
  * A connection definition - union of all connection types
  */
-export type ConnectionDefinition = KafkaConnectionDefinition | S3ConnectionDefinition;
+export type ConnectionDefinition =
+  | KafkaConnectionDefinition
+  | S3ConnectionDefinition
+  | GCSConnectionDefinition;
 
 function validateConnectionName(name: string): void {
   if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) {
@@ -166,6 +192,32 @@ export function defineS3Connection(
   };
 }
 
+/**
+ * Define a GCS connection
+ *
+ * @param name - The connection name (must be valid identifier)
+ * @param options - GCS connection configuration
+ * @returns A connection definition that can be used in a project
+ */
+export function defineGCSConnection(
+  name: string,
+  options: GCSConnectionOptions
+): GCSConnectionDefinition {
+  validateConnectionName(name);
+
+  if (!options.serviceAccountCredentialsJson.trim()) {
+    throw new Error("GCS connection `serviceAccountCredentialsJson` is required.");
+  }
+
+  return {
+    [CONNECTION_BRAND]: true,
+    _name: name,
+    _type: "connection",
+    _connectionType: "gcs",
+    options,
+  };
+}
+
 /**
  * Check if a value is a connection definition
  */
@@ -192,6 +244,13 @@ export function isS3ConnectionDefinition(value: unknown): value is S3ConnectionD
   return isConnectionDefinition(value) && value._connectionType === "s3";
 }
 
+/**
+ * Check if a value is a GCS connection definition
+ */
+export function isGCSConnectionDefinition(value: unknown): value is GCSConnectionDefinition {
+  return isConnectionDefinition(value) && value._connectionType === "gcs";
+}
+
 /**
  * Get the connection type from a connection definition
  */

package/src/schema/datasource.test.ts

@@ -9,7 +9,7 @@ import {
 } from "./datasource.js";
 import { t } from "./types.js";
 import { engine } from "./engines.js";
-import { defineKafkaConnection, defineS3Connection } from "./connection.js";
+import { defineKafkaConnection, defineS3Connection, defineGCSConnection } from "./connection.js";
 
 describe("Datasource Schema", () => {
   describe("defineDatasource", () => {
@@ -88,7 +88,7 @@ describe("Datasource Schema", () => {
       expect(ds2._name).toBe("events_v2");
     });
 
-    it("throws when both kafka and s3 ingestion are configured", () => {
+    it("throws when multiple ingestion configs are configured", () => {
       const kafkaConn = defineKafkaConnection("my_kafka", {
         bootstrapServers: "kafka.example.com:9092",
       });
@@ -96,6 +96,9 @@ describe("Datasource Schema", () => {
         region: "us-east-1",
         arn: "arn:aws:iam::123456789012:role/tinybird-s3-access",
       });
+      const gcsConn = defineGCSConnection("my_gcs", {
+        serviceAccountCredentialsJson: '{{ tb_secret("GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON") }}',
+      });
 
       expect(() =>
         defineDatasource("events", {
@@ -109,7 +112,92 @@ describe("Datasource Schema", () => {
             bucketUri: "s3://my-bucket/events/*.csv",
           },
         })
-      ).toThrow("Datasource cannot define both `kafka` and `s3` ingestion options.");
+      ).toThrow("Datasource can only define one ingestion option: `kafka`, `s3`, or `gcs`.");
+
+      expect(() =>
+        defineDatasource("events_gcs", {
+          schema: { id: t.string() },
+          kafka: {
+            connection: kafkaConn,
+            topic: "events",
+          },
+          gcs: {
+            connection: gcsConn,
+            bucketUri: "gs://my-bucket/events/*.csv",
+          },
+        })
+      ).toThrow("Datasource can only define one ingestion option: `kafka`, `s3`, or `gcs`.");
+    });
+
+    it("accepts gcs ingestion configuration", () => {
+      const gcsConn = defineGCSConnection("my_gcs", {
+        serviceAccountCredentialsJson: '{{ tb_secret("GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON") }}',
+      });
+
+      const ds = defineDatasource("events_gcs", {
+        schema: { id: t.string() },
+        gcs: {
+          connection: gcsConn,
+          bucketUri: "gs://my-bucket/events/*.csv",
+          schedule: "@auto",
+        },
+      });
+
+      expect(ds.options.gcs?.connection._name).toBe("my_gcs");
+      expect(ds.options.gcs?.bucketUri).toBe("gs://my-bucket/events/*.csv");
+    });
+
+    it("accepts datasource indexes", () => {
+      const ds = defineDatasource("events", {
+        schema: { id: t.string() },
+        indexes: [
+          {
+            name: "id_set",
+            expr: "id",
+            type: "set(100)",
+            granularity: 1,
+          },
+        ],
+      });
+
+      expect(ds.options.indexes).toEqual([
+        {
+          name: "id_set",
+          expr: "id",
+          type: "set(100)",
+          granularity: 1,
+        },
+      ]);
+    });
+
+    it("validates datasource index fields", () => {
+      expect(() =>
+        defineDatasource("events", {
+          schema: { id: t.string() },
+          indexes: [{ name: "invalid name", expr: "id", type: "set(100)", granularity: 1 }],
+        })
+      ).toThrow("Invalid datasource index name");
+
+      expect(() =>
+        defineDatasource("events", {
+          schema: { id: t.string() },
+          indexes: [{ name: "id_set", expr: "", type: "set(100)", granularity: 1 }],
+        })
+      ).toThrow('Invalid datasource index "id_set": expr is required.');
+
+      expect(() =>
+        defineDatasource("events", {
+          schema: { id: t.string() },
+          indexes: [{ name: "id_set", expr: "id", type: "", granularity: 1 }],
+        })
+      ).toThrow('Invalid datasource index "id_set": type is required.');
+
+      expect(() =>
+        defineDatasource("events", {
+          schema: { id: t.string() },
+          indexes: [{ name: "id_set", expr: "id", type: "set(100)", granularity: 0 }],
+        })
+      ).toThrow('Invalid datasource index "id_set": granularity must be a positive integer.');
     });
   });
 

package/src/schema/datasource.ts

@@ -5,7 +5,11 @@
 
 import { getModifiers, isTypeValidator, type AnyTypeValidator } from "./types.js";
 import type { EngineConfig } from "./engines.js";
-import type { KafkaConnectionDefinition, S3ConnectionDefinition } from "./connection.js";
+import type {
+  KafkaConnectionDefinition,
+  S3ConnectionDefinition,
+  GCSConnectionDefinition,
+} from "./connection.js";
 import type { TokenDefinition, DatasourceTokenScope } from "./token.js";
 
 // Symbol for brand typing - use Symbol.for() for global registry
@@ -84,6 +88,35 @@ export interface S3Config {
   fromTimestamp?: string;
 }
 
+/**
+ * GCS import configuration for a datasource
+ */
+export interface GCSConfig {
+  /** GCS connection to use */
+  connection: GCSConnectionDefinition;
+  /** GCS bucket URI, for example: gs://my-bucket/path/*.csv */
+  bucketUri: string;
+  /** Import schedule, for example: @auto or @once */
+  schedule?: string;
+  /** Incremental import lower bound timestamp expression */
+  fromTimestamp?: string;
+}
+
+/**
+ * Datasource index configuration.
+ * Emits as: `<name> <expr> TYPE <type> GRANULARITY <n>`
+ */
+export interface DatasourceIndex {
+  /** Index name */
+  name: string;
+  /** Index expression */
+  expr: string;
+  /** Index type and parameters (for example: `set(100)`) */
+  type: string;
+  /** Index granularity */
+  granularity: number;
+}
+
 /**
  * Options for defining a datasource
  */
@@ -109,10 +142,14 @@ export interface DatasourceOptions<TSchema extends SchemaDefinition> {
    * This should be the SELECT clause only (no FROM/WHERE).
    */
   forwardQuery?: string;
+  /** Secondary indexes for MergeTree-family engines */
+  indexes?: readonly DatasourceIndex[];
   /** Kafka ingestion configuration */
   kafka?: KafkaConfig;
   /** S3 ingestion configuration */
   s3?: S3Config;
+  /** GCS ingestion configuration */
+  gcs?: GCSConfig;
 }
 
 /**
@@ -170,8 +207,30 @@ export function defineDatasource<TSchema extends SchemaDefinition>(
     );
   }
 
-  if (options.kafka && options.s3) {
-    throw new Error("Datasource cannot define both `kafka` and `s3` ingestion options.");
+  const ingestionConfigCount = [options.kafka, options.s3, options.gcs].filter(Boolean).length;
+  if (ingestionConfigCount > 1) {
+    throw new Error("Datasource can only define one ingestion option: `kafka`, `s3`, or `gcs`.");
+  }
+
+  if (options.indexes) {
+    for (const index of options.indexes) {
+      if (!index.name || /\s/.test(index.name)) {
+        throw new Error(
+          `Invalid datasource index name: "${index.name}". Index names must be non-empty and cannot contain whitespace.`
+        );
+      }
+      if (!index.expr?.trim()) {
+        throw new Error(`Invalid datasource index "${index.name}": expr is required.`);
+      }
+      if (!index.type?.trim()) {
+        throw new Error(`Invalid datasource index "${index.name}": type is required.`);
+      }
+      if (!Number.isInteger(index.granularity) || index.granularity <= 0) {
+        throw new Error(
+          `Invalid datasource index "${index.name}": granularity must be a positive integer.`
+        );
+      }
+    }
   }
 
   return {