@tinybirdco/sdk 0.0.1 → 0.0.3

package/src/generator/index.ts

@@ -6,7 +6,8 @@
 import { loadSchema, loadEntities, entitiesToProject, type LoadedEntities } from "./loader.js";
 import { generateAllDatasources, type GeneratedDatasource } from "./datasource.js";
 import { generateAllPipes, type GeneratedPipe } from "./pipe.js";
-import type { ProjectDefinition, DatasourcesDefinition, PipesDefinition } from "../schema/project.js";
+import { generateAllConnections, type GeneratedConnection } from "./connection.js";
+import type { ProjectDefinition, DatasourcesDefinition, PipesDefinition, ConnectionsDefinition } from "../schema/project.js";
 
 /**
  * Generated resources ready for API push
@@ -16,6 +17,8 @@ export interface GeneratedResources {
   datasources: GeneratedDatasource[];
   /** Generated pipe files */
   pipes: GeneratedPipe[];
+  /** Generated connection files */
+  connections: GeneratedConnection[];
 }
 
 /**
@@ -34,6 +37,7 @@ export interface BuildResult {
   stats: {
     datasourceCount: number;
     pipeCount: number;
+    connectionCount: number;
   };
 }
 
@@ -46,10 +50,12 @@ export interface BuildResult {
 export function generateResources(project: ProjectDefinition): GeneratedResources {
   const datasources = generateAllDatasources(project.datasources);
   const pipes = generateAllPipes(project.pipes);
+  const connections = generateAllConnections(project.connections);
 
   return {
     datasources,
     pipes,
+    connections,
   };
 }
 
@@ -107,6 +113,7 @@ export async function build(options: BuildOptions): Promise<BuildResult> {
     stats: {
       datasourceCount: resources.datasources.length,
       pipeCount: resources.pipes.length,
+      connectionCount: resources.connections.length,
     },
   };
 }
@@ -133,6 +140,7 @@ export interface BuildFromIncludeResult {
   stats: {
     datasourceCount: number;
     pipeCount: number;
+    connectionCount: number;
   };
 }
 
@@ -141,11 +149,13 @@ export interface BuildFromIncludeResult {
  */
 export function generateResourcesFromEntities(
   datasources: DatasourcesDefinition,
-  pipes: PipesDefinition
+  pipes: PipesDefinition,
+  connections: ConnectionsDefinition = {}
 ): GeneratedResources {
   return {
     datasources: generateAllDatasources(datasources),
     pipes: generateAllPipes(pipes),
+    connections: generateAllConnections(connections),
   };
 }
 
@@ -181,10 +191,10 @@ export async function buildFromInclude(
   });
 
   // Convert to format for generators
-  const { datasources, pipes } = entitiesToProject(entities);
+  const { datasources, pipes, connections } = entitiesToProject(entities);
 
   // Generate resources
-  const resources = generateResourcesFromEntities(datasources, pipes);
+  const resources = generateResourcesFromEntities(datasources, pipes, connections);
 
   return {
     resources,
@@ -192,6 +202,7 @@ export async function buildFromInclude(
     stats: {
       datasourceCount: resources.datasources.length,
       pipeCount: resources.pipes.length,
+      connectionCount: resources.connections.length,
     },
   };
 }
@@ -200,4 +211,5 @@ export async function buildFromInclude(
 export { loadSchema, loadEntities, entitiesToProject, type LoaderOptions, type LoadedSchema, type LoadedEntities, type LoadEntitiesOptions } from "./loader.js";
 export { generateDatasource, generateAllDatasources, type GeneratedDatasource } from "./datasource.js";
 export { generatePipe, generateAllPipes, type GeneratedPipe } from "./pipe.js";
+export { generateConnection, generateAllConnections, type GeneratedConnection } from "./connection.js";
 export { generateClientFile, type GenerateClientOptions, type GeneratedClient } from "./client.js";

package/src/generator/loader.ts

@@ -7,9 +7,10 @@ import * as esbuild from "esbuild";
 import * as path from "path";
 import * as fs from "fs";
 import { watch as chokidarWatch, type FSWatcher } from "chokidar";
-import { isProjectDefinition, type ProjectDefinition, type DatasourcesDefinition, type PipesDefinition } from "../schema/project.js";
+import { isProjectDefinition, type ProjectDefinition, type DatasourcesDefinition, type PipesDefinition, type ConnectionsDefinition } from "../schema/project.js";
 import { isDatasourceDefinition, type DatasourceDefinition } from "../schema/datasource.js";
 import { isPipeDefinition, type PipeDefinition } from "../schema/pipe.js";
+import { isConnectionDefinition, type ConnectionDefinition } from "../schema/connection.js";
 
 /**
  * Result of loading a schema file
@@ -157,6 +158,8 @@ export interface LoadedEntities {
   datasources: Record<string, { definition: DatasourceDefinition; info: EntityInfo }>;
   /** Discovered pipes with their metadata */
   pipes: Record<string, { definition: PipeDefinition; info: EntityInfo }>;
+  /** Discovered connections with their metadata */
+  connections: Record<string, { definition: ConnectionDefinition; info: EntityInfo }>;
   /** All source files that were scanned */
   sourceFiles: string[];
 }
@@ -195,6 +198,7 @@ export async function loadEntities(options: LoadEntitiesOptions): Promise<Loaded
   const result: LoadedEntities = {
     datasources: {},
     pipes: {},
+    connections: {},
     sourceFiles: [],
   };
 
@@ -237,7 +241,7 @@ export async function loadEntities(options: LoadEntitiesOptions): Promise<Loaded
       const moduleUrl = `file://${outfile}`;
       const module = await import(moduleUrl);
 
-      // Scan all exports for datasources and pipes
+      // Scan all exports for datasources, pipes, and connections
       for (const [exportName, value] of Object.entries(module)) {
         if (isDatasourceDefinition(value)) {
           result.datasources[exportName] = {
@@ -255,6 +259,14 @@ export async function loadEntities(options: LoadEntitiesOptions): Promise<Loaded
               sourceFile: includePath,
             },
           };
+        } else if (isConnectionDefinition(value)) {
+          result.connections[exportName] = {
+            definition: value,
+            info: {
+              exportName,
+              sourceFile: includePath,
+            },
+          };
         }
       }
     } finally {
@@ -282,9 +294,11 @@ export async function loadEntities(options: LoadEntitiesOptions): Promise<Loaded
 export function entitiesToProject(entities: LoadedEntities): {
   datasources: DatasourcesDefinition;
   pipes: PipesDefinition;
+  connections: ConnectionsDefinition;
 } {
   const datasources: DatasourcesDefinition = {};
   const pipes: PipesDefinition = {};
+  const connections: ConnectionsDefinition = {};
 
   for (const [name, { definition }] of Object.entries(entities.datasources)) {
     datasources[name] = definition;
@@ -294,7 +308,11 @@ export function entitiesToProject(entities: LoadedEntities): {
     pipes[name] = definition;
   }
 
-  return { datasources, pipes };
+  for (const [name, { definition }] of Object.entries(entities.connections)) {
+    connections[name] = definition;
+  }
+
+  return { datasources, pipes, connections };
 }
 
 /**

package/src/index.ts

@@ -106,8 +106,19 @@ export type {
   ColumnDefinition,
   TokenConfig,
   ExtractSchema,
+  KafkaConfig,
 } from "./schema/datasource.js";
 
+// ============ Connection ============
+export { createKafkaConnection, isConnectionDefinition, isKafkaConnectionDefinition, getConnectionType } from "./schema/connection.js";
+export type {
+  ConnectionDefinition,
+  KafkaConnectionDefinition,
+  KafkaConnectionOptions,
+  KafkaSecurityProtocol,
+  KafkaSaslMechanism,
+} from "./schema/connection.js";
+
 // ============ Pipe ============
 export {
   definePipe,
@@ -153,6 +164,7 @@ export type {
   TinybirdClientConfig,
   DatasourcesDefinition,
   PipesDefinition,
+  ConnectionsDefinition,
   ExtractDatasources,
   ExtractPipes,
   DataModel,

package/src/schema/connection.test.ts

@@ -0,0 +1,149 @@
+import { describe, it, expect } from "vitest";
+import {
+  createKafkaConnection,
+  isConnectionDefinition,
+  isKafkaConnectionDefinition,
+  getConnectionType,
+} from "./connection.js";
+
+describe("Connection Schema", () => {
+  describe("createKafkaConnection", () => {
+    it("creates a Kafka connection with required fields", () => {
+      const conn = createKafkaConnection("my_kafka", {
+        bootstrapServers: "kafka.example.com:9092",
+      });
+
+      expect(conn._name).toBe("my_kafka");
+      expect(conn._type).toBe("connection");
+      expect(conn._connectionType).toBe("kafka");
+      expect(conn.options.bootstrapServers).toBe("kafka.example.com:9092");
+    });
+
+    it("creates a Kafka connection with all options", () => {
+      const conn = createKafkaConnection("my_kafka", {
+        bootstrapServers: "kafka.example.com:9092",
+        securityProtocol: "SASL_SSL",
+        saslMechanism: "PLAIN",
+        key: '{{ tb_secret("KAFKA_KEY") }}',
+        secret: '{{ tb_secret("KAFKA_SECRET") }}',
+        sslCaPem: '{{ tb_secret("KAFKA_CA_CERT") }}',
+      });
+
+      expect(conn.options.securityProtocol).toBe("SASL_SSL");
+      expect(conn.options.saslMechanism).toBe("PLAIN");
+      expect(conn.options.key).toBe('{{ tb_secret("KAFKA_KEY") }}');
+      expect(conn.options.secret).toBe('{{ tb_secret("KAFKA_SECRET") }}');
+      expect(conn.options.sslCaPem).toBe('{{ tb_secret("KAFKA_CA_CERT") }}');
+    });
+
+    it("supports different SASL mechanisms", () => {
+      const scramConn = createKafkaConnection("scram_kafka", {
+        bootstrapServers: "kafka.example.com:9092",
+        saslMechanism: "SCRAM-SHA-256",
+      });
+      expect(scramConn.options.saslMechanism).toBe("SCRAM-SHA-256");
+
+      const scram512Conn = createKafkaConnection("scram512_kafka", {
+        bootstrapServers: "kafka.example.com:9092",
+        saslMechanism: "SCRAM-SHA-512",
+      });
+      expect(scram512Conn.options.saslMechanism).toBe("SCRAM-SHA-512");
+
+      const oauthConn = createKafkaConnection("oauth_kafka", {
+        bootstrapServers: "kafka.example.com:9092",
+        saslMechanism: "OAUTHBEARER",
+      });
+      expect(oauthConn.options.saslMechanism).toBe("OAUTHBEARER");
+    });
+
+    it("supports different security protocols", () => {
+      const plaintext = createKafkaConnection("plaintext_kafka", {
+        bootstrapServers: "localhost:9092",
+        securityProtocol: "PLAINTEXT",
+      });
+      expect(plaintext.options.securityProtocol).toBe("PLAINTEXT");
+
+      const saslPlaintext = createKafkaConnection("sasl_plaintext_kafka", {
+        bootstrapServers: "localhost:9092",
+        securityProtocol: "SASL_PLAINTEXT",
+      });
+      expect(saslPlaintext.options.securityProtocol).toBe("SASL_PLAINTEXT");
+    });
+
+    it("throws error for invalid connection name", () => {
+      expect(() =>
+        createKafkaConnection("123invalid", {
+          bootstrapServers: "kafka.example.com:9092",
+        })
+      ).toThrow("Invalid connection name");
+
+      expect(() =>
+        createKafkaConnection("my-connection", {
+          bootstrapServers: "kafka.example.com:9092",
+        })
+      ).toThrow("Invalid connection name");
+
+      expect(() =>
+        createKafkaConnection("", {
+          bootstrapServers: "kafka.example.com:9092",
+        })
+      ).toThrow("Invalid connection name");
+    });
+
+    it("allows valid naming patterns", () => {
+      const conn1 = createKafkaConnection("_private_kafka", {
+        bootstrapServers: "kafka.example.com:9092",
+      });
+      expect(conn1._name).toBe("_private_kafka");
+
+      const conn2 = createKafkaConnection("kafka_v2", {
+        bootstrapServers: "kafka.example.com:9092",
+      });
+      expect(conn2._name).toBe("kafka_v2");
+    });
+  });
+
+  describe("isConnectionDefinition", () => {
+    it("returns true for valid connection", () => {
+      const conn = createKafkaConnection("my_kafka", {
+        bootstrapServers: "kafka.example.com:9092",
+      });
+
+      expect(isConnectionDefinition(conn)).toBe(true);
+    });
+
+    it("returns false for non-connection objects", () => {
+      expect(isConnectionDefinition({})).toBe(false);
+      expect(isConnectionDefinition(null)).toBe(false);
+      expect(isConnectionDefinition(undefined)).toBe(false);
+      expect(isConnectionDefinition("string")).toBe(false);
+      expect(isConnectionDefinition(123)).toBe(false);
+      expect(isConnectionDefinition({ _name: "test" })).toBe(false);
+    });
+  });
+
+  describe("isKafkaConnectionDefinition", () => {
+    it("returns true for Kafka connection", () => {
+      const conn = createKafkaConnection("my_kafka", {
+        bootstrapServers: "kafka.example.com:9092",
+      });
+
+      expect(isKafkaConnectionDefinition(conn)).toBe(true);
+    });
+
+    it("returns false for non-Kafka objects", () => {
+      expect(isKafkaConnectionDefinition({})).toBe(false);
+      expect(isKafkaConnectionDefinition(null)).toBe(false);
+    });
+  });
+
+  describe("getConnectionType", () => {
+    it("returns the connection type", () => {
+      const conn = createKafkaConnection("my_kafka", {
+        bootstrapServers: "kafka.example.com:9092",
+      });
+
+      expect(getConnectionType(conn)).toBe("kafka");
+    });
+  });
+});

package/src/schema/connection.ts

@@ -0,0 +1,123 @@
+/**
+ * Connection definition for Tinybird
+ * Define external connections (Kafka, etc.) as TypeScript with full type safety
+ */
+
+// Symbol for brand typing
+const CONNECTION_BRAND = Symbol("tinybird.connection");
+
+/**
+ * Kafka security protocol options
+ */
+export type KafkaSecurityProtocol = "SASL_SSL" | "PLAINTEXT" | "SASL_PLAINTEXT";
+
+/**
+ * Kafka SASL mechanism options
+ */
+export type KafkaSaslMechanism = "PLAIN" | "SCRAM-SHA-256" | "SCRAM-SHA-512" | "OAUTHBEARER";
+
+/**
+ * Options for creating a Kafka connection
+ */
+export interface KafkaConnectionOptions {
+  /** Kafka bootstrap servers (host:port) */
+  bootstrapServers: string;
+  /** Security protocol (default: 'SASL_SSL') */
+  securityProtocol?: KafkaSecurityProtocol;
+  /** SASL mechanism for authentication */
+  saslMechanism?: KafkaSaslMechanism;
+  /** Kafka key/username - can use {{ tb_secret(...) }} */
+  key?: string;
+  /** Kafka secret/password - can use {{ tb_secret(...) }} */
+  secret?: string;
+  /** SSL CA certificate PEM - for private CA certs */
+  sslCaPem?: string;
+}
+
+/**
+ * Kafka-specific connection definition
+ */
+export interface KafkaConnectionDefinition {
+  readonly [CONNECTION_BRAND]: true;
+  /** Connection name */
+  readonly _name: string;
+  /** Type marker for inference */
+  readonly _type: "connection";
+  /** Connection type */
+  readonly _connectionType: "kafka";
+  /** Kafka options */
+  readonly options: KafkaConnectionOptions;
+}
+
+/**
+ * A connection definition - union of all connection types
+ */
+export type ConnectionDefinition = KafkaConnectionDefinition;
+
+/**
+ * Create a Kafka connection
+ *
+ * @param name - The connection name (must be valid identifier)
+ * @param options - Kafka connection configuration
+ * @returns A connection definition that can be used in a project
+ *
+ * @example
+ * ```ts
+ * import { createKafkaConnection } from '@tinybirdco/sdk';
+ *
+ * export const myKafka = createKafkaConnection('my_kafka', {
+ *   bootstrapServers: 'kafka.example.com:9092',
+ *   securityProtocol: 'SASL_SSL',
+ *   saslMechanism: 'PLAIN',
+ *   key: '{{ tb_secret("KAFKA_KEY") }}',
+ *   secret: '{{ tb_secret("KAFKA_SECRET") }}',
+ * });
+ * ```
+ */
+export function createKafkaConnection(
+  name: string,
+  options: KafkaConnectionOptions
+): KafkaConnectionDefinition {
+  // Validate name is a valid identifier
+  if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) {
+    throw new Error(
+      `Invalid connection name: "${name}". Must start with a letter or underscore and contain only alphanumeric characters and underscores.`
+    );
+  }
+
+  return {
+    [CONNECTION_BRAND]: true,
+    _name: name,
+    _type: "connection",
+    _connectionType: "kafka",
+    options,
+  };
+}
+
+/**
+ * Check if a value is a connection definition
+ */
+export function isConnectionDefinition(value: unknown): value is ConnectionDefinition {
+  return (
+    typeof value === "object" &&
+    value !== null &&
+    CONNECTION_BRAND in value &&
+    (value as Record<symbol, unknown>)[CONNECTION_BRAND] === true
+  );
+}
+
+/**
+ * Check if a value is a Kafka connection definition
+ */
+export function isKafkaConnectionDefinition(value: unknown): value is KafkaConnectionDefinition {
+  return isConnectionDefinition(value) && value._connectionType === "kafka";
+}
+
+/**
+ * Get the connection type from a connection definition
+ */
+export function getConnectionType<T extends ConnectionDefinition>(
+  connection: T
+): T["_connectionType"] {
+  return connection._connectionType;
+}

package/src/schema/datasource.ts

@@ -5,6 +5,7 @@
 
 import type { AnyTypeValidator } from "./types.js";
 import type { EngineConfig } from "./engines.js";
+import type { KafkaConnectionDefinition } from "./connection.js";
 
 // Symbol for brand typing
 const DATASOURCE_BRAND = Symbol("tinybird.datasource");
@@ -35,6 +36,20 @@ export interface TokenConfig {
   permissions: readonly ("READ" | "APPEND")[];
 }
 
+/**
+ * Kafka ingestion configuration for a datasource
+ */
+export interface KafkaConfig {
+  /** Kafka connection to use */
+  connection: KafkaConnectionDefinition;
+  /** Kafka topic to consume from */
+  topic: string;
+  /** Consumer group ID (optional) */
+  groupId?: string;
+  /** Where to start reading: 'earliest' or 'latest' (default: 'latest') */
+  autoOffsetReset?: "earliest" | "latest";
+}
+
 /**
  * Options for defining a datasource
  */
@@ -55,6 +70,8 @@ export interface DatasourceOptions<TSchema extends SchemaDefinition> {
    * Defaults to true.
    */
   jsonPaths?: boolean;
+  /** Kafka ingestion configuration */
+  kafka?: KafkaConfig;
 }
 
 /**

package/src/schema/project.ts

@@ -5,6 +5,7 @@
 
 import type { DatasourceDefinition, SchemaDefinition } from "./datasource.js";
 import type { PipeDefinition, ParamsDefinition, OutputDefinition } from "./pipe.js";
+import type { ConnectionDefinition } from "./connection.js";
 import { getEndpointConfig } from "./pipe.js";
 import type { TinybirdClient } from "../client/base.js";
 import type { QueryResult } from "../client/types.js";
@@ -23,6 +24,11 @@ export type DatasourcesDefinition = Record<string, DatasourceDefinition<SchemaDe
  */
 export type PipesDefinition = Record<string, PipeDefinition<ParamsDefinition, OutputDefinition>>;
 
+/**
+ * Collection of connection definitions
+ */
+export type ConnectionsDefinition = Record<string, ConnectionDefinition>;
+
 /**
  * Type for a single query method
  */
@@ -101,12 +107,15 @@ export interface TinybirdClientConfig<
  */
 export interface ProjectConfig<
   TDatasources extends DatasourcesDefinition = DatasourcesDefinition,
-  TPipes extends PipesDefinition = PipesDefinition
+  TPipes extends PipesDefinition = PipesDefinition,
+  TConnections extends ConnectionsDefinition = ConnectionsDefinition
 > {
   /** All datasources in this project */
   datasources?: TDatasources;
   /** All pipes in this project */
   pipes?: TPipes;
+  /** All connections in this project */
+  connections?: TConnections;
 }
 
 /**
@@ -114,7 +123,8 @@ export interface ProjectConfig<
  */
 export interface ProjectDefinition<
   TDatasources extends DatasourcesDefinition = DatasourcesDefinition,
-  TPipes extends PipesDefinition = PipesDefinition
+  TPipes extends PipesDefinition = PipesDefinition,
+  TConnections extends ConnectionsDefinition = ConnectionsDefinition
 > {
   readonly [PROJECT_BRAND]: true;
   /** Type marker for inference */
@@ -123,6 +133,8 @@ export interface ProjectDefinition<
   readonly datasources: TDatasources;
   /** All pipes */
   readonly pipes: TPipes;
+  /** All connections */
+  readonly connections: TConnections;
   /** Typed Tinybird client */
   readonly tinybird: ProjectClient<TDatasources, TPipes>;
 }
@@ -157,12 +169,14 @@ export interface ProjectDefinition<
  */
 export function defineProject<
   TDatasources extends DatasourcesDefinition,
-  TPipes extends PipesDefinition
+  TPipes extends PipesDefinition,
+  TConnections extends ConnectionsDefinition
 >(
-  config: ProjectConfig<TDatasources, TPipes>
-): ProjectDefinition<TDatasources, TPipes> {
+  config: ProjectConfig<TDatasources, TPipes, TConnections>
+): ProjectDefinition<TDatasources, TPipes, TConnections> {
   const datasources = (config.datasources ?? {}) as TDatasources;
   const pipes = (config.pipes ?? {}) as TPipes;
+  const connections = (config.connections ?? {}) as TConnections;
 
   // Use the shared client builder
   const tinybird = buildProjectClient(datasources, pipes);
@@ -172,6 +186,7 @@ export function defineProject<
     _type: "project",
     datasources,
     pipes,
+    connections,
     tinybird,
   };
 }