@tinybirdco/sdk 0.0.2 → 0.0.3

package/src/generator/connection.ts

@@ -0,0 +1,104 @@
+/**
+ * Connection content generator
+ * Converts ConnectionDefinition to native .connection file format
+ */
+
+import type { ConnectionDefinition, KafkaConnectionDefinition } from "../schema/connection.js";
+
+/**
+ * Generated connection content
+ */
+export interface GeneratedConnection {
+  /** Connection name */
+  name: string;
+  /** The generated .connection file content */
+  content: string;
+}
+
+/**
+ * Generate a Kafka connection content
+ */
+function generateKafkaConnection(connection: KafkaConnectionDefinition): string {
+  const parts: string[] = [];
+  const options = connection.options;
+
+  parts.push("TYPE kafka");
+  parts.push(`KAFKA_BOOTSTRAP_SERVERS ${options.bootstrapServers}`);
+
+  if (options.securityProtocol) {
+    parts.push(`KAFKA_SECURITY_PROTOCOL ${options.securityProtocol}`);
+  }
+
+  if (options.saslMechanism) {
+    parts.push(`KAFKA_SASL_MECHANISM ${options.saslMechanism}`);
+  }
+
+  if (options.key) {
+    parts.push(`KAFKA_KEY ${options.key}`);
+  }
+
+  if (options.secret) {
+    parts.push(`KAFKA_SECRET ${options.secret}`);
+  }
+
+  if (options.sslCaPem) {
+    parts.push(`KAFKA_SSL_CA_PEM ${options.sslCaPem}`);
+  }
+
+  return parts.join("\n");
+}
+
+/**
+ * Generate a .connection file content from a ConnectionDefinition
+ *
+ * @param connection - The connection definition
+ * @returns Generated connection content
+ *
+ * @example
+ * ```ts
+ * const myKafka = createKafkaConnection('my_kafka', {
+ *   bootstrapServers: 'kafka.example.com:9092',
+ *   securityProtocol: 'SASL_SSL',
+ *   saslMechanism: 'PLAIN',
+ *   key: '{{ tb_secret("KAFKA_KEY") }}',
+ *   secret: '{{ tb_secret("KAFKA_SECRET") }}',
+ * });
+ *
+ * const { content } = generateConnection(myKafka);
+ * // Returns:
+ * // TYPE kafka
+ * // KAFKA_BOOTSTRAP_SERVERS kafka.example.com:9092
+ * // KAFKA_SECURITY_PROTOCOL SASL_SSL
+ * // KAFKA_SASL_MECHANISM PLAIN
+ * // KAFKA_KEY {{ tb_secret("KAFKA_KEY") }}
+ * // KAFKA_SECRET {{ tb_secret("KAFKA_SECRET") }}
+ * ```
+ */
+export function generateConnection(
+  connection: ConnectionDefinition
+): GeneratedConnection {
+  let content: string;
+
+  if (connection._connectionType === "kafka") {
+    content = generateKafkaConnection(connection as KafkaConnectionDefinition);
+  } else {
+    throw new Error(`Unsupported connection type: ${connection._connectionType}`);
+  }
+
+  return {
+    name: connection._name,
+    content,
+  };
+}
+
+/**
+ * Generate .connection files for all connections in a project
+ *
+ * @param connections - Record of connection definitions
+ * @returns Array of generated connection content
+ */
+export function generateAllConnections(
+  connections: Record<string, ConnectionDefinition>
+): GeneratedConnection[] {
+  return Object.values(connections).map(generateConnection);
+}

package/src/generator/datasource.test.ts

@@ -1,6 +1,7 @@
 import { describe, it, expect } from 'vitest';
 import { generateDatasource, generateAllDatasources } from './datasource.js';
 import { defineDatasource } from '../schema/datasource.js';
+import { createKafkaConnection } from '../schema/connection.js';
 import { t } from '../schema/types.js';
 import { engine } from '../schema/engines.js';
 
@@ -294,4 +295,111 @@ describe('Datasource Generator', () => {
       expect(result.content).toContain('ENGINE_TTL "timestamp + INTERVAL 90 DAY"');
     });
   });
+
+  describe('Kafka configuration', () => {
+    it('includes Kafka connection name and topic', () => {
+      const kafkaConn = createKafkaConnection('my_kafka', {
+        bootstrapServers: 'kafka.example.com:9092',
+      });
+
+      const ds = defineDatasource('kafka_events', {
+        schema: {
+          timestamp: t.dateTime(),
+          event: t.string(),
+        },
+        engine: engine.mergeTree({ sortingKey: ['timestamp'] }),
+        kafka: {
+          connection: kafkaConn,
+          topic: 'events',
+        },
+      });
+
+      const result = generateDatasource(ds);
+
+      expect(result.content).toContain('KAFKA_CONNECTION_NAME my_kafka');
+      expect(result.content).toContain('KAFKA_TOPIC events');
+    });
+
+    it('includes Kafka group ID when provided', () => {
+      const kafkaConn = createKafkaConnection('my_kafka', {
+        bootstrapServers: 'kafka.example.com:9092',
+      });
+
+      const ds = defineDatasource('kafka_events', {
+        schema: {
+          timestamp: t.dateTime(),
+          event: t.string(),
+        },
+        engine: engine.mergeTree({ sortingKey: ['timestamp'] }),
+        kafka: {
+          connection: kafkaConn,
+          topic: 'events',
+          groupId: 'my-consumer-group',
+        },
+      });
+
+      const result = generateDatasource(ds);
+
+      expect(result.content).toContain('KAFKA_GROUP_ID my-consumer-group');
+    });
+
+    it('includes auto offset reset when provided', () => {
+      const kafkaConn = createKafkaConnection('my_kafka', {
+        bootstrapServers: 'kafka.example.com:9092',
+      });
+
+      const ds = defineDatasource('kafka_events', {
+        schema: {
+          timestamp: t.dateTime(),
+          event: t.string(),
+        },
+        engine: engine.mergeTree({ sortingKey: ['timestamp'] }),
+        kafka: {
+          connection: kafkaConn,
+          topic: 'events',
+          autoOffsetReset: 'earliest',
+        },
+      });
+
+      const result = generateDatasource(ds);
+
+      expect(result.content).toContain('KAFKA_AUTO_OFFSET_RESET earliest');
+    });
+
+    it('generates complete Kafka datasource with all options', () => {
+      const kafkaConn = createKafkaConnection('my_kafka', {
+        bootstrapServers: 'kafka.example.com:9092',
+        securityProtocol: 'SASL_SSL',
+        saslMechanism: 'PLAIN',
+      });
+
+      const ds = defineDatasource('kafka_events', {
+        description: 'Events from Kafka',
+        schema: {
+          timestamp: t.dateTime(),
+          event_type: t.string(),
+          payload: t.string(),
+        },
+        engine: engine.mergeTree({ sortingKey: ['timestamp'] }),
+        kafka: {
+          connection: kafkaConn,
+          topic: 'events',
+          groupId: 'my-consumer-group',
+          autoOffsetReset: 'earliest',
+        },
+      });
+
+      const result = generateDatasource(ds);
+
+      expect(result.name).toBe('kafka_events');
+      expect(result.content).toContain('DESCRIPTION >');
+      expect(result.content).toContain('Events from Kafka');
+      expect(result.content).toContain('SCHEMA >');
+      expect(result.content).toContain('ENGINE "MergeTree"');
+      expect(result.content).toContain('KAFKA_CONNECTION_NAME my_kafka');
+      expect(result.content).toContain('KAFKA_TOPIC events');
+      expect(result.content).toContain('KAFKA_GROUP_ID my-consumer-group');
+      expect(result.content).toContain('KAFKA_AUTO_OFFSET_RESET earliest');
+    });
+  });
 });

package/src/generator/datasource.ts

@@ -3,7 +3,7 @@
  * Converts DatasourceDefinition to native .datasource file format
  */
 
-import type { DatasourceDefinition, SchemaDefinition, ColumnDefinition } from "../schema/datasource.js";
+import type { DatasourceDefinition, SchemaDefinition, ColumnDefinition, KafkaConfig } from "../schema/datasource.js";
 import type { AnyTypeValidator, TypeModifiers } from "../schema/types.js";
 import { getColumnType, getColumnJsonPath } from "../schema/datasource.js";
 import { getEngineClause, type EngineConfig } from "../schema/engines.js";
@@ -143,6 +143,26 @@ function generateEngineConfig(engine?: EngineConfig): string {
   return getEngineClause(engine);
 }
 
+/**
+ * Generate Kafka configuration lines
+ */
+function generateKafkaConfig(kafka: KafkaConfig): string {
+  const parts: string[] = [];
+
+  parts.push(`KAFKA_CONNECTION_NAME ${kafka.connection._name}`);
+  parts.push(`KAFKA_TOPIC ${kafka.topic}`);
+
+  if (kafka.groupId) {
+    parts.push(`KAFKA_GROUP_ID ${kafka.groupId}`);
+  }
+
+  if (kafka.autoOffsetReset) {
+    parts.push(`KAFKA_AUTO_OFFSET_RESET ${kafka.autoOffsetReset}`);
+  }
+
+  return parts.join("\n");
+}
+
 /**
  * Generate a .datasource file content from a DatasourceDefinition
  *
@@ -198,6 +218,12 @@ export function generateDatasource(
   // Add engine configuration
   parts.push(generateEngineConfig(datasource.options.engine));
 
+  // Add Kafka configuration if present
+  if (datasource.options.kafka) {
+    parts.push("");
+    parts.push(generateKafkaConfig(datasource.options.kafka));
+  }
+
   return {
     name: datasource._name,
     content: parts.join("\n"),

package/src/generator/index.ts

@@ -6,7 +6,8 @@
 import { loadSchema, loadEntities, entitiesToProject, type LoadedEntities } from "./loader.js";
 import { generateAllDatasources, type GeneratedDatasource } from "./datasource.js";
 import { generateAllPipes, type GeneratedPipe } from "./pipe.js";
-import type { ProjectDefinition, DatasourcesDefinition, PipesDefinition } from "../schema/project.js";
+import { generateAllConnections, type GeneratedConnection } from "./connection.js";
+import type { ProjectDefinition, DatasourcesDefinition, PipesDefinition, ConnectionsDefinition } from "../schema/project.js";
 
 /**
  * Generated resources ready for API push
@@ -16,6 +17,8 @@ export interface GeneratedResources {
   datasources: GeneratedDatasource[];
   /** Generated pipe files */
   pipes: GeneratedPipe[];
+  /** Generated connection files */
+  connections: GeneratedConnection[];
 }
 
 /**
@@ -34,6 +37,7 @@ export interface BuildResult {
   stats: {
     datasourceCount: number;
     pipeCount: number;
+    connectionCount: number;
   };
 }
 
@@ -46,10 +50,12 @@ export interface BuildResult {
 export function generateResources(project: ProjectDefinition): GeneratedResources {
   const datasources = generateAllDatasources(project.datasources);
   const pipes = generateAllPipes(project.pipes);
+  const connections = generateAllConnections(project.connections);
 
   return {
     datasources,
     pipes,
+    connections,
   };
 }
 
@@ -107,6 +113,7 @@ export async function build(options: BuildOptions): Promise<BuildResult> {
     stats: {
       datasourceCount: resources.datasources.length,
       pipeCount: resources.pipes.length,
+      connectionCount: resources.connections.length,
     },
   };
 }
@@ -133,6 +140,7 @@ export interface BuildFromIncludeResult {
   stats: {
     datasourceCount: number;
     pipeCount: number;
+    connectionCount: number;
   };
 }
 
@@ -141,11 +149,13 @@ export interface BuildFromIncludeResult {
  */
 export function generateResourcesFromEntities(
   datasources: DatasourcesDefinition,
-  pipes: PipesDefinition
+  pipes: PipesDefinition,
+  connections: ConnectionsDefinition = {}
 ): GeneratedResources {
   return {
     datasources: generateAllDatasources(datasources),
     pipes: generateAllPipes(pipes),
+    connections: generateAllConnections(connections),
   };
 }
 
@@ -181,10 +191,10 @@ export async function buildFromInclude(
   });
 
   // Convert to format for generators
-  const { datasources, pipes } = entitiesToProject(entities);
+  const { datasources, pipes, connections } = entitiesToProject(entities);
 
   // Generate resources
-  const resources = generateResourcesFromEntities(datasources, pipes);
+  const resources = generateResourcesFromEntities(datasources, pipes, connections);
 
   return {
     resources,
@@ -192,6 +202,7 @@ export async function buildFromInclude(
     stats: {
       datasourceCount: resources.datasources.length,
       pipeCount: resources.pipes.length,
+      connectionCount: resources.connections.length,
     },
   };
 }
@@ -200,4 +211,5 @@ export async function buildFromInclude(
 export { loadSchema, loadEntities, entitiesToProject, type LoaderOptions, type LoadedSchema, type LoadedEntities, type LoadEntitiesOptions } from "./loader.js";
 export { generateDatasource, generateAllDatasources, type GeneratedDatasource } from "./datasource.js";
 export { generatePipe, generateAllPipes, type GeneratedPipe } from "./pipe.js";
+export { generateConnection, generateAllConnections, type GeneratedConnection } from "./connection.js";
 export { generateClientFile, type GenerateClientOptions, type GeneratedClient } from "./client.js";

package/src/generator/loader.ts

@@ -7,9 +7,10 @@ import * as esbuild from "esbuild";
 import * as path from "path";
 import * as fs from "fs";
 import { watch as chokidarWatch, type FSWatcher } from "chokidar";
-import { isProjectDefinition, type ProjectDefinition, type DatasourcesDefinition, type PipesDefinition } from "../schema/project.js";
+import { isProjectDefinition, type ProjectDefinition, type DatasourcesDefinition, type PipesDefinition, type ConnectionsDefinition } from "../schema/project.js";
 import { isDatasourceDefinition, type DatasourceDefinition } from "../schema/datasource.js";
 import { isPipeDefinition, type PipeDefinition } from "../schema/pipe.js";
+import { isConnectionDefinition, type ConnectionDefinition } from "../schema/connection.js";
 
 /**
  * Result of loading a schema file
@@ -157,6 +158,8 @@ export interface LoadedEntities {
   datasources: Record<string, { definition: DatasourceDefinition; info: EntityInfo }>;
   /** Discovered pipes with their metadata */
   pipes: Record<string, { definition: PipeDefinition; info: EntityInfo }>;
+  /** Discovered connections with their metadata */
+  connections: Record<string, { definition: ConnectionDefinition; info: EntityInfo }>;
   /** All source files that were scanned */
   sourceFiles: string[];
 }
@@ -195,6 +198,7 @@ export async function loadEntities(options: LoadEntitiesOptions): Promise<Loaded
   const result: LoadedEntities = {
     datasources: {},
     pipes: {},
+    connections: {},
     sourceFiles: [],
   };
 
@@ -237,7 +241,7 @@ export async function loadEntities(options: LoadEntitiesOptions): Promise<Loaded
       const moduleUrl = `file://${outfile}`;
       const module = await import(moduleUrl);
 
-      // Scan all exports for datasources and pipes
+      // Scan all exports for datasources, pipes, and connections
       for (const [exportName, value] of Object.entries(module)) {
         if (isDatasourceDefinition(value)) {
           result.datasources[exportName] = {
@@ -255,6 +259,14 @@ export async function loadEntities(options: LoadEntitiesOptions): Promise<Loaded
               sourceFile: includePath,
             },
           };
+        } else if (isConnectionDefinition(value)) {
+          result.connections[exportName] = {
+            definition: value,
+            info: {
+              exportName,
+              sourceFile: includePath,
+            },
+          };
         }
       }
     } finally {
@@ -282,9 +294,11 @@ export async function loadEntities(options: LoadEntitiesOptions): Promise<Loaded
 export function entitiesToProject(entities: LoadedEntities): {
   datasources: DatasourcesDefinition;
   pipes: PipesDefinition;
+  connections: ConnectionsDefinition;
 } {
   const datasources: DatasourcesDefinition = {};
   const pipes: PipesDefinition = {};
+  const connections: ConnectionsDefinition = {};
 
   for (const [name, { definition }] of Object.entries(entities.datasources)) {
     datasources[name] = definition;
@@ -294,7 +308,11 @@ export function entitiesToProject(entities: LoadedEntities): {
     pipes[name] = definition;
   }
 
-  return { datasources, pipes };
+  for (const [name, { definition }] of Object.entries(entities.connections)) {
+    connections[name] = definition;
+  }
+
+  return { datasources, pipes, connections };
 }
 
 /**

package/src/index.ts

@@ -106,8 +106,19 @@ export type {
   ColumnDefinition,
   TokenConfig,
   ExtractSchema,
+  KafkaConfig,
 } from "./schema/datasource.js";
 
+// ============ Connection ============
+export { createKafkaConnection, isConnectionDefinition, isKafkaConnectionDefinition, getConnectionType } from "./schema/connection.js";
+export type {
+  ConnectionDefinition,
+  KafkaConnectionDefinition,
+  KafkaConnectionOptions,
+  KafkaSecurityProtocol,
+  KafkaSaslMechanism,
+} from "./schema/connection.js";
+
 // ============ Pipe ============
 export {
   definePipe,
@@ -153,6 +164,7 @@ export type {
   TinybirdClientConfig,
   DatasourcesDefinition,
   PipesDefinition,
+  ConnectionsDefinition,
   ExtractDatasources,
   ExtractPipes,
   DataModel,

package/src/schema/connection.test.ts

@@ -0,0 +1,149 @@
+import { describe, it, expect } from "vitest";
+import {
+  createKafkaConnection,
+  isConnectionDefinition,
+  isKafkaConnectionDefinition,
+  getConnectionType,
+} from "./connection.js";
+
+describe("Connection Schema", () => {
+  describe("createKafkaConnection", () => {
+    it("creates a Kafka connection with required fields", () => {
+      const conn = createKafkaConnection("my_kafka", {
+        bootstrapServers: "kafka.example.com:9092",
+      });
+
+      expect(conn._name).toBe("my_kafka");
+      expect(conn._type).toBe("connection");
+      expect(conn._connectionType).toBe("kafka");
+      expect(conn.options.bootstrapServers).toBe("kafka.example.com:9092");
+    });
+
+    it("creates a Kafka connection with all options", () => {
+      const conn = createKafkaConnection("my_kafka", {
+        bootstrapServers: "kafka.example.com:9092",
+        securityProtocol: "SASL_SSL",
+        saslMechanism: "PLAIN",
+        key: '{{ tb_secret("KAFKA_KEY") }}',
+        secret: '{{ tb_secret("KAFKA_SECRET") }}',
+        sslCaPem: '{{ tb_secret("KAFKA_CA_CERT") }}',
+      });
+
+      expect(conn.options.securityProtocol).toBe("SASL_SSL");
+      expect(conn.options.saslMechanism).toBe("PLAIN");
+      expect(conn.options.key).toBe('{{ tb_secret("KAFKA_KEY") }}');
+      expect(conn.options.secret).toBe('{{ tb_secret("KAFKA_SECRET") }}');
+      expect(conn.options.sslCaPem).toBe('{{ tb_secret("KAFKA_CA_CERT") }}');
+    });
+
+    it("supports different SASL mechanisms", () => {
+      const scramConn = createKafkaConnection("scram_kafka", {
+        bootstrapServers: "kafka.example.com:9092",
+        saslMechanism: "SCRAM-SHA-256",
+      });
+      expect(scramConn.options.saslMechanism).toBe("SCRAM-SHA-256");
+
+      const scram512Conn = createKafkaConnection("scram512_kafka", {
+        bootstrapServers: "kafka.example.com:9092",
+        saslMechanism: "SCRAM-SHA-512",
+      });
+      expect(scram512Conn.options.saslMechanism).toBe("SCRAM-SHA-512");
+
+      const oauthConn = createKafkaConnection("oauth_kafka", {
+        bootstrapServers: "kafka.example.com:9092",
+        saslMechanism: "OAUTHBEARER",
+      });
+      expect(oauthConn.options.saslMechanism).toBe("OAUTHBEARER");
+    });
+
+    it("supports different security protocols", () => {
+      const plaintext = createKafkaConnection("plaintext_kafka", {
+        bootstrapServers: "localhost:9092",
+        securityProtocol: "PLAINTEXT",
+      });
+      expect(plaintext.options.securityProtocol).toBe("PLAINTEXT");
+
+      const saslPlaintext = createKafkaConnection("sasl_plaintext_kafka", {
+        bootstrapServers: "localhost:9092",
+        securityProtocol: "SASL_PLAINTEXT",
+      });
+      expect(saslPlaintext.options.securityProtocol).toBe("SASL_PLAINTEXT");
+    });
+
+    it("throws error for invalid connection name", () => {
+      expect(() =>
+        createKafkaConnection("123invalid", {
+          bootstrapServers: "kafka.example.com:9092",
+        })
+      ).toThrow("Invalid connection name");
+
+      expect(() =>
+        createKafkaConnection("my-connection", {
+          bootstrapServers: "kafka.example.com:9092",
+        })
+      ).toThrow("Invalid connection name");
+
+      expect(() =>
+        createKafkaConnection("", {
+          bootstrapServers: "kafka.example.com:9092",
+        })
+      ).toThrow("Invalid connection name");
+    });
+
+    it("allows valid naming patterns", () => {
+      const conn1 = createKafkaConnection("_private_kafka", {
+        bootstrapServers: "kafka.example.com:9092",
+      });
+      expect(conn1._name).toBe("_private_kafka");
+
+      const conn2 = createKafkaConnection("kafka_v2", {
+        bootstrapServers: "kafka.example.com:9092",
+      });
+      expect(conn2._name).toBe("kafka_v2");
+    });
+  });
+
+  describe("isConnectionDefinition", () => {
+    it("returns true for valid connection", () => {
+      const conn = createKafkaConnection("my_kafka", {
+        bootstrapServers: "kafka.example.com:9092",
+      });
+
+      expect(isConnectionDefinition(conn)).toBe(true);
+    });
+
+    it("returns false for non-connection objects", () => {
+      expect(isConnectionDefinition({})).toBe(false);
+      expect(isConnectionDefinition(null)).toBe(false);
+      expect(isConnectionDefinition(undefined)).toBe(false);
+      expect(isConnectionDefinition("string")).toBe(false);
+      expect(isConnectionDefinition(123)).toBe(false);
+      expect(isConnectionDefinition({ _name: "test" })).toBe(false);
+    });
+  });
+
+  describe("isKafkaConnectionDefinition", () => {
+    it("returns true for Kafka connection", () => {
+      const conn = createKafkaConnection("my_kafka", {
+        bootstrapServers: "kafka.example.com:9092",
+      });
+
+      expect(isKafkaConnectionDefinition(conn)).toBe(true);
+    });
+
+    it("returns false for non-Kafka objects", () => {
+      expect(isKafkaConnectionDefinition({})).toBe(false);
+      expect(isKafkaConnectionDefinition(null)).toBe(false);
+    });
+  });
+
+  describe("getConnectionType", () => {
+    it("returns the connection type", () => {
+      const conn = createKafkaConnection("my_kafka", {
+        bootstrapServers: "kafka.example.com:9092",
+      });
+
+      expect(getConnectionType(conn)).toBe("kafka");
+    });
+  });
+});