@tinybirdco/sdk 0.0.1 → 0.0.3

package/src/api/build.ts

@@ -88,6 +88,8 @@ export interface BuildApiResult {
   datasourceCount: number;
   /** Number of pipes deployed */
   pipeCount: number;
+  /** Number of connections deployed */
+  connectionCount: number;
   /** Build ID if successful */
   buildId?: string;
   /** Pipe changes in this build */
@@ -166,6 +168,21 @@ export async function buildToTinybird(
     );
   }
 
+  // Add connections
+  for (const conn of resources.connections ?? []) {
+    const fieldName = `data_project://`;
+    const fileName = `${conn.name}.connection`;
+    if (debug) {
+      console.log(`[debug] Adding connection: ${fieldName} (filename: ${fileName})`);
+      console.log(`[debug] Content:\n${conn.content}\n`);
+    }
+    formData.append(
+      fieldName,
+      new Blob([conn.content], { type: "text/plain" }),
+      fileName
+    );
+  }
+
   // Make the request
   const url = `${config.baseUrl.replace(/\/$/, "")}/v1/build`;
 
@@ -217,6 +234,7 @@ export async function buildToTinybird(
       error: formatErrors(),
       datasourceCount: resources.datasources.length,
       pipeCount: resources.pipes.length,
+      connectionCount: resources.connections?.length ?? 0,
     };
   }
 
@@ -228,6 +246,7 @@ export async function buildToTinybird(
       error: formatErrors(),
       datasourceCount: resources.datasources.length,
       pipeCount: resources.pipes.length,
+      connectionCount: resources.connections?.length ?? 0,
     };
   }
 
@@ -236,6 +255,7 @@ export async function buildToTinybird(
     result: body.result,
     datasourceCount: resources.datasources.length,
     pipeCount: resources.pipes.length,
+    connectionCount: resources.connections?.length ?? 0,
     buildId: body.build?.id,
     pipes: {
       changed: body.build?.changed_pipe_names ?? [],

package/src/api/deploy.test.ts

@@ -31,6 +31,7 @@ describe("Deploy API", () => {
     pipes: [
       { name: "top_events", content: "NODE main\nSQL > SELECT * FROM events" },
     ],
+    connections: [],
   };
 
   describe("deployToMain", () => {

package/src/api/deploy.ts

@@ -126,6 +126,7 @@ export async function deployToMain(
       error: formatErrors(),
       datasourceCount: resources.datasources.length,
       pipeCount: resources.pipes.length,
+      connectionCount: resources.connections?.length ?? 0,
     };
   }
 
@@ -137,6 +138,7 @@ export async function deployToMain(
       error: formatErrors(),
       datasourceCount: resources.datasources.length,
       pipeCount: resources.pipes.length,
+      connectionCount: resources.connections?.length ?? 0,
     };
   }
 
@@ -145,6 +147,7 @@ export async function deployToMain(
     result: body.result,
     datasourceCount: resources.datasources.length,
     pipeCount: resources.pipes.length,
+    connectionCount: resources.connections?.length ?? 0,
     buildId: body.build?.id,
     pipes: {
       changed: body.build?.changed_pipe_names ?? [],

package/src/cli/commands/init.test.ts

@@ -228,6 +228,78 @@ describe("Init Command", () => {
     });
   });
 
+  describe("package.json scripts", () => {
+    it("adds tinybird:dev and tinybird:build scripts to existing package.json", async () => {
+      const packageJson = { name: "test-project", scripts: { dev: "next dev" } };
+      fs.writeFileSync(
+        path.join(tempDir, "package.json"),
+        JSON.stringify(packageJson, null, 2)
+      );
+
+      const result = await runInit({ cwd: tempDir, skipLogin: true });
+
+      expect(result.success).toBe(true);
+      expect(result.created).toContain("package.json (added tinybird scripts)");
+
+      const updatedPackageJson = JSON.parse(
+        fs.readFileSync(path.join(tempDir, "package.json"), "utf-8")
+      );
+      expect(updatedPackageJson.scripts["tinybird:dev"]).toBe("tinybird dev");
+      expect(updatedPackageJson.scripts["tinybird:build"]).toBe("tinybird build");
+      expect(updatedPackageJson.scripts.dev).toBe("next dev"); // preserved
+    });
+
+    it("does not overwrite existing tinybird scripts", async () => {
+      const packageJson = {
+        name: "test-project",
+        scripts: {
+          "tinybird:dev": "custom dev command",
+          "tinybird:build": "custom build command",
+        },
+      };
+      fs.writeFileSync(
+        path.join(tempDir, "package.json"),
+        JSON.stringify(packageJson, null, 2)
+      );
+
+      const result = await runInit({ cwd: tempDir, skipLogin: true });
+
+      expect(result.success).toBe(true);
+      expect(result.created).not.toContain("package.json (added tinybird scripts)");
+
+      const updatedPackageJson = JSON.parse(
+        fs.readFileSync(path.join(tempDir, "package.json"), "utf-8")
+      );
+      expect(updatedPackageJson.scripts["tinybird:dev"]).toBe("custom dev command");
+      expect(updatedPackageJson.scripts["tinybird:build"]).toBe("custom build command");
+    });
+
+    it("creates scripts object if package.json has no scripts", async () => {
+      const packageJson = { name: "test-project" };
+      fs.writeFileSync(
+        path.join(tempDir, "package.json"),
+        JSON.stringify(packageJson, null, 2)
+      );
+
+      const result = await runInit({ cwd: tempDir, skipLogin: true });
+
+      expect(result.success).toBe(true);
+
+      const updatedPackageJson = JSON.parse(
+        fs.readFileSync(path.join(tempDir, "package.json"), "utf-8")
+      );
+      expect(updatedPackageJson.scripts["tinybird:dev"]).toBe("tinybird dev");
+      expect(updatedPackageJson.scripts["tinybird:build"]).toBe("tinybird build");
+    });
+
+    it("does not fail if no package.json exists", async () => {
+      const result = await runInit({ cwd: tempDir, skipLogin: true });
+
+      expect(result.success).toBe(true);
+      expect(result.created).not.toContain("package.json (added tinybird scripts)");
+    });
+  });
+
   describe("directory creation", () => {
     it("creates tinybird directory if it does not exist", async () => {
       expect(fs.existsSync(path.join(tempDir, "tinybird"))).toBe(false);

package/src/cli/commands/init.ts

@@ -267,6 +267,36 @@ export async function runInit(options: InitOptions = {}): Promise<InitResult> {
     }
   }
 
+  // Add scripts to package.json if it exists
+  const packageJsonPath = path.join(cwd, "package.json");
+  if (fs.existsSync(packageJsonPath)) {
+    try {
+      const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, "utf-8"));
+      let modified = false;
+
+      if (!packageJson.scripts) {
+        packageJson.scripts = {};
+      }
+
+      if (!packageJson.scripts["tinybird:dev"]) {
+        packageJson.scripts["tinybird:dev"] = "tinybird dev";
+        modified = true;
+      }
+
+      if (!packageJson.scripts["tinybird:build"]) {
+        packageJson.scripts["tinybird:build"] = "tinybird build";
+        modified = true;
+      }
+
+      if (modified) {
+        fs.writeFileSync(packageJsonPath, JSON.stringify(packageJson, null, 2) + "\n");
+        created.push("package.json (added tinybird scripts)");
+      }
+    } catch {
+      // Silently ignore package.json errors - not critical
+    }
+  }
+
   // Check if login is needed
   if (!skipLogin && !hasValidToken(cwd)) {
     console.log("\nNo authentication found. Starting login flow...\n");

package/src/cli/index.ts

@@ -4,6 +4,12 @@
  * Commands for building and deploying Tinybird projects
  */
 
+import { config } from "dotenv";
+
+// Load .env files in priority order (later files don't override earlier ones)
+config({ path: ".env.local" });
+config({ path: ".env" });
+
 import { readFileSync } from "node:fs";
 import { fileURLToPath } from "node:url";
 import { dirname, resolve } from "node:path";

package/src/generator/connection.test.ts

@@ -0,0 +1,135 @@
+import { describe, it, expect } from "vitest";
+import { generateConnection, generateAllConnections } from "./connection.js";
+import { createKafkaConnection } from "../schema/connection.js";
+
+describe("Connection Generator", () => {
+  describe("generateConnection", () => {
+    it("generates basic Kafka connection with required fields", () => {
+      const conn = createKafkaConnection("my_kafka", {
+        bootstrapServers: "kafka.example.com:9092",
+      });
+
+      const result = generateConnection(conn);
+
+      expect(result.name).toBe("my_kafka");
+      expect(result.content).toContain("TYPE kafka");
+      expect(result.content).toContain("KAFKA_BOOTSTRAP_SERVERS kafka.example.com:9092");
+    });
+
+    it("includes security protocol when provided", () => {
+      const conn = createKafkaConnection("my_kafka", {
+        bootstrapServers: "kafka.example.com:9092",
+        securityProtocol: "SASL_SSL",
+      });
+
+      const result = generateConnection(conn);
+
+      expect(result.content).toContain("KAFKA_SECURITY_PROTOCOL SASL_SSL");
+    });
+
+    it("includes SASL mechanism when provided", () => {
+      const conn = createKafkaConnection("my_kafka", {
+        bootstrapServers: "kafka.example.com:9092",
+        saslMechanism: "PLAIN",
+      });
+
+      const result = generateConnection(conn);
+
+      expect(result.content).toContain("KAFKA_SASL_MECHANISM PLAIN");
+    });
+
+    it("includes key and secret when provided", () => {
+      const conn = createKafkaConnection("my_kafka", {
+        bootstrapServers: "kafka.example.com:9092",
+        key: '{{ tb_secret("KAFKA_KEY") }}',
+        secret: '{{ tb_secret("KAFKA_SECRET") }}',
+      });
+
+      const result = generateConnection(conn);
+
+      expect(result.content).toContain('KAFKA_KEY {{ tb_secret("KAFKA_KEY") }}');
+      expect(result.content).toContain('KAFKA_SECRET {{ tb_secret("KAFKA_SECRET") }}');
+    });
+
+    it("includes SSL CA PEM when provided", () => {
+      const conn = createKafkaConnection("my_kafka", {
+        bootstrapServers: "kafka.example.com:9092",
+        sslCaPem: '{{ tb_secret("KAFKA_CA_CERT") }}',
+      });
+
+      const result = generateConnection(conn);
+
+      expect(result.content).toContain('KAFKA_SSL_CA_PEM {{ tb_secret("KAFKA_CA_CERT") }}');
+    });
+
+    it("generates full Kafka connection with all options", () => {
+      const conn = createKafkaConnection("my_kafka", {
+        bootstrapServers: "kafka.example.com:9092",
+        securityProtocol: "SASL_SSL",
+        saslMechanism: "SCRAM-SHA-256",
+        key: '{{ tb_secret("KAFKA_KEY") }}',
+        secret: '{{ tb_secret("KAFKA_SECRET") }}',
+        sslCaPem: '{{ tb_secret("KAFKA_CA_CERT") }}',
+      });
+
+      const result = generateConnection(conn);
+
+      expect(result.name).toBe("my_kafka");
+      expect(result.content).toContain("TYPE kafka");
+      expect(result.content).toContain("KAFKA_BOOTSTRAP_SERVERS kafka.example.com:9092");
+      expect(result.content).toContain("KAFKA_SECURITY_PROTOCOL SASL_SSL");
+      expect(result.content).toContain("KAFKA_SASL_MECHANISM SCRAM-SHA-256");
+      expect(result.content).toContain('KAFKA_KEY {{ tb_secret("KAFKA_KEY") }}');
+      expect(result.content).toContain('KAFKA_SECRET {{ tb_secret("KAFKA_SECRET") }}');
+      expect(result.content).toContain('KAFKA_SSL_CA_PEM {{ tb_secret("KAFKA_CA_CERT") }}');
+    });
+
+    it("supports PLAINTEXT security protocol", () => {
+      const conn = createKafkaConnection("local_kafka", {
+        bootstrapServers: "localhost:9092",
+        securityProtocol: "PLAINTEXT",
+      });
+
+      const result = generateConnection(conn);
+
+      expect(result.content).toContain("KAFKA_SECURITY_PROTOCOL PLAINTEXT");
+    });
+
+    it("supports different SASL mechanisms", () => {
+      const mechanisms = ["PLAIN", "SCRAM-SHA-256", "SCRAM-SHA-512", "OAUTHBEARER"] as const;
+
+      mechanisms.forEach((mechanism) => {
+        const conn = createKafkaConnection("my_kafka", {
+          bootstrapServers: "kafka.example.com:9092",
+          saslMechanism: mechanism,
+        });
+
+        const result = generateConnection(conn);
+
+        expect(result.content).toContain(`KAFKA_SASL_MECHANISM ${mechanism}`);
+      });
+    });
+  });
+
+  describe("generateAllConnections", () => {
+    it("generates all connections", () => {
+      const conn1 = createKafkaConnection("kafka1", {
+        bootstrapServers: "kafka1.example.com:9092",
+      });
+      const conn2 = createKafkaConnection("kafka2", {
+        bootstrapServers: "kafka2.example.com:9092",
+      });
+
+      const results = generateAllConnections({ kafka1: conn1, kafka2: conn2 });
+
+      expect(results).toHaveLength(2);
+      expect(results.map((r) => r.name).sort()).toEqual(["kafka1", "kafka2"]);
+    });
+
+    it("returns empty array for empty connections", () => {
+      const results = generateAllConnections({});
+
+      expect(results).toHaveLength(0);
+    });
+  });
+});

package/src/generator/connection.ts

@@ -0,0 +1,104 @@
+/**
+ * Connection content generator
+ * Converts ConnectionDefinition to native .connection file format
+ */
+
+import type { ConnectionDefinition, KafkaConnectionDefinition } from "../schema/connection.js";
+
+/**
+ * Generated connection content
+ */
+export interface GeneratedConnection {
+  /** Connection name */
+  name: string;
+  /** The generated .connection file content */
+  content: string;
+}
+
+/**
+ * Generate a Kafka connection content
+ */
+function generateKafkaConnection(connection: KafkaConnectionDefinition): string {
+  const parts: string[] = [];
+  const options = connection.options;
+
+  parts.push("TYPE kafka");
+  parts.push(`KAFKA_BOOTSTRAP_SERVERS ${options.bootstrapServers}`);
+
+  if (options.securityProtocol) {
+    parts.push(`KAFKA_SECURITY_PROTOCOL ${options.securityProtocol}`);
+  }
+
+  if (options.saslMechanism) {
+    parts.push(`KAFKA_SASL_MECHANISM ${options.saslMechanism}`);
+  }
+
+  if (options.key) {
+    parts.push(`KAFKA_KEY ${options.key}`);
+  }
+
+  if (options.secret) {
+    parts.push(`KAFKA_SECRET ${options.secret}`);
+  }
+
+  if (options.sslCaPem) {
+    parts.push(`KAFKA_SSL_CA_PEM ${options.sslCaPem}`);
+  }
+
+  return parts.join("\n");
+}
+
+/**
+ * Generate a .connection file content from a ConnectionDefinition
+ *
+ * @param connection - The connection definition
+ * @returns Generated connection content
+ *
+ * @example
+ * ```ts
+ * const myKafka = createKafkaConnection('my_kafka', {
+ *   bootstrapServers: 'kafka.example.com:9092',
+ *   securityProtocol: 'SASL_SSL',
+ *   saslMechanism: 'PLAIN',
+ *   key: '{{ tb_secret("KAFKA_KEY") }}',
+ *   secret: '{{ tb_secret("KAFKA_SECRET") }}',
+ * });
+ *
+ * const { content } = generateConnection(myKafka);
+ * // Returns:
+ * // TYPE kafka
+ * // KAFKA_BOOTSTRAP_SERVERS kafka.example.com:9092
+ * // KAFKA_SECURITY_PROTOCOL SASL_SSL
+ * // KAFKA_SASL_MECHANISM PLAIN
+ * // KAFKA_KEY {{ tb_secret("KAFKA_KEY") }}
+ * // KAFKA_SECRET {{ tb_secret("KAFKA_SECRET") }}
+ * ```
+ */
+export function generateConnection(
+  connection: ConnectionDefinition
+): GeneratedConnection {
+  let content: string;
+
+  if (connection._connectionType === "kafka") {
+    content = generateKafkaConnection(connection as KafkaConnectionDefinition);
+  } else {
+    throw new Error(`Unsupported connection type: ${connection._connectionType}`);
+  }
+
+  return {
+    name: connection._name,
+    content,
+  };
+}
+
+/**
+ * Generate .connection files for all connections in a project
+ *
+ * @param connections - Record of connection definitions
+ * @returns Array of generated connection content
+ */
+export function generateAllConnections(
+  connections: Record<string, ConnectionDefinition>
+): GeneratedConnection[] {
+  return Object.values(connections).map(generateConnection);
+}

package/src/generator/datasource.test.ts

@@ -1,6 +1,7 @@
 import { describe, it, expect } from 'vitest';
 import { generateDatasource, generateAllDatasources } from './datasource.js';
 import { defineDatasource } from '../schema/datasource.js';
+import { createKafkaConnection } from '../schema/connection.js';
 import { t } from '../schema/types.js';
 import { engine } from '../schema/engines.js';
 
@@ -294,4 +295,111 @@ describe('Datasource Generator', () => {
       expect(result.content).toContain('ENGINE_TTL "timestamp + INTERVAL 90 DAY"');
     });
   });
+
+  describe('Kafka configuration', () => {
+    it('includes Kafka connection name and topic', () => {
+      const kafkaConn = createKafkaConnection('my_kafka', {
+        bootstrapServers: 'kafka.example.com:9092',
+      });
+
+      const ds = defineDatasource('kafka_events', {
+        schema: {
+          timestamp: t.dateTime(),
+          event: t.string(),
+        },
+        engine: engine.mergeTree({ sortingKey: ['timestamp'] }),
+        kafka: {
+          connection: kafkaConn,
+          topic: 'events',
+        },
+      });
+
+      const result = generateDatasource(ds);
+
+      expect(result.content).toContain('KAFKA_CONNECTION_NAME my_kafka');
+      expect(result.content).toContain('KAFKA_TOPIC events');
+    });
+
+    it('includes Kafka group ID when provided', () => {
+      const kafkaConn = createKafkaConnection('my_kafka', {
+        bootstrapServers: 'kafka.example.com:9092',
+      });
+
+      const ds = defineDatasource('kafka_events', {
+        schema: {
+          timestamp: t.dateTime(),
+          event: t.string(),
+        },
+        engine: engine.mergeTree({ sortingKey: ['timestamp'] }),
+        kafka: {
+          connection: kafkaConn,
+          topic: 'events',
+          groupId: 'my-consumer-group',
+        },
+      });
+
+      const result = generateDatasource(ds);
+
+      expect(result.content).toContain('KAFKA_GROUP_ID my-consumer-group');
+    });
+
+    it('includes auto offset reset when provided', () => {
+      const kafkaConn = createKafkaConnection('my_kafka', {
+        bootstrapServers: 'kafka.example.com:9092',
+      });
+
+      const ds = defineDatasource('kafka_events', {
+        schema: {
+          timestamp: t.dateTime(),
+          event: t.string(),
+        },
+        engine: engine.mergeTree({ sortingKey: ['timestamp'] }),
+        kafka: {
+          connection: kafkaConn,
+          topic: 'events',
+          autoOffsetReset: 'earliest',
+        },
+      });
+
+      const result = generateDatasource(ds);
+
+      expect(result.content).toContain('KAFKA_AUTO_OFFSET_RESET earliest');
+    });
+
+    it('generates complete Kafka datasource with all options', () => {
+      const kafkaConn = createKafkaConnection('my_kafka', {
+        bootstrapServers: 'kafka.example.com:9092',
+        securityProtocol: 'SASL_SSL',
+        saslMechanism: 'PLAIN',
+      });
+
+      const ds = defineDatasource('kafka_events', {
+        description: 'Events from Kafka',
+        schema: {
+          timestamp: t.dateTime(),
+          event_type: t.string(),
+          payload: t.string(),
+        },
+        engine: engine.mergeTree({ sortingKey: ['timestamp'] }),
+        kafka: {
+          connection: kafkaConn,
+          topic: 'events',
+          groupId: 'my-consumer-group',
+          autoOffsetReset: 'earliest',
+        },
+      });
+
+      const result = generateDatasource(ds);
+
+      expect(result.name).toBe('kafka_events');
+      expect(result.content).toContain('DESCRIPTION >');
+      expect(result.content).toContain('Events from Kafka');
+      expect(result.content).toContain('SCHEMA >');
+      expect(result.content).toContain('ENGINE "MergeTree"');
+      expect(result.content).toContain('KAFKA_CONNECTION_NAME my_kafka');
+      expect(result.content).toContain('KAFKA_TOPIC events');
+      expect(result.content).toContain('KAFKA_GROUP_ID my-consumer-group');
+      expect(result.content).toContain('KAFKA_AUTO_OFFSET_RESET earliest');
+    });
+  });
 });

package/src/generator/datasource.ts

@@ -3,7 +3,7 @@
  * Converts DatasourceDefinition to native .datasource file format
  */
 
-import type { DatasourceDefinition, SchemaDefinition, ColumnDefinition } from "../schema/datasource.js";
+import type { DatasourceDefinition, SchemaDefinition, ColumnDefinition, KafkaConfig } from "../schema/datasource.js";
 import type { AnyTypeValidator, TypeModifiers } from "../schema/types.js";
 import { getColumnType, getColumnJsonPath } from "../schema/datasource.js";
 import { getEngineClause, type EngineConfig } from "../schema/engines.js";
@@ -143,6 +143,26 @@ function generateEngineConfig(engine?: EngineConfig): string {
   return getEngineClause(engine);
 }
 
+/**
+ * Generate Kafka configuration lines
+ */
+function generateKafkaConfig(kafka: KafkaConfig): string {
+  const parts: string[] = [];
+
+  parts.push(`KAFKA_CONNECTION_NAME ${kafka.connection._name}`);
+  parts.push(`KAFKA_TOPIC ${kafka.topic}`);
+
+  if (kafka.groupId) {
+    parts.push(`KAFKA_GROUP_ID ${kafka.groupId}`);
+  }
+
+  if (kafka.autoOffsetReset) {
+    parts.push(`KAFKA_AUTO_OFFSET_RESET ${kafka.autoOffsetReset}`);
+  }
+
+  return parts.join("\n");
+}
+
 /**
  * Generate a .datasource file content from a DatasourceDefinition
  *
@@ -198,6 +218,12 @@ export function generateDatasource(
   // Add engine configuration
   parts.push(generateEngineConfig(datasource.options.engine));
 
+  // Add Kafka configuration if present
+  if (datasource.options.kafka) {
+    parts.push("");
+    parts.push(generateKafkaConfig(datasource.options.kafka));
+  }
+
   return {
     name: datasource._name,
     content: parts.join("\n"),