@tinybirdco/sdk 0.0.2 → 0.0.3

package/src/schema/connection.ts

@@ -0,0 +1,123 @@
+/**
+ * Connection definition for Tinybird
+ * Define external connections (Kafka, etc.) as TypeScript with full type safety
+ */
+
+// Symbol for brand typing
+const CONNECTION_BRAND = Symbol("tinybird.connection");
+
+/**
+ * Kafka security protocol options
+ */
+export type KafkaSecurityProtocol = "SASL_SSL" | "PLAINTEXT" | "SASL_PLAINTEXT";
+
+/**
+ * Kafka SASL mechanism options
+ */
+export type KafkaSaslMechanism = "PLAIN" | "SCRAM-SHA-256" | "SCRAM-SHA-512" | "OAUTHBEARER";
+
+/**
+ * Options for creating a Kafka connection
+ */
+export interface KafkaConnectionOptions {
+  /** Kafka bootstrap servers (host:port) */
+  bootstrapServers: string;
+  /** Security protocol (default: 'SASL_SSL') */
+  securityProtocol?: KafkaSecurityProtocol;
+  /** SASL mechanism for authentication */
+  saslMechanism?: KafkaSaslMechanism;
+  /** Kafka key/username - can use {{ tb_secret(...) }} */
+  key?: string;
+  /** Kafka secret/password - can use {{ tb_secret(...) }} */
+  secret?: string;
+  /** SSL CA certificate PEM - for private CA certs */
+  sslCaPem?: string;
+}
+
+/**
+ * Kafka-specific connection definition
+ */
+export interface KafkaConnectionDefinition {
+  readonly [CONNECTION_BRAND]: true;
+  /** Connection name */
+  readonly _name: string;
+  /** Type marker for inference */
+  readonly _type: "connection";
+  /** Connection type */
+  readonly _connectionType: "kafka";
+  /** Kafka options */
+  readonly options: KafkaConnectionOptions;
+}
+
+/**
+ * A connection definition - union of all connection types
+ */
+export type ConnectionDefinition = KafkaConnectionDefinition;
+
+/**
+ * Create a Kafka connection
+ *
+ * @param name - The connection name (must be valid identifier)
+ * @param options - Kafka connection configuration
+ * @returns A connection definition that can be used in a project
+ *
+ * @example
+ * ```ts
+ * import { createKafkaConnection } from '@tinybirdco/sdk';
+ *
+ * export const myKafka = createKafkaConnection('my_kafka', {
+ *   bootstrapServers: 'kafka.example.com:9092',
+ *   securityProtocol: 'SASL_SSL',
+ *   saslMechanism: 'PLAIN',
+ *   key: '{{ tb_secret("KAFKA_KEY") }}',
+ *   secret: '{{ tb_secret("KAFKA_SECRET") }}',
+ * });
+ * ```
+ */
+export function createKafkaConnection(
+  name: string,
+  options: KafkaConnectionOptions
+): KafkaConnectionDefinition {
+  // Validate name is a valid identifier
+  if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) {
+    throw new Error(
+      `Invalid connection name: "${name}". Must start with a letter or underscore and contain only alphanumeric characters and underscores.`
+    );
+  }
+
+  return {
+    [CONNECTION_BRAND]: true,
+    _name: name,
+    _type: "connection",
+    _connectionType: "kafka",
+    options,
+  };
+}
+
+/**
+ * Check if a value is a connection definition
+ */
+export function isConnectionDefinition(value: unknown): value is ConnectionDefinition {
+  return (
+    typeof value === "object" &&
+    value !== null &&
+    CONNECTION_BRAND in value &&
+    (value as Record<symbol, unknown>)[CONNECTION_BRAND] === true
+  );
+}
+
+/**
+ * Check if a value is a Kafka connection definition
+ */
+export function isKafkaConnectionDefinition(value: unknown): value is KafkaConnectionDefinition {
+  return isConnectionDefinition(value) && value._connectionType === "kafka";
+}
+
+/**
+ * Get the connection type from a connection definition
+ */
+export function getConnectionType<T extends ConnectionDefinition>(
+  connection: T
+): T["_connectionType"] {
+  return connection._connectionType;
+}

package/src/schema/datasource.ts

@@ -5,6 +5,7 @@
 
 import type { AnyTypeValidator } from "./types.js";
 import type { EngineConfig } from "./engines.js";
+import type { KafkaConnectionDefinition } from "./connection.js";
 
 // Symbol for brand typing
 const DATASOURCE_BRAND = Symbol("tinybird.datasource");
@@ -35,6 +36,20 @@ export interface TokenConfig {
   permissions: readonly ("READ" | "APPEND")[];
 }
 
+/**
+ * Kafka ingestion configuration for a datasource
+ */
+export interface KafkaConfig {
+  /** Kafka connection to use */
+  connection: KafkaConnectionDefinition;
+  /** Kafka topic to consume from */
+  topic: string;
+  /** Consumer group ID (optional) */
+  groupId?: string;
+  /** Where to start reading: 'earliest' or 'latest' (default: 'latest') */
+  autoOffsetReset?: "earliest" | "latest";
+}
+
 /**
  * Options for defining a datasource
  */
@@ -55,6 +70,8 @@ export interface DatasourceOptions<TSchema extends SchemaDefinition> {
    * Defaults to true.
    */
   jsonPaths?: boolean;
+  /** Kafka ingestion configuration */
+  kafka?: KafkaConfig;
 }
 
 /**

package/src/schema/project.ts

@@ -5,6 +5,7 @@
 
 import type { DatasourceDefinition, SchemaDefinition } from "./datasource.js";
 import type { PipeDefinition, ParamsDefinition, OutputDefinition } from "./pipe.js";
+import type { ConnectionDefinition } from "./connection.js";
 import { getEndpointConfig } from "./pipe.js";
 import type { TinybirdClient } from "../client/base.js";
 import type { QueryResult } from "../client/types.js";
@@ -23,6 +24,11 @@ export type DatasourcesDefinition = Record<string, DatasourceDefinition<SchemaDe
  */
 export type PipesDefinition = Record<string, PipeDefinition<ParamsDefinition, OutputDefinition>>;
 
+/**
+ * Collection of connection definitions
+ */
+export type ConnectionsDefinition = Record<string, ConnectionDefinition>;
+
 /**
  * Type for a single query method
  */
@@ -101,12 +107,15 @@ export interface TinybirdClientConfig<
  */
 export interface ProjectConfig<
   TDatasources extends DatasourcesDefinition = DatasourcesDefinition,
-  TPipes extends PipesDefinition = PipesDefinition
+  TPipes extends PipesDefinition = PipesDefinition,
+  TConnections extends ConnectionsDefinition = ConnectionsDefinition
 > {
   /** All datasources in this project */
   datasources?: TDatasources;
   /** All pipes in this project */
   pipes?: TPipes;
+  /** All connections in this project */
+  connections?: TConnections;
 }
 
 /**
@@ -114,7 +123,8 @@ export interface ProjectConfig<
  */
 export interface ProjectDefinition<
   TDatasources extends DatasourcesDefinition = DatasourcesDefinition,
-  TPipes extends PipesDefinition = PipesDefinition
+  TPipes extends PipesDefinition = PipesDefinition,
+  TConnections extends ConnectionsDefinition = ConnectionsDefinition
 > {
   readonly [PROJECT_BRAND]: true;
   /** Type marker for inference */
@@ -123,6 +133,8 @@ export interface ProjectDefinition<
   readonly datasources: TDatasources;
   /** All pipes */
   readonly pipes: TPipes;
+  /** All connections */
+  readonly connections: TConnections;
   /** Typed Tinybird client */
   readonly tinybird: ProjectClient<TDatasources, TPipes>;
 }
@@ -157,12 +169,14 @@ export interface ProjectDefinition<
  */
 export function defineProject<
   TDatasources extends DatasourcesDefinition,
-  TPipes extends PipesDefinition
+  TPipes extends PipesDefinition,
+  TConnections extends ConnectionsDefinition
 >(
-  config: ProjectConfig<TDatasources, TPipes>
-): ProjectDefinition<TDatasources, TPipes> {
+  config: ProjectConfig<TDatasources, TPipes, TConnections>
+): ProjectDefinition<TDatasources, TPipes, TConnections> {
   const datasources = (config.datasources ?? {}) as TDatasources;
   const pipes = (config.pipes ?? {}) as TPipes;
+  const connections = (config.connections ?? {}) as TConnections;
 
   // Use the shared client builder
   const tinybird = buildProjectClient(datasources, pipes);
@@ -172,6 +186,7 @@ export function defineProject<
     _type: "project",
     datasources,
     pipes,
+    connections,
     tinybird,
   };
 }