@tinybirdco/sdk 0.0.47 → 0.0.49

package/src/cli/commands/migrate.ts

@@ -112,6 +112,9 @@ export async function runMigrate(
   const migrated: ParsedResource[] = [];
   const migratedConnectionNames = new Set<string>();
   const migratedDatasourceNames = new Set<string>();
+  const parsedConnectionTypeByName = new Map(
+    parsedConnections.map((connection) => [connection.name, connection.connectionType] as const)
+  );
 
   for (const connection of parsedConnections) {
     try {
@@ -160,6 +163,38 @@ export async function runMigrate(
   }
 
   for (const pipe of parsedPipes) {
+    if (pipe.type === "sink") {
+      const sinkConnectionName = pipe.sink?.connectionName;
+      if (!sinkConnectionName || !migratedConnectionNames.has(sinkConnectionName)) {
+        errors.push({
+          filePath: pipe.filePath,
+          resourceName: pipe.name,
+          resourceKind: pipe.kind,
+          message: `Sink pipe references missing/unmigrated connection "${sinkConnectionName ?? "(none)"}".`,
+        });
+        continue;
+      }
+      const sinkConnectionType = parsedConnectionTypeByName.get(sinkConnectionName);
+      if (!sinkConnectionType) {
+        errors.push({
+          filePath: pipe.filePath,
+          resourceName: pipe.name,
+          resourceKind: pipe.kind,
+          message: `Sink pipe connection "${sinkConnectionName}" could not be resolved.`,
+        });
+        continue;
+      }
+      if (sinkConnectionType !== pipe.sink?.service) {
+        errors.push({
+          filePath: pipe.filePath,
+          resourceName: pipe.name,
+          resourceKind: pipe.kind,
+          message: `Sink pipe service "${pipe.sink?.service}" is incompatible with connection "${sinkConnectionName}" type "${sinkConnectionType}".`,
+        });
+        continue;
+      }
+    }
+
     if (
       pipe.type === "materialized" &&
       (!pipe.materializedDatasource ||

package/src/generator/connection.test.ts

@@ -51,6 +51,19 @@ describe("Connection Generator", () => {
       expect(result.content).toContain('KAFKA_SECRET {{ tb_secret("KAFKA_SECRET") }}');
     });
 
+    it("includes schema registry URL when provided", () => {
+      const conn = defineKafkaConnection("my_kafka", {
+        bootstrapServers: "kafka.example.com:9092",
+        schemaRegistryUrl: "https://registry-user:registry-pass@registry.example.com",
+      });
+
+      const result = generateConnection(conn);
+
+      expect(result.content).toContain(
+        "KAFKA_SCHEMA_REGISTRY_URL https://registry-user:registry-pass@registry.example.com"
+      );
+    });
+
     it("includes SSL CA PEM when provided", () => {
       const conn = defineKafkaConnection("my_kafka", {
         bootstrapServers: "kafka.example.com:9092",

package/src/generator/connection.ts

@@ -42,6 +42,10 @@ function generateKafkaConnection(connection: KafkaConnectionDefinition): string
     parts.push(`KAFKA_SECRET ${options.secret}`);
   }
 
+  if (options.schemaRegistryUrl) {
+    parts.push(`KAFKA_SCHEMA_REGISTRY_URL ${options.schemaRegistryUrl}`);
+  }
+
   if (options.sslCaPem) {
     parts.push(`KAFKA_SSL_CA_PEM ${options.sslCaPem}`);
   }

package/src/generator/datasource.test.ts

@@ -263,6 +263,43 @@ describe('Datasource Generator', () => {
       expect(schemaLines[1]).toContain(',');
       expect(schemaLines[2]).not.toContain(',');
     });
+
+    it('autogenerates jsonPath when jsonPaths is enabled and no explicit path is set', () => {
+      const ds = defineDatasource('test_ds', {
+        schema: {
+          event_id: t.string().nullable(),
+        },
+      });
+
+      const result = generateDatasource(ds);
+      expect(result.content).toContain('event_id Nullable(String) `json:$.event_id`');
+    });
+
+    it('uses explicit jsonPath from validator modifier when jsonPaths is enabled', () => {
+      const ds = defineDatasource('test_ds', {
+        schema: {
+          event_id: t.string().nullable().jsonPath('$.explicit_path'),
+        },
+      });
+
+      const result = generateDatasource(ds);
+      expect(result.content).toContain('event_id Nullable(String) `json:$.explicit_path`');
+      expect(result.content).not.toContain('`json:$.event_id`');
+    });
+
+    it('omits json paths when jsonPaths is false even if column has explicit jsonPath modifier', () => {
+      const ds = defineDatasource('test_ds', {
+        jsonPaths: false,
+        schema: {
+          event_id: t.string().nullable().jsonPath('$.explicit_path'),
+        },
+      });
+
+      const result = generateDatasource(ds);
+      expect(result.content).toContain('event_id Nullable(String)');
+      expect(result.content).not.toContain('`json:$.explicit_path`');
+      expect(result.content).not.toContain('`json:$.event_id`');
+    });
   });
 
   describe('generateAllDatasources', () => {
@@ -380,6 +417,29 @@ describe('Datasource Generator', () => {
       expect(result.content).toContain('KAFKA_AUTO_OFFSET_RESET earliest');
     });
 
+    it('includes store raw value when provided', () => {
+      const kafkaConn = defineKafkaConnection('my_kafka', {
+        bootstrapServers: 'kafka.example.com:9092',
+      });
+
+      const ds = defineDatasource('kafka_events', {
+        schema: {
+          timestamp: t.dateTime(),
+          event: t.string(),
+        },
+        engine: engine.mergeTree({ sortingKey: ['timestamp'] }),
+        kafka: {
+          connection: kafkaConn,
+          topic: 'events',
+          storeRawValue: true,
+        },
+      });
+
+      const result = generateDatasource(ds);
+
+      expect(result.content).toContain('KAFKA_STORE_RAW_VALUE True');
+    });
+
     it('generates complete Kafka datasource with all options', () => {
       const kafkaConn = defineKafkaConnection('my_kafka', {
         bootstrapServers: 'kafka.example.com:9092',

package/src/generator/datasource.ts

@@ -166,6 +166,9 @@ function generateKafkaConfig(kafka: KafkaConfig): string {
   if (kafka.autoOffsetReset) {
     parts.push(`KAFKA_AUTO_OFFSET_RESET ${kafka.autoOffsetReset}`);
   }
+  if (kafka.storeRawValue !== undefined) {
+    parts.push(`KAFKA_STORE_RAW_VALUE ${kafka.storeRawValue ? "True" : "False"}`);
+  }
 
   return parts.join("\n");
 }

package/src/generator/pipe.test.ts

@@ -1,7 +1,8 @@
 import { describe, it, expect } from 'vitest';
 import { generatePipe, generateAllPipes } from './pipe.js';
-import { definePipe, defineMaterializedView, node } from '../schema/pipe.js';
+import { definePipe, defineMaterializedView, defineSinkPipe, node } from '../schema/pipe.js';
 import { defineDatasource } from '../schema/datasource.js';
+import { defineKafkaConnection, defineS3Connection } from '../schema/connection.js';
 import { defineToken } from '../schema/token.js';
 import { t } from '../schema/types.js';
 import { p } from '../schema/params.js';
@@ -472,6 +473,60 @@ GROUP BY day, country
     });
   });
 
+  describe('Sink configuration', () => {
+    it('generates Kafka sink directives', () => {
+      const kafka = defineKafkaConnection('events_kafka', {
+        bootstrapServers: 'localhost:9092',
+      });
+
+      const pipe = defineSinkPipe('events_sink', {
+        nodes: [node({ name: 'publish', sql: 'SELECT * FROM events' })],
+        sink: {
+          connection: kafka,
+          topic: 'events_out',
+          schedule: '@on-demand',
+        },
+      });
+
+      const result = generatePipe(pipe);
+      expect(result.content).toContain('TYPE sink');
+      expect(result.content).toContain('EXPORT_CONNECTION_NAME events_kafka');
+      expect(result.content).toContain('EXPORT_KAFKA_TOPIC events_out');
+      expect(result.content).toContain('EXPORT_SCHEDULE @on-demand');
+      expect(result.content).not.toContain('EXPORT_STRATEGY');
+    });
+
+    it('generates S3 sink directives', () => {
+      const s3 = defineS3Connection('exports_s3', {
+        region: 'us-east-1',
+        arn: 'arn:aws:iam::123456789012:role/tinybird-s3-access',
+      });
+
+      const pipe = defineSinkPipe('events_s3_sink', {
+        nodes: [node({ name: 'export', sql: 'SELECT * FROM events' })],
+        sink: {
+          connection: s3,
+          bucketUri: 's3://bucket/events/',
+          fileTemplate: 'events_{date}',
+          format: 'csv',
+          schedule: '@once',
+          compression: 'gzip',
+          strategy: 'replace',
+        },
+      });
+
+      const result = generatePipe(pipe);
+      expect(result.content).toContain('TYPE sink');
+      expect(result.content).toContain('EXPORT_CONNECTION_NAME exports_s3');
+      expect(result.content).toContain('EXPORT_BUCKET_URI s3://bucket/events/');
+      expect(result.content).toContain('EXPORT_FILE_TEMPLATE events_{date}');
+      expect(result.content).toContain('EXPORT_FORMAT csv');
+      expect(result.content).toContain('EXPORT_SCHEDULE @once');
+      expect(result.content).toContain('EXPORT_STRATEGY replace');
+      expect(result.content).toContain('EXPORT_COMPRESSION gzip');
+    });
+  });
+
   describe('Token generation', () => {
     it('generates TOKEN lines with inline config', () => {
       const pipe = definePipe('test_pipe', {

package/src/generator/pipe.ts

@@ -9,9 +9,15 @@ import type {
   EndpointConfig,
   MaterializedConfig,
   CopyConfig,
+  SinkConfig,
   PipeTokenConfig,
 } from "../schema/pipe.js";
-import { getEndpointConfig, getMaterializedConfig, getCopyConfig } from "../schema/pipe.js";
+import {
+  getEndpointConfig,
+  getMaterializedConfig,
+  getCopyConfig,
+  getSinkConfig,
+} from "../schema/pipe.js";
 
 /**
  * Generated pipe content
@@ -114,6 +120,33 @@ function generateCopy(config: CopyConfig): string {
   return parts.join("\n");
 }
 
+/**
+ * Generate the TYPE sink section
+ */
+function generateSink(config: SinkConfig): string {
+  const parts: string[] = ["TYPE sink"];
+
+  parts.push(`EXPORT_CONNECTION_NAME ${config.connection._name}`);
+
+  if ("topic" in config) {
+    parts.push(`EXPORT_KAFKA_TOPIC ${config.topic}`);
+    parts.push(`EXPORT_SCHEDULE ${config.schedule}`);
+  } else {
+    parts.push(`EXPORT_BUCKET_URI ${config.bucketUri}`);
+    parts.push(`EXPORT_FILE_TEMPLATE ${config.fileTemplate}`);
+    parts.push(`EXPORT_SCHEDULE ${config.schedule}`);
+    parts.push(`EXPORT_FORMAT ${config.format}`);
+    if (config.strategy) {
+      parts.push(`EXPORT_STRATEGY ${config.strategy}`);
+    }
+    if (config.compression) {
+      parts.push(`EXPORT_COMPRESSION ${config.compression}`);
+    }
+  }
+
+  return parts.join("\n");
+}
+
 /**
  * Generate TOKEN lines for a pipe
  */
@@ -220,6 +253,13 @@ export function generatePipe(pipe: PipeDefinition): GeneratedPipe {
     parts.push(generateCopy(copyConfig));
   }
 
+  // Add sink configuration if this is a sink pipe
+  const sinkConfig = getSinkConfig(pipe);
+  if (sinkConfig) {
+    parts.push("");
+    parts.push(generateSink(sinkConfig));
+  }
+
   // Add tokens if present
   const tokenLines = generateTokens(pipe.options.tokens);
   if (tokenLines.length > 0) {

package/src/index.test.ts

@@ -10,4 +10,8 @@ describe("root public exports", () => {
     expect(typeof sdk.defineProject).toBe("function");
     expect(typeof sdk.Tinybird).toBe("function");
   });
+
+  it("exports secret utility", () => {
+    expect(typeof sdk.secret).toBe("function");
+  });
 });

package/src/index.ts

@@ -97,6 +97,9 @@ export type {
   VersionedCollapsingMergeTreeConfig,
 } from "./schema/engines.js";
 
+// ============ Utilities ============
+export { secret } from "./schema/secret.js";
+
 // ============ Datasource ============
 export { defineDatasource, isDatasourceDefinition, column, getColumnType, getColumnJsonPath, getColumnNames } from "./schema/datasource.js";
 export type {
@@ -146,14 +149,17 @@ export {
   defineEndpoint,
   defineMaterializedView,
   defineCopyPipe,
+  defineSinkPipe,
   node,
   isPipeDefinition,
   isNodeDefinition,
   getEndpointConfig,
   getMaterializedConfig,
   getCopyConfig,
+  getSinkConfig,
   isMaterializedView,
   isCopyPipe,
+  isSinkPipe,
   getNodeNames,
   getNode,
   sql,
@@ -163,7 +169,13 @@ export type {
   PipeOptions,
   EndpointOptions,
   CopyPipeOptions,
+  SinkPipeOptions,
   CopyConfig,
+  SinkConfig,
+  SinkStrategy,
+  SinkCompression,
+  KafkaSinkConfig,
+  S3SinkConfig,
   NodeDefinition,
   NodeOptions,
   ParamsDefinition,