@tinybirdco/sdk 0.0.48 → 0.0.49

package/src/generator/pipe.test.ts

@@ -1,7 +1,8 @@
 import { describe, it, expect } from 'vitest';
 import { generatePipe, generateAllPipes } from './pipe.js';
-import { definePipe, defineMaterializedView, node } from '../schema/pipe.js';
+import { definePipe, defineMaterializedView, defineSinkPipe, node } from '../schema/pipe.js';
 import { defineDatasource } from '../schema/datasource.js';
+import { defineKafkaConnection, defineS3Connection } from '../schema/connection.js';
 import { defineToken } from '../schema/token.js';
 import { t } from '../schema/types.js';
 import { p } from '../schema/params.js';
@@ -472,6 +473,60 @@ GROUP BY day, country
     });
   });
 
+  describe('Sink configuration', () => {
+    it('generates Kafka sink directives', () => {
+      const kafka = defineKafkaConnection('events_kafka', {
+        bootstrapServers: 'localhost:9092',
+      });
+
+      const pipe = defineSinkPipe('events_sink', {
+        nodes: [node({ name: 'publish', sql: 'SELECT * FROM events' })],
+        sink: {
+          connection: kafka,
+          topic: 'events_out',
+          schedule: '@on-demand',
+        },
+      });
+
+      const result = generatePipe(pipe);
+      expect(result.content).toContain('TYPE sink');
+      expect(result.content).toContain('EXPORT_CONNECTION_NAME events_kafka');
+      expect(result.content).toContain('EXPORT_KAFKA_TOPIC events_out');
+      expect(result.content).toContain('EXPORT_SCHEDULE @on-demand');
+      expect(result.content).not.toContain('EXPORT_STRATEGY');
+    });
+
+    it('generates S3 sink directives', () => {
+      const s3 = defineS3Connection('exports_s3', {
+        region: 'us-east-1',
+        arn: 'arn:aws:iam::123456789012:role/tinybird-s3-access',
+      });
+
+      const pipe = defineSinkPipe('events_s3_sink', {
+        nodes: [node({ name: 'export', sql: 'SELECT * FROM events' })],
+        sink: {
+          connection: s3,
+          bucketUri: 's3://bucket/events/',
+          fileTemplate: 'events_{date}',
+          format: 'csv',
+          schedule: '@once',
+          compression: 'gzip',
+          strategy: 'replace',
+        },
+      });
+
+      const result = generatePipe(pipe);
+      expect(result.content).toContain('TYPE sink');
+      expect(result.content).toContain('EXPORT_CONNECTION_NAME exports_s3');
+      expect(result.content).toContain('EXPORT_BUCKET_URI s3://bucket/events/');
+      expect(result.content).toContain('EXPORT_FILE_TEMPLATE events_{date}');
+      expect(result.content).toContain('EXPORT_FORMAT csv');
+      expect(result.content).toContain('EXPORT_SCHEDULE @once');
+      expect(result.content).toContain('EXPORT_STRATEGY replace');
+      expect(result.content).toContain('EXPORT_COMPRESSION gzip');
+    });
+  });
+
   describe('Token generation', () => {
     it('generates TOKEN lines with inline config', () => {
       const pipe = definePipe('test_pipe', {

package/src/generator/pipe.ts

@@ -9,9 +9,15 @@ import type {
   EndpointConfig,
   MaterializedConfig,
   CopyConfig,
+  SinkConfig,
   PipeTokenConfig,
 } from "../schema/pipe.js";
-import { getEndpointConfig, getMaterializedConfig, getCopyConfig } from "../schema/pipe.js";
+import {
+  getEndpointConfig,
+  getMaterializedConfig,
+  getCopyConfig,
+  getSinkConfig,
+} from "../schema/pipe.js";
 
 /**
  * Generated pipe content
@@ -114,6 +120,33 @@ function generateCopy(config: CopyConfig): string {
   return parts.join("\n");
 }
 
+/**
+ * Generate the TYPE sink section
+ */
+function generateSink(config: SinkConfig): string {
+  const parts: string[] = ["TYPE sink"];
+
+  parts.push(`EXPORT_CONNECTION_NAME ${config.connection._name}`);
+
+  if ("topic" in config) {
+    parts.push(`EXPORT_KAFKA_TOPIC ${config.topic}`);
+    parts.push(`EXPORT_SCHEDULE ${config.schedule}`);
+  } else {
+    parts.push(`EXPORT_BUCKET_URI ${config.bucketUri}`);
+    parts.push(`EXPORT_FILE_TEMPLATE ${config.fileTemplate}`);
+    parts.push(`EXPORT_SCHEDULE ${config.schedule}`);
+    parts.push(`EXPORT_FORMAT ${config.format}`);
+    if (config.strategy) {
+      parts.push(`EXPORT_STRATEGY ${config.strategy}`);
+    }
+    if (config.compression) {
+      parts.push(`EXPORT_COMPRESSION ${config.compression}`);
+    }
+  }
+
+  return parts.join("\n");
+}
+
 /**
  * Generate TOKEN lines for a pipe
  */
@@ -220,6 +253,13 @@ export function generatePipe(pipe: PipeDefinition): GeneratedPipe {
     parts.push(generateCopy(copyConfig));
   }
 
+  // Add sink configuration if this is a sink pipe
+  const sinkConfig = getSinkConfig(pipe);
+  if (sinkConfig) {
+    parts.push("");
+    parts.push(generateSink(sinkConfig));
+  }
+
   // Add tokens if present
   const tokenLines = generateTokens(pipe.options.tokens);
   if (tokenLines.length > 0) {

package/src/index.ts

@@ -149,14 +149,17 @@ export {
   defineEndpoint,
   defineMaterializedView,
   defineCopyPipe,
+  defineSinkPipe,
   node,
   isPipeDefinition,
   isNodeDefinition,
   getEndpointConfig,
   getMaterializedConfig,
   getCopyConfig,
+  getSinkConfig,
   isMaterializedView,
   isCopyPipe,
+  isSinkPipe,
   getNodeNames,
   getNode,
   sql,
@@ -166,7 +169,13 @@ export type {
   PipeOptions,
   EndpointOptions,
   CopyPipeOptions,
+  SinkPipeOptions,
   CopyConfig,
+  SinkConfig,
+  SinkStrategy,
+  SinkCompression,
+  KafkaSinkConfig,
+  S3SinkConfig,
   NodeDefinition,
   NodeOptions,
   ParamsDefinition,

package/src/migrate/emit-ts.ts

@@ -3,6 +3,7 @@ import { toCamelCase } from "../codegen/utils.js";
 import { parseLiteralFromDatafile, toTsLiteral } from "./parser-utils.js";
 import type {
   DatasourceModel,
+  DatasourceEngineModel,
   KafkaConnectionModel,
   ParsedResource,
   PipeModel,
@@ -151,7 +152,7 @@ function strictParamBaseValidator(type: string): string {
 function applyParamOptional(
   baseValidator: string,
   required: boolean,
-  defaultValue: string | number | undefined
+  defaultValue: string | number | boolean | undefined
 ): string {
   const withDefault = defaultValue !== undefined;
   if (!withDefault && required) {
@@ -168,6 +169,13 @@ function applyParamOptional(
   return `${baseValidator}${optionalSuffix}`;
 }
 
+function applyParamDescription(validator: string, description: string | undefined): string {
+  if (description === undefined) {
+    return validator;
+  }
+  return `${validator}.describe(${JSON.stringify(description)})`;
+}
+
 function engineFunctionName(type: string): string {
   const map: Record<string, string> = {
     MergeTree: "mergeTree",
@@ -184,9 +192,8 @@ function engineFunctionName(type: string): string {
   return functionName;
 }
 
-function emitEngineOptions(ds: DatasourceModel): string {
+function emitEngineOptions(engine: DatasourceEngineModel): string {
   const options: string[] = [];
-  const { engine } = ds;
 
   if (engine.sortingKey.length === 1) {
     options.push(`sortingKey: ${escapeString(engine.sortingKey[0]!)}`);
@@ -256,7 +263,7 @@ function emitDatasource(ds: DatasourceModel): string {
   }
 
   lines.push(`export const ${variableName} = defineDatasource(${escapeString(ds.name)}, {`);
-  if (ds.description) {
+  if (ds.description !== undefined) {
     lines.push(`  description: ${emitStringOrSecret(ds.description)},`);
   }
   if (!hasJsonPath) {
@@ -295,7 +302,9 @@ function emitDatasource(ds: DatasourceModel): string {
     lines.push(`    ${columnKey}: ${validator},`);
   }
   lines.push("  },");
-  lines.push(`  engine: ${emitEngineOptions(ds)},`);
+  if (ds.engine) {
+    lines.push(`  engine: ${emitEngineOptions(ds.engine)},`);
+  }
 
   if (ds.kafka) {
     const connectionVar = toCamelCase(ds.kafka.connectionName);
@@ -423,24 +432,27 @@ function emitPipe(pipe: PipeModel): string {
     lines.push(`export const ${variableName} = defineMaterializedView(${escapeString(pipe.name)}, {`);
   } else if (pipe.type === "copy") {
     lines.push(`export const ${variableName} = defineCopyPipe(${escapeString(pipe.name)}, {`);
+  } else if (pipe.type === "sink") {
+    lines.push(`export const ${variableName} = defineSinkPipe(${escapeString(pipe.name)}, {`);
   } else {
     lines.push(`export const ${variableName} = definePipe(${escapeString(pipe.name)}, {`);
   }
 
-  if (pipe.description) {
+  if (pipe.description !== undefined) {
     lines.push(`  description: ${escapeString(pipe.description)},`);
   }
 
-  if (pipe.type === "pipe" || pipe.type === "endpoint") {
+  if (pipe.type === "pipe" || pipe.type === "endpoint" || pipe.type === "sink") {
     if (pipe.params.length > 0) {
       lines.push("  params: {");
       for (const param of pipe.params) {
         const baseValidator = strictParamBaseValidator(param.type);
-        const validator = applyParamOptional(
+        const validatorWithOptional = applyParamOptional(
           baseValidator,
           param.required,
           param.defaultValue
         );
+        const validator = applyParamDescription(validatorWithOptional, param.description);
         lines.push(`    ${emitObjectKey(param.name)}: ${validator},`);
       }
       lines.push("  },");
@@ -464,11 +476,35 @@ function emitPipe(pipe: PipeModel): string {
     }
   }
 
+  if (pipe.type === "sink") {
+    if (!pipe.sink) {
+      throw new Error(`Sink pipe "${pipe.name}" is missing sink configuration.`);
+    }
+    lines.push("  sink: {");
+    lines.push(`    connection: ${toCamelCase(pipe.sink.connectionName)},`);
+    if (pipe.sink.service === "kafka") {
+      lines.push(`    topic: ${escapeString(pipe.sink.topic)},`);
+      lines.push(`    schedule: ${escapeString(pipe.sink.schedule)},`);
+    } else {
+      lines.push(`    bucketUri: ${escapeString(pipe.sink.bucketUri)},`);
+      lines.push(`    fileTemplate: ${escapeString(pipe.sink.fileTemplate)},`);
+      lines.push(`    schedule: ${escapeString(pipe.sink.schedule)},`);
+      lines.push(`    format: ${escapeString(pipe.sink.format)},`);
+      if (pipe.sink.strategy) {
+        lines.push(`    strategy: ${escapeString(pipe.sink.strategy)},`);
+      }
+      if (pipe.sink.compression) {
+        lines.push(`    compression: ${escapeString(pipe.sink.compression)},`);
+      }
+    }
+    lines.push("  },");
+  }
+
   lines.push("  nodes: [");
   for (const node of pipe.nodes) {
     lines.push("    node({");
     lines.push(`      name: ${escapeString(node.name)},`);
-    if (node.description) {
+    if (node.description !== undefined) {
       lines.push(`      description: ${escapeString(node.description)},`);
     }
     lines.push("      sql: `");
@@ -526,7 +562,6 @@ export function emitMigrationFileContent(resources: ParsedResource[]): string {
     "defineCopyPipe",
     "node",
     "t",
-    "engine",
   ]);
   if (connections.some((connection) => connection.connectionType === "kafka")) {
     imports.add("defineKafkaConnection");
@@ -537,6 +572,12 @@ export function emitMigrationFileContent(resources: ParsedResource[]): string {
   if (needsParams) {
     imports.add("p");
   }
+  if (pipes.some((pipe) => pipe.type === "sink")) {
+    imports.add("defineSinkPipe");
+  }
+  if (datasources.some((datasource) => datasource.engine !== undefined)) {
+    imports.add("engine");
+  }
   if (needsSecret) {
     imports.add("secret");
   }
@@ -548,6 +589,7 @@ export function emitMigrationFileContent(resources: ParsedResource[]): string {
     "definePipe",
     "defineMaterializedView",
     "defineCopyPipe",
+    "defineSinkPipe",
     "node",
     "t",
     "engine",

package/src/migrate/parse-datasource.ts

@@ -4,45 +4,44 @@ import {
   isBlank,
   parseDirectiveLine,
   parseQuotedValue,
+  readDirectiveBlock,
   splitCommaSeparated,
   splitLines,
   splitTopLevelComma,
-  stripIndent,
 } from "./parser-utils.js";
 
-interface BlockReadResult {
-  lines: string[];
-  nextIndex: number;
-}
-
-function readIndentedBlock(lines: string[], startIndex: number): BlockReadResult {
-  const collected: string[] = [];
-  let i = startIndex;
-
-  while (i < lines.length) {
-    const line = lines[i] ?? "";
-    if (line.startsWith("    ")) {
-      collected.push(stripIndent(line));
-      i += 1;
-      continue;
-    }
-
-    if (isBlank(line)) {
-      let j = i + 1;
-      while (j < lines.length && isBlank(lines[j] ?? "")) {
-        j += 1;
-      }
-      if (j < lines.length && (lines[j] ?? "").startsWith("    ")) {
-        collected.push("");
-        i += 1;
-        continue;
-      }
-    }
-
-    break;
+const DATASOURCE_DIRECTIVES = new Set([
+  "DESCRIPTION",
+  "SCHEMA",
+  "FORWARD_QUERY",
+  "SHARED_WITH",
+  "ENGINE",
+  "ENGINE_SORTING_KEY",
+  "ENGINE_PARTITION_KEY",
+  "ENGINE_PRIMARY_KEY",
+  "ENGINE_TTL",
+  "ENGINE_VER",
+  "ENGINE_SIGN",
+  "ENGINE_VERSION",
+  "ENGINE_SUMMING_COLUMNS",
+  "ENGINE_SETTINGS",
+  "KAFKA_CONNECTION_NAME",
+  "KAFKA_TOPIC",
+  "KAFKA_GROUP_ID",
+  "KAFKA_AUTO_OFFSET_RESET",
+  "IMPORT_CONNECTION_NAME",
+  "IMPORT_BUCKET_URI",
+  "IMPORT_SCHEDULE",
+  "IMPORT_FROM_TIMESTAMP",
+  "TOKEN",
+]);
+
+function isDatasourceDirectiveLine(line: string): boolean {
+  if (!line) {
+    return false;
   }
-
-  return { lines: collected, nextIndex: i };
+  const { key } = parseDirectiveLine(line);
+  return DATASOURCE_DIRECTIVES.has(key);
 }
 
 function findTokenOutsideContexts(input: string, token: string): number {
@@ -197,7 +196,15 @@ function parseEngineSettings(value: string): Record<string, string | number | bo
 }
 
 function parseToken(filePath: string, resourceName: string, value: string): DatasourceTokenModel {
-  const parts = value.split(/\s+/).filter(Boolean);
+  const trimmed = value.trim();
+  const quotedMatch = trimmed.match(/^"([^"]+)"\s+(READ|APPEND)$/);
+  if (quotedMatch) {
+    const name = quotedMatch[1];
+    const scope = quotedMatch[2] as "READ" | "APPEND";
+    return { name, scope };
+  }
+
+  const parts = trimmed.split(/\s+/).filter(Boolean);
   if (parts.length < 2) {
     throw new MigrationParseError(
       filePath,
@@ -216,7 +223,11 @@ function parseToken(filePath: string, resourceName: string, value: string): Data
     );
   }
 
-  const name = parts[0];
+  const rawName = parts[0] ?? "";
+  const name =
+    rawName.startsWith('"') && rawName.endsWith('"') && rawName.length >= 2
+      ? rawName.slice(1, -1)
+      : rawName;
   const scope = parts[1];
   if (scope !== "READ" && scope !== "APPEND") {
     throw new MigrationParseError(
@@ -271,22 +282,14 @@ export function parseDatasourceFile(resource: ResourceFile): DatasourceModel {
     }
 
     if (line === "DESCRIPTION >") {
-      const block = readIndentedBlock(lines, i + 1);
-      if (block.lines.length === 0) {
-        throw new MigrationParseError(
-          resource.filePath,
-          "datasource",
-          resource.name,
-          "DESCRIPTION block is empty."
-        );
-      }
+      const block = readDirectiveBlock(lines, i + 1, isDatasourceDirectiveLine);
       description = block.lines.join("\n");
       i = block.nextIndex;
       continue;
     }
 
     if (line === "SCHEMA >") {
-      const block = readIndentedBlock(lines, i + 1);
+      const block = readDirectiveBlock(lines, i + 1, isDatasourceDirectiveLine);
       if (block.lines.length === 0) {
         throw new MigrationParseError(
           resource.filePath,
@@ -306,7 +309,7 @@ export function parseDatasourceFile(resource: ResourceFile): DatasourceModel {
     }
 
     if (line === "FORWARD_QUERY >") {
-      const block = readIndentedBlock(lines, i + 1);
+      const block = readDirectiveBlock(lines, i + 1, isDatasourceDirectiveLine);
       if (block.lines.length === 0) {
         throw new MigrationParseError(
           resource.filePath,
@@ -321,7 +324,7 @@ export function parseDatasourceFile(resource: ResourceFile): DatasourceModel {
     }
 
     if (line === "SHARED_WITH >") {
-      const block = readIndentedBlock(lines, i + 1);
+      const block = readDirectiveBlock(lines, i + 1, isDatasourceDirectiveLine);
       for (const sharedLine of block.lines) {
         const normalized = sharedLine.trim().replace(/,$/, "");
         if (normalized) {
@@ -449,16 +452,25 @@ export function parseDatasourceFile(resource: ResourceFile): DatasourceModel {
     );
   }
 
-  if (!engineType) {
-    throw new MigrationParseError(
-      resource.filePath,
-      "datasource",
-      resource.name,
-      "ENGINE directive is required."
-    );
+  const hasEngineDirectives =
+    sortingKey.length > 0 ||
+    partitionKey !== undefined ||
+    (primaryKey !== undefined && primaryKey.length > 0) ||
+    ttl !== undefined ||
+    ver !== undefined ||
+    isDeleted !== undefined ||
+    sign !== undefined ||
+    version !== undefined ||
+    (summingColumns !== undefined && summingColumns.length > 0) ||
+    settings !== undefined;
+
+  if (!engineType && hasEngineDirectives) {
+    // Tinybird defaults to MergeTree when ENGINE is omitted.
+    // If engine-specific options are present, preserve them by inferring MergeTree.
+    engineType = "MergeTree";
   }
 
-  if (sortingKey.length === 0) {
+  if (engineType && sortingKey.length === 0) {
     throw new MigrationParseError(
       resource.filePath,
       "datasource",
@@ -525,19 +537,21 @@ export function parseDatasourceFile(resource: ResourceFile): DatasourceModel {
     filePath: resource.filePath,
     description,
     columns,
-    engine: {
-      type: engineType,
-      sortingKey,
-      partitionKey,
-      primaryKey,
-      ttl,
-      ver,
-      isDeleted,
-      sign,
-      version,
-      summingColumns,
-      settings,
-    },
+    engine: engineType
+      ? {
+          type: engineType,
+          sortingKey,
+          partitionKey,
+          primaryKey,
+          ttl,
+          ver,
+          isDeleted,
+          sign,
+          version,
+          summingColumns,
+          settings,
+        }
+      : undefined,
     kafka,
     s3,
     forwardQuery,