@tinybirdco/sdk 0.0.47 → 0.0.49

package/src/migrate/parse-pipe.ts

@@ -3,49 +3,40 @@ import {
   MigrationParseError,
   isBlank,
   parseDirectiveLine,
+  parseQuotedValue,
+  readDirectiveBlock,
   splitLines,
   splitTopLevelComma,
-  stripIndent,
 } from "./parser-utils.js";
 
-interface BlockReadResult {
-  lines: string[];
-  nextIndex: number;
-}
-
-function readIndentedBlock(lines: string[], startIndex: number): BlockReadResult {
-  const collected: string[] = [];
-  let i = startIndex;
-
-  while (i < lines.length) {
-    const line = lines[i] ?? "";
-    if (line.startsWith("    ")) {
-      collected.push(stripIndent(line));
-      i += 1;
-      continue;
-    }
-
-    if (isBlank(line)) {
-      let j = i + 1;
-      while (j < lines.length && isBlank(lines[j] ?? "")) {
-        j += 1;
-      }
-      if (j < lines.length && (lines[j] ?? "").startsWith("    ")) {
-        collected.push("");
-        i += 1;
-        continue;
-      }
-    }
-
-    break;
+const PIPE_DIRECTIVES = new Set([
+  "DESCRIPTION",
+  "NODE",
+  "SQL",
+  "TYPE",
+  "CACHE",
+  "DATASOURCE",
+  "DEPLOYMENT_METHOD",
+  "TARGET_DATASOURCE",
+  "COPY_SCHEDULE",
+  "COPY_MODE",
+  "TOKEN",
+]);
+
+function isPipeDirectiveLine(line: string): boolean {
+  if (!line) {
+    return false;
   }
-
-  return { lines: collected, nextIndex: i };
+  const { key } = parseDirectiveLine(line);
+  return PIPE_DIRECTIVES.has(key);
 }
 
 function nextNonBlank(lines: string[], startIndex: number): number {
   let i = startIndex;
-  while (i < lines.length && isBlank(lines[i] ?? "")) {
+  while (
+    i < lines.length &&
+    (isBlank(lines[i] ?? "") || (lines[i] ?? "").trim().startsWith("#"))
+  ) {
     i += 1;
   }
   return i;
@@ -114,11 +105,14 @@ function mapTemplateFunctionToParamType(func: string): string | null {
   return null;
 }
 
-function parseParamDefault(rawValue: string): string | number {
+function parseParamDefault(rawValue: string): string | number | boolean {
   const trimmed = rawValue.trim();
   if (/^-?\d+(\.\d+)?$/.test(trimmed)) {
     return Number(trimmed);
   }
+  if (/^(true|false)$/i.test(trimmed)) {
+    return trimmed.toLowerCase() === "true";
+  }
   if (
     (trimmed.startsWith("'") && trimmed.endsWith("'")) ||
     (trimmed.startsWith('"') && trimmed.endsWith('"'))
@@ -128,6 +122,92 @@ function parseParamDefault(rawValue: string): string | number {
   throw new Error(`Unsupported parameter default value: "${rawValue}"`);
 }
 
+function parseKeywordArgument(rawArg: string): { key: string; value: string } | null {
+  const equalsIndex = rawArg.indexOf("=");
+  if (equalsIndex <= 0) {
+    return null;
+  }
+
+  const key = rawArg.slice(0, equalsIndex).trim();
+  if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(key)) {
+    return null;
+  }
+
+  const value = rawArg.slice(equalsIndex + 1).trim();
+  if (!value) {
+    return null;
+  }
+
+  return { key, value };
+}
+
+function parseRequiredFlag(rawValue: string): boolean {
+  const normalized = rawValue.trim().toLowerCase();
+  if (normalized === "true" || normalized === "1") {
+    return true;
+  }
+  if (normalized === "false" || normalized === "0") {
+    return false;
+  }
+  throw new Error(`Unsupported required value: "${rawValue}"`);
+}
+
+function parseParamOptions(rawArgs: string[]): {
+  defaultValue?: string | number | boolean;
+  required?: boolean;
+  description?: string;
+} {
+  let positionalDefault: string | number | boolean | undefined;
+  let keywordDefault: string | number | boolean | undefined;
+  let required: boolean | undefined;
+  let description: string | undefined;
+
+  for (const rawArg of rawArgs) {
+    const trimmed = rawArg.trim();
+    if (!trimmed) {
+      continue;
+    }
+
+    const keyword = parseKeywordArgument(trimmed);
+    if (!keyword) {
+      if (positionalDefault === undefined) {
+        positionalDefault = parseParamDefault(trimmed);
+      }
+      continue;
+    }
+
+    const keyLower = keyword.key.toLowerCase();
+    if (keyLower === "default") {
+      keywordDefault = parseParamDefault(keyword.value);
+      continue;
+    }
+    if (keyLower === "required") {
+      required = parseRequiredFlag(keyword.value);
+      continue;
+    }
+    if (keyLower === "description") {
+      const parsedDescription = parseParamDefault(keyword.value);
+      if (typeof parsedDescription !== "string") {
+        throw new Error(`Unsupported description value: "${keyword.value}"`);
+      }
+      description = parsedDescription;
+      continue;
+    }
+  }
+
+  let defaultValue = keywordDefault ?? positionalDefault;
+  if (keywordDefault !== undefined && positionalDefault !== undefined) {
+    if (keywordDefault !== positionalDefault) {
+      throw new Error(
+        `Parameter has conflicting positional and keyword defaults: "${positionalDefault}" and "${keywordDefault}".`
+      );
+    }
+    defaultValue = positionalDefault;
+  }
+
+  return { defaultValue, required, description };
+}
+
 function inferParamsFromSql(
   sql: string,
   filePath: string,
@@ -170,10 +250,15 @@ function inferParamsFromSql(
       );
     }
 
-    let defaultValue: string | number | undefined;
+    let defaultValue: string | number | boolean | undefined;
+    let required: boolean | undefined;
+    let description: string | undefined;
     if (args.length > 1) {
       try {
-        defaultValue = parseParamDefault(args[1] ?? "");
+        const parsedOptions = parseParamOptions(args.slice(1));
+        defaultValue = parsedOptions.defaultValue;
+        required = parsedOptions.required;
+        description = parsedOptions.description;
       } catch (error) {
         throw new MigrationParseError(
           filePath,
@@ -206,14 +291,21 @@ function inferParamsFromSql(
       }
       if (existing.defaultValue === undefined && defaultValue !== undefined) {
         existing.defaultValue = defaultValue;
-        existing.required = false;
       }
+      if (existing.description === undefined && description !== undefined) {
+        existing.description = description;
+      }
+      const optionalInAnyUsage =
+        existing.required === false || required === false || defaultValue !== undefined;
+      existing.required = !optionalInAnyUsage;
     } else {
+      const isRequired = required ?? defaultValue === undefined;
       params.set(paramName, {
         name: paramName,
         type: mappedType,
-        required: defaultValue === undefined,
+        required: isRequired,
         defaultValue,
+        description,
       });
     }
 
@@ -224,7 +316,13 @@ function inferParamsFromSql(
 }
 
 function parseToken(filePath: string, resourceName: string, value: string): PipeTokenModel {
-  const parts = value.split(/\s+/).filter(Boolean);
+  const trimmed = value.trim();
+  const quotedMatch = trimmed.match(/^"([^"]+)"(?:\s+(READ))?$/);
+  if (quotedMatch) {
+    return { name: quotedMatch[1] ?? "", scope: "READ" };
+  }
+
+  const parts = trimmed.split(/\s+/).filter(Boolean);
   if (parts.length === 0) {
     throw new MigrationParseError(filePath, "pipe", resourceName, "Invalid TOKEN line.");
   }
@@ -237,7 +335,13 @@ function parseToken(filePath: string, resourceName: string, value: string): Pipe
     );
   }
 
-  const tokenName = parts[0];
+  const rawTokenName = parts[0] ?? "";
+  const tokenName =
+    rawTokenName.startsWith('"') &&
+    rawTokenName.endsWith('"') &&
+    rawTokenName.length >= 2
+      ? rawTokenName.slice(1, -1)
+      : rawTokenName;
   const scope = parts[1] ?? "READ";
   if (scope !== "READ") {
     throw new MigrationParseError(
@@ -263,27 +367,27 @@ export function parsePipeFile(resource: ResourceFile): PipeModel {
   let copyTargetDatasource: string | undefined;
   let copySchedule: string | undefined;
   let copyMode: "append" | "replace" | undefined;
+  let exportService: "kafka" | "s3" | undefined;
+  let exportConnectionName: string | undefined;
+  let exportTopic: string | undefined;
+  let exportBucketUri: string | undefined;
+  let exportFileTemplate: string | undefined;
+  let exportFormat: string | undefined;
+  let exportSchedule: string | undefined;
+  let exportStrategy: "create_new" | "replace" | undefined;
+  let exportCompression: "none" | "gzip" | "snappy" | undefined;
 
   let i = 0;
   while (i < lines.length) {
     const line = (lines[i] ?? "").trim();
-    if (!line) {
+    if (!line || line.startsWith("#")) {
       i += 1;
       continue;
     }
 
     if (line === "DESCRIPTION >") {
-      const block = readIndentedBlock(lines, i + 1);
-      if (block.lines.length === 0) {
-        throw new MigrationParseError(
-          resource.filePath,
-          "pipe",
-          resource.name,
-          "DESCRIPTION block is empty."
-        );
-      }
-
-      if (!description) {
+      const block = readDirectiveBlock(lines, i + 1, isPipeDirectiveLine);
+      if (description === undefined) {
         description = block.lines.join("\n");
       } else if (nodes.length > 0) {
         nodes[nodes.length - 1] = {
@@ -318,15 +422,7 @@ export function parsePipeFile(resource: ResourceFile): PipeModel {
 
       let nodeDescription: string | undefined;
       if ((lines[i] ?? "").trim() === "DESCRIPTION >") {
-        const descriptionBlock = readIndentedBlock(lines, i + 1);
-        if (descriptionBlock.lines.length === 0) {
-          throw new MigrationParseError(
-            resource.filePath,
-            "pipe",
-            resource.name,
-            `Node "${nodeName}" has an empty DESCRIPTION block.`
-          );
-        }
+        const descriptionBlock = readDirectiveBlock(lines, i + 1, isPipeDirectiveLine);
         nodeDescription = descriptionBlock.lines.join("\n");
         i = descriptionBlock.nextIndex;
         i = nextNonBlank(lines, i);
@@ -340,7 +436,7 @@ export function parsePipeFile(resource: ResourceFile): PipeModel {
           `Node "${nodeName}" is missing SQL > block.`
         );
       }
-      const sqlBlock = readIndentedBlock(lines, i + 1);
+      const sqlBlock = readDirectiveBlock(lines, i + 1, isPipeDirectiveLine);
       if (sqlBlock.lines.length === 0) {
         throw new MigrationParseError(
           resource.filePath,
@@ -374,22 +470,26 @@ export function parsePipeFile(resource: ResourceFile): PipeModel {
 
     const { key, value } = parseDirectiveLine(line);
     switch (key) {
-      case "TYPE":
-        if (value === "endpoint") {
+      case "TYPE": {
+        const normalizedType = parseQuotedValue(value).toLowerCase();
+        if (normalizedType === "endpoint") {
           pipeType = "endpoint";
-        } else if (value === "MATERIALIZED") {
+        } else if (normalizedType === "materialized") {
           pipeType = "materialized";
-        } else if (value === "COPY") {
+        } else if (normalizedType === "copy") {
           pipeType = "copy";
+        } else if (normalizedType === "sink") {
+          pipeType = "sink";
         } else {
           throw new MigrationParseError(
             resource.filePath,
             "pipe",
             resource.name,
-            `Unsupported TYPE value in strict mode: "${value}"`
+            `Unsupported TYPE value in strict mode: "${parseQuotedValue(value)}"`
           );
         }
         break;
+      }
       case "CACHE": {
         const ttl = Number(value);
         if (!Number.isFinite(ttl) || ttl < 0) {
@@ -434,6 +534,63 @@ export function parsePipeFile(resource: ResourceFile): PipeModel {
         }
         copyMode = value;
         break;
+      case "EXPORT_SERVICE": {
+        const normalized = parseQuotedValue(value).toLowerCase();
+        if (normalized !== "kafka" && normalized !== "s3") {
+          throw new MigrationParseError(
+            resource.filePath,
+            "pipe",
+            resource.name,
+            `Unsupported EXPORT_SERVICE in strict mode: "${value}"`
+          );
+        }
+        exportService = normalized;
+        break;
+      }
+      case "EXPORT_CONNECTION_NAME":
+        exportConnectionName = parseQuotedValue(value);
+        break;
+      case "EXPORT_KAFKA_TOPIC":
+        exportTopic = parseQuotedValue(value);
+        break;
+      case "EXPORT_BUCKET_URI":
+        exportBucketUri = parseQuotedValue(value);
+        break;
+      case "EXPORT_FILE_TEMPLATE":
+        exportFileTemplate = parseQuotedValue(value);
+        break;
+      case "EXPORT_FORMAT":
+        exportFormat = parseQuotedValue(value);
+        break;
+      case "EXPORT_SCHEDULE":
+        exportSchedule = parseQuotedValue(value);
+        break;
+      case "EXPORT_STRATEGY": {
+        const normalized = parseQuotedValue(value).toLowerCase();
+        if (normalized !== "create_new" && normalized !== "replace") {
+          throw new MigrationParseError(
+            resource.filePath,
+            "pipe",
+            resource.name,
+            `Unsupported EXPORT_STRATEGY in strict mode: "${value}"`
+          );
+        }
+        exportStrategy = normalized;
+        break;
+      }
+      case "EXPORT_COMPRESSION": {
+        const normalized = parseQuotedValue(value).toLowerCase();
+        if (normalized !== "none" && normalized !== "gzip" && normalized !== "snappy") {
+          throw new MigrationParseError(
+            resource.filePath,
+            "pipe",
+            resource.name,
+            `Unsupported EXPORT_COMPRESSION in strict mode: "${value}"`
+          );
+        }
+        exportCompression = normalized;
+        break;
+      }
       case "TOKEN":
         tokens.push(parseToken(resource.filePath, resource.name, value));
         break;
@@ -485,6 +642,144 @@ export function parsePipeFile(resource: ResourceFile): PipeModel {
     );
   }
 
+  const hasSinkDirectives =
+    exportService !== undefined ||
+    exportConnectionName !== undefined ||
+    exportTopic !== undefined ||
+    exportBucketUri !== undefined ||
+    exportFileTemplate !== undefined ||
+    exportFormat !== undefined ||
+    exportSchedule !== undefined ||
+    exportStrategy !== undefined ||
+    exportCompression !== undefined;
+
+  if (pipeType !== "sink" && hasSinkDirectives) {
+    throw new MigrationParseError(
+      resource.filePath,
+      "pipe",
+      resource.name,
+      "EXPORT_* directives are only supported for TYPE sink."
+    );
+  }
+
+  let sink: PipeModel["sink"];
+  if (pipeType === "sink") {
+    if (!exportConnectionName) {
+      throw new MigrationParseError(
+        resource.filePath,
+        "pipe",
+        resource.name,
+        "EXPORT_CONNECTION_NAME is required for TYPE sink."
+      );
+    }
+
+    const hasKafkaDirectives = exportTopic !== undefined;
+    const hasS3Directives =
+      exportBucketUri !== undefined ||
+      exportFileTemplate !== undefined ||
+      exportFormat !== undefined ||
+      exportCompression !== undefined;
+
+    if (hasKafkaDirectives && hasS3Directives) {
+      throw new MigrationParseError(
+        resource.filePath,
+        "pipe",
+        resource.name,
+        "Sink pipe cannot mix Kafka and S3 export directives."
+      );
+    }
+
+    const inferredService =
+      exportService ?? (hasKafkaDirectives ? "kafka" : hasS3Directives ? "s3" : undefined);
+
+    if (!inferredService) {
+      throw new MigrationParseError(
+        resource.filePath,
+        "pipe",
+        resource.name,
+        "Sink pipe must define EXPORT_SERVICE or include service-specific export directives."
+      );
+    }
+
+    if (inferredService === "kafka") {
+      if (hasS3Directives) {
+        throw new MigrationParseError(
+          resource.filePath,
+          "pipe",
+          resource.name,
+          "S3 export directives are not valid for Kafka sinks."
+        );
+      }
+      if (!exportTopic) {
+        throw new MigrationParseError(
+          resource.filePath,
+          "pipe",
+          resource.name,
+          "EXPORT_KAFKA_TOPIC is required for Kafka sinks."
+        );
+      }
+      if (!exportSchedule) {
+        throw new MigrationParseError(
+          resource.filePath,
+          "pipe",
+          resource.name,
+          "EXPORT_SCHEDULE is required for Kafka sinks."
+        );
+      }
+      if (exportStrategy !== undefined) {
+        throw new MigrationParseError(
+          resource.filePath,
+          "pipe",
+          resource.name,
+          "EXPORT_STRATEGY is only valid for S3 sinks."
+        );
+      }
+      if (exportCompression !== undefined) {
+        throw new MigrationParseError(
+          resource.filePath,
+          "pipe",
+          resource.name,
+          "EXPORT_COMPRESSION is only valid for S3 sinks."
+        );
+      }
+
+      sink = {
+        service: "kafka",
+        connectionName: exportConnectionName,
+        topic: exportTopic,
+        schedule: exportSchedule,
+      };
+    } else {
+      if (hasKafkaDirectives) {
+        throw new MigrationParseError(
+          resource.filePath,
+          "pipe",
+          resource.name,
+          "Kafka export directives are not valid for S3 sinks."
+        );
+      }
+      if (!exportBucketUri || !exportFileTemplate || !exportFormat || !exportSchedule) {
+        throw new MigrationParseError(
+          resource.filePath,
+          "pipe",
+          resource.name,
+          "S3 sinks require EXPORT_BUCKET_URI, EXPORT_FILE_TEMPLATE, EXPORT_FORMAT, and EXPORT_SCHEDULE."
+        );
+      }
+
+      sink = {
+        service: "s3",
+        connectionName: exportConnectionName,
+        bucketUri: exportBucketUri,
+        fileTemplate: exportFileTemplate,
+        format: exportFormat,
+        schedule: exportSchedule,
+        strategy: exportStrategy,
+        compression: exportCompression,
+      };
+    }
+  }
+
   const params =
     pipeType === "materialized" || pipeType === "copy"
       ? []
@@ -510,9 +805,9 @@ export function parsePipeFile(resource: ResourceFile): PipeModel {
     copyTargetDatasource,
     copySchedule,
     copyMode,
+    sink,
     tokens,
     params,
     inferredOutputColumns,
   };
 }
-

package/src/migrate/parser-utils.ts

@@ -27,6 +27,42 @@ export function stripIndent(line: string): string {
   return line.trimStart();
 }
 
+export interface BlockReadResult {
+  lines: string[];
+  nextIndex: number;
+}
+
+export function readDirectiveBlock(
+  lines: string[],
+  startIndex: number,
+  isDirectiveLine: (line: string) => boolean
+): BlockReadResult {
+  const collected: string[] = [];
+  let i = startIndex;
+
+  while (i < lines.length) {
+    const line = (lines[i] ?? "").trim();
+    if (isDirectiveLine(line)) {
+      break;
+    }
+    collected.push(line);
+    i += 1;
+  }
+
+  let first = 0;
+  while (first < collected.length && collected[first] === "") {
+    first += 1;
+  }
+
+  let last = collected.length - 1;
+  while (last >= first && collected[last] === "") {
+    last -= 1;
+  }
+
+  const normalized = first <= last ? collected.slice(first, last + 1) : [];
+  return { lines: normalized, nextIndex: i };
+}
+
 export function splitCommaSeparated(input: string): string[] {
   return input
     .split(",")
@@ -157,4 +193,3 @@ export function splitTopLevelComma(input: string): string[] {
 
   return parts;
 }
-

package/src/migrate/types.ts

@@ -30,6 +30,7 @@ export interface DatasourceEngineModel {
   primaryKey?: string[];
   ttl?: string;
   ver?: string;
+  isDeleted?: string;
   sign?: string;
   version?: string;
   summingColumns?: string[];
@@ -41,6 +42,7 @@ export interface DatasourceKafkaModel {
   topic: string;
   groupId?: string;
   autoOffsetReset?: "earliest" | "latest";
+  storeRawValue?: boolean;
 }
 
 export interface DatasourceS3Model {
@@ -61,7 +63,7 @@ export interface DatasourceModel {
   filePath: string;
   description?: string;
   columns: DatasourceColumnModel[];
-  engine: DatasourceEngineModel;
+  engine?: DatasourceEngineModel;
   kafka?: DatasourceKafkaModel;
   s3?: DatasourceS3Model;
   forwardQuery?: string;
@@ -80,13 +82,34 @@ export interface PipeTokenModel {
   scope: "READ";
 }
 
-export type PipeTypeModel = "pipe" | "endpoint" | "materialized" | "copy";
+export type PipeTypeModel = "pipe" | "endpoint" | "materialized" | "copy" | "sink";
+
+export interface PipeKafkaSinkModel {
+  service: "kafka";
+  connectionName: string;
+  topic: string;
+  schedule: string;
+}
+
+export interface PipeS3SinkModel {
+  service: "s3";
+  connectionName: string;
+  bucketUri: string;
+  fileTemplate: string;
+  format: string;
+  schedule: string;
+  strategy?: "create_new" | "replace";
+  compression?: "none" | "gzip" | "snappy";
+}
+
+export type PipeSinkModel = PipeKafkaSinkModel | PipeS3SinkModel;
 
 export interface PipeParamModel {
   name: string;
   type: string;
   required: boolean;
-  defaultValue?: string | number;
+  defaultValue?: string | number | boolean;
+  description?: string;
 }
 
 export interface PipeModel {
@@ -102,6 +125,7 @@ export interface PipeModel {
   copyTargetDatasource?: string;
   copySchedule?: string;
   copyMode?: "append" | "replace";
+  sink?: PipeSinkModel;
   tokens: PipeTokenModel[];
   params: PipeParamModel[];
   inferredOutputColumns: string[];
@@ -117,6 +141,7 @@ export interface KafkaConnectionModel {
   saslMechanism?: "PLAIN" | "SCRAM-SHA-256" | "SCRAM-SHA-512" | "OAUTHBEARER";
   key?: string;
   secret?: string;
+  schemaRegistryUrl?: string;
   sslCaPem?: string;
 }
 

package/src/schema/connection.ts

@@ -31,6 +31,8 @@ export interface KafkaConnectionOptions {
   key?: string;
   /** Kafka secret/password - can use {{ tb_secret(...) }} */
   secret?: string;
+  /** Schema Registry URL (optionally with embedded auth credentials) */
+  schemaRegistryUrl?: string;
   /** SSL CA certificate PEM - for private CA certs */
   sslCaPem?: string;
 }