@tinybirdco/sdk 0.0.50 → 0.0.52

package/src/migrate/parse-pipe.ts

@@ -70,35 +70,41 @@ function inferOutputColumnsFromSql(sql: string): string[] {
 }
 
 function mapTemplateFunctionToParamType(func: string): string | null {
-  const known = new Set([
-    "String",
-    "UUID",
-    "Int8",
-    "Int16",
-    "Int32",
-    "Int64",
-    "UInt8",
-    "UInt16",
-    "UInt32",
-    "UInt64",
-    "Float32",
-    "Float64",
-    "Boolean",
-    "Bool",
-    "Date",
-    "DateTime",
-    "DateTime64",
-    "Array",
-  ]);
-
-  if (known.has(func)) {
-    return func;
+  const lower = func.toLowerCase();
+  const aliases: Record<string, string> = {
+    string: "String",
+    uuid: "UUID",
+    int: "Int32",
+    integer: "Int32",
+    int8: "Int8",
+    int16: "Int16",
+    int32: "Int32",
+    int64: "Int64",
+    uint8: "UInt8",
+    uint16: "UInt16",
+    uint32: "UInt32",
+    uint64: "UInt64",
+    float32: "Float32",
+    float64: "Float64",
+    boolean: "Boolean",
+    bool: "Boolean",
+    date: "Date",
+    datetime: "DateTime",
+    datetime64: "DateTime64",
+    array: "Array",
+    column: "column",
+    json: "JSON",
+  };
+
+  const mapped = aliases[lower];
+  if (mapped) {
+    return mapped;
   }
 
-  if (func.startsWith("DateTime64")) {
+  if (lower.startsWith("datetime64")) {
     return "DateTime64";
   }
-  if (func.startsWith("DateTime")) {
+  if (lower.startsWith("datetime")) {
     return "DateTime";
   }
 
@@ -157,8 +163,7 @@ function parseParamOptions(rawArgs: string[]): {
   required?: boolean;
   description?: string;
 } {
-  let positionalDefault: string | number | boolean | undefined;
-  let keywordDefault: string | number | boolean | undefined;
+  let defaultValue: string | number | boolean | undefined;
   let required: boolean | undefined;
   let description: string | undefined;
 
@@ -170,15 +175,13 @@ function parseParamOptions(rawArgs: string[]): {
 
     const keyword = parseKeywordArgument(trimmed);
     if (!keyword) {
-      if (positionalDefault === undefined) {
-        positionalDefault = parseParamDefault(trimmed);
-      }
+      defaultValue = parseParamDefault(trimmed);
       continue;
     }
 
     const keyLower = keyword.key.toLowerCase();
     if (keyLower === "default") {
-      keywordDefault = parseParamDefault(keyword.value);
+      defaultValue = parseParamDefault(keyword.value);
       continue;
     }
     if (keyLower === "required") {
@@ -195,17 +198,122 @@ function parseParamOptions(rawArgs: string[]): {
     }
   }
 
-  let defaultValue = keywordDefault ?? positionalDefault;
-  if (keywordDefault !== undefined && positionalDefault !== undefined) {
-    if (keywordDefault !== positionalDefault) {
-      throw new Error(
-        `Parameter has conflicting positional and keyword defaults: "${positionalDefault}" and "${keywordDefault}".`
-      );
+  return { defaultValue, required, description };
+}
+
+function extractTemplateFunctionCalls(expression: string): Array<{
+  functionName: string;
+  argsRaw: string;
+  fullCall: string;
+  start: number;
+  end: number;
+}> {
+  const maskParenthesesInsideQuotes = (value: string): string => {
+    let output = "";
+    let inSingleQuote = false;
+    let inDoubleQuote = false;
+
+    for (let i = 0; i < value.length; i += 1) {
+      const char = value[i] ?? "";
+      const prev = i > 0 ? value[i - 1] ?? "" : "";
+
+      if (char === "'" && !inDoubleQuote && prev !== "\\") {
+        inSingleQuote = !inSingleQuote;
+        output += char;
+        continue;
+      }
+      if (char === '"' && !inSingleQuote && prev !== "\\") {
+        inDoubleQuote = !inDoubleQuote;
+        output += char;
+        continue;
+      }
+
+      if ((inSingleQuote || inDoubleQuote) && (char === "(" || char === ")")) {
+        output += " ";
+        continue;
+      }
+
+      output += char;
     }
-    defaultValue = positionalDefault;
+
+    return output;
+  };
+
+  const maskedExpression = maskParenthesesInsideQuotes(expression);
+  const callRegex = /([a-zA-Z_][a-zA-Z0-9_]*)\s*\(([^()]*)\)/g;
+  const calls: Array<{
+    functionName: string;
+    argsRaw: string;
+    fullCall: string;
+    start: number;
+    end: number;
+  }> = [];
+  let match: RegExpExecArray | null = callRegex.exec(maskedExpression);
+  while (match) {
+    const start = match.index;
+    const fullCall = expression.slice(start, start + (match[0]?.length ?? 0));
+    const openParen = fullCall.indexOf("(");
+    const closeParen = fullCall.lastIndexOf(")");
+
+    calls.push({
+      functionName: match[1] ?? "",
+      argsRaw: openParen >= 0 && closeParen > openParen ? fullCall.slice(openParen + 1, closeParen) : "",
+      fullCall,
+      start,
+      end: start + fullCall.length,
+    });
+    match = callRegex.exec(maskedExpression);
   }
+  return calls;
+}
 
-  return { defaultValue, required, description };
+function shouldParseTemplateFunctionAsParam(mappedType: string): boolean {
+  return mappedType !== "Array";
+}
+
+function normalizeSqlPlaceholders(sql: string): string {
+  const placeholderRegex = /\{\{\s*([^{}]+?)\s*\}\}/g;
+  return sql.replace(placeholderRegex, (fullMatch, rawExpression) => {
+    const expression = String(rawExpression);
+    const calls = extractTemplateFunctionCalls(expression);
+    if (calls.length === 0) {
+      return fullMatch;
+    }
+
+    let rewritten = "";
+    let cursor = 0;
+    let changed = false;
+    for (const call of calls) {
+      rewritten += expression.slice(cursor, call.start);
+
+      let replacement = call.fullCall;
+      const normalizedFunction = String(call.functionName).toLowerCase();
+      if (normalizedFunction !== "error" && normalizedFunction !== "custom_error") {
+        const mappedType = mapTemplateFunctionToParamType(String(call.functionName));
+        if (mappedType && shouldParseTemplateFunctionAsParam(mappedType)) {
+          const args = splitTopLevelComma(String(call.argsRaw));
+          if (args.length > 0) {
+            const paramName = args[0]?.trim() ?? "";
+            if (/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(paramName)) {
+              replacement = `${String(call.functionName)}(${paramName})`;
+            }
+          }
+        }
+      }
+
+      if (replacement !== call.fullCall) {
+        changed = true;
+      }
+      rewritten += replacement;
+      cursor = call.end;
+    }
+    rewritten += expression.slice(cursor);
+
+    if (!changed) {
+      return fullMatch;
+    }
+    return `{{ ${rewritten.trim()} }}`;
+  });
 }
 
 function inferParamsFromSql(
@@ -213,100 +321,105 @@ function inferParamsFromSql(
   filePath: string,
   resourceName: string
 ): PipeParamModel[] {
-  const regex = /\{\{\s*([a-zA-Z_][a-zA-Z0-9_]*)\(([^{}]*)\)\s*\}\}/g;
+  const regex = /\{\{\s*([^{}]+?)\s*\}\}/g;
   const params = new Map<string, PipeParamModel>();
   let match: RegExpExecArray | null = regex.exec(sql);
 
   while (match) {
-    const templateFunction = match[1] ?? "";
-    const argsRaw = match[2] ?? "";
-    const args = splitTopLevelComma(argsRaw);
-    if (args.length === 0) {
-      throw new MigrationParseError(
-        filePath,
-        "pipe",
-        resourceName,
-        `Invalid template placeholder: "${match[0]}"`
-      );
-    }
-
-    const paramName = args[0]?.trim();
-    if (!paramName || !/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(paramName)) {
-      throw new MigrationParseError(
-        filePath,
-        "pipe",
-        resourceName,
-        `Unsupported parameter name in placeholder: "${match[0]}"`
-      );
-    }
+    const expression = match[1] ?? "";
+    const calls = extractTemplateFunctionCalls(expression);
+
+    for (const call of calls) {
+      const templateFunction = call.functionName;
+      const normalizedTemplateFunction = templateFunction.toLowerCase();
+      if (normalizedTemplateFunction === "error" || normalizedTemplateFunction === "custom_error") {
+        continue;
+      }
 
-    const mappedType = mapTemplateFunctionToParamType(templateFunction);
-    if (!mappedType) {
-      throw new MigrationParseError(
-        filePath,
-        "pipe",
-        resourceName,
-        `Unsupported placeholder function in strict mode: "${templateFunction}"`
-      );
-    }
+      const mappedType = mapTemplateFunctionToParamType(templateFunction);
+      if (!mappedType) {
+        throw new MigrationParseError(
+          filePath,
+          "pipe",
+          resourceName,
+          `Unsupported placeholder function in strict mode: "${templateFunction}"`
+        );
+      }
 
-    let defaultValue: string | number | boolean | undefined;
-    let required: boolean | undefined;
-    let description: string | undefined;
-    if (args.length > 1) {
-      try {
-        const parsedOptions = parseParamOptions(args.slice(1));
-        defaultValue = parsedOptions.defaultValue;
-        required = parsedOptions.required;
-        description = parsedOptions.description;
-      } catch (error) {
+      const args = splitTopLevelComma(call.argsRaw);
+      if (args.length === 0) {
         throw new MigrationParseError(
           filePath,
           "pipe",
           resourceName,
-          (error as Error).message
+          `Invalid template placeholder: "${call.fullCall}"`
         );
       }
-    }
 
-    const existing = params.get(paramName);
-    if (existing) {
-      if (existing.type !== mappedType) {
+      const paramName = args[0]?.trim() ?? "";
+      const isIdentifier = /^[a-zA-Z_][a-zA-Z0-9_]*$/.test(paramName);
+      if (!isIdentifier) {
+        if (mappedType === "column") {
+          continue;
+        }
         throw new MigrationParseError(
           filePath,
           "pipe",
           resourceName,
-          `Parameter "${paramName}" is used with multiple types: "${existing.type}" and "${mappedType}".`
+          `Unsupported parameter name in placeholder: "{{ ${call.fullCall} }}"`
         );
       }
-      if (existing.defaultValue !== undefined && defaultValue !== undefined) {
-        if (existing.defaultValue !== defaultValue) {
+
+      let defaultValue: string | number | boolean | undefined;
+      let required: boolean | undefined;
+      let description: string | undefined;
+      if (args.length > 1 && shouldParseTemplateFunctionAsParam(mappedType)) {
+        try {
+          const parsedOptions = parseParamOptions(args.slice(1));
+          defaultValue = parsedOptions.defaultValue;
+          required = parsedOptions.required;
+          description = parsedOptions.description;
+        } catch (error) {
           throw new MigrationParseError(
             filePath,
             "pipe",
             resourceName,
-            `Parameter "${paramName}" uses multiple defaults: "${existing.defaultValue}" and "${defaultValue}".`
+            (error as Error).message
           );
         }
       }
-      if (existing.defaultValue === undefined && defaultValue !== undefined) {
-        existing.defaultValue = defaultValue;
-      }
-      if (existing.description === undefined && description !== undefined) {
-        existing.description = description;
+
+      const existing = params.get(paramName);
+      if (existing) {
+        if (existing.type !== mappedType) {
+          // Keep the last explicit type seen in SQL.
+          existing.type = mappedType;
+        }
+
+        // Match backend merge semantics: prefer the latest truthy value.
+        if (defaultValue !== undefined || existing.defaultValue !== undefined) {
+          existing.defaultValue =
+            (defaultValue as string | number | boolean | undefined) || existing.defaultValue;
+        }
+        if (description !== undefined || existing.description !== undefined) {
+          existing.description = description || existing.description;
+        }
+        const optionalInAnyUsage =
+          existing.required === false ||
+          required === false ||
+          existing.defaultValue !== undefined ||
+          defaultValue !== undefined;
+        existing.required = !optionalInAnyUsage;
+      } else {
+        const isRequired = required ?? defaultValue === undefined;
+        params.set(paramName, {
+          name: paramName,
+          type: mappedType,
+          required: isRequired,
+          defaultValue,
+          description,
+        });
       }
-      const optionalInAnyUsage =
-        existing.required === false || required === false || defaultValue !== undefined;
-      existing.required = !optionalInAnyUsage;
-    } else {
-      const isRequired = required ?? defaultValue === undefined;
-      params.set(paramName, {
-        name: paramName,
-        type: mappedType,
-        required: isRequired,
-        defaultValue,
-        description,
-      });
     }
 
     match = regex.exec(sql);
@@ -355,9 +468,21 @@ function parseToken(filePath: string, resourceName: string, value: string): Pipe
   return { name: tokenName, scope: "READ" };
 }
 
+function normalizeExportStrategy(rawValue: string): "create_new" | "replace" {
+  const normalized = parseQuotedValue(rawValue).toLowerCase();
+  if (normalized === "create_new") {
+    return "create_new";
+  }
+  if (normalized === "replace" || normalized === "truncate") {
+    return "replace";
+  }
+  throw new Error(`Unsupported sink strategy in strict mode: "${rawValue}"`);
+}
+
 export function parsePipeFile(resource: ResourceFile): PipeModel {
   const lines = splitLines(resource.content);
   const nodes: PipeModel["nodes"] = [];
+  const rawNodeSqls: string[] = [];
   const tokens: PipeTokenModel[] = [];
   let description: string | undefined;
   let pipeType: PipeModel["type"] = "pipe";
@@ -458,10 +583,11 @@ export function parsePipeFile(resource: ResourceFile): PipeModel {
         );
       }
 
+      rawNodeSqls.push(sql);
       nodes.push({
         name: nodeName,
         description: nodeDescription,
-        sql,
+        sql: normalizeSqlPlaceholders(sql),
       });
 
       i = sqlBlock.nextIndex;
@@ -566,8 +692,9 @@ export function parsePipeFile(resource: ResourceFile): PipeModel {
         exportSchedule = parseQuotedValue(value);
         break;
       case "EXPORT_STRATEGY": {
-        const normalized = parseQuotedValue(value).toLowerCase();
-        if (normalized !== "create_new" && normalized !== "replace") {
+        try {
+          exportStrategy = normalizeExportStrategy(value);
+        } catch {
           throw new MigrationParseError(
             resource.filePath,
             "pipe",
@@ -575,7 +702,19 @@ export function parsePipeFile(resource: ResourceFile): PipeModel {
             `Unsupported EXPORT_STRATEGY in strict mode: "${value}"`
           );
         }
-        exportStrategy = normalized;
+        break;
+      }
+      case "EXPORT_WRITE_STRATEGY": {
+        try {
+          exportStrategy = normalizeExportStrategy(value);
+        } catch {
+          throw new MigrationParseError(
+            resource.filePath,
+            "pipe",
+            resource.name,
+            `Unsupported EXPORT_WRITE_STRATEGY in strict mode: "${value}"`
+          );
+        }
         break;
       }
       case "EXPORT_COMPRESSION": {
@@ -784,7 +923,7 @@ export function parsePipeFile(resource: ResourceFile): PipeModel {
     pipeType === "materialized" || pipeType === "copy"
       ? []
       : inferParamsFromSql(
-          nodes.map((node) => node.sql).join("\n"),
+          rawNodeSqls.join("\n"),
           resource.filePath,
           resource.name
         );

package/src/migrate/parser-utils.ts

@@ -72,7 +72,11 @@ export function splitCommaSeparated(input: string): string[] {
 
 export function parseQuotedValue(input: string): string {
   const trimmed = input.trim();
-  if (trimmed.startsWith('"') && trimmed.endsWith('"') && trimmed.length >= 2) {
+  if (
+    trimmed.length >= 2 &&
+    ((trimmed.startsWith('"') && trimmed.endsWith('"')) ||
+      (trimmed.startsWith("'") && trimmed.endsWith("'")))
+  ) {
     return trimmed.slice(1, -1);
   }
   return trimmed;

package/src/migrate/types.ts

@@ -64,6 +64,13 @@ export interface DatasourceTokenModel {
   scope: "READ" | "APPEND";
 }
 
+export interface DatasourceIndexModel {
+  name: string;
+  expr: string;
+  type: string;
+  granularity: number;
+}
+
 export interface DatasourceModel {
   kind: "datasource";
   name: string;
@@ -71,6 +78,7 @@ export interface DatasourceModel {
   description?: string;
   columns: DatasourceColumnModel[];
   engine?: DatasourceEngineModel;
+  indexes: DatasourceIndexModel[];
   kafka?: DatasourceKafkaModel;
   s3?: DatasourceS3Model;
   gcs?: DatasourceGCSModel;

package/src/schema/datasource.test.ts

@@ -146,6 +146,59 @@ describe("Datasource Schema", () => {
       expect(ds.options.gcs?.connection._name).toBe("my_gcs");
       expect(ds.options.gcs?.bucketUri).toBe("gs://my-bucket/events/*.csv");
     });
+
+    it("accepts datasource indexes", () => {
+      const ds = defineDatasource("events", {
+        schema: { id: t.string() },
+        indexes: [
+          {
+            name: "id_set",
+            expr: "id",
+            type: "set(100)",
+            granularity: 1,
+          },
+        ],
+      });
+
+      expect(ds.options.indexes).toEqual([
+        {
+          name: "id_set",
+          expr: "id",
+          type: "set(100)",
+          granularity: 1,
+        },
+      ]);
+    });
+
+    it("validates datasource index fields", () => {
+      expect(() =>
+        defineDatasource("events", {
+          schema: { id: t.string() },
+          indexes: [{ name: "invalid name", expr: "id", type: "set(100)", granularity: 1 }],
+        })
+      ).toThrow("Invalid datasource index name");
+
+      expect(() =>
+        defineDatasource("events", {
+          schema: { id: t.string() },
+          indexes: [{ name: "id_set", expr: "", type: "set(100)", granularity: 1 }],
+        })
+      ).toThrow('Invalid datasource index "id_set": expr is required.');
+
+      expect(() =>
+        defineDatasource("events", {
+          schema: { id: t.string() },
+          indexes: [{ name: "id_set", expr: "id", type: "", granularity: 1 }],
+        })
+      ).toThrow('Invalid datasource index "id_set": type is required.');
+
+      expect(() =>
+        defineDatasource("events", {
+          schema: { id: t.string() },
+          indexes: [{ name: "id_set", expr: "id", type: "set(100)", granularity: 0 }],
+        })
+      ).toThrow('Invalid datasource index "id_set": granularity must be a positive integer.');
+    });
   });
 
   describe("isDatasourceDefinition", () => {

package/src/schema/datasource.ts

@@ -102,6 +102,21 @@ export interface GCSConfig {
   fromTimestamp?: string;
 }
 
+/**
+ * Datasource index configuration.
+ * Emits as: `<name> <expr> TYPE <type> GRANULARITY <n>`
+ */
+export interface DatasourceIndex {
+  /** Index name */
+  name: string;
+  /** Index expression */
+  expr: string;
+  /** Index type and parameters (for example: `set(100)`) */
+  type: string;
+  /** Index granularity */
+  granularity: number;
+}
+
 /**
  * Options for defining a datasource
  */
@@ -127,6 +142,8 @@ export interface DatasourceOptions<TSchema extends SchemaDefinition> {
    * This should be the SELECT clause only (no FROM/WHERE).
    */
   forwardQuery?: string;
+  /** Secondary indexes for MergeTree-family engines */
+  indexes?: readonly DatasourceIndex[];
   /** Kafka ingestion configuration */
   kafka?: KafkaConfig;
   /** S3 ingestion configuration */
@@ -195,6 +212,27 @@ export function defineDatasource<TSchema extends SchemaDefinition>(
     throw new Error("Datasource can only define one ingestion option: `kafka`, `s3`, or `gcs`.");
   }
 
+  if (options.indexes) {
+    for (const index of options.indexes) {
+      if (!index.name || /\s/.test(index.name)) {
+        throw new Error(
+          `Invalid datasource index name: "${index.name}". Index names must be non-empty and cannot contain whitespace.`
+        );
+      }
+      if (!index.expr?.trim()) {
+        throw new Error(`Invalid datasource index "${index.name}": expr is required.`);
+      }
+      if (!index.type?.trim()) {
+        throw new Error(`Invalid datasource index "${index.name}": type is required.`);
+      }
+      if (!Number.isInteger(index.granularity) || index.granularity <= 0) {
+        throw new Error(
+          `Invalid datasource index "${index.name}": granularity must be a positive integer.`
+        );
+      }
+    }
+  }
+
   return {
     [DATASOURCE_BRAND]: true,
     _name: name,