@tinybirdco/sdk 0.0.49 → 0.0.51

package/src/cli/commands/migrate.test.ts

@@ -107,7 +107,7 @@ export const eventsEndpoint = definePipe("events_endpoint", {
 SELECT event_id, user_id, payload
 FROM events
 WHERE user_id = {{UInt64(user_id)}}
-AND env = {{String(env, 'prod')}}
+AND env = {{ String(env) }}
       \`,
     }),
     node({
@@ -168,7 +168,7 @@ GROUP BY user_id
       sql: \`
 SELECT user_id, total
 FROM agg
-WHERE total > {{UInt32(min_total, 10)}}
+WHERE total > {{ UInt32(min_total) }}
       \`,
     }),
   ],
@@ -616,6 +616,144 @@ IMPORT_FROM_TIMESTAMP 2024-01-01T00:00:00Z
     expect(output).toContain('fromTimestamp: "2024-01-01T00:00:00Z"');
   });
 
+  it("migrates gcs connection and import datasource directives", async () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-migrate-"));
+    tempDirs.push(tempDir);
+
+    writeFile(
+      tempDir,
+      "gcsample.connection",
+      `TYPE gcs
+GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON {{ tb_secret("GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON") }}
+`
+    );
+
+    writeFile(
+      tempDir,
+      "events_gcs_landing.datasource",
+      `SCHEMA >
+    timestamp DateTime,
+    session_id String
+
+ENGINE "MergeTree"
+ENGINE_SORTING_KEY "timestamp"
+IMPORT_CONNECTION_NAME gcsample
+IMPORT_BUCKET_URI gs://my-gcs-bucket/events/*.csv
+IMPORT_SCHEDULE @auto
+IMPORT_FROM_TIMESTAMP 2024-01-01T00:00:00Z
+`
+    );
+
+    const result = await runMigrate({
+      cwd: tempDir,
+      patterns: ["."],
+      strict: true,
+    });
+
+    expect(result.success).toBe(true);
+    expect(result.errors).toHaveLength(0);
+    expect(result.migrated.filter((resource) => resource.kind === "connection")).toHaveLength(1);
+    expect(result.migrated.filter((resource) => resource.kind === "datasource")).toHaveLength(1);
+
+    const output = fs.readFileSync(result.outputPath, "utf-8");
+    expect(output).toContain("defineGCSConnection");
+    expect(output).toContain('export const gcsample = defineGCSConnection("gcsample", {');
+    expect(output).toContain(
+      'serviceAccountCredentialsJson: secret("GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON"),'
+    );
+    expect(output).toContain("gcs: {");
+    expect(output).toContain("connection: gcsample");
+    expect(output).toContain('bucketUri: "gs://my-gcs-bucket/events/*.csv"');
+    expect(output).toContain('schedule: "@auto"');
+    expect(output).toContain('fromTimestamp: "2024-01-01T00:00:00Z"');
+  });
+
+  it("migrates single-quoted gcs import directives", async () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-migrate-"));
+    tempDirs.push(tempDir);
+
+    writeFile(
+      tempDir,
+      "gcp_billing_reports.connection",
+      `TYPE gcs
+GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON {{tb_secret("GCP_SERVICE_ACCOUNT_CREDENTIALS_JSON")}}
+`
+    );
+
+    writeFile(
+      tempDir,
+      "gcp_production_billing_account_landing.datasource",
+      `SCHEMA >
+    usage_start_time DateTime64(3)
+
+ENGINE "MergeTree"
+ENGINE_SORTING_KEY "usage_start_time"
+IMPORT_CONNECTION_NAME 'gcp_billing_reports'
+IMPORT_BUCKET_URI 'gs://tinybird-oa-sot-fwd/gcp_production_billing_reports/billing_*.csv.gz'
+IMPORT_SCHEDULE '@on-demand'
+`
+    );
+
+    const result = await runMigrate({
+      cwd: tempDir,
+      patterns: ["."],
+      strict: true,
+    });
+
+    expect(result.success).toBe(true);
+    expect(result.errors).toHaveLength(0);
+
+    const output = fs.readFileSync(result.outputPath, "utf-8");
+    expect(output).toContain(
+      'export const gcpBillingReports = defineGCSConnection("gcp_billing_reports", {'
+    );
+    expect(output).toContain("gcs: {");
+    expect(output).toContain("connection: gcpBillingReports");
+    expect(output).toContain(
+      'bucketUri: "gs://tinybird-oa-sot-fwd/gcp_production_billing_reports/billing_*.csv.gz"'
+    );
+    expect(output).toContain('schedule: "@on-demand"');
+  });
+
+  it("reports an error when import directives use a non-bucket connection type", async () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-migrate-"));
+    tempDirs.push(tempDir);
+
+    writeFile(
+      tempDir,
+      "stream.connection",
+      `TYPE kafka
+KAFKA_BOOTSTRAP_SERVERS localhost:9092
+`
+    );
+
+    writeFile(
+      tempDir,
+      "events_landing.datasource",
+      `SCHEMA >
+    id String
+
+ENGINE "MergeTree"
+ENGINE_SORTING_KEY "id"
+IMPORT_CONNECTION_NAME stream
+IMPORT_BUCKET_URI gs://my-gcs-bucket/events/*.csv
+`
+    );
+
+    const result = await runMigrate({
+      cwd: tempDir,
+      patterns: ["."],
+      strict: true,
+    });
+
+    expect(result.success).toBe(false);
+    expect(result.errors.map((error) => error.message)).toEqual(
+      expect.arrayContaining([
+        'Datasource import directives require an s3 or gcs connection, found "kafka".',
+      ])
+    );
+  });
+
 
   it("migrates KAFKA_STORE_RAW_VALUE datasource directive", async () => {
     const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-migrate-"));
@@ -659,6 +797,44 @@ KAFKA_STORE_RAW_VALUE True
     expect(output).toContain("storeRawValue: true");
   });
 
+  it("migrates INDEXES datasource block", async () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-migrate-"));
+    tempDirs.push(tempDir);
+
+    writeFile(
+      tempDir,
+      "events.datasource",
+      `SCHEMA >
+    id String,
+    pipe_name String
+
+ENGINE "MergeTree"
+ENGINE_SORTING_KEY "id"
+INDEXES >
+    pipe_name_set pipe_name TYPE set(100) GRANULARITY 1
+    id_bf lower(id) TYPE bloom_filter(0.001) GRANULARITY 4
+`
+    );
+
+    const result = await runMigrate({
+      cwd: tempDir,
+      patterns: ["."],
+      strict: true,
+    });
+
+    expect(result.success).toBe(true);
+    expect(result.errors).toHaveLength(0);
+
+    const output = fs.readFileSync(result.outputPath, "utf-8");
+    expect(output).toContain("indexes: [");
+    expect(output).toContain(
+      '{ name: "pipe_name_set", expr: "pipe_name", type: "set(100)", granularity: 1 }'
+    );
+    expect(output).toContain(
+      '{ name: "id_bf", expr: "lower(id)", type: "bloom_filter(0.001)", granularity: 4 }'
+    );
+  });
+
   it("migrates kafka schema registry and engine is deleted directives", async () => {
     const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-migrate-"));
     tempDirs.push(tempDir);
@@ -809,6 +985,201 @@ TYPE ENDPOINT
     expect(output).toContain("endpoint: true,");
   });
 
+  it("supports Int/Integer aliases and column placeholder params", async () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-migrate-"));
+    tempDirs.push(tempDir);
+
+    writeFile(
+      tempDir,
+      "placeholder_aliases.pipe",
+      `NODE endpoint
+SQL >
+    SELECT event_id AS event_id
+    FROM events
+    WHERE score >= {{int(min_score, 10)}}
+    ORDER BY {{column(sort_col)}} DESC
+    LIMIT {{ Integer(limit, 50) }}
+TYPE endpoint
+`
+    );
+
+    const result = await runMigrate({
+      cwd: tempDir,
+      patterns: ["."],
+      strict: true,
+    });
+
+    expect(result.success).toBe(true);
+    expect(result.errors).toHaveLength(0);
+
+    const output = fs.readFileSync(result.outputPath, "utf-8");
+    expect(output).toContain('export const placeholderAliases = definePipe("placeholder_aliases", {');
+    expect(output).toContain("params: {");
+    expect(output).toContain("limit: p.int32().optional(50),");
+    expect(output).toContain("min_score: p.int32().optional(10),");
+    expect(output).toContain("sort_col: p.column(),");
+  });
+
+  it("supports error() placeholder helper without treating it as a param", async () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-migrate-"));
+    tempDirs.push(tempDir);
+
+    writeFile(
+      tempDir,
+      "placeholder_error.pipe",
+      `NODE endpoint
+SQL >
+    %
+    {% if not defined(start_date) and not defined(end_date) %}
+        {{ error('start_date and end_date are required parameters') }}
+    {% end %}
+    SELECT 1 AS id
+    WHERE now() >= {{DateTime(start_date, '2025-01-01 00:00:00')}}
+TYPE endpoint
+`
+    );
+
+    const result = await runMigrate({
+      cwd: tempDir,
+      patterns: ["."],
+      strict: true,
+    });
+
+    expect(result.success).toBe(true);
+    expect(result.errors).toHaveLength(0);
+
+    const output = fs.readFileSync(result.outputPath, "utf-8");
+    expect(output).toContain('export const placeholderError = definePipe("placeholder_error", {');
+    expect(output).toContain("params: {");
+    expect(output).toContain('start_date: p.dateTime().optional("2025-01-01 00:00:00"),');
+    expect(output).not.toContain("error:");
+  });
+
+  it("uses the last explicit default and strips inline defaults from SQL", async () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-migrate-"));
+    tempDirs.push(tempDir);
+
+    writeFile(
+      tempDir,
+      "last_default.pipe",
+      `NODE endpoint
+SQL >
+    SELECT 1
+    WHERE d >= {{ String(start_date, '2025-03-01') }}
+      AND d <= {{ String(start_date, '2025-04-01') }}
+TYPE endpoint
+`
+    );
+
+    const result = await runMigrate({
+      cwd: tempDir,
+      patterns: ["."],
+      strict: true,
+    });
+
+    expect(result.success).toBe(true);
+    expect(result.errors).toHaveLength(0);
+
+    const output = fs.readFileSync(result.outputPath, "utf-8");
+    expect(output).toContain('start_date: p.string().optional("2025-04-01"),');
+    expect(output).toContain("{{ String(start_date) }}");
+    expect(output).not.toContain("{{ String(start_date, '2025-03-01') }}");
+    expect(output).not.toContain("{{ String(start_date, '2025-04-01') }}");
+  });
+
+  it("keeps the previous truthy default when a later duplicate uses a falsy default", async () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-migrate-"));
+    tempDirs.push(tempDir);
+
+    writeFile(
+      tempDir,
+      "truthy_default_precedence.pipe",
+      `NODE endpoint
+SQL >
+    SELECT 1
+    WHERE d >= {{ String(start_date, '2025-03-01') }}
+      AND d <= {{ String(start_date, '') }}
+TYPE endpoint
+`
+    );
+
+    const result = await runMigrate({
+      cwd: tempDir,
+      patterns: ["."],
+      strict: true,
+    });
+
+    expect(result.success).toBe(true);
+    expect(result.errors).toHaveLength(0);
+
+    const output = fs.readFileSync(result.outputPath, "utf-8");
+    expect(output).toContain('start_date: p.string().optional("2025-03-01"),');
+    expect(output).toContain("{{ String(start_date) }}");
+  });
+
+  it("extracts params from placeholder expressions with multiple function calls", async () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-migrate-"));
+    tempDirs.push(tempDir);
+
+    writeFile(
+      tempDir,
+      "expression_params.pipe",
+      `NODE endpoint
+SQL >
+    SELECT 1
+    LIMIT {{ Int32(limit, 20) }}
+    OFFSET {{ Int32(page, 0) * Int32(limit, 20) }}
+TYPE endpoint
+`
+    );
+
+    const result = await runMigrate({
+      cwd: tempDir,
+      patterns: ["."],
+      strict: true,
+    });
+
+    expect(result.success).toBe(true);
+    expect(result.errors).toHaveLength(0);
+
+    const output = fs.readFileSync(result.outputPath, "utf-8");
+    expect(output).toContain("limit: p.int32().optional(20),");
+    expect(output).toContain("page: p.int32().optional(0),");
+    expect(output).toContain("{{ Int32(limit) }}");
+    expect(output).toContain("{{ Int32(page) * Int32(limit) }}");
+  });
+
+  it("uses the last explicit type when a param appears with mixed placeholder types", async () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-migrate-"));
+    tempDirs.push(tempDir);
+
+    writeFile(
+      tempDir,
+      "last_type.pipe",
+      `NODE endpoint
+SQL >
+    SELECT 1
+    WHERE d >= {{ Date(start_date) }}
+      AND d <= {{ String(start_date, '2025-04-01') }}
+TYPE endpoint
+`
+    );
+
+    const result = await runMigrate({
+      cwd: tempDir,
+      patterns: ["."],
+      strict: true,
+    });
+
+    expect(result.success).toBe(true);
+    expect(result.errors).toHaveLength(0);
+
+    const output = fs.readFileSync(result.outputPath, "utf-8");
+    expect(output).toContain('start_date: p.string().optional("2025-04-01"),');
+    expect(output).toContain("{{ Date(start_date) }}");
+    expect(output).toContain("{{ String(start_date) }}");
+  });
+
   it("parses multiline datasource blocks with flexible indentation", async () => {
     const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-migrate-"));
     tempDirs.push(tempDir);
@@ -1107,6 +1478,38 @@ TYPE endpoint
     );
   });
 
+  it("ignores function-like text inside quoted descriptions when parsing placeholders", async () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-migrate-"));
+    tempDirs.push(tempDir);
+
+    writeFile(
+      tempDir,
+      "description_parentheses.pipe",
+      `NODE endpoint
+SQL >
+  %
+  SELECT 1 AS id
+  WHERE ts >= now() - interval {{Int32(days, 1, description="Number of days to analyze (defaults to 1 day)")}} day
+TYPE endpoint
+`
+    );
+
+    const result = await runMigrate({
+      cwd: tempDir,
+      patterns: ["."],
+      strict: true,
+    });
+
+    expect(result.success).toBe(true);
+    expect(result.errors).toHaveLength(0);
+
+    const output = fs.readFileSync(result.outputPath, "utf-8");
+    expect(output).toContain(
+      'days: p.int32().optional(1).describe("Number of days to analyze (defaults to 1 day)"),'
+    );
+    expect(output).toContain("{{ Int32(days) }}");
+  });
+
   it("migrates datasource with mixed explicit and default json paths", async () => {
     const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-migrate-"));
     tempDirs.push(tempDir);
@@ -1358,6 +1761,49 @@ EXPORT_COMPRESSION gzip
     expect(output).toContain('compression: "gzip"');
   });
 
+  it("migrates legacy EXPORT_WRITE_STRATEGY sink directives", async () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-migrate-"));
+    tempDirs.push(tempDir);
+
+    writeFile(
+      tempDir,
+      "exports_s3.connection",
+      `TYPE s3
+S3_REGION "us-east-1"
+S3_ARN "arn:aws:iam::123456789012:role/tinybird-s3-access"
+`
+    );
+
+    writeFile(
+      tempDir,
+      "events_s3_sink.pipe",
+      `NODE export
+SQL >
+    SELECT * FROM events
+TYPE sink
+EXPORT_CONNECTION_NAME exports_s3
+EXPORT_BUCKET_URI s3://exports/events/
+EXPORT_FILE_TEMPLATE events_{date}
+EXPORT_SCHEDULE @once
+EXPORT_FORMAT ndjson
+EXPORT_WRITE_STRATEGY truncate
+`
+    );
+
+    const result = await runMigrate({
+      cwd: tempDir,
+      patterns: ["."],
+      strict: true,
+    });
+
+    expect(result.success).toBe(true);
+    expect(result.errors).toHaveLength(0);
+
+    const output = fs.readFileSync(result.outputPath, "utf-8");
+    expect(output).toContain('export const eventsS3Sink = defineSinkPipe("events_s3_sink", {');
+    expect(output).toContain('strategy: "replace"');
+  });
+
   it("reports an error when sink pipe references a missing connection", async () => {
     const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-migrate-"));
     tempDirs.push(tempDir);

package/src/cli/commands/migrate.ts

@@ -133,7 +133,9 @@ export async function runMigrate(
 
   for (const datasource of parsedDatasources) {
     const referencedConnectionName =
-      datasource.kafka?.connectionName ?? datasource.s3?.connectionName;
+      datasource.kafka?.connectionName ??
+      datasource.s3?.connectionName ??
+      datasource.gcs?.connectionName;
 
     if (
       referencedConnectionName &&
@@ -148,6 +150,42 @@ export async function runMigrate(
       continue;
     }
 
+    if (datasource.kafka) {
+      const kafkaConnectionType = parsedConnectionTypeByName.get(datasource.kafka.connectionName);
+      if (kafkaConnectionType !== "kafka") {
+        errors.push({
+          filePath: datasource.filePath,
+          resourceName: datasource.name,
+          resourceKind: datasource.kind,
+          message: `Datasource kafka ingestion requires a kafka connection, found "${kafkaConnectionType ?? "(none)"}".`,
+        });
+        continue;
+      }
+    }
+
+    const importConfig = datasource.s3 ?? datasource.gcs;
+    if (importConfig) {
+      const importConnectionType = parsedConnectionTypeByName.get(importConfig.connectionName);
+      if (importConnectionType !== "s3" && importConnectionType !== "gcs") {
+        errors.push({
+          filePath: datasource.filePath,
+          resourceName: datasource.name,
+          resourceKind: datasource.kind,
+          message:
+            `Datasource import directives require an s3 or gcs connection, found "${importConnectionType ?? "(none)"}".`,
+        });
+        continue;
+      }
+
+      if (importConnectionType === "gcs") {
+        datasource.gcs = { ...importConfig };
+        datasource.s3 = undefined;
+      } else {
+        datasource.s3 = { ...importConfig };
+        datasource.gcs = undefined;
+      }
+    }
+
     try {
       validateResourceForEmission(datasource);
       migrated.push(datasource);

package/src/codegen/type-mapper.test.ts

@@ -132,6 +132,18 @@ describe("clickhouseTypeToValidator", () => {
         "t.map(t.string(), t.int32())"
       );
     });
+
+    it("handles Tuple(T1, T2, ...)", () => {
+      expect(clickhouseTypeToValidator("Tuple(String, Float64, String)")).toBe(
+        "t.tuple(t.string(), t.float64(), t.string())"
+      );
+    });
+
+    it("handles Array(Tuple(...))", () => {
+      expect(clickhouseTypeToValidator("Array(Tuple(String, Float64, String))")).toBe(
+        "t.array(t.tuple(t.string(), t.float64(), t.string()))"
+      );
+    });
   });
 
   describe("enum types", () => {
@@ -160,6 +172,12 @@ describe("clickhouseTypeToValidator", () => {
         't.aggregateFunction("uniq", t.string())'
       );
     });
+
+    it("handles AggregateFunction(count) without explicit state type", () => {
+      expect(clickhouseTypeToValidator("AggregateFunction(count)")).toBe(
+        't.aggregateFunction("count", t.uint64())'
+      );
+    });
   });
 
   describe("unknown types", () => {

package/src/codegen/type-mapper.ts

@@ -16,6 +16,61 @@ function parseEnumValues(enumContent: string): string[] {
   return values;
 }
 
+function splitTopLevelComma(input: string): string[] {
+  const parts: string[] = [];
+  let current = "";
+  let depth = 0;
+  let inSingleQuote = false;
+  let inDoubleQuote = false;
+
+  for (let i = 0; i < input.length; i += 1) {
+    const char = input[i];
+    const prev = i > 0 ? input[i - 1] : "";
+
+    if (char === "'" && !inDoubleQuote && prev !== "\\") {
+      inSingleQuote = !inSingleQuote;
+      current += char;
+      continue;
+    }
+
+    if (char === '"' && !inSingleQuote && prev !== "\\") {
+      inDoubleQuote = !inDoubleQuote;
+      current += char;
+      continue;
+    }
+
+    if (!inSingleQuote && !inDoubleQuote) {
+      if (char === "(") {
+        depth += 1;
+        current += char;
+        continue;
+      }
+      if (char === ")") {
+        depth -= 1;
+        current += char;
+        continue;
+      }
+      if (char === "," && depth === 0) {
+        const trimmed = current.trim();
+        if (trimmed.length > 0) {
+          parts.push(trimmed);
+        }
+        current = "";
+        continue;
+      }
+    }
+
+    current += char;
+  }
+
+  const trimmed = current.trim();
+  if (trimmed.length > 0) {
+    parts.push(trimmed);
+  }
+
+  return parts;
+}
+
 /**
  * Map a ClickHouse type to a t.* validator call
  *
@@ -138,19 +193,26 @@ export function clickhouseTypeToValidator(chType: string): string {
     return `t.array(${innerType})`;
   }
 
-  // Tuple(T1, T2, ...) - simplified handling
+  // Tuple(T1, T2, ...)
   const tupleMatch = chType.match(/^Tuple\((.+)\)$/);
   if (tupleMatch) {
-    // For complex tuples, we just use a JSON type
-    // TODO: Could parse and generate t.tuple() for simple cases
-    return `t.json() /* Tuple: ${chType} */`;
+    const tupleArgs = splitTopLevelComma(tupleMatch[1]);
+    if (tupleArgs.length === 0) {
+      return `t.string() /* TODO: Unknown type: ${chType} */`;
+    }
+    const tupleTypes = tupleArgs.map((arg) => clickhouseTypeToValidator(arg));
+    return `t.tuple(${tupleTypes.join(", ")})`;
   }
 
   // Map(K, V)
-  const mapMatch = chType.match(/^Map\(([^,]+),\s*(.+)\)$/);
+  const mapMatch = chType.match(/^Map\((.+)\)$/);
   if (mapMatch) {
-    const keyType = clickhouseTypeToValidator(mapMatch[1]);
-    const valueType = clickhouseTypeToValidator(mapMatch[2]);
+    const mapArgs = splitTopLevelComma(mapMatch[1]);
+    if (mapArgs.length !== 2) {
+      return `t.string() /* TODO: Unknown type: ${chType} */`;
+    }
+    const keyType = clickhouseTypeToValidator(mapArgs[0]);
+    const valueType = clickhouseTypeToValidator(mapArgs[1]);
     return `t.map(${keyType}, ${valueType})`;
   }
 
@@ -190,6 +252,16 @@ export function clickhouseTypeToValidator(chType: string): string {
     return `t.aggregateFunction("${func}", ${innerType})`;
   }
 
+  // AggregateFunction(count)
+  const aggNoArgMatch = chType.match(/^AggregateFunction\((\w+)\)$/);
+  if (aggNoArgMatch) {
+    const func = aggNoArgMatch[1];
+    if (func === "count") {
+      return 't.aggregateFunction("count", t.uint64())';
+    }
+    return `t.string() /* TODO: Unknown type: ${chType} */`;
+  }
+
   // Nested - treat as JSON
   if (chType.startsWith("Nested(")) {
     return `t.json() /* ${chType} */`;