@tinybirdco/sdk 0.0.48 → 0.0.50

package/src/cli/commands/migrate.test.ts

@@ -107,7 +107,7 @@ export const eventsEndpoint = definePipe("events_endpoint", {
 SELECT event_id, user_id, payload
 FROM events
 WHERE user_id = {{UInt64(user_id)}}
-  AND env = {{String(env, 'prod')}}
+AND env = {{String(env, 'prod')}}
       \`,
     }),
     node({
@@ -616,6 +616,98 @@ IMPORT_FROM_TIMESTAMP 2024-01-01T00:00:00Z
     expect(output).toContain('fromTimestamp: "2024-01-01T00:00:00Z"');
   });
 
+  it("migrates gcs connection and import datasource directives", async () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-migrate-"));
+    tempDirs.push(tempDir);
+
+    writeFile(
+      tempDir,
+      "gcsample.connection",
+      `TYPE gcs
+GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON {{ tb_secret("GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON") }}
+`
+    );
+
+    writeFile(
+      tempDir,
+      "events_gcs_landing.datasource",
+      `SCHEMA >
+    timestamp DateTime,
+    session_id String
+
+ENGINE "MergeTree"
+ENGINE_SORTING_KEY "timestamp"
+IMPORT_CONNECTION_NAME gcsample
+IMPORT_BUCKET_URI gs://my-gcs-bucket/events/*.csv
+IMPORT_SCHEDULE @auto
+IMPORT_FROM_TIMESTAMP 2024-01-01T00:00:00Z
+`
+    );
+
+    const result = await runMigrate({
+      cwd: tempDir,
+      patterns: ["."],
+      strict: true,
+    });
+
+    expect(result.success).toBe(true);
+    expect(result.errors).toHaveLength(0);
+    expect(result.migrated.filter((resource) => resource.kind === "connection")).toHaveLength(1);
+    expect(result.migrated.filter((resource) => resource.kind === "datasource")).toHaveLength(1);
+
+    const output = fs.readFileSync(result.outputPath, "utf-8");
+    expect(output).toContain("defineGCSConnection");
+    expect(output).toContain('export const gcsample = defineGCSConnection("gcsample", {');
+    expect(output).toContain(
+      'serviceAccountCredentialsJson: secret("GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON"),'
+    );
+    expect(output).toContain("gcs: {");
+    expect(output).toContain("connection: gcsample");
+    expect(output).toContain('bucketUri: "gs://my-gcs-bucket/events/*.csv"');
+    expect(output).toContain('schedule: "@auto"');
+    expect(output).toContain('fromTimestamp: "2024-01-01T00:00:00Z"');
+  });
+
+  it("reports an error when import directives use a non-bucket connection type", async () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-migrate-"));
+    tempDirs.push(tempDir);
+
+    writeFile(
+      tempDir,
+      "stream.connection",
+      `TYPE kafka
+KAFKA_BOOTSTRAP_SERVERS localhost:9092
+`
+    );
+
+    writeFile(
+      tempDir,
+      "events_landing.datasource",
+      `SCHEMA >
+    id String
+
+ENGINE "MergeTree"
+ENGINE_SORTING_KEY "id"
+IMPORT_CONNECTION_NAME stream
+IMPORT_BUCKET_URI gs://my-gcs-bucket/events/*.csv
+`
+    );
+
+    const result = await runMigrate({
+      cwd: tempDir,
+      patterns: ["."],
+      strict: true,
+    });
+
+    expect(result.success).toBe(false);
+    expect(result.errors.map((error) => error.message)).toEqual(
+      expect.arrayContaining([
+        'Datasource import directives require an s3 or gcs connection, found "kafka".',
+      ])
+    );
+  });
+
+
   it("migrates KAFKA_STORE_RAW_VALUE datasource directive", async () => {
     const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-migrate-"));
     tempDirs.push(tempDir);
@@ -737,6 +829,375 @@ DATASOURCE events_state
     );
   });
 
+  it("migrates TYPE copy in lowercase", async () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-migrate-"));
+    tempDirs.push(tempDir);
+
+    writeFile(
+      tempDir,
+      "copy_target.datasource",
+      `SCHEMA >
+    id String
+
+ENGINE "MergeTree"
+ENGINE_SORTING_KEY "id"
+`
+    );
+
+    writeFile(
+      tempDir,
+      "copy_lower.pipe",
+      `NODE copy_node
+SQL >
+    SELECT id
+    FROM copy_target
+TYPE copy
+TARGET_DATASOURCE copy_target
+`
+    );
+
+    const result = await runMigrate({
+      cwd: tempDir,
+      patterns: ["."],
+      strict: true,
+    });
+
+    expect(result.success).toBe(true);
+    expect(result.errors).toHaveLength(0);
+
+    const output = fs.readFileSync(result.outputPath, "utf-8");
+    expect(output).toContain('export const copyLower = defineCopyPipe("copy_lower", {');
+    expect(output).toContain("datasource: copyTarget,");
+  });
+
+  it("migrates TYPE ENDPOINT in uppercase", async () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-migrate-"));
+    tempDirs.push(tempDir);
+
+    writeFile(
+      tempDir,
+      "endpoint_upper.pipe",
+      `NODE endpoint
+SQL >
+    SELECT 1 AS id
+    FROM system.numbers
+    LIMIT 1
+TYPE ENDPOINT
+`
+    );
+
+    const result = await runMigrate({
+      cwd: tempDir,
+      patterns: ["."],
+      strict: true,
+    });
+
+    expect(result.success).toBe(true);
+    expect(result.errors).toHaveLength(0);
+
+    const output = fs.readFileSync(result.outputPath, "utf-8");
+    expect(output).toContain('export const endpointUpper = definePipe("endpoint_upper", {');
+    expect(output).toContain("endpoint: true,");
+  });
+
+  it("parses multiline datasource blocks with flexible indentation", async () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-migrate-"));
+    tempDirs.push(tempDir);
+
+    writeFile(
+      tempDir,
+      "flex.datasource",
+      `DESCRIPTION >
+  Flexible indentation description
+
+SCHEMA >
+  id String,
+  env String
+
+ENGINE "MergeTree"
+ENGINE_SORTING_KEY "id"
+FORWARD_QUERY >
+  SELECT id, env
+  FROM upstream_ds
+SHARED_WITH >
+  workspace_a,
+  workspace_b
+`
+    );
+
+    const result = await runMigrate({
+      cwd: tempDir,
+      patterns: ["."],
+      strict: true,
+    });
+
+    expect(result.success).toBe(true);
+    expect(result.errors).toHaveLength(0);
+
+    const output = fs.readFileSync(result.outputPath, "utf-8");
+    expect(output).toContain('description: "Flexible indentation description"');
+    expect(output).toContain("forwardQuery: `");
+    expect(output).toContain("SELECT id, env");
+    expect(output).toContain("FROM upstream_ds");
+    expect(output).toContain('sharedWith: ["workspace_a", "workspace_b"],');
+  });
+
+  it("parses node SQL blocks with flexible indentation", async () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-migrate-"));
+    tempDirs.push(tempDir);
+
+    writeFile(
+      tempDir,
+      "flex_pipe.pipe",
+      `NODE endpoint_node
+DESCRIPTION >
+  Endpoint node description
+SQL >
+  %
+  SELECT 1 AS id
+  FROM system.numbers
+TYPE endpoint
+`
+    );
+
+    const result = await runMigrate({
+      cwd: tempDir,
+      patterns: ["."],
+      strict: true,
+    });
+
+    expect(result.success).toBe(true);
+    expect(result.errors).toHaveLength(0);
+
+    const output = fs.readFileSync(result.outputPath, "utf-8");
+    expect(output).toContain('description: "Endpoint node description"');
+    expect(output).toContain("SELECT 1 AS id");
+    expect(output).toContain("FROM system.numbers");
+    expect(output).toContain("endpoint: true,");
+  });
+
+  it("migrates datasource without engine directives", async () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-migrate-"));
+    tempDirs.push(tempDir);
+
+    writeFile(
+      tempDir,
+      "no_engine.datasource",
+      `SCHEMA >
+  id String,
+  name String
+`
+    );
+
+    const result = await runMigrate({
+      cwd: tempDir,
+      patterns: ["."],
+      strict: true,
+    });
+
+    expect(result.success).toBe(true);
+    expect(result.errors).toHaveLength(0);
+
+    const output = fs.readFileSync(result.outputPath, "utf-8");
+    expect(output).toContain('export const noEngine = defineDatasource("no_engine", {');
+    expect(output).not.toContain("engine:");
+    expect(output).not.toContain(", engine,");
+  });
+
+  it("infers MergeTree when engine options exist without ENGINE directive", async () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-migrate-"));
+    tempDirs.push(tempDir);
+
+    writeFile(
+      tempDir,
+      "implicit_engine.datasource",
+      `SCHEMA >
+  id String,
+  event_date Date
+
+ENGINE_SORTING_KEY "id"
+ENGINE_PARTITION_KEY "toYYYYMM(event_date)"
+`
+    );
+
+    const result = await runMigrate({
+      cwd: tempDir,
+      patterns: ["."],
+      strict: true,
+    });
+
+    expect(result.success).toBe(true);
+    expect(result.errors).toHaveLength(0);
+
+    const output = fs.readFileSync(result.outputPath, "utf-8");
+    expect(output).toContain('export const implicitEngine = defineDatasource("implicit_engine", {');
+    expect(output).toContain(
+      'engine: engine.mergeTree({ sortingKey: "id", partitionKey: "toYYYYMM(event_date)" }),'
+    );
+  });
+
+  it("supports quoted datasource token names with spaces", async () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-migrate-"));
+    tempDirs.push(tempDir);
+
+    writeFile(
+      tempDir,
+      "token_spaces.datasource",
+      `TOKEN "ingestion (Data Source append)" APPEND
+
+SCHEMA >
+  id String
+
+ENGINE "MergeTree"
+ENGINE_SORTING_KEY "id"
+`
+    );
+
+    const result = await runMigrate({
+      cwd: tempDir,
+      patterns: ["."],
+      strict: true,
+    });
+
+    expect(result.success).toBe(true);
+    expect(result.errors).toHaveLength(0);
+
+    const output = fs.readFileSync(result.outputPath, "utf-8");
+    expect(output).toContain(
+      '{ name: "ingestion (Data Source append)", permissions: ["APPEND"] },'
+    );
+  });
+
+  it("rejects unquoted datasource token names with spaces", async () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-migrate-"));
+    tempDirs.push(tempDir);
+
+    writeFile(
+      tempDir,
+      "bad_token.datasource",
+      `TOKEN ingestion data APPEND
+
+SCHEMA >
+  id String
+
+ENGINE "MergeTree"
+ENGINE_SORTING_KEY "id"
+`
+    );
+
+    const result = await runMigrate({
+      cwd: tempDir,
+      patterns: ["."],
+      strict: true,
+    });
+
+    expect(result.success).toBe(false);
+    expect(result.errors).toHaveLength(1);
+    expect(result.errors[0]?.message).toContain("Unsupported TOKEN syntax in strict mode");
+  });
+
+  it("allows empty datasource DESCRIPTION block", async () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-migrate-"));
+    tempDirs.push(tempDir);
+
+    writeFile(
+      tempDir,
+      "empty_ds_desc.datasource",
+      `DESCRIPTION >
+
+SCHEMA >
+  id String
+
+ENGINE "MergeTree"
+ENGINE_SORTING_KEY "id"
+`
+    );
+
+    const result = await runMigrate({
+      cwd: tempDir,
+      patterns: ["."],
+      strict: true,
+    });
+
+    expect(result.success).toBe(true);
+    expect(result.errors).toHaveLength(0);
+
+    const output = fs.readFileSync(result.outputPath, "utf-8");
+    expect(output).toContain(
+      'export const emptyDsDesc = defineDatasource("empty_ds_desc", {'
+    );
+    expect(output).toContain('description: "",');
+  });
+
+  it("allows empty node DESCRIPTION block", async () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-migrate-"));
+    tempDirs.push(tempDir);
+
+    writeFile(
+      tempDir,
+      "empty_node_desc.pipe",
+      `NODE helper
+DESCRIPTION >
+
+SQL >
+  SELECT 1 AS id
+TYPE endpoint
+`
+    );
+
+    const result = await runMigrate({
+      cwd: tempDir,
+      patterns: ["."],
+      strict: true,
+    });
+
+    expect(result.success).toBe(true);
+    expect(result.errors).toHaveLength(0);
+
+    const output = fs.readFileSync(result.outputPath, "utf-8");
+    expect(output).toContain('export const emptyNodeDesc = definePipe("empty_node_desc", {');
+    expect(output).toContain('description: "",');
+  });
+
+  it("supports keyword-style pipe param arguments", async () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-migrate-"));
+    tempDirs.push(tempDir);
+
+    writeFile(
+      tempDir,
+      "keyword_params.pipe",
+      `NODE endpoint
+SQL >
+  SELECT 1 AS id
+  WHERE tenant_id = {{ String(tenant_id, description="Tenant ID to filter", default="") }}
+    AND event_date >= {{ Date(from_date, description="Starting date", required=False) }}
+    AND days >= {{ Int32(days, 1, description="Days to include", required=True) }}
+TYPE endpoint
+`
+    );
+
+    const result = await runMigrate({
+      cwd: tempDir,
+      patterns: ["."],
+      strict: true,
+    });
+
+    expect(result.success).toBe(true);
+    expect(result.errors).toHaveLength(0);
+
+    const output = fs.readFileSync(result.outputPath, "utf-8");
+    expect(output).toContain('export const keywordParams = definePipe("keyword_params", {');
+    expect(output).toContain(
+      'tenant_id: p.string().optional("").describe("Tenant ID to filter"),'
+    );
+    expect(output).toContain(
+      'from_date: p.date().optional().describe("Starting date"),'
+    );
+    expect(output).toContain(
+      'days: p.int32().optional(1).describe("Days to include"),'
+    );
+  });
+
   it("migrates datasource with mixed explicit and default json paths", async () => {
     const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-migrate-"));
     tempDirs.push(tempDir);
@@ -887,4 +1348,213 @@ KAFKA_GROUP_ID events-group
     expect(output).not.toContain("const secret = (name: string, defaultValue?: string) =>");
     expect(output).toContain('groupId: "events-group",');
   });
+
+  it("migrates Kafka sink pipes and emits sink config in TypeScript", async () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-migrate-"));
+    tempDirs.push(tempDir);
+
+    writeFile(
+      tempDir,
+      "events_kafka.connection",
+      `TYPE kafka
+KAFKA_BOOTSTRAP_SERVERS localhost:9092
+`
+    );
+
+    writeFile(
+      tempDir,
+      "events_sink.pipe",
+      `NODE publish
+SQL >
+    SELECT *
+    FROM events
+    WHERE env = {{String(env, 'prod')}}
+TYPE sink
+EXPORT_CONNECTION_NAME events_kafka
+EXPORT_KAFKA_TOPIC events_out
+EXPORT_SCHEDULE @on-demand
+`
+    );
+
+    const result = await runMigrate({
+      cwd: tempDir,
+      patterns: ["."],
+      strict: true,
+    });
+
+    expect(result.success).toBe(true);
+    expect(result.errors).toHaveLength(0);
+    expect(result.migrated.filter((resource) => resource.kind === "connection")).toHaveLength(1);
+    expect(result.migrated.filter((resource) => resource.kind === "pipe")).toHaveLength(1);
+
+    const output = fs.readFileSync(result.outputPath, "utf-8");
+    expect(output).toContain('export const eventsSink = defineSinkPipe("events_sink", {');
+    expect(output).toContain("params: {");
+    expect(output).toContain('env: p.string().optional("prod"),');
+    expect(output).toContain("sink: {");
+    expect(output).toContain("connection: eventsKafka");
+    expect(output).toContain('topic: "events_out"');
+    expect(output).toContain('schedule: "@on-demand"');
+    expect(output).not.toContain('strategy:');
+  });
+
+  it("migrates S3 sink pipes and emits compression/strategy config in TypeScript", async () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-migrate-"));
+    tempDirs.push(tempDir);
+
+    writeFile(
+      tempDir,
+      "exports_s3.connection",
+      `TYPE s3
+S3_REGION "us-east-1"
+S3_ARN "arn:aws:iam::123456789012:role/tinybird-s3-access"
+`
+    );
+
+    writeFile(
+      tempDir,
+      "events_s3_sink.pipe",
+      `NODE export
+SQL >
+    SELECT * FROM events
+TYPE sink
+EXPORT_CONNECTION_NAME exports_s3
+EXPORT_BUCKET_URI s3://exports/events/
+EXPORT_FILE_TEMPLATE events_{date}
+EXPORT_SCHEDULE @once
+EXPORT_FORMAT ndjson
+EXPORT_STRATEGY create_new
+EXPORT_COMPRESSION gzip
+`
+    );
+
+    const result = await runMigrate({
+      cwd: tempDir,
+      patterns: ["."],
+      strict: true,
+    });
+
+    expect(result.success).toBe(true);
+    expect(result.errors).toHaveLength(0);
+
+    const output = fs.readFileSync(result.outputPath, "utf-8");
+    expect(output).toContain('export const eventsS3Sink = defineSinkPipe("events_s3_sink", {');
+    expect(output).toContain("sink: {");
+    expect(output).toContain("connection: exportsS3");
+    expect(output).toContain('bucketUri: "s3://exports/events/"');
+    expect(output).toContain('fileTemplate: "events_{date}"');
+    expect(output).toContain('schedule: "@once"');
+    expect(output).toContain('format: "ndjson"');
+    expect(output).toContain('strategy: "create_new"');
+    expect(output).toContain('compression: "gzip"');
+  });
+
+  it("reports an error when sink pipe references a missing connection", async () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-migrate-"));
+    tempDirs.push(tempDir);
+
+    writeFile(
+      tempDir,
+      "events_sink.pipe",
+      `NODE publish
+SQL >
+    SELECT * FROM events
+TYPE sink
+EXPORT_CONNECTION_NAME missing_connection
+EXPORT_KAFKA_TOPIC events_out
+EXPORT_SCHEDULE @on-demand
+`
+    );
+
+    const result = await runMigrate({
+      cwd: tempDir,
+      patterns: ["."],
+      strict: true,
+    });
+
+    expect(result.success).toBe(false);
+    expect(result.errors.map((error) => error.message)).toEqual(
+      expect.arrayContaining([
+        'Sink pipe references missing/unmigrated connection "missing_connection".',
+      ])
+    );
+  });
+
+  it("reports an error when sink connection type does not match sink service", async () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-migrate-"));
+    tempDirs.push(tempDir);
+
+    writeFile(
+      tempDir,
+      "events_kafka.connection",
+      `TYPE kafka
+KAFKA_BOOTSTRAP_SERVERS localhost:9092
+`
+    );
+
+    writeFile(
+      tempDir,
+      "events_s3_sink.pipe",
+      `NODE export
+SQL >
+    SELECT * FROM events
+TYPE sink
+EXPORT_CONNECTION_NAME events_kafka
+EXPORT_BUCKET_URI s3://exports/events/
+EXPORT_FILE_TEMPLATE events_{date}
+EXPORT_SCHEDULE @once
+EXPORT_FORMAT csv
+`
+    );
+
+    const result = await runMigrate({
+      cwd: tempDir,
+      patterns: ["."],
+      strict: true,
+    });
+
+    expect(result.success).toBe(false);
+    expect(result.errors.map((error) => error.message)).toEqual(
+      expect.arrayContaining([
+        'Sink pipe service "s3" is incompatible with connection "events_kafka" type "kafka".',
+      ])
+    );
+  });
+
+  it("supports comments in sink pipe files", async () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-migrate-"));
+    tempDirs.push(tempDir);
+
+    writeFile(
+      tempDir,
+      "events_kafka.connection",
+      `TYPE kafka
+KAFKA_BOOTSTRAP_SERVERS localhost:9092
+`
+    );
+
+    writeFile(
+      tempDir,
+      "events_sink.pipe",
+      `# this pipe publishes records
+NODE publish
+SQL >
+    SELECT * FROM events
+TYPE sink
+# Kafka target
+EXPORT_CONNECTION_NAME events_kafka
+EXPORT_KAFKA_TOPIC events_out
+EXPORT_SCHEDULE @on-demand
+`
+    );
+
+    const result = await runMigrate({
+      cwd: tempDir,
+      patterns: ["."],
+      strict: true,
+    });
+
+    expect(result.success).toBe(true);
+    expect(result.errors).toHaveLength(0);
+  });
 });