@tinybirdco/sdk 0.0.41 → 0.0.42

package/src/cli/commands/migrate.test.ts

@@ -0,0 +1,564 @@
+import * as fs from "node:fs";
+import * as os from "node:os";
+import * as path from "node:path";
+import { afterEach, describe, expect, it } from "vitest";
+import { runMigrate } from "./migrate.js";
+
+function writeFile(dir: string, relativePath: string, content: string): void {
+  const fullPath = path.join(dir, relativePath);
+  fs.mkdirSync(path.dirname(fullPath), { recursive: true });
+  fs.writeFileSync(fullPath, content);
+}
+
+const EXPECTED_COMPLEX_OUTPUT = `/**
+ * Generated by tinybird migrate.
+ * Review endpoint output schemas and any defaults before production use.
+ */
+
+import { createKafkaConnection, defineDatasource, definePipe, defineMaterializedView, defineCopyPipe, node, t, engine, column, p } from "@tinybirdco/sdk";
+
+// Connections
+
+export const stream = createKafkaConnection("stream", {
+  bootstrapServers: "localhost:9092",
+  securityProtocol: "SASL_SSL",
+  saslMechanism: "PLAIN",
+  key: "api-key",
+  secret: "api-secret",
+  sslCaPem: "ca-pem-content",
+});
+
+// Datasources
+
+/**
+ * Events from Kafka stream
+ */
+export const events = defineDatasource("events", {
+  description: "Events from Kafka stream",
+  schema: {
+    event_id: column(t.string(), { jsonPath: "$.event_id" }),
+    user_id: column(t.uint64(), { jsonPath: "$.user.id" }),
+    env: column(t.string().default("prod"), { jsonPath: "$.env" }),
+    is_test: column(t.bool().default(false), { jsonPath: "$.meta.is_test" }),
+    updated_at: column(t.dateTime(), { jsonPath: "$.updated_at" }),
+    payload: column(t.string().default("{}").codec("ZSTD(1)"), { jsonPath: "$.payload" }),
+  },
+  engine: engine.replacingMergeTree({ sortingKey: ["event_id", "user_id"], partitionKey: "toYYYYMM(updated_at)", primaryKey: "event_id", ttl: "updated_at + toIntervalDay(30)", ver: "updated_at", settings: { "index_granularity": 8192, "enable_mixed_granularity_parts": true } }),
+  kafka: {
+    connection: stream,
+    topic: "events_topic",
+    groupId: "events-consumer",
+    autoOffsetReset: "earliest",
+  },
+  forwardQuery: \`
+SELECT *
+FROM events_mv
+  \`,
+  tokens: [
+    { name: "events_read", permissions: ["READ"] },
+    { name: "events_append", permissions: ["APPEND"] },
+  ],
+  sharedWith: ["workspace_a", "workspace_b"],
+});
+
+export const eventsRollup = defineDatasource("events_rollup", {
+  jsonPaths: false,
+  schema: {
+    user_id: t.uint64(),
+    total: t.uint64(),
+  },
+  engine: engine.summingMergeTree({ sortingKey: "user_id", columns: ["total"] }),
+});
+
+// Pipes
+
+export const copyEvents = defineCopyPipe("copy_events", {
+  datasource: eventsRollup,
+  copy_mode: "replace",
+  copy_schedule: "@on-demand",
+  nodes: [
+    node({
+      name: "copy_node",
+      sql: \`
+SELECT event_id, user_id
+FROM events
+      \`,
+    }),
+  ],
+  tokens: [
+    { name: "copy_token" },
+  ],
+});
+
+/**
+ * Endpoint for filtered events
+ */
+export const eventsEndpoint = definePipe("events_endpoint", {
+  description: "Endpoint for filtered events",
+  params: {
+    env: p.string().optional("prod"),
+    user_id: p.uint64(),
+  },
+  nodes: [
+    node({
+      name: "base",
+      description: "Base filter",
+      sql: \`
+SELECT event_id, user_id, payload
+FROM events
+WHERE user_id = {{UInt64(user_id)}}
+  AND env = {{String(env, 'prod')}}
+      \`,
+    }),
+    node({
+      name: "endpoint",
+      sql: \`
+SELECT event_id AS event_id, user_id AS user_id
+FROM base
+      \`,
+    }),
+  ],
+  endpoint: { enabled: true, cache: { enabled: true, ttl: 120 } },
+  output: {
+    event_id: t.string(),
+    user_id: t.string(),
+  },
+  tokens: [
+    { name: "endpoint_token" },
+  ],
+});
+
+/**
+ * Materialized rollup
+ */
+export const eventsMv = defineMaterializedView("events_mv", {
+  description: "Materialized rollup",
+  datasource: eventsRollup,
+  deploymentMethod: "alter",
+  nodes: [
+    node({
+      name: "rollup",
+      sql: \`
+SELECT user_id, count() AS total
+FROM events
+GROUP BY user_id
+      \`,
+    }),
+  ],
+  tokens: [
+    { name: "mv_token" },
+  ],
+});
+
+export const statsPipe = definePipe("stats_pipe", {
+  params: {
+    min_total: p.uint32().optional(10),
+  },
+  nodes: [
+    node({
+      name: "agg",
+      sql: \`
+SELECT user_id, count() AS total
+FROM events
+GROUP BY user_id
+      \`,
+    }),
+    node({
+      name: "final",
+      sql: \`
+SELECT user_id, total
+FROM agg
+WHERE total > {{UInt32(min_total, 10)}}
+      \`,
+    }),
+  ],
+  tokens: [
+    { name: "stats_token" },
+  ],
+});
+`;
+
+const EXPECTED_PARTIAL_OUTPUT = `/**
+ * Generated by tinybird migrate.
+ * Review endpoint output schemas and any defaults before production use.
+ */
+
+import { createKafkaConnection, defineDatasource, definePipe, defineMaterializedView, defineCopyPipe, node, t, engine, p } from "@tinybirdco/sdk";
+
+// Connections
+
+export const stream = createKafkaConnection("stream", {
+  bootstrapServers: "localhost:9092",
+});
+
+// Datasources
+
+export const events = defineDatasource("events", {
+  jsonPaths: false,
+  schema: {
+    event_id: t.string(),
+    user_id: t.uint64(),
+    created_at: t.dateTime(),
+  },
+  engine: engine.mergeTree({ sortingKey: "event_id" }),
+  kafka: {
+    connection: stream,
+    topic: "events_topic",
+  },
+});
+
+// Pipes
+
+export const eventsEndpoint = definePipe("events_endpoint", {
+  params: {
+    user_id: p.uint64(),
+  },
+  nodes: [
+    node({
+      name: "source",
+      sql: \`
+SELECT event_id, user_id
+FROM events
+      \`,
+    }),
+    node({
+      name: "endpoint",
+      sql: \`
+SELECT event_id AS event_id, user_id AS user_id
+FROM source
+WHERE user_id = {{UInt64(user_id)}}
+      \`,
+    }),
+  ],
+  endpoint: true,
+  output: {
+    event_id: t.string(),
+    user_id: t.string(),
+  },
+  tokens: [
+    { name: "endpoint_token" },
+  ],
+});
+`;
+
+describe("runMigrate", () => {
+  const tempDirs: string[] = [];
+
+  afterEach(() => {
+    for (const dir of tempDirs) {
+      try {
+        fs.rmSync(dir, { recursive: true });
+      } catch {
+        // Ignore cleanup failures
+      }
+    }
+    tempDirs.length = 0;
+  });
+
+  it("migrates complex resources including endpoint, materialized, and copy pipes", async () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-migrate-"));
+    tempDirs.push(tempDir);
+
+    writeFile(
+      tempDir,
+      "stream.connection",
+      `TYPE kafka
+KAFKA_BOOTSTRAP_SERVERS localhost:9092
+KAFKA_SECURITY_PROTOCOL SASL_SSL
+KAFKA_SASL_MECHANISM PLAIN
+KAFKA_KEY api-key
+KAFKA_SECRET api-secret
+KAFKA_SSL_CA_PEM ca-pem-content
+`
+    );
+
+    writeFile(
+      tempDir,
+      "events.datasource",
+      `DESCRIPTION >
+    Events from Kafka stream
+SCHEMA >
+    event_id String \`json:$.event_id\`,
+    user_id UInt64 \`json:$.user.id\`,
+    env String \`json:$.env\` DEFAULT 'prod',
+    is_test Bool \`json:$.meta.is_test\` DEFAULT 0,
+    updated_at DateTime \`json:$.updated_at\`,
+    payload String \`json:$.payload\` DEFAULT '{}' CODEC(ZSTD(1))
+
+ENGINE "ReplacingMergeTree"
+ENGINE_SORTING_KEY "event_id, user_id"
+ENGINE_PARTITION_KEY "toYYYYMM(updated_at)"
+ENGINE_PRIMARY_KEY "event_id"
+ENGINE_TTL "updated_at + toIntervalDay(30)"
+ENGINE_VER "updated_at"
+ENGINE_SETTINGS "index_granularity=8192, enable_mixed_granularity_parts=true"
+KAFKA_CONNECTION_NAME stream
+KAFKA_TOPIC events_topic
+KAFKA_GROUP_ID events-consumer
+KAFKA_AUTO_OFFSET_RESET earliest
+TOKEN events_read READ
+TOKEN events_append APPEND
+SHARED_WITH >
+    workspace_a,
+    workspace_b
+FORWARD_QUERY >
+    SELECT *
+    FROM events_mv
+`
+    );
+
+    writeFile(
+      tempDir,
+      "events_rollup.datasource",
+      `SCHEMA >
+    user_id UInt64,
+    total UInt64
+
+ENGINE "SummingMergeTree"
+ENGINE_SORTING_KEY "user_id"
+ENGINE_SUMMING_COLUMNS "total"
+`
+    );
+
+    writeFile(
+      tempDir,
+      "events_endpoint.pipe",
+      `DESCRIPTION >
+    Endpoint for filtered events
+NODE base
+DESCRIPTION >
+    Base filter
+SQL >
+    %
+    SELECT event_id, user_id, payload
+    FROM events
+    WHERE user_id = {{UInt64(user_id)}}
+      AND env = {{String(env, 'prod')}}
+NODE endpoint
+SQL >
+    SELECT event_id AS event_id, user_id AS user_id
+    FROM base
+TYPE endpoint
+CACHE 120
+TOKEN endpoint_token READ
+`
+    );
+
+    writeFile(
+      tempDir,
+      "events_mv.pipe",
+      `DESCRIPTION >
+    Materialized rollup
+NODE rollup
+SQL >
+    SELECT user_id, count() AS total
+    FROM events
+    GROUP BY user_id
+TYPE MATERIALIZED
+DATASOURCE events_rollup
+DEPLOYMENT_METHOD alter
+TOKEN mv_token READ
+`
+    );
+
+    writeFile(
+      tempDir,
+      "copy_events.pipe",
+      `NODE copy_node
+SQL >
+    SELECT event_id, user_id
+    FROM events
+TYPE COPY
+TARGET_DATASOURCE events_rollup
+COPY_SCHEDULE @on-demand
+COPY_MODE replace
+TOKEN copy_token READ
+`
+    );
+
+    writeFile(
+      tempDir,
+      "stats_pipe.pipe",
+      `NODE agg
+SQL >
+    SELECT user_id, count() AS total
+    FROM events
+    GROUP BY user_id
+NODE final
+SQL >
+    SELECT user_id, total
+    FROM agg
+    WHERE total > {{UInt32(min_total, 10)}}
+TOKEN stats_token READ
+`
+    );
+
+    const result = await runMigrate({
+      cwd: tempDir,
+      patterns: ["."],
+      strict: true,
+    });
+
+    expect(result.success).toBe(true);
+    expect(result.errors).toHaveLength(0);
+    expect(result.migrated).toHaveLength(7);
+    expect(result.migrated.filter((resource) => resource.kind === "connection")).toHaveLength(1);
+    expect(result.migrated.filter((resource) => resource.kind === "datasource")).toHaveLength(2);
+    expect(result.migrated.filter((resource) => resource.kind === "pipe")).toHaveLength(4);
+    expect(path.basename(result.outputPath)).toBe("tinybird.migration.ts");
+    expect(fs.existsSync(result.outputPath)).toBe(true);
+
+    const output = fs.readFileSync(result.outputPath, "utf-8");
+    expect(output).toBe(EXPECTED_COMPLEX_OUTPUT);
+  });
+
+  it("continues processing and reports all errors while writing complex migratable resources", async () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-migrate-"));
+    tempDirs.push(tempDir);
+
+    writeFile(
+      tempDir,
+      "stream.connection",
+      `TYPE kafka
+KAFKA_BOOTSTRAP_SERVERS localhost:9092
+`
+    );
+
+    writeFile(
+      tempDir,
+      "events.datasource",
+      `SCHEMA >
+    event_id String,
+    user_id UInt64,
+    created_at DateTime
+
+ENGINE "MergeTree"
+ENGINE_SORTING_KEY "event_id"
+KAFKA_CONNECTION_NAME stream
+KAFKA_TOPIC events_topic
+`
+    );
+
+    writeFile(
+      tempDir,
+      "events_endpoint.pipe",
+      `NODE source
+SQL >
+    SELECT event_id, user_id
+    FROM events
+NODE endpoint
+SQL >
+    SELECT event_id AS event_id, user_id AS user_id
+    FROM source
+    WHERE user_id = {{UInt64(user_id)}}
+TYPE endpoint
+TOKEN endpoint_token READ
+`
+    );
+
+    writeFile(
+      tempDir,
+      "events_mv.pipe",
+      `NODE rollup
+SQL >
+    SELECT user_id, count() AS total
+    FROM events
+    GROUP BY user_id
+TYPE MATERIALIZED
+DATASOURCE missing_ds
+`
+    );
+
+    writeFile(
+      tempDir,
+      "broken.pipe",
+      `NODE broken
+SQL >
+    SELECT *
+    FROM events
+TYPE endpoint
+UNSUPPORTED_DIRECTIVE true
+`
+    );
+
+    const result = await runMigrate({
+      cwd: tempDir,
+      patterns: ["."],
+      strict: true,
+    });
+
+    expect(result.success).toBe(false);
+    expect(result.errors).toHaveLength(2);
+    expect(result.errors.map((error) => error.message)).toEqual(
+      expect.arrayContaining([
+        'Unsupported pipe directive in strict mode: "UNSUPPORTED_DIRECTIVE true"',
+        'Materialized pipe references missing/unmigrated datasource "missing_ds".',
+      ])
+    );
+    expect(result.migrated.filter((resource) => resource.kind === "connection")).toHaveLength(1);
+    expect(result.migrated.filter((resource) => resource.kind === "datasource")).toHaveLength(1);
+    expect(result.migrated.filter((resource) => resource.kind === "pipe")).toHaveLength(1);
+    expect(fs.existsSync(result.outputPath)).toBe(true);
+
+    const output = fs.readFileSync(result.outputPath, "utf-8");
+    expect(output).toBe(EXPECTED_PARTIAL_OUTPUT);
+  });
+
+  it("returns exact output content in dry-run mode for complex migratable resources", async () => {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-migrate-"));
+    tempDirs.push(tempDir);
+
+    writeFile(
+      tempDir,
+      "stream.connection",
+      `TYPE kafka
+KAFKA_BOOTSTRAP_SERVERS localhost:9092
+`
+    );
+
+    writeFile(
+      tempDir,
+      "events.datasource",
+      `SCHEMA >
+    event_id String,
+    user_id UInt64,
+    created_at DateTime
+
+ENGINE "MergeTree"
+ENGINE_SORTING_KEY "event_id"
+KAFKA_CONNECTION_NAME stream
+KAFKA_TOPIC events_topic
+`
+    );
+
+    writeFile(
+      tempDir,
+      "events_endpoint.pipe",
+      `NODE source
+SQL >
+    SELECT event_id, user_id
+    FROM events
+NODE endpoint
+SQL >
+    SELECT event_id AS event_id, user_id AS user_id
+    FROM source
+    WHERE user_id = {{UInt64(user_id)}}
+TYPE endpoint
+TOKEN endpoint_token READ
+`
+    );
+
+    const result = await runMigrate({
+      cwd: tempDir,
+      patterns: ["."],
+      strict: true,
+      dryRun: true,
+    });
+
+    expect(result.success).toBe(true);
+    expect(result.errors).toHaveLength(0);
+    expect(result.migrated).toHaveLength(3);
+    expect(result.outputContent).toBe(EXPECTED_PARTIAL_OUTPUT);
+    expect(fs.existsSync(result.outputPath)).toBe(false);
+  });
+});