@tinybirdco/sdk 0.0.40 → 0.0.42

package/dist/cli/commands/migrate.test.js

@@ -0,0 +1,473 @@
+import * as fs from "node:fs";
+import * as os from "node:os";
+import * as path from "node:path";
+import { afterEach, describe, expect, it } from "vitest";
+import { runMigrate } from "./migrate.js";
+function writeFile(dir, relativePath, content) {
+    const fullPath = path.join(dir, relativePath);
+    fs.mkdirSync(path.dirname(fullPath), { recursive: true });
+    fs.writeFileSync(fullPath, content);
+}
+const EXPECTED_COMPLEX_OUTPUT = `/**
+ * Generated by tinybird migrate.
+ * Review endpoint output schemas and any defaults before production use.
+ */
+
+import { createKafkaConnection, defineDatasource, definePipe, defineMaterializedView, defineCopyPipe, node, t, engine, column, p } from "@tinybirdco/sdk";
+
+// Connections
+
+export const stream = createKafkaConnection("stream", {
+  bootstrapServers: "localhost:9092",
+  securityProtocol: "SASL_SSL",
+  saslMechanism: "PLAIN",
+  key: "api-key",
+  secret: "api-secret",
+  sslCaPem: "ca-pem-content",
+});
+
+// Datasources
+
+/**
+ * Events from Kafka stream
+ */
+export const events = defineDatasource("events", {
+  description: "Events from Kafka stream",
+  schema: {
+    event_id: column(t.string(), { jsonPath: "$.event_id" }),
+    user_id: column(t.uint64(), { jsonPath: "$.user.id" }),
+    env: column(t.string().default("prod"), { jsonPath: "$.env" }),
+    is_test: column(t.bool().default(false), { jsonPath: "$.meta.is_test" }),
+    updated_at: column(t.dateTime(), { jsonPath: "$.updated_at" }),
+    payload: column(t.string().default("{}").codec("ZSTD(1)"), { jsonPath: "$.payload" }),
+  },
+  engine: engine.replacingMergeTree({ sortingKey: ["event_id", "user_id"], partitionKey: "toYYYYMM(updated_at)", primaryKey: "event_id", ttl: "updated_at + toIntervalDay(30)", ver: "updated_at", settings: { "index_granularity": 8192, "enable_mixed_granularity_parts": true } }),
+  kafka: {
+    connection: stream,
+    topic: "events_topic",
+    groupId: "events-consumer",
+    autoOffsetReset: "earliest",
+  },
+  forwardQuery: \`
+SELECT *
+FROM events_mv
+  \`,
+  tokens: [
+    { name: "events_read", permissions: ["READ"] },
+    { name: "events_append", permissions: ["APPEND"] },
+  ],
+  sharedWith: ["workspace_a", "workspace_b"],
+});
+
+export const eventsRollup = defineDatasource("events_rollup", {
+  jsonPaths: false,
+  schema: {
+    user_id: t.uint64(),
+    total: t.uint64(),
+  },
+  engine: engine.summingMergeTree({ sortingKey: "user_id", columns: ["total"] }),
+});
+
+// Pipes
+
+export const copyEvents = defineCopyPipe("copy_events", {
+  datasource: eventsRollup,
+  copy_mode: "replace",
+  copy_schedule: "@on-demand",
+  nodes: [
+    node({
+      name: "copy_node",
+      sql: \`
+SELECT event_id, user_id
+FROM events
+      \`,
+    }),
+  ],
+  tokens: [
+    { name: "copy_token" },
+  ],
+});
+
+/**
+ * Endpoint for filtered events
+ */
+export const eventsEndpoint = definePipe("events_endpoint", {
+  description: "Endpoint for filtered events",
+  params: {
+    env: p.string().optional("prod"),
+    user_id: p.uint64(),
+  },
+  nodes: [
+    node({
+      name: "base",
+      description: "Base filter",
+      sql: \`
+SELECT event_id, user_id, payload
+FROM events
+WHERE user_id = {{UInt64(user_id)}}
+  AND env = {{String(env, 'prod')}}
+      \`,
+    }),
+    node({
+      name: "endpoint",
+      sql: \`
+SELECT event_id AS event_id, user_id AS user_id
+FROM base
+      \`,
+    }),
+  ],
+  endpoint: { enabled: true, cache: { enabled: true, ttl: 120 } },
+  output: {
+    event_id: t.string(),
+    user_id: t.string(),
+  },
+  tokens: [
+    { name: "endpoint_token" },
+  ],
+});
+
+/**
+ * Materialized rollup
+ */
+export const eventsMv = defineMaterializedView("events_mv", {
+  description: "Materialized rollup",
+  datasource: eventsRollup,
+  deploymentMethod: "alter",
+  nodes: [
+    node({
+      name: "rollup",
+      sql: \`
+SELECT user_id, count() AS total
+FROM events
+GROUP BY user_id
+      \`,
+    }),
+  ],
+  tokens: [
+    { name: "mv_token" },
+  ],
+});
+
+export const statsPipe = definePipe("stats_pipe", {
+  params: {
+    min_total: p.uint32().optional(10),
+  },
+  nodes: [
+    node({
+      name: "agg",
+      sql: \`
+SELECT user_id, count() AS total
+FROM events
+GROUP BY user_id
+      \`,
+    }),
+    node({
+      name: "final",
+      sql: \`
+SELECT user_id, total
+FROM agg
+WHERE total > {{UInt32(min_total, 10)}}
+      \`,
+    }),
+  ],
+  tokens: [
+    { name: "stats_token" },
+  ],
+});
+`;
+const EXPECTED_PARTIAL_OUTPUT = `/**
+ * Generated by tinybird migrate.
+ * Review endpoint output schemas and any defaults before production use.
+ */
+
+import { createKafkaConnection, defineDatasource, definePipe, defineMaterializedView, defineCopyPipe, node, t, engine, p } from "@tinybirdco/sdk";
+
+// Connections
+
+export const stream = createKafkaConnection("stream", {
+  bootstrapServers: "localhost:9092",
+});
+
+// Datasources
+
+export const events = defineDatasource("events", {
+  jsonPaths: false,
+  schema: {
+    event_id: t.string(),
+    user_id: t.uint64(),
+    created_at: t.dateTime(),
+  },
+  engine: engine.mergeTree({ sortingKey: "event_id" }),
+  kafka: {
+    connection: stream,
+    topic: "events_topic",
+  },
+});
+
+// Pipes
+
+export const eventsEndpoint = definePipe("events_endpoint", {
+  params: {
+    user_id: p.uint64(),
+  },
+  nodes: [
+    node({
+      name: "source",
+      sql: \`
+SELECT event_id, user_id
+FROM events
+      \`,
+    }),
+    node({
+      name: "endpoint",
+      sql: \`
+SELECT event_id AS event_id, user_id AS user_id
+FROM source
+WHERE user_id = {{UInt64(user_id)}}
+      \`,
+    }),
+  ],
+  endpoint: true,
+  output: {
+    event_id: t.string(),
+    user_id: t.string(),
+  },
+  tokens: [
+    { name: "endpoint_token" },
+  ],
+});
+`;
+describe("runMigrate", () => {
+    const tempDirs = [];
+    afterEach(() => {
+        for (const dir of tempDirs) {
+            try {
+                fs.rmSync(dir, { recursive: true });
+            }
+            catch {
+                // Ignore cleanup failures
+            }
+        }
+        tempDirs.length = 0;
+    });
+    it("migrates complex resources including endpoint, materialized, and copy pipes", async () => {
+        const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-migrate-"));
+        tempDirs.push(tempDir);
+        writeFile(tempDir, "stream.connection", `TYPE kafka
+KAFKA_BOOTSTRAP_SERVERS localhost:9092
+KAFKA_SECURITY_PROTOCOL SASL_SSL
+KAFKA_SASL_MECHANISM PLAIN
+KAFKA_KEY api-key
+KAFKA_SECRET api-secret
+KAFKA_SSL_CA_PEM ca-pem-content
+`);
+        writeFile(tempDir, "events.datasource", `DESCRIPTION >
+    Events from Kafka stream
+SCHEMA >
+    event_id String \`json:$.event_id\`,
+    user_id UInt64 \`json:$.user.id\`,
+    env String \`json:$.env\` DEFAULT 'prod',
+    is_test Bool \`json:$.meta.is_test\` DEFAULT 0,
+    updated_at DateTime \`json:$.updated_at\`,
+    payload String \`json:$.payload\` DEFAULT '{}' CODEC(ZSTD(1))
+
+ENGINE "ReplacingMergeTree"
+ENGINE_SORTING_KEY "event_id, user_id"
+ENGINE_PARTITION_KEY "toYYYYMM(updated_at)"
+ENGINE_PRIMARY_KEY "event_id"
+ENGINE_TTL "updated_at + toIntervalDay(30)"
+ENGINE_VER "updated_at"
+ENGINE_SETTINGS "index_granularity=8192, enable_mixed_granularity_parts=true"
+KAFKA_CONNECTION_NAME stream
+KAFKA_TOPIC events_topic
+KAFKA_GROUP_ID events-consumer
+KAFKA_AUTO_OFFSET_RESET earliest
+TOKEN events_read READ
+TOKEN events_append APPEND
+SHARED_WITH >
+    workspace_a,
+    workspace_b
+FORWARD_QUERY >
+    SELECT *
+    FROM events_mv
+`);
+        writeFile(tempDir, "events_rollup.datasource", `SCHEMA >
+    user_id UInt64,
+    total UInt64
+
+ENGINE "SummingMergeTree"
+ENGINE_SORTING_KEY "user_id"
+ENGINE_SUMMING_COLUMNS "total"
+`);
+        writeFile(tempDir, "events_endpoint.pipe", `DESCRIPTION >
+    Endpoint for filtered events
+NODE base
+DESCRIPTION >
+    Base filter
+SQL >
+    %
+    SELECT event_id, user_id, payload
+    FROM events
+    WHERE user_id = {{UInt64(user_id)}}
+      AND env = {{String(env, 'prod')}}
+NODE endpoint
+SQL >
+    SELECT event_id AS event_id, user_id AS user_id
+    FROM base
+TYPE endpoint
+CACHE 120
+TOKEN endpoint_token READ
+`);
+        writeFile(tempDir, "events_mv.pipe", `DESCRIPTION >
+    Materialized rollup
+NODE rollup
+SQL >
+    SELECT user_id, count() AS total
+    FROM events
+    GROUP BY user_id
+TYPE MATERIALIZED
+DATASOURCE events_rollup
+DEPLOYMENT_METHOD alter
+TOKEN mv_token READ
+`);
+        writeFile(tempDir, "copy_events.pipe", `NODE copy_node
+SQL >
+    SELECT event_id, user_id
+    FROM events
+TYPE COPY
+TARGET_DATASOURCE events_rollup
+COPY_SCHEDULE @on-demand
+COPY_MODE replace
+TOKEN copy_token READ
+`);
+        writeFile(tempDir, "stats_pipe.pipe", `NODE agg
+SQL >
+    SELECT user_id, count() AS total
+    FROM events
+    GROUP BY user_id
+NODE final
+SQL >
+    SELECT user_id, total
+    FROM agg
+    WHERE total > {{UInt32(min_total, 10)}}
+TOKEN stats_token READ
+`);
+        const result = await runMigrate({
+            cwd: tempDir,
+            patterns: ["."],
+            strict: true,
+        });
+        expect(result.success).toBe(true);
+        expect(result.errors).toHaveLength(0);
+        expect(result.migrated).toHaveLength(7);
+        expect(result.migrated.filter((resource) => resource.kind === "connection")).toHaveLength(1);
+        expect(result.migrated.filter((resource) => resource.kind === "datasource")).toHaveLength(2);
+        expect(result.migrated.filter((resource) => resource.kind === "pipe")).toHaveLength(4);
+        expect(path.basename(result.outputPath)).toBe("tinybird.migration.ts");
+        expect(fs.existsSync(result.outputPath)).toBe(true);
+        const output = fs.readFileSync(result.outputPath, "utf-8");
+        expect(output).toBe(EXPECTED_COMPLEX_OUTPUT);
+    });
+    it("continues processing and reports all errors while writing complex migratable resources", async () => {
+        const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-migrate-"));
+        tempDirs.push(tempDir);
+        writeFile(tempDir, "stream.connection", `TYPE kafka
+KAFKA_BOOTSTRAP_SERVERS localhost:9092
+`);
+        writeFile(tempDir, "events.datasource", `SCHEMA >
+    event_id String,
+    user_id UInt64,
+    created_at DateTime
+
+ENGINE "MergeTree"
+ENGINE_SORTING_KEY "event_id"
+KAFKA_CONNECTION_NAME stream
+KAFKA_TOPIC events_topic
+`);
+        writeFile(tempDir, "events_endpoint.pipe", `NODE source
+SQL >
+    SELECT event_id, user_id
+    FROM events
+NODE endpoint
+SQL >
+    SELECT event_id AS event_id, user_id AS user_id
+    FROM source
+    WHERE user_id = {{UInt64(user_id)}}
+TYPE endpoint
+TOKEN endpoint_token READ
+`);
+        writeFile(tempDir, "events_mv.pipe", `NODE rollup
+SQL >
+    SELECT user_id, count() AS total
+    FROM events
+    GROUP BY user_id
+TYPE MATERIALIZED
+DATASOURCE missing_ds
+`);
+        writeFile(tempDir, "broken.pipe", `NODE broken
+SQL >
+    SELECT *
+    FROM events
+TYPE endpoint
+UNSUPPORTED_DIRECTIVE true
+`);
+        const result = await runMigrate({
+            cwd: tempDir,
+            patterns: ["."],
+            strict: true,
+        });
+        expect(result.success).toBe(false);
+        expect(result.errors).toHaveLength(2);
+        expect(result.errors.map((error) => error.message)).toEqual(expect.arrayContaining([
+            'Unsupported pipe directive in strict mode: "UNSUPPORTED_DIRECTIVE true"',
+            'Materialized pipe references missing/unmigrated datasource "missing_ds".',
+        ]));
+        expect(result.migrated.filter((resource) => resource.kind === "connection")).toHaveLength(1);
+        expect(result.migrated.filter((resource) => resource.kind === "datasource")).toHaveLength(1);
+        expect(result.migrated.filter((resource) => resource.kind === "pipe")).toHaveLength(1);
+        expect(fs.existsSync(result.outputPath)).toBe(true);
+        const output = fs.readFileSync(result.outputPath, "utf-8");
+        expect(output).toBe(EXPECTED_PARTIAL_OUTPUT);
+    });
+    it("returns exact output content in dry-run mode for complex migratable resources", async () => {
+        const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-migrate-"));
+        tempDirs.push(tempDir);
+        writeFile(tempDir, "stream.connection", `TYPE kafka
+KAFKA_BOOTSTRAP_SERVERS localhost:9092
+`);
+        writeFile(tempDir, "events.datasource", `SCHEMA >
+    event_id String,
+    user_id UInt64,
+    created_at DateTime
+
+ENGINE "MergeTree"
+ENGINE_SORTING_KEY "event_id"
+KAFKA_CONNECTION_NAME stream
+KAFKA_TOPIC events_topic
+`);
+        writeFile(tempDir, "events_endpoint.pipe", `NODE source
+SQL >
+    SELECT event_id, user_id
+    FROM events
+NODE endpoint
+SQL >
+    SELECT event_id AS event_id, user_id AS user_id
+    FROM source
+    WHERE user_id = {{UInt64(user_id)}}
+TYPE endpoint
+TOKEN endpoint_token READ
+`);
+        const result = await runMigrate({
+            cwd: tempDir,
+            patterns: ["."],
+            strict: true,
+            dryRun: true,
+        });
+        expect(result.success).toBe(true);
+        expect(result.errors).toHaveLength(0);
+        expect(result.migrated).toHaveLength(3);
+        expect(result.outputContent).toBe(EXPECTED_PARTIAL_OUTPUT);
+        expect(fs.existsSync(result.outputPath)).toBe(false);
+    });
+});
+//# sourceMappingURL=migrate.test.js.map

package/dist/cli/commands/migrate.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"migrate.test.js","sourceRoot":"","sources":["../../../src/cli/commands/migrate.test.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AACzD,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAE1C,SAAS,SAAS,CAAC,GAAW,EAAE,YAAoB,EAAE,OAAe;IACnE,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;IAC9C,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1D,EAAE,CAAC,aAAa,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;AACtC,CAAC;AAED,MAAM,uBAAuB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAsK/B,CAAC;AAEF,MAAM,uBAAuB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA6D/B,CAAC;AAEF,QAAQ,CAAC,YAAY,EAAE,GAAG,EAAE;IAC1B,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,SAAS,CAAC,GAAG,EAAE;QACb,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;YAC3B,IAAI,CAAC;gBACH,EAAE,CAAC,MAAM,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACtC,CAAC;YAAC,MAAM,CAAC;gBACP,0BAA0B;YAC5B,CAAC;QACH,CAAC;QACD,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;IACtB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6EAA6E,EAAE,KAAK,IAAI,EAAE;QAC3F,MAAM,OAAO,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,mBAAmB,CAAC,CAAC,CAAC;QAC5E,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAEvB,SAAS,CACP,OAAO,EACP,mBAAmB,EACnB;;;;;;;CAOL,CACI,CAAC;QAEF,SAAS,CACP,OAAO,EACP,mBAAmB,EACnB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA6BL,CACI,CAAC;QAEF,SAAS,CACP,OAAO,EACP,0BAA0B,EAC1B;;;;;;;CAOL,CACI,CAAC;QAEF,SAAS,CACP,OAAO,EACP,sBAAsB,EACtB;;;;;;;;;;;;;;;;;;CAkBL,CACI,CAAC;QAEF,SAAS,CACP,OAAO,EACP,gBAAgB,EAChB;;;;;;;;;;;CAWL,CACI,CAAC;QAEF,SAAS,CACP,OAAO,EACP,kBAAkB,EAClB;;;;;;;;;CASL,CACI,CAAC;QAEF,SAAS,CACP,OAAO,EACP,iBAAiB,EACjB;;;;;;;;;;;CAWL,CACI,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC;YAC9B,GAAG,EAAE,OAAO;YACZ,QAAQ,EAAE,CAAC,GAAG,CAAC;YACf,MAAM,EAAE,IAAI;SACb,CAAC,CAAC;QAEH,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACtC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACxC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,KAAK,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC7F,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,KAAK,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC7F,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACvF,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;QACvE,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEpD,MAAM,MAAM,GAAG,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QAC3D,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wFAAwF,EAAE,KAAK,IAAI,EAAE;QACtG,MAAM,OAAO,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,mBAAmB,CAAC,CAAC,CAAC;QAC5E,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAEvB,SAAS,CACP,OAAO,EACP,mBAAmB,EACnB;;CAEL,CACI,CAAC;QAEF,SAAS,CACP,OAAO,EACP,mBAAmB,EACnB;;;;;;;;;CASL,CACI,CAAC;QAEF,SAAS,CACP,OAAO,EACP,sBAAsB,EACtB;;;;;;;;;;;CAWL,CACI,CAAC;QAEF,SAAS,CACP,OAAO,EACP,gBAAgB,EAChB;;;;;;;CAOL,CACI,CAAC;QAEF,SAAS,CACP,OAAO,EACP,aAAa,EACb;;;;;;CAML,CACI,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC;YAC9B,GAAG,EAAE,OAAO;YACZ,QAAQ,EAAE,CAAC,GAAG,CAAC;YACf,MAAM,EAAE,IAAI;SACb,CAAC,CAAC;QAEH,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACtC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CACzD,MAAM,CAAC,eAAe,CAAC;YACrB,yEAAyE;YACzE,0EAA0E;SAC3E,CAAC,CACH,CAAC;QACF,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,KAAK,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC7F,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,KAAK,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC7F,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACvF,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEpD,MAAM,MAAM,GAAG,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QAC3D,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+EAA+E,EAAE,KAAK,IAAI,EAAE;QAC7F,MAAM,OAAO,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,mBAAmB,CAAC,CAAC,CAAC;QAC5E,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAEvB,SAAS,CACP,OAAO,EACP,mBAAmB,EACnB;;CAEL,CACI,CAAC;QAEF,SAAS,CACP,OAAO,EACP,mBAAmB,EACnB;;;;;;;;;CASL,CACI,CAAC;QAEF,SAAS,CACP,OAAO,EACP,sBAAsB,EACtB;;;;;;;;;;;CAWL,CACI,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC;YAC9B,GAAG,EAAE,OAAO;YACZ,QAAQ,EAAE,CAAC,GAAG,CAAC;YACf,MAAM,EAAE,IAAI;YACZ,MAAM,EAAE,IAAI;SACb,CAAC,CAAC;QAEH,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACtC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACxC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;QAC3D,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACvD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/cli/commands/pull.d.ts

@@ -0,0 +1,59 @@
+/**
+ * Pull command - downloads all cloud resources as Tinybird datafiles
+ */
+import { type ResourceFileType } from "../../api/resources.js";
+/**
+ * Pull command options
+ */
+export interface PullCommandOptions {
+    /** Working directory (defaults to cwd) */
+    cwd?: string;
+    /** Output directory for pulled files (defaults to current directory) */
+    outputDir?: string;
+    /** Whether to overwrite existing files (defaults to false) */
+    overwrite?: boolean;
+}
+/**
+ * Single file written by pull
+ */
+export interface PulledFileResult {
+    /** Resource name */
+    name: string;
+    /** Resource type */
+    type: ResourceFileType;
+    /** Filename written */
+    filename: string;
+    /** Absolute path written */
+    path: string;
+    /** Path relative to cwd */
+    relativePath: string;
+    /** Whether this file was newly created or overwritten */
+    status: "created" | "overwritten";
+}
+/**
+ * Pull command result
+ */
+export interface PullCommandResult {
+    /** Whether pull was successful */
+    success: boolean;
+    /** Output directory used */
+    outputDir?: string;
+    /** Files written */
+    files?: PulledFileResult[];
+    /** Pull statistics */
+    stats?: {
+        datasources: number;
+        pipes: number;
+        connections: number;
+        total: number;
+    };
+    /** Error message if failed */
+    error?: string;
+    /** Duration in milliseconds */
+    durationMs: number;
+}
+/**
+ * Pull all resources from Tinybird and write them as datafiles
+ */
+export declare function runPull(options?: PullCommandOptions): Promise<PullCommandResult>;
+//# sourceMappingURL=pull.d.ts.map

package/dist/cli/commands/pull.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pull.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/pull.ts"],"names":[],"mappings":"AAAA;;GAEG;AAKH,OAAO,EAIL,KAAK,gBAAgB,EACtB,MAAM,wBAAwB,CAAC;AAEhC;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,0CAA0C;IAC1C,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,wEAAwE;IACxE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,8DAA8D;IAC9D,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,oBAAoB;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,oBAAoB;IACpB,IAAI,EAAE,gBAAgB,CAAC;IACvB,uBAAuB;IACvB,QAAQ,EAAE,MAAM,CAAC;IACjB,4BAA4B;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,2BAA2B;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,yDAAyD;IACzD,MAAM,EAAE,SAAS,GAAG,aAAa,CAAC;CACnC;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,kCAAkC;IAClC,OAAO,EAAE,OAAO,CAAC;IACjB,4BAA4B;IAC5B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,oBAAoB;IACpB,KAAK,CAAC,EAAE,gBAAgB,EAAE,CAAC;IAC3B,sBAAsB;IACtB,KAAK,CAAC,EAAE;QACN,WAAW,EAAE,MAAM,CAAC;QACpB,KAAK,EAAE,MAAM,CAAC;QACd,WAAW,EAAE,MAAM,CAAC;QACpB,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;IACF,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+BAA+B;IAC/B,UAAU,EAAE,MAAM,CAAC;CACpB;AASD;;GAEG;AACH,wBAAsB,OAAO,CAC3B,OAAO,GAAE,kBAAuB,GAC/B,OAAO,CAAC,iBAAiB,CAAC,CAiG5B"}

package/dist/cli/commands/pull.js

@@ -0,0 +1,104 @@
+/**
+ * Pull command - downloads all cloud resources as Tinybird datafiles
+ */
+import * as fs from "node:fs/promises";
+import * as path from "node:path";
+import { loadConfigAsync } from "../config.js";
+import { pullAllResourceFiles, } from "../../api/resources.js";
+/**
+ * Convert grouped resources to a flat file list
+ */
+function flattenResources(resources) {
+    return [...resources.datasources, ...resources.pipes, ...resources.connections];
+}
+/**
+ * Pull all resources from Tinybird and write them as datafiles
+ */
+export async function runPull(options = {}) {
+    const startTime = Date.now();
+    const cwd = options.cwd ?? process.cwd();
+    const outputDir = path.resolve(cwd, options.outputDir ?? ".");
+    const overwrite = options.overwrite ?? false;
+    let config;
+    try {
+        config = await loadConfigAsync(cwd);
+    }
+    catch (error) {
+        return {
+            success: false,
+            error: error.message,
+            durationMs: Date.now() - startTime,
+        };
+    }
+    let pulled;
+    try {
+        pulled = await pullAllResourceFiles({
+            baseUrl: config.baseUrl,
+            token: config.token,
+        });
+    }
+    catch (error) {
+        return {
+            success: false,
+            error: `Pull failed: ${error.message}`,
+            durationMs: Date.now() - startTime,
+        };
+    }
+    const allFiles = flattenResources(pulled).sort((a, b) => a.filename.localeCompare(b.filename));
+    try {
+        await fs.mkdir(outputDir, { recursive: true });
+        const writtenFiles = [];
+        for (const file of allFiles) {
+            const absolutePath = path.join(outputDir, file.filename);
+            let existed = false;
+            try {
+                await fs.access(absolutePath);
+                existed = true;
+            }
+            catch {
+                existed = false;
+            }
+            await fs.writeFile(absolutePath, file.content, {
+                encoding: "utf-8",
+                flag: overwrite ? "w" : "wx",
+            });
+            writtenFiles.push({
+                name: file.name,
+                type: file.type,
+                filename: file.filename,
+                path: absolutePath,
+                relativePath: path.relative(cwd, absolutePath),
+                status: existed ? "overwritten" : "created",
+            });
+        }
+        return {
+            success: true,
+            outputDir,
+            files: writtenFiles,
+            stats: {
+                datasources: pulled.datasources.length,
+                pipes: pulled.pipes.length,
+                connections: pulled.connections.length,
+                total: writtenFiles.length,
+            },
+            durationMs: Date.now() - startTime,
+        };
+    }
+    catch (error) {
+        const err = error;
+        if (err.code === "EEXIST") {
+            return {
+                success: false,
+                error: `File already exists: ${err.path ?? "unknown"}. ` +
+                    "Use --force to overwrite existing files.",
+                durationMs: Date.now() - startTime,
+            };
+        }
+        return {
+            success: false,
+            error: `Failed to write files: ${error.message}`,
+            durationMs: Date.now() - startTime,
+        };
+    }
+}
+//# sourceMappingURL=pull.js.map

package/dist/cli/commands/pull.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pull.js","sourceRoot":"","sources":["../../../src/cli/commands/pull.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACvC,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,EACL,oBAAoB,GAIrB,MAAM,wBAAwB,CAAC;AAuDhC;;GAEG;AACH,SAAS,gBAAgB,CAAC,SAA8B;IACtD,OAAO,CAAC,GAAG,SAAS,CAAC,WAAW,EAAE,GAAG,SAAS,CAAC,KAAK,EAAE,GAAG,SAAS,CAAC,WAAW,CAAC,CAAC;AAClF,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,UAA8B,EAAE;IAEhC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;IACzC,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,SAAS,IAAI,GAAG,CAAC,CAAC;IAC9D,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,KAAK,CAAC;IAE7C,IAAI,MAAmD,CAAC;IACxD,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,eAAe,CAAC,GAAG,CAAC,CAAC;IACtC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK,EAAG,KAAe,CAAC,OAAO;YAC/B,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;SACnC,CAAC;IACJ,CAAC;IAED,IAAI,MAA2B,CAAC;IAChC,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,oBAAoB,CAAC;YAClC,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,KAAK,EAAE,MAAM,CAAC,KAAK;SACpB,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK,EAAE,gBAAiB,KAAe,CAAC,OAAO,EAAE;YACjD,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;SACnC,CAAC;IACJ,CAAC;IAED,MAAM,QAAQ,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACtD,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,CACrC,CAAC;IAEF,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAE/C,MAAM,YAAY,GAAuB,EAAE,CAAC;QAE5C,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC5B,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;YACzD,IAAI,OAAO,GAAG,KAAK,CAAC;YAEpB,IAAI,CAAC;gBACH,MAAM,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;gBAC9B,OAAO,GAAG,IAAI,CAAC;YACjB,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,GAAG,KAAK,CAAC;YAClB,CAAC;YAED,MAAM,EAAE,CAAC,SAAS,CAAC,YAAY,EAAE,IAAI,CAAC,OAAO,EAAE;gBAC7C,QAAQ,EAAE,OAAO;gBACjB,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI;aAC7B,CAAC,CAAC;YAEH,YAAY,CAAC,IAAI,CAAC;gBAChB,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,IAAI,EAAE,YAAY;gBAClB,YAAY,EAAE,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,YAAY,CAAC;gBAC9C,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS;aAC5C,CAAC,CAAC;QACL,CAAC;QAED,OAAO;YACL,OAAO,EAAE,IAAI;YACb,SAAS;YACT,KAAK,EAAE,YAAY;YACnB,KAAK,EAAE;gBACL,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM;gBACtC,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM;gBAC1B,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM;gBACtC,KAAK,EAAE,YAAY,CAAC,MAAM;aAC3B;YACD,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;SACnC,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,GAAG,GAAG,KAA8B,CAAC;QAE3C,IAAI,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC1B,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EACH,wBAAwB,GAAG,CAAC,IAAI,IAAI,SAAS,IAAI;oBACjD,0CAA0C;gBAC5C,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;aACnC,CAAC;QACJ,CAAC;QAED,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK,EAAE,0BAA2B,KAAe,CAAC,OAAO,EAAE;YAC3D,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;SACnC,CAAC;IACJ,CAAC;AACH,CAAC"}

package/dist/cli/commands/pull.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=pull.test.d.ts.map

package/dist/cli/commands/pull.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pull.test.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/pull.test.ts"],"names":[],"mappings":""}