@tinybirdco/sdk 0.0.41 → 0.0.43

package/src/cli/commands/migrate.ts

@@ -0,0 +1,240 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { discoverResourceFiles } from "../../migrate/discovery.js";
+import { emitMigrationFileContent, validateResourceForEmission } from "../../migrate/emit-ts.js";
+import { parseResourceFile } from "../../migrate/parse.js";
+import { MigrationParseError } from "../../migrate/parser-utils.js";
+import type {
+  MigrationError,
+  MigrationResult,
+  ParsedResource,
+  ResourceFile,
+} from "../../migrate/types.js";
+
+export interface MigrateCommandOptions {
+  cwd?: string;
+  patterns: string[];
+  out?: string;
+  strict?: boolean;
+  dryRun?: boolean;
+  force?: boolean;
+}
+
+function toMigrationError(resource: ResourceFile, error: unknown): MigrationError {
+  const message = (error as Error).message || String(error);
+  return {
+    filePath: resource.filePath,
+    resourceName: resource.name,
+    resourceKind: resource.kind,
+    message,
+  };
+}
+
+function sortResourcesForOutput(resources: ParsedResource[]): ParsedResource[] {
+  const order: Record<ParsedResource["kind"], number> = {
+    connection: 0,
+    datasource: 1,
+    pipe: 2,
+  };
+  return [...resources].sort((a, b) => {
+    const byType = order[a.kind] - order[b.kind];
+    if (byType !== 0) {
+      return byType;
+    }
+    return a.name.localeCompare(b.name);
+  });
+}
+
+export async function runMigrate(
+  options: MigrateCommandOptions
+): Promise<MigrationResult> {
+  const cwd = options.cwd ?? process.cwd();
+  const strict = options.strict ?? true;
+  const dryRun = options.dryRun ?? false;
+  const force = options.force ?? false;
+  const outputPath = path.isAbsolute(options.out ?? "")
+    ? (options.out as string)
+    : path.resolve(cwd, options.out ?? "tinybird.migration.ts");
+
+  const errors: MigrationError[] = [];
+
+  if (!options.patterns || options.patterns.length === 0) {
+    return {
+      success: false,
+      outputPath,
+      migrated: [],
+      errors: [
+        {
+          filePath: ".",
+          resourceName: "patterns",
+          resourceKind: "datasource",
+          message: "At least one file, directory, or glob pattern is required.",
+        },
+      ],
+      dryRun,
+    };
+  }
+
+  const discovered = discoverResourceFiles(options.patterns, cwd);
+  errors.push(...discovered.errors);
+
+  const parsedResources: ParsedResource[] = [];
+  for (const resource of discovered.resources) {
+    try {
+      parsedResources.push(parseResourceFile(resource));
+    } catch (error) {
+      if (error instanceof MigrationParseError) {
+        errors.push({
+          filePath: error.filePath,
+          resourceName: error.resourceName,
+          resourceKind: error.resourceKind,
+          message: error.message,
+        });
+      } else {
+        errors.push(toMigrationError(resource, error));
+      }
+    }
+  }
+
+  const parsedConnections = parsedResources.filter(
+    (resource): resource is Extract<ParsedResource, { kind: "connection" }> =>
+      resource.kind === "connection"
+  );
+  const parsedDatasources = parsedResources.filter(
+    (resource): resource is Extract<ParsedResource, { kind: "datasource" }> =>
+      resource.kind === "datasource"
+  );
+  const parsedPipes = parsedResources.filter(
+    (resource): resource is Extract<ParsedResource, { kind: "pipe" }> =>
+      resource.kind === "pipe"
+  );
+
+  const migrated: ParsedResource[] = [];
+  const migratedConnectionNames = new Set<string>();
+  const migratedDatasourceNames = new Set<string>();
+
+  for (const connection of parsedConnections) {
+    try {
+      validateResourceForEmission(connection);
+      migrated.push(connection);
+      migratedConnectionNames.add(connection.name);
+    } catch (error) {
+      errors.push({
+        filePath: connection.filePath,
+        resourceName: connection.name,
+        resourceKind: connection.kind,
+        message: (error as Error).message,
+      });
+    }
+  }
+
+  for (const datasource of parsedDatasources) {
+    if (
+      datasource.kafka &&
+      !migratedConnectionNames.has(datasource.kafka.connectionName)
+    ) {
+      errors.push({
+        filePath: datasource.filePath,
+        resourceName: datasource.name,
+        resourceKind: datasource.kind,
+        message: `Datasource references missing/unmigrated connection "${datasource.kafka.connectionName}".`,
+      });
+      continue;
+    }
+
+    try {
+      validateResourceForEmission(datasource);
+      migrated.push(datasource);
+      migratedDatasourceNames.add(datasource.name);
+    } catch (error) {
+      errors.push({
+        filePath: datasource.filePath,
+        resourceName: datasource.name,
+        resourceKind: datasource.kind,
+        message: (error as Error).message,
+      });
+    }
+  }
+
+  for (const pipe of parsedPipes) {
+    if (
+      pipe.type === "materialized" &&
+      (!pipe.materializedDatasource ||
+        !migratedDatasourceNames.has(pipe.materializedDatasource))
+    ) {
+      errors.push({
+        filePath: pipe.filePath,
+        resourceName: pipe.name,
+        resourceKind: pipe.kind,
+        message: `Materialized pipe references missing/unmigrated datasource "${pipe.materializedDatasource ?? "(none)"}".`,
+      });
+      continue;
+    }
+
+    if (
+      pipe.type === "copy" &&
+      (!pipe.copyTargetDatasource ||
+        !migratedDatasourceNames.has(pipe.copyTargetDatasource))
+    ) {
+      errors.push({
+        filePath: pipe.filePath,
+        resourceName: pipe.name,
+        resourceKind: pipe.kind,
+        message: `Copy pipe references missing/unmigrated datasource "${pipe.copyTargetDatasource ?? "(none)"}".`,
+      });
+      continue;
+    }
+
+    try {
+      validateResourceForEmission(pipe);
+      migrated.push(pipe);
+    } catch (error) {
+      errors.push({
+        filePath: pipe.filePath,
+        resourceName: pipe.name,
+        resourceKind: pipe.kind,
+        message: (error as Error).message,
+      });
+    }
+  }
+
+  const sortedMigrated = sortResourcesForOutput(migrated);
+  let outputContent: string | undefined;
+
+  if (sortedMigrated.length > 0) {
+    try {
+      outputContent = emitMigrationFileContent(sortedMigrated);
+    } catch (error) {
+      errors.push({
+        filePath: ".",
+        resourceName: "output",
+        resourceKind: "datasource",
+        message: `Failed to emit migration output: ${(error as Error).message}`,
+      });
+    }
+  }
+
+  if (!dryRun && outputContent) {
+    if (fs.existsSync(outputPath) && !force) {
+      errors.push({
+        filePath: path.relative(cwd, outputPath),
+        resourceName: path.basename(outputPath),
+        resourceKind: "datasource",
+        message: `Output file already exists: ${outputPath}. Use --force to overwrite.`,
+      });
+    } else {
+      fs.writeFileSync(outputPath, outputContent);
+    }
+  }
+
+  const success = strict ? errors.length === 0 : true;
+  return {
+    success,
+    outputPath,
+    migrated: sortedMigrated,
+    errors,
+    dryRun,
+    outputContent,
+  };
+}
+

package/src/cli/commands/pull.test.ts

@@ -0,0 +1,173 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
+import * as fs from "node:fs/promises";
+import * as os from "node:os";
+import * as path from "node:path";
+import { runPull } from "./pull.js";
+
+vi.mock("../config.js", () => ({
+  loadConfigAsync: vi.fn(),
+}));
+
+vi.mock("../../api/resources.js", () => ({
+  pullAllResourceFiles: vi.fn(),
+}));
+
+import { loadConfigAsync } from "../config.js";
+import { pullAllResourceFiles } from "../../api/resources.js";
+
+const mockedLoadConfigAsync = vi.mocked(loadConfigAsync);
+const mockedPullAllResourceFiles = vi.mocked(pullAllResourceFiles);
+
+describe("Pull Command", () => {
+  let tempDir: string;
+
+  beforeEach(async () => {
+    vi.clearAllMocks();
+    tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "tinybird-pull-test-"));
+
+    mockedLoadConfigAsync.mockResolvedValue({
+      include: ["src/lib/tinybird.ts"],
+      token: "p.test-token",
+      baseUrl: "https://api.tinybird.co",
+      configPath: path.join(tempDir, "tinybird.config.json"),
+      cwd: tempDir,
+      gitBranch: "feature/pull",
+      tinybirdBranch: "feature_pull",
+      isMainBranch: false,
+      devMode: "branch",
+    });
+  });
+
+  afterEach(async () => {
+    await fs.rm(tempDir, { recursive: true, force: true });
+    vi.resetAllMocks();
+  });
+
+  it("writes pulled datasource, pipe, and connection files", async () => {
+    mockedPullAllResourceFiles.mockResolvedValue({
+      datasources: [
+        {
+          name: "events",
+          type: "datasource",
+          filename: "events.datasource",
+          content: "SCHEMA >\n    timestamp DateTime",
+        },
+      ],
+      pipes: [
+        {
+          name: "top_events",
+          type: "pipe",
+          filename: "top_events.pipe",
+          content: "NODE endpoint\nSQL >\n    SELECT 1",
+        },
+      ],
+      connections: [
+        {
+          name: "main_kafka",
+          type: "connection",
+          filename: "main_kafka.connection",
+          content: "TYPE kafka",
+        },
+      ],
+    });
+
+    const result = await runPull({ cwd: tempDir, outputDir: "pulled" });
+
+    expect(result.success).toBe(true);
+    expect(result.stats).toEqual({
+      datasources: 1,
+      pipes: 1,
+      connections: 1,
+      total: 3,
+    });
+
+    const outputPath = path.join(tempDir, "pulled");
+    await expect(fs.readFile(path.join(outputPath, "events.datasource"), "utf-8")).resolves.toContain(
+      "SCHEMA >"
+    );
+    await expect(fs.readFile(path.join(outputPath, "top_events.pipe"), "utf-8")).resolves.toContain(
+      "NODE endpoint"
+    );
+    await expect(
+      fs.readFile(path.join(outputPath, "main_kafka.connection"), "utf-8")
+    ).resolves.toContain("TYPE kafka");
+  });
+
+  it("returns error when a file exists and overwrite is disabled", async () => {
+    mockedPullAllResourceFiles.mockResolvedValue({
+      datasources: [
+        {
+          name: "events",
+          type: "datasource",
+          filename: "events.datasource",
+          content: "SCHEMA >\n    timestamp DateTime",
+        },
+      ],
+      pipes: [],
+      connections: [],
+    });
+
+    const outputPath = path.join(tempDir, "pulled");
+    await fs.mkdir(outputPath, { recursive: true });
+    await fs.writeFile(path.join(outputPath, "events.datasource"), "old", "utf-8");
+
+    const result = await runPull({
+      cwd: tempDir,
+      outputDir: "pulled",
+      overwrite: false,
+    });
+
+    expect(result.success).toBe(false);
+    expect(result.error).toContain("File already exists");
+  });
+
+  it("overwrites existing files when overwrite is enabled", async () => {
+    mockedPullAllResourceFiles.mockResolvedValue({
+      datasources: [
+        {
+          name: "events",
+          type: "datasource",
+          filename: "events.datasource",
+          content: "new-content",
+        },
+      ],
+      pipes: [],
+      connections: [],
+    });
+
+    const outputPath = path.join(tempDir, "pulled");
+    await fs.mkdir(outputPath, { recursive: true });
+    await fs.writeFile(path.join(outputPath, "events.datasource"), "old-content", "utf-8");
+
+    const result = await runPull({
+      cwd: tempDir,
+      outputDir: "pulled",
+      overwrite: true,
+    });
+
+    expect(result.success).toBe(true);
+    expect(result.files?.[0]?.status).toBe("overwritten");
+    await expect(fs.readFile(path.join(outputPath, "events.datasource"), "utf-8")).resolves.toBe(
+      "new-content"
+    );
+  });
+
+  it("returns error when config loading fails", async () => {
+    mockedLoadConfigAsync.mockRejectedValue(new Error("No config found"));
+
+    const result = await runPull({ cwd: tempDir });
+
+    expect(result.success).toBe(false);
+    expect(result.error).toContain("No config found");
+  });
+
+  it("returns error when pull API fails", async () => {
+    mockedPullAllResourceFiles.mockRejectedValue(new Error("Unauthorized"));
+
+    const result = await runPull({ cwd: tempDir });
+
+    expect(result.success).toBe(false);
+    expect(result.error).toContain("Pull failed");
+    expect(result.error).toContain("Unauthorized");
+  });
+});

package/src/cli/commands/pull.ts

@@ -0,0 +1,177 @@
+/**
+ * Pull command - downloads all cloud resources as Tinybird datafiles
+ */
+
+import * as fs from "node:fs/promises";
+import * as path from "node:path";
+import { loadConfigAsync } from "../config.js";
+import {
+  pullAllResourceFiles,
+  type PulledResourceFiles,
+  type ResourceFile,
+  type ResourceFileType,
+} from "../../api/resources.js";
+
+/**
+ * Pull command options
+ */
+export interface PullCommandOptions {
+  /** Working directory (defaults to cwd) */
+  cwd?: string;
+  /** Output directory for pulled files (defaults to current directory) */
+  outputDir?: string;
+  /** Whether to overwrite existing files (defaults to false) */
+  overwrite?: boolean;
+}
+
+/**
+ * Single file written by pull
+ */
+export interface PulledFileResult {
+  /** Resource name */
+  name: string;
+  /** Resource type */
+  type: ResourceFileType;
+  /** Filename written */
+  filename: string;
+  /** Absolute path written */
+  path: string;
+  /** Path relative to cwd */
+  relativePath: string;
+  /** Whether this file was newly created or overwritten */
+  status: "created" | "overwritten";
+}
+
+/**
+ * Pull command result
+ */
+export interface PullCommandResult {
+  /** Whether pull was successful */
+  success: boolean;
+  /** Output directory used */
+  outputDir?: string;
+  /** Files written */
+  files?: PulledFileResult[];
+  /** Pull statistics */
+  stats?: {
+    datasources: number;
+    pipes: number;
+    connections: number;
+    total: number;
+  };
+  /** Error message if failed */
+  error?: string;
+  /** Duration in milliseconds */
+  durationMs: number;
+}
+
+/**
+ * Convert grouped resources to a flat file list
+ */
+function flattenResources(resources: PulledResourceFiles): ResourceFile[] {
+  return [...resources.datasources, ...resources.pipes, ...resources.connections];
+}
+
+/**
+ * Pull all resources from Tinybird and write them as datafiles
+ */
+export async function runPull(
+  options: PullCommandOptions = {}
+): Promise<PullCommandResult> {
+  const startTime = Date.now();
+  const cwd = options.cwd ?? process.cwd();
+  const outputDir = path.resolve(cwd, options.outputDir ?? ".");
+  const overwrite = options.overwrite ?? false;
+
+  let config: Awaited<ReturnType<typeof loadConfigAsync>>;
+  try {
+    config = await loadConfigAsync(cwd);
+  } catch (error) {
+    return {
+      success: false,
+      error: (error as Error).message,
+      durationMs: Date.now() - startTime,
+    };
+  }
+
+  let pulled: PulledResourceFiles;
+  try {
+    pulled = await pullAllResourceFiles({
+      baseUrl: config.baseUrl,
+      token: config.token,
+    });
+  } catch (error) {
+    return {
+      success: false,
+      error: `Pull failed: ${(error as Error).message}`,
+      durationMs: Date.now() - startTime,
+    };
+  }
+
+  const allFiles = flattenResources(pulled).sort((a, b) =>
+    a.filename.localeCompare(b.filename)
+  );
+
+  try {
+    await fs.mkdir(outputDir, { recursive: true });
+
+    const writtenFiles: PulledFileResult[] = [];
+
+    for (const file of allFiles) {
+      const absolutePath = path.join(outputDir, file.filename);
+      let existed = false;
+
+      try {
+        await fs.access(absolutePath);
+        existed = true;
+      } catch {
+        existed = false;
+      }
+
+      await fs.writeFile(absolutePath, file.content, {
+        encoding: "utf-8",
+        flag: overwrite ? "w" : "wx",
+      });
+
+      writtenFiles.push({
+        name: file.name,
+        type: file.type,
+        filename: file.filename,
+        path: absolutePath,
+        relativePath: path.relative(cwd, absolutePath),
+        status: existed ? "overwritten" : "created",
+      });
+    }
+
+    return {
+      success: true,
+      outputDir,
+      files: writtenFiles,
+      stats: {
+        datasources: pulled.datasources.length,
+        pipes: pulled.pipes.length,
+        connections: pulled.connections.length,
+        total: writtenFiles.length,
+      },
+      durationMs: Date.now() - startTime,
+    };
+  } catch (error) {
+    const err = error as NodeJS.ErrnoException;
+
+    if (err.code === "EEXIST") {
+      return {
+        success: false,
+        error:
+          `File already exists: ${err.path ?? "unknown"}. ` +
+          "Use --force to overwrite existing files.",
+        durationMs: Date.now() - startTime,
+      };
+    }
+
+    return {
+      success: false,
+      error: `Failed to write files: ${(error as Error).message}`,
+      durationMs: Date.now() - startTime,
+    };
+  }
+}

package/src/cli/config.ts

@@ -4,6 +4,7 @@
 
 import * as fs from "fs";
 import * as path from "path";
+import { config as loadDotenv } from "dotenv";
 import { getCurrentGitBranch, isMainBranch, getTinybirdBranchName } from "./git.js";
 
 // Re-export types from config-types.ts (separate file to avoid bundling esbuild)
@@ -70,6 +71,28 @@ const DEFAULT_CONFIG_FILE = "tinybird.config.json";
  */
 const TINYBIRD_FILE = "lib/tinybird.ts";
 
+/**
+ * Load .env files from a directory.
+ *
+ * Priority:
+ * 1. .env.local
+ * 2. .env
+ *
+ * Existing process.env values are preserved (dotenv default behavior).
+ */
+export function loadEnvFiles(directory: string): void {
+  const envLocalPath = path.join(directory, ".env.local");
+  const envPath = path.join(directory, ".env");
+
+  if (fs.existsSync(envLocalPath)) {
+    loadDotenv({ path: envLocalPath });
+  }
+
+  if (fs.existsSync(envPath)) {
+    loadDotenv({ path: envPath });
+  }
+}
+
 /**
  * Detect if project has a src folder
  */
@@ -201,6 +224,9 @@ function resolveConfig(config: TinybirdConfig, configPath: string): ResolvedConf
   // Get the directory containing the config file
   const configDir = path.dirname(configPath);
 
+  // Load environment files next to the config before interpolating values.
+  loadEnvFiles(configDir);
+
   // Resolve token (may contain env vars)
   let resolvedToken: string;
   try {