@tinybirdco/sdk 0.0.41 → 0.0.42

package/src/cli/index.ts

@@ -20,6 +20,7 @@ import { runBuild } from "./commands/build.js";
 import { runDeploy } from "./commands/deploy.js";
 import { runPreview } from "./commands/preview.js";
 import { runDev } from "./commands/dev.js";
+import { runMigrate } from "./commands/migrate.js";
 import { runLogin } from "./commands/login.js";
 import {
   runBranchList,
@@ -28,6 +29,7 @@ import {
 } from "./commands/branch.js";
 import { runClear } from "./commands/clear.js";
 import { runInfo } from "./commands/info.js";
+import { runPull } from "./commands/pull.js";
 import {
   runOpenDashboard,
   type Environment,
@@ -179,6 +181,50 @@ function createCli(): Command {
       }
     });
 
+  // Pull command
+  program
+    .command("pull")
+    .description(
+      "Download datasources, pipes, and connections from Tinybird as datafiles"
+    )
+    .option(
+      "-o, --output-dir <path>",
+      "Directory where pulled files are written",
+      "."
+    )
+    .option("-f, --force", "Overwrite existing files")
+    .action(async (options) => {
+      output.highlight("Pulling resources from Tinybird...");
+
+      const result = await runPull({
+        outputDir: options.outputDir,
+        overwrite: options.force,
+      });
+
+      if (!result.success) {
+        output.error(result.error ?? "Pull failed");
+        process.exit(1);
+      }
+
+      const files = result.files ?? [];
+      const stats = result.stats ?? {
+        datasources: 0,
+        pipes: 0,
+        connections: 0,
+        total: files.length,
+      };
+
+      for (const file of files) {
+        const status = file.status === "created" ? "created" : "changed";
+        output.showResourceChange(file.relativePath, status);
+      }
+
+      output.success(
+        `\nPulled ${stats.total} file(s): ${stats.datasources} datasource(s), ${stats.pipes} pipe(s), ${stats.connections} connection(s)`
+      );
+      output.success(`Completed in ${output.formatDuration(result.durationMs)}`);
+    });
+
   // Open command
   program
     .command("open")
@@ -314,6 +360,72 @@ function createCli(): Command {
       }
     });
 
+  // Migrate command
+  program
+    .command("migrate")
+    .description(
+      "Migrate .datasource/.pipe/.connection files into TypeScript definitions"
+    )
+    .argument(
+      "<patterns...>",
+      "One or more file paths, directories, or glob patterns"
+    )
+    .option(
+      "-o, --out <path>",
+      "Output TypeScript path (default: ./tinybird.migration.ts)"
+    )
+    .option("--dry-run", "Preview migration without writing files")
+    .option("--force", "Overwrite output file if it already exists")
+    .option(
+      "--no-strict",
+      "Do not fail command when some resources cannot be migrated"
+    )
+    .action(async (patterns: string[], options) => {
+      const result = await runMigrate({
+        patterns,
+        out: options.out,
+        strict: options.strict,
+        dryRun: options.dryRun,
+        force: options.force,
+      });
+
+      const migratedCounts = {
+        datasources: result.migrated.filter((resource) => resource.kind === "datasource")
+          .length,
+        pipes: result.migrated.filter((resource) => resource.kind === "pipe").length,
+        connections: result.migrated.filter(
+          (resource) => resource.kind === "connection"
+        ).length,
+      };
+
+      console.log(
+        `Migrated: ${migratedCounts.datasources} datasource(s), ${migratedCounts.pipes} pipe(s), ${migratedCounts.connections} connection(s)`
+      );
+
+      if (result.migrated.length > 0) {
+        if (result.dryRun) {
+          console.log(`[Dry run] Output would be written to: ${result.outputPath}`);
+        } else {
+          console.log(`Output written to: ${result.outputPath}`);
+        }
+      } else {
+        console.log("No resources were migrated.");
+      }
+
+      if (result.errors.length > 0) {
+        console.log("\nErrors:");
+        for (const error of result.errors) {
+          console.log(
+            `- ${error.filePath} (${error.resourceKind}:${error.resourceName}): ${error.message}`
+          );
+        }
+      }
+
+      if (!result.success) {
+        process.exit(1);
+      }
+    });
+
   // Deploy command
   program
     .command("deploy")

package/src/migrate/discovery.ts

@@ -0,0 +1,151 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { resolveIncludeFiles } from "../generator/include-paths.js";
+import type { MigrationError, ResourceFile, ResourceKind } from "./types.js";
+
+const SUPPORTED_EXTENSIONS = new Map<string, ResourceKind>([
+  [".datasource", "datasource"],
+  [".pipe", "pipe"],
+  [".connection", "connection"],
+]);
+
+function normalizePath(filePath: string): string {
+  return filePath.replace(/\\/g, "/");
+}
+
+function getKindFromPath(filePath: string): ResourceKind | null {
+  const ext = path.extname(filePath).toLowerCase();
+  return SUPPORTED_EXTENSIONS.get(ext) ?? null;
+}
+
+function collectDirectoryFiles(directory: string): string[] {
+  const entries = fs.readdirSync(directory, { withFileTypes: true });
+  const files: string[] = [];
+
+  for (const entry of entries) {
+    const fullPath = path.join(directory, entry.name);
+
+    if (entry.isDirectory()) {
+      if (entry.name === "node_modules" || entry.name === ".git") {
+        continue;
+      }
+      files.push(...collectDirectoryFiles(fullPath));
+      continue;
+    }
+
+    if (entry.isFile()) {
+      files.push(fullPath);
+    }
+  }
+
+  return files;
+}
+
+export interface DiscoverResourcesResult {
+  resources: ResourceFile[];
+  errors: MigrationError[];
+}
+
+export function discoverResourceFiles(
+  patterns: string[],
+  cwd: string
+): DiscoverResourcesResult {
+  const resources: ResourceFile[] = [];
+  const errors: MigrationError[] = [];
+  const seen = new Set<string>();
+
+  for (const pattern of patterns) {
+    const absolutePattern = path.isAbsolute(pattern)
+      ? pattern
+      : path.resolve(cwd, pattern);
+
+    if (fs.existsSync(absolutePattern)) {
+      const stat = fs.statSync(absolutePattern);
+
+      if (stat.isDirectory()) {
+        const directoryFiles = collectDirectoryFiles(absolutePattern);
+        for (const absoluteFilePath of directoryFiles) {
+          const kind = getKindFromPath(absoluteFilePath);
+          if (!kind) {
+            continue;
+          }
+
+          const key = normalizePath(absoluteFilePath);
+          if (seen.has(key)) {
+            continue;
+          }
+          seen.add(key);
+
+          const relativePath = normalizePath(path.relative(cwd, absoluteFilePath));
+          const name = path.basename(absoluteFilePath, path.extname(absoluteFilePath));
+          resources.push({
+            kind,
+            filePath: relativePath,
+            absolutePath: absoluteFilePath,
+            name,
+            content: fs.readFileSync(absoluteFilePath, "utf-8"),
+          });
+        }
+        continue;
+      }
+
+      const kind = getKindFromPath(absolutePattern);
+      if (!kind) {
+        errors.push({
+          filePath: pattern,
+          resourceName: path.basename(absolutePattern),
+          resourceKind: "datasource",
+          message: `Unsupported file extension: ${path.extname(absolutePattern) || "(none)"}. Use .datasource, .pipe, or .connection.`,
+        });
+        continue;
+      }
+
+      const key = normalizePath(absolutePattern);
+      if (seen.has(key)) {
+        continue;
+      }
+      seen.add(key);
+      resources.push({
+        kind,
+        filePath: normalizePath(path.relative(cwd, absolutePattern)),
+        absolutePath: absolutePattern,
+        name: path.basename(absolutePattern, path.extname(absolutePattern)),
+        content: fs.readFileSync(absolutePattern, "utf-8"),
+      });
+      continue;
+    }
+
+    try {
+      const matched = resolveIncludeFiles([pattern], cwd);
+      for (const entry of matched) {
+        const kind = getKindFromPath(entry.absolutePath);
+        if (!kind) {
+          continue;
+        }
+        const key = normalizePath(entry.absolutePath);
+        if (seen.has(key)) {
+          continue;
+        }
+        seen.add(key);
+        resources.push({
+          kind,
+          filePath: normalizePath(entry.sourcePath),
+          absolutePath: entry.absolutePath,
+          name: path.basename(entry.absolutePath, path.extname(entry.absolutePath)),
+          content: fs.readFileSync(entry.absolutePath, "utf-8"),
+        });
+      }
+    } catch (error) {
+      errors.push({
+        filePath: pattern,
+        resourceName: path.basename(pattern),
+        resourceKind: "datasource",
+        message: (error as Error).message,
+      });
+    }
+  }
+
+  resources.sort((a, b) => a.filePath.localeCompare(b.filePath));
+  return { resources, errors };
+}
+