@tinybirdco/sdk 0.0.40 → 0.0.42

package/src/cli/index.ts

@@ -20,6 +20,7 @@ import { runBuild } from "./commands/build.js";
 import { runDeploy } from "./commands/deploy.js";
 import { runPreview } from "./commands/preview.js";
 import { runDev } from "./commands/dev.js";
+import { runMigrate } from "./commands/migrate.js";
 import { runLogin } from "./commands/login.js";
 import {
   runBranchList,
@@ -28,6 +29,7 @@ import {
 } from "./commands/branch.js";
 import { runClear } from "./commands/clear.js";
 import { runInfo } from "./commands/info.js";
+import { runPull } from "./commands/pull.js";
 import {
   runOpenDashboard,
   type Environment,
@@ -179,6 +181,50 @@ function createCli(): Command {
       }
     });
 
+  // Pull command
+  program
+    .command("pull")
+    .description(
+      "Download datasources, pipes, and connections from Tinybird as datafiles"
+    )
+    .option(
+      "-o, --output-dir <path>",
+      "Directory where pulled files are written",
+      "."
+    )
+    .option("-f, --force", "Overwrite existing files")
+    .action(async (options) => {
+      output.highlight("Pulling resources from Tinybird...");
+
+      const result = await runPull({
+        outputDir: options.outputDir,
+        overwrite: options.force,
+      });
+
+      if (!result.success) {
+        output.error(result.error ?? "Pull failed");
+        process.exit(1);
+      }
+
+      const files = result.files ?? [];
+      const stats = result.stats ?? {
+        datasources: 0,
+        pipes: 0,
+        connections: 0,
+        total: files.length,
+      };
+
+      for (const file of files) {
+        const status = file.status === "created" ? "created" : "changed";
+        output.showResourceChange(file.relativePath, status);
+      }
+
+      output.success(
+        `\nPulled ${stats.total} file(s): ${stats.datasources} datasource(s), ${stats.pipes} pipe(s), ${stats.connections} connection(s)`
+      );
+      output.success(`Completed in ${output.formatDuration(result.durationMs)}`);
+    });
+
   // Open command
   program
     .command("open")
@@ -314,6 +360,72 @@ function createCli(): Command {
       }
     });
 
+  // Migrate command
+  program
+    .command("migrate")
+    .description(
+      "Migrate .datasource/.pipe/.connection files into TypeScript definitions"
+    )
+    .argument(
+      "<patterns...>",
+      "One or more file paths, directories, or glob patterns"
+    )
+    .option(
+      "-o, --out <path>",
+      "Output TypeScript path (default: ./tinybird.migration.ts)"
+    )
+    .option("--dry-run", "Preview migration without writing files")
+    .option("--force", "Overwrite output file if it already exists")
+    .option(
+      "--no-strict",
+      "Do not fail command when some resources cannot be migrated"
+    )
+    .action(async (patterns: string[], options) => {
+      const result = await runMigrate({
+        patterns,
+        out: options.out,
+        strict: options.strict,
+        dryRun: options.dryRun,
+        force: options.force,
+      });
+
+      const migratedCounts = {
+        datasources: result.migrated.filter((resource) => resource.kind === "datasource")
+          .length,
+        pipes: result.migrated.filter((resource) => resource.kind === "pipe").length,
+        connections: result.migrated.filter(
+          (resource) => resource.kind === "connection"
+        ).length,
+      };
+
+      console.log(
+        `Migrated: ${migratedCounts.datasources} datasource(s), ${migratedCounts.pipes} pipe(s), ${migratedCounts.connections} connection(s)`
+      );
+
+      if (result.migrated.length > 0) {
+        if (result.dryRun) {
+          console.log(`[Dry run] Output would be written to: ${result.outputPath}`);
+        } else {
+          console.log(`Output written to: ${result.outputPath}`);
+        }
+      } else {
+        console.log("No resources were migrated.");
+      }
+
+      if (result.errors.length > 0) {
+        console.log("\nErrors:");
+        for (const error of result.errors) {
+          console.log(
+            `- ${error.filePath} (${error.resourceKind}:${error.resourceName}): ${error.message}`
+          );
+        }
+      }
+
+      if (!result.success) {
+        process.exit(1);
+      }
+    });
+
   // Deploy command
   program
     .command("deploy")

package/src/client/base.test.ts

@@ -149,13 +149,15 @@ describe("TinybirdClient", () => {
       expect(client.datasources).toBeDefined();
     });
 
-    it("datasources namespace has append method", () => {
+    it("datasources namespace has ingest/append/replace/delete/truncate methods", () => {
       const client = createClient({
         baseUrl: "https://api.tinybird.co",
         token: "test-token",
       });
 
+      expect(typeof client.datasources.ingest).toBe("function");
       expect(typeof client.datasources.append).toBe("function");
+      expect(typeof client.datasources.replace).toBe("function");
       expect(typeof client.datasources.delete).toBe("function");
       expect(typeof client.datasources.truncate).toBe("function");
     });
@@ -168,7 +170,9 @@ describe("TinybirdClient", () => {
 
       const datasources: DatasourcesNamespace = client.datasources;
       expect(datasources).toBeDefined();
+      expect(typeof datasources.ingest).toBe("function");
       expect(typeof datasources.append).toBe("function");
+      expect(typeof datasources.replace).toBe("function");
       expect(typeof datasources.delete).toBe("function");
       expect(typeof datasources.truncate).toBe("function");
     });

package/src/client/base.ts

@@ -66,7 +66,7 @@ export class TinybirdClient {
   private resolvedContext: ClientContext | null = null;
 
   /**
-   * Datasources namespace for import operations
+   * Datasources namespace for ingest and import operations
    */
   readonly datasources: DatasourcesNamespace;
 
@@ -90,9 +90,19 @@ export class TinybirdClient {
 
     // Initialize datasources namespace
     this.datasources = {
+      ingest: <T extends Record<string, unknown>>(
+        datasourceName: string,
+        event: T,
+        options: IngestOptions = {}
+      ): Promise<IngestResult> => {
+        return this.ingestDatasource(datasourceName, event, options);
+      },
       append: (datasourceName: string, options: AppendOptions): Promise<AppendResult> => {
         return this.appendDatasource(datasourceName, options);
       },
+      replace: (datasourceName: string, options: AppendOptions): Promise<AppendResult> => {
+        return this.replaceDatasource(datasourceName, options);
+      },
       delete: (datasourceName: string, options: DeleteOptions): Promise<DeleteResult> => {
         return this.deleteDatasource(datasourceName, options);
       },
@@ -146,6 +156,28 @@ export class TinybirdClient {
     }
   }
 
+  /**
+   * Replace datasource rows from a URL or local file
+   *
+   * @param datasourceName - Name of the datasource
+   * @param options - Replace options including url or file source
+   * @returns Append-style result
+   */
+  private async replaceDatasource(
+    datasourceName: string,
+    options: AppendOptions
+  ): Promise<AppendResult> {
+    const token = await this.getToken();
+
+    try {
+      return await this.getApi(token).appendDatasource(datasourceName, options, {
+        mode: "replace",
+      });
+    } catch (error) {
+      this.rethrowApiError(error);
+    }
+  }
+
   /**
    * Delete rows from a datasource using a SQL condition
    *
@@ -186,6 +218,28 @@ export class TinybirdClient {
     }
   }
 
+  /**
+   * Ingest a single event to a datasource
+   *
+   * @param datasourceName - Name of the datasource
+   * @param event - Event data to ingest
+   * @param options - Additional request options
+   * @returns Ingest result
+   */
+  private async ingestDatasource<T extends Record<string, unknown>>(
+    datasourceName: string,
+    event: T,
+    options: IngestOptions = {}
+  ): Promise<IngestResult> {
+    const token = await this.getToken();
+
+    try {
+      return await this.getApi(token).ingestBatch(datasourceName, [event], options);
+    } catch (error) {
+      this.rethrowApiError(error);
+    }
+  }
+
   /**
    * Get the effective token, resolving branch token in dev mode if needed
    */
@@ -351,7 +405,7 @@ export class TinybirdClient {
     event: T,
     options: IngestOptions = {}
   ): Promise<IngestResult> {
-    return this.ingestBatch(datasourceName, [event], options);
+    return this.datasources.ingest(datasourceName, event, options);
   }
 
   /**

package/src/client/types.ts

@@ -290,8 +290,16 @@ export interface TruncateResult {
  * Datasources namespace interface for raw client
  */
 export interface DatasourcesNamespace {
+  /** Ingest a single event into a datasource */
+  ingest<T extends Record<string, unknown>>(
+    datasourceName: string,
+    event: T,
+    options?: IngestOptions
+  ): Promise<IngestResult>;
   /** Append data to a datasource from a URL or file */
   append(datasourceName: string, options: AppendOptions): Promise<AppendResult>;
+  /** Replace all datasource rows with data from a URL or file */
+  replace(datasourceName: string, options: AppendOptions): Promise<AppendResult>;
   /** Delete rows from a datasource using a SQL condition */
   delete(datasourceName: string, options: DeleteOptions): Promise<DeleteResult>;
   /** Truncate all rows from a datasource */

package/src/migrate/discovery.ts

@@ -0,0 +1,151 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { resolveIncludeFiles } from "../generator/include-paths.js";
+import type { MigrationError, ResourceFile, ResourceKind } from "./types.js";
+
+const SUPPORTED_EXTENSIONS = new Map<string, ResourceKind>([
+  [".datasource", "datasource"],
+  [".pipe", "pipe"],
+  [".connection", "connection"],
+]);
+
+function normalizePath(filePath: string): string {
+  return filePath.replace(/\\/g, "/");
+}
+
+function getKindFromPath(filePath: string): ResourceKind | null {
+  const ext = path.extname(filePath).toLowerCase();
+  return SUPPORTED_EXTENSIONS.get(ext) ?? null;
+}
+
+function collectDirectoryFiles(directory: string): string[] {
+  const entries = fs.readdirSync(directory, { withFileTypes: true });
+  const files: string[] = [];
+
+  for (const entry of entries) {
+    const fullPath = path.join(directory, entry.name);
+
+    if (entry.isDirectory()) {
+      if (entry.name === "node_modules" || entry.name === ".git") {
+        continue;
+      }
+      files.push(...collectDirectoryFiles(fullPath));
+      continue;
+    }
+
+    if (entry.isFile()) {
+      files.push(fullPath);
+    }
+  }
+
+  return files;
+}
+
+export interface DiscoverResourcesResult {
+  resources: ResourceFile[];
+  errors: MigrationError[];
+}
+
+export function discoverResourceFiles(
+  patterns: string[],
+  cwd: string
+): DiscoverResourcesResult {
+  const resources: ResourceFile[] = [];
+  const errors: MigrationError[] = [];
+  const seen = new Set<string>();
+
+  for (const pattern of patterns) {
+    const absolutePattern = path.isAbsolute(pattern)
+      ? pattern
+      : path.resolve(cwd, pattern);
+
+    if (fs.existsSync(absolutePattern)) {
+      const stat = fs.statSync(absolutePattern);
+
+      if (stat.isDirectory()) {
+        const directoryFiles = collectDirectoryFiles(absolutePattern);
+        for (const absoluteFilePath of directoryFiles) {
+          const kind = getKindFromPath(absoluteFilePath);
+          if (!kind) {
+            continue;
+          }
+
+          const key = normalizePath(absoluteFilePath);
+          if (seen.has(key)) {
+            continue;
+          }
+          seen.add(key);
+
+          const relativePath = normalizePath(path.relative(cwd, absoluteFilePath));
+          const name = path.basename(absoluteFilePath, path.extname(absoluteFilePath));
+          resources.push({
+            kind,
+            filePath: relativePath,
+            absolutePath: absoluteFilePath,
+            name,
+            content: fs.readFileSync(absoluteFilePath, "utf-8"),
+          });
+        }
+        continue;
+      }
+
+      const kind = getKindFromPath(absolutePattern);
+      if (!kind) {
+        errors.push({
+          filePath: pattern,
+          resourceName: path.basename(absolutePattern),
+          resourceKind: "datasource",
+          message: `Unsupported file extension: ${path.extname(absolutePattern) || "(none)"}. Use .datasource, .pipe, or .connection.`,
+        });
+        continue;
+      }
+
+      const key = normalizePath(absolutePattern);
+      if (seen.has(key)) {
+        continue;
+      }
+      seen.add(key);
+      resources.push({
+        kind,
+        filePath: normalizePath(path.relative(cwd, absolutePattern)),
+        absolutePath: absolutePattern,
+        name: path.basename(absolutePattern, path.extname(absolutePattern)),
+        content: fs.readFileSync(absolutePattern, "utf-8"),
+      });
+      continue;
+    }
+
+    try {
+      const matched = resolveIncludeFiles([pattern], cwd);
+      for (const entry of matched) {
+        const kind = getKindFromPath(entry.absolutePath);
+        if (!kind) {
+          continue;
+        }
+        const key = normalizePath(entry.absolutePath);
+        if (seen.has(key)) {
+          continue;
+        }
+        seen.add(key);
+        resources.push({
+          kind,
+          filePath: normalizePath(entry.sourcePath),
+          absolutePath: entry.absolutePath,
+          name: path.basename(entry.absolutePath, path.extname(entry.absolutePath)),
+          content: fs.readFileSync(entry.absolutePath, "utf-8"),
+        });
+      }
+    } catch (error) {
+      errors.push({
+        filePath: pattern,
+        resourceName: path.basename(pattern),
+        resourceKind: "datasource",
+        message: (error as Error).message,
+      });
+    }
+  }
+
+  resources.sort((a, b) => a.filePath.localeCompare(b.filePath));
+  return { resources, errors };
+}
+