@tinybirdco/sdk 0.0.56 → 0.0.58

package/src/cli/commands/generate.test.ts

@@ -0,0 +1,180 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import { runGenerate } from "./generate.js";
+
+vi.mock("../config.js", () => ({
+  loadConfigAsync: vi.fn(),
+}));
+
+vi.mock("../../generator/index.js", () => ({
+  buildFromInclude: vi.fn(),
+}));
+
+describe("Generate command", () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  afterEach(() => {
+    vi.resetAllMocks();
+  });
+
+  it("returns generated artifacts with stable relative paths", async () => {
+    const { loadConfigAsync } = await import("../config.js");
+    const { buildFromInclude } = await import("../../generator/index.js");
+
+    vi.mocked(loadConfigAsync).mockResolvedValue({
+      include: ["lib/tinybird.ts"],
+      token: "p.test-token",
+      baseUrl: "https://api.tinybird.co",
+      configPath: "/tmp/tinybird.config.json",
+      cwd: "/tmp",
+      gitBranch: "feature-x",
+      tinybirdBranch: "feature_x",
+      isMainBranch: false,
+      devMode: "branch",
+    });
+
+    vi.mocked(buildFromInclude).mockResolvedValue({
+      resources: {
+        datasources: [{ name: "events", content: "SCHEMA >" }],
+        pipes: [{ name: "events_endpoint", content: "TYPE endpoint" }],
+        connections: [{ name: "kafka_main", content: "TYPE kafka" }],
+      },
+      entities: {
+        datasources: {},
+        pipes: {},
+        connections: {},
+        rawDatasources: [],
+        rawPipes: [],
+        sourceFiles: [],
+      },
+      stats: {
+        datasourceCount: 1,
+        pipeCount: 1,
+        connectionCount: 1,
+      },
+    });
+
+    const result = await runGenerate();
+
+    expect(result.success).toBe(true);
+    expect(result.artifacts).toEqual([
+      {
+        type: "datasource",
+        name: "events",
+        relativePath: "datasources/events.datasource",
+        content: "SCHEMA >",
+      },
+      {
+        type: "pipe",
+        name: "events_endpoint",
+        relativePath: "pipes/events_endpoint.pipe",
+        content: "TYPE endpoint",
+      },
+      {
+        type: "connection",
+        name: "kafka_main",
+        relativePath: "connections/kafka_main.connection",
+        content: "TYPE kafka",
+      },
+    ]);
+    expect(result.stats?.totalCount).toBe(3);
+  });
+
+  it("returns an error when loading config fails", async () => {
+    const { loadConfigAsync } = await import("../config.js");
+    vi.mocked(loadConfigAsync).mockRejectedValue(new Error("No tinybird config"));
+
+    const result = await runGenerate();
+
+    expect(result.success).toBe(false);
+    expect(result.error).toContain("No tinybird config");
+  });
+
+  it("writes artifacts to outputDir when requested", async () => {
+    const { loadConfigAsync } = await import("../config.js");
+    const { buildFromInclude } = await import("../../generator/index.js");
+
+    vi.mocked(loadConfigAsync).mockResolvedValue({
+      include: ["lib/tinybird.ts"],
+      token: "p.test-token",
+      baseUrl: "https://api.tinybird.co",
+      configPath: "/tmp/tinybird.config.json",
+      cwd: "/tmp",
+      gitBranch: "feature-x",
+      tinybirdBranch: "feature_x",
+      isMainBranch: false,
+      devMode: "branch",
+    });
+
+    vi.mocked(buildFromInclude).mockResolvedValue({
+      resources: {
+        datasources: [{ name: "events", content: "SCHEMA >" }],
+        pipes: [{ name: "events_endpoint", content: "TYPE endpoint" }],
+        connections: [{ name: "kafka_main", content: "TYPE kafka" }],
+      },
+      entities: {
+        datasources: {},
+        pipes: {},
+        connections: {},
+        rawDatasources: [],
+        rawPipes: [],
+        sourceFiles: [],
+      },
+      stats: {
+        datasourceCount: 1,
+        pipeCount: 1,
+        connectionCount: 1,
+      },
+    });
+
+    const outputDir = await mkdtemp(join(tmpdir(), "tb-generate-test-"));
+    try {
+      const result = await runGenerate({ outputDir });
+
+      expect(result.success).toBe(true);
+      expect(
+        await readFile(join(outputDir, "datasources/events.datasource"), "utf-8")
+      ).toBe("SCHEMA >");
+      expect(
+        await readFile(join(outputDir, "pipes/events_endpoint.pipe"), "utf-8")
+      ).toBe("TYPE endpoint");
+      expect(
+        await readFile(
+          join(outputDir, "connections/kafka_main.connection"),
+          "utf-8"
+        )
+      ).toBe("TYPE kafka");
+    } finally {
+      await rm(outputDir, { recursive: true, force: true });
+    }
+  });
+
+  it("returns an error when buildFromInclude fails", async () => {
+    const { loadConfigAsync } = await import("../config.js");
+    const { buildFromInclude } = await import("../../generator/index.js");
+
+    vi.mocked(loadConfigAsync).mockResolvedValue({
+      include: ["lib/tinybird.ts"],
+      token: "p.test-token",
+      baseUrl: "https://api.tinybird.co",
+      configPath: "/tmp/tinybird.config.json",
+      cwd: "/tmp",
+      gitBranch: "feature-x",
+      tinybirdBranch: "feature_x",
+      isMainBranch: false,
+      devMode: "branch",
+    });
+    vi.mocked(buildFromInclude).mockRejectedValue(
+      new Error("generator failed")
+    );
+
+    const result = await runGenerate();
+
+    expect(result.success).toBe(false);
+    expect(result.error).toContain("generator failed");
+  });
+});

package/src/cli/commands/generate.ts

@@ -0,0 +1,121 @@
+/**
+ * Generate command - build Tinybird resources from TypeScript include paths
+ * and expose a stable artifact contract for external consumers.
+ */
+
+import { mkdir, writeFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { loadConfigAsync } from "../config.js";
+import { buildFromInclude, type BuildFromIncludeResult } from "../../generator/index.js";
+
+export type GeneratedResourceType = "datasource" | "pipe" | "connection";
+
+export interface GeneratedResourceArtifact {
+  type: GeneratedResourceType;
+  name: string;
+  relativePath: string;
+  content: string;
+}
+
+export interface GenerateCommandOptions {
+  cwd?: string;
+  outputDir?: string;
+}
+
+export interface GenerateCommandResult {
+  success: boolean;
+  artifacts?: GeneratedResourceArtifact[];
+  stats?: {
+    datasourceCount: number;
+    pipeCount: number;
+    connectionCount: number;
+    totalCount: number;
+  };
+  outputDir?: string;
+  configPath?: string;
+  error?: string;
+  durationMs: number;
+}
+
+function toArtifacts(build: BuildFromIncludeResult): GeneratedResourceArtifact[] {
+  const artifacts: GeneratedResourceArtifact[] = [];
+
+  for (const datasource of build.resources.datasources) {
+    artifacts.push({
+      type: "datasource",
+      name: datasource.name,
+      relativePath: `datasources/${datasource.name}.datasource`,
+      content: datasource.content,
+    });
+  }
+
+  for (const pipe of build.resources.pipes) {
+    artifacts.push({
+      type: "pipe",
+      name: pipe.name,
+      relativePath: `pipes/${pipe.name}.pipe`,
+      content: pipe.content,
+    });
+  }
+
+  for (const connection of build.resources.connections) {
+    artifacts.push({
+      type: "connection",
+      name: connection.name,
+      relativePath: `connections/${connection.name}.connection`,
+      content: connection.content,
+    });
+  }
+
+  return artifacts;
+}
+
+async function writeArtifacts(outputDir: string, artifacts: GeneratedResourceArtifact[]): Promise<void> {
+  for (const artifact of artifacts) {
+    const targetPath = join(outputDir, artifact.relativePath);
+    const targetDir = dirname(targetPath);
+    await mkdir(targetDir, { recursive: true });
+    await writeFile(targetPath, artifact.content, "utf-8");
+  }
+}
+
+export async function runGenerate(
+  options: GenerateCommandOptions = {}
+): Promise<GenerateCommandResult> {
+  const startTime = Date.now();
+  const cwd = options.cwd ?? process.cwd();
+
+  try {
+    const config = await loadConfigAsync(cwd);
+    const build = await buildFromInclude({
+      includePaths: config.include,
+      cwd: config.cwd,
+    });
+
+    const artifacts = toArtifacts(build);
+
+    if (options.outputDir) {
+      await writeArtifacts(options.outputDir, artifacts);
+    }
+
+    return {
+      success: true,
+      artifacts,
+      stats: {
+        datasourceCount: build.stats.datasourceCount,
+        pipeCount: build.stats.pipeCount,
+        connectionCount: build.stats.connectionCount,
+        totalCount: artifacts.length,
+      },
+      outputDir: options.outputDir,
+      configPath: config.configPath,
+      durationMs: Date.now() - startTime,
+    };
+  } catch (error) {
+    return {
+      success: false,
+      error: (error as Error).message,
+      durationMs: Date.now() - startTime,
+    };
+  }
+}

package/src/cli/index.ts

@@ -18,6 +18,7 @@ import pc from "picocolors";
 import { runInit } from "./commands/init.js";
 import { runBuild } from "./commands/build.js";
 import { runDeploy } from "./commands/deploy.js";
+import { runGenerate } from "./commands/generate.js";
 import { runPreview } from "./commands/preview.js";
 import { runDev } from "./commands/dev.js";
 import { runMigrate } from "./commands/migrate.js";
@@ -360,6 +361,46 @@ function createCli(): Command {
       }
     });
 
+  // Generate command
+  program
+    .command("generate")
+    .description("Generate Tinybird datafiles from TypeScript definitions")
+    .option("--json", "Output JSON artifacts (for external consumers)")
+    .option(
+      "-o, --output-dir <path>",
+      "Write generated files to a target directory"
+    )
+    .action(async (options) => {
+      const result = await runGenerate({
+        outputDir: options.outputDir,
+      });
+
+      if (!result.success) {
+        console.error(`Error: ${result.error}`);
+        process.exit(1);
+      }
+
+      if (options.json) {
+        console.log(JSON.stringify(result, null, 2));
+        return;
+      }
+
+      const stats = result.stats ?? {
+        datasourceCount: 0,
+        pipeCount: 0,
+        connectionCount: 0,
+        totalCount: 0,
+      };
+
+      console.log(
+        `Generated ${stats.totalCount} resources (${stats.datasourceCount} datasources, ${stats.pipeCount} pipes, ${stats.connectionCount} connections)`
+      );
+      if (result.outputDir) {
+        console.log(`Written to: ${result.outputDir}`);
+      }
+      console.log(`Completed in ${output.formatDuration(result.durationMs)}`);
+    });
+
   // Migrate command
   program
     .command("migrate")

package/src/client/types.ts

@@ -152,6 +152,11 @@ export interface IngestOptions {
   signal?: AbortSignal;
   /** Wait for the ingestion to complete before returning */
   wait?: boolean;
+  /**
+   * Number of retry attempts after the first request.
+   * Retries are disabled by default when undefined.
+   */
+  maxRetries?: number;
 }
 
 /**