@tinybirdco/sdk 0.0.36 → 0.0.37

package/src/api/api.test.ts

@@ -292,4 +292,226 @@ describe("TinybirdApi", () => {
       },
     });
   });
+
+  describe("appendDatasource", () => {
+    it("appends data from URL", async () => {
+      let datasourceName: string | null = null;
+      let modeParam: string | null = null;
+      let formatParam: string | null = null;
+      let contentType: string | null = null;
+      let requestBody: string | null = null;
+
+      server.use(
+        http.post(`${BASE_URL}/v0/datasources`, async ({ request }) => {
+          const url = new URL(request.url);
+          datasourceName = url.searchParams.get("name");
+          modeParam = url.searchParams.get("mode");
+          formatParam = url.searchParams.get("format");
+          contentType = request.headers.get("content-type");
+          requestBody = await request.text();
+
+          return HttpResponse.json({
+            successful_rows: 100,
+            quarantined_rows: 0,
+            import_id: "import_123",
+          });
+        })
+      );
+
+      const api = createTinybirdApi({
+        baseUrl: BASE_URL,
+        token: "p.default-token",
+      });
+
+      const result = await api.appendDatasource("events", {
+        url: "https://example.com/data.csv",
+      });
+
+      expect(result).toEqual({
+        successful_rows: 100,
+        quarantined_rows: 0,
+        import_id: "import_123",
+      });
+      expect(datasourceName).toBe("events");
+      expect(modeParam).toBe("append");
+      expect(formatParam).toBe("csv");
+      expect(contentType).toBe("application/x-www-form-urlencoded");
+      expect(requestBody).toBe("url=https%3A%2F%2Fexample.com%2Fdata.csv");
+    });
+
+    it("detects ndjson format from URL extension", async () => {
+      let formatParam: string | null = null;
+
+      server.use(
+        http.post(`${BASE_URL}/v0/datasources`, async ({ request }) => {
+          const url = new URL(request.url);
+          formatParam = url.searchParams.get("format");
+          return HttpResponse.json({ successful_rows: 1, quarantined_rows: 0 });
+        })
+      );
+
+      const api = createTinybirdApi({
+        baseUrl: BASE_URL,
+        token: "p.default-token",
+      });
+
+      await api.appendDatasource("events", {
+        url: "https://example.com/data.ndjson",
+      });
+
+      expect(formatParam).toBe("ndjson");
+    });
+
+    it("detects jsonl as ndjson format", async () => {
+      let formatParam: string | null = null;
+
+      server.use(
+        http.post(`${BASE_URL}/v0/datasources`, async ({ request }) => {
+          const url = new URL(request.url);
+          formatParam = url.searchParams.get("format");
+          return HttpResponse.json({ successful_rows: 1, quarantined_rows: 0 });
+        })
+      );
+
+      const api = createTinybirdApi({
+        baseUrl: BASE_URL,
+        token: "p.default-token",
+      });
+
+      await api.appendDatasource("events", {
+        url: "https://example.com/data.jsonl",
+      });
+
+      expect(formatParam).toBe("ndjson");
+    });
+
+    it("detects parquet format from URL extension", async () => {
+      let formatParam: string | null = null;
+
+      server.use(
+        http.post(`${BASE_URL}/v0/datasources`, async ({ request }) => {
+          const url = new URL(request.url);
+          formatParam = url.searchParams.get("format");
+          return HttpResponse.json({ successful_rows: 1, quarantined_rows: 0 });
+        })
+      );
+
+      const api = createTinybirdApi({
+        baseUrl: BASE_URL,
+        token: "p.default-token",
+      });
+
+      await api.appendDatasource("events", {
+        url: "https://example.com/data.parquet",
+      });
+
+      expect(formatParam).toBe("parquet");
+    });
+
+    it("strips query string when detecting format", async () => {
+      let formatParam: string | null = null;
+
+      server.use(
+        http.post(`${BASE_URL}/v0/datasources`, async ({ request }) => {
+          const url = new URL(request.url);
+          formatParam = url.searchParams.get("format");
+          return HttpResponse.json({ successful_rows: 1, quarantined_rows: 0 });
+        })
+      );
+
+      const api = createTinybirdApi({
+        baseUrl: BASE_URL,
+        token: "p.default-token",
+      });
+
+      await api.appendDatasource("events", {
+        url: "https://example.com/data.csv?token=abc",
+      });
+
+      expect(formatParam).toBe("csv");
+    });
+
+    it("includes CSV dialect options", async () => {
+      let delimiterParam: string | null = null;
+      let newLineParam: string | null = null;
+      let escapeCharParam: string | null = null;
+
+      server.use(
+        http.post(`${BASE_URL}/v0/datasources`, async ({ request }) => {
+          const url = new URL(request.url);
+          delimiterParam = url.searchParams.get("dialect_delimiter");
+          newLineParam = url.searchParams.get("dialect_new_line");
+          escapeCharParam = url.searchParams.get("dialect_escapechar");
+          return HttpResponse.json({ successful_rows: 1, quarantined_rows: 0 });
+        })
+      );
+
+      const api = createTinybirdApi({
+        baseUrl: BASE_URL,
+        token: "p.default-token",
+      });
+
+      await api.appendDatasource("events", {
+        url: "https://example.com/data.csv",
+        csvDialect: {
+          delimiter: ";",
+          newLine: "\r\n",
+          escapeChar: "\\",
+        },
+      });
+
+      expect(delimiterParam).toBe(";");
+      expect(newLineParam).toBe("\r\n");
+      expect(escapeCharParam).toBe("\\");
+    });
+
+    it("throws error when neither url nor file is provided", async () => {
+      const api = createTinybirdApi({
+        baseUrl: BASE_URL,
+        token: "p.default-token",
+      });
+
+      await expect(api.appendDatasource("events", {})).rejects.toThrow(
+        "Either 'url' or 'file' must be provided in options"
+      );
+    });
+
+    it("throws error when both url and file are provided", async () => {
+      const api = createTinybirdApi({
+        baseUrl: BASE_URL,
+        token: "p.default-token",
+      });
+
+      await expect(
+        api.appendDatasource("events", {
+          url: "https://example.com/data.csv",
+          file: "./data.csv",
+        })
+      ).rejects.toThrow("Only one of 'url' or 'file' can be provided, not both");
+    });
+
+    it("allows per-request token override", async () => {
+      let authorizationHeader: string | null = null;
+
+      server.use(
+        http.post(`${BASE_URL}/v0/datasources`, async ({ request }) => {
+          authorizationHeader = request.headers.get("Authorization");
+          return HttpResponse.json({ successful_rows: 1, quarantined_rows: 0 });
+        })
+      );
+
+      const api = createTinybirdApi({
+        baseUrl: BASE_URL,
+        token: "p.default-token",
+      });
+
+      await api.appendDatasource(
+        "events",
+        { url: "https://example.com/data.csv" },
+        { token: "p.override-token" }
+      );
+
+      expect(authorizationHeader).toBe("Bearer p.override-token");
+    });
+  });
 });

package/src/api/api.ts

@@ -1,5 +1,7 @@
 import { createTinybirdFetcher, type TinybirdFetch } from "./fetcher.js";
 import type {
+  AppendOptions,
+  AppendResult,
   IngestOptions,
   IngestResult,
   QueryOptions,
@@ -41,6 +43,11 @@ export interface TinybirdApiIngestOptions extends IngestOptions {
   token?: string;
 }
 
+export interface TinybirdApiAppendOptions extends Omit<AppendOptions, 'url' | 'file'> {
+  /** Optional token override for this request */
+  token?: string;
+}
+
 /**
  * Error thrown by TinybirdApi when a response is not OK
  */
@@ -242,6 +249,116 @@ export class TinybirdApi {
     return (await response.json()) as QueryResult<T>;
   }
 
+  /**
+   * Append data to a datasource from a URL or local file
+   */
+  async appendDatasource(
+    datasourceName: string,
+    options: AppendOptions,
+    apiOptions: TinybirdApiAppendOptions = {}
+  ): Promise<AppendResult> {
+    const { url: sourceUrl, file: filePath } = options;
+
+    if (!sourceUrl && !filePath) {
+      throw new Error("Either 'url' or 'file' must be provided in options");
+    }
+
+    if (sourceUrl && filePath) {
+      throw new Error("Only one of 'url' or 'file' can be provided, not both");
+    }
+
+    const url = new URL("/v0/datasources", `${this.baseUrl}/`);
+    url.searchParams.set("name", datasourceName);
+    url.searchParams.set("mode", "append");
+
+    // Auto-detect format from file/url extension
+    const format = this.detectFormat(sourceUrl ?? filePath!);
+    if (format) {
+      url.searchParams.set("format", format);
+    }
+
+    // Add CSV dialect options if applicable
+    if (options.csvDialect) {
+      if (options.csvDialect.delimiter) {
+        url.searchParams.set("dialect_delimiter", options.csvDialect.delimiter);
+      }
+      if (options.csvDialect.newLine) {
+        url.searchParams.set("dialect_new_line", options.csvDialect.newLine);
+      }
+      if (options.csvDialect.escapeChar) {
+        url.searchParams.set("dialect_escapechar", options.csvDialect.escapeChar);
+      }
+    }
+
+    let response: Response;
+
+    if (sourceUrl) {
+      // Remote URL: send as form-urlencoded with url parameter
+      response = await this.request(url.toString(), {
+        method: "POST",
+        token: apiOptions.token,
+        headers: {
+          "Content-Type": "application/x-www-form-urlencoded",
+        },
+        body: `url=${encodeURIComponent(sourceUrl)}`,
+        signal: this.createAbortSignal(options.timeout ?? apiOptions.timeout, options.signal ?? apiOptions.signal),
+      });
+    } else {
+      // Local file: send as multipart form data
+      const formData = await this.createFileFormData(filePath!);
+      response = await this.request(url.toString(), {
+        method: "POST",
+        token: apiOptions.token,
+        body: formData,
+        signal: this.createAbortSignal(options.timeout ?? apiOptions.timeout, options.signal ?? apiOptions.signal),
+      });
+    }
+
+    if (!response.ok) {
+      await this.handleErrorResponse(response);
+    }
+
+    return (await response.json()) as AppendResult;
+  }
+
+  /**
+   * Detect format from file path or URL extension
+   */
+  private detectFormat(source: string): "csv" | "ndjson" | "parquet" | undefined {
+    // Remove query string if present
+    const pathOnly = source.split("?")[0];
+    const extension = pathOnly.split(".").pop()?.toLowerCase();
+
+    switch (extension) {
+      case "csv":
+        return "csv";
+      case "ndjson":
+      case "jsonl":
+        return "ndjson";
+      case "parquet":
+        return "parquet";
+      default:
+        return undefined;
+    }
+  }
+
+  /**
+   * Create FormData for file upload
+   */
+  private async createFileFormData(filePath: string): Promise<FormData> {
+    // Dynamic import for Node.js fs module (browser-safe)
+    const fs = await import("node:fs");
+    const path = await import("node:path");
+
+    const fileContent = await fs.promises.readFile(filePath);
+    const fileName = path.basename(filePath);
+
+    const formData = new FormData();
+    formData.append("csv", new Blob([fileContent]), fileName);
+
+    return formData;
+  }
+
   private createAbortSignal(
     timeout?: number,
     existingSignal?: AbortSignal

package/src/cli/commands/init.ts

@@ -22,7 +22,6 @@ import { getGitRoot } from "../git.js";
 import { fetchAllResources } from "../../api/resources.js";
 import { generateCombinedFile } from "../../codegen/index.js";
 import { execSync } from "child_process";
-import { setTimeout as sleep } from "node:timers/promises";
 import {
   detectPackageManager,
   getPackageManagerAddCmd,
@@ -313,6 +312,8 @@ export interface InitOptions {
   includeCdWorkflow?: boolean;
   /** Git provider for workflow templates */
   workflowProvider?: "github" | "gitlab";
+  /** Skip auto-installing @tinybirdco/sdk dependency */
+  skipDependencyInstall?: boolean;
 }
 
 /**
@@ -415,6 +416,8 @@ export async function runInit(options: InitOptions = {}): Promise<InitResult> {
   const cwd = options.cwd ?? process.cwd();
   const force = options.force ?? false;
   const skipLogin = options.skipLogin ?? false;
+  const skipDependencyInstall =
+    options.skipDependencyInstall ?? Boolean(process.env.VITEST);
 
   const created: string[] = [];
   const skipped: string[] = [];
@@ -739,14 +742,13 @@ export async function runInit(options: InitOptions = {}): Promise<InitResult> {
   }
 
   // Install @tinybirdco/sdk if not already installed
-  if (!hasTinybirdSdkDependency(cwd)) {
+  if (!skipDependencyInstall && !hasTinybirdSdkDependency(cwd)) {
     const s = p.spinner();
     s.start("Installing dependencies");
     const packageManager = detectPackageManager(cwd);
     const addCmd = getPackageManagerAddCmd(packageManager);
     try {
       execSync(`${addCmd} @tinybirdco/sdk`, { cwd, stdio: "pipe" });
-      await sleep(1000);
       s.stop("Installed dependencies");
       created.push("@tinybirdco/sdk");
     } catch (error) {

package/src/client/base.test.ts

@@ -1,5 +1,6 @@
 import { describe, it, expect } from "vitest";
 import { TinybirdClient, createClient } from "./base.js";
+import type { DatasourcesNamespace } from "./types.js";
 
 describe("TinybirdClient", () => {
   describe("constructor", () => {
@@ -137,4 +138,35 @@ describe("TinybirdClient", () => {
       expect(context.baseUrl).toBe("https://api.tinybird.co");
     });
   });
+
+  describe("datasources", () => {
+    it("exposes datasources namespace", () => {
+      const client = createClient({
+        baseUrl: "https://api.tinybird.co",
+        token: "test-token",
+      });
+
+      expect(client.datasources).toBeDefined();
+    });
+
+    it("datasources namespace has append method", () => {
+      const client = createClient({
+        baseUrl: "https://api.tinybird.co",
+        token: "test-token",
+      });
+
+      expect(typeof client.datasources.append).toBe("function");
+    });
+
+    it("datasources conforms to DatasourcesNamespace interface", () => {
+      const client = createClient({
+        baseUrl: "https://api.tinybird.co",
+        token: "test-token",
+      });
+
+      const datasources: DatasourcesNamespace = client.datasources;
+      expect(datasources).toBeDefined();
+      expect(typeof datasources.append).toBe("function");
+    });
+  });
 });

package/src/client/base.ts

@@ -3,8 +3,11 @@
  */
 
 import type {
+  AppendOptions,
+  AppendResult,
   ClientConfig,
   ClientContext,
+  DatasourcesNamespace,
   QueryResult,
   IngestResult,
   QueryOptions,
@@ -57,6 +60,11 @@ export class TinybirdClient {
   private contextPromise: Promise<ClientContext> | null = null;
   private resolvedContext: ClientContext | null = null;
 
+  /**
+   * Datasources namespace for import operations
+   */
+  readonly datasources: DatasourcesNamespace;
+
   constructor(config: ClientConfig) {
     // Validate required config
     if (!config.baseUrl) {
@@ -71,6 +79,46 @@ export class TinybirdClient {
       ...config,
       baseUrl: config.baseUrl.replace(/\/$/, ""),
     };
+
+    // Initialize datasources namespace
+    this.datasources = {
+      append: (datasourceName: string, options: AppendOptions): Promise<AppendResult> => {
+        return this.appendDatasource(datasourceName, options);
+      },
+    };
+  }
+
+  /**
+   * Append data to a datasource from a URL or local file
+   *
+   * @param datasourceName - Name of the datasource
+   * @param options - Append options including url or file source
+   * @returns Append result
+   *
+   * @example
+   * ```ts
+   * // Append from URL
+   * await client.datasources.append('events', {
+   *   url: 'https://example.com/data.csv',
+   * });
+   *
+   * // Append from local file
+   * await client.datasources.append('events', {
+   *   file: './data/events.ndjson',
+   * });
+   * ```
+   */
+  private async appendDatasource(
+    datasourceName: string,
+    options: AppendOptions
+  ): Promise<AppendResult> {
+    const token = await this.getToken();
+
+    try {
+      return await this.getApi(token).appendDatasource(datasourceName, options);
+    } catch (error) {
+      this.rethrowApiError(error);
+    }
   }
 
   /**

package/src/client/types.ts

@@ -189,3 +189,57 @@ export interface TypedDatasourceIngest<TRow> {
   /** Send multiple events in a batch */
   sendBatch(events: TRow[], options?: IngestOptions): Promise<IngestResult>;
 }
+
+/**
+ * Data format for append operations
+ */
+export type AppendFormat = "csv" | "ndjson" | "parquet";
+
+/**
+ * CSV dialect options for customizing CSV parsing
+ */
+export interface CsvDialectOptions {
+  /** Field delimiter character (default: ',') */
+  delimiter?: string;
+  /** New line character(s) (default: '\n') */
+  newLine?: string;
+  /** Escape character for special characters */
+  escapeChar?: string;
+}
+
+/**
+ * Options for append operation
+ * Exactly one of `url` or `file` must be provided
+ */
+export interface AppendOptions {
+  /** Remote URL to import data from */
+  url?: string;
+  /** Local file path to import data from */
+  file?: string;
+  /** CSV dialect options (only applicable for csv format) */
+  csvDialect?: CsvDialectOptions;
+  /** Request timeout in milliseconds */
+  timeout?: number;
+  /** AbortController signal for cancellation */
+  signal?: AbortSignal;
+}
+
+/**
+ * Result of an append operation
+ */
+export interface AppendResult {
+  /** Number of rows successfully appended */
+  successful_rows: number;
+  /** Number of rows that failed to append (quarantined) */
+  quarantined_rows: number;
+  /** Import ID for tracking */
+  import_id?: string;
+}
+
+/**
+ * Datasources namespace interface for raw client
+ */
+export interface DatasourcesNamespace {
+  /** Append data to a datasource from a URL or file */
+  append(datasourceName: string, options: AppendOptions): Promise<AppendResult>;
+}

package/src/index.ts

@@ -202,8 +202,12 @@ export type {
 export { TinybirdClient, createClient } from "./client/base.js";
 export { TinybirdError } from "./client/types.js";
 export type {
+  AppendOptions,
+  AppendResult,
   ClientConfig,
   ClientContext,
+  CsvDialectOptions,
+  DatasourcesNamespace,
   QueryResult,
   IngestResult,
   QueryOptions,
@@ -226,6 +230,7 @@ export type {
   TinybirdApiConfig,
   TinybirdApiQueryOptions,
   TinybirdApiIngestOptions,
+  TinybirdApiAppendOptions,
   TinybirdApiRequestInit,
 } from "./api/api.js";
 

package/src/schema/project.test.ts

@@ -93,6 +93,37 @@ describe("Project Schema", () => {
       expect(typeof project.tinybird.ingest.eventsBatch).toBe("function");
     });
 
+    it("creates datasource accessors with append method", () => {
+      const events = defineDatasource("events", {
+        schema: { timestamp: t.dateTime() },
+      });
+
+      const project = defineProject({
+        datasources: { events },
+      });
+
+      expect(project.tinybird.events).toBeDefined();
+      expect(typeof project.tinybird.events.append).toBe("function");
+    });
+
+    it("creates multiple datasource accessors", () => {
+      const events = defineDatasource("events", {
+        schema: { timestamp: t.dateTime() },
+      });
+      const pageViews = defineDatasource("page_views", {
+        schema: { pathname: t.string() },
+      });
+
+      const project = defineProject({
+        datasources: { events, pageViews },
+      });
+
+      expect(project.tinybird.events).toBeDefined();
+      expect(project.tinybird.pageViews).toBeDefined();
+      expect(typeof project.tinybird.events.append).toBe("function");
+      expect(typeof project.tinybird.pageViews.append).toBe("function");
+    });
+
     it("throws error when accessing client before initialization", () => {
       const project = defineProject({});
 
@@ -269,6 +300,39 @@ describe("Project Schema", () => {
       expect(typeof client.ingest.eventsBatch).toBe("function");
     });
 
+    it("creates datasource accessors with append method", () => {
+      const events = defineDatasource("events", {
+        schema: { id: t.string() },
+      });
+
+      const client = createTinybirdClient({
+        datasources: { events },
+        pipes: {},
+      });
+
+      expect(client.events).toBeDefined();
+      expect(typeof client.events.append).toBe("function");
+    });
+
+    it("creates multiple datasource accessors", () => {
+      const events = defineDatasource("events", {
+        schema: { id: t.string() },
+      });
+      const pageViews = defineDatasource("page_views", {
+        schema: { pathname: t.string() },
+      });
+
+      const client = createTinybirdClient({
+        datasources: { events, pageViews },
+        pipes: {},
+      });
+
+      expect(client.events).toBeDefined();
+      expect(client.pageViews).toBeDefined();
+      expect(typeof client.events.append).toBe("function");
+      expect(typeof client.pageViews.append).toBe("function");
+    });
+
     it("accepts devMode option", () => {
       const events = defineDatasource("events", {
         schema: { id: t.string() },