@tinybirdco/sdk 0.0.57 → 0.0.59

package/src/api/api.test.ts

@@ -207,6 +207,395 @@ describe("TinybirdApi", () => {
     ).rejects.toThrow("Date values are not supported in ingest payloads");
   });
 
+  it("does not retry ingest on 503 when retry is disabled", async () => {
+    let attempts = 0;
+
+    server.use(
+      http.post(`${BASE_URL}/v0/events`, () => {
+        attempts += 1;
+        return new HttpResponse("Service unavailable", { status: 503 });
+      })
+    );
+
+    const api = createTinybirdApi({
+      baseUrl: BASE_URL,
+      token: "p.default-token",
+    });
+
+    await expect(
+      api.ingest("events", { timestamp: "2024-01-01 00:00:00" })
+    ).rejects.toMatchObject({
+      name: "TinybirdApiError",
+      statusCode: 503,
+    });
+    expect(attempts).toBe(1);
+  });
+
+  it("retries ingest on 503 with exponential backoff", async () => {
+    let attempts = 0;
+
+    server.use(
+      http.post(`${BASE_URL}/v0/events`, () => {
+        attempts += 1;
+        if (attempts === 1) {
+          return new HttpResponse("Service unavailable", { status: 503 });
+        }
+
+        return HttpResponse.json({
+          successful_rows: 1,
+          quarantined_rows: 0,
+        });
+      })
+    );
+
+    const api = createTinybirdApi({
+      baseUrl: BASE_URL,
+      token: "p.default-token",
+    });
+
+    const result = await api.ingest(
+      "events",
+      { timestamp: "2024-01-01 00:00:00" },
+      {
+        maxRetries: 1,
+      }
+    );
+
+    expect(result).toEqual({ successful_rows: 1, quarantined_rows: 0 });
+    expect(attempts).toBe(2);
+  });
+
+  it("retries ingest on 429 with retry-after header and succeeds", async () => {
+    let attempts = 0;
+
+    server.use(
+      http.post(`${BASE_URL}/v0/events`, () => {
+        attempts += 1;
+        if (attempts === 1) {
+          return new HttpResponse("Rate limited", {
+            status: 429,
+            headers: {
+              "Retry-After": "0",
+            },
+          });
+        }
+
+        return HttpResponse.json({
+          successful_rows: 1,
+          quarantined_rows: 0,
+        });
+      })
+    );
+
+    const api = createTinybirdApi({
+      baseUrl: BASE_URL,
+      token: "p.default-token",
+    });
+
+    const result = await api.ingest(
+      "events",
+      { timestamp: "2024-01-01 00:00:00" },
+      {
+        maxRetries: 1,
+      }
+    );
+
+    expect(result).toEqual({ successful_rows: 1, quarantined_rows: 0 });
+    expect(attempts).toBe(2);
+  });
+
+  it("drains retryable 429 response body before retrying", async () => {
+    let attempts = 0;
+    let firstResponse: Response | undefined;
+
+    const customFetch: typeof fetch = async () => {
+      attempts += 1;
+      if (attempts === 1) {
+        const stream = new ReadableStream<Uint8Array>({
+          start(controller) {
+            controller.enqueue(new TextEncoder().encode("rate limited"));
+            controller.close();
+          },
+        });
+
+        firstResponse = new Response(stream, {
+          status: 429,
+          headers: {
+            "Retry-After": "0",
+          },
+        });
+        return firstResponse;
+      }
+
+      return new Response(
+        JSON.stringify({
+          successful_rows: 1,
+          quarantined_rows: 0,
+        }),
+        {
+          status: 200,
+          headers: {
+            "Content-Type": "application/json",
+          },
+        }
+      );
+    };
+
+    const api = createTinybirdApi({
+      baseUrl: BASE_URL,
+      token: "p.default-token",
+      fetch: customFetch,
+    });
+
+    const result = await api.ingest(
+      "events",
+      { timestamp: "2024-01-01 00:00:00" },
+      {
+        maxRetries: 1,
+      }
+    );
+
+    expect(result).toEqual({ successful_rows: 1, quarantined_rows: 0 });
+    expect(attempts).toBe(2);
+    expect(firstResponse?.bodyUsed).toBe(true);
+  });
+
+  it("does not retry 429 when rate-limit delay headers are missing", async () => {
+    let attempts = 0;
+
+    server.use(
+      http.post(`${BASE_URL}/v0/events`, () => {
+        attempts += 1;
+        return new HttpResponse("Rate limited", { status: 429 });
+      })
+    );
+
+    const api = createTinybirdApi({
+      baseUrl: BASE_URL,
+      token: "p.default-token",
+    });
+
+    await expect(
+      api.ingest(
+        "events",
+        { timestamp: "2024-01-01 00:00:00" },
+        {
+          maxRetries: 3,
+        }
+      )
+    ).rejects.toMatchObject({
+      name: "TinybirdApiError",
+      statusCode: 429,
+    });
+    expect(attempts).toBe(1);
+  });
+
+  it("does not retry ingest on non-retryable status by default", async () => {
+    let attempts = 0;
+
+    server.use(
+      http.post(`${BASE_URL}/v0/events`, () => {
+        attempts += 1;
+        return HttpResponse.json({ error: "Invalid payload" }, { status: 400 });
+      })
+    );
+
+    const api = createTinybirdApi({
+      baseUrl: BASE_URL,
+      token: "p.default-token",
+    });
+
+    await expect(
+      api.ingest(
+        "events",
+        { timestamp: "2024-01-01 00:00:00" },
+        {
+          maxRetries: 3,
+        }
+      )
+    ).rejects.toMatchObject({
+      name: "TinybirdApiError",
+      statusCode: 400,
+    });
+
+    expect(attempts).toBe(1);
+  });
+
+  it("stops retrying ingest after maxRetries on 429", async () => {
+    let attempts = 0;
+
+    server.use(
+      http.post(`${BASE_URL}/v0/events`, () => {
+        attempts += 1;
+        return new HttpResponse("Rate limited", {
+          status: 429,
+          headers: {
+            "Retry-After": "0",
+          },
+        });
+      })
+    );
+
+    const api = createTinybirdApi({
+      baseUrl: BASE_URL,
+      token: "p.default-token",
+    });
+
+    await expect(
+      api.ingest(
+        "events",
+        { timestamp: "2024-01-01 00:00:00" },
+        {
+          maxRetries: 2,
+        }
+      )
+    ).rejects.toMatchObject({
+      name: "TinybirdApiError",
+      statusCode: 429,
+    });
+
+    expect(attempts).toBe(3);
+  });
+
+  it("stops retrying ingest after maxRetries on 503", async () => {
+    let attempts = 0;
+
+    server.use(
+      http.post(`${BASE_URL}/v0/events`, () => {
+        attempts += 1;
+        return new HttpResponse("Service unavailable", { status: 503 });
+      })
+    );
+
+    const api = createTinybirdApi({
+      baseUrl: BASE_URL,
+      token: "p.default-token",
+    });
+
+    await expect(
+      api.ingest(
+        "events",
+        { timestamp: "2024-01-01 00:00:00" },
+        {
+          maxRetries: 2,
+        }
+      )
+    ).rejects.toMatchObject({
+      name: "TinybirdApiError",
+      statusCode: 503,
+    });
+
+    expect(attempts).toBe(3);
+  });
+
+  it("retries ingest on 503 when wait is false", async () => {
+    let attempts = 0;
+
+    server.use(
+      http.post(`${BASE_URL}/v0/events`, () => {
+        attempts += 1;
+        if (attempts === 1) {
+          return new HttpResponse("Service unavailable", { status: 503 });
+        }
+
+        return HttpResponse.json({
+          successful_rows: 1,
+          quarantined_rows: 0,
+        });
+      })
+    );
+
+    const api = createTinybirdApi({
+      baseUrl: BASE_URL,
+      token: "p.default-token",
+    });
+
+    const result = await api.ingest(
+      "events",
+      { timestamp: "2024-01-01 00:00:00" },
+      {
+        wait: false,
+        maxRetries: 1,
+      }
+    );
+
+    expect(result).toEqual({
+      successful_rows: 1,
+      quarantined_rows: 0,
+    });
+    expect(attempts).toBe(2);
+  });
+
+  it("does not retry 500 even when wait is false", async () => {
+    let attempts = 0;
+
+    server.use(
+      http.post(`${BASE_URL}/v0/events`, () => {
+        attempts += 1;
+        return new HttpResponse("Internal error", { status: 500 });
+      })
+    );
+
+    const api = createTinybirdApi({
+      baseUrl: BASE_URL,
+      token: "p.default-token",
+    });
+
+    await expect(
+      api.ingest(
+        "events",
+        { timestamp: "2024-01-01 00:00:00" },
+        {
+          wait: false,
+          maxRetries: 3,
+        }
+      )
+    ).rejects.toMatchObject({
+      name: "TinybirdApiError",
+      statusCode: 500,
+    });
+
+    expect(attempts).toBe(1);
+  });
+
+  it("does not retry ingest on transient network errors", async () => {
+    let fetchAttempts = 0;
+
+    server.use(
+      http.post(`${BASE_URL}/v0/events`, () => {
+        return HttpResponse.json({
+          successful_rows: 1,
+          quarantined_rows: 0,
+        });
+      })
+    );
+
+    const flakyFetch: typeof fetch = async (input, init) => {
+      fetchAttempts += 1;
+      if (fetchAttempts === 1) {
+        throw new TypeError("fetch failed");
+      }
+      return fetch(input, init);
+    };
+
+    const api = createTinybirdApi({
+      baseUrl: BASE_URL,
+      token: "p.default-token",
+      fetch: flakyFetch,
+    });
+
+    await expect(
+      api.ingest(
+        "events",
+        { timestamp: "2024-01-01 00:00:00" },
+        {
+          maxRetries: 1,
+        }
+      )
+    ).rejects.toThrow("fetch failed");
+    expect(fetchAttempts).toBe(1);
+  });
+
   it("executes raw SQL via tinybirdApi.sql", async () => {
     let rawSql: string | null = null;
     let contentType: string | null = null;

package/src/api/api.ts

@@ -14,6 +14,8 @@ import type {
 } from "../client/types.js";
 
 const DEFAULT_TIMEOUT = 30000;
+const DEFAULT_INGEST_RETRY_503_BASE_DELAY_MS = 200;
+const DEFAULT_INGEST_RETRY_503_MAX_DELAY_MS = 3000;
 
 /**
  * Public, decoupled Tinybird API wrapper configuration
@@ -279,22 +281,49 @@ export class TinybirdApi {
     const ndjson = events
       .map((event) => JSON.stringify(this.serializeEvent(event)))
       .join("\n");
+    const signal = this.createAbortSignal(options.timeout, options.signal);
+    const maxRetries = this.resolveIngestMaxRetries(options.maxRetries);
+    let retryCount = 0;
+
+    while (true) {
+      let response: Response;
+
+      try {
+        response = await this.request(url.toString(), {
+          method: "POST",
+          token: options.token,
+          headers: {
+            "Content-Type": "application/x-ndjson",
+          },
+          body: ndjson,
+          signal,
+        });
+      } catch (error) {
+        throw error;
+      }
 
-    const response = await this.request(url.toString(), {
-      method: "POST",
-      token: options.token,
-      headers: {
-        "Content-Type": "application/x-ndjson",
-      },
-      body: ndjson,
-      signal: this.createAbortSignal(options.timeout, options.signal),
-    });
+      if (response.ok) {
+        return (await response.json()) as IngestResult;
+      }
+
+      const retry429Delay = this.resolveRetry429Delay(response, maxRetries, retryCount);
+      if (retry429Delay !== undefined) {
+        await this.discardResponseBody(response);
+        await this.sleep(retry429Delay, signal);
+        retryCount += 1;
+        continue;
+      }
+
+      const retry503Delay = this.resolveRetry503Delay(response, maxRetries, retryCount);
+      if (retry503Delay !== undefined) {
+        await this.discardResponseBody(response);
+        await this.sleep(retry503Delay, signal);
+        retryCount += 1;
+        continue;
+      }
 
-    if (!response.ok) {
       await this.handleErrorResponse(response);
     }
-
-    return (await response.json()) as IngestResult;
   }
 
   /**
@@ -575,6 +604,164 @@ export class TinybirdApi {
     return AbortSignal.any([timeoutSignal, existingSignal]);
   }
 
+  private resolveIngestMaxRetries(
+    maxRetries: TinybirdApiIngestOptions["maxRetries"]
+  ): number | undefined {
+    if (maxRetries === undefined) {
+      return undefined;
+    }
+
+    if (!Number.isFinite(maxRetries)) {
+      throw new Error("'maxRetries' must be a finite number");
+    }
+
+    return Math.max(0, Math.floor(maxRetries));
+  }
+
+  private resolveRetry429Delay(
+    response: Response,
+    maxRetries: number | undefined,
+    retryCount: number
+  ): number | undefined {
+    if (maxRetries === undefined) {
+      return undefined;
+    }
+
+    if (response.status !== 429) {
+      return undefined;
+    }
+
+    if (retryCount >= maxRetries) {
+      return undefined;
+    }
+
+    return this.resolveRetryDelayFromHeaders(response);
+  }
+
+  private resolveRetry503Delay(
+    response: Response,
+    maxRetries: number | undefined,
+    retryCount: number
+  ): number | undefined {
+    if (maxRetries === undefined) {
+      return undefined;
+    }
+
+    if (response.status !== 503) {
+      return undefined;
+    }
+
+    if (retryCount >= maxRetries) {
+      return undefined;
+    }
+
+    return this.calculateRetry503DelayMs(retryCount);
+  }
+
+  private resolveRetryDelayFromHeaders(response: Response): number | undefined {
+    const retryAfter = response.headers.get("retry-after");
+    const retryAfterDelay = this.parseRetryAfterDelayMs(retryAfter);
+    if (retryAfterDelay !== undefined) {
+      return retryAfterDelay;
+    }
+
+    const rateLimitReset = response.headers.get("x-ratelimit-reset");
+    const rateLimitResetDelay = this.parseRateLimitResetDelayMs(rateLimitReset);
+    if (rateLimitResetDelay !== undefined) {
+      return rateLimitResetDelay;
+    }
+    return undefined;
+  }
+
+  private parseRetryAfterDelayMs(value: string | null): number | undefined {
+    if (!value) {
+      return undefined;
+    }
+
+    const trimmed = value.trim();
+    const seconds = Number(trimmed);
+    if (Number.isFinite(seconds)) {
+      return Math.max(0, Math.floor(seconds * 1000));
+    }
+
+    const retryDateMs = Date.parse(trimmed);
+    if (Number.isNaN(retryDateMs)) {
+      return undefined;
+    }
+
+    return Math.max(0, retryDateMs - Date.now());
+  }
+
+  private parseRateLimitResetDelayMs(value: string | null): number | undefined {
+    if (!value) {
+      return undefined;
+    }
+
+    const numericValue = Number(value.trim());
+    if (!Number.isFinite(numericValue)) {
+      return undefined;
+    }
+
+    return Math.max(0, Math.floor(numericValue * 1000));
+  }
+
+  private calculateRetry503DelayMs(retryCount: number): number {
+    return Math.min(
+      DEFAULT_INGEST_RETRY_503_MAX_DELAY_MS,
+      DEFAULT_INGEST_RETRY_503_BASE_DELAY_MS * 2 ** retryCount
+    );
+  }
+
+  private async discardResponseBody(response: Response): Promise<void> {
+    if (response.bodyUsed || !response.body) {
+      return;
+    }
+
+    try {
+      await response.arrayBuffer();
+    } catch {
+      try {
+        await response.body.cancel();
+      } catch {
+        // Best effort cleanup only; never mask retry/error flow.
+      }
+    }
+  }
+
+  private async sleep(delayMs: number, signal?: AbortSignal): Promise<void> {
+    if (delayMs <= 0) {
+      return;
+    }
+
+    await new Promise<void>((resolve, reject) => {
+      const timer = setTimeout(() => {
+        cleanup();
+        resolve();
+      }, delayMs);
+
+      const onAbort = () => {
+        cleanup();
+        reject(signal?.reason ?? new DOMException("The operation was aborted.", "AbortError"));
+      };
+
+      const cleanup = () => {
+        clearTimeout(timer);
+        signal?.removeEventListener("abort", onAbort);
+      };
+
+      if (!signal) {
+        return;
+      }
+
+      if (signal.aborted) {
+        onAbort();
+        return;
+      }
+
+      signal.addEventListener("abort", onAbort, { once: true });
+    });
+  }
+
   private serializeEvent(
     event: Record<string, unknown>
   ): Record<string, unknown> {

package/src/client/types.ts

@@ -152,6 +152,11 @@ export interface IngestOptions {
   signal?: AbortSignal;
   /** Wait for the ingestion to complete before returning */
   wait?: boolean;
+  /**
+   * Number of retry attempts after the first request.
+   * Retries are disabled by default when undefined.
+   */
+  maxRetries?: number;
 }
 
 /**

package/src/schema/datasource.test.ts

@@ -7,7 +7,7 @@ import {
   getColumnNames,
   column,
 } from "./datasource.js";
-import { t } from "./types.js";
+import { t, type AnyTypeValidator } from "./types.js";
 import { engine } from "./engines.js";
 import { defineKafkaConnection, defineS3Connection, defineGCSConnection } from "./connection.js";
 
@@ -259,6 +259,20 @@ describe("Datasource Schema", () => {
       expect(result).toBeUndefined();
     });
 
+    it("never returns a function when validator branding is missing", () => {
+      const validator = t.string();
+      // Simulate a validator-like object where isTypeValidator() fails by
+      // removing symbol keys (including the validator brand).
+      const unbrandedValidator = Object.fromEntries(
+        Object.entries(validator as unknown as Record<string, unknown>)
+      ) as unknown as AnyTypeValidator;
+
+      const result = getColumnJsonPath(unbrandedValidator);
+
+      expect(result).toBeUndefined();
+      expect(typeof result).not.toBe("function");
+    });
+
     it("returns jsonPath from validator modifier", () => {
       const validator = t.string().jsonPath("$.user.id");
       const result = getColumnJsonPath(validator);

package/src/schema/datasource.ts

@@ -272,7 +272,9 @@ export function getColumnJsonPath(column: AnyTypeValidator | ColumnDefinition):
     return getModifiers(column).jsonPath;
   }
 
-  if (column.jsonPath !== undefined) {
+  // Check typeof to avoid returning the jsonPath method from validators
+  // if isTypeValidator incorrectly returns false (e.g., cross-module Symbol issues)
+  if (typeof column.jsonPath === "string") {
     return column.jsonPath;
   }