@tinybirdco/sdk 0.0.47 → 0.0.49

package/src/schema/datasource.test.ts

@@ -24,6 +24,7 @@ describe("Datasource Schema", () => {
       expect(ds._name).toBe("events");
       expect(ds._type).toBe("datasource");
       expect(ds.options.schema).toBeDefined();
+      expect(ds.options.engine).toBeUndefined();
     });
 
     it("creates a datasource with description", () => {
@@ -169,6 +170,22 @@ describe("Datasource Schema", () => {
 
       expect(result).toBeUndefined();
     });
+
+    it("returns jsonPath from validator modifier", () => {
+      const validator = t.string().jsonPath("$.user.id");
+      const result = getColumnJsonPath(validator);
+
+      expect(result).toBe("$.user.id");
+    });
+
+    it("prefers column definition jsonPath over validator modifier", () => {
+      const col = column(t.string().jsonPath("$.from_validator"), {
+        jsonPath: "$.from_column",
+      });
+      const result = getColumnJsonPath(col);
+
+      expect(result).toBe("$.from_column");
+    });
   });
 
   describe("getColumnNames", () => {

package/src/schema/datasource.ts

@@ -3,7 +3,7 @@
  * Define table schemas as TypeScript with full type safety
  */
 
-import type { AnyTypeValidator } from "./types.js";
+import { getModifiers, isTypeValidator, type AnyTypeValidator } from "./types.js";
 import type { EngineConfig } from "./engines.js";
 import type { KafkaConnectionDefinition, S3ConnectionDefinition } from "./connection.js";
 import type { TokenDefinition, DatasourceTokenScope } from "./token.js";
@@ -66,6 +66,8 @@ export interface KafkaConfig {
   groupId?: string;
   /** Where to start reading: 'earliest' or 'latest' (default: 'latest') */
   autoOffsetReset?: "earliest" | "latest";
+  /** Whether to store the raw Kafka value payload */
+  storeRawValue?: boolean;
 }
 
 /**
@@ -207,9 +209,18 @@ export function getColumnType(column: AnyTypeValidator | ColumnDefinition): AnyT
  * Get the JSON path for a column if defined
  */
 export function getColumnJsonPath(column: AnyTypeValidator | ColumnDefinition): string | undefined {
-  if ("jsonPath" in column) {
+  if (isTypeValidator(column)) {
+    return getModifiers(column).jsonPath;
+  }
+
+  if (column.jsonPath !== undefined) {
     return column.jsonPath;
   }
+
+  if (isTypeValidator(column.type)) {
+    return getModifiers(column.type).jsonPath;
+  }
+
   return undefined;
 }
 

package/src/schema/engines.test.ts

@@ -44,6 +44,14 @@ describe('Engine Configurations', () => {
       });
       expect(config.ver).toBe('updated_at');
     });
+
+    it('supports isDeleted column', () => {
+      const config = engine.replacingMergeTree({
+        sortingKey: ['id'],
+        isDeleted: '_is_deleted',
+      });
+      expect(config.isDeleted).toBe('_is_deleted');
+    });
   });
 
   describe('SummingMergeTree', () => {
@@ -138,6 +146,16 @@ describe('Engine Configurations', () => {
       expect(clause).toContain('ENGINE_VER "updated_at"');
     });
 
+    it('includes ReplacingMergeTree isDeleted column', () => {
+      const config = engine.replacingMergeTree({
+        sortingKey: ['id'],
+        isDeleted: '_is_deleted',
+      });
+      const clause = getEngineClause(config);
+      expect(clause).toContain('ENGINE "ReplacingMergeTree"');
+      expect(clause).toContain('ENGINE_IS_DELETED "_is_deleted"');
+    });
+
     it('includes SummingMergeTree columns', () => {
       const config = engine.summingMergeTree({
         sortingKey: ['date'],

package/src/schema/engines.ts

@@ -241,6 +241,9 @@ export function getEngineClause(config: EngineConfig): string {
   if (config.type === "ReplacingMergeTree" && config.ver) {
     parts.push(`ENGINE_VER "${config.ver}"`);
   }
+  if (config.type === "ReplacingMergeTree" && config.isDeleted) {
+    parts.push(`ENGINE_IS_DELETED "${config.isDeleted}"`);
+  }
 
   if (config.type === "CollapsingMergeTree" || config.type === "VersionedCollapsingMergeTree") {
     parts.push(`ENGINE_SIGN "${config.sign}"`);

package/src/schema/pipe.test.ts

@@ -1,17 +1,21 @@
 import { describe, it, expect } from "vitest";
 import {
   definePipe,
+  defineSinkPipe,
   defineMaterializedView,
   node,
   isPipeDefinition,
   getEndpointConfig,
   getMaterializedConfig,
+  getSinkConfig,
   isMaterializedView,
+  isSinkPipe,
   getNodeNames,
   getNode,
   sql,
 } from "./pipe.js";
 import { defineDatasource } from "./datasource.js";
+import { defineKafkaConnection, defineS3Connection } from "./connection.js";
 import { t } from "./types.js";
 import { p } from "./params.js";
 import { engine } from "./engines.js";
@@ -154,6 +158,91 @@ describe("Pipe Schema", () => {
     });
   });
 
+  describe("Sink pipes", () => {
+    it("creates a Kafka sink pipe", () => {
+      const kafka = defineKafkaConnection("events_kafka", {
+        bootstrapServers: "localhost:9092",
+      });
+
+      const pipe = defineSinkPipe("events_sink", {
+        nodes: [node({ name: "publish", sql: "SELECT * FROM events" })],
+        sink: {
+          connection: kafka,
+          topic: "events_out",
+          schedule: "@on-demand",
+        },
+      });
+
+      const sink = getSinkConfig(pipe);
+      expect(sink).toBeTruthy();
+      expect(sink && "topic" in sink ? sink.topic : undefined).toBe("events_out");
+      expect(isSinkPipe(pipe)).toBe(true);
+    });
+
+    it("creates an S3 sink pipe", () => {
+      const s3 = defineS3Connection("exports_s3", {
+        region: "us-east-1",
+        arn: "arn:aws:iam::123456789012:role/tinybird-s3-access",
+      });
+
+      const pipe = defineSinkPipe("exports_sink", {
+        nodes: [node({ name: "export", sql: "SELECT * FROM events" })],
+        sink: {
+          connection: s3,
+          bucketUri: "s3://exports/events/",
+          fileTemplate: "events_{date}",
+          schedule: "@once",
+          format: "csv",
+          strategy: "create_new",
+          compression: "gzip",
+        },
+      });
+
+      const sink = getSinkConfig(pipe);
+      expect(sink).toBeTruthy();
+      expect(sink && "bucketUri" in sink ? sink.bucketUri : undefined).toBe("s3://exports/events/");
+      expect(isSinkPipe(pipe)).toBe(true);
+    });
+
+    it("throws when Kafka sink connection type is invalid", () => {
+      const s3 = defineS3Connection("exports_s3", {
+        region: "us-east-1",
+        arn: "arn:aws:iam::123456789012:role/tinybird-s3-access",
+      });
+
+      expect(() =>
+        defineSinkPipe("bad_sink", {
+          nodes: [node({ name: "export", sql: "SELECT * FROM events" })],
+          sink: {
+            // Runtime validation rejects mismatched connection/type
+            connection: s3 as unknown as ReturnType<typeof defineKafkaConnection>,
+            topic: "events_out",
+            schedule: "@on-demand",
+          },
+        })
+      ).toThrow("requires a Kafka connection");
+    });
+
+    it("throws when sink configuration is passed to definePipe", () => {
+      const kafka = defineKafkaConnection("events_kafka", {
+        bootstrapServers: "localhost:9092",
+      });
+
+      expect(() =>
+        definePipe(
+          "bad_via_define_pipe",
+          {
+            nodes: [node({ name: "export", sql: "SELECT * FROM events" })],
+            sink: {
+              connection: kafka,
+              topic: "events_out",
+            },
+          } as unknown as Parameters<typeof definePipe>[1]
+        )
+      ).toThrow("must be created with defineSinkPipe");
+    });
+  });
+
   describe("getEndpointConfig", () => {
     it("returns null when endpoint is false", () => {
       const pipe = definePipe("my_pipe", {

package/src/schema/pipe.ts

@@ -9,6 +9,8 @@ import type { DatasourceDefinition, SchemaDefinition, ColumnDefinition } from ".
 import { getColumnType } from "./datasource.js";
 import { getTinybirdType } from "./types.js";
 import type { TokenDefinition, PipeTokenScope } from "./token.js";
+import type { KafkaConnectionDefinition, S3ConnectionDefinition } from "./connection.js";
+import { isKafkaConnectionDefinition, isS3ConnectionDefinition } from "./connection.js";
 
 /** Symbol for brand typing pipes - use Symbol.for() for global registry */
 export const PIPE_BRAND = Symbol.for("tinybird.pipe");
@@ -155,6 +157,55 @@ export interface CopyConfig<
   copy_schedule?: string;
 }
 
+/**
+ * Sink export strategy.
+ * - 'create_new': write new files on each run
+ * - 'replace': replace destination data on each run
+ */
+export type SinkStrategy = "create_new" | "replace";
+
+/**
+ * S3 sink compression codec.
+ */
+export type SinkCompression = "none" | "gzip" | "snappy";
+
+/**
+ * Kafka sink configuration
+ */
+export interface KafkaSinkConfig {
+  /** Kafka connection used to publish records */
+  connection: KafkaConnectionDefinition;
+  /** Destination Kafka topic */
+  topic: string;
+  /** Sink schedule (for example: @on-demand, @once, cron expression) */
+  schedule: string;
+}
+
+/**
+ * S3 sink configuration
+ */
+export interface S3SinkConfig {
+  /** S3 connection used to write exported files */
+  connection: S3ConnectionDefinition;
+  /** Destination bucket URI (for example: s3://bucket/prefix/) */
+  bucketUri: string;
+  /** Output filename template (supports Tinybird placeholders) */
+  fileTemplate: string;
+  /** Output format (for example: csv, ndjson) */
+  format: string;
+  /** Sink schedule (for example: @on-demand, @once, cron expression) */
+  schedule: string;
+  /** Export strategy */
+  strategy?: SinkStrategy;
+  /** Compression codec */
+  compression?: SinkCompression;
+}
+
+/**
+ * Sink pipe configuration (Kafka or S3 only)
+ */
+export type SinkConfig = KafkaSinkConfig | S3SinkConfig;
+
 /**
  * Inline token configuration for pipe access
  */
@@ -194,9 +245,9 @@ export interface PipeOptions<
   nodes: readonly NodeDefinition[];
   /** Output schema (optional for reusable pipes, required for endpoints) */
   output?: TOutput;
-  /** Whether this pipe is an API endpoint (shorthand for { enabled: true }). Mutually exclusive with materialized and copy. */
+  /** Whether this pipe is an API endpoint (shorthand for { enabled: true }). Mutually exclusive with materialized, copy, and sink. */
   endpoint?: boolean | EndpointConfig;
-  /** Materialized view configuration. Mutually exclusive with endpoint and copy. */
+  /** Materialized view configuration. Mutually exclusive with endpoint, copy, and sink. */
   materialized?: MaterializedConfig;
   /** Copy pipe configuration. Mutually exclusive with endpoint and materialized. */
   copy?: CopyConfig;
@@ -204,6 +255,33 @@ export interface PipeOptions<
   tokens?: readonly PipeTokenConfig[];
 }
 
+/**
+ * Options for defining a sink pipe
+ */
+export interface SinkPipeOptions<TParams extends ParamsDefinition> {
+  /** Human-readable description of the sink pipe */
+  description?: string;
+  /** Parameter definitions for query inputs */
+  params?: TParams;
+  /** Nodes in the transformation pipeline */
+  nodes: readonly NodeDefinition[];
+  /** Sink export configuration */
+  sink: SinkConfig;
+  /** Sink pipes are not endpoints */
+  endpoint?: never;
+  /** Sink pipes are not materialized views */
+  materialized?: never;
+  /** Sink pipes are not copy pipes */
+  copy?: never;
+  /** Access tokens for this sink pipe */
+  tokens?: readonly PipeTokenConfig[];
+}
+
+type PipeRuntimeOptions<
+  TParams extends ParamsDefinition,
+  TOutput extends OutputDefinition
+> = (PipeOptions<TParams, TOutput> & { sink?: never }) | SinkPipeOptions<TParams>;
+
 /**
  * Options for defining an endpoint (API-exposed pipe)
  */
@@ -277,7 +355,7 @@ export interface PipeDefinition<
   /** Output schema (optional for reusable pipes) */
   readonly _output?: TOutput;
   /** Full options */
-  readonly options: PipeOptions<TParams, TOutput>;
+  readonly options: PipeRuntimeOptions<TParams, TOutput>;
 }
 
 /**
@@ -428,6 +506,52 @@ function validateMaterializedSchema(
   }
 }
 
+function validateSinkConfig(pipeName: string, sink: SinkConfig): void {
+  if ("topic" in sink) {
+    if (!isKafkaConnectionDefinition(sink.connection)) {
+      throw new Error(
+        `Pipe "${pipeName}" sink with topic requires a Kafka connection.`
+      );
+    }
+    if (typeof sink.topic !== "string" || !sink.topic.trim()) {
+      throw new Error(`Pipe "${pipeName}" sink topic cannot be empty.`);
+    }
+    if (typeof sink.schedule !== "string" || !sink.schedule.trim()) {
+      throw new Error(`Pipe "${pipeName}" sink schedule cannot be empty.`);
+    }
+    return;
+  }
+
+  if (!isS3ConnectionDefinition(sink.connection)) {
+    throw new Error(
+      `Pipe "${pipeName}" S3 sink requires an S3 connection.`
+    );
+  }
+  if (typeof sink.bucketUri !== "string" || !sink.bucketUri.trim()) {
+    throw new Error(`Pipe "${pipeName}" sink bucketUri cannot be empty.`);
+  }
+  if (typeof sink.fileTemplate !== "string" || !sink.fileTemplate.trim()) {
+    throw new Error(`Pipe "${pipeName}" sink fileTemplate cannot be empty.`);
+  }
+  if (typeof sink.format !== "string" || !sink.format.trim()) {
+    throw new Error(`Pipe "${pipeName}" sink format cannot be empty.`);
+  }
+  if (typeof sink.schedule !== "string" || !sink.schedule.trim()) {
+    throw new Error(`Pipe "${pipeName}" sink schedule cannot be empty.`);
+  }
+  if (sink.strategy && sink.strategy !== "create_new" && sink.strategy !== "replace") {
+    throw new Error(`Pipe "${pipeName}" sink strategy must be "create_new" or "replace".`);
+  }
+  if (
+    sink.compression &&
+    sink.compression !== "none" &&
+    sink.compression !== "gzip" &&
+    sink.compression !== "snappy"
+  ) {
+    throw new Error(`Pipe "${pipeName}" sink compression must be "none", "gzip", or "snappy".`);
+  }
+}
+
 export function definePipe<
   TParams extends ParamsDefinition,
   TOutput extends OutputDefinition
@@ -447,6 +571,12 @@ export function definePipe<
     throw new Error(`Pipe "${name}" must have at least one node.`);
   }
 
+  if ("sink" in (options as unknown as object)) {
+    throw new Error(
+      `Pipe "${name}" sink configuration must be created with defineSinkPipe().`
+    );
+  }
+
   // Validate output is provided for endpoints and materialized views
   if ((options.endpoint || options.materialized) && (!options.output || Object.keys(options.output).length === 0)) {
     throw new Error(
@@ -480,7 +610,47 @@ export function definePipe<
     options: {
       ...options,
       params,
-    },
+    } as PipeRuntimeOptions<TParams, TOutput>,
+  };
+}
+
+/**
+ * Define a Tinybird sink pipe
+ *
+ * Sink pipes export query results to external systems via Kafka or S3.
+ *
+ * @param name - The sink pipe name (must be valid identifier)
+ * @param options - Sink pipe configuration
+ * @returns A pipe definition configured as a sink pipe
+ */
+export function defineSinkPipe<TParams extends ParamsDefinition>(
+  name: string,
+  options: SinkPipeOptions<TParams>
+): PipeDefinition<TParams, Record<string, never>> {
+  if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) {
+    throw new Error(
+      `Invalid pipe name: "${name}". Must start with a letter or underscore and contain only alphanumeric characters and underscores.`
+    );
+  }
+
+  if (!options.nodes || options.nodes.length === 0) {
+    throw new Error(`Pipe "${name}" must have at least one node.`);
+  }
+
+  validateSinkConfig(name, options.sink);
+
+  const params = (options.params ?? {}) as TParams;
+
+  return {
+    [PIPE_BRAND]: true,
+    _name: name,
+    _type: "pipe",
+    _params: params,
+    _output: undefined,
+    options: {
+      ...options,
+      params,
+    } as PipeRuntimeOptions<TParams, Record<string, never>>,
   };
 }
 
@@ -790,6 +960,20 @@ export function isCopyPipe(pipe: PipeDefinition): boolean {
   return pipe.options.copy !== undefined;
 }
 
+/**
+ * Get the sink configuration from a pipe
+ */
+export function getSinkConfig(pipe: PipeDefinition): SinkConfig | null {
+  return "sink" in pipe.options ? (pipe.options.sink ?? null) : null;
+}
+
+/**
+ * Check if a pipe is a sink pipe
+ */
+export function isSinkPipe(pipe: PipeDefinition): boolean {
+  return pipe.options.sink !== undefined;
+}
+
 /**
  * Get all node names from a pipe
  */

package/src/schema/secret.test.ts

@@ -0,0 +1,19 @@
+import { describe, expect, it } from "vitest";
+import { secret } from "./secret.js";
+
+describe("secret helper", () => {
+  it("creates a secret template without default", () => {
+    expect(secret("KAFKA_KEY")).toBe('{{ tb_secret("KAFKA_KEY") }}');
+  });
+
+  it("creates a secret template with default", () => {
+    expect(secret("KAFKA_GROUP_ID", "events_group")).toBe(
+      '{{ tb_secret("KAFKA_GROUP_ID", "events_group") }}'
+    );
+  });
+
+  it("throws on empty secret name", () => {
+    expect(() => secret("")).toThrow("Secret name must be a non-empty string.");
+  });
+});
+

package/src/schema/secret.ts

@@ -0,0 +1,16 @@
+/**
+ * Secret template helper.
+ * Produces Tinybird-compatible `tb_secret(...)` template strings.
+ */
+export function secret(name: string, defaultValue?: string): string {
+  if (!name || name.trim().length === 0) {
+    throw new Error("Secret name must be a non-empty string.");
+  }
+
+  if (defaultValue === undefined) {
+    return `{{ tb_secret("${name}") }}`;
+  }
+
+  return `{{ tb_secret("${name}", "${defaultValue}") }}`;
+}
+

package/src/schema/types.test.ts

@@ -115,6 +115,20 @@ describe('Type Validators (t.*)', () => {
     });
   });
 
+  describe('jsonPath modifier', () => {
+    it('sets jsonPath in modifiers', () => {
+      const type = t.string().jsonPath('$.payload.id');
+      expect(type._modifiers.jsonPath).toBe('$.payload.id');
+    });
+
+    it('supports chaining with other modifiers', () => {
+      const type = t.string().nullable().jsonPath('$.user.name');
+      expect(type._tinybirdType).toBe('Nullable(String)');
+      expect(type._modifiers.nullable).toBe(true);
+      expect(type._modifiers.jsonPath).toBe('$.user.name');
+    });
+  });
+
   describe('Complex types', () => {
     it('generates Array type', () => {
       const type = t.array(t.string());

package/src/schema/types.ts

@@ -32,6 +32,8 @@ export interface TypeValidator<
   default(value: TType): TypeValidator<TType, TTinybirdType, TModifiers & { hasDefault: true; defaultValue: TType }>;
   /** Set a codec for compression */
   codec(codec: string): TypeValidator<TType, TTinybirdType, TModifiers & { codec: string }>;
+  /** Set an explicit JSON path for extraction (overrides autogenerated path) */
+  jsonPath(path: string): TypeValidator<TType, TTinybirdType, TModifiers & { jsonPath: string }>;
 }
 
 export interface TypeModifiers {
@@ -40,6 +42,7 @@ export interface TypeModifiers {
   hasDefault?: boolean;
   defaultValue?: unknown;
   codec?: string;
+  jsonPath?: string;
 }
 
 // Internal implementation
@@ -110,6 +113,13 @@ function createValidator<TType, TTinybirdType extends string>(
         codec,
       }) as TypeValidator<TType, TTinybirdType, TypeModifiers & { codec: string }>;
     },
+
+    jsonPath(path: string) {
+      return createValidator<TType, TTinybirdType>(tinybirdType, {
+        ...modifiers,
+        jsonPath: path,
+      }) as TypeValidator<TType, TTinybirdType, TypeModifiers & { jsonPath: string }>;
+    },
   };
 
   return validator;