@tinybirdco/sdk 0.0.1

package/src/schema/datasource.ts

@@ -0,0 +1,195 @@
+/**
+ * Datasource definition for Tinybird
+ * Define table schemas as TypeScript with full type safety
+ */
+
+import type { AnyTypeValidator } from "./types.js";
+import type { EngineConfig } from "./engines.js";
+
+// Symbol for brand typing
+const DATASOURCE_BRAND = Symbol("tinybird.datasource");
+
+/**
+ * A column can be defined as just a type validator,
+ * or with additional options like JSON path or default value
+ */
+export interface ColumnDefinition<T extends AnyTypeValidator = AnyTypeValidator> {
+  /** The column type */
+  type: T;
+  /** JSON path for extracting from nested JSON (e.g., '$.user.id') */
+  jsonPath?: string;
+}
+
+/**
+ * Schema definition is a record of column names to type validators or column definitions
+ */
+export type SchemaDefinition = Record<string, AnyTypeValidator | ColumnDefinition>;
+
+/**
+ * Token configuration for datasource access
+ */
+export interface TokenConfig {
+  /** Token name */
+  name: string;
+  /** Permissions granted to this token */
+  permissions: readonly ("READ" | "APPEND")[];
+}
+
+/**
+ * Options for defining a datasource
+ */
+export interface DatasourceOptions<TSchema extends SchemaDefinition> {
+  /** Human-readable description of the datasource */
+  description?: string;
+  /** Column schema definition */
+  schema: TSchema;
+  /** Table engine configuration */
+  engine?: EngineConfig;
+  /** Access tokens for this datasource */
+  tokens?: readonly TokenConfig[];
+  /** Workspaces to share this datasource with */
+  sharedWith?: readonly string[];
+  /**
+   * Whether to generate JSON path expressions for columns.
+   * Set to false for datasources that are targets of materialized views.
+   * Defaults to true.
+   */
+  jsonPaths?: boolean;
+}
+
+/**
+ * A datasource definition with full type information
+ */
+export interface DatasourceDefinition<TSchema extends SchemaDefinition = SchemaDefinition> {
+  readonly [DATASOURCE_BRAND]: true;
+  /** Datasource name */
+  readonly _name: string;
+  /** Type marker for inference */
+  readonly _type: "datasource";
+  /** Schema definition */
+  readonly _schema: TSchema;
+  /** Full options */
+  readonly options: DatasourceOptions<TSchema>;
+}
+
+/**
+ * Define a Tinybird datasource
+ *
+ * @param name - The datasource name (must be valid identifier)
+ * @param options - Datasource configuration including schema and engine
+ * @returns A datasource definition that can be used in a project
+ *
+ * @example
+ * ```ts
+ * import { defineDatasource, t, engine } from '@tinybirdco/sdk';
+ *
+ * export const events = defineDatasource('events', {
+ *   description: 'User event tracking data',
+ *   schema: {
+ *     timestamp: t.dateTime(),
+ *     event_id: t.uuid(),
+ *     user_id: t.string(),
+ *     event_type: t.string().lowCardinality(),
+ *     properties: t.json(),
+ *     session_id: t.string().nullable(),
+ *   },
+ *   engine: engine.mergeTree({
+ *     sortingKey: ['user_id', 'timestamp'],
+ *     partitionKey: 'toYYYYMM(timestamp)',
+ *     ttl: 'timestamp + INTERVAL 90 DAY',
+ *   }),
+ * });
+ * ```
+ */
+export function defineDatasource<TSchema extends SchemaDefinition>(
+  name: string,
+  options: DatasourceOptions<TSchema>
+): DatasourceDefinition<TSchema> {
+  // Validate name is a valid identifier
+  if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) {
+    throw new Error(
+      `Invalid datasource name: "${name}". Must start with a letter or underscore and contain only alphanumeric characters and underscores.`
+    );
+  }
+
+  return {
+    [DATASOURCE_BRAND]: true,
+    _name: name,
+    _type: "datasource",
+    _schema: options.schema,
+    options,
+  };
+}
+
+/**
+ * Check if a value is a datasource definition
+ */
+export function isDatasourceDefinition(value: unknown): value is DatasourceDefinition {
+  return (
+    typeof value === "object" &&
+    value !== null &&
+    DATASOURCE_BRAND in value &&
+    (value as Record<symbol, unknown>)[DATASOURCE_BRAND] === true
+  );
+}
+
+/**
+ * Get the column type for a schema entry (handles both raw validators and column definitions)
+ */
+export function getColumnType(column: AnyTypeValidator | ColumnDefinition): AnyTypeValidator {
+  if ("type" in column && typeof column.type === "object") {
+    return column.type;
+  }
+  return column as AnyTypeValidator;
+}
+
+/**
+ * Get the JSON path for a column if defined
+ */
+export function getColumnJsonPath(column: AnyTypeValidator | ColumnDefinition): string | undefined {
+  if ("jsonPath" in column) {
+    return column.jsonPath;
+  }
+  return undefined;
+}
+
+/**
+ * Get all column names from a schema
+ */
+export function getColumnNames<TSchema extends SchemaDefinition>(
+  schema: TSchema
+): (keyof TSchema)[] {
+  return Object.keys(schema) as (keyof TSchema)[];
+}
+
+/**
+ * Helper type to extract the schema from a datasource definition
+ */
+export type ExtractSchema<T> = T extends DatasourceDefinition<infer S> ? S : never;
+
+/**
+ * Column definition helper for complex column configurations
+ *
+ * @example
+ * ```ts
+ * import { defineDatasource, t, column } from '@tinybirdco/sdk';
+ *
+ * export const events = defineDatasource('events', {
+ *   schema: {
+ *     // Simple column
+ *     id: t.string(),
+ *     // Column with JSON extraction
+ *     user_id: column(t.string(), { jsonPath: '$.user.id' }),
+ *   },
+ * });
+ * ```
+ */
+export function column<T extends AnyTypeValidator>(
+  type: T,
+  options?: Omit<ColumnDefinition<T>, "type">
+): ColumnDefinition<T> {
+  return {
+    type,
+    ...options,
+  };
+}

package/src/schema/engines.test.ts

@@ -0,0 +1,247 @@
+import { describe, it, expect } from 'vitest';
+import { engine, getEngineClause, getSortingKey, getPrimaryKey } from './engines.js';
+
+describe('Engine Configurations', () => {
+  describe('MergeTree', () => {
+    it('creates MergeTree config', () => {
+      const config = engine.mergeTree({ sortingKey: ['id'] });
+      expect(config.type).toBe('MergeTree');
+      expect(config.sortingKey).toEqual(['id']);
+    });
+
+    it('supports string sortingKey', () => {
+      const config = engine.mergeTree({ sortingKey: 'id' });
+      expect(config.sortingKey).toBe('id');
+    });
+
+    it('supports partitionKey', () => {
+      const config = engine.mergeTree({
+        sortingKey: ['id'],
+        partitionKey: 'toYYYYMM(timestamp)',
+      });
+      expect(config.partitionKey).toBe('toYYYYMM(timestamp)');
+    });
+
+    it('supports TTL', () => {
+      const config = engine.mergeTree({
+        sortingKey: ['id'],
+        ttl: 'timestamp + INTERVAL 90 DAY',
+      });
+      expect(config.ttl).toBe('timestamp + INTERVAL 90 DAY');
+    });
+  });
+
+  describe('ReplacingMergeTree', () => {
+    it('creates ReplacingMergeTree config', () => {
+      const config = engine.replacingMergeTree({ sortingKey: ['id'] });
+      expect(config.type).toBe('ReplacingMergeTree');
+    });
+
+    it('supports version column', () => {
+      const config = engine.replacingMergeTree({
+        sortingKey: ['id'],
+        ver: 'updated_at',
+      });
+      expect(config.ver).toBe('updated_at');
+    });
+  });
+
+  describe('SummingMergeTree', () => {
+    it('creates SummingMergeTree config', () => {
+      const config = engine.summingMergeTree({ sortingKey: ['id'] });
+      expect(config.type).toBe('SummingMergeTree');
+    });
+
+    it('supports columns to sum', () => {
+      const config = engine.summingMergeTree({
+        sortingKey: ['date'],
+        columns: ['count', 'total'],
+      });
+      expect(config.columns).toEqual(['count', 'total']);
+    });
+  });
+
+  describe('AggregatingMergeTree', () => {
+    it('creates AggregatingMergeTree config', () => {
+      const config = engine.aggregatingMergeTree({ sortingKey: ['id'] });
+      expect(config.type).toBe('AggregatingMergeTree');
+    });
+  });
+
+  describe('CollapsingMergeTree', () => {
+    it('creates CollapsingMergeTree config', () => {
+      const config = engine.collapsingMergeTree({
+        sortingKey: ['id'],
+        sign: 'sign_col',
+      });
+      expect(config.type).toBe('CollapsingMergeTree');
+      expect(config.sign).toBe('sign_col');
+    });
+  });
+
+  describe('VersionedCollapsingMergeTree', () => {
+    it('creates VersionedCollapsingMergeTree config', () => {
+      const config = engine.versionedCollapsingMergeTree({
+        sortingKey: ['id'],
+        sign: 'sign_col',
+        version: 'version_col',
+      });
+      expect(config.type).toBe('VersionedCollapsingMergeTree');
+      expect(config.sign).toBe('sign_col');
+      expect(config.version).toBe('version_col');
+    });
+  });
+
+  describe('getEngineClause', () => {
+    it('generates basic MergeTree clause', () => {
+      const config = engine.mergeTree({ sortingKey: ['id'] });
+      const clause = getEngineClause(config);
+      expect(clause).toContain('ENGINE "MergeTree"');
+      expect(clause).toContain('ENGINE_SORTING_KEY "id"');
+    });
+
+    it('includes partition key', () => {
+      const config = engine.mergeTree({
+        sortingKey: ['id'],
+        partitionKey: 'toYYYYMM(timestamp)',
+      });
+      const clause = getEngineClause(config);
+      expect(clause).toContain('ENGINE_PARTITION_KEY "toYYYYMM(timestamp)"');
+    });
+
+    it('includes TTL', () => {
+      const config = engine.mergeTree({
+        sortingKey: ['id'],
+        ttl: 'timestamp + INTERVAL 90 DAY',
+      });
+      const clause = getEngineClause(config);
+      expect(clause).toContain('ENGINE_TTL "timestamp + INTERVAL 90 DAY"');
+    });
+
+    it('includes primary key when different from sorting key', () => {
+      const config = engine.mergeTree({
+        sortingKey: ['id', 'timestamp'],
+        primaryKey: ['id'],
+      });
+      const clause = getEngineClause(config);
+      expect(clause).toContain('ENGINE_SORTING_KEY "id, timestamp"');
+      expect(clause).toContain('ENGINE_PRIMARY_KEY "id"');
+    });
+
+    it('includes ReplacingMergeTree version column', () => {
+      const config = engine.replacingMergeTree({
+        sortingKey: ['id'],
+        ver: 'updated_at',
+      });
+      const clause = getEngineClause(config);
+      expect(clause).toContain('ENGINE "ReplacingMergeTree"');
+      expect(clause).toContain('ENGINE_VER "updated_at"');
+    });
+
+    it('includes SummingMergeTree columns', () => {
+      const config = engine.summingMergeTree({
+        sortingKey: ['date'],
+        columns: ['count', 'total'],
+      });
+      const clause = getEngineClause(config);
+      expect(clause).toContain('ENGINE "SummingMergeTree"');
+      expect(clause).toContain('ENGINE_SUMMING_COLUMNS "count, total"');
+    });
+
+    it('includes CollapsingMergeTree sign column', () => {
+      const config = engine.collapsingMergeTree({
+        sortingKey: ['id'],
+        sign: 'sign_col',
+      });
+      const clause = getEngineClause(config);
+      expect(clause).toContain('ENGINE "CollapsingMergeTree"');
+      expect(clause).toContain('ENGINE_SIGN "sign_col"');
+    });
+
+    it('includes VersionedCollapsingMergeTree sign and version', () => {
+      const config = engine.versionedCollapsingMergeTree({
+        sortingKey: ['id'],
+        sign: 'sign_col',
+        version: 'version_col',
+      });
+      const clause = getEngineClause(config);
+      expect(clause).toContain('ENGINE "VersionedCollapsingMergeTree"');
+      expect(clause).toContain('ENGINE_SIGN "sign_col"');
+      expect(clause).toContain('ENGINE_VERSION "version_col"');
+    });
+  });
+
+  describe('Helper functions', () => {
+    it('getSortingKey returns array from string', () => {
+      const config = engine.mergeTree({ sortingKey: 'id' });
+      expect(getSortingKey(config)).toEqual(['id']);
+    });
+
+    it('getSortingKey returns array from array', () => {
+      const config = engine.mergeTree({ sortingKey: ['id', 'timestamp'] });
+      expect(getSortingKey(config)).toEqual(['id', 'timestamp']);
+    });
+
+    it('getPrimaryKey defaults to sorting key', () => {
+      const config = engine.mergeTree({ sortingKey: ['id'] });
+      expect(getPrimaryKey(config)).toEqual(['id']);
+    });
+
+    it('getPrimaryKey returns explicit primary key', () => {
+      const config = engine.mergeTree({
+        sortingKey: ['id', 'timestamp'],
+        primaryKey: ['id'],
+      });
+      expect(getPrimaryKey(config)).toEqual(['id']);
+    });
+  });
+
+  describe('ENGINE_SETTINGS string escaping', () => {
+    it('quotes string values in settings', () => {
+      const config = engine.mergeTree({
+        sortingKey: ['id'],
+        settings: {
+          storage_policy: 'tiered',
+        },
+      });
+      const clause = getEngineClause(config);
+      expect(clause).toContain("storage_policy='tiered'");
+    });
+
+    it('does not quote numeric values in settings', () => {
+      const config = engine.mergeTree({
+        sortingKey: ['id'],
+        settings: {
+          index_granularity: 8192,
+        },
+      });
+      const clause = getEngineClause(config);
+      expect(clause).toContain('index_granularity=8192');
+      expect(clause).not.toContain("index_granularity='8192'");
+    });
+
+    it('escapes single quotes in string values', () => {
+      const config = engine.mergeTree({
+        sortingKey: ['id'],
+        settings: {
+          comment: "it's a test",
+        },
+      });
+      const clause = getEngineClause(config);
+      expect(clause).toContain("comment='it\\'s a test'");
+    });
+
+    it('handles mixed string and numeric settings', () => {
+      const config = engine.mergeTree({
+        sortingKey: ['id'],
+        settings: {
+          storage_policy: 'default',
+          index_granularity: 8192,
+        },
+      });
+      const clause = getEngineClause(config);
+      expect(clause).toContain("storage_policy='default'");
+      expect(clause).toContain('index_granularity=8192');
+    });
+  });
+});

package/src/schema/engines.ts

@@ -0,0 +1,271 @@
+/**
+ * Engine configurations for Tinybird datasources
+ * ClickHouse table engines determine how data is stored and queried
+ */
+
+/**
+ * Base configuration shared by all MergeTree engines
+ */
+export interface BaseMergeTreeConfig {
+  /** Columns used for sorting data within parts (required for all MergeTree engines) */
+  sortingKey: string | readonly string[];
+  /** Expression for partitioning data (e.g., 'toYYYYMM(timestamp)') */
+  partitionKey?: string;
+  /** Primary key columns (defaults to sortingKey if not specified) */
+  primaryKey?: string | readonly string[];
+  /** TTL expression for automatic data expiration (e.g., 'timestamp + INTERVAL 90 DAY') */
+  ttl?: string;
+  /** Additional engine settings */
+  settings?: Record<string, string | number | boolean>;
+}
+
+/**
+ * MergeTree engine configuration
+ * The most universal and functional table engine for high-load tasks
+ */
+export interface MergeTreeConfig extends BaseMergeTreeConfig {
+  type: "MergeTree";
+}
+
+/**
+ * ReplacingMergeTree engine configuration
+ * Removes duplicate rows with the same sorting key during merges
+ */
+export interface ReplacingMergeTreeConfig extends BaseMergeTreeConfig {
+  type: "ReplacingMergeTree";
+  /** Optional version column - rows with highest version are kept */
+  ver?: string;
+  /** Optional flag to enable clean mode (ClickHouse 23.2+) */
+  isDeleted?: string;
+}
+
+/**
+ * SummingMergeTree engine configuration
+ * Sums numeric columns for rows with the same sorting key during merges
+ */
+export interface SummingMergeTreeConfig extends BaseMergeTreeConfig {
+  type: "SummingMergeTree";
+  /** Columns to sum (if not specified, all numeric columns are summed) */
+  columns?: readonly string[];
+}
+
+/**
+ * AggregatingMergeTree engine configuration
+ * For incremental data aggregation with AggregateFunction columns
+ */
+export interface AggregatingMergeTreeConfig extends BaseMergeTreeConfig {
+  type: "AggregatingMergeTree";
+}
+
+/**
+ * CollapsingMergeTree engine configuration
+ * For collapsing rows that cancel each other out
+ */
+export interface CollapsingMergeTreeConfig extends BaseMergeTreeConfig {
+  type: "CollapsingMergeTree";
+  /** Column containing sign (1 for state, -1 for cancel) */
+  sign: string;
+}
+
+/**
+ * VersionedCollapsingMergeTree engine configuration
+ * For collapsing with versioning when events may arrive out of order
+ */
+export interface VersionedCollapsingMergeTreeConfig extends BaseMergeTreeConfig {
+  type: "VersionedCollapsingMergeTree";
+  /** Column containing sign (1 for state, -1 for cancel) */
+  sign: string;
+  /** Column containing version number */
+  version: string;
+}
+
+/**
+ * Union type of all engine configurations
+ */
+export type EngineConfig =
+  | MergeTreeConfig
+  | ReplacingMergeTreeConfig
+  | SummingMergeTreeConfig
+  | AggregatingMergeTreeConfig
+  | CollapsingMergeTreeConfig
+  | VersionedCollapsingMergeTreeConfig;
+
+/**
+ * Helper to normalize sorting key to array format
+ */
+function normalizeSortingKey(key: string | readonly string[]): readonly string[] {
+  return typeof key === "string" ? [key] : key;
+}
+
+/**
+ * Engine configuration builders
+ *
+ * @example
+ * ```ts
+ * import { engine } from '@tinybirdco/sdk';
+ *
+ * // Basic MergeTree
+ * engine.mergeTree({
+ *   sortingKey: ['user_id', 'timestamp'],
+ *   partitionKey: 'toYYYYMM(timestamp)',
+ * });
+ *
+ * // ReplacingMergeTree for upserts
+ * engine.replacingMergeTree({
+ *   sortingKey: ['id'],
+ *   ver: 'updated_at',
+ * });
+ *
+ * // SummingMergeTree for counters
+ * engine.summingMergeTree({
+ *   sortingKey: ['date', 'metric_name'],
+ *   columns: ['value'],
+ * });
+ * ```
+ */
+export const engine = {
+  /**
+   * MergeTree - The most universal engine for high-load tasks
+   * Best for: General-purpose analytics, logs, events
+   */
+  mergeTree: (config: Omit<MergeTreeConfig, "type">): MergeTreeConfig => ({
+    type: "MergeTree",
+    ...config,
+  }),
+
+  /**
+   * ReplacingMergeTree - Removes duplicates during background merges
+   * Best for: Maintaining latest state, upserts, slowly changing dimensions
+   *
+   * @param config.ver - Optional version column. Rows with highest version are kept.
+   */
+  replacingMergeTree: (
+    config: Omit<ReplacingMergeTreeConfig, "type">
+  ): ReplacingMergeTreeConfig => ({
+    type: "ReplacingMergeTree",
+    ...config,
+  }),
+
+  /**
+   * SummingMergeTree - Sums numeric columns during background merges
+   * Best for: Counters, metrics aggregation, pre-aggregated data
+   *
+   * @param config.columns - Columns to sum. If not specified, all numeric columns are summed.
+   */
+  summingMergeTree: (
+    config: Omit<SummingMergeTreeConfig, "type">
+  ): SummingMergeTreeConfig => ({
+    type: "SummingMergeTree",
+    ...config,
+  }),
+
+  /**
+   * AggregatingMergeTree - For incremental aggregation with AggregateFunction columns
+   * Best for: Materialized views, incremental aggregation pipelines
+   */
+  aggregatingMergeTree: (
+    config: Omit<AggregatingMergeTreeConfig, "type">
+  ): AggregatingMergeTreeConfig => ({
+    type: "AggregatingMergeTree",
+    ...config,
+  }),
+
+  /**
+   * CollapsingMergeTree - For collapsing state/cancel row pairs
+   * Best for: Changelog-style updates, mutable data with deletes
+   *
+   * @param config.sign - Column containing 1 (state) or -1 (cancel)
+   */
+  collapsingMergeTree: (
+    config: Omit<CollapsingMergeTreeConfig, "type">
+  ): CollapsingMergeTreeConfig => ({
+    type: "CollapsingMergeTree",
+    ...config,
+  }),
+
+  /**
+   * VersionedCollapsingMergeTree - Collapsing with versioning for out-of-order events
+   * Best for: Changelog-style updates with potential out-of-order arrival
+   *
+   * @param config.sign - Column containing 1 (state) or -1 (cancel)
+   * @param config.version - Column containing version number for ordering
+   */
+  versionedCollapsingMergeTree: (
+    config: Omit<VersionedCollapsingMergeTreeConfig, "type">
+  ): VersionedCollapsingMergeTreeConfig => ({
+    type: "VersionedCollapsingMergeTree",
+    ...config,
+  }),
+} as const;
+
+/**
+ * Get the sorting key as an array
+ */
+export function getSortingKey(config: EngineConfig): readonly string[] {
+  return normalizeSortingKey(config.sortingKey);
+}
+
+/**
+ * Get the primary key as an array (defaults to sorting key)
+ */
+export function getPrimaryKey(config: EngineConfig): readonly string[] {
+  if (config.primaryKey) {
+    return normalizeSortingKey(config.primaryKey);
+  }
+  return getSortingKey(config);
+}
+
+/**
+ * Generate the engine clause for a datasource file
+ */
+export function getEngineClause(config: EngineConfig): string {
+  const parts: string[] = [`ENGINE "${config.type}"`];
+
+  if (config.partitionKey) {
+    parts.push(`ENGINE_PARTITION_KEY "${config.partitionKey}"`);
+  }
+
+  const sortingKey = getSortingKey(config);
+  parts.push(`ENGINE_SORTING_KEY "${sortingKey.join(", ")}"`);
+
+  if (config.primaryKey) {
+    const primaryKey = getPrimaryKey(config);
+    parts.push(`ENGINE_PRIMARY_KEY "${primaryKey.join(", ")}"`);
+  }
+
+  if (config.ttl) {
+    parts.push(`ENGINE_TTL "${config.ttl}"`);
+  }
+
+  // Engine-specific options
+  if (config.type === "ReplacingMergeTree" && config.ver) {
+    parts.push(`ENGINE_VER "${config.ver}"`);
+  }
+
+  if (config.type === "CollapsingMergeTree" || config.type === "VersionedCollapsingMergeTree") {
+    parts.push(`ENGINE_SIGN "${config.sign}"`);
+  }
+
+  if (config.type === "VersionedCollapsingMergeTree") {
+    parts.push(`ENGINE_VERSION "${config.version}"`);
+  }
+
+  if (config.type === "SummingMergeTree" && config.columns && config.columns.length > 0) {
+    parts.push(`ENGINE_SUMMING_COLUMNS "${config.columns.join(", ")}"`);
+  }
+
+  if (config.settings && Object.keys(config.settings).length > 0) {
+    const settingsStr = Object.entries(config.settings)
+      .map(([k, v]) => {
+        if (typeof v === "string") {
+          const escaped = v.replace(/'/g, "\\'");
+          return `${k}='${escaped}'`;
+        }
+        return `${k}=${v}`;
+      })
+      .join(", ");
+    parts.push(`ENGINE_SETTINGS "${settingsStr}"`);
+  }
+
+  return parts.join("\n");
+}