@tinybirdco/sdk 0.0.1

package/src/generator/pipe.test.ts

@@ -0,0 +1,441 @@
+import { describe, it, expect } from 'vitest';
+import { generatePipe, generateAllPipes } from './pipe.js';
+import { definePipe, defineMaterializedView, node } from '../schema/pipe.js';
+import { defineDatasource } from '../schema/datasource.js';
+import { t } from '../schema/types.js';
+import { p } from '../schema/params.js';
+import { engine } from '../schema/engines.js';
+
+// Helper to create a simple output schema for tests
+const simpleOutput = { result: t.int32() };
+
+describe('Pipe Generator', () => {
+  describe('generatePipe', () => {
+    it('generates basic pipe with node', () => {
+      const pipe = definePipe('test_pipe', {
+        nodes: [
+          node({
+            name: 'endpoint',
+            sql: 'SELECT * FROM table',
+          }),
+        ],
+        output: simpleOutput,
+        endpoint: true,
+      });
+
+      const result = generatePipe(pipe);
+      expect(result.name).toBe('test_pipe');
+      expect(result.content).toContain('NODE endpoint');
+      expect(result.content).toContain('SQL >');
+      expect(result.content).toContain('SELECT * FROM table');
+    });
+
+    it('includes description when provided', () => {
+      const pipe = definePipe('test_pipe', {
+        description: 'Test pipe description',
+        nodes: [node({ name: 'endpoint', sql: 'SELECT 1' })],
+        output: simpleOutput,
+        endpoint: true,
+      });
+
+      const result = generatePipe(pipe);
+      expect(result.content).toContain('DESCRIPTION >');
+      expect(result.content).toContain('Test pipe description');
+    });
+
+    it('includes TYPE endpoint when endpoint is true', () => {
+      const pipe = definePipe('test_pipe', {
+        nodes: [node({ name: 'endpoint', sql: 'SELECT 1' })],
+        output: simpleOutput,
+        endpoint: true,
+      });
+
+      const result = generatePipe(pipe);
+      expect(result.content).toContain('TYPE endpoint');
+    });
+
+    it('does not include TYPE endpoint when endpoint is false', () => {
+      const pipe = definePipe('test_pipe', {
+        nodes: [node({ name: 'endpoint', sql: 'SELECT 1' })],
+        output: simpleOutput,
+        endpoint: false,
+      });
+
+      const result = generatePipe(pipe);
+      expect(result.content).not.toContain('TYPE endpoint');
+    });
+  });
+
+  describe('Dynamic SQL detection', () => {
+    it('adds % on its own line for SQL with template parameters', () => {
+      const pipe = definePipe('test_pipe', {
+        nodes: [
+          node({
+            name: 'endpoint',
+            sql: 'SELECT * FROM table WHERE id = {{Int32(id)}}',
+          }),
+        ],
+        output: simpleOutput,
+        endpoint: true,
+      });
+
+      const result = generatePipe(pipe);
+      expect(result.content).toContain('SQL >\n    %\n    SELECT');
+    });
+
+    it('adds % for SQL with DateTime parameter', () => {
+      const pipe = definePipe('test_pipe', {
+        nodes: [
+          node({
+            name: 'endpoint',
+            sql: 'SELECT * FROM table WHERE timestamp >= {{DateTime(start_date)}}',
+          }),
+        ],
+        output: simpleOutput,
+        endpoint: true,
+      });
+
+      const result = generatePipe(pipe);
+      expect(result.content).toContain('    %\n');
+    });
+
+    it('does not add % for SQL without template parameters', () => {
+      const pipe = definePipe('test_pipe', {
+        nodes: [
+          node({
+            name: 'endpoint',
+            sql: 'SELECT * FROM table',
+          }),
+        ],
+        output: simpleOutput,
+        endpoint: true,
+      });
+
+      const result = generatePipe(pipe);
+      expect(result.content).not.toContain('%');
+    });
+
+    it('does not add % for SQL with curly braces that are not parameters', () => {
+      const pipe = definePipe('test_pipe', {
+        nodes: [
+          node({
+            name: 'endpoint',
+            sql: "SELECT JSONExtract(data, 'field', 'String') FROM table",
+          }),
+        ],
+        output: simpleOutput,
+        endpoint: true,
+      });
+
+      const result = generatePipe(pipe);
+      expect(result.content).not.toContain('%');
+    });
+  });
+
+  describe('Multiple nodes', () => {
+    it('generates all nodes with separation', () => {
+      const pipe = definePipe('test_pipe', {
+        nodes: [
+          node({ name: 'first', sql: 'SELECT * FROM table1' }),
+          node({ name: 'second', sql: 'SELECT * FROM first' }),
+        ],
+        output: simpleOutput,
+        endpoint: true,
+      });
+
+      const result = generatePipe(pipe);
+      expect(result.content).toContain('NODE first');
+      expect(result.content).toContain('NODE second');
+      expect(result.content).toContain('SELECT * FROM table1');
+      expect(result.content).toContain('SELECT * FROM first');
+    });
+
+    it('includes node descriptions', () => {
+      const pipe = definePipe('test_pipe', {
+        nodes: [
+          node({
+            name: 'endpoint',
+            description: 'This is a test node',
+            sql: 'SELECT 1',
+          }),
+        ],
+        output: simpleOutput,
+        endpoint: true,
+      });
+
+      const result = generatePipe(pipe);
+      expect(result.content).toContain('NODE endpoint');
+      expect(result.content).toContain('DESCRIPTION >');
+      expect(result.content).toContain('This is a test node');
+    });
+  });
+
+  describe('Endpoint configuration', () => {
+    it('includes cache when enabled', () => {
+      const pipe = definePipe('test_pipe', {
+        nodes: [node({ name: 'endpoint', sql: 'SELECT 1' })],
+        output: simpleOutput,
+        endpoint: {
+          enabled: true,
+          cache: { enabled: true, ttl: 300 },
+        },
+      });
+
+      const result = generatePipe(pipe);
+      expect(result.content).toContain('TYPE endpoint');
+      expect(result.content).toContain('CACHE 300');
+    });
+
+    it('uses default cache TTL when not specified', () => {
+      const pipe = definePipe('test_pipe', {
+        nodes: [node({ name: 'endpoint', sql: 'SELECT 1' })],
+        output: simpleOutput,
+        endpoint: {
+          enabled: true,
+          cache: { enabled: true },
+        },
+      });
+
+      const result = generatePipe(pipe);
+      expect(result.content).toContain('CACHE 60');
+    });
+  });
+
+  describe('generateAllPipes', () => {
+    it('generates all pipes', () => {
+      const pipe1 = definePipe('pipe1', {
+        nodes: [node({ name: 'endpoint', sql: 'SELECT 1' })],
+        output: simpleOutput,
+        endpoint: true,
+      });
+      const pipe2 = definePipe('pipe2', {
+        nodes: [node({ name: 'endpoint', sql: 'SELECT 2' })],
+        output: simpleOutput,
+        endpoint: true,
+      });
+
+      const results = generateAllPipes({ pipe1, pipe2 });
+      expect(results).toHaveLength(2);
+      expect(results.map(r => r.name).sort()).toEqual(['pipe1', 'pipe2']);
+    });
+  });
+
+  describe('Full integration', () => {
+    it('generates complete pipe file', () => {
+      const pipe = definePipe('top_pages', {
+        description: 'Get the most visited pages',
+        params: {
+          start_date: p.dateTime(),
+          end_date: p.dateTime(),
+          limit: p.int32().optional(10),
+        },
+        nodes: [
+          node({
+            name: 'aggregated',
+            sql: `
+SELECT
+  pathname,
+  count() AS views
+FROM page_views
+WHERE timestamp >= {{DateTime(start_date)}}
+  AND timestamp <= {{DateTime(end_date)}}
+GROUP BY pathname
+ORDER BY views DESC
+LIMIT {{Int32(limit, 10)}}
+            `.trim(),
+          }),
+        ],
+        output: {
+          pathname: t.string(),
+          views: t.uint64(),
+        },
+        endpoint: true,
+      });
+
+      const result = generatePipe(pipe);
+
+      expect(result.name).toBe('top_pages');
+      expect(result.content).toContain('DESCRIPTION >');
+      expect(result.content).toContain('Get the most visited pages');
+      expect(result.content).toContain('NODE aggregated');
+      expect(result.content).toContain('SQL >');
+      expect(result.content).toContain('    %\n');
+      expect(result.content).toContain('pathname');
+      expect(result.content).toContain('{{DateTime(start_date)}}');
+      expect(result.content).toContain('{{Int32(limit, 10)}}');
+      expect(result.content).toContain('TYPE endpoint');
+    });
+  });
+
+  describe('Materialized Views', () => {
+    const salesByHour = defineDatasource('sales_by_hour', {
+      schema: {
+        day: t.date(),
+        country: t.string().lowCardinality(),
+        total_sales: t.simpleAggregateFunction('sum', t.uint64()),
+      },
+      engine: engine.aggregatingMergeTree({
+        sortingKey: ['day', 'country'],
+      }),
+    });
+
+    it('generates TYPE MATERIALIZED and DATASOURCE with datasource', () => {
+      const pipe = definePipe('sales_by_hour_mv', {
+        nodes: [
+          node({
+            name: 'daily_sales',
+            sql: 'SELECT toStartOfDay(date) as day, country, sum(sales) as total_sales FROM teams GROUP BY day, country',
+          }),
+        ],
+        output: {
+          day: t.date(),
+          country: t.string().lowCardinality(),
+          total_sales: t.simpleAggregateFunction('sum', t.uint64()),
+        },
+        materialized: {
+          datasource: salesByHour,
+        },
+      });
+
+      const result = generatePipe(pipe);
+
+      expect(result.content).toContain('TYPE MATERIALIZED');
+      expect(result.content).toContain('DATASOURCE sales_by_hour');
+      expect(result.content).not.toContain('TYPE endpoint');
+    });
+
+    it('generates DEPLOYMENT_METHOD alter when specified', () => {
+      const pipe = definePipe('sales_by_hour_mv', {
+        nodes: [
+          node({
+            name: 'daily_sales',
+            sql: 'SELECT toStartOfDay(date) as day, country, sum(sales) as total_sales FROM teams GROUP BY day, country',
+          }),
+        ],
+        output: {
+          day: t.date(),
+          country: t.string().lowCardinality(),
+          total_sales: t.simpleAggregateFunction('sum', t.uint64()),
+        },
+        materialized: {
+          datasource: salesByHour,
+          deploymentMethod: 'alter',
+        },
+      });
+
+      const result = generatePipe(pipe);
+
+      expect(result.content).toContain('TYPE MATERIALIZED');
+      expect(result.content).toContain('DATASOURCE sales_by_hour');
+      expect(result.content).toContain('DEPLOYMENT_METHOD alter');
+    });
+
+    it('does not include DEPLOYMENT_METHOD when not specified', () => {
+      const pipe = definePipe('sales_by_hour_mv', {
+        nodes: [
+          node({
+            name: 'daily_sales',
+            sql: 'SELECT toStartOfDay(date) as day, country, sum(sales) as total_sales FROM teams GROUP BY day, country',
+          }),
+        ],
+        output: {
+          day: t.date(),
+          country: t.string().lowCardinality(),
+          total_sales: t.simpleAggregateFunction('sum', t.uint64()),
+        },
+        materialized: {
+          datasource: salesByHour,
+        },
+      });
+
+      const result = generatePipe(pipe);
+
+      expect(result.content).not.toContain('DEPLOYMENT_METHOD');
+    });
+
+    it('generates complete materialized view pipe file', () => {
+      const pipe = definePipe('sales_by_hour_mv', {
+        description: 'Aggregate sales per hour',
+        nodes: [
+          node({
+            name: 'daily_sales',
+            sql: `
+SELECT
+  toStartOfDay(starting_date) as day,
+  country,
+  sum(sales) as total_sales
+FROM teams
+GROUP BY day, country
+            `.trim(),
+          }),
+        ],
+        output: {
+          day: t.date(),
+          country: t.string().lowCardinality(),
+          total_sales: t.simpleAggregateFunction('sum', t.uint64()),
+        },
+        materialized: {
+          datasource: salesByHour,
+          deploymentMethod: 'alter',
+        },
+      });
+
+      const result = generatePipe(pipe);
+
+      expect(result.name).toBe('sales_by_hour_mv');
+      expect(result.content).toContain('DESCRIPTION >');
+      expect(result.content).toContain('Aggregate sales per hour');
+      expect(result.content).toContain('NODE daily_sales');
+      expect(result.content).toContain('SQL >');
+      expect(result.content).toContain('toStartOfDay(starting_date) as day');
+      expect(result.content).toContain('TYPE MATERIALIZED');
+      expect(result.content).toContain('DATASOURCE sales_by_hour');
+      expect(result.content).toContain('DEPLOYMENT_METHOD alter');
+    });
+
+    it('works with defineMaterializedView helper using datasource', () => {
+      const pipe = defineMaterializedView('sales_mv', {
+        description: 'Sales materialized view',
+        datasource: salesByHour,
+        nodes: [
+          node({
+            name: 'daily_sales',
+            sql: 'SELECT toStartOfDay(date) as day, country, sum(sales) as total_sales FROM events GROUP BY day, country',
+          }),
+        ],
+        deploymentMethod: 'alter',
+      });
+
+      const result = generatePipe(pipe);
+
+      expect(result.name).toBe('sales_mv');
+      expect(result.content).toContain('TYPE MATERIALIZED');
+      expect(result.content).toContain('DATASOURCE sales_by_hour');
+      expect(result.content).toContain('DEPLOYMENT_METHOD alter');
+    });
+
+    it('generates DATASOURCE correctly with datasource field', () => {
+      const pipe = definePipe('sales_by_hour_mv_2', {
+        nodes: [
+          node({
+            name: 'daily_sales',
+            sql: 'SELECT toStartOfDay(date) as day, country, sum(sales) as total_sales FROM teams GROUP BY day, country',
+          }),
+        ],
+        output: {
+          day: t.date(),
+          country: t.string().lowCardinality(),
+          total_sales: t.simpleAggregateFunction('sum', t.uint64()),
+        },
+        materialized: {
+          datasource: salesByHour,
+        },
+      });
+
+      const result = generatePipe(pipe);
+
+      expect(result.content).toContain('TYPE MATERIALIZED');
+      expect(result.content).toContain('DATASOURCE sales_by_hour');
+    });
+  });
+});

package/src/generator/pipe.ts

@@ -0,0 +1,220 @@
+/**
+ * Pipe content generator
+ * Converts PipeDefinition to native .pipe file format
+ */
+
+import type {
+  PipeDefinition,
+  NodeDefinition,
+  EndpointConfig,
+  MaterializedConfig,
+  CopyConfig,
+} from "../schema/pipe.js";
+import { getEndpointConfig, getMaterializedConfig, getCopyConfig } from "../schema/pipe.js";
+
+/**
+ * Generated pipe content
+ */
+export interface GeneratedPipe {
+  /** Pipe name */
+  name: string;
+  /** The generated .pipe file content */
+  content: string;
+}
+
+/**
+ * Check if SQL contains template parameters like {{...}}
+ */
+function hasDynamicParameters(sql: string): boolean {
+  return /\{\{[^}]+\}\}/.test(sql);
+}
+
+/**
+ * Generate a NODE section for the pipe
+ */
+function generateNode(node: NodeDefinition): string {
+  const parts: string[] = [];
+
+  parts.push(`NODE ${node._name}`);
+
+  if (node.description) {
+    parts.push(`DESCRIPTION >`);
+    parts.push(`    ${node.description}`);
+  }
+
+  parts.push(`SQL >`);
+
+  // Check if SQL has dynamic parameters - if so, add % on its own line
+  const isDynamic = hasDynamicParameters(node.sql);
+  if (isDynamic) {
+    parts.push(`    %`);
+  }
+
+  const sqlLines = node.sql.trim().split("\n");
+  sqlLines.forEach((line) => {
+    parts.push(`    ${line}`);
+  });
+
+  return parts.join("\n");
+}
+
+/**
+ * Generate the TYPE endpoint section
+ */
+function generateEndpoint(endpoint: EndpointConfig): string {
+  const parts: string[] = ["TYPE endpoint"];
+
+  if (endpoint.cache?.enabled) {
+    if (endpoint.cache.ttl !== undefined) {
+      parts.push(`CACHE ${endpoint.cache.ttl}`);
+    } else {
+      parts.push("CACHE 60"); // Default cache TTL
+    }
+  }
+
+  return parts.join("\n");
+}
+
+/**
+ * Generate the TYPE MATERIALIZED section
+ */
+function generateMaterialized(config: MaterializedConfig): string {
+  const parts: string[] = ["TYPE MATERIALIZED"];
+
+  // The config is normalized by definePipe to always have `datasource` set.
+  // Use non-null assertion since we know it's always present after normalization.
+  const datasourceName = config.datasource!._name;
+  parts.push(`DATASOURCE ${datasourceName}`);
+
+  if (config.deploymentMethod === "alter") {
+    parts.push("DEPLOYMENT_METHOD alter");
+  }
+
+  return parts.join("\n");
+}
+
+/**
+ * Generate the TYPE COPY section
+ */
+function generateCopy(config: CopyConfig): string {
+  const parts: string[] = ["TYPE COPY"];
+
+  const datasourceName = config.datasource._name;
+  parts.push(`TARGET_DATASOURCE ${datasourceName}`);
+
+  if (config.copy_schedule) {
+    parts.push(`COPY_SCHEDULE ${config.copy_schedule}`);
+  }
+
+  if (config.copy_mode) {
+    parts.push(`COPY_MODE ${config.copy_mode}`);
+  }
+
+  return parts.join("\n");
+}
+
+/**
+ * Generate a .pipe file content from a PipeDefinition
+ *
+ * @param pipe - The pipe definition
+ * @returns Generated pipe content
+ *
+ * @example
+ * ```ts
+ * const topEvents = definePipe('top_events', {
+ *   description: 'Get top events by count',
+ *   params: {
+ *     start_date: p.dateTime(),
+ *     limit: p.int32().optional(10),
+ *   },
+ *   nodes: [
+ *     node({
+ *       name: 'endpoint',
+ *       sql: `
+ *         SELECT event_type, count() as count
+ *         FROM events
+ *         WHERE timestamp >= {{DateTime(start_date)}}
+ *         ORDER BY count DESC
+ *         LIMIT {{Int32(limit, 10)}}
+ *       `,
+ *     }),
+ *   ],
+ *   output: {
+ *     event_type: t.string(),
+ *     count: t.uint64(),
+ *   },
+ *   endpoint: true,
+ * });
+ *
+ * const { content } = generatePipe(topEvents);
+ * // Returns:
+ * // DESCRIPTION >
+ * //     Get top events by count
+ * //
+ * // NODE endpoint
+ * // SQL >
+ * //     SELECT event_type, count() as count
+ * //     FROM events
+ * //     WHERE timestamp >= {{DateTime(start_date)}}
+ * //     ORDER BY count DESC
+ * //     LIMIT {{Int32(limit, 10)}}
+ * //
+ * // TYPE endpoint
+ * ```
+ */
+export function generatePipe(pipe: PipeDefinition): GeneratedPipe {
+  const parts: string[] = [];
+
+  // Add description if present
+  if (pipe.options.description) {
+    parts.push(`DESCRIPTION >\n    ${pipe.options.description}`);
+    parts.push("");
+  }
+
+  // Add all nodes
+  pipe.options.nodes.forEach((node, index) => {
+    parts.push(generateNode(node));
+    // Add empty line between nodes
+    if (index < pipe.options.nodes.length - 1) {
+      parts.push("");
+    }
+  });
+
+  // Add endpoint configuration if this is an endpoint
+  const endpointConfig = getEndpointConfig(pipe);
+  if (endpointConfig) {
+    parts.push("");
+    parts.push(generateEndpoint(endpointConfig));
+  }
+
+  // Add materialized view configuration if this is a materialized view
+  const materializedConfig = getMaterializedConfig(pipe);
+  if (materializedConfig) {
+    parts.push("");
+    parts.push(generateMaterialized(materializedConfig));
+  }
+
+  // Add copy pipe configuration if this is a copy pipe
+  const copyConfig = getCopyConfig(pipe);
+  if (copyConfig) {
+    parts.push("");
+    parts.push(generateCopy(copyConfig));
+  }
+
+  return {
+    name: pipe._name,
+    content: parts.join("\n"),
+  };
+}
+
+/**
+ * Generate .pipe files for all pipes in a project
+ *
+ * @param pipes - Record of pipe definitions
+ * @returns Array of generated pipe content
+ */
+export function generateAllPipes(
+  pipes: Record<string, PipeDefinition>
+): GeneratedPipe[] {
+  return Object.values(pipes).map(generatePipe);
+}