@tinybirdco/sdk 0.0.1

package/src/generator/datasource.ts

@@ -0,0 +1,217 @@
+/**
+ * Datasource content generator
+ * Converts DatasourceDefinition to native .datasource file format
+ */
+
+import type { DatasourceDefinition, SchemaDefinition, ColumnDefinition } from "../schema/datasource.js";
+import type { AnyTypeValidator, TypeModifiers } from "../schema/types.js";
+import { getColumnType, getColumnJsonPath } from "../schema/datasource.js";
+import { getEngineClause, type EngineConfig } from "../schema/engines.js";
+
+/**
+ * Generated datasource content
+ */
+export interface GeneratedDatasource {
+  /** Datasource name */
+  name: string;
+  /** The generated .datasource file content */
+  content: string;
+}
+
+/**
+ * Get the Tinybird type string from a type validator
+ * Handles the internal structure of validators
+ */
+function getTinybirdTypeFromValidator(validator: AnyTypeValidator): string {
+  // The validator has _tinybirdType as the type string
+  return validator._tinybirdType;
+}
+
+/**
+ * Get modifiers from a validator
+ */
+function getModifiersFromValidator(validator: AnyTypeValidator): TypeModifiers {
+  return validator._modifiers;
+}
+
+/**
+ * Format a default value for the datasource file
+ */
+function formatDefaultValue(value: unknown, tinybirdType: string): string {
+  if (value === null) {
+    return "NULL";
+  }
+
+  if (typeof value === "string") {
+    // Escape single quotes
+    return `'${value.replace(/'/g, "\\'")}'`;
+  }
+
+  if (typeof value === "number" || typeof value === "bigint") {
+    return String(value);
+  }
+
+  if (typeof value === "boolean") {
+    return value ? "1" : "0";
+  }
+
+  if (value instanceof Date) {
+    // Format based on type
+    if (tinybirdType.startsWith("Date") && !tinybirdType.includes("Time")) {
+      return `'${value.toISOString().split("T")[0]}'`;
+    }
+    return `'${value.toISOString().replace("T", " ").slice(0, 19)}'`;
+  }
+
+  // For arrays and objects, use raw JSON (no quotes)
+  if (Array.isArray(value)) {
+    return JSON.stringify(value);
+  }
+
+  if (typeof value === "object" && value !== null) {
+    return JSON.stringify(value);
+  }
+
+  // Fallback for other types - stringify as string literal
+  return `'${String(value).replace(/'/g, "\\'")}'`;
+}
+
+/**
+ * Generate a column definition line for the schema
+ */
+function generateColumnLine(
+  columnName: string,
+  column: AnyTypeValidator | ColumnDefinition,
+  includeJsonPaths: boolean
+): string {
+  const validator = getColumnType(column);
+  const jsonPath = getColumnJsonPath(column);
+  const tinybirdType = getTinybirdTypeFromValidator(validator);
+  const modifiers = getModifiersFromValidator(validator);
+
+  const parts: string[] = [`    ${columnName} ${tinybirdType}`];
+
+  // Add JSON path for Events API ingestion support if enabled
+  // Use explicit jsonPath if defined, otherwise default to $.columnName
+  if (includeJsonPaths) {
+    const effectiveJsonPath = jsonPath ?? `$.${columnName}`;
+    parts.push(`\`json:${effectiveJsonPath}\``);
+  }
+
+  // Add default value if defined
+  if (modifiers.hasDefault && modifiers.defaultValue !== undefined) {
+    const defaultStr = formatDefaultValue(modifiers.defaultValue, tinybirdType);
+    parts.push(`DEFAULT ${defaultStr}`);
+  }
+
+  // Add codec if defined
+  if (modifiers.codec) {
+    parts.push(`CODEC(${modifiers.codec})`);
+  }
+
+  return parts.join(" ");
+}
+
+/**
+ * Generate the SCHEMA section
+ */
+function generateSchema(schema: SchemaDefinition, includeJsonPaths: boolean): string {
+  const lines = ["SCHEMA >"];
+
+  const columnNames = Object.keys(schema);
+  columnNames.forEach((name, index) => {
+    const column = schema[name];
+    const line = generateColumnLine(name, column, includeJsonPaths);
+    // Add comma if not the last column
+    const suffix = index < columnNames.length - 1 ? "," : "";
+    lines.push(line + suffix);
+  });
+
+  return lines.join("\n");
+}
+
+/**
+ * Generate the engine configuration
+ * Uses the helper from engines.ts if an engine is provided
+ */
+function generateEngineConfig(engine?: EngineConfig): string {
+  if (!engine) {
+    // Default to MergeTree with first column as sorting key
+    return 'ENGINE "MergeTree"';
+  }
+
+  return getEngineClause(engine);
+}
+
+/**
+ * Generate a .datasource file content from a DatasourceDefinition
+ *
+ * @param datasource - The datasource definition
+ * @returns Generated datasource content
+ *
+ * @example
+ * ```ts
+ * const events = defineDatasource('events', {
+ *   description: 'User events',
+ *   schema: {
+ *     timestamp: t.dateTime(),
+ *     user_id: t.string(),
+ *     event: t.string(),
+ *   },
+ *   engine: engine.mergeTree({
+ *     sortingKey: ['user_id', 'timestamp'],
+ *   }),
+ * });
+ *
+ * const { content } = generateDatasource(events);
+ * // Returns:
+ * // DESCRIPTION >
+ * //     User events
+ * //
+ * // SCHEMA >
+ * //     timestamp DateTime,
+ * //     user_id String,
+ * //     event String
+ * //
+ * // ENGINE "MergeTree"
+ * // ENGINE_SORTING_KEY "user_id, timestamp"
+ * ```
+ */
+export function generateDatasource(
+  datasource: DatasourceDefinition
+): GeneratedDatasource {
+  const parts: string[] = [];
+
+  // Add description if present
+  if (datasource.options.description) {
+    parts.push(`DESCRIPTION >\n    ${datasource.options.description}`);
+    parts.push("");
+  }
+
+  // Check if JSON paths should be included (defaults to true)
+  const includeJsonPaths = datasource.options.jsonPaths !== false;
+
+  // Add schema
+  parts.push(generateSchema(datasource._schema, includeJsonPaths));
+  parts.push("");
+
+  // Add engine configuration
+  parts.push(generateEngineConfig(datasource.options.engine));
+
+  return {
+    name: datasource._name,
+    content: parts.join("\n"),
+  };
+}
+
+/**
+ * Generate .datasource files for all datasources in a project
+ *
+ * @param datasources - Record of datasource definitions
+ * @returns Array of generated datasource content
+ */
+export function generateAllDatasources(
+  datasources: Record<string, DatasourceDefinition>
+): GeneratedDatasource[] {
+  return Object.values(datasources).map(generateDatasource);
+}

package/src/generator/index.test.ts

@@ -0,0 +1,209 @@
+import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import { generateResources, build } from "./index.js";
+import { defineProject } from "../schema/project.js";
+import { defineDatasource } from "../schema/datasource.js";
+import { definePipe, node } from "../schema/pipe.js";
+import { t } from "../schema/types.js";
+import { engine } from "../schema/engines.js";
+import * as fs from "fs";
+import * as path from "path";
+import * as os from "os";
+
+describe("Generator Index", () => {
+  describe("generateResources", () => {
+    it("generates resources from a project definition", () => {
+      const events = defineDatasource("events", {
+        schema: {
+          timestamp: t.dateTime(),
+          event_name: t.string(),
+        },
+      });
+
+      const topEvents = definePipe("top_events", {
+        nodes: [
+          node({
+            name: "endpoint",
+            sql: "SELECT event_name, count() as cnt FROM events GROUP BY event_name",
+          }),
+        ],
+        output: {
+          event_name: t.string(),
+          cnt: t.uint64(),
+        },
+        endpoint: true,
+      });
+
+      const project = defineProject({
+        datasources: { events },
+        pipes: { topEvents },
+      });
+
+      const result = generateResources(project);
+
+      expect(result.datasources).toHaveLength(1);
+      expect(result.datasources[0].name).toBe("events");
+      expect(result.datasources[0].content).toContain("timestamp DateTime");
+
+      expect(result.pipes).toHaveLength(1);
+      expect(result.pipes[0].name).toBe("top_events");
+      expect(result.pipes[0].content).toContain("SELECT event_name");
+    });
+
+    it("handles empty project", () => {
+      const project = defineProject({
+        datasources: {},
+        pipes: {},
+      });
+
+      const result = generateResources(project);
+
+      expect(result.datasources).toHaveLength(0);
+      expect(result.pipes).toHaveLength(0);
+    });
+
+    it("generates multiple datasources and pipes", () => {
+      const ds1 = defineDatasource("ds1", { schema: { id: t.string() } });
+      const ds2 = defineDatasource("ds2", { schema: { name: t.string() } });
+      const ds3 = defineDatasource("ds3", { schema: { count: t.int32() } });
+
+      const pipe1 = definePipe("pipe1", {
+        nodes: [node({ name: "n", sql: "SELECT * FROM ds1" })],
+        output: { id: t.string() },
+        endpoint: true,
+      });
+      const pipe2 = definePipe("pipe2", {
+        nodes: [node({ name: "n", sql: "SELECT * FROM ds2" })],
+        output: { name: t.string() },
+        endpoint: true,
+      });
+
+      const project = defineProject({
+        datasources: { ds1, ds2, ds3 },
+        pipes: { pipe1, pipe2 },
+      });
+
+      const result = generateResources(project);
+
+      expect(result.datasources).toHaveLength(3);
+      expect(result.pipes).toHaveLength(2);
+    });
+
+    it("generates datasources with full options", () => {
+      const events = defineDatasource("events", {
+        description: "Event tracking data",
+        schema: {
+          timestamp: t.dateTime(),
+          event_name: t.string(),
+          user_id: t.string().nullable(),
+          metadata: t.string().default("{}"),
+        },
+        engine: engine.mergeTree({
+          sortingKey: ["timestamp", "event_name"],
+          partitionKey: "toYYYYMM(timestamp)",
+        }),
+      });
+
+      const project = defineProject({
+        datasources: { events },
+        pipes: {},
+      });
+
+      const result = generateResources(project);
+
+      expect(result.datasources).toHaveLength(1);
+      expect(result.datasources[0].content).toContain("DESCRIPTION >");
+      expect(result.datasources[0].content).toContain("Event tracking data");
+      expect(result.datasources[0].content).toContain("ENGINE_SORTING_KEY");
+      expect(result.datasources[0].content).toContain("ENGINE_PARTITION_KEY");
+    });
+
+    it("generates pipes with endpoint config", () => {
+      const stats = definePipe("stats", {
+        description: "Get stats",
+        nodes: [
+          node({
+            name: "calc",
+            description: "Calculate statistics",
+            sql: "SELECT count() as total FROM events",
+          }),
+        ],
+        output: { total: t.uint64() },
+        endpoint: {
+          enabled: true,
+          cache: { enabled: true, ttl: 120 },
+        },
+      });
+
+      const project = defineProject({
+        datasources: {},
+        pipes: { stats },
+      });
+
+      const result = generateResources(project);
+
+      expect(result.pipes).toHaveLength(1);
+      expect(result.pipes[0].content).toContain("DESCRIPTION >");
+      expect(result.pipes[0].content).toContain("Get stats");
+      expect(result.pipes[0].content).toContain("TYPE endpoint");
+      expect(result.pipes[0].content).toContain("CACHE 120");
+    });
+
+    it("generates non-endpoint pipes", () => {
+      const materialize = definePipe("materialize", {
+        nodes: [
+          node({
+            name: "aggregate",
+            sql: "SELECT event_name, count() FROM events GROUP BY event_name",
+          }),
+        ],
+        output: { event_name: t.string() },
+        endpoint: false,
+      });
+
+      const project = defineProject({
+        datasources: {},
+        pipes: { materialize },
+      });
+
+      const result = generateResources(project);
+
+      expect(result.pipes).toHaveLength(1);
+      expect(result.pipes[0].content).not.toContain("TYPE endpoint");
+    });
+  });
+
+  describe("build", () => {
+    let tempDir: string;
+
+    beforeEach(() => {
+      tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-test-"));
+    });
+
+    afterEach(() => {
+      try {
+        fs.rmSync(tempDir, { recursive: true });
+      } catch {
+        // Ignore cleanup errors
+      }
+    });
+
+    it("throws error for non-existent schema file", async () => {
+      const schemaPath = path.join(tempDir, "nonexistent.ts");
+
+      await expect(build({ schemaPath })).rejects.toThrow("Schema file not found");
+    });
+
+    it("throws error when no project definition is exported", async () => {
+      const schemaContent = `
+export const notAProject = { foo: "bar" };
+`;
+
+      const schemaPath = path.join(tempDir, "invalid-schema.ts");
+      fs.writeFileSync(schemaPath, schemaContent);
+
+      await expect(build({ schemaPath })).rejects.toThrow(
+        "No ProjectDefinition found"
+      );
+    });
+  });
+});

package/src/generator/index.ts

@@ -0,0 +1,203 @@
+/**
+ * Main generator module
+ * Orchestrates loading schema and generating all resources
+ */
+
+import { loadSchema, loadEntities, entitiesToProject, type LoadedEntities } from "./loader.js";
+import { generateAllDatasources, type GeneratedDatasource } from "./datasource.js";
+import { generateAllPipes, type GeneratedPipe } from "./pipe.js";
+import type { ProjectDefinition, DatasourcesDefinition, PipesDefinition } from "../schema/project.js";
+
+/**
+ * Generated resources ready for API push
+ */
+export interface GeneratedResources {
+  /** Generated datasource files */
+  datasources: GeneratedDatasource[];
+  /** Generated pipe files */
+  pipes: GeneratedPipe[];
+}
+
+/**
+ * Build result with metadata
+ */
+export interface BuildResult {
+  /** The generated resources */
+  resources: GeneratedResources;
+  /** The loaded project definition (for validation) */
+  project: ProjectDefinition;
+  /** Path to the schema file */
+  schemaPath: string;
+  /** Directory containing the schema */
+  schemaDir: string;
+  /** Statistics about the build */
+  stats: {
+    datasourceCount: number;
+    pipeCount: number;
+  };
+}
+
+/**
+ * Generate resources from a loaded project definition
+ *
+ * @param project - The project definition
+ * @returns Generated resources
+ */
+export function generateResources(project: ProjectDefinition): GeneratedResources {
+  const datasources = generateAllDatasources(project.datasources);
+  const pipes = generateAllPipes(project.pipes);
+
+  return {
+    datasources,
+    pipes,
+  };
+}
+
+/**
+ * Build options
+ */
+export interface BuildOptions {
+  /** Path to the schema file */
+  schemaPath: string;
+  /** Working directory (defaults to cwd) */
+  cwd?: string;
+}
+
+/**
+ * Build all resources from a TypeScript schema
+ *
+ * This is the main entry point for the generator.
+ * It loads the schema, generates all resources, and returns them
+ * ready for API push.
+ *
+ * @param options - Build options
+ * @returns Build result with generated resources
+ *
+ * @example
+ * ```ts
+ * const result = await build({
+ *   schemaPath: 'src/tinybird/schema.ts',
+ * });
+ *
+ * console.log(`Generated ${result.stats.datasourceCount} datasources`);
+ * console.log(`Generated ${result.stats.pipeCount} pipes`);
+ *
+ * // Resources are ready to push to API
+ * result.resources.datasources.forEach(ds => {
+ *   console.log(`${ds.name}.datasource:`);
+ *   console.log(ds.content);
+ * });
+ * ```
+ */
+export async function build(options: BuildOptions): Promise<BuildResult> {
+  // Load the schema
+  const loaded = await loadSchema({
+    schemaPath: options.schemaPath,
+    cwd: options.cwd,
+  });
+
+  // Generate resources
+  const resources = generateResources(loaded.project);
+
+  return {
+    resources,
+    project: loaded.project,
+    schemaPath: loaded.schemaPath,
+    schemaDir: loaded.schemaDir,
+    stats: {
+      datasourceCount: resources.datasources.length,
+      pipeCount: resources.pipes.length,
+    },
+  };
+}
+
+/**
+ * Build options using include paths
+ */
+export interface BuildFromIncludeOptions {
+  /** Array of file paths to scan for datasources and pipes */
+  includePaths: string[];
+  /** Working directory (defaults to cwd) */
+  cwd?: string;
+}
+
+/**
+ * Build result from include paths
+ */
+export interface BuildFromIncludeResult {
+  /** The generated resources */
+  resources: GeneratedResources;
+  /** Loaded entities from source files */
+  entities: LoadedEntities;
+  /** Statistics about the build */
+  stats: {
+    datasourceCount: number;
+    pipeCount: number;
+  };
+}
+
+/**
+ * Generate resources from entities
+ */
+export function generateResourcesFromEntities(
+  datasources: DatasourcesDefinition,
+  pipes: PipesDefinition
+): GeneratedResources {
+  return {
+    datasources: generateAllDatasources(datasources),
+    pipes: generateAllPipes(pipes),
+  };
+}
+
+/**
+ * Build all resources from include paths
+ *
+ * This is the main entry point for the generator that works with
+ * auto-discovered entities. It loads datasources and pipes from the
+ * include paths and generates Tinybird resources ready to deploy.
+ *
+ * @param options - Build options with include paths
+ * @returns Build result with generated resources
+ *
+ * @example
+ * ```ts
+ * const result = await buildFromInclude({
+ *   includePaths: ['src/tinybird/datasources.ts', 'src/tinybird/pipes.ts'],
+ * });
+ *
+ * // Push resources to Tinybird
+ * await deploy(result.resources);
+ * ```
+ */
+export async function buildFromInclude(
+  options: BuildFromIncludeOptions
+): Promise<BuildFromIncludeResult> {
+  const cwd = options.cwd ?? process.cwd();
+
+  // Load entities from include paths
+  const entities = await loadEntities({
+    includePaths: options.includePaths,
+    cwd,
+  });
+
+  // Convert to format for generators
+  const { datasources, pipes } = entitiesToProject(entities);
+
+  // Generate resources
+  const resources = generateResourcesFromEntities(datasources, pipes);
+
+  return {
+    resources,
+    entities,
+    stats: {
+      datasourceCount: resources.datasources.length,
+      pipeCount: resources.pipes.length,
+    },
+  };
+}
+
+// Re-export types and utilities
+export { loadSchema, loadEntities, entitiesToProject, type LoaderOptions, type LoadedSchema, type LoadedEntities, type LoadEntitiesOptions } from "./loader.js";
+export { generateDatasource, generateAllDatasources, type GeneratedDatasource } from "./datasource.js";
+export { generatePipe, generateAllPipes, type GeneratedPipe } from "./pipe.js";
+export { generateClientFile, type GenerateClientOptions, type GeneratedClient } from "./client.js";