@tinybirdco/sdk 0.0.4 → 0.0.7

package/src/generator/index.test.ts

@@ -1,5 +1,5 @@
 import { describe, it, expect, beforeEach, afterEach } from "vitest";
-import { generateResources, build } from "./index.js";
+import { generateResources, build, buildFromInclude } from "./index.js";
 import { defineProject } from "../schema/project.js";
 import { defineDatasource } from "../schema/datasource.js";
 import { definePipe, node } from "../schema/pipe.js";
@@ -206,4 +206,124 @@ export const notAProject = { foo: "bar" };
       );
     });
   });
+
+  describe("buildFromInclude with raw datafiles", () => {
+    let tempDir: string;
+
+    beforeEach(() => {
+      tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinybird-raw-test-"));
+    });
+
+    afterEach(() => {
+      try {
+        fs.rmSync(tempDir, { recursive: true });
+      } catch {
+        // Ignore cleanup errors
+      }
+    });
+
+    it("includes raw .datasource files directly", async () => {
+      // Create a raw datasource file
+      const datasourceContent = `SCHEMA >
+    timestamp DateTime,
+    user_id String
+
+ENGINE "MergeTree"
+ENGINE_SORTING_KEY "timestamp"
+`;
+      const datasourcePath = path.join(tempDir, "events.datasource");
+      fs.writeFileSync(datasourcePath, datasourceContent);
+
+      const result = await buildFromInclude({
+        includePaths: [datasourcePath],
+        cwd: tempDir,
+      });
+
+      expect(result.resources.datasources).toHaveLength(1);
+      expect(result.resources.datasources[0].name).toBe("events");
+      expect(result.resources.datasources[0].content).toBe(datasourceContent);
+      expect(result.stats.datasourceCount).toBe(1);
+    });
+
+    it("includes raw .pipe files directly", async () => {
+      // Create a raw pipe file
+      const pipeContent = `NODE endpoint
+SQL >
+    SELECT count() AS total FROM events
+
+TYPE endpoint
+`;
+      const pipePath = path.join(tempDir, "stats.pipe");
+      fs.writeFileSync(pipePath, pipeContent);
+
+      const result = await buildFromInclude({
+        includePaths: [pipePath],
+        cwd: tempDir,
+      });
+
+      expect(result.resources.pipes).toHaveLength(1);
+      expect(result.resources.pipes[0].name).toBe("stats");
+      expect(result.resources.pipes[0].content).toBe(pipeContent);
+      expect(result.stats.pipeCount).toBe(1);
+    });
+
+    it("includes multiple raw datasource and pipe files", async () => {
+      // Create multiple raw datasource files
+      const datasource1Content = `SCHEMA >
+    event_id String,
+    timestamp DateTime
+
+ENGINE "MergeTree"
+ENGINE_SORTING_KEY "timestamp"
+`;
+      const datasource1Path = path.join(tempDir, "raw_events.datasource");
+      fs.writeFileSync(datasource1Path, datasource1Content);
+
+      const datasource2Content = `SCHEMA >
+    user_id String,
+    name String
+
+ENGINE "MergeTree"
+ENGINE_SORTING_KEY "user_id"
+`;
+      const datasource2Path = path.join(tempDir, "users.datasource");
+      fs.writeFileSync(datasource2Path, datasource2Content);
+
+      // Create a raw pipe file
+      const rawPipeContent = `NODE main
+SQL >
+    SELECT * FROM raw_events
+
+TYPE endpoint
+`;
+      const rawPipePath = path.join(tempDir, "raw_endpoint.pipe");
+      fs.writeFileSync(rawPipePath, rawPipeContent);
+
+      const result = await buildFromInclude({
+        includePaths: [datasource1Path, datasource2Path, rawPipePath],
+        cwd: tempDir,
+      });
+
+      // Should have 2 datasources (both from raw)
+      expect(result.resources.datasources).toHaveLength(2);
+      expect(result.stats.datasourceCount).toBe(2);
+
+      // Should have 1 pipe (from raw)
+      expect(result.resources.pipes).toHaveLength(1);
+      expect(result.stats.pipeCount).toBe(1);
+
+      // Check the raw files are included with correct content
+      const rawDs1 = result.resources.datasources.find(d => d.name === "raw_events");
+      expect(rawDs1).toBeDefined();
+      expect(rawDs1!.content).toBe(datasource1Content);
+
+      const rawDs2 = result.resources.datasources.find(d => d.name === "users");
+      expect(rawDs2).toBeDefined();
+      expect(rawDs2!.content).toBe(datasource2Content);
+
+      const rawPipe = result.resources.pipes.find(p => p.name === "raw_endpoint");
+      expect(rawPipe).toBeDefined();
+      expect(rawPipe!.content).toBe(rawPipeContent);
+    });
+  });
 });

package/src/generator/index.ts

@@ -166,6 +166,9 @@ export function generateResourcesFromEntities(
  * auto-discovered entities. It loads datasources and pipes from the
  * include paths and generates Tinybird resources ready to deploy.
  *
+ * Supports both TypeScript files (processed through esbuild) and raw
+ * .datasource/.pipe files (included directly).
+ *
  * @param options - Build options with include paths
  * @returns Build result with generated resources
  *
@@ -193,9 +196,24 @@ export async function buildFromInclude(
   // Convert to format for generators
   const { datasources, pipes, connections } = entitiesToProject(entities);
 
-  // Generate resources
+  // Generate resources from TypeScript definitions
   const resources = generateResourcesFromEntities(datasources, pipes, connections);
 
+  // Merge in raw datafiles (loaded directly without processing)
+  for (const raw of entities.rawDatasources) {
+    resources.datasources.push({
+      name: raw.name,
+      content: raw.content,
+    });
+  }
+
+  for (const raw of entities.rawPipes) {
+    resources.pipes.push({
+      name: raw.name,
+      content: raw.content,
+    });
+  }
+
   return {
     resources,
     entities,

package/src/generator/loader.ts

@@ -150,6 +150,18 @@ export interface EntityInfo {
   sourceFile: string;
 }
 
+/**
+ * Raw datafile loaded directly from disk
+ */
+export interface RawDatafile {
+  /** The file name (without extension) */
+  name: string;
+  /** The raw content of the file */
+  content: string;
+  /** The source file path (relative to cwd) */
+  sourceFile: string;
+}
+
 /**
  * Result of loading entities from multiple files
  */
@@ -160,6 +172,10 @@ export interface LoadedEntities {
   pipes: Record<string, { definition: PipeDefinition; info: EntityInfo }>;
   /** Discovered connections with their metadata */
   connections: Record<string, { definition: ConnectionDefinition; info: EntityInfo }>;
+  /** Raw .datasource files loaded directly */
+  rawDatasources: RawDatafile[];
+  /** Raw .pipe files loaded directly */
+  rawPipes: RawDatafile[];
   /** All source files that were scanned */
   sourceFiles: string[];
 }
@@ -199,6 +215,8 @@ export async function loadEntities(options: LoadEntitiesOptions): Promise<Loaded
     datasources: {},
     pipes: {},
     connections: {},
+    rawDatasources: [],
+    rawPipes: [],
     sourceFiles: [],
   };
 
@@ -213,6 +231,31 @@ export async function loadEntities(options: LoadEntitiesOptions): Promise<Loaded
     }
 
     result.sourceFiles.push(includePath);
+
+    // Check if this is a raw datafile (.datasource or .pipe)
+    if (includePath.endsWith(".datasource")) {
+      const content = fs.readFileSync(absolutePath, "utf-8");
+      const name = path.basename(includePath, ".datasource");
+      result.rawDatasources.push({
+        name,
+        content,
+        sourceFile: includePath,
+      });
+      continue;
+    }
+
+    if (includePath.endsWith(".pipe")) {
+      const content = fs.readFileSync(absolutePath, "utf-8");
+      const name = path.basename(includePath, ".pipe");
+      result.rawPipes.push({
+        name,
+        content,
+        sourceFile: includePath,
+      });
+      continue;
+    }
+
+    // TypeScript file - process through esbuild
     const fileDir = path.dirname(absolutePath);
 
     // Create a temporary output file for the bundle

package/src/schema/connection.ts

@@ -3,8 +3,9 @@
  * Define external connections (Kafka, etc.) as TypeScript with full type safety
  */
 
-// Symbol for brand typing
-const CONNECTION_BRAND = Symbol("tinybird.connection");
+// Symbol for brand typing - use Symbol.for() for global registry
+// This ensures the same symbol is used across module instances
+const CONNECTION_BRAND = Symbol.for("tinybird.connection");
 
 /**
  * Kafka security protocol options

package/src/schema/datasource.ts

@@ -7,8 +7,9 @@ import type { AnyTypeValidator } from "./types.js";
 import type { EngineConfig } from "./engines.js";
 import type { KafkaConnectionDefinition } from "./connection.js";
 
-// Symbol for brand typing
-const DATASOURCE_BRAND = Symbol("tinybird.datasource");
+// Symbol for brand typing - use Symbol.for() for global registry
+// This ensures the same symbol is used across module instances
+const DATASOURCE_BRAND = Symbol.for("tinybird.datasource");
 
 /**
  * A column can be defined as just a type validator,

package/src/schema/params.ts

@@ -3,8 +3,9 @@
  * Similar to the column type validators but for query parameters
  */
 
-// Symbol for brand typing
-const PARAM_BRAND = Symbol("tinybird.param");
+// Symbol for brand typing - use Symbol.for() for global registry
+// This ensures the same symbol is used across module instances
+const PARAM_BRAND = Symbol.for("tinybird.param");
 
 /**
  * Base interface for parameter validators

package/src/schema/pipe.ts

@@ -9,10 +9,10 @@ import type { DatasourceDefinition, SchemaDefinition, ColumnDefinition } from ".
 import { getColumnType } from "./datasource.js";
 import { getTinybirdType } from "./types.js";
 
-/** Symbol for brand typing pipes */
-export const PIPE_BRAND: unique symbol = Symbol("tinybird.pipe");
-/** Symbol for brand typing nodes */
-export const NODE_BRAND: unique symbol = Symbol("tinybird.node");
+/** Symbol for brand typing pipes - use Symbol.for() for global registry */
+export const PIPE_BRAND = Symbol.for("tinybird.pipe");
+/** Symbol for brand typing nodes - use Symbol.for() for global registry */
+export const NODE_BRAND = Symbol.for("tinybird.node");
 
 /**
  * Parameter definition for a pipe

package/src/schema/project.ts

@@ -11,8 +11,9 @@ import type { TinybirdClient } from "../client/base.js";
 import type { QueryResult } from "../client/types.js";
 import type { InferRow, InferParams, InferOutputRow } from "../infer/index.js";
 
-// Symbol for brand typing
-const PROJECT_BRAND = Symbol("tinybird.project");
+// Symbol for brand typing - use Symbol.for() for global registry
+// This ensures the same symbol is used across module instances
+const PROJECT_BRAND = Symbol.for("tinybird.project");
 
 /**
  * Collection of datasource definitions

package/src/schema/types.ts

@@ -3,8 +3,9 @@
  * Similar to Convex's `v.*` pattern, but for ClickHouse types
  */
 
-// Symbol for brand typing
-const VALIDATOR_BRAND = Symbol("tinybird.validator");
+// Symbol for brand typing - use Symbol.for() for global registry
+// This ensures the same symbol is used across module instances
+const VALIDATOR_BRAND = Symbol.for("tinybird.validator");
 
 /**
  * Base interface for all type validators

package/src/test/handlers.ts

@@ -63,6 +63,59 @@ export function createNoChangesResponse() {
   };
 }
 
+/**
+ * Create deploy success response (for /v1/deploy endpoint)
+ * This returns a deployment object, not a build object
+ */
+export function createDeploySuccessResponse(options?: {
+  deploymentId?: string;
+  status?: string;
+}) {
+  return {
+    result: "success",
+    deployment: {
+      id: options?.deploymentId ?? "deploy-123",
+      status: options?.status ?? "pending",
+    },
+  };
+}
+
+/**
+ * Create deployment status response (for /v1/deployments/:id endpoint)
+ */
+export function createDeploymentStatusResponse(options?: {
+  deploymentId?: string;
+  status?: string;
+}) {
+  return {
+    result: "success",
+    deployment: {
+      id: options?.deploymentId ?? "deploy-123",
+      status: options?.status ?? "data_ready",
+    },
+  };
+}
+
+/**
+ * Create set-live success response (for /v1/deployments/:id/set-live endpoint)
+ */
+export function createSetLiveSuccessResponse() {
+  return {
+    result: "success",
+  };
+}
+
+/**
+ * Create deployments list response (for /v1/deployments endpoint)
+ */
+export function createDeploymentsListResponse(options?: {
+  deployments?: Array<{ id: string; status: string; live: boolean }>;
+}) {
+  return {
+    deployments: options?.deployments ?? [],
+  };
+}
+
 /**
  * Default handlers for build and deploy endpoints
  */
@@ -76,4 +129,9 @@ export const handlers = [
   http.post(`${BASE_URL}/v1/deploy`, () => {
     return HttpResponse.json(createBuildSuccessResponse());
   }),
+
+  // Deployments list endpoint - empty list by default (no stale deployments)
+  http.get(`${BASE_URL}/v1/deployments`, () => {
+    return HttpResponse.json(createDeploymentsListResponse());
+  }),
 ];