@tinybirdco/sdk 0.0.38 → 0.0.40

package/src/generator/include-paths.ts

@@ -0,0 +1,234 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
+
+export interface ResolvedIncludeFile {
+  sourcePath: string;
+  absolutePath: string;
+}
+
+const GLOB_SEGMENT_REGEX = /[*?[]/;
+const IGNORED_DIRECTORIES = new Set([".git", "node_modules"]);
+const SEGMENT_REGEX_CACHE = new Map<string, RegExp>();
+
+function hasGlobPattern(value: string): boolean {
+  return GLOB_SEGMENT_REGEX.test(value);
+}
+
+function normalizePath(value: string): string {
+  return value.replace(/\\/g, "/");
+}
+
+function splitAbsolutePath(filePath: string): { root: string; segments: string[] } {
+  const absolutePath = path.resolve(filePath);
+  const root = path.parse(absolutePath).root;
+  const relative = path.relative(root, absolutePath);
+
+  return {
+    root: normalizePath(root),
+    segments: normalizePath(relative).split("/").filter(Boolean),
+  };
+}
+
+function segmentMatcher(segment: string): RegExp {
+  const cached = SEGMENT_REGEX_CACHE.get(segment);
+  if (cached) {
+    return cached;
+  }
+
+  const escaped = segment
+    .replace(/[.+^${}()|\\]/g, "\\$&")
+    .replace(/\*/g, "[^/]*")
+    .replace(/\?/g, "[^/]");
+
+  const matcher = new RegExp(`^${escaped}$`);
+  SEGMENT_REGEX_CACHE.set(segment, matcher);
+  return matcher;
+}
+
+function matchSegment(patternSegment: string, valueSegment: string): boolean {
+  if (!hasGlobPattern(patternSegment)) {
+    return patternSegment === valueSegment;
+  }
+
+  return segmentMatcher(patternSegment).test(valueSegment);
+}
+
+function matchGlobSegments(
+  patternSegments: string[],
+  pathSegments: string[],
+  patternIndex: number,
+  pathIndex: number,
+  memo: Map<string, boolean>
+): boolean {
+  const key = `${patternIndex}:${pathIndex}`;
+  const cached = memo.get(key);
+  if (cached !== undefined) {
+    return cached;
+  }
+
+  if (patternIndex === patternSegments.length) {
+    const matches = pathIndex === pathSegments.length;
+    memo.set(key, matches);
+    return matches;
+  }
+
+  const patternSegment = patternSegments[patternIndex];
+  let matches = false;
+
+  if (patternSegment === "**") {
+    matches = matchGlobSegments(patternSegments, pathSegments, patternIndex + 1, pathIndex, memo);
+
+    if (!matches && pathIndex < pathSegments.length) {
+      matches = matchGlobSegments(patternSegments, pathSegments, patternIndex, pathIndex + 1, memo);
+    }
+  } else if (
+    pathIndex < pathSegments.length &&
+    matchSegment(patternSegment, pathSegments[pathIndex])
+  ) {
+    matches = matchGlobSegments(patternSegments, pathSegments, patternIndex + 1, pathIndex + 1, memo);
+  }
+
+  memo.set(key, matches);
+  return matches;
+}
+
+function matchGlobPath(absolutePattern: string, absolutePath: string): boolean {
+  const patternParts = splitAbsolutePath(absolutePattern);
+  const pathParts = splitAbsolutePath(absolutePath);
+
+  if (patternParts.root.toLowerCase() !== pathParts.root.toLowerCase()) {
+    return false;
+  }
+
+  return matchGlobSegments(
+    patternParts.segments,
+    pathParts.segments,
+    0,
+    0,
+    new Map<string, boolean>()
+  );
+}
+
+function getGlobRootDirectory(absolutePattern: string): string {
+  const { root, segments } = splitAbsolutePath(absolutePattern);
+  const firstGlobIndex = segments.findIndex((segment) => hasGlobPattern(segment));
+  const baseSegments =
+    firstGlobIndex === -1 ? segments : segments.slice(0, firstGlobIndex);
+
+  return path.join(root, ...baseSegments);
+}
+
+function collectFilesRecursive(directory: string, result: string[]): void {
+  const entries = fs.readdirSync(directory, { withFileTypes: true });
+
+  for (const entry of entries) {
+    const fullPath = path.join(directory, entry.name);
+
+    if (entry.isDirectory()) {
+      if (IGNORED_DIRECTORIES.has(entry.name)) {
+        continue;
+      }
+
+      collectFilesRecursive(fullPath, result);
+      continue;
+    }
+
+    if (entry.isFile()) {
+      result.push(fullPath);
+    }
+  }
+}
+
+function expandGlobPattern(absolutePattern: string): string[] {
+  const rootDirectory = getGlobRootDirectory(absolutePattern);
+
+  if (!fs.existsSync(rootDirectory)) {
+    return [];
+  }
+
+  if (!fs.statSync(rootDirectory).isDirectory()) {
+    return [];
+  }
+
+  const files: string[] = [];
+  collectFilesRecursive(rootDirectory, files);
+
+  return files
+    .filter((filePath) => matchGlobPath(absolutePattern, filePath))
+    .sort((a, b) => a.localeCompare(b));
+}
+
+export function resolveIncludeFiles(
+  includePaths: string[],
+  cwd: string
+): ResolvedIncludeFile[] {
+  const resolved: ResolvedIncludeFile[] = [];
+  const seen = new Set<string>();
+
+  for (const includePath of includePaths) {
+    const absoluteIncludePath = path.isAbsolute(includePath)
+      ? includePath
+      : path.resolve(cwd, includePath);
+
+    if (hasGlobPattern(includePath)) {
+      const matchedFiles = expandGlobPattern(absoluteIncludePath);
+
+      if (matchedFiles.length === 0) {
+        throw new Error(`Include pattern matched no files: ${includePath}`);
+      }
+
+      for (const matchedFile of matchedFiles) {
+        if (seen.has(matchedFile)) {
+          continue;
+        }
+
+        seen.add(matchedFile);
+        resolved.push({
+          sourcePath: path.isAbsolute(includePath)
+            ? matchedFile
+            : path.relative(cwd, matchedFile),
+          absolutePath: matchedFile,
+        });
+      }
+      continue;
+    }
+
+    if (!fs.existsSync(absoluteIncludePath)) {
+      throw new Error(`Include file not found: ${absoluteIncludePath}`);
+    }
+
+    if (seen.has(absoluteIncludePath)) {
+      continue;
+    }
+
+    seen.add(absoluteIncludePath);
+    resolved.push({
+      sourcePath: includePath,
+      absolutePath: absoluteIncludePath,
+    });
+  }
+
+  return resolved;
+}
+
+export function getIncludeWatchDirectories(
+  includePaths: string[],
+  cwd: string
+): string[] {
+  const watchDirs = new Set<string>();
+
+  for (const includePath of includePaths) {
+    const absoluteIncludePath = path.isAbsolute(includePath)
+      ? includePath
+      : path.resolve(cwd, includePath);
+
+    if (hasGlobPattern(includePath)) {
+      watchDirs.add(getGlobRootDirectory(absoluteIncludePath));
+      continue;
+    }
+
+    watchDirs.add(path.dirname(absoluteIncludePath));
+  }
+
+  return Array.from(watchDirs);
+}

package/src/generator/index.test.ts

@@ -325,5 +325,49 @@ TYPE endpoint
       expect(rawPipe).toBeDefined();
       expect(rawPipe!.content).toBe(rawPipeContent);
     });
+
+    it("supports glob include patterns", async () => {
+      const nestedDir = path.join(tempDir, "tinybird", "legacy");
+      fs.mkdirSync(nestedDir, { recursive: true });
+
+      const datasourceContent = `SCHEMA >
+    id String
+
+ENGINE "MergeTree"
+ENGINE_SORTING_KEY "id"
+`;
+      const datasourcePath = path.join(nestedDir, "events.datasource");
+      fs.writeFileSync(datasourcePath, datasourceContent);
+
+      const pipeContent = `NODE endpoint
+SQL >
+    SELECT * FROM events
+
+TYPE endpoint
+`;
+      const pipePath = path.join(nestedDir, "events.pipe");
+      fs.writeFileSync(pipePath, pipeContent);
+
+      const result = await buildFromInclude({
+        includePaths: ["tinybird/**/*.datasource", "tinybird/**/*.pipe"],
+        cwd: tempDir,
+      });
+
+      expect(result.resources.datasources).toHaveLength(1);
+      expect(result.resources.datasources[0].name).toBe("events");
+      expect(result.resources.datasources[0].content).toBe(datasourceContent);
+      expect(result.resources.pipes).toHaveLength(1);
+      expect(result.resources.pipes[0].name).toBe("events");
+      expect(result.resources.pipes[0].content).toBe(pipeContent);
+    });
+
+    it("throws when include glob matches no files", async () => {
+      await expect(
+        buildFromInclude({
+          includePaths: ["tinybird/**/*.datasource"],
+          cwd: tempDir,
+        })
+      ).rejects.toThrow("Include pattern matched no files");
+    });
   });
 });

package/src/generator/index.ts

@@ -122,7 +122,7 @@ export async function build(options: BuildOptions): Promise<BuildResult> {
  * Build options using include paths
  */
 export interface BuildFromIncludeOptions {
-  /** Array of file paths to scan for datasources and pipes */
+  /** Array of file paths or glob patterns to scan for datasources and pipes */
   includePaths: string[];
   /** Working directory (defaults to cwd) */
   cwd?: string;

package/src/generator/loader.ts

@@ -11,6 +11,7 @@ import { isProjectDefinition, type ProjectDefinition, type DatasourcesDefinition
 import { isDatasourceDefinition, type DatasourceDefinition } from "../schema/datasource.js";
 import { isPipeDefinition, type PipeDefinition } from "../schema/pipe.js";
 import { isConnectionDefinition, type ConnectionDefinition } from "../schema/connection.js";
+import { resolveIncludeFiles } from "./include-paths.js";
 
 /**
  * Result of loading a schema file
@@ -184,7 +185,7 @@ export interface LoadedEntities {
  * Options for loading entities
  */
 export interface LoadEntitiesOptions {
-  /** Array of file paths to scan (can be relative or absolute) */
+  /** Array of file paths or glob patterns to scan (can be relative or absolute) */
   includePaths: string[];
   /** The working directory for resolution (defaults to cwd) */
   cwd?: string;
@@ -211,6 +212,7 @@ export interface LoadEntitiesOptions {
  */
 export async function loadEntities(options: LoadEntitiesOptions): Promise<LoadedEntities> {
   const cwd = options.cwd ?? process.cwd();
+  const includeFiles = resolveIncludeFiles(options.includePaths, cwd);
   const result: LoadedEntities = {
     datasources: {},
     pipes: {},
@@ -220,15 +222,9 @@ export async function loadEntities(options: LoadEntitiesOptions): Promise<Loaded
     sourceFiles: [],
   };
 
-  for (const includePath of options.includePaths) {
-    const absolutePath = path.isAbsolute(includePath)
-      ? includePath
-      : path.resolve(cwd, includePath);
-
-    // Verify the file exists
-    if (!fs.existsSync(absolutePath)) {
-      throw new Error(`Include file not found: ${absolutePath}`);
-    }
+  for (const includeFile of includeFiles) {
+    const includePath = includeFile.sourcePath;
+    const absolutePath = includeFile.absolutePath;
 
     result.sourceFiles.push(includePath);
 

package/src/index.ts

@@ -208,10 +208,14 @@ export type {
   ClientContext,
   CsvDialectOptions,
   DatasourcesNamespace,
+  DeleteOptions,
+  DeleteResult,
   QueryResult,
   IngestResult,
   QueryOptions,
   IngestOptions,
+  TruncateOptions,
+  TruncateResult,
   ColumnMeta,
   QueryStatistics,
   TinybirdErrorResponse,
@@ -231,6 +235,8 @@ export type {
   TinybirdApiQueryOptions,
   TinybirdApiIngestOptions,
   TinybirdApiAppendOptions,
+  TinybirdApiDeleteOptions,
+  TinybirdApiTruncateOptions,
   TinybirdApiRequestInit,
   TinybirdApiTokenScope,
   TinybirdApiCreateTokenRequest,

package/src/schema/project.test.ts

@@ -94,7 +94,7 @@ describe("Project Schema", () => {
       expect((project.tinybird as unknown as Record<string, unknown>).pipes).toBeUndefined();
     });
 
-    it("creates datasource accessors with append method", () => {
+    it("creates datasource accessors with append/delete/truncate methods", () => {
       const events = defineDatasource("events", {
         schema: { timestamp: t.dateTime() },
       });
@@ -105,6 +105,8 @@ describe("Project Schema", () => {
 
       expect(project.tinybird.events).toBeDefined();
       expect(typeof project.tinybird.events.append).toBe("function");
+      expect(typeof project.tinybird.events.delete).toBe("function");
+      expect(typeof project.tinybird.events.truncate).toBe("function");
     });
 
     it("creates multiple datasource accessors", () => {
@@ -123,6 +125,10 @@ describe("Project Schema", () => {
       expect(project.tinybird.pageViews).toBeDefined();
       expect(typeof project.tinybird.events.append).toBe("function");
       expect(typeof project.tinybird.pageViews.append).toBe("function");
+      expect(typeof project.tinybird.events.delete).toBe("function");
+      expect(typeof project.tinybird.pageViews.delete).toBe("function");
+      expect(typeof project.tinybird.events.truncate).toBe("function");
+      expect(typeof project.tinybird.pageViews.truncate).toBe("function");
     });
 
     it("throws error when accessing client before initialization", () => {
@@ -304,7 +310,7 @@ describe("Project Schema", () => {
       expect((client as unknown as Record<string, unknown>).pipes).toBeUndefined();
     });
 
-    it("creates datasource accessors with append method", () => {
+    it("creates datasource accessors with append/delete/truncate methods", () => {
       const events = defineDatasource("events", {
         schema: { id: t.string() },
       });
@@ -316,6 +322,8 @@ describe("Project Schema", () => {
 
       expect(client.events).toBeDefined();
       expect(typeof client.events.append).toBe("function");
+      expect(typeof client.events.delete).toBe("function");
+      expect(typeof client.events.truncate).toBe("function");
     });
 
     it("creates multiple datasource accessors", () => {
@@ -335,6 +343,10 @@ describe("Project Schema", () => {
       expect(client.pageViews).toBeDefined();
       expect(typeof client.events.append).toBe("function");
       expect(typeof client.pageViews.append).toBe("function");
+      expect(typeof client.events.delete).toBe("function");
+      expect(typeof client.pageViews.delete).toBe("function");
+      expect(typeof client.events.truncate).toBe("function");
+      expect(typeof client.pageViews.truncate).toBe("function");
     });
 
     it("accepts devMode option", () => {

package/src/schema/project.ts

@@ -12,8 +12,12 @@ import type {
   AppendOptions,
   AppendResult,
   DatasourcesNamespace,
+  DeleteOptions,
+  DeleteResult,
   QueryOptions,
   QueryResult,
+  TruncateOptions,
+  TruncateResult,
 } from "../client/types.js";
 import type { InferRow, InferParams, InferOutputRow } from "../infer/index.js";
 import type { TokensNamespace } from "../client/tokens.js";
@@ -82,16 +86,20 @@ type IngestMethods<T extends DatasourcesDefinition> = {
 };
 
 /**
- * Type for a datasource accessor with append method
+ * Type for a datasource accessor with import/mutation methods
  */
 type DatasourceAccessor = {
   /** Append data from a URL or file */
   append(options: AppendOptions): Promise<AppendResult>;
+  /** Delete rows using a SQL condition */
+  delete(options: DeleteOptions): Promise<DeleteResult>;
+  /** Truncate all rows */
+  truncate(options?: TruncateOptions): Promise<TruncateResult>;
 };
 
 /**
  * Type for datasource accessors object
- * Maps each datasource to an accessor with append method
+ * Maps each datasource to an accessor with import/mutation methods
  */
 type DatasourceAccessors<T extends DatasourcesDefinition> = {
   [K in keyof T]: DatasourceAccessor;
@@ -105,7 +113,7 @@ interface ProjectClientBase<TDatasources extends DatasourcesDefinition> {
   ingest: IngestMethods<TDatasources>;
   /** Token operations (JWT creation, etc.) */
   readonly tokens: TokensNamespace;
-  /** Datasource operations (append from URL/file) */
+  /** Datasource operations (append/delete/truncate) */
   readonly datasources: DatasourcesNamespace;
   /** Execute raw SQL queries */
   sql<T = unknown>(sql: string, options?: QueryOptions): Promise<QueryResult<T>>;
@@ -374,6 +382,14 @@ function buildProjectClient<
         const client = await getClient();
         return client.datasources.append(tinybirdName, options);
       },
+      delete: async (options: DeleteOptions) => {
+        const client = await getClient();
+        return client.datasources.delete(tinybirdName, options);
+      },
+      truncate: async (options: TruncateOptions = {}) => {
+        const client = await getClient();
+        return client.datasources.truncate(tinybirdName, options);
+      },
     };
   }