@tinybirdco/sdk 0.0.50 → 0.0.51

package/src/generator/datasource.test.ts

@@ -95,6 +95,28 @@ describe('Datasource Generator', () => {
       expect(result.content).toContain('FORWARD_QUERY >');
       expect(result.content).toContain('    SELECT id');
     });
+
+    it("includes indexes block when provided", () => {
+      const ds = defineDatasource("test_ds", {
+        schema: {
+          id: t.string(),
+          pipe_name: t.string(),
+        },
+        indexes: [
+          { name: "pipe_name_set", expr: "pipe_name", type: "set(100)", granularity: 1 },
+          { name: "id_bf", expr: "lower(id)", type: "bloom_filter(0.001)", granularity: 4 },
+        ],
+      });
+
+      const result = generateDatasource(ds);
+      expect(result.content).toContain("INDEXES >");
+      expect(result.content).toContain(
+        "pipe_name_set pipe_name TYPE set(100) GRANULARITY 1"
+      );
+      expect(result.content).toContain(
+        "id_bf lower(id) TYPE bloom_filter(0.001) GRANULARITY 4"
+      );
+    });
   });
 
   describe('Column formatting', () => {

package/src/generator/datasource.ts

@@ -11,6 +11,7 @@ import type {
   S3Config,
   GCSConfig,
   TokenConfig,
+  DatasourceIndex,
 } from "../schema/datasource.js";
 import type { AnyTypeValidator, TypeModifiers } from "../schema/types.js";
 import { getColumnType, getColumnJsonPath } from "../schema/datasource.js";
@@ -211,6 +212,23 @@ function generateForwardQuery(forwardQuery?: string): string | null {
   return ["FORWARD_QUERY >", ...lines.map((line) => `    ${line}`)].join("\n");
 }
 
+/**
+ * Generate INDEXES section
+ */
+function generateIndexes(indexes?: readonly DatasourceIndex[]): string | null {
+  if (!indexes || indexes.length === 0) {
+    return null;
+  }
+
+  const lines = ["INDEXES >"];
+  for (const index of indexes) {
+    lines.push(
+      `    ${index.name} ${index.expr} TYPE ${index.type} GRANULARITY ${index.granularity}`
+    );
+  }
+  return lines.join("\n");
+}
+
 /**
  * Generate SHARED_WITH section for sharing datasource with other workspaces
  */
@@ -317,6 +335,13 @@ export function generateDatasource(
   // Add engine configuration
   parts.push(generateEngineConfig(datasource.options.engine));
 
+  // Add indexes if present
+  const indexes = generateIndexes(datasource.options.indexes);
+  if (indexes) {
+    parts.push("");
+    parts.push(indexes);
+  }
+
   // Add Kafka configuration if present
   if (datasource.options.kafka) {
     parts.push("");

package/src/generator/pipe.test.ts

@@ -164,6 +164,27 @@ describe('Pipe Generator', () => {
       const result = generatePipe(pipe);
       expect(result.content).toContain('    %\n');
     });
+
+    it('injects param defaults into placeholders when SQL omits them', () => {
+      const pipe = definePipe('defaults_pipe', {
+        params: {
+          start_date: p.date().optional('2025-03-01'),
+          page: p.int32().optional(0),
+        },
+        nodes: [
+          node({
+            name: 'endpoint',
+            sql: 'SELECT * FROM events WHERE d >= {{Date(start_date)}} LIMIT 10 OFFSET {{Int32(page)}}',
+          }),
+        ],
+        output: simpleOutput,
+        endpoint: true,
+      });
+
+      const result = generatePipe(pipe);
+      expect(result.content).toContain("{{ Date(start_date, '2025-03-01') }}");
+      expect(result.content).toContain('{{ Int32(page, 0) }}');
+    });
   });
 
   describe('Multiple nodes', () => {

package/src/generator/pipe.ts

@@ -18,6 +18,8 @@ import {
   getCopyConfig,
   getSinkConfig,
 } from "../schema/pipe.js";
+import type { AnyParamValidator } from "../schema/params.js";
+import { getParamDefault } from "../schema/params.js";
 
 /**
  * Generated pipe content
@@ -36,10 +38,123 @@ function hasDynamicParameters(sql: string): boolean {
   return /\{\{[^}]+\}\}/.test(sql) || /\{%[^%]+%\}/.test(sql);
 }
 
+function splitTopLevelComma(input: string): string[] {
+  const parts: string[] = [];
+  let current = "";
+  let depth = 0;
+  let inSingleQuote = false;
+  let inDoubleQuote = false;
+
+  for (let i = 0; i < input.length; i += 1) {
+    const char = input[i];
+    const prev = i > 0 ? input[i - 1] : "";
+
+    if (char === "'" && !inDoubleQuote && prev !== "\\") {
+      inSingleQuote = !inSingleQuote;
+      current += char;
+      continue;
+    }
+
+    if (char === '"' && !inSingleQuote && prev !== "\\") {
+      inDoubleQuote = !inDoubleQuote;
+      current += char;
+      continue;
+    }
+
+    if (!inSingleQuote && !inDoubleQuote) {
+      if (char === "(") {
+        depth += 1;
+        current += char;
+        continue;
+      }
+      if (char === ")" && depth > 0) {
+        depth -= 1;
+        current += char;
+        continue;
+      }
+      if (char === "," && depth === 0) {
+        parts.push(current.trim());
+        current = "";
+        continue;
+      }
+    }
+
+    current += char;
+  }
+
+  if (current.trim()) {
+    parts.push(current.trim());
+  }
+
+  return parts;
+}
+
+function toTemplateDefaultLiteral(value: string | number | boolean): string {
+  if (typeof value === "string") {
+    return `'${value.replace(/\\/g, "\\\\").replace(/'/g, "\\'")}'`;
+  }
+  return String(value);
+}
+
+function applyParamDefaultsToSql(
+  sql: string,
+  params?: Record<string, AnyParamValidator>
+): string {
+  if (!params) {
+    return sql;
+  }
+
+  const defaults = new Map<string, string>();
+  for (const [name, validator] of Object.entries(params)) {
+    const defaultValue = getParamDefault(validator);
+    if (defaultValue !== undefined) {
+      defaults.set(name, toTemplateDefaultLiteral(defaultValue as string | number | boolean));
+    }
+  }
+
+  if (defaults.size === 0) {
+    return sql;
+  }
+
+  const placeholderRegex = /\{\{\s*([^{}]+?)\s*\}\}/g;
+  return sql.replace(placeholderRegex, (fullMatch, rawExpression) => {
+    const expression = String(rawExpression);
+    const rewritten = expression.replace(
+      /([a-zA-Z_][a-zA-Z0-9_]*)\s*\(([^()]*)\)/g,
+      (call, _functionName, rawArgs) => {
+        const args = splitTopLevelComma(String(rawArgs));
+        if (args.length !== 1) {
+          return call;
+        }
+
+        const paramName = args[0]?.trim() ?? "";
+        if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(paramName)) {
+          return call;
+        }
+
+        const defaultLiteral = defaults.get(paramName);
+        if (!defaultLiteral) {
+          return call;
+        }
+
+        return call.replace(/\)\s*$/, `, ${defaultLiteral})`);
+      }
+    );
+
+    if (rewritten === expression) {
+      return fullMatch;
+    }
+    return `{{ ${rewritten.trim()} }}`;
+  });
+}
+
 /**
  * Generate a NODE section for the pipe
  */
-function generateNode(node: NodeDefinition): string {
+function generateNode(
+  node: NodeDefinition,
+  params?: Record<string, AnyParamValidator>
+): string {
   const parts: string[] = [];
 
   parts.push(`NODE ${node._name}`);
@@ -57,7 +172,8 @@ function generateNode(node: NodeDefinition): string {
     parts.push(`    %`);
   }
 
-  const sqlLines = node.sql.trim().split("\n");
+  const sqlWithDefaults = applyParamDefaultsToSql(node.sql, params);
+  const sqlLines = sqlWithDefaults.trim().split("\n");
   sqlLines.forEach((line) => {
     parts.push(`    ${line}`);
   });
@@ -225,7 +341,7 @@ export function generatePipe(pipe: PipeDefinition): GeneratedPipe {
 
   // Add all nodes
   pipe.options.nodes.forEach((node, index) => {
-    parts.push(generateNode(node));
+    parts.push(generateNode(node, pipe.options.params as Record<string, AnyParamValidator> | undefined));
     // Add empty line between nodes
     if (index < pipe.options.nodes.length - 1) {
       parts.push("");

package/src/index.ts

@@ -114,6 +114,7 @@ export type {
   KafkaConfig,
   S3Config,
   GCSConfig,
+  DatasourceIndex,
 } from "./schema/datasource.js";
 
 // ============ Connection ============

package/src/migrate/emit-ts.ts

@@ -133,6 +133,8 @@ function strictParamBaseValidator(type: string): string {
   const map: Record<string, string> = {
     String: "p.string()",
     UUID: "p.uuid()",
+    Int: "p.int32()",
+    Integer: "p.int32()",
     Int8: "p.int8()",
     Int16: "p.int16()",
     Int32: "p.int32()",
@@ -149,6 +151,8 @@ function strictParamBaseValidator(type: string): string {
     DateTime: "p.dateTime()",
     DateTime64: "p.dateTime64()",
     Array: "p.array(p.string())",
+    column: "p.column()",
+    JSON: "p.json()",
   };
   const validator = map[type];
   if (!validator) {
@@ -313,6 +317,15 @@ function emitDatasource(ds: DatasourceModel): string {
   if (ds.engine) {
     lines.push(`  engine: ${emitEngineOptions(ds.engine)},`);
   }
+  if (ds.indexes.length > 0) {
+    lines.push("  indexes: [");
+    for (const index of ds.indexes) {
+      lines.push(
+        `    { name: ${escapeString(index.name)}, expr: ${escapeString(index.expr)}, type: ${escapeString(index.type)}, granularity: ${index.granularity} },`
+      );
+    }
+    lines.push("  ],");
+  }
 
   if (ds.kafka) {
     const connectionVar = toCamelCase(ds.kafka.connectionName);

package/src/migrate/parse-datasource.ts

@@ -1,4 +1,9 @@
-import type { DatasourceModel, DatasourceTokenModel, ResourceFile } from "./types.js";
+import type {
+  DatasourceIndexModel,
+  DatasourceModel,
+  DatasourceTokenModel,
+  ResourceFile,
+} from "./types.js";
 import {
   MigrationParseError,
   isBlank,
@@ -14,6 +19,7 @@ const DATASOURCE_DIRECTIVES = new Set([
   "DESCRIPTION",
   "SCHEMA",
   "FORWARD_QUERY",
+  "INDEXES",
   "SHARED_WITH",
   "ENGINE",
   "ENGINE_SORTING_KEY",
@@ -241,9 +247,53 @@ function parseToken(filePath: string, resourceName: string, value: string): Data
   return { name, scope };
 }
 
+function parseIndexLine(
+  filePath: string,
+  resourceName: string,
+  rawLine: string
+): DatasourceIndexModel {
+  const line = rawLine.trim().replace(/,$/, "");
+  if (!line) {
+    throw new MigrationParseError(
+      filePath,
+      "datasource",
+      resourceName,
+      "Empty INDEXES line."
+    );
+  }
+
+  const match = line.match(/^(\S+)\s+(.+?)\s+TYPE\s+(.+?)\s+GRANULARITY\s+(\d+)$/i);
+  if (!match) {
+    throw new MigrationParseError(
+      filePath,
+      "datasource",
+      resourceName,
+      `Invalid INDEXES definition: "${rawLine}"`
+    );
+  }
+
+  const granularity = Number(match[4]);
+  if (!Number.isInteger(granularity) || granularity <= 0) {
+    throw new MigrationParseError(
+      filePath,
+      "datasource",
+      resourceName,
+      `Invalid INDEXES GRANULARITY value: "${match[4]}"`
+    );
+  }
+
+  return {
+    name: match[1],
+    expr: match[2].trim(),
+    type: match[3].trim(),
+    granularity,
+  };
+}
+
 export function parseDatasourceFile(resource: ResourceFile): DatasourceModel {
   const lines = splitLines(resource.content);
   const columns = [];
+  const indexes: DatasourceIndexModel[] = [];
   const tokens: DatasourceTokenModel[] = [];
   const sharedWith: string[] = [];
   let description: string | undefined;
@@ -323,6 +373,26 @@ export function parseDatasourceFile(resource: ResourceFile): DatasourceModel {
       continue;
     }
 
+    if (line === "INDEXES >") {
+      const block = readDirectiveBlock(lines, i + 1, isDatasourceDirectiveLine);
+      if (block.lines.length === 0) {
+        throw new MigrationParseError(
+          resource.filePath,
+          "datasource",
+          resource.name,
+          "INDEXES block is empty."
+        );
+      }
+      for (const indexLine of block.lines) {
+        if (isBlank(indexLine) || indexLine.trim().startsWith("#")) {
+          continue;
+        }
+        indexes.push(parseIndexLine(resource.filePath, resource.name, indexLine));
+      }
+      i = block.nextIndex;
+      continue;
+    }
+
     if (line === "SHARED_WITH >") {
       const block = readDirectiveBlock(lines, i + 1, isDatasourceDirectiveLine);
       for (const sharedLine of block.lines) {
@@ -552,6 +622,7 @@ export function parseDatasourceFile(resource: ResourceFile): DatasourceModel {
           settings,
         }
       : undefined,
+    indexes,
     kafka,
     s3,
     forwardQuery,