@tinybirdco/sdk 0.0.59 → 0.0.61

package/src/migrate/emit-ts.ts

@@ -288,20 +288,24 @@ function emitDatasource(ds: DatasourceModel): string {
     const columnKey = emitObjectKey(column.name);
 
     if (column.defaultExpression !== undefined) {
-      const parsedDefault = parseLiteralFromDatafile(column.defaultExpression);
-      let literalValue = parsedDefault;
-      if (typeof parsedDefault === "number" && isBooleanType(column.type)) {
-        if (parsedDefault === 0 || parsedDefault === 1) {
-          literalValue = parsedDefault === 1;
-        } else {
-          throw new Error(
-            `Boolean default value must be 0 or 1 for column "${column.name}" in datasource "${ds.name}".`
-          );
+      try {
+        const parsedDefault = parseLiteralFromDatafile(column.defaultExpression);
+        let literalValue = parsedDefault;
+        if (typeof parsedDefault === "number" && isBooleanType(column.type)) {
+          if (parsedDefault === 0 || parsedDefault === 1) {
+            literalValue = parsedDefault === 1;
+          } else {
+            throw new Error(
+              `Boolean default value must be 0 or 1 for column "${column.name}" in datasource "${ds.name}".`
+            );
+          }
         }
+        validator += `.default(${toTsLiteral(
+          literalValue as string | number | boolean | null | Record<string, unknown> | unknown[]
+        )})`;
+      } catch {
+        validator += `.defaultExpr(${escapeString(column.defaultExpression)})`;
       }
-      validator += `.default(${toTsLiteral(
-        literalValue as string | number | boolean | null | Record<string, unknown> | unknown[]
-      )})`;
     }
 
     if (column.codec) {

package/src/migrate/parse-connection.test.ts

@@ -0,0 +1,98 @@
+import { describe, it, expect } from "vitest";
+import { parseConnectionFile } from "./parse-connection.js";
+import type { ResourceFile } from "./types.js";
+
+function resource(name: string, content: string): ResourceFile {
+  return {
+    kind: "connection",
+    name,
+    filePath: `${name}.connection`,
+    absolutePath: `/tmp/${name}.connection`,
+    content,
+  };
+}
+
+describe("parseConnectionFile", () => {
+  it("parses basic Kafka connection", () => {
+    const result = parseConnectionFile(
+      resource(
+        "my_kafka",
+        `TYPE kafka
+KAFKA_BOOTSTRAP_SERVERS localhost:9092`
+      )
+    );
+
+    expect(result.connectionType).toBe("kafka");
+    expect(result).toHaveProperty("bootstrapServers", "localhost:9092");
+  });
+
+  it("parses single-quoted TYPE", () => {
+    const result = parseConnectionFile(
+      resource(
+        "my_kafka",
+        `TYPE 'kafka'
+KAFKA_BOOTSTRAP_SERVERS localhost:9092
+KAFKA_SCHEMA_REGISTRY_URL https://registry.example.com`
+      )
+    );
+
+    expect(result.connectionType).toBe("kafka");
+    expect(result).toHaveProperty("schemaRegistryUrl", "https://registry.example.com");
+  });
+
+  it("parses multiline SSL CA PEM with > syntax", () => {
+    const result = parseConnectionFile(
+      resource(
+        "my_kafka",
+        `TYPE kafka
+KAFKA_BOOTSTRAP_SERVERS localhost:9092
+KAFKA_SECURITY_PROTOCOL SASL_SSL
+KAFKA_SSL_CA_PEM >
+    -----BEGIN CERTIFICATE-----
+    MIIDXTCCAkWgAwIBAgIJAM
+    -----END CERTIFICATE-----`
+      )
+    );
+
+    expect(result).toHaveProperty(
+      "sslCaPem",
+      "-----BEGIN CERTIFICATE-----\nMIIDXTCCAkWgAwIBAgIJAM\n-----END CERTIFICATE-----"
+    );
+  });
+
+  it("parses multiline SSL CA PEM with directives after the block", () => {
+    const result = parseConnectionFile(
+      resource(
+        "my_kafka",
+        `TYPE kafka
+KAFKA_BOOTSTRAP_SERVERS localhost:9092
+KAFKA_SSL_CA_PEM >
+    -----BEGIN CERTIFICATE-----
+    MIIDXTCCAkWgAwIBAgIJAM
+    -----END CERTIFICATE-----
+KAFKA_SECURITY_PROTOCOL SASL_SSL
+KAFKA_KEY mykey`
+      )
+    );
+
+    expect(result).toHaveProperty(
+      "sslCaPem",
+      "-----BEGIN CERTIFICATE-----\nMIIDXTCCAkWgAwIBAgIJAM\n-----END CERTIFICATE-----"
+    );
+    expect(result).toHaveProperty("securityProtocol", "SASL_SSL");
+    expect(result).toHaveProperty("key", "mykey");
+  });
+
+  it("parses single-line SSL CA PEM (e.g. a secret reference)", () => {
+    const result = parseConnectionFile(
+      resource(
+        "my_kafka",
+        `TYPE kafka
+KAFKA_BOOTSTRAP_SERVERS localhost:9092
+KAFKA_SSL_CA_PEM {{ tb_secret('KAFKA_SSL_CA_PEM') }}`
+      )
+    );
+
+    expect(result).toHaveProperty("sslCaPem", "{{ tb_secret('KAFKA_SSL_CA_PEM') }}");
+  });
+});

package/src/migrate/parse-connection.ts

@@ -9,9 +9,34 @@ import {
   isBlank,
   parseDirectiveLine,
   parseQuotedValue,
+  readDirectiveBlock,
   splitLines,
 } from "./parser-utils.js";
 
+const CONNECTION_DIRECTIVES = new Set([
+  "TYPE",
+  "KAFKA_BOOTSTRAP_SERVERS",
+  "KAFKA_SECURITY_PROTOCOL",
+  "KAFKA_SASL_MECHANISM",
+  "KAFKA_KEY",
+  "KAFKA_SECRET",
+  "KAFKA_SCHEMA_REGISTRY_URL",
+  "KAFKA_SSL_CA_PEM",
+  "S3_REGION",
+  "S3_ARN",
+  "S3_ACCESS_KEY",
+  "S3_SECRET",
+  "GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON",
+]);
+
+function isConnectionDirectiveLine(line: string): boolean {
+  const trimmed = line.trim();
+  if (!trimmed || trimmed.startsWith("#")) return false;
+  const firstSpace = trimmed.indexOf(" ");
+  const word = firstSpace === -1 ? trimmed : trimmed.slice(0, firstSpace);
+  return CONNECTION_DIRECTIVES.has(word);
+}
+
 export function parseConnectionFile(
   resource: ResourceFile
 ): KafkaConnectionModel | S3ConnectionModel | GCSConnectionModel {
@@ -41,9 +66,12 @@ export function parseConnectionFile(
   let accessSecret: string | undefined;
   let serviceAccountCredentialsJson: string | undefined;
 
-  for (const rawLine of lines) {
+  let i = 0;
+  while (i < lines.length) {
+    const rawLine = lines[i] ?? "";
     const line = rawLine.trim();
     if (isBlank(line) || line.startsWith("#")) {
+      i += 1;
       continue;
     }
 
@@ -92,6 +120,12 @@ export function parseConnectionFile(
         schemaRegistryUrl = value;
         break;
       case "KAFKA_SSL_CA_PEM":
+        if (value === ">") {
+          const block = readDirectiveBlock(lines, i + 1, isConnectionDirectiveLine);
+          sslCaPem = block.lines.join("\n");
+          i = block.nextIndex;
+          continue;
+        }
         sslCaPem = value;
         break;
       case "S3_REGION":
@@ -107,6 +141,12 @@ export function parseConnectionFile(
         accessSecret = parseQuotedValue(value);
         break;
       case "GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON":
+        if (value === ">") {
+          const block = readDirectiveBlock(lines, i + 1, isConnectionDirectiveLine);
+          serviceAccountCredentialsJson = block.lines.join("\n");
+          i = block.nextIndex;
+          continue;
+        }
         serviceAccountCredentialsJson = parseQuotedValue(value);
         break;
       default:
@@ -117,6 +157,7 @@ export function parseConnectionFile(
           `Unsupported connection directive in strict mode: "${line}"`
         );
     }
+    i += 1;
   }
 
   if (!connectionType) {

package/src/schema/types.test.ts

@@ -109,6 +109,24 @@ describe("Type Validators (t.*)", () => {
       const type = t.int32().default(42);
       expect(type._modifiers.defaultValue).toBe(42);
     });
+
+    it("stores default SQL expression in modifiers", () => {
+      const type = t.uuid().defaultExpr("generateUUIDv4()");
+      expect(type._modifiers.hasDefault).toBe(true);
+      expect(type._modifiers.defaultExpression).toBe("generateUUIDv4()");
+      expect(type._modifiers.defaultValue).toBeUndefined();
+    });
+
+    it("trims default SQL expression", () => {
+      const type = t.uuid().defaultExpr("  generateUUIDv4()  ");
+      expect(type._modifiers.defaultExpression).toBe("generateUUIDv4()");
+    });
+
+    it("throws on empty default SQL expression", () => {
+      expect(() => t.uuid().defaultExpr("   ")).toThrow(
+        "Default expression cannot be empty."
+      );
+    });
   });
 
   describe("Codec modifier", () => {

package/src/schema/types.ts

@@ -44,6 +44,14 @@ export interface TypeValidator<
     TTinybirdType,
     TModifiers & { hasDefault: true; defaultValue: TType }
   >;
+  /** Set a default SQL expression for the column (for example: generateUUIDv4()) */
+  defaultExpr(
+    expression: string,
+  ): TypeValidator<
+    TType,
+    TTinybirdType,
+    TModifiers & { hasDefault: true; defaultExpression: string }
+  >;
   /** Set a codec for compression */
   codec(
     codec: string,
@@ -59,6 +67,7 @@ export interface TypeModifiers {
   lowCardinality?: boolean;
   hasDefault?: boolean;
   defaultValue?: unknown;
+  defaultExpression?: string;
   codec?: string;
   jsonPath?: string;
 }
@@ -144,6 +153,7 @@ function createValidator<TType, TTinybirdType extends string>(
         ...modifiers,
         hasDefault: true,
         defaultValue: value,
+        defaultExpression: undefined,
       }) as TypeValidator<
         TType,
         TTinybirdType,
@@ -151,6 +161,23 @@ function createValidator<TType, TTinybirdType extends string>(
       >;
     },
 
+    defaultExpr(expression: string) {
+      const trimmed = expression.trim();
+      if (!trimmed) {
+        throw new Error("Default expression cannot be empty.");
+      }
+      return createValidator<TType, TTinybirdType>(tinybirdType, {
+        ...modifiers,
+        hasDefault: true,
+        defaultValue: undefined,
+        defaultExpression: trimmed,
+      }) as TypeValidator<
+        TType,
+        TTinybirdType,
+        TypeModifiers & { hasDefault: true; defaultExpression: string }
+      >;
+    },
+
     codec(codec: string) {
       return createValidator<TType, TTinybirdType>(tinybirdType, {
         ...modifiers,