@tinybirdco/sdk 0.0.48 → 0.0.50

package/src/migrate/emit-ts.ts

@@ -3,6 +3,8 @@ import { toCamelCase } from "../codegen/utils.js";
 import { parseLiteralFromDatafile, toTsLiteral } from "./parser-utils.js";
 import type {
   DatasourceModel,
+  DatasourceEngineModel,
+  GCSConnectionModel,
   KafkaConnectionModel,
   ParsedResource,
   PipeModel,
@@ -58,11 +60,13 @@ function hasSecretTemplate(resources: ParsedResource[]): boolean {
         if (resource.secret) values.push(resource.secret);
         if (resource.sslCaPem) values.push(resource.sslCaPem);
         if (resource.schemaRegistryUrl) values.push(resource.schemaRegistryUrl);
-      } else {
+      } else if (resource.connectionType === "s3") {
         values.push(resource.region);
         if (resource.arn) values.push(resource.arn);
         if (resource.accessKey) values.push(resource.accessKey);
         if (resource.secret) values.push(resource.secret);
+      } else {
+        values.push(resource.serviceAccountCredentialsJson);
       }
       continue;
     }
@@ -79,6 +83,11 @@ function hasSecretTemplate(resources: ParsedResource[]): boolean {
         if (resource.s3.schedule) values.push(resource.s3.schedule);
         if (resource.s3.fromTimestamp) values.push(resource.s3.fromTimestamp);
       }
+      if (resource.gcs) {
+        values.push(resource.gcs.bucketUri);
+        if (resource.gcs.schedule) values.push(resource.gcs.schedule);
+        if (resource.gcs.fromTimestamp) values.push(resource.gcs.fromTimestamp);
+      }
       continue;
     }
   }
@@ -151,7 +160,7 @@ function strictParamBaseValidator(type: string): string {
 function applyParamOptional(
   baseValidator: string,
   required: boolean,
-  defaultValue: string | number | undefined
+  defaultValue: string | number | boolean | undefined
 ): string {
   const withDefault = defaultValue !== undefined;
   if (!withDefault && required) {
@@ -168,6 +177,13 @@ function applyParamOptional(
   return `${baseValidator}${optionalSuffix}`;
 }
 
+function applyParamDescription(validator: string, description: string | undefined): string {
+  if (description === undefined) {
+    return validator;
+  }
+  return `${validator}.describe(${JSON.stringify(description)})`;
+}
+
 function engineFunctionName(type: string): string {
   const map: Record<string, string> = {
     MergeTree: "mergeTree",
@@ -184,9 +200,8 @@ function engineFunctionName(type: string): string {
   return functionName;
 }
 
-function emitEngineOptions(ds: DatasourceModel): string {
+function emitEngineOptions(engine: DatasourceEngineModel): string {
   const options: string[] = [];
-  const { engine } = ds;
 
   if (engine.sortingKey.length === 1) {
     options.push(`sortingKey: ${escapeString(engine.sortingKey[0]!)}`);
@@ -256,7 +271,7 @@ function emitDatasource(ds: DatasourceModel): string {
   }
 
   lines.push(`export const ${variableName} = defineDatasource(${escapeString(ds.name)}, {`);
-  if (ds.description) {
+  if (ds.description !== undefined) {
     lines.push(`  description: ${emitStringOrSecret(ds.description)},`);
   }
   if (!hasJsonPath) {
@@ -295,7 +310,9 @@ function emitDatasource(ds: DatasourceModel): string {
     lines.push(`    ${columnKey}: ${validator},`);
   }
   lines.push("  },");
-  lines.push(`  engine: ${emitEngineOptions(ds)},`);
+  if (ds.engine) {
+    lines.push(`  engine: ${emitEngineOptions(ds.engine)},`);
+  }
 
   if (ds.kafka) {
     const connectionVar = toCamelCase(ds.kafka.connectionName);
@@ -328,6 +345,20 @@ function emitDatasource(ds: DatasourceModel): string {
     lines.push("  },");
   }
 
+  if (ds.gcs) {
+    const connectionVar = toCamelCase(ds.gcs.connectionName);
+    lines.push("  gcs: {");
+    lines.push(`    connection: ${connectionVar},`);
+    lines.push(`    bucketUri: ${emitStringOrSecret(ds.gcs.bucketUri)},`);
+    if (ds.gcs.schedule) {
+      lines.push(`    schedule: ${emitStringOrSecret(ds.gcs.schedule)},`);
+    }
+    if (ds.gcs.fromTimestamp) {
+      lines.push(`    fromTimestamp: ${emitStringOrSecret(ds.gcs.fromTimestamp)},`);
+    }
+    lines.push("  },");
+  }
+
   if (ds.forwardQuery) {
     lines.push("  forwardQuery: `");
     lines.push(ds.forwardQuery.replace(/`/g, "\\`").replace(/\${/g, "\\${"));
@@ -355,7 +386,9 @@ function emitDatasource(ds: DatasourceModel): string {
   return lines.join("\n");
 }
 
-function emitConnection(connection: KafkaConnectionModel | S3ConnectionModel): string {
+function emitConnection(
+  connection: KafkaConnectionModel | S3ConnectionModel | GCSConnectionModel
+): string {
   const variableName = toCamelCase(connection.name);
   const lines: string[] = [];
 
@@ -387,19 +420,31 @@ function emitConnection(connection: KafkaConnectionModel | S3ConnectionModel): s
     return lines.join("\n");
   }
 
+  if (connection.connectionType === "s3") {
+    lines.push(
+      `export const ${variableName} = defineS3Connection(${escapeString(connection.name)}, {`
+    );
+    lines.push(`  region: ${emitStringOrSecret(connection.region)},`);
+    if (connection.arn) {
+      lines.push(`  arn: ${emitStringOrSecret(connection.arn)},`);
+    }
+    if (connection.accessKey) {
+      lines.push(`  accessKey: ${emitStringOrSecret(connection.accessKey)},`);
+    }
+    if (connection.secret) {
+      lines.push(`  secret: ${emitStringOrSecret(connection.secret)},`);
+    }
+    lines.push("});");
+    lines.push("");
+    return lines.join("\n");
+  }
+
   lines.push(
-    `export const ${variableName} = defineS3Connection(${escapeString(connection.name)}, {`
+    `export const ${variableName} = defineGCSConnection(${escapeString(connection.name)}, {`
+  );
+  lines.push(
+    `  serviceAccountCredentialsJson: ${emitStringOrSecret(connection.serviceAccountCredentialsJson)},`
   );
-  lines.push(`  region: ${emitStringOrSecret(connection.region)},`);
-  if (connection.arn) {
-    lines.push(`  arn: ${emitStringOrSecret(connection.arn)},`);
-  }
-  if (connection.accessKey) {
-    lines.push(`  accessKey: ${emitStringOrSecret(connection.accessKey)},`);
-  }
-  if (connection.secret) {
-    lines.push(`  secret: ${emitStringOrSecret(connection.secret)},`);
-  }
   lines.push("});");
   lines.push("");
   return lines.join("\n");
@@ -423,24 +468,27 @@ function emitPipe(pipe: PipeModel): string {
     lines.push(`export const ${variableName} = defineMaterializedView(${escapeString(pipe.name)}, {`);
   } else if (pipe.type === "copy") {
     lines.push(`export const ${variableName} = defineCopyPipe(${escapeString(pipe.name)}, {`);
+  } else if (pipe.type === "sink") {
+    lines.push(`export const ${variableName} = defineSinkPipe(${escapeString(pipe.name)}, {`);
   } else {
     lines.push(`export const ${variableName} = definePipe(${escapeString(pipe.name)}, {`);
   }
 
-  if (pipe.description) {
+  if (pipe.description !== undefined) {
     lines.push(`  description: ${escapeString(pipe.description)},`);
   }
 
-  if (pipe.type === "pipe" || pipe.type === "endpoint") {
+  if (pipe.type === "pipe" || pipe.type === "endpoint" || pipe.type === "sink") {
     if (pipe.params.length > 0) {
       lines.push("  params: {");
       for (const param of pipe.params) {
         const baseValidator = strictParamBaseValidator(param.type);
-        const validator = applyParamOptional(
+        const validatorWithOptional = applyParamOptional(
           baseValidator,
           param.required,
           param.defaultValue
         );
+        const validator = applyParamDescription(validatorWithOptional, param.description);
         lines.push(`    ${emitObjectKey(param.name)}: ${validator},`);
       }
       lines.push("  },");
@@ -464,11 +512,35 @@ function emitPipe(pipe: PipeModel): string {
     }
   }
 
+  if (pipe.type === "sink") {
+    if (!pipe.sink) {
+      throw new Error(`Sink pipe "${pipe.name}" is missing sink configuration.`);
+    }
+    lines.push("  sink: {");
+    lines.push(`    connection: ${toCamelCase(pipe.sink.connectionName)},`);
+    if (pipe.sink.service === "kafka") {
+      lines.push(`    topic: ${escapeString(pipe.sink.topic)},`);
+      lines.push(`    schedule: ${escapeString(pipe.sink.schedule)},`);
+    } else {
+      lines.push(`    bucketUri: ${escapeString(pipe.sink.bucketUri)},`);
+      lines.push(`    fileTemplate: ${escapeString(pipe.sink.fileTemplate)},`);
+      lines.push(`    schedule: ${escapeString(pipe.sink.schedule)},`);
+      lines.push(`    format: ${escapeString(pipe.sink.format)},`);
+      if (pipe.sink.strategy) {
+        lines.push(`    strategy: ${escapeString(pipe.sink.strategy)},`);
+      }
+      if (pipe.sink.compression) {
+        lines.push(`    compression: ${escapeString(pipe.sink.compression)},`);
+      }
+    }
+    lines.push("  },");
+  }
+
   lines.push("  nodes: [");
   for (const node of pipe.nodes) {
     lines.push("    node({");
     lines.push(`      name: ${escapeString(node.name)},`);
-    if (node.description) {
+    if (node.description !== undefined) {
       lines.push(`      description: ${escapeString(node.description)},`);
     }
     lines.push("      sql: `");
@@ -506,7 +578,7 @@ function emitPipe(pipe: PipeModel): string {
 
 export function emitMigrationFileContent(resources: ParsedResource[]): string {
   const connections = resources.filter(
-    (resource): resource is KafkaConnectionModel | S3ConnectionModel =>
+    (resource): resource is KafkaConnectionModel | S3ConnectionModel | GCSConnectionModel =>
       resource.kind === "connection"
   );
   const datasources = resources.filter(
@@ -526,7 +598,6 @@ export function emitMigrationFileContent(resources: ParsedResource[]): string {
     "defineCopyPipe",
     "node",
     "t",
-    "engine",
   ]);
   if (connections.some((connection) => connection.connectionType === "kafka")) {
     imports.add("defineKafkaConnection");
@@ -534,9 +605,18 @@ export function emitMigrationFileContent(resources: ParsedResource[]): string {
   if (connections.some((connection) => connection.connectionType === "s3")) {
     imports.add("defineS3Connection");
   }
+  if (connections.some((connection) => connection.connectionType === "gcs")) {
+    imports.add("defineGCSConnection");
+  }
   if (needsParams) {
     imports.add("p");
   }
+  if (pipes.some((pipe) => pipe.type === "sink")) {
+    imports.add("defineSinkPipe");
+  }
+  if (datasources.some((datasource) => datasource.engine !== undefined)) {
+    imports.add("engine");
+  }
   if (needsSecret) {
     imports.add("secret");
   }
@@ -544,10 +624,12 @@ export function emitMigrationFileContent(resources: ParsedResource[]): string {
   const orderedImports = [
     "defineKafkaConnection",
     "defineS3Connection",
+    "defineGCSConnection",
     "defineDatasource",
     "definePipe",
     "defineMaterializedView",
     "defineCopyPipe",
+    "defineSinkPipe",
     "node",
     "t",
     "engine",

package/src/migrate/parse-connection.ts

@@ -1,4 +1,9 @@
-import type { KafkaConnectionModel, ResourceFile, S3ConnectionModel } from "./types.js";
+import type {
+  GCSConnectionModel,
+  KafkaConnectionModel,
+  ResourceFile,
+  S3ConnectionModel,
+} from "./types.js";
 import {
   MigrationParseError,
   isBlank,
@@ -9,7 +14,7 @@ import {
 
 export function parseConnectionFile(
   resource: ResourceFile
-): KafkaConnectionModel | S3ConnectionModel {
+): KafkaConnectionModel | S3ConnectionModel | GCSConnectionModel {
   const lines = splitLines(resource.content);
   let connectionType: string | undefined;
 
@@ -34,6 +39,7 @@ export function parseConnectionFile(
   let arn: string | undefined;
   let accessKey: string | undefined;
   let accessSecret: string | undefined;
+  let serviceAccountCredentialsJson: string | undefined;
 
   for (const rawLine of lines) {
     const line = rawLine.trim();
@@ -100,6 +106,9 @@ export function parseConnectionFile(
       case "S3_SECRET":
         accessSecret = parseQuotedValue(value);
         break;
+      case "GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON":
+        serviceAccountCredentialsJson = parseQuotedValue(value);
+        break;
       default:
         throw new MigrationParseError(
           resource.filePath,
@@ -120,12 +129,12 @@ export function parseConnectionFile(
   }
 
   if (connectionType === "kafka") {
-    if (region || arn || accessKey || accessSecret) {
+    if (region || arn || accessKey || accessSecret || serviceAccountCredentialsJson) {
       throw new MigrationParseError(
         resource.filePath,
         "connection",
         resource.name,
-        "S3 directives are not valid for kafka connections."
+        "S3/GCS directives are not valid for kafka connections."
       );
     }
 
@@ -161,13 +170,14 @@ export function parseConnectionFile(
       key ||
       secret ||
       schemaRegistryUrl ||
-      sslCaPem
+      sslCaPem ||
+      serviceAccountCredentialsJson
     ) {
       throw new MigrationParseError(
         resource.filePath,
         "connection",
         resource.name,
-        "Kafka directives are not valid for s3 connections."
+        "Kafka/GCS directives are not valid for s3 connections."
       );
     }
 
@@ -210,6 +220,46 @@ export function parseConnectionFile(
     };
   }
 
+  if (connectionType === "gcs") {
+    if (
+      bootstrapServers ||
+      securityProtocol ||
+      saslMechanism ||
+      key ||
+      secret ||
+      schemaRegistryUrl ||
+      sslCaPem ||
+      region ||
+      arn ||
+      accessKey ||
+      accessSecret
+    ) {
+      throw new MigrationParseError(
+        resource.filePath,
+        "connection",
+        resource.name,
+        "Kafka/S3 directives are not valid for gcs connections."
+      );
+    }
+
+    if (!serviceAccountCredentialsJson) {
+      throw new MigrationParseError(
+        resource.filePath,
+        "connection",
+        resource.name,
+        "GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON is required for gcs connections."
+      );
+    }
+
+    return {
+      kind: "connection",
+      name: resource.name,
+      filePath: resource.filePath,
+      connectionType: "gcs",
+      serviceAccountCredentialsJson,
+    };
+  }
+
   throw new MigrationParseError(
     resource.filePath,
     "connection",

package/src/migrate/parse-datasource.ts

@@ -4,45 +4,44 @@ import {
   isBlank,
   parseDirectiveLine,
   parseQuotedValue,
+  readDirectiveBlock,
   splitCommaSeparated,
   splitLines,
   splitTopLevelComma,
-  stripIndent,
 } from "./parser-utils.js";
 
-interface BlockReadResult {
-  lines: string[];
-  nextIndex: number;
-}
-
-function readIndentedBlock(lines: string[], startIndex: number): BlockReadResult {
-  const collected: string[] = [];
-  let i = startIndex;
-
-  while (i < lines.length) {
-    const line = lines[i] ?? "";
-    if (line.startsWith("    ")) {
-      collected.push(stripIndent(line));
-      i += 1;
-      continue;
-    }
-
-    if (isBlank(line)) {
-      let j = i + 1;
-      while (j < lines.length && isBlank(lines[j] ?? "")) {
-        j += 1;
-      }
-      if (j < lines.length && (lines[j] ?? "").startsWith("    ")) {
-        collected.push("");
-        i += 1;
-        continue;
-      }
-    }
-
-    break;
+const DATASOURCE_DIRECTIVES = new Set([
+  "DESCRIPTION",
+  "SCHEMA",
+  "FORWARD_QUERY",
+  "SHARED_WITH",
+  "ENGINE",
+  "ENGINE_SORTING_KEY",
+  "ENGINE_PARTITION_KEY",
+  "ENGINE_PRIMARY_KEY",
+  "ENGINE_TTL",
+  "ENGINE_VER",
+  "ENGINE_SIGN",
+  "ENGINE_VERSION",
+  "ENGINE_SUMMING_COLUMNS",
+  "ENGINE_SETTINGS",
+  "KAFKA_CONNECTION_NAME",
+  "KAFKA_TOPIC",
+  "KAFKA_GROUP_ID",
+  "KAFKA_AUTO_OFFSET_RESET",
+  "IMPORT_CONNECTION_NAME",
+  "IMPORT_BUCKET_URI",
+  "IMPORT_SCHEDULE",
+  "IMPORT_FROM_TIMESTAMP",
+  "TOKEN",
+]);
+
+function isDatasourceDirectiveLine(line: string): boolean {
+  if (!line) {
+    return false;
   }
-
-  return { lines: collected, nextIndex: i };
+  const { key } = parseDirectiveLine(line);
+  return DATASOURCE_DIRECTIVES.has(key);
 }
 
 function findTokenOutsideContexts(input: string, token: string): number {
@@ -197,7 +196,15 @@ function parseEngineSettings(value: string): Record<string, string | number | bo
 }
 
 function parseToken(filePath: string, resourceName: string, value: string): DatasourceTokenModel {
-  const parts = value.split(/\s+/).filter(Boolean);
+  const trimmed = value.trim();
+  const quotedMatch = trimmed.match(/^"([^"]+)"\s+(READ|APPEND)$/);
+  if (quotedMatch) {
+    const name = quotedMatch[1];
+    const scope = quotedMatch[2] as "READ" | "APPEND";
+    return { name, scope };
+  }
+
+  const parts = trimmed.split(/\s+/).filter(Boolean);
   if (parts.length < 2) {
     throw new MigrationParseError(
       filePath,
@@ -216,7 +223,11 @@ function parseToken(filePath: string, resourceName: string, value: string): Data
     );
   }
 
-  const name = parts[0];
+  const rawName = parts[0] ?? "";
+  const name =
+    rawName.startsWith('"') && rawName.endsWith('"') && rawName.length >= 2
+      ? rawName.slice(1, -1)
+      : rawName;
   const scope = parts[1];
   if (scope !== "READ" && scope !== "APPEND") {
     throw new MigrationParseError(
@@ -271,22 +282,14 @@ export function parseDatasourceFile(resource: ResourceFile): DatasourceModel {
     }
 
     if (line === "DESCRIPTION >") {
-      const block = readIndentedBlock(lines, i + 1);
-      if (block.lines.length === 0) {
-        throw new MigrationParseError(
-          resource.filePath,
-          "datasource",
-          resource.name,
-          "DESCRIPTION block is empty."
-        );
-      }
+      const block = readDirectiveBlock(lines, i + 1, isDatasourceDirectiveLine);
       description = block.lines.join("\n");
       i = block.nextIndex;
       continue;
     }
 
     if (line === "SCHEMA >") {
-      const block = readIndentedBlock(lines, i + 1);
+      const block = readDirectiveBlock(lines, i + 1, isDatasourceDirectiveLine);
       if (block.lines.length === 0) {
         throw new MigrationParseError(
           resource.filePath,
@@ -306,7 +309,7 @@ export function parseDatasourceFile(resource: ResourceFile): DatasourceModel {
     }
 
     if (line === "FORWARD_QUERY >") {
-      const block = readIndentedBlock(lines, i + 1);
+      const block = readDirectiveBlock(lines, i + 1, isDatasourceDirectiveLine);
       if (block.lines.length === 0) {
         throw new MigrationParseError(
           resource.filePath,
@@ -321,7 +324,7 @@ export function parseDatasourceFile(resource: ResourceFile): DatasourceModel {
     }
 
     if (line === "SHARED_WITH >") {
-      const block = readIndentedBlock(lines, i + 1);
+      const block = readDirectiveBlock(lines, i + 1, isDatasourceDirectiveLine);
       for (const sharedLine of block.lines) {
         const normalized = sharedLine.trim().replace(/,$/, "");
         if (normalized) {
@@ -449,16 +452,25 @@ export function parseDatasourceFile(resource: ResourceFile): DatasourceModel {
     );
   }
 
-  if (!engineType) {
-    throw new MigrationParseError(
-      resource.filePath,
-      "datasource",
-      resource.name,
-      "ENGINE directive is required."
-    );
+  const hasEngineDirectives =
+    sortingKey.length > 0 ||
+    partitionKey !== undefined ||
+    (primaryKey !== undefined && primaryKey.length > 0) ||
+    ttl !== undefined ||
+    ver !== undefined ||
+    isDeleted !== undefined ||
+    sign !== undefined ||
+    version !== undefined ||
+    (summingColumns !== undefined && summingColumns.length > 0) ||
+    settings !== undefined;
+
+  if (!engineType && hasEngineDirectives) {
+    // Tinybird defaults to MergeTree when ENGINE is omitted.
+    // If engine-specific options are present, preserve them by inferring MergeTree.
+    engineType = "MergeTree";
   }
 
-  if (sortingKey.length === 0) {
+  if (engineType && sortingKey.length === 0) {
     throw new MigrationParseError(
       resource.filePath,
       "datasource",
@@ -506,7 +518,7 @@ export function parseDatasourceFile(resource: ResourceFile): DatasourceModel {
       resource.filePath,
       "datasource",
       resource.name,
-      "IMPORT_CONNECTION_NAME and IMPORT_BUCKET_URI are required when S3 import directives are used."
+      "IMPORT_CONNECTION_NAME and IMPORT_BUCKET_URI are required when import directives are used."
     );
   }
 
@@ -515,7 +527,7 @@ export function parseDatasourceFile(resource: ResourceFile): DatasourceModel {
       resource.filePath,
       "datasource",
       resource.name,
-      "Datasource cannot mix Kafka directives with S3 import directives."
+      "Datasource cannot mix Kafka directives with import directives."
     );
   }
 
@@ -525,19 +537,21 @@ export function parseDatasourceFile(resource: ResourceFile): DatasourceModel {
     filePath: resource.filePath,
     description,
     columns,
-    engine: {
-      type: engineType,
-      sortingKey,
-      partitionKey,
-      primaryKey,
-      ttl,
-      ver,
-      isDeleted,
-      sign,
-      version,
-      summingColumns,
-      settings,
-    },
+    engine: engineType
+      ? {
+          type: engineType,
+          sortingKey,
+          partitionKey,
+          primaryKey,
+          ttl,
+          ver,
+          isDeleted,
+          sign,
+          version,
+          summingColumns,
+          settings,
+        }
+      : undefined,
     kafka,
     s3,
     forwardQuery,