@tinybirdco/sdk 0.0.49 → 0.0.51

package/src/migrate/parse-connection.ts

@@ -1,4 +1,9 @@
-import type { KafkaConnectionModel, ResourceFile, S3ConnectionModel } from "./types.js";
+import type {
+  GCSConnectionModel,
+  KafkaConnectionModel,
+  ResourceFile,
+  S3ConnectionModel,
+} from "./types.js";
 import {
   MigrationParseError,
   isBlank,
@@ -9,7 +14,7 @@ import {
 
 export function parseConnectionFile(
   resource: ResourceFile
-): KafkaConnectionModel | S3ConnectionModel {
+): KafkaConnectionModel | S3ConnectionModel | GCSConnectionModel {
   const lines = splitLines(resource.content);
   let connectionType: string | undefined;
 
@@ -34,6 +39,7 @@ export function parseConnectionFile(
   let arn: string | undefined;
   let accessKey: string | undefined;
   let accessSecret: string | undefined;
+  let serviceAccountCredentialsJson: string | undefined;
 
   for (const rawLine of lines) {
     const line = rawLine.trim();
@@ -100,6 +106,9 @@ export function parseConnectionFile(
       case "S3_SECRET":
         accessSecret = parseQuotedValue(value);
         break;
+      case "GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON":
+        serviceAccountCredentialsJson = parseQuotedValue(value);
+        break;
       default:
         throw new MigrationParseError(
           resource.filePath,
@@ -120,12 +129,12 @@ export function parseConnectionFile(
   }
 
   if (connectionType === "kafka") {
-    if (region || arn || accessKey || accessSecret) {
+    if (region || arn || accessKey || accessSecret || serviceAccountCredentialsJson) {
       throw new MigrationParseError(
         resource.filePath,
         "connection",
         resource.name,
-        "S3 directives are not valid for kafka connections."
+        "S3/GCS directives are not valid for kafka connections."
       );
     }
 
@@ -161,13 +170,14 @@ export function parseConnectionFile(
       key ||
       secret ||
       schemaRegistryUrl ||
-      sslCaPem
+      sslCaPem ||
+      serviceAccountCredentialsJson
     ) {
       throw new MigrationParseError(
         resource.filePath,
         "connection",
         resource.name,
-        "Kafka directives are not valid for s3 connections."
+        "Kafka/GCS directives are not valid for s3 connections."
       );
     }
 
@@ -210,6 +220,46 @@ export function parseConnectionFile(
     };
   }
 
+  if (connectionType === "gcs") {
+    if (
+      bootstrapServers ||
+      securityProtocol ||
+      saslMechanism ||
+      key ||
+      secret ||
+      schemaRegistryUrl ||
+      sslCaPem ||
+      region ||
+      arn ||
+      accessKey ||
+      accessSecret
+    ) {
+      throw new MigrationParseError(
+        resource.filePath,
+        "connection",
+        resource.name,
+        "Kafka/S3 directives are not valid for gcs connections."
+      );
+    }
+
+    if (!serviceAccountCredentialsJson) {
+      throw new MigrationParseError(
+        resource.filePath,
+        "connection",
+        resource.name,
+        "GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON is required for gcs connections."
+      );
+    }
+
+    return {
+      kind: "connection",
+      name: resource.name,
+      filePath: resource.filePath,
+      connectionType: "gcs",
+      serviceAccountCredentialsJson,
+    };
+  }
+
   throw new MigrationParseError(
     resource.filePath,
     "connection",

package/src/migrate/parse-datasource.ts

@@ -1,4 +1,9 @@
-import type { DatasourceModel, DatasourceTokenModel, ResourceFile } from "./types.js";
+import type {
+  DatasourceIndexModel,
+  DatasourceModel,
+  DatasourceTokenModel,
+  ResourceFile,
+} from "./types.js";
 import {
   MigrationParseError,
   isBlank,
@@ -14,6 +19,7 @@ const DATASOURCE_DIRECTIVES = new Set([
   "DESCRIPTION",
   "SCHEMA",
   "FORWARD_QUERY",
+  "INDEXES",
   "SHARED_WITH",
   "ENGINE",
   "ENGINE_SORTING_KEY",
@@ -241,9 +247,53 @@ function parseToken(filePath: string, resourceName: string, value: string): Data
   return { name, scope };
 }
 
+function parseIndexLine(
+  filePath: string,
+  resourceName: string,
+  rawLine: string
+): DatasourceIndexModel {
+  const line = rawLine.trim().replace(/,$/, "");
+  if (!line) {
+    throw new MigrationParseError(
+      filePath,
+      "datasource",
+      resourceName,
+      "Empty INDEXES line."
+    );
+  }
+
+  const match = line.match(/^(\S+)\s+(.+?)\s+TYPE\s+(.+?)\s+GRANULARITY\s+(\d+)$/i);
+  if (!match) {
+    throw new MigrationParseError(
+      filePath,
+      "datasource",
+      resourceName,
+      `Invalid INDEXES definition: "${rawLine}"`
+    );
+  }
+
+  const granularity = Number(match[4]);
+  if (!Number.isInteger(granularity) || granularity <= 0) {
+    throw new MigrationParseError(
+      filePath,
+      "datasource",
+      resourceName,
+      `Invalid INDEXES GRANULARITY value: "${match[4]}"`
+    );
+  }
+
+  return {
+    name: match[1],
+    expr: match[2].trim(),
+    type: match[3].trim(),
+    granularity,
+  };
+}
+
 export function parseDatasourceFile(resource: ResourceFile): DatasourceModel {
   const lines = splitLines(resource.content);
   const columns = [];
+  const indexes: DatasourceIndexModel[] = [];
   const tokens: DatasourceTokenModel[] = [];
   const sharedWith: string[] = [];
   let description: string | undefined;
@@ -323,6 +373,26 @@ export function parseDatasourceFile(resource: ResourceFile): DatasourceModel {
       continue;
     }
 
+    if (line === "INDEXES >") {
+      const block = readDirectiveBlock(lines, i + 1, isDatasourceDirectiveLine);
+      if (block.lines.length === 0) {
+        throw new MigrationParseError(
+          resource.filePath,
+          "datasource",
+          resource.name,
+          "INDEXES block is empty."
+        );
+      }
+      for (const indexLine of block.lines) {
+        if (isBlank(indexLine) || indexLine.trim().startsWith("#")) {
+          continue;
+        }
+        indexes.push(parseIndexLine(resource.filePath, resource.name, indexLine));
+      }
+      i = block.nextIndex;
+      continue;
+    }
+
     if (line === "SHARED_WITH >") {
       const block = readDirectiveBlock(lines, i + 1, isDatasourceDirectiveLine);
       for (const sharedLine of block.lines) {
@@ -518,7 +588,7 @@ export function parseDatasourceFile(resource: ResourceFile): DatasourceModel {
       resource.filePath,
       "datasource",
       resource.name,
-      "IMPORT_CONNECTION_NAME and IMPORT_BUCKET_URI are required when S3 import directives are used."
+      "IMPORT_CONNECTION_NAME and IMPORT_BUCKET_URI are required when import directives are used."
     );
   }
 
@@ -527,7 +597,7 @@ export function parseDatasourceFile(resource: ResourceFile): DatasourceModel {
       resource.filePath,
       "datasource",
       resource.name,
-      "Datasource cannot mix Kafka directives with S3 import directives."
+      "Datasource cannot mix Kafka directives with import directives."
     );
   }
 
@@ -552,6 +622,7 @@ export function parseDatasourceFile(resource: ResourceFile): DatasourceModel {
           settings,
         }
       : undefined,
+    indexes,
     kafka,
     s3,
     forwardQuery,