@tinybirdco/sdk 0.0.47 → 0.0.49

package/src/migrate/emit-ts.ts

@@ -3,6 +3,7 @@ import { toCamelCase } from "../codegen/utils.js";
 import { parseLiteralFromDatafile, toTsLiteral } from "./parser-utils.js";
 import type {
   DatasourceModel,
+  DatasourceEngineModel,
   KafkaConnectionModel,
   ParsedResource,
   PipeModel,
@@ -13,6 +14,79 @@ function escapeString(value: string): string {
   return JSON.stringify(value);
 }
 
+function emitObjectKey(key: string): string {
+  return /^[A-Za-z_$][A-Za-z0-9_$]*$/.test(key) ? key : escapeString(key);
+}
+
+interface ParsedSecretTemplate {
+  name: string;
+  defaultValue?: string;
+}
+
+function parseTbSecretTemplate(value: string): ParsedSecretTemplate | null {
+  const trimmed = value.trim();
+  const regex =
+    /^\{\{\s*tb_secret\(\s*["']([^"']+)["'](?:\s*,\s*["']([^"']*)["'])?\s*\)\s*\}\}$/;
+  const match = trimmed.match(regex);
+  if (!match) {
+    return null;
+  }
+  return {
+    name: match[1] ?? "",
+    defaultValue: match[2],
+  };
+}
+
+function emitStringOrSecret(value: string): string {
+  const parsed = parseTbSecretTemplate(value);
+  if (!parsed) {
+    return escapeString(value);
+  }
+  if (parsed.defaultValue !== undefined) {
+    return `secret(${escapeString(parsed.name)}, ${escapeString(parsed.defaultValue)})`;
+  }
+  return `secret(${escapeString(parsed.name)})`;
+}
+
+function hasSecretTemplate(resources: ParsedResource[]): boolean {
+  const values: string[] = [];
+
+  for (const resource of resources) {
+    if (resource.kind === "connection") {
+      if (resource.connectionType === "kafka") {
+        values.push(resource.bootstrapServers);
+        if (resource.key) values.push(resource.key);
+        if (resource.secret) values.push(resource.secret);
+        if (resource.sslCaPem) values.push(resource.sslCaPem);
+        if (resource.schemaRegistryUrl) values.push(resource.schemaRegistryUrl);
+      } else {
+        values.push(resource.region);
+        if (resource.arn) values.push(resource.arn);
+        if (resource.accessKey) values.push(resource.accessKey);
+        if (resource.secret) values.push(resource.secret);
+      }
+      continue;
+    }
+
+    if (resource.kind === "datasource") {
+      if (resource.description) values.push(resource.description);
+      if (resource.kafka) {
+        values.push(resource.kafka.topic);
+        if (resource.kafka.groupId) values.push(resource.kafka.groupId);
+        if (resource.kafka.autoOffsetReset) values.push(resource.kafka.autoOffsetReset);
+      }
+      if (resource.s3) {
+        values.push(resource.s3.bucketUri);
+        if (resource.s3.schedule) values.push(resource.s3.schedule);
+        if (resource.s3.fromTimestamp) values.push(resource.s3.fromTimestamp);
+      }
+      continue;
+    }
+  }
+
+  return values.some((value) => parseTbSecretTemplate(value) !== null);
+}
+
 function normalizedBaseType(type: string): string {
   let current = type.trim();
   let updated = true;
@@ -78,7 +152,7 @@ function strictParamBaseValidator(type: string): string {
 function applyParamOptional(
   baseValidator: string,
   required: boolean,
-  defaultValue: string | number | undefined
+  defaultValue: string | number | boolean | undefined
 ): string {
   const withDefault = defaultValue !== undefined;
   if (!withDefault && required) {
@@ -95,6 +169,13 @@ function applyParamOptional(
   return `${baseValidator}${optionalSuffix}`;
 }
 
+function applyParamDescription(validator: string, description: string | undefined): string {
+  if (description === undefined) {
+    return validator;
+  }
+  return `${validator}.describe(${JSON.stringify(description)})`;
+}
+
 function engineFunctionName(type: string): string {
   const map: Record<string, string> = {
     MergeTree: "mergeTree",
@@ -111,9 +192,8 @@ function engineFunctionName(type: string): string {
   return functionName;
 }
 
-function emitEngineOptions(ds: DatasourceModel): string {
+function emitEngineOptions(engine: DatasourceEngineModel): string {
   const options: string[] = [];
-  const { engine } = ds;
 
   if (engine.sortingKey.length === 1) {
     options.push(`sortingKey: ${escapeString(engine.sortingKey[0]!)}`);
@@ -141,6 +221,9 @@ function emitEngineOptions(ds: DatasourceModel): string {
   if (engine.ver) {
     options.push(`ver: ${escapeString(engine.ver)}`);
   }
+  if (engine.isDeleted) {
+    options.push(`isDeleted: ${escapeString(engine.isDeleted)}`);
+  }
   if (engine.sign) {
     options.push(`sign: ${escapeString(engine.sign)}`);
   }
@@ -170,13 +253,6 @@ function emitDatasource(ds: DatasourceModel): string {
   const variableName = toCamelCase(ds.name);
   const lines: string[] = [];
   const hasJsonPath = ds.columns.some((column) => column.jsonPath !== undefined);
-  const hasMissingJsonPath = ds.columns.some((column) => column.jsonPath === undefined);
-
-  if (hasJsonPath && hasMissingJsonPath) {
-    throw new Error(
-      `Datasource "${ds.name}" has mixed json path usage. This is not representable in strict mode.`
-    );
-  }
 
   if (ds.description) {
     lines.push("/**");
@@ -187,8 +263,8 @@ function emitDatasource(ds: DatasourceModel): string {
   }
 
   lines.push(`export const ${variableName} = defineDatasource(${escapeString(ds.name)}, {`);
-  if (ds.description) {
-    lines.push(`  description: ${escapeString(ds.description)},`);
+  if (ds.description !== undefined) {
+    lines.push(`  description: ${emitStringOrSecret(ds.description)},`);
   }
   if (!hasJsonPath) {
     lines.push("  jsonPaths: false,");
@@ -197,6 +273,7 @@ function emitDatasource(ds: DatasourceModel): string {
   lines.push("  schema: {");
   for (const column of ds.columns) {
     let validator = strictColumnTypeToValidator(column.type);
+    const columnKey = emitObjectKey(column.name);
 
     if (column.defaultExpression !== undefined) {
       const parsedDefault = parseLiteralFromDatafile(column.defaultExpression);
@@ -220,26 +297,28 @@ function emitDatasource(ds: DatasourceModel): string {
     }
 
     if (column.jsonPath) {
-      lines.push(
-        `    ${column.name}: column(${validator}, { jsonPath: ${escapeString(column.jsonPath)} }),`
-      );
-    } else {
-      lines.push(`    ${column.name}: ${validator},`);
+      validator += `.jsonPath(${escapeString(column.jsonPath)})`;
     }
+    lines.push(`    ${columnKey}: ${validator},`);
   }
   lines.push("  },");
-  lines.push(`  engine: ${emitEngineOptions(ds)},`);
+  if (ds.engine) {
+    lines.push(`  engine: ${emitEngineOptions(ds.engine)},`);
+  }
 
   if (ds.kafka) {
     const connectionVar = toCamelCase(ds.kafka.connectionName);
     lines.push("  kafka: {");
     lines.push(`    connection: ${connectionVar},`);
-    lines.push(`    topic: ${escapeString(ds.kafka.topic)},`);
+    lines.push(`    topic: ${emitStringOrSecret(ds.kafka.topic)},`);
     if (ds.kafka.groupId) {
-      lines.push(`    groupId: ${escapeString(ds.kafka.groupId)},`);
+      lines.push(`    groupId: ${emitStringOrSecret(ds.kafka.groupId)},`);
     }
     if (ds.kafka.autoOffsetReset) {
-      lines.push(`    autoOffsetReset: ${escapeString(ds.kafka.autoOffsetReset)},`);
+      lines.push(`    autoOffsetReset: ${emitStringOrSecret(ds.kafka.autoOffsetReset)},`);
+    }
+    if (ds.kafka.storeRawValue !== undefined) {
+      lines.push(`    storeRawValue: ${ds.kafka.storeRawValue},`);
     }
     lines.push("  },");
   }
@@ -248,12 +327,12 @@ function emitDatasource(ds: DatasourceModel): string {
     const connectionVar = toCamelCase(ds.s3.connectionName);
     lines.push("  s3: {");
     lines.push(`    connection: ${connectionVar},`);
-    lines.push(`    bucketUri: ${escapeString(ds.s3.bucketUri)},`);
+    lines.push(`    bucketUri: ${emitStringOrSecret(ds.s3.bucketUri)},`);
     if (ds.s3.schedule) {
-      lines.push(`    schedule: ${escapeString(ds.s3.schedule)},`);
+      lines.push(`    schedule: ${emitStringOrSecret(ds.s3.schedule)},`);
     }
     if (ds.s3.fromTimestamp) {
-      lines.push(`    fromTimestamp: ${escapeString(ds.s3.fromTimestamp)},`);
+      lines.push(`    fromTimestamp: ${emitStringOrSecret(ds.s3.fromTimestamp)},`);
     }
     lines.push("  },");
   }
@@ -293,21 +372,24 @@ function emitConnection(connection: KafkaConnectionModel | S3ConnectionModel): s
     lines.push(
       `export const ${variableName} = defineKafkaConnection(${escapeString(connection.name)}, {`
     );
-    lines.push(`  bootstrapServers: ${escapeString(connection.bootstrapServers)},`);
+    lines.push(`  bootstrapServers: ${emitStringOrSecret(connection.bootstrapServers)},`);
     if (connection.securityProtocol) {
-      lines.push(`  securityProtocol: ${escapeString(connection.securityProtocol)},`);
+      lines.push(`  securityProtocol: ${emitStringOrSecret(connection.securityProtocol)},`);
     }
     if (connection.saslMechanism) {
-      lines.push(`  saslMechanism: ${escapeString(connection.saslMechanism)},`);
+      lines.push(`  saslMechanism: ${emitStringOrSecret(connection.saslMechanism)},`);
     }
     if (connection.key) {
-      lines.push(`  key: ${escapeString(connection.key)},`);
+      lines.push(`  key: ${emitStringOrSecret(connection.key)},`);
     }
     if (connection.secret) {
-      lines.push(`  secret: ${escapeString(connection.secret)},`);
+      lines.push(`  secret: ${emitStringOrSecret(connection.secret)},`);
+    }
+    if (connection.schemaRegistryUrl) {
+      lines.push(`  schemaRegistryUrl: ${emitStringOrSecret(connection.schemaRegistryUrl)},`);
     }
     if (connection.sslCaPem) {
-      lines.push(`  sslCaPem: ${escapeString(connection.sslCaPem)},`);
+      lines.push(`  sslCaPem: ${emitStringOrSecret(connection.sslCaPem)},`);
     }
     lines.push("});");
     lines.push("");
@@ -317,15 +399,15 @@ function emitConnection(connection: KafkaConnectionModel | S3ConnectionModel): s
   lines.push(
     `export const ${variableName} = defineS3Connection(${escapeString(connection.name)}, {`
   );
-  lines.push(`  region: ${escapeString(connection.region)},`);
+  lines.push(`  region: ${emitStringOrSecret(connection.region)},`);
   if (connection.arn) {
-    lines.push(`  arn: ${escapeString(connection.arn)},`);
+    lines.push(`  arn: ${emitStringOrSecret(connection.arn)},`);
   }
   if (connection.accessKey) {
-    lines.push(`  accessKey: ${escapeString(connection.accessKey)},`);
+    lines.push(`  accessKey: ${emitStringOrSecret(connection.accessKey)},`);
   }
   if (connection.secret) {
-    lines.push(`  secret: ${escapeString(connection.secret)},`);
+    lines.push(`  secret: ${emitStringOrSecret(connection.secret)},`);
   }
   lines.push("});");
   lines.push("");
@@ -350,25 +432,28 @@ function emitPipe(pipe: PipeModel): string {
     lines.push(`export const ${variableName} = defineMaterializedView(${escapeString(pipe.name)}, {`);
   } else if (pipe.type === "copy") {
     lines.push(`export const ${variableName} = defineCopyPipe(${escapeString(pipe.name)}, {`);
+  } else if (pipe.type === "sink") {
+    lines.push(`export const ${variableName} = defineSinkPipe(${escapeString(pipe.name)}, {`);
   } else {
     lines.push(`export const ${variableName} = definePipe(${escapeString(pipe.name)}, {`);
   }
 
-  if (pipe.description) {
+  if (pipe.description !== undefined) {
     lines.push(`  description: ${escapeString(pipe.description)},`);
   }
 
-  if (pipe.type === "pipe" || pipe.type === "endpoint") {
+  if (pipe.type === "pipe" || pipe.type === "endpoint" || pipe.type === "sink") {
     if (pipe.params.length > 0) {
       lines.push("  params: {");
       for (const param of pipe.params) {
         const baseValidator = strictParamBaseValidator(param.type);
-        const validator = applyParamOptional(
+        const validatorWithOptional = applyParamOptional(
           baseValidator,
           param.required,
           param.defaultValue
         );
-        lines.push(`    ${param.name}: ${validator},`);
+        const validator = applyParamDescription(validatorWithOptional, param.description);
+        lines.push(`    ${emitObjectKey(param.name)}: ${validator},`);
       }
       lines.push("  },");
     }
@@ -391,11 +476,35 @@ function emitPipe(pipe: PipeModel): string {
     }
   }
 
+  if (pipe.type === "sink") {
+    if (!pipe.sink) {
+      throw new Error(`Sink pipe "${pipe.name}" is missing sink configuration.`);
+    }
+    lines.push("  sink: {");
+    lines.push(`    connection: ${toCamelCase(pipe.sink.connectionName)},`);
+    if (pipe.sink.service === "kafka") {
+      lines.push(`    topic: ${escapeString(pipe.sink.topic)},`);
+      lines.push(`    schedule: ${escapeString(pipe.sink.schedule)},`);
+    } else {
+      lines.push(`    bucketUri: ${escapeString(pipe.sink.bucketUri)},`);
+      lines.push(`    fileTemplate: ${escapeString(pipe.sink.fileTemplate)},`);
+      lines.push(`    schedule: ${escapeString(pipe.sink.schedule)},`);
+      lines.push(`    format: ${escapeString(pipe.sink.format)},`);
+      if (pipe.sink.strategy) {
+        lines.push(`    strategy: ${escapeString(pipe.sink.strategy)},`);
+      }
+      if (pipe.sink.compression) {
+        lines.push(`    compression: ${escapeString(pipe.sink.compression)},`);
+      }
+    }
+    lines.push("  },");
+  }
+
   lines.push("  nodes: [");
   for (const node of pipe.nodes) {
     lines.push("    node({");
     lines.push(`      name: ${escapeString(node.name)},`);
-    if (node.description) {
+    if (node.description !== undefined) {
       lines.push(`      description: ${escapeString(node.description)},`);
     }
     lines.push("      sql: `");
@@ -413,7 +522,7 @@ function emitPipe(pipe: PipeModel): string {
     }
     lines.push("  output: {");
     for (const columnName of endpointOutputColumns) {
-      lines.push(`    ${columnName}: t.string(),`);
+      lines.push(`    ${emitObjectKey(columnName)}: t.string(),`);
     }
     lines.push("  },");
   }
@@ -443,10 +552,8 @@ export function emitMigrationFileContent(resources: ParsedResource[]): string {
     (resource): resource is PipeModel => resource.kind === "pipe"
   );
 
-  const needsColumn = datasources.some((ds) =>
-    ds.columns.some((column) => column.jsonPath !== undefined)
-  );
   const needsParams = pipes.some((pipe) => pipe.params.length > 0);
+  const needsSecret = hasSecretTemplate(resources);
 
   const imports = new Set<string>([
     "defineDatasource",
@@ -455,7 +562,6 @@ export function emitMigrationFileContent(resources: ParsedResource[]): string {
     "defineCopyPipe",
     "node",
     "t",
-    "engine",
   ]);
   if (connections.some((connection) => connection.connectionType === "kafka")) {
     imports.add("defineKafkaConnection");
@@ -463,12 +569,18 @@ export function emitMigrationFileContent(resources: ParsedResource[]): string {
   if (connections.some((connection) => connection.connectionType === "s3")) {
     imports.add("defineS3Connection");
   }
-  if (needsColumn) {
-    imports.add("column");
-  }
   if (needsParams) {
     imports.add("p");
   }
+  if (pipes.some((pipe) => pipe.type === "sink")) {
+    imports.add("defineSinkPipe");
+  }
+  if (datasources.some((datasource) => datasource.engine !== undefined)) {
+    imports.add("engine");
+  }
+  if (needsSecret) {
+    imports.add("secret");
+  }
 
   const orderedImports = [
     "defineKafkaConnection",
@@ -477,10 +589,11 @@ export function emitMigrationFileContent(resources: ParsedResource[]): string {
     "definePipe",
     "defineMaterializedView",
     "defineCopyPipe",
+    "defineSinkPipe",
     "node",
     "t",
     "engine",
-    "column",
+    "secret",
     "p",
   ].filter((name) => imports.has(name));
 

package/src/migrate/parse-connection.ts

@@ -27,6 +27,7 @@ export function parseConnectionFile(
     | undefined;
   let key: string | undefined;
   let secret: string | undefined;
+  let schemaRegistryUrl: string | undefined;
   let sslCaPem: string | undefined;
 
   let region: string | undefined;
@@ -36,7 +37,7 @@ export function parseConnectionFile(
 
   for (const rawLine of lines) {
     const line = rawLine.trim();
-    if (isBlank(line)) {
+    if (isBlank(line) || line.startsWith("#")) {
       continue;
     }
 
@@ -81,6 +82,9 @@ export function parseConnectionFile(
       case "KAFKA_SECRET":
         secret = value;
         break;
+      case "KAFKA_SCHEMA_REGISTRY_URL":
+        schemaRegistryUrl = value;
+        break;
       case "KAFKA_SSL_CA_PEM":
         sslCaPem = value;
         break;
@@ -144,12 +148,21 @@ export function parseConnectionFile(
       saslMechanism,
       key,
       secret,
+      schemaRegistryUrl,
       sslCaPem,
     };
   }
 
   if (connectionType === "s3") {
-    if (bootstrapServers || securityProtocol || saslMechanism || key || secret || sslCaPem) {
+    if (
+      bootstrapServers ||
+      securityProtocol ||
+      saslMechanism ||
+      key ||
+      secret ||
+      schemaRegistryUrl ||
+      sslCaPem
+    ) {
       throw new MigrationParseError(
         resource.filePath,
         "connection",