@tinybirdco/sdk 0.0.69 → 0.0.71

package/src/infer/index.ts

@@ -82,6 +82,37 @@ type OptionalParamKeys<T extends ParamsDefinition> = {
   [K in keyof T]: T[K] extends ParamValidator<unknown, string, false> ? K : never;
 }[keyof T];
 
+/**
+ * Extract the required output keys from an output definition
+ */
+type RequiredOutputKeys<T extends OutputDefinition> = {
+  [K in keyof T]: T[K] extends TypeValidator<unknown, string, infer M>
+    ? M extends { optional: true }
+      ? never
+      : K
+    : K;
+}[keyof T];
+
+/**
+ * Extract the optional output keys from an output definition
+ */
+type OptionalOutputKeys<T extends OutputDefinition> = {
+  [K in keyof T]: T[K] extends TypeValidator<unknown, string, infer M>
+    ? M extends { optional: true }
+      ? K
+      : never
+    : never;
+}[keyof T];
+
+/**
+ * Extract a single output row type from an output definition
+ */
+type InferOutputObject<T extends OutputDefinition> = {
+  [K in RequiredOutputKeys<T>]: InferColumn<T[K]>;
+} & {
+  [K in OptionalOutputKeys<T>]?: InferColumn<T[K]>;
+};
+
 /**
  * Extract the params type from a pipe definition
  *
@@ -131,14 +162,14 @@ export type InferParams<T> = T extends PipeDefinition<infer P, OutputDefinition>
  * ```
  */
 export type InferOutput<T> = T extends PipeDefinition<ParamsDefinition, infer O>
-  ? { [K in keyof O]: InferColumn<O[K]> }[]
+  ? InferOutputObject<O>[]
   : never;
 
 /**
  * Extract a single output row type (without array wrapper)
  */
 export type InferOutputRow<T> = T extends PipeDefinition<ParamsDefinition, infer O>
-  ? { [K in keyof O]: InferColumn<O[K]> }
+  ? InferOutputObject<O>
   : never;
 
 /**

package/src/migrate/emit-ts.ts

@@ -15,6 +15,10 @@ function escapeString(value: string): string {
   return JSON.stringify(value);
 }
 
+function escapeTemplateLiteral(value: string): string {
+  return value.replace(/\\/g, "\\\\").replace(/`/g, "\\`").replace(/\${/g, "\\${");
+}
+
 function emitObjectKey(key: string): string {
   return /^[A-Za-z_$][A-Za-z0-9_$]*$/.test(key) ? key : escapeString(key);
 }
@@ -196,6 +200,7 @@ function engineFunctionName(type: string): string {
     AggregatingMergeTree: "aggregatingMergeTree",
     CollapsingMergeTree: "collapsingMergeTree",
     VersionedCollapsingMergeTree: "versionedCollapsingMergeTree",
+    Null: "null",
   };
   const functionName = map[type];
   if (!functionName) {
@@ -205,6 +210,10 @@ function engineFunctionName(type: string): string {
 }
 
 function emitEngineOptions(engine: DatasourceEngineModel): string {
+  if (engine.type === "Null") {
+    return "engine.null()";
+  }
+
   const options: string[] = [];
 
   if (engine.sortingKey.length === 1) {
@@ -378,7 +387,7 @@ function emitDatasource(ds: DatasourceModel): string {
 
   if (ds.forwardQuery) {
     lines.push("  forwardQuery: `");
-    lines.push(ds.forwardQuery.replace(/`/g, "\\`").replace(/\${/g, "\\${"));
+    lines.push(escapeTemplateLiteral(ds.forwardQuery));
     lines.push("  `,");
   }
 
@@ -561,7 +570,7 @@ function emitPipe(pipe: PipeModel): string {
       lines.push(`      description: ${escapeString(node.description)},`);
     }
     lines.push("      sql: `");
-    lines.push(node.sql.replace(/`/g, "\\`").replace(/\${/g, "\\${"));
+    lines.push(escapeTemplateLiteral(node.sql));
     lines.push("      `,");
     lines.push("    }),");
   }

package/src/migrate/parse-datasource.ts

@@ -540,7 +540,7 @@ export function parseDatasourceFile(resource: ResourceFile): DatasourceModel {
     engineType = "MergeTree";
   }
 
-  if (engineType && sortingKey.length === 0) {
+  if (engineType && engineType !== "Null" && sortingKey.length === 0) {
     throw new MigrationParseError(
       resource.filePath,
       "datasource",

package/src/schema/engines.test.ts

@@ -100,6 +100,13 @@ describe('Engine Configurations', () => {
     });
   });
 
+  describe('Null', () => {
+    it('creates Null config', () => {
+      const config = engine.null();
+      expect(config.type).toBe('Null');
+    });
+  });
+
   describe('getEngineClause', () => {
     it('generates basic MergeTree clause', () => {
       const config = engine.mergeTree({ sortingKey: ['id'] });
@@ -187,6 +194,12 @@ describe('Engine Configurations', () => {
       expect(clause).toContain('ENGINE_SIGN "sign_col"');
       expect(clause).toContain('ENGINE_VERSION "version_col"');
     });
+
+    it('generates Null engine without MergeTree directives', () => {
+      const clause = getEngineClause(engine.null());
+      expect(clause).toBe('ENGINE Null');
+      expect(clause).not.toContain('ENGINE_SORTING_KEY');
+    });
   });
 
   describe('Helper functions', () => {

package/src/schema/engines.ts

@@ -79,10 +79,18 @@ export interface VersionedCollapsingMergeTreeConfig extends BaseMergeTreeConfig
   version: string;
 }
 
+/**
+ * Null engine configuration
+ * Discards inserted rows and returns an empty response when read
+ */
+export interface NullEngineConfig {
+  type: "Null";
+}
+
 /**
  * Union type of all engine configurations
  */
-export type EngineConfig =
+export type MergeTreeEngineConfig =
   | MergeTreeConfig
   | ReplacingMergeTreeConfig
   | SummingMergeTreeConfig
@@ -90,6 +98,8 @@ export type EngineConfig =
   | CollapsingMergeTreeConfig
   | VersionedCollapsingMergeTreeConfig;
 
+export type EngineConfig = MergeTreeEngineConfig | NullEngineConfig;
+
 /**
  * Helper to normalize sorting key to array format
  */
@@ -121,6 +131,9 @@ function normalizeSortingKey(key: string | readonly string[]): readonly string[]
  *   sortingKey: ['date', 'metric_name'],
  *   columns: ['value'],
  * });
+ *
+ * // Null engine for materialized view source tables
+ * engine.null();
  * ```
  */
 export const engine = {
@@ -196,19 +209,27 @@ export const engine = {
     type: "VersionedCollapsingMergeTree",
     ...config,
   }),
+
+  /**
+   * Null - Discards inserted rows and returns no rows when read
+   * Best for: Materialized view source tables that transform and discard raw input
+   */
+  null: (): NullEngineConfig => ({
+    type: "Null",
+  }),
 } as const;
 
 /**
  * Get the sorting key as an array
  */
-export function getSortingKey(config: EngineConfig): readonly string[] {
+export function getSortingKey(config: MergeTreeEngineConfig): readonly string[] {
   return normalizeSortingKey(config.sortingKey);
 }
 
 /**
  * Get the primary key as an array (defaults to sorting key)
  */
-export function getPrimaryKey(config: EngineConfig): readonly string[] {
+export function getPrimaryKey(config: MergeTreeEngineConfig): readonly string[] {
   if (config.primaryKey) {
     return normalizeSortingKey(config.primaryKey);
   }
@@ -219,6 +240,10 @@ export function getPrimaryKey(config: EngineConfig): readonly string[] {
  * Generate the engine clause for a datasource file
  */
 export function getEngineClause(config: EngineConfig): string {
+  if (config.type === "Null") {
+    return "ENGINE Null";
+  }
+
   const parts: string[] = [`ENGINE "${config.type}"`];
 
   if (config.partitionKey) {

package/src/schema/pipe.test.ts

@@ -1,4 +1,4 @@
-import { describe, it, expect } from "vitest";
+import { describe, it, expect, expectTypeOf } from "vitest";
 import {
   definePipe,
   defineSinkPipe,
@@ -19,6 +19,7 @@ import { defineKafkaConnection, defineS3Connection } from "./connection.js";
 import { t } from "./types.js";
 import { p } from "./params.js";
 import { engine } from "./engines.js";
+import type { InferOutputRow } from "../infer/index.js";
 
 describe("Pipe Schema", () => {
   describe("node", () => {
@@ -82,6 +83,30 @@ describe("Pipe Schema", () => {
       expect(pipe.options.description).toBe("A test pipe");
     });
 
+    it("infers optional output fields as optional object properties", () => {
+      const pipe = definePipe("my_pipe", {
+        nodes: [node({ name: "endpoint", sql: "SELECT 1" })],
+        output: {
+          id: t.uint64(),
+          subtitle: t.string().optional(),
+          metadata: t.string().nullable().optional(),
+        },
+        endpoint: true,
+      });
+
+      type Row = InferOutputRow<typeof pipe>;
+
+      const requiredOnly: Row = { id: 1 };
+      const withOptionals: Row = { id: 1, subtitle: "hello", metadata: null };
+
+      expectTypeOf<Row["id"]>().toEqualTypeOf<number>();
+      expectTypeOf<Row["subtitle"]>().toEqualTypeOf<string | undefined>();
+      expectTypeOf<Row["metadata"]>().toEqualTypeOf<string | null | undefined>();
+      expect(requiredOnly).toEqual({ id: 1 });
+      expect(withOptionals).toEqual({ id: 1, subtitle: "hello", metadata: null });
+      expect(pipe._output?.subtitle?._modifiers.optional).toBe(true);
+    });
+
     it("throws error for invalid pipe name", () => {
       expect(() =>
         definePipe("123invalid", {

package/src/schema/types.test.ts

@@ -64,6 +64,21 @@ describe("Type Validators (t.*)", () => {
     });
   });
 
+  describe("Optional modifier", () => {
+    it("marks field as optional without changing the Tinybird type", () => {
+      const type = t.string().optional();
+      expect(type._tinybirdType).toBe("String");
+      expect(type._modifiers.optional).toBe(true);
+    });
+
+    it("can be combined with nullable fields", () => {
+      const type = t.string().nullable().optional();
+      expect(type._tinybirdType).toBe("Nullable(String)");
+      expect(type._modifiers.nullable).toBe(true);
+      expect(type._modifiers.optional).toBe(true);
+    });
+  });
+
   describe("LowCardinality modifier", () => {
     it("wraps type in LowCardinality", () => {
       const type = t.string().lowCardinality();

package/src/schema/types.ts

@@ -24,6 +24,12 @@ export interface TypeValidator<
   /** Metadata about modifiers applied */
   readonly _modifiers: TModifiers;
 
+  /** Mark this output field as optional (it may be absent from endpoint responses) */
+  optional(): TypeValidator<
+    TType,
+    TTinybirdType,
+    TModifiers & { optional: true }
+  >;
   /** Make this column nullable */
   nullable(): TypeValidator<
     TType | null,
@@ -63,6 +69,7 @@ export interface TypeValidator<
 }
 
 export interface TypeModifiers {
+  optional?: boolean;
   nullable?: boolean;
   lowCardinality?: boolean;
   hasDefault?: boolean;
@@ -94,6 +101,17 @@ function createValidator<TType, TTinybirdType extends string>(
     tinybirdType,
     modifiers,
 
+    optional() {
+      return createValidator<TType, TTinybirdType>(tinybirdType, {
+        ...modifiers,
+        optional: true,
+      }) as TypeValidator<
+        TType,
+        TTinybirdType,
+        TypeModifiers & { optional: true }
+      >;
+    },
+
     nullable() {
       // If already has LowCardinality, we need to move Nullable inside
       // ClickHouse requires: LowCardinality(Nullable(X)), not Nullable(LowCardinality(X))