@superblocksteam/sdk-api 0.0.4

package/src/integrations/kafka/client.ts

@@ -0,0 +1,215 @@
+/**
+ * Kafka client implementation.
+ *
+ * Constructs request objects matching the expected backend format.
+ */
+
+import type { z } from "zod";
+import type { IntegrationConfig, IntegrationClientImpl } from "../types.js";
+import type { QueryExecutor, TraceMetadata } from "../registry.js";
+import { RestApiValidationError } from "../../errors.js";
+import { IntegrationError } from "../../runtime/errors.js";
+import type {
+  KafkaClient,
+  KafkaConsumeParams,
+  KafkaProduceParams,
+} from "./types.js";
+
+/**
+ * Consume request structure.
+ */
+interface KafkaConsumeRequest {
+  operation: "OPERATION_CONSUME";
+  consume: {
+    topic: string;
+    from: string;
+    groupId?: string;
+    clientId?: string;
+    readUncommitted?: boolean;
+    seek?: {
+      topic?: string;
+      offset?: string;
+      partition?: number;
+    };
+  };
+}
+
+/**
+ * Produce request structure.
+ */
+interface KafkaProduceRequest {
+  operation: "OPERATION_PRODUCE";
+  produce: {
+    messages: string;
+    acks?: string;
+    clientId?: string;
+    timeout?: number;
+    compression?: string;
+    transactionId?: string;
+    autoCreateTopic?: boolean;
+    idempotent?: boolean;
+    transaction?: boolean;
+  };
+}
+
+/**
+ * Kafka plugin request - discriminated union on operation.
+ */
+type KafkaRequest = KafkaConsumeRequest | KafkaProduceRequest;
+
+/**
+ * Operation string values expected by the Kafka plugin.
+ */
+const OPERATION = {
+  CONSUME: "OPERATION_CONSUME",
+  PRODUCE: "OPERATION_PRODUCE",
+} as const;
+
+/**
+ * Consume from string values.
+ */
+const CONSUME_FROM = {
+  BEGINNING: "FROM_BEGINNING",
+  LATEST: "FROM_LATEST",
+  SEEK: "FROM_SEEK",
+} as const;
+
+/**
+ * Acks string values.
+ */
+const ACKS = {
+  NONE: "ACKS_NONE",
+  LEADER: "ACKS_LEADER",
+  ALL: "ACKS_ALL",
+} as const;
+
+/**
+ * Compression string values.
+ */
+const COMPRESSION = {
+  GZIP: "COMPRESSION_GZIP",
+  SNAPPY: "COMPRESSION_SNAPPY",
+  LZ4: "COMPRESSION_LZ4",
+  ZSTD: "COMPRESSION_ZSTD",
+} as const;
+
+/**
+ * Internal implementation of KafkaClient.
+ */
+export class KafkaClientImpl implements KafkaClient, IntegrationClientImpl {
+  readonly config: IntegrationConfig;
+  private readonly executeQuery: QueryExecutor;
+
+  constructor(config: IntegrationConfig, executeQuery: QueryExecutor) {
+    this.config = config;
+    this.executeQuery = executeQuery;
+  }
+
+  get name(): string {
+    return this.config.name;
+  }
+
+  get pluginId(): string {
+    return this.config.pluginId;
+  }
+
+  private buildConsumeRequest(params: KafkaConsumeParams): KafkaConsumeRequest {
+    const request: KafkaConsumeRequest = {
+      operation: OPERATION.CONSUME,
+      consume: {
+        topic: params.topic,
+        from: params.from ? CONSUME_FROM[params.from] : CONSUME_FROM.LATEST,
+        groupId: params.groupId,
+        clientId: params.clientId,
+        readUncommitted: params.readUncommitted ?? false,
+      },
+    };
+
+    if (params.seek && request.consume) {
+      request.consume.seek = {
+        topic: params.seek.topic,
+        offset: String(params.seek.offset),
+        partition: params.seek.partition,
+      };
+    }
+
+    return request;
+  }
+
+  private buildProduceRequest(params: KafkaProduceParams): KafkaProduceRequest {
+    const request: KafkaProduceRequest = {
+      operation: OPERATION.PRODUCE,
+      produce: {
+        messages: params.messages,
+        acks: params.acks ? ACKS[params.acks] : ACKS.ALL,
+        clientId: params.clientId,
+        timeout: params.timeout,
+        compression: params.compression
+          ? COMPRESSION[params.compression]
+          : undefined,
+        transactionId: params.transactionId,
+        autoCreateTopic: params.autoCreateTopic ?? false,
+        idempotent: params.idempotent ?? false,
+        transaction: params.transaction ?? false,
+      },
+    };
+
+    return request;
+  }
+
+  private async executeAndValidate<T>(
+    request: KafkaRequest,
+    schema: z.ZodSchema<T>,
+    operation: string,
+    metadata?: TraceMetadata,
+  ): Promise<T> {
+    try {
+      const result = await this.executeQuery(
+        request as unknown as Record<string, unknown>,
+        undefined,
+        metadata,
+      );
+
+      const parseResult = schema.safeParse(result);
+
+      if (!parseResult.success) {
+        throw new RestApiValidationError(
+          `Result validation failed: ${parseResult.error.message}`,
+          {
+            zodError: parseResult.error,
+            data: result,
+          },
+        );
+      }
+
+      return parseResult.data;
+    } catch (error) {
+      if (
+        error instanceof RestApiValidationError ||
+        error instanceof IntegrationError
+      ) {
+        throw error;
+      }
+
+      throw new IntegrationError(this.config.name, operation, error);
+    }
+  }
+
+  async consume<T>(
+    schema: z.ZodSchema<T>,
+    params: KafkaConsumeParams,
+    metadata?: TraceMetadata,
+  ): Promise<T> {
+    const request = this.buildConsumeRequest(params);
+    return this.executeAndValidate(request, schema, "consume", metadata);
+  }
+
+  async produce<T>(
+    schema: z.ZodSchema<T>,
+    params: KafkaProduceParams,
+    metadata?: TraceMetadata,
+  ): Promise<T> {
+    const request = this.buildProduceRequest(params);
+    return this.executeAndValidate(request, schema, "produce", metadata);
+  }
+}

package/src/integrations/kafka/index.ts

@@ -0,0 +1,14 @@
+/**
+ * Kafka integration client.
+ *
+ * @module
+ */
+
+export type {
+  KafkaClient,
+  KafkaOperation,
+  KafkaSeek,
+  KafkaConsumeParams,
+  KafkaProduceParams,
+} from "./types.js";
+export { KafkaClientImpl } from "./client.js";

package/src/integrations/kafka/types.ts

@@ -0,0 +1,127 @@
+/**
+ * Kafka client types.
+ */
+
+import type { z } from "zod";
+import type { BaseIntegrationClient } from "../../types.js";
+import type { TraceMetadata } from "../registry.js";
+
+/**
+ * Kafka operation types.
+ */
+export type KafkaOperation = "CONSUME" | "PRODUCE";
+
+/**
+ * Seek position for consuming from a specific offset.
+ */
+export interface KafkaSeek {
+  /** Topic name */
+  topic: string;
+  /** Offset to start from */
+  offset: number;
+  /** Partition to consume from */
+  partition: number;
+}
+
+/**
+ * Parameters for Kafka consume operation.
+ */
+export interface KafkaConsumeParams {
+  /** Topic to consume from */
+  topic: string;
+  /** Where to start consuming from */
+  from?: "BEGINNING" | "LATEST" | "SEEK";
+  /** Consumer group ID */
+  groupId?: string;
+  /** Client ID */
+  clientId?: string;
+  /** Seek position for SEEK mode */
+  seek?: KafkaSeek;
+  /** Whether to read uncommitted messages */
+  readUncommitted?: boolean;
+}
+
+/**
+ * Parameters for Kafka produce operation.
+ */
+export interface KafkaProduceParams {
+  /** Messages to produce (JSON string) */
+  messages: string;
+  /** Acknowledgment level */
+  acks?: "NONE" | "LEADER" | "ALL";
+  /** Client ID */
+  clientId?: string;
+  /** Timeout in milliseconds */
+  timeout?: number;
+  /** Compression type */
+  compression?: "GZIP" | "SNAPPY" | "LZ4" | "ZSTD";
+  /** Transaction ID for transactional produce */
+  transactionId?: string;
+  /** Whether to auto-create topic */
+  autoCreateTopic?: boolean;
+  /** Whether to enable idempotent delivery */
+  idempotent?: boolean;
+  /** Whether to use transactions */
+  transaction?: boolean;
+}
+
+/**
+ * Kafka client for message streaming operations.
+ *
+ * Provides consume() and produce() methods for Kafka operations.
+ *
+ * @example
+ * ```typescript
+ * // Declare in api(): integrations: { kafka: kafka(INTEGRATION_ID) }
+ * // In run(), access via ctx.integrations.kafka
+ *
+ * // Consume messages
+ * const MessageSchema = z.array(z.object({
+ *   topic: z.string(),
+ *   partition: z.number(),
+ *   offset: z.number(),
+ *   value: z.unknown(),
+ * }));
+ *
+ * const messages = await kafka.consume(MessageSchema, {
+ *   topic: 'my-topic',
+ *   from: 'LATEST',
+ *   groupId: 'my-consumer-group',
+ * });
+ *
+ * // Produce messages
+ * const result = await kafka.produce(z.any(), {
+ *   messages: JSON.stringify([
+ *     { topic: 'my-topic', value: { key: 'value' } }
+ *   ]),
+ * });
+ * ```
+ */
+export interface KafkaClient extends BaseIntegrationClient {
+  /**
+   * Consume messages from Kafka.
+   *
+   * @param schema - Zod schema for validating the result
+   * @param params - Consume parameters
+   * @returns The validated result
+   */
+  consume<T>(
+    schema: z.ZodSchema<T>,
+    params: KafkaConsumeParams,
+    metadata?: TraceMetadata,
+  ): Promise<T>;
+
+  /**
+   * Produce messages to Kafka.
+   *
+   * @param schema - Zod schema for validating the result
+   * @param params - Produce parameters
+   * @param metadata - Optional trace metadata for diagnostics
+   * @returns The validated result
+   */
+  produce<T>(
+    schema: z.ZodSchema<T>,
+    params: KafkaProduceParams,
+    metadata?: TraceMetadata,
+  ): Promise<T>;
+}

package/src/integrations/kinesis/README.md

@@ -0,0 +1,153 @@
+# Kinesis Client
+
+Write records to and read records from Amazon Kinesis Data Streams.
+
+## Methods
+
+| Method                          | Description                               |
+| ------------------------------- | ----------------------------------------- |
+| `putRecord(params, metadata?)`  | Write a single record to a Kinesis stream |
+| `getRecords(params, metadata?)` | Read records from a Kinesis stream shard  |
+
+## Usage
+
+### Put Record
+
+```typescript
+import { api, z, kinesis } from "@superblocksteam/sdk-api";
+
+// Integration ID from the integrations panel
+const PROD_KINESIS = "a1b2c3d4-5678-90ab-cdef-kinesis00001";
+
+export default api({
+  integrations: {
+    kinesis: kinesis(PROD_KINESIS),
+  },
+  name: "KinesisExample",
+  input: z.object({
+    userId: z.string(),
+    event: z.string(),
+  }),
+  output: z.object({
+    success: z.boolean(),
+  }),
+  async run(ctx, { userId, event }) {
+    await kinesis.putRecord({
+      data: JSON.stringify({ event, userId, timestamp: Date.now() }),
+      partitionKey: userId,
+      streamName: "user-events",
+    });
+
+    return { success: true };
+  },
+});
+```
+
+### Put Record with Stream ARN
+
+```typescript
+await kinesis.putRecord({
+  data: JSON.stringify({ event: "signup" }),
+  partitionKey: "user-123",
+  streamIdentifierType: "STREAM_ARN",
+  streamArn: "arn:aws:kinesis:us-east-1:123456789:stream/user-events",
+});
+```
+
+### Get Records
+
+```typescript
+const records = await kinesis.getRecords({
+  shardId: "shardId-000000000000",
+  shardIteratorType: "LATEST",
+  streamName: "user-events",
+  limit: 10,
+});
+```
+
+### Get Records from Specific Sequence Number
+
+```typescript
+const records = await kinesis.getRecords({
+  shardId: "shardId-000000000000",
+  shardIteratorType: "AFTER_SEQUENCE_NUMBER",
+  startingSequenceNumber:
+    "49590338271490256608559692538361571095921575989136588898",
+  streamName: "user-events",
+});
+```
+
+### Get Records at Timestamp
+
+```typescript
+const records = await kinesis.getRecords({
+  shardId: "shardId-000000000000",
+  shardIteratorType: "AT_TIMESTAMP",
+  timestamp: "2024-01-15T00:00:00Z",
+  streamName: "user-events",
+});
+```
+
+## Parameters
+
+### KinesisPutParams
+
+| Parameter              | Type                            | Required | Description                                  |
+| ---------------------- | ------------------------------- | -------- | -------------------------------------------- |
+| `data`                 | `string`                        | Yes      | The data payload (JSON string or raw string) |
+| `partitionKey`         | `string`                        | Yes      | The partition key for the record             |
+| `streamIdentifierType` | `'STREAM_NAME' \| 'STREAM_ARN'` | No       | How to identify the stream                   |
+| `streamName`           | `string`                        | No       | Stream name (when using STREAM_NAME)         |
+| `streamArn`            | `string`                        | No       | Stream ARN (when using STREAM_ARN)           |
+
+### KinesisGetParams
+
+| Parameter                | Type                            | Required | Description                                    |
+| ------------------------ | ------------------------------- | -------- | ---------------------------------------------- |
+| `shardId`                | `string`                        | Yes      | The shard ID to read from                      |
+| `shardIteratorType`      | `ShardIteratorType`             | Yes      | How to position the shard iterator             |
+| `limit`                  | `number`                        | No       | Maximum number of records to return            |
+| `pollingCooldownMs`      | `number`                        | No       | Polling cooldown in milliseconds               |
+| `startingSequenceNumber` | `string`                        | No       | Sequence number (for AT/AFTER_SEQUENCE_NUMBER) |
+| `timestamp`              | `string`                        | No       | ISO timestamp (for AT_TIMESTAMP)               |
+| `streamIdentifierType`   | `'STREAM_NAME' \| 'STREAM_ARN'` | No       | How to identify the stream                     |
+| `streamName`             | `string`                        | No       | Stream name (when using STREAM_NAME)           |
+| `streamArn`              | `string`                        | No       | Stream ARN (when using STREAM_ARN)             |
+
+### Shard Iterator Types
+
+| Type                    | Description                               |
+| ----------------------- | ----------------------------------------- |
+| `AT_SEQUENCE_NUMBER`    | Start at the specified sequence number    |
+| `AFTER_SEQUENCE_NUMBER` | Start after the specified sequence number |
+| `AT_TIMESTAMP`          | Start at the specified timestamp          |
+| `TRIM_HORIZON`          | Start at the oldest record in the shard   |
+| `LATEST`                | Start at the most recent record           |
+
+## Trace Metadata
+
+All methods accept an optional `metadata` parameter as the last argument for diagnostics labeling. See the [root SDK README](../../../README.md#trace-metadata) for details.
+
+## Error Handling
+
+```typescript
+import { IntegrationError } from "@superblocksteam/sdk-api";
+
+try {
+  await kinesis.putRecord({
+    data: JSON.stringify({ event: "test" }),
+    partitionKey: "key-1",
+    streamName: "my-stream",
+  });
+} catch (error) {
+  if (error instanceof IntegrationError) {
+    console.error("Kinesis error:", error.message);
+  }
+}
+```
+
+## API Reference
+
+- [Amazon Kinesis Data Streams](https://docs.aws.amazon.com/streams/latest/dev/introduction.html)
+- [PutRecord](https://docs.aws.amazon.com/kinesis/latest/APIReference/API_PutRecord.html)
+- [GetRecords](https://docs.aws.amazon.com/kinesis/latest/APIReference/API_GetRecords.html)

package/src/integrations/kinesis/client.ts

@@ -0,0 +1,146 @@
+/**
+ * Kinesis client implementation.
+ *
+ * Uses the native Kinesis plugin type from @superblocksteam/types.
+ * Supports PUT (write records) and GET (read records) operations.
+ */
+
+import type { IntegrationConfig, IntegrationClientImpl } from "../types.js";
+import type { QueryExecutor, TraceMetadata } from "../registry.js";
+import { IntegrationError } from "../../runtime/errors.js";
+import type {
+  KinesisClient,
+  KinesisPutParams,
+  KinesisGetParams,
+} from "./types.js";
+
+/**
+ * Kinesis operation type enum values matching the proto.
+ */
+const OPERATION_TYPE = {
+  PUT: "OPERATION_TYPE_PUT",
+  GET: "OPERATION_TYPE_GET",
+} as const;
+
+/**
+ * Internal implementation of KinesisClient.
+ *
+ * Sends requests matching the Kinesis plugin proto structure:
+ * - `operationType`: PUT or GET
+ * - `put`: PUT operation parameters
+ * - `get`: GET operation parameters
+ */
+export class KinesisClientImpl implements KinesisClient, IntegrationClientImpl {
+  readonly config: IntegrationConfig;
+  private readonly executeQuery: QueryExecutor;
+
+  constructor(config: IntegrationConfig, executeQuery: QueryExecutor) {
+    this.config = config;
+    this.executeQuery = executeQuery;
+  }
+
+  get name(): string {
+    return this.config.name;
+  }
+
+  get pluginId(): string {
+    return this.config.pluginId;
+  }
+
+  /**
+   * Execute a request and wrap errors.
+   */
+  private async exec(
+    request: Record<string, unknown>,
+    operation: string,
+    metadata?: TraceMetadata,
+  ): Promise<unknown> {
+    try {
+      return await this.executeQuery(request, undefined, metadata);
+    } catch (error) {
+      if (error instanceof IntegrationError) {
+        throw error;
+      }
+
+      throw new IntegrationError(this.config.name, operation, error);
+    }
+  }
+
+  async putRecord(
+    params: KinesisPutParams,
+    metadata?: TraceMetadata,
+  ): Promise<unknown> {
+    const put: Record<string, unknown> = {
+      data: params.data,
+      partitionKey: params.partitionKey,
+    };
+
+    if (params.streamIdentifierType) {
+      put.streamIdentifierType = params.streamIdentifierType;
+    }
+
+    if (params.streamName) {
+      put.streamName = params.streamName;
+    }
+
+    if (params.streamArn) {
+      put.streamArn = params.streamArn;
+    }
+
+    return this.exec(
+      {
+        operationType: OPERATION_TYPE.PUT,
+        put,
+      },
+      "putRecord",
+      metadata,
+    );
+  }
+
+  async getRecords(
+    params: KinesisGetParams,
+    metadata?: TraceMetadata,
+  ): Promise<unknown> {
+    const get: Record<string, unknown> = {
+      shardId: params.shardId,
+      shardIteratorType: params.shardIteratorType,
+    };
+
+    if (params.limit !== undefined) {
+      get.limit = params.limit;
+    }
+
+    if (params.pollingCooldownMs !== undefined) {
+      get.pollingCooldownMs = params.pollingCooldownMs;
+    }
+
+    if (params.startingSequenceNumber) {
+      get.startingSequenceNumber = params.startingSequenceNumber;
+    }
+
+    if (params.timestamp) {
+      get.timestamp = params.timestamp;
+    }
+
+    if (params.streamIdentifierType) {
+      get.streamIdentifierType = params.streamIdentifierType;
+    }
+
+    if (params.streamName) {
+      get.streamName = params.streamName;
+    }
+
+    if (params.streamArn) {
+      get.streamArn = params.streamArn;
+    }
+
+    return this.exec(
+      {
+        operationType: OPERATION_TYPE.GET,
+        get,
+      },
+      "getRecords",
+      metadata,
+    );
+  }
+}

package/src/integrations/kinesis/index.ts

@@ -0,0 +1,14 @@
+/**
+ * Kinesis integration client.
+ *
+ * @module
+ */
+
+export type {
+  KinesisClient,
+  KinesisPutParams,
+  KinesisGetParams,
+  KinesisStreamIdentifier,
+  KinesisShardIteratorType,
+} from "./types.js";
+export { KinesisClientImpl } from "./client.js";