@eventualize/dynamodb-storage-adapter 1.0.0

package/models/events-table-schema.json

@@ -0,0 +1,50 @@
+{
+    "TableName": "events",
+    "AttributeDefinitions": [
+        {
+            "AttributeName": "stream_address",
+            "AttributeType": "S"
+        },
+        {
+            "AttributeName": "offset",
+            "AttributeType": "N"
+        },
+        {
+            "AttributeName": "event_type",
+            "AttributeType": "S"
+        },
+        {
+            "AttributeName": "captured_at",
+            "AttributeType": "S"
+        }
+    ],
+    "KeySchema": [
+        {
+            "AttributeName": "stream_address",
+            "KeyType": "HASH"
+        },
+        {
+            "AttributeName": "offset",
+            "KeyType": "RANGE"
+        }
+    ],
+    "BillingMode": "PAY_PER_REQUEST",
+    "GlobalSecondaryIndexes": [
+        {
+            "IndexName": "event_type__captured_at",
+            "KeySchema": [
+                {
+                    "AttributeName": "event_type",
+                    "KeyType": "HASH"
+                },
+                {
+                    "AttributeName": "captured_at",
+                    "KeyType": "RANGE"
+                }
+            ],
+            "Projection": {
+                "ProjectionType": "ALL"
+            }
+        }
+    ]
+}

package/models/messages-table-schema.json

@@ -0,0 +1,24 @@
+{
+    "TableName": "messages",
+    "AttributeDefinitions": [
+        {
+            "AttributeName": "message_addres",
+            "AttributeType": "S"
+        },
+        {
+            "AttributeName": "captured_at",
+            "AttributeType": "S"
+        }
+    ],
+    "KeySchema": [
+        {
+            "AttributeName": "message_addres",
+            "KeyType": "HASH"
+        },
+        {
+            "AttributeName": "captured_at",
+            "KeyType": "RANGE"
+        }
+    ],
+    "BillingMode": "PAY_PER_REQUEST"
+}

package/models/snapshots-table-schema.json

@@ -0,0 +1,24 @@
+{
+    "TableName": "snapshots",
+    "AttributeDefinitions": [
+        {
+            "AttributeName": "view_address",
+            "AttributeType": "S"
+        },
+        {
+            "AttributeName": "offset",
+            "AttributeType": "N"
+        }
+    ],
+    "KeySchema": [
+        {
+            "AttributeName": "view_address",
+            "KeyType": "HASH"
+        },
+        {
+            "AttributeName": "offset",
+            "KeyType": "RANGE"
+        }
+    ],
+    "BillingMode": "PAY_PER_REQUEST"
+}

package/package.json

@@ -0,0 +1,29 @@
+{
+  "name": "@eventualize/dynamodb-storage-adapter",
+  "version": "1.0.0",
+  "description": "",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    },
+    "./*": "./dist/*.js"
+  },
+  "scripts": {
+    "build": "tsc --build",
+    "clean": "rimraf dist",
+    "rebuild": "npm run clean && npm run build",
+    "test": "echo not tests implemented yet."
+  },
+  "license": "ISC",
+  "type": "module",
+  "devDependencies": {
+    "@types/node": "^24.10.1"
+  },
+  "dependencies": {
+    "@aws-sdk/client-dynamodb": "^3.948.0",
+    "@aws-sdk/util-dynamodb": "^3.948.0"
+  }
+}

package/src/DynamoDbClient.ts

@@ -0,0 +1,32 @@
+// src/dynamo-client.ts
+import { DynamoDBClient, DynamoDBClientConfig, ListTablesCommand } from "@aws-sdk/client-dynamodb";
+
+export async function listTables(client: DynamoDBClient) {
+    try {
+        const command = new ListTablesCommand({});
+        const response = await client.send(command);
+        console.log("Tables in DynamoDB Local:", response.TableNames);
+        return response.TableNames;
+    } catch (error) {
+        console.error("Error listing tables:", error);
+        throw error;
+    }
+}
+
+export function createDynamoDBClient(): DynamoDBClient {
+    const endpoint = process.env.DYNAMODB_URL;
+    const accessKeyId = process.env.AWS_ACCESS_KEY_ID;
+    const secretAccessKey = process.env.AWS_SECRET_ACCESS_KEY;
+    const region = process.env.AWS_REGION || 'us-east-1';
+
+    if (!endpoint || !accessKeyId || !secretAccessKey) {
+        const envVars = { endpoint, accessKeyId, secretAccessKey };
+        throw new Error("AWS credentials are not set in environment variables: " + JSON.stringify(envVars));
+    }
+
+    const config: DynamoDBClientConfig = {};
+    config.endpoint = endpoint;
+    config.credentials = { accessKeyId, secretAccessKey };
+    config.region = region;
+    return new DynamoDBClient(config);
+}

package/src/EvDBDynamoDBAdmin.ts

@@ -0,0 +1,115 @@
+import { AttributeValue, BatchWriteItemCommand, DynamoDBClient, ScanCommand, WriteRequest } from "@aws-sdk/client-dynamodb";
+import IEvDbStorageAdmin from "@eventualize/types/IEvDbStorageAdmin";
+import { createDynamoDBClient } from "./DynamoDbClient.js";
+
+export default class EvDbDynamoDbAdmin implements IEvDbStorageAdmin {
+    private dynamoDbClient: DynamoDBClient
+    constructor() {
+        this.dynamoDbClient = createDynamoDBClient();
+    };
+
+    // Helper function to extract keys in raw DynamoDB format
+    private extractKeys(tableName: string, items: Record<string, AttributeValue>[]): WriteRequest[] {
+        return items.map(item => {
+            const deleteRequest: WriteRequest = {
+                DeleteRequest: {
+                    Key: {} // Must build the Key object manually
+                }
+            };
+
+            if (tableName === "events") {
+                deleteRequest.DeleteRequest!.Key!["stream_address"] = item["stream_address"];
+                deleteRequest.DeleteRequest!.Key!["offset"] = item["offset"];
+            } else if (tableName === "snapshots") {
+                deleteRequest.DeleteRequest!.Key!["view_address"] = item["view_address"];
+                deleteRequest.DeleteRequest!.Key!["offset"] = item["offset"];
+            } else if (tableName === "messages") {
+                // Assuming the schema typo was fixed to message_address
+                deleteRequest.DeleteRequest!.Key!["message_address"] = item["message_address"];
+                deleteRequest.DeleteRequest!.Key!["captured_at"] = item["captured_at"];
+            }
+
+            return deleteRequest;
+        });
+    }
+
+    /**
+    * Scans a table for all items and deletes them in batches of 25.
+    * @param tableName The name of the table to clear.
+    */
+    private async clearTableItems(tableName: string): Promise<void> {
+        let items;
+        let exclusiveStartKey: Record<string, AttributeValue> | undefined = undefined;
+        const BATCH_SIZE = 25;
+
+        do {
+            // Define ProjectionExpression and ExpressionAttributeNames dynamically
+            let projectionExpression: string;
+            let expressionAttributeNames: Record<string, string> | undefined = undefined;
+
+            if (tableName === "events") {
+                projectionExpression = "stream_address, #o";
+                expressionAttributeNames = { "#o": "offset" };
+            } else if (tableName === "snapshots") {
+                projectionExpression = "view_address, #o";
+                expressionAttributeNames = { "#o": "offset" };
+            } else { // "messages" table
+                projectionExpression = "message_address, captured_at";
+                // No offset in the projection for messages table, so no #o alias is needed
+            }
+
+            const scanCommand: ScanCommand = new ScanCommand({
+                TableName: tableName,
+                ProjectionExpression: projectionExpression,
+                ExpressionAttributeNames: expressionAttributeNames, // Pass the dynamic object
+
+                Limit: BATCH_SIZE,
+                ExclusiveStartKey: exclusiveStartKey,
+            });
+
+            // ... (rest of the function for scanning and batch writing remains the same) ...
+            const scanResult = await this.dynamoDbClient.send(scanCommand);
+            items = scanResult.Items;
+
+            if (items && items.length > 0) {
+                const deleteRequests = this.extractKeys(tableName, items);
+
+                const batchWriteCommand = new BatchWriteItemCommand({
+                    RequestItems: {
+                        [tableName]: deleteRequests
+                    }
+                });
+                await this.dynamoDbClient.send(batchWriteCommand);
+                console.log(`Deleted ${items.length} items from ${tableName}.`);
+            }
+
+            exclusiveStartKey = scanResult.LastEvaluatedKey;
+
+        } while (exclusiveStartKey);
+
+        console.log(`Finished item deletion for table: ${tableName}`);
+    }
+
+    createEnvironmentAsync(): Promise<void> {
+        throw new Error("Method not implemented.");
+    }
+    destroyEnvironmentAsync(): Promise<void> {
+        throw new Error("Method not implemented.");
+    }
+    public async clearEnvironmentAsync(): Promise<void> {
+        await this.clearTableItems("events");
+        await this.clearTableItems("snapshots");
+        await this.clearTableItems("messages");
+    }
+    dispose?(): void {
+        throw new Error("Method not implemented.");
+    }
+    disposeAsync(): Promise<void> {
+        throw new Error("Method not implemented.");
+    }
+
+    public async close(): Promise<void> {
+        return;
+    }
+
+}

package/src/EvDbDynamoDbStorageAdapter.ts

@@ -0,0 +1,250 @@
+import { unmarshall } from "@aws-sdk/util-dynamodb";
+
+import { IEvDbPayloadData } from '@eventualize/types/IEvDbEventPayload';
+import IEvDbEventMetadata from '@eventualize/types/IEvDbEventMetadata';
+import EvDbStreamCursor from '@eventualize/types/EvDbStreamCursor';
+import EvDbMessage from '@eventualize/types/EvDbMessage';
+import IEvDbStorageSnapshotAdapter from '@eventualize/types/IEvDbStorageSnapshotAdapter';
+import IEvDbStorageStreamAdapter from '@eventualize/types/IEvDbStorageStreamAdapter';
+import EvDbStreamAddress from '@eventualize/types/EvDbStreamAddress';
+import EvDbViewAddress from '@eventualize/types/EvDbViewAddress';
+import { EvDbStoredSnapshotResultRaw } from '@eventualize/types/EvDbStoredSnapshotResult';
+import { EvDbStoredSnapshotData } from '@eventualize/types/EvDbStoredSnapshotData';
+import EvDbEvent from '@eventualize/types/EvDbEvent';
+import StreamStoreAffected from '@eventualize/types/StreamStoreAffected';
+import EvDbContinuousFetchOptions from '@eventualize/types/EvDbContinuousFetchOptions';
+import EvDbMessageFilter from '@eventualize/types/EvDbMessageFilter';
+import { EvDbShardName } from '@eventualize/types/primitiveTypes';
+
+
+import { createDynamoDBClient, listTables } from './DynamoDbClient.js';
+import QueryProvider, { deserializeStreamAddress, EventRecord, MessageRecord } from './EvDbDynamoDbStorageAdapterQueries.js'
+import { DynamoDBClient, TransactionCanceledException, TransactWriteItemsCommand } from '@aws-sdk/client-dynamodb';
+
+// Type definitions for records
+export interface EvDbEventRecord extends IEvDbEventMetadata {
+    id: string;
+    payload: IEvDbPayloadData;
+}
+
+export interface EvDbSnapshotRecord {
+    id: string;
+    streamType: string;
+    streamId: string;
+    viewName: string;
+    offset: bigint;
+    state: IEvDbPayloadData;
+}
+
+export interface IEvDbOutboxTransformer {
+    transform(message: EvDbMessage): EvDbMessage;
+}
+
+export interface EvDbStorageContext {
+    schema?: string;
+    shortId: string;
+    id: string;
+}
+
+const serializePayload = (payload: IEvDbPayloadData) => Buffer.from(JSON.stringify(payload), 'utf-8');
+const deserializePayload = (payload: any): IEvDbPayloadData => {
+    if (!!payload && typeof payload == 'object') {
+        return payload;
+    }
+    return {};
+}
+
+/**
+ * Prisma-based storage adapter for EvDb
+ * Replaces SQL Server-specific adapter with database-agnostic Prisma implementation
+ */
+export default class EvDbDynamoDbStorageAdapter implements IEvDbStorageSnapshotAdapter, IEvDbStorageStreamAdapter {
+    constructor(private dynamoDbClient: DynamoDBClient = createDynamoDBClient()) {
+    }
+    getFromOutbox(filter: EvDbMessageFilter, options?: EvDbContinuousFetchOptions | null): Promise<AsyncIterable<EvDbMessage>> {
+        throw new Error('Method not implemented.');
+    }
+    getFromOutboxAsync(shard: EvDbShardName, filter: EvDbMessageFilter, options?: EvDbContinuousFetchOptions | null, cancellation?: AbortSignal): AsyncIterable<EvDbMessage> {
+        throw new Error('Method not implemented.');
+    }
+    getRecordsFromOutboxAsync(filter: EvDbMessageFilter, options?: EvDbContinuousFetchOptions | null, cancellation?: AbortSignal): AsyncIterable<EvDbMessage>;
+    getRecordsFromOutboxAsync(shard: EvDbShardName, filter: EvDbMessageFilter, options?: EvDbContinuousFetchOptions | null, cancellation?: AbortSignal): AsyncIterable<EvDbMessage>;
+    getRecordsFromOutboxAsync(shard: unknown, filter?: unknown, options?: unknown, cancellation?: unknown): AsyncIterable<EvDbMessage> {
+        throw new Error('Method not implemented.');
+    }
+    subscribeToMessageAsync(handler: (message: EvDbMessage) => Promise<void>, filter: EvDbMessageFilter, options?: EvDbContinuousFetchOptions | null): Promise<void>;
+    subscribeToMessageAsync(handler: (message: EvDbMessage) => Promise<void>, shard: EvDbShardName, filter: EvDbMessageFilter, options?: EvDbContinuousFetchOptions | null): Promise<void>;
+    subscribeToMessageAsync(handler: unknown, shard: unknown, filter?: unknown, options?: unknown): Promise<void> {
+        throw new Error('Method not implemented.');
+    }
+
+    /**
+     * Store stream events in a transaction
+     */
+    async storeStreamAsync(
+        events: ReadonlyArray<EvDbEvent>,
+        messages: ReadonlyArray<EvDbMessage>,
+    ): Promise<StreamStoreAffected> {
+        try {
+            await listTables(this.dynamoDbClient);
+            const eventsToInsert: EventRecord[] = events.map((event) =>
+                EventRecord.createFromEvent(event));
+
+            const messagesToInsert: MessageRecord[] = messages.map(message => {
+                return {
+                    id: crypto.randomUUID(),
+                    stream_cursor: message.streamCursor,
+                    channel: message.channel,
+                    message_type: message.messageType,
+                    event_type: message.eventType,
+                    captured_by: message.capturedBy,
+                    captured_at: message.capturedAt,
+                    payload: message.payload,
+                }
+            })
+
+            const storeEventsQuery = QueryProvider.saveEvents(eventsToInsert);
+            const storeMessagesQuery = QueryProvider.saveMessages(messagesToInsert);
+
+            const transactItems = { TransactItems: [...storeEventsQuery, ...storeMessagesQuery] };
+
+            const command = new TransactWriteItemsCommand(transactItems)
+            await this.dynamoDbClient.send(command);
+
+            const numEvents = eventsToInsert.length;
+            const numMessages = messagesToInsert
+                .reduce((prev, { message_type: t }) =>
+                    Object.assign(prev, { [t]: (prev[t] ?? 0) + 1 }), {} as Record<string, number>);
+            return new StreamStoreAffected(numEvents, new Map(Object.entries(numMessages)));
+        } catch (error) {
+            if (this.isOccException(error)) {
+                throw new Error('OPTIMISTIC_CONCURRENCY_VIOLATION');
+            }
+            throw error;
+        }
+    }
+
+    /**
+     * Store outbox messages in a transaction
+     */
+    async storeOutboxMessagesAsync(
+        shardName: EvDbShardName,
+        records: EvDbMessage[],
+    ): Promise<number> {
+        throw new Error('Method not implemented.');
+    }
+
+    /**
+     * Get the last offset for a stream
+     */
+    async getLastOffsetAsync(
+        streamAddress: EvDbStreamAddress
+    ): Promise<number> {
+        const query = QueryProvider.getLastOffset(streamAddress);
+        const response = await this.dynamoDbClient.send(query);
+        if (!response.Items) {
+            return -1;
+        }
+        return parseInt(response.Items[0]?.offset.N ?? '-1', 10);
+    }
+
+    /**
+     * Get events for a stream since a specific offset
+     */
+    async *getEventsAsync(
+        streamCursor: EvDbStreamCursor,
+        pageSize: number = 100
+    ): AsyncGenerator<EvDbEvent, void, undefined> {
+        let queryCursor: Record<string, any> | undefined = undefined;
+
+        do {
+            const getEventsCommand = QueryProvider.getEvents(streamCursor);
+            const response = await this.dynamoDbClient.send(getEventsCommand);
+
+            if (response.Items && response.Items.length > 0) {
+                for (const item of response.Items) {
+                    const res = unmarshall(item);
+                    const streamAddress = deserializeStreamAddress(res.stream_address)
+                    const r: EventRecord = new EventRecord(
+                        crypto.randomUUID(),
+                        new EvDbStreamCursor(streamAddress.streamType, streamAddress.streamId, res.offset),
+                        res.event_type,
+                        res.captured_by,
+                        new Date(res.captured_at),
+                        res.payload,
+                        new Date(res.stored_at)
+                    );
+                    yield r.toEvDbEvent();
+                }
+            }
+
+            queryCursor = response.LastEvaluatedKey;
+
+        } while (queryCursor)
+    }
+
+    /**
+     * Get snapshot for a stream view
+     */
+    async getSnapshotAsync(
+        viewAddress: EvDbViewAddress
+    ): Promise<EvDbStoredSnapshotResultRaw> {
+        const { streamType, streamId, viewName } = viewAddress;
+        try {
+            const query = QueryProvider.getSnapshot(viewAddress);
+            const response = await this.dynamoDbClient.send(query);
+
+            if (!response.Items) {
+                return EvDbStoredSnapshotResultRaw.Empty;
+            }
+
+            const snapshot = unmarshall(response.Items[0]);
+
+            return new EvDbStoredSnapshotResultRaw(
+                snapshot.offset,
+                new Date(Number(snapshot.stored_at)),
+                snapshot.state,
+            );
+        } catch (error) {
+            throw error;
+        }
+    }
+
+    /**
+     * Save a snapshot
+     */
+    async storeSnapshotAsync(record: EvDbStoredSnapshotData): Promise<void> {
+        try {
+            const command = QueryProvider.saveSnapshot(record);
+            await this.dynamoDbClient.send(command);
+
+        } catch (error) {
+            throw error;
+        }
+    }
+
+    /**
+     * Check if an exception is an optimistic concurrency conflict
+     */
+    private isOccException(error: unknown): boolean {
+        if (!(error instanceof TransactionCanceledException)) {
+            return false;
+        }
+        return !!(error as TransactionCanceledException)
+            .CancellationReasons?.some(({ Code }) => Code === 'ConditionalCheckFailed')
+    }
+
+    /**
+     * Get table name for shard
+     */
+    private getTableNameForShard(shardName: EvDbShardName): string {
+        throw new Error('Method not implemented.');
+    }
+
+    /**
+     * Close the database connection
+     */
+    async close(): Promise<void> {
+        return;
+    }
+}