@outbox-event-bus/dynamodb-aws-sdk-outbox 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +469 -0
- package/dist/index.cjs +224 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +85 -0
- package/dist/index.d.cts.map +1 -0
- package/dist/index.d.mts +85 -0
- package/dist/index.d.mts.map +1 -0
- package/dist/index.mjs +221 -0
- package/dist/index.mjs.map +1 -0
- package/package.json +56 -0
- package/src/dynamodb-aws-sdk-outbox.test.ts +117 -0
- package/src/dynamodb-aws-sdk-outbox.ts +325 -0
- package/src/index.ts +2 -0
- package/src/integration.e2e.ts +465 -0
- package/src/transaction-storage.ts +25 -0
- package/src/transactional.test.ts +98 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.cjs","names":["DynamoDBDocumentClient","PollingService","BatchSizeLimitError","EventStatus","TransactWriteCommand","DocQueryCommand","event: FailedBusEvent","UpdateCommand","event: BusEvent","ConditionalCheckFailedException","expressionAttributeValues: Record<string, any>","AsyncLocalStorage","items: TransactWriteItem[]","collector: DynamoDBAwsSdkTransactionCollector"],"sources":["../src/dynamodb-aws-sdk-outbox.ts","../src/transaction-storage.ts"],"sourcesContent":["import { ConditionalCheckFailedException, type DynamoDBClient } from \"@aws-sdk/client-dynamodb\"\nimport {\n QueryCommand as DocQueryCommand,\n DynamoDBDocumentClient,\n TransactWriteCommand,\n type TransactWriteCommandInput,\n UpdateCommand,\n} from \"@aws-sdk/lib-dynamodb\"\nimport {\n BatchSizeLimitError,\n type BusEvent,\n type ErrorHandler,\n EventStatus,\n type FailedBusEvent,\n formatErrorMessage,\n type IOutbox,\n type OutboxConfig,\n PollingService,\n reportEventError,\n} from \"outbox-event-bus\"\n\n// DynamoDB has a hard limit of 100 items per transaction\nconst DYNAMODB_TRANSACTION_LIMIT = 100\n\nexport type TransactWriteItem = NonNullable<TransactWriteCommandInput[\"TransactItems\"]>[number]\n\nexport type DynamoDBAwsSdkTransactionCollector = {\n push: (item: TransactWriteItem) => void\n items?: TransactWriteItem[]\n}\n\nexport interface DynamoDBAwsSdkOutboxConfig extends OutboxConfig {\n client: DynamoDBClient\n tableName: string\n statusIndexName?: string\n processingTimeoutMs?: number // Time before a PROCESSING event is considered stuck\n getCollector?: (() => DynamoDBAwsSdkTransactionCollector | undefined) | undefined\n}\n\nexport class DynamoDBAwsSdkOutbox implements IOutbox<DynamoDBAwsSdkTransactionCollector> {\n private readonly config: Required<DynamoDBAwsSdkOutboxConfig>\n private readonly docClient: DynamoDBDocumentClient\n private readonly poller: PollingService\n\n constructor(config: DynamoDBAwsSdkOutboxConfig) {\n this.config = {\n batchSize: config.batchSize ?? 50,\n pollIntervalMs: config.pollIntervalMs ?? 1000,\n maxRetries: config.maxRetries ?? 5,\n baseBackoffMs: config.baseBackoffMs ?? 1000,\n processingTimeoutMs: config.processingTimeoutMs ?? 30000,\n maxErrorBackoffMs: config.maxErrorBackoffMs ?? 30000,\n tableName: config.tableName,\n statusIndexName: config.statusIndexName ?? \"status-gsiSortKey-index\",\n client: config.client,\n getCollector: config.getCollector,\n }\n\n this.docClient = DynamoDBDocumentClient.from(config.client, {\n marshallOptions: { removeUndefinedValues: true },\n })\n\n this.poller = new PollingService({\n pollIntervalMs: this.config.pollIntervalMs,\n baseBackoffMs: this.config.baseBackoffMs,\n maxErrorBackoffMs: this.config.maxErrorBackoffMs,\n performMaintenance: () => this.recoverStuckEvents(),\n processBatch: (handler) => this.processBatch(handler),\n })\n }\n\n async publish(\n events: BusEvent[],\n transaction?: DynamoDBAwsSdkTransactionCollector\n ): Promise<void> {\n if (events.length === 0) return\n\n if (events.length > DYNAMODB_TRANSACTION_LIMIT) {\n throw new BatchSizeLimitError(DYNAMODB_TRANSACTION_LIMIT, events.length)\n }\n\n const collector = transaction ?? this.config.getCollector?.()\n\n const items = events.map((event) => {\n return {\n Put: {\n TableName: this.config.tableName,\n Item: {\n id: event.id,\n type: event.type,\n payload: event.payload,\n occurredAt: event.occurredAt.toISOString(),\n status: EventStatus.CREATED,\n retryCount: 0,\n gsiSortKey: event.occurredAt.getTime(),\n },\n },\n }\n })\n\n if (collector) {\n const itemsInCollector = collector.items?.length ?? 0\n\n if (itemsInCollector + items.length > DYNAMODB_TRANSACTION_LIMIT) {\n throw new BatchSizeLimitError(DYNAMODB_TRANSACTION_LIMIT, itemsInCollector + items.length)\n }\n\n for (const item of items) {\n collector.push(item)\n }\n } else {\n await this.docClient.send(\n new TransactWriteCommand({\n TransactItems: items,\n })\n )\n }\n }\n\n async getFailedEvents(): Promise<FailedBusEvent[]> {\n const result = await this.docClient.send(\n new DocQueryCommand({\n TableName: this.config.tableName,\n IndexName: this.config.statusIndexName,\n KeyConditionExpression: \"#status = :status\",\n ExpressionAttributeNames: { \"#status\": \"status\" },\n ExpressionAttributeValues: { \":status\": EventStatus.FAILED },\n Limit: 100,\n ScanIndexForward: false,\n })\n )\n\n if (!result.Items) return []\n\n return result.Items.map((item) => {\n const event: FailedBusEvent = {\n id: item.id,\n type: item.type,\n payload: item.payload,\n occurredAt: this.parseOccurredAt(item.occurredAt),\n retryCount: item.retryCount || 0,\n }\n if (item.lastError) event.error = item.lastError\n if (item.startedOn) event.lastAttemptAt = new Date(item.startedOn)\n return event\n })\n }\n\n async retryEvents(eventIds: string[]): Promise<void> {\n if (eventIds.length === 0) return\n\n await Promise.all(\n eventIds.map((id) =>\n this.docClient.send(\n new UpdateCommand({\n TableName: this.config.tableName,\n Key: { id },\n UpdateExpression:\n \"SET #status = :pending, retryCount = :zero, gsiSortKey = :now REMOVE lastError, nextRetryAt, startedOn\",\n ExpressionAttributeNames: { \"#status\": \"status\" },\n ExpressionAttributeValues: {\n \":pending\": EventStatus.CREATED,\n \":zero\": 0,\n \":now\": Date.now(),\n },\n })\n )\n )\n )\n }\n\n start(handler: (event: BusEvent) => Promise<void>, onError: ErrorHandler): void {\n this.poller.start(handler, onError)\n }\n\n async stop(): Promise<void> {\n await this.poller.stop()\n }\n\n private async processBatch(handler: (event: BusEvent) => Promise<void>) {\n const now = Date.now()\n\n const result = await this.docClient.send(\n new DocQueryCommand({\n TableName: this.config.tableName,\n IndexName: this.config.statusIndexName,\n KeyConditionExpression: \"#status = :status AND gsiSortKey <= :now\",\n ExpressionAttributeNames: {\n \"#status\": \"status\",\n },\n ExpressionAttributeValues: {\n \":status\": EventStatus.CREATED,\n \":now\": now,\n },\n Limit: this.config.batchSize,\n })\n )\n\n if (!result.Items || result.Items.length === 0) return\n\n for (const item of result.Items) {\n const event: BusEvent = {\n id: item.id,\n type: item.type,\n payload: item.payload,\n occurredAt: this.parseOccurredAt(item.occurredAt),\n }\n\n try {\n await this.markEventAsProcessing(item.id, now)\n await handler(event)\n await this.markEventAsCompleted(event.id)\n } catch (error) {\n if (error instanceof ConditionalCheckFailedException) {\n continue\n }\n\n const newRetryCount = (item.retryCount || 0) + 1\n reportEventError(this.poller.onError, error, event, newRetryCount, this.config.maxRetries)\n\n await this.markEventAsFailed(item.id, newRetryCount, error)\n }\n }\n }\n\n private parseOccurredAt(occurredAt: unknown): Date {\n if (occurredAt instanceof Date) {\n return occurredAt\n }\n if (typeof occurredAt === \"string\") {\n return new Date(occurredAt)\n }\n return new Date()\n }\n\n private async markEventAsProcessing(id: string, now: number): Promise<void> {\n await this.docClient.send(\n new UpdateCommand({\n TableName: this.config.tableName,\n Key: { id },\n UpdateExpression: \"SET #status = :processing, gsiSortKey = :timeoutAt, startedOn = :now\",\n ExpressionAttributeNames: { \"#status\": \"status\" },\n ExpressionAttributeValues: {\n \":processing\": EventStatus.ACTIVE,\n \":timeoutAt\": now + this.config.processingTimeoutMs,\n \":now\": now,\n },\n })\n )\n }\n\n private async markEventAsCompleted(id: string): Promise<void> {\n await this.docClient.send(\n new UpdateCommand({\n TableName: this.config.tableName,\n Key: { id },\n UpdateExpression: \"SET #status = :completed, completedOn = :now REMOVE gsiSortKey\",\n ExpressionAttributeNames: { \"#status\": \"status\" },\n ExpressionAttributeValues: {\n \":completed\": EventStatus.COMPLETED,\n \":now\": Date.now(),\n },\n })\n )\n }\n\n private async markEventAsFailed(id: string, retryCount: number, error: unknown): Promise<void> {\n const isFinalFailure = retryCount >= this.config.maxRetries\n const status = isFinalFailure ? EventStatus.FAILED : EventStatus.CREATED\n const errorMsg = formatErrorMessage(error)\n const now = Date.now()\n\n const updateExpression = isFinalFailure\n ? \"SET #status = :status, retryCount = :rc, lastError = :err, nextRetryAt = :now REMOVE gsiSortKey\"\n : \"SET #status = :status, retryCount = :rc, lastError = :err, gsiSortKey = :nextAttempt\"\n\n const expressionAttributeValues: Record<string, any> = {\n \":status\": status,\n \":rc\": retryCount,\n \":err\": errorMsg,\n }\n\n if (isFinalFailure) {\n expressionAttributeValues[\":now\"] = now\n } else {\n const delay = this.poller.calculateBackoff(retryCount)\n expressionAttributeValues[\":nextAttempt\"] = now + delay\n }\n\n await this.docClient.send(\n new UpdateCommand({\n TableName: this.config.tableName,\n Key: { id },\n UpdateExpression: updateExpression,\n ExpressionAttributeNames: { \"#status\": \"status\" },\n ExpressionAttributeValues: expressionAttributeValues,\n })\n )\n }\n\n private async recoverStuckEvents() {\n const now = Date.now()\n\n const result = await this.docClient.send(\n new DocQueryCommand({\n TableName: this.config.tableName,\n IndexName: this.config.statusIndexName,\n KeyConditionExpression: \"#status = :status AND gsiSortKey <= :now\",\n ExpressionAttributeNames: { \"#status\": \"status\" },\n ExpressionAttributeValues: {\n \":status\": EventStatus.ACTIVE,\n \":now\": now,\n },\n })\n )\n\n if (result.Items && result.Items.length > 0) {\n await Promise.all(\n result.Items.map((item) =>\n this.markEventAsFailed(item.id, (item.retryCount || 0) + 1, \"Processing timeout\")\n )\n )\n }\n }\n}\n","import { AsyncLocalStorage } from \"node:async_hooks\"\nimport type {\n DynamoDBAwsSdkTransactionCollector,\n TransactWriteItem,\n} from \"./dynamodb-aws-sdk-outbox\"\n\nexport const dynamodbAwsSdkTransactionStorage =\n new AsyncLocalStorage<DynamoDBAwsSdkTransactionCollector>()\n\nexport async function withDynamoDBAwsSdkTransaction<T>(\n fn: (collector: DynamoDBAwsSdkTransactionCollector) => Promise<T>\n): Promise<T> {\n const items: TransactWriteItem[] = []\n const collector: DynamoDBAwsSdkTransactionCollector = {\n push: (item: TransactWriteItem) => items.push(item),\n get items() {\n return items\n },\n }\n return dynamodbAwsSdkTransactionStorage.run(collector, () => fn(collector))\n}\n\nexport function getDynamoDBAwsSdkCollector(): () => DynamoDBAwsSdkTransactionCollector | undefined {\n return () => dynamodbAwsSdkTransactionStorage.getStore()\n}\n"],"mappings":";;;;;;AAsBA,MAAM,6BAA6B;AAiBnC,IAAa,uBAAb,MAAyF;CACvF,AAAiB;CACjB,AAAiB;CACjB,AAAiB;CAEjB,YAAY,QAAoC;AAC9C,OAAK,SAAS;GACZ,WAAW,OAAO,aAAa;GAC/B,gBAAgB,OAAO,kBAAkB;GACzC,YAAY,OAAO,cAAc;GACjC,eAAe,OAAO,iBAAiB;GACvC,qBAAqB,OAAO,uBAAuB;GACnD,mBAAmB,OAAO,qBAAqB;GAC/C,WAAW,OAAO;GAClB,iBAAiB,OAAO,mBAAmB;GAC3C,QAAQ,OAAO;GACf,cAAc,OAAO;GACtB;AAED,OAAK,YAAYA,6CAAuB,KAAK,OAAO,QAAQ,EAC1D,iBAAiB,EAAE,uBAAuB,MAAM,EACjD,CAAC;AAEF,OAAK,SAAS,IAAIC,gCAAe;GAC/B,gBAAgB,KAAK,OAAO;GAC5B,eAAe,KAAK,OAAO;GAC3B,mBAAmB,KAAK,OAAO;GAC/B,0BAA0B,KAAK,oBAAoB;GACnD,eAAe,YAAY,KAAK,aAAa,QAAQ;GACtD,CAAC;;CAGJ,MAAM,QACJ,QACA,aACe;AACf,MAAI,OAAO,WAAW,EAAG;AAEzB,MAAI,OAAO,SAAS,2BAClB,OAAM,IAAIC,qCAAoB,4BAA4B,OAAO,OAAO;EAG1E,MAAM,YAAY,eAAe,KAAK,OAAO,gBAAgB;EAE7D,MAAM,QAAQ,OAAO,KAAK,UAAU;AAClC,UAAO,EACL,KAAK;IACH,WAAW,KAAK,OAAO;IACvB,MAAM;KACJ,IAAI,MAAM;KACV,MAAM,MAAM;KACZ,SAAS,MAAM;KACf,YAAY,MAAM,WAAW,aAAa;KAC1C,QAAQC,6BAAY;KACpB,YAAY;KACZ,YAAY,MAAM,WAAW,SAAS;KACvC;IACF,EACF;IACD;AAEF,MAAI,WAAW;GACb,MAAM,mBAAmB,UAAU,OAAO,UAAU;AAEpD,OAAI,mBAAmB,MAAM,SAAS,2BACpC,OAAM,IAAID,qCAAoB,4BAA4B,mBAAmB,MAAM,OAAO;AAG5F,QAAK,MAAM,QAAQ,MACjB,WAAU,KAAK,KAAK;QAGtB,OAAM,KAAK,UAAU,KACnB,IAAIE,2CAAqB,EACvB,eAAe,OAChB,CAAC,CACH;;CAIL,MAAM,kBAA6C;EACjD,MAAM,SAAS,MAAM,KAAK,UAAU,KAClC,IAAIC,mCAAgB;GAClB,WAAW,KAAK,OAAO;GACvB,WAAW,KAAK,OAAO;GACvB,wBAAwB;GACxB,0BAA0B,EAAE,WAAW,UAAU;GACjD,2BAA2B,EAAE,WAAWF,6BAAY,QAAQ;GAC5D,OAAO;GACP,kBAAkB;GACnB,CAAC,CACH;AAED,MAAI,CAAC,OAAO,MAAO,QAAO,EAAE;AAE5B,SAAO,OAAO,MAAM,KAAK,SAAS;GAChC,MAAMG,QAAwB;IAC5B,IAAI,KAAK;IACT,MAAM,KAAK;IACX,SAAS,KAAK;IACd,YAAY,KAAK,gBAAgB,KAAK,WAAW;IACjD,YAAY,KAAK,cAAc;IAChC;AACD,OAAI,KAAK,UAAW,OAAM,QAAQ,KAAK;AACvC,OAAI,KAAK,UAAW,OAAM,gBAAgB,IAAI,KAAK,KAAK,UAAU;AAClE,UAAO;IACP;;CAGJ,MAAM,YAAY,UAAmC;AACnD,MAAI,SAAS,WAAW,EAAG;AAE3B,QAAM,QAAQ,IACZ,SAAS,KAAK,OACZ,KAAK,UAAU,KACb,IAAIC,oCAAc;GAChB,WAAW,KAAK,OAAO;GACvB,KAAK,EAAE,IAAI;GACX,kBACE;GACF,0BAA0B,EAAE,WAAW,UAAU;GACjD,2BAA2B;IACzB,YAAYJ,6BAAY;IACxB,SAAS;IACT,QAAQ,KAAK,KAAK;IACnB;GACF,CAAC,CACH,CACF,CACF;;CAGH,MAAM,SAA6C,SAA6B;AAC9E,OAAK,OAAO,MAAM,SAAS,QAAQ;;CAGrC,MAAM,OAAsB;AAC1B,QAAM,KAAK,OAAO,MAAM;;CAG1B,MAAc,aAAa,SAA6C;EACtE,MAAM,MAAM,KAAK,KAAK;EAEtB,MAAM,SAAS,MAAM,KAAK,UAAU,KAClC,IAAIE,mCAAgB;GAClB,WAAW,KAAK,OAAO;GACvB,WAAW,KAAK,OAAO;GACvB,wBAAwB;GACxB,0BAA0B,EACxB,WAAW,UACZ;GACD,2BAA2B;IACzB,WAAWF,6BAAY;IACvB,QAAQ;IACT;GACD,OAAO,KAAK,OAAO;GACpB,CAAC,CACH;AAED,MAAI,CAAC,OAAO,SAAS,OAAO,MAAM,WAAW,EAAG;AAEhD,OAAK,MAAM,QAAQ,OAAO,OAAO;GAC/B,MAAMK,QAAkB;IACtB,IAAI,KAAK;IACT,MAAM,KAAK;IACX,SAAS,KAAK;IACd,YAAY,KAAK,gBAAgB,KAAK,WAAW;IAClD;AAED,OAAI;AACF,UAAM,KAAK,sBAAsB,KAAK,IAAI,IAAI;AAC9C,UAAM,QAAQ,MAAM;AACpB,UAAM,KAAK,qBAAqB,MAAM,GAAG;YAClC,OAAO;AACd,QAAI,iBAAiBC,yDACnB;IAGF,MAAM,iBAAiB,KAAK,cAAc,KAAK;AAC/C,2CAAiB,KAAK,OAAO,SAAS,OAAO,OAAO,eAAe,KAAK,OAAO,WAAW;AAE1F,UAAM,KAAK,kBAAkB,KAAK,IAAI,eAAe,MAAM;;;;CAKjE,AAAQ,gBAAgB,YAA2B;AACjD,MAAI,sBAAsB,KACxB,QAAO;AAET,MAAI,OAAO,eAAe,SACxB,QAAO,IAAI,KAAK,WAAW;AAE7B,yBAAO,IAAI,MAAM;;CAGnB,MAAc,sBAAsB,IAAY,KAA4B;AAC1E,QAAM,KAAK,UAAU,KACnB,IAAIF,oCAAc;GAChB,WAAW,KAAK,OAAO;GACvB,KAAK,EAAE,IAAI;GACX,kBAAkB;GAClB,0BAA0B,EAAE,WAAW,UAAU;GACjD,2BAA2B;IACzB,eAAeJ,6BAAY;IAC3B,cAAc,MAAM,KAAK,OAAO;IAChC,QAAQ;IACT;GACF,CAAC,CACH;;CAGH,MAAc,qBAAqB,IAA2B;AAC5D,QAAM,KAAK,UAAU,KACnB,IAAII,oCAAc;GAChB,WAAW,KAAK,OAAO;GACvB,KAAK,EAAE,IAAI;GACX,kBAAkB;GAClB,0BAA0B,EAAE,WAAW,UAAU;GACjD,2BAA2B;IACzB,cAAcJ,6BAAY;IAC1B,QAAQ,KAAK,KAAK;IACnB;GACF,CAAC,CACH;;CAGH,MAAc,kBAAkB,IAAY,YAAoB,OAA+B;EAC7F,MAAM,iBAAiB,cAAc,KAAK,OAAO;EACjD,MAAM,SAAS,iBAAiBA,6BAAY,SAASA,6BAAY;EACjE,MAAM,oDAA8B,MAAM;EAC1C,MAAM,MAAM,KAAK,KAAK;EAEtB,MAAM,mBAAmB,iBACrB,oGACA;EAEJ,MAAMO,4BAAiD;GACrD,WAAW;GACX,OAAO;GACP,QAAQ;GACT;AAED,MAAI,eACF,2BAA0B,UAAU;MAGpC,2BAA0B,kBAAkB,MAD9B,KAAK,OAAO,iBAAiB,WAAW;AAIxD,QAAM,KAAK,UAAU,KACnB,IAAIH,oCAAc;GAChB,WAAW,KAAK,OAAO;GACvB,KAAK,EAAE,IAAI;GACX,kBAAkB;GAClB,0BAA0B,EAAE,WAAW,UAAU;GACjD,2BAA2B;GAC5B,CAAC,CACH;;CAGH,MAAc,qBAAqB;EACjC,MAAM,MAAM,KAAK,KAAK;EAEtB,MAAM,SAAS,MAAM,KAAK,UAAU,KAClC,IAAIF,mCAAgB;GAClB,WAAW,KAAK,OAAO;GACvB,WAAW,KAAK,OAAO;GACvB,wBAAwB;GACxB,0BAA0B,EAAE,WAAW,UAAU;GACjD,2BAA2B;IACzB,WAAWF,6BAAY;IACvB,QAAQ;IACT;GACF,CAAC,CACH;AAED,MAAI,OAAO,SAAS,OAAO,MAAM,SAAS,EACxC,OAAM,QAAQ,IACZ,OAAO,MAAM,KAAK,SAChB,KAAK,kBAAkB,KAAK,KAAK,KAAK,cAAc,KAAK,GAAG,qBAAqB,CAClF,CACF;;;;;;AC3TP,MAAa,mCACX,IAAIQ,oCAAuD;AAE7D,eAAsB,8BACpB,IACY;CACZ,MAAMC,QAA6B,EAAE;CACrC,MAAMC,YAAgD;EACpD,OAAO,SAA4B,MAAM,KAAK,KAAK;EACnD,IAAI,QAAQ;AACV,UAAO;;EAEV;AACD,QAAO,iCAAiC,IAAI,iBAAiB,GAAG,UAAU,CAAC;;AAG7E,SAAgB,6BAAmF;AACjG,cAAa,iCAAiC,UAAU"}
|
package/dist/index.d.cts
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import { DynamoDBClient } from "@aws-sdk/client-dynamodb";
|
|
2
|
+
import { TransactWriteCommandInput } from "@aws-sdk/lib-dynamodb";
|
|
3
|
+
import { AsyncLocalStorage } from "node:async_hooks";
|
|
4
|
+
|
|
5
|
+
//#region ../../core/src/errors/errors.d.ts
|
|
6
|
+
interface OutboxErrorContext {
|
|
7
|
+
event?: BusEvent | FailedBusEvent;
|
|
8
|
+
cause?: unknown;
|
|
9
|
+
[key: string]: unknown;
|
|
10
|
+
}
|
|
11
|
+
declare abstract class OutboxError<TContext extends OutboxErrorContext = OutboxErrorContext> extends Error {
|
|
12
|
+
context?: TContext | undefined;
|
|
13
|
+
constructor(message: string, name: string, context?: TContext);
|
|
14
|
+
}
|
|
15
|
+
//#endregion
|
|
16
|
+
//#region ../../core/src/types/types.d.ts
|
|
17
|
+
type BusEvent<T extends string = string, P = unknown> = {
|
|
18
|
+
id: string;
|
|
19
|
+
type: T;
|
|
20
|
+
payload: P;
|
|
21
|
+
occurredAt: Date;
|
|
22
|
+
metadata?: Record<string, unknown>;
|
|
23
|
+
};
|
|
24
|
+
type FailedBusEvent<T extends string = string, P = unknown> = BusEvent<T, P> & {
|
|
25
|
+
error?: string;
|
|
26
|
+
retryCount: number;
|
|
27
|
+
lastAttemptAt?: Date;
|
|
28
|
+
};
|
|
29
|
+
type ErrorHandler = (error: OutboxError) => void;
|
|
30
|
+
//#endregion
|
|
31
|
+
//#region ../../core/src/types/interfaces.d.ts
|
|
32
|
+
interface IOutbox<TTransaction> {
|
|
33
|
+
publish: (events: BusEvent[], transaction?: TTransaction) => Promise<void>;
|
|
34
|
+
start: (handler: (event: BusEvent) => Promise<void>, onError: ErrorHandler) => void;
|
|
35
|
+
stop: () => Promise<void>;
|
|
36
|
+
getFailedEvents: () => Promise<FailedBusEvent[]>;
|
|
37
|
+
retryEvents: (eventIds: string[]) => Promise<void>;
|
|
38
|
+
}
|
|
39
|
+
interface OutboxConfig {
|
|
40
|
+
maxRetries?: number;
|
|
41
|
+
baseBackoffMs?: number;
|
|
42
|
+
pollIntervalMs?: number;
|
|
43
|
+
batchSize?: number;
|
|
44
|
+
processingTimeoutMs?: number;
|
|
45
|
+
maxErrorBackoffMs?: number;
|
|
46
|
+
}
|
|
47
|
+
//#endregion
|
|
48
|
+
//#region src/dynamodb-aws-sdk-outbox.d.ts
|
|
49
|
+
type TransactWriteItem = NonNullable<TransactWriteCommandInput["TransactItems"]>[number];
|
|
50
|
+
type DynamoDBAwsSdkTransactionCollector = {
|
|
51
|
+
push: (item: TransactWriteItem) => void;
|
|
52
|
+
items?: TransactWriteItem[];
|
|
53
|
+
};
|
|
54
|
+
interface DynamoDBAwsSdkOutboxConfig extends OutboxConfig {
|
|
55
|
+
client: DynamoDBClient;
|
|
56
|
+
tableName: string;
|
|
57
|
+
statusIndexName?: string;
|
|
58
|
+
processingTimeoutMs?: number;
|
|
59
|
+
getCollector?: (() => DynamoDBAwsSdkTransactionCollector | undefined) | undefined;
|
|
60
|
+
}
|
|
61
|
+
declare class DynamoDBAwsSdkOutbox implements IOutbox<DynamoDBAwsSdkTransactionCollector> {
|
|
62
|
+
private readonly config;
|
|
63
|
+
private readonly docClient;
|
|
64
|
+
private readonly poller;
|
|
65
|
+
constructor(config: DynamoDBAwsSdkOutboxConfig);
|
|
66
|
+
publish(events: BusEvent[], transaction?: DynamoDBAwsSdkTransactionCollector): Promise<void>;
|
|
67
|
+
getFailedEvents(): Promise<FailedBusEvent[]>;
|
|
68
|
+
retryEvents(eventIds: string[]): Promise<void>;
|
|
69
|
+
start(handler: (event: BusEvent) => Promise<void>, onError: ErrorHandler): void;
|
|
70
|
+
stop(): Promise<void>;
|
|
71
|
+
private processBatch;
|
|
72
|
+
private parseOccurredAt;
|
|
73
|
+
private markEventAsProcessing;
|
|
74
|
+
private markEventAsCompleted;
|
|
75
|
+
private markEventAsFailed;
|
|
76
|
+
private recoverStuckEvents;
|
|
77
|
+
}
|
|
78
|
+
//#endregion
|
|
79
|
+
//#region src/transaction-storage.d.ts
|
|
80
|
+
declare const dynamodbAwsSdkTransactionStorage: AsyncLocalStorage<DynamoDBAwsSdkTransactionCollector>;
|
|
81
|
+
declare function withDynamoDBAwsSdkTransaction<T>(fn: (collector: DynamoDBAwsSdkTransactionCollector) => Promise<T>): Promise<T>;
|
|
82
|
+
declare function getDynamoDBAwsSdkCollector(): () => DynamoDBAwsSdkTransactionCollector | undefined;
|
|
83
|
+
//#endregion
|
|
84
|
+
export { DynamoDBAwsSdkOutbox, DynamoDBAwsSdkOutboxConfig, DynamoDBAwsSdkTransactionCollector, TransactWriteItem, dynamodbAwsSdkTransactionStorage, getDynamoDBAwsSdkCollector, withDynamoDBAwsSdkTransaction };
|
|
85
|
+
//# sourceMappingURL=index.d.cts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.cts","names":[],"sources":["../../../core/src/errors/errors.ts","../../../core/src/types/types.ts","../../../core/src/types/interfaces.ts","../src/dynamodb-aws-sdk-outbox.ts","../src/transaction-storage.ts"],"sourcesContent":[],"mappings":";;;;;UAEiB,kBAAA;UACP,WAAW;;;AADrB;AAMsB,uBAAA,WAAW,CAAA,iBACd,kBADc,GACO,kBADP,CAAA,SAEvB,KAAA,CAFuB;EACd,OAAA,CAAA,EAEA,QAFA,GAAA,SAAA;EAAqB,WAAA,CAAA,OAAA,EAAA,MAAA,EAAA,IAAA,EAAA,MAAA,EAAA,OAAA,CAAA,EAIe,QAJf;;;;KCL5B;;QAEJ;EDJS,OAAA,ECKN,CDLM;EAMK,UAAA,ECAR,IDAmB;EACd,QAAA,CAAA,ECAN,MDAM,CAAA,MAAA,EAAA,OAAA,CAAA;CAAqB;AAErB,KCCP,cDDO,CAAA,UAAA,MAAA,GAAA,MAAA,EAAA,IAAA,OAAA,CAAA,GCCkD,QDDlD,CCC2D,CDD3D,ECC8D,CDD9D,CAAA,GAAA;EAEoC,KAAA,CAAA,EAAA,MAAA;EAH7C,UAAA,EAAA,MAAA;EAAK,aAAA,CAAA,ECKG,IDLH;;ACJP,KAuBI,YAAA,GAvBJ,CAAA,KAAA,EAuB2B,WAvB3B,EAAA,GAAA,IAAA;;;UCJS;oBACG,0BAA0B,iBAAiB;2BACpC,aAAa,wBAAwB;cAClD;EFHG,eAAA,EAAA,GAAA,GEIQ,OFJU,CEIF,cFHZ,EAAA,CAAA;EAKC,WAAA,EAAA,CAAW,QAAA,EAAA,MAAA,EAAA,EAAA,GEDM,OFCN,CAAA,IAAA,CAAA;;UEsDhB,YAAA;EA5DA,UAAO,CAAA,EAAA,MAAA;EACJ,aAAA,CAAA,EAAA,MAAA;EAA0B,cAAA,CAAA,EAAA,MAAA;EAAiB,SAAA,CAAA,EAAA,MAAA;EACpC,mBAAA,CAAA,EAAA,MAAA;EAAa,iBAAA,CAAA,EAAA,MAAA;;;;KCoB5B,iBAAA,GAAoB,YAAY;KAEhC,kCAAA;EHxBK,IAAA,EAAA,CAAA,IAAA,EGyBF,iBHzBoB,EAAA,GACzB,IAAA;EAKY,KAAA,CAAA,EGoBZ,iBHpBuB,EAAA;CACd;AAAqB,UGsBvB,0BAAA,SAAmC,YHtBZ,CAAA;EAErB,MAAA,EGqBT,cHrBS;EAEoC,SAAA,EAAA,MAAA;EAH7C,eAAA,CAAA,EAAA,MAAA;EAAK,mBAAA,CAAA,EAAA,MAAA;wBG0BS;;cAGX,oBAAA,YAAgC,QAAQ;EFnCzC,iBAAQ,MAAA;EAEZ,iBAAA,SAAA;EACG,iBAAA,MAAA;EACG,WAAA,CAAA,MAAA,EEoCQ,0BFpCR;EACD,OAAA,CAAA,MAAA,EE+DD,QF/DC,EAAA,EAAA,WAAA,CAAA,EEgEK,kCFhEL,CAAA,EEiER,OFjEQ,CAAA,IAAA,CAAA;EAAM,eAAA,CAAA,CAAA,EE8GQ,OF9GR,CE8GgB,cF9GhB,EAAA,CAAA;EAGP,WAAA,CAAA,QAAc,EAAA,MAAA,EAAA,CAAA,EEwIe,OFxIf,CAAA,IAAA,CAAA;EAAoD,KAAA,CAAA,OAAA,EAAA,CAAA,KAAA,EE+JrD,QF/JqD,EAAA,GE+JxC,OF/JwC,CAAA,IAAA,CAAA,EAAA,OAAA,EE+JhB,YF/JgB,CAAA,EAAA,IAAA;EAAG,IAAA,CAAA,CAAA,EEmKjE,OFnKiE,CAAA,IAAA,CAAA;EAAZ,QAAA,YAAA;EAGnD,QAAA,eAAA;EAAI,QAAA,qBAAA;EAcV,QAAA,oBAAuB;;;;;;cGvBtB,kCAAgC,kBAAA;iBAGvB,iDACJ,uCAAuC,QAAQ,KAC9D,QAAQ;iBAWK,0BAAA,CAAA,SAAoC"}
|
package/dist/index.d.mts
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import { DynamoDBClient } from "@aws-sdk/client-dynamodb";
|
|
2
|
+
import { TransactWriteCommandInput } from "@aws-sdk/lib-dynamodb";
|
|
3
|
+
import { AsyncLocalStorage } from "node:async_hooks";
|
|
4
|
+
|
|
5
|
+
//#region ../../core/src/errors/errors.d.ts
|
|
6
|
+
interface OutboxErrorContext {
|
|
7
|
+
event?: BusEvent | FailedBusEvent;
|
|
8
|
+
cause?: unknown;
|
|
9
|
+
[key: string]: unknown;
|
|
10
|
+
}
|
|
11
|
+
declare abstract class OutboxError<TContext extends OutboxErrorContext = OutboxErrorContext> extends Error {
|
|
12
|
+
context?: TContext | undefined;
|
|
13
|
+
constructor(message: string, name: string, context?: TContext);
|
|
14
|
+
}
|
|
15
|
+
//#endregion
|
|
16
|
+
//#region ../../core/src/types/types.d.ts
|
|
17
|
+
type BusEvent<T extends string = string, P = unknown> = {
|
|
18
|
+
id: string;
|
|
19
|
+
type: T;
|
|
20
|
+
payload: P;
|
|
21
|
+
occurredAt: Date;
|
|
22
|
+
metadata?: Record<string, unknown>;
|
|
23
|
+
};
|
|
24
|
+
type FailedBusEvent<T extends string = string, P = unknown> = BusEvent<T, P> & {
|
|
25
|
+
error?: string;
|
|
26
|
+
retryCount: number;
|
|
27
|
+
lastAttemptAt?: Date;
|
|
28
|
+
};
|
|
29
|
+
type ErrorHandler = (error: OutboxError) => void;
|
|
30
|
+
//#endregion
|
|
31
|
+
//#region ../../core/src/types/interfaces.d.ts
|
|
32
|
+
interface IOutbox<TTransaction> {
|
|
33
|
+
publish: (events: BusEvent[], transaction?: TTransaction) => Promise<void>;
|
|
34
|
+
start: (handler: (event: BusEvent) => Promise<void>, onError: ErrorHandler) => void;
|
|
35
|
+
stop: () => Promise<void>;
|
|
36
|
+
getFailedEvents: () => Promise<FailedBusEvent[]>;
|
|
37
|
+
retryEvents: (eventIds: string[]) => Promise<void>;
|
|
38
|
+
}
|
|
39
|
+
interface OutboxConfig {
|
|
40
|
+
maxRetries?: number;
|
|
41
|
+
baseBackoffMs?: number;
|
|
42
|
+
pollIntervalMs?: number;
|
|
43
|
+
batchSize?: number;
|
|
44
|
+
processingTimeoutMs?: number;
|
|
45
|
+
maxErrorBackoffMs?: number;
|
|
46
|
+
}
|
|
47
|
+
//#endregion
|
|
48
|
+
//#region src/dynamodb-aws-sdk-outbox.d.ts
|
|
49
|
+
type TransactWriteItem = NonNullable<TransactWriteCommandInput["TransactItems"]>[number];
|
|
50
|
+
type DynamoDBAwsSdkTransactionCollector = {
|
|
51
|
+
push: (item: TransactWriteItem) => void;
|
|
52
|
+
items?: TransactWriteItem[];
|
|
53
|
+
};
|
|
54
|
+
interface DynamoDBAwsSdkOutboxConfig extends OutboxConfig {
|
|
55
|
+
client: DynamoDBClient;
|
|
56
|
+
tableName: string;
|
|
57
|
+
statusIndexName?: string;
|
|
58
|
+
processingTimeoutMs?: number;
|
|
59
|
+
getCollector?: (() => DynamoDBAwsSdkTransactionCollector | undefined) | undefined;
|
|
60
|
+
}
|
|
61
|
+
declare class DynamoDBAwsSdkOutbox implements IOutbox<DynamoDBAwsSdkTransactionCollector> {
|
|
62
|
+
private readonly config;
|
|
63
|
+
private readonly docClient;
|
|
64
|
+
private readonly poller;
|
|
65
|
+
constructor(config: DynamoDBAwsSdkOutboxConfig);
|
|
66
|
+
publish(events: BusEvent[], transaction?: DynamoDBAwsSdkTransactionCollector): Promise<void>;
|
|
67
|
+
getFailedEvents(): Promise<FailedBusEvent[]>;
|
|
68
|
+
retryEvents(eventIds: string[]): Promise<void>;
|
|
69
|
+
start(handler: (event: BusEvent) => Promise<void>, onError: ErrorHandler): void;
|
|
70
|
+
stop(): Promise<void>;
|
|
71
|
+
private processBatch;
|
|
72
|
+
private parseOccurredAt;
|
|
73
|
+
private markEventAsProcessing;
|
|
74
|
+
private markEventAsCompleted;
|
|
75
|
+
private markEventAsFailed;
|
|
76
|
+
private recoverStuckEvents;
|
|
77
|
+
}
|
|
78
|
+
//#endregion
|
|
79
|
+
//#region src/transaction-storage.d.ts
|
|
80
|
+
declare const dynamodbAwsSdkTransactionStorage: AsyncLocalStorage<DynamoDBAwsSdkTransactionCollector>;
|
|
81
|
+
declare function withDynamoDBAwsSdkTransaction<T>(fn: (collector: DynamoDBAwsSdkTransactionCollector) => Promise<T>): Promise<T>;
|
|
82
|
+
declare function getDynamoDBAwsSdkCollector(): () => DynamoDBAwsSdkTransactionCollector | undefined;
|
|
83
|
+
//#endregion
|
|
84
|
+
export { DynamoDBAwsSdkOutbox, DynamoDBAwsSdkOutboxConfig, DynamoDBAwsSdkTransactionCollector, TransactWriteItem, dynamodbAwsSdkTransactionStorage, getDynamoDBAwsSdkCollector, withDynamoDBAwsSdkTransaction };
|
|
85
|
+
//# sourceMappingURL=index.d.mts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.mts","names":[],"sources":["../../../core/src/errors/errors.ts","../../../core/src/types/types.ts","../../../core/src/types/interfaces.ts","../src/dynamodb-aws-sdk-outbox.ts","../src/transaction-storage.ts"],"sourcesContent":[],"mappings":";;;;;UAEiB,kBAAA;UACP,WAAW;;;AADrB;AAMsB,uBAAA,WAAW,CAAA,iBACd,kBADc,GACO,kBADP,CAAA,SAEvB,KAAA,CAFuB;EACd,OAAA,CAAA,EAEA,QAFA,GAAA,SAAA;EAAqB,WAAA,CAAA,OAAA,EAAA,MAAA,EAAA,IAAA,EAAA,MAAA,EAAA,OAAA,CAAA,EAIe,QAJf;;;;KCL5B;;QAEJ;EDJS,OAAA,ECKN,CDLM;EAMK,UAAA,ECAR,IDAmB;EACd,QAAA,CAAA,ECAN,MDAM,CAAA,MAAA,EAAA,OAAA,CAAA;CAAqB;AAErB,KCCP,cDDO,CAAA,UAAA,MAAA,GAAA,MAAA,EAAA,IAAA,OAAA,CAAA,GCCkD,QDDlD,CCC2D,CDD3D,ECC8D,CDD9D,CAAA,GAAA;EAEoC,KAAA,CAAA,EAAA,MAAA;EAH7C,UAAA,EAAA,MAAA;EAAK,aAAA,CAAA,ECKG,IDLH;;ACJP,KAuBI,YAAA,GAvBJ,CAAA,KAAA,EAuB2B,WAvB3B,EAAA,GAAA,IAAA;;;UCJS;oBACG,0BAA0B,iBAAiB;2BACpC,aAAa,wBAAwB;cAClD;EFHG,eAAA,EAAA,GAAA,GEIQ,OFJU,CEIF,cFHZ,EAAA,CAAA;EAKC,WAAA,EAAA,CAAW,QAAA,EAAA,MAAA,EAAA,EAAA,GEDM,OFCN,CAAA,IAAA,CAAA;;UEsDhB,YAAA;EA5DA,UAAO,CAAA,EAAA,MAAA;EACJ,aAAA,CAAA,EAAA,MAAA;EAA0B,cAAA,CAAA,EAAA,MAAA;EAAiB,SAAA,CAAA,EAAA,MAAA;EACpC,mBAAA,CAAA,EAAA,MAAA;EAAa,iBAAA,CAAA,EAAA,MAAA;;;;KCoB5B,iBAAA,GAAoB,YAAY;KAEhC,kCAAA;EHxBK,IAAA,EAAA,CAAA,IAAA,EGyBF,iBHzBoB,EAAA,GACzB,IAAA;EAKY,KAAA,CAAA,EGoBZ,iBHpBuB,EAAA;CACd;AAAqB,UGsBvB,0BAAA,SAAmC,YHtBZ,CAAA;EAErB,MAAA,EGqBT,cHrBS;EAEoC,SAAA,EAAA,MAAA;EAH7C,eAAA,CAAA,EAAA,MAAA;EAAK,mBAAA,CAAA,EAAA,MAAA;wBG0BS;;cAGX,oBAAA,YAAgC,QAAQ;EFnCzC,iBAAQ,MAAA;EAEZ,iBAAA,SAAA;EACG,iBAAA,MAAA;EACG,WAAA,CAAA,MAAA,EEoCQ,0BFpCR;EACD,OAAA,CAAA,MAAA,EE+DD,QF/DC,EAAA,EAAA,WAAA,CAAA,EEgEK,kCFhEL,CAAA,EEiER,OFjEQ,CAAA,IAAA,CAAA;EAAM,eAAA,CAAA,CAAA,EE8GQ,OF9GR,CE8GgB,cF9GhB,EAAA,CAAA;EAGP,WAAA,CAAA,QAAc,EAAA,MAAA,EAAA,CAAA,EEwIe,OFxIf,CAAA,IAAA,CAAA;EAAoD,KAAA,CAAA,OAAA,EAAA,CAAA,KAAA,EE+JrD,QF/JqD,EAAA,GE+JxC,OF/JwC,CAAA,IAAA,CAAA,EAAA,OAAA,EE+JhB,YF/JgB,CAAA,EAAA,IAAA;EAAG,IAAA,CAAA,CAAA,EEmKjE,OFnKiE,CAAA,IAAA,CAAA;EAAZ,QAAA,YAAA;EAGnD,QAAA,eAAA;EAAI,QAAA,qBAAA;EAcV,QAAA,oBAAuB;;;;;;cGvBtB,kCAAgC,kBAAA;iBAGvB,iDACJ,uCAAuC,QAAQ,KAC9D,QAAQ;iBAWK,0BAAA,CAAA,SAAoC"}
|
package/dist/index.mjs
ADDED
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
import { ConditionalCheckFailedException } from "@aws-sdk/client-dynamodb";
|
|
2
|
+
import { DynamoDBDocumentClient, QueryCommand, TransactWriteCommand, UpdateCommand } from "@aws-sdk/lib-dynamodb";
|
|
3
|
+
import { BatchSizeLimitError, EventStatus, PollingService, formatErrorMessage, reportEventError } from "outbox-event-bus";
|
|
4
|
+
import { AsyncLocalStorage } from "node:async_hooks";
|
|
5
|
+
|
|
6
|
+
//#region src/dynamodb-aws-sdk-outbox.ts
|
|
7
|
+
const DYNAMODB_TRANSACTION_LIMIT = 100;
|
|
8
|
+
var DynamoDBAwsSdkOutbox = class {
|
|
9
|
+
config;
|
|
10
|
+
docClient;
|
|
11
|
+
poller;
|
|
12
|
+
constructor(config) {
|
|
13
|
+
this.config = {
|
|
14
|
+
batchSize: config.batchSize ?? 50,
|
|
15
|
+
pollIntervalMs: config.pollIntervalMs ?? 1e3,
|
|
16
|
+
maxRetries: config.maxRetries ?? 5,
|
|
17
|
+
baseBackoffMs: config.baseBackoffMs ?? 1e3,
|
|
18
|
+
processingTimeoutMs: config.processingTimeoutMs ?? 3e4,
|
|
19
|
+
maxErrorBackoffMs: config.maxErrorBackoffMs ?? 3e4,
|
|
20
|
+
tableName: config.tableName,
|
|
21
|
+
statusIndexName: config.statusIndexName ?? "status-gsiSortKey-index",
|
|
22
|
+
client: config.client,
|
|
23
|
+
getCollector: config.getCollector
|
|
24
|
+
};
|
|
25
|
+
this.docClient = DynamoDBDocumentClient.from(config.client, { marshallOptions: { removeUndefinedValues: true } });
|
|
26
|
+
this.poller = new PollingService({
|
|
27
|
+
pollIntervalMs: this.config.pollIntervalMs,
|
|
28
|
+
baseBackoffMs: this.config.baseBackoffMs,
|
|
29
|
+
maxErrorBackoffMs: this.config.maxErrorBackoffMs,
|
|
30
|
+
performMaintenance: () => this.recoverStuckEvents(),
|
|
31
|
+
processBatch: (handler) => this.processBatch(handler)
|
|
32
|
+
});
|
|
33
|
+
}
|
|
34
|
+
async publish(events, transaction) {
|
|
35
|
+
if (events.length === 0) return;
|
|
36
|
+
if (events.length > DYNAMODB_TRANSACTION_LIMIT) throw new BatchSizeLimitError(DYNAMODB_TRANSACTION_LIMIT, events.length);
|
|
37
|
+
const collector = transaction ?? this.config.getCollector?.();
|
|
38
|
+
const items = events.map((event) => {
|
|
39
|
+
return { Put: {
|
|
40
|
+
TableName: this.config.tableName,
|
|
41
|
+
Item: {
|
|
42
|
+
id: event.id,
|
|
43
|
+
type: event.type,
|
|
44
|
+
payload: event.payload,
|
|
45
|
+
occurredAt: event.occurredAt.toISOString(),
|
|
46
|
+
status: EventStatus.CREATED,
|
|
47
|
+
retryCount: 0,
|
|
48
|
+
gsiSortKey: event.occurredAt.getTime()
|
|
49
|
+
}
|
|
50
|
+
} };
|
|
51
|
+
});
|
|
52
|
+
if (collector) {
|
|
53
|
+
const itemsInCollector = collector.items?.length ?? 0;
|
|
54
|
+
if (itemsInCollector + items.length > DYNAMODB_TRANSACTION_LIMIT) throw new BatchSizeLimitError(DYNAMODB_TRANSACTION_LIMIT, itemsInCollector + items.length);
|
|
55
|
+
for (const item of items) collector.push(item);
|
|
56
|
+
} else await this.docClient.send(new TransactWriteCommand({ TransactItems: items }));
|
|
57
|
+
}
|
|
58
|
+
async getFailedEvents() {
|
|
59
|
+
const result = await this.docClient.send(new QueryCommand({
|
|
60
|
+
TableName: this.config.tableName,
|
|
61
|
+
IndexName: this.config.statusIndexName,
|
|
62
|
+
KeyConditionExpression: "#status = :status",
|
|
63
|
+
ExpressionAttributeNames: { "#status": "status" },
|
|
64
|
+
ExpressionAttributeValues: { ":status": EventStatus.FAILED },
|
|
65
|
+
Limit: 100,
|
|
66
|
+
ScanIndexForward: false
|
|
67
|
+
}));
|
|
68
|
+
if (!result.Items) return [];
|
|
69
|
+
return result.Items.map((item) => {
|
|
70
|
+
const event = {
|
|
71
|
+
id: item.id,
|
|
72
|
+
type: item.type,
|
|
73
|
+
payload: item.payload,
|
|
74
|
+
occurredAt: this.parseOccurredAt(item.occurredAt),
|
|
75
|
+
retryCount: item.retryCount || 0
|
|
76
|
+
};
|
|
77
|
+
if (item.lastError) event.error = item.lastError;
|
|
78
|
+
if (item.startedOn) event.lastAttemptAt = new Date(item.startedOn);
|
|
79
|
+
return event;
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
async retryEvents(eventIds) {
|
|
83
|
+
if (eventIds.length === 0) return;
|
|
84
|
+
await Promise.all(eventIds.map((id) => this.docClient.send(new UpdateCommand({
|
|
85
|
+
TableName: this.config.tableName,
|
|
86
|
+
Key: { id },
|
|
87
|
+
UpdateExpression: "SET #status = :pending, retryCount = :zero, gsiSortKey = :now REMOVE lastError, nextRetryAt, startedOn",
|
|
88
|
+
ExpressionAttributeNames: { "#status": "status" },
|
|
89
|
+
ExpressionAttributeValues: {
|
|
90
|
+
":pending": EventStatus.CREATED,
|
|
91
|
+
":zero": 0,
|
|
92
|
+
":now": Date.now()
|
|
93
|
+
}
|
|
94
|
+
}))));
|
|
95
|
+
}
|
|
96
|
+
start(handler, onError) {
|
|
97
|
+
this.poller.start(handler, onError);
|
|
98
|
+
}
|
|
99
|
+
async stop() {
|
|
100
|
+
await this.poller.stop();
|
|
101
|
+
}
|
|
102
|
+
async processBatch(handler) {
|
|
103
|
+
const now = Date.now();
|
|
104
|
+
const result = await this.docClient.send(new QueryCommand({
|
|
105
|
+
TableName: this.config.tableName,
|
|
106
|
+
IndexName: this.config.statusIndexName,
|
|
107
|
+
KeyConditionExpression: "#status = :status AND gsiSortKey <= :now",
|
|
108
|
+
ExpressionAttributeNames: { "#status": "status" },
|
|
109
|
+
ExpressionAttributeValues: {
|
|
110
|
+
":status": EventStatus.CREATED,
|
|
111
|
+
":now": now
|
|
112
|
+
},
|
|
113
|
+
Limit: this.config.batchSize
|
|
114
|
+
}));
|
|
115
|
+
if (!result.Items || result.Items.length === 0) return;
|
|
116
|
+
for (const item of result.Items) {
|
|
117
|
+
const event = {
|
|
118
|
+
id: item.id,
|
|
119
|
+
type: item.type,
|
|
120
|
+
payload: item.payload,
|
|
121
|
+
occurredAt: this.parseOccurredAt(item.occurredAt)
|
|
122
|
+
};
|
|
123
|
+
try {
|
|
124
|
+
await this.markEventAsProcessing(item.id, now);
|
|
125
|
+
await handler(event);
|
|
126
|
+
await this.markEventAsCompleted(event.id);
|
|
127
|
+
} catch (error) {
|
|
128
|
+
if (error instanceof ConditionalCheckFailedException) continue;
|
|
129
|
+
const newRetryCount = (item.retryCount || 0) + 1;
|
|
130
|
+
reportEventError(this.poller.onError, error, event, newRetryCount, this.config.maxRetries);
|
|
131
|
+
await this.markEventAsFailed(item.id, newRetryCount, error);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
parseOccurredAt(occurredAt) {
|
|
136
|
+
if (occurredAt instanceof Date) return occurredAt;
|
|
137
|
+
if (typeof occurredAt === "string") return new Date(occurredAt);
|
|
138
|
+
return /* @__PURE__ */ new Date();
|
|
139
|
+
}
|
|
140
|
+
async markEventAsProcessing(id, now) {
|
|
141
|
+
await this.docClient.send(new UpdateCommand({
|
|
142
|
+
TableName: this.config.tableName,
|
|
143
|
+
Key: { id },
|
|
144
|
+
UpdateExpression: "SET #status = :processing, gsiSortKey = :timeoutAt, startedOn = :now",
|
|
145
|
+
ExpressionAttributeNames: { "#status": "status" },
|
|
146
|
+
ExpressionAttributeValues: {
|
|
147
|
+
":processing": EventStatus.ACTIVE,
|
|
148
|
+
":timeoutAt": now + this.config.processingTimeoutMs,
|
|
149
|
+
":now": now
|
|
150
|
+
}
|
|
151
|
+
}));
|
|
152
|
+
}
|
|
153
|
+
async markEventAsCompleted(id) {
|
|
154
|
+
await this.docClient.send(new UpdateCommand({
|
|
155
|
+
TableName: this.config.tableName,
|
|
156
|
+
Key: { id },
|
|
157
|
+
UpdateExpression: "SET #status = :completed, completedOn = :now REMOVE gsiSortKey",
|
|
158
|
+
ExpressionAttributeNames: { "#status": "status" },
|
|
159
|
+
ExpressionAttributeValues: {
|
|
160
|
+
":completed": EventStatus.COMPLETED,
|
|
161
|
+
":now": Date.now()
|
|
162
|
+
}
|
|
163
|
+
}));
|
|
164
|
+
}
|
|
165
|
+
async markEventAsFailed(id, retryCount, error) {
|
|
166
|
+
const isFinalFailure = retryCount >= this.config.maxRetries;
|
|
167
|
+
const status = isFinalFailure ? EventStatus.FAILED : EventStatus.CREATED;
|
|
168
|
+
const errorMsg = formatErrorMessage(error);
|
|
169
|
+
const now = Date.now();
|
|
170
|
+
const updateExpression = isFinalFailure ? "SET #status = :status, retryCount = :rc, lastError = :err, nextRetryAt = :now REMOVE gsiSortKey" : "SET #status = :status, retryCount = :rc, lastError = :err, gsiSortKey = :nextAttempt";
|
|
171
|
+
const expressionAttributeValues = {
|
|
172
|
+
":status": status,
|
|
173
|
+
":rc": retryCount,
|
|
174
|
+
":err": errorMsg
|
|
175
|
+
};
|
|
176
|
+
if (isFinalFailure) expressionAttributeValues[":now"] = now;
|
|
177
|
+
else expressionAttributeValues[":nextAttempt"] = now + this.poller.calculateBackoff(retryCount);
|
|
178
|
+
await this.docClient.send(new UpdateCommand({
|
|
179
|
+
TableName: this.config.tableName,
|
|
180
|
+
Key: { id },
|
|
181
|
+
UpdateExpression: updateExpression,
|
|
182
|
+
ExpressionAttributeNames: { "#status": "status" },
|
|
183
|
+
ExpressionAttributeValues: expressionAttributeValues
|
|
184
|
+
}));
|
|
185
|
+
}
|
|
186
|
+
async recoverStuckEvents() {
|
|
187
|
+
const now = Date.now();
|
|
188
|
+
const result = await this.docClient.send(new QueryCommand({
|
|
189
|
+
TableName: this.config.tableName,
|
|
190
|
+
IndexName: this.config.statusIndexName,
|
|
191
|
+
KeyConditionExpression: "#status = :status AND gsiSortKey <= :now",
|
|
192
|
+
ExpressionAttributeNames: { "#status": "status" },
|
|
193
|
+
ExpressionAttributeValues: {
|
|
194
|
+
":status": EventStatus.ACTIVE,
|
|
195
|
+
":now": now
|
|
196
|
+
}
|
|
197
|
+
}));
|
|
198
|
+
if (result.Items && result.Items.length > 0) await Promise.all(result.Items.map((item) => this.markEventAsFailed(item.id, (item.retryCount || 0) + 1, "Processing timeout")));
|
|
199
|
+
}
|
|
200
|
+
};
|
|
201
|
+
|
|
202
|
+
//#endregion
|
|
203
|
+
//#region src/transaction-storage.ts
|
|
204
|
+
const dynamodbAwsSdkTransactionStorage = new AsyncLocalStorage();
|
|
205
|
+
async function withDynamoDBAwsSdkTransaction(fn) {
|
|
206
|
+
const items = [];
|
|
207
|
+
const collector = {
|
|
208
|
+
push: (item) => items.push(item),
|
|
209
|
+
get items() {
|
|
210
|
+
return items;
|
|
211
|
+
}
|
|
212
|
+
};
|
|
213
|
+
return dynamodbAwsSdkTransactionStorage.run(collector, () => fn(collector));
|
|
214
|
+
}
|
|
215
|
+
function getDynamoDBAwsSdkCollector() {
|
|
216
|
+
return () => dynamodbAwsSdkTransactionStorage.getStore();
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
//#endregion
|
|
220
|
+
export { DynamoDBAwsSdkOutbox, dynamodbAwsSdkTransactionStorage, getDynamoDBAwsSdkCollector, withDynamoDBAwsSdkTransaction };
|
|
221
|
+
//# sourceMappingURL=index.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.mjs","names":["DocQueryCommand","event: FailedBusEvent","event: BusEvent","expressionAttributeValues: Record<string, any>","items: TransactWriteItem[]","collector: DynamoDBAwsSdkTransactionCollector"],"sources":["../src/dynamodb-aws-sdk-outbox.ts","../src/transaction-storage.ts"],"sourcesContent":["import { ConditionalCheckFailedException, type DynamoDBClient } from \"@aws-sdk/client-dynamodb\"\nimport {\n QueryCommand as DocQueryCommand,\n DynamoDBDocumentClient,\n TransactWriteCommand,\n type TransactWriteCommandInput,\n UpdateCommand,\n} from \"@aws-sdk/lib-dynamodb\"\nimport {\n BatchSizeLimitError,\n type BusEvent,\n type ErrorHandler,\n EventStatus,\n type FailedBusEvent,\n formatErrorMessage,\n type IOutbox,\n type OutboxConfig,\n PollingService,\n reportEventError,\n} from \"outbox-event-bus\"\n\n// DynamoDB has a hard limit of 100 items per transaction\nconst DYNAMODB_TRANSACTION_LIMIT = 100\n\nexport type TransactWriteItem = NonNullable<TransactWriteCommandInput[\"TransactItems\"]>[number]\n\nexport type DynamoDBAwsSdkTransactionCollector = {\n push: (item: TransactWriteItem) => void\n items?: TransactWriteItem[]\n}\n\nexport interface DynamoDBAwsSdkOutboxConfig extends OutboxConfig {\n client: DynamoDBClient\n tableName: string\n statusIndexName?: string\n processingTimeoutMs?: number // Time before a PROCESSING event is considered stuck\n getCollector?: (() => DynamoDBAwsSdkTransactionCollector | undefined) | undefined\n}\n\nexport class DynamoDBAwsSdkOutbox implements IOutbox<DynamoDBAwsSdkTransactionCollector> {\n private readonly config: Required<DynamoDBAwsSdkOutboxConfig>\n private readonly docClient: DynamoDBDocumentClient\n private readonly poller: PollingService\n\n constructor(config: DynamoDBAwsSdkOutboxConfig) {\n this.config = {\n batchSize: config.batchSize ?? 50,\n pollIntervalMs: config.pollIntervalMs ?? 1000,\n maxRetries: config.maxRetries ?? 5,\n baseBackoffMs: config.baseBackoffMs ?? 1000,\n processingTimeoutMs: config.processingTimeoutMs ?? 30000,\n maxErrorBackoffMs: config.maxErrorBackoffMs ?? 30000,\n tableName: config.tableName,\n statusIndexName: config.statusIndexName ?? \"status-gsiSortKey-index\",\n client: config.client,\n getCollector: config.getCollector,\n }\n\n this.docClient = DynamoDBDocumentClient.from(config.client, {\n marshallOptions: { removeUndefinedValues: true },\n })\n\n this.poller = new PollingService({\n pollIntervalMs: this.config.pollIntervalMs,\n baseBackoffMs: this.config.baseBackoffMs,\n maxErrorBackoffMs: this.config.maxErrorBackoffMs,\n performMaintenance: () => this.recoverStuckEvents(),\n processBatch: (handler) => this.processBatch(handler),\n })\n }\n\n async publish(\n events: BusEvent[],\n transaction?: DynamoDBAwsSdkTransactionCollector\n ): Promise<void> {\n if (events.length === 0) return\n\n if (events.length > DYNAMODB_TRANSACTION_LIMIT) {\n throw new BatchSizeLimitError(DYNAMODB_TRANSACTION_LIMIT, events.length)\n }\n\n const collector = transaction ?? this.config.getCollector?.()\n\n const items = events.map((event) => {\n return {\n Put: {\n TableName: this.config.tableName,\n Item: {\n id: event.id,\n type: event.type,\n payload: event.payload,\n occurredAt: event.occurredAt.toISOString(),\n status: EventStatus.CREATED,\n retryCount: 0,\n gsiSortKey: event.occurredAt.getTime(),\n },\n },\n }\n })\n\n if (collector) {\n const itemsInCollector = collector.items?.length ?? 0\n\n if (itemsInCollector + items.length > DYNAMODB_TRANSACTION_LIMIT) {\n throw new BatchSizeLimitError(DYNAMODB_TRANSACTION_LIMIT, itemsInCollector + items.length)\n }\n\n for (const item of items) {\n collector.push(item)\n }\n } else {\n await this.docClient.send(\n new TransactWriteCommand({\n TransactItems: items,\n })\n )\n }\n }\n\n async getFailedEvents(): Promise<FailedBusEvent[]> {\n const result = await this.docClient.send(\n new DocQueryCommand({\n TableName: this.config.tableName,\n IndexName: this.config.statusIndexName,\n KeyConditionExpression: \"#status = :status\",\n ExpressionAttributeNames: { \"#status\": \"status\" },\n ExpressionAttributeValues: { \":status\": EventStatus.FAILED },\n Limit: 100,\n ScanIndexForward: false,\n })\n )\n\n if (!result.Items) return []\n\n return result.Items.map((item) => {\n const event: FailedBusEvent = {\n id: item.id,\n type: item.type,\n payload: item.payload,\n occurredAt: this.parseOccurredAt(item.occurredAt),\n retryCount: item.retryCount || 0,\n }\n if (item.lastError) event.error = item.lastError\n if (item.startedOn) event.lastAttemptAt = new Date(item.startedOn)\n return event\n })\n }\n\n async retryEvents(eventIds: string[]): Promise<void> {\n if (eventIds.length === 0) return\n\n await Promise.all(\n eventIds.map((id) =>\n this.docClient.send(\n new UpdateCommand({\n TableName: this.config.tableName,\n Key: { id },\n UpdateExpression:\n \"SET #status = :pending, retryCount = :zero, gsiSortKey = :now REMOVE lastError, nextRetryAt, startedOn\",\n ExpressionAttributeNames: { \"#status\": \"status\" },\n ExpressionAttributeValues: {\n \":pending\": EventStatus.CREATED,\n \":zero\": 0,\n \":now\": Date.now(),\n },\n })\n )\n )\n )\n }\n\n start(handler: (event: BusEvent) => Promise<void>, onError: ErrorHandler): void {\n this.poller.start(handler, onError)\n }\n\n async stop(): Promise<void> {\n await this.poller.stop()\n }\n\n private async processBatch(handler: (event: BusEvent) => Promise<void>) {\n const now = Date.now()\n\n const result = await this.docClient.send(\n new DocQueryCommand({\n TableName: this.config.tableName,\n IndexName: this.config.statusIndexName,\n KeyConditionExpression: \"#status = :status AND gsiSortKey <= :now\",\n ExpressionAttributeNames: {\n \"#status\": \"status\",\n },\n ExpressionAttributeValues: {\n \":status\": EventStatus.CREATED,\n \":now\": now,\n },\n Limit: this.config.batchSize,\n })\n )\n\n if (!result.Items || result.Items.length === 0) return\n\n for (const item of result.Items) {\n const event: BusEvent = {\n id: item.id,\n type: item.type,\n payload: item.payload,\n occurredAt: this.parseOccurredAt(item.occurredAt),\n }\n\n try {\n await this.markEventAsProcessing(item.id, now)\n await handler(event)\n await this.markEventAsCompleted(event.id)\n } catch (error) {\n if (error instanceof ConditionalCheckFailedException) {\n continue\n }\n\n const newRetryCount = (item.retryCount || 0) + 1\n reportEventError(this.poller.onError, error, event, newRetryCount, this.config.maxRetries)\n\n await this.markEventAsFailed(item.id, newRetryCount, error)\n }\n }\n }\n\n private parseOccurredAt(occurredAt: unknown): Date {\n if (occurredAt instanceof Date) {\n return occurredAt\n }\n if (typeof occurredAt === \"string\") {\n return new Date(occurredAt)\n }\n return new Date()\n }\n\n private async markEventAsProcessing(id: string, now: number): Promise<void> {\n await this.docClient.send(\n new UpdateCommand({\n TableName: this.config.tableName,\n Key: { id },\n UpdateExpression: \"SET #status = :processing, gsiSortKey = :timeoutAt, startedOn = :now\",\n ExpressionAttributeNames: { \"#status\": \"status\" },\n ExpressionAttributeValues: {\n \":processing\": EventStatus.ACTIVE,\n \":timeoutAt\": now + this.config.processingTimeoutMs,\n \":now\": now,\n },\n })\n )\n }\n\n private async markEventAsCompleted(id: string): Promise<void> {\n await this.docClient.send(\n new UpdateCommand({\n TableName: this.config.tableName,\n Key: { id },\n UpdateExpression: \"SET #status = :completed, completedOn = :now REMOVE gsiSortKey\",\n ExpressionAttributeNames: { \"#status\": \"status\" },\n ExpressionAttributeValues: {\n \":completed\": EventStatus.COMPLETED,\n \":now\": Date.now(),\n },\n })\n )\n }\n\n private async markEventAsFailed(id: string, retryCount: number, error: unknown): Promise<void> {\n const isFinalFailure = retryCount >= this.config.maxRetries\n const status = isFinalFailure ? EventStatus.FAILED : EventStatus.CREATED\n const errorMsg = formatErrorMessage(error)\n const now = Date.now()\n\n const updateExpression = isFinalFailure\n ? \"SET #status = :status, retryCount = :rc, lastError = :err, nextRetryAt = :now REMOVE gsiSortKey\"\n : \"SET #status = :status, retryCount = :rc, lastError = :err, gsiSortKey = :nextAttempt\"\n\n const expressionAttributeValues: Record<string, any> = {\n \":status\": status,\n \":rc\": retryCount,\n \":err\": errorMsg,\n }\n\n if (isFinalFailure) {\n expressionAttributeValues[\":now\"] = now\n } else {\n const delay = this.poller.calculateBackoff(retryCount)\n expressionAttributeValues[\":nextAttempt\"] = now + delay\n }\n\n await this.docClient.send(\n new UpdateCommand({\n TableName: this.config.tableName,\n Key: { id },\n UpdateExpression: updateExpression,\n ExpressionAttributeNames: { \"#status\": \"status\" },\n ExpressionAttributeValues: expressionAttributeValues,\n })\n )\n }\n\n private async recoverStuckEvents() {\n const now = Date.now()\n\n const result = await this.docClient.send(\n new DocQueryCommand({\n TableName: this.config.tableName,\n IndexName: this.config.statusIndexName,\n KeyConditionExpression: \"#status = :status AND gsiSortKey <= :now\",\n ExpressionAttributeNames: { \"#status\": \"status\" },\n ExpressionAttributeValues: {\n \":status\": EventStatus.ACTIVE,\n \":now\": now,\n },\n })\n )\n\n if (result.Items && result.Items.length > 0) {\n await Promise.all(\n result.Items.map((item) =>\n this.markEventAsFailed(item.id, (item.retryCount || 0) + 1, \"Processing timeout\")\n )\n )\n }\n }\n}\n","import { AsyncLocalStorage } from \"node:async_hooks\"\nimport type {\n DynamoDBAwsSdkTransactionCollector,\n TransactWriteItem,\n} from \"./dynamodb-aws-sdk-outbox\"\n\nexport const dynamodbAwsSdkTransactionStorage =\n new AsyncLocalStorage<DynamoDBAwsSdkTransactionCollector>()\n\nexport async function withDynamoDBAwsSdkTransaction<T>(\n fn: (collector: DynamoDBAwsSdkTransactionCollector) => Promise<T>\n): Promise<T> {\n const items: TransactWriteItem[] = []\n const collector: DynamoDBAwsSdkTransactionCollector = {\n push: (item: TransactWriteItem) => items.push(item),\n get items() {\n return items\n },\n }\n return dynamodbAwsSdkTransactionStorage.run(collector, () => fn(collector))\n}\n\nexport function getDynamoDBAwsSdkCollector(): () => DynamoDBAwsSdkTransactionCollector | undefined {\n return () => dynamodbAwsSdkTransactionStorage.getStore()\n}\n"],"mappings":";;;;;;AAsBA,MAAM,6BAA6B;AAiBnC,IAAa,uBAAb,MAAyF;CACvF,AAAiB;CACjB,AAAiB;CACjB,AAAiB;CAEjB,YAAY,QAAoC;AAC9C,OAAK,SAAS;GACZ,WAAW,OAAO,aAAa;GAC/B,gBAAgB,OAAO,kBAAkB;GACzC,YAAY,OAAO,cAAc;GACjC,eAAe,OAAO,iBAAiB;GACvC,qBAAqB,OAAO,uBAAuB;GACnD,mBAAmB,OAAO,qBAAqB;GAC/C,WAAW,OAAO;GAClB,iBAAiB,OAAO,mBAAmB;GAC3C,QAAQ,OAAO;GACf,cAAc,OAAO;GACtB;AAED,OAAK,YAAY,uBAAuB,KAAK,OAAO,QAAQ,EAC1D,iBAAiB,EAAE,uBAAuB,MAAM,EACjD,CAAC;AAEF,OAAK,SAAS,IAAI,eAAe;GAC/B,gBAAgB,KAAK,OAAO;GAC5B,eAAe,KAAK,OAAO;GAC3B,mBAAmB,KAAK,OAAO;GAC/B,0BAA0B,KAAK,oBAAoB;GACnD,eAAe,YAAY,KAAK,aAAa,QAAQ;GACtD,CAAC;;CAGJ,MAAM,QACJ,QACA,aACe;AACf,MAAI,OAAO,WAAW,EAAG;AAEzB,MAAI,OAAO,SAAS,2BAClB,OAAM,IAAI,oBAAoB,4BAA4B,OAAO,OAAO;EAG1E,MAAM,YAAY,eAAe,KAAK,OAAO,gBAAgB;EAE7D,MAAM,QAAQ,OAAO,KAAK,UAAU;AAClC,UAAO,EACL,KAAK;IACH,WAAW,KAAK,OAAO;IACvB,MAAM;KACJ,IAAI,MAAM;KACV,MAAM,MAAM;KACZ,SAAS,MAAM;KACf,YAAY,MAAM,WAAW,aAAa;KAC1C,QAAQ,YAAY;KACpB,YAAY;KACZ,YAAY,MAAM,WAAW,SAAS;KACvC;IACF,EACF;IACD;AAEF,MAAI,WAAW;GACb,MAAM,mBAAmB,UAAU,OAAO,UAAU;AAEpD,OAAI,mBAAmB,MAAM,SAAS,2BACpC,OAAM,IAAI,oBAAoB,4BAA4B,mBAAmB,MAAM,OAAO;AAG5F,QAAK,MAAM,QAAQ,MACjB,WAAU,KAAK,KAAK;QAGtB,OAAM,KAAK,UAAU,KACnB,IAAI,qBAAqB,EACvB,eAAe,OAChB,CAAC,CACH;;CAIL,MAAM,kBAA6C;EACjD,MAAM,SAAS,MAAM,KAAK,UAAU,KAClC,IAAIA,aAAgB;GAClB,WAAW,KAAK,OAAO;GACvB,WAAW,KAAK,OAAO;GACvB,wBAAwB;GACxB,0BAA0B,EAAE,WAAW,UAAU;GACjD,2BAA2B,EAAE,WAAW,YAAY,QAAQ;GAC5D,OAAO;GACP,kBAAkB;GACnB,CAAC,CACH;AAED,MAAI,CAAC,OAAO,MAAO,QAAO,EAAE;AAE5B,SAAO,OAAO,MAAM,KAAK,SAAS;GAChC,MAAMC,QAAwB;IAC5B,IAAI,KAAK;IACT,MAAM,KAAK;IACX,SAAS,KAAK;IACd,YAAY,KAAK,gBAAgB,KAAK,WAAW;IACjD,YAAY,KAAK,cAAc;IAChC;AACD,OAAI,KAAK,UAAW,OAAM,QAAQ,KAAK;AACvC,OAAI,KAAK,UAAW,OAAM,gBAAgB,IAAI,KAAK,KAAK,UAAU;AAClE,UAAO;IACP;;CAGJ,MAAM,YAAY,UAAmC;AACnD,MAAI,SAAS,WAAW,EAAG;AAE3B,QAAM,QAAQ,IACZ,SAAS,KAAK,OACZ,KAAK,UAAU,KACb,IAAI,cAAc;GAChB,WAAW,KAAK,OAAO;GACvB,KAAK,EAAE,IAAI;GACX,kBACE;GACF,0BAA0B,EAAE,WAAW,UAAU;GACjD,2BAA2B;IACzB,YAAY,YAAY;IACxB,SAAS;IACT,QAAQ,KAAK,KAAK;IACnB;GACF,CAAC,CACH,CACF,CACF;;CAGH,MAAM,SAA6C,SAA6B;AAC9E,OAAK,OAAO,MAAM,SAAS,QAAQ;;CAGrC,MAAM,OAAsB;AAC1B,QAAM,KAAK,OAAO,MAAM;;CAG1B,MAAc,aAAa,SAA6C;EACtE,MAAM,MAAM,KAAK,KAAK;EAEtB,MAAM,SAAS,MAAM,KAAK,UAAU,KAClC,IAAID,aAAgB;GAClB,WAAW,KAAK,OAAO;GACvB,WAAW,KAAK,OAAO;GACvB,wBAAwB;GACxB,0BAA0B,EACxB,WAAW,UACZ;GACD,2BAA2B;IACzB,WAAW,YAAY;IACvB,QAAQ;IACT;GACD,OAAO,KAAK,OAAO;GACpB,CAAC,CACH;AAED,MAAI,CAAC,OAAO,SAAS,OAAO,MAAM,WAAW,EAAG;AAEhD,OAAK,MAAM,QAAQ,OAAO,OAAO;GAC/B,MAAME,QAAkB;IACtB,IAAI,KAAK;IACT,MAAM,KAAK;IACX,SAAS,KAAK;IACd,YAAY,KAAK,gBAAgB,KAAK,WAAW;IAClD;AAED,OAAI;AACF,UAAM,KAAK,sBAAsB,KAAK,IAAI,IAAI;AAC9C,UAAM,QAAQ,MAAM;AACpB,UAAM,KAAK,qBAAqB,MAAM,GAAG;YAClC,OAAO;AACd,QAAI,iBAAiB,gCACnB;IAGF,MAAM,iBAAiB,KAAK,cAAc,KAAK;AAC/C,qBAAiB,KAAK,OAAO,SAAS,OAAO,OAAO,eAAe,KAAK,OAAO,WAAW;AAE1F,UAAM,KAAK,kBAAkB,KAAK,IAAI,eAAe,MAAM;;;;CAKjE,AAAQ,gBAAgB,YAA2B;AACjD,MAAI,sBAAsB,KACxB,QAAO;AAET,MAAI,OAAO,eAAe,SACxB,QAAO,IAAI,KAAK,WAAW;AAE7B,yBAAO,IAAI,MAAM;;CAGnB,MAAc,sBAAsB,IAAY,KAA4B;AAC1E,QAAM,KAAK,UAAU,KACnB,IAAI,cAAc;GAChB,WAAW,KAAK,OAAO;GACvB,KAAK,EAAE,IAAI;GACX,kBAAkB;GAClB,0BAA0B,EAAE,WAAW,UAAU;GACjD,2BAA2B;IACzB,eAAe,YAAY;IAC3B,cAAc,MAAM,KAAK,OAAO;IAChC,QAAQ;IACT;GACF,CAAC,CACH;;CAGH,MAAc,qBAAqB,IAA2B;AAC5D,QAAM,KAAK,UAAU,KACnB,IAAI,cAAc;GAChB,WAAW,KAAK,OAAO;GACvB,KAAK,EAAE,IAAI;GACX,kBAAkB;GAClB,0BAA0B,EAAE,WAAW,UAAU;GACjD,2BAA2B;IACzB,cAAc,YAAY;IAC1B,QAAQ,KAAK,KAAK;IACnB;GACF,CAAC,CACH;;CAGH,MAAc,kBAAkB,IAAY,YAAoB,OAA+B;EAC7F,MAAM,iBAAiB,cAAc,KAAK,OAAO;EACjD,MAAM,SAAS,iBAAiB,YAAY,SAAS,YAAY;EACjE,MAAM,WAAW,mBAAmB,MAAM;EAC1C,MAAM,MAAM,KAAK,KAAK;EAEtB,MAAM,mBAAmB,iBACrB,oGACA;EAEJ,MAAMC,4BAAiD;GACrD,WAAW;GACX,OAAO;GACP,QAAQ;GACT;AAED,MAAI,eACF,2BAA0B,UAAU;MAGpC,2BAA0B,kBAAkB,MAD9B,KAAK,OAAO,iBAAiB,WAAW;AAIxD,QAAM,KAAK,UAAU,KACnB,IAAI,cAAc;GAChB,WAAW,KAAK,OAAO;GACvB,KAAK,EAAE,IAAI;GACX,kBAAkB;GAClB,0BAA0B,EAAE,WAAW,UAAU;GACjD,2BAA2B;GAC5B,CAAC,CACH;;CAGH,MAAc,qBAAqB;EACjC,MAAM,MAAM,KAAK,KAAK;EAEtB,MAAM,SAAS,MAAM,KAAK,UAAU,KAClC,IAAIH,aAAgB;GAClB,WAAW,KAAK,OAAO;GACvB,WAAW,KAAK,OAAO;GACvB,wBAAwB;GACxB,0BAA0B,EAAE,WAAW,UAAU;GACjD,2BAA2B;IACzB,WAAW,YAAY;IACvB,QAAQ;IACT;GACF,CAAC,CACH;AAED,MAAI,OAAO,SAAS,OAAO,MAAM,SAAS,EACxC,OAAM,QAAQ,IACZ,OAAO,MAAM,KAAK,SAChB,KAAK,kBAAkB,KAAK,KAAK,KAAK,cAAc,KAAK,GAAG,qBAAqB,CAClF,CACF;;;;;;AC3TP,MAAa,mCACX,IAAI,mBAAuD;AAE7D,eAAsB,8BACpB,IACY;CACZ,MAAMI,QAA6B,EAAE;CACrC,MAAMC,YAAgD;EACpD,OAAO,SAA4B,MAAM,KAAK,KAAK;EACnD,IAAI,QAAQ;AACV,UAAO;;EAEV;AACD,QAAO,iCAAiC,IAAI,iBAAiB,GAAG,UAAU,CAAC;;AAG7E,SAAgB,6BAAmF;AACjG,cAAa,iCAAiC,UAAU"}
|
package/package.json
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@outbox-event-bus/dynamodb-aws-sdk-outbox",
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"private": false,
|
|
5
|
+
"license": "MIT",
|
|
6
|
+
"repository": {
|
|
7
|
+
"type": "git",
|
|
8
|
+
"url": "https://github.com/dunika/outbox-event-bus.git",
|
|
9
|
+
"directory": "adapters/dynamodb-aws-sdk"
|
|
10
|
+
},
|
|
11
|
+
"homepage": "https://github.com/dunika/outbox-event-bus/tree/main/adapters/dynamodb-aws-sdk#readme",
|
|
12
|
+
"type": "module",
|
|
13
|
+
"main": "./dist/index.cjs",
|
|
14
|
+
"module": "./dist/index.mjs",
|
|
15
|
+
"types": "./dist/index.d.mts",
|
|
16
|
+
"exports": {
|
|
17
|
+
".": {
|
|
18
|
+
"import": {
|
|
19
|
+
"types": "./dist/index.d.mts",
|
|
20
|
+
"default": "./dist/index.mjs"
|
|
21
|
+
},
|
|
22
|
+
"require": {
|
|
23
|
+
"types": "./dist/index.d.cts",
|
|
24
|
+
"default": "./dist/index.cjs"
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
},
|
|
28
|
+
"files": [
|
|
29
|
+
"dist",
|
|
30
|
+
"src",
|
|
31
|
+
"README.md",
|
|
32
|
+
"LICENSE"
|
|
33
|
+
],
|
|
34
|
+
"scripts": {
|
|
35
|
+
"build": "tsdown",
|
|
36
|
+
"test": "vitest run",
|
|
37
|
+
"test:e2e": "docker-compose up -d --wait && vitest run -c vitest.e2e.config.ts; status=$?; docker-compose down; exit $status"
|
|
38
|
+
},
|
|
39
|
+
"dependencies": {
|
|
40
|
+
"outbox-event-bus": "workspace:*"
|
|
41
|
+
},
|
|
42
|
+
"peerDependencies": {
|
|
43
|
+
"@aws-sdk/client-dynamodb": "^3.956.0",
|
|
44
|
+
"@aws-sdk/lib-dynamodb": "^3.956.0"
|
|
45
|
+
},
|
|
46
|
+
"devDependencies": {
|
|
47
|
+
"@aws-sdk/client-dynamodb": "^3.956.0",
|
|
48
|
+
"@aws-sdk/lib-dynamodb": "^3.956.0",
|
|
49
|
+
"@outbox-event-bus/config": "workspace:*",
|
|
50
|
+
"@types/node": "catalog:",
|
|
51
|
+
"testcontainers": "catalog:",
|
|
52
|
+
"tsdown": "catalog:",
|
|
53
|
+
"typescript": "catalog:",
|
|
54
|
+
"vitest": "catalog:"
|
|
55
|
+
}
|
|
56
|
+
}
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
import { DynamoDBClient } from "@aws-sdk/client-dynamodb"
|
|
2
|
+
import { BatchSizeLimitError, EventStatus } from "outbox-event-bus"
|
|
3
|
+
import { beforeEach, describe, expect, it, vi } from "vitest"
|
|
4
|
+
import { DynamoDBAwsSdkOutbox } from "./index"
|
|
5
|
+
|
|
6
|
+
const mockSend = vi.fn()
|
|
7
|
+
|
|
8
|
+
vi.mock("@aws-sdk/lib-dynamodb", async () => {
|
|
9
|
+
const actual = (await vi.importActual("@aws-sdk/lib-dynamodb")) as any
|
|
10
|
+
return {
|
|
11
|
+
...actual,
|
|
12
|
+
DynamoDBDocumentClient: {
|
|
13
|
+
from: vi.fn(() => ({
|
|
14
|
+
send: mockSend,
|
|
15
|
+
})),
|
|
16
|
+
},
|
|
17
|
+
}
|
|
18
|
+
})
|
|
19
|
+
|
|
20
|
+
describe("DynamoDBAwsSdkOutbox Unit Tests", () => {
|
|
21
|
+
let outbox: DynamoDBAwsSdkOutbox
|
|
22
|
+
let client: DynamoDBClient
|
|
23
|
+
|
|
24
|
+
beforeEach(() => {
|
|
25
|
+
vi.clearAllMocks()
|
|
26
|
+
client = new DynamoDBClient({})
|
|
27
|
+
outbox = new DynamoDBAwsSdkOutbox({
|
|
28
|
+
client,
|
|
29
|
+
tableName: "test-table",
|
|
30
|
+
statusIndexName: "status-index",
|
|
31
|
+
})
|
|
32
|
+
})
|
|
33
|
+
|
|
34
|
+
it("should publish events", async () => {
|
|
35
|
+
const events = [
|
|
36
|
+
{ id: "e1", type: "t1", payload: { a: 1 }, occurredAt: new Date() },
|
|
37
|
+
{ id: "e2", type: "t2", payload: { b: 2 }, occurredAt: new Date() },
|
|
38
|
+
]
|
|
39
|
+
|
|
40
|
+
await outbox.publish(events)
|
|
41
|
+
|
|
42
|
+
expect(mockSend).toHaveBeenCalledWith(expect.any(Object))
|
|
43
|
+
})
|
|
44
|
+
|
|
45
|
+
it("should process a batch of events", async () => {
|
|
46
|
+
const mockItems = [
|
|
47
|
+
{
|
|
48
|
+
id: "e1",
|
|
49
|
+
type: "t1",
|
|
50
|
+
payload: { a: 1 },
|
|
51
|
+
occurredAt: new Date().toISOString(),
|
|
52
|
+
status: EventStatus.CREATED,
|
|
53
|
+
},
|
|
54
|
+
]
|
|
55
|
+
|
|
56
|
+
mockSend
|
|
57
|
+
.mockResolvedValueOnce({ Items: [] }) // recoverStuckEvents
|
|
58
|
+
.mockResolvedValueOnce({ Items: mockItems }) // fetch PENDING
|
|
59
|
+
.mockResolvedValueOnce({}) // claim update
|
|
60
|
+
|
|
61
|
+
outbox.start(
|
|
62
|
+
async () => {},
|
|
63
|
+
(err) => console.error(err)
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
// Wait for one poll cycle
|
|
67
|
+
await new Promise((res) => setTimeout(res, 200))
|
|
68
|
+
|
|
69
|
+
expect(mockSend).toHaveBeenCalled()
|
|
70
|
+
await outbox.stop()
|
|
71
|
+
})
|
|
72
|
+
|
|
73
|
+
it("should handle processing failure", async () => {
|
|
74
|
+
const mockItems = [
|
|
75
|
+
{
|
|
76
|
+
id: "e1",
|
|
77
|
+
type: "t1",
|
|
78
|
+
payload: { a: 1 },
|
|
79
|
+
occurredAt: new Date().toISOString(),
|
|
80
|
+
status: EventStatus.CREATED,
|
|
81
|
+
retryCount: 0,
|
|
82
|
+
},
|
|
83
|
+
]
|
|
84
|
+
|
|
85
|
+
mockSend
|
|
86
|
+
.mockResolvedValueOnce({ Items: [] }) // recoverStuckEvents
|
|
87
|
+
.mockResolvedValueOnce({ Items: mockItems }) // fetch PENDING
|
|
88
|
+
.mockResolvedValueOnce({}) // claim update
|
|
89
|
+
|
|
90
|
+
outbox.start(
|
|
91
|
+
async () => {
|
|
92
|
+
throw new Error("Failed")
|
|
93
|
+
},
|
|
94
|
+
(err) => console.error(err)
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
await new Promise((res) => setTimeout(res, 200))
|
|
98
|
+
await outbox.stop()
|
|
99
|
+
|
|
100
|
+
// Should have called update for retry
|
|
101
|
+
const retryUpdate = mockSend.mock.calls.find((call) =>
|
|
102
|
+
call[0].input?.UpdateExpression?.includes("retryCount")
|
|
103
|
+
)
|
|
104
|
+
expect(retryUpdate).toBeDefined()
|
|
105
|
+
})
|
|
106
|
+
|
|
107
|
+
it("should throw BatchSizeLimitError when publishing more than 100 events", async () => {
|
|
108
|
+
const events = Array.from({ length: 101 }, (_, i) => ({
|
|
109
|
+
id: `e${i}`,
|
|
110
|
+
type: "t",
|
|
111
|
+
payload: {},
|
|
112
|
+
occurredAt: new Date(),
|
|
113
|
+
}))
|
|
114
|
+
|
|
115
|
+
await expect(outbox.publish(events)).rejects.toThrow(BatchSizeLimitError)
|
|
116
|
+
})
|
|
117
|
+
})
|