@causa/runtime-google 0.33.1 → 0.34.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -2
- package/dist/pubsub/publisher.d.ts +3 -2
- package/dist/pubsub/publisher.js +20 -10
- package/dist/spanner/entity-manager.d.ts +22 -18
- package/dist/spanner/entity-manager.js +6 -6
- package/dist/spanner/index.d.ts +1 -1
- package/dist/transaction/index.d.ts +3 -0
- package/dist/transaction/index.js +3 -0
- package/dist/transaction/spanner-outbox/event.d.ts +27 -0
- package/dist/transaction/spanner-outbox/event.js +63 -0
- package/dist/transaction/spanner-outbox/index.d.ts +5 -0
- package/dist/transaction/spanner-outbox/index.js +4 -0
- package/dist/transaction/spanner-outbox/module.d.ts +28 -0
- package/dist/transaction/spanner-outbox/module.js +91 -0
- package/dist/transaction/spanner-outbox/runner.d.ts +19 -0
- package/dist/transaction/spanner-outbox/runner.js +31 -0
- package/dist/transaction/spanner-outbox/sender.d.ts +95 -0
- package/dist/transaction/spanner-outbox/sender.js +150 -0
- package/dist/transaction/spanner-pubsub/index.d.ts +0 -2
- package/dist/transaction/spanner-pubsub/index.js +0 -2
- package/dist/transaction/spanner-pubsub/runner.d.ts +3 -3
- package/dist/transaction/spanner-pubsub/runner.js +8 -25
- package/dist/transaction/{spanner-pubsub/state-transaction.d.ts → spanner-state-transaction.d.ts} +4 -5
- package/dist/transaction/{spanner-pubsub/state-transaction.js → spanner-state-transaction.js} +2 -3
- package/dist/transaction/spanner-transaction.d.ts +16 -0
- package/dist/transaction/spanner-transaction.js +20 -0
- package/dist/transaction/spanner-utils.d.ts +9 -0
- package/dist/transaction/spanner-utils.js +31 -0
- package/package.json +11 -11
- package/dist/transaction/spanner-pubsub/transaction.d.ts +0 -17
- package/dist/transaction/spanner-pubsub/transaction.js +0 -21
package/README.md
CHANGED
|
@@ -102,12 +102,24 @@ For testing, the `createDatabase` utility creates a temporary database, copying
|
|
|
102
102
|
|
|
103
103
|
### GCP-based Causa transaction runners
|
|
104
104
|
|
|
105
|
-
This package provides
|
|
105
|
+
This package provides the following `TransactionRunner`s:
|
|
106
106
|
|
|
107
|
-
|
|
107
|
+
- `SpannerPubSubTransactionRunner`
|
|
108
|
+
- `FirestorePubSubTransactionRunner`
|
|
109
|
+
- `SpannerOutboxTransactionRunner`
|
|
110
|
+
|
|
111
|
+
The first two use Pub/Sub and a `BufferEventTransaction` to publish events. They only differ by the service used to store the state.
|
|
112
|
+
|
|
113
|
+
The `SpannerPubSubTransactionRunner` uses a Spanner transaction as the underlying transaction for the `SpannerTransaction`, while the `FirestorePubSubTransactionRunner` uses a Firestore transaction for the `FirestorePubSubTransaction`. Both state transactions implement the `FindReplaceStateTransaction` interface, and therefore the runners can be used with the `VersionedEntityManager`.
|
|
108
114
|
|
|
109
115
|
One feature sets the `FirestorePubSubTransactionRunner` and its `FirestoreStateTransaction` apart: the handling of deleted entities using a separate, "soft-deleted document collection". Entities with a non-null `deletedAt` property are moved to a collection suffixed with `$deleted`, and an `_expirationDate` field is added to them. A TTL is expected to be set on this field. The `@SoftDeletedFirestoreCollection` decorator must be added to document classes that are meant to be handled using the `FirestorePubSubTransactionRunner`.
|
|
110
116
|
|
|
117
|
+
> [!CAUTION]
|
|
118
|
+
>
|
|
119
|
+
> `SpannerPubSubTransactionRunner` and `FirestorePubSubTransactionRunner` do not provide atomic guarantees between the state and the events being committed. This could result in events being lost, as they are published once the state transaction successfully committed. Prefer the `SpannerOutboxTransactionRunner` when applicable.
|
|
120
|
+
|
|
121
|
+
`SpannerOutboxTransactionRunner` implements the outbox pattern (from the base runtime's `OutboxTransactionRunner`), and uses the default injected `EventPublisher` (which can be the `PubSubPublisher`, if the corresponding module is imported). It requires an outbox table to be created in each database using the runner. See the documentation of the `SpannerOutboxEvent` for more information.
|
|
122
|
+
|
|
111
123
|
### Validation
|
|
112
124
|
|
|
113
125
|
The `@IsValidFirestoreId` validation decorator checks that a property is a string which is not `.` or `..`, and does not contain forward slashes. This ensures the property's value can be used as a Firestore document ID.
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type EventPublisher, type ObjectSerializer, type PublishOptions } from '@causa/runtime';
|
|
1
|
+
import { type EventPublisher, type ObjectSerializer, type PreparedEvent, type PublishOptions } from '@causa/runtime';
|
|
2
2
|
import { PubSub, type PublishOptions as TopicPublishOptions } from '@google-cloud/pubsub';
|
|
3
3
|
import type { OnApplicationShutdown } from '@nestjs/common';
|
|
4
4
|
import type { Logger } from 'pino';
|
|
@@ -88,7 +88,8 @@ export declare class PubSubPublisher implements EventPublisher, OnApplicationShu
|
|
|
88
88
|
* @returns The Pub/Sub {@link Topic} to which messages can be published.
|
|
89
89
|
*/
|
|
90
90
|
private getTopic;
|
|
91
|
-
|
|
91
|
+
prepare(topic: string, event: object, options?: PublishOptions): Promise<PreparedEvent>;
|
|
92
|
+
publish(topicOrPreparedEvent: string | PreparedEvent, event?: object, options?: PublishOptions): Promise<void>;
|
|
92
93
|
flush(): Promise<void>;
|
|
93
94
|
onApplicationShutdown(): Promise<void>;
|
|
94
95
|
}
|
package/dist/pubsub/publisher.js
CHANGED
|
@@ -85,17 +85,11 @@ export class PubSubPublisher {
|
|
|
85
85
|
this.topicCache[topicName] = topic;
|
|
86
86
|
return topic;
|
|
87
87
|
}
|
|
88
|
-
async
|
|
88
|
+
async prepare(topic, event, options = {}) {
|
|
89
89
|
const data = await this.serializer.serialize(event);
|
|
90
|
-
const pubSubTopic = this.getTopic(topic);
|
|
91
90
|
const defaultAttributes = {};
|
|
92
|
-
const baseLogData = {
|
|
93
|
-
topic,
|
|
94
|
-
pubSubTopic: pubSubTopic.name,
|
|
95
|
-
};
|
|
96
91
|
if ('id' in event && typeof event.id === 'string') {
|
|
97
92
|
defaultAttributes.eventId = event.id;
|
|
98
|
-
baseLogData.eventId = event.id;
|
|
99
93
|
}
|
|
100
94
|
if ('producedAt' in event && event.producedAt instanceof Date) {
|
|
101
95
|
defaultAttributes.producedAt = event.producedAt.toISOString();
|
|
@@ -105,14 +99,29 @@ export class PubSubPublisher {
|
|
|
105
99
|
}
|
|
106
100
|
const attributes = {
|
|
107
101
|
...defaultAttributes,
|
|
108
|
-
...options
|
|
102
|
+
...options?.attributes,
|
|
103
|
+
};
|
|
104
|
+
const key = options?.key;
|
|
105
|
+
return { topic, data, attributes, key };
|
|
106
|
+
}
|
|
107
|
+
async publish(topicOrPreparedEvent, event, options = {}) {
|
|
108
|
+
const isPrepared = typeof topicOrPreparedEvent !== 'string';
|
|
109
|
+
const { topic, data, attributes, key } = isPrepared
|
|
110
|
+
? topicOrPreparedEvent
|
|
111
|
+
: await this.prepare(topicOrPreparedEvent, event, options);
|
|
112
|
+
const pubSubTopic = this.getTopic(topic);
|
|
113
|
+
const baseLogData = {
|
|
114
|
+
topic,
|
|
115
|
+
pubSubTopic: pubSubTopic.name,
|
|
109
116
|
};
|
|
110
|
-
|
|
117
|
+
if (attributes && 'eventId' in attributes) {
|
|
118
|
+
baseLogData.eventId = attributes.eventId;
|
|
119
|
+
}
|
|
111
120
|
try {
|
|
112
121
|
const pubSubMessageId = await pubSubTopic.publishMessage({
|
|
113
122
|
data,
|
|
114
123
|
attributes,
|
|
115
|
-
orderingKey,
|
|
124
|
+
orderingKey: key,
|
|
116
125
|
});
|
|
117
126
|
this.logger.info({ ...baseLogData, pubSubMessageId }, 'Published message to Pub/Sub.');
|
|
118
127
|
}
|
|
@@ -121,6 +130,7 @@ export class PubSubPublisher {
|
|
|
121
130
|
...baseLogData,
|
|
122
131
|
pubSubMessage: data.toString('base64'),
|
|
123
132
|
pubSubAttributes: attributes,
|
|
133
|
+
pubSubOrderingKey: key,
|
|
124
134
|
errorMessage: error.message,
|
|
125
135
|
errorStack: error.stack,
|
|
126
136
|
}, 'Failed to publish message to Pub/Sub.');
|
|
@@ -8,6 +8,10 @@ import type { RecursivePartialEntity } from './types.js';
|
|
|
8
8
|
* Any Spanner transaction that can be used for reading.
|
|
9
9
|
*/
|
|
10
10
|
export type SpannerReadOnlyTransaction = Snapshot | Transaction;
|
|
11
|
+
/**
|
|
12
|
+
* A Spanner transaction that can be used for reading and writing.
|
|
13
|
+
*/
|
|
14
|
+
export type SpannerReadWriteTransaction = Transaction;
|
|
11
15
|
/**
|
|
12
16
|
* A key for a Spanner row.
|
|
13
17
|
*/
|
|
@@ -17,16 +21,16 @@ export type SpannerKey = (string | null)[];
|
|
|
17
21
|
*/
|
|
18
22
|
type WriteOperationOptions = {
|
|
19
23
|
/**
|
|
20
|
-
* The {@link
|
|
24
|
+
* The {@link SpannerReadWriteTransaction} to use.
|
|
21
25
|
*/
|
|
22
|
-
transaction?:
|
|
26
|
+
transaction?: SpannerReadWriteTransaction;
|
|
23
27
|
};
|
|
24
28
|
/**
|
|
25
29
|
* Base options for all read operations.
|
|
26
30
|
*/
|
|
27
31
|
type ReadOperationOptions = {
|
|
28
32
|
/**
|
|
29
|
-
* The {@link
|
|
33
|
+
* The {@link SpannerReadOnlyTransaction} to use.
|
|
30
34
|
*/
|
|
31
35
|
transaction?: SpannerReadOnlyTransaction;
|
|
32
36
|
};
|
|
@@ -42,7 +46,7 @@ type SnapshotOptions = {
|
|
|
42
46
|
/**
|
|
43
47
|
* A function that can be passed to the {@link SpannerEntityManager.snapshot} method.
|
|
44
48
|
*/
|
|
45
|
-
export type SnapshotFunction<T> = (snapshot:
|
|
49
|
+
export type SnapshotFunction<T> = (snapshot: SpannerReadOnlyTransaction) => Promise<T>;
|
|
46
50
|
/**
|
|
47
51
|
* A SQL statement run using {@link SpannerEntityManager.query}.
|
|
48
52
|
*/
|
|
@@ -196,16 +200,16 @@ export declare class SpannerEntityManager {
|
|
|
196
200
|
*/
|
|
197
201
|
findOneByKeyOrFail<T>(entityType: Type<T>, key: SpannerKey | SpannerKey[number], options?: FindOptions): Promise<T>;
|
|
198
202
|
/**
|
|
199
|
-
* Runs the provided function in a (read write) {@link
|
|
203
|
+
* Runs the provided function in a (read write) {@link SpannerReadWriteTransaction}.
|
|
200
204
|
* The function itself should not commit or rollback the transaction.
|
|
201
205
|
* If the function throws an error, the transaction will be rolled back.
|
|
202
206
|
*
|
|
203
207
|
* @param runFn The function to run in the transaction.
|
|
204
208
|
* @returns The return value of the function.
|
|
205
209
|
*/
|
|
206
|
-
transaction<T>(runFn: (transaction:
|
|
210
|
+
transaction<T>(runFn: (transaction: SpannerReadWriteTransaction) => Promise<T>): Promise<T>;
|
|
207
211
|
/**
|
|
208
|
-
* Runs the provided function in a
|
|
212
|
+
* Runs the provided function in a {@link SpannerReadOnlyTransaction}.
|
|
209
213
|
* The snapshot will be automatically released when the function returns.
|
|
210
214
|
*
|
|
211
215
|
* @param runFn The function to run in the transaction.
|
|
@@ -213,7 +217,7 @@ export declare class SpannerEntityManager {
|
|
|
213
217
|
*/
|
|
214
218
|
snapshot<T>(runFn: SnapshotFunction<T>): Promise<T>;
|
|
215
219
|
/**
|
|
216
|
-
* Runs the provided function in a
|
|
220
|
+
* Runs the provided function in a {@link SpannerReadOnlyTransaction}.
|
|
217
221
|
* The snapshot will be automatically released when the function returns.
|
|
218
222
|
*
|
|
219
223
|
* @param options The options to use when creating the snapshot.
|
|
@@ -230,8 +234,8 @@ export declare class SpannerEntityManager {
|
|
|
230
234
|
clear(entityType: Type, options?: WriteOperationOptions): Promise<void>;
|
|
231
235
|
/**
|
|
232
236
|
* Runs the given SQL statement in the database.
|
|
233
|
-
* By default, the statement is run in a
|
|
234
|
-
*
|
|
237
|
+
* By default, the statement is run in a {@link SpannerReadOnlyTransaction}. To perform a write operation, pass a
|
|
238
|
+
* {@link SpannerReadWriteTransaction} in the options.
|
|
235
239
|
*
|
|
236
240
|
* @param options Options for the operation.
|
|
237
241
|
* @param statement The SQL statement to run.
|
|
@@ -241,7 +245,7 @@ export declare class SpannerEntityManager {
|
|
|
241
245
|
query<T>(options: QueryOptions<T>, statement: SqlStatement): Promise<T[]>;
|
|
242
246
|
/**
|
|
243
247
|
* Runs the given SQL statement in the database.
|
|
244
|
-
* The statement is run in a
|
|
248
|
+
* The statement is run in a {@link SpannerReadOnlyTransaction}.
|
|
245
249
|
*
|
|
246
250
|
* @param statement The SQL statement to run.
|
|
247
251
|
* @returns The rows returned by the query.
|
|
@@ -331,22 +335,22 @@ export declare class SpannerEntityManager {
|
|
|
331
335
|
validateFn?: (entity: T) => void;
|
|
332
336
|
}): Promise<T>;
|
|
333
337
|
/**
|
|
334
|
-
* Runs the given "read-write" function on a transaction. If a transaction is not passed, a new
|
|
335
|
-
* created instead.
|
|
338
|
+
* Runs the given "read-write" function on a transaction. If a transaction is not passed, a new
|
|
339
|
+
* {@link SpannerReadWriteTransaction} is created instead.
|
|
336
340
|
*
|
|
337
341
|
* @param transaction The transaction to use. If `undefined`, a new transaction is created.
|
|
338
342
|
* @param fn The function to run on the transaction.
|
|
339
343
|
* @returns The result of the function.
|
|
340
344
|
*/
|
|
341
|
-
runInExistingOrNewTransaction<T>(transaction:
|
|
345
|
+
runInExistingOrNewTransaction<T>(transaction: SpannerReadWriteTransaction | undefined, fn: (transaction: SpannerReadWriteTransaction) => Promise<T>): Promise<T>;
|
|
342
346
|
/**
|
|
343
|
-
* Runs the given "read-only" function on a transaction. If a transaction is not passed, a new
|
|
344
|
-
* created instead.
|
|
347
|
+
* Runs the given "read-only" function on a transaction. If a transaction is not passed, a new
|
|
348
|
+
* {@link SpannerReadOnlyTransaction} is created instead.
|
|
345
349
|
*
|
|
346
|
-
* @param transaction The transaction to use. If `undefined`, a new {@link
|
|
350
|
+
* @param transaction The transaction to use. If `undefined`, a new {@link SpannerReadOnlyTransaction} is created.
|
|
347
351
|
* @param fn The function to run on the transaction.
|
|
348
352
|
* @returns The result of the function.
|
|
349
353
|
*/
|
|
350
|
-
runInExistingOrNewReadOnlyTransaction<T>(transaction: SpannerReadOnlyTransaction | undefined, fn:
|
|
354
|
+
runInExistingOrNewReadOnlyTransaction<T>(transaction: SpannerReadOnlyTransaction | undefined, fn: SnapshotFunction<T>): Promise<T>;
|
|
351
355
|
}
|
|
352
356
|
export {};
|
|
@@ -202,7 +202,7 @@ let SpannerEntityManager = class SpannerEntityManager {
|
|
|
202
202
|
return entity;
|
|
203
203
|
}
|
|
204
204
|
/**
|
|
205
|
-
* Runs the provided function in a (read write) {@link
|
|
205
|
+
* Runs the provided function in a (read write) {@link SpannerReadWriteTransaction}.
|
|
206
206
|
* The function itself should not commit or rollback the transaction.
|
|
207
207
|
* If the function throws an error, the transaction will be rolled back.
|
|
208
208
|
*
|
|
@@ -440,8 +440,8 @@ let SpannerEntityManager = class SpannerEntityManager {
|
|
|
440
440
|
});
|
|
441
441
|
}
|
|
442
442
|
/**
|
|
443
|
-
* Runs the given "read-write" function on a transaction. If a transaction is not passed, a new
|
|
444
|
-
* created instead.
|
|
443
|
+
* Runs the given "read-write" function on a transaction. If a transaction is not passed, a new
|
|
444
|
+
* {@link SpannerReadWriteTransaction} is created instead.
|
|
445
445
|
*
|
|
446
446
|
* @param transaction The transaction to use. If `undefined`, a new transaction is created.
|
|
447
447
|
* @param fn The function to run on the transaction.
|
|
@@ -459,10 +459,10 @@ let SpannerEntityManager = class SpannerEntityManager {
|
|
|
459
459
|
return this.transaction(fn);
|
|
460
460
|
}
|
|
461
461
|
/**
|
|
462
|
-
* Runs the given "read-only" function on a transaction. If a transaction is not passed, a new
|
|
463
|
-
* created instead.
|
|
462
|
+
* Runs the given "read-only" function on a transaction. If a transaction is not passed, a new
|
|
463
|
+
* {@link SpannerReadOnlyTransaction} is created instead.
|
|
464
464
|
*
|
|
465
|
-
* @param transaction The transaction to use. If `undefined`, a new {@link
|
|
465
|
+
* @param transaction The transaction to use. If `undefined`, a new {@link SpannerReadOnlyTransaction} is created.
|
|
466
466
|
* @param fn The function to run on the transaction.
|
|
467
467
|
* @returns The result of the function.
|
|
468
468
|
*/
|
package/dist/spanner/index.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
export { SpannerColumn } from './column.decorator.js';
|
|
2
2
|
export { SPANNER_SESSION_POOL_OPTIONS_FOR_CLOUD_FUNCTIONS, SPANNER_SESSION_POOL_OPTIONS_FOR_SERVICE, catchSpannerDatabaseErrors, getDefaultSpannerDatabaseForCloudFunction, } from './database.js';
|
|
3
3
|
export { SpannerEntityManager } from './entity-manager.js';
|
|
4
|
-
export type { SpannerKey, SpannerReadOnlyTransaction, } from './entity-manager.js';
|
|
4
|
+
export type { SpannerKey, SpannerReadOnlyTransaction, SpannerReadWriteTransaction, } from './entity-manager.js';
|
|
5
5
|
export * from './errors.js';
|
|
6
6
|
export { SpannerHealthIndicator } from './healthcheck.js';
|
|
7
7
|
export { SpannerModule } from './module.js';
|
|
@@ -1,2 +1,5 @@
|
|
|
1
1
|
export * from './firestore-pubsub/index.js';
|
|
2
|
+
export * from './spanner-outbox/index.js';
|
|
2
3
|
export * from './spanner-pubsub/index.js';
|
|
4
|
+
export { SpannerStateTransaction } from './spanner-state-transaction.js';
|
|
5
|
+
export { SpannerTransaction } from './spanner-transaction.js';
|
|
@@ -1,2 +1,5 @@
|
|
|
1
1
|
export * from './firestore-pubsub/index.js';
|
|
2
|
+
export * from './spanner-outbox/index.js';
|
|
2
3
|
export * from './spanner-pubsub/index.js';
|
|
4
|
+
export { SpannerStateTransaction } from './spanner-state-transaction.js';
|
|
5
|
+
export { SpannerTransaction } from './spanner-transaction.js';
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import type { EventAttributes, OutboxEvent } from '@causa/runtime';
|
|
2
|
+
/**
|
|
3
|
+
* A Spanner table that implements the {@link OutboxEvent} interface, such that it can be used to store outbox event.
|
|
4
|
+
*
|
|
5
|
+
* The full DDL for the table to be used by the outbox transaction runner is:
|
|
6
|
+
*
|
|
7
|
+
* ```sql
|
|
8
|
+
* CREATE TABLE OutboxEvent (
|
|
9
|
+
* id STRING(36) NOT NULL,
|
|
10
|
+
* topic STRING(MAX) NOT NULL,
|
|
11
|
+
* data BYTES(MAX) NOT NULL,
|
|
12
|
+
* attributes JSON NOT NULL,
|
|
13
|
+
* leaseExpiration TIMESTAMP,
|
|
14
|
+
* -- 20 is the number of shards.
|
|
15
|
+
* shard INT64 NOT NULL AS (MOD(ABS(FARM_FINGERPRINT(id)), 20)),
|
|
16
|
+
* ) PRIMARY KEY (id);
|
|
17
|
+
* CREATE INDEX OutboxEventsByShardAndLeaseExpiration ON OutboxEvent(shard, leaseExpiration)
|
|
18
|
+
* ```
|
|
19
|
+
*/
|
|
20
|
+
export declare class SpannerOutboxEvent implements OutboxEvent {
|
|
21
|
+
constructor(init: SpannerOutboxEvent);
|
|
22
|
+
readonly id: string;
|
|
23
|
+
readonly topic: string;
|
|
24
|
+
readonly data: Buffer;
|
|
25
|
+
readonly attributes: EventAttributes;
|
|
26
|
+
readonly leaseExpiration: Date | null;
|
|
27
|
+
}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
2
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
3
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
4
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
5
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
6
|
+
};
|
|
7
|
+
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
8
|
+
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
9
|
+
};
|
|
10
|
+
import { SpannerColumn, SpannerTable } from '../../spanner/index.js';
|
|
11
|
+
/**
|
|
12
|
+
* A Spanner table that implements the {@link OutboxEvent} interface, such that it can be used to store outbox event.
|
|
13
|
+
*
|
|
14
|
+
* The full DDL for the table to be used by the outbox transaction runner is:
|
|
15
|
+
*
|
|
16
|
+
* ```sql
|
|
17
|
+
* CREATE TABLE OutboxEvent (
|
|
18
|
+
* id STRING(36) NOT NULL,
|
|
19
|
+
* topic STRING(MAX) NOT NULL,
|
|
20
|
+
* data BYTES(MAX) NOT NULL,
|
|
21
|
+
* attributes JSON NOT NULL,
|
|
22
|
+
* leaseExpiration TIMESTAMP,
|
|
23
|
+
* -- 20 is the number of shards.
|
|
24
|
+
* shard INT64 NOT NULL AS (MOD(ABS(FARM_FINGERPRINT(id)), 20)),
|
|
25
|
+
* ) PRIMARY KEY (id);
|
|
26
|
+
* CREATE INDEX OutboxEventsByShardAndLeaseExpiration ON OutboxEvent(shard, leaseExpiration)
|
|
27
|
+
* ```
|
|
28
|
+
*/
|
|
29
|
+
let SpannerOutboxEvent = class SpannerOutboxEvent {
|
|
30
|
+
constructor(init) {
|
|
31
|
+
Object.assign(this, init);
|
|
32
|
+
}
|
|
33
|
+
id;
|
|
34
|
+
topic;
|
|
35
|
+
data;
|
|
36
|
+
attributes;
|
|
37
|
+
leaseExpiration;
|
|
38
|
+
};
|
|
39
|
+
__decorate([
|
|
40
|
+
SpannerColumn(),
|
|
41
|
+
__metadata("design:type", String)
|
|
42
|
+
], SpannerOutboxEvent.prototype, "id", void 0);
|
|
43
|
+
__decorate([
|
|
44
|
+
SpannerColumn(),
|
|
45
|
+
__metadata("design:type", String)
|
|
46
|
+
], SpannerOutboxEvent.prototype, "topic", void 0);
|
|
47
|
+
__decorate([
|
|
48
|
+
SpannerColumn(),
|
|
49
|
+
__metadata("design:type", Buffer)
|
|
50
|
+
], SpannerOutboxEvent.prototype, "data", void 0);
|
|
51
|
+
__decorate([
|
|
52
|
+
SpannerColumn({ isJson: true }),
|
|
53
|
+
__metadata("design:type", Object)
|
|
54
|
+
], SpannerOutboxEvent.prototype, "attributes", void 0);
|
|
55
|
+
__decorate([
|
|
56
|
+
SpannerColumn(),
|
|
57
|
+
__metadata("design:type", Object)
|
|
58
|
+
], SpannerOutboxEvent.prototype, "leaseExpiration", void 0);
|
|
59
|
+
SpannerOutboxEvent = __decorate([
|
|
60
|
+
SpannerTable({ name: 'OutboxEvent', primaryKey: ['id'] }),
|
|
61
|
+
__metadata("design:paramtypes", [SpannerOutboxEvent])
|
|
62
|
+
], SpannerOutboxEvent);
|
|
63
|
+
export { SpannerOutboxEvent };
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
export { SpannerOutboxEvent } from './event.js';
|
|
2
|
+
export { SpannerOutboxTransactionModule } from './module.js';
|
|
3
|
+
export { SpannerOutboxTransactionRunner } from './runner.js';
|
|
4
|
+
export type { SpannerOutboxTransaction } from './runner.js';
|
|
5
|
+
export { SpannerOutboxSender } from './sender.js';
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import type { OutboxEvent } from '@causa/runtime';
|
|
2
|
+
import type { DynamicModule, Type } from '@nestjs/common';
|
|
3
|
+
import { type SpannerOutboxSenderOptions } from './sender.js';
|
|
4
|
+
/**
|
|
5
|
+
* Options for the {@link SpannerOutboxTransactionModule}.
|
|
6
|
+
*/
|
|
7
|
+
export type SpannerOutboxTransactionModuleOptions = SpannerOutboxSenderOptions & {
|
|
8
|
+
/**
|
|
9
|
+
* The type of {@link OutboxEvent} used by the {@link SpannerOutboxTransactionRunner}.
|
|
10
|
+
* This should be a valid class decorated with `@SpannerTable`.
|
|
11
|
+
* Defaults to {@link SpannerOutboxEvent}.
|
|
12
|
+
*/
|
|
13
|
+
outboxEventType?: Type<OutboxEvent>;
|
|
14
|
+
};
|
|
15
|
+
/**
|
|
16
|
+
* The module providing the {@link SpannerOutboxTransactionRunner}.
|
|
17
|
+
* This assumes the `SpannerModule` and an {@link EventPublisher} are available (as well as the `LoggerModule`).
|
|
18
|
+
*/
|
|
19
|
+
export declare class SpannerOutboxTransactionModule {
|
|
20
|
+
/**
|
|
21
|
+
* Initializes the {@link SpannerOutboxTransactionModule} with the given options.
|
|
22
|
+
* The returned module is always global.
|
|
23
|
+
*
|
|
24
|
+
* @param options Options for the {@link SpannerOutboxTransactionModule}.
|
|
25
|
+
* @returns The module.
|
|
26
|
+
*/
|
|
27
|
+
static forRoot(options?: SpannerOutboxTransactionModuleOptions): DynamicModule;
|
|
28
|
+
}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import { EVENT_PUBLISHER_INJECTION_NAME, Logger } from '@causa/runtime/nestjs';
|
|
2
|
+
import { ConfigService } from '@nestjs/config';
|
|
3
|
+
import { SpannerEntityManager } from '../../spanner/index.js';
|
|
4
|
+
import { SpannerOutboxEvent } from './event.js';
|
|
5
|
+
import { SpannerOutboxTransactionRunner } from './runner.js';
|
|
6
|
+
import { SpannerOutboxSender, } from './sender.js';
|
|
7
|
+
/**
|
|
8
|
+
* Combines options passed to the module with the configuration (from the environment).
|
|
9
|
+
*
|
|
10
|
+
* @param options Options passed to the module.
|
|
11
|
+
* @param config The {@link ConfigService} to use.
|
|
12
|
+
* @returns The parsed {@link SpannerOutboxSenderOptions}.
|
|
13
|
+
*/
|
|
14
|
+
function parseSenderOptions(options, config) {
|
|
15
|
+
function validateIntOrUndefined(name) {
|
|
16
|
+
const strValue = config.get(name);
|
|
17
|
+
if (strValue === undefined) {
|
|
18
|
+
return undefined;
|
|
19
|
+
}
|
|
20
|
+
const value = parseInt(strValue);
|
|
21
|
+
if (isNaN(value)) {
|
|
22
|
+
throw new Error(`Environment variable ${name} must be a number.`);
|
|
23
|
+
}
|
|
24
|
+
return value;
|
|
25
|
+
}
|
|
26
|
+
const batchSize = validateIntOrUndefined('SPANNER_OUTBOX_BATCH_SIZE');
|
|
27
|
+
const pollingInterval = validateIntOrUndefined('SPANNER_OUTBOX_POLLING_INTERVAL');
|
|
28
|
+
const idColumn = config.get('SPANNER_OUTBOX_ID_COLUMN');
|
|
29
|
+
const leaseExpirationColumn = config.get('SPANNER_OUTBOX_LEASE_EXPIRATION_COLUMN');
|
|
30
|
+
const index = config.get('SPANNER_OUTBOX_INDEX');
|
|
31
|
+
const shardingColumn = config.get('SPANNER_OUTBOX_SHARDING_COLUMN');
|
|
32
|
+
const shardingCount = validateIntOrUndefined('SPANNER_OUTBOX_SHARDING_COUNT');
|
|
33
|
+
const leaseDuration = validateIntOrUndefined('SPANNER_OUTBOX_LEASE_DURATION');
|
|
34
|
+
const sharding = shardingColumn && shardingCount
|
|
35
|
+
? { column: shardingColumn, count: shardingCount }
|
|
36
|
+
: undefined;
|
|
37
|
+
const envOptions = {
|
|
38
|
+
batchSize,
|
|
39
|
+
pollingInterval,
|
|
40
|
+
idColumn,
|
|
41
|
+
leaseExpirationColumn,
|
|
42
|
+
index,
|
|
43
|
+
sharding,
|
|
44
|
+
leaseDuration,
|
|
45
|
+
};
|
|
46
|
+
return { ...envOptions, ...options };
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* The module providing the {@link SpannerOutboxTransactionRunner}.
|
|
50
|
+
* This assumes the `SpannerModule` and an {@link EventPublisher} are available (as well as the `LoggerModule`).
|
|
51
|
+
*/
|
|
52
|
+
export class SpannerOutboxTransactionModule {
|
|
53
|
+
/**
|
|
54
|
+
* Initializes the {@link SpannerOutboxTransactionModule} with the given options.
|
|
55
|
+
* The returned module is always global.
|
|
56
|
+
*
|
|
57
|
+
* @param options Options for the {@link SpannerOutboxTransactionModule}.
|
|
58
|
+
* @returns The module.
|
|
59
|
+
*/
|
|
60
|
+
static forRoot(options = {}) {
|
|
61
|
+
const { outboxEventType, ...senderOptions } = {
|
|
62
|
+
outboxEventType: SpannerOutboxEvent,
|
|
63
|
+
...options,
|
|
64
|
+
};
|
|
65
|
+
return {
|
|
66
|
+
module: SpannerOutboxTransactionModule,
|
|
67
|
+
global: true,
|
|
68
|
+
providers: [
|
|
69
|
+
{
|
|
70
|
+
provide: SpannerOutboxSender,
|
|
71
|
+
useFactory: (entityManager, publisher, logger, config) => {
|
|
72
|
+
const options = parseSenderOptions(senderOptions, config);
|
|
73
|
+
return new SpannerOutboxSender(entityManager, outboxEventType, publisher, logger, options);
|
|
74
|
+
},
|
|
75
|
+
inject: [
|
|
76
|
+
SpannerEntityManager,
|
|
77
|
+
EVENT_PUBLISHER_INJECTION_NAME,
|
|
78
|
+
Logger,
|
|
79
|
+
ConfigService,
|
|
80
|
+
],
|
|
81
|
+
},
|
|
82
|
+
{
|
|
83
|
+
provide: SpannerOutboxTransactionRunner,
|
|
84
|
+
useFactory: (entityManager, sender, logger) => new SpannerOutboxTransactionRunner(entityManager, outboxEventType, sender, logger),
|
|
85
|
+
inject: [SpannerEntityManager, SpannerOutboxSender, Logger],
|
|
86
|
+
},
|
|
87
|
+
],
|
|
88
|
+
exports: [SpannerOutboxTransactionRunner],
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { OutboxTransactionRunner, type OutboxEvent, type OutboxEventTransaction } from '@causa/runtime';
|
|
2
|
+
import { Logger } from '@causa/runtime/nestjs';
|
|
3
|
+
import type { Type } from '@nestjs/common';
|
|
4
|
+
import { SpannerEntityManager } from '../../spanner/index.js';
|
|
5
|
+
import { SpannerTransaction } from '../spanner-transaction.js';
|
|
6
|
+
import { SpannerOutboxSender } from './sender.js';
|
|
7
|
+
/**
|
|
8
|
+
* A {@link SpannerTransaction} that uses an {@link OutboxEventTransaction}.
|
|
9
|
+
*/
|
|
10
|
+
export type SpannerOutboxTransaction = SpannerTransaction<OutboxEventTransaction>;
|
|
11
|
+
/**
|
|
12
|
+
* An {@link OutboxTransactionRunner} that uses a {@link SpannerTransaction} to run transactions.
|
|
13
|
+
* Events are stored in a Spanner table before being published.
|
|
14
|
+
*/
|
|
15
|
+
export declare class SpannerOutboxTransactionRunner extends OutboxTransactionRunner<SpannerOutboxTransaction> {
|
|
16
|
+
readonly entityManager: SpannerEntityManager;
|
|
17
|
+
constructor(entityManager: SpannerEntityManager, outboxEventType: Type<OutboxEvent>, sender: SpannerOutboxSender, logger: Logger);
|
|
18
|
+
protected runStateTransaction<RT>(eventTransaction: OutboxEventTransaction, runFn: (transaction: SpannerOutboxTransaction) => Promise<RT>): Promise<RT>;
|
|
19
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { OutboxTransactionRunner, } from '@causa/runtime';
|
|
2
|
+
import { Logger } from '@causa/runtime/nestjs';
|
|
3
|
+
import { SpannerEntityManager } from '../../spanner/index.js';
|
|
4
|
+
import { SpannerStateTransaction } from '../spanner-state-transaction.js';
|
|
5
|
+
import { SpannerTransaction } from '../spanner-transaction.js';
|
|
6
|
+
import { throwRetryableInTransactionIfNeeded } from '../spanner-utils.js';
|
|
7
|
+
import { SpannerOutboxSender } from './sender.js';
|
|
8
|
+
/**
|
|
9
|
+
* An {@link OutboxTransactionRunner} that uses a {@link SpannerTransaction} to run transactions.
|
|
10
|
+
* Events are stored in a Spanner table before being published.
|
|
11
|
+
*/
|
|
12
|
+
export class SpannerOutboxTransactionRunner extends OutboxTransactionRunner {
|
|
13
|
+
entityManager;
|
|
14
|
+
constructor(entityManager, outboxEventType, sender, logger) {
|
|
15
|
+
super(outboxEventType, sender, logger);
|
|
16
|
+
this.entityManager = entityManager;
|
|
17
|
+
}
|
|
18
|
+
async runStateTransaction(eventTransaction, runFn) {
|
|
19
|
+
return await this.entityManager.transaction(async (dbTransaction) => {
|
|
20
|
+
const stateTransaction = new SpannerStateTransaction(this.entityManager, dbTransaction);
|
|
21
|
+
const transaction = new SpannerTransaction(stateTransaction, eventTransaction);
|
|
22
|
+
try {
|
|
23
|
+
return await runFn(transaction);
|
|
24
|
+
}
|
|
25
|
+
catch (error) {
|
|
26
|
+
await throwRetryableInTransactionIfNeeded(error);
|
|
27
|
+
throw error;
|
|
28
|
+
}
|
|
29
|
+
});
|
|
30
|
+
}
|
|
31
|
+
}
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
import { OutboxEventSender, type EventPublisher, type OutboxEvent, type OutboxEventPublishResult, type OutboxEventSenderOptions } from '@causa/runtime';
|
|
2
|
+
import { Logger } from '@causa/runtime/nestjs';
|
|
3
|
+
import type { Type } from '@nestjs/common';
|
|
4
|
+
import { SpannerEntityManager } from '../../spanner/index.js';
|
|
5
|
+
/**
|
|
6
|
+
* Sharding options for the {@link SpannerOutboxSender}.
|
|
7
|
+
*/
|
|
8
|
+
export type SpannerOutboxSenderShardingOptions = {
|
|
9
|
+
/**
|
|
10
|
+
* The name of the column used for sharding.
|
|
11
|
+
*/
|
|
12
|
+
readonly column: string;
|
|
13
|
+
/**
|
|
14
|
+
* The number of shards.
|
|
15
|
+
*/
|
|
16
|
+
readonly count: number;
|
|
17
|
+
};
|
|
18
|
+
/**
|
|
19
|
+
* Options for the {@link SpannerOutboxSender}.
|
|
20
|
+
*/
|
|
21
|
+
export type SpannerOutboxSenderOptions = OutboxEventSenderOptions & {
|
|
22
|
+
/**
|
|
23
|
+
* Sharding options.
|
|
24
|
+
* If not set, queries to fetch events will not use sharding.
|
|
25
|
+
*/
|
|
26
|
+
readonly sharding?: SpannerOutboxSenderShardingOptions;
|
|
27
|
+
/**
|
|
28
|
+
* The name of the column used to store the event ID.
|
|
29
|
+
* Defaults to `id`.
|
|
30
|
+
*/
|
|
31
|
+
readonly idColumn?: string;
|
|
32
|
+
/**
|
|
33
|
+
* The name of the column used to store the lease expiration.
|
|
34
|
+
* Defaults to `leaseExpiration`.
|
|
35
|
+
*/
|
|
36
|
+
readonly leaseExpirationColumn?: string;
|
|
37
|
+
/**
|
|
38
|
+
* The index used to fetch events.
|
|
39
|
+
*/
|
|
40
|
+
readonly index?: string;
|
|
41
|
+
};
|
|
42
|
+
/**
|
|
43
|
+
* An {@link OutboxEventSender} that uses a Spanner table to store events.
|
|
44
|
+
*/
|
|
45
|
+
export declare class SpannerOutboxSender extends OutboxEventSender {
|
|
46
|
+
readonly entityManager: SpannerEntityManager;
|
|
47
|
+
readonly outboxEventType: Type<OutboxEvent>;
|
|
48
|
+
/**
|
|
49
|
+
* Sharding options.
|
|
50
|
+
* If `null`, queries to fetch events will not use sharding.
|
|
51
|
+
*/
|
|
52
|
+
readonly sharding: SpannerOutboxSenderShardingOptions | undefined;
|
|
53
|
+
/**
|
|
54
|
+
* The name of the column used for the {@link OutboxEvent.id} property.
|
|
55
|
+
*/
|
|
56
|
+
readonly idColumn: string;
|
|
57
|
+
/**
|
|
58
|
+
* The name of the column used for the {@link OutboxEvent.leaseExpiration} property.
|
|
59
|
+
*/
|
|
60
|
+
readonly leaseExpirationColumn: string;
|
|
61
|
+
/**
|
|
62
|
+
* The index used to fetch events.
|
|
63
|
+
*/
|
|
64
|
+
readonly index: string | undefined;
|
|
65
|
+
/**
|
|
66
|
+
* The SQL query used to fetch events from the outbox.
|
|
67
|
+
*/
|
|
68
|
+
readonly fetchEventsSql: string;
|
|
69
|
+
/**
|
|
70
|
+
* The SQL query used to update events in the outbox after they have been successfully published.
|
|
71
|
+
*/
|
|
72
|
+
readonly successfulUpdateSql: string;
|
|
73
|
+
/**
|
|
74
|
+
* The SQL query used to update events in the outbox after they have failed to be published.
|
|
75
|
+
*/
|
|
76
|
+
readonly failedUpdateSql: string;
|
|
77
|
+
/**
|
|
78
|
+
* Creates a new {@link SpannerOutboxSender}.
|
|
79
|
+
*
|
|
80
|
+
* @param entityManager The {@link SpannerEntityManager} to use to access the outbox.
|
|
81
|
+
* @param outboxEventType The type for the Spanner table used to store outbox events.
|
|
82
|
+
* @param publisher The {@link EventPublisher} to use to publish events.
|
|
83
|
+
* @param logger The {@link Logger} to use.
|
|
84
|
+
* @param options Options for the {@link SpannerOutboxSender}.
|
|
85
|
+
*/
|
|
86
|
+
constructor(entityManager: SpannerEntityManager, outboxEventType: Type<OutboxEvent>, publisher: EventPublisher, logger: Logger, options?: SpannerOutboxSenderOptions);
|
|
87
|
+
/**
|
|
88
|
+
* Builds the SQL queries used to fetch and update events in the outbox based on the options.
|
|
89
|
+
*
|
|
90
|
+
* @returns The SQL queries used to fetch and update events in the outbox.
|
|
91
|
+
*/
|
|
92
|
+
protected buildSql(): Pick<SpannerOutboxSender, 'fetchEventsSql' | 'successfulUpdateSql' | 'failedUpdateSql'>;
|
|
93
|
+
protected fetchEvents(): Promise<OutboxEvent[]>;
|
|
94
|
+
protected updateOutbox(result: OutboxEventPublishResult): Promise<void>;
|
|
95
|
+
}
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
import { OutboxEventSender, } from '@causa/runtime';
|
|
2
|
+
import { Logger } from '@causa/runtime/nestjs';
|
|
3
|
+
import { SpannerEntityManager } from '../../spanner/index.js';
|
|
4
|
+
/**
|
|
5
|
+
* The default name for the {@link OutboxEvent.id} column.
|
|
6
|
+
*/
|
|
7
|
+
const DEFAULT_ID_COLUMN = 'id';
|
|
8
|
+
/**
|
|
9
|
+
* The default name for the {@link OutboxEvent.leaseExpiration} column.
|
|
10
|
+
*/
|
|
11
|
+
const DEFAULT_LEASE_EXPIRATION_COLUMN = 'leaseExpiration';
|
|
12
|
+
/**
|
|
13
|
+
* An {@link OutboxEventSender} that uses a Spanner table to store events.
|
|
14
|
+
*/
|
|
15
|
+
export class SpannerOutboxSender extends OutboxEventSender {
|
|
16
|
+
entityManager;
|
|
17
|
+
outboxEventType;
|
|
18
|
+
/**
|
|
19
|
+
* Sharding options.
|
|
20
|
+
* If `null`, queries to fetch events will not use sharding.
|
|
21
|
+
*/
|
|
22
|
+
sharding;
|
|
23
|
+
/**
|
|
24
|
+
* The name of the column used for the {@link OutboxEvent.id} property.
|
|
25
|
+
*/
|
|
26
|
+
idColumn;
|
|
27
|
+
/**
|
|
28
|
+
* The name of the column used for the {@link OutboxEvent.leaseExpiration} property.
|
|
29
|
+
*/
|
|
30
|
+
leaseExpirationColumn;
|
|
31
|
+
/**
|
|
32
|
+
* The index used to fetch events.
|
|
33
|
+
*/
|
|
34
|
+
index;
|
|
35
|
+
/**
|
|
36
|
+
* The SQL query used to fetch events from the outbox.
|
|
37
|
+
*/
|
|
38
|
+
fetchEventsSql;
|
|
39
|
+
/**
|
|
40
|
+
* The SQL query used to update events in the outbox after they have been successfully published.
|
|
41
|
+
*/
|
|
42
|
+
successfulUpdateSql;
|
|
43
|
+
/**
|
|
44
|
+
* The SQL query used to update events in the outbox after they have failed to be published.
|
|
45
|
+
*/
|
|
46
|
+
failedUpdateSql;
|
|
47
|
+
/**
|
|
48
|
+
* Creates a new {@link SpannerOutboxSender}.
|
|
49
|
+
*
|
|
50
|
+
* @param entityManager The {@link SpannerEntityManager} to use to access the outbox.
|
|
51
|
+
* @param outboxEventType The type for the Spanner table used to store outbox events.
|
|
52
|
+
* @param publisher The {@link EventPublisher} to use to publish events.
|
|
53
|
+
* @param logger The {@link Logger} to use.
|
|
54
|
+
* @param options Options for the {@link SpannerOutboxSender}.
|
|
55
|
+
*/
|
|
56
|
+
constructor(entityManager, outboxEventType, publisher, logger, options = {}) {
|
|
57
|
+
super(publisher, logger, options);
|
|
58
|
+
this.entityManager = entityManager;
|
|
59
|
+
this.outboxEventType = outboxEventType;
|
|
60
|
+
this.sharding = options.sharding;
|
|
61
|
+
this.idColumn = options.idColumn ?? DEFAULT_ID_COLUMN;
|
|
62
|
+
this.leaseExpirationColumn =
|
|
63
|
+
options.leaseExpirationColumn ?? DEFAULT_LEASE_EXPIRATION_COLUMN;
|
|
64
|
+
this.index = options.index;
|
|
65
|
+
({
|
|
66
|
+
fetchEventsSql: this.fetchEventsSql,
|
|
67
|
+
successfulUpdateSql: this.successfulUpdateSql,
|
|
68
|
+
failedUpdateSql: this.failedUpdateSql,
|
|
69
|
+
} = this.buildSql());
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Builds the SQL queries used to fetch and update events in the outbox based on the options.
|
|
73
|
+
*
|
|
74
|
+
* @returns The SQL queries used to fetch and update events in the outbox.
|
|
75
|
+
*/
|
|
76
|
+
buildSql() {
|
|
77
|
+
const table = this.entityManager.sqlTableName(this.outboxEventType);
|
|
78
|
+
const tableWithIndex = this.entityManager.sqlTableName(this.outboxEventType, { index: this.index });
|
|
79
|
+
let filter = `${this.leaseExpirationColumn} IS NULL OR ${this.leaseExpirationColumn} < @currentTime`;
|
|
80
|
+
if (this.sharding) {
|
|
81
|
+
const { column, count } = this.sharding;
|
|
82
|
+
filter = `${column} BETWEEN 0 AND ${count - 1} AND (${filter})`;
|
|
83
|
+
}
|
|
84
|
+
const fetchEventsSql = `
|
|
85
|
+
UPDATE
|
|
86
|
+
${table}
|
|
87
|
+
SET
|
|
88
|
+
\`${this.leaseExpirationColumn}\` = @leaseExpiration
|
|
89
|
+
WHERE
|
|
90
|
+
\`${this.idColumn}\` IN (
|
|
91
|
+
SELECT
|
|
92
|
+
\`${this.idColumn}\`
|
|
93
|
+
FROM
|
|
94
|
+
${tableWithIndex}
|
|
95
|
+
WHERE
|
|
96
|
+
${filter}
|
|
97
|
+
LIMIT
|
|
98
|
+
@batchSize
|
|
99
|
+
)
|
|
100
|
+
THEN RETURN
|
|
101
|
+
${this.entityManager.sqlColumns(this.outboxEventType)}`;
|
|
102
|
+
const successfulUpdateSql = `
|
|
103
|
+
DELETE FROM
|
|
104
|
+
${table}
|
|
105
|
+
WHERE
|
|
106
|
+
\`${this.idColumn}\` IN UNNEST(@ids)`;
|
|
107
|
+
const failedUpdateSql = `
|
|
108
|
+
UPDATE
|
|
109
|
+
${table}
|
|
110
|
+
SET
|
|
111
|
+
\`${this.leaseExpirationColumn}\` = NULL
|
|
112
|
+
WHERE
|
|
113
|
+
\`${this.idColumn}\` IN UNNEST(@ids)`;
|
|
114
|
+
return { fetchEventsSql, successfulUpdateSql, failedUpdateSql };
|
|
115
|
+
}
|
|
116
|
+
async fetchEvents() {
|
|
117
|
+
return await this.entityManager.transaction(async (transaction) => {
|
|
118
|
+
const currentTime = new Date();
|
|
119
|
+
const leaseExpiration = new Date(currentTime.getTime() + this.leaseDuration);
|
|
120
|
+
const params = {
|
|
121
|
+
leaseExpiration,
|
|
122
|
+
currentTime,
|
|
123
|
+
batchSize: this.batchSize,
|
|
124
|
+
};
|
|
125
|
+
return await this.entityManager.query({ transaction, entityType: this.outboxEventType }, { sql: this.fetchEventsSql, params });
|
|
126
|
+
});
|
|
127
|
+
}
|
|
128
|
+
async updateOutbox(result) {
|
|
129
|
+
const successfulSends = [];
|
|
130
|
+
const failedSends = [];
|
|
131
|
+
Object.entries(result).forEach(([id, success]) => (success ? successfulSends : failedSends).push(id));
|
|
132
|
+
const batchUpdates = [];
|
|
133
|
+
if (successfulSends.length > 0) {
|
|
134
|
+
batchUpdates.push({
|
|
135
|
+
sql: this.successfulUpdateSql,
|
|
136
|
+
params: { ids: successfulSends },
|
|
137
|
+
});
|
|
138
|
+
}
|
|
139
|
+
if (failedSends.length > 0) {
|
|
140
|
+
batchUpdates.push({
|
|
141
|
+
sql: this.failedUpdateSql,
|
|
142
|
+
params: { ids: failedSends },
|
|
143
|
+
});
|
|
144
|
+
}
|
|
145
|
+
if (batchUpdates.length === 0) {
|
|
146
|
+
return;
|
|
147
|
+
}
|
|
148
|
+
await this.entityManager.transaction((transaction) => transaction.batchUpdate(batchUpdates));
|
|
149
|
+
}
|
|
150
|
+
}
|
|
@@ -2,12 +2,12 @@ import { TransactionRunner } from '@causa/runtime';
|
|
|
2
2
|
import { Logger } from '@causa/runtime/nestjs';
|
|
3
3
|
import { PubSubPublisher } from '../../pubsub/index.js';
|
|
4
4
|
import { SpannerEntityManager } from '../../spanner/index.js';
|
|
5
|
-
import {
|
|
5
|
+
import { SpannerTransaction } from '../spanner-transaction.js';
|
|
6
6
|
/**
|
|
7
7
|
* A {@link TransactionRunner} that uses Spanner for state and Pub/Sub for events.
|
|
8
8
|
* A Spanner transaction is used as the main transaction. If it succeeds, events are published to Pub/Sub outside of it.
|
|
9
9
|
*/
|
|
10
|
-
export declare class SpannerPubSubTransactionRunner extends TransactionRunner<
|
|
10
|
+
export declare class SpannerPubSubTransactionRunner extends TransactionRunner<SpannerTransaction> {
|
|
11
11
|
readonly entityManager: SpannerEntityManager;
|
|
12
12
|
readonly publisher: PubSubPublisher;
|
|
13
13
|
private readonly logger;
|
|
@@ -19,5 +19,5 @@ export declare class SpannerPubSubTransactionRunner extends TransactionRunner<Sp
|
|
|
19
19
|
* @param logger The {@link Logger} to use.
|
|
20
20
|
*/
|
|
21
21
|
constructor(entityManager: SpannerEntityManager, publisher: PubSubPublisher, logger: Logger);
|
|
22
|
-
run<T>(runFn: (transaction:
|
|
22
|
+
run<T>(runFn: (transaction: SpannerTransaction) => Promise<T>): Promise<[T]>;
|
|
23
23
|
}
|
|
@@ -7,18 +7,14 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
|
|
|
7
7
|
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
8
8
|
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
9
9
|
};
|
|
10
|
-
import { BufferEventTransaction,
|
|
10
|
+
import { BufferEventTransaction, TransactionRunner } from '@causa/runtime';
|
|
11
11
|
import { Logger } from '@causa/runtime/nestjs';
|
|
12
12
|
import { Injectable } from '@nestjs/common';
|
|
13
|
-
import { setTimeout } from 'timers/promises';
|
|
14
13
|
import { PubSubPublisher } from '../../pubsub/index.js';
|
|
15
|
-
import { SpannerEntityManager
|
|
16
|
-
import { SpannerStateTransaction } from '
|
|
17
|
-
import {
|
|
18
|
-
|
|
19
|
-
* The delay, in milliseconds, over which a timestamp issue is deemed irrecoverable.
|
|
20
|
-
*/
|
|
21
|
-
const ACCEPTABLE_PAST_DATE_DELAY = 25000;
|
|
14
|
+
import { SpannerEntityManager } from '../../spanner/index.js';
|
|
15
|
+
import { SpannerStateTransaction } from '../spanner-state-transaction.js';
|
|
16
|
+
import { SpannerTransaction } from '../spanner-transaction.js';
|
|
17
|
+
import { throwRetryableInTransactionIfNeeded } from '../spanner-utils.js';
|
|
22
18
|
/**
|
|
23
19
|
* A {@link TransactionRunner} that uses Spanner for state and Pub/Sub for events.
|
|
24
20
|
* A Spanner transaction is used as the main transaction. If it succeeds, events are published to Pub/Sub outside of it.
|
|
@@ -46,28 +42,15 @@ let SpannerPubSubTransactionRunner = class SpannerPubSubTransactionRunner extend
|
|
|
46
42
|
const stateTransaction = new SpannerStateTransaction(this.entityManager, dbTransaction);
|
|
47
43
|
// This must be inside the Spanner transaction because staged messages should be cleared when the transaction is retried.
|
|
48
44
|
const eventTransaction = new BufferEventTransaction(this.publisher);
|
|
49
|
-
const transaction = new
|
|
45
|
+
const transaction = new SpannerTransaction(stateTransaction, eventTransaction);
|
|
50
46
|
try {
|
|
51
47
|
const result = await runFn(transaction);
|
|
52
48
|
this.logger.info('Committing the Spanner transaction.');
|
|
53
49
|
return { result, eventTransaction };
|
|
54
50
|
}
|
|
55
51
|
catch (error) {
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
// Throwing a `SpannerTransactionOldTimestampError` will cause the transaction to be retried with a newer
|
|
59
|
-
// timestamp.
|
|
60
|
-
if (!(error instanceof TransactionOldTimestampError)) {
|
|
61
|
-
throw error;
|
|
62
|
-
}
|
|
63
|
-
const delay = error.delay ?? Infinity;
|
|
64
|
-
if (delay >= ACCEPTABLE_PAST_DATE_DELAY) {
|
|
65
|
-
throw error;
|
|
66
|
-
}
|
|
67
|
-
if (delay > 0) {
|
|
68
|
-
await setTimeout(delay);
|
|
69
|
-
}
|
|
70
|
-
throw TemporarySpannerError.retryableInTransaction(error.message);
|
|
52
|
+
await throwRetryableInTransactionIfNeeded(error);
|
|
53
|
+
throw error;
|
|
71
54
|
}
|
|
72
55
|
});
|
|
73
56
|
this.logger.info('Publishing Pub/Sub events.');
|
package/dist/transaction/{spanner-pubsub/state-transaction.d.ts → spanner-state-transaction.d.ts}
RENAMED
|
@@ -1,20 +1,19 @@
|
|
|
1
1
|
import type { FindReplaceStateTransaction } from '@causa/runtime';
|
|
2
|
-
import { Transaction as SpannerTransaction } from '@google-cloud/spanner';
|
|
3
2
|
import type { Type } from '@nestjs/common';
|
|
4
|
-
import { SpannerEntityManager } from '
|
|
3
|
+
import { SpannerEntityManager, type SpannerReadWriteTransaction } from '../spanner/index.js';
|
|
5
4
|
/**
|
|
6
5
|
* A {@link FindReplaceStateTransaction} that uses Spanner for state storage.
|
|
7
6
|
*/
|
|
8
7
|
export declare class SpannerStateTransaction implements FindReplaceStateTransaction {
|
|
9
8
|
readonly entityManager: SpannerEntityManager;
|
|
10
|
-
readonly transaction:
|
|
9
|
+
readonly transaction: SpannerReadWriteTransaction;
|
|
11
10
|
/**
|
|
12
11
|
* Creates a new {@link SpannerStateTransaction}.
|
|
13
12
|
*
|
|
14
13
|
* @param entityManager The {@link SpannerEntityManager} to use to access entities in the state.
|
|
15
|
-
* @param transaction The {@link
|
|
14
|
+
* @param transaction The {@link SpannerReadWriteTransaction} to use for the transaction.
|
|
16
15
|
*/
|
|
17
|
-
constructor(entityManager: SpannerEntityManager, transaction:
|
|
16
|
+
constructor(entityManager: SpannerEntityManager, transaction: SpannerReadWriteTransaction);
|
|
18
17
|
replace<T extends object>(entity: T): Promise<void>;
|
|
19
18
|
deleteWithSameKeyAs<T extends object>(type: Type<T>, key: Partial<T>): Promise<void>;
|
|
20
19
|
findOneWithSameKeyAs<T extends object>(type: Type<T>, entity: Partial<T>): Promise<T | undefined>;
|
package/dist/transaction/{spanner-pubsub/state-transaction.js → spanner-state-transaction.js}
RENAMED
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { SpannerEntityManager } from '../../spanner/index.js';
|
|
1
|
+
import { SpannerEntityManager, } from '../spanner/index.js';
|
|
3
2
|
/**
|
|
4
3
|
* A {@link FindReplaceStateTransaction} that uses Spanner for state storage.
|
|
5
4
|
*/
|
|
@@ -10,7 +9,7 @@ export class SpannerStateTransaction {
|
|
|
10
9
|
* Creates a new {@link SpannerStateTransaction}.
|
|
11
10
|
*
|
|
12
11
|
* @param entityManager The {@link SpannerEntityManager} to use to access entities in the state.
|
|
13
|
-
* @param transaction The {@link
|
|
12
|
+
* @param transaction The {@link SpannerReadWriteTransaction} to use for the transaction.
|
|
14
13
|
*/
|
|
15
14
|
constructor(entityManager, transaction) {
|
|
16
15
|
this.entityManager = entityManager;
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { type EventTransaction, Transaction } from '@causa/runtime';
|
|
2
|
+
import { SpannerEntityManager, type SpannerReadWriteTransaction } from '../spanner/index.js';
|
|
3
|
+
import { SpannerStateTransaction } from './spanner-state-transaction.js';
|
|
4
|
+
/**
|
|
5
|
+
* A {@link Transaction} that uses Spanner for state storage, and any available {@link EventTransaction} implementation.
|
|
6
|
+
*/
|
|
7
|
+
export declare class SpannerTransaction<ET extends EventTransaction = EventTransaction> extends Transaction<SpannerStateTransaction, ET> {
|
|
8
|
+
/**
|
|
9
|
+
* The underlying {@link SpannerTransaction} used by the state transaction.
|
|
10
|
+
*/
|
|
11
|
+
get spannerTransaction(): SpannerReadWriteTransaction;
|
|
12
|
+
/**
|
|
13
|
+
* The underlying {@link SpannerEntityManager} used by the state transaction.
|
|
14
|
+
*/
|
|
15
|
+
get entityManager(): SpannerEntityManager;
|
|
16
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { Transaction } from '@causa/runtime';
|
|
2
|
+
import { SpannerEntityManager, } from '../spanner/index.js';
|
|
3
|
+
import { SpannerStateTransaction } from './spanner-state-transaction.js';
|
|
4
|
+
/**
|
|
5
|
+
* A {@link Transaction} that uses Spanner for state storage, and any available {@link EventTransaction} implementation.
|
|
6
|
+
*/
|
|
7
|
+
export class SpannerTransaction extends Transaction {
|
|
8
|
+
/**
|
|
9
|
+
* The underlying {@link SpannerTransaction} used by the state transaction.
|
|
10
|
+
*/
|
|
11
|
+
get spannerTransaction() {
|
|
12
|
+
return this.stateTransaction.transaction;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* The underlying {@link SpannerEntityManager} used by the state transaction.
|
|
16
|
+
*/
|
|
17
|
+
get entityManager() {
|
|
18
|
+
return this.stateTransaction.entityManager;
|
|
19
|
+
}
|
|
20
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Checks if the given error is a `TransactionOldTimestampError` and throws a `TemporarySpannerError` such that the
|
|
3
|
+
* transaction can be retried by the runner.
|
|
4
|
+
* If the {@link TransactionOldTimestampError.delay} is too large, the error is deemed irrecoverable and nothing is
|
|
5
|
+
* thrown (it is up to the caller to handle the error).
|
|
6
|
+
*
|
|
7
|
+
* @param error The error to test.
|
|
8
|
+
*/
|
|
9
|
+
export declare function throwRetryableInTransactionIfNeeded(error: unknown): Promise<void>;
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { TransactionOldTimestampError } from '@causa/runtime';
|
|
2
|
+
import { setTimeout } from 'timers/promises';
|
|
3
|
+
import { TemporarySpannerError } from '../spanner/index.js';
|
|
4
|
+
/**
|
|
5
|
+
* The delay, in milliseconds, over which a timestamp issue is deemed irrecoverable.
|
|
6
|
+
*/
|
|
7
|
+
const ACCEPTABLE_PAST_DATE_DELAY = 25000;
|
|
8
|
+
/**
|
|
9
|
+
* Checks if the given error is a `TransactionOldTimestampError` and throws a `TemporarySpannerError` such that the
|
|
10
|
+
* transaction can be retried by the runner.
|
|
11
|
+
* If the {@link TransactionOldTimestampError.delay} is too large, the error is deemed irrecoverable and nothing is
|
|
12
|
+
* thrown (it is up to the caller to handle the error).
|
|
13
|
+
*
|
|
14
|
+
* @param error The error to test.
|
|
15
|
+
*/
|
|
16
|
+
export async function throwRetryableInTransactionIfNeeded(error) {
|
|
17
|
+
// `TransactionOldTimestampError`s indicate that the transaction is using a timestamp older than what is
|
|
18
|
+
// observed in the state (Spanner).
|
|
19
|
+
if (!(error instanceof TransactionOldTimestampError)) {
|
|
20
|
+
return;
|
|
21
|
+
}
|
|
22
|
+
const delay = error.delay ?? Infinity;
|
|
23
|
+
if (delay >= ACCEPTABLE_PAST_DATE_DELAY) {
|
|
24
|
+
return;
|
|
25
|
+
}
|
|
26
|
+
if (delay > 0) {
|
|
27
|
+
await setTimeout(delay);
|
|
28
|
+
}
|
|
29
|
+
// Throwing a `TemporarySpannerError` will cause the transaction to be retried with a newer timestamp.
|
|
30
|
+
throw TemporarySpannerError.retryableInTransaction(error.message);
|
|
31
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@causa/runtime-google",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.34.0",
|
|
4
4
|
"description": "An extension to the Causa runtime SDK (`@causa/runtime`), providing Google-specific features.",
|
|
5
5
|
"repository": "github:causa-io/runtime-typescript-google",
|
|
6
6
|
"license": "ISC",
|
|
@@ -29,21 +29,21 @@
|
|
|
29
29
|
"test:cov": "npm run test -- --coverage"
|
|
30
30
|
},
|
|
31
31
|
"dependencies": {
|
|
32
|
-
"@causa/runtime": ">= 0.
|
|
32
|
+
"@causa/runtime": ">= 0.24.0 < 1.0.0",
|
|
33
33
|
"@google-cloud/precise-date": "^4.0.0",
|
|
34
34
|
"@google-cloud/pubsub": "^4.9.0",
|
|
35
35
|
"@google-cloud/spanner": "^7.16.0",
|
|
36
36
|
"@google-cloud/tasks": "^5.5.1",
|
|
37
37
|
"@grpc/grpc-js": "^1.12.2",
|
|
38
|
-
"@nestjs/common": "^10.4.
|
|
38
|
+
"@nestjs/common": "^10.4.11",
|
|
39
39
|
"@nestjs/config": "^3.3.0",
|
|
40
|
-
"@nestjs/core": "^10.4.
|
|
40
|
+
"@nestjs/core": "^10.4.11",
|
|
41
41
|
"@nestjs/passport": "^10.0.3",
|
|
42
42
|
"@nestjs/terminus": "^10.2.3",
|
|
43
43
|
"class-transformer": "^0.5.1",
|
|
44
44
|
"class-validator": "^0.14.1",
|
|
45
45
|
"express": "^4.21.1",
|
|
46
|
-
"firebase-admin": "^13.0.
|
|
46
|
+
"firebase-admin": "^13.0.1",
|
|
47
47
|
"jsonwebtoken": "^9.0.2",
|
|
48
48
|
"nestjs-pino": "^4.1.0",
|
|
49
49
|
"passport-http-bearer": "^1.0.1",
|
|
@@ -51,18 +51,18 @@
|
|
|
51
51
|
"reflect-metadata": "^0.2.2"
|
|
52
52
|
},
|
|
53
53
|
"devDependencies": {
|
|
54
|
-
"@nestjs/testing": "^10.4.
|
|
55
|
-
"@swc/core": "^1.9.
|
|
54
|
+
"@nestjs/testing": "^10.4.11",
|
|
55
|
+
"@swc/core": "^1.9.3",
|
|
56
56
|
"@swc/jest": "^0.2.37",
|
|
57
57
|
"@tsconfig/node22": "^22.0.0",
|
|
58
58
|
"@types/jest": "^29.5.14",
|
|
59
59
|
"@types/jsonwebtoken": "^9.0.7",
|
|
60
|
-
"@types/node": "^22.
|
|
60
|
+
"@types/node": "^22.10.0",
|
|
61
61
|
"@types/passport-http-bearer": "^1.0.41",
|
|
62
62
|
"@types/supertest": "^6.0.2",
|
|
63
63
|
"@types/uuid": "^10.0.0",
|
|
64
64
|
"dotenv": "^16.4.5",
|
|
65
|
-
"eslint": "^9.
|
|
65
|
+
"eslint": "^9.15.0",
|
|
66
66
|
"eslint-config-prettier": "^9.1.0",
|
|
67
67
|
"eslint-plugin-prettier": "^5.2.1",
|
|
68
68
|
"jest": "^29.7.0",
|
|
@@ -71,8 +71,8 @@
|
|
|
71
71
|
"supertest": "^7.0.0",
|
|
72
72
|
"ts-jest": "^29.2.5",
|
|
73
73
|
"ts-node": "^10.9.2",
|
|
74
|
-
"typescript": "^5.
|
|
75
|
-
"typescript-eslint": "^8.
|
|
74
|
+
"typescript": "^5.7.2",
|
|
75
|
+
"typescript-eslint": "^8.16.0",
|
|
76
76
|
"uuid": "^11.0.3"
|
|
77
77
|
}
|
|
78
78
|
}
|
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
import { BufferEventTransaction, Transaction } from '@causa/runtime';
|
|
2
|
-
import { Transaction as SpannerTransaction } from '@google-cloud/spanner';
|
|
3
|
-
import { SpannerEntityManager } from '../../spanner/index.js';
|
|
4
|
-
import { SpannerStateTransaction } from './state-transaction.js';
|
|
5
|
-
/**
|
|
6
|
-
* A {@link Transaction} that uses Spanner for state storage and Pub/Sub for event publishing.
|
|
7
|
-
*/
|
|
8
|
-
export declare class SpannerPubSubTransaction extends Transaction<SpannerStateTransaction, BufferEventTransaction> {
|
|
9
|
-
/**
|
|
10
|
-
* The underlying {@link SpannerTransaction} used by the state transaction.
|
|
11
|
-
*/
|
|
12
|
-
get spannerTransaction(): SpannerTransaction;
|
|
13
|
-
/**
|
|
14
|
-
* The underlying {@link SpannerEntityManager} used by the state transaction.
|
|
15
|
-
*/
|
|
16
|
-
get entityManager(): SpannerEntityManager;
|
|
17
|
-
}
|
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
import { BufferEventTransaction, Transaction } from '@causa/runtime';
|
|
2
|
-
import { Transaction as SpannerTransaction } from '@google-cloud/spanner';
|
|
3
|
-
import { SpannerEntityManager } from '../../spanner/index.js';
|
|
4
|
-
import { SpannerStateTransaction } from './state-transaction.js';
|
|
5
|
-
/**
|
|
6
|
-
* A {@link Transaction} that uses Spanner for state storage and Pub/Sub for event publishing.
|
|
7
|
-
*/
|
|
8
|
-
export class SpannerPubSubTransaction extends Transaction {
|
|
9
|
-
/**
|
|
10
|
-
* The underlying {@link SpannerTransaction} used by the state transaction.
|
|
11
|
-
*/
|
|
12
|
-
get spannerTransaction() {
|
|
13
|
-
return this.stateTransaction.transaction;
|
|
14
|
-
}
|
|
15
|
-
/**
|
|
16
|
-
* The underlying {@link SpannerEntityManager} used by the state transaction.
|
|
17
|
-
*/
|
|
18
|
-
get entityManager() {
|
|
19
|
-
return this.stateTransaction.entityManager;
|
|
20
|
-
}
|
|
21
|
-
}
|