@causa/runtime-google 1.0.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/pubsub/publisher.js +8 -23
- package/dist/spanner/entity-manager.d.ts +34 -4
- package/dist/spanner/entity-manager.js +39 -14
- package/dist/spanner/table-cache.d.ts +2 -10
- package/dist/spanner/table-cache.js +3 -7
- package/dist/spanner/types.d.ts +13 -1
- package/dist/spanner/types.js +2 -1
- package/dist/transaction/firestore-pubsub/runner.d.ts +1 -1
- package/dist/transaction/firestore-pubsub/runner.js +1 -1
- package/dist/transaction/spanner-outbox/module.js +6 -1
- package/dist/transaction/spanner-outbox/runner.d.ts +16 -2
- package/dist/transaction/spanner-outbox/runner.js +4 -1
- package/dist/transaction/spanner-outbox/sender.d.ts +23 -3
- package/dist/transaction/spanner-outbox/sender.js +39 -11
- package/package.json +24 -26
package/dist/pubsub/publisher.js
CHANGED
|
@@ -5,11 +5,10 @@ import { getConfigurationKeyForTopic } from './configuration.js';
|
|
|
5
5
|
import { PubSubTopicNotConfiguredError } from './errors.js';
|
|
6
6
|
/**
|
|
7
7
|
* The default options to use when publishing messages.
|
|
8
|
-
* Batching is
|
|
9
|
-
* important than throughput.
|
|
8
|
+
* Batching is minimal, as latency should be prioritized over throughput.
|
|
10
9
|
*/
|
|
11
10
|
const DEFAULT_PUBLISH_OPTIONS = {
|
|
12
|
-
batching: {
|
|
11
|
+
batching: { maxMilliseconds: 5 },
|
|
13
12
|
};
|
|
14
13
|
/**
|
|
15
14
|
* An implementation of the {@link EventPublisher} using Google Pub/Sub as the broker.
|
|
@@ -117,26 +116,12 @@ export class PubSubPublisher {
|
|
|
117
116
|
if (attributes && 'eventId' in attributes) {
|
|
118
117
|
messageInfo.eventId = attributes.eventId;
|
|
119
118
|
}
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
this.logger.info({ publishedMessage: { ...messageInfo, messageId } }, 'Published message to Pub/Sub.');
|
|
127
|
-
}
|
|
128
|
-
catch (error) {
|
|
129
|
-
this.logger.error({
|
|
130
|
-
failedMessage: {
|
|
131
|
-
...messageInfo,
|
|
132
|
-
data: data.toString('base64'),
|
|
133
|
-
attributes,
|
|
134
|
-
key,
|
|
135
|
-
},
|
|
136
|
-
error: error.stack,
|
|
137
|
-
}, 'Failed to publish message to Pub/Sub.');
|
|
138
|
-
throw error;
|
|
139
|
-
}
|
|
119
|
+
const messageId = await pubSubTopic.publishMessage({
|
|
120
|
+
data,
|
|
121
|
+
attributes,
|
|
122
|
+
orderingKey: key,
|
|
123
|
+
});
|
|
124
|
+
this.logger.info({ publishedMessage: { ...messageInfo, messageId } }, 'Published message to Pub/Sub.');
|
|
140
125
|
}
|
|
141
126
|
async flush() {
|
|
142
127
|
await Promise.all(Object.values(this.topicCache).map((topic) => topic.flush()));
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { Database, Snapshot, Transaction } from '@google-cloud/spanner';
|
|
2
2
|
import { type Type as ParamType } from '@google-cloud/spanner/build/src/codec.js';
|
|
3
|
-
import type { TimestampBounds } from '@google-cloud/spanner/build/src/transaction.js';
|
|
3
|
+
import type { ExecuteSqlRequest, TimestampBounds } from '@google-cloud/spanner/build/src/transaction.js';
|
|
4
4
|
import { type Type } from '@nestjs/common';
|
|
5
5
|
import { SpannerTableCache } from './table-cache.js';
|
|
6
6
|
import type { SpannerReadOnlyTransactionOption, SpannerReadWriteTransactionOption } from './types.js';
|
|
@@ -53,7 +53,7 @@ export type SqlStatement = {
|
|
|
53
53
|
/**
|
|
54
54
|
* Options for {@link SpannerEntityManager.query}.
|
|
55
55
|
*/
|
|
56
|
-
export type QueryOptions<T> = SpannerReadOnlyTransactionOption & {
|
|
56
|
+
export type QueryOptions<T> = SpannerReadOnlyTransactionOption & Pick<ExecuteSqlRequest, 'requestOptions'> & {
|
|
57
57
|
/**
|
|
58
58
|
* The type of entity to return in the list of results.
|
|
59
59
|
*/
|
|
@@ -62,7 +62,7 @@ export type QueryOptions<T> = SpannerReadOnlyTransactionOption & {
|
|
|
62
62
|
/**
|
|
63
63
|
* Options when reading entities.
|
|
64
64
|
*/
|
|
65
|
-
type FindOptions = SpannerReadOnlyTransactionOption & {
|
|
65
|
+
type FindOptions = SpannerReadOnlyTransactionOption & Pick<ExecuteSqlRequest, 'requestOptions'> & {
|
|
66
66
|
/**
|
|
67
67
|
* The index to use to look up the entity.
|
|
68
68
|
*/
|
|
@@ -114,6 +114,8 @@ export declare class SpannerEntityManager {
|
|
|
114
114
|
/**
|
|
115
115
|
* Returns the (quoted) name of the table for the given entity type.
|
|
116
116
|
*
|
|
117
|
+
* @deprecated Use {@link sqlTable} instead.
|
|
118
|
+
*
|
|
117
119
|
* @param entityTypeOrTable The type of entity, or the unquoted table name.
|
|
118
120
|
* @param options Options when constructing the table name (e.g. the index to use).
|
|
119
121
|
* @returns The name of the table, quoted with backticks.
|
|
@@ -130,6 +132,25 @@ export declare class SpannerEntityManager {
|
|
|
130
132
|
*/
|
|
131
133
|
disableQueryNullFilteredIndexEmulatorCheck?: boolean;
|
|
132
134
|
}): string;
|
|
135
|
+
/**
|
|
136
|
+
* Returns the (quoted) name of the table for the given entity type.
|
|
137
|
+
*
|
|
138
|
+
* @param entityTypeOrTable The type of entity, or the unquoted table name.
|
|
139
|
+
* @param options Options when constructing the table name (e.g. the index to use).
|
|
140
|
+
* @returns The name of the table, quoted with backticks.
|
|
141
|
+
*/
|
|
142
|
+
sqlTable(entityTypeOrTable: Type | string, options?: {
|
|
143
|
+
/**
|
|
144
|
+
* Sets a table hint to indicate which index to use when querying the table.
|
|
145
|
+
* The value will be quoted with backticks.
|
|
146
|
+
*/
|
|
147
|
+
index?: string;
|
|
148
|
+
/**
|
|
149
|
+
* Sets a table hint to disable the check that prevents queries from using null-filtered indexes.
|
|
150
|
+
* This is useful when using the emulator, which does not support null-filtered indexes.
|
|
151
|
+
*/
|
|
152
|
+
disableQueryNullFilteredIndexEmulatorCheck?: boolean;
|
|
153
|
+
}): string;
|
|
133
154
|
/**
|
|
134
155
|
* Returns the (quoted) list of columns for the given entity type or list of columns.
|
|
135
156
|
*
|
|
@@ -139,7 +160,16 @@ export declare class SpannerEntityManager {
|
|
|
139
160
|
* @param entityTypeOrColumns The type of entity, or the unquoted list of columns.
|
|
140
161
|
* @returns The list of columns, quoted with backticks and joined.
|
|
141
162
|
*/
|
|
142
|
-
sqlColumns(entityTypeOrColumns: Type | string[]
|
|
163
|
+
sqlColumns<T = unknown>(entityTypeOrColumns: Type<T> | string[], options?: {
|
|
164
|
+
/**
|
|
165
|
+
* If `entityTypeOrColumns` is a type, only the columns for the given properties will be included.
|
|
166
|
+
*/
|
|
167
|
+
forProperties?: T extends object ? (keyof T & string)[] : never;
|
|
168
|
+
/**
|
|
169
|
+
* The alias with which to prefix each column.
|
|
170
|
+
*/
|
|
171
|
+
alias?: string;
|
|
172
|
+
}): string;
|
|
143
173
|
/**
|
|
144
174
|
* Fetches a single row from the database using its key (either primary or for a secondary index).
|
|
145
175
|
*
|
|
@@ -73,11 +73,23 @@ let SpannerEntityManager = class SpannerEntityManager {
|
|
|
73
73
|
/**
|
|
74
74
|
* Returns the (quoted) name of the table for the given entity type.
|
|
75
75
|
*
|
|
76
|
+
* @deprecated Use {@link sqlTable} instead.
|
|
77
|
+
*
|
|
76
78
|
* @param entityTypeOrTable The type of entity, or the unquoted table name.
|
|
77
79
|
* @param options Options when constructing the table name (e.g. the index to use).
|
|
78
80
|
* @returns The name of the table, quoted with backticks.
|
|
79
81
|
*/
|
|
80
82
|
sqlTableName(entityTypeOrTable, options = {}) {
|
|
83
|
+
return this.sqlTable(entityTypeOrTable, options);
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Returns the (quoted) name of the table for the given entity type.
|
|
87
|
+
*
|
|
88
|
+
* @param entityTypeOrTable The type of entity, or the unquoted table name.
|
|
89
|
+
* @param options Options when constructing the table name (e.g. the index to use).
|
|
90
|
+
* @returns The name of the table, quoted with backticks.
|
|
91
|
+
*/
|
|
92
|
+
sqlTable(entityTypeOrTable, options = {}) {
|
|
81
93
|
const tableHints = {};
|
|
82
94
|
if (options.index) {
|
|
83
95
|
tableHints.FORCE_INDEX = `\`${options.index}\``;
|
|
@@ -89,9 +101,10 @@ let SpannerEntityManager = class SpannerEntityManager {
|
|
|
89
101
|
const tableHintsString = Object.entries(tableHints)
|
|
90
102
|
.map(([k, v]) => `${k}=${v}`)
|
|
91
103
|
.join(',');
|
|
92
|
-
const
|
|
93
|
-
?
|
|
94
|
-
: this.tableCache.getMetadata(entityTypeOrTable).
|
|
104
|
+
const tableName = typeof entityTypeOrTable === 'string'
|
|
105
|
+
? entityTypeOrTable
|
|
106
|
+
: this.tableCache.getMetadata(entityTypeOrTable).tableName;
|
|
107
|
+
const quotedTableName = `\`${tableName}\``;
|
|
95
108
|
return tableHintsString.length > 0
|
|
96
109
|
? `${quotedTableName}@{${tableHintsString}}`
|
|
97
110
|
: quotedTableName;
|
|
@@ -105,11 +118,22 @@ let SpannerEntityManager = class SpannerEntityManager {
|
|
|
105
118
|
* @param entityTypeOrColumns The type of entity, or the unquoted list of columns.
|
|
106
119
|
* @returns The list of columns, quoted with backticks and joined.
|
|
107
120
|
*/
|
|
108
|
-
sqlColumns(entityTypeOrColumns) {
|
|
121
|
+
sqlColumns(entityTypeOrColumns, options = {}) {
|
|
122
|
+
let columns;
|
|
109
123
|
if (Array.isArray(entityTypeOrColumns)) {
|
|
110
|
-
|
|
124
|
+
columns = entityTypeOrColumns;
|
|
111
125
|
}
|
|
112
|
-
|
|
126
|
+
else {
|
|
127
|
+
const { columnNames } = this.tableCache.getMetadata(entityTypeOrColumns);
|
|
128
|
+
columns = options.forProperties
|
|
129
|
+
? options.forProperties.map((p) => columnNames[p])
|
|
130
|
+
: Object.values(columnNames);
|
|
131
|
+
}
|
|
132
|
+
columns = columns.map((c) => `\`${c}\``);
|
|
133
|
+
if (options.alias) {
|
|
134
|
+
columns = columns.map((c) => `\`${options.alias}\`.${c}`);
|
|
135
|
+
}
|
|
136
|
+
return columns.join(', ');
|
|
113
137
|
}
|
|
114
138
|
/**
|
|
115
139
|
* Fetches a single row from the database using its key (either primary or for a secondary index).
|
|
@@ -131,8 +155,9 @@ let SpannerEntityManager = class SpannerEntityManager {
|
|
|
131
155
|
if (!Array.isArray(key)) {
|
|
132
156
|
key = [key];
|
|
133
157
|
}
|
|
134
|
-
const { tableName,
|
|
135
|
-
const columns = options.columns ??
|
|
158
|
+
const { tableName, columnNames, primaryKeyColumns, softDeleteColumn } = this.tableCache.getMetadata(entityType);
|
|
159
|
+
const columns = options.columns ??
|
|
160
|
+
(options.index ? primaryKeyColumns : Object.values(columnNames));
|
|
136
161
|
return await this.snapshot({ transaction: options.transaction }, async (transaction) => {
|
|
137
162
|
const [rows] = await transaction.read(tableName, {
|
|
138
163
|
keys: [key],
|
|
@@ -141,6 +166,7 @@ let SpannerEntityManager = class SpannerEntityManager {
|
|
|
141
166
|
json: true,
|
|
142
167
|
jsonOptions: { wrapNumbers: true },
|
|
143
168
|
index: options.index,
|
|
169
|
+
requestOptions: options.requestOptions,
|
|
144
170
|
});
|
|
145
171
|
const row = rows[0];
|
|
146
172
|
if (row && options.index && !options.columns) {
|
|
@@ -210,7 +236,7 @@ let SpannerEntityManager = class SpannerEntityManager {
|
|
|
210
236
|
if (options.transaction) {
|
|
211
237
|
return await runFn(options.transaction);
|
|
212
238
|
}
|
|
213
|
-
return await this.database.runTransactionAsync(async (transaction) => {
|
|
239
|
+
return await this.database.runTransactionAsync({ requestOptions: { transactionTag: options.tag } }, async (transaction) => {
|
|
214
240
|
try {
|
|
215
241
|
const result = await runFn(transaction);
|
|
216
242
|
if (transaction.ended) {
|
|
@@ -265,20 +291,19 @@ let SpannerEntityManager = class SpannerEntityManager {
|
|
|
265
291
|
* @param options The options to use when running the operation.
|
|
266
292
|
*/
|
|
267
293
|
async clear(entityType, options = {}) {
|
|
268
|
-
const {
|
|
269
|
-
await this.transaction(options, (transaction) => transaction.runUpdate({
|
|
270
|
-
sql: `DELETE FROM ${quotedTableName} WHERE TRUE`,
|
|
271
|
-
}));
|
|
294
|
+
const { tableName } = this.tableCache.getMetadata(entityType);
|
|
295
|
+
await this.transaction(options, (transaction) => transaction.runUpdate(`DELETE FROM \`${tableName}\` WHERE TRUE`));
|
|
272
296
|
}
|
|
273
297
|
async query(optionsOrStatement, statement) {
|
|
274
298
|
const options = statement
|
|
275
299
|
? optionsOrStatement
|
|
276
300
|
: {};
|
|
277
301
|
const sqlStatement = statement ?? optionsOrStatement;
|
|
278
|
-
const { entityType } = options;
|
|
302
|
+
const { entityType, requestOptions } = options;
|
|
279
303
|
return await this.snapshot({ transaction: options.transaction }, async (transaction) => {
|
|
280
304
|
const [rows] = await transaction.run({
|
|
281
305
|
...sqlStatement,
|
|
306
|
+
requestOptions,
|
|
282
307
|
json: true,
|
|
283
308
|
jsonOptions: { wrapNumbers: entityType != null },
|
|
284
309
|
});
|
|
@@ -7,22 +7,14 @@ export type CachedSpannerTableMetadata = {
|
|
|
7
7
|
* The name of the table.
|
|
8
8
|
*/
|
|
9
9
|
tableName: string;
|
|
10
|
-
/**
|
|
11
|
-
* The name of the table, quoted with backticks for use in queries.
|
|
12
|
-
*/
|
|
13
|
-
quotedTableName: string;
|
|
14
10
|
/**
|
|
15
11
|
* The (ordered) list of columns that are part of the primary key.
|
|
16
12
|
*/
|
|
17
13
|
primaryKeyColumns: string[];
|
|
18
14
|
/**
|
|
19
|
-
*
|
|
20
|
-
*/
|
|
21
|
-
columns: string[];
|
|
22
|
-
/**
|
|
23
|
-
* The list of all columns in the table, quoted with backticks and joined for use in queries.
|
|
15
|
+
* A map from class property names to Spanner column names.
|
|
24
16
|
*/
|
|
25
|
-
|
|
17
|
+
columnNames: Record<string, string>;
|
|
26
18
|
/**
|
|
27
19
|
* The name of the column used to mark soft deletes, if any.
|
|
28
20
|
*/
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { getSpannerColumnsMetadata } from './column.decorator.js';
|
|
2
2
|
import { InvalidEntityDefinitionError } from './errors.js';
|
|
3
3
|
import { getSpannerTableMetadataFromType } from './table.decorator.js';
|
|
4
4
|
/**
|
|
@@ -21,7 +21,6 @@ export class SpannerTableCache {
|
|
|
21
21
|
throw new InvalidEntityDefinitionError(entityType);
|
|
22
22
|
}
|
|
23
23
|
const tableName = tableMetadata.name;
|
|
24
|
-
const quotedTableName = `\`${tableName}\``;
|
|
25
24
|
const primaryKeyColumns = tableMetadata.primaryKey;
|
|
26
25
|
const columnsMetadata = getSpannerColumnsMetadata(entityType);
|
|
27
26
|
const softDeleteColumns = Object.values(columnsMetadata).filter((metadata) => metadata.softDelete);
|
|
@@ -29,15 +28,12 @@ export class SpannerTableCache {
|
|
|
29
28
|
throw new InvalidEntityDefinitionError(entityType, `Only one column can be marked as soft delete.`);
|
|
30
29
|
}
|
|
31
30
|
const softDeleteColumn = softDeleteColumns[0]?.name ?? null;
|
|
32
|
-
const
|
|
33
|
-
const quotedColumns = columns.map((c) => `\`${c}\``).join(', ');
|
|
31
|
+
const columnNames = Object.fromEntries(Object.entries(columnsMetadata).map(([prop, { name }]) => [prop, name]));
|
|
34
32
|
return {
|
|
35
33
|
tableName,
|
|
36
|
-
quotedTableName,
|
|
37
34
|
primaryKeyColumns,
|
|
38
|
-
columns,
|
|
39
|
-
quotedColumns,
|
|
40
35
|
softDeleteColumn,
|
|
36
|
+
columnNames,
|
|
41
37
|
};
|
|
42
38
|
}
|
|
43
39
|
/**
|
package/dist/spanner/types.d.ts
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
|
+
import { protos } from '@google-cloud/spanner';
|
|
1
2
|
import type { SpannerReadOnlyTransaction, SpannerReadWriteTransaction } from './entity-manager.js';
|
|
3
|
+
export declare const SpannerRequestPriority: typeof protos.google.spanner.v1.RequestOptions.Priority;
|
|
2
4
|
/**
|
|
3
5
|
* Option for a function that accepts a Spanner read-only transaction.
|
|
4
6
|
*/
|
|
@@ -15,5 +17,15 @@ export type SpannerReadWriteTransactionOption = {
|
|
|
15
17
|
/**
|
|
16
18
|
* The transaction to use.
|
|
17
19
|
*/
|
|
18
|
-
readonly transaction
|
|
20
|
+
readonly transaction: SpannerReadWriteTransaction;
|
|
21
|
+
} | {
|
|
22
|
+
/**
|
|
23
|
+
* The transaction to use.
|
|
24
|
+
*/
|
|
25
|
+
readonly transaction?: undefined;
|
|
26
|
+
/**
|
|
27
|
+
* A tag to assign to the transaction.
|
|
28
|
+
* This can only be provided when creating a new transaction.
|
|
29
|
+
*/
|
|
30
|
+
readonly tag?: string;
|
|
19
31
|
};
|
package/dist/spanner/types.js
CHANGED
|
@@ -1 +1,2 @@
|
|
|
1
|
-
|
|
1
|
+
import { protos } from '@google-cloud/spanner';
|
|
2
|
+
export const SpannerRequestPriority = protos.google.spanner.v1.RequestOptions.Priority;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { TransactionRunner, type ReadWriteTransactionOptions, type TransactionFn } from '@causa/runtime';
|
|
2
2
|
import { Logger } from '@causa/runtime/nestjs';
|
|
3
|
-
import { Firestore } from '
|
|
3
|
+
import { Firestore } from 'firebase-admin/firestore';
|
|
4
4
|
import { PubSubPublisher } from '../../pubsub/index.js';
|
|
5
5
|
import { FirestoreReadOnlyStateTransaction } from './readonly-state-transaction.js';
|
|
6
6
|
import { FirestorePubSubTransaction } from './transaction.js';
|
|
@@ -10,8 +10,8 @@ var __metadata = (this && this.__metadata) || function (k, v) {
|
|
|
10
10
|
var FirestorePubSubTransactionRunner_1;
|
|
11
11
|
import { OutboxEventTransaction, TransactionRunner, } from '@causa/runtime';
|
|
12
12
|
import { Logger } from '@causa/runtime/nestjs';
|
|
13
|
-
import { Firestore } from '@google-cloud/firestore';
|
|
14
13
|
import { Injectable } from '@nestjs/common';
|
|
14
|
+
import { Firestore } from 'firebase-admin/firestore';
|
|
15
15
|
import { wrapFirestoreOperation } from '../../firestore/index.js';
|
|
16
16
|
import { PubSubPublisher } from '../../pubsub/index.js';
|
|
17
17
|
import { FirestoreReadOnlyStateTransaction } from './readonly-state-transaction.js';
|
|
@@ -32,9 +32,14 @@ function parseSenderOptions(defaultOptions, customOptions, config) {
|
|
|
32
32
|
const index = config.get('SPANNER_OUTBOX_INDEX');
|
|
33
33
|
const shardingColumn = config.get('SPANNER_OUTBOX_SHARDING_COLUMN');
|
|
34
34
|
const shardingCount = validateIntOrUndefined('SPANNER_OUTBOX_SHARDING_COUNT');
|
|
35
|
+
const shardingDisableRoundRobin = !!config.get('SPANNER_OUTBOX_SHARDING_DISABLE_ROUND_ROBIN');
|
|
35
36
|
const leaseDuration = validateIntOrUndefined('SPANNER_OUTBOX_LEASE_DURATION');
|
|
36
37
|
const sharding = shardingColumn && shardingCount
|
|
37
|
-
? {
|
|
38
|
+
? {
|
|
39
|
+
column: shardingColumn,
|
|
40
|
+
count: shardingCount,
|
|
41
|
+
roundRobin: !shardingDisableRoundRobin,
|
|
42
|
+
}
|
|
38
43
|
: undefined;
|
|
39
44
|
const envOptionsWithUndefined = {
|
|
40
45
|
batchSize,
|
|
@@ -1,10 +1,19 @@
|
|
|
1
|
-
import { OutboxTransactionRunner, type OutboxEvent, type OutboxEventTransaction, type ReadWriteTransactionOptions, type TransactionFn } from '@causa/runtime';
|
|
1
|
+
import { OutboxTransactionRunner, type OutboxEvent, type OutboxEventTransaction, type ReadOnlyTransactionOption, type ReadWriteTransactionOptions, type TransactionFn, type TransactionOption } from '@causa/runtime';
|
|
2
2
|
import { Logger } from '@causa/runtime/nestjs';
|
|
3
3
|
import type { Type } from '@nestjs/common';
|
|
4
4
|
import { SpannerEntityManager } from '../../spanner/index.js';
|
|
5
5
|
import { SpannerReadOnlyStateTransaction } from './readonly-transaction.js';
|
|
6
6
|
import { SpannerOutboxSender } from './sender.js';
|
|
7
7
|
import { SpannerOutboxTransaction } from './transaction.js';
|
|
8
|
+
/**
|
|
9
|
+
* Options for a Spanner outbox transaction.
|
|
10
|
+
*/
|
|
11
|
+
export type SpannerOutboxReadWriteTransactionOptions = ReadWriteTransactionOptions & {
|
|
12
|
+
/**
|
|
13
|
+
* The Spanner tag to assign to the transaction.
|
|
14
|
+
*/
|
|
15
|
+
tag?: string;
|
|
16
|
+
};
|
|
8
17
|
/**
|
|
9
18
|
* An {@link OutboxTransactionRunner} that uses a {@link SpannerOutboxTransaction} to run transactions.
|
|
10
19
|
* Events are stored in a Spanner table before being published.
|
|
@@ -13,5 +22,10 @@ export declare class SpannerOutboxTransactionRunner extends OutboxTransactionRun
|
|
|
13
22
|
readonly entityManager: SpannerEntityManager;
|
|
14
23
|
constructor(entityManager: SpannerEntityManager, outboxEventType: Type<OutboxEvent>, sender: SpannerOutboxSender, logger: Logger);
|
|
15
24
|
protected runReadOnly<RT>(runFn: TransactionFn<SpannerReadOnlyStateTransaction, RT>): Promise<RT>;
|
|
16
|
-
protected runStateTransaction<RT>(eventTransactionFactory: () => OutboxEventTransaction, options:
|
|
25
|
+
protected runStateTransaction<RT>(eventTransactionFactory: () => OutboxEventTransaction, options: SpannerOutboxReadWriteTransactionOptions, runFn: TransactionFn<SpannerOutboxTransaction, RT>): Promise<RT>;
|
|
26
|
+
run<RT>(runFn: TransactionFn<SpannerOutboxTransaction, RT>): Promise<RT>;
|
|
27
|
+
run<RT>(options: TransactionOption<SpannerOutboxTransaction> | SpannerOutboxReadWriteTransactionOptions, runFn: TransactionFn<SpannerOutboxTransaction, RT>): Promise<RT>;
|
|
28
|
+
run<RT>(options: ReadOnlyTransactionOption<SpannerReadOnlyStateTransaction> & {
|
|
29
|
+
readOnly: true;
|
|
30
|
+
}, runFn: TransactionFn<SpannerReadOnlyStateTransaction, RT>): Promise<RT>;
|
|
17
31
|
}
|
|
@@ -23,7 +23,7 @@ export class SpannerOutboxTransactionRunner extends OutboxTransactionRunner {
|
|
|
23
23
|
});
|
|
24
24
|
}
|
|
25
25
|
async runStateTransaction(eventTransactionFactory, options, runFn) {
|
|
26
|
-
return await this.entityManager.transaction(async (dbTransaction) => {
|
|
26
|
+
return await this.entityManager.transaction({ tag: options.tag }, async (dbTransaction) => {
|
|
27
27
|
const stateTransaction = new SpannerStateTransaction(this.entityManager, dbTransaction);
|
|
28
28
|
const eventTransaction = eventTransactionFactory();
|
|
29
29
|
const transaction = new SpannerOutboxTransaction(stateTransaction, eventTransaction, options.publishOptions);
|
|
@@ -36,4 +36,7 @@ export class SpannerOutboxTransactionRunner extends OutboxTransactionRunner {
|
|
|
36
36
|
}
|
|
37
37
|
});
|
|
38
38
|
}
|
|
39
|
+
async run(optionsOrRunFn, runFn) {
|
|
40
|
+
return super.run(optionsOrRunFn, runFn);
|
|
41
|
+
}
|
|
39
42
|
}
|
|
@@ -3,9 +3,9 @@ import { Logger } from '@causa/runtime/nestjs';
|
|
|
3
3
|
import type { Type } from '@nestjs/common';
|
|
4
4
|
import { SpannerEntityManager } from '../../spanner/index.js';
|
|
5
5
|
/**
|
|
6
|
-
*
|
|
6
|
+
* The sharding configuration for the {@link SpannerOutboxSender}.
|
|
7
7
|
*/
|
|
8
|
-
|
|
8
|
+
type SpannerOutboxSenderSharding = {
|
|
9
9
|
/**
|
|
10
10
|
* The name of the column used for sharding.
|
|
11
11
|
*/
|
|
@@ -14,7 +14,16 @@ export type SpannerOutboxSenderShardingOptions = {
|
|
|
14
14
|
* The number of shards.
|
|
15
15
|
*/
|
|
16
16
|
readonly count: number;
|
|
17
|
+
/**
|
|
18
|
+
* Whether events are fetched one shard at a time in a round-robin fashion, or all shards at once.
|
|
19
|
+
* Defaults to `true`.
|
|
20
|
+
*/
|
|
21
|
+
readonly roundRobin: boolean;
|
|
17
22
|
};
|
|
23
|
+
/**
|
|
24
|
+
* Sharding options for the {@link SpannerOutboxSender}.
|
|
25
|
+
*/
|
|
26
|
+
export type SpannerOutboxSenderShardingOptions = Pick<SpannerOutboxSenderSharding, 'column' | 'count'> & Partial<SpannerOutboxSenderSharding>;
|
|
18
27
|
/**
|
|
19
28
|
* Options for the {@link SpannerOutboxSender}.
|
|
20
29
|
*/
|
|
@@ -49,7 +58,17 @@ export declare class SpannerOutboxSender extends OutboxEventSender {
|
|
|
49
58
|
* Sharding options.
|
|
50
59
|
* If `null`, queries to fetch events will not use sharding.
|
|
51
60
|
*/
|
|
52
|
-
readonly sharding:
|
|
61
|
+
readonly sharding: SpannerOutboxSenderSharding | undefined;
|
|
62
|
+
/**
|
|
63
|
+
* If sharding round-robin is enabled, the permutation of shard indices determining the order in which shards are
|
|
64
|
+
* queried by {@link SpannerOutboxSender.fetchEvents}.
|
|
65
|
+
*/
|
|
66
|
+
private readonly shardsPermutation;
|
|
67
|
+
/**
|
|
68
|
+
* If sharding round-robin is enabled, the index of the next shard to query in
|
|
69
|
+
* {@link SpannerOutboxSender.fetchEvents}.
|
|
70
|
+
*/
|
|
71
|
+
private shardsPermutationIndex;
|
|
53
72
|
/**
|
|
54
73
|
* The name of the column used for the {@link OutboxEvent.id} property.
|
|
55
74
|
*/
|
|
@@ -99,3 +118,4 @@ export declare class SpannerOutboxSender extends OutboxEventSender {
|
|
|
99
118
|
protected fetchEvents(): Promise<OutboxEvent[]>;
|
|
100
119
|
protected updateOutbox(result: OutboxEventPublishResult): Promise<void>;
|
|
101
120
|
}
|
|
121
|
+
export {};
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { OutboxEventSender, } from '@causa/runtime';
|
|
2
2
|
import { Logger } from '@causa/runtime/nestjs';
|
|
3
|
-
import { SpannerEntityManager } from '../../spanner/index.js';
|
|
3
|
+
import { SpannerEntityManager, SpannerRequestPriority, } from '../../spanner/index.js';
|
|
4
4
|
/**
|
|
5
5
|
* The default name for the {@link OutboxEvent.id} column.
|
|
6
6
|
*/
|
|
@@ -20,6 +20,16 @@ export class SpannerOutboxSender extends OutboxEventSender {
|
|
|
20
20
|
* If `null`, queries to fetch events will not use sharding.
|
|
21
21
|
*/
|
|
22
22
|
sharding;
|
|
23
|
+
/**
|
|
24
|
+
* If sharding round-robin is enabled, the permutation of shard indices determining the order in which shards are
|
|
25
|
+
* queried by {@link SpannerOutboxSender.fetchEvents}.
|
|
26
|
+
*/
|
|
27
|
+
shardsPermutation;
|
|
28
|
+
/**
|
|
29
|
+
* If sharding round-robin is enabled, the index of the next shard to query in
|
|
30
|
+
* {@link SpannerOutboxSender.fetchEvents}.
|
|
31
|
+
*/
|
|
32
|
+
shardsPermutationIndex;
|
|
23
33
|
/**
|
|
24
34
|
* The name of the column used for the {@link OutboxEvent.id} property.
|
|
25
35
|
*/
|
|
@@ -63,11 +73,19 @@ export class SpannerOutboxSender extends OutboxEventSender {
|
|
|
63
73
|
super(publisher, logger, options);
|
|
64
74
|
this.entityManager = entityManager;
|
|
65
75
|
this.outboxEventType = outboxEventType;
|
|
66
|
-
this.sharding = options.sharding
|
|
76
|
+
this.sharding = options.sharding
|
|
77
|
+
? { roundRobin: true, ...options.sharding }
|
|
78
|
+
: undefined;
|
|
67
79
|
this.idColumn = options.idColumn ?? DEFAULT_ID_COLUMN;
|
|
68
80
|
this.leaseExpirationColumn =
|
|
69
81
|
options.leaseExpirationColumn ?? DEFAULT_LEASE_EXPIRATION_COLUMN;
|
|
70
82
|
this.index = options.index;
|
|
83
|
+
if (this.sharding?.roundRobin) {
|
|
84
|
+
this.shardsPermutation = Array.from({ length: this.sharding.count }, (_, i) => [i, Math.random()])
|
|
85
|
+
.sort(([, a], [, b]) => a - b)
|
|
86
|
+
.map(([i]) => i);
|
|
87
|
+
this.shardsPermutationIndex = 0;
|
|
88
|
+
}
|
|
71
89
|
({
|
|
72
90
|
fetchEventsSql: this.fetchEventsSql,
|
|
73
91
|
acquireLeaseSql: this.acquireLeaseSql,
|
|
@@ -81,13 +99,18 @@ export class SpannerOutboxSender extends OutboxEventSender {
|
|
|
81
99
|
* @returns The SQL queries used to fetch and update events in the outbox.
|
|
82
100
|
*/
|
|
83
101
|
buildSql() {
|
|
84
|
-
const table = this.entityManager.
|
|
85
|
-
const tableWithIndex = this.entityManager.
|
|
102
|
+
const table = this.entityManager.sqlTable(this.outboxEventType);
|
|
103
|
+
const tableWithIndex = this.entityManager.sqlTable(this.outboxEventType, {
|
|
104
|
+
index: this.index,
|
|
105
|
+
});
|
|
86
106
|
const noLeaseFilter = `\`${this.leaseExpirationColumn}\` IS NULL OR \`${this.leaseExpirationColumn}\` < @currentTime`;
|
|
87
107
|
let fetchFilter = noLeaseFilter;
|
|
88
108
|
if (this.sharding) {
|
|
89
|
-
const { column, count } = this.sharding;
|
|
90
|
-
|
|
109
|
+
const { column, count, roundRobin } = this.sharding;
|
|
110
|
+
const shardFilter = (roundRobin ?? true)
|
|
111
|
+
? `\`${column}\` = @shard`
|
|
112
|
+
: `\`${column}\` BETWEEN 0 AND ${count - 1}`;
|
|
113
|
+
fetchFilter = `(${shardFilter}) AND (${fetchFilter})`;
|
|
91
114
|
}
|
|
92
115
|
const fetchEventsSql = `
|
|
93
116
|
SELECT
|
|
@@ -131,12 +154,17 @@ export class SpannerOutboxSender extends OutboxEventSender {
|
|
|
131
154
|
};
|
|
132
155
|
}
|
|
133
156
|
async fetchEvents() {
|
|
157
|
+
const params = {
|
|
158
|
+
currentTime: new Date(),
|
|
159
|
+
batchSize: this.batchSize,
|
|
160
|
+
};
|
|
161
|
+
if (this.shardsPermutationIndex !== undefined && this.shardsPermutation) {
|
|
162
|
+
params.shard = this.shardsPermutation[this.shardsPermutationIndex];
|
|
163
|
+
this.shardsPermutationIndex =
|
|
164
|
+
(this.shardsPermutationIndex + 1) % this.shardsPermutation.length;
|
|
165
|
+
}
|
|
134
166
|
// Event IDs are first acquired in an (implicit) read-only transaction, to avoid a lock on the entire table.
|
|
135
|
-
const
|
|
136
|
-
const eventIds = await this.entityManager.query({
|
|
137
|
-
sql: this.fetchEventsSql,
|
|
138
|
-
params: { currentTime, batchSize: this.batchSize },
|
|
139
|
-
});
|
|
167
|
+
const eventIds = await this.entityManager.query({ requestOptions: { priority: SpannerRequestPriority.PRIORITY_MEDIUM } }, { sql: this.fetchEventsSql, params });
|
|
140
168
|
if (eventIds.length === 0) {
|
|
141
169
|
return [];
|
|
142
170
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@causa/runtime-google",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.2.0",
|
|
4
4
|
"description": "An extension to the Causa runtime SDK (`@causa/runtime`), providing Google-specific features.",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -32,50 +32,48 @@
|
|
|
32
32
|
"test:cov": "npm run test -- --coverage"
|
|
33
33
|
},
|
|
34
34
|
"dependencies": {
|
|
35
|
-
"@causa/runtime": "^1.
|
|
35
|
+
"@causa/runtime": "^1.1.0",
|
|
36
36
|
"@google-cloud/precise-date": "^5.0.0",
|
|
37
|
-
"@google-cloud/pubsub": "^5.
|
|
38
|
-
"@google-cloud/spanner": "^8.
|
|
37
|
+
"@google-cloud/pubsub": "^5.2.0",
|
|
38
|
+
"@google-cloud/spanner": "^8.2.1",
|
|
39
39
|
"@google-cloud/tasks": "^6.2.0",
|
|
40
|
-
"@grpc/grpc-js": "^1.
|
|
41
|
-
"@nestjs/common": "^11.1.
|
|
40
|
+
"@grpc/grpc-js": "^1.14.0",
|
|
41
|
+
"@nestjs/common": "^11.1.6",
|
|
42
42
|
"@nestjs/config": "^4.0.2",
|
|
43
|
-
"@nestjs/core": "^11.1.
|
|
43
|
+
"@nestjs/core": "^11.1.6",
|
|
44
44
|
"@nestjs/passport": "^11.0.5",
|
|
45
45
|
"@nestjs/terminus": "^11.0.0",
|
|
46
46
|
"class-transformer": "^0.5.1",
|
|
47
47
|
"class-validator": "^0.14.2",
|
|
48
48
|
"express": "^5.1.0",
|
|
49
|
-
"firebase-admin": "^13.
|
|
49
|
+
"firebase-admin": "^13.5.0",
|
|
50
50
|
"jsonwebtoken": "^9.0.2",
|
|
51
51
|
"passport-http-bearer": "^1.0.1",
|
|
52
|
-
"pino": "^9.
|
|
52
|
+
"pino": "^9.12.0",
|
|
53
53
|
"reflect-metadata": "^0.2.2"
|
|
54
54
|
},
|
|
55
55
|
"devDependencies": {
|
|
56
|
-
"@nestjs/testing": "^11.1.
|
|
57
|
-
"@swc/core": "^1.13.
|
|
56
|
+
"@nestjs/testing": "^11.1.6",
|
|
57
|
+
"@swc/core": "^1.13.5",
|
|
58
58
|
"@swc/jest": "^0.2.39",
|
|
59
59
|
"@tsconfig/node22": "^22.0.2",
|
|
60
60
|
"@types/jest": "^30.0.0",
|
|
61
61
|
"@types/jsonwebtoken": "^9.0.10",
|
|
62
|
-
"@types/node": "^22.
|
|
63
|
-
"@types/passport-http-bearer": "^1.0.
|
|
62
|
+
"@types/node": "^22.18.8",
|
|
63
|
+
"@types/passport-http-bearer": "^1.0.42",
|
|
64
64
|
"@types/supertest": "^6.0.3",
|
|
65
|
-
"@types/uuid": "^
|
|
66
|
-
"dotenv": "^17.2.
|
|
67
|
-
"eslint": "^9.
|
|
68
|
-
"eslint-config-prettier": "^10.1.
|
|
69
|
-
"eslint-plugin-prettier": "^5.5.
|
|
70
|
-
"jest": "^30.0
|
|
65
|
+
"@types/uuid": "^11.0.0",
|
|
66
|
+
"dotenv": "^17.2.3",
|
|
67
|
+
"eslint": "^9.36.0",
|
|
68
|
+
"eslint-config-prettier": "^10.1.8",
|
|
69
|
+
"eslint-plugin-prettier": "^5.5.4",
|
|
70
|
+
"jest": "^30.2.0",
|
|
71
71
|
"jest-extended": "^6.0.0",
|
|
72
|
-
"pino-pretty": "^13.
|
|
72
|
+
"pino-pretty": "^13.1.1",
|
|
73
73
|
"rimraf": "^6.0.1",
|
|
74
|
-
"supertest": "^7.1.
|
|
75
|
-
"
|
|
76
|
-
"
|
|
77
|
-
"
|
|
78
|
-
"typescript-eslint": "^8.37.0",
|
|
79
|
-
"uuid": "^11.1.0"
|
|
74
|
+
"supertest": "^7.1.4",
|
|
75
|
+
"typescript": "^5.9.3",
|
|
76
|
+
"typescript-eslint": "^8.45.0",
|
|
77
|
+
"uuid": "^13.0.0"
|
|
80
78
|
}
|
|
81
79
|
}
|