@reventlessdev/reventless-aws 3.0.0-alpha.174 → 3.0.0-alpha.176
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/package.json +10 -10
- package/src/Platform.res +115 -31
- package/src/Platform.res.mjs +217 -44
- package/src/adapter/DcbEventLog/DcbBackend.res +2 -2
- package/src/adapter/EventCollector/EventCollectorChannel_DynamoDbStream.res +4 -1
- package/src/adapter/EventCollector/EventCollectorChannel_DynamoDbStream.res.mjs +1 -1
- package/src/adapter/EventCollector/EventCollectorChannel_DynamoDbStream_Runtime.res +14 -4
- package/src/adapter/EventCollector/EventCollectorChannel_DynamoDbStream_Runtime.res.mjs +19 -14
- package/src/adapter/EventLog/EventLogBackend.res +82 -0
- package/src/adapter/EventLog/EventLogBackend.res.mjs +53 -0
- package/src/adapter/EventLog/EventLogStorage.res +15 -0
- package/src/adapter/EventLog/EventLogStorage.res.mjs +20 -1
- package/src/adapter/EventLog/EventLogStorage_Postgres.res +37 -0
- package/src/adapter/EventLog/EventLogStorage_Postgres.res.mjs +21 -0
- package/src/adapter/Postgres/PgChangeFeedRelay_Builder.res +26 -11
- package/src/adapter/Postgres/PgChangeFeedRelay_Builder.res.mjs +15 -6
- package/src/adapter/Postgres/PgChangeFeedRelay_Runtime.res +78 -7
- package/src/adapter/Postgres/PgChangeFeedRelay_Runtime.res.mjs +30 -2
- package/src/adapter/Postgres/PgProjectionFeed.res +100 -0
- package/src/adapter/Postgres/PgProjectionFeed.res.mjs +54 -0
- package/src/adapter/QueryDb/PgQueryResolver_Lambda.res +192 -0
- package/src/adapter/QueryDb/PgQueryResolver_Lambda.res.mjs +141 -0
- package/src/adapter/QueryDb/QueryDbBackend.res +52 -0
- package/src/adapter/QueryDb/QueryDbBackend.res.mjs +44 -0
- package/src/adapter/QueryDb/QueryDbResolvers.res +48 -0
- package/src/adapter/QueryDb/QueryDbResolvers.res.mjs +17 -1
- package/src/adapter/QueryDb/QueryDbStorage.res +44 -0
- package/src/adapter/QueryDb/QueryDbStorage.res.mjs +34 -1
- package/src/adapter/QueryDb/QueryDbStorage_Postgres.res +43 -0
- package/src/adapter/QueryDb/QueryDbStorage_Postgres.res.mjs +21 -0
- package/src/adapter/QueryDb/QueryDbStorage_Postgres_Runtime.res +19 -0
- package/src/adapter/QueryDb/QueryDbStorage_Postgres_Runtime.res.mjs +15 -0
- package/src/adapter/QueryEngine/QueryEngine_DynamoDb.res +12 -1
- package/src/adapter/QueryEngine/QueryEngine_DynamoDb.res.mjs +2 -1
- package/src/adapter/Runtime/AggregateRuntime_Builder_Single.res +83 -3
- package/src/adapter/Runtime/AggregateRuntime_Builder_Single.res.mjs +46 -3
- package/src/adapter/Runtime/AggregateRuntime_Builder_Single_Async.res +83 -3
- package/src/adapter/Runtime/AggregateRuntime_Builder_Single_Async.res.mjs +48 -3
- package/src/adapter/Runtime/EventCollectorRuntime_Builder_Single.res +130 -5
- package/src/adapter/Runtime/EventCollectorRuntime_Builder_Single.res.mjs +66 -6
- package/src/adapter/Runtime/PgChangeFeedRelayEntryPoint.mjs +15 -10
- package/src/adapter/Runtime/ReadModelEntryPoint.mjs +32 -15
- package/src/adapter/Runtime/StateViewSliceEntryPoint.mjs +26 -12
- package/src/adapter/Runtime/StateViewSliceRuntime_Builder_Single.res +138 -8
- package/src/adapter/Runtime/StateViewSliceRuntime_Builder_Single.res.mjs +76 -11
- package/src/components/Aggregate_Builder_Single.res +12 -5
- package/src/components/Aggregate_Builder_Single.res.mjs +12 -6
- package/src/components/Aggregate_Builder_Single_Async.res +12 -5
- package/src/components/Aggregate_Builder_Single_Async.res.mjs +12 -6
- package/src/components/ReadModel_Builder_NoResolver.res.mjs +1 -1
- package/src/components/ReadModel_Builder_NoResolver_Stream.res +13 -4
- package/src/components/ReadModel_Builder_NoResolver_Stream.res.mjs +14 -9
- package/src/components/ReadModel_Builder_Single.res +14 -5
- package/src/components/ReadModel_Builder_Single.res.mjs +15 -12
- package/src/components/ReadModel_Builder_Single_Stream.res +14 -5
- package/src/components/ReadModel_Builder_Single_Stream.res.mjs +15 -12
- package/src/components/StateViewSlice_Builder.res +10 -4
- package/src/components/StateViewSlice_Builder.res.mjs +8 -10
- package/src/components/StateViewSlice_Builder_Stream.res +10 -4
- package/src/components/StateViewSlice_Builder_Stream.res.mjs +8 -10
- package/src/plugin/runtime/PluginRuntime_Builder.res +12 -0
- package/src/plugin/runtime/PluginRuntime_Builder.res.mjs +4 -0
- package/tests/EventCollectorChannelStreamRuntimeTest.res +55 -0
- package/tests/EventCollectorChannelStreamRuntimeTest.res.mjs +78 -0
- package/tests/PgChangeFeedRelay_IntegrationTest.res +259 -0
- package/tests/PgChangeFeedRelay_IntegrationTest.res.mjs +238 -0
- package/tests/PgPipeline_IntegrationTest.res +208 -0
- package/tests/PgPipeline_IntegrationTest.res.mjs +260 -0
- package/tests/PgQueryResolver_LambdaTest.res +206 -0
- package/tests/PgQueryResolver_LambdaTest.res.mjs +259 -0
|
@@ -10,6 +10,10 @@ type readModelInfo = {
|
|
|
10
10
|
specModulePath: string,
|
|
11
11
|
mappingsModulePath: string,
|
|
12
12
|
queryDbTableName: Pulumi.Output.t<string>,
|
|
13
|
+
/** B3.1: this read model's QueryDb is Postgres-backed — its HANDLER_CONFIG
|
|
14
|
+
entry carries a `pgConnection` (admin-exempt read models stay DynamoDB,
|
|
15
|
+
so the flag is per read model, not per Lambda). */
|
|
16
|
+
pgBacked: bool,
|
|
13
17
|
}
|
|
14
18
|
|
|
15
19
|
let readModelInfos: dict<readModelInfo> = Dict.make()
|
|
@@ -19,10 +23,11 @@ let registerReadModel = (
|
|
|
19
23
|
~specModulePath,
|
|
20
24
|
~mappingsModulePath,
|
|
21
25
|
~queryDbTableName,
|
|
26
|
+
~pgBacked=false,
|
|
22
27
|
) =>
|
|
23
28
|
readModelInfos->Dict.set(
|
|
24
29
|
readModelName,
|
|
25
|
-
{specModulePath, mappingsModulePath, queryDbTableName},
|
|
30
|
+
{specModulePath, mappingsModulePath, queryDbTableName, pgBacked},
|
|
26
31
|
)
|
|
27
32
|
|
|
28
33
|
type storedSpec = {
|
|
@@ -110,6 +115,54 @@ let finish = () =>
|
|
|
110
115
|
switch grandParent.contents {
|
|
111
116
|
| Some(parent) =>
|
|
112
117
|
let opts = {Pulumi.ComponentResource.parent: parent}
|
|
118
|
+
let customOpts =
|
|
119
|
+
opts->ReventlessCore.Util.Pulumi.ComponentResourceOptions.toCustomResourceOptions
|
|
120
|
+
|
|
121
|
+
// B3.0: Postgres-backed source logs have no DynamoDB stream — provision
|
|
122
|
+
// the SQS feed queue the change-feed relay fans events into. Its ARN
|
|
123
|
+
// doubles as the handlers' sourceUrn dispatch key (per-projection
|
|
124
|
+
// filtering makes over-delivery a no-op).
|
|
125
|
+
let feedQueue =
|
|
126
|
+
EventLogBackend.isPostgres() || DcbBackend.isPostgres()
|
|
127
|
+
? Some(
|
|
128
|
+
PgProjectionFeed.makeQueue(
|
|
129
|
+
~name="AllReadModelsFeed",
|
|
130
|
+
~scope="aws-rm-feed",
|
|
131
|
+
~includeClassic=true,
|
|
132
|
+
~includeDcb=true,
|
|
133
|
+
~opts=customOpts,
|
|
134
|
+
),
|
|
135
|
+
)
|
|
136
|
+
: None
|
|
137
|
+
let feedArnOutput = switch feedQueue {
|
|
138
|
+
| Some(queue) => queue.arn
|
|
139
|
+
| None => Pulumi.Output.make("")
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
// B3.1: Postgres-backed read models get a `pgConnection` in their handler
|
|
143
|
+
// entry (per read model — admin-exempt ones stay DynamoDB in the same
|
|
144
|
+
// Lambda). The Lambda itself goes in-VPC when any handler is pg-backed.
|
|
145
|
+
let qdbSelection = QueryDbBackend.get()
|
|
146
|
+
let anyPgBacked =
|
|
147
|
+
readModelInfos->Dict.valuesToArray->Array.some(info => info.pgBacked)
|
|
148
|
+
let pgConnectionFragment = switch (qdbSelection, anyPgBacked) {
|
|
149
|
+
| (Some(sel), true) =>
|
|
150
|
+
sel.connectionConfig->Pulumi.Output.apply(cc => {
|
|
151
|
+
let pgConnectionJson =
|
|
152
|
+
[
|
|
153
|
+
("host", cc.host->JSON.Encode.string),
|
|
154
|
+
("port", cc.port->Int.toFloat->JSON.Encode.float),
|
|
155
|
+
("database", cc.database->JSON.Encode.string),
|
|
156
|
+
("username", cc.username->JSON.Encode.string),
|
|
157
|
+
("secretArn", cc.secretArn->JSON.Encode.string),
|
|
158
|
+
]
|
|
159
|
+
->Dict.fromArray
|
|
160
|
+
->JSON.Encode.object
|
|
161
|
+
->JSON.stringify
|
|
162
|
+
`,"pgConnection":${pgConnectionJson}`
|
|
163
|
+
})
|
|
164
|
+
| _ => Pulumi.Output.make("")
|
|
165
|
+
}
|
|
113
166
|
|
|
114
167
|
let handlerOutputs: array<Pulumi.Output.t<string>> = []
|
|
115
168
|
let packageDirs: dict<string> = Dict.make()
|
|
@@ -136,11 +189,21 @@ let finish = () =>
|
|
|
136
189
|
let mappingsModule =
|
|
137
190
|
info.mappingsModulePath->JSON.stringifyAny->Option.getOr(`""`)
|
|
138
191
|
|
|
192
|
+
let handlerPgFragment = info.pgBacked
|
|
193
|
+
? pgConnectionFragment
|
|
194
|
+
: Pulumi.Output.make("")
|
|
139
195
|
let handlerJson =
|
|
140
|
-
Pulumi.Output.
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
196
|
+
Pulumi.Output.all4((
|
|
197
|
+
info.queryDbTableName,
|
|
198
|
+
spec.sourceUrns,
|
|
199
|
+
feedArnOutput,
|
|
200
|
+
handlerPgFragment,
|
|
201
|
+
))
|
|
202
|
+
->Pulumi.Output.apply(((tableName, urns, feedArn, pgFragment)) => {
|
|
203
|
+
// No stream resource (Postgres-backed source) → dispatch on the
|
|
204
|
+
// feed queue's ARN instead of a stream URN.
|
|
205
|
+
let sourceUrn = urns->Array.get(0)->Option.getOr(feedArn)
|
|
206
|
+
`{"specModule":${specModule},"mappingsModule":${mappingsModule},"queryDbTableName":"${tableName}","sourceUrn":"${sourceUrn}"${pgFragment}}`
|
|
144
207
|
})
|
|
145
208
|
let _ = handlerOutputs->Array.push(handlerJson)
|
|
146
209
|
| None =>
|
|
@@ -165,6 +228,25 @@ let finish = () =>
|
|
|
165
228
|
~packageDirs,
|
|
166
229
|
)
|
|
167
230
|
|
|
231
|
+
// Postgres-backed read models → the projection Lambda writes to RDS, so
|
|
232
|
+
// it must land in-VPC on the DB-access security group + private subnets.
|
|
233
|
+
let vpcConfig = switch (qdbSelection, anyPgBacked) {
|
|
234
|
+
| (Some(sel), true) =>
|
|
235
|
+
Some(
|
|
236
|
+
sel.securityGroupId
|
|
237
|
+
->Pulumi.Output.apply(sgId =>
|
|
238
|
+
(
|
|
239
|
+
{
|
|
240
|
+
PulumiAws.Lambda.Function.subnetIds: sel.subnetIds->Pulumi.Input.make,
|
|
241
|
+
securityGroupIds: [sgId->Pulumi.Input.make]->Pulumi.Input.make,
|
|
242
|
+
}: PulumiAws.Lambda.Function.vpcConfig
|
|
243
|
+
)
|
|
244
|
+
)
|
|
245
|
+
->Pulumi.Output.asInput,
|
|
246
|
+
)
|
|
247
|
+
| _ => None
|
|
248
|
+
}
|
|
249
|
+
|
|
168
250
|
let runtime = RuntimeEnvironment_Lambda.makeFromCodeAsset(
|
|
169
251
|
~name="AllReadModels",
|
|
170
252
|
~code,
|
|
@@ -172,9 +254,39 @@ let finish = () =>
|
|
|
172
254
|
~envVars,
|
|
173
255
|
~memorySize=maxMemorySize,
|
|
174
256
|
~timeout=maxTimeout,
|
|
257
|
+
~vpcConfig=?vpcConfig,
|
|
175
258
|
~opts,
|
|
176
259
|
)
|
|
177
260
|
|
|
261
|
+
// Grant the Postgres-backed projection Lambda read access to the
|
|
262
|
+
// RDS-managed master secret so PgRuntime can resolve the DB password.
|
|
263
|
+
switch (qdbSelection, anyPgBacked) {
|
|
264
|
+
| (Some(sel), true) =>
|
|
265
|
+
let _ = sel.connectionConfig->Pulumi.Output.apply(cc => {
|
|
266
|
+
open PulumiAws.PolicyDocument
|
|
267
|
+
PulumiAws.IAM.RolePolicy.make(
|
|
268
|
+
~name="AllReadModels-pgSecret",
|
|
269
|
+
~args={
|
|
270
|
+
policy: PulumiAws.PolicyDocument.make(
|
|
271
|
+
~id="AllReadModels-pgSecretPolicy",
|
|
272
|
+
~statements=[
|
|
273
|
+
{
|
|
274
|
+
sid: "AllowGetPgSecret",
|
|
275
|
+
effect: Allow,
|
|
276
|
+
actions: Action("secretsmanager:GetSecretValue"),
|
|
277
|
+
resources: Resource(cc.secretArn),
|
|
278
|
+
},
|
|
279
|
+
],
|
|
280
|
+
)
|
|
281
|
+
->PulumiAws.PolicyDocument.toJsonString
|
|
282
|
+
->Pulumi.Input.make,
|
|
283
|
+
role: runtime.parts.lambdaRole.id->Pulumi.Output.asInput,
|
|
284
|
+
},
|
|
285
|
+
)
|
|
286
|
+
})
|
|
287
|
+
| _ => ()
|
|
288
|
+
}
|
|
289
|
+
|
|
178
290
|
let channelSpecs = storedSpecs->Array.map(({channelSpec}) => channelSpec)
|
|
179
291
|
let _connectResources = EventCollectorChannel.connect(
|
|
180
292
|
~name="AllReadModels",
|
|
@@ -182,6 +294,19 @@ let finish = () =>
|
|
|
182
294
|
~runtime,
|
|
183
295
|
~opts,
|
|
184
296
|
)
|
|
297
|
+
|
|
298
|
+
// B3.0: wire the Postgres feed queue into the Lambda (ESM + receive IAM).
|
|
299
|
+
switch feedQueue {
|
|
300
|
+
| Some(queue) =>
|
|
301
|
+
PgProjectionFeed.connect(
|
|
302
|
+
~name="AllReadModelsFeed",
|
|
303
|
+
~queue,
|
|
304
|
+
~lambda=runtime.parts.lambda,
|
|
305
|
+
~lambdaRole=runtime.parts.lambdaRole,
|
|
306
|
+
~opts=customOpts,
|
|
307
|
+
)
|
|
308
|
+
| None => ()
|
|
309
|
+
}
|
|
185
310
|
| None =>
|
|
186
311
|
log.warn(
|
|
187
312
|
~comp="EventCollectorRuntime_Builder_Single",
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
// Generated by ReScript, PLEASE EDIT WITH CARE
|
|
2
2
|
|
|
3
|
+
import * as Aws from "@pulumi/aws";
|
|
3
4
|
import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
|
|
4
5
|
import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
|
|
5
6
|
import * as Pulumi from "@pulumi/pulumi";
|
|
@@ -7,7 +8,13 @@ import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
|
|
|
7
8
|
import * as Primitive_string from "@rescript/runtime/lib/es6/Primitive_string.js";
|
|
8
9
|
import * as Logger$ReventlessCore from "@reventlessdev/reventless-core/src/util/Logger.res.mjs";
|
|
9
10
|
import * as Component$ReventlessCore from "@reventlessdev/reventless-core/src/components/Component.res.mjs";
|
|
11
|
+
import * as DcbBackend$ReventlessAws from "../DcbEventLog/DcbBackend.res.mjs";
|
|
12
|
+
import * as PolicyDocument$PulumiAws from "@reventlessdev/rescript-pulumi-aws/src/IAM/PolicyDocument.res.mjs";
|
|
10
13
|
import * as Util_Bundle$ReventlessAws from "../../util/Util_Bundle.res.mjs";
|
|
14
|
+
import * as Util_Pulumi$ReventlessCore from "@reventlessdev/reventless-core/src/util/Util_Pulumi.res.mjs";
|
|
15
|
+
import * as QueryDbBackend$ReventlessAws from "../QueryDb/QueryDbBackend.res.mjs";
|
|
16
|
+
import * as EventLogBackend$ReventlessAws from "../EventLog/EventLogBackend.res.mjs";
|
|
17
|
+
import * as PgProjectionFeed$ReventlessAws from "../Postgres/PgProjectionFeed.res.mjs";
|
|
11
18
|
import * as RuntimeEnvironment_Lambda$ReventlessAws from "./RuntimeEnvironment_Lambda.res.mjs";
|
|
12
19
|
import * as EventCollectorChannel_DynamoDbStream$ReventlessAws from "../EventCollector/EventCollectorChannel_DynamoDbStream.res.mjs";
|
|
13
20
|
|
|
@@ -15,11 +22,13 @@ let log = Logger$ReventlessCore.fromEnv();
|
|
|
15
22
|
|
|
16
23
|
let readModelInfos = {};
|
|
17
24
|
|
|
18
|
-
function registerReadModel(readModelName, specModulePath, mappingsModulePath, queryDbTableName) {
|
|
25
|
+
function registerReadModel(readModelName, specModulePath, mappingsModulePath, queryDbTableName, pgBackedOpt) {
|
|
26
|
+
let pgBacked = pgBackedOpt !== undefined ? pgBackedOpt : false;
|
|
19
27
|
readModelInfos[readModelName] = {
|
|
20
28
|
specModulePath: specModulePath,
|
|
21
29
|
mappingsModulePath: mappingsModulePath,
|
|
22
|
-
queryDbTableName: queryDbTableName
|
|
30
|
+
queryDbTableName: queryDbTableName,
|
|
31
|
+
pgBacked: pgBacked
|
|
23
32
|
};
|
|
24
33
|
}
|
|
25
34
|
|
|
@@ -81,6 +90,36 @@ function finish() {
|
|
|
81
90
|
let opts = {
|
|
82
91
|
parent: opts_parent
|
|
83
92
|
};
|
|
93
|
+
let customOpts = Util_Pulumi$ReventlessCore.ComponentResourceOptions.toCustomResourceOptions(opts);
|
|
94
|
+
let feedQueue = EventLogBackend$ReventlessAws.isPostgres() || DcbBackend$ReventlessAws.isPostgres() ? PgProjectionFeed$ReventlessAws.makeQueue("AllReadModelsFeed", "aws-rm-feed", true, true, customOpts) : undefined;
|
|
95
|
+
let feedArnOutput = feedQueue !== undefined ? feedQueue.arn : Pulumi.output("");
|
|
96
|
+
let qdbSelection = QueryDbBackend$ReventlessAws.get();
|
|
97
|
+
let anyPgBacked = Object.values(readModelInfos).some(info => info.pgBacked);
|
|
98
|
+
let pgConnectionFragment = qdbSelection !== undefined && anyPgBacked ? qdbSelection.connectionConfig.apply(cc => {
|
|
99
|
+
let pgConnectionJson = JSON.stringify(Object.fromEntries([
|
|
100
|
+
[
|
|
101
|
+
"host",
|
|
102
|
+
cc.host
|
|
103
|
+
],
|
|
104
|
+
[
|
|
105
|
+
"port",
|
|
106
|
+
cc.port
|
|
107
|
+
],
|
|
108
|
+
[
|
|
109
|
+
"database",
|
|
110
|
+
cc.database
|
|
111
|
+
],
|
|
112
|
+
[
|
|
113
|
+
"username",
|
|
114
|
+
cc.username
|
|
115
|
+
],
|
|
116
|
+
[
|
|
117
|
+
"secretArn",
|
|
118
|
+
cc.secretArn
|
|
119
|
+
]
|
|
120
|
+
]));
|
|
121
|
+
return `,"pgConnection":` + pgConnectionJson;
|
|
122
|
+
}) : Pulumi.output("");
|
|
84
123
|
let handlerOutputs = [];
|
|
85
124
|
let packageDirs = {};
|
|
86
125
|
let sortedSpecs = storedSpecs.toSorted((a, b) => Primitive_string.compare(a.componentName, b.componentName));
|
|
@@ -95,12 +134,15 @@ function finish() {
|
|
|
95
134
|
packageDirs[mappingsPkg] = Util_Bundle$ReventlessAws.resolvePackageRoot(mappingsPkg);
|
|
96
135
|
let specModule = Stdlib_Option.getOr(JSON.stringify(info.specModulePath), `""`);
|
|
97
136
|
let mappingsModule = Stdlib_Option.getOr(JSON.stringify(info.mappingsModulePath), `""`);
|
|
137
|
+
let handlerPgFragment = info.pgBacked ? pgConnectionFragment : Pulumi.output("");
|
|
98
138
|
let handlerJson = Pulumi.all([
|
|
99
139
|
info.queryDbTableName,
|
|
100
|
-
spec.sourceUrns
|
|
140
|
+
spec.sourceUrns,
|
|
141
|
+
feedArnOutput,
|
|
142
|
+
handlerPgFragment
|
|
101
143
|
]).apply(param => {
|
|
102
|
-
let sourceUrn = param[1][0];
|
|
103
|
-
return `{"specModule":` + specModule + `,"mappingsModule":` + mappingsModule + `,"queryDbTableName":"` + param[0] + `","sourceUrn":"` + sourceUrn + `"}`;
|
|
144
|
+
let sourceUrn = Stdlib_Option.getOr(param[1][0], param[2]);
|
|
145
|
+
return `{"specModule":` + specModule + `,"mappingsModule":` + mappingsModule + `,"queryDbTableName":"` + param[0] + `","sourceUrn":"` + sourceUrn + `"` + param[3] + `}`;
|
|
104
146
|
});
|
|
105
147
|
handlerOutputs.push(handlerJson);
|
|
106
148
|
});
|
|
@@ -108,9 +150,27 @@ function finish() {
|
|
|
108
150
|
let envVars = {};
|
|
109
151
|
envVars["HANDLER_CONFIG"] = handlerConfigOutput;
|
|
110
152
|
let match$1 = Util_Bundle$ReventlessAws.buildCodeArchive("@reventlessdev/reventless-aws/src/adapter/Runtime/ReadModelEntryPoint.mjs", packageDirs, undefined);
|
|
111
|
-
let
|
|
153
|
+
let vpcConfig = qdbSelection !== undefined && anyPgBacked ? qdbSelection.securityGroupId.apply(sgId => ({
|
|
154
|
+
subnetIds: qdbSelection.subnetIds,
|
|
155
|
+
securityGroupIds: [sgId]
|
|
156
|
+
})) : undefined;
|
|
157
|
+
let runtime = RuntimeEnvironment_Lambda$ReventlessAws.makeFromCodeAsset("AllReadModels", match$1.code, match$1.sourceCodeHash, envVars, match[0], match[1], undefined, undefined, undefined, undefined, vpcConfig, opts);
|
|
158
|
+
if (qdbSelection !== undefined && anyPgBacked) {
|
|
159
|
+
qdbSelection.connectionConfig.apply(cc => new (Aws.iam.RolePolicy)("AllReadModels-pgSecret", {
|
|
160
|
+
policy: PolicyDocument$PulumiAws.toJsonString(PolicyDocument$PulumiAws.make(undefined, "AllReadModels-pgSecretPolicy", [{
|
|
161
|
+
Sid: "AllowGetPgSecret",
|
|
162
|
+
Effect: "Allow",
|
|
163
|
+
Action: "secretsmanager:GetSecretValue",
|
|
164
|
+
Resource: cc.secretArn
|
|
165
|
+
}])),
|
|
166
|
+
role: runtime.parts.lambdaRole.id
|
|
167
|
+
}));
|
|
168
|
+
}
|
|
112
169
|
let channelSpecs = storedSpecs.map(param => param.channelSpec);
|
|
113
170
|
EventCollectorChannel_DynamoDbStream$ReventlessAws.connect("AllReadModels", channelSpecs, runtime, opts);
|
|
171
|
+
if (feedQueue !== undefined) {
|
|
172
|
+
PgProjectionFeed$ReventlessAws.connect("AllReadModelsFeed", feedQueue, runtime.parts.lambda, runtime.parts.lambdaRole, customOpts);
|
|
173
|
+
}
|
|
114
174
|
} else {
|
|
115
175
|
log.warn("EventCollectorRuntime_Builder_Single", undefined, `finish: grandParent not set`);
|
|
116
176
|
}
|
|
@@ -1,20 +1,22 @@
|
|
|
1
1
|
// PgChangeFeedRelay Lambda entry point (scheduled poll — B2.3).
|
|
2
2
|
//
|
|
3
3
|
// Triggered on a schedule (EventBridge rate rule). Each invocation drains every
|
|
4
|
-
// Postgres
|
|
5
|
-
//
|
|
6
|
-
//
|
|
7
|
-
// command topics, and
|
|
8
|
-
//
|
|
4
|
+
// Postgres log listed in HANDLER_CONFIG from its checkpoint — DCB (`dcb_event`)
|
|
5
|
+
// and classic aggregate (`event_log`) logs alike — and relays the events,
|
|
6
|
+
// transformed into the {id, meta, event} shape, onto the plugin EventCollector
|
|
7
|
+
// SQS queue — which then fans out to projections, aggregate command topics, and
|
|
8
|
+
// the cross-plugin SNS EventTopic.
|
|
9
|
+
// See docs/plans/done/aws-postgres-change-feed-bridge.md.
|
|
9
10
|
//
|
|
10
11
|
// HANDLER_CONFIG shape:
|
|
11
12
|
// { "logs": [ { "pgConnection": {host,port,database,username,secretArn},
|
|
12
|
-
// "logName": string, // dcb_event
|
|
13
|
-
// "subscriber": string, //
|
|
13
|
+
// "logName": string, // dcb_event/event_log log_name
|
|
14
|
+
// "subscriber": string, // per-log checkpoint key
|
|
14
15
|
// "targetQueueUrl": string, // plugin EventCollector SQS URL
|
|
15
|
-
// "
|
|
16
|
+
// "kind"?: "classic", // absent → DCB
|
|
17
|
+
// "partitionTag"?: <derivedPartitionTag> } ] } // DCB only
|
|
16
18
|
|
|
17
|
-
import { relay } from "@reventlessdev/reventless-aws/src/adapter/Postgres/PgChangeFeedRelay_Runtime.res.mjs";
|
|
19
|
+
import { relay, relayClassic } from "@reventlessdev/reventless-aws/src/adapter/Postgres/PgChangeFeedRelay_Runtime.res.mjs";
|
|
18
20
|
import { sendMessage } from "@reventlessdev/reventless-aws/src/util/Util_SQS_Runtime.res.mjs";
|
|
19
21
|
import { makeQueueRef, log } from "./HandlerFactoryHelpers.mjs";
|
|
20
22
|
|
|
@@ -37,7 +39,10 @@ export async function handler(_event, _context) {
|
|
|
37
39
|
logs.map(async (l) => {
|
|
38
40
|
const sendBatch = makeSendBatch(l.targetQueueUrl);
|
|
39
41
|
try {
|
|
40
|
-
const count =
|
|
42
|
+
const count =
|
|
43
|
+
l.kind === "classic"
|
|
44
|
+
? await relayClassic(l.pgConnection, l.logName, l.subscriber, sendBatch)
|
|
45
|
+
: await relay(l.pgConnection, l.logName, l.subscriber, l.partitionTag, sendBatch);
|
|
41
46
|
log.debug("relayed " + count + " event(s) for " + l.logName, { comp: "PgChangeFeedRelay" });
|
|
42
47
|
} catch (err) {
|
|
43
48
|
// Log and continue — a failed log this tick is retried next tick from its
|
|
@@ -7,6 +7,7 @@ import { patchSpecId, makeTableRef, log, pluginName } from "./HandlerFactoryHelp
|
|
|
7
7
|
import { tag as requestContextTag } from "@reventlessdev/reventless-core/src/RequestContext.res.mjs";
|
|
8
8
|
import { Make as readModelCallbackMake } from "@reventlessdev/reventless-core/src/components/ReadModel/ReadModel_Callback.res.mjs";
|
|
9
9
|
import { load as qdbLoad, loadStream as qdbLoadStream, save as qdbSave, saveBatch as qdbSaveBatch, count as qdbCount, $$delete as qdbDelete, deleteBatch as qdbDeleteBatch } from "@reventlessdev/reventless-aws/src/adapter/QueryDb/QueryDbStorage_DynamoDb_Runtime.res.mjs";
|
|
10
|
+
import { opsFor as pgQdbOpsFor } from "@reventlessdev/reventless-aws/src/adapter/QueryDb/QueryDbStorage_Postgres_Runtime.res.mjs";
|
|
10
11
|
import { handleStreamEvent } from "@reventlessdev/reventless-aws/src/adapter/EventCollector/EventCollectorChannel_DynamoDbStream_Runtime.res.mjs";
|
|
11
12
|
|
|
12
13
|
const dynamicImport = (specifier) => import('/var/task/node_modules/' + specifier);
|
|
@@ -69,21 +70,37 @@ function groupBySource(records) {
|
|
|
69
70
|
return dict;
|
|
70
71
|
}
|
|
71
72
|
|
|
72
|
-
|
|
73
|
+
// `pgConnection`, when present, is the resolved PgConnection.connectionConfig the
|
|
74
|
+
// runtime builder injected for a Postgres-backed read model: bind the Postgres
|
|
75
|
+
// operation set (queryDbTableName is then the read-model spec name, the shared
|
|
76
|
+
// `qdb_<name>` discriminator). Absent → the DynamoDB path is byte-identical.
|
|
77
|
+
function buildReadModelHandler(specModule, mappingsModule, queryDbTableName, pgConnection) {
|
|
73
78
|
const patchedSpec = patchSpecId(specModule);
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
79
|
+
let operations;
|
|
80
|
+
if (pgConnection) {
|
|
81
|
+
const indexes = (patchedSpec.config && patchedSpec.config.indexes) || [];
|
|
82
|
+
const subIdField = patchedSpec.subIdConfig ? patchedSpec.subIdConfig.subIdField : undefined;
|
|
83
|
+
const pgOps = pgQdbOpsFor(pgConnection, queryDbTableName, indexes, subIdField);
|
|
84
|
+
operations = {
|
|
85
|
+
...pgOps,
|
|
86
|
+
save: mkInjectIdSave(pgOps.save),
|
|
87
|
+
saveBatch: mkInjectIdSaveBatch(pgOps.saveBatch),
|
|
88
|
+
};
|
|
89
|
+
} else {
|
|
90
|
+
const table = makeTableRef(queryDbTableName);
|
|
91
|
+
const rawSave = qdbSave(table);
|
|
92
|
+
const rawSaveBatch = qdbSaveBatch(table);
|
|
93
|
+
|
|
94
|
+
operations = {
|
|
95
|
+
load: qdbLoad(table),
|
|
96
|
+
loadStream: qdbLoadStream(table),
|
|
97
|
+
save: mkInjectIdSave(rawSave),
|
|
98
|
+
saveBatch: mkInjectIdSaveBatch(rawSaveBatch),
|
|
99
|
+
count: qdbCount(table),
|
|
100
|
+
delete: qdbDelete(table),
|
|
101
|
+
deleteBatch: qdbDeleteBatch(table),
|
|
102
|
+
};
|
|
103
|
+
}
|
|
87
104
|
|
|
88
105
|
const effectiveMappings = fixMappingsModule(mappingsModule);
|
|
89
106
|
const callback = readModelCallbackMake(patchedSpec)(effectiveMappings)({
|
|
@@ -102,7 +119,7 @@ async function buildAllHandlers() {
|
|
|
102
119
|
await Promise.all(config.handlers.map(async h => {
|
|
103
120
|
const specModule = await dynamicImport(h.specModule);
|
|
104
121
|
const mappingsModule = await dynamicImport(h.mappingsModule);
|
|
105
|
-
const handler = buildReadModelHandler(specModule, mappingsModule, h.queryDbTableName);
|
|
122
|
+
const handler = buildReadModelHandler(specModule, mappingsModule, h.queryDbTableName, h.pgConnection);
|
|
106
123
|
// Multiple read models can share one source stream — e.g. every admin read
|
|
107
124
|
// model (Plugins, PluginHistory, PlatformEventGraph, UIFragmentRegistry)
|
|
108
125
|
// projects the Plugin aggregate's EventLog stream. Accumulate ALL handlers
|
|
@@ -10,6 +10,7 @@ import { makeTableRef, log, pluginName } from "./HandlerFactoryHelpers.mjs";
|
|
|
10
10
|
import { tag as requestContextTag } from "@reventlessdev/reventless-core/src/RequestContext.res.mjs";
|
|
11
11
|
import { handleAction } from "@reventlessdev/reventless-core/src/Projection.res.mjs";
|
|
12
12
|
import { load, loadStream, save, saveBatch, count, $$delete, deleteBatch } from "@reventlessdev/reventless-aws/src/adapter/QueryDb/QueryDbStorage_DynamoDb_Runtime.res.mjs";
|
|
13
|
+
import { opsFor as pgQdbOpsFor } from "@reventlessdev/reventless-aws/src/adapter/QueryDb/QueryDbStorage_Postgres_Runtime.res.mjs";
|
|
13
14
|
import { handleStreamEvent } from "@reventlessdev/reventless-aws/src/adapter/EventCollector/EventCollectorChannel_DynamoDbStream_Runtime.res.mjs";
|
|
14
15
|
|
|
15
16
|
const dynamicImport = (specifier) => import('/var/task/node_modules/' + specifier);
|
|
@@ -40,17 +41,27 @@ function groupBySource(records) {
|
|
|
40
41
|
return dict;
|
|
41
42
|
}
|
|
42
43
|
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
44
|
+
// `pgConnection`, when present, selects the Postgres QueryDb runtime for this
|
|
45
|
+
// slice's view table (`queryDbTableName` is then the slice spec name, the shared
|
|
46
|
+
// `qdb_<name>` discriminator). Absent → the DynamoDB path is byte-identical.
|
|
47
|
+
function buildJsonEventsHandler(specModule, projectionModule, queryDbTableName, pgConnection) {
|
|
48
|
+
let queryDbOps;
|
|
49
|
+
if (pgConnection) {
|
|
50
|
+
const indexes = (specModule.config && specModule.config.indexes) || [];
|
|
51
|
+
const subIdField = specModule.subIdConfig ? specModule.subIdConfig.subIdField : undefined;
|
|
52
|
+
queryDbOps = pgQdbOpsFor(pgConnection, queryDbTableName, indexes, subIdField);
|
|
53
|
+
} else {
|
|
54
|
+
const table = makeTableRef(queryDbTableName);
|
|
55
|
+
queryDbOps = {
|
|
56
|
+
load: load(table),
|
|
57
|
+
loadStream: loadStream(table),
|
|
58
|
+
save: save(table),
|
|
59
|
+
saveBatch: saveBatch(table),
|
|
60
|
+
count: count(table),
|
|
61
|
+
delete: $$delete(table),
|
|
62
|
+
deleteBatch: deleteBatch(table),
|
|
63
|
+
};
|
|
64
|
+
}
|
|
54
65
|
// Spec module exports `consumedEventSchema`; the `project` function lives in
|
|
55
66
|
// the sibling projection module (`<Name>_Projection.res`).
|
|
56
67
|
const eventSchema = specModule.consumedEventSchema;
|
|
@@ -91,6 +102,9 @@ function expandHandlers(config) {
|
|
|
91
102
|
projectionModule: base + h.p,
|
|
92
103
|
queryDbTableName: h.q,
|
|
93
104
|
sourceUrn: h.u !== undefined ? h.u : sharedUrn,
|
|
105
|
+
// Shared across a plugin's slices (all follow the platform backend toggle);
|
|
106
|
+
// hoisted to the top level of the compressed config like base/sourceUrn.
|
|
107
|
+
pgConnection: config.pgConnection,
|
|
94
108
|
};
|
|
95
109
|
});
|
|
96
110
|
}
|
|
@@ -104,7 +118,7 @@ async function buildAllHandlers() {
|
|
|
104
118
|
dynamicImport(h.specModule),
|
|
105
119
|
dynamicImport(h.projectionModule),
|
|
106
120
|
]);
|
|
107
|
-
const jsonEventsHandler = buildJsonEventsHandler(specModule, projectionModule, h.queryDbTableName);
|
|
121
|
+
const jsonEventsHandler = buildJsonEventsHandler(specModule, projectionModule, h.queryDbTableName, h.pgConnection);
|
|
108
122
|
const streamHandler = (event, context) => handleStreamEvent(jsonEventsHandler, event, context);
|
|
109
123
|
const existing = handlers[h.sourceUrn] || [];
|
|
110
124
|
existing.push(streamHandler);
|