@reventlessdev/reventless-aws 3.0.0-alpha.175 → 3.0.0-alpha.176
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/package.json +10 -10
- package/src/Platform.res +115 -31
- package/src/Platform.res.mjs +217 -44
- package/src/adapter/DcbEventLog/DcbBackend.res +2 -2
- package/src/adapter/EventCollector/EventCollectorChannel_DynamoDbStream.res +4 -1
- package/src/adapter/EventCollector/EventCollectorChannel_DynamoDbStream.res.mjs +1 -1
- package/src/adapter/EventCollector/EventCollectorChannel_DynamoDbStream_Runtime.res +14 -4
- package/src/adapter/EventCollector/EventCollectorChannel_DynamoDbStream_Runtime.res.mjs +19 -14
- package/src/adapter/EventLog/EventLogBackend.res +82 -0
- package/src/adapter/EventLog/EventLogBackend.res.mjs +53 -0
- package/src/adapter/EventLog/EventLogStorage.res +15 -0
- package/src/adapter/EventLog/EventLogStorage.res.mjs +20 -1
- package/src/adapter/EventLog/EventLogStorage_Postgres.res +37 -0
- package/src/adapter/EventLog/EventLogStorage_Postgres.res.mjs +21 -0
- package/src/adapter/Postgres/PgChangeFeedRelay_Builder.res +26 -11
- package/src/adapter/Postgres/PgChangeFeedRelay_Builder.res.mjs +15 -6
- package/src/adapter/Postgres/PgChangeFeedRelay_Runtime.res +54 -0
- package/src/adapter/Postgres/PgChangeFeedRelay_Runtime.res.mjs +23 -0
- package/src/adapter/Postgres/PgProjectionFeed.res +100 -0
- package/src/adapter/Postgres/PgProjectionFeed.res.mjs +54 -0
- package/src/adapter/QueryDb/PgQueryResolver_Lambda.res +192 -0
- package/src/adapter/QueryDb/PgQueryResolver_Lambda.res.mjs +141 -0
- package/src/adapter/QueryDb/QueryDbBackend.res +52 -0
- package/src/adapter/QueryDb/QueryDbBackend.res.mjs +44 -0
- package/src/adapter/QueryDb/QueryDbResolvers.res +48 -0
- package/src/adapter/QueryDb/QueryDbResolvers.res.mjs +17 -1
- package/src/adapter/QueryDb/QueryDbStorage.res +44 -0
- package/src/adapter/QueryDb/QueryDbStorage.res.mjs +34 -1
- package/src/adapter/QueryDb/QueryDbStorage_Postgres.res +43 -0
- package/src/adapter/QueryDb/QueryDbStorage_Postgres.res.mjs +21 -0
- package/src/adapter/QueryDb/QueryDbStorage_Postgres_Runtime.res +19 -0
- package/src/adapter/QueryDb/QueryDbStorage_Postgres_Runtime.res.mjs +15 -0
- package/src/adapter/QueryEngine/QueryEngine_DynamoDb.res +12 -1
- package/src/adapter/QueryEngine/QueryEngine_DynamoDb.res.mjs +2 -1
- package/src/adapter/Runtime/AggregateRuntime_Builder_Single.res +83 -3
- package/src/adapter/Runtime/AggregateRuntime_Builder_Single.res.mjs +46 -3
- package/src/adapter/Runtime/AggregateRuntime_Builder_Single_Async.res +83 -3
- package/src/adapter/Runtime/AggregateRuntime_Builder_Single_Async.res.mjs +48 -3
- package/src/adapter/Runtime/EventCollectorRuntime_Builder_Single.res +130 -5
- package/src/adapter/Runtime/EventCollectorRuntime_Builder_Single.res.mjs +66 -6
- package/src/adapter/Runtime/PgChangeFeedRelayEntryPoint.mjs +15 -10
- package/src/adapter/Runtime/ReadModelEntryPoint.mjs +32 -15
- package/src/adapter/Runtime/StateViewSliceEntryPoint.mjs +26 -12
- package/src/adapter/Runtime/StateViewSliceRuntime_Builder_Single.res +138 -8
- package/src/adapter/Runtime/StateViewSliceRuntime_Builder_Single.res.mjs +76 -11
- package/src/components/Aggregate_Builder_Single.res +12 -5
- package/src/components/Aggregate_Builder_Single.res.mjs +12 -6
- package/src/components/Aggregate_Builder_Single_Async.res +12 -5
- package/src/components/Aggregate_Builder_Single_Async.res.mjs +12 -6
- package/src/components/ReadModel_Builder_NoResolver.res.mjs +1 -1
- package/src/components/ReadModel_Builder_NoResolver_Stream.res +13 -4
- package/src/components/ReadModel_Builder_NoResolver_Stream.res.mjs +14 -9
- package/src/components/ReadModel_Builder_Single.res +14 -5
- package/src/components/ReadModel_Builder_Single.res.mjs +15 -12
- package/src/components/ReadModel_Builder_Single_Stream.res +14 -5
- package/src/components/ReadModel_Builder_Single_Stream.res.mjs +15 -12
- package/src/components/StateViewSlice_Builder.res +10 -4
- package/src/components/StateViewSlice_Builder.res.mjs +8 -10
- package/src/components/StateViewSlice_Builder_Stream.res +10 -4
- package/src/components/StateViewSlice_Builder_Stream.res.mjs +8 -10
- package/src/plugin/runtime/PluginRuntime_Builder.res +12 -0
- package/src/plugin/runtime/PluginRuntime_Builder.res.mjs +4 -0
- package/tests/EventCollectorChannelStreamRuntimeTest.res +55 -0
- package/tests/EventCollectorChannelStreamRuntimeTest.res.mjs +78 -0
- package/tests/PgChangeFeedRelay_IntegrationTest.res +115 -4
- package/tests/PgChangeFeedRelay_IntegrationTest.res.mjs +84 -0
- package/tests/PgPipeline_IntegrationTest.res +208 -0
- package/tests/PgPipeline_IntegrationTest.res.mjs +260 -0
- package/tests/PgQueryResolver_LambdaTest.res +206 -0
- package/tests/PgQueryResolver_LambdaTest.res.mjs +259 -0
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
// Generated by ReScript, PLEASE EDIT WITH CARE
|
|
2
2
|
|
|
3
|
+
import * as Aws from "@pulumi/aws";
|
|
3
4
|
import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
|
|
4
5
|
import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
|
|
5
6
|
import * as Pulumi from "@pulumi/pulumi";
|
|
@@ -7,7 +8,13 @@ import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
|
|
|
7
8
|
import * as Primitive_string from "@rescript/runtime/lib/es6/Primitive_string.js";
|
|
8
9
|
import * as Logger$ReventlessCore from "@reventlessdev/reventless-core/src/util/Logger.res.mjs";
|
|
9
10
|
import * as Component$ReventlessCore from "@reventlessdev/reventless-core/src/components/Component.res.mjs";
|
|
11
|
+
import * as DcbBackend$ReventlessAws from "../DcbEventLog/DcbBackend.res.mjs";
|
|
12
|
+
import * as PolicyDocument$PulumiAws from "@reventlessdev/rescript-pulumi-aws/src/IAM/PolicyDocument.res.mjs";
|
|
10
13
|
import * as Util_Bundle$ReventlessAws from "../../util/Util_Bundle.res.mjs";
|
|
14
|
+
import * as Util_Pulumi$ReventlessCore from "@reventlessdev/reventless-core/src/util/Util_Pulumi.res.mjs";
|
|
15
|
+
import * as QueryDbBackend$ReventlessAws from "../QueryDb/QueryDbBackend.res.mjs";
|
|
16
|
+
import * as EventLogBackend$ReventlessAws from "../EventLog/EventLogBackend.res.mjs";
|
|
17
|
+
import * as PgProjectionFeed$ReventlessAws from "../Postgres/PgProjectionFeed.res.mjs";
|
|
11
18
|
import * as RuntimeEnvironment_Lambda$ReventlessAws from "./RuntimeEnvironment_Lambda.res.mjs";
|
|
12
19
|
import * as EventCollectorChannel_DynamoDbStream$ReventlessAws from "../EventCollector/EventCollectorChannel_DynamoDbStream.res.mjs";
|
|
13
20
|
|
|
@@ -15,11 +22,13 @@ let log = Logger$ReventlessCore.fromEnv();
|
|
|
15
22
|
|
|
16
23
|
let readModelInfos = {};
|
|
17
24
|
|
|
18
|
-
function registerReadModel(readModelName, specModulePath, mappingsModulePath, queryDbTableName) {
|
|
25
|
+
function registerReadModel(readModelName, specModulePath, mappingsModulePath, queryDbTableName, pgBackedOpt) {
|
|
26
|
+
let pgBacked = pgBackedOpt !== undefined ? pgBackedOpt : false;
|
|
19
27
|
readModelInfos[readModelName] = {
|
|
20
28
|
specModulePath: specModulePath,
|
|
21
29
|
mappingsModulePath: mappingsModulePath,
|
|
22
|
-
queryDbTableName: queryDbTableName
|
|
30
|
+
queryDbTableName: queryDbTableName,
|
|
31
|
+
pgBacked: pgBacked
|
|
23
32
|
};
|
|
24
33
|
}
|
|
25
34
|
|
|
@@ -81,6 +90,36 @@ function finish() {
|
|
|
81
90
|
let opts = {
|
|
82
91
|
parent: opts_parent
|
|
83
92
|
};
|
|
93
|
+
let customOpts = Util_Pulumi$ReventlessCore.ComponentResourceOptions.toCustomResourceOptions(opts);
|
|
94
|
+
let feedQueue = EventLogBackend$ReventlessAws.isPostgres() || DcbBackend$ReventlessAws.isPostgres() ? PgProjectionFeed$ReventlessAws.makeQueue("AllReadModelsFeed", "aws-rm-feed", true, true, customOpts) : undefined;
|
|
95
|
+
let feedArnOutput = feedQueue !== undefined ? feedQueue.arn : Pulumi.output("");
|
|
96
|
+
let qdbSelection = QueryDbBackend$ReventlessAws.get();
|
|
97
|
+
let anyPgBacked = Object.values(readModelInfos).some(info => info.pgBacked);
|
|
98
|
+
let pgConnectionFragment = qdbSelection !== undefined && anyPgBacked ? qdbSelection.connectionConfig.apply(cc => {
|
|
99
|
+
let pgConnectionJson = JSON.stringify(Object.fromEntries([
|
|
100
|
+
[
|
|
101
|
+
"host",
|
|
102
|
+
cc.host
|
|
103
|
+
],
|
|
104
|
+
[
|
|
105
|
+
"port",
|
|
106
|
+
cc.port
|
|
107
|
+
],
|
|
108
|
+
[
|
|
109
|
+
"database",
|
|
110
|
+
cc.database
|
|
111
|
+
],
|
|
112
|
+
[
|
|
113
|
+
"username",
|
|
114
|
+
cc.username
|
|
115
|
+
],
|
|
116
|
+
[
|
|
117
|
+
"secretArn",
|
|
118
|
+
cc.secretArn
|
|
119
|
+
]
|
|
120
|
+
]));
|
|
121
|
+
return `,"pgConnection":` + pgConnectionJson;
|
|
122
|
+
}) : Pulumi.output("");
|
|
84
123
|
let handlerOutputs = [];
|
|
85
124
|
let packageDirs = {};
|
|
86
125
|
let sortedSpecs = storedSpecs.toSorted((a, b) => Primitive_string.compare(a.componentName, b.componentName));
|
|
@@ -95,12 +134,15 @@ function finish() {
|
|
|
95
134
|
packageDirs[mappingsPkg] = Util_Bundle$ReventlessAws.resolvePackageRoot(mappingsPkg);
|
|
96
135
|
let specModule = Stdlib_Option.getOr(JSON.stringify(info.specModulePath), `""`);
|
|
97
136
|
let mappingsModule = Stdlib_Option.getOr(JSON.stringify(info.mappingsModulePath), `""`);
|
|
137
|
+
let handlerPgFragment = info.pgBacked ? pgConnectionFragment : Pulumi.output("");
|
|
98
138
|
let handlerJson = Pulumi.all([
|
|
99
139
|
info.queryDbTableName,
|
|
100
|
-
spec.sourceUrns
|
|
140
|
+
spec.sourceUrns,
|
|
141
|
+
feedArnOutput,
|
|
142
|
+
handlerPgFragment
|
|
101
143
|
]).apply(param => {
|
|
102
|
-
let sourceUrn = param[1][0];
|
|
103
|
-
return `{"specModule":` + specModule + `,"mappingsModule":` + mappingsModule + `,"queryDbTableName":"` + param[0] + `","sourceUrn":"` + sourceUrn + `"}`;
|
|
144
|
+
let sourceUrn = Stdlib_Option.getOr(param[1][0], param[2]);
|
|
145
|
+
return `{"specModule":` + specModule + `,"mappingsModule":` + mappingsModule + `,"queryDbTableName":"` + param[0] + `","sourceUrn":"` + sourceUrn + `"` + param[3] + `}`;
|
|
104
146
|
});
|
|
105
147
|
handlerOutputs.push(handlerJson);
|
|
106
148
|
});
|
|
@@ -108,9 +150,27 @@ function finish() {
|
|
|
108
150
|
let envVars = {};
|
|
109
151
|
envVars["HANDLER_CONFIG"] = handlerConfigOutput;
|
|
110
152
|
let match$1 = Util_Bundle$ReventlessAws.buildCodeArchive("@reventlessdev/reventless-aws/src/adapter/Runtime/ReadModelEntryPoint.mjs", packageDirs, undefined);
|
|
111
|
-
let
|
|
153
|
+
let vpcConfig = qdbSelection !== undefined && anyPgBacked ? qdbSelection.securityGroupId.apply(sgId => ({
|
|
154
|
+
subnetIds: qdbSelection.subnetIds,
|
|
155
|
+
securityGroupIds: [sgId]
|
|
156
|
+
})) : undefined;
|
|
157
|
+
let runtime = RuntimeEnvironment_Lambda$ReventlessAws.makeFromCodeAsset("AllReadModels", match$1.code, match$1.sourceCodeHash, envVars, match[0], match[1], undefined, undefined, undefined, undefined, vpcConfig, opts);
|
|
158
|
+
if (qdbSelection !== undefined && anyPgBacked) {
|
|
159
|
+
qdbSelection.connectionConfig.apply(cc => new (Aws.iam.RolePolicy)("AllReadModels-pgSecret", {
|
|
160
|
+
policy: PolicyDocument$PulumiAws.toJsonString(PolicyDocument$PulumiAws.make(undefined, "AllReadModels-pgSecretPolicy", [{
|
|
161
|
+
Sid: "AllowGetPgSecret",
|
|
162
|
+
Effect: "Allow",
|
|
163
|
+
Action: "secretsmanager:GetSecretValue",
|
|
164
|
+
Resource: cc.secretArn
|
|
165
|
+
}])),
|
|
166
|
+
role: runtime.parts.lambdaRole.id
|
|
167
|
+
}));
|
|
168
|
+
}
|
|
112
169
|
let channelSpecs = storedSpecs.map(param => param.channelSpec);
|
|
113
170
|
EventCollectorChannel_DynamoDbStream$ReventlessAws.connect("AllReadModels", channelSpecs, runtime, opts);
|
|
171
|
+
if (feedQueue !== undefined) {
|
|
172
|
+
PgProjectionFeed$ReventlessAws.connect("AllReadModelsFeed", feedQueue, runtime.parts.lambda, runtime.parts.lambdaRole, customOpts);
|
|
173
|
+
}
|
|
114
174
|
} else {
|
|
115
175
|
log.warn("EventCollectorRuntime_Builder_Single", undefined, `finish: grandParent not set`);
|
|
116
176
|
}
|
|
@@ -1,20 +1,22 @@
|
|
|
1
1
|
// PgChangeFeedRelay Lambda entry point (scheduled poll — B2.3).
|
|
2
2
|
//
|
|
3
3
|
// Triggered on a schedule (EventBridge rate rule). Each invocation drains every
|
|
4
|
-
// Postgres
|
|
5
|
-
//
|
|
6
|
-
//
|
|
7
|
-
// command topics, and
|
|
8
|
-
//
|
|
4
|
+
// Postgres log listed in HANDLER_CONFIG from its checkpoint — DCB (`dcb_event`)
|
|
5
|
+
// and classic aggregate (`event_log`) logs alike — and relays the events,
|
|
6
|
+
// transformed into the {id, meta, event} shape, onto the plugin EventCollector
|
|
7
|
+
// SQS queue — which then fans out to projections, aggregate command topics, and
|
|
8
|
+
// the cross-plugin SNS EventTopic.
|
|
9
|
+
// See docs/plans/done/aws-postgres-change-feed-bridge.md.
|
|
9
10
|
//
|
|
10
11
|
// HANDLER_CONFIG shape:
|
|
11
12
|
// { "logs": [ { "pgConnection": {host,port,database,username,secretArn},
|
|
12
|
-
// "logName": string, // dcb_event
|
|
13
|
-
// "subscriber": string, //
|
|
13
|
+
// "logName": string, // dcb_event/event_log log_name
|
|
14
|
+
// "subscriber": string, // per-log checkpoint key
|
|
14
15
|
// "targetQueueUrl": string, // plugin EventCollector SQS URL
|
|
15
|
-
// "
|
|
16
|
+
// "kind"?: "classic", // absent → DCB
|
|
17
|
+
// "partitionTag"?: <derivedPartitionTag> } ] } // DCB only
|
|
16
18
|
|
|
17
|
-
import { relay } from "@reventlessdev/reventless-aws/src/adapter/Postgres/PgChangeFeedRelay_Runtime.res.mjs";
|
|
19
|
+
import { relay, relayClassic } from "@reventlessdev/reventless-aws/src/adapter/Postgres/PgChangeFeedRelay_Runtime.res.mjs";
|
|
18
20
|
import { sendMessage } from "@reventlessdev/reventless-aws/src/util/Util_SQS_Runtime.res.mjs";
|
|
19
21
|
import { makeQueueRef, log } from "./HandlerFactoryHelpers.mjs";
|
|
20
22
|
|
|
@@ -37,7 +39,10 @@ export async function handler(_event, _context) {
|
|
|
37
39
|
logs.map(async (l) => {
|
|
38
40
|
const sendBatch = makeSendBatch(l.targetQueueUrl);
|
|
39
41
|
try {
|
|
40
|
-
const count =
|
|
42
|
+
const count =
|
|
43
|
+
l.kind === "classic"
|
|
44
|
+
? await relayClassic(l.pgConnection, l.logName, l.subscriber, sendBatch)
|
|
45
|
+
: await relay(l.pgConnection, l.logName, l.subscriber, l.partitionTag, sendBatch);
|
|
41
46
|
log.debug("relayed " + count + " event(s) for " + l.logName, { comp: "PgChangeFeedRelay" });
|
|
42
47
|
} catch (err) {
|
|
43
48
|
// Log and continue — a failed log this tick is retried next tick from its
|
|
@@ -7,6 +7,7 @@ import { patchSpecId, makeTableRef, log, pluginName } from "./HandlerFactoryHelp
|
|
|
7
7
|
import { tag as requestContextTag } from "@reventlessdev/reventless-core/src/RequestContext.res.mjs";
|
|
8
8
|
import { Make as readModelCallbackMake } from "@reventlessdev/reventless-core/src/components/ReadModel/ReadModel_Callback.res.mjs";
|
|
9
9
|
import { load as qdbLoad, loadStream as qdbLoadStream, save as qdbSave, saveBatch as qdbSaveBatch, count as qdbCount, $$delete as qdbDelete, deleteBatch as qdbDeleteBatch } from "@reventlessdev/reventless-aws/src/adapter/QueryDb/QueryDbStorage_DynamoDb_Runtime.res.mjs";
|
|
10
|
+
import { opsFor as pgQdbOpsFor } from "@reventlessdev/reventless-aws/src/adapter/QueryDb/QueryDbStorage_Postgres_Runtime.res.mjs";
|
|
10
11
|
import { handleStreamEvent } from "@reventlessdev/reventless-aws/src/adapter/EventCollector/EventCollectorChannel_DynamoDbStream_Runtime.res.mjs";
|
|
11
12
|
|
|
12
13
|
const dynamicImport = (specifier) => import('/var/task/node_modules/' + specifier);
|
|
@@ -69,21 +70,37 @@ function groupBySource(records) {
|
|
|
69
70
|
return dict;
|
|
70
71
|
}
|
|
71
72
|
|
|
72
|
-
|
|
73
|
+
// `pgConnection`, when present, is the resolved PgConnection.connectionConfig the
|
|
74
|
+
// runtime builder injected for a Postgres-backed read model: bind the Postgres
|
|
75
|
+
// operation set (queryDbTableName is then the read-model spec name, the shared
|
|
76
|
+
// `qdb_<name>` discriminator). Absent → the DynamoDB path is byte-identical.
|
|
77
|
+
function buildReadModelHandler(specModule, mappingsModule, queryDbTableName, pgConnection) {
|
|
73
78
|
const patchedSpec = patchSpecId(specModule);
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
79
|
+
let operations;
|
|
80
|
+
if (pgConnection) {
|
|
81
|
+
const indexes = (patchedSpec.config && patchedSpec.config.indexes) || [];
|
|
82
|
+
const subIdField = patchedSpec.subIdConfig ? patchedSpec.subIdConfig.subIdField : undefined;
|
|
83
|
+
const pgOps = pgQdbOpsFor(pgConnection, queryDbTableName, indexes, subIdField);
|
|
84
|
+
operations = {
|
|
85
|
+
...pgOps,
|
|
86
|
+
save: mkInjectIdSave(pgOps.save),
|
|
87
|
+
saveBatch: mkInjectIdSaveBatch(pgOps.saveBatch),
|
|
88
|
+
};
|
|
89
|
+
} else {
|
|
90
|
+
const table = makeTableRef(queryDbTableName);
|
|
91
|
+
const rawSave = qdbSave(table);
|
|
92
|
+
const rawSaveBatch = qdbSaveBatch(table);
|
|
93
|
+
|
|
94
|
+
operations = {
|
|
95
|
+
load: qdbLoad(table),
|
|
96
|
+
loadStream: qdbLoadStream(table),
|
|
97
|
+
save: mkInjectIdSave(rawSave),
|
|
98
|
+
saveBatch: mkInjectIdSaveBatch(rawSaveBatch),
|
|
99
|
+
count: qdbCount(table),
|
|
100
|
+
delete: qdbDelete(table),
|
|
101
|
+
deleteBatch: qdbDeleteBatch(table),
|
|
102
|
+
};
|
|
103
|
+
}
|
|
87
104
|
|
|
88
105
|
const effectiveMappings = fixMappingsModule(mappingsModule);
|
|
89
106
|
const callback = readModelCallbackMake(patchedSpec)(effectiveMappings)({
|
|
@@ -102,7 +119,7 @@ async function buildAllHandlers() {
|
|
|
102
119
|
await Promise.all(config.handlers.map(async h => {
|
|
103
120
|
const specModule = await dynamicImport(h.specModule);
|
|
104
121
|
const mappingsModule = await dynamicImport(h.mappingsModule);
|
|
105
|
-
const handler = buildReadModelHandler(specModule, mappingsModule, h.queryDbTableName);
|
|
122
|
+
const handler = buildReadModelHandler(specModule, mappingsModule, h.queryDbTableName, h.pgConnection);
|
|
106
123
|
// Multiple read models can share one source stream — e.g. every admin read
|
|
107
124
|
// model (Plugins, PluginHistory, PlatformEventGraph, UIFragmentRegistry)
|
|
108
125
|
// projects the Plugin aggregate's EventLog stream. Accumulate ALL handlers
|
|
@@ -10,6 +10,7 @@ import { makeTableRef, log, pluginName } from "./HandlerFactoryHelpers.mjs";
|
|
|
10
10
|
import { tag as requestContextTag } from "@reventlessdev/reventless-core/src/RequestContext.res.mjs";
|
|
11
11
|
import { handleAction } from "@reventlessdev/reventless-core/src/Projection.res.mjs";
|
|
12
12
|
import { load, loadStream, save, saveBatch, count, $$delete, deleteBatch } from "@reventlessdev/reventless-aws/src/adapter/QueryDb/QueryDbStorage_DynamoDb_Runtime.res.mjs";
|
|
13
|
+
import { opsFor as pgQdbOpsFor } from "@reventlessdev/reventless-aws/src/adapter/QueryDb/QueryDbStorage_Postgres_Runtime.res.mjs";
|
|
13
14
|
import { handleStreamEvent } from "@reventlessdev/reventless-aws/src/adapter/EventCollector/EventCollectorChannel_DynamoDbStream_Runtime.res.mjs";
|
|
14
15
|
|
|
15
16
|
const dynamicImport = (specifier) => import('/var/task/node_modules/' + specifier);
|
|
@@ -40,17 +41,27 @@ function groupBySource(records) {
|
|
|
40
41
|
return dict;
|
|
41
42
|
}
|
|
42
43
|
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
44
|
+
// `pgConnection`, when present, selects the Postgres QueryDb runtime for this
|
|
45
|
+
// slice's view table (`queryDbTableName` is then the slice spec name, the shared
|
|
46
|
+
// `qdb_<name>` discriminator). Absent → the DynamoDB path is byte-identical.
|
|
47
|
+
function buildJsonEventsHandler(specModule, projectionModule, queryDbTableName, pgConnection) {
|
|
48
|
+
let queryDbOps;
|
|
49
|
+
if (pgConnection) {
|
|
50
|
+
const indexes = (specModule.config && specModule.config.indexes) || [];
|
|
51
|
+
const subIdField = specModule.subIdConfig ? specModule.subIdConfig.subIdField : undefined;
|
|
52
|
+
queryDbOps = pgQdbOpsFor(pgConnection, queryDbTableName, indexes, subIdField);
|
|
53
|
+
} else {
|
|
54
|
+
const table = makeTableRef(queryDbTableName);
|
|
55
|
+
queryDbOps = {
|
|
56
|
+
load: load(table),
|
|
57
|
+
loadStream: loadStream(table),
|
|
58
|
+
save: save(table),
|
|
59
|
+
saveBatch: saveBatch(table),
|
|
60
|
+
count: count(table),
|
|
61
|
+
delete: $$delete(table),
|
|
62
|
+
deleteBatch: deleteBatch(table),
|
|
63
|
+
};
|
|
64
|
+
}
|
|
54
65
|
// Spec module exports `consumedEventSchema`; the `project` function lives in
|
|
55
66
|
// the sibling projection module (`<Name>_Projection.res`).
|
|
56
67
|
const eventSchema = specModule.consumedEventSchema;
|
|
@@ -91,6 +102,9 @@ function expandHandlers(config) {
|
|
|
91
102
|
projectionModule: base + h.p,
|
|
92
103
|
queryDbTableName: h.q,
|
|
93
104
|
sourceUrn: h.u !== undefined ? h.u : sharedUrn,
|
|
105
|
+
// Shared across a plugin's slices (all follow the platform backend toggle);
|
|
106
|
+
// hoisted to the top level of the compressed config like base/sourceUrn.
|
|
107
|
+
pgConnection: config.pgConnection,
|
|
94
108
|
};
|
|
95
109
|
});
|
|
96
110
|
}
|
|
@@ -104,7 +118,7 @@ async function buildAllHandlers() {
|
|
|
104
118
|
dynamicImport(h.specModule),
|
|
105
119
|
dynamicImport(h.projectionModule),
|
|
106
120
|
]);
|
|
107
|
-
const jsonEventsHandler = buildJsonEventsHandler(specModule, projectionModule, h.queryDbTableName);
|
|
121
|
+
const jsonEventsHandler = buildJsonEventsHandler(specModule, projectionModule, h.queryDbTableName, h.pgConnection);
|
|
108
122
|
const streamHandler = (event, context) => handleStreamEvent(jsonEventsHandler, event, context);
|
|
109
123
|
const existing = handlers[h.sourceUrn] || [];
|
|
110
124
|
existing.push(streamHandler);
|
|
@@ -129,18 +129,44 @@ let buildLambda = (
|
|
|
129
129
|
~handlerOutputs: array<Pulumi.Output.t<handlerEntry>>,
|
|
130
130
|
~packageDirs,
|
|
131
131
|
~channelSpecs,
|
|
132
|
+
~feedQueue: option<PulumiAws.SQS.Queue.t>=None,
|
|
132
133
|
~memorySize=1024,
|
|
133
134
|
~timeout=30,
|
|
134
135
|
) => {
|
|
135
136
|
let opts = {Pulumi.ComponentResource.parent: parent}
|
|
137
|
+
|
|
138
|
+
// B3.1: on a Postgres-backed platform every slice view table lives in
|
|
139
|
+
// Postgres (slices are never admin-exempt) — one shared `pgConnection` is
|
|
140
|
+
// hoisted to the top level of the compressed config, like base/sourceUrn.
|
|
141
|
+
let qdbSelection = QueryDbBackend.get()
|
|
142
|
+
let pgConnectionFragment = switch qdbSelection {
|
|
143
|
+
| Some(sel) =>
|
|
144
|
+
sel.connectionConfig->Pulumi.Output.apply(cc => {
|
|
145
|
+
let pgConnectionJson =
|
|
146
|
+
[
|
|
147
|
+
("host", cc.host->JSON.Encode.string),
|
|
148
|
+
("port", cc.port->Int.toFloat->JSON.Encode.float),
|
|
149
|
+
("database", cc.database->JSON.Encode.string),
|
|
150
|
+
("username", cc.username->JSON.Encode.string),
|
|
151
|
+
("secretArn", cc.secretArn->JSON.Encode.string),
|
|
152
|
+
]
|
|
153
|
+
->Dict.fromArray
|
|
154
|
+
->JSON.Encode.object
|
|
155
|
+
->JSON.stringify
|
|
156
|
+
`,"pgConnection":${pgConnectionJson}`
|
|
157
|
+
})
|
|
158
|
+
| None => Pulumi.Output.make("")
|
|
159
|
+
}
|
|
160
|
+
|
|
136
161
|
// Compress HANDLER_CONFIG to stay under AWS Lambda's 4KB env-var ceiling:
|
|
137
162
|
// hoist the common module-path prefix (`base`) and the shared DCB stream
|
|
138
163
|
// `sourceUrn` (identical across a plugin's slices) out of the per-handler
|
|
139
164
|
// entries, and use short keys (s/p/q/u). StateViewSliceEntryPoint re-expands
|
|
140
165
|
// these. Durable fix (externalize to S3/SSM) tracked in docs/plans.
|
|
141
166
|
let handlerConfigOutput =
|
|
142
|
-
Pulumi.Output.all(handlerOutputs)
|
|
143
|
-
->Pulumi.Output.
|
|
167
|
+
(Pulumi.Output.all(handlerOutputs), pgConnectionFragment)
|
|
168
|
+
->Pulumi.Output.all2
|
|
169
|
+
->Pulumi.Output.apply(((handlers, pgFragment)) => {
|
|
144
170
|
let paths = []
|
|
145
171
|
handlers->Array.forEach(h => {
|
|
146
172
|
paths->Array.push(h.specModule)->ignore
|
|
@@ -175,7 +201,7 @@ let buildLambda = (
|
|
|
175
201
|
->Array.join(",")
|
|
176
202
|
let baseJson = base->JSON.stringifyAny->Option.getOr(`""`)
|
|
177
203
|
let urnJson = sharedUrn->JSON.stringifyAny->Option.getOr(`""`)
|
|
178
|
-
`{"v":2,"base":${baseJson},"sourceUrn":${urnJson},"handlers":[${entries}]}`
|
|
204
|
+
`{"v":2,"base":${baseJson},"sourceUrn":${urnJson},"handlers":[${entries}]${pgFragment}}`
|
|
179
205
|
})
|
|
180
206
|
|
|
181
207
|
let envVars: dict<Pulumi.Input.t<string>> = Dict.make()
|
|
@@ -201,6 +227,25 @@ let buildLambda = (
|
|
|
201
227
|
~packageDirs,
|
|
202
228
|
)
|
|
203
229
|
|
|
230
|
+
// Postgres-backed view tables → the projection Lambda writes to RDS, so it
|
|
231
|
+
// must land in-VPC on the DB-access security group + private subnets.
|
|
232
|
+
let vpcConfig = switch qdbSelection {
|
|
233
|
+
| Some(sel) =>
|
|
234
|
+
Some(
|
|
235
|
+
sel.securityGroupId
|
|
236
|
+
->Pulumi.Output.apply(sgId =>
|
|
237
|
+
(
|
|
238
|
+
{
|
|
239
|
+
PulumiAws.Lambda.Function.subnetIds: sel.subnetIds->Pulumi.Input.make,
|
|
240
|
+
securityGroupIds: [sgId->Pulumi.Input.make]->Pulumi.Input.make,
|
|
241
|
+
}: PulumiAws.Lambda.Function.vpcConfig
|
|
242
|
+
)
|
|
243
|
+
)
|
|
244
|
+
->Pulumi.Output.asInput,
|
|
245
|
+
)
|
|
246
|
+
| None => None
|
|
247
|
+
}
|
|
248
|
+
|
|
204
249
|
let runtime = RuntimeEnvironment_Lambda.makeFromCodeAsset(
|
|
205
250
|
~name="AllStateViewSlices",
|
|
206
251
|
~code,
|
|
@@ -208,15 +253,58 @@ let buildLambda = (
|
|
|
208
253
|
~envVars,
|
|
209
254
|
~memorySize,
|
|
210
255
|
~timeout,
|
|
256
|
+
~vpcConfig=?vpcConfig,
|
|
211
257
|
~opts,
|
|
212
258
|
)
|
|
213
259
|
|
|
260
|
+
// Grant the Postgres-backed projection Lambda read access to the RDS-managed
|
|
261
|
+
// master secret so PgRuntime can resolve the DB password at cold start.
|
|
262
|
+
switch qdbSelection {
|
|
263
|
+
| Some(sel) =>
|
|
264
|
+
let _ = sel.connectionConfig->Pulumi.Output.apply(cc => {
|
|
265
|
+
open PulumiAws.PolicyDocument
|
|
266
|
+
PulumiAws.IAM.RolePolicy.make(
|
|
267
|
+
~name="AllStateViewSlices-pgSecret",
|
|
268
|
+
~args={
|
|
269
|
+
policy: PulumiAws.PolicyDocument.make(
|
|
270
|
+
~id="AllStateViewSlices-pgSecretPolicy",
|
|
271
|
+
~statements=[
|
|
272
|
+
{
|
|
273
|
+
sid: "AllowGetPgSecret",
|
|
274
|
+
effect: Allow,
|
|
275
|
+
actions: Action("secretsmanager:GetSecretValue"),
|
|
276
|
+
resources: Resource(cc.secretArn),
|
|
277
|
+
},
|
|
278
|
+
],
|
|
279
|
+
)
|
|
280
|
+
->PulumiAws.PolicyDocument.toJsonString
|
|
281
|
+
->Pulumi.Input.make,
|
|
282
|
+
role: runtime.parts.lambdaRole.id->Pulumi.Output.asInput,
|
|
283
|
+
},
|
|
284
|
+
)
|
|
285
|
+
})
|
|
286
|
+
| None => ()
|
|
287
|
+
}
|
|
288
|
+
|
|
214
289
|
let _connectResources = EventCollectorChannel.connect(
|
|
215
290
|
~name="AllStateViewSlices",
|
|
216
291
|
~channelSpecs,
|
|
217
292
|
~runtime,
|
|
218
293
|
~opts,
|
|
219
294
|
)
|
|
295
|
+
|
|
296
|
+
// B3.0: wire the Postgres feed queue into the Lambda (ESM + receive IAM).
|
|
297
|
+
switch feedQueue {
|
|
298
|
+
| Some(queue) =>
|
|
299
|
+
PgProjectionFeed.connect(
|
|
300
|
+
~name="AllStateViewSlicesFeed",
|
|
301
|
+
~queue,
|
|
302
|
+
~lambda=runtime.parts.lambda,
|
|
303
|
+
~lambdaRole=runtime.parts.lambdaRole,
|
|
304
|
+
~opts=opts->ReventlessCore.Util.Pulumi.ComponentResourceOptions.toCustomResourceOptions,
|
|
305
|
+
)
|
|
306
|
+
| None => ()
|
|
307
|
+
}
|
|
220
308
|
}
|
|
221
309
|
|
|
222
310
|
let finishWithDcbEventLog = (dcbEventLog: ReventlessCore.DcbEventLog.component) =>
|
|
@@ -236,6 +324,25 @@ let finishWithDcbEventLog = (dcbEventLog: ReventlessCore.DcbEventLog.component)
|
|
|
236
324
|
~opts={Pulumi.ComponentResource.parent: parent},
|
|
237
325
|
)
|
|
238
326
|
|
|
327
|
+
// B3.0: a Postgres-backed DCB log has no stream resource — provision the
|
|
328
|
+
// SQS feed queue the change-feed relay fans DCB events into. Its ARN is
|
|
329
|
+
// the handlers' sourceUrn dispatch key.
|
|
330
|
+
let feedQueue = DcbBackend.isPostgres()
|
|
331
|
+
? Some(
|
|
332
|
+
PgProjectionFeed.makeQueue(
|
|
333
|
+
~name="AllStateViewSlicesFeed",
|
|
334
|
+
~scope="aws-svs-feed",
|
|
335
|
+
~includeClassic=false,
|
|
336
|
+
~includeDcb=true,
|
|
337
|
+
~opts={Pulumi.CustomResourceOptions.parent: parent},
|
|
338
|
+
),
|
|
339
|
+
)
|
|
340
|
+
: None
|
|
341
|
+
let feedArnOutput = switch feedQueue {
|
|
342
|
+
| Some(queue) => queue.arn
|
|
343
|
+
| None => Pulumi.Output.make("")
|
|
344
|
+
}
|
|
345
|
+
|
|
239
346
|
let handlerOutputs: array<Pulumi.Output.t<handlerEntry>> = []
|
|
240
347
|
let packageDirs: dict<string> = Dict.make()
|
|
241
348
|
let allQueryDbResources: array<ReventlessInfra.Adapter.resource> = []
|
|
@@ -248,8 +355,11 @@ let finishWithDcbEventLog = (dcbEventLog: ReventlessCore.DcbEventLog.component)
|
|
|
248
355
|
packageDirs->Dict.set(projectionPkg, Util_Bundle.resolvePackageRoot(projectionPkg))
|
|
249
356
|
|
|
250
357
|
let etResources: array<ReventlessInfra.Adapter.resource> = dcbOutputs.eventTopic.resources
|
|
251
|
-
|
|
252
|
-
let sourceUrn =
|
|
358
|
+
// No stream resource (Postgres DCB) → dispatch on the feed queue ARN.
|
|
359
|
+
let sourceUrn = switch etResources->Array.get(0) {
|
|
360
|
+
| Some(resource) => resource.urn
|
|
361
|
+
| None => feedArnOutput
|
|
362
|
+
}
|
|
253
363
|
|
|
254
364
|
let handlerJson =
|
|
255
365
|
Pulumi.Output.all2((info.queryDbTableName, sourceUrn))
|
|
@@ -270,6 +380,7 @@ let finishWithDcbEventLog = (dcbEventLog: ReventlessCore.DcbEventLog.component)
|
|
|
270
380
|
~handlerOutputs,
|
|
271
381
|
~packageDirs,
|
|
272
382
|
~channelSpecs=[{channel: channel, eventTopics, resources: allQueryDbResources}],
|
|
383
|
+
~feedQueue,
|
|
273
384
|
)
|
|
274
385
|
| None =>
|
|
275
386
|
log.warn(
|
|
@@ -300,6 +411,23 @@ let finish = () =>
|
|
|
300
411
|
let handlerOutputs: array<Pulumi.Output.t<handlerEntry>> = []
|
|
301
412
|
let packageDirs: dict<string> = Dict.make()
|
|
302
413
|
|
|
414
|
+
// B3.0: Postgres-backed source logs have no stream — feed queue instead.
|
|
415
|
+
let feedQueue = DcbBackend.isPostgres()
|
|
416
|
+
? Some(
|
|
417
|
+
PgProjectionFeed.makeQueue(
|
|
418
|
+
~name="AllStateViewSlicesFeed",
|
|
419
|
+
~scope="aws-svs-feed",
|
|
420
|
+
~includeClassic=false,
|
|
421
|
+
~includeDcb=true,
|
|
422
|
+
~opts={Pulumi.CustomResourceOptions.parent: parent},
|
|
423
|
+
),
|
|
424
|
+
)
|
|
425
|
+
: None
|
|
426
|
+
let feedArnOutput = switch feedQueue {
|
|
427
|
+
| Some(queue) => queue.arn
|
|
428
|
+
| None => Pulumi.Output.make("")
|
|
429
|
+
}
|
|
430
|
+
|
|
303
431
|
storedSpecs->Array.forEach(spec => {
|
|
304
432
|
switch sliceInfos->Dict.get(spec.componentName) {
|
|
305
433
|
| Some(info) =>
|
|
@@ -309,13 +437,14 @@ let finish = () =>
|
|
|
309
437
|
packageDirs->Dict.set(projectionPkg, Util_Bundle.resolvePackageRoot(projectionPkg))
|
|
310
438
|
|
|
311
439
|
let handlerJson =
|
|
312
|
-
Pulumi.Output.
|
|
313
|
-
->Pulumi.Output.apply(((tableName, urns)) => {
|
|
440
|
+
Pulumi.Output.all3((info.queryDbTableName, spec.sourceUrns, feedArnOutput))
|
|
441
|
+
->Pulumi.Output.apply(((tableName, urns, feedArn)) => {
|
|
314
442
|
let entry: handlerEntry = {
|
|
315
443
|
specModule: info.specModulePath,
|
|
316
444
|
projectionModule: info.projectionModulePath,
|
|
317
445
|
queryDbTableName: tableName,
|
|
318
|
-
|
|
446
|
+
// No stream resource (Postgres) → dispatch on the feed queue ARN.
|
|
447
|
+
sourceUrn: urns->Array.get(0)->Option.getOr(feedArn),
|
|
319
448
|
}
|
|
320
449
|
entry
|
|
321
450
|
})
|
|
@@ -333,6 +462,7 @@ let finish = () =>
|
|
|
333
462
|
~handlerOutputs,
|
|
334
463
|
~packageDirs,
|
|
335
464
|
~channelSpecs=storedSpecs->Array.map(({channelSpec}) => channelSpec),
|
|
465
|
+
~feedQueue,
|
|
336
466
|
~memorySize=maxMemorySize,
|
|
337
467
|
~timeout=maxTimeout,
|
|
338
468
|
)
|