@reventlessdev/reventless-aws 3.0.0-alpha.175 → 3.0.0-alpha.176
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/package.json +10 -10
- package/src/Platform.res +115 -31
- package/src/Platform.res.mjs +217 -44
- package/src/adapter/DcbEventLog/DcbBackend.res +2 -2
- package/src/adapter/EventCollector/EventCollectorChannel_DynamoDbStream.res +4 -1
- package/src/adapter/EventCollector/EventCollectorChannel_DynamoDbStream.res.mjs +1 -1
- package/src/adapter/EventCollector/EventCollectorChannel_DynamoDbStream_Runtime.res +14 -4
- package/src/adapter/EventCollector/EventCollectorChannel_DynamoDbStream_Runtime.res.mjs +19 -14
- package/src/adapter/EventLog/EventLogBackend.res +82 -0
- package/src/adapter/EventLog/EventLogBackend.res.mjs +53 -0
- package/src/adapter/EventLog/EventLogStorage.res +15 -0
- package/src/adapter/EventLog/EventLogStorage.res.mjs +20 -1
- package/src/adapter/EventLog/EventLogStorage_Postgres.res +37 -0
- package/src/adapter/EventLog/EventLogStorage_Postgres.res.mjs +21 -0
- package/src/adapter/Postgres/PgChangeFeedRelay_Builder.res +26 -11
- package/src/adapter/Postgres/PgChangeFeedRelay_Builder.res.mjs +15 -6
- package/src/adapter/Postgres/PgChangeFeedRelay_Runtime.res +54 -0
- package/src/adapter/Postgres/PgChangeFeedRelay_Runtime.res.mjs +23 -0
- package/src/adapter/Postgres/PgProjectionFeed.res +100 -0
- package/src/adapter/Postgres/PgProjectionFeed.res.mjs +54 -0
- package/src/adapter/QueryDb/PgQueryResolver_Lambda.res +192 -0
- package/src/adapter/QueryDb/PgQueryResolver_Lambda.res.mjs +141 -0
- package/src/adapter/QueryDb/QueryDbBackend.res +52 -0
- package/src/adapter/QueryDb/QueryDbBackend.res.mjs +44 -0
- package/src/adapter/QueryDb/QueryDbResolvers.res +48 -0
- package/src/adapter/QueryDb/QueryDbResolvers.res.mjs +17 -1
- package/src/adapter/QueryDb/QueryDbStorage.res +44 -0
- package/src/adapter/QueryDb/QueryDbStorage.res.mjs +34 -1
- package/src/adapter/QueryDb/QueryDbStorage_Postgres.res +43 -0
- package/src/adapter/QueryDb/QueryDbStorage_Postgres.res.mjs +21 -0
- package/src/adapter/QueryDb/QueryDbStorage_Postgres_Runtime.res +19 -0
- package/src/adapter/QueryDb/QueryDbStorage_Postgres_Runtime.res.mjs +15 -0
- package/src/adapter/QueryEngine/QueryEngine_DynamoDb.res +12 -1
- package/src/adapter/QueryEngine/QueryEngine_DynamoDb.res.mjs +2 -1
- package/src/adapter/Runtime/AggregateRuntime_Builder_Single.res +83 -3
- package/src/adapter/Runtime/AggregateRuntime_Builder_Single.res.mjs +46 -3
- package/src/adapter/Runtime/AggregateRuntime_Builder_Single_Async.res +83 -3
- package/src/adapter/Runtime/AggregateRuntime_Builder_Single_Async.res.mjs +48 -3
- package/src/adapter/Runtime/EventCollectorRuntime_Builder_Single.res +130 -5
- package/src/adapter/Runtime/EventCollectorRuntime_Builder_Single.res.mjs +66 -6
- package/src/adapter/Runtime/PgChangeFeedRelayEntryPoint.mjs +15 -10
- package/src/adapter/Runtime/ReadModelEntryPoint.mjs +32 -15
- package/src/adapter/Runtime/StateViewSliceEntryPoint.mjs +26 -12
- package/src/adapter/Runtime/StateViewSliceRuntime_Builder_Single.res +138 -8
- package/src/adapter/Runtime/StateViewSliceRuntime_Builder_Single.res.mjs +76 -11
- package/src/components/Aggregate_Builder_Single.res +12 -5
- package/src/components/Aggregate_Builder_Single.res.mjs +12 -6
- package/src/components/Aggregate_Builder_Single_Async.res +12 -5
- package/src/components/Aggregate_Builder_Single_Async.res.mjs +12 -6
- package/src/components/ReadModel_Builder_NoResolver.res.mjs +1 -1
- package/src/components/ReadModel_Builder_NoResolver_Stream.res +13 -4
- package/src/components/ReadModel_Builder_NoResolver_Stream.res.mjs +14 -9
- package/src/components/ReadModel_Builder_Single.res +14 -5
- package/src/components/ReadModel_Builder_Single.res.mjs +15 -12
- package/src/components/ReadModel_Builder_Single_Stream.res +14 -5
- package/src/components/ReadModel_Builder_Single_Stream.res.mjs +15 -12
- package/src/components/StateViewSlice_Builder.res +10 -4
- package/src/components/StateViewSlice_Builder.res.mjs +8 -10
- package/src/components/StateViewSlice_Builder_Stream.res +10 -4
- package/src/components/StateViewSlice_Builder_Stream.res.mjs +8 -10
- package/src/plugin/runtime/PluginRuntime_Builder.res +12 -0
- package/src/plugin/runtime/PluginRuntime_Builder.res.mjs +4 -0
- package/tests/EventCollectorChannelStreamRuntimeTest.res +55 -0
- package/tests/EventCollectorChannelStreamRuntimeTest.res.mjs +78 -0
- package/tests/PgChangeFeedRelay_IntegrationTest.res +115 -4
- package/tests/PgChangeFeedRelay_IntegrationTest.res.mjs +84 -0
- package/tests/PgPipeline_IntegrationTest.res +208 -0
- package/tests/PgPipeline_IntegrationTest.res.mjs +260 -0
- package/tests/PgQueryResolver_LambdaTest.res +206 -0
- package/tests/PgQueryResolver_LambdaTest.res.mjs +259 -0
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
// AWS-side selection for the classic (aggregate) EventLog storage backend — the
|
|
2
|
+
// aggregate analogue of `DcbBackend` (B2.3c).
|
|
3
|
+
//
|
|
4
|
+
// When `Platform.MakeWithConfig` is given a `~pgConnection`, it records the resolved
|
|
5
|
+
// selection here BEFORE any admin/plugin `construct` runs. Read by:
|
|
6
|
+
// - `EventLogStorage.Selectable` — route classic EventLog storage to Postgres (no
|
|
7
|
+
// DynamoDB table/stream) and bind the Postgres operation set;
|
|
8
|
+
// - `Aggregate_Builder_Single(_Async)` — substitute the stable `event_log.log_name`
|
|
9
|
+
// for the DynamoDB table name registered with the runtime builder;
|
|
10
|
+
// - `AggregateRuntime_Builder_Single(_Async).finish()` — inject `pgConnection` into
|
|
11
|
+
// each handler's HANDLER_CONFIG entry (activates the AggregateEntryPoint.mjs
|
|
12
|
+
// Postgres branch) and put the AllAggregates Lambda in-VPC with secret access;
|
|
13
|
+
// - `Platform.provisionPgChangeFeedRelay` — relay each classic `event_log` log to
|
|
14
|
+
// its plugin's EventCollector SQS queue.
|
|
15
|
+
//
|
|
16
|
+
// All classic EventLogs in the platform share one selection, mirroring the DCB
|
|
17
|
+
// toggle. Only the Single (sync/async) aggregate strategies honour it; the
|
|
18
|
+
// PerAggregate and Micro strategies stay DynamoDB-only.
|
|
19
|
+
|
|
20
|
+
type selection = {
|
|
21
|
+
connectionConfig: Pulumi.Output.t<PgConnection.connectionConfig>,
|
|
22
|
+
/** DB-access security group every aggregate/relay Lambda attaches (PgConnection.securityGroupId). */
|
|
23
|
+
securityGroupId: Pulumi.Output.t<string>,
|
|
24
|
+
/** Private subnets for the in-VPC aggregate/relay Lambdas (PgConnection.subnetIds). */
|
|
25
|
+
subnetIds: array<Pulumi.Input.t<string>>,
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
let selectionRef: ref<option<selection>> = ref(None)
|
|
29
|
+
|
|
30
|
+
/** Record the classic-EventLog Postgres selection. Called once by the platform before construct. */
|
|
31
|
+
let set = (sel: selection) => selectionRef := Some(sel)
|
|
32
|
+
|
|
33
|
+
/** The active classic-EventLog Postgres selection, if the platform was given a `PgConnection`. */
|
|
34
|
+
let get = (): option<selection> => selectionRef.contents
|
|
35
|
+
|
|
36
|
+
/** Whether classic EventLog storage is Postgres-backed on this platform. */
|
|
37
|
+
let isPostgres = (): bool => selectionRef.contents->Option.isSome
|
|
38
|
+
|
|
39
|
+
// ---------------------------------------------------------------------------
|
|
40
|
+
// Relay-log registry — classic counterpart of DcbBackend's (B2.3d).
|
|
41
|
+
//
|
|
42
|
+
// One entry per Postgres-backed classic log (one per aggregate), assembled across
|
|
43
|
+
// the plugin build in two steps so `makePlatform` can provision the change-feed
|
|
44
|
+
// relay from complete inputs:
|
|
45
|
+
// 1. `EventLogStorage_Postgres.make` registers `{logName, aggregateName}` when
|
|
46
|
+
// the classic storage is created (logName = `<Aggregate>EventLog`, the
|
|
47
|
+
// ComponentType-derived name the deploy-time ops and the entry point share).
|
|
48
|
+
// 2. `PluginRuntime_Builder.forPluginEventCollector` attaches that plugin's
|
|
49
|
+
// EventCollector SQS queue — its `~eventTopics` dict is keyed by aggregate
|
|
50
|
+
// name, so each key attaches exactly the logs whose DynamoDB stream the
|
|
51
|
+
// collector would have subscribed to on the DynamoDB path.
|
|
52
|
+
// ---------------------------------------------------------------------------
|
|
53
|
+
type relayLogEntry = {
|
|
54
|
+
logName: string,
|
|
55
|
+
aggregateName: string,
|
|
56
|
+
mutable collectorQueueUrl: option<Pulumi.Output.t<string>>,
|
|
57
|
+
mutable collectorQueueArn: option<Pulumi.Output.t<string>>,
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
let relayLogs: array<relayLogEntry> = []
|
|
61
|
+
|
|
62
|
+
/** Step 1: record a Postgres classic log (from the storage maker). */
|
|
63
|
+
let registerRelayLog = (~logName: string, ~aggregateName: string) =>
|
|
64
|
+
relayLogs
|
|
65
|
+
->Array.push({logName, aggregateName, collectorQueueUrl: None, collectorQueueArn: None})
|
|
66
|
+
->ignore
|
|
67
|
+
|
|
68
|
+
/** Step 2: attach a plugin's EventCollector SQS queue to one aggregate's relay entry. */
|
|
69
|
+
let attachCollectorQueue = (
|
|
70
|
+
~aggregateName: string,
|
|
71
|
+
~url: Pulumi.Output.t<string>,
|
|
72
|
+
~arn: Pulumi.Output.t<string>,
|
|
73
|
+
) =>
|
|
74
|
+
relayLogs
|
|
75
|
+
->Array.find(e => e.aggregateName == aggregateName)
|
|
76
|
+
->Option.forEach(e => {
|
|
77
|
+
e.collectorQueueUrl = Some(url)
|
|
78
|
+
e.collectorQueueArn = Some(arn)
|
|
79
|
+
})
|
|
80
|
+
|
|
81
|
+
/** All registered relay logs (read by `makePlatform` to provision the relay). */
|
|
82
|
+
let getRelayLogs = (): array<relayLogEntry> => relayLogs
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
// Generated by ReScript, PLEASE EDIT WITH CARE
|
|
2
|
+
|
|
3
|
+
import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
|
|
4
|
+
|
|
5
|
+
let selectionRef = {
|
|
6
|
+
contents: undefined
|
|
7
|
+
};
|
|
8
|
+
|
|
9
|
+
function set(sel) {
|
|
10
|
+
selectionRef.contents = sel;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
function get() {
|
|
14
|
+
return selectionRef.contents;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
function isPostgres() {
|
|
18
|
+
return Stdlib_Option.isSome(selectionRef.contents);
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
let relayLogs = [];
|
|
22
|
+
|
|
23
|
+
function registerRelayLog(logName, aggregateName) {
|
|
24
|
+
relayLogs.push({
|
|
25
|
+
logName: logName,
|
|
26
|
+
aggregateName: aggregateName,
|
|
27
|
+
collectorQueueUrl: undefined,
|
|
28
|
+
collectorQueueArn: undefined
|
|
29
|
+
});
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function attachCollectorQueue(aggregateName, url, arn) {
|
|
33
|
+
Stdlib_Option.forEach(relayLogs.find(e => e.aggregateName === aggregateName), e => {
|
|
34
|
+
e.collectorQueueUrl = url;
|
|
35
|
+
e.collectorQueueArn = arn;
|
|
36
|
+
});
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function getRelayLogs() {
|
|
40
|
+
return relayLogs;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export {
|
|
44
|
+
selectionRef,
|
|
45
|
+
set,
|
|
46
|
+
get,
|
|
47
|
+
isPostgres,
|
|
48
|
+
relayLogs,
|
|
49
|
+
registerRelayLog,
|
|
50
|
+
attachCollectorQueue,
|
|
51
|
+
getRelayLogs,
|
|
52
|
+
}
|
|
53
|
+
/* No side effect */
|
|
@@ -1,3 +1,18 @@
|
|
|
1
1
|
module DynamoDb = EventLogStorage_DynamoDb
|
|
2
2
|
module DynamoDbStream = EventLogStorage_DynamoDbStream
|
|
3
3
|
module DynamoDb_Runtime = EventLogStorage_DynamoDb_Runtime
|
|
4
|
+
module Postgres = EventLogStorage_Postgres
|
|
5
|
+
|
|
6
|
+
// Build-time classic storage selector — the aggregate analogue of
|
|
7
|
+
// `DcbEventLogStorage.Selectable`. Reads `EventLogBackend` — set by the platform
|
|
8
|
+
// when a `PgConnection` is supplied — and dispatches to Postgres (no table/stream)
|
|
9
|
+
// or the streamed DynamoDB maker. Used by the Single (sync/async) aggregate
|
|
10
|
+
// builders; PerAggregate and Micro stay on `DynamoDbStream` directly.
|
|
11
|
+
module Selectable = {
|
|
12
|
+
let make: ReventlessCore.EventLog_Adapter.storageMaker = (~name, ~opts) =>
|
|
13
|
+
if EventLogBackend.isPostgres() {
|
|
14
|
+
Postgres.make(~name, ~opts)
|
|
15
|
+
} else {
|
|
16
|
+
DynamoDbStream.make(~name, ~opts)
|
|
17
|
+
}
|
|
18
|
+
}
|
|
@@ -1,5 +1,20 @@
|
|
|
1
1
|
// Generated by ReScript, PLEASE EDIT WITH CARE
|
|
2
2
|
|
|
3
|
+
import * as EventLogBackend$ReventlessAws from "./EventLogBackend.res.mjs";
|
|
4
|
+
import * as EventLogStorage_Postgres$ReventlessAws from "./EventLogStorage_Postgres.res.mjs";
|
|
5
|
+
import * as EventLogStorage_DynamoDbStream$ReventlessAws from "./EventLogStorage_DynamoDbStream.res.mjs";
|
|
6
|
+
|
|
7
|
+
function make(name, opts) {
|
|
8
|
+
if (EventLogBackend$ReventlessAws.isPostgres()) {
|
|
9
|
+
return EventLogStorage_Postgres$ReventlessAws.make(name, opts);
|
|
10
|
+
} else {
|
|
11
|
+
return EventLogStorage_DynamoDbStream$ReventlessAws.make(name, opts);
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
let Selectable = {
|
|
16
|
+
make: make
|
|
17
|
+
};
|
|
3
18
|
|
|
4
19
|
let DynamoDb;
|
|
5
20
|
|
|
@@ -7,9 +22,13 @@ let DynamoDbStream;
|
|
|
7
22
|
|
|
8
23
|
let DynamoDb_Runtime;
|
|
9
24
|
|
|
25
|
+
let Postgres;
|
|
26
|
+
|
|
10
27
|
export {
|
|
11
28
|
DynamoDb,
|
|
12
29
|
DynamoDbStream,
|
|
13
30
|
DynamoDb_Runtime,
|
|
31
|
+
Postgres,
|
|
32
|
+
Selectable,
|
|
14
33
|
}
|
|
15
|
-
/*
|
|
34
|
+
/* EventLogStorage_Postgres-ReventlessAws Not a pure module */
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
// Deploy-time classic (aggregate) EventLog storage for the Postgres backend — the
|
|
2
|
+
// aggregate analogue of `DcbEventLogStorage_Postgres` (B2.3c).
|
|
3
|
+
//
|
|
4
|
+
// Creates NO DynamoDB table and NO stream: classic events live in Postgres
|
|
5
|
+
// (`event_log`, discriminated by `log_name`) and the shared `PgChangeFeedRelay`
|
|
6
|
+
// drives propagation. Returns:
|
|
7
|
+
// - `resources: []` — so the aggregate's EventTopic `DynamoDbStream` publisher
|
|
8
|
+
// becomes a passive marker with nothing to subscribe to, and
|
|
9
|
+
// - `operations` — the classic runtime op set bound to the shared `PgConnection`
|
|
10
|
+
// pool (lazy: no connection opens during `pulumi up`).
|
|
11
|
+
//
|
|
12
|
+
// Selected by `EventLogStorage.Selectable` when `EventLogBackend` holds a selection.
|
|
13
|
+
|
|
14
|
+
let make: ReventlessCore.EventLog_Adapter.storageMaker = (~name, ~opts as _) => {
|
|
15
|
+
// `name` is the ComponentType-derived `<Aggregate>EventLog` — the canonical
|
|
16
|
+
// `event_log.log_name`. Register it so `makePlatform` can wire the change-feed
|
|
17
|
+
// relay; the collector queue is attached later in forPluginEventCollector,
|
|
18
|
+
// keyed by the aggregate name (the `~eventTopics` dict key).
|
|
19
|
+
let aggregateName = name->String.endsWith("EventLog")
|
|
20
|
+
? name->String.slice(~start=0, ~end=name->String.length - 8)
|
|
21
|
+
: name
|
|
22
|
+
EventLogBackend.registerRelayLog(~logName=name, ~aggregateName)
|
|
23
|
+
|
|
24
|
+
let operations = switch EventLogBackend.get() {
|
|
25
|
+
| Some({connectionConfig}) =>
|
|
26
|
+
connectionConfig->Pulumi.Output.apply(config =>
|
|
27
|
+
EventLogStorage_Postgres_Runtime.opsFor(config, ~logName=name)
|
|
28
|
+
)
|
|
29
|
+
| None =>
|
|
30
|
+
// Selectable only routes here when a selection is set; guard defensively.
|
|
31
|
+
JsError.throwWithMessage("EventLogStorage_Postgres.make called without an EventLogBackend selection")
|
|
32
|
+
}
|
|
33
|
+
{
|
|
34
|
+
ReventlessCore.EventLog_Adapter.resources: [],
|
|
35
|
+
operations,
|
|
36
|
+
}
|
|
37
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
// Generated by ReScript, PLEASE EDIT WITH CARE
|
|
2
|
+
|
|
3
|
+
import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
|
|
4
|
+
import * as EventLogBackend$ReventlessAws from "./EventLogBackend.res.mjs";
|
|
5
|
+
import * as EventLogStorage_Postgres_Runtime$ReventlessAws from "./EventLogStorage_Postgres_Runtime.res.mjs";
|
|
6
|
+
|
|
7
|
+
function make(name, param) {
|
|
8
|
+
let aggregateName = name.endsWith("EventLog") ? name.slice(0, name.length - 8 | 0) : name;
|
|
9
|
+
EventLogBackend$ReventlessAws.registerRelayLog(name, aggregateName);
|
|
10
|
+
let match = EventLogBackend$ReventlessAws.get();
|
|
11
|
+
let operations = match !== undefined ? match.connectionConfig.apply(config => EventLogStorage_Postgres_Runtime$ReventlessAws.opsFor(config, name)) : Stdlib_JsError.throwWithMessage("EventLogStorage_Postgres.make called without an EventLogBackend selection");
|
|
12
|
+
return {
|
|
13
|
+
resources: [],
|
|
14
|
+
operations: operations
|
|
15
|
+
};
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export {
|
|
19
|
+
make,
|
|
20
|
+
}
|
|
21
|
+
/* EventLogStorage_Postgres_Runtime-ReventlessAws Not a pure module */
|
|
@@ -11,16 +11,24 @@
|
|
|
11
11
|
// >= 1 minute. Sub-minute latency needs a self-invoking loop / EventBridge
|
|
12
12
|
// Scheduler (follow-up) or the Fargate-LISTEN upgrade.
|
|
13
13
|
|
|
14
|
-
//
|
|
14
|
+
// Which Postgres feed a relay log drains.
|
|
15
|
+
type feed =
|
|
16
|
+
/** `dcb_event` log; the partition tag is sury-encoded into HANDLER_CONFIG so the
|
|
17
|
+
relay computes the same `id` (partition key) the DynamoDB-stream path would. */
|
|
18
|
+
| Dcb({partitionTag: Reventless.DcbTag.derivedPartitionTag})
|
|
19
|
+
/** Classic `event_log` log (one per aggregate); the stored payload already is
|
|
20
|
+
the DynamoDB item shape, so no extra config is needed. */
|
|
21
|
+
| Classic
|
|
22
|
+
|
|
23
|
+
// One Postgres-backed log to relay.
|
|
15
24
|
type relayLog = {
|
|
16
25
|
connectionConfig: Pulumi.Output.t<PgConnection.connectionConfig>,
|
|
17
|
-
/** dcb_event.log_name
|
|
26
|
+
/** dcb_event.log_name / event_log.log_name discriminator. */
|
|
18
27
|
logName: string,
|
|
19
|
-
/** dcb_subscription checkpoint key for this relay.
|
|
28
|
+
/** dcb_subscription / event_log_subscription checkpoint key for this relay.
|
|
29
|
+
Both tables key by subscriber alone, so this MUST be unique per log. */
|
|
20
30
|
subscriber: string,
|
|
21
|
-
|
|
22
|
-
computes the same `id` (partition key) the DynamoDB-stream path would. */
|
|
23
|
-
partitionTag: Reventless.DcbTag.derivedPartitionTag,
|
|
31
|
+
feed: feed,
|
|
24
32
|
/** Plugin EventCollector SQS queue this log's events are relayed to. */
|
|
25
33
|
targetQueueUrl: Pulumi.Output.t<string>,
|
|
26
34
|
targetQueueArn: Pulumi.Output.t<string>,
|
|
@@ -60,16 +68,23 @@ let make = (
|
|
|
60
68
|
]
|
|
61
69
|
->Dict.fromArray
|
|
62
70
|
->JSON.Encode.object
|
|
63
|
-
[
|
|
71
|
+
let entries = [
|
|
64
72
|
("pgConnection", pgConnectionJson),
|
|
65
73
|
("logName", l.logName->JSON.Encode.string),
|
|
66
74
|
("subscriber", l.subscriber->JSON.Encode.string),
|
|
67
|
-
(
|
|
68
|
-
"partitionTag",
|
|
69
|
-
l.partitionTag->S.reverseConvertToJsonOrThrow(Reventless.DcbTag.derivedPartitionTagSchema),
|
|
70
|
-
),
|
|
71
75
|
("targetQueueUrl", queueUrl->JSON.Encode.string),
|
|
72
76
|
]
|
|
77
|
+
switch l.feed {
|
|
78
|
+
| Dcb({partitionTag}) =>
|
|
79
|
+
entries
|
|
80
|
+
->Array.push((
|
|
81
|
+
"partitionTag",
|
|
82
|
+
partitionTag->S.reverseConvertToJsonOrThrow(Reventless.DcbTag.derivedPartitionTagSchema),
|
|
83
|
+
))
|
|
84
|
+
->ignore
|
|
85
|
+
| Classic => entries->Array.push(("kind", "classic"->JSON.Encode.string))->ignore
|
|
86
|
+
}
|
|
87
|
+
entries
|
|
73
88
|
->Dict.fromArray
|
|
74
89
|
->JSON.Encode.object
|
|
75
90
|
->JSON.stringify
|
|
@@ -49,7 +49,7 @@ function make(name, logs, securityGroupId, subnetIds, intervalMinutesOpt, opts)
|
|
|
49
49
|
cc.secretArn
|
|
50
50
|
]
|
|
51
51
|
]);
|
|
52
|
-
|
|
52
|
+
let entries = [
|
|
53
53
|
[
|
|
54
54
|
"pgConnection",
|
|
55
55
|
pgConnectionJson
|
|
@@ -62,15 +62,24 @@ function make(name, logs, securityGroupId, subnetIds, intervalMinutesOpt, opts)
|
|
|
62
62
|
"subscriber",
|
|
63
63
|
l.subscriber
|
|
64
64
|
],
|
|
65
|
-
[
|
|
66
|
-
"partitionTag",
|
|
67
|
-
S.reverseConvertToJsonOrThrow(l.partitionTag, DcbTag$Reventless.derivedPartitionTagSchema)
|
|
68
|
-
],
|
|
69
65
|
[
|
|
70
66
|
"targetQueueUrl",
|
|
71
67
|
param[1]
|
|
72
68
|
]
|
|
73
|
-
]
|
|
69
|
+
];
|
|
70
|
+
let match = l.feed;
|
|
71
|
+
if (typeof match !== "object") {
|
|
72
|
+
entries.push([
|
|
73
|
+
"kind",
|
|
74
|
+
"classic"
|
|
75
|
+
]);
|
|
76
|
+
} else {
|
|
77
|
+
entries.push([
|
|
78
|
+
"partitionTag",
|
|
79
|
+
S.reverseConvertToJsonOrThrow(match.partitionTag, DcbTag$Reventless.derivedPartitionTagSchema)
|
|
80
|
+
]);
|
|
81
|
+
}
|
|
82
|
+
return JSON.stringify(Object.fromEntries(entries));
|
|
74
83
|
}));
|
|
75
84
|
let handlerConfigOutput = Pulumi.all(logJsonOutputs).apply(items => `{"logs":[` + items.join(",") + `]}`);
|
|
76
85
|
let envVars = Object.fromEntries([[
|
|
@@ -87,3 +87,57 @@ let relay = async (
|
|
|
87
87
|
let pool = PgRuntime.poolFor(config)
|
|
88
88
|
await relayWithPool(~pool, ~logName, ~subscriber, ~partitionTagJson?, ~sendBatch)
|
|
89
89
|
}
|
|
90
|
+
|
|
91
|
+
// ─── Classic (aggregate) event_log relay ───
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Transform one classic feed event into the EventCollector JSON body. The stored
|
|
95
|
+
* `payload` IS the flat item the DynamoDB append would have put (id, position,
|
|
96
|
+
* event, data, decomposed meta) — classic appends write the serialized event JSON
|
|
97
|
+
* verbatim on both backends — so the transform is exactly the DynamoDB-stream
|
|
98
|
+
* decoder and the SQS handler cannot tell the difference. Returns `None` for a
|
|
99
|
+
* malformed row (mirrors `buildJsonEvent'`; snapshots live in a separate Postgres
|
|
100
|
+
* table and never enter the feed).
|
|
101
|
+
*/
|
|
102
|
+
let toClassicEventCollectorJson = (
|
|
103
|
+
event: ReventlessPostgres.EventLogChangeFeed.classicEvent,
|
|
104
|
+
): option<JSON.t> =>
|
|
105
|
+
event.payload
|
|
106
|
+
->JSON.Decode.object
|
|
107
|
+
->Option.flatMap(Util_DynamoDbStream_Runtime.buildJsonEvent')
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* Drain a classic Postgres `event_log` log from its checkpoint on the given `pool`
|
|
111
|
+
* and relay its events to the EventCollector via the injected `sendBatch`. Same
|
|
112
|
+
* injection seam and at-least-once semantics as `relayWithPool`.
|
|
113
|
+
*/
|
|
114
|
+
let relayClassicWithPool = async (
|
|
115
|
+
~pool: ReventlessPostgres.PgDriver.pool,
|
|
116
|
+
~logName: string,
|
|
117
|
+
~subscriber: string,
|
|
118
|
+
~sendBatch: array<JSON.t> => promise<unit>,
|
|
119
|
+
): int =>
|
|
120
|
+
await ReventlessPostgres.EventLogChangeFeed.drain(
|
|
121
|
+
pool,
|
|
122
|
+
~logName,
|
|
123
|
+
~subscriber,
|
|
124
|
+
~handle=async events => {
|
|
125
|
+
let jsons = events->Array.filterMap(toClassicEventCollectorJson)
|
|
126
|
+
if jsons->Array.length > 0 {
|
|
127
|
+
await sendBatch(jsons)
|
|
128
|
+
}
|
|
129
|
+
},
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* Deployed-Lambda entry for a classic log. Thin wrapper over `relayClassicWithPool`.
|
|
134
|
+
*/
|
|
135
|
+
let relayClassic = async (
|
|
136
|
+
~config: PgConnection.connectionConfig,
|
|
137
|
+
~logName: string,
|
|
138
|
+
~subscriber: string,
|
|
139
|
+
~sendBatch: array<JSON.t> => promise<unit>,
|
|
140
|
+
): int => {
|
|
141
|
+
let pool = PgRuntime.poolFor(config)
|
|
142
|
+
await relayClassicWithPool(~pool, ~logName, ~subscriber, ~sendBatch)
|
|
143
|
+
}
|
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
// Generated by ReScript, PLEASE EDIT WITH CARE
|
|
2
2
|
|
|
3
3
|
import * as S from "sury/src/S.res.mjs";
|
|
4
|
+
import * as Stdlib_JSON from "@rescript/runtime/lib/es6/Stdlib_JSON.js";
|
|
4
5
|
import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
|
|
5
6
|
import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
|
|
6
7
|
import * as DcbTag$Reventless from "@reventlessdev/reventless-spec/src/components/DcbTag.res.mjs";
|
|
7
8
|
import * as Message$ReventlessCore from "@reventlessdev/reventless-core/src/Message.res.mjs";
|
|
8
9
|
import * as PgRuntime$ReventlessAws from "./PgRuntime.res.mjs";
|
|
9
10
|
import * as PgChangeFeed$ReventlessPostgres from "@reventlessdev/reventless-postgres/src/PgChangeFeed.res.mjs";
|
|
11
|
+
import * as EventLogChangeFeed$ReventlessPostgres from "@reventlessdev/reventless-postgres/src/EventLogChangeFeed.res.mjs";
|
|
10
12
|
import * as Util_DynamoDbStream_Runtime$ReventlessAws from "../../util/Util_DynamoDbStream_Runtime.res.mjs";
|
|
11
13
|
import * as DcbEventLogStorage_DynamoDb_Runtime$ReventlessAws from "../DcbEventLog/DcbEventLogStorage_DynamoDb_Runtime.res.mjs";
|
|
12
14
|
|
|
@@ -37,9 +39,30 @@ async function relay(config, logName, subscriber, partitionTagJson, sendBatch) {
|
|
|
37
39
|
return await relayWithPool(pool, logName, subscriber, partitionTagJson, sendBatch);
|
|
38
40
|
}
|
|
39
41
|
|
|
42
|
+
function toClassicEventCollectorJson(event) {
|
|
43
|
+
return Stdlib_Option.flatMap(Stdlib_JSON.Decode.object(event.payload), Util_DynamoDbStream_Runtime$ReventlessAws.buildJsonEvent$p);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
async function relayClassicWithPool(pool, logName, subscriber, sendBatch) {
|
|
47
|
+
return await EventLogChangeFeed$ReventlessPostgres.drain(pool, logName, subscriber, undefined, async events => {
|
|
48
|
+
let jsons = Stdlib_Array.filterMap(events, toClassicEventCollectorJson);
|
|
49
|
+
if (jsons.length !== 0) {
|
|
50
|
+
return await sendBatch(jsons);
|
|
51
|
+
}
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
async function relayClassic(config, logName, subscriber, sendBatch) {
|
|
56
|
+
let pool = PgRuntime$ReventlessAws.poolFor(config);
|
|
57
|
+
return await relayClassicWithPool(pool, logName, subscriber, sendBatch);
|
|
58
|
+
}
|
|
59
|
+
|
|
40
60
|
export {
|
|
41
61
|
toEventCollectorJson,
|
|
42
62
|
relayWithPool,
|
|
43
63
|
relay,
|
|
64
|
+
toClassicEventCollectorJson,
|
|
65
|
+
relayClassicWithPool,
|
|
66
|
+
relayClassic,
|
|
44
67
|
}
|
|
45
68
|
/* S Not a pure module */
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
// B3.0 — projection delivery on Postgres platforms
|
|
2
|
+
// (docs/plans/aws-postgres-querydb-adapter.md).
|
|
3
|
+
//
|
|
4
|
+
// On DynamoDB platforms the ReadModel / StateViewSlice Lambdas consume their
|
|
5
|
+
// source event logs via DynamoDB-stream event-source mappings. Postgres logs have
|
|
6
|
+
// no streams — instead the shared `PgChangeFeedRelay` fans every Postgres log's
|
|
7
|
+
// events out to one SQS "feed queue" per consumer Lambda, registered here and
|
|
8
|
+
// read by `Platform.provisionPgChangeFeedRelay`. Handlers are keyed by the feed
|
|
9
|
+
// queue's ARN (the sourceUrn analogue of the stream ARN), and per-projection
|
|
10
|
+
// filtering happens in the callbacks (meta.service / event-type matching), so
|
|
11
|
+
// delivering a log's events to a consumer that doesn't project them is a no-op —
|
|
12
|
+
// the same property the DynamoDB path relies on when several read models share
|
|
13
|
+
// one source stream.
|
|
14
|
+
//
|
|
15
|
+
// Checkpoints: each (feed queue, log) pair drains under subscriber
|
|
16
|
+
// `<scope>:<logName>` — per-log isolation, same rule as the EventCollector relay
|
|
17
|
+
// targets (the subscription tables key by subscriber alone).
|
|
18
|
+
|
|
19
|
+
type feedQueue = {
|
|
20
|
+
/** Subscriber prefix — the checkpoint key is `<scope>:<logName>` per relayed log. */
|
|
21
|
+
scope: string,
|
|
22
|
+
/** Relay classic `event_log` logs into this queue. */
|
|
23
|
+
includeClassic: bool,
|
|
24
|
+
/** Relay DCB `dcb_event` logs into this queue. */
|
|
25
|
+
includeDcb: bool,
|
|
26
|
+
url: Pulumi.Output.t<string>,
|
|
27
|
+
arn: Pulumi.Output.t<string>,
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
let feedQueues: array<feedQueue> = []
|
|
31
|
+
|
|
32
|
+
/** All registered feed queues (read by `Platform.provisionPgChangeFeedRelay`). */
|
|
33
|
+
let getFeedQueues = (): array<feedQueue> => feedQueues
|
|
34
|
+
|
|
35
|
+
/** Create a consumer Lambda's feed queue and register it for relay fan-out.
|
|
36
|
+
ESM + IAM attach separately via `connect` once the Lambda exists. */
|
|
37
|
+
let makeQueue = (
|
|
38
|
+
~name: string,
|
|
39
|
+
~scope: string,
|
|
40
|
+
~includeClassic: bool,
|
|
41
|
+
~includeDcb: bool,
|
|
42
|
+
~opts: Pulumi.CustomResourceOptions.t,
|
|
43
|
+
): PulumiAws.SQS.Queue.t => {
|
|
44
|
+
let queue = PulumiAws.SQS.Queue.make(
|
|
45
|
+
~name,
|
|
46
|
+
~args={
|
|
47
|
+
PulumiAws.SQS.Queue.visibilityTimeoutSeconds: 120->Pulumi.Input.make,
|
|
48
|
+
redrivePolicy: Util_DeadLetterQueue.queue.arn
|
|
49
|
+
->Pulumi.Output.apply(dlqArn =>
|
|
50
|
+
PulumiAws.SQS.Queue.RedrivePolicy.make(~deadLetterTargetArn=dlqArn, ~maxReceiveCount=5)
|
|
51
|
+
)
|
|
52
|
+
->Pulumi.Output.asInput,
|
|
53
|
+
sqsManagedSseEnabled: false->Pulumi.Input.make,
|
|
54
|
+
},
|
|
55
|
+
~opts,
|
|
56
|
+
)
|
|
57
|
+
feedQueues
|
|
58
|
+
->Array.push({scope, includeClassic, includeDcb, url: queue.id, arn: queue.arn})
|
|
59
|
+
->ignore
|
|
60
|
+
queue
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/** Wire a feed queue into its consumer Lambda: SQS event-source mapping +
|
|
64
|
+
receive IAM on the Lambda role. */
|
|
65
|
+
let connect = (
|
|
66
|
+
~name: string,
|
|
67
|
+
~queue: PulumiAws.SQS.Queue.t,
|
|
68
|
+
~lambda: Pulumi.Output.t<PulumiAws.Lambda.Function.t>,
|
|
69
|
+
~lambdaRole: PulumiAws.IAM.Role.t,
|
|
70
|
+
~opts: Pulumi.CustomResourceOptions.t,
|
|
71
|
+
) => {
|
|
72
|
+
let _esm = Util_EventSourceMapping.subscribeSqs(~lambda, ~name, ~queue, ~opts)
|
|
73
|
+
let _policy = queue.arn->Pulumi.Output.apply(queueArn => {
|
|
74
|
+
open PulumiAws.PolicyDocument
|
|
75
|
+
PulumiAws.IAM.RolePolicy.make(
|
|
76
|
+
~name=`${name}Receive`,
|
|
77
|
+
~args={
|
|
78
|
+
policy: PulumiAws.PolicyDocument.make(
|
|
79
|
+
~id=`${name}ReceivePolicy`,
|
|
80
|
+
~statements=[
|
|
81
|
+
{
|
|
82
|
+
sid: "AllowReceiveFeedQueue",
|
|
83
|
+
effect: Allow,
|
|
84
|
+
actions: Actions([
|
|
85
|
+
"sqs:ReceiveMessage",
|
|
86
|
+
"sqs:DeleteMessage",
|
|
87
|
+
"sqs:GetQueueAttributes",
|
|
88
|
+
]),
|
|
89
|
+
resources: Resource(queueArn),
|
|
90
|
+
},
|
|
91
|
+
],
|
|
92
|
+
)
|
|
93
|
+
->PulumiAws.PolicyDocument.toJsonString
|
|
94
|
+
->Pulumi.Input.make,
|
|
95
|
+
role: lambdaRole.id->Pulumi.Output.asInput,
|
|
96
|
+
},
|
|
97
|
+
~opts,
|
|
98
|
+
)
|
|
99
|
+
})
|
|
100
|
+
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
// Generated by ReScript, PLEASE EDIT WITH CARE
|
|
2
|
+
|
|
3
|
+
import * as Aws from "@pulumi/aws";
|
|
4
|
+
import * as SQS_Queue$PulumiAws from "@reventlessdev/rescript-pulumi-aws/src/SQS/SQS_Queue.res.mjs";
|
|
5
|
+
import * as PolicyDocument$PulumiAws from "@reventlessdev/rescript-pulumi-aws/src/IAM/PolicyDocument.res.mjs";
|
|
6
|
+
import * as Util_DeadLetterQueue$ReventlessAws from "../../util/Util_DeadLetterQueue.res.mjs";
|
|
7
|
+
import * as Util_EventSourceMapping$ReventlessAws from "../../util/Util_EventSourceMapping.res.mjs";
|
|
8
|
+
|
|
9
|
+
let feedQueues = [];
|
|
10
|
+
|
|
11
|
+
function getFeedQueues() {
|
|
12
|
+
return feedQueues;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
function makeQueue(name, scope, includeClassic, includeDcb, opts) {
|
|
16
|
+
let queue = new (Aws.sqs.Queue)(name, {
|
|
17
|
+
redrivePolicy: Util_DeadLetterQueue$ReventlessAws.queue.arn.apply(dlqArn => SQS_Queue$PulumiAws.RedrivePolicy.make(dlqArn, 5)),
|
|
18
|
+
visibilityTimeoutSeconds: 120,
|
|
19
|
+
sqsManagedSseEnabled: false
|
|
20
|
+
}, opts);
|
|
21
|
+
feedQueues.push({
|
|
22
|
+
scope: scope,
|
|
23
|
+
includeClassic: includeClassic,
|
|
24
|
+
includeDcb: includeDcb,
|
|
25
|
+
url: queue.id,
|
|
26
|
+
arn: queue.arn
|
|
27
|
+
});
|
|
28
|
+
return queue;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function connect(name, queue, lambda, lambdaRole, opts) {
|
|
32
|
+
Util_EventSourceMapping$ReventlessAws.subscribeSqs(lambda, name, queue, undefined, opts);
|
|
33
|
+
queue.arn.apply(queueArn => new (Aws.iam.RolePolicy)(name + `Receive`, {
|
|
34
|
+
policy: PolicyDocument$PulumiAws.toJsonString(PolicyDocument$PulumiAws.make(undefined, name + `ReceivePolicy`, [{
|
|
35
|
+
Sid: "AllowReceiveFeedQueue",
|
|
36
|
+
Effect: "Allow",
|
|
37
|
+
Action: [
|
|
38
|
+
"sqs:ReceiveMessage",
|
|
39
|
+
"sqs:DeleteMessage",
|
|
40
|
+
"sqs:GetQueueAttributes"
|
|
41
|
+
],
|
|
42
|
+
Resource: queueArn
|
|
43
|
+
}])),
|
|
44
|
+
role: lambdaRole.id
|
|
45
|
+
}, opts));
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export {
|
|
49
|
+
feedQueues,
|
|
50
|
+
getFeedQueues,
|
|
51
|
+
makeQueue,
|
|
52
|
+
connect,
|
|
53
|
+
}
|
|
54
|
+
/* @pulumi/aws Not a pure module */
|