@reventlessdev/reventless-core 3.0.0-alpha.341 → 3.0.0-alpha.343
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/package.json +7 -7
- package/src/Projection.res +32 -4
- package/src/Projection.res.mjs +51 -4
- package/src/ViewChange.res +1 -9
- package/src/ViewChange.res.mjs +2 -13
- package/src/admin/Platform_ComponentDefinitionsApi.res +29 -1
- package/src/admin/Platform_ComponentDefinitionsApi.res.mjs +47 -1
- package/src/admin/Platform_PluginStructuresApi.res +4 -3
- package/src/admin/Platform_PluginStructuresApi.res.mjs +5 -1
- package/src/components/Api/SuryToJsonSchema.res +7 -0
- package/src/components/Api/SuryToJsonSchema.res.mjs +18 -0
- package/src/components/Dcb/Dcb_Builder.res +59 -0
- package/src/components/Dcb/Dcb_Builder.res.mjs +48 -3
- package/src/components/DcbEventLog/DcbEventLog.res +2 -0
- package/src/components/DcbEventLog/DcbEventLog_Adapter.res +7 -0
- package/src/components/DcbEventLog/DcbEventLog_Builder.res +12 -3
- package/src/components/DcbEventLog/DcbEventLog_Builder.res.mjs +5 -3
- package/src/components/DcbEventLog/DcbEventLog_Operations.res +3 -0
- package/src/components/DcbEventLog/DcbEventLog_Operations.res.mjs +3 -1
- package/src/components/OutboundTranslationSlice/OutboundTranslationSlice_Builder.res.mjs +2 -2
- package/src/components/OutboundTranslationSlice/OutboundTranslationSlice_Callback.res +156 -107
- package/src/components/OutboundTranslationSlice/OutboundTranslationSlice_Callback.res.mjs +174 -154
- package/src/components/StateChangeSlice/StateChangeSlice.res +3 -0
- package/src/components/StateChangeSlice/StateChangeSlice_Builder.res +6 -0
- package/src/components/StateChangeSlice/StateChangeSlice_Builder.res.mjs +5 -4
- package/src/components/StateChangeSlice/StateChangeSlice_Callback.res +369 -279
- package/src/components/StateChangeSlice/StateChangeSlice_Callback.res.mjs +210 -166
- package/src/plugin/component/Plugin_Structure.res +45 -0
- package/src/plugin/component/Plugin_Structure.res.mjs +42 -1
- package/src/util/Logger.res +5 -5
- package/src/util/Logger.res.mjs +3 -3
- package/tests/admin/AdminApiSchemaDriftTest.res +23 -0
- package/tests/admin/AdminApiSchemaDriftTest.res.mjs +21 -1
- package/tests/admin/Platform_ComponentDefinitionsApiTest.res +1 -0
- package/tests/admin/Platform_ComponentDefinitionsApiTest.res.mjs +1 -0
- package/tests/admin/Platform_PluginStructuresApiTest.res +1 -0
- package/tests/admin/Platform_PluginStructuresApiTest.res.mjs +1 -0
- package/tests/api/QuantitySemanticTest.res +68 -0
- package/tests/api/QuantitySemanticTest.res.mjs +68 -0
- package/tests/commandgenerator/OwnerActingTest.res.mjs +1 -1
- package/tests/commandtopic/CommandOutcomeHookTest.res.mjs +1 -1
- package/tests/dcb/DcbFenceCountReadTest.res +213 -0
- package/tests/dcb/DcbFenceCountReadTest.res.mjs +358 -0
- package/tests/dcb/DcbPiiSliceTest.res.mjs +1 -1
- package/tests/dcb/DcbReadsNothingTest.res.mjs +1 -1
- package/tests/dcb/DcbScopeInferenceTest.res +143 -2
- package/tests/dcb/DcbScopeInferenceTest.res.mjs +192 -2
- package/tests/dcb/DcbStateChangeSliceTest.res.mjs +22 -22
- package/tests/dcb/DcbTagTest.res +68 -0
- package/tests/dcb/DcbTagTest.res.mjs +86 -10
- package/tests/dcb/DcbValidationTest.res.mjs +1 -1
- package/tests/logger/LoggerLazyTest.res.mjs +3 -3
- package/tests/plugin/PluginStructureAccessTest.res +1 -0
- package/tests/plugin/PluginStructureAccessTest.res.mjs +1 -1
- package/tests/plugin/PluginStructureTest.res +61 -0
- package/tests/plugin/PluginStructureTest.res.mjs +78 -32
- package/tests/plugin/StateChange/PsRateSession.res +37 -0
- package/tests/plugin/StateChange/PsRateSession.res.mjs +80 -0
- package/tests/plugin/pluginDefinitionRequiredScalars.txt +10 -0
- package/tests/projection/ViewChangeTest.res +32 -0
- package/tests/projection/ViewChangeTest.res.mjs +57 -0
|
@@ -229,9 +229,54 @@ function Make(DcbEventLogStorage) {
|
|
|
229
229
|
let scopeIssues = DcbValidation$Reventless.validateScopeVsInference(scopeAnnotations, inferred);
|
|
230
230
|
scopeIssues.contradictions.forEach(e => log.warn("Dcb_Builder", undefined, `DCB scope contradiction (` + e.sliceName + `): ` + e.message));
|
|
231
231
|
scopeIssues.redundancies.forEach(e => log.info("Dcb_Builder", undefined, `DCB scope (` + e.sliceName + `): ` + e.message));
|
|
232
|
+
let armIssues = DcbValidation$Reventless.validateCrossPartitionArms(inferenceShapes, inferred);
|
|
233
|
+
armIssues.redundancies.forEach(e => log.info("Dcb_Builder", undefined, `DCB scope (` + e.sliceName + `): ` + e.message));
|
|
234
|
+
if (armIssues.contradictions.length !== 0) {
|
|
235
|
+
let detail$1 = armIssues.contradictions.map(e => e.sliceName + `: ` + e.message).join(" | ");
|
|
236
|
+
log.error("Dcb_Builder", undefined, `DCB scope contradiction (` + name + `): ` + detail$1);
|
|
237
|
+
Stdlib_JsError.throwWithMessage(`DCB scope contradiction (` + name + `): ` + detail$1);
|
|
238
|
+
}
|
|
232
239
|
let effective = DcbTag$Reventless.deriveEffectiveScope(sliceSchemas);
|
|
233
240
|
let effectiveCrossPartitionTagKeys = effective.crossPartitionTagKeys;
|
|
234
241
|
let effectiveTagKeysByEventType = effective.tagKeysByEventType;
|
|
242
|
+
effective.crossPartitionPairs.forEach(pair => {
|
|
243
|
+
pair.writers.forEach(writer => {
|
|
244
|
+
let writes = s => s.sliceName === writer;
|
|
245
|
+
let otherPairs = effective.crossPartitionPairs.some(p => {
|
|
246
|
+
if (p.writers.includes(writer)) {
|
|
247
|
+
if (p.eventType !== pair.eventType) {
|
|
248
|
+
return true;
|
|
249
|
+
} else {
|
|
250
|
+
return p.key !== pair.key;
|
|
251
|
+
}
|
|
252
|
+
} else {
|
|
253
|
+
return false;
|
|
254
|
+
}
|
|
255
|
+
});
|
|
256
|
+
let carriesCrossPartition = Stdlib_Option.mapOr(inferenceShapes.find(writes), false, s => DcbScopeInference$Reventless.producedKeys(s).some(k => effectiveCrossPartitionTagKeys.includes(k)));
|
|
257
|
+
let match = DcbTag$Reventless.slicePartitionTag(boundaryPartition, writer);
|
|
258
|
+
let composite;
|
|
259
|
+
if (match !== undefined) {
|
|
260
|
+
switch (match.TAG) {
|
|
261
|
+
case "Composite" :
|
|
262
|
+
composite = true;
|
|
263
|
+
break;
|
|
264
|
+
case "Simple" :
|
|
265
|
+
case "ByEventType" :
|
|
266
|
+
composite = false;
|
|
267
|
+
break;
|
|
268
|
+
}
|
|
269
|
+
} else {
|
|
270
|
+
composite = false;
|
|
271
|
+
}
|
|
272
|
+
let message = `DCB pair (` + name + `): ` + writer + ` now updates fence ` + pair.key + ` for ` + pair.reader + `, which reads ` + pair.eventType + ` by ` + pair.key + ` across partitions`;
|
|
273
|
+
if (otherPairs || carriesCrossPartition || composite) {
|
|
274
|
+
return log.warn("Dcb_Builder", undefined, message + `; its appends already carry other cross-partition, composite or pair fences, and each adds an item to a transaction capped at 100`);
|
|
275
|
+
} else {
|
|
276
|
+
return log.info("Dcb_Builder", undefined, message);
|
|
277
|
+
}
|
|
278
|
+
});
|
|
279
|
+
});
|
|
235
280
|
DcbValidation$Reventless.validateCompositeReads(stateChangeSlices.map(Sc => [
|
|
236
281
|
Sc.Spec.name,
|
|
237
282
|
Sc.Spec.commandSchema,
|
|
@@ -244,7 +289,7 @@ function Make(DcbEventLogStorage) {
|
|
|
244
289
|
log.error("Dcb_Builder", undefined, `DCB scope degraded (` + name + `): cross-partition reads of [` + effective.droppedCrossPartitionTagKeys.join(", ") + `] were derived but dropped — an ambiguous slice forced the boundary onto @crossPartition annotations that do not carry them. Slices referencing another entity by these keys will decide against an empty history and reject valid commands. Resolve: ` + effective.ambiguities.map(param => param[0] + ` (` + param[1] + `)`).join(" | "));
|
|
245
290
|
}
|
|
246
291
|
let DcbEventLog = DcbEventLog_Builder$ReventlessCore.Make(DcbEventLogStorage)(DcbEventTopicPublisher);
|
|
247
|
-
let dcbEventLog = DcbEventLog.make(name, indexes$1, partitionTag, effectiveCrossPartitionTagKeys, opts);
|
|
292
|
+
let dcbEventLog = DcbEventLog.make(name, indexes$1, partitionTag, effectiveCrossPartitionTagKeys, DcbScopeInference$Reventless.pairFenceTypes(effective.crossPartitionPairs), opts);
|
|
248
293
|
Stdlib_Option.forEach(HooksConfig.hooks.onDcbEventLogCreated, hook => hook(dcbEventLog));
|
|
249
294
|
let commandSchema = Sury.json;
|
|
250
295
|
let DcbCommandTopic = CommandTopic_Builder$ReventlessCore.Make({
|
|
@@ -278,7 +323,7 @@ function Make(DcbEventLogStorage) {
|
|
|
278
323
|
asyncDcbCommandTopicOpt = undefined;
|
|
279
324
|
}
|
|
280
325
|
let stateChangeSlicesOutputs = Object.fromEntries(syncSlices.map(StateChangeSlice => {
|
|
281
|
-
let ch = StateChangeSlice.make(dcbEventLog, publishJsons, effectiveTagKeysByEventType, effectiveCrossPartitionTagKeys, DcbTag$Reventless.slicePartitionTag(boundaryPartition, StateChangeSlice.Spec.name), componentRuntime[StateChangeSlice.Spec.name], opts);
|
|
326
|
+
let ch = StateChangeSlice.make(dcbEventLog, publishJsons, effectiveTagKeysByEventType, effectiveCrossPartitionTagKeys, effective.crossPartitionPairs, DcbTag$Reventless.slicePartitionTag(boundaryPartition, StateChangeSlice.Spec.name), componentRuntime[StateChangeSlice.Spec.name], opts);
|
|
282
327
|
return [
|
|
283
328
|
StateChangeSlice.Spec.name,
|
|
284
329
|
Component$ReventlessCore.outputs(ch)
|
|
@@ -288,7 +333,7 @@ function Make(DcbEventLogStorage) {
|
|
|
288
333
|
if (asyncDcbCommandTopicOpt !== undefined) {
|
|
289
334
|
let asyncPublishJsons = Component$ReventlessCore.operations(Primitive_option.valFromOption(asyncDcbCommandTopicOpt)).apply(ops => ops.publishJsons);
|
|
290
335
|
asyncStateChangeSlicesOutputs = Object.fromEntries(asyncSlices.map(StateChangeSlice => {
|
|
291
|
-
let ch = StateChangeSlice.make(dcbEventLog, asyncPublishJsons, effectiveTagKeysByEventType, effectiveCrossPartitionTagKeys, DcbTag$Reventless.slicePartitionTag(boundaryPartition, StateChangeSlice.Spec.name), componentRuntime[StateChangeSlice.Spec.name], opts);
|
|
336
|
+
let ch = StateChangeSlice.make(dcbEventLog, asyncPublishJsons, effectiveTagKeysByEventType, effectiveCrossPartitionTagKeys, effective.crossPartitionPairs, DcbTag$Reventless.slicePartitionTag(boundaryPartition, StateChangeSlice.Spec.name), componentRuntime[StateChangeSlice.Spec.name], opts);
|
|
292
337
|
return [
|
|
293
338
|
StateChangeSlice.Spec.name,
|
|
294
339
|
Component$ReventlessCore.outputs(ch)
|
|
@@ -11,6 +11,7 @@ type read = ReventlessInfra.DcbEventLog.read
|
|
|
11
11
|
type append = ReventlessInfra.DcbEventLog.append
|
|
12
12
|
type readStream = ReventlessInfra.DcbEventLog.readStream
|
|
13
13
|
type appendStream = ReventlessInfra.DcbEventLog.appendStream
|
|
14
|
+
type readFences = ReventlessInfra.DcbEventLog.readFences
|
|
14
15
|
type operations = ReventlessInfra.DcbEventLog.operations
|
|
15
16
|
type component = ReventlessInfra.DcbEventLog.component
|
|
16
17
|
|
|
@@ -38,6 +39,7 @@ module type T = {
|
|
|
38
39
|
~indexes: array<string>=?,
|
|
39
40
|
~partitionTag: Reventless.DcbTag.derivedPartitionTag,
|
|
40
41
|
~crossPartitionTagKeys: array<string>=?,
|
|
42
|
+
~pairFenceTypes: dict<array<string>>=?,
|
|
41
43
|
~opts: Pulumi.ComponentResource.options=?,
|
|
42
44
|
) => component
|
|
43
45
|
}
|
|
@@ -33,6 +33,10 @@ type operations = {
|
|
|
33
33
|
~after: Reventless.DcbTag.sequencePosition=?,
|
|
34
34
|
~strongConsistency: bool=?,
|
|
35
35
|
) => Stream.t<rawSequencedEvent, string, unit>,
|
|
36
|
+
/** Reads the fences of cross-partition tags, strongly consistent, for a reader
|
|
37
|
+
of an eventually consistent index to check its read against. Absent on a
|
|
38
|
+
store whose reads are never stale. */
|
|
39
|
+
readFences?: array<Reventless.DcbTag.tag> => promise<array<Reventless.DcbFenceCount.fence>>,
|
|
36
40
|
}
|
|
37
41
|
|
|
38
42
|
type storage = {
|
|
@@ -49,6 +53,9 @@ type storageMaker = (
|
|
|
49
53
|
(per-tag GSI) and its consistency fence is bumped by every carrier. Derived
|
|
50
54
|
once at build time so read-scope = fence-scope per tag. */
|
|
51
55
|
~crossPartitionTagKeys: array<string>=?,
|
|
56
|
+
/** Key -> the event types with a pair fence on it (`DcbScopeInference.pairFenceTypes`):
|
|
57
|
+
every append of such a type carrying the key moves that fence. */
|
|
58
|
+
~pairFenceTypes: dict<array<string>>=?,
|
|
52
59
|
~opts: Pulumi.CustomResourceOptions.t,
|
|
53
60
|
) => storage
|
|
54
61
|
|
|
@@ -12,7 +12,7 @@ module Make = (
|
|
|
12
12
|
|
|
13
13
|
type component = ReventlessInfra.DcbEventLog.component
|
|
14
14
|
|
|
15
|
-
let construct = (indexes, partitionTag, crossPartitionTagKeys, self, name) => {
|
|
15
|
+
let construct = (indexes, partitionTag, crossPartitionTagKeys, pairFenceTypes, self, name) => {
|
|
16
16
|
let opts = {Pulumi.CustomResourceOptions.parent: self->Component.toPulumiResource}
|
|
17
17
|
|
|
18
18
|
let logName = name->ComponentType.name(DcbEventLog.componentType)
|
|
@@ -24,6 +24,7 @@ module Make = (
|
|
|
24
24
|
~indexes,
|
|
25
25
|
~partitionTag,
|
|
26
26
|
~crossPartitionTagKeys,
|
|
27
|
+
~pairFenceTypes,
|
|
27
28
|
~opts,
|
|
28
29
|
)
|
|
29
30
|
|
|
@@ -65,6 +66,7 @@ module Make = (
|
|
|
65
66
|
append: Ops.append,
|
|
66
67
|
readStream: Ops.readStream,
|
|
67
68
|
appendStream: Ops.appendStream,
|
|
69
|
+
readFences: ?Ops.readFences,
|
|
68
70
|
}
|
|
69
71
|
ops
|
|
70
72
|
}),
|
|
@@ -77,11 +79,18 @@ module Make = (
|
|
|
77
79
|
self->Component.setOutputs(outputs)
|
|
78
80
|
}
|
|
79
81
|
|
|
80
|
-
let make = (
|
|
82
|
+
let make = (
|
|
83
|
+
~name,
|
|
84
|
+
~indexes=[],
|
|
85
|
+
~partitionTag,
|
|
86
|
+
~crossPartitionTagKeys=[],
|
|
87
|
+
~pairFenceTypes=Dict.make(),
|
|
88
|
+
~opts=?,
|
|
89
|
+
): component =>
|
|
81
90
|
Component.make(
|
|
82
91
|
~componentType=DcbEventLog.componentType->ComponentType.toString,
|
|
83
92
|
~name,
|
|
84
|
-
~construct=construct(indexes, partitionTag, crossPartitionTagKeys, ...),
|
|
93
|
+
~construct=construct(indexes, partitionTag, crossPartitionTagKeys, pairFenceTypes, ...),
|
|
85
94
|
~opts,
|
|
86
95
|
)
|
|
87
96
|
}
|
|
@@ -14,9 +14,10 @@ import * as DcbEventLog_Operations$ReventlessCore from "./DcbEventLog_Operations
|
|
|
14
14
|
function Make(Storage) {
|
|
15
15
|
return EventTopicPublisher => {
|
|
16
16
|
let eventSchema = Sury.json;
|
|
17
|
-
let make = (name, indexesOpt, partitionTag, crossPartitionTagKeysOpt, opts) => {
|
|
17
|
+
let make = (name, indexesOpt, partitionTag, crossPartitionTagKeysOpt, pairFenceTypesOpt, opts) => {
|
|
18
18
|
let indexes = indexesOpt !== undefined ? indexesOpt : [];
|
|
19
19
|
let crossPartitionTagKeys = crossPartitionTagKeysOpt !== undefined ? crossPartitionTagKeysOpt : [];
|
|
20
|
+
let pairFenceTypes = pairFenceTypesOpt !== undefined ? pairFenceTypesOpt : ({});
|
|
20
21
|
return Component$ReventlessCore.make(ComponentType$ReventlessCore.toString(DcbEventLog$ReventlessCore.componentType), name, (extra, extra$1) => {
|
|
21
22
|
let opts_parent = Component$ReventlessCore.toPulumiResource(extra);
|
|
22
23
|
let opts = {
|
|
@@ -27,7 +28,7 @@ function Make(Storage) {
|
|
|
27
28
|
kind: "Plugin",
|
|
28
29
|
name: extra$1
|
|
29
30
|
};
|
|
30
|
-
let storage = Storage.make(logName, indexes, partitionTag, crossPartitionTagKeys, opts);
|
|
31
|
+
let storage = Storage.make(logName, indexes, partitionTag, crossPartitionTagKeys, pairFenceTypes, opts);
|
|
31
32
|
EventLogProvisioning$ReventlessCore.notify("Dcb", logName, owner, storage.resources, opts);
|
|
32
33
|
let SpecificEventTopic = EventTopic_Builder$ReventlessCore.Make({
|
|
33
34
|
Id: Id$Reventless.$$String,
|
|
@@ -51,7 +52,8 @@ function Make(Storage) {
|
|
|
51
52
|
read: Ops.read,
|
|
52
53
|
append: Ops.append,
|
|
53
54
|
readStream: Ops.readStream,
|
|
54
|
-
appendStream: Ops.appendStream
|
|
55
|
+
appendStream: Ops.appendStream,
|
|
56
|
+
readFences: Ops.readFences
|
|
55
57
|
};
|
|
56
58
|
}));
|
|
57
59
|
let outputs_resources = storage.resources;
|
|
@@ -14,6 +14,7 @@ module type T = {
|
|
|
14
14
|
let append: DcbEventLog.append
|
|
15
15
|
let readStream: DcbEventLog.readStream
|
|
16
16
|
let appendStream: DcbEventLog.appendStream
|
|
17
|
+
let readFences: option<DcbEventLog.readFences>
|
|
17
18
|
}
|
|
18
19
|
|
|
19
20
|
module Make = (Ops: Ops): T => {
|
|
@@ -164,4 +165,6 @@ module Make = (Ops: Ops): T => {
|
|
|
164
165
|
->Stream.map((rawEvent): DcbEventLog_Adapter.rawStoredEvent => rawEvent->Obj.magic)
|
|
165
166
|
->Stream.runCollect
|
|
166
167
|
->Effect.flatMap(rawEvents => Effect.promise(() => Ops.storage.append(rawEvents, ~condition?)))
|
|
168
|
+
|
|
169
|
+
let readFences = Ops.storage.readFences
|
|
167
170
|
}
|
|
@@ -111,11 +111,13 @@ function Make(Ops) {
|
|
|
111
111
|
};
|
|
112
112
|
let readStream = (query, after, strongConsistency) => Stream.map(Ops.storage.readStream(query, after, strongConsistency), raw => raw);
|
|
113
113
|
let appendStream = (stream, condition) => Effect$1.flatMap(Stream$1.runCollect(Stream.map(stream, rawEvent => rawEvent)), rawEvents => Effect$1.promise(() => Ops.storage.append(rawEvents, condition)));
|
|
114
|
+
let readFences = Ops.storage.readFences;
|
|
114
115
|
return {
|
|
115
116
|
read: read,
|
|
116
117
|
append: append,
|
|
117
118
|
readStream: readStream,
|
|
118
|
-
appendStream: appendStream
|
|
119
|
+
appendStream: appendStream,
|
|
120
|
+
readFences: readFences
|
|
119
121
|
};
|
|
120
122
|
}
|
|
121
123
|
|
|
@@ -118,7 +118,7 @@ function Make(RuntimeEnvironment) {
|
|
|
118
118
|
param[1]
|
|
119
119
|
]), eventsArr.map(param => param[2]), Capabilities.capabilities());
|
|
120
120
|
await syncToQueryDb(queryDbOps);
|
|
121
|
-
Stdlib_Promise.$$catch(Callback.phase2(publishJsonsFn, Capabilities.capabilities(), publishJsonsAndWait).then(() => syncToQueryDb(queryDbOps)), exn => {
|
|
121
|
+
Stdlib_Promise.$$catch(Callback.phase2(publishJsonsFn, Capabilities.capabilities(), publishJsonsAndWait, undefined).then(() => syncToQueryDb(queryDbOps)), exn => {
|
|
122
122
|
let errMsg = Stdlib_Option.getOr(Stdlib_Option.flatMap(Stdlib_JsExn.fromException(exn), Stdlib_JsExn.message), "unknown");
|
|
123
123
|
Effect.runSync(EffectLogger$ReventlessCore.logError(`OutboundTranslationSlice(` + Spec.name + `)`, undefined, undefined, `detached phase 2 error: ` + errMsg));
|
|
124
124
|
return Promise.resolve();
|
|
@@ -138,7 +138,7 @@ function Make(RuntimeEnvironment) {
|
|
|
138
138
|
let publishJsonsFn = param[1];
|
|
139
139
|
return {
|
|
140
140
|
enqueueEvent: param[0].enqueueEvent,
|
|
141
|
-
translatePending: async () => await Callback.phase2(publishJsonsFn, Capabilities.capabilities(), publishJsonsAndWait)
|
|
141
|
+
translatePending: async () => await Callback.phase2(publishJsonsFn, Capabilities.capabilities(), publishJsonsAndWait, undefined)
|
|
142
142
|
};
|
|
143
143
|
}));
|
|
144
144
|
let aggregatedResources = Object.values(eventTopics).flatMap(t => t.resources);
|
|
@@ -32,6 +32,9 @@ type todoRow = {
|
|
|
32
32
|
/** The event that created the row, so the command sent for it continues its
|
|
33
33
|
flow. Absent on rows written before it existed. */
|
|
34
34
|
source?: Message.flowSource,
|
|
35
|
+
/** Until when the runtime that claimed the row to translate it holds it. Past
|
|
36
|
+
it, a `Processing` row is one whose runtime stopped, and may be claimed again. */
|
|
37
|
+
claimedUntil?: string,
|
|
35
38
|
}
|
|
36
39
|
|
|
37
40
|
/** The item schema seen through the `JSON.t` the row actually stores.
|
|
@@ -66,6 +69,7 @@ let todoRowSchemaFor = (itemSchema: S.t<'item>): S.t<todoRow> =>
|
|
|
66
69
|
maxRetries: ?s.matches(S.option(S.int)),
|
|
67
70
|
lastError: ?s.matches(S.option(S.string)),
|
|
68
71
|
source: ?s.matches(S.option(Message.flowSourceSchema)),
|
|
72
|
+
claimedUntil: ?s.matches(S.option(S.string)),
|
|
69
73
|
})
|
|
70
74
|
|
|
71
75
|
/**
|
|
@@ -89,6 +93,26 @@ type phase1<'event> = (
|
|
|
89
93
|
~capabilities: Reventless.Capabilities.t,
|
|
90
94
|
) => promise<unit>
|
|
91
95
|
|
|
96
|
+
/**
|
|
97
|
+
Where rows are shared by runtimes that run at once, as Lambdas share a table, each
|
|
98
|
+
row is claimed before it is translated, so no two translate it. `claim(id, row)`
|
|
99
|
+
stores `row` (already `Processing`, with its `claimedUntil`) only if the stored row
|
|
100
|
+
is still the one this runtime read, and says whether it did. A row it could not
|
|
101
|
+
claim is another runtime's: phase 2 drops it from its own list. Without a claim,
|
|
102
|
+
the list is this runtime's alone.
|
|
103
|
+
*/
|
|
104
|
+
type claim = (string, todoRow) => promise<bool>
|
|
105
|
+
|
|
106
|
+
/** How long a claim holds: past the longest a runtime may run. */
|
|
107
|
+
let claimMs =
|
|
108
|
+
15. *. 60. *. 1000.
|
|
109
|
+
|
|
110
|
+
/** Whether a `Processing` row's claim has run out at `now`. A row with no claim is
|
|
111
|
+
one an older build left `Processing`: its runtime is gone. */
|
|
112
|
+
let claimRanOut = (row: todoRow, ~now: Date.t) =>
|
|
113
|
+
row.status == Processing &&
|
|
114
|
+
row.claimedUntil->Option.mapOr(true, until => Date.fromString(until) < now)
|
|
115
|
+
|
|
92
116
|
/** Phase 2: translate all pending items, optionally publishing commands.
|
|
93
117
|
|
|
94
118
|
`~capabilities` is supplied by whoever drives the callback — the in-process
|
|
@@ -99,6 +123,7 @@ type phase2 = (
|
|
|
99
123
|
ReventlessInfra.CommandTopic.publishJsons,
|
|
100
124
|
~capabilities: Reventless.Capabilities.t,
|
|
101
125
|
~publishJsonsAndWait: CommandTopic_Helpers.publishJsonsAndWait=?,
|
|
126
|
+
~claim: claim=?,
|
|
102
127
|
) => promise<unit>
|
|
103
128
|
|
|
104
129
|
/** The whole protocol as a value — what a driver holding a built callback needs,
|
|
@@ -317,6 +342,7 @@ module Make = (
|
|
|
317
342
|
publishJsons: ReventlessInfra.CommandTopic.publishJsons,
|
|
318
343
|
~capabilities: Reventless.Capabilities.t,
|
|
319
344
|
~publishJsonsAndWait: option<CommandTopic_Helpers.publishJsonsAndWait>=?,
|
|
345
|
+
~claim: option<claim>=?,
|
|
320
346
|
) => {
|
|
321
347
|
let publishJsons = CommandSent.announcing(
|
|
322
348
|
~comp=`OutboundTranslationSlice(${Spec.name})`,
|
|
@@ -347,155 +373,178 @@ module Make = (
|
|
|
347
373
|
todoItems
|
|
348
374
|
->Dict.toArray
|
|
349
375
|
->Array.filter(((_, row)) =>
|
|
350
|
-
row.status == Pending ||
|
|
376
|
+
row.status == Pending ||
|
|
377
|
+
row.status == Failed && row.retryCount < Spec.maxRetries ||
|
|
378
|
+
claimRanOut(row, ~now=Date.make())
|
|
351
379
|
)
|
|
352
380
|
|
|
353
381
|
// Process each item individually -- each translate call is independent
|
|
354
|
-
let _ = await pending->Array.reduce(Promise.resolve(), async (prev, (id,
|
|
382
|
+
let _ = await pending->Array.reduce(Promise.resolve(), async (prev, (id, read)) => {
|
|
355
383
|
let _ = await prev
|
|
356
384
|
|
|
357
|
-
//
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
| Error(e) => Error(#Unopened(e->Reventless.Pii.errorMessage))
|
|
368
|
-
}
|
|
369
|
-
| other => Ok(other)
|
|
385
|
+
// Claimed first, so no other runtime translates it too.
|
|
386
|
+
let claimed = {
|
|
387
|
+
...read,
|
|
388
|
+
status: Processing,
|
|
389
|
+
processedAt: now(),
|
|
390
|
+
claimedUntil: Date.fromTime(Date.now() +. claimMs)->Date.toISOString,
|
|
391
|
+
}
|
|
392
|
+
let ours = switch claim {
|
|
393
|
+
| None => true
|
|
394
|
+
| Some(claim) => await claim(id, claimed)
|
|
370
395
|
}
|
|
371
|
-
|
|
372
|
-
| Error(#Gone(field)) =>
|
|
396
|
+
if !ours {
|
|
373
397
|
EffectLogger.logInfo(
|
|
374
398
|
~comp=`OutboundTranslationSlice(${Spec.name})`,
|
|
375
|
-
~ids=?
|
|
376
|
-
`${id}:
|
|
399
|
+
~ids=?read.source->Option.map(EffectLogger.idsOfSource),
|
|
400
|
+
`${id}: another runtime holds it; left to that one`,
|
|
377
401
|
)->Effect.runSync
|
|
378
|
-
todoItems->Dict.
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
402
|
+
todoItems->Dict.delete(id)
|
|
403
|
+
} else {
|
|
404
|
+
todoItems->Dict.set(id, claimed)
|
|
405
|
+
let row = read
|
|
406
|
+
|
|
407
|
+
// Opened first: a personal field is sealed in the row, and one whose
|
|
408
|
+
// subject was erased since leaves nothing to send.
|
|
409
|
+
let opened = switch row.item {
|
|
410
|
+
| Object(data) if personalFields != [] =>
|
|
411
|
+
switch await Reventless.Pii.openRecord(Spec.outboundItemSchema, data) {
|
|
412
|
+
| Ok(data) =>
|
|
413
|
+
switch Reventless.Pii.goneField(Spec.outboundItemSchema, data) {
|
|
414
|
+
| Some(field) => Error(#Gone(field))
|
|
415
|
+
| None => Ok(JSON.Object(data))
|
|
416
|
+
}
|
|
417
|
+
| Error(e) => Error(#Unopened(e->Reventless.Pii.errorMessage))
|
|
418
|
+
}
|
|
419
|
+
| other => Ok(other)
|
|
390
420
|
}
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
421
|
+
let item = switch opened {
|
|
422
|
+
| Error(#Gone(field)) =>
|
|
423
|
+
EffectLogger.logInfo(
|
|
424
|
+
~comp=`OutboundTranslationSlice(${Spec.name})`,
|
|
425
|
+
~ids=?row.source->Option.map(EffectLogger.idsOfSource),
|
|
426
|
+
`${id}: its ${field} was erased; nothing to do`,
|
|
427
|
+
)->Effect.runSync
|
|
428
|
+
todoItems->Dict.set(id, {...row, status: Completed, completedAt: now()})
|
|
429
|
+
None
|
|
430
|
+
| Error(#Unopened(why)) =>
|
|
397
431
|
EffectLogger.logError(
|
|
398
432
|
~comp=`OutboundTranslationSlice(${Spec.name})`,
|
|
399
433
|
~ids=?row.source->Option.map(EffectLogger.idsOfSource),
|
|
400
|
-
|
|
434
|
+
`${id}: its work item does not open: ${why}`,
|
|
401
435
|
)->Effect.runSync
|
|
436
|
+
let updated = row->afterFailedAttempt(~lastError=why)
|
|
437
|
+
todoItems->Dict.set(id, updated)
|
|
438
|
+
if updated.status == Abandoned {
|
|
439
|
+
newlyAbandoned->Array.push(id)
|
|
440
|
+
}
|
|
402
441
|
None
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
switch item {
|
|
407
|
-
| None => ()
|
|
408
|
-
| Some(item) =>
|
|
409
|
-
todoItems->Dict.set(id, {...row, status: Processing, processedAt: now()})
|
|
410
|
-
|
|
411
|
-
let result = try await Translation.translate(id, item, ~capabilities) catch {
|
|
412
|
-
| exn =>
|
|
413
|
-
let msg =
|
|
414
|
-
exn
|
|
415
|
-
->JsExn.fromException
|
|
416
|
-
->Option.flatMap(JsExn.message)
|
|
417
|
-
->Option.getOr("unknown error")
|
|
418
|
-
Error(msg)
|
|
419
|
-
}
|
|
420
|
-
|
|
421
|
-
switch result {
|
|
422
|
-
| Ok(Some((targetId, cmd))) =>
|
|
423
|
-
// Encode command and publish
|
|
424
|
-
let commandJson = try cmd
|
|
425
|
-
->Reventless.Util_Sury.toJson(Spec.inboundCommandSchema)
|
|
426
|
-
->Some catch {
|
|
442
|
+
| Ok(itemJson) =>
|
|
443
|
+
try itemJson->Reventless.Util_Sury.fromJson(Spec.outboundItemSchema)->Some catch {
|
|
427
444
|
| exn =>
|
|
428
445
|
let errMsg =
|
|
429
446
|
exn->JsExn.fromException->Option.flatMap(JsExn.message)->Option.getOr("unknown")
|
|
430
447
|
EffectLogger.logError(
|
|
431
448
|
~comp=`OutboundTranslationSlice(${Spec.name})`,
|
|
432
449
|
~ids=?row.source->Option.map(EffectLogger.idsOfSource),
|
|
433
|
-
`failed to
|
|
450
|
+
`failed to decode outboundItem: ${errMsg}`,
|
|
434
451
|
)->Effect.runSync
|
|
435
452
|
None
|
|
436
453
|
}
|
|
454
|
+
}
|
|
437
455
|
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
switch refusal {
|
|
458
|
-
| Some((cause, lastError)) =>
|
|
459
|
-
let updated = row->afterRefusal(~cause, ~lastError)
|
|
460
|
-
todoItems->Dict.set(id, updated)
|
|
461
|
-
if updated.status == Abandoned {
|
|
462
|
-
newlyAbandoned->Array.push(id)
|
|
463
|
-
}
|
|
464
|
-
| None => todoItems->Dict.set(id, {...row, status: Completed, completedAt: now()})
|
|
465
|
-
}
|
|
466
|
-
} catch {
|
|
456
|
+
switch item {
|
|
457
|
+
| None => ()
|
|
458
|
+
| Some(item) =>
|
|
459
|
+
let result = try await Translation.translate(id, item, ~capabilities) catch {
|
|
460
|
+
| exn =>
|
|
461
|
+
let msg =
|
|
462
|
+
exn
|
|
463
|
+
->JsExn.fromException
|
|
464
|
+
->Option.flatMap(JsExn.message)
|
|
465
|
+
->Option.getOr("unknown error")
|
|
466
|
+
Error(msg)
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
switch result {
|
|
470
|
+
| Ok(Some((targetId, cmd))) =>
|
|
471
|
+
// Encode command and publish
|
|
472
|
+
let commandJson = try cmd
|
|
473
|
+
->Reventless.Util_Sury.toJson(Spec.inboundCommandSchema)
|
|
474
|
+
->Some catch {
|
|
467
475
|
| exn =>
|
|
468
476
|
let errMsg =
|
|
469
477
|
exn->JsExn.fromException->Option.flatMap(JsExn.message)->Option.getOr("unknown")
|
|
470
478
|
EffectLogger.logError(
|
|
471
479
|
~comp=`OutboundTranslationSlice(${Spec.name})`,
|
|
472
480
|
~ids=?row.source->Option.map(EffectLogger.idsOfSource),
|
|
473
|
-
`failed to
|
|
481
|
+
`failed to encode inbound command: ${errMsg}`,
|
|
474
482
|
)->Effect.runSync
|
|
475
|
-
|
|
483
|
+
None
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
switch commandJson {
|
|
487
|
+
| Some(commandJson) =>
|
|
488
|
+
try {
|
|
489
|
+
let msg: Reventless.Message.commandJson = {
|
|
490
|
+
id: targetId,
|
|
491
|
+
meta: makeMeta(row),
|
|
492
|
+
commandJson,
|
|
493
|
+
}
|
|
494
|
+
let refusal = switch publishJsonsAndWait {
|
|
495
|
+
| Some(wait) =>
|
|
496
|
+
switch (await wait([msg]))->Array.get(0) {
|
|
497
|
+
| Some(Rejected({errorCode, errorDetail, cause})) =>
|
|
498
|
+
Some((cause, CommandTopic_Helpers.describeRefusal(~errorCode, ~errorDetail)))
|
|
499
|
+
| Some(Accepted(_) | Pending(_)) | None => None
|
|
500
|
+
}
|
|
501
|
+
| None =>
|
|
502
|
+
await publishJsons([msg])
|
|
503
|
+
None
|
|
504
|
+
}
|
|
505
|
+
switch refusal {
|
|
506
|
+
| Some((cause, lastError)) =>
|
|
507
|
+
let updated = row->afterRefusal(~cause, ~lastError)
|
|
508
|
+
todoItems->Dict.set(id, updated)
|
|
509
|
+
if updated.status == Abandoned {
|
|
510
|
+
newlyAbandoned->Array.push(id)
|
|
511
|
+
}
|
|
512
|
+
| None => todoItems->Dict.set(id, {...row, status: Completed, completedAt: now()})
|
|
513
|
+
}
|
|
514
|
+
} catch {
|
|
515
|
+
| exn =>
|
|
516
|
+
let errMsg =
|
|
517
|
+
exn->JsExn.fromException->Option.flatMap(JsExn.message)->Option.getOr("unknown")
|
|
518
|
+
EffectLogger.logError(
|
|
519
|
+
~comp=`OutboundTranslationSlice(${Spec.name})`,
|
|
520
|
+
~ids=?row.source->Option.map(EffectLogger.idsOfSource),
|
|
521
|
+
`failed to publish command: ${errMsg}`,
|
|
522
|
+
)->Effect.runSync
|
|
523
|
+
let updated = row->afterFailedAttempt(~lastError=errMsg)
|
|
524
|
+
todoItems->Dict.set(id, updated)
|
|
525
|
+
if updated.status == Abandoned {
|
|
526
|
+
newlyAbandoned->Array.push(id)
|
|
527
|
+
}
|
|
528
|
+
}
|
|
529
|
+
| None =>
|
|
530
|
+
let updated = row->afterFailedAttempt
|
|
476
531
|
todoItems->Dict.set(id, updated)
|
|
477
532
|
if updated.status == Abandoned {
|
|
478
533
|
newlyAbandoned->Array.push(id)
|
|
479
534
|
}
|
|
480
535
|
}
|
|
481
|
-
|
|
482
|
-
|
|
536
|
+
|
|
537
|
+
| Ok(None) =>
|
|
538
|
+
// Fire-and-forget: no command to publish
|
|
539
|
+
todoItems->Dict.set(id, {...row, status: Completed, completedAt: now()})
|
|
540
|
+
|
|
541
|
+
| Error(msg) =>
|
|
542
|
+
let updated = row->afterFailedAttempt(~lastError=msg)
|
|
483
543
|
todoItems->Dict.set(id, updated)
|
|
484
544
|
if updated.status == Abandoned {
|
|
485
545
|
newlyAbandoned->Array.push(id)
|
|
486
546
|
}
|
|
487
547
|
}
|
|
488
|
-
|
|
489
|
-
| Ok(None) =>
|
|
490
|
-
// Fire-and-forget: no command to publish
|
|
491
|
-
todoItems->Dict.set(id, {...row, status: Completed, completedAt: now()})
|
|
492
|
-
|
|
493
|
-
| Error(msg) =>
|
|
494
|
-
let updated = row->afterFailedAttempt(~lastError=msg)
|
|
495
|
-
todoItems->Dict.set(id, updated)
|
|
496
|
-
if updated.status == Abandoned {
|
|
497
|
-
newlyAbandoned->Array.push(id)
|
|
498
|
-
}
|
|
499
548
|
}
|
|
500
549
|
}
|
|
501
550
|
})
|