@reventlessdev/reventless-core 3.0.0-alpha.320 → 3.0.0-alpha.322
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/package.json +8 -8
- package/src/components/AutomationSlice/AutomationSlice_Builder.res +12 -2
- package/src/components/AutomationSlice/AutomationSlice_Builder.res.mjs +4 -4
- package/src/components/AutomationSlice/AutomationSlice_Callback.res +179 -53
- package/src/components/AutomationSlice/AutomationSlice_Callback.res.mjs +221 -102
- package/src/components/OutboundTranslationSlice/OutboundTranslationSlice_Builder.res +1 -1
- package/src/components/OutboundTranslationSlice/OutboundTranslationSlice_Builder.res.mjs +1 -1
- package/src/components/OutboundTranslationSlice/OutboundTranslationSlice_Callback.res +119 -23
- package/src/components/OutboundTranslationSlice/OutboundTranslationSlice_Callback.res.mjs +144 -52
- package/tests/pii/PiiMember.res +5 -0
- package/tests/pii/PiiMember.res.mjs +6 -0
- package/tests/pii/PiiPpxTest.res +8 -0
- package/tests/pii/PiiPpxTest.res.mjs +10 -0
|
@@ -3,13 +3,14 @@
|
|
|
3
3
|
import * as Sury from "sury";
|
|
4
4
|
import * as Stdlib_Dict from "@rescript/runtime/lib/es6/Stdlib_Dict.js";
|
|
5
5
|
import * as Stdlib_JSON from "@rescript/runtime/lib/es6/Stdlib_JSON.js";
|
|
6
|
-
import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
|
|
7
6
|
import * as Stdlib_JsExn from "@rescript/runtime/lib/es6/Stdlib_JsExn.js";
|
|
8
7
|
import * as Clock from "effect/Clock";
|
|
9
8
|
import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
|
|
9
|
+
import * as Stdlib_Result from "@rescript/runtime/lib/es6/Stdlib_Result.js";
|
|
10
10
|
import * as Effect from "effect/Effect";
|
|
11
|
+
import * as Pii$Reventless from "@reventlessdev/reventless-spec/src/components/Pii.res.mjs";
|
|
11
12
|
import * as Clock$Reventless from "@reventlessdev/reventless-spec/src/components/Clock.res.mjs";
|
|
12
|
-
import * as
|
|
13
|
+
import * as Primitive_object from "@rescript/runtime/lib/es6/Primitive_object.js";
|
|
13
14
|
import * as Message$Reventless from "@reventlessdev/reventless-spec/src/types/Message.res.mjs";
|
|
14
15
|
import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
|
|
15
16
|
import * as Util_Sury$Reventless from "@reventlessdev/reventless-spec/src/util/Util_Sury.res.mjs";
|
|
@@ -84,6 +85,98 @@ function Make(Spec) {
|
|
|
84
85
|
};
|
|
85
86
|
let readsClock = Automation.mappings.some(M => M.sourceName === Clock$Reventless.name);
|
|
86
87
|
let makeMeta = () => Message$ReventlessCore.generateMeta(Spec.targetName, undefined, undefined, undefined, undefined, undefined, undefined);
|
|
88
|
+
let personalFields = Pii$Reventless.recordFields(Spec.todoItemSchema);
|
|
89
|
+
let keyField = "$key";
|
|
90
|
+
let fingerprintPurpose = `automationWorkItem:` + Spec.name;
|
|
91
|
+
let warnedUnkeyed = {
|
|
92
|
+
contents: false
|
|
93
|
+
};
|
|
94
|
+
let storedKey = (key, capabilities) => {
|
|
95
|
+
if (Primitive_object.equal(personalFields, [])) {
|
|
96
|
+
return key;
|
|
97
|
+
}
|
|
98
|
+
let fingerprint = capabilities.secrets.fingerprint(fingerprintPurpose, key);
|
|
99
|
+
if (fingerprint.TAG === "Ok") {
|
|
100
|
+
return fingerprint._0;
|
|
101
|
+
}
|
|
102
|
+
let why = fingerprint._0._0;
|
|
103
|
+
if (Pii$Reventless.Provider.isRequired()) {
|
|
104
|
+
Effect.runSync(EffectLogger$ReventlessCore.logError(`AutomationSlice(` + Spec.name + `)`, undefined, `a work item was not stored: its row id cannot be fingerprinted (` + why + `)`));
|
|
105
|
+
return;
|
|
106
|
+
} else {
|
|
107
|
+
if (!warnedUnkeyed.contents) {
|
|
108
|
+
warnedUnkeyed.contents = true;
|
|
109
|
+
Effect.runSync(EffectLogger$ReventlessCore.logWarn(`AutomationSlice(` + Spec.name + `)`, undefined, `work items carrying personal data are keyed in the clear: no key to fingerprint them with (` + why + `)`));
|
|
110
|
+
}
|
|
111
|
+
return key;
|
|
112
|
+
}
|
|
113
|
+
};
|
|
114
|
+
let storedItem = async (key, item) => {
|
|
115
|
+
let json = Util_Sury$Reventless.toJson(item, Spec.todoItemSchema);
|
|
116
|
+
let match = personalFields[0];
|
|
117
|
+
if (match === undefined) {
|
|
118
|
+
return {
|
|
119
|
+
TAG: "Ok",
|
|
120
|
+
_0: json
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
if (typeof json !== "object" || json === null || Array.isArray(json)) {
|
|
124
|
+
return {
|
|
125
|
+
TAG: "Ok",
|
|
126
|
+
_0: json
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
let data = Object.assign({}, json);
|
|
130
|
+
data[keyField] = key;
|
|
131
|
+
return Stdlib_Result.mapError(Stdlib_Result.map(await Pii$Reventless.sealFields(personalFields.concat([[
|
|
132
|
+
keyField,
|
|
133
|
+
match[1]
|
|
134
|
+
]]), Spec.name, undefined, true, data), d => (d)), Pii$Reventless.errorMessage);
|
|
135
|
+
};
|
|
136
|
+
let openRow = async (id, row) => {
|
|
137
|
+
let decode = (json, key) => {
|
|
138
|
+
let item;
|
|
139
|
+
try {
|
|
140
|
+
item = Util_Sury$Reventless.fromJson(json, Spec.todoItemSchema);
|
|
141
|
+
} catch (raw_exn) {
|
|
142
|
+
let exn = Primitive_exceptions.internalToException(raw_exn);
|
|
143
|
+
return {
|
|
144
|
+
TAG: "Unopened",
|
|
145
|
+
_0: Stdlib_Option.getOr(Stdlib_Option.flatMap(Stdlib_JsExn.fromException(exn), Stdlib_JsExn.message), "unknown")
|
|
146
|
+
};
|
|
147
|
+
}
|
|
148
|
+
return {
|
|
149
|
+
TAG: "Open",
|
|
150
|
+
key: key,
|
|
151
|
+
item: item
|
|
152
|
+
};
|
|
153
|
+
};
|
|
154
|
+
let data = row.item;
|
|
155
|
+
if (typeof data !== "object" || data === null || Array.isArray(data)) {
|
|
156
|
+
return decode(data, id);
|
|
157
|
+
}
|
|
158
|
+
if (!Primitive_object.notequal(personalFields, [])) {
|
|
159
|
+
return decode(data, id);
|
|
160
|
+
}
|
|
161
|
+
let e = await Pii$Reventless.openFields(personalFields.map(param => param[0]).concat([keyField]), data);
|
|
162
|
+
if (e.TAG !== "Ok") {
|
|
163
|
+
return {
|
|
164
|
+
TAG: "Unopened",
|
|
165
|
+
_0: Pii$Reventless.errorMessage(e._0)
|
|
166
|
+
};
|
|
167
|
+
}
|
|
168
|
+
let data$1 = e._0;
|
|
169
|
+
let field = Pii$Reventless.goneField(Spec.todoItemSchema, data$1);
|
|
170
|
+
if (field !== undefined) {
|
|
171
|
+
return {
|
|
172
|
+
TAG: "Gone",
|
|
173
|
+
_0: field
|
|
174
|
+
};
|
|
175
|
+
}
|
|
176
|
+
let key = Stdlib_Option.getOr(Stdlib_Option.flatMap(data$1[keyField], Stdlib_JSON.Decode.string), id);
|
|
177
|
+
Stdlib_Dict.$$delete(data$1, keyField);
|
|
178
|
+
return decode(data$1, key);
|
|
179
|
+
};
|
|
87
180
|
let processingTimeoutMs = 15 * 60 * 1000;
|
|
88
181
|
let afterFailedAttempt = row => {
|
|
89
182
|
let retryCount = row.retryCount + 1 | 0;
|
|
@@ -92,6 +185,31 @@ function Make(Spec) {
|
|
|
92
185
|
newrecord.status = retryCount >= Spec.maxRetries ? "Abandoned" : "Failed";
|
|
93
186
|
return newrecord;
|
|
94
187
|
};
|
|
188
|
+
let abandonmentCommand = async (id, row) => {
|
|
189
|
+
let match = await openRow(id, row);
|
|
190
|
+
switch (match.TAG) {
|
|
191
|
+
case "Open" :
|
|
192
|
+
return Stdlib_Option.flatMap(Automation.onExhausted(match.key, match.item), param => {
|
|
193
|
+
let commandJson;
|
|
194
|
+
try {
|
|
195
|
+
commandJson = Util_Sury$Reventless.toJson(param[1], Spec.commandSchema);
|
|
196
|
+
} catch (exn) {
|
|
197
|
+
Effect.runSync(EffectLogger$ReventlessCore.logError(`AutomationSlice(` + Spec.name + `)`, undefined, `retries exhausted for ` + id + `, and the abandonment command could not be encoded`));
|
|
198
|
+
return;
|
|
199
|
+
}
|
|
200
|
+
return {
|
|
201
|
+
id: param[0],
|
|
202
|
+
meta: makeMeta(),
|
|
203
|
+
commandJson: commandJson
|
|
204
|
+
};
|
|
205
|
+
});
|
|
206
|
+
case "Gone" :
|
|
207
|
+
return;
|
|
208
|
+
case "Unopened" :
|
|
209
|
+
Effect.runSync(EffectLogger$ReventlessCore.logError(`AutomationSlice(` + Spec.name + `)`, undefined, `retries exhausted for ` + id + `, and its item could not be decoded to announce it`));
|
|
210
|
+
return;
|
|
211
|
+
}
|
|
212
|
+
};
|
|
95
213
|
let Route = AutomationSlice_Route$ReventlessCore.Make(Spec)(Automation);
|
|
96
214
|
let recordedAtOf = time => {
|
|
97
215
|
let millis = new Date(time).getTime();
|
|
@@ -101,26 +219,32 @@ function Make(Spec) {
|
|
|
101
219
|
return millis;
|
|
102
220
|
}
|
|
103
221
|
};
|
|
104
|
-
let apply = param => {
|
|
105
|
-
param.collected
|
|
106
|
-
|
|
107
|
-
let match =
|
|
108
|
-
|
|
109
|
-
|
|
222
|
+
let apply = async (param, capabilities) => {
|
|
223
|
+
let collected = param.collected;
|
|
224
|
+
for (let i = 0, i_finish = collected.length; i < i_finish; ++i) {
|
|
225
|
+
let match = collected[i];
|
|
226
|
+
let key = match[0];
|
|
227
|
+
let id = storedKey(key, capabilities);
|
|
228
|
+
if (id !== undefined && !Stdlib_Option.isSome(todoItems[id])) {
|
|
229
|
+
let why = await storedItem(key, match[1]);
|
|
230
|
+
if (why.TAG === "Ok") {
|
|
231
|
+
let row_item = why._0;
|
|
232
|
+
let row_createdAt = new Date().toISOString();
|
|
233
|
+
let row_maxRetries = Spec.maxRetries;
|
|
234
|
+
let row = {
|
|
235
|
+
item: row_item,
|
|
236
|
+
status: "Pending",
|
|
237
|
+
createdAt: row_createdAt,
|
|
238
|
+
retryCount: 0,
|
|
239
|
+
maxRetries: row_maxRetries
|
|
240
|
+
};
|
|
241
|
+
todoItems[id] = row;
|
|
242
|
+
} else {
|
|
243
|
+
Effect.runSync(EffectLogger$ReventlessCore.logError(`AutomationSlice(` + Spec.name + `)`, undefined, `a work item was not stored: ` + why._0));
|
|
244
|
+
}
|
|
110
245
|
}
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
let row_maxRetries = Spec.maxRetries;
|
|
114
|
-
let row = {
|
|
115
|
-
item: row_item,
|
|
116
|
-
status: "Pending",
|
|
117
|
-
createdAt: row_createdAt,
|
|
118
|
-
retryCount: 0,
|
|
119
|
-
maxRetries: row_maxRetries
|
|
120
|
-
};
|
|
121
|
-
todoItems[id] = row;
|
|
122
|
-
});
|
|
123
|
-
Stdlib_Option.forEach(param.resolved, id => Stdlib_Option.forEach(todoItems[id], row => {
|
|
246
|
+
}
|
|
247
|
+
return Stdlib_Option.forEach(Stdlib_Option.flatMap(param.resolved, key => storedKey(key, capabilities)), id => Stdlib_Option.forEach(todoItems[id], row => {
|
|
124
248
|
let newrecord = {...row};
|
|
125
249
|
newrecord.completedAt = new Date().toISOString();
|
|
126
250
|
newrecord.status = "Completed";
|
|
@@ -143,8 +267,9 @@ function Make(Spec) {
|
|
|
143
267
|
});
|
|
144
268
|
});
|
|
145
269
|
};
|
|
146
|
-
let phase1 = (events, sliceCtx) => {
|
|
147
|
-
events.
|
|
270
|
+
let phase1 = async (events, sliceCtx, capabilities) => {
|
|
271
|
+
for (let i = 0, i_finish = events.length; i < i_finish; ++i) {
|
|
272
|
+
let json = events[i];
|
|
148
273
|
let context = Message$ReventlessCore.decode(json, Message$Reventless.contextSchema);
|
|
149
274
|
let recordedAt = recordedAtOf(context.meta.time);
|
|
150
275
|
let ctx;
|
|
@@ -159,12 +284,14 @@ function Make(Spec) {
|
|
|
159
284
|
let dict = Stdlib_JSON.Decode.object(json);
|
|
160
285
|
let eventPayload = dict !== undefined ? Stdlib_Option.getOr(dict["event"], json) : json;
|
|
161
286
|
let routed = Route.route(eventPayload, sourceName, context.id, ctx);
|
|
162
|
-
routed.
|
|
287
|
+
for (let j = 0, j_finish = routed.length; j < j_finish; ++j) {
|
|
288
|
+
await apply(routed[j], capabilities);
|
|
289
|
+
}
|
|
163
290
|
reschedule(routed, context.meta.msgId, Stdlib_Option.getOr(ctx.recordedAt, Date.now()));
|
|
164
291
|
if (sourceName === Clock$Reventless.name) {
|
|
165
|
-
|
|
292
|
+
AutomationSlice_Schedules$ReventlessCore.fired(schedules, context.id);
|
|
166
293
|
}
|
|
167
|
-
}
|
|
294
|
+
}
|
|
168
295
|
};
|
|
169
296
|
let rebuildSchedules = events => {
|
|
170
297
|
Object.keys(schedules).forEach(key => Stdlib_Dict.$$delete(schedules, key));
|
|
@@ -229,60 +356,64 @@ function Make(Spec) {
|
|
|
229
356
|
}
|
|
230
357
|
});
|
|
231
358
|
let commands = [];
|
|
232
|
-
pending.
|
|
233
|
-
let
|
|
234
|
-
let
|
|
235
|
-
let
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
359
|
+
for (let i = 0, i_finish = pending.length; i < i_finish; ++i) {
|
|
360
|
+
let match = pending[i];
|
|
361
|
+
let row = match[1];
|
|
362
|
+
let id = match[0];
|
|
363
|
+
let field = await openRow(id, row);
|
|
364
|
+
switch (field.TAG) {
|
|
365
|
+
case "Open" :
|
|
366
|
+
let match$1 = Automation.process(field.key, field.item, currentTime);
|
|
367
|
+
if (match$1 !== undefined) {
|
|
368
|
+
let targetId = match$1[0];
|
|
369
|
+
let newrecord = {...row};
|
|
370
|
+
newrecord.processedAt = new Date().toISOString();
|
|
371
|
+
newrecord.status = "Processing";
|
|
372
|
+
todoItems[id] = newrecord;
|
|
373
|
+
let commandJson;
|
|
374
|
+
try {
|
|
375
|
+
commandJson = Util_Sury$Reventless.toJson(match$1[1], Spec.commandSchema);
|
|
376
|
+
} catch (raw_exn) {
|
|
377
|
+
let exn = Primitive_exceptions.internalToException(raw_exn);
|
|
378
|
+
let errMsg = Stdlib_Option.getOr(Stdlib_Option.flatMap(Stdlib_JsExn.fromException(exn), Stdlib_JsExn.message), "unknown");
|
|
379
|
+
Effect.runSync(EffectLogger$ReventlessCore.logError(`AutomationSlice(` + Spec.name + `)`, undefined, `failed to encode command: ` + errMsg));
|
|
380
|
+
let updated = afterFailedAttempt(row);
|
|
381
|
+
todoItems[id] = updated;
|
|
382
|
+
if (updated.status === "Abandoned") {
|
|
383
|
+
newlyAbandoned.push(id);
|
|
384
|
+
}
|
|
385
|
+
commandJson = undefined;
|
|
386
|
+
}
|
|
387
|
+
Stdlib_Option.forEach(commandJson, commandJson => {
|
|
388
|
+
let msg_meta = makeMeta();
|
|
389
|
+
let msg = {
|
|
390
|
+
id: targetId,
|
|
391
|
+
meta: msg_meta,
|
|
392
|
+
commandJson: commandJson
|
|
393
|
+
};
|
|
394
|
+
commands.push(msg);
|
|
395
|
+
});
|
|
265
396
|
}
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
let
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
}
|
|
278
|
-
}
|
|
397
|
+
break;
|
|
398
|
+
case "Gone" :
|
|
399
|
+
Effect.runSync(EffectLogger$ReventlessCore.logInfo(`AutomationSlice(` + Spec.name + `)`, undefined, id + `: its ` + field._0 + ` was erased; nothing to do`));
|
|
400
|
+
let newrecord$1 = {...row};
|
|
401
|
+
newrecord$1.completedAt = new Date().toISOString();
|
|
402
|
+
newrecord$1.status = "Completed";
|
|
403
|
+
todoItems[id] = newrecord$1;
|
|
404
|
+
break;
|
|
405
|
+
case "Unopened" :
|
|
406
|
+
Effect.runSync(EffectLogger$ReventlessCore.logError(`AutomationSlice(` + Spec.name + `)`, undefined, `failed to decode todoItem: ` + field._0));
|
|
407
|
+
break;
|
|
408
|
+
}
|
|
409
|
+
}
|
|
279
410
|
if (commands.length !== 0) {
|
|
280
411
|
try {
|
|
281
412
|
await publishJsons(commands);
|
|
282
|
-
} catch (raw_exn) {
|
|
283
|
-
let exn = Primitive_exceptions.internalToException(raw_exn);
|
|
284
|
-
let errMsg = Stdlib_Option.getOr(Stdlib_Option.flatMap(Stdlib_JsExn.fromException(exn), Stdlib_JsExn.message), "unknown");
|
|
285
|
-
Effect.runSync(EffectLogger$ReventlessCore.logError(`AutomationSlice(` + Spec.name + `)`, undefined, `failed to publish commands: ` + errMsg));
|
|
413
|
+
} catch (raw_exn$1) {
|
|
414
|
+
let exn$1 = Primitive_exceptions.internalToException(raw_exn$1);
|
|
415
|
+
let errMsg$1 = Stdlib_Option.getOr(Stdlib_Option.flatMap(Stdlib_JsExn.fromException(exn$1), Stdlib_JsExn.message), "unknown");
|
|
416
|
+
Effect.runSync(EffectLogger$ReventlessCore.logError(`AutomationSlice(` + Spec.name + `)`, undefined, `failed to publish commands: ` + errMsg$1));
|
|
286
417
|
pending.forEach(param => {
|
|
287
418
|
let id = param[0];
|
|
288
419
|
let current = todoItems[id];
|
|
@@ -301,38 +432,26 @@ function Make(Spec) {
|
|
|
301
432
|
});
|
|
302
433
|
}
|
|
303
434
|
}
|
|
304
|
-
let abandonmentCommands =
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
return Stdlib_Option.flatMap(Automation.onExhausted(id, item), param => {
|
|
313
|
-
let commandJson;
|
|
314
|
-
try {
|
|
315
|
-
commandJson = Util_Sury$Reventless.toJson(param[1], Spec.commandSchema);
|
|
316
|
-
} catch (exn) {
|
|
317
|
-
Effect.runSync(EffectLogger$ReventlessCore.logError(`AutomationSlice(` + Spec.name + `)`, undefined, `retries exhausted for ` + id + `, and the abandonment command could not be encoded`));
|
|
318
|
-
return;
|
|
435
|
+
let abandonmentCommands = [];
|
|
436
|
+
for (let i$1 = 0, i_finish$1 = newlyAbandoned.length; i$1 < i_finish$1; ++i$1) {
|
|
437
|
+
let id$1 = newlyAbandoned[i$1];
|
|
438
|
+
let row$1 = todoItems[id$1];
|
|
439
|
+
if (row$1 !== undefined) {
|
|
440
|
+
let command = await abandonmentCommand(id$1, row$1);
|
|
441
|
+
if (command !== undefined) {
|
|
442
|
+
abandonmentCommands.push(command);
|
|
319
443
|
}
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
meta: makeMeta(),
|
|
323
|
-
commandJson: commandJson
|
|
324
|
-
};
|
|
325
|
-
});
|
|
326
|
-
}));
|
|
444
|
+
}
|
|
445
|
+
}
|
|
327
446
|
if (abandonmentCommands.length === 0) {
|
|
328
447
|
return;
|
|
329
448
|
}
|
|
330
449
|
try {
|
|
331
450
|
return await publishJsons(abandonmentCommands);
|
|
332
|
-
} catch (raw_exn$
|
|
333
|
-
let exn$
|
|
334
|
-
let errMsg$
|
|
335
|
-
return Effect.runSync(EffectLogger$ReventlessCore.logError(`AutomationSlice(` + Spec.name + `)`, undefined, `failed to publish abandonment commands: ` + errMsg$
|
|
451
|
+
} catch (raw_exn$2) {
|
|
452
|
+
let exn$2 = Primitive_exceptions.internalToException(raw_exn$2);
|
|
453
|
+
let errMsg$2 = Stdlib_Option.getOr(Stdlib_Option.flatMap(Stdlib_JsExn.fromException(exn$2), Stdlib_JsExn.message), "unknown");
|
|
454
|
+
return Effect.runSync(EffectLogger$ReventlessCore.logError(`AutomationSlice(` + Spec.name + `)`, undefined, `failed to publish abandonment commands: ` + errMsg$2));
|
|
336
455
|
}
|
|
337
456
|
};
|
|
338
457
|
return {
|
|
@@ -189,7 +189,7 @@ module Make = (
|
|
|
189
189
|
Effect.promise(
|
|
190
190
|
async () => {
|
|
191
191
|
// Phase 1: collect outbound items
|
|
192
|
-
Callback.phase1(eventsArr)
|
|
192
|
+
await Callback.phase1(eventsArr, ~capabilities=Capabilities.capabilities())
|
|
193
193
|
// Sync the post-phase-1 TODO state so consumers awaiting the
|
|
194
194
|
// originating publishEvent observe Pending rows immediately.
|
|
195
195
|
await syncToQueryDb(queryDbOps)
|
|
@@ -108,7 +108,7 @@ function Make(RuntimeEnvironment) {
|
|
|
108
108
|
return [];
|
|
109
109
|
}
|
|
110
110
|
})), events => Stream$1.fromIterable(events))), eventsArr => Effect.promise(async () => {
|
|
111
|
-
Callback.phase1(eventsArr);
|
|
111
|
+
await Callback.phase1(eventsArr, Capabilities.capabilities());
|
|
112
112
|
await syncToQueryDb(queryDbOps);
|
|
113
113
|
Stdlib_Promise.$$catch(Callback.phase2(publishJsonsFn, Capabilities.capabilities()).then(() => syncToQueryDb(queryDbOps)), exn => {
|
|
114
114
|
let errMsg = Stdlib_Option.getOr(Stdlib_Option.flatMap(Stdlib_JsExn.fromException(exn), Stdlib_JsExn.message), "unknown");
|
|
@@ -79,7 +79,10 @@ the AWS entry point kept passing bare events and nothing complained, because its
|
|
|
79
79
|
hand-written copy was internally consistent. The mismatch surfaced only at
|
|
80
80
|
runtime, as `undefined` destructured from a tuple that was never a tuple.
|
|
81
81
|
*/
|
|
82
|
-
type phase1<'event> =
|
|
82
|
+
type phase1<'event> = (
|
|
83
|
+
array<(string, 'event)>,
|
|
84
|
+
~capabilities: Reventless.Capabilities.t,
|
|
85
|
+
) => promise<unit>
|
|
83
86
|
|
|
84
87
|
/** Phase 2: translate all pending items, optionally publishing commands.
|
|
85
88
|
|
|
@@ -109,7 +112,8 @@ module type T = {
|
|
|
109
112
|
|
|
110
113
|
/** Phase 1: update TODO list from a batch of `(sourceId, event)` pairs
|
|
111
114
|
(collect only, no resolve). `sourceId` is the envelope id of the entity the
|
|
112
|
-
event was published for.
|
|
115
|
+
event was published for. `~capabilities` supplies the key a personal item's
|
|
116
|
+
row id is fingerprinted with. */
|
|
113
117
|
let phase1: phase1<Spec.consumedEvent>
|
|
114
118
|
|
|
115
119
|
/** Phase 2: translate all pending items, optionally publishing commands via publishJsons. */
|
|
@@ -201,25 +205,82 @@ module Make = (
|
|
|
201
205
|
)->Effect.runSync
|
|
202
206
|
}
|
|
203
207
|
|
|
204
|
-
|
|
205
|
-
|
|
208
|
+
// A work item that carries personal data is stored the way an event is: its
|
|
209
|
+
// marked fields sealed, and its row keyed by a fingerprint of the key `collect`
|
|
210
|
+
// chose, which may hold the address the work is about. A table key can be
|
|
211
|
+
// neither encrypted nor erased.
|
|
212
|
+
let personalFields = Reventless.Pii.recordFields(Spec.outboundItemSchema)
|
|
213
|
+
let fingerprintPurpose = `outboundWorkItem:${Spec.name}`
|
|
214
|
+
let warnedUnkeyed = ref(false)
|
|
215
|
+
|
|
216
|
+
/** The id a row is stored under, or `None` when it cannot be stored safely. */
|
|
217
|
+
let storedKey = (key, ~capabilities: Reventless.Capabilities.t): option<string> =>
|
|
218
|
+
if personalFields == [] {
|
|
219
|
+
Some(key)
|
|
220
|
+
} else {
|
|
221
|
+
switch capabilities.secrets.fingerprint(~purpose=fingerprintPurpose, key) {
|
|
222
|
+
| Ok(fingerprint) => Some(fingerprint)
|
|
223
|
+
| Error(Unavailable(why)) =>
|
|
224
|
+
if Reventless.Pii.Provider.isRequired() {
|
|
225
|
+
EffectLogger.logError(
|
|
226
|
+
~comp=`OutboundTranslationSlice(${Spec.name})`,
|
|
227
|
+
`a work item was not stored: its row id cannot be fingerprinted (${why})`,
|
|
228
|
+
)->Effect.runSync
|
|
229
|
+
None
|
|
230
|
+
} else {
|
|
231
|
+
if !warnedUnkeyed.contents {
|
|
232
|
+
warnedUnkeyed := true
|
|
233
|
+
EffectLogger.logWarn(
|
|
234
|
+
~comp=`OutboundTranslationSlice(${Spec.name})`,
|
|
235
|
+
`work items carrying personal data are keyed in the clear: no key to fingerprint them with (${why})`,
|
|
236
|
+
)->Effect.runSync
|
|
237
|
+
}
|
|
238
|
+
Some(key)
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
let phase1 = async (
|
|
244
|
+
events: array<(string, Spec.consumedEvent)>,
|
|
245
|
+
~capabilities: Reventless.Capabilities.t,
|
|
246
|
+
) =>
|
|
247
|
+
for i in 0 to events->Array.length - 1 {
|
|
248
|
+
let (sourceId, event) = events->Array.getUnsafe(i)
|
|
206
249
|
// Collect new outbound items
|
|
207
|
-
Translation.collect(event, ~sourceId)
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
250
|
+
let collected = Translation.collect(event, ~sourceId)
|
|
251
|
+
for j in 0 to collected->Array.length - 1 {
|
|
252
|
+
let (key, item) = collected->Array.getUnsafe(j)
|
|
253
|
+
switch storedKey(key, ~capabilities) {
|
|
254
|
+
| None => ()
|
|
255
|
+
| Some(id) if todoItems->Dict.get(id)->Option.isSome => () // idempotent
|
|
256
|
+
| Some(id) =>
|
|
257
|
+
let json = item->Reventless.Util_Sury.toJson(Spec.outboundItemSchema)
|
|
258
|
+
let stored = switch json {
|
|
259
|
+
| Object(data) if personalFields != [] =>
|
|
260
|
+
(
|
|
261
|
+
await Reventless.Pii.sealRecord(Spec.outboundItemSchema, data)
|
|
262
|
+
)->Result.map(d => JSON.Object(d))
|
|
263
|
+
| other => Ok(other)
|
|
264
|
+
}
|
|
265
|
+
switch stored {
|
|
266
|
+
| Error(e) =>
|
|
267
|
+
EffectLogger.logError(
|
|
268
|
+
~comp=`OutboundTranslationSlice(${Spec.name})`,
|
|
269
|
+
`a work item was not stored: ${e->Reventless.Pii.errorMessage}`,
|
|
270
|
+
)->Effect.runSync
|
|
271
|
+
| Ok(stored) =>
|
|
272
|
+
let row: todoRow = {
|
|
273
|
+
item: stored,
|
|
274
|
+
status: Pending,
|
|
275
|
+
createdAt: now(),
|
|
276
|
+
retryCount: 0,
|
|
277
|
+
maxRetries: Spec.maxRetries,
|
|
278
|
+
}
|
|
279
|
+
todoItems->Dict.set(id, row)
|
|
217
280
|
}
|
|
218
|
-
todoItems->Dict.set(id, row)
|
|
219
281
|
}
|
|
220
|
-
}
|
|
221
|
-
}
|
|
222
|
-
}
|
|
282
|
+
}
|
|
283
|
+
}
|
|
223
284
|
|
|
224
285
|
let phase2 = async (
|
|
225
286
|
publishJsons: ReventlessInfra.CommandTopic.publishJsons,
|
|
@@ -253,15 +314,50 @@ module Make = (
|
|
|
253
314
|
let _ = await pending->Array.reduce(Promise.resolve(), async (prev, (id, row)) => {
|
|
254
315
|
let _ = await prev
|
|
255
316
|
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
317
|
+
// Opened first: a personal field is sealed in the row, and one whose
|
|
318
|
+
// subject was erased since leaves nothing to send.
|
|
319
|
+
let opened = switch row.item {
|
|
320
|
+
| Object(data) if personalFields != [] =>
|
|
321
|
+
switch await Reventless.Pii.openRecord(Spec.outboundItemSchema, data) {
|
|
322
|
+
| Ok(data) =>
|
|
323
|
+
switch Reventless.Pii.goneField(Spec.outboundItemSchema, data) {
|
|
324
|
+
| Some(field) => Error(#Gone(field))
|
|
325
|
+
| None => Ok(JSON.Object(data))
|
|
326
|
+
}
|
|
327
|
+
| Error(e) => Error(#Unopened(e->Reventless.Pii.errorMessage))
|
|
328
|
+
}
|
|
329
|
+
| other => Ok(other)
|
|
330
|
+
}
|
|
331
|
+
let item = switch opened {
|
|
332
|
+
| Error(#Gone(field)) =>
|
|
333
|
+
EffectLogger.logInfo(
|
|
334
|
+
~comp=`OutboundTranslationSlice(${Spec.name})`,
|
|
335
|
+
`${id}: its ${field} was erased; nothing to do`,
|
|
336
|
+
)->Effect.runSync
|
|
337
|
+
todoItems->Dict.set(id, {...row, status: Completed, completedAt: now()})
|
|
338
|
+
None
|
|
339
|
+
| Error(#Unopened(why)) =>
|
|
260
340
|
EffectLogger.logError(
|
|
261
341
|
~comp=`OutboundTranslationSlice(${Spec.name})`,
|
|
262
|
-
|
|
342
|
+
`${id}: its work item does not open: ${why}`,
|
|
263
343
|
)->Effect.runSync
|
|
344
|
+
let updated = row->afterFailedAttempt(~lastError=why)
|
|
345
|
+
todoItems->Dict.set(id, updated)
|
|
346
|
+
if updated.status == Abandoned {
|
|
347
|
+
newlyAbandoned->Array.push(id)
|
|
348
|
+
}
|
|
264
349
|
None
|
|
350
|
+
| Ok(itemJson) =>
|
|
351
|
+
try itemJson->Reventless.Util_Sury.fromJson(Spec.outboundItemSchema)->Some catch {
|
|
352
|
+
| exn =>
|
|
353
|
+
let errMsg =
|
|
354
|
+
exn->JsExn.fromException->Option.flatMap(JsExn.message)->Option.getOr("unknown")
|
|
355
|
+
EffectLogger.logError(
|
|
356
|
+
~comp=`OutboundTranslationSlice(${Spec.name})`,
|
|
357
|
+
`failed to decode outboundItem: ${errMsg}`,
|
|
358
|
+
)->Effect.runSync
|
|
359
|
+
None
|
|
360
|
+
}
|
|
265
361
|
}
|
|
266
362
|
|
|
267
363
|
switch item {
|