@reventlessdev/reventless-core 3.0.0-alpha.320 → 3.0.0-alpha.322

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,13 +3,14 @@
3
3
  import * as Sury from "sury";
4
4
  import * as Stdlib_Dict from "@rescript/runtime/lib/es6/Stdlib_Dict.js";
5
5
  import * as Stdlib_JSON from "@rescript/runtime/lib/es6/Stdlib_JSON.js";
6
- import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
7
6
  import * as Stdlib_JsExn from "@rescript/runtime/lib/es6/Stdlib_JsExn.js";
8
7
  import * as Clock from "effect/Clock";
9
8
  import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
9
+ import * as Stdlib_Result from "@rescript/runtime/lib/es6/Stdlib_Result.js";
10
10
  import * as Effect from "effect/Effect";
11
+ import * as Pii$Reventless from "@reventlessdev/reventless-spec/src/components/Pii.res.mjs";
11
12
  import * as Clock$Reventless from "@reventlessdev/reventless-spec/src/components/Clock.res.mjs";
12
- import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
13
+ import * as Primitive_object from "@rescript/runtime/lib/es6/Primitive_object.js";
13
14
  import * as Message$Reventless from "@reventlessdev/reventless-spec/src/types/Message.res.mjs";
14
15
  import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
15
16
  import * as Util_Sury$Reventless from "@reventlessdev/reventless-spec/src/util/Util_Sury.res.mjs";
@@ -84,6 +85,98 @@ function Make(Spec) {
84
85
  };
85
86
  let readsClock = Automation.mappings.some(M => M.sourceName === Clock$Reventless.name);
86
87
  let makeMeta = () => Message$ReventlessCore.generateMeta(Spec.targetName, undefined, undefined, undefined, undefined, undefined, undefined);
88
+ let personalFields = Pii$Reventless.recordFields(Spec.todoItemSchema);
89
+ let keyField = "$key";
90
+ let fingerprintPurpose = `automationWorkItem:` + Spec.name;
91
+ let warnedUnkeyed = {
92
+ contents: false
93
+ };
94
+ let storedKey = (key, capabilities) => {
95
+ if (Primitive_object.equal(personalFields, [])) {
96
+ return key;
97
+ }
98
+ let fingerprint = capabilities.secrets.fingerprint(fingerprintPurpose, key);
99
+ if (fingerprint.TAG === "Ok") {
100
+ return fingerprint._0;
101
+ }
102
+ let why = fingerprint._0._0;
103
+ if (Pii$Reventless.Provider.isRequired()) {
104
+ Effect.runSync(EffectLogger$ReventlessCore.logError(`AutomationSlice(` + Spec.name + `)`, undefined, `a work item was not stored: its row id cannot be fingerprinted (` + why + `)`));
105
+ return;
106
+ } else {
107
+ if (!warnedUnkeyed.contents) {
108
+ warnedUnkeyed.contents = true;
109
+ Effect.runSync(EffectLogger$ReventlessCore.logWarn(`AutomationSlice(` + Spec.name + `)`, undefined, `work items carrying personal data are keyed in the clear: no key to fingerprint them with (` + why + `)`));
110
+ }
111
+ return key;
112
+ }
113
+ };
114
+ let storedItem = async (key, item) => {
115
+ let json = Util_Sury$Reventless.toJson(item, Spec.todoItemSchema);
116
+ let match = personalFields[0];
117
+ if (match === undefined) {
118
+ return {
119
+ TAG: "Ok",
120
+ _0: json
121
+ };
122
+ }
123
+ if (typeof json !== "object" || json === null || Array.isArray(json)) {
124
+ return {
125
+ TAG: "Ok",
126
+ _0: json
127
+ };
128
+ }
129
+ let data = Object.assign({}, json);
130
+ data[keyField] = key;
131
+ return Stdlib_Result.mapError(Stdlib_Result.map(await Pii$Reventless.sealFields(personalFields.concat([[
132
+ keyField,
133
+ match[1]
134
+ ]]), Spec.name, undefined, true, data), d => (d)), Pii$Reventless.errorMessage);
135
+ };
136
+ let openRow = async (id, row) => {
137
+ let decode = (json, key) => {
138
+ let item;
139
+ try {
140
+ item = Util_Sury$Reventless.fromJson(json, Spec.todoItemSchema);
141
+ } catch (raw_exn) {
142
+ let exn = Primitive_exceptions.internalToException(raw_exn);
143
+ return {
144
+ TAG: "Unopened",
145
+ _0: Stdlib_Option.getOr(Stdlib_Option.flatMap(Stdlib_JsExn.fromException(exn), Stdlib_JsExn.message), "unknown")
146
+ };
147
+ }
148
+ return {
149
+ TAG: "Open",
150
+ key: key,
151
+ item: item
152
+ };
153
+ };
154
+ let data = row.item;
155
+ if (typeof data !== "object" || data === null || Array.isArray(data)) {
156
+ return decode(data, id);
157
+ }
158
+ if (!Primitive_object.notequal(personalFields, [])) {
159
+ return decode(data, id);
160
+ }
161
+ let e = await Pii$Reventless.openFields(personalFields.map(param => param[0]).concat([keyField]), data);
162
+ if (e.TAG !== "Ok") {
163
+ return {
164
+ TAG: "Unopened",
165
+ _0: Pii$Reventless.errorMessage(e._0)
166
+ };
167
+ }
168
+ let data$1 = e._0;
169
+ let field = Pii$Reventless.goneField(Spec.todoItemSchema, data$1);
170
+ if (field !== undefined) {
171
+ return {
172
+ TAG: "Gone",
173
+ _0: field
174
+ };
175
+ }
176
+ let key = Stdlib_Option.getOr(Stdlib_Option.flatMap(data$1[keyField], Stdlib_JSON.Decode.string), id);
177
+ Stdlib_Dict.$$delete(data$1, keyField);
178
+ return decode(data$1, key);
179
+ };
87
180
  let processingTimeoutMs = 15 * 60 * 1000;
88
181
  let afterFailedAttempt = row => {
89
182
  let retryCount = row.retryCount + 1 | 0;
@@ -92,6 +185,31 @@ function Make(Spec) {
92
185
  newrecord.status = retryCount >= Spec.maxRetries ? "Abandoned" : "Failed";
93
186
  return newrecord;
94
187
  };
188
+ let abandonmentCommand = async (id, row) => {
189
+ let match = await openRow(id, row);
190
+ switch (match.TAG) {
191
+ case "Open" :
192
+ return Stdlib_Option.flatMap(Automation.onExhausted(match.key, match.item), param => {
193
+ let commandJson;
194
+ try {
195
+ commandJson = Util_Sury$Reventless.toJson(param[1], Spec.commandSchema);
196
+ } catch (exn) {
197
+ Effect.runSync(EffectLogger$ReventlessCore.logError(`AutomationSlice(` + Spec.name + `)`, undefined, `retries exhausted for ` + id + `, and the abandonment command could not be encoded`));
198
+ return;
199
+ }
200
+ return {
201
+ id: param[0],
202
+ meta: makeMeta(),
203
+ commandJson: commandJson
204
+ };
205
+ });
206
+ case "Gone" :
207
+ return;
208
+ case "Unopened" :
209
+ Effect.runSync(EffectLogger$ReventlessCore.logError(`AutomationSlice(` + Spec.name + `)`, undefined, `retries exhausted for ` + id + `, and its item could not be decoded to announce it`));
210
+ return;
211
+ }
212
+ };
95
213
  let Route = AutomationSlice_Route$ReventlessCore.Make(Spec)(Automation);
96
214
  let recordedAtOf = time => {
97
215
  let millis = new Date(time).getTime();
@@ -101,26 +219,32 @@ function Make(Spec) {
101
219
  return millis;
102
220
  }
103
221
  };
104
- let apply = param => {
105
- param.collected.forEach(param => {
106
- let id = param[0];
107
- let match = todoItems[id];
108
- if (match !== undefined) {
109
- return;
222
+ let apply = async (param, capabilities) => {
223
+ let collected = param.collected;
224
+ for (let i = 0, i_finish = collected.length; i < i_finish; ++i) {
225
+ let match = collected[i];
226
+ let key = match[0];
227
+ let id = storedKey(key, capabilities);
228
+ if (id !== undefined && !Stdlib_Option.isSome(todoItems[id])) {
229
+ let why = await storedItem(key, match[1]);
230
+ if (why.TAG === "Ok") {
231
+ let row_item = why._0;
232
+ let row_createdAt = new Date().toISOString();
233
+ let row_maxRetries = Spec.maxRetries;
234
+ let row = {
235
+ item: row_item,
236
+ status: "Pending",
237
+ createdAt: row_createdAt,
238
+ retryCount: 0,
239
+ maxRetries: row_maxRetries
240
+ };
241
+ todoItems[id] = row;
242
+ } else {
243
+ Effect.runSync(EffectLogger$ReventlessCore.logError(`AutomationSlice(` + Spec.name + `)`, undefined, `a work item was not stored: ` + why._0));
244
+ }
110
245
  }
111
- let row_item = Util_Sury$Reventless.toJson(param[1], Spec.todoItemSchema);
112
- let row_createdAt = new Date().toISOString();
113
- let row_maxRetries = Spec.maxRetries;
114
- let row = {
115
- item: row_item,
116
- status: "Pending",
117
- createdAt: row_createdAt,
118
- retryCount: 0,
119
- maxRetries: row_maxRetries
120
- };
121
- todoItems[id] = row;
122
- });
123
- Stdlib_Option.forEach(param.resolved, id => Stdlib_Option.forEach(todoItems[id], row => {
246
+ }
247
+ return Stdlib_Option.forEach(Stdlib_Option.flatMap(param.resolved, key => storedKey(key, capabilities)), id => Stdlib_Option.forEach(todoItems[id], row => {
124
248
  let newrecord = {...row};
125
249
  newrecord.completedAt = new Date().toISOString();
126
250
  newrecord.status = "Completed";
@@ -143,8 +267,9 @@ function Make(Spec) {
143
267
  });
144
268
  });
145
269
  };
146
- let phase1 = (events, sliceCtx) => {
147
- events.forEach(json => {
270
+ let phase1 = async (events, sliceCtx, capabilities) => {
271
+ for (let i = 0, i_finish = events.length; i < i_finish; ++i) {
272
+ let json = events[i];
148
273
  let context = Message$ReventlessCore.decode(json, Message$Reventless.contextSchema);
149
274
  let recordedAt = recordedAtOf(context.meta.time);
150
275
  let ctx;
@@ -159,12 +284,14 @@ function Make(Spec) {
159
284
  let dict = Stdlib_JSON.Decode.object(json);
160
285
  let eventPayload = dict !== undefined ? Stdlib_Option.getOr(dict["event"], json) : json;
161
286
  let routed = Route.route(eventPayload, sourceName, context.id, ctx);
162
- routed.forEach(apply);
287
+ for (let j = 0, j_finish = routed.length; j < j_finish; ++j) {
288
+ await apply(routed[j], capabilities);
289
+ }
163
290
  reschedule(routed, context.meta.msgId, Stdlib_Option.getOr(ctx.recordedAt, Date.now()));
164
291
  if (sourceName === Clock$Reventless.name) {
165
- return AutomationSlice_Schedules$ReventlessCore.fired(schedules, context.id);
292
+ AutomationSlice_Schedules$ReventlessCore.fired(schedules, context.id);
166
293
  }
167
- });
294
+ }
168
295
  };
169
296
  let rebuildSchedules = events => {
170
297
  Object.keys(schedules).forEach(key => Stdlib_Dict.$$delete(schedules, key));
@@ -229,60 +356,64 @@ function Make(Spec) {
229
356
  }
230
357
  });
231
358
  let commands = [];
232
- pending.forEach(param => {
233
- let row = param[1];
234
- let id = param[0];
235
- let item;
236
- try {
237
- item = Primitive_option.some(Util_Sury$Reventless.fromJson(row.item, Spec.todoItemSchema));
238
- } catch (raw_exn) {
239
- let exn = Primitive_exceptions.internalToException(raw_exn);
240
- let errMsg = Stdlib_Option.getOr(Stdlib_Option.flatMap(Stdlib_JsExn.fromException(exn), Stdlib_JsExn.message), "unknown");
241
- Effect.runSync(EffectLogger$ReventlessCore.logError(`AutomationSlice(` + Spec.name + `)`, undefined, `failed to decode todoItem: ` + errMsg));
242
- item = undefined;
243
- }
244
- Stdlib_Option.forEach(item, item => {
245
- let match = Automation.process(id, item, currentTime);
246
- if (match === undefined) {
247
- return;
248
- }
249
- let targetId = match[0];
250
- let newrecord = {...row};
251
- newrecord.processedAt = new Date().toISOString();
252
- newrecord.status = "Processing";
253
- todoItems[id] = newrecord;
254
- let commandJson;
255
- try {
256
- commandJson = Util_Sury$Reventless.toJson(match[1], Spec.commandSchema);
257
- } catch (raw_exn) {
258
- let exn = Primitive_exceptions.internalToException(raw_exn);
259
- let errMsg = Stdlib_Option.getOr(Stdlib_Option.flatMap(Stdlib_JsExn.fromException(exn), Stdlib_JsExn.message), "unknown");
260
- Effect.runSync(EffectLogger$ReventlessCore.logError(`AutomationSlice(` + Spec.name + `)`, undefined, `failed to encode command: ` + errMsg));
261
- let updated = afterFailedAttempt(row);
262
- todoItems[id] = updated;
263
- if (updated.status === "Abandoned") {
264
- newlyAbandoned.push(id);
359
+ for (let i = 0, i_finish = pending.length; i < i_finish; ++i) {
360
+ let match = pending[i];
361
+ let row = match[1];
362
+ let id = match[0];
363
+ let field = await openRow(id, row);
364
+ switch (field.TAG) {
365
+ case "Open" :
366
+ let match$1 = Automation.process(field.key, field.item, currentTime);
367
+ if (match$1 !== undefined) {
368
+ let targetId = match$1[0];
369
+ let newrecord = {...row};
370
+ newrecord.processedAt = new Date().toISOString();
371
+ newrecord.status = "Processing";
372
+ todoItems[id] = newrecord;
373
+ let commandJson;
374
+ try {
375
+ commandJson = Util_Sury$Reventless.toJson(match$1[1], Spec.commandSchema);
376
+ } catch (raw_exn) {
377
+ let exn = Primitive_exceptions.internalToException(raw_exn);
378
+ let errMsg = Stdlib_Option.getOr(Stdlib_Option.flatMap(Stdlib_JsExn.fromException(exn), Stdlib_JsExn.message), "unknown");
379
+ Effect.runSync(EffectLogger$ReventlessCore.logError(`AutomationSlice(` + Spec.name + `)`, undefined, `failed to encode command: ` + errMsg));
380
+ let updated = afterFailedAttempt(row);
381
+ todoItems[id] = updated;
382
+ if (updated.status === "Abandoned") {
383
+ newlyAbandoned.push(id);
384
+ }
385
+ commandJson = undefined;
386
+ }
387
+ Stdlib_Option.forEach(commandJson, commandJson => {
388
+ let msg_meta = makeMeta();
389
+ let msg = {
390
+ id: targetId,
391
+ meta: msg_meta,
392
+ commandJson: commandJson
393
+ };
394
+ commands.push(msg);
395
+ });
265
396
  }
266
- commandJson = undefined;
267
- }
268
- Stdlib_Option.forEach(commandJson, commandJson => {
269
- let msg_meta = makeMeta();
270
- let msg = {
271
- id: targetId,
272
- meta: msg_meta,
273
- commandJson: commandJson
274
- };
275
- commands.push(msg);
276
- });
277
- });
278
- });
397
+ break;
398
+ case "Gone" :
399
+ Effect.runSync(EffectLogger$ReventlessCore.logInfo(`AutomationSlice(` + Spec.name + `)`, undefined, id + `: its ` + field._0 + ` was erased; nothing to do`));
400
+ let newrecord$1 = {...row};
401
+ newrecord$1.completedAt = new Date().toISOString();
402
+ newrecord$1.status = "Completed";
403
+ todoItems[id] = newrecord$1;
404
+ break;
405
+ case "Unopened" :
406
+ Effect.runSync(EffectLogger$ReventlessCore.logError(`AutomationSlice(` + Spec.name + `)`, undefined, `failed to decode todoItem: ` + field._0));
407
+ break;
408
+ }
409
+ }
279
410
  if (commands.length !== 0) {
280
411
  try {
281
412
  await publishJsons(commands);
282
- } catch (raw_exn) {
283
- let exn = Primitive_exceptions.internalToException(raw_exn);
284
- let errMsg = Stdlib_Option.getOr(Stdlib_Option.flatMap(Stdlib_JsExn.fromException(exn), Stdlib_JsExn.message), "unknown");
285
- Effect.runSync(EffectLogger$ReventlessCore.logError(`AutomationSlice(` + Spec.name + `)`, undefined, `failed to publish commands: ` + errMsg));
413
+ } catch (raw_exn$1) {
414
+ let exn$1 = Primitive_exceptions.internalToException(raw_exn$1);
415
+ let errMsg$1 = Stdlib_Option.getOr(Stdlib_Option.flatMap(Stdlib_JsExn.fromException(exn$1), Stdlib_JsExn.message), "unknown");
416
+ Effect.runSync(EffectLogger$ReventlessCore.logError(`AutomationSlice(` + Spec.name + `)`, undefined, `failed to publish commands: ` + errMsg$1));
286
417
  pending.forEach(param => {
287
418
  let id = param[0];
288
419
  let current = todoItems[id];
@@ -301,38 +432,26 @@ function Make(Spec) {
301
432
  });
302
433
  }
303
434
  }
304
- let abandonmentCommands = Stdlib_Array.filterMap(newlyAbandoned, id => Stdlib_Option.flatMap(todoItems[id], row => {
305
- let item;
306
- try {
307
- item = Util_Sury$Reventless.fromJson(row.item, Spec.todoItemSchema);
308
- } catch (exn) {
309
- Effect.runSync(EffectLogger$ReventlessCore.logError(`AutomationSlice(` + Spec.name + `)`, undefined, `retries exhausted for ` + id + `, and its item could not be decoded to announce it`));
310
- return;
311
- }
312
- return Stdlib_Option.flatMap(Automation.onExhausted(id, item), param => {
313
- let commandJson;
314
- try {
315
- commandJson = Util_Sury$Reventless.toJson(param[1], Spec.commandSchema);
316
- } catch (exn) {
317
- Effect.runSync(EffectLogger$ReventlessCore.logError(`AutomationSlice(` + Spec.name + `)`, undefined, `retries exhausted for ` + id + `, and the abandonment command could not be encoded`));
318
- return;
435
+ let abandonmentCommands = [];
436
+ for (let i$1 = 0, i_finish$1 = newlyAbandoned.length; i$1 < i_finish$1; ++i$1) {
437
+ let id$1 = newlyAbandoned[i$1];
438
+ let row$1 = todoItems[id$1];
439
+ if (row$1 !== undefined) {
440
+ let command = await abandonmentCommand(id$1, row$1);
441
+ if (command !== undefined) {
442
+ abandonmentCommands.push(command);
319
443
  }
320
- return {
321
- id: param[0],
322
- meta: makeMeta(),
323
- commandJson: commandJson
324
- };
325
- });
326
- }));
444
+ }
445
+ }
327
446
  if (abandonmentCommands.length === 0) {
328
447
  return;
329
448
  }
330
449
  try {
331
450
  return await publishJsons(abandonmentCommands);
332
- } catch (raw_exn$1) {
333
- let exn$1 = Primitive_exceptions.internalToException(raw_exn$1);
334
- let errMsg$1 = Stdlib_Option.getOr(Stdlib_Option.flatMap(Stdlib_JsExn.fromException(exn$1), Stdlib_JsExn.message), "unknown");
335
- return Effect.runSync(EffectLogger$ReventlessCore.logError(`AutomationSlice(` + Spec.name + `)`, undefined, `failed to publish abandonment commands: ` + errMsg$1));
451
+ } catch (raw_exn$2) {
452
+ let exn$2 = Primitive_exceptions.internalToException(raw_exn$2);
453
+ let errMsg$2 = Stdlib_Option.getOr(Stdlib_Option.flatMap(Stdlib_JsExn.fromException(exn$2), Stdlib_JsExn.message), "unknown");
454
+ return Effect.runSync(EffectLogger$ReventlessCore.logError(`AutomationSlice(` + Spec.name + `)`, undefined, `failed to publish abandonment commands: ` + errMsg$2));
336
455
  }
337
456
  };
338
457
  return {
@@ -189,7 +189,7 @@ module Make = (
189
189
  Effect.promise(
190
190
  async () => {
191
191
  // Phase 1: collect outbound items
192
- Callback.phase1(eventsArr)
192
+ await Callback.phase1(eventsArr, ~capabilities=Capabilities.capabilities())
193
193
  // Sync the post-phase-1 TODO state so consumers awaiting the
194
194
  // originating publishEvent observe Pending rows immediately.
195
195
  await syncToQueryDb(queryDbOps)
@@ -108,7 +108,7 @@ function Make(RuntimeEnvironment) {
108
108
  return [];
109
109
  }
110
110
  })), events => Stream$1.fromIterable(events))), eventsArr => Effect.promise(async () => {
111
- Callback.phase1(eventsArr);
111
+ await Callback.phase1(eventsArr, Capabilities.capabilities());
112
112
  await syncToQueryDb(queryDbOps);
113
113
  Stdlib_Promise.$$catch(Callback.phase2(publishJsonsFn, Capabilities.capabilities()).then(() => syncToQueryDb(queryDbOps)), exn => {
114
114
  let errMsg = Stdlib_Option.getOr(Stdlib_Option.flatMap(Stdlib_JsExn.fromException(exn), Stdlib_JsExn.message), "unknown");
@@ -79,7 +79,10 @@ the AWS entry point kept passing bare events and nothing complained, because its
79
79
  hand-written copy was internally consistent. The mismatch surfaced only at
80
80
  runtime, as `undefined` destructured from a tuple that was never a tuple.
81
81
  */
82
- type phase1<'event> = array<(string, 'event)> => unit
82
+ type phase1<'event> = (
83
+ array<(string, 'event)>,
84
+ ~capabilities: Reventless.Capabilities.t,
85
+ ) => promise<unit>
83
86
 
84
87
  /** Phase 2: translate all pending items, optionally publishing commands.
85
88
 
@@ -109,7 +112,8 @@ module type T = {
109
112
 
110
113
  /** Phase 1: update TODO list from a batch of `(sourceId, event)` pairs
111
114
  (collect only, no resolve). `sourceId` is the envelope id of the entity the
112
- event was published for. */
115
+ event was published for. `~capabilities` supplies the key a personal item's
116
+ row id is fingerprinted with. */
113
117
  let phase1: phase1<Spec.consumedEvent>
114
118
 
115
119
  /** Phase 2: translate all pending items, optionally publishing commands via publishJsons. */
@@ -201,25 +205,82 @@ module Make = (
201
205
  )->Effect.runSync
202
206
  }
203
207
 
204
- let phase1 = (events: array<(string, Spec.consumedEvent)>) => {
205
- events->Array.forEach(((sourceId, event)) => {
208
+ // A work item that carries personal data is stored the way an event is: its
209
+ // marked fields sealed, and its row keyed by a fingerprint of the key `collect`
210
+ // chose, which may hold the address the work is about. A table key can be
211
+ // neither encrypted nor erased.
212
+ let personalFields = Reventless.Pii.recordFields(Spec.outboundItemSchema)
213
+ let fingerprintPurpose = `outboundWorkItem:${Spec.name}`
214
+ let warnedUnkeyed = ref(false)
215
+
216
+ /** The id a row is stored under, or `None` when it cannot be stored safely. */
217
+ let storedKey = (key, ~capabilities: Reventless.Capabilities.t): option<string> =>
218
+ if personalFields == [] {
219
+ Some(key)
220
+ } else {
221
+ switch capabilities.secrets.fingerprint(~purpose=fingerprintPurpose, key) {
222
+ | Ok(fingerprint) => Some(fingerprint)
223
+ | Error(Unavailable(why)) =>
224
+ if Reventless.Pii.Provider.isRequired() {
225
+ EffectLogger.logError(
226
+ ~comp=`OutboundTranslationSlice(${Spec.name})`,
227
+ `a work item was not stored: its row id cannot be fingerprinted (${why})`,
228
+ )->Effect.runSync
229
+ None
230
+ } else {
231
+ if !warnedUnkeyed.contents {
232
+ warnedUnkeyed := true
233
+ EffectLogger.logWarn(
234
+ ~comp=`OutboundTranslationSlice(${Spec.name})`,
235
+ `work items carrying personal data are keyed in the clear: no key to fingerprint them with (${why})`,
236
+ )->Effect.runSync
237
+ }
238
+ Some(key)
239
+ }
240
+ }
241
+ }
242
+
243
+ let phase1 = async (
244
+ events: array<(string, Spec.consumedEvent)>,
245
+ ~capabilities: Reventless.Capabilities.t,
246
+ ) =>
247
+ for i in 0 to events->Array.length - 1 {
248
+ let (sourceId, event) = events->Array.getUnsafe(i)
206
249
  // Collect new outbound items
207
- Translation.collect(event, ~sourceId)->Array.forEach(((id, item)) => {
208
- switch todoItems->Dict.get(id) {
209
- | Some(_) => () // Already exists -- skip (idempotent)
210
- | None =>
211
- let row: todoRow = {
212
- item: item->Reventless.Util_Sury.toJson(Spec.outboundItemSchema),
213
- status: Pending,
214
- createdAt: now(),
215
- retryCount: 0,
216
- maxRetries: Spec.maxRetries,
250
+ let collected = Translation.collect(event, ~sourceId)
251
+ for j in 0 to collected->Array.length - 1 {
252
+ let (key, item) = collected->Array.getUnsafe(j)
253
+ switch storedKey(key, ~capabilities) {
254
+ | None => ()
255
+ | Some(id) if todoItems->Dict.get(id)->Option.isSome => () // idempotent
256
+ | Some(id) =>
257
+ let json = item->Reventless.Util_Sury.toJson(Spec.outboundItemSchema)
258
+ let stored = switch json {
259
+ | Object(data) if personalFields != [] =>
260
+ (
261
+ await Reventless.Pii.sealRecord(Spec.outboundItemSchema, data)
262
+ )->Result.map(d => JSON.Object(d))
263
+ | other => Ok(other)
264
+ }
265
+ switch stored {
266
+ | Error(e) =>
267
+ EffectLogger.logError(
268
+ ~comp=`OutboundTranslationSlice(${Spec.name})`,
269
+ `a work item was not stored: ${e->Reventless.Pii.errorMessage}`,
270
+ )->Effect.runSync
271
+ | Ok(stored) =>
272
+ let row: todoRow = {
273
+ item: stored,
274
+ status: Pending,
275
+ createdAt: now(),
276
+ retryCount: 0,
277
+ maxRetries: Spec.maxRetries,
278
+ }
279
+ todoItems->Dict.set(id, row)
217
280
  }
218
- todoItems->Dict.set(id, row)
219
281
  }
220
- })
221
- })
222
- }
282
+ }
283
+ }
223
284
 
224
285
  let phase2 = async (
225
286
  publishJsons: ReventlessInfra.CommandTopic.publishJsons,
@@ -253,15 +314,50 @@ module Make = (
253
314
  let _ = await pending->Array.reduce(Promise.resolve(), async (prev, (id, row)) => {
254
315
  let _ = await prev
255
316
 
256
- let item = try row.item->Reventless.Util_Sury.fromJson(Spec.outboundItemSchema)->Some catch {
257
- | exn =>
258
- let errMsg =
259
- exn->JsExn.fromException->Option.flatMap(JsExn.message)->Option.getOr("unknown")
317
+ // Opened first: a personal field is sealed in the row, and one whose
318
+ // subject was erased since leaves nothing to send.
319
+ let opened = switch row.item {
320
+ | Object(data) if personalFields != [] =>
321
+ switch await Reventless.Pii.openRecord(Spec.outboundItemSchema, data) {
322
+ | Ok(data) =>
323
+ switch Reventless.Pii.goneField(Spec.outboundItemSchema, data) {
324
+ | Some(field) => Error(#Gone(field))
325
+ | None => Ok(JSON.Object(data))
326
+ }
327
+ | Error(e) => Error(#Unopened(e->Reventless.Pii.errorMessage))
328
+ }
329
+ | other => Ok(other)
330
+ }
331
+ let item = switch opened {
332
+ | Error(#Gone(field)) =>
333
+ EffectLogger.logInfo(
334
+ ~comp=`OutboundTranslationSlice(${Spec.name})`,
335
+ `${id}: its ${field} was erased; nothing to do`,
336
+ )->Effect.runSync
337
+ todoItems->Dict.set(id, {...row, status: Completed, completedAt: now()})
338
+ None
339
+ | Error(#Unopened(why)) =>
260
340
  EffectLogger.logError(
261
341
  ~comp=`OutboundTranslationSlice(${Spec.name})`,
262
- `failed to decode outboundItem: ${errMsg}`,
342
+ `${id}: its work item does not open: ${why}`,
263
343
  )->Effect.runSync
344
+ let updated = row->afterFailedAttempt(~lastError=why)
345
+ todoItems->Dict.set(id, updated)
346
+ if updated.status == Abandoned {
347
+ newlyAbandoned->Array.push(id)
348
+ }
264
349
  None
350
+ | Ok(itemJson) =>
351
+ try itemJson->Reventless.Util_Sury.fromJson(Spec.outboundItemSchema)->Some catch {
352
+ | exn =>
353
+ let errMsg =
354
+ exn->JsExn.fromException->Option.flatMap(JsExn.message)->Option.getOr("unknown")
355
+ EffectLogger.logError(
356
+ ~comp=`OutboundTranslationSlice(${Spec.name})`,
357
+ `failed to decode outboundItem: ${errMsg}`,
358
+ )->Effect.runSync
359
+ None
360
+ }
265
361
  }
266
362
 
267
363
  switch item {