@reventlessdev/reventless-core 3.0.0-alpha.320 → 3.0.0-alpha.322

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -3,6 +3,20 @@
3
3
  All notable changes to this project will be documented in this file.
4
4
  See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
5
5
 
6
+ # 3.0.0-alpha.322 (2026-10-30)
7
+
8
+ ### Features
9
+
10
+ * **automation:** a work item with personal data is stored like an event ([f9e173f](https://github.com/ReventlessDev/reventless-core/commit/f9e173fb6f1dc671900052f2c4b6590ee99b5bdf))
11
+
12
+
13
+ # 3.0.0-alpha.321 (2026-10-30)
14
+
15
+ ### Features
16
+
17
+ * **outbound:** a work item with personal data is stored like an event ([5588bbe](https://github.com/ReventlessDev/reventless-core/commit/5588bbeaccc5032800d02871243c70147c669ebd))
18
+
19
+
6
20
  # 3.0.0-alpha.320 (2026-10-30)
7
21
 
8
22
  ### Features
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@reventlessdev/reventless-core",
3
- "version": "3.0.0-alpha.320",
3
+ "version": "3.0.0-alpha.322",
4
4
  "description": "Core package for Reventless framework",
5
5
  "license": "Apache-2.0",
6
6
  "jest": {
@@ -29,16 +29,16 @@
29
29
  "sury": "11.0.0",
30
30
  "uuid": "^13.0.0",
31
31
  "@reventlessdev/rescript-effect": "1.0.0-alpha.1",
32
+ "@reventlessdev/rescript-fast-csv": "2.0.0-alpha.24",
32
33
  "@reventlessdev/rescript-hash-object": "1.2.0-alpha.15",
33
- "@reventlessdev/rescript-node": "2.0.0-alpha.24",
34
34
  "@reventlessdev/rescript-jest": "1.0.0-alpha.13",
35
- "@reventlessdev/rescript-fast-csv": "2.0.0-alpha.24",
36
- "@reventlessdev/rescript-ssh2": "2.0.0-alpha.25",
37
- "@reventlessdev/rescript-uuid": "2.0.0-alpha.1",
38
- "@reventlessdev/reventless-infra": "3.0.0-alpha.217",
39
- "@reventlessdev/reventless-spec": "3.0.0-alpha.188",
35
+ "@reventlessdev/rescript-node": "2.0.0-alpha.24",
40
36
  "@reventlessdev/rescript-pulumi-pulumi": "3.0.0-alpha.0",
41
- "@reventlessdev/reventless-interop": "3.0.0-alpha.42"
37
+ "@reventlessdev/reventless-infra": "3.0.0-alpha.219",
38
+ "@reventlessdev/rescript-ssh2": "2.0.0-alpha.25",
39
+ "@reventlessdev/reventless-interop": "3.0.0-alpha.42",
40
+ "@reventlessdev/reventless-spec": "3.0.0-alpha.190",
41
+ "@reventlessdev/rescript-uuid": "2.0.0-alpha.1"
42
42
  },
43
43
  "devDependencies": {
44
44
  "rescript": "12.3.1",
@@ -26,6 +26,12 @@ module Make = (
26
26
  Schedules: {
27
27
  let platformFor: (~slice: string) => ReventlessInfra.AutomationSlice.platformSchedules
28
28
  },
29
+ // What the platform provisioned: the key a work item carrying personal data is
30
+ // fingerprinted with. A thunk, as on the outbound builder, so a platform
31
+ // resolves it when the runtime exists.
32
+ Capabilities: {
33
+ let capabilities: unit => Reventless.Capabilities.t
34
+ },
29
35
  ) => {
30
36
  let finish = EventCollectorRuntimeBuilder.finish
31
37
  module Make = (
@@ -227,7 +233,7 @@ module Make = (
227
233
  Effect.promise(
228
234
  async () => {
229
235
  // Phase 1: per-source decode + collect/resolve, threaded with context
230
- Callback.phase1(jsons, context)
236
+ await Callback.phase1(jsons, context, ~capabilities=Capabilities.capabilities())
231
237
  // Sync the post-phase-1 TODO state so consumers awaiting the
232
238
  // originating publishEvent observe Pending rows immediately.
233
239
  await syncToQueryDb(queryDbOps)
@@ -307,7 +313,11 @@ module Make = (
307
313
  // As the AWS sweep does: the rows phase 2 moved are written back.
308
314
  processPending,
309
315
  tick: async (~key, ~at) => {
310
- Callback.phase1([AutomationSlice_Callback.tickEvent(~key, ~at)], context)
316
+ await Callback.phase1(
317
+ [AutomationSlice_Callback.tickEvent(~key, ~at)],
318
+ context,
319
+ ~capabilities=Capabilities.capabilities(),
320
+ )
311
321
  await syncToQueryDb(queryDbOps)
312
322
  // A once-only schedule's row went when it fired, and a Clock mapping
313
323
  // may change schedules too.
@@ -28,7 +28,7 @@ import * as AutomationSlice_Callback$ReventlessCore from "./AutomationSlice_Call
28
28
  import * as AutomationSlice_Schedules$ReventlessCore from "./AutomationSlice_Schedules.res.mjs";
29
29
 
30
30
  function Make(RuntimeEnvironment) {
31
- return QueryDbStorage => (QueryDbResolvers => (EventCollectorChannel => (EventCollectorRuntimeBuilder => (Api => (Schedules => {
31
+ return QueryDbStorage => (QueryDbResolvers => (EventCollectorChannel => (EventCollectorRuntimeBuilder => (Api => (Schedules => (Capabilities => {
32
32
  let Make = Spec => (Automation => {
33
33
  let Callback = AutomationSlice_Callback$ReventlessCore.Make(Spec)(Automation);
34
34
  let queryDbName = Spec.name + "Todo";
@@ -131,7 +131,7 @@ function Make(RuntimeEnvironment) {
131
131
  name: Spec.name
132
132
  }, opts);
133
133
  let jsonEventsHandler = stream => Effect.flatMap(Stream.runCollect(stream), jsons => Effect.promise(async () => {
134
- Callback.phase1(jsons, context);
134
+ await Callback.phase1(jsons, context, Capabilities.capabilities());
135
135
  await syncToQueryDb(queryDbOps);
136
136
  let changes = Callback.takeScheduleChanges();
137
137
  if (changes.length !== 0) {
@@ -172,7 +172,7 @@ function Make(RuntimeEnvironment) {
172
172
  enqueueEvent: param[0].enqueueEvent,
173
173
  processPending: processPending,
174
174
  tick: async (key, at) => {
175
- Callback.phase1([AutomationSlice_Callback$ReventlessCore.tickEvent(key, at)], context);
175
+ await Callback.phase1([AutomationSlice_Callback$ReventlessCore.tickEvent(key, at)], context, Capabilities.capabilities());
176
176
  await syncToQueryDb(queryDbOps);
177
177
  await syncSchedules(schedulesOps, storedSchedules);
178
178
  await AutomationSlice_Schedules$ReventlessCore.tell(platform, Callback.takeScheduleChanges());
@@ -208,7 +208,7 @@ function Make(RuntimeEnvironment) {
208
208
  finish: EventCollectorRuntimeBuilder.finish,
209
209
  Make: Make
210
210
  };
211
- })))));
211
+ }))))));
212
212
  }
213
213
 
214
214
  export {
@@ -82,7 +82,11 @@ Unlike outbound, phase 1 here takes *raw* envelope JSONs: the callback routes by
82
82
  `meta.service` and unwraps `{id, meta, event}` itself, per-source, through the
83
83
  registered Mappings.
84
84
  */
85
- type phase1 = (array<JSON.t>, Reventless.AutomationSlice.context) => unit
85
+ type phase1 = (
86
+ array<JSON.t>,
87
+ Reventless.AutomationSlice.context,
88
+ ~capabilities: Reventless.Capabilities.t,
89
+ ) => promise<unit>
86
90
 
87
91
  /** Phase 2: process all pending items, publishing commands. */
88
92
  type phase2 = ReventlessInfra.CommandTopic.publishJsons => promise<unit>
@@ -178,6 +182,98 @@ module Make = (
178
182
  // the slice here makes the target's own events unmatchable, silently.
179
183
  let makeMeta = (): Reventless.Message.meta => Message.generateMeta(~service=Spec.targetName)
180
184
 
185
+ // A work item that carries personal data is stored the way an event is: its
186
+ // marked fields sealed, its row keyed by a fingerprint of the key the mapping
187
+ // chose, and that key sealed inside the item under `keyField`, so `process`,
188
+ // `resolve` and `onExhausted` still see the key the slice wrote.
189
+ let personalFields = Reventless.Pii.recordFields(Spec.todoItemSchema)
190
+ let keyField = "$key"
191
+ let fingerprintPurpose = `automationWorkItem:${Spec.name}`
192
+ let warnedUnkeyed = ref(false)
193
+
194
+ /** The id a row is stored under, or `None` when it cannot be stored safely. */
195
+ let storedKey = (key, ~capabilities: Reventless.Capabilities.t): option<string> =>
196
+ if personalFields == [] {
197
+ Some(key)
198
+ } else {
199
+ switch capabilities.secrets.fingerprint(~purpose=fingerprintPurpose, key) {
200
+ | Ok(fingerprint) => Some(fingerprint)
201
+ | Error(Unavailable(why)) =>
202
+ if Reventless.Pii.Provider.isRequired() {
203
+ EffectLogger.logError(
204
+ ~comp=`AutomationSlice(${Spec.name})`,
205
+ `a work item was not stored: its row id cannot be fingerprinted (${why})`,
206
+ )->Effect.runSync
207
+ None
208
+ } else {
209
+ if !warnedUnkeyed.contents {
210
+ warnedUnkeyed := true
211
+ EffectLogger.logWarn(
212
+ ~comp=`AutomationSlice(${Spec.name})`,
213
+ `work items carrying personal data are keyed in the clear: no key to fingerprint them with (${why})`,
214
+ )->Effect.runSync
215
+ }
216
+ Some(key)
217
+ }
218
+ }
219
+ }
220
+
221
+ /** The item as a row stores it: sealed, with the key it was collected under. */
222
+ let storedItem = async (~key, item: Spec.todoItem): result<JSON.t, string> => {
223
+ let json = item->Reventless.Util_Sury.toJson(Spec.todoItemSchema)
224
+ switch (json, personalFields->Array.get(0)) {
225
+ | (_, None) => Ok(json)
226
+ | (Object(data), Some((_, marker))) =>
227
+ let data = data->Dict.copy
228
+ data->Dict.set(keyField, JSON.Encode.string(key))
229
+
230
+ (
231
+ await Reventless.Pii.sealFields(
232
+ ~fields=personalFields->Array.concat([(keyField, marker)]),
233
+ ~label=Spec.name,
234
+ ~bare=true,
235
+ data,
236
+ )
237
+ )
238
+ ->Result.map(d => JSON.Object(d))
239
+ ->Result.mapError(Reventless.Pii.errorMessage)
240
+ | (other, _) => Ok(other)
241
+ }
242
+ }
243
+
244
+ type opened =
245
+ | Open({key: string, item: Spec.todoItem})
246
+ | Gone(string)
247
+ | Unopened(string)
248
+
249
+ /** A row's item and the key it was collected under, opened. */
250
+ let openRow = async (~id, row: todoRow): opened => {
251
+ let decode = (json, ~key) =>
252
+ switch json->Reventless.Util_Sury.fromJson(Spec.todoItemSchema) {
253
+ | item => Open({key, item})
254
+ | exception exn =>
255
+ Unopened(exn->JsExn.fromException->Option.flatMap(JsExn.message)->Option.getOr("unknown"))
256
+ }
257
+ switch row.item {
258
+ | Object(data) if personalFields != [] =>
259
+ switch await Reventless.Pii.openFields(
260
+ ~fields=personalFields->Array.map(((f, _)) => f)->Array.concat([keyField]),
261
+ data,
262
+ ) {
263
+ | Error(e) => Unopened(e->Reventless.Pii.errorMessage)
264
+ | Ok(data) =>
265
+ switch Reventless.Pii.goneField(Spec.todoItemSchema, data) {
266
+ | Some(field) => Gone(field)
267
+ | None =>
268
+ let key = data->Dict.get(keyField)->Option.flatMap(JSON.Decode.string)->Option.getOr(id)
269
+ data->Dict.delete(keyField)
270
+ JSON.Object(data)->decode(~key)
271
+ }
272
+ }
273
+ | json => json->decode(~key=id)
274
+ }
275
+ }
276
+
181
277
  /** How long a row may stay `Processing` before its command is sent again. */
182
278
  let processingTimeoutMs =
183
279
  15. *. 60. *. 1000.
@@ -193,10 +289,12 @@ module Make = (
193
289
 
194
290
  /** The command a slice wants published now that this item will never be
195
291
  processed again, if it wants one. The row is `Abandoned` either way. */
196
- let abandonmentCommand = (~id: string, ~row: todoRow): option<Reventless.Message.commandJson> =>
197
- switch row.item->Reventless.Util_Sury.fromJson(Spec.todoItemSchema) {
198
- | item =>
199
- Automation.onExhausted(id, item)->Option.flatMap(((targetId, command)) =>
292
+ let abandonmentCommand = async (~id: string, ~row: todoRow): option<
293
+ Reventless.Message.commandJson,
294
+ > =>
295
+ switch await openRow(~id, row) {
296
+ | Open({key, item}) =>
297
+ Automation.onExhausted(key, item)->Option.flatMap(((targetId, command)) =>
200
298
  switch command->Reventless.Util_Sury.toJson(Spec.commandSchema) {
201
299
  | commandJson => Some({Reventless.Message.id: targetId, meta: makeMeta(), commandJson})
202
300
  | exception _ =>
@@ -207,7 +305,8 @@ module Make = (
207
305
  None
208
306
  }
209
307
  )
210
- | exception _ =>
308
+ | Gone(_) => None
309
+ | Unopened(_) =>
211
310
  EffectLogger.logError(
212
311
  ~comp=`AutomationSlice(${Spec.name})`,
213
312
  `retries exhausted for ${id}, and its item could not be decoded to announce it`,
@@ -224,24 +323,37 @@ module Make = (
224
323
  | millis => Some(millis)
225
324
  }
226
325
 
227
- let apply = ({collected, resolved}: AutomationSlice_Route.routed<Spec.todoItem>) => {
228
- // Collect — append new items keyed by ID; first writer wins (idempotent).
229
- collected->Array.forEach(((id, item)) => {
230
- switch todoItems->Dict.get(id) {
231
- | Some(_) => () // Already exists — skip
232
- | None =>
233
- let row: todoRow = {
234
- item: item->Reventless.Util_Sury.toJson(Spec.todoItemSchema),
235
- status: Pending,
236
- createdAt: now(),
237
- retryCount: 0,
238
- maxRetries: Spec.maxRetries,
326
+ let apply = async (
327
+ {collected, resolved}: AutomationSlice_Route.routed<Spec.todoItem>,
328
+ ~capabilities,
329
+ ) => {
330
+ for i in 0 to collected->Array.length - 1 {
331
+ let (key, item) = collected->Array.getUnsafe(i)
332
+ switch storedKey(key, ~capabilities) {
333
+ | None => ()
334
+ | Some(id) if todoItems->Dict.get(id)->Option.isSome => () // Already exists — skip
335
+ | Some(id) =>
336
+ switch await storedItem(~key, item) {
337
+ | Error(why) =>
338
+ EffectLogger.logError(
339
+ ~comp=`AutomationSlice(${Spec.name})`,
340
+ `a work item was not stored: ${why}`,
341
+ )->Effect.runSync
342
+ | Ok(stored) =>
343
+ let row: todoRow = {
344
+ item: stored,
345
+ status: Pending,
346
+ createdAt: now(),
347
+ retryCount: 0,
348
+ maxRetries: Spec.maxRetries,
349
+ }
350
+ todoItems->Dict.set(id, row)
239
351
  }
240
- todoItems->Dict.set(id, row)
241
352
  }
242
- })
243
- // Resolve — mark a pending item as completed.
244
- resolved->Option.forEach(id =>
353
+ }
354
+ resolved
355
+ ->Option.flatMap(key => storedKey(key, ~capabilities))
356
+ ->Option.forEach(id =>
245
357
  todoItems
246
358
  ->Dict.get(id)
247
359
  ->Option.forEach(row => {
@@ -276,8 +388,13 @@ module Make = (
276
388
  )
277
389
  )
278
390
 
279
- let phase1 = (events: array<JSON.t>, sliceCtx: Reventless.AutomationSlice.context) => {
280
- events->Array.forEach(json => {
391
+ let phase1 = async (
392
+ events: array<JSON.t>,
393
+ sliceCtx: Reventless.AutomationSlice.context,
394
+ ~capabilities: Reventless.Capabilities.t,
395
+ ) =>
396
+ for i in 0 to events->Array.length - 1 {
397
+ let json = events->Array.getUnsafe(i)
281
398
  let context = json->Message.decode(Reventless.Message.contextSchema)
282
399
  let ctx = switch recordedAtOf(context.meta.time) {
283
400
  | Some(recordedAt) => {...sliceCtx, recordedAt}
@@ -293,7 +410,9 @@ module Make = (
293
410
  }
294
411
  // Multiple mappings can share a sourceName; all matching dispatches run.
295
412
  let routed = Route.route(eventPayload, ~sourceName, ~sourceId=context.id, ctx)
296
- routed->Array.forEach(apply)
413
+ for j in 0 to routed->Array.length - 1 {
414
+ await apply(routed->Array.getUnsafe(j), ~capabilities)
415
+ }
297
416
  reschedule(
298
417
  routed,
299
418
  ~setBy=context.meta.msgId,
@@ -304,8 +423,7 @@ module Make = (
304
423
  if sourceName == Reventless.Clock.name {
305
424
  schedules->AutomationSlice_Schedules.fired(~key=context.id)
306
425
  }
307
- })
308
- }
426
+ }
309
427
 
310
428
  let rebuildSchedules = (events: array<JSON.t>) => {
311
429
  schedules->Dict.keysToArray->Array.forEach(key => schedules->Dict.delete(key))
@@ -367,20 +485,22 @@ module Make = (
367
485
 
368
486
  let commands = []
369
487
 
370
- pending->Array.forEach(((id, row)) => {
371
- let item = try row.item->Reventless.Util_Sury.fromJson(Spec.todoItemSchema)->Some catch {
372
- | exn =>
373
- let errMsg =
374
- exn->JsExn.fromException->Option.flatMap(JsExn.message)->Option.getOr("unknown")
488
+ for i in 0 to pending->Array.length - 1 {
489
+ let (id, row) = pending->Array.getUnsafe(i)
490
+ switch await openRow(~id, row) {
491
+ | Gone(field) =>
492
+ EffectLogger.logInfo(
493
+ ~comp=`AutomationSlice(${Spec.name})`,
494
+ `${id}: its ${field} was erased; nothing to do`,
495
+ )->Effect.runSync
496
+ todoItems->Dict.set(id, {...row, status: Completed, completedAt: now()})
497
+ | Unopened(errMsg) =>
375
498
  EffectLogger.logError(
376
499
  ~comp=`AutomationSlice(${Spec.name})`,
377
500
  `failed to decode todoItem: ${errMsg}`,
378
501
  )->Effect.runSync
379
- None
380
- }
381
-
382
- item->Option.forEach(item => {
383
- switch Automation.process(id, item, ~now=currentTime) {
502
+ | Open({key, item}) =>
503
+ switch Automation.process(key, item, ~now=currentTime) {
384
504
  | Some((targetId, command)) =>
385
505
  todoItems->Dict.set(id, {...row, status: Processing, processedAt: now()})
386
506
  let commandJson = try command
@@ -400,20 +520,18 @@ module Make = (
400
520
  }
401
521
  None
402
522
  }
403
- commandJson->Option.forEach(
404
- commandJson => {
405
- let msg: Reventless.Message.commandJson = {
406
- id: targetId,
407
- meta: makeMeta(),
408
- commandJson,
409
- }
410
- let _ = commands->Array.push(msg)
411
- },
412
- )
523
+ commandJson->Option.forEach(commandJson => {
524
+ let msg: Reventless.Message.commandJson = {
525
+ id: targetId,
526
+ meta: makeMeta(),
527
+ commandJson,
528
+ }
529
+ let _ = commands->Array.push(msg)
530
+ })
413
531
  | None => () // Skip — process returned None
414
532
  }
415
- })
416
- })
533
+ }
534
+ }
417
535
 
418
536
  if commands->Array.length > 0 {
419
537
  try await publishJsons(commands) catch {
@@ -442,10 +560,18 @@ module Make = (
442
560
  // Announced after the batch, so a publish failure above has already had its
443
561
  // say on which rows ran out of budget. A failure here cannot mark anything for
444
562
  // retry — the budget is what ran out — so the rows stay `Abandoned` regardless.
445
- let abandonmentCommands =
446
- newlyAbandoned->Array.filterMap(id =>
447
- todoItems->Dict.get(id)->Option.flatMap(row => abandonmentCommand(~id, ~row))
448
- )
563
+ let abandonmentCommands = []
564
+ for i in 0 to newlyAbandoned->Array.length - 1 {
565
+ let id = newlyAbandoned->Array.getUnsafe(i)
566
+ switch todoItems->Dict.get(id) {
567
+ | Some(row) =>
568
+ switch await abandonmentCommand(~id, ~row) {
569
+ | Some(command) => abandonmentCommands->Array.push(command)
570
+ | None => ()
571
+ }
572
+ | None => ()
573
+ }
574
+ }
449
575
  if abandonmentCommands->Array.length > 0 {
450
576
  try await publishJsons(abandonmentCommands) catch {
451
577
  | exn =>