@rikalabs/akter 0.1.0-alpha.0 → 0.1.0-alpha.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (308) hide show
  1. package/CHANGELOG.md +46 -3
  2. package/README.md +2 -2
  3. package/dist/actor/definition.d.ts +13 -9
  4. package/dist/actor/descriptor.d.ts +2 -2
  5. package/dist/actor/descriptor.js +4 -4
  6. package/dist/client/calls.js +1 -1
  7. package/dist/client/index.d.ts +1 -1
  8. package/dist/client/index.js +1 -1
  9. package/dist/errors/actor.d.ts +12 -42
  10. package/dist/errors/actor.js +13 -42
  11. package/dist/index.d.ts +1 -1
  12. package/dist/index.js +1 -1
  13. package/dist/members/payload.d.ts +1 -1
  14. package/dist/protocol/inspection.d.ts +221 -1
  15. package/dist/protocol/inspection.js +151 -2
  16. package/dist/protocol/wire.d.ts +6 -49
  17. package/dist/protocol/wire.js +1 -10
  18. package/dist/runtime/actors.d.ts +3 -0
  19. package/dist/runtime/adoption/backfill.js +1 -1
  20. package/dist/runtime/adoption/enforce.js +2 -2
  21. package/dist/runtime/adoption/observe.js +1 -1
  22. package/dist/runtime/adoption/plan.d.ts +1 -1
  23. package/dist/runtime/adoption/startup.js +1 -1
  24. package/dist/runtime/adoption/status.d.ts +1 -1
  25. package/dist/runtime/app.d.ts +65 -0
  26. package/dist/runtime/app.js +38 -0
  27. package/dist/runtime/connections/holder.d.ts +11 -1
  28. package/dist/runtime/connections/holder.js +8 -4
  29. package/dist/runtime/connections/streams.d.ts +4 -4
  30. package/dist/runtime/content/store.js +15 -11
  31. package/dist/runtime/database/bounded.d.ts +4 -18
  32. package/dist/runtime/database/bounded.js +36 -22
  33. package/dist/runtime/database/coordination.d.ts +12 -3
  34. package/dist/runtime/database/coordination.js +29 -36
  35. package/dist/runtime/database/migrations.d.ts +61 -29
  36. package/dist/runtime/database/migrations.js +375 -139
  37. package/dist/runtime/database/replica.d.ts +4 -1
  38. package/dist/runtime/database/replica.js +4 -1
  39. package/dist/runtime/database/schema.d.ts +13 -0
  40. package/dist/runtime/database/schema.js +13 -0
  41. package/dist/runtime/database/shards.d.ts +31 -13
  42. package/dist/runtime/database/shards.js +38 -26
  43. package/dist/runtime/entity/register.d.ts +8 -1
  44. package/dist/runtime/entity/register.js +58 -7
  45. package/dist/runtime/fleet/checks.js +4 -4
  46. package/dist/runtime/fleet/maintainer.d.ts +1 -1
  47. package/dist/runtime/fleet/maintainer.js +1 -1
  48. package/dist/runtime/index.d.ts +2 -0
  49. package/dist/runtime/index.js +1 -0
  50. package/dist/runtime/inspector/layer.d.ts +20 -0
  51. package/dist/runtime/inspector/layer.js +224 -5
  52. package/dist/runtime/inspector/queries.d.ts +54 -8
  53. package/dist/runtime/inspector/queries.js +84 -15
  54. package/dist/runtime/layer.d.ts +48 -39
  55. package/dist/runtime/layer.js +124 -56
  56. package/dist/runtime/operators/audit.d.ts +3 -0
  57. package/dist/runtime/operators/audit.js +6 -2
  58. package/dist/runtime/operators/repair.js +30 -10
  59. package/dist/runtime/operators/seed.js +4 -2
  60. package/dist/runtime/payloads/versions.d.ts +4 -4
  61. package/dist/runtime/payloads/versions.js +24 -15
  62. package/dist/runtime/queries.js +4 -2
  63. package/dist/runtime/registration.d.ts +5 -2
  64. package/dist/runtime/registration.js +3 -3
  65. package/dist/runtime/runner.d.ts +12 -0
  66. package/dist/runtime/runner.js +12 -0
  67. package/dist/runtime/storage/codec.d.ts +19 -7
  68. package/dist/runtime/storage/codec.js +23 -7
  69. package/dist/runtime/storage/placements.d.ts +6 -1
  70. package/dist/runtime/storage/placements.js +98 -6
  71. package/dist/runtime/storage/retention.d.ts +6 -0
  72. package/dist/runtime/storage/retention.js +25 -13
  73. package/dist/runtime/subscriptions/relay.js +47 -44
  74. package/dist/runtime/subscriptions/storage.d.ts +3 -2
  75. package/dist/runtime/subscriptions/storage.js +9 -6
  76. package/dist/runtime/telemetry/live.d.ts +255 -0
  77. package/dist/runtime/telemetry/live.js +540 -0
  78. package/dist/runtime/telemetry/metrics.d.ts +2 -0
  79. package/dist/runtime/telemetry/metrics.js +4 -1
  80. package/dist/runtime/telemetry/sampler.js +33 -13
  81. package/dist/runtime/telemetry/spans.d.ts +1 -1
  82. package/dist/runtime/telemetry/spans.js +1 -1
  83. package/dist/runtime/topology/locks.d.ts +6 -3
  84. package/dist/runtime/topology/locks.js +27 -4
  85. package/dist/runtime/turn/execute.d.ts +2 -0
  86. package/dist/runtime/turn/execute.js +5 -2
  87. package/dist/runtime/turn/pipeline.d.ts +5 -12
  88. package/dist/runtime/turn/pipeline.js +11 -11
  89. package/dist/runtime/turn/relay.d.ts +13 -3
  90. package/dist/runtime/turn/relay.js +27 -14
  91. package/dist/runtime/turn/rows.js +4 -4
  92. package/dist/runtime/workflows/compatibility.d.ts +1 -1
  93. package/dist/runtime/workflows/compatibility.js +85 -31
  94. package/dist/runtime/workflows/engine.js +20 -17
  95. package/dist/serve/layer.js +3 -2
  96. package/dist/tables/owned.d.ts +1 -1
  97. package/dist/tables/owned.js +1 -1
  98. package/dist/testing/actor-test.d.ts +2 -24
  99. package/dist/testing/actor-test.js +1 -27
  100. package/dist/testing/index.d.ts +0 -12
  101. package/dist/testing/index.js +0 -6
  102. package/package.json +1 -1
  103. package/dist/runtime/database/neki/migrations.d.ts +0 -31
  104. package/dist/runtime/database/neki/migrations.js +0 -276
  105. package/dist/runtime/database/neki/session.d.ts +0 -23
  106. package/dist/runtime/database/neki/session.js +0 -31
  107. package/dist/testing/cluster.d.ts +0 -93
  108. package/dist/testing/cluster.js +0 -296
  109. package/dist/testing/conformance/access.d.ts +0 -5
  110. package/dist/testing/conformance/access.js +0 -531
  111. package/dist/testing/conformance/admission.d.ts +0 -11
  112. package/dist/testing/conformance/admission.js +0 -182
  113. package/dist/testing/conformance/adoption.d.ts +0 -2
  114. package/dist/testing/conformance/adoption.js +0 -1019
  115. package/dist/testing/conformance/assertions.d.ts +0 -202
  116. package/dist/testing/conformance/assertions.js +0 -849
  117. package/dist/testing/conformance/batches.d.ts +0 -41
  118. package/dist/testing/conformance/batches.js +0 -388
  119. package/dist/testing/conformance/blobs.d.ts +0 -14
  120. package/dist/testing/conformance/blobs.js +0 -405
  121. package/dist/testing/conformance/capacity.d.ts +0 -8
  122. package/dist/testing/conformance/capacity.js +0 -433
  123. package/dist/testing/conformance/client.d.ts +0 -5
  124. package/dist/testing/conformance/client.js +0 -719
  125. package/dist/testing/conformance/cold-serve.d.ts +0 -16
  126. package/dist/testing/conformance/cold-serve.js +0 -254
  127. package/dist/testing/conformance/connections/actors.d.ts +0 -1143
  128. package/dist/testing/conformance/connections/actors.js +0 -180
  129. package/dist/testing/conformance/connections/harness.d.ts +0 -1754
  130. package/dist/testing/conformance/connections/harness.js +0 -142
  131. package/dist/testing/conformance/connections/holders.d.ts +0 -4
  132. package/dist/testing/conformance/connections/holders.js +0 -165
  133. package/dist/testing/conformance/connections/lifecycle.d.ts +0 -4
  134. package/dist/testing/conformance/connections/lifecycle.js +0 -113
  135. package/dist/testing/conformance/connections/limits.d.ts +0 -4
  136. package/dist/testing/conformance/connections/limits.js +0 -127
  137. package/dist/testing/conformance/connections/reauthorization.d.ts +0 -4
  138. package/dist/testing/conformance/connections/reauthorization.js +0 -170
  139. package/dist/testing/conformance/connections/resync.d.ts +0 -4
  140. package/dist/testing/conformance/connections/resync.js +0 -360
  141. package/dist/testing/conformance/connections/takeover.d.ts +0 -4
  142. package/dist/testing/conformance/connections/takeover.js +0 -287
  143. package/dist/testing/conformance/connections.d.ts +0 -4
  144. package/dist/testing/conformance/connections.js +0 -15
  145. package/dist/testing/conformance/content-blobs.d.ts +0 -16
  146. package/dist/testing/conformance/content-blobs.js +0 -534
  147. package/dist/testing/conformance/counter.d.ts +0 -16
  148. package/dist/testing/conformance/counter.js +0 -813
  149. package/dist/testing/conformance/cron.d.ts +0 -5
  150. package/dist/testing/conformance/cron.js +0 -665
  151. package/dist/testing/conformance/drain.d.ts +0 -26
  152. package/dist/testing/conformance/drain.js +0 -540
  153. package/dist/testing/conformance/effect-control.d.ts +0 -38
  154. package/dist/testing/conformance/effect-control.js +0 -1059
  155. package/dist/testing/conformance/effects.d.ts +0 -23
  156. package/dist/testing/conformance/effects.js +0 -432
  157. package/dist/testing/conformance/events.d.ts +0 -28
  158. package/dist/testing/conformance/events.js +0 -335
  159. package/dist/testing/conformance/export.d.ts +0 -2
  160. package/dist/testing/conformance/export.js +0 -368
  161. package/dist/testing/conformance/fleet.d.ts +0 -444
  162. package/dist/testing/conformance/fleet.js +0 -1009
  163. package/dist/testing/conformance/groups.d.ts +0 -7
  164. package/dist/testing/conformance/groups.js +0 -353
  165. package/dist/testing/conformance/heap.d.ts +0 -3
  166. package/dist/testing/conformance/heap.js +0 -149
  167. package/dist/testing/conformance/http.d.ts +0 -2809
  168. package/dist/testing/conformance/http.js +0 -1234
  169. package/dist/testing/conformance/inspection-views.d.ts +0 -8
  170. package/dist/testing/conformance/inspection-views.js +0 -272
  171. package/dist/testing/conformance/inspector.d.ts +0 -7
  172. package/dist/testing/conformance/inspector.js +0 -650
  173. package/dist/testing/conformance/mint.d.ts +0 -58
  174. package/dist/testing/conformance/mint.js +0 -692
  175. package/dist/testing/conformance/multi-runner.d.ts +0 -8
  176. package/dist/testing/conformance/multi-runner.js +0 -403
  177. package/dist/testing/conformance/neki/plan.d.ts +0 -19
  178. package/dist/testing/conformance/neki/plan.js +0 -30
  179. package/dist/testing/conformance/observability.d.ts +0 -3
  180. package/dist/testing/conformance/observability.js +0 -343
  181. package/dist/testing/conformance/offline.d.ts +0 -4
  182. package/dist/testing/conformance/offline.js +0 -474
  183. package/dist/testing/conformance/operator-harness.d.ts +0 -57
  184. package/dist/testing/conformance/operator-harness.js +0 -78
  185. package/dist/testing/conformance/operator.d.ts +0 -3
  186. package/dist/testing/conformance/operator.js +0 -379
  187. package/dist/testing/conformance/outbox-cross-shard.d.ts +0 -14
  188. package/dist/testing/conformance/outbox-cross-shard.js +0 -160
  189. package/dist/testing/conformance/outbox.d.ts +0 -36
  190. package/dist/testing/conformance/outbox.js +0 -437
  191. package/dist/testing/conformance/payload-migrations.d.ts +0 -3
  192. package/dist/testing/conformance/payload-migrations.js +0 -612
  193. package/dist/testing/conformance/pipeline.d.ts +0 -3
  194. package/dist/testing/conformance/pipeline.js +0 -1040
  195. package/dist/testing/conformance/placement.d.ts +0 -22
  196. package/dist/testing/conformance/placement.js +0 -539
  197. package/dist/testing/conformance/platform.d.ts +0 -40
  198. package/dist/testing/conformance/platform.js +0 -101
  199. package/dist/testing/conformance/progress.d.ts +0 -39
  200. package/dist/testing/conformance/progress.js +0 -666
  201. package/dist/testing/conformance/properties.d.ts +0 -6
  202. package/dist/testing/conformance/properties.js +0 -295
  203. package/dist/testing/conformance/protocols.d.ts +0 -4
  204. package/dist/testing/conformance/protocols.js +0 -618
  205. package/dist/testing/conformance/provider/advertise.d.ts +0 -21
  206. package/dist/testing/conformance/provider/advertise.js +0 -44
  207. package/dist/testing/conformance/read-your-writes.d.ts +0 -20
  208. package/dist/testing/conformance/read-your-writes.js +0 -192
  209. package/dist/testing/conformance/reducers.d.ts +0 -8
  210. package/dist/testing/conformance/reducers.js +0 -258
  211. package/dist/testing/conformance/relay.d.ts +0 -42
  212. package/dist/testing/conformance/relay.js +0 -1115
  213. package/dist/testing/conformance/restore.d.ts +0 -43
  214. package/dist/testing/conformance/restore.js +0 -341
  215. package/dist/testing/conformance/retention.d.ts +0 -14
  216. package/dist/testing/conformance/retention.js +0 -463
  217. package/dist/testing/conformance/rls.d.ts +0 -24
  218. package/dist/testing/conformance/rls.js +0 -576
  219. package/dist/testing/conformance/simulation.d.ts +0 -31
  220. package/dist/testing/conformance/simulation.js +0 -167
  221. package/dist/testing/conformance/single-shard.d.ts +0 -2
  222. package/dist/testing/conformance/single-shard.js +0 -125
  223. package/dist/testing/conformance/singleton.d.ts +0 -3
  224. package/dist/testing/conformance/singleton.js +0 -293
  225. package/dist/testing/conformance/statements.d.ts +0 -60
  226. package/dist/testing/conformance/statements.js +0 -108
  227. package/dist/testing/conformance/streams.d.ts +0 -7
  228. package/dist/testing/conformance/streams.js +0 -280
  229. package/dist/testing/conformance/subscriptions/access.d.ts +0 -4
  230. package/dist/testing/conformance/subscriptions/access.js +0 -98
  231. package/dist/testing/conformance/subscriptions/actors.d.ts +0 -4910
  232. package/dist/testing/conformance/subscriptions/actors.js +0 -372
  233. package/dist/testing/conformance/subscriptions/cluster.d.ts +0 -4
  234. package/dist/testing/conformance/subscriptions/cluster.js +0 -89
  235. package/dist/testing/conformance/subscriptions/delivery.d.ts +0 -4
  236. package/dist/testing/conformance/subscriptions/delivery.js +0 -253
  237. package/dist/testing/conformance/subscriptions/deploys.d.ts +0 -4
  238. package/dist/testing/conformance/subscriptions/deploys.js +0 -162
  239. package/dist/testing/conformance/subscriptions/epochs.d.ts +0 -4
  240. package/dist/testing/conformance/subscriptions/epochs.js +0 -169
  241. package/dist/testing/conformance/subscriptions/failures.d.ts +0 -4
  242. package/dist/testing/conformance/subscriptions/failures.js +0 -108
  243. package/dist/testing/conformance/subscriptions/feeds.d.ts +0 -4
  244. package/dist/testing/conformance/subscriptions/feeds.js +0 -218
  245. package/dist/testing/conformance/subscriptions/harness.d.ts +0 -124
  246. package/dist/testing/conformance/subscriptions/harness.js +0 -143
  247. package/dist/testing/conformance/subscriptions/operator.d.ts +0 -4
  248. package/dist/testing/conformance/subscriptions/operator.js +0 -128
  249. package/dist/testing/conformance/subscriptions/retention.d.ts +0 -4
  250. package/dist/testing/conformance/subscriptions/retention.js +0 -288
  251. package/dist/testing/conformance/subscriptions/routing.d.ts +0 -4
  252. package/dist/testing/conformance/subscriptions/routing.js +0 -179
  253. package/dist/testing/conformance/subscriptions.d.ts +0 -4
  254. package/dist/testing/conformance/subscriptions.js +0 -19
  255. package/dist/testing/conformance/tables.d.ts +0 -125
  256. package/dist/testing/conformance/tables.js +0 -664
  257. package/dist/testing/conformance/transports/actors.d.ts +0 -1907
  258. package/dist/testing/conformance/transports/actors.js +0 -163
  259. package/dist/testing/conformance/transports/client.d.ts +0 -3
  260. package/dist/testing/conformance/transports/client.js +0 -237
  261. package/dist/testing/conformance/transports/feeds.d.ts +0 -3
  262. package/dist/testing/conformance/transports/feeds.js +0 -232
  263. package/dist/testing/conformance/transports/harness.d.ts +0 -93
  264. package/dist/testing/conformance/transports/harness.js +0 -84
  265. package/dist/testing/conformance/transports/streams.d.ts +0 -3
  266. package/dist/testing/conformance/transports/streams.js +0 -96
  267. package/dist/testing/conformance/transports/websocket.d.ts +0 -3
  268. package/dist/testing/conformance/transports/websocket.js +0 -384
  269. package/dist/testing/conformance/transports/wire.d.ts +0 -141
  270. package/dist/testing/conformance/transports/wire.js +0 -204
  271. package/dist/testing/conformance/transports.d.ts +0 -3
  272. package/dist/testing/conformance/transports.js +0 -11
  273. package/dist/testing/conformance/watch.d.ts +0 -59
  274. package/dist/testing/conformance/watch.js +0 -879
  275. package/dist/testing/conformance/workflow-engine.d.ts +0 -97
  276. package/dist/testing/conformance/workflow-engine.js +0 -253
  277. package/dist/testing/conformance/workflow-versions.d.ts +0 -3
  278. package/dist/testing/conformance/workflow-versions.js +0 -545
  279. package/dist/testing/conformance/workflows/access.d.ts +0 -4
  280. package/dist/testing/conformance/workflows/access.js +0 -147
  281. package/dist/testing/conformance/workflows/actors.d.ts +0 -2317
  282. package/dist/testing/conformance/workflows/actors.js +0 -305
  283. package/dist/testing/conformance/workflows/basics.d.ts +0 -4
  284. package/dist/testing/conformance/workflows/basics.js +0 -144
  285. package/dist/testing/conformance/workflows/engine.d.ts +0 -4
  286. package/dist/testing/conformance/workflows/engine.js +0 -80
  287. package/dist/testing/conformance/workflows/guards.d.ts +0 -4
  288. package/dist/testing/conformance/workflows/guards.js +0 -172
  289. package/dist/testing/conformance/workflows/harness.d.ts +0 -14
  290. package/dist/testing/conformance/workflows/harness.js +0 -44
  291. package/dist/testing/conformance/workflows/recovery.d.ts +0 -4
  292. package/dist/testing/conformance/workflows/recovery.js +0 -181
  293. package/dist/testing/conformance/workflows/starts.d.ts +0 -4
  294. package/dist/testing/conformance/workflows/starts.js +0 -57
  295. package/dist/testing/conformance/workflows/suspension.d.ts +0 -4
  296. package/dist/testing/conformance/workflows/suspension.js +0 -106
  297. package/dist/testing/conformance/workflows/waits.d.ts +0 -4
  298. package/dist/testing/conformance/workflows/waits.js +0 -284
  299. package/dist/testing/conformance/workflows.d.ts +0 -4
  300. package/dist/testing/conformance/workflows.js +0 -19
  301. package/dist/testing/conformance.d.ts +0 -330
  302. package/dist/testing/conformance.js +0 -364
  303. package/dist/testing/foundation.d.ts +0 -41
  304. package/dist/testing/foundation.js +0 -517
  305. package/dist/testing/simulate-cluster.d.ts +0 -112
  306. package/dist/testing/simulate-cluster.js +0 -257
  307. package/dist/testing/simulate.d.ts +0 -90
  308. package/dist/testing/simulate.js +0 -127
@@ -1,257 +0,0 @@
1
- import { Cause, Config, Console, DateTime, Duration, Effect, Exit, Fiber, Option, Predicate, Random, Schedule, Schema, } from "effect";
2
- import { SqlError } from "effect/sql";
3
- import { Actor, Actors } from "../index.js";
4
- import { ActorError } from "../errors/actor.js";
5
- import { ActorTest } from "./actor-test.js";
6
- import { ActorCluster } from "./cluster.js";
7
- import { conclude, generator, SETTLE_ROUNDS } from "./simulate.js";
8
- const isActorError = Schema.is(ActorError);
9
- /**
10
- * Runs `program` on `cluster` under a fault schedule drawn from
11
- * `options.seed`, then checks that every command committed exactly one
12
- * receipt and no due or attempted outbox row was left undelivered. A failure
13
- * dies with the seed. Rerunning it repeats the program's choices and each
14
- * command's entry runner and fault; which runner owns an actor depends on
15
- * runner addresses, which change from run to run, and command ids are minted
16
- * fresh. Settling advances every runner's clock, so simulate once per
17
- * cluster after any runner has restarted: a runner that starts after the
18
- * settling has an earlier clock than the others, and a command id minted on a
19
- * runner ahead of it is in the future to it.
20
- */
21
- export const simulateCluster = (cluster) => (options, program) => Effect.gen(function* () {
22
- const draw = generator(options.seed);
23
- const rate = options.faultRate ?? 0.5;
24
- const attempt = Duration.fromInputUnsafe(options.attempt ?? "10 seconds");
25
- const within = Duration.fromInputUnsafe(options.within ?? "2 minutes");
26
- const settle = options.settle ?? "1 minute";
27
- const steps = [];
28
- const violations = [];
29
- const runners = Array.from({ length: cluster.runners }, (_, index) => index);
30
- const on = cluster.on;
31
- const planned = options.faults.includes("primaryFailover");
32
- const others = options.faults.filter((fault) => fault !== "primaryFailover");
33
- if (runners.length < 2)
34
- return yield* Effect.die(new Error("ActorTest.simulateCluster needs at least two runners"));
35
- if (planned && options.primary === undefined)
36
- return yield* Effect.die(new Error("primaryFailover needs the primary option"));
37
- if (planned && (options.commands === undefined || options.commands < 1))
38
- return yield* Effect.die(new Error("primaryFailover needs the number of commands"));
39
- const failoverAt = planned ? Math.floor(draw() * options.commands) : -1;
40
- const eachRunner = (effect) => Effect.forEach(runners, (runner) => on(runner)(effect));
41
- const clearFaults = ActorTest.use((test) => test.clearFaults);
42
- const clocks = (yield* eachRunner(ActorTest.use((test) => test.now))).map(DateTime.toEpochMillis);
43
- if (Math.max(...clocks) - Math.min(...clocks) > 1_000)
44
- return yield* Effect.die(new Error("ActorTest.simulateCluster needs runners whose clocks agree; an earlier simulation settled some runners' clocks ahead of a restarted one, so simulate once per cluster"));
45
- /** The cluster's own pool reconnects through a failover like any other, so a read that meets one is retried. */
46
- const read = (query) => query.pipe(Effect.retry({ schedule: Schedule.spaced("200 millis"), times: 50 }), Effect.orDie);
47
- /**
48
- * Minting a command id reads the database clock, which a failover
49
- * briefly takes away, so it is retried. A crash fault is queued on every
50
- * runner, and one of them takes it; if all of them still hold theirs,
51
- * the turn never reached its point.
52
- */
53
- const command = (label, effect) => Effect.gen(function* () {
54
- const fault = steps.length === failoverAt
55
- ? "primaryFailover"
56
- : others.length === 0 || draw() >= rate
57
- ? "none"
58
- : others[Math.floor(draw() * others.length)];
59
- const entry = Math.floor(draw() * runners.length);
60
- const point = fault === "crashAfterCommit" || (fault === "runnerKill" && draw() < 0.5)
61
- ? "afterCommit"
62
- : "beforeCommit";
63
- const landed = (fault === "connectionLoss" || fault === "primaryFailover") && draw() < 0.5;
64
- const commandId = yield* on(entry)(Effect.gen(function* () {
65
- return yield* (yield* Actors).mintCommandId;
66
- })).pipe(Effect.retry({
67
- while: (error) => error.isRetryable,
68
- schedule: Schedule.spaced("200 millis"),
69
- times: 100,
70
- }), Effect.orDie);
71
- steps.push({ label, commandId, fault, runner: entry, point, landed });
72
- const call = effect.pipe(Actor.commandId(commandId));
73
- const send = (runner) => on(runner)(call).pipe(Effect.timeout(attempt));
74
- /**
75
- * Sends the same id through each runner in turn, skipping `avoid`,
76
- * until a receipt answers; only the command's own declared failures end it.
77
- */
78
- const deliver = (from, avoid) => {
79
- const order = runners.filter((runner) => runner !== avoid);
80
- const start = Math.max(order.indexOf(from), 0);
81
- return Effect.gen(function* () {
82
- for (let round = 0;; round++) {
83
- const exit = yield* Effect.exit(send(order[(start + round) % order.length]));
84
- if (Exit.isSuccess(exit))
85
- return exit.value;
86
- const error = Cause.findErrorOption(exit.cause);
87
- const resendable = Option.isNone(error) ||
88
- Predicate.isTagged(error.value, "TimeoutError") ||
89
- (isActorError(error.value) && error.value.isRetryable);
90
- if (!resendable)
91
- return yield* Effect.failCause(exit.cause);
92
- yield* Effect.sleep("100 millis");
93
- }
94
- });
95
- };
96
- /**
97
- * Whichever runner runs this command's turn reaches its pause, so
98
- * every runner holds one and the others' are cleared once the owner is known.
99
- */
100
- const holdTurn = Effect.gen(function* () {
101
- const paused = yield* Effect.forEach(runners, (runner) => on(runner)(ActorTest.use((test) => test.pauseNext(point, { commandId }))));
102
- const first = yield* Effect.forkChild(Effect.exit(send(entry)));
103
- const owner = yield* Effect.raceAll(paused.map((pause, runner) => Effect.as(pause.reached, runner))).pipe(Effect.timeoutOrElse({
104
- duration: within,
105
- orElse: () => Effect.die(new Error(`${label} (${commandId}) never reached ${point}`)),
106
- }));
107
- yield* Effect.forEach(runners.filter((runner) => runner !== owner), (runner) => on(runner)(clearFaults));
108
- return { owner, first, release: paused[owner].release };
109
- });
110
- const survivor = (owner) => entry === owner ? (owner + 1) % runners.length : entry;
111
- if (fault === "none")
112
- return yield* deliver(entry);
113
- if (fault === "dropReply") {
114
- yield* Effect.exit(send(entry));
115
- return yield* deliver(entry);
116
- }
117
- if (fault === "crashBeforeCommit" || fault === "crashAfterCommit") {
118
- yield* eachRunner(ActorTest.use((test) => test.crashNext(point, { commandId })));
119
- const result = yield* deliver(entry);
120
- const left = (yield* eachRunner(clearFaults)).flat();
121
- if (left.length === runners.length)
122
- violations.push(`${label} (${commandId}) never reached its ${point} crash`);
123
- return result;
124
- }
125
- if (fault === "runnerKill") {
126
- const { owner, first, release } = yield* holdTurn;
127
- yield* cluster.kill(owner);
128
- yield* release;
129
- const result = yield* deliver(survivor(owner), owner);
130
- yield* Fiber.interrupt(first);
131
- yield* cluster.restart(owner);
132
- yield* cluster.ready;
133
- return result;
134
- }
135
- if (fault === "heartbeatLoss") {
136
- const { owner, first, release } = yield* holdTurn;
137
- const heartbeat = yield* cluster.pauseHeartbeat(owner);
138
- const result = yield* deliver(survivor(owner), owner);
139
- yield* release;
140
- yield* Fiber.join(first);
141
- yield* heartbeat.resume;
142
- yield* cluster.ready;
143
- return result;
144
- }
145
- const { first, release } = yield* holdTurn;
146
- if (!landed) {
147
- yield* cluster.failover;
148
- yield* release;
149
- }
150
- else {
151
- yield* cluster.holdReplies;
152
- yield* release;
153
- yield* read(cluster.sql `
154
- SELECT count(*)::integer AS count FROM actor_receipts
155
- WHERE tenant_id = ${cluster.tenant} AND command_id = ${commandId}`).pipe(Effect.repeat({
156
- until: ([receipts]) => receipts.count > 0,
157
- schedule: Schedule.spaced("20 millis"),
158
- }), Effect.timeoutOrElse({
159
- duration: within,
160
- orElse: () => Effect.die(new Error(`${label} (${commandId}) never committed`)),
161
- }));
162
- yield* cluster.failover;
163
- }
164
- if (fault === "primaryFailover")
165
- yield* options.primary;
166
- const settled = yield* Fiber.join(first);
167
- return Exit.isSuccess(settled) ? settled.value : yield* deliver(entry);
168
- }).pipe(Effect.timeoutOrElse({
169
- duration: within,
170
- orElse: () => Effect.die(new Error(`${label} did not commit within ${Duration.format(within)}`)),
171
- }));
172
- const simulation = {
173
- seed: options.seed,
174
- int: (min, max) => Effect.sync(() => min + Math.floor(draw() * (max - min + 1))),
175
- pick: (values) => Effect.sync(() => values[Math.floor(draw() * values.length)]),
176
- command,
177
- };
178
- const run = Effect.gen(function* () {
179
- yield* program(simulation);
180
- if (steps.length === 0)
181
- violations.push("the program sent no command");
182
- if (planned && steps.length <= failoverAt)
183
- violations.push(`the primary failover planned for command ${failoverAt} never happened`);
184
- /**
185
- * A row that was ever attempted is still in flight, while one never
186
- * attempted and not yet due is a timer the program scheduled.
187
- */
188
- const pending = read(Effect.gen(function* () {
189
- const now = DateTime.toEpochMillis(yield* on(0)(ActorTest.use((test) => test.now)));
190
- const [rows] = yield* cluster.sql `
191
- SELECT count(*)::integer AS count FROM actor_outbox
192
- WHERE tenant_id = ${cluster.tenant} AND (due_at_ms <= ${now} OR attempts > 0)`;
193
- return rows.count;
194
- }));
195
- let unsettled = Number.POSITIVE_INFINITY;
196
- for (let round = 0; round < SETTLE_ROUNDS && unsettled > 0; round++) {
197
- yield* eachRunner(ActorTest.use((test) => test.advance(settle)));
198
- unsettled = yield* pending.pipe(Effect.repeat({
199
- until: (count) => count === 0,
200
- schedule: Schedule.spaced("100 millis"),
201
- }), Effect.timeoutOrElse({ duration: "5 seconds", orElse: () => pending }));
202
- }
203
- if (unsettled > 0)
204
- violations.push(`${unsettled} due or attempted outbox rows were never delivered`);
205
- const left = (yield* eachRunner(clearFaults)).flat();
206
- if (left.length > 0)
207
- violations.push(`faults at ${left.join(", ")} were never reached`);
208
- for (const step of steps) {
209
- const [receipts] = yield* read(cluster.sql `
210
- SELECT count(*)::integer AS count FROM actor_receipts
211
- WHERE tenant_id = ${cluster.tenant} AND command_id = ${step.commandId}`);
212
- if (receipts.count !== 1)
213
- violations.push(`${step.label} (${step.commandId}) has ${receipts.count} receipts`);
214
- }
215
- });
216
- return yield* conclude({
217
- seed: options.seed,
218
- steps,
219
- violations,
220
- run,
221
- cleanup: eachRunner(clearFaults).pipe(Effect.ignore),
222
- });
223
- });
224
- /** Seeds pull requests run on a cluster: `0` and `1`. */
225
- export const CLUSTER_SIMULATION_SEEDS = 2;
226
- /** Seeds the nightly run covers on a cluster: about a minute each. */
227
- export const NIGHTLY_CLUSTER_SIMULATION_SEEDS = 6;
228
- /** Seeds pull requests run through a real primary failover, each on a primary and standby of its own: `0`. */
229
- export const FAILOVER_SIMULATION_SEEDS = 1;
230
- /** Seeds the nightly run puts through a real primary failover: under a minute each. */
231
- export const NIGHTLY_FAILOVER_SIMULATION_SEEDS = 3;
232
- const seedsOf = (countName, pullRequest, nightly) => Effect.gen(function* () {
233
- const property = yield* Config.String("PROPERTY_SEED").pipe(Config.withDefault(""));
234
- const random = property === "random";
235
- const count = yield* Config.Int(countName).pipe(Config.withDefault(random ? nightly : pullRequest));
236
- const configured = yield* Config.Int("SIMULATION_SEED_BASE").pipe(Config.option);
237
- const base = Option.isSome(configured)
238
- ? configured.value
239
- : random
240
- ? yield* Random.nextIntBetween(0, 2 ** 31 - 1 - count)
241
- : 0;
242
- if (random && Option.isNone(configured))
243
- yield* Console.error(`${countName} seeds from ${base}`);
244
- return Array.from({ length: count }, (_, index) => String(base + index));
245
- }).pipe(Effect.orDie);
246
- /**
247
- * The seeds this run simulates on a cluster: `CLUSTER_SIMULATION_SEEDS` of
248
- * them counted up from `SIMULATION_SEED_BASE`. A run whose `PROPERTY_SEED` is
249
- * `random`, as the nightly run's is, defaults to
250
- * `NIGHTLY_CLUSTER_SIMULATION_SEEDS` seeds from a base it draws and prints,
251
- * so each night covers seeds no other night did; every other run defaults to
252
- * `CLUSTER_SIMULATION_SEEDS` from 0. A failure names its seed; rerun it with
253
- * `CLUSTER_SIMULATION_SEEDS=1 SIMULATION_SEED_BASE=<seed>`.
254
- */
255
- export const clusterSimulationSeeds = seedsOf("CLUSTER_SIMULATION_SEEDS", CLUSTER_SIMULATION_SEEDS, NIGHTLY_CLUSTER_SIMULATION_SEEDS);
256
- /** The seeds for the failover drill, chosen like `clusterSimulationSeeds` from `FAILOVER_SIMULATION_SEEDS`. */
257
- export const failoverSimulationSeeds = seedsOf("FAILOVER_SIMULATION_SEEDS", FAILOVER_SIMULATION_SEEDS, NIGHTLY_FAILOVER_SIMULATION_SEEDS);
@@ -1,90 +0,0 @@
1
- import { Duration, Effect } from "effect";
2
- import { SqlClient } from "effect/sql";
3
- import { Actors } from "../index.ts";
4
- import type { ActorTest } from "./actor-test.ts";
5
- /**
6
- * A fault `ActorTest.simulate` injects into one command:
7
- * - `crashBeforeCommit`: the turn dies before COMMIT, so it rolls back and the handle retries.
8
- * - `crashAfterCommit`: the turn commits and the reply is lost, so the retry reads the receipt.
9
- * - `dropReply`: the caller discards the reply it got and sends the same command id again.
10
- * - `relayCrash`: the next relay delivery dies before deleting its outbox row, so the row
11
- * is redelivered once its claim lease ends. Drawn only for commands marked `relays`.
12
- * - `clockSkew`: the framework clock jumps ahead of the database clock before the command.
13
- */
14
- export type SimulationFault = "crashBeforeCommit" | "crashAfterCommit" | "dropReply" | "relayCrash" | "clockSkew";
15
- export interface SimulationOptions {
16
- /** Every draw, and so the whole fault schedule, follows from it. */
17
- readonly seed: string;
18
- readonly faults: ReadonlyArray<SimulationFault>;
19
- /** Chance that a command gets a fault. Default 0.5. */
20
- readonly faultRate?: number;
21
- /** Largest `clockSkew` jump. Default 5 minutes. */
22
- readonly maxSkew?: Duration.Input;
23
- /** How far the clock moves to settle outstanding deliveries. Default 1 minute per round. */
24
- readonly settle?: Duration.Input;
25
- }
26
- export interface SimulationStep {
27
- readonly label: string;
28
- readonly commandId: string;
29
- readonly fault: SimulationFault | "none";
30
- /** The `clockSkew` jump in milliseconds, 0 otherwise. */
31
- readonly skewMs: number;
32
- }
33
- export interface SimulationReport {
34
- readonly seed: string;
35
- readonly steps: ReadonlyArray<SimulationStep>;
36
- }
37
- export interface Simulation {
38
- readonly seed: string;
39
- /**
40
- * Runs one command under the seed's next fault, with a command id minted
41
- * for it. `effect` must be a single command call, so it commits one receipt.
42
- * `relays` marks a command whose turn stages an intent, so a relay crash
43
- * drawn for it must be reached by that intent's delivery.
44
- */
45
- readonly command: <A, E, R>(label: string, effect: Effect.Effect<A, E, R>, options?: {
46
- readonly relays?: boolean;
47
- }) => Effect.Effect<A, E, R>;
48
- /** A seeded integer in `[min, max]`. */
49
- readonly int: (min: number, max: number) => Effect.Effect<number>;
50
- /** A seeded element of a non-empty list. */
51
- readonly pick: <A>(values: readonly [A, ...Array<A>]) => Effect.Effect<A>;
52
- }
53
- /** A small seeded generator, so the schedule never depends on the runtime's own randomness. */
54
- export declare const generator: (seed: string) => () => number;
55
- /** Settling rounds before undelivered outbox rows count as lost. */
56
- export declare const SETTLE_ROUNDS = 5;
57
- /**
58
- * Runs a simulation's `run`, then reports it: a defect or a violation dies
59
- * with the seed and the schedule drawn, and `cleanup` runs however `run` ends.
60
- */
61
- export declare const conclude: <Step extends {
62
- readonly label: string;
63
- readonly fault: string;
64
- }, E, R>({ seed, steps, violations, run, cleanup, }: {
65
- readonly seed: string;
66
- readonly steps: ReadonlyArray<Step>;
67
- readonly violations: ReadonlyArray<string>;
68
- readonly run: Effect.Effect<void, E, R>;
69
- readonly cleanup: Effect.Effect<unknown>;
70
- }) => Effect.Effect<{
71
- readonly seed: string;
72
- readonly steps: ReadonlyArray<Step>;
73
- }, never, R>;
74
- /**
75
- * Runs `program` against `test`'s runtime under a fault
76
- * schedule drawn from `options.seed`, then settles the relay and checks that
77
- * every command committed exactly one receipt and no due or attempted outbox
78
- * row was left undelivered. Any failure dies with the seed, so rerunning that
79
- * seed repeats the program's choices and the fault on each step; command ids
80
- * are minted fresh on every run. A crashed delivery pushes its row's due time to the end of its claim lease, so an attempted row counts as in flight while an unattempted row that is not yet due is a timer the program scheduled.
81
- */
82
- export declare const simulate: (test: ActorTest["Service"]) => <E, R>(options: SimulationOptions, program: (simulation: Simulation) => Effect.Effect<void, E, R>) => Effect.Effect<SimulationReport, never, R | Actors | SqlClient.SqlClient>;
83
- /** Seeds pull requests run: `0` to `19`. */
84
- export declare const SIMULATION_SEEDS = 20;
85
- /**
86
- * The seeds this run simulates: `SIMULATION_SEEDS` of them (default 20)
87
- * counted up from `SIMULATION_SEED_BASE` (default 0), so a nightly run sets
88
- * both and a failure names the seed to rerun.
89
- */
90
- export declare const simulationSeeds: Effect.Effect<string[], never, never>;
@@ -1,127 +0,0 @@
1
- import { Cause, Config, DateTime, Duration, Effect } from "effect";
2
- import { SqlClient } from "effect/sql";
3
- import { Actor, Actors } from "../index.js";
4
- /** A small seeded generator, so the schedule never depends on the runtime's own randomness. */
5
- export const generator = (seed) => {
6
- let state = 2166136261;
7
- for (const char of seed)
8
- state = Math.imul(state ^ char.charCodeAt(0), 16777619);
9
- return () => {
10
- state = (state + 0x6d2b79f5) | 0;
11
- let mixed = Math.imul(state ^ (state >>> 15), 1 | state);
12
- mixed = (mixed + Math.imul(mixed ^ (mixed >>> 7), 61 | mixed)) ^ mixed;
13
- return ((mixed ^ (mixed >>> 14)) >>> 0) / 4294967296;
14
- };
15
- };
16
- const crashPoint = {
17
- crashBeforeCommit: "beforeCommit",
18
- crashAfterCommit: "afterCommit",
19
- relayCrash: "beforeOutboxDelete",
20
- };
21
- /** Settling rounds before undelivered outbox rows count as lost. */
22
- export const SETTLE_ROUNDS = 5;
23
- const schedule = (steps) => steps.map(({ label, fault }) => `${label}:${fault}`).join(" ");
24
- /**
25
- * Runs a simulation's `run`, then reports it: a defect or a violation dies
26
- * with the seed and the schedule drawn, and `cleanup` runs however `run` ends.
27
- */
28
- export const conclude = ({ seed, steps, violations, run, cleanup, }) => Effect.gen(function* () {
29
- yield* run.pipe(Effect.ensuring(cleanup), Effect.catchCause((cause) => Effect.die(new Error(`Simulation failed with seed ${seed} after ${steps.length} steps\n${Cause.pretty(cause)}\nschedule: ${schedule(steps)}`))));
30
- if (violations.length > 0)
31
- return yield* Effect.die(new Error(`Simulation failed with seed ${seed}\n${violations.join("\n")}\nschedule: ${schedule(steps)}`));
32
- return { seed, steps };
33
- });
34
- /**
35
- * Runs `program` against `test`'s runtime under a fault
36
- * schedule drawn from `options.seed`, then settles the relay and checks that
37
- * every command committed exactly one receipt and no due or attempted outbox
38
- * row was left undelivered. Any failure dies with the seed, so rerunning that
39
- * seed repeats the program's choices and the fault on each step; command ids
40
- * are minted fresh on every run. A crashed delivery pushes its row's due time to the end of its claim lease, so an attempted row counts as in flight while an unattempted row that is not yet due is a timer the program scheduled.
41
- */
42
- export const simulate = (test) => (options, program) => Effect.gen(function* () {
43
- const actors = yield* Actors;
44
- const sql = yield* SqlClient.SqlClient;
45
- const draw = generator(options.seed);
46
- const rate = options.faultRate ?? 0.5;
47
- const maxSkewMs = Duration.toMillis(Duration.fromInputUnsafe(options.maxSkew ?? "5 minutes"));
48
- const settle = options.settle ?? "1 minute";
49
- const steps = [];
50
- const int = (min, max) => Effect.sync(() => min + Math.floor(draw() * (max - min + 1)));
51
- const direct = options.faults.filter((fault) => fault !== "relayCrash");
52
- const nextFault = (relays) => Effect.sync(() => {
53
- const faults = relays ? options.faults : direct;
54
- if (faults.length === 0 || draw() >= rate)
55
- return "none";
56
- return faults[Math.floor(draw() * faults.length)];
57
- });
58
- const simulation = {
59
- seed: options.seed,
60
- int,
61
- pick: (values) => Effect.sync(() => values[Math.floor(draw() * values.length)]),
62
- command: (label, effect, commandOptions) => Effect.gen(function* () {
63
- const fault = yield* nextFault(commandOptions?.relays === true);
64
- const skewMs = fault === "clockSkew" ? Math.floor(draw() * maxSkewMs) : 0;
65
- const commandId = yield* actors.mintCommandId.pipe(Effect.orDie);
66
- steps.push({ label, commandId, fault, skewMs });
67
- const call = effect.pipe(Actor.commandId(commandId));
68
- const point = crashPoint[fault];
69
- if (point !== undefined)
70
- yield* test.crashNext(point);
71
- if (skewMs > 0)
72
- yield* test.advance(skewMs);
73
- if (fault === "dropReply")
74
- yield* Effect.exit(call);
75
- return yield* call;
76
- }),
77
- };
78
- const violations = [];
79
- const run = Effect.gen(function* () {
80
- yield* program(simulation);
81
- if (steps.length === 0)
82
- violations.push("the program sent no command");
83
- let unsettled = Number.POSITIVE_INFINITY;
84
- for (let round = 0; round < SETTLE_ROUNDS && unsettled > 0; round++) {
85
- yield* test.advance(settle);
86
- const now = DateTime.toEpochMillis(yield* test.now);
87
- const [pending] = yield* sql `
88
- SELECT count(*)::integer AS count FROM actor_outbox
89
- WHERE tenant_id = ${test.tenant} AND (due_at_ms <= ${now} OR attempts > 0)`;
90
- unsettled = pending.count;
91
- }
92
- if (unsettled > 0)
93
- violations.push(`${unsettled} due or attempted outbox rows were never delivered`);
94
- const left = yield* test.clearFaults;
95
- if (left.length > 0)
96
- violations.push(`faults at ${left.join(", ")} were never reached`);
97
- for (const step of steps) {
98
- const [receipts] = yield* sql `
99
- SELECT count(*)::integer AS count FROM actor_receipts
100
- WHERE tenant_id = ${test.tenant} AND command_id = ${step.commandId}`;
101
- if (receipts.count !== 1)
102
- violations.push(`${step.label} (${step.commandId}) has ${receipts.count} receipts`);
103
- }
104
- });
105
- const queued = yield* test.clearFaults;
106
- if (queued.length > 0)
107
- return yield* Effect.die(new Error(`ActorTest.simulate injects every fault itself; faults at ${queued.join(", ")} were already queued`));
108
- return yield* conclude({
109
- seed: options.seed,
110
- steps,
111
- violations,
112
- run,
113
- cleanup: test.clearFaults,
114
- });
115
- });
116
- /** Seeds pull requests run: `0` to `19`. */
117
- export const SIMULATION_SEEDS = 20;
118
- /**
119
- * The seeds this run simulates: `SIMULATION_SEEDS` of them (default 20)
120
- * counted up from `SIMULATION_SEED_BASE` (default 0), so a nightly run sets
121
- * both and a failure names the seed to rerun.
122
- */
123
- export const simulationSeeds = Effect.gen(function* () {
124
- const count = yield* Config.Int("SIMULATION_SEEDS").pipe(Config.withDefault(SIMULATION_SEEDS));
125
- const base = yield* Config.Int("SIMULATION_SEED_BASE").pipe(Config.withDefault(0));
126
- return Array.from({ length: count }, (_, index) => String(base + index));
127
- }).pipe(Effect.orDie);