@effect/cluster 0.37.2 → 0.38.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ClusterCron/package.json +6 -0
- package/dist/cjs/ClusterCron.js +86 -0
- package/dist/cjs/ClusterCron.js.map +1 -0
- package/dist/cjs/ClusterSchema.js +9 -1
- package/dist/cjs/ClusterSchema.js.map +1 -1
- package/dist/cjs/ClusterWorkflowEngine.js +21 -6
- package/dist/cjs/ClusterWorkflowEngine.js.map +1 -1
- package/dist/cjs/Entity.js +6 -1
- package/dist/cjs/Entity.js.map +1 -1
- package/dist/cjs/EntityAddress.js +8 -1
- package/dist/cjs/EntityAddress.js.map +1 -1
- package/dist/cjs/MessageStorage.js +6 -4
- package/dist/cjs/MessageStorage.js.map +1 -1
- package/dist/cjs/Runner.js +15 -0
- package/dist/cjs/Runner.js.map +1 -1
- package/dist/cjs/RunnerAddress.js +8 -1
- package/dist/cjs/RunnerAddress.js.map +1 -1
- package/dist/cjs/Runners.js +5 -0
- package/dist/cjs/Runners.js.map +1 -1
- package/dist/cjs/ShardId.js +75 -7
- package/dist/cjs/ShardId.js.map +1 -1
- package/dist/cjs/ShardManager.js +63 -43
- package/dist/cjs/ShardManager.js.map +1 -1
- package/dist/cjs/ShardStorage.js +45 -36
- package/dist/cjs/ShardStorage.js.map +1 -1
- package/dist/cjs/Sharding.js +45 -37
- package/dist/cjs/Sharding.js.map +1 -1
- package/dist/cjs/ShardingConfig.js +9 -2
- package/dist/cjs/ShardingConfig.js.map +1 -1
- package/dist/cjs/Singleton.js +2 -2
- package/dist/cjs/Singleton.js.map +1 -1
- package/dist/cjs/SingletonAddress.js +2 -2
- package/dist/cjs/SingletonAddress.js.map +1 -1
- package/dist/cjs/SqlMessageStorage.js +32 -27
- package/dist/cjs/SqlMessageStorage.js.map +1 -1
- package/dist/cjs/SqlShardStorage.js +14 -14
- package/dist/cjs/SqlShardStorage.js.map +1 -1
- package/dist/cjs/index.js +3 -1
- package/dist/cjs/internal/entityManager.js +2 -1
- package/dist/cjs/internal/entityManager.js.map +1 -1
- package/dist/cjs/internal/shardManager.js +138 -37
- package/dist/cjs/internal/shardManager.js.map +1 -1
- package/dist/dts/ClusterCron.d.ts +37 -0
- package/dist/dts/ClusterCron.d.ts.map +1 -0
- package/dist/dts/ClusterSchema.d.ts +8 -0
- package/dist/dts/ClusterSchema.d.ts.map +1 -1
- package/dist/dts/ClusterWorkflowEngine.d.ts.map +1 -1
- package/dist/dts/Entity.d.ts +10 -0
- package/dist/dts/Entity.d.ts.map +1 -1
- package/dist/dts/EntityAddress.d.ts +9 -3
- package/dist/dts/EntityAddress.d.ts.map +1 -1
- package/dist/dts/MessageStorage.d.ts +3 -3
- package/dist/dts/MessageStorage.d.ts.map +1 -1
- package/dist/dts/Runner.d.ts +15 -0
- package/dist/dts/Runner.d.ts.map +1 -1
- package/dist/dts/RunnerAddress.d.ts +5 -0
- package/dist/dts/RunnerAddress.d.ts.map +1 -1
- package/dist/dts/Runners.d.ts.map +1 -1
- package/dist/dts/ShardId.d.ts +60 -6
- package/dist/dts/ShardId.d.ts.map +1 -1
- package/dist/dts/ShardManager.d.ts +13 -13
- package/dist/dts/ShardManager.d.ts.map +1 -1
- package/dist/dts/ShardStorage.d.ts +11 -14
- package/dist/dts/ShardStorage.d.ts.map +1 -1
- package/dist/dts/Sharding.d.ts +4 -2
- package/dist/dts/Sharding.d.ts.map +1 -1
- package/dist/dts/ShardingConfig.d.ts +32 -6
- package/dist/dts/ShardingConfig.d.ts.map +1 -1
- package/dist/dts/Singleton.d.ts +3 -1
- package/dist/dts/Singleton.d.ts.map +1 -1
- package/dist/dts/SingletonAddress.d.ts +4 -3
- package/dist/dts/SingletonAddress.d.ts.map +1 -1
- package/dist/dts/SqlMessageStorage.d.ts +3 -2
- package/dist/dts/SqlMessageStorage.d.ts.map +1 -1
- package/dist/dts/SqlShardStorage.d.ts +1 -1
- package/dist/dts/index.d.ts +4 -0
- package/dist/dts/index.d.ts.map +1 -1
- package/dist/esm/ClusterCron.js +77 -0
- package/dist/esm/ClusterCron.js.map +1 -0
- package/dist/esm/ClusterSchema.js +7 -0
- package/dist/esm/ClusterSchema.js.map +1 -1
- package/dist/esm/ClusterWorkflowEngine.js +21 -6
- package/dist/esm/ClusterWorkflowEngine.js.map +1 -1
- package/dist/esm/Entity.js +6 -1
- package/dist/esm/Entity.js.map +1 -1
- package/dist/esm/EntityAddress.js +8 -1
- package/dist/esm/EntityAddress.js.map +1 -1
- package/dist/esm/MessageStorage.js +6 -4
- package/dist/esm/MessageStorage.js.map +1 -1
- package/dist/esm/Runner.js +15 -0
- package/dist/esm/Runner.js.map +1 -1
- package/dist/esm/RunnerAddress.js +8 -1
- package/dist/esm/RunnerAddress.js.map +1 -1
- package/dist/esm/Runners.js +5 -0
- package/dist/esm/Runners.js.map +1 -1
- package/dist/esm/ShardId.js +73 -6
- package/dist/esm/ShardId.js.map +1 -1
- package/dist/esm/ShardManager.js +64 -45
- package/dist/esm/ShardManager.js.map +1 -1
- package/dist/esm/ShardStorage.js +44 -36
- package/dist/esm/ShardStorage.js.map +1 -1
- package/dist/esm/Sharding.js +45 -37
- package/dist/esm/Sharding.js.map +1 -1
- package/dist/esm/ShardingConfig.js +9 -2
- package/dist/esm/ShardingConfig.js.map +1 -1
- package/dist/esm/Singleton.js +2 -2
- package/dist/esm/Singleton.js.map +1 -1
- package/dist/esm/SingletonAddress.js +2 -2
- package/dist/esm/SingletonAddress.js.map +1 -1
- package/dist/esm/SqlMessageStorage.js +32 -27
- package/dist/esm/SqlMessageStorage.js.map +1 -1
- package/dist/esm/SqlShardStorage.js +14 -14
- package/dist/esm/SqlShardStorage.js.map +1 -1
- package/dist/esm/index.js +4 -0
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/internal/entityManager.js +2 -1
- package/dist/esm/internal/entityManager.js.map +1 -1
- package/dist/esm/internal/shardManager.js +136 -36
- package/dist/esm/internal/shardManager.js.map +1 -1
- package/package.json +12 -4
- package/src/ClusterCron.ts +129 -0
- package/src/ClusterSchema.ts +9 -0
- package/src/ClusterWorkflowEngine.ts +37 -6
- package/src/Entity.ts +20 -1
- package/src/EntityAddress.ts +11 -1
- package/src/MessageStorage.ts +12 -7
- package/src/Runner.ts +18 -0
- package/src/RunnerAddress.ts +9 -1
- package/src/Runners.ts +5 -0
- package/src/ShardId.ts +81 -11
- package/src/ShardManager.ts +74 -45
- package/src/ShardStorage.ts +51 -47
- package/src/Sharding.ts +45 -39
- package/src/ShardingConfig.ts +36 -7
- package/src/Singleton.ts +5 -2
- package/src/SingletonAddress.ts +2 -2
- package/src/SqlMessageStorage.ts +36 -30
- package/src/SqlShardStorage.ts +15 -15
- package/src/index.ts +5 -0
- package/src/internal/entityManager.ts +2 -1
- package/src/internal/shardManager.ts +158 -52
package/package.json
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
{
|
2
2
|
"name": "@effect/cluster",
|
3
|
-
"version": "0.
|
3
|
+
"version": "0.38.0",
|
4
4
|
"description": "Unified interfaces for common cluster-specific services",
|
5
5
|
"license": "MIT",
|
6
6
|
"repository": {
|
@@ -11,9 +11,9 @@
|
|
11
11
|
"sideEffects": [],
|
12
12
|
"homepage": "https://effect.website",
|
13
13
|
"peerDependencies": {
|
14
|
-
"@effect/platform": "^0.84.
|
15
|
-
"@effect/rpc": "^0.61.
|
16
|
-
"@effect/sql": "^0.37.
|
14
|
+
"@effect/platform": "^0.84.5",
|
15
|
+
"@effect/rpc": "^0.61.5",
|
16
|
+
"@effect/sql": "^0.37.5",
|
17
17
|
"@effect/workflow": "^0.1.2",
|
18
18
|
"effect": "^3.16.3"
|
19
19
|
},
|
@@ -30,6 +30,11 @@
|
|
30
30
|
"import": "./dist/esm/index.js",
|
31
31
|
"default": "./dist/cjs/index.js"
|
32
32
|
},
|
33
|
+
"./ClusterCron": {
|
34
|
+
"types": "./dist/dts/ClusterCron.d.ts",
|
35
|
+
"import": "./dist/esm/ClusterCron.js",
|
36
|
+
"default": "./dist/cjs/ClusterCron.js"
|
37
|
+
},
|
33
38
|
"./ClusterError": {
|
34
39
|
"types": "./dist/dts/ClusterError.d.ts",
|
35
40
|
"import": "./dist/esm/ClusterError.js",
|
@@ -218,6 +223,9 @@
|
|
218
223
|
},
|
219
224
|
"typesVersions": {
|
220
225
|
"*": {
|
226
|
+
"ClusterCron": [
|
227
|
+
"./dist/dts/ClusterCron.d.ts"
|
228
|
+
],
|
221
229
|
"ClusterError": [
|
222
230
|
"./dist/dts/ClusterError.d.ts"
|
223
231
|
],
|
@@ -0,0 +1,129 @@
|
|
1
|
+
/**
|
2
|
+
* @since 1.0.0
|
3
|
+
*/
|
4
|
+
import * as Rpc from "@effect/rpc/Rpc"
|
5
|
+
import * as Cron from "effect/Cron"
|
6
|
+
import * as DateTime from "effect/DateTime"
|
7
|
+
import * as Duration from "effect/Duration"
|
8
|
+
import * as Effect from "effect/Effect"
|
9
|
+
import * as Layer from "effect/Layer"
|
10
|
+
import * as Option from "effect/Option"
|
11
|
+
import * as PrimaryKey from "effect/PrimaryKey"
|
12
|
+
import * as Schedule from "effect/Schedule"
|
13
|
+
import * as Schema from "effect/Schema"
|
14
|
+
import type { Scope } from "effect/Scope"
|
15
|
+
import * as ClusterSchema from "./ClusterSchema.js"
|
16
|
+
import { Persisted, Uninterruptible } from "./ClusterSchema.js"
|
17
|
+
import * as DeliverAt from "./DeliverAt.js"
|
18
|
+
import * as Entity from "./Entity.js"
|
19
|
+
import type { Sharding } from "./Sharding.js"
|
20
|
+
import * as Singleton from "./Singleton.js"
|
21
|
+
|
22
|
+
/**
|
23
|
+
* @since 1.0.0
|
24
|
+
* @category Constructors
|
25
|
+
*/
|
26
|
+
export const make = <E, R>(options: {
|
27
|
+
readonly name: string
|
28
|
+
readonly cron: Cron.Cron
|
29
|
+
readonly execute: Effect.Effect<void, E, R>
|
30
|
+
|
31
|
+
/**
|
32
|
+
* Choose a shard group to run this cron job on.
|
33
|
+
*/
|
34
|
+
readonly shardGroup?: string | undefined
|
35
|
+
|
36
|
+
/**
|
37
|
+
* Whether to run the next cron job based from the time of the previous run.
|
38
|
+
*
|
39
|
+
* Defaults to `false`, meaning the next run will be calculated from the
|
40
|
+
* current time.
|
41
|
+
*/
|
42
|
+
readonly calculateNextRunFromPrevious?: boolean | undefined
|
43
|
+
|
44
|
+
/**
|
45
|
+
* If set, the cron job will skip execution if the scheduled time is older
|
46
|
+
* than this duration.
|
47
|
+
*
|
48
|
+
* This is useful to prevent running jobs that were scheduled too far in the
|
49
|
+
* past.
|
50
|
+
*
|
51
|
+
* Defaults to "1 day".
|
52
|
+
*/
|
53
|
+
readonly skipIfOlderThan?: Duration.DurationInput | undefined
|
54
|
+
}): Layer.Layer<never, never, Sharding | Exclude<R, Scope>> => {
|
55
|
+
const CronEntity = Entity.make(`ClusterCron/${options.name}`, [
|
56
|
+
Rpc.make("run", {
|
57
|
+
payload: CronPayload
|
58
|
+
})
|
59
|
+
.annotate(Persisted, true)
|
60
|
+
.annotate(Uninterruptible, true)
|
61
|
+
]).annotate(ClusterSchema.ShardGroup, () => options.shardGroup ?? "default")
|
62
|
+
|
63
|
+
const InitialRun = Singleton.make(
|
64
|
+
`ClusterCron/${options.name}`,
|
65
|
+
Effect.gen(function*() {
|
66
|
+
const client = (yield* CronEntity.client)("initial")
|
67
|
+
const now = yield* DateTime.now
|
68
|
+
const next = Cron.next(options.cron, now)
|
69
|
+
yield* client.run({
|
70
|
+
dateTime: DateTime.unsafeFromDate(next)
|
71
|
+
}, { discard: true })
|
72
|
+
}),
|
73
|
+
{ shardGroup: options.shardGroup }
|
74
|
+
)
|
75
|
+
|
76
|
+
const skipIfOlderThan = Option.fromNullable(options.skipIfOlderThan).pipe(
|
77
|
+
Option.map(Duration.decode),
|
78
|
+
Option.getOrElse(() => Duration.days(1))
|
79
|
+
)
|
80
|
+
|
81
|
+
const effect = Effect.fnUntraced(function*(dateTime: DateTime.Utc) {
|
82
|
+
const now = yield* DateTime.now
|
83
|
+
if (DateTime.lessThan(dateTime, DateTime.subtractDuration(now, skipIfOlderThan))) {
|
84
|
+
return
|
85
|
+
}
|
86
|
+
return yield* options.execute
|
87
|
+
}, Effect.orDie)
|
88
|
+
|
89
|
+
const EntityLayer = CronEntity.toLayer(Effect.gen(function*() {
|
90
|
+
const makeClient = yield* CronEntity.client
|
91
|
+
return {
|
92
|
+
run(request) {
|
93
|
+
return Effect.ensuring(
|
94
|
+
effect(request.payload.dateTime),
|
95
|
+
Effect.gen(function*() {
|
96
|
+
const now = yield* DateTime.now
|
97
|
+
const next = DateTime.unsafeFromDate(Cron.next(
|
98
|
+
options.cron,
|
99
|
+
options.calculateNextRunFromPrevious ? request.payload.dateTime : now
|
100
|
+
))
|
101
|
+
const client = makeClient(DateTime.formatIso(next))
|
102
|
+
return yield* client.run({ dateTime: next }, { discard: true })
|
103
|
+
}).pipe(
|
104
|
+
Effect.sandbox,
|
105
|
+
Effect.retry(retryPolicy),
|
106
|
+
Effect.orDie
|
107
|
+
)
|
108
|
+
)
|
109
|
+
}
|
110
|
+
}
|
111
|
+
}))
|
112
|
+
|
113
|
+
return Layer.merge(InitialRun, EntityLayer)
|
114
|
+
}
|
115
|
+
|
116
|
+
const retryPolicy = Schedule.exponential(200, 1.5).pipe(
|
117
|
+
Schedule.union(Schedule.spaced("1 minute"))
|
118
|
+
)
|
119
|
+
|
120
|
+
class CronPayload extends Schema.Class<CronPayload>("@effect/cluster/ClusterCron/CronPayload")({
|
121
|
+
dateTime: Schema.DateTimeUtc
|
122
|
+
}) {
|
123
|
+
[PrimaryKey.symbol]() {
|
124
|
+
return ""
|
125
|
+
}
|
126
|
+
[DeliverAt.symbol]() {
|
127
|
+
return this.dateTime
|
128
|
+
}
|
129
|
+
}
|
package/src/ClusterSchema.ts
CHANGED
@@ -3,6 +3,7 @@
|
|
3
3
|
*/
|
4
4
|
import * as Context from "effect/Context"
|
5
5
|
import { constFalse } from "effect/Function"
|
6
|
+
import type { EntityId } from "./EntityId.js"
|
6
7
|
|
7
8
|
/**
|
8
9
|
* @since 1.0.0
|
@@ -21,3 +22,11 @@ export class Uninterruptible
|
|
21
22
|
defaultValue: constFalse
|
22
23
|
})
|
23
24
|
{}
|
25
|
+
|
26
|
+
/**
|
27
|
+
* @since 1.0.0
|
28
|
+
* @category Annotations
|
29
|
+
*/
|
30
|
+
export class ShardGroup extends Context.Reference<ShardGroup>()("@effect/cluster/ClusterSchema/ShardGroup", {
|
31
|
+
defaultValue: (): (entityId: EntityId) => string => (_) => "default"
|
32
|
+
}) {}
|
@@ -37,6 +37,7 @@ export const make = Effect.gen(function*() {
|
|
37
37
|
const sharding = yield* Sharding.Sharding
|
38
38
|
const storage = yield* MessageStorage
|
39
39
|
|
40
|
+
const workflows = new Map<string, Workflow.Any>()
|
40
41
|
const entities = new Map<
|
41
42
|
string,
|
42
43
|
Entity.Entity<
|
@@ -67,16 +68,20 @@ export const make = Effect.gen(function*() {
|
|
67
68
|
const deferredClient = yield* DeferredEntity.client
|
68
69
|
|
69
70
|
const requestIdFor = Effect.fnUntraced(function*(options: {
|
71
|
+
readonly workflow: Workflow.Any
|
70
72
|
readonly entityType: string
|
71
73
|
readonly executionId: string
|
72
74
|
readonly tag: string
|
73
75
|
readonly id: string
|
74
76
|
}) {
|
77
|
+
const shardGroup = Context.get(options.workflow.annotations, ClusterSchema.ShardGroup)(
|
78
|
+
options.executionId as EntityId
|
79
|
+
)
|
75
80
|
const entityId = EntityId.make(options.executionId)
|
76
81
|
const address = new EntityAddress({
|
77
82
|
entityType: EntityType.make(options.entityType),
|
78
83
|
entityId,
|
79
|
-
shardId: sharding.getShardId(entityId)
|
84
|
+
shardId: sharding.getShardId(entityId, shardGroup)
|
80
85
|
})
|
81
86
|
return yield* storage.requestIdForPrimaryKey({ address, tag: options.tag, id: options.id })
|
82
87
|
})
|
@@ -92,6 +97,7 @@ export const make = Effect.gen(function*() {
|
|
92
97
|
})
|
93
98
|
|
94
99
|
const requestReply = Effect.fnUntraced(function*(options: {
|
100
|
+
readonly workflow: Workflow.Any
|
95
101
|
readonly entityType: string
|
96
102
|
readonly executionId: string
|
97
103
|
readonly tag: string
|
@@ -112,6 +118,7 @@ export const make = Effect.gen(function*() {
|
|
112
118
|
readonly attempt: number
|
113
119
|
}) {
|
114
120
|
const requestId = yield* requestIdFor({
|
121
|
+
workflow: options.workflow,
|
115
122
|
entityType: `Workflow/${options.workflow.name}`,
|
116
123
|
executionId: options.executionId,
|
117
124
|
tag: "activity",
|
@@ -128,10 +135,14 @@ export const make = Effect.gen(function*() {
|
|
128
135
|
)
|
129
136
|
|
130
137
|
const clearClock = Effect.fnUntraced(function*(options: {
|
138
|
+
readonly workflow: Workflow.Any
|
131
139
|
readonly executionId: string
|
132
140
|
}) {
|
141
|
+
const shardGroup = Context.get(options.workflow.annotations, ClusterSchema.ShardGroup)(
|
142
|
+
options.executionId as EntityId
|
143
|
+
)
|
133
144
|
const entityId = EntityId.make(options.executionId)
|
134
|
-
const shardId = sharding.getShardId(entityId)
|
145
|
+
const shardId = sharding.getShardId(entityId, shardGroup)
|
135
146
|
const clockAddress = new EntityAddress({
|
136
147
|
entityType: ClockEntity.type,
|
137
148
|
entityId,
|
@@ -149,6 +160,7 @@ export const make = Effect.gen(function*() {
|
|
149
160
|
return Effect.dieMessage(`Workflow ${workflow.name} already registered`)
|
150
161
|
}
|
151
162
|
const entity = makeWorkflowEntity(workflow)
|
163
|
+
workflows.set(workflow.name, workflow)
|
152
164
|
entities.set(workflow.name, entity as any)
|
153
165
|
return sharding.registerEntity(
|
154
166
|
entity,
|
@@ -174,7 +186,7 @@ export const make = Effect.gen(function*() {
|
|
174
186
|
}
|
175
187
|
instance.suspended = false
|
176
188
|
return Effect.zipRight(
|
177
|
-
Effect.ignore(clearClock({ executionId })),
|
189
|
+
Effect.ignore(clearClock({ workflow, executionId })),
|
178
190
|
Effect.interrupt
|
179
191
|
)
|
180
192
|
}),
|
@@ -229,6 +241,7 @@ export const make = Effect.gen(function*() {
|
|
229
241
|
interrupt: Effect.fnUntraced(
|
230
242
|
function*(this: WorkflowEngine["Type"], workflow, executionId) {
|
231
243
|
const requestId = yield* requestIdFor({
|
244
|
+
workflow,
|
232
245
|
entityType: `Workflow/${workflow.name}`,
|
233
246
|
executionId,
|
234
247
|
tag: "run",
|
@@ -262,7 +275,12 @@ export const make = Effect.gen(function*() {
|
|
262
275
|
|
263
276
|
resume: Effect.fnUntraced(
|
264
277
|
function*(workflowName: string, executionId: string) {
|
278
|
+
const workflow = workflows.get(workflowName)
|
279
|
+
if (!workflow) {
|
280
|
+
return yield* Effect.dieMessage(`WorkflowEngine.resume: ${workflowName} not registered`)
|
281
|
+
}
|
265
282
|
const maybeReply = yield* requestReply({
|
283
|
+
workflow,
|
266
284
|
entityType: `Workflow/${workflowName}`,
|
267
285
|
executionId,
|
268
286
|
tag: "run",
|
@@ -316,6 +334,7 @@ export const make = Effect.gen(function*() {
|
|
316
334
|
WorkflowInstance.pipe(
|
317
335
|
Effect.flatMap((instance) =>
|
318
336
|
requestReply({
|
337
|
+
workflow: instance.workflow,
|
319
338
|
entityType: DeferredEntity.type,
|
320
339
|
executionId: instance.executionId,
|
321
340
|
tag: "set",
|
@@ -356,6 +375,17 @@ export const make = Effect.gen(function*() {
|
|
356
375
|
})
|
357
376
|
})
|
358
377
|
|
378
|
+
const retryPolicy = Schedule.exponential(200, 1.5).pipe(
|
379
|
+
Schedule.union(Schedule.spaced("1 minute"))
|
380
|
+
)
|
381
|
+
|
382
|
+
const ensureSuccess = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
|
383
|
+
effect.pipe(
|
384
|
+
Effect.sandbox,
|
385
|
+
Effect.retry(retryPolicy),
|
386
|
+
Effect.orDie
|
387
|
+
)
|
388
|
+
|
359
389
|
const ActivityRpc = Rpc.make("activity", {
|
360
390
|
payload: {
|
361
391
|
name: Schema.String,
|
@@ -380,6 +410,7 @@ const makeWorkflowEntity = (workflow: Workflow.Any) =>
|
|
380
410
|
}),
|
381
411
|
ActivityRpc
|
382
412
|
])
|
413
|
+
.annotateContext(workflow.annotations)
|
383
414
|
.annotateRpcs(ClusterSchema.Persisted, true)
|
384
415
|
.annotateRpcs(ClusterSchema.Uninterruptible, true)
|
385
416
|
|
@@ -420,7 +451,7 @@ const DeferredEntityLayer = DeferredEntity.toLayer(Effect.gen(function*() {
|
|
420
451
|
return {
|
421
452
|
set: (request) =>
|
422
453
|
Effect.as(
|
423
|
-
|
454
|
+
ensureSuccess(client.resume(request.payload, { discard: true })),
|
424
455
|
request.payload.exit
|
425
456
|
),
|
426
457
|
resume: (request) => engine.resume(request.payload.workflowName, executionId)
|
@@ -453,12 +484,12 @@ const ClockEntityLayer = ClockEntity.toLayer(Effect.gen(function*() {
|
|
453
484
|
return {
|
454
485
|
run(request) {
|
455
486
|
const deferred = DurableClock.make({ name: request.payload.name, duration: Duration.zero }).deferred
|
456
|
-
return engine.deferredDone({
|
487
|
+
return ensureSuccess(engine.deferredDone({
|
457
488
|
workflowName: request.payload.workflowName,
|
458
489
|
executionId,
|
459
490
|
deferred,
|
460
491
|
exit: { _tag: "Success", value: void 0 }
|
461
|
-
})
|
492
|
+
}))
|
462
493
|
}
|
463
494
|
}
|
464
495
|
}))
|
package/src/Entity.ts
CHANGED
@@ -26,6 +26,7 @@ import type {
|
|
26
26
|
MailboxFull,
|
27
27
|
PersistenceError
|
28
28
|
} from "./ClusterError.js"
|
29
|
+
import { ShardGroup } from "./ClusterSchema.js"
|
29
30
|
import { EntityAddress } from "./EntityAddress.js"
|
30
31
|
import type { EntityId } from "./EntityId.js"
|
31
32
|
import { EntityType } from "./EntityType.js"
|
@@ -68,6 +69,16 @@ export interface Entity<in out Rpcs extends Rpc.Any> extends Equal.Equal {
|
|
68
69
|
*/
|
69
70
|
readonly protocol: RpcGroup.RpcGroup<Rpcs>
|
70
71
|
|
72
|
+
/**
|
73
|
+
* Get the shard group for the given EntityId.
|
74
|
+
*/
|
75
|
+
getShardGroup(entityId: EntityId): string
|
76
|
+
|
77
|
+
/**
|
78
|
+
* Get the ShardId for the given EntityId.
|
79
|
+
*/
|
80
|
+
getShardId(entityId: EntityId): Effect.Effect<ShardId.ShardId, never, Sharding>
|
81
|
+
|
71
82
|
/**
|
72
83
|
* Annotate the entity with a value.
|
73
84
|
*/
|
@@ -205,6 +216,9 @@ const Proto = {
|
|
205
216
|
annotateRpcsContext<S>(this: Entity<any>, context: Context.Context<S>) {
|
206
217
|
return fromRpcGroup(this.type, this.protocol.annotateRpcsContext(context))
|
207
218
|
},
|
219
|
+
getShardId(this: Entity<any>, entityId: EntityId) {
|
220
|
+
return Effect.map(shardingTag, (sharding) => sharding.getShardId(entityId, this.getShardGroup(entityId)))
|
221
|
+
},
|
208
222
|
get client() {
|
209
223
|
return shardingTag.pipe(
|
210
224
|
Effect.flatMap((sharding) => sharding.makeClient(this as any))
|
@@ -341,6 +355,7 @@ export const fromRpcGroup = <Rpcs extends Rpc.Any>(
|
|
341
355
|
const self = Object.create(Proto)
|
342
356
|
self.type = EntityType.make(type)
|
343
357
|
self.protocol = protocol
|
358
|
+
self.getShardGroup = Context.get(protocol.annotations, ShardGroup)
|
344
359
|
return self
|
345
360
|
}
|
346
361
|
|
@@ -470,7 +485,11 @@ export const makeTestClient: <Rpcs extends Rpc.Any, LA, LE, LR>(
|
|
470
485
|
layer: Layer.Layer<LA, LE, LR>
|
471
486
|
) {
|
472
487
|
const config = yield* ShardingConfig
|
473
|
-
const makeShardId = (entityId: string) =>
|
488
|
+
const makeShardId = (entityId: string) =>
|
489
|
+
ShardId.make(
|
490
|
+
entity.getShardGroup(entityId as EntityId),
|
491
|
+
(Math.abs(hashString(entityId) % config.shardsPerGroup)) + 1
|
492
|
+
)
|
474
493
|
const snowflakeGen = yield* Snowflake.makeGenerator
|
475
494
|
const runnerAddress = new RunnerAddress({ host: "localhost", port: 3000 })
|
476
495
|
const entityMap = new Map<string, {
|
package/src/EntityAddress.ts
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
/**
|
2
2
|
* @since 1.0.0
|
3
3
|
*/
|
4
|
+
import * as Equal from "effect/Equal"
|
4
5
|
import * as Hash from "effect/Hash"
|
5
6
|
import * as Schema from "effect/Schema"
|
6
7
|
import { EntityId } from "./EntityId.js"
|
@@ -36,11 +37,20 @@ export class EntityAddress extends Schema.Class<EntityAddress>(SymbolKey)({
|
|
36
37
|
* @since 1.0.0
|
37
38
|
*/
|
38
39
|
readonly [TypeId] = TypeId;
|
40
|
+
|
41
|
+
/**
|
42
|
+
* @since 1.0.0
|
43
|
+
*/
|
44
|
+
[Equal.symbol](that: EntityAddress): boolean {
|
45
|
+
return this.entityType === that.entityType && this.entityId === that.entityId &&
|
46
|
+
this.shardId[Equal.symbol](that.shardId)
|
47
|
+
}
|
48
|
+
|
39
49
|
/**
|
40
50
|
* @since 1.0.0
|
41
51
|
*/
|
42
52
|
[Hash.symbol]() {
|
43
|
-
return Hash.cached(this
|
53
|
+
return Hash.cached(this, Hash.string(`${this.entityType}:${this.entityId}:${this.shardId.toString()}`))
|
44
54
|
}
|
45
55
|
}
|
46
56
|
|
package/src/MessageStorage.ts
CHANGED
@@ -22,7 +22,7 @@ import type { EntityAddress } from "./EntityAddress.js"
|
|
22
22
|
import * as Envelope from "./Envelope.js"
|
23
23
|
import * as Message from "./Message.js"
|
24
24
|
import * as Reply from "./Reply.js"
|
25
|
-
import
|
25
|
+
import { ShardId } from "./ShardId.js"
|
26
26
|
import type { ShardingConfig } from "./ShardingConfig.js"
|
27
27
|
import * as Snowflake from "./Snowflake.js"
|
28
28
|
|
@@ -266,7 +266,7 @@ export type Encoded = {
|
|
266
266
|
* - All Interrupt's for unprocessed requests
|
267
267
|
*/
|
268
268
|
readonly unprocessedMessages: (
|
269
|
-
shardIds: ReadonlyArray<
|
269
|
+
shardIds: ReadonlyArray<string>,
|
270
270
|
now: number
|
271
271
|
) => Effect.Effect<
|
272
272
|
Array<{
|
@@ -308,7 +308,7 @@ export type Encoded = {
|
|
308
308
|
* Reset the mailbox state for the provided shards.
|
309
309
|
*/
|
310
310
|
readonly resetShards: (
|
311
|
-
shardIds: ReadonlyArray<
|
311
|
+
shardIds: ReadonlyArray<string>
|
312
312
|
) => Effect.Effect<void, PersistenceError>
|
313
313
|
}
|
314
314
|
|
@@ -441,7 +441,9 @@ export const makeEncoded: (encoded: Encoded) => Effect.Effect<
|
|
441
441
|
const shards = Array.from(shardIds)
|
442
442
|
if (shards.length === 0) return Effect.succeed([])
|
443
443
|
return Effect.flatMap(
|
444
|
-
Effect.suspend(() =>
|
444
|
+
Effect.suspend(() =>
|
445
|
+
encoded.unprocessedMessages(shards.map((id) => id.toString()), clock.unsafeCurrentTimeMillis())
|
446
|
+
),
|
445
447
|
decodeMessages
|
446
448
|
)
|
447
449
|
},
|
@@ -455,7 +457,7 @@ export const makeEncoded: (encoded: Encoded) => Effect.Effect<
|
|
455
457
|
},
|
456
458
|
resetAddress: encoded.resetAddress,
|
457
459
|
clearAddress: encoded.clearAddress,
|
458
|
-
resetShards: (shardIds) => encoded.resetShards(Array.from(shardIds))
|
460
|
+
resetShards: (shardIds) => encoded.resetShards(Array.from(shardIds, (id) => id.toString()))
|
459
461
|
})
|
460
462
|
|
461
463
|
const decodeMessages = (
|
@@ -666,7 +668,9 @@ export class MemoryDriver extends Effect.Service<MemoryDriver>()("@effect/cluste
|
|
666
668
|
if (existing) {
|
667
669
|
return SaveResultEncoded.Duplicate({
|
668
670
|
originalId: Snowflake.Snowflake(existing.envelope.requestId),
|
669
|
-
lastReceivedReply: existing.
|
671
|
+
lastReceivedReply: existing.replies.length === 1 && existing.replies[0]._tag === "WithExit"
|
672
|
+
? Option.some(existing.replies[0])
|
673
|
+
: existing.lastReceivedChunk
|
670
674
|
})
|
671
675
|
}
|
672
676
|
if (envelope._tag === "Request") {
|
@@ -726,7 +730,8 @@ export class MemoryDriver extends Effect.Service<MemoryDriver>()("@effect/cluste
|
|
726
730
|
let index = journal.indexOf(Iterable.unsafeHead(unprocessed))
|
727
731
|
for (; index < journal.length; index++) {
|
728
732
|
const envelope = journal[index]
|
729
|
-
|
733
|
+
const shardId = ShardId.make(envelope.address.shardId)
|
734
|
+
if (!shardIds.includes(shardId.toString())) {
|
730
735
|
continue
|
731
736
|
}
|
732
737
|
if (envelope._tag === "Request") {
|
package/src/Runner.ts
CHANGED
@@ -1,6 +1,8 @@
|
|
1
1
|
/**
|
2
2
|
* @since 1.0.0
|
3
3
|
*/
|
4
|
+
import * as Equal from "effect/Equal"
|
5
|
+
import * as Hash from "effect/Hash"
|
4
6
|
import { NodeInspectSymbol } from "effect/Inspectable"
|
5
7
|
import * as Pretty from "effect/Pretty"
|
6
8
|
import * as Schema from "effect/Schema"
|
@@ -35,6 +37,7 @@ export type TypeId = typeof TypeId
|
|
35
37
|
*/
|
36
38
|
export class Runner extends Schema.Class<Runner>(SymbolKey)({
|
37
39
|
address: RunnerAddress,
|
40
|
+
groups: Schema.Array(Schema.String),
|
38
41
|
version: Schema.Int
|
39
42
|
}) {
|
40
43
|
/**
|
@@ -63,6 +66,20 @@ export class Runner extends Schema.Class<Runner>(SymbolKey)({
|
|
63
66
|
[NodeInspectSymbol](): string {
|
64
67
|
return this.toString()
|
65
68
|
}
|
69
|
+
|
70
|
+
/**
|
71
|
+
* @since 1.0.0
|
72
|
+
*/
|
73
|
+
[Equal.symbol](that: Runner): boolean {
|
74
|
+
return this.address[Equal.symbol](that.address) && this.version === that.version
|
75
|
+
}
|
76
|
+
|
77
|
+
/**
|
78
|
+
* @since 1.0.0
|
79
|
+
*/
|
80
|
+
[Hash.symbol](): number {
|
81
|
+
return Hash.cached(this, Hash.string(`${this.address.toString()}:${this.version}`))
|
82
|
+
}
|
66
83
|
}
|
67
84
|
|
68
85
|
/**
|
@@ -80,5 +97,6 @@ export class Runner extends Schema.Class<Runner>(SymbolKey)({
|
|
80
97
|
*/
|
81
98
|
export const make = (props: {
|
82
99
|
readonly address: RunnerAddress
|
100
|
+
readonly groups: ReadonlyArray<string>
|
83
101
|
readonly version: number
|
84
102
|
}): Runner => new Runner(props)
|
package/src/RunnerAddress.ts
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
/**
|
2
2
|
* @since 1.0.0
|
3
3
|
*/
|
4
|
+
import * as Equal from "effect/Equal"
|
4
5
|
import * as Hash from "effect/Hash"
|
5
6
|
import { NodeInspectSymbol } from "effect/Inspectable"
|
6
7
|
import * as Schema from "effect/Schema"
|
@@ -32,11 +33,18 @@ export class RunnerAddress extends Schema.Class<RunnerAddress>(SymbolKey)({
|
|
32
33
|
*/
|
33
34
|
readonly [TypeId] = TypeId;
|
34
35
|
|
36
|
+
/**
|
37
|
+
* @since 1.0.0
|
38
|
+
*/
|
39
|
+
[Equal.symbol](that: RunnerAddress): boolean {
|
40
|
+
return this.host === that.host && this.port === that.port
|
41
|
+
}
|
42
|
+
|
35
43
|
/**
|
36
44
|
* @since 1.0.0
|
37
45
|
*/
|
38
46
|
[Hash.symbol]() {
|
39
|
-
return Hash.cached(this
|
47
|
+
return Hash.cached(this, Hash.string(this.toString()))
|
40
48
|
}
|
41
49
|
|
42
50
|
/**
|
package/src/Runners.ts
CHANGED
@@ -164,6 +164,11 @@ export const make: (options: Omit<Runners["Type"], "sendLocal" | "notifyLocal">)
|
|
164
164
|
MessageStorage.SaveResult.$match({
|
165
165
|
Success: () => afterPersist(message, false),
|
166
166
|
Duplicate: ({ lastReceivedReply, originalId }) => {
|
167
|
+
// If the last received reply is an exit, we can just return it
|
168
|
+
// as the response.
|
169
|
+
if (Option.isSome(lastReceivedReply) && lastReceivedReply.value._tag === "WithExit") {
|
170
|
+
return message.respond(lastReceivedReply.value.withRequestId(message.envelope.requestId))
|
171
|
+
}
|
167
172
|
requestIdRewrites.set(message.envelope.requestId, originalId)
|
168
173
|
return afterPersist(
|
169
174
|
new Message.OutgoingRequest({
|
package/src/ShardId.ts
CHANGED
@@ -1,27 +1,97 @@
|
|
1
1
|
/**
|
2
2
|
* @since 1.0.0
|
3
3
|
*/
|
4
|
-
import * as
|
4
|
+
import * as Equal from "effect/Equal"
|
5
|
+
import * as Hash from "effect/Hash"
|
6
|
+
import * as S from "effect/Schema"
|
5
7
|
|
6
8
|
/**
|
7
9
|
* @since 1.0.0
|
8
|
-
* @category
|
10
|
+
* @category Symbols
|
9
11
|
*/
|
10
|
-
export const
|
11
|
-
Schema.brand("ShardId"),
|
12
|
-
Schema.annotations({
|
13
|
-
pretty: () => (shardId) => `ShardId(${shardId})`
|
14
|
-
})
|
15
|
-
)
|
12
|
+
export const TypeId: unique symbol = Symbol.for("@effect/cluster/ShardId")
|
16
13
|
|
17
14
|
/**
|
18
15
|
* @since 1.0.0
|
19
|
-
* @category
|
16
|
+
* @category Symbols
|
20
17
|
*/
|
21
|
-
export type
|
18
|
+
export type TypeId = typeof TypeId
|
19
|
+
|
20
|
+
const constDisableValidation = { disableValidation: true }
|
22
21
|
|
23
22
|
/**
|
24
23
|
* @since 1.0.0
|
25
24
|
* @category Constructors
|
26
25
|
*/
|
27
|
-
export const make = (
|
26
|
+
export const make = (group: string, id: number): ShardId => new ShardId({ group, id }, constDisableValidation)
|
27
|
+
|
28
|
+
/**
|
29
|
+
* @since 1.0.0
|
30
|
+
* @category Models
|
31
|
+
*/
|
32
|
+
export class ShardId extends S.Class<ShardId>("@effect/cluster/ShardId")({
|
33
|
+
group: S.String,
|
34
|
+
id: S.Int
|
35
|
+
}) {
|
36
|
+
/**
|
37
|
+
* @since 1.0.0
|
38
|
+
*/
|
39
|
+
readonly [TypeId]: TypeId = TypeId;
|
40
|
+
|
41
|
+
/**
|
42
|
+
* @since 1.0.0
|
43
|
+
*/
|
44
|
+
[Equal.symbol](that: ShardId): boolean {
|
45
|
+
return this.group === that.group && this.id === that.id
|
46
|
+
}
|
47
|
+
|
48
|
+
/**
|
49
|
+
* @since 1.0.0
|
50
|
+
*/
|
51
|
+
[Hash.symbol](): number {
|
52
|
+
return Hash.cached(this, Hash.string(this.toString()))
|
53
|
+
}
|
54
|
+
|
55
|
+
/**
|
56
|
+
* @since 1.0.0
|
57
|
+
*/
|
58
|
+
toString(): string {
|
59
|
+
return `${this.group}:${this.id}`
|
60
|
+
}
|
61
|
+
|
62
|
+
/**
|
63
|
+
* @since 1.0.0
|
64
|
+
*/
|
65
|
+
static toString(shardId: {
|
66
|
+
readonly group: string
|
67
|
+
readonly id: number
|
68
|
+
}): string {
|
69
|
+
return `${shardId.group}:${shardId.id}`
|
70
|
+
}
|
71
|
+
|
72
|
+
/**
|
73
|
+
* @since 1.0.0
|
74
|
+
*/
|
75
|
+
static fromStringEncoded(s: string): {
|
76
|
+
readonly group: string
|
77
|
+
readonly id: number
|
78
|
+
} {
|
79
|
+
const index = s.lastIndexOf(":")
|
80
|
+
if (index === -1) {
|
81
|
+
throw new Error(`Invalid ShardId format`)
|
82
|
+
}
|
83
|
+
const group = s.substring(0, index)
|
84
|
+
const id = Number(s.substring(index + 1))
|
85
|
+
if (isNaN(id)) {
|
86
|
+
throw new Error(`ShardId id must be a number`)
|
87
|
+
}
|
88
|
+
return { group, id }
|
89
|
+
}
|
90
|
+
|
91
|
+
/**
|
92
|
+
* @since 1.0.0
|
93
|
+
*/
|
94
|
+
static fromString(s: string): ShardId {
|
95
|
+
return new ShardId(ShardId.fromStringEncoded(s), constDisableValidation)
|
96
|
+
}
|
97
|
+
}
|