@effect/cluster 0.50.6 → 0.51.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (223) hide show
  1. package/RunnerStorage/package.json +6 -0
  2. package/SqlRunnerStorage/package.json +6 -0
  3. package/dist/cjs/ClusterError.js +2 -24
  4. package/dist/cjs/ClusterError.js.map +1 -1
  5. package/dist/cjs/ClusterMetrics.js +13 -15
  6. package/dist/cjs/ClusterMetrics.js.map +1 -1
  7. package/dist/cjs/ClusterWorkflowEngine.js +41 -81
  8. package/dist/cjs/ClusterWorkflowEngine.js.map +1 -1
  9. package/dist/cjs/Entity.js.map +1 -1
  10. package/dist/cjs/EntityAddress.js +9 -1
  11. package/dist/cjs/EntityAddress.js.map +1 -1
  12. package/dist/cjs/EntityId.js +7 -1
  13. package/dist/cjs/EntityId.js.map +1 -1
  14. package/dist/cjs/EntityProxy.js +1 -1
  15. package/dist/cjs/EntityProxy.js.map +1 -1
  16. package/dist/cjs/HttpRunner.js +69 -43
  17. package/dist/cjs/HttpRunner.js.map +1 -1
  18. package/dist/cjs/MessageStorage.js +64 -16
  19. package/dist/cjs/MessageStorage.js.map +1 -1
  20. package/dist/cjs/Runner.js +3 -3
  21. package/dist/cjs/Runner.js.map +1 -1
  22. package/dist/cjs/RunnerAddress.js +7 -0
  23. package/dist/cjs/RunnerAddress.js.map +1 -1
  24. package/dist/cjs/RunnerHealth.js +91 -32
  25. package/dist/cjs/RunnerHealth.js.map +1 -1
  26. package/dist/cjs/RunnerServer.js +38 -24
  27. package/dist/cjs/RunnerServer.js.map +1 -1
  28. package/dist/cjs/RunnerStorage.js +100 -0
  29. package/dist/cjs/RunnerStorage.js.map +1 -0
  30. package/dist/cjs/Runners.js +18 -22
  31. package/dist/cjs/Runners.js.map +1 -1
  32. package/dist/cjs/ShardId.js +17 -7
  33. package/dist/cjs/ShardId.js.map +1 -1
  34. package/dist/cjs/Sharding.js +435 -318
  35. package/dist/cjs/Sharding.js.map +1 -1
  36. package/dist/cjs/ShardingConfig.js +10 -14
  37. package/dist/cjs/ShardingConfig.js.map +1 -1
  38. package/dist/cjs/Snowflake.js +1 -1
  39. package/dist/cjs/SocketRunner.js +1 -1
  40. package/dist/cjs/SocketRunner.js.map +1 -1
  41. package/dist/cjs/SqlMessageStorage.js +22 -28
  42. package/dist/cjs/SqlMessageStorage.js.map +1 -1
  43. package/dist/cjs/SqlRunnerStorage.js +378 -0
  44. package/dist/cjs/SqlRunnerStorage.js.map +1 -0
  45. package/dist/cjs/index.js +5 -15
  46. package/dist/cjs/internal/entityManager.js +40 -9
  47. package/dist/cjs/internal/entityManager.js.map +1 -1
  48. package/dist/dts/ClusterError.d.ts +0 -22
  49. package/dist/dts/ClusterError.d.ts.map +1 -1
  50. package/dist/dts/ClusterMetrics.d.ts +4 -14
  51. package/dist/dts/ClusterMetrics.d.ts.map +1 -1
  52. package/dist/dts/ClusterWorkflowEngine.d.ts.map +1 -1
  53. package/dist/dts/Entity.d.ts +2 -2
  54. package/dist/dts/Entity.d.ts.map +1 -1
  55. package/dist/dts/EntityAddress.d.ts +11 -0
  56. package/dist/dts/EntityAddress.d.ts.map +1 -1
  57. package/dist/dts/EntityId.d.ts +5 -0
  58. package/dist/dts/EntityId.d.ts.map +1 -1
  59. package/dist/dts/EntityProxy.d.ts +5 -6
  60. package/dist/dts/EntityProxy.d.ts.map +1 -1
  61. package/dist/dts/HttpRunner.d.ts +48 -25
  62. package/dist/dts/HttpRunner.d.ts.map +1 -1
  63. package/dist/dts/MessageStorage.d.ts +13 -5
  64. package/dist/dts/MessageStorage.d.ts.map +1 -1
  65. package/dist/dts/Runner.d.ts +4 -4
  66. package/dist/dts/Runner.d.ts.map +1 -1
  67. package/dist/dts/RunnerAddress.d.ts +5 -0
  68. package/dist/dts/RunnerAddress.d.ts.map +1 -1
  69. package/dist/dts/RunnerHealth.d.ts +24 -16
  70. package/dist/dts/RunnerHealth.d.ts.map +1 -1
  71. package/dist/dts/RunnerServer.d.ts +5 -4
  72. package/dist/dts/RunnerServer.d.ts.map +1 -1
  73. package/dist/dts/{ShardStorage.d.ts → RunnerStorage.d.ts} +41 -54
  74. package/dist/dts/RunnerStorage.d.ts.map +1 -0
  75. package/dist/dts/Runners.d.ts +15 -11
  76. package/dist/dts/Runners.d.ts.map +1 -1
  77. package/dist/dts/ShardId.d.ts +1 -1
  78. package/dist/dts/ShardId.d.ts.map +1 -1
  79. package/dist/dts/Sharding.d.ts +20 -10
  80. package/dist/dts/Sharding.d.ts.map +1 -1
  81. package/dist/dts/ShardingConfig.d.ts +40 -14
  82. package/dist/dts/ShardingConfig.d.ts.map +1 -1
  83. package/dist/dts/SocketRunner.d.ts +4 -3
  84. package/dist/dts/SocketRunner.d.ts.map +1 -1
  85. package/dist/dts/SqlMessageStorage.d.ts +2 -3
  86. package/dist/dts/SqlMessageStorage.d.ts.map +1 -1
  87. package/dist/dts/SqlRunnerStorage.d.ts +40 -0
  88. package/dist/dts/SqlRunnerStorage.d.ts.map +1 -0
  89. package/dist/dts/index.d.ts +4 -24
  90. package/dist/dts/index.d.ts.map +1 -1
  91. package/dist/esm/ClusterError.js +0 -21
  92. package/dist/esm/ClusterError.js.map +1 -1
  93. package/dist/esm/ClusterMetrics.js +12 -14
  94. package/dist/esm/ClusterMetrics.js.map +1 -1
  95. package/dist/esm/ClusterWorkflowEngine.js +41 -81
  96. package/dist/esm/ClusterWorkflowEngine.js.map +1 -1
  97. package/dist/esm/Entity.js.map +1 -1
  98. package/dist/esm/EntityAddress.js +7 -0
  99. package/dist/esm/EntityAddress.js.map +1 -1
  100. package/dist/esm/EntityId.js +5 -0
  101. package/dist/esm/EntityId.js.map +1 -1
  102. package/dist/esm/EntityProxy.js +2 -2
  103. package/dist/esm/EntityProxy.js.map +1 -1
  104. package/dist/esm/HttpRunner.js +62 -39
  105. package/dist/esm/HttpRunner.js.map +1 -1
  106. package/dist/esm/MessageStorage.js +65 -17
  107. package/dist/esm/MessageStorage.js.map +1 -1
  108. package/dist/esm/Runner.js +3 -3
  109. package/dist/esm/Runner.js.map +1 -1
  110. package/dist/esm/RunnerAddress.js +7 -0
  111. package/dist/esm/RunnerAddress.js.map +1 -1
  112. package/dist/esm/RunnerHealth.js +88 -30
  113. package/dist/esm/RunnerHealth.js.map +1 -1
  114. package/dist/esm/RunnerServer.js +38 -24
  115. package/dist/esm/RunnerServer.js.map +1 -1
  116. package/dist/esm/RunnerStorage.js +90 -0
  117. package/dist/esm/RunnerStorage.js.map +1 -0
  118. package/dist/esm/Runners.js +19 -23
  119. package/dist/esm/Runners.js.map +1 -1
  120. package/dist/esm/ShardId.js +16 -6
  121. package/dist/esm/ShardId.js.map +1 -1
  122. package/dist/esm/Sharding.js +438 -321
  123. package/dist/esm/Sharding.js.map +1 -1
  124. package/dist/esm/ShardingConfig.js +10 -14
  125. package/dist/esm/ShardingConfig.js.map +1 -1
  126. package/dist/esm/Snowflake.js +1 -1
  127. package/dist/esm/SocketRunner.js +1 -1
  128. package/dist/esm/SocketRunner.js.map +1 -1
  129. package/dist/esm/SqlMessageStorage.js +22 -28
  130. package/dist/esm/SqlMessageStorage.js.map +1 -1
  131. package/dist/esm/SqlRunnerStorage.js +369 -0
  132. package/dist/esm/SqlRunnerStorage.js.map +1 -0
  133. package/dist/esm/index.js +4 -24
  134. package/dist/esm/index.js.map +1 -1
  135. package/dist/esm/internal/entityManager.js +40 -9
  136. package/dist/esm/internal/entityManager.js.map +1 -1
  137. package/package.json +20 -60
  138. package/src/ClusterError.ts +0 -24
  139. package/src/ClusterMetrics.ts +12 -16
  140. package/src/ClusterWorkflowEngine.ts +38 -78
  141. package/src/Entity.ts +2 -7
  142. package/src/EntityAddress.ts +10 -0
  143. package/src/EntityId.ts +6 -0
  144. package/src/EntityProxy.ts +10 -10
  145. package/src/HttpRunner.ts +132 -67
  146. package/src/MessageStorage.ts +89 -24
  147. package/src/Runner.ts +4 -4
  148. package/src/RunnerAddress.ts +8 -0
  149. package/src/RunnerHealth.ts +119 -56
  150. package/src/RunnerServer.ts +64 -47
  151. package/src/RunnerStorage.ts +218 -0
  152. package/src/Runners.ts +32 -45
  153. package/src/ShardId.ts +14 -3
  154. package/src/Sharding.ts +548 -413
  155. package/src/ShardingConfig.ts +39 -31
  156. package/src/Snowflake.ts +1 -1
  157. package/src/SocketRunner.ts +6 -4
  158. package/src/SqlMessageStorage.ts +28 -30
  159. package/src/SqlRunnerStorage.ts +541 -0
  160. package/src/index.ts +4 -29
  161. package/src/internal/entityManager.ts +44 -10
  162. package/HttpCommon/package.json +0 -6
  163. package/HttpShardManager/package.json +0 -6
  164. package/ShardManager/package.json +0 -6
  165. package/ShardStorage/package.json +0 -6
  166. package/SocketShardManager/package.json +0 -6
  167. package/SqlShardStorage/package.json +0 -6
  168. package/SynchronizedClock/package.json +0 -6
  169. package/dist/cjs/HttpCommon.js +0 -48
  170. package/dist/cjs/HttpCommon.js.map +0 -1
  171. package/dist/cjs/HttpShardManager.js +0 -139
  172. package/dist/cjs/HttpShardManager.js.map +0 -1
  173. package/dist/cjs/ShardManager.js +0 -549
  174. package/dist/cjs/ShardManager.js.map +0 -1
  175. package/dist/cjs/ShardStorage.js +0 -151
  176. package/dist/cjs/ShardStorage.js.map +0 -1
  177. package/dist/cjs/SocketShardManager.js +0 -32
  178. package/dist/cjs/SocketShardManager.js.map +0 -1
  179. package/dist/cjs/SqlShardStorage.js +0 -253
  180. package/dist/cjs/SqlShardStorage.js.map +0 -1
  181. package/dist/cjs/SynchronizedClock.js +0 -65
  182. package/dist/cjs/SynchronizedClock.js.map +0 -1
  183. package/dist/cjs/internal/shardManager.js +0 -353
  184. package/dist/cjs/internal/shardManager.js.map +0 -1
  185. package/dist/dts/HttpCommon.d.ts +0 -25
  186. package/dist/dts/HttpCommon.d.ts.map +0 -1
  187. package/dist/dts/HttpShardManager.d.ts +0 -119
  188. package/dist/dts/HttpShardManager.d.ts.map +0 -1
  189. package/dist/dts/ShardManager.d.ts +0 -459
  190. package/dist/dts/ShardManager.d.ts.map +0 -1
  191. package/dist/dts/ShardStorage.d.ts.map +0 -1
  192. package/dist/dts/SocketShardManager.d.ts +0 -17
  193. package/dist/dts/SocketShardManager.d.ts.map +0 -1
  194. package/dist/dts/SqlShardStorage.d.ts +0 -38
  195. package/dist/dts/SqlShardStorage.d.ts.map +0 -1
  196. package/dist/dts/SynchronizedClock.d.ts +0 -19
  197. package/dist/dts/SynchronizedClock.d.ts.map +0 -1
  198. package/dist/dts/internal/shardManager.d.ts +0 -2
  199. package/dist/dts/internal/shardManager.d.ts.map +0 -1
  200. package/dist/esm/HttpCommon.js +0 -38
  201. package/dist/esm/HttpCommon.js.map +0 -1
  202. package/dist/esm/HttpShardManager.js +0 -128
  203. package/dist/esm/HttpShardManager.js.map +0 -1
  204. package/dist/esm/ShardManager.js +0 -535
  205. package/dist/esm/ShardManager.js.map +0 -1
  206. package/dist/esm/ShardStorage.js +0 -141
  207. package/dist/esm/ShardStorage.js.map +0 -1
  208. package/dist/esm/SocketShardManager.js +0 -24
  209. package/dist/esm/SocketShardManager.js.map +0 -1
  210. package/dist/esm/SqlShardStorage.js +0 -244
  211. package/dist/esm/SqlShardStorage.js.map +0 -1
  212. package/dist/esm/SynchronizedClock.js +0 -57
  213. package/dist/esm/SynchronizedClock.js.map +0 -1
  214. package/dist/esm/internal/shardManager.js +0 -342
  215. package/dist/esm/internal/shardManager.js.map +0 -1
  216. package/src/HttpCommon.ts +0 -73
  217. package/src/HttpShardManager.ts +0 -273
  218. package/src/ShardManager.ts +0 -823
  219. package/src/ShardStorage.ts +0 -297
  220. package/src/SocketShardManager.ts +0 -48
  221. package/src/SqlShardStorage.ts +0 -329
  222. package/src/SynchronizedClock.ts +0 -82
  223. package/src/internal/shardManager.ts +0 -412
@@ -0,0 +1,541 @@
1
+ /**
2
+ * @since 1.0.0
3
+ */
4
+ import * as SqlClient from "@effect/sql/SqlClient"
5
+ import type { SqlError } from "@effect/sql/SqlError"
6
+ import type * as Statement from "@effect/sql/Statement"
7
+ import * as Arr from "effect/Array"
8
+ import * as Duration from "effect/Duration"
9
+ import * as Effect from "effect/Effect"
10
+ import * as Layer from "effect/Layer"
11
+ import * as ScopedRef from "effect/ScopedRef"
12
+ import { PersistenceError } from "./ClusterError.js"
13
+ import * as RunnerStorage from "./RunnerStorage.js"
14
+ import * as ShardId from "./ShardId.js"
15
+ import * as ShardingConfig from "./ShardingConfig.js"
16
+
17
+ const withTracerDisabled = Effect.withTracerEnabled(false)
18
+
19
+ /**
20
+ * @since 1.0.0
21
+ * @category Constructors
22
+ */
23
+ export const make = Effect.fnUntraced(function*(options: {
24
+ readonly prefix?: string | undefined
25
+ }) {
26
+ const config = yield* ShardingConfig.ShardingConfig
27
+ const sql = (yield* SqlClient.SqlClient).withoutTransforms()
28
+ const prefix = options?.prefix ?? "cluster"
29
+ const table = (name: string) => `${prefix}_${name}`
30
+ const lockConnRef = yield* sql.onDialectOrElse({
31
+ sqlite: () => Effect.void,
32
+
33
+ orElse: () => ScopedRef.fromAcquire(sql.reserve)
34
+ })
35
+
36
+ const runnersTable = table("runners")
37
+ const runnersTableSql = sql(runnersTable)
38
+
39
+ // Migrate old tables if they exist
40
+ // TODO: Remove in next major version
41
+ const hasOldTables = yield* sql`SELECT shard_id FROM ${sql(table("shards"))} LIMIT 1`.pipe(
42
+ Effect.isSuccess
43
+ )
44
+ if (hasOldTables) {
45
+ yield* sql`DROP TABLE ${sql(table("shards"))}`.pipe(Effect.ignore)
46
+ yield* sql`DROP TABLE ${runnersTableSql}`.pipe(Effect.ignore)
47
+ }
48
+
49
+ yield* sql.onDialectOrElse({
50
+ mssql: () =>
51
+ sql`
52
+ IF OBJECT_ID(N'${runnersTableSql}', N'U') IS NULL
53
+ CREATE TABLE ${runnersTableSql} (
54
+ machine_id INT IDENTITY PRIMARY KEY,
55
+ address VARCHAR(255) NOT NULL,
56
+ runner TEXT NOT NULL,
57
+ healthy BIT NOT NULL DEFAULT 1,
58
+ last_heartbeat DATETIME NOT NULL DEFAULT GETDATE(),
59
+ UNIQUE(address)
60
+ )
61
+ `,
62
+ mysql: () =>
63
+ sql`
64
+ CREATE TABLE IF NOT EXISTS ${runnersTableSql} (
65
+ machine_id INT AUTO_INCREMENT PRIMARY KEY,
66
+ address VARCHAR(255) NOT NULL,
67
+ runner TEXT NOT NULL,
68
+ healthy BOOLEAN NOT NULL DEFAULT TRUE,
69
+ last_heartbeat DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
70
+ UNIQUE(address)
71
+ )
72
+ `,
73
+ pg: () =>
74
+ sql`
75
+ CREATE TABLE IF NOT EXISTS ${runnersTableSql} (
76
+ machine_id SERIAL PRIMARY KEY,
77
+ address VARCHAR(255) NOT NULL,
78
+ runner TEXT NOT NULL,
79
+ healthy BOOLEAN NOT NULL DEFAULT TRUE,
80
+ last_heartbeat TIMESTAMP NOT NULL DEFAULT NOW(),
81
+ UNIQUE(address)
82
+ )
83
+ `,
84
+ orElse: () =>
85
+ // sqlite
86
+ sql`
87
+ CREATE TABLE IF NOT EXISTS ${runnersTableSql} (
88
+ machine_id INTEGER PRIMARY KEY AUTOINCREMENT,
89
+ address TEXT NOT NULL,
90
+ runner TEXT NOT NULL,
91
+ healthy INTEGER NOT NULL DEFAULT 1,
92
+ last_heartbeat DATETIME NOT NULL DEFAULT (CURRENT_TIMESTAMP),
93
+ UNIQUE(address)
94
+ )
95
+ `
96
+ })
97
+
98
+ const locksTable = table("locks")
99
+ const locksTableSql = sql(locksTable)
100
+
101
+ yield* sql.onDialectOrElse({
102
+ mssql: () =>
103
+ sql`
104
+ IF OBJECT_ID(N'${locksTableSql}', N'U') IS NULL
105
+ CREATE TABLE ${locksTableSql} (
106
+ shard_id VARCHAR(50) PRIMARY KEY,
107
+ address VARCHAR(255) NOT NULL,
108
+ acquired_at DATETIME NOT NULL
109
+ )
110
+ `,
111
+ mysql: () => Effect.void,
112
+ pg: () => Effect.void,
113
+ orElse: () =>
114
+ // sqlite
115
+ sql`
116
+ CREATE TABLE IF NOT EXISTS ${locksTableSql} (
117
+ shard_id TEXT PRIMARY KEY,
118
+ address TEXT NOT NULL,
119
+ acquired_at DATETIME NOT NULL
120
+ )
121
+ `
122
+ })
123
+
124
+ const sqlNowString = sql.onDialectOrElse({
125
+ pg: () => "NOW()",
126
+ mysql: () => "NOW()",
127
+ mssql: () => "GETDATE()",
128
+ orElse: () => "CURRENT_TIMESTAMP"
129
+ })
130
+ const sqlNow = sql.literal(sqlNowString)
131
+
132
+ const expiresSeconds = sql.literal(Math.ceil(Duration.toSeconds(config.shardLockExpiration)).toString())
133
+ const lockExpiresAt = sql.onDialectOrElse({
134
+ pg: () => sql`${sqlNow} - INTERVAL '${expiresSeconds} seconds'`,
135
+ mysql: () => sql`DATE_SUB(${sqlNow}, INTERVAL ${expiresSeconds} SECOND)`,
136
+ mssql: () => sql`DATEADD(SECOND, -${expiresSeconds}, ${sqlNow})`,
137
+ orElse: () => sql`datetime(${sqlNow}, '-${expiresSeconds} seconds')`
138
+ })
139
+
140
+ const encodeBoolean = sql.onDialectOrElse({
141
+ mssql: () => (b: boolean) => (b ? 1 : 0),
142
+ sqlite: () => (b: boolean) => (b ? 1 : 0),
143
+ orElse: () => (b: boolean) => b
144
+ })
145
+
146
+ // Upsert runner and return machine_id
147
+ const insertRunner = sql.onDialectOrElse({
148
+ mssql: () => (address: string, runner: string, healthy: boolean) =>
149
+ sql`
150
+ MERGE ${runnersTableSql} AS target
151
+ USING (SELECT ${address} AS address, ${runner} AS runner, ${sqlNow} AS last_heartbeat, ${
152
+ encodeBoolean(healthy)
153
+ } AS healthy) AS source
154
+ ON target.address = source.address
155
+ WHEN MATCHED THEN
156
+ UPDATE SET runner = source.runner, last_heartbeat = source.last_heartbeat, healthy = source.healthy
157
+ WHEN NOT MATCHED THEN
158
+ INSERT (address, runner, last_heartbeat, healthy)
159
+ VALUES (source.address, source.runner, source.last_heartbeat, source.healthy)
160
+ OUTPUT INSERTED.machine_id;
161
+ `.values,
162
+ mysql: () => (address: string, runner: string, healthy: boolean) =>
163
+ sql<{ machine_id: number }>`
164
+ INSERT INTO ${runnersTableSql} (address, runner, last_heartbeat, healthy)
165
+ VALUES (${address}, ${runner}, ${sqlNow}, ${healthy})
166
+ ON DUPLICATE KEY UPDATE
167
+ runner = VALUES(runner),
168
+ last_heartbeat = VALUES(last_heartbeat),
169
+ healthy = VALUES(healthy);
170
+ SELECT machine_id FROM ${runnersTableSql} WHERE address = ${address};
171
+ `.unprepared.pipe(
172
+ Effect.map((results: any) => [[results[1][0].machine_id]])
173
+ ),
174
+ pg: () => (address: string, runner: string, healthy: boolean) =>
175
+ sql`
176
+ INSERT INTO ${runnersTableSql} (address, runner, last_heartbeat, healthy)
177
+ VALUES (${address}, ${runner}, ${sqlNow}, ${healthy})
178
+ ON CONFLICT (address) DO UPDATE
179
+ SET runner = EXCLUDED.runner,
180
+ last_heartbeat = EXCLUDED.last_heartbeat,
181
+ healthy = EXCLUDED.healthy
182
+ RETURNING machine_id
183
+ `.values,
184
+ orElse: () => (address: string, runner: string, healthy: boolean) =>
185
+ // sqlite
186
+ sql`
187
+ INSERT INTO ${runnersTableSql} (address, runner, last_heartbeat, healthy)
188
+ VALUES (${address}, ${runner}, ${sqlNow}, ${encodeBoolean(healthy)})
189
+ ON CONFLICT(address) DO UPDATE SET
190
+ runner = excluded.runner,
191
+ last_heartbeat = excluded.last_heartbeat,
192
+ healthy = excluded.healthy
193
+ RETURNING machine_id;
194
+ `.values
195
+ })
196
+
197
+ const execWithLockConn = <A>(effect: Statement.Statement<A>): Effect.Effect<unknown, SqlError> => {
198
+ if (!lockConnRef) return effect
199
+ const [query, params] = effect.compile()
200
+ return ScopedRef.get(lockConnRef).pipe(
201
+ Effect.flatMap((conn) => conn.executeRaw(query, params)),
202
+ Effect.onError(() => resetLockConn)
203
+ )
204
+ }
205
+ const execWithLockConnValues = <A>(
206
+ effect: Statement.Statement<A>
207
+ ): Effect.Effect<ReadonlyArray<ReadonlyArray<any>>, SqlError> => {
208
+ if (!lockConnRef) return effect.values
209
+ const [query, params] = effect.compile()
210
+ return ScopedRef.get(lockConnRef).pipe(
211
+ Effect.flatMap((conn) => conn.executeValues(query, params)),
212
+ Effect.onError(() => resetLockConn)
213
+ )
214
+ }
215
+ const resetLockConn = sql.onDialectOrElse({
216
+ pg: () =>
217
+ Effect.gen(function*() {
218
+ const conn = yield* ScopedRef.get(lockConnRef!)
219
+ yield* Effect.ignore(conn.executeRaw("SELECT pg_advisory_unlock_all()", []))
220
+ yield* Effect.orDie(ScopedRef.set(lockConnRef!, sql.reserve))
221
+ }),
222
+ mysql: () =>
223
+ Effect.gen(function*() {
224
+ const conn = yield* ScopedRef.get(lockConnRef!)
225
+ yield* Effect.ignore(conn.executeRaw("SELECT RELEASE_ALL_LOCKS()", []))
226
+ yield* Effect.orDie(ScopedRef.set(lockConnRef!, sql.reserve))
227
+ }),
228
+ orElse: () => Effect.void
229
+ })
230
+
231
+ const acquireLock = sql.onDialectOrElse({
232
+ pg: () =>
233
+ Effect.fnUntraced(function*(_address: string, shardIds: ReadonlyArray<string>) {
234
+ const conn = yield* ScopedRef.get(lockConnRef!)
235
+ const acquiredShardIds: Array<string> = []
236
+ const toAcquire = new Map(shardIds.map((shardId) => [lockNumbers.get(shardId)!, shardId]))
237
+ const takenLocks = yield* conn.executeValues(
238
+ `SELECT objid FROM pg_locks WHERE locktype = 'advisory' AND granted = true AND pid = pg_backend_pid() ORDER BY objid`,
239
+ []
240
+ )
241
+ for (let i = 0; i < takenLocks.length; i++) {
242
+ const lockNum = takenLocks[i][0] as number
243
+ acquiredShardIds.push(lockNumbersReverse.get(lockNum)!)
244
+ toAcquire.delete(lockNum)
245
+ }
246
+ if (toAcquire.size === 0) {
247
+ return acquiredShardIds
248
+ }
249
+ const results = (yield* conn.executeUnprepared(`SELECT ${pgLocks(toAcquire)}`, [], undefined))[0] as Record<
250
+ string,
251
+ boolean
252
+ >
253
+ for (const shardId in results) {
254
+ if (results[shardId]) {
255
+ acquiredShardIds.push(shardId)
256
+ }
257
+ }
258
+ return acquiredShardIds
259
+ }, Effect.onError(() => resetLockConn)),
260
+
261
+ mysql: () =>
262
+ Effect.fnUntraced(function*(_address: string, shardIds: ReadonlyArray<string>) {
263
+ const conn = yield* ScopedRef.get(lockConnRef!)
264
+ const takenLocks = (yield* conn.executeUnprepared(`SELECT ${allMySqlTakenLocks}`, [], undefined))[0] as Record<
265
+ string,
266
+ 1 | null
267
+ >
268
+ const acquiredShardIds: Array<string> = []
269
+ const toAcquire: Array<string> = []
270
+ for (const shardId in takenLocks) {
271
+ if (takenLocks[shardId] === 1) {
272
+ acquiredShardIds.push(shardId)
273
+ } else if (shardIds.includes(shardId)) {
274
+ toAcquire.push(shardId)
275
+ }
276
+ }
277
+ if (toAcquire.length === 0) {
278
+ return acquiredShardIds
279
+ }
280
+ const results = (yield* conn.executeUnprepared(`SELECT ${mysqlLocks(toAcquire)}`, [], undefined))[0] as Record<
281
+ string,
282
+ number
283
+ >
284
+ for (const shardId in results) {
285
+ if (results[shardId] === 1) {
286
+ acquiredShardIds.push(shardId)
287
+ }
288
+ }
289
+ return acquiredShardIds
290
+ }, Effect.onError(() => resetLockConn)),
291
+
292
+ mssql: () => (address: string, shardIds: ReadonlyArray<string>) => {
293
+ const values = shardIds.map((shardId) => sql`(${stringLiteral(shardId)}, ${stringLiteral(address)}, ${sqlNow})`)
294
+ return sql`
295
+ MERGE ${locksTableSql} WITH (HOLDLOCK) AS target
296
+ USING (SELECT * FROM (VALUES ${sql.csv(values)})) AS source (shard_id, address, acquired_at)
297
+ ON target.shard_id = source.shard_id
298
+ WHEN MATCHED AND (target.address = source.address OR DATEDIFF(SECOND, target.acquired_at, ${sqlNow}) > ${expiresSeconds}) THEN
299
+ UPDATE SET address = source.address, acquired_at = source.acquired_at
300
+ WHEN NOT MATCHED THEN
301
+ INSERT (shard_id, address, acquired_at)
302
+ VALUES (source.shard_id, source.address, source.acquired_at);
303
+ `.pipe(
304
+ Effect.andThen(acquiredLocks(address, shardIds)),
305
+ sql.withTransaction
306
+ )
307
+ },
308
+
309
+ orElse: () => (address: string, shardIds: ReadonlyArray<string>) => {
310
+ const values = shardIds.map((shardId) => sql`(${stringLiteral(shardId)}, ${stringLiteral(address)}, ${sqlNow})`)
311
+ return sql`
312
+ WITH source(shard_id, address, acquired_at) AS (VALUES ${sql.csv(values)})
313
+ INSERT INTO ${locksTableSql} (shard_id, address, acquired_at)
314
+ SELECT source.shard_id, source.address, source.acquired_at
315
+ FROM source
316
+ WHERE NOT EXISTS (
317
+ SELECT 1 FROM ${locksTableSql}
318
+ WHERE shard_id = source.shard_id
319
+ AND address != ${address}
320
+ AND (strftime('%s', ${sqlNow}) - strftime('%s', acquired_at)) <= ${expiresSeconds}
321
+ )
322
+ ON CONFLICT(shard_id) DO UPDATE
323
+ SET address = ${address}, acquired_at = ${sqlNow}
324
+ `.pipe(
325
+ Effect.andThen(acquiredLocks(address, shardIds)),
326
+ sql.withTransaction
327
+ )
328
+ }
329
+ })
330
+
331
+ const lockNumbers = new Map<string, number>()
332
+ const lockNumbersReverse = new Map<number, string>()
333
+ for (let i = 0; i < config.shardGroups.length; i++) {
334
+ const group = config.shardGroups[i]
335
+ const base = (i + 1) * 1000000
336
+ for (let shard = 1; shard <= config.shardsPerGroup; shard++) {
337
+ const shardId = ShardId.make(group, shard).toString()
338
+ const lockNum = base + shard
339
+ lockNumbers.set(shardId, lockNum)
340
+ lockNumbersReverse.set(lockNum, shardId)
341
+ }
342
+ }
343
+
344
+ const lockNames = new Map<string, string>()
345
+ const lockNamesReverse = new Map<string, string>()
346
+ for (let i = 0; i < config.shardGroups.length; i++) {
347
+ const group = config.shardGroups[i]
348
+ for (let shard = 1; shard <= config.shardsPerGroup; shard++) {
349
+ const shardId = ShardId.make(group, shard).toString()
350
+ const lockName = `${prefix}.${shardId}`
351
+ lockNames.set(shardId, lockName)
352
+ lockNamesReverse.set(lockName, shardId)
353
+ }
354
+ }
355
+
356
+ const pgLocks = (shardIdsMap: Map<number, string>) =>
357
+ Array.from(
358
+ shardIdsMap.entries(),
359
+ ([lockNum, shardId]) => `pg_try_advisory_lock(${lockNum}) AS "${shardId}"`
360
+ ).join(", ")
361
+
362
+ const mysqlLocks = (shardIds: ReadonlyArray<string>) =>
363
+ shardIds.map((shardId) => `GET_LOCK('${lockNames.get(shardId)!}', 0) AS "${shardId}"`).join(", ")
364
+
365
+ const allMySqlTakenLocks = Array.from(
366
+ lockNames.entries(),
367
+ ([shardId, lockName]) => `IS_USED_LOCK('${lockName}') = CONNECTION_ID() AS "${shardId}"`
368
+ ).join(", ")
369
+
370
+ const acquiredLocks = (address: string, shardIds: ReadonlyArray<string>) =>
371
+ sql<{ shard_id: string }>`
372
+ SELECT shard_id FROM ${sql(locksTable)}
373
+ WHERE address = ${address}
374
+ AND acquired_at >= ${lockExpiresAt}
375
+ AND shard_id IN ${stringLiteralArr(shardIds)}
376
+ `.values.pipe(
377
+ Effect.map((rows) => rows.map((row) => row[0] as string))
378
+ )
379
+
380
+ const wrapString = sql.onDialectOrElse({
381
+ mssql: () => (s: string) => `N'${s}'`,
382
+ orElse: () => (s: string) => `'${s}'`
383
+ })
384
+ const stringLiteral = (s: string) => sql.literal(wrapString(s))
385
+ const stringLiteralArr = (arr: ReadonlyArray<string>) => sql.literal(`(${arr.map(wrapString).join(",")})`)
386
+
387
+ const refreshShards = sql.onDialectOrElse({
388
+ pg: () => acquireLock,
389
+ mysql: () => acquireLock,
390
+ mssql: () => (address: string, shardIds: ReadonlyArray<string>) =>
391
+ sql`
392
+ UPDATE ${locksTableSql}
393
+ SET acquired_at = ${sqlNow}
394
+ OUTPUT inserted.shard_id
395
+ WHERE address = ${address} AND shard_id IN ${stringLiteralArr(shardIds)}
396
+ `.pipe(execWithLockConnValues, Effect.map((rows) => rows.map((row) => row[0] as string))),
397
+ orElse: () => (address: string, shardIds: ReadonlyArray<string>) =>
398
+ sql`
399
+ UPDATE ${locksTableSql}
400
+ SET acquired_at = ${sqlNow}
401
+ WHERE address = ${address} AND shard_id IN ${stringLiteralArr(shardIds)}
402
+ RETURNING shard_id
403
+ `.pipe(execWithLockConnValues, Effect.map((rows) => rows.map((row) => row[0] as string)))
404
+ })
405
+
406
+ return RunnerStorage.makeEncoded({
407
+ getRunners: sql`SELECT runner, healthy FROM ${runnersTableSql} WHERE last_heartbeat > ${lockExpiresAt}`.values.pipe(
408
+ PersistenceError.refail,
409
+ Effect.map(Arr.map(([runner, healthy]) => [String(runner), Boolean(healthy)] as const)),
410
+ withTracerDisabled
411
+ ),
412
+
413
+ register: (address, runner, healthy) =>
414
+ insertRunner(address, runner, healthy).pipe(
415
+ Effect.map((rows: any) => Number(rows[0][0])),
416
+ PersistenceError.refail,
417
+ withTracerDisabled
418
+ ),
419
+
420
+ unregister: (address) =>
421
+ sql`DELETE FROM ${runnersTableSql} WHERE address = ${address} OR last_heartbeat < ${lockExpiresAt}`.pipe(
422
+ Effect.asVoid,
423
+ PersistenceError.refail,
424
+ withTracerDisabled
425
+ ),
426
+
427
+ setRunnerHealth: (address, healthy) =>
428
+ sql`UPDATE ${runnersTableSql} SET healthy = ${encodeBoolean(healthy)} WHERE address = ${address}`
429
+ .pipe(
430
+ Effect.asVoid,
431
+ PersistenceError.refail,
432
+ withTracerDisabled
433
+ ),
434
+
435
+ acquire: (address, shardIds) =>
436
+ acquireLock(address, shardIds).pipe(
437
+ PersistenceError.refail,
438
+ withTracerDisabled
439
+ ),
440
+
441
+ refresh: (address, shardIds) =>
442
+ sql`UPDATE ${runnersTableSql} SET last_heartbeat = ${sqlNow} WHERE address = ${address}`.pipe(
443
+ execWithLockConn,
444
+ shardIds.length > 0 ?
445
+ Effect.andThen(refreshShards(address, shardIds)) :
446
+ Effect.as([]),
447
+ PersistenceError.refail
448
+ ),
449
+
450
+ release: sql.onDialectOrElse({
451
+ pg: () =>
452
+ Effect.fnUntraced(
453
+ function*(_address, shardId) {
454
+ const lockNum = lockNumbers.get(shardId)!
455
+ const conn = yield* ScopedRef.get(lockConnRef!)
456
+ const release = conn.executeRaw(`SELECT pg_advisory_unlock(${lockNum})`, [])
457
+ const check = conn.executeValues(
458
+ `SELECT 1 FROM pg_locks WHERE locktype = 'advisory' AND granted = true AND pid = pg_backend_pid() AND objid = ${lockNum}`,
459
+ []
460
+ )
461
+ while (true) {
462
+ yield* release
463
+ const takenLocks = yield* check
464
+ if (takenLocks.length === 0) return
465
+ }
466
+ },
467
+ Effect.onError(() => resetLockConn),
468
+ Effect.asVoid,
469
+ PersistenceError.refail,
470
+ withTracerDisabled
471
+ ),
472
+ mysql: () =>
473
+ Effect.fnUntraced(
474
+ function*(_address, shardId) {
475
+ const conn = yield* ScopedRef.get(lockConnRef!)
476
+ const lockName = lockNames.get(shardId)!
477
+ const release = conn.executeRaw(`SELECT RELEASE_LOCK('${lockName}')`, [])
478
+ const check = conn.executeValues(
479
+ `SELECT IS_USED_LOCK('${lockName}') = CONNECTION_ID() AS is_taken`,
480
+ []
481
+ )
482
+ while (true) {
483
+ yield* release
484
+ const takenLocks = yield* check
485
+ if (takenLocks.length === 0 || takenLocks[0][0] !== 1) return
486
+ }
487
+ },
488
+ Effect.onError(() => resetLockConn),
489
+ Effect.asVoid,
490
+ PersistenceError.refail,
491
+ withTracerDisabled
492
+ ),
493
+ orElse: () => (address, shardId) =>
494
+ sql`DELETE FROM ${locksTableSql} WHERE address = ${address} AND shard_id = ${shardId}`.pipe(
495
+ PersistenceError.refail,
496
+ withTracerDisabled
497
+ )
498
+ }),
499
+
500
+ releaseAll: sql.onDialectOrElse({
501
+ pg: () => (_address) =>
502
+ sql`SELECT pg_advisory_unlock_all()`.pipe(
503
+ execWithLockConn,
504
+ Effect.asVoid,
505
+ PersistenceError.refail,
506
+ withTracerDisabled
507
+ ),
508
+ mysql: () => (_address) =>
509
+ sql`SELECT RELEASE_ALL_LOCKS()`.pipe(
510
+ execWithLockConn,
511
+ Effect.asVoid,
512
+ PersistenceError.refail,
513
+ withTracerDisabled
514
+ ),
515
+ orElse: () => (address) =>
516
+ sql`DELETE FROM ${locksTableSql} WHERE address = ${address}`.pipe(
517
+ PersistenceError.refail,
518
+ withTracerDisabled
519
+ )
520
+ })
521
+ })
522
+ }, withTracerDisabled)
523
+
524
+ /**
525
+ * @since 1.0.0
526
+ * @category Layers
527
+ */
528
+ export const layer: Layer.Layer<
529
+ RunnerStorage.RunnerStorage,
530
+ SqlError,
531
+ SqlClient.SqlClient | ShardingConfig.ShardingConfig
532
+ > = Layer.scoped(RunnerStorage.RunnerStorage)(make({}))
533
+
534
+ /**
535
+ * @since 1.0.0
536
+ * @category Layers
537
+ */
538
+ export const layerWith = (options: {
539
+ readonly prefix?: string | undefined
540
+ }): Layer.Layer<RunnerStorage.RunnerStorage, SqlError, SqlClient.SqlClient | ShardingConfig.ShardingConfig> =>
541
+ Layer.scoped(RunnerStorage.RunnerStorage)(make(options))
package/src/index.ts CHANGED
@@ -63,21 +63,11 @@ export * as EntityType from "./EntityType.js"
63
63
  */
64
64
  export * as Envelope from "./Envelope.js"
65
65
 
66
- /**
67
- * @since 1.0.0
68
- */
69
- export * as HttpCommon from "./HttpCommon.js"
70
-
71
66
  /**
72
67
  * @since 1.0.0
73
68
  */
74
69
  export * as HttpRunner from "./HttpRunner.js"
75
70
 
76
- /**
77
- * @since 1.0.0
78
- */
79
- export * as HttpShardManager from "./HttpShardManager.js"
80
-
81
71
  /**
82
72
  * @since 1.0.0
83
73
  */
@@ -121,22 +111,17 @@ export * as RunnerServer from "./RunnerServer.js"
121
111
  /**
122
112
  * @since 1.0.0
123
113
  */
124
- export * as Runners from "./Runners.js"
125
-
126
- /**
127
- * @since 1.0.0
128
- */
129
- export * as ShardId from "./ShardId.js"
114
+ export * as RunnerStorage from "./RunnerStorage.js"
130
115
 
131
116
  /**
132
117
  * @since 1.0.0
133
118
  */
134
- export * as ShardManager from "./ShardManager.js"
119
+ export * as Runners from "./Runners.js"
135
120
 
136
121
  /**
137
122
  * @since 1.0.0
138
123
  */
139
- export * as ShardStorage from "./ShardStorage.js"
124
+ export * as ShardId from "./ShardId.js"
140
125
 
141
126
  /**
142
127
  * @since 1.0.0
@@ -173,11 +158,6 @@ export * as Snowflake from "./Snowflake.js"
173
158
  */
174
159
  export * as SocketRunner from "./SocketRunner.js"
175
160
 
176
- /**
177
- * @since 1.0.0
178
- */
179
- export * as SocketShardManager from "./SocketShardManager.js"
180
-
181
161
  /**
182
162
  * @since 1.0.0
183
163
  */
@@ -186,9 +166,4 @@ export * as SqlMessageStorage from "./SqlMessageStorage.js"
186
166
  /**
187
167
  * @since 1.0.0
188
168
  */
189
- export * as SqlShardStorage from "./SqlShardStorage.js"
190
-
191
- /**
192
- * @since 1.0.0
193
- */
194
- export * as SynchronizedClock from "./SynchronizedClock.js"
169
+ export * as SqlRunnerStorage from "./SqlRunnerStorage.js"