@effect/cluster 0.50.6 → 0.52.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (232) hide show
  1. package/RunnerStorage/package.json +6 -0
  2. package/SqlRunnerStorage/package.json +6 -0
  3. package/dist/cjs/ClusterError.js +2 -24
  4. package/dist/cjs/ClusterError.js.map +1 -1
  5. package/dist/cjs/ClusterMetrics.js +13 -15
  6. package/dist/cjs/ClusterMetrics.js.map +1 -1
  7. package/dist/cjs/ClusterSchema.js +17 -2
  8. package/dist/cjs/ClusterSchema.js.map +1 -1
  9. package/dist/cjs/ClusterWorkflowEngine.js +50 -83
  10. package/dist/cjs/ClusterWorkflowEngine.js.map +1 -1
  11. package/dist/cjs/Entity.js +1 -13
  12. package/dist/cjs/Entity.js.map +1 -1
  13. package/dist/cjs/EntityAddress.js +9 -1
  14. package/dist/cjs/EntityAddress.js.map +1 -1
  15. package/dist/cjs/EntityId.js +7 -1
  16. package/dist/cjs/EntityId.js.map +1 -1
  17. package/dist/cjs/EntityProxy.js +1 -1
  18. package/dist/cjs/EntityProxy.js.map +1 -1
  19. package/dist/cjs/HttpRunner.js +69 -43
  20. package/dist/cjs/HttpRunner.js.map +1 -1
  21. package/dist/cjs/MessageStorage.js +64 -16
  22. package/dist/cjs/MessageStorage.js.map +1 -1
  23. package/dist/cjs/Runner.js +3 -3
  24. package/dist/cjs/Runner.js.map +1 -1
  25. package/dist/cjs/RunnerAddress.js +7 -0
  26. package/dist/cjs/RunnerAddress.js.map +1 -1
  27. package/dist/cjs/RunnerHealth.js +91 -32
  28. package/dist/cjs/RunnerHealth.js.map +1 -1
  29. package/dist/cjs/RunnerServer.js +38 -24
  30. package/dist/cjs/RunnerServer.js.map +1 -1
  31. package/dist/cjs/RunnerStorage.js +100 -0
  32. package/dist/cjs/RunnerStorage.js.map +1 -0
  33. package/dist/cjs/Runners.js +18 -22
  34. package/dist/cjs/Runners.js.map +1 -1
  35. package/dist/cjs/ShardId.js +17 -7
  36. package/dist/cjs/ShardId.js.map +1 -1
  37. package/dist/cjs/Sharding.js +444 -320
  38. package/dist/cjs/Sharding.js.map +1 -1
  39. package/dist/cjs/ShardingConfig.js +10 -14
  40. package/dist/cjs/ShardingConfig.js.map +1 -1
  41. package/dist/cjs/Snowflake.js +1 -1
  42. package/dist/cjs/SocketRunner.js +1 -1
  43. package/dist/cjs/SocketRunner.js.map +1 -1
  44. package/dist/cjs/SqlMessageStorage.js +22 -28
  45. package/dist/cjs/SqlMessageStorage.js.map +1 -1
  46. package/dist/cjs/SqlRunnerStorage.js +375 -0
  47. package/dist/cjs/SqlRunnerStorage.js.map +1 -0
  48. package/dist/cjs/index.js +5 -15
  49. package/dist/cjs/internal/entityManager.js +42 -23
  50. package/dist/cjs/internal/entityManager.js.map +1 -1
  51. package/dist/dts/ClusterError.d.ts +0 -22
  52. package/dist/dts/ClusterError.d.ts.map +1 -1
  53. package/dist/dts/ClusterMetrics.d.ts +4 -14
  54. package/dist/dts/ClusterMetrics.d.ts.map +1 -1
  55. package/dist/dts/ClusterSchema.d.ts +9 -1
  56. package/dist/dts/ClusterSchema.d.ts.map +1 -1
  57. package/dist/dts/ClusterWorkflowEngine.d.ts.map +1 -1
  58. package/dist/dts/Entity.d.ts +3 -14
  59. package/dist/dts/Entity.d.ts.map +1 -1
  60. package/dist/dts/EntityAddress.d.ts +11 -0
  61. package/dist/dts/EntityAddress.d.ts.map +1 -1
  62. package/dist/dts/EntityId.d.ts +5 -0
  63. package/dist/dts/EntityId.d.ts.map +1 -1
  64. package/dist/dts/EntityProxy.d.ts +5 -6
  65. package/dist/dts/EntityProxy.d.ts.map +1 -1
  66. package/dist/dts/HttpRunner.d.ts +48 -25
  67. package/dist/dts/HttpRunner.d.ts.map +1 -1
  68. package/dist/dts/MessageStorage.d.ts +13 -5
  69. package/dist/dts/MessageStorage.d.ts.map +1 -1
  70. package/dist/dts/Runner.d.ts +4 -4
  71. package/dist/dts/Runner.d.ts.map +1 -1
  72. package/dist/dts/RunnerAddress.d.ts +5 -0
  73. package/dist/dts/RunnerAddress.d.ts.map +1 -1
  74. package/dist/dts/RunnerHealth.d.ts +24 -16
  75. package/dist/dts/RunnerHealth.d.ts.map +1 -1
  76. package/dist/dts/RunnerServer.d.ts +5 -4
  77. package/dist/dts/RunnerServer.d.ts.map +1 -1
  78. package/dist/dts/{ShardStorage.d.ts → RunnerStorage.d.ts} +41 -54
  79. package/dist/dts/RunnerStorage.d.ts.map +1 -0
  80. package/dist/dts/Runners.d.ts +15 -11
  81. package/dist/dts/Runners.d.ts.map +1 -1
  82. package/dist/dts/ShardId.d.ts +1 -1
  83. package/dist/dts/ShardId.d.ts.map +1 -1
  84. package/dist/dts/Sharding.d.ts +20 -10
  85. package/dist/dts/Sharding.d.ts.map +1 -1
  86. package/dist/dts/ShardingConfig.d.ts +40 -14
  87. package/dist/dts/ShardingConfig.d.ts.map +1 -1
  88. package/dist/dts/SocketRunner.d.ts +4 -3
  89. package/dist/dts/SocketRunner.d.ts.map +1 -1
  90. package/dist/dts/SqlMessageStorage.d.ts +2 -3
  91. package/dist/dts/SqlMessageStorage.d.ts.map +1 -1
  92. package/dist/dts/SqlRunnerStorage.d.ts +40 -0
  93. package/dist/dts/SqlRunnerStorage.d.ts.map +1 -0
  94. package/dist/dts/index.d.ts +4 -24
  95. package/dist/dts/index.d.ts.map +1 -1
  96. package/dist/esm/ClusterError.js +0 -21
  97. package/dist/esm/ClusterError.js.map +1 -1
  98. package/dist/esm/ClusterMetrics.js +12 -14
  99. package/dist/esm/ClusterMetrics.js.map +1 -1
  100. package/dist/esm/ClusterSchema.js +17 -2
  101. package/dist/esm/ClusterSchema.js.map +1 -1
  102. package/dist/esm/ClusterWorkflowEngine.js +50 -83
  103. package/dist/esm/ClusterWorkflowEngine.js.map +1 -1
  104. package/dist/esm/Entity.js +0 -12
  105. package/dist/esm/Entity.js.map +1 -1
  106. package/dist/esm/EntityAddress.js +7 -0
  107. package/dist/esm/EntityAddress.js.map +1 -1
  108. package/dist/esm/EntityId.js +5 -0
  109. package/dist/esm/EntityId.js.map +1 -1
  110. package/dist/esm/EntityProxy.js +2 -2
  111. package/dist/esm/EntityProxy.js.map +1 -1
  112. package/dist/esm/HttpRunner.js +62 -39
  113. package/dist/esm/HttpRunner.js.map +1 -1
  114. package/dist/esm/MessageStorage.js +65 -17
  115. package/dist/esm/MessageStorage.js.map +1 -1
  116. package/dist/esm/Runner.js +3 -3
  117. package/dist/esm/Runner.js.map +1 -1
  118. package/dist/esm/RunnerAddress.js +7 -0
  119. package/dist/esm/RunnerAddress.js.map +1 -1
  120. package/dist/esm/RunnerHealth.js +88 -30
  121. package/dist/esm/RunnerHealth.js.map +1 -1
  122. package/dist/esm/RunnerServer.js +38 -24
  123. package/dist/esm/RunnerServer.js.map +1 -1
  124. package/dist/esm/RunnerStorage.js +90 -0
  125. package/dist/esm/RunnerStorage.js.map +1 -0
  126. package/dist/esm/Runners.js +19 -23
  127. package/dist/esm/Runners.js.map +1 -1
  128. package/dist/esm/ShardId.js +16 -6
  129. package/dist/esm/ShardId.js.map +1 -1
  130. package/dist/esm/Sharding.js +447 -323
  131. package/dist/esm/Sharding.js.map +1 -1
  132. package/dist/esm/ShardingConfig.js +10 -14
  133. package/dist/esm/ShardingConfig.js.map +1 -1
  134. package/dist/esm/Snowflake.js +1 -1
  135. package/dist/esm/SocketRunner.js +1 -1
  136. package/dist/esm/SocketRunner.js.map +1 -1
  137. package/dist/esm/SqlMessageStorage.js +22 -28
  138. package/dist/esm/SqlMessageStorage.js.map +1 -1
  139. package/dist/esm/SqlRunnerStorage.js +366 -0
  140. package/dist/esm/SqlRunnerStorage.js.map +1 -0
  141. package/dist/esm/index.js +4 -24
  142. package/dist/esm/index.js.map +1 -1
  143. package/dist/esm/internal/entityManager.js +41 -22
  144. package/dist/esm/internal/entityManager.js.map +1 -1
  145. package/package.json +20 -60
  146. package/src/ClusterError.ts +0 -24
  147. package/src/ClusterMetrics.ts +12 -16
  148. package/src/ClusterSchema.ts +17 -2
  149. package/src/ClusterWorkflowEngine.ts +48 -80
  150. package/src/Entity.ts +3 -21
  151. package/src/EntityAddress.ts +10 -0
  152. package/src/EntityId.ts +6 -0
  153. package/src/EntityProxy.ts +10 -10
  154. package/src/HttpRunner.ts +132 -67
  155. package/src/MessageStorage.ts +89 -24
  156. package/src/Runner.ts +4 -4
  157. package/src/RunnerAddress.ts +8 -0
  158. package/src/RunnerHealth.ts +119 -56
  159. package/src/RunnerServer.ts +64 -47
  160. package/src/RunnerStorage.ts +218 -0
  161. package/src/Runners.ts +32 -45
  162. package/src/ShardId.ts +14 -3
  163. package/src/Sharding.ts +561 -417
  164. package/src/ShardingConfig.ts +39 -31
  165. package/src/Snowflake.ts +1 -1
  166. package/src/SocketRunner.ts +6 -4
  167. package/src/SqlMessageStorage.ts +28 -30
  168. package/src/SqlRunnerStorage.ts +537 -0
  169. package/src/index.ts +4 -29
  170. package/src/internal/entityManager.ts +45 -29
  171. package/HttpCommon/package.json +0 -6
  172. package/HttpShardManager/package.json +0 -6
  173. package/ShardManager/package.json +0 -6
  174. package/ShardStorage/package.json +0 -6
  175. package/SocketShardManager/package.json +0 -6
  176. package/SqlShardStorage/package.json +0 -6
  177. package/SynchronizedClock/package.json +0 -6
  178. package/dist/cjs/HttpCommon.js +0 -48
  179. package/dist/cjs/HttpCommon.js.map +0 -1
  180. package/dist/cjs/HttpShardManager.js +0 -139
  181. package/dist/cjs/HttpShardManager.js.map +0 -1
  182. package/dist/cjs/ShardManager.js +0 -549
  183. package/dist/cjs/ShardManager.js.map +0 -1
  184. package/dist/cjs/ShardStorage.js +0 -151
  185. package/dist/cjs/ShardStorage.js.map +0 -1
  186. package/dist/cjs/SocketShardManager.js +0 -32
  187. package/dist/cjs/SocketShardManager.js.map +0 -1
  188. package/dist/cjs/SqlShardStorage.js +0 -253
  189. package/dist/cjs/SqlShardStorage.js.map +0 -1
  190. package/dist/cjs/SynchronizedClock.js +0 -65
  191. package/dist/cjs/SynchronizedClock.js.map +0 -1
  192. package/dist/cjs/internal/shardManager.js +0 -353
  193. package/dist/cjs/internal/shardManager.js.map +0 -1
  194. package/dist/dts/HttpCommon.d.ts +0 -25
  195. package/dist/dts/HttpCommon.d.ts.map +0 -1
  196. package/dist/dts/HttpShardManager.d.ts +0 -119
  197. package/dist/dts/HttpShardManager.d.ts.map +0 -1
  198. package/dist/dts/ShardManager.d.ts +0 -459
  199. package/dist/dts/ShardManager.d.ts.map +0 -1
  200. package/dist/dts/ShardStorage.d.ts.map +0 -1
  201. package/dist/dts/SocketShardManager.d.ts +0 -17
  202. package/dist/dts/SocketShardManager.d.ts.map +0 -1
  203. package/dist/dts/SqlShardStorage.d.ts +0 -38
  204. package/dist/dts/SqlShardStorage.d.ts.map +0 -1
  205. package/dist/dts/SynchronizedClock.d.ts +0 -19
  206. package/dist/dts/SynchronizedClock.d.ts.map +0 -1
  207. package/dist/dts/internal/shardManager.d.ts +0 -2
  208. package/dist/dts/internal/shardManager.d.ts.map +0 -1
  209. package/dist/esm/HttpCommon.js +0 -38
  210. package/dist/esm/HttpCommon.js.map +0 -1
  211. package/dist/esm/HttpShardManager.js +0 -128
  212. package/dist/esm/HttpShardManager.js.map +0 -1
  213. package/dist/esm/ShardManager.js +0 -535
  214. package/dist/esm/ShardManager.js.map +0 -1
  215. package/dist/esm/ShardStorage.js +0 -141
  216. package/dist/esm/ShardStorage.js.map +0 -1
  217. package/dist/esm/SocketShardManager.js +0 -24
  218. package/dist/esm/SocketShardManager.js.map +0 -1
  219. package/dist/esm/SqlShardStorage.js +0 -244
  220. package/dist/esm/SqlShardStorage.js.map +0 -1
  221. package/dist/esm/SynchronizedClock.js +0 -57
  222. package/dist/esm/SynchronizedClock.js.map +0 -1
  223. package/dist/esm/internal/shardManager.js +0 -342
  224. package/dist/esm/internal/shardManager.js.map +0 -1
  225. package/src/HttpCommon.ts +0 -73
  226. package/src/HttpShardManager.ts +0 -273
  227. package/src/ShardManager.ts +0 -823
  228. package/src/ShardStorage.ts +0 -297
  229. package/src/SocketShardManager.ts +0 -48
  230. package/src/SqlShardStorage.ts +0 -329
  231. package/src/SynchronizedClock.ts +0 -82
  232. package/src/internal/shardManager.ts +0 -412
@@ -0,0 +1,537 @@
1
+ /**
2
+ * @since 1.0.0
3
+ */
4
+ import * as SqlClient from "@effect/sql/SqlClient"
5
+ import type { SqlError } from "@effect/sql/SqlError"
6
+ import type * as Statement from "@effect/sql/Statement"
7
+ import * as Arr from "effect/Array"
8
+ import * as Duration from "effect/Duration"
9
+ import * as Effect from "effect/Effect"
10
+ import * as Layer from "effect/Layer"
11
+ import * as Resource from "effect/Resource"
12
+ import { PersistenceError } from "./ClusterError.js"
13
+ import * as RunnerStorage from "./RunnerStorage.js"
14
+ import * as ShardId from "./ShardId.js"
15
+ import * as ShardingConfig from "./ShardingConfig.js"
16
+
17
+ const withTracerDisabled = Effect.withTracerEnabled(false)
18
+
19
+ /**
20
+ * @since 1.0.0
21
+ * @category Constructors
22
+ */
23
+ export const make = Effect.fnUntraced(function*(options: {
24
+ readonly prefix?: string | undefined
25
+ }) {
26
+ const config = yield* ShardingConfig.ShardingConfig
27
+ const sql = (yield* SqlClient.SqlClient).withoutTransforms()
28
+ const prefix = options?.prefix ?? "cluster"
29
+ const table = (name: string) => `${prefix}_${name}`
30
+ const acquireLockConn = sql.onDialectOrElse({
31
+ pg: () =>
32
+ Effect.gen(function*() {
33
+ const conn = yield* Effect.orDie(sql.reserve)
34
+ yield* Effect.addFinalizer(() => Effect.orDie(conn.executeRaw("SELECT pg_advisory_unlock_all()", [])))
35
+ return conn
36
+ }),
37
+ mysql: () =>
38
+ Effect.gen(function*() {
39
+ const conn = yield* Effect.orDie(sql.reserve)
40
+ yield* Effect.addFinalizer(() => Effect.orDie(conn.executeRaw("SELECT RELEASE_ALL_LOCKS()", [])))
41
+ return conn
42
+ }),
43
+ orElse: () => undefined
44
+ })
45
+ const lockConnRef = acquireLockConn && (yield* Resource.manual(acquireLockConn))
46
+
47
+ const runnersTable = table("runners")
48
+ const runnersTableSql = sql(runnersTable)
49
+
50
+ // Migrate old tables if they exist
51
+ // TODO: Remove in next major version
52
+ const hasOldTables = yield* sql`SELECT shard_id FROM ${sql(table("shards"))} LIMIT 1`.pipe(
53
+ Effect.isSuccess
54
+ )
55
+ if (hasOldTables) {
56
+ yield* sql`DROP TABLE ${sql(table("shards"))}`.pipe(Effect.ignore)
57
+ yield* sql`DROP TABLE ${runnersTableSql}`.pipe(Effect.ignore)
58
+ }
59
+
60
+ yield* sql.onDialectOrElse({
61
+ mssql: () =>
62
+ sql`
63
+ IF OBJECT_ID(N'${runnersTableSql}', N'U') IS NULL
64
+ CREATE TABLE ${runnersTableSql} (
65
+ machine_id INT IDENTITY PRIMARY KEY,
66
+ address VARCHAR(255) NOT NULL,
67
+ runner TEXT NOT NULL,
68
+ healthy BIT NOT NULL DEFAULT 1,
69
+ last_heartbeat DATETIME NOT NULL DEFAULT GETDATE(),
70
+ UNIQUE(address)
71
+ )
72
+ `,
73
+ mysql: () =>
74
+ sql`
75
+ CREATE TABLE IF NOT EXISTS ${runnersTableSql} (
76
+ machine_id INT AUTO_INCREMENT PRIMARY KEY,
77
+ address VARCHAR(255) NOT NULL,
78
+ runner TEXT NOT NULL,
79
+ healthy BOOLEAN NOT NULL DEFAULT TRUE,
80
+ last_heartbeat DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
81
+ UNIQUE(address)
82
+ )
83
+ `,
84
+ pg: () =>
85
+ sql`
86
+ CREATE TABLE IF NOT EXISTS ${runnersTableSql} (
87
+ machine_id SERIAL PRIMARY KEY,
88
+ address VARCHAR(255) NOT NULL,
89
+ runner TEXT NOT NULL,
90
+ healthy BOOLEAN NOT NULL DEFAULT TRUE,
91
+ last_heartbeat TIMESTAMP NOT NULL DEFAULT NOW(),
92
+ UNIQUE(address)
93
+ )
94
+ `,
95
+ orElse: () =>
96
+ // sqlite
97
+ sql`
98
+ CREATE TABLE IF NOT EXISTS ${runnersTableSql} (
99
+ machine_id INTEGER PRIMARY KEY AUTOINCREMENT,
100
+ address TEXT NOT NULL,
101
+ runner TEXT NOT NULL,
102
+ healthy INTEGER NOT NULL DEFAULT 1,
103
+ last_heartbeat DATETIME NOT NULL DEFAULT (CURRENT_TIMESTAMP),
104
+ UNIQUE(address)
105
+ )
106
+ `
107
+ })
108
+
109
+ const locksTable = table("locks")
110
+ const locksTableSql = sql(locksTable)
111
+
112
+ yield* sql.onDialectOrElse({
113
+ mssql: () =>
114
+ sql`
115
+ IF OBJECT_ID(N'${locksTableSql}', N'U') IS NULL
116
+ CREATE TABLE ${locksTableSql} (
117
+ shard_id VARCHAR(50) PRIMARY KEY,
118
+ address VARCHAR(255) NOT NULL,
119
+ acquired_at DATETIME NOT NULL
120
+ )
121
+ `,
122
+ mysql: () => Effect.void,
123
+ pg: () => Effect.void,
124
+ orElse: () =>
125
+ // sqlite
126
+ sql`
127
+ CREATE TABLE IF NOT EXISTS ${locksTableSql} (
128
+ shard_id TEXT PRIMARY KEY,
129
+ address TEXT NOT NULL,
130
+ acquired_at DATETIME NOT NULL
131
+ )
132
+ `
133
+ })
134
+
135
+ const sqlNowString = sql.onDialectOrElse({
136
+ pg: () => "NOW()",
137
+ mysql: () => "NOW()",
138
+ mssql: () => "GETDATE()",
139
+ orElse: () => "CURRENT_TIMESTAMP"
140
+ })
141
+ const sqlNow = sql.literal(sqlNowString)
142
+
143
+ const expiresSeconds = sql.literal(Math.ceil(Duration.toSeconds(config.shardLockExpiration)).toString())
144
+ const lockExpiresAt = sql.onDialectOrElse({
145
+ pg: () => sql`${sqlNow} - INTERVAL '${expiresSeconds} seconds'`,
146
+ mysql: () => sql`DATE_SUB(${sqlNow}, INTERVAL ${expiresSeconds} SECOND)`,
147
+ mssql: () => sql`DATEADD(SECOND, -${expiresSeconds}, ${sqlNow})`,
148
+ orElse: () => sql`datetime(${sqlNow}, '-${expiresSeconds} seconds')`
149
+ })
150
+
151
+ const encodeBoolean = sql.onDialectOrElse({
152
+ mssql: () => (b: boolean) => (b ? 1 : 0),
153
+ sqlite: () => (b: boolean) => (b ? 1 : 0),
154
+ orElse: () => (b: boolean) => b
155
+ })
156
+
157
+ // Upsert runner and return machine_id
158
+ const insertRunner = sql.onDialectOrElse({
159
+ mssql: () => (address: string, runner: string, healthy: boolean) =>
160
+ sql`
161
+ MERGE ${runnersTableSql} AS target
162
+ USING (SELECT ${address} AS address, ${runner} AS runner, ${sqlNow} AS last_heartbeat, ${
163
+ encodeBoolean(healthy)
164
+ } AS healthy) AS source
165
+ ON target.address = source.address
166
+ WHEN MATCHED THEN
167
+ UPDATE SET runner = source.runner, last_heartbeat = source.last_heartbeat, healthy = source.healthy
168
+ WHEN NOT MATCHED THEN
169
+ INSERT (address, runner, last_heartbeat, healthy)
170
+ VALUES (source.address, source.runner, source.last_heartbeat, source.healthy)
171
+ OUTPUT INSERTED.machine_id;
172
+ `.values,
173
+ mysql: () => (address: string, runner: string, healthy: boolean) =>
174
+ sql<{ machine_id: number }>`
175
+ INSERT INTO ${runnersTableSql} (address, runner, last_heartbeat, healthy)
176
+ VALUES (${address}, ${runner}, ${sqlNow}, ${healthy})
177
+ ON DUPLICATE KEY UPDATE
178
+ runner = VALUES(runner),
179
+ last_heartbeat = VALUES(last_heartbeat),
180
+ healthy = VALUES(healthy);
181
+ SELECT machine_id FROM ${runnersTableSql} WHERE address = ${address};
182
+ `.unprepared.pipe(
183
+ Effect.map((results: any) => [[results[1][0].machine_id]])
184
+ ),
185
+ pg: () => (address: string, runner: string, healthy: boolean) =>
186
+ sql`
187
+ INSERT INTO ${runnersTableSql} (address, runner, last_heartbeat, healthy)
188
+ VALUES (${address}, ${runner}, ${sqlNow}, ${healthy})
189
+ ON CONFLICT (address) DO UPDATE
190
+ SET runner = EXCLUDED.runner,
191
+ last_heartbeat = EXCLUDED.last_heartbeat,
192
+ healthy = EXCLUDED.healthy
193
+ RETURNING machine_id
194
+ `.values,
195
+ orElse: () => (address: string, runner: string, healthy: boolean) =>
196
+ // sqlite
197
+ sql`
198
+ INSERT INTO ${runnersTableSql} (address, runner, last_heartbeat, healthy)
199
+ VALUES (${address}, ${runner}, ${sqlNow}, ${encodeBoolean(healthy)})
200
+ ON CONFLICT(address) DO UPDATE SET
201
+ runner = excluded.runner,
202
+ last_heartbeat = excluded.last_heartbeat,
203
+ healthy = excluded.healthy
204
+ RETURNING machine_id;
205
+ `.values
206
+ })
207
+
208
+ const execWithLockConn = <A>(effect: Statement.Statement<A>): Effect.Effect<unknown, SqlError> => {
209
+ if (!lockConnRef) return effect
210
+ const [query, params] = effect.compile()
211
+ return Resource.get(lockConnRef).pipe(
212
+ Effect.flatMap((conn) => conn.executeRaw(query, params)),
213
+ Effect.onError(() => Resource.refresh(lockConnRef!))
214
+ )
215
+ }
216
+ const execWithLockConnValues = <A>(
217
+ effect: Statement.Statement<A>
218
+ ): Effect.Effect<ReadonlyArray<ReadonlyArray<any>>, SqlError> => {
219
+ if (!lockConnRef) return effect.values
220
+ const [query, params] = effect.compile()
221
+ return Resource.get(lockConnRef).pipe(
222
+ Effect.flatMap((conn) => conn.executeValues(query, params)),
223
+ Effect.onError(() => Resource.refresh(lockConnRef!))
224
+ )
225
+ }
226
+
227
+ const acquireLock = sql.onDialectOrElse({
228
+ pg: () =>
229
+ Effect.fnUntraced(function*(_address: string, shardIds: ReadonlyArray<string>) {
230
+ const conn = yield* Resource.get(lockConnRef!)
231
+ const acquiredShardIds: Array<string> = []
232
+ const toAcquire = new Map(shardIds.map((shardId) => [lockNumbers.get(shardId)!, shardId]))
233
+ const takenLocks = yield* conn.executeValues(
234
+ `SELECT objid FROM pg_locks WHERE locktype = 'advisory' AND granted = true AND pid = pg_backend_pid() ORDER BY objid`,
235
+ []
236
+ )
237
+ for (let i = 0; i < takenLocks.length; i++) {
238
+ const lockNum = takenLocks[i][0] as number
239
+ acquiredShardIds.push(lockNumbersReverse.get(lockNum)!)
240
+ toAcquire.delete(lockNum)
241
+ }
242
+ if (toAcquire.size === 0) {
243
+ return acquiredShardIds
244
+ }
245
+ const results = (yield* conn.executeUnprepared(`SELECT ${pgLocks(toAcquire)}`, [], undefined))[0] as Record<
246
+ string,
247
+ boolean
248
+ >
249
+ for (const shardId in results) {
250
+ if (results[shardId]) {
251
+ acquiredShardIds.push(shardId)
252
+ }
253
+ }
254
+ return acquiredShardIds
255
+ }, Effect.onError(() => Resource.refresh(lockConnRef!))),
256
+
257
+ mysql: () =>
258
+ Effect.fnUntraced(function*(_address: string, shardIds: ReadonlyArray<string>) {
259
+ const conn = yield* Resource.get(lockConnRef!)
260
+ const takenLocks = (yield* conn.executeUnprepared(`SELECT ${allMySqlTakenLocks}`, [], undefined))[0] as Record<
261
+ string,
262
+ 1 | null
263
+ >
264
+ const acquiredShardIds: Array<string> = []
265
+ const toAcquire: Array<string> = []
266
+ for (const shardId in takenLocks) {
267
+ if (takenLocks[shardId] === 1) {
268
+ acquiredShardIds.push(shardId)
269
+ } else if (shardIds.includes(shardId)) {
270
+ toAcquire.push(shardId)
271
+ }
272
+ }
273
+ if (toAcquire.length === 0) {
274
+ return acquiredShardIds
275
+ }
276
+ const results = (yield* conn.executeUnprepared(`SELECT ${mysqlLocks(toAcquire)}`, [], undefined))[0] as Record<
277
+ string,
278
+ number
279
+ >
280
+ for (const shardId in results) {
281
+ if (results[shardId] === 1) {
282
+ acquiredShardIds.push(shardId)
283
+ }
284
+ }
285
+ return acquiredShardIds
286
+ }, Effect.onError(() => Resource.refresh(lockConnRef!))),
287
+
288
+ mssql: () => (address: string, shardIds: ReadonlyArray<string>) => {
289
+ const values = shardIds.map((shardId) => sql`(${stringLiteral(shardId)}, ${stringLiteral(address)}, ${sqlNow})`)
290
+ return sql`
291
+ MERGE ${locksTableSql} WITH (HOLDLOCK) AS target
292
+ USING (SELECT * FROM (VALUES ${sql.csv(values)})) AS source (shard_id, address, acquired_at)
293
+ ON target.shard_id = source.shard_id
294
+ WHEN MATCHED AND (target.address = source.address OR DATEDIFF(SECOND, target.acquired_at, ${sqlNow}) > ${expiresSeconds}) THEN
295
+ UPDATE SET address = source.address, acquired_at = source.acquired_at
296
+ WHEN NOT MATCHED THEN
297
+ INSERT (shard_id, address, acquired_at)
298
+ VALUES (source.shard_id, source.address, source.acquired_at);
299
+ `.pipe(
300
+ Effect.andThen(acquiredLocks(address, shardIds)),
301
+ sql.withTransaction
302
+ )
303
+ },
304
+
305
+ orElse: () => (address: string, shardIds: ReadonlyArray<string>) => {
306
+ const values = shardIds.map((shardId) => sql`(${stringLiteral(shardId)}, ${stringLiteral(address)}, ${sqlNow})`)
307
+ return sql`
308
+ WITH source(shard_id, address, acquired_at) AS (VALUES ${sql.csv(values)})
309
+ INSERT INTO ${locksTableSql} (shard_id, address, acquired_at)
310
+ SELECT source.shard_id, source.address, source.acquired_at
311
+ FROM source
312
+ WHERE NOT EXISTS (
313
+ SELECT 1 FROM ${locksTableSql}
314
+ WHERE shard_id = source.shard_id
315
+ AND address != ${address}
316
+ AND (strftime('%s', ${sqlNow}) - strftime('%s', acquired_at)) <= ${expiresSeconds}
317
+ )
318
+ ON CONFLICT(shard_id) DO UPDATE
319
+ SET address = ${address}, acquired_at = ${sqlNow}
320
+ `.pipe(
321
+ Effect.andThen(acquiredLocks(address, shardIds)),
322
+ sql.withTransaction
323
+ )
324
+ }
325
+ })
326
+
327
+ const lockNumbers = new Map<string, number>()
328
+ const lockNumbersReverse = new Map<number, string>()
329
+ for (let i = 0; i < config.shardGroups.length; i++) {
330
+ const group = config.shardGroups[i]
331
+ const base = (i + 1) * 1000000
332
+ for (let shard = 1; shard <= config.shardsPerGroup; shard++) {
333
+ const shardId = ShardId.make(group, shard).toString()
334
+ const lockNum = base + shard
335
+ lockNumbers.set(shardId, lockNum)
336
+ lockNumbersReverse.set(lockNum, shardId)
337
+ }
338
+ }
339
+
340
+ const lockNames = new Map<string, string>()
341
+ const lockNamesReverse = new Map<string, string>()
342
+ for (let i = 0; i < config.shardGroups.length; i++) {
343
+ const group = config.shardGroups[i]
344
+ for (let shard = 1; shard <= config.shardsPerGroup; shard++) {
345
+ const shardId = ShardId.make(group, shard).toString()
346
+ const lockName = `${prefix}.${shardId}`
347
+ lockNames.set(shardId, lockName)
348
+ lockNamesReverse.set(lockName, shardId)
349
+ }
350
+ }
351
+
352
+ const pgLocks = (shardIdsMap: Map<number, string>) =>
353
+ Array.from(
354
+ shardIdsMap.entries(),
355
+ ([lockNum, shardId]) => `pg_try_advisory_lock(${lockNum}) AS "${shardId}"`
356
+ ).join(", ")
357
+
358
+ const mysqlLocks = (shardIds: ReadonlyArray<string>) =>
359
+ shardIds.map((shardId) => `GET_LOCK('${lockNames.get(shardId)!}', 0) AS "${shardId}"`).join(", ")
360
+
361
+ const allMySqlTakenLocks = Array.from(
362
+ lockNames.entries(),
363
+ ([shardId, lockName]) => `IS_USED_LOCK('${lockName}') = CONNECTION_ID() AS "${shardId}"`
364
+ ).join(", ")
365
+
366
+ const acquiredLocks = (address: string, shardIds: ReadonlyArray<string>) =>
367
+ sql<{ shard_id: string }>`
368
+ SELECT shard_id FROM ${sql(locksTable)}
369
+ WHERE address = ${address}
370
+ AND acquired_at >= ${lockExpiresAt}
371
+ AND shard_id IN ${stringLiteralArr(shardIds)}
372
+ `.values.pipe(
373
+ Effect.map((rows) => rows.map((row) => row[0] as string))
374
+ )
375
+
376
+ const wrapString = sql.onDialectOrElse({
377
+ mssql: () => (s: string) => `N'${s}'`,
378
+ orElse: () => (s: string) => `'${s}'`
379
+ })
380
+ const stringLiteral = (s: string) => sql.literal(wrapString(s))
381
+ const stringLiteralArr = (arr: ReadonlyArray<string>) => sql.literal(`(${arr.map(wrapString).join(",")})`)
382
+
383
+ const refreshShards = sql.onDialectOrElse({
384
+ pg: () => acquireLock,
385
+ mysql: () => acquireLock,
386
+ mssql: () => (address: string, shardIds: ReadonlyArray<string>) =>
387
+ sql`
388
+ UPDATE ${locksTableSql}
389
+ SET acquired_at = ${sqlNow}
390
+ OUTPUT inserted.shard_id
391
+ WHERE address = ${address} AND shard_id IN ${stringLiteralArr(shardIds)}
392
+ `.pipe(execWithLockConnValues, Effect.map((rows) => rows.map((row) => row[0] as string))),
393
+ orElse: () => (address: string, shardIds: ReadonlyArray<string>) =>
394
+ sql`
395
+ UPDATE ${locksTableSql}
396
+ SET acquired_at = ${sqlNow}
397
+ WHERE address = ${address} AND shard_id IN ${stringLiteralArr(shardIds)}
398
+ RETURNING shard_id
399
+ `.pipe(execWithLockConnValues, Effect.map((rows) => rows.map((row) => row[0] as string)))
400
+ })
401
+
402
+ return RunnerStorage.makeEncoded({
403
+ getRunners: sql`SELECT runner, healthy FROM ${runnersTableSql} WHERE last_heartbeat > ${lockExpiresAt}`.values.pipe(
404
+ PersistenceError.refail,
405
+ Effect.map(Arr.map(([runner, healthy]) => [String(runner), Boolean(healthy)] as const)),
406
+ withTracerDisabled
407
+ ),
408
+
409
+ register: (address, runner, healthy) =>
410
+ insertRunner(address, runner, healthy).pipe(
411
+ Effect.map((rows: any) => Number(rows[0][0])),
412
+ PersistenceError.refail,
413
+ withTracerDisabled
414
+ ),
415
+
416
+ unregister: (address) =>
417
+ sql`DELETE FROM ${runnersTableSql} WHERE address = ${address} OR last_heartbeat < ${lockExpiresAt}`.pipe(
418
+ Effect.asVoid,
419
+ PersistenceError.refail,
420
+ withTracerDisabled
421
+ ),
422
+
423
+ setRunnerHealth: (address, healthy) =>
424
+ sql`UPDATE ${runnersTableSql} SET healthy = ${encodeBoolean(healthy)} WHERE address = ${address}`
425
+ .pipe(
426
+ Effect.asVoid,
427
+ PersistenceError.refail,
428
+ withTracerDisabled
429
+ ),
430
+
431
+ acquire: (address, shardIds) =>
432
+ acquireLock(address, shardIds).pipe(
433
+ PersistenceError.refail,
434
+ withTracerDisabled
435
+ ),
436
+
437
+ refresh: (address, shardIds) =>
438
+ sql`UPDATE ${runnersTableSql} SET last_heartbeat = ${sqlNow} WHERE address = ${address}`.pipe(
439
+ execWithLockConn,
440
+ shardIds.length > 0 ?
441
+ Effect.andThen(refreshShards(address, shardIds)) :
442
+ Effect.as([]),
443
+ PersistenceError.refail
444
+ ),
445
+
446
+ release: sql.onDialectOrElse({
447
+ pg: () =>
448
+ Effect.fnUntraced(
449
+ function*(_address, shardId) {
450
+ const lockNum = lockNumbers.get(shardId)!
451
+ const conn = yield* Resource.get(lockConnRef!)
452
+ const release = conn.executeRaw(`SELECT pg_advisory_unlock(${lockNum})`, [])
453
+ const check = conn.executeValues(
454
+ `SELECT 1 FROM pg_locks WHERE locktype = 'advisory' AND granted = true AND pid = pg_backend_pid() AND objid = ${lockNum}`,
455
+ []
456
+ )
457
+ while (true) {
458
+ yield* release
459
+ const takenLocks = yield* check
460
+ if (takenLocks.length === 0) return
461
+ }
462
+ },
463
+ Effect.onError(() => Resource.refresh(lockConnRef!)),
464
+ Effect.asVoid,
465
+ PersistenceError.refail,
466
+ withTracerDisabled
467
+ ),
468
+ mysql: () =>
469
+ Effect.fnUntraced(
470
+ function*(_address, shardId) {
471
+ const conn = yield* Resource.get(lockConnRef!)
472
+ const lockName = lockNames.get(shardId)!
473
+ const release = conn.executeRaw(`SELECT RELEASE_LOCK('${lockName}')`, [])
474
+ const check = conn.executeValues(
475
+ `SELECT IS_USED_LOCK('${lockName}') = CONNECTION_ID() AS is_taken`,
476
+ []
477
+ )
478
+ while (true) {
479
+ yield* release
480
+ const takenLocks = yield* check
481
+ if (takenLocks.length === 0 || takenLocks[0][0] !== 1) return
482
+ }
483
+ },
484
+ Effect.onError(() => Resource.refresh(lockConnRef!)),
485
+ Effect.asVoid,
486
+ PersistenceError.refail,
487
+ withTracerDisabled
488
+ ),
489
+ orElse: () => (address, shardId) =>
490
+ sql`DELETE FROM ${locksTableSql} WHERE address = ${address} AND shard_id = ${shardId}`.pipe(
491
+ PersistenceError.refail,
492
+ withTracerDisabled
493
+ )
494
+ }),
495
+
496
+ releaseAll: sql.onDialectOrElse({
497
+ pg: () => (_address) =>
498
+ sql`SELECT pg_advisory_unlock_all()`.pipe(
499
+ execWithLockConn,
500
+ Effect.asVoid,
501
+ PersistenceError.refail,
502
+ withTracerDisabled
503
+ ),
504
+ mysql: () => (_address) =>
505
+ sql`SELECT RELEASE_ALL_LOCKS()`.pipe(
506
+ execWithLockConn,
507
+ Effect.asVoid,
508
+ PersistenceError.refail,
509
+ withTracerDisabled
510
+ ),
511
+ orElse: () => (address) =>
512
+ sql`DELETE FROM ${locksTableSql} WHERE address = ${address}`.pipe(
513
+ PersistenceError.refail,
514
+ withTracerDisabled
515
+ )
516
+ })
517
+ })
518
+ }, withTracerDisabled)
519
+
520
+ /**
521
+ * @since 1.0.0
522
+ * @category Layers
523
+ */
524
+ export const layer: Layer.Layer<
525
+ RunnerStorage.RunnerStorage,
526
+ SqlError,
527
+ SqlClient.SqlClient | ShardingConfig.ShardingConfig
528
+ > = Layer.scoped(RunnerStorage.RunnerStorage)(make({}))
529
+
530
+ /**
531
+ * @since 1.0.0
532
+ * @category Layers
533
+ */
534
+ export const layerWith = (options: {
535
+ readonly prefix?: string | undefined
536
+ }): Layer.Layer<RunnerStorage.RunnerStorage, SqlError, SqlClient.SqlClient | ShardingConfig.ShardingConfig> =>
537
+ Layer.scoped(RunnerStorage.RunnerStorage)(make(options))
package/src/index.ts CHANGED
@@ -63,21 +63,11 @@ export * as EntityType from "./EntityType.js"
63
63
  */
64
64
  export * as Envelope from "./Envelope.js"
65
65
 
66
- /**
67
- * @since 1.0.0
68
- */
69
- export * as HttpCommon from "./HttpCommon.js"
70
-
71
66
  /**
72
67
  * @since 1.0.0
73
68
  */
74
69
  export * as HttpRunner from "./HttpRunner.js"
75
70
 
76
- /**
77
- * @since 1.0.0
78
- */
79
- export * as HttpShardManager from "./HttpShardManager.js"
80
-
81
71
  /**
82
72
  * @since 1.0.0
83
73
  */
@@ -121,22 +111,17 @@ export * as RunnerServer from "./RunnerServer.js"
121
111
  /**
122
112
  * @since 1.0.0
123
113
  */
124
- export * as Runners from "./Runners.js"
125
-
126
- /**
127
- * @since 1.0.0
128
- */
129
- export * as ShardId from "./ShardId.js"
114
+ export * as RunnerStorage from "./RunnerStorage.js"
130
115
 
131
116
  /**
132
117
  * @since 1.0.0
133
118
  */
134
- export * as ShardManager from "./ShardManager.js"
119
+ export * as Runners from "./Runners.js"
135
120
 
136
121
  /**
137
122
  * @since 1.0.0
138
123
  */
139
- export * as ShardStorage from "./ShardStorage.js"
124
+ export * as ShardId from "./ShardId.js"
140
125
 
141
126
  /**
142
127
  * @since 1.0.0
@@ -173,11 +158,6 @@ export * as Snowflake from "./Snowflake.js"
173
158
  */
174
159
  export * as SocketRunner from "./SocketRunner.js"
175
160
 
176
- /**
177
- * @since 1.0.0
178
- */
179
- export * as SocketShardManager from "./SocketShardManager.js"
180
-
181
161
  /**
182
162
  * @since 1.0.0
183
163
  */
@@ -186,9 +166,4 @@ export * as SqlMessageStorage from "./SqlMessageStorage.js"
186
166
  /**
187
167
  * @since 1.0.0
188
168
  */
189
- export * as SqlShardStorage from "./SqlShardStorage.js"
190
-
191
- /**
192
- * @since 1.0.0
193
- */
194
- export * as SynchronizedClock from "./SynchronizedClock.js"
169
+ export * as SqlRunnerStorage from "./SqlRunnerStorage.js"