@effect/cluster 0.50.6 → 0.52.0

package/src/ShardingConfig.ts

@@ -34,9 +34,15 @@ export class ShardingConfig extends Context.Tag("@effect/cluster/ShardingConfig"
    */
   readonly runnerListenAddress: Option.Option<RunnerAddress>
   /**
-   * The version of the current runner.
+   * A number that determines how many shards this runner will be assigned
+   * relative to other runners.
+   *
+   * Defaults to `1`.
+   *
+   * A value of `2` means that this runner should be assigned twice as many
+   * shards as a runner with a weight of `1`.
    */
-  readonly serverVersion: number
+  readonly runnerShardWeight: number
   /**
    * The shard groups that are assigned to this runner.
    *
@@ -50,14 +56,13 @@ export class ShardingConfig extends Context.Tag("@effect/cluster/ShardingConfig"
    */
   readonly shardsPerGroup: number
   /**
-   * The address of the shard manager.
+   * Shard lock refresh interval.
    */
-  readonly shardManagerAddress: RunnerAddress
+  readonly shardLockRefreshInterval: DurationInput
   /**
-   * If the shard manager is unavailable for this duration, all the shard
-   * assignments will be reset.
+   * Shard lock expiration duration.
    */
-  readonly shardManagerUnavailableTimeout: DurationInput
+  readonly shardLockExpiration: DurationInput
   /**
    * The default capacity of the mailbox for entities.
    */
@@ -81,12 +86,19 @@ export class ShardingConfig extends Context.Tag("@effect/cluster/ShardingConfig"
    * The interval at which to poll for client replies from storage.
    */
   readonly entityReplyPollInterval: DurationInput
+  /**
+   * The interval at which to poll for new runners and refresh shard
+   * assignments.
+   */
   readonly refreshAssignmentsInterval: DurationInput
   /**
    * The interval to retry a send if EntityNotAssignedToRunner is returned.
    */
   readonly sendRetryInterval: DurationInput
-  // readonly unhealthyRunnerReportInterval: Duration.Duration
+  /**
+   * The interval at which to check for unhealthy runners and report them
+   */
+  readonly runnerHealthCheckInterval: DurationInput
   /**
    * Simulate serialization and deserialization to remote runners for local
    * entities.
@@ -103,18 +115,19 @@ const defaultRunnerAddress = RunnerAddress.make({ host: "localhost", port: 34431
 export const defaults: ShardingConfig["Type"] = {
   runnerAddress: Option.some(defaultRunnerAddress),
   runnerListenAddress: Option.none(),
-  serverVersion: 1,
+  runnerShardWeight: 1,
   shardsPerGroup: 300,
-  shardManagerAddress: RunnerAddress.make({ host: "localhost", port: 8080 }),
-  shardManagerUnavailableTimeout: Duration.minutes(10),
   shardGroups: ["default"],
+  shardLockRefreshInterval: Duration.seconds(10),
+  shardLockExpiration: Duration.seconds(35),
   entityMailboxCapacity: 4096,
   entityMaxIdleTime: Duration.minutes(1),
   entityTerminationTimeout: Duration.seconds(15),
   entityMessagePollInterval: Duration.seconds(10),
   entityReplyPollInterval: Duration.millis(200),
   sendRetryInterval: Duration.millis(100),
-  refreshAssignmentsInterval: Duration.minutes(5),
+  refreshAssignmentsInterval: Duration.seconds(3),
+  runnerHealthCheckInterval: Duration.minutes(1),
   simulateRemoteSerialization: true
 }
 
@@ -155,9 +168,8 @@ export const config: Config.Config<ShardingConfig["Type"]> = Config.all({
       Config.withDescription("The port to listen on.")
     )
   }).pipe(Config.map((options) => RunnerAddress.make(options)), Config.option),
-  serverVersion: Config.integer("serverVersion").pipe(
-    Config.withDefault(defaults.serverVersion),
-    Config.withDescription("The version of the current runner.")
+  runnerShardWeight: Config.integer("runnerShardWeight").pipe(
+    Config.withDefault(defaults.runnerShardWeight)
   ),
   shardGroups: Config.array(Config.string("shardGroups")).pipe(
     Config.withDefault(["default"]),
@@ -167,21 +179,13 @@ export const config: Config.Config<ShardingConfig["Type"]> = Config.all({
     Config.withDefault(defaults.shardsPerGroup),
     Config.withDescription("The number of shards to allocate per shard group.")
   ),
-  shardManagerAddress: Config.all({
-    host: Config.string("shardManagerHost").pipe(
-      Config.withDefault(defaults.shardManagerAddress.host),
-      Config.withDescription("The host of the shard manager.")
-    ),
-    port: Config.integer("shardManagerPort").pipe(
-      Config.withDefault(defaults.shardManagerAddress.port),
-      Config.withDescription("The port of the shard manager.")
-    )
-  }).pipe(Config.map((options) => RunnerAddress.make(options))),
-  shardManagerUnavailableTimeout: Config.duration("shardManagerUnavailableTimeout").pipe(
-    Config.withDefault(defaults.shardManagerUnavailableTimeout),
-    Config.withDescription(
-      "If the shard is unavilable for this duration, all the shard assignments will be reset."
-    )
+  shardLockRefreshInterval: Config.duration("shardLockRefreshInterval").pipe(
+    Config.withDefault(defaults.shardLockRefreshInterval),
+    Config.withDescription("Shard lock refresh interval.")
+  ),
+  shardLockExpiration: Config.duration("shardLockExpiration").pipe(
+    Config.withDefault(defaults.shardLockExpiration),
+    Config.withDescription("Shard lock expiration duration.")
   ),
   entityMailboxCapacity: Config.integer("entityMailboxCapacity").pipe(
     Config.withDefault(defaults.entityMailboxCapacity),
@@ -207,12 +211,16 @@ export const config: Config.Config<ShardingConfig["Type"]> = Config.all({
   ),
   sendRetryInterval: Config.duration("sendRetryInterval").pipe(
     Config.withDefault(defaults.sendRetryInterval),
-    Config.withDescription("The interval to retry a send if EntityNotManagedByRunner is returned.")
+    Config.withDescription("The interval to retry a send if EntityNotAssignedToRunner is returned.")
   ),
   refreshAssignmentsInterval: Config.duration("refreshAssignmentsInterval").pipe(
     Config.withDefault(defaults.refreshAssignmentsInterval),
     Config.withDescription("The interval at which to refresh shard assignments.")
   ),
+  runnerHealthCheckInterval: Config.duration("runnerHealthCheckInterval").pipe(
+    Config.withDefault(defaults.runnerHealthCheckInterval),
+    Config.withDescription("The interval at which to check for unhealthy runners and report them.")
+  ),
   simulateRemoteSerialization: Config.boolean("simulateRemoteSerialization").pipe(
     Config.withDefault(defaults.simulateRemoteSerialization),
     Config.withDescription("Simulate serialization and deserialization to remote runners for local entities.")

package/src/Snowflake.ts

@@ -162,7 +162,7 @@ export const makeGenerator: Effect.Effect<Snowflake.Generator> = Effect.gen(func
         // reset sequence if we're in a new millisecond
         sequence = 0
         sequenceAt = now
-      } else if (sequence >= 1024) {
+      } else if (sequence >= 4096) {
         // if we've hit the max sequence for this millisecond, go to the next
         // millisecond
         sequenceAt++

package/src/SocketRunner.ts

@@ -7,11 +7,12 @@ import * as RpcServer from "@effect/rpc/RpcServer"
 import * as Effect from "effect/Effect"
 import * as Layer from "effect/Layer"
 import type { MessageStorage } from "./MessageStorage.js"
+import type { RunnerHealth } from "./RunnerHealth.js"
 import type * as Runners from "./Runners.js"
 import * as RunnerServer from "./RunnerServer.js"
+import type * as RunnerStorage from "./RunnerStorage.js"
 import type * as Sharding from "./Sharding.js"
 import type { ShardingConfig } from "./ShardingConfig.js"
-import type * as ShardStorage from "./ShardStorage.js"
 
 const withLogAddress = <A, E, R>(layer: Layer.Layer<A, E, R>): Layer.Layer<A, E, R | SocketServer> =>
   Layer.effectDiscard(Effect.gen(function*() {
@@ -37,10 +38,11 @@ export const layer: Layer.Layer<
   | RpcSerialization.RpcSerialization
   | SocketServer
   | MessageStorage
-  | ShardStorage.ShardStorage
+  | RunnerStorage.RunnerStorage
+  | RunnerHealth
 > = RunnerServer.layerWithClients.pipe(
   withLogAddress,
-  Layer.provide(Layer.fresh(RpcServer.layerProtocolSocketServer))
+  Layer.provide(RpcServer.layerProtocolSocketServer)
 )
 
 /**
@@ -50,5 +52,5 @@ export const layer: Layer.Layer<
 export const layerClientOnly: Layer.Layer<
   Sharding.Sharding | Runners.Runners,
   never,
-  Runners.RpcClientProtocol | ShardingConfig | MessageStorage
+  Runners.RpcClientProtocol | ShardingConfig | MessageStorage | RunnerStorage.RunnerStorage
 > = RunnerServer.layerClientOnly

package/src/SqlMessageStorage.ts

@@ -6,7 +6,6 @@ import * as SqlClient from "@effect/sql/SqlClient"
 import type { Row } from "@effect/sql/SqlConnection"
 import type { SqlError } from "@effect/sql/SqlError"
 import * as Arr from "effect/Array"
-import type { DurationInput } from "effect/Duration"
 import * as Effect from "effect/Effect"
 import * as Layer from "effect/Layer"
 import * as Option from "effect/Option"
@@ -41,6 +40,7 @@ export const make = Effect.fnUntraced(function*(options?: {
   )
 
   const messageKindAckChunk = sql.literal(String(messageKind.AckChunk))
+  const messageKindInterrupt = sql.literal(String(messageKind.Interrupt))
   const replyKindWithExit = sql.literal(String(replyKind.WithExit))
 
   const messagesTable = table("messages")
@@ -284,11 +284,11 @@ export const make = Effect.fnUntraced(function*(options?: {
       )
   })
 
-  const fiveMinutesAgo = sql.onDialectOrElse({
-    mssql: () => sql.literal(`DATEADD(MINUTE, -5, GETDATE())`),
-    mysql: () => sql.literal(`NOW() - INTERVAL 5 MINUTE`),
-    pg: () => sql.literal(`NOW() - INTERVAL '5 minutes'`),
-    orElse: () => sql.literal(`DATETIME('now', '-5 minute')`)
+  const tenMinutesAgo = sql.onDialectOrElse({
+    mssql: () => sql.literal(`DATEADD(MINUTE, -10, GETDATE())`),
+    mysql: () => sql.literal(`NOW() - INTERVAL 10 MINUTE`),
+    pg: () => sql.literal(`NOW() - INTERVAL '10 minutes'`),
+    orElse: () => sql.literal(`DATETIME('now', '-10 minute')`)
   })
   const sqlNowString = sql.onDialectOrElse({
     pg: () => "NOW()",
@@ -318,7 +318,7 @@ export const make = Effect.fnUntraced(function*(options?: {
             AND (kind = ${replyKindWithExit} OR acked = ${sqlFalse})
           )
           AND m.processed = ${sqlFalse}
-          AND (m.last_read IS NULL OR m.last_read < ${fiveMinutesAgo})
+          AND (m.last_read IS NULL OR m.last_read < ${tenMinutesAgo})
           AND (m.deliver_at IS NULL OR m.deliver_at <= ${sql.literal(String(now))})
           ORDER BY m.rowid ASC
           FOR UPDATE
@@ -339,7 +339,7 @@ export const make = Effect.fnUntraced(function*(options?: {
           AND (kind = ${replyKindWithExit} OR acked = ${sqlFalse})
         )
         AND processed = ${sqlFalse}
-        AND (m.last_read IS NULL OR m.last_read < ${fiveMinutesAgo})
+        AND (m.last_read IS NULL OR m.last_read < ${tenMinutesAgo})
         AND (m.deliver_at IS NULL OR m.deliver_at <= ${sql.literal(String(now))})
         ORDER BY m.rowid ASC
       `.unprepared.pipe(
@@ -400,7 +400,8 @@ export const make = Effect.fnUntraced(function*(options?: {
         )
       }).pipe(
         Effect.provideService(SqlClient.SafeIntegers, true),
-        PersistenceError.refail
+        PersistenceError.refail,
+        withTracerDisabled
       ),
 
     saveReply: (reply) =>
@@ -415,18 +416,23 @@ export const make = Effect.fnUntraced(function*(options?: {
         )
       }).pipe(
         Effect.asVoid,
-        PersistenceError.refail
+        PersistenceError.refail,
+        withTracerDisabled
       ),
 
     clearReplies: Effect.fnUntraced(
       function*(requestId) {
-        yield* sql`DELETE FROM ${repliesTableSql} WHERE request_id = ${String(requestId)}`
+        yield* sql`DELETE FROM ${repliesTableSql} WHERE request_id = ${String(requestId)} AND kind = 0`
+        yield* sql`DELETE FROM ${messagesTableSql} WHERE request_id = ${
+          String(requestId)
+        } AND kind = ${messageKindInterrupt}`
         yield* sql`UPDATE ${messagesTableSql} SET processed = ${sqlFalse}, last_reply_id = NULL, last_read = NULL WHERE request_id = ${
           String(requestId)
         }`
       },
       sql.withTransaction,
-      PersistenceError.refail
+      PersistenceError.refail,
+      withTracerDisabled
     ),
 
     requestIdForPrimaryKey: (primaryKey) =>
@@ -437,7 +443,8 @@ export const make = Effect.fnUntraced(function*(options?: {
           )
         ),
         Effect.provideService(SqlClient.SafeIntegers, true),
-        PersistenceError.refail
+        PersistenceError.refail,
+        withTracerDisabled
       ),
 
     repliesFor: (requestIds) =>
@@ -590,7 +597,6 @@ export const layer: Layer.Layer<
  */
 export const layerWith = (options: {
   readonly prefix?: string | undefined
-  readonly replyPollInterval?: DurationInput | undefined
 }): Layer.Layer<MessageStorage.MessageStorage, never, SqlClient.SqlClient | ShardingConfig> =>
   Layer.scoped(MessageStorage.MessageStorage, make(options)).pipe(
     Layer.provide(Snowflake.layerGenerator)
@@ -638,8 +644,7 @@ const migrations = (options?: {
               last_reply_id BIGINT,
               last_read DATETIME,
               deliver_at BIGINT,
-              UNIQUE (message_id),
-              FOREIGN KEY (request_id) REFERENCES ${messagesTableSql} (id) ON DELETE CASCADE
+              UNIQUE (message_id)
             )
           `,
         mysql: () =>
@@ -665,8 +670,7 @@ const migrations = (options?: {
               last_read DATETIME,
               deliver_at BIGINT,
               UNIQUE (id),
-              UNIQUE (message_id),
-              FOREIGN KEY (request_id) REFERENCES ${messagesTableSql} (id) ON DELETE CASCADE
+              UNIQUE (message_id)
             )
           `,
         pg: () =>
@@ -691,8 +695,7 @@ const migrations = (options?: {
               last_reply_id BIGINT,
               last_read TIMESTAMP,
               deliver_at BIGINT,
-              UNIQUE (message_id),
-              FOREIGN KEY (request_id) REFERENCES ${messagesTableSql} (id) ON DELETE CASCADE
+              UNIQUE (message_id)
             )
           `.pipe(Effect.ignore),
         orElse: () =>
@@ -717,8 +720,7 @@ const migrations = (options?: {
               last_reply_id INTEGER,
               last_read TEXT,
               deliver_at INTEGER,
-              UNIQUE (message_id),
-              FOREIGN KEY (request_id) REFERENCES ${messagesTableSql} (id) ON DELETE CASCADE
+              UNIQUE (message_id)
             )
           `
       })
@@ -790,8 +792,7 @@ const migrations = (options?: {
               sequence INT,
               acked BIT NOT NULL DEFAULT 0,
               CONSTRAINT ${sql(repliesTable + "_one_exit")} UNIQUE (request_id, kind),
-              CONSTRAINT ${sql(repliesTable + "_sequence")} UNIQUE (request_id, sequence),
-              FOREIGN KEY (request_id) REFERENCES ${messagesTableSql} (id) ON DELETE CASCADE
+              CONSTRAINT ${sql(repliesTable + "_sequence")} UNIQUE (request_id, sequence)
             )
           `,
         mysql: () =>
@@ -806,8 +807,7 @@ const migrations = (options?: {
               acked BOOLEAN NOT NULL DEFAULT FALSE,
               UNIQUE (id),
               UNIQUE (request_id, kind),
-              UNIQUE (request_id, sequence),
-              FOREIGN KEY (request_id) REFERENCES ${messagesTableSql} (id) ON DELETE CASCADE
+              UNIQUE (request_id, sequence)
             )
           `,
         pg: () =>
@@ -821,8 +821,7 @@ const migrations = (options?: {
               sequence INT,
               acked BOOLEAN NOT NULL DEFAULT FALSE,
               UNIQUE (request_id, kind),
-              UNIQUE (request_id, sequence),
-              FOREIGN KEY (request_id) REFERENCES ${messagesTableSql} (id) ON DELETE CASCADE
+              UNIQUE (request_id, sequence)
             )
           `,
         orElse: () =>
@@ -836,8 +835,7 @@ const migrations = (options?: {
               sequence INTEGER,
               acked BOOLEAN NOT NULL DEFAULT FALSE,
               UNIQUE (request_id, kind),
-              UNIQUE (request_id, sequence),
-              FOREIGN KEY (request_id) REFERENCES ${messagesTableSql} (id) ON DELETE CASCADE
+              UNIQUE (request_id, sequence)
             )
           `
       })