@effect/cluster 0.50.6 → 0.52.0

package/src/ShardStorage.ts

@@ -1,297 +0,0 @@
-/**
- * @since 1.0.0
- */
-import * as Arr from "effect/Array"
-import * as Context from "effect/Context"
-import * as Effect from "effect/Effect"
-import * as Layer from "effect/Layer"
-import * as MutableHashMap from "effect/MutableHashMap"
-import * as Option from "effect/Option"
-import type { PersistenceError } from "./ClusterError.js"
-import { Runner } from "./Runner.js"
-import { RunnerAddress } from "./RunnerAddress.js"
-import { ShardId } from "./ShardId.js"
-
-/**
- * Represents a generic interface to the persistent storage required by the
- * cluster.
- *
- * @since 1.0.0
- * @category models
- */
-export class ShardStorage extends Context.Tag("@effect/cluster/ShardStorage")<ShardStorage, {
-  /**
-   * Get the current assignments of shards to runners.
-   */
-  readonly getAssignments: Effect.Effect<
-    Array<[ShardId, Option.Option<RunnerAddress>]>,
-    PersistenceError
-  >
-
-  /**
-   * Save the current state of shards assignments to runners.
-   */
-  readonly saveAssignments: (
-    assignments: Iterable<readonly [ShardId, Option.Option<RunnerAddress>]>
-  ) => Effect.Effect<void, PersistenceError>
-
-  /**
-   * Get all runners registered with the cluster.
-   */
-  readonly getRunners: Effect.Effect<Array<[RunnerAddress, Runner]>, PersistenceError>
-
-  /**
-   * Save the current runners registered with the cluster.
-   */
-  readonly saveRunners: (runners: Iterable<readonly [RunnerAddress, Runner]>) => Effect.Effect<void, PersistenceError>
-
-  /**
-   * Try to acquire the given shard ids for processing.
-   *
-   * It returns an array of shards it was able to acquire.
-   */
-  readonly acquire: (
-    address: RunnerAddress,
-    shardIds: Iterable<ShardId>
-  ) => Effect.Effect<Array<ShardId>, PersistenceError>
-
-  /**
-   * Refresh the locks owned by the given runner.
-   *
-   * Locks expire after 15 seconds, so this method should be called every 10
-   * seconds to keep the locks alive.
-   */
-  readonly refresh: (
-    address: RunnerAddress,
-    shardIds: Iterable<ShardId>
-  ) => Effect.Effect<Array<ShardId>, PersistenceError>
-
-  /**
-   * Release the given shard ids.
-   */
-  readonly release: (
-    address: RunnerAddress,
-    shardId: ShardId
-  ) => Effect.Effect<void, PersistenceError>
-
-  /**
-   * Release all the shards assigned to the given runner.
-   */
-  readonly releaseAll: (address: RunnerAddress) => Effect.Effect<void, PersistenceError>
-}>() {}
-
-/**
- * @since 1.0.0
- * @category Encoded
- */
-export interface Encoded {
-  /**
-   * Get the current assignments of shards to runners.
-   */
-  readonly getAssignments: Effect.Effect<
-    Array<
-      readonly [
-        shardId: string,
-        runnerAddress: string | null
-      ]
-    >,
-    PersistenceError
-  >
-
-  /**
-   * Save the current state of shards assignments to runners.
-   */
-  readonly saveAssignments: (
-    assignments: Array<readonly [shardId: string, RunnerAddress: string | null]>
-  ) => Effect.Effect<void, PersistenceError>
-
-  /**
-   * Get all runners registered with the cluster.
-   */
-  readonly getRunners: Effect.Effect<Array<readonly [address: string, runner: string]>, PersistenceError>
-
-  /**
-   * Save the current runners registered with the cluster.
-   */
-  readonly saveRunners: (
-    runners: Array<readonly [address: string, runner: string]>
-  ) => Effect.Effect<void, PersistenceError>
-
-  /**
-   * Acquire the lock on the given shards, returning the shards that were
-   * successfully locked.
-   */
-  readonly acquire: (
-    address: string,
-    shardIds: ReadonlyArray<string>
-  ) => Effect.Effect<Array<string>, PersistenceError>
-
-  /**
-   * Refresh the lock on the given shards, returning the shards that were
-   * successfully locked.
-   */
-  readonly refresh: (
-    address: string,
-    shardIds: ReadonlyArray<string>
-  ) => Effect.Effect<Array<string>, PersistenceError>
-
-  /**
-   * Release the lock on the given shard.
-   */
-  readonly release: (
-    address: string,
-    shardId: string
-  ) => Effect.Effect<void, PersistenceError>
-
-  /**
-   * Release the lock on all shards for the given runner.
-   */
-  readonly releaseAll: (address: string) => Effect.Effect<void, PersistenceError>
-}
-
-/**
- * @since 1.0.0
- * @category layers
- */
-export const makeEncoded = (encoded: Encoded) =>
-  ShardStorage.of({
-    getAssignments: Effect.map(encoded.getAssignments, (assignments) => {
-      const arr = Arr.empty<[ShardId, Option.Option<RunnerAddress>]>()
-      for (const [shardId, runnerAddress] of assignments) {
-        arr.push([
-          ShardId.fromString(shardId),
-          runnerAddress === null ? Option.none() : Option.some(decodeRunnerAddress(runnerAddress))
-        ])
-      }
-      return arr
-    }),
-    saveAssignments: (assignments) => {
-      const arr = Arr.empty<readonly [string, string | null]>()
-      for (const [shardId, runnerAddress] of assignments) {
-        arr.push([
-          shardId.toString(),
-          Option.isNone(runnerAddress) ? null : encodeRunnerAddress(runnerAddress.value)
-        ])
-      }
-      return encoded.saveAssignments(arr)
-    },
-    getRunners: Effect.gen(function*() {
-      const runners = yield* encoded.getRunners
-      const results: Array<[RunnerAddress, Runner]> = []
-      for (let i = 0; i < runners.length; i++) {
-        const [address, runner] = runners[i]
-        try {
-          results.push([decodeRunnerAddress(address), Runner.decodeSync(runner)])
-        } catch {
-          //
-        }
-      }
-      return results
-    }),
-    saveRunners: (runners) =>
-      Effect.suspend(() =>
-        encoded.saveRunners(
-          Array.from(runners, ([address, runner]) => [encodeRunnerAddress(address), Runner.encodeSync(runner)])
-        )
-      ),
-    acquire: (address, shardIds) => {
-      const arr = Array.from(shardIds, (id) => id.toString())
-      return encoded.acquire(encodeRunnerAddress(address), arr).pipe(
-        Effect.map((shards) => shards.map(ShardId.fromString))
-      )
-    },
-    refresh: (address, shardIds) => {
-      const arr = Array.from(shardIds, (id) => id.toString())
-      return encoded.refresh(encodeRunnerAddress(address), arr).pipe(
-        Effect.map((shards) => shards.map(ShardId.fromString))
-      )
-    },
-    release(address, shardId) {
-      return encoded.release(encodeRunnerAddress(address), shardId.toString())
-    },
-    releaseAll(address) {
-      return encoded.releaseAll(encodeRunnerAddress(address))
-    }
-  })
-
-/**
- * @since 1.0.0
- * @category layers
- */
-export const layerNoop: Layer.Layer<ShardStorage> = Layer.sync(
-  ShardStorage,
-  () => {
-    let acquired: Array<ShardId> = []
-    return ShardStorage.of({
-      getAssignments: Effect.sync(() => []),
-      saveAssignments: () => Effect.void,
-      getRunners: Effect.sync(() => []),
-      saveRunners: () => Effect.void,
-      acquire: (_address, shards) => {
-        acquired = Array.from(shards)
-        return Effect.succeed(Array.from(shards))
-      },
-      refresh: () => Effect.sync(() => acquired),
-      release: () => Effect.void,
-      releaseAll: () => Effect.void
-    })
-  }
-)
-
-/**
- * @since 1.0.0
- * @category constructors
- */
-export const makeMemory = Effect.gen(function*() {
-  const assignments = MutableHashMap.empty<ShardId, Option.Option<RunnerAddress>>()
-  const runners = MutableHashMap.empty<RunnerAddress, Runner>()
-
-  function saveAssignments(value: Iterable<readonly [ShardId, Option.Option<RunnerAddress>]>) {
-    return Effect.sync(() => {
-      for (const [shardId, runnerAddress] of value) {
-        MutableHashMap.set(assignments, shardId, runnerAddress)
-      }
-    })
-  }
-
-  function saveRunners(value: Iterable<readonly [RunnerAddress, Runner]>) {
-    return Effect.sync(() => {
-      for (const [address, runner] of value) {
-        MutableHashMap.set(runners, address, runner)
-      }
-    })
-  }
-
-  let acquired: Array<ShardId> = []
-
-  return ShardStorage.of({
-    getAssignments: Effect.sync(() => Array.from(assignments)),
-    saveAssignments,
-    getRunners: Effect.sync(() => Array.from(runners)),
-    saveRunners,
-    acquire: (_address, shardIds) => {
-      acquired = Array.from(shardIds)
-      return Effect.succeed(Array.from(shardIds))
-    },
-    refresh: () => Effect.sync(() => acquired),
-    release: () => Effect.void,
-    releaseAll: () => Effect.void
-  })
-})
-
-/**
- * @since 1.0.0
- * @category layers
- */
-export const layerMemory: Layer.Layer<ShardStorage> = Layer.effect(ShardStorage, makeMemory)
-
-// -------------------------------------------------------------------------------------
-// internal
-// -------------------------------------------------------------------------------------
-
-const encodeRunnerAddress = (runnerAddress: RunnerAddress) => `${runnerAddress.host}:${runnerAddress.port}`
-
-const decodeRunnerAddress = (runnerAddress: string): RunnerAddress => {
-  const [host, port] = runnerAddress.split(":")
-  return new RunnerAddress({ host, port: Number(port) })
-}

package/src/SocketShardManager.ts

@@ -1,48 +0,0 @@
-/**
- * @since 1.0.0
- */
-import { SocketServer } from "@effect/platform/SocketServer"
-import type { RpcSerialization } from "@effect/rpc/RpcSerialization"
-import * as RpcServer from "@effect/rpc/RpcServer"
-import * as Effect from "effect/Effect"
-import * as Layer from "effect/Layer"
-import * as MessageStorage from "./MessageStorage.js"
-import type { RunnerHealth } from "./RunnerHealth.js"
-import * as Runners from "./Runners.js"
-import type { ShardingConfig } from "./ShardingConfig.js"
-import * as ShardManager from "./ShardManager.js"
-import type { ShardStorage } from "./ShardStorage.js"
-
-const withLogAddress = <A, E, R>(layer: Layer.Layer<A, E, R>): Layer.Layer<A, E, R | SocketServer> =>
-  Layer.effectDiscard(Effect.gen(function*() {
-    const server = yield* SocketServer
-    const address = server.address._tag === "UnixAddress"
-      ? server.address.path
-      : `${server.address.hostname}:${server.address.port}`
-    yield* Effect.annotateLogs(Effect.logInfo(`Listening on: ${address}`), {
-      package: "@effect/cluster",
-      service: "ShardManager"
-    })
-  })).pipe(Layer.provideMerge(layer))
-
-/**
- * @since 1.0.0
- * @category Layers
- */
-export const layer: Layer.Layer<
-  ShardManager.ShardManager,
-  never,
-  | ShardStorage
-  | SocketServer
-  | Runners.RpcClientProtocol
-  | RpcSerialization
-  | RunnerHealth
-  | ShardManager.Config
-  | ShardingConfig
-> = ShardManager.layerServer.pipe(
-  withLogAddress,
-  Layer.provide(Layer.fresh(RpcServer.layerProtocolSocketServer)),
-  Layer.provideMerge(ShardManager.layer),
-  Layer.provide(Runners.layerRpc),
-  Layer.provide(MessageStorage.layerNoop)
-)

package/src/SqlShardStorage.ts

@@ -1,329 +0,0 @@
-/**
- * @since 1.0.0
- */
-import * as SqlClient from "@effect/sql/SqlClient"
-import type { SqlError } from "@effect/sql/SqlError"
-import * as Arr from "effect/Array"
-import * as Effect from "effect/Effect"
-import * as Layer from "effect/Layer"
-import { PersistenceError } from "./ClusterError.js"
-import * as ShardStorage from "./ShardStorage.js"
-
-const withTracerDisabled = Effect.withTracerEnabled(false)
-
-/**
- * @since 1.0.0
- * @category Constructors
- */
-export const make = Effect.fnUntraced(function*(options?: {
-  readonly prefix?: string | undefined
-}) {
-  const sql = (yield* SqlClient.SqlClient).withoutTransforms()
-  const prefix = options?.prefix ?? "cluster"
-  const table = (name: string) => `${prefix}_${name}`
-
-  const runnersTable = table("runners")
-  const runnersTableSql = sql(runnersTable)
-
-  yield* sql.onDialectOrElse({
-    mssql: () =>
-      sql`
-        IF OBJECT_ID(N'${runnersTableSql}', N'U') IS NULL
-        CREATE TABLE ${runnersTableSql} (
-          address VARCHAR(255) PRIMARY KEY,
-          runner TEXT NOT NULL
-        )
-      `,
-    mysql: () =>
-      sql`
-        CREATE TABLE IF NOT EXISTS ${runnersTableSql} (
-          address VARCHAR(255) PRIMARY KEY,
-          runner TEXT NOT NULL
-        )
-      `,
-    pg: () =>
-      sql`
-        CREATE TABLE IF NOT EXISTS ${runnersTableSql} (
-          address VARCHAR(255) PRIMARY KEY,
-          runner TEXT NOT NULL
-        )
-      `,
-    orElse: () =>
-      // sqlite
-      sql`
-        CREATE TABLE IF NOT EXISTS ${runnersTableSql} (
-          address TEXT PRIMARY KEY,
-          runner TEXT NOT NULL
-        )
-      `
-  })
-
-  const shardsTable = table("shards")
-  const shardsTableSql = sql(shardsTable)
-
-  yield* sql.onDialectOrElse({
-    mssql: () =>
-      sql`
-        IF OBJECT_ID(N'${shardsTableSql}', N'U') IS NULL
-        CREATE TABLE ${shardsTableSql} (
-          shard_id VARCHAR(50) PRIMARY KEY,
-          address VARCHAR(255)
-        )
-      `,
-    mysql: () =>
-      sql`
-        CREATE TABLE IF NOT EXISTS ${shardsTableSql} (
-          shard_id VARCHAR(50) PRIMARY KEY,
-          address VARCHAR(255)
-        )
-      `,
-    pg: () =>
-      sql`
-        CREATE TABLE IF NOT EXISTS ${shardsTableSql} (
-          shard_id VARCHAR(50) PRIMARY KEY,
-          address VARCHAR(255)
-        )
-      `,
-    orElse: () =>
-      // sqlite
-      sql`
-        CREATE TABLE IF NOT EXISTS ${shardsTableSql} (
-          shard_id TEXT PRIMARY KEY,
-          address TEXT
-        )
-      `
-  })
-
-  const locksTable = table("locks")
-  const locksTableSql = sql(locksTable)
-
-  yield* sql.onDialectOrElse({
-    mssql: () =>
-      sql`
-        IF OBJECT_ID(N'${locksTableSql}', N'U') IS NULL
-        CREATE TABLE ${locksTableSql} (
-          shard_id VARCHAR(50) PRIMARY KEY,
-          address VARCHAR(255) NOT NULL,
-          acquired_at DATETIME NOT NULL
-        )
-      `,
-    mysql: () =>
-      sql`
-        CREATE TABLE IF NOT EXISTS ${locksTableSql} (
-          shard_id VARCHAR(50) PRIMARY KEY,
-          address VARCHAR(255) NOT NULL,
-          acquired_at DATETIME NOT NULL
-        )
-      `,
-    pg: () =>
-      sql`
-        CREATE TABLE IF NOT EXISTS ${locksTableSql} (
-          shard_id VARCHAR(50) PRIMARY KEY,
-          address VARCHAR(255) NOT NULL,
-          acquired_at TIMESTAMP NOT NULL
-        )
-      `,
-    orElse: () =>
-      // sqlite
-      sql`
-        CREATE TABLE IF NOT EXISTS ${locksTableSql} (
-          shard_id TEXT PRIMARY KEY,
-          address TEXT NOT NULL,
-          acquired_at DATETIME NOT NULL
-        )
-      `
-  })
-
-  const sqlNowString = sql.onDialectOrElse({
-    pg: () => "NOW()",
-    mysql: () => "NOW()",
-    mssql: () => "GETDATE()",
-    orElse: () => "CURRENT_TIMESTAMP"
-  })
-  const sqlNow = sql.literal(sqlNowString)
-
-  const lockExpiresAt = sql.onDialectOrElse({
-    pg: () => sql`${sqlNow} - INTERVAL '5 seconds'`,
-    mysql: () => sql`DATE_SUB(${sqlNow}, INTERVAL 5 SECOND)`,
-    mssql: () => sql`DATEADD(SECOND, -5, ${sqlNow})`,
-    orElse: () => sql`datetime(${sqlNow}, '-5 seconds')`
-  })
-
-  const acquireLock = sql.onDialectOrElse({
-    pg: () => (address: string, values: Array<any>) =>
-      sql`
-        INSERT INTO ${locksTableSql} (shard_id, address, acquired_at) VALUES ${sql.csv(values)}
-        ON CONFLICT (shard_id) DO UPDATE
-        SET address = ${address}, acquired_at = ${sqlNow}
-        WHERE ${locksTableSql}.address = ${address}
-          OR ${locksTableSql}.acquired_at < ${lockExpiresAt}
-      `,
-    mysql: () => (_address: string, values: Array<any>) =>
-      sql`
-        INSERT INTO ${locksTableSql} (shard_id, address, acquired_at) VALUES ${sql.csv(values)}
-        ON DUPLICATE KEY UPDATE
-        address = IF(address = VALUES(address) OR acquired_at < ${lockExpiresAt}, VALUES(address), address),
-        acquired_at = IF(address = VALUES(address) OR acquired_at < ${lockExpiresAt}, VALUES(acquired_at), acquired_at)
-      `.unprepared,
-    mssql: () => (_address: string, values: Array<any>) =>
-      sql`
-        MERGE ${locksTableSql} WITH (HOLDLOCK) AS target
-        USING (SELECT * FROM (VALUES ${sql.csv(values)})) AS source (shard_id, address, acquired_at)
-        ON target.shard_id = source.shard_id
-        WHEN MATCHED AND (target.address = source.address OR DATEDIFF(SECOND, target.acquired_at, ${sqlNow}) > 5) THEN
-          UPDATE SET address = source.address, acquired_at = source.acquired_at
-        WHEN NOT MATCHED THEN
-          INSERT (shard_id, address, acquired_at)
-          VALUES (source.shard_id, source.address, source.acquired_at);
-      `,
-    orElse: () => (address: string, values: Array<any>) =>
-      // sqlite
-      sql`
-        WITH source(shard_id, address, acquired_at) AS (VALUES ${sql.csv(values)})
-        INSERT INTO ${locksTableSql} (shard_id, address, acquired_at)
-        SELECT source.shard_id, source.address, source.acquired_at
-        FROM source
-        WHERE NOT EXISTS (
-          SELECT 1 FROM ${locksTableSql}
-          WHERE shard_id = source.shard_id
-          AND address != ${address}
-          AND (strftime('%s', ${sqlNow}) - strftime('%s', acquired_at)) <= 5
-        )
-        ON CONFLICT(shard_id) DO UPDATE
-        SET address = ${address}, acquired_at = ${sqlNow}
-      `
-  })
-
-  const wrapString = sql.onDialectOrElse({
-    mssql: () => (s: string) => `N'${s}'`,
-    orElse: () => (s: string) => `'${s}'`
-  })
-  const wrapStringArr = (arr: ReadonlyArray<string>) => sql.literal(arr.map(wrapString).join(", "))
-
-  const refreshShards = sql.onDialectOrElse({
-    mysql: () => (address: string, shardIds: ReadonlyArray<string>) => {
-      const shardIdsStr = wrapStringArr(shardIds)
-      return sql<Array<{ shard_id: string }>>`
-        UPDATE ${locksTableSql}
-        SET acquired_at = ${sqlNow}
-        WHERE address = ${address} AND shard_id IN (${shardIdsStr});
-        SELECT shard_id FROM ${locksTableSql} WHERE address = ${address} AND shard_id IN (${shardIdsStr})
-      `.unprepared.pipe(
-        Effect.map((rows) => rows[1].map((row) => [row.shard_id]))
-      )
-    },
-    mssql: () => (address: string, shardIds: ReadonlyArray<string>) =>
-      sql`
-        UPDATE ${locksTableSql}
-        SET acquired_at = ${sqlNow}
-        OUTPUT inserted.shard_id
-        WHERE address = ${address} AND shard_id IN (${wrapStringArr(shardIds)})
-      `.values,
-    orElse: () => (address: string, shardIds: ReadonlyArray<string>) =>
-      sql`
-        UPDATE ${locksTableSql}
-        SET acquired_at = ${sqlNow}
-        WHERE address = ${address} AND shard_id IN (${wrapStringArr(shardIds)})
-        RETURNING shard_id
-      `.values
-  })
-
-  return ShardStorage.makeEncoded({
-    getAssignments: sql`SELECT shard_id, address FROM ${shardsTableSql} ORDER BY shard_id`.values.pipe(
-      PersistenceError.refail,
-      withTracerDisabled
-    ) as any,
-
-    saveAssignments: (assignments) => {
-      const remove = sql`DELETE FROM ${shardsTableSql}`
-      if (assignments.length === 0) {
-        return PersistenceError.refail(remove)
-      }
-      const values = assignments.map(([shardId, address]) => sql`(${shardId}, ${address})`)
-      return remove.pipe(
-        Effect.andThen(sql`INSERT INTO ${shardsTableSql} (shard_id, address) VALUES ${sql.csv(values)}`.unprepared),
-        sql.withTransaction,
-        PersistenceError.refail,
-        withTracerDisabled
-      )
-    },
-
-    getRunners: sql`SELECT address, runner FROM ${runnersTableSql}`.values.pipe(
-      PersistenceError.refail,
-      Effect.map(Arr.map(([address, runner]) => [String(address), String(runner)] as const)),
-      withTracerDisabled
-    ),
-
-    saveRunners: (runners) => {
-      const remove = sql`DELETE FROM ${runnersTableSql}`
-      if (runners.length === 0) {
-        return PersistenceError.refail(remove)
-      }
-      const values = runners.map(([address, runner]) => sql`(${address}, ${runner})`)
-      const insert = sql`INSERT INTO ${runnersTableSql} (address, runner) VALUES ${sql.csv(values)}`.unprepared
-      return remove.pipe(
-        Effect.andThen(insert),
-        sql.withTransaction,
-        PersistenceError.refail,
-        withTracerDisabled
-      )
-    },
-
-    acquire: Effect.fnUntraced(
-      function*(address, shardIds) {
-        if (shardIds.length > 0) {
-          const values = shardIds.map((shardId) => sql`(${shardId}, ${address}, ${sqlNow})`)
-          yield* acquireLock(address, values)
-        }
-        const currentLocks = yield* sql<{ shard_id: string }>`
-          SELECT shard_id FROM ${sql(locksTable)}
-          WHERE address = ${address} AND acquired_at >= ${lockExpiresAt}
-        `.values
-        return currentLocks.map((row) => row[0] as string)
-      },
-      sql.withTransaction,
-      PersistenceError.refail,
-      withTracerDisabled
-    ),
-
-    refresh: (address, shardIds) =>
-      shardIds.length === 0
-        ? Effect.succeed([])
-        : refreshShards(address, shardIds).pipe(
-          Effect.map((rows) => rows.map((row) => row[0] as string)),
-          PersistenceError.refail,
-          withTracerDisabled
-        ),
-
-    release: (address, shardId) =>
-      sql`DELETE FROM ${locksTableSql} WHERE address = ${address} AND shard_id = ${shardId}`.pipe(
-        PersistenceError.refail,
-        withTracerDisabled
-      ),
-
-    releaseAll: (address) =>
-      sql`DELETE FROM ${locksTableSql} WHERE address = ${address}`.pipe(
-        PersistenceError.refail,
-        withTracerDisabled
-      )
-  })
-}, withTracerDisabled)
-
-/**
- * @since 1.0.0
- * @category Layers
- */
-export const layer: Layer.Layer<
-  ShardStorage.ShardStorage,
-  SqlError,
-  SqlClient.SqlClient
-> = Layer.effect(ShardStorage.ShardStorage, make())
-
-/**
- * @since 1.0.0
- * @category Layers
- */
-export const layerWith = (options: {
-  readonly prefix?: string | undefined
-}): Layer.Layer<ShardStorage.ShardStorage, SqlError, SqlClient.SqlClient> =>
-  Layer.scoped(ShardStorage.ShardStorage, make(options))