effect 4.0.7 → 4.1.0

package/src/cluster/SqlMessageStorage.ts

@@ -11,10 +11,10 @@
  * `layer`, `layerWith`, `make`, and `makeEncoded` run the migrations before
  * building the storage. To run migrations with a different connection, such as
  * a schema owner in a deploy step, use `layerMigrations` there and
- * `layerStorage` in the runtime.
+ * `layerStorage` or `makeEncodedStorage` in the runtime.
  *
  * Request deduplication keys that exceed the 255-character `message_id`
- * column are hashed with the `Crypto` service before they are written, so
+ * column are hashed with the `Digest.Digest` service before they are written, so
  * composed keys of any length are supported; shorter keys are stored as
  * plaintext, byte-compatible with rows written by previous versions.
  *
@@ -23,12 +23,10 @@
  */
 // eslint-disable effect/no-bigint-literals
 import * as Arr from "../Array.ts"
-import * as Crypto from "../Crypto.ts"
+import * as Digest from "../crypto/Digest.ts"
 import * as Effect from "../Effect.ts"
-import * as Hex from "../encoding/Hex.ts"
 import * as Layer from "../Layer.ts"
 import * as Option from "../Option.ts"
-import type * as PlatformError from "../PlatformError.ts"
 import * as Migrator from "../sql/Migrator.ts"
 import * as SqlClient from "../sql/SqlClient.ts"
 import type { Row } from "../sql/SqlConnection.ts"
@@ -66,6 +64,7 @@ const withTracerDisabled = Effect.withTracerEnabled(false)
  * tables, including the migration history table.
  *
  * @see {@link make} for the decoded `MessageStorage` constructor
+ * @see {@link makeEncodedStorage} for construction without migrations
  *
  * @stability unstable
  * @category constructors
@@ -76,16 +75,49 @@ export const makeEncoded: (options?: {
 }) => Effect.Effect<
   MessageStorage.Encoded,
   never,
-  SqlClient.SqlClient | Crypto.Crypto
+  SqlClient.SqlClient | Digest.Digest
 > = (options) => Effect.andThen(runMessageMigrations(options), makeEncodedStorage(options))
 
-const makeEncodedStorage = Effect.fnUntraced(function*(
-  options: {
+/**
+ * Creates a SQL-backed encoded message storage driver without running migrations
+ * or executing startup SQL.
+ *
+ * **When to use**
+ *
+ * Use when composing a custom encoded adapter with a runtime SQL connection
+ * that does not have schema-management privileges.
+ *
+ * **Details**
+ *
+ * The optional `prefix` controls the message and reply table names; when omitted,
+ * `cluster` is used.
+ *
+ * **Gotchas**
+ *
+ * Run `layerMigrations` separately with the same prefix before using the driver.
+ * This constructor does not create tables or check that the schema is up to date.
+ *
+ * @see {@link makeEncoded} for construction that runs migrations
+ * @see {@link layerMigrations} for running migrations separately
+ * @see {@link layerStorage} for providing decoded storage without migrations
+ *
+ * @stability unstable
+ * @category constructors
+ * @since 4.1.0
+ */
+export const makeEncodedStorage: (options?: {
+  readonly prefix?: string | undefined
+}) => Effect.Effect<
+  MessageStorage.Encoded,
+  never,
+  SqlClient.SqlClient | Digest.Digest
+> = Effect.fnUntraced(function*(
+  options?: {
     readonly prefix?: string | undefined
-  } | undefined
+  }
 ) {
   const sql = (yield* SqlClient.SqlClient).withoutTransforms()
-  const crypto = yield* Crypto.Crypto
+  const digest = yield* Digest.Digest
   const prefix = options?.prefix ?? "cluster"
   const table = (name: string) => `${prefix}_${name}`
 
@@ -108,10 +140,10 @@ const makeEncodedStorage = Effect.fnUntraced(function*(
   // birthday bound). Digests never contain "/" while composed keys always do,
   // so the two encodings cannot collide.
   const encoder = new TextEncoder()
-  const messageIdForPrimaryKey = (primaryKey: string): Effect.Effect<string, PlatformError.PlatformError> =>
+  const messageIdForPrimaryKey = (primaryKey: string): Effect.Effect<string> =>
     primaryKey.length <= 255
       ? Effect.succeed(primaryKey)
-      : Effect.map(crypto.digest("SHA-256", encoder.encode(primaryKey)), Hex.encode)
+      : digest.digestHex(Digest.sha256, encoder.encode(primaryKey))
 
   const messageIdEnforcesWidth = sql.onDialectOrElse({
     mssql: () => true,
@@ -525,7 +557,7 @@ const makeEncodedStorage = Effect.fnUntraced(function*(
   const encoded: MessageStorage.Encoded = {
     saveEnvelope: ({ deliverAt, envelope, primaryKey }) =>
       Effect.suspend(() => {
-        let insert: Effect.Effect<ReadonlyArray<Row>, SqlError | PlatformError.PlatformError>
+        let insert: Effect.Effect<ReadonlyArray<Row>, SqlError>
         if (primaryKey !== null) {
           insert = Effect.flatMap(messageIdForPrimaryKey(primaryKey), (messageId) => {
             const row = envelopeToRow(envelope, messageId, deliverAt)
@@ -811,7 +843,7 @@ export const make: (options?: {
 }) => Effect.Effect<
   MessageStorage.MessageStorage["Service"],
   never,
-  SqlClient.SqlClient | Snowflake.Generator | Crypto.Crypto
+  SqlClient.SqlClient | Snowflake.Generator | Digest.Digest
 > = (options) => Effect.flatMap(makeEncoded(options), MessageStorage.makeEncoded)
 
 /**
@@ -1165,7 +1197,7 @@ export const layerMigrations = (options: {
  */
 export const layerStorage = (options: {
   readonly prefix?: string | undefined
-}): Layer.Layer<MessageStorage.MessageStorage, never, SqlClient.SqlClient | ShardingConfig | Crypto.Crypto> =>
+}): Layer.Layer<MessageStorage.MessageStorage, never, SqlClient.SqlClient | ShardingConfig | Digest.Digest> =>
   Layer.effect(
     MessageStorage.MessageStorage,
     Effect.flatMap(makeEncodedStorage(options), MessageStorage.makeEncoded)
@@ -1188,7 +1220,7 @@ export const layerStorage = (options: {
  */
 export const layerWith = (options: {
   readonly prefix?: string | undefined
-}): Layer.Layer<MessageStorage.MessageStorage, never, SqlClient.SqlClient | ShardingConfig | Crypto.Crypto> =>
+}): Layer.Layer<MessageStorage.MessageStorage, never, SqlClient.SqlClient | ShardingConfig | Digest.Digest> =>
   layerStorage(options).pipe(
     Layer.provide(layerMigrations(options))
   )
@@ -1210,7 +1242,7 @@ export const layerWith = (options: {
 export const layer: Layer.Layer<
   MessageStorage.MessageStorage,
   never,
-  SqlClient.SqlClient | ShardingConfig | Crypto.Crypto
+  SqlClient.SqlClient | ShardingConfig | Digest.Digest
 > = layerWith({})
 
 // -------------------------------------------------------------------------------------------------

package/src/cluster/SqlRunnerStorage.ts

@@ -98,6 +98,9 @@ const makeStorage = Effect.fnUntraced(function*(options: {
     Math.ceil(Duration.toMillis(Duration.fromInputUnsafe(config.shardLockExpiration))),
     2147483647
   )
+  const shardLockExpirationSeconds = Math.ceil(Duration.toSeconds(config.shardLockExpiration))
+  // Between the smallest and largest values MySQL accepts.
+  const waitTimeout = Math.min(Math.max(1, shardLockExpirationSeconds), 31536000)
 
   // Keep all PostgreSQL and MySQL shard-lock operations on a rebuildable
   // reserved connection, including when advisory locks are disabled.
@@ -107,26 +110,29 @@ const makeStorage = Effect.fnUntraced(function*(options: {
         const conn = yield* Effect.orDie(sql.reserve).pipe(
           Scope.provide(scope)
         )
+        if (disableAdvisoryLocks) return [conn, { pid: 0, pinLock: undefined }] as const
+        // The connection returns to the shared pool, so restore the session's
+        // idle_session_timeout along with releasing its locks. Registered
+        // before the timeout is set, so a failed setup restores it too.
+        yield* Scope.addFinalizerExit(scope, () =>
+          conn.executeRaw(
+            "SELECT pg_advisory_unlock_all(), (SELECT set_config(name, reset_val, false) FROM pg_settings WHERE name = 'idle_session_timeout')",
+            []
+          ).pipe(
+            Effect.timeout(lockOperationInterval),
+            Effect.interruptible,
+            Effect.ignoreCause
+          ))
         // Advisory locks belong to the session, so a frozen runner whose
         // connection stays open would hold them indefinitely. End the session
         // once it has been idle for shardLockExpiration, which releases them.
         // Runners holding shards use it at least every shardLockExpiration / 3.
         // Servers without idle_session_timeout (before PostgreSQL 14) keep the
         // session open.
-        const pid = disableAdvisoryLocks
-          ? 0
-          : (yield* conn.executeValues(
-            `SELECT pg_backend_pid(), (SELECT set_config(name, '${idleSessionTimeout}', false) FROM pg_settings WHERE name = 'idle_session_timeout')`,
-            []
-          ))[0][0] as number
-        if (!disableAdvisoryLocks) {
-          yield* Scope.addFinalizerExit(scope, () =>
-            conn.executeRaw("SELECT pg_advisory_unlock_all()", []).pipe(
-              Effect.timeout(lockOperationInterval),
-              Effect.interruptible,
-              Effect.ignoreCause
-            ))
-        }
+        const pid = (yield* conn.executeValues(
+          `SELECT pg_backend_pid(), (SELECT set_config(name, '${idleSessionTimeout}', false) FROM pg_settings WHERE name = 'idle_session_timeout')`,
+          []
+        ))[0][0] as number
         return [conn, { pid, pinLock: undefined }] as const
       }, Effect.orDie),
     mysql: () =>
@@ -135,17 +141,25 @@ const makeStorage = Effect.fnUntraced(function*(options: {
           Scope.provide(scope)
         )
         if (disableAdvisoryLocks) return [conn, { pid: 0, pinLock: undefined }] as const
+        // The connection goes back to the shared pool, so leave it as it was
+        // borrowed, even if setup fails below.
+        const borrowedWaitTimeout = Number((yield* conn.executeValues("SELECT @@SESSION.wait_timeout", []))[0][0])
+        yield* Scope.addFinalizerExit(scope, () =>
+          conn.executeRaw("SELECT RELEASE_ALL_LOCKS()", []).pipe(
+            Effect.andThen(conn.executeRaw(`SET SESSION wait_timeout = ${borrowedWaitTimeout}`, [])),
+            Effect.timeout(lockOperationInterval),
+            Effect.interruptible,
+            Effect.ignoreCause
+          ))
+        // Named locks belong to the connection, so a frozen runner whose
+        // connection stays open would hold them indefinitely. Have the server
+        // close the connection once it has been idle for shardLockExpiration,
+        // which releases them. vtgate accepts but ignores wait_timeout, so
+        // behind Vitess the locks stay held until the connection closes.
+        yield* conn.executeRaw(`SET SESSION wait_timeout = ${waitTimeout}`, [])
         // we need to get the connection id using IS_USED_LOCK to properly
         // support vitess
         const session: LockSession = yield* pinLockSession(conn)
-        if (!disableAdvisoryLocks) {
-          yield* Scope.addFinalizerExit(scope, () =>
-            conn.executeRaw("SELECT RELEASE_ALL_LOCKS()", []).pipe(
-              Effect.timeout(lockOperationInterval),
-              Effect.interruptible,
-              Effect.ignoreCause
-            ))
-        }
         return [conn, session] as const
       }, Effect.orDie),
     orElse: () => undefined
@@ -211,11 +225,7 @@ const makeStorage = Effect.fnUntraced(function*(options: {
   // avoid deadlocks between acquisition, refresh and bulk release.
   const pgLockOrder = sql.literal(`ORDER BY shard_id COLLATE "C"`)
 
-  const expiresSeconds = sql.literal(
-    Math.ceil(Duration.toSeconds(
-      Duration.fromInputUnsafe(config.shardLockExpiration)
-    )).toString()
-  )
+  const expiresSeconds = sql.literal(shardLockExpirationSeconds.toString())
   const lockExpiresAt = sql.onDialectOrElse({
     pg: () => sql`${sqlNow} - INTERVAL '${expiresSeconds} seconds'`,
     mysql: () => sql`DATE_SUB(${sqlNow}, INTERVAL ${expiresSeconds} SECOND)`,