effect 4.0.7 → 4.1.1

package/src/cluster/SqlMessageStorage.ts

@@ -11,10 +11,10 @@
  * `layer`, `layerWith`, `make`, and `makeEncoded` run the migrations before
  * building the storage. To run migrations with a different connection, such as
  * a schema owner in a deploy step, use `layerMigrations` there and
- * `layerStorage` in the runtime.
+ * `layerStorage` or `makeEncodedStorage` in the runtime.
  *
  * Request deduplication keys that exceed the 255-character `message_id`
- * column are hashed with the `Crypto` service before they are written, so
+ * column are hashed with the `Digest.Digest` service before they are written, so
  * composed keys of any length are supported; shorter keys are stored as
  * plaintext, byte-compatible with rows written by previous versions.
  *
@@ -23,12 +23,10 @@
  */
 // eslint-disable effect/no-bigint-literals
 import * as Arr from "../Array.ts"
-import * as Crypto from "../Crypto.ts"
+import * as Digest from "../crypto/Digest.ts"
 import * as Effect from "../Effect.ts"
-import * as Hex from "../encoding/Hex.ts"
 import * as Layer from "../Layer.ts"
 import * as Option from "../Option.ts"
-import type * as PlatformError from "../PlatformError.ts"
 import * as Migrator from "../sql/Migrator.ts"
 import * as SqlClient from "../sql/SqlClient.ts"
 import type { Row } from "../sql/SqlConnection.ts"
@@ -66,6 +64,7 @@ const withTracerDisabled = Effect.withTracerEnabled(false)
  * tables, including the migration history table.
  *
  * @see {@link make} for the decoded `MessageStorage` constructor
+ * @see {@link makeEncodedStorage} for construction without migrations
  *
  * @stability unstable
  * @category constructors
@@ -76,16 +75,49 @@ export const makeEncoded: (options?: {
 }) => Effect.Effect<
   MessageStorage.Encoded,
   never,
-  SqlClient.SqlClient | Crypto.Crypto
+  SqlClient.SqlClient | Digest.Digest
 > = (options) => Effect.andThen(runMessageMigrations(options), makeEncodedStorage(options))
 
-const makeEncodedStorage = Effect.fnUntraced(function*(
-  options: {
+/**
+ * Creates a SQL-backed encoded message storage driver without running migrations
+ * or executing startup SQL.
+ *
+ * **When to use**
+ *
+ * Use when composing a custom encoded adapter with a runtime SQL connection
+ * that does not have schema-management privileges.
+ *
+ * **Details**
+ *
+ * The optional `prefix` controls the message and reply table names; when omitted,
+ * `cluster` is used.
+ *
+ * **Gotchas**
+ *
+ * Run `layerMigrations` separately with the same prefix before using the driver.
+ * This constructor does not create tables or check that the schema is up to date.
+ *
+ * @see {@link makeEncoded} for construction that runs migrations
+ * @see {@link layerMigrations} for running migrations separately
+ * @see {@link layerStorage} for providing decoded storage without migrations
+ *
+ * @stability unstable
+ * @category constructors
+ * @since 4.1.0
+ */
+export const makeEncodedStorage: (options?: {
+  readonly prefix?: string | undefined
+}) => Effect.Effect<
+  MessageStorage.Encoded,
+  never,
+  SqlClient.SqlClient | Digest.Digest
+> = Effect.fnUntraced(function*(
+  options?: {
     readonly prefix?: string | undefined
-  } | undefined
+  }
 ) {
   const sql = (yield* SqlClient.SqlClient).withoutTransforms()
-  const crypto = yield* Crypto.Crypto
+  const digest = yield* Digest.Digest
   const prefix = options?.prefix ?? "cluster"
   const table = (name: string) => `${prefix}_${name}`
 
@@ -108,10 +140,10 @@ const makeEncodedStorage = Effect.fnUntraced(function*(
   // birthday bound). Digests never contain "/" while composed keys always do,
   // so the two encodings cannot collide.
   const encoder = new TextEncoder()
-  const messageIdForPrimaryKey = (primaryKey: string): Effect.Effect<string, PlatformError.PlatformError> =>
+  const messageIdForPrimaryKey = (primaryKey: string): Effect.Effect<string> =>
     primaryKey.length <= 255
       ? Effect.succeed(primaryKey)
-      : Effect.map(crypto.digest("SHA-256", encoder.encode(primaryKey)), Hex.encode)
+      : digest.digestHex(Digest.sha256, encoder.encode(primaryKey))
 
   const messageIdEnforcesWidth = sql.onDialectOrElse({
     mssql: () => true,
@@ -525,7 +557,7 @@ const makeEncodedStorage = Effect.fnUntraced(function*(
   const encoded: MessageStorage.Encoded = {
     saveEnvelope: ({ deliverAt, envelope, primaryKey }) =>
       Effect.suspend(() => {
-        let insert: Effect.Effect<ReadonlyArray<Row>, SqlError | PlatformError.PlatformError>
+        let insert: Effect.Effect<ReadonlyArray<Row>, SqlError>
         if (primaryKey !== null) {
           insert = Effect.flatMap(messageIdForPrimaryKey(primaryKey), (messageId) => {
             const row = envelopeToRow(envelope, messageId, deliverAt)
@@ -811,7 +843,7 @@ export const make: (options?: {
 }) => Effect.Effect<
   MessageStorage.MessageStorage["Service"],
   never,
-  SqlClient.SqlClient | Snowflake.Generator | Crypto.Crypto
+  SqlClient.SqlClient | Snowflake.Generator | Digest.Digest
 > = (options) => Effect.flatMap(makeEncoded(options), MessageStorage.makeEncoded)
 
 /**
@@ -1165,7 +1197,7 @@ export const layerMigrations = (options: {
  */
 export const layerStorage = (options: {
   readonly prefix?: string | undefined
-}): Layer.Layer<MessageStorage.MessageStorage, never, SqlClient.SqlClient | ShardingConfig | Crypto.Crypto> =>
+}): Layer.Layer<MessageStorage.MessageStorage, never, SqlClient.SqlClient | ShardingConfig | Digest.Digest> =>
   Layer.effect(
     MessageStorage.MessageStorage,
     Effect.flatMap(makeEncodedStorage(options), MessageStorage.makeEncoded)
@@ -1188,7 +1220,7 @@ export const layerStorage = (options: {
  */
 export const layerWith = (options: {
   readonly prefix?: string | undefined
-}): Layer.Layer<MessageStorage.MessageStorage, never, SqlClient.SqlClient | ShardingConfig | Crypto.Crypto> =>
+}): Layer.Layer<MessageStorage.MessageStorage, never, SqlClient.SqlClient | ShardingConfig | Digest.Digest> =>
   layerStorage(options).pipe(
     Layer.provide(layerMigrations(options))
   )
@@ -1210,7 +1242,7 @@ export const layerWith = (options: {
 export const layer: Layer.Layer<
   MessageStorage.MessageStorage,
   never,
-  SqlClient.SqlClient | ShardingConfig | Crypto.Crypto
+  SqlClient.SqlClient | ShardingConfig | Digest.Digest
 > = layerWith({})
 
 // -------------------------------------------------------------------------------------------------

package/src/cluster/SqlRunnerStorage.ts

@@ -98,6 +98,9 @@ const makeStorage = Effect.fnUntraced(function*(options: {
     Math.ceil(Duration.toMillis(Duration.fromInputUnsafe(config.shardLockExpiration))),
     2147483647
   )
+  const shardLockExpirationSeconds = Math.ceil(Duration.toSeconds(config.shardLockExpiration))
+  // Between the smallest and largest values MySQL accepts.
+  const waitTimeout = Math.min(Math.max(1, shardLockExpirationSeconds), 31536000)
 
   // Keep all PostgreSQL and MySQL shard-lock operations on a rebuildable
   // reserved connection, including when advisory locks are disabled.
@@ -107,26 +110,29 @@ const makeStorage = Effect.fnUntraced(function*(options: {
         const conn = yield* Effect.orDie(sql.reserve).pipe(
           Scope.provide(scope)
         )
+        if (disableAdvisoryLocks) return [conn, { pid: 0, pinLock: undefined }] as const
+        // The connection returns to the shared pool, so restore the session's
+        // idle_session_timeout along with releasing its locks. Registered
+        // before the timeout is set, so a failed setup restores it too.
+        yield* Scope.addFinalizerExit(scope, () =>
+          conn.executeRaw(
+            "SELECT pg_advisory_unlock_all(), (SELECT set_config(name, reset_val, false) FROM pg_settings WHERE name = 'idle_session_timeout')",
+            []
+          ).pipe(
+            Effect.timeout(lockOperationInterval),
+            Effect.interruptible,
+            Effect.ignoreCause
+          ))
         // Advisory locks belong to the session, so a frozen runner whose
         // connection stays open would hold them indefinitely. End the session
         // once it has been idle for shardLockExpiration, which releases them.
         // Runners holding shards use it at least every shardLockExpiration / 3.
         // Servers without idle_session_timeout (before PostgreSQL 14) keep the
         // session open.
-        const pid = disableAdvisoryLocks
-          ? 0
-          : (yield* conn.executeValues(
-            `SELECT pg_backend_pid(), (SELECT set_config(name, '${idleSessionTimeout}', false) FROM pg_settings WHERE name = 'idle_session_timeout')`,
-            []
-          ))[0][0] as number
-        if (!disableAdvisoryLocks) {
-          yield* Scope.addFinalizerExit(scope, () =>
-            conn.executeRaw("SELECT pg_advisory_unlock_all()", []).pipe(
-              Effect.timeout(lockOperationInterval),
-              Effect.interruptible,
-              Effect.ignoreCause
-            ))
-        }
+        const pid = (yield* conn.executeValues(
+          `SELECT pg_backend_pid(), (SELECT set_config(name, '${idleSessionTimeout}', false) FROM pg_settings WHERE name = 'idle_session_timeout')`,
+          []
+        ))[0][0] as number
         return [conn, { pid, pinLock: undefined }] as const
       }, Effect.orDie),
     mysql: () =>
@@ -135,17 +141,25 @@ const makeStorage = Effect.fnUntraced(function*(options: {
           Scope.provide(scope)
         )
         if (disableAdvisoryLocks) return [conn, { pid: 0, pinLock: undefined }] as const
+        // The connection goes back to the shared pool, so leave it as it was
+        // borrowed, even if setup fails below.
+        const borrowedWaitTimeout = Number((yield* conn.executeValues("SELECT @@SESSION.wait_timeout", []))[0][0])
+        yield* Scope.addFinalizerExit(scope, () =>
+          conn.executeRaw("SELECT RELEASE_ALL_LOCKS()", []).pipe(
+            Effect.andThen(conn.executeRaw(`SET SESSION wait_timeout = ${borrowedWaitTimeout}`, [])),
+            Effect.timeout(lockOperationInterval),
+            Effect.interruptible,
+            Effect.ignoreCause
+          ))
+        // Named locks belong to the connection, so a frozen runner whose
+        // connection stays open would hold them indefinitely. Have the server
+        // close the connection once it has been idle for shardLockExpiration,
+        // which releases them. vtgate accepts but ignores wait_timeout, so
+        // behind Vitess the locks stay held until the connection closes.
+        yield* conn.executeRaw(`SET SESSION wait_timeout = ${waitTimeout}`, [])
         // we need to get the connection id using IS_USED_LOCK to properly
         // support vitess
         const session: LockSession = yield* pinLockSession(conn)
-        if (!disableAdvisoryLocks) {
-          yield* Scope.addFinalizerExit(scope, () =>
-            conn.executeRaw("SELECT RELEASE_ALL_LOCKS()", []).pipe(
-              Effect.timeout(lockOperationInterval),
-              Effect.interruptible,
-              Effect.ignoreCause
-            ))
-        }
         return [conn, session] as const
       }, Effect.orDie),
     orElse: () => undefined
@@ -211,11 +225,7 @@ const makeStorage = Effect.fnUntraced(function*(options: {
   // avoid deadlocks between acquisition, refresh and bulk release.
   const pgLockOrder = sql.literal(`ORDER BY shard_id COLLATE "C"`)
 
-  const expiresSeconds = sql.literal(
-    Math.ceil(Duration.toSeconds(
-      Duration.fromInputUnsafe(config.shardLockExpiration)
-    )).toString()
-  )
+  const expiresSeconds = sql.literal(shardLockExpirationSeconds.toString())
   const lockExpiresAt = sql.onDialectOrElse({
     pg: () => sql`${sqlNow} - INTERVAL '${expiresSeconds} seconds'`,
     mysql: () => sql`DATE_SUB(${sqlNow}, INTERVAL ${expiresSeconds} SECOND)`,

package/src/cluster/internal/entityManager.ts

@@ -369,8 +369,11 @@ export const make = Effect.fnUntraced(function*<
                     )
                   }
                 }
+                // A failed save can abort the request transaction, so a retry
+                // could store a success outside it. Save once and let the
+                // failed transaction replay the request.
                 const write = retryRespond(
-                  4,
+                  persisted && request.transaction && Exit.isSuccess(response.exit) ? 0 : 4,
                   Effect.suspend(() =>
                     request.message.respond(
                       new Reply.WithExit({
@@ -407,8 +410,25 @@ export const make = Effect.fnUntraced(function*<
                   }
                   removeActiveRequest(Snowflake.Snowflake(response.requestId))
                 })
+                // Let callers read the saved reply instead of waiting on a
+                // handler that will never be resumed.
+                const completeFromStorage = Effect.andThen(
+                  complete,
+                  options.storage.unregisterReplyHandler(request.message.envelope.requestId)
+                )
                 const respond = Effect.andThen(save, complete).pipe(
                   Effect.catchTag("EntityNotAssignedToRunner", () => abandonFenced),
+                  // A save can commit and still fail, e.g. when the COMMIT
+                  // acknowledgement is lost. Volatile requests skip the lookup:
+                  // their replies are never stored, and a storage read could
+                  // stall a custom responder's failure.
+                  Effect.catchTag("PersistenceError", (error) =>
+                    persisted
+                      ? Effect.flatMap(
+                        hasStoredExit(request.message.envelope.requestId),
+                        (committed) => committed ? completeFromStorage : Effect.fail(error)
+                      )
+                      : Effect.fail(error)),
                   Effect.orDie
                 )
                 const transaction = request.transaction
@@ -427,16 +447,10 @@ export const make = Effect.fnUntraced(function*<
                     // A rejected save failed the transaction, rolling back the
                     // handler's writes; the request is left for the owner.
                     if (fenced) return abandonFenced
+                    // The commit can succeed despite the error.
                     return Effect.flatMap(
                       hasStoredExit(request.message.envelope.requestId),
-                      (committed) => {
-                        if (!committed) return restartFrom(outcome.cause)
-                        // Commit succeeded despite the error; let callers read the saved reply.
-                        return Effect.andThen(
-                          complete,
-                          options.storage.unregisterReplyHandler(request.message.envelope.requestId)
-                        )
-                      }
+                      (committed) => committed ? completeFromStorage : restartFrom(outcome.cause)
                     )
                   }
                   return Effect.orDie(save)
@@ -977,7 +991,7 @@ const makeMessageDecode = <Rpcs extends Rpc.Any>(entityRpcs: Map<string, Rpcs>)
         )
       )
     }
-    return decodeRequest(message, rpc) as Effect.Effect<
+    return decodeRequest(message, rpc) as any as Effect.Effect<
       {
         readonly _tag: "IncomingRequest"
         readonly envelope: Envelope.Request.Any