effect 4.0.7 → 4.1.0

package/src/persistence/PersistedQueue.ts

@@ -17,6 +17,7 @@
 import * as Cause from "../Cause.ts"
 import * as Clock from "../Clock.ts"
 import * as Context from "../Context.ts"
+import * as SecureRandom from "../crypto/SecureRandom.ts"
 import * as Duration from "../Duration.ts"
 import * as Effect from "../Effect.ts"
 import { flow } from "../Function.ts"
@@ -250,7 +251,7 @@ export const makeFactory = Effect.gen(function*() {
         [TypeId]: TypeId,
         offer: (value, opts) =>
           Effect.suspend(() => {
-            const id = opts?.id ?? crypto.randomUUID()
+            const id = opts?.id ?? SecureRandom.uuidV4Unsafe()
             const delay = positiveDelay(opts?.delay)
             if (delay !== undefined && !Duration.isFinite(delay)) {
               return Effect.succeed(id)
@@ -695,6 +696,15 @@ export const layerStoreMemory: Layer.Layer<
  * queue. Delayed offers share the retry sorted set: they are scored with the
  * offering process's clock and released by the polling worker's clock, so
  * clock skew between hosts shifts their delivery.
+ * A handler whose lock lapsed can still settle or renew its claim until a
+ * reset recovers the element.
+ *
+ * **Gotchas**
+ *
+ * The lock format is incompatible with older workers on the same prefix: their
+ * settlements are not fenced and can corrupt newer claims. Stop every old
+ * worker and let its locks expire before starting upgraded workers. Keep the
+ * existing Redis data rather than flushing it.
  *
  * @stability unstable
  * @category constructors
@@ -728,17 +738,15 @@ export const makeStoreRedis = Effect.fnUntraced(function*(
     1
   )
   const prefix = options?.prefix ?? "effectq:"
-  const keyLock = (id: string) => `${prefix}${id}:lock`
   const keysFor = (name: string) => ({
     queue: `${prefix}${name}`,
     pending: `${prefix}${name}:pending`,
     failed: `${prefix}${name}:failed`,
     delayed: `${prefix}${name}:delayed`,
     attempts: `${prefix}${name}:attempts`,
-    ids: `${prefix}${name}:ids`
+    ids: `${prefix}${name}:ids`,
+    owners: `${prefix}${name}:owners`
   })
-  const workerId = crypto.randomUUID()
-
   const ackRetrySchedule = makeAckRetrySchedule(lockExpirationMillis)
 
   type Element = {
@@ -747,6 +755,8 @@ export const makeStoreRedis = Effect.fnUntraced(function*(
     readonly attempts: number
     // the raw wire payload, so requeue and retry do not re-stringify it
     readonly payload: string
+    // unique per acquisition, and recorded in the owners hash as the claim token
+    readonly lock: string
   }
 
   const requeue = redis.eval(requeueRedis)
@@ -780,8 +790,9 @@ export const makeStoreRedis = Effect.fnUntraced(function*(
                 requeue(
                   keys.queue,
                   keys.pending,
-                  keyLock(element.id),
+                  element.lock,
                   keys.attempts,
+                  keys.owners,
                   element.id,
                   element.payload
                 ), { concurrency: "unbounded", discard: true })
@@ -796,7 +807,7 @@ export const makeStoreRedis = Effect.fnUntraced(function*(
           keys.attempts,
           keys.failed,
           keys.ids,
-          prefix,
+          keys.owners,
           state.maxAttempts,
           clock.currentTimeMillisUnsafe()
         )
@@ -813,8 +824,8 @@ export const makeStoreRedis = Effect.fnUntraced(function*(
             keys.pending,
             keys.delayed,
             keys.attempts,
-            prefix,
-            workerId,
+            keys.owners,
+            `${prefix}lock:${SecureRandom.uuidV4Unsafe()}:`,
             size,
             lockExpirationMillis,
             clock.currentTimeMillisUnsafe()
@@ -833,14 +844,15 @@ export const makeStoreRedis = Effect.fnUntraced(function*(
           }
           takenLatch.closeUnsafe()
           const elements: Array<Element> = []
-          for (let i = 0; i < results.length; i += 2) {
+          for (let i = 0; i < results.length; i += 3) {
             const payload = results[i] as string
             const parsed = JSON.parse(payload)
             elements.push({
               id: parsed.id,
               element: parsed.element,
               attempts: Number(results[i + 1]),
-              payload
+              payload,
+              lock: results[i + 2] as string
             })
           }
           yield* Queue.offerAll(queue, elements)
@@ -860,12 +872,12 @@ export const makeStoreRedis = Effect.fnUntraced(function*(
     idleTimeToLive: Duration.seconds(30)
   })
 
-  const activeLockKeys = new Set<string>()
+  const activeLocks = new Map<string, { readonly owners: string; readonly id: string }>()
 
   yield* Effect.gen(function*() {
     while (true) {
       yield* Effect.sleep(lockRefreshMillis)
-      yield* Effect.ignore(expireAll(Array.from(activeLockKeys), lockExpirationMillis))
+      yield* Effect.ignore(expireAll(activeLocks, lockExpirationMillis))
     }
   }).pipe(
     Effect.forkScoped,
@@ -952,19 +964,19 @@ export const makeStoreRedis = Effect.fnUntraced(function*(
           Effect.scoped,
           Effect.tap((element) => {
             const keys = keysFor(options.name)
-            const lock = keyLock(element.id)
-            activeLockKeys.add(lock)
+            const lock = element.lock
+            activeLocks.set(lock, { owners: keys.owners, id: element.id })
             const ack = (effect: Effect.Effect<unknown, Redis.RedisError>) =>
               effect.pipe(
                 Effect.retry(ackRetrySchedule),
                 Effect.orDie,
-                Effect.ensuring(Effect.sync(() => activeLockKeys.delete(lock)))
+                Effect.ensuring(Effect.sync(() => activeLocks.delete(lock)))
               )
             return Effect.addFinalizer((exit) =>
               Effect.suspend(() => {
                 const now = clock.currentTimeMillisUnsafe()
                 if (exit._tag === "Success") {
-                  return ack(complete(keys.pending, lock, keys.attempts, keys.ids, element.id, now))
+                  return ack(complete(keys.pending, lock, keys.attempts, keys.ids, keys.owners, element.id, now))
                 }
                 const dead = deadLetterFromCause(exit.cause)
                 const failElement = (cause: Cause.Cause<unknown>) =>
@@ -974,6 +986,7 @@ export const makeStoreRedis = Effect.fnUntraced(function*(
                     keys.failed,
                     keys.attempts,
                     keys.ids,
+                    keys.owners,
                     element.id,
                     JSON.stringify({
                       id: element.id,
@@ -987,7 +1000,9 @@ export const makeStoreRedis = Effect.fnUntraced(function*(
                   return failElement(dead.cause)
                 }
                 if (Cause.hasInterruptsOnly(exit.cause)) {
-                  return ack(requeue(keys.queue, keys.pending, lock, keys.attempts, element.id, element.payload))
+                  return ack(
+                    requeue(keys.queue, keys.pending, lock, keys.attempts, keys.owners, element.id, element.payload)
+                  )
                 }
                 if (element.attempts >= options.maxAttempts) {
                   return failElement(exit.cause)
@@ -997,6 +1012,7 @@ export const makeStoreRedis = Effect.fnUntraced(function*(
                     keys.pending,
                     lock,
                     keys.delayed,
+                    keys.owners,
                     element.id,
                     element.payload,
                     now + Duration.toMillis(delay)
@@ -1087,7 +1103,7 @@ const resetQueueRedis = Redis.script(
       keyAttempts: string,
       keyFailed: string,
       keyIds: string,
-      prefix: string,
+      keyOwners: string,
       maxAttempts: number,
       now: number
     ]
@@ -1099,17 +1115,17 @@ local key_pending = KEYS[2]
 local key_attempts = KEYS[3]
 local key_failed = KEYS[4]
 local key_ids = KEYS[5]
-local prefix = ARGV[1]
-local max_attempts = tonumber(ARGV[2])
-local now = ARGV[3]
+local key_owners = KEYS[6]
+local max_attempts = tonumber(ARGV[1])
+local now = ARGV[2]
 
 local entries = redis.call("HGETALL", key_pending)
+-- recovery revokes ownership, so late settlements from the old claim are no-ops
 for i = 1, #entries, 2 do
   local id = entries[i]
   local payload = entries[i + 1]
-  local lock_key = prefix .. id .. ":lock"
-  local exists = redis.call("EXISTS", lock_key)
-  if exists == 0 then
+  local key_lock = redis.call("HGET", key_owners, id)
+  if not key_lock or redis.call("EXISTS", key_lock) == 0 then
     local attempts = tonumber(redis.call("HGET", key_attempts, id) or "0")
     if attempts >= max_attempts then
       -- compose the failed record by hand so the element payload does not go
@@ -1124,17 +1140,37 @@ for i = 1, #entries, 2 do
     else
       redis.call("RPUSH", key_queue, payload)
     end
+    redis.call("HDEL", key_owners, id)
     redis.call("HDEL", key_pending, id)
   end
 end
 `,
-    numberOfKeys: 5
+    numberOfKeys: 6
   }
 )
 
+// Settlements are fenced on the acquisition's lock key, so a handler whose claim
+// was recovered cannot release a newer claim on the same id.
+const releaseClaimLua = `if redis.call("HGET", key_owners, id) ~= key_lock then
+  return 0
+end
+
+redis.call("HDEL", key_owners, id)
+redis.call("DEL", key_lock)
+redis.call("HDEL", key_pending, id)
+`
+
 const requeueRedis = Redis.script(
   (
-    ...args: [keyQueue: string, keyPending: string, keyLock: string, keyAttempts: string, id: string, payload: string]
+    ...args: [
+      keyQueue: string,
+      keyPending: string,
+      keyLock: string,
+      keyAttempts: string,
+      keyOwners: string,
+      id: string,
+      payload: string
+    ]
   ) => args,
   {
     lua: `
@@ -1142,60 +1178,77 @@ local key_queue = KEYS[1]
 local key_pending = KEYS[2]
 local key_lock = KEYS[3]
 local key_attempts = KEYS[4]
+local key_owners = KEYS[5]
 local id = ARGV[1]
 local payload = ARGV[2]
 
-redis.call("DEL", key_lock)
-redis.call("HDEL", key_pending, id)
+${releaseClaimLua}
 local attempts = redis.call("HINCRBY", key_attempts, id, -1)
 if attempts <= 0 then
   redis.call("HDEL", key_attempts, id)
 end
 redis.call("RPUSH", key_queue, payload)
 `,
-    numberOfKeys: 4
+    numberOfKeys: 5
   }
 )
 
 const completeRedis = Redis.script(
-  (...args: [keyPending: string, keyLock: string, keyAttempts: string, keyIds: string, id: string, now: number]) =>
-    args,
+  (
+    ...args: [
+      keyPending: string,
+      keyLock: string,
+      keyAttempts: string,
+      keyIds: string,
+      keyOwners: string,
+      id: string,
+      now: number
+    ]
+  ) => args,
   {
     lua: `
 local key_pending = KEYS[1]
 local key_lock = KEYS[2]
 local key_attempts = KEYS[3]
 local key_ids = KEYS[4]
+local key_owners = KEYS[5]
 local id = ARGV[1]
 local now = ARGV[2]
 
-redis.call("DEL", key_lock)
-redis.call("HDEL", key_pending, id)
+${releaseClaimLua}
 redis.call("HDEL", key_attempts, id)
 redis.call("ZADD", key_ids, "XX", now, id)
 `,
-    numberOfKeys: 4
+    numberOfKeys: 5
   }
 )
 
 const retryRedis = Redis.script(
   (
-    ...args: [keyPending: string, keyLock: string, keyDelayed: string, id: string, payload: string, visibleAt: number]
+    ...args: [
+      keyPending: string,
+      keyLock: string,
+      keyDelayed: string,
+      keyOwners: string,
+      id: string,
+      payload: string,
+      visibleAt: number
+    ]
   ) => args,
   {
     lua: `
 local key_pending = KEYS[1]
 local key_lock = KEYS[2]
 local key_delayed = KEYS[3]
+local key_owners = KEYS[4]
 local id = ARGV[1]
 local payload = ARGV[2]
 local visible_at = ARGV[3]
 
-redis.call("DEL", key_lock)
-redis.call("HDEL", key_pending, id)
+${releaseClaimLua}
 redis.call("ZADD", key_delayed, visible_at, payload)
 `,
-    numberOfKeys: 3
+    numberOfKeys: 4
   }
 )
 
@@ -1207,6 +1260,7 @@ const failedRedis = Redis.script(
       keyFailed: string,
       keyAttempts: string,
       keyIds: string,
+      keyOwners: string,
       id: string,
       payload: string
     ]
@@ -1218,18 +1272,18 @@ local key_lock = KEYS[2]
 local key_failed = KEYS[3]
 local key_attempts = KEYS[4]
 local key_ids = KEYS[5]
+local key_owners = KEYS[6]
 local id = ARGV[1]
 local payload = ARGV[2]
 
-redis.call("DEL", key_lock)
-redis.call("HDEL", key_pending, id)
+${releaseClaimLua}
 redis.call("HDEL", key_attempts, id)
 redis.call("RPUSH", key_failed, payload)
 -- failed ids keep their dedupe entry until failedTimeToLive removes the
 -- dead-letter record, so park them outside the timeToLive trim range
 redis.call("ZADD", key_ids, "XX", "+inf", id)
 `,
-    numberOfKeys: 5
+    numberOfKeys: 6
   }
 )
 
@@ -1261,8 +1315,8 @@ const takeRedis = Redis.script(
       keyPending: string,
       keyDelayed: string,
       keyAttempts: string,
-      prefix: string,
-      workerId: string,
+      keyOwners: string,
+      lockPrefix: string,
       batchSize: number,
       pttl: number,
       now: number
@@ -1274,11 +1328,11 @@ local key_queue = KEYS[1]
 local key_pending = KEYS[2]
 local key_delayed = KEYS[3]
 local key_attempts = KEYS[4]
-local prefix = ARGV[1]
-local worker_id = ARGV[2]
-local batch_size = tonumber(ARGV[3])
-local pttl = ARGV[4]
-local now = ARGV[5]
+local key_owners = KEYS[5]
+local lock_prefix = ARGV[1]
+local batch_size = tonumber(ARGV[2])
+local pttl = ARGV[3]
+local now = ARGV[4]
 
 ${queueRecordIdRedis}
 
@@ -1305,28 +1359,43 @@ redis.call("LTRIM", key_queue, #payloads, -1)
 local result = {}
 for i, id in ipairs(ids) do
   local payload = payloads[i]
-  local key_lock = prefix .. id .. ":lock"
-  redis.call("SET", key_lock, worker_id, "PX", pttl)
+  -- the lock key is unique per acquisition and doubles as the ownership token
+  local key_lock = lock_prefix .. i
+  redis.call("SET", key_lock, 1, "PX", pttl)
+  redis.call("HSET", key_owners, id, key_lock)
   redis.call("HSET", key_pending, id, payload)
   local attempts = redis.call("HINCRBY", key_attempts, id, 1)
-  result[i * 2 - 1] = payload
-  result[i * 2] = attempts
+  result[i * 3 - 2] = payload
+  result[i * 3 - 1] = attempts
+  result[i * 3] = key_lock
 end
 
 return result
 `,
-    numberOfKeys: 4
+    numberOfKeys: 5
   }
 ).withReturnType<ReadonlyArray<string | number> | null>()
 
 const expireAllRedis = Redis.script(
-  (keys: ReadonlyArray<string>, ttl: number) => [...keys, ttl],
+  (claims: ReadonlyMap<string, { readonly owners: string; readonly id: string }>, ttl: number) => {
+    const keys: Array<string> = []
+    const ids: Array<string> = []
+    for (const [lock, { id, owners }] of claims) {
+      keys.push(lock, owners)
+      ids.push(id)
+    }
+    return [...keys, ttl, ...ids]
+  },
   {
-    numberOfKeys: (keys) => keys.length,
+    numberOfKeys: (claims) => claims.size * 2,
     lua: `
 local ttl = ARGV[1]
-for i, key in ipairs(KEYS) do
-  redis.call("PEXPIRE", key, ttl)
+-- a lapsed lock can be recreated only while the claim has not been recovered
+for i = 2, #ARGV do
+  local key_lock = KEYS[i * 2 - 3]
+  if redis.call("HGET", KEYS[i * 2 - 2], ARGV[i]) == key_lock then
+    redis.call("SET", key_lock, 1, "PX", ttl)
+  end
 end
 `
   }
@@ -1767,7 +1836,7 @@ export const makeStoreSql: (
           yield* pollLatch.await
           yield* Effect.yieldNow
           nudge.closeUnsafe()
-          const lock = crypto.randomUUID()
+          const lock = SecureRandom.uuidV4Unsafe()
           const results = takers.current === 0 ? [] : yield* poll(takers.current, stringLiteral(lock))
           if (results.length === 0) {
             yield* Effect.race(Effect.sleep(pollInterval), nudge.await)

package/src/persistence/Persistence.ts

@@ -16,7 +16,7 @@ import * as Duration from "../Duration.ts"
 import * as Effect from "../Effect.ts"
 import * as Exit from "../Exit.ts"
 import { identity } from "../Function.ts"
-import { sqlCleanupBatchSize } from "../internal/persistence.ts"
+import { escapeRedisGlob, sqlCleanupBatchSize, storeNamespace } from "../internal/persistence.ts"
 import * as Layer from "../Layer.ts"
 import * as PrimaryKey from "../PrimaryKey.ts"
 import * as Schedule from "../Schedule.ts"
@@ -377,7 +377,7 @@ export const layerBackingSqlMultiTable: Layer.Layer<
       )
 
       type UpsertFn = (
-        entries: Array<{ id: string; value: string; expires: number | null }>
+        entries: Arr.NonEmptyReadonlyArray<{ id: string; value: string; expires: number | null }>
       ) => Effect.Effect<unknown, SqlError>
 
       const upsert = sql.onDialectOrElse({
@@ -492,7 +492,7 @@ export const layerBackingSqlMultiTable: Layer.Layer<
         setMany: (entries) =>
           Effect.suspend(() => {
             try {
-              const encoded = entries.map(([key, value, ttl]) => ({
+              const encoded = Arr.map(entries, ([key, value, ttl]) => ({
                 id: key,
                 value: JSON.stringify(value),
                 expires: unsafeTtlToExpires(clock, ttl)
@@ -716,7 +716,7 @@ export const layerBackingSql: Layer.Layer<
   )
 
   type UpsertFn = (
-    entries: Array<{ store_id: string; id: string; value: string; expires: number | null }>
+    entries: Arr.NonEmptyReadonlyArray<{ store_id: string; id: string; value: string; expires: number | null }>
   ) => Effect.Effect<unknown, SqlError>
 
   const upsert = sql.onDialectOrElse({
@@ -841,7 +841,7 @@ export const layerBackingSql: Layer.Layer<
         setMany: (entries) =>
           Effect.suspend(() => {
             try {
-              const encoded = entries.map(([key, value, ttl]) => ({
+              const encoded = Arr.map(entries, ([key, value, ttl]) => ({
                 store_id: storeId,
                 id: key,
                 value: JSON.stringify(value),
@@ -894,8 +894,10 @@ export const layerBackingSql: Layer.Layer<
  *
  * **Details**
  *
- * Each store id is used as a key prefix, values are JSON-encoded, and finite
- * TTLs are stored with Redis expiration.
+ * Entries are stored under `<storeId>:<key>`, with `%` and `:` in the store id
+ * percent-encoded so store namespaces cannot overlap. Values are JSON-encoded,
+ * finite TTLs are stored with Redis expiration, and `clear` only removes keys
+ * in the store's namespace.
  *
  * @stability unstable
  * @category layers
@@ -910,9 +912,10 @@ export const layerBackingRedis: Layer.Layer<
   const setMany = redis.eval(setManyRedis)
 
   return BackingPersistence.of({
-    make: (prefix) =>
+    make: (storeId) =>
       Effect.sync(() => {
-        const prefixed = (key: string) => `${prefix}:${key}`
+        const namespace = storeNamespace(storeId)
+        const prefixed = (key: string) => `${namespace}${key}`
         const parse = (str: string | null) => {
           if (str === null) {
             return Effect.undefined
@@ -1007,7 +1010,7 @@ export const layerBackingRedis: Layer.Layer<
               redis.send("DEL", prefixed(key)),
               ({ cause }) => new PersistenceError({ message: `Failed to remove key ${key} from Redis`, cause })
             ),
-          clear: redis.send<Array<string>>("KEYS", `${prefix}:*`).pipe(
+          clear: redis.send<Array<string>>("KEYS", `${escapeRedisGlob(namespace)}*`).pipe(
             Effect.flatMap((keys) => keys.length === 0 ? Effect.void : redis.send("DEL", ...keys)),
             Effect.mapError(({ cause }) =>
               new PersistenceError({
@@ -1062,8 +1065,12 @@ end
  *
  * **Details**
  *
- * Each store id becomes a key prefix, and values are stored as JSON with
- * optional expiration timestamps.
+ * Entries are stored under `<storeId>:<key>`, with `%` and `:` in the store id
+ * percent-encoded so store namespaces cannot overlap. Values are stored as JSON
+ * with optional expiration timestamps.
+ *
+ * `clear` uses `KeyValueStore.clearPrefix` to remove only the store's entries,
+ * so the backing store must support it.
  *
  * @stability unstable
  * @category layers
@@ -1079,7 +1086,7 @@ export const layerBackingKvs: Layer.Layer<
   return BackingPersistence.of({
     make: (storeId) =>
       Effect.sync(() => {
-        const store = KeyValueStore.prefix(backing, storeId)
+        const store = KeyValueStore.prefix(backing, storeNamespace(storeId))
         const get = (key: string) =>
           Effect.flatMap(
             Effect.mapError(