effect 4.0.6 → 4.0.7

package/src/cli/internal/completions/bash.ts

@@ -51,6 +51,10 @@ const buildFlagGroupDeclarations = (
   lines.push(``)
 }
 
+// Quoting these substitutions breaks quote matching on Bash 3.2.
+const removeQuotes = (variable: string): string =>
+  [`\\\\`, `\\"`, `\\'`].map((quote) => `${variable}=\${${variable}//${quote}/}`).join("; ")
+
 /**
  * Complete choices without `compgen -W`, which reparses quoted values. Escape
  * only the portion readline replaces for the active quote context.
@@ -71,9 +75,8 @@ const choicesHelper = (helperName: string, lines: Array<string>): void => {
   lines.push(`  esac`)
   lines.push(``)
   lines.push(`  local _prefix="$_cur" _committed="$_head"`)
-  // Quoting these substitutions breaks quote matching on Bash 3.2.
-  lines.push(`  _prefix=\${_prefix//\\\\/}; _prefix=\${_prefix//\\"/}; _prefix=\${_prefix//\\'/}`)
-  lines.push(`  _committed=\${_committed//\\\\/}; _committed=\${_committed//\\"/}; _committed=\${_committed//\\'/}`)
+  lines.push(`  ${removeQuotes("_prefix")}`)
+  lines.push(`  ${removeQuotes("_committed")}`)
   lines.push(``)
   lines.push(`  COMPREPLY=()`)
   lines.push(`  local _choice _rest _match`)
@@ -111,30 +114,76 @@ const choicesHelper = (helperName: string, lines: Array<string>): void => {
   lines.push(``)
 }
 
-const choiceCompletion = (helperName: string, values: ReadonlyArray<string>): string =>
-  `${helperName} "$cur" "$_comp_word" ${values.map((value) => `'${escapeForBash(value)}'`).join(" ")}`
+interface Helpers {
+  readonly choices: string
+  readonly paths: string
+}
 
-const flagValueCompletion = (type: Completions.FlagType, helperName: string): string | undefined => {
-  switch (type._tag) {
-    case "Boolean":
-      return undefined
-    case "Choice":
-      return choiceCompletion(helperName, type.values)
-    case "Path":
-      if (type.pathType === "directory") return `COMPREPLY=( $(compgen -d -- "$cur") )`
-      return `COMPREPLY=( $(compgen -f -- "$cur") )`
-    default:
-      return undefined
-  }
+/**
+ * Complete paths through the choice helper, one `compgen` line per candidate.
+ * The typed directory is kept as written so expansions in it still apply; only
+ * the completed name is quoted.
+ */
+const pathsHelper = (helpers: Helpers, lines: Array<string>): void => {
+  lines.push(`${helpers.paths}()`)
+  lines.push(`{`)
+  lines.push(`  local _kind="$1" _cur="$2" _word="$3"`)
+  lines.push(``)
+  lines.push(`  local _prefix="$_cur" _committed="\${_cur%"$_word"}"`)
+  lines.push(`  ${removeQuotes("_prefix")}`)
+  lines.push(`  ${removeQuotes("_committed")}`)
+  lines.push(``)
+  // Track a quote the typed directory leaves open.
+  lines.push(`  local _typed="" _dir="$_committed" _open="" _i`)
+  lines.push(`  if [[ "$_word" == */* ]]; then`)
+  lines.push(`    _typed="\${_word%/*}/"`)
+  lines.push(`    _dir="\${_prefix%/*}/"`)
+  lines.push(`    if [[ "$_cur" == "$_word" ]]; then`)
+  lines.push(`      for ((_i = 0; _i < \${#_typed}; _i++)); do`)
+  lines.push(`        case "$_open\${_typed:_i:1}" in`)
+  lines.push(`          \\\\|\\"\\\\) ((_i += 1)) ;;`)
+  lines.push(`          \\"|\\') _open="\${_typed:_i:1}" ;;`)
+  lines.push(`          \\"\\"|\\'\\') _open="" ;;`)
+  lines.push(`        esac`)
+  lines.push(`      done`)
+  lines.push(`    fi`)
+  lines.push(`  fi`)
+  lines.push(``)
+  lines.push(`  local _path _names=() _rest="\${_word#"$_typed"}"`)
+  lines.push(`  while IFS= read -r _path; do`)
+  lines.push(`    if [[ "$_path" == "$_dir"* ]]; then`)
+  lines.push(`      _names+=("$_committed\${_path#"$_dir"}")`)
+  lines.push(`    elif [[ -z "$_committed" ]]; then`)
+  // Bash 3.2 returns expanded paths, which replace the whole word.
+  lines.push(`      _names+=("$_path")`)
+  lines.push(`      _typed="" _rest="" _open=""`)
+  lines.push(`    fi`)
+  lines.push(`  done < <(compgen "$_kind" -- "$_prefix")`)
+  // Close the quote before the name so a common prefix stays balanced.
+  lines.push(`  if [[ -n "$_open" ]]; then`)
+  lines.push(`    _typed+="$_open"`)
+  lines.push(`    _rest="\${_prefix##*/}"`)
+  lines.push(`  fi`)
+  lines.push(``)
+  lines.push(`  ${helpers.choices} "\${_cur%"$_word"}$_rest" "$_rest" "\${_names[@]}"`)
+  lines.push(`  for ((_i = 0; _i < \${#COMPREPLY[@]}; _i++)); do`)
+  lines.push(`    COMPREPLY[_i]="$_typed\${COMPREPLY[_i]}"`)
+  lines.push(`  done`)
+  lines.push(`}`)
+  lines.push(``)
 }
 
-const argCompletion = (type: Completions.ArgumentType, helperName: string): string | undefined => {
+const valueCompletion = (
+  type: Completions.FlagType | Completions.ArgumentType,
+  helpers: Helpers
+): string | undefined => {
   switch (type._tag) {
-    case "Choice":
-      return choiceCompletion(helperName, type.values)
+    case "Choice": {
+      const values = type.values.map((value) => `'${escapeForBash(value)}'`).join(" ")
+      return `${helpers.choices} "$cur" "$_comp_word" ${values}`
+    }
     case "Path":
-      if (type.pathType === "directory") return `COMPREPLY=( $(compgen -d -- "$cur") )`
-      return `COMPREPLY=( $(compgen -f -- "$cur") )`
+      return `${helpers.paths} ${type.pathType === "directory" ? "-d" : "-f"} "$cur" "$_comp_word"`
     default:
       return undefined
   }
@@ -148,7 +197,7 @@ const generateFunction = (
   descriptor: Completions.CommandDescriptor,
   parentPath: ReadonlyArray<string>,
   lines: Array<string>,
-  helperName: string
+  helpers: Helpers
 ): void => {
   const currentPath = [...parentPath, descriptor.name]
   const funcName = `_${currentPath.map(sanitizeFunctionName).join("_")}`
@@ -175,7 +224,7 @@ const generateFunction = (
       for (const alias of flag.aliases) {
         longNames.push(alias.length === 1 ? `-${alias}` : `--${alias}`)
       }
-      const completion = flagValueCompletion(flag.type, helperName)
+      const completion = valueCompletion(flag.type, helpers)
       if (completion) {
         lines.push(`    ${longNames.join("|")})`)
         lines.push(`      ${completion}`)
@@ -231,7 +280,7 @@ const generateFunction = (
 
   // Positional argument completion
   const argsWithCompletions = descriptor.arguments.flatMap((argument, index) => {
-    const completion = argCompletion(argument.type, helperName)
+    const completion = valueCompletion(argument.type, helpers)
     return completion === undefined ? [] : [{ argument, completion, index }]
   })
   if (argsWithCompletions.length > 0) {
@@ -287,7 +336,7 @@ const generateFunction = (
 
   // Recurse into subcommands
   for (const sub of descriptor.subcommands) {
-    generateFunction(sub, currentPath, lines, helperName)
+    generateFunction(sub, currentPath, lines, helpers)
   }
 }
 
@@ -301,7 +350,7 @@ export const generate = (
   const lines: Array<string> = []
   const safeName = sanitizeFunctionName(executableName)
   // Sanitized command names cannot contain `-`.
-  const helperName = `_${safeName}--choices`
+  const helpers: Helpers = { choices: `_${safeName}--choices`, paths: `_${safeName}--paths` }
 
   lines.push(`###-begin-${escapeForBash(executableName)}-completions-###`)
   lines.push(`#`)
@@ -341,8 +390,9 @@ export const generate = (
   lines.push(`fi`)
   lines.push(``)
 
-  choicesHelper(helperName, lines)
-  generateFunction(descriptor, [], lines, helperName)
+  choicesHelper(helpers.choices, lines)
+  pathsHelper(helpers, lines)
+  generateFunction(descriptor, [], lines, helpers)
 
   lines.push(`complete -F _${safeName} ${escapeForBash(executableName)}`)
   lines.push(`###-end-${escapeForBash(executableName)}-completions-###`)

package/src/cluster/ClusterWorkflowEngine.ts

@@ -7,6 +7,7 @@
  * @stability unstable
  * @since 4.0.0
  */
+import * as Cause from "../Cause.ts"
 import * as Context from "../Context.ts"
 import * as DateTime from "../DateTime.ts"
 import * as Deferred from "../Deferred.ts"
@@ -539,9 +540,26 @@ export const make = Effect.gen(function*() {
                       !instance.abandoned &&
                       !(suspendOnFailure && exit._tag === "Failure")
                     ) {
-                      return parent && !parent.detached
+                      const resumeParent = parent && !parent.detached
                         ? ensureSuccess(sendResumeParent(parent, executionId))
                         : Effect.void
+                      if (exit._tag === "Success" || !Cause.hasInterruptsOnly(exit.cause)) {
+                        return resumeParent
+                      }
+                      // `interruptUnsafe` cuts the run off without a result and
+                      // its replay applies the interrupt. Retain the scope like a
+                      // suspended run, so `closeRuns` supersedes it and only the
+                      // final run's compensations observe the interrupt.
+                      return engine.deferredResult(InterruptSignal).pipe(
+                        Effect.flatMap((maybeExit) => {
+                          if (Option.isNone(maybeExit)) {
+                            return resumeParent
+                          }
+                          instance.suspended = true
+                          return Effect.void
+                        }),
+                        Effect.orDie
+                      )
                     }
                     return engine.deferredResult(InterruptSignal).pipe(
                       Effect.flatMap((maybeExit) => {
@@ -797,25 +815,28 @@ export const make = Effect.gen(function*() {
           if (exit) {
             return Option.some(exit)
           }
-          return yield* requestReply({
+          const requestId = yield* requestIdFor({
             workflow: instance.workflow,
             entityType: `Workflow/${instance.workflow._tag}`,
             executionId: instance.executionId,
             tag: "deferred",
             id: deferred.name
-          }).pipe(
-            Effect.map((reply) => {
-              if (Option.isNone(reply)) {
-                return Option.none<Exit.Exit<unknown, unknown>>()
-              }
-              const decoded = decodeDeferredWithExit(reply.value as any)
-              return Option.some(
-                decoded.exit._tag === "Success"
-                  ? decoded.exit.value
-                  : decoded.exit
-              )
-            })
-          )
+          })
+          if (Option.isNone(requestId)) {
+            return Option.none<Exit.Exit<unknown, unknown>>()
+          }
+          const reply = yield* replyForRequestId(requestId.value)
+          if (Option.isSome(reply)) {
+            return Option.some(deferredReplyExit(reply.value))
+          }
+          // The first completion saved under the deferred name wins, so a
+          // request the owner has not processed yet already holds the result.
+          const [request] = yield* storage.unprocessedMessagesById([requestId.value])
+          if (request?.envelope._tag === "Request") {
+            return Option.some(decodeDeferredPayload(request.envelope.payload).exit)
+          }
+          // It was processed after the first read, so its reply is stored now.
+          return Option.map(yield* replyForRequestId(requestId.value), deferredReplyExit)
         })),
         Effect.retry({
           while: (e) => e._tag === "PersistenceError",
@@ -896,10 +917,14 @@ const retryPolicy = Schedule.min([
   Schedule.spaced("1 minute")
 ])
 
+// A cluster abandonment means this owner can no longer complete the effect, so
+// retrying it would only wait on a retired run. Re-raise it as an annotated
+// interrupt, so the activation can unwind and its shard can be released.
 const ensureSuccess = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
   effect.pipe(
     Effect.sandbox,
-    Effect.retry(retryPolicy),
+    Effect.retry({ schedule: retryPolicy, while: (cause) => !ClusterAbandon.isCause(cause) }),
+    Effect.catchIf(ClusterAbandon.isCause, () => ClusterAbandon.interrupt),
     Effect.orDie
   )
 
@@ -959,6 +984,11 @@ const isFailedReply = (reply: WithExitEncoded<Workflow.ResultEncoded<any, any>>)
   reply.exit._tag === "Success" && reply.exit.value._tag === "Suspended" && reply.exit.value.failed === true
 
 const decodeDeferredWithExit = Schema.decodeSync(Schema.toCodecJson(Reply.WithExit.schema(DeferredRpc)))
+const decodeDeferredPayload = Schema.decodeUnknownSync(Schema.toCodecJson(DeferredRpc.payloadSchema))
+const deferredReplyExit = (reply: WithExitEncoded<Workflow.ResultEncoded<any, any>>): Exit.Exit<unknown, unknown> => {
+  const decoded = decodeDeferredWithExit(reply as any)
+  return decoded.exit._tag === "Success" ? decoded.exit.value : decoded.exit
+}
 
 const ResumeRpc = Rpc.make("resume", {
   // Older persisted resume envelopes have an empty payload and use the empty key.

package/src/cluster/MessageStorage.ts

@@ -133,8 +133,15 @@ export class MessageStorage extends Context.Service<MessageStorage, {
 
   /**
    * Unregister the reply handler for the specified message.
+   *
+   * By default the waiters are resumed with `EntityNotAssignedToRunner`. With
+   * `interrupt: true` they are interrupted with a cluster-abandon annotation
+   * instead, for when the request will not be served by this runner.
    */
-  readonly unregisterReplyHandler: (requestId: Snowflake.Snowflake) => Effect.Effect<void>
+  readonly unregisterReplyHandler: (
+    requestId: Snowflake.Snowflake,
+    options?: { readonly interrupt?: boolean | undefined }
+  ) => Effect.Effect<void>
 
   /**
    * Unregister the reply handlers for the specified ShardId.
@@ -569,6 +576,14 @@ export const make = (
     }
     const replyHandlers = new Map<Snowflake.Snowflake, Array<ReplyHandler>>()
     const replyHandlersShard = new Map<string, Set<ReplyHandler>>()
+    const resumeUnregistered = (handler: ReplyHandler, options?: { readonly interrupt?: boolean | undefined }) =>
+      handler.resume(
+        options?.interrupt ? ClusterAbandon.interrupt : Effect.fail(
+          new EntityNotAssignedToRunner({
+            address: handler.message.envelope.address
+          })
+        )
+      )
     return MessageStorage.of({
       ...storage,
       // Nested transactions defer notifications to the outermost commit.
@@ -614,7 +629,7 @@ export const make = (
           })
         })
       },
-      unregisterReplyHandler: (requestId) =>
+      unregisterReplyHandler: (requestId, options) =>
         Effect.sync(() => {
           const handlers = replyHandlers.get(requestId)
           if (!handlers) return
@@ -622,11 +637,7 @@ export const make = (
           for (let i = 0; i < handlers.length; i++) {
             const handler = handlers[i]
             handler.shardSet.delete(handler)
-            handler.resume(Effect.fail(
-              new EntityNotAssignedToRunner({
-                address: handler.message.envelope.address
-              })
-            ))
+            resumeUnregistered(handler, options)
           }
         }),
       unregisterShardReplyHandlers: (shardId, options) =>
@@ -637,13 +648,7 @@ export const make = (
           replyHandlersShard.delete(id)
           shardSet.forEach((handler) => {
             replyHandlers.delete(handler.message.envelope.requestId)
-            handler.resume(
-              options?.interrupt ? ClusterAbandon.interrupt : Effect.fail(
-                new EntityNotAssignedToRunner({
-                  address: handler.message.envelope.address
-                })
-              )
-            )
+            resumeUnregistered(handler, options)
           })
         }),
       saveReply(reply) {

package/src/cluster/Sharding.ts

@@ -290,6 +290,24 @@ const make = Effect.gen(function*() {
     updateReleasingShardsGauge()
   }
   const activeShardsLatch = yield* Latch.make(false)
+
+  // Applies the result of a lock refresh: refreshed shards it did not return,
+  // and returned shards no longer assigned to this runner, move from
+  // `acquiredShards` to `releasingShards`. Shards acquired during the refresh
+  // are absent from its response, so only the refreshed ones are considered.
+  const releaseLostShards = (refreshed: ReadonlyArray<ShardId>, acquired: ReadonlyArray<ShardId>) => {
+    for (const shardId of refreshed) {
+      if (
+        MutableHashSet.has(acquiredShards, shardId) &&
+        (!acquired.includes(shardId) || !MutableHashSet.has(selfShards, shardId))
+      ) {
+        MutableHashSet.remove(acquiredShards, shardId)
+        MutableHashSet.add(releasingShards, shardId)
+      }
+    }
+    updateShardGauges()
+    if (MutableHashSet.size(releasingShards) > 0) activeShardsLatch.openUnsafe()
+  }
   // open once shutdown starts handing every shard off
   const handoffLatch = Latch.makeUnsafe(false)
   // open once the handoff has released every shard lock
@@ -690,27 +708,12 @@ const make = Effect.gen(function*() {
       return Effect.logInfo("Shard lock storage has recovered")
     })
 
-    const refreshShardLocks = Effect.gen(function*() {
+    const refreshShardLocks = Effect.suspend(() => {
       const refreshed = [...acquiredShards, ...releasingShards]
-      const acquired = yield* runnerStorage.refresh(selfAddress, refreshed)
-      // Shards acquired during the request are absent from its response.
-      for (const shardId of refreshed) {
-        if (MutableHashSet.has(acquiredShards, shardId) && !acquired.includes(shardId)) {
-          MutableHashSet.remove(acquiredShards, shardId)
-          MutableHashSet.add(releasingShards, shardId)
-        }
-      }
-      for (let i = 0; i < acquired.length; i++) {
-        const shardId = acquired[i]
-        if (!MutableHashSet.has(selfShards, shardId)) {
-          MutableHashSet.remove(acquiredShards, shardId)
-          MutableHashSet.add(releasingShards, shardId)
-        }
-      }
-      updateShardGauges()
-      if (MutableHashSet.size(releasingShards) > 0) {
-        yield* activeShardsLatch.open
-      }
+      return Effect.map(
+        runnerStorage.refresh(selfAddress, refreshed),
+        (acquired) => releaseLostShards(refreshed, acquired)
+      )
     }).pipe(
       Effect.retry({
         times: 5,
@@ -1541,6 +1544,27 @@ const make = Effect.gen(function*() {
   }
   const clientRequests = new Map<Snowflake.Snowflake, ClientRequestEntry>()
 
+  // During shutdown, abandon local callers parked on persisted requests that a
+  // retiring entity never started. They are served by the next owner. Callers
+  // on a runner that keeps running are left waiting, so they stay durable.
+  const abandonUndelivered = (
+    address: EntityAddress,
+    isDelivered: (requestId: Snowflake.Snowflake) => boolean
+  ): Effect.Effect<void> =>
+    Effect.suspend(() => {
+      if (!MutableRef.get(isShutdown)) return Effect.void
+      const abandon: Array<Effect.Effect<void>> = []
+      for (const [requestId, entry] of clientRequests) {
+        if (
+          !Context.get(entry.message.annotations, Persisted) ||
+          !Equal.equals(entry.message.envelope.address, address) ||
+          isDelivered(requestId)
+        ) continue
+        abandon.push(storage.unregisterReplyHandler(requestId, { interrupt: true }))
+      }
+      return Effect.all(abandon, { discard: true })
+    })
+
   const clients: ResourceMap<
     Entity<any, any>,
     (entityId: string) => RpcClient.RpcClient<
@@ -1878,6 +1902,42 @@ const make = Effect.gen(function*() {
   const reaper = yield* EntityReaper
   const entityManagerLatches = new Map<string, Latch.Latch>()
 
+  // This runner's view of its shards lags lock storage by up to one refresh,
+  // or longer while the process is paused. So the final reply of a persisted
+  // request is stored only once a refresh of its shard's lock confirms it.
+  // Anything short of that, including a refresh that failed, timed out, was
+  // interrupted, or never started because shutdown closed the scope, leaves
+  // the reply for the shard's next owner. Concurrent writers on a shard share
+  // one refresh, which outlives an interrupted writer. Entities draining from
+  // a released shard still hold its lock.
+  const pendingLockChecks = new Map<ShardId, Fiber.Fiber<boolean>>()
+  const confirmShardLock = (runnerAddress: RunnerAddress, shardId: ShardId): Effect.Effect<boolean> =>
+    Effect.suspend(() => {
+      if (!MutableHashSet.has(acquiredShards, shardId) && !MutableHashSet.has(releasingShards, shardId)) {
+        return Effect.succeed(false)
+      }
+      const pending = pendingLockChecks.get(shardId)
+      const check = pending ? Effect.succeed(pending) : runnerStorage.refresh(runnerAddress, [shardId]).pipe(
+        Effect.map((acquired) => {
+          releaseLostShards([shardId], acquired)
+          return acquired.includes(shardId)
+        }),
+        Effect.timeoutOption(shardLockInterval),
+        Effect.map(Option.getOrElse(() => false)),
+        Effect.catch(() => Effect.succeed(false)),
+        Effect.forkIn(shardingScope, { startImmediately: true }),
+        Effect.map((fiber) => {
+          pendingLockChecks.set(shardId, fiber)
+          fiber.addObserver(() => pendingLockChecks.delete(shardId))
+          return fiber
+        })
+      )
+      return Effect.flatMap(
+        check,
+        (fiber) => Effect.map(Fiber.await(fiber), (exit) => exit._tag === "Success" && exit.value)
+      )
+    })
+
   const registerEntity: Sharding["Service"]["registerEntity"] = Effect.fnUntraced(
     function*(entity, build, options) {
       const runnerAddress = getRunnerAddress()
@@ -1895,7 +1955,9 @@ const make = Effect.gen(function*() {
         storage,
         runnerAddress,
         residency,
-        sharding
+        sharding,
+        abandonUndelivered,
+        isShardOwned: (shardId) => confirmShardLock(runnerAddress, shardId)
       }).pipe(
         Effect.provideContext(services.pipe(
           // Registration overrides construction fallbacks, except for runner-owned services.

package/src/cluster/ShardingConfig.ts

@@ -81,6 +81,11 @@ export class ShardingConfig extends Context.Service<ShardingConfig, {
   readonly shardLockRefreshInterval: Duration.Input
   /**
    * Shard lock expiration duration.
+   *
+   * With PostgreSQL advisory locks (PostgreSQL 14+), this also bounds how long
+   * a runner that stops refreshing, for example because its process is frozen,
+   * keeps its shards: the server ends its idle lock session after this
+   * duration, releasing the locks.
    */
   readonly shardLockExpiration: Duration.Input
   /**

package/src/cluster/Snowflake.ts

@@ -257,18 +257,14 @@ export const makeGenerator: Effect.Effect<Snowflake.Generator> = Effect.gen(func
         machineId = newMachineId
       }),
     nextUnsafe() {
-      let now = Math.floor(clock.currentTimeMillisUnsafe())
+      const now = Math.floor(clock.currentTimeMillisUnsafe())
 
-      // account for clock drift, only allow time to move forward
-      if (now < sequenceAt) {
-        now = sequenceAt
-      } else if (now > sequenceAt) {
+      if (now > sequenceAt) {
         // reset sequence if we're in a new millisecond
         sequence = 0
         sequenceAt = now
       } else if (sequence >= 4096) {
-        // if we've hit the max sequence for this millisecond, go to the next
-        // millisecond
+        // sequence exhausted and the clock hasn't advanced, so borrow the next millisecond
         sequenceAt++
         sequence = 0
       }