effect 4.0.1 → 4.0.3

package/src/cluster/internal/entityManager.ts

@@ -86,6 +86,10 @@ export interface Residency {
   readonly releaseUnsafe: () => void
 }
 
+interface RequestTransaction {
+  settle?: ((outcome: Exit.Exit<unknown, unknown>) => Effect.Effect<void>) | undefined
+}
+
 // Represents the entities managed by this entity manager
 /**
  * @internal
@@ -100,10 +104,13 @@ export type EntityState = {
     /** Excludes requests awaiting their first dispatch from replay. */
     delivered: boolean
     sentExit: boolean
+    /** Treat early termination interrupts like shutdown interrupts. */
+    terminating?: boolean | undefined
     lastSentChunk: Option.Option<Reply.Chunk<Rpc.Any>>
     sequence: number
     /** Set when the request should not outlive its caller. */
     callerScope?: Scope.Scope | undefined
+    transaction?: RequestTransaction | undefined
   }>
   lastActiveCheck: number
   write: RpcServer.RpcServer<any>["write"]
@@ -152,9 +159,13 @@ export const make = Effect.fnUntraced(function*<
   entityRpcs.set(KeepAliveRpc._tag, KeepAliveRpc as any)
 
   const activeServers = new Map<EntityId, EntityState>()
+  // Entities finishing their in-flight requests after removal, which still
+  // accept interrupts and acks
+  const drainingServers = new Map<EntityId, EntityState>()
   const serverCloseLatches = new Map<EntityAddress, {
     readonly closed: Latch.Latch
     readonly force: Latch.Latch
+    closing: boolean
   }>()
   const processedRequestIds = new Set<Snowflake.Snowflake>()
 
@@ -181,7 +192,8 @@ export const make = Effect.fnUntraced(function*<
     const keepAliveLatch = Latch.makeUnsafe()
     const closeLatches = {
       closed: Latch.makeUnsafe(),
-      force: Latch.makeUnsafe()
+      force: Latch.makeUnsafe(),
+      closing: false
     }
 
     yield* Scope.addFinalizer(
@@ -264,22 +276,23 @@ export const make = Effect.fnUntraced(function*<
                 // They will be retried when the entity is restarted.
                 // Also, if the request is uninterruptible, we ignore the
                 // interrupt.
+                const persisted = storageEnabled && Context.get(request.message.annotations, Persisted)
                 if (
-                  storageEnabled &&
-                  Context.get(request.message.annotations, Persisted) &&
+                  persisted &&
                   Exit.hasInterrupts(response.exit) &&
-                  (isShuttingDown || isUninterruptibleForServer(request.message.annotations))
+                  (isShuttingDown || request.terminating || isUninterruptibleForServer(request.message.annotations))
                 ) {
-                  if (!isShuttingDown) {
+                  if (!isShuttingDown && !request.terminating) {
                     request.sentExit = false
                     return server.write(0, requestEnvelope(request), requestWriteOptions(request)).pipe(
+                      Effect.setContext(handlerContext),
                       Effect.forkIn(handlerScope)
                     )
                   }
                   activeRequests.delete(Snowflake.Snowflake(response.requestId))
                   return options.storage.unregisterReplyHandler(request.message.envelope.requestId)
                 }
-                return retryRespond(
+                const save = retryRespond(
                   4,
                   Effect.suspend(() =>
                     request.message.respond(
@@ -290,22 +303,53 @@ export const make = Effect.fnUntraced(function*<
                       })
                     )
                   )
-                ).pipe(
-                  Effect.flatMap(() => {
-                    if (storageEnabled) {
-                      processedRequestIds.add(request.message.envelope.requestId)
-                    }
-                    activeRequests.delete(Snowflake.Snowflake(response.requestId))
+                )
+                const complete = Effect.sync(() => {
+                  if (storageEnabled) {
+                    processedRequestIds.add(request.message.envelope.requestId)
+                  }
+                  activeRequests.delete(Snowflake.Snowflake(response.requestId))
 
-                    // Start the idle timer when the last request completes.
-                    if (activeRequests.size === 0) {
-                      state.lastActiveCheck = clock.currentTimeMillisUnsafe()
-                    }
+                  // Start the idle timer when the last request completes.
+                  if (activeRequests.size === 0) {
+                    state.lastActiveCheck = clock.currentTimeMillisUnsafe()
+                  }
+                })
+                const respond = Effect.orDie(Effect.andThen(save, complete))
+                const transaction = request.transaction
+                if (!transaction) return respond
+                const exit = response.exit
+
+                if (!persisted) {
+                  if (Exit.isSuccess(exit)) return respond
+                  transaction.settle = () => respond
+                  return Effect.void
+                }
 
-                    return Effect.void
-                  }),
-                  Effect.orDie
-                )
+                if (Exit.isSuccess(exit)) {
+                  transaction.settle = (outcome) =>
+                    Exit.isSuccess(outcome) ? complete : Effect.flatMap(
+                      hasStoredExit(request.message.envelope.requestId),
+                      (committed) => {
+                        if (!committed) return restartFrom(outcome.cause)
+                        // Commit succeeded despite the error; let callers read the saved reply.
+                        return Effect.andThen(
+                          complete,
+                          options.storage.unregisterReplyHandler(request.message.envelope.requestId)
+                        )
+                      }
+                    )
+                  return Effect.orDie(save)
+                }
+                transaction.settle = (outcome) =>
+                  // Transaction wrappers can add defects, but not typed errors; interrupts may be repeated.
+                  Exit.isSuccess(outcome) || !outcome.cause.reasons.some((reason) =>
+                      Cause.isDieReason(reason) &&
+                      !exit.cause.reasons.some((r) => Cause.isDieReason(r) && r.defect === reason.defect)
+                    )
+                    ? respond
+                    : restartFrom(outcome.cause)
+                return Effect.void
               }
               case "Chunk": {
                 const request = activeRequests.get(Snowflake.Snowflake(response.requestId))
@@ -330,10 +374,7 @@ export const make = Effect.fnUntraced(function*<
                 ))
               }
               case "Defect": {
-                if (!isActive()) return endLatch.open
-                const rebuild = writeRef.rebuildUnsafe({ from: server.write, prepare: replay })
-                if (!rebuild) return Effect.void
-                return Effect.forkIn(restart(Cause.die(response.defect), rebuild), managerScope)
+                return restartFrom(Cause.die(response.defect))
               }
               case "ClientEnd": {
                 return endLatch.open
@@ -345,6 +386,21 @@ export const make = Effect.fnUntraced(function*<
           Effect.setContext(Context.merge(handlerContext, handlers))
         )
 
+        // Replay if the stored reply cannot be read.
+        const hasStoredExit = (requestId: Snowflake.Snowflake): Effect.Effect<boolean> =>
+          options.storage.repliesForUnfiltered([requestId]).pipe(
+            Effect.map((replies) => replies.some((reply) => reply._tag === "WithExit")),
+            Effect.catchCause(() => Effect.succeed(false))
+          )
+
+        // Do not inherit the failed handler's transaction context.
+        const restartFrom = (cause: Cause.Cause<unknown>): Effect.Effect<void> => {
+          if (!isActive()) return endLatch.open
+          const rebuild = writeRef.rebuildUnsafe({ from: server.write, prepare: replay })
+          if (!rebuild) return Effect.void
+          return Effect.forkIn(Effect.setContext(restart(cause, rebuild), handlerContext), managerScope)
+        }
+
         yield* Scope.addFinalizer(
           handlerScope,
           Effect.sync(() => {
@@ -415,15 +471,32 @@ export const make = Effect.fnUntraced(function*<
       Effect.suspend(() => {
         activeServers.delete(address.entityId)
         retired.openUnsafe()
+        closeLatches.closing = true
         acquireEntity(address)
+        drainingServers.set(address.entityId, state)
         return Effect.raceFirst(
-          state.write(0, { _tag: "Eof" }).pipe(
+          Effect.gen(function*() {
+            for (const [requestId, request] of activeRequests) {
+              if (
+                !request.delivered || request.sentExit ||
+                !Context.get(request.message.annotations, ClusterSchema.InterruptOnTermination) ||
+                isUninterruptibleForServer(request.message.annotations)
+              ) continue
+              request.terminating = true
+              yield* state.write(0, { _tag: "Interrupt", requestId: requestId as any, interruptors: [] })
+            }
+            yield* state.write(0, { _tag: "Eof" })
+          }).pipe(
             Effect.andThen(endLatch.await),
             Effect.timeoutOption(config.entityTerminationTimeout),
             Effect.interruptible
           ),
           Effect.interruptible(closeLatches.force.await)
-        )
+        ).pipe(Effect.ensuring(Effect.sync(() => {
+          if (drainingServers.get(address.entityId) === state) {
+            drainingServers.delete(address.entityId)
+          }
+        })))
       })
     )
     if (!options.sharding.hasShardId(address.shardId)) {
@@ -458,9 +531,38 @@ export const make = Effect.fnUntraced(function*<
     Effect.forkIn(managerScope)
   )
 
+  function sendEnvelope(server: EntityState, message: Message.IncomingEnvelope): Effect.Effect<void> {
+    const entry = server.activeRequests.get(message.envelope.requestId)
+    if (!entry) {
+      return Effect.void
+    } else if (
+      message.envelope._tag === "AckChunk" &&
+      Option.isSome(entry.lastSentChunk) &&
+      message.envelope.replyId !== entry.lastSentChunk.value.id
+    ) {
+      return Effect.void
+    }
+    return server.write(
+      0,
+      message.envelope._tag === "AckChunk"
+        ? { _tag: "Ack", requestId: message.envelope.requestId as any }
+        : {
+          _tag: "Interrupt",
+          requestId: message.envelope.requestId as any,
+          interruptors: []
+        }
+    )
+  }
+
   function sendLocal<R extends Rpc.Any>(
     message: Message.IncomingLocal<R>
   ): Effect.Effect<void, EntityNotAssignedToRunner | MailboxFull | AlreadyProcessingMessage> {
+    if (message._tag === "IncomingEnvelope") {
+      const draining = drainingServers.get(message.envelope.address.entityId)
+      if (draining?.activeRequests.has(message.envelope.requestId)) {
+        return sendEnvelope(draining, message)
+      }
+    }
     return Effect.provideService(
       Effect.flatMap(
         entities.get(message.envelope.address),
@@ -557,26 +659,7 @@ export const make = Effect.fnUntraced(function*<
               return server.write(0, requestEnvelope(entry), requestWriteOptions(entry))
             }
             case "IncomingEnvelope": {
-              const entry = server.activeRequests.get(message.envelope.requestId)
-              if (!entry) {
-                return Effect.void
-              } else if (
-                message.envelope._tag === "AckChunk" &&
-                Option.isSome(entry.lastSentChunk) &&
-                message.envelope.replyId !== entry.lastSentChunk.value.id
-              ) {
-                return Effect.void
-              }
-              return server.write(
-                0,
-                message.envelope._tag === "AckChunk"
-                  ? { _tag: "Ack", requestId: message.envelope.requestId as any }
-                  : {
-                    _tag: "Interrupt",
-                    requestId: message.envelope.requestId as any,
-                    interruptors: []
-                  }
-              )
+              return sendEnvelope(server, message)
             }
           }
         }
@@ -596,11 +679,24 @@ export const make = Effect.fnUntraced(function*<
     })
 
   const requestWriteOptions = (
-    entry: { readonly message: Message.IncomingRequestLocal<any>; readonly callerScope?: Scope.Scope | undefined }
+    entry: {
+      readonly message: Message.IncomingRequestLocal<any>
+      readonly callerScope?: Scope.Scope | undefined
+      transaction?: RequestTransaction | undefined
+    }
   ): Parameters<EntityState["write"]>[2] => {
-    const onTransaction = Context.get(entry.message.annotations, WithTransaction)
-      ? options.storage.withTransaction
-      : undefined
+    const onTransaction = !Context.get(entry.message.annotations, WithTransaction)
+      ? undefined
+      : <A, E, R>(effect: Effect.Effect<A, E, R>) =>
+        // Interrupts during commit must not hide the outcome from settle.
+        Effect.uninterruptibleMask((restore) => {
+          const transaction: RequestTransaction = {}
+          entry.transaction = transaction
+          return Effect.onExit(options.storage.withTransaction(restore(effect)), (outcome) => {
+            if (entry.transaction === transaction) entry.transaction = undefined
+            return transaction.settle ? transaction.settle(outcome) : Effect.void
+          })
+        })
     const onCaller = entry.callerScope && bindToCaller(entry.callerScope)
     if (!onCaller) return onTransaction && { onRequest: onTransaction }
     return { onRequest: onTransaction ? (effect) => onCaller(onTransaction(effect)) : onCaller }
@@ -621,13 +717,13 @@ export const make = Effect.fnUntraced(function*<
             }
           })
         }
-        activeServers.forEach((state) => {
-          if (shardId[Equal.symbol](state.address.shardId)) {
-            fibers.push(runFork(entities.removeIgnore(state.address)))
-          }
-        })
+        // Look entities up by address: `activeServers` is keyed by entity id,
+        // so an entity with the same id on another shard can hide this one.
         serverCloseLatches.forEach((latches, address) => {
           if (shardId[Equal.symbol](address.shardId)) {
+            if (!latches.closing) {
+              fibers.push(runFork(entities.removeIgnore(address)))
+            }
             fibers.push(runFork(latches.closed.await))
           }
         })

package/src/encoding/Base64.ts

@@ -1,6 +1,7 @@
 /**
  * Base64 encoding and decoding helpers.
  *
+ * @stability stable
  * @since 4.0.0
  */
 import * as Result from "../Result.ts"
@@ -34,6 +35,7 @@ import { EncodingError } from "./EncodingError.ts"
  * @see {@link decode} for decoding standard Base64 to bytes
  * @see {@link decodeString} for decoding standard Base64 to UTF-8 text
  *
+ * @stability stable
  * @category encoding
  * @since 4.0.0
  */
@@ -62,6 +64,7 @@ export const encode: (input: Uint8Array | string) => string = (input) =>
  * Base64.decode("SGVsbG8=") // => Result.succeed(new Uint8Array([72, 101, 108, 108, 111]))
  * ```
  *
+ * @stability stable
  * @category decoding
  * @since 4.0.0
  */
@@ -138,6 +141,7 @@ export const decode = (str: string): Result.Result<Uint8Array, EncodingError> =>
  * Base64.decodeString("aGVsbG8=") // => Result.succeed("hello")
  * ```
  *
+ * @stability stable
  * @category decoding
  * @since 4.0.0
  */

package/src/encoding/Base64Url.ts

@@ -1,6 +1,7 @@
 /**
  * URL-safe Base64 encoding and decoding helpers.
  *
+ * @stability stable
  * @since 4.0.0
  */
 import * as Result from "../Result.ts"
@@ -34,6 +35,7 @@ import { EncodingError } from "./EncodingError.ts"
  * @see {@link decode} for decoding URL-safe Base64 to bytes
  * @see {@link decodeString} for decoding URL-safe Base64 to UTF-8 text
  *
+ * @stability stable
  * @category encoding
  * @since 4.0.0
  */
@@ -62,6 +64,7 @@ export const encode: (input: Uint8Array | string) => string = (input) =>
  * Base64Url.decode("SGVsbG8_") // => Result.succeed(new Uint8Array([72, 101, 108, 108, 111, 63]))
  * ```
  *
+ * @stability stable
  * @category decoding
  * @since 4.0.0
  */
@@ -116,6 +119,7 @@ export const decode = (str: string): Result.Result<Uint8Array, EncodingError> =>
  * Base64Url.decodeString("aGVsbG8_") // => Result.succeed("hello?")
  * ```
  *
+ * @stability stable
  * @category decoding
  * @since 4.0.0
  */

package/src/encoding/EncodingError.ts

@@ -1,6 +1,7 @@
 /**
  * Shared error type for encoding and decoding operations.
  *
+ * @stability stable
  * @since 4.0.0
  */
 import * as Data from "../Data.ts"
@@ -22,6 +23,7 @@ import { hasProperty } from "../Predicate.ts"
  *
  * @see {@link isEncodingError} for the public guard that checks this marker
  *
+ * @stability stable
  * @category type IDs
  * @since 4.0.0
  */
@@ -34,6 +36,7 @@ export const EncodingErrorTypeId = "~effect/encoding/EncodingError" as const
  *
  * Use to type the marker carried by `EncodingError` values.
  *
+ * @stability stable
  * @category type IDs
  * @since 4.0.0
  */
@@ -55,6 +58,7 @@ export type EncodingErrorTypeId = typeof EncodingErrorTypeId
  *
  * @see {@link isEncodingError} for checking whether a value is an EncodingError
  *
+ * @stability stable
  * @category errors
  * @since 4.0.0
  */
@@ -92,6 +96,7 @@ export class EncodingError extends Data.TaggedError("EncodingError")<{
  * @see {@link EncodingError} for the structured error produced by failed
  * encoding and decoding operations
  *
+ * @stability stable
  * @category guards
  * @since 4.0.0
  */

package/src/encoding/Hex.ts

@@ -1,6 +1,7 @@
 /**
  * Hexadecimal encoding, decoding, and random value helpers.
  *
+ * @stability stable
  * @since 4.0.0
  */
 import * as Result from "../Result.ts"
@@ -26,6 +27,7 @@ import { EncodingError } from "./EncodingError.ts"
  * Hex.encode(bytes) // => "48656c6c6f"
  * ```
  *
+ * @stability stable
  * @category encoding
  * @since 4.0.0
  */
@@ -47,6 +49,7 @@ export const encode: (input: Uint8Array | string) => string = (input) =>
  * security-sensitive values, use the `Crypto.Crypto` service's `randomBytes`
  * method and encode the result with {@link encode}.
  *
+ * @stability stable
  * @category encoding
  * @since 4.0.0
  */
@@ -173,6 +176,7 @@ const random32 = (): string => {
  * Hex.decode("48656c6c6f") // => Result.succeed(new Uint8Array([72, 101, 108, 108, 111]))
  * ```
  *
+ * @stability stable
  * @category decoding
  * @since 4.0.0
  */
@@ -233,6 +237,7 @@ export const decode = (str: string): Result.Result<Uint8Array, EncodingError> =>
  * Hex.decodeString("68656c6c6f") // => Result.succeed("hello")
  * ```
  *
+ * @stability stable
  * @category decoding
  * @since 4.0.0
  */

package/src/encoding/Toml.ts

@@ -283,7 +283,7 @@ class TomlParser {
     if (multiline) {
       this.expect("\"")
       this.expect("\"")
-      if (this.peek() === "\n") this.advance()
+      this.advance(this.input.startsWith("\r\n", this.index) ? 2 : this.peek() === "\n" ? 1 : 0)
     }
     let output = ""
     while (!this.done) {
@@ -347,7 +347,7 @@ class TomlParser {
     if (multiline) {
       this.expect("'")
       this.expect("'")
-      if (this.peek() === "\n") this.advance()
+      this.advance(this.input.startsWith("\r\n", this.index) ? 2 : this.peek() === "\n" ? 1 : 0)
     }
     const start = this.index
     while (!this.done) {

package/src/encoding/Yaml.ts

@@ -523,6 +523,12 @@ class YamlParser {
     }
     this.index = end
 
+    let contentEnd = content.length
+    while (contentEnd > 0 && content[contentEnd - 1].length === 0) contentEnd--
+    const trailingBreaks = Math.max(0, content.length - contentEnd - (end === this.lines.length ? 1 : 0)) +
+      (contentEnd > 0 && start + contentEnd < this.lines.length ? 1 : 0)
+    content.length = contentEnd
+
     let output = ""
     if (style === "|") {
       output = content.join("\n")
@@ -543,11 +549,9 @@ class YamlParser {
           previousMoreIndented = moreIndented
         }
       }
-      output += "\n".repeat(blankLines)
     }
-    if (chomp === "keep") return output.endsWith("\n") ? output : `${output}\n`
-    output = output.replace(/\n+$/, "")
-    return chomp === "strip" ? output : `${output}\n`
+    if (chomp === "keep") return output + "\n".repeat(trailingBreaks)
+    return chomp === "clip" && contentEnd > 0 && trailingBreaks > 0 ? `${output}\n` : output
   }
 
   private skipBlankLines(): number {

package/src/encoding/index.ts

@@ -6,21 +6,25 @@
 // @barrel: Auto-generated exports. Do not edit manually.
 
 /**
+ * @stability stable
  * @since 4.0.0
  */
 export * as Base64 from "./Base64.ts"
 
 /**
+ * @stability stable
  * @since 4.0.0
  */
 export * as Base64Url from "./Base64Url.ts"
 
 /**
+ * @stability stable
  * @since 4.0.0
  */
 export * as EncodingError from "./EncodingError.ts"
 
 /**
+ * @stability stable
  * @since 4.0.0
  */
 export * as Hex from "./Hex.ts"

package/src/eventlog/SqlEventJournal.ts

@@ -56,7 +56,7 @@ export const make = (options?: {
       pg: () =>
         sql`
           CREATE TABLE IF NOT EXISTS ${entryTableSql} (
-            id UUID PRIMARY KEY,
+            id BYTEA PRIMARY KEY,
             event TEXT NOT NULL,
             primary_key TEXT NOT NULL,
             payload BYTEA NOT NULL,
@@ -95,8 +95,8 @@ export const make = (options?: {
       pg: () =>
         sql`
           CREATE TABLE IF NOT EXISTS ${remotesTableSql} (
-            remote_id UUID NOT NULL,
-            entry_id UUID NOT NULL,
+            remote_id BYTEA NOT NULL,
+            entry_id BYTEA NOT NULL,
             sequence INT NOT NULL,
             PRIMARY KEY (remote_id, entry_id)
           )`,
@@ -343,12 +343,26 @@ const SqlUint8Array = Schema.Union([Schema.Uint8Array, ArrayBuffer]).pipe(
 
 const SqlEntryId = SqlUint8Array.pipe(Schema.decodeTo(EventJournal.EntryId))
 
+const SqlNatural = Schema.Union([
+  Schema.Natural,
+  Schema.FiniteFromString.check(Schema.isInt(), Schema.isGreaterThanOrEqualTo(0)),
+  Schema.BigInt.pipe(
+    Schema.decodeTo(
+      Schema.Natural,
+      SchemaTransformation.transform({
+        decode: Number,
+        encode: BigInt
+      })
+    )
+  )
+])
+
 const EntryRow = Schema.Struct({
   id: SqlEntryId,
   event: Schema.String,
   primary_key: Schema.String,
   payload: SqlUint8Array,
-  timestamp: Schema.Int
+  timestamp: SqlNatural
 })
 
 const EntryRowArray = Schema.Array(EntryRow)