effect 4.0.6 → 4.0.7

package/src/cluster/SqlRunnerStorage.ts

@@ -38,6 +38,14 @@ import * as ShardingConfig from "./ShardingConfig.ts"
 
 const withTracerDisabled = Effect.withTracerEnabled(false)
 
+// The session holding a storage's advisory locks, as reported to other
+// sessions.
+interface LockSession {
+  pid: number
+  // MySQL only: the lock that pins the session's lock connection
+  readonly pinLock: string | undefined
+}
+
 // This exact FNV-1a hash, including its tag and UTF-8 encoding, is a persistent
 // advisory-lock wire format and must never change.
 const postgresLockNamespace = (prefix: string): number => {
@@ -69,6 +77,28 @@ const makeStorage = Effect.fnUntraced(function*(options: {
   // PostgreSQL exposes the signed int4 lock key through pg_locks as an unsigned oid.
   const pgLockNamespaceOid = pgLockNamespace >>> 0
 
+  // MySQL identifies a lock holder by connection id. Behind Vitess that is the
+  // id of vtgate's lock connection for the session, which vtgate replaces
+  // once the session releases all of its locks. A lock of the session's own
+  // pins that connection and reads its id.
+  const pinLockSession = Effect.fnUntraced(function*(conn: Connection, pinLock?: string) {
+    while (true) {
+      const name = pinLock ?? `cluster:pid:${(Math.random() * Number.MAX_SAFE_INTEGER) | 0}`
+      const taken = (yield* conn.executeValues(
+        `SELECT GET_LOCK('${name}', 10), IS_USED_LOCK('${name}')`,
+        []
+      ))[0] as [1 | null, number]
+      if (taken[0] === null) continue
+      return { pid: Number(taken[1]), pinLock: name }
+    }
+  })
+
+  // In milliseconds, capped to the largest value PostgreSQL accepts.
+  const idleSessionTimeout = Math.min(
+    Math.ceil(Duration.toMillis(Duration.fromInputUnsafe(config.shardLockExpiration))),
+    2147483647
+  )
+
   // Keep all PostgreSQL and MySQL shard-lock operations on a rebuildable
   // reserved connection, including when advisory locks are disabled.
   const acquireLockConn = singleSession ? undefined : sql.onDialectOrElse({
@@ -77,9 +107,18 @@ const makeStorage = Effect.fnUntraced(function*(options: {
         const conn = yield* Effect.orDie(sql.reserve).pipe(
           Scope.provide(scope)
         )
+        // Advisory locks belong to the session, so a frozen runner whose
+        // connection stays open would hold them indefinitely. End the session
+        // once it has been idle for shardLockExpiration, which releases them.
+        // Runners holding shards use it at least every shardLockExpiration / 3.
+        // Servers without idle_session_timeout (before PostgreSQL 14) keep the
+        // session open.
         const pid = disableAdvisoryLocks
           ? 0
-          : (yield* conn.executeValues("SELECT pg_backend_pid()", []))[0][0] as number
+          : (yield* conn.executeValues(
+            `SELECT pg_backend_pid(), (SELECT set_config(name, '${idleSessionTimeout}', false) FROM pg_settings WHERE name = 'idle_session_timeout')`,
+            []
+          ))[0][0] as number
         if (!disableAdvisoryLocks) {
           yield* Scope.addFinalizerExit(scope, () =>
             conn.executeRaw("SELECT pg_advisory_unlock_all()", []).pipe(
@@ -88,26 +127,17 @@ const makeStorage = Effect.fnUntraced(function*(options: {
               Effect.ignoreCause
             ))
         }
-        return [conn, pid] as const
+        return [conn, { pid, pinLock: undefined }] as const
       }, Effect.orDie),
     mysql: () =>
       Effect.fnUntraced(function*(scope: Scope.Scope) {
         const conn = yield* Effect.orDie(sql.reserve).pipe(
           Scope.provide(scope)
         )
-        if (disableAdvisoryLocks) return [conn, 0] as const
+        if (disableAdvisoryLocks) return [conn, { pid: 0, pinLock: undefined }] as const
         // we need to get the connection id using IS_USED_LOCK to properly
         // support vitess
-        let pid: number | undefined = undefined
-        while (pid === undefined) {
-          const address = `cluster:pid:${(Math.random() * Number.MAX_SAFE_INTEGER) | 0}`
-          const taken = (yield* conn.executeValues(
-            `SELECT GET_LOCK('${address}', 10), IS_USED_LOCK('${address}')`,
-            []
-          ))[0] as [1 | null, number]
-          if (taken[0] === null) continue
-          pid = taken[1]
-        }
+        const session: LockSession = yield* pinLockSession(conn)
         if (!disableAdvisoryLocks) {
           yield* Scope.addFinalizerExit(scope, () =>
             conn.executeRaw("SELECT RELEASE_ALL_LOCKS()", []).pipe(
@@ -116,7 +146,7 @@ const makeStorage = Effect.fnUntraced(function*(options: {
               Effect.ignoreCause
             ))
         }
-        return [conn, pid] as const
+        return [conn, session] as const
       }, Effect.orDie),
     orElse: () => undefined
   })
@@ -140,7 +170,7 @@ const makeStorage = Effect.fnUntraced(function*(options: {
     )
   })
 
-  const rebuildLockConn = (from?: readonly [Connection, number]) => {
+  const rebuildLockConn = (from?: readonly [Connection, LockSession]) => {
     const rebuild = lockConn?.rebuildUnsafe({ from })
     if (!rebuild) return Effect.void
     // Bound release so a stalled connection cannot block subsequent rebuilds.
@@ -152,13 +182,13 @@ const makeStorage = Effect.fnUntraced(function*(options: {
   }
   // SQL failures replace the held connection; deadlines are handled by withLockOperationDeadline.
   const useLockConn = <A, E, R>(
-    f: (conn: Connection, pid: number) => Effect.Effect<A, E, R>
+    f: (conn: Connection, pid: number, session: LockSession) => Effect.Effect<A, E, R>
   ): Effect.Effect<A, E, R> =>
     Effect.flatMap(
       lockConn!.await,
       (held) =>
         Effect.onError(
-          f(held[0], held[1]),
+          f(held[0], held[1].pid, held[1]),
           (cause) => Cause.hasInterruptsOnly(cause) ? Effect.void : rebuildLockConn(held)
         )
     )
@@ -270,6 +300,40 @@ const makeStorage = Effect.fnUntraced(function*(options: {
     return useLockConn((conn) => conn.executeValues(query, params))
   }
 
+  // Splits shardIds into the advisory locks held by the lock session and the rest.
+  const pgHeldLocks = Effect.fnUntraced(function*(conn: Connection, pid: number, shardIds: ReadonlyArray<string>) {
+    const held: Array<string> = []
+    const notHeld = new Map(shardIds.map((shardId) => [lockNumbers.get(shardId)!, shardId]))
+    const takenLocks = yield* conn.executeValues(
+      `SELECT objid FROM pg_locks WHERE locktype = 'advisory' AND granted = true AND classid = ${pgLockNamespaceOid} AND objsubid = 2 AND pid = ${pid} ORDER BY objid`,
+      []
+    )
+    for (let i = 0; i < takenLocks.length; i++) {
+      const lockNum = takenLocks[i][0] as number
+      const shardId = notHeld.get(lockNum)
+      if (shardId === undefined) continue
+      held.push(shardId)
+      notHeld.delete(lockNum)
+    }
+    return [held, notHeld] as const
+  })
+
+  const mysqlHeldLocks = Effect.fnUntraced(function*(conn: Connection, pid: number, shardIds: ReadonlyArray<string>) {
+    const takenLocks = (yield* conn.executeValues(`SELECT ${allMySqlTakenLocks}`, []))[0] as Array<number | null>
+    const held: Array<string> = []
+    const notHeld: Array<string> = []
+    for (let i = 0; i < shardIds.length; i++) {
+      const shardId = shardIds[i]
+      const lockTakenBy = takenLocks[shardIdsIndex.get(shardId)!]
+      if (lockTakenBy !== null && Number(lockTakenBy) === pid) {
+        held.push(shardId)
+      } else {
+        notHeld.push(shardId)
+      }
+    }
+    return [held, notHeld] as const
+  })
+
   const acquireLock = sql.onDialectOrElse({
     pg: () => {
       if (disableAdvisoryLocks) {
@@ -293,19 +357,7 @@ const makeStorage = Effect.fnUntraced(function*(options: {
       }
       return (_address: string, shardIds: ReadonlyArray<string>) =>
         useLockConn(Effect.fnUntraced(function*(conn, pid) {
-          const acquiredShardIds: Array<string> = []
-          const toAcquire = new Map(shardIds.map((shardId) => [lockNumbers.get(shardId)!, shardId]))
-          const takenLocks = yield* conn.executeValues(
-            `SELECT objid FROM pg_locks WHERE locktype = 'advisory' AND granted = true AND classid = ${pgLockNamespaceOid} AND objsubid = 2 AND pid = ${pid} ORDER BY objid`,
-            []
-          )
-          for (let i = 0; i < takenLocks.length; i++) {
-            const lockNum = takenLocks[i][0] as number
-            const shardId = toAcquire.get(lockNum)
-            if (shardId === undefined) continue
-            acquiredShardIds.push(shardId)
-            toAcquire.delete(lockNum)
-          }
+          const [acquiredShardIds, toAcquire] = yield* pgHeldLocks(conn, pid, shardIds)
           if (toAcquire.size === 0) {
             return acquiredShardIds
           }
@@ -339,18 +391,7 @@ const makeStorage = Effect.fnUntraced(function*(options: {
       }
       return (_address: string, shardIds: ReadonlyArray<string>) =>
         useLockConn(Effect.fnUntraced(function*(conn, pid) {
-          const takenLocks = (yield* conn.executeValues(`SELECT ${allMySqlTakenLocks}`, []))[0] as Array<number | null>
-          const acquiredShardIds: Array<string> = []
-          const toAcquire: Array<string> = []
-          for (let i = 0; i < shardIds.length; i++) {
-            const shardId = shardIds[i]
-            const lockTakenBy = takenLocks[shardIdsIndex.get(shardId)!]
-            if (lockTakenBy === pid) {
-              acquiredShardIds.push(shardId)
-            } else if (shardIds.includes(shardId)) {
-              toAcquire.push(shardId)
-            }
-          }
+          const [acquiredShardIds, toAcquire] = yield* mysqlHeldLocks(conn, pid, shardIds)
           if (toAcquire.length === 0) {
             return acquiredShardIds
           }
@@ -463,9 +504,15 @@ const makeStorage = Effect.fnUntraced(function*(options: {
   const stringLiteral = (s: string) => sql.literal(wrapString(s))
   const stringLiteralArr = (arr: ReadonlyArray<string>) => sql.literal(`(${arr.map(wrapString).join(",")})`)
 
+  // An advisory lock refresh only confirms the locks the current lock session
+  // holds. A replaced session has lost its predecessor's locks, possibly to
+  // another runner in between, so reacquiring them here would hide the loss.
   const refreshShards = sql.onDialectOrElse({
     pg: () => {
-      if (!disableAdvisoryLocks) return acquireLock
+      if (!disableAdvisoryLocks) {
+        return (_address: string, shardIds: ReadonlyArray<string>) =>
+          useLockConn((conn, pid) => Effect.map(pgHeldLocks(conn, pid, shardIds), ([held]) => held))
+      }
       return (address: string, shardIds: ReadonlyArray<string>) =>
         sql`
           WITH locked AS (
@@ -483,7 +530,10 @@ const makeStorage = Effect.fnUntraced(function*(options: {
         )
     },
     mysql: () => {
-      if (!disableAdvisoryLocks) return acquireLock
+      if (!disableAdvisoryLocks) {
+        return (_address: string, shardIds: ReadonlyArray<string>) =>
+          useLockConn((conn, pid) => Effect.map(mysqlHeldLocks(conn, pid, shardIds), ([held]) => held))
+      }
       return (address: string, shardIds: ReadonlyArray<string>) => {
         const shardIdsStr = stringLiteralArr(shardIds)
         return sql<Array<{ shard_id: string }>>`
@@ -558,7 +608,7 @@ const makeStorage = Effect.fnUntraced(function*(options: {
               `SELECT IS_USED_LOCK('${lockName}')`,
               []
             )
-            if (takenLocks.length === 0 || takenLocks[0][0] !== pid) return
+            if (takenLocks.length === 0 || Number(takenLocks[0][0]) !== pid) return
           }
         }))
     },
@@ -579,7 +629,12 @@ const makeStorage = Effect.fnUntraced(function*(options: {
     mysql: () => (address: string) =>
       disableAdvisoryLocks
         ? sql`DELETE FROM ${locksTableSql} WHERE address = ${address}`.pipe(execWithLockConn)
-        : sql`SELECT RELEASE_ALL_LOCKS()`.pipe(execWithLockConn, Effect.asVoid),
+        : useLockConn(Effect.fnUntraced(function*(conn, _pid, session) {
+          yield* conn.executeRaw("SELECT RELEASE_ALL_LOCKS()", [])
+          // Releasing every lock also releases the pin, so take it again.
+          const pinned = yield* pinLockSession(conn, session.pinLock)
+          session.pid = pinned.pid
+        })),
     orElse: () => (address: string) => sql`DELETE FROM ${locksTableSql} WHERE address = ${address}`
   })
 
@@ -612,8 +667,16 @@ const makeStorage = Effect.fnUntraced(function*(options: {
           withTracerDisabled
         ),
 
+    // A runner without shards leaves its lock connection idle, so the server
+    // may have ended the session, and the first operation on it fails and
+    // replaces it. Acquiring does not depend on the previous session, so retry
+    // once on the replacement.
     acquire: (address, shardIds) =>
-      withLockOperationDeadline(acquireLock(address, shardIds)).pipe(
+      withLockOperationDeadline(
+        lockConn
+          ? Effect.catch(acquireLock(address, shardIds), () => acquireLock(address, shardIds))
+          : acquireLock(address, shardIds)
+      ).pipe(
         PersistenceError.refail,
         withTracerDisabled
       ),

package/src/cluster/internal/entityManager.ts

@@ -20,6 +20,7 @@ import * as Schedule from "../../Schedule.ts"
 import * as Schema from "../../Schema.ts"
 import * as SchemaIssue from "../../SchemaIssue.ts"
 import * as Scope from "../../Scope.ts"
+import type { PersistenceError } from "../ClusterError.ts"
 import { AlreadyProcessingMessage, EntityNotAssignedToRunner, MailboxFull, MalformedMessage } from "../ClusterError.ts"
 import * as ClusterMetrics from "../ClusterMetrics.ts"
 import { isUninterruptibleForServer, Persisted, WithTransaction } from "../ClusterSchema.ts"
@@ -37,6 +38,7 @@ import type { ShardId } from "../ShardId.ts"
 import type { Sharding } from "../Sharding.ts"
 import { ShardingConfig } from "../ShardingConfig.ts"
 import * as Snowflake from "../Snowflake.ts"
+import * as ClusterAbandon from "./clusterAbandon.ts"
 import { CurrentActivationScope } from "./entityActivation.ts"
 import { EntityReaper } from "./entityReaper.ts"
 import { acquireEntity, releaseEntity } from "./interruptors.ts"
@@ -144,6 +146,19 @@ export const make = Effect.fnUntraced(function*<
     readonly disableFatalDefects?: boolean | undefined
     readonly defectRetryPolicy?: Schedule.Schedule<any, unknown, never, never> | undefined
     readonly spanAttributes?: Record<string, string> | undefined
+    /**
+     * Called once a retiring entity stops accepting requests, so callers
+     * waiting on requests it has not started can be released.
+     */
+    readonly abandonUndelivered: (
+      address: EntityAddress,
+      isDelivered: (requestId: Snowflake.Snowflake) => boolean
+    ) => Effect.Effect<void>
+    /**
+     * Whether this runner still owns the shard, checked before storing the
+     * final reply of a persisted request.
+     */
+    readonly isShardOwned: (shardId: ShardId) => Effect.Effect<boolean>
   }
 ) {
   const config = yield* ShardingConfig
@@ -248,6 +263,14 @@ export const make = Effect.fnUntraced(function*<
     // Finalizers run in reverse, so the mailbox series and the resident
     // mailbox are removed only after the drain finalizer below has finished.
     const activeRequests: EntityState["activeRequests"] = new Map()
+    // Drops a request that has left this entity, starting the idle timer when
+    // it was the last one.
+    const removeActiveRequest = (requestId: Snowflake.Snowflake) => {
+      activeRequests.delete(requestId)
+      if (activeRequests.size === 0) {
+        state.lastActiveCheck = clock.currentTimeMillisUnsafe()
+      }
+    }
     let mailboxSeries: Metric.Registration | undefined
     yield* Scope.addFinalizer(
       scope,
@@ -326,27 +349,27 @@ export const make = Effect.fnUntraced(function*<
                   return Effect.void
                 }
 
-                // For durable messages, ignore interrupts during shutdown.
-                // They will be retried when the entity is restarted.
-                // Also, if the request is uninterruptible, we ignore the
-                // interrupt.
                 const persisted = storageEnabled && Context.get(request.message.annotations, Persisted)
-                if (
-                  persisted &&
-                  Exit.hasInterrupts(response.exit) &&
-                  (isShuttingDown || request.terminating || isUninterruptibleForServer(request.message.annotations))
-                ) {
-                  if (!isShuttingDown && !request.terminating) {
+                if (persisted && Exit.isFailure(response.exit) && Cause.hasInterrupts(response.exit.cause)) {
+                  // A durable request interrupted by shutdown, or abandoned
+                  // because this owner can no longer serve it, is left for
+                  // replay instead of being answered.
+                  const abandoned = ClusterAbandon.isCause(response.exit.cause) &&
+                    (!isActive() || !options.sharding.hasShardId(address.shardId))
+                  if (isShuttingDown || request.terminating || abandoned) {
+                    removeActiveRequest(Snowflake.Snowflake(response.requestId))
+                    return options.storage.unregisterReplyHandler(request.message.envelope.requestId)
+                  }
+                  // An uninterruptible request ignores the interrupt and runs again.
+                  if (isUninterruptibleForServer(request.message.annotations)) {
                     request.sentExit = false
                     return server.write(0, requestEnvelope(request), requestWriteOptions(request)).pipe(
                       Effect.setContext(handlerContext),
                       Effect.forkIn(handlerScope)
                     )
                   }
-                  activeRequests.delete(Snowflake.Snowflake(response.requestId))
-                  return options.storage.unregisterReplyHandler(request.message.envelope.requestId)
                 }
-                const save = retryRespond(
+                const write = retryRespond(
                   4,
                   Effect.suspend(() =>
                     request.message.respond(
@@ -358,18 +381,36 @@ export const make = Effect.fnUntraced(function*<
                     )
                   )
                 )
+                // A runner that lost the shard must not decide the stored
+                // result. A rejection is not retried.
+                let fenced = false
+                const save = !persisted ? write : Effect.flatMap(
+                  options.isShardOwned(address.shardId),
+                  (owned): Effect.Effect<void, MalformedMessage | PersistenceError | EntityNotAssignedToRunner> => {
+                    if (owned) return write
+                    fenced = true
+                    return Effect.fail(new EntityNotAssignedToRunner({ address }))
+                  }
+                )
+                // Leave the request for the shard's owner: release the claim
+                // so its next poll sees it, and let local callers re-route.
+                const abandonFenced = Effect.suspend(() => {
+                  removeActiveRequest(Snowflake.Snowflake(response.requestId))
+                  return Effect.logDebug("Abandoning a request whose shard this runner no longer owns").pipe(
+                    Effect.andThen(options.storage.unregisterReplyHandler(request.message.envelope.requestId)),
+                    Effect.andThen(Effect.ignore(options.storage.resetRequests([request.message.envelope.requestId])))
+                  )
+                })
                 const complete = Effect.sync(() => {
                   if (storageEnabled) {
                     processedRequestIds.add(request.message.envelope.requestId)
                   }
-                  activeRequests.delete(Snowflake.Snowflake(response.requestId))
-
-                  // Start the idle timer when the last request completes.
-                  if (activeRequests.size === 0) {
-                    state.lastActiveCheck = clock.currentTimeMillisUnsafe()
-                  }
+                  removeActiveRequest(Snowflake.Snowflake(response.requestId))
                 })
-                const respond = Effect.orDie(Effect.andThen(save, complete))
+                const respond = Effect.andThen(save, complete).pipe(
+                  Effect.catchTag("EntityNotAssignedToRunner", () => abandonFenced),
+                  Effect.orDie
+                )
                 const transaction = request.transaction
                 if (!transaction) return respond
                 const exit = response.exit
@@ -381,8 +422,12 @@ export const make = Effect.fnUntraced(function*<
                 }
 
                 if (Exit.isSuccess(exit)) {
-                  transaction.settle = (outcome) =>
-                    Exit.isSuccess(outcome) ? complete : Effect.flatMap(
+                  transaction.settle = (outcome) => {
+                    if (Exit.isSuccess(outcome)) return complete
+                    // A rejected save failed the transaction, rolling back the
+                    // handler's writes; the request is left for the owner.
+                    if (fenced) return abandonFenced
+                    return Effect.flatMap(
                       hasStoredExit(request.message.envelope.requestId),
                       (committed) => {
                         if (!committed) return restartFrom(outcome.cause)
@@ -393,6 +438,7 @@ export const make = Effect.fnUntraced(function*<
                         )
                       }
                     )
+                  }
                   return Effect.orDie(save)
                 }
                 transaction.settle = (outcome) =>
@@ -539,6 +585,13 @@ export const make = Effect.fnUntraced(function*<
               request.terminating = true
               yield* state.write(0, { _tag: "Interrupt", requestId: requestId as any, interruptors: [] })
             }
+            // A handler this drain waits for may be parked on a request the
+            // retired entity will never start, such as its own queued request.
+            // Release callers before `Eof`, which waits for handler acquisition.
+            yield* options.abandonUndelivered(
+              address,
+              (requestId) => activeRequests.get(requestId)?.delivered === true
+            )
             yield* state.write(0, { _tag: "Eof" })
           }).pipe(
             Effect.andThen(endLatch.await),

package/src/devtools/DevToolsClient.ts

@@ -219,6 +219,7 @@ const makeTracerEffect = Effect.gen(function*() {
       client.sendUnsafe(spanSnapshot(span))
       const oldEvent = span.event
       span.event = function(this: Tracer.Span, name, startTime, attributes) {
+        if (span.status._tag === "Ended") return
         client.sendUnsafe({
           _tag: "SpanEvent",
           traceId: span.traceId,
@@ -232,6 +233,7 @@ const makeTracerEffect = Effect.gen(function*() {
 
       const oldEnd = span.end
       span.end = function(this: Tracer.Span, endTime, exit) {
+        if (span.status._tag === "Ended") return
         oldEnd.call(this, endTime, exit)
         client.sendUnsafe(spanSnapshot(span))
       }

package/src/encoding/Base64.ts

@@ -148,6 +148,16 @@ export const decode = (str: string): Result.Result<Uint8Array, EncodingError> =>
 export const decodeString = (str: string) => Result.map(decode(str), (_) => decoder.decode(_))
 
 const encodeBytes = (bytes: Uint8Array): string => {
+  const length = bytes.length
+  // Looked up on the prototype at call time, so own or subclass overrides are ignored.
+  const toBase64 = (Uint8Array.prototype as { readonly toBase64?: (this: Uint8Array) => string }).toBase64
+  // The size guard also keeps detached buffers (length 0) away from native `toBase64`, which throws on them.
+  if (length >= 32 && typeof toBase64 === "function") return toBase64.call(bytes)
+  // Below ~128 bytes, appending to a string beats filling a byte array and decoding it once.
+  return length < 128 ? encodeShort(bytes) : encodeLong(bytes)
+}
+
+const encodeShort = (bytes: Uint8Array): string => {
   const length = bytes.length
   let result = ""
   let i: number
@@ -171,10 +181,38 @@ const encodeBytes = (bytes: Uint8Array): string => {
   return result
 }
 
+const encodeLong = (bytes: Uint8Array): string => {
+  const length = bytes.length
+  const result = new Uint8Array(Math.ceil(length / 3) * 4)
+  let i: number
+  let j = 0
+  for (i = 2; i < length; i += 3, j += 4) {
+    result[j] = alphabetCodes[bytes[i - 2] >> 2]
+    result[j + 1] = alphabetCodes[((bytes[i - 2] & 0x03) << 4) | (bytes[i - 1] >> 4)]
+    result[j + 2] = alphabetCodes[((bytes[i - 1] & 0x0f) << 2) | (bytes[i] >> 6)]
+    result[j + 3] = alphabetCodes[bytes[i] & 0x3f]
+  }
+  if (i === length + 1) {
+    result[j] = alphabetCodes[bytes[i - 2] >> 2]
+    result[j + 1] = alphabetCodes[(bytes[i - 2] & 0x03) << 4]
+    result[j + 2] = paddingCode
+    result[j + 3] = paddingCode
+  }
+  if (i === length) {
+    result[j] = alphabetCodes[bytes[i - 2] >> 2]
+    result[j + 1] = alphabetCodes[((bytes[i - 2] & 0x03) << 4) | (bytes[i - 1] >> 4)]
+    result[j + 2] = alphabetCodes[(bytes[i - 1] & 0x0f) << 2]
+    result[j + 3] = paddingCode
+  }
+  return decoder.decode(result)
+}
+
 const encoder = new TextEncoder()
 const decoder = new TextDecoder()
 const stripCrlf = (str: string) => str.replace(/[\n\r]/g, "")
 const alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"
+const alphabetCodes = encoder.encode(alphabet)
+const paddingCode = "=".charCodeAt(0)
 const codes = new Uint8Array(123).fill(255)
 for (let i = 0; i < alphabet.length; i++) codes[alphabet.charCodeAt(i)] = i
 codes["=".charCodeAt(0)] = 0

package/src/encoding/Sse.ts

@@ -308,9 +308,10 @@ export function makeParser(onParse: (event: AnyEvent) => void, options?: DecodeO
   }
 
   function feed(chunk: string): SseError | undefined {
-    // Strip any UTF-8 byte order mark (BOM) at the start of the stream.
-    // Note that we do not strip any non - UTF8 BOM, as eventsource streams are
-    // always decoded as UTF8 as per the specification.
+    // Empty chunks must not consume the leading BOM check below
+    if (chunk.length === 0) return
+
+    // Strip a UTF-8 byte order mark (BOM) at the start of the stream
     if (isFirstChunk && chunk.startsWith(BOM)) {
       chunk = chunk.slice(BOM.length)
     }

package/src/encoding/Toml.ts

@@ -47,6 +47,8 @@ class TomlParser {
   private line = 1
   private column = 1
   private readonly explicitTables = new Set<Table>()
+  // Tables defined by dotted keys; a header cannot define them again.
+  private readonly dottedTables = new Set<Table>()
 
   constructor(input: string) {
     this.input = input
@@ -87,7 +89,7 @@ class TomlParser {
     this.finishStatement()
 
     const table = this.resolveTable(path, array)
-    if (!array && this.explicitTables.has(table)) {
+    if (!array && (this.explicitTables.has(table) || this.dottedTables.has(table))) {
       this.fail(`Cannot redefine table '${path.join(".")}'`)
     }
     if (!array) {
@@ -120,7 +122,7 @@ class TomlParser {
         value = makeTable()
         table[key] = value
       }
-      if (Array.isArray(value)) {
+      if (Array.isArray(value) && !last) {
         value = value[value.length - 1]
       }
       if (!isTable(value)) {
@@ -140,11 +142,12 @@ class TomlParser {
         const child = makeTable()
         table[key] = child
         table = child
-      } else if (isTable(existing)) {
+      } else if (isTable(existing) && !this.explicitTables.has(existing)) {
         table = existing
       } else {
         this.fail(`Cannot redefine existing key '${keys.slice(0, index + 1).join(".")}'`)
       }
+      this.dottedTables.add(table)
     }
     const key = keys[keys.length - 1]
     if (hasOwn.call(table, key)) {
@@ -255,17 +258,17 @@ class TomlParser {
       if (normalized.endsWith("nan")) return Number.NaN
       return normalized[0] === "-" ? Number.NEGATIVE_INFINITY : Number.POSITIVE_INFINITY
     }
-    if (/^0x[0-9A-Fa-f](?:_?[0-9A-Fa-f])*$/.test(token)) {
-      return Number.parseInt(normalized.slice(2), 16)
-    }
-    if (/^0o[0-7](?:_?[0-7])*$/.test(token)) {
-      return Number.parseInt(normalized.slice(2), 8)
-    }
-    if (/^0b[01](?:_?[01])*$/.test(token)) {
-      return Number.parseInt(normalized.slice(2), 2)
-    }
-    if (/^[+-]?(?:0|[1-9](?:_?\d)*)$/.test(token)) {
-      return Number(normalized)
+    if (
+      /^0x[0-9A-Fa-f](?:_?[0-9A-Fa-f])*$/.test(token) ||
+      /^0o[0-7](?:_?[0-7])*$/.test(token) ||
+      /^0b[01](?:_?[01])*$/.test(token) ||
+      /^[+-]?(?:0|[1-9](?:_?\d)*)$/.test(token)
+    ) {
+      const value = Number(normalized)
+      if (!Number.isSafeInteger(value)) {
+        this.fail(`Integer out of range '${token}'`)
+      }
+      return value
     }
     if (
       /^[+-]?(?:(?:0|[1-9](?:_?\d)*)\.\d(?:_?\d)*(?:[eE][+-]?\d(?:_?\d)*)?|(?:0|[1-9](?:_?\d)*)[eE][+-]?\d(?:_?\d)*)$/
@@ -330,10 +333,15 @@ class TomlParser {
       } else if (escape === "u" || escape === "U") {
         const length = escape === "u" ? 4 : 8
         const hex = this.input.slice(this.index, this.index + length)
-        if (!new RegExp(`^[0-9A-Fa-f]{${length}}$`).test(hex)) {
+        const codePoint = Number.parseInt(hex, 16)
+        if (
+          !new RegExp(`^[0-9A-Fa-f]{${length}}$`).test(hex) ||
+          (codePoint >= 0xD800 && codePoint <= 0xDFFF) ||
+          codePoint > 0x10FFFF
+        ) {
           this.fail("Invalid unicode escape")
         }
-        output += String.fromCodePoint(Number.parseInt(hex, 16))
+        output += String.fromCodePoint(codePoint)
         this.advance(length)
       } else {
         this.fail(`Invalid escape '\\${escape}'`)
@@ -470,9 +478,18 @@ class TomlParser {
 
   private advance(count = 1): void {
     for (let offset = 0; offset < count; offset++) {
+      const code = this.input.charCodeAt(this.index)
+      if (
+        (code < 0x20 && code !== 0x09 && code !== 0x0A && !(code === 0x0D && this.input[this.index + 1] === "\n")) ||
+        code === 0x7F
+      ) {
+        this.fail("Invalid control character")
+      }
       if (this.input[this.index] === "\n") {
         this.line++
         this.column = 1
+      } else if (this.input[this.index] === "\r" && this.input[this.index + 1] !== "\n") {
+        this.fail("Bare carriage return")
       } else {
         this.column++
       }