effect 4.0.0-beta.22 → 4.0.0-beta.24

package/src/SchemaAST.ts

@@ -2028,7 +2028,7 @@ type Type =
 /** @internal */
 export type Sentinel = {
   readonly key: PropertyKey
-  readonly literal: LiteralValue
+  readonly literal: LiteralValue | symbol
 }
 
 function getCandidateTypes(ast: AST): ReadonlyArray<Type> {
@@ -2081,28 +2081,33 @@ function getCandidateTypes(ast: AST): ReadonlyArray<Type> {
 }
 
 /** @internal */
-export function collectSentinels(ast: AST): Array<Sentinel> | undefined {
+export function collectSentinels(ast: AST): Array<Sentinel> {
   switch (ast._tag) {
+    default:
+      return []
     case "Declaration": {
       const s = ast.annotations?.["~sentinels"]
-      return Array.isArray(s) && s.length ? s : undefined
+      return Array.isArray(s) ? s : []
     }
-    case "Objects": {
-      const v = ast.propertySignatures.flatMap((ps) =>
-        isLiteral(ps.type) && !isOptional(ps.type)
-          ? [{ key: ps.name, literal: ps.type.literal }]
-          : []
-      )
-      return v.length ? v : undefined
-    }
-    case "Arrays": {
-      const v = ast.elements.flatMap((e, i) =>
-        isLiteral(e) && !isOptional(e)
+    case "Objects":
+      return ast.propertySignatures.flatMap((ps): Array<Sentinel> => {
+        const type = ps.type
+        if (!isOptional(type)) {
+          if (isLiteral(type)) {
+            return [{ key: ps.name, literal: type.literal }]
+          }
+          if (isUniqueSymbol(type)) {
+            return [{ key: ps.name, literal: type.symbol }]
+          }
+        }
+        return []
+      })
+    case "Arrays":
+      return ast.elements.flatMap((e, i) => {
+        return isLiteral(e) && !isOptional(e)
           ? [{ key: i, literal: e.literal }]
           : []
-      )
-      return v.length ? v : undefined
-    }
+      })
     case "Suspend":
       return collectSentinels(ast.thunk())
   }
@@ -2110,7 +2115,7 @@ export function collectSentinels(ast: AST): Array<Sentinel> | undefined {
 
 type CandidateIndex = {
   byType?: { [K in Type]?: Array<AST> }
-  bySentinel?: Map<PropertyKey, Map<LiteralValue, Array<AST>>>
+  bySentinel?: Map<PropertyKey, Map<LiteralValue | symbol, Array<AST>>>
   otherwise?: { [K in Type]?: Array<AST> }
 }
 
@@ -2132,7 +2137,7 @@ function getIndex(types: ReadonlyArray<AST>): CandidateIndex {
     idx.byType ??= {}
     for (const t of types) (idx.byType[t] ??= []).push(a)
 
-    if (sentinels?.length) { // discriminated variants
+    if (sentinels.length > 0) { // discriminated variants
       idx.bySentinel ??= new Map()
       for (const { key, literal } of sentinels) {
         let m = idx.bySentinel.get(key)
@@ -2640,6 +2645,11 @@ export function replaceContext<A extends AST>(ast: A, context: Context | undefin
   })
 }
 
+/** @internal */
+export function getLastEncoding(ast: AST): AST {
+  return ast.encoding ? getLastEncoding(ast.encoding[ast.encoding.length - 1].to) : ast
+}
+
 /** @internal */
 export function annotate<A extends AST>(ast: A, annotations: Schema.Annotations.Annotations): A {
   if (ast.checks) {

package/src/SchemaRepresentation.ts

@@ -1771,6 +1771,8 @@ export const toSchemaDefaultReviver: Reviver<Schema.Top> = (s, recur) => {
         return Schema.Result(typeParameters[0], typeParameters[1])
       case "effect/HashSet":
         return Schema.HashSet(typeParameters[0])
+      case "effect/Chunk":
+        return Schema.Chunk(typeParameters[0])
     }
   }
 }

package/src/ServiceMap.ts

@@ -321,6 +321,7 @@ export interface Reference<in out Shape> extends Service<never, Shape> {
   readonly [ReferenceTypeId]: typeof ReferenceTypeId
   readonly defaultValue: () => Shape
   [Symbol.iterator](): EffectIterator<Reference<Shape>>
+  new(_: never): {}
 }
 
 /**

package/src/internal/schema/representation.ts

@@ -24,80 +24,13 @@ export function fromASTs(asts: readonly [AST.AST, ...Array<AST.AST>]): SchemaRep
 
   const referenceMap = new Map<AST.AST, string>()
   const uniqueReferences = new Set<string>()
-  const usedReferences = new Set<string>()
+  const visiting = new Set<AST.AST>()
 
   const schemas = Arr.map(asts, (ast) => recur(ast))
 
   return {
-    representations: Arr.map(schemas, compact),
-    references: Rec.map(Rec.filter(references, (_, k) => !isCompactable(k)), compact)
-  }
-
-  function isCompactable($ref: string): boolean {
-    return !usedReferences.has($ref)
-  }
-
-  function compact(s: SchemaRepresentation.Representation): SchemaRepresentation.Representation {
-    switch (s._tag) {
-      default:
-        return s
-      case "Declaration":
-        return {
-          ...s,
-          typeParameters: s.typeParameters.map(compact),
-          encodedSchema: compact(s.encodedSchema)
-        }
-      case "Reference": {
-        if (isCompactable(s.$ref)) {
-          return compact(references[s.$ref])
-        }
-        return s
-      }
-      case "Suspend":
-        return { ...s, thunk: compact(s.thunk) }
-      case "String":
-        return {
-          ...s,
-          ...(s.contentSchema ? { contentSchema: compact(s.contentSchema) } : undefined)
-        }
-      case "TemplateLiteral":
-        return { ...s, parts: s.parts.map(compact) }
-      case "Arrays":
-        return {
-          ...s,
-          elements: s.elements.map((e) => ({ ...e, type: compact(e.type) })),
-          rest: s.rest.map(compact)
-        }
-      case "Objects":
-        return {
-          ...s,
-          checks: s.checks.map(compactCheck),
-          propertySignatures: s.propertySignatures.map((ps) => ({ ...ps, type: compact(ps.type) })),
-          indexSignatures: s.indexSignatures.map((is) => ({
-            ...is,
-            parameter: compact(is.parameter),
-            type: compact(is.type)
-          }))
-        }
-      case "Union":
-        return { ...s, types: s.types.map(compact) }
-    }
-  }
-
-  function compactCheck<M extends SchemaRepresentation.Meta>(
-    check: SchemaRepresentation.Check<M>
-  ): SchemaRepresentation.Check<M> {
-    switch (check._tag) {
-      case "Filter":
-        return {
-          ...check,
-          meta: check.meta._tag === "isPropertyNames"
-            ? { _tag: "isPropertyNames", propertyNames: compact(check.meta.propertyNames) } as M
-            : check.meta
-        }
-      case "FilterGroup":
-        return { ...check, checks: Arr.map(check.checks, compactCheck) }
-    }
+    representations: schemas,
+    references
   }
 
   function gen(prefix: string = "_"): string {
@@ -115,32 +48,45 @@ export function fromASTs(asts: readonly [AST.AST, ...Array<AST.AST>]): SchemaRep
   function recur(ast: AST.AST, prefix?: string): SchemaRepresentation.Representation {
     const found = referenceMap.get(ast)
     if (found !== undefined) {
-      usedReferences.add(found)
       return { _tag: "Reference", $ref: found }
     }
 
-    const last = getLastEncoding(ast)
-
-    if (ast === last) {
-      const reference = ast._tag === "Declaration"
-        ? gen(ast._tag)
-        : gen(InternalAnnotations.resolveIdentifier(ast) ?? prefix ?? `${ast._tag}_`)
+    const last = AST.getLastEncoding(ast)
+    const identifier = InternalAnnotations.resolveIdentifier(ast) ?? prefix
 
-      const encodedSchemaPrefix = ast._tag === "Declaration"
-        ? InternalAnnotations.resolveIdentifier(ast) ?? prefix
-        : prefix
+    if (ast !== last) {
+      return recur(last, identifier)
+    }
 
+    // Has identifier → always create reference
+    if (identifier !== undefined) {
+      const reference = gen(identifier)
       referenceMap.set(ast, reference)
-      const out = on(ast, encodedSchemaPrefix)
+      const out = on(ast)
       references[reference] = out
       return { _tag: "Reference", $ref: reference }
-    } else {
-      return recur(last, InternalAnnotations.resolveIdentifier(ast) ?? prefix)
     }
-  }
 
-  function getLastEncoding(ast: AST.AST): AST.AST {
-    return ast.encoding ? ast.encoding[ast.encoding.length - 1].to : ast
+    // Recursion detected → create reference
+    if (visiting.has(ast)) {
+      const reference = gen(`${ast._tag}_`)
+      referenceMap.set(ast, reference)
+      return { _tag: "Reference", $ref: reference }
+    }
+
+    // Normal case → inline
+    visiting.add(ast)
+    const out = on(ast)
+    visiting.delete(ast)
+
+    // A descendant triggered reference creation (recursion)
+    const ref = referenceMap.get(ast)
+    if (ref !== undefined) {
+      references[ref] = out
+      return { _tag: "Reference", $ref: ref }
+    }
+
+    return out
   }
 
   function getEncodedSchema(last: AST.Declaration): AST.AST {
@@ -153,12 +99,12 @@ export function fromASTs(asts: readonly [AST.AST, ...Array<AST.AST>]): SchemaRep
     return AST.null
   }
 
-  function on(last: AST.AST, encodedSchemaPrefix?: string): SchemaRepresentation.Representation {
+  function on(last: AST.AST): SchemaRepresentation.Representation {
     const annotations = fromASTAnnotations(last.annotations)
     switch (last._tag) {
       case "Declaration": {
         // this must be executed before transforming the type parameters
-        const encodedSchema = recur(getEncodedSchema(last), encodedSchemaPrefix)
+        const encodedSchema = recur(getEncodedSchema(last))
         return {
           _tag: "Declaration",
           typeParameters: last.typeParameters.map((ast) => recur(ast)),
@@ -228,7 +174,7 @@ export function fromASTs(asts: readonly [AST.AST, ...Array<AST.AST>]): SchemaRep
         return {
           _tag: last._tag,
           elements: last.elements.map((e) => {
-            const last = getLastEncoding(e)
+            const last = AST.getLastEncoding(e)
             return {
               isOptional: AST.isOptional(last),
               type: recur(e),
@@ -243,7 +189,7 @@ export function fromASTs(asts: readonly [AST.AST, ...Array<AST.AST>]): SchemaRep
         return {
           _tag: last._tag,
           propertySignatures: last.propertySignatures.map((ps) => {
-            const last = getLastEncoding(ps.type)
+            const last = AST.getLastEncoding(ps.type)
             return {
               name: ps.name,
               type: recur(ps.type),
@@ -322,6 +268,7 @@ export function fromASTs(asts: readonly [AST.AST, ...Array<AST.AST>]): SchemaRep
 export const fromASTBlacklist: Set<string> = new Set([
   // `expected` is preserved because is useful to generate descriptions in JSON Schemas
   "~structural",
+  "~sentinels",
   "meta",
   "toArbitrary",
   "toArbitraryConstraint",

package/src/unstable/http/HttpClient.ts

@@ -3,9 +3,11 @@
  */
 import type { NonEmptyReadonlyArray } from "../../Array.ts"
 import * as Cause from "../../Cause.ts"
+import { Clock } from "../../Clock.ts"
+import * as Duration from "../../Duration.ts"
 import * as Effect from "../../Effect.ts"
 import * as Exit from "../../Exit.ts"
-import type { Fiber } from "../../Fiber.ts"
+import * as Fiber from "../../Fiber.ts"
 import { constFalse, constTrue, dual, flow, identity } from "../../Function.ts"
 import * as Inspectable from "../../Inspectable.ts"
 import * as Layer from "../../Layer.ts"
@@ -19,6 +21,7 @@ import * as ServiceMap from "../../ServiceMap.ts"
 import * as Stream from "../../Stream.ts"
 import * as Tracer from "../../Tracer.ts"
 import type { EqualsWith, ExcludeTag, ExtractTag, NoExcessProperties, NoInfer, Tags } from "../../Types.ts"
+import type * as RateLimiter from "../persistence/RateLimiter.ts"
 import * as Cookies from "./Cookies.ts"
 import * as Headers from "./Headers.ts"
 import * as Error from "./HttpClientError.ts"
@@ -630,7 +633,7 @@ export const make = (
     request: HttpClientRequest.HttpClientRequest,
     url: URL,
     signal: AbortSignal,
-    fiber: Fiber<HttpClientResponse.HttpClientResponse, Error.HttpClientError>
+    fiber: Fiber.Fiber<HttpClientResponse.HttpClientResponse, Error.HttpClientError>
   ) => Effect.Effect<HttpClientResponse.HttpClientResponse, Error.HttpClientError>
 ): HttpClient =>
   makeWith((effect) =>
@@ -1072,6 +1075,286 @@ export const retryTransient: {
   }
 )
 
+/**
+ * @since 4.0.0
+ * @category rate limiting
+ */
+export declare namespace WithRateLimiter {
+  /**
+   * @since 4.0.0
+   * @category rate limiting
+   */
+  export interface Options {
+    /**
+     * The `RateLimiter` service to use for rate limiting.
+     */
+    readonly limiter: RateLimiter.RateLimiter
+    /**
+     * The initial rate limit window duration.
+     */
+    readonly window: Duration.Input
+    /**
+     * The initial maximum number of allowed requests in the window.
+     */
+    readonly limit: number
+    /**
+     * The key to identify the rate limit. Requests with the same key will share
+     * the same rate limit. This can be used to implement per-user or
+     * per-endpoint rate limits.
+     */
+    readonly key: string | ((request: HttpClientRequest.HttpClientRequest) => string)
+    /**
+     * Defaults to `"fixed-window"`.
+     */
+    readonly algorithm?: "fixed-window" | "token-bucket" | undefined
+    /**
+     * Defaults to `1`.
+     */
+    readonly tokens?: number | ((request: HttpClientRequest.HttpClientRequest) => number) | undefined
+    /**
+     * Disable automatic limits updates from response headers.
+     */
+    readonly disableResponseInspection?: boolean | undefined
+  }
+}
+
+/**
+ * Applies request rate limiting using the `RateLimiter` service.
+ *
+ * It can update limits by inspecting common rate limit response headers and
+ * automatically retries HTTP `429` responses (or `HttpClientError` values
+ * wrapping a `429` response) by forcing the retry back through the limiter.
+ *
+ * @since 4.0.0
+ * @category rate limiting
+ */
+export const withRateLimiter: {
+  /**
+   * Applies request rate limiting using the `RateLimiter` service.
+   *
+   * It can update limits by inspecting common rate limit response headers and
+   * automatically retries HTTP `429` responses (or `HttpClientError` values
+   * wrapping a `429` response) by forcing the retry back through the limiter.
+   *
+   * @since 4.0.0
+   * @category rate limiting
+   */
+  (options: WithRateLimiter.Options): <E, R>(
+    self: HttpClient.With<E, R>
+  ) => HttpClient.With<E | RateLimiter.RateLimiterError, R>
+  /**
+   * Applies request rate limiting using the `RateLimiter` service.
+   *
+   * It can update limits by inspecting common rate limit response headers and
+   * automatically retries HTTP `429` responses (or `HttpClientError` values
+   * wrapping a `429` response) by forcing the retry back through the limiter.
+   *
+   * @since 4.0.0
+   * @category rate limiting
+   */
+  <E, R>(self: HttpClient.With<E, R>, options: WithRateLimiter.Options): HttpClient.With<E | RateLimiter.RateLimiterError, R>
+} = dual(2, <E, R>(
+  self: HttpClient.With<E, R>,
+  options: WithRateLimiter.Options
+): HttpClient.With<E | RateLimiter.RateLimiterError, R> => {
+  const initialState: RateLimiterState = {
+    limit: options.limit,
+    window: Duration.max(Duration.fromInputUnsafe(options.window), Duration.millis(1))
+  }
+  const states = new Map<string, RateLimiterState>()
+
+  const keyOption = options.key
+  const resolveKey: (request: HttpClientRequest.HttpClientRequest) => string = typeof keyOption === "function"
+    ? keyOption
+    : () => keyOption
+  const tokensOption = options.tokens
+  const resolveTokens: (request: HttpClientRequest.HttpClientRequest) => number | undefined =
+    typeof tokensOption === "function" ? tokensOption : () => tokensOption
+
+  const getState = (key: string): RateLimiterState => {
+    const current = states.get(key)
+    if (current !== undefined) {
+      return current
+    }
+    states.set(key, initialState)
+    return initialState
+  }
+
+  const onResponse = options.disableResponseInspection
+    ? undefined
+    : (clock: Clock, key: string, headers: Headers.Headers, tokens: number | undefined) => {
+      const current = getState(key)
+      const next = parseRateLimiterState(current, clock, headers, tokens)
+      if (next.limit !== current.limit || !Duration.equals(next.window, current.window)) {
+        states.set(key, next)
+      }
+    }
+
+  return transform(self, function loop(effect, request): Effect.Effect<
+    HttpClientResponse.HttpClientResponse,
+    E | RateLimiter.RateLimiterError,
+    R
+  > {
+    const fiber = Fiber.getCurrent()!
+    const clock = fiber.getRef(Clock)
+    const key = resolveKey(request)
+    const tokens = resolveTokens(request)
+    const current = getState(key)
+    return Effect.flatMap(
+      options.limiter.consume({
+        algorithm: options.algorithm,
+        onExceeded: "delay",
+        key,
+        limit: current.limit,
+        window: current.window,
+        tokens
+      }),
+      ({ delay }) => {
+        const run = Effect.matchEffect(effect, {
+          onSuccess(response) {
+            onResponse?.(clock, key, response.headers, tokens)
+            return response.status === 429 ? loop(effect, request) : Effect.succeed(response)
+          },
+          onFailure(error) {
+            if (isTooManyRequestsHttpClientError(error)) {
+              onResponse?.(clock, key, error.reason.response.headers, tokens)
+              return loop(effect, request)
+            }
+            return Effect.fail(error)
+          }
+        })
+        return Duration.isZero(delay) ? run : Effect.delay(run, delay)
+      }
+    )
+  })
+})
+
+interface RateLimiterState {
+  readonly limit: number
+  readonly window: Duration.Duration
+}
+
+const parseRateLimiterState = (
+  state: RateLimiterState,
+  clock: Clock,
+  headers: Headers.Headers,
+  tokens: number | undefined
+): RateLimiterState => {
+  const limit = parseRateLimitLimit(headers, tokens) ?? state.limit
+  const window = parseRateLimitWindow(clock, headers) ?? state.window
+  if (limit === state.limit && Duration.equals(window, state.window)) {
+    return state
+  }
+  return { limit, window }
+}
+
+const parseRateLimitLimit = (headers: Headers.Headers, tokens: number | undefined): number | undefined => {
+  const raw = getHeader(headers, "ratelimit-limit", "x-ratelimit-limit")
+  const value = parseNumberHeader(raw)
+  if (value !== undefined && value > 0) {
+    return value
+  }
+  const remaining = parseRateLimitRemaining(headers)
+  if (remaining === undefined) {
+    return undefined
+  }
+  return Math.max(remaining + (tokens !== undefined && tokens > 0 ? tokens : 1), 1)
+}
+
+const parseRateLimitRemaining = (headers: Headers.Headers): number | undefined => {
+  const raw = getHeader(headers, "ratelimit-remaining", "x-ratelimit-remaining")
+  const value = parseNumberHeader(raw)
+  return value !== undefined && value >= 0 ? value : undefined
+}
+
+const parseRateLimitWindow = (
+  clock: Clock,
+  headers: Headers.Headers
+): Duration.Duration | undefined => {
+  const retryAfter = parseRetryAfter(
+    clock,
+    getHeader(headers, "retry-after")
+  )
+  if (retryAfter !== undefined) {
+    return retryAfter
+  }
+  const resetAfter = parseResetAfter(getHeader(headers, "ratelimit-reset-after", "x-ratelimit-reset-after"))
+  if (resetAfter !== undefined) {
+    return resetAfter
+  }
+  return parseResetHeader(clock, getHeader(headers, "ratelimit-reset", "x-ratelimit-reset"))
+}
+
+const parseRetryAfter = (
+  clock: Clock,
+  value: string | undefined
+): Duration.Duration | undefined => {
+  if (value === undefined) {
+    return undefined
+  }
+  const numeric = parseNumberHeader(value)
+  if (numeric !== undefined) {
+    return Duration.max(Duration.seconds(numeric), Duration.millis(1))
+  }
+  const parsedDate = Date.parse(value)
+  if (Number.isNaN(parsedDate)) {
+    return undefined
+  }
+  const millis = parsedDate - clock.currentTimeMillisUnsafe()
+  if (millis <= 0) {
+    return Duration.millis(1)
+  }
+  return Duration.millis(millis)
+}
+
+const parseResetAfter = (value: string | undefined): Duration.Duration | undefined => {
+  const numeric = parseNumberHeader(value)
+  if (numeric === undefined || numeric <= 0) {
+    return undefined
+  }
+  return Duration.max(Duration.seconds(numeric), Duration.millis(1))
+}
+
+const parseResetHeader = (
+  clock: Clock,
+  value: string | undefined
+): Duration.Duration | undefined => {
+  const numeric = parseNumberHeader(value)
+  if (numeric === undefined || numeric <= 0) {
+    return undefined
+  }
+  const nowMillis = clock.currentTimeMillisUnsafe()
+  if (numeric > 1_000_000_000_000) {
+    return Duration.millis(Math.max(numeric - nowMillis, 1))
+  }
+  if (numeric > 1_000_000_000) {
+    return Duration.millis(Math.max((numeric * 1_000) - nowMillis, 1))
+  }
+  return Duration.max(Duration.seconds(numeric), Duration.millis(1))
+}
+
+const parseNumberHeader = (value: string | undefined): number | undefined => {
+  if (value === undefined) {
+    return undefined
+  }
+  const match = /-?\d+(?:\.\d+)?/.exec(value)
+  if (match === null) {
+    return undefined
+  }
+  const parsed = Number(match[0])
+  return Number.isFinite(parsed) ? parsed : undefined
+}
+
+const getHeader = (headers: Headers.Headers, ...keys: Array<string>): string | undefined => {
+  for (let i = 0; i < keys.length; i++) {
+    const value = headers[keys[i]]
+    if (value !== undefined) {
+      return value
+    }
+  }
+  return undefined
+}
+
 /**
  * Performs an additional effect after a successful request.
  *
@@ -1462,6 +1745,11 @@ const isTransientHttpError = (error: unknown) =>
   (error.reason._tag === "TransportError" ||
     (error.reason._tag === "StatusCodeError" && isTransientResponse(error.reason.response)))
 
+const isTooManyRequestsHttpClientError = (
+  error: unknown
+): error is Error.HttpClientError & { readonly reason: Error.StatusCodeError } =>
+  Error.isHttpClientError(error) && error.reason._tag === "StatusCodeError" && error.reason.response.status === 429
+
 const isTransientResponse = (response: HttpClientResponse.HttpClientResponse) =>
   response.status === 408 ||
   response.status === 429 ||

package/src/unstable/httpapi/HttpApi.ts

@@ -57,7 +57,7 @@ export interface HttpApi<
   /**
    * Prefix all endpoints in the `HttpApi`.
    */
-  prefix<const Prefix extends PathInput>(prefix: Prefix): HttpApi<Id, Groups>
+  prefix<const Prefix extends PathInput>(prefix: Prefix): HttpApi<Id, HttpApiGroup.AddPrefix<Groups, Prefix>>
 
   /**
    * Add a middleware to a `HttpApi`. It will be applied to all endpoints in the

package/src/unstable/httpapi/OpenApi.ts

@@ -363,7 +363,7 @@ export function fromApi<Id extends string, Groups extends HttpApiGroup.Any>(
 
       function processParameters(schema: Schema.Top | undefined, i: OpenAPISpecParameter["in"]) {
         if (schema) {
-          const ast = AST.toEncoded(schema.ast)
+          const ast = AST.getLastEncoding(schema.ast)
           if (AST.isObjects(ast)) {
             for (const ps of ast.propertySignatures) {
               op.parameters.push({