effect 4.0.5 → 4.0.7

package/src/http-api/HttpApiTest.ts

@@ -11,19 +11,21 @@
  */
 import * as Context from "../Context.ts"
 import * as Effect from "../Effect.ts"
+import * as Exit from "../Exit.ts"
 import type { FileSystem } from "../FileSystem.ts"
 import type { Generator } from "../http/Etag.ts"
 import * as HttpClient from "../http/HttpClient.ts"
+import * as HttpEffect from "../http/HttpEffect.ts"
 import type { HttpPlatform } from "../http/HttpPlatform.ts"
 import * as HttpRouter from "../http/HttpRouter.ts"
 import * as HttpServerRequest from "../http/HttpServerRequest.ts"
 import * as HttpServerResponse from "../http/HttpServerResponse.ts"
-import * as preResponseHandler from "../http/internal/preResponseHandler.ts"
+import { CurrentRequestState, RequestState } from "../http/internal/requestState.ts"
 import * as Layer from "../Layer.ts"
 import type { Path } from "../Path.ts"
-import type { Scope } from "../Scope.ts"
+import * as Scope from "../Scope.ts"
 import type * as HttpApi from "./HttpApi.ts"
-import type { HandlerRuntime } from "./HttpApiBuilder.ts"
+import type { GroupService, HandlerRuntime } from "./HttpApiBuilder.ts"
 import * as HttpApiBuilder from "./HttpApiBuilder.ts"
 import * as HttpApiClient from "./HttpApiClient.ts"
 import type * as HttpApiEndpoint from "./HttpApiEndpoint.ts"
@@ -37,6 +39,17 @@ import type * as HttpApiGroup from "./HttpApiGroup.ts"
  * Handlers for the selected groups are taken from the environment; unselected
  * groups are wired with placeholder handlers that fail if called.
  *
+ * Each call runs its handler in a request scope that is a child of the
+ * `Scope` the client was created in. A streaming response keeps that scope
+ * open until its body ends, and a body that is never read is closed with the
+ * owning `Scope`.
+ *
+ * A `Stream` payload reaches the handler unbuffered: each element is pulled
+ * from the caller's stream as the handler reads it. If the caller's stream
+ * fails, reading the request body fails with it. Client aborts and socket
+ * behavior such as draining unread bodies, closing the connection, or
+ * answering with `499` are not emulated.
+ *
  * @stability unstable
  * @category testing
  * @since 4.0.0
@@ -62,7 +75,7 @@ export const groups = Effect.fnUntraced(function*<
   | Generator
   | HttpPlatform
   | Path
-  | Scope
+  | Scope.Scope
 > {
   let context = yield* Effect.context<HttpApiGroup.ToService<ApiId, SelectedGroups>>()
 
@@ -85,7 +98,8 @@ export const groups = Effect.fnUntraced(function*<
       handlers.set(endpointIdentifier, handler)
       routes.push(HttpApiBuilder.handlerToRoute(api, group as any, handler, context))
     }
-    context = Context.add(context, group as any, { handlers, routes })
+    const service: GroupService = { handlers, routes }
+    context = Context.add(context, group as any, service)
   }
 
   const layer: Layer.Layer<
@@ -100,18 +114,29 @@ export const groups = Effect.fnUntraced(function*<
     Layer.provide(Layer.succeedContext(context))
   ) as any
   const handler = yield* HttpRouter.toHttpEffect(layer)
-  const httpClient = HttpClient.make(Effect.fnUntraced(function*(request) {
+  const scope = yield* Effect.scope
+  const httpClient = HttpClient.make((request) => {
+    const requestScope = Scope.forkUnsafe(scope)
     const serverRequest = HttpServerRequest.fromClientRequest(request)
-    const response = yield* handler.pipe(
+    const state = new RequestState()
+    let transferred = false
+    return handler.pipe(
       Effect.flatMap((response) => {
-        const preResponse = preResponseHandler.requestPreResponseHandlers.get(serverRequest.source)
+        const preResponse = state.preResponseHandler
         return preResponse === undefined ? Effect.succeed(response) : preResponse(serverRequest, response)
       }),
+      Effect.map((response) => {
+        const transfer = HttpEffect.scopeTransferToStream(response)
+        transferred = transfer !== response
+        return HttpServerResponse.toClientResponse(transfer, { request })
+      }),
       Effect.provideService(HttpServerRequest.HttpServerRequest, serverRequest),
-      Effect.orDie
+      Effect.provideService(CurrentRequestState, state),
+      Effect.provideService(Scope.Scope, requestScope),
+      Effect.orDie,
+      Effect.onExit((exit) => transferred && Exit.isSuccess(exit) ? Effect.void : Scope.close(requestScope, exit))
     )
-    return HttpServerResponse.toClientResponse(response, { request })
-  }, Effect.scoped))
+  })
 
   return yield* HttpApiClient.makeWith(api, {
     httpClient,

package/src/http-api/OpenApi.ts

@@ -229,9 +229,20 @@ export const annotations: (
   transform: Transform
 })
 
-const defaultOptions: SchemaRepresentation.ToRepresentationOptions = {}
+/**
+ * Options for {@link fromApi}.
+ *
+ * @stability unstable
+ * @category models
+ * @since 4.0.6
+ */
+export interface FromApiOptions
+  extends SchemaRepresentation.ToRepresentationOptions, Pick<Schema.ToJsonSchemaOptions, "includeAnnotationKey">
+{}
+
+const defaultOptions: FromApiOptions = {}
 const apiCache = new WeakMap<
-  SchemaRepresentation.ToRepresentationOptions,
+  FromApiOptions,
   WeakMap<HttpApi.Constraint, OpenAPISpec>
 >()
 
@@ -285,7 +296,8 @@ function processAnnotation<Services, S, I>(
  * integrates annotations like descriptions, summaries, external documentation,
  * and overrides. The optional reference policy receives canonical JSON encoded
  * ASTs and controls which schemas are extracted into components. By default, only candidates with resolved identifiers
- * become references; anonymous non-recursive schemas remain inline.
+ * become references; anonymous non-recursive schemas remain inline. The optional `includeAnnotationKey` predicate selects
+ * custom schema annotation keys, such as `x-*` vendor extensions, to include in the generated schemas.
  *
  * **Gotchas**
  *
@@ -299,7 +311,7 @@ function processAnnotation<Services, S, I>(
  */
 export function fromApi<Id extends string, Groups extends HttpApiGroup.Constraint>(
   api: HttpApi.HttpApi<Id, Groups>,
-  options?: SchemaRepresentation.ToRepresentationOptions
+  options?: FromApiOptions
 ): OpenAPISpec {
   const resolvedOptions = options ?? defaultOptions
   let cache = apiCache.get(resolvedOptions)
@@ -329,7 +341,7 @@ type SchemaOp =
 
 function makeOpenApi<Id extends string, Groups extends HttpApiGroup.Constraint>(
   api: HttpApi.HttpApi<Id, Groups>,
-  options: SchemaRepresentation.ToRepresentationOptions
+  options: FromApiOptions
 ): OpenAPISpec {
   let spec: OpenAPISpec = {
     openapi: "3.1.0",
@@ -423,10 +435,7 @@ function makeOpenApi<Id extends string, Groups extends HttpApiGroup.Constraint>(
       }
 
       const paramNames = HttpApiPath.getParamNames(endpoint.params)
-      const path = endpoint.path.replace(
-        /:(\w+)\??/g,
-        (match, key: string) => paramNames === undefined || paramNames.has(key) ? `{${key}}` : match
-      )
+      const path = HttpApiPath.replaceParams(endpoint.path, paramNames, (key) => `{${key}}`)
       const method = endpoint.method.toLowerCase() as Lowercase<HttpMethod.HttpMethod>
       const isQuery = method === "query"
       const operationKey = isQuery ? "QUERY" : method
@@ -467,32 +476,44 @@ function makeOpenApi<Id extends string, Groups extends HttpApiGroup.Constraint>(
           if (streamContent !== undefined) {
             streamContent.forEach((stream, contentType) => {
               op.responses[status].content ??= {}
-              if (HttpApiSchema.isStreamSse(stream)) {
-                InternalRecord.assignProperty(op.responses[status].content, contentType, {
-                  schema: registerSchema(stream.events.ast),
-                  "x-effect-stream": {
-                    encoding: "sse",
-                    causeSchema: registerSchema(Schema.Cause(stream.error, Schema.Defect()).ast),
-                    errorSchema: registerSchema(stream.error.ast),
-                    failureEvent: reservedStreamFailureEvent
-                  }
-                })
-              } else {
-                InternalRecord.assignProperty(op.responses[status].content, contentType, {
-                  schema: {
-                    type: "string",
-                    format: "binary"
-                  },
-                  "x-effect-stream": {
-                    encoding: "uint8array"
-                  }
-                })
-              }
+              InternalRecord.assignProperty(op.responses[status].content, contentType, makeStreamMediaType(stream))
             })
           }
         }
       }
 
+      function makeStreamMediaType(stream: HttpApiSchema.StreamSchema): OpenApiSpecMediaType {
+        if (HttpApiSchema.isStreamSse(stream)) {
+          return {
+            schema: registerSchema(stream.events.ast),
+            "x-effect-stream": {
+              encoding: "sse",
+              causeSchema: registerSchema(Schema.Cause(stream.error, Schema.Defect()).ast),
+              errorSchema: registerSchema(stream.error.ast),
+              failureEvent: reservedStreamFailureEvent
+            }
+          }
+        }
+        if (HttpApiSchema.isStreamNdjson(stream)) {
+          // Each line is one element, so the media type documents the element schema.
+          return {
+            schema: registerSchema(stream.schema.ast),
+            "x-effect-stream": {
+              encoding: "ndjson"
+            }
+          }
+        }
+        return {
+          schema: {
+            type: "string",
+            format: "binary"
+          },
+          "x-effect-stream": {
+            encoding: "uint8array"
+          }
+        }
+      }
+
       function processParameters(schema: Schema.Constraint | undefined, i: OpenAPISpecParameter["in"]) {
         if (schema) {
           schemaOps.push({
@@ -527,10 +548,13 @@ function makeOpenApi<Id extends string, Groups extends HttpApiGroup.Constraint>(
         if (!HttpApiMiddleware.isSecurity(middleware)) {
           return
         }
-        for (const [name, security] of Object.entries(middleware.security)) {
-          processHttpApiSecurity(name, security)
-          op.security.push({ [name]: [] })
+        const names = Object.keys(middleware.security)
+        for (const name of names) {
+          processHttpApiSecurity(name, middleware.security[name])
         }
+        // Stacked middleware must all pass (AND); schemes within one middleware are alternatives (OR).
+        const requirements = op.security.length === 0 ? [{}] : op.security
+        op.security = requirements.flatMap((requirement) => names.map((name) => ({ ...requirement, [name]: [] })))
       })
 
       function processHttpApiSecurity(
@@ -554,27 +578,41 @@ function makeOpenApi<Id extends string, Groups extends HttpApiGroup.Constraint>(
 
       const hasBody = HttpMethod.hasBody(endpoint.method)
       if (hasBody) {
-        const schemasByContentType = new Map<string, {
-          readonly encoding: HttpApiSchema.PayloadEncoding
-          readonly schemas: Array<Schema.Top>
-        }>()
+        const schemasByContentType = new Map<
+          string,
+          {
+            readonly encoding: HttpApiSchema.PayloadEncoding
+            readonly schemas: Array<Schema.Top>
+          } | HttpApiSchema.StreamSchema
+        >()
         for (const schema of HttpApiEndpoint.getPayloadSchemas(endpoint)) {
+          if (HttpApiSchema.isStreamSchema(schema)) {
+            // Endpoint construction rejects a buffered payload sharing a streaming payload's media type.
+            for (const contentType of schema.contentTypes) {
+              schemasByContentType.set(contentType, schema)
+            }
+            continue
+          }
           if (HttpApiSchema.isNoContent(schema.ast)) continue
           const encoding = HttpApiSchema.getPayloadEncoding(schema.ast, endpoint.method)
           const existing = schemasByContentType.get(encoding.contentType)
           if (existing === undefined) {
             schemasByContentType.set(encoding.contentType, { encoding, schemas: [schema] })
-          } else {
+          } else if (!HttpApiSchema.isStreamSchema(existing)) {
             existing.schemas.push(schema)
           }
         }
         if (schemasByContentType.size > 0) {
           const content: OpenApiSpecContent = {}
-          for (const [contentType, { encoding, schemas }] of schemasByContentType) {
-            const asts = schemas.map(SchemaAST.getAST)
+          for (const [contentType, entry] of schemasByContentType) {
+            if (HttpApiSchema.isStreamSchema(entry)) {
+              InternalRecord.assignProperty(content, contentType, makeStreamMediaType(entry))
+              continue
+            }
+            const asts = entry.schemas.map(SchemaAST.getAST)
             const ast = asts.length === 1 ? asts[0] : new SchemaAST.Union(asts)
             InternalRecord.assignProperty(content, contentType, {
-              schema: registerSchema(ast, encoding._tag)
+              schema: registerSchema(ast, entry.encoding._tag)
             })
           }
           op.requestBody = { content, required: true }
@@ -678,7 +716,7 @@ function makeOpenApi<Id extends string, Groups extends HttpApiGroup.Constraint>(
 
 function compileSchemaOps(
   schemaOps: Array<SchemaOp>,
-  options: SchemaRepresentation.ToRepresentationOptions,
+  options: FromApiOptions,
   components: JsonSchema.Definitions
 ): void {
   if (!Arr.isArrayNonEmpty(schemaOps)) return
@@ -697,7 +735,7 @@ function compileSchemaOps(
   const jsonSchemaMultiDocument = JsonSchema.toMultiDocumentOpenApi3_1(
     InternalToJsonSchemaDocument.toJsonSchemaMultiDocument(
       { representations, references },
-      { onExcessProperty: "error" }
+      { onExcessProperty: "error", includeAnnotationKey: options.includeAnnotationKey }
     )
   )
 
@@ -792,6 +830,9 @@ function selectReferences(
 
     pushChecks(representation.checks)
     switch (representation._tag) {
+      case "String":
+        if (representation.content?.schema !== undefined) stack.push(representation.content.schema)
+        break
       case "Declaration":
         stack.push(...representation.typeParameters)
         break
@@ -861,7 +902,13 @@ function extractResponseBodies(
     const headers = HttpApiSchema.isWithHeaders(schema) ? schema.headers : annotation?.headersCodec
     const status = getStatus(schema)
     const ast = body.ast
-    if (HttpApiSchema.isStreamSchema(body)) {
+    if (HttpApiSchema.isUpgrade(body)) {
+      addNoContent(status, getDescription(ast) ?? "Switching Protocols")
+      map.get(status)!.headers.push(
+        Schema.Struct({ connection: Schema.Literal("Upgrade"), upgrade: Schema.Literal(body.protocol) })
+      )
+      if (body.headers !== undefined) map.get(status)!.headers.push(body.headers)
+    } else if (HttpApiSchema.isStreamSchema(body)) {
       addStreamContent(body, status)
     } else if (HttpApiSchema.isNoContent(ast)) {
       addNoContent(status, getDescription(schema.ast) ?? getDescription(ast) ?? "<No Content>")
@@ -937,20 +984,14 @@ function extractResponseBodies(
     stream: HttpApiSchema.StreamSchema,
     status: number
   ) {
-    const statusMap = map.get(status)
+    let statusMap = map.get(status)
     if (statusMap === undefined) {
-      map.set(status, {
-        descriptions: new Set(),
-        content: undefined,
-        headers: [],
-        streamContent: new Map([[stream.contentType, stream]])
-      })
-    } else {
-      if (statusMap.streamContent === undefined) {
-        statusMap.streamContent = new Map([[stream.contentType, stream]])
-      } else {
-        statusMap.streamContent.set(stream.contentType, stream)
-      }
+      statusMap = { descriptions: new Set(), content: undefined, headers: [], streamContent: undefined }
+      map.set(status, statusMap)
+    }
+    const streamContent = statusMap.streamContent ??= new Map()
+    for (const contentType of stream.contentTypes) {
+      streamContent.set(contentType, stream)
     }
   }
 }
@@ -1259,7 +1300,8 @@ export interface OpenApiSpecMediaType {
 }
 
 /**
- * Effect-specific metadata for generated streaming response media types.
+ * Effect-specific metadata for generated streaming request and response media
+ * types.
  *
  * @stability unstable
  * @category models
@@ -1275,6 +1317,9 @@ export type OpenApiSpecEffectStream =
   | {
     encoding: "uint8array"
   }
+  | {
+    encoding: "ndjson"
+  }
 
 /**
  * Generated OpenAPI request body object for endpoint payloads.

package/src/http-api/internal/path.ts

@@ -1,3 +1,4 @@
+import { getClosingParenthensePosition } from "../../http/FindMyWay/internal/router.ts"
 import type * as Schema from "../../Schema.ts"
 import * as SchemaAST from "../../SchemaAST.ts"
 
@@ -24,17 +25,8 @@ export function toRouterPath(path: string, schema: Schema.Constraint | undefined
   if (paramNames === undefined) return path
 
   let out = ""
-  let depth = 0
-  let regexStart = -1
   for (let i = 0; i < path.length; i++) {
     const char = path[i]
-    if (depth > 0 || i === regexStart) {
-      out += char
-      if (char === "\\") out += path[++i] ?? ""
-      else if (char === "(") depth++
-      else if (char === ")") depth--
-      continue
-    }
     if (char !== ":") {
       out += char
       continue
@@ -50,12 +42,48 @@ export function toRouterPath(path: string, schema: Schema.Constraint | undefined
       out += param[0]
       i += param[0].length - 1
       const next = path[i + 1]
-      if (next === "(") regexStart = i + 1
-      // FindMyWay needs a regex to end the parameter name here.
-      else if (next !== undefined && !"/.-?".includes(next)) out += "(.*?)"
+      if (next === "(") {
+        const end = getClosingParenthensePosition(path, i + 1)
+        out += path.slice(i + 1, end + 1)
+        i = end
+      } else if (next !== undefined && !"/.-?".includes(next)) {
+        // FindMyWay needs a regex to end the parameter name here.
+        out += "(.*?)"
+      }
     } else {
       out += "::"
     }
   }
   return out
 }
+
+/**
+ * Replaces declared `:name` params, including any regex constraint and `?`.
+ * Returning `undefined` also removes a preceding `/`.
+ *
+ * @internal
+ */
+export function replaceParams(
+  path: string,
+  paramNames: ReadonlySet<string> | undefined,
+  replace: (name: string, optional: boolean) => string | undefined
+): string {
+  const params = /::|:(\w+)/g
+  let out = ""
+  let last = 0
+  for (let param = params.exec(path); param !== null; param = params.exec(path)) {
+    const name = param[1]
+    if (name === undefined || (paramNames !== undefined && !paramNames.has(name))) continue
+    let end = params.lastIndex
+    if (path[end] === "(") end = getClosingParenthensePosition(path, end) + 1
+    const optional = path[end] === "?"
+    if (optional) end++
+    const value = replace(name, optional)
+    let start = param.index
+    if (value === undefined && path[start - 1] === "/") start--
+    out += path.slice(last, start) + (value ?? "")
+    params.lastIndex = end
+    last = end
+  }
+  return out + path.slice(last)
+}

package/src/http-api/internal/toCodecHttpProperties.ts

@@ -0,0 +1,56 @@
+import { memoize } from "../../Function.ts"
+import * as Option from "../../Option.ts"
+import * as Predicate from "../../Predicate.ts"
+import * as Schema from "../../Schema.ts"
+import * as SchemaAST from "../../SchemaAST.ts"
+import * as SchemaGetter from "../../SchemaGetter.ts"
+import * as SchemaTransformation from "../../SchemaTransformation.ts"
+
+/** @internal */
+export function toCodecHttpProperties<S extends Schema.Constraint>(schema: S): Schema.toCodecStringTree<S> {
+  const codec = Schema.toCodecStringTree(schema)
+  return codec.rebuild(normalizeProperties(codec.ast))
+}
+
+const omitUndefined = new SchemaTransformation.Transformation(
+  SchemaGetter.passthrough(),
+  SchemaGetter.transformOptional(Option.filter(Predicate.isNotUndefined))
+)
+
+// Only descend through unions and suspensions, leaving nested values alone.
+const present = memoize((ast: SchemaAST.AST): SchemaAST.AST => {
+  if (ast._tag === "Undefined") return new SchemaAST.Never(ast.annotations, undefined, undefined, ast.context)
+  if (ast._tag === "Suspend") return ast.recur(present)
+  if (ast._tag !== "Union") return ast
+
+  const members = SchemaAST.mapOrSame(ast.types, present)
+  if (members === ast.types) return ast
+  const types = members.filter((member) => member._tag !== "Never")
+  if (types.length === 1 && ast.checks === undefined && ast.annotations === undefined) {
+    return SchemaAST.replaceContext(types[0], ast.context)
+  }
+  return new SchemaAST.Union(types, ast.options, ast.annotations, ast.checks, undefined, ast.context)
+})
+
+const normalizeProperties = SchemaAST.applyToSelfOrLastLinkEncodingIdempotent((ast) => {
+  if (ast._tag === "Union" || ast._tag === "Suspend") return ast.recur(normalizeProperties)
+  if (ast._tag !== "Objects") return ast
+
+  const properties = SchemaAST.mapOrSame(ast.propertySignatures, (ps) => {
+    const encoded = SchemaAST.toEncoded(ps.type)
+    if (!SchemaAST.isOptional(encoded)) return ps
+    const wire = present(encoded)
+    return wire === encoded
+      ? ps
+      : new SchemaAST.PropertySignature(ps.name, SchemaAST.decodeTo(wire, ps.type, omitUndefined))
+  })
+  return properties === ast.propertySignatures ? ast : new SchemaAST.Objects(
+    properties,
+    ast.indexSignatures,
+    ast.annotations,
+    ast.checks,
+    undefined,
+    ast.context,
+    ast.encodingChecks
+  )
+})

package/src/index.ts

@@ -80,6 +80,12 @@ export * as Boolean from "./Boolean.ts"
  */
 export * as Brand from "./Brand.ts"
 
+/**
+ * @stability unstable
+ * @since 4.0.6
+ */
+export * as Bytes from "./Bytes.ts"
+
 /**
  * @stability unstable
  * @since 4.0.0
@@ -698,6 +704,18 @@ export * as ScopedRef from "./ScopedRef.ts"
  */
 export * as Semaphore from "./Semaphore.ts"
 
+/**
+ * @stability unstable
+ * @since 4.0.6
+ */
+export * as SemVer from "./SemVer.ts"
+
+/**
+ * @stability unstable
+ * @since 4.0.6
+ */
+export * as SemVerRange from "./SemVerRange.ts"
+
 /**
  * @stability stable
  * @since 2.0.0
@@ -764,6 +782,12 @@ export * as Take from "./Take.ts"
  */
 export * as Terminal from "./Terminal.ts"
 
+/**
+ * @stability experimental
+ * @since 4.0.6
+ */
+export * as TextDiff from "./TextDiff.ts"
+
 /**
  * @stability stable
  * @since 2.0.0