effect 4.0.0-beta.103 → 4.0.0-beta.104

package/src/unstable/http/MultipartParser/internal/multipart.ts

@@ -0,0 +1,262 @@
+import type { Config, MultipartError, PartInfo } from "../../MultipartParser.ts"
+import * as CT from "./contentType.ts"
+import * as HP from "./headers.ts"
+import * as Search from "./search.ts"
+
+const State = {
+  headers: 0,
+  body: 1
+} as const
+type State = (typeof State)[keyof typeof State]
+
+const errInvalidDisposition: MultipartError = { _tag: "InvalidDisposition" }
+const errEndNotReached: MultipartError = { _tag: "EndNotReached" }
+const errMaxParts: MultipartError = { _tag: "ReachedLimit", limit: "MaxParts" }
+const errMaxTotalSize: MultipartError = {
+  _tag: "ReachedLimit",
+  limit: "MaxTotalSize"
+}
+const errMaxPartSize: MultipartError = {
+  _tag: "ReachedLimit",
+  limit: "MaxPartSize"
+}
+const errMaxFieldSize: MultipartError = {
+  _tag: "ReachedLimit",
+  limit: "MaxFieldSize"
+}
+
+const constCR = new TextEncoder().encode("\r\n")
+
+export function defaultIsFile(info: PartInfo) {
+  return (
+    info.filename !== undefined ||
+    info.contentType === "application/octet-stream"
+  )
+}
+
+function parseBoundary(headers: Record<string, string>) {
+  const contentType = CT.parse(headers["content-type"])
+  return contentType.parameters.boundary
+}
+
+function noopOnChunk(_chunk: Uint8Array | null) {}
+
+export function make({
+  headers,
+  onFile: onPart,
+  onField,
+  onError,
+  onDone,
+  isFile = defaultIsFile,
+  maxParts = Infinity,
+  maxTotalSize = Infinity,
+  maxPartSize = Infinity,
+  maxFieldSize = 1024 * 1024
+}: Config) {
+  const boundary = parseBoundary(headers)
+  if (boundary === undefined) {
+    onError({ _tag: "InvalidBoundary" })
+    return {
+      write: noopOnChunk,
+      end() {}
+    }
+  }
+
+  const state = {
+    state: State.headers as State,
+    index: 0,
+    parts: 0,
+    onChunk: noopOnChunk,
+    info: undefined as any as PartInfo,
+    headerSkip: 0,
+    partSize: 0,
+    totalSize: 0,
+    isFile: false,
+    fieldChunks: [] as Array<Uint8Array>,
+    fieldSize: 0,
+    done: false
+  }
+
+  function skipBody() {
+    state.state = State.body
+    state.isFile = true
+    state.onChunk = noopOnChunk
+  }
+
+  const headerParser = HP.make()
+
+  const split = Search.make(
+    `\r\n--${boundary}`,
+    function(index, chunk) {
+      if (index === 0) {
+        // data before the first boundary
+        skipBody()
+        return
+      } else if (index !== state.index) {
+        if (state.index > 0) {
+          if (state.isFile) {
+            state.onChunk(null)
+          } else {
+            if (state.fieldChunks.length === 1) {
+              onField(state.info, state.fieldChunks[0])
+            } else {
+              const buf = new Uint8Array(state.fieldSize)
+              let offset = 0
+              for (let i = 0; i < state.fieldChunks.length; i++) {
+                const chunk = state.fieldChunks[i]
+                buf.set(chunk, offset)
+                offset += chunk.length
+              }
+              onField(state.info, buf)
+            }
+            state.fieldSize = 0
+            state.fieldChunks = []
+          }
+        }
+        state.partSize = 0
+
+        state.state = State.headers
+        state.index = index
+        state.headerSkip = 2 // skip the first \r\n
+
+        // trailing --
+        if (chunk[0] === 45 && chunk[1] === 45) {
+          state.done = true
+          return onDone()
+        }
+
+        state.parts++
+        if (state.parts > maxParts) {
+          onError(errMaxParts)
+        }
+      }
+
+      if ((state.partSize += chunk.length) > maxPartSize) {
+        onError(errMaxPartSize)
+      }
+
+      if (state.state === State.headers) {
+        const result = headerParser(chunk, state.headerSkip)
+        state.headerSkip = 0
+
+        if (result._tag === "Continue") {
+          return
+        } else if (result._tag === "Failure") {
+          skipBody()
+          return onError({ _tag: "BadHeaders", error: result })
+        }
+
+        const contentType = CT.parse(result.headers["content-type"] as string)
+        const contentDisposition = CT.parse(
+          result.headers["content-disposition"] as string,
+          true
+        )
+
+        if (
+          "form-data" === contentDisposition.value &&
+          !("name" in contentDisposition.parameters)
+        ) {
+          skipBody()
+          return onError(errInvalidDisposition)
+        }
+
+        let encodedFilename: string | undefined
+        if ("filename*" in contentDisposition.parameters) {
+          const parts = contentDisposition.parameters["filename*"].split("''")
+          if (parts.length === 2) {
+            try {
+              encodedFilename = decodeURIComponent(parts[1])
+            } catch {
+              encodedFilename = parts[1]
+            }
+          }
+        }
+
+        state.info = {
+          name: contentDisposition.parameters.name ?? "",
+          filename: encodedFilename ?? contentDisposition.parameters.filename,
+          contentType: contentType.value === ""
+            ? contentDisposition.parameters.filename !== undefined
+              ? "application/octet-stream"
+              : "text/plain"
+            : contentType.value,
+          contentTypeParameters: contentType.parameters,
+          contentDisposition: contentDisposition.value,
+          contentDispositionParameters: contentDisposition.parameters as any,
+          headers: result.headers
+        }
+
+        state.state = State.body
+        state.isFile = isFile(state.info)
+
+        if (state.isFile) {
+          state.onChunk = onPart(state.info)
+        }
+
+        if (result.endPosition < chunk.length) {
+          if (state.isFile) {
+            state.onChunk(chunk.subarray(result.endPosition))
+          } else {
+            const buf = chunk.subarray(result.endPosition)
+            if ((state.fieldSize += buf.length) > maxFieldSize) {
+              onError(errMaxFieldSize)
+            }
+            state.fieldChunks.push(buf)
+          }
+        }
+      } else if (state.isFile) {
+        state.onChunk(chunk)
+      } else {
+        if ((state.fieldSize += chunk.length) > maxFieldSize) {
+          onError(errMaxFieldSize)
+        }
+        state.fieldChunks.push(chunk)
+      }
+    },
+    constCR,
+    2
+  )
+
+  return {
+    write(chunk: Uint8Array) {
+      if ((state.totalSize += chunk.length) > maxTotalSize) {
+        return onError(errMaxTotalSize)
+      }
+      return split.write(chunk)
+    },
+    end() {
+      split.end()
+      if (!state.done) {
+        onError(errEndNotReached)
+      }
+
+      state.state = State.headers
+      state.index = 0
+      state.parts = 0
+      state.onChunk = noopOnChunk
+      state.info = undefined as any as PartInfo
+      state.totalSize = 0
+      state.partSize = 0
+      state.fieldChunks = []
+      state.fieldSize = 0
+      state.done = false
+    }
+  } as const
+}
+
+const utf8Decoder = new TextDecoder("utf-8")
+function getDecoder(charset: string) {
+  if (charset === "utf-8" || charset === "utf8" || charset === "") {
+    return utf8Decoder
+  }
+
+  try {
+    return new TextDecoder(charset)
+  } catch (error) {
+    return utf8Decoder
+  }
+}
+
+export function decodeField(info: PartInfo, value: Uint8Array): string {
+  return getDecoder(info.contentTypeParameters.charset ?? "utf-8").decode(value)
+}

package/src/unstable/http/MultipartParser/internal/search.ts

@@ -0,0 +1,167 @@
+interface SearchState {
+  readonly needle: Uint8Array
+  readonly needleLength: number
+  readonly indexes: Record<number, ReadonlyArray<number>>
+  readonly firstByte: number
+
+  previousChunk: Uint8Array | undefined
+  previousChunkLength: number
+  matchIndex: number
+}
+
+function makeState(needle_: string): SearchState {
+  const needle = new TextEncoder().encode(needle_)
+  const needleLength = needle.length
+
+  const indexes: Record<number, Array<number>> = {}
+  for (let i = 0; i < needleLength; i++) {
+    const b = needle[i]
+    if (indexes[b] === undefined) indexes[b] = []
+    indexes[b].push(i)
+  }
+
+  return {
+    needle,
+    needleLength,
+    indexes,
+    firstByte: needle[0],
+    previousChunk: undefined,
+    previousChunkLength: 0,
+    matchIndex: 0
+  }
+}
+
+export function make(
+  needle: string,
+  callback: (index: number, chunk: Uint8Array) => void,
+  seed?: Uint8Array,
+  minimumChunkLength?: number
+) {
+  const state = makeState(needle)
+  const minChunkLength = minimumChunkLength ?? state.needleLength
+  if (seed !== undefined) {
+    state.previousChunk = seed
+    state.previousChunkLength = seed.length
+  }
+
+  function makeIndexOf(): (
+    chunk: Uint8Array,
+    needle: Uint8Array,
+    fromIndex: number
+  ) => number {
+    // on node.js use the Buffer api
+    if (
+      "Buffer" in globalThis &&
+      !("Bun" in globalThis || "Deno" in globalThis)
+    ) {
+      return function(chunk, needle, fromIndex) {
+        return Buffer.prototype.indexOf.call(chunk, needle, fromIndex)
+      }
+    }
+
+    const skipTable = new Uint8Array(256).fill(state.needle.length)
+    for (let i = 0, lastIndex = state.needle.length - 1; i < lastIndex; ++i) {
+      skipTable[state.needle[i]] = lastIndex - i
+    }
+
+    return function(chunk, needle, fromIndex) {
+      const lengthTotal = chunk.length
+      let i = fromIndex + state.needleLength - 1
+
+      while (i < lengthTotal) {
+        for (
+          let j = state.needleLength - 1, k = i;
+          j >= 0 && chunk[k] === needle[j];
+          j--, k--
+        ) {
+          if (j === 0) return k
+        }
+        i += skipTable[chunk[i]]
+      }
+
+      return -1
+    }
+  }
+
+  const indexOf = makeIndexOf()
+
+  function write(chunk: Uint8Array): void {
+    let chunkLength = chunk.length
+
+    if (state.previousChunk !== undefined) {
+      const newChunk = new Uint8Array(state.previousChunkLength + chunkLength)
+      newChunk.set(state.previousChunk)
+      newChunk.set(chunk, state.previousChunkLength)
+      chunk = newChunk
+      chunkLength = state.previousChunkLength + chunkLength
+      state.previousChunk = undefined
+    }
+
+    let pos = 0
+    while (pos < chunkLength) {
+      const remaining = chunkLength - pos
+      if (remaining < minChunkLength) {
+        state.previousChunk = chunk.subarray(pos)
+        state.previousChunkLength = remaining
+        return
+      }
+
+      const match = indexOf(chunk, state.needle, pos)
+
+      if (match > -1) {
+        if (match > pos) {
+          callback(state.matchIndex, chunk.subarray(pos, match))
+        }
+        state.matchIndex += 1
+        pos = match + state.needleLength
+        continue
+      } else if (chunk[chunkLength - 1] in state.indexes) {
+        const indexes = state.indexes[chunk[chunkLength - 1]]
+        let earliestIndex = -1
+        for (let i = 0, len = indexes.length; i < len; i++) {
+          const index = indexes[i]
+          if (
+            chunk[chunkLength - 1 - index] === state.firstByte &&
+            i > earliestIndex
+          ) {
+            earliestIndex = index
+          }
+        }
+        if (earliestIndex === -1) {
+          if (pos === 0) {
+            callback(state.matchIndex, chunk)
+          } else {
+            callback(state.matchIndex, chunk.subarray(pos))
+          }
+        } else {
+          if (chunkLength - 1 - earliestIndex > pos) {
+            callback(
+              state.matchIndex,
+              chunk.subarray(pos, chunkLength - 1 - earliestIndex)
+            )
+          }
+          state.previousChunk = chunk.subarray(chunkLength - 1 - earliestIndex)
+          state.previousChunkLength = earliestIndex + 1
+        }
+      } else if (pos === 0) {
+        callback(state.matchIndex, chunk)
+      } else {
+        callback(state.matchIndex, chunk.subarray(pos))
+      }
+
+      break
+    }
+  }
+
+  function end(): void {
+    if (state.previousChunk !== undefined && state.previousChunk !== seed) {
+      callback(state.matchIndex, state.previousChunk)
+    }
+
+    state.previousChunk = seed
+    state.previousChunkLength = seed?.length ?? 0
+    state.matchIndex = 0
+  }
+
+  return { write, end } as const
+}

package/src/unstable/http/MultipartParser.ts

@@ -0,0 +1,115 @@
+/**
+ * Low-level parser for HTTP `multipart/form-data` bodies.
+ *
+ * @since 4.0.0
+ */
+import type * as HeadersParser from "./MultipartParser/HeadersParser.ts"
+import * as internal from "./MultipartParser/internal/multipart.ts"
+
+/**
+ * Metadata describing a multipart form part.
+ *
+ * @category models
+ * @since 4.0.0
+ */
+export interface PartInfo {
+  readonly name: string
+  readonly filename?: string | undefined
+  readonly contentType: string
+  readonly contentTypeParameters: Record<string, string>
+  readonly contentDisposition: string
+  readonly contentDispositionParameters: Record<string, string>
+  readonly headers: Record<string, string | Array<string>>
+}
+
+/**
+ * An error produced while parsing a multipart body.
+ *
+ * @category errors
+ * @since 4.0.0
+ */
+export type MultipartError =
+  | {
+    readonly _tag: "InvalidBoundary"
+  }
+  | {
+    readonly _tag: "BadHeaders"
+    readonly error: HeadersParser.Failure
+  }
+  | {
+    readonly _tag: "InvalidDisposition"
+  }
+  | {
+    readonly _tag: "ReachedLimit"
+    readonly limit:
+      | "MaxParts"
+      | "MaxTotalSize"
+      | "MaxPartSize"
+      | "MaxFieldSize"
+  }
+  | {
+    readonly _tag: "EndNotReached"
+  }
+
+/**
+ * Shared multipart parser configuration.
+ *
+ * @category models
+ * @since 4.0.0
+ */
+export type BaseConfig = {
+  readonly headers: Record<string, string>
+  readonly isFile?: ((info: PartInfo) => boolean) | undefined
+  readonly maxParts?: number | undefined
+  readonly maxTotalSize?: number | undefined
+  readonly maxPartSize?: number | undefined
+  readonly maxFieldSize?: number | undefined
+}
+
+/**
+ * Multipart parser configuration with event callbacks.
+ *
+ * @category models
+ * @since 4.0.0
+ */
+export type Config = BaseConfig & {
+  readonly onField: (info: PartInfo, value: Uint8Array) => void
+  readonly onFile: (info: PartInfo) => (chunk: Uint8Array | null) => void
+  readonly onError: (error: MultipartError) => void
+  readonly onDone: () => void
+}
+
+/**
+ * A streaming multipart parser.
+ *
+ * @category models
+ * @since 4.0.0
+ */
+export interface Parser {
+  readonly write: (chunk: Uint8Array) => void
+  readonly end: () => void
+}
+
+/**
+ * Creates a streaming multipart parser.
+ *
+ * @category constructors
+ * @since 4.0.0
+ */
+export const make: (options: Config) => Parser = internal.make
+
+/**
+ * Determines whether a multipart part should be treated as a file.
+ *
+ * @category utilities
+ * @since 4.0.0
+ */
+export const defaultIsFile: (info: PartInfo) => boolean = internal.defaultIsFile
+
+/**
+ * Decodes a multipart field using its declared character set.
+ *
+ * @category utilities
+ * @since 4.0.0
+ */
+export const decodeField: (info: PartInfo, value: Uint8Array) => string = internal.decodeField

package/src/unstable/http/Template.ts

@@ -215,11 +215,12 @@ export function stream<A extends ReadonlyArray<InterpolatedWithStream>>(
     buffer = ""
   }
 
-  return Stream.flatMap(
-    Stream.fromIterable(chunks),
-    (chunk) =>
-      typeof chunk === "string" ? Stream.succeed(chunk) : Effect.isEffect(chunk) ? Stream.fromEffect(chunk) : chunk,
-    { concurrency: "unbounded" }
+  return Stream.fromIterable(chunks).pipe(
+    Stream.mapEffect(
+      (chunk) => Effect.isEffect(chunk) ? chunk : Effect.succeed(chunk),
+      { concurrency: "unbounded" }
+    ),
+    Stream.flatMap((chunk) => typeof chunk === "string" ? Stream.succeed(chunk) : chunk)
   )
 }
 

package/src/unstable/http/index.ts

@@ -127,7 +127,7 @@ export * as Multipart from "./Multipart.ts"
 /**
  * @since 4.0.0
  */
-export * as Multipasta from "./Multipasta.ts"
+export * as MultipartParser from "./MultipartParser.ts"
 
 /**
  * @since 4.0.0

package/src/unstable/http/internal/httpBody.ts

@@ -0,0 +1,15 @@
+import * as Headers from "../Headers.ts"
+import type * as HttpBody from "../HttpBody.ts"
+
+/** @internal */
+export const updateHeaders = (headers: Headers.Headers, body: HttpBody.HttpBody): Headers.Headers => {
+  if (body._tag === "Empty" || body._tag === "FormData") {
+    return Headers.remove(Headers.remove(headers, "content-type"), "content-length")
+  }
+  headers = body.contentType === undefined
+    ? Headers.remove(headers, "content-type")
+    : Headers.set(headers, "content-type", body.contentType)
+  return body.contentLength === undefined
+    ? Headers.remove(headers, "content-length")
+    : Headers.set(headers, "content-length", body.contentLength.toString())
+}

package/src/unstable/httpapi/HttpApi.ts

@@ -15,7 +15,6 @@ import { type Pipeable, pipeArguments } from "../../Pipeable.ts"
 import * as Predicate from "../../Predicate.ts"
 import * as Record from "../../Record.ts"
 import type * as Schema from "../../Schema.ts"
-import type * as SchemaAST from "../../SchemaAST.ts"
 import type { PathInput } from "../http/HttpRouter.ts"
 import * as HttpApiEndpoint from "./HttpApiEndpoint.ts"
 import type * as HttpApiGroup from "./HttpApiGroup.ts"
@@ -290,11 +289,11 @@ export const reflect = <Id extends string, Groups extends HttpApiGroup.Constrain
         mergedAnnotations: Context.merge(groupAnnotations, endpoint.annotations),
         successes: extractResponseContent(
           HttpApiEndpoint.getSuccessSchemas(endpoint),
-          HttpApiSchema.getStatusSuccess
+          HttpApiSchema.getStatusSuccessSchema
         ),
         errors: extractResponseContent(
           HttpApiEndpoint.getErrorSchemas(endpoint),
-          HttpApiSchema.getStatusError
+          HttpApiSchema.getStatusErrorSchema
         )
       })
     }
@@ -305,7 +304,7 @@ export const reflect = <Id extends string, Groups extends HttpApiGroup.Constrain
 
 const extractResponseContent = (
   schemas: Array<Schema.Top>,
-  getStatus: (ast: SchemaAST.AST) => number
+  getStatus: (schema: Schema.Constraint) => number
 ): ReadonlyMap<number, [Schema.Top, ...Array<Schema.Top>]> => {
   const map = new Map<number, [Schema.Top, ...Array<Schema.Top>]>()
 
@@ -314,9 +313,9 @@ const extractResponseContent = (
   return map
 
   function add(schema: Schema.Top) {
-    if (HttpApiSchema.isStreamSchema(schema)) return
-    const ast = schema.ast
-    const status = getStatus(ast)
+    const body = HttpApiSchema.isWithHeaders(schema) ? schema.schema : schema
+    if (HttpApiSchema.isStreamSchema(body)) return
+    const status = getStatus(schema)
     const schemas = map.get(status)
     if (schemas === undefined) {
       map.set(status, [schema])