@panorama-industries/effect-vercel-queue 0.0.0-stage → 0.1.0

package/src/internal/cloud-event.ts

@@ -0,0 +1,244 @@
+import type { Headers } from "effect/http";
+
+import { VercelRegion } from "@panorama-industries/effect-vercel-core";
+import { DateTime, Duration, Effect, Option, Predicate, Schema } from "effect";
+
+import type { LeaseDeadline } from "./lease.ts";
+
+import {
+  ConsumerGroupName,
+  MessageHandle,
+  MessageId,
+  MessageMetadata,
+  ReceiptHandle,
+  ReceivedMessage,
+  TopicName,
+} from "../contracts/queue-domain.ts";
+
+export const CLOUD_EVENT_TYPE_V1BETA = "com.vercel.queue.v1beta";
+export const CLOUD_EVENT_TYPE_V2BETA = "com.vercel.queue.v2beta";
+
+const REGION_HEADER_PATTERN = /^[a-z]{2,5}[0-9]{1,2}$/;
+const DEFAULT_RETENTION = Duration.hours(24);
+
+/** A push delivery the queue could not have sent: answered with HTTP 400. */
+export class MalformedCloudEvent extends Schema.TaggedError<MalformedCloudEvent>()(
+  "MalformedCloudEvent",
+  { message: Schema.String },
+) {}
+
+export interface CallbackRoute {
+  readonly topic: TopicName;
+  readonly consumerGroup: ConsumerGroupName;
+  readonly messageId: MessageId;
+  readonly region: Option.Option<VercelRegion>;
+}
+
+/** Binary-mode delivery whose lease and payload arrived with the request. */
+export interface InlineLease {
+  readonly receiptHandle: ReceiptHandle;
+  readonly deliveryCount: Option.Option<string>;
+  readonly createdAt: Option.Option<string>;
+  readonly expiresAt: Option.Option<string>;
+  readonly contentType: Option.Option<string>;
+  readonly visibilityDeadline: Option.Option<string>;
+  readonly handoffCount: Option.Option<number>;
+}
+
+export interface ParsedCallback extends CallbackRoute {
+  readonly inline: Option.Option<InlineLease>;
+}
+
+const malformed = (message: string) => Effect.fail(new MalformedCloudEvent({ message }));
+
+const header = (headers: Headers.Headers, name: string): Option.Option<string> =>
+  Option.fromNullishOr(headers[name]);
+
+const nonEmptyHeader = (headers: Headers.Headers, name: string): Option.Option<string> =>
+  Option.filter(header(headers, name), (value) => value.length > 0);
+
+/** Reads `Vqs-Experimental-Handoff-Count`-style values: only positive integers count. */
+export const parseHandoffCount = (value: Option.Option<string>): Option.Option<number> =>
+  Option.flatMap(value, (raw) => {
+    if (raw.length === 0) return Option.none();
+    const count = Number(raw);
+    return Number.isInteger(count) && count > 0 ? Option.some(count) : Option.none();
+  });
+
+const brand = <S extends Schema.Top & Schema.Decoder<unknown>>(
+  schema: S,
+  field: string,
+  value: string,
+) =>
+  Schema.decodeUnknownEffect(schema)(value).pipe(
+    Effect.catch(() =>
+      malformed(`Invalid CloudEvent: ${field} ${JSON.stringify(value)} is not valid`),
+    ),
+  );
+
+const route = Effect.fnUntraced(function* (
+  topic: string,
+  consumerGroup: string,
+  messageId: string,
+  region: Option.Option<VercelRegion>,
+) {
+  return {
+    topic: yield* brand(TopicName, "queueName", topic),
+    consumerGroup: yield* brand(ConsumerGroupName, "consumerGroup", consumerGroup),
+    messageId: yield* brand(MessageId, "messageId", messageId),
+    region,
+  } satisfies CallbackRoute;
+});
+
+const parseStructured = Effect.fnUntraced(function* (
+  body: unknown,
+  contentType: Option.Option<string>,
+) {
+  if (Option.isNone(contentType) || !contentType.value.includes("application/cloudevents+json")) {
+    return yield* malformed("Invalid content type: expected 'application/cloudevents+json'");
+  }
+  const event = Predicate.isObjectOrArray(body) ? (body as Record<string, unknown>) : undefined;
+  const data = Predicate.isObjectOrArray(event?.["data"])
+    ? (event["data"] as Record<string, unknown>)
+    : undefined;
+  if (event === undefined || !event["type"] || !event["source"] || !event["id"] || !data) {
+    return yield* malformed("Invalid CloudEvent: missing required fields");
+  }
+  if (event["type"] !== CLOUD_EVENT_TYPE_V1BETA) {
+    return yield* malformed(
+      // oxlint-disable-next-line typescript/no-base-to-string -- matches @vercel/queue's message
+      `Invalid CloudEvent type: expected '${CLOUD_EVENT_TYPE_V1BETA}', got '${String(event["type"])}'`,
+    );
+  }
+  const missing = ["queueName", "consumerGroup", "messageId"].filter((field) => !(field in data));
+  if (missing.length > 0) {
+    return yield* malformed(`Missing required CloudEvent data fields: ${missing.join(", ")}`);
+  }
+  const parsed = yield* route(
+    String(data["queueName"]),
+    String(data["consumerGroup"]),
+    String(data["messageId"]),
+    Option.none(),
+  );
+  return { ...parsed, inline: Option.none() } satisfies ParsedCallback;
+});
+
+const parseBinary = Effect.fnUntraced(function* (headers: Headers.Headers) {
+  const queueName = nonEmptyHeader(headers, "ce-vqsqueuename");
+  const consumerGroup = nonEmptyHeader(headers, "ce-vqsconsumergroup");
+  const messageId = nonEmptyHeader(headers, "ce-vqsmessageid");
+  if (Option.isNone(queueName) || Option.isNone(consumerGroup) || Option.isNone(messageId)) {
+    const missing = [
+      Option.isNone(queueName) ? ["ce-vqsqueuename"] : [],
+      Option.isNone(consumerGroup) ? ["ce-vqsconsumergroup"] : [],
+      Option.isNone(messageId) ? ["ce-vqsmessageid"] : [],
+    ].flat();
+    return yield* malformed(`Missing required CloudEvent headers: ${missing.join(", ")}`);
+  }
+
+  const rawRegion = header(headers, "ce-vqsregion");
+  let region = Option.none<VercelRegion>();
+  if (Option.isSome(rawRegion)) {
+    const decoded = REGION_HEADER_PATTERN.test(rawRegion.value)
+      ? Schema.decodeUnknownOption(VercelRegion)(rawRegion.value)
+      : Option.none();
+    if (Option.isNone(decoded)) {
+      return yield* malformed(
+        `Invalid ce-vqsregion header: ${JSON.stringify(rawRegion.value)}. ` +
+          `Region must match /^[a-z]{2,5}[0-9]{1,2}$/ (e.g. "iad1", "lhr1").`,
+      );
+    }
+    region = decoded;
+  }
+
+  const base = yield* route(queueName.value, consumerGroup.value, messageId.value, region);
+  const receiptHandle = nonEmptyHeader(headers, "ce-vqsreceipthandle");
+  if (Option.isNone(receiptHandle)) {
+    return { ...base, inline: Option.none() } satisfies ParsedCallback;
+  }
+
+  const inline: InlineLease = {
+    receiptHandle: yield* brand(ReceiptHandle, "receiptHandle", receiptHandle.value),
+    deliveryCount: nonEmptyHeader(headers, "ce-vqsdeliverycount"),
+    createdAt: nonEmptyHeader(headers, "ce-vqscreatedat"),
+    expiresAt: nonEmptyHeader(headers, "ce-vqsexpiresat"),
+    contentType: nonEmptyHeader(headers, "content-type"),
+    visibilityDeadline: nonEmptyHeader(headers, "ce-vqsvisibilitydeadline"),
+    handoffCount: parseHandoffCount(header(headers, "ce-vqsexperimentalhandoffcount")),
+  };
+  return { ...base, inline: Option.some(inline) } satisfies ParsedCallback;
+});
+
+const UnknownFromJson = Schema.fromJsonString(Schema.Unknown);
+
+/**
+ * Parses a push delivery like `@vercel/queue`'s `parseCallback`: binary mode
+ * (`com.vercel.queue.v2beta`, metadata in `ce-*` headers) or structured mode (JSON CloudEvent).
+ */
+export const parseCallback = (
+  headers: Headers.Headers,
+  text: Effect.Effect<string, unknown>,
+): Effect.Effect<ParsedCallback, MalformedCloudEvent> => {
+  if (headers["ce-type"] === CLOUD_EVENT_TYPE_V2BETA) return parseBinary(headers);
+  return text.pipe(
+    Effect.flatMap(Schema.decodeUnknownEffect(UnknownFromJson)),
+    Effect.catch(() => malformed("Failed to parse CloudEvent from request body")),
+    Effect.flatMap((body) => parseStructured(body, header(headers, "content-type"))),
+  );
+};
+
+const parseDate = (value: Option.Option<string>): Option.Option<DateTime.Utc> =>
+  Option.flatMap(value, (raw) => DateTime.make(raw)).pipe(Option.map(DateTime.toUtc));
+
+/** Builds the leased message a binary-mode delivery carries inline. */
+export const inlineDelivery = Effect.fnUntraced(function* (
+  parsed: CallbackRoute,
+  inline: InlineLease,
+  region: VercelRegion,
+  defaultContentType: string,
+  body: Uint8Array,
+) {
+  const now = yield* DateTime.now;
+  const createdAt = Option.getOrElse(parseDate(inline.createdAt), () => now);
+  const deliveryCount = Option.getOrElse(
+    Option.filter(
+      Option.map(inline.deliveryCount, (raw) => Number.parseInt(raw, 10)),
+      (count) => !Number.isNaN(count),
+    ),
+    () => 1,
+  );
+  const handle = new MessageHandle({
+    topic: parsed.topic,
+    consumerGroup: parsed.consumerGroup,
+    messageId: parsed.messageId,
+    receiptHandle: inline.receiptHandle,
+    region,
+  });
+  const metadata = new MessageMetadata({
+    messageId: parsed.messageId,
+    deliveryCount,
+    createdAt,
+    expiresAt: Option.getOrElse(parseDate(inline.expiresAt), () =>
+      DateTime.addDuration(createdAt, DEFAULT_RETENTION),
+    ),
+    topic: parsed.topic,
+    consumerGroup: parsed.consumerGroup,
+    region,
+    ...Option.match(inline.handoffCount, {
+      onNone: () => ({}),
+      onSome: (experimentalHandoffCount) => ({ experimentalHandoffCount }),
+    }),
+  });
+  const deadline: LeaseDeadline = Option.map(inline.visibilityDeadline, (raw) =>
+    Option.getOrElse(parseDate(Option.some(raw)), () => "expired" as const),
+  );
+  return {
+    message: new ReceivedMessage({
+      handle,
+      metadata,
+      contentType: Option.getOrElse(inline.contentType, () => defaultContentType),
+      body,
+    }),
+    deadline,
+  };
+});

package/src/internal/lease.ts

@@ -0,0 +1,84 @@
+import type { DateTime, Scope } from "effect";
+
+import { Clock, Duration, Effect, Option } from "effect";
+
+import type { QueueApi } from "../contracts/queue-api.ts";
+import type { MessageHandle } from "../contracts/queue-domain.ts";
+
+const DEFAULT_VISIBILITY_TIMEOUT = Duration.minutes(5);
+const MIN_VISIBILITY_TIMEOUT = Duration.seconds(30);
+const MIN_RENEWAL_INTERVAL = Duration.seconds(10);
+const MAX_RENEWAL_INTERVAL = Duration.seconds(60);
+const RENEWAL_RETRY_INTERVAL = Duration.seconds(3);
+
+export const leaseVisibilityTimeout = (input: Duration.Input | undefined): Duration.Duration =>
+  Duration.max(
+    MIN_VISIBILITY_TIMEOUT,
+    input === undefined ? DEFAULT_VISIBILITY_TIMEOUT : Duration.fromInputUnsafe(input),
+  );
+
+export const renewalInterval = (timeout: Duration.Duration): Duration.Duration =>
+  Duration.clamp(Duration.millis(Duration.toMillis(timeout) / 5), {
+    minimum: MIN_RENEWAL_INTERVAL,
+    maximum: MAX_RENEWAL_INTERVAL,
+  });
+
+export type LeaseDeadline = Option.Option<DateTime.Utc | "expired">;
+
+const firstRenewalDelay = Effect.fnUntraced(function* (
+  timeout: Duration.Duration,
+  deadline: LeaseDeadline,
+) {
+  if (Option.isNone(deadline)) return renewalInterval(timeout);
+  if (deadline.value === "expired") return Duration.zero;
+  const now = yield* Clock.currentTimeMillis;
+  const remaining = deadline.value.epochMilliseconds - now;
+  return remaining > 0 ? renewalInterval(Duration.millis(remaining)) : Duration.zero;
+});
+
+/**
+ * Keeps a lease alive until the enclosing scope closes. Transient failures retry every 3 seconds;
+ * a lost or rejected lease (400, 401, 403, 404, 409) ends renewal.
+ */
+export const keepLeaseAlive = Effect.fnUntraced(
+  function* (
+    api: QueueApi["Service"],
+    handle: MessageHandle,
+    timeout: Duration.Duration,
+    deadline: LeaseDeadline,
+  ): Effect.fn.Return<void, never, Scope.Scope> {
+    const interval = renewalInterval(timeout);
+    const stop = (error: unknown) =>
+      Effect.logError("Lease renewal failed with a client error; stopping renewal", error).pipe(
+        Effect.as(Option.none<Duration.Duration>()),
+      );
+
+    const renew = api.changeVisibility(handle, timeout).pipe(
+      Effect.as(Option.some(interval)),
+      Effect.catchTags({ MessageNotFound: stop, MessageNotAvailable: stop }),
+      Effect.catchReasons("VercelApiError", {
+        ApiBadRequest: (_, error) => stop(error),
+        ApiUnauthorized: (_, error) => stop(error),
+        ApiForbidden: (_, error) => stop(error),
+      }),
+      Effect.catch((error) =>
+        Effect.logError("Lease renewal failed; retrying in 3 seconds", error).pipe(
+          Effect.as(Option.some(RENEWAL_RETRY_INTERVAL)),
+        ),
+      ),
+    );
+
+    const loop = Effect.fnUntraced(function* (initialDelay: Duration.Duration) {
+      let delay: Option.Option<Duration.Duration> = Option.some(initialDelay);
+      while (Option.isSome(delay)) {
+        yield* Effect.sleep(delay.value);
+        delay = yield* renew;
+      }
+    });
+
+    yield* Effect.forkScoped(loop(yield* firstRenewalDelay(timeout, deadline)), {
+      startImmediately: true,
+    });
+  },
+  Effect.annotateLogs({ module: "QueueLease" }),
+);

package/src/internal/multipart.ts

@@ -0,0 +1,240 @@
+import { Data, Effect, Option, Stream } from "effect";
+
+export interface MultipartPart {
+  /** Header names in lower case; repeated headers are joined with `, `. */
+  readonly headers: Readonly<Record<string, string>>;
+  readonly body: Uint8Array;
+}
+
+export interface OutgoingPart {
+  readonly headers: ReadonlyArray<readonly [name: string, value: string]>;
+  readonly body: Uint8Array;
+}
+
+export class MultipartMixedError extends Data.TaggedError("MultipartMixedError")<{
+  readonly message: string;
+}> {}
+
+const maxHeaderSize = 65_536;
+
+const encoder = new TextEncoder();
+const latin1 = new TextDecoder("latin1");
+
+export const boundaryFromContentType = (contentType: string): Option.Option<string> => {
+  const match = /boundary=(?:"([^"]+)"|([^;]+))/i.exec(contentType);
+  return Option.fromNullishOr(match?.[1] ?? match?.[2]);
+};
+
+const concatBytes = (chunks: ReadonlyArray<Uint8Array>): Uint8Array => {
+  const out = new Uint8Array(chunks.reduce((total, chunk) => total + chunk.length, 0));
+  let offset = 0;
+  for (const chunk of chunks) {
+    out.set(chunk, offset);
+    offset += chunk.length;
+  }
+  return out;
+};
+
+export const encodeMultipartMixed = (
+  boundary: string,
+  parts: ReadonlyArray<OutgoingPart>,
+): Uint8Array =>
+  concatBytes([
+    ...parts.flatMap((part) => [
+      encoder.encode(
+        `--${boundary}\r\n${part.headers.map(([name, value]) => `${name}: ${value}\r\n`).join("")}\r\n`,
+      ),
+      part.body,
+      encoder.encode("\r\n"),
+    ]),
+    encoder.encode(`--${boundary}--\r\n`),
+  ]);
+
+const indexOf = (haystack: Uint8Array, needle: Uint8Array, from: number): number => {
+  const last = haystack.length - needle.length;
+  const first = needle[0];
+  outer: for (let i = Math.max(0, from); i <= last; i++) {
+    if (haystack[i] !== first) continue;
+    for (let j = 1; j < needle.length; j++) {
+      if (haystack[i + j] !== needle[j]) continue outer;
+    }
+    return i;
+  }
+  return -1;
+};
+
+const startsWith = (haystack: Uint8Array, needle: Uint8Array): boolean =>
+  haystack.length >= needle.length && indexOf(haystack.subarray(0, needle.length), needle, 0) === 0;
+
+const parseHeaders = (bytes: Uint8Array): Record<string, string> => {
+  const headers: Record<string, string> = {};
+  for (const line of latin1.decode(bytes).trim().split(/\r?\n/)) {
+    const colon = line.indexOf(":");
+    if (colon <= 0) continue;
+    const name = line.slice(0, colon).trim().toLowerCase();
+    const value = line.slice(colon + 1).trim();
+    headers[name] = name in headers ? `${headers[name]}, ${value}` : value;
+  }
+  return headers;
+};
+
+const crlfcrlf = encoder.encode("\r\n\r\n");
+const lflf = encoder.encode("\n\n");
+
+type State = "start" | "afterBoundary" | "headers" | "body" | "done";
+
+const growableBuffer = () => {
+  let storage = new Uint8Array(0);
+  let start = 0;
+  let end = 0;
+  return {
+    view: () => storage.subarray(start, end),
+    append: (chunk: Uint8Array) => {
+      const length = end - start;
+      if (end + chunk.length > storage.length) {
+        const next = new Uint8Array(Math.max(storage.length * 2, length + chunk.length, 1024));
+        next.set(storage.subarray(start, end));
+        storage = next;
+        start = 0;
+        end = length;
+      }
+      storage.set(chunk, end);
+      end += chunk.length;
+    },
+    consume: (count: number) => {
+      start += count;
+      if (start === end) {
+        start = 0;
+        end = 0;
+      }
+    },
+    clear: () => {
+      start = 0;
+      end = 0;
+    },
+  };
+};
+
+const mixedParser = (boundary: string) => {
+  const opening = encoder.encode(`--${boundary}`);
+  const delimiter = encoder.encode(`\r\n--${boundary}`);
+  const buffer = growableBuffer();
+  let state: State = "start";
+  let searchFrom = 0;
+  let headers: Record<string, string> = {};
+
+  const failure = (message: string) => Effect.fail(new MultipartMixedError({ message }));
+
+  const step = (parts: Array<MultipartPart>): Option.Option<string> | "wait" => {
+    const bytes = buffer.view();
+    switch (state) {
+      case "start": {
+        if (bytes.length < opening.length) return "wait";
+        if (!startsWith(bytes, opening)) {
+          return Option.some("Invalid multipart stream: missing initial boundary");
+        }
+        buffer.consume(opening.length);
+        state = "afterBoundary";
+        return Option.none();
+      }
+      case "afterBoundary": {
+        if (bytes.length < 2) return "wait";
+        if (bytes[0] === 45 && bytes[1] === 45) {
+          state = "done";
+          buffer.clear();
+          return "wait";
+        }
+        if (bytes[0] === 13 && bytes[1] === 10) {
+          buffer.consume(2);
+        } else if (bytes[0] === 10) {
+          buffer.consume(1);
+        } else {
+          return Option.some("Invalid character after multipart boundary");
+        }
+        state = "headers";
+        return Option.none();
+      }
+      case "headers": {
+        let end = -1;
+        let skip = 0;
+        if (bytes[0] === 13 && bytes[1] === 10) {
+          end = 0;
+          skip = 2;
+        } else if (bytes[0] === 10) {
+          end = 0;
+          skip = 1;
+        } else {
+          const crlf = indexOf(bytes, crlfcrlf, 0);
+          const lf = indexOf(bytes, lflf, 0);
+          if (crlf !== -1 && (lf === -1 || crlf < lf)) {
+            end = crlf;
+            skip = 4;
+          } else {
+            end = lf;
+            skip = 2;
+          }
+        }
+        if (end === -1) {
+          return bytes.length > maxHeaderSize
+            ? Option.some("Multipart part headers are too large")
+            : "wait";
+        }
+        headers = parseHeaders(bytes.subarray(0, end));
+        buffer.consume(end + skip);
+        searchFrom = 0;
+        state = "body";
+        return Option.none();
+      }
+      case "body": {
+        const end = indexOf(bytes, delimiter, searchFrom);
+        if (end === -1) {
+          searchFrom = Math.max(0, bytes.length - delimiter.length + 1);
+          return "wait";
+        }
+        parts.push({ headers, body: bytes.slice(0, end) });
+        buffer.consume(end + delimiter.length);
+        state = "afterBoundary";
+        return Option.none();
+      }
+      case "done":
+        return "wait";
+    }
+  };
+
+  const push = (chunk: Uint8Array): Effect.Effect<Array<MultipartPart>, MultipartMixedError> => {
+    if (state === "done") return Effect.succeed([]);
+    buffer.append(chunk);
+    const parts: Array<MultipartPart> = [];
+    while (true) {
+      const result = step(parts);
+      if (result === "wait") return Effect.succeed(parts);
+      if (Option.isSome(result)) return failure(result.value);
+    }
+  };
+
+  const finish = Effect.suspend(() => {
+    if (state === "done") return Effect.void;
+    return failure(
+      state === "start"
+        ? "Invalid multipart stream: missing initial boundary"
+        : "Unexpected end of multipart stream",
+    );
+  });
+
+  return { push, finish };
+};
+
+/** Parses a `multipart/mixed` body as it streams in, emitting each part once it is complete. */
+export const decodeMultipartMixed =
+  (boundary: string) =>
+  <E, R>(
+    source: Stream.Stream<Uint8Array, E, R>,
+  ): Stream.Stream<MultipartPart, E | MultipartMixedError, R> =>
+    Stream.suspend(() => {
+      const parser = mixedParser(boundary);
+      return source.pipe(
+        Stream.mapEffect(parser.push),
+        Stream.flattenIterable,
+        Stream.concat(Stream.fromEffectDrain(parser.finish)),
+      );
+    });