@panorama-industries/effect-vercel-queue 0.0.0-stage → 0.2.0

package/src/internal/process.ts

@@ -0,0 +1,194 @@
+import type { Tracer } from "effect";
+
+import { Duration, Effect, Option, Schema } from "effect";
+import { Headers, HttpTraceContext } from "effect/http";
+
+import type {
+  ConsumeFailure,
+  ConsumeOptions,
+  MessageHandler,
+} from "../contracts/message-handling.ts";
+import type { QueueApi, QueueLeaseError } from "../contracts/queue-api.ts";
+import type { ReceivedMessage } from "../contracts/queue-domain.ts";
+import type { Topic } from "../contracts/topic.ts";
+import type { LeaseDeadline } from "./lease.ts";
+
+import { RetryDirective } from "../contracts/message-handling.ts";
+import { MessageMetadata, PayloadDecodeFailed } from "../contracts/queue-domain.ts";
+import { keepLeaseAlive, leaseVisibilityTimeout } from "./lease.ts";
+
+export interface Delivery {
+  readonly message: ReceivedMessage;
+  readonly deadline: LeaseDeadline;
+}
+
+/** Errors a processed delivery can end with once its error policy answered `Fail`. */
+export type ProcessError<E> = E | PayloadDecodeFailed | QueueLeaseError;
+
+interface Escalation<E> {
+  readonly forPolicy: Option.Option<E | ConsumeFailure>;
+  readonly error: ProcessError<E>;
+}
+
+const escalate = <E>(
+  forPolicy: Option.Option<E | ConsumeFailure>,
+  error: ProcessError<E>,
+): Escalation<E> => ({ forPolicy, error });
+
+const toPolicy = <E>(error: E | PayloadDecodeFailed): Escalation<E> =>
+  escalate<E>(Option.some(error), error);
+
+/**
+ * Leases, decodes, handles and settles one delivered message, following `@vercel/queue`'s
+ * `ConsumerGroup.processMessage`.
+ */
+export const processDelivery = <A, I, E, R, R2>(
+  api: QueueApi["Service"],
+  topic: Topic<A, I>,
+  handler: MessageHandler<A, E, R>,
+  options: ConsumeOptions<E, R2>,
+  delivery: Delivery,
+): Effect.Effect<void, ProcessError<E>, R | R2> => {
+  const { handle, metadata, body } = delivery.message;
+  const timeout = leaseVisibilityTimeout(options.visibilityTimeout);
+  const maxReruns = Math.max(0, Math.floor(options.experimentalMaxCoalescedReruns ?? 0));
+
+  const decode = Schema.decodeEffect(topic.wire)(body).pipe(
+    Effect.mapError(
+      (error) =>
+        new PayloadDecodeFailed({
+          topic: handle.topic,
+          messageId: handle.messageId,
+          message: error.message,
+        }),
+    ),
+  );
+
+  const runHandler = (payload: A, meta: MessageMetadata, deadline: LeaseDeadline) =>
+    Effect.scoped(
+      keepLeaseAlive(api, handle, timeout, deadline).pipe(Effect.andThen(handler(payload, meta))),
+    ).pipe(Effect.mapError(toPolicy));
+
+  const settleFailure = (error: QueueLeaseError) => Effect.fail(escalate<E>(Option.none(), error));
+
+  const acknowledge = (rerun: number, coalescedCount: number) =>
+    api
+      .acknowledge(
+        handle,
+        coalescedCount > 0 ? { experimentalIfCoalesced: coalescedCount } : undefined,
+      )
+      .pipe(
+        Effect.as(Option.none<number>()),
+        Effect.catchReason("MessageNotAvailable", "ConcurrencyCoalescedPending", (reason, error) =>
+          rerun > maxReruns
+            ? Effect.fail(escalate<E>(Option.some(reason), error))
+            : Effect.succeed(Option.some(reason.coalescedCount)),
+        ),
+        Effect.catchTags({
+          MessageNotAvailable: settleFailure,
+          MessageNotFound: settleFailure,
+          VercelApiError: settleFailure,
+          ConsumerRegistryNotConfigured: settleFailure,
+        }),
+      );
+
+  const attempt = Effect.gen(function* () {
+    const payload = yield* Effect.mapError(decode, toPolicy);
+    yield* runHandler(payload, metadata, delivery.deadline);
+    let coalescedCount = 0;
+    for (let rerun = 1; ; rerun++) {
+      const refused = yield* acknowledge(rerun, coalescedCount);
+      if (Option.isNone(refused)) return;
+      coalescedCount = refused.value;
+      yield* runHandler(
+        payload,
+        new MessageMetadata({
+          messageId: metadata.messageId,
+          deliveryCount: metadata.deliveryCount,
+          createdAt: metadata.createdAt,
+          expiresAt: metadata.expiresAt,
+          topic: metadata.topic,
+          consumerGroup: metadata.consumerGroup,
+          region: metadata.region,
+          ...(metadata.experimentalHandoffCount === undefined
+            ? {}
+            : { experimentalHandoffCount: metadata.experimentalHandoffCount }),
+          experimentalCoalesced: { rerun, coalescedCount },
+        }),
+        Option.some("expired"),
+      );
+    }
+  });
+
+  const chooseDirective = (error: E | ConsumeFailure) =>
+    options.onError === undefined
+      ? Effect.succeed(RetryDirective.Fail())
+      : options
+          .onError(error, metadata)
+          .pipe(
+            Effect.catchDefect((defect) =>
+              Effect.logWarning("Queue error policy failed; failing the delivery", defect).pipe(
+                Effect.as(RetryDirective.Fail()),
+              ),
+            ),
+          );
+
+  const applyDirective = (directive: RetryDirective, error: ProcessError<E>) =>
+    RetryDirective.$match(directive, {
+      Acknowledge: () =>
+        api
+          .acknowledge(handle)
+          .pipe(Effect.catch((cause) => Effect.logWarning("Failed to acknowledge message", cause))),
+      RetryAfter: ({ delay }) =>
+        api.changeVisibility(handle, delay).pipe(
+          Effect.asVoid,
+          Effect.catch((cause) =>
+            Effect.logWarning("Failed to reschedule message for retry", cause),
+          ),
+        ),
+      ExperimentalHandoff: () =>
+        api.changeVisibility(handle, Duration.zero, { experimentalHandoff: true }).pipe(
+          Effect.asVoid,
+          Effect.catch((cause) => Effect.logWarning("Failed to hand off message", cause)),
+        ),
+      Fail: () => Effect.fail(error),
+    });
+
+  return attempt.pipe(
+    Effect.catch(({ forPolicy, error }) =>
+      Option.match(forPolicy, {
+        onNone: () => Effect.fail(error),
+        onSome: (cause) =>
+          chooseDirective(cause).pipe(
+            Effect.flatMap((directive) => applyDirective(directive, error)),
+          ),
+      }),
+    ),
+    Effect.annotateLogs({ module: "QueueConsumer", messageId: handle.messageId }),
+  );
+};
+
+const PRODUCER_TRACEPARENT = "x-vercel-queue-traceparent";
+
+/** Links the producer's span, delivered in `x-vercel-queue-traceparent`, without parenting to it. */
+export const producerLinks = (headers: Headers.Headers): ReadonlyArray<Tracer.SpanLink> => {
+  const traceparent = headers[PRODUCER_TRACEPARENT];
+  if (!traceparent) return [];
+  return Option.match(HttpTraceContext.w3c(Headers.fromInput({ traceparent })), {
+    onNone: () => [],
+    onSome: (span) => [{ span, attributes: {} }],
+  });
+};
+
+/** Runs `self` in a consumer-kind `vqs.process` span, as `@vercel/queue`'s `withConsumerSpan`. */
+export const withProcessSpan =
+  (links: ReadonlyArray<Tracer.SpanLink>) =>
+  <A, E, R>(self: Effect.Effect<A, E, R>): Effect.Effect<A, E, R> =>
+    Effect.withSpan(self, "vqs.process", {
+      kind: "consumer",
+      links,
+      attributes: {
+        "messaging.system": "vercel-queue",
+        "messaging.operation": "process",
+      },
+    });

package/src/internal/queue-protocol.ts

@@ -0,0 +1,169 @@
+import { DateTime, Duration, Option, Predicate, Schema } from "effect";
+
+import type { OutgoingMessage } from "../contracts/queue-api.ts";
+
+export const errorCodeHeader = "Vqs-Error-Code";
+
+export const apiBaseUrl = (url: URL): string =>
+  `${url.origin}${url.pathname.replace(/\/+$/, "")}/api/v3/topic`;
+
+export const topicPath = (topic: string, ...segments: ReadonlyArray<string>): string =>
+  [topic, ...segments].map((segment) => `/${encodeURIComponent(segment)}`).join("");
+
+export const wholeSeconds = (input: Duration.Input): number =>
+  Math.ceil(Duration.toMillis(Duration.fromInputUnsafe(input)) / 1000);
+
+const trimHeaderValue = (value: string) => value.replace(/^[\t\n\r ]+|[\t\n\r ]+$/g, "");
+
+/** A case-insensitive header set with the same overwrite semantics as `Headers.set`. */
+export class HeaderSet {
+  private readonly entries = new Map<string, string>();
+
+  constructor(init: Readonly<Record<string, string>> = {}) {
+    for (const [name, value] of Object.entries(init)) this.set(name, value);
+  }
+
+  set(name: string, value: string): this {
+    this.entries.set(name.toLowerCase(), trimHeaderValue(value));
+    return this;
+  }
+
+  delete(name: string): this {
+    this.entries.delete(name.toLowerCase());
+    return this;
+  }
+
+  get(name: string): string | undefined {
+    return this.entries.get(name.toLowerCase());
+  }
+
+  toRecord(): Record<string, string> {
+    return Object.fromEntries(this.entries);
+  }
+
+  /** Entries sorted by name, as `Headers` iterates them. */
+  sorted(): Array<readonly [string, string]> {
+    return [...this.entries].toSorted(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
+  }
+}
+
+const isReservedMessageHeader = (name: string) => {
+  const lower = name.toLowerCase();
+  return lower === "authorization" || lower === "content-type" || lower.startsWith("vqs-");
+};
+
+export const messageHeaders = (
+  custom: Readonly<Record<string, string>>,
+  message: OutgoingMessage,
+): HeaderSet => {
+  const headers = new HeaderSet(custom).delete("authorization");
+  for (const [name, value] of Object.entries(message.headers ?? {})) {
+    if (!isReservedMessageHeader(name)) headers.set(name, value);
+  }
+  headers.set("content-type", message.contentType);
+  if (message.idempotencyKey) headers.set("vqs-idempotency-key", message.idempotencyKey);
+  if (message.retention !== undefined) {
+    headers.set("vqs-retention-seconds", String(wholeSeconds(message.retention)));
+  }
+  if (message.delay !== undefined) {
+    headers.set("vqs-delay-seconds", String(wholeSeconds(message.delay)));
+  }
+  if (message.experimentalConcurrencyKey) {
+    headers.set("vqs-experimental-concurrency-key", message.experimentalConcurrencyKey);
+  }
+  return headers;
+};
+
+export const retryAfterFromHeader = (
+  value: string | undefined,
+  nowMillis: number,
+): Option.Option<Duration.Duration> => {
+  if (!value) return Option.none();
+  const seconds = Number(value);
+  if (Number.isFinite(seconds) && seconds >= 0) return Option.some(Duration.seconds(seconds));
+  const dateMillis = Date.parse(value);
+  return Number.isNaN(dateMillis)
+    ? Option.none()
+    : Option.some(Duration.millis(Math.max(0, dateMillis - nowMillis)));
+};
+
+export interface ErrorBody {
+  readonly text: string;
+  readonly message: string;
+  readonly originalMessageId: Option.Option<string>;
+}
+
+const decodeJsonObject = Schema.decodeUnknownOption(
+  Schema.fromJsonString(Schema.Record(Schema.String, Schema.Unknown)),
+);
+
+export const errorBodyOf = (text: string, contentType: string | undefined): ErrorBody => {
+  const plain: ErrorBody = { text, message: text, originalMessageId: Option.none() };
+  if (!contentType?.toLowerCase().includes("json")) return plain;
+  return Option.match(decodeJsonObject(text), {
+    onNone: () => plain,
+    onSome: (body) => {
+      const error = body["error"];
+      const message = Predicate.isString(error)
+        ? error
+        : Predicate.isObject(error) && Predicate.isString(error["message"])
+          ? error["message"]
+          : text;
+      const original = body["originalMessageId"];
+      return {
+        text,
+        message,
+        originalMessageId: Predicate.isString(original) ? Option.some(original) : Option.none(),
+      };
+    },
+  });
+};
+
+const positiveInt = (value: unknown): Option.Option<number> =>
+  Predicate.isNumber(value) && Number.isInteger(value) && value > 0
+    ? Option.some(value)
+    : Option.none();
+
+export const coalescedPendingCount = (
+  errorCode: string | undefined,
+  coalescedHeader: string | undefined,
+  text: string,
+): Option.Option<number> => {
+  if (errorCode !== "concurrency_coalesced_pending") return Option.none();
+  return Option.orElse(positiveInt(Number(coalescedHeader ?? null)), () =>
+    Option.flatMap(decodeJsonObject(text), (body) => positiveInt(body["coalescedCount"])),
+  );
+};
+
+export interface QueuePartHeaders {
+  readonly messageId: string;
+  readonly receiptHandle: string;
+  readonly deliveryCount: number;
+  readonly timestamp: string;
+  readonly expiresAt: Option.Option<string>;
+  readonly contentType: string;
+  readonly handoffCount: Option.Option<number>;
+}
+
+export const queuePartHeaders = (
+  headers: Readonly<Record<string, string>>,
+): Option.Option<QueuePartHeaders> => {
+  const messageId = headers["vqs-message-id"];
+  const timestamp = headers["vqs-timestamp"];
+  const receiptHandle = headers["vqs-receipt-handle"];
+  if (!messageId || !timestamp || !receiptHandle) return Option.none();
+  const deliveryCount = Number.parseInt(headers["vqs-delivery-count"] || "0", 10);
+  if (Number.isNaN(deliveryCount)) return Option.none();
+  return Option.some({
+    messageId,
+    receiptHandle,
+    deliveryCount,
+    timestamp,
+    expiresAt: Option.fromUndefinedOr(headers["vqs-expires-at"] || undefined),
+    contentType: headers["content-type"] || "application/octet-stream",
+    handoffCount: positiveInt(Number(headers["vqs-experimental-handoff-count"] || Number.NaN)),
+  });
+};
+
+export const parseDate = (value: string): Option.Option<DateTime.Utc> =>
+  DateTime.make(new Date(value));

package/src/internal/version.ts

@@ -0,0 +1,5 @@
+export const packageName = "@panorama-industries/effect-vercel-queue";
+
+export const packageVersion = "0.2.0";
+
+export const userAgent = `${packageName}/${packageVersion}`;

package/src/retry-policy.ts

@@ -0,0 +1,48 @@
+import { Clock, Effect, Option, Pull, Schedule } from "effect";
+
+import type { MessageMetadata } from "./contracts/queue-domain.ts";
+
+import { RetryDirective } from "./contracts/message-handling.ts";
+
+/**
+ * Build an `onError` policy that redelivers failed messages on a `Schedule`.
+ *
+ * The n-th failed delivery (`deliveryCount` minus experimental handoffs, which do not count as
+ * retries) waits for the schedule's n-th delay: the policy answers `RetryAfter(delay)` while the
+ * schedule continues and `onExhausted` once it is done. The schedule receives the error as input
+ * and is stepped from a fresh state on every call, all at the current time, so elapsed-time
+ * schedules see no elapsed time.
+ *
+ * ```ts
+ * consumer.handler(Orders, fulfil, {
+ *   onError: retryOnSchedule(Schedule.exponential("1 second").pipe(Schedule.upTo({ times: 5 }))),
+ * });
+ * ```
+ *
+ * @param schedule - Delay of each retry; its outputs are ignored.
+ * @param options.onExhausted - Directive once the schedule is done. Default: `Acknowledge`, so the
+ *   message is dropped.
+ */
+export const retryOnSchedule =
+  <Out, In, R>(
+    schedule: Schedule.Schedule<Out, In, never, R>,
+    options?: { readonly onExhausted?: RetryDirective },
+  ) =>
+  (error: In, metadata: MessageMetadata): Effect.Effect<RetryDirective, never, R> =>
+    Effect.gen(function* () {
+      const step = yield* Schedule.toStep(schedule);
+      const now = yield* Clock.currentTimeMillis;
+      const attempt = Math.max(
+        1,
+        metadata.deliveryCount - (metadata.experimentalHandoffCount ?? 0),
+      );
+      let delay = Option.none<RetryDirective>();
+      for (let n = 0; n < attempt; n++) {
+        delay = yield* step(now, error).pipe(
+          Effect.map(([, duration]) => Option.some(RetryDirective.RetryAfter({ delay: duration }))),
+          Pull.catchDone(() => Effect.succeed(Option.none<RetryDirective>())),
+        );
+        if (Option.isNone(delay)) break;
+      }
+      return Option.getOrElse(delay, () => options?.onExhausted ?? RetryDirective.Acknowledge());
+    });

package/src/testing/index.ts

@@ -0,0 +1 @@
+export * from "./test-queue.ts";