@panorama-industries/effect-vercel-queue 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +13 -2
- package/dist/contracts/message-handling.d.ts +128 -0
- package/dist/contracts/message-handling.d.ts.map +1 -0
- package/dist/contracts/message-handling.js +4 -0
- package/dist/contracts/message-handling.js.map +1 -0
- package/dist/contracts/queue-api.d.ts +255 -0
- package/dist/contracts/queue-api.d.ts.map +1 -0
- package/dist/contracts/queue-api.js +18 -0
- package/dist/contracts/queue-api.js.map +1 -0
- package/dist/contracts/queue-config.d.ts +87 -0
- package/dist/contracts/queue-config.d.ts.map +1 -0
- package/dist/contracts/queue-config.js +7 -0
- package/dist/contracts/queue-config.js.map +1 -0
- package/dist/contracts/queue-consumer.d.ts +30 -0
- package/dist/contracts/queue-consumer.d.ts.map +1 -0
- package/dist/contracts/queue-consumer.js +15 -0
- package/dist/contracts/queue-consumer.js.map +1 -0
- package/dist/contracts/queue-domain.d.ts +171 -0
- package/dist/contracts/queue-domain.d.ts.map +1 -0
- package/dist/contracts/queue-domain.js +137 -0
- package/dist/contracts/queue-domain.js.map +1 -0
- package/dist/contracts/queue-poller.d.ts +169 -0
- package/dist/contracts/queue-poller.d.ts.map +1 -0
- package/dist/contracts/queue-poller.js +10 -0
- package/dist/contracts/queue-poller.js.map +1 -0
- package/dist/contracts/queue-producer.d.ts +23 -0
- package/dist/contracts/queue-producer.d.ts.map +1 -0
- package/dist/contracts/queue-producer.js +5 -0
- package/dist/contracts/queue-producer.js.map +1 -0
- package/dist/contracts/topic.d.ts +47 -0
- package/dist/contracts/topic.d.ts.map +1 -0
- package/dist/contracts/topic.js +34 -0
- package/dist/contracts/topic.js.map +1 -0
- package/dist/implementation/queue-api-live.d.ts +16 -0
- package/dist/implementation/queue-api-live.d.ts.map +1 -0
- package/dist/implementation/queue-api-live.js +326 -0
- package/dist/implementation/queue-api-live.js.map +1 -0
- package/dist/implementation/queue-config-live.d.ts +37 -0
- package/dist/implementation/queue-config-live.d.ts.map +1 -0
- package/dist/implementation/queue-config-live.js +99 -0
- package/dist/implementation/queue-config-live.js.map +1 -0
- package/dist/implementation/queue-consumer-live.d.ts +14 -0
- package/dist/implementation/queue-consumer-live.d.ts.map +1 -0
- package/dist/implementation/queue-consumer-live.js +56 -0
- package/dist/implementation/queue-consumer-live.js.map +1 -0
- package/dist/implementation/queue-live.d.ts +15 -0
- package/dist/implementation/queue-live.d.ts.map +1 -0
- package/dist/implementation/queue-live.js +16 -0
- package/dist/implementation/queue-live.js.map +1 -0
- package/dist/implementation/queue-poller-live.d.ts +14 -0
- package/dist/implementation/queue-poller-live.d.ts.map +1 -0
- package/dist/implementation/queue-poller-live.js +66 -0
- package/dist/implementation/queue-poller-live.js.map +1 -0
- package/dist/implementation/queue-producer-live.d.ts +9 -0
- package/dist/implementation/queue-producer-live.d.ts.map +1 -0
- package/dist/implementation/queue-producer-live.js +54 -0
- package/dist/implementation/queue-producer-live.js.map +1 -0
- package/dist/index.d.ts +16 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +16 -0
- package/dist/index.js.map +1 -0
- package/dist/internal/cloud-event.d.ts +46 -0
- package/dist/internal/cloud-event.d.ts.map +1 -0
- package/dist/internal/cloud-event.js +140 -0
- package/dist/internal/cloud-event.js.map +1 -0
- package/dist/internal/lease.d.ts +23 -0
- package/dist/internal/lease.d.ts.map +1 -0
- package/dist/internal/lease.js +44 -0
- package/dist/internal/lease.js.map +1 -0
- package/dist/internal/multipart.d.ts +23 -0
- package/dist/internal/multipart.d.ts.map +1 -0
- package/dist/internal/multipart.js +205 -0
- package/dist/internal/multipart.js.map +1 -0
- package/dist/internal/process.d.ts +25 -0
- package/dist/internal/process.d.ts.map +1 -0
- package/dist/internal/process.js +95 -0
- package/dist/internal/process.js.map +1 -0
- package/dist/internal/queue-protocol.d.ts +38 -0
- package/dist/internal/queue-protocol.d.ts.map +1 -0
- package/dist/internal/queue-protocol.js +119 -0
- package/dist/internal/queue-protocol.js.map +1 -0
- package/dist/internal/version.d.ts +4 -0
- package/dist/internal/version.d.ts.map +1 -0
- package/dist/internal/version.js +4 -0
- package/dist/internal/version.js.map +1 -0
- package/dist/retry-policy.d.ts +26 -0
- package/dist/retry-policy.d.ts.map +1 -0
- package/dist/retry-policy.js +34 -0
- package/dist/retry-policy.js.map +1 -0
- package/dist/testing/index.d.ts +2 -0
- package/dist/testing/index.d.ts.map +1 -0
- package/dist/testing/index.js +2 -0
- package/dist/testing/index.js.map +1 -0
- package/dist/testing/test-queue.d.ts +74 -0
- package/dist/testing/test-queue.d.ts.map +1 -0
- package/dist/testing/test-queue.js +253 -0
- package/dist/testing/test-queue.js.map +1 -0
- package/package.json +59 -3
- package/src/contracts/message-handling.ts +68 -0
- package/src/contracts/queue-api.ts +199 -0
- package/src/contracts/queue-config.ts +47 -0
- package/src/contracts/queue-consumer.ts +40 -0
- package/src/contracts/queue-domain.ts +202 -0
- package/src/contracts/queue-poller.ts +76 -0
- package/src/contracts/queue-producer.ts +40 -0
- package/src/contracts/topic.ts +76 -0
- package/src/implementation/queue-api-live.ts +637 -0
- package/src/implementation/queue-config-live.ts +178 -0
- package/src/implementation/queue-consumer-live.ts +112 -0
- package/src/implementation/queue-live.ts +32 -0
- package/src/implementation/queue-poller-live.ts +130 -0
- package/src/implementation/queue-producer-live.ts +89 -0
- package/src/index.ts +19 -0
- package/src/internal/cloud-event.ts +244 -0
- package/src/internal/lease.ts +84 -0
- package/src/internal/multipart.ts +240 -0
- package/src/internal/process.ts +194 -0
- package/src/internal/queue-protocol.ts +169 -0
- package/src/internal/version.ts +5 -0
- package/src/retry-policy.ts +48 -0
- package/src/testing/index.ts +1 -0
- package/src/testing/test-queue.ts +469 -0
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
import type { DeploymentId } from "@panorama-industries/effect-vercel-core";
|
|
2
|
+
|
|
3
|
+
import { VercelRegion, VercelSystemEnv } from "@panorama-industries/effect-vercel-core";
|
|
4
|
+
import { Config, ConfigProvider, Effect, Layer, Option, Redacted, Schema } from "effect";
|
|
5
|
+
|
|
6
|
+
import { QueueAuth, QueueConfig } from "../contracts/queue-config.ts";
|
|
7
|
+
|
|
8
|
+
/** Values that replace what {@link QueueConfigLive} derives from the environment. */
|
|
9
|
+
export interface QueueConfigOptions {
|
|
10
|
+
/** Default region; otherwise `VERCEL_REGION`, then `iad1`. */
|
|
11
|
+
readonly region?: VercelRegion;
|
|
12
|
+
|
|
13
|
+
/** Base URL per region; replaces both the regional default and the local broker URL. */
|
|
14
|
+
readonly baseUrl?: (region: VercelRegion) => URL;
|
|
15
|
+
|
|
16
|
+
/** Fixed bearer token used instead of OIDC or the local broker token. */
|
|
17
|
+
readonly token?: string | Redacted.Redacted<string>;
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Deployment that sends are pinned to and consumer calls name; otherwise `VERCEL_DEPLOYMENT_ID`.
|
|
21
|
+
* `"unpinned"` sends no deployment at all, so any deployment's consumers may receive messages.
|
|
22
|
+
*/
|
|
23
|
+
readonly deploymentId?: DeploymentId | "unpinned";
|
|
24
|
+
|
|
25
|
+
/** Extra headers sent with every request. */
|
|
26
|
+
readonly headers?: Readonly<Record<string, string>>;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
const defaultRegion = VercelRegion.make("iad1");
|
|
30
|
+
|
|
31
|
+
const semver =
|
|
32
|
+
/^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-([\da-zA-Z-]+(?:\.[\da-zA-Z-]+)*))?(?:\+[\da-zA-Z-]+(?:\.[\da-zA-Z-]+)*)?$/;
|
|
33
|
+
|
|
34
|
+
const supportsLocalQueueBroker = (version: string): boolean => {
|
|
35
|
+
if (version.length > 256) return false;
|
|
36
|
+
const match = semver.exec(version);
|
|
37
|
+
if (match === null) return false;
|
|
38
|
+
const major = Number(match[1]);
|
|
39
|
+
const minor = Number(match[2]);
|
|
40
|
+
const patch = Number(match[3]);
|
|
41
|
+
const prerelease = match[4];
|
|
42
|
+
if (
|
|
43
|
+
![major, minor, patch].every(Number.isSafeInteger) ||
|
|
44
|
+
prerelease?.split(".").some((part) => /^0\d+$/.test(part))
|
|
45
|
+
) {
|
|
46
|
+
return false;
|
|
47
|
+
}
|
|
48
|
+
return (
|
|
49
|
+
major > 59 ||
|
|
50
|
+
(major === 59 && (minor > 26 || (minor === 26 && (patch > 0 || prerelease === undefined))))
|
|
51
|
+
);
|
|
52
|
+
};
|
|
53
|
+
|
|
54
|
+
const configError = (message: string) =>
|
|
55
|
+
new Config.ConfigError(new ConfigProvider.SourceError({ message }));
|
|
56
|
+
|
|
57
|
+
const nonEmpty = (name: string) =>
|
|
58
|
+
Config.option(Config.String(name)).pipe(Config.map(Option.filter((value) => value !== "")));
|
|
59
|
+
|
|
60
|
+
const decodeUrl = Schema.decodeUnknownOption(Schema.URLFromString);
|
|
61
|
+
|
|
62
|
+
const regionalBaseUrl = (region: VercelRegion) => new URL(`https://${region}.vercel-queue.com`);
|
|
63
|
+
|
|
64
|
+
const stripTrailingSlashes = (path: string) => path.replace(/\/+$/, "");
|
|
65
|
+
|
|
66
|
+
const sameEndpoint = (a: URL, b: URL) =>
|
|
67
|
+
a.origin === b.origin && stripTrailingSlashes(a.pathname) === stripTrailingSlashes(b.pathname);
|
|
68
|
+
|
|
69
|
+
const missingDeploymentId = configError(
|
|
70
|
+
"No deployment ID available. VERCEL_DEPLOYMENT_ID is not set.\n\n" +
|
|
71
|
+
"This usually means the code is running outside a Vercel deployment " +
|
|
72
|
+
"(e.g. during build or in a non-Vercel environment).\n\n" +
|
|
73
|
+
"To fix this, provide an explicit deployment ID:\n" +
|
|
74
|
+
' queueConfigLayer({ deploymentId: DeploymentId.make("dpl_xxx") })\n' +
|
|
75
|
+
"Or explicitly opt out of deployment pinning:\n" +
|
|
76
|
+
' queueConfigLayer({ deploymentId: "unpinned" })',
|
|
77
|
+
);
|
|
78
|
+
|
|
79
|
+
const missingLocalToken = configError(
|
|
80
|
+
"Missing local queue token. VERCEL_QUEUE_BASE_URL selects a local broker, " +
|
|
81
|
+
"but VERCEL_QUEUE_TOKEN is not set. Start your app with a Vercel CLI " +
|
|
82
|
+
"version that supports local Queues, or provide an explicit token.",
|
|
83
|
+
);
|
|
84
|
+
|
|
85
|
+
const localToken = Config.option(Config.Redacted("VERCEL_QUEUE_TOKEN")).pipe(
|
|
86
|
+
Effect.map(Option.filter((token) => Redacted.value(token) !== "")),
|
|
87
|
+
Effect.flatMap(
|
|
88
|
+
Option.match({ onNone: () => Effect.fail(missingLocalToken), onSome: Effect.succeed }),
|
|
89
|
+
),
|
|
90
|
+
);
|
|
91
|
+
|
|
92
|
+
const invalidLocalUrl = configError("Invalid VERCEL_QUEUE_BASE_URL: expected an absolute URL");
|
|
93
|
+
|
|
94
|
+
const resolveQueueConfig = Effect.fnUntraced(function* (options: QueueConfigOptions) {
|
|
95
|
+
const env = yield* VercelSystemEnv;
|
|
96
|
+
const devMode = env.isDevelopment && Option.isNone(env.deploymentId);
|
|
97
|
+
|
|
98
|
+
const region =
|
|
99
|
+
options.region ??
|
|
100
|
+
Option.getOrUndefined(env.region) ??
|
|
101
|
+
(yield* Effect.as(
|
|
102
|
+
devMode
|
|
103
|
+
? Effect.void
|
|
104
|
+
: Effect.logWarning(
|
|
105
|
+
`Queue region not detected; defaulting to "${defaultRegion}". On Vercel it comes ` +
|
|
106
|
+
"from VERCEL_REGION. Set it explicitly with queueConfigLayer({ region }) to " +
|
|
107
|
+
"silence this warning.",
|
|
108
|
+
),
|
|
109
|
+
defaultRegion,
|
|
110
|
+
));
|
|
111
|
+
|
|
112
|
+
const injectedUrl =
|
|
113
|
+
devMode && Option.exists(env.cliVersion, supportsLocalQueueBroker)
|
|
114
|
+
? yield* nonEmpty("VERCEL_QUEUE_BASE_URL")
|
|
115
|
+
: Option.none<string>();
|
|
116
|
+
const localUrl = Option.flatMap(injectedUrl, decodeUrl);
|
|
117
|
+
if (Option.isSome(injectedUrl) && Option.isNone(localUrl) && options.baseUrl === undefined) {
|
|
118
|
+
return yield* Effect.fail(invalidLocalUrl);
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
const baseUrl =
|
|
122
|
+
options.baseUrl ??
|
|
123
|
+
Option.match(localUrl, {
|
|
124
|
+
onNone: () => regionalBaseUrl,
|
|
125
|
+
onSome: (url) => () => new URL(url),
|
|
126
|
+
});
|
|
127
|
+
const usesLocalBroker = Option.exists(localUrl, (url) => sameEndpoint(baseUrl(region), url));
|
|
128
|
+
|
|
129
|
+
const auth =
|
|
130
|
+
options.token !== undefined
|
|
131
|
+
? QueueAuth.Token({
|
|
132
|
+
token: Redacted.isRedacted(options.token) ? options.token : Redacted.make(options.token),
|
|
133
|
+
})
|
|
134
|
+
: usesLocalBroker
|
|
135
|
+
? QueueAuth.Token({ token: yield* localToken })
|
|
136
|
+
: QueueAuth.Oidc();
|
|
137
|
+
|
|
138
|
+
const deploymentId =
|
|
139
|
+
options.deploymentId === "unpinned" || usesLocalBroker || devMode
|
|
140
|
+
? Option.none<DeploymentId>()
|
|
141
|
+
: Option.some(
|
|
142
|
+
yield* Option.match(
|
|
143
|
+
Option.orElse(Option.fromUndefinedOr(options.deploymentId), () => env.deploymentId),
|
|
144
|
+
{ onNone: () => Effect.fail(missingDeploymentId), onSome: Effect.succeed },
|
|
145
|
+
),
|
|
146
|
+
);
|
|
147
|
+
|
|
148
|
+
return QueueConfig.of({
|
|
149
|
+
region,
|
|
150
|
+
baseUrl,
|
|
151
|
+
auth,
|
|
152
|
+
deploymentId,
|
|
153
|
+
pinSends: Option.isSome(deploymentId),
|
|
154
|
+
headers: options.headers ?? {},
|
|
155
|
+
});
|
|
156
|
+
});
|
|
157
|
+
|
|
158
|
+
/**
|
|
159
|
+
* Builds {@link QueueConfig} from the environment, with any given values taking precedence.
|
|
160
|
+
*
|
|
161
|
+
* Outside development, construction fails with a `ConfigError` when no deployment ID is known and
|
|
162
|
+
* the options do not opt out with `deploymentId: "unpinned"`.
|
|
163
|
+
*/
|
|
164
|
+
export const queueConfigLayer = (
|
|
165
|
+
options: QueueConfigOptions = {},
|
|
166
|
+
): Layer.Layer<QueueConfig, Config.ConfigError, VercelSystemEnv> =>
|
|
167
|
+
Layer.effect(QueueConfig, resolveQueueConfig(options));
|
|
168
|
+
|
|
169
|
+
/**
|
|
170
|
+
* Derives {@link QueueConfig} from `VercelSystemEnv`, `VERCEL_QUEUE_BASE_URL` and
|
|
171
|
+
* `VERCEL_QUEUE_TOKEN`.
|
|
172
|
+
*
|
|
173
|
+
* On Vercel it targets `https://{region}.vercel-queue.com` with OIDC and pins sends to the
|
|
174
|
+
* deployment, failing with a `ConfigError` when `VERCEL_DEPLOYMENT_ID` is missing. Under
|
|
175
|
+
* `vercel dev` (CLI 59.26.0 or later) it targets the local broker with its token.
|
|
176
|
+
*/
|
|
177
|
+
export const QueueConfigLive: Layer.Layer<QueueConfig, Config.ConfigError, VercelSystemEnv> =
|
|
178
|
+
queueConfigLayer();
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
import type { HttpServerError } from "effect/http";
|
|
2
|
+
|
|
3
|
+
import { Cause, Effect, Layer, Option, Predicate } from "effect";
|
|
4
|
+
import { HttpServerRequest, HttpServerResponse } from "effect/http";
|
|
5
|
+
|
|
6
|
+
import type { ConsumeOptions, MessageHandler } from "../contracts/message-handling.ts";
|
|
7
|
+
import type { QueueReceiveByIdError } from "../contracts/queue-api.ts";
|
|
8
|
+
import type { MessageId } from "../contracts/queue-domain.ts";
|
|
9
|
+
import type { Topic } from "../contracts/topic.ts";
|
|
10
|
+
import type { Delivery } from "../internal/process.ts";
|
|
11
|
+
|
|
12
|
+
import { QueueApi } from "../contracts/queue-api.ts";
|
|
13
|
+
import { QueueConfig } from "../contracts/queue-config.ts";
|
|
14
|
+
import { QueueConsumer } from "../contracts/queue-consumer.ts";
|
|
15
|
+
import { inlineDelivery, MalformedCloudEvent, parseCallback } from "../internal/cloud-event.ts";
|
|
16
|
+
import { leaseVisibilityTimeout } from "../internal/lease.ts";
|
|
17
|
+
import { processDelivery, producerLinks, withProcessSpan } from "../internal/process.ts";
|
|
18
|
+
|
|
19
|
+
const skipped = (reason: "already_processed" | "not_available", messageId: MessageId) =>
|
|
20
|
+
Effect.logDebug(
|
|
21
|
+
`Queue callback skipped (${reason}) for message ${messageId}: ` +
|
|
22
|
+
"the message was already handled or is no longer available to this consumer.",
|
|
23
|
+
).pipe(Effect.as(HttpServerResponse.jsonUnsafe({ status: "skipped", reason, messageId })));
|
|
24
|
+
|
|
25
|
+
const failed = (cause: Cause.Cause<unknown>) =>
|
|
26
|
+
Cause.hasInterruptsOnly(cause)
|
|
27
|
+
? Effect.failCause(cause as Cause.Cause<never>)
|
|
28
|
+
: Effect.logError("Queue callback error", cause).pipe(
|
|
29
|
+
Effect.as(
|
|
30
|
+
HttpServerResponse.jsonUnsafe(
|
|
31
|
+
{ error: "Failed to process queue message" },
|
|
32
|
+
{ status: 500 },
|
|
33
|
+
),
|
|
34
|
+
),
|
|
35
|
+
);
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Implements {@link QueueConsumer} for push delivery, as `@vercel/queue`'s `handleCallback`.
|
|
39
|
+
*
|
|
40
|
+
* Requires `QueueApi` and `QueueConfig`. Leases default to 5 minutes (at least 30 seconds) and are
|
|
41
|
+
* renewed while the handler runs. Responses: 200 `{status:"success"}`; 200 `{status:"skipped"}`
|
|
42
|
+
* when the message was already processed (410) or is held elsewhere (409); 400 `{error}` for a
|
|
43
|
+
* malformed CloudEvent; 500 otherwise, so the queue redelivers.
|
|
44
|
+
*/
|
|
45
|
+
export const QueueConsumerLive = Layer.effect(
|
|
46
|
+
QueueConsumer,
|
|
47
|
+
Effect.gen(function* () {
|
|
48
|
+
const api = yield* QueueApi;
|
|
49
|
+
const config = yield* QueueConfig;
|
|
50
|
+
|
|
51
|
+
const handler = <A, I, E, R, R2 = never>(
|
|
52
|
+
topic: Topic<A, I>,
|
|
53
|
+
handle: MessageHandler<A, E, R>,
|
|
54
|
+
options: ConsumeOptions<E, R2> = {},
|
|
55
|
+
) =>
|
|
56
|
+
Effect.gen(function* () {
|
|
57
|
+
const request = yield* HttpServerRequest.HttpServerRequest;
|
|
58
|
+
const parsed = yield* parseCallback(request.headers, request.text);
|
|
59
|
+
const region = Option.getOrElse(parsed.region, () => config.region);
|
|
60
|
+
|
|
61
|
+
const delivery: Effect.Effect<
|
|
62
|
+
Delivery,
|
|
63
|
+
QueueReceiveByIdError | HttpServerError.HttpServerError | MalformedCloudEvent
|
|
64
|
+
> = Option.match(parsed.inline, {
|
|
65
|
+
onNone: () =>
|
|
66
|
+
api
|
|
67
|
+
.receiveById(parsed.topic, parsed.consumerGroup, parsed.messageId, {
|
|
68
|
+
visibilityTimeout: leaseVisibilityTimeout(options.visibilityTimeout),
|
|
69
|
+
region,
|
|
70
|
+
})
|
|
71
|
+
.pipe(Effect.map((message) => ({ message, deadline: Option.none() }))),
|
|
72
|
+
onSome: (inline) =>
|
|
73
|
+
Effect.flatMap(request.arrayBuffer, (buffer) =>
|
|
74
|
+
buffer.byteLength === 0
|
|
75
|
+
? Effect.fail(
|
|
76
|
+
new MalformedCloudEvent({
|
|
77
|
+
message: "Binary mode callback with receipt handle is missing payload",
|
|
78
|
+
}),
|
|
79
|
+
)
|
|
80
|
+
: inlineDelivery(parsed, inline, region, topic.contentType, new Uint8Array(buffer)),
|
|
81
|
+
),
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
return yield* delivery.pipe(
|
|
85
|
+
Effect.flatMap((received) => processDelivery(api, topic, handle, options, received)),
|
|
86
|
+
withProcessSpan(producerLinks(request.headers)),
|
|
87
|
+
Effect.as(HttpServerResponse.jsonUnsafe({ status: "success" })),
|
|
88
|
+
Effect.catch((error) => {
|
|
89
|
+
if (Predicate.isTagged(error, "MessageAlreadyProcessed")) {
|
|
90
|
+
return skipped("already_processed", parsed.messageId);
|
|
91
|
+
}
|
|
92
|
+
if (Predicate.isTagged(error, "MessageNotAvailable")) {
|
|
93
|
+
return skipped("not_available", parsed.messageId);
|
|
94
|
+
}
|
|
95
|
+
return Effect.fail(error);
|
|
96
|
+
}),
|
|
97
|
+
);
|
|
98
|
+
}).pipe(
|
|
99
|
+
Effect.catch((error) =>
|
|
100
|
+
error instanceof MalformedCloudEvent
|
|
101
|
+
? Effect.logError("Queue callback error", error).pipe(
|
|
102
|
+
Effect.as(HttpServerResponse.jsonUnsafe({ error: error.message }, { status: 400 })),
|
|
103
|
+
)
|
|
104
|
+
: Effect.fail(error),
|
|
105
|
+
),
|
|
106
|
+
Effect.catchCause(failed),
|
|
107
|
+
Effect.annotateLogs({ module: "QueueConsumer" }),
|
|
108
|
+
);
|
|
109
|
+
|
|
110
|
+
return QueueConsumer.of({ handler });
|
|
111
|
+
}),
|
|
112
|
+
);
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import type { Config } from "effect";
|
|
2
|
+
import type { HttpClient } from "effect/http";
|
|
3
|
+
|
|
4
|
+
import { VercelOidcLive, VercelSystemEnvLive } from "@panorama-industries/effect-vercel-core";
|
|
5
|
+
import { Layer } from "effect";
|
|
6
|
+
|
|
7
|
+
import type { QueueConsumer } from "../contracts/queue-consumer.ts";
|
|
8
|
+
import type { QueuePoller } from "../contracts/queue-poller.ts";
|
|
9
|
+
import type { QueueProducer } from "../contracts/queue-producer.ts";
|
|
10
|
+
|
|
11
|
+
import { QueueApiLive } from "./queue-api-live.ts";
|
|
12
|
+
import { QueueConfigLive } from "./queue-config-live.ts";
|
|
13
|
+
import { QueueConsumerLive } from "./queue-consumer-live.ts";
|
|
14
|
+
import { QueuePollerLive } from "./queue-poller-live.ts";
|
|
15
|
+
import { QueueProducerLive } from "./queue-producer-live.ts";
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Producer, push consumer and poller configured from the Vercel environment, authenticated with
|
|
19
|
+
* the function's OIDC token. Provide an `HttpClient`, e.g. `FetchHttpClient.layer`.
|
|
20
|
+
*
|
|
21
|
+
* Compose the individual layers instead to override configuration (`queueConfigLayer`) or
|
|
22
|
+
* credentials.
|
|
23
|
+
*/
|
|
24
|
+
export const QueueLive: Layer.Layer<
|
|
25
|
+
QueueProducer | QueueConsumer | QueuePoller,
|
|
26
|
+
Config.ConfigError,
|
|
27
|
+
HttpClient.HttpClient
|
|
28
|
+
> = Layer.mergeAll(QueueProducerLive, QueueConsumerLive, QueuePollerLive).pipe(
|
|
29
|
+
Layer.provideMerge(QueueApiLive),
|
|
30
|
+
Layer.provide(Layer.mergeAll(QueueConfigLive, VercelOidcLive)),
|
|
31
|
+
Layer.provide(VercelSystemEnvLive),
|
|
32
|
+
);
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
import { Effect, Exit, Layer, Option, Predicate, Scope } from "effect";
|
|
2
|
+
|
|
3
|
+
import type { ConsumeOptions, MessageHandler } from "../contracts/message-handling.ts";
|
|
4
|
+
import type {
|
|
5
|
+
ConsumerGroupName,
|
|
6
|
+
MessageAlreadyProcessed,
|
|
7
|
+
MessageId,
|
|
8
|
+
MessageNotAvailable,
|
|
9
|
+
MessageNotFound,
|
|
10
|
+
ReceivedMessage,
|
|
11
|
+
} from "../contracts/queue-domain.ts";
|
|
12
|
+
import type { PollByIdOptions, PollOptions } from "../contracts/queue-poller.ts";
|
|
13
|
+
import type { Topic } from "../contracts/topic.ts";
|
|
14
|
+
|
|
15
|
+
import { QueueApi } from "../contracts/queue-api.ts";
|
|
16
|
+
import { MessageCorrupted, PayloadDecodeFailed } from "../contracts/queue-domain.ts";
|
|
17
|
+
import { QueuePoller, ReceiveOutcome } from "../contracts/queue-poller.ts";
|
|
18
|
+
import { keepLeaseAlive, leaseVisibilityTimeout } from "../internal/lease.ts";
|
|
19
|
+
import { processDelivery, withProcessSpan } from "../internal/process.ts";
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Implements {@link QueuePoller}, as `@vercel/queue`'s `PollingQueueClient.receive`.
|
|
23
|
+
*
|
|
24
|
+
* Requires `QueueApi`. Leases default to 5 minutes (at least 30 seconds) and are renewed while each
|
|
25
|
+
* handler runs. A batch is handled concurrently up to `concurrency` (unbounded by default), and
|
|
26
|
+
* messages waiting their turn keep their leases renewed. When handlers fail, every message is still
|
|
27
|
+
* settled and the first failure in delivery order is returned. An undecodable payload whose error
|
|
28
|
+
* policy answers `Fail` is reported as `MessageCorrupted`.
|
|
29
|
+
*/
|
|
30
|
+
export const QueuePollerLive = Layer.effect(
|
|
31
|
+
QueuePoller,
|
|
32
|
+
Effect.gen(function* () {
|
|
33
|
+
const api = yield* QueueApi;
|
|
34
|
+
|
|
35
|
+
const handleOne = <A, I, E, R, R2>(
|
|
36
|
+
topic: Topic<A, I>,
|
|
37
|
+
handler: MessageHandler<A, E, R>,
|
|
38
|
+
options: ConsumeOptions<E, R2>,
|
|
39
|
+
message: ReceivedMessage,
|
|
40
|
+
) =>
|
|
41
|
+
processDelivery(api, topic, handler, options, {
|
|
42
|
+
message,
|
|
43
|
+
deadline: Option.none(),
|
|
44
|
+
}).pipe(
|
|
45
|
+
withProcessSpan([]),
|
|
46
|
+
Effect.mapError((error) =>
|
|
47
|
+
error instanceof PayloadDecodeFailed
|
|
48
|
+
? new MessageCorrupted({ messageId: error.messageId, message: error.message })
|
|
49
|
+
: error,
|
|
50
|
+
),
|
|
51
|
+
);
|
|
52
|
+
|
|
53
|
+
const receive = <A, I, E, R, R2 = never>(
|
|
54
|
+
topic: Topic<A, I>,
|
|
55
|
+
consumerGroup: ConsumerGroupName,
|
|
56
|
+
handler: MessageHandler<A, E, R>,
|
|
57
|
+
options: PollOptions<E, R2> = {},
|
|
58
|
+
) =>
|
|
59
|
+
Effect.gen(function* () {
|
|
60
|
+
const messages = yield* api.receive(topic.name, consumerGroup, {
|
|
61
|
+
visibilityTimeout: leaseVisibilityTimeout(options.visibilityTimeout),
|
|
62
|
+
...(options.limit === undefined ? {} : { limit: options.limit }),
|
|
63
|
+
...(options.cursorFallback === undefined
|
|
64
|
+
? {}
|
|
65
|
+
: { cursorFallback: options.cursorFallback }),
|
|
66
|
+
...(options.region === undefined ? {} : { region: options.region }),
|
|
67
|
+
});
|
|
68
|
+
if (messages.length === 0) return ReceiveOutcome.Empty();
|
|
69
|
+
const timeout = leaseVisibilityTimeout(options.visibilityTimeout);
|
|
70
|
+
const waiting = yield* Effect.forEach(messages, (message) =>
|
|
71
|
+
Effect.gen(function* () {
|
|
72
|
+
const scope = yield* Scope.make();
|
|
73
|
+
yield* keepLeaseAlive(api, message.handle, timeout, Option.none()).pipe(
|
|
74
|
+
Scope.provide(scope),
|
|
75
|
+
);
|
|
76
|
+
return { message, scope };
|
|
77
|
+
}),
|
|
78
|
+
);
|
|
79
|
+
const exits = yield* Effect.forEach(
|
|
80
|
+
waiting,
|
|
81
|
+
({ message, scope }) =>
|
|
82
|
+
Scope.close(scope, Exit.void).pipe(
|
|
83
|
+
Effect.andThen(Effect.exit(handleOne(topic, handler, options, message))),
|
|
84
|
+
),
|
|
85
|
+
{ concurrency: options.concurrency ?? "unbounded" },
|
|
86
|
+
).pipe(
|
|
87
|
+
Effect.ensuring(Effect.forEach(waiting, ({ scope }) => Scope.close(scope, Exit.void))),
|
|
88
|
+
);
|
|
89
|
+
const failure = exits.find(Exit.isFailure);
|
|
90
|
+
if (failure !== undefined) return yield* Effect.failCause(failure.cause);
|
|
91
|
+
return ReceiveOutcome.Processed({ count: messages.length });
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
const receiveById = <A, I, E, R, R2 = never>(
|
|
95
|
+
topic: Topic<A, I>,
|
|
96
|
+
consumerGroup: ConsumerGroupName,
|
|
97
|
+
messageId: MessageId,
|
|
98
|
+
handler: MessageHandler<A, E, R>,
|
|
99
|
+
options: PollByIdOptions<E, R2> = {},
|
|
100
|
+
) =>
|
|
101
|
+
api
|
|
102
|
+
.receiveById(topic.name, consumerGroup, messageId, {
|
|
103
|
+
visibilityTimeout: leaseVisibilityTimeout(options.visibilityTimeout),
|
|
104
|
+
...(options.region === undefined ? {} : { region: options.region }),
|
|
105
|
+
})
|
|
106
|
+
.pipe(
|
|
107
|
+
Effect.flatMap((message) => handleOne(topic, handler, options, message)),
|
|
108
|
+
Effect.as(ReceiveOutcome.Processed({ count: 1 })),
|
|
109
|
+
Effect.catch((error) => {
|
|
110
|
+
if (Predicate.isTagged(error, "MessageNotFound")) {
|
|
111
|
+
return Effect.succeed<ReceiveOutcome>(ReceiveOutcome.NotFound({ messageId }));
|
|
112
|
+
}
|
|
113
|
+
if (Predicate.isTagged(error, "MessageNotAvailable")) {
|
|
114
|
+
return Effect.succeed<ReceiveOutcome>(ReceiveOutcome.NotAvailable({ messageId }));
|
|
115
|
+
}
|
|
116
|
+
if (Predicate.isTagged(error, "MessageAlreadyProcessed")) {
|
|
117
|
+
return Effect.succeed<ReceiveOutcome>(ReceiveOutcome.AlreadyProcessed({ messageId }));
|
|
118
|
+
}
|
|
119
|
+
return Effect.fail(
|
|
120
|
+
error as Exclude<
|
|
121
|
+
typeof error,
|
|
122
|
+
MessageNotFound | MessageNotAvailable | MessageAlreadyProcessed
|
|
123
|
+
>,
|
|
124
|
+
);
|
|
125
|
+
}),
|
|
126
|
+
);
|
|
127
|
+
|
|
128
|
+
return QueuePoller.of({ receive, receiveById });
|
|
129
|
+
}),
|
|
130
|
+
);
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import { Array, Effect, Layer, Schema } from "effect";
|
|
2
|
+
|
|
3
|
+
import type { PublishOptions } from "../contracts/queue-api.ts";
|
|
4
|
+
import type { BatchEntry } from "../contracts/queue-producer.ts";
|
|
5
|
+
import type { Topic } from "../contracts/topic.ts";
|
|
6
|
+
|
|
7
|
+
import { QueueApi, SendResult } from "../contracts/queue-api.ts";
|
|
8
|
+
import { BatchSizeOutOfRange, PayloadEncodeFailed } from "../contracts/queue-domain.ts";
|
|
9
|
+
import { QueueProducer } from "../contracts/queue-producer.ts";
|
|
10
|
+
|
|
11
|
+
const maxBatchSize = 100;
|
|
12
|
+
|
|
13
|
+
const encode = <A, I>(topic: Topic<A, I>, payload: A) =>
|
|
14
|
+
Schema.encodeEffect(topic.wire)(payload).pipe(
|
|
15
|
+
Effect.mapError(
|
|
16
|
+
(error) => new PayloadEncodeFailed({ topic: topic.name, message: error.message }),
|
|
17
|
+
),
|
|
18
|
+
);
|
|
19
|
+
|
|
20
|
+
const publishOptions = (options: PublishOptions): PublishOptions => ({
|
|
21
|
+
...(options.idempotencyKey !== undefined && { idempotencyKey: options.idempotencyKey }),
|
|
22
|
+
...(options.retention !== undefined && { retention: options.retention }),
|
|
23
|
+
...(options.delay !== undefined && { delay: options.delay }),
|
|
24
|
+
...(options.headers !== undefined && { headers: options.headers }),
|
|
25
|
+
...(options.experimentalConcurrencyKey !== undefined && {
|
|
26
|
+
experimentalConcurrencyKey: options.experimentalConcurrencyKey,
|
|
27
|
+
}),
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
const sendSpan = (topic: Topic.Any) =>
|
|
31
|
+
Effect.withSpan("vqs.send", {
|
|
32
|
+
kind: "producer",
|
|
33
|
+
attributes: {
|
|
34
|
+
"messaging.system": "vercel-queue",
|
|
35
|
+
"messaging.destination.name": topic.name,
|
|
36
|
+
},
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Implements {@link QueueProducer} over `QueueApi`: encodes payloads with the topic's schema and
|
|
41
|
+
* publishes them in a `vqs.send` producer span.
|
|
42
|
+
*/
|
|
43
|
+
export const QueueProducerLive: Layer.Layer<QueueProducer, never, QueueApi> = Layer.effect(
|
|
44
|
+
QueueProducer,
|
|
45
|
+
Effect.gen(function* () {
|
|
46
|
+
const api = yield* QueueApi;
|
|
47
|
+
|
|
48
|
+
const send = Effect.fnUntraced(function* <A, I>(
|
|
49
|
+
topic: Topic<A, I>,
|
|
50
|
+
payload: A,
|
|
51
|
+
options: PublishOptions = {},
|
|
52
|
+
) {
|
|
53
|
+
const body = yield* encode(topic, payload);
|
|
54
|
+
return yield* api
|
|
55
|
+
.send(topic.name, { ...publishOptions(options), body, contentType: topic.contentType })
|
|
56
|
+
.pipe(
|
|
57
|
+
Effect.tap(
|
|
58
|
+
SendResult.$match({
|
|
59
|
+
Sent: ({ messageId }) =>
|
|
60
|
+
Effect.annotateCurrentSpan("messaging.message.id", messageId),
|
|
61
|
+
Deferred: () => Effect.void,
|
|
62
|
+
}),
|
|
63
|
+
),
|
|
64
|
+
sendSpan(topic),
|
|
65
|
+
);
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
const sendBatch = Effect.fnUntraced(function* <A, I>(
|
|
69
|
+
topic: Topic<A, I>,
|
|
70
|
+
entries: Array.NonEmptyReadonlyArray<BatchEntry<A>>,
|
|
71
|
+
) {
|
|
72
|
+
if (entries.length === 0 || entries.length > maxBatchSize) {
|
|
73
|
+
return yield* new BatchSizeOutOfRange({ size: entries.length, maximum: maxBatchSize });
|
|
74
|
+
}
|
|
75
|
+
const toMessage = (entry: BatchEntry<A>) =>
|
|
76
|
+
Effect.map(encode(topic, entry.payload), (body) => ({
|
|
77
|
+
...publishOptions(entry),
|
|
78
|
+
body,
|
|
79
|
+
contentType: topic.contentType,
|
|
80
|
+
}));
|
|
81
|
+
const [head, ...tail] = entries;
|
|
82
|
+
const first = yield* toMessage(head);
|
|
83
|
+
const rest = yield* Effect.forEach(tail, toMessage);
|
|
84
|
+
return yield* api.sendBatch(topic.name, [first, ...rest]).pipe(sendSpan(topic));
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
return QueueProducer.of({ send, sendBatch });
|
|
88
|
+
}),
|
|
89
|
+
);
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
export * from "./contracts/message-handling.ts";
|
|
2
|
+
export * from "./contracts/queue-api.ts";
|
|
3
|
+
export * from "./contracts/queue-config.ts";
|
|
4
|
+
export * from "./contracts/queue-consumer.ts";
|
|
5
|
+
export * from "./contracts/queue-domain.ts";
|
|
6
|
+
export * from "./contracts/queue-poller.ts";
|
|
7
|
+
export * from "./contracts/queue-producer.ts";
|
|
8
|
+
export * from "./contracts/topic.ts";
|
|
9
|
+
export { QueueApiLive } from "./implementation/queue-api-live.ts";
|
|
10
|
+
export {
|
|
11
|
+
QueueConfigLive,
|
|
12
|
+
type QueueConfigOptions,
|
|
13
|
+
queueConfigLayer,
|
|
14
|
+
} from "./implementation/queue-config-live.ts";
|
|
15
|
+
export { QueueConsumerLive } from "./implementation/queue-consumer-live.ts";
|
|
16
|
+
export { QueueLive } from "./implementation/queue-live.ts";
|
|
17
|
+
export { QueuePollerLive } from "./implementation/queue-poller-live.ts";
|
|
18
|
+
export { QueueProducerLive } from "./implementation/queue-producer-live.ts";
|
|
19
|
+
export { retryOnSchedule } from "./retry-policy.ts";
|