@ayepi/aws 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/ayepi-aws.md +434 -0
- package/package.json +9 -8
package/README.md
CHANGED
|
@@ -84,7 +84,7 @@ This package ships dense, machine-oriented reference docs written for **AI codin
|
|
|
84
84
|
|
|
85
85
|
- [`ayepi-aws.md`](./ayepi-aws.md)
|
|
86
86
|
|
|
87
|
-
They
|
|
87
|
+
They ship with this package and also live in the [repo](https://github.com/ClickerMonkey/ayepi/tree/main/packages/aws).
|
|
88
88
|
|
|
89
89
|
## License
|
|
90
90
|
|
package/ayepi-aws.md
ADDED
|
@@ -0,0 +1,434 @@
|
|
|
1
|
+
<!--
|
|
2
|
+
ayepi-aws.md — reference for `@ayepi/aws`, written for coding agents.
|
|
3
|
+
|
|
4
|
+
Copy this file into any project that depends on `@ayepi/aws` (e.g. into your repo's
|
|
5
|
+
`docs/` or `.claude/` directory) and reference it from your agents and slash commands.
|
|
6
|
+
It documents the public API, the patterns the package expects, and how it works under the
|
|
7
|
+
hood, with copy-pasteable examples. Keep it in sync with the installed package version.
|
|
8
|
+
-->
|
|
9
|
+
|
|
10
|
+
# `@ayepi/aws`
|
|
11
|
+
|
|
12
|
+
AWS backends for ayepi: an **S3-backed [`@ayepi/files`](./ayepi-files.md) `FileStore` +
|
|
13
|
+
`Presigner`** (`@ayepi/aws/s3`) and an **SQS-backed [`@ayepi/work`](./ayepi-work.md) `Queue`**
|
|
14
|
+
(`@ayepi/aws/sqs`) with transparent S3 offload of large message bodies. Every AWS call goes
|
|
15
|
+
through [`@ayepi/core`](./ayepi-core.md) `retry`, so SQS/S3 throttling (rate limits) is absorbed
|
|
16
|
+
under load; on exhaustion the error is reported to an `onError` hook and rethrown. Reach for it
|
|
17
|
+
when you want your ayepi work queue and file storage to ride on AWS managed services rather than
|
|
18
|
+
self-hosted Redis/disk.
|
|
19
|
+
|
|
20
|
+
The AWS SDK v3 clients are **optional peer dependencies** (`^3`) — you install and own them
|
|
21
|
+
(region, credentials, endpoint, lifecycle). The package never constructs a client; it talks to
|
|
22
|
+
the one you pass via `client.send(command)`. Install only what you use:
|
|
23
|
+
|
|
24
|
+
```sh
|
|
25
|
+
pnpm add @ayepi/aws @aws-sdk/client-s3 @aws-sdk/client-sqs @aws-sdk/lib-storage @aws-sdk/s3-request-presigner
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
The package ships three export subpaths:
|
|
29
|
+
|
|
30
|
+
| Subpath | Exports | AWS SDK packages used |
|
|
31
|
+
|---|---|---|
|
|
32
|
+
| `@ayepi/aws/s3` | `s3Files`, `S3FilesOptions` | `@aws-sdk/client-s3`, `@aws-sdk/lib-storage`, `@aws-sdk/s3-request-presigner` |
|
|
33
|
+
| `@ayepi/aws/sqs` | `sqsQueue`, `SqsQueueOptions`, `LargePayloadOptions` | `@aws-sdk/client-sqs` (+ an `@ayepi/files` store for offload) |
|
|
34
|
+
| `@ayepi/aws` (root) | `AwsClient`, `ResilientOptions`, `makeRun` | — (shared retry seam) |
|
|
35
|
+
|
|
36
|
+
## Public API
|
|
37
|
+
|
|
38
|
+
### `@ayepi/aws/s3` — `s3Files(opts)`
|
|
39
|
+
|
|
40
|
+
```ts
|
|
41
|
+
function s3Files(opts: S3FilesOptions): FileStore & Presigner;
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
Creates an S3-backed [`@ayepi/files`](./ayepi-files.md) `FileStore` that also implements
|
|
45
|
+
`Presigner`. Pass a configured `S3Client` and a `bucket`; `prefix` namespaces every key.
|
|
46
|
+
|
|
47
|
+
```ts
|
|
48
|
+
interface S3FilesOptions extends ResilientOptions {
|
|
49
|
+
/** A configured `@aws-sdk/client-s3` `S3Client`. */
|
|
50
|
+
readonly client: S3Client;
|
|
51
|
+
/** Target bucket. */
|
|
52
|
+
readonly bucket: string;
|
|
53
|
+
/** Key prefix prepended to every key (default `''`). */
|
|
54
|
+
readonly prefix?: string;
|
|
55
|
+
/** @internal Upload seam (default: `@aws-sdk/lib-storage` multipart `Upload`) — injectable for tests. */
|
|
56
|
+
readonly upload?: (key: string, body: FileBody, contentType?: string, metadata?: Record<string, string>) => Promise<void>;
|
|
57
|
+
/** @internal Presign seam (default: `@aws-sdk/s3-request-presigner`) — injectable for tests. */
|
|
58
|
+
readonly presign?: (kind: 'get' | 'put', key: string, expiresIn: number, contentType?: string) => Promise<string>;
|
|
59
|
+
}
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
- `client` — a concrete `@aws-sdk/client-s3` `S3Client`. (Unlike the SQS queue's structural
|
|
63
|
+
`AwsClient`, the store needs the concrete client because multipart `Upload` and `getSignedUrl`
|
|
64
|
+
bind to it.)
|
|
65
|
+
- `bucket` — the target S3 bucket.
|
|
66
|
+
- `prefix` (default `''`) — prepended to every key on write and stripped from keys returned by
|
|
67
|
+
`list`, so callers work in a clean namespace (e.g. `prefix: 'docs/'`, you `get('a.txt')`,
|
|
68
|
+
the object lives at `docs/a.txt`).
|
|
69
|
+
- `retry` / `onError` — inherited from `ResilientOptions` (see the root section). Every store
|
|
70
|
+
operation runs inside `retry`.
|
|
71
|
+
- `upload` / `presign` — **internal injectable seams**. They default to the real SDK glue
|
|
72
|
+
(lib-storage `Upload` multipart, `s3-request-presigner` `getSignedUrl`); you only override
|
|
73
|
+
them in tests or to customize the SDK call. The defaults are exercised against real S3 in the
|
|
74
|
+
integration test, not in unit tests.
|
|
75
|
+
|
|
76
|
+
The returned object is a `FileStore & Presigner` (the `@ayepi/files` contract):
|
|
77
|
+
|
|
78
|
+
```ts
|
|
79
|
+
interface FileStore {
|
|
80
|
+
put(key: string, body: FileBody, opts?: PutOptions): Promise<FileInfo>;
|
|
81
|
+
get(key: string): Promise<FileObject | undefined>;
|
|
82
|
+
head(key: string): Promise<FileInfo | undefined>;
|
|
83
|
+
delete(key: string): Promise<boolean>;
|
|
84
|
+
list(prefix: string, opts?: ListOptions): Promise<ListResult>;
|
|
85
|
+
}
|
|
86
|
+
interface Presigner {
|
|
87
|
+
presignDownload(key: string, opts?: PresignDownloadOptions): Promise<string>;
|
|
88
|
+
presignUpload(key: string, opts?: PresignUploadOptions): Promise<string>;
|
|
89
|
+
}
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
Behavior of each method:
|
|
93
|
+
|
|
94
|
+
- **`put(key, body, opts?)`** — streams `body` to S3 via the upload seam (multipart `Upload`,
|
|
95
|
+
passing `ContentType` and `Metadata`), then `head`s the key and returns the resulting
|
|
96
|
+
`FileInfo`. `body` is any `@ayepi/files` `FileBody` (`ReadableStream` / `Uint8Array` / `Blob`
|
|
97
|
+
/ `string`).
|
|
98
|
+
- **`get(key)`** — issues `GetObjectCommand`; returns a `FileObject` whose `info` is built from
|
|
99
|
+
`ContentLength` / `ContentType` / `ETag` / `LastModified` / `Metadata`, and whose body is lazy:
|
|
100
|
+
`stream()` → `transformToWebStream()`, `bytes()` → `transformToByteArray()`, `text()` →
|
|
101
|
+
`transformToString()`. **Read the body once** — pick one accessor. A missing key resolves to
|
|
102
|
+
`undefined`.
|
|
103
|
+
- **`head(key)`** — issues `HeadObjectCommand`; returns the `FileInfo` or `undefined` if missing.
|
|
104
|
+
- **`delete(key)`** — `head`s first to report prior existence, then issues `DeleteObjectCommand`;
|
|
105
|
+
resolves `true` if the object existed, `false` otherwise.
|
|
106
|
+
- **`list(prefix, opts?)`** — issues `ListObjectsV2Command` with `Prefix` = `ns + prefix`,
|
|
107
|
+
`MaxKeys` = `opts.limit`, `ContinuationToken` = `opts.cursor`. Returns `{ files, cursor }`
|
|
108
|
+
where `cursor` is the SDK's `NextContinuationToken` (absent when the listing is complete) and
|
|
109
|
+
each file's key has the store `prefix` **stripped**. List entries carry `size`, `etag`,
|
|
110
|
+
`modifiedAt` but **no `contentType`** (ListObjectsV2 doesn't return it).
|
|
111
|
+
- **`presignDownload(key, opts?)`** — a presigned `GET` URL (default expiry **900 s**).
|
|
112
|
+
- **`presignUpload(key, opts?)`** — a presigned `PUT` URL (default expiry **900 s**); `opts.contentType`
|
|
113
|
+
pins the `Content-Type` the upload must use.
|
|
114
|
+
|
|
115
|
+
**Not-found handling.** S3 surfaces a missing key differently per op (`NoSuchKey` on GET,
|
|
116
|
+
`NotFound` on HEAD). The store treats an error as not-found when its `name` is `NoSuchKey` or
|
|
117
|
+
`NotFound`, **or** when `$metadata.httpStatusCode` is `404` — those become `undefined`
|
|
118
|
+
(`get`/`head`) or `false` (`delete`). Any other error propagates (after retries).
|
|
119
|
+
|
|
120
|
+
### `@ayepi/aws/sqs` — `sqsQueue(opts)`
|
|
121
|
+
|
|
122
|
+
```ts
|
|
123
|
+
function sqsQueue(opts: SqsQueueOptions): Queue;
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
Creates an [`@ayepi/work`](./ayepi-work-ports.md) `Queue` over SQS. SQS's native
|
|
127
|
+
visibility-timeout model maps directly onto the work `Queue` lease contract.
|
|
128
|
+
|
|
129
|
+
```ts
|
|
130
|
+
interface SqsQueueOptions extends ResilientOptions {
|
|
131
|
+
/** A configured `@aws-sdk/client-sqs` `SQSClient`. */
|
|
132
|
+
readonly client: SQSClient;
|
|
133
|
+
/** The target queue URL. */
|
|
134
|
+
readonly queueUrl: string;
|
|
135
|
+
/** Long-poll seconds for `pop` (0–20, default 0). */
|
|
136
|
+
readonly waitTimeSeconds?: number;
|
|
137
|
+
/** Transparently offload large bodies to S3. */
|
|
138
|
+
readonly largePayload?: LargePayloadOptions;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
interface LargePayloadOptions {
|
|
142
|
+
/** The store oversized bodies are written to (e.g. `s3Files({...})`). */
|
|
143
|
+
readonly store: FileStore;
|
|
144
|
+
/** Offload bodies larger than this many bytes (default ~240 KB). */
|
|
145
|
+
readonly threshold?: number;
|
|
146
|
+
/** Key prefix for offloaded bodies (default `'sqs-payloads/'`). */
|
|
147
|
+
readonly prefix?: string;
|
|
148
|
+
}
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
- `client` — a configured `@aws-sdk/client-sqs` `SQSClient` (structurally an `AwsClient`).
|
|
152
|
+
- `queueUrl` — the target SQS queue URL.
|
|
153
|
+
- `waitTimeSeconds` (default `0`) — SQS long-poll seconds for `pop` (`0`–`20`). Set it (e.g. `10`)
|
|
154
|
+
to long-poll and reduce empty receives / API calls.
|
|
155
|
+
- `largePayload` — optional; see below.
|
|
156
|
+
- `retry` / `onError` — from `ResilientOptions`.
|
|
157
|
+
|
|
158
|
+
The returned [`Queue`](./ayepi-work-ports.md#queue--the-durable-work-log) maps onto SQS
|
|
159
|
+
(all durations in the contract are **milliseconds**; SQS wants seconds, so values are
|
|
160
|
+
`Math.ceil`-converted):
|
|
161
|
+
|
|
162
|
+
| `Queue` method | SQS call | Mapping |
|
|
163
|
+
|---|---|---|
|
|
164
|
+
| `push(body, { delay? })` | `SendMessage` | `MessageBody` = body (or pointer); `DelaySeconds` = `ceil(delay/1000)`, **clamped to ≤ 900 s** |
|
|
165
|
+
| `pop(max, visibility)` | `ReceiveMessage` | `MaxNumberOfMessages` = `min(max, 10)`; `VisibilityTimeout` = `ceil(visibility/1000)`, **clamped to ≤ 43200 s**; `WaitTimeSeconds` = `waitTimeSeconds`; reads `ApproximateReceiveCount` |
|
|
166
|
+
| `heartbeat(pulled, visibility)` | `ChangeMessageVisibility` | extends the lease to `ceil(visibility/1000)` s, **clamped to ≤ 43200 s** |
|
|
167
|
+
| `ack(pulled)` | `DeleteMessage` | permanently removes the message (+ deletes the offloaded S3 body) |
|
|
168
|
+
| `fail(pulled, delay?)` | `ChangeMessageVisibility` | returns the message early: `VisibilityTimeout` = `ceil((delay ?? 0)/1000)` s, **clamped to ≤ 43200 s** |
|
|
169
|
+
|
|
170
|
+
**SQS range clamping (far-future scheduling).** SQS rejects out-of-range values, so the queue
|
|
171
|
+
clamps each duration into SQS's allowed range: `DelaySeconds` to **0–900 s** (15 min) on `push`,
|
|
172
|
+
and `VisibilityTimeout` to **0–43200 s** (12 h) on `pop`/`heartbeat`/`fail`. This is what lets
|
|
173
|
+
the `@ayepi/work` engine schedule items **arbitrarily far** in the future on SQS: a long `delay`
|
|
174
|
+
or `startAt` is clamped to the cap rather than erroring, so a far-future item is delivered early,
|
|
175
|
+
the engine sees it isn't due yet and **puts it back** (via `fail` with the remaining delay, again
|
|
176
|
+
clamped), and it **bounces** — every ≤ 900 s after a `push` and every ≤ 12 h after a re-defer —
|
|
177
|
+
until its scheduled time finally arrives. See the engine's
|
|
178
|
+
[early-arrival re-defer](./ayepi-work-ports.md#early-arrival-re-defer-far-future-scheduling).
|
|
179
|
+
The cost is **polling**: a far-future item is received and re-deferred on that cadence the whole
|
|
180
|
+
time it waits (each bounce is a `ReceiveMessage` + `ChangeMessageVisibility`), so prefer modest
|
|
181
|
+
horizons or keep distant schedules sparse.
|
|
182
|
+
|
|
183
|
+
- **`pop`** returns at most **10** messages per call (SQS's `ReceiveMessage` cap), regardless of
|
|
184
|
+
`max`. Each `PulledWork.attempt` comes from the message's `ApproximateReceiveCount` (delivery
|
|
185
|
+
count, starting at 1); `handle` is an opaque `{ receiptHandle, s3Key? }` you round-trip to
|
|
186
|
+
`heartbeat`/`ack`/`fail`.
|
|
187
|
+
- **Dead-lettering** is the queue's own SQS **redrive policy** — configured on the queue (or its
|
|
188
|
+
DLQ) in AWS, not here. After `maxReceiveCount` failed deliveries SQS moves the message to the
|
|
189
|
+
DLQ natively. (`sqsQueue` does not implement the optional `Queue.deadLetter` hook.)
|
|
190
|
+
|
|
191
|
+
**`largePayload` — S3 offload (SQS caps a message at 256 KB).** When configured, on `push` a
|
|
192
|
+
body whose length exceeds `threshold` (default `240 * 1024`, comfortably under the 256 KB cap) is
|
|
193
|
+
written to `store` under `` `${prefix}${randomUUID()}` `` (prefix default `'sqs-payloads/'`) with
|
|
194
|
+
`contentType: 'application/json'`, and the SQS message carries a small JSON pointer
|
|
195
|
+
`{ "__ayepiS3__": "<key>" }` instead of the body. On `pop`, a message body that parses to such a
|
|
196
|
+
pointer is detected and the offloaded body is fetched from `store` and inlined back into
|
|
197
|
+
`PulledWork.body`; the `s3Key` rides along in the handle. On `ack`, the offloaded S3 object is
|
|
198
|
+
deleted. Notes:
|
|
199
|
+
|
|
200
|
+
- With `largePayload` **off**, a body that happens to look like a pointer is passed through
|
|
201
|
+
untouched (no inlining). With it **on**, a non-JSON or non-pointer body is treated as a plain
|
|
202
|
+
message (passthrough).
|
|
203
|
+
- A vanished offloaded body (key missing in `store`) inlines as an empty string `''` rather than
|
|
204
|
+
throwing.
|
|
205
|
+
- `store` is any [`@ayepi/files`](./ayepi-files.md) `FileStore` — typically an `s3Files({...})`,
|
|
206
|
+
but a filesystem store or your own works too.
|
|
207
|
+
|
|
208
|
+
### `@ayepi/aws` (root) — the retry seam
|
|
209
|
+
|
|
210
|
+
The root subpath exports the shared resilience machinery both backends build on. You rarely call
|
|
211
|
+
these directly; they're documented so you understand `retry`/`onError` and can pass a structural
|
|
212
|
+
client.
|
|
213
|
+
|
|
214
|
+
#### `AwsClient`
|
|
215
|
+
|
|
216
|
+
```ts
|
|
217
|
+
interface AwsClient {
|
|
218
|
+
send(command: unknown): Promise<unknown>;
|
|
219
|
+
}
|
|
220
|
+
```
|
|
221
|
+
|
|
222
|
+
The minimal AWS SDK v3 client surface used internally — just `send(command)`. The real `S3Client`
|
|
223
|
+
/ `SQSClient` satisfy it structurally; a test can pass `{ send: vi.fn() }`. (Presigning and
|
|
224
|
+
multipart upload need the *concrete* `S3Client`, which is why `s3Files` takes `S3Client` directly
|
|
225
|
+
rather than `AwsClient`.)
|
|
226
|
+
|
|
227
|
+
#### `ResilientOptions`
|
|
228
|
+
|
|
229
|
+
```ts
|
|
230
|
+
interface ResilientOptions {
|
|
231
|
+
/** Retry policy for each AWS call (core `retry` — `attempts`/`base`/`factor`/`max`/`jitter`/…). Defaults absorb throttling. */
|
|
232
|
+
readonly retry?: Omit<RetryOptions, 'errorResult'>;
|
|
233
|
+
/** Notified when a call fails after exhausting retries (the error then propagates). Off by default; must not throw. */
|
|
234
|
+
readonly onError?: (err: unknown) => void;
|
|
235
|
+
}
|
|
236
|
+
```
|
|
237
|
+
|
|
238
|
+
Shared by `S3FilesOptions` and `SqsQueueOptions`.
|
|
239
|
+
|
|
240
|
+
- **`retry`** — the `@ayepi/core` `RetryOptions` (minus `errorResult`, which the wrapper owns):
|
|
241
|
+
`attempts`, `base`, `factor`, `max`, `jitter`, `sleep`, etc. The defaults are tuned to absorb
|
|
242
|
+
throttling; tune `attempts` up for aggressive rate limits.
|
|
243
|
+
- **`onError`** — called **once** when a call finally gives up (after all retries), then the error
|
|
244
|
+
propagates. Off by default. It must not throw — if it does, the throw is swallowed so it can't
|
|
245
|
+
mask the original AWS error.
|
|
246
|
+
|
|
247
|
+
#### `makeRun(opts)`
|
|
248
|
+
|
|
249
|
+
```ts
|
|
250
|
+
function makeRun(opts: ResilientOptions): <T>(fn: () => Promise<T>) => Promise<T>;
|
|
251
|
+
```
|
|
252
|
+
|
|
253
|
+
Builds the retry-wrapping runner each backend uses: it runs `fn` under `@ayepi/core` `retry` with
|
|
254
|
+
your `retry` options, and on final failure reports the error to `onError` (guarded) before
|
|
255
|
+
rethrowing. Both `s3Files` and `sqsQueue` wrap **every** AWS `send` in this runner. You'd only
|
|
256
|
+
call `makeRun` yourself if you were building a third AWS backend on the same seam.
|
|
257
|
+
|
|
258
|
+
## Examples
|
|
259
|
+
|
|
260
|
+
### S3 file store — round-trip + presign
|
|
261
|
+
|
|
262
|
+
```ts
|
|
263
|
+
import { S3Client } from '@aws-sdk/client-s3';
|
|
264
|
+
import { s3Files } from '@ayepi/aws/s3';
|
|
265
|
+
import { toStream, collect } from '@ayepi/files';
|
|
266
|
+
|
|
267
|
+
const files = s3Files({
|
|
268
|
+
client: new S3Client({ region: 'us-east-1' }),
|
|
269
|
+
bucket: 'my-bucket',
|
|
270
|
+
prefix: 'docs/', // every key namespaced under docs/
|
|
271
|
+
});
|
|
272
|
+
|
|
273
|
+
// write a stream with a content type + metadata
|
|
274
|
+
await files.put('a.txt', toStream('hello s3'), { contentType: 'text/plain', metadata: { owner: 'ada' } });
|
|
275
|
+
|
|
276
|
+
// read it back (body read once)
|
|
277
|
+
const obj = (await files.get('a.txt'))!;
|
|
278
|
+
console.log(obj.info.size); // 8
|
|
279
|
+
console.log(new TextDecoder().decode(await collect(obj.stream())));
|
|
280
|
+
|
|
281
|
+
// list by prefix — returned keys have the store prefix stripped
|
|
282
|
+
const { files: page, cursor } = await files.list('a', { limit: 50 });
|
|
283
|
+
console.log(page.map((f) => f.key)); // ['a.txt']
|
|
284
|
+
|
|
285
|
+
// mint a presigned download URL (default 900 s; override per call)
|
|
286
|
+
const url = await files.presignDownload('a.txt', { expiresIn: 300 });
|
|
287
|
+
|
|
288
|
+
// delete reports prior existence
|
|
289
|
+
console.log(await files.delete('a.txt')); // true
|
|
290
|
+
console.log(await files.get('a.txt')); // undefined
|
|
291
|
+
```
|
|
292
|
+
|
|
293
|
+
### SQS queue wired into `createWork`
|
|
294
|
+
|
|
295
|
+
```ts
|
|
296
|
+
import { SQSClient } from '@aws-sdk/client-sqs';
|
|
297
|
+
import { sqsQueue } from '@ayepi/aws/sqs';
|
|
298
|
+
import { createWork } from '@ayepi/work';
|
|
299
|
+
|
|
300
|
+
const queue = sqsQueue({
|
|
301
|
+
client: new SQSClient({ region: 'us-east-1' }),
|
|
302
|
+
queueUrl: process.env.SQS_URL!,
|
|
303
|
+
waitTimeSeconds: 10, // long-poll
|
|
304
|
+
});
|
|
305
|
+
|
|
306
|
+
// supply pubsub + store from another backend (e.g. @ayepi/redis) to go fully distributed
|
|
307
|
+
const w = createWork({ queue, pubsub, store, work: [/* ...defineWork() */] as const });
|
|
308
|
+
```
|
|
309
|
+
|
|
310
|
+
> A fully distributed `createWork` needs all three ports (`queue`, `pubsub`, `store`). `@ayepi/aws`
|
|
311
|
+
> supplies the **queue**; pair it with a `PubSub`/`Store` (e.g. from
|
|
312
|
+
> [`@ayepi/redis`](./ayepi-redis.md)). See [ayepi-work-ports.md](./ayepi-work-ports.md).
|
|
313
|
+
|
|
314
|
+
### Large-payload offload using `s3Files` as the store
|
|
315
|
+
|
|
316
|
+
```ts
|
|
317
|
+
import { S3Client } from '@aws-sdk/client-s3';
|
|
318
|
+
import { SQSClient } from '@aws-sdk/client-sqs';
|
|
319
|
+
import { s3Files } from '@ayepi/aws/s3';
|
|
320
|
+
import { sqsQueue } from '@ayepi/aws/sqs';
|
|
321
|
+
|
|
322
|
+
const s3 = new S3Client({ region: 'us-east-1' });
|
|
323
|
+
const sqs = new SQSClient({ region: 'us-east-1' });
|
|
324
|
+
|
|
325
|
+
const queue = sqsQueue({
|
|
326
|
+
client: sqs,
|
|
327
|
+
queueUrl: process.env.SQS_URL!,
|
|
328
|
+
largePayload: {
|
|
329
|
+
store: s3Files({ client: s3, bucket: 'work-payloads' }),
|
|
330
|
+
threshold: 240 * 1024, // offload bodies over ~240 KB (default)
|
|
331
|
+
prefix: 'sqs-payloads/', // S3 key prefix for offloaded bodies (default)
|
|
332
|
+
},
|
|
333
|
+
});
|
|
334
|
+
|
|
335
|
+
await queue.push('small'); // sent inline
|
|
336
|
+
await queue.push('x'.repeat(300_000)); // > threshold → written to S3, message carries the pointer
|
|
337
|
+
|
|
338
|
+
const items = await queue.pop(10, 30_000); // 30 s lease (ms)
|
|
339
|
+
for (const it of items) {
|
|
340
|
+
console.log(it.body); // the large body is inlined back from S3 transparently
|
|
341
|
+
await it.attempt; // delivery count from ApproximateReceiveCount
|
|
342
|
+
await queue.ack(it); // DeleteMessage + delete the offloaded S3 object
|
|
343
|
+
}
|
|
344
|
+
```
|
|
345
|
+
|
|
346
|
+
### Configuring retry for aggressive throttling
|
|
347
|
+
|
|
348
|
+
```ts
|
|
349
|
+
import { s3Files } from '@ayepi/aws/s3';
|
|
350
|
+
|
|
351
|
+
const files = s3Files({
|
|
352
|
+
client: s3,
|
|
353
|
+
bucket: 'my-bucket',
|
|
354
|
+
retry: { attempts: 10 }, // more attempts before giving up (core RetryOptions)
|
|
355
|
+
onError: (err) => console.error('[s3] gave up after retries:', err),
|
|
356
|
+
});
|
|
357
|
+
```
|
|
358
|
+
|
|
359
|
+
The same `retry` / `onError` options apply to `sqsQueue`. On final failure `onError` fires once
|
|
360
|
+
(it must not throw) and the AWS error is rethrown.
|
|
361
|
+
|
|
362
|
+
## How it works under the hood
|
|
363
|
+
|
|
364
|
+
- **One retry seam, two backends.** Both `s3Files` and `sqsQueue` build a runner via `makeRun`
|
|
365
|
+
and wrap **every** `client.send(...)` in `@ayepi/core` `retry`. A throttled reply is retried per
|
|
366
|
+
your `retry` policy; only after attempts are exhausted does the error reach `onError` and
|
|
367
|
+
propagate. `onError` is guarded so a throwing reporter can't mask the AWS error.
|
|
368
|
+
- **S3 store is plain command dispatch.** `get`/`head`/`delete`/`list` issue
|
|
369
|
+
`GetObject`/`HeadObject`/`DeleteObject`/`ListObjectsV2` and map the response fields onto
|
|
370
|
+
`@ayepi/files` shapes; `put` uses lib-storage's multipart `Upload` then re-`head`s for the
|
|
371
|
+
authoritative `FileInfo`; presigning uses `s3-request-presigner`'s `getSignedUrl`. The upload
|
|
372
|
+
and presign steps are injectable seams (`opts.upload` / `opts.presign`) so unit tests can run
|
|
373
|
+
without real AWS; the real SDK glue is covered by the LocalStack integration test.
|
|
374
|
+
- **SQS queue is the visibility-timeout lease.** `pop`'s `ReceiveMessage` *is* the lease (hidden
|
|
375
|
+
for `VisibilityTimeout`); `heartbeat` extends it with `ChangeMessageVisibility`; `ack`
|
|
376
|
+
`DeleteMessage`s; `fail` shortens visibility so the message reappears after `delay`. A worker
|
|
377
|
+
that dies lets visibility lapse and SQS redelivers with a bumped `ApproximateReceiveCount` →
|
|
378
|
+
`attempt`. This is exactly the work `Queue` contract, served natively by SQS.
|
|
379
|
+
- **Large-payload offload is a pointer swap.** Over-threshold bodies are stored in the provided
|
|
380
|
+
`FileStore` and replaced in the message with `{ "__ayepiS3__": key }`. `pop` detects the
|
|
381
|
+
pointer (JSON-parse + marker check), fetches and inlines the body; `ack` deletes the S3 object.
|
|
382
|
+
Detection is best-effort: a body that doesn't parse to the marker shape is treated as a plain
|
|
383
|
+
message.
|
|
384
|
+
- **Dead-lettering is native.** There's no app-level DLQ here — SQS's redrive policy on the queue
|
|
385
|
+
moves a message to its configured dead-letter queue after `maxReceiveCount` deliveries.
|
|
386
|
+
|
|
387
|
+
## Gotchas / constraints
|
|
388
|
+
|
|
389
|
+
- **SQS's 256 KB message limit & why offload exists.** SQS rejects messages over 256 KB. Without
|
|
390
|
+
`largePayload`, a large work envelope fails to `push`. Enable `largePayload` (threshold default
|
|
391
|
+
~240 KB, safely under the cap) to transparently offload to S3. The offloaded objects accumulate
|
|
392
|
+
in your bucket — they're deleted on `ack`, but a message that never acks (e.g. ends up in the
|
|
393
|
+
DLQ) leaves its S3 object behind; consider a bucket lifecycle rule on the offload prefix.
|
|
394
|
+
- **Visibility timeout vs heartbeat.** The `visibility` you pass to `pop` is the lease length; the
|
|
395
|
+
engine must `heartbeat` (extend visibility) before it lapses or SQS will redeliver the still-running
|
|
396
|
+
item to another worker (at-least-once). Pick a `visibility` comfortably longer than your
|
|
397
|
+
heartbeat interval. Durations in the `Queue` contract are **milliseconds**; SQS uses seconds
|
|
398
|
+
(the queue `Math.ceil`-converts), and SQS's own max visibility is 12 hours.
|
|
399
|
+
- **`pop` returns ≤ 10 per call.** SQS caps `ReceiveMessage` at 10 messages regardless of the `max`
|
|
400
|
+
you request. The engine polls in a loop, so this is fine — just don't expect one `pop` to drain
|
|
401
|
+
a large backlog.
|
|
402
|
+
- **Far-future schedules bounce (and cost polling).** SQS caps a single `DelaySeconds` at 15 min and
|
|
403
|
+
a visibility at 12 h, so the queue clamps both. A far-future `runAt` / `WorkDelayError` deferral is
|
|
404
|
+
therefore honored by **re-deferral**: the item is received early, the engine re-checks `startAt` and
|
|
405
|
+
puts it back, and it bounces every ≤ 900 s (after `push`) / ≤ 12 h (after a re-defer) until due. It
|
|
406
|
+
fires at the right time, but each bounce is a `ReceiveMessage` + `ChangeMessageVisibility` — so a
|
|
407
|
+
large number of distant-future items incurs ongoing polling. See
|
|
408
|
+
[ayepi-work-ports.md → early-arrival re-defer](./ayepi-work-ports.md#early-arrival-re-defer-far-future-scheduling).
|
|
409
|
+
- **You own the SDK clients & their lifecycle.** `@ayepi/aws` never constructs or closes a client.
|
|
410
|
+
Configure region/credentials/endpoint yourself and `destroy()` the clients on shutdown. The AWS
|
|
411
|
+
SDK packages are optional peer deps — install the ones you use, or the import will fail.
|
|
412
|
+
- **DLQ is configured on the queue, not here.** Set the redrive policy (`maxReceiveCount` + DLQ
|
|
413
|
+
ARN) on the SQS queue in AWS. `sqsQueue` doesn't implement `Queue.deadLetter`; failures redeliver
|
|
414
|
+
until SQS dead-letters them natively.
|
|
415
|
+
- **Eventual consistency.** S3 is read-after-write consistent for new objects, but `list` and
|
|
416
|
+
cross-region/replication views can lag; an offloaded body written then immediately popped on a
|
|
417
|
+
different host is generally fine, but don't assume strong global ordering. A `get` on a key that
|
|
418
|
+
S3 reports as missing returns `undefined`, not an error.
|
|
419
|
+
- **Read an S3 `FileObject`'s body once.** `stream()` / `bytes()` / `text()` consume the same
|
|
420
|
+
underlying SDK stream — call exactly one. To read again, `get` again.
|
|
421
|
+
- **`list` entries have no `contentType`.** ListObjectsV2 doesn't return it; `head`/`get` do.
|
|
422
|
+
- **Silent by default.** Without `onError`, a call that exhausts retries throws but reports nothing
|
|
423
|
+
extra. Pass `onError` in production to surface give-ups.
|
|
424
|
+
|
|
425
|
+
## See also
|
|
426
|
+
|
|
427
|
+
- [ayepi-files.md](./ayepi-files.md) — the `FileStore` / `Presigner` contract `s3Files`
|
|
428
|
+
implements, plus stream helpers (`toStream`, `collect`).
|
|
429
|
+
- [ayepi-work.md](./ayepi-work.md) — `createWork`, `defineWork`, and the work engine `sqsQueue`
|
|
430
|
+
plugs into as the `Queue` port.
|
|
431
|
+
- [ayepi-work-ports.md](./ayepi-work-ports.md) — the `Queue` / `Store` / `PubSub` port contracts
|
|
432
|
+
and the visibility-timeout lease model `sqsQueue` maps SQS onto.
|
|
433
|
+
- [ayepi-redis.md](./ayepi-redis.md) — a `PubSub`/`Store` (and `Broker`) backend to pair with the
|
|
434
|
+
SQS queue for a fully distributed `createWork`.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ayepi/aws",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2.0",
|
|
4
4
|
"description": "AWS backends for ayepi — an SQS @ayepi/work queue (large payloads offloaded to S3) and an S3 @ayepi/files store, all retry-wrapped for throttle resilience",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"publishConfig": {
|
|
@@ -18,7 +18,8 @@
|
|
|
18
18
|
"type": "module",
|
|
19
19
|
"sideEffects": false,
|
|
20
20
|
"files": [
|
|
21
|
-
"dist"
|
|
21
|
+
"dist",
|
|
22
|
+
"ayepi-*.md"
|
|
22
23
|
],
|
|
23
24
|
"exports": {
|
|
24
25
|
".": {
|
|
@@ -61,9 +62,9 @@
|
|
|
61
62
|
"@aws-sdk/client-sqs": "^3",
|
|
62
63
|
"@aws-sdk/lib-storage": "^3",
|
|
63
64
|
"@aws-sdk/s3-request-presigner": "^3",
|
|
64
|
-
"@ayepi/core": "^0.
|
|
65
|
-
"@ayepi/
|
|
66
|
-
"@ayepi/
|
|
65
|
+
"@ayepi/core": "^0.2.0",
|
|
66
|
+
"@ayepi/work": "^0.2.0",
|
|
67
|
+
"@ayepi/files": "^0.2.0"
|
|
67
68
|
},
|
|
68
69
|
"peerDependenciesMeta": {
|
|
69
70
|
"@ayepi/work": {
|
|
@@ -96,9 +97,9 @@
|
|
|
96
97
|
"tsdown": "^0.12.0",
|
|
97
98
|
"vitest": "^2.1.8",
|
|
98
99
|
"zod": "^4.4.3",
|
|
99
|
-
"@ayepi/
|
|
100
|
-
"@ayepi/
|
|
101
|
-
"@ayepi/
|
|
100
|
+
"@ayepi/core": "0.2.0",
|
|
101
|
+
"@ayepi/work": "0.2.0",
|
|
102
|
+
"@ayepi/files": "0.2.0"
|
|
102
103
|
},
|
|
103
104
|
"keywords": [
|
|
104
105
|
"ayepi",
|