@vouchington/aws 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +57 -3
- package/dist/errors.mjs +6 -0
- package/dist/firehose-batcher.d.mts +25 -0
- package/dist/firehose-batcher.mjs +94 -0
- package/dist/firehose-records.d.mts +6 -0
- package/dist/firehose-records.mjs +42 -0
- package/dist/firehose.d.mts +2 -5
- package/dist/firehose.mjs +2 -42
- package/dist/testing/emulator-docker.d.mts +22 -0
- package/dist/testing/emulator-docker.mjs +102 -0
- package/dist/testing/emulator-package.d.mts +23 -0
- package/dist/testing/emulator-package.mjs +66 -0
- package/dist/testing/emulator.d.mts +28 -0
- package/dist/testing/emulator.mjs +108 -0
- package/dist/testing/firehose.d.mts +18 -0
- package/dist/testing/firehose.mjs +63 -0
- package/dist/testing/index.d.mts +6 -0
- package/dist/testing/index.mjs +6 -0
- package/dist/testing/s3.d.mts +39 -0
- package/dist/testing/s3.mjs +102 -0
- package/dist/testing/sqs.d.mts +6 -0
- package/dist/testing/sqs.mjs +24 -0
- package/package.json +26 -11
package/README.md
CHANGED
|
@@ -15,12 +15,31 @@ Each client package is an optional peer; install only the services you use. S3 s
|
|
|
15
15
|
`chunkSendEntries`, `chunkDeleteEntries`, `buildSendMessageBatchCommand`,
|
|
16
16
|
`buildDeleteMessageBatchCommand`, `buildReceiveMessageCommand`, `classifyBatchFailures`.
|
|
17
17
|
- `/firehose`: `encodeJsonRecord`, `chunkFirehoseRecords`, `buildPutRecordBatchCommand`,
|
|
18
|
-
`getFailedFirehoseRecords
|
|
18
|
+
`getFailedFirehoseRecords`, and `createFirehoseBatcher({ putRecordBatch, onError })`, which
|
|
19
|
+
buffers records per delivery stream (`write`, `writeJson`), flushes on 500 records, 4 MiB, or a
|
|
20
|
+
timer (`flushIntervalMs`, default 1000), and retries records Firehose rejects once
|
|
21
|
+
(`retryDelayMs`, default 100) before reporting them to `onError`. Stream names and the send
|
|
22
|
+
function are inputs; `flush()` drains everything and never rejects for send failures.
|
|
19
23
|
- `/cloudwatch`: `chunkMetricData`, `buildPutMetricDataCommand`.
|
|
20
24
|
- `/request-handler`: `createAwsRequestHandler` (default connection and socket deadlines,
|
|
21
25
|
timeout constants, `AWS_DUALSTACK_CLIENT_CONFIG`.
|
|
22
|
-
- `/errors`: `isAwsThrottlingError` and `getAwsErrorStatus`.
|
|
23
|
-
|
|
26
|
+
- `/errors`: `isAwsThrottlingError` and `getAwsErrorStatus`. `isAwsThrottlingError` also matches
|
|
27
|
+
SES's HTTP 400 "Maximum sending rate exceeded" message. Pass `getAwsErrorStatus` as `getStatus`
|
|
28
|
+
to `@vouchington/queue-errors` helpers; this package does not duplicate them. `wrapHttpForRetry`
|
|
29
|
+
treats that SES 400 as terminal, so check `isAwsThrottlingError` first and rethrow a match
|
|
30
|
+
unchanged to keep the queue's normal retry policy:
|
|
31
|
+
|
|
32
|
+
```ts
|
|
33
|
+
import { getAwsErrorStatus, isAwsThrottlingError } from '@vouchington/aws/errors'
|
|
34
|
+
import { wrapHttpForRetry } from '@vouchington/queue-errors'
|
|
35
|
+
|
|
36
|
+
await sendEmail().catch((error: unknown) => {
|
|
37
|
+
if (isAwsThrottlingError(error)) throw error
|
|
38
|
+
wrapHttpForRetry(error, { getStatus: getAwsErrorStatus })
|
|
39
|
+
})
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
- `/testing`: AWS Emulator harness for Vitest or any test runner; see below.
|
|
24
43
|
|
|
25
44
|
## Example
|
|
26
45
|
|
|
@@ -36,3 +55,38 @@ const ses = new SESClient({
|
|
|
36
55
|
})
|
|
37
56
|
await ses.send(buildEmailCommand({ source, to, subject, text }))
|
|
38
57
|
```
|
|
58
|
+
|
|
59
|
+
## Testing harness (`@vouchington/aws/testing`)
|
|
60
|
+
|
|
61
|
+
Starts the AWS Emulator from a pinned, digest-verified package and
|
|
62
|
+
provides seeding and inspection helpers. It runs natively on Linux and in a Docker container
|
|
63
|
+
elsewhere. It imports the S3, IAM, SQS and Firehose client packages, so install all four to use
|
|
64
|
+
this subpath; they stay optional peers, so consumers of the other subpaths never need S3 or IAM. Image tags, version pins, bucket, queue and stream
|
|
65
|
+
names, cache paths, and environment variable names are all inputs.
|
|
66
|
+
|
|
67
|
+
```ts
|
|
68
|
+
import {
|
|
69
|
+
startAwsEmulator,
|
|
70
|
+
trustAwsEmulatorCertificate,
|
|
71
|
+
provisionFirehoseStreams,
|
|
72
|
+
} from '@vouchington/aws/testing'
|
|
73
|
+
|
|
74
|
+
const emulator = await startAwsEmulator({
|
|
75
|
+
package: { pin: { version, sha256: { amd64, arm64 } }, cacheDir: '.cache/aws-emulator' },
|
|
76
|
+
dockerImage: 'node:24-trixie-slim@sha256:...', // runs the Linux binary on macOS
|
|
77
|
+
})
|
|
78
|
+
trustAwsEmulatorCertificate(emulator.caPem)
|
|
79
|
+
await provisionFirehoseStreams(emulator, { streamNames: ['events-test'] })
|
|
80
|
+
// ... run tests against emulator.endpoint with emulator.accessKeyId/secretAccessKey ...
|
|
81
|
+
await emulator.stop()
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
- `startAwsEmulator`, `trustAwsEmulatorCertificate`, `ensureAwsEmulatorBinary`, `debArch`,
|
|
85
|
+
`startAwsEmulatorInDocker`, `readAnnouncedPort`: lifecycle (`trustAwsEmulatorCertificate` needs Node 24.5 or newer). The download `fetch` is injectable
|
|
86
|
+
for proxies; the caller records the version and SHA-256 pins after verifying AWS's signature.
|
|
87
|
+
- `createTestS3Bucket`, `uniqueS3Prefix`, `putS3Object`, `readS3ObjectText`, `readS3ObjectBytes`,
|
|
88
|
+
`s3ObjectExists`, `listS3Keys`, `listS3Versions`, `deleteS3Object`, `sendToPresignedS3Url`:
|
|
89
|
+
helpers that take the test's own `S3Client`.
|
|
90
|
+
- `createTestSqsQueue`, `sendTestSqsMessage`, `countTestSqsMessages`: helpers that take the test's
|
|
91
|
+
own `SQSClient` (build it with `useQueueUrlAsEndpoint: false`).
|
|
92
|
+
- `provisionFirehoseStreams`: bucket, delivery role, and one ACTIVE DirectPut stream per name.
|
package/dist/errors.mjs
CHANGED
|
@@ -14,6 +14,8 @@ const THROTTLING_NAMES = new Set([
|
|
|
14
14
|
'ThrottlingException',
|
|
15
15
|
'TooManyRequestsException',
|
|
16
16
|
]);
|
|
17
|
+
// SES reports a sending-rate throttle as HTTP 400, sometimes with only this message to identify it.
|
|
18
|
+
const SES_SENDING_RATE_EXCEEDED = /maximum sending rate exceeded/i;
|
|
17
19
|
function asShape(error) {
|
|
18
20
|
return typeof error === 'object' && error !== null ? error : undefined;
|
|
19
21
|
}
|
|
@@ -23,6 +25,8 @@ export function getAwsErrorStatus(error) {
|
|
|
23
25
|
return typeof status === 'number' ? status : undefined;
|
|
24
26
|
}
|
|
25
27
|
// Detects rate-limit style failures across SES, SQS, Firehose, and CloudWatch error shapes.
|
|
28
|
+
// Returns a boolean, not a type guard: every Error fits the optional fields it checks, so a guard
|
|
29
|
+
// would narrow a non-throttling Error to `never`.
|
|
26
30
|
export function isAwsThrottlingError(error) {
|
|
27
31
|
const shape = asShape(error);
|
|
28
32
|
if (!shape)
|
|
@@ -31,5 +35,7 @@ export function isAwsThrottlingError(error) {
|
|
|
31
35
|
return true;
|
|
32
36
|
if (getAwsErrorStatus(error) === 429)
|
|
33
37
|
return true;
|
|
38
|
+
if (typeof shape.message === 'string' && SES_SENDING_RATE_EXCEEDED.test(shape.message))
|
|
39
|
+
return true;
|
|
34
40
|
return [shape.name, shape.Code, shape.code].some((value) => typeof value === 'string' && THROTTLING_NAMES.has(value));
|
|
35
41
|
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type { PutRecordBatchOutput } from '@aws-sdk/client-firehose';
|
|
2
|
+
export interface FirehoseBatcherOptions {
|
|
3
|
+
/** Sends one PutRecordBatch request; the caller owns the client, credentials, and region. */
|
|
4
|
+
putRecordBatch: (input: {
|
|
5
|
+
DeliveryStreamName: string;
|
|
6
|
+
Records: {
|
|
7
|
+
Data: Uint8Array;
|
|
8
|
+
}[];
|
|
9
|
+
}) => Promise<Partial<Pick<PutRecordBatchOutput, 'FailedPutCount' | 'RequestResponses'>>>;
|
|
10
|
+
/** Receives oversize drops, send failures, records dropped after the retry, and timer flush errors. */
|
|
11
|
+
onError: (error: Error) => void;
|
|
12
|
+
/** Delay before buffered records are flushed. Defaults to 1000. */
|
|
13
|
+
flushIntervalMs?: number;
|
|
14
|
+
/** Delay before the single retry of records Firehose rejected. Defaults to 100. */
|
|
15
|
+
retryDelayMs?: number;
|
|
16
|
+
}
|
|
17
|
+
export interface FirehoseBatcher {
|
|
18
|
+
/** Buffers one record. Oversize records are reported to onError and dropped. */
|
|
19
|
+
write: (streamName: string, data: Uint8Array) => void;
|
|
20
|
+
/** Serializes a value as a delimiter-terminated JSON record and buffers it. */
|
|
21
|
+
writeJson: (streamName: string, value: unknown, delimiter?: string) => void;
|
|
22
|
+
/** Sends everything buffered, including records written while the flush runs. */
|
|
23
|
+
flush: () => Promise<void>;
|
|
24
|
+
}
|
|
25
|
+
export declare function createFirehoseBatcher(options: FirehoseBatcherOptions): FirehoseBatcher;
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
import { chunkFirehoseRecords, encodeJsonRecord, FIREHOSE_MAX_RECORD_BYTES, getFailedFirehoseRecords, } from './firehose-records.mjs';
|
|
2
|
+
const FIREHOSE_FLUSH_RECORDS = 500;
|
|
3
|
+
const FIREHOSE_FLUSH_BYTES = 4 * 1024 * 1024;
|
|
4
|
+
const toError = (value) => value instanceof Error ? value : new Error(String(value));
|
|
5
|
+
const pause = (ms) => ms > 0 ? new Promise((resolve) => setTimeout(resolve, ms)) : Promise.resolve();
|
|
6
|
+
// Buffers records per delivery stream and flushes on count, size, or time. Records Firehose
|
|
7
|
+
// rejects are retried once; send errors never reject flush(), they go to onError.
|
|
8
|
+
export function createFirehoseBatcher(options) {
|
|
9
|
+
const { putRecordBatch, onError } = options;
|
|
10
|
+
const flushIntervalMs = options.flushIntervalMs ?? 1000;
|
|
11
|
+
const retryDelayMs = options.retryDelayMs ?? 100;
|
|
12
|
+
let buffer = [];
|
|
13
|
+
let bufferedBytes = 0;
|
|
14
|
+
let timer;
|
|
15
|
+
let active;
|
|
16
|
+
function clearTimer() {
|
|
17
|
+
if (timer !== undefined)
|
|
18
|
+
clearTimeout(timer);
|
|
19
|
+
timer = undefined;
|
|
20
|
+
}
|
|
21
|
+
async function sendOnce(streamName, records) {
|
|
22
|
+
try {
|
|
23
|
+
const output = await putRecordBatch({
|
|
24
|
+
DeliveryStreamName: streamName,
|
|
25
|
+
Records: records.map((Data) => ({ Data })),
|
|
26
|
+
});
|
|
27
|
+
return getFailedFirehoseRecords(records, output);
|
|
28
|
+
}
|
|
29
|
+
catch (error) {
|
|
30
|
+
onError(toError(error));
|
|
31
|
+
return records;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
async function sendWithRetry(streamName, records) {
|
|
35
|
+
const failed = await sendOnce(streamName, records);
|
|
36
|
+
if (failed.length === 0)
|
|
37
|
+
return;
|
|
38
|
+
await pause(retryDelayMs);
|
|
39
|
+
const dropped = await sendOnce(streamName, failed);
|
|
40
|
+
if (dropped.length > 0) {
|
|
41
|
+
onError(new Error(`Dropped ${dropped.length} Firehose records for stream ${streamName}`));
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
async function drain() {
|
|
45
|
+
const records = buffer;
|
|
46
|
+
buffer = [];
|
|
47
|
+
bufferedBytes = 0;
|
|
48
|
+
try {
|
|
49
|
+
const groups = Map.groupBy(records, (record) => record.streamName);
|
|
50
|
+
await Promise.all([...groups].flatMap(([streamName, group]) => chunkFirehoseRecords(group.map((record) => record.data)).map((batch) => sendWithRetry(streamName, batch))));
|
|
51
|
+
}
|
|
52
|
+
finally {
|
|
53
|
+
active = undefined;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
function flush() {
|
|
57
|
+
clearTimer();
|
|
58
|
+
if (active)
|
|
59
|
+
return active.then(flush);
|
|
60
|
+
if (buffer.length === 0)
|
|
61
|
+
return Promise.resolve();
|
|
62
|
+
active = drain();
|
|
63
|
+
return active.then(flush);
|
|
64
|
+
}
|
|
65
|
+
function flushInBackground() {
|
|
66
|
+
flush().catch((error) => onError(toError(error)));
|
|
67
|
+
}
|
|
68
|
+
function write(streamName, data) {
|
|
69
|
+
if (data.byteLength > FIREHOSE_MAX_RECORD_BYTES) {
|
|
70
|
+
onError(new RangeError(`Firehose record of ${data.byteLength} bytes exceeds the limit for stream ${streamName}`));
|
|
71
|
+
return;
|
|
72
|
+
}
|
|
73
|
+
buffer.push({ streamName, data });
|
|
74
|
+
bufferedBytes += data.byteLength;
|
|
75
|
+
if (buffer.length >= FIREHOSE_FLUSH_RECORDS || bufferedBytes >= FIREHOSE_FLUSH_BYTES) {
|
|
76
|
+
// A running drain re-flushes when it settles, so it also takes this record.
|
|
77
|
+
if (!active)
|
|
78
|
+
flushInBackground();
|
|
79
|
+
return;
|
|
80
|
+
}
|
|
81
|
+
if (timer !== undefined)
|
|
82
|
+
return;
|
|
83
|
+
timer = setTimeout(() => {
|
|
84
|
+
timer = undefined;
|
|
85
|
+
flushInBackground();
|
|
86
|
+
}, flushIntervalMs);
|
|
87
|
+
timer.unref();
|
|
88
|
+
}
|
|
89
|
+
return {
|
|
90
|
+
write,
|
|
91
|
+
writeJson: (streamName, value, delimiter) => write(streamName, encodeJsonRecord(value, delimiter)),
|
|
92
|
+
flush,
|
|
93
|
+
};
|
|
94
|
+
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import { PutRecordBatchCommand, type PutRecordBatchOutput } from '@aws-sdk/client-firehose';
|
|
2
|
+
export declare const FIREHOSE_MAX_RECORD_BYTES: number;
|
|
3
|
+
export declare function encodeJsonRecord(value: unknown, delimiter?: string): Uint8Array;
|
|
4
|
+
export declare function chunkFirehoseRecords(records: readonly Uint8Array[]): Uint8Array[][];
|
|
5
|
+
export declare function buildPutRecordBatchCommand(deliveryStreamName: string, records: readonly Uint8Array[]): PutRecordBatchCommand;
|
|
6
|
+
export declare function getFailedFirehoseRecords(records: readonly Uint8Array[], output: Partial<Pick<PutRecordBatchOutput, 'FailedPutCount' | 'RequestResponses'>>): Uint8Array[];
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { PutRecordBatchCommand, } from '@aws-sdk/client-firehose';
|
|
2
|
+
import { byteLength, chunkBySize } from './chunk.mjs';
|
|
3
|
+
const FIREHOSE_MAX_RECORDS_PER_BATCH = 500;
|
|
4
|
+
const FIREHOSE_MAX_BATCH_BYTES = 4 * 1024 * 1024;
|
|
5
|
+
export const FIREHOSE_MAX_RECORD_BYTES = 1_000 * 1024;
|
|
6
|
+
// Serializes a value as a delimiter-terminated JSON record (newline by default for S3 sinks).
|
|
7
|
+
export function encodeJsonRecord(value, delimiter = '\n') {
|
|
8
|
+
const json = JSON.stringify(value);
|
|
9
|
+
if (json === undefined)
|
|
10
|
+
throw new TypeError('Value is not JSON serializable');
|
|
11
|
+
return Buffer.from(`${json}${delimiter}`, 'utf8');
|
|
12
|
+
}
|
|
13
|
+
// Splits records by Firehose count and byte limits. A record over the per-record limit throws.
|
|
14
|
+
export function chunkFirehoseRecords(records) {
|
|
15
|
+
return chunkBySize(records, {
|
|
16
|
+
maxItems: FIREHOSE_MAX_RECORDS_PER_BATCH,
|
|
17
|
+
maxBytes: FIREHOSE_MAX_BATCH_BYTES,
|
|
18
|
+
sizeOf: (record) => {
|
|
19
|
+
const size = byteLength(record);
|
|
20
|
+
if (size > FIREHOSE_MAX_RECORD_BYTES) {
|
|
21
|
+
throw new RangeError(`Firehose record of ${size} bytes exceeds ${FIREHOSE_MAX_RECORD_BYTES}`);
|
|
22
|
+
}
|
|
23
|
+
return size;
|
|
24
|
+
},
|
|
25
|
+
});
|
|
26
|
+
}
|
|
27
|
+
export function buildPutRecordBatchCommand(deliveryStreamName, records) {
|
|
28
|
+
const Records = records.map((Data) => ({ Data }));
|
|
29
|
+
return new PutRecordBatchCommand({ DeliveryStreamName: deliveryStreamName, Records });
|
|
30
|
+
}
|
|
31
|
+
// Returns the input records Firehose rejected, aligned by index. The caller decides whether to retry.
|
|
32
|
+
export function getFailedFirehoseRecords(records, output) {
|
|
33
|
+
if (!output.FailedPutCount)
|
|
34
|
+
return [];
|
|
35
|
+
const failed = [];
|
|
36
|
+
for (const [index, response] of (output.RequestResponses ?? []).entries()) {
|
|
37
|
+
const record = records[index];
|
|
38
|
+
if (response.ErrorCode && record)
|
|
39
|
+
failed.push(record);
|
|
40
|
+
}
|
|
41
|
+
return failed;
|
|
42
|
+
}
|
package/dist/firehose.d.mts
CHANGED
|
@@ -1,5 +1,2 @@
|
|
|
1
|
-
|
|
2
|
-
export
|
|
3
|
-
export declare function chunkFirehoseRecords(records: readonly Uint8Array[]): Uint8Array[][];
|
|
4
|
-
export declare function buildPutRecordBatchCommand(deliveryStreamName: string, records: readonly Uint8Array[]): PutRecordBatchCommand;
|
|
5
|
-
export declare function getFailedFirehoseRecords(records: readonly Uint8Array[], output: Partial<Pick<PutRecordBatchOutput, 'FailedPutCount' | 'RequestResponses'>>): Uint8Array[];
|
|
1
|
+
export * from './firehose-records.mts';
|
|
2
|
+
export * from './firehose-batcher.mts';
|
package/dist/firehose.mjs
CHANGED
|
@@ -1,42 +1,2 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
const FIREHOSE_MAX_RECORDS_PER_BATCH = 500;
|
|
4
|
-
const FIREHOSE_MAX_BATCH_BYTES = 4 * 1024 * 1024;
|
|
5
|
-
const FIREHOSE_MAX_RECORD_BYTES = 1_000 * 1024;
|
|
6
|
-
// Serializes a value as a delimiter-terminated JSON record (newline by default for S3 sinks).
|
|
7
|
-
export function encodeJsonRecord(value, delimiter = '\n') {
|
|
8
|
-
const json = JSON.stringify(value);
|
|
9
|
-
if (json === undefined)
|
|
10
|
-
throw new TypeError('Value is not JSON serializable');
|
|
11
|
-
return Buffer.from(`${json}${delimiter}`, 'utf8');
|
|
12
|
-
}
|
|
13
|
-
// Splits records by Firehose count and byte limits. A record over the per-record limit throws.
|
|
14
|
-
export function chunkFirehoseRecords(records) {
|
|
15
|
-
return chunkBySize(records, {
|
|
16
|
-
maxItems: FIREHOSE_MAX_RECORDS_PER_BATCH,
|
|
17
|
-
maxBytes: FIREHOSE_MAX_BATCH_BYTES,
|
|
18
|
-
sizeOf: (record) => {
|
|
19
|
-
const size = byteLength(record);
|
|
20
|
-
if (size > FIREHOSE_MAX_RECORD_BYTES) {
|
|
21
|
-
throw new RangeError(`Firehose record of ${size} bytes exceeds ${FIREHOSE_MAX_RECORD_BYTES}`);
|
|
22
|
-
}
|
|
23
|
-
return size;
|
|
24
|
-
},
|
|
25
|
-
});
|
|
26
|
-
}
|
|
27
|
-
export function buildPutRecordBatchCommand(deliveryStreamName, records) {
|
|
28
|
-
const Records = records.map((Data) => ({ Data }));
|
|
29
|
-
return new PutRecordBatchCommand({ DeliveryStreamName: deliveryStreamName, Records });
|
|
30
|
-
}
|
|
31
|
-
// Returns the input records Firehose rejected, aligned by index. The caller decides whether to retry.
|
|
32
|
-
export function getFailedFirehoseRecords(records, output) {
|
|
33
|
-
if (!output.FailedPutCount)
|
|
34
|
-
return [];
|
|
35
|
-
const failed = [];
|
|
36
|
-
for (const [index, response] of (output.RequestResponses ?? []).entries()) {
|
|
37
|
-
const record = records[index];
|
|
38
|
-
if (response.ErrorCode && record)
|
|
39
|
-
failed.push(record);
|
|
40
|
-
}
|
|
41
|
-
return failed;
|
|
42
|
-
}
|
|
1
|
+
export * from './firehose-records.mjs';
|
|
2
|
+
export * from './firehose-batcher.mjs';
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { type ChildProcess } from 'node:child_process';
|
|
2
|
+
export declare const AWS_EMULATOR_IDLE_TIMEOUT_SECONDS = 3600;
|
|
3
|
+
export interface StartedAwsEmulatorProcess {
|
|
4
|
+
port: Promise<number>;
|
|
5
|
+
/** The in-container port names the state directory when Docker remaps the host port. */
|
|
6
|
+
statePort?: number;
|
|
7
|
+
stop: () => Promise<void>;
|
|
8
|
+
}
|
|
9
|
+
export interface StartAwsEmulatorInDockerOptions {
|
|
10
|
+
/** Directory the emulator uses as HOME; mounted at the same path inside the container. */
|
|
11
|
+
home: string;
|
|
12
|
+
/** Host path of the Linux emulator binary, mounted read-only. */
|
|
13
|
+
binary: string;
|
|
14
|
+
/** A Linux image able to run the binary (a Debian-based Node image works). Pin it by digest. */
|
|
15
|
+
image: string;
|
|
16
|
+
/** Prefix of the container name. Defaults to `aws-emulator`. */
|
|
17
|
+
containerNamePrefix?: string;
|
|
18
|
+
}
|
|
19
|
+
/** Runs the Linux binary in a container, for hosts (macOS) that cannot execute it. */
|
|
20
|
+
export declare function startAwsEmulatorInDocker(options: StartAwsEmulatorInDockerOptions): Promise<StartedAwsEmulatorProcess>;
|
|
21
|
+
/** Resolves with the port in the emulator's startup log line, or rejects past the deadline. */
|
|
22
|
+
export declare function readAnnouncedPort(child: ChildProcess): Promise<number>;
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import { execFile, spawn } from 'node:child_process';
|
|
2
|
+
import { resolve } from 'node:path';
|
|
3
|
+
import { promisify } from 'node:util';
|
|
4
|
+
const execFileAsync = promisify(execFile);
|
|
5
|
+
const EMULATOR_PORT_IN_CONTAINER = 9297;
|
|
6
|
+
const DOCKER_START_DEADLINE_MS = 180_000;
|
|
7
|
+
const READY_DEADLINE_MS = 60_000;
|
|
8
|
+
const DOCKER_COMMAND_DEADLINE_MS = 30_000;
|
|
9
|
+
// A crashed test runner must not leave an emulator holding memory forever.
|
|
10
|
+
export const AWS_EMULATOR_IDLE_TIMEOUT_SECONDS = 3_600;
|
|
11
|
+
/** Runs the Linux binary in a container, for hosts (macOS) that cannot execute it. */
|
|
12
|
+
export async function startAwsEmulatorInDocker(options) {
|
|
13
|
+
const { home, binary, image } = options;
|
|
14
|
+
const name = `${options.containerNamePrefix ?? 'aws-emulator'}-${process.pid}-${Date.now()}`;
|
|
15
|
+
await execFileAsync('docker', [
|
|
16
|
+
'run',
|
|
17
|
+
'--rm',
|
|
18
|
+
'--detach',
|
|
19
|
+
'--name',
|
|
20
|
+
name,
|
|
21
|
+
'--memory=2g',
|
|
22
|
+
'--memory-swap=2g',
|
|
23
|
+
'--user',
|
|
24
|
+
`${process.getuid?.() ?? 1000}:${process.getgid?.() ?? 1000}`,
|
|
25
|
+
'--publish',
|
|
26
|
+
`127.0.0.1::${EMULATOR_PORT_IN_CONTAINER}`,
|
|
27
|
+
'--volume',
|
|
28
|
+
`${resolve(binary)}:/usr/local/bin/aws-emulator:ro`,
|
|
29
|
+
'--volume',
|
|
30
|
+
`${home}:${home}`,
|
|
31
|
+
'--env',
|
|
32
|
+
`HOME=${home}`,
|
|
33
|
+
'--env',
|
|
34
|
+
'EMULATOR_TELEMETRY=off',
|
|
35
|
+
'--entrypoint',
|
|
36
|
+
'/usr/local/bin/aws-emulator',
|
|
37
|
+
image,
|
|
38
|
+
'--bind',
|
|
39
|
+
'0.0.0.0',
|
|
40
|
+
'--port',
|
|
41
|
+
String(EMULATOR_PORT_IN_CONTAINER),
|
|
42
|
+
'--telemetry',
|
|
43
|
+
'off',
|
|
44
|
+
'--idle-timeout',
|
|
45
|
+
String(AWS_EMULATOR_IDLE_TIMEOUT_SECONDS),
|
|
46
|
+
], { timeout: DOCKER_START_DEADLINE_MS });
|
|
47
|
+
const stop = async () => {
|
|
48
|
+
await execFileAsync('docker', ['rm', '--force', name], {
|
|
49
|
+
timeout: DOCKER_COMMAND_DEADLINE_MS,
|
|
50
|
+
}).catch(() => undefined);
|
|
51
|
+
};
|
|
52
|
+
const port = (async () => {
|
|
53
|
+
// Follow the container log until the server announces it listens; the state files exist then.
|
|
54
|
+
const logs = spawn('docker', ['logs', '--follow', name], { stdio: ['ignore', 'pipe', 'pipe'] });
|
|
55
|
+
try {
|
|
56
|
+
await readAnnouncedPort(logs);
|
|
57
|
+
}
|
|
58
|
+
finally {
|
|
59
|
+
logs.kill();
|
|
60
|
+
}
|
|
61
|
+
const { stdout } = await execFileAsync('docker', ['port', name, `${EMULATOR_PORT_IN_CONTAINER}/tcp`], { timeout: DOCKER_COMMAND_DEADLINE_MS });
|
|
62
|
+
const match = /:(\d+)\s*$/m.exec(stdout);
|
|
63
|
+
if (!match)
|
|
64
|
+
throw new Error(`Could not read the published AWS Emulator port: ${stdout}`);
|
|
65
|
+
return Number(match[1]);
|
|
66
|
+
})();
|
|
67
|
+
return { port, statePort: EMULATOR_PORT_IN_CONTAINER, stop };
|
|
68
|
+
}
|
|
69
|
+
/** Resolves with the port in the emulator's startup log line, or rejects past the deadline. */
|
|
70
|
+
export function readAnnouncedPort(child) {
|
|
71
|
+
return new Promise((resolvePort, reject) => {
|
|
72
|
+
let output = '';
|
|
73
|
+
// The runtime owns the timer behind AbortSignal.timeout, so the deadline is an abort event.
|
|
74
|
+
const deadline = AbortSignal.timeout(READY_DEADLINE_MS);
|
|
75
|
+
const settle = new AbortController();
|
|
76
|
+
deadline.addEventListener('abort', () => reject(new Error(`AWS Emulator did not announce its endpoint:\n${output}`)), { once: true, signal: settle.signal });
|
|
77
|
+
const onData = (chunk) => {
|
|
78
|
+
output += chunk.toString();
|
|
79
|
+
const match = /"endpoint_url":"https:\/\/[^"]+:(\d+)"/.exec(output);
|
|
80
|
+
if (!match)
|
|
81
|
+
return;
|
|
82
|
+
settle.abort();
|
|
83
|
+
child.stdout?.off('data', onData);
|
|
84
|
+
child.stderr?.off('data', onData);
|
|
85
|
+
// Keep draining so a full pipe never blocks the emulator.
|
|
86
|
+
child.stdout?.resume();
|
|
87
|
+
child.stderr?.resume();
|
|
88
|
+
resolvePort(Number(match[1]));
|
|
89
|
+
};
|
|
90
|
+
child.stdout?.on('data', onData);
|
|
91
|
+
child.stderr?.on('data', onData);
|
|
92
|
+
child.once('exit', (code) => {
|
|
93
|
+
settle.abort();
|
|
94
|
+
reject(new Error(`AWS Emulator exited with ${code} before it was ready:\n${output}`));
|
|
95
|
+
});
|
|
96
|
+
// A binary that cannot start emits 'error', and may never emit 'exit'.
|
|
97
|
+
child.once('error', (error) => {
|
|
98
|
+
settle.abort();
|
|
99
|
+
reject(new Error(`AWS Emulator failed to start:\n${output}`, { cause: error }));
|
|
100
|
+
});
|
|
101
|
+
});
|
|
102
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
export type DebArch = 'amd64' | 'arm64';
|
|
2
|
+
export interface AwsEmulatorPackagePin {
|
|
3
|
+
version: string;
|
|
4
|
+
/** SHA-256 hex digest of each architecture's .deb, recorded after verifying AWS's signature. */
|
|
5
|
+
sha256: Record<DebArch, string>;
|
|
6
|
+
}
|
|
7
|
+
export interface EnsureAwsEmulatorBinaryOptions {
|
|
8
|
+
/** The pinned package; callers own the version bump and the recorded digests. */
|
|
9
|
+
pin: AwsEmulatorPackagePin;
|
|
10
|
+
/** Directory the verified binary is cached in, one subdirectory per version and architecture. */
|
|
11
|
+
cacheDir: string;
|
|
12
|
+
/** Defaults to the host architecture. */
|
|
13
|
+
target?: DebArch;
|
|
14
|
+
/** Defaults to the global fetch; inject one that honors a proxy where the network requires it. */
|
|
15
|
+
fetch?: (url: string, init: {
|
|
16
|
+
signal: AbortSignal;
|
|
17
|
+
}) => Promise<Response>;
|
|
18
|
+
/** Package index. Defaults to the AWS download host. */
|
|
19
|
+
baseUrl?: string;
|
|
20
|
+
}
|
|
21
|
+
export declare function debArch(): DebArch;
|
|
22
|
+
/** Downloads, digest-verifies and extracts the pinned binary once per cache directory. */
|
|
23
|
+
export declare function ensureAwsEmulatorBinary(options: EnsureAwsEmulatorBinaryOptions): Promise<string>;
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import { execFile } from 'node:child_process';
|
|
2
|
+
import { createHash } from 'node:crypto';
|
|
3
|
+
import { createReadStream, createWriteStream, existsSync } from 'node:fs';
|
|
4
|
+
import { chmod, mkdir, mkdtemp, rename, rm } from 'node:fs/promises';
|
|
5
|
+
import { arch } from 'node:os';
|
|
6
|
+
import { join } from 'node:path';
|
|
7
|
+
import { Readable } from 'node:stream';
|
|
8
|
+
import { pipeline } from 'node:stream/promises';
|
|
9
|
+
import { promisify } from 'node:util';
|
|
10
|
+
const execFileAsync = promisify(execFile);
|
|
11
|
+
const DOWNLOAD_DEADLINE_MS = 180_000;
|
|
12
|
+
export function debArch() {
|
|
13
|
+
const value = arch();
|
|
14
|
+
if (value === 'x64')
|
|
15
|
+
return 'amd64';
|
|
16
|
+
if (value === 'arm64')
|
|
17
|
+
return 'arm64';
|
|
18
|
+
throw new Error(`The AWS Emulator ships no Linux package for ${value}`);
|
|
19
|
+
}
|
|
20
|
+
/** Downloads, digest-verifies and extracts the pinned binary once per cache directory. */
|
|
21
|
+
export async function ensureAwsEmulatorBinary(options) {
|
|
22
|
+
const { pin, cacheDir } = options;
|
|
23
|
+
const target = options.target ?? debArch();
|
|
24
|
+
const doFetch = options.fetch ?? fetch;
|
|
25
|
+
const baseUrl = options.baseUrl ?? 'https://downloads.aws.com/aws-emulator/packages/aws-emulator/LTS';
|
|
26
|
+
const dir = join(cacheDir, pin.version, target);
|
|
27
|
+
const binary = join(dir, 'aws-emulator');
|
|
28
|
+
if (existsSync(binary))
|
|
29
|
+
return binary;
|
|
30
|
+
await mkdir(dir, { recursive: true });
|
|
31
|
+
const staging = await mkdtemp(join(dir, 'staging-'));
|
|
32
|
+
try {
|
|
33
|
+
const deb = join(staging, 'aws-emulator.deb');
|
|
34
|
+
const url = `${baseUrl}/aws-emulator_${pin.version}-1_${target}.deb`;
|
|
35
|
+
const response = await doFetch(url, { signal: AbortSignal.timeout(DOWNLOAD_DEADLINE_MS) });
|
|
36
|
+
if (!response.ok || !response.body) {
|
|
37
|
+
throw new Error(`AWS Emulator download failed: ${response.status} ${url}`);
|
|
38
|
+
}
|
|
39
|
+
await pipeline(Readable.fromWeb(response.body), createWriteStream(deb));
|
|
40
|
+
const digest = await sha256File(deb);
|
|
41
|
+
const expected = pin.sha256[target];
|
|
42
|
+
if (digest !== expected) {
|
|
43
|
+
throw new Error(`AWS Emulator package digest ${digest} does not match pinned ${expected}`);
|
|
44
|
+
}
|
|
45
|
+
// A .deb is an ar archive; the binary is the only file needed from its data tarball.
|
|
46
|
+
await execFileAsync('sh', [
|
|
47
|
+
'-c',
|
|
48
|
+
'ar p "$0" data.tar.xz | tar -xJf - -C "$1" ./opt/aws-emulator/bin/aws-emulator',
|
|
49
|
+
deb,
|
|
50
|
+
staging,
|
|
51
|
+
], { timeout: DOWNLOAD_DEADLINE_MS });
|
|
52
|
+
const extracted = join(staging, 'opt', 'aws-emulator', 'bin', 'aws-emulator');
|
|
53
|
+
await chmod(extracted, 0o755);
|
|
54
|
+
// Concurrent runs may race to this point; rename is atomic and either copy is identical.
|
|
55
|
+
await rename(extracted, binary);
|
|
56
|
+
return binary;
|
|
57
|
+
}
|
|
58
|
+
finally {
|
|
59
|
+
await rm(staging, { recursive: true, force: true });
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
async function sha256File(path) {
|
|
63
|
+
const hash = createHash('sha256');
|
|
64
|
+
await pipeline(createReadStream(path), hash);
|
|
65
|
+
return hash.digest('hex');
|
|
66
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { type EnsureAwsEmulatorBinaryOptions } from './emulator-package.mts';
|
|
2
|
+
export interface AwsEmulatorConnection {
|
|
3
|
+
endpoint: string;
|
|
4
|
+
region: string;
|
|
5
|
+
caPem: string;
|
|
6
|
+
accessKeyId: string;
|
|
7
|
+
secretAccessKey: string;
|
|
8
|
+
}
|
|
9
|
+
export interface RunningAwsEmulator extends AwsEmulatorConnection {
|
|
10
|
+
stop: () => Promise<void>;
|
|
11
|
+
}
|
|
12
|
+
export interface StartAwsEmulatorOptions {
|
|
13
|
+
/** The pinned package to download and verify. */
|
|
14
|
+
package: EnsureAwsEmulatorBinaryOptions;
|
|
15
|
+
/** Image used to run the Linux binary on hosts that cannot execute it natively (macOS). */
|
|
16
|
+
dockerImage: string;
|
|
17
|
+
/** Prefix for the temporary HOME directory and the container name. Defaults to `aws-emulator`. */
|
|
18
|
+
namePrefix?: string;
|
|
19
|
+
/** Region the emulator reports. Defaults to `us-west-2`. */
|
|
20
|
+
region?: string;
|
|
21
|
+
}
|
|
22
|
+
/** Starts a private emulator on a free port with telemetry off and returns its connection. */
|
|
23
|
+
export declare function startAwsEmulator(options: StartAwsEmulatorOptions): Promise<RunningAwsEmulator>;
|
|
24
|
+
/**
|
|
25
|
+
* Adds the run's self-signed emulator certificate to Node's default trust store, which every TLS
|
|
26
|
+
* client in this process (the SDK's handler, `fetch`) reads unless it passes its own `ca`.
|
|
27
|
+
*/
|
|
28
|
+
export declare function trustAwsEmulatorCertificate(caPem: string): void;
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import { spawn } from 'node:child_process';
|
|
2
|
+
import { once } from 'node:events';
|
|
3
|
+
import { mkdtemp, readFile, rm } from 'node:fs/promises';
|
|
4
|
+
import { request } from 'node:https';
|
|
5
|
+
import { platform, tmpdir } from 'node:os';
|
|
6
|
+
import { join } from 'node:path';
|
|
7
|
+
import tls from 'node:tls';
|
|
8
|
+
import { AWS_EMULATOR_IDLE_TIMEOUT_SECONDS, readAnnouncedPort, startAwsEmulatorInDocker, } from './emulator-docker.mjs';
|
|
9
|
+
import { ensureAwsEmulatorBinary, } from './emulator-package.mjs';
|
|
10
|
+
const READY_DEADLINE_MS = 60_000;
|
|
11
|
+
/** Starts a private emulator on a free port with telemetry off and returns its connection. */
|
|
12
|
+
export async function startAwsEmulator(options) {
|
|
13
|
+
const prefix = options.namePrefix ?? 'aws-emulator';
|
|
14
|
+
const binary = await ensureAwsEmulatorBinary(options.package);
|
|
15
|
+
const home = await mkdtemp(join(tmpdir(), `${prefix}-`));
|
|
16
|
+
const running = platform() === 'linux'
|
|
17
|
+
? startNative(binary, home)
|
|
18
|
+
: await startAwsEmulatorInDocker({
|
|
19
|
+
home,
|
|
20
|
+
binary,
|
|
21
|
+
image: options.dockerImage,
|
|
22
|
+
containerNamePrefix: prefix,
|
|
23
|
+
});
|
|
24
|
+
try {
|
|
25
|
+
const port = await running.port;
|
|
26
|
+
const stateDir = join(home, '.aws', 'emulator', String(running.statePort ?? port));
|
|
27
|
+
const caPem = await readFile(join(stateDir, 'ca.pem'), 'utf8');
|
|
28
|
+
const credentials = parseCredentials(await readFile(join(stateDir, 'emulator_credentials'), 'utf8'));
|
|
29
|
+
const endpoint = `https://127.0.0.1:${port}`;
|
|
30
|
+
await checkHealth(endpoint, caPem);
|
|
31
|
+
return {
|
|
32
|
+
endpoint,
|
|
33
|
+
region: options.region ?? 'us-west-2',
|
|
34
|
+
caPem,
|
|
35
|
+
...credentials,
|
|
36
|
+
stop: async () => {
|
|
37
|
+
await running.stop();
|
|
38
|
+
await rm(home, { recursive: true, force: true });
|
|
39
|
+
},
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
catch (err) {
|
|
43
|
+
await running.stop();
|
|
44
|
+
await rm(home, { recursive: true, force: true });
|
|
45
|
+
throw err;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
function startNative(binary, home) {
|
|
49
|
+
const child = spawn(binary, [
|
|
50
|
+
'--port',
|
|
51
|
+
'0',
|
|
52
|
+
'--telemetry',
|
|
53
|
+
'off',
|
|
54
|
+
'--idle-timeout',
|
|
55
|
+
String(AWS_EMULATOR_IDLE_TIMEOUT_SECONDS),
|
|
56
|
+
], {
|
|
57
|
+
env: { PATH: process.env.PATH, HOME: home, EMULATOR_TELEMETRY: 'off' },
|
|
58
|
+
stdio: ['ignore', 'pipe', 'pipe'],
|
|
59
|
+
});
|
|
60
|
+
return { port: readAnnouncedPort(child), stop: () => stopChild(child) };
|
|
61
|
+
}
|
|
62
|
+
async function stopChild(child) {
|
|
63
|
+
// A failed spawn has no pid and never emits 'exit'.
|
|
64
|
+
if (child.pid === undefined || child.exitCode !== null || child.signalCode !== null)
|
|
65
|
+
return;
|
|
66
|
+
const exited = once(child, 'exit');
|
|
67
|
+
child.kill('SIGTERM');
|
|
68
|
+
// The runtime owns the timer behind AbortSignal.timeout; a stuck emulator is killed outright.
|
|
69
|
+
const forceKill = AbortSignal.timeout(5_000);
|
|
70
|
+
const onForceKill = () => child.kill('SIGKILL');
|
|
71
|
+
forceKill.addEventListener('abort', onForceKill, { once: true });
|
|
72
|
+
await exited;
|
|
73
|
+
forceKill.removeEventListener('abort', onForceKill);
|
|
74
|
+
}
|
|
75
|
+
// The emulator logs its endpoint once it listens, so one request confirms it serves.
|
|
76
|
+
async function checkHealth(endpoint, caPem) {
|
|
77
|
+
const status = await new Promise((resolveStatus, reject) => {
|
|
78
|
+
const req = request(`${endpoint}/__admin/health`, { ca: caPem, signal: AbortSignal.timeout(READY_DEADLINE_MS) }, (res) => {
|
|
79
|
+
res.resume();
|
|
80
|
+
resolveStatus(res.statusCode);
|
|
81
|
+
});
|
|
82
|
+
req.once('error', reject);
|
|
83
|
+
req.end();
|
|
84
|
+
});
|
|
85
|
+
if (status !== 200)
|
|
86
|
+
throw new Error(`AWS Emulator at ${endpoint} answered health with ${status}`);
|
|
87
|
+
}
|
|
88
|
+
function parseCredentials(file) {
|
|
89
|
+
const accessKeyId = /^\s*aws_access_key_id\s*=\s*(\S+)/m.exec(file)?.[1];
|
|
90
|
+
const secretAccessKey = /^\s*aws_secret_access_key\s*=\s*(\S+)/m.exec(file)?.[1];
|
|
91
|
+
if (!accessKeyId || !secretAccessKey)
|
|
92
|
+
throw new Error('AWS Emulator wrote no credentials');
|
|
93
|
+
return { accessKeyId, secretAccessKey };
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* Adds the run's self-signed emulator certificate to Node's default trust store, which every TLS
|
|
97
|
+
* client in this process (the SDK's handler, `fetch`) reads unless it passes its own `ca`.
|
|
98
|
+
*/
|
|
99
|
+
export function trustAwsEmulatorCertificate(caPem) {
|
|
100
|
+
// Added in Node 24.5; older runtimes can pass the certificate as `ca` or set NODE_EXTRA_CA_CERTS.
|
|
101
|
+
if (typeof tls.setDefaultCACertificates !== 'function') {
|
|
102
|
+
throw new Error('trustAwsEmulatorCertificate needs Node 24.5 or newer; pass caPem as `ca` instead');
|
|
103
|
+
}
|
|
104
|
+
const current = tls.getCACertificates('default');
|
|
105
|
+
if (current.includes(caPem))
|
|
106
|
+
return;
|
|
107
|
+
tls.setDefaultCACertificates([...current, caPem]);
|
|
108
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { AwsEmulatorConnection } from './emulator.mts';
|
|
2
|
+
export interface ProvisionFirehoseStreamsOptions {
|
|
3
|
+
/** Delivery stream names to create; the application under test derives them from its own config. */
|
|
4
|
+
streamNames: readonly string[];
|
|
5
|
+
/** Destination bucket and delivery role names. Default to `test-firehose`. */
|
|
6
|
+
bucketName?: string;
|
|
7
|
+
roleName?: string;
|
|
8
|
+
/** How long to wait for every stream to become ACTIVE. Defaults to 30000. */
|
|
9
|
+
activeDeadlineMs?: number;
|
|
10
|
+
/** Delay between status checks. Defaults to 1000. */
|
|
11
|
+
pollIntervalMs?: number;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Creates one active DirectPut delivery stream per name, delivering into a destination bucket.
|
|
15
|
+
* The emulator rejects a stream whose role does not exist and accepts records only once the
|
|
16
|
+
* stream is ACTIVE (about three seconds after creation), so this waits for every stream.
|
|
17
|
+
*/
|
|
18
|
+
export declare function provisionFirehoseStreams(emulator: AwsEmulatorConnection, options: ProvisionFirehoseStreamsOptions): Promise<void>;
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import { CreateBucketCommand, S3Client } from '@aws-sdk/client-s3';
|
|
2
|
+
import { CreateDeliveryStreamCommand, DescribeDeliveryStreamCommand, FirehoseClient, } from '@aws-sdk/client-firehose';
|
|
3
|
+
import { CreateRoleCommand, IAMClient } from '@aws-sdk/client-iam';
|
|
4
|
+
/**
|
|
5
|
+
* Creates one active DirectPut delivery stream per name, delivering into a destination bucket.
|
|
6
|
+
* The emulator rejects a stream whose role does not exist and accepts records only once the
|
|
7
|
+
* stream is ACTIVE (about three seconds after creation), so this waits for every stream.
|
|
8
|
+
*/
|
|
9
|
+
export async function provisionFirehoseStreams(emulator, options) {
|
|
10
|
+
const bucket = options.bucketName ?? 'test-firehose';
|
|
11
|
+
const config = {
|
|
12
|
+
endpoint: emulator.endpoint,
|
|
13
|
+
region: emulator.region,
|
|
14
|
+
credentials: { accessKeyId: emulator.accessKeyId, secretAccessKey: emulator.secretAccessKey },
|
|
15
|
+
};
|
|
16
|
+
const s3 = new S3Client(config);
|
|
17
|
+
const iam = new IAMClient(config);
|
|
18
|
+
const firehose = new FirehoseClient(config);
|
|
19
|
+
try {
|
|
20
|
+
await s3.send(new CreateBucketCommand({ Bucket: bucket }));
|
|
21
|
+
const { Role } = await iam.send(new CreateRoleCommand({
|
|
22
|
+
RoleName: options.roleName ?? 'test-firehose',
|
|
23
|
+
AssumeRolePolicyDocument: JSON.stringify({
|
|
24
|
+
Version: '2012-10-17',
|
|
25
|
+
Statement: [
|
|
26
|
+
{
|
|
27
|
+
Effect: 'Allow',
|
|
28
|
+
Principal: { Service: 'firehose.amazonaws.com' },
|
|
29
|
+
Action: 'sts:AssumeRole',
|
|
30
|
+
},
|
|
31
|
+
],
|
|
32
|
+
}),
|
|
33
|
+
}));
|
|
34
|
+
await Promise.all(options.streamNames.map((DeliveryStreamName) => firehose.send(new CreateDeliveryStreamCommand({
|
|
35
|
+
DeliveryStreamName,
|
|
36
|
+
DeliveryStreamType: 'DirectPut',
|
|
37
|
+
ExtendedS3DestinationConfiguration: {
|
|
38
|
+
RoleARN: Role?.Arn,
|
|
39
|
+
BucketARN: `arn:aws:s3:::${bucket}`,
|
|
40
|
+
Prefix: `${DeliveryStreamName}/`,
|
|
41
|
+
},
|
|
42
|
+
}))));
|
|
43
|
+
const deadline = Date.now() + (options.activeDeadlineMs ?? 30_000);
|
|
44
|
+
await Promise.all(options.streamNames.map((name) => waitUntilActive(firehose, name, deadline, options.pollIntervalMs ?? 1_000)));
|
|
45
|
+
}
|
|
46
|
+
finally {
|
|
47
|
+
s3.destroy();
|
|
48
|
+
iam.destroy();
|
|
49
|
+
firehose.destroy();
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
async function waitUntilActive(client, DeliveryStreamName, deadline, pollIntervalMs) {
|
|
53
|
+
for (;;) {
|
|
54
|
+
const { DeliveryStreamDescription } = await client.send(new DescribeDeliveryStreamCommand({ DeliveryStreamName }));
|
|
55
|
+
const status = DeliveryStreamDescription?.DeliveryStreamStatus;
|
|
56
|
+
if (status === 'ACTIVE')
|
|
57
|
+
return;
|
|
58
|
+
if (Date.now() >= deadline) {
|
|
59
|
+
throw new Error(`Delivery stream ${DeliveryStreamName} is ${status}, not ACTIVE`);
|
|
60
|
+
}
|
|
61
|
+
await new Promise((resolve) => setTimeout(resolve, pollIntervalMs));
|
|
62
|
+
}
|
|
63
|
+
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
export { startAwsEmulator, trustAwsEmulatorCertificate, type AwsEmulatorConnection, type RunningAwsEmulator, type StartAwsEmulatorOptions, } from './emulator.mts';
|
|
2
|
+
export { AWS_EMULATOR_IDLE_TIMEOUT_SECONDS, readAnnouncedPort, startAwsEmulatorInDocker, type StartAwsEmulatorInDockerOptions, type StartedAwsEmulatorProcess, } from './emulator-docker.mts';
|
|
3
|
+
export { debArch, ensureAwsEmulatorBinary, type AwsEmulatorPackagePin, type DebArch, type EnsureAwsEmulatorBinaryOptions, } from './emulator-package.mts';
|
|
4
|
+
export { provisionFirehoseStreams, type ProvisionFirehoseStreamsOptions } from './firehose.mts';
|
|
5
|
+
export { createTestS3Bucket, deleteS3Object, listS3Keys, listS3Versions, putS3Object, readS3ObjectBytes, readS3ObjectText, s3ObjectExists, sendToPresignedS3Url, uniqueS3Prefix, type S3VersionEntry, } from './s3.mts';
|
|
6
|
+
export { countTestSqsMessages, createTestSqsQueue, sendTestSqsMessage } from './sqs.mts';
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
export { startAwsEmulator, trustAwsEmulatorCertificate, } from './emulator.mjs';
|
|
2
|
+
export { AWS_EMULATOR_IDLE_TIMEOUT_SECONDS, readAnnouncedPort, startAwsEmulatorInDocker, } from './emulator-docker.mjs';
|
|
3
|
+
export { debArch, ensureAwsEmulatorBinary, } from './emulator-package.mjs';
|
|
4
|
+
export { provisionFirehoseStreams } from './firehose.mjs';
|
|
5
|
+
export { createTestS3Bucket, deleteS3Object, listS3Keys, listS3Versions, putS3Object, readS3ObjectBytes, readS3ObjectText, s3ObjectExists, sendToPresignedS3Url, uniqueS3Prefix, } from './s3.mjs';
|
|
6
|
+
export { countTestSqsMessages, createTestSqsQueue, sendTestSqsMessage } from './sqs.mjs';
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { type PutObjectCommandInput, type S3Client } from '@aws-sdk/client-s3';
|
|
2
|
+
/** A key prefix no other test or concurrent fork can produce. */
|
|
3
|
+
export declare function uniqueS3Prefix(label: string): string;
|
|
4
|
+
/** A bucket owned by one test file, for code that reads its bucket name from the environment. */
|
|
5
|
+
export declare function createTestS3Bucket(client: S3Client, label: string, options?: {
|
|
6
|
+
versioned?: boolean;
|
|
7
|
+
}): Promise<string>;
|
|
8
|
+
export declare function putS3Object(client: S3Client, input: Omit<PutObjectCommandInput, 'Body'> & {
|
|
9
|
+
Body: string | Uint8Array;
|
|
10
|
+
}): Promise<{
|
|
11
|
+
versionId?: string;
|
|
12
|
+
}>;
|
|
13
|
+
/** The object's text, or undefined when the key does not exist. */
|
|
14
|
+
export declare function readS3ObjectText(client: S3Client, Bucket: string, Key: string): Promise<string | undefined>;
|
|
15
|
+
/** The object's bytes, or undefined when the key does not exist. */
|
|
16
|
+
export declare function readS3ObjectBytes(client: S3Client, Bucket: string, Key: string): Promise<Buffer | undefined>;
|
|
17
|
+
export declare function s3ObjectExists(client: S3Client, Bucket: string, Key: string): Promise<boolean>;
|
|
18
|
+
/** Every current key under a prefix, sorted. */
|
|
19
|
+
export declare function listS3Keys(client: S3Client, Bucket: string, Prefix: string): Promise<string[]>;
|
|
20
|
+
export interface S3VersionEntry {
|
|
21
|
+
key: string;
|
|
22
|
+
versionId: string;
|
|
23
|
+
deleteMarker: boolean;
|
|
24
|
+
}
|
|
25
|
+
/** Every version and delete marker under a prefix, for versioned buckets. */
|
|
26
|
+
export declare function listS3Versions(client: S3Client, Bucket: string, Prefix: string): Promise<S3VersionEntry[]>;
|
|
27
|
+
export declare function deleteS3Object(client: S3Client, Bucket: string, Key: string): Promise<void>;
|
|
28
|
+
/**
|
|
29
|
+
* Sends a request to a presigned URL the way a browser would. The fetch is injectable for
|
|
30
|
+
* environments that need a custom dispatcher; it defaults to the global fetch.
|
|
31
|
+
*/
|
|
32
|
+
export declare function sendToPresignedS3Url(url: string, init: {
|
|
33
|
+
method: 'GET' | 'PUT';
|
|
34
|
+
contentType?: string;
|
|
35
|
+
body?: Uint8Array;
|
|
36
|
+
}, doFetch?: (url: string, init: RequestInit) => Promise<Response>): Promise<{
|
|
37
|
+
status: number;
|
|
38
|
+
body: string;
|
|
39
|
+
}>;
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import { CreateBucketCommand, DeleteObjectCommand, GetObjectCommand, HeadObjectCommand, ListObjectsV2Command, ListObjectVersionsCommand, PutBucketVersioningCommand, PutObjectCommand, } from '@aws-sdk/client-s3';
|
|
3
|
+
// Every helper takes the S3 client the test already points at the emulator, so a test can seed
|
|
4
|
+
// and inspect state without going through the code under test.
|
|
5
|
+
/** A key prefix no other test or concurrent fork can produce. */
|
|
6
|
+
export function uniqueS3Prefix(label) {
|
|
7
|
+
return `test/${label}/${randomUUID()}/`;
|
|
8
|
+
}
|
|
9
|
+
/** A bucket owned by one test file, for code that reads its bucket name from the environment. */
|
|
10
|
+
export async function createTestS3Bucket(client, label, options = {}) {
|
|
11
|
+
// Truncate the label, never the random suffix, so concurrent tests keep distinct buckets.
|
|
12
|
+
const Bucket = `test-${label.slice(0, 21)}-${randomUUID()}`.toLowerCase();
|
|
13
|
+
await client.send(new CreateBucketCommand({ Bucket }));
|
|
14
|
+
if (options.versioned) {
|
|
15
|
+
await client.send(new PutBucketVersioningCommand({ Bucket, VersioningConfiguration: { Status: 'Enabled' } }));
|
|
16
|
+
}
|
|
17
|
+
return Bucket;
|
|
18
|
+
}
|
|
19
|
+
export async function putS3Object(client, input) {
|
|
20
|
+
const result = await client.send(new PutObjectCommand(input));
|
|
21
|
+
return result.VersionId === undefined ? {} : { versionId: result.VersionId };
|
|
22
|
+
}
|
|
23
|
+
async function readBody(client, Bucket, Key, read) {
|
|
24
|
+
try {
|
|
25
|
+
const body = await getBody(client, Bucket, Key);
|
|
26
|
+
return body ? await read(body) : undefined;
|
|
27
|
+
}
|
|
28
|
+
catch (err) {
|
|
29
|
+
if (err.name === 'NoSuchKey')
|
|
30
|
+
return undefined;
|
|
31
|
+
throw err;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
async function getBody(client, Bucket, Key) {
|
|
35
|
+
return (await client.send(new GetObjectCommand({ Bucket, Key }))).Body;
|
|
36
|
+
}
|
|
37
|
+
/** The object's text, or undefined when the key does not exist. */
|
|
38
|
+
export function readS3ObjectText(client, Bucket, Key) {
|
|
39
|
+
return readBody(client, Bucket, Key, (body) => body.transformToString());
|
|
40
|
+
}
|
|
41
|
+
/** The object's bytes, or undefined when the key does not exist. */
|
|
42
|
+
export function readS3ObjectBytes(client, Bucket, Key) {
|
|
43
|
+
return readBody(client, Bucket, Key, async (body) => Buffer.from(await body.transformToByteArray()));
|
|
44
|
+
}
|
|
45
|
+
export async function s3ObjectExists(client, Bucket, Key) {
|
|
46
|
+
try {
|
|
47
|
+
await client.send(new HeadObjectCommand({ Bucket, Key }));
|
|
48
|
+
return true;
|
|
49
|
+
}
|
|
50
|
+
catch (err) {
|
|
51
|
+
if (err.$metadata?.httpStatusCode === 404) {
|
|
52
|
+
return false;
|
|
53
|
+
}
|
|
54
|
+
throw err;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
/** Every current key under a prefix, sorted. */
|
|
58
|
+
export async function listS3Keys(client, Bucket, Prefix) {
|
|
59
|
+
const keys = [];
|
|
60
|
+
let ContinuationToken;
|
|
61
|
+
do {
|
|
62
|
+
const page = await client.send(new ListObjectsV2Command({ Bucket, Prefix, ContinuationToken }));
|
|
63
|
+
keys.push(...(page.Contents ?? []).flatMap((object) => (object.Key ? [object.Key] : [])));
|
|
64
|
+
ContinuationToken = page.NextContinuationToken;
|
|
65
|
+
} while (ContinuationToken);
|
|
66
|
+
return keys.toSorted();
|
|
67
|
+
}
|
|
68
|
+
/** Every version and delete marker under a prefix, for versioned buckets. */
|
|
69
|
+
export async function listS3Versions(client, Bucket, Prefix) {
|
|
70
|
+
const versions = [];
|
|
71
|
+
let KeyMarker;
|
|
72
|
+
let VersionIdMarker;
|
|
73
|
+
let truncated;
|
|
74
|
+
do {
|
|
75
|
+
const page = await client.send(new ListObjectVersionsCommand({ Bucket, Prefix, KeyMarker, VersionIdMarker }));
|
|
76
|
+
for (const v of page.Versions ?? []) {
|
|
77
|
+
versions.push({ key: v.Key, versionId: v.VersionId, deleteMarker: false });
|
|
78
|
+
}
|
|
79
|
+
for (const v of page.DeleteMarkers ?? []) {
|
|
80
|
+
versions.push({ key: v.Key, versionId: v.VersionId, deleteMarker: true });
|
|
81
|
+
}
|
|
82
|
+
KeyMarker = page.NextKeyMarker;
|
|
83
|
+
VersionIdMarker = page.NextVersionIdMarker;
|
|
84
|
+
truncated = page.IsTruncated;
|
|
85
|
+
} while (truncated);
|
|
86
|
+
return versions;
|
|
87
|
+
}
|
|
88
|
+
export async function deleteS3Object(client, Bucket, Key) {
|
|
89
|
+
await client.send(new DeleteObjectCommand({ Bucket, Key }));
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* Sends a request to a presigned URL the way a browser would. The fetch is injectable for
|
|
93
|
+
* environments that need a custom dispatcher; it defaults to the global fetch.
|
|
94
|
+
*/
|
|
95
|
+
export async function sendToPresignedS3Url(url, init, doFetch = fetch) {
|
|
96
|
+
const response = await doFetch(url, {
|
|
97
|
+
method: init.method,
|
|
98
|
+
...(init.contentType ? { headers: { 'content-type': init.contentType } } : {}),
|
|
99
|
+
...(init.body ? { body: init.body } : {}),
|
|
100
|
+
});
|
|
101
|
+
return { status: response.status, body: await response.text() };
|
|
102
|
+
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import { type SQSClient } from '@aws-sdk/client-sqs';
|
|
2
|
+
/** A queue owned by one test file, for workers that read their queue URL from the environment. */
|
|
3
|
+
export declare function createTestSqsQueue(client: SQSClient, label: string): Promise<string>;
|
|
4
|
+
export declare function sendTestSqsMessage(client: SQSClient, QueueUrl: string, MessageBody: string): Promise<void>;
|
|
5
|
+
/** Messages neither visible nor in flight are gone; the sum is what a consumer has not deleted. */
|
|
6
|
+
export declare function countTestSqsMessages(client: SQSClient, QueueUrl: string): Promise<number>;
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import { CreateQueueCommand, GetQueueAttributesCommand, SendMessageCommand, } from '@aws-sdk/client-sqs';
|
|
3
|
+
// Every helper takes the SQS client the test already points at the emulator. Build it with
|
|
4
|
+
// `useQueueUrlAsEndpoint: false`: queue URLs carry the real AWS host, and the explicit endpoint
|
|
5
|
+
// keeps requests on the emulator.
|
|
6
|
+
/** A queue owned by one test file, for workers that read their queue URL from the environment. */
|
|
7
|
+
export async function createTestSqsQueue(client, label) {
|
|
8
|
+
const { QueueUrl } = await client.send(new CreateQueueCommand({ QueueName: `test-${label.slice(0, 38)}-${randomUUID()}` }));
|
|
9
|
+
if (!QueueUrl)
|
|
10
|
+
throw new Error('The AWS Emulator returned no queue URL');
|
|
11
|
+
return QueueUrl;
|
|
12
|
+
}
|
|
13
|
+
export async function sendTestSqsMessage(client, QueueUrl, MessageBody) {
|
|
14
|
+
await client.send(new SendMessageCommand({ QueueUrl, MessageBody }));
|
|
15
|
+
}
|
|
16
|
+
/** Messages neither visible nor in flight are gone; the sum is what a consumer has not deleted. */
|
|
17
|
+
export async function countTestSqsMessages(client, QueueUrl) {
|
|
18
|
+
const { Attributes } = await client.send(new GetQueueAttributesCommand({
|
|
19
|
+
QueueUrl,
|
|
20
|
+
AttributeNames: ['ApproximateNumberOfMessages', 'ApproximateNumberOfMessagesNotVisible'],
|
|
21
|
+
}));
|
|
22
|
+
return (Number(Attributes?.ApproximateNumberOfMessages ?? 0) +
|
|
23
|
+
Number(Attributes?.ApproximateNumberOfMessagesNotVisible ?? 0));
|
|
24
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@vouchington/aws",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.2",
|
|
4
4
|
"description": "AWS SDK v3 command builders, batching, codecs, and error classifiers for SES, SQS, Firehose, and CloudWatch.",
|
|
5
5
|
"homepage": "https://github.com/vouchington/vouchington-platform/tree/main/packages/aws#readme",
|
|
6
6
|
"bugs": {
|
|
@@ -49,24 +49,33 @@
|
|
|
49
49
|
"types": "./dist/errors.d.mts",
|
|
50
50
|
"import": "./dist/errors.mjs",
|
|
51
51
|
"default": "./dist/errors.mjs"
|
|
52
|
+
},
|
|
53
|
+
"./testing": {
|
|
54
|
+
"types": "./dist/testing/index.d.mts",
|
|
55
|
+
"import": "./dist/testing/index.mjs",
|
|
56
|
+
"default": "./dist/testing/index.mjs"
|
|
52
57
|
}
|
|
53
58
|
},
|
|
54
59
|
"publishConfig": {
|
|
55
60
|
"access": "public"
|
|
56
61
|
},
|
|
57
62
|
"devDependencies": {
|
|
58
|
-
"@aws-sdk/client-cloudwatch": "^3.
|
|
59
|
-
"@aws-sdk/client-firehose": "^3.
|
|
60
|
-
"@aws-sdk/client-
|
|
61
|
-
"@aws-sdk/client-
|
|
62
|
-
"@
|
|
63
|
+
"@aws-sdk/client-cloudwatch": "^3.1165.0",
|
|
64
|
+
"@aws-sdk/client-firehose": "^3.1165.0",
|
|
65
|
+
"@aws-sdk/client-iam": "^3.1175.0",
|
|
66
|
+
"@aws-sdk/client-s3": "^3.1175.0",
|
|
67
|
+
"@aws-sdk/client-ses": "^3.1165.0",
|
|
68
|
+
"@aws-sdk/client-sqs": "^3.1165.0",
|
|
69
|
+
"@smithy/node-http-handler": "^4.14.2"
|
|
63
70
|
},
|
|
64
71
|
"peerDependencies": {
|
|
65
|
-
"@aws-sdk/client-cloudwatch": "^3.
|
|
66
|
-
"@aws-sdk/client-firehose": "^3.
|
|
67
|
-
"@aws-sdk/client-
|
|
68
|
-
"@aws-sdk/client-
|
|
69
|
-
"@
|
|
72
|
+
"@aws-sdk/client-cloudwatch": "^3.1165.0",
|
|
73
|
+
"@aws-sdk/client-firehose": "^3.1165.0",
|
|
74
|
+
"@aws-sdk/client-iam": "^3.1175.0",
|
|
75
|
+
"@aws-sdk/client-s3": "^3.1175.0",
|
|
76
|
+
"@aws-sdk/client-ses": "^3.1165.0",
|
|
77
|
+
"@aws-sdk/client-sqs": "^3.1165.0",
|
|
78
|
+
"@smithy/node-http-handler": "^4.14.2"
|
|
70
79
|
},
|
|
71
80
|
"peerDependenciesMeta": {
|
|
72
81
|
"@aws-sdk/client-cloudwatch": {
|
|
@@ -75,6 +84,12 @@
|
|
|
75
84
|
"@aws-sdk/client-firehose": {
|
|
76
85
|
"optional": true
|
|
77
86
|
},
|
|
87
|
+
"@aws-sdk/client-iam": {
|
|
88
|
+
"optional": true
|
|
89
|
+
},
|
|
90
|
+
"@aws-sdk/client-s3": {
|
|
91
|
+
"optional": true
|
|
92
|
+
},
|
|
78
93
|
"@aws-sdk/client-ses": {
|
|
79
94
|
"optional": true
|
|
80
95
|
},
|