@defra/grants-config-utils 1.1.2 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/package.json +1 -1
- package/src/s3/s3-interactions.js +13 -6
- package/src/s3/s3-interactions.test.js +158 -13
- package/src/sqs/sqs-subscriber.js +3 -1
- package/src/sqs/sqs-subscriber.test.js +1 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,17 @@
|
|
|
1
1
|
# @defra/grants-config-utils
|
|
2
2
|
|
|
3
|
+
## 1.2.1
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- 0842ad8: Include messageId in attributes for onMessage
|
|
8
|
+
|
|
9
|
+
## 1.2.0
|
|
10
|
+
|
|
11
|
+
### Minor Changes
|
|
12
|
+
|
|
13
|
+
- a4e8348: Allow external configuration of S3 client
|
|
14
|
+
|
|
3
15
|
## 1.1.2
|
|
4
16
|
|
|
5
17
|
### Patch Changes
|
package/package.json
CHANGED
|
@@ -4,16 +4,23 @@ import { ListObjectsV2Command, PutObjectCommand } from "@aws-sdk/client-s3";
|
|
|
4
4
|
|
|
5
5
|
let s3client;
|
|
6
6
|
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
const initialiseClient = (
|
|
7
|
+
let bucketName;
|
|
8
|
+
|
|
9
|
+
export const initialiseClient = ({
|
|
10
|
+
region,
|
|
11
|
+
endpoint,
|
|
12
|
+
forcePathStyle,
|
|
13
|
+
bucketNameOverride,
|
|
14
|
+
} = {}) => {
|
|
10
15
|
if (!s3client) {
|
|
11
16
|
s3client = createS3Client({
|
|
12
|
-
region: config.get("aws.region"),
|
|
13
|
-
endpoint: config.get("aws.endpointUrl"),
|
|
14
|
-
forcePathStyle: config.get("aws.s3.forcePathStyle"),
|
|
17
|
+
region: region ?? config.get("aws.region"),
|
|
18
|
+
endpoint: endpoint ?? config.get("aws.endpointUrl"),
|
|
19
|
+
forcePathStyle: forcePathStyle ?? config.get("aws.s3.forcePathStyle"),
|
|
15
20
|
});
|
|
21
|
+
bucketName = bucketNameOverride ?? config.get("aws.s3.bucketName");
|
|
16
22
|
}
|
|
23
|
+
|
|
17
24
|
return s3client;
|
|
18
25
|
};
|
|
19
26
|
|
|
@@ -1,23 +1,111 @@
|
|
|
1
|
-
import { getBucketName, listFiles, uploadBlob } from "./s3-interactions.js";
|
|
2
1
|
import { createS3Client } from "./s3-client.js";
|
|
3
|
-
import { ListObjectsV2Command } from "@aws-sdk/client-s3";
|
|
2
|
+
import { ListObjectsV2Command, PutObjectCommand } from "@aws-sdk/client-s3";
|
|
4
3
|
|
|
5
4
|
vi.mock("./s3-client.js");
|
|
6
5
|
|
|
7
|
-
const
|
|
6
|
+
const defaultBucketName = "configs-bucket";
|
|
8
7
|
|
|
9
8
|
describe("s3-interactions", () => {
|
|
10
9
|
const mockS3Client = { send: vi.fn() };
|
|
11
10
|
const mockLogger = { info: vi.fn(), warn: vi.fn(), error: vi.fn() };
|
|
11
|
+
let initialiseClient, listFiles, uploadBlob, getBucketName;
|
|
12
12
|
|
|
13
|
-
beforeEach(() => {
|
|
14
|
-
|
|
13
|
+
beforeEach(async () => {
|
|
14
|
+
vi.resetModules();
|
|
15
|
+
vi.clearAllMocks();
|
|
16
|
+
|
|
17
|
+
createS3Client.mockReturnValue(mockS3Client);
|
|
18
|
+
|
|
19
|
+
const s3Interactions = await import("./s3-interactions.js");
|
|
20
|
+
initialiseClient = s3Interactions.initialiseClient;
|
|
21
|
+
listFiles = s3Interactions.listFiles;
|
|
22
|
+
uploadBlob = s3Interactions.uploadBlob;
|
|
23
|
+
getBucketName = s3Interactions.getBucketName;
|
|
15
24
|
});
|
|
16
25
|
|
|
17
26
|
afterEach(() => {
|
|
18
27
|
vi.clearAllMocks();
|
|
19
28
|
});
|
|
20
29
|
|
|
30
|
+
describe("initialiseClient", () => {
|
|
31
|
+
it("should initialise S3 client with default config values when no options provided", () => {
|
|
32
|
+
const client = initialiseClient();
|
|
33
|
+
|
|
34
|
+
expect(createS3Client).toHaveBeenCalledWith({
|
|
35
|
+
region: "eu-west-2",
|
|
36
|
+
endpoint: null,
|
|
37
|
+
forcePathStyle: true,
|
|
38
|
+
});
|
|
39
|
+
expect(client).toBe(mockS3Client);
|
|
40
|
+
expect(getBucketName()).toBe(defaultBucketName);
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
it("should initialise S3 client with default config values when empty object provided", () => {
|
|
44
|
+
const client = initialiseClient({});
|
|
45
|
+
|
|
46
|
+
expect(createS3Client).toHaveBeenCalledWith({
|
|
47
|
+
region: "eu-west-2",
|
|
48
|
+
endpoint: null,
|
|
49
|
+
forcePathStyle: true,
|
|
50
|
+
});
|
|
51
|
+
expect(client).toBe(mockS3Client);
|
|
52
|
+
expect(getBucketName()).toBe(defaultBucketName);
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
it("should initialise S3 client with custom optional configuration options", () => {
|
|
56
|
+
const customOptions = {
|
|
57
|
+
region: "us-east-1",
|
|
58
|
+
endpoint: "http://localhost:4566",
|
|
59
|
+
forcePathStyle: false,
|
|
60
|
+
bucketNameOverride: "custom-bucket",
|
|
61
|
+
};
|
|
62
|
+
|
|
63
|
+
const client = initialiseClient(customOptions);
|
|
64
|
+
|
|
65
|
+
expect(createS3Client).toHaveBeenCalledWith({
|
|
66
|
+
region: "us-east-1",
|
|
67
|
+
endpoint: "http://localhost:4566",
|
|
68
|
+
forcePathStyle: false,
|
|
69
|
+
});
|
|
70
|
+
expect(client).toBe(mockS3Client);
|
|
71
|
+
expect(getBucketName()).toBe("custom-bucket");
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
it("should fallback omitted options to default config values when partial options provided", () => {
|
|
75
|
+
const partialOptions = {
|
|
76
|
+
region: "ap-southeast-1",
|
|
77
|
+
bucketNameOverride: "my-custom-bucket",
|
|
78
|
+
};
|
|
79
|
+
|
|
80
|
+
const client = initialiseClient(partialOptions);
|
|
81
|
+
|
|
82
|
+
expect(createS3Client).toHaveBeenCalledWith({
|
|
83
|
+
region: "ap-southeast-1",
|
|
84
|
+
endpoint: null,
|
|
85
|
+
forcePathStyle: true,
|
|
86
|
+
});
|
|
87
|
+
expect(client).toBe(mockS3Client);
|
|
88
|
+
expect(getBucketName()).toBe("my-custom-bucket");
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
it("should return the existing client on subsequent calls and not re-initialise", () => {
|
|
92
|
+
const client1 = initialiseClient({
|
|
93
|
+
region: "us-west-2",
|
|
94
|
+
bucketNameOverride: "initial-bucket",
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
const client2 = initialiseClient({
|
|
98
|
+
region: "eu-west-1",
|
|
99
|
+
bucketNameOverride: "second-bucket",
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
expect(createS3Client).toHaveBeenCalledTimes(1);
|
|
103
|
+
expect(client1).toBe(mockS3Client);
|
|
104
|
+
expect(client2).toBe(mockS3Client);
|
|
105
|
+
expect(getBucketName()).toBe("initial-bucket");
|
|
106
|
+
});
|
|
107
|
+
});
|
|
108
|
+
|
|
21
109
|
describe("listFiles", () => {
|
|
22
110
|
it("should list files with a given prefix", async () => {
|
|
23
111
|
const prefix = "test-prefix";
|
|
@@ -35,7 +123,7 @@ describe("s3-interactions", () => {
|
|
|
35
123
|
expect(mockS3Client.send).toHaveBeenCalledWith(
|
|
36
124
|
expect.objectContaining({
|
|
37
125
|
input: {
|
|
38
|
-
Bucket:
|
|
126
|
+
Bucket: defaultBucketName,
|
|
39
127
|
Prefix: prefix,
|
|
40
128
|
},
|
|
41
129
|
}),
|
|
@@ -50,6 +138,29 @@ describe("s3-interactions", () => {
|
|
|
50
138
|
);
|
|
51
139
|
});
|
|
52
140
|
|
|
141
|
+
it("should list files using custom bucketNameOverride if initialised beforehand", async () => {
|
|
142
|
+
initialiseClient({ bucketNameOverride: "custom-list-bucket" });
|
|
143
|
+
|
|
144
|
+
const prefix = "custom-prefix";
|
|
145
|
+
const mockListResponse = {
|
|
146
|
+
Contents: [{ Key: "custom-prefix/doc.json" }],
|
|
147
|
+
};
|
|
148
|
+
|
|
149
|
+
mockS3Client.send.mockResolvedValueOnce(mockListResponse);
|
|
150
|
+
|
|
151
|
+
const result = await listFiles(mockLogger, prefix);
|
|
152
|
+
|
|
153
|
+
expect(mockS3Client.send).toHaveBeenCalledWith(
|
|
154
|
+
expect.objectContaining({
|
|
155
|
+
input: {
|
|
156
|
+
Bucket: "custom-list-bucket",
|
|
157
|
+
Prefix: prefix,
|
|
158
|
+
},
|
|
159
|
+
}),
|
|
160
|
+
);
|
|
161
|
+
expect(result).toEqual(mockListResponse.Contents);
|
|
162
|
+
});
|
|
163
|
+
|
|
53
164
|
it("should return an empty array if no files are found", async () => {
|
|
54
165
|
const prefix = "empty-prefix";
|
|
55
166
|
const mockListResponse = {
|
|
@@ -91,37 +202,59 @@ describe("s3-interactions", () => {
|
|
|
91
202
|
expect(mockS3Client.send).toHaveBeenCalledWith(
|
|
92
203
|
expect.objectContaining({
|
|
93
204
|
input: {
|
|
94
|
-
Bucket:
|
|
205
|
+
Bucket: defaultBucketName,
|
|
95
206
|
Key: key,
|
|
96
207
|
Body: body,
|
|
97
208
|
},
|
|
98
209
|
}),
|
|
99
210
|
);
|
|
211
|
+
expect(mockS3Client.send).toHaveBeenCalledWith(
|
|
212
|
+
expect.any(PutObjectCommand),
|
|
213
|
+
);
|
|
100
214
|
expect(result).toEqual(mockPutObjectResponse);
|
|
101
215
|
expect(mockLogger.info).toHaveBeenCalledWith(
|
|
102
216
|
`Uploaded document: ${key}, ETag: ${mockPutObjectResponse.ETag}`,
|
|
103
217
|
);
|
|
104
218
|
});
|
|
105
219
|
|
|
220
|
+
it("should upload blob using custom bucketNameOverride if initialised beforehand", async () => {
|
|
221
|
+
initialiseClient({ bucketNameOverride: "custom-upload-bucket" });
|
|
222
|
+
|
|
223
|
+
const key = "test-custom-key";
|
|
224
|
+
const body = "test-custom-data";
|
|
225
|
+
|
|
226
|
+
const mockPutObjectResponse = { ETag: '"custom-etag"' };
|
|
227
|
+
mockS3Client.send.mockResolvedValueOnce(mockPutObjectResponse);
|
|
228
|
+
|
|
229
|
+
const result = await uploadBlob(mockLogger, key, body);
|
|
230
|
+
|
|
231
|
+
expect(mockS3Client.send).toHaveBeenCalledWith(
|
|
232
|
+
expect.objectContaining({
|
|
233
|
+
input: {
|
|
234
|
+
Bucket: "custom-upload-bucket",
|
|
235
|
+
Key: key,
|
|
236
|
+
Body: body,
|
|
237
|
+
},
|
|
238
|
+
}),
|
|
239
|
+
);
|
|
240
|
+
expect(result).toEqual(mockPutObjectResponse);
|
|
241
|
+
});
|
|
242
|
+
|
|
106
243
|
it("should throw an error if the upload fails", async () => {
|
|
107
244
|
const key = "test-key";
|
|
108
245
|
const body = "test-data";
|
|
109
246
|
|
|
110
247
|
const mockError = new Error("Upload failed");
|
|
111
|
-
mockS3Client.send.mockRejectedValueOnce(mockError);
|
|
112
|
-
createS3Client.mockReturnValueOnce(mockS3Client);
|
|
113
|
-
|
|
114
248
|
mockS3Client.send.mockRejectedValueOnce(mockError);
|
|
115
249
|
|
|
116
250
|
await expect(uploadBlob(mockLogger, key, body)).rejects.toThrow(
|
|
117
251
|
"Upload failed",
|
|
118
252
|
);
|
|
119
253
|
|
|
120
|
-
expect(createS3Client).not.toHaveBeenCalled();
|
|
121
254
|
expect(mockS3Client.send).toHaveBeenCalledWith(
|
|
122
255
|
expect.objectContaining({
|
|
123
256
|
input: {
|
|
124
|
-
Bucket:
|
|
257
|
+
Bucket: defaultBucketName,
|
|
125
258
|
Key: key,
|
|
126
259
|
Body: body,
|
|
127
260
|
},
|
|
@@ -131,9 +264,21 @@ describe("s3-interactions", () => {
|
|
|
131
264
|
});
|
|
132
265
|
|
|
133
266
|
describe("getBucketName", () => {
|
|
134
|
-
it("should return
|
|
267
|
+
it("should return undefined before initialisation", () => {
|
|
268
|
+
const result = getBucketName();
|
|
269
|
+
expect(result).toBeUndefined();
|
|
270
|
+
});
|
|
271
|
+
|
|
272
|
+
it("should return the default bucket name after default initialisation", () => {
|
|
273
|
+
initialiseClient();
|
|
135
274
|
const result = getBucketName();
|
|
136
275
|
expect(result).toEqual("configs-bucket");
|
|
137
276
|
});
|
|
277
|
+
|
|
278
|
+
it("should return the overridden bucket name after custom initialisation", () => {
|
|
279
|
+
initialiseClient({ bucketNameOverride: "overridden-bucket-name" });
|
|
280
|
+
const result = getBucketName();
|
|
281
|
+
expect(result).toEqual("overridden-bucket-name");
|
|
282
|
+
});
|
|
138
283
|
});
|
|
139
284
|
});
|
|
@@ -104,7 +104,9 @@ export class SqsSubscriber {
|
|
|
104
104
|
}
|
|
105
105
|
|
|
106
106
|
extractMessageAttributes(message) {
|
|
107
|
-
const attributes = {
|
|
107
|
+
const attributes = {
|
|
108
|
+
messageId: message.MessageId,
|
|
109
|
+
};
|
|
108
110
|
for (const key in message.MessageAttributes) {
|
|
109
111
|
if (Object.hasOwn(message.MessageAttributes, key)) {
|
|
110
112
|
const attr = message.MessageAttributes[key];
|