@defra/grants-config-utils 1.2.1 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,5 +1,11 @@
1
1
  # @defra/grants-config-utils
2
2
 
3
+ ## 1.3.0
4
+
5
+ ### Minor Changes
6
+
7
+ - 38fc923: Export s3 client. Add paginated list all files
8
+
3
9
  ## 1.2.1
4
10
 
5
11
  ### Patch Changes
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@defra/grants-config-utils",
3
- "version": "1.2.1",
3
+ "version": "1.3.0",
4
4
  "description": "NPM module for providing utils for publishing grants config",
5
5
  "engines": {
6
6
  "node": ">=24",
@@ -30,6 +30,7 @@
30
30
  "main": "src/service/config-store-and-inform.js",
31
31
  "exports": {
32
32
  ".": "./src/service/config-store-and-inform.js",
33
+ "./s3-client": "./src/s3/s3-client.js",
33
34
  "./s3-interactions": "./src/s3/s3-interactions.js",
34
35
  "./sns-client": "./src/sns/sns-client.js",
35
36
  "./sqs-subscriber": "./src/sqs/sqs-subscriber.js",
@@ -1,6 +1,10 @@
1
1
  import { config } from "../config/config.js";
2
2
  import { createS3Client } from "./s3-client.js";
3
- import { ListObjectsV2Command, PutObjectCommand } from "@aws-sdk/client-s3";
3
+ import {
4
+ ListObjectsV2Command,
5
+ paginateListObjectsV2,
6
+ PutObjectCommand,
7
+ } from "@aws-sdk/client-s3";
4
8
 
5
9
  let s3client;
6
10
 
@@ -39,6 +43,23 @@ export const listFiles = async (logger, prefix) => {
39
43
  return result.Contents ?? [];
40
44
  };
41
45
 
46
+ export const listAllFiles = async (logger) => {
47
+ const client = initialiseClient();
48
+ const paginator = paginateListObjectsV2(
49
+ { client },
50
+ {
51
+ Bucket: bucketName,
52
+ },
53
+ );
54
+
55
+ const objectKeys = [];
56
+ for await (const { Contents } of paginator) {
57
+ objectKeys.push(...Contents.map((obj) => ({ Key: obj.Key })));
58
+ }
59
+ logger.info(`Found ${objectKeys.length ?? 0} files in bucket ${bucketName}`);
60
+ return objectKeys;
61
+ };
62
+
42
63
  export const uploadBlob = async (logger, filename, contents) => {
43
64
  const client = initialiseClient();
44
65
 
@@ -1,14 +1,20 @@
1
1
  import { createS3Client } from "./s3-client.js";
2
- import { ListObjectsV2Command, PutObjectCommand } from "@aws-sdk/client-s3";
2
+ import {
3
+ S3Client,
4
+ ListObjectsV2Command,
5
+ PutObjectCommand,
6
+ } from "@aws-sdk/client-s3";
3
7
 
4
8
  vi.mock("./s3-client.js");
5
9
 
6
10
  const defaultBucketName = "configs-bucket";
7
11
 
8
12
  describe("s3-interactions", () => {
9
- const mockS3Client = { send: vi.fn() };
13
+ const mockS3Client = Object.assign(Object.create(S3Client.prototype), {
14
+ send: vi.fn(),
15
+ });
10
16
  const mockLogger = { info: vi.fn(), warn: vi.fn(), error: vi.fn() };
11
- let initialiseClient, listFiles, uploadBlob, getBucketName;
17
+ let initialiseClient, listFiles, listAllFiles, uploadBlob, getBucketName;
12
18
 
13
19
  beforeEach(async () => {
14
20
  vi.resetModules();
@@ -19,6 +25,7 @@ describe("s3-interactions", () => {
19
25
  const s3Interactions = await import("./s3-interactions.js");
20
26
  initialiseClient = s3Interactions.initialiseClient;
21
27
  listFiles = s3Interactions.listFiles;
28
+ listAllFiles = s3Interactions.listAllFiles;
22
29
  uploadBlob = s3Interactions.uploadBlob;
23
30
  getBucketName = s3Interactions.getBucketName;
24
31
  });
@@ -189,6 +196,115 @@ describe("s3-interactions", () => {
189
196
  });
190
197
  });
191
198
 
199
+ describe("listAllFiles", () => {
200
+ it("should list all files in the bucket", async () => {
201
+ const mockListResponse = {
202
+ Contents: [{ Key: "file1.txt" }, { Key: "file2.txt" }],
203
+ IsTruncated: false,
204
+ };
205
+
206
+ mockS3Client.send.mockResolvedValueOnce(mockListResponse);
207
+
208
+ const result = await listAllFiles(mockLogger);
209
+
210
+ expect(mockS3Client.send).toHaveBeenCalledWith(
211
+ expect.objectContaining({
212
+ input: {
213
+ Bucket: defaultBucketName,
214
+ },
215
+ }),
216
+ );
217
+ expect(result).toEqual([{ Key: "file1.txt" }, { Key: "file2.txt" }]);
218
+ expect(mockLogger.info).toHaveBeenCalledWith(
219
+ `Found 2 files in bucket ${defaultBucketName}`,
220
+ );
221
+ });
222
+
223
+ it("should paginate across multiple pages of files", async () => {
224
+ const page1 = {
225
+ Contents: [{ Key: "file1.txt" }],
226
+ IsTruncated: true,
227
+ NextContinuationToken: "token-page-2",
228
+ };
229
+ const page2 = {
230
+ Contents: [{ Key: "file2.txt" }, { Key: "file3.txt" }],
231
+ IsTruncated: false,
232
+ };
233
+
234
+ mockS3Client.send
235
+ .mockResolvedValueOnce(page1)
236
+ .mockResolvedValueOnce(page2);
237
+
238
+ const result = await listAllFiles(mockLogger);
239
+
240
+ expect(mockS3Client.send).toHaveBeenCalledTimes(2);
241
+ expect(mockS3Client.send).toHaveBeenCalledWith(
242
+ expect.objectContaining({
243
+ input: expect.objectContaining({
244
+ Bucket: defaultBucketName,
245
+ }),
246
+ }),
247
+ );
248
+ expect(result).toEqual([
249
+ { Key: "file1.txt" },
250
+ { Key: "file2.txt" },
251
+ { Key: "file3.txt" },
252
+ ]);
253
+ expect(mockLogger.info).toHaveBeenCalledWith(
254
+ `Found 3 files in bucket ${defaultBucketName}`,
255
+ );
256
+ });
257
+
258
+ it("should list all files using custom bucketNameOverride if initialised beforehand", async () => {
259
+ initialiseClient({ bucketNameOverride: "custom-list-all-bucket" });
260
+
261
+ const mockListResponse = {
262
+ Contents: [{ Key: "doc.json" }],
263
+ IsTruncated: false,
264
+ };
265
+
266
+ mockS3Client.send.mockResolvedValueOnce(mockListResponse);
267
+
268
+ const result = await listAllFiles(mockLogger);
269
+
270
+ expect(mockS3Client.send).toHaveBeenCalledWith(
271
+ expect.objectContaining({
272
+ input: {
273
+ Bucket: "custom-list-all-bucket",
274
+ },
275
+ }),
276
+ );
277
+ expect(result).toEqual([{ Key: "doc.json" }]);
278
+ expect(mockLogger.info).toHaveBeenCalledWith(
279
+ "Found 1 files in bucket custom-list-all-bucket",
280
+ );
281
+ });
282
+
283
+ it("should return an empty array if bucket has no files", async () => {
284
+ const mockListResponse = {
285
+ Contents: [],
286
+ IsTruncated: false,
287
+ };
288
+
289
+ mockS3Client.send.mockResolvedValueOnce(mockListResponse);
290
+
291
+ const result = await listAllFiles(mockLogger);
292
+
293
+ expect(result).toEqual([]);
294
+ expect(mockLogger.info).toHaveBeenCalledWith(
295
+ `Found 0 files in bucket ${defaultBucketName}`,
296
+ );
297
+ });
298
+
299
+ it("should throw an error if listing fails", async () => {
300
+ const mockError = new Error("List all failed");
301
+
302
+ mockS3Client.send.mockRejectedValueOnce(mockError);
303
+
304
+ await expect(listAllFiles(mockLogger)).rejects.toThrow("List all failed");
305
+ });
306
+ });
307
+
192
308
  describe("uploadBlob", () => {
193
309
  it("should upload a blob to the S3 bucket with specified key", async () => {
194
310
  const key = "test-key";