@defra/grants-config-utils 1.2.1 → 1.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -54,19 +54,18 @@ jobs:
54
54
 
55
55
  - name: Create Release Pull Request or Release
56
56
  id: changesets
57
- uses: changesets/action@a45c4d594aa4e2c509dc14a9f2b3b67ba3780d0d #v1.9.0
57
+ uses: changesets/action@8488615a623b1b9c987934bb89eae8af6a946ac1 #v2.1.1
58
58
  with:
59
- commitMode: 'github-api'
60
- publish: npm run github:release
61
- commit: 'Create new config version'
62
- title: 'Create new config version'
59
+ publish-script: npm run github:release
60
+ commit-message: 'Create new config version'
61
+ pr-title: 'Create new config version'
63
62
 
64
63
  - name: Trigger required checks on release PR, then merge to main, and delete release branch
65
64
  if: ${{ !startsWith(github.event_name == 'push' && github.event.head_commit.message || inputs.commit_message, 'Create new config version') }}
66
65
  uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 #v9.0.0
67
66
  with:
68
67
  script: |
69
- const prNumber = Number("${{ steps.changesets.outputs.pullRequestNumber }}");
68
+ const prNumber = Number("${{ steps.changesets.outputs.pr-number }}");
70
69
  const { data: releasePR } = await github.rest.pulls.get({
71
70
  owner: context.repo.owner,
72
71
  repo: context.repo.repo,
package/CHANGELOG.md CHANGED
@@ -1,5 +1,17 @@
1
1
  # @defra/grants-config-utils
2
2
 
3
+ ## 1.3.1
4
+
5
+ ### Patch Changes
6
+
7
+ - 6ad0254: Dependency updates and changesets major version upgrade
8
+
9
+ ## 1.3.0
10
+
11
+ ### Minor Changes
12
+
13
+ - 38fc923: Export s3 client. Add paginated list all files
14
+
3
15
  ## 1.2.1
4
16
 
5
17
  ### Patch Changes
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@defra/grants-config-utils",
3
- "version": "1.2.1",
3
+ "version": "1.3.1",
4
4
  "description": "NPM module for providing utils for publishing grants config",
5
5
  "engines": {
6
6
  "node": ">=24",
@@ -30,6 +30,7 @@
30
30
  "main": "src/service/config-store-and-inform.js",
31
31
  "exports": {
32
32
  ".": "./src/service/config-store-and-inform.js",
33
+ "./s3-client": "./src/s3/s3-client.js",
33
34
  "./s3-interactions": "./src/s3/s3-interactions.js",
34
35
  "./sns-client": "./src/sns/sns-client.js",
35
36
  "./sqs-subscriber": "./src/sqs/sqs-subscriber.js",
@@ -45,10 +46,10 @@
45
46
  },
46
47
  "homepage": "https://github.com/DEFRA/grants-config-utils#readme",
47
48
  "dependencies": {
48
- "@aws-sdk/client-s3": "3.1102.0",
49
- "@aws-sdk/client-sns": "3.1102.0",
50
- "@aws-sdk/client-sqs": "3.1102.0",
51
- "@aws-sdk/client-sts": "3.1102.0",
49
+ "@aws-sdk/client-s3": "3.1112.0",
50
+ "@aws-sdk/client-sns": "3.1112.0",
51
+ "@aws-sdk/client-sqs": "3.1112.0",
52
+ "@aws-sdk/client-sts": "3.1112.0",
52
53
  "brace-expansion": "5.0.9",
53
54
  "convict": "6.2.5",
54
55
  "convict-format-with-validator": "6.2.0"
@@ -56,8 +57,8 @@
56
57
  "devDependencies": {
57
58
  "@babel/core": "8.0.1",
58
59
  "@babel/preset-env": "8.0.2",
59
- "@changesets/changelog-github": "0.7.0",
60
- "@changesets/cli": "2.31.1",
60
+ "@changesets/changelog-github": "1.0.0",
61
+ "@changesets/cli": "3.0.1",
61
62
  "@vitest/coverage-v8": "4.1.10",
62
63
  "eslint": "10.7.0",
63
64
  "eslint-config-prettier": "10.1.1",
@@ -1,6 +1,10 @@
1
1
  import { config } from "../config/config.js";
2
2
  import { createS3Client } from "./s3-client.js";
3
- import { ListObjectsV2Command, PutObjectCommand } from "@aws-sdk/client-s3";
3
+ import {
4
+ ListObjectsV2Command,
5
+ paginateListObjectsV2,
6
+ PutObjectCommand,
7
+ } from "@aws-sdk/client-s3";
4
8
 
5
9
  let s3client;
6
10
 
@@ -39,6 +43,23 @@ export const listFiles = async (logger, prefix) => {
39
43
  return result.Contents ?? [];
40
44
  };
41
45
 
46
+ export const listAllFiles = async (logger) => {
47
+ const client = initialiseClient();
48
+ const paginator = paginateListObjectsV2(
49
+ { client },
50
+ {
51
+ Bucket: bucketName,
52
+ },
53
+ );
54
+
55
+ const objectKeys = [];
56
+ for await (const { Contents } of paginator) {
57
+ objectKeys.push(...Contents.map((obj) => ({ Key: obj.Key })));
58
+ }
59
+ logger.info(`Found ${objectKeys.length ?? 0} files in bucket ${bucketName}`);
60
+ return objectKeys;
61
+ };
62
+
42
63
  export const uploadBlob = async (logger, filename, contents) => {
43
64
  const client = initialiseClient();
44
65
 
@@ -1,14 +1,20 @@
1
1
  import { createS3Client } from "./s3-client.js";
2
- import { ListObjectsV2Command, PutObjectCommand } from "@aws-sdk/client-s3";
2
+ import {
3
+ S3Client,
4
+ ListObjectsV2Command,
5
+ PutObjectCommand,
6
+ } from "@aws-sdk/client-s3";
3
7
 
4
8
  vi.mock("./s3-client.js");
5
9
 
6
10
  const defaultBucketName = "configs-bucket";
7
11
 
8
12
  describe("s3-interactions", () => {
9
- const mockS3Client = { send: vi.fn() };
13
+ const mockS3Client = Object.assign(Object.create(S3Client.prototype), {
14
+ send: vi.fn(),
15
+ });
10
16
  const mockLogger = { info: vi.fn(), warn: vi.fn(), error: vi.fn() };
11
- let initialiseClient, listFiles, uploadBlob, getBucketName;
17
+ let initialiseClient, listFiles, listAllFiles, uploadBlob, getBucketName;
12
18
 
13
19
  beforeEach(async () => {
14
20
  vi.resetModules();
@@ -19,6 +25,7 @@ describe("s3-interactions", () => {
19
25
  const s3Interactions = await import("./s3-interactions.js");
20
26
  initialiseClient = s3Interactions.initialiseClient;
21
27
  listFiles = s3Interactions.listFiles;
28
+ listAllFiles = s3Interactions.listAllFiles;
22
29
  uploadBlob = s3Interactions.uploadBlob;
23
30
  getBucketName = s3Interactions.getBucketName;
24
31
  });
@@ -189,6 +196,115 @@ describe("s3-interactions", () => {
189
196
  });
190
197
  });
191
198
 
199
+ describe("listAllFiles", () => {
200
+ it("should list all files in the bucket", async () => {
201
+ const mockListResponse = {
202
+ Contents: [{ Key: "file1.txt" }, { Key: "file2.txt" }],
203
+ IsTruncated: false,
204
+ };
205
+
206
+ mockS3Client.send.mockResolvedValueOnce(mockListResponse);
207
+
208
+ const result = await listAllFiles(mockLogger);
209
+
210
+ expect(mockS3Client.send).toHaveBeenCalledWith(
211
+ expect.objectContaining({
212
+ input: {
213
+ Bucket: defaultBucketName,
214
+ },
215
+ }),
216
+ );
217
+ expect(result).toEqual([{ Key: "file1.txt" }, { Key: "file2.txt" }]);
218
+ expect(mockLogger.info).toHaveBeenCalledWith(
219
+ `Found 2 files in bucket ${defaultBucketName}`,
220
+ );
221
+ });
222
+
223
+ it("should paginate across multiple pages of files", async () => {
224
+ const page1 = {
225
+ Contents: [{ Key: "file1.txt" }],
226
+ IsTruncated: true,
227
+ NextContinuationToken: "token-page-2",
228
+ };
229
+ const page2 = {
230
+ Contents: [{ Key: "file2.txt" }, { Key: "file3.txt" }],
231
+ IsTruncated: false,
232
+ };
233
+
234
+ mockS3Client.send
235
+ .mockResolvedValueOnce(page1)
236
+ .mockResolvedValueOnce(page2);
237
+
238
+ const result = await listAllFiles(mockLogger);
239
+
240
+ expect(mockS3Client.send).toHaveBeenCalledTimes(2);
241
+ expect(mockS3Client.send).toHaveBeenCalledWith(
242
+ expect.objectContaining({
243
+ input: expect.objectContaining({
244
+ Bucket: defaultBucketName,
245
+ }),
246
+ }),
247
+ );
248
+ expect(result).toEqual([
249
+ { Key: "file1.txt" },
250
+ { Key: "file2.txt" },
251
+ { Key: "file3.txt" },
252
+ ]);
253
+ expect(mockLogger.info).toHaveBeenCalledWith(
254
+ `Found 3 files in bucket ${defaultBucketName}`,
255
+ );
256
+ });
257
+
258
+ it("should list all files using custom bucketNameOverride if initialised beforehand", async () => {
259
+ initialiseClient({ bucketNameOverride: "custom-list-all-bucket" });
260
+
261
+ const mockListResponse = {
262
+ Contents: [{ Key: "doc.json" }],
263
+ IsTruncated: false,
264
+ };
265
+
266
+ mockS3Client.send.mockResolvedValueOnce(mockListResponse);
267
+
268
+ const result = await listAllFiles(mockLogger);
269
+
270
+ expect(mockS3Client.send).toHaveBeenCalledWith(
271
+ expect.objectContaining({
272
+ input: {
273
+ Bucket: "custom-list-all-bucket",
274
+ },
275
+ }),
276
+ );
277
+ expect(result).toEqual([{ Key: "doc.json" }]);
278
+ expect(mockLogger.info).toHaveBeenCalledWith(
279
+ "Found 1 files in bucket custom-list-all-bucket",
280
+ );
281
+ });
282
+
283
+ it("should return an empty array if bucket has no files", async () => {
284
+ const mockListResponse = {
285
+ Contents: [],
286
+ IsTruncated: false,
287
+ };
288
+
289
+ mockS3Client.send.mockResolvedValueOnce(mockListResponse);
290
+
291
+ const result = await listAllFiles(mockLogger);
292
+
293
+ expect(result).toEqual([]);
294
+ expect(mockLogger.info).toHaveBeenCalledWith(
295
+ `Found 0 files in bucket ${defaultBucketName}`,
296
+ );
297
+ });
298
+
299
+ it("should throw an error if listing fails", async () => {
300
+ const mockError = new Error("List all failed");
301
+
302
+ mockS3Client.send.mockRejectedValueOnce(mockError);
303
+
304
+ await expect(listAllFiles(mockLogger)).rejects.toThrow("List all failed");
305
+ });
306
+ });
307
+
192
308
  describe("uploadBlob", () => {
193
309
  it("should upload a blob to the S3 bucket with specified key", async () => {
194
310
  const key = "test-key";