@defra/grants-config-utils 1.2.1 → 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/release.yml +5 -6
- package/CHANGELOG.md +12 -0
- package/package.json +8 -7
- package/src/s3/s3-interactions.js +22 -1
- package/src/s3/s3-interactions.test.js +119 -3
|
@@ -54,19 +54,18 @@ jobs:
|
|
|
54
54
|
|
|
55
55
|
- name: Create Release Pull Request or Release
|
|
56
56
|
id: changesets
|
|
57
|
-
uses: changesets/action@
|
|
57
|
+
uses: changesets/action@8488615a623b1b9c987934bb89eae8af6a946ac1 #v2.1.1
|
|
58
58
|
with:
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
title: 'Create new config version'
|
|
59
|
+
publish-script: npm run github:release
|
|
60
|
+
commit-message: 'Create new config version'
|
|
61
|
+
pr-title: 'Create new config version'
|
|
63
62
|
|
|
64
63
|
- name: Trigger required checks on release PR, then merge to main, and delete release branch
|
|
65
64
|
if: ${{ !startsWith(github.event_name == 'push' && github.event.head_commit.message || inputs.commit_message, 'Create new config version') }}
|
|
66
65
|
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 #v9.0.0
|
|
67
66
|
with:
|
|
68
67
|
script: |
|
|
69
|
-
const prNumber = Number("${{ steps.changesets.outputs.
|
|
68
|
+
const prNumber = Number("${{ steps.changesets.outputs.pr-number }}");
|
|
70
69
|
const { data: releasePR } = await github.rest.pulls.get({
|
|
71
70
|
owner: context.repo.owner,
|
|
72
71
|
repo: context.repo.repo,
|
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,17 @@
|
|
|
1
1
|
# @defra/grants-config-utils
|
|
2
2
|
|
|
3
|
+
## 1.3.1
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- 6ad0254: Dependency updates and changesets major version upgrade
|
|
8
|
+
|
|
9
|
+
## 1.3.0
|
|
10
|
+
|
|
11
|
+
### Minor Changes
|
|
12
|
+
|
|
13
|
+
- 38fc923: Export s3 client. Add paginated list all files
|
|
14
|
+
|
|
3
15
|
## 1.2.1
|
|
4
16
|
|
|
5
17
|
### Patch Changes
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@defra/grants-config-utils",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.3.1",
|
|
4
4
|
"description": "NPM module for providing utils for publishing grants config",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=24",
|
|
@@ -30,6 +30,7 @@
|
|
|
30
30
|
"main": "src/service/config-store-and-inform.js",
|
|
31
31
|
"exports": {
|
|
32
32
|
".": "./src/service/config-store-and-inform.js",
|
|
33
|
+
"./s3-client": "./src/s3/s3-client.js",
|
|
33
34
|
"./s3-interactions": "./src/s3/s3-interactions.js",
|
|
34
35
|
"./sns-client": "./src/sns/sns-client.js",
|
|
35
36
|
"./sqs-subscriber": "./src/sqs/sqs-subscriber.js",
|
|
@@ -45,10 +46,10 @@
|
|
|
45
46
|
},
|
|
46
47
|
"homepage": "https://github.com/DEFRA/grants-config-utils#readme",
|
|
47
48
|
"dependencies": {
|
|
48
|
-
"@aws-sdk/client-s3": "3.
|
|
49
|
-
"@aws-sdk/client-sns": "3.
|
|
50
|
-
"@aws-sdk/client-sqs": "3.
|
|
51
|
-
"@aws-sdk/client-sts": "3.
|
|
49
|
+
"@aws-sdk/client-s3": "3.1112.0",
|
|
50
|
+
"@aws-sdk/client-sns": "3.1112.0",
|
|
51
|
+
"@aws-sdk/client-sqs": "3.1112.0",
|
|
52
|
+
"@aws-sdk/client-sts": "3.1112.0",
|
|
52
53
|
"brace-expansion": "5.0.9",
|
|
53
54
|
"convict": "6.2.5",
|
|
54
55
|
"convict-format-with-validator": "6.2.0"
|
|
@@ -56,8 +57,8 @@
|
|
|
56
57
|
"devDependencies": {
|
|
57
58
|
"@babel/core": "8.0.1",
|
|
58
59
|
"@babel/preset-env": "8.0.2",
|
|
59
|
-
"@changesets/changelog-github": "0.
|
|
60
|
-
"@changesets/cli": "
|
|
60
|
+
"@changesets/changelog-github": "1.0.0",
|
|
61
|
+
"@changesets/cli": "3.0.1",
|
|
61
62
|
"@vitest/coverage-v8": "4.1.10",
|
|
62
63
|
"eslint": "10.7.0",
|
|
63
64
|
"eslint-config-prettier": "10.1.1",
|
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
import { config } from "../config/config.js";
|
|
2
2
|
import { createS3Client } from "./s3-client.js";
|
|
3
|
-
import {
|
|
3
|
+
import {
|
|
4
|
+
ListObjectsV2Command,
|
|
5
|
+
paginateListObjectsV2,
|
|
6
|
+
PutObjectCommand,
|
|
7
|
+
} from "@aws-sdk/client-s3";
|
|
4
8
|
|
|
5
9
|
let s3client;
|
|
6
10
|
|
|
@@ -39,6 +43,23 @@ export const listFiles = async (logger, prefix) => {
|
|
|
39
43
|
return result.Contents ?? [];
|
|
40
44
|
};
|
|
41
45
|
|
|
46
|
+
export const listAllFiles = async (logger) => {
|
|
47
|
+
const client = initialiseClient();
|
|
48
|
+
const paginator = paginateListObjectsV2(
|
|
49
|
+
{ client },
|
|
50
|
+
{
|
|
51
|
+
Bucket: bucketName,
|
|
52
|
+
},
|
|
53
|
+
);
|
|
54
|
+
|
|
55
|
+
const objectKeys = [];
|
|
56
|
+
for await (const { Contents } of paginator) {
|
|
57
|
+
objectKeys.push(...Contents.map((obj) => ({ Key: obj.Key })));
|
|
58
|
+
}
|
|
59
|
+
logger.info(`Found ${objectKeys.length ?? 0} files in bucket ${bucketName}`);
|
|
60
|
+
return objectKeys;
|
|
61
|
+
};
|
|
62
|
+
|
|
42
63
|
export const uploadBlob = async (logger, filename, contents) => {
|
|
43
64
|
const client = initialiseClient();
|
|
44
65
|
|
|
@@ -1,14 +1,20 @@
|
|
|
1
1
|
import { createS3Client } from "./s3-client.js";
|
|
2
|
-
import {
|
|
2
|
+
import {
|
|
3
|
+
S3Client,
|
|
4
|
+
ListObjectsV2Command,
|
|
5
|
+
PutObjectCommand,
|
|
6
|
+
} from "@aws-sdk/client-s3";
|
|
3
7
|
|
|
4
8
|
vi.mock("./s3-client.js");
|
|
5
9
|
|
|
6
10
|
const defaultBucketName = "configs-bucket";
|
|
7
11
|
|
|
8
12
|
describe("s3-interactions", () => {
|
|
9
|
-
const mockS3Client =
|
|
13
|
+
const mockS3Client = Object.assign(Object.create(S3Client.prototype), {
|
|
14
|
+
send: vi.fn(),
|
|
15
|
+
});
|
|
10
16
|
const mockLogger = { info: vi.fn(), warn: vi.fn(), error: vi.fn() };
|
|
11
|
-
let initialiseClient, listFiles, uploadBlob, getBucketName;
|
|
17
|
+
let initialiseClient, listFiles, listAllFiles, uploadBlob, getBucketName;
|
|
12
18
|
|
|
13
19
|
beforeEach(async () => {
|
|
14
20
|
vi.resetModules();
|
|
@@ -19,6 +25,7 @@ describe("s3-interactions", () => {
|
|
|
19
25
|
const s3Interactions = await import("./s3-interactions.js");
|
|
20
26
|
initialiseClient = s3Interactions.initialiseClient;
|
|
21
27
|
listFiles = s3Interactions.listFiles;
|
|
28
|
+
listAllFiles = s3Interactions.listAllFiles;
|
|
22
29
|
uploadBlob = s3Interactions.uploadBlob;
|
|
23
30
|
getBucketName = s3Interactions.getBucketName;
|
|
24
31
|
});
|
|
@@ -189,6 +196,115 @@ describe("s3-interactions", () => {
|
|
|
189
196
|
});
|
|
190
197
|
});
|
|
191
198
|
|
|
199
|
+
describe("listAllFiles", () => {
|
|
200
|
+
it("should list all files in the bucket", async () => {
|
|
201
|
+
const mockListResponse = {
|
|
202
|
+
Contents: [{ Key: "file1.txt" }, { Key: "file2.txt" }],
|
|
203
|
+
IsTruncated: false,
|
|
204
|
+
};
|
|
205
|
+
|
|
206
|
+
mockS3Client.send.mockResolvedValueOnce(mockListResponse);
|
|
207
|
+
|
|
208
|
+
const result = await listAllFiles(mockLogger);
|
|
209
|
+
|
|
210
|
+
expect(mockS3Client.send).toHaveBeenCalledWith(
|
|
211
|
+
expect.objectContaining({
|
|
212
|
+
input: {
|
|
213
|
+
Bucket: defaultBucketName,
|
|
214
|
+
},
|
|
215
|
+
}),
|
|
216
|
+
);
|
|
217
|
+
expect(result).toEqual([{ Key: "file1.txt" }, { Key: "file2.txt" }]);
|
|
218
|
+
expect(mockLogger.info).toHaveBeenCalledWith(
|
|
219
|
+
`Found 2 files in bucket ${defaultBucketName}`,
|
|
220
|
+
);
|
|
221
|
+
});
|
|
222
|
+
|
|
223
|
+
it("should paginate across multiple pages of files", async () => {
|
|
224
|
+
const page1 = {
|
|
225
|
+
Contents: [{ Key: "file1.txt" }],
|
|
226
|
+
IsTruncated: true,
|
|
227
|
+
NextContinuationToken: "token-page-2",
|
|
228
|
+
};
|
|
229
|
+
const page2 = {
|
|
230
|
+
Contents: [{ Key: "file2.txt" }, { Key: "file3.txt" }],
|
|
231
|
+
IsTruncated: false,
|
|
232
|
+
};
|
|
233
|
+
|
|
234
|
+
mockS3Client.send
|
|
235
|
+
.mockResolvedValueOnce(page1)
|
|
236
|
+
.mockResolvedValueOnce(page2);
|
|
237
|
+
|
|
238
|
+
const result = await listAllFiles(mockLogger);
|
|
239
|
+
|
|
240
|
+
expect(mockS3Client.send).toHaveBeenCalledTimes(2);
|
|
241
|
+
expect(mockS3Client.send).toHaveBeenCalledWith(
|
|
242
|
+
expect.objectContaining({
|
|
243
|
+
input: expect.objectContaining({
|
|
244
|
+
Bucket: defaultBucketName,
|
|
245
|
+
}),
|
|
246
|
+
}),
|
|
247
|
+
);
|
|
248
|
+
expect(result).toEqual([
|
|
249
|
+
{ Key: "file1.txt" },
|
|
250
|
+
{ Key: "file2.txt" },
|
|
251
|
+
{ Key: "file3.txt" },
|
|
252
|
+
]);
|
|
253
|
+
expect(mockLogger.info).toHaveBeenCalledWith(
|
|
254
|
+
`Found 3 files in bucket ${defaultBucketName}`,
|
|
255
|
+
);
|
|
256
|
+
});
|
|
257
|
+
|
|
258
|
+
it("should list all files using custom bucketNameOverride if initialised beforehand", async () => {
|
|
259
|
+
initialiseClient({ bucketNameOverride: "custom-list-all-bucket" });
|
|
260
|
+
|
|
261
|
+
const mockListResponse = {
|
|
262
|
+
Contents: [{ Key: "doc.json" }],
|
|
263
|
+
IsTruncated: false,
|
|
264
|
+
};
|
|
265
|
+
|
|
266
|
+
mockS3Client.send.mockResolvedValueOnce(mockListResponse);
|
|
267
|
+
|
|
268
|
+
const result = await listAllFiles(mockLogger);
|
|
269
|
+
|
|
270
|
+
expect(mockS3Client.send).toHaveBeenCalledWith(
|
|
271
|
+
expect.objectContaining({
|
|
272
|
+
input: {
|
|
273
|
+
Bucket: "custom-list-all-bucket",
|
|
274
|
+
},
|
|
275
|
+
}),
|
|
276
|
+
);
|
|
277
|
+
expect(result).toEqual([{ Key: "doc.json" }]);
|
|
278
|
+
expect(mockLogger.info).toHaveBeenCalledWith(
|
|
279
|
+
"Found 1 files in bucket custom-list-all-bucket",
|
|
280
|
+
);
|
|
281
|
+
});
|
|
282
|
+
|
|
283
|
+
it("should return an empty array if bucket has no files", async () => {
|
|
284
|
+
const mockListResponse = {
|
|
285
|
+
Contents: [],
|
|
286
|
+
IsTruncated: false,
|
|
287
|
+
};
|
|
288
|
+
|
|
289
|
+
mockS3Client.send.mockResolvedValueOnce(mockListResponse);
|
|
290
|
+
|
|
291
|
+
const result = await listAllFiles(mockLogger);
|
|
292
|
+
|
|
293
|
+
expect(result).toEqual([]);
|
|
294
|
+
expect(mockLogger.info).toHaveBeenCalledWith(
|
|
295
|
+
`Found 0 files in bucket ${defaultBucketName}`,
|
|
296
|
+
);
|
|
297
|
+
});
|
|
298
|
+
|
|
299
|
+
it("should throw an error if listing fails", async () => {
|
|
300
|
+
const mockError = new Error("List all failed");
|
|
301
|
+
|
|
302
|
+
mockS3Client.send.mockRejectedValueOnce(mockError);
|
|
303
|
+
|
|
304
|
+
await expect(listAllFiles(mockLogger)).rejects.toThrow("List all failed");
|
|
305
|
+
});
|
|
306
|
+
});
|
|
307
|
+
|
|
192
308
|
describe("uploadBlob", () => {
|
|
193
309
|
it("should upload a blob to the S3 bucket with specified key", async () => {
|
|
194
310
|
const key = "test-key";
|