@defra/grants-config-utils 1.0.0 → 1.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,5 +1,17 @@
1
1
  # @defra/grants-config-utils
2
2
 
3
+ ## 1.1.1
4
+
5
+ ### Patch Changes
6
+
7
+ - 435e2a9: Export trace parent
8
+
9
+ ## 1.1.0
10
+
11
+ ### Minor Changes
12
+
13
+ - 385953a: Add SQS subscriber. Provide deployer name for config publishes
14
+
3
15
  ## 1.0.0
4
16
 
5
17
  ### Major Changes
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@defra/grants-config-utils",
3
- "version": "1.0.0",
3
+ "version": "1.1.1",
4
4
  "description": "NPM module for providing utils for publishing grants config",
5
5
  "engines": {
6
6
  "node": ">=24",
@@ -32,6 +32,8 @@
32
32
  ".": "./src/service/config-store-and-inform.js",
33
33
  "./s3-interactions": "./src/s3/s3-interactions.js",
34
34
  "./sns-client": "./src/sns/sns-client.js",
35
+ "./sqs-subscriber": "./src/sqs/sqs-subscriber.js",
36
+ "./trace-parent": "./src/sqs/trace-parent.js",
35
37
  "./config": "./src/config/config.js",
36
38
  "./broker": "./src/broker/broker-auth-helper.js",
37
39
  "./sts-client-plugin": "./src/sts/sts-client-plugin.js",
@@ -43,29 +45,31 @@
43
45
  },
44
46
  "homepage": "https://github.com/DEFRA/grants-config-utils#readme",
45
47
  "dependencies": {
46
- "@aws-sdk/client-s3": "3.1091.0",
47
- "@aws-sdk/client-sns": "3.1091.0",
48
- "@aws-sdk/client-sts": "3.1091.0",
48
+ "@aws-sdk/client-s3": "3.1095.0",
49
+ "@aws-sdk/client-sns": "3.1095.0",
50
+ "@aws-sdk/client-sqs": "3.1095.0",
51
+ "@aws-sdk/client-sts": "3.1095.0",
52
+ "brace-expansion": "5.0.9",
49
53
  "convict": "6.2.5",
50
54
  "convict-format-with-validator": "6.2.0"
51
55
  },
52
56
  "devDependencies": {
53
- "@babel/core": "7.29.7",
54
- "@babel/preset-env": "7.29.7",
57
+ "@babel/core": "8.0.1",
58
+ "@babel/preset-env": "8.0.2",
55
59
  "@changesets/changelog-github": "0.7.0",
56
60
  "@changesets/cli": "2.31.1",
57
61
  "@vitest/coverage-v8": "4.1.10",
58
62
  "eslint": "10.7.0",
59
63
  "eslint-config-prettier": "10.1.1",
60
64
  "husky": "9.1.7",
61
- "lint-staged": "17.1.0",
65
+ "lint-staged": "17.2.0",
62
66
  "neostandard": "0.13.0",
63
- "prettier": "3.9.5",
67
+ "prettier": "3.9.6",
64
68
  "vitest": "4.1.10",
65
69
  "vitest-fetch-mock": "0.4.5"
66
70
  },
67
71
  "overrides": {
68
- "brace-expansion": "5.0.8",
72
+ "brace-expansion": "5.0.9",
69
73
  "eslint": "10.7.0"
70
74
  },
71
75
  "release-it": {
@@ -77,6 +77,12 @@ export const config = convict({
77
77
  default: "active",
78
78
  env: "GRANTS_CONFIG_PUBLISH_STATUS",
79
79
  },
80
+ user: {
81
+ doc: "Config published by user",
82
+ format: String,
83
+ default: "system",
84
+ env: "SERVICE_DEPLOYED_BY",
85
+ },
80
86
  },
81
87
  });
82
88
 
@@ -106,6 +106,7 @@ const notifyConfigBrokerServiceVersionAvailable = async (
106
106
  ) => {
107
107
  const configSNSTopic = config.get("aws.sns.configVersionTopicArn");
108
108
  const configPublishStatus = config.get("configPublish.status");
109
+ const configPublishUser = config.get("configPublish.user");
109
110
 
110
111
  // iterate each grant configuration, notify config available at current service version
111
112
  for (const configAtServiceVersion of configsAtServiceVersion) {
@@ -113,6 +114,7 @@ const notifyConfigBrokerServiceVersionAvailable = async (
113
114
  configSNSTopic,
114
115
  configAtServiceVersion,
115
116
  configPublishStatus,
117
+ configPublishUser,
116
118
  logger,
117
119
  );
118
120
  }
@@ -122,21 +124,19 @@ const sendConfigMessageToBroker = async (
122
124
  configSNSTopic,
123
125
  configAtServiceVersion,
124
126
  configPublishStatus,
127
+ configPublishUser,
125
128
  logger,
126
129
  ) => {
127
130
  const { grant, version, files } = configAtServiceVersion;
128
131
  // files is an array of tuples, we only want the S3 paths here
129
132
  const s3Paths = files.map(([_, s3Path]) => s3Path);
130
133
 
131
- // When CDP is updated to inject user that did deployment of container, reference that injected variable from config here
132
- const user = "system";
133
-
134
134
  const payload = {
135
135
  grant,
136
136
  version,
137
137
  files: s3Paths,
138
138
  status: configPublishStatus,
139
- user,
139
+ user: configPublishUser,
140
140
  };
141
141
 
142
142
  if (setToUseSNS(configSNSTopic)) {
@@ -50,6 +50,7 @@ describe("storeConfigVersionAndInformBroker", () => {
50
50
  config.set("serviceName", "grants-config-playground");
51
51
  config.set("configBroker.apiEndpoint", "https://broker.unit.test");
52
52
  config.set("configPublish.status", "active");
53
+ config.set("configPublish.user", "tom_the_cat");
53
54
  config.set(
54
55
  "aws.sns.configVersionTopicArn",
55
56
  "arn:aws:sns:us-east-1:123456789012:config-version-topic",
@@ -113,7 +114,7 @@ describe("storeConfigVersionAndInformBroker", () => {
113
114
  version: "1.2.3",
114
115
  files: ["grant-1/1.2.3/main.json"],
115
116
  status: "active",
116
- user: "system",
117
+ user: "tom_the_cat",
117
118
  });
118
119
  });
119
120
 
@@ -0,0 +1,122 @@
1
+ import {
2
+ DeleteMessageCommand,
3
+ ReceiveMessageCommand,
4
+ SendMessageCommand,
5
+ SQSClient,
6
+ } from "@aws-sdk/client-sqs";
7
+ import { setTimeout } from "node:timers/promises";
8
+ import { withTraceParent } from "./trace-parent.js";
9
+ const DEFAULT_POLL_TIMEOUT_MS = 30000;
10
+
11
+ export class SqsSubscriber {
12
+ constructor(options) {
13
+ this.queueUrl = options.queueUrl;
14
+ this.onMessage = options.onMessage;
15
+ this.logger = options.logger;
16
+ this.isRunning = false;
17
+ this.timeoutOnPollErrorMs =
18
+ options.timeoutOnErrorMs || DEFAULT_POLL_TIMEOUT_MS;
19
+
20
+ this.sqsClient = new SQSClient({
21
+ region: options.region,
22
+ endpoint: options.awsEndpointUrl,
23
+ });
24
+ }
25
+
26
+ async start() {
27
+ this.isRunning = true;
28
+ await this.poll();
29
+ }
30
+
31
+ async stop() {
32
+ this.isRunning = false;
33
+ }
34
+
35
+ async poll() {
36
+ this.logger.info(`Started polling SQS queue: ${this.queueUrl}`);
37
+
38
+ while (this.isRunning) {
39
+ try {
40
+ const messages = await this.getMessages();
41
+ await Promise.all(messages.map((m) => this.processMessage(m)));
42
+ } catch (err) {
43
+ this.logger.error(
44
+ `Error polling SQS queue ${this.queueUrl}: ${err.message}`,
45
+ );
46
+ await setTimeout(this.timeoutOnPollErrorMs);
47
+ }
48
+ }
49
+
50
+ this.logger.info(`Stopped polling SQS queue: ${this.queueUrl}`);
51
+ }
52
+
53
+ async processMessage(message) {
54
+ this.logger.info(`Processing SQS message ${message.MessageId}`);
55
+
56
+ try {
57
+ const body = JSON.parse(message.Body);
58
+ await withTraceParent(body.traceparent, () =>
59
+ this.onMessage(
60
+ body,
61
+ this.extractMessageAttributes(message),
62
+ message.Attributes.SentTimestamp,
63
+ ),
64
+ );
65
+ await this.deleteMessage(message);
66
+ } catch (error) {
67
+ this.logger.error(
68
+ { error },
69
+ `Error processing SQS message ${message.MessageId}`,
70
+ );
71
+ }
72
+ }
73
+
74
+ async getMessages() {
75
+ const response = await this.sqsClient.send(
76
+ new ReceiveMessageCommand({
77
+ QueueUrl: this.queueUrl,
78
+ MaxNumberOfMessages: 10,
79
+ WaitTimeSeconds: 20,
80
+ AttributeNames: ["All"],
81
+ MessageAttributeNames: ["All"],
82
+ }),
83
+ );
84
+
85
+ return response.Messages || [];
86
+ }
87
+
88
+ async sendMessage(message) {
89
+ // we could pass in attributes and send them on, but we don't need them yet
90
+ const params = {
91
+ QueueUrl: this.queueUrl,
92
+ MessageBody: JSON.stringify(message),
93
+ };
94
+ await this.sqsClient.send(new SendMessageCommand(params));
95
+ }
96
+
97
+ async deleteMessage(message) {
98
+ await this.sqsClient.send(
99
+ new DeleteMessageCommand({
100
+ QueueUrl: this.queueUrl,
101
+ ReceiptHandle: message.ReceiptHandle,
102
+ }),
103
+ );
104
+ }
105
+
106
+ extractMessageAttributes(message) {
107
+ const attributes = {};
108
+ for (const key in message.MessageAttributes) {
109
+ if (Object.hasOwn(message.MessageAttributes, key)) {
110
+ const attr = message.MessageAttributes[key];
111
+ if (attr.DataType === "String" || attr.DataType === "Number") {
112
+ attributes[key] = attr.StringValue;
113
+ } else if (attr.DataType === "String.Array") {
114
+ attributes[key] = JSON.parse(attr.StringValue);
115
+ } else {
116
+ attributes[key] = attr.BinaryValue;
117
+ }
118
+ }
119
+ }
120
+ return attributes;
121
+ }
122
+ }
@@ -0,0 +1,236 @@
1
+ import {
2
+ DeleteMessageCommand,
3
+ ReceiveMessageCommand,
4
+ SendMessageCommand,
5
+ SQSClient,
6
+ } from "@aws-sdk/client-sqs";
7
+ import { SqsSubscriber } from "./sqs-subscriber.js";
8
+
9
+ const mockLogger = {
10
+ info: vi.fn(),
11
+ error: vi.fn(),
12
+ };
13
+
14
+ vi.mock("@aws-sdk/client-sqs");
15
+
16
+ const timeoutSleepMs = 100;
17
+
18
+ let consumer;
19
+ let onMessage;
20
+
21
+ beforeEach(async () => {
22
+ onMessage = vi.fn().mockResolvedValue();
23
+
24
+ consumer = new SqsSubscriber({
25
+ queueUrl: "https://sqs.eu-west-2.amazonaws.com/123456789012/test-queue",
26
+ onMessage,
27
+ logger: mockLogger,
28
+ region: "eu-west-2",
29
+ awsEndpointUrl: "http://localhost:4566",
30
+ timeoutOnErrorMs: timeoutSleepMs,
31
+ });
32
+ });
33
+
34
+ describe("constructor", () => {
35
+ it("instantiates with options", () => {
36
+ expect(consumer.queueUrl).toBe(
37
+ "https://sqs.eu-west-2.amazonaws.com/123456789012/test-queue",
38
+ );
39
+ expect(consumer.onMessage).toBe(onMessage);
40
+ expect(consumer.isRunning).toBe(false);
41
+ expect(SQSClient).toHaveBeenCalledWith({
42
+ endpoint: "http://localhost:4566",
43
+ region: "eu-west-2",
44
+ });
45
+ });
46
+ });
47
+
48
+ describe("start", () => {
49
+ it("sets isRunning to true and starts polling", async () => {
50
+ consumer.poll = vi.fn().mockResolvedValue();
51
+
52
+ await consumer.start();
53
+
54
+ expect(consumer.isRunning).toBe(true);
55
+ expect(consumer.poll).toHaveBeenCalled();
56
+ });
57
+ });
58
+
59
+ describe("stop", () => {
60
+ it("sets isRunning to false", async () => {
61
+ consumer.isRunning = true;
62
+
63
+ await consumer.stop();
64
+
65
+ expect(consumer.isRunning).toBe(false);
66
+ });
67
+ });
68
+
69
+ describe("error on polling", () => {
70
+ it("error when polling waits for timeout period before running again, and logs errors", async () => {
71
+ const timeCaptures = [];
72
+ consumer.sqsClient.send.mockImplementationOnce(() => {
73
+ timeCaptures.push(Date.now());
74
+ throw new Error("Test polling error");
75
+ });
76
+
77
+ consumer.sqsClient.send.mockImplementationOnce(() => {
78
+ timeCaptures.push(Date.now());
79
+ consumer.isRunning = false; // stop now
80
+ throw new Error("Test polling error");
81
+ });
82
+
83
+ await consumer.start();
84
+
85
+ expect(timeCaptures[1] - timeCaptures[0]).toBeGreaterThanOrEqual(
86
+ timeoutSleepMs - 2,
87
+ ); //2ms tolerance
88
+ expect(mockLogger.error).toHaveBeenCalledWith(
89
+ "Error polling SQS queue https://sqs.eu-west-2.amazonaws.com/123456789012/test-queue: Test polling error",
90
+ );
91
+ });
92
+ });
93
+
94
+ describe("message processing", () => {
95
+ it("processes and deletes messages correctly", async () => {
96
+ const mockMessages = [
97
+ {
98
+ MessageId: "msg-1",
99
+ Body: '{ "message" : "Test message 1" }',
100
+ ReceiptHandle: "receipt-1",
101
+ MessageAttributes: {
102
+ Attribute1: {
103
+ StringValue: "Value1",
104
+ DataType: "String",
105
+ },
106
+ Attribute2: {
107
+ StringValue: "1",
108
+ DataType: "Number",
109
+ },
110
+ Attribute3: {
111
+ BinaryValue: new TextEncoder().encode("Value3"),
112
+ DataType: "Binary",
113
+ },
114
+ },
115
+ Attributes: {
116
+ SentTimestamp: "1780599163000",
117
+ },
118
+ },
119
+ ];
120
+
121
+ consumer.sqsClient.send.mockImplementation(async (command) => {
122
+ if (command instanceof ReceiveMessageCommand) {
123
+ return { Messages: mockMessages };
124
+ }
125
+ if (command instanceof DeleteMessageCommand) {
126
+ return {};
127
+ }
128
+ return {};
129
+ });
130
+
131
+ consumer.onMessage.mockReset();
132
+
133
+ consumer.onMessage.mockImplementationOnce(() => {
134
+ consumer.isRunning = false; // just take 1 message
135
+ return Promise.resolve("processed");
136
+ });
137
+
138
+ await consumer.start();
139
+
140
+ expect(ReceiveMessageCommand).toHaveBeenCalledWith({
141
+ QueueUrl: consumer.queueUrl,
142
+ MaxNumberOfMessages: 10,
143
+ WaitTimeSeconds: 20,
144
+ AttributeNames: ["All"],
145
+ MessageAttributeNames: ["All"],
146
+ });
147
+
148
+ expect(onMessage).toHaveBeenCalledWith(
149
+ { message: "Test message 1" },
150
+ {
151
+ Attribute1: "Value1",
152
+ Attribute2: "1",
153
+ Attribute3: new TextEncoder().encode("Value3"),
154
+ },
155
+ "1780599163000",
156
+ );
157
+
158
+ expect(DeleteMessageCommand).toHaveBeenCalledWith({
159
+ QueueUrl: consumer.queueUrl,
160
+ ReceiptHandle: "receipt-1",
161
+ });
162
+ });
163
+
164
+ it("handles errors gracefully in processing", async () => {
165
+ const mockMessages = [
166
+ {
167
+ MessageId: "msg-1",
168
+ Body: '{ "message" : "Test message 1" }',
169
+ ReceiptHandle: "receipt-1",
170
+ Attributes: {
171
+ SentTimestamp: "1780599163000",
172
+ },
173
+ },
174
+ ];
175
+
176
+ // mock the fetch part to return our test message
177
+ consumer.sqsClient.send.mockImplementation(async (command) => {
178
+ if (command instanceof ReceiveMessageCommand) {
179
+ return { Messages: mockMessages };
180
+ }
181
+ return {};
182
+ });
183
+
184
+ consumer.onMessage.mockReset();
185
+
186
+ const err = new Error("Test error");
187
+
188
+ consumer.onMessage.mockImplementationOnce(() => {
189
+ consumer.isRunning = false; // just take 1 message
190
+ return Promise.reject(err);
191
+ });
192
+
193
+ await consumer.start();
194
+
195
+ expect(mockLogger.error).toHaveBeenCalledWith(
196
+ { error: err },
197
+ "Error processing SQS message msg-1",
198
+ );
199
+ });
200
+ });
201
+
202
+ describe("sendMessage", () => {
203
+ it("sends a message via client", async () => {
204
+ const mockMessage = {
205
+ grant: "grant-1",
206
+ version: "1.2.3",
207
+ };
208
+
209
+ await consumer.sendMessage(mockMessage);
210
+
211
+ expect(SendMessageCommand).toHaveBeenCalledWith({
212
+ QueueUrl: consumer.queueUrl,
213
+ MessageBody: '{"grant":"grant-1","version":"1.2.3"}',
214
+ });
215
+ expect(consumer.sqsClient.send).toHaveBeenCalledWith(
216
+ expect.any(SendMessageCommand),
217
+ );
218
+ });
219
+ });
220
+
221
+ describe("deleteMessage", () => {
222
+ it("deletes a message", async () => {
223
+ const mockMessage = {
224
+ MessageId: "msg-1",
225
+ ReceiptHandle: "receipt-1",
226
+ };
227
+
228
+ await consumer.deleteMessage(mockMessage);
229
+
230
+ expect(DeleteMessageCommand).toHaveBeenCalledWith({
231
+ QueueUrl: consumer.queueUrl,
232
+ ReceiptHandle: "receipt-1",
233
+ });
234
+ expect(consumer.sqsClient.send).toHaveBeenCalled();
235
+ });
236
+ });
@@ -0,0 +1,8 @@
1
+ import { AsyncLocalStorage } from "node:async_hooks";
2
+
3
+ const asyncLocalStorage = new AsyncLocalStorage();
4
+
5
+ export const withTraceParent = (traceParent, fn) =>
6
+ asyncLocalStorage.run(traceParent, fn);
7
+
8
+ export const getTraceParent = () => asyncLocalStorage.getStore();
@@ -0,0 +1,13 @@
1
+ import { setTimeout } from "node:timers/promises";
2
+ import { getTraceParent, withTraceParent } from "./trace-parent.js";
3
+
4
+ describe("traceParent", () => {
5
+ it("runs a function with traceParent in async scope", async () => {
6
+ const traceParent = await withTraceParent("1234-0987", async () => {
7
+ await setTimeout(1);
8
+ return getTraceParent();
9
+ });
10
+
11
+ expect(traceParent).toEqual("1234-0987");
12
+ });
13
+ });