@adobe/spacecat-shared-data-access 2.89.0 → 2.91.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/package.json +2 -1
- package/src/index.js +5 -0
- package/src/models/audit/audit.model.js +2 -0
- package/src/models/base/base.collection.js +11 -0
- package/src/models/base/entity.registry.js +14 -6
- package/src/models/configuration/configuration.collection.js +174 -14
- package/src/models/configuration/configuration.model.js +92 -7
- package/src/models/configuration/configuration.schema.js +29 -58
- package/src/models/configuration/index.d.ts +28 -16
- package/src/service/index.js +39 -2
- package/src/util/index.js +10 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,17 @@
|
|
|
1
|
+
# [@adobe/spacecat-shared-data-access-v2.91.0](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-data-access-v2.90.0...@adobe/spacecat-shared-data-access-v2.91.0) (2025-12-10)
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
### Features
|
|
5
|
+
|
|
6
|
+
* add unused content fragment audit type ([#1135](https://github.com/adobe/spacecat-shared/issues/1135)) ([dfdd01e](https://github.com/adobe/spacecat-shared/commit/dfdd01e2603ab737da2f1935139b66693edb9d70))
|
|
7
|
+
|
|
8
|
+
# [@adobe/spacecat-shared-data-access-v2.90.0](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-data-access-v2.89.0...@adobe/spacecat-shared-data-access-v2.90.0) (2025-12-08)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
### Features
|
|
12
|
+
|
|
13
|
+
* **data-access:** handle configuration via s3 ([#1216](https://github.com/adobe/spacecat-shared/issues/1216)) ([ef03ab7](https://github.com/adobe/spacecat-shared/commit/ef03ab705bae8653162de12405bcbc65c32bb5a8))
|
|
14
|
+
|
|
1
15
|
# [@adobe/spacecat-shared-data-access-v2.89.0](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-data-access-v2.88.9...@adobe/spacecat-shared-data-access-v2.89.0) (2025-12-08)
|
|
2
16
|
|
|
3
17
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@adobe/spacecat-shared-data-access",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.91.0",
|
|
4
4
|
"description": "Shared modules of the Spacecat Services - Data Access",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"engines": {
|
|
@@ -40,6 +40,7 @@
|
|
|
40
40
|
"dependencies": {
|
|
41
41
|
"@adobe/spacecat-shared-utils": "1.81.1",
|
|
42
42
|
"@aws-sdk/client-dynamodb": "3.940.0",
|
|
43
|
+
"@aws-sdk/client-s3": "^3.940.0",
|
|
43
44
|
"@aws-sdk/lib-dynamodb": "3.940.0",
|
|
44
45
|
"@types/joi": "17.2.3",
|
|
45
46
|
"aws-xray-sdk": "3.12.0",
|
package/src/index.js
CHANGED
|
@@ -26,6 +26,7 @@ export default function dataAccessWrapper(fn) {
|
|
|
26
26
|
* Wrapper for data access layer. This wrapper will create a data access layer if it is not
|
|
27
27
|
* already created. It requires the context to have a log object. It will also use the
|
|
28
28
|
* DYNAMO_TABLE_NAME_DATA environment variable to create the data access layer.
|
|
29
|
+
* Optionally, it will use the ENV and AWS_REGION environment variables
|
|
29
30
|
*
|
|
30
31
|
* @param {object} request - The request object
|
|
31
32
|
* @param {object} context - The context object
|
|
@@ -37,10 +38,14 @@ export default function dataAccessWrapper(fn) {
|
|
|
37
38
|
|
|
38
39
|
const {
|
|
39
40
|
DYNAMO_TABLE_NAME_DATA = TABLE_NAME_DATA,
|
|
41
|
+
S3_CONFIG_BUCKET: s3Bucket,
|
|
42
|
+
AWS_REGION: region,
|
|
40
43
|
} = context.env;
|
|
41
44
|
|
|
42
45
|
context.dataAccess = createDataAccess({
|
|
43
46
|
tableNameData: DYNAMO_TABLE_NAME_DATA,
|
|
47
|
+
s3Bucket,
|
|
48
|
+
region,
|
|
44
49
|
}, log);
|
|
45
50
|
}
|
|
46
51
|
|
|
@@ -41,6 +41,8 @@ class Audit extends BaseModel {
|
|
|
41
41
|
REDIRECT_CHAINS: 'redirect-chains',
|
|
42
42
|
BROKEN_BACKLINKS: 'broken-backlinks',
|
|
43
43
|
BROKEN_INTERNAL_LINKS: 'broken-internal-links',
|
|
44
|
+
CONTENT_FRAGMENT_UNUSED: 'content-fragment-unused',
|
|
45
|
+
CONTENT_FRAGMENT_UNUSED_AUTO_FIX: 'content-fragment-unused-auto-fix',
|
|
44
46
|
EXPERIMENTATION: 'experimentation',
|
|
45
47
|
CONVERSION: 'conversion',
|
|
46
48
|
ORGANIC_KEYWORDS: 'organic-keywords',
|
|
@@ -27,6 +27,7 @@ import {
|
|
|
27
27
|
entityNameToAllPKValue,
|
|
28
28
|
removeElectroProperties,
|
|
29
29
|
} from '../../util/util.js';
|
|
30
|
+
import { DATASTORE_TYPE } from '../../util/index.js';
|
|
30
31
|
|
|
31
32
|
function isValidParent(parent, child) {
|
|
32
33
|
if (!hasText(parent.entityName)) {
|
|
@@ -55,6 +56,13 @@ class BaseCollection {
|
|
|
55
56
|
*/
|
|
56
57
|
static COLLECTION_NAME = undefined;
|
|
57
58
|
|
|
59
|
+
/**
|
|
60
|
+
* The datastore type for this collection. Defaults to DYNAMO.
|
|
61
|
+
* Override in subclasses to use a different datastore (e.g., S3).
|
|
62
|
+
* @type {string}
|
|
63
|
+
*/
|
|
64
|
+
static DATASTORE_TYPE = DATASTORE_TYPE.DYNAMO;
|
|
65
|
+
|
|
58
66
|
/**
|
|
59
67
|
* Constructs an instance of BaseCollection.
|
|
60
68
|
* @constructor
|
|
@@ -245,6 +253,7 @@ class BaseCollection {
|
|
|
245
253
|
order: options.order || 'desc',
|
|
246
254
|
...options.limit && { limit: options.limit },
|
|
247
255
|
...options.attributes && { attributes: options.attributes },
|
|
256
|
+
/* c8 ignore next */
|
|
248
257
|
...options.cursor && { cursor: options.cursor },
|
|
249
258
|
};
|
|
250
259
|
|
|
@@ -289,6 +298,7 @@ class BaseCollection {
|
|
|
289
298
|
return allData.length ? this.#createInstance(allData[0]) : null;
|
|
290
299
|
} else {
|
|
291
300
|
const instances = this.#createInstances(allData);
|
|
301
|
+
/* c8 ignore next 2 */
|
|
292
302
|
return shouldReturnCursor
|
|
293
303
|
? { data: instances, cursor: result.cursor || null }
|
|
294
304
|
: instances;
|
|
@@ -417,6 +427,7 @@ class BaseCollection {
|
|
|
417
427
|
.filter((entity) => entity !== null);
|
|
418
428
|
|
|
419
429
|
// Extract unprocessed keys
|
|
430
|
+
/* c8 ignore next 3 */
|
|
420
431
|
const unprocessed = result.unprocessed
|
|
421
432
|
? result.unprocessed.map((item) => item)
|
|
422
433
|
: [];
|
|
@@ -47,7 +47,6 @@ import ApiKeySchema from '../api-key/api-key.schema.js';
|
|
|
47
47
|
import AsyncJobSchema from '../async-job/async-job.schema.js';
|
|
48
48
|
import AuditSchema from '../audit/audit.schema.js';
|
|
49
49
|
import AuditUrlSchema from '../audit-url/audit-url.schema.js';
|
|
50
|
-
import ConfigurationSchema from '../configuration/configuration.schema.js';
|
|
51
50
|
import EntitlementSchema from '../entitlement/entitlement.schema.js';
|
|
52
51
|
import FixEntitySchema from '../fix-entity/fix-entity.schema.js';
|
|
53
52
|
import FixEntitySuggestionSchema from '../fix-entity-suggestion/fix-entity-suggestion.schema.js';
|
|
@@ -84,11 +83,13 @@ class EntityRegistry {
|
|
|
84
83
|
/**
|
|
85
84
|
* Constructs an instance of EntityRegistry.
|
|
86
85
|
* @constructor
|
|
87
|
-
* @param {Object}
|
|
86
|
+
* @param {Object} services - Dictionary of services keyed by datastore type.
|
|
87
|
+
* @param {Object} services.dynamo - The ElectroDB service instance for DynamoDB operations.
|
|
88
|
+
* @param {{s3Client: S3Client, s3Bucket: string}|null} [services.s3] - S3 service configuration.
|
|
88
89
|
* @param {Object} log - A logger for capturing and logging information.
|
|
89
90
|
*/
|
|
90
|
-
constructor(
|
|
91
|
-
this.
|
|
91
|
+
constructor(services, log) {
|
|
92
|
+
this.services = services;
|
|
92
93
|
this.log = log;
|
|
93
94
|
this.collections = new Map();
|
|
94
95
|
|
|
@@ -98,13 +99,20 @@ class EntityRegistry {
|
|
|
98
99
|
/**
|
|
99
100
|
* Initializes the collections managed by the EntityRegistry.
|
|
100
101
|
* This method creates instances of each collection and stores them in an internal map.
|
|
102
|
+
* ElectroDB-based collections are initialized with the dynamo service.
|
|
103
|
+
* Configuration is handled specially as it's a standalone S3-based collection.
|
|
101
104
|
* @private
|
|
102
105
|
*/
|
|
103
106
|
#initialize() {
|
|
107
|
+
// Initialize ElectroDB-based collections
|
|
104
108
|
Object.values(EntityRegistry.entities).forEach(({ collection: Collection, schema }) => {
|
|
105
|
-
const collection = new Collection(this.
|
|
109
|
+
const collection = new Collection(this.services.dynamo, this, schema, this.log);
|
|
106
110
|
this.collections.set(Collection.COLLECTION_NAME, collection);
|
|
107
111
|
});
|
|
112
|
+
|
|
113
|
+
// Initialize Configuration collection separately (standalone S3-based)
|
|
114
|
+
const configCollection = new ConfigurationCollection(this.services.s3, this.log);
|
|
115
|
+
this.collections.set(ConfigurationCollection.COLLECTION_NAME, configCollection);
|
|
108
116
|
}
|
|
109
117
|
|
|
110
118
|
/**
|
|
@@ -142,11 +150,11 @@ class EntityRegistry {
|
|
|
142
150
|
}
|
|
143
151
|
}
|
|
144
152
|
|
|
153
|
+
// Register ElectroDB-based entities only (Configuration is handled separately)
|
|
145
154
|
EntityRegistry.registerEntity(ApiKeySchema, ApiKeyCollection);
|
|
146
155
|
EntityRegistry.registerEntity(AsyncJobSchema, AsyncJobCollection);
|
|
147
156
|
EntityRegistry.registerEntity(AuditSchema, AuditCollection);
|
|
148
157
|
EntityRegistry.registerEntity(AuditUrlSchema, AuditUrlCollection);
|
|
149
|
-
EntityRegistry.registerEntity(ConfigurationSchema, ConfigurationCollection);
|
|
150
158
|
EntityRegistry.registerEntity(EntitlementSchema, EntitlementCollection);
|
|
151
159
|
EntityRegistry.registerEntity(FixEntitySchema, FixEntityCollection);
|
|
152
160
|
EntityRegistry.registerEntity(FixEntitySuggestionSchema, FixEntitySuggestionCollection);
|
|
@@ -10,36 +10,196 @@
|
|
|
10
10
|
* governing permissions and limitations under the License.
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
|
-
import {
|
|
14
|
-
|
|
13
|
+
import {
|
|
14
|
+
GetObjectCommand,
|
|
15
|
+
PutObjectCommand,
|
|
16
|
+
} from '@aws-sdk/client-s3';
|
|
17
|
+
|
|
18
|
+
import DataAccessError from '../../errors/data-access.error.js';
|
|
19
|
+
import { sanitizeIdAndAuditFields } from '../../util/util.js';
|
|
20
|
+
import Configuration from './configuration.model.js';
|
|
21
|
+
import { checkConfiguration } from './configuration.schema.js';
|
|
22
|
+
|
|
23
|
+
const S3_CONFIG_KEY = 'config/spacecat/global-config.json';
|
|
15
24
|
|
|
16
25
|
/**
|
|
17
|
-
* ConfigurationCollection - A collection class
|
|
18
|
-
*
|
|
19
|
-
* Configuration
|
|
26
|
+
* ConfigurationCollection - A standalone collection class for managing Configuration entities.
|
|
27
|
+
* Unlike other collections, this does not use ElectroDB or DynamoDB.
|
|
28
|
+
* Configuration is stored as a versioned JSON object in S3.
|
|
20
29
|
*
|
|
21
30
|
* @class ConfigurationCollection
|
|
22
|
-
* @extends BaseCollection
|
|
23
31
|
*/
|
|
24
|
-
class ConfigurationCollection
|
|
32
|
+
class ConfigurationCollection {
|
|
25
33
|
static COLLECTION_NAME = 'ConfigurationCollection';
|
|
26
34
|
|
|
35
|
+
/**
|
|
36
|
+
* Constructs an instance of ConfigurationCollection.
|
|
37
|
+
* @constructor
|
|
38
|
+
* @param {{s3Client: S3Client, s3Bucket: string}|null} s3Config - S3 configuration.
|
|
39
|
+
* @param {Object} log - A logger for capturing logging information.
|
|
40
|
+
*/
|
|
41
|
+
constructor(s3Config, log) {
|
|
42
|
+
this.log = log;
|
|
43
|
+
|
|
44
|
+
if (s3Config) {
|
|
45
|
+
this.s3Client = s3Config.s3Client;
|
|
46
|
+
this.s3Bucket = s3Config.s3Bucket;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Validates that S3 is configured. Throws an error if not.
|
|
52
|
+
* @private
|
|
53
|
+
* @throws {DataAccessError} If S3 is not configured.
|
|
54
|
+
*/
|
|
55
|
+
#requireS3() {
|
|
56
|
+
if (!this.s3Client || !this.s3Bucket) {
|
|
57
|
+
throw new DataAccessError(
|
|
58
|
+
'S3 configuration is required for Configuration storage. '
|
|
59
|
+
+ 'Ensure S3_CONFIG_BUCKET environment variable is set.',
|
|
60
|
+
this,
|
|
61
|
+
);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Creates an instance of the Configuration model from data and versionId.
|
|
67
|
+
* @private
|
|
68
|
+
* @param {Object} data - The configuration data.
|
|
69
|
+
* @param {string} versionId - The S3 VersionId.
|
|
70
|
+
* @returns {Configuration} - The Configuration model instance.
|
|
71
|
+
*/
|
|
72
|
+
#createInstance(data, versionId) {
|
|
73
|
+
return new Configuration(data, versionId, this, this.log);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Creates a new configuration version and stores it in S3.
|
|
78
|
+
* S3 versioning handles the version history automatically.
|
|
79
|
+
* The S3 VersionId is used as the configurationId.
|
|
80
|
+
*
|
|
81
|
+
* @param {Object} data - The configuration data to store.
|
|
82
|
+
* @returns {Promise<Configuration>} - The created Configuration instance.
|
|
83
|
+
* @throws {DataAccessError} If S3 is not configured or the operation fails.
|
|
84
|
+
*/
|
|
27
85
|
async create(data) {
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
86
|
+
this.#requireS3();
|
|
87
|
+
|
|
88
|
+
try {
|
|
89
|
+
const sanitizedData = sanitizeIdAndAuditFields('Configuration', data);
|
|
90
|
+
|
|
91
|
+
const now = new Date().toISOString();
|
|
92
|
+
const configData = {
|
|
93
|
+
...sanitizedData,
|
|
94
|
+
createdAt: now,
|
|
95
|
+
updatedAt: now,
|
|
96
|
+
updatedBy: sanitizedData.updatedBy || 'system',
|
|
97
|
+
};
|
|
98
|
+
|
|
99
|
+
// Validate the configuration against the schema
|
|
100
|
+
checkConfiguration(configData);
|
|
101
|
+
|
|
102
|
+
const command = new PutObjectCommand({
|
|
103
|
+
Bucket: this.s3Bucket,
|
|
104
|
+
Key: S3_CONFIG_KEY,
|
|
105
|
+
Body: JSON.stringify(configData),
|
|
106
|
+
ContentType: 'application/json',
|
|
107
|
+
});
|
|
31
108
|
|
|
32
|
-
|
|
109
|
+
const response = await this.s3Client.send(command);
|
|
110
|
+
const { VersionId: versionId } = response;
|
|
33
111
|
|
|
34
|
-
|
|
112
|
+
this.log.info(`Configuration stored in S3 with VersionId ${versionId}`);
|
|
113
|
+
|
|
114
|
+
return this.#createInstance(configData, versionId);
|
|
115
|
+
} catch (error) {
|
|
116
|
+
if (error instanceof DataAccessError) {
|
|
117
|
+
throw error;
|
|
118
|
+
}
|
|
119
|
+
const message = `Failed to create configuration in S3: ${error.message}`;
|
|
120
|
+
this.log.error(message, error);
|
|
121
|
+
throw new DataAccessError(message, this, error);
|
|
122
|
+
}
|
|
35
123
|
}
|
|
36
124
|
|
|
125
|
+
/**
|
|
126
|
+
* Finds a configuration by S3 VersionId.
|
|
127
|
+
*
|
|
128
|
+
* @param {string} version - The S3 VersionId.
|
|
129
|
+
* @returns {Promise<Configuration|null>} - The Configuration instance or null if not found.
|
|
130
|
+
* @throws {DataAccessError} If S3 is not configured or the operation fails.
|
|
131
|
+
*/
|
|
37
132
|
async findByVersion(version) {
|
|
38
|
-
|
|
133
|
+
this.#requireS3();
|
|
134
|
+
|
|
135
|
+
const versionId = String(version);
|
|
136
|
+
|
|
137
|
+
try {
|
|
138
|
+
const command = new GetObjectCommand({
|
|
139
|
+
Bucket: this.s3Bucket,
|
|
140
|
+
Key: S3_CONFIG_KEY,
|
|
141
|
+
VersionId: versionId,
|
|
142
|
+
});
|
|
143
|
+
|
|
144
|
+
const response = await this.s3Client.send(command);
|
|
145
|
+
const bodyString = await response.Body.transformToString();
|
|
146
|
+
const configData = JSON.parse(bodyString);
|
|
147
|
+
|
|
148
|
+
return this.#createInstance(configData, versionId);
|
|
149
|
+
} catch (error) {
|
|
150
|
+
if (error.name === 'NoSuchKey' || error.name === 'NoSuchVersion') {
|
|
151
|
+
this.log.info(`Configuration with version ${versionId} not found in S3`);
|
|
152
|
+
return null;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
if (error instanceof DataAccessError) {
|
|
156
|
+
throw error;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
const message = `Failed to retrieve configuration with version ${versionId} from S3: ${error.message}`;
|
|
160
|
+
this.log.error(message, error);
|
|
161
|
+
throw new DataAccessError(message, this, error);
|
|
162
|
+
}
|
|
39
163
|
}
|
|
40
164
|
|
|
165
|
+
/**
|
|
166
|
+
* Retrieves the latest configuration from S3.
|
|
167
|
+
* S3 automatically returns the latest version when versioning is enabled.
|
|
168
|
+
*
|
|
169
|
+
* @returns {Promise<Configuration|null>} - The latest Configuration instance
|
|
170
|
+
* or null if not found.
|
|
171
|
+
* @throws {DataAccessError} If S3 is not configured or the operation fails.
|
|
172
|
+
*/
|
|
41
173
|
async findLatest() {
|
|
42
|
-
|
|
174
|
+
this.#requireS3();
|
|
175
|
+
|
|
176
|
+
try {
|
|
177
|
+
const command = new GetObjectCommand({
|
|
178
|
+
Bucket: this.s3Bucket,
|
|
179
|
+
Key: S3_CONFIG_KEY,
|
|
180
|
+
});
|
|
181
|
+
|
|
182
|
+
const response = await this.s3Client.send(command);
|
|
183
|
+
const bodyString = await response.Body.transformToString();
|
|
184
|
+
const configData = JSON.parse(bodyString);
|
|
185
|
+
const { VersionId: versionId } = response;
|
|
186
|
+
|
|
187
|
+
return this.#createInstance(configData, versionId);
|
|
188
|
+
} catch (error) {
|
|
189
|
+
// If the object doesn't exist, return null (first-time setup)
|
|
190
|
+
if (error.name === 'NoSuchKey') {
|
|
191
|
+
this.log.info('No configuration found in S3, returning null');
|
|
192
|
+
return null;
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
if (error instanceof DataAccessError) {
|
|
196
|
+
throw error;
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
const message = `Failed to retrieve configuration from S3: ${error.message}`;
|
|
200
|
+
this.log.error(message, error);
|
|
201
|
+
throw new DataAccessError(message, this, error);
|
|
202
|
+
}
|
|
43
203
|
}
|
|
44
204
|
}
|
|
45
205
|
|
|
@@ -13,17 +13,16 @@
|
|
|
13
13
|
import { isNonEmptyObject, isNonEmptyArray } from '@adobe/spacecat-shared-utils';
|
|
14
14
|
|
|
15
15
|
import { sanitizeIdAndAuditFields } from '../../util/util.js';
|
|
16
|
-
import BaseModel from '../base/base.model.js';
|
|
17
16
|
import { Entitlement } from '../entitlement/index.js';
|
|
18
17
|
|
|
19
18
|
/**
|
|
20
|
-
* Configuration - A class representing
|
|
21
|
-
*
|
|
19
|
+
* Configuration - A standalone class representing the global Configuration entity.
|
|
20
|
+
* This is a singleton entity stored in S3 with versioning.
|
|
21
|
+
* Unlike other entities, Configuration does not use ElectroDB or DynamoDB.
|
|
22
22
|
*
|
|
23
23
|
* @class Configuration
|
|
24
|
-
* @extends BaseModel
|
|
25
24
|
*/
|
|
26
|
-
class Configuration
|
|
25
|
+
class Configuration {
|
|
27
26
|
static ENTITY_NAME = 'Configuration';
|
|
28
27
|
|
|
29
28
|
static JOB_GROUPS = {
|
|
@@ -53,7 +52,78 @@ class Configuration extends BaseModel {
|
|
|
53
52
|
|
|
54
53
|
static AUDIT_NAME_MAX_LENGTH = 37;
|
|
55
54
|
|
|
56
|
-
|
|
55
|
+
constructor(data, versionId, collection, log) {
|
|
56
|
+
this.handlers = data.handlers;
|
|
57
|
+
this.jobs = data.jobs;
|
|
58
|
+
this.queues = data.queues;
|
|
59
|
+
this.slackRoles = data.slackRoles;
|
|
60
|
+
this.createdAt = data.createdAt;
|
|
61
|
+
this.updatedAt = data.updatedAt;
|
|
62
|
+
this.updatedBy = data.updatedBy;
|
|
63
|
+
this.versionId = versionId;
|
|
64
|
+
this.collection = collection;
|
|
65
|
+
this.log = log;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
getId() {
|
|
69
|
+
return this.versionId;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
getConfigurationId() {
|
|
73
|
+
return this.versionId;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
getVersion() {
|
|
77
|
+
return this.versionId;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
getCreatedAt() {
|
|
81
|
+
return this.createdAt;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
getUpdatedAt() {
|
|
85
|
+
return this.updatedAt;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
getUpdatedBy() {
|
|
89
|
+
return this.updatedBy;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
setUpdatedBy(updatedBy) {
|
|
93
|
+
this.updatedBy = updatedBy;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
getHandlers() {
|
|
97
|
+
return this.handlers;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
setHandlers(handlers) {
|
|
101
|
+
this.handlers = handlers;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
getJobs() {
|
|
105
|
+
return this.jobs;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
setJobs(jobs) {
|
|
109
|
+
this.jobs = jobs;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
getQueues() {
|
|
113
|
+
return this.queues;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
setQueues(queues) {
|
|
117
|
+
this.queues = queues;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
getSlackRoles() {
|
|
121
|
+
return this.slackRoles;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
setSlackRoles(slackRoles) {
|
|
125
|
+
this.slackRoles = slackRoles;
|
|
126
|
+
}
|
|
57
127
|
|
|
58
128
|
getHandler(type) {
|
|
59
129
|
return this.getHandlers()?.[type];
|
|
@@ -161,6 +231,7 @@ class Configuration extends BaseModel {
|
|
|
161
231
|
if (enabled) {
|
|
162
232
|
if (handler.enabledByDefault) {
|
|
163
233
|
handler.disabled[entityKey] = handler.disabled[entityKey]
|
|
234
|
+
/* c8 ignore next */
|
|
164
235
|
.filter((id) => id !== entityId) || [];
|
|
165
236
|
} else {
|
|
166
237
|
handler.enabled[entityKey] = Array
|
|
@@ -170,6 +241,7 @@ class Configuration extends BaseModel {
|
|
|
170
241
|
handler.disabled[entityKey] = Array
|
|
171
242
|
.from(new Set([...(handler.disabled[entityKey] || []), entityId]));
|
|
172
243
|
} else {
|
|
244
|
+
/* c8 ignore next */
|
|
173
245
|
handler.enabled[entityKey] = handler.enabled[entityKey].filter((id) => id !== entityId) || [];
|
|
174
246
|
}
|
|
175
247
|
|
|
@@ -270,6 +342,7 @@ class Configuration extends BaseModel {
|
|
|
270
342
|
if (!isNonEmptyObject(queues)) {
|
|
271
343
|
throw new Error('Queues configuration cannot be empty');
|
|
272
344
|
}
|
|
345
|
+
/* c8 ignore next */
|
|
273
346
|
const existingQueues = this.getQueues() || {};
|
|
274
347
|
const mergedQueues = { ...existingQueues, ...queues };
|
|
275
348
|
this.setQueues(mergedQueues);
|
|
@@ -496,7 +569,19 @@ class Configuration extends BaseModel {
|
|
|
496
569
|
}
|
|
497
570
|
|
|
498
571
|
async save() {
|
|
499
|
-
return this.collection.create(sanitizeIdAndAuditFields(
|
|
572
|
+
return this.collection.create(sanitizeIdAndAuditFields('Configuration', this.toJSON()));
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
toJSON() {
|
|
576
|
+
return {
|
|
577
|
+
handlers: this.handlers,
|
|
578
|
+
jobs: this.jobs,
|
|
579
|
+
queues: this.queues,
|
|
580
|
+
slackRoles: this.slackRoles,
|
|
581
|
+
createdAt: this.createdAt,
|
|
582
|
+
updatedAt: this.updatedAt,
|
|
583
|
+
updatedBy: this.updatedBy,
|
|
584
|
+
};
|
|
500
585
|
}
|
|
501
586
|
}
|
|
502
587
|
|
|
@@ -10,18 +10,21 @@
|
|
|
10
10
|
* governing permissions and limitations under the License.
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
13
|
+
/**
|
|
14
|
+
* Configuration Schema
|
|
15
|
+
*
|
|
16
|
+
* Unlike other entities, Configuration does not use ElectroDB and is stored in S3.
|
|
17
|
+
* Validation is handled by Joi schemas defined below.
|
|
18
|
+
*/
|
|
16
19
|
|
|
17
20
|
import Joi from 'joi';
|
|
18
21
|
|
|
19
|
-
import SchemaBuilder from '../base/schema.builder.js';
|
|
20
22
|
import Configuration from './configuration.model.js';
|
|
21
|
-
import ConfigurationCollection from './configuration.collection.js';
|
|
22
|
-
import { zeroPad } from '../../util/util.js';
|
|
23
23
|
|
|
24
|
-
const
|
|
24
|
+
const validJobGroups = Object.values(Configuration.JOB_GROUPS);
|
|
25
|
+
const validJobIntervals = Object.values(Configuration.JOB_INTERVALS);
|
|
26
|
+
|
|
27
|
+
export const handlerSchema = Joi.object().pattern(Joi.string(), Joi.object(
|
|
25
28
|
{
|
|
26
29
|
enabled: Joi.object({
|
|
27
30
|
sites: Joi.array().items(Joi.string()),
|
|
@@ -44,17 +47,32 @@ const handlerSchema = Joi.object().pattern(Joi.string(), Joi.object(
|
|
|
44
47
|
},
|
|
45
48
|
)).unknown(true);
|
|
46
49
|
|
|
47
|
-
const
|
|
50
|
+
export const jobSchema = Joi.object({
|
|
51
|
+
group: Joi.string().valid(...validJobGroups).required(),
|
|
52
|
+
type: Joi.string().required(),
|
|
53
|
+
interval: Joi.string().valid(...validJobIntervals).required(),
|
|
54
|
+
}).unknown(true);
|
|
55
|
+
|
|
56
|
+
export const jobsSchema = Joi.array().items(jobSchema).required();
|
|
57
|
+
|
|
58
|
+
export const queueSchema = Joi.object().min(1).required();
|
|
48
59
|
|
|
49
|
-
const
|
|
60
|
+
export const slackRolesSchema = Joi.object().min(1);
|
|
50
61
|
|
|
51
|
-
const configurationSchema = Joi.object({
|
|
52
|
-
version: Joi.number().required(),
|
|
62
|
+
export const configurationSchema = Joi.object({
|
|
53
63
|
queues: queueSchema,
|
|
54
64
|
handlers: handlerSchema,
|
|
55
65
|
jobs: jobsSchema,
|
|
66
|
+
slackRoles: slackRolesSchema,
|
|
56
67
|
}).unknown(true);
|
|
57
68
|
|
|
69
|
+
/**
|
|
70
|
+
* Validates a configuration object against the schema.
|
|
71
|
+
* @param {object} data - The configuration data to validate.
|
|
72
|
+
* @param {object} [schema=configurationSchema] - The Joi schema to validate against.
|
|
73
|
+
* @returns {object} The validated configuration data.
|
|
74
|
+
* @throws {Error} If validation fails.
|
|
75
|
+
*/
|
|
58
76
|
export const checkConfiguration = (data, schema = configurationSchema) => {
|
|
59
77
|
const { error, value } = schema.validate(data);
|
|
60
78
|
|
|
@@ -64,50 +82,3 @@ export const checkConfiguration = (data, schema = configurationSchema) => {
|
|
|
64
82
|
|
|
65
83
|
return value;
|
|
66
84
|
};
|
|
67
|
-
|
|
68
|
-
/*
|
|
69
|
-
Schema Doc: https://electrodb.dev/en/modeling/schema/
|
|
70
|
-
Attribute Doc: https://electrodb.dev/en/modeling/attributes/
|
|
71
|
-
Indexes Doc: https://electrodb.dev/en/modeling/indexes/
|
|
72
|
-
*/
|
|
73
|
-
|
|
74
|
-
const schema = new SchemaBuilder(Configuration, ConfigurationCollection)
|
|
75
|
-
.addAttribute('handlers', {
|
|
76
|
-
type: 'any',
|
|
77
|
-
validate: (value) => !value || checkConfiguration(value, handlerSchema),
|
|
78
|
-
})
|
|
79
|
-
.addAttribute('jobs', {
|
|
80
|
-
type: 'list',
|
|
81
|
-
items: {
|
|
82
|
-
type: 'map',
|
|
83
|
-
properties: {
|
|
84
|
-
group: { type: Object.values(Configuration.JOB_GROUPS), required: true },
|
|
85
|
-
type: { type: 'string', required: true },
|
|
86
|
-
interval: { type: Object.values(Configuration.JOB_INTERVALS), required: true },
|
|
87
|
-
},
|
|
88
|
-
},
|
|
89
|
-
})
|
|
90
|
-
.addAttribute('queues', {
|
|
91
|
-
type: 'any',
|
|
92
|
-
required: true,
|
|
93
|
-
validate: (value) => isNonEmptyObject(value),
|
|
94
|
-
})
|
|
95
|
-
.addAttribute('slackRoles', {
|
|
96
|
-
type: 'any',
|
|
97
|
-
validate: (value) => !value || isNonEmptyObject(value),
|
|
98
|
-
})
|
|
99
|
-
.addAttribute('version', {
|
|
100
|
-
type: 'number',
|
|
101
|
-
required: true,
|
|
102
|
-
readOnly: true,
|
|
103
|
-
})
|
|
104
|
-
.addAttribute('versionString', { // used for indexing/sorting
|
|
105
|
-
type: 'string',
|
|
106
|
-
required: true,
|
|
107
|
-
readOnly: true,
|
|
108
|
-
default: '0', // setting the default forces set() to run, to transform the version number to a string
|
|
109
|
-
set: (value, all) => zeroPad(all.version, 10),
|
|
110
|
-
})
|
|
111
|
-
.addAllIndex(['versionString']);
|
|
112
|
-
|
|
113
|
-
export default schema.build();
|
|
@@ -10,40 +10,52 @@
|
|
|
10
10
|
* governing permissions and limitations under the License.
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
|
-
import type {
|
|
14
|
-
BaseCollection, BaseModel, Organization, Site,
|
|
15
|
-
} from '../index';
|
|
13
|
+
import type { Organization, Site } from '../index';
|
|
16
14
|
|
|
17
|
-
export interface Configuration
|
|
15
|
+
export interface Configuration {
|
|
18
16
|
addHandler(type: string, handler: object): void;
|
|
19
|
-
|
|
17
|
+
disableHandlerForOrg(type: string, org: Organization): void;
|
|
20
18
|
disableHandlerForSite(type: string, site: Site): void;
|
|
21
|
-
|
|
19
|
+
enableHandlerForOrg(type: string, org: Organization): void;
|
|
22
20
|
enableHandlerForSite(type: string, site: Site): void;
|
|
23
21
|
getConfigurationId(): string;
|
|
24
|
-
|
|
25
|
-
getEnabledAuditsForSite(site: Site): string[];
|
|
22
|
+
getCreatedAt(): string;
|
|
26
23
|
getDisabledAuditsForSite(site: Site): string[];
|
|
24
|
+
getEnabledAuditsForSite(site: Site): string[];
|
|
25
|
+
getEnabledSiteIdsForHandler(type: string): string[];
|
|
27
26
|
getHandler(type: string): object | undefined;
|
|
28
27
|
getHandlers(): object;
|
|
29
|
-
|
|
28
|
+
getId(): string;
|
|
29
|
+
getJobs(): object[];
|
|
30
30
|
getQueues(): object;
|
|
31
31
|
getSlackRoleMembersByRole(role: string): string[];
|
|
32
32
|
getSlackRoles(): object;
|
|
33
|
-
|
|
34
|
-
|
|
33
|
+
getUpdatedAt(): string;
|
|
34
|
+
getUpdatedBy(): string;
|
|
35
|
+
getVersion(): string;
|
|
36
|
+
isHandlerDependencyMetForOrg(type: string, org: Organization): true | string[];
|
|
37
|
+
isHandlerDependencyMetForSite(type: string, site: Site): true | string[];
|
|
38
|
+
isHandlerEnabledForOrg(type: string, org: Organization): boolean;
|
|
35
39
|
isHandlerEnabledForSite(type: string, site: Site): boolean;
|
|
40
|
+
registerAudit(type: string, enabledByDefault?: boolean, interval?: string, productCodes?: string[]): void;
|
|
41
|
+
save(): Promise<Configuration>;
|
|
36
42
|
setHandlers(handlers: object): void;
|
|
37
|
-
setJobs(jobs: object): void;
|
|
43
|
+
setJobs(jobs: object[]): void;
|
|
38
44
|
setQueues(queues: object): void;
|
|
39
45
|
setSlackRoles(slackRoles: object): void;
|
|
46
|
+
setUpdatedBy(updatedBy: string): void;
|
|
47
|
+
toJSON(): object;
|
|
48
|
+
unregisterAudit(type: string): void;
|
|
49
|
+
updateConfiguration(data: object): void;
|
|
40
50
|
updateHandlerOrgs(type: string, orgId: string, enabled: boolean): void;
|
|
51
|
+
updateHandlerProperties(type: string, properties: object): void;
|
|
41
52
|
updateHandlerSites(type: string, siteId: string, enabled: boolean): void;
|
|
42
|
-
|
|
43
|
-
|
|
53
|
+
updateJob(type: string, properties: { interval?: string; group?: string }): void;
|
|
54
|
+
updateQueues(queues: object): void;
|
|
44
55
|
}
|
|
45
56
|
|
|
46
|
-
export interface ConfigurationCollection
|
|
47
|
-
|
|
57
|
+
export interface ConfigurationCollection {
|
|
58
|
+
create(data: object): Promise<Configuration>;
|
|
59
|
+
findByVersion(version: string): Promise<Configuration | null>;
|
|
48
60
|
findLatest(): Promise<Configuration | null>;
|
|
49
61
|
}
|
package/src/service/index.js
CHANGED
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
|
|
13
13
|
import { DynamoDB } from '@aws-sdk/client-dynamodb';
|
|
14
14
|
import { DynamoDBDocument } from '@aws-sdk/lib-dynamodb';
|
|
15
|
+
import { S3Client } from '@aws-sdk/client-s3';
|
|
15
16
|
import { Service } from 'electrodb';
|
|
16
17
|
|
|
17
18
|
import { instrumentAWSClient } from '@adobe/spacecat-shared-utils';
|
|
@@ -65,10 +66,45 @@ const createElectroService = (client, config, log) => {
|
|
|
65
66
|
);
|
|
66
67
|
};
|
|
67
68
|
|
|
69
|
+
/**
|
|
70
|
+
* Creates an S3 service configuration if bucket configuration is provided.
|
|
71
|
+
*
|
|
72
|
+
* @param {object} config - Configuration object
|
|
73
|
+
* @param {string} [config.s3Bucket] - S3 bucket name
|
|
74
|
+
* @param {string} [config.region] - AWS region
|
|
75
|
+
* @returns {{s3Client: S3Client, s3Bucket: string}|null} - S3 client and bucket or null
|
|
76
|
+
*/
|
|
77
|
+
const createS3Service = (config) => {
|
|
78
|
+
const { s3Bucket, region } = config;
|
|
79
|
+
|
|
80
|
+
if (!s3Bucket) {
|
|
81
|
+
return null;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
const options = region ? { region } : {};
|
|
85
|
+
const s3Client = instrumentAWSClient(new S3Client(options));
|
|
86
|
+
|
|
87
|
+
return { s3Client, s3Bucket };
|
|
88
|
+
};
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Creates a services dictionary containing all datastore services.
|
|
92
|
+
* Each collection can declare which service it needs via its DATASTORE_TYPE.
|
|
93
|
+
*
|
|
94
|
+
* @param {object} electroService - The ElectroDB service for DynamoDB operations
|
|
95
|
+
* @param {object} config - Configuration object
|
|
96
|
+
* @returns {object} Services dictionary with dynamo and s3 services
|
|
97
|
+
*/
|
|
98
|
+
const createServices = (electroService, config) => ({
|
|
99
|
+
dynamo: electroService,
|
|
100
|
+
s3: createS3Service(config),
|
|
101
|
+
});
|
|
102
|
+
|
|
68
103
|
/**
|
|
69
104
|
* Creates a data access layer for interacting with DynamoDB using ElectroDB.
|
|
70
105
|
*
|
|
71
|
-
* @param {{tableNameData: string}} config - Configuration
|
|
106
|
+
* @param {{tableNameData: string, s3Bucket?: string, region?: string}} config - Configuration
|
|
107
|
+
* object containing table name and optional S3 configuration
|
|
72
108
|
* @param {object} log - Logger instance, defaults to console
|
|
73
109
|
* @param {DynamoDB} [client] - Optional custom DynamoDB client instance
|
|
74
110
|
* @returns {object} Data access collections for interacting with entities
|
|
@@ -78,7 +114,8 @@ export const createDataAccess = (config, log = console, client = undefined) => {
|
|
|
78
114
|
|
|
79
115
|
const rawClient = createRawClient(client);
|
|
80
116
|
const electroService = createElectroService(rawClient, config, log);
|
|
81
|
-
const
|
|
117
|
+
const services = createServices(electroService, config);
|
|
118
|
+
const entityRegistry = new EntityRegistry(services, log);
|
|
82
119
|
|
|
83
120
|
return entityRegistry.getCollections();
|
|
84
121
|
};
|
package/src/util/index.js
CHANGED
|
@@ -26,3 +26,13 @@ export {
|
|
|
26
26
|
registerLogger,
|
|
27
27
|
getLogger,
|
|
28
28
|
} from './logger-registry.js';
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Datastore types that collections can use to declare their storage backend.
|
|
32
|
+
* @readonly
|
|
33
|
+
* @enum {string}
|
|
34
|
+
*/
|
|
35
|
+
export const DATASTORE_TYPE = Object.freeze({
|
|
36
|
+
DYNAMO: 'dynamo',
|
|
37
|
+
S3: 's3',
|
|
38
|
+
});
|