@adobe/spacecat-shared-data-access 2.109.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/README.md +260 -245
- package/docker-compose.test.yml +39 -0
- package/package.json +5 -6
- package/src/index.js +10 -5
- package/src/models/audit/audit.collection.js +6 -20
- package/src/models/base/base.collection.js +699 -395
- package/src/models/base/base.model.js +13 -6
- package/src/models/base/entity.registry.js +11 -4
- package/src/models/base/schema.builder.js +2 -0
- package/src/models/base/schema.js +15 -4
- package/src/models/fix-entity-suggestion/fix-entity-suggestion.model.js +15 -0
- package/src/models/fix-entity-suggestion/fix-entity-suggestion.schema.js +12 -0
- package/src/models/key-event/key-event.collection.js +23 -1
- package/src/models/latest-audit/latest-audit.collection.js +77 -3
- package/src/models/scrape-job/scrape-job.collection.js +38 -0
- package/src/models/scrape-job/scrape-job.schema.js +4 -2
- package/src/models/site/site.schema.js +1 -0
- package/src/models/site-enrollment/site-enrollment.collection.js +11 -1
- package/src/models/trial-user/trial-user.collection.js +0 -2
- package/src/models/trial-user/trial-user.schema.js +5 -1
- package/src/service/index.d.ts +7 -1
- package/src/service/index.js +32 -51
- package/src/util/index.js +2 -1
- package/src/util/logger-registry.js +12 -0
- package/src/util/patcher.js +64 -112
- package/src/util/postgrest.utils.js +203 -0
- package/tsconfig.json +24 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,17 @@
|
|
|
1
|
+
## [@adobe/spacecat-shared-data-access-v3.0.0](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-data-access-v2.109.0...@adobe/spacecat-shared-data-access-v3.0.0) (2026-02-16)
|
|
2
|
+
|
|
3
|
+
### ⚠ BREAKING CHANGES
|
|
4
|
+
|
|
5
|
+
* **data-access:** data-access v3 migrates from DynamoDB/ElectroDB to Postgres/PostgREST
|
|
6
|
+
|
|
7
|
+
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
|
|
8
|
+
* data access v3 (#1349)
|
|
9
|
+
|
|
10
|
+
### Features
|
|
11
|
+
|
|
12
|
+
* data access v3 ([#1349](https://github.com/adobe/spacecat-shared/issues/1349)) ([6db6b79](https://github.com/adobe/spacecat-shared/commit/6db6b79ecb7813acad798a2435fe317cd9d1731c))
|
|
13
|
+
* **data-access:** v3 Postgres/PostgREST migration ([b79725f](https://github.com/adobe/spacecat-shared/commit/b79725f6a53bc0baac14a64acd8105a3eec56d99))
|
|
14
|
+
|
|
1
15
|
# [@adobe/spacecat-shared-data-access-v2.109.0](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-data-access-v2.108.0...@adobe/spacecat-shared-data-access-v2.109.0) (2026-02-16)
|
|
2
16
|
|
|
3
17
|
|
package/README.md
CHANGED
|
@@ -1,6 +1,11 @@
|
|
|
1
|
-
#
|
|
1
|
+
# Spacecat Shared Data Access (v3)
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
`@adobe/spacecat-shared-data-access` is the shared data-access layer used by Spacecat services.
|
|
4
|
+
|
|
5
|
+
This package is **v3 Postgres-first**:
|
|
6
|
+
- Primary datastore: **Postgres via PostgREST** (`@supabase/postgrest-js`)
|
|
7
|
+
- Optional secondary datastore: **S3** (for `Configuration`)
|
|
8
|
+
- No ElectroDB/DynamoDB runtime dependency in v3 behavior
|
|
4
9
|
|
|
5
10
|
## Installation
|
|
6
11
|
|
|
@@ -8,293 +13,303 @@ This Node.js module, `spacecat-shared-data-access`, is a data access layer for m
|
|
|
8
13
|
npm install @adobe/spacecat-shared-data-access
|
|
9
14
|
```
|
|
10
15
|
|
|
16
|
+
## What You Get
|
|
17
|
+
|
|
18
|
+
The package provides:
|
|
19
|
+
- `createDataAccess(config, log?, client?)`
|
|
20
|
+
- `dataAccessWrapper(fn)` (default export) for Helix/Lambda style handlers
|
|
21
|
+
- Entity collections/models with stable external API shape for services
|
|
22
|
+
|
|
23
|
+
## Quick Start
|
|
24
|
+
|
|
25
|
+
```js
|
|
26
|
+
import { createDataAccess } from '@adobe/spacecat-shared-data-access';
|
|
27
|
+
|
|
28
|
+
const dataAccess = createDataAccess({
|
|
29
|
+
postgrestUrl: process.env.POSTGREST_URL,
|
|
30
|
+
postgrestSchema: process.env.POSTGREST_SCHEMA || 'public',
|
|
31
|
+
postgrestApiKey: process.env.POSTGREST_API_KEY,
|
|
32
|
+
// Only needed if you use Configuration entity:
|
|
33
|
+
s3Bucket: process.env.S3_CONFIG_BUCKET,
|
|
34
|
+
region: process.env.AWS_REGION,
|
|
35
|
+
}, console);
|
|
36
|
+
|
|
37
|
+
const site = await dataAccess.Site.findById('0983c6da-0dee-45cc-b897-3f1fed6b460b');
|
|
38
|
+
console.log(site?.getBaseURL());
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
## Configuration
|
|
42
|
+
|
|
43
|
+
### `createDataAccess` config
|
|
44
|
+
|
|
45
|
+
- `postgrestUrl` (required): Base URL of PostgREST server
|
|
46
|
+
- `postgrestSchema` (optional): Postgres schema exposed by PostgREST, default `public`
|
|
47
|
+
- `postgrestApiKey` (optional): Added as `apikey` and `Authorization: Bearer ...`
|
|
48
|
+
- `postgrestHeaders` (optional): Extra headers for PostgREST client
|
|
49
|
+
- `s3Bucket` (optional): Required only for `Configuration` entity
|
|
50
|
+
- `region` (optional): AWS region for S3 client
|
|
51
|
+
|
|
52
|
+
### Custom PostgREST client
|
|
53
|
+
|
|
54
|
+
You can inject an already-constructed PostgREST client as third argument:
|
|
55
|
+
|
|
56
|
+
```js
|
|
57
|
+
import { PostgrestClient } from '@supabase/postgrest-js';
|
|
58
|
+
import { createDataAccess } from '@adobe/spacecat-shared-data-access';
|
|
59
|
+
|
|
60
|
+
const client = new PostgrestClient(process.env.POSTGREST_URL, { schema: 'public' });
|
|
61
|
+
const dataAccess = createDataAccess({ postgrestUrl: process.env.POSTGREST_URL }, console, client);
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
## Wrapper Usage
|
|
65
|
+
|
|
66
|
+
Default export is a wrapper that attaches `context.dataAccess`.
|
|
67
|
+
|
|
68
|
+
```js
|
|
69
|
+
import wrap from '@adobe/helix-shared-wrap';
|
|
70
|
+
import dataAccessWrapper from '@adobe/spacecat-shared-data-access';
|
|
71
|
+
|
|
72
|
+
async function run(request, context) {
|
|
73
|
+
const { dataAccess } = context;
|
|
74
|
+
const site = await dataAccess.Site.findById(request.params.siteId);
|
|
75
|
+
return {
|
|
76
|
+
statusCode: site ? 200 : 404,
|
|
77
|
+
body: site ? site.toJSON() : { error: 'not found' },
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
export const main = wrap(run)
|
|
82
|
+
.with(dataAccessWrapper);
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
The wrapper reads from `context.env`:
|
|
86
|
+
- `POSTGREST_URL` (default fallback: `http://localhost:3000`)
|
|
87
|
+
- `POSTGREST_SCHEMA`
|
|
88
|
+
- `POSTGREST_API_KEY`
|
|
89
|
+
- `S3_CONFIG_BUCKET`
|
|
90
|
+
- `AWS_REGION`
|
|
91
|
+
|
|
92
|
+
## Field Mapping Behavior
|
|
93
|
+
|
|
94
|
+
Public model API remains camelCase while Postgres/PostgREST tables are snake_case.
|
|
95
|
+
|
|
96
|
+
Examples:
|
|
97
|
+
- `site.siteId` <-> `sites.id`
|
|
98
|
+
- `site.baseURL` <-> `sites.base_url`
|
|
99
|
+
|
|
100
|
+
The mapping is handled in the base PostgREST utilities and applied on both read and write paths.
|
|
101
|
+
|
|
11
102
|
## Entities
|
|
12
103
|
|
|
13
|
-
|
|
14
|
-
-
|
|
15
|
-
-
|
|
16
|
-
-
|
|
17
|
-
-
|
|
18
|
-
-
|
|
19
|
-
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
-
|
|
23
|
-
-
|
|
24
|
-
-
|
|
25
|
-
-
|
|
26
|
-
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
-
|
|
30
|
-
-
|
|
31
|
-
-
|
|
32
|
-
-
|
|
33
|
-
-
|
|
34
|
-
-
|
|
35
|
-
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
-
|
|
39
|
-
-
|
|
40
|
-
-
|
|
41
|
-
-
|
|
42
|
-
-
|
|
43
|
-
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
-
|
|
59
|
-
-
|
|
60
|
-
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
-
|
|
65
|
-
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
-
|
|
76
|
-
-
|
|
77
|
-
-
|
|
78
|
-
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
### FixEntity
|
|
89
|
-
- **fixEntityId** (String): Unique identifier for the fix entity.
|
|
90
|
-
- **opportunityId** (String): ID of the associated opportunity.
|
|
91
|
-
- **createdAt** (String): Timestamp of creation.
|
|
92
|
-
- **updatedAt** (String): Timestamp of the last update.
|
|
93
|
-
- **type** (String): Type of the fix entity (from Suggestion.TYPES).
|
|
94
|
-
- **status** (String): Status of the fix entity (PENDING, DEPLOYED, PUBLISHED, FAILED, ROLLED_BACK).
|
|
95
|
-
- **executedBy** (String): Who executed the fix.
|
|
96
|
-
- **executedAt** (String): When the fix was executed.
|
|
97
|
-
- **publishedAt** (String): When the fix was published.
|
|
98
|
-
- **changeDetails** (Object): Details of the changes made.
|
|
99
|
-
|
|
100
|
-
### Suggestion
|
|
101
|
-
- **suggestionId** (String): Unique identifier for the suggestion.
|
|
102
|
-
- **opportunityId** (String): ID of the associated opportunity.
|
|
103
|
-
- **updatedAt** (String): Timestamp of the last update.
|
|
104
|
-
- **createdAt** (String): Timestamp of creation.
|
|
105
|
-
- **status** (String): Status of the suggestion (NEW, APPROVED, IN_PROGRESS, SKIPPED, FIXED, ERROR, OUTDATED, PENDING_VALIDATION, REJECTED).
|
|
106
|
-
- **type** (String): Type of the suggestion (CODE_CHANGE, CONTENT_UPDATE, REDIRECT_UPDATE, METADATA_UPDATE, AI_INSIGHTS, CONFIG_UPDATE).
|
|
107
|
-
- **rank** (Number): Rank/priority of the suggestion.
|
|
108
|
-
- **data** (Object): Data payload for the suggestion.
|
|
109
|
-
- **kpiDeltas** (Object): KPI delta information (optional).
|
|
110
|
-
|
|
111
|
-
### FixEntitySuggestion
|
|
112
|
-
- **suggestionId** (String): ID of the associated suggestion (primary partition key).
|
|
113
|
-
- **fixEntityId** (String): ID of the associated fix entity (primary sort key).
|
|
114
|
-
- **opportunityId** (String): ID of the associated opportunity.
|
|
115
|
-
- **fixEntityCreatedAt** (String): Creation timestamp of the fix entity.
|
|
116
|
-
- **fixEntityCreatedDate** (String): Date portion of fixEntityCreatedAt (auto-generated).
|
|
117
|
-
- **createdAt** (String): Timestamp of creation.
|
|
118
|
-
- **updatedAt** (String): Timestamp of the last update.
|
|
119
|
-
|
|
120
|
-
## DynamoDB Data Model
|
|
121
|
-
|
|
122
|
-
The module is designed to work with the following DynamoDB tables:
|
|
123
|
-
|
|
124
|
-
1. **Sites Table**: Manages site records.
|
|
125
|
-
2. **Audits Table**: Stores audit information for each site.
|
|
126
|
-
3. **Latest Audits Table**: Holds only the latest audit for each site for quick access.
|
|
127
|
-
4. **Site Candidates Table**: Manages site candidates.
|
|
128
|
-
5. **Site Top Pages Table**: Stores top pages for each site.
|
|
129
|
-
|
|
130
|
-
Each table is designed with scalability and efficient querying in mind, utilizing both key and non-key attributes effectively.
|
|
131
|
-
|
|
132
|
-
For a detailed schema, refer to `docs/schema.json`. This schema is importable to Amazon NoSQL Workbench and used by the integration tests.
|
|
133
|
-
|
|
134
|
-
## Integration Testing
|
|
135
|
-
|
|
136
|
-
The module includes comprehensive integration tests embedding a local DynamoDB server with in-memory storage for testing:
|
|
104
|
+
Current exported entities include:
|
|
105
|
+
- `ApiKey`
|
|
106
|
+
- `AsyncJob`
|
|
107
|
+
- `Audit`
|
|
108
|
+
- `AuditUrl`
|
|
109
|
+
- `Configuration`
|
|
110
|
+
- `Entitlement`
|
|
111
|
+
- `Experiment`
|
|
112
|
+
- `FixEntity`
|
|
113
|
+
- `FixEntitySuggestion`
|
|
114
|
+
- `ImportJob`
|
|
115
|
+
- `ImportUrl`
|
|
116
|
+
- `KeyEvent`
|
|
117
|
+
- `LatestAudit`
|
|
118
|
+
- `Opportunity`
|
|
119
|
+
- `Organization`
|
|
120
|
+
- `PageCitability`
|
|
121
|
+
- `PageIntent`
|
|
122
|
+
- `Project`
|
|
123
|
+
- `Report`
|
|
124
|
+
- `ScrapeJob`
|
|
125
|
+
- `ScrapeUrl`
|
|
126
|
+
- `SentimentGuideline`
|
|
127
|
+
- `SentimentTopic`
|
|
128
|
+
- `Site`
|
|
129
|
+
- `SiteCandidate`
|
|
130
|
+
- `SiteEnrollment`
|
|
131
|
+
- `SiteTopForm`
|
|
132
|
+
- `SiteTopPage`
|
|
133
|
+
- `Suggestion`
|
|
134
|
+
- `TrialUser`
|
|
135
|
+
- `TrialUserActivity`
|
|
136
|
+
|
|
137
|
+
## V3 Behavior Notes
|
|
138
|
+
|
|
139
|
+
- `Configuration` remains S3-backed in v3.
|
|
140
|
+
- `KeyEvent` is deprecated in v3 and intentionally throws on access/mutation methods.
|
|
141
|
+
- `LatestAudit` is virtual in v3 and derived from `Audit` queries (no dedicated table required).
|
|
142
|
+
|
|
143
|
+
## Migrating from V2
|
|
144
|
+
|
|
145
|
+
If you are upgrading from DynamoDB/ElectroDB-based v2:
|
|
146
|
+
|
|
147
|
+
### What stays the same
|
|
148
|
+
|
|
149
|
+
- You still use `createDataAccess(...)`.
|
|
150
|
+
- You still access collections through `dataAccess.<Entity>` (for example `dataAccess.Site`).
|
|
151
|
+
- Model/collection APIs are intended to stay stable for service callers.
|
|
152
|
+
|
|
153
|
+
### What changes
|
|
154
|
+
|
|
155
|
+
- Backing store is now Postgres via PostgREST, not DynamoDB/ElectroDB.
|
|
156
|
+
- You must provide `postgrestUrl` (or `POSTGREST_URL` via wrapper env).
|
|
157
|
+
- `Configuration` remains S3-backed (requires `s3Bucket`/`S3_CONFIG_BUCKET` when used).
|
|
158
|
+
- `KeyEvent` is deprecated in v3 and now throws.
|
|
159
|
+
- `LatestAudit` is no longer a dedicated table and is computed from `Audit` queries.
|
|
160
|
+
|
|
161
|
+
### Required environment/config updates
|
|
162
|
+
|
|
163
|
+
- Replace old Dynamo-specific configuration with:
|
|
164
|
+
- `POSTGREST_URL`
|
|
165
|
+
- optional `POSTGREST_SCHEMA`
|
|
166
|
+
- optional `POSTGREST_API_KEY`
|
|
167
|
+
- Keep S3 config envs only if using `Configuration`:
|
|
168
|
+
- `S3_CONFIG_BUCKET`
|
|
169
|
+
- `AWS_REGION`
|
|
170
|
+
|
|
171
|
+
## Development
|
|
172
|
+
|
|
173
|
+
## Local Development
|
|
174
|
+
|
|
175
|
+
### First-time setup
|
|
176
|
+
|
|
177
|
+
From the monorepo root:
|
|
137
178
|
|
|
138
179
|
```bash
|
|
139
|
-
npm
|
|
180
|
+
npm install
|
|
140
181
|
```
|
|
141
182
|
|
|
142
|
-
|
|
183
|
+
Optional: verify package tooling from this workspace:
|
|
143
184
|
|
|
144
|
-
|
|
185
|
+
```bash
|
|
186
|
+
cd packages/spacecat-shared-data-access
|
|
187
|
+
node -v
|
|
188
|
+
npm -v
|
|
189
|
+
```
|
|
145
190
|
|
|
146
|
-
|
|
191
|
+
### Day-to-day workflow
|
|
147
192
|
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
- `getSiteByBaseURLWithAuditInfo`
|
|
154
|
-
- `getSiteByBaseURLWithAudits`
|
|
155
|
-
- `getSiteByBaseURLWithLatestAudit`
|
|
156
|
-
- `addSite`
|
|
157
|
-
- `updateSite`
|
|
158
|
-
- `removeSite`
|
|
159
|
-
- `findByPreviewURL`
|
|
160
|
-
- `findByExternalOwnerIdAndExternalSiteId`
|
|
193
|
+
1. Create/switch to a feature branch.
|
|
194
|
+
2. Make code changes in `src/` and tests in `test/unit` and `test/it`.
|
|
195
|
+
3. Run unit tests while iterating.
|
|
196
|
+
4. Run integration tests before opening/merging a PR.
|
|
197
|
+
5. Run lint and fix issues.
|
|
161
198
|
|
|
162
|
-
###
|
|
163
|
-
- `getSiteCandidateByBaseURL`
|
|
164
|
-
- `upsertSiteCandidate`
|
|
165
|
-
- `siteCandidateExists`
|
|
166
|
-
- `updateSiteCandidate`
|
|
199
|
+
### Common commands (from `packages/spacecat-shared-data-access`)
|
|
167
200
|
|
|
168
|
-
###
|
|
169
|
-
- `getAuditsForSite`
|
|
170
|
-
- `getAuditForSite`
|
|
171
|
-
- `getLatestAudits`
|
|
172
|
-
- `getLatestAuditForSite`
|
|
173
|
-
- `addAudit`
|
|
201
|
+
### Run unit tests
|
|
174
202
|
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
203
|
+
```bash
|
|
204
|
+
npm test
|
|
205
|
+
```
|
|
178
206
|
|
|
179
|
-
###
|
|
180
|
-
- `getSuggestionsByFixEntityId` - Gets all suggestions associated with a specific FixEntity
|
|
181
|
-
- `setSuggestionsForFixEntity` - Sets suggestions for a FixEntity by managing junction table relationships
|
|
207
|
+
### Run unit tests with debugger
|
|
182
208
|
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
209
|
+
```bash
|
|
210
|
+
npm run test:debug
|
|
211
|
+
```
|
|
186
212
|
|
|
187
|
-
###
|
|
188
|
-
- `allBySuggestionId` - Gets all junction records associated with a specific Suggestion
|
|
189
|
-
- `allByFixEntityId` - Gets all junction records associated with a specific FixEntity
|
|
213
|
+
### Run integration tests
|
|
190
214
|
|
|
191
|
-
|
|
215
|
+
```bash
|
|
216
|
+
npm run test:it
|
|
217
|
+
```
|
|
192
218
|
|
|
193
|
-
|
|
194
|
-
This integration allows your Lambda functions to access and manipulate data.
|
|
219
|
+
### Run lint
|
|
195
220
|
|
|
196
|
-
|
|
221
|
+
```bash
|
|
222
|
+
npm run lint
|
|
223
|
+
```
|
|
197
224
|
|
|
198
|
-
|
|
225
|
+
### Auto-fix lint issues
|
|
199
226
|
|
|
200
|
-
|
|
227
|
+
```bash
|
|
228
|
+
npm run lint:fix
|
|
229
|
+
```
|
|
201
230
|
|
|
202
|
-
|
|
203
|
-
import dataAccessWrapper from '@adobe/spacecat-shared-data-access';
|
|
204
|
-
```
|
|
231
|
+
### Clean local install artifacts
|
|
205
232
|
|
|
206
|
-
|
|
233
|
+
```bash
|
|
234
|
+
npm run clean
|
|
235
|
+
```
|
|
207
236
|
|
|
208
|
-
|
|
209
|
-
secret assigned to your Lambda function.
|
|
237
|
+
The integration suite under `test/it` is PostgREST-based and runs via Docker.
|
|
210
238
|
|
|
211
|
-
|
|
212
|
-
const { DYNAMO_TABLE_NAME_DATA } = context.env;
|
|
213
|
-
```
|
|
239
|
+
## Integration Tests
|
|
214
240
|
|
|
215
|
-
|
|
241
|
+
Integration tests run a local Postgres + PostgREST stack via Docker Compose and execute
|
|
242
|
+
the mocha suite under `test/it`.
|
|
216
243
|
|
|
217
|
-
|
|
244
|
+
### Prerequisites
|
|
218
245
|
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
.with(dataAccessWrapper) // Add this line
|
|
223
|
-
.with(sqs)
|
|
224
|
-
.with(secrets)
|
|
225
|
-
.with(helixStatus);
|
|
226
|
-
```
|
|
246
|
+
- Docker Desktop (or equivalent Docker daemon)
|
|
247
|
+
- AWS CLI configured with credentials that can access the Spacecat Development AWS account
|
|
248
|
+
ECR repository (only needed when pulling the default private ECR image)
|
|
227
249
|
|
|
228
|
-
|
|
250
|
+
### Default image used by IT harness
|
|
229
251
|
|
|
230
|
-
|
|
252
|
+
- Repository: `682033462621.dkr.ecr.us-east-1.amazonaws.com/mysticat-data-service`
|
|
253
|
+
- Tag: `v1.11.0` (override via env var)
|
|
231
254
|
|
|
232
|
-
|
|
233
|
-
async function run(message, context) {
|
|
234
|
-
const { dataAccess } = context;
|
|
235
|
-
|
|
236
|
-
// Example: Retrieve all sites
|
|
237
|
-
const sites = await dataAccess.Site.getSites();
|
|
238
|
-
// ... more logic ...
|
|
239
|
-
}
|
|
240
|
-
```
|
|
255
|
+
### Authenticate Docker to ECR
|
|
241
256
|
|
|
242
|
-
|
|
257
|
+
The default image is in a private ECR repo in:
|
|
258
|
+
- **SpaceCat Development (AWS3338)**
|
|
243
259
|
|
|
244
|
-
|
|
260
|
+
If you are setting this up for the first time:
|
|
261
|
+
1. Get AWS credentials for **SpaceCat Development (AWS3338)** from `klam.corp.adobe.com`.
|
|
262
|
+
2. Add them to `~/.aws/credentials` under a profile name you choose.
|
|
263
|
+
3. Use that profile in the ECR login command.
|
|
245
264
|
|
|
246
|
-
|
|
247
|
-
import wrap from '@adobe/helix-shared-wrap';
|
|
248
|
-
import dataAccessWrapper from '@adobe/spacecat-shared-data-access';
|
|
249
|
-
import sqsEventAdapter from './sqsEventAdapter';
|
|
250
|
-
import sqs from './sqs';
|
|
251
|
-
import secrets from '@adobe/helix-shared-secrets';
|
|
252
|
-
import helixStatus from '@adobe/helix-status';
|
|
265
|
+
Example `~/.aws/credentials` entry:
|
|
253
266
|
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
} catch (error) {
|
|
260
|
-
// Error handling
|
|
261
|
-
}
|
|
262
|
-
}
|
|
267
|
+
```ini
|
|
268
|
+
[spacecat-dev]
|
|
269
|
+
aws_access_key_id = <your-access-key-id>
|
|
270
|
+
aws_secret_access_key = <your-secret-access-key>
|
|
271
|
+
```
|
|
263
272
|
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
273
|
+
Repository:
|
|
274
|
+
|
|
275
|
+
- `682033462621.dkr.ecr.us-east-1.amazonaws.com/mysticat-data-service`
|
|
276
|
+
|
|
277
|
+
Then authenticate Docker to ECR:
|
|
278
|
+
|
|
279
|
+
```bash
|
|
280
|
+
aws ecr get-login-password --profile spacecat-dev --region us-east-1 \
|
|
281
|
+
| docker login --username AWS --password-stdin 682033462621.dkr.ecr.us-east-1.amazonaws.com
|
|
270
282
|
```
|
|
271
283
|
|
|
272
|
-
|
|
284
|
+
### Run
|
|
273
285
|
|
|
274
|
-
|
|
286
|
+
```bash
|
|
287
|
+
npm run test:it
|
|
288
|
+
```
|
|
275
289
|
|
|
276
|
-
|
|
290
|
+
### Useful overrides
|
|
277
291
|
|
|
278
|
-
|
|
292
|
+
- `MYSTICAT_DATA_SERVICE_TAG`: override image tag (recommended for version bumps)
|
|
293
|
+
- `MYSTICAT_DATA_SERVICE_REPOSITORY`: override image repository
|
|
294
|
+
- `MYSTICAT_DATA_SERVICE_PLATFORM`: override container platform (default `linux/amd64`)
|
|
295
|
+
- `IT_POSTGREST_PORT`: override exposed PostgREST port (default `3300`)
|
|
296
|
+
- `IT_POSTGRES_PORT`: override exposed Postgres port (default `55432`)
|
|
279
297
|
|
|
280
|
-
|
|
298
|
+
```bash
|
|
299
|
+
export MYSTICAT_DATA_SERVICE_TAG=v1.11.0
|
|
300
|
+
export MYSTICAT_DATA_SERVICE_PLATFORM=linux/amd64
|
|
301
|
+
# optional if repository changes
|
|
302
|
+
export MYSTICAT_DATA_SERVICE_REPOSITORY=682033462621.dkr.ecr.us-east-1.amazonaws.com/mysticat-data-service
|
|
303
|
+
```
|
|
281
304
|
|
|
282
|
-
|
|
283
|
-
2. Get the commit ID of your push
|
|
284
|
-
3. In your dependent project, temporarily modify the package.json dependency:
|
|
305
|
+
## TypeScript
|
|
285
306
|
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
// To:
|
|
291
|
-
"@adobe/spacecat-shared-data-access": "https://gitpkg.now.sh/adobe/spacecat-shared/packages/spacecat-shared-data-access?YOUR_COMMIT_ID",
|
|
292
|
-
```
|
|
307
|
+
Type definitions are shipped from:
|
|
308
|
+
- `src/index.d.ts`
|
|
309
|
+
- `src/models/**/index.d.ts`
|
|
293
310
|
|
|
294
|
-
|
|
295
|
-
5. Test your changes
|
|
296
|
-
6. Once testing is complete and your PR is merged, update the dependent project to use the released version
|
|
311
|
+
Use the package directly in TS projects; no extra setup required.
|
|
297
312
|
|
|
298
313
|
## License
|
|
299
314
|
|
|
300
|
-
|
|
315
|
+
Apache-2.0
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# Docker Compose for integration testing — PostgreSQL + PostgREST
|
|
2
|
+
name: spacecat-data-access-test
|
|
3
|
+
|
|
4
|
+
services:
|
|
5
|
+
db:
|
|
6
|
+
image: postgres:16-alpine
|
|
7
|
+
container_name: spacecat-test-db
|
|
8
|
+
environment:
|
|
9
|
+
POSTGRES_USER: postgres
|
|
10
|
+
POSTGRES_PASSWORD: postgres
|
|
11
|
+
POSTGRES_DB: spacecat_test
|
|
12
|
+
ports:
|
|
13
|
+
- "54320:5432"
|
|
14
|
+
tmpfs:
|
|
15
|
+
- /var/lib/postgresql/data
|
|
16
|
+
volumes:
|
|
17
|
+
- ./test/it/v3/init-db.sql:/docker-entrypoint-initdb.d/01-init.sql:ro
|
|
18
|
+
healthcheck:
|
|
19
|
+
test: ["CMD-SHELL", "pg_isready -U postgres"]
|
|
20
|
+
interval: 2s
|
|
21
|
+
timeout: 3s
|
|
22
|
+
retries: 10
|
|
23
|
+
|
|
24
|
+
postgrest:
|
|
25
|
+
image: postgrest/postgrest:v14.4
|
|
26
|
+
container_name: spacecat-test-postgrest
|
|
27
|
+
depends_on:
|
|
28
|
+
db:
|
|
29
|
+
condition: service_healthy
|
|
30
|
+
environment:
|
|
31
|
+
PGRST_DB_URI: postgres://postgrest_authenticator:postgrest@db:5432/spacecat_test
|
|
32
|
+
PGRST_DB_SCHEMAS: public
|
|
33
|
+
PGRST_DB_ANON_ROLE: postgrest_anon
|
|
34
|
+
PGRST_DB_EXTRA_SEARCH_PATH: ""
|
|
35
|
+
PGRST_LOG_LEVEL: warn
|
|
36
|
+
PGRST_ADMIN_SERVER_PORT: 3001
|
|
37
|
+
ports:
|
|
38
|
+
- "3456:3000"
|
|
39
|
+
- "3457:3001"
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@adobe/spacecat-shared-data-access",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "3.0.0",
|
|
4
4
|
"description": "Shared modules of the Spacecat Services - Data Access",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"engines": {
|
|
@@ -10,7 +10,9 @@
|
|
|
10
10
|
"main": "src/index.js",
|
|
11
11
|
"types": "src/index.d.ts",
|
|
12
12
|
"scripts": {
|
|
13
|
-
"test:it": "
|
|
13
|
+
"test:it": "npm run test:it:legacy && npm run test:it:postgrest",
|
|
14
|
+
"test:it:legacy": "IT_SEED_MODE=none mocha --require ./test/it/fixtures.js --spec \"test/it/**/*.test.js\" --ignore \"test/it/postgrest/**/*.test.js\"",
|
|
15
|
+
"test:it:postgrest": "IT_SEED_MODE=tenant-sql mocha --require ./test/it/fixtures.js --spec \"test/it/postgrest/**/*.test.js\"",
|
|
14
16
|
"test": "c8 mocha --spec \"test/unit/**/*.test.js\"",
|
|
15
17
|
"test:debug": "mocha --inspect-brk --require ./test/setup-env.js --spec \"test/unit/**/*.test.js\"",
|
|
16
18
|
"test:debug:config": "mocha --inspect-brk --require ./test/setup-env.js --spec \"test/unit/models/site/config.test.js\"",
|
|
@@ -39,19 +41,16 @@
|
|
|
39
41
|
},
|
|
40
42
|
"dependencies": {
|
|
41
43
|
"@adobe/spacecat-shared-utils": "1.81.1",
|
|
42
|
-
"@
|
|
44
|
+
"@supabase/postgrest-js": "^1.21.4",
|
|
43
45
|
"@aws-sdk/client-s3": "^3.940.0",
|
|
44
|
-
"@aws-sdk/lib-dynamodb": "3.940.0",
|
|
45
46
|
"@types/joi": "17.2.3",
|
|
46
47
|
"aws-xray-sdk": "3.12.0",
|
|
47
|
-
"electrodb": "3.5.0",
|
|
48
48
|
"joi": "18.0.2",
|
|
49
49
|
"pluralize": "8.0.0"
|
|
50
50
|
},
|
|
51
51
|
"devDependencies": {
|
|
52
52
|
"chai": "6.2.1",
|
|
53
53
|
"chai-as-promised": "8.0.2",
|
|
54
|
-
"dynamo-db-local": "9.6.0",
|
|
55
54
|
"nock": "14.0.10",
|
|
56
55
|
"sinon": "21.0.0",
|
|
57
56
|
"sinon-chai": "4.0.1"
|