@bernierllc/contentful-sync-service 1.5.0 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -4
- package/dist/contentful-sync-service.d.ts +24 -5
- package/dist/contentful-sync-service.js +84 -25
- package/dist/index.d.ts +1 -1
- package/dist/types.d.ts +11 -0
- package/package.json +1 -2
package/README.md
CHANGED
|
@@ -186,9 +186,9 @@ const result = await service.incrementalSync();
|
|
|
186
186
|
console.log(result.stats.itemsDeleted); // Number of items deleted
|
|
187
187
|
```
|
|
188
188
|
|
|
189
|
-
##### `processWebhook(body: string, headers: Record<string, string>): Promise<
|
|
189
|
+
##### `processWebhook(body: string, headers: Record<string, string>): Promise<WebhookApplyResult>`
|
|
190
190
|
|
|
191
|
-
|
|
191
|
+
Verify a Contentful webhook, then run `incrementalSync()` for it (see `applyWebhookEvent` below). It returns `{ topic, id, outcome }`, where `outcome` is `'synced'` or `'ignored'`. A failed sync throws, so answer 500 and Contentful retries the delivery.
|
|
192
192
|
|
|
193
193
|
```typescript
|
|
194
194
|
app.post('/webhook', async (req, res) => {
|
|
@@ -200,6 +200,21 @@ app.post('/webhook', async (req, res) => {
|
|
|
200
200
|
});
|
|
201
201
|
```
|
|
202
202
|
|
|
203
|
+
##### `applyWebhookEvent(topic: string, payload: ContentfulWebhookPayload): Promise<WebhookApplyResult>`
|
|
204
|
+
|
|
205
|
+
Handle an already-verified event. Use it when your own receiver checks signatures. The webhook body is never written to storage: Contentful delivers webhooks unordered, with retries and duplicates, while the CDA sync API is ordered and authoritative. So a delivery change triggers `incrementalSync()`, and storage only ever reflects what the sync returns.
|
|
206
|
+
|
|
207
|
+
| Topic | Effect |
|
|
208
|
+
|-------|--------|
|
|
209
|
+
| `Entry`/`Asset` `.publish`, `.unpublish`, `.archive`, `.delete` | Runs `incrementalSync()` (`synced`) |
|
|
210
|
+
| Draft topics (`create`, `save`, `auto_save`, `unarchive`), `ContentType.*`, events for another space or environment | Nothing (`ignored`) |
|
|
211
|
+
|
|
212
|
+
Syncs never overlap: while one runs, every new request waits for one follow-up sync that starts after it, so at most one sync runs and one more is queued at any time, and none races on the sync token. `startAutoSync()` uses the same path.
|
|
213
|
+
|
|
214
|
+
The CDA can lag the webhook by a few seconds. A change the follow-up sync does not see yet is picked up by the next webhook or auto-sync, so keep `startAutoSync()` running.
|
|
215
|
+
|
|
216
|
+
Both `ContentManagement.Entry.publish` (the `X-Contentful-Topic` header) and `Entry.publish` work.
|
|
217
|
+
|
|
203
218
|
##### `startAutoSync(): void`
|
|
204
219
|
|
|
205
220
|
Start automatic incremental sync at the configured interval.
|
|
@@ -332,11 +347,12 @@ if (!result.success) {
|
|
|
332
347
|
}
|
|
333
348
|
```
|
|
334
349
|
|
|
335
|
-
`processWebhook()`
|
|
350
|
+
`processWebhook()` and `applyWebhookEvent()` throw a `ContentfulSyncServiceError`:
|
|
336
351
|
|
|
337
352
|
| Code | When |
|
|
338
353
|
|------|------|
|
|
339
|
-
| `INVALID_INPUT` | No `webhookSecret` was configured,
|
|
354
|
+
| `INVALID_INPUT` | No `webhookSecret` was configured, the signature or payload is invalid, or the topic or `sys.id` is missing |
|
|
355
|
+
| `OPERATION_FAILED` | The sync for the event failed (the sync's `ContentfulSyncServiceError` in `cause`) |
|
|
340
356
|
|
|
341
357
|
Sync failures are logged with these codes:
|
|
342
358
|
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { ContentfulWebhookPayload } from '@bernierllc/contentful-types';
|
|
2
|
+
import type { SyncServiceConfig, SyncResult, WebhookApplyResult } from './types';
|
|
2
3
|
/**
|
|
3
4
|
* ContentfulSyncService
|
|
4
5
|
*
|
|
@@ -10,10 +11,11 @@ export declare class ContentfulSyncService {
|
|
|
10
11
|
private readonly exportAdapter;
|
|
11
12
|
private readonly cdaClient;
|
|
12
13
|
private readonly webhookHandler?;
|
|
13
|
-
private readonly queue;
|
|
14
14
|
private readonly logger;
|
|
15
15
|
private readonly neverhub;
|
|
16
16
|
private syncIntervalId?;
|
|
17
|
+
private syncRun;
|
|
18
|
+
private syncFollowUp;
|
|
17
19
|
constructor(config: SyncServiceConfig);
|
|
18
20
|
/**
|
|
19
21
|
* Register with NeverHub. register() no-ops when NeverHub is unreachable,
|
|
@@ -31,16 +33,33 @@ export declare class ContentfulSyncService {
|
|
|
31
33
|
*/
|
|
32
34
|
initialSync(managementToken: string): Promise<SyncResult>;
|
|
33
35
|
/**
|
|
34
|
-
* Perform incremental sync using CDA Sync API
|
|
36
|
+
* Perform incremental sync using CDA Sync API. A call made while a sync is running waits for
|
|
37
|
+
* one follow-up run, so every caller gets a sync that started after its call.
|
|
35
38
|
*/
|
|
36
39
|
incrementalSync(): Promise<SyncResult>;
|
|
40
|
+
private syncOnce;
|
|
41
|
+
private runIncrementalSync;
|
|
37
42
|
/**
|
|
38
|
-
*
|
|
43
|
+
* Verify a webhook and apply it to storage (see applyWebhookEvent)
|
|
39
44
|
*
|
|
40
45
|
* @param body - Webhook request body
|
|
41
46
|
* @param headers - Webhook request headers
|
|
42
47
|
*/
|
|
43
|
-
processWebhook(body: string, headers: Record<string, string>): Promise<
|
|
48
|
+
processWebhook(body: string, headers: Record<string, string>): Promise<WebhookApplyResult>;
|
|
49
|
+
/**
|
|
50
|
+
* Bring storage up to date for a verified webhook event. A publish, unpublish, archive or
|
|
51
|
+
* delete of an Entry or Asset in this space and environment runs `incrementalSync()`; the
|
|
52
|
+
* payload itself is never written, because the CDA sync is ordered and Contentful's webhook
|
|
53
|
+
* deliveries are not (retries and duplicates can arrive late). Draft topics (create, save,
|
|
54
|
+
* auto_save, unarchive), ContentType topics and events for another space or environment run
|
|
55
|
+
* nothing and come back as `ignored`.
|
|
56
|
+
*
|
|
57
|
+
* @param topic - `x-contentful-topic`, e.g. `ContentManagement.Entry.publish` or `Entry.publish`
|
|
58
|
+
* @param payload - Webhook body, already verified by the caller
|
|
59
|
+
* @throws ContentfulSyncServiceError `INVALID_INPUT` for a malformed topic or payload,
|
|
60
|
+
* `OPERATION_FAILED` (sync error in `cause`) when the sync fails, so the webhook is retried
|
|
61
|
+
*/
|
|
62
|
+
applyWebhookEvent(topic: string, payload: ContentfulWebhookPayload): Promise<WebhookApplyResult>;
|
|
44
63
|
/**
|
|
45
64
|
* Start automatic sync at configured interval
|
|
46
65
|
*/
|
|
@@ -11,7 +11,6 @@ exports.ContentfulSyncService = void 0;
|
|
|
11
11
|
const contentful_export_adapter_1 = require("@bernierllc/contentful-export-adapter");
|
|
12
12
|
const contentful_cda_client_1 = require("@bernierllc/contentful-cda-client");
|
|
13
13
|
const contentful_webhook_handler_1 = require("@bernierllc/contentful-webhook-handler");
|
|
14
|
-
const message_queue_1 = require("@bernierllc/message-queue");
|
|
15
14
|
const logger_1 = require("@bernierllc/logger");
|
|
16
15
|
const neverhub_adapter_1 = require("@bernierllc/neverhub-adapter");
|
|
17
16
|
const errors_1 = require("./errors");
|
|
@@ -19,7 +18,6 @@ const errors_1 = require("./errors");
|
|
|
19
18
|
// eslint-disable-next-line @typescript-eslint/no-require-imports -- JSON read at runtime, outside rootDir
|
|
20
19
|
const pkg = require('../package.json');
|
|
21
20
|
let warnedNeverhubEnabled = false;
|
|
22
|
-
/** Wrap anything that is not already ours, so every logged failure carries code and cause. */
|
|
23
21
|
function toServiceError(error, message, context) {
|
|
24
22
|
if (error instanceof errors_1.ContentfulSyncServiceError) {
|
|
25
23
|
return error;
|
|
@@ -41,10 +39,11 @@ class ContentfulSyncService {
|
|
|
41
39
|
exportAdapter;
|
|
42
40
|
cdaClient;
|
|
43
41
|
webhookHandler;
|
|
44
|
-
queue;
|
|
45
42
|
logger;
|
|
46
43
|
neverhub = new neverhub_adapter_1.NeverHubAdapter();
|
|
47
44
|
syncIntervalId;
|
|
45
|
+
syncRun = null;
|
|
46
|
+
syncFollowUp = null;
|
|
48
47
|
constructor(config) {
|
|
49
48
|
this.config = {
|
|
50
49
|
syncInterval: 300000, // 5 minutes default
|
|
@@ -59,7 +58,6 @@ class ContentfulSyncService {
|
|
|
59
58
|
if (config.webhookSecret) {
|
|
60
59
|
this.webhookHandler = new contentful_webhook_handler_1.ContentfulWebhookHandler({ webhookSecret: config.webhookSecret });
|
|
61
60
|
}
|
|
62
|
-
this.queue = new message_queue_1.MessageQueue({ name: 'contentful-sync' });
|
|
63
61
|
this.logger = (0, logger_1.createLogger)({
|
|
64
62
|
context: {
|
|
65
63
|
service: 'ContentfulSyncService',
|
|
@@ -197,9 +195,28 @@ class ContentfulSyncService {
|
|
|
197
195
|
}
|
|
198
196
|
}
|
|
199
197
|
/**
|
|
200
|
-
* Perform incremental sync using CDA Sync API
|
|
198
|
+
* Perform incremental sync using CDA Sync API. A call made while a sync is running waits for
|
|
199
|
+
* one follow-up run, so every caller gets a sync that started after its call.
|
|
201
200
|
*/
|
|
202
201
|
async incrementalSync() {
|
|
202
|
+
return (await this.syncOnce()).result;
|
|
203
|
+
}
|
|
204
|
+
// Single-flight: overlapping syncs would race on the sync token and could write an older delta last.
|
|
205
|
+
syncOnce() {
|
|
206
|
+
if (!this.syncRun) {
|
|
207
|
+
this.syncRun = this.runIncrementalSync().finally(() => {
|
|
208
|
+
this.syncRun = null;
|
|
209
|
+
});
|
|
210
|
+
return this.syncRun;
|
|
211
|
+
}
|
|
212
|
+
// The running sync may have fetched before the caller's change landed, so queue one more run after it.
|
|
213
|
+
this.syncFollowUp ??= this.syncRun.then(() => {
|
|
214
|
+
this.syncFollowUp = null;
|
|
215
|
+
return this.syncOnce();
|
|
216
|
+
});
|
|
217
|
+
return this.syncFollowUp;
|
|
218
|
+
}
|
|
219
|
+
async runIncrementalSync() {
|
|
203
220
|
const startTime = Date.now();
|
|
204
221
|
this.logger.info('Starting incremental sync', { spaceId: this.config.spaceId });
|
|
205
222
|
try {
|
|
@@ -245,10 +262,7 @@ class ContentfulSyncService {
|
|
|
245
262
|
timestamp: new Date()
|
|
246
263
|
};
|
|
247
264
|
this.logger.info('Incremental sync completed', stats);
|
|
248
|
-
return {
|
|
249
|
-
success: true,
|
|
250
|
-
stats
|
|
251
|
-
};
|
|
265
|
+
return { result: { success: true, stats } };
|
|
252
266
|
}
|
|
253
267
|
catch (error) {
|
|
254
268
|
const duration = Date.now() - startTime;
|
|
@@ -259,20 +273,23 @@ class ContentfulSyncService {
|
|
|
259
273
|
errorChain: (0, logger_1.getErrorChainEntries)(failure)
|
|
260
274
|
});
|
|
261
275
|
return {
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
276
|
+
result: {
|
|
277
|
+
success: false,
|
|
278
|
+
stats: {
|
|
279
|
+
entriesSynced: 0,
|
|
280
|
+
assetsSynced: 0,
|
|
281
|
+
itemsDeleted: 0,
|
|
282
|
+
duration,
|
|
283
|
+
timestamp: new Date()
|
|
284
|
+
},
|
|
285
|
+
error: errorMessage
|
|
269
286
|
},
|
|
270
|
-
|
|
287
|
+
failure
|
|
271
288
|
};
|
|
272
289
|
}
|
|
273
290
|
}
|
|
274
291
|
/**
|
|
275
|
-
*
|
|
292
|
+
* Verify a webhook and apply it to storage (see applyWebhookEvent)
|
|
276
293
|
*
|
|
277
294
|
* @param body - Webhook request body
|
|
278
295
|
* @param headers - Webhook request headers
|
|
@@ -291,12 +308,55 @@ class ContentfulSyncService {
|
|
|
291
308
|
context: { topic: result.topic }
|
|
292
309
|
});
|
|
293
310
|
}
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
311
|
+
return this.applyWebhookEvent(result.topic ?? '', result.payload);
|
|
312
|
+
}
|
|
313
|
+
/**
|
|
314
|
+
* Bring storage up to date for a verified webhook event. A publish, unpublish, archive or
|
|
315
|
+
* delete of an Entry or Asset in this space and environment runs `incrementalSync()`; the
|
|
316
|
+
* payload itself is never written, because the CDA sync is ordered and Contentful's webhook
|
|
317
|
+
* deliveries are not (retries and duplicates can arrive late). Draft topics (create, save,
|
|
318
|
+
* auto_save, unarchive), ContentType topics and events for another space or environment run
|
|
319
|
+
* nothing and come back as `ignored`.
|
|
320
|
+
*
|
|
321
|
+
* @param topic - `x-contentful-topic`, e.g. `ContentManagement.Entry.publish` or `Entry.publish`
|
|
322
|
+
* @param payload - Webhook body, already verified by the caller
|
|
323
|
+
* @throws ContentfulSyncServiceError `INVALID_INPUT` for a malformed topic or payload,
|
|
324
|
+
* `OPERATION_FAILED` (sync error in `cause`) when the sync fails, so the webhook is retried
|
|
325
|
+
*/
|
|
326
|
+
async applyWebhookEvent(topic, payload) {
|
|
327
|
+
// Contentful sends `ContentManagement.<Resource>.<action>`; the short `<Resource>.<action>` form is accepted too.
|
|
328
|
+
const [resource, action] = topic.split('.').slice(-2);
|
|
329
|
+
const id = payload?.sys?.id;
|
|
330
|
+
if (!resource || !action || !id) {
|
|
331
|
+
throw new errors_1.ContentfulSyncServiceError('Webhook topic or payload is malformed', {
|
|
332
|
+
code: 'INVALID_INPUT',
|
|
333
|
+
context: { topic, id }
|
|
334
|
+
});
|
|
335
|
+
}
|
|
336
|
+
const ignored = (reason) => {
|
|
337
|
+
this.logger.info('Webhook event ignored', { topic, id, reason });
|
|
338
|
+
return { topic, id, outcome: 'ignored' };
|
|
339
|
+
};
|
|
340
|
+
// Optional chaining: the payload comes off the wire, so the declared type is not a guarantee.
|
|
341
|
+
if (payload.sys.space?.sys.id !== this.config.spaceId || payload.sys.environment?.sys.id !== this.config.environmentId) {
|
|
342
|
+
return ignored('other space or environment');
|
|
343
|
+
}
|
|
344
|
+
if (resource !== 'Entry' && resource !== 'Asset') {
|
|
345
|
+
return ignored('resource not synced');
|
|
346
|
+
}
|
|
347
|
+
if (!['publish', 'unpublish', 'archive', 'delete'].includes(action)) {
|
|
348
|
+
return ignored('not a delivery change');
|
|
349
|
+
}
|
|
350
|
+
const { failure } = await this.syncOnce();
|
|
351
|
+
if (failure) {
|
|
352
|
+
throw new errors_1.ContentfulSyncServiceError('Sync for webhook event failed', {
|
|
353
|
+
code: 'OPERATION_FAILED',
|
|
354
|
+
cause: failure,
|
|
355
|
+
context: { topic, id }
|
|
356
|
+
});
|
|
357
|
+
}
|
|
358
|
+
this.logger.info('Webhook event synced', { topic, id });
|
|
359
|
+
return { topic, id, outcome: 'synced' };
|
|
300
360
|
}
|
|
301
361
|
/**
|
|
302
362
|
* Start automatic sync at configured interval
|
|
@@ -327,7 +387,6 @@ class ContentfulSyncService {
|
|
|
327
387
|
async shutdown() {
|
|
328
388
|
this.logger.info('Shutting down ContentfulSyncService');
|
|
329
389
|
this.stopAutoSync();
|
|
330
|
-
// Note: MessageQueue doesn't expose a close() method in current version
|
|
331
390
|
this.logger.info('ContentfulSyncService shutdown complete');
|
|
332
391
|
}
|
|
333
392
|
}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
export { ContentfulSyncService } from './contentful-sync-service';
|
|
2
2
|
export { ContentfulSyncServiceError, type ContentfulSyncServiceErrorCode, type ContentfulSyncServiceErrorOptions } from './errors';
|
|
3
3
|
export { InMemoryContentStorage, InMemorySyncStateStorage } from './in-memory-storage';
|
|
4
|
-
export type { ContentStorage, SyncStateStorage, SyncServiceConfig, SyncStats, SyncResult } from './types';
|
|
4
|
+
export type { ContentStorage, SyncStateStorage, SyncServiceConfig, SyncStats, WebhookApplyResult, SyncResult } from './types';
|
package/dist/types.d.ts
CHANGED
|
@@ -143,3 +143,14 @@ export interface SyncResult {
|
|
|
143
143
|
*/
|
|
144
144
|
error?: string;
|
|
145
145
|
}
|
|
146
|
+
/**
|
|
147
|
+
* What applyWebhookEvent() did with one webhook event
|
|
148
|
+
*/
|
|
149
|
+
export interface WebhookApplyResult {
|
|
150
|
+
/** Topic as received */
|
|
151
|
+
topic: string;
|
|
152
|
+
/** Entry or asset id */
|
|
153
|
+
id: string;
|
|
154
|
+
/** `synced` (an incremental sync ran and succeeded) or `ignored` (no delivery change for this space and environment) */
|
|
155
|
+
outcome: 'synced' | 'ignored';
|
|
156
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bernierllc/contentful-sync-service",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.6.0",
|
|
4
4
|
"description": "Full export + incremental webhook sync to local storage for Contentful content",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"contentful",
|
|
@@ -25,7 +25,6 @@
|
|
|
25
25
|
"@bernierllc/contentful-types": "^1.5.0",
|
|
26
26
|
"@bernierllc/contentful-webhook-handler": "^1.3.3",
|
|
27
27
|
"@bernierllc/logger": "^2.0.1",
|
|
28
|
-
"@bernierllc/message-queue": "^1.5.2",
|
|
29
28
|
"@bernierllc/neverhub-adapter": "^0.4.1"
|
|
30
29
|
},
|
|
31
30
|
"devDependencies": {
|