@stackbit/cms-contentful 0.4.49-staging.1 → 0.4.49-staging.2

package/src/contentful-content-source.ts

@@ -76,10 +76,10 @@ import {
     convertDocumentVersions
 } from './contentful-entries-converter';
 import { convertAndFilterScheduledActions } from './contentful-scheduled-actions-converter';
-import { ContentPoller, SyncResult } from './content-poller';
+import { ContentPoller, ContentPollerSyncResult, ContentPollerSyncContext } from './content-poller';
 import { Readable } from 'stream';
 import { CONTENTFUL_BUILT_IN_IMAGE_SOURCES, CONTENTFUL_BYNDER_APP, CONTENTFUL_CLOUDINARY_APP } from './contentful-consts';
-import { getLastUpdatedEntityDate, SyncContext } from './utils';
+import { getLastUpdatedEntityDate } from './utils';
 
 export interface ContentSourceOptions {
     /** Contentful Space ID */
@@ -109,7 +109,9 @@ type UserContext = {
     accessToken: string;
 };
 
-export type SchemaContext = unknown;
+export type SchemaContext = {
+    lastUpdatedContentTypeDate?: string;
+};
 
 type EntityError = {
     name: string;
@@ -138,7 +140,6 @@ export class ContentfulContentSource implements ContentSourceTypes.ContentSource
     private webhookUrl?: string;
     private devAppRestartNeeded?: () => void;
     private cache!: ContentSourceTypes.Cache<SchemaContext, DocumentContext, AssetContext>;
-    private syncContext!: SyncContext;
     private taskQueue: TaskQueue;
 
     constructor(options: ContentSourceOptions) {
@@ -197,8 +198,6 @@ export class ContentfulContentSource implements ContentSourceTypes.ContentSource
             this.userLogger.info('Using webhook for content updates');
         }
 
-        this.syncContext = ((await this.cache.get('syncContext')) as SyncContext) ?? {};
-
         this.plainClient = createPlainApiClient({
             spaceId: this.spaceId,
             accessToken: this.accessToken,
@@ -217,15 +216,6 @@ export class ContentfulContentSource implements ContentSourceTypes.ContentSource
         const appInstallations: AppInstallationProps[] = await fetchAllAppInstallations(this.plainClient);
         await this.fetchUsers();
 
-        // The contentPoller updates its syncContext property internally when
-        // it gets an updated content. But, the actual cached data is not
-        // updated. Therefore, when content source module is restarted, we need
-        // to reset the contentPoller's internal syncContext, so it will be
-        // able to fetch all the updated content from its cached state.
-        if (this.contentPoller && this.contentPoller.pollType === 'date') {
-            this.contentPoller.setSyncContext(this.syncContext);
-        }
-
         // replace all data at once in atomic action
         this.locales = locales.filter((locale) => {
             // filter out disabled locales
@@ -339,11 +329,11 @@ export class ContentfulContentSource implements ContentSourceTypes.ContentSource
             previewToken: this.previewToken,
             managementToken: this.accessToken,
             pollType: 'date',
-            syncContext: this.syncContext,
+            syncContext: this.getSyncContextForContentPollerFromCache(),
             notificationCallback: async (syncResult) => {
                 if (syncResult.contentTypes.length) {
                     this.logger.debug('content type was changed, invalidate schema');
-                    this.cache.invalidateSchema();
+                    await this.cache.invalidateSchema();
                 } else {
                     const result = await this.convertSyncResult(syncResult);
                     await this.cache.updateContent(result);
@@ -367,7 +357,32 @@ export class ContentfulContentSource implements ContentSourceTypes.ContentSource
         this.userMap = _.keyBy(users, 'sys.id');
     }
 
-    private async convertSyncResult(syncResult: SyncResult) {
+    private async fetchUsersIfNeeded(entities: (EntryProps | AssetProps)[]) {
+        const entityHasNoCachedAuthor = entities.some((entity) => !this.userMap[entity.sys.updatedBy?.sys.id ?? '']);
+        if (entityHasNoCachedAuthor) {
+            await this.fetchUsers();
+        }
+    }
+
+    private updateContentPollerSyncContext(syncContext: ContentPollerSyncContext) {
+        if (this.contentPoller && this.contentPoller.pollType === 'date') {
+            this.contentPoller.setSyncContext({
+                ...this.getSyncContextForContentPollerFromCache(),
+                ...syncContext
+            });
+        }
+    }
+
+    private getSyncContextForContentPollerFromCache() {
+        const cacheSyncContext = this.cache.getSyncContext();
+        return {
+            lastUpdatedEntryDate: cacheSyncContext.documentsSyncContext as string | undefined,
+            lastUpdatedAssetDate: cacheSyncContext.assetsSyncContext as string | undefined,
+            lastUpdatedContentTypeDate: this.cache.getSchema().context?.lastUpdatedContentTypeDate
+        };
+    }
+
+    private async convertSyncResult(syncResult: ContentPollerSyncResult) {
         // remove deleted entries and assets from fieldData
         // generally, the "sync" method of the preview API never notifies of deleted objects, therefore we rely on
         // the deleteObject method to notify the user that restart is needed. Then, once user restarts the SSG, it
@@ -380,11 +395,7 @@ export class ContentfulContentSource implements ContentSourceTypes.ContentSource
             deletedAssets: syncResult.deletedAssets.length
         });
 
-        const resultHasNoCachedAuthor = [...syncResult.entries, ...syncResult.assets].some((data) => !this.userMap[data.sys.updatedBy?.sys.id ?? '']);
-
-        if (resultHasNoCachedAuthor) {
-            await this.fetchUsers();
-        }
+        await this.fetchUsersIfNeeded([...syncResult.entries, ...syncResult.assets]);
 
         // convert updated/created entries and assets
         const documents = this.convertEntries(syncResult.entries, this.cache.getModelByName);
@@ -413,90 +424,107 @@ export class ContentfulContentSource implements ContentSourceTypes.ContentSource
             bynderImagesAsList: this.bynderImagesAsList
         });
 
+        const prevLastUpdatedContentTypeDate = this.cache.getSchema().context?.lastUpdatedContentTypeDate;
+
         // Check if one of the content types was changed from the last time the
         // content types were fetched, in which case remove all cached content.
         const lastUpdatedContentTypeDate = getLastUpdatedEntityDate(contentTypes);
-        if (this.syncContext.lastUpdatedContentTypeDate !== lastUpdatedContentTypeDate) {
+        if (prevLastUpdatedContentTypeDate !== lastUpdatedContentTypeDate) {
             this.logger.debug(
-                `last updated content type date '${lastUpdatedContentTypeDate}' is different from cached ` +
-                    `syncContext.lastUpdatedContentTypeDate '${this.syncContext.lastUpdatedContentTypeDate}', clearing cache`
+                `last updated content type date '${lastUpdatedContentTypeDate}' is different ` +
+                    `from the cached date '${prevLastUpdatedContentTypeDate}', clearing cache`
             );
-            await this.cache.remove('entries');
-            await this.cache.remove('assets');
-            this.syncContext = { lastUpdatedContentTypeDate };
-            await this.cache.set('syncContext', this.syncContext);
+            await this.cache.clearSyncContext({
+                clearDocumentsSyncContext: true,
+                clearAssetsSyncContext: false
+            });
         }
 
+        this.updateContentPollerSyncContext({ lastUpdatedContentTypeDate });
+
         return {
             models,
             locales: this.locales.map((locale) => ({
                 code: locale.code,
                 default: locale.default
             })),
-            context: {}
+            context: {
+                lastUpdatedContentTypeDate: lastUpdatedContentTypeDate
+            }
         };
     }
 
-    async getDocuments(): Promise<ContextualDocument[]> {
+    async getDocuments(options?: { syncContext?: string }): Promise<ContextualDocument[] | { documents: ContextualDocument[]; syncContext?: string }> {
         this.logger.debug('getDocuments');
+        let lastUpdatedEntryDate = options?.syncContext;
         let entries: EntryProps[];
-        const cachedEntries = ((await this.cache.get('entries')) as EntryProps[]) ?? [];
         try {
-            if (this.syncContext.lastUpdatedEntryDate && !_.isEmpty(cachedEntries)) {
-                this.logger.debug(`got ${cachedEntries.length} entries from cache, fetching entries updated after ${this.syncContext.lastUpdatedEntryDate}`);
-                const updatedEntries = await fetchEntriesUpdatedAfter(this.plainClient, this.syncContext.lastUpdatedEntryDate, this.userLogger);
-                this.logger.debug(`got ${updatedEntries.length} updated/created entries after ${this.syncContext.lastUpdatedEntryDate}`);
-                if (updatedEntries.length) {
-                    this.syncContext.lastUpdatedEntryDate = getLastUpdatedEntityDate(updatedEntries);
+            if (lastUpdatedEntryDate) {
+                this.logger.debug(`fetching entries updated after ${lastUpdatedEntryDate}`);
+                entries = await fetchEntriesUpdatedAfter(this.plainClient, lastUpdatedEntryDate, this.userLogger);
+                this.logger.debug(`got ${entries.length} updated/created entries after ${lastUpdatedEntryDate}`);
+                if (entries.length) {
+                    lastUpdatedEntryDate = getLastUpdatedEntityDate(entries);
                 }
-                entries = _.unionBy(updatedEntries, cachedEntries, 'sys.id');
             } else {
                 entries = await fetchAllEntries(this.plainClient, this.userLogger);
-                this.syncContext.lastUpdatedEntryDate = getLastUpdatedEntityDate(entries);
+                lastUpdatedEntryDate = getLastUpdatedEntityDate(entries);
             }
         } catch (error: any) {
             // Stackbit won't be able to work properly even if one of the entries was not fetched.
             // All fetch methods use Contentful's API client that handles errors and retries automatically.
             this.logger.error(`Failed fetching documents from Contentful, error: ${error.message}`);
-            return [];
+            // By returning the original syncContext we are ensuring that next
+            // time the getDocuments is called, it will try to get the documents
+            // using the same syncContext
+            return { documents: [], syncContext: lastUpdatedEntryDate };
         }
-        await this.cache.set('entries', entries);
-        await this.cache.set('syncContext', this.syncContext);
+
+        this.updateContentPollerSyncContext({ lastUpdatedEntryDate });
+
         this.logger.debug(
-            `got ${entries.length} entries from space ${this.spaceId}, environment ${this.environment}, lastUpdatedEntryDate: ${this.syncContext.lastUpdatedEntryDate}`
+            `got ${entries.length} entries from space ${this.spaceId}, environment ${this.environment}, lastUpdatedEntryDate: ${lastUpdatedEntryDate}`
         );
-        return this.convertEntries(entries, this.cache.getModelByName);
+        await this.fetchUsersIfNeeded(entries);
+
+        const documents = this.convertEntries(entries, this.cache.getModelByName);
+
+        return { documents, syncContext: lastUpdatedEntryDate };
     }
 
-    async getAssets() {
+    async getAssets(options?: { syncContext?: string }): Promise<ContextualAsset[] | { assets: ContextualAsset[]; syncContext?: string }> {
         this.logger.debug('getAssets');
-        let assets: AssetProps[];
-        const cachedAssets = ((await this.cache.get('assets')) as AssetProps[]) ?? [];
+        let lastUpdatedAssetDate = options?.syncContext;
+        let ctflAssets: AssetProps[];
         try {
-            if (this.syncContext.lastUpdatedAssetDate && !_.isEmpty(cachedAssets)) {
-                this.logger.debug(`got ${cachedAssets.length} assets from cache, fetching assets updated after ${this.syncContext.lastUpdatedAssetDate}`);
-                const updatedAssets = await fetchAssetsUpdatedAfter(this.plainClient, this.syncContext.lastUpdatedAssetDate, this.userLogger);
-                this.logger.debug(`got ${updatedAssets.length} updated/created assets after ${this.syncContext.lastUpdatedAssetDate}`);
-                if (updatedAssets.length) {
-                    this.syncContext.lastUpdatedAssetDate = getLastUpdatedEntityDate(updatedAssets);
+            if (lastUpdatedAssetDate) {
+                this.logger.debug(`fetching assets updated after ${lastUpdatedAssetDate}`);
+                ctflAssets = await fetchAssetsUpdatedAfter(this.plainClient, lastUpdatedAssetDate, this.userLogger);
+                this.logger.debug(`got ${ctflAssets.length} updated/created assets after ${lastUpdatedAssetDate}`);
+                if (ctflAssets.length) {
+                    lastUpdatedAssetDate = getLastUpdatedEntityDate(ctflAssets);
                 }
-                assets = _.unionBy(updatedAssets, cachedAssets, 'sys.id');
             } else {
-                assets = await fetchAllAssets(this.plainClient, this.userLogger);
-                this.syncContext.lastUpdatedAssetDate = getLastUpdatedEntityDate(assets);
+                ctflAssets = await fetchAllAssets(this.plainClient, this.userLogger);
+                lastUpdatedAssetDate = getLastUpdatedEntityDate(ctflAssets);
             }
         } catch (error: any) {
             // Stackbit won't be able to work properly even if one of the entries or the assets was not fetched.
             // All fetch methods use Contentful's API client that handles errors and retries automatically.
             this.logger.error(`Failed fetching assets from Contentful, error: ${error.message}`);
-            return [];
+            return { assets: [], syncContext: lastUpdatedAssetDate };
         }
-        await this.cache.set('assets', assets);
-        await this.cache.set('syncContext', this.syncContext);
+
+        this.updateContentPollerSyncContext({ lastUpdatedAssetDate });
+
         this.logger.debug(
-            `got ${assets.length} assets from space ${this.spaceId}, environment ${this.environment}, lastUpdatedAssetDate: ${this.syncContext.lastUpdatedAssetDate}`
+            `got ${ctflAssets.length} assets from space ${this.spaceId}, environment ${this.environment}, lastUpdatedAssetDate: ${lastUpdatedAssetDate}`
         );
-        return this.convertAssets(assets);
+        await this.fetchUsersIfNeeded(ctflAssets);
+
+        const assets = this.convertAssets(ctflAssets);
+
+        return { assets, syncContext: lastUpdatedAssetDate };
     }
 
     async hasAccess({ userContext }: { userContext?: UserContext }): Promise<{ hasConnection: boolean; hasPermissions: boolean }> {

package/src/utils.ts

@@ -1,11 +1,5 @@
 import { BasicMetaSysProps } from 'contentful-management';
 
-export type SyncContext = {
-    lastUpdatedEntryDate?: string;
-    lastUpdatedAssetDate?: string;
-    lastUpdatedContentTypeDate?: string;
-};
-
 export function getLastUpdatedEntityDate(entities: { sys: BasicMetaSysProps }[]): string | undefined {
     let lastUpdatedDate: string = '';
     for (const entity of entities) {