@stackbit/cms-contentful 0.4.49-staging.1 → 0.4.49

package/src/contentful-api-client.ts

@@ -39,11 +39,17 @@ interface PlainApiClientOptions {
     accessToken: string;
     spaceId: string;
     environment: string;
+    managementHost?: string;
+    uploadHost?: string;
 }
 
-export function createPlainApiClient({ accessToken, spaceId, environment }: PlainApiClientOptions) {
+export function createPlainApiClient({ accessToken, spaceId, environment, managementHost, uploadHost }: PlainApiClientOptions) {
     return createClient(
-        { accessToken: accessToken },
+        {
+            accessToken: accessToken,
+            ...(managementHost ? { host: managementHost } : null),
+            ...(uploadHost ? { hostUpload: uploadHost } : null)
+        },
         {
             type: 'plain',
             defaults: {
@@ -178,7 +184,7 @@ export async function fetchEntriesUpdatedAfter(client: PlainClientAPI, date: str
     return fetchAllItems<EntryProps>({
         getMany: client.entry.getMany,
         logger,
-        params: {
+        query: {
             'sys.updatedAt[gt]': date
         }
     });
@@ -192,7 +198,7 @@ export async function fetchAssetsUpdatedAfter(client: PlainClientAPI, date: stri
     return fetchAllItems<AssetProps>({
         getMany: client.asset.getMany,
         logger,
-        params: {
+        query: {
             'sys.updatedAt[gt]': date
         }
     });
@@ -210,11 +216,23 @@ export async function fetchAllContentTypes(client: PlainClientAPI, logger?: Logg
     return fetchAllItems<ContentTypeProps>({ getMany: client.contentType.getMany, logger });
 }
 
-export async function fetchAllContentTypesAfter(client: PlainClientAPI, date: string, logger?: Logger) {
+export async function hasContentTypesUpdatedAfter(client: PlainClientAPI, date: string, logger?: Logger): Promise<boolean> {
+    const updatedContentTypes = await fetchAllItems<ContentTypeProps>({
+        getMany: client.contentType.getMany,
+        logger,
+        query: {
+            'sys.updatedAt[gt]': date,
+            limit: 1
+        }
+    });
+    return updatedContentTypes.length > 0;
+}
+
+export async function fetchContentTypesUpdatedAfter(client: PlainClientAPI, date: string, logger?: Logger) {
     return fetchAllItems<ContentTypeProps>({
         getMany: client.contentType.getMany,
         logger,
-        params: {
+        query: {
             'sys.updatedAt[gt]': date
         }
     });
@@ -366,11 +384,11 @@ export async function getScheduledAction(client: PlainClientAPI, scheduledAction
 export async function fetchAllItems<T>({
     getMany,
     logger,
-    params
+    query
 }: {
     getMany: (params: QueryParams) => Promise<CollectionProp<T>>;
     logger?: Logger;
-    params?: Record<string, any>;
+    query?: QueryOptions;
 }): Promise<T[]> {
     let limit = 1000;
     let items: T[] = [];
@@ -382,7 +400,7 @@ export async function fetchAllItems<T>({
                 query: {
                     skip: items.length,
                     limit: limit,
-                    ...params
+                    ...query
                 }
             });
             items = items.concat(result.items);

package/src/contentful-content-source.ts

@@ -76,10 +76,10 @@ import {
     convertDocumentVersions
 } from './contentful-entries-converter';
 import { convertAndFilterScheduledActions } from './contentful-scheduled-actions-converter';
-import { ContentPoller, SyncResult } from './content-poller';
+import { ContentPoller, ContentPollerSyncResult, ContentPollerSyncContext } from './content-poller';
 import { Readable } from 'stream';
 import { CONTENTFUL_BUILT_IN_IMAGE_SOURCES, CONTENTFUL_BYNDER_APP, CONTENTFUL_CLOUDINARY_APP } from './contentful-consts';
-import { getLastUpdatedEntityDate, SyncContext } from './utils';
+import { getLastUpdatedEntityDate } from './utils';
 
 export interface ContentSourceOptions {
     /** Contentful Space ID */
@@ -103,13 +103,50 @@ export interface ContentSourceOptions {
      * Use webhook for content updates.
      */
     useWebhookForContentUpdates?: boolean;
+    /**
+     * Use accessToken instead of userContext.accessToken for write operations.
+     */
+    useAccessTokenForUpdates?: boolean;
+    /**
+     * Switches the ContentfulContentSource to work with EU data regions.
+     * When set to true, the following updated accordingly:
+     * previewHost: 'preview.eu.contentful.com'
+     * manageHost: 'api.eu.contentful.com'
+     * uploadHost: 'upload.eu.contentful.com'
+     */
+    useEURegion?: boolean;
+    /**
+     * The host of the Contentful's preview API.
+     * By default, this value is set to 'preview.contentful.com'.
+     * If `useEURegion` is set, uses 'preview.eu.contentful.com'.
+     */
+    previewHost?: string;
+    /**
+     * The host of the Contentful's management API.
+     * By default, this value is `undefined` which makes the contentful-management
+     * use the 'api.contentful.com'.
+     * If `useEURegion` is set, this value is set to 'api.eu.contentful.com'.
+     */
+    managementHost?: string;
+    /**
+     * The host used to upload assets to Contentful.
+     * By default, this value is `undefined` which makes the contentful-management
+     * use the 'upload.contentful.com'.
+     * If `useEURegion` is set, this value is set to 'upload.eu.contentful.com'.
+     */
+    uploadHost?: string;
 }
 
 type UserContext = {
+    // The user's accessToken is provided by the "Netlify Create" Contentful
+    // OAuth Application when users connect their Netlify Create and Contentful
+    // accounts via OAuth flow.
     accessToken: string;
 };
 
-export type SchemaContext = unknown;
+export type SchemaContext = {
+    lastUpdatedContentTypeDate?: string;
+};
 
 type EntityError = {
     name: string;
@@ -129,6 +166,11 @@ export class ContentfulContentSource implements ContentSourceTypes.ContentSource
     private localDev!: boolean;
     private contentPoller: ContentPoller | null = null;
     private useWebhookForContentUpdates: boolean;
+    private useAccessTokenForUpdates: boolean;
+    private useEURegion: boolean;
+    private previewHost?: string;
+    private managementHost?: string;
+    private uploadHost?: string;
     private locales: LocaleProps[] = [];
     private defaultLocale?: LocaleProps;
     private userMap: Record<string, UserProps> = {};
@@ -138,7 +180,6 @@ export class ContentfulContentSource implements ContentSourceTypes.ContentSource
     private webhookUrl?: string;
     private devAppRestartNeeded?: () => void;
     private cache!: ContentSourceTypes.Cache<SchemaContext, DocumentContext, AssetContext>;
-    private syncContext!: SyncContext;
     private taskQueue: TaskQueue;
 
     constructor(options: ContentSourceOptions) {
@@ -148,6 +189,11 @@ export class ContentfulContentSource implements ContentSourceTypes.ContentSource
         this.previewToken = options.previewToken ?? process.env.CONTENTFUL_PREVIEW_TOKEN;
         this.useLocalizedAssetFields = options.useLocalizedAssetFields ?? false;
         this.useWebhookForContentUpdates = options.useWebhookForContentUpdates ?? false;
+        this.useAccessTokenForUpdates = options.useAccessTokenForUpdates ?? false;
+        this.useEURegion = options.useEURegion ?? false;
+        this.previewHost = options.previewHost ?? (this.useEURegion ? 'preview.eu.contentful.com' : undefined);
+        this.managementHost = options.managementHost ?? (this.useEURegion ? 'api.eu.contentful.com' : undefined);
+        this.uploadHost = options.uploadHost ?? (this.useEURegion ? 'upload.eu.contentful.com' : undefined);
         // Max active bulk actions per space is limited to 5.
         // Max of 200 items per bulk action.
         this.taskQueue = new TaskQueue({ limit: 5 });
@@ -158,7 +204,7 @@ export class ContentfulContentSource implements ContentSourceTypes.ContentSource
     }
 
     getContentSourceType(): string {
-        return 'contentful';
+        return this.useEURegion ? 'contentful-eu' : 'contentful';
     }
 
     getProjectId(): string {
@@ -197,12 +243,12 @@ export class ContentfulContentSource implements ContentSourceTypes.ContentSource
             this.userLogger.info('Using webhook for content updates');
         }
 
-        this.syncContext = ((await this.cache.get('syncContext')) as SyncContext) ?? {};
-
         this.plainClient = createPlainApiClient({
             spaceId: this.spaceId,
             accessToken: this.accessToken,
-            environment: this.environment
+            environment: this.environment,
+            managementHost: this.managementHost,
+            uploadHost: this.uploadHost
         });
 
         await this.validateConfig();
@@ -217,15 +263,6 @@ export class ContentfulContentSource implements ContentSourceTypes.ContentSource
         const appInstallations: AppInstallationProps[] = await fetchAllAppInstallations(this.plainClient);
         await this.fetchUsers();
 
-        // The contentPoller updates its syncContext property internally when
-        // it gets an updated content. But, the actual cached data is not
-        // updated. Therefore, when content source module is restarted, we need
-        // to reset the contentPoller's internal syncContext, so it will be
-        // able to fetch all the updated content from its cached state.
-        if (this.contentPoller && this.contentPoller.pollType === 'date') {
-            this.contentPoller.setSyncContext(this.syncContext);
-        }
-
         // replace all data at once in atomic action
         this.locales = locales.filter((locale) => {
             // filter out disabled locales
@@ -338,12 +375,15 @@ export class ContentfulContentSource implements ContentSourceTypes.ContentSource
             environment: this.environment,
             previewToken: this.previewToken,
             managementToken: this.accessToken,
+            previewHost: this.previewHost,
+            managementHost: this.managementHost,
+            uploadHost: this.uploadHost,
             pollType: 'date',
-            syncContext: this.syncContext,
+            syncContext: this.getSyncContextForContentPollerFromCache(),
             notificationCallback: async (syncResult) => {
                 if (syncResult.contentTypes.length) {
                     this.logger.debug('content type was changed, invalidate schema');
-                    this.cache.invalidateSchema();
+                    await this.cache.invalidateSchema();
                 } else {
                     const result = await this.convertSyncResult(syncResult);
                     await this.cache.updateContent(result);
@@ -367,7 +407,32 @@ export class ContentfulContentSource implements ContentSourceTypes.ContentSource
         this.userMap = _.keyBy(users, 'sys.id');
     }
 
-    private async convertSyncResult(syncResult: SyncResult) {
+    private async fetchUsersIfNeeded(entities: (EntryProps | AssetProps)[]) {
+        const entityHasNoCachedAuthor = entities.some((entity) => !this.userMap[entity.sys.updatedBy?.sys.id ?? '']);
+        if (entityHasNoCachedAuthor) {
+            await this.fetchUsers();
+        }
+    }
+
+    private updateContentPollerSyncContext(syncContext: ContentPollerSyncContext) {
+        if (this.contentPoller && this.contentPoller.pollType === 'date') {
+            this.contentPoller.setSyncContext({
+                ...this.getSyncContextForContentPollerFromCache(),
+                ...syncContext
+            });
+        }
+    }
+
+    private getSyncContextForContentPollerFromCache() {
+        const cacheSyncContext = this.cache.getSyncContext();
+        return {
+            lastUpdatedEntryDate: cacheSyncContext.documentsSyncContext as string | undefined,
+            lastUpdatedAssetDate: cacheSyncContext.assetsSyncContext as string | undefined,
+            lastUpdatedContentTypeDate: this.cache.getSchema().context?.lastUpdatedContentTypeDate
+        };
+    }
+
+    private async convertSyncResult(syncResult: ContentPollerSyncResult) {
         // remove deleted entries and assets from fieldData
         // generally, the "sync" method of the preview API never notifies of deleted objects, therefore we rely on
         // the deleteObject method to notify the user that restart is needed. Then, once user restarts the SSG, it
@@ -380,11 +445,7 @@ export class ContentfulContentSource implements ContentSourceTypes.ContentSource
             deletedAssets: syncResult.deletedAssets.length
         });
 
-        const resultHasNoCachedAuthor = [...syncResult.entries, ...syncResult.assets].some((data) => !this.userMap[data.sys.updatedBy?.sys.id ?? '']);
-
-        if (resultHasNoCachedAuthor) {
-            await this.fetchUsers();
-        }
+        await this.fetchUsersIfNeeded([...syncResult.entries, ...syncResult.assets]);
 
         // convert updated/created entries and assets
         const documents = this.convertEntries(syncResult.entries, this.cache.getModelByName);
@@ -413,94 +474,111 @@ export class ContentfulContentSource implements ContentSourceTypes.ContentSource
             bynderImagesAsList: this.bynderImagesAsList
         });
 
+        const prevLastUpdatedContentTypeDate = this.cache.getSchema().context?.lastUpdatedContentTypeDate;
+
         // Check if one of the content types was changed from the last time the
         // content types were fetched, in which case remove all cached content.
         const lastUpdatedContentTypeDate = getLastUpdatedEntityDate(contentTypes);
-        if (this.syncContext.lastUpdatedContentTypeDate !== lastUpdatedContentTypeDate) {
+        if (prevLastUpdatedContentTypeDate !== lastUpdatedContentTypeDate) {
             this.logger.debug(
-                `last updated content type date '${lastUpdatedContentTypeDate}' is different from cached ` +
-                    `syncContext.lastUpdatedContentTypeDate '${this.syncContext.lastUpdatedContentTypeDate}', clearing cache`
+                `last updated content type date '${lastUpdatedContentTypeDate}' is different ` +
+                    `from the cached date '${prevLastUpdatedContentTypeDate}', clearing cache`
             );
-            await this.cache.remove('entries');
-            await this.cache.remove('assets');
-            this.syncContext = { lastUpdatedContentTypeDate };
-            await this.cache.set('syncContext', this.syncContext);
+            await this.cache.clearSyncContext({
+                clearDocumentsSyncContext: true,
+                clearAssetsSyncContext: false
+            });
         }
 
+        this.updateContentPollerSyncContext({ lastUpdatedContentTypeDate });
+
         return {
             models,
             locales: this.locales.map((locale) => ({
                 code: locale.code,
                 default: locale.default
             })),
-            context: {}
+            context: {
+                lastUpdatedContentTypeDate: lastUpdatedContentTypeDate
+            }
         };
     }
 
-    async getDocuments(): Promise<ContextualDocument[]> {
+    async getDocuments(options?: { syncContext?: string }): Promise<ContextualDocument[] | { documents: ContextualDocument[]; syncContext?: string }> {
         this.logger.debug('getDocuments');
+        let lastUpdatedEntryDate = options?.syncContext;
         let entries: EntryProps[];
-        const cachedEntries = ((await this.cache.get('entries')) as EntryProps[]) ?? [];
         try {
-            if (this.syncContext.lastUpdatedEntryDate && !_.isEmpty(cachedEntries)) {
-                this.logger.debug(`got ${cachedEntries.length} entries from cache, fetching entries updated after ${this.syncContext.lastUpdatedEntryDate}`);
-                const updatedEntries = await fetchEntriesUpdatedAfter(this.plainClient, this.syncContext.lastUpdatedEntryDate, this.userLogger);
-                this.logger.debug(`got ${updatedEntries.length} updated/created entries after ${this.syncContext.lastUpdatedEntryDate}`);
-                if (updatedEntries.length) {
-                    this.syncContext.lastUpdatedEntryDate = getLastUpdatedEntityDate(updatedEntries);
+            if (lastUpdatedEntryDate) {
+                this.logger.debug(`fetching entries updated after ${lastUpdatedEntryDate}`);
+                entries = await fetchEntriesUpdatedAfter(this.plainClient, lastUpdatedEntryDate, this.userLogger);
+                this.logger.debug(`got ${entries.length} updated/created entries after ${lastUpdatedEntryDate}`);
+                if (entries.length) {
+                    lastUpdatedEntryDate = getLastUpdatedEntityDate(entries);
                 }
-                entries = _.unionBy(updatedEntries, cachedEntries, 'sys.id');
             } else {
                 entries = await fetchAllEntries(this.plainClient, this.userLogger);
-                this.syncContext.lastUpdatedEntryDate = getLastUpdatedEntityDate(entries);
+                lastUpdatedEntryDate = getLastUpdatedEntityDate(entries);
             }
         } catch (error: any) {
             // Stackbit won't be able to work properly even if one of the entries was not fetched.
             // All fetch methods use Contentful's API client that handles errors and retries automatically.
             this.logger.error(`Failed fetching documents from Contentful, error: ${error.message}`);
-            return [];
+            // By returning the original syncContext we are ensuring that next
+            // time the getDocuments is called, it will try to get the documents
+            // using the same syncContext
+            return { documents: [], syncContext: lastUpdatedEntryDate };
         }
-        await this.cache.set('entries', entries);
-        await this.cache.set('syncContext', this.syncContext);
+
+        this.updateContentPollerSyncContext({ lastUpdatedEntryDate });
+
         this.logger.debug(
-            `got ${entries.length} entries from space ${this.spaceId}, environment ${this.environment}, lastUpdatedEntryDate: ${this.syncContext.lastUpdatedEntryDate}`
+            `got ${entries.length} entries from space ${this.spaceId}, environment ${this.environment}, lastUpdatedEntryDate: ${lastUpdatedEntryDate}`
         );
-        return this.convertEntries(entries, this.cache.getModelByName);
+        await this.fetchUsersIfNeeded(entries);
+
+        const documents = this.convertEntries(entries, this.cache.getModelByName);
+
+        return { documents, syncContext: lastUpdatedEntryDate };
     }
 
-    async getAssets() {
+    async getAssets(options?: { syncContext?: string }): Promise<ContextualAsset[] | { assets: ContextualAsset[]; syncContext?: string }> {
         this.logger.debug('getAssets');
-        let assets: AssetProps[];
-        const cachedAssets = ((await this.cache.get('assets')) as AssetProps[]) ?? [];
+        let lastUpdatedAssetDate = options?.syncContext;
+        let ctflAssets: AssetProps[];
         try {
-            if (this.syncContext.lastUpdatedAssetDate && !_.isEmpty(cachedAssets)) {
-                this.logger.debug(`got ${cachedAssets.length} assets from cache, fetching assets updated after ${this.syncContext.lastUpdatedAssetDate}`);
-                const updatedAssets = await fetchAssetsUpdatedAfter(this.plainClient, this.syncContext.lastUpdatedAssetDate, this.userLogger);
-                this.logger.debug(`got ${updatedAssets.length} updated/created assets after ${this.syncContext.lastUpdatedAssetDate}`);
-                if (updatedAssets.length) {
-                    this.syncContext.lastUpdatedAssetDate = getLastUpdatedEntityDate(updatedAssets);
+            if (lastUpdatedAssetDate) {
+                this.logger.debug(`fetching assets updated after ${lastUpdatedAssetDate}`);
+                ctflAssets = await fetchAssetsUpdatedAfter(this.plainClient, lastUpdatedAssetDate, this.userLogger);
+                this.logger.debug(`got ${ctflAssets.length} updated/created assets after ${lastUpdatedAssetDate}`);
+                if (ctflAssets.length) {
+                    lastUpdatedAssetDate = getLastUpdatedEntityDate(ctflAssets);
                 }
-                assets = _.unionBy(updatedAssets, cachedAssets, 'sys.id');
             } else {
-                assets = await fetchAllAssets(this.plainClient, this.userLogger);
-                this.syncContext.lastUpdatedAssetDate = getLastUpdatedEntityDate(assets);
+                ctflAssets = await fetchAllAssets(this.plainClient, this.userLogger);
+                lastUpdatedAssetDate = getLastUpdatedEntityDate(ctflAssets);
             }
         } catch (error: any) {
             // Stackbit won't be able to work properly even if one of the entries or the assets was not fetched.
             // All fetch methods use Contentful's API client that handles errors and retries automatically.
             this.logger.error(`Failed fetching assets from Contentful, error: ${error.message}`);
-            return [];
+            return { assets: [], syncContext: lastUpdatedAssetDate };
         }
-        await this.cache.set('assets', assets);
-        await this.cache.set('syncContext', this.syncContext);
+
+        this.updateContentPollerSyncContext({ lastUpdatedAssetDate });
+
         this.logger.debug(
-            `got ${assets.length} assets from space ${this.spaceId}, environment ${this.environment}, lastUpdatedAssetDate: ${this.syncContext.lastUpdatedAssetDate}`
+            `got ${ctflAssets.length} assets from space ${this.spaceId}, environment ${this.environment}, lastUpdatedAssetDate: ${lastUpdatedAssetDate}`
         );
-        return this.convertAssets(assets);
+        await this.fetchUsersIfNeeded(ctflAssets);
+
+        const assets = this.convertAssets(ctflAssets);
+
+        return { assets, syncContext: lastUpdatedAssetDate };
     }
 
     async hasAccess({ userContext }: { userContext?: UserContext }): Promise<{ hasConnection: boolean; hasPermissions: boolean }> {
-        if (!this.localDev && !userContext?.accessToken) {
+        if (!this.localDev && !this.useAccessTokenForUpdates && !userContext?.accessToken) {
             return {
                 hasConnection: false,
                 hasPermissions: false
@@ -1011,7 +1089,7 @@ export class ContentfulContentSource implements ContentSourceTypes.ContentSource
     }
 
     private getPlainApiClientForUser({ userContext }: { userContext?: UserContext }): PlainClientAPI {
-        if (this.localDev) {
+        if (this.localDev || this.useAccessTokenForUpdates) {
             return this.plainClient;
         }
         const userAccessToken = userContext?.accessToken;
@@ -1021,7 +1099,9 @@ export class ContentfulContentSource implements ContentSourceTypes.ContentSource
         return createPlainApiClient({
             spaceId: this.spaceId,
             accessToken: userAccessToken,
-            environment: this.environment
+            environment: this.environment,
+            managementHost: this.managementHost,
+            uploadHost: this.uploadHost
         });
     }
 

package/src/utils.ts

@@ -1,11 +1,5 @@
 import { BasicMetaSysProps } from 'contentful-management';
 
-export type SyncContext = {
-    lastUpdatedEntryDate?: string;
-    lastUpdatedAssetDate?: string;
-    lastUpdatedContentTypeDate?: string;
-};
-
 export function getLastUpdatedEntityDate(entities: { sys: BasicMetaSysProps }[]): string | undefined {
     let lastUpdatedDate: string = '';
     for (const entity of entities) {