@agility/content-sync 1.0.3 → 1.1.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agility/content-sync",
-  "version": "1.0.3",
+  "version": "1.1.1",
   "description": "JavaScript SDK for synchronizing content from Agility CMS",
   "main": "dist/agility-sync-sdk.node.js",
   "scripts": {

package/src/methods/syncContent.js

@@ -12,11 +12,12 @@ export default async function (languageCode, token) {
 	let itemCount = 0
 	let busy = false
 	let waitMS = 0
-	const waitMaxMS = 30000
-	const waitIntervalMS = 500
+	const waitMaxMS = 60000
+	const waitIntervalMS = 1000
 
 	do {
 
+
 		//sync content items...
 		const syncRet = await this.agilityClient.getSyncContent({
 			syncToken: token,
@@ -25,8 +26,8 @@ export default async function (languageCode, token) {
 
 		});
 
-		if (syncRet.busy !== undefined
-			&& syncRet.busy === true) {
+
+		if (syncRet === undefined || syncRet === null || (syncRet.busy !== undefined && syncRet.busy === true)) {
 			//if the api is being updated, wait a few ms and try again...
 			waitMS += waitIntervalMS
 			if (waitMS > waitMaxMS) {
@@ -53,15 +54,19 @@ export default async function (languageCode, token) {
 		const syncItems = syncRet.items;
 
 		//if we don't get anything back, kick out
-		if (syncItems.length === 0) {
-			break;
+		if (syncItems.length > 0 ) {
+
+			for (let index = 0; index < syncItems.length; index++) {
+				await storeInterface.saveContentItem({ contentItem: syncItems[index], languageCode });
+			}
 		}
 
-		for (let index = 0; index < syncItems.length; index++) {
-			await storeInterface.saveContentItem({ contentItem: syncItems[index], languageCode });
+		if (syncRet.syncToken > token) {
+			token = syncRet.syncToken;
+		} else {
+			break;
 		}
 
-		token = syncRet.syncToken;
 		itemCount += syncItems.length;
 
 	} while (token > 0 || busy === true)

package/src/methods/syncPages.js

@@ -1,15 +1,15 @@
 import { logInfo, logWarning, sleep } from '../util'
 
 export default async function (languageCode, token) {
-    const storeInterface = this.store;
+	const storeInterface = this.store;
 	if (!token) token = 0;
 
 	let itemCount = 0;
 
 	let busy = false
 	let waitMS = 0
-	const waitMaxMS = 30000
-	const waitIntervalMS = 500
+	const waitMaxMS = 60000
+	const waitIntervalMS = 1000
 
 
 	do {
@@ -20,8 +20,7 @@ export default async function (languageCode, token) {
 			languageCode: languageCode
 		});
 
-		if (syncRet.busy !== undefined
-			&& syncRet.busy === true) {
+		if (syncRet === undefined || syncRet === null || (syncRet.busy !== undefined && syncRet.busy === true)) {
 			//if the api is being updated, wait a few ms and try again...
 			waitMS += waitIntervalMS
 			if (waitMS > waitMaxMS) {
@@ -29,7 +28,7 @@ export default async function (languageCode, token) {
 				break
 			}
 
-			if (! busy) {
+			if (!busy) {
 				busy = true
 				logInfo("Sync API is busy.  Waiting...")
 			}
@@ -46,16 +45,21 @@ export default async function (languageCode, token) {
 
 		const syncItems = syncRet.items;
 
-		//if we don't get anything back, kick out
-		if (syncItems.length === 0) {
-			break;
+		//if we have something...
+		if (syncItems.length > 0) {
+
+
+			for (let index = 0; index < syncItems.length; index++) {
+				await storeInterface.savePageItem({ pageItem: syncItems[index], languageCode });
+			}
 		}
 
-		for (let index = 0; index < syncItems.length; index++) {
-			await storeInterface.savePageItem({ pageItem: syncItems[index], languageCode });
+		if (syncRet.syncToken > token) {
+			token = syncRet.syncToken;
+		} else {
+			break;
 		}
 
-		token = syncRet.syncToken;
 		itemCount += syncItems.length;
 
 

package/src/store-interface-console.js

@@ -1,23 +1,64 @@
+/**
+ * The function to handle saving/updating an item to your storage. This could be a Content Item, Page, Url Redirections, Sync State (state), or Sitemap.
+ * @param {Object} params - The parameters object
+ * @param {Object} params.options - A flexible object that can contain any properties specifically related to this interface
+ * @param {Object} params.item - The object representing the Content Item, Page, Url Redirections, Sync State (state), or Sitemap that needs to be saved/updated
+ * @param {String} params.itemType - The type of item being saved/updated, expected values are `item`, `page`, `sitemap`, `nestedsitemap`, `state`, `urlredirections`
+ * @param {String} params.languageCode - The locale code associated to the item being saved/updated
+ * @param {(String|Number)} params.itemID - The ID of the item being saved/updated - this could be a string or number depending on the itemType
+ * @returns {Void}
+ */
 const saveItem = async ({ options, item, itemType, languageCode, itemID }) => {
     console.log(`Console Interface: saveItem has been called`);
     return null;
 }
-
+/**
+ * The function to handle deleting an item to your storage. This could be a Content Item, Page, Url Redirections, Sync State (state), or Sitemap.
+ * @param {Object} params - The parameters object
+ * @param {Object} params.options - A flexible object that can contain any properties specifically related to this interface
+ * @param {String} params.itemType - The type of item being deleted, expected values are `item`, `page`, `sitemap`, `nestedsitemap`, `state`, `urlredirections`
+ * @param {String} params.languageCode - The locale code associated to the item being saved/updated
+ * @param {(String|Number)} params.itemID - The ID of the item being deleted - this could be a string or number depending on the itemType
+ * @returns {Void}
+ */
 const deleteItem = async ({ options, itemType, languageCode, itemID }) => {
     console.log(`Console Interface: deleteItem has been called`);
     return null;
 }
-
+/**
+ * The function to handle updating and placing a Content Item into a "list" so that you can handle querying a collection of items.
+ * @param {Object} params - The parameters object
+ * @param {Object} params.options - A flexible object that can contain any properties specifically related to this interface
+ * @param {Object} params.item - The object representing the Content Item
+ * @param {String} params.languageCode - The locale code associated to the item being saved/updated 
+ * @param {(String|Number)} params.itemID - The ID of the item being updated - this could be a string or number depending on the itemType
+ * @param {String} params.referenceName - The reference name of the Content List that this Content Item should be added to
+ * @param {String} params.definitionName - The Model name that the Content Item is based on
+ * @returns {Void}
+ */
 const mergeItemToList = async ({ options, item, languageCode, itemID, referenceName, definitionName }) => {
 	console.log(`Console Interface: mergeItemToList has been called`);
     return null;
 }
-
+/**
+ * The function to handle retrieving a Content Item, Page, Url Redirections, Sync State (state), or Sitemap
+ * @param {Object} params - The parameters object
+ * @param {Object} params.options - A flexible object that can contain any properties specifically related to this interface
+ * @param {String} params.itemType - The type of item being accessed, expected values are `item`, `list`, `page`, `sitemap`, `nestedsitemap`, `state`, `urlredirections`
+ * @param {String} params.languageCode - The locale code associated to the item being accessed
+ * @param {(String|Number)} params.itemID - The ID of the item being accessed - this could be a string or number depending on the itemType
+ * @returns {Object}
+ */
 const getItem = async ({ options, itemType, languageCode, itemID }) => {
     console.log(`Console Interface: getItem has been called`)
     return null;
 }
-
+/**
+ * The function to handle clearing the cache of synchronized data from the CMS
+ * @param {Object} params - The parameters object
+ * @param {Object} params.options - A flexible object that can contain any properties specifically related to this interface
+ * @returns {Void}
+ */
 const clearItems = async ({ options }) => {
     console.log(`Console Interface: clearItem has been called`)
     return null;

package/src/store-interface-filesystem.js

@@ -1,21 +1,25 @@
 const fs = require('fs')
 const os = require('os')
 const path = require('path')
-const { sleep } = require("./util")
-const { lockSync, unlockSync, checkSync, check } = require("proper-lockfile")
+const {sleep} = require("./util")
+const { lockSync, unlockSync, checkSync, check }  = require("proper-lockfile")
 
 
 require("dotenv").config({
 	path: `.env.${process.env.NODE_ENV}`,
 })
 
-
-const getFilePath = ({ options, itemType, languageCode, itemID }) => {
-	const fileName = `${itemID}.json`;
-
-	return path.join(options.rootPath, languageCode, itemType, fileName);
-}
-
+/**
+ * The function to handle saving/updating an item to your storage. This could be a Content Item, Page, Url Redirections, Sync State (state), or Sitemap.
+ * @param {Object} params - The parameters object
+ * @param {Object} params.options - A flexible object that can contain any properties specifically related to this interface
+ * @param {String} params.options.rootPath - The path to store/access the content as JSON 
+ * @param {Object} params.item - The object representing the Content Item, Page, Url Redirections, Sync State (state), or Sitemap that needs to be saved/updated
+ * @param {String} params.itemType - The type of item being saved/updated, expected values are `item`, `page`, `sitemap`, `nestedsitemap`, `state`, `urlredirections`
+ * @param {String} params.languageCode - The locale code associated to the item being saved/updated
+ * @param {(String|Number)} params.itemID - The ID of the item being saved/updated - this could be a string or number depending on the itemType
+ * @returns {Void}
+ */
 const saveItem = async ({ options, item, itemType, languageCode, itemID }) => {
 
 	let filePath = getFilePath({ options, itemType, languageCode, itemID });
@@ -30,7 +34,16 @@ const saveItem = async ({ options, item, itemType, languageCode, itemID }) => {
 	let json = JSON.stringify(item);
 	fs.writeFileSync(filePath, json);
 }
-
+/**
+ * The function to handle deleting an item to your storage. This could be a Content Item, Page, Url Redirections, Sync State (state), or Sitemap.
+ * @param {Object} params - The parameters object
+ * @param {Object} params.options - A flexible object that can contain any properties specifically related to this interface
+ * @param {String} params.options.rootPath - The path to store/access the content as JSON 
+ * @param {String} params.itemType - The type of item being deleted, expected values are `item`, `page`, `sitemap`, `nestedsitemap`, `state`, `urlredirections`
+ * @param {String} params.languageCode - The locale code associated to the item being saved/updated
+ * @param {(String|Number)} params.itemID - The ID of the item being deleted - this could be a string or number depending on the itemType
+ * @returns {Void}
+ */
 const deleteItem = async ({ options, itemType, languageCode, itemID }) => {
 
 	let filePath = getFilePath({ options, itemType, languageCode, itemID });
@@ -40,44 +53,62 @@ const deleteItem = async ({ options, itemType, languageCode, itemID }) => {
 	}
 
 }
-
+/**
+ * The function to handle updating and placing a Content Item into a "list" so that you can handle querying a collection of items.
+ * @param {Object} params - The parameters object
+ * @param {Object} params.options - A flexible object that can contain any properties specifically related to this interface
+ * @param {String} params.options.rootPath - The path to store/access the content as JSON 
+ * @param {Object} params.item - The object representing the Content Item
+ * @param {String} params.languageCode - The locale code associated to the item being saved/updated 
+ * @param {(String|Number)} params.itemID - The ID of the item being updated - this could be a string or number depending on the itemType
+ * @param {String} params.referenceName - The reference name of the Content List that this Content Item should be added to
+ * @param {String} params.definitionName - The Model name that the Content Item is based on
+ * @returns {Void}
+ */
 const mergeItemToList = async ({ options, item, languageCode, itemID, referenceName, definitionName }) => {
 
 	let contentList = await getItem({ options, itemType: "list", languageCode, itemID: referenceName });
 
 	if (contentList == null) {
 		//initialize the list
-		contentList = [];
-	}
-
-
-	//replace the item...
-	const cIndex = contentList.findIndex((ci) => {
-		return ci.contentID === itemID;
-	});
-
-	if (item.properties.state === 3) {
-		//*** deleted item (remove from the list) ***
-		if (cIndex >= 0) {
-			//remove the item
-			contentList.splice(cIndex, 1);
-		}
-
+		contentList = [item];
 	} else {
-		//*** regular item (merge) ***
-		if (cIndex >= 0) {
-			//replace the existing item
-			contentList[cIndex] = item;
+		//replace the item...
+		const cIndex = contentList.findIndex((ci) => {
+			return ci.contentID === itemID;
+		});
+
+		if (item.properties.state === 3) {
+			//*** deleted item (remove from the list) ***
+			if (cIndex >= 0) {
+				//remove the item
+				contentList.splice(cIndex, 1);
+			}
+
 		} else {
-			//and it to the end of the
-			contentList.push(item);
+			//*** regular item (merge) ***
+			if (cIndex >= 0) {
+				//replace the existing item
+				contentList[cIndex] = item;
+			} else {
+				//and it to the end of the
+				contentList.push(item);
+			}
 		}
-
 	}
 
 	await saveItem({ options, item: contentList, itemType: "list", languageCode, itemID: referenceName });
 }
-
+/**
+ * The function to handle retrieving a Content Item, Page, Url Redirections, Sync State (state), or Sitemap
+ * @param {Object} params - The parameters object
+ * @param {Object} params.options - A flexible object that can contain any properties specifically related to this interface
+ * @param {String} params.options.rootPath - The path to store/access the content as JSON 
+ * @param {String} params.itemType - The type of item being accessed, expected values are `item`, `list`, `page`, `sitemap`, `nestedsitemap`, `state`, `urlredirections`
+ * @param {String} params.languageCode - The locale code associated to the item being accessed
+ * @param {(String|Number)} params.itemID - The ID of the item being accessed - this could be a string or number depending on the itemType
+ * @returns {Object}
+ */
 const getItem = async ({ options, itemType, languageCode, itemID }) => {
 	let filePath = getFilePath({ options, itemType, languageCode, itemID });
 
@@ -87,24 +118,29 @@ const getItem = async ({ options, itemType, languageCode, itemID }) => {
 	return JSON.parse(json);
 }
 
+/**
+ * The function to handle clearing the cache of synchronized data from the CMS
+ * @param {Object} params - The parameters object
+ * @param {Object} params.options - A flexible object that can contain any properties specifically related to this interface
+ * @param {String} params.options.rootPath - The path to store/access the content as JSON 
+ * @returns {Void}
+ */
 const clearItems = async ({ options }) => {
 	fs.rmdirSync(options.rootPath, { recursive: true })
 }
 
-const waitOnLock = async (lockFile) => {
 
-	while (await check(lockFile)) {
-		await sleep(100)
-	}
-
-}
 
+/**
+ * The function to handle multi-threaded Syncs that may be happening at the same time. If you need to prevent a sync from happening and let it wait until another sync has finished use this.
+ * @returns {Promise}
+ */
 const mutexLock = async () => {
 
 
 	const dir = os.tmpdir();
 	const lockFile = `${dir}/${"agility-sync"}.mutex`
-	if (!fs.existsSync(lockFile)) {
+	if (! fs.existsSync(lockFile)) {
 		fs.writeFileSync(lockFile, "agility-sync");
 	}
 
@@ -139,7 +175,18 @@ const mutexLock = async () => {
 }
 
 
+//private function to get a wait on a lock file
+const waitOnLock = async (lockFile) => {
+	while (await check(lockFile)) {
+		await sleep(100)
+	}
+}
 
+//private function to get path of an item
+const getFilePath = ({ options, itemType, languageCode, itemID }) => {
+	const fileName = `${itemID}.json`;
+	return path.join(options.rootPath, languageCode, itemType, fileName);
+}
 
 
 module.exports = {

package/src/store-interface.js

@@ -82,20 +82,19 @@ const saveContentItem = async ({ contentItem, languageCode }) => {
 			languageCode,
 			itemID: contentItem.contentID,
 		});
-
 		if (currentItem) {
+
 			//if the item is deleted, we need to grab the def and ref name from the current
 			definitionName = sanitizeName(currentItem.properties.definitionName)
 			referenceName = currentItem.properties.referenceName
-		}
-
-		await store.deleteItem({
-			options,
-			itemType: "item",
-			languageCode,
-			itemID: contentItem.contentID,
-		});
 
+			await store.deleteItem({
+				options,
+				itemType: "item",
+				languageCode,
+				itemID: contentItem.contentID,
+			});
+		}
 	} else {
 		//regular item
 		if (!contentItem.properties.definitionName
@@ -221,7 +220,14 @@ const getSyncState = async (languageCode) => {
  * @param {boolean} [requestParams.expandAllContentLinks] - Whether or not to expand entire linked content references, includings lists and items that are rendered in the CMS as Grid or Link. Default is **false**
  * @returns {Promise<Object>} - Returns a content item object.
 */
-const getContentItem = async ({ contentID, languageCode, depth = 2, expandAllContentLinks = false }) => {
+const getContentItem = async ({ contentID, languageCode, depth, contentLinkDepth, expandAllContentLinks = false }) => {
+
+	if (depth === undefined && contentLinkDepth !== undefined) {
+		depth = contentLinkDepth
+	} else if (depth === undefined && contentLinkDepth === undefined) {
+		depth = 2
+	}
+
 	const contentItem = await store.getItem({
 		options,
 		itemType: "item",
@@ -251,33 +257,92 @@ const expandContentItem = async ({ contentItem, languageCode, depth, expandAllCo
 					depth: depth - 1,
 				});
 				if (childItem != null) fields[fieldName] = childItem;
-			} else if (fieldValue.sortids && fieldValue.sortids.split) {
-				//multi linked item
-				const sortIDAry = fieldValue.sortids.split(",");
+			} else if (fieldValue.fulllist === true
+							&& fieldValue.referencename
+							&& expandAllContentLinks === true) {
+
+				//LINK TO THE FULL LIST
+				const referenceName = fieldValue.referencename
+
+				const skip = 0
+				const take = 50
+
+				const list = await getContentList({
+					referenceName,
+					languageCode,
+					depth: depth - 1,
+					expandAllContentLinks,
+					skip,
+					take
+				})
+
+				let sortIDAry = []
+
+				if (fieldValue.sortids && fieldValue.sortids.split) {
+					sortIDAry = fieldValue.sortids.split(",");
+				}
+
+				let itemCount = 0
 				const childItems = [];
 				for (const childItemID of sortIDAry) {
+					itemCount++
 					const childItem = await getContentItem({
 						contentID: childItemID,
 						languageCode,
 						depth: depth - 1,
+						expandAllContentLinks,
+						skip,
+						take
+					});
+
+					if (childItem != null) {
+						childItems.push(childItem);
+					}
+				}
+
+				for (const listItem of list) {
+					itemCount++
+					if (itemCount > 50) break;
+
+					const listItemContentID = listItem.contentID
+					if (sortIDAry.includes(`${listItemContentID}`)) {
+						continue;
+					}
+
+					const childItem = await getContentItem({
+						contentID: listItemContentID,
+						languageCode,
+						depth: depth - 1,
 						expandAllContentLinks
 					});
 					if (childItem != null) childItems.push(childItem);
 				}
+
 				fields[fieldName] = childItems;
-			} else if (fieldValue.referencename
-				&& expandAllContentLinks === true) {
-					//if we are expanding ALL content links...
 
-					const childList = await getContentList({
-						referenceName: fieldValue.referencename,
+			} else if (fieldValue.sortids && fieldValue.sortids.split && fieldValue.fulllist !== true) {
+				//MULTI LINKED ITEM
+				let sortIDAry = []
+				if (fieldValue.sortids && fieldValue.sortids.split) {
+					sortIDAry = fieldValue.sortids.split(",");
+				}
+
+				const skip = expandAllContentLinks ? 0 : undefined
+				const take = expandAllContentLinks ? 50 : undefined
+
+				const childItems = [];
+				for (const childItemID of sortIDAry) {
+					const childItem = await getContentItem({
+						contentID: childItemID,
 						languageCode,
 						depth: depth - 1,
 						expandAllContentLinks,
-						take: 50
-					})
-
-					fields[fieldName] = childList
+						skip,
+						take
+					});
+					if (childItem != null) childItems.push(childItem);
+				}
+				fields[fieldName] = childItems;
 
 			}
 		}
@@ -358,7 +423,14 @@ const getContentList = async ({ referenceName, languageCode, depth = 0, expandAl
  * @param {*} { pageID, languageCode, depth = 3 }
  * @returns
  */
-const getPage = async ({ pageID, languageCode, depth = 3 }) => {
+const getPage = async ({ pageID, languageCode, depth, contentLinkDepth, expandAllContentLinks = false }) => {
+
+	if (depth === undefined && contentLinkDepth !== undefined) {
+		depth = contentLinkDepth
+	} else if (depth === undefined && contentLinkDepth === undefined) {
+		depth = 2
+	}
+
 	let pageItem = await store.getItem({
 		options,
 		itemType: "page",
@@ -377,6 +449,7 @@ const getPage = async ({ pageID, languageCode, depth = 3 }) => {
 					contentID: mod.item.contentid,
 					languageCode,
 					depth: depth - 1,
+					expandAllContentLinks
 				});
 				mod.item = moduleItem;
 			}

package/src/sync-client.js

@@ -9,7 +9,7 @@ import storeInterfaceFileSystem from './store-interface-filesystem'
 
 function getSyncClient (userConfig) {
     validateConfigParams(userConfig);
-    return createSyncCient(userConfig);
+    return createSyncClient(userConfig);
 }
 
 function validateConfigParams(configParams) {
@@ -39,11 +39,11 @@ const defaultConfig = {
     }
 };
 
-function createSyncCient(userConfig) {
+function createSyncClient(userConfig) {
     let config = {
         ...defaultConfig,
         ...userConfig
-        
+
     }
 
     const agilityClient = agility.getApi({
@@ -60,7 +60,7 @@ function createSyncCient(userConfig) {
 
     //set the sync storage interface provider, it will also validate it
     storeInterface.setStore(store, config.store.options);
-    
+
     return {
         config,
         agilityClient,