@kwiz/common 1.0.4 → 1.0.6

package/dist/utils/sharepoint.rest/listutils/GetListItemsByCaml.js

@@ -1,13 +1,4 @@
 "use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.GetListItemsByCaml = void 0;
 const _dependencies_1 = require("../../_dependencies");
@@ -27,138 +18,136 @@ const lookupOrUserFieldLimit = 11;
  *  - view.ViewQuery
  *  - wrapped view.ViewQuery in a <View>: "&lt;View Scope='RecursiveAll'>&lt;Query>&lt;/Query>&lt;RowLimit>1000&lt;/RowLimit>&lt;/View>" or send rowLimit in options.
  */
-function GetListItemsByCaml(siteUrl, listIdOrTitle, camlQuery, options) {
-    return __awaiter(this, void 0, void 0, function* () {
-        siteUrl = (0, common_1.GetSiteUrl)(siteUrl);
-        if (!camlQuery.toLowerCase().startsWith("<view")) {
-            camlQuery = `<View Scope='RecursiveAll'><Query>${camlQuery}</Query></View>`;
-        }
-        let xmlParser = new DOMParser();
-        let xmlDoc = xmlParser.parseFromString(camlQuery, "text/xml");
-        let viewNode = xmlDoc.querySelector("View, view");
-        let queryNode = viewNode && viewNode.querySelector("Query, query");
-        let rowLimitNode = viewNode && viewNode.querySelector("RowLimit, rowlimit");
-        let viewFieldsNode = viewNode && viewNode.querySelector("ViewFields, viewfields");
-        let parseRowLimitNodeResult = _parseRowLimitNode(rowLimitNode, viewNode, options.rowLimit);
-        if (!(0, _dependencies_1.isNullOrUndefined)(parseRowLimitNodeResult)) {
-            options.rowLimit = parseRowLimitNodeResult.rowLimit;
-            camlQuery = parseRowLimitNodeResult.camlQuery;
+async function GetListItemsByCaml(siteUrl, listIdOrTitle, camlQuery, options) {
+    siteUrl = (0, common_1.GetSiteUrl)(siteUrl);
+    if (!camlQuery.toLowerCase().startsWith("<view")) {
+        camlQuery = `<View Scope='RecursiveAll'><Query>${camlQuery}</Query></View>`;
+    }
+    let xmlParser = new DOMParser();
+    let xmlDoc = xmlParser.parseFromString(camlQuery, "text/xml");
+    let viewNode = xmlDoc.querySelector("View, view");
+    let queryNode = viewNode && viewNode.querySelector("Query, query");
+    let rowLimitNode = viewNode && viewNode.querySelector("RowLimit, rowlimit");
+    let viewFieldsNode = viewNode && viewNode.querySelector("ViewFields, viewfields");
+    let parseRowLimitNodeResult = _parseRowLimitNode(rowLimitNode, viewNode, options.rowLimit);
+    if (!(0, _dependencies_1.isNullOrUndefined)(parseRowLimitNodeResult)) {
+        options.rowLimit = parseRowLimitNodeResult.rowLimit;
+        camlQuery = parseRowLimitNodeResult.camlQuery;
+    }
+    let removeSingleAndConditionsResult = _removeSingleAndConditions(viewNode, queryNode);
+    if (!(0, _dependencies_1.isNullOrUndefined)(removeSingleAndConditionsResult)) {
+        camlQuery = removeSingleAndConditionsResult.camlQuery;
+    }
+    let getAllItems = (0, _dependencies_1.isNullOrEmptyString)(options.rowLimit) || options.rowLimit < 1;
+    let totalItemsInList = 99999;
+    let maxBatchSize = 5000;
+    let batchSize = options.rowLimit > 0 && options.rowLimit < maxBatchSize ? options.rowLimit : maxBatchSize;
+    let totalNumberOfItemsToGet = getAllItems ? totalItemsInList : options.rowLimit > maxBatchSize ? options.rowLimit : batchSize;
+    let requestUrl = `${(0, list_1.GetListRestUrl)(siteUrl, listIdOrTitle)}/GetItems?$expand=FieldValuesAsText`;
+    let allListFieldsHash = (0, _dependencies_1.toHash)(await (0, list_1.GetListFields)(siteUrl, listIdOrTitle), f => f.InternalName);
+    let expandFields = [];
+    let orderByStatement = (0, _dependencies_1.GetOrderByFromCaml)(camlQuery);
+    if ((0, _dependencies_1.isNullOrUndefined)(options.columns)) {
+        options.columns = [];
+    }
+    let columns = options.columns;
+    columns = _ensureOrderByColumns(orderByStatement, columns);
+    columns = _ensureViewFields(viewFieldsNode, columns, allListFieldsHash);
+    columns = _normalizeColumns(columns, allListFieldsHash);
+    if (columns.length > options.columns.length) {
+        logger.warn(`added ${columns.length - options.columns.length} to query`);
+    }
+    options.columns = columns;
+    // Store the result of processing the request
+    let rtrnProcessRequestResult;
+    let lookupOrUserFieldsInColumns = options.columns.filter((columnName) => {
+        let field = allListFieldsHash[columnName];
+        return _isLookupOrUserField(field);
+    });
+    let isThrottled = lookupOrUserFieldsInColumns.length >= lookupOrUserFieldLimit;
+    try {
+        if (isThrottled) {
+            //ISSUE: 1565
+            rtrnProcessRequestResult = await _processLookupThresholdCamlRequest(orderByStatement, allListFieldsHash, options, camlQuery, requestUrl, expandFields, siteUrl, totalNumberOfItemsToGet, batchSize);
         }
-        let removeSingleAndConditionsResult = _removeSingleAndConditions(viewNode, queryNode);
-        if (!(0, _dependencies_1.isNullOrUndefined)(removeSingleAndConditionsResult)) {
-            camlQuery = removeSingleAndConditionsResult.camlQuery;
+        else {
+            rtrnProcessRequestResult = await _processNormalCamlRequest(orderByStatement, allListFieldsHash, options, camlQuery, requestUrl, expandFields, siteUrl, totalNumberOfItemsToGet, batchSize);
         }
-        let getAllItems = (0, _dependencies_1.isNullOrEmptyString)(options.rowLimit) || options.rowLimit < 1;
-        let totalItemsInList = 99999;
-        let maxBatchSize = 5000;
-        let batchSize = options.rowLimit > 0 && options.rowLimit < maxBatchSize ? options.rowLimit : maxBatchSize;
-        let totalNumberOfItemsToGet = getAllItems ? totalItemsInList : options.rowLimit > maxBatchSize ? options.rowLimit : batchSize;
-        let requestUrl = `${(0, list_1.GetListRestUrl)(siteUrl, listIdOrTitle)}/GetItems?$expand=FieldValuesAsText`;
-        let allListFieldsHash = (0, _dependencies_1.toHash)(yield (0, list_1.GetListFields)(siteUrl, listIdOrTitle), f => f.InternalName);
-        let expandFields = [];
-        let orderByStatement = (0, _dependencies_1.GetOrderByFromCaml)(camlQuery);
-        if ((0, _dependencies_1.isNullOrUndefined)(options.columns)) {
-            options.columns = [];
+        const { items, needContentTypes, postProcessOrderBy } = rtrnProcessRequestResult;
+        let itemsResult = (0, common_2.__fixGetListItemsResults)(siteUrl, listIdOrTitle, items, options.foldersBehaviour);
+        let itemsWithOutContentType = [];
+        if (needContentTypes) {
+            itemsResult.forEach((item) => {
+                if ((0, _dependencies_1.isNullOrUndefined)(item["ContentType"])) {
+                    itemsWithOutContentType.push(item.Id);
+                }
+                else if (!(0, _dependencies_1.isNullOrUndefined)(item["ContentType"].Name)) {
+                    item["ContentType"] = item["ContentType"].Name;
+                }
+            });
         }
-        let columns = options.columns;
-        columns = _ensureOrderByColumns(orderByStatement, columns);
-        columns = _ensureViewFields(viewFieldsNode, columns, allListFieldsHash);
-        columns = _normalizeColumns(columns, allListFieldsHash);
-        if (columns.length > options.columns.length) {
-            logger.warn(`added ${columns.length - options.columns.length} to query`);
+        if (itemsWithOutContentType.length > 0) {
+            logger.time("Getting content types");
+            //Issue 1465 content types no longer come back from get items request...
+            //Make a separate request to get this info
+            let ctypes = (await (0, GetListItemsById_1.GetItemsById)(siteUrl, listIdOrTitle, itemsWithOutContentType, {
+                expand: ['ContentType/Name'],
+                select: ['ContentType/Name', 'Id'],
+                jsonMetadata: _dependencies_1.jsonTypes.nometadata
+            }));
+            itemsResult.forEach(item => {
+                if (!(0, _dependencies_1.isNullOrUndefined)(ctypes[item.Id])) {
+                    item["ContentType"] = ctypes[item.Id].ContentType.Name;
+                }
+            });
+            logger.timeEnd("Getting content types");
         }
-        options.columns = columns;
-        // Store the result of processing the request
-        let rtrnProcessRequestResult;
-        let lookupOrUserFieldsInColumns = options.columns.filter((columnName) => {
-            let field = allListFieldsHash[columnName];
-            return _isLookupOrUserField(field);
-        });
-        let isThrottled = lookupOrUserFieldsInColumns.length >= lookupOrUserFieldLimit;
-        try {
-            if (isThrottled) {
-                //ISSUE: 1565
-                rtrnProcessRequestResult = yield _processLookupThresholdCamlRequest(orderByStatement, allListFieldsHash, options, camlQuery, requestUrl, expandFields, siteUrl, totalNumberOfItemsToGet, batchSize);
-            }
-            else {
-                rtrnProcessRequestResult = yield _processNormalCamlRequest(orderByStatement, allListFieldsHash, options, camlQuery, requestUrl, expandFields, siteUrl, totalNumberOfItemsToGet, batchSize);
-            }
-            const { items, needContentTypes, postProcessOrderBy } = rtrnProcessRequestResult;
-            let itemsResult = (0, common_2.__fixGetListItemsResults)(siteUrl, listIdOrTitle, items, options.foldersBehaviour);
-            let itemsWithOutContentType = [];
-            if (needContentTypes) {
-                itemsResult.forEach((item) => {
-                    if ((0, _dependencies_1.isNullOrUndefined)(item["ContentType"])) {
-                        itemsWithOutContentType.push(item.Id);
-                    }
-                    else if (!(0, _dependencies_1.isNullOrUndefined)(item["ContentType"].Name)) {
-                        item["ContentType"] = item["ContentType"].Name;
-                    }
-                });
+        if (postProcessOrderBy) {
+            //re-apply sort
+            if (_dependencies_1.IsLocalDev) {
+                logger.table(itemsResult.map(i => {
+                    let row = {
+                        Id: i.Id,
+                        Title: i.Title
+                    };
+                    orderByStatement.forEach(s => { row[`${s.Name} ${s.IsAscending ? 'asc' : 'desc'}`] = i[s.Name]; });
+                    return row;
+                }), "before sort", true);
             }
-            if (itemsWithOutContentType.length > 0) {
-                logger.time("Getting content types");
-                //Issue 1465 content types no longer come back from get items request...
-                //Make a separate request to get this info
-                let ctypes = (yield (0, GetListItemsById_1.GetItemsById)(siteUrl, listIdOrTitle, itemsWithOutContentType, {
-                    expand: ['ContentType/Name'],
-                    select: ['ContentType/Name', 'Id'],
-                    jsonMetadata: _dependencies_1.jsonTypes.nometadata
-                }));
-                itemsResult.forEach(item => {
-                    if (!(0, _dependencies_1.isNullOrUndefined)(ctypes[item.Id])) {
-                        item["ContentType"] = ctypes[item.Id].ContentType.Name;
+            itemsResult.sort((a, b) => {
+                for (let i = 0; i < orderByStatement.length; i++) {
+                    let ob = orderByStatement[i];
+                    let v1 = a[ob.Name];
+                    let v2 = b[ob.Name];
+                    if (v1 === v2) {
+                        //these are equal - continue to second sort statement
                     }
-                });
-                logger.timeEnd("Getting content types");
-            }
-            if (postProcessOrderBy) {
-                //re-apply sort
-                if (_dependencies_1.IsLocalDev) {
-                    logger.table(itemsResult.map(i => {
-                        let row = {
-                            Id: i.Id,
-                            Title: i.Title
-                        };
-                        orderByStatement.forEach(s => { row[`${s.Name} ${s.IsAscending ? 'asc' : 'desc'}`] = i[s.Name]; });
-                        return row;
-                    }), "before sort", true);
-                }
-                itemsResult.sort((a, b) => {
-                    for (let i = 0; i < orderByStatement.length; i++) {
-                        let ob = orderByStatement[i];
-                        let v1 = a[ob.Name];
-                        let v2 = b[ob.Name];
-                        if (v1 === v2) {
-                            //these are equal - continue to second sort statement
-                        }
-                        else {
-                            if (ob.IsAscending)
-                                return (v1 > v2) ? 1 : -1;
-                            else
-                                return (v2 > v1) ? 1 : -1;
-                        }
+                    else {
+                        if (ob.IsAscending)
+                            return (v1 > v2) ? 1 : -1;
+                        else
+                            return (v2 > v1) ? 1 : -1;
                     }
-                    return 0;
-                });
-                if (_dependencies_1.IsLocalDev) {
-                    logger.table(itemsResult.map(i => {
-                        let row = {
-                            Id: i.Id,
-                            Title: i.Title
-                        };
-                        orderByStatement.forEach(s => { row[`${s.Name} ${s.IsAscending ? 'asc' : 'desc'}`] = i[s.Name]; });
-                        return row;
-                    }), "after sort", true);
                 }
+                return 0;
+            });
+            if (_dependencies_1.IsLocalDev) {
+                logger.table(itemsResult.map(i => {
+                    let row = {
+                        Id: i.Id,
+                        Title: i.Title
+                    };
+                    orderByStatement.forEach(s => { row[`${s.Name} ${s.IsAscending ? 'asc' : 'desc'}`] = i[s.Name]; });
+                    return row;
+                }), "after sort", true);
             }
-            return itemsResult;
-        }
-        catch (ex) {
-            console.log(`isThrottled: ${isThrottled}`);
         }
-        return null;
-    });
+        return itemsResult;
+    }
+    catch (ex) {
+        console.log(`isThrottled: ${isThrottled}`);
+    }
+    return null;
 }
 exports.GetListItemsByCaml = GetListItemsByCaml;
 // (window as any).runtTest = true;
@@ -172,161 +161,94 @@ exports.GetListItemsByCaml = GetListItemsByCaml;
 //     });
 // }, 500);
 //ISSUE: 1565
-function _processLookupThresholdCamlRequest(orderByStatement, allListFieldsHash, options, camlQuery, requestUrl, expandFields, siteUrl, totalNumberOfItemsToGet, batchSize) {
-    return __awaiter(this, void 0, void 0, function* () {
-        let rtrnProcessRequestResult = {
-            items: [],
-            postProcessOrderBy: false,
-            needContentTypes: false
-        };
-        let orderByFields = orderByStatement.map((orderByField) => {
-            let field = allListFieldsHash[orderByField.Name];
-            return field;
-        });
-        let lookupOrUserFieldsInOrderBy = orderByFields.filter((orderByField) => {
-            return _isLookupOrUserField(orderByField);
-        });
-        let lookupOrUserFieldsToChunk = options.columns.filter((columnName) => {
-            let field = allListFieldsHash[columnName];
-            if (_isLookupOrUserField(field)) {
-                let col = (0, _dependencies_1.firstOrNull)(lookupOrUserFieldsInOrderBy, (orderByField) => {
-                    return field.InternalName !== orderByField.InternalName;
-                });
-                return col === null;
-            }
-            return false;
-        });
-        let otherFieldNames = options.columns.filter((columnName) => {
-            let field = allListFieldsHash[columnName];
-            let exists = !(0, _dependencies_1.isNullOrUndefined)(field);
-            let isLookupField = exists && _isLookupOrUserField(field);
-            let isInOrderBy = exists && (0, _dependencies_1.firstOrNull)(orderByFields, (orderByField) => orderByField.InternalName === field.InternalName) !== null;
-            return exists && !isLookupField && !isInOrderBy;
-        });
-        //Split the requests into equally chunked arrays based on the lookup field count
-        //add lookup fields in chunk but leave room to add lookup fields from the order by statement 
-        //since they are mandatory    
-        let requestChunks = (0, _dependencies_1.chunkArray)(lookupOrUserFieldsToChunk, lookupOrUserFieldLimit - lookupOrUserFieldsInOrderBy.length);
-        let otherFieldsChunkSize = Math.ceil(otherFieldNames.length / requestChunks.length);
-        let otherFieldsChunks = (0, _dependencies_1.chunkArray)(otherFieldNames, otherFieldsChunkSize);
-        requestChunks.forEach((chunk, index) => {
-            //Add all order by fields to each request, this will include the lookup
-            //fields from the order by statement that we left room for previously
-            requestChunks[index] = chunk.concat(orderByFields.map((orderByField) => {
-                return orderByField.InternalName;
-            }));
-            //Add the other fields but split them across the requests so we don't request duplicate data
-            if (otherFieldsChunks[index]) {
-                requestChunks[index] = chunk.concat(otherFieldsChunks[index]);
-            }
-        });
-        //requestChunks should now have about the same number of fields in each chunk and each
-        //chunk will have all the order by fields.            
-        let queries = requestChunks.map((requestChunk) => {
-            let camlQueryClone = camlQuery;
-            let processColumnsResult = _processColumns(requestChunk, expandFields, allListFieldsHash);
-            //Id field must be included in oder to merge the items correctly
-            let viewFields = processColumnsResult.viewFields;
-            if (viewFields.length && !viewFields.includes("Id")) {
-                viewFields.push("Id");
-            }
-            let selectFields = processColumnsResult.selectFields;
-            if (selectFields.length && !selectFields.includes("Id")) {
-                selectFields.push("Id");
-            }
-            rtrnProcessRequestResult.needContentTypes = rtrnProcessRequestResult.needContentTypes || processColumnsResult.needContentTypes;
-            expandFields = processColumnsResult.expandFields;
-            if ((0, _dependencies_1.isNotEmptyArray)(viewFields)) {
-                camlQueryClone = (0, _dependencies_1.EnsureViewFields)(camlQueryClone, viewFields, true);
-            }
-            // if (isDebug()) {
-            //     let lfields = viewFields.filter((fieldName) => {
-            //         let field = allListFieldsHash[fieldName];
-            //         return _isLookupOrUserField(field);
-            //     });
-            //     let xmlDoc = new DOMParser().parseFromString(camlQueryClone, "text/xml");
-            //     let viewNode = xmlDoc.querySelector("View, view");
-            //     let viewFieldsNode = viewNode && viewNode.querySelector("ViewFields, viewfields");
-            //     let viewFields2 = Array.from(viewFieldsNode.children).map((viewFieldNode) => {
-            //         let name = viewFieldNode.getAttribute("Name") || viewFieldNode.getAttribute("name");
-            //         return name;
-            //     });
-            //     let lfields2 = viewFields2.filter((fieldName) => {
-            //         let field = allListFieldsHash[fieldName];
-            //         return _isLookupOrUserField(field);
-            //     });
-            //     if (lfields2.length !== lfields.length) {
-            //         logger.warn("Lookup fields in caml query do not match look up fields in view fields.");
-            //         logger.warn(`Lookup fields in caml query: ${lfields2}`)
-            //         logger.warn(`Lookup fields in in view fields: ${lfields}`)
-            //     }
-            // }
-            // Prepare the REST URL with select fields
-            let restUrlWithSelect = requestUrl;
-            if (expandFields.length > 0) {
-                restUrlWithSelect += ',' + expandFields.join(',');
-            }
-            // Include the lookup user select fields with the other select fields
-            restUrlWithSelect += '&$select=' + selectFields.join(',');
-            return { camlQueryClone, restUrlWithSelect, viewFields };
-        });
-        let mergedItems = null;
-        for (let i = 0; i < queries.length; i++) {
-            const { camlQueryClone, restUrlWithSelect, viewFields } = queries[i];
-            let loopResult = yield processRequestResult(requestUrl, restUrlWithSelect, camlQueryClone, Object.assign(Object.assign({}, options), { columns: viewFields }), siteUrl, totalNumberOfItemsToGet, batchSize, orderByStatement);
-            if ((0, _dependencies_1.isNullOrUndefined)(mergedItems)) {
-                mergedItems = loopResult.items;
-            }
-            else {
-                for (let restItemIndex = 0; restItemIndex < loopResult.items.length; restItemIndex++) {
-                    //The item chunks (loopResult) should be in the same order as mergedItems because each request should have the 
-                    //same number of items returned in the same order. The only difference between requests is which fields are present.
-                    //So request 1 will have the 1st set of fields, request 2 will have the 2nd set. etc.
-                    //This means that the restItemIndex should be the same as existingItemIndex for each iteration. But, we do these extra 
-                    //checks just in case.
-                    let restItem = loopResult.items[restItemIndex];
-                    let existingItem = mergedItems[restItemIndex];
-                    let existingItemIndex = restItemIndex;
-                    if ((0, _dependencies_1.isNullOrUndefined)(existingItem) || existingItem.Id !== restItem.Id) {
-                        existingItemIndex = (0, _dependencies_1.firstIndexOf)(mergedItems, (mergedItem) => {
-                            return mergedItem.Id === restItem.Id;
-                        });
-                    }
-                    if (existingItemIndex === -1) {
-                        //We shouldn't get here. Each chunk should have the same items in the same order.
-                        logger.warn("_processLookupThresholdCamlRequest results are out of sync");
-                        mergedItems.push(restItem);
-                    }
-                    else {
-                        let existingItem = mergedItems[existingItemIndex];
-                        let FieldValuesAsText = Object.assign(Object.assign({}, (existingItem.FieldValuesAsText || {})), (restItem.FieldValuesAsText || {}));
-                        let FieldValuesForEdit = Object.assign(Object.assign({}, (existingItem.FieldValuesForEdit || {})), (restItem.FieldValuesForEdit || {}));
-                        existingItem = Object.assign(Object.assign({}, existingItem), restItem);
-                        existingItem.FieldValuesAsText = FieldValuesAsText;
-                        existingItem.FieldValuesForEdit = FieldValuesForEdit;
-                        mergedItems[existingItemIndex] = existingItem;
-                    }
-                }
-            }
-            // only need to put this true if it happens once
-            if (loopResult.postProcessOrderBy) {
-                rtrnProcessRequestResult.postProcessOrderBy = true;
-            }
+async function _processLookupThresholdCamlRequest(orderByStatement, allListFieldsHash, options, camlQuery, requestUrl, expandFields, siteUrl, totalNumberOfItemsToGet, batchSize) {
+    let rtrnProcessRequestResult = {
+        items: [],
+        postProcessOrderBy: false,
+        needContentTypes: false
+    };
+    let orderByFields = orderByStatement.map((orderByField) => {
+        let field = allListFieldsHash[orderByField.Name];
+        return field;
+    });
+    let lookupOrUserFieldsInOrderBy = orderByFields.filter((orderByField) => {
+        return _isLookupOrUserField(orderByField);
+    });
+    let lookupOrUserFieldsToChunk = options.columns.filter((columnName) => {
+        let field = allListFieldsHash[columnName];
+        if (_isLookupOrUserField(field)) {
+            let col = (0, _dependencies_1.firstOrNull)(lookupOrUserFieldsInOrderBy, (orderByField) => {
+                return field.InternalName !== orderByField.InternalName;
+            });
+            return col === null;
         }
-        rtrnProcessRequestResult.items = mergedItems;
-        return rtrnProcessRequestResult;
+        return false;
     });
-}
-function _processNormalCamlRequest(orderByStatement, allListFieldsHash, options, camlQuery, requestUrl, expandFields, siteUrl, totalNumberOfItemsToGet, batchSize) {
-    return __awaiter(this, void 0, void 0, function* () {
-        let processColumnsResult = _processColumns(options.columns, expandFields, allListFieldsHash);
+    let otherFieldNames = options.columns.filter((columnName) => {
+        let field = allListFieldsHash[columnName];
+        let exists = !(0, _dependencies_1.isNullOrUndefined)(field);
+        let isLookupField = exists && _isLookupOrUserField(field);
+        let isInOrderBy = exists && (0, _dependencies_1.firstOrNull)(orderByFields, (orderByField) => orderByField.InternalName === field.InternalName) !== null;
+        return exists && !isLookupField && !isInOrderBy;
+    });
+    //Split the requests into equally chunked arrays based on the lookup field count
+    //add lookup fields in chunk but leave room to add lookup fields from the order by statement 
+    //since they are mandatory    
+    let requestChunks = (0, _dependencies_1.chunkArray)(lookupOrUserFieldsToChunk, lookupOrUserFieldLimit - lookupOrUserFieldsInOrderBy.length);
+    let otherFieldsChunkSize = Math.ceil(otherFieldNames.length / requestChunks.length);
+    let otherFieldsChunks = (0, _dependencies_1.chunkArray)(otherFieldNames, otherFieldsChunkSize);
+    requestChunks.forEach((chunk, index) => {
+        //Add all order by fields to each request, this will include the lookup
+        //fields from the order by statement that we left room for previously
+        requestChunks[index] = chunk.concat(orderByFields.map((orderByField) => {
+            return orderByField.InternalName;
+        }));
+        //Add the other fields but split them across the requests so we don't request duplicate data
+        if (otherFieldsChunks[index]) {
+            requestChunks[index] = chunk.concat(otherFieldsChunks[index]);
+        }
+    });
+    //requestChunks should now have about the same number of fields in each chunk and each
+    //chunk will have all the order by fields.            
+    let queries = requestChunks.map((requestChunk) => {
+        let camlQueryClone = camlQuery;
+        let processColumnsResult = _processColumns(requestChunk, expandFields, allListFieldsHash);
+        //Id field must be included in oder to merge the items correctly
         let viewFields = processColumnsResult.viewFields;
+        if (viewFields.length && !viewFields.includes("Id")) {
+            viewFields.push("Id");
+        }
         let selectFields = processColumnsResult.selectFields;
+        if (selectFields.length && !selectFields.includes("Id")) {
+            selectFields.push("Id");
+        }
+        rtrnProcessRequestResult.needContentTypes = rtrnProcessRequestResult.needContentTypes || processColumnsResult.needContentTypes;
         expandFields = processColumnsResult.expandFields;
-        let needContentTypes = processColumnsResult.needContentTypes;
         if ((0, _dependencies_1.isNotEmptyArray)(viewFields)) {
-            camlQuery = (0, _dependencies_1.EnsureViewFields)(camlQuery, viewFields, true);
+            camlQueryClone = (0, _dependencies_1.EnsureViewFields)(camlQueryClone, viewFields, true);
         }
+        // if (isDebug()) {
+        //     let lfields = viewFields.filter((fieldName) => {
+        //         let field = allListFieldsHash[fieldName];
+        //         return _isLookupOrUserField(field);
+        //     });
+        //     let xmlDoc = new DOMParser().parseFromString(camlQueryClone, "text/xml");
+        //     let viewNode = xmlDoc.querySelector("View, view");
+        //     let viewFieldsNode = viewNode && viewNode.querySelector("ViewFields, viewfields");
+        //     let viewFields2 = Array.from(viewFieldsNode.children).map((viewFieldNode) => {
+        //         let name = viewFieldNode.getAttribute("Name") || viewFieldNode.getAttribute("name");
+        //         return name;
+        //     });
+        //     let lfields2 = viewFields2.filter((fieldName) => {
+        //         let field = allListFieldsHash[fieldName];
+        //         return _isLookupOrUserField(field);
+        //     });
+        //     if (lfields2.length !== lfields.length) {
+        //         logger.warn("Lookup fields in caml query do not match look up fields in view fields.");
+        //         logger.warn(`Lookup fields in caml query: ${lfields2}`)
+        //         logger.warn(`Lookup fields in in view fields: ${lfields}`)
+        //     }
+        // }
         // Prepare the REST URL with select fields
         let restUrlWithSelect = requestUrl;
         if (expandFields.length > 0) {
@@ -334,10 +256,82 @@ function _processNormalCamlRequest(orderByStatement, allListFieldsHash, options,
         }
         // Include the lookup user select fields with the other select fields
         restUrlWithSelect += '&$select=' + selectFields.join(',');
-        // Process the request and get the result
-        let result = yield processRequestResult(requestUrl, restUrlWithSelect, camlQuery, options, siteUrl, totalNumberOfItemsToGet, batchSize, orderByStatement);
-        return Object.assign(Object.assign({}, result), { needContentTypes });
+        return { camlQueryClone, restUrlWithSelect, viewFields };
     });
+    let mergedItems = null;
+    for (let i = 0; i < queries.length; i++) {
+        const { camlQueryClone, restUrlWithSelect, viewFields } = queries[i];
+        let loopResult = await processRequestResult(requestUrl, restUrlWithSelect, camlQueryClone, { ...options, columns: viewFields }, siteUrl, totalNumberOfItemsToGet, batchSize, orderByStatement);
+        if ((0, _dependencies_1.isNullOrUndefined)(mergedItems)) {
+            mergedItems = loopResult.items;
+        }
+        else {
+            for (let restItemIndex = 0; restItemIndex < loopResult.items.length; restItemIndex++) {
+                //The item chunks (loopResult) should be in the same order as mergedItems because each request should have the 
+                //same number of items returned in the same order. The only difference between requests is which fields are present.
+                //So request 1 will have the 1st set of fields, request 2 will have the 2nd set. etc.
+                //This means that the restItemIndex should be the same as existingItemIndex for each iteration. But, we do these extra 
+                //checks just in case.
+                let restItem = loopResult.items[restItemIndex];
+                let existingItem = mergedItems[restItemIndex];
+                let existingItemIndex = restItemIndex;
+                if ((0, _dependencies_1.isNullOrUndefined)(existingItem) || existingItem.Id !== restItem.Id) {
+                    existingItemIndex = (0, _dependencies_1.firstIndexOf)(mergedItems, (mergedItem) => {
+                        return mergedItem.Id === restItem.Id;
+                    });
+                }
+                if (existingItemIndex === -1) {
+                    //We shouldn't get here. Each chunk should have the same items in the same order.
+                    logger.warn("_processLookupThresholdCamlRequest results are out of sync");
+                    mergedItems.push(restItem);
+                }
+                else {
+                    let existingItem = mergedItems[existingItemIndex];
+                    let FieldValuesAsText = {
+                        ...(existingItem.FieldValuesAsText || {}),
+                        ...(restItem.FieldValuesAsText || {})
+                    };
+                    let FieldValuesForEdit = {
+                        ...(existingItem.FieldValuesForEdit || {}),
+                        ...(restItem.FieldValuesForEdit || {})
+                    };
+                    existingItem = { ...existingItem, ...restItem };
+                    existingItem.FieldValuesAsText = FieldValuesAsText;
+                    existingItem.FieldValuesForEdit = FieldValuesForEdit;
+                    mergedItems[existingItemIndex] = existingItem;
+                }
+            }
+        }
+        // only need to put this true if it happens once
+        if (loopResult.postProcessOrderBy) {
+            rtrnProcessRequestResult.postProcessOrderBy = true;
+        }
+    }
+    rtrnProcessRequestResult.items = mergedItems;
+    return rtrnProcessRequestResult;
+}
+async function _processNormalCamlRequest(orderByStatement, allListFieldsHash, options, camlQuery, requestUrl, expandFields, siteUrl, totalNumberOfItemsToGet, batchSize) {
+    let processColumnsResult = _processColumns(options.columns, expandFields, allListFieldsHash);
+    let viewFields = processColumnsResult.viewFields;
+    let selectFields = processColumnsResult.selectFields;
+    expandFields = processColumnsResult.expandFields;
+    let needContentTypes = processColumnsResult.needContentTypes;
+    if ((0, _dependencies_1.isNotEmptyArray)(viewFields)) {
+        camlQuery = (0, _dependencies_1.EnsureViewFields)(camlQuery, viewFields, true);
+    }
+    // Prepare the REST URL with select fields
+    let restUrlWithSelect = requestUrl;
+    if (expandFields.length > 0) {
+        restUrlWithSelect += ',' + expandFields.join(',');
+    }
+    // Include the lookup user select fields with the other select fields
+    restUrlWithSelect += '&$select=' + selectFields.join(',');
+    // Process the request and get the result
+    let result = await processRequestResult(requestUrl, restUrlWithSelect, camlQuery, options, siteUrl, totalNumberOfItemsToGet, batchSize, orderByStatement);
+    return {
+        ...result,
+        needContentTypes
+    };
 }
 function _ensureViewFields(viewFieldsNode, columns, allListFieldsHash) {
     //ISSUE: 1565
@@ -538,92 +532,90 @@ function _isLookupOrUserField(field) {
             return false;
     }
 }
-function processRequestResult(requestUrl, restUrlWithSelect, camlQuery, options, siteUrl, totalNumberOfItemsToGet, batchSize, orderByStatement) {
-    return __awaiter(this, void 0, void 0, function* () {
-        //issue 6150: if there are too many fields, url will be too long. just get all columns without adding a $select
-        if (restUrlWithSelect.length < 2000)
-            requestUrl = restUrlWithSelect;
-        let query = { ViewXml: camlQuery.replace("</View>", `<RowLimit>${batchSize}</RowLimit></View>`) };
-        if (!(0, _dependencies_1.isNullOrEmptyString)(options.FolderServerRelativeUrl))
-            query.FolderServerRelativeUrl = options.FolderServerRelativeUrl;
-        let data = { query: query };
-        let items = [];
-        //let triedWithoutViewFields = false;
-        let postProcessOrderBy = false;
-        do {
-            try {
-                let requestResult = yield (0, rest_1.GetJson)(requestUrl, JSON.stringify(data), {
-                    allowCache: options.refreshCache !== true,
-                    postCacheKey: JSON.stringify(data.query),
-                    jsonMetadata: _dependencies_1.jsonTypes.nometadata,
-                    spWebUrl: siteUrl
-                });
-                if (requestResult && requestResult.value)
-                    items.push(...requestResult.value);
-                let itemsLeftToGet = totalNumberOfItemsToGet - items.length;
-                if (itemsLeftToGet > 0 && //we need more items
-                    requestResult.value.length === batchSize //we might have more on server since we got the full batch size we asked for
-                ) {
-                    let lastItem = items[items.length - 1];
-                    let lastItemId = lastItem.Id;
-                    let pagingInfoSort = "";
-                    //Issue 7542 need to add order by value to the paging info if it is in the query
-                    if (!postProcessOrderBy && orderByStatement && orderByStatement.length) {
-                        let orderFieldName = orderByStatement[0].Name;
-                        let orderFieldValue = lastItem[orderFieldName];
-                        //if field is ID - do not do it.
-                        if (orderFieldName.toLowerCase() !== "id" && !(0, _dependencies_1.isNullOrUndefined)(orderFieldValue)) {
-                            //if value is date - we need it in ISO and in this format: yyyyMMdd HH:mm:ss
-                            try {
-                                //Numbers cast to date properly but they are not dates. ignore number values.
-                                let orderFieldValueAsDate = (0, _dependencies_1.isNumeric)(orderFieldValue) ? null : new Date(orderFieldValue);
-                                if ((0, _dependencies_1.isDate)(orderFieldValueAsDate)) {
-                                    try {
-                                        //issue 7599 date only field on different time zone...
-                                        orderFieldValue = (0, web_1.SPServerLocalTimeToUTCSync)(siteUrl, orderFieldValueAsDate).replace(/T/i, " ").replace(/Z/i, " ");
-                                    }
-                                    catch (e) { }
+async function processRequestResult(requestUrl, restUrlWithSelect, camlQuery, options, siteUrl, totalNumberOfItemsToGet, batchSize, orderByStatement) {
+    //issue 6150: if there are too many fields, url will be too long. just get all columns without adding a $select
+    if (restUrlWithSelect.length < 2000)
+        requestUrl = restUrlWithSelect;
+    let query = { ViewXml: camlQuery.replace("</View>", `<RowLimit>${batchSize}</RowLimit></View>`) };
+    if (!(0, _dependencies_1.isNullOrEmptyString)(options.FolderServerRelativeUrl))
+        query.FolderServerRelativeUrl = options.FolderServerRelativeUrl;
+    let data = { query: query };
+    let items = [];
+    //let triedWithoutViewFields = false;
+    let postProcessOrderBy = false;
+    do {
+        try {
+            let requestResult = await (0, rest_1.GetJson)(requestUrl, JSON.stringify(data), {
+                allowCache: options.refreshCache !== true,
+                postCacheKey: JSON.stringify(data.query),
+                jsonMetadata: _dependencies_1.jsonTypes.nometadata,
+                spWebUrl: siteUrl
+            });
+            if (requestResult && requestResult.value)
+                items.push(...requestResult.value);
+            let itemsLeftToGet = totalNumberOfItemsToGet - items.length;
+            if (itemsLeftToGet > 0 && //we need more items
+                requestResult.value.length === batchSize //we might have more on server since we got the full batch size we asked for
+            ) {
+                let lastItem = items[items.length - 1];
+                let lastItemId = lastItem.Id;
+                let pagingInfoSort = "";
+                //Issue 7542 need to add order by value to the paging info if it is in the query
+                if (!postProcessOrderBy && orderByStatement && orderByStatement.length) {
+                    let orderFieldName = orderByStatement[0].Name;
+                    let orderFieldValue = lastItem[orderFieldName];
+                    //if field is ID - do not do it.
+                    if (orderFieldName.toLowerCase() !== "id" && !(0, _dependencies_1.isNullOrUndefined)(orderFieldValue)) {
+                        //if value is date - we need it in ISO and in this format: yyyyMMdd HH:mm:ss
+                        try {
+                            //Numbers cast to date properly but they are not dates. ignore number values.
+                            let orderFieldValueAsDate = (0, _dependencies_1.isNumeric)(orderFieldValue) ? null : new Date(orderFieldValue);
+                            if ((0, _dependencies_1.isDate)(orderFieldValueAsDate)) {
+                                try {
+                                    //issue 7599 date only field on different time zone...
+                                    orderFieldValue = (0, web_1.SPServerLocalTimeToUTCSync)(siteUrl, orderFieldValueAsDate).replace(/T/i, " ").replace(/Z/i, " ");
                                 }
+                                catch (e) { }
                             }
-                            catch (e) { }
-                            pagingInfoSort = `&p_${orderFieldName}=${(0, _dependencies_1.encodeURIComponentEX)(orderFieldValue)}`;
                         }
+                        catch (e) { }
+                        pagingInfoSort = `&p_${orderFieldName}=${(0, _dependencies_1.encodeURIComponentEX)(orderFieldValue)}`;
                     }
-                    data.query.ListItemCollectionPosition = {
-                        "PagingInfo": `Paged=TRUE${pagingInfoSort}&p_ID=${lastItemId}`
-                    };
-                    if (itemsLeftToGet < batchSize) //the last batch should be smaller, update the row limit
-                        data.query.ViewXml = camlQuery.replace("</View>", `<RowLimit>${itemsLeftToGet}</RowLimit></View>`);
                 }
-                else
-                    data = null;
+                data.query.ListItemCollectionPosition = {
+                    "PagingInfo": `Paged=TRUE${pagingInfoSort}&p_ID=${lastItemId}`
+                };
+                if (itemsLeftToGet < batchSize) //the last batch should be smaller, update the row limit
+                    data.query.ViewXml = camlQuery.replace("</View>", `<RowLimit>${itemsLeftToGet}</RowLimit></View>`);
             }
-            catch (e) {
-                if ((0, common_1.__getSPRestErrorData)(e).code.indexOf('SPQueryThrottledException')) {
-                    //test again - on awaiting score view, this will work but will ONLY return the order by fields...
-                    // if (!triedWithoutViewFields) {
-                    //     //Our own issues list had too many lookup fields.
-                    //     logger.info("Query throttled, trying again without view fields... some fields might be missing from results.");
-                    //     triedWithoutViewFields = true;
-                    //     camlQuery = EnsureViewFields(camlQuery, orderByStatement.map(o => o.Name), false, true);
-                    //     data.query.ViewXml = camlQuery.replace("</View>", `<RowLimit>${batchSize}</RowLimit></View>`);
-                    // }
-                    // else
-                    if (!postProcessOrderBy && orderByStatement.length > 0) {
-                        logger.warn("Query throttled, trying again without order by...");
-                        postProcessOrderBy = true;
-                        camlQuery = (0, _dependencies_1.RemoveOrderByFromCaml)(camlQuery);
-                        camlQuery = (0, _dependencies_1.EnsureViewFields)(camlQuery, [], false, true);
-                        data.query.ViewXml = camlQuery.replace("</View>", `<RowLimit>${batchSize}</RowLimit></View>`);
-                    }
-                    else
-                        throw e; //still throttled? might be due to filter query
+            else
+                data = null;
+        }
+        catch (e) {
+            if ((0, common_1.__getSPRestErrorData)(e).code.indexOf('SPQueryThrottledException')) {
+                //test again - on awaiting score view, this will work but will ONLY return the order by fields...
+                // if (!triedWithoutViewFields) {
+                //     //Our own issues list had too many lookup fields.
+                //     logger.info("Query throttled, trying again without view fields... some fields might be missing from results.");
+                //     triedWithoutViewFields = true;
+                //     camlQuery = EnsureViewFields(camlQuery, orderByStatement.map(o => o.Name), false, true);
+                //     data.query.ViewXml = camlQuery.replace("</View>", `<RowLimit>${batchSize}</RowLimit></View>`);
+                // }
+                // else
+                if (!postProcessOrderBy && orderByStatement.length > 0) {
+                    logger.warn("Query throttled, trying again without order by...");
+                    postProcessOrderBy = true;
+                    camlQuery = (0, _dependencies_1.RemoveOrderByFromCaml)(camlQuery);
+                    camlQuery = (0, _dependencies_1.EnsureViewFields)(camlQuery, [], false, true);
+                    data.query.ViewXml = camlQuery.replace("</View>", `<RowLimit>${batchSize}</RowLimit></View>`);
                 }
                 else
-                    throw e; //different error
+                    throw e; //still throttled? might be due to filter query
             }
-        } while (!(0, _dependencies_1.isNullOrEmptyString)(data));
-        return { items: items, postProcessOrderBy: postProcessOrderBy };
-    });
+            else
+                throw e; //different error
+        }
+    } while (!(0, _dependencies_1.isNullOrEmptyString)(data));
+    return { items: items, postProcessOrderBy: postProcessOrderBy };
 }
 //# sourceMappingURL=GetListItemsByCaml.js.map