@kwiz/common 1.0.78 → 1.0.79

package/src/utils/sharepoint.rest/listutils/GetListItemsByCaml.ts

@@ -1,775 +1,775 @@
-import { IsLocalDev } from "../../../_dependencies";
-import { chunkArray, firstIndexOf, firstOrNull, makeUniqueArray, toHash } from "../../../helpers/collections.base";
-import { EnsureViewFields, GetOrderByFromCaml, RemoveOrderByFromCaml, getFieldOutputType } from "../../../helpers/sharepoint";
-import { isDate, isNotEmptyArray, isNullOrEmptyArray, isNullOrEmptyString, isNullOrNaN, isNullOrUndefined, isNumeric } from "../../../helpers/typecheckers";
-import { encodeURIComponentEX } from "../../../helpers/url";
-import { IDictionary } from "../../../types/common.types";
-import { jsonTypes } from "../../../types/rest.types";
-import { IFieldInfoEX } from "../../../types/sharepoint.types";
-import { GeListItemsFoldersBehaviour, IRestItem } from "../../../types/sharepoint.utils.types";
-import { ConsoleLogger } from "../../consolelogger";
-import { GetJson } from "../../rest";
-import { GetSiteUrl, __getSPRestErrorData } from "../common";
-import { GetListFields, GetListRestUrl } from "../list";
-import { SPServerLocalTimeToUTCSync } from "../web";
-import { GetItemsById } from "./GetListItemsById";
-import { SkipFields, __fixGetListItemsResults } from "./common";
-
-const logger = ConsoleLogger.get("sharepoint.rest/list/GetListItemsByCaml");
-
-interface ICamlOptions {
-    /** Optional, default: 1000. 0: get all items. */
-    rowLimit?: number;
-    /** Id, Title, Modified, FileLeafRef, FileDirRef, FileRef, FileSystemObjectType */
-    columns: string[];
-    foldersBehaviour?: GeListItemsFoldersBehaviour;
-    /** set query to get items from this folder only */
-    FolderServerRelativeUrl?: string;
-    refreshCache?: boolean;
-}
-
-//Lookup threshold limit is 12 but we use a smaller number here to ensure there are no issues
-const lookupOrUserFieldLimit = 11;
-/** returns the items or NULL, never errors out since it is used in aggregator.
- *  set throwErrors = true if you want errors to be thrown instead of returning null.
- * camlQuery: one of:
- *  - view.ViewQuery
- *  - wrapped view.ViewQuery in a <View>: "&lt;View Scope='RecursiveAll'>&lt;Query>&lt;/Query>&lt;RowLimit>1000&lt;/RowLimit>&lt;/View>" or send rowLimit in options.
- */
-export async function GetListItemsByCaml(siteUrl: string, listIdOrTitle: string, camlQuery: string, options: ICamlOptions): Promise<IRestItem[]> {
-    siteUrl = GetSiteUrl(siteUrl);
-
-    if (!camlQuery.toLowerCase().startsWith("<view")) {
-        camlQuery = `<View Scope='RecursiveAll'><Query>${camlQuery}</Query></View>`;
-    }
-
-    let xmlParser = new DOMParser();
-    let xmlDoc = xmlParser.parseFromString(camlQuery, "text/xml");
-    let viewNode = xmlDoc.querySelector("View, view");
-    let queryNode = viewNode && viewNode.querySelector("Query, query");
-    let rowLimitNode = viewNode && viewNode.querySelector("RowLimit, rowlimit");
-    let viewFieldsNode = viewNode && viewNode.querySelector("ViewFields, viewfields");
-
-    let parseRowLimitNodeResult = _parseRowLimitNode(rowLimitNode, viewNode, options.rowLimit);
-    if (!isNullOrUndefined(parseRowLimitNodeResult)) {
-        options.rowLimit = parseRowLimitNodeResult.rowLimit;
-        camlQuery = parseRowLimitNodeResult.camlQuery;
-    }
-
-    let removeSingleAndConditionsResult = _removeSingleAndConditions(viewNode, queryNode);
-    if (!isNullOrUndefined(removeSingleAndConditionsResult)) {
-        camlQuery = removeSingleAndConditionsResult.camlQuery;
-    }
-
-    let getAllItems = isNullOrEmptyString(options.rowLimit) || options.rowLimit < 1;
-    let totalItemsInList = 99999;
-
-    let maxBatchSize = 5000;
-    let batchSize = options.rowLimit > 0 && options.rowLimit < maxBatchSize ? options.rowLimit : maxBatchSize;
-    let totalNumberOfItemsToGet = getAllItems ? totalItemsInList : options.rowLimit > maxBatchSize ? options.rowLimit : batchSize;
-
-    let requestUrl = `${GetListRestUrl(siteUrl, listIdOrTitle)}/GetItems?$expand=FieldValuesAsText`;
-
-    let allListFieldsHash = toHash(await GetListFields(siteUrl, listIdOrTitle), f => f.InternalName);
-
-    let expandFields: string[] = [];
-
-    let orderByStatement = GetOrderByFromCaml(camlQuery);
-
-    if (isNullOrUndefined(options.columns)) {
-        options.columns = [];
-    }
-
-    let columns = options.columns;
-    columns = _ensureOrderByColumns(orderByStatement, columns);
-    columns = _ensureViewFields(viewFieldsNode, columns, allListFieldsHash);
-    columns = _normalizeColumns(columns, allListFieldsHash);
-
-    if (columns.length > options.columns.length) {
-        logger.warn(`added ${columns.length - options.columns.length} to query`);
-    }
-
-    options.columns = columns;
-
-    // Store the result of processing the request
-    let rtrnProcessRequestResult: { items: IRestItem[]; postProcessOrderBy: boolean; needContentTypes: boolean };
-
-    let lookupOrUserFieldsInColumns = options.columns.filter((columnName) => {
-        let field = allListFieldsHash[columnName];
-        return _isLookupOrUserField(field);
-    });
-
-    let isThrottled = lookupOrUserFieldsInColumns.length >= lookupOrUserFieldLimit;
-
-    try {
-        if (isThrottled) {
-            //ISSUE: 1565
-            rtrnProcessRequestResult = await _processLookupThresholdCamlRequest(
-                orderByStatement,
-                allListFieldsHash,
-                options,
-                camlQuery,
-                requestUrl,
-                expandFields,
-                siteUrl,
-                totalNumberOfItemsToGet,
-                batchSize);
-        } else {
-            rtrnProcessRequestResult = await _processNormalCamlRequest(
-                orderByStatement,
-                allListFieldsHash,
-                options,
-                camlQuery,
-                requestUrl,
-                expandFields,
-                siteUrl,
-                totalNumberOfItemsToGet,
-                batchSize);
-        }
-
-        const { items, needContentTypes, postProcessOrderBy } = rtrnProcessRequestResult
-
-        let itemsResult = __fixGetListItemsResults(siteUrl, listIdOrTitle, items, options.foldersBehaviour);
-
-        let itemsWithOutContentType: number[] = [];
-        if (needContentTypes) {
-            itemsResult.forEach((item) => {
-                if (isNullOrUndefined(item["ContentType"])) {
-                    itemsWithOutContentType.push(item.Id);
-                } else if (!isNullOrUndefined(item["ContentType"].Name)) {
-                    item["ContentType"] = item["ContentType"].Name;
-                }
-            });
-        }
-
-        if (itemsWithOutContentType.length > 0) {
-            logger.time("Getting content types");
-            //Issue 1465 content types no longer come back from get items request...
-            //Make a separate request to get this info
-            let ctypes = (await GetItemsById(siteUrl, listIdOrTitle, itemsWithOutContentType, {
-                expand: ['ContentType/Name'],
-                select: ['ContentType/Name', 'Id'],
-                jsonMetadata: jsonTypes.nometadata
-            })) as any as { Id: string, ContentType: { Name: string } }[];
-
-            itemsResult.forEach(item => {
-                if (!isNullOrUndefined(ctypes[item.Id])) {
-                    item["ContentType"] = ctypes[item.Id].ContentType.Name
-                }
-            });
-            logger.timeEnd("Getting content types");
-        }
-
-        if (postProcessOrderBy) {
-            //re-apply sort
-            if (IsLocalDev) {
-                logger.table(itemsResult.map(i => {
-                    let row = {
-                        Id: i.Id,
-                        Title: i.Title
-                    };
-                    orderByStatement.forEach(s => { row[`${s.Name} ${s.IsAscending ? 'asc' : 'desc'}`] = i[s.Name]; });
-                    return row;
-                }), "before sort", true);
-            }
-            itemsResult.sort((a, b) => {
-                for (let i = 0; i < orderByStatement.length; i++) {
-                    let ob = orderByStatement[i];
-                    let v1 = a[ob.Name];
-                    let v2 = b[ob.Name];
-                    if (v1 === v2) {
-                        //these are equal - continue to second sort statement
-                    }
-                    else {
-                        if (ob.IsAscending)
-                            return (v1 > v2) ? 1 : -1;
-                        else
-                            return (v2 > v1) ? 1 : -1;
-                    }
-                }
-                return 0;
-            });
-
-            if (IsLocalDev) {
-                logger.table(itemsResult.map(i => {
-                    let row = {
-                        Id: i.Id,
-                        Title: i.Title
-                    };
-                    orderByStatement.forEach(s => { row[`${s.Name} ${s.IsAscending ? 'asc' : 'desc'}`] = i[s.Name]; });
-                    return row;
-                }), "after sort", true);
-            }
-
-        }
-
-        return itemsResult;
-    } catch (ex) {
-        console.log(`isThrottled: ${isThrottled}`);
-    }
-
-    return null;
-}
-
-// (window as any).runtTest = true;
-// window.setTimeout(() => {
-//     if ((window as any).runtTest == false) return;
-//     (window as any).runtTest = false;
-//     GetListItemsByCaml(null, "aec7756b-daa0-4da1-88ba-c66cb572e816", "", {
-//         columns: [],
-//         refreshCache: true
-//     }).then((r) => {
-
-//     });
-// }, 500);
-
-//ISSUE: 1565
-async function _processLookupThresholdCamlRequest(
-    orderByStatement: { Name: string; IsAscending: boolean; }[],
-    allListFieldsHash: IDictionary<IFieldInfoEX>,
-    options: ICamlOptions,
-    camlQuery: string,
-    requestUrl: string,
-    expandFields: string[],
-    siteUrl: string,
-    totalNumberOfItemsToGet: number,
-    batchSize: number) {
-    let rtrnProcessRequestResult = {
-        items: [],
-        postProcessOrderBy: false,
-        needContentTypes: false
-    };
-
-    let orderByFields = orderByStatement.map((orderByField) => {
-        let field = allListFieldsHash[orderByField.Name];
-        return field;
-    });
-
-    let lookupOrUserFieldsInOrderBy = orderByFields.filter((orderByField) => {
-        return _isLookupOrUserField(orderByField);
-    });
-
-    let lookupOrUserFieldsToChunk = options.columns.filter((columnName) => {
-        let field = allListFieldsHash[columnName];
-        if (_isLookupOrUserField(field)) {
-            let col = firstOrNull(lookupOrUserFieldsInOrderBy, (orderByField) => {
-                return field.InternalName !== orderByField.InternalName;
-            });
-
-            return col === null;
-        }
-        return false;
-    });
-
-    let otherFieldNames = options.columns.filter((columnName) => {
-        let field = allListFieldsHash[columnName];
-        return !isNullOrUndefined(field) && !_isLookupOrUserField(field);
-    });
-
-    //The number of lookup columns in each request is based on the lookupOrUserFieldLimit.     
-    //Lookup fields in the order by statement must be sent with each request.
-    //For example, we have 20 lookup columns and 3 of them are in the order by statement.
-    //The 3 order by lookup columns are sent with each request. The remaining lookups are split into chunks.    
-    //The request will split into 3 requests
-    //The first request will have 11 lookup columns (8 standard + 3 order by lookup columns)
-    //The second request will have 11 lookup columns (8 standard + 3 order by lookup columns)
-    //The third request will have 4 lookup columns (1 standard + 3 order by lookup columns)
-    let requestChunks = chunkArray(lookupOrUserFieldsToChunk, lookupOrUserFieldLimit - lookupOrUserFieldsInOrderBy.length);
-    let otherFieldsChunkSize = Math.ceil(otherFieldNames.length / requestChunks.length);
-    let otherFieldsChunks = chunkArray(otherFieldNames, otherFieldsChunkSize);
-
-    requestChunks.forEach((chunk, index) => {
-        //Add all order by fields to each request, this will include the lookup
-        //fields from the order by statement that we left room for previously
-        requestChunks[index] = chunk.concat(orderByFields.map((orderByField) => {
-            return orderByField.InternalName;
-        }));
-
-        //Add the other fields but split them across the requests so we don't request duplicate data
-        if (otherFieldsChunks[index]) {
-            requestChunks[index] = chunk.concat(otherFieldsChunks[index]);
-        }
-    });
-
-    //requestChunks should now have about the same number of fields in each chunk and each
-    //chunk will have all the order by fields.            
-    let queries = requestChunks.map((requestChunk) => {
-        let camlQueryClone = camlQuery;
-
-        let processColumnsResult = _processColumns(requestChunk, expandFields, allListFieldsHash);
-
-        //Id field must be included in oder to merge the items correctly
-        let viewFields = processColumnsResult.viewFields;
-        if (viewFields.length && !viewFields.includes("Id")) {
-            viewFields.push("Id");
-        }
-
-        let selectFields = processColumnsResult.selectFields
-        if (selectFields.length && !selectFields.includes("Id")) {
-            selectFields.push("Id");
-        }
-
-        rtrnProcessRequestResult.needContentTypes = rtrnProcessRequestResult.needContentTypes || processColumnsResult.needContentTypes;
-
-        expandFields = processColumnsResult.expandFields;
-
-        if (isNotEmptyArray(viewFields)) {
-            camlQueryClone = EnsureViewFields(camlQueryClone, viewFields, true);
-        }
-
-        // if (isDebug()) {
-        //     let lfields = viewFields.filter((fieldName) => {
-        //         let field = allListFieldsHash[fieldName];
-        //         return _isLookupOrUserField(field);
-        //     });
-
-        //     let xmlDoc = new DOMParser().parseFromString(camlQueryClone, "text/xml");
-        //     let viewNode = xmlDoc.querySelector("View, view");
-        //     let viewFieldsNode = viewNode && viewNode.querySelector("ViewFields, viewfields");
-
-        //     let viewFields2 = Array.from(viewFieldsNode.children).map((viewFieldNode) => {
-        //         let name = viewFieldNode.getAttribute("Name") || viewFieldNode.getAttribute("name");
-        //         return name;
-        //     });
-
-        //     let lfields2 = viewFields2.filter((fieldName) => {
-        //         let field = allListFieldsHash[fieldName];
-        //         return _isLookupOrUserField(field);
-        //     });
-
-        //     if (lfields2.length !== lfields.length) {
-        //         logger.warn("Lookup fields in caml query do not match look up fields in view fields.");
-        //         logger.warn(`Lookup fields in caml query: ${lfields2}`)
-        //         logger.warn(`Lookup fields in in view fields: ${lfields}`)
-        //     }
-        // }
-
-        // Prepare the REST URL with select fields
-        let restUrlWithSelect = requestUrl;
-        if (expandFields.length > 0) {
-            restUrlWithSelect += ',' + expandFields.join(',');
-        }
-        // Include the lookup user select fields with the other select fields
-        restUrlWithSelect += '&$select=' + selectFields.join(',');
-        return { camlQueryClone, restUrlWithSelect, viewFields };
-    });
-
-    let mergedItems: IRestItem[] = null;
-    for (let i = 0; i < queries.length; i++) {
-        const { camlQueryClone, restUrlWithSelect, viewFields } = queries[i];
-
-        let loopResult = await processRequestResult(requestUrl,
-            restUrlWithSelect,
-            camlQueryClone,
-            { ...options, columns: viewFields },
-            siteUrl,
-            totalNumberOfItemsToGet,
-            batchSize,
-            orderByStatement);
-
-        if (isNullOrUndefined(mergedItems)) {
-            mergedItems = loopResult.items;
-        } else {
-            for (let restItemIndex = 0; restItemIndex < loopResult.items.length; restItemIndex++) {
-                //The item chunks (loopResult) should be in the same order as mergedItems because each request should have the 
-                //same number of items returned in the same order. The only difference between requests is which fields are present.
-                //So request 1 will have the 1st set of fields, request 2 will have the 2nd set. etc.
-                //This means that the restItemIndex should be the same as existingItemIndex for each iteration. But, we do these extra 
-                //checks just in case.
-                let restItem = loopResult.items[restItemIndex];
-                let existingItem = mergedItems[restItemIndex];
-                let existingItemIndex = restItemIndex;
-                if (isNullOrUndefined(existingItem) || existingItem.Id !== restItem.Id) {
-                    existingItemIndex = firstIndexOf(mergedItems, (mergedItem) => {
-                        return mergedItem.Id === restItem.Id;
-                    });
-                }
-
-                if (existingItemIndex === -1) {
-                    //We shouldn't get here. Each chunk should have the same items in the same order.
-                    logger.warn("_processLookupThresholdCamlRequest results are out of sync");
-                    mergedItems.push(restItem);
-                } else {
-                    let existingItem = mergedItems[existingItemIndex];
-                    let FieldValuesAsText = {
-                        ...(existingItem.FieldValuesAsText || {}),
-                        ...(restItem.FieldValuesAsText || {})
-                    };
-                    let FieldValuesForEdit = {
-                        ...(existingItem.FieldValuesForEdit || {}),
-                        ...(restItem.FieldValuesForEdit || {})
-                    };
-                    existingItem = { ...existingItem, ...restItem };
-                    existingItem.FieldValuesAsText = FieldValuesAsText;
-                    existingItem.FieldValuesForEdit = FieldValuesForEdit;
-                    mergedItems[existingItemIndex] = existingItem;
-                }
-            }
-        }
-
-        // only need to put this true if it happens once
-        if (loopResult.postProcessOrderBy) {
-            rtrnProcessRequestResult.postProcessOrderBy = true;
-        }
-    }
-
-    rtrnProcessRequestResult.items = mergedItems;
-
-    return rtrnProcessRequestResult;
-}
-
-async function _processNormalCamlRequest(orderByStatement: { Name: string; IsAscending: boolean; }[],
-    allListFieldsHash: IDictionary<IFieldInfoEX>,
-    options: ICamlOptions,
-    camlQuery: string,
-    requestUrl: string,
-    expandFields: string[],
-    siteUrl: string,
-    totalNumberOfItemsToGet: number,
-    batchSize: number) {
-    let processColumnsResult = _processColumns(options.columns, expandFields, allListFieldsHash);
-    let viewFields = processColumnsResult.viewFields;
-    let selectFields = processColumnsResult.selectFields;
-    expandFields = processColumnsResult.expandFields;
-    let needContentTypes = processColumnsResult.needContentTypes;
-
-    if (isNotEmptyArray(viewFields)) {
-        camlQuery = EnsureViewFields(camlQuery, viewFields, true);
-    }
-
-    // Prepare the REST URL with select fields
-    let restUrlWithSelect = requestUrl;
-    if (expandFields.length > 0) {
-        restUrlWithSelect += ',' + expandFields.join(',');
-    }
-    // Include the lookup user select fields with the other select fields
-    restUrlWithSelect += '&$select=' + selectFields.join(',');
-
-    // Process the request and get the result
-    let result = await processRequestResult(requestUrl, restUrlWithSelect, camlQuery, options, siteUrl, totalNumberOfItemsToGet, batchSize, orderByStatement);
-
-    return {
-        ...result,
-        needContentTypes
-    };
-}
-
-function _ensureViewFields(viewFieldsNode: Element, columns: string[], allListFieldsHash: IDictionary<IFieldInfoEX>) {
-    //ISSUE: 1565
-    //Cases
-    //1. Empty view fields element so ALL columns will be requested. We must include all list fields in 
-    //columns parsing so that we get an accurate count of how many lookup/user fields there are
-    //2. View fields element with a some field names that must also be included in the column parsing so we add them here.
-    if (isNullOrUndefined(viewFieldsNode) && isNullOrEmptyArray(columns)) {
-        Object.keys(allListFieldsHash).forEach((fieldName) => {
-            let field = allListFieldsHash[fieldName];
-            if (!isNullOrUndefined(field)
-                && !field.Hidden
-                && !SkipFields.includes(fieldName.toLowerCase())) {
-                columns.push(field.InternalName);
-            }
-        });
-    } else if (!isNullOrUndefined(viewFieldsNode)) {
-        let fieldRefNodes = Array.from(viewFieldsNode.querySelectorAll("FieldRef"));
-        if (isNotEmptyArray(fieldRefNodes)) {
-            fieldRefNodes.forEach((fieldRefNode) => {
-                let name = fieldRefNode.getAttribute("Name") || fieldRefNode.getAttribute("name");
-                if (!isNullOrEmptyString(name)) {
-                    columns.push(name);
-                }
-            });
-        }
-    }
-
-    return columns;
-}
-
-function _ensureOrderByColumns(orderByStatement: { Name: string; IsAscending: boolean; }[], columns: string[]) {
-    //Issue 548: Ensure that the order by field is in the view fields and columns collection so 
-    //that paging works correctly when there is an order by clause    
-    if (orderByStatement.length > 0 && !isNullOrEmptyString(orderByStatement[0].Name)) {
-        // just add them to columns, and we will add them to view fields below.
-        // camlQuery = EnsureViewFields(camlQuery, orderByStatement.orderBy.map(o => o.Name), false);       
-        orderByStatement.forEach(o => {
-            if (columns.indexOf(o.Name) === -1) {
-                columns.push(o.Name);
-            }
-        });
-    }
-
-    return columns;
-}
-
-function _normalizeColumns(columns: string[], allListFieldsHash: IDictionary<IFieldInfoEX>) {
-    return makeUniqueArray(columns.map((column) => {
-        //some columns will come in lower case (from dvp), normalize them so that we can remove duplicates
-        let name = Object.keys(allListFieldsHash).filter((fieldName) => {
-            return fieldName.toLowerCase() === column.toLowerCase();
-        })[0];
-        if (!isNullOrEmptyString(name)) {
-            let field = allListFieldsHash[name];
-            return field;
-        }
-        return null;
-    }).filter((field) => {
-        return !isNullOrUndefined(field);
-    }).map((field) => {
-        return field.InternalName;
-    }).concat(['Title', 'Id', 'FileLeafRef', 'FileDirRef', 'FileRef', 'FileSystemObjectType']));
-}
-
-function _removeSingleAndConditions(viewNode: Element, queryNode: Element) {
-    if (!isNullOrUndefined(viewNode.querySelector("And, and"))) {
-        try {
-            //Issue 8063: calendar list, will wrongly add a wrapping <and> statement for a single condition
-            //will result in error 500
-            //ie: <View Scope='RecursiveAll'><Query><Where><And><Eq><FieldRef Name=\"Title\" /><Value Type=\"Text\">Holiday</Value></Eq></And></Where>
-            //</Query><RowLimit>5000</RowLimit></View>
-            let whereNode = queryNode && queryNode.querySelector("Where, where");
-
-            if (!isNullOrUndefined(whereNode)) {
-                let firstCondition = whereNode.firstElementChild;
-                if (firstCondition.tagName.toLowerCase() === "and" && firstCondition.children.length < 2) {
-                    //this is the bug, <and> tag must have 2 conditions - get rid of it!
-                    whereNode.innerHTML = firstCondition.innerHTML;
-                    return {
-                        camlQuery: viewNode.outerHTML
-                    };
-                }
-            }
-        } catch (e) { }
-    }
-
-    return null;
-}
-
-function _parseRowLimitNode(rowLimitNode: Element, viewNode: Element, rowLimit: number) {
-    if (!isNullOrUndefined(rowLimitNode)) {
-        let value = rowLimitNode.textContent;
-        //if not provided by options - use it
-        if (isNullOrNaN(rowLimit) && isNumeric(value) && Number(value) > 0) {
-            rowLimit = Number(value);
-        }
-        //remove it
-        viewNode.removeChild(rowLimitNode);
-        return {
-            camlQuery: viewNode.outerHTML,
-            rowLimit: rowLimit
-        };
-    }
-    return null;
-}
-
-function _processColumns(columns: string[], expandFields: string[], allListFieldsHash: IDictionary<IFieldInfoEX>) {
-    let thresholdLimitLookupCount = 0;
-    let thresholdLimitLookupHit = false;
-    let selectFields: string[] = [];
-    let viewFields: string[] = [];
-    let needContentTypes = false;
-
-    //column parsing
-    columns.forEach(viewField => {
-        let viewFieldLower = viewField.toLowerCase();
-        if (viewFieldLower === 'contenttype' || viewFieldLower === 'contenttypeid') {
-            needContentTypes = true;
-        } else if (viewFieldLower === '_moderationstatus') {
-            selectFields.push(`FieldValuesAsText/${viewField}`);
-            viewFields.push('_moderationstatus');
-        } else if (viewFieldLower === '_moderationcomments') {
-            selectFields.push('OData_' + viewField);
-            viewFields.push('_moderationcomments');
-        } else if (viewFieldLower === "filesystemobjecttype") {
-            selectFields.push("FileSystemObjectType");
-        } else if (viewFieldLower === "fileref" || viewFieldLower === "filedirref") {
-            //treat them similar to lookup fields
-            selectFields.push(viewField);
-            selectFields.push(`FieldValuesAsText/${viewField}`);
-            viewFields.push(`${viewField}`);
-            viewFields.push(`${viewField}Id`);
-        } else {
-            //prefer to get columns not from FieldValuesAsText, unless special data type requires it (date, lookup, boolean, etc...)
-            //make the select url shorter
-            let foundField = allListFieldsHash[viewField];
-            if (foundField) {
-                let foundFieldInternalName = foundField.InternalName;
-                viewFields.push(foundFieldInternalName);
-
-                let urlField: IFieldInfoEX = null;
-
-                if (foundField.TypeAsString === "URL" && foundField.InternalName === "URL") {
-                    urlField = foundField;
-                } else if ((viewFieldLower === "urlnomenu"
-                    || viewFieldLower === "urlwmenu2"
-                    || viewFieldLower === "urlwmenu")
-                    && foundField.TypeAsString === "Computed"
-                    && !isNullOrUndefined(allListFieldsHash["URL"])
-                    && allListFieldsHash["URL"].TypeAsString === "URL") {
-                    urlField = allListFieldsHash["URL"];
-                }
-
-                if (!isNullOrUndefined(urlField) && urlField.TypeAsString === "URL") {
-                    viewFields.push(urlField.InternalName);
-                    selectFields.push(urlField.InternalName);
-                    selectFields.push(`FieldValuesAsText/${urlField.InternalName}`);
-                } else {
-                    //Issue 828, 336
-                    if (foundFieldInternalName.startsWith("_")) {
-                        foundFieldInternalName = `OData_${foundFieldInternalName}`;
-                    }
-
-                    let outputType = getFieldOutputType(foundField);
-
-                    switch (outputType) {
-                        case "Lookup":
-                        case "LookupMulti":
-                        case "User":
-                        case "UserMulti":
-                            thresholdLimitLookupCount += 1;
-                            if (thresholdLimitLookupCount >= lookupOrUserFieldLimit) {
-                                thresholdLimitLookupHit = true;
-                            }
-                            //lookup raw values comes with Id appended
-                            selectFields.push(`FieldValuesAsText/${foundFieldInternalName}`);
-                            selectFields.push(`${foundFieldInternalName}Id`);
-                            break;
-                        case "Boolean":
-                        case "Attachments":
-                        case "AllDayEvent":
-                        case "Recurrence":
-                            selectFields.push(`FieldValuesAsText/${foundFieldInternalName}`);
-                            selectFields.push(foundFieldInternalName);
-                            break;
-                        default:
-                            selectFields.push(foundFieldInternalName);
-                            break;
-                    }
-                }
-            }
-            else {
-                selectFields.push(viewField);
-            }
-        }
-    });
-
-    if (needContentTypes) {
-        expandFields.push("ContentType");
-        selectFields.push("ContentType/Name");
-        selectFields.push("ContentTypeId");
-        viewFields.push("ContentTypeId");
-    }
-
-    return {
-        thresholdLimitLookupHit,
-        viewFields: makeUniqueArray(viewFields),
-        selectFields: makeUniqueArray(selectFields),
-        expandFields: makeUniqueArray(expandFields),
-        needContentTypes
-    };
-}
-
-function _isLookupOrUserField(field: IFieldInfoEX) {
-    if (isNullOrUndefined(field)) {
-        return false;
-    }
-    let outputType = getFieldOutputType(field);
-
-    switch (outputType) {
-        case "Lookup":
-        case "LookupMulti":
-        case "User":
-        case "UserMulti":
-            return true;
-        default:
-            return false;
-    }
-}
-
-async function processRequestResult(requestUrl: string, restUrlWithSelect: string, camlQuery: string, options: ICamlOptions, siteUrl: string, totalNumberOfItemsToGet: number, batchSize: number, orderByStatement): Promise<{ items: IRestItem[]; postProcessOrderBy: boolean; }> {
-    //issue 6150: if there are too many fields, url will be too long. just get all columns without adding a $select
-    if (restUrlWithSelect.length < 2000)
-        requestUrl = restUrlWithSelect;
-
-    let query: any = { ViewXml: camlQuery.replace("</View>", `<RowLimit>${batchSize}</RowLimit></View>`) };
-
-    if (!isNullOrEmptyString(options.FolderServerRelativeUrl))
-        query.FolderServerRelativeUrl = options.FolderServerRelativeUrl;
-
-    let data = { query: query };
-
-    let items: IRestItem[] = [];
-    //let triedWithoutViewFields = false;
-    let postProcessOrderBy = false;
-    do {
-        try {
-            let requestResult = await GetJson<{ value: IRestItem[]; }>(requestUrl, JSON.stringify(data),
-                {
-                    allowCache: options.refreshCache !== true,
-                    postCacheKey: JSON.stringify(data.query),
-                    jsonMetadata: jsonTypes.nometadata,
-                    spWebUrl: siteUrl
-                });
-            if (requestResult && requestResult.value)
-                items.push(...requestResult.value);
-
-            let itemsLeftToGet = totalNumberOfItemsToGet - items.length;
-            if (itemsLeftToGet > 0 &&//we need more items
-                requestResult.value.length === batchSize//we might have more on server since we got the full batch size we asked for
-            ) {
-                let lastItem = items[items.length - 1];
-                let lastItemId = lastItem.Id;
-                let pagingInfoSort = "";
-                //Issue 7542 need to add order by value to the paging info if it is in the query
-                if (!postProcessOrderBy && orderByStatement && orderByStatement.length) {
-                    let orderFieldName = orderByStatement[0].Name;
-                    let orderFieldValue = lastItem[orderFieldName];
-
-                    //if field is ID - do not do it.
-                    if (orderFieldName.toLowerCase() !== "id" && !isNullOrUndefined(orderFieldValue)) {
-                        //if value is date - we need it in ISO and in this format: yyyyMMdd HH:mm:ss
-                        try {
-                            //Numbers cast to date properly but they are not dates. ignore number values.
-                            let orderFieldValueAsDate = isNumeric(orderFieldValue) ? null : new Date(orderFieldValue);
-                            if (isDate(orderFieldValueAsDate)) {
-                                try {
-                                    //issue 7599 date only field on different time zone...
-                                    orderFieldValue = SPServerLocalTimeToUTCSync(siteUrl, orderFieldValueAsDate).replace(/T/i, " ").replace(/Z/i, " ");
-                                } catch (e) { }
-                            }
-
-                        } catch (e) { }
-                        pagingInfoSort = `&p_${orderFieldName}=${encodeURIComponentEX(orderFieldValue)}`;
-                    }
-                }
-                data.query.ListItemCollectionPosition = {
-                    "PagingInfo": `Paged=TRUE${pagingInfoSort}&p_ID=${lastItemId}`
-                };
-                if (itemsLeftToGet < batchSize)//the last batch should be smaller, update the row limit
-                    data.query.ViewXml = camlQuery.replace("</View>", `<RowLimit>${itemsLeftToGet}</RowLimit></View>`);
-            }
-            else
-                data = null;
-        } catch (e) {
-            if (__getSPRestErrorData(e).code.indexOf('SPQueryThrottledException')) {
-                //test again - on awaiting score view, this will work but will ONLY return the order by fields...
-                // if (!triedWithoutViewFields) {
-                //     //Our own issues list had too many lookup fields.
-                //     logger.info("Query throttled, trying again without view fields... some fields might be missing from results.");
-                //     triedWithoutViewFields = true;
-                //     camlQuery = EnsureViewFields(camlQuery, orderByStatement.map(o => o.Name), false, true);
-                //     data.query.ViewXml = camlQuery.replace("</View>", `<RowLimit>${batchSize}</RowLimit></View>`);
-                // }
-                // else
-                if (!postProcessOrderBy && orderByStatement.length > 0) {
-                    logger.warn("Query throttled, trying again without order by...");
-                    postProcessOrderBy = true;
-                    camlQuery = RemoveOrderByFromCaml(camlQuery);
-                    camlQuery = EnsureViewFields(camlQuery, [], false, true);
-                    data.query.ViewXml = camlQuery.replace("</View>", `<RowLimit>${batchSize}</RowLimit></View>`);
-                }
-                else throw e;//still throttled? might be due to filter query
-            }
-            else throw e;//different error
-        }
-    } while (!isNullOrEmptyString(data));
-    return { items: items, postProcessOrderBy: postProcessOrderBy };
+import { IsLocalDev } from "../../../_dependencies";
+import { chunkArray, firstIndexOf, firstOrNull, makeUniqueArray, toHash } from "../../../helpers/collections.base";
+import { EnsureViewFields, GetOrderByFromCaml, RemoveOrderByFromCaml, getFieldOutputType } from "../../../helpers/sharepoint";
+import { isDate, isNotEmptyArray, isNullOrEmptyArray, isNullOrEmptyString, isNullOrNaN, isNullOrUndefined, isNumeric } from "../../../helpers/typecheckers";
+import { encodeURIComponentEX } from "../../../helpers/url";
+import { IDictionary } from "../../../types/common.types";
+import { jsonTypes } from "../../../types/rest.types";
+import { IFieldInfoEX } from "../../../types/sharepoint.types";
+import { GeListItemsFoldersBehaviour, IRestItem } from "../../../types/sharepoint.utils.types";
+import { ConsoleLogger } from "../../consolelogger";
+import { GetJson } from "../../rest";
+import { GetSiteUrl, __getSPRestErrorData } from "../common";
+import { GetListFields, GetListRestUrl } from "../list";
+import { SPServerLocalTimeToUTCSync } from "../web";
+import { GetItemsById } from "./GetListItemsById";
+import { SkipFields, __fixGetListItemsResults } from "./common";
+
+const logger = ConsoleLogger.get("sharepoint.rest/list/GetListItemsByCaml");
+
+interface ICamlOptions {
+    /** Optional, default: 1000. 0: get all items. */
+    rowLimit?: number;
+    /** Id, Title, Modified, FileLeafRef, FileDirRef, FileRef, FileSystemObjectType */
+    columns: string[];
+    foldersBehaviour?: GeListItemsFoldersBehaviour;
+    /** set query to get items from this folder only */
+    FolderServerRelativeUrl?: string;
+    refreshCache?: boolean;
+}
+
+//Lookup threshold limit is 12 but we use a smaller number here to ensure there are no issues
+const lookupOrUserFieldLimit = 11;
+/** returns the items or NULL, never errors out since it is used in aggregator.
+ *  set throwErrors = true if you want errors to be thrown instead of returning null.
+ * camlQuery: one of:
+ *  - view.ViewQuery
+ *  - wrapped view.ViewQuery in a <View>: "&lt;View Scope='RecursiveAll'>&lt;Query>&lt;/Query>&lt;RowLimit>1000&lt;/RowLimit>&lt;/View>" or send rowLimit in options.
+ */
+export async function GetListItemsByCaml(siteUrl: string, listIdOrTitle: string, camlQuery: string, options: ICamlOptions): Promise<IRestItem[]> {
+    siteUrl = GetSiteUrl(siteUrl);
+
+    if (!camlQuery.toLowerCase().startsWith("<view")) {
+        camlQuery = `<View Scope='RecursiveAll'><Query>${camlQuery}</Query></View>`;
+    }
+
+    let xmlParser = new DOMParser();
+    let xmlDoc = xmlParser.parseFromString(camlQuery, "text/xml");
+    let viewNode = xmlDoc.querySelector("View, view");
+    let queryNode = viewNode && viewNode.querySelector("Query, query");
+    let rowLimitNode = viewNode && viewNode.querySelector("RowLimit, rowlimit");
+    let viewFieldsNode = viewNode && viewNode.querySelector("ViewFields, viewfields");
+
+    let parseRowLimitNodeResult = _parseRowLimitNode(rowLimitNode, viewNode, options.rowLimit);
+    if (!isNullOrUndefined(parseRowLimitNodeResult)) {
+        options.rowLimit = parseRowLimitNodeResult.rowLimit;
+        camlQuery = parseRowLimitNodeResult.camlQuery;
+    }
+
+    let removeSingleAndConditionsResult = _removeSingleAndConditions(viewNode, queryNode);
+    if (!isNullOrUndefined(removeSingleAndConditionsResult)) {
+        camlQuery = removeSingleAndConditionsResult.camlQuery;
+    }
+
+    let getAllItems = isNullOrEmptyString(options.rowLimit) || options.rowLimit < 1;
+    let totalItemsInList = 99999;
+
+    let maxBatchSize = 5000;
+    let batchSize = options.rowLimit > 0 && options.rowLimit < maxBatchSize ? options.rowLimit : maxBatchSize;
+    let totalNumberOfItemsToGet = getAllItems ? totalItemsInList : options.rowLimit > maxBatchSize ? options.rowLimit : batchSize;
+
+    let requestUrl = `${GetListRestUrl(siteUrl, listIdOrTitle)}/GetItems?$expand=FieldValuesAsText`;
+
+    let allListFieldsHash = toHash(await GetListFields(siteUrl, listIdOrTitle), f => f.InternalName);
+
+    let expandFields: string[] = [];
+
+    let orderByStatement = GetOrderByFromCaml(camlQuery);
+
+    if (isNullOrUndefined(options.columns)) {
+        options.columns = [];
+    }
+
+    let columns = options.columns;
+    columns = _ensureOrderByColumns(orderByStatement, columns);
+    columns = _ensureViewFields(viewFieldsNode, columns, allListFieldsHash);
+    columns = _normalizeColumns(columns, allListFieldsHash);
+
+    if (columns.length > options.columns.length) {
+        logger.warn(`added ${columns.length - options.columns.length} to query`);
+    }
+
+    options.columns = columns;
+
+    // Store the result of processing the request
+    let rtrnProcessRequestResult: { items: IRestItem[]; postProcessOrderBy: boolean; needContentTypes: boolean };
+
+    let lookupOrUserFieldsInColumns = options.columns.filter((columnName) => {
+        let field = allListFieldsHash[columnName];
+        return _isLookupOrUserField(field);
+    });
+
+    let isThrottled = lookupOrUserFieldsInColumns.length >= lookupOrUserFieldLimit;
+
+    try {
+        if (isThrottled) {
+            //ISSUE: 1565
+            rtrnProcessRequestResult = await _processLookupThresholdCamlRequest(
+                orderByStatement,
+                allListFieldsHash,
+                options,
+                camlQuery,
+                requestUrl,
+                expandFields,
+                siteUrl,
+                totalNumberOfItemsToGet,
+                batchSize);
+        } else {
+            rtrnProcessRequestResult = await _processNormalCamlRequest(
+                orderByStatement,
+                allListFieldsHash,
+                options,
+                camlQuery,
+                requestUrl,
+                expandFields,
+                siteUrl,
+                totalNumberOfItemsToGet,
+                batchSize);
+        }
+
+        const { items, needContentTypes, postProcessOrderBy } = rtrnProcessRequestResult
+
+        let itemsResult = __fixGetListItemsResults(siteUrl, listIdOrTitle, items, options.foldersBehaviour);
+
+        let itemsWithOutContentType: number[] = [];
+        if (needContentTypes) {
+            itemsResult.forEach((item) => {
+                if (isNullOrUndefined(item["ContentType"])) {
+                    itemsWithOutContentType.push(item.Id);
+                } else if (!isNullOrUndefined(item["ContentType"].Name)) {
+                    item["ContentType"] = item["ContentType"].Name;
+                }
+            });
+        }
+
+        if (itemsWithOutContentType.length > 0) {
+            logger.time("Getting content types");
+            //Issue 1465 content types no longer come back from get items request...
+            //Make a separate request to get this info
+            let ctypes = (await GetItemsById(siteUrl, listIdOrTitle, itemsWithOutContentType, {
+                expand: ['ContentType/Name'],
+                select: ['ContentType/Name', 'Id'],
+                jsonMetadata: jsonTypes.nometadata
+            })) as any as { Id: string, ContentType: { Name: string } }[];
+
+            itemsResult.forEach(item => {
+                if (!isNullOrUndefined(ctypes[item.Id])) {
+                    item["ContentType"] = ctypes[item.Id].ContentType.Name
+                }
+            });
+            logger.timeEnd("Getting content types");
+        }
+
+        if (postProcessOrderBy) {
+            //re-apply sort
+            if (IsLocalDev) {
+                logger.table(itemsResult.map(i => {
+                    let row = {
+                        Id: i.Id,
+                        Title: i.Title
+                    };
+                    orderByStatement.forEach(s => { row[`${s.Name} ${s.IsAscending ? 'asc' : 'desc'}`] = i[s.Name]; });
+                    return row;
+                }), "before sort", true);
+            }
+            itemsResult.sort((a, b) => {
+                for (let i = 0; i < orderByStatement.length; i++) {
+                    let ob = orderByStatement[i];
+                    let v1 = a[ob.Name];
+                    let v2 = b[ob.Name];
+                    if (v1 === v2) {
+                        //these are equal - continue to second sort statement
+                    }
+                    else {
+                        if (ob.IsAscending)
+                            return (v1 > v2) ? 1 : -1;
+                        else
+                            return (v2 > v1) ? 1 : -1;
+                    }
+                }
+                return 0;
+            });
+
+            if (IsLocalDev) {
+                logger.table(itemsResult.map(i => {
+                    let row = {
+                        Id: i.Id,
+                        Title: i.Title
+                    };
+                    orderByStatement.forEach(s => { row[`${s.Name} ${s.IsAscending ? 'asc' : 'desc'}`] = i[s.Name]; });
+                    return row;
+                }), "after sort", true);
+            }
+
+        }
+
+        return itemsResult;
+    } catch (ex) {
+        console.log(`isThrottled: ${isThrottled}`);
+    }
+
+    return null;
+}
+
+// (window as any).runtTest = true;
+// window.setTimeout(() => {
+//     if ((window as any).runtTest == false) return;
+//     (window as any).runtTest = false;
+//     GetListItemsByCaml(null, "aec7756b-daa0-4da1-88ba-c66cb572e816", "", {
+//         columns: [],
+//         refreshCache: true
+//     }).then((r) => {
+
+//     });
+// }, 500);
+
+//ISSUE: 1565
+async function _processLookupThresholdCamlRequest(
+    orderByStatement: { Name: string; IsAscending: boolean; }[],
+    allListFieldsHash: IDictionary<IFieldInfoEX>,
+    options: ICamlOptions,
+    camlQuery: string,
+    requestUrl: string,
+    expandFields: string[],
+    siteUrl: string,
+    totalNumberOfItemsToGet: number,
+    batchSize: number) {
+    let rtrnProcessRequestResult = {
+        items: [],
+        postProcessOrderBy: false,
+        needContentTypes: false
+    };
+
+    let orderByFields = orderByStatement.map((orderByField) => {
+        let field = allListFieldsHash[orderByField.Name];
+        return field;
+    });
+
+    let lookupOrUserFieldsInOrderBy = orderByFields.filter((orderByField) => {
+        return _isLookupOrUserField(orderByField);
+    });
+
+    let lookupOrUserFieldsToChunk = options.columns.filter((columnName) => {
+        let field = allListFieldsHash[columnName];
+        if (_isLookupOrUserField(field)) {
+            let col = firstOrNull(lookupOrUserFieldsInOrderBy, (orderByField) => {
+                return field.InternalName !== orderByField.InternalName;
+            });
+
+            return col === null;
+        }
+        return false;
+    });
+
+    let otherFieldNames = options.columns.filter((columnName) => {
+        let field = allListFieldsHash[columnName];
+        return !isNullOrUndefined(field) && !_isLookupOrUserField(field);
+    });
+
+    //The number of lookup columns in each request is based on the lookupOrUserFieldLimit.     
+    //Lookup fields in the order by statement must be sent with each request.
+    //For example, we have 20 lookup columns and 3 of them are in the order by statement.
+    //The 3 order by lookup columns are sent with each request. The remaining lookups are split into chunks.    
+    //The request will split into 3 requests
+    //The first request will have 11 lookup columns (8 standard + 3 order by lookup columns)
+    //The second request will have 11 lookup columns (8 standard + 3 order by lookup columns)
+    //The third request will have 4 lookup columns (1 standard + 3 order by lookup columns)
+    let requestChunks = chunkArray(lookupOrUserFieldsToChunk, lookupOrUserFieldLimit - lookupOrUserFieldsInOrderBy.length);
+    let otherFieldsChunkSize = Math.ceil(otherFieldNames.length / requestChunks.length);
+    let otherFieldsChunks = chunkArray(otherFieldNames, otherFieldsChunkSize);
+
+    requestChunks.forEach((chunk, index) => {
+        //Add all order by fields to each request, this will include the lookup
+        //fields from the order by statement that we left room for previously
+        requestChunks[index] = chunk.concat(orderByFields.map((orderByField) => {
+            return orderByField.InternalName;
+        }));
+
+        //Add the other fields but split them across the requests so we don't request duplicate data
+        if (otherFieldsChunks[index]) {
+            requestChunks[index] = chunk.concat(otherFieldsChunks[index]);
+        }
+    });
+
+    //requestChunks should now have about the same number of fields in each chunk and each
+    //chunk will have all the order by fields.            
+    let queries = requestChunks.map((requestChunk) => {
+        let camlQueryClone = camlQuery;
+
+        let processColumnsResult = _processColumns(requestChunk, expandFields, allListFieldsHash);
+
+        //Id field must be included in oder to merge the items correctly
+        let viewFields = processColumnsResult.viewFields;
+        if (viewFields.length && !viewFields.includes("Id")) {
+            viewFields.push("Id");
+        }
+
+        let selectFields = processColumnsResult.selectFields
+        if (selectFields.length && !selectFields.includes("Id")) {
+            selectFields.push("Id");
+        }
+
+        rtrnProcessRequestResult.needContentTypes = rtrnProcessRequestResult.needContentTypes || processColumnsResult.needContentTypes;
+
+        expandFields = processColumnsResult.expandFields;
+
+        if (isNotEmptyArray(viewFields)) {
+            camlQueryClone = EnsureViewFields(camlQueryClone, viewFields, true);
+        }
+
+        // if (isDebug()) {
+        //     let lfields = viewFields.filter((fieldName) => {
+        //         let field = allListFieldsHash[fieldName];
+        //         return _isLookupOrUserField(field);
+        //     });
+
+        //     let xmlDoc = new DOMParser().parseFromString(camlQueryClone, "text/xml");
+        //     let viewNode = xmlDoc.querySelector("View, view");
+        //     let viewFieldsNode = viewNode && viewNode.querySelector("ViewFields, viewfields");
+
+        //     let viewFields2 = Array.from(viewFieldsNode.children).map((viewFieldNode) => {
+        //         let name = viewFieldNode.getAttribute("Name") || viewFieldNode.getAttribute("name");
+        //         return name;
+        //     });
+
+        //     let lfields2 = viewFields2.filter((fieldName) => {
+        //         let field = allListFieldsHash[fieldName];
+        //         return _isLookupOrUserField(field);
+        //     });
+
+        //     if (lfields2.length !== lfields.length) {
+        //         logger.warn("Lookup fields in caml query do not match look up fields in view fields.");
+        //         logger.warn(`Lookup fields in caml query: ${lfields2}`)
+        //         logger.warn(`Lookup fields in in view fields: ${lfields}`)
+        //     }
+        // }
+
+        // Prepare the REST URL with select fields
+        let restUrlWithSelect = requestUrl;
+        if (expandFields.length > 0) {
+            restUrlWithSelect += ',' + expandFields.join(',');
+        }
+        // Include the lookup user select fields with the other select fields
+        restUrlWithSelect += '&$select=' + selectFields.join(',');
+        return { camlQueryClone, restUrlWithSelect, viewFields };
+    });
+
+    let mergedItems: IRestItem[] = null;
+    for (let i = 0; i < queries.length; i++) {
+        const { camlQueryClone, restUrlWithSelect, viewFields } = queries[i];
+
+        let loopResult = await processRequestResult(requestUrl,
+            restUrlWithSelect,
+            camlQueryClone,
+            { ...options, columns: viewFields },
+            siteUrl,
+            totalNumberOfItemsToGet,
+            batchSize,
+            orderByStatement);
+
+        if (isNullOrUndefined(mergedItems)) {
+            mergedItems = loopResult.items;
+        } else {
+            for (let restItemIndex = 0; restItemIndex < loopResult.items.length; restItemIndex++) {
+                //The item chunks (loopResult) should be in the same order as mergedItems because each request should have the 
+                //same number of items returned in the same order. The only difference between requests is which fields are present.
+                //So request 1 will have the 1st set of fields, request 2 will have the 2nd set. etc.
+                //This means that the restItemIndex should be the same as existingItemIndex for each iteration. But, we do these extra 
+                //checks just in case.
+                let restItem = loopResult.items[restItemIndex];
+                let existingItem = mergedItems[restItemIndex];
+                let existingItemIndex = restItemIndex;
+                if (isNullOrUndefined(existingItem) || existingItem.Id !== restItem.Id) {
+                    existingItemIndex = firstIndexOf(mergedItems, (mergedItem) => {
+                        return mergedItem.Id === restItem.Id;
+                    });
+                }
+
+                if (existingItemIndex === -1) {
+                    //We shouldn't get here. Each chunk should have the same items in the same order.
+                    logger.warn("_processLookupThresholdCamlRequest results are out of sync");
+                    mergedItems.push(restItem);
+                } else {
+                    let existingItem = mergedItems[existingItemIndex];
+                    let FieldValuesAsText = {
+                        ...(existingItem.FieldValuesAsText || {}),
+                        ...(restItem.FieldValuesAsText || {})
+                    };
+                    let FieldValuesForEdit = {
+                        ...(existingItem.FieldValuesForEdit || {}),
+                        ...(restItem.FieldValuesForEdit || {})
+                    };
+                    existingItem = { ...existingItem, ...restItem };
+                    existingItem.FieldValuesAsText = FieldValuesAsText;
+                    existingItem.FieldValuesForEdit = FieldValuesForEdit;
+                    mergedItems[existingItemIndex] = existingItem;
+                }
+            }
+        }
+
+        // only need to put this true if it happens once
+        if (loopResult.postProcessOrderBy) {
+            rtrnProcessRequestResult.postProcessOrderBy = true;
+        }
+    }
+
+    rtrnProcessRequestResult.items = mergedItems;
+
+    return rtrnProcessRequestResult;
+}
+
+async function _processNormalCamlRequest(orderByStatement: { Name: string; IsAscending: boolean; }[],
+    allListFieldsHash: IDictionary<IFieldInfoEX>,
+    options: ICamlOptions,
+    camlQuery: string,
+    requestUrl: string,
+    expandFields: string[],
+    siteUrl: string,
+    totalNumberOfItemsToGet: number,
+    batchSize: number) {
+    let processColumnsResult = _processColumns(options.columns, expandFields, allListFieldsHash);
+    let viewFields = processColumnsResult.viewFields;
+    let selectFields = processColumnsResult.selectFields;
+    expandFields = processColumnsResult.expandFields;
+    let needContentTypes = processColumnsResult.needContentTypes;
+
+    if (isNotEmptyArray(viewFields)) {
+        camlQuery = EnsureViewFields(camlQuery, viewFields, true);
+    }
+
+    // Prepare the REST URL with select fields
+    let restUrlWithSelect = requestUrl;
+    if (expandFields.length > 0) {
+        restUrlWithSelect += ',' + expandFields.join(',');
+    }
+    // Include the lookup user select fields with the other select fields
+    restUrlWithSelect += '&$select=' + selectFields.join(',');
+
+    // Process the request and get the result
+    let result = await processRequestResult(requestUrl, restUrlWithSelect, camlQuery, options, siteUrl, totalNumberOfItemsToGet, batchSize, orderByStatement);
+
+    return {
+        ...result,
+        needContentTypes
+    };
+}
+
+function _ensureViewFields(viewFieldsNode: Element, columns: string[], allListFieldsHash: IDictionary<IFieldInfoEX>) {
+    //ISSUE: 1565
+    //Cases
+    //1. Empty view fields element so ALL columns will be requested. We must include all list fields in 
+    //columns parsing so that we get an accurate count of how many lookup/user fields there are
+    //2. View fields element with a some field names that must also be included in the column parsing so we add them here.
+    if (isNullOrUndefined(viewFieldsNode) && isNullOrEmptyArray(columns)) {
+        Object.keys(allListFieldsHash).forEach((fieldName) => {
+            let field = allListFieldsHash[fieldName];
+            if (!isNullOrUndefined(field)
+                && !field.Hidden
+                && !SkipFields.includes(fieldName.toLowerCase())) {
+                columns.push(field.InternalName);
+            }
+        });
+    } else if (!isNullOrUndefined(viewFieldsNode)) {
+        let fieldRefNodes = Array.from(viewFieldsNode.querySelectorAll("FieldRef"));
+        if (isNotEmptyArray(fieldRefNodes)) {
+            fieldRefNodes.forEach((fieldRefNode) => {
+                let name = fieldRefNode.getAttribute("Name") || fieldRefNode.getAttribute("name");
+                if (!isNullOrEmptyString(name)) {
+                    columns.push(name);
+                }
+            });
+        }
+    }
+
+    return columns;
+}
+
+function _ensureOrderByColumns(orderByStatement: { Name: string; IsAscending: boolean; }[], columns: string[]) {
+    //Issue 548: Ensure that the order by field is in the view fields and columns collection so 
+    //that paging works correctly when there is an order by clause    
+    if (orderByStatement.length > 0 && !isNullOrEmptyString(orderByStatement[0].Name)) {
+        // just add them to columns, and we will add them to view fields below.
+        // camlQuery = EnsureViewFields(camlQuery, orderByStatement.orderBy.map(o => o.Name), false);       
+        orderByStatement.forEach(o => {
+            if (columns.indexOf(o.Name) === -1) {
+                columns.push(o.Name);
+            }
+        });
+    }
+
+    return columns;
+}
+
+function _normalizeColumns(columns: string[], allListFieldsHash: IDictionary<IFieldInfoEX>) {
+    return makeUniqueArray(columns.map((column) => {
+        //some columns will come in lower case (from dvp), normalize them so that we can remove duplicates
+        let name = Object.keys(allListFieldsHash).filter((fieldName) => {
+            return fieldName.toLowerCase() === column.toLowerCase();
+        })[0];
+        if (!isNullOrEmptyString(name)) {
+            let field = allListFieldsHash[name];
+            return field;
+        }
+        return null;
+    }).filter((field) => {
+        return !isNullOrUndefined(field);
+    }).map((field) => {
+        return field.InternalName;
+    }).concat(['Title', 'Id', 'FileLeafRef', 'FileDirRef', 'FileRef', 'FileSystemObjectType']));
+}
+
+function _removeSingleAndConditions(viewNode: Element, queryNode: Element) {
+    if (!isNullOrUndefined(viewNode.querySelector("And, and"))) {
+        try {
+            //Issue 8063: calendar list, will wrongly add a wrapping <and> statement for a single condition
+            //will result in error 500
+            //ie: <View Scope='RecursiveAll'><Query><Where><And><Eq><FieldRef Name=\"Title\" /><Value Type=\"Text\">Holiday</Value></Eq></And></Where>
+            //</Query><RowLimit>5000</RowLimit></View>
+            let whereNode = queryNode && queryNode.querySelector("Where, where");
+
+            if (!isNullOrUndefined(whereNode)) {
+                let firstCondition = whereNode.firstElementChild;
+                if (firstCondition.tagName.toLowerCase() === "and" && firstCondition.children.length < 2) {
+                    //this is the bug, <and> tag must have 2 conditions - get rid of it!
+                    whereNode.innerHTML = firstCondition.innerHTML;
+                    return {
+                        camlQuery: viewNode.outerHTML
+                    };
+                }
+            }
+        } catch (e) { }
+    }
+
+    return null;
+}
+
+function _parseRowLimitNode(rowLimitNode: Element, viewNode: Element, rowLimit: number) {
+    if (!isNullOrUndefined(rowLimitNode)) {
+        let value = rowLimitNode.textContent;
+        //if not provided by options - use it
+        if (isNullOrNaN(rowLimit) && isNumeric(value) && Number(value) > 0) {
+            rowLimit = Number(value);
+        }
+        //remove it
+        viewNode.removeChild(rowLimitNode);
+        return {
+            camlQuery: viewNode.outerHTML,
+            rowLimit: rowLimit
+        };
+    }
+    return null;
+}
+
+function _processColumns(columns: string[], expandFields: string[], allListFieldsHash: IDictionary<IFieldInfoEX>) {
+    let thresholdLimitLookupCount = 0;
+    let thresholdLimitLookupHit = false;
+    let selectFields: string[] = [];
+    let viewFields: string[] = [];
+    let needContentTypes = false;
+
+    //column parsing
+    columns.forEach(viewField => {
+        let viewFieldLower = viewField.toLowerCase();
+        if (viewFieldLower === 'contenttype' || viewFieldLower === 'contenttypeid') {
+            needContentTypes = true;
+        } else if (viewFieldLower === '_moderationstatus') {
+            selectFields.push(`FieldValuesAsText/${viewField}`);
+            viewFields.push('_moderationstatus');
+        } else if (viewFieldLower === '_moderationcomments') {
+            selectFields.push('OData_' + viewField);
+            viewFields.push('_moderationcomments');
+        } else if (viewFieldLower === "filesystemobjecttype") {
+            selectFields.push("FileSystemObjectType");
+        } else if (viewFieldLower === "fileref" || viewFieldLower === "filedirref") {
+            //treat them similar to lookup fields
+            selectFields.push(viewField);
+            selectFields.push(`FieldValuesAsText/${viewField}`);
+            viewFields.push(`${viewField}`);
+            viewFields.push(`${viewField}Id`);
+        } else {
+            //prefer to get columns not from FieldValuesAsText, unless special data type requires it (date, lookup, boolean, etc...)
+            //make the select url shorter
+            let foundField = allListFieldsHash[viewField];
+            if (foundField) {
+                let foundFieldInternalName = foundField.InternalName;
+                viewFields.push(foundFieldInternalName);
+
+                let urlField: IFieldInfoEX = null;
+
+                if (foundField.TypeAsString === "URL" && foundField.InternalName === "URL") {
+                    urlField = foundField;
+                } else if ((viewFieldLower === "urlnomenu"
+                    || viewFieldLower === "urlwmenu2"
+                    || viewFieldLower === "urlwmenu")
+                    && foundField.TypeAsString === "Computed"
+                    && !isNullOrUndefined(allListFieldsHash["URL"])
+                    && allListFieldsHash["URL"].TypeAsString === "URL") {
+                    urlField = allListFieldsHash["URL"];
+                }
+
+                if (!isNullOrUndefined(urlField) && urlField.TypeAsString === "URL") {
+                    viewFields.push(urlField.InternalName);
+                    selectFields.push(urlField.InternalName);
+                    selectFields.push(`FieldValuesAsText/${urlField.InternalName}`);
+                } else {
+                    //Issue 828, 336
+                    if (foundFieldInternalName.startsWith("_")) {
+                        foundFieldInternalName = `OData_${foundFieldInternalName}`;
+                    }
+
+                    let outputType = getFieldOutputType(foundField);
+
+                    switch (outputType) {
+                        case "Lookup":
+                        case "LookupMulti":
+                        case "User":
+                        case "UserMulti":
+                            thresholdLimitLookupCount += 1;
+                            if (thresholdLimitLookupCount >= lookupOrUserFieldLimit) {
+                                thresholdLimitLookupHit = true;
+                            }
+                            //lookup raw values comes with Id appended
+                            selectFields.push(`FieldValuesAsText/${foundFieldInternalName}`);
+                            selectFields.push(`${foundFieldInternalName}Id`);
+                            break;
+                        case "Boolean":
+                        case "Attachments":
+                        case "AllDayEvent":
+                        case "Recurrence":
+                            selectFields.push(`FieldValuesAsText/${foundFieldInternalName}`);
+                            selectFields.push(foundFieldInternalName);
+                            break;
+                        default:
+                            selectFields.push(foundFieldInternalName);
+                            break;
+                    }
+                }
+            }
+            else {
+                selectFields.push(viewField);
+            }
+        }
+    });
+
+    if (needContentTypes) {
+        expandFields.push("ContentType");
+        selectFields.push("ContentType/Name");
+        selectFields.push("ContentTypeId");
+        viewFields.push("ContentTypeId");
+    }
+
+    return {
+        thresholdLimitLookupHit,
+        viewFields: makeUniqueArray(viewFields),
+        selectFields: makeUniqueArray(selectFields),
+        expandFields: makeUniqueArray(expandFields),
+        needContentTypes
+    };
+}
+
+function _isLookupOrUserField(field: IFieldInfoEX) {
+    if (isNullOrUndefined(field)) {
+        return false;
+    }
+    let outputType = getFieldOutputType(field);
+
+    switch (outputType) {
+        case "Lookup":
+        case "LookupMulti":
+        case "User":
+        case "UserMulti":
+            return true;
+        default:
+            return false;
+    }
+}
+
+async function processRequestResult(requestUrl: string, restUrlWithSelect: string, camlQuery: string, options: ICamlOptions, siteUrl: string, totalNumberOfItemsToGet: number, batchSize: number, orderByStatement): Promise<{ items: IRestItem[]; postProcessOrderBy: boolean; }> {
+    //issue 6150: if there are too many fields, url will be too long. just get all columns without adding a $select
+    if (restUrlWithSelect.length < 2000)
+        requestUrl = restUrlWithSelect;
+
+    let query: any = { ViewXml: camlQuery.replace("</View>", `<RowLimit>${batchSize}</RowLimit></View>`) };
+
+    if (!isNullOrEmptyString(options.FolderServerRelativeUrl))
+        query.FolderServerRelativeUrl = options.FolderServerRelativeUrl;
+
+    let data = { query: query };
+
+    let items: IRestItem[] = [];
+    //let triedWithoutViewFields = false;
+    let postProcessOrderBy = false;
+    do {
+        try {
+            let requestResult = await GetJson<{ value: IRestItem[]; }>(requestUrl, JSON.stringify(data),
+                {
+                    allowCache: options.refreshCache !== true,
+                    postCacheKey: JSON.stringify(data.query),
+                    jsonMetadata: jsonTypes.nometadata,
+                    spWebUrl: siteUrl
+                });
+            if (requestResult && requestResult.value)
+                items.push(...requestResult.value);
+
+            let itemsLeftToGet = totalNumberOfItemsToGet - items.length;
+            if (itemsLeftToGet > 0 &&//we need more items
+                requestResult.value.length === batchSize//we might have more on server since we got the full batch size we asked for
+            ) {
+                let lastItem = items[items.length - 1];
+                let lastItemId = lastItem.Id;
+                let pagingInfoSort = "";
+                //Issue 7542 need to add order by value to the paging info if it is in the query
+                if (!postProcessOrderBy && orderByStatement && orderByStatement.length) {
+                    let orderFieldName = orderByStatement[0].Name;
+                    let orderFieldValue = lastItem[orderFieldName];
+
+                    //if field is ID - do not do it.
+                    if (orderFieldName.toLowerCase() !== "id" && !isNullOrUndefined(orderFieldValue)) {
+                        //if value is date - we need it in ISO and in this format: yyyyMMdd HH:mm:ss
+                        try {
+                            //Numbers cast to date properly but they are not dates. ignore number values.
+                            let orderFieldValueAsDate = isNumeric(orderFieldValue) ? null : new Date(orderFieldValue);
+                            if (isDate(orderFieldValueAsDate)) {
+                                try {
+                                    //issue 7599 date only field on different time zone...
+                                    orderFieldValue = SPServerLocalTimeToUTCSync(siteUrl, orderFieldValueAsDate).replace(/T/i, " ").replace(/Z/i, " ");
+                                } catch (e) { }
+                            }
+
+                        } catch (e) { }
+                        pagingInfoSort = `&p_${orderFieldName}=${encodeURIComponentEX(orderFieldValue)}`;
+                    }
+                }
+                data.query.ListItemCollectionPosition = {
+                    "PagingInfo": `Paged=TRUE${pagingInfoSort}&p_ID=${lastItemId}`
+                };
+                if (itemsLeftToGet < batchSize)//the last batch should be smaller, update the row limit
+                    data.query.ViewXml = camlQuery.replace("</View>", `<RowLimit>${itemsLeftToGet}</RowLimit></View>`);
+            }
+            else
+                data = null;
+        } catch (e) {
+            if (__getSPRestErrorData(e).code.indexOf('SPQueryThrottledException')) {
+                //test again - on awaiting score view, this will work but will ONLY return the order by fields...
+                // if (!triedWithoutViewFields) {
+                //     //Our own issues list had too many lookup fields.
+                //     logger.info("Query throttled, trying again without view fields... some fields might be missing from results.");
+                //     triedWithoutViewFields = true;
+                //     camlQuery = EnsureViewFields(camlQuery, orderByStatement.map(o => o.Name), false, true);
+                //     data.query.ViewXml = camlQuery.replace("</View>", `<RowLimit>${batchSize}</RowLimit></View>`);
+                // }
+                // else
+                if (!postProcessOrderBy && orderByStatement.length > 0) {
+                    logger.warn("Query throttled, trying again without order by...");
+                    postProcessOrderBy = true;
+                    camlQuery = RemoveOrderByFromCaml(camlQuery);
+                    camlQuery = EnsureViewFields(camlQuery, [], false, true);
+                    data.query.ViewXml = camlQuery.replace("</View>", `<RowLimit>${batchSize}</RowLimit></View>`);
+                }
+                else throw e;//still throttled? might be due to filter query
+            }
+            else throw e;//different error
+        }
+    } while (!isNullOrEmptyString(data));
+    return { items: items, postProcessOrderBy: postProcessOrderBy };
 }