@kwiz/common 1.0.1

package/src/utils/sharepoint.rest/listutils/GetListItemsByCaml.ts

@@ -0,0 +1,750 @@
+import { EnsureViewFields, GeListItemsFoldersBehaviour, GetOrderByFromCaml, IDictionary, IFieldInfoEX, IRestItem, IsLocalDev, RemoveOrderByFromCaml, chunkArray, encodeURIComponentEX, firstIndexOf, firstOrNull, getFieldOutputType, isDate, isNotEmptyArray, isNullOrEmptyArray, isNullOrEmptyString, isNullOrNaN, isNullOrUndefined, isNumeric, jsonTypes, makeUniqueArray, toHash } from "../../_dependencies";
+import { ConsoleLogger } from "../../consolelogger";
+import { GetJson } from "../../rest";
+import { GetSiteUrl, __getSPRestErrorData } from "../common";
+import { GetListFields, GetListRestUrl } from "../list";
+import { SPServerLocalTimeToUTCSync } from "../web";
+import { GetItemsById } from "./GetListItemsById";
+import { SkipFields, __fixGetListItemsResults } from "./common";
+
+const logger = ConsoleLogger.get("sharepoint.rest/list/GetListItemsByCaml");
+
+interface ICamlOptions {
+    /** Optional, default: 1000. 0: get all items. */
+    rowLimit?: number;
+    /** Id, Title, Modified, FileLeafRef, FileDirRef, FileRef, FileSystemObjectType */
+    columns: string[];
+    foldersBehaviour?: GeListItemsFoldersBehaviour;
+    /** set query to get items from this folder only */
+    FolderServerRelativeUrl?: string;
+    refreshCache?: boolean;
+}
+
+//Lookup threshold limit is 12 but we use a smaller number here to ensure there are no issues
+const lookupOrUserFieldLimit = 11;
+/** returns the items or NULL, never errors out since it is used in aggregator.
+ *  set throwErrors = true if you want errors to be thrown instead of returning null.
+ * camlQuery: one of:
+ *  - view.ViewQuery
+ *  - wrapped view.ViewQuery in a <View>: "&lt;View Scope='RecursiveAll'>&lt;Query>&lt;/Query>&lt;RowLimit>1000&lt;/RowLimit>&lt;/View>" or send rowLimit in options.
+ */
+export async function GetListItemsByCaml(siteUrl: string, listIdOrTitle: string, camlQuery: string, options: ICamlOptions): Promise<IRestItem[]> {
+    siteUrl = GetSiteUrl(siteUrl);
+
+    if (!camlQuery.toLowerCase().startsWith("<view")) {
+        camlQuery = `<View Scope='RecursiveAll'><Query>${camlQuery}</Query></View>`;
+    }
+
+    let xmlParser = new DOMParser();
+    let xmlDoc = xmlParser.parseFromString(camlQuery, "text/xml");
+    let viewNode = xmlDoc.querySelector("View, view");
+    let queryNode = viewNode && viewNode.querySelector("Query, query");
+    let rowLimitNode = viewNode && viewNode.querySelector("RowLimit, rowlimit");
+    let viewFieldsNode = viewNode && viewNode.querySelector("ViewFields, viewfields");
+
+    let parseRowLimitNodeResult = _parseRowLimitNode(rowLimitNode, viewNode, options.rowLimit);
+    if (!isNullOrUndefined(parseRowLimitNodeResult)) {
+        options.rowLimit = parseRowLimitNodeResult.rowLimit;
+        camlQuery = parseRowLimitNodeResult.camlQuery;
+    }
+
+    let removeSingleAndConditionsResult = _removeSingleAndConditions(viewNode, queryNode);
+    if (!isNullOrUndefined(removeSingleAndConditionsResult)) {
+        camlQuery = removeSingleAndConditionsResult.camlQuery;
+    }
+
+    let getAllItems = isNullOrEmptyString(options.rowLimit) || options.rowLimit < 1;
+    let totalItemsInList = 99999;
+
+    let maxBatchSize = 5000;
+    let batchSize = options.rowLimit > 0 && options.rowLimit < maxBatchSize ? options.rowLimit : maxBatchSize;
+    let totalNumberOfItemsToGet = getAllItems ? totalItemsInList : options.rowLimit > maxBatchSize ? options.rowLimit : batchSize;
+
+    let requestUrl = `${GetListRestUrl(siteUrl, listIdOrTitle)}/GetItems?$expand=FieldValuesAsText`;
+
+    let allListFieldsHash = toHash(await GetListFields(siteUrl, listIdOrTitle), f => f.InternalName);
+
+    let expandFields: string[] = [];
+
+    let orderByStatement = GetOrderByFromCaml(camlQuery);
+
+    if (isNullOrUndefined(options.columns)) {
+        options.columns = [];
+    }
+
+    let columns = options.columns;
+    columns = _ensureOrderByColumns(orderByStatement, columns);
+    columns = _ensureViewFields(viewFieldsNode, columns, allListFieldsHash);
+    columns = _normalizeColumns(columns, allListFieldsHash);
+
+    if (columns.length > options.columns.length) {
+        logger.warn(`added ${columns.length - options.columns.length} to query`);
+    }
+
+    options.columns = columns;
+
+    // Store the result of processing the request
+    let rtrnProcessRequestResult: { items: IRestItem[]; postProcessOrderBy: boolean; needContentTypes: boolean };
+
+    let lookupOrUserFieldsInColumns = options.columns.filter((columnName) => {
+        let field = allListFieldsHash[columnName];
+        return _isLookupOrUserField(field);
+    });
+
+    let isThrottled = lookupOrUserFieldsInColumns.length >= lookupOrUserFieldLimit;
+
+    try {
+        if (isThrottled) {
+            //ISSUE: 1565
+            rtrnProcessRequestResult = await _processLookupThresholdCamlRequest(
+                orderByStatement,
+                allListFieldsHash,
+                options,
+                camlQuery,
+                requestUrl,
+                expandFields,
+                siteUrl,
+                totalNumberOfItemsToGet,
+                batchSize);
+        } else {
+            rtrnProcessRequestResult = await _processNormalCamlRequest(
+                orderByStatement,
+                allListFieldsHash,
+                options,
+                camlQuery,
+                requestUrl,
+                expandFields,
+                siteUrl,
+                totalNumberOfItemsToGet,
+                batchSize);
+        }
+
+        const { items, needContentTypes, postProcessOrderBy } = rtrnProcessRequestResult
+
+        let itemsResult = __fixGetListItemsResults(siteUrl, listIdOrTitle, items, options.foldersBehaviour);
+
+        let itemsWithOutContentType: number[] = [];
+        if (needContentTypes) {
+            itemsResult.forEach((item) => {
+                if (isNullOrUndefined(item["ContentType"])) {
+                    itemsWithOutContentType.push(item.Id);
+                } else if (!isNullOrUndefined(item["ContentType"].Name)) {
+                    item["ContentType"] = item["ContentType"].Name;
+                }
+            });
+        }
+
+        if (itemsWithOutContentType.length > 0) {
+            logger.time("Getting content types");
+            //Issue 1465 content types no longer come back from get items request...
+            //Make a separate request to get this info
+            let ctypes = (await GetItemsById(siteUrl, listIdOrTitle, itemsWithOutContentType, {
+                expand: ['ContentType/Name'],
+                select: ['ContentType/Name', 'Id'],
+                jsonMetadata: jsonTypes.nometadata
+            })) as any as { Id: string, ContentType: { Name: string } }[];
+
+            itemsResult.forEach(item => {
+                if (!isNullOrUndefined(ctypes[item.Id])) {
+                    item["ContentType"] = ctypes[item.Id].ContentType.Name
+                }
+            });
+            logger.timeEnd("Getting content types");
+        }
+
+        if (postProcessOrderBy) {
+            //re-apply sort
+            if (IsLocalDev) {
+                logger.table(itemsResult.map(i => {
+                    let row = {
+                        Id: i.Id,
+                        Title: i.Title
+                    };
+                    orderByStatement.forEach(s => { row[`${s.Name} ${s.IsAscending ? 'asc' : 'desc'}`] = i[s.Name]; });
+                    return row;
+                }), "before sort", true);
+            }
+            itemsResult.sort((a, b) => {
+                for (let i = 0; i < orderByStatement.length; i++) {
+                    let ob = orderByStatement[i];
+                    let v1 = a[ob.Name];
+                    let v2 = b[ob.Name];
+                    if (v1 === v2) {
+                        //these are equal - continue to second sort statement
+                    }
+                    else {
+                        if (ob.IsAscending)
+                            return (v1 > v2) ? 1 : -1;
+                        else
+                            return (v2 > v1) ? 1 : -1;
+                    }
+                }
+                return 0;
+            });
+
+            if (IsLocalDev) {
+                logger.table(itemsResult.map(i => {
+                    let row = {
+                        Id: i.Id,
+                        Title: i.Title
+                    };
+                    orderByStatement.forEach(s => { row[`${s.Name} ${s.IsAscending ? 'asc' : 'desc'}`] = i[s.Name]; });
+                    return row;
+                }), "after sort", true);
+            }
+
+        }
+
+        return itemsResult;
+    } catch (ex) {
+        console.log(`isThrottled: ${isThrottled}`);
+    }
+
+    return null;
+}
+
+// (window as any).runtTest = true;
+// window.setTimeout(() => {
+//     if ((window as any).runtTest == false) return;
+//     (window as any).runtTest = false;
+//     GetListItemsByCaml(null, "aec7756b-daa0-4da1-88ba-c66cb572e816", "", {
+//         columns: [],
+//         refreshCache: true
+//     }).then((r) => {
+
+//     });
+// }, 500);
+
+//ISSUE: 1565
+async function _processLookupThresholdCamlRequest(
+    orderByStatement: { Name: string; IsAscending: boolean; }[],
+    allListFieldsHash: IDictionary<IFieldInfoEX>,
+    options: ICamlOptions,
+    camlQuery: string,
+    requestUrl: string,
+    expandFields: string[],
+    siteUrl: string,
+    totalNumberOfItemsToGet: number,
+    batchSize: number) {
+    let rtrnProcessRequestResult = {
+        items: [],
+        postProcessOrderBy: false,
+        needContentTypes: false
+    };
+
+    let orderByFields = orderByStatement.map((orderByField) => {
+        let field = allListFieldsHash[orderByField.Name];
+        return field;
+    });
+
+    let lookupOrUserFieldsInOrderBy = orderByFields.filter((orderByField) => {
+        return _isLookupOrUserField(orderByField);
+    });
+
+    let lookupOrUserFieldsToChunk = options.columns.filter((columnName) => {
+        let field = allListFieldsHash[columnName];
+        if (_isLookupOrUserField(field)) {
+            let col = firstOrNull(lookupOrUserFieldsInOrderBy, (orderByField) => {
+                return field.InternalName !== orderByField.InternalName;
+            });
+
+            return col === null;
+        }
+        return false;
+    });
+
+    let otherFieldNames = options.columns.filter((columnName) => {
+        let field = allListFieldsHash[columnName];
+        let exists = !isNullOrUndefined(field);
+        let isLookupField = exists && _isLookupOrUserField(field);
+        let isInOrderBy = exists && firstOrNull(orderByFields, (orderByField) => orderByField.InternalName === field.InternalName) !== null
+        return exists && !isLookupField && !isInOrderBy;
+    });
+
+    //Split the requests into equally chunked arrays based on the lookup field count
+    //add lookup fields in chunk but leave room to add lookup fields from the order by statement 
+    //since they are mandatory    
+    let requestChunks = chunkArray(lookupOrUserFieldsToChunk, lookupOrUserFieldLimit - lookupOrUserFieldsInOrderBy.length);
+    let otherFieldsChunkSize = Math.ceil(otherFieldNames.length / requestChunks.length);
+    let otherFieldsChunks = chunkArray(otherFieldNames, otherFieldsChunkSize);
+
+    requestChunks.forEach((chunk, index) => {
+        //Add all order by fields to each request, this will include the lookup
+        //fields from the order by statement that we left room for previously
+        requestChunks[index] = chunk.concat(orderByFields.map((orderByField) => {
+            return orderByField.InternalName;
+        }));
+
+        //Add the other fields but split them across the requests so we don't request duplicate data
+        if (otherFieldsChunks[index]) {
+            requestChunks[index] = chunk.concat(otherFieldsChunks[index]);
+        }
+    });
+
+    //requestChunks should now have about the same number of fields in each chunk and each
+    //chunk will have all the order by fields.            
+    let queries = requestChunks.map((requestChunk) => {
+        let camlQueryClone = camlQuery;
+
+        let processColumnsResult = _processColumns(requestChunk, expandFields, allListFieldsHash);
+
+        //Id field must be included in oder to merge the items correctly
+        let viewFields = processColumnsResult.viewFields;
+        if (viewFields.length && !viewFields.includes("Id")) {
+            viewFields.push("Id");
+        }
+
+        let selectFields = processColumnsResult.selectFields
+        if (selectFields.length && !selectFields.includes("Id")) {
+            selectFields.push("Id");
+        }
+
+        rtrnProcessRequestResult.needContentTypes = rtrnProcessRequestResult.needContentTypes || processColumnsResult.needContentTypes;
+
+        expandFields = processColumnsResult.expandFields;
+
+        if (isNotEmptyArray(viewFields)) {
+            camlQueryClone = EnsureViewFields(camlQueryClone, viewFields, true);
+        }
+
+        // if (isDebug()) {
+        //     let lfields = viewFields.filter((fieldName) => {
+        //         let field = allListFieldsHash[fieldName];
+        //         return _isLookupOrUserField(field);
+        //     });
+
+        //     let xmlDoc = new DOMParser().parseFromString(camlQueryClone, "text/xml");
+        //     let viewNode = xmlDoc.querySelector("View, view");
+        //     let viewFieldsNode = viewNode && viewNode.querySelector("ViewFields, viewfields");
+
+        //     let viewFields2 = Array.from(viewFieldsNode.children).map((viewFieldNode) => {
+        //         let name = viewFieldNode.getAttribute("Name") || viewFieldNode.getAttribute("name");
+        //         return name;
+        //     });
+
+        //     let lfields2 = viewFields2.filter((fieldName) => {
+        //         let field = allListFieldsHash[fieldName];
+        //         return _isLookupOrUserField(field);
+        //     });
+
+        //     if (lfields2.length !== lfields.length) {
+        //         logger.warn("Lookup fields in caml query do not match look up fields in view fields.");
+        //         logger.warn(`Lookup fields in caml query: ${lfields2}`)
+        //         logger.warn(`Lookup fields in in view fields: ${lfields}`)
+        //     }
+        // }
+
+        // Prepare the REST URL with select fields
+        let restUrlWithSelect = requestUrl;
+        if (expandFields.length > 0) {
+            restUrlWithSelect += ',' + expandFields.join(',');
+        }
+        // Include the lookup user select fields with the other select fields
+        restUrlWithSelect += '&$select=' + selectFields.join(',');
+        return { camlQueryClone, restUrlWithSelect, viewFields };
+    });
+
+    let mergedItems: IRestItem[] = null;
+    for (let i = 0; i < queries.length; i++) {
+        const { camlQueryClone, restUrlWithSelect, viewFields } = queries[i];
+
+        let loopResult = await processRequestResult(requestUrl,
+            restUrlWithSelect,
+            camlQueryClone,
+            { ...options, columns: viewFields },
+            siteUrl,
+            totalNumberOfItemsToGet,
+            batchSize,
+            orderByStatement);
+
+        if (isNullOrUndefined(mergedItems)) {
+            mergedItems = loopResult.items;
+        } else {
+            for (let restItemIndex = 0; restItemIndex < loopResult.items.length; restItemIndex++) {
+                //The item chunks (loopResult) should be in the same order as mergedItems because each request should have the 
+                //same number of items returned in the same order. The only difference between requests is which fields are present.
+                //So request 1 will have the 1st set of fields, request 2 will have the 2nd set. etc.
+                //This means that the restItemIndex should be the same as existingItemIndex for each iteration. But, we do these extra 
+                //checks just in case.
+                let restItem = loopResult.items[restItemIndex];
+                let existingItem = mergedItems[restItemIndex];
+                let existingItemIndex = restItemIndex;
+                if (isNullOrUndefined(existingItem) || existingItem.Id !== restItem.Id) {
+                    existingItemIndex = firstIndexOf(mergedItems, (mergedItem) => {
+                        return mergedItem.Id === restItem.Id;
+                    });
+                }
+
+                if (existingItemIndex === -1) {
+                    //We shouldn't get here. Each chunk should have the same items in the same order.
+                    logger.warn("_processLookupThresholdCamlRequest results are out of sync");
+                    mergedItems.push(restItem);
+                } else {
+                    let existingItem = mergedItems[existingItemIndex];
+                    let FieldValuesAsText = {
+                        ...(existingItem.FieldValuesAsText || {}),
+                        ...(restItem.FieldValuesAsText || {})
+                    };
+                    let FieldValuesForEdit = {
+                        ...(existingItem.FieldValuesForEdit || {}),
+                        ...(restItem.FieldValuesForEdit || {})
+                    };
+                    existingItem = { ...existingItem, ...restItem };
+                    existingItem.FieldValuesAsText = FieldValuesAsText;
+                    existingItem.FieldValuesForEdit = FieldValuesForEdit;
+                    mergedItems[existingItemIndex] = existingItem;
+                }
+            }
+        }
+
+        // only need to put this true if it happens once
+        if (loopResult.postProcessOrderBy) {
+            rtrnProcessRequestResult.postProcessOrderBy = true;
+        }
+    }
+
+    rtrnProcessRequestResult.items = mergedItems;
+
+    return rtrnProcessRequestResult;
+}
+
+async function _processNormalCamlRequest(orderByStatement: { Name: string; IsAscending: boolean; }[],
+    allListFieldsHash: IDictionary<IFieldInfoEX>,
+    options: ICamlOptions,
+    camlQuery: string,
+    requestUrl: string,
+    expandFields: string[],
+    siteUrl: string,
+    totalNumberOfItemsToGet: number,
+    batchSize: number) {
+    let processColumnsResult = _processColumns(options.columns, expandFields, allListFieldsHash);
+    let viewFields = processColumnsResult.viewFields;
+    let selectFields = processColumnsResult.selectFields;
+    expandFields = processColumnsResult.expandFields;
+    let needContentTypes = processColumnsResult.needContentTypes;
+
+    if (isNotEmptyArray(viewFields)) {
+        camlQuery = EnsureViewFields(camlQuery, viewFields, true);
+    }
+
+    // Prepare the REST URL with select fields
+    let restUrlWithSelect = requestUrl;
+    if (expandFields.length > 0) {
+        restUrlWithSelect += ',' + expandFields.join(',');
+    }
+    // Include the lookup user select fields with the other select fields
+    restUrlWithSelect += '&$select=' + selectFields.join(',');
+
+    // Process the request and get the result
+    let result = await processRequestResult(requestUrl, restUrlWithSelect, camlQuery, options, siteUrl, totalNumberOfItemsToGet, batchSize, orderByStatement);
+
+    return {
+        ...result,
+        needContentTypes
+    };
+}
+
+function _ensureViewFields(viewFieldsNode: Element, columns: string[], allListFieldsHash: IDictionary<IFieldInfoEX>) {
+    //ISSUE: 1565
+    //Cases
+    //1. Empty view fields element so ALL columns will be requested. We must include all list fields in 
+    //columns parsing so that we get an accurate count of how many lookup/user fields there are
+    //2. View fields element with a some field names that must also be included in the column parsing so we add them here.
+    if (isNullOrUndefined(viewFieldsNode) && isNullOrEmptyArray(columns)) {
+        Object.keys(allListFieldsHash).forEach((fieldName) => {
+            let field = allListFieldsHash[fieldName];
+            if (!isNullOrUndefined(field)
+                && !field.Hidden
+                && !SkipFields.includes(fieldName.toLowerCase())) {
+                columns.push(field.InternalName);
+            }
+        });
+    } else if (!isNullOrUndefined(viewFieldsNode)) {
+        let fieldRefNodes = Array.from(viewFieldsNode.querySelectorAll("FieldRef"));
+        if (isNotEmptyArray(fieldRefNodes)) {
+            fieldRefNodes.forEach((fieldRefNode) => {
+                let name = fieldRefNode.getAttribute("Name") || fieldRefNode.getAttribute("name");
+                if (!isNullOrEmptyString(name)) {
+                    columns.push(name);
+                }
+            });
+        }
+    }
+
+    return columns;
+}
+
+function _ensureOrderByColumns(orderByStatement: { Name: string; IsAscending: boolean; }[], columns: string[]) {
+    //Issue 548: Ensure that the order by field is in the view fields and columns collection so 
+    //that paging works correctly when there is an order by clause    
+    if (orderByStatement.length > 0 && !isNullOrEmptyString(orderByStatement[0].Name)) {
+        // just add them to columns, and we will add them to view fields below.
+        // camlQuery = EnsureViewFields(camlQuery, orderByStatement.orderBy.map(o => o.Name), false);       
+        orderByStatement.forEach(o => {
+            if (columns.indexOf(o.Name) === -1) {
+                columns.push(o.Name);
+            }
+        });
+    }
+
+    return columns;
+}
+
+function _normalizeColumns(columns: string[], allListFieldsHash: IDictionary<IFieldInfoEX>) {
+    return makeUniqueArray(columns.map((column) => {
+        //some columns will come in lower case (from dvp), normalize them so that we can remove duplicates
+        let name = Object.keys(allListFieldsHash).filter((fieldName) => {
+            return fieldName.toLowerCase() === column.toLowerCase();
+        })[0];
+        if (!isNullOrEmptyString(name)) {
+            let field = allListFieldsHash[name];
+            return field;
+        }
+        return null;
+    }).filter((field) => {
+        return !isNullOrUndefined(field);
+    }).map((field) => {
+        return field.InternalName;
+    }).concat(['Title', 'Id', 'FileLeafRef', 'FileDirRef', 'FileRef', 'FileSystemObjectType']));
+}
+
+function _removeSingleAndConditions(viewNode: Element, queryNode: Element) {
+    if (!isNullOrUndefined(viewNode.querySelector("And, and"))) {
+        try {
+            //Issue 8063: calendar list, will wrongly add a wrapping <and> statement for a single condition
+            //will result in error 500
+            //ie: <View Scope='RecursiveAll'><Query><Where><And><Eq><FieldRef Name=\"Title\" /><Value Type=\"Text\">Holiday</Value></Eq></And></Where>
+            //</Query><RowLimit>5000</RowLimit></View>
+            let whereNode = queryNode && queryNode.querySelector("Where, where");
+
+            if (!isNullOrUndefined(whereNode)) {
+                let firstCondition = whereNode.firstElementChild;
+                if (firstCondition.tagName.toLowerCase() === "and" && firstCondition.children.length < 2) {
+                    //this is the bug, <and> tag must have 2 conditions - get rid of it!
+                    whereNode.innerHTML = firstCondition.innerHTML;
+                    return {
+                        camlQuery: viewNode.outerHTML
+                    };
+                }
+            }
+        } catch (e) { }
+    }
+
+    return null;
+}
+
+function _parseRowLimitNode(rowLimitNode: Element, viewNode: Element, rowLimit: number) {
+    if (!isNullOrUndefined(rowLimitNode)) {
+        let value = rowLimitNode.textContent;
+        //if not provided by options - use it
+        if (isNullOrNaN(rowLimit) && isNumeric(value) && Number(value) > 0) {
+            rowLimit = Number(value);
+        }
+        //remove it
+        viewNode.removeChild(rowLimitNode);
+        return {
+            camlQuery: viewNode.outerHTML,
+            rowLimit: rowLimit
+        };
+    }
+    return null;
+}
+
+function _processColumns(columns: string[], expandFields: string[], allListFieldsHash: IDictionary<IFieldInfoEX>) {
+    let thresholdLimitLookupCount = 0;
+    let thresholdLimitLookupHit = false;
+    let selectFields: string[] = [];
+    let viewFields: string[] = [];
+    let needContentTypes = false;
+
+    //column parsing
+    columns.forEach(viewField => {
+        if (viewField.toLowerCase() === 'contenttype' || viewField.toLowerCase() === 'contenttypeid') {
+            needContentTypes = true;
+        }
+        else if (viewField.toLowerCase() === '_moderationstatus') {
+            selectFields.push(`FieldValuesAsText/${viewField}`);
+            viewFields.push('_moderationstatus');
+        }
+        else if (viewField.toLowerCase() === '_moderationcomments') {
+            selectFields.push('OData_' + viewField);
+            viewFields.push('_moderationcomments');
+        }
+        else if (viewField.toLowerCase() === "filesystemobjecttype") {
+            selectFields.push("FileSystemObjectType");
+        }
+        else if (viewField.toLowerCase() === "fileref" || viewField.toLowerCase() === "filedirref") {
+            //treat them similar to lookup fields
+            selectFields.push(viewField);
+            selectFields.push(`FieldValuesAsText/${viewField}`);
+            viewFields.push(`${viewField}`);
+            viewFields.push(`${viewField}Id`);
+        }
+        else {
+            //prefer to get columns not from FieldValuesAsText, unless special data type requires it (date, lookup, boolean, etc...)
+            //make the select url shorter
+            let foundField = allListFieldsHash[viewField];
+            if (foundField) {
+                let foundFieldInternalName = foundField.InternalName;
+                viewFields.push(foundFieldInternalName);
+
+                //Issue 828, 336
+                if (foundFieldInternalName.startsWith("_")) {
+                    foundFieldInternalName = `OData_${foundFieldInternalName}`;
+                }
+
+                let outputType = getFieldOutputType(foundField);
+
+                switch (outputType) {
+                    case "Lookup":
+                    case "LookupMulti":
+                    case "User":
+                    case "UserMulti":
+                        thresholdLimitLookupCount += 1;
+                        if (thresholdLimitLookupCount >= lookupOrUserFieldLimit) {
+                            thresholdLimitLookupHit = true;
+                        }
+                        //lookup raw values comes with Id appended
+                        selectFields.push(`FieldValuesAsText/${foundFieldInternalName}`);
+                        selectFields.push(`${foundFieldInternalName}Id`);
+                        break;
+                    case "Boolean":
+                    case "Attachments":
+                    case "AllDayEvent":
+                    case "Recurrence":
+                        selectFields.push(`FieldValuesAsText/${foundFieldInternalName}`);
+                        selectFields.push(foundFieldInternalName);
+                        break;
+                    default:
+                        selectFields.push(foundFieldInternalName);
+                        break;
+                }
+            }
+            else {
+                selectFields.push(viewField);
+            }
+        }
+    });
+
+    if (needContentTypes) {
+        expandFields.push("ContentType");
+        selectFields.push("ContentType/Name");
+        selectFields.push("ContentTypeId");
+        viewFields.push("ContentTypeId");
+    }
+
+    return {
+        thresholdLimitLookupHit,
+        viewFields: makeUniqueArray(viewFields),
+        selectFields: makeUniqueArray(selectFields),
+        expandFields: makeUniqueArray(expandFields),
+        needContentTypes
+    };
+}
+
+function _isLookupOrUserField(field: IFieldInfoEX) {
+    if (isNullOrUndefined(field)) {
+        return false;
+    }
+    let outputType = getFieldOutputType(field);
+
+    switch (outputType) {
+        case "Lookup":
+        case "LookupMulti":
+        case "User":
+        case "UserMulti":
+            return true;
+        default:
+            return false;
+    }
+}
+
+async function processRequestResult(requestUrl: string, restUrlWithSelect: string, camlQuery: string, options: ICamlOptions, siteUrl: string, totalNumberOfItemsToGet: number, batchSize: number, orderByStatement): Promise<{ items: IRestItem[]; postProcessOrderBy: boolean; }> {
+    //issue 6150: if there are too many fields, url will be too long. just get all columns without adding a $select
+    if (restUrlWithSelect.length < 2000)
+        requestUrl = restUrlWithSelect;
+
+    let query: any = { ViewXml: camlQuery.replace("</View>", `<RowLimit>${batchSize}</RowLimit></View>`) };
+
+    if (!isNullOrEmptyString(options.FolderServerRelativeUrl))
+        query.FolderServerRelativeUrl = options.FolderServerRelativeUrl;
+
+    let data = { query: query };
+
+    let items: IRestItem[] = [];
+    //let triedWithoutViewFields = false;
+    let postProcessOrderBy = false;
+    do {
+        try {
+            let requestResult = await GetJson<{ value: IRestItem[]; }>(requestUrl, JSON.stringify(data),
+                {
+                    allowCache: options.refreshCache !== true,
+                    postCacheKey: JSON.stringify(data.query),
+                    jsonMetadata: jsonTypes.nometadata,
+                    spWebUrl: siteUrl
+                });
+            if (requestResult && requestResult.value)
+                items.push(...requestResult.value);
+
+            let itemsLeftToGet = totalNumberOfItemsToGet - items.length;
+            if (itemsLeftToGet > 0 &&//we need more items
+                requestResult.value.length === batchSize//we might have more on server since we got the full batch size we asked for
+            ) {
+                let lastItem = items[items.length - 1];
+                let lastItemId = lastItem.Id;
+                let pagingInfoSort = "";
+                //Issue 7542 need to add order by value to the paging info if it is in the query
+                if (!postProcessOrderBy && orderByStatement && orderByStatement.length) {
+                    let orderFieldName = orderByStatement[0].Name;
+                    let orderFieldValue = lastItem[orderFieldName];
+
+                    //if field is ID - do not do it.
+                    if (orderFieldName.toLowerCase() !== "id" && !isNullOrUndefined(orderFieldValue)) {
+                        //if value is date - we need it in ISO and in this format: yyyyMMdd HH:mm:ss
+                        try {
+                            //Numbers cast to date properly but they are not dates. ignore number values.
+                            let orderFieldValueAsDate = isNumeric(orderFieldValue) ? null : new Date(orderFieldValue);
+                            if (isDate(orderFieldValueAsDate)) {
+                                try {
+                                    //issue 7599 date only field on different time zone...
+                                    orderFieldValue = SPServerLocalTimeToUTCSync(siteUrl, orderFieldValueAsDate).replace(/T/i, " ").replace(/Z/i, " ");
+                                } catch (e) { }
+                            }
+
+                        } catch (e) { }
+                        pagingInfoSort = `&p_${orderFieldName}=${encodeURIComponentEX(orderFieldValue)}`;
+                    }
+                }
+                data.query.ListItemCollectionPosition = {
+                    "PagingInfo": `Paged=TRUE${pagingInfoSort}&p_ID=${lastItemId}`
+                };
+                if (itemsLeftToGet < batchSize)//the last batch should be smaller, update the row limit
+                    data.query.ViewXml = camlQuery.replace("</View>", `<RowLimit>${itemsLeftToGet}</RowLimit></View>`);
+            }
+            else
+                data = null;
+        } catch (e) {
+            if (__getSPRestErrorData(e).code.indexOf('SPQueryThrottledException')) {
+                //test again - on awaiting score view, this will work but will ONLY return the order by fields...
+                // if (!triedWithoutViewFields) {
+                //     //Our own issues list had too many lookup fields.
+                //     logger.info("Query throttled, trying again without view fields... some fields might be missing from results.");
+                //     triedWithoutViewFields = true;
+                //     camlQuery = EnsureViewFields(camlQuery, orderByStatement.map(o => o.Name), false, true);
+                //     data.query.ViewXml = camlQuery.replace("</View>", `<RowLimit>${batchSize}</RowLimit></View>`);
+                // }
+                // else
+                if (!postProcessOrderBy && orderByStatement.length > 0) {
+                    logger.warn("Query throttled, trying again without order by...");
+                    postProcessOrderBy = true;
+                    camlQuery = RemoveOrderByFromCaml(camlQuery);
+                    camlQuery = EnsureViewFields(camlQuery, [], false, true);
+                    data.query.ViewXml = camlQuery.replace("</View>", `<RowLimit>${batchSize}</RowLimit></View>`);
+                }
+                else throw e;//still throttled? might be due to filter query
+            }
+            else throw e;//different error
+        }
+    } while (!isNullOrEmptyString(data));
+    return { items: items, postProcessOrderBy: postProcessOrderBy };
+}