@kwiz/common 1.0.1

package/dist/utils/sharepoint.rest/listutils/GetListItemsByCaml.js

@@ -0,0 +1,629 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.GetListItemsByCaml = void 0;
+const _dependencies_1 = require("../../_dependencies");
+const consolelogger_1 = require("../../consolelogger");
+const rest_1 = require("../../rest");
+const common_1 = require("../common");
+const list_1 = require("../list");
+const web_1 = require("../web");
+const GetListItemsById_1 = require("./GetListItemsById");
+const common_2 = require("./common");
+const logger = consolelogger_1.ConsoleLogger.get("sharepoint.rest/list/GetListItemsByCaml");
+//Lookup threshold limit is 12 but we use a smaller number here to ensure there are no issues
+const lookupOrUserFieldLimit = 11;
+/** returns the items or NULL, never errors out since it is used in aggregator.
+ *  set throwErrors = true if you want errors to be thrown instead of returning null.
+ * camlQuery: one of:
+ *  - view.ViewQuery
+ *  - wrapped view.ViewQuery in a <View>: "&lt;View Scope='RecursiveAll'>&lt;Query>&lt;/Query>&lt;RowLimit>1000&lt;/RowLimit>&lt;/View>" or send rowLimit in options.
+ */
+function GetListItemsByCaml(siteUrl, listIdOrTitle, camlQuery, options) {
+    return __awaiter(this, void 0, void 0, function* () {
+        siteUrl = (0, common_1.GetSiteUrl)(siteUrl);
+        if (!camlQuery.toLowerCase().startsWith("<view")) {
+            camlQuery = `<View Scope='RecursiveAll'><Query>${camlQuery}</Query></View>`;
+        }
+        let xmlParser = new DOMParser();
+        let xmlDoc = xmlParser.parseFromString(camlQuery, "text/xml");
+        let viewNode = xmlDoc.querySelector("View, view");
+        let queryNode = viewNode && viewNode.querySelector("Query, query");
+        let rowLimitNode = viewNode && viewNode.querySelector("RowLimit, rowlimit");
+        let viewFieldsNode = viewNode && viewNode.querySelector("ViewFields, viewfields");
+        let parseRowLimitNodeResult = _parseRowLimitNode(rowLimitNode, viewNode, options.rowLimit);
+        if (!(0, _dependencies_1.isNullOrUndefined)(parseRowLimitNodeResult)) {
+            options.rowLimit = parseRowLimitNodeResult.rowLimit;
+            camlQuery = parseRowLimitNodeResult.camlQuery;
+        }
+        let removeSingleAndConditionsResult = _removeSingleAndConditions(viewNode, queryNode);
+        if (!(0, _dependencies_1.isNullOrUndefined)(removeSingleAndConditionsResult)) {
+            camlQuery = removeSingleAndConditionsResult.camlQuery;
+        }
+        let getAllItems = (0, _dependencies_1.isNullOrEmptyString)(options.rowLimit) || options.rowLimit < 1;
+        let totalItemsInList = 99999;
+        let maxBatchSize = 5000;
+        let batchSize = options.rowLimit > 0 && options.rowLimit < maxBatchSize ? options.rowLimit : maxBatchSize;
+        let totalNumberOfItemsToGet = getAllItems ? totalItemsInList : options.rowLimit > maxBatchSize ? options.rowLimit : batchSize;
+        let requestUrl = `${(0, list_1.GetListRestUrl)(siteUrl, listIdOrTitle)}/GetItems?$expand=FieldValuesAsText`;
+        let allListFieldsHash = (0, _dependencies_1.toHash)(yield (0, list_1.GetListFields)(siteUrl, listIdOrTitle), f => f.InternalName);
+        let expandFields = [];
+        let orderByStatement = (0, _dependencies_1.GetOrderByFromCaml)(camlQuery);
+        if ((0, _dependencies_1.isNullOrUndefined)(options.columns)) {
+            options.columns = [];
+        }
+        let columns = options.columns;
+        columns = _ensureOrderByColumns(orderByStatement, columns);
+        columns = _ensureViewFields(viewFieldsNode, columns, allListFieldsHash);
+        columns = _normalizeColumns(columns, allListFieldsHash);
+        if (columns.length > options.columns.length) {
+            logger.warn(`added ${columns.length - options.columns.length} to query`);
+        }
+        options.columns = columns;
+        // Store the result of processing the request
+        let rtrnProcessRequestResult;
+        let lookupOrUserFieldsInColumns = options.columns.filter((columnName) => {
+            let field = allListFieldsHash[columnName];
+            return _isLookupOrUserField(field);
+        });
+        let isThrottled = lookupOrUserFieldsInColumns.length >= lookupOrUserFieldLimit;
+        try {
+            if (isThrottled) {
+                //ISSUE: 1565
+                rtrnProcessRequestResult = yield _processLookupThresholdCamlRequest(orderByStatement, allListFieldsHash, options, camlQuery, requestUrl, expandFields, siteUrl, totalNumberOfItemsToGet, batchSize);
+            }
+            else {
+                rtrnProcessRequestResult = yield _processNormalCamlRequest(orderByStatement, allListFieldsHash, options, camlQuery, requestUrl, expandFields, siteUrl, totalNumberOfItemsToGet, batchSize);
+            }
+            const { items, needContentTypes, postProcessOrderBy } = rtrnProcessRequestResult;
+            let itemsResult = (0, common_2.__fixGetListItemsResults)(siteUrl, listIdOrTitle, items, options.foldersBehaviour);
+            let itemsWithOutContentType = [];
+            if (needContentTypes) {
+                itemsResult.forEach((item) => {
+                    if ((0, _dependencies_1.isNullOrUndefined)(item["ContentType"])) {
+                        itemsWithOutContentType.push(item.Id);
+                    }
+                    else if (!(0, _dependencies_1.isNullOrUndefined)(item["ContentType"].Name)) {
+                        item["ContentType"] = item["ContentType"].Name;
+                    }
+                });
+            }
+            if (itemsWithOutContentType.length > 0) {
+                logger.time("Getting content types");
+                //Issue 1465 content types no longer come back from get items request...
+                //Make a separate request to get this info
+                let ctypes = (yield (0, GetListItemsById_1.GetItemsById)(siteUrl, listIdOrTitle, itemsWithOutContentType, {
+                    expand: ['ContentType/Name'],
+                    select: ['ContentType/Name', 'Id'],
+                    jsonMetadata: _dependencies_1.jsonTypes.nometadata
+                }));
+                itemsResult.forEach(item => {
+                    if (!(0, _dependencies_1.isNullOrUndefined)(ctypes[item.Id])) {
+                        item["ContentType"] = ctypes[item.Id].ContentType.Name;
+                    }
+                });
+                logger.timeEnd("Getting content types");
+            }
+            if (postProcessOrderBy) {
+                //re-apply sort
+                if (_dependencies_1.IsLocalDev) {
+                    logger.table(itemsResult.map(i => {
+                        let row = {
+                            Id: i.Id,
+                            Title: i.Title
+                        };
+                        orderByStatement.forEach(s => { row[`${s.Name} ${s.IsAscending ? 'asc' : 'desc'}`] = i[s.Name]; });
+                        return row;
+                    }), "before sort", true);
+                }
+                itemsResult.sort((a, b) => {
+                    for (let i = 0; i < orderByStatement.length; i++) {
+                        let ob = orderByStatement[i];
+                        let v1 = a[ob.Name];
+                        let v2 = b[ob.Name];
+                        if (v1 === v2) {
+                            //these are equal - continue to second sort statement
+                        }
+                        else {
+                            if (ob.IsAscending)
+                                return (v1 > v2) ? 1 : -1;
+                            else
+                                return (v2 > v1) ? 1 : -1;
+                        }
+                    }
+                    return 0;
+                });
+                if (_dependencies_1.IsLocalDev) {
+                    logger.table(itemsResult.map(i => {
+                        let row = {
+                            Id: i.Id,
+                            Title: i.Title
+                        };
+                        orderByStatement.forEach(s => { row[`${s.Name} ${s.IsAscending ? 'asc' : 'desc'}`] = i[s.Name]; });
+                        return row;
+                    }), "after sort", true);
+                }
+            }
+            return itemsResult;
+        }
+        catch (ex) {
+            console.log(`isThrottled: ${isThrottled}`);
+        }
+        return null;
+    });
+}
+exports.GetListItemsByCaml = GetListItemsByCaml;
+// (window as any).runtTest = true;
+// window.setTimeout(() => {
+//     if ((window as any).runtTest == false) return;
+//     (window as any).runtTest = false;
+//     GetListItemsByCaml(null, "aec7756b-daa0-4da1-88ba-c66cb572e816", "", {
+//         columns: [],
+//         refreshCache: true
+//     }).then((r) => {
+//     });
+// }, 500);
+//ISSUE: 1565
+function _processLookupThresholdCamlRequest(orderByStatement, allListFieldsHash, options, camlQuery, requestUrl, expandFields, siteUrl, totalNumberOfItemsToGet, batchSize) {
+    return __awaiter(this, void 0, void 0, function* () {
+        let rtrnProcessRequestResult = {
+            items: [],
+            postProcessOrderBy: false,
+            needContentTypes: false
+        };
+        let orderByFields = orderByStatement.map((orderByField) => {
+            let field = allListFieldsHash[orderByField.Name];
+            return field;
+        });
+        let lookupOrUserFieldsInOrderBy = orderByFields.filter((orderByField) => {
+            return _isLookupOrUserField(orderByField);
+        });
+        let lookupOrUserFieldsToChunk = options.columns.filter((columnName) => {
+            let field = allListFieldsHash[columnName];
+            if (_isLookupOrUserField(field)) {
+                let col = (0, _dependencies_1.firstOrNull)(lookupOrUserFieldsInOrderBy, (orderByField) => {
+                    return field.InternalName !== orderByField.InternalName;
+                });
+                return col === null;
+            }
+            return false;
+        });
+        let otherFieldNames = options.columns.filter((columnName) => {
+            let field = allListFieldsHash[columnName];
+            let exists = !(0, _dependencies_1.isNullOrUndefined)(field);
+            let isLookupField = exists && _isLookupOrUserField(field);
+            let isInOrderBy = exists && (0, _dependencies_1.firstOrNull)(orderByFields, (orderByField) => orderByField.InternalName === field.InternalName) !== null;
+            return exists && !isLookupField && !isInOrderBy;
+        });
+        //Split the requests into equally chunked arrays based on the lookup field count
+        //add lookup fields in chunk but leave room to add lookup fields from the order by statement 
+        //since they are mandatory    
+        let requestChunks = (0, _dependencies_1.chunkArray)(lookupOrUserFieldsToChunk, lookupOrUserFieldLimit - lookupOrUserFieldsInOrderBy.length);
+        let otherFieldsChunkSize = Math.ceil(otherFieldNames.length / requestChunks.length);
+        let otherFieldsChunks = (0, _dependencies_1.chunkArray)(otherFieldNames, otherFieldsChunkSize);
+        requestChunks.forEach((chunk, index) => {
+            //Add all order by fields to each request, this will include the lookup
+            //fields from the order by statement that we left room for previously
+            requestChunks[index] = chunk.concat(orderByFields.map((orderByField) => {
+                return orderByField.InternalName;
+            }));
+            //Add the other fields but split them across the requests so we don't request duplicate data
+            if (otherFieldsChunks[index]) {
+                requestChunks[index] = chunk.concat(otherFieldsChunks[index]);
+            }
+        });
+        //requestChunks should now have about the same number of fields in each chunk and each
+        //chunk will have all the order by fields.            
+        let queries = requestChunks.map((requestChunk) => {
+            let camlQueryClone = camlQuery;
+            let processColumnsResult = _processColumns(requestChunk, expandFields, allListFieldsHash);
+            //Id field must be included in oder to merge the items correctly
+            let viewFields = processColumnsResult.viewFields;
+            if (viewFields.length && !viewFields.includes("Id")) {
+                viewFields.push("Id");
+            }
+            let selectFields = processColumnsResult.selectFields;
+            if (selectFields.length && !selectFields.includes("Id")) {
+                selectFields.push("Id");
+            }
+            rtrnProcessRequestResult.needContentTypes = rtrnProcessRequestResult.needContentTypes || processColumnsResult.needContentTypes;
+            expandFields = processColumnsResult.expandFields;
+            if ((0, _dependencies_1.isNotEmptyArray)(viewFields)) {
+                camlQueryClone = (0, _dependencies_1.EnsureViewFields)(camlQueryClone, viewFields, true);
+            }
+            // if (isDebug()) {
+            //     let lfields = viewFields.filter((fieldName) => {
+            //         let field = allListFieldsHash[fieldName];
+            //         return _isLookupOrUserField(field);
+            //     });
+            //     let xmlDoc = new DOMParser().parseFromString(camlQueryClone, "text/xml");
+            //     let viewNode = xmlDoc.querySelector("View, view");
+            //     let viewFieldsNode = viewNode && viewNode.querySelector("ViewFields, viewfields");
+            //     let viewFields2 = Array.from(viewFieldsNode.children).map((viewFieldNode) => {
+            //         let name = viewFieldNode.getAttribute("Name") || viewFieldNode.getAttribute("name");
+            //         return name;
+            //     });
+            //     let lfields2 = viewFields2.filter((fieldName) => {
+            //         let field = allListFieldsHash[fieldName];
+            //         return _isLookupOrUserField(field);
+            //     });
+            //     if (lfields2.length !== lfields.length) {
+            //         logger.warn("Lookup fields in caml query do not match look up fields in view fields.");
+            //         logger.warn(`Lookup fields in caml query: ${lfields2}`)
+            //         logger.warn(`Lookup fields in in view fields: ${lfields}`)
+            //     }
+            // }
+            // Prepare the REST URL with select fields
+            let restUrlWithSelect = requestUrl;
+            if (expandFields.length > 0) {
+                restUrlWithSelect += ',' + expandFields.join(',');
+            }
+            // Include the lookup user select fields with the other select fields
+            restUrlWithSelect += '&$select=' + selectFields.join(',');
+            return { camlQueryClone, restUrlWithSelect, viewFields };
+        });
+        let mergedItems = null;
+        for (let i = 0; i < queries.length; i++) {
+            const { camlQueryClone, restUrlWithSelect, viewFields } = queries[i];
+            let loopResult = yield processRequestResult(requestUrl, restUrlWithSelect, camlQueryClone, Object.assign(Object.assign({}, options), { columns: viewFields }), siteUrl, totalNumberOfItemsToGet, batchSize, orderByStatement);
+            if ((0, _dependencies_1.isNullOrUndefined)(mergedItems)) {
+                mergedItems = loopResult.items;
+            }
+            else {
+                for (let restItemIndex = 0; restItemIndex < loopResult.items.length; restItemIndex++) {
+                    //The item chunks (loopResult) should be in the same order as mergedItems because each request should have the 
+                    //same number of items returned in the same order. The only difference between requests is which fields are present.
+                    //So request 1 will have the 1st set of fields, request 2 will have the 2nd set. etc.
+                    //This means that the restItemIndex should be the same as existingItemIndex for each iteration. But, we do these extra 
+                    //checks just in case.
+                    let restItem = loopResult.items[restItemIndex];
+                    let existingItem = mergedItems[restItemIndex];
+                    let existingItemIndex = restItemIndex;
+                    if ((0, _dependencies_1.isNullOrUndefined)(existingItem) || existingItem.Id !== restItem.Id) {
+                        existingItemIndex = (0, _dependencies_1.firstIndexOf)(mergedItems, (mergedItem) => {
+                            return mergedItem.Id === restItem.Id;
+                        });
+                    }
+                    if (existingItemIndex === -1) {
+                        //We shouldn't get here. Each chunk should have the same items in the same order.
+                        logger.warn("_processLookupThresholdCamlRequest results are out of sync");
+                        mergedItems.push(restItem);
+                    }
+                    else {
+                        let existingItem = mergedItems[existingItemIndex];
+                        let FieldValuesAsText = Object.assign(Object.assign({}, (existingItem.FieldValuesAsText || {})), (restItem.FieldValuesAsText || {}));
+                        let FieldValuesForEdit = Object.assign(Object.assign({}, (existingItem.FieldValuesForEdit || {})), (restItem.FieldValuesForEdit || {}));
+                        existingItem = Object.assign(Object.assign({}, existingItem), restItem);
+                        existingItem.FieldValuesAsText = FieldValuesAsText;
+                        existingItem.FieldValuesForEdit = FieldValuesForEdit;
+                        mergedItems[existingItemIndex] = existingItem;
+                    }
+                }
+            }
+            // only need to put this true if it happens once
+            if (loopResult.postProcessOrderBy) {
+                rtrnProcessRequestResult.postProcessOrderBy = true;
+            }
+        }
+        rtrnProcessRequestResult.items = mergedItems;
+        return rtrnProcessRequestResult;
+    });
+}
+function _processNormalCamlRequest(orderByStatement, allListFieldsHash, options, camlQuery, requestUrl, expandFields, siteUrl, totalNumberOfItemsToGet, batchSize) {
+    return __awaiter(this, void 0, void 0, function* () {
+        let processColumnsResult = _processColumns(options.columns, expandFields, allListFieldsHash);
+        let viewFields = processColumnsResult.viewFields;
+        let selectFields = processColumnsResult.selectFields;
+        expandFields = processColumnsResult.expandFields;
+        let needContentTypes = processColumnsResult.needContentTypes;
+        if ((0, _dependencies_1.isNotEmptyArray)(viewFields)) {
+            camlQuery = (0, _dependencies_1.EnsureViewFields)(camlQuery, viewFields, true);
+        }
+        // Prepare the REST URL with select fields
+        let restUrlWithSelect = requestUrl;
+        if (expandFields.length > 0) {
+            restUrlWithSelect += ',' + expandFields.join(',');
+        }
+        // Include the lookup user select fields with the other select fields
+        restUrlWithSelect += '&$select=' + selectFields.join(',');
+        // Process the request and get the result
+        let result = yield processRequestResult(requestUrl, restUrlWithSelect, camlQuery, options, siteUrl, totalNumberOfItemsToGet, batchSize, orderByStatement);
+        return Object.assign(Object.assign({}, result), { needContentTypes });
+    });
+}
+function _ensureViewFields(viewFieldsNode, columns, allListFieldsHash) {
+    //ISSUE: 1565
+    //Cases
+    //1. Empty view fields element so ALL columns will be requested. We must include all list fields in 
+    //columns parsing so that we get an accurate count of how many lookup/user fields there are
+    //2. View fields element with a some field names that must also be included in the column parsing so we add them here.
+    if ((0, _dependencies_1.isNullOrUndefined)(viewFieldsNode) && (0, _dependencies_1.isNullOrEmptyArray)(columns)) {
+        Object.keys(allListFieldsHash).forEach((fieldName) => {
+            let field = allListFieldsHash[fieldName];
+            if (!(0, _dependencies_1.isNullOrUndefined)(field)
+                && !field.Hidden
+                && !common_2.SkipFields.includes(fieldName.toLowerCase())) {
+                columns.push(field.InternalName);
+            }
+        });
+    }
+    else if (!(0, _dependencies_1.isNullOrUndefined)(viewFieldsNode)) {
+        let fieldRefNodes = Array.from(viewFieldsNode.querySelectorAll("FieldRef"));
+        if ((0, _dependencies_1.isNotEmptyArray)(fieldRefNodes)) {
+            fieldRefNodes.forEach((fieldRefNode) => {
+                let name = fieldRefNode.getAttribute("Name") || fieldRefNode.getAttribute("name");
+                if (!(0, _dependencies_1.isNullOrEmptyString)(name)) {
+                    columns.push(name);
+                }
+            });
+        }
+    }
+    return columns;
+}
+function _ensureOrderByColumns(orderByStatement, columns) {
+    //Issue 548: Ensure that the order by field is in the view fields and columns collection so 
+    //that paging works correctly when there is an order by clause    
+    if (orderByStatement.length > 0 && !(0, _dependencies_1.isNullOrEmptyString)(orderByStatement[0].Name)) {
+        // just add them to columns, and we will add them to view fields below.
+        // camlQuery = EnsureViewFields(camlQuery, orderByStatement.orderBy.map(o => o.Name), false);       
+        orderByStatement.forEach(o => {
+            if (columns.indexOf(o.Name) === -1) {
+                columns.push(o.Name);
+            }
+        });
+    }
+    return columns;
+}
+function _normalizeColumns(columns, allListFieldsHash) {
+    return (0, _dependencies_1.makeUniqueArray)(columns.map((column) => {
+        //some columns will come in lower case (from dvp), normalize them so that we can remove duplicates
+        let name = Object.keys(allListFieldsHash).filter((fieldName) => {
+            return fieldName.toLowerCase() === column.toLowerCase();
+        })[0];
+        if (!(0, _dependencies_1.isNullOrEmptyString)(name)) {
+            let field = allListFieldsHash[name];
+            return field;
+        }
+        return null;
+    }).filter((field) => {
+        return !(0, _dependencies_1.isNullOrUndefined)(field);
+    }).map((field) => {
+        return field.InternalName;
+    }).concat(['Title', 'Id', 'FileLeafRef', 'FileDirRef', 'FileRef', 'FileSystemObjectType']));
+}
+function _removeSingleAndConditions(viewNode, queryNode) {
+    if (!(0, _dependencies_1.isNullOrUndefined)(viewNode.querySelector("And, and"))) {
+        try {
+            //Issue 8063: calendar list, will wrongly add a wrapping <and> statement for a single condition
+            //will result in error 500
+            //ie: <View Scope='RecursiveAll'><Query><Where><And><Eq><FieldRef Name=\"Title\" /><Value Type=\"Text\">Holiday</Value></Eq></And></Where>
+            //</Query><RowLimit>5000</RowLimit></View>
+            let whereNode = queryNode && queryNode.querySelector("Where, where");
+            if (!(0, _dependencies_1.isNullOrUndefined)(whereNode)) {
+                let firstCondition = whereNode.firstElementChild;
+                if (firstCondition.tagName.toLowerCase() === "and" && firstCondition.children.length < 2) {
+                    //this is the bug, <and> tag must have 2 conditions - get rid of it!
+                    whereNode.innerHTML = firstCondition.innerHTML;
+                    return {
+                        camlQuery: viewNode.outerHTML
+                    };
+                }
+            }
+        }
+        catch (e) { }
+    }
+    return null;
+}
+function _parseRowLimitNode(rowLimitNode, viewNode, rowLimit) {
+    if (!(0, _dependencies_1.isNullOrUndefined)(rowLimitNode)) {
+        let value = rowLimitNode.textContent;
+        //if not provided by options - use it
+        if ((0, _dependencies_1.isNullOrNaN)(rowLimit) && (0, _dependencies_1.isNumeric)(value) && Number(value) > 0) {
+            rowLimit = Number(value);
+        }
+        //remove it
+        viewNode.removeChild(rowLimitNode);
+        return {
+            camlQuery: viewNode.outerHTML,
+            rowLimit: rowLimit
+        };
+    }
+    return null;
+}
+function _processColumns(columns, expandFields, allListFieldsHash) {
+    let thresholdLimitLookupCount = 0;
+    let thresholdLimitLookupHit = false;
+    let selectFields = [];
+    let viewFields = [];
+    let needContentTypes = false;
+    //column parsing
+    columns.forEach(viewField => {
+        if (viewField.toLowerCase() === 'contenttype' || viewField.toLowerCase() === 'contenttypeid') {
+            needContentTypes = true;
+        }
+        else if (viewField.toLowerCase() === '_moderationstatus') {
+            selectFields.push(`FieldValuesAsText/${viewField}`);
+            viewFields.push('_moderationstatus');
+        }
+        else if (viewField.toLowerCase() === '_moderationcomments') {
+            selectFields.push('OData_' + viewField);
+            viewFields.push('_moderationcomments');
+        }
+        else if (viewField.toLowerCase() === "filesystemobjecttype") {
+            selectFields.push("FileSystemObjectType");
+        }
+        else if (viewField.toLowerCase() === "fileref" || viewField.toLowerCase() === "filedirref") {
+            //treat them similar to lookup fields
+            selectFields.push(viewField);
+            selectFields.push(`FieldValuesAsText/${viewField}`);
+            viewFields.push(`${viewField}`);
+            viewFields.push(`${viewField}Id`);
+        }
+        else {
+            //prefer to get columns not from FieldValuesAsText, unless special data type requires it (date, lookup, boolean, etc...)
+            //make the select url shorter
+            let foundField = allListFieldsHash[viewField];
+            if (foundField) {
+                let foundFieldInternalName = foundField.InternalName;
+                viewFields.push(foundFieldInternalName);
+                //Issue 828, 336
+                if (foundFieldInternalName.startsWith("_")) {
+                    foundFieldInternalName = `OData_${foundFieldInternalName}`;
+                }
+                let outputType = (0, _dependencies_1.getFieldOutputType)(foundField);
+                switch (outputType) {
+                    case "Lookup":
+                    case "LookupMulti":
+                    case "User":
+                    case "UserMulti":
+                        thresholdLimitLookupCount += 1;
+                        if (thresholdLimitLookupCount >= lookupOrUserFieldLimit) {
+                            thresholdLimitLookupHit = true;
+                        }
+                        //lookup raw values comes with Id appended
+                        selectFields.push(`FieldValuesAsText/${foundFieldInternalName}`);
+                        selectFields.push(`${foundFieldInternalName}Id`);
+                        break;
+                    case "Boolean":
+                    case "Attachments":
+                    case "AllDayEvent":
+                    case "Recurrence":
+                        selectFields.push(`FieldValuesAsText/${foundFieldInternalName}`);
+                        selectFields.push(foundFieldInternalName);
+                        break;
+                    default:
+                        selectFields.push(foundFieldInternalName);
+                        break;
+                }
+            }
+            else {
+                selectFields.push(viewField);
+            }
+        }
+    });
+    if (needContentTypes) {
+        expandFields.push("ContentType");
+        selectFields.push("ContentType/Name");
+        selectFields.push("ContentTypeId");
+        viewFields.push("ContentTypeId");
+    }
+    return {
+        thresholdLimitLookupHit,
+        viewFields: (0, _dependencies_1.makeUniqueArray)(viewFields),
+        selectFields: (0, _dependencies_1.makeUniqueArray)(selectFields),
+        expandFields: (0, _dependencies_1.makeUniqueArray)(expandFields),
+        needContentTypes
+    };
+}
+function _isLookupOrUserField(field) {
+    if ((0, _dependencies_1.isNullOrUndefined)(field)) {
+        return false;
+    }
+    let outputType = (0, _dependencies_1.getFieldOutputType)(field);
+    switch (outputType) {
+        case "Lookup":
+        case "LookupMulti":
+        case "User":
+        case "UserMulti":
+            return true;
+        default:
+            return false;
+    }
+}
+function processRequestResult(requestUrl, restUrlWithSelect, camlQuery, options, siteUrl, totalNumberOfItemsToGet, batchSize, orderByStatement) {
+    return __awaiter(this, void 0, void 0, function* () {
+        //issue 6150: if there are too many fields, url will be too long. just get all columns without adding a $select
+        if (restUrlWithSelect.length < 2000)
+            requestUrl = restUrlWithSelect;
+        let query = { ViewXml: camlQuery.replace("</View>", `<RowLimit>${batchSize}</RowLimit></View>`) };
+        if (!(0, _dependencies_1.isNullOrEmptyString)(options.FolderServerRelativeUrl))
+            query.FolderServerRelativeUrl = options.FolderServerRelativeUrl;
+        let data = { query: query };
+        let items = [];
+        //let triedWithoutViewFields = false;
+        let postProcessOrderBy = false;
+        do {
+            try {
+                let requestResult = yield (0, rest_1.GetJson)(requestUrl, JSON.stringify(data), {
+                    allowCache: options.refreshCache !== true,
+                    postCacheKey: JSON.stringify(data.query),
+                    jsonMetadata: _dependencies_1.jsonTypes.nometadata,
+                    spWebUrl: siteUrl
+                });
+                if (requestResult && requestResult.value)
+                    items.push(...requestResult.value);
+                let itemsLeftToGet = totalNumberOfItemsToGet - items.length;
+                if (itemsLeftToGet > 0 && //we need more items
+                    requestResult.value.length === batchSize //we might have more on server since we got the full batch size we asked for
+                ) {
+                    let lastItem = items[items.length - 1];
+                    let lastItemId = lastItem.Id;
+                    let pagingInfoSort = "";
+                    //Issue 7542 need to add order by value to the paging info if it is in the query
+                    if (!postProcessOrderBy && orderByStatement && orderByStatement.length) {
+                        let orderFieldName = orderByStatement[0].Name;
+                        let orderFieldValue = lastItem[orderFieldName];
+                        //if field is ID - do not do it.
+                        if (orderFieldName.toLowerCase() !== "id" && !(0, _dependencies_1.isNullOrUndefined)(orderFieldValue)) {
+                            //if value is date - we need it in ISO and in this format: yyyyMMdd HH:mm:ss
+                            try {
+                                //Numbers cast to date properly but they are not dates. ignore number values.
+                                let orderFieldValueAsDate = (0, _dependencies_1.isNumeric)(orderFieldValue) ? null : new Date(orderFieldValue);
+                                if ((0, _dependencies_1.isDate)(orderFieldValueAsDate)) {
+                                    try {
+                                        //issue 7599 date only field on different time zone...
+                                        orderFieldValue = (0, web_1.SPServerLocalTimeToUTCSync)(siteUrl, orderFieldValueAsDate).replace(/T/i, " ").replace(/Z/i, " ");
+                                    }
+                                    catch (e) { }
+                                }
+                            }
+                            catch (e) { }
+                            pagingInfoSort = `&p_${orderFieldName}=${(0, _dependencies_1.encodeURIComponentEX)(orderFieldValue)}`;
+                        }
+                    }
+                    data.query.ListItemCollectionPosition = {
+                        "PagingInfo": `Paged=TRUE${pagingInfoSort}&p_ID=${lastItemId}`
+                    };
+                    if (itemsLeftToGet < batchSize) //the last batch should be smaller, update the row limit
+                        data.query.ViewXml = camlQuery.replace("</View>", `<RowLimit>${itemsLeftToGet}</RowLimit></View>`);
+                }
+                else
+                    data = null;
+            }
+            catch (e) {
+                if ((0, common_1.__getSPRestErrorData)(e).code.indexOf('SPQueryThrottledException')) {
+                    //test again - on awaiting score view, this will work but will ONLY return the order by fields...
+                    // if (!triedWithoutViewFields) {
+                    //     //Our own issues list had too many lookup fields.
+                    //     logger.info("Query throttled, trying again without view fields... some fields might be missing from results.");
+                    //     triedWithoutViewFields = true;
+                    //     camlQuery = EnsureViewFields(camlQuery, orderByStatement.map(o => o.Name), false, true);
+                    //     data.query.ViewXml = camlQuery.replace("</View>", `<RowLimit>${batchSize}</RowLimit></View>`);
+                    // }
+                    // else
+                    if (!postProcessOrderBy && orderByStatement.length > 0) {
+                        logger.warn("Query throttled, trying again without order by...");
+                        postProcessOrderBy = true;
+                        camlQuery = (0, _dependencies_1.RemoveOrderByFromCaml)(camlQuery);
+                        camlQuery = (0, _dependencies_1.EnsureViewFields)(camlQuery, [], false, true);
+                        data.query.ViewXml = camlQuery.replace("</View>", `<RowLimit>${batchSize}</RowLimit></View>`);
+                    }
+                    else
+                        throw e; //still throttled? might be due to filter query
+                }
+                else
+                    throw e; //different error
+            }
+        } while (!(0, _dependencies_1.isNullOrEmptyString)(data));
+        return { items: items, postProcessOrderBy: postProcessOrderBy };
+    });
+}
+//# sourceMappingURL=GetListItemsByCaml.js.map