n8n-nodes-base 1.14.1 → 1.15.1

package/dist/nodes/HttpRequest/V3/HttpRequestV3.node.js

@@ -2,8 +2,8 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.HttpRequestV3 = void 0;
 const n8n_workflow_1 = require("n8n-workflow");
-const utilities_1 = require("../../../utils/utilities");
 const GenericFunctions_1 = require("../GenericFunctions");
+const utilities_1 = require("../../../utils/utilities");
 function toText(data) {
     if (typeof data === 'object' && data !== null) {
         return JSON.stringify(data);
@@ -18,7 +18,7 @@ class HttpRequestV3 {
             version: [3, 4, 4.1],
             defaults: {
                 name: 'HTTP Request',
-                color: '#2200DD',
+                color: '#0004F5',
             },
             inputs: ['main'],
             outputs: ['main'],
@@ -873,6 +873,225 @@ class HttpRequestV3 {
                                 },
                             ],
                         },
+                        {
+                            displayName: 'Pagination',
+                            name: 'pagination',
+                            placeholder: 'Add pagination',
+                            type: 'fixedCollection',
+                            typeOptions: {
+                                multipleValues: false,
+                            },
+                            default: {
+                                pagination: {},
+                            },
+                            options: [
+                                {
+                                    displayName: 'Pagination',
+                                    name: 'pagination',
+                                    values: [
+                                        {
+                                            displayName: 'Pagination Mode',
+                                            name: 'paginationMode',
+                                            type: 'options',
+                                            typeOptions: {
+                                                noDataExpression: true,
+                                            },
+                                            options: [
+                                                {
+                                                    name: 'Off',
+                                                    value: 'off',
+                                                },
+                                                {
+                                                    name: 'Update a Parameter in Each Request',
+                                                    value: 'updateAParameterInEachRequest',
+                                                },
+                                                {
+                                                    name: 'Response Contains Next URL',
+                                                    value: 'responseContainsNextURL',
+                                                },
+                                            ],
+                                            default: 'updateAParameterInEachRequest',
+                                            description: 'If pagination should be used',
+                                        },
+                                        {
+                                            displayName: 'Use the $response variables to access the data of the previous response. <a href="https://docs.n8n.io/code/builtin/http-node-variables/?utm_source=n8n_app&utm_medium=node_settings_modal-credential_link&utm_campaign=n8n-nodes-base.httpRequest" target="_blank">More info</a>',
+                                            name: 'webhookNotice',
+                                            displayOptions: {
+                                                hide: {
+                                                    paginationMode: ['off'],
+                                                },
+                                            },
+                                            type: 'notice',
+                                            default: '',
+                                        },
+                                        {
+                                            displayName: 'Next URL',
+                                            name: 'nextURL',
+                                            type: 'string',
+                                            displayOptions: {
+                                                show: {
+                                                    paginationMode: ['responseContainsNextURL'],
+                                                },
+                                            },
+                                            default: '',
+                                            description: 'Should evaluate to true when pagination is complete. More info.',
+                                        },
+                                        {
+                                            displayName: 'Parameters',
+                                            name: 'parameters',
+                                            type: 'fixedCollection',
+                                            displayOptions: {
+                                                show: {
+                                                    paginationMode: ['updateAParameterInEachRequest'],
+                                                },
+                                            },
+                                            typeOptions: {
+                                                multipleValues: true,
+                                                noExpression: true,
+                                            },
+                                            placeholder: 'Add Parameter',
+                                            default: {
+                                                parameters: [
+                                                    {
+                                                        type: 'qs',
+                                                        name: '',
+                                                        value: '',
+                                                    },
+                                                ],
+                                            },
+                                            options: [
+                                                {
+                                                    name: 'parameters',
+                                                    displayName: 'Parameter',
+                                                    values: [
+                                                        {
+                                                            displayName: 'Type',
+                                                            name: 'type',
+                                                            type: 'options',
+                                                            options: [
+                                                                {
+                                                                    name: 'Body',
+                                                                    value: 'body',
+                                                                },
+                                                                {
+                                                                    name: 'Header',
+                                                                    value: 'headers',
+                                                                },
+                                                                {
+                                                                    name: 'Query',
+                                                                    value: 'qs',
+                                                                },
+                                                            ],
+                                                            default: 'qs',
+                                                            description: 'Where the parameter should be set',
+                                                        },
+                                                        {
+                                                            displayName: 'Name',
+                                                            name: 'name',
+                                                            type: 'string',
+                                                            default: '',
+                                                        },
+                                                        {
+                                                            displayName: 'Value',
+                                                            name: 'value',
+                                                            type: 'string',
+                                                            default: '',
+                                                        },
+                                                    ],
+                                                },
+                                            ],
+                                        },
+                                        {
+                                            displayName: 'Pagination Complete When',
+                                            name: 'paginationCompleteWhen',
+                                            type: 'options',
+                                            typeOptions: {
+                                                noDataExpression: true,
+                                            },
+                                            displayOptions: {
+                                                hide: {
+                                                    paginationMode: ['off'],
+                                                },
+                                            },
+                                            options: [
+                                                {
+                                                    name: 'Response Is Empty',
+                                                    value: 'responseIsEmpty',
+                                                },
+                                                {
+                                                    name: 'Receive Specific Status Code(s)',
+                                                    value: 'receiveSpecificStatusCodes',
+                                                },
+                                                {
+                                                    name: 'Other',
+                                                    value: 'other',
+                                                },
+                                            ],
+                                            default: 'responseIsEmpty',
+                                            description: 'When should no further requests be made?',
+                                        },
+                                        {
+                                            displayName: 'Status Code(s) when Complete',
+                                            name: 'statusCodesWhenComplete',
+                                            type: 'string',
+                                            typeOptions: {
+                                                noDataExpression: true,
+                                            },
+                                            displayOptions: {
+                                                show: {
+                                                    paginationCompleteWhen: ['receiveSpecificStatusCodes'],
+                                                },
+                                            },
+                                            default: '',
+                                            description: 'Accepts comma-separated values',
+                                        },
+                                        {
+                                            displayName: 'Complete Expression',
+                                            name: 'completeExpression',
+                                            type: 'string',
+                                            displayOptions: {
+                                                show: {
+                                                    paginationCompleteWhen: ['other'],
+                                                },
+                                            },
+                                            default: '',
+                                            description: 'Should evaluate to true when pagination is complete. More info.',
+                                        },
+                                        {
+                                            displayName: 'Limit Pages Fetched',
+                                            name: 'limitPagesFetched',
+                                            type: 'boolean',
+                                            typeOptions: {
+                                                noDataExpression: true,
+                                            },
+                                            displayOptions: {
+                                                hide: {
+                                                    paginationMode: ['off'],
+                                                },
+                                            },
+                                            default: false,
+                                            noDataExpression: true,
+                                            description: 'Whether the number of requests should be limited',
+                                        },
+                                        {
+                                            displayName: 'Max Pages',
+                                            name: 'maxRequests',
+                                            type: 'number',
+                                            typeOptions: {
+                                                noDataExpression: true,
+                                            },
+                                            displayOptions: {
+                                                show: {
+                                                    limitPagesFetched: [true],
+                                                },
+                                            },
+                                            default: 100,
+                                            description: 'Maximum amount of request to be make',
+                                        },
+                                    ],
+                                },
+                            ],
+                        },
                         {
                             displayName: 'Proxy',
                             name: 'proxy',
@@ -978,6 +1197,9 @@ class HttpRequestV3 {
         const requestPromises = [];
         let fullResponse = false;
         let autoDetectResponseFormat = false;
+        const pagination = this.getNodeParameter('options.pagination.pagination', 0, null, {
+            rawExpressions: true,
+        });
         for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
             const requestMethod = this.getNodeParameter('method', itemIndex);
             const sendQuery = this.getNodeParameter('sendQuery', itemIndex, false);
@@ -1013,10 +1235,8 @@ class HttpRequestV3 {
                 gzip: true,
                 rejectUnauthorized: !allowUnauthorizedCerts || false,
                 followRedirect: false,
+                resolveWithFullResponse: true,
             };
-            if (autoDetectResponseFormat || fullResponse) {
-                requestOptions.resolveWithFullResponse = true;
-            }
             if (requestOptions.method !== 'GET' && nodeVersion >= 4.1) {
                 requestOptions = { ...requestOptions, followAllRedirects: false };
             }
@@ -1243,7 +1463,50 @@ class HttpRequestV3 {
                 this.sendMessageToUI((0, GenericFunctions_1.sanitizeUiMessage)(requestOptions, authDataKeys));
             }
             catch (e) { }
-            if (authentication === 'genericCredentialType' || authentication === 'none') {
+            if (pagination && pagination.paginationMode !== 'off') {
+                let continueExpression = '={{false}}';
+                if (pagination.paginationCompleteWhen === 'receiveSpecificStatusCodes') {
+                    const statusCodesWhenCompleted = pagination.statusCodesWhenComplete
+                        .split(',')
+                        .map((item) => parseInt(item.trim()));
+                    continueExpression = `={{ !${JSON.stringify(statusCodesWhenCompleted)}.includes($response.statusCode) }}`;
+                }
+                else if (pagination.paginationCompleteWhen === 'responseIsEmpty') {
+                    continueExpression =
+                        '={{ Array.isArray($response.body) ? $response.body.length : !!$response.body }}';
+                }
+                else {
+                    if (!pagination.completeExpression.length || pagination.completeExpression[0] !== '=') {
+                        throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Invalid or empty Complete Expression');
+                    }
+                    continueExpression = `={{ !(${pagination.completeExpression.trim().slice(3, -2)}) }}`;
+                }
+                const paginationData = {
+                    continue: continueExpression,
+                    request: {},
+                };
+                if (pagination.paginationMode === 'updateAParameterInEachRequest') {
+                    paginationData.request = {};
+                    pagination.parameters.parameters.forEach((parameter) => {
+                        if (!paginationData.request[parameter.type]) {
+                            paginationData.request[parameter.type] = {};
+                        }
+                        paginationData.request[parameter.type][parameter.name] = parameter.value;
+                    });
+                }
+                else if (pagination.paginationMode === 'responseContainsNextURL') {
+                    paginationData.request.url = pagination.nextURL;
+                }
+                if (pagination.limitPagesFetched) {
+                    paginationData.maxRequests = pagination.maxRequests;
+                }
+                if (responseFormat === 'file') {
+                    paginationData.binaryResult = true;
+                }
+                const requestPromise = this.helpers.requestWithAuthenticationPaginated.call(this, requestOptions, itemIndex, paginationData, nodeCredentialType);
+                requestPromises.push(requestPromise);
+            }
+            else if (authentication === 'genericCredentialType' || authentication === 'none') {
                 if (oAuth1Api) {
                     const requestOAuth1 = this.helpers.requestOAuth1.call(this, 'oAuth1Api', requestOptions);
                     requestOAuth1.catch(() => { });
@@ -1270,25 +1533,25 @@ class HttpRequestV3 {
             }
         }
         const promisesResponses = await Promise.allSettled(requestPromises);
-        let response;
+        let responseData;
         for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
-            response = promisesResponses.shift();
-            if (response.status !== 'fulfilled') {
-                if (response.reason.statusCode === 429) {
-                    response.reason.message =
+            responseData = promisesResponses.shift();
+            if (responseData.status !== 'fulfilled') {
+                if (responseData.reason.statusCode === 429) {
+                    responseData.reason.message =
                         "Try spacing your requests out using the batching settings under 'Options'";
                 }
                 if (!this.continueOnFail()) {
-                    if (autoDetectResponseFormat && response.reason.error instanceof Buffer) {
-                        response.reason.error = Buffer.from(response.reason.error).toString();
+                    if (autoDetectResponseFormat && responseData.reason.error instanceof Buffer) {
+                        responseData.reason.error = Buffer.from(responseData.reason.error).toString();
                     }
-                    throw new n8n_workflow_1.NodeApiError(this.getNode(), response, { itemIndex });
+                    throw new n8n_workflow_1.NodeApiError(this.getNode(), responseData, { itemIndex });
                 }
                 else {
-                    (0, n8n_workflow_1.removeCircularRefs)(response.reason);
+                    (0, n8n_workflow_1.removeCircularRefs)(responseData.reason);
                     returnItems.push({
                         json: {
-                            error: response.reason,
+                            error: responseData.reason,
                         },
                         pairedItem: {
                             item: itemIndex,
@@ -1297,147 +1560,171 @@ class HttpRequestV3 {
                     continue;
                 }
             }
-            response = response.value;
-            let responseFormat = this.getNodeParameter('options.response.response.responseFormat', 0, 'autodetect');
-            fullResponse = this.getNodeParameter('options.response.response.fullResponse', 0, false);
-            if (autoDetectResponseFormat) {
-                const responseContentType = (_k = response.headers['content-type']) !== null && _k !== void 0 ? _k : '';
-                if (responseContentType.includes('application/json')) {
-                    responseFormat = 'json';
-                    const neverError = this.getNodeParameter('options.response.response.neverError', 0, false);
-                    const data = await this.helpers
-                        .binaryToBuffer(response.body)
-                        .then((body) => body.toString());
-                    response.body = (0, n8n_workflow_1.jsonParse)(data, {
-                        ...(neverError
-                            ? { fallbackValue: {} }
-                            : { errorMessage: 'Invalid JSON in response body' }),
-                    });
-                }
-                else if (GenericFunctions_1.binaryContentTypes.some((e) => responseContentType.includes(e))) {
-                    responseFormat = 'file';
-                }
-                else {
-                    responseFormat = 'text';
-                    const data = await this.helpers
-                        .binaryToBuffer(response.body)
-                        .then((body) => body.toString());
-                    response.body = !data ? undefined : data;
-                }
+            let responses;
+            if (Array.isArray(responseData.value)) {
+                responses = responseData.value;
             }
-            if (autoDetectResponseFormat && !fullResponse) {
-                delete response.headers;
-                delete response.statusCode;
-                delete response.statusMessage;
-                response = response.body;
-                requestOptions.resolveWithFullResponse = false;
+            else {
+                responses = [responseData.value];
             }
-            if (responseFormat === 'file') {
-                const outputPropertyName = this.getNodeParameter('options.response.response.outputPropertyName', 0, 'data');
-                const newItem = {
-                    json: {},
-                    binary: {},
-                    pairedItem: {
-                        item: itemIndex,
-                    },
-                };
-                if (items[itemIndex].binary !== undefined) {
-                    Object.assign(newItem.binary, items[itemIndex].binary);
+            let responseFormat = this.getNodeParameter('options.response.response.responseFormat', 0, 'autodetect');
+            fullResponse = this.getNodeParameter('options.response.response.fullResponse', 0, false);
+            for (let [index, response] of Object.entries(responses)) {
+                if (this.getMode() === 'manual' && index === '0') {
+                    const nodeContext = this.getContext('node');
+                    if (pagination && pagination.paginationMode !== 'off') {
+                        nodeContext.response = responseData.value[0];
+                    }
+                    else {
+                        nodeContext.response = responseData.value;
+                    }
                 }
-                let binaryData;
-                if (fullResponse) {
-                    const returnItem = {};
-                    for (const property of fullResponseProperties) {
-                        if (property === 'body') {
-                            continue;
+                if (autoDetectResponseFormat) {
+                    const responseContentType = (_k = response.headers['content-type']) !== null && _k !== void 0 ? _k : '';
+                    if (responseContentType.includes('application/json')) {
+                        responseFormat = 'json';
+                        if (!response.__bodyResolved) {
+                            const neverError = this.getNodeParameter('options.response.response.neverError', 0, false);
+                            const data = await this.helpers
+                                .binaryToBuffer(response.body)
+                                .then((body) => body.toString());
+                            response.body = (0, n8n_workflow_1.jsonParse)(data, {
+                                ...(neverError
+                                    ? { fallbackValue: {} }
+                                    : { errorMessage: 'Invalid JSON in response body' }),
+                            });
                         }
-                        returnItem[property] = response[property];
                     }
-                    newItem.json = returnItem;
-                    binaryData = response.body;
-                }
-                else {
-                    newItem.json = items[itemIndex].json;
-                    binaryData = response;
-                }
-                newItem.binary[outputPropertyName] = await this.helpers.prepareBinaryData(binaryData);
-                returnItems.push(newItem);
-            }
-            else if (responseFormat === 'text') {
-                const outputPropertyName = this.getNodeParameter('options.response.response.outputPropertyName', 0, 'data');
-                if (fullResponse) {
-                    const returnItem = {};
-                    for (const property of fullResponseProperties) {
-                        if (property === 'body') {
-                            returnItem[outputPropertyName] = toText(response[property]);
-                            continue;
+                    else if (GenericFunctions_1.binaryContentTypes.some((e) => responseContentType.includes(e))) {
+                        responseFormat = 'file';
+                    }
+                    else {
+                        responseFormat = 'text';
+                        if (!response.__bodyResolved) {
+                            const data = await this.helpers
+                                .binaryToBuffer(response.body)
+                                .then((body) => body.toString());
+                            response.body = !data ? undefined : data;
                         }
-                        returnItem[property] = response[property];
                     }
-                    returnItems.push({
-                        json: returnItem,
-                        pairedItem: {
-                            item: itemIndex,
-                        },
-                    });
                 }
-                else {
-                    returnItems.push({
-                        json: {
-                            [outputPropertyName]: toText(response),
-                        },
+                if (autoDetectResponseFormat && !fullResponse) {
+                    delete response.headers;
+                    delete response.statusCode;
+                    delete response.statusMessage;
+                }
+                if (!fullResponse) {
+                    response = response.body;
+                }
+                if (responseFormat === 'file') {
+                    const outputPropertyName = this.getNodeParameter('options.response.response.outputPropertyName', 0, 'data');
+                    const newItem = {
+                        json: {},
+                        binary: {},
                         pairedItem: {
                             item: itemIndex,
                         },
-                    });
-                }
-            }
-            else {
-                if (requestOptions.resolveWithFullResponse === true) {
-                    const returnItem = {};
-                    for (const property of fullResponseProperties) {
-                        returnItem[property] = response[property];
+                    };
+                    if (items[itemIndex].binary !== undefined) {
+                        Object.assign(newItem.binary, items[itemIndex].binary);
                     }
-                    if (responseFormat === 'json' && typeof returnItem.body === 'string') {
-                        try {
-                            returnItem.body = JSON.parse(returnItem.body);
-                        }
-                        catch (error) {
-                            throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Response body is not valid JSON. Change "Response Format" to "Text"', { itemIndex });
+                    let binaryData;
+                    if (fullResponse) {
+                        const returnItem = {};
+                        for (const property of fullResponseProperties) {
+                            if (property === 'body') {
+                                continue;
+                            }
+                            returnItem[property] = response[property];
                         }
+                        newItem.json = returnItem;
+                        binaryData = response === null || response === void 0 ? void 0 : response.body;
                     }
-                    returnItems.push({
-                        json: returnItem,
-                        pairedItem: {
-                            item: itemIndex,
-                        },
-                    });
+                    else {
+                        newItem.json = items[itemIndex].json;
+                        binaryData = response;
+                    }
+                    newItem.binary[outputPropertyName] = await this.helpers.prepareBinaryData(binaryData);
+                    returnItems.push(newItem);
                 }
-                else {
-                    if (responseFormat === 'json' && typeof response === 'string') {
-                        try {
-                            response = JSON.parse(response);
-                        }
-                        catch (error) {
-                            throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Response body is not valid JSON. Change "Response Format" to "Text"', { itemIndex });
+                else if (responseFormat === 'text') {
+                    const outputPropertyName = this.getNodeParameter('options.response.response.outputPropertyName', 0, 'data');
+                    if (fullResponse) {
+                        const returnItem = {};
+                        for (const property of fullResponseProperties) {
+                            if (property === 'body') {
+                                returnItem[outputPropertyName] = toText(response[property]);
+                                continue;
+                            }
+                            returnItem[property] = response[property];
                         }
-                    }
-                    if (Array.isArray(response)) {
-                        response.forEach((item) => returnItems.push({
-                            json: item,
+                        returnItems.push({
+                            json: returnItem,
                             pairedItem: {
                                 item: itemIndex,
                             },
-                        }));
+                        });
                     }
                     else {
                         returnItems.push({
-                            json: response,
+                            json: {
+                                [outputPropertyName]: toText(response),
+                            },
+                            pairedItem: {
+                                item: itemIndex,
+                            },
+                        });
+                    }
+                }
+                else {
+                    if (fullResponse) {
+                        const returnItem = {};
+                        for (const property of fullResponseProperties) {
+                            returnItem[property] = response[property];
+                        }
+                        if (responseFormat === 'json' && typeof returnItem.body === 'string') {
+                            try {
+                                returnItem.body = JSON.parse(returnItem.body);
+                            }
+                            catch (error) {
+                                throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Response body is not valid JSON. Change "Response Format" to "Text"', { itemIndex });
+                            }
+                        }
+                        returnItems.push({
+                            json: returnItem,
                             pairedItem: {
                                 item: itemIndex,
                             },
                         });
                     }
+                    else {
+                        if (responseFormat === 'json' && typeof response === 'string') {
+                            try {
+                                if (typeof response !== 'object') {
+                                    response = JSON.parse(response);
+                                }
+                            }
+                            catch (error) {
+                                throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Response body is not valid JSON. Change "Response Format" to "Text"', { itemIndex });
+                            }
+                        }
+                        if (Array.isArray(response)) {
+                            response.forEach((item) => returnItems.push({
+                                json: item,
+                                pairedItem: {
+                                    item: itemIndex,
+                                },
+                            }));
+                        }
+                        else {
+                            returnItems.push({
+                                json: response,
+                                pairedItem: {
+                                    item: itemIndex,
+                                },
+                            });
+                        }
+                    }
                 }
             }
         }