@elastic/elasticsearch 7.7.0-rc.2 → 7.8.0

package/lib/Helpers.js

@@ -4,13 +4,16 @@
 
 'use strict'
 
+/* eslint camelcase: 0 */
+
+const { Readable } = require('stream')
 const { promisify } = require('util')
 const { ResponseError, ConfigurationError } = require('./errors')
 
 const pImmediate = promisify(setImmediate)
 const sleep = promisify(setTimeout)
-const kGetHits = Symbol('elasticsearch-get-hits')
 const kClient = Symbol('elasticsearch-client')
+/* istanbul ignore next */
 const noop = () => {}
 
 class Helpers {
@@ -19,23 +22,22 @@ class Helpers {
     this.maxRetries = opts.maxRetries
   }
 
-  [kGetHits] (body) {
-    if (body.hits && body.hits.hits) {
-      return body.hits.hits.map(d => d._source)
-    }
-    return []
-  }
-
   /**
    * Runs a search operation. The only difference between client.search and this utility,
    * is that we are only returning the hits to the user and not the full ES response.
+   * This helper automatically adds `filter_path=hits.hits._source` to the querystring,
+   * as it will only need the documents source.
    * @param {object} params - The Elasticsearch's search parameters.
    * @param {object} options - The client optional configuration for this request.
    * @return {array} The documents that matched the request.
    */
   async search (params, options) {
-    const response = await this[kClient].search(params, options)
-    return this[kGetHits](response.body)
+    appendFilterPath('hits.hits._source', params, true)
+    const { body } = await this[kClient].search(params, options)
+    if (body.hits && body.hits.hits) {
+      return body.hits.hits.map(d => d._source)
+    }
+    return []
   }
 
   /**
@@ -63,6 +65,8 @@ class Helpers {
       options.ignore = [429]
     }
     params.scroll = params.scroll || '1m'
+    appendFilterPath('_scroll_id', params, false)
+    const { method, body, index, ...querystring } = params
 
     let response = null
     for (let i = 0; i < maxRetries; i++) {
@@ -74,33 +78,31 @@ class Helpers {
       throw new ResponseError(response)
     }
 
-    let scrollId = response.body._scroll_id
+    let scroll_id = response.body._scroll_id
     let stop = false
     const clear = async () => {
       stop = true
       await this[kClient].clearScroll(
-        { body: { scroll_id: scrollId } },
+        { body: { scroll_id } },
         { ignore: [400] }
       )
     }
 
-    while (response.body.hits.hits.length > 0) {
-      scrollId = response.body._scroll_id
+    while (response.body.hits && response.body.hits.hits.length > 0) {
+      scroll_id = response.body._scroll_id
       response.clear = clear
-      response.documents = this[kGetHits](response.body)
+      addDocumentsGetter(response)
 
       yield response
 
-      if (!scrollId || stop === true) {
+      if (!scroll_id || stop === true) {
         break
       }
 
       for (let i = 0; i < maxRetries; i++) {
         response = await this[kClient].scroll({
-          scroll: params.scroll,
-          body: {
-            scroll_id: scrollId
-          }
+          ...querystring,
+          body: { scroll_id }
         }, options)
         if (response.statusCode !== 429) break
         await sleep(wait)
@@ -120,11 +122,14 @@ class Helpers {
    * }
    * ```
    * Each document is what you will find by running a scrollSearch and iterating on the hits array.
+   * This helper automatically adds `filter_path=hits.hits._source` to the querystring,
+   * as it will only need the documents source.
    * @param {object} params - The Elasticsearch's search parameters.
    * @param {object} options - The client optional configuration for this request.
    * @return {iterator} the async iterator
    */
   async * scrollDocuments (params, options) {
+    appendFilterPath('hits.hits._source', params, true)
     for await (const { documents } of this.scrollSearch(params)) {
       for (const document of documents) {
         yield document
@@ -132,21 +137,266 @@ class Helpers {
     }
   }
 
+  /**
+   * Creates a msearch helper instance. Once you configure it, you can use the provided
+   * `search` method to add new searches in the queue.
+   * @param {object} options - The configuration of the msearch operations.
+   * @return {object} The possible operations to run.
+   */
+  msearch (options = {}) {
+    const client = this[kClient]
+    const {
+      operations = 5,
+      concurrency = 5,
+      flushInterval = 500,
+      retries = this.maxRetries,
+      wait = 5000,
+      ...msearchOptions
+    } = options
+
+    let stopReading = false
+    let stopError = null
+    let timeoutRef = null
+    const operationsStream = new Readable({
+      objectMode: true,
+      read (size) {}
+    })
+
+    const p = iterate()
+    const helper = {
+      then (onFulfilled, onRejected) {
+        return p.then(onFulfilled, onRejected)
+      },
+      catch (onRejected) {
+        return p.catch(onRejected)
+      },
+      stop (error = null) {
+        if (stopReading === true) return
+        stopReading = true
+        stopError = error
+        operationsStream.push(null)
+      },
+      // TODO: support abort a single search?
+      // NOTE: the validation checks are synchronous and the callback/promise will
+      //       be resolved in the same tick. We might want to fix this in the future.
+      search (header, body, callback) {
+        if (stopReading === true) {
+          const error = stopError === null
+            ? new ConfigurationError('The msearch processor has been stopped')
+            : stopError
+          return callback ? callback(error, {}) : Promise.reject(error)
+        }
+
+        if (!(typeof header === 'object' && header !== null && !Array.isArray(header))) {
+          const error = new ConfigurationError('The header should be an object')
+          return callback ? callback(error, {}) : Promise.reject(error)
+        }
+
+        if (!(typeof body === 'object' && body !== null && !Array.isArray(body))) {
+          const error = new ConfigurationError('The body should be an object')
+          return callback ? callback(error, {}) : Promise.reject(error)
+        }
+
+        let promise = null
+        if (callback === undefined) {
+          let onFulfilled = null
+          let onRejected = null
+          promise = new Promise((resolve, reject) => {
+            onFulfilled = resolve
+            onRejected = reject
+          })
+          callback = function callback (err, result) {
+            err ? onRejected(err) : onFulfilled(result)
+          }
+        }
+
+        operationsStream.push([header, body, callback])
+
+        if (promise !== null) {
+          return promise
+        }
+      }
+    }
+
+    return helper
+
+    async function iterate () {
+      const { semaphore, finish } = buildSemaphore()
+      const msearchBody = []
+      const callbacks = []
+      let loadedOperations = 0
+      timeoutRef = setTimeout(onFlushTimeout, flushInterval)
+
+      for await (const operation of operationsStream) {
+        timeoutRef.refresh()
+        loadedOperations += 1
+        msearchBody.push(operation[0], operation[1])
+        callbacks.push(operation[2])
+        if (loadedOperations >= operations) {
+          const send = await semaphore()
+          send(msearchBody.slice(), callbacks.slice())
+          msearchBody.length = 0
+          callbacks.length = 0
+          loadedOperations = 0
+        }
+      }
+
+      clearTimeout(timeoutRef)
+      // In some cases the previos http call does not have finished,
+      // or we didn't reach the flush bytes threshold, so we force one last operation.
+      if (loadedOperations > 0) {
+        const send = await semaphore()
+        send(msearchBody, callbacks)
+      }
+
+      await finish()
+
+      if (stopError !== null) {
+        throw stopError
+      }
+
+      async function onFlushTimeout () {
+        if (loadedOperations === 0) return
+        const msearchBodyCopy = msearchBody.slice()
+        const callbacksCopy = callbacks.slice()
+        msearchBody.length = 0
+        callbacks.length = 0
+        loadedOperations = 0
+        try {
+          const send = await semaphore()
+          send(msearchBodyCopy, callbacksCopy)
+        } catch (err) {
+          /* istanbul ignore next */
+          helper.stop(err)
+        }
+      }
+    }
+
+    // This function builds a semaphore using the concurrency
+    // options of the msearch helper. It is used inside the iterator
+    // to guarantee that no more than the number of operations
+    // allowed to run at the same time are executed.
+    // It returns a semaphore function which resolves in the next tick
+    // if we didn't reach the maximim concurrency yet, otherwise it returns
+    // a promise that resolves as soon as one of the running request has finshed.
+    // The semaphore function resolves a send function, which will be used
+    // to send the actual msearch request.
+    // It also returns a finish function, which returns a promise that is resolved
+    // when there are no longer request running.
+    function buildSemaphore () {
+      let resolveSemaphore = null
+      let resolveFinish = null
+      let running = 0
+
+      return { semaphore, finish }
+
+      function finish () {
+        return new Promise((resolve, reject) => {
+          if (running === 0) {
+            resolve()
+          } else {
+            resolveFinish = resolve
+          }
+        })
+      }
+
+      function semaphore () {
+        if (running < concurrency) {
+          running += 1
+          return pImmediate(send)
+        } else {
+          return new Promise((resolve, reject) => {
+            resolveSemaphore = resolve
+          })
+        }
+      }
+
+      function send (msearchBody, callbacks) {
+        /* istanbul ignore if */
+        if (running > concurrency) {
+          throw new Error('Max concurrency reached')
+        }
+        msearchOperation(msearchBody, callbacks, () => {
+          running -= 1
+          if (resolveSemaphore) {
+            running += 1
+            resolveSemaphore(send)
+            resolveSemaphore = null
+          } else if (resolveFinish && running === 0) {
+            resolveFinish()
+          }
+        })
+      }
+    }
+
+    function msearchOperation (msearchBody, callbacks, done) {
+      let retryCount = retries
+
+      // Instead of going full on async-await, which would make the code easier to read,
+      // we have decided to use callback style instead.
+      // This because every time we use async await, V8 will create multiple promises
+      // behind the scenes, making the code slightly slower.
+      tryMsearch(msearchBody, callbacks, retrySearch)
+      function retrySearch (msearchBody, callbacks) {
+        if (msearchBody.length > 0 && retryCount > 0) {
+          retryCount -= 1
+          setTimeout(tryMsearch, wait, msearchBody, callbacks, retrySearch)
+          return
+        }
+
+        done()
+      }
+
+      // This function never returns an error, if the msearch operation fails,
+      // the error is dispatched to all search executors.
+      function tryMsearch (msearchBody, callbacks, done) {
+        client.msearch(Object.assign({}, msearchOptions, { body: msearchBody }), (err, results) => {
+          const retryBody = []
+          const retryCallbacks = []
+          if (err) {
+            addDocumentsGetter(results)
+            for (const callback of callbacks) {
+              callback(err, results)
+            }
+            return done(retryBody, retryCallbacks)
+          }
+          const { responses } = results.body
+          for (let i = 0, len = responses.length; i < len; i++) {
+            const response = responses[i]
+            if (response.status === 429 && retryCount > 0) {
+              retryBody.push(msearchBody[i * 2])
+              retryBody.push(msearchBody[(i * 2) + 1])
+              retryCallbacks.push(callbacks[i])
+              continue
+            }
+            const result = { ...results, body: response }
+            addDocumentsGetter(result)
+            if (response.status >= 400) {
+              callbacks[i](new ResponseError(result), result)
+            } else {
+              callbacks[i](null, result)
+            }
+          }
+          done(retryBody, retryCallbacks)
+        })
+      }
+    }
+  }
+
   /**
    * Creates a bulk helper instance. Once you configure it, you can pick which operation
    * to execute with the given dataset, index, create, update, and delete.
    * @param {object} options - The configuration of the bulk operation.
-   * @return {object} The possible orations to run with the datasource.
+   * @return {object} The possible operations to run with the datasource.
    */
   bulk (options) {
-    // TODO: add an interval to force flush the body
-    //       to handle the slow producer problem
     const client = this[kClient]
     const { serialize, deserialize } = client.serializer
     const {
       datasource,
       onDocument,
       flushBytes = 5000000,
+      flushInterval = 30000,
       concurrency = 5,
       retries = this.maxRetries,
       wait = 5000,
@@ -166,6 +416,7 @@ class Helpers {
     }
 
     let shouldAbort = false
+    let timeoutRef = null
     const stats = {
       total: 0,
       failed: 0,
@@ -177,8 +428,7 @@ class Helpers {
     }
 
     const p = iterate()
-
-    return {
+    const helper = {
       then (onFulfilled, onRejected) {
         return p.then(onFulfilled, onRejected)
       },
@@ -186,12 +436,15 @@ class Helpers {
         return p.catch(onRejected)
       },
       abort () {
+        clearTimeout(timeoutRef)
         shouldAbort = true
         stats.aborted = true
         return this
       }
     }
 
+    return helper
+
     /**
      * Function that iterates over the given datasource and start a bulk operation as soon
      * as it reaches the configured bulk size. It's designed to use the Node.js asynchronous
@@ -208,9 +461,11 @@ class Helpers {
       let actionBody = ''
       let payloadBody = ''
       let chunkBytes = 0
+      timeoutRef = setTimeout(onFlushTimeout, flushInterval)
 
       for await (const chunk of datasource) {
         if (shouldAbort === true) break
+        timeoutRef.refresh()
         const action = onDocument(chunk)
         const operation = Array.isArray(action)
           ? Object.keys(action[0])[0]
@@ -219,21 +474,20 @@ class Helpers {
           actionBody = serialize(action)
           payloadBody = typeof chunk === 'string' ? chunk : serialize(chunk)
           chunkBytes += Buffer.byteLength(actionBody) + Buffer.byteLength(payloadBody)
-          bulkBody.push(actionBody)
-          bulkBody.push(payloadBody)
+          bulkBody.push(actionBody, payloadBody)
         } else if (operation === 'update') {
           actionBody = serialize(action[0])
           payloadBody = typeof chunk === 'string'
-            ? `{doc:${chunk}}`
+            ? `{"doc":${chunk}}`
             : serialize({ doc: chunk, ...action[1] })
           chunkBytes += Buffer.byteLength(actionBody) + Buffer.byteLength(payloadBody)
-          bulkBody.push(actionBody)
-          bulkBody.push(payloadBody)
+          bulkBody.push(actionBody, payloadBody)
         } else if (operation === 'delete') {
           actionBody = serialize(action)
           chunkBytes += Buffer.byteLength(actionBody)
           bulkBody.push(actionBody)
         } else {
+          clearTimeout(timeoutRef)
           throw new ConfigurationError(`Bulk helper invalid action: '${operation}'`)
         }
 
@@ -246,6 +500,7 @@ class Helpers {
         }
       }
 
+      clearTimeout(timeoutRef)
       // In some cases the previos http call does not have finished,
       // or we didn't reach the flush bytes threshold, so we force one last operation.
       if (shouldAbort === false && chunkBytes > 0) {
@@ -268,6 +523,21 @@ class Helpers {
       stats.total = stats.successful + stats.failed
 
       return stats
+
+      async function onFlushTimeout () {
+        if (chunkBytes === 0) return
+        stats.bytes += chunkBytes
+        const bulkBodyCopy = bulkBody.slice()
+        bulkBody.length = 0
+        chunkBytes = 0
+        try {
+          const send = await semaphore()
+          send(bulkBodyCopy)
+        } catch (err) {
+          /* istanbul ignore next */
+          helper.abort()
+        }
+      }
     }
 
     // This function builds a semaphore using the concurrency
@@ -308,6 +578,7 @@ class Helpers {
 
       function semaphore () {
         if (running < concurrency) {
+          running += 1
           return pImmediate(send)
         } else {
           return new Promise((resolve, reject) => {
@@ -318,10 +589,9 @@ class Helpers {
 
       function send (bulkBody) {
         /* istanbul ignore if */
-        if (running >= concurrency) {
+        if (running > concurrency) {
           throw new Error('Max concurrency reached')
         }
-        running += 1
         bulkOperation(bulkBody, err => {
           running -= 1
           if (err) {
@@ -329,6 +599,7 @@ class Helpers {
             error = err
           }
           if (resolveSemaphore) {
+            running += 1
             resolveSemaphore(send)
             resolveSemaphore = null
           } else if (resolveFinish && running === 0) {
@@ -371,6 +642,7 @@ class Helpers {
               operation: deserialize(bulkBody[i]),
               document: operation !== 'delete'
                 ? deserialize(bulkBody[i + 1])
+                /* istanbul ignore next */
                 : null,
               retried: isRetrying
             })
@@ -402,6 +674,7 @@ class Helpers {
               // but the ES node were handling too many operations.
               if (status === 429) {
                 retry.push(bulkBody[indexSlice])
+                /* istanbul ignore next */
                 if (operation !== 'delete') {
                   retry.push(bulkBody[indexSlice + 1])
                 }
@@ -428,4 +701,27 @@ class Helpers {
   }
 }
 
+// Using a getter will improve the overall performances of the code,
+// as we will reed the documents only if needed.
+function addDocumentsGetter (result) {
+  Object.defineProperty(result, 'documents', {
+    get () {
+      if (this.body.hits && this.body.hits.hits) {
+        return this.body.hits.hits.map(d => d._source)
+      }
+      return []
+    }
+  })
+}
+
+function appendFilterPath (filter, params, force) {
+  if (params.filter_path !== undefined) {
+    params.filter_path += ',' + filter
+  } else if (params.filterPath !== undefined) {
+    params.filterPath += ',' + filter
+  } else if (force === true) {
+    params.filter_path = filter
+  }
+}
+
 module.exports = Helpers

package/lib/Transport.js

@@ -22,7 +22,7 @@ const clientVersion = require('../package.json').version
 const userAgent = `elasticsearch-js/${clientVersion} (${os.platform()} ${os.release()}-${os.arch()}; Node.js ${process.version})`
 
 class Transport {
-  constructor (opts = {}) {
+  constructor (opts) {
     if (typeof opts.compression === 'string' && opts.compression !== 'gzip') {
       throw new ConfigurationError(`Invalid compression: '${opts.compression}'`)
     }
@@ -34,9 +34,9 @@ class Transport {
     this.suggestCompression = opts.suggestCompression === true
     this.compression = opts.compression || false
     this.headers = Object.assign({},
-      { 'User-Agent': userAgent },
-      opts.suggestCompression === true ? { 'Accept-Encoding': 'gzip,deflate' } : null,
-      opts.headers
+      { 'user-agent': userAgent },
+      opts.suggestCompression === true ? { 'accept-encoding': 'gzip,deflate' } : null,
+      lowerCaseHeaders(opts.headers)
     )
     this.sniffInterval = opts.sniffInterval
     this.sniffOnConnectionFault = opts.sniffOnConnectionFault
@@ -51,7 +51,6 @@ class Transport {
     } else if (opts.nodeSelector === 'round-robin') {
       this.nodeSelector = roundRobinSelector()
     } else if (opts.nodeSelector === 'random') {
-      /* istanbul ignore next */
       this.nodeSelector = randomSelector
     } else {
       this.nodeSelector = roundRobinSelector()
@@ -243,10 +242,10 @@ class Transport {
       })
     }
 
-    const headers = Object.assign({}, this.headers, options.headers)
+    const headers = Object.assign({}, this.headers, lowerCaseHeaders(options.headers))
 
     if (options.opaqueId !== undefined) {
-      headers['X-Opaque-Id'] = this.opaqueIdPrefix !== null
+      headers['x-opaque-id'] = this.opaqueIdPrefix !== null
         ? this.opaqueIdPrefix + options.opaqueId
         : options.opaqueId
     }
@@ -262,7 +261,7 @@ class Transport {
       }
 
       if (params.body !== '') {
-        headers['Content-Type'] = headers['Content-Type'] || 'application/json'
+        headers['content-type'] = headers['content-type'] || 'application/json'
       }
 
     // handle ndjson body
@@ -277,7 +276,7 @@ class Transport {
         params.body = params.bulkBody
       }
       if (params.body !== '') {
-        headers['Content-Type'] = headers['Content-Type'] || 'application/x-ndjson'
+        headers['content-type'] = headers['content-type'] || 'application/x-ndjson'
       }
     }
 
@@ -301,7 +300,7 @@ class Transport {
     if (params.body !== '' && params.body != null) {
       if (isStream(params.body) === true) {
         if (compression === 'gzip') {
-          params.headers['Content-Encoding'] = compression
+          params.headers['content-encoding'] = compression
           params.body = params.body.pipe(createGzip())
         }
         makeRequest()
@@ -311,13 +310,13 @@ class Transport {
           if (err) {
             return callback(err, result)
           }
-          params.headers['Content-Encoding'] = compression
-          params.headers['Content-Length'] = '' + Buffer.byteLength(buffer)
+          params.headers['content-encoding'] = compression
+          params.headers['content-length'] = '' + Buffer.byteLength(buffer)
           params.body = buffer
           makeRequest()
         })
       } else {
-        params.headers['Content-Length'] = '' + Buffer.byteLength(params.body)
+        params.headers['content-length'] = '' + Buffer.byteLength(params.body)
         makeRequest()
       }
     } else {
@@ -385,7 +384,7 @@ class Transport {
       }
 
       debug('Sniffing ended successfully', result.body)
-      const protocol = result.meta.connection.url.protocol || 'http:'
+      const protocol = result.meta.connection.url.protocol || /* istanbul ignore next */ 'http:'
       const hosts = this.connectionPool.nodesToHost(result.body.nodes, protocol)
       this.connectionPool.update(hosts)
 
@@ -453,5 +452,21 @@ function generateRequestId () {
     return (nextReqId = (nextReqId + 1) & maxInt)
   }
 }
+
+function lowerCaseHeaders (oldHeaders) {
+  if (oldHeaders == null) return oldHeaders
+  const newHeaders = {}
+  for (const header in oldHeaders) {
+    newHeaders[header.toLowerCase()] = oldHeaders[header]
+  }
+  return newHeaders
+}
+
 module.exports = Transport
-module.exports.internals = { defaultNodeFilter, roundRobinSelector, randomSelector, generateRequestId }
+module.exports.internals = {
+  defaultNodeFilter,
+  roundRobinSelector,
+  randomSelector,
+  generateRequestId,
+  lowerCaseHeaders
+}

package/lib/pool/BaseConnectionPool.js

@@ -52,6 +52,7 @@ class BaseConnectionPool {
     }
 
     if (opts.ssl == null) opts.ssl = this._ssl
+    /* istanbul ignore else */
     if (opts.agent == null) opts.agent = this._agent
 
     const connection = new this.Connection(opts)
@@ -201,6 +202,7 @@ class BaseConnectionPool {
       }
 
       address = address.slice(0, 4) === 'http'
+        /* istanbul ignore next */
         ? address
         : `${protocol}//${address}`
       const roles = node.roles.reduce((acc, role) => {

package/lib/pool/CloudConnectionPool.js

@@ -7,7 +7,7 @@
 const BaseConnectionPool = require('./BaseConnectionPool')
 
 class CloudConnectionPool extends BaseConnectionPool {
-  constructor (opts = {}) {
+  constructor (opts) {
     super(opts)
     this.cloudConnection = null
   }