@macula-io/martha 0.0.0 → 1.0.0

package/node_modules/minipass-fetch/lib/index.js

@@ -0,0 +1,376 @@
+'use strict'
+const { URL } = require('url')
+const http = require('http')
+const https = require('https')
+const zlib = require('minizlib')
+const { Minipass } = require('minipass')
+
+const Body = require('./body.js')
+const { writeToStream, getTotalBytes } = Body
+const Response = require('./response.js')
+const Headers = require('./headers.js')
+const { createHeadersLenient } = Headers
+const Request = require('./request.js')
+const { getNodeRequestOptions } = Request
+const FetchError = require('./fetch-error.js')
+const AbortError = require('./abort-error.js')
+
+// XXX this should really be split up and unit-ized for easier testing
+// and better DRY implementation of data/http request aborting
+const fetch = async (url, opts) => {
+  if (/^data:/.test(url)) {
+    const request = new Request(url, opts)
+    // delay 1 promise tick so that the consumer can abort right away
+    return Promise.resolve().then(() => new Promise((resolve, reject) => {
+      let type, data
+      try {
+        const { pathname, search } = new URL(url)
+        const split = pathname.split(',')
+        if (split.length < 2) {
+          throw new Error('invalid data: URI')
+        }
+        const mime = split.shift()
+        const base64 = /;base64$/.test(mime)
+        type = base64 ? mime.slice(0, -1 * ';base64'.length) : mime
+        const rawData = decodeURIComponent(split.join(',') + search)
+        data = base64 ? Buffer.from(rawData, 'base64') : Buffer.from(rawData)
+      } catch (er) {
+        return reject(new FetchError(`[${request.method}] ${
+          request.url} invalid URL, ${er.message}`, 'system', er))
+      }
+
+      const { signal } = request
+      if (signal && signal.aborted) {
+        return reject(new AbortError('The user aborted a request.'))
+      }
+
+      const headers = { 'Content-Length': data.length }
+      if (type) {
+        headers['Content-Type'] = type
+      }
+      return resolve(new Response(data, { headers }))
+    }))
+  }
+
+  return new Promise((resolve, reject) => {
+    // build request object
+    const request = new Request(url, opts)
+    let options
+    try {
+      options = getNodeRequestOptions(request)
+    } catch (er) {
+      return reject(er)
+    }
+
+    const send = (options.protocol === 'https:' ? https : http).request
+    const { signal } = request
+    let response = null
+    const abort = () => {
+      const error = new AbortError('The user aborted a request.')
+      reject(error)
+      if (Minipass.isStream(request.body) &&
+          typeof request.body.destroy === 'function') {
+        request.body.destroy(error)
+      }
+      if (response && response.body) {
+        response.body.emit('error', error)
+      }
+    }
+
+    if (signal && signal.aborted) {
+      return abort()
+    }
+
+    const abortAndFinalize = () => {
+      abort()
+      finalize()
+    }
+
+    const finalize = () => {
+      req.abort()
+      if (signal) {
+        signal.removeEventListener('abort', abortAndFinalize)
+      }
+      clearTimeout(reqTimeout)
+    }
+
+    // send request
+    const req = send(options)
+
+    if (signal) {
+      signal.addEventListener('abort', abortAndFinalize)
+    }
+
+    let reqTimeout = null
+    if (request.timeout) {
+      req.once('socket', () => {
+        reqTimeout = setTimeout(() => {
+          reject(new FetchError(`network timeout at: ${
+            request.url}`, 'request-timeout'))
+          finalize()
+        }, request.timeout)
+      })
+    }
+
+    req.on('error', er => {
+      // if a 'response' event is emitted before the 'error' event, then by the
+      // time this handler is run it's too late to reject the Promise for the
+      // response. instead, we forward the error event to the response stream
+      // so that the error will surface to the user when they try to consume
+      // the body. this is done as a side effect of aborting the request except
+      // for in windows, where we must forward the event manually, otherwise
+      // there is no longer a ref'd socket attached to the request and the
+      // stream never ends so the event loop runs out of work and the process
+      // exits without warning.
+      // coverage skipped here due to the difficulty in testing
+      // istanbul ignore next
+      if (req.res) {
+        req.res.emit('error', er)
+      }
+      reject(new FetchError(`request to ${request.url} failed, reason: ${
+        er.message}`, 'system', er))
+      finalize()
+    })
+
+    req.on('response', res => {
+      clearTimeout(reqTimeout)
+
+      const headers = createHeadersLenient(res.headers)
+
+      // HTTP fetch step 5
+      if (fetch.isRedirect(res.statusCode)) {
+        // HTTP fetch step 5.2
+        const location = headers.get('Location')
+
+        // HTTP fetch step 5.3
+        let locationURL = null
+        try {
+          locationURL = location === null ? null : new URL(location, request.url).toString()
+        } catch {
+          // error here can only be invalid URL in Location: header
+          // do not throw when options.redirect == manual
+          // let the user extract the errorneous redirect URL
+          if (request.redirect !== 'manual') {
+            /* eslint-disable-next-line max-len */
+            reject(new FetchError(`uri requested responds with an invalid redirect URL: ${location}`, 'invalid-redirect'))
+            finalize()
+            return
+          }
+        }
+
+        // HTTP fetch step 5.5
+        if (request.redirect === 'error') {
+          reject(new FetchError('uri requested responds with a redirect, ' +
+            `redirect mode is set to error: ${request.url}`, 'no-redirect'))
+          finalize()
+          return
+        } else if (request.redirect === 'manual') {
+          // node-fetch-specific step: make manual redirect a bit easier to
+          // use by setting the Location header value to the resolved URL.
+          if (locationURL !== null) {
+            // handle corrupted header
+            try {
+              headers.set('Location', locationURL)
+            } catch (err) {
+              /* istanbul ignore next: nodejs server prevent invalid
+                 response headers, we can't test this through normal
+                 request */
+              reject(err)
+            }
+          }
+        } else if (request.redirect === 'follow' && locationURL !== null) {
+          // HTTP-redirect fetch step 5
+          if (request.counter >= request.follow) {
+            reject(new FetchError(`maximum redirect reached at: ${
+              request.url}`, 'max-redirect'))
+            finalize()
+            return
+          }
+
+          // HTTP-redirect fetch step 9
+          if (res.statusCode !== 303 &&
+              request.body &&
+              getTotalBytes(request) === null) {
+            reject(new FetchError(
+              'Cannot follow redirect with body being a readable stream',
+              'unsupported-redirect'
+            ))
+            finalize()
+            return
+          }
+
+          // Update host due to redirection
+          request.headers.set('host', (new URL(locationURL)).host)
+
+          // HTTP-redirect fetch step 6 (counter increment)
+          // Create a new Request object.
+          const requestOpts = {
+            headers: new Headers(request.headers),
+            follow: request.follow,
+            counter: request.counter + 1,
+            agent: request.agent,
+            compress: request.compress,
+            method: request.method,
+            body: request.body,
+            signal: request.signal,
+            timeout: request.timeout,
+          }
+
+          // if the redirect is to a new hostname, strip the authorization and cookie headers
+          const parsedOriginal = new URL(request.url)
+          const parsedRedirect = new URL(locationURL)
+          if (parsedOriginal.hostname !== parsedRedirect.hostname) {
+            requestOpts.headers.delete('authorization')
+            requestOpts.headers.delete('cookie')
+          }
+
+          // HTTP-redirect fetch step 11
+          if (res.statusCode === 303 || (
+            (res.statusCode === 301 || res.statusCode === 302) &&
+              request.method === 'POST'
+          )) {
+            requestOpts.method = 'GET'
+            requestOpts.body = undefined
+            requestOpts.headers.delete('content-length')
+          }
+
+          // HTTP-redirect fetch step 15
+          resolve(fetch(new Request(locationURL, requestOpts)))
+          finalize()
+          return
+        }
+      } // end if(isRedirect)
+
+      // prepare response
+      res.once('end', () =>
+        signal && signal.removeEventListener('abort', abortAndFinalize))
+
+      const body = new Minipass()
+      // if an error occurs, either on the response stream itself, on one of the
+      // decoder streams, or a response length timeout from the Body class, we
+      // forward the error through to our internal body stream. If we see an
+      // error event on that, we call finalize to abort the request and ensure
+      // we don't leave a socket believing a request is in flight.
+      // this is difficult to test, so lacks specific coverage.
+      body.on('error', finalize)
+      // exceedingly rare that the stream would have an error,
+      // but just in case we proxy it to the stream in use.
+      res.on('error', /* istanbul ignore next */ er => body.emit('error', er))
+      res.on('data', (chunk) => body.write(chunk))
+      res.on('end', () => body.end())
+
+      const responseOptions = {
+        url: request.url,
+        status: res.statusCode,
+        statusText: res.statusMessage,
+        headers: headers,
+        size: request.size,
+        timeout: request.timeout,
+        counter: request.counter,
+        trailer: new Promise(resolveTrailer =>
+          res.on('end', () => resolveTrailer(createHeadersLenient(res.trailers)))),
+      }
+
+      // HTTP-network fetch step 12.1.1.3
+      const codings = headers.get('Content-Encoding')
+
+      // HTTP-network fetch step 12.1.1.4: handle content codings
+
+      // in following scenarios we ignore compression support
+      // 1. compression support is disabled
+      // 2. HEAD request
+      // 3. no Content-Encoding header
+      // 4. no content response (204)
+      // 5. content not modified response (304)
+      if (!request.compress ||
+          request.method === 'HEAD' ||
+          codings === null ||
+          res.statusCode === 204 ||
+          res.statusCode === 304) {
+        response = new Response(body, responseOptions)
+        resolve(response)
+        return
+      }
+
+      // Be less strict when decoding compressed responses, since sometimes
+      // servers send slightly invalid responses that are still accepted
+      // by common browsers.
+      // Always using Z_SYNC_FLUSH is what cURL does.
+      const zlibOptions = {
+        flush: zlib.constants.Z_SYNC_FLUSH,
+        finishFlush: zlib.constants.Z_SYNC_FLUSH,
+      }
+
+      // for gzip
+      if (codings === 'gzip' || codings === 'x-gzip') {
+        const unzip = new zlib.Gunzip(zlibOptions)
+        response = new Response(
+          // exceedingly rare that the stream would have an error,
+          // but just in case we proxy it to the stream in use.
+          body.on('error', /* istanbul ignore next */ er => unzip.emit('error', er)).pipe(unzip),
+          responseOptions
+        )
+        resolve(response)
+        return
+      }
+
+      // for deflate
+      if (codings === 'deflate' || codings === 'x-deflate') {
+        // handle the infamous raw deflate response from old servers
+        // a hack for old IIS and Apache servers
+        res.once('data', chunk => {
+          // see http://stackoverflow.com/questions/37519828
+          const decoder = (chunk[0] & 0x0F) === 0x08
+            ? new zlib.Inflate()
+            : new zlib.InflateRaw()
+          // exceedingly rare that the stream would have an error,
+          // but just in case we proxy it to the stream in use.
+          body.on('error', /* istanbul ignore next */ er => decoder.emit('error', er)).pipe(decoder)
+          response = new Response(decoder, responseOptions)
+          resolve(response)
+        })
+        return
+      }
+
+      // for br
+      if (codings === 'br') {
+        // ignoring coverage so tests don't have to fake support (or lack of) for brotli
+        // istanbul ignore next
+        try {
+          var decoder = new zlib.BrotliDecompress()
+        } catch (err) {
+          reject(err)
+          finalize()
+          return
+        }
+        // exceedingly rare that the stream would have an error,
+        // but just in case we proxy it to the stream in use.
+        body.on('error', /* istanbul ignore next */ er => decoder.emit('error', er)).pipe(decoder)
+        response = new Response(decoder, responseOptions)
+        resolve(response)
+        return
+      }
+
+      // otherwise, use response as-is
+      response = new Response(body, responseOptions)
+      resolve(response)
+    })
+
+    writeToStream(req, request)
+  })
+}
+
+module.exports = fetch
+
+fetch.isRedirect = code =>
+  code === 301 ||
+  code === 302 ||
+  code === 303 ||
+  code === 307 ||
+  code === 308
+
+fetch.Headers = Headers
+fetch.Request = Request
+fetch.Response = Response
+fetch.FetchError = FetchError
+fetch.AbortError = AbortError

package/node_modules/minipass-fetch/lib/request.js

@@ -0,0 +1,282 @@
+'use strict'
+const { URL } = require('url')
+const { Minipass } = require('minipass')
+const Headers = require('./headers.js')
+const { exportNodeCompatibleHeaders } = Headers
+const Body = require('./body.js')
+const { clone, extractContentType, getTotalBytes } = Body
+
+const version = require('../package.json').version
+const defaultUserAgent =
+  `minipass-fetch/${version} (+https://github.com/isaacs/minipass-fetch)`
+
+const INTERNALS = Symbol('Request internals')
+
+const isRequest = input =>
+  typeof input === 'object' && typeof input[INTERNALS] === 'object'
+
+const isAbortSignal = signal => {
+  const proto = (
+    signal
+    && typeof signal === 'object'
+    && Object.getPrototypeOf(signal)
+  )
+  return !!(proto && proto.constructor.name === 'AbortSignal')
+}
+
+class Request extends Body {
+  constructor (input, init = {}) {
+    const parsedURL = isRequest(input) ? new URL(input.url)
+      : input && input.href ? new URL(input.href)
+      : new URL(`${input}`)
+
+    if (isRequest(input)) {
+      init = { ...input[INTERNALS], ...init }
+    } else if (!input || typeof input === 'string') {
+      input = {}
+    }
+
+    const method = (init.method || input.method || 'GET').toUpperCase()
+    const isGETHEAD = method === 'GET' || method === 'HEAD'
+
+    if ((init.body !== null && init.body !== undefined ||
+        isRequest(input) && input.body !== null) && isGETHEAD) {
+      throw new TypeError('Request with GET/HEAD method cannot have body')
+    }
+
+    const inputBody = init.body !== null && init.body !== undefined ? init.body
+      : isRequest(input) && input.body !== null ? clone(input)
+      : null
+
+    super(inputBody, {
+      timeout: init.timeout || input.timeout || 0,
+      size: init.size || input.size || 0,
+    })
+
+    const headers = new Headers(init.headers || input.headers || {})
+
+    if (inputBody !== null && inputBody !== undefined &&
+        !headers.has('Content-Type')) {
+      const contentType = extractContentType(inputBody)
+      if (contentType) {
+        headers.append('Content-Type', contentType)
+      }
+    }
+
+    const signal = 'signal' in init ? init.signal
+      : null
+
+    if (signal !== null && signal !== undefined && !isAbortSignal(signal)) {
+      throw new TypeError('Expected signal must be an instanceof AbortSignal')
+    }
+
+    // TLS specific options that are handled by node
+    const {
+      ca,
+      cert,
+      ciphers,
+      clientCertEngine,
+      crl,
+      dhparam,
+      ecdhCurve,
+      family,
+      honorCipherOrder,
+      key,
+      passphrase,
+      pfx,
+      rejectUnauthorized = process.env.NODE_TLS_REJECT_UNAUTHORIZED !== '0',
+      secureOptions,
+      secureProtocol,
+      servername,
+      sessionIdContext,
+    } = init
+
+    this[INTERNALS] = {
+      method,
+      redirect: init.redirect || input.redirect || 'follow',
+      headers,
+      parsedURL,
+      signal,
+      ca,
+      cert,
+      ciphers,
+      clientCertEngine,
+      crl,
+      dhparam,
+      ecdhCurve,
+      family,
+      honorCipherOrder,
+      key,
+      passphrase,
+      pfx,
+      rejectUnauthorized,
+      secureOptions,
+      secureProtocol,
+      servername,
+      sessionIdContext,
+    }
+
+    // node-fetch-only options
+    this.follow = init.follow !== undefined ? init.follow
+      : input.follow !== undefined ? input.follow
+      : 20
+    this.compress = init.compress !== undefined ? init.compress
+      : input.compress !== undefined ? input.compress
+      : true
+    this.counter = init.counter || input.counter || 0
+    this.agent = init.agent || input.agent
+  }
+
+  get method () {
+    return this[INTERNALS].method
+  }
+
+  get url () {
+    return this[INTERNALS].parsedURL.toString()
+  }
+
+  get headers () {
+    return this[INTERNALS].headers
+  }
+
+  get redirect () {
+    return this[INTERNALS].redirect
+  }
+
+  get signal () {
+    return this[INTERNALS].signal
+  }
+
+  clone () {
+    return new Request(this)
+  }
+
+  get [Symbol.toStringTag] () {
+    return 'Request'
+  }
+
+  static getNodeRequestOptions (request) {
+    const parsedURL = request[INTERNALS].parsedURL
+    const headers = new Headers(request[INTERNALS].headers)
+
+    // fetch step 1.3
+    if (!headers.has('Accept')) {
+      headers.set('Accept', '*/*')
+    }
+
+    // Basic fetch
+    if (!/^https?:$/.test(parsedURL.protocol)) {
+      throw new TypeError('Only HTTP(S) protocols are supported')
+    }
+
+    if (request.signal &&
+        Minipass.isStream(request.body) &&
+        typeof request.body.destroy !== 'function') {
+      throw new Error(
+        'Cancellation of streamed requests with AbortSignal is not supported')
+    }
+
+    // HTTP-network-or-cache fetch steps 2.4-2.7
+    const contentLengthValue =
+      (request.body === null || request.body === undefined) &&
+        /^(POST|PUT)$/i.test(request.method) ? '0'
+      : request.body !== null && request.body !== undefined
+        ? getTotalBytes(request)
+        : null
+
+    if (contentLengthValue) {
+      headers.set('Content-Length', contentLengthValue + '')
+    }
+
+    // HTTP-network-or-cache fetch step 2.11
+    if (!headers.has('User-Agent')) {
+      headers.set('User-Agent', defaultUserAgent)
+    }
+
+    // HTTP-network-or-cache fetch step 2.15
+    if (request.compress && !headers.has('Accept-Encoding')) {
+      headers.set('Accept-Encoding', 'gzip,deflate')
+    }
+
+    const agent = typeof request.agent === 'function'
+      ? request.agent(parsedURL)
+      : request.agent
+
+    if (!headers.has('Connection') && !agent) {
+      headers.set('Connection', 'close')
+    }
+
+    // TLS specific options that are handled by node
+    const {
+      ca,
+      cert,
+      ciphers,
+      clientCertEngine,
+      crl,
+      dhparam,
+      ecdhCurve,
+      family,
+      honorCipherOrder,
+      key,
+      passphrase,
+      pfx,
+      rejectUnauthorized,
+      secureOptions,
+      secureProtocol,
+      servername,
+      sessionIdContext,
+    } = request[INTERNALS]
+
+    // HTTP-network fetch step 4.2
+    // chunked encoding is handled by Node.js
+
+    // we cannot spread parsedURL directly, so we have to read each property one-by-one
+    // and map them to the equivalent https?.request() method options
+    const urlProps = {
+      auth: parsedURL.username || parsedURL.password
+        ? `${parsedURL.username}:${parsedURL.password}`
+        : '',
+      host: parsedURL.host,
+      hostname: parsedURL.hostname,
+      path: `${parsedURL.pathname}${parsedURL.search}`,
+      port: parsedURL.port,
+      protocol: parsedURL.protocol,
+    }
+
+    return {
+      ...urlProps,
+      method: request.method,
+      headers: exportNodeCompatibleHeaders(headers),
+      agent,
+      ca,
+      cert,
+      ciphers,
+      clientCertEngine,
+      crl,
+      dhparam,
+      ecdhCurve,
+      family,
+      honorCipherOrder,
+      key,
+      passphrase,
+      pfx,
+      rejectUnauthorized,
+      secureOptions,
+      secureProtocol,
+      servername,
+      sessionIdContext,
+      timeout: request.timeout,
+    }
+  }
+}
+
+module.exports = Request
+
+Object.defineProperties(Request.prototype, {
+  method: { enumerable: true },
+  url: { enumerable: true },
+  headers: { enumerable: true },
+  redirect: { enumerable: true },
+  clone: { enumerable: true },
+  signal: { enumerable: true },
+})