com.googler.python 1.0.7 → 1.0.9

package/python3.4.2/lib/python3.4/site-packages/pip/_vendor/cachecontrol/controller.py

@@ -0,0 +1,373 @@
+"""
+The httplib2 algorithms ported for use with requests.
+"""
+import logging
+import re
+import calendar
+import time
+from email.utils import parsedate_tz
+
+from pip._vendor.requests.structures import CaseInsensitiveDict
+
+from .cache import DictCache
+from .serialize import Serializer
+
+
+logger = logging.getLogger(__name__)
+
+URI = re.compile(r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?")
+
+
+def parse_uri(uri):
+    """Parses a URI using the regex given in Appendix B of RFC 3986.
+
+        (scheme, authority, path, query, fragment) = parse_uri(uri)
+    """
+    groups = URI.match(uri).groups()
+    return (groups[1], groups[3], groups[4], groups[6], groups[8])
+
+
+class CacheController(object):
+    """An interface to see if request should cached or not.
+    """
+    def __init__(self, cache=None, cache_etags=True, serializer=None,
+                 status_codes=None):
+        self.cache = cache or DictCache()
+        self.cache_etags = cache_etags
+        self.serializer = serializer or Serializer()
+        self.cacheable_status_codes = status_codes or (200, 203, 300, 301)
+
+    @classmethod
+    def _urlnorm(cls, uri):
+        """Normalize the URL to create a safe key for the cache"""
+        (scheme, authority, path, query, fragment) = parse_uri(uri)
+        if not scheme or not authority:
+            raise Exception("Only absolute URIs are allowed. uri = %s" % uri)
+
+        scheme = scheme.lower()
+        authority = authority.lower()
+
+        if not path:
+            path = "/"
+
+        # Could do syntax based normalization of the URI before
+        # computing the digest. See Section 6.2.2 of Std 66.
+        request_uri = query and "?".join([path, query]) or path
+        defrag_uri = scheme + "://" + authority + request_uri
+
+        return defrag_uri
+
+    @classmethod
+    def cache_url(cls, uri):
+        return cls._urlnorm(uri)
+
+    def parse_cache_control(self, headers):
+        known_directives = {
+            # https://tools.ietf.org/html/rfc7234#section-5.2
+            'max-age': (int, True,),
+            'max-stale': (int, False,),
+            'min-fresh': (int, True,),
+            'no-cache': (None, False,),
+            'no-store': (None, False,),
+            'no-transform': (None, False,),
+            'only-if-cached' : (None, False,),
+            'must-revalidate': (None, False,),
+            'public': (None, False,),
+            'private': (None, False,),
+            'proxy-revalidate': (None, False,),
+            's-maxage': (int, True,)
+        }
+
+        cc_headers = headers.get('cache-control',
+                                 headers.get('Cache-Control', ''))
+
+        retval = {}
+
+        for cc_directive in cc_headers.split(','):
+            parts = cc_directive.split('=', 1)
+            directive = parts[0].strip()
+
+            try:
+                typ, required = known_directives[directive]
+            except KeyError:
+                logger.debug('Ignoring unknown cache-control directive: %s',
+                             directive)
+                continue
+
+            if not typ or not required:
+                retval[directive] = None
+            if typ:
+                try:
+                    retval[directive] = typ(parts[1].strip())
+                except IndexError:
+                    if required:
+                        logger.debug('Missing value for cache-control '
+                                     'directive: %s', directive)
+                except ValueError:
+                    logger.debug('Invalid value for cache-control directive '
+                                 '%s, must be %s', directive, typ.__name__)
+
+        return retval
+
+    def cached_request(self, request):
+        """
+        Return a cached response if it exists in the cache, otherwise
+        return False.
+        """
+        cache_url = self.cache_url(request.url)
+        logger.debug('Looking up "%s" in the cache', cache_url)
+        cc = self.parse_cache_control(request.headers)
+
+        # Bail out if the request insists on fresh data
+        if 'no-cache' in cc:
+            logger.debug('Request header has "no-cache", cache bypassed')
+            return False
+
+        if 'max-age' in cc and cc['max-age'] == 0:
+            logger.debug('Request header has "max_age" as 0, cache bypassed')
+            return False
+
+        # Request allows serving from the cache, let's see if we find something
+        cache_data = self.cache.get(cache_url)
+        if cache_data is None:
+            logger.debug('No cache entry available')
+            return False
+
+        # Check whether it can be deserialized
+        resp = self.serializer.loads(request, cache_data)
+        if not resp:
+            logger.warning('Cache entry deserialization failed, entry ignored')
+            return False
+
+        # If we have a cached 301, return it immediately. We don't
+        # need to test our response for other headers b/c it is
+        # intrinsically "cacheable" as it is Permanent.
+        # See:
+        #   https://tools.ietf.org/html/rfc7231#section-6.4.2
+        #
+        # Client can try to refresh the value by repeating the request
+        # with cache busting headers as usual (ie no-cache).
+        if resp.status == 301:
+            msg = ('Returning cached "301 Moved Permanently" response '
+                   '(ignoring date and etag information)')
+            logger.debug(msg)
+            return resp
+
+        headers = CaseInsensitiveDict(resp.headers)
+        if not headers or 'date' not in headers:
+            if 'etag' not in headers:
+                # Without date or etag, the cached response can never be used
+                # and should be deleted.
+                logger.debug('Purging cached response: no date or etag')
+                self.cache.delete(cache_url)
+            logger.debug('Ignoring cached response: no date')
+            return False
+
+        now = time.time()
+        date = calendar.timegm(
+            parsedate_tz(headers['date'])
+        )
+        current_age = max(0, now - date)
+        logger.debug('Current age based on date: %i', current_age)
+
+        # TODO: There is an assumption that the result will be a
+        #       urllib3 response object. This may not be best since we
+        #       could probably avoid instantiating or constructing the
+        #       response until we know we need it.
+        resp_cc = self.parse_cache_control(headers)
+
+        # determine freshness
+        freshness_lifetime = 0
+
+        # Check the max-age pragma in the cache control header
+        if 'max-age' in resp_cc:
+            freshness_lifetime = resp_cc['max-age']
+            logger.debug('Freshness lifetime from max-age: %i',
+                         freshness_lifetime)
+
+        # If there isn't a max-age, check for an expires header
+        elif 'expires' in headers:
+            expires = parsedate_tz(headers['expires'])
+            if expires is not None:
+                expire_time = calendar.timegm(expires) - date
+                freshness_lifetime = max(0, expire_time)
+                logger.debug("Freshness lifetime from expires: %i",
+                             freshness_lifetime)
+
+        # Determine if we are setting freshness limit in the
+        # request. Note, this overrides what was in the response.
+        if 'max-age' in cc:
+            freshness_lifetime = cc['max-age']
+            logger.debug('Freshness lifetime from request max-age: %i',
+                         freshness_lifetime)
+
+        if 'min-fresh' in cc:
+            min_fresh = cc['min-fresh']
+            # adjust our current age by our min fresh
+            current_age += min_fresh
+            logger.debug('Adjusted current age from min-fresh: %i',
+                         current_age)
+
+        # Return entry if it is fresh enough
+        if freshness_lifetime > current_age:
+            logger.debug('The response is "fresh", returning cached response')
+            logger.debug('%i > %i', freshness_lifetime, current_age)
+            return resp
+
+        # we're not fresh. If we don't have an Etag, clear it out
+        if 'etag' not in headers:
+            logger.debug(
+                'The cached response is "stale" with no etag, purging'
+            )
+            self.cache.delete(cache_url)
+
+        # return the original handler
+        return False
+
+    def conditional_headers(self, request):
+        cache_url = self.cache_url(request.url)
+        resp = self.serializer.loads(request, self.cache.get(cache_url))
+        new_headers = {}
+
+        if resp:
+            headers = CaseInsensitiveDict(resp.headers)
+
+            if 'etag' in headers:
+                new_headers['If-None-Match'] = headers['ETag']
+
+            if 'last-modified' in headers:
+                new_headers['If-Modified-Since'] = headers['Last-Modified']
+
+        return new_headers
+
+    def cache_response(self, request, response, body=None,
+                       status_codes=None):
+        """
+        Algorithm for caching requests.
+
+        This assumes a requests Response object.
+        """
+        # From httplib2: Don't cache 206's since we aren't going to
+        #                handle byte range requests
+        cacheable_status_codes = status_codes or self.cacheable_status_codes
+        if response.status not in cacheable_status_codes:
+            logger.debug(
+                'Status code %s not in %s',
+                response.status,
+                cacheable_status_codes
+            )
+            return
+
+        response_headers = CaseInsensitiveDict(response.headers)
+
+        # If we've been given a body, our response has a Content-Length, that
+        # Content-Length is valid then we can check to see if the body we've
+        # been given matches the expected size, and if it doesn't we'll just
+        # skip trying to cache it.
+        if (body is not None and
+                "content-length" in response_headers and
+                response_headers["content-length"].isdigit() and
+                int(response_headers["content-length"]) != len(body)):
+            return
+
+        cc_req = self.parse_cache_control(request.headers)
+        cc = self.parse_cache_control(response_headers)
+
+        cache_url = self.cache_url(request.url)
+        logger.debug('Updating cache with response from "%s"', cache_url)
+
+        # Delete it from the cache if we happen to have it stored there
+        no_store = False
+        if 'no-store' in cc:
+            no_store = True
+            logger.debug('Response header has "no-store"')
+        if 'no-store' in cc_req:
+            no_store = True
+            logger.debug('Request header has "no-store"')
+        if no_store and self.cache.get(cache_url):
+            logger.debug('Purging existing cache entry to honor "no-store"')
+            self.cache.delete(cache_url)
+
+        # If we've been given an etag, then keep the response
+        if self.cache_etags and 'etag' in response_headers:
+            logger.debug('Caching due to etag')
+            self.cache.set(
+                cache_url,
+                self.serializer.dumps(request, response, body=body),
+            )
+
+        # Add to the cache any 301s. We do this before looking that
+        # the Date headers.
+        elif response.status == 301:
+            logger.debug('Caching permanant redirect')
+            self.cache.set(
+                cache_url,
+                self.serializer.dumps(request, response)
+            )
+
+        # Add to the cache if the response headers demand it. If there
+        # is no date header then we can't do anything about expiring
+        # the cache.
+        elif 'date' in response_headers:
+            # cache when there is a max-age > 0
+            if 'max-age' in cc and cc['max-age'] > 0:
+                logger.debug('Caching b/c date exists and max-age > 0')
+                self.cache.set(
+                    cache_url,
+                    self.serializer.dumps(request, response, body=body),
+                )
+
+            # If the request can expire, it means we should cache it
+            # in the meantime.
+            elif 'expires' in response_headers:
+                if response_headers['expires']:
+                    logger.debug('Caching b/c of expires header')
+                    self.cache.set(
+                        cache_url,
+                        self.serializer.dumps(request, response, body=body),
+                    )
+
+    def update_cached_response(self, request, response):
+        """On a 304 we will get a new set of headers that we want to
+        update our cached value with, assuming we have one.
+
+        This should only ever be called when we've sent an ETag and
+        gotten a 304 as the response.
+        """
+        cache_url = self.cache_url(request.url)
+
+        cached_response = self.serializer.loads(
+            request,
+            self.cache.get(cache_url)
+        )
+
+        if not cached_response:
+            # we didn't have a cached response
+            return response
+
+        # Lets update our headers with the headers from the new request:
+        # http://tools.ietf.org/html/draft-ietf-httpbis-p4-conditional-26#section-4.1
+        #
+        # The server isn't supposed to send headers that would make
+        # the cached body invalid. But... just in case, we'll be sure
+        # to strip out ones we know that might be problmatic due to
+        # typical assumptions.
+        excluded_headers = [
+            "content-length",
+        ]
+
+        cached_response.headers.update(
+            dict((k, v) for k, v in response.headers.items()
+                 if k.lower() not in excluded_headers)
+        )
+
+        # we want a 200 b/c we have content via the cache
+        cached_response.status = 200
+
+        # update our cache
+        self.cache.set(
+            cache_url,
+            self.serializer.dumps(request, cached_response),
+        )
+
+        return cached_response

package/python3.4.2/lib/python3.4/site-packages/pip/_vendor/cachecontrol/filewrapper.py

@@ -0,0 +1,78 @@
+from io import BytesIO
+
+
+class CallbackFileWrapper(object):
+    """
+    Small wrapper around a fp object which will tee everything read into a
+    buffer, and when that file is closed it will execute a callback with the
+    contents of that buffer.
+
+    All attributes are proxied to the underlying file object.
+
+    This class uses members with a double underscore (__) leading prefix so as
+    not to accidentally shadow an attribute.
+    """
+
+    def __init__(self, fp, callback):
+        self.__buf = BytesIO()
+        self.__fp = fp
+        self.__callback = callback
+
+    def __getattr__(self, name):
+        # The vaguaries of garbage collection means that self.__fp is
+        # not always set.  By using __getattribute__ and the private
+        # name[0] allows looking up the attribute value and raising an
+        # AttributeError when it doesn't exist. This stop thigns from
+        # infinitely recursing calls to getattr in the case where
+        # self.__fp hasn't been set.
+        #
+        # [0] https://docs.python.org/2/reference/expressions.html#atom-identifiers
+        fp = self.__getattribute__('_CallbackFileWrapper__fp')
+        return getattr(fp, name)
+
+    def __is_fp_closed(self):
+        try:
+            return self.__fp.fp is None
+        except AttributeError:
+            pass
+
+        try:
+            return self.__fp.closed
+        except AttributeError:
+            pass
+
+        # We just don't cache it then.
+        # TODO: Add some logging here...
+        return False
+
+    def _close(self):
+        if self.__callback:
+            self.__callback(self.__buf.getvalue())
+
+        # We assign this to None here, because otherwise we can get into
+        # really tricky problems where the CPython interpreter dead locks
+        # because the callback is holding a reference to something which
+        # has a __del__ method. Setting this to None breaks the cycle
+        # and allows the garbage collector to do it's thing normally.
+        self.__callback = None
+
+    def read(self, amt=None):
+        data = self.__fp.read(amt)
+        self.__buf.write(data)
+        if self.__is_fp_closed():
+            self._close()
+
+        return data
+
+    def _safe_read(self, amt):
+        data = self.__fp._safe_read(amt)
+        if amt == 2 and data == b'\r\n':
+            # urllib executes this read to toss the CRLF at the end
+            # of the chunk.
+            return data
+
+        self.__buf.write(data)
+        if self.__is_fp_closed():
+            self._close()
+
+        return data

package/python3.4.2/lib/python3.4/site-packages/pip/_vendor/cachecontrol/heuristics.py

@@ -0,0 +1,138 @@
+import calendar
+import time
+
+from email.utils import formatdate, parsedate, parsedate_tz
+
+from datetime import datetime, timedelta
+
+TIME_FMT = "%a, %d %b %Y %H:%M:%S GMT"
+
+
+def expire_after(delta, date=None):
+    date = date or datetime.utcnow()
+    return date + delta
+
+
+def datetime_to_header(dt):
+    return formatdate(calendar.timegm(dt.timetuple()))
+
+
+class BaseHeuristic(object):
+
+    def warning(self, response):
+        """
+        Return a valid 1xx warning header value describing the cache
+        adjustments.
+
+        The response is provided too allow warnings like 113
+        http://tools.ietf.org/html/rfc7234#section-5.5.4 where we need
+        to explicitly say response is over 24 hours old.
+        """
+        return '110 - "Response is Stale"'
+
+    def update_headers(self, response):
+        """Update the response headers with any new headers.
+
+        NOTE: This SHOULD always include some Warning header to
+              signify that the response was cached by the client, not
+              by way of the provided headers.
+        """
+        return {}
+
+    def apply(self, response):
+        updated_headers = self.update_headers(response)
+
+        if updated_headers:
+            response.headers.update(updated_headers)
+            warning_header_value = self.warning(response)
+            if warning_header_value is not None:
+                response.headers.update({'Warning': warning_header_value})
+
+        return response
+
+
+class OneDayCache(BaseHeuristic):
+    """
+    Cache the response by providing an expires 1 day in the
+    future.
+    """
+    def update_headers(self, response):
+        headers = {}
+
+        if 'expires' not in response.headers:
+            date = parsedate(response.headers['date'])
+            expires = expire_after(timedelta(days=1),
+                                   date=datetime(*date[:6]))
+            headers['expires'] = datetime_to_header(expires)
+            headers['cache-control'] = 'public'
+        return headers
+
+
+class ExpiresAfter(BaseHeuristic):
+    """
+    Cache **all** requests for a defined time period.
+    """
+
+    def __init__(self, **kw):
+        self.delta = timedelta(**kw)
+
+    def update_headers(self, response):
+        expires = expire_after(self.delta)
+        return {
+            'expires': datetime_to_header(expires),
+            'cache-control': 'public',
+        }
+
+    def warning(self, response):
+        tmpl = '110 - Automatically cached for %s. Response might be stale'
+        return tmpl % self.delta
+
+
+class LastModified(BaseHeuristic):
+    """
+    If there is no Expires header already, fall back on Last-Modified
+    using the heuristic from
+    http://tools.ietf.org/html/rfc7234#section-4.2.2
+    to calculate a reasonable value.
+
+    Firefox also does something like this per
+    https://developer.mozilla.org/en-US/docs/Web/HTTP/Caching_FAQ
+    http://lxr.mozilla.org/mozilla-release/source/netwerk/protocol/http/nsHttpResponseHead.cpp#397
+    Unlike mozilla we limit this to 24-hr.
+    """
+    cacheable_by_default_statuses = set([
+        200, 203, 204, 206, 300, 301, 404, 405, 410, 414, 501
+    ])
+
+    def update_headers(self, resp):
+        headers = resp.headers
+
+        if 'expires' in headers:
+            return {}
+
+        if 'cache-control' in headers and headers['cache-control'] != 'public':
+            return {}
+
+        if resp.status not in self.cacheable_by_default_statuses:
+            return {}
+
+        if 'date' not in headers or 'last-modified' not in headers:
+            return {}
+
+        date = calendar.timegm(parsedate_tz(headers['date']))
+        last_modified = parsedate(headers['last-modified'])
+        if date is None or last_modified is None:
+            return {}
+
+        now = time.time()
+        current_age = max(0, now - date)
+        delta = date - calendar.timegm(last_modified)
+        freshness_lifetime = max(0, min(delta / 10, 24 * 3600))
+        if freshness_lifetime <= current_age:
+            return {}
+
+        expires = date + freshness_lifetime
+        return {'expires': time.strftime(TIME_FMT, time.gmtime(expires))}
+
+    def warning(self, resp):
+        return None