com.googler.python 1.1.2 → 1.1.4

package/python3.6.2/lib/python3.6/http/cookiejar.py

@@ -1,2098 +0,0 @@
-r"""HTTP cookie handling for web clients.
-
-This module has (now fairly distant) origins in Gisle Aas' Perl module
-HTTP::Cookies, from the libwww-perl library.
-
-Docstrings, comments and debug strings in this code refer to the
-attributes of the HTTP cookie system as cookie-attributes, to distinguish
-them clearly from Python attributes.
-
-Class diagram (note that BSDDBCookieJar and the MSIE* classes are not
-distributed with the Python standard library, but are available from
-http://wwwsearch.sf.net/):
-
-                        CookieJar____
-                        /     \      \
-            FileCookieJar      \      \
-             /    |   \         \      \
- MozillaCookieJar | LWPCookieJar \      \
-                  |               |      \
-                  |   ---MSIEBase |       \
-                  |  /      |     |        \
-                  | /   MSIEDBCookieJar BSDDBCookieJar
-                  |/
-               MSIECookieJar
-
-"""
-
-__all__ = ['Cookie', 'CookieJar', 'CookiePolicy', 'DefaultCookiePolicy',
-           'FileCookieJar', 'LWPCookieJar', 'LoadError', 'MozillaCookieJar']
-
-import copy
-import datetime
-import re
-import time
-import urllib.parse, urllib.request
-try:
-    import threading as _threading
-except ImportError:
-    import dummy_threading as _threading
-import http.client  # only for the default HTTP port
-from calendar import timegm
-
-debug = False   # set to True to enable debugging via the logging module
-logger = None
-
-def _debug(*args):
-    if not debug:
-        return
-    global logger
-    if not logger:
-        import logging
-        logger = logging.getLogger("http.cookiejar")
-    return logger.debug(*args)
-
-
-DEFAULT_HTTP_PORT = str(http.client.HTTP_PORT)
-MISSING_FILENAME_TEXT = ("a filename was not supplied (nor was the CookieJar "
-                         "instance initialised with one)")
-
-def _warn_unhandled_exception():
-    # There are a few catch-all except: statements in this module, for
-    # catching input that's bad in unexpected ways.  Warn if any
-    # exceptions are caught there.
-    import io, warnings, traceback
-    f = io.StringIO()
-    traceback.print_exc(None, f)
-    msg = f.getvalue()
-    warnings.warn("http.cookiejar bug!\n%s" % msg, stacklevel=2)
-
-
-# Date/time conversion
-# -----------------------------------------------------------------------------
-
-EPOCH_YEAR = 1970
-def _timegm(tt):
-    year, month, mday, hour, min, sec = tt[:6]
-    if ((year >= EPOCH_YEAR) and (1 <= month <= 12) and (1 <= mday <= 31) and
-        (0 <= hour <= 24) and (0 <= min <= 59) and (0 <= sec <= 61)):
-        return timegm(tt)
-    else:
-        return None
-
-DAYS = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"]
-MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun",
-          "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]
-MONTHS_LOWER = []
-for month in MONTHS: MONTHS_LOWER.append(month.lower())
-
-def time2isoz(t=None):
-    """Return a string representing time in seconds since epoch, t.
-
-    If the function is called without an argument, it will use the current
-    time.
-
-    The format of the returned string is like "YYYY-MM-DD hh:mm:ssZ",
-    representing Universal Time (UTC, aka GMT).  An example of this format is:
-
-    1994-11-24 08:49:37Z
-
-    """
-    if t is None:
-        dt = datetime.datetime.utcnow()
-    else:
-        dt = datetime.datetime.utcfromtimestamp(t)
-    return "%04d-%02d-%02d %02d:%02d:%02dZ" % (
-        dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second)
-
-def time2netscape(t=None):
-    """Return a string representing time in seconds since epoch, t.
-
-    If the function is called without an argument, it will use the current
-    time.
-
-    The format of the returned string is like this:
-
-    Wed, DD-Mon-YYYY HH:MM:SS GMT
-
-    """
-    if t is None:
-        dt = datetime.datetime.utcnow()
-    else:
-        dt = datetime.datetime.utcfromtimestamp(t)
-    return "%s, %02d-%s-%04d %02d:%02d:%02d GMT" % (
-        DAYS[dt.weekday()], dt.day, MONTHS[dt.month-1],
-        dt.year, dt.hour, dt.minute, dt.second)
-
-
-UTC_ZONES = {"GMT": None, "UTC": None, "UT": None, "Z": None}
-
-TIMEZONE_RE = re.compile(r"^([-+])?(\d\d?):?(\d\d)?$", re.ASCII)
-def offset_from_tz_string(tz):
-    offset = None
-    if tz in UTC_ZONES:
-        offset = 0
-    else:
-        m = TIMEZONE_RE.search(tz)
-        if m:
-            offset = 3600 * int(m.group(2))
-            if m.group(3):
-                offset = offset + 60 * int(m.group(3))
-            if m.group(1) == '-':
-                offset = -offset
-    return offset
-
-def _str2time(day, mon, yr, hr, min, sec, tz):
-    yr = int(yr)
-    if yr > datetime.MAXYEAR:
-        return None
-
-    # translate month name to number
-    # month numbers start with 1 (January)
-    try:
-        mon = MONTHS_LOWER.index(mon.lower())+1
-    except ValueError:
-        # maybe it's already a number
-        try:
-            imon = int(mon)
-        except ValueError:
-            return None
-        if 1 <= imon <= 12:
-            mon = imon
-        else:
-            return None
-
-    # make sure clock elements are defined
-    if hr is None: hr = 0
-    if min is None: min = 0
-    if sec is None: sec = 0
-
-    day = int(day)
-    hr = int(hr)
-    min = int(min)
-    sec = int(sec)
-
-    if yr < 1000:
-        # find "obvious" year
-        cur_yr = time.localtime(time.time())[0]
-        m = cur_yr % 100
-        tmp = yr
-        yr = yr + cur_yr - m
-        m = m - tmp
-        if abs(m) > 50:
-            if m > 0: yr = yr + 100
-            else: yr = yr - 100
-
-    # convert UTC time tuple to seconds since epoch (not timezone-adjusted)
-    t = _timegm((yr, mon, day, hr, min, sec, tz))
-
-    if t is not None:
-        # adjust time using timezone string, to get absolute time since epoch
-        if tz is None:
-            tz = "UTC"
-        tz = tz.upper()
-        offset = offset_from_tz_string(tz)
-        if offset is None:
-            return None
-        t = t - offset
-
-    return t
-
-STRICT_DATE_RE = re.compile(
-    r"^[SMTWF][a-z][a-z], (\d\d) ([JFMASOND][a-z][a-z]) "
-    r"(\d\d\d\d) (\d\d):(\d\d):(\d\d) GMT$", re.ASCII)
-WEEKDAY_RE = re.compile(
-    r"^(?:Sun|Mon|Tue|Wed|Thu|Fri|Sat)[a-z]*,?\s*", re.I | re.ASCII)
-LOOSE_HTTP_DATE_RE = re.compile(
-    r"""^
-    (\d\d?)            # day
-       (?:\s+|[-\/])
-    (\w+)              # month
-        (?:\s+|[-\/])
-    (\d+)              # year
-    (?:
-          (?:\s+|:)    # separator before clock
-       (\d\d?):(\d\d)  # hour:min
-       (?::(\d\d))?    # optional seconds
-    )?                 # optional clock
-       \s*
-    ([-+]?\d{2,4}|(?![APap][Mm]\b)[A-Za-z]+)? # timezone
-       \s*
-    (?:\(\w+\))?       # ASCII representation of timezone in parens.
-       \s*$""", re.X | re.ASCII)
-def http2time(text):
-    """Returns time in seconds since epoch of time represented by a string.
-
-    Return value is an integer.
-
-    None is returned if the format of str is unrecognized, the time is outside
-    the representable range, or the timezone string is not recognized.  If the
-    string contains no timezone, UTC is assumed.
-
-    The timezone in the string may be numerical (like "-0800" or "+0100") or a
-    string timezone (like "UTC", "GMT", "BST" or "EST").  Currently, only the
-    timezone strings equivalent to UTC (zero offset) are known to the function.
-
-    The function loosely parses the following formats:
-
-    Wed, 09 Feb 1994 22:23:32 GMT       -- HTTP format
-    Tuesday, 08-Feb-94 14:15:29 GMT     -- old rfc850 HTTP format
-    Tuesday, 08-Feb-1994 14:15:29 GMT   -- broken rfc850 HTTP format
-    09 Feb 1994 22:23:32 GMT            -- HTTP format (no weekday)
-    08-Feb-94 14:15:29 GMT              -- rfc850 format (no weekday)
-    08-Feb-1994 14:15:29 GMT            -- broken rfc850 format (no weekday)
-
-    The parser ignores leading and trailing whitespace.  The time may be
-    absent.
-
-    If the year is given with only 2 digits, the function will select the
-    century that makes the year closest to the current date.
-
-    """
-    # fast exit for strictly conforming string
-    m = STRICT_DATE_RE.search(text)
-    if m:
-        g = m.groups()
-        mon = MONTHS_LOWER.index(g[1].lower()) + 1
-        tt = (int(g[2]), mon, int(g[0]),
-              int(g[3]), int(g[4]), float(g[5]))
-        return _timegm(tt)
-
-    # No, we need some messy parsing...
-
-    # clean up
-    text = text.lstrip()
-    text = WEEKDAY_RE.sub("", text, 1)  # Useless weekday
-
-    # tz is time zone specifier string
-    day, mon, yr, hr, min, sec, tz = [None]*7
-
-    # loose regexp parse
-    m = LOOSE_HTTP_DATE_RE.search(text)
-    if m is not None:
-        day, mon, yr, hr, min, sec, tz = m.groups()
-    else:
-        return None  # bad format
-
-    return _str2time(day, mon, yr, hr, min, sec, tz)
-
-ISO_DATE_RE = re.compile(
-    r"""^
-    (\d{4})              # year
-       [-\/]?
-    (\d\d?)              # numerical month
-       [-\/]?
-    (\d\d?)              # day
-   (?:
-         (?:\s+|[-:Tt])  # separator before clock
-      (\d\d?):?(\d\d)    # hour:min
-      (?::?(\d\d(?:\.\d*)?))?  # optional seconds (and fractional)
-   )?                    # optional clock
-      \s*
-   ([-+]?\d\d?:?(:?\d\d)?
-    |Z|z)?               # timezone  (Z is "zero meridian", i.e. GMT)
-      \s*$""", re.X | re. ASCII)
-def iso2time(text):
-    """
-    As for http2time, but parses the ISO 8601 formats:
-
-    1994-02-03 14:15:29 -0100    -- ISO 8601 format
-    1994-02-03 14:15:29          -- zone is optional
-    1994-02-03                   -- only date
-    1994-02-03T14:15:29          -- Use T as separator
-    19940203T141529Z             -- ISO 8601 compact format
-    19940203                     -- only date
-
-    """
-    # clean up
-    text = text.lstrip()
-
-    # tz is time zone specifier string
-    day, mon, yr, hr, min, sec, tz = [None]*7
-
-    # loose regexp parse
-    m = ISO_DATE_RE.search(text)
-    if m is not None:
-        # XXX there's an extra bit of the timezone I'm ignoring here: is
-        #   this the right thing to do?
-        yr, mon, day, hr, min, sec, tz, _ = m.groups()
-    else:
-        return None  # bad format
-
-    return _str2time(day, mon, yr, hr, min, sec, tz)
-
-
-# Header parsing
-# -----------------------------------------------------------------------------
-
-def unmatched(match):
-    """Return unmatched part of re.Match object."""
-    start, end = match.span(0)
-    return match.string[:start]+match.string[end:]
-
-HEADER_TOKEN_RE =        re.compile(r"^\s*([^=\s;,]+)")
-HEADER_QUOTED_VALUE_RE = re.compile(r"^\s*=\s*\"([^\"\\]*(?:\\.[^\"\\]*)*)\"")
-HEADER_VALUE_RE =        re.compile(r"^\s*=\s*([^\s;,]*)")
-HEADER_ESCAPE_RE = re.compile(r"\\(.)")
-def split_header_words(header_values):
-    r"""Parse header values into a list of lists containing key,value pairs.
-
-    The function knows how to deal with ",", ";" and "=" as well as quoted
-    values after "=".  A list of space separated tokens are parsed as if they
-    were separated by ";".
-
-    If the header_values passed as argument contains multiple values, then they
-    are treated as if they were a single value separated by comma ",".
-
-    This means that this function is useful for parsing header fields that
-    follow this syntax (BNF as from the HTTP/1.1 specification, but we relax
-    the requirement for tokens).
-
-      headers           = #header
-      header            = (token | parameter) *( [";"] (token | parameter))
-
-      token             = 1*<any CHAR except CTLs or separators>
-      separators        = "(" | ")" | "<" | ">" | "@"
-                        | "," | ";" | ":" | "\" | <">
-                        | "/" | "[" | "]" | "?" | "="
-                        | "{" | "}" | SP | HT
-
-      quoted-string     = ( <"> *(qdtext | quoted-pair ) <"> )
-      qdtext            = <any TEXT except <">>
-      quoted-pair       = "\" CHAR
-
-      parameter         = attribute "=" value
-      attribute         = token
-      value             = token | quoted-string
-
-    Each header is represented by a list of key/value pairs.  The value for a
-    simple token (not part of a parameter) is None.  Syntactically incorrect
-    headers will not necessarily be parsed as you would want.
-
-    This is easier to describe with some examples:
-
-    >>> split_header_words(['foo="bar"; port="80,81"; discard, bar=baz'])
-    [[('foo', 'bar'), ('port', '80,81'), ('discard', None)], [('bar', 'baz')]]
-    >>> split_header_words(['text/html; charset="iso-8859-1"'])
-    [[('text/html', None), ('charset', 'iso-8859-1')]]
-    >>> split_header_words([r'Basic realm="\"foo\bar\""'])
-    [[('Basic', None), ('realm', '"foobar"')]]
-
-    """
-    assert not isinstance(header_values, str)
-    result = []
-    for text in header_values:
-        orig_text = text
-        pairs = []
-        while text:
-            m = HEADER_TOKEN_RE.search(text)
-            if m:
-                text = unmatched(m)
-                name = m.group(1)
-                m = HEADER_QUOTED_VALUE_RE.search(text)
-                if m:  # quoted value
-                    text = unmatched(m)
-                    value = m.group(1)
-                    value = HEADER_ESCAPE_RE.sub(r"\1", value)
-                else:
-                    m = HEADER_VALUE_RE.search(text)
-                    if m:  # unquoted value
-                        text = unmatched(m)
-                        value = m.group(1)
-                        value = value.rstrip()
-                    else:
-                        # no value, a lone token
-                        value = None
-                pairs.append((name, value))
-            elif text.lstrip().startswith(","):
-                # concatenated headers, as per RFC 2616 section 4.2
-                text = text.lstrip()[1:]
-                if pairs: result.append(pairs)
-                pairs = []
-            else:
-                # skip junk
-                non_junk, nr_junk_chars = re.subn(r"^[=\s;]*", "", text)
-                assert nr_junk_chars > 0, (
-                    "split_header_words bug: '%s', '%s', %s" %
-                    (orig_text, text, pairs))
-                text = non_junk
-        if pairs: result.append(pairs)
-    return result
-
-HEADER_JOIN_ESCAPE_RE = re.compile(r"([\"\\])")
-def join_header_words(lists):
-    """Do the inverse (almost) of the conversion done by split_header_words.
-
-    Takes a list of lists of (key, value) pairs and produces a single header
-    value.  Attribute values are quoted if needed.
-
-    >>> join_header_words([[("text/plain", None), ("charset", "iso-8859-1")]])
-    'text/plain; charset="iso-8859-1"'
-    >>> join_header_words([[("text/plain", None)], [("charset", "iso-8859-1")]])
-    'text/plain, charset="iso-8859-1"'
-
-    """
-    headers = []
-    for pairs in lists:
-        attr = []
-        for k, v in pairs:
-            if v is not None:
-                if not re.search(r"^\w+$", v):
-                    v = HEADER_JOIN_ESCAPE_RE.sub(r"\\\1", v)  # escape " and \
-                    v = '"%s"' % v
-                k = "%s=%s" % (k, v)
-            attr.append(k)
-        if attr: headers.append("; ".join(attr))
-    return ", ".join(headers)
-
-def strip_quotes(text):
-    if text.startswith('"'):
-        text = text[1:]
-    if text.endswith('"'):
-        text = text[:-1]
-    return text
-
-def parse_ns_headers(ns_headers):
-    """Ad-hoc parser for Netscape protocol cookie-attributes.
-
-    The old Netscape cookie format for Set-Cookie can for instance contain
-    an unquoted "," in the expires field, so we have to use this ad-hoc
-    parser instead of split_header_words.
-
-    XXX This may not make the best possible effort to parse all the crap
-    that Netscape Cookie headers contain.  Ronald Tschalar's HTTPClient
-    parser is probably better, so could do worse than following that if
-    this ever gives any trouble.
-
-    Currently, this is also used for parsing RFC 2109 cookies.
-
-    """
-    known_attrs = ("expires", "domain", "path", "secure",
-                   # RFC 2109 attrs (may turn up in Netscape cookies, too)
-                   "version", "port", "max-age")
-
-    result = []
-    for ns_header in ns_headers:
-        pairs = []
-        version_set = False
-
-        # XXX: The following does not strictly adhere to RFCs in that empty
-        # names and values are legal (the former will only appear once and will
-        # be overwritten if multiple occurrences are present). This is
-        # mostly to deal with backwards compatibility.
-        for ii, param in enumerate(ns_header.split(';')):
-            param = param.strip()
-
-            key, sep, val = param.partition('=')
-            key = key.strip()
-
-            if not key:
-                if ii == 0:
-                    break
-                else:
-                    continue
-
-            # allow for a distinction between present and empty and missing
-            # altogether
-            val = val.strip() if sep else None
-
-            if ii != 0:
-                lc = key.lower()
-                if lc in known_attrs:
-                    key = lc
-
-                if key == "version":
-                    # This is an RFC 2109 cookie.
-                    if val is not None:
-                        val = strip_quotes(val)
-                    version_set = True
-                elif key == "expires":
-                    # convert expires date to seconds since epoch
-                    if val is not None:
-                        val = http2time(strip_quotes(val))  # None if invalid
-            pairs.append((key, val))
-
-        if pairs:
-            if not version_set:
-                pairs.append(("version", "0"))
-            result.append(pairs)
-
-    return result
-
-
-IPV4_RE = re.compile(r"\.\d+$", re.ASCII)
-def is_HDN(text):
-    """Return True if text is a host domain name."""
-    # XXX
-    # This may well be wrong.  Which RFC is HDN defined in, if any (for
-    #  the purposes of RFC 2965)?
-    # For the current implementation, what about IPv6?  Remember to look
-    #  at other uses of IPV4_RE also, if change this.
-    if IPV4_RE.search(text):
-        return False
-    if text == "":
-        return False
-    if text[0] == "." or text[-1] == ".":
-        return False
-    return True
-
-def domain_match(A, B):
-    """Return True if domain A domain-matches domain B, according to RFC 2965.
-
-    A and B may be host domain names or IP addresses.
-
-    RFC 2965, section 1:
-
-    Host names can be specified either as an IP address or a HDN string.
-    Sometimes we compare one host name with another.  (Such comparisons SHALL
-    be case-insensitive.)  Host A's name domain-matches host B's if
-
-         *  their host name strings string-compare equal; or
-
-         * A is a HDN string and has the form NB, where N is a non-empty
-            name string, B has the form .B', and B' is a HDN string.  (So,
-            x.y.com domain-matches .Y.com but not Y.com.)
-
-    Note that domain-match is not a commutative operation: a.b.c.com
-    domain-matches .c.com, but not the reverse.
-
-    """
-    # Note that, if A or B are IP addresses, the only relevant part of the
-    # definition of the domain-match algorithm is the direct string-compare.
-    A = A.lower()
-    B = B.lower()
-    if A == B:
-        return True
-    if not is_HDN(A):
-        return False
-    i = A.rfind(B)
-    if i == -1 or i == 0:
-        # A does not have form NB, or N is the empty string
-        return False
-    if not B.startswith("."):
-        return False
-    if not is_HDN(B[1:]):
-        return False
-    return True
-
-def liberal_is_HDN(text):
-    """Return True if text is a sort-of-like a host domain name.
-
-    For accepting/blocking domains.
-
-    """
-    if IPV4_RE.search(text):
-        return False
-    return True
-
-def user_domain_match(A, B):
-    """For blocking/accepting domains.
-
-    A and B may be host domain names or IP addresses.
-
-    """
-    A = A.lower()
-    B = B.lower()
-    if not (liberal_is_HDN(A) and liberal_is_HDN(B)):
-        if A == B:
-            # equal IP addresses
-            return True
-        return False
-    initial_dot = B.startswith(".")
-    if initial_dot and A.endswith(B):
-        return True
-    if not initial_dot and A == B:
-        return True
-    return False
-
-cut_port_re = re.compile(r":\d+$", re.ASCII)
-def request_host(request):
-    """Return request-host, as defined by RFC 2965.
-
-    Variation from RFC: returned value is lowercased, for convenient
-    comparison.
-
-    """
-    url = request.get_full_url()
-    host = urllib.parse.urlparse(url)[1]
-    if host == "":
-        host = request.get_header("Host", "")
-
-    # remove port, if present
-    host = cut_port_re.sub("", host, 1)
-    return host.lower()
-
-def eff_request_host(request):
-    """Return a tuple (request-host, effective request-host name).
-
-    As defined by RFC 2965, except both are lowercased.
-
-    """
-    erhn = req_host = request_host(request)
-    if req_host.find(".") == -1 and not IPV4_RE.search(req_host):
-        erhn = req_host + ".local"
-    return req_host, erhn
-
-def request_path(request):
-    """Path component of request-URI, as defined by RFC 2965."""
-    url = request.get_full_url()
-    parts = urllib.parse.urlsplit(url)
-    path = escape_path(parts.path)
-    if not path.startswith("/"):
-        # fix bad RFC 2396 absoluteURI
-        path = "/" + path
-    return path
-
-def request_port(request):
-    host = request.host
-    i = host.find(':')
-    if i >= 0:
-        port = host[i+1:]
-        try:
-            int(port)
-        except ValueError:
-            _debug("nonnumeric port: '%s'", port)
-            return None
-    else:
-        port = DEFAULT_HTTP_PORT
-    return port
-
-# Characters in addition to A-Z, a-z, 0-9, '_', '.', and '-' that don't
-# need to be escaped to form a valid HTTP URL (RFCs 2396 and 1738).
-HTTP_PATH_SAFE = "%/;:@&=+$,!~*'()"
-ESCAPED_CHAR_RE = re.compile(r"%([0-9a-fA-F][0-9a-fA-F])")
-def uppercase_escaped_char(match):
-    return "%%%s" % match.group(1).upper()
-def escape_path(path):
-    """Escape any invalid characters in HTTP URL, and uppercase all escapes."""
-    # There's no knowing what character encoding was used to create URLs
-    # containing %-escapes, but since we have to pick one to escape invalid
-    # path characters, we pick UTF-8, as recommended in the HTML 4.0
-    # specification:
-    # http://www.w3.org/TR/REC-html40/appendix/notes.html#h-B.2.1
-    # And here, kind of: draft-fielding-uri-rfc2396bis-03
-    # (And in draft IRI specification: draft-duerst-iri-05)
-    # (And here, for new URI schemes: RFC 2718)
-    path = urllib.parse.quote(path, HTTP_PATH_SAFE)
-    path = ESCAPED_CHAR_RE.sub(uppercase_escaped_char, path)
-    return path
-
-def reach(h):
-    """Return reach of host h, as defined by RFC 2965, section 1.
-
-    The reach R of a host name H is defined as follows:
-
-       *  If
-
-          -  H is the host domain name of a host; and,
-
-          -  H has the form A.B; and
-
-          -  A has no embedded (that is, interior) dots; and
-
-          -  B has at least one embedded dot, or B is the string "local".
-             then the reach of H is .B.
-
-       *  Otherwise, the reach of H is H.
-
-    >>> reach("www.acme.com")
-    '.acme.com'
-    >>> reach("acme.com")
-    'acme.com'
-    >>> reach("acme.local")
-    '.local'
-
-    """
-    i = h.find(".")
-    if i >= 0:
-        #a = h[:i]  # this line is only here to show what a is
-        b = h[i+1:]
-        i = b.find(".")
-        if is_HDN(h) and (i >= 0 or b == "local"):
-            return "."+b
-    return h
-
-def is_third_party(request):
-    """
-
-    RFC 2965, section 3.3.6:
-
-        An unverifiable transaction is to a third-party host if its request-
-        host U does not domain-match the reach R of the request-host O in the
-        origin transaction.
-
-    """
-    req_host = request_host(request)
-    if not domain_match(req_host, reach(request.origin_req_host)):
-        return True
-    else:
-        return False
-
-
-class Cookie:
-    """HTTP Cookie.
-
-    This class represents both Netscape and RFC 2965 cookies.
-
-    This is deliberately a very simple class.  It just holds attributes.  It's
-    possible to construct Cookie instances that don't comply with the cookie
-    standards.  CookieJar.make_cookies is the factory function for Cookie
-    objects -- it deals with cookie parsing, supplying defaults, and
-    normalising to the representation used in this class.  CookiePolicy is
-    responsible for checking them to see whether they should be accepted from
-    and returned to the server.
-
-    Note that the port may be present in the headers, but unspecified ("Port"
-    rather than"Port=80", for example); if this is the case, port is None.
-
-    """
-
-    def __init__(self, version, name, value,
-                 port, port_specified,
-                 domain, domain_specified, domain_initial_dot,
-                 path, path_specified,
-                 secure,
-                 expires,
-                 discard,
-                 comment,
-                 comment_url,
-                 rest,
-                 rfc2109=False,
-                 ):
-
-        if version is not None: version = int(version)
-        if expires is not None: expires = int(float(expires))
-        if port is None and port_specified is True:
-            raise ValueError("if port is None, port_specified must be false")
-
-        self.version = version
-        self.name = name
-        self.value = value
-        self.port = port
-        self.port_specified = port_specified
-        # normalise case, as per RFC 2965 section 3.3.3
-        self.domain = domain.lower()
-        self.domain_specified = domain_specified
-        # Sigh.  We need to know whether the domain given in the
-        # cookie-attribute had an initial dot, in order to follow RFC 2965
-        # (as clarified in draft errata).  Needed for the returned $Domain
-        # value.
-        self.domain_initial_dot = domain_initial_dot
-        self.path = path
-        self.path_specified = path_specified
-        self.secure = secure
-        self.expires = expires
-        self.discard = discard
-        self.comment = comment
-        self.comment_url = comment_url
-        self.rfc2109 = rfc2109
-
-        self._rest = copy.copy(rest)
-
-    def has_nonstandard_attr(self, name):
-        return name in self._rest
-    def get_nonstandard_attr(self, name, default=None):
-        return self._rest.get(name, default)
-    def set_nonstandard_attr(self, name, value):
-        self._rest[name] = value
-
-    def is_expired(self, now=None):
-        if now is None: now = time.time()
-        if (self.expires is not None) and (self.expires <= now):
-            return True
-        return False
-
-    def __str__(self):
-        if self.port is None: p = ""
-        else: p = ":"+self.port
-        limit = self.domain + p + self.path
-        if self.value is not None:
-            namevalue = "%s=%s" % (self.name, self.value)
-        else:
-            namevalue = self.name
-        return "<Cookie %s for %s>" % (namevalue, limit)
-
-    def __repr__(self):
-        args = []
-        for name in ("version", "name", "value",
-                     "port", "port_specified",
-                     "domain", "domain_specified", "domain_initial_dot",
-                     "path", "path_specified",
-                     "secure", "expires", "discard", "comment", "comment_url",
-                     ):
-            attr = getattr(self, name)
-            args.append("%s=%s" % (name, repr(attr)))
-        args.append("rest=%s" % repr(self._rest))
-        args.append("rfc2109=%s" % repr(self.rfc2109))
-        return "%s(%s)" % (self.__class__.__name__, ", ".join(args))
-
-
-class CookiePolicy:
-    """Defines which cookies get accepted from and returned to server.
-
-    May also modify cookies, though this is probably a bad idea.
-
-    The subclass DefaultCookiePolicy defines the standard rules for Netscape
-    and RFC 2965 cookies -- override that if you want a customized policy.
-
-    """
-    def set_ok(self, cookie, request):
-        """Return true if (and only if) cookie should be accepted from server.
-
-        Currently, pre-expired cookies never get this far -- the CookieJar
-        class deletes such cookies itself.
-
-        """
-        raise NotImplementedError()
-
-    def return_ok(self, cookie, request):
-        """Return true if (and only if) cookie should be returned to server."""
-        raise NotImplementedError()
-
-    def domain_return_ok(self, domain, request):
-        """Return false if cookies should not be returned, given cookie domain.
-        """
-        return True
-
-    def path_return_ok(self, path, request):
-        """Return false if cookies should not be returned, given cookie path.
-        """
-        return True
-
-
-class DefaultCookiePolicy(CookiePolicy):
-    """Implements the standard rules for accepting and returning cookies."""
-
-    DomainStrictNoDots = 1
-    DomainStrictNonDomain = 2
-    DomainRFC2965Match = 4
-
-    DomainLiberal = 0
-    DomainStrict = DomainStrictNoDots|DomainStrictNonDomain
-
-    def __init__(self,
-                 blocked_domains=None, allowed_domains=None,
-                 netscape=True, rfc2965=False,
-                 rfc2109_as_netscape=None,
-                 hide_cookie2=False,
-                 strict_domain=False,
-                 strict_rfc2965_unverifiable=True,
-                 strict_ns_unverifiable=False,
-                 strict_ns_domain=DomainLiberal,
-                 strict_ns_set_initial_dollar=False,
-                 strict_ns_set_path=False,
-                 ):
-        """Constructor arguments should be passed as keyword arguments only."""
-        self.netscape = netscape
-        self.rfc2965 = rfc2965
-        self.rfc2109_as_netscape = rfc2109_as_netscape
-        self.hide_cookie2 = hide_cookie2
-        self.strict_domain = strict_domain
-        self.strict_rfc2965_unverifiable = strict_rfc2965_unverifiable
-        self.strict_ns_unverifiable = strict_ns_unverifiable
-        self.strict_ns_domain = strict_ns_domain
-        self.strict_ns_set_initial_dollar = strict_ns_set_initial_dollar
-        self.strict_ns_set_path = strict_ns_set_path
-
-        if blocked_domains is not None:
-            self._blocked_domains = tuple(blocked_domains)
-        else:
-            self._blocked_domains = ()
-
-        if allowed_domains is not None:
-            allowed_domains = tuple(allowed_domains)
-        self._allowed_domains = allowed_domains
-
-    def blocked_domains(self):
-        """Return the sequence of blocked domains (as a tuple)."""
-        return self._blocked_domains
-    def set_blocked_domains(self, blocked_domains):
-        """Set the sequence of blocked domains."""
-        self._blocked_domains = tuple(blocked_domains)
-
-    def is_blocked(self, domain):
-        for blocked_domain in self._blocked_domains:
-            if user_domain_match(domain, blocked_domain):
-                return True
-        return False
-
-    def allowed_domains(self):
-        """Return None, or the sequence of allowed domains (as a tuple)."""
-        return self._allowed_domains
-    def set_allowed_domains(self, allowed_domains):
-        """Set the sequence of allowed domains, or None."""
-        if allowed_domains is not None:
-            allowed_domains = tuple(allowed_domains)
-        self._allowed_domains = allowed_domains
-
-    def is_not_allowed(self, domain):
-        if self._allowed_domains is None:
-            return False
-        for allowed_domain in self._allowed_domains:
-            if user_domain_match(domain, allowed_domain):
-                return False
-        return True
-
-    def set_ok(self, cookie, request):
-        """
-        If you override .set_ok(), be sure to call this method.  If it returns
-        false, so should your subclass (assuming your subclass wants to be more
-        strict about which cookies to accept).
-
-        """
-        _debug(" - checking cookie %s=%s", cookie.name, cookie.value)
-
-        assert cookie.name is not None
-
-        for n in "version", "verifiability", "name", "path", "domain", "port":
-            fn_name = "set_ok_"+n
-            fn = getattr(self, fn_name)
-            if not fn(cookie, request):
-                return False
-
-        return True
-
-    def set_ok_version(self, cookie, request):
-        if cookie.version is None:
-            # Version is always set to 0 by parse_ns_headers if it's a Netscape
-            # cookie, so this must be an invalid RFC 2965 cookie.
-            _debug("   Set-Cookie2 without version attribute (%s=%s)",
-                   cookie.name, cookie.value)
-            return False
-        if cookie.version > 0 and not self.rfc2965:
-            _debug("   RFC 2965 cookies are switched off")
-            return False
-        elif cookie.version == 0 and not self.netscape:
-            _debug("   Netscape cookies are switched off")
-            return False
-        return True
-
-    def set_ok_verifiability(self, cookie, request):
-        if request.unverifiable and is_third_party(request):
-            if cookie.version > 0 and self.strict_rfc2965_unverifiable:
-                _debug("   third-party RFC 2965 cookie during "
-                             "unverifiable transaction")
-                return False
-            elif cookie.version == 0 and self.strict_ns_unverifiable:
-                _debug("   third-party Netscape cookie during "
-                             "unverifiable transaction")
-                return False
-        return True
-
-    def set_ok_name(self, cookie, request):
-        # Try and stop servers setting V0 cookies designed to hack other
-        # servers that know both V0 and V1 protocols.
-        if (cookie.version == 0 and self.strict_ns_set_initial_dollar and
-            cookie.name.startswith("$")):
-            _debug("   illegal name (starts with '$'): '%s'", cookie.name)
-            return False
-        return True
-
-    def set_ok_path(self, cookie, request):
-        if cookie.path_specified:
-            req_path = request_path(request)
-            if ((cookie.version > 0 or
-                 (cookie.version == 0 and self.strict_ns_set_path)) and
-                not req_path.startswith(cookie.path)):
-                _debug("   path attribute %s is not a prefix of request "
-                       "path %s", cookie.path, req_path)
-                return False
-        return True
-
-    def set_ok_domain(self, cookie, request):
-        if self.is_blocked(cookie.domain):
-            _debug("   domain %s is in user block-list", cookie.domain)
-            return False
-        if self.is_not_allowed(cookie.domain):
-            _debug("   domain %s is not in user allow-list", cookie.domain)
-            return False
-        if cookie.domain_specified:
-            req_host, erhn = eff_request_host(request)
-            domain = cookie.domain
-            if self.strict_domain and (domain.count(".") >= 2):
-                # XXX This should probably be compared with the Konqueror
-                # (kcookiejar.cpp) and Mozilla implementations, but it's a
-                # losing battle.
-                i = domain.rfind(".")
-                j = domain.rfind(".", 0, i)
-                if j == 0:  # domain like .foo.bar
-                    tld = domain[i+1:]
-                    sld = domain[j+1:i]
-                    if sld.lower() in ("co", "ac", "com", "edu", "org", "net",
-                       "gov", "mil", "int", "aero", "biz", "cat", "coop",
-                       "info", "jobs", "mobi", "museum", "name", "pro",
-                       "travel", "eu") and len(tld) == 2:
-                        # domain like .co.uk
-                        _debug("   country-code second level domain %s", domain)
-                        return False
-            if domain.startswith("."):
-                undotted_domain = domain[1:]
-            else:
-                undotted_domain = domain
-            embedded_dots = (undotted_domain.find(".") >= 0)
-            if not embedded_dots and domain != ".local":
-                _debug("   non-local domain %s contains no embedded dot",
-                       domain)
-                return False
-            if cookie.version == 0:
-                if (not erhn.endswith(domain) and
-                    (not erhn.startswith(".") and
-                     not ("."+erhn).endswith(domain))):
-                    _debug("   effective request-host %s (even with added "
-                           "initial dot) does not end with %s",
-                           erhn, domain)
-                    return False
-            if (cookie.version > 0 or
-                (self.strict_ns_domain & self.DomainRFC2965Match)):
-                if not domain_match(erhn, domain):
-                    _debug("   effective request-host %s does not domain-match "
-                           "%s", erhn, domain)
-                    return False
-            if (cookie.version > 0 or
-                (self.strict_ns_domain & self.DomainStrictNoDots)):
-                host_prefix = req_host[:-len(domain)]
-                if (host_prefix.find(".") >= 0 and
-                    not IPV4_RE.search(req_host)):
-                    _debug("   host prefix %s for domain %s contains a dot",
-                           host_prefix, domain)
-                    return False
-        return True
-
-    def set_ok_port(self, cookie, request):
-        if cookie.port_specified:
-            req_port = request_port(request)
-            if req_port is None:
-                req_port = "80"
-            else:
-                req_port = str(req_port)
-            for p in cookie.port.split(","):
-                try:
-                    int(p)
-                except ValueError:
-                    _debug("   bad port %s (not numeric)", p)
-                    return False
-                if p == req_port:
-                    break
-            else:
-                _debug("   request port (%s) not found in %s",
-                       req_port, cookie.port)
-                return False
-        return True
-
-    def return_ok(self, cookie, request):
-        """
-        If you override .return_ok(), be sure to call this method.  If it
-        returns false, so should your subclass (assuming your subclass wants to
-        be more strict about which cookies to return).
-
-        """
-        # Path has already been checked by .path_return_ok(), and domain
-        # blocking done by .domain_return_ok().
-        _debug(" - checking cookie %s=%s", cookie.name, cookie.value)
-
-        for n in "version", "verifiability", "secure", "expires", "port", "domain":
-            fn_name = "return_ok_"+n
-            fn = getattr(self, fn_name)
-            if not fn(cookie, request):
-                return False
-        return True
-
-    def return_ok_version(self, cookie, request):
-        if cookie.version > 0 and not self.rfc2965:
-            _debug("   RFC 2965 cookies are switched off")
-            return False
-        elif cookie.version == 0 and not self.netscape:
-            _debug("   Netscape cookies are switched off")
-            return False
-        return True
-
-    def return_ok_verifiability(self, cookie, request):
-        if request.unverifiable and is_third_party(request):
-            if cookie.version > 0 and self.strict_rfc2965_unverifiable:
-                _debug("   third-party RFC 2965 cookie during unverifiable "
-                       "transaction")
-                return False
-            elif cookie.version == 0 and self.strict_ns_unverifiable:
-                _debug("   third-party Netscape cookie during unverifiable "
-                       "transaction")
-                return False
-        return True
-
-    def return_ok_secure(self, cookie, request):
-        if cookie.secure and request.type != "https":
-            _debug("   secure cookie with non-secure request")
-            return False
-        return True
-
-    def return_ok_expires(self, cookie, request):
-        if cookie.is_expired(self._now):
-            _debug("   cookie expired")
-            return False
-        return True
-
-    def return_ok_port(self, cookie, request):
-        if cookie.port:
-            req_port = request_port(request)
-            if req_port is None:
-                req_port = "80"
-            for p in cookie.port.split(","):
-                if p == req_port:
-                    break
-            else:
-                _debug("   request port %s does not match cookie port %s",
-                       req_port, cookie.port)
-                return False
-        return True
-
-    def return_ok_domain(self, cookie, request):
-        req_host, erhn = eff_request_host(request)
-        domain = cookie.domain
-
-        # strict check of non-domain cookies: Mozilla does this, MSIE5 doesn't
-        if (cookie.version == 0 and
-            (self.strict_ns_domain & self.DomainStrictNonDomain) and
-            not cookie.domain_specified and domain != erhn):
-            _debug("   cookie with unspecified domain does not string-compare "
-                   "equal to request domain")
-            return False
-
-        if cookie.version > 0 and not domain_match(erhn, domain):
-            _debug("   effective request-host name %s does not domain-match "
-                   "RFC 2965 cookie domain %s", erhn, domain)
-            return False
-        if cookie.version == 0 and not ("."+erhn).endswith(domain):
-            _debug("   request-host %s does not match Netscape cookie domain "
-                   "%s", req_host, domain)
-            return False
-        return True
-
-    def domain_return_ok(self, domain, request):
-        # Liberal check of.  This is here as an optimization to avoid
-        # having to load lots of MSIE cookie files unless necessary.
-        req_host, erhn = eff_request_host(request)
-        if not req_host.startswith("."):
-            req_host = "."+req_host
-        if not erhn.startswith("."):
-            erhn = "."+erhn
-        if not (req_host.endswith(domain) or erhn.endswith(domain)):
-            #_debug("   request domain %s does not match cookie domain %s",
-            #       req_host, domain)
-            return False
-
-        if self.is_blocked(domain):
-            _debug("   domain %s is in user block-list", domain)
-            return False
-        if self.is_not_allowed(domain):
-            _debug("   domain %s is not in user allow-list", domain)
-            return False
-
-        return True
-
-    def path_return_ok(self, path, request):
-        _debug("- checking cookie path=%s", path)
-        req_path = request_path(request)
-        if not req_path.startswith(path):
-            _debug("  %s does not path-match %s", req_path, path)
-            return False
-        return True
-
-
-def vals_sorted_by_key(adict):
-    keys = sorted(adict.keys())
-    return map(adict.get, keys)
-
-def deepvalues(mapping):
-    """Iterates over nested mapping, depth-first, in sorted order by key."""
-    values = vals_sorted_by_key(mapping)
-    for obj in values:
-        mapping = False
-        try:
-            obj.items
-        except AttributeError:
-            pass
-        else:
-            mapping = True
-            yield from deepvalues(obj)
-        if not mapping:
-            yield obj
-
-
-# Used as second parameter to dict.get() method, to distinguish absent
-# dict key from one with a None value.
-class Absent: pass
-
-class CookieJar:
-    """Collection of HTTP cookies.
-
-    You may not need to know about this class: try
-    urllib.request.build_opener(HTTPCookieProcessor).open(url).
-    """
-
-    non_word_re = re.compile(r"\W")
-    quote_re = re.compile(r"([\"\\])")
-    strict_domain_re = re.compile(r"\.?[^.]*")
-    domain_re = re.compile(r"[^.]*")
-    dots_re = re.compile(r"^\.+")
-
-    magic_re = re.compile(r"^\#LWP-Cookies-(\d+\.\d+)", re.ASCII)
-
-    def __init__(self, policy=None):
-        if policy is None:
-            policy = DefaultCookiePolicy()
-        self._policy = policy
-
-        self._cookies_lock = _threading.RLock()
-        self._cookies = {}
-
-    def set_policy(self, policy):
-        self._policy = policy
-
-    def _cookies_for_domain(self, domain, request):
-        cookies = []
-        if not self._policy.domain_return_ok(domain, request):
-            return []
-        _debug("Checking %s for cookies to return", domain)
-        cookies_by_path = self._cookies[domain]
-        for path in cookies_by_path.keys():
-            if not self._policy.path_return_ok(path, request):
-                continue
-            cookies_by_name = cookies_by_path[path]
-            for cookie in cookies_by_name.values():
-                if not self._policy.return_ok(cookie, request):
-                    _debug("   not returning cookie")
-                    continue
-                _debug("   it's a match")
-                cookies.append(cookie)
-        return cookies
-
-    def _cookies_for_request(self, request):
-        """Return a list of cookies to be returned to server."""
-        cookies = []
-        for domain in self._cookies.keys():
-            cookies.extend(self._cookies_for_domain(domain, request))
-        return cookies
-
-    def _cookie_attrs(self, cookies):
-        """Return a list of cookie-attributes to be returned to server.
-
-        like ['foo="bar"; $Path="/"', ...]
-
-        The $Version attribute is also added when appropriate (currently only
-        once per request).
-
-        """
-        # add cookies in order of most specific (ie. longest) path first
-        cookies.sort(key=lambda a: len(a.path), reverse=True)
-
-        version_set = False
-
-        attrs = []
-        for cookie in cookies:
-            # set version of Cookie header
-            # XXX
-            # What should it be if multiple matching Set-Cookie headers have
-            #  different versions themselves?
-            # Answer: there is no answer; was supposed to be settled by
-            #  RFC 2965 errata, but that may never appear...
-            version = cookie.version
-            if not version_set:
-                version_set = True
-                if version > 0:
-                    attrs.append("$Version=%s" % version)
-
-            # quote cookie value if necessary
-            # (not for Netscape protocol, which already has any quotes
-            #  intact, due to the poorly-specified Netscape Cookie: syntax)
-            if ((cookie.value is not None) and
-                self.non_word_re.search(cookie.value) and version > 0):
-                value = self.quote_re.sub(r"\\\1", cookie.value)
-            else:
-                value = cookie.value
-
-            # add cookie-attributes to be returned in Cookie header
-            if cookie.value is None:
-                attrs.append(cookie.name)
-            else:
-                attrs.append("%s=%s" % (cookie.name, value))
-            if version > 0:
-                if cookie.path_specified:
-                    attrs.append('$Path="%s"' % cookie.path)
-                if cookie.domain.startswith("."):
-                    domain = cookie.domain
-                    if (not cookie.domain_initial_dot and
-                        domain.startswith(".")):
-                        domain = domain[1:]
-                    attrs.append('$Domain="%s"' % domain)
-                if cookie.port is not None:
-                    p = "$Port"
-                    if cookie.port_specified:
-                        p = p + ('="%s"' % cookie.port)
-                    attrs.append(p)
-
-        return attrs
-
-    def add_cookie_header(self, request):
-        """Add correct Cookie: header to request (urllib.request.Request object).
-
-        The Cookie2 header is also added unless policy.hide_cookie2 is true.
-
-        """
-        _debug("add_cookie_header")
-        self._cookies_lock.acquire()
-        try:
-
-            self._policy._now = self._now = int(time.time())
-
-            cookies = self._cookies_for_request(request)
-
-            attrs = self._cookie_attrs(cookies)
-            if attrs:
-                if not request.has_header("Cookie"):
-                    request.add_unredirected_header(
-                        "Cookie", "; ".join(attrs))
-
-            # if necessary, advertise that we know RFC 2965
-            if (self._policy.rfc2965 and not self._policy.hide_cookie2 and
-                not request.has_header("Cookie2")):
-                for cookie in cookies:
-                    if cookie.version != 1:
-                        request.add_unredirected_header("Cookie2", '$Version="1"')
-                        break
-
-        finally:
-            self._cookies_lock.release()
-
-        self.clear_expired_cookies()
-
-    def _normalized_cookie_tuples(self, attrs_set):
-        """Return list of tuples containing normalised cookie information.
-
-        attrs_set is the list of lists of key,value pairs extracted from
-        the Set-Cookie or Set-Cookie2 headers.
-
-        Tuples are name, value, standard, rest, where name and value are the
-        cookie name and value, standard is a dictionary containing the standard
-        cookie-attributes (discard, secure, version, expires or max-age,
-        domain, path and port) and rest is a dictionary containing the rest of
-        the cookie-attributes.
-
-        """
-        cookie_tuples = []
-
-        boolean_attrs = "discard", "secure"
-        value_attrs = ("version",
-                       "expires", "max-age",
-                       "domain", "path", "port",
-                       "comment", "commenturl")
-
-        for cookie_attrs in attrs_set:
-            name, value = cookie_attrs[0]
-
-            # Build dictionary of standard cookie-attributes (standard) and
-            # dictionary of other cookie-attributes (rest).
-
-            # Note: expiry time is normalised to seconds since epoch.  V0
-            # cookies should have the Expires cookie-attribute, and V1 cookies
-            # should have Max-Age, but since V1 includes RFC 2109 cookies (and
-            # since V0 cookies may be a mish-mash of Netscape and RFC 2109), we
-            # accept either (but prefer Max-Age).
-            max_age_set = False
-
-            bad_cookie = False
-
-            standard = {}
-            rest = {}
-            for k, v in cookie_attrs[1:]:
-                lc = k.lower()
-                # don't lose case distinction for unknown fields
-                if lc in value_attrs or lc in boolean_attrs:
-                    k = lc
-                if k in boolean_attrs and v is None:
-                    # boolean cookie-attribute is present, but has no value
-                    # (like "discard", rather than "port=80")
-                    v = True
-                if k in standard:
-                    # only first value is significant
-                    continue
-                if k == "domain":
-                    if v is None:
-                        _debug("   missing value for domain attribute")
-                        bad_cookie = True
-                        break
-                    # RFC 2965 section 3.3.3
-                    v = v.lower()
-                if k == "expires":
-                    if max_age_set:
-                        # Prefer max-age to expires (like Mozilla)
-                        continue
-                    if v is None:
-                        _debug("   missing or invalid value for expires "
-                              "attribute: treating as session cookie")
-                        continue
-                if k == "max-age":
-                    max_age_set = True
-                    try:
-                        v = int(v)
-                    except ValueError:
-                        _debug("   missing or invalid (non-numeric) value for "
-                              "max-age attribute")
-                        bad_cookie = True
-                        break
-                    # convert RFC 2965 Max-Age to seconds since epoch
-                    # XXX Strictly you're supposed to follow RFC 2616
-                    #   age-calculation rules.  Remember that zero Max-Age
-                    #   is a request to discard (old and new) cookie, though.
-                    k = "expires"
-                    v = self._now + v
-                if (k in value_attrs) or (k in boolean_attrs):
-                    if (v is None and
-                        k not in ("port", "comment", "commenturl")):
-                        _debug("   missing value for %s attribute" % k)
-                        bad_cookie = True
-                        break
-                    standard[k] = v
-                else:
-                    rest[k] = v
-
-            if bad_cookie:
-                continue
-
-            cookie_tuples.append((name, value, standard, rest))
-
-        return cookie_tuples
-
-    def _cookie_from_cookie_tuple(self, tup, request):
-        # standard is dict of standard cookie-attributes, rest is dict of the
-        # rest of them
-        name, value, standard, rest = tup
-
-        domain = standard.get("domain", Absent)
-        path = standard.get("path", Absent)
-        port = standard.get("port", Absent)
-        expires = standard.get("expires", Absent)
-
-        # set the easy defaults
-        version = standard.get("version", None)
-        if version is not None:
-            try:
-                version = int(version)
-            except ValueError:
-                return None  # invalid version, ignore cookie
-        secure = standard.get("secure", False)
-        # (discard is also set if expires is Absent)
-        discard = standard.get("discard", False)
-        comment = standard.get("comment", None)
-        comment_url = standard.get("commenturl", None)
-
-        # set default path
-        if path is not Absent and path != "":
-            path_specified = True
-            path = escape_path(path)
-        else:
-            path_specified = False
-            path = request_path(request)
-            i = path.rfind("/")
-            if i != -1:
-                if version == 0:
-                    # Netscape spec parts company from reality here
-                    path = path[:i]
-                else:
-                    path = path[:i+1]
-            if len(path) == 0: path = "/"
-
-        # set default domain
-        domain_specified = domain is not Absent
-        # but first we have to remember whether it starts with a dot
-        domain_initial_dot = False
-        if domain_specified:
-            domain_initial_dot = bool(domain.startswith("."))
-        if domain is Absent:
-            req_host, erhn = eff_request_host(request)
-            domain = erhn
-        elif not domain.startswith("."):
-            domain = "."+domain
-
-        # set default port
-        port_specified = False
-        if port is not Absent:
-            if port is None:
-                # Port attr present, but has no value: default to request port.
-                # Cookie should then only be sent back on that port.
-                port = request_port(request)
-            else:
-                port_specified = True
-                port = re.sub(r"\s+", "", port)
-        else:
-            # No port attr present.  Cookie can be sent back on any port.
-            port = None
-
-        # set default expires and discard
-        if expires is Absent:
-            expires = None
-            discard = True
-        elif expires <= self._now:
-            # Expiry date in past is request to delete cookie.  This can't be
-            # in DefaultCookiePolicy, because can't delete cookies there.
-            try:
-                self.clear(domain, path, name)
-            except KeyError:
-                pass
-            _debug("Expiring cookie, domain='%s', path='%s', name='%s'",
-                   domain, path, name)
-            return None
-
-        return Cookie(version,
-                      name, value,
-                      port, port_specified,
-                      domain, domain_specified, domain_initial_dot,
-                      path, path_specified,
-                      secure,
-                      expires,
-                      discard,
-                      comment,
-                      comment_url,
-                      rest)
-
-    def _cookies_from_attrs_set(self, attrs_set, request):
-        cookie_tuples = self._normalized_cookie_tuples(attrs_set)
-
-        cookies = []
-        for tup in cookie_tuples:
-            cookie = self._cookie_from_cookie_tuple(tup, request)
-            if cookie: cookies.append(cookie)
-        return cookies
-
-    def _process_rfc2109_cookies(self, cookies):
-        rfc2109_as_ns = getattr(self._policy, 'rfc2109_as_netscape', None)
-        if rfc2109_as_ns is None:
-            rfc2109_as_ns = not self._policy.rfc2965
-        for cookie in cookies:
-            if cookie.version == 1:
-                cookie.rfc2109 = True
-                if rfc2109_as_ns:
-                    # treat 2109 cookies as Netscape cookies rather than
-                    # as RFC2965 cookies
-                    cookie.version = 0
-
-    def make_cookies(self, response, request):
-        """Return sequence of Cookie objects extracted from response object."""
-        # get cookie-attributes for RFC 2965 and Netscape protocols
-        headers = response.info()
-        rfc2965_hdrs = headers.get_all("Set-Cookie2", [])
-        ns_hdrs = headers.get_all("Set-Cookie", [])
-
-        rfc2965 = self._policy.rfc2965
-        netscape = self._policy.netscape
-
-        if ((not rfc2965_hdrs and not ns_hdrs) or
-            (not ns_hdrs and not rfc2965) or
-            (not rfc2965_hdrs and not netscape) or
-            (not netscape and not rfc2965)):
-            return []  # no relevant cookie headers: quick exit
-
-        try:
-            cookies = self._cookies_from_attrs_set(
-                split_header_words(rfc2965_hdrs), request)
-        except Exception:
-            _warn_unhandled_exception()
-            cookies = []
-
-        if ns_hdrs and netscape:
-            try:
-                # RFC 2109 and Netscape cookies
-                ns_cookies = self._cookies_from_attrs_set(
-                    parse_ns_headers(ns_hdrs), request)
-            except Exception:
-                _warn_unhandled_exception()
-                ns_cookies = []
-            self._process_rfc2109_cookies(ns_cookies)
-
-            # Look for Netscape cookies (from Set-Cookie headers) that match
-            # corresponding RFC 2965 cookies (from Set-Cookie2 headers).
-            # For each match, keep the RFC 2965 cookie and ignore the Netscape
-            # cookie (RFC 2965 section 9.1).  Actually, RFC 2109 cookies are
-            # bundled in with the Netscape cookies for this purpose, which is
-            # reasonable behaviour.
-            if rfc2965:
-                lookup = {}
-                for cookie in cookies:
-                    lookup[(cookie.domain, cookie.path, cookie.name)] = None
-
-                def no_matching_rfc2965(ns_cookie, lookup=lookup):
-                    key = ns_cookie.domain, ns_cookie.path, ns_cookie.name
-                    return key not in lookup
-                ns_cookies = filter(no_matching_rfc2965, ns_cookies)
-
-            if ns_cookies:
-                cookies.extend(ns_cookies)
-
-        return cookies
-
-    def set_cookie_if_ok(self, cookie, request):
-        """Set a cookie if policy says it's OK to do so."""
-        self._cookies_lock.acquire()
-        try:
-            self._policy._now = self._now = int(time.time())
-
-            if self._policy.set_ok(cookie, request):
-                self.set_cookie(cookie)
-
-
-        finally:
-            self._cookies_lock.release()
-
-    def set_cookie(self, cookie):
-        """Set a cookie, without checking whether or not it should be set."""
-        c = self._cookies
-        self._cookies_lock.acquire()
-        try:
-            if cookie.domain not in c: c[cookie.domain] = {}
-            c2 = c[cookie.domain]
-            if cookie.path not in c2: c2[cookie.path] = {}
-            c3 = c2[cookie.path]
-            c3[cookie.name] = cookie
-        finally:
-            self._cookies_lock.release()
-
-    def extract_cookies(self, response, request):
-        """Extract cookies from response, where allowable given the request."""
-        _debug("extract_cookies: %s", response.info())
-        self._cookies_lock.acquire()
-        try:
-            self._policy._now = self._now = int(time.time())
-
-            for cookie in self.make_cookies(response, request):
-                if self._policy.set_ok(cookie, request):
-                    _debug(" setting cookie: %s", cookie)
-                    self.set_cookie(cookie)
-        finally:
-            self._cookies_lock.release()
-
-    def clear(self, domain=None, path=None, name=None):
-        """Clear some cookies.
-
-        Invoking this method without arguments will clear all cookies.  If
-        given a single argument, only cookies belonging to that domain will be
-        removed.  If given two arguments, cookies belonging to the specified
-        path within that domain are removed.  If given three arguments, then
-        the cookie with the specified name, path and domain is removed.
-
-        Raises KeyError if no matching cookie exists.
-
-        """
-        if name is not None:
-            if (domain is None) or (path is None):
-                raise ValueError(
-                    "domain and path must be given to remove a cookie by name")
-            del self._cookies[domain][path][name]
-        elif path is not None:
-            if domain is None:
-                raise ValueError(
-                    "domain must be given to remove cookies by path")
-            del self._cookies[domain][path]
-        elif domain is not None:
-            del self._cookies[domain]
-        else:
-            self._cookies = {}
-
-    def clear_session_cookies(self):
-        """Discard all session cookies.
-
-        Note that the .save() method won't save session cookies anyway, unless
-        you ask otherwise by passing a true ignore_discard argument.
-
-        """
-        self._cookies_lock.acquire()
-        try:
-            for cookie in self:
-                if cookie.discard:
-                    self.clear(cookie.domain, cookie.path, cookie.name)
-        finally:
-            self._cookies_lock.release()
-
-    def clear_expired_cookies(self):
-        """Discard all expired cookies.
-
-        You probably don't need to call this method: expired cookies are never
-        sent back to the server (provided you're using DefaultCookiePolicy),
-        this method is called by CookieJar itself every so often, and the
-        .save() method won't save expired cookies anyway (unless you ask
-        otherwise by passing a true ignore_expires argument).
-
-        """
-        self._cookies_lock.acquire()
-        try:
-            now = time.time()
-            for cookie in self:
-                if cookie.is_expired(now):
-                    self.clear(cookie.domain, cookie.path, cookie.name)
-        finally:
-            self._cookies_lock.release()
-
-    def __iter__(self):
-        return deepvalues(self._cookies)
-
-    def __len__(self):
-        """Return number of contained cookies."""
-        i = 0
-        for cookie in self: i = i + 1
-        return i
-
-    def __repr__(self):
-        r = []
-        for cookie in self: r.append(repr(cookie))
-        return "<%s[%s]>" % (self.__class__.__name__, ", ".join(r))
-
-    def __str__(self):
-        r = []
-        for cookie in self: r.append(str(cookie))
-        return "<%s[%s]>" % (self.__class__.__name__, ", ".join(r))
-
-
-# derives from OSError for backwards-compatibility with Python 2.4.0
-class LoadError(OSError): pass
-
-class FileCookieJar(CookieJar):
-    """CookieJar that can be loaded from and saved to a file."""
-
-    def __init__(self, filename=None, delayload=False, policy=None):
-        """
-        Cookies are NOT loaded from the named file until either the .load() or
-        .revert() method is called.
-
-        """
-        CookieJar.__init__(self, policy)
-        if filename is not None:
-            try:
-                filename+""
-            except:
-                raise ValueError("filename must be string-like")
-        self.filename = filename
-        self.delayload = bool(delayload)
-
-    def save(self, filename=None, ignore_discard=False, ignore_expires=False):
-        """Save cookies to a file."""
-        raise NotImplementedError()
-
-    def load(self, filename=None, ignore_discard=False, ignore_expires=False):
-        """Load cookies from a file."""
-        if filename is None:
-            if self.filename is not None: filename = self.filename
-            else: raise ValueError(MISSING_FILENAME_TEXT)
-
-        with open(filename) as f:
-            self._really_load(f, filename, ignore_discard, ignore_expires)
-
-    def revert(self, filename=None,
-               ignore_discard=False, ignore_expires=False):
-        """Clear all cookies and reload cookies from a saved file.
-
-        Raises LoadError (or OSError) if reversion is not successful; the
-        object's state will not be altered if this happens.
-
-        """
-        if filename is None:
-            if self.filename is not None: filename = self.filename
-            else: raise ValueError(MISSING_FILENAME_TEXT)
-
-        self._cookies_lock.acquire()
-        try:
-
-            old_state = copy.deepcopy(self._cookies)
-            self._cookies = {}
-            try:
-                self.load(filename, ignore_discard, ignore_expires)
-            except OSError:
-                self._cookies = old_state
-                raise
-
-        finally:
-            self._cookies_lock.release()
-
-
-def lwp_cookie_str(cookie):
-    """Return string representation of Cookie in the LWP cookie file format.
-
-    Actually, the format is extended a bit -- see module docstring.
-
-    """
-    h = [(cookie.name, cookie.value),
-         ("path", cookie.path),
-         ("domain", cookie.domain)]
-    if cookie.port is not None: h.append(("port", cookie.port))
-    if cookie.path_specified: h.append(("path_spec", None))
-    if cookie.port_specified: h.append(("port_spec", None))
-    if cookie.domain_initial_dot: h.append(("domain_dot", None))
-    if cookie.secure: h.append(("secure", None))
-    if cookie.expires: h.append(("expires",
-                               time2isoz(float(cookie.expires))))
-    if cookie.discard: h.append(("discard", None))
-    if cookie.comment: h.append(("comment", cookie.comment))
-    if cookie.comment_url: h.append(("commenturl", cookie.comment_url))
-
-    keys = sorted(cookie._rest.keys())
-    for k in keys:
-        h.append((k, str(cookie._rest[k])))
-
-    h.append(("version", str(cookie.version)))
-
-    return join_header_words([h])
-
-class LWPCookieJar(FileCookieJar):
-    """
-    The LWPCookieJar saves a sequence of "Set-Cookie3" lines.
-    "Set-Cookie3" is the format used by the libwww-perl library, not known
-    to be compatible with any browser, but which is easy to read and
-    doesn't lose information about RFC 2965 cookies.
-
-    Additional methods
-
-    as_lwp_str(ignore_discard=True, ignore_expired=True)
-
-    """
-
-    def as_lwp_str(self, ignore_discard=True, ignore_expires=True):
-        """Return cookies as a string of "\\n"-separated "Set-Cookie3" headers.
-
-        ignore_discard and ignore_expires: see docstring for FileCookieJar.save
-
-        """
-        now = time.time()
-        r = []
-        for cookie in self:
-            if not ignore_discard and cookie.discard:
-                continue
-            if not ignore_expires and cookie.is_expired(now):
-                continue
-            r.append("Set-Cookie3: %s" % lwp_cookie_str(cookie))
-        return "\n".join(r+[""])
-
-    def save(self, filename=None, ignore_discard=False, ignore_expires=False):
-        if filename is None:
-            if self.filename is not None: filename = self.filename
-            else: raise ValueError(MISSING_FILENAME_TEXT)
-
-        with open(filename, "w") as f:
-            # There really isn't an LWP Cookies 2.0 format, but this indicates
-            # that there is extra information in here (domain_dot and
-            # port_spec) while still being compatible with libwww-perl, I hope.
-            f.write("#LWP-Cookies-2.0\n")
-            f.write(self.as_lwp_str(ignore_discard, ignore_expires))
-
-    def _really_load(self, f, filename, ignore_discard, ignore_expires):
-        magic = f.readline()
-        if not self.magic_re.search(magic):
-            msg = ("%r does not look like a Set-Cookie3 (LWP) format "
-                   "file" % filename)
-            raise LoadError(msg)
-
-        now = time.time()
-
-        header = "Set-Cookie3:"
-        boolean_attrs = ("port_spec", "path_spec", "domain_dot",
-                         "secure", "discard")
-        value_attrs = ("version",
-                       "port", "path", "domain",
-                       "expires",
-                       "comment", "commenturl")
-
-        try:
-            while 1:
-                line = f.readline()
-                if line == "": break
-                if not line.startswith(header):
-                    continue
-                line = line[len(header):].strip()
-
-                for data in split_header_words([line]):
-                    name, value = data[0]
-                    standard = {}
-                    rest = {}
-                    for k in boolean_attrs:
-                        standard[k] = False
-                    for k, v in data[1:]:
-                        if k is not None:
-                            lc = k.lower()
-                        else:
-                            lc = None
-                        # don't lose case distinction for unknown fields
-                        if (lc in value_attrs) or (lc in boolean_attrs):
-                            k = lc
-                        if k in boolean_attrs:
-                            if v is None: v = True
-                            standard[k] = v
-                        elif k in value_attrs:
-                            standard[k] = v
-                        else:
-                            rest[k] = v
-
-                    h = standard.get
-                    expires = h("expires")
-                    discard = h("discard")
-                    if expires is not None:
-                        expires = iso2time(expires)
-                    if expires is None:
-                        discard = True
-                    domain = h("domain")
-                    domain_specified = domain.startswith(".")
-                    c = Cookie(h("version"), name, value,
-                               h("port"), h("port_spec"),
-                               domain, domain_specified, h("domain_dot"),
-                               h("path"), h("path_spec"),
-                               h("secure"),
-                               expires,
-                               discard,
-                               h("comment"),
-                               h("commenturl"),
-                               rest)
-                    if not ignore_discard and c.discard:
-                        continue
-                    if not ignore_expires and c.is_expired(now):
-                        continue
-                    self.set_cookie(c)
-        except OSError:
-            raise
-        except Exception:
-            _warn_unhandled_exception()
-            raise LoadError("invalid Set-Cookie3 format file %r: %r" %
-                            (filename, line))
-
-
-class MozillaCookieJar(FileCookieJar):
-    """
-
-    WARNING: you may want to backup your browser's cookies file if you use
-    this class to save cookies.  I *think* it works, but there have been
-    bugs in the past!
-
-    This class differs from CookieJar only in the format it uses to save and
-    load cookies to and from a file.  This class uses the Mozilla/Netscape
-    `cookies.txt' format.  lynx uses this file format, too.
-
-    Don't expect cookies saved while the browser is running to be noticed by
-    the browser (in fact, Mozilla on unix will overwrite your saved cookies if
-    you change them on disk while it's running; on Windows, you probably can't
-    save at all while the browser is running).
-
-    Note that the Mozilla/Netscape format will downgrade RFC2965 cookies to
-    Netscape cookies on saving.
-
-    In particular, the cookie version and port number information is lost,
-    together with information about whether or not Path, Port and Discard were
-    specified by the Set-Cookie2 (or Set-Cookie) header, and whether or not the
-    domain as set in the HTTP header started with a dot (yes, I'm aware some
-    domains in Netscape files start with a dot and some don't -- trust me, you
-    really don't want to know any more about this).
-
-    Note that though Mozilla and Netscape use the same format, they use
-    slightly different headers.  The class saves cookies using the Netscape
-    header by default (Mozilla can cope with that).
-
-    """
-    magic_re = re.compile("#( Netscape)? HTTP Cookie File")
-    header = """\
-# Netscape HTTP Cookie File
-# http://curl.haxx.se/rfc/cookie_spec.html
-# This is a generated file!  Do not edit.
-
-"""
-
-    def _really_load(self, f, filename, ignore_discard, ignore_expires):
-        now = time.time()
-
-        magic = f.readline()
-        if not self.magic_re.search(magic):
-            raise LoadError(
-                "%r does not look like a Netscape format cookies file" %
-                filename)
-
-        try:
-            while 1:
-                line = f.readline()
-                if line == "": break
-
-                # last field may be absent, so keep any trailing tab
-                if line.endswith("\n"): line = line[:-1]
-
-                # skip comments and blank lines XXX what is $ for?
-                if (line.strip().startswith(("#", "$")) or
-                    line.strip() == ""):
-                    continue
-
-                domain, domain_specified, path, secure, expires, name, value = \
-                        line.split("\t")
-                secure = (secure == "TRUE")
-                domain_specified = (domain_specified == "TRUE")
-                if name == "":
-                    # cookies.txt regards 'Set-Cookie: foo' as a cookie
-                    # with no name, whereas http.cookiejar regards it as a
-                    # cookie with no value.
-                    name = value
-                    value = None
-
-                initial_dot = domain.startswith(".")
-                assert domain_specified == initial_dot
-
-                discard = False
-                if expires == "":
-                    expires = None
-                    discard = True
-
-                # assume path_specified is false
-                c = Cookie(0, name, value,
-                           None, False,
-                           domain, domain_specified, initial_dot,
-                           path, False,
-                           secure,
-                           expires,
-                           discard,
-                           None,
-                           None,
-                           {})
-                if not ignore_discard and c.discard:
-                    continue
-                if not ignore_expires and c.is_expired(now):
-                    continue
-                self.set_cookie(c)
-
-        except OSError:
-            raise
-        except Exception:
-            _warn_unhandled_exception()
-            raise LoadError("invalid Netscape format cookies file %r: %r" %
-                            (filename, line))
-
-    def save(self, filename=None, ignore_discard=False, ignore_expires=False):
-        if filename is None:
-            if self.filename is not None: filename = self.filename
-            else: raise ValueError(MISSING_FILENAME_TEXT)
-
-        with open(filename, "w") as f:
-            f.write(self.header)
-            now = time.time()
-            for cookie in self:
-                if not ignore_discard and cookie.discard:
-                    continue
-                if not ignore_expires and cookie.is_expired(now):
-                    continue
-                if cookie.secure: secure = "TRUE"
-                else: secure = "FALSE"
-                if cookie.domain.startswith("."): initial_dot = "TRUE"
-                else: initial_dot = "FALSE"
-                if cookie.expires is not None:
-                    expires = str(cookie.expires)
-                else:
-                    expires = ""
-                if cookie.value is None:
-                    # cookies.txt regards 'Set-Cookie: foo' as a cookie
-                    # with no name, whereas http.cookiejar regards it as a
-                    # cookie with no value.
-                    name = ""
-                    value = cookie.name
-                else:
-                    name = cookie.name
-                    value = cookie.value
-                f.write(
-                    "\t".join([cookie.domain, initial_dot, cookie.path,
-                               secure, expires, name, value])+
-                    "\n")