kibi-swipl-linux-x64-gnu 1.0.0

package/lib/swipl/library/ext/http/http/http_open.pl

@@ -0,0 +1,1821 @@
+/*  Part of SWI-Prolog
+
+    Author:        Jan Wielemaker
+    E-mail:        J.Wielemaker@vu.nl
+    WWW:           http://www.swi-prolog.org
+    Copyright (c)  2002-2024, University of Amsterdam
+                              VU University Amsterdam
+                              CWI, Amsterdam
+                              SWI-Prolog Solutions b.v.
+    All rights reserved.
+
+    Redistribution and use in source and binary forms, with or without
+    modification, are permitted provided that the following conditions
+    are met:
+
+    1. Redistributions of source code must retain the above copyright
+       notice, this list of conditions and the following disclaimer.
+
+    2. Redistributions in binary form must reproduce the above copyright
+       notice, this list of conditions and the following disclaimer in
+       the documentation and/or other materials provided with the
+       distribution.
+
+    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+    FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+    COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+    INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+    BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+    LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+    CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+    LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+    ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+    POSSIBILITY OF SUCH DAMAGE.
+*/
+
+:- module(http_open,
+          [ http_open/3,                % +URL, -Stream, +Options
+            http_set_authorization/2,   % +URL, +Authorization
+            http_close_keep_alive/1     % +Address
+          ]).
+:- autoload(library(aggregate),[aggregate_all/3]).
+:- autoload(library(apply),[foldl/4,include/3]).
+:- autoload(library(base64),[base64/3]).
+:- use_module(library(debug),[debug/3,debugging/1]).
+:- autoload(library(error),
+	    [ domain_error/2, must_be/2, existence_error/2, instantiation_error/1
+	    ]).
+:- autoload(library(lists),[last/2,member/2]).
+:- autoload(library(option),
+	    [ meta_options/3, option/2, select_option/4, merge_options/3,
+	      option/3, select_option/3
+	    ]).
+:- autoload(library(readutil),[read_line_to_codes/2]).
+:- autoload(library(uri),
+	    [ uri_resolve/3, uri_components/2, uri_data/3,
+              uri_authority_components/2, uri_authority_data/3,
+	      uri_encoded/3, uri_query_components/2, uri_is_global/1
+	    ]).
+:- autoload(library(http/http_header),
+            [ http_parse_header/2, http_post_data/3 ]).
+:- autoload(library(http/http_stream),[stream_range_open/3]).
+:- if(exists_source(library(ssl))).
+:- autoload(library(ssl), [ssl_upgrade_legacy_options/2]).
+:- endif.
+:- use_module(library(socket)).
+:- use_module(library(settings)).
+
+:- setting(http:max_keep_alive_idle, number, 2,
+           "Time to keep idle keep alive connections around").
+:- setting(http:max_keep_alive_connections, integer, 10,
+           "Maximum number of client keep alive connections").
+:- setting(http:max_keep_alive_host_connections, integer, 2,
+           "Maximum number of client keep alive to a single host").
+
+/** <module> HTTP client library
+
+This library defines http_open/3, which opens an URL as a Prolog stream.
+The functionality of the  library  can   be  extended  by  loading two
+additional modules that act as plugins:
+
+    * library(http/http_ssl_plugin)
+    Loading this library causes http_open/3 to handle HTTPS connections.
+    Relevant options for SSL certificate handling are handed to
+    ssl_context/3. This plugin is loaded automatically if the scheme
+    `https` is requested using a default SSL context. See the plugin for
+    additional information regarding security.
+
+    * library(zlib)
+    Loading this library supports the `gzip` transfer encoding.  This
+    plugin is lazily loaded if a connection is opened that claims this
+    transfer encoding.
+
+    * library(http/http_cookie)
+    Loading this library adds tracking cookies to http_open/3. Returned
+    cookies are collected in the Prolog database and supplied for
+    subsequent requests.
+
+    * library(http/http_stream)
+    This library adds support for _chunked_ encoding. It is lazily
+    loaded if the server sends a ``Transfer-encoding: chunked`` header.
+
+
+Here is a simple example to fetch a web-page:
+
+```
+?- http_open('http://www.google.com/search?q=prolog', In, []),
+   copy_stream_data(In, user_output),
+   close(In).
+<!doctype html><head><title>prolog - Google Search</title><script>
+...
+```
+
+The example below fetches the modification time of a web-page. Note that
+=|Modified|= is =|''|= (the empty atom) if the  web-server does not provide a
+time-stamp for the resource. See also parse_time/2.
+
+```
+modified(URL, Stamp) :-
+       http_open(URL, In,
+                 [ method(head),
+                   header(last_modified, Modified)
+                 ]),
+       close(In),
+       Modified \== '',
+       parse_time(Modified, Stamp).
+```
+
+Then next example uses Google search. It exploits library(uri) to manage
+URIs, library(sgml) to load  an  HTML   document  and  library(xpath) to
+navigate the parsed HTML. Note that  you   may  need to adjust the XPath
+queries if the data returned by Google changes (this example indeed
+no longer works and currently fails at the first xpath/3 call)
+
+```
+:- use_module(library(http/http_open)).
+:- use_module(library(xpath)).
+:- use_module(library(sgml)).
+:- use_module(library(uri)).
+
+google(For, Title, HREF) :-
+        uri_encoded(query_value, For, Encoded),
+        atom_concat('http://www.google.com/search?q=', Encoded, URL),
+        http_open(URL, In, []),
+        call_cleanup(
+            load_html(In, DOM, []),
+            close(In)),
+        xpath(DOM, //h3(@class=r), Result),
+        xpath(Result, //a(@href=HREF0, text), Title),
+        uri_components(HREF0, Components),
+        uri_data(search, Components, Query),
+        uri_query_components(Query, Parts),
+        memberchk(q=HREF, Parts).
+```
+
+An example query is below:
+
+```
+?- google(prolog, Title, HREF).
+Title = 'SWI-Prolog',
+HREF = 'http://www.swi-prolog.org/' ;
+Title = 'Prolog - Wikipedia',
+HREF = 'https://nl.wikipedia.org/wiki/Prolog' ;
+Title = 'Prolog - Wikipedia, the free encyclopedia',
+HREF = 'https://en.wikipedia.org/wiki/Prolog' ;
+Title = 'Pro-Log is logistiek dienstverlener m.b.t. vervoer over water.',
+HREF = 'http://www.pro-log.nl/' ;
+Title = 'Learn Prolog Now!',
+HREF = 'http://www.learnprolognow.org/' ;
+Title = 'Free Online Version - Learn Prolog
+...
+```
+
+@see load_html/3 and xpath/3 can be used to parse and navigate HTML
+     documents.
+@see http_get/3 and http_post/4 provide an alternative interface that
+     convert the reply depending on the =|Content-Type|= header.
+*/
+
+:- multifile
+    http:encoding_filter/3,           % +Encoding, +In0, -In
+    http:current_transfer_encoding/1, % ?Encoding
+    http:disable_encoding_filter/1,   % +ContentType
+    http:http_protocol_hook/5,        % +Protocol, +Parts, +StreamPair,
+                                      % -NewStreamPair, +Options
+    http:open_options/2,              % +Parts, -Options
+    http:write_cookies/3,             % +Out, +Parts, +Options
+    http:update_cookies/3,            % +CookieLine, +Parts, +Options
+    http:authenticate_client/2,       % +URL, +Action
+    http:http_connection_over_proxy/6.
+
+:- meta_predicate
+    http_open(+,-,:).
+
+:- predicate_options(http_open/3, 3,
+                     [ authorization(compound),
+                       final_url(-atom),
+                       header(+atom, -atom),
+                       headers(-list),
+                       raw_headers(-list(string)),
+                       connection(+atom),
+                       method(oneof([delete,get,put,purge,head,
+                                     post,patch,options])),
+                       size(-integer),
+                       status_code(-integer),
+                       output(-stream),
+                       timeout(number),
+                       unix_socket(+atom),
+                       proxy(atom, integer),
+                       proxy_authorization(compound),
+                       bypass_proxy(boolean),
+                       request_header(any),
+                       user_agent(atom),
+                       version(-compound),
+        % The option below applies if library(http/http_header) is loaded
+                       post(any),
+        % The options below apply if library(http/http_ssl_plugin)) is loaded
+                       pem_password_hook(callable),
+                       cacert_file(atom),
+                       cert_verify_hook(callable)
+                     ]).
+
+%!  user_agent(-Agent) is det.
+%
+%   Default value for =|User-Agent|=,  can   be  overruled using the
+%   option user_agent(Agent) of http_open/3.
+
+user_agent('SWI-Prolog').
+
+%!  http_open(+URL, -Stream, +Options) is det.
+%
+%   Open the data at the HTTP  server   as  a  Prolog stream. URL is
+%   either an atom  specifying  a  URL   or  a  list  representing a
+%   broken-down  URL  as  specified  below.   After  this  predicate
+%   succeeds the data can be read from Stream. After completion this
+%   stream must be  closed  using   the  built-in  Prolog  predicate
+%   close/1. Options provides additional options:
+%
+%     * authenticate(+Boolean)
+%     If `false` (default `true`), do _not_ try to automatically
+%     authenticate the client if a 401 (Unauthorized) status code
+%     is received.
+%
+%     * authorization(+Term)
+%     Send authorization. See also http_set_authorization/2. Supported
+%     schemes:
+%
+%       - basic(+User, +Password)
+%       HTTP Basic authentication.
+%       - bearer(+Token)
+%       HTTP Bearer authentication.
+%       - digest(+User, +Password)
+%       HTTP Digest authentication.  This option is only provided
+%       if the plugin library(http/http_digest) is also loaded.
+%
+%     * unix_socket(+Path)
+%     Connect to the given Unix domain socket.  In this scenario
+%     the host name and port or ignored.  If the server replies
+%     with a _redirect_ message and the host differs from the
+%     original host as normal TCP connection is used to handle
+%     the redirect.  This option is inspired by curl(1)'s option
+%     `--unix-socket`.
+%
+%     * connection(+Connection)
+%     Specify the =Connection= header.  Default is =close=.  The
+%     alternative is =|Keep-alive|=.  This maintains a pool of
+%     available connections as determined by keep_connection/1.
+%     The library(http/websockets) uses =|Keep-alive, Upgrade|=.
+%     Keep-alive connections can be closed explicitly using
+%     http_close_keep_alive/1. Keep-alive connections may
+%     significantly improve repetitive requests on the same server,
+%     especially if the IP route is long, HTTPS is used or the
+%     connection uses a proxy.
+%
+%     * final_url(-FinalURL)
+%     Unify FinalURL with the final   destination. This differs from
+%     the  original  URL  if  the  returned  head  of  the  original
+%     indicates an HTTP redirect (codes 301,  302 or 303). Without a
+%     redirect, FinalURL is the same as URL if  URL is an atom, or a
+%     URL constructed from the parts.
+%
+%     * header(Name, -AtomValue)
+%     If provided, AtomValue is  unified  with   the  value  of  the
+%     indicated  field  in  the  reply    header.  Name  is  matched
+%     case-insensitive and the underscore  (_)   matches  the hyphen
+%     (-). Multiple of these options  may   be  provided  to extract
+%     multiple  header  fields.  If  the  header  is  not  available
+%     AtomValue is unified to the empty atom ('').
+%
+%     * headers(-List)
+%     If provided,  List is unified  with a list of  Name(Value) pairs
+%     corresponding to  fields in  the reply  header.  Name  and Value
+%     follow  the  same  conventions used  by  the  header(Name,Value)
+%     option.  A  pseudo header status_code(Code) is  added to provide
+%     the  HTTP status  as  an integer.   See also  raw_headers(-List)
+%     which  provides  the  entire   HTTP  reply  header  in  unparsed
+%     representation.
+%
+%     * method(+Method)
+%     One of =get= (default), =head=, =delete=, =post=,   =put=   or
+%     =patch=.
+%     The  =head= message can be
+%     used in combination with  the   header(Name,  Value) option to
+%     access information on the resource   without actually fetching
+%     the resource itself.  The  returned   stream  must  be  closed
+%     immediately.
+%
+%     If post(Data) is provided, the default is =post=.
+%
+%     * size(-Size)
+%     Size is unified with the   integer value of =|Content-Length|=
+%     in the reply header.
+%
+%     * version(-Version)
+%     Version is a _pair_ `Major-Minor`, where `Major` and `Minor`
+%     are integers representing the HTTP version in the reply header.
+%
+%     * range(+Range)
+%     Ask for partial content. Range   is  a term _|Unit(From,To)|_,
+%     where `From` is an integer and `To`   is  either an integer or
+%     the atom `end`. HTTP 1.1 only   supports Unit = `bytes`. E.g.,
+%     to   ask   for    bytes    1000-1999,     use    the    option
+%     range(bytes(1000,1999))
+%
+%     * raw_encoding(+Encoding)
+%     Do not install a decoding filter for Encoding.  For example,
+%     using raw_encoding('applocation/gzip') the system will not
+%     decompress the stream if it is compressed using `gzip`.
+%
+%     * raw_headers(-Lines)
+%     Unify Lines with a list of strings that represents the complete
+%     reply header returned by the server.  See also headers(-List).
+%
+%     * redirect(+Boolean)
+%     If `false` (default `true`), do _not_ automatically redirect
+%     if a 3XX code is received.  Must be combined with
+%     status_code(Code) and one of the header options to read the
+%     redirect reply. In particular, without status_code(Code) a
+%     redirect is mapped to an exception.
+%
+%     * status_code(-Code)
+%     If this option is  present  and   Code  unifies  with the HTTP
+%     status code, do *not* translate errors (4xx, 5xx) into an
+%     exception. Instead, http_open/3 behaves as if 2xx (success) is
+%     returned, providing the application to read the error document
+%     from the returned stream.
+%
+%     * output(-Out)
+%     Unify the output stream with Out and do not close it. This can
+%     be used to upgrade a connection.
+%
+%     * timeout(+Timeout)
+%     If provided, set a timeout on   the stream using set_stream/2.
+%     With this option if no new data arrives within Timeout seconds
+%     the stream raises an exception.  Default   is  to wait forever
+%     (=infinite=).
+%
+%     * post(+Data)
+%     Issue a =POST= request on the HTTP server.  Data is
+%     handed to http_post_data/3.
+%
+%     * proxy(+Host:Port)
+%     Use an HTTP proxy to connect to the outside world.  See also
+%     socket:proxy_for_url/3.  This option overrules the proxy
+%     specification defined by socket:proxy_for_url/3.
+%
+%     * proxy(+Host, +Port)
+%     Synonym for proxy(+Host:Port).  Deprecated.
+%
+%     * proxy_authorization(+Authorization)
+%     Send authorization to the proxy.  Otherwise   the  same as the
+%     =authorization= option.
+%
+%     * bypass_proxy(+Boolean)
+%     If =true=, bypass proxy hooks.  Default is =false=.
+%
+%     * request_header(Name = Value)
+%     Additional  name-value  parts  are  added   in  the  order  of
+%     appearance to the HTTP request   header.  No interpretation is
+%     done.
+%
+%     * max_redirect(+Max)
+%     Sets the maximum length of a redirection chain.  This is needed
+%     for some IRIs that redirect indefinitely to other IRIs without
+%     looping (e.g., redirecting to IRIs with a random element in them).
+%     Max must be either a non-negative integer or the atom `infinite`.
+%     The default value is `10`.
+%
+%     * user_agent(+Agent)
+%     Defines the value of the  =|User-Agent|=   field  of  the HTTP
+%     header. Default is =SWI-Prolog=.
+%
+%   The hook http:open_options/2 can  be   used  to  provide default
+%   options   based   on   the   broken-down     URL.   The   option
+%   status_code(-Code)  is  particularly  useful   to  query  *REST*
+%   interfaces that commonly return status   codes  other than `200`
+%   that need to be be processed by the client code.
+%
+%   @param URL is either an atom or string (url) or a list of _parts_.
+%
+%               When provided, this list may contain the fields
+%               =scheme=, =user=, =password=, =host=, =port=, =path=
+%               and either =query_string= (whose argument is an atom)
+%               or =search= (whose argument is a list of
+%               =|Name(Value)|= or =|Name=Value|= compound terms).
+%               Only =host= is mandatory.  The example below opens the
+%               URL =|http://www.example.com/my/path?q=Hello%20World&lang=en|=.
+%               Note that values must *not* be quoted because the
+%               library inserts the required quotes.
+%
+%               ```
+%               http_open([ host('www.example.com'),
+%                           path('/my/path'),
+%                           search([ q='Hello world',
+%                                    lang=en
+%                                  ])
+%                         ])
+%               ```
+%
+%   @throws error(existence_error(url, Id),Context) is raised if the
+%   HTTP result code is not in the range 200..299. Context has the
+%   shape context(Message, status(Code, TextCode)), where `Code` is the
+%   numeric HTTP code and `TextCode` is the textual description thereof
+%   provided by the server. `Message` may provide additional details or
+%   may be unbound.
+%
+%   @see ssl_context/3 for SSL related options if
+%   library(http/http_ssl_plugin) is loaded.
+
+:- multifile
+    socket:proxy_for_url/3.           % +URL, +Host, -ProxyList
+
+http_open(URL, Stream, QOptions) :-
+    meta_options(is_meta, QOptions, Options0),
+    (   atomic(URL)
+    ->  parse_url_ex(URL, Parts)
+    ;   Parts = URL
+    ),
+    autoload_https(Parts),
+    upgrade_ssl_options(Parts, Options0, Options),
+    add_authorization(Parts, Options, Options1),
+    findall(HostOptions, hooked_options(Parts, HostOptions), AllHostOptions),
+    foldl(merge_options_rev, AllHostOptions, Options1, Options2),
+    (   option(bypass_proxy(true), Options)
+    ->  try_http_proxy(direct, Parts, Stream, Options2)
+    ;   term_variables(Options2, Vars2),
+        findall(Result-Vars2,
+                try_a_proxy(Parts, Result, Options2),
+                ResultList),
+        last(ResultList, Status-Vars2)
+    ->  (   Status = true(_Proxy, Stream)
+        ->  true
+        ;   throw(error(proxy_error(tried(ResultList)), _))
+        )
+    ;   try_http_proxy(direct, Parts, Stream, Options2)
+    ).
+
+try_a_proxy(Parts, Result, Options) :-
+    parts_uri(Parts, AtomicURL),
+    option(host(Host), Parts),
+    (   option(unix_socket(Path), Options)
+    ->  Proxy = unix_socket(Path)
+    ;   (   option(proxy(ProxyHost:ProxyPort), Options)
+        ;   is_list(Options),
+            memberchk(proxy(ProxyHost,ProxyPort), Options)
+        )
+    ->  Proxy = proxy(ProxyHost, ProxyPort)
+    ;   socket:proxy_for_url(AtomicURL, Host, Proxy)
+    ),
+    debug(http(proxy),
+          'http_open: Connecting via ~w to ~w', [Proxy, AtomicURL]),
+    (   catch(try_http_proxy(Proxy, Parts, Stream, Options), E, true)
+    ->  (   var(E)
+        ->  !, Result = true(Proxy, Stream)
+        ;   Result = error(Proxy, E)
+        )
+    ;   Result = false(Proxy)
+    ),
+    debug(http(proxy), 'http_open: ~w: ~p', [Proxy, Result]).
+
+try_http_proxy(Method, Parts, Stream, Options0) :-
+    option(host(Host), Parts),
+    proxy_request_uri(Method, Parts, RequestURI),
+    select_option(visited(Visited0), Options0, OptionsV, []),
+    Options = [visited([Parts|Visited0])|OptionsV],
+    parts_scheme(Parts, Scheme),
+    default_port(Scheme, DefPort),
+    url_part(port(Port), Parts, DefPort),
+    host_and_port(Host, DefPort, Port, HostPort),
+    (   option(connection(Connection), Options0),
+        keep_alive(Connection),
+        get_from_pool(Host:Port, StreamPair),
+        debug(http(keep_alive), 'Trying Keep-alive to ~p using ~p',
+              [ Host:Port, StreamPair ]),
+        catch(send_rec_header(StreamPair, Stream, HostPort,
+                              RequestURI, Parts, Options),
+              Error,
+              keep_alive_error(Error, StreamPair))
+    ->  true
+    ;   http:http_connection_over_proxy(Method, Parts, Host:Port,
+                                        SocketStreamPair, Options, Options1),
+        (   catch(http:http_protocol_hook(Scheme, Parts,
+                                          SocketStreamPair,
+                                          StreamPair, Options),
+                  Error,
+                  ( close(SocketStreamPair, [force(true)]),
+                    throw(Error)))
+        ->  true
+        ;   StreamPair = SocketStreamPair
+        ),
+        send_rec_header(StreamPair, Stream, HostPort,
+                        RequestURI, Parts, Options1)
+    ),
+    return_final_url(Options).
+
+proxy_request_uri(direct, Parts, RequestURI) :-
+    !,
+    parts_request_uri(Parts, RequestURI).
+proxy_request_uri(unix_socket(_), Parts, RequestURI) :-
+    !,
+    parts_request_uri(Parts, RequestURI).
+proxy_request_uri(_, Parts, RequestURI) :-
+    parts_uri(Parts, RequestURI).
+
+http:http_connection_over_proxy(unix_socket(Path), _, _,
+                                StreamPair, Options, Options) :-
+    !,
+    unix_domain_socket(Socket),
+    tcp_connect(Socket, Path),
+    tcp_open_socket(Socket, In, Out),
+    stream_pair(StreamPair, In, Out).
+http:http_connection_over_proxy(direct, _, Host:Port,
+                                StreamPair, Options, Options) :-
+    !,
+    open_socket(Host:Port, StreamPair, Options).
+http:http_connection_over_proxy(proxy(ProxyHost, ProxyPort), Parts, _,
+                                StreamPair, Options, Options) :-
+    \+ ( memberchk(scheme(Scheme), Parts),
+         secure_scheme(Scheme)
+       ),
+    !,
+    % We do not want any /more/ proxy after this
+    open_socket(ProxyHost:ProxyPort, StreamPair,
+                [bypass_proxy(true)|Options]).
+http:http_connection_over_proxy(socks(SocksHost, SocksPort), _Parts, Host:Port,
+                                StreamPair, Options, Options) :-
+    !,
+    tcp_connect(SocksHost:SocksPort, StreamPair, [bypass_proxy(true)]),
+    catch(negotiate_socks_connection(Host:Port, StreamPair),
+          Error,
+          ( close(StreamPair, [force(true)]),
+            throw(Error)
+          )).
+
+%!  hooked_options(+Parts, -Options) is nondet.
+%
+%   Calls  http:open_options/2  and  if  necessary    upgrades  old  SSL
+%   cacerts_file(File) option to a cacerts(List) option to ensure proper
+%   merging of options.
+
+hooked_options(Parts, Options) :-
+    http:open_options(Parts, Options0),
+    upgrade_ssl_options(Parts, Options0, Options).
+
+:- if(current_predicate(ssl_upgrade_legacy_options/2)).
+upgrade_ssl_options(Parts, Options0, Options) :-
+    requires_ssl(Parts),
+    !,
+    ssl_upgrade_legacy_options(Options0, Options).
+:- endif.
+upgrade_ssl_options(_, Options, Options).
+
+merge_options_rev(Old, New, Merged) :-
+    merge_options(New, Old, Merged).
+
+is_meta(pem_password_hook).             % SSL plugin callbacks
+is_meta(cert_verify_hook).
+
+
+http:http_protocol_hook(http, _, StreamPair, StreamPair, _).
+
+default_port(https, 443) :- !.
+default_port(wss,   443) :- !.
+default_port(_,     80).
+
+host_and_port(Host, DefPort, DefPort, Host) :- !.
+host_and_port(Host, _,       Port,    Host:Port).
+
+%!  autoload_https(+Parts) is det.
+%
+%   If the requested scheme is https or wss, load the HTTPS plugin.
+
+autoload_https(Parts) :-
+    requires_ssl(Parts),
+    memberchk(scheme(S), Parts),
+    \+ clause(http:http_protocol_hook(S, _, StreamPair, StreamPair, _),_),
+    exists_source(library(http/http_ssl_plugin)),
+    !,
+    use_module(library(http/http_ssl_plugin)).
+autoload_https(_).
+
+requires_ssl(Parts) :-
+    memberchk(scheme(S), Parts),
+    secure_scheme(S).
+
+secure_scheme(https).
+secure_scheme(wss).
+
+%!  send_rec_header(+StreamPair, -Stream,
+%!                  +Host, +RequestURI, +Parts, +Options) is det.
+%
+%   Send header to Out and process reply.  If there is an error or
+%   failure, close In and Out and return the error or failure.
+
+send_rec_header(StreamPair, Stream, Host, RequestURI, Parts, Options) :-
+    (   catch(guarded_send_rec_header(StreamPair, Stream,
+                                      Host, RequestURI, Parts, Options),
+              E, true)
+    ->  (   var(E)
+        ->  (   option(output(StreamPair), Options)
+            ->  true
+            ;   true
+            )
+        ;   close(StreamPair, [force(true)]),
+            throw(E)
+        )
+    ;   close(StreamPair, [force(true)]),
+        fail
+    ).
+
+guarded_send_rec_header(StreamPair, Stream, Host, RequestURI, Parts, Options) :-
+    user_agent(Agent, Options),
+    method(Options, MNAME),
+    http_version(Version),
+    option(connection(Connection), Options, close),
+    debug(http(send_request), "> ~w ~w HTTP/~w", [MNAME, RequestURI, Version]),
+    debug(http(send_request), "> Host: ~w", [Host]),
+    debug(http(send_request), "> User-Agent: ~w", [Agent]),
+    debug(http(send_request), "> Connection: ~w", [Connection]),
+    format(StreamPair,
+           '~w ~w HTTP/~w\r\n\c
+               Host: ~w\r\n\c
+               User-Agent: ~w\r\n\c
+               Connection: ~w\r\n',
+           [MNAME, RequestURI, Version, Host, Agent, Connection]),
+    parts_uri(Parts, URI),
+    x_headers(Options, URI, StreamPair),
+    write_cookies(StreamPair, Parts, Options),
+    (   option(post(PostData), Options)
+    ->  http_post_data(PostData, StreamPair, [])
+    ;   format(StreamPair, '\r\n', [])
+    ),
+    flush_output(StreamPair),
+                                    % read the reply header
+    read_header(StreamPair, Parts, ReplyVersion, Code, Comment, Lines),
+    update_cookies(Lines, Parts, Options),
+    reply_header(Lines, Options),
+    do_open(ReplyVersion, Code, Comment, Lines, Options, Parts, Host,
+            StreamPair, Stream).
+
+
+%!  http_version(-Version:atom) is det.
+%
+%   HTTP version we publish. We  can  only   use  1.1  if we support
+%   chunked encoding.
+
+http_version('1.1') :-
+    http:current_transfer_encoding(chunked),
+    !.
+http_version('1.1') :-
+    autoload_encoding(chunked),
+    !.
+http_version('1.0').
+
+method(Options, MNAME) :-
+    option(post(_), Options),
+    !,
+    option(method(M), Options, post),
+    (   map_method(M, MNAME0)
+    ->  MNAME = MNAME0
+    ;   domain_error(method, M)
+    ).
+method(Options, MNAME) :-
+    option(method(M), Options, get),
+    (   map_method(M, MNAME0)
+    ->  MNAME = MNAME0
+    ;   map_method(_, M)
+    ->  MNAME = M
+    ;   domain_error(method, M)
+    ).
+
+%!  map_method(+MethodID, -Method)
+%
+%   Support additional ``METHOD`` keywords.  Default   are  the official
+%   HTTP methods as defined by the various RFCs.
+
+:- multifile
+    map_method/2.
+
+map_method(delete,  'DELETE').
+map_method(get,     'GET').
+map_method(head,    'HEAD').
+map_method(post,    'POST').
+map_method(put,     'PUT').
+map_method(patch,   'PATCH').
+map_method(options, 'OPTIONS').
+
+%!  x_headers(+Options, +URI, +Out) is det.
+%
+%   Emit extra headers from   request_header(Name=Value)  options in
+%   Options.
+%
+%   @tbd Use user/password fields
+
+x_headers(Options, URI, Out) :-
+    x_headers_(Options, [url(URI)|Options], Out).
+
+x_headers_([], _, _).
+x_headers_([H|T], Options, Out) :-
+    x_header(H, Options, Out),
+    x_headers_(T, Options, Out).
+
+x_header(request_header(Name=Value), _, Out) :-
+    !,
+    debug(http(send_request), "> ~w: ~w", [Name, Value]),
+    format(Out, '~w: ~w\r\n', [Name, Value]).
+x_header(proxy_authorization(ProxyAuthorization), Options, Out) :-
+    !,
+    auth_header(ProxyAuthorization, Options, 'Proxy-Authorization', Out).
+x_header(authorization(Authorization), Options, Out) :-
+    !,
+    auth_header(Authorization, Options, 'Authorization', Out).
+x_header(range(Spec), _, Out) :-
+    !,
+    Spec =.. [Unit, From, To],
+    (   To == end
+    ->  ToT = ''
+    ;   must_be(integer, To),
+        ToT = To
+    ),
+    debug(http(send_request), "> Range: ~w=~d-~w", [Unit, From, ToT]),
+    format(Out, 'Range: ~w=~d-~w\r\n', [Unit, From, ToT]).
+x_header(_, _, _).
+
+%!  auth_header(+AuthOption, +Options, +HeaderName, +Out)
+
+auth_header(basic(User, Password), _, Header, Out) :-
+    !,
+    format(codes(Codes), '~w:~w', [User, Password]),
+    phrase(base64(Codes), Base64Codes),
+    debug(http(send_request), "> ~w: Basic ~s", [Header, Base64Codes]),
+    format(Out, '~w: Basic ~s\r\n', [Header, Base64Codes]).
+auth_header(bearer(Token), _, Header, Out) :-
+    !,
+    debug(http(send_request), "> ~w: Bearer ~w", [Header,Token]),
+    format(Out, '~w: Bearer ~w\r\n', [Header, Token]).
+auth_header(Auth, Options, _, Out) :-
+    option(url(URL), Options),
+    add_method(Options, Options1),
+    http:authenticate_client(URL, send_auth_header(Auth, Out, Options1)),
+    !.
+auth_header(Auth, _, _, _) :-
+    domain_error(authorization, Auth).
+
+user_agent(Agent, Options) :-
+    (   option(user_agent(Agent), Options)
+    ->  true
+    ;   user_agent(Agent)
+    ).
+
+add_method(Options0, Options) :-
+    option(method(_), Options0),
+    !,
+    Options = Options0.
+add_method(Options0, Options) :-
+    option(post(_), Options0),
+    !,
+    Options = [method(post)|Options0].
+add_method(Options0, [method(get)|Options0]).
+
+%!  do_open(+HTTPVersion, +HTTPStatusCode, +HTTPStatusComment, +Header,
+%!          +Options, +Parts, +Host, +In, -FinalIn) is det.
+%
+%   Handle the HTTP status once available. If   200-299, we are ok. If a
+%   redirect, redo the open,  returning  a   new  stream.  Else issue an
+%   error.
+%
+%   @error  existence_error(url, URL)
+
+                                        % Redirections
+do_open(_, Code, _, Lines, Options0, Parts, _, In, Stream) :-
+    redirect_code(Code),
+    option(redirect(true), Options0, true),
+    location(Lines, RequestURI),
+    !,
+    debug(http(redirect), 'http_open: redirecting to ~w', [RequestURI]),
+    close(In),
+    parts_uri(Parts, Base),
+    uri_resolve(RequestURI, Base, Redirected),
+    parse_url_ex(Redirected, RedirectedParts),
+    (   redirect_limit_exceeded(Options0, Max)
+    ->  format(atom(Comment), 'max_redirect (~w) limit exceeded', [Max]),
+        throw(error(permission_error(redirect, http, Redirected),
+                    context(_, Comment)))
+    ;   redirect_loop(RedirectedParts, Options0)
+    ->  throw(error(permission_error(redirect, http, Redirected),
+                    context(_, 'Redirection loop')))
+    ;   true
+    ),
+    redirect_options(Parts, RedirectedParts, Options0, Options),
+    http_open(RedirectedParts, Stream, Options).
+                                        % Need authentication
+do_open(_Version, Code, _Comment, Lines, Options0, Parts, _Host, In0, Stream) :-
+    authenticate_code(Code),
+    option(authenticate(true), Options0, true),
+    parts_uri(Parts, URI),
+    parse_headers(Lines, Headers),
+    http:authenticate_client(
+             URI,
+             auth_reponse(Headers, Options0, Options)),
+    !,
+    close(In0),
+    http_open(Parts, Stream, Options).
+                                        % Accepted codes
+do_open(Version, Code, _, Lines, Options, Parts, Host, In0, In) :-
+    (   option(status_code(Code), Options),
+        Lines \== []
+    ->  true
+    ;   successful_code(Code)
+    ),
+    !,
+    parts_uri(Parts, URI),
+    parse_headers(Lines, Headers),
+    return_version(Options, Version),
+    return_size(Options, Headers),
+    return_fields(Options, Headers),
+    return_headers(Options, [status_code(Code)|Headers]),
+    consider_keep_alive(Lines, Parts, Host, In0, In1, Options),
+    transfer_encoding_filter(Lines, In1, In, Options),
+                                    % properly re-initialise the stream
+    set_stream(In, file_name(URI)),
+    set_stream(In, record_position(true)).
+do_open(_, _, _, [], Options, _, _, _, _) :-
+    option(connection(Connection), Options),
+    keep_alive(Connection),
+    !,
+    throw(error(keep_alive(closed),_)).
+                                        % report anything else as error
+do_open(_Version, Code, Comment, _,  _, Parts, _, _, _) :-
+    parts_uri(Parts, URI),
+    (   map_error_code(Code, Error)
+    ->  Formal =.. [Error, url, URI]
+    ;   Formal = existence_error(url, URI)
+    ),
+    throw(error(Formal, context(_, status(Code, Comment)))).
+
+
+successful_code(Code) :-
+    between(200, 299, Code).
+
+%!  redirect_limit_exceeded(+Options:list(compound), -Max:nonneg) is semidet.
+%
+%   True if we have exceeded the maximum redirection length (default 10).
+
+redirect_limit_exceeded(Options, Max) :-
+    option(visited(Visited), Options, []),
+    length(Visited, N),
+    option(max_redirect(Max), Options, 10),
+    (Max == infinite -> fail ; N > Max).
+
+
+%!  redirect_loop(+Parts, +Options) is semidet.
+%
+%   True if we are in  a  redirection   loop.  Note  that some sites
+%   redirect once to the same place using  cookies or similar, so we
+%   allow for two tries. In fact,   we  should probably test whether
+%   authorization or cookie headers have changed.
+
+redirect_loop(Parts, Options) :-
+    option(visited(Visited), Options, []),
+    include(==(Parts), Visited, Same),
+    length(Same, Count),
+    Count > 2.
+
+
+%!  redirect_options(+Parts, +RedirectedParts, +Options0, -Options) is det.
+%
+%   A redirect from a POST should do  a   GET  on the returned URI. This
+%   means we must remove the method(post)   and  post(Data) options from
+%   the original option-list.
+%
+%   If we are connecting over a Unix   domain socket we drop this option
+%   if the redirect host does not match the initial host.
+
+redirect_options(Parts, RedirectedParts, Options0, Options) :-
+    select_option(unix_socket(_), Options0, Options1),
+    memberchk(host(Host), Parts),
+    memberchk(host(RHost), RedirectedParts),
+    debug(http(redirect), 'http_open: redirecting AF_UNIX ~w to ~w',
+          [Host, RHost]),
+    Host \== RHost,
+    !,
+    redirect_options(Options1, Options).
+redirect_options(_, _, Options0, Options) :-
+    redirect_options(Options0, Options).
+
+redirect_options(Options0, Options) :-
+    (   select_option(post(_), Options0, Options1)
+    ->  true
+    ;   Options1 = Options0
+    ),
+    (   select_option(method(Method), Options1, Options),
+        \+ redirect_method(Method)
+    ->  true
+    ;   Options = Options1
+    ).
+
+redirect_method(delete).
+redirect_method(get).
+redirect_method(head).
+
+
+%!  map_error_code(+HTTPCode, -PrologError) is semidet.
+%
+%   Map HTTP error codes to Prolog errors.
+%
+%   @tbd    Many more maps. Unfortunately many have no sensible Prolog
+%           counterpart.
+
+map_error_code(401, permission_error).
+map_error_code(403, permission_error).
+map_error_code(404, existence_error).
+map_error_code(405, permission_error).
+map_error_code(407, permission_error).
+map_error_code(410, existence_error).
+
+redirect_code(301).                     % Moved Permanently
+redirect_code(302).                     % Found (previously "Moved Temporary")
+redirect_code(303).                     % See Other
+redirect_code(307).                     % Temporary Redirect
+
+authenticate_code(401).
+
+%!  open_socket(+Address, -StreamPair, +Options) is det.
+%
+%   Create and connect a client socket to Address.  Options
+%
+%       * timeout(+Timeout)
+%       Sets timeout on the stream, *after* connecting the
+%       socket.
+%
+%   @tbd    Make timeout also work on tcp_connect/4.
+%   @tbd    This is the same as do_connect/4 in http_client.pl
+
+open_socket(Address, StreamPair, Options) :-
+    debug(http(open), 'http_open: Connecting to ~p ...', [Address]),
+    tcp_connect(Address, StreamPair, Options),
+    stream_pair(StreamPair, In, Out),
+    debug(http(open), '\tok ~p ---> ~p', [In, Out]),
+    set_stream(In, record_position(false)),
+    (   option(timeout(Timeout), Options)
+    ->  set_stream(In, timeout(Timeout))
+    ;   true
+    ).
+
+
+return_version(Options, Major-Minor) :-
+    option(version(Major-Minor), Options, _).
+
+return_size(Options, Headers) :-
+    (   memberchk(content_length(Size), Headers)
+    ->  option(size(Size), Options, _)
+    ;   true
+    ).
+
+return_fields([], _).
+return_fields([header(Name, Value)|T], Headers) :-
+    !,
+    (   Term =.. [Name,Value],
+        memberchk(Term, Headers)
+    ->  true
+    ;   Value = ''
+    ),
+    return_fields(T, Headers).
+return_fields([_|T], Lines) :-
+    return_fields(T, Lines).
+
+return_headers(Options, Headers) :-
+    option(headers(Headers), Options, _).
+
+%!  parse_headers(+Lines, -Headers:list(compound)) is det.
+%
+%   Parse the header lines for   the  headers(-List) option. Invalid
+%   header   lines   are   skipped,   printing   a   warning   using
+%   print_message/2.
+
+parse_headers([], []) :- !.
+parse_headers([Line|Lines], Headers) :-
+    catch(http_parse_header(Line, [Header]), Error, true),
+    (   var(Error)
+    ->  Headers = [Header|More]
+    ;   print_message(warning, Error),
+        Headers = More
+    ),
+    parse_headers(Lines, More).
+
+
+%!  return_final_url(+Options) is semidet.
+%
+%   If Options contains final_url(URL), unify URL with the final
+%   URL after redirections.
+
+return_final_url(Options) :-
+    option(final_url(URL), Options),
+    var(URL),
+    !,
+    option(visited([Parts|_]), Options),
+    parts_uri(Parts, URL).
+return_final_url(_).
+
+
+%!  transfer_encoding_filter(+Lines, +In0, -In, +Options) is det.
+%
+%   Install filters depending on the transfer  encoding. If In0 is a
+%   stream-pair, we close the output   side. If transfer-encoding is
+%   not specified, the content-encoding is  interpreted as a synonym
+%   for transfer-encoding, because many   servers incorrectly depend
+%   on  this.  Exceptions  to  this   are  content-types  for  which
+%   disable_encoding_filter/1 holds.
+
+transfer_encoding_filter(Lines, In0, In, Options) :-
+    transfer_encoding(Lines, Encoding),
+    !,
+    transfer_encoding_filter_(Encoding, In0, In, Options).
+transfer_encoding_filter(Lines, In0, In, Options) :-
+    content_encoding(Lines, Encoding),
+    content_type(Lines, Type),
+    \+ http:disable_encoding_filter(Type),
+    !,
+    transfer_encoding_filter_(Encoding, In0, In, Options).
+transfer_encoding_filter(_, In, In, _Options).
+
+transfer_encoding_filter_(Encoding, In0, In, Options) :-
+    option(raw_encoding(Encoding), Options),
+    !,
+    In = In0.
+transfer_encoding_filter_(Encoding, In0, In, _Options) :-
+    stream_pair(In0, In1, Out),
+    (   http:encoding_filter(Encoding, In1, In2)
+    ->  true
+    ;   autoload_encoding(Encoding),
+        http:encoding_filter(Encoding, In1, In2)
+    ->  true
+    ;   domain_error(http_encoding, Encoding)
+    ),
+    (   var(Out)
+    ->  In = In2
+    ;   stream_pair(In, In2, Out)
+    ).
+
+:- multifile
+    autoload_encoding/1.
+
+:- if(exists_source(library(zlib))).
+autoload_encoding(gzip) :-
+    use_module(library(zlib)).
+:- endif.
+:- if(exists_source(library(http/http_stream))).
+autoload_encoding(chunked) :-
+    use_module(library(http/http_stream)).
+:- endif.
+
+content_type(Lines, Type) :-
+    member(Line, Lines),
+    phrase(field('content-type'), Line, Rest),
+    !,
+    atom_codes(Type, Rest).
+
+%!  http:disable_encoding_filter(+ContentType) is semidet.
+%
+%   Do not use  the   =|Content-encoding|=  as =|Transfer-encoding|=
+%   encoding for specific values of   ContentType. This predicate is
+%   multifile and can thus be extended by the user.
+
+http:disable_encoding_filter('application/x-gzip').
+http:disable_encoding_filter('application/x-tar').
+http:disable_encoding_filter('x-world/x-vrml').
+http:disable_encoding_filter('application/zip').
+http:disable_encoding_filter('application/x-gzip').
+http:disable_encoding_filter('application/x-zip-compressed').
+http:disable_encoding_filter('application/x-compress').
+http:disable_encoding_filter('application/x-compressed').
+http:disable_encoding_filter('application/x-spoon').
+
+%!  transfer_encoding(+Lines, -Encoding) is semidet.
+%
+%   True if Encoding  is  the   value  of  the =|Transfer-encoding|=
+%   header.
+
+transfer_encoding(Lines, Encoding) :-
+    what_encoding(transfer_encoding, Lines, Encoding).
+
+what_encoding(What, Lines, Encoding) :-
+    member(Line, Lines),
+    phrase(encoding_(What, Debug), Line, Rest),
+    !,
+    atom_codes(Encoding, Rest),
+    debug(http(What), '~w: ~p', [Debug, Rest]).
+
+encoding_(content_encoding, 'Content-encoding') -->
+    field('content-encoding').
+encoding_(transfer_encoding, 'Transfer-encoding') -->
+    field('transfer-encoding').
+
+%!  content_encoding(+Lines, -Encoding) is semidet.
+%
+%   True if Encoding is the value of the =|Content-encoding|=
+%   header.
+
+content_encoding(Lines, Encoding) :-
+    what_encoding(content_encoding, Lines, Encoding).
+
+%!  read_header(+In:stream, +Parts, -Version, -Code:int,
+%!  -Comment:atom, -Lines:list) is det.
+%
+%   Read the HTTP reply-header.  If the reply is completely empty
+%   an existence error is thrown.  If the replied header is
+%   otherwise invalid a 500 HTTP error is simulated, having the
+%   comment =|Invalid reply header|=.
+%
+%   @param Parts    A list of compound terms that describe the
+%                   parsed request URI.
+%   @param Version  HTTP reply version as Major-Minor pair
+%   @param Code     Numeric HTTP reply-code
+%   @param Comment  Comment of reply-code as atom
+%   @param Lines    Remaining header lines as code-lists.
+%
+%   @error existence_error(http_reply, Uri)
+
+read_header(In, Parts, Major-Minor, Code, Comment, Lines) :-
+    read_line_to_codes(In, Line),
+    (   Line == end_of_file
+    ->  parts_uri(Parts, Uri),
+        existence_error(http_reply,Uri)
+    ;   true
+    ),
+    Line \== end_of_file,
+    phrase(first_line(Major-Minor, Code, Comment), Line),
+    debug(http(open), 'HTTP/~d.~d ~w ~w', [Major, Minor, Code, Comment]),
+    read_line_to_codes(In, Line2),
+    rest_header(Line2, In, Lines),
+    !,
+    (   debugging(http(open))
+    ->  forall(member(HL, Lines),
+               debug(http(open), '~s', [HL]))
+    ;   true
+    ).
+read_header(_, _, 1-1, 500, 'Invalid reply header', []).
+
+rest_header([], _, []) :- !.            % blank line: end of header
+rest_header(L0, In, [L0|L]) :-
+    read_line_to_codes(In, L1),
+    rest_header(L1, In, L).
+
+%!  content_length(+Header, -Length:int) is semidet.
+%
+%   Find the Content-Length in an HTTP reply-header.
+
+content_length(Lines, Length) :-
+    member(Line, Lines),
+    phrase(content_length(Length0), Line),
+    !,
+    Length = Length0.
+
+location(Lines, RequestURI) :-
+    member(Line, Lines),
+    phrase(atom_field(location, RequestURI), Line),
+    !.
+
+connection(Lines, Connection) :-
+    member(Line, Lines),
+    phrase(atom_field(connection, Connection0), Line),
+    !,
+    Connection = Connection0.
+
+first_line(Major-Minor, Code, Comment) -->
+    "HTTP/", integer(Major), ".", integer(Minor),
+    skip_blanks,
+    integer(Code),
+    skip_blanks,
+    rest(Comment).
+
+atom_field(Name, Value) -->
+    field(Name),
+    rest(Value).
+
+content_length(Len) -->
+    field('content-length'),
+    integer(Len).
+
+field(Name) -->
+    { atom_codes(Name, Codes) },
+    field_codes(Codes).
+
+field_codes([]) -->
+    ":",
+    skip_blanks.
+field_codes([H|T]) -->
+    [C],
+    { match_header_char(H, C)
+    },
+    field_codes(T).
+
+match_header_char(C, C) :- !.
+match_header_char(C, U) :-
+    code_type(C, to_lower(U)),
+    !.
+match_header_char(0'_, 0'-).
+
+
+skip_blanks -->
+    [C],
+    { code_type(C, white)
+    },
+    !,
+    skip_blanks.
+skip_blanks -->
+    [].
+
+%!  integer(-Int)//
+%
+%   Read 1 or more digits and return as integer.
+
+integer(Code) -->
+    digit(D0),
+    digits(D),
+    { number_codes(Code, [D0|D])
+    }.
+
+digit(C) -->
+    [C],
+    { code_type(C, digit)
+    }.
+
+digits([D0|D]) -->
+    digit(D0),
+    !,
+    digits(D).
+digits([]) -->
+    [].
+
+%!  rest(-Atom:atom)//
+%
+%   Get rest of input as an atom.
+
+rest(Atom) --> call(rest_(Atom)).
+
+rest_(Atom, L, []) :-
+    atom_codes(Atom, L).
+
+
+%!  reply_header(+Lines, +Options) is det.
+%
+%   Return the entire reply header as  a list of strings to the option
+%   raw_headers(-Headers).
+
+reply_header(Lines, Options) :-
+    option(raw_headers(Headers), Options),
+    !,
+    maplist(string_codes, Headers, Lines).
+reply_header(_, _).
+
+
+                 /*******************************
+                 *   AUTHORIZATION MANAGEMENT   *
+                 *******************************/
+
+%!  http_set_authorization(+URL, +Authorization) is det.
+%
+%   Set user/password to supply with URLs   that have URL as prefix.
+%   If  Authorization  is  the   atom    =|-|=,   possibly   defined
+%   authorization is cleared.  For example:
+%
+%   ```
+%   ?- http_set_authorization('http://www.example.com/private/',
+%                             basic('John', 'Secret'))
+%   ```
+%
+%   @tbd    Move to a separate module, so http_get/3, etc. can use this
+%           too.
+
+:- dynamic
+    stored_authorization/2,
+    cached_authorization/2.
+
+http_set_authorization(URL, Authorization) :-
+    must_be(atom, URL),
+    retractall(stored_authorization(URL, _)),
+    (   Authorization = (-)
+    ->  true
+    ;   check_authorization(Authorization),
+        assert(stored_authorization(URL, Authorization))
+    ),
+    retractall(cached_authorization(_,_)).
+
+check_authorization(Var) :-
+    var(Var),
+    !,
+    instantiation_error(Var).
+check_authorization(basic(User, Password)) :-
+    must_be(atom, User),
+    must_be(text, Password).
+check_authorization(digest(User, Password)) :-
+    must_be(atom, User),
+    must_be(text, Password).
+
+%!  authorization(+URL, -Authorization) is semidet.
+%
+%   True if Authorization must be supplied for URL.
+%
+%   @tbd    Cleanup cache if it gets too big.
+
+authorization(_, _) :-
+    \+ stored_authorization(_, _),
+    !,
+    fail.
+authorization(URL, Authorization) :-
+    cached_authorization(URL, Authorization),
+    !,
+    Authorization \== (-).
+authorization(URL, Authorization) :-
+    (   stored_authorization(Prefix, Authorization),
+        sub_atom(URL, 0, _, _, Prefix)
+    ->  assert(cached_authorization(URL, Authorization))
+    ;   assert(cached_authorization(URL, -)),
+        fail
+    ).
+
+add_authorization(_, Options, Options) :-
+    option(authorization(_), Options),
+    !.
+add_authorization(Parts, Options0, Options) :-
+    url_part(user(User), Parts),
+    url_part(password(Passwd), Parts),
+    !,
+    Options = [authorization(basic(User,Passwd))|Options0].
+add_authorization(Parts, Options0, Options) :-
+    stored_authorization(_, _) ->   % quick test to avoid work
+    parts_uri(Parts, URL),
+    authorization(URL, Auth),
+    !,
+    Options = [authorization(Auth)|Options0].
+add_authorization(_, Options, Options).
+
+
+%!  parse_url_ex(+URL, -Parts)
+%
+%   Parts:  Scheme,  Host,  Port,    User:Password,  RequestURI  (no
+%   fragment).
+
+parse_url_ex(URL, [uri(URL)|Parts]) :-
+    uri_components(URL, Components),
+    phrase(components(Components), Parts),
+    (   option(host(_), Parts)
+    ->  true
+    ;   domain_error(url, URL)
+    ).
+
+components(Components) -->
+    uri_scheme(Components),
+    uri_path(Components),
+    uri_authority(Components),
+    uri_request_uri(Components).
+
+uri_scheme(Components) -->
+    { uri_data(scheme, Components, Scheme), nonvar(Scheme) },
+    !,
+    [ scheme(Scheme)
+    ].
+uri_scheme(_) --> [].
+
+uri_path(Components) -->
+    { uri_data(path, Components, Path0), nonvar(Path0),
+      (   Path0 == ''
+      ->  Path = (/)
+      ;   Path = Path0
+      )
+    },
+    !,
+    [ path(Path)
+    ].
+uri_path(_) --> [].
+
+uri_authority(Components) -->
+    { uri_data(authority, Components, Auth), nonvar(Auth),
+      !,
+      uri_authority_components(Auth, Data)
+    },
+    [ authority(Auth) ],
+    auth_field(user, Data),
+    auth_field(password, Data),
+    auth_field(host, Data),
+    auth_field(port, Data).
+uri_authority(_) --> [].
+
+auth_field(Field, Data) -->
+    { uri_authority_data(Field, Data, EncValue), nonvar(EncValue),
+      !,
+      (   atom(EncValue)
+      ->  uri_encoded(query_value, Value, EncValue)
+      ;   Value = EncValue
+      ),
+      Part =.. [Field,Value]
+    },
+    [ Part ].
+auth_field(_, _) --> [].
+
+uri_request_uri(Components) -->
+    { uri_data(path, Components, Path0),
+      uri_data(search, Components, Search),
+      (   Path0 == ''
+      ->  Path = (/)
+      ;   Path = Path0
+      ),
+      uri_data(path, Components2, Path),
+      uri_data(search, Components2, Search),
+      uri_components(RequestURI, Components2)
+    },
+    [ request_uri(RequestURI)
+    ].
+
+%!  parts_scheme(+Parts, -Scheme) is det.
+%!  parts_uri(+Parts, -URI) is det.
+%!  parts_request_uri(+Parts, -RequestURI) is det.
+%!  parts_search(+Parts, -Search) is det.
+%!  parts_authority(+Parts, -Authority) is semidet.
+
+parts_scheme(Parts, Scheme) :-
+    url_part(scheme(Scheme), Parts),
+    !.
+parts_scheme(Parts, Scheme) :-          % compatibility with library(url)
+    url_part(protocol(Scheme), Parts),
+    !.
+parts_scheme(_, http).
+
+parts_authority(Parts, Auth) :-
+    url_part(authority(Auth), Parts),
+    !.
+parts_authority(Parts, Auth) :-
+    url_part(host(Host), Parts, _),
+    url_part(port(Port), Parts, _),
+    url_part(user(User), Parts, _),
+    url_part(password(Password), Parts, _),
+    uri_authority_components(Auth,
+                             uri_authority(User, Password, Host, Port)).
+
+parts_request_uri(Parts, RequestURI) :-
+    option(request_uri(RequestURI), Parts),
+    !.
+parts_request_uri(Parts, RequestURI) :-
+    url_part(path(Path), Parts, /),
+    ignore(parts_search(Parts, Search)),
+    uri_data(path, Data, Path),
+    uri_data(search, Data, Search),
+    uri_components(RequestURI, Data).
+
+parts_search(Parts, Search) :-
+    option(query_string(Search), Parts),
+    !.
+parts_search(Parts, Search) :-
+    option(search(Fields), Parts),
+    !,
+    uri_query_components(Search, Fields).
+
+
+parts_uri(Parts, URI) :-
+    option(uri(URI), Parts),
+    !.
+parts_uri(Parts, URI) :-
+    parts_scheme(Parts, Scheme),
+    ignore(parts_authority(Parts, Auth)),
+    parts_request_uri(Parts, RequestURI),
+    uri_components(RequestURI, Data),
+    uri_data(scheme, Data, Scheme),
+    uri_data(authority, Data, Auth),
+    uri_components(URI, Data).
+
+parts_port(Parts, Port) :-
+    parts_scheme(Parts, Scheme),
+    default_port(Scheme, DefPort),
+    url_part(port(Port), Parts, DefPort).
+
+url_part(Part, Parts) :-
+    Part =.. [Name,Value],
+    Gen =.. [Name,RawValue],
+    option(Gen, Parts),
+    !,
+    Value = RawValue.
+
+url_part(Part, Parts, Default) :-
+    Part =.. [Name,Value],
+    Gen =.. [Name,RawValue],
+    (   option(Gen, Parts)
+    ->  Value = RawValue
+    ;   Value = Default
+    ).
+
+
+                 /*******************************
+                 *            COOKIES           *
+                 *******************************/
+
+write_cookies(Out, Parts, Options) :-
+    http:write_cookies(Out, Parts, Options),
+    !.
+write_cookies(_, _, _).
+
+update_cookies(_, _, _) :-
+    predicate_property(http:update_cookies(_,_,_), number_of_clauses(0)),
+    !.
+update_cookies(Lines, Parts, Options) :-
+    (   member(Line, Lines),
+        phrase(atom_field('set_cookie', CookieData), Line),
+        http:update_cookies(CookieData, Parts, Options),
+        fail
+    ;   true
+    ).
+
+
+                 /*******************************
+                 *           OPEN ANY           *
+                 *******************************/
+
+:- multifile iostream:open_hook/6.
+
+%!  iostream:open_hook(+Spec, +Mode, -Stream, -Close,
+%!                     +Options0, -Options) is semidet.
+%
+%   Hook implementation that makes  open_any/5   support  =http= and
+%   =https= URLs for =|Mode == read|=.
+
+iostream:open_hook(URL, read, Stream, Close, Options0, Options) :-
+    (atom(URL) -> true ; string(URL)),
+    uri_is_global(URL),
+    uri_components(URL, Components),
+    uri_data(scheme, Components, Scheme),
+    http_scheme(Scheme),
+    !,
+    Options = Options0,
+    Close = close(Stream),
+    http_open(URL, Stream, Options0).
+
+http_scheme(http).
+http_scheme(https).
+
+
+                 /*******************************
+                 *          KEEP-ALIVE          *
+                 *******************************/
+
+%!  consider_keep_alive(+HeaderLines, +Parts, +Host,
+%!                      +Stream0, -Stream,
+%!                      +Options) is det.
+%
+%   If we have agree on a Keep-alive   connection, return a range stream
+%   rather than the original stream. We also  use the content length and
+%   a range stream if we are dealing   with an HTTPS connection. This is
+%   because not all servers seem to  complete the TLS closing handshake.
+%   If the server does not complete  this   we  receive  a TLS handshake
+%   error on end-of-file, causing the read to fail.
+
+consider_keep_alive(Lines, Parts, Host, StreamPair, In, Options) :-
+    option(connection(Asked), Options),
+    keep_alive(Asked),
+    connection(Lines, Given),
+    keep_alive(Given),
+    content_length(Lines, Bytes),
+    !,
+    stream_pair(StreamPair, In0, _),
+    connection_address(Host, Parts, HostPort),
+    debug(http(keep_alive),
+          'Keep-alive to ~w (~D bytes)', [HostPort, Bytes]),
+    stream_range_open(In0, In,
+                      [ size(Bytes),
+                        onclose(keep_alive(StreamPair, HostPort))
+                      ]).
+consider_keep_alive(Lines, Parts, _Host, StreamPair, In, _Options) :-
+    memberchk(scheme(https), Parts),
+    content_length(Lines, Bytes),
+    !,
+    stream_pair(StreamPair, In0, _),
+    stream_range_open(In0, In,
+                      [ size(Bytes),
+                        onclose(close_range(StreamPair))
+                      ]).
+consider_keep_alive(_, _, _, Stream, Stream, _).
+
+connection_address(Host, _, Host) :-
+    Host = _:_,
+    !.
+connection_address(Host, Parts, Host:Port) :-
+    parts_port(Parts, Port).
+
+keep_alive(keep_alive) :- !.
+keep_alive(Connection) :-
+    downcase_atom(Connection, 'keep-alive').
+
+%!  keep_alive(+StreamPair, +Host, +In, -Left) is det.
+%
+%   Callback when closing the range stream   used to process the content
+%   of the reply. This callback makes   the  stream available for future
+%   keep-alive connections or closes the stream. The stream is closed if
+%
+%     - There are too many bytes left unprocessed in the range stream.
+%     - There are too many pooled connections.
+
+:- public keep_alive/4.
+:- det(keep_alive/4).
+
+keep_alive(StreamPair, Host, _In, 0) :-
+    !,
+    add_to_pool_or_close(Host, StreamPair).
+keep_alive(StreamPair, Host, In, Left) :-
+    (   Left < 100,
+        debug(http(connection), 'Reading ~D left bytes', [Left]),
+        read_incomplete(In, Left)
+    ->  add_to_pool_or_close(Host, StreamPair)
+    ;   debug(http(connection),
+              'Closing connection due to excessive unprocessed input', []),
+        close_keep_alive(StreamPair)
+    ).
+
+add_to_pool_or_close(Host, StreamPair) :-
+    add_to_pool(Host, StreamPair),
+    !,
+    debug(http(connection), 'Added connection to ~p to pool', [Host]).
+add_to_pool_or_close(Host, StreamPair) :-
+    close_keep_alive(StreamPair),
+    debug(http(connection), 'Closed connection to ~p [pool full]', [Host]).
+
+close_keep_alive(StreamPair) :-
+    (   debugging(http(connection))
+    ->  catch(close(StreamPair), E,
+              print_message(warning, E))
+    ;   close(StreamPair, [force(true)])
+    ).
+
+:- public close_range/3.
+close_range(StreamPair, _Raw, _BytesLeft) :-
+    close(StreamPair, [force(true)]).
+
+%!  read_incomplete(+In, +Left) is semidet.
+%
+%   If we have not all input from  a Keep-alive connection, read the
+%   remainder if it is short. Else, we fail and close the stream.
+
+read_incomplete(In, Left) :-
+    catch(setup_call_cleanup(
+              open_null_stream(Null),
+              copy_stream_data(In, Null, Left),
+              close(Null)),
+          error(_,_),
+          fail).
+
+:- dynamic
+    connection_pool/4,              % Hash, Address, Stream, Time
+    connection_gc_time/1.
+
+%!  add_to_pool(+Address, +StreamPair) is semidet.
+%
+%   Add a connection  to  the  keep-alive   pool  after  completing  the
+%   interaction. Fails if there are already  too many connections in the
+%   pool.
+
+add_to_pool(Address, StreamPair) :-
+    keep_connection(Address),
+    get_time(Now),
+    term_hash(Address, Hash),
+    assertz(connection_pool(Hash, Address, StreamPair, Now)).
+
+%!  get_from_pool(+Address, -StreamPair) is nondet.
+%
+%   Get an existing Keep-Alive connection to  Address as StreamPair. The
+%   caller relies on non-determinism of this   predicate  to try another
+%   connection if the returned one is  already   closed  by the peer. We
+%   cannot rely on the non-determinism of retract/1 as that respects the
+%   _logical update view_. Therefore, we must   use retract/1 and commit
+%   as retract/1 guarantees that  the  first   retracted  clause  is not
+%   already retracted.
+
+get_from_pool(Address, StreamPair) :-
+    term_hash(Address, Hash),
+    repeat,
+    (   retract(connection_pool(Hash, Address, StreamPair, _))
+    ->  true
+    ;   !,
+        fail
+    ).
+
+%!  keep_connection(+Address) is semidet.
+%
+%   Succeeds if we want to keep the connection open. We currently keep a
+%   maximum of `http:max_keep_alive_connections` connections waiting and
+%   a maximum of `http:max_keep_alive_host_connections`  waiting for the
+%   same  address.  Connections  older  than  `http:max_keep_alive_idle`
+%   seconds are closed.
+
+keep_connection(Address) :-
+    setting(http:max_keep_alive_idle, Time),
+    close_old_connections(Time),
+    predicate_property(connection_pool(_,_,_,_), number_of_clauses(C)),
+    setting(http:max_keep_alive_connections, MaxConnections),
+    C =< MaxConnections,
+    term_hash(Address, Hash),
+    aggregate_all(count, connection_pool(Hash, Address, _, _), Count),
+    setting(http:max_keep_alive_host_connections, MaxHostConnections),
+    Count =< MaxHostConnections.
+
+close_old_connections(Timeout) :-
+    get_time(Now),
+    Before is Now - Timeout,
+    (   connection_gc_time(GC),
+        GC > Before
+    ->  true
+    ;   (   retractall(connection_gc_time(_)),
+            asserta(connection_gc_time(Now)),
+            connection_pool(Hash, Address, StreamPair, Added),
+            Added < Before,
+            retract(connection_pool(Hash, Address, StreamPair, Added)),
+            debug(http(connection),
+                  'Closing inactive keep-alive to ~p', [Address]),
+            close(StreamPair, [force(true)]),
+            fail
+        ;   true
+        )
+    ).
+
+
+%!  http_close_keep_alive(+Address) is det.
+%
+%   Close all keep-alive connections matching Address. Address is of
+%   the  form  Host:Port.  In  particular,  http_close_keep_alive(_)
+%   closes all currently known keep-alive connections.
+
+http_close_keep_alive(Address) :-
+    forall(get_from_pool(Address, StreamPair),
+           close(StreamPair, [force(true)])).
+
+%!  keep_alive_error(+Error, +StreamPair)
+%
+%   Deal with an error from  reusing   a  keep-alive  connection. If the
+%   error is due to an I/O error  or end-of-file, fail to backtrack over
+%   get_from_pool/2. Otherwise it is a real   error and we thus re-raise
+%   it. In all cases we close StreamPair rather than returning it to the
+%   pool as we may have done a partial read and thus be out of sync wrt.
+%   the HTTP protocol.
+
+keep_alive_error(error(keep_alive(closed), _), _) :-
+    !,
+    debug(http(connection), 'Keep-alive connection was closed', []),
+    fail.
+keep_alive_error(error(io_error(_,_), _), StreamPair) :-
+    !,
+    close(StreamPair, [force(true)]),
+    debug(http(connection), 'IO error on Keep-alive connection', []),
+    fail.
+keep_alive_error(error(existence_error(http_reply, _URL), _), _) :-
+    !,
+    debug(http(connection), 'Got empty reply on Keep-alive connection', []),
+    fail.
+keep_alive_error(Error, StreamPair) :-
+    close(StreamPair, [force(true)]),
+    throw(Error).
+
+
+                 /*******************************
+                 *     HOOK DOCUMENTATION       *
+                 *******************************/
+
+%!  http:open_options(+Parts, -Options) is nondet.
+%
+%   This hook is used by the HTTP   client library to define default
+%   options based on the the broken-down request-URL.  The following
+%   example redirects all traffic, except for localhost over a proxy:
+%
+%       ```
+%       :- multifile
+%           http:open_options/2.
+%
+%       http:open_options(Parts, Options) :-
+%           option(host(Host), Parts),
+%           Host \== localhost,
+%           Options = [proxy('proxy.local', 3128)].
+%       ```
+%
+%   This hook may return multiple   solutions.  The returned options
+%   are  combined  using  merge_options/3  where  earlier  solutions
+%   overrule later solutions.
+
+%!  http:write_cookies(+Out, +Parts, +Options) is semidet.
+%
+%   Emit a =|Cookie:|= header for the  current connection. Out is an
+%   open stream to the HTTP server, Parts is the broken-down request
+%   (see uri_components/2) and Options is the list of options passed
+%   to http_open.  The predicate is called as if using ignore/1.
+%
+%   @see complements http:update_cookies/3.
+%   @see library(http/http_cookie) implements cookie handling on
+%   top of these hooks.
+
+%!  http:update_cookies(+CookieData, +Parts, +Options) is semidet.
+%
+%   Update the cookie database.  CookieData  is   the  value  of the
+%   =|Set-Cookie|= field, Parts is  the   broken-down  request  (see
+%   uri_components/2) and Options is the list   of options passed to
+%   http_open.
+%
+%   @see complements http:write_cookies
+%   @see library(http/http_cookies) implements cookie handling on
+%   top of these hooks.