httpclient 2.3.0.1 → 2.8.3

data/lib/httpclient/connection.rb

@@ -1,5 +1,5 @@
 # HTTPClient - HTTP client library.
-# Copyright (C) 2000-2009  NAKAMURA, Hiroshi  <nahi@ruby-lang.org>.
+# Copyright (C) 2000-2015  NAKAMURA, Hiroshi  <nahi@ruby-lang.org>.
 #
 # This program is copyrighted free software by NAKAMURA, Hiroshi.  You can
 # redistribute it and/or modify it under the same terms of Ruby's license;

data/lib/httpclient/cookie.rb

@@ -1,573 +1,220 @@
-# cookie.rb is redistributed file which is originally included in Webagent
-# version 0.6.2 by TAKAHASHI `Maki' Masayoshi.  And it contains some bug fixes.
-# You can download the entire package of Webagent from
-# http://www.rubycolor.org/arc/.
+# do not override if httpclient/webagent-cookie is loaded already
+unless defined?(HTTPClient::CookieManager)
+begin # for catching LoadError and load webagent-cookie instead
 
-
-# Cookie class
-#
-# I refered to w3m's source to make these classes. Some comments
-# are quoted from it. I'm thanksful for author(s) of it.
-#
-#    w3m homepage:  http://ei5nazha.yz.yamagata-u.ac.jp/~aito/w3m/eng/
-
-require 'time'
-require 'monitor'
+require 'http-cookie'
 require 'httpclient/util'
 
-class WebAgent
-
-  module CookieUtils
+class HTTPClient
+  class CookieManager
+    include HTTPClient::Util
 
-    def head_match?(str1, str2)
-      str1 == str2[0, str1.length]
-    end
+    attr_reader :format, :jar
+    attr_accessor :cookies_file
 
-    def tail_match?(str1, str2)
-      if str1.length > 0
-	str1 == str2[-str1.length..-1].to_s
-      else
-	true
-      end
+    def initialize(cookies_file = nil, format = WebAgentSaver, jar = HTTP::CookieJar.new)
+      @cookies_file = cookies_file
+      @format = format
+      @jar = jar
+      load_cookies if @cookies_file
+    end
+
+    def load_cookies
+      check_cookies_file
+      @jar.clear
+      @jar.load(@cookies_file, :format => @format)
+    end
+
+    def save_cookies(session = false)
+      check_cookies_file
+      @jar.save(@cookies_file, :format => @format, :session => session)
+    end
+
+    def cookies(uri = nil)
+      cookies = @jar.cookies(uri)
+      # TODO: return HTTP::Cookie in the future
+      cookies.map { |cookie|
+        WebAgent::Cookie.new(
+          :name => cookie.name,
+          :value => cookie.value,
+          :domain => cookie.domain,
+          :path => cookie.path,
+          :origin => cookie.origin,
+          :for_domain => cookie.for_domain,
+          :expires => cookie.expires,
+          :httponly => cookie.httponly,
+          :secure => cookie.secure
+        )
+      }
     end
 
-    def domain_match(host, domain)
-      domainname = domain.sub(/\.\z/, '').downcase
-      hostname = host.sub(/\.\z/, '').downcase
-      case domain
-      when /\d+\.\d+\.\d+\.\d+/
-	return (hostname == domainname)
-      when '.' 
-	return true
-      when /^\./
-        # allows; host == rubyforge.org, domain == .rubyforge.org
-	return tail_match?(domainname, '.' + hostname)
-      else
-	return (hostname == domainname)
+    def cookie_value(uri)
+      cookies = self.cookies(uri)
+      unless cookies.empty?
+        HTTP::Cookie.cookie_value(cookies)
       end
     end
 
-    def total_dot_num(string)
-      string.scan(/\./).length()
+    def parse(value, uri)
+      @jar.parse(value, uri)
     end
 
-  end
-
-  class Cookie
-    include CookieUtils
-
-    attr_accessor :name, :value
-    attr_accessor :domain, :path
-    attr_accessor :expires      ## for Netscape Cookie
-    attr_accessor :url
-    attr_writer :use, :secure, :http_only, :discard, :domain_orig, :path_orig, :override
-
-    USE = 1
-    SECURE = 2
-    DOMAIN = 4
-    PATH = 8
-    DISCARD = 16
-    OVERRIDE = 32
-    OVERRIDE_OK = 32
-    HTTP_ONLY = 64
-
-    def initialize()
-      @name = @value = @domain = @path = nil
-      @expires = nil
-      @url = nil
-      @use = @secure = @http_only = @discard = @domain_orig = @path_orig = @override = nil
-    end
-
-    def discard?
-      @discard
-    end
-
-    def use?
-      @use
-    end
-
-    def secure?
-      @secure
-    end
-
-    def http_only?
-      @http_only
-    end
-
-    def domain_orig?
-      @domain_orig
-    end
-
-    def path_orig?
-      @path_orig
+    def cookies=(cookies)
+      @jar.clear
+      cookies.each do |cookie|
+        add(cookie)
+      end
     end
 
-    def override?
-      @override
+    def add(cookie)
+      @jar.add(cookie)
     end
 
-    def flag
-      flg = 0
-      flg += USE  if @use
-      flg += SECURE  if @secure
-      flg += HTTP_ONLY  if @http_only
-      flg += DOMAIN  if @domain_orig
-      flg += PATH  if @path_orig
-      flg += DISCARD if @discard
-      flg += OVERRIDE if @override
-      flg
+    def find(uri)
+      warning('CookieManager#find is deprecated and will be removed in near future. Use HTTP::Cookie.cookie_value(CookieManager#cookies) instead')
+      if cookie = cookies(uri)
+        HTTP::Cookie.cookie_value(cookie)
+      end
     end
 
-    def set_flag(flag)
-      flag = flag.to_i
-      @use = true      if flag & USE > 0
-      @secure = true   if flag & SECURE > 0
-      @http_only = true   if flag & HTTP_ONLY > 0
-      @domain_orig = true if flag & DOMAIN > 0
-      @path_orig = true if flag & PATH > 0
-      @discard  = true if flag & DISCARD > 0
-      @override = true if flag & OVERRIDE > 0
-    end
+  private
 
-    def match?(url)
-      domainname = url.host
-      if (!domainname ||
-	  !domain_match(domainname, @domain) ||
-	  (@path && !head_match?(@path, url.path.empty? ? '/' : url.path)) ||
-	  (@secure && (url.scheme != 'https')) )
-	return false
-      else
-	return true
+    def check_cookies_file
+      unless @cookies_file
+        raise ArgumentError.new('Cookies file not specified')
       end
     end
+  end
 
-    def join_quotedstr(array, sep)
-      ret = Array.new()
-      old_elem = nil
-      array.each{|elem|
-	if (elem.scan(/"/).length % 2) == 0
-	  if old_elem
-	    old_elem << sep << elem
-	  else
-	    ret << elem
-	    old_elem = nil
-	  end  
-	else
-	  if old_elem
-	    old_elem << sep << elem
-	    ret << old_elem
-	    old_elem = nil
-	  else
-	    old_elem = elem.dup
-	  end
-	end
-      }
-      ret
+  class WebAgentSaver < HTTP::CookieJar::AbstractSaver
+    # no option
+    def default_options
+      {}
     end
 
-    def parse(str, url)
-      @url = url
-      # TODO: should not depend on join_quotedstr. scan with escape like CSV.
-      cookie_elem = str.split(/;/)
-      cookie_elem = join_quotedstr(cookie_elem, ';')
-      cookie_elem -= [""] # del empty elements, a cookie might included ";;"
-      first_elem = cookie_elem.shift
-      if first_elem !~ /([^=]*)(\=(.*))?/
-	return
-	## raise ArgumentError 'invalid cookie value'
-      end
-      @name = $1.strip
-      @value = normalize_cookie_value($3)
-      cookie_elem.each{|pair|
-	key, value = pair.split(/=/, 2)  ## value may nil
-	key.strip!
-        value = normalize_cookie_value(value)
-	case key.downcase
-	when 'domain'
-	  @domain = value
-	when 'expires'
-          @expires = nil
-	  begin
-	    @expires = Time.parse(value).gmtime() if value
-	  rescue ArgumentError
-	  end
-	when 'path'
-	  @path = value
-	when 'secure'
-	  @secure = true  ## value may nil, but must 'true'.
-	when 'httponly'
-	  @http_only = true  ## value may nil, but must 'true'.
-	else
-	  ## ignore
-	end
+    # same as HTTP::CookieJar::CookiestxtSaver
+    def save(io, jar)
+      jar.each { |cookie|
+        next if !@session && cookie.session?
+        io.print cookie_to_record(cookie)
       }
     end
 
-    def normalize_cookie_value(value)
-      if value
-        value = value.strip.sub(/\A"(.*)"\z/) { $1 }
-        value = nil if value.empty?
-      end
-      value
+    # same as HTTP::CookieJar::CookiestxtSaver
+    def load(io, jar)
+      io.each_line { |line|
+        cookie = parse_record(line) and jar.add(cookie)
+      }
     end
-    private :normalize_cookie_value
-  end
 
-  class CookieManager
-    include CookieUtils
-
-    ### errors
-    class Error < StandardError; end
-    class ErrorOverrideOK < Error; end
-    class SpecialError < Error; end
+  private
 
-    attr_reader :cookies
-    attr_accessor :cookies_file
-    attr_accessor :accept_domains, :reject_domains
-
-    # for conformance to http://wp.netscape.com/newsref/std/cookie_spec.html
-    attr_accessor :netscape_rule
-    SPECIAL_DOMAIN = [".com",".edu",".gov",".mil",".net",".org",".int"]
-
-    def initialize(file=nil)
-      @cookies = Array.new()
-      @cookies.extend(MonitorMixin)
-      @cookies_file = file
-      @is_saved = true
-      @reject_domains = Array.new()
-      @accept_domains = Array.new()
-      @netscape_rule = false
+    def cookie_to_record(cookie)
+      [
+        cookie.origin,
+        cookie.name, 
+        cookie.value,
+        cookie.expires.to_i,
+        cookie.dot_domain,
+        cookie.path,
+        self.class.flag(cookie)
+      ].join("\t") + "\n"
     end
 
-    def cookies=(cookies)
-      @cookies = cookies
-      @cookies.extend(MonitorMixin)
-    end
+    def parse_record(line)
+      return nil if /\A#/ =~ line
+      col = line.chomp.split(/\t/)
 
-    def save_all_cookies(force = nil, save_unused = true, save_discarded = true)
-      @cookies.synchronize do
-        check_expired_cookies()
-        if @is_saved and !force
-          return
-        end
-        File.open(@cookies_file, 'w') do |f|
-          @cookies.each do |cookie|
-            if (cookie.use? or save_unused) and
-                (!cookie.discard? or save_discarded)
-              f.print(cookie.url.to_s,"\t",
-                      cookie.name,"\t",
-                      cookie.value,"\t",
-                      cookie.expires.to_i,"\t",
-                      cookie.domain,"\t",
-                      cookie.path,"\t",
-                      cookie.flag,"\n")
-            end
-          end
-        end
+      origin = col[0]
+      name = col[1]
+      value = col[2]
+      value.chomp!
+      if col[3].empty? or col[3] == '0'
+        expires = nil
+      else
+        expires = Time.at(col[3].to_i)
+        return nil if expires < Time.now
       end
-      @is_saved = true
-    end
+      domain = col[4]
+      path = col[5]
 
-    def save_cookies(force = nil)
-      save_all_cookies(force, false, false)
+      cookie = WebAgent::Cookie.new(name, value,
+        :origin => origin,
+        :domain => domain,
+        :path => path,
+        :expires => expires
+      )
+      self.class.set_flag(cookie, col[6].to_i)
+      cookie
     end
 
-    def check_expired_cookies()
-      @cookies.reject!{|cookie|
-        is_expired = (cookie.expires && (cookie.expires < Time.now.gmtime))
-        if is_expired && !cookie.discard?
-          @is_saved = false
-        end
-        is_expired
-      }
-    end
+    USE = 1
+    SECURE = 2
+    DOMAIN = 4
+    PATH = 8
+    HTTP_ONLY = 64
 
-    def parse(str, url)
-      cookie = WebAgent::Cookie.new()
-      cookie.parse(str, url)
-      add(cookie)
+    def self.flag(cookie)
+      flg = 0
+      flg += USE # not used
+      flg += SECURE  if cookie.secure?
+      flg += DOMAIN  if cookie.for_domain?
+      flg += HTTP_ONLY  if cookie.httponly?
+      flg += PATH  if cookie.path # not used
+      flg
     end
 
-    def make_cookie_str(cookie_list)
-      if cookie_list.empty?
-	return nil
-      end
-
-      ret = ''
-      c = cookie_list.shift
-      ret += "#{c.name}=#{c.value}"
-      cookie_list.each{|cookie|
-	ret += "; #{cookie.name}=#{cookie.value}"
-      }
-      return ret
+    def self.set_flag(cookie, flag)
+      cookie.secure = true if flag & SECURE > 0
+      cookie.for_domain = true if flag & DOMAIN > 0
+      cookie.httponly = true if flag & HTTP_ONLY > 0
     end
-    private :make_cookie_str
+  end
+end
 
+# for backward compatibility
+class WebAgent
+  CookieManager = ::HTTPClient::CookieManager
 
-    def find(url)
-      return nil if @cookies.empty?
+  class Cookie < HTTP::Cookie
+    include HTTPClient::Util
 
-      cookie_list = Array.new()
-      @cookies.each{|cookie|
-        is_expired = (cookie.expires && (cookie.expires < Time.now.gmtime))
-        if cookie.use? && !is_expired && cookie.match?(url)
-          if cookie_list.select{|c1| c1.name == cookie.name}.empty?
-            cookie_list << cookie
-          end
-        end
-      }
-      return make_cookie_str(cookie_list)
+    def url
+      deprecated('url', 'origin')
+      self.origin
     end
 
-    def find_cookie_info(domain, path, name)
-      @cookies.find{|c|
-	c.domain == domain && c.path == path && c.name == name
-      }
+    def url=(url)
+      deprecated('url=', 'origin=')
+      self.origin = url
     end
-    private :find_cookie_info
 
-    # not tested well; used only netscape_rule = true.
-    def cookie_error(err, override)
-      if !err.kind_of?(ErrorOverrideOK) || !override
-	raise err
-      end
+    def http_only?
+      deprecated('http_only?', 'httponly?')
+      self.httponly?
     end
-    private :cookie_error
-
-    def add(cookie)
-      url = cookie.url
-      name, value = cookie.name, cookie.value
-      expires, domain, path = 
-	cookie.expires, cookie.domain, cookie.path
-      secure, http_only, domain_orig, path_orig = 
-	cookie.secure?, cookie.http_only?, cookie.domain_orig?, cookie.path_orig?
-      discard, override = 
-	cookie.discard?, cookie.override?
-
-      domainname = url.host
-      domain_orig, path_orig = domain, path
-
-      if domain
-
-	# [DRAFT 12] s. 4.2.2 (does not apply in the case that
-	# host name is the same as domain attribute for version 0
-	# cookie)
-	# I think that this rule has almost the same effect as the
-	# tail match of [NETSCAPE].
-	if domain !~ /^\./ && domainname != domain
-	  domain = '.'+domain
-	end
-
-        # [NETSCAPE] rule
-        if @netscape_rule
-          n = total_dot_num(domain)
-          if n < 2
-            cookie_error(SpecialError.new(), override)
-          elsif n == 2
-            ## [NETSCAPE] rule
-            ok = SPECIAL_DOMAIN.select{|sdomain|
-              sdomain == domain[-(sdomain.length)..-1]
-            }
-            if ok.empty?
-              cookie_error(SpecialError.new(), override)
-            end
-          end
-        end
-
-        # this implementation does not check RFC2109 4.3.2 case 2;
-        # the portion of host not in domain does not contain a dot.
-        # according to nsCookieService.cpp in Firefox 3.0.4, Firefox 3.0.4
-        # and IE does not check, too.
-      end
 
-      path ||= url.path.sub(%r|/[^/]*\z|, '')
-      domain ||= domainname
-      @cookies.synchronize do
-        cookie = find_cookie_info(domain, path, name)
-        if !cookie
-          cookie = WebAgent::Cookie.new()
-          cookie.use = true
-          @cookies << cookie
-        end
-        check_expired_cookies()
-      end
+    alias original_domain domain
 
-      cookie.url = url
-      cookie.name = name
-      cookie.value = value
-      cookie.expires = expires
-      cookie.domain = domain
-      cookie.path = path
-
-      ## for flag
-      cookie.secure = secure
-      cookie.http_only = http_only
-      cookie.domain_orig = domain_orig
-      cookie.path_orig = path_orig
-      if discard || cookie.expires == nil
-	cookie.discard = true
-      else
-	cookie.discard = false
-	@is_saved = false
-      end
+    def domain
+      warning('Cookie#domain returns dot-less domain name now. Use Cookie#dot_domain if you need "." at the beginning.')
+      self.original_domain
     end
 
-    def load_cookies()
-      return if !File.readable?(@cookies_file)
-      @cookies.synchronize do
-        @cookies.clear
-        File.open(@cookies_file,'r'){|f|
-          while line = f.gets
-            cookie = WebAgent::Cookie.new()
-            @cookies << cookie
-            col = line.chomp.split(/\t/)
-            cookie.url = HTTPClient::Util.urify(col[0])
-            cookie.name = col[1]
-            cookie.value = col[2]
-            if col[3].empty? or col[3] == '0'
-              cookie.expires = nil
-            else
-              cookie.expires = Time.at(col[3].to_i).gmtime
-            end
-            cookie.domain = col[4]
-            cookie.path = col[5]
-            cookie.set_flag(col[6])
-          end
-        }
-      end
+    def flag
+      deprecated('flag', 'secure, for_domain, etc.')
+      HTTPClient::WebAgentSaver.flag(self)
     end
 
-    def check_cookie_accept_domain(domain)
-      unless domain
-	return false
-      end
-      @accept_domains.each{|dom|
-	if domain_match(domain, dom)
-	  return true
-	end
-      }
-      @reject_domains.each{|dom|
-	if domain_match(domain, dom)
-	  return false
-	end
-      }
-      return true
+  private
+
+    def deprecated(old, new)
+      warning("WebAgent::Cookie is deprecated and will be replaced with HTTP::Cookie in the near future. Please use Cookie##{new} instead of Cookie##{old} for the replacement.")
     end
   end
 end
 
-__END__
-
-=begin
-
-== WebAgent::CookieManager Class
-
-Load, save, parse and send cookies.
-
-=== Usage
-
-  ## initialize
-  cm = WebAgent::CookieManager.new("/home/foo/bar/cookie")
-
-  ## load cookie data
-  cm.load_cookies()
-
-  ## parse cookie from string (maybe "Set-Cookie:" header)
-  cm.parse(str)
-
-  ## send cookie data to url
-  f.write(cm.find(url))
-
-  ## save cookie to cookiefile
-  cm.save_cookies()
-
-
-=== Class Methods
-
- -- CookieManager::new(file=nil)
-
-     create new CookieManager. If a file is provided,
-     use it as cookies' file.
-
-=== Methods
-
- -- CookieManager#save_cookies(force = nil)
-
-     save cookies' data into file. if argument is true,
-     save data although data is not modified.
-
- -- CookieManager#parse(str, url)
-
-     parse string and store cookie (to parse HTTP response header).
-
- -- CookieManager#find(url)
-
-     get cookies and make into string (to send as HTTP request header).
-
- -- CookieManager#add(cookie)
-
-     add new cookie.
-
- -- CookieManager#load_cookies()
-
-     load cookies' data from file.
-
-
-== WebAgent::CookieUtils Module
-
- -- CookieUtils::head_match?(str1, str2)
- -- CookieUtils::tail_match?(str1, str2)
- -- CookieUtils::domain_match(host, domain)
- -- CookieUtils::total_dot_num(str)
-
-
-== WebAgent::Cookie Class
-
-=== Class Methods
-
- -- Cookie::new()
-
-      create new cookie.
-
-=== Methods
-
- -- Cookie#match?(url)
-
-       match cookie by url. if match, return true. otherwise,
-       return false.
-
- -- Cookie#name
- -- Cookie#name=(name)
- -- Cookie#value
- -- Cookie#value=(value)
- -- Cookie#domain
- -- Cookie#domain=(domain)
- -- Cookie#path
- -- Cookie#path=(path)
- -- Cookie#expires
- -- Cookie#expires=(expires)
- -- Cookie#url
- -- Cookie#url=(url)
-
-      accessor methods for cookie's items.
-
- -- Cookie#discard?
- -- Cookie#discard=(discard)
- -- Cookie#use?
- -- Cookie#use=(use)
- -- Cookie#secure?
- -- Cookie#secure=(secure)
- -- Cookie#http_only?
- -- Cookie#http_only=(http_only)
- -- Cookie#domain_orig?
- -- Cookie#domain_orig=(domain_orig)
- -- Cookie#path_orig?
- -- Cookie#path_orig=(path_orig)
- -- Cookie#override?
- -- Cookie#override=(override)
- -- Cookie#flag
- -- Cookie#set_flag(flag_num)
-
-      accessor methods for flags.
-
-=end
+rescue LoadError
+  require 'httpclient/webagent-cookie'
+end
+end