agentx 0.0.2

data/lib/agentx/html.rb

@@ -0,0 +1,82 @@
+
+module AgentX
+
+  class HTML
+    def initialize(html)
+      if html.kind_of?(String)
+        @html = Nokogiri::HTML(html)
+      else
+        @html = html
+      end
+    end
+
+    def to_html
+      @html.to_html
+    end
+
+    def first(selector)
+      (e = @html.css(selector).first) && HTML.new(e)
+    end
+
+    def all(selector)
+      @html.css(selector).map { |e| HTML.new(e) }
+    end
+
+    def parent
+      HTML.new(@html.parent)
+    end
+
+    def children
+      @html.children.map { |e| HTML.new(e) }
+    end
+
+    def next
+      HTML.new(@html.next)
+    end
+
+    def previous
+      HTML.new(@html.previous)
+    end
+
+    def attributes
+      h = {}
+      @html.attribute_nodes.each { |n| h[n.name] = n.value }
+      h
+    end
+
+    def [](attr)
+      @html[attr.to_s]
+    end
+
+    def form_to_hash(opts={})
+      h = {}
+      all('input').each do |input|
+        h[input['name']] = input['value']
+      end
+      opts.each do |k,v|
+        h[k.to_s] = v
+      end
+      h
+    end
+
+    def to_s
+      to_html
+    end
+
+    def inspect
+      to_html
+    end
+
+    def to_nokogiri
+      @html
+    end
+
+    NBSP = Nokogiri::HTML(' ').text
+
+    def text
+      @html.text.gsub(NBSP, ' ')
+    end
+  end
+
+end
+

data/lib/agentx/request.rb

@@ -0,0 +1,288 @@
+
+module AgentX
+
+  class Request
+    attr_reader :url, :method
+
+    def initialize(session, url, params={})
+      @session = session
+      @url, @params = url, params
+      @headers = session.headers
+      @body = {}
+    end
+
+    def headers(headers={})
+      if @method
+        @headers
+      else
+        headers.each { |k,v| set_header(k, v) }
+        self
+      end
+    end
+
+    def body(body=nil)
+      if @method
+        @body
+      else
+        @body.merge!(body) if body
+        self
+      end
+    end
+
+    def params(params=nil)
+      if @method
+        @params
+      else
+        @params.merge!(params) if params
+        self
+      end
+    end
+
+    def head(params={})
+      params(params)
+      http(:head)
+    end
+
+    def get(params={})
+      params(params)
+      http(:get)
+    end
+
+    def post(body={})
+      body(body)
+      http(:post)
+    end
+
+    def put(body={})
+      body(body)
+      http(:put)
+    end
+
+    def cacheable?
+      method == 'GET' || method == 'HEAD'
+    end
+
+    # Force: parse response as json (normally type is determined by correctly
+    # set headers.
+
+    def json
+      @parse_type = :json
+
+      self
+    end
+
+    def json?
+      @parse_type == :json
+    end
+
+    def html
+      @parse_type = :html
+
+      self
+    end
+
+    def html?
+      @parse_type == :html
+    end
+
+    def xml
+      @parse_type = :xml
+
+      self
+    end
+
+    def xml?
+      @parse_type == :xml
+    end
+
+    def inspect
+      if @method
+        "(Request #{@method.to_s.upcase} #{url})"
+      else
+        "(Request #{url})"
+      end
+    end
+
+    def host
+      uri.host
+    end
+
+    def scheme
+      uri.scheme
+    end
+
+    def port
+      uri.port
+    end
+
+    def base_url
+      return @session.base_url if @session.base_url
+
+      if uri.port != AgentX::Session::DEFAULT_PORT[uri.scheme]
+        "#{uri.scheme}://#{uri.host}:#{uri.port}"
+      else
+        "#{uri.scheme}://#{uri.host}"
+      end
+    end
+
+    def full_url
+      @full_url ||= case
+        when url.start_with?('/')
+          "#{@session.base_url}#{url}" 
+        when url.start_with?('http://', 'https://') 
+          url
+        else "#{@session.relative_base_url}#{url}"
+      end
+    end
+
+    def cache_key
+      k = [full_url, method, body, params]
+      Digest::MD5.hexdigest(Oj.dump(k, mode: :compat))
+    end
+
+    def timings
+      ts = {}
+      (@times || {}).each do |k,v|
+        ts[k] = "#{'%.2f' % (v * 1000)}ms"
+      end
+      ts
+    end
+
+    def request_time
+      timings[:http_request]
+    end
+
+    private
+
+    HEADER_MAP = {
+      accept:              'Accept',
+      accept_charset:      'Accept-Charset',
+      accept_encoding:     'Accept-Encoding',
+      accept_language:     'Accept-Language',
+      accept_datetime:     'Accept-Datetime',
+      authorization:       'Authorization',
+      cache_control:       'Cache-Control',
+      connection:          'Connection',
+      cookie:              'Cookie',
+      content_length:      'Content-Length',
+      date:                'Date',
+      expect:              'Expect',
+      from:                'From',
+      host:                'Host',
+      if_match:            'If-Match',
+      if_modified_since:   'If-Modified-Since',
+      if_none_match:       'If-None-Match',
+      if_range:            'If-Range',
+      if_unmodified_since: 'If-Unmodified-Since',
+      max_forwards:        'Max-Forwards',
+      origin:              'Origin',
+      pragma:              'Pragma',
+      proxy_authorization: 'Proxy-Authorization',
+      range:               'Range',
+      referer:             'Referer',
+      referrer:            'Referer',
+      te:                  'TE',
+      user_agent:          'User-Agent',
+      upgrade:             'Upgrade',
+      via:                 'Via',
+      warning:             'Warning'
+    }
+
+    def set_header(k, v)
+      @headers[HEADER_MAP[k] || k.to_s] = v
+    end
+
+    def http(*args)
+      r = nil
+      time(:http_request) do
+        r = untimed_http(*args)
+      end
+      AgentX.logger.info([
+        request_time, 
+        @session.history.last.response.code,
+        method, 
+        full_url].join(' '))
+      r
+    end
+
+    def untimed_http(verb, params=@params, body=@body, headers=@headers)
+      @method = verb.to_s.upcase
+
+      if cookies = HTTP::Cookie.cookie_value(@session.jar.cookies(full_url))
+        set_header(:cookie, cookies)
+      end
+
+      response = nil
+
+      if cacheable? && (response = Cache.read(self))
+        if response.fresh?
+          AgentX.logger.debug("cache fresh")
+        else
+          AgentX.logger.debug("cache validate")
+          response = validate(response)
+        end
+      end
+
+      unless response
+        AgentX.logger.debug("cache miss")
+        response = response_from_easy
+      end
+
+      @session.history.add(self, response)
+      response.cookies.each do |cookie|
+        @session.jar.parse(cookie, full_url)
+      end
+
+      if response.headers.location
+        @session[response.headers.location].get
+      else
+        response.parse(@parse_type)
+      end
+    end
+
+    def validate(response)
+      if response.headers.last_modified
+        set_header(:if_modified_since, response.headers.last_modified)
+      end
+
+      if response.headers.etag
+        set_header(:if_none_match, response.headers.etag)
+      end
+
+      response_from_easy(response)
+    end
+
+    def response_from_easy(response=nil)
+      easy = Ethon::Easy.new
+
+      AgentX.logger.info("easy: #{method} #{full_url}")
+
+      easy.http_request(full_url, method, 
+        params: params, body: body, headers: headers)
+
+      unless easy.perform == :ok
+        raise "Error: #{easy.return_code}"
+      end
+
+      r = Response.from_easy(easy, response)
+
+      Cache.write(self, r) if cacheable? && r.cacheable?
+
+      r
+    end
+
+    def uri
+      @uri ||= URI(full_url)
+    end
+
+    def time(name)
+      @times ||= {}
+      start = Time.now
+      r = yield
+      @times[name] = Time.now - start
+      r
+    end
+
+  end
+end
+

data/lib/agentx/response.rb

@@ -0,0 +1,261 @@
+
+module AgentX
+
+  class Response
+    attr_reader :code, :body, :headers
+
+    def initialize(code, body, headers)
+      @code, @body, @headers = code, body, Headers.parse(headers)
+    end
+
+    def self.from_easy(easy, response=nil)
+      headers = Headers.parse(easy.response_headers)
+
+      r = new(easy.response_code, easy.response_body, headers)
+
+      if response && r.not_modified?
+        r = new(response.code, response.body, response.headers.merge(r.headers))
+      end
+
+      r
+    end
+
+    def cookies
+      Array(headers.set_cookie || [])
+    end
+
+    def ok?
+      code == 200
+    end
+
+    def not_modified?
+      code == 304
+    end
+
+    def fresh?
+      headers.ttl && headers.ttl > 0
+    end
+
+    def expires_at
+      headers.ttl ? Time.now + headers.ttl : Time.at(0)
+    end
+
+    CACHEABLE_CODES = [200, 203, 300, 301, 302, 404, 410]
+
+    def cacheable?
+      return false if headers.cache_control && headers.cache_control.no_store?
+      return false unless CACHEABLE_CODES.include?(code)
+
+      !! (headers.etag || headers.last_modified || fresh?)
+    end
+
+    def inspect
+      "(Response #{code})"
+    end
+
+    def to_hash
+      { 'code'    => code,
+        'body'    => body,
+        'headers' => headers.to_hash }
+    end
+
+    def self.from_hash(h)
+      new(h['code'], h['body'], h['headers'])
+    end
+
+    def parse(type=nil)
+      case
+        when type == :json || headers.json? then Oj.load(body)
+        when type == :html || headers.html? then HTML.new(body)
+        when type == :xml  || headers.xml?  then XML.new(body)
+                                            else body
+      end
+    end
+
+    class Headers
+      include Enumerable
+
+      def initialize(hash={})
+        @hash = hash
+        @hash['Date'] ||= Time.now.httpdate
+        @normalized = {}
+        @hash.each do |k,v|
+          @normalized[k.to_s.downcase] = v
+        end
+      end
+
+      def self.parse(str)
+        return new(str) if str.kind_of?(Hash)
+        return str      if str.kind_of?(Headers)
+
+        hash = {}
+
+        str.lines.each do |line|
+          next if line =~ /^HTTP\/\d/
+          k, v = line.split(':', 2).map { |s| s.strip }
+
+          if k && v
+            if hash[k]
+              hash[k] = Array(hash[k])
+              hash[k] << v
+            else
+              hash[k] = v
+            end
+          end
+        end
+
+        new(hash)
+      end
+
+      def merge(headers)
+        Headers.new(to_hash.merge(headers.to_hash))
+      end
+
+      def inspect
+        "(Headers #{@normalized})"
+      end
+
+      def each(&block)
+        @normalized.each(&block)
+      end
+
+      def [](k)
+        @hash[k] || @normalized[k.to_s.downcase]
+      end
+
+      def server
+        @normalized['server']
+      end
+
+      def date
+        Time.parse(@normalized['date'])
+      end
+
+      def age
+        (@normalized['age'] || (Time.now - date)).to_i
+      end
+
+      def expires
+        if d = @normalized['expires']
+          Time.parse(d)
+        end
+      end
+
+      def max_age
+        (cache_control && 
+         (cache_control.shared_max_age || cache_control.max_age)) ||
+        (expires && (expires - Time.now))
+      end
+
+      def ttl
+        max_age && (max_age - age)
+      end
+
+      def last_modified
+        @normalized['last-modified']
+      end
+
+      def etag
+        @normalized['etag']
+      end
+
+      def content_type
+        @normalized['content-type']
+      end
+
+      def json?
+        content_type.to_s.downcase['json']
+      end
+
+      def html?
+        content_type.to_s.downcase['html']
+      end
+
+      def xml?
+        content_type.to_s.downcase['xml']
+      end
+
+      def content_length
+        if (length = @normalized['content-length']) && length =~ /^\d+$/
+          length.to_i
+        end
+      end
+
+      def cache_control
+        if @normalized['cache-control']
+          @cache_control ||= CacheControl.parse(@normalized['cache-control'])
+        end
+      end
+
+      def set_cookie
+        @normalized['set-cookie']
+      end
+
+      def location
+        @normalized['location']
+      end
+
+      def to_hash
+        @hash
+      end
+
+      class CacheControl
+        attr_reader :directives
+
+        def initialize(directives)
+          @directives = directives
+        end
+
+        def public?
+          @directives['public']
+        end
+
+        def private?
+          @directives['private']
+        end
+
+        def no_store?
+          @directives['no-store']
+        end
+
+        def no_cache?
+          @directives['no-cache']
+        end
+
+        def must_revalidate?
+          @directives['must-revalidate']
+        end
+
+        def max_age
+          @directives['max-age'] && @directives['max-age'].to_i
+        end
+
+        def shared_max_age
+          @directives['s-max-age'] && @directives['s-max-age'].to_i
+        end
+
+        def to_s
+          directives.map { |k,v| v == true ? k : "#{k}=#{v}" }.join(', ')
+        end
+
+        def inspect
+          "(CacheControl #{directives})"
+        end
+
+        def self.parse(s)
+          h = {}
+
+          Array(s).join(',').gsub(' ', '').split(',').each do |p|
+            k, v = p.split('=')
+
+            h[k.downcase] = (v || true)
+          end
+
+          new(h)
+        end
+      end
+    end
+  end
+
+end
+