mingle_events 0.1.3 → 0.1.4

data/lib/mingle_events/feed/page.rb

@@ -14,27 +14,27 @@ module MingleEvents
       end
   
       def entries
-        @entries ||= page_as_document.search('entry').map do |entry_element|
+        @entries ||= page_as_document.select_all('./atom:feed/atom:entry').map do |entry_element|
           Entry.new(entry_element)
         end
       end
   
       def next
-        next_url_element = page_as_document.at("link[@rel='next']")
+        next_url_element = page_as_document.select("./atom:feed/atom:link[@rel='next']")
         if next_url_element.nil?
           nil
         else
-          Page.new(next_url_element["href"], @mingle_access)
+          Page.new(next_url_element.attr("href"), @mingle_access)
         end
       end
   
       private    
   
       def page_as_document
-        @page_as_document ||= Nokogiri::XML(@mingle_access.fetch_page(@url)).remove_namespaces!
+        @page_as_document ||= Xml.parse(@mingle_access.fetch_page(@url), ATOM_AND_MINGLE_NS)
       end
     
     end
     
   end
-end
+end

data/lib/mingle_events/http.rb

@@ -0,0 +1,50 @@
+module MingleEvents
+  module Http
+    extend self
+
+    MAX_RETRY_TIMES = 5
+
+    # get response body for a url, a block can be passed in for request pre-processing
+    def get(url, retry_count=0, &block)
+      rsp = fetch_page_response(url, &block)
+      case rsp
+      when Net::HTTPSuccess
+        rsp.body
+      when Net::HTTPUnauthorized
+        raise HttpError.new(rsp, url, %{
+If you think you are passing correct credentials, please check
+that you have enabled Mingle for basic authentication.
+See <http://www.thoughtworks-studios.com/mingle/3.3/help/configuring_mingle_authentication.html>.})
+      when Net::HTTPBadGateway, Net::HTTPServiceUnavailable, Net::HTTPGatewayTimeOut
+        raise HttpError.new(rsp, url) if retry_count >= MAX_RETRY_TIMES
+        cooldown = retry_count * 2
+        MingleEvents.log.info "Getting service error when get page at #{url}, retry after #{cooldown}s..."
+        sleep cooldown
+        get(url, retry_count + 1, &block)
+      else
+        raise HttpError.new(rsp, url)
+      end
+    end
+
+    private
+    def fetch_page_response(url, &block)
+      uri = URI.parse(url)
+      http = Net::HTTP.new(uri.host, uri.port)
+      path = uri.request_uri
+
+      if uri.scheme == 'https'
+        http.use_ssl = true
+        http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      end
+
+      MingleEvents.log.info "Fetching page at #{path}..."
+
+      start = Time.now
+      req = Net::HTTP::Get.new(path)
+      yield(req) if block_given?
+      rsp = http.request(req)
+      MingleEvents.log.info "...#{path} fetched in #{Time.now - start} seconds."
+      rsp
+    end
+  end
+end

data/lib/mingle_events/mingle_basic_auth_access.rb

@@ -1,70 +1,37 @@
-module MingleEvents 
-  
+module MingleEvents
+
   # Supports fetching of Mingle resources using HTTP basic auth.
   # Please only use this class to access resources over HTTPS so
   # as not to send credentials over plain-text connections.
   class MingleBasicAuthAccess
-    
-    attr_reader :base_url
-        
-    BASIC_AUTH_HTTP_WARNING = %{     
+    BASIC_AUTH_HTTP_WARNING = %{
 WARNING!!!
-It looks like you are using basic authentication over a plain-text HTTP connection. 
+It looks like you are using basic authentication over a plain-text HTTP connection.
 We HIGHLY recommend AGAINST this practice. You should only use basic authentication over
 a secure HTTPS connection. Instructions for enabling HTTPS/SSL in Mingle can be found at
 <http://www.thoughtworks-studios.com/mingle/3.3/help/advanced_mingle_configuration.html>
 WARNING!!
 }
+    attr_reader :base_url
 
-    def initialize(base_url, username, password)
+    def initialize(base_url, username, password, http=Http)
       @base_url = base_url
       @username = username
       @password = password
+      @http = http
     end
-    
-    # Fetch the content at location via HTTP. Throws error if non-200 response.
-    def fetch_page(location) 
-      rsp = fetch_page_response(location)    
-      case rsp
-      when Net::HTTPSuccess
-        rsp.body
-      when Net::HTTPUnauthorized
-        raise HttpError.new(rsp, location, %{
-If you think you are passing correct credentials, please check 
-that you have enabled Mingle for basic authentication. 
-See <http://www.thoughtworks-studios.com/mingle/3.3/help/configuring_mingle_authentication.html>.})
-      else
-        raise HttpError.new(rsp, location) 
+
+    def fetch_page(location)
+      location = @base_url + location if location[0..0] == '/'
+      @http.get(location) do |req|
+        MingleEvents.log.warn(BASIC_AUTH_HTTP_WARNING) if URI.parse(location).scheme == 'http'
+        req['authorization'] = basic_encode(@username, @password)
       end
     end
-    
-    private
-    
-    def fetch_page_response(location)
-      location = @base_url + location if location[0..0] == '/' 
-            
-      uri = URI.parse(location)
-      http = Net::HTTP.new(uri.host, uri.port)
-      
-      if uri.scheme == 'https'
-        http.use_ssl = true
-        http.verify_mode = OpenSSL::SSL::VERIFY_NONE
-      else
-        MingleEvents.log.warn BASIC_AUTH_HTTP_WARNING
-      end
-
-      path = uri.path
-      path += "?#{uri.query}" if uri.query
-      MingleEvents.log.info "Fetching page at #{path}..."
-      
-      start = Time.now
-      req = Net::HTTP::Get.new(path)
-      req.basic_auth(@username, @password)
-      rsp = http.request(req)
-      MingleEvents.log.info "...#{path} fetched in #{Time.now - start} seconds."
 
-      rsp
+    private
+    def basic_encode(account, password)
+      'Basic ' + ["#{account}:#{password}"].pack('m').delete("\r\n")
     end
-
   end
-end
+end

data/lib/mingle_events/mingle_hmac_auth_access.rb

@@ -0,0 +1,22 @@
+module MingleEvents
+  # Client for Mingle's experimental HMAC api auth support
+  class MingleHmacAuthAccess
+    attr_reader :base_url
+
+    def initialize(base_url, login, api_key, http=Http)
+      @base_url = base_url
+      @login = login
+      @api_key = api_key
+      @http = http
+    end
+
+    def fetch_page(location)
+      location  = @base_url + location if location[0..0] == '/'
+      @http.get(location) do |req|
+        ApiAuth.sign!(req, @login, @api_key)
+      end
+    end
+  end
+
+
+end

data/lib/mingle_events/mingle_oauth_access.rb

@@ -1,40 +1,20 @@
-module MingleEvents 
-  
+module MingleEvents
+
   # Client for Mingle's experimental OAuth 2.0 support in 3.0
-  #--
-  # TODO: Update error handling and support of fetching response
-  # objects to that of MingleBasicAuthAccess
   class MingleOauthAccess
+    attr_reader :base_url
 
-    def initialize(base_url, token)
+    def initialize(base_url, token, http=Http)
       @base_url = base_url
       @token = token
+      @http = http
     end
 
     def fetch_page(location)
-      location = @base_url + location if location[0..0] == '/' 
-      
-      uri = URI.parse(location)
-      http = Net::HTTP.new(uri.host, uri.port)
-      http.use_ssl = true
-      http.verify_mode = OpenSSL::SSL::VERIFY_NONE
-      headers = {
-        'Authorization' => %{Token token="#{@token}"}
-      }
-
-      path = uri.path
-      path += "?#{uri.query}" if uri.query
-      MingleEvents.log.info "Fetching page at #{path}..."
-      
-      start = Time.now
-      response = http.get(path, headers)
-      MingleEvents.log.info "... #{path} fetched in #{Time.now - start} seconds."
-
-      # todo: what's the right way to raise on non 200 ?
-      # raise StandardError.new(response.body) unless response == Net::HTTPSuccess
-      
-      response.body
+      location  = @base_url + location if location[0..0] == '/'
+      @http.get(location) do |req|
+        req['Authorization'] = %{Token token="#{@token}"}
+      end
     end
-
   end
-end
+end

data/lib/mingle_events/poller.rb

@@ -5,17 +5,18 @@ module MingleEvents
     # configured for specified mingle projects. processors_by_project_identifier should
     # be a hash where the keys are mingle project identifiers and the values are
     # lists of event processors.
-    def initialize(mingle_access, processors_by_project_identifier)
+    def initialize(mingle_access, processors_by_project_identifier, state_dir=nil)
       @mingle_access = mingle_access
       @processors_by_project_identifier = processors_by_project_identifier
+      @state_dir = state_dir
     end
 
     # Run a single poll for each project configured with processor(s) and 
     # broadcast each event to each processor.
-    def run_once(options={})
+    def run_once
       MingleEvents.log.info("MingleEvents::Poller about to poll once...")
       @processors_by_project_identifier.each do |project_identifier, processors|
-        fetcher = ProjectEventFetcher.new(project_identifier, @mingle_access)
+        fetcher = ProjectEventFetcher.new(project_identifier, @mingle_access, @state_dir)
         fetcher.set_current_state_to_now_if_no_current_state 
         latest_events = fetcher.fetch_latest.to_a
         processors.each{|p| p.process_events(latest_events)}        

data/lib/mingle_events/processors/author_filter.rb

@@ -36,14 +36,14 @@ module MingleEvents
         
         def lookup_author_uri
           team_resource = "/api/v2/projects/#{@project_identifier}/team.xml"
-          @raw_xml ||= @mingle_access.fetch_page(URI.escape(team_resource))
-          @doc ||= Nokogiri::XML(@raw_xml)
+          @raw_xml ||= @mingle_access.fetch_page(URIParser.escape(team_resource))
+          @doc ||= Xml.parse(@raw_xml)
 
-          users = @doc.search('/projects_members/projects_member/user').map do |user|
+          users = @doc.select_all('/projects_members/projects_member/user').map do |user|
             {
-              :url => user.attribute('url').inner_text,
-              :login => user.at('login').inner_text,
-              :email => user.at('email').inner_text
+              :url => user.attr('url'),
+              :login => user.inner_text('login'),
+              :email => user.inner_text('email')
             }
           end
           
@@ -59,4 +59,4 @@ module MingleEvents
     
     end
   end
-end
+end

data/lib/mingle_events/processors/card_data.rb

@@ -52,7 +52,7 @@ module MingleEvents
         # if there's a real problem. In most polling scenarios, this is a highly unlikely
         # problem as there will usually be 1 or a few events.
         begin
-          raw_xml = @mingle_access.fetch_page(URI.escape(path))
+          raw_xml = @mingle_access.fetch_page(URIParser.escape(path))
         rescue
           msg = %{
 
@@ -71,21 +71,21 @@ Stack Trace:
           MingleEvents.log.info(msg)
           return
         end
-        doc = Nokogiri::XML(raw_xml)
+        doc = Xml.parse(raw_xml)
         
-        doc.search('/results/result').map do |card_result|
-          card_number = card_result.at('number').inner_text.to_i
-          card_version = card_result.at('version').inner_text.to_i
+        doc.select_all('/results/result').map do |card_result|
+          card_number = card_result.inner_text('number').to_i
+          card_version = card_result.inner_text('version').to_i
           custom_properties = {}
           @card_data_by_number_and_version[data_key(card_number, card_version)] = {
             :number => card_number,
             :version => card_version,
-            :card_type_name => card_result.at('card_type_name').inner_text,
+            :card_type_name => card_result.inner_text('card_type_name'),
             :custom_properties => custom_properties
           }
           card_result.children.each do |child|
-            if child.name.index("cp_") == 0
-              custom_properties[@custom_properties.property_name_for_column(child.name)] = 
+            if child.tag_name.index("cp_") == 0
+              custom_properties[@custom_properties.property_name_for_column(child.tag_name)] = 
                 nullable_value_from_element(child)
             end
           end
@@ -95,17 +95,17 @@ Stack Trace:
       def load_card_data_for_event(card_event)
         begin
           page_xml = @mingle_access.fetch_page(card_event.card_version_resource_uri)
-          doc = Nokogiri::XML(page_xml)
+          doc = Xml.parse(page_xml)
           custom_properties = {}
           result = {
             :number => card_event.card_number,
             :version => card_event.version,
-            :card_type_name => doc.at('/card/card_type/name').inner_text,
+            :card_type_name => doc.select('/card/card_type/name').inner_text,
             :custom_properties => custom_properties
           }
-          doc.search('/card/properties/property').each do |property|
-            custom_properties[property.at('name').inner_text] = 
-              nullable_value_from_element(property.at('value'))
+          doc.select_all('/card/properties/property').each do |property|
+            custom_properties[property.inner_text('name')] =
+              nullable_value_from_element(property.select('value'))
           end
           
           result
@@ -116,9 +116,9 @@ Stack Trace:
       end
   
       def nullable_value_from_element(element)
-        element['nil'] == 'true' ? nil : element.inner_text
+        element.attr('nil') == 'true' ? nil : element.inner_text
       end
     end
     
   end
-end
+end

data/lib/mingle_events/project_custom_properties.rb

@@ -1,33 +1,33 @@
 module MingleEvents
-  
+
   class ProjectCustomProperties
-    
+
     def initialize(mingle_access, project_identifier)
       @mingle_access = mingle_access
       @project_identifier = project_identifier
     end
-    
+
     def property_name_for_column(column_name)
       property_names_by_column_name[column_name]
     end
-    
+
     private
-    
+
     def property_names_by_column_name
       @property_names_by_column_name ||= lookup_property_names_by_column_name
     end
-    
-    def lookup_property_names_by_column_name      
-      as_document.search('/property_definitions/property_definition').inject({}) do |mapping, element|
-        mapping[element.at('column_name').inner_text] = element.at('name').inner_text        
+
+    def lookup_property_names_by_column_name
+      as_document.select_all('/property_definitions/property_definition').inject({}) do |mapping, element|
+        mapping[element.inner_text('column_name')] = element.inner_text('name')
         mapping
       end
     end
-    
+
     def as_document
-      @as_document ||= Nokogiri::XML(@mingle_access.fetch_page("/api/v2/projects/#{@project_identifier}/property_definitions.xml"))
-    end  
-    
+      @as_document ||= Xml.parse(@mingle_access.fetch_page("/api/v2/projects/#{@project_identifier}/property_definitions.xml"))
+    end
+
   end
-  
-end
+
+end

data/lib/mingle_events/project_event_fetcher.rb

@@ -7,160 +7,71 @@ module MingleEvents
   # one at at time, reading it off disk, to avoid massive memory consumption.
   class ProjectEventFetcher
     
+    attr_reader :entry_cache
+    
     def initialize(project_identifier, mingle_access, state_dir=nil)
       @project_identifier = project_identifier
       @mingle_access = mingle_access
       base_uri = URI.parse(mingle_access.base_url)
-      @state_dir = File.expand_path(state_dir || File.join('~', '.mingle_events', base_uri.host, base_uri.port.to_s, project_identifier, 'fetched_events'))
+      @state_dir = state_dir || File.join('~', '.mingle_events', base_uri.host, base_uri.port.to_s)
+      @state_dir = File.expand_path(File.join(@state_dir, project_identifier, 'fetched_events'))
+      @entry_cache = EntryCache.new(@state_dir)
     end
     
     # blow away any existing state, when next used to fetch events from mingle
     # will crawl all the way back to time zero
     def reset
-      FileUtils.rm_rf(@state_dir)
+      @entry_cache.clear
     end
     
     def set_current_state_to_now_if_no_current_state
-      return if has_current_state?
-      
-      latest_event = page_with_latest_entries.entries.first
-      return if latest_event.nil?
-      write_entry_to_disk(latest_event, nil)
-      update_current_state(latest_event, latest_event)
+      return if @entry_cache.has_current_state?
+      @entry_cache.set_current_state(page_with_latest_entries.entries.first)
     end
     
     # fetch the latest events from mingle, i.e., the ones not previously seen
     def fetch_latest
       page = page_with_latest_entries
       most_recent_new_entry = page.entries.first
-      last_fetched_entry = load_last_fetched_entry
+      last_fetched_entry = @entry_cache.latest
       last_fetched_entry_seen = false      
       next_entry = nil
       while !last_fetched_entry_seen && page
         page.entries.each do |entry|
                     
+          @entry_cache.write(entry, next_entry)
           if last_fetched_entry && entry.entry_id == last_fetched_entry.entry_id
             last_fetched_entry_seen = true
             break
           end
-          
-          write_entry_to_disk(entry, next_entry)
+
           next_entry = entry
-        
         end
         page = page.next
       end
                   
-      update_current_state(next_entry, most_recent_new_entry)
-      Entries.new(file_for_entry(next_entry), file_for_entry(most_recent_new_entry))
+      @entry_cache.update_current_state(next_entry, most_recent_new_entry)
+      @entry_cache.entries(next_entry, most_recent_new_entry)
     end
     
     # returns all previously fetched entries; can be used to reprocess the events for, say,
     # various historical analyses
     def all_fetched_entries
-      current_state = load_current_state
-      Entries.new(current_state[:first_fetched_entry_info_file], current_state[:last_fetched_entry_info_file])
-    end 
-    
-    def first_entry_fetched
-      current_state_entry(:first_fetched_entry_info_file)
+      @entry_cache.all_entries
     end
     
-    def last_entry_fetched
-      current_state_entry(:last_fetched_entry_info_file)
+    def first_entry_fetched      
+      @entry_cache.first
     end
     
-    # only public to facilitate testing
-    def update_current_state(oldest_new_entry, most_recent_new_entry)
-      current_state = load_current_state
-      # if most_recent_new_entry
-        current_state.merge!(:last_fetched_entry_info_file => file_for_entry(most_recent_new_entry))
-        if current_state[:first_fetched_entry_info_file].nil?
-          current_state.merge!(:first_fetched_entry_info_file => file_for_entry(oldest_new_entry))
-        end
-        File.open(current_state_file, 'w'){|out| YAML.dump(current_state, out)}
-      # end
-    end 
-    
-    # only public to facilitate testing
-    def write_entry_to_disk(entry, next_entry)
-      file = file_for_entry(entry)
-      FileUtils.mkdir_p(File.dirname(file))
-      file_content = {:entry_xml => entry.raw_xml, :next_entry_file_path => file_for_entry(next_entry)}
-      File.open(file, 'w'){|out| YAML.dump(file_content, out)}
+    def last_entry_fetched
+      @entry_cache.latest
     end
-           
+                   
     private
     
     def page_with_latest_entries
       Feed::Page.new("/api/v2/projects/#{@project_identifier}/feeds/events.xml", @mingle_access)
-    end
-    
-    def current_state_entry(info_file_key)
-      if info_file = load_current_state[info_file_key]
-        Feed::Entry.from_snippet(YAML.load(File.new(info_file))[:entry_xml])
-      else
-        nil
-      end
-    end
-
-    def file_for_entry(entry)
-      return nil if entry.nil?
-      
-      entry_id_as_uri = URI.parse(entry.entry_id)
-      relative_path_parts = entry_id_as_uri.path.split('/')
-      entry_id_int = relative_path_parts.last
-      insertions = ["#{entry_id_int.to_i/16384}", "#{entry_id_int.to_i%16384}"]
-      relative_path_parts = relative_path_parts[0..-2] + insertions + ["#{entry_id_int}.yml"]  
-      File.join(@state_dir, *relative_path_parts)
-    end
-    
-    def has_current_state?
-      File.exist?(current_state_file)
-    end
-    
-    def current_state_file
-      File.expand_path(File.join(@state_dir, 'current_state.yml'))
-    end
-    
-    def load_last_fetched_entry
-      current_state = load_current_state
-      last_fetched_entry = if current_state[:last_fetched_entry_info_file]
-        last_fetched_entry_info = YAML.load(File.new(current_state[:last_fetched_entry_info_file]))
-        Feed::Entry.from_snippet(last_fetched_entry_info[:entry_xml])
-      else 
-        nil
-      end
-    end
-        
-    def load_current_state
-      if has_current_state?
-        YAML.load(File.new(current_state_file))
-      else
-        {:last_fetched_entry_info_file => nil, :first_fetched_entry_info_file => nil}
-      end
-    end
-    
-    class Entries
-      
-      include Enumerable
-      
-      def initialize(first_info_file, last_info_file)
-        @first_info_file = first_info_file
-        @last_info_file = last_info_file
-      end
-          
-      def each(&block)
-        current_file = @first_info_file
-        while current_file
-          current_entry_info = YAML.load(File.new(current_file))
-          yield(Feed::Entry.from_snippet(current_entry_info[:entry_xml]))
-          break if File.expand_path(current_file) == File.expand_path(@last_info_file)
-          current_file = current_entry_info[:next_entry_file_path]
-        end
-      end
-      
-    end
-          
+    end        
   end
-end
+end