mingle_events 0.0.4 → 0.0.6

data/README.textile

@@ -114,4 +114,21 @@ Each event that is passed to the processor is an instance of type MingleEvents::
 
 h2. Retry & Error handling
 
-As of now, retry is not implemented. If an error occurs during event processing, the error will be logged and processing will stop. The next run will re-start at the point of the last error.
+As of now, retry is not implemented. If an error occurs during event processing, the error will be logged and processing will stop. The next run will re-start at the point of the last error.
+
+h2. Historical analysis
+
+One of the great usages of event playback is historical analysis. E.g., if you wanted to see the number of stories added each day to your project, you could write a processor that counts new stories and takes a snapshot at the end of each day, where the end of the day is the first time you see an event for the next day. If you are doing this sort of analysis you will not want to poll the entire project's event history every time as that is quite time consuming. 
+
+It is possible to repeatedly run analysis against previously fetched events using the ProjectEventFetcher class that is only used in the internals of the standard polling mechanism.  ProjectEventFetcher does most of the real work of reading new events from the server and making them available for local processing, so it's not a bad class to get to know.
+
+<pre>
+event_fetcher = MingleEvents::ProjectEventFetcher.new('mingle', mingle_access)
+processor = MingleEvents::Processors::PutsPublisher.new
+event_fetcher.all_fetched_entries.each do |e|
+  processor.process_events([e])
+end  
+</pre>
+
+That's not an ideal interface.  It's a bit awkward to push a single event into an array for processing.  We'll likely look to make this much better as we bring back batch processing of events.
+

data/lib/mingle_events/feed/entry.rb

@@ -72,6 +72,20 @@ module MingleEvents
       def to_s
         "Entry[entry_id=#{entry_id}, updated=#{updated}]"
       end
+      
+      def eql?(object)
+        if object.equal?(self)
+         return true
+        elsif !self.class.equal?(object.class)
+         return false
+        end
+      
+        return object.entry_id == entry_id
+      end
+      
+      def ==(object)
+        eql?(object)
+      end
   
       private
   

data/lib/mingle_events/mingle_basic_auth_access.rb

@@ -24,7 +24,6 @@ WARNING!!
     
     # Fetch the content at location via HTTP. Throws error if non-200 response.
     def fetch_page(location) 
-      MingleEvents.log.info("About to fetch #{location}...")
       rsp = fetch_page_response(location)    
       case rsp
       when Net::HTTPSuccess
@@ -51,18 +50,18 @@ See <http://www.thoughtworks-studios.com/mingle/3.3/help/configuring_mingle_auth
         http.use_ssl = true
         http.verify_mode = OpenSSL::SSL::VERIFY_NONE
       else
-        puts BASIC_AUTH_HTTP_WARNING
+        MingleEvents.log.warn BASIC_AUTH_HTTP_WARNING
       end
 
       path = uri.path
       path += "?#{uri.query}" if uri.query
-      puts "Fetching page at #{path}..."
+      MingleEvents.log.info "Fetching page at #{path}..."
       
       start = Time.now
       req = Net::HTTP::Get.new(path)
       req.basic_auth(@username, @password)
       rsp = http.request(req)
-      puts "...#{Time.now - start}"
+      MingleEvents.log.info "...#{path} fetched in #{Time.now - start} seconds."
 
       rsp
     end

data/lib/mingle_events/mingle_oauth_access.rb

@@ -24,11 +24,11 @@ module MingleEvents
 
       path = uri.path
       path += "?#{uri.query}" if uri.query
-      puts "Fetching page at #{path}..."
+      MingleEvents.log.info "Fetching page at #{path}..."
       
       start = Time.now
       response = http.get(path, headers)
-      puts "...#{Time.now - start}"
+      MingleEvents.log.info "... #{path} fetched in #{Time.now - start} seconds."
 
       # todo: what's the right way to raise on non 200 ?
       # raise StandardError.new(response.body) unless response == Net::HTTPSuccess

data/lib/mingle_events/poller.rb

@@ -17,13 +17,9 @@ module MingleEvents
       @processors_by_project_identifier.each do |project_identifier, processors|
         fetcher = ProjectEventFetcher.new(project_identifier, @mingle_access)
         fetcher.reset if options[:clean]
-        info_file_for_new_event = fetcher.fetch_latest 
-        while info_file_for_new_event  
-          entry_info = YAML.load(File.new(info_file_for_new_event))
-          entry = Feed::Entry.new(Nokogiri::XML(entry_info[:entry_xml]).at('/entry'))
+        fetcher.fetch_latest.each do |entry|
           MingleEvents.log.info("About to process event #{entry.entry_id}...")
           processors.each{|p| p.process_events([entry])}
-          info_file_for_new_event = entry_info[:next_entry_file_path]
         end
       end
     end

data/lib/mingle_events/project_event_fetcher.rb

@@ -10,22 +10,24 @@ module MingleEvents
       @state_dir = File.expand_path(state_dir || File.join('~', '.mingle_events', base_uri.host, base_uri.port.to_s, project_identifier, 'fetched_events'))
     end
     
+    # blow away any existing state, when next used to fetch events from mingle
+    # will crawl all the way back to time zero
     def reset
       FileUtils.rm_rf(@state_dir)
     end
     
+    # fetch the latest events from mingle, i.e., the ones not previously seen
     def fetch_latest
       page = Feed::Page.new("/api/v2/projects/#{@project_identifier}/feeds/events.xml", @mingle_access)
       most_recent_new_entry = page.entries.first
-      
       last_fetched_entry = load_last_fetched_entry
-      fetched_previously_seen_event = false      
+      last_fetched_entry_seen = false      
       next_entry = nil
-      while !fetched_previously_seen_event && page
+      while !last_fetched_entry_seen && page
         page.entries.each do |entry|
                     
           if last_fetched_entry && entry.entry_id == last_fetched_entry.entry_id
-            fetched_previously_seen_event = true
+            last_fetched_entry_seen = true
             break
           end
           
@@ -35,12 +37,56 @@ module MingleEvents
         end
         page = page.next
       end
-            
-      update_current_state(most_recent_new_entry)
-      
-      file_for_entry(next_entry)
+                  
+      update_current_state(next_entry, most_recent_new_entry)
+      Entries.new(file_for_entry(next_entry), file_for_entry(most_recent_new_entry))
+    end
+    
+    # returns all previously fetched entries; can be used to reprocess the events for, say,
+    # various historical analyses
+    def all_fetched_entries
+      current_state = load_current_state
+      Entries.new(current_state[:first_fetched_entry_info_file], current_state[:last_fetched_entry_info_file])
+    end 
+    
+    def first_entry_fetched
+      current_state_entry(:first_fetched_entry_info_file)
+    end
+    
+    def last_entry_fetched
+      current_state_entry(:last_fetched_entry_info_file)
+    end
+    
+    # only public to facilitate testing
+    def update_current_state(oldest_new_entry, most_recent_new_entry)
+      current_state = load_current_state
+      if most_recent_new_entry
+        current_state.merge!(:last_fetched_entry_info_file => file_for_entry(most_recent_new_entry))
+        if current_state[:first_fetched_entry_info_file].nil?
+          current_state.merge!(:first_fetched_entry_info_file => file_for_entry(oldest_new_entry))
+        end
+        File.open(current_state_file, 'w'){|out| YAML.dump(current_state, out)}
+      end
+    end 
+    
+    # only public to facilitate testing
+    def write_entry_to_disk(entry, next_entry)
+      file = file_for_entry(entry)
+      FileUtils.mkdir_p(File.dirname(file))
+      file_content = {:entry_xml => entry.raw_xml, :next_entry_file_path => file_for_entry(next_entry)}
+      File.open(file, 'w'){|out| YAML.dump(file_content, out)}
     end
+           
+    private
     
+    def current_state_entry(info_file_key)
+      if info_file = load_current_state[info_file_key]
+        entry_for_xml(YAML.load(File.new(info_file))[:entry_xml])
+      else
+        nil
+      end
+    end
+
     def file_for_entry(entry)
       return nil if entry.nil?
       
@@ -55,15 +101,9 @@ module MingleEvents
     def current_state_file
       File.expand_path(File.join(@state_dir, 'current_state.yml'))
     end
-        
-    private 
     
     def load_last_fetched_entry
-      current_state = if File.exist?(current_state_file)
-        YAML.load(File.new(current_state_file))
-      else
-        {:last_fetched_entry_info_file => nil}
-      end
+      current_state = load_current_state
       last_fetched_entry = if current_state[:last_fetched_entry_info_file]
         last_fetched_entry_info = YAML.load(File.new(current_state[:last_fetched_entry_info_file]))
         Feed::Entry.new(Nokogiri::XML(last_fetched_entry_info[:entry_xml]).at('/entry'))
@@ -71,26 +111,43 @@ module MingleEvents
         nil
       end
     end
-    
-    def update_current_state(most_recent_new_entry)
-      if most_recent_new_entry
-        File.open(current_state_file, 'w') do |out|
-          YAML.dump({:last_fetched_entry_info_file => file_for_entry(most_recent_new_entry)}, out)
-        end
-      end
-    end
         
-    def write_entry_to_disk(entry, next_entry)
-      file = file_for_entry(entry)
-      FileUtils.mkdir_p(File.dirname(file))
-      file_content = {
-        :entry_xml => entry.raw_xml,
-        :next_entry_file_path => file_for_entry(next_entry)
-      }
-      File.open(file, 'w') do |out|
-        YAML.dump(file_content, out)
+    def load_current_state
+      if File.exist?(current_state_file)
+        YAML.load(File.new(current_state_file))
+      else
+        {:last_fetched_entry_info_file => nil, :first_fetched_entry_info_file => nil}
       end
     end
     
+    def entry_for_xml(entry_xml)
+      Feed::Entry.new(Nokogiri::XML(entry_xml).at('/entry'))
+    end
+    
+    class Entries
+      
+      include Enumerable
+      
+      def initialize(first_info_file, last_info_file)
+        @first_info_file = first_info_file
+        @last_info_file = last_info_file
+      end
+      
+      def entry_for_xml(entry_xml)
+        Feed::Entry.new(Nokogiri::XML(entry_xml).at('/entry'))
+      end
+      
+      def each(&block)
+        current_file = @first_info_file
+        while current_file
+          current_entry_info = YAML.load(File.new(current_file))
+          yield(entry_for_xml(current_entry_info[:entry_xml]))
+          break if File.expand_path(current_file) == File.expand_path(@last_info_file)
+          current_file = current_entry_info[:next_entry_file_path]
+        end
+      end
+      
+    end
+          
   end
 end

data/test/mingle_events/feed/entry_test.rb

@@ -134,6 +134,34 @@ module MingleEvents
         assert_equal(entry.entry_id, entry.event_id)
       end
       
+      def test_entry_id_determines_equality
+        element_xml_text_1 = %{
+          <entry xmlns:mingle="http://www.thoughtworks-studios.com/ns/mingle">
+            <id>https://mingle.example.com/projects/mingle/events/index/234443</id>
+            <category term="page" scheme="http://www.thoughtworks-studios.com/ns/mingle#categories"/>
+          </entry>}
+        entry_1 = Entry.new(Nokogiri::XML(element_xml_text_1))
+        
+        element_xml_text_2 = %{
+          <entry xmlns:mingle="http://www.thoughtworks-studios.com/ns/mingle">
+            <id>https://mingle.example.com/projects/mingle/events/index/234443</id>
+            <category term="card" scheme="http://www.thoughtworks-studios.com/ns/mingle#categories"/>
+          </entry>}
+        entry_2 = Entry.new(Nokogiri::XML(element_xml_text_2))
+        
+        element_xml_text_3 = %{
+          <entry xmlns:mingle="http://www.thoughtworks-studios.com/ns/mingle">
+            <id>https://mingle.example.com/projects/mingle/events/index/234</id>
+            <category term="card" scheme="http://www.thoughtworks-studios.com/ns/mingle#categories"/>
+          </entry>}
+        entry_3 = Entry.new(Nokogiri::XML(element_xml_text_3))
+        
+        assert entry_1.eql?(entry_2)
+        assert entry_1 == entry_2
+        assert !entry_2.eql?(entry_3)
+        assert entry_2 != entry_3
+      end
+      
     end
   
   end

data/test/mingle_events/project_event_fetcher_test.rb

@@ -7,60 +7,34 @@ module MingleEvents
       state_dir = temp_dir
       fetcher = ProjectEventFetcher.new('atlas', stub_mingle_access, state_dir)  
       
-      file_for_first_new_entry = fetcher.fetch_latest
-      
-      expected_entry_ids = [
-        'https://mingle.example.com/projects/atlas/events/index/23',
-        'https://mingle.example.com/projects/atlas/events/index/97',
-        'https://mingle.example.com/projects/atlas/events/index/98',
-        'https://mingle.example.com/projects/atlas/events/index/99',
-        'https://mingle.example.com/projects/atlas/events/index/100',
-        'https://mingle.example.com/projects/atlas/events/index/101',
-        'https://mingle.example.com/projects/atlas/events/index/103'
-      ]
-      
-      assert_expected_entry_chain_written_to_disk(expected_entry_ids, file_for_first_new_entry, fetcher)      
+      latest_entries = fetcher.fetch_latest
+      expected_latest_entries = [23, 97, 98, 99, 100, 101, 103].map{|n| entry(n)}
+      assert_equal(expected_latest_entries, latest_entries.to_a)
+      assert_equal entry(23), fetcher.first_entry_fetched
+      assert_equal entry(103), fetcher.last_entry_fetched
     end
     
     def test_can_fetch_all_entries_and_write_to_disk_when_existing_state
       state_dir = temp_dir
       fetcher = ProjectEventFetcher.new('atlas', stub_mingle_access, state_dir)
-      setup_current_state(%{
-<entry>
-  <id>https://mingle.example.com/projects/atlas/events/index/97</id>
-  <title>entry 97</title>
-  <updated>2011-02-03T01:10:52Z</updated>
-  <author><name>Harry</name></author>
-</entry>}, fetcher)
       
-      file_for_first_new_entry = fetcher.fetch_latest
+      setup_current_state(23, 97, 97, fetcher)
       
-      expected_entry_ids = [
-        'https://mingle.example.com/projects/atlas/events/index/98',
-        'https://mingle.example.com/projects/atlas/events/index/99',
-        'https://mingle.example.com/projects/atlas/events/index/100',
-        'https://mingle.example.com/projects/atlas/events/index/101',
-        'https://mingle.example.com/projects/atlas/events/index/103'
-      ]
-      
-      assert_expected_entry_chain_written_to_disk(expected_entry_ids, file_for_first_new_entry, fetcher)      
+      latest_entries = fetcher.fetch_latest
+      expected_latest_entries = [98, 99, 100, 101, 103].map{|n| entry(n)}
+      assert_equal(expected_latest_entries, latest_entries.to_a)
+      assert_equal entry(23), fetcher.first_entry_fetched
+      assert_equal entry(103), fetcher.last_entry_fetched
     end
     
     def test_no_new_entries_with_current_state
       state_dir = temp_dir
       fetcher = ProjectEventFetcher.new('atlas', stub_mingle_access, state_dir)
-      setup_current_state(%{
-<entry>
-  <id>https://mingle.example.com/projects/atlas/events/index/103</id>
-  <title>entry 103</title>
-  <updated>2011-02-03T01:10:52Z</updated>
-  <author><name>Harry</name></author>
-</entry>}, fetcher)
+      setup_current_state(23, 97, 103, fetcher)
 
-      assert_nil fetcher.fetch_latest
-      
-      current_state = YAML.load(File.new(File.join(state_dir, 'current_state.yml')))
-      assert_equal(@last_fetched_entry_info_file, current_state[:last_fetched_entry_info_file])
+      assert fetcher.fetch_latest.to_a.empty?
+      assert_equal entry(23), fetcher.first_entry_fetched
+      assert_equal entry(103), fetcher.last_entry_fetched
     end
     
     def test_no_new_entries_with_no_current_state
@@ -77,46 +51,32 @@ module MingleEvents
 </feed>})
       fetcher = ProjectEventFetcher.new('atlas', mingle_access, state_dir)
       
-      assert_nil fetcher.fetch_latest
-      assert !File.exist?(File.join(state_dir, 'current_state.yml'))
+      assert fetcher.fetch_latest.to_a.empty?
+      assert_nil fetcher.first_entry_fetched
+      assert_nil fetcher.last_entry_fetched
     end
     
     private
     
-    def setup_current_state(last_fetched_entry_xml, fetcher)
-      last_fetched_entry_info = {
-        :entry_xml => last_fetched_entry_xml,
-        :next_entry_file_path => nil
-      }
-      @last_fetched_entry_info_file = fetcher.file_for_entry(Feed::Entry.new(Nokogiri::XML(last_fetched_entry_xml).at('/entry')))
-      FileUtils.mkdir_p(File.dirname(@last_fetched_entry_info_file))
-      File.open(@last_fetched_entry_info_file, 'w') do |out|
-        YAML.dump(last_fetched_entry_info, out)
-      end
-      current_state = {:last_fetched_entry_info_file => @last_fetched_entry_info_file}
-      File.open(fetcher.current_state_file, 'w') do |out|
-        YAML.dump(current_state, out)
-      end
-
+    def setup_current_state(first_entry_id, second_entry_id, last_entry_id, fetcher)    
+      first_entry = entry(first_entry_id)
+      second_entry = entry(second_entry_id)
+      last_entry = entry(last_entry_id)
+      fetcher.write_entry_to_disk(first_entry, second_entry)
+      fetcher.write_entry_to_disk(last_entry, nil)  
+      fetcher.update_current_state(first_entry, last_entry)
     end
-    
-    def assert_expected_entry_chain_written_to_disk(expected_entry_ids, file_for_first_new_entry, fetcher)
-      
-      file_for_new_entry = file_for_first_new_entry
-      file_for_last_inspected_entry = nil
-      expected_entry_ids.each_with_index do |expected_entry_id, index|
-        file_for_last_inspected_entry = file_for_new_entry
-        new_entry_info = YAML.load(File.new(file_for_new_entry))
-        new_entry = Feed::Entry.new(Nokogiri::XML(new_entry_info[:entry_xml]).at('/entry'))
-        assert_equal(expected_entry_id, new_entry.entry_id)
-        file_for_new_entry = new_entry_info[:next_entry_file_path]
-      end
-      
-      current_state = YAML.load(File.new(fetcher.current_state_file))
-      assert_equal(file_for_last_inspected_entry, current_state[:last_fetched_entry_info_file])
-      
+            
+    def entry(entry_id)
+      entry_xml = %{
+        <entry>
+          <id>https://mingle.example.com/projects/atlas/events/index/#{entry_id}</id>
+          <title>entry #{entry_id}</title>
+          <updated>2011-02-03T01:10:52Z</updated>
+          <author><name>Bob</name></author>
+        </entry>
+      }
+      Feed::Entry.new(Nokogiri::XML(entry_xml).at('/entry'))
     end
-    
-      
   end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: mingle_events
 version: !ruby/object:Gem::Version 
-  hash: 23
+  hash: 19
   prerelease: 
   segments: 
   - 0
   - 0
-  - 4
-  version: 0.0.4
+  - 6
+  version: 0.0.6
 platform: ruby
 authors: 
 - David Rice