stats_combiner 0.0.1

data/.gitignore

@@ -0,0 +1,8 @@
+tpm_combiner.rb
+*.DS_Store
+*stats_db.sqlite3
+spec/spec_credentials.rb
+rewrites.rb
+test/*
+pkg/*
+*.gem

data/README.md

@@ -0,0 +1,85 @@
+# StatsCombiner
+
+StatsCombiner is a Ruby gem for generating most-viewed widgets from the [Chartbeat API](http://chartbeat.pbworks.com/). Unlike most analytics systems, Chartbeat doesn't give you cumulative visitor counts. Rather, it takes live snapshots of people sitting on pages at a given time. StatsCombiner asks Chartbeat what these numbers are every `n` minutes during a given `ttl` and combines visitor counts where it finds matching `<title>`s, to allow popular stories to bubble up the list. When `ttl` expires, it will publish out a static HTML file with your top ten list to a location of your choosing, trash its database and start collecting again.
+
+This gem is a rewrite from the PHP implementation I wrote about [here](http://blog.chartbeat.com/2009/08/04/guest-post-how-talking-points-memo-uses-chartbeat/).
+
+## Installation
+
+`gem install stats_combiner`
+
+## Usage
+
+Write a short combiner script that tells StatsCombiner your Chartbeat API parameters, how long it should combine for and where to put the flat file. Add filters to manipulate the data it will publish. 
+
+Here's an example:
+
+    require 'rubygems'
+    require 'stats_combiner'
+    
+    # initialize the script
+    s = StatsCombiner::Combiner.new({
+      :ttl => 3600, #1 hour from first run
+      :host => 'yourdomain.com',
+      :api_key => 'YOURKEY',
+      :flat_file => '/path/to/staticfile/top_ten.html'
+    })
+        
+    # create a filters object and add some filters
+    e = StatsCombiner::Filterer.new
+    e.add :path_regex => /(\/$|\/index.php$)/, :exclude => true
+    
+    # run it!
+    s.run({
+      :filters => e.filters,
+      :verbose => true  #this option reports combining status and TTL when true
+     })
+
+Then add this script to your crontab. I recommend running it every 5 minutes. Just be a good API-consumer when setting your cron:
+
+    */5 * * * *       cd /path/to/my_combiner && ruby my_combiner.rb
+
+### Filters
+
+    http://{prefix}.{host}/{path}{suffix}
+
+search by..
+
+    :title_regex => regexp                # Filter on a title pattern
+    :path_regex=> regexp                  # Filter on a path pattern
+
+..to add a:
+
+    :suffix => string or regexp           # a path modification
+    :prefix => string                     # a subdomain
+    :modify_title => bool or regexp       # Modify the title inline 
+                                          # (true replaces title match with '')
+
+..or, to ignore the entry:
+
+    :exclude => bool                      # Exclude matching URLs from the top ten list
+    
+Some examples from TPM:
+     
+    e.add :prefix => 'tpmdc', :title_regex => /\| TPMDC/, :modify_title => true
+    e.add :path_regex => /((\?|&)ref=.*)/, :suffix => ''
+    e.add :path_regex => /(\?(page|img)=(.*)($|&))/, :suffix => '?\2=1'
+    e.add :path_regex => /(\/$|\/index.php$)/, :exclude => true
+
+### A note on testing
+
+You may need to `sudo spec` if your user doesn't have write access to the gems directory.
+
+## Author
+
+Al Shaw (al@talkingpointsmemo.com)
+
+## License (MIT)
+
+Copyright (c) 2010 TPM Media LLC
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,25 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "stats_combiner"
+    gem.summary = %Q{StatsCombiner creates most-viewed story widgets from the Chartbeat API}
+    gem.description = %Q{A tool to create most-viewed story widgets from the Chartbeat API.}
+    gem.email = "almshaw@gmail.com"
+    gem.homepage = "http://github.com/tpm/stats_combiner"
+    gem.authors = ["Al Shaw"]
+    gem.add_dependency 'crack'
+    gem.add_dependency 'sequel'
+    gem.add_development_dependency "rspec"
+    gem.add_development_dependency "fakeweb"
+    gem.add_development_dependency "timecop"
+    gem.add_development_dependency "hpricot"
+
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
+end

data/VERSION

@@ -0,0 +1 @@
+0.0.1

data/combiner_sample.rb

@@ -0,0 +1,26 @@
+require 'rubygems'
+require 'stats_combiner'
+
+s = StatsCombiner::Combiner.new({
+	:ttl => 3600, 
+	:host => 'yourhost.com',
+	:api_key => 'YOURKEY',
+	:flat_file => '/path/to/top_ten.html'
+})
+
+e = StatsCombiner::Filterer.new
+e.add :prefix => 'tpmdc', :title_regex => /\| TPMDC/, :modify_title => true
+e.add :prefix => 'tpmmuckraker', :title_regex => /\| TPMMuckraker/, :modify_title => true
+e.add :prefix => 'tpmtv', :title_regex => /\| TPMTV/, :modify_title => true
+e.add :prefix => 'tpmcafe', :title_regex => /\| TPMCafe/, :modify_title => true
+e.add :prefix => 'tpmlivewire', :title_regex => /\| TPM LiveWire/, :modify_title => true
+e.add :prefix => 'tpmpolltracker', :title_regex => /\| TPM PollTracker/, :modify_title => true
+e.add :prefix => 'www', :title_regex => /\|.*$/, :modify_title => true
+
+#put excluders last
+e.add :path_regex => /(\/$|\/index.php$)/, :exclude => true
+
+s.run({
+	:filters => e.filters,
+	:verbose => true
+})

data/lib/stats_combiner.rb

@@ -0,0 +1,182 @@
+require 'open-uri'
+require 'fileutils'
+require 'crack/json'
+require 'sequel'
+require 'stats_combiner/filterer'
+
+module StatsCombiner
+	
+  class Combiner
+
+    # Usage:
+    #
+    #  s = StatsCombiner::Combiner.new({
+    #       :api_key => 'your_key',
+    #       :host => 'talkingpointsmemo.com',
+    #       :ttl => 3600,
+    #       :flat_file => '/var/www/html/topten.html'
+    #     })  
+    def initialize(opts = {})
+      @init_options = {
+        :ttl => 3600, #one hour by default
+        :story_count => 10,
+        :flat_file => 'topten.html',
+      }.merge!(opts)
+      @db_file = "#{@init_options[:host]}_stats_db.sqlite3"
+    end
+
+    # check where we are in the cycle
+    # and run the necessary functions
+    # call this after initializing StatsCombiner
+    # 
+    # Usage:
+    #  s.run({
+    #   :filters => e.filters (result of a StatsCombiner::Filterer filters hash)
+    #   :verbose => true
+    #  })
+    def run(opts = {})
+      { :filters => nil,
+        :verbose => false
+      }.merge!(opts)
+      
+      @filters = opts[:filters]
+      
+      now = Time.now
+      if File::exists?(@db_file)
+        @db = Sequel.sqlite(@db_file)
+        @table_create_time = self.table_create_time
+        @table_destroy_time = @table_create_time + @init_options[:ttl]
+        @ttl = @table_destroy_time.to_i - Time.now.to_i
+        
+        if now.to_i < @table_destroy_time.to_i 
+         self.combine
+         if opts[:verbose]
+          puts "Combining. DB has #{@ttl} seconds to live"
+         end
+        else
+         self.report_and_cleanup
+         if opts[:verbose]
+          puts "ttl expired. reporting and cleaning up"
+          end
+        end
+      else
+        self.setup
+        if opts[:verbose]
+          puts "No DB detected. I set one up"
+        end
+      end
+    end
+    
+  
+  protected
+  
+    # Set up the database.
+    # This is done once every timeout cycle
+    def setup
+      @db = Sequel.sqlite(@db_file)
+      @db.create_table :stories do
+        primary_key    :id,       :type => Integer
+        String         :title,    :text => true
+        String         :path,     :text => true
+        Fixnum         :visitors
+        DateTime       :created_at
+      end
+      
+      @db.create_table :create_time do
+        primary_key    :id,       :type => Integer
+        DateTime       :timestamp
+      end
+      
+      now = Time.now
+      time_table = @db[:create_time]
+      time_table.insert(:timestamp => now)
+    end
+    
+    def table_create_time
+      @db[:create_time].select(:timestamp).first[:timestamp]
+    end
+    
+    def destroy
+      FileUtils.rm_rf(@db_file)
+    end
+    
+    # grab the data, and parse it into something we can use
+    # and combine it
+    def combine
+      host = @init_options[:host]
+      api_key = @init_options[:api_key]
+      @url = "http://api.chartbeat.com/toppages/?host=#{host}&limit=50&apikey=#{api_key}"
+      
+      @data = open(@url).read
+      @data = Crack::JSON.parse(@data)
+    
+      @data.each do |datum|
+        visitors = datum['visitors']
+        path = datum['path']
+        title = datum['i']
+        
+        # if the story is already in the db, combine visitor count
+        # otherwise insert a new row
+        existing_story = @db[:stories].where(:title => title).first || ''
+        
+        if not existing_story.empty?
+          existing_visitors = existing_story[:visitors]
+          @db[:stories].where(:title => title).update :visitors => existing_visitors + visitors
+        else
+          @db[:stories].insert({
+            :title => title,
+            :visitors => visitors,
+            :path => path
+           })
+        end
+        
+      end
+    end
+    
+    # Pull data out of the db, apply filters and write the flat file and dump the db.
+    # This is done once every timeout cycle.
+    def report_and_cleanup
+      stories = @db[:stories].order(:visitors.desc).all
+
+     #filter the array if applicable, then narrow down to top ten
+     if @filters
+        stories.each do |story|         
+          StatsCombiner::Filterer.apply_filters!(@filters,story)
+        end       
+     end
+     
+     #sweep away the excludes
+     stories.reject! {|story| story if story[:title].nil? || story[:path].nil? }
+     
+     top_ten = stories[0..9]
+     now = Time.now
+     flat_file = @init_options[:flat_file]
+     host = @init_options[:host]
+     
+     #write it out
+     html = '<ol>'
+  
+     top_ten.each do |story|
+       title = story[:title]
+       path = story[:path]
+       visitors = story[:visitors]
+       
+       if not story[:prefix].nil?
+        prefix = story[:prefix] + '.'
+       end
+       
+       html << "<li><a href=\"http://#{prefix}#{host}#{path}\">#{title}</a></li> <!-- #{visitors} -->"
+     end
+      
+     html << '</ol>'
+     html << "<!-- This report was generated at #{now} -->"
+    
+     flat_file = File.new(flat_file, "w+")
+     flat_file.write(html)
+     flat_file.close
+     
+     #Destroy the DB and start the journey over again.
+     self.destroy
+    end
+  end
+end

data/lib/stats_combiner/filterer.rb

@@ -0,0 +1,111 @@
+module StatsCombiner
+
+  class Filterer 
+    
+    attr_accessor :filters
+    
+    # Initialize a filters object:
+    #  e = StatsCombiner::Filterer.new
+    #
+    def initialize()
+      @filters ||= []
+    end 
+    
+    
+    # Add a filter that StatsCombiner can use to manipulate paths and titles it
+    # gets from Chartbeat.
+    # 
+    # Options:                            Pattern: <tt>http://{prefix}.{host}/{path}{suffix}</tt>
+    #
+    # search by..
+    #  title_regex => nil                 Filter on a title pattern
+    #  path_regex=> nil                   Filter on a path pattern
+    #
+    # ..to add a:
+    #  suffix => nil                      a path modification
+    #  prefix => nil                      a subdomain
+    #  modify_title => bool or regexp     Modify the title inline
+    #
+    # Or, to ignore the entry:
+    #  exclude => true                    Exclude this pattern from the top ten list
+    # 
+    # Some examples from TPM:
+    #  e.add :prefix => 'tpmdc', :title_regex => /\| TPMDC/, :modify_title => true
+    #  e.add :path_regex => /(\?ref=.*$|\&ref=.*$|)/, :suffix => '', :modify_path => true
+    #  e.add :path_regex => /(\?(page|img)=(.*)($|&))/, :suffix => '?\2=1'
+    #  e.add :path_regex => /(\/$|\/index.php$)/, :exclude => true
+    def add(options={})   
+      { :prefix => nil,
+        :suffix => nil,
+        :title_regex => nil,
+        :path_regex => nil,
+        :modify_title => false, 
+        :exclude => false,
+      }.merge!(options)
+      
+      filter = {}
+      filter[:rule] = {}.merge!(options)
+            
+      @filters << filter
+    end
+  
+    # sanity check
+    def list_filters
+      @filters.each do |filter|
+        p filter[:rule]
+      end
+    end
+    
+    # a datum comes in from chartbeat data, and is manipulated
+    # with the apply_filters method, and sent back to Combiner to write out
+    #
+    # Grab filters with <tt>e.filters</tt>
+    def self.apply_filters!(filters,datum={})      
+      { :title => nil,
+        :path => nil,
+        :prefix => nil
+      }.merge!(datum)
+      
+      filters.each do |filter|
+      
+        # set prefixes where they match title regexes
+        # /\| TPMDC/ => http://tpmdc
+        if (filter[:rule][:prefix] && filter[:rule][:title_regex]) && datum[:title].match(filter[:rule][:title_regex])
+            datum[:prefix] = filter[:rule][:prefix]
+        end
+        
+        # modify path => '?q=new_suffix'
+        # append to path with regex replacement variables => '\1&new_suffix'
+        if (filter[:rule][:suffix] && filter[:rule][:path_regex]) && datum[:path].match(filter[:rule][:path_regex])
+              datum[:path].gsub!(filter[:rule][:path_regex],filter[:rule][:suffix])
+        end
+        
+        # apply title mods
+        # modify_title => true /==> 
+        # modify_title => "DC Central"
+        # title_regex => /| TPMDC/,  modify_title => '\1 Central' ==> TPMDC Central
+        if filter[:rule][:modify_title]
+          filter[:rule][:modify_title] = '' unless filter[:rule][:modify_title].is_a?(String)
+          datum[:title].gsub!(filter[:rule][:title_regex], filter[:rule][:modify_title])
+          datum[:title].strip!
+        end
+        
+        # apply excludes. 
+        # this should take out the whole record if it matches a path or title regex
+        if filter[:rule][:exclude] && ((filter[:rule][:path_regex].is_a?(Regexp) && datum[:path].match(filter[:rule][:path_regex])) || (filter[:rule][:title_regex].is_a?(Regexp) && datum[:title].match(filter[:rule][:title_regex])))        
+            # nil out datum.
+            # StatsCombiner::Combiner will sweep away the nils later
+            datum[:title] = datum[:path] = datum[:prefix] = nil
+        end
+      
+      end
+    
+      datum
+     
+    end
+  
+  
+  end
+
+end
+

data/spec/stats_combiner_spec.rb

@@ -0,0 +1,294 @@
+require '../lib/stats_combiner'
+require 'test_data'
+
+require 'spec'
+require 'timecop'
+require 'hpricot'
+
+HOST = 'fake.com'
+KEY = 'fake_key'
+
+describe "an unfiltered StatsCombiner cycle" do
+
+  before :each do
+    @flat_file = File.dirname(__FILE__) + '/test_flat_file.html'
+    @ttl = 3600
+    @s = StatsCombiner::Combiner.new({
+      :ttl => @ttl, 
+      :host=> HOST,
+      :api_key=> KEY,
+      :flat_file => @flat_file,
+    })
+    @db_file = "#{HOST}_stats_db.sqlite3"
+  end
+  
+  it 'should do a first-time run, setting up the db' do
+    @s.run()
+    
+    
+    File.exist?(@db_file).should == true
+    
+    @db = Sequel.sqlite(@db_file)
+    @db[:stories].all.should be_a(Array)
+    @db[:create_time].all.should be_a(Array)
+    
+    #allow for a 2 second variation in timestamp
+    @db[:create_time].select(:timestamp).first[:timestamp].to_i.should be_close((Time.now.to_i - 2),(Time.now.to_i + 2))
+    
+    @first_run_time = Time.now
+  end
+  
+  it 'should do a second-time run, capturing data' do
+    #set Time.now to 5 seconds from now
+    t = Time.now
+    Timecop.travel(t + 5)
+    
+    @s.run()
+
+    File.exist?(@db_file).should == true
+
+    @db = Sequel.sqlite(@db_file)   
+    @db[:stories].all.length.should be >= 1
+    first_story = @db[:stories].first
+    first_story[:visitors].should be_a(Fixnum)
+    first_story[:title].should be_a(String)
+    first_story[:path].should be_a(String)
+    
+    #save first_story title & first_story visitors for use in the combiner test
+    FIRST_STORY_TITLE = first_story[:title]
+    FIRST_STORY_VISITORS = first_story[:visitors]
+    Timecop.return
+  end
+  
+  it 'should do a combining data capture' do
+    @s.run()
+
+    File.exist?(@db_file).should == true
+    @db = Sequel.sqlite(@db_file)   
+    
+    # let's check to see that the titles array is unique
+    # i.e., that dupes have been added together
+    test_titles = []
+    stories = @db[:stories].all
+    stories.each do |story|
+      test_titles << story[:title]
+    end
+    test_titles.uniq.size.should eql(stories.size)
+    
+    # now let's check that visit counts have been combined
+    # for this, we'll *assume* that the first story in the array
+    # (which has the higest visitor ct) will be combined.
+    # We'll use @first_story_title we got in the second-time-run test
+    @combined_story = @db[:stories].where(:title => FIRST_STORY_TITLE).first
+    @combined_story[:title].should be_a(String)
+    @combined_story[:visitors].should > FIRST_STORY_VISITORS
+  end
+  
+  it 'should report data and dump db' do
+    # set Time.now to 5 seconds past ttl
+    t = Time.now
+    Timecop.travel(t + @ttl + 5)
+  
+    @s.run()
+    
+    File.exist?(@flat_file).should == true
+    File.exist?(@db_file).should == false
+
+    Timecop.return
+  end
+  
+  
+  after :all do
+    FileUtils.rm_rf(@db_file)
+    FileUtils.rm_rf(@flat_file)
+  end
+  
+end
+
+describe "basic Filterer filtering" do
+   
+  before :each do
+    @f = StatsCombiner::Filterer.new
+  end
+  
+  it 'should add a rule to the filters array' do
+    @f.add :prefix => 'tpmdc', :title_regex => /\| TPMDC/, :modify_title => true
+    
+    @f.filters.size.should eql(1)
+  end
+  
+  # various filter cases
+  it 'should set a prefix according to a title_regex' do
+    @f.add :prefix => 'tpmdc', :title_regex => /\| TPMDC/
+    datum = {:visitors=>366, :created_at=>nil, :path=>"/2010/05/with-specter-suffering-white-house-and-gop-looking-at-surging-sestak.php", :id=>3, :title=>"With Specter Suffering, White House And GOP Looking At Surging Sestak | TPMDC"}
+
+    result = StatsCombiner::Filterer.apply_filters!(@f.filters,datum)
+    result[:prefix].should eql("tpmdc")
+  end
+
+  it 'should modify a title based on a title_regex and a modify_title boolean' do
+    @f.add :prefix => 'tpmdc', :title_regex => /\| TPMDC/, :modify_title => true
+    
+    datum = {:visitors=>366, :created_at=>nil, :path=>"/2010/05/with-specter-suffering-white-house-and-gop-looking-at-surging-sestak.php", :id=>3, :title=>"With Specter Suffering, White House And GOP Looking At Surging Sestak | TPMDC"}
+
+    result = StatsCombiner::Filterer.apply_filters!(@f.filters,datum)
+    result[:title].should eql("With Specter Suffering, White House And GOP Looking At Surging Sestak")
+  end
+
+  it 'should modify a title based on a title_regex and a modify_title regex' do
+    @f.add :prefix => 'tpmdc', :title_regex => /(\| TPMDC)$/, :modify_title => '\1 Central'
+    
+    datum = {:visitors=>366, :created_at=>nil, :path=>"/2010/05/with-specter-suffering-white-house-and-gop-looking-at-surging-sestak.php", :id=>3, :title=>"With Specter Suffering, White House And GOP Looking At Surging Sestak | TPMDC"}
+
+    result = StatsCombiner::Filterer.apply_filters!(@f.filters,datum)
+    result[:title].should eql("With Specter Suffering, White House And GOP Looking At Surging Sestak | TPMDC Central")
+  end
+  
+  it 'should set a suffix where it matches a path_regex and modify a path according to a suffix regex' do
+    @f.add :path_regex => /(\?ref=.*)$/, :suffix => '\1&foo=bar', :modify_path => true
+  
+    datum = {:visitors=>366, :created_at=>nil, :path=>"/2010/05/with-specter-suffering-white-house-and-gop-looking-at-surging-sestak.php?ref=fpa", :id=>3, :title=>"With Specter Suffering, White House And GOP Looking At Surging Sestak | TPMDC"}
+    
+    result = StatsCombiner::Filterer.apply_filters!(@f.filters,datum)
+    result[:path].should eql("/2010/05/with-specter-suffering-white-house-and-gop-looking-at-surging-sestak.php?ref=fpa&foo=bar")
+    
+    @f.filters.clear
+    
+    #another example to kill unwanted query strings
+    @f.add :path_regex => /((\?|&)ref=.*)/, :suffix => '', :modify_path => true
+ 
+     datum = {:visitors=>366, :created_at=>nil, :path=>"/2010/05/with-specter-suffering-white-house-and-gop-looking-at-surging-sestak.php?id=keepme&ref=killme", :id=>3, :title=>"With Specter Suffering, White House And GOP Looking At Surging Sestak | TPMDC"}
+    
+    result = StatsCombiner::Filterer.apply_filters!(@f.filters,datum)
+    result[:path].should eql("/2010/05/with-specter-suffering-white-house-and-gop-looking-at-surging-sestak.php?id=keepme")
+  end
+
+  
+  it 'should nil out data where exclude is true and path or title regexes are matched' do
+    #first, two examples of a matching regex
+    
+    @f.add :path_regex => /(\/$|\/index.php$)/, :exclude => true
+    
+    datum = {:visitors=>3090, :created_at=>nil, :path=>"/", :id=>1, :title=>"Talking Points Memo | Breaking News and Analysis"}
+    
+    result = StatsCombiner::Filterer.apply_filters!(@f.filters,datum)
+    result[:title].should be_nil
+    result[:path].should be_nil
+    
+    @f.filters.clear
+
+    @f.add :path_regex => /talk\/blogs/, :exclude => true
+    
+    datum = {:visitors=>6, :created_at=>nil, :path=>"/talk/blogs/a/m/americandad/2010/03/an-open-letter-to-conservative.php/", :id=>31, :title=>"An open letter to conservatives | AmericanDad's Blog"}
+    
+    result = StatsCombiner::Filterer.apply_filters!(@f.filters,datum)
+    result[:title].should be_nil
+    result[:path].should be_nil
+   
+    @f.filters.clear
+    
+    #now, let's look for invalid data. run a regex against a nonmatch
+    @f.add :path_regex => /(\/$|\/index.php$)/, :exclude => true
+    
+    datum = {:visitors=>6, :created_at=>nil, :path=>"/talk/blogs/a/m/americandad/2010/03/an-open-letter-to-conservative.php", :id=>31, :title=>"An open letter to conservatives | AmericanDad's Blog"}    
+    
+    result = StatsCombiner::Filterer.apply_filters!(@f.filters,datum)
+    result[:title].should_not be_nil
+    result[:path].should_not be_nil
+    
+    @f.filters.clear
+    
+    #try a title match
+    @f.add :title_regex => /(Breaking News and Analysis)/, :exclude => true
+    
+    datum = {:visitors=>3090, :created_at=>nil, :path=>"/", :id=>1, :title=>"Talking Points Memo | Breaking News and Analysis"}
+    
+    result = StatsCombiner::Filterer.apply_filters!(@f.filters,datum)
+    result[:title].should be_nil
+    result[:path].should be_nil    
+    
+  end
+  
+end
+
+# best way to do this (without prying open the Class) might be 
+# to define a set of filters, timetravel our way to the flat file stage, 
+# open it and parse the HTML with hpricot for rules.
+# Caveats - filters will be pretty specific 
+# to whatever account is running this suite. So, proceed with caution.
+describe "filtered StatsCombining" do
+
+  before :each do
+    @flat_file = File.dirname(__FILE__) + '/test_flat_file.html'
+    @ttl = 3600
+    @s = StatsCombiner::Combiner.new({
+      :ttl => @ttl, 
+      :host=> HOST,
+      :api_key=> KEY,
+      :flat_file => @flat_file,
+    })
+    @db_file = "#{HOST}_stats_db.sqlite3"
+  end
+
+  it 'should run its way through the cycle and publish out a top ten list according to filter rules' do
+  
+    # first, let's set the filters we want to apply
+    e = StatsCombiner::Filterer.new
+    e.add :prefix => 'tpmdc', :title_regex => /\| TPMDC/, :modify_title => true
+    e.add :prefix => 'tpmmuckraker', :title_regex => /\| TPMMuckraker/, :modify_title => true
+    e.add :prefix => 'tpmtv', :title_regex => /\| TPMTV/, :modify_title => true
+    e.add :prefix => 'tpmcafe', :title_regex => /\| TPMCafe/, :modify_title => true
+    e.add :prefix => 'tpmlivewire', :title_regex => /\| TPM LiveWire/, :modify_title => true
+    e.add :prefix => 'polltracker', :title_regex => /\| TPM PollTracker/, :modify_title => true
+    e.add :prefix => 'www', :title_regex => /\|.*$/, :modify_title => true  
+    e.add :path_regex => /(\/$|\/index.php$)/, :exclude => true  
+    
+    # now, let's go through the rigamarole to get this thing pubbed.
+    # run to setup db
+    @s.run :filters => e.filters
+    # run again to start publishing
+    @s.run :filters => e.filters
+    # timetravel to pub time and do it.
+    # * set Time.now to 5 seconds past ttl
+    t = Time.now
+    Timecop.travel(t + @ttl + 5)
+    
+    # add filters
+    @s.run :filters => e.filters
+    
+    # sanity check
+    File.exist?(@flat_file).should == true
+    
+    # open the file we just made
+    list = File.open(@flat_file).read
+    list = Hpricot(list)
+    
+    # collect urls and titles
+    urls = []
+    titles = []
+    list.search("a").each do |a|
+      urls << a.attributes['href']
+      titles << a.inner_html
+    end  
+    
+    # let's make sure we have 10 stories
+    urls.size.should eql(10)
+    
+    # pull prefixes from rules array
+    prefixes = e.filters.collect { |filter| filter[:rule][:prefix] }
+    
+    # test prefixes against subdomains
+    urls.each do |url|
+        subdomain = URI.parse(url).host.split('.')[0]
+        prefixes.include?(subdomain).should == true
+    end
+    
+    Timecop.return
+  end
+
+  after :all do
+    FileUtils.rm_rf(@db_file)
+    FileUtils.rm_rf(@flat_file)
+  end
+
+end

data/spec/test_data.rb

@@ -0,0 +1,9 @@
+require 'fakeweb'
+
+FakeWeb.allow_net_connect = false
+
+TEST_DATA = <<-DOCUMENT
+[{"i": "Talking Points Memo | Breaking News and Analysis", "path": "\/", "visitors": 529}, {"i": "Theories of the Fall | Talking Points Memo", "path": "\/archives\/2010\/05\/theories_of_the_fall.php", "visitors": 66}, {"i": "What A Week: Rand Paul Takes The National Stage | TPMDC", "path": "\/2010\/05\/what-a-week-rand-paul-takes-the-national-stage.php?ref=fpa", "visitors": 37}, {"i": "Texas Board Of Ed Approves Right-Wing History Textbook Standards (VIDEO) | TPMMuckraker", "path": "\/2010\/05\/texas_history_textbooks_final_vote.php", "visitors": 15}, {"i": "Ron Paul Appeared On Meet The Press In '07 And Spoke Out Against Civil Rights Act (VIDEO) | TPMDC", "path": "\/2010\/05\/ron-paul-appeared-on-meet-the-press-in-07-and-spoke-out-against-civil-rights-act-video.php?ref=fpb", "visitors": 12}, {"i": "Texas Board Of Ed Approves Right-Wing History Textbook Standards (VIDEO) | TPMMuckraker", "path": "\/2010\/05\/texas_history_textbooks_final_vote.php?ref=fpb", "visitors": 11}, {"i": "CNN Headline: 'Miss USA: Muslim Trailblazer Or Hezbollah Spy?' | TPM LiveWire", "path": "\/2010\/05\/cnn-headline-miss-usa-muslim-trailblazer-or-hezbollah-spy.php?ref=fpb", "visitors": 8}, {"i": "GOP Kills Science Jobs Bill By Forcing Dems To Vote For Porn | TPMDC", "path": "\/2010\/05\/gop-kills-science-jobs-bill-by-forcing-dems-to-vote-for-porn.php", "visitors": 7}, {"i": "Texas Board Of Ed Approves Right-Wing History Textbook Standards (VIDEO) | TPMMuckraker", "path": "\/2010\/05\/texas_history_textbooks_final_vote.php?ref=fpa", "visitors": 7}, {"i": "Will Wall Street Reform Negotiators Publicly Weaken Derivative-Trading Regulations? (VIDEO) | TPMDC", "path": "\/2010\/05\/will-wall-street-reform-negotiators-publicly-weaken-derivative-trading-regulations-video.php?ref=fpb", "visitors": 6}, {"i": "What A Week: Rand Paul Takes The National Stage | TPMDC", "path": "\/2010\/05\/what-a-week-rand-paul-takes-the-national-stage.php?ref=tn", "visitors": 6}, {"i": "Unity In Kentucky As GOPers Rally 'Round Rand | TPMDC", "path": "\/2010\/05\/this-is-what-kentucky-unity-looks-like-gopers-rally-round-rand.php?ref=fpb", "visitors": 6}, {"i": "So Not Ready For Prime Time (or Even Sunday) | Talking Points Memo", "path": "\/archives\/2010\/05\/so_not_ready_for_prime_time_or_even_sunday.php", "visitors": 6}, {"i": "Paul Backs Out Of Meet The Press Appearance | TPMDC", "path": "\/2010\/05\/paul-backs-out-of-meet-the-press-appearance.php?ref=fpb", "visitors": 6}, {"i": "Jack Conway To TPMDC: Paul Civil Rights Comments 'Relevant' To General Election Campaign | TPMDC", "path": "\/2010\/05\/jack-conway-to-tpmdc-paul-civil-rights-comments-relevant-to-general-election-campaign.php?ref=fpb", "visitors": 5}, {"i": "TPMDC Saturday Roundup | TPMDC", "path": "\/2010\/05\/obama-announces-commission-on-oil-spill.php", "visitors": 5}, {"i": "What A Week: Rand Paul Takes The National Stage | TPMDC", "path": "\/2010\/05\/what-a-week-rand-paul-takes-the-national-stage.php", "visitors": 5}, {"i": "Dems Brace For Loss Of Usually Ultra-Safe Hawaii House Seat On Saturday | TPMDC", "path": "\/2010\/05\/dems-brace-for-loss-of-usually-ultra-safe-hawaii-house-seat-on-saturday.php", "visitors": 4}, {"i": "That Other Thing: Blumenthal And The Swim Team | TPMMuckraker", "path": "\/2010\/05\/that_other_thing_blumenthal_and_the_swim_team.php?ref=fpc", "visitors": 4}, {"i": "Franken Raises Money To Oppose Rand Paul -- And He Likes Brunch | TPMDC", "path": "\/2010\/05\/franken-raises-money-to-oppose-rand-paul----and-he-likes-brunch.php?ref=fpi", "visitors": 4}, {"i": "What Did Rand Paul Really Say On Maddow Last Night? | TPMDC", "path": "\/2010\/05\/what-did-rand-paul-really-say-on-maddow-last-night.php", "visitors": 4}, {"i": "Like a Friggin' Rock | Talking Points Memo", "path": "\/archives\/2010\/05\/like_a_friggin_rock.php", "visitors": 4}, {"i": "Franken Raises Money To Oppose Rand Paul -- And He Likes Brunch | TPMDC", "path": "\/2010\/05\/franken-raises-money-to-oppose-rand-paul----and-he-likes-brunch.php", "visitors": 4}, {"i": "Unity In Kentucky As GOPers Rally 'Round Rand | TPMDC", "path": "\/2010\/05\/this-is-what-kentucky-unity-looks-like-gopers-rally-round-rand.php?ref=fpa", "visitors": 4}, {"i": "Rand Paul: Economic Collapse Could Lead To 'A Hitler' Coming To Power | TPMMuckraker", "path": "\/2010\/05\/rand_paul_economic_collapse_could_lead_to_a_hitler.php", "visitors": 4}, {"i": "Rand Paul: Obama's BP Comments Sound 'Really Un-American' (VIDEO) | TPM LiveWire", "path": "\/2010\/05\/rand-paul-obamas-bp-comments-sound-really-un-american-video.php?ref=fpblg", "visitors": 4}, {"i": "CNN Headline: 'Miss USA: Muslim Trailblazer Or Hezbollah Spy?' | TPM LiveWire", "path": "\/2010\/05\/cnn-headline-miss-usa-muslim-trailblazer-or-hezbollah-spy.php", "visitors": 3}, {"i": "A Red Tulip Confirms All Crows Are Black | Fred Moolten's Blog", "path": "\/talk\/blogs\/fredmoolten\/2009\/06\/a-red-tulip-confirms-all-crows.php", "visitors": 3}, {"i": "Peter Beinart Unbound? | TPMCafe", "path": "\/2010\/05\/19\/beinart_unbound_here_and_in_bookforumcom\/", "visitors": 3}, {"i": "Bill Clinton To Campaign For Blanche Lincoln In Arkansas | TPMDC", "path": "\/2010\/05\/bill-clinton-to-campaign-for-blanche-lincoln-in-arkansas.php?ref=fpb", "visitors": 3}, {"i": "Texas Board Of Ed Approves Right-Wing History Textbook Standards (VIDEO) | TPMMuckraker", "path": "\/2010\/05\/texas_history_textbooks_final_vote.php?ref=tn", "visitors": 3}, {"i": "What A Week: Rand Paul Takes The National Stage | TPMDC", "path": "\/2010\/05\/what-a-week-rand-paul-takes-the-national-stage.php?ref=dcblt", "visitors": 3}, {"i": "Bwakfat's Dashboard | All", "path": "\/cgi-bin\/mt-current\/mt-cp.cgi?__mode=my_dashboard", "visitors": 3}, {"i": "Paul Has Been Guest Of Conspiracy Theorist Shock Jock Alex Jones (VIDEO) | TPMMuckraker", "path": "\/2010\/05\/paul_has_been_guest_of_conspiracy_theorist_shock_j.php", "visitors": 3}, {"i": "Full Text Of Newsmax Column Suggesting Military Coup Against Obama | TPM News Pages", "path": "\/news\/2009\/09\/full_text_of_newsmax_column_suggesting_military_co.php\/", "visitors": 3}, {"i": "Ron Paul Appeared On Meet The Press In '07 And Spoke Out Against Civil Rights Act (VIDEO) | TPMDC", "path": "\/2010\/05\/ron-paul-appeared-on-meet-the-press-in-07-and-spoke-out-against-civil-rights-act-video.php", "visitors": 3}, {"i": "Rand Paul In '08: Beware The NAFTA Superhighway! (VIDEO) | TPMMuckraker", "path": "\/2010\/05\/rand_paul_beware_the_nafta_superhighway_video.php", "visitors": 3}, {"i": "Jack Conway To TPMDC: Paul Civil Rights Comments 'Relevant' To General Election Campaign | TPMDC", "path": "\/2010\/05\/jack-conway-to-tpmdc-paul-civil-rights-comments-relevant-to-general-election-campaign.php?ref=tn", "visitors": 3}, {"i": "Ahistorical | Talking Points Memo", "path": "\/archives\/2010\/05\/ahistorical.php", "visitors": 3}, {"i": "Harvard Needs To Fire Dershowitz | TPMCafe", "path": "\/2010\/05\/22\/harvard_needs_to_fire_dershowitz\/", "visitors": 3}, {"i": "Scientists see video, adjust Gulf leak estimates | TPM News Pages", "path": "\/news\/2010\/05\/scientists_see_video_adjust_gulf_leak_estimates.php?ref=fpa", "visitors": 3}, {"i": "Rand Paul Defends Criticism Of Civil Rights Act To Rachel Maddow (VIDEO) | TPM LiveWire", "path": "\/2010\/05\/rand-paul-defends-criticism-of-civil-rights-act-to-rachel-maddow.php?ref=fpa", "visitors": 3}, {"i": "Dems Brace For Loss Of Usually Ultra-Safe Hawaii House Seat On Saturday | TPMDC", "path": "\/2010\/05\/dems-brace-for-loss-of-usually-ultra-safe-hawaii-house-seat-on-saturday.php?ref=fpb", "visitors": 3}, {"i": "LisB Blows A Gasket | LisB's Blog", "path": "\/talk\/blogs\/l\/i\/lisb\/2010\/05\/lisb-blows-a-gasket.php?ref=reccafe", "visitors": 2}, {"i": "Rand Paul In '08: Beware The NAFTA Superhighway! (VIDEO) | TPMMuckraker", "path": "\/2010\/05\/rand_paul_beware_the_nafta_superhighway_video.php?ref=fpblg", "visitors": 2}, {"i": "Texas Textbook Hearings | TPMMuckraker", "path": "\/texas_textbook_hearings\/", "visitors": 2}, {"i": "California Gov. Candidate Pushes Plan For 'Pedophile Island' | TPM LiveWire", "path": "\/2010\/05\/california-gov-candidate-pushes-plan-for-pedophile-island.php?ref=fpblg", "visitors": 2}, {"i": "Jack Conway Strikes Back At Paul's 'Un-American' Comment | TPM LiveWire", "path": "\/2010\/05\/jack-conway-strikes-back-at-pauls-un-american-comment.php?ref=fpa", "visitors": 2}, {"i": "Theories of the Fall | Talking Points Memo", "path": "\/archives\/2010\/05\/theories_of_the_fall.php?ref=fpblg", "visitors": 2}, {"i": "Jack Conway Strikes Back At Paul's 'Un-American' Comment | TPM LiveWire", "path": "\/2010\/05\/jack-conway-strikes-back-at-pauls-un-american-comment.php", "visitors": 2}]
+DOCUMENT
+
+FakeWeb.register_uri(:get, "http://api.chartbeat.com/toppages/?host=fake.com&limit=50&apikey=fake_key", :body => TEST_DATA)

data/stats_combiner.gemspec

@@ -0,0 +1,68 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{stats_combiner}
+  s.version = "0.0.1"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Al Shaw"]
+  s.date = %q{2010-05-24}
+  s.description = %q{A tool to create most-viewed story widgets from the Chartbeat API.}
+  s.email = %q{almshaw@gmail.com}
+  s.extra_rdoc_files = [
+    "README.md"
+  ]
+  s.files = [
+    ".gitignore",
+     "README.md",
+     "Rakefile",
+     "VERSION",
+     "combiner_sample.rb",
+     "lib/stats_combiner.rb",
+     "lib/stats_combiner/filterer.rb",
+     "spec/stats_combiner_spec.rb",
+     "spec/test_data.rb",
+     "stats_combiner.gemspec"
+  ]
+  s.homepage = %q{http://github.com/tpm/stats_combiner}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.6}
+  s.summary = %q{StatsCombiner creates most-viewed story widgets from the Chartbeat API}
+  s.test_files = [
+    "spec/stats_combiner_spec.rb",
+     "spec/test_data.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<crack>, [">= 0"])
+      s.add_runtime_dependency(%q<sequel>, [">= 0"])
+      s.add_development_dependency(%q<rspec>, [">= 0"])
+      s.add_development_dependency(%q<fakeweb>, [">= 0"])
+      s.add_development_dependency(%q<timecop>, [">= 0"])
+      s.add_development_dependency(%q<hpricot>, [">= 0"])
+    else
+      s.add_dependency(%q<crack>, [">= 0"])
+      s.add_dependency(%q<sequel>, [">= 0"])
+      s.add_dependency(%q<rspec>, [">= 0"])
+      s.add_dependency(%q<fakeweb>, [">= 0"])
+      s.add_dependency(%q<timecop>, [">= 0"])
+      s.add_dependency(%q<hpricot>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<crack>, [">= 0"])
+    s.add_dependency(%q<sequel>, [">= 0"])
+    s.add_dependency(%q<rspec>, [">= 0"])
+    s.add_dependency(%q<fakeweb>, [">= 0"])
+    s.add_dependency(%q<timecop>, [">= 0"])
+    s.add_dependency(%q<hpricot>, [">= 0"])
+  end
+end
+

metadata

@@ -0,0 +1,143 @@
+--- !ruby/object:Gem::Specification 
+name: stats_combiner
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Al Shaw
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-05-24 00:00:00 -04:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: crack
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: sequel
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: fakeweb
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: timecop
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id005
+- !ruby/object:Gem::Dependency 
+  name: hpricot
+  prerelease: false
+  requirement: &id006 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id006
+description: A tool to create most-viewed story widgets from the Chartbeat API.
+email: almshaw@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.md
+files: 
+- .gitignore
+- README.md
+- Rakefile
+- VERSION
+- combiner_sample.rb
+- lib/stats_combiner.rb
+- lib/stats_combiner/filterer.rb
+- spec/stats_combiner_spec.rb
+- spec/test_data.rb
+- stats_combiner.gemspec
+has_rdoc: true
+homepage: http://github.com/tpm/stats_combiner
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.6
+signing_key: 
+specification_version: 3
+summary: StatsCombiner creates most-viewed story widgets from the Chartbeat API
+test_files: 
+- spec/stats_combiner_spec.rb
+- spec/test_data.rb