trumpcare_tracker 0.1.9 → 0.1.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fe0ebb8e71f2a9e43a10fadc44bdfdba50ef3611
-  data.tar.gz: a6557125ce42eac222e47e67644b327ed384e6ec
+  metadata.gz: 141bd2e1843ec05907a0eb07c7ab97befa22f14b
+  data.tar.gz: 0ed7d0da381a117a102f9936092772ea3ef57799
 SHA512:
-  metadata.gz: 6a798fc04fda293a830e127f8254f8e02294c6ee0df9f6d8242fc07a0f3cc958df1ab677143241546b0df341e092f329ca5689b952ae00fead9c0942135b0d6f
-  data.tar.gz: c4ed4f819da83b93bcba2850039778e192c7b6b7d2c244497bb13fee8028f9c16fd07ae71a49d56d50a15f5380dec86fdfaa7a55651aeea6a7206ca7dcb7a426
+  metadata.gz: 2fdc6ed15df67450b7bb5f71a01ae2a9d1eef1534ad5f5649c6149fcafe02980f0bc9e9878f3f09b522acebbc55b2350c5a22d5c027d228fd4f48313a2c46f89
+  data.tar.gz: 46712d4277da517da3d3ae112e5b8918e879064b2da700f73dc89313f1a7916fe3c649fecf7b31c0f375d510f7b6b8b7713a231781dea84323e2bed422be6eac

data/README.md

@@ -45,11 +45,14 @@ TrumpcareTracker::Reporters.new
 run the tasks like so
 
 ```
-bundle exec rake tracker:tweet_bot # Audit tweets and post to Twitter timeline with Senator's phone numbers
+bundle exec rake tracker:tweet_bot:democrats
+bundle exec rake tracker:tweet_bot:republicans # Audit tweets and post to Twitter timeline with Senator's phone numbers
 
-bundle exec rake tracker:export # Audit tweets and export to CSV
+bundle exec rake tracker:export:democrats
+bundle exec rake tracker:export:republicans # Audit tweets and export to CSV
 
-bundle exec rake tracker:homepage_scraper # Audit websites and export to CSV
+bundle exec rake tracker:homepage_scraper:democrats
+bundle exec rake tracker:homepage_scraper:republicans # Audit websites and export to CSV
 ```
 
 ## Development

data/lib/trumpcare_tracker.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require 'date'
 require 'trumpcare_tracker/version'
 require 'twitter'
 
@@ -10,7 +11,7 @@ class TrumpcareTracker
   ACCESS_TOKEN        = ENV['TCT_ACCESS_TOKEN']
   ACCESS_TOKEN_SECRET = ENV['TCT_ACCESS_TOKEN_SECRET']
 
-  attr_reader :user, :screen_name, :alt_screen_name
+  attr_reader :user, :screen_name, :alt_screen_name, :requests
 
   def self.ratio(numerator, denominator)
     return 0.0 if denominator.zero?
@@ -22,17 +23,23 @@ class TrumpcareTracker
   end
 
   def self.trumpcare_keyword_regex
-    /(ahca|trumpcare|healthcare|health|care|drug|medication|medicaid|prescription|vaccine|obamacare|cbo|premiums|insurance|deductibles|aca|o-care|a.h.c.a|a.c.a|pre-existing conditions|hhs)/
+    /(ahca|trumpcare|healthcare|health|care|drug|medic|prescription|vaccin|obamacare|cbo|premiums|insurance|deductibles|aca.| aca |aca |o-care|a.h.c.a|a.c.a|pre-existing conditions|hhs|showusthebill|show us the bill)/
   end
 
   def self.russia_keyword_regex
-    /(russia|comey|sessions|mueller|fbi|flynn|obstruction of justice|collusion|putin|kremlin)/
+    /(russia|comey|sessions|mueller|fbi|flynn|obstruction of justice|collusion|putin|kremlin|kislyak|attorney general| intelligence committee|rosenstein|witch hunt|impeach|perjur|under oath)/
   end
 
-  def initialize(user, screen_name, alt_screen_name = nil)
+  def initialize(user, screen_name, alt_screen_name = '')
+    @audited         = false
+    @requests        = 0
     @user            = user
-    @screen_name     = client.user screen_name
-    @alt_screen_name = client.user alt_screen_name if alt_screen_name
+    @screen_name     = screen_name.to_s
+    @alt_screen_name = alt_screen_name.to_s
+  end
+
+  def audited?
+    @audited
   end
 
   # Instantiate a Twitter Rest Client with API authorization
@@ -51,8 +58,10 @@ class TrumpcareTracker
 
   # Make two cursored API calls to fetch the 400 most recent tweets
   def fetch_timeline(screen_name)
-    return [] unless screen_name
+    return [] if screen_name.to_s.empty?
+    @requests += 2
     timeline = client.user_timeline(screen_name, exclude_replies: true, count: 200)
+    return timeline if timeline.empty?
     timeline + client.user_timeline(
       screen_name,
       exclude_replies: true,
@@ -69,11 +78,22 @@ class TrumpcareTracker
     end
   end
 
-  def reduce_by_keywords(regex)
-    recent_tweets.each_with_object([]) do |tweet, memo|
-      tweet_with_full_text = client.status(tweet.id, tweet_mode: 'extended')
-      memo << tweet if tweet_match?(tweet_with_full_text, regex)
+  def filter_by_keywords
+    recent_tweets.each do |tweet|
+      tweet_with_full_text = fetch_tweet_with_full_text(tweet)
+      if tweet_match?(tweet_with_full_text, self.class.trumpcare_keyword_regex)
+        stats[:trumpcare_tweets] << tweet
+      elsif tweet_match?(tweet_with_full_text, self.class.russia_keyword_regex)
+        stats[:russia_tweets] << tweet
+      end
     end
+
+    stats
+  end
+
+  def fetch_tweet_with_full_text(tweet)
+    @requests += 1
+    client.status(tweet.id, tweet_mode: 'extended')
   end
 
   def tweet_match?(tweet, regex)
@@ -81,7 +101,7 @@ class TrumpcareTracker
     if full_text.downcase.match?(regex)
       true
     elsif tweet.quoted_tweet?
-      quoted_tweet = client.status(tweet.quoted_tweet.id, tweet_mode: 'extended')
+      quoted_tweet = fetch_tweet_with_full_text(tweet.quoted_tweet)
       tweet_match?(quoted_tweet, regex)
     else
       false
@@ -89,19 +109,21 @@ class TrumpcareTracker
   end
 
   def trumpcare_tweets
-    @_trumpcare_tweets ||= reduce_by_keywords(self.class.trumpcare_keyword_regex)
+    stats[:trumpcare_tweets]
   end
 
   def russia_tweets
-    @_russia_tweets ||= reduce_by_keywords(self.class.russia_keyword_regex)
+    stats[:russia_tweets]
   end
 
   def audit
-    puts to_s
+    filter_by_keywords unless audited?
+    @audited = true
   end
 
   def to_s
-    @_to_s ||= "@#{screen_name.screen_name}'s last 7 days\n#{recent_tweets_count} tweets\n"\
+    audit unless audited?
+    @_to_s ||= "@#{screen_name}'s last 7 days\n#{recent_tweets_count} tweets\n"\
       "#{trumpcare_tweets_count} TrumpCare - #{trumpcare_tweets_percentage}%\n"\
       "#{russia_tweets_count} Russia - #{russia_tweets_percentage}%\n"\
       "#{trumpcare_to_russia_tweets_ratio} TrumpCare tweets for every Russia tweet"
@@ -111,6 +133,13 @@ class TrumpcareTracker
     @_recent_tweets_count ||= recent_tweets.count
   end
 
+  def stats
+    @_stats ||= {
+      trumpcare_tweets: [],
+      russia_tweets: []
+    }
+  end
+
   def trumpcare_tweets_count
     @_trumpcare_tweets_count ||= trumpcare_tweets.count
   end
@@ -120,22 +149,22 @@ class TrumpcareTracker
   end
 
   def trumpcare_tweets_percentage
-    self.class.percentage(trumpcare_tweets.count, recent_tweets.count)
+    self.class.percentage(trumpcare_tweets_count, recent_tweets_count)
   end
 
   def russia_tweets_percentage
-    self.class.percentage(russia_tweets.count, recent_tweets.count)
+    self.class.percentage(russia_tweets_count, recent_tweets_count)
   end
 
   def trumpcare_to_russia_tweets_ratio
-    self.class.ratio(trumpcare_tweets.count, russia_tweets.count)
+    self.class.ratio(trumpcare_tweets_count, russia_tweets_count)
   end
 
   def to_h
     {
       senator: user,
-      official_user_name: screen_name.screen_name,
-      alt_user_name: alt_screen_name_screen_name,
+      official_user_name: screen_name,
+      alt_user_name: alt_screen_name,
       tweets_in_last_seven_days: recent_tweets.count,
       trumpcare_tweet_count: trumpcare_tweets.count,
       tct_percentage: trumpcare_tweets_percentage,
@@ -147,12 +176,8 @@ class TrumpcareTracker
     }
   end
 
-  def alt_screen_name_screen_name
-    return '' unless alt_screen_name
-    alt_screen_name.screen_name
-  end
-
   def to_tweet(options = {})
+    @requests += 1
     client.update(".#{self}", options)
   end
 end

data/lib/trumpcare_tracker/rake_task.rb

@@ -9,56 +9,97 @@ require 'trumpcare_tracker'
 class TrumpcareTracker
   # Base rake task class
   class RakeTask < Rake::TaskLib
-    module Methods
-      def reps
-        democrats = PYR.reps do |r|
-          r.democrat = true
-          r.chamber  = 'upper'
-        end
-
-        independents = PYR.reps do |r|
-          r.independent = true
-          r.chamber     = 'upper'
-        end
-
-        democrats.objects.to_a + independents.objects.to_a
+    def democrats(chamber = '')
+      chamber = chamber.to_s
+
+      democrats = PYR.reps do |r|
+        r.democrat = true
+        r.chamber  = chamber unless chamber.empty?
       end
 
-      def handles
-        @_handles ||= CSV.read(File.expand_path('../twitter_handles.csv', __FILE__), headers: true)
+      independents = PYR.reps do |r|
+        r.independent = true
+        r.chamber     = chamber unless chamber.empty?
       end
 
-      def audit_rep(i, rep, &block)
-        puts "Sending requests to Twitter API for #{rep.official_full}"
-        alt_screen_name = handles.detect do |handle|
-          handle['official'].downcase.strip == rep.twitter.downcase.strip
-        end['personal/campaign']
-        tracker = TrumpcareTracker.new(rep.official_full, rep.twitter, alt_screen_name)
-        tracker.audit
-        block.call(tracker, rep) if block_given?
-        puts "#{i + 1} down. Pausing for 45 seconds to avoid hitting API rate limit."
-        sleep(45)
-      rescue Twitter::Error::TooManyRequests
-        puts 'Rate limit exceeded, waiting 5 minutes'
-        sleep(300)
-        audit_rep(i, rep, &block)
-      rescue Twitter::Error => e
-        puts e.message
-        audit_rep(i, rep, &block)
+      democrats.objects.to_a + independents.objects.to_a
+    end
+
+    def senate_democrats
+      democrats('upper')
+    end
+
+    def house_democrats
+      democrats('lower')
+    end
+
+    def republicans(chamber = '')
+      chamber = chamber.to_s
+
+      republicans = PYR.reps do |r|
+        r.republican = true
+        r.chamber = chamber unless chamber.empty?
+      end
+
+      republicans.objects
+    end
+
+    def senate_republicans
+      republicans('upper')
+    end
+
+    def house_republicans
+      republicans('lower')
+    end
+
+    def handles
+      @_handles ||= CSV.read(File.expand_path('../twitter_handles.csv', __FILE__), headers: true)
+    end
+
+    def audit_rep(i, rep, &block)
+      puts "Sending requests to Twitter API for #{rep.official_full}"
+      rep_handles = handles.detect do |handle|
+        handle['official'].downcase.strip == rep.twitter.downcase.strip
+      end
+
+      alt_screen_name = if rep_handles
+        rep_handles['personal/campaign']
       end
 
-      def audit_tweets(reps, &block)
-        reps.each_with_index do |rep, i|
-          next if rep.twitter.nil?
-          audit_rep(i, rep, &block)
-        end
-      ensure
-        `say "beep beep beep beep beep"`
+      tracker = TrumpcareTracker.new(rep.official_full, rep.twitter, alt_screen_name)
+      start_time = Time.now
+      tracker.audit
+      duration = (Time.now - start_time).round(2)
+      puts tracker.to_s
+      block.call(tracker, rep) if block_given?
+
+      puts "#{i + 1} down. #{tracker.requests} requests took #{duration} seconds."
+      interval = tracker.requests - duration
+      if interval.positive?
+        puts "Waiting #{interval} seconds to avoid hitting API limit"
+        sleep(interval)
       end
 
-      def mentions_mapper(doc, regex)
-        doc.text.split("\n").select { |string| string.downcase.match?(regex) }
+    rescue Twitter::Error::TooManyRequests
+      puts 'Rate limit exceeded, waiting 5 minutes'
+      sleep(300)
+      audit_rep(i, rep, &block)
+    rescue Twitter::Error => e
+      puts e.message
+      audit_rep(i, rep, &block)
+    end
+
+    def audit_tweets(reps, &block)
+      reps.each_with_index do |rep, i|
+        next if rep.twitter.nil?
+        audit_rep(i, rep, &block)
       end
+    ensure
+      `say "beep beep beep beep beep"`
+    end
+
+    def mentions_mapper(doc, regex)
+      doc.text.split("\n").select { |string| string.downcase.match?(regex) }
     end
   end
 end

data/lib/trumpcare_tracker/reporters.rb

@@ -10,72 +10,115 @@ class TrumpcareTracker
   # require 'trumpcare_tracker/reporters'
   # TrumpcareTracker::Reporters.new
   #
-  # $ bundle exec rake tracker:export
-  # $ bundle exec rake tracker:homepage_scraper
-  class Reporters < RakeTask
-    include RakeTask::Methods
-
+  # $ rake tracker:export:senate_democrats
+  # $ rake tracker:export:house_democrats
+  # $ rake tracker:export:senate_republicans
+  # $ rake tracker:export:house_republicans
+  # $ rake tracker:export:all
+  #
+  # $ rake tracker:homepage_scraper:senate_democrats
+  # $ rake tracker:homepage_scraper:house_democrats
+  # $ rake tracker:homepage_scraper:senate_republicans
+  # $ rake tracker:homepage_scraper:house_republicans
+  # $ rake tracker:homepage_scraper:all
+class Reporters < RakeTask
     def initialize
       export_task
       homepage_scraper_task
     end
 
     def export_task
-      namespace :tracker do
-        desc 'Output a report of Senate Democrats Trumpcare tweets'
-        task :export do
-          csv_tweet_report = CSV.generate do |csv|
-            csv << %w[
-              senator
-              official_user_name
-              alt_user_name
-              tweets_in_last_seven_days
-              trumpcare_tweets
-              tct_percentage
-              russia_tweets
-              rt_percentage
-              tct_to_rt_ratio
-              trumpcare_tweet_urls
-              russia_tweet_urls
-            ]
-            audit_tweets(reps) { |tracker| csv << tracker.to_h.values }
-          end
-
-          File.open('trumpcare_tweet_report.csv', 'w') do |file|
-            file.write csv_tweet_report
-          end
+      namespace(:tracker) do
+        namespace(:export) do
+          desc 'Output a report of Senate Democrats Trumpcare tweets'
+          task(:senate_democrats) { export(:senate_democrats) }
+
+          desc 'Output a report of House Democrats Trumpcare tweets'
+          task(:house_democrats) { export(:house_democrats) }
+
+          desc 'Output a report of Senate Republicans Trumpcare tweets'
+          task(:senate_republicans) { export(:senate_republicans) }
+
+          desc 'Output a report of House Republicans Trumpcare tweets'
+          task(:house_republicans) { export(:house_republicans) }
+
+          all_tasks
         end
       end
     end
 
+    def export(caucus)
+      csv_tweet_report = CSV.generate do |csv|
+        csv << %w[
+          senator
+          official_user_name
+          alt_user_name
+          tweets_in_last_seven_days
+          trumpcare_tweets
+          tct_percentage
+          russia_tweets
+          rt_percentage
+          tct_to_rt_ratio
+          trumpcare_tweet_urls
+          russia_tweet_urls
+        ]
+        audit_tweets(send(caucus)) { |tracker| csv << tracker.to_h.values }
+      end
+
+      File.open("trumpcare_tweet_report_#{caucus}.csv", 'w') do |file|
+        file.write csv_tweet_report
+      end
+    end
+
 
 
     def homepage_scraper_task
-      namespace :tracker do
+      namespace(:tracker) do
         desc 'Search Senators\' official homepage for TrumpCare and Russia Mentions'
-        task :homepage_scraper do
-          csv_homepage_report = CSV.generate do |csv|
-            csv << %w[senator url trumpcare_mentions russia_mentions trumpcare_to_russia_ratio]
-            reps.each do |rep|
-              doc = Nokogiri::HTML(open(rep.url))
-              trumpcare_mentions_count = mentions_mapper(doc, TrumpcareTracker.trumpcare_keyword_regex).count
-              russia_mentions_count = mentions_mapper(doc, TrumpcareTracker.russia_keyword_regex).count
-              csv << [
-                  rep.official_full,
-                  rep.url,
-                  trumpcare_mentions_count,
-                  russia_mentions_count,
-                  TrumpcareTracker.ratio(trumpcare_mentions_count, russia_mentions_count)
-              ]
-              puts "Scraped #{rep.official_full}'s homepage"
-            end
-          end
-
-          File.open('trumpcare_homepage_report.csv', 'w') do |file|
-            file.write csv_homepage_report
-          end
+        namespace(:homepage_scraper) do
+          desc 'Search Senate Democrat\'s homepages'
+          task(:senate_democrats) { homepage_scraper(:senate_democrats) }
+
+          desc 'Search House Democrat\'s homepages'
+          task(:house_democrats) { homepage_scraper(:house_democrats) }
+
+          desc 'Search Senate Republican\'s homepages'
+          task(:senate_republicans) { homepage_scraper(:senate_republicans) }
+
+          desc 'Search House Republican\'s homepages'
+          task(:house_republicans) { homepage_scraper(:house_republicans) }
+
+          all_tasks
         end
       end
     end
+
+    def homepage_scraper(caucus)
+      csv_homepage_report = CSV.generate do |csv|
+        csv << %w[senator url trumpcare_mentions russia_mentions trumpcare_to_russia_ratio]
+        send(caucus).each do |rep|
+          doc = Nokogiri::HTML(open(rep.url))
+          trumpcare_mentions_count = mentions_mapper(doc, TrumpcareTracker.trumpcare_keyword_regex).count
+          russia_mentions_count = mentions_mapper(doc, TrumpcareTracker.russia_keyword_regex).count
+          csv << [
+            rep.official_full,
+            rep.url,
+            trumpcare_mentions_count,
+            russia_mentions_count,
+            TrumpcareTracker.ratio(trumpcare_mentions_count, russia_mentions_count)
+          ]
+          puts "Scraped #{rep.official_full}'s homepage"
+        end
+      end
+
+      File.open("trumpcare_homepage_report_#{caucus}.csv", 'w') do |file|
+        file.write csv_homepage_report
+      end
+    end
+
+    def all_tasks
+      desc 'Run for each caucus'
+      task(all: [:senate_democrats, :house_democrats, :senate_republicans, :house_republicans])
+    end
   end
 end