statement 2.0.3 → 2.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 88a5cc8cc4a20b541a5cadd65e4ab3c139f5cf9f
-  data.tar.gz: 37d62f7c98c3a92b09980c14a5dde0d3c2c809ba
+  metadata.gz: c771519d9fc8c1d3b906d0b09695205a92f893f2
+  data.tar.gz: 737c1f6530c5d7d9424be1f5629df0042c6abc8c
 SHA512:
-  metadata.gz: 4ac80b83893c08c95d6d8a3623ca6a0859cddbfdba8e00004a8ff8370c6f87bc393d059f900eb60484993a70ea0a363a1033305d68a883fd4fe8fa61dbf27755
-  data.tar.gz: b95a2cba9cd02eb533d4fe73b681c180eb23d28eeb52491174548c59df794667b8fd194397699426cfc94918a32813de297789bedf62c8256b38e6b9196f05ca
+  metadata.gz: c1c53d467e8b878ff6995b6aacbc1618904a263bf55fe2d50e702334afbfbd0d323979d26be9e1d39abf83c2e21dc9258dcce98330cfc8a962620714bebd5707
+  data.tar.gz: e2ef4d6db0d2276a089c8e4bace60d859bc60d584177c4ec862bc55ddf2973ea3aa1b10aecf19769b4e86a5cc6143ae7206eef7688fe81c0a4f3ea57a1505827

data/lib/statement/scraper.rb

@@ -32,7 +32,7 @@ module Statement
       [:crenshaw, :capuano, :cold_fusion, :conaway, :chabot, :freshman_senators, :klobuchar, :billnelson, :crapo, :boxer,
       :vitter, :inhofe, :document_query, :swalwell, :fischer, :clark, :edwards, :culberson_chabot_grisham, :barton,
       :welch, :sessions, :gabbard, :costa, :farr, :mcclintock, :olson, :schumer, :lamborn, :walden, :boehner,
-      :bennie_thompson, :speier, :poe, :grassley, :bennet, :shaheen, :keating, :drupal, :jenkins, :durbin_burr]
+      :bennie_thompson, :speier, :poe, :grassley, :bennet, :shaheen, :keating, :drupal, :jenkins, :durbin_burr, :rand_paul]
     end
 
     def self.committee_methods
@@ -42,10 +42,11 @@ module Statement
     def self.member_scrapers
       year = Date.today.year
       results = [crenshaw, capuano, cold_fusion(year, nil), conaway, chabot, klobuchar(year), billnelson(page=0),
-        document_query(page=1), document_query(page=2), swalwell(page=1), crapo, boxer, grassley(page=0), 
+        document_query(page=1), document_query(page=2), swalwell(page=1), crapo, boxer, grassley(page=0),
         vitter(year=year), inhofe(year=year), fischer, clark(year=year), edwards, culberson_chabot_grisham(page=1), barton, welch,
         sessions(year=year), gabbard, costa, farr, olson, schumer, lamborn(limit=10), walden, bennie_thompson, speier,
-        poe(year=year, month=0), bennet(page=1), shaheen(page=1), perlmutter, keating, drupal, jenkins, durbin_burr(page=1)].flatten
+        poe(year=year, month=0), bennet(page=1), shaheen(page=1), perlmutter, keating, drupal, jenkins, durbin_burr(page=1),
+        rand_paul(page = 1)].flatten
       results = results.compact
       Utils.remove_generic_urls!(results)
     end
@@ -416,6 +417,51 @@ module Statement
       results
     end
 
+    def self.rand_paul(page = 1)
+      # each page contains a max of 20 results
+      page_url = "http://www.paul.senate.gov/news/press?PageNum_rs=#{page}"
+      doc = open_html(page_url)
+      return if doc.nil?
+      results = doc.search('#press .title').inject([]) do |arr, title|
+        article_url = URI.join(page_url, title.search('a')[0]['href'])
+        article_datestr = title.previous_element.text # e.g. "05.11.15"
+        arr << {
+          :source => page_url,
+          :url => article_url.to_s,
+          :domain => article_url.host,
+          :title => title.text,
+          :date => Date.strptime(article_datestr, '%m.%d.%y')
+        }
+      end
+      results
+    end
+
+
+    def self.patrick_meehan(page = 0)
+      # This is a Drupal page and it uses the View plugin, but unlike the other
+      # Drupal pages, it does not make use of .views-field-created, and instead, the
+      # only Month-Year is given (03 Feb).
+      page_url = "https://meehan.house.gov/media-center/press-releases?page=#{page}"
+      doc = open_html(page_url)
+      return if doc.nil?
+      results = doc.search('.view-congress-press-releases .views-row').inject([]) do |arr, article|
+        title = article.search('.views-field-title a')[0]
+        article_url = URI.join(page_url, title['href'])
+        raise "Date still needs to be parsed; thanks a lot Drupal"
+        article_datestr = title.previous_element.text
+        arr << {
+          :source => page_url,
+          :url => article_url.to_s,
+          :domain => article_url.host,
+          :title => title.text,
+          :date => Date.strptime(article_datestr, 'SOMETHING')
+        }
+      end
+
+      results
+    end
+
+
     # fetches the latest 1000 releases, can be altered
     def self.lautenberg(rows=1000)
       results = []
@@ -811,24 +857,36 @@ module Statement
       results
     end
 
-    def self.backfill_bilirakis
+    def self.backfill_bilirakis(page=1)
       results = []
       domain = 'bilirakis.house.gov'
-      url = 'http://bilirakis.house.gov/press-releases/'
+      url = "https://bilirakis.house.gov/press-releases?page=#{page}"
       doc = open_html(url)
       return if doc.nil?
-      doc.css("ul li[@class='article articleright']").each do |row|
-        results << {:source => url, :url => 'http://bilirakis.house.gov' + row.children[3].children[1]['href'], :title => row.children[3].text.strip, :date => Date.parse(row.children[5].text), :domain => domain }
+      doc.css("#region-content .views-row").each do |row|
+          title_anchor = row.css("h3 a")
+          title = title_anchor.text
+          release_url = "http://#{domain + title_anchor.attr('href')}"
+            raw_date = row.css(".views-field-created").text
+          results << { :source => url,
+                       :url => release_url,
+                       :title => title,
+                       :date => begin Date.parse(raw_date) rescue nil end,
+                       :domain => domain }
       end
+      results
     end
 
-    def self.backfill_boustany
+    def self.backfill_boustany(congress)
       results = []
       domain = 'boustany.house.gov'
-      url = 'http://boustany.house.gov/113th-congress/showallitems/'
+      url = "http://boustany.house.gov/#{congress}th-congress/showallitems/"
       doc = open_html(url)
       return if doc.nil?
-
+      (doc/:ul)[13].search(:li).each do |row|
+        results << {:source => url, :url => 'http://boustany.house.gov' + row.children.search(:a)[0]['href'], :title => row.children.search(:a)[0].text, :date => Date.parse(row.children[5].text), :domain => domain }
+      end
+      results
     end
 
     def self.perlmutter
@@ -875,7 +933,9 @@ module Statement
             "https://bilirakis.house.gov/press-releases",
             "http://quigley.house.gov/media-center/press-releases",
             "https://denham.house.gov/media-center/press-releases",
-            "https://sewell.house.gov/media-center/press-releases"
+            "https://sewell.house.gov/media-center/press-releases",
+            "https://buchanan.house.gov/media-center/press-releases",
+            "https://meehan.house.gov/media-center/press-releases"
         ]
       end
 

data/lib/statement/version.rb

@@ -1,3 +1,3 @@
 module Statement
-  VERSION = "2.0.3"
+  VERSION = "2.0.4"
 end