CLI_Headline_Scraper 0.1.6 → 0.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c531dc5cb0962a2a016687894ad6212e6b7bf34b
-  data.tar.gz: 14e4d1939afadf7e741ecad229e6710b55d809d4
+  metadata.gz: 4575df135e79c6ac587405d19faf26ae4bd0ba62
+  data.tar.gz: e249f38021d93f7e95d454b614bed0ae99cd5bd8
 SHA512:
-  metadata.gz: 91ea48e86a9152864abcf1f187c76eb034a9a87464dfc23002b6b774b904a9fe7fcee952ae58145c38c289609e040e541ee89f7efda50eb184d26c9b74690360
-  data.tar.gz: bb58536699d79b58ffb88fa06a63d43aab7392cb6edba027ffe0f86eefc0c9b354a778515069e983cd4f175e45d72f20f033bcf94825801abdf9aedc25f1b44b
+  metadata.gz: e1e0a9f6e8612f3073561bae8e2fcb45235b78af5eff98299c8c2865bfda2f1982b9f3b047e3035f47184a0d001b6e8cf5b46ef37e6b01a80fa11afbe9fd14f5
+  data.tar.gz: f16967e39c2a8020912c1b94adc75cec15c866714a6e420f02a414c6aea277c3867c33e494186fc84b2c3171177d0f10ed89316ea44fab37b354c9efcaceffdb

data/bin/console

@@ -11,10 +11,4 @@ require "environment"
 # Pry.start
 require "irb"
 
-###Testing Conditions###
-
-
-
-
-###end testing conditions###
 IRB.start(__FILE__)

data/bin/headline_scraper

@@ -1,13 +1,5 @@
 #!/usr/bin/env ruby
-require 'environment'
-
-###testing conditions###
-
-
-###end testing conditions###
-
-Scraper.msnbc_homepage
-Scraper.fox_homepage
-Scraper.reuters_homepage
+puts "starting"
+require_relative '../lib/environment'
 
 CLI.new.call

data/lib/CLI_Headline_Scraper/Article.rb

@@ -32,15 +32,23 @@ class Article
     self.all.select{|item| item.network_name == network_name}
   end
 
+  def self.find_by_summary(word)
+
+    #cycle through all articles.
+    #look at each article's summary
+    #if summary contains word, add summary to a new array.
+    #after finished with all articles, display array.
+    self.all.select { |article| article.summary.downcase.include?(word.downcase) }
+
+
+  end
+
 
   def populate_metadata()
     #retreives metadata of reuters article -- right now just time/date.
     #1. Scrapes data from the selected article's url.(separate)
     #3. Uses that data to populate article.authors, article.date_posted, article.text.
-
     Scraper.reuters_article(self)
-
-
     article = Article.find_by_headline(headline)
 
   end

data/lib/CLI_Headline_Scraper/CLI.rb

@@ -1,4 +1,6 @@
 #Our CLI Controller
+
+require 'pry'
 class CLI
 
   attr_reader :time
@@ -8,6 +10,12 @@ class CLI
   end
 
   def call
+    puts "Initializing..."
+    Scraper.msnbc_homepage
+    Scraper.fox_homepage
+    Scraper.reuters_homepage
+    puts("done")
+    puts("")
 
     self.greet
     self.display_menu #initial menu selection of what you want to see
@@ -19,7 +27,7 @@ class CLI
     puts "Welcome to Headline Scraper"
     sleep(1)
     puts "Please select which of the following articles you would like to view:"
-    sleep(1.5)  
+    sleep(1.5)
     puts ""
   end
 
@@ -116,8 +124,6 @@ class CLI
         the_article = the_network.articles[selection[1]-1]
         self.article_options_menu(the_article)
       end
-
-
   end
 
   def selection_exists?(selection) #post-screens entries to make sure the valid entry actually refers to an existing item
@@ -144,8 +150,6 @@ class CLI
     end
   end
 
-
-
   def select_scrape_method(article)
 
     case article.network_name
@@ -159,11 +163,6 @@ class CLI
     end
   end
 
-
-
-
-
-
   def article_options_menu(article)
     #takes article object as an argument
     #automatically displays article headline, network name, and article metadata (i.e. author, date & time posted, number of comments, tags etc.)
@@ -186,7 +185,6 @@ class CLI
     puts "1. View article in browser."
     puts "2. Return to previous menu."
     puts "Or type 'exit'."
-
     input = gets.strip.upcase
     case input
     when "1"

data/lib/CLI_Headline_Scraper/Scraper.rb

@@ -7,13 +7,14 @@ class Scraper
 #<<<<<<<<<<<<<<<<<<REUTERS SCRAPING METHODS>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
 
 def self.reuters_homepage
-
+  puts "scraping Reuters homepage"
   url = "https://www.reuters.com"
   homepage = self.get_page(url)
   reuters = Network.create_with_url("REUTERS", url)
   reuters.home_html = homepage
   self.scrape_reuters_articles.each{|article| article = Article.create_with_url(article[0],"REUTERS", article[1])}
 
+
 end
 
 
@@ -33,7 +34,7 @@ end
 
 
 def self.check_reuters_urls(articles)
-   #checks for and corrects common issue where MSNBC uses partial urls for internal links
+   #checks for and corrects common issue where a website uses partial urls for internal links
 
   articles.each do |article|
     if !article[1].include?("www")
@@ -59,6 +60,7 @@ end
 #<<<<<<<<<<<<<<<<<<FOX SCRAPING METHODS>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
 
 def self.fox_homepage
+  puts "scraping Fox homepage"
   url = "http://www.foxnews.com"
   homepage = self.get_page(url)
   fox = Network.create_with_url("FOX NEWS", url)
@@ -70,16 +72,17 @@ end
 def self.scrape_fox_articles
 
   html = Network.find_by_name("FOX NEWS").home_html
+    leader = [html.css("div.collection.collection-spotlight article.article.story-1 header a").text.strip, html.css("div.collection.collection-spotlight article.article.story-1 header a").attribute("href")]
 
-  leader = [html.css("div.primary h1 a").text, html.css("div.primary h1 a").attribute("href").value]
-  second = [html.css("div.top-stories a h3").first.text, html.css("div.top-stories li").first.css("a").attribute("href").value]
+    second = [html.css("div.main.main-secondary article.article.story-1 h2.title a").text, html.css("div.main.main-secondary article.article.story-1 h2.title a").attribute("href").value]
 
-  third = [html.css("div.top-stories a h3")[1].text, html.css("div.top-stories li[data-vr-contentbox = ''] a")[4].attribute("href").value]
+    third = [html.css("div.main.main-secondary article.article.story-2 h2.title a").text, html.css("div.main.main-secondary article.article.story-2 h2.title a").attribute("href").value]
 
   articles = [leader, second, third]
 
 end
 
+
 def self.fox_article(article)
   article.html = self.get_page(article.url)
   article.summary = article.html.css("meta[name='description']").attribute("content").value
@@ -92,6 +95,7 @@ end
 #<<<<<<<<<<<<<<<MSNBC SCRAPING METHODS>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
 
   def self.msnbc_homepage
+    puts "scraping MSNBC homepage"
     url = "http://www.msnbc.com"
     homepage = self.get_page(url)
     msnbc = Network.create_with_url("MSNBC", url)

data/lib/CLI_Headline_Scraper/version.rb

@@ -1,3 +1,3 @@
 module CLIHeadlineScraper
-  VERSION = "0.1.6"
+  VERSION = "0.1.7"
 end

data/lib/environment.rb

@@ -1,4 +1,3 @@
-require 'require_all'
 require 'launchy'
 require 'open-uri'
 require 'nokogiri'

data/refactor_notes.txt

@@ -0,0 +1,4 @@
+1. Make separate scraper classes for each publication (supported by a scraper module)
+2. Make it so that each Object only knows about a max of one other object.
+3. Eliminate magic numbers.
+4. Don't use 'singleton' classes.  Make a new instance of each scraper object(one for each website) and store things like its html in instance variables like @doc.

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: CLI_Headline_Scraper
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.7
 platform: ruby
 authors:
 - Jim Stricker
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-09-06 00:00:00.000000000 Z
+date: 2017-11-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -126,6 +126,7 @@ files:
 - ".gitignore"
 - ".rspec"
 - ".travis.yml"
+- CLI_Headline_Scraper-0.1.6.gem
 - CLI_Headline_Scraper.gemspec
 - Gemfile
 - LICENSE.txt
@@ -140,6 +141,7 @@ files:
 - lib/CLI_Headline_Scraper/Scraper.rb
 - lib/CLI_Headline_Scraper/version.rb
 - lib/environment.rb
+- refactor_notes.txt
 - selection
 - spec.md
 homepage: https://github.com/jmstrick93/CLI_Headline_Scraper