wikihow 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7049014cdd5bf0f414c1d25b67ff23cb510071007c60cae0310588da402a7564
-  data.tar.gz: d8c84ca96eda9d23f318b5dc4b6b276d332aec6d68d8f405cfdfc87eb8e88965
+  metadata.gz: c9ddfdd1a9076593a482625840ecb771a52fc2a5e087de6d5f4c214334aee9ae
+  data.tar.gz: 525467dc49b64bb6a42a45240aa072934378a9499241ccac780e8016f2e351fe
 SHA512:
-  metadata.gz: 6cd866fa89a9175087255dc9f538c2b8550681d747030a1a679fdc38cc72a0e424ff86c12cc501b2ff9f89ca06d1472c44182ff1f3e0af0ef2a0b4118b91ba32
-  data.tar.gz: 49ee6a9276be12580ac75bf880f1ce0ff51598125b71c0d50190380d240d37990c291769e7a0d67c3aa0eeef76a8a9f442303f4b9169e47b1a9133e70e1a83eb
+  metadata.gz: 68cd4e28703925cef8b23db2a12628aa2d652ac76cf1298c16e70dc33d28e0b1c1e80927583a4642409d580662a63948708a017e198854e3b01c698005ace4a1
+  data.tar.gz: 0a47cb859671095a0befc78e077cb88fcc203958d32bd523af0545246441d897673a87013944507e448b762652fc52a90169fd1eb6e2910db18df66557baa449

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    wikihow (0.1.0)
+    wikihow (0.1.1)
       nokogiri
 
 GEM

data/README.md

@@ -22,7 +22,7 @@ Or install it yourself as:
 
 Type the below and follow the on screen prompts.
 
-./bin/wikihow
+$ wikihow
 
 ## Development
 

data/lib/wikihow/category.rb

@@ -18,20 +18,8 @@ class Wikihow::Category
 
   def self.get_or_create_categories
     if self.all == []
-      self.scrape_for_categories.each{|category| self.new(category)}
+      Wikihow::Scraper.scrape_for_categories.each{|category| self.new(category)}
     end
-    #binding.pry
     self.all
   end
-
-  def self.scrape_for_categories
-    doc = Nokogiri::HTML(open("https://www.wikihow.com/Main-Page"))
-    categories_array = []
-    doc.search("#hp_categories a").each do |category|
-      title = category.text
-      url = category.attr("href")
-      categories_array << {:title => title,:url => url}
-    end
-    categories_array
-  end
 end

data/lib/wikihow/scraper.rb

@@ -0,0 +1,47 @@
+class Wikihow::Scraper
+  def self.scrape_for_categories
+    doc = Nokogiri::HTML(open("https://www.wikihow.com/Main-Page"))
+    categories_array = []
+    doc.search("#hp_categories a").each do |category|
+      title = category.text
+      url = category.attr("href")
+      categories_array << {:title => title,:url => url}
+    end
+    categories_array
+  end
+
+  def self.scrape_for_topics(category)
+    doc = Nokogiri::HTML(open("https://www.wikihow.com" + category.url))
+    topics_array = []
+    doc.search("#cat_container #cat_all a").each do |topic|
+      title = topic.search("span").text.strip
+      url = topic.attr("href")
+      topics_array << {:title => title,:url => url} if title != ""
+    end
+    topics_array
+  end
+
+  def self.scrape_topic(topic)
+    doc = Nokogiri::HTML(open(topic.url))
+
+    topic.intro = doc.search("#intro p").last.text
+    sections_array = []
+    doc.search("#intro #method_toc .toc_method").each do |method|
+      sections_array << {:section_title => method.text, :section_steps => []}
+    end
+
+    sections_array.each.with_index do |section, i|
+      doc.search(".steps_list_2")[i].search(".step").each do |section_li|
+        step_description = [section_li.search(".whb").text.strip + " " + section_li.search("> text()").text.strip]
+        section_li.search("> ul > li").each do |step_li|
+          bullet_point = [step_li.search("> text(), a").text.strip]
+          sub_bullet_point = step_li.search("> ul > li").collect {|bullet_point_li|bullet_point_li.search("> text()").text.strip}
+          bullet_point << sub_bullet_point if sub_bullet_point !=[]
+          step_description << bullet_point if bullet_point != []
+        end
+        section[:section_steps] << step_description
+      end
+    end
+    sections_array
+  end
+end

data/lib/wikihow/topic.rb

@@ -15,7 +15,7 @@ class Wikihow::Topic
 
   def sections
     if @sections == []
-      @sections = self.scrape_topic
+      @sections = Wikihow::Scraper.scrape_topic(self)
     end
     @sections
   end
@@ -24,53 +24,10 @@ class Wikihow::Topic
     @sections = sections
   end
 
-  def scrape_topic
-    doc = Nokogiri::HTML(open(self.url))
-
-    self.intro = doc.search("#intro p").last.text
-    sections_array = []
-    doc.search("#intro #method_toc .toc_method").each do |method|
-      sections_array << {:section_title => method.text, :section_steps => []}
-    end
-
-    sections_array.each.with_index do |section, i|
-      doc.search(".steps_list_2")[i].search(".step").each do |section_li|
-        step_description = [section_li.search(".whb").text.strip + " " + section_li.search("> text()").text.strip]
-        section_li.search("> ul > li").each do |step_li|
-          bullet_point = [step_li.search("> text(), a").text.strip]
-          sub_bullet_point = step_li.search("> ul > li").collect {|bullet_point_li|bullet_point_li.search("> text()").text.strip}
-          bullet_point << sub_bullet_point if sub_bullet_point !=[]
-          step_description << bullet_point if bullet_point != []
-        end
-        section[:section_steps] << step_description
-      end
-    end
-    sections_array
-  end
-
-  # def self.sentence_to_snake_case(string)
-  #   string.gsub(" ","_")
-  # end
-  #
-  # def self.snake_case_to_sentence(string)
-  #   string.gsub("_"," ")
-  # end
-
   def self.get_or_create_topics_from_category(category)
     if category.topics == []
-      self.scrape_for_topics(category).each{|topic_hash|self.new(topic_hash, category)}
+      Wikihow::Scraper.scrape_for_topics(category).each{|topic_hash|self.new(topic_hash, category)}
     end
     category.topics
   end
-
-  def self.scrape_for_topics(category)
-    doc = Nokogiri::HTML(open("https://www.wikihow.com" + category.url))
-    topics_array = []
-    doc.search("#cat_container #cat_all a").each do |topic|
-      title = topic.search("span").text.strip
-      url = topic.attr("href")
-      topics_array << {:title => title,:url => url} if title != ""
-    end
-    topics_array
-  end
 end

data/lib/wikihow/version.rb

@@ -1,3 +1,3 @@
 module Wikihow
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

data/lib/wikihow.rb

@@ -5,3 +5,4 @@ require_relative './wikihow/version'
 require_relative './wikihow/cli'
 require_relative './wikihow/category'
 require_relative './wikihow/topic'
+require_relative './wikihow/scraper'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: wikihow
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - "'Jared Grove'"
@@ -90,6 +90,7 @@ files:
 - lib/wikihow.rb
 - lib/wikihow/category.rb
 - lib/wikihow/cli.rb
+- lib/wikihow/scraper.rb
 - lib/wikihow/topic.rb
 - lib/wikihow/version.rb
 - wikihow.gemspec