scraping 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b9aff8bf11dbde49763faa84c830518cb308daab
-  data.tar.gz: ba701a4490c085c2278ff93efbdeb244a9b74b12
+  metadata.gz: d078337c4224a96b2587427a06556877d69eb656
+  data.tar.gz: dc8bdf3cc7a53568c99d869dfdda769c17c79d7b
 SHA512:
-  metadata.gz: b6bc072d8df959e32b8cd5230db684ebffdb08d23ce3a2633a683042eed6f92a5f0c8e61b4c35ecd570975461d417a93f6941b1688d328eafb76cf958f247627
-  data.tar.gz: 0149081ebc7cd6073e2a178f10c1904e46a3e3cab8e2fa9f35fc82387b1a7938d4f48428a9eb58d1adc835680c5dc0248b76bd562fb7b1e8288cc24efca46154
+  metadata.gz: ca76e4ac712fc15fea39a3f7d76298bf20e651410ec8a515a63345d4563740e2f59caede6f6b0a314cbb7335c7a72888c336ef0ef18bd32ee10bfe7fc21da921
+  data.tar.gz: c42df98a5a21a72b6cb7397dd34a099ec40ef36567c3f7a7b76dfcb3a4a864c8fe034276dd6ecf53d347e442292e939471c43ac0704d4080c302062f40dfdcb9

data/README.md

@@ -35,15 +35,16 @@ You can also scrape arrays, objects, and arrays of objects. `elements` and `elem
 ```ruby
 class YouCan
   include Scraping
+
   elements :scrape, '.scrape'
 
-  elements :also_scrape, '.also-scrape li' do
+  sections :also_scrape, '.also-scrape li' do
     element :name, 'a'
     element :link, 'a/@href'
     elements :numbers, 'span'
   end
 
-  elements_of :nested_scrape do
+  section :nested_scrape do
     element :data, '.data'
   end
 end
@@ -89,6 +90,10 @@ class Advanced
 
   element :birthday, '.birthday', as: :date
 
+  elements :numbers, 'span' do |node|
+    node.text.to_i * 10
+  end
+
   private
 
   def extract_date(node)
@@ -99,15 +104,39 @@ end
 advanced = Advanced.new(<<-EOF)
   <h1 class="name">Millard Fillmore</h1>
   <h2 class="birthday">7-1-1800</h2>
+  <span>1</span>
+  <span>2</span>
 EOF
 
 advanced.first_name #=> 'Millard'
 advanced.birthday #=> #<Date: 1800-01-07>
+advanced.numbers #=> [10, 20]
+```
+
+## HTTP
+
+Scraping is totally agnostic of HTTP, but if you need a suggestion, check out [HTTParty](https://github.com/jnunemaker/httparty).
+
+```ruby
+class HackerNews
+  include HTTParty
+  include Scraping
+
+  base_uri 'https://news.ycombinator.com'
+  elements :stories, '.athing .title > a'
+
+  def self.scrape
+    super get('/').body
+  end
+end
+
+news = HackerNews.scrape
+puts news.stories.inspect
 ```
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/scraping.
+Bug reports and pull requests are welcome on GitHub at https://github.com/promptworks/scraping.
 
 ## License
 

data/lib/scraping.rb

@@ -2,8 +2,9 @@ require 'nokogiri'
 require 'scraping/version'
 require 'scraping/dsl'
 require 'scraping/rules/element'
-require 'scraping/rules/elements_of'
 require 'scraping/rules/elements'
+require 'scraping/rules/section'
+require 'scraping/rules/sections'
 
 module Scraping
   def self.included(base)
@@ -38,12 +39,17 @@ module Scraping
       super
     end
 
-    def elements_of(name)
+    def elements(name, *)
       attr_accessor name
       super
     end
 
-    def elements(name, *)
+    def section(name, *)
+      attr_accessor name
+      super
+    end
+
+    def sections(name, *)
       attr_accessor name
       super
     end

data/lib/scraping/dsl.rb

@@ -8,12 +8,16 @@ module Scraping
       rules[name] = Rules::Element.new(name, selector, options, &block)
     end
 
-    def elements_of(name, &block)
-      rules[name] = Rules::ElementsOf.new(name).evaluate(&block)
+    def elements(name, selector, options = {}, &block)
+      rules[name] = Rules::Elements.new(name, selector, options, &block)
     end
 
-    def elements(name, selector, options = {}, &block)
-      rules[name] = Rules::Elements.new(name, selector, options).evaluate(&block)
+    def section(name, selector = '.', &block)
+      rules[name] = Rules::Section.new(name, selector).evaluate(&block)
+    end
+
+    def sections(name, selector, &block)
+      rules[name] = Rules::Sections.new(name, selector).evaluate(&block)
     end
   end
 end

data/lib/scraping/rules/elements.rb

@@ -1,27 +1,18 @@
+require 'scraping/rules/element'
+
 module Scraping
   module Rules
-    class Elements
-      attr_reader :name, :selector, :rule, :options
-
-      def initialize(name, selector, options = {})
-        @name = name
-        @selector = selector
-        @options = options
-      end
-
-      def evaluate(&block)
-        if block_given?
-          @rule = ElementsOf.new(name).evaluate(&block)
-        else
-          @rule = Element.new(name, '.', options)
-        end
+    class Elements < Element
+      attr_reader :multiselector
 
-        self
+      def initialize(name, selector, options = {}, &extract)
+        super(name, '.', options, &extract)
+        @multiselector = selector
       end
 
       def call(scraper, node)
-        node.search(selector).map do |item|
-          rule.call(scraper, item)
+        node.search(multiselector).map do |item|
+          super scraper, item
         end
       end
     end

data/lib/scraping/rules/{elements_of.rb → section.rb}

@@ -2,12 +2,13 @@ require 'ostruct'
 
 module Scraping
   module Rules
-    class ElementsOf
+    class Section
       include DSL
-      attr_reader :name
+      attr_reader :name, :selector
 
-      def initialize(name)
+      def initialize(name, selector = '.')
         @name = name
+        @selector = selector
       end
 
       def evaluate(&block)
@@ -17,7 +18,7 @@ module Scraping
 
       def call(scraper, node)
         rules.inject(OpenStruct.new) do |obj, (name, rule)|
-          obj[name] = rule.call(scraper, node)
+          obj[name] = rule.call scraper, node.at(selector)
           obj
         end
       end

data/lib/scraping/rules/sections.rb

@@ -0,0 +1,20 @@
+require 'scraping/rules/section'
+
+module Scraping
+  module Rules
+    class Sections < Section
+      attr_reader :multiselector
+
+      def initialize(name, selector)
+        super name, '.'
+        @multiselector = selector
+      end
+
+      def call(scraper, node)
+        node.search(multiselector).map do |item|
+          super scraper, item
+        end
+      end
+    end
+  end
+end

data/lib/scraping/version.rb

@@ -1,3 +1,3 @@
 module Scraping
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scraping
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Ray Zane
@@ -83,7 +83,8 @@ files:
 - lib/scraping/dsl.rb
 - lib/scraping/rules/element.rb
 - lib/scraping/rules/elements.rb
-- lib/scraping/rules/elements_of.rb
+- lib/scraping/rules/section.rb
+- lib/scraping/rules/sections.rb
 - lib/scraping/version.rb
 - scraping.gemspec
 homepage: https://github.com/rzane/scraping