web_scraper 1.0.1 → 1.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a3e01f01d20813809f915bc0b97280f3d35c2153
-  data.tar.gz: e73cf357e0151f499414ba21ce178a8724271e9c
+  metadata.gz: 33e7eb69bd6cea28712f7f778792cb45539ad6d4
+  data.tar.gz: 46874fa0d02cf37e67bbf22ce28d457c51126535
 SHA512:
-  metadata.gz: d38b93e448b86791f93918226d9dbadd6304667b4b7a2c1ef794802eb47cbc21d98331d2af86ccebb0d5011c36d0578bb861a7e8663c91e832c77a2356d0ba8d
-  data.tar.gz: 4c5213c3d410bb89ad093da6fc9f578c637e6bade5dbf94b5d4e93a891c1b94ef9ce6dbb9d35197bd0b728bb11e20076d3a21837e18120e6660a60d157cc150b
+  metadata.gz: bee62dc14b4540bbeb5f975a0a88eea729c1f4cb815954b61be533d04c4a94046ce54bf0bdba94d728e2281b26e6ff01683d627be5a9f63e3b5cbf5351dbf617
+  data.tar.gz: db2358697c31df901f36a8e6dedd0a140bc3c1af65d83dab07ccafbb3f670b70c8dc8c278ae718359e98bd1e1e6d52c7e030624deeb754742becea7047ac34c4

data/lib/web_scraper.rb

@@ -1,25 +1,74 @@
 require 'open-uri'
 require 'nokogiri'
 
+##
+# WebScraper allows you to describe html structure declaratively,
+# get appropriate blocks, and work with them as with ruby objects.
+# @example
+#  class Article < WebScraper
+#    resource 'http://hbswk.hbs.edu/topics/it.html'
+#
+#    base css: '.tile-medium'
+#
+#    property :title,       xpath: './/h4/a/text()'
+#    property :date,        xpath: './/li[1]/text()'
+#    property :category,    xpath: './/li[2]/a/text()'
+#    property :description, xpath: './/p/text()'
+#
+#    key :title
+#  end
+#
+#  puts "#{Article.count} articles were found"
+#  puts
+#
+#  articles = Article.all
+#
+#  articles.each do |article|
+#    header = article.title
+#    puts header
+#    puts '=' * header.length
+#    puts
+#
+#    subheader = "#{article.date} #{article.category}"
+#    puts subheader
+#    puts '-' * subheader.length
+#    puts
+#
+#    puts article.description
+#    puts
+#  end
+#
+#  article =  Article.find('Tech Investment the Wise Way')
+#
+#  puts article.description
 class WebScraper
+  ##
+  # The error raises when a user tries to call a class method
+  # when not all required attributes were defined.
   class ConfigurationError < RuntimeError
     def message
       'resource, base, properties and key should be defined'
     end
   end
 
+  ##
+  # The error raises when a user tries to define resource improperly.
   class ResourceDefentitionError < RuntimeError
     def message
       'resource should be a string'
     end
   end
 
+  ##
+  # The error raises when a user tries to define base improperly.
   class BaseDefentitionError < RuntimeError
     def message
       'base should be a selector (:css|:xpath => String)'
     end
   end
 
+  ##
+  # The error raises when a user tries to define propery improperly.
   class PropertyDefentitionError < RuntimeError
     def message
       'property is a name (with type optionally) ' +
@@ -27,6 +76,8 @@ class WebScraper
     end
   end
 
+  ##
+  # The error raises when a user tries to define key improperly.
   class KeyDefentitionError < RuntimeError
     def message
       'key should be a name of a defined property'
@@ -34,6 +85,12 @@ class WebScraper
   end
 
   class << self
+    ##
+    # Loads html page, detects appropriate blocks,
+    # wraps them in objects.
+    # The result will be cached.
+    # @example
+    #  articles = Article.all
     def all
       raise ConfigurationError unless valid?
 
@@ -41,18 +98,38 @@ class WebScraper
                .send(*_base).map { |node| new(node) }
     end
 
+    ##
+    # Returns number of objects found.
+    # @example
+    #  puts "#{Article.count} articles were found"
     def count
       all.size
     end
 
-    def expire
+    ##
+    # Resets cache of the html data.
+    # @example
+    #  Article.reset
+    def reset
       @all = nil
     end
 
+    ##
+    # Finds first object with required key.
+    # @example
+    #  article = Article.find('Tech Investment the Wise Way')
     def find(key)
       all.find { |e| e.send(_key) == key }
     end
 
+    ##
+    # Defines resource -- url of the html page.
+    # @example
+    #  class Article < WebScraper
+    #    ...
+    #    resource 'http://hbswk.hbs.edu/topics/it.html'
+    #    ...
+    #  end
     def resource(_resource)
       raise ResourceDefentitionError unless _resource.is_a? String
 
@@ -61,6 +138,15 @@ class WebScraper
 
     attr_reader :_resource
 
+    ##
+    # Defines base -- selector which determines blocks of content.
+    # You can use css or xpath selectors.
+    # @example
+    #  class Article < WebScraper
+    #    ...
+    #    base css: '.tile-medium'
+    #    ...
+    #  end
     def base(_base)
       raise BaseDefentitionError unless valid_selector? _base
 
@@ -69,6 +155,19 @@ class WebScraper
 
     attr_reader :_base
 
+    ##
+    # Defines property -- name (and type optionally) and selector.
+    # You can use css or xpath selectors.
+    # Types determine returning values.
+    # Available types (default is string): string, integer, float, node.
+    # The node option means nokogiri node.
+    # @example
+    #  class Article < WebScraper
+    #    ...
+    #    property :title,           xpath: './/h4/a/text()'
+    #    property  views: :integer, xpath: './/h4/span/text()'
+    #    ...
+    #  end
     def property(*args)
       @properties ||= {}
 
@@ -101,6 +200,14 @@ class WebScraper
 
     attr_reader :properties
 
+    ##
+    # Defines key -- property which will be used in find method.
+    # @example
+    #  class Article < WebScraper
+    #    ...
+    #    key :title
+    #    ...
+    #  end
     def key(_key)
       raise KeyDefentitionError unless properties.keys.include? _key
 
@@ -109,10 +216,14 @@ class WebScraper
 
     attr_reader :_key
 
+    ##
+    # Checks if all attributes were set.
     def valid?
       _resource && _base && _key
     end
 
+    ##
+    # Checks if selector was defined correctly.
     def valid_selector?(selector)
       (selector.is_a? Hash) &&
       (selector.size == 1) &&
@@ -120,6 +231,8 @@ class WebScraper
       (selector.values.first.is_a? String)
     end
 
+    ##
+    # Checks if property information (i.e. name and type) were defined correctly.
     def valid_info?(info)
       (info.is_a? Hash) &&
       (info.size == 1) &&
@@ -130,20 +243,31 @@ class WebScraper
     private :new
   end
 
+  ##
+  # Sets nokogiri node. It's private method.
   def initialize(node)
     @node = node
   end
 
-  attr_reader :node
-
+  ##
+  # Allows you to use nokogiri css method directly on your object.
+  # It proxies it to nokogiri node.
   def css(*args)
-    node.css(*args)
+    @node.css(*args)
   end
 
+  ##
+  # Allows you to use nokogiri xpath method directly on your object.
+  # It proxies it to nokogiri node.
   def xpath(*args)
-    node.xpath(*args)
+    @node.xpath(*args)
   end
 
+  ##
+  # Returns appropriate value for property if found.
+  # Converts it to the defined type.
+  # @example
+  #  puts article.description
   def method_missing(name, *args, &block)
     if self.class.properties.key? name
       property = self.class.properties[name]

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: web_scraper
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.0.2
 platform: ruby
 authors:
 - Speransky Danil
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-03-16 00:00:00.000000000 Z
+date: 2014-03-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -24,7 +24,8 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: ''
+description: Web Scraper is a library to build APIs by scraping static sites and use
+  data as models.
 email: speranskydanil@gmail.com
 executables: []
 extensions: []
@@ -54,6 +55,7 @@ rubyforge_project:
 rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
-summary: ''
+summary: Web Scraper is a library to build APIs by scraping static sites and use data
+  as models.
 test_files: []
 has_rdoc: