web_parser 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 5460b12d5b804540dbe145c157897d92eeaa68ea
+  data.tar.gz: 79d9afacf57e5424b808cc90fc7e1fafa9549170
+SHA512:
+  metadata.gz: cabaeb1fe86d0e180642245630176bbcec94a797cf7a53fe036baf89396dfbc701db93f522fb793d2796902dc1ff676fc7a7f26a0a689a5f61499f55d5531a86
+  data.tar.gz: 1f8209b5ab0164d5759dab4bd70dac5ccb51af5d63906c506acd386f50ed3d3fc22457f178dc0110df28fd9a669bd37f3a888f335af093386428899bc4bb3709

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in web_parser.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Daniel Mrozek
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,116 @@
+# WebParser
+
+Simple gem for easy information fetching from web pages.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'web_parser'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install web_parser
+
+## Example usage
+
+Just write your own class and include WebParser::Doc.
+
+```ruby
+class YahooSearch
+  # include WebParser
+  include WebParser::Doc
+
+  # define recipes
+  recipes do
+    # simplest way to define recipe
+    query          :xpath, '//input[@id="yschsp"]/@value'
+    # You can use simple normalization as last parameter
+    query_downcase :xpath, '//input[@id="yschsp"]/@value',
+                   ->(value) { value.text.downcase }
+    # You can also provide just method name for normalization
+    page_number    :css, '#pg > strong', :to_i
+    # Or you can do whatever you want to obtain value, just provide lambda as
+    # parameter
+    first_page?    :lambda, ->(doc) {
+      doc.css('#pg > strong').text.to_i == 1
+    }
+    # Nesting
+    right_links do
+      sign_in :css, '#yucs-profile', :strip
+      mail    :css, '#yucs-mail_link_id', :strip
+    end
+    # Array, usefull for example when parsing eshops
+    results :css, '#web > ol > li' do
+      name :css, '> .res h3'
+      url  :xpath, './/h3[1]/a/@href'
+    end
+  end
+end
+```
+
+Then just initialize your class and call `parse`.
+
+```ruby
+require 'open-uri'
+
+html_page = open('http://search.yahoo.com/search?p=Ruby').read
+
+YahooSearch.new(html_page).parse
+=> {:query=>"Ruby",
+    :query_downcase=>"ruby",
+    :page_number=>1,
+    :first_page?=>true,
+    :right_links=>{:sign_in=>"Sign In", :mail=>"Mail"},
+    :results=>
+      [
+        {
+          :name=>"Ruby Programming Language",
+          :url=>"https://www.ruby-lang.org/en/" },
+        {
+          :name=>"Ruby - Wikipedia, the free encyclopedia",
+          :url=>"http://en.wikipedia.org/wiki/Ruby"},
+        {
+          :name=>"Ruby - Image Results",
+          :url=>"http://images.search.yahoo.com/search/images?_adv_prop=image&va=Ruby"},
+        {
+          :name=>"Ruby (programming language) - Wikipedia, the free
+                encyclopedia",
+          :url=>"http://en.wikipedia.org/wiki/Ruby_(programming_language)"},
+        {
+          :name=>"‘Ruby’ Today: Reality Star Dishes on Show’s Failure ...",
+          :url=>"http://abcnews.go.com/blogs/entertainment/2013/01/ruby-today-reality-star-dishes-on-shows-failure/"},
+        {
+          :name=>"Download Ruby",
+          :url=>"https://www.ruby-lang.org/en/downloads/"},
+        {
+          :name=>"Ruby: The gemstone Ruby information and pictures",
+          :url=>"http://www.minerals.net/gemstone/ruby_gemstone.aspx"},
+        {
+          :name=>"Ruby - Gemstone",
+          :url=>"http://www.gemstone.org/index.php?option=com_content&view=article&id=85:ruby&catid=1:gem-by-gem&Itemid=14"},
+        {
+          :name=>"Buy Loose Precious Ruby Gemstones at Wholesale Prices from ...",
+          :url=>"http://www.gemselect.com/ruby/ruby.php"},
+        {
+          :name=>"Ruby on Rails",
+          :url=>"http://rubyonrails.org/"},
+        {
+          :name=>"Ruby (Adventures) - Bulbapedia, the community-driven Pokémon ...",
+          :url=>"http://bulbapedia.bulbagarden.net/wiki/Ruby_(Adventures)"}
+      ]
+    }
+
+```
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,14 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+
+require './lib/web_parser.rb'
+
+task :console do
+  require 'pry'
+  ARGV.clear
+  Pry.start WebParser
+end
+task :c => :console
+

data/examples/yahoo_search.rb

@@ -0,0 +1,43 @@
+require 'open-uri'
+$LOAD_PATH.unshift "#{File.dirname(__FILE__)}/../lib"
+require 'web_parser'
+
+class YahooSearch
+  # include WebParser
+  include WebParser::Doc
+
+  # define recipes
+  recipes do
+    # simplest way to define recipe
+    query          :xpath, '//input[@id="yschsp"]/@value'
+    # You can use simple normalization as last parameter
+    query_downcase :xpath, '//input[@id="yschsp"]/@value',
+                   ->(value) { value.text.downcase }
+    # You can also provide just method name for normalization
+    page_number    :css, '#pg > strong', :to_i
+    # Or you can do whatever you want to obtain value, just provide lambda as
+    # parameter
+    first_page?    :lambda, ->(doc) {
+      doc.css('#pg > strong').text.to_i == 1
+    }
+    # Nesting
+    right_links do
+      sign_in :css, '#yucs-profile', :strip
+      mail    :css, '#yucs-mail_link_id', :strip
+    end
+    # Array, usefull for example when parsing eshops
+    results :css, '#web > ol > li' do
+      name :css, '> .res h3'
+      url  :xpath, './/h3[1]/a/@href'
+    end
+  end
+end
+
+html_page = open('http://search.yahoo.com/search?p=Ruby').read
+
+rslt = YahooSearch.new(html_page).parse
+
+require 'pp'
+
+pp rslt
+

data/lib/web_parser/doc.rb

@@ -0,0 +1,58 @@
+module WebParser
+  module Doc
+    class XPathsNotSet < StandardError; end
+
+    def self.included base
+      base.extend ClassMethods
+    end
+
+    module ClassMethods
+      def recipes &block
+        @recipes = Recipes.new(&block) if block_given?
+        @recipes
+      end
+    end
+
+    # @example
+    #   recipes do
+    #     id          :css,   '.title > h1'
+    #     description :xpath, '.desc > p', ->(value) { value.gsub('-', '') }
+    #     price       :xpath, '.price > .vat', :to_f
+    #     summary     :lambda, ->(doc) {
+    #       doc.css('a#total_downloads').gsub(',', '').to_f
+    #     }
+    #     additional_info do
+    #       vat :css, '.vat', :to_i
+    #       fee :css, '.fee', :to_f
+    #     end
+    #     categories :css, '.categories > .category', :to_s do
+    #       name :xpath, './li/a'
+    #       url  :xpath, './li/a/@href'
+    #       count :lambda, ->(doc) {
+    #         doc.xpath('./li/a').text =~ /(\d+)$/ && $1
+    #       }
+    #     end
+    #   end
+
+    # Creates a new page parser
+    # @param [String] doc Nokogiri object with page we would parsing
+    def initialize doc, parser=Nokogiri::HTML
+      @doc = parser.parse(doc)
+      raise XPathsNotSet, "no recipes defined!" unless self.class.recipes
+    end
+
+    # Main method for parsing document
+    # @return [Hash] Parsed informations from page in a hash
+    def parse
+      get_parsed
+    end
+
+    private
+
+    def get_parsed
+      self.class.recipes.apply(@doc)
+    end
+  end
+
+end
+

data/lib/web_parser/recipe.rb

@@ -0,0 +1,32 @@
+module WebParser
+  class Recipe
+    attr_reader :type, :value, :normalize_method
+
+    def initialize name, *args
+      @name, @type, @value, @normalize_method = name, *args
+      @value, @type = @type, @value if args.size == 1
+    end
+
+    def apply doc
+      case @type
+      when :val    then @value
+      when :css    then normalize(doc.css(@value))
+      when :xpath  then normalize(doc.xpath(@value))
+      when :lambda then @value.call(doc)
+      else raise "uknown recipe type '#{@type}'!"
+      end
+    end
+
+    private
+
+    def normalize value
+      if @normalize_method.respond_to?(:call)
+        @normalize_method.call(value)
+      else
+        value = value.text.gsub("\u00a0", ' ') # replace nbsp with normal space
+        @normalize_method ? value.send(@normalize_method) : value
+      end
+    end
+  end
+end
+

data/lib/web_parser/recipes.rb

@@ -0,0 +1,50 @@
+module WebParser
+  class Recipes
+    attr_reader :recipes, :type, :recipe
+
+    def initialize type=:single, recipe=nil, &block
+      raise "recipe must be set for type :each" if type == :each && !recipe
+      @recipes, @type, @recipe = {}, type, recipe
+      instance_eval(&block) if block_given?
+    end
+
+    def apply doc
+      recipes.inject({}) do |mem, (name, val)|
+        if val.is_a?(Recipe)
+          mem[name] = apply_recipe(doc, val)
+        elsif val.is_a?(Recipes)
+          mem[name] = apply_recipes(doc, val)
+        end
+        mem
+      end
+    end
+
+    def method_missing name, *args, &block
+      if    block_given? && args.size == 0
+        @recipes[name] = Recipes.new(&block)
+      elsif block_given? && (args.size == 2 || args.size == 3)
+        args[2] ||= -> (val) { val } # just return array of elements
+        @recipes[name] = Recipes.new(:each, Recipe.new(name, *args), &block)
+      elsif !block_given? && (args.size == 2 || args.size == 3)
+        @recipes[name] = Recipe.new(name, *args, &block)
+      else
+        super
+      end
+    end
+
+    private
+
+    def apply_recipe doc, recipe
+      recipe.apply(doc)
+    end
+
+    def apply_recipes doc, recipes
+      if recipes.type == :each
+        recipes.recipe.apply(doc).map {|subdoc| recipes.apply(subdoc) }
+      else
+        recipes.apply(doc)
+      end
+    end
+  end
+end
+

data/lib/web_parser/version.rb

@@ -0,0 +1,3 @@
+module WebParser
+  VERSION = "0.0.1"
+end

data/lib/web_parser.rb

@@ -0,0 +1,11 @@
+require "web_parser/version"
+
+require 'nokogiri'
+
+require_relative 'web_parser/recipe'
+require_relative 'web_parser/recipes'
+require_relative 'web_parser/doc'
+
+module WebParser
+end
+

data/web_parser.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'web_parser/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "web_parser"
+  spec.version       = WebParser::VERSION
+  spec.authors       = ["Daniel Mrozek"]
+  spec.email         = ["mrazicz@gmail.com"]
+  spec.description   = %q{Simple gem for easy web page parsing.}
+  spec.summary       = %q{Simple gem for easy web page parsing. Just set xpaths, provide HTML, and get hash with informations.}
+  spec.homepage      = "https://github.com/mrazicz/web_parser"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "nokogiri", "~> 1.6.3.1"
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "pry"
+  spec.add_development_dependency "rake"
+end

metadata

@@ -0,0 +1,114 @@
+--- !ruby/object:Gem::Specification
+name: web_parser
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Daniel Mrozek
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-08-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.6.3.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.6.3.1
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Simple gem for easy web page parsing.
+email:
+- mrazicz@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- examples/yahoo_search.rb
+- lib/web_parser.rb
+- lib/web_parser/doc.rb
+- lib/web_parser/recipe.rb
+- lib/web_parser/recipes.rb
+- lib/web_parser/version.rb
+- web_parser.gemspec
+homepage: https://github.com/mrazicz/web_parser
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.1.4
+signing_key: 
+specification_version: 4
+summary: Simple gem for easy web page parsing. Just set xpaths, provide HTML, and
+  get hash with informations.
+test_files: []
+has_rdoc: