ninja2k 0.0.1

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Randy Morgan
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,81 @@
+# Ninja2k
+
+This gem is a tool for Ninja2k that allows you to scrape Nokogiri parsable resources for specified
+clues and add hooks to define how those clues are processed. It also
+lets you export the results into an xlsx file.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'ninja2k'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install ninja2k
+
+## Usage
+
+### Basic Scraping
+
+    require 'ninja2k'
+
+    clues = ['Operating system', 'Processors', 'Chipset', 'Memory type', 'Hard drive', 'Graphics',
+             'Ports', 'Webcam', 'Pointing device', 'Keyboard', 'Network interface', 'Chipset', 'Wireless',
+             'Power supply type', 'Energy efficiency', 'Weight', 'Minimum dimensions (W x D x H)',
+             'Warranty', 'Software included', 'Product color']
+
+    url =  "http://h10010.www1.hp.com/wwpc/ie/en/ho/WF06b/321957-321957-3329742-89318-89318-5186820-5231694.html?dnr=1"
+    selector =  "//td[text()='%s']/following-sibling::td"
+
+    scraper = Ninja2k::Scraper.new(url, selector, :clues => clues)
+    scraper.to_xlsx('my_spreadsheet.xlsx')
+
+
+### With Hooks and Styles
+
+    require 'ninja2k'
+
+    clues = ['Operating system', 'Processors', 'Chipset', 'Memory type', 'Hard drive', 'Graphics',
+             'Ports', 'Webcam', 'Pointing device', 'Keyboard', 'Network interface', 'Chipset', 'Wireless',
+             'Power supply type', 'Energy efficiency', 'Weight', 'Minimum dimensions (W x D x H)',
+             'Warranty', 'Software included', 'Product color']
+
+    url =  "http://h10010.www1.hp.com/wwpc/ie/en/ho/WF06b/321957-321957-3329742-89318-89318-5186820-5231694.html?dnr=1"
+    selector =  "//td[text()='%s']/following-sibling::td"
+
+    os_hook =  Proc.new do |element|
+                 element.inner_html.split('<br>').each do |datum|
+                   datum.strip!.upcase!
+                 end
+               end
+
+
+    scraper = Ninja2k::Scraper.new(url, selector, :clues => clues, :hooks => { 'Operating system' => os_hook })
+
+    # You can also alter the xlsx spreadsheet before serializing.
+    # See https://github.com/randym/axlsx
+
+    package = scraper.to_xlsx
+    clue_style = package.workbook.styles.add_style :fg_color => 'FF0000'
+    package.workbook.worksheets.first.col_style(0, clue_style)
+    package.serialize('styled.xlsx')
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+## Copyright and License
+----------
+
+Ninja2k &copy; 2012 by [Randy Morgan](mailto:digial.ipseity@gmail.com). 
+
+Ninja2k is licensed under the MIT license. Please see the LICENSE document for more information.

data/Rakefile

@@ -0,0 +1,13 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+task :test do
+     require 'rake/testtask'
+     Rake::TestTask.new do |t|
+       t.libs << 'test'
+       t.test_files = FileList['test/**/tc_*.rb']
+       t.verbose = false
+       t.warning = true
+     end
+end
+
+task :default => :test

data/lib/ninja2k/scraper.rb

@@ -0,0 +1,168 @@
+require 'nokogiri'
+require 'open-uri'
+require 'axlsx'
+
+module Ninja2k
+
+
+  # Scraper will load up a specified resource, and search the page using a combination of your seletor and any clues given.
+  # It provides a hooking mechanism so you can override the default parsing action (split on <br>, one row for each item found)
+  #
+  # @example 
+  #     clues = ['Operating system', 'Processors', 'Chipset', 'Memory type', 'Hard drive', 'Graphics',
+  #       'Ports', 'Webcam', 'Pointing device', 'Keyboard', 'Network interface', 'Chipset', 'Wireless',
+  #       'Power supply type', 'Energy efficiency', 'Weight', 'Minimum dimensions (W x D x H)',
+  #       'Warranty', 'Software included', 'Product color']
+  #
+  #     url =  "http://h10010.www1.hp.com/wwpc/ie/en/ho/WF06b/321957-321957-3329742-89318-89318-5186820-5231694.html?dnr=1"
+  #     selector =  "//td[text()='%s']/following-sibling::td"
+  #
+  #     scraper = Ninja2k::Scraper.new(url, selector, :clues => clues)
+  #     scraper.to_xlsx('my_spreadsheet.xlsx')
+  class Scraper
+
+    # Creates a new Scraper
+    #
+    # @param [String] url The resource to scrape
+    #
+    # @param [String] selector The xpath select to use when searching for clues. Use %s in the selector to interpolate each clue
+    #
+    # @param [Hash] options each option will be evaluated against a attr_writer using respond_to? If a writer exists, the value for the option is passed to the writer.
+    #
+    # @option [Array] clues The clues to search for
+    #
+    # @option [Hash] hooks A hash of hooks where the key is the clue name the Proc value will be caled against.
+    def initialize(url, selector, options={})
+      self.url = url
+      self.selector = selector
+      options.each do |o|
+        self.send("#{o[0]}=", o[1]) if self.respond_to? "#{o[0]}="
+      end
+    end
+
+    # The url we will scrape from
+    # @return [String]
+    attr_accessor :url
+
+    # The xpath selector to use when searching for clues
+    # @return [String]
+    attr_accessor :selector
+
+    # The output from scraping as an array
+    # This is populated by the scrape or to_xlsx methods
+    #
+    # @return [Array]
+    def output
+      @output ||= []
+    end
+
+    # A hash of Proc object to call when parsing each item found by the selector and clue combination.
+    # The element found will be passed to the member of this hash that uses the clue as a key
+    #
+    # @see example/example.rb
+    #
+    # @return [Hash]
+    def hooks
+      @hooks ||= {}
+    end
+
+    # @see hooks
+    def hooks=(hash)
+      raise ArgumentError, 'Hooks must be a hash of procs to call when scraping each clue' unless hash.is_a?(Hash)
+      @hooks = hash
+    end
+
+    # Adds a hook to the hook hash
+    #
+    # @param [String] clue the clue this hook will be called for
+    #
+    # @param [Proc] p_roc the Proc to call when the clue is found
+    def add_hook(clue, p_roc)
+      hooks[clue] = p_roc
+    end
+
+    # Scrapes the resourse using the clues and hooks provided
+    #
+    # @return [Array]
+    def scrape
+      @package = nil
+      @output = []
+      clues.each do |clue|
+        if detail = parse_clue(clue)
+          output << [clue, detail.pop]
+          detail.each { |datum| output << ['', datum] }
+        end
+      end
+      output
+    end
+
+    # seralizes the output to xlsx. If you do not specify the file_name parameter
+    # The package will be created, but not serialized to disk. This means you can use the return value
+    # to stream the data using to_xlsx(false).to_stream.read
+    #
+    # @param [String] filename the filename to use in output
+    #
+    # @return [Axlsx::Package]
+    def to_xlsx(filename=false)
+      scrape
+      serialize(filename)
+    end
+    
+    # The clues we are going to look for with the selector in the document returned by url
+    #
+    # @return [Array]
+    def clues
+      @clues ||= []
+    end
+
+    # Sets the clues for the scraper
+    #
+    # @param [Arrray] value The clues to look for.
+    def clues=(value)
+      raise ArugmentError, 'clues must be an array of strings to search for with your selector' unless value.is_a?(Array)
+      @clues = value
+    end
+
+    # The axlsx package used for xlsx serialization
+    #
+    # @return [Axlsx::Package]
+    def package
+      @package ||= Axlsx::Package.new
+    end
+
+    private
+
+    def doc
+      @doc ||= begin 
+                 Nokogiri::HTML(open(@url))
+               rescue
+                 raise ArgumentError, 'Invalid URL - Nothing to parse'
+               end
+    end
+
+    def selector_for_clue(clue)
+      @selector % clue
+    end
+
+    def parse_clue(clue)
+      if element = doc.at(selector_for_clue(clue))
+        call_hook(clue, element) || element.inner_html.split('<br>').each(&:strip!)
+      end
+    end
+
+    def call_hook(clue, element)
+      if hooks[clue].is_a? Proc
+        value = hooks[clue].call(element)
+        value.is_a?(Array) ? value : [value]
+      end
+    end
+
+    def serialize(file_name)
+      package.workbook.add_worksheet do |sheet|
+        output.each { |datum| sheet.add_row datum }
+      end
+      package.serialize(file_name) if file_name
+      package
+    end
+  end
+end

data/lib/ninja2k/version.rb

@@ -0,0 +1,3 @@
+module Ninja2k
+    VERSION = "0.0.1"
+end

data/lib/ninja2k.rb

@@ -0,0 +1,2 @@
+require "ninja2k/version"
+require "ninja2k/scraper.rb"

metadata

@@ -0,0 +1,117 @@
+--- !ruby/object:Gem::Specification
+name: ninja2k
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Randy Morgan
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-08-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: axlsx
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: kramdown
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A web resource scraper that lets you define xpath selectors, clues and
+  hooks for custom parsing as well as export to xlsx.
+email:
+- digital.ipseity@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/ninja2k/scraper.rb
+- lib/ninja2k/version.rb
+- lib/ninja2k.rb
+- README.md
+- LICENSE
+- Rakefile
+homepage: https://github.com/randym/ninja2k
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: ninja2k is a gem for ninja2k
+test_files: []
+has_rdoc: