macaron 1.0.0

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2012 Dale Ma
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,13 @@
+# Macaron
+Macaron is a simple web scraper implemented in ruby. It's used for service alive testing.
+
+## Install
+    gem install e-macaron
+
+## Example
+```ruby
+macaron http://www.google.com/
+```
+
+## License
+MIT LICENSE, please refer to the LICENSE file.

data/bin/macaron

@@ -0,0 +1,13 @@
+#!/user/bin/ruby
+
+require File.dirname(__FILE__) + '/../lib/macaron'
+
+mother = Spawner.new({ 
+  :thread_timeout_seconds => 999,
+  :in_site_crawling => true,
+  :with_waltir => false
+})
+mother.dig('http://rubyconf.tw/2012/', 2)
+# mother.dig('http://www.sakura.idv.tw/', 2) # url, depth
+puts "Success times: #{mother.success_times}"
+puts "Fail times: #{mother.fail_times}"

data/lib/macaron.rb

@@ -0,0 +1,6 @@
+$: << File.dirname(__FILE__)
+
+require 'macaron/processor'
+require 'macaron/spawner'
+require 'macaron/scraper'
+include Macaron

data/lib/macaron/processor.rb

@@ -0,0 +1,50 @@
+require 'threadpool'
+
+module Macaron
+  class Processor < Job
+    @@output_lock = Mutex.new
+    
+    def run
+      begin
+        url = @args.shift
+        depth = @args.shift
+        html = @args.shift
+        return if depth <= 0
+        scraper = Scraper.new
+        scraper.analyze(url, html)
+
+        @@result[url] = {:anchors => scraper.anchors}
+
+        # do some additional analyzes
+        run_sub_tasks(scraper)
+
+        links = nil
+        if @@options[:in_site_crawling]
+          links = scraper.internal_anchors
+        else
+          links = scraper.absolute_anchors
+        end
+
+        links.each { |a| 
+          next if @@parsed_urls.include?(a)
+          @@task_map = @@task_map.put(a, depth - 1)
+        }
+
+        @@mutex.synchronize {
+          @@success_times += 1
+        }
+      rescue Exception => e
+        @@mutex.synchronize {
+          @@fail_times += 1
+        }
+        print "Error on job: #{url}, msg: #{e.message}\n"
+      end
+    end
+
+    private
+    def run_sub_tasks(scraper)
+      # p scraper.image_urls
+    end
+
+  end
+end

data/lib/macaron/scraper.rb

@@ -0,0 +1,78 @@
+require 'rubygems'
+require 'nokogiri'
+require 'open-uri'
+require 'benchmark'
+require 'timeout'
+require 'watir-webdriver'
+
+module Macaron
+  class Scraper
+
+    def analyze(host, html)
+      @host = host
+      @html = html
+      
+      elapsed_seconds = 0
+      begin
+        timeout(@@options[:nokogiri_timeout_seconds]) do
+          elapsed_seconds = Benchmark.realtime { fetch_dom }
+        end
+      rescue Timeout::Error
+        print "Timeout on #{host}\n"
+        @@mutex.synchronize {
+          @@fail_times += 1
+        }
+      end
+
+      @all_links = absolute_anchors
+
+      print ">> elapsed #{elapsed_seconds} seconds to get '#{host}'\n"
+    end
+
+    def anchors
+      @dom.css('a')
+    end
+
+    def absolute_anchors
+      make_absolute_anchors(anchors)      
+    end
+
+    def internal_anchors
+      root = URI.parse(@host).host
+      @all_links.select {|l| URI.parse(l).host == root}
+    end
+
+    def external_anchors
+      root = URI.parse(@host).host
+      @all_links.select {|l| URI.parse(l).host != root}
+    end
+
+    def images
+      @dom.css('img')
+    end
+
+    def image_urls
+      images.map { |img| make_absolute(img['src']) }.compact
+    end
+
+    def fetch_dom
+      if @@options[:with_waltir]
+        @dom = Nokogiri::HTML(@html)
+      else
+        @dom = Nokogiri::HTML(open(@host))
+      end
+    end
+
+    def make_absolute_anchors(nodes)
+      nodes.map {|n| make_absolute(n['href']) }.compact
+    end
+
+    def make_absolute(href)
+      begin
+        URI.parse(@host).merge(URI.parse(href)).to_s
+      rescue
+        nil
+      end
+    end
+  end
+end

data/lib/macaron/spawner.rb

@@ -0,0 +1,77 @@
+require 'rubygems'
+require 'threadpool'
+require 'hamster'
+
+module Macaron
+  @@result = {}
+  @@parsed_urls = Hamster.set
+  @@task_map = Hamster.hash
+  @@options = {}
+  @@success_times = 0
+  @@fail_times = 0
+  @@mutex = Mutex.new
+
+  class Spawner
+    DEFALUT_OPTIONS = {
+      :nokogiri_timeout_seconds => 30,
+      :thread_timeout_seconds => 40,
+      :pages => 1000,
+      :initial_workers => 1,
+      :maximum_workers => 1,
+      :in_site_crawling => true,
+      :with_waltir => false
+    }.freeze
+
+    def initialize(options = {})
+      @@options = DEFALUT_OPTIONS.merge(options)
+      @threadpool = Threadpool.new(
+        @@options[:initial_workers], 
+        @@options[:maximum_workers], 
+        @@options[:thread_timeout_seconds]
+      )
+    end
+
+    def success_times
+      @@success_times
+    end
+
+    def fail_times
+      @@fail_times
+    end
+
+    def dig(url, init_depth)
+      @@task_map = @@task_map.put(url, init_depth)
+      loop do
+        @@task_map = @@task_map.remove {|url, depth| 
+          @@parsed_urls = @@parsed_urls.add(url)
+
+          if @@options[:with_waltir]
+            html = get_html_via_waltir(url)
+            @threadpool.load(Processor.new(url, depth, html))  
+          else
+            @threadpool.load(Processor.new(url, depth))
+          end          
+        }
+
+        break if @threadpool.busy_workers_count == 0 && @@task_map.empty?
+        
+        if @@success_times > @@options[:pages]
+          print "Fetched pages exceeds the limit #{@@options[:pages]}\n"
+          break
+        end
+      end
+
+      @bot.close unless @bot.nil?
+
+      # puts "result: #{@@result.size}, #{@@result.keys}"
+    end
+
+    private
+    def get_html_via_waltir(url)
+      @bot ||= Watir::Browser.new
+      @bot.goto(url)
+      @bot.html
+    end
+
+  end
+end

metadata

@@ -0,0 +1,51 @@
+--- !ruby/object:Gem::Specification
+name: macaron
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+  prerelease: 
+platform: ruby
+authors:
+- Dale Ma
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-11-17 00:00:00.000000000 Z
+dependencies: []
+description: 
+email: dalema22@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/macaron
+- lib/macaron/processor.rb
+- lib/macaron/scraper.rb
+- lib/macaron/spawner.rb
+- lib/macaron.rb
+- LICENSE
+- README.md
+homepage: http://github.com/eguitarz/macaron
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.19
+signing_key: 
+specification_version: 3
+summary: Ruby based web scraper
+test_files: []