macaron 1.0.1 → 1.0.2

data/bin/macaron

@@ -17,7 +17,11 @@ options_parser = OptionParser.new do |opts|
   end
 
   opts.on("-j", "--javascript", "Open javascript support mode") do |j|
-    options[:waltir] = j
+    options[:with_waltir] = j
+  end
+
+  opts.on("-s", "--save", "Save html") do |s|
+    options[:save] = s
   end
 
   opts.on_tail("-h", "--help", "Show this message") do

data/lib/macaron/processor.rb

@@ -1,3 +1,4 @@
+require 'rubygems'
 require 'threadpool'
 
 module Macaron
@@ -13,7 +14,8 @@ module Macaron
         scraper = Scraper.new
         scraper.analyze(url, html)
 
-        @@result[url] = {:anchors => scraper.anchors}
+        # @@result[url] = {:anchors => scraper.anchors}
+        @@result[url] = true;
 
         # do some additional analyzes
         run_sub_tasks(scraper)
@@ -24,9 +26,11 @@ module Macaron
         else
           links = scraper.absolute_anchors
         end
+        puts "found #{links.size} links on #{url}" if @@options[:debug]
 
-        links.each { |a| 
+        links.each { |a|
           next if @@parsed_urls.include?(a)
+          p "put #{a} into tasks" if @@options[:debug]
           @@task_map = @@task_map.put(a, depth - 1)
         }
 
@@ -44,6 +48,14 @@ module Macaron
     private
     def run_sub_tasks(scraper)
       # p scraper.image_urls
+      
+      if @@options[:save]
+        dir = @@options[:dir] || '/tmp'
+        filename = scraper.host.gsub('/', '\\')
+        File.open(File.join(dir, filename), "w+") do |f|
+          f.write(scraper.dom)
+        end
+      end
     end
 
   end

data/lib/macaron/scraper.rb

@@ -7,6 +7,7 @@ require 'watir-webdriver'
 
 module Macaron
   class Scraper
+    attr_reader :dom, :host
 
     def analyze(host, html)
       @host = host
@@ -56,11 +57,11 @@ module Macaron
     end
 
     def fetch_dom
-      if @@options[:with_waltir]
-        @dom = Nokogiri::HTML(@html)
-      else
-        @dom = Nokogiri::HTML(open(@host))
+      unless @@options[:with_waltir]
+        @html = open(@host)
       end
+
+      @dom = Nokogiri::HTML(@html)
     end
 
     def make_absolute_anchors(nodes)

data/lib/macaron/spawner.rb

@@ -16,10 +16,11 @@ module Macaron
       :nokogiri_timeout_seconds => 30,
       :thread_timeout_seconds => 40,
       :pages => 1000,
-      :initial_workers => 1,
-      :maximum_workers => 1,
+      :initial_workers => 4,
+      :maximum_workers => 4,
       :in_site_crawling => true,
-      :with_waltir => false
+      :with_waltir => false,
+      :debug => false
     }.freeze
 
     def initialize(options = {})
@@ -42,7 +43,7 @@ module Macaron
     def dig(url, init_depth=3)
       @@task_map = @@task_map.put(url, init_depth)
       loop do
-        @@task_map = @@task_map.remove {|url, depth| 
+        @@task_map.each {|url, depth|
           @@parsed_urls = @@parsed_urls.add(url)
 
           if @@options[:with_waltir]
@@ -50,7 +51,9 @@ module Macaron
             @threadpool.load(Processor.new(url, depth, html))  
           else
             @threadpool.load(Processor.new(url, depth))
-          end          
+          end
+
+          @@task_map = @@task_map.delete(url)
         }
 
         break if @threadpool.busy_workers_count == 0 && @@task_map.empty?

metadata

@@ -1,53 +1,61 @@
 --- !ruby/object:Gem::Specification
 name: macaron
 version: !ruby/object:Gem::Version
-  version: 1.0.1
-  prerelease: 
+  prerelease:
+  version: 1.0.2
 platform: ruby
 authors:
 - Dale Ma
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2012-11-17 00:00:00.000000000 Z
+date: 2012-12-06 00:00:00.000000000 Z
 dependencies: []
-description: 
+description:
 email: dalema22@gmail.com
 executables:
 - macaron
 extensions: []
 extra_rdoc_files: []
 files:
-- bin/macaron
-- lib/macaron/processor.rb
-- lib/macaron/scraper.rb
-- lib/macaron/spawner.rb
-- lib/macaron/version.rb
-- lib/macaron.rb
+- !binary |-
+  YmluL21hY2Fyb24=
+- !binary |-
+  bGliL21hY2Fyb24ucmI=
+- !binary |-
+  bGliL21hY2Fyb24vcHJvY2Vzc29yLnJi
+- !binary |-
+  bGliL21hY2Fyb24vc2NyYXBlci5yYg==
+- !binary |-
+  bGliL21hY2Fyb24vc3Bhd25lci5yYg==
+- !binary |-
+  bGliL21hY2Fyb24vdmVyc2lvbi5yYg==
 - LICENSE
 - README.md
 homepage: http://github.com/eguitarz/macaron
 licenses: []
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
-      version: '0'
-required_rubygems_version: !ruby/object:Gem::Requirement
+      version: !binary |-
+        MA==
   none: false
+required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
-      version: '0'
+      version: !binary |-
+        MA==
+  none: false
 requirements: []
-rubyforge_project: 
-rubygems_version: 1.8.19
-signing_key: 
+rubyforge_project:
+rubygems_version: 1.8.24
+signing_key:
 specification_version: 3
 summary: Ruby based web scraper
 test_files: []