embed_html 0.2.2 → 0.2.3

data/Rakefile

@@ -3,7 +3,7 @@ require 'rubygems'
 require 'rake'
 require 'echoe'
 
-Echoe.new('embed_html', '0.2.2') do |p|
+Echoe.new('embed_html', '0.2.3') do |p|
   p.description    = "Download and embed images in html using base64 data encoding"
   p.summary        = "Download or process a HTML page, find images there, download them and embed it into the HTML using Base64 data encoding"
   p.url            = "http://github.com/siuying/embed_html"

data/bin/eurl

@@ -2,12 +2,13 @@ require 'embed_html'
 
 url = ARGV[0]
 file = ARGV[1]
+concurrency = ARGV[2].nil? ? 5 : ARGV[2].to_i
 
 if url && file
 	log = Logger.new($stdout)
 	log.level = Logger::INFO
 
-	html = EmbedHtml::Embeder.new(url, log).process
+	html = EmbedHtml::Embeder.new(url, log, concurrency).process
 	File.open(file, 'w') {|f| f.write(html)}
 
 else

data/embed_html.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |s|
   s.name = %q{embed_html}
-  s.version = "0.2.2"
+  s.version = "0.2.3"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
   s.authors = ["Francis Chong"]

data/lib/embed_html/embeder.rb

@@ -11,10 +11,12 @@ module EmbedHtml
     
     attr_accessor :url
     attr_accessor :logger
+    attr_accessor :concurrency
     
-    def initialize(url, logger=Logger.new($stdout))
+    def initialize(url, logger=Logger.new($stdout), concurrency=MAX_CONCURRENCY)
       @logger = logger
       @url = url
+      @concurrency = concurrency
     end
     
     def process
@@ -22,20 +24,22 @@ module EmbedHtml
       html = Typhoeus::Request.get(@url.to_s).body
       doc = Hpricot(html)
       
-      hydra = Typhoeus::Hydra.new(:max_concurrency => MAX_CONCURRENCY)
+      hydra = Typhoeus::Hydra.new(:max_concurrency => @concurrency)
       doc.search("//img").each do |img|                
         begin
           hydra.queue create_fetch_file_request(img, 'src')
         rescue StandardError => e
-          @logger.error "failed download image: #{img['src']}"
+          @logger.error "failed download image: #{img['src']} #{e.inspect}"
         end
       end
 
       doc.search("//script").each do |script|                
         begin
-          hydra.queue create_fetch_file_request(script, 'src')
+          if script['src']
+            hydra.queue create_fetch_file_request(script, 'src')
+          end
         rescue StandardError => e
-          @logger.error "failed download script: #{script['src']}"
+          @logger.error "failed download script: #{script['src']} #{e.inspect}"
         end
       end
 
@@ -43,7 +47,7 @@ module EmbedHtml
         begin
           hydra.queue create_fetch_file_request(link, 'href')
         rescue StandardError => e
-          @logger.error "failed download linked resource: #{link['href']}"
+          @logger.error "failed download linked resource: #{link['href']} #{e.inspect}"
         end
       end
       

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 2
-  - 2
-  version: 0.2.2
+  - 3
+  version: 0.2.3
 platform: ruby
 authors: 
 - Francis Chong