embed_html 0.1.0 → 0.2.0

data/Rakefile

@@ -3,7 +3,7 @@ require 'rubygems'
 require 'rake'
 require 'echoe'
 
-Echoe.new('embed_html', '0.1.0') do |p|
+Echoe.new('embed_html', '0.2.0') do |p|
   p.description    = "Download and embed images in html using base64 data encoding"
   p.summary        = "Download or process a HTML page, find images there, download them and embed it into the HTML using Base64 data encoding"
   p.url            = "http://github.com/siuying/embed_html"

data/embed_html.gemspec

@@ -2,11 +2,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{embed_html}
-  s.version = "0.1.0"
+  s.version = "0.2.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
   s.authors = ["Francis Chong"]
-  s.date = %q{2010-05-20}
+  s.date = %q{2010-06-22}
   s.default_executable = %q{eurl}
   s.description = %q{Download and embed images in html using base64 data encoding}
   s.email = %q{francis@ignition.hk}

data/lib/embed_html/embeder.rb

@@ -25,26 +25,50 @@ module EmbedHtml
       hydra = Typhoeus::Hydra.new(:max_concurrency => MAX_CONCURRENCY)
       doc.search("//img").each do |img|                
         begin
-          image_url = URI.join(@url, img.attributes['src'])
-          @logger.debug "queue download image: #{image_url}"
+          hydra.queue create_fetch_file_request(img, 'src')
+        rescue StandardError => e
+          @logger.error "failed download image: #{img['src']}"
+        end
+      end
+
+      doc.search("//script").each do |script|                
+        begin
+          hydra.queue create_fetch_file_request(script, 'src')
+        rescue StandardError => e
+          @logger.error "failed download script: #{script['src']}"
+        end
+      end
 
-          request = Typhoeus::Request.new(image_url.to_s)
-          request.on_complete do |response|
-            data = response.body
-            type = response.headers_hash["Content-Type"]
-            if data && type
-              data_b64 = Base64.encode64(data)
-              img.attributes['src'] = "data:#{type};base64,#{data_b64}"
-            end  
-          end
-          hydra.queue request
+      doc.search("//link").each do |link|
+        begin
+          hydra.queue create_fetch_file_request(link, 'href')
         rescue StandardError => e
-          @logger.error "failed downloading image: #{image_url} (#{e.message})"
+          @logger.error "failed download linked resource: #{link['href']}"
         end
       end
+      
       hydra.run
+
       @logger.info "done"            
       doc.to_html      
     end
+    
+    private
+    def create_fetch_file_request(element, field)
+      file_url = URI.join(@url, element.attributes[field])
+      @logger.debug "queue download file: #{file_url}"
+
+      request = Typhoeus::Request.new(file_url.to_s)
+      request.on_complete do |response|
+        data = response.body
+        type = response.headers_hash["Content-Type"]
+        if data && type
+          data_b64 = Base64.encode64(data)
+          element.attributes[field] = "data:#{type};base64,#{data_b64}"
+        end  
+      end
+      return request
+    end
+    
   end
 end

metadata

@@ -4,9 +4,9 @@ version: !ruby/object:Gem::Version
   prerelease: false
   segments: 
   - 0
-  - 1
+  - 2
   - 0
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors: 
 - Francis Chong
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-05-20 00:00:00 +08:00
+date: 2010-06-22 00:00:00 +08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency