embed_html 0.3.0 → 0.3.2

data/README.markdown

@@ -2,6 +2,7 @@ Dependencies
 ============
 
  * Hpricot
+ * Typhoeus
 
 Install
 =======

data/Rakefile

@@ -3,13 +3,13 @@ require 'rubygems'
 require 'rake'
 require 'echoe'
 
-Echoe.new('embed_html', '0.3.0') do |p|
+Echoe.new('embed_html', '0.3.2') do |p|
   p.description    = "Download and embed images in html using base64 data encoding"
   p.summary        = "Download or process a HTML page, find images there, download them and embed it into the HTML using Base64 data encoding"
   p.url            = "http://github.com/siuying/embed_html"
   p.author         = "Francis Chong"
   p.email          = "francis@ignition.hk"
   p.ignore_pattern = ["tmp/*", "script/*", "*.html"]
-  p.runtime_dependencies = ["hpricot", "mime-types"]
+  p.runtime_dependencies = ["hpricot", "typhoeus", "mime-types"]
 end
 

data/embed_html.gemspec

@@ -1,38 +1,39 @@
 # -*- encoding: utf-8 -*-
 
 Gem::Specification.new do |s|
-  s.name = %q{embed_html}
-  s.version = "0.3.0"
+  s.name = "embed_html"
+  s.version = "0.3.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
   s.authors = ["Francis Chong"]
-  s.date = %q{2010-06-25}
-  s.default_executable = %q{eurl}
-  s.description = %q{Download and embed images in html using base64 data encoding}
-  s.email = %q{francis@ignition.hk}
+  s.date = "2012-02-01"
+  s.description = "Download and embed images in html using base64 data encoding"
+  s.email = "francis@ignition.hk"
   s.executables = ["eurl"]
   s.extra_rdoc_files = ["README.markdown", "bin/eurl", "lib/embed_html.rb", "lib/embed_html/embeder.rb"]
   s.files = ["Manifest", "README.markdown", "Rakefile", "bin/eurl", "lib/embed_html.rb", "lib/embed_html/embeder.rb", "embed_html.gemspec"]
-  s.homepage = %q{http://github.com/siuying/embed_html}
+  s.homepage = "http://github.com/siuying/embed_html"
   s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Embed_html", "--main", "README.markdown"]
   s.require_paths = ["lib"]
-  s.rubyforge_project = %q{embed_html}
-  s.rubygems_version = %q{1.3.6}
-  s.summary = %q{Download or process a HTML page, find images there, download them and embed it into the HTML using Base64 data encoding}
+  s.rubyforge_project = "embed_html"
+  s.rubygems_version = "1.8.10"
+  s.summary = "Download or process a HTML page, find images there, download them and embed it into the HTML using Base64 data encoding"
 
   if s.respond_to? :specification_version then
-    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
     s.specification_version = 3
 
-    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
       s.add_runtime_dependency(%q<hpricot>, [">= 0"])
+      s.add_runtime_dependency(%q<typhoeus>, [">= 0"])
       s.add_runtime_dependency(%q<mime-types>, [">= 0"])
     else
       s.add_dependency(%q<hpricot>, [">= 0"])
+      s.add_dependency(%q<typhoeus>, [">= 0"])
       s.add_dependency(%q<mime-types>, [">= 0"])
     end
   else
     s.add_dependency(%q<hpricot>, [">= 0"])
+    s.add_dependency(%q<typhoeus>, [">= 0"])
     s.add_dependency(%q<mime-types>, [">= 0"])
   end
 end

data/lib/embed_html/embeder.rb

@@ -10,25 +10,30 @@ module EmbedHtml
   class Embeder
     MAX_CONCURRENCY = 5
     
-    attr_accessor :url
+    attr_accessor :url_or_html
     attr_accessor :logger
     attr_accessor :concurrency
+    attr_accessor :base_dirname
     
-    def initialize(url, logger=Logger.new($stdout), concurrency=MAX_CONCURRENCY)
+    def initialize(url_or_html, logger=Logger.new($stdout), concurrency=MAX_CONCURRENCY)
       @logger = logger
-      @url = url
+      @url_or_html = url_or_html
       @concurrency = concurrency
     end
     
     def process
-      @logger.info "downloading url: #{@url}"
-      html = Typhoeus::Request.get(@url.to_s).body
+      # @logger.info "downloading url: #{@url_or_html}"
+      html = (@url_or_html =~ /$http/) ? Typhoeus::Request.get(@url_or_html.to_s).body : @url_or_html
       doc = Hpricot(html)
       
       hydra = Typhoeus::Hydra.new(:max_concurrency => @concurrency)
       doc.search("//img").each do |img|                
         begin
-          hydra.queue create_fetch_file_request(img, 'src')
+          if img['src']=~ /^http/
+            hydra.queue create_fetch_file_request(img, 'src')
+          else
+            fetch_file(img, 'src')
+          end
         rescue StandardError => e
           @logger.error "failed download image: #{img['src']} #{e.inspect}"
         end
@@ -36,8 +41,10 @@ module EmbedHtml
 
       doc.search("//script").each do |script|                
         begin
-          if script['src']
+          if script['src'] and script['src'] =~ /^http/
             hydra.queue create_fetch_file_request(script, 'src')
+          elsif script['src']
+            fetch_file(script, 'src')
           end
         rescue StandardError => e
           @logger.error "failed download script: #{script['src']} #{e.inspect}"
@@ -46,7 +53,12 @@ module EmbedHtml
 
       doc.search("//link").each do |link|
         begin
-          hydra.queue create_fetch_file_request(link, 'href')
+          url = link['href']
+          if url =~ /^http/
+            hydra.queue create_fetch_file_request(link, 'href')
+          else
+            fetch_file(link, 'href')
+          end
         rescue StandardError => e
           @logger.error "failed download linked resource: #{link['href']} #{e.inspect}"
         end
@@ -54,13 +66,13 @@ module EmbedHtml
       
       hydra.run
 
-      @logger.info "done"            
+      # @logger.info "done"            
       doc.to_html      
     end
     
     def process_local
-      @logger.info "downloading url: #{@url}"
-      html = open(@url).read
+      # @logger.info "downloading url: #{@url_or_html}"
+      html = open(@url_or_html).read
       doc = Hpricot(html)
 
       doc.search("//img").each do |img|                
@@ -87,13 +99,13 @@ module EmbedHtml
         end
       end
 
-      @logger.info "done"            
+      # @logger.info "done"            
       doc.to_html      
     end
     
     private
     def create_fetch_file_request(element, field)
-      file_url = URI.join(@url, element.attributes[field])
+      file_url = (@url_or_html =~ /^http/) ? URI.join(@url_or_html, element.attributes[field]) : element.attributes[field]
       @logger.debug "queue download file: #{file_url}"
 
       request = Typhoeus::Request.new(file_url.to_s)
@@ -109,8 +121,9 @@ module EmbedHtml
     end
 
     def fetch_file(element, field)
-      file_url = element.attributes[field]
+      file_url = @base_dirname ? "#{@base_dirname.to_s}/#{element.attributes[field]}" : element.attributes[field]
       @logger.debug "queue download file: #{file_url}"
+      return unless File.exists?(file_url)
       
       type = MIME::Types.type_for(file_url).first.to_s rescue "application/data"
       data = open(file_url.to_s).read

metadata

@@ -1,58 +1,60 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: embed_html
-version: !ruby/object:Gem::Version 
-  prerelease: false
-  segments: 
-  - 0
-  - 3
-  - 0
-  version: 0.3.0
+version: !ruby/object:Gem::Version
+  version: 0.3.2
+  prerelease: 
 platform: ruby
-authors: 
+authors:
 - Francis Chong
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2010-06-25 00:00:00 +08:00
-default_executable: 
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2012-02-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: hpricot
+  requirement: &70320708644640 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
   prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 0
-        version: "0"
+  version_requirements: *70320708644640
+- !ruby/object:Gem::Dependency
+  name: typhoeus
+  requirement: &70320708641840 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: mime-types
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 0
-        version: "0"
+  version_requirements: *70320708641840
+- !ruby/object:Gem::Dependency
+  name: mime-types
+  requirement: &70320708641160 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
-  version_requirements: *id002
+  prerelease: false
+  version_requirements: *70320708641160
 description: Download and embed images in html using base64 data encoding
 email: francis@ignition.hk
-executables: 
+executables:
 - eurl
 extensions: []
-
-extra_rdoc_files: 
+extra_rdoc_files:
 - README.markdown
 - bin/eurl
 - lib/embed_html.rb
 - lib/embed_html/embeder.rb
-files: 
+files:
 - Manifest
 - README.markdown
 - Rakefile
@@ -60,41 +62,35 @@ files:
 - lib/embed_html.rb
 - lib/embed_html/embeder.rb
 - embed_html.gemspec
-has_rdoc: true
 homepage: http://github.com/siuying/embed_html
 licenses: []
-
 post_install_message: 
-rdoc_options: 
+rdoc_options:
 - --line-numbers
 - --inline-source
 - --title
 - Embed_html
 - --main
 - README.markdown
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      segments: 
-      - 1
-      - 2
-      version: "1.2"
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '1.2'
 requirements: []
-
 rubyforge_project: embed_html
-rubygems_version: 1.3.6
+rubygems_version: 1.8.10
 signing_key: 
 specification_version: 3
-summary: Download or process a HTML page, find images there, download them and embed it into the HTML using Base64 data encoding
+summary: Download or process a HTML page, find images there, download them and embed
+  it into the HTML using Base64 data encoding
 test_files: []
-