embed_html 0.1.0

data/Manifest

@@ -0,0 +1,6 @@
+Manifest
+README.markdown
+Rakefile
+bin/eurl
+lib/embed_html.rb
+lib/embed_html/embeder.rb

data/README.markdown

@@ -0,0 +1,28 @@
+Dependencies
+============
+
+ * Hpricot
+ * Typhoeus
+
+Install
+=======
+
+> gem install embed_html
+
+Usage
+=======
+
+> eurl _<input-url>_ _<output-file>_
+
+Example:
+
+> eurl "http://en.wikipedia.org/wiki/Wiki" wiki.html
+
+Output:
+
+>	I, [2010-05-20T15:56:58.315769 #26393]  INFO -- : downloading url: http://en.wikipedia.org/wiki/Wiki
+
+>	I, [2010-05-20T15:57:02.033068 #26393]  INFO -- : done
+	
+The file wiki.html then contains the specified page, with all of the images embeded
+

data/Rakefile

@@ -0,0 +1,15 @@
+# Rakefile
+require 'rubygems'
+require 'rake'
+require 'echoe'
+
+Echoe.new('embed_html', '0.1.0') do |p|
+  p.description    = "Download and embed images in html using base64 data encoding"
+  p.summary        = "Download or process a HTML page, find images there, download them and embed it into the HTML using Base64 data encoding"
+  p.url            = "http://github.com/siuying/embed_html"
+  p.author         = "Francis Chong"
+  p.email          = "francis@ignition.hk"
+  p.ignore_pattern = ["tmp/*", "script/*", "*.html"]
+  p.runtime_dependencies = ["hpricot", "typhoeus"]
+end
+

data/bin/eurl

@@ -0,0 +1,16 @@
+require 'embed_html'
+
+url = ARGV[0]
+file = ARGV[1]
+
+if url && file
+	log = Logger.new($stdout)
+	log.level = Logger::INFO
+
+	html = EmbedHtml::Embeder.new(url, log).process
+	File.open(file, 'w') {|f| f.write(html)}
+
+else
+	puts "usage: eurl <URL> <OUTPUT_FILE>"
+
+end

data/embed_html.gemspec

@@ -0,0 +1,38 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{embed_html}
+  s.version = "0.1.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Francis Chong"]
+  s.date = %q{2010-05-20}
+  s.default_executable = %q{eurl}
+  s.description = %q{Download and embed images in html using base64 data encoding}
+  s.email = %q{francis@ignition.hk}
+  s.executables = ["eurl"]
+  s.extra_rdoc_files = ["README.markdown", "bin/eurl", "lib/embed_html.rb", "lib/embed_html/embeder.rb"]
+  s.files = ["Manifest", "README.markdown", "Rakefile", "bin/eurl", "lib/embed_html.rb", "lib/embed_html/embeder.rb", "embed_html.gemspec"]
+  s.homepage = %q{http://github.com/siuying/embed_html}
+  s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Embed_html", "--main", "README.markdown"]
+  s.require_paths = ["lib"]
+  s.rubyforge_project = %q{embed_html}
+  s.rubygems_version = %q{1.3.6}
+  s.summary = %q{Download or process a HTML page, find images there, download them and embed it into the HTML using Base64 data encoding}
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<hpricot>, [">= 0"])
+      s.add_runtime_dependency(%q<typhoeus>, [">= 0"])
+    else
+      s.add_dependency(%q<hpricot>, [">= 0"])
+      s.add_dependency(%q<typhoeus>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<hpricot>, [">= 0"])
+    s.add_dependency(%q<typhoeus>, [">= 0"])
+  end
+end

data/lib/embed_html/embeder.rb

@@ -0,0 +1,50 @@
+require 'logger'
+require 'open-uri'
+require 'hpricot'
+require 'uri'
+require 'base64'
+require 'typhoeus'
+
+module EmbedHtml
+  class Embeder
+    MAX_CONCURRENCY = 5
+    
+    attr_accessor :url
+    attr_accessor :logger
+    
+    def initialize(url, logger=Logger.new($stdout))
+      @logger = logger
+      @url = url
+    end
+    
+    def process
+      @logger.info "downloading url: #{@url}"
+      html = Typhoeus::Request.get(@url.to_s).body
+      doc = Hpricot(html)
+      
+      hydra = Typhoeus::Hydra.new(:max_concurrency => MAX_CONCURRENCY)
+      doc.search("//img").each do |img|                
+        begin
+          image_url = URI.join(@url, img.attributes['src'])
+          @logger.debug "queue download image: #{image_url}"
+
+          request = Typhoeus::Request.new(image_url.to_s)
+          request.on_complete do |response|
+            data = response.body
+            type = response.headers_hash["Content-Type"]
+            if data && type
+              data_b64 = Base64.encode64(data)
+              img.attributes['src'] = "data:#{type};base64,#{data_b64}"
+            end  
+          end
+          hydra.queue request
+        rescue StandardError => e
+          @logger.error "failed downloading image: #{image_url} (#{e.message})"
+        end
+      end
+      hydra.run
+      @logger.info "done"            
+      doc.to_html      
+    end
+  end
+end

data/lib/embed_html.rb

@@ -0,0 +1,4 @@
+path = File.dirname(__FILE__)
+$:.unshift(path) unless $:.include?(path)
+
+require 'embed_html/embeder'

metadata

@@ -0,0 +1,100 @@
+--- !ruby/object:Gem::Specification 
+name: embed_html
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors: 
+- Francis Chong
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-05-20 00:00:00 +08:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: hpricot
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: typhoeus
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id002
+description: Download and embed images in html using base64 data encoding
+email: francis@ignition.hk
+executables: 
+- eurl
+extensions: []
+
+extra_rdoc_files: 
+- README.markdown
+- bin/eurl
+- lib/embed_html.rb
+- lib/embed_html/embeder.rb
+files: 
+- Manifest
+- README.markdown
+- Rakefile
+- bin/eurl
+- lib/embed_html.rb
+- lib/embed_html/embeder.rb
+- embed_html.gemspec
+has_rdoc: true
+homepage: http://github.com/siuying/embed_html
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --line-numbers
+- --inline-source
+- --title
+- Embed_html
+- --main
+- README.markdown
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 1
+      - 2
+      version: "1.2"
+requirements: []
+
+rubyforge_project: embed_html
+rubygems_version: 1.3.6
+signing_key: 
+specification_version: 3
+summary: Download or process a HTML page, find images there, download them and embed it into the HTML using Base64 data encoding
+test_files: []
+