idl 1.0.0

data/bin/idl

@@ -0,0 +1,19 @@
+#!/usr/bin/env ruby
+
+require 'docopt'
+require 'idl'
+
+options = Docopt::docopt """
+IDl - Download all images from a webpage at given URL
+
+Usage:
+  idl [options] <URL> [directory]
+
+Options:
+  -h --help     Print this help message and exit
+  -l --linked   Download linked-to images instead of included ones
+"""
+
+if options['<URL>']
+  IDl.new.harvest options['<URL>'], options['--linked'], options['directory'] || './'
+end

data/lib/idl.rb

@@ -0,0 +1,79 @@
+require 'open-uri'
+require 'nokogiri'
+require 'eventmachine'
+require 'em-http-request'
+
+module EventMachine
+  class HttpClient
+    attr_accessor :file
+  end
+end
+
+class IDl
+  def initialize
+    @image_extensions = %w(jpg jpeg png gif tif tiff).collect {|ext| ".#{ext}"}
+  end
+
+  def harvest(url, linked=false, target_dir='./')
+    doc = Nokogiri.parse open url
+
+    if linked
+      urls = []
+      doc.css('a[href]').each do |a|
+        if path = URI(a['href']).path and path.downcase.end_with?(*@image_extensions)
+          urls << URI(url).merge(a['href']).to_s
+        end
+      end
+    else
+      urls = doc.css('img[src]').collect do |img|
+        URI(url).merge(img['src']).to_s
+      end
+    end
+
+    if urls
+      self.fetch urls, target_dir
+    end
+  end
+
+  def fetch(urls, target_dir='./')
+    EM.run do
+      request_pool = EM::MultiRequest.new
+
+      request_pool.callback do
+        puts 'All requests finished.'
+        EM.stop
+      end
+
+      urls.uniq.each do |url|
+        puts "Enqueuing [#{url}]"
+        request = EM::HttpRequest.new(url).get
+
+        request.stream do |chunk|
+          FileUtils.mkdir_p target_dir unless File.exists? target_dir
+          request.file = File.open unique_filepath(url, target_dir), 'wb' unless request.file
+          request.file.write chunk
+        end
+
+        request.callback do
+          puts "Image [#{url}] was downloaded successfully."
+        end
+
+        request_pool.add request.object_id, request
+      end
+    end
+  end
+
+  def unique_filepath(url, target_dir)
+    filename = File.basename URI(url).path
+    filepath = target_dir + filename
+    ext = File.extname filename
+    suffix = 1
+
+    while File.exists? filepath
+      suffix = suffix + 1
+      filepath = target_dir + File.basename(filename, ext) + '-' + suffix.to_s + ext
+    end
+
+    return filepath
+  end
+end

metadata

@@ -0,0 +1,114 @@
+--- !ruby/object:Gem::Specification
+name: idl
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+  prerelease: 
+platform: ruby
+authors:
+- Ilya Vassilevsky
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-12-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: eventmachine
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: em-http-request
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: docopt
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Provides a class that can download images from webpages and a command-line
+  tool
+email: vassilevsky@gmail.com
+executables:
+- idl
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/idl.rb
+- bin/idl
+homepage: https://github.com/vassilevsky/idl
+licenses: []
+post_install_message: Now you can run `idl http://www.example.com/path/to/page.html`
+  to download all images from the page to the ./page/ directory.
+rdoc_options: []
+require_paths:
+- lib
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '1.9'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Downloads all images from a webpage in parallel
+test_files: []