download_files 0.0.2 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2cc5ee3b9444c2c755a9be44f0e4f3245a2ecbcb
-  data.tar.gz: 13d1f4e3d59ecbb1cbf72318779fb6e8bb65a9a5
+  metadata.gz: 581b5cc0afb407f886aaebd62edb94c6d318b44c
+  data.tar.gz: 2677502f3b63318fd23855a130f55cad73619c9a
 SHA512:
-  metadata.gz: 17f4bf978f62f18889d74bc034527fd262beee9fa5f8c5d15ac76eb4ae653e18353d46ccfc79b7175296834f16341d1b3546ebd41c248f08323b49e1ac8ec6e3
-  data.tar.gz: 715bf6f10e6a4187d575764825a8cc90ae86317cc7d0672be253df285d0017a19d7da2549b853d3398a22671f7d1e013c16bc3ad028ce390d6428c73259fb844
+  metadata.gz: 7cfe3e0320976ff4b1d032ad8c64e0651f02905213a43784a95850c56dfec8b8792db3fe86bb680090d53daeaea99bc9b886de85767a4f0bced5f5a0f5120c78
+  data.tar.gz: 57e593f0d696f73c06b57f73cac499ee8091e7cc1e2cc9a8ce4ff900f023b0079c02ca92f9d76a45b7cd2fa123c67ab59db9ac943d9bf3f2de582744064974a0

data/bin/download_files

@@ -9,23 +9,33 @@ class App
   include Methadone::Main
   include Methadone::CLILogging
 
-  main do |page_address, pattern, directory|
-    dir = if directory
-      FileUtils.mkdir_p(directory) unless File.directory?(directory)
-    else
-      FileUtils.pwd
-    end
-    DownloadFiles.download_files(page_address, pattern, dir, logger)
+  main do |page_address, pattern|
+    FileUtils.mkdir_p(options['directory']) unless File.directory?(options['directory'])
+    DownloadFiles.download_files(
+      page_address,
+      pattern,
+      options['directory'],
+      options['ssl-verify'],
+      options['verbose'],
+      logger
+    )
   end
 
-  # Declare command-line interface
   description "Downloads files from the web page at the given address on the given domain that match the given regular expression pattern."
-  
-  # Require an argument
+
   arg :page_address
   arg :pattern
   arg :directory, :optional
-  
+
+  options['directory'] = FileUtils.pwd
+  on('-d DIRECTORY', '--directory', 'Directory for scrapped content')
+
+  options['ssl-verify'] = true
+  on "--[no]-ssl-verify", 'Do no SSL verification of target website.'
+
+  options['verbose'] = false
+  on '-v', "--verbose", 'Verbose output'
+
   version DownloadFiles::VERSION
 
   use_log_level_option

data/download_files.gemspec

@@ -16,8 +16,7 @@ Gem::Specification.new do |gem|
   gem.version       = DownloadFiles::VERSION
 
   gem.add_development_dependency('rake', '~> 0.9.2')
-  
-  gem.add_dependency('methadone', '~> 1.2.5')
-  gem.add_dependency('mechanize')
-  gem.add_dependency('nullobject')
+
+  gem.add_dependency('methadone', '~> 1.4.0')
+  gem.add_dependency('mechanize', '~> 2.7.3')
 end

data/lib/download_files.rb

@@ -1,15 +1,15 @@
 require_relative "download_files/version"
 
 require 'mechanize'
-require 'nullobject'
 
 class Mechanize
   def self.start
     yield(new)
   end
 
-  def self.go(page_address)
+  def self.go(page_address, ssl_verify)
     start do |agent|
+      agent.verify_mode = OpenSSL::SSL::VERIFY_NONE unless ssl_verify
       agent.get(page_address) do |page|
         yield agent, page
       end
@@ -18,24 +18,55 @@ class Mechanize
 end
 
 module DownloadFiles
-  def self.download_files(page_address, pattern, dir=FileUtils.pwd, logger=Null::Object.instance)
-    logger.debug "DownloadFiles.download_files: " + {page_address: page_address, pattern: pattern, dir: dir, logger: logger}.inspect
+  def self.download_files(
+    page_address,
+    pattern,
+    dir,
+    ssl_verify,
+    is_verbose,
+    logger
+  )
+    logger.debug(
+      "DownloadFiles.download_files: " +
+      {
+        page_address: page_address,
+        pattern: pattern,
+        dir: dir,
+        ssl_verify: ssl_verify,
+        is_verbose: is_verbose,
+        logger: logger
+      }.inspect
+    )
 
-    abs_dir = File.expand_path(dir, FileUtils.pwd)
+    abs_dir = File.absolute_path(dir)
 
-    Mechanize.go(page_address) do |agent, page|
+    logger.info(
+      "Downloading files from \"#{page_address}\" to \"#{abs_dir}\" that match /#{pattern}/ " +
+      (if ssl_verify then 'requiring' else 'without requiring' end + ' SSL verification') +
+      '...'
+    )
+
+    Mechanize.go(page_address, ssl_verify) do |agent, page|
       logger.debug page.inspect
-      logger.info "On #{page.uri}"
+
+      if is_verbose
+        logger.info "On #{page.uri}"
+      end
 
       agent.pluggable_parser.default = Mechanize::Download
 
       page.links.each do |link|
-        logger.debug "Checking link #{link.href}"
+        if is_verbose
+          logger.info "Checking link #{link.href}"
+        end
 
         next unless link.href && link.href.match(pattern)
 
         full_file_name = File.expand_path(File.basename(link.href), abs_dir)
-        logger.info "Downloading #{link.click.uri} to #{full_file_name}"
+
+        if is_verbose
+          logger.info "Downloading #{link.click.uri} to #{full_file_name}"
+        end
 
         agent.get(link.click.uri).save(full_file_name)
       end

data/lib/download_files/version.rb

@@ -1,3 +1,3 @@
 module DownloadFiles
-  VERSION = "0.0.2"
+  VERSION = "1.0.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: download_files
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 1.0.0
 platform: ruby
 authors:
 - Dustin Morrill
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-04-23 00:00:00.000000000 Z
+date: 2014-05-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -30,42 +30,28 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.2.5
+        version: 1.4.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.2.5
+        version: 1.4.0
 - !ruby/object:Gem::Dependency
   name: mechanize
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: nullobject
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 2.7.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 2.7.3
 description: Download all files at a particular URL that match a certain pattern.
 email:
 - dmorrill10@gmail.com
@@ -107,3 +93,4 @@ signing_key:
 specification_version: 4
 summary: Download all files at a particular URL that match a certain pattern.
 test_files: []
+has_rdoc: