pmirror 0.0.4 → 0.1.1

data/.travis.yml

@@ -1,4 +1,12 @@
 language: ruby
-rvm: 
-  - 1.9.3
-  - 2.1.0
+rvm:
+- 1.9.3
+- 2.1.0
+deploy:
+  provider: rubygems
+  api_key:
+    secure: T0PsyTzE9ZIhs48xHxMUNIjzGS9pu5iYVYR8K6sUjcv2H5zmg/R3PZ3g6DK8tvnLHCgI5nrZ0Nb+BJgH+65yRMJrrFAjJISDTh/3gWq+nlTq4IFxgYOvFrCQXl2IyhjpkFktUxHBWMnz7v6iTZJt/Ej+My4iQ8X6kl4WiAnKJJ4=
+  gem: pmirror
+  on:
+    tags: true
+    repo: adnichols/pmirror

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    pmirror (0.0.4)
+    pmirror (0.1.1)
       methadone (~> 1.3.1)
       nokogiri
       progressbar

data/README.md

@@ -54,13 +54,12 @@ Usage: pmirror [options] url
 
 Options:
     -h, --help                       Show command line help
-    -p, --pattern PAT1,PAT2,PAT3     Regex to match files in remote dir,
-may specify multiple patterns
-    -u --url                         One or more URL's to check for
-files that match the defined patterns
+    -p, --pattern PAT1,PAT2,PAT3     Regex to match files in remote dir, may specify multiple patterns
+    -u --url                         One or more URL's to check for files that match the defined patterns
     -l, --localdir DIR               Local directory to mirror files to
     -e, --exec CMD                   Execute command after completion
-    -d, --debug                      Enable debugging
+        --log-level                  Set the logging level to one of [ debug, info, warn, error]
+    -c, --config FILE                Config file to read command line options from (yaml)
     -v, --version                    Show version
 ```
 
@@ -74,14 +73,31 @@ the same filename is matched across multiple URL's only the first will
 be downloaded, subsequent files will see that there is already a local
 file with the same name and will not download. 
 
+Example:
+```
+pmirror --url http://someurl.com    # Single URL
+pmirror --url http://someurl.com,http://someotherurl.com # Multiple URLs
+```
+
 `--pattern` allows you to specify a comma separated list of patterns to
 match on the remote directly. We will iterate over each pattern and
 build up the file list to fetch from the remote url. 
 
+Example:
+```
+pmirror --pattern foo               # Single pattern
+pmirror --pattern foo,bar           # Multiple patterns
+```
+
 `--localdir` is the location you want files downloaded to. It is also
 the directory in which any commands specified with `--exec` will be
 performed.
 
+Example:
+```
+pmirror --localdir /tmp/foo
+```
+
 `--exec` is used to perform actions on the download directory. The
 envisioned use case right now is simply to run a createrepo when the
 downloads are complete. This will change PWD to whatever directory is
@@ -89,7 +105,30 @@ specified in `--localdir` and then will run the command you specify.
 There is not currently a way to pass the value of `--localdir` into a
 command other than to specify `.` in your command. 
 
-`--debug` would enable some extra output
+Example:
+```
+pmirror --exec 'createrepo -c .cache -d .'
+```
+
+`--log-level` will set the logging level when you run the application.
+This enables additional messaging. The most useful log levels right now
+are `info` and `debug`. 
+
+`--config` allows you to put all of your command line options into a
+YAML formatted configuration file. Unlike the command line, lists of
+options here (like pattern: or url:) are YAML lists. Each key must be
+the long name of the option to use. There is no default config file. 
+
+Example config:
+```
+localdir: ../foo
+pattern:
+  - ^floo.*
+  - ^mah.*
+url:
+  - http://localhost:55555
+  - http://localhost:55555
+```
 
 `--version` provides the version info
 

data/features/pmirror.feature

@@ -11,13 +11,12 @@ Feature: My bootstrapped app kinda works
     And the banner should document that this app takes options
     And the following options should be documented:
       |--pattern|
-      |--debug|
       |--localdir|
       |--exec|
       |--version|
       |--url|
+      |--log-level|
       |-p|
-      |-d|
       |-l|
       |-e|
       |-u|
@@ -69,3 +68,12 @@ Feature: My bootstrapped app kinda works
     And the following files should not exist:
       | ../foo/meh.txt|
 
+  Scenario: Use a config file
+    When I successfully run `pmirror -c ../../pmirror_test_config.yaml`
+    Then the exit status should be 0
+    And the following files should exist:
+      | ../foo/floober.txt|
+      | ../foo/floobah.txt|
+      | ../foo/mah.txt|
+    And the following files should not exist:
+      | ../foo/meh.txt|

data/lib/pmirror.rb

@@ -11,12 +11,19 @@ module Pmirror
     version(::Pmirror::VERSION)
 
     main do
-      d "Inside main"
+      debug "Inside main"
+
+      parse_config(options[:config]) if options[:config]
+      normalize_defaults
 
       if options[:url] && options[:pattern] && options[:localdir]
         download_list = get_download_list(options[:url], options[:pattern])
-        d "download_list: #{download_list.inspect}"
-        download_files(options[:localdir], download_list)
+        debug "download_list: #{download_list.inspect}"
+        if download_list
+          download_files(options[:localdir], download_list)
+        else
+          info "No files to download"
+        end
 
         execute(options[:exec]) if options[:exec]
       else
@@ -31,67 +38,73 @@ module Pmirror
       )
     on("-l", "--localdir DIR", "Local directory to mirror files to")
     on("-e", "--exec CMD", "Execute command after completion")
-    on("-d", "--debug", "Enable debugging")
     on("-u", "--url URL,URL", Array, "Url or remote site")
+    on("-c", "--config FILE", "Config file (yaml) to use instead of command line options")
 
-    def self.d(msg)
-      if options[:debug]
-        puts "[DEBUG]: #{msg}"
+    use_log_level_option
+
+    def self.parse_config(config_file)
+      debug "In parse_config"
+      parsed = YAML::load_file(config_file)
+      if parsed.kind_of? Hash
+        parsed.each do |option,value|
+          debug "Storing option '#{option}' with value '#{value.inspect}'"
+          options[option] = value
+        end
       end
     end
 
     def self.get_download_list(url_list, pattern)
-      d "inside get_download_list"
+      debug "inside get_download_list"
       downloads = {}
       url_list.each do |single_url|
         downloads[single_url] = []
-        d "Getting download list for url: #{single_url}"
+        info "Getting download list for url: #{single_url}"
         page = Nokogiri::HTML(open(single_url))
 
         page.css("a").each do |link|
           file_name = link.attributes['href'].value
           pattern.each do |matcher|
             if /#{matcher}/.match(file_name)
-              d "Found match: #{file_name}"
+              debug "Found match: #{file_name}"
               downloads[single_url] << file_name
             end
           end
         end
-        d "Returning downloads: #{downloads.inspect}"
+        debug "Returning downloads: #{downloads.inspect}"
       end
       downloads
     end
 
     def self.download_files(local_dir, url_hash={})
-      d "Inside download_files"
+      debug "Inside download_files"
       url_hash.each_key do |single_url|
-        d "Working on #{single_url}"
+        debug "Working on #{single_url}"
         url_hash[single_url].each do |file|
           local_fn = "#{local_dir}/#{file}"
 
           unless Dir.exist? options[:localdir]
-            d "PWD: #{Dir.pwd}"
-            puts Dir.open(Dir.pwd).read
-            puts "Destination directory '#{options[:localdir]}' does not exist!"
+            debug "PWD: #{Dir.pwd}"
+            info "Destination directory '#{options[:localdir]}' does not exist!"
             exit 1
           end
 
           remote_fn = "#{single_url}/#{file}"
           unless File.exist?(local_fn)
-            puts "Downloading File: #{file}"
-            puts "#{remote_fn} ==> #{local_fn}"
+            info "Downloading File: #{file}"
+            info "#{remote_fn} ==> #{local_fn}"
             http_to_file(local_fn, remote_fn)
             # File.write(local_fn, open(remote_fn).read)
-            puts "Download Complete for #{file}"
+            info "Download Complete for #{file}"
           else
-            puts "Skipping #{file}, already exists"
+            info "Skipping #{file}, already exists"
           end
         end
       end
     end
 
     def self.http_to_file(filename,url)
-      d "Inside http_to_file"
+      debug "Inside http_to_file"
       pbar = nil
       File.open(filename, 'wb') do |save_file|
         open(url, 'rb',
@@ -105,12 +118,12 @@ module Pmirror
           pbar.set s if pbar
         }) {|f| save_file.write(f.read) }
       end
-      puts
+      info ""
     end
 
     def self.execute(cmd)
-      d "Inside execute"
-      puts "Executing: #{cmd}"
+      debug "Inside execute"
+      info "Executing: #{cmd}"
       sh("cd #{options[:localdir]} && #{cmd}")
     end
 

data/lib/pmirror/version.rb

@@ -1,3 +1,3 @@
 module Pmirror
-  VERSION = "0.0.4"
+  VERSION = "0.1.1"
 end

data/pmirror.gemspec

@@ -10,7 +10,7 @@ Gem::Specification.new do |spec|
   spec.email         = ["anichols@trumped.org"]
   spec.description   = %q{Mirror files from remote http server by pattern}
   spec.summary       = %q{Mirror files from remote http server by pattern}
-  spec.homepage      = ""
+  spec.homepage      = "http://github.com/adnichols/pmirror"
   spec.license       = "MIT"
 
   spec.files         = `git ls-files`.split($/)

data/pmirror_test_config.yaml

@@ -0,0 +1,7 @@
+localdir: ../foo
+pattern: 
+  - ^floo.*
+  - ^mah.*
+url: 
+  - http://localhost:55555
+  - http://localhost:55555

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pmirror
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.1.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-01-23 00:00:00.000000000 Z
+date: 2014-01-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -178,12 +178,13 @@ files:
 - lib/pmirror.rb
 - lib/pmirror/version.rb
 - pmirror.gemspec
+- pmirror_test_config.yaml
 - spec/something_spec.rb
 - web/floobah.txt
 - web/floober.txt
 - web/mah.txt
 - web/meh.txt
-homepage: ''
+homepage: http://github.com/adnichols/pmirror
 licenses:
 - MIT
 post_install_message: