pmirror 0.0.1 → 0.0.2

data/.gitignore

@@ -2,3 +2,4 @@
 results.html
 pkg
 html
+*.gem

data/README.md

@@ -17,6 +17,9 @@ find it.
 The tool is very new so it only has the bare minimum feature set:
 
 - Specify multiple regex patterns to match against the remote directory
+- Specify a list of URL's to look for patterns across, this means you
+  can look for the same patterns on multiple urls and aggregate those
+  files into one place
 - Provides a progressbar download status indicator
 - Specify a local directory to download files to
 - Specify a command to execute on the local directory once all files are
@@ -53,6 +56,8 @@ Options:
     -h, --help                       Show command line help
     -p, --pattern PAT1,PAT2,PAT3     Regex to match files in remote dir,
 may specify multiple patterns
+    -u --url                         One or more URL's to check for
+files that match the defined patterns
     -l, --localdir DIR               Local directory to mirror files to
     -e, --exec CMD                   Execute command after completion
     -d, --debug                      Enable debugging
@@ -61,9 +66,13 @@ may specify multiple patterns
 
 Usage should be pretty self explanatory but here are the details:
 
-`url` is the remote URL that you want to fetch files from. Right now
+`--url` is the remote URL that you want to fetch files from. Right now
 this is assumed to be an un-authenticated url. We do not recurse into
-directories looking for files.
+directories looking for files. You may specify more than one url and we
+will look at each url and download any matching files from that url. If
+the same filename is matched across multiple URL's only the first will
+be downloaded, subsequent files will see that there is already a local
+file with the same name and will not download. 
 
 `--pattern` allows you to specify a comma separated list of patterns to
 match on the remote directly. We will iterate over each pattern and

data/features/pmirror.feature

@@ -14,25 +14,28 @@ Feature: My bootstrapped app kinda works
       |--localdir|
       |--exec|
       |--version|
-    And the banner should document that this app's arguments are:
-      |url|which is required|
-
+      |--url|
+      |-p|
+      |-d|
+      |-l|
+      |-e|
+      |-u|
 
   Scenario: Download a file
-    When I successfully run `pmirror -p meh -l ../foo http://localhost:55555`
+    When I successfully run `pmirror -p meh -l ../foo -u http://localhost:55555`
     Then the exit status should be 0
     And the following files should exist:
       |../foo/meh.txt|
 
   Scenario: Execute on local directory
-    When I successfully run `pmirror -p meh -l ../foo -e "touch test" http://localhost:55555`
+    When I successfully run `pmirror -p meh -l ../foo -e "touch test" -u http://localhost:55555`
     Then the exit status should be 0
     And the following files should exist:
       |../foo/meh.txt|
       |../foo/test   |
 
   Scenario: Match multiple files
-    When I successfully run `pmirror -p floo -l ../foo http://localhost:55555`
+    When I successfully run `pmirror -p floo -l ../foo -u http://localhost:55555`
     Then the exit status should be 0
     And the following files should exist:
       | ../foo/floober.txt|
@@ -42,7 +45,7 @@ Feature: My bootstrapped app kinda works
       | ../foo/meh.txt|
 
   Scenario: Match multiple patterns
-    When I successfully run `pmirror -p '^floo.*','^mah.*' -l ../foo http://localhost:55555`
+    When I successfully run `pmirror -p '^floo.*','^mah.*' -l ../foo -u http://localhost:55555`
     Then the exit status should be 0
     And the following files should exist:
       | ../foo/floober.txt|
@@ -51,5 +54,13 @@ Feature: My bootstrapped app kinda works
     And the following files should not exist:
       | ../foo/meh.txt|
 
-
+  Scenario: Match multiple urls
+    When I successfully run `pmirror -p '^floo.*','^mah.*' -l ../foo -u http://localhost:55555,http://localhost:55555`
+    Then the exit status should be 0
+    And the following files should exist:
+      | ../foo/floober.txt|
+      | ../foo/floobah.txt|
+      | ../foo/mah.txt|
+    And the following files should not exist:
+      | ../foo/meh.txt|
 

data/lib/pmirror.rb

@@ -9,15 +9,17 @@ module Pmirror
     include Methadone::CLILogging
     include Methadone::SH
 
-    main do |url|
+    main do
+      d "Inside main"
 
-      download_files( url, options[:localdir], get_download_list(url, options[:pattern]) )
+      download_list = get_download_list(options[:url], options[:pattern])
+      d "download_list: #{download_list.inspect}"
+      download_files(options[:localdir], download_list)
       execute(options[:exec]) if options[:exec]
 
     end
 
     description "Mirror files on a remote http server based on pattern match"
-    arg("url", "Url or remote site", :one, :required)
     on("-p", "--pattern PAT1,PAT2,PAT3", Array,
        "Regex to match files in remote dir, may specify multiple patterns"
       )
@@ -25,48 +27,66 @@ module Pmirror
     on("-e", "--exec CMD", "Execute command after completion")
     on("-d", "--debug", "Enable debugging")
     on("-v", "--version", "Show version")
+    on("-u", "--url URL,URL", Array, "Url or remote site")
 
-    def self.get_download_list(url, pattern)
-      info "Getting download list"
-      page = Nokogiri::HTML(open(url))
-      downloads = []
+    def self.d(msg)
+      if options[:debug]
+        puts "[DEBUG]: #{msg}"
+      end
+    end
+
+    def self.get_download_list(url_list, pattern)
+      d "inside get_download_list"
+      downloads = {}
+      url_list.each do |single_url|
+        downloads[single_url] = []
+        d "Getting download list for url: #{single_url}"
+        page = Nokogiri::HTML(open(single_url))
 
-      page.css("a").each do |link|
-        rpm_name = link.attributes['href'].value
-        pattern.each do |matcher|
-          if /#{matcher}/.match(rpm_name)
-            downloads << rpm_name
+        page.css("a").each do |link|
+          file_name = link.attributes['href'].value
+          pattern.each do |matcher|
+            if /#{matcher}/.match(file_name)
+              d "Found match: #{file_name}"
+              downloads[single_url] << file_name
+            end
           end
         end
+        d "Returning downloads: #{downloads.inspect}"
       end
       downloads
     end
 
-    def self.download_files(url, local_dir, file_list=[])
-      file_list.each do |file|
-        local_fn = "#{local_dir}/#{file}"
+    def self.download_files(local_dir, url_hash={})
+      d "Inside download_files"
+      url_hash.each_key do |single_url|
+        d "Working on #{single_url}"
+        url_hash[single_url].each do |file|
+          local_fn = "#{local_dir}/#{file}"
 
-        unless Dir.exist? options[:localdir]
-          puts "PWD: #{Dir.pwd}"
-          puts Dir.open(Dir.pwd).read
-          puts "Destination directory '#{options[:localdir]}' does not exist!"
-          exit 1
-        end
+          unless Dir.exist? options[:localdir]
+            d "PWD: #{Dir.pwd}"
+            puts Dir.open(Dir.pwd).read
+            puts "Destination directory '#{options[:localdir]}' does not exist!"
+            exit 1
+          end
 
-        remote_fn = "#{url}/#{file}"
-        unless File.exist?(local_fn)
-          puts "Downloading File: #{file}"
-          puts "#{remote_fn} ==> #{local_fn}"
-          http_to_file(local_fn, remote_fn)
-          # File.write(local_fn, open(remote_fn).read)
-          puts "Download Complete for #{file}"
-        else
-          puts "Skipping #{file}, already exists"
+          remote_fn = "#{single_url}/#{file}"
+          unless File.exist?(local_fn)
+            puts "Downloading File: #{file}"
+            puts "#{remote_fn} ==> #{local_fn}"
+            http_to_file(local_fn, remote_fn)
+            # File.write(local_fn, open(remote_fn).read)
+            puts "Download Complete for #{file}"
+          else
+            puts "Skipping #{file}, already exists"
+          end
         end
       end
     end
 
     def self.http_to_file(filename,url)
+      d "Inside http_to_file"
       pbar = nil
       File.open(filename, 'wb') do |save_file|
         open(url, 'rb',
@@ -84,6 +104,7 @@ module Pmirror
     end
 
     def self.execute(cmd)
+      d "Inside execute"
       puts "Executing: #{cmd}"
       sh("cd #{options[:localdir]} && #{cmd}")
     end

data/lib/pmirror/version.rb

@@ -1,3 +1,3 @@
 module Pmirror
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pmirror
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-01-19 00:00:00.000000000 Z
+date: 2014-01-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler